Merge tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi

Pull SCSI updates from James Bottomley:
 "This is mostly updates to the usual drivers: ufs, qedf, mpt3sas, lpfc,
  xfcp, hisi_sas, cxlflash, qla2xxx.

  In the absence of Nic, we're also taking target updates which are
  mostly minor except for the tcmu refactor.

  The only real core change to worry about is the removal of high page
  bouncing (in sas, storvsc and iscsi). This has been well tested and no
  problems have shown up so far"

* tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi: (268 commits)
  scsi: lpfc: update driver version to 12.0.0.4
  scsi: lpfc: Fix port initialization failure.
  scsi: lpfc: Fix 16gb hbas failing cq create.
  scsi: lpfc: Fix crash in blk_mq layer when executing modprobe -r lpfc
  scsi: lpfc: correct oversubscription of nvme io requests for an adapter
  scsi: lpfc: Fix MDS diagnostics failure (Rx < Tx)
  scsi: hisi_sas: Mark PHY as in reset for nexus reset
  scsi: hisi_sas: Fix return value when get_free_slot() failed
  scsi: hisi_sas: Terminate STP reject quickly for v2 hw
  scsi: hisi_sas: Add v2 hw force PHY function for internal ATA command
  scsi: hisi_sas: Include TMF elements in struct hisi_sas_slot
  scsi: hisi_sas: Try wait commands before before controller reset
  scsi: hisi_sas: Init disks after controller reset
  scsi: hisi_sas: Create a scsi_host_template per HW module
  scsi: hisi_sas: Reset disks when discovered
  scsi: hisi_sas: Add LED feature for v3 hw
  scsi: hisi_sas: Change common allocation mode of device id
  scsi: hisi_sas: change slot index allocation mode
  scsi: hisi_sas: Introduce hisi_sas_phy_set_linkrate()
  scsi: hisi_sas: fix a typo in hisi_sas_task_prep()
  ...

139 files changed, 7070 insertions, 1972 deletions

diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index ce5019db50fd..6a91d04351d9 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -5054,6 +5054,18 @@ int ata_sas_port_init(struct ata_port *ap)
 }
 EXPORT_SYMBOL_GPL(ata_sas_port_init);
 
+int ata_sas_tport_add(struct device *parent, struct ata_port *ap)
+{
+        return ata_tport_add(parent, ap);
+}
+EXPORT_SYMBOL_GPL(ata_sas_tport_add);
+
+void ata_sas_tport_delete(struct ata_port *ap)
+{
+        ata_tport_delete(ap);
+}
+EXPORT_SYMBOL_GPL(ata_sas_tport_delete);
+
 /**
  *      ata_sas_port_destroy - Destroy a SATA port allocated by ata_sas_port_alloc
  *      @ap: SATA port to destroy
diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c
index 8699bb969e7e..3c836c099a8f 100644
--- a/drivers/hv/ring_buffer.c
+++ b/drivers/hv/ring_buffer.c
@@ -227,6 +227,8 @@ int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
         ring_info->ring_buffer->feature_bits.value = 1;
 
         ring_info->ring_size = page_cnt << PAGE_SHIFT;
+        ring_info->ring_size_div10_reciprocal =
+                reciprocal_value(ring_info->ring_size / 10);
         ring_info->ring_datasize = ring_info->ring_size -
                 sizeof(struct hv_ring_buffer);
 
diff --git a/drivers/message/fusion/lsi/mpi_cnfg.h b/drivers/message/fusion/lsi/mpi_cnfg.h
index 4e9c0ce94f27..059997f8ebce 100644
--- a/drivers/message/fusion/lsi/mpi_cnfg.h
+++ b/drivers/message/fusion/lsi/mpi_cnfg.h
@@ -1802,13 +1802,13 @@ typedef struct _CONFIG_PAGE_FC_PORT_0
 #define MPI_FCPORTPAGE0_SUPPORT_CLASS_2                 (0x00000002)
 #define MPI_FCPORTPAGE0_SUPPORT_CLASS_3                 (0x00000004)
 
-#define MPI_FCPORTPAGE0_SUPPORT_SPEED_UKNOWN            (0x00000000) /* (SNIA)HBA_PORTSPEED_UNKNOWN 0   Unknown - transceiver incapable of reporting */
+#define MPI_FCPORTPAGE0_SUPPORT_SPEED_UNKNOWN           (0x00000000) /* (SNIA)HBA_PORTSPEED_UNKNOWN 0   Unknown - transceiver incapable of reporting */
 #define MPI_FCPORTPAGE0_SUPPORT_1GBIT_SPEED             (0x00000001) /* (SNIA)HBA_PORTSPEED_1GBIT   1   1 GBit/sec */
 #define MPI_FCPORTPAGE0_SUPPORT_2GBIT_SPEED             (0x00000002) /* (SNIA)HBA_PORTSPEED_2GBIT   2   2 GBit/sec */
 #define MPI_FCPORTPAGE0_SUPPORT_10GBIT_SPEED            (0x00000004) /* (SNIA)HBA_PORTSPEED_10GBIT  4  10 GBit/sec */
 #define MPI_FCPORTPAGE0_SUPPORT_4GBIT_SPEED             (0x00000008) /* (SNIA)HBA_PORTSPEED_4GBIT   8   4 GBit/sec */
 
-#define MPI_FCPORTPAGE0_CURRENT_SPEED_UKNOWN            MPI_FCPORTPAGE0_SUPPORT_SPEED_UKNOWN
+#define MPI_FCPORTPAGE0_CURRENT_SPEED_UNKNOWN           MPI_FCPORTPAGE0_SUPPORT_SPEED_UNKNOWN
 #define MPI_FCPORTPAGE0_CURRENT_SPEED_1GBIT             MPI_FCPORTPAGE0_SUPPORT_1GBIT_SPEED
 #define MPI_FCPORTPAGE0_CURRENT_SPEED_2GBIT             MPI_FCPORTPAGE0_SUPPORT_2GBIT_SPEED
 #define MPI_FCPORTPAGE0_CURRENT_SPEED_10GBIT            MPI_FCPORTPAGE0_SUPPORT_10GBIT_SPEED
diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c
index a746ccdd630a..a625ac4e2872 100644
--- a/drivers/message/fusion/mptbase.c
+++ b/drivers/message/fusion/mptbase.c
@@ -7600,7 +7600,7 @@ mpt_display_event_info(MPT_ADAPTER *ioc, EventNotificationReply_t *pEventReply)
 
                 snprintf(evStr, EVENT_DESCR_STR_SZ,
                     "SAS Initiator Device Table Overflow: max initiators=%02d "
-                    "current initators=%02d",
+                    "current initiators=%02d",
                     max_init, current_init);
                 break;
         }
diff --git a/drivers/message/fusion/mptfc.c b/drivers/message/fusion/mptfc.c
index 6d461ca97150..06b175420be9 100644
--- a/drivers/message/fusion/mptfc.c
+++ b/drivers/message/fusion/mptfc.c
@@ -693,7 +693,7 @@ mptfc_display_port_link_speed(MPT_ADAPTER *ioc, int portnum, FCPortPage0_t *pp0d
         state = pp0dest->PortState;
 
         if (state != MPI_FCPORTPAGE0_PORTSTATE_OFFLINE &&
-            new_speed != MPI_FCPORTPAGE0_CURRENT_SPEED_UKNOWN) {
+            new_speed != MPI_FCPORTPAGE0_CURRENT_SPEED_UNKNOWN) {
 
                 old = old_speed == MPI_FCPORTPAGE0_CURRENT_SPEED_1GBIT ? "1 Gbps" :
                        old_speed == MPI_FCPORTPAGE0_CURRENT_SPEED_2GBIT ? "2 Gbps" :
diff --git a/drivers/message/fusion/mptlan.c b/drivers/message/fusion/mptlan.c
index 55dd71bbdc2a..4cbed4d06aa7 100644
--- a/drivers/message/fusion/mptlan.c
+++ b/drivers/message/fusion/mptlan.c
@@ -670,7 +670,7 @@ out:
 }
 
 /*=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=*/
-static int
+static netdev_tx_t
 mpt_lan_sdu_send (struct sk_buff *skb, struct net_device *dev)
 {
         struct mpt_lan_priv *priv = netdev_priv(dev);
diff --git a/drivers/message/fusion/mptsas.c b/drivers/message/fusion/mptsas.c
index 19a5aa70ecda..76a66da33996 100644
--- a/drivers/message/fusion/mptsas.c
+++ b/drivers/message/fusion/mptsas.c
@@ -4320,7 +4320,7 @@ mptsas_hotplug_work(MPT_ADAPTER *ioc, struct fw_event_work *fw_event,
                         if (ioc->raid_data.pIocPg2->RaidVolume[i].VolumeID ==
                             hot_plug_info->id) {
                                 printk(MYIOC_s_WARN_FMT "firmware bug: unable "
-                                    "to add hidden disk - target_id matchs "
+                                    "to add hidden disk - target_id matches "
                                     "volume_id\n", ioc->name);
                                 mptsas_free_fw_event(ioc, fw_event);
                                 return;
diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h
index 99d8e7398a5b..23304aca25f9 100644
--- a/drivers/net/hyperv/hyperv_net.h
+++ b/drivers/net/hyperv/hyperv_net.h
@@ -189,7 +189,6 @@ struct netvsc_device;
 struct net_device_context;
 
 extern u32 netvsc_ring_bytes;
-extern struct reciprocal_value netvsc_ring_reciprocal;
 
 struct netvsc_device *netvsc_device_add(struct hv_device *device,
                                         const struct netvsc_device_info *info);
diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c
index d2ee66c259a7..5d5bd513847f 100644
--- a/drivers/net/hyperv/netvsc.c
+++ b/drivers/net/hyperv/netvsc.c
@@ -31,7 +31,6 @@
 #include <linux/vmalloc.h>
 #include <linux/rtnetlink.h>
 #include <linux/prefetch.h>
-#include <linux/reciprocal_div.h>
 
 #include <asm/sync_bitops.h>
 
@@ -635,17 +634,6 @@ void netvsc_device_remove(struct hv_device *device)
 #define RING_AVAIL_PERCENT_HIWATER 20
 #define RING_AVAIL_PERCENT_LOWATER 10
 
-/*
- * Get the percentage of available bytes to write in the ring.
- * The return value is in range from 0 to 100.
- */
-static u32 hv_ringbuf_avail_percent(const struct hv_ring_buffer_info *ring_info)
-{
-        u32 avail_write = hv_get_bytes_to_write(ring_info);
-
-        return reciprocal_divide(avail_write  * 100, netvsc_ring_reciprocal);
-}
-
 static inline void netvsc_free_send_slot(struct netvsc_device *net_device,
                                          u32 index)
 {
@@ -694,8 +682,8 @@ static void netvsc_send_tx_complete(struct net_device *ndev,
                 struct netdev_queue *txq = netdev_get_tx_queue(ndev, q_idx);
 
                 if (netif_tx_queue_stopped(txq) &&
-                    (hv_ringbuf_avail_percent(&channel->outbound) > RING_AVAIL_PERCENT_HIWATER ||
-                     queue_sends < 1)) {
+                    (hv_get_avail_to_write_percent(&channel->outbound) >
+                     RING_AVAIL_PERCENT_HIWATER || queue_sends < 1)) {
                         netif_tx_wake_queue(txq);
                         ndev_ctx->eth_stats.wake_queue++;
                 }
@@ -802,7 +790,7 @@ static inline int netvsc_send_pkt(
         struct netdev_queue *txq = netdev_get_tx_queue(ndev, packet->q_idx);
         u64 req_id;
         int ret;
-        u32 ring_avail = hv_ringbuf_avail_percent(&out_channel->outbound);
+        u32 ring_avail = hv_get_avail_to_write_percent(&out_channel->outbound);
 
         nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;
         if (skb)
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index bef4d55a108c..1e9bde68698d 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -35,7 +35,6 @@
 #include <linux/slab.h>
 #include <linux/rtnetlink.h>
 #include <linux/netpoll.h>
-#include <linux/reciprocal_div.h>
 
 #include <net/arp.h>
 #include <net/route.h>
@@ -59,7 +58,6 @@ static unsigned int ring_size __ro_after_init = 128;
 module_param(ring_size, uint, 0444);
 MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)");
 unsigned int netvsc_ring_bytes __ro_after_init;
-struct reciprocal_value netvsc_ring_reciprocal __ro_after_init;
 
 static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE |
                                 NETIF_MSG_LINK | NETIF_MSG_IFUP |
@@ -2130,7 +2128,6 @@ static int __init netvsc_drv_init(void)
                         ring_size);
         }
         netvsc_ring_bytes = ring_size * PAGE_SIZE;
-        netvsc_ring_reciprocal = reciprocal_value(netvsc_ring_bytes);
 
         ret = vmbus_driver_register(&netvsc_drv);
         if (ret)
diff --git a/drivers/s390/scsi/zfcp_dbf.c b/drivers/s390/scsi/zfcp_dbf.c
index 18c4f933e8b9..3b368fcf13f4 100644
--- a/drivers/s390/scsi/zfcp_dbf.c
+++ b/drivers/s390/scsi/zfcp_dbf.c
@@ -285,6 +285,8 @@ void zfcp_dbf_rec_trig(char *tag, struct zfcp_adapter *adapter,
         struct list_head *entry;
         unsigned long flags;
 
+        lockdep_assert_held(&adapter->erp_lock);
+
         if (unlikely(!debug_level_enabled(dbf->rec, level)))
                 return;
 
@@ -599,16 +601,18 @@ void zfcp_dbf_san_in_els(char *tag, struct zfcp_fsf_req *fsf)
 }
 
 /**
- * zfcp_dbf_scsi - trace event for scsi commands
- * @tag: identifier for event
- * @sc: pointer to struct scsi_cmnd
- * @fsf: pointer to struct zfcp_fsf_req
+ * zfcp_dbf_scsi_common() - Common trace event helper for scsi.
+ * @tag: Identifier for event.
+ * @level: trace level of event.
+ * @sdev: Pointer to SCSI device as context for this event.
+ * @sc: Pointer to SCSI command, or NULL with task management function (TMF).
+ * @fsf: Pointer to FSF request, or NULL.
  */
-void zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *sc,
-                   struct zfcp_fsf_req *fsf)
+void zfcp_dbf_scsi_common(char *tag, int level, struct scsi_device *sdev,
+                          struct scsi_cmnd *sc, struct zfcp_fsf_req *fsf)
 {
         struct zfcp_adapter *adapter =
-                (struct zfcp_adapter *) sc->device->host->hostdata[0];
+                (struct zfcp_adapter *) sdev->host->hostdata[0];
         struct zfcp_dbf *dbf = adapter->dbf;
         struct zfcp_dbf_scsi *rec = &dbf->scsi_buf;
         struct fcp_resp_with_ext *fcp_rsp;
@@ -620,16 +624,28 @@ void zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *sc,
 
         memcpy(rec->tag, tag, ZFCP_DBF_TAG_LEN);
         rec->id = ZFCP_DBF_SCSI_CMND;
-        rec->scsi_result = sc->result;
-        rec->scsi_retries = sc->retries;
-        rec->scsi_allowed = sc->allowed;
-        rec->scsi_id = sc->device->id;
-        rec->scsi_lun = (u32)sc->device->lun;
-        rec->scsi_lun_64_hi = (u32)(sc->device->lun >> 32);
-        rec->host_scribble = (unsigned long)sc->host_scribble;
-
-        memcpy(rec->scsi_opcode, sc->cmnd,
-               min((int)sc->cmd_len, ZFCP_DBF_SCSI_OPCODE));
+        if (sc) {
+                rec->scsi_result = sc->result;
+                rec->scsi_retries = sc->retries;
+                rec->scsi_allowed = sc->allowed;
+                rec->scsi_id = sc->device->id;
+                rec->scsi_lun = (u32)sc->device->lun;
+                rec->scsi_lun_64_hi = (u32)(sc->device->lun >> 32);
+                rec->host_scribble = (unsigned long)sc->host_scribble;
+
+                memcpy(rec->scsi_opcode, sc->cmnd,
+                       min_t(int, sc->cmd_len, ZFCP_DBF_SCSI_OPCODE));
+        } else {
+                rec->scsi_result = ~0;
+                rec->scsi_retries = ~0;
+                rec->scsi_allowed = ~0;
+                rec->scsi_id = sdev->id;
+                rec->scsi_lun = (u32)sdev->lun;
+                rec->scsi_lun_64_hi = (u32)(sdev->lun >> 32);
+                rec->host_scribble = ~0;
+
+                memset(rec->scsi_opcode, 0xff, ZFCP_DBF_SCSI_OPCODE);
+        }
 
         if (fsf) {
                 rec->fsf_req_id = fsf->req_id;
@@ -664,6 +680,46 @@ void zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *sc,
         spin_unlock_irqrestore(&dbf->scsi_lock, flags);
 }
 
+/**
+ * zfcp_dbf_scsi_eh() - Trace event for special cases of scsi_eh callbacks.
+ * @tag: Identifier for event.
+ * @adapter: Pointer to zfcp adapter as context for this event.
+ * @scsi_id: SCSI ID/target to indicate scope of task management function (TMF).
+ * @ret: Return value of calling function.
+ *
+ * This SCSI trace variant does not depend on any of:
+ * scsi_cmnd, zfcp_fsf_req, scsi_device.
+ */
+void zfcp_dbf_scsi_eh(char *tag, struct zfcp_adapter *adapter,
+                      unsigned int scsi_id, int ret)
+{
+        struct zfcp_dbf *dbf = adapter->dbf;
+        struct zfcp_dbf_scsi *rec = &dbf->scsi_buf;
+        unsigned long flags;
+        static int const level = 1;
+
+        if (unlikely(!debug_level_enabled(adapter->dbf->scsi, level)))
+                return;
+
+        spin_lock_irqsave(&dbf->scsi_lock, flags);
+        memset(rec, 0, sizeof(*rec));
+
+        memcpy(rec->tag, tag, ZFCP_DBF_TAG_LEN);
+        rec->id = ZFCP_DBF_SCSI_CMND;
+        rec->scsi_result = ret; /* re-use field, int is 4 bytes and fits */
+        rec->scsi_retries = ~0;
+        rec->scsi_allowed = ~0;
+        rec->fcp_rsp_info = ~0;
+        rec->scsi_id = scsi_id;
+        rec->scsi_lun = (u32)ZFCP_DBF_INVALID_LUN;
+        rec->scsi_lun_64_hi = (u32)(ZFCP_DBF_INVALID_LUN >> 32);
+        rec->host_scribble = ~0;
+        memset(rec->scsi_opcode, 0xff, ZFCP_DBF_SCSI_OPCODE);
+
+        debug_event(dbf->scsi, level, rec, sizeof(*rec));
+        spin_unlock_irqrestore(&dbf->scsi_lock, flags);
+}
+
 static debug_info_t *zfcp_dbf_reg(const char *name, int size, int rec_size)
 {
         struct debug_info *d;
diff --git a/drivers/s390/scsi/zfcp_dbf.h b/drivers/s390/scsi/zfcp_dbf.h
index e2a973cd2573..d116c07ed77a 100644
--- a/drivers/s390/scsi/zfcp_dbf.h
+++ b/drivers/s390/scsi/zfcp_dbf.h
@@ -359,7 +359,7 @@ void _zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *scmd,
                                         scmd->device->host->hostdata[0];
 
         if (debug_level_enabled(adapter->dbf->scsi, level))
-                zfcp_dbf_scsi(tag, level, scmd, req);
+                zfcp_dbf_scsi_common(tag, level, scmd->device, scmd, req);
 }
 
 /**
@@ -402,16 +402,23 @@ void zfcp_dbf_scsi_abort(char *tag, struct scsi_cmnd *scmd,
 }
 
 /**
- * zfcp_dbf_scsi_devreset - trace event for Logical Unit or Target Reset
- * @tag: tag indicating success or failure of reset operation
- * @scmnd: SCSI command which caused this error recovery
- * @flag: indicates type of reset (Target Reset, Logical Unit Reset)
+ * zfcp_dbf_scsi_devreset() - Trace event for Logical Unit or Target Reset.
+ * @tag: Tag indicating success or failure of reset operation.
+ * @sdev: Pointer to SCSI device as context for this event.
+ * @flag: Indicates type of reset (Target Reset, Logical Unit Reset).
+ * @fsf_req: Pointer to FSF request representing the TMF, or NULL.
  */
 static inline
-void zfcp_dbf_scsi_devreset(char *tag, struct scsi_cmnd *scmnd, u8 flag,
+void zfcp_dbf_scsi_devreset(char *tag, struct scsi_device *sdev, u8 flag,
                             struct zfcp_fsf_req *fsf_req)
 {
+        struct zfcp_adapter *adapter = (struct zfcp_adapter *)
+                                        sdev->host->hostdata[0];
         char tmp_tag[ZFCP_DBF_TAG_LEN];
+        static int const level = 1;
+
+        if (unlikely(!debug_level_enabled(adapter->dbf->scsi, level)))
+                return;
 
         if (flag == FCP_TMF_TGT_RESET)
                 memcpy(tmp_tag, "tr_", 3);
@@ -419,7 +426,7 @@ void zfcp_dbf_scsi_devreset(char *tag, struct scsi_cmnd *scmnd, u8 flag,
                 memcpy(tmp_tag, "lr_", 3);
 
         memcpy(&tmp_tag[3], tag, 4);
-        _zfcp_dbf_scsi(tmp_tag, 1, scmnd, fsf_req);
+        zfcp_dbf_scsi_common(tmp_tag, level, sdev, NULL, fsf_req);
 }
 
 /**
diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c
index 1d91a32db08e..e7e6b63905e2 100644
--- a/drivers/s390/scsi/zfcp_erp.c
+++ b/drivers/s390/scsi/zfcp_erp.c
@@ -19,7 +19,6 @@
 enum zfcp_erp_act_flags {
         ZFCP_STATUS_ERP_TIMEDOUT        = 0x10000000,
         ZFCP_STATUS_ERP_CLOSE_ONLY      = 0x01000000,
-        ZFCP_STATUS_ERP_DISMISSING      = 0x00100000,
         ZFCP_STATUS_ERP_DISMISSED       = 0x00200000,
         ZFCP_STATUS_ERP_LOWMEM          = 0x00400000,
         ZFCP_STATUS_ERP_NO_REF          = 0x00800000,
@@ -27,7 +26,6 @@ enum zfcp_erp_act_flags {
 
 enum zfcp_erp_steps {
         ZFCP_ERP_STEP_UNINITIALIZED     = 0x0000,
-        ZFCP_ERP_STEP_FSF_XCONFIG       = 0x0001,
         ZFCP_ERP_STEP_PHYS_PORT_CLOSING = 0x0010,
         ZFCP_ERP_STEP_PORT_CLOSING      = 0x0100,
         ZFCP_ERP_STEP_PORT_OPENING      = 0x0800,
@@ -35,16 +33,28 @@ enum zfcp_erp_steps {
         ZFCP_ERP_STEP_LUN_OPENING       = 0x2000,
 };
 
+/**
+ * enum zfcp_erp_act_type - Type of ERP action object.
+ * @ZFCP_ERP_ACTION_REOPEN_LUN: LUN recovery.
+ * @ZFCP_ERP_ACTION_REOPEN_PORT: Port recovery.
+ * @ZFCP_ERP_ACTION_REOPEN_PORT_FORCED: Forced port recovery.
+ * @ZFCP_ERP_ACTION_REOPEN_ADAPTER: Adapter recovery.
+ * @ZFCP_ERP_ACTION_NONE: Eyecatcher pseudo flag to bitwise or-combine with
+ *                        either of the first four enum values.
+ *                        Used to indicate that an ERP action could not be
+ *                        set up despite a detected need for some recovery.
+ * @ZFCP_ERP_ACTION_FAILED: Eyecatcher pseudo flag to bitwise or-combine with
+ *                          either of the first four enum values.
+ *                          Used to indicate that ERP not needed because
+ *                          the object has ZFCP_STATUS_COMMON_ERP_FAILED.
+ */
 enum zfcp_erp_act_type {
         ZFCP_ERP_ACTION_REOPEN_LUN         = 1,
         ZFCP_ERP_ACTION_REOPEN_PORT        = 2,
         ZFCP_ERP_ACTION_REOPEN_PORT_FORCED = 3,
         ZFCP_ERP_ACTION_REOPEN_ADAPTER     = 4,
-};
-
-enum zfcp_erp_act_state {
-        ZFCP_ERP_ACTION_RUNNING = 1,
-        ZFCP_ERP_ACTION_READY   = 2,
+        ZFCP_ERP_ACTION_NONE               = 0xc0,
+        ZFCP_ERP_ACTION_FAILED             = 0xe0,
 };
 
 enum zfcp_erp_act_result {
@@ -62,14 +72,14 @@ static void zfcp_erp_adapter_block(struct zfcp_adapter *adapter, int mask)
                                        ZFCP_STATUS_COMMON_UNBLOCKED | mask);
 }
 
-static int zfcp_erp_action_exists(struct zfcp_erp_action *act)
+static bool zfcp_erp_action_is_running(struct zfcp_erp_action *act)
 {
         struct zfcp_erp_action *curr_act;
 
         list_for_each_entry(curr_act, &act->adapter->erp_running_head, list)
                 if (act == curr_act)
-                        return ZFCP_ERP_ACTION_RUNNING;
-        return 0;
+                        return true;
+        return false;
 }
 
 static void zfcp_erp_action_ready(struct zfcp_erp_action *act)
@@ -85,7 +95,7 @@ static void zfcp_erp_action_ready(struct zfcp_erp_action *act)
 static void zfcp_erp_action_dismiss(struct zfcp_erp_action *act)
 {
         act->status |= ZFCP_STATUS_ERP_DISMISSED;
-        if (zfcp_erp_action_exists(act) == ZFCP_ERP_ACTION_RUNNING)
+        if (zfcp_erp_action_is_running(act))
                 zfcp_erp_action_ready(act);
 }
 
@@ -126,6 +136,49 @@ static void zfcp_erp_action_dismiss_adapter(struct zfcp_adapter *adapter)
         }
 }
 
+static int zfcp_erp_handle_failed(int want, struct zfcp_adapter *adapter,
+                                  struct zfcp_port *port,
+                                  struct scsi_device *sdev)
+{
+        int need = want;
+        struct zfcp_scsi_dev *zsdev;
+
+        switch (want) {
+        case ZFCP_ERP_ACTION_REOPEN_LUN:
+                zsdev = sdev_to_zfcp(sdev);
+                if (atomic_read(&zsdev->status) & ZFCP_STATUS_COMMON_ERP_FAILED)
+                        need = 0;
+                break;
+        case ZFCP_ERP_ACTION_REOPEN_PORT_FORCED:
+                if (atomic_read(&port->status) & ZFCP_STATUS_COMMON_ERP_FAILED)
+                        need = 0;
+                break;
+        case ZFCP_ERP_ACTION_REOPEN_PORT:
+                if (atomic_read(&port->status) &
+                    ZFCP_STATUS_COMMON_ERP_FAILED) {
+                        need = 0;
+                        /* ensure propagation of failed status to new devices */
+                        zfcp_erp_set_port_status(
+                                port, ZFCP_STATUS_COMMON_ERP_FAILED);
+                }
+                break;
+        case ZFCP_ERP_ACTION_REOPEN_ADAPTER:
+                if (atomic_read(&adapter->status) &
+                    ZFCP_STATUS_COMMON_ERP_FAILED) {
+                        need = 0;
+                        /* ensure propagation of failed status to new devices */
+                        zfcp_erp_set_adapter_status(
+                                adapter, ZFCP_STATUS_COMMON_ERP_FAILED);
+                }
+                break;
+        default:
+                need = 0;
+                break;
+        }
+
+        return need;
+}
+
 static int zfcp_erp_required_act(int want, struct zfcp_adapter *adapter,
                                  struct zfcp_port *port,
                                  struct scsi_device *sdev)
@@ -241,48 +294,70 @@ static struct zfcp_erp_action *zfcp_erp_setup_act(int need, u32 act_status,
         return erp_action;
 }
 
-static int zfcp_erp_action_enqueue(int want, struct zfcp_adapter *adapter,
-                                   struct zfcp_port *port,
-                                   struct scsi_device *sdev,
-                                   char *id, u32 act_status)
+static void zfcp_erp_action_enqueue(int want, struct zfcp_adapter *adapter,
+                                    struct zfcp_port *port,
+                                    struct scsi_device *sdev,
+                                    char *id, u32 act_status)
 {
-        int retval = 1, need;
+        int need;
         struct zfcp_erp_action *act;
 
-        if (!adapter->erp_thread)
-                return -EIO;
+        need = zfcp_erp_handle_failed(want, adapter, port, sdev);
+        if (!need) {
+                need = ZFCP_ERP_ACTION_FAILED; /* marker for trace */
+                goto out;
+        }
+
+        if (!adapter->erp_thread) {
+                need = ZFCP_ERP_ACTION_NONE; /* marker for trace */
+                goto out;
+        }
 
         need = zfcp_erp_required_act(want, adapter, port, sdev);
         if (!need)
                 goto out;
 
         act = zfcp_erp_setup_act(need, act_status, adapter, port, sdev);
-        if (!act)
+        if (!act) {
+                need |= ZFCP_ERP_ACTION_NONE; /* marker for trace */
                 goto out;
+        }
         atomic_or(ZFCP_STATUS_ADAPTER_ERP_PENDING, &adapter->status);
         ++adapter->erp_total_count;
         list_add_tail(&act->list, &adapter->erp_ready_head);
         wake_up(&adapter->erp_ready_wq);
-        retval = 0;
  out:
         zfcp_dbf_rec_trig(id, adapter, port, sdev, want, need);
-        return retval;
 }
 
-static int _zfcp_erp_adapter_reopen(struct zfcp_adapter *adapter,
+void zfcp_erp_port_forced_no_port_dbf(char *id, struct zfcp_adapter *adapter,
+                                      u64 port_name, u32 port_id)
+{
+        unsigned long flags;
+        static /* don't waste stack */ struct zfcp_port tmpport;
+
+        write_lock_irqsave(&adapter->erp_lock, flags);
+        /* Stand-in zfcp port with fields just good enough for
+         * zfcp_dbf_rec_trig() and zfcp_dbf_set_common().
+         * Under lock because tmpport is static.
+         */
+        atomic_set(&tmpport.status, -1); /* unknown */
+        tmpport.wwpn = port_name;
+        tmpport.d_id = port_id;
+        zfcp_dbf_rec_trig(id, adapter, &tmpport, NULL,
+                          ZFCP_ERP_ACTION_REOPEN_PORT_FORCED,
+                          ZFCP_ERP_ACTION_NONE);
+        write_unlock_irqrestore(&adapter->erp_lock, flags);
+}
+
+static void _zfcp_erp_adapter_reopen(struct zfcp_adapter *adapter,
                                     int clear_mask, char *id)
 {
         zfcp_erp_adapter_block(adapter, clear_mask);
         zfcp_scsi_schedule_rports_block(adapter);
 
-        /* ensure propagation of failed status to new devices */
-        if (atomic_read(&adapter->status) & ZFCP_STATUS_COMMON_ERP_FAILED) {
-                zfcp_erp_set_adapter_status(adapter,
-                                            ZFCP_STATUS_COMMON_ERP_FAILED);
-                return -EIO;
-        }
-        return zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_ADAPTER,
-                                       adapter, NULL, NULL, id, 0);
+        zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_ADAPTER,
+                                adapter, NULL, NULL, id, 0);
 }
 
 /**
@@ -299,12 +374,8 @@ void zfcp_erp_adapter_reopen(struct zfcp_adapter *adapter, int clear, char *id)
         zfcp_scsi_schedule_rports_block(adapter);
 
         write_lock_irqsave(&adapter->erp_lock, flags);
-        if (atomic_read(&adapter->status) & ZFCP_STATUS_COMMON_ERP_FAILED)
-                zfcp_erp_set_adapter_status(adapter,
-                                            ZFCP_STATUS_COMMON_ERP_FAILED);
-        else
-                zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_ADAPTER, adapter,
-                                        NULL, NULL, id, 0);
+        zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_ADAPTER, adapter,
+                                NULL, NULL, id, 0);
         write_unlock_irqrestore(&adapter->erp_lock, flags);
 }
 
@@ -345,9 +416,6 @@ static void _zfcp_erp_port_forced_reopen(struct zfcp_port *port, int clear,
         zfcp_erp_port_block(port, clear);
         zfcp_scsi_schedule_rport_block(port);
 
-        if (atomic_read(&port->status) & ZFCP_STATUS_COMMON_ERP_FAILED)
-                return;
-
         zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_PORT_FORCED,
                                 port->adapter, port, NULL, id, 0);
 }
@@ -368,19 +436,13 @@ void zfcp_erp_port_forced_reopen(struct zfcp_port *port, int clear, char *id)
         write_unlock_irqrestore(&adapter->erp_lock, flags);
 }
 
-static int _zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id)
+static void _zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id)
 {
         zfcp_erp_port_block(port, clear);
         zfcp_scsi_schedule_rport_block(port);
 
-        if (atomic_read(&port->status) & ZFCP_STATUS_COMMON_ERP_FAILED) {
-                /* ensure propagation of failed status to new devices */
-                zfcp_erp_set_port_status(port, ZFCP_STATUS_COMMON_ERP_FAILED);
-                return -EIO;
-        }
-
-        return zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_PORT,
-                                       port->adapter, port, NULL, id, 0);
+        zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_PORT,
+                                port->adapter, port, NULL, id, 0);
 }
 
 /**
@@ -388,20 +450,15 @@ static int _zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id)
  * @port: port to recover
  * @clear_mask: flags in port status to be cleared
  * @id: Id for debug trace event.
- *
- * Returns 0 if recovery has been triggered, < 0 if not.
  */
-int zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id)
+void zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id)
 {
-        int retval;
         unsigned long flags;
         struct zfcp_adapter *adapter = port->adapter;
 
         write_lock_irqsave(&adapter->erp_lock, flags);
-        retval = _zfcp_erp_port_reopen(port, clear, id);
+        _zfcp_erp_port_reopen(port, clear, id);
         write_unlock_irqrestore(&adapter->erp_lock, flags);
-
-        return retval;
 }
 
 static void zfcp_erp_lun_block(struct scsi_device *sdev, int clear_mask)
@@ -418,9 +475,6 @@ static void _zfcp_erp_lun_reopen(struct scsi_device *sdev, int clear, char *id,
 
         zfcp_erp_lun_block(sdev, clear);
 
-        if (atomic_read(&zfcp_sdev->status) & ZFCP_STATUS_COMMON_ERP_FAILED)
-                return;
-
         zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_LUN, adapter,
                                 zfcp_sdev->port, sdev, id, act_status);
 }
@@ -482,21 +536,23 @@ void zfcp_erp_lun_shutdown_wait(struct scsi_device *sdev, char *id)
         zfcp_erp_wait(adapter);
 }
 
-static int status_change_set(unsigned long mask, atomic_t *status)
+static int zfcp_erp_status_change_set(unsigned long mask, atomic_t *status)
 {
         return (atomic_read(status) ^ mask) & mask;
 }
 
 static void zfcp_erp_adapter_unblock(struct zfcp_adapter *adapter)
 {
-        if (status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED, &adapter->status))
+        if (zfcp_erp_status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED,
+                                       &adapter->status))
                 zfcp_dbf_rec_run("eraubl1", &adapter->erp_action);
         atomic_or(ZFCP_STATUS_COMMON_UNBLOCKED, &adapter->status);
 }
 
 static void zfcp_erp_port_unblock(struct zfcp_port *port)
 {
-        if (status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED, &port->status))
+        if (zfcp_erp_status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED,
+                                       &port->status))
                 zfcp_dbf_rec_run("erpubl1", &port->erp_action);
         atomic_or(ZFCP_STATUS_COMMON_UNBLOCKED, &port->status);
 }
@@ -505,7 +561,8 @@ static void zfcp_erp_lun_unblock(struct scsi_device *sdev)
 {
         struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev);
 
-        if (status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED, &zfcp_sdev->status))
+        if (zfcp_erp_status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED,
+                                       &zfcp_sdev->status))
                 zfcp_dbf_rec_run("erlubl1", &sdev_to_zfcp(sdev)->erp_action);
         atomic_or(ZFCP_STATUS_COMMON_UNBLOCKED, &zfcp_sdev->status);
 }
@@ -553,7 +610,7 @@ void zfcp_erp_notify(struct zfcp_erp_action *erp_action, unsigned long set_mask)
         unsigned long flags;
 
         write_lock_irqsave(&adapter->erp_lock, flags);
-        if (zfcp_erp_action_exists(erp_action) == ZFCP_ERP_ACTION_RUNNING) {
+        if (zfcp_erp_action_is_running(erp_action)) {
                 erp_action->status |= set_mask;
                 zfcp_erp_action_ready(erp_action);
         }
@@ -1634,3 +1691,14 @@ void zfcp_erp_clear_lun_status(struct scsi_device *sdev, u32 mask)
                 atomic_set(&zfcp_sdev->erp_counter, 0);
 }
 
+/**
+ * zfcp_erp_adapter_reset_sync() - Really reopen adapter and wait.
+ * @adapter: Pointer to zfcp_adapter to reopen.
+ * @id: Trace tag string of length %ZFCP_DBF_TAG_LEN.
+ */
+void zfcp_erp_adapter_reset_sync(struct zfcp_adapter *adapter, char *id)
+{
+        zfcp_erp_set_adapter_status(adapter, ZFCP_STATUS_COMMON_RUNNING);
+        zfcp_erp_adapter_reopen(adapter, ZFCP_STATUS_COMMON_ERP_FAILED, id);
+        zfcp_erp_wait(adapter);
+}
diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h
index e5eed8aac0ce..bd0c5a9f04cb 100644
--- a/drivers/s390/scsi/zfcp_ext.h
+++ b/drivers/s390/scsi/zfcp_ext.h
@@ -50,17 +50,23 @@ extern void zfcp_dbf_hba_basic(char *, struct zfcp_adapter *);
 extern void zfcp_dbf_san_req(char *, struct zfcp_fsf_req *, u32);
 extern void zfcp_dbf_san_res(char *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_san_in_els(char *, struct zfcp_fsf_req *);
-extern void zfcp_dbf_scsi(char *, int, struct scsi_cmnd *,
-                          struct zfcp_fsf_req *);
+extern void zfcp_dbf_scsi_common(char *tag, int level, struct scsi_device *sdev,
+                                 struct scsi_cmnd *sc,
+                                 struct zfcp_fsf_req *fsf);
+extern void zfcp_dbf_scsi_eh(char *tag, struct zfcp_adapter *adapter,
+                             unsigned int scsi_id, int ret);
 
 /* zfcp_erp.c */
 extern void zfcp_erp_set_adapter_status(struct zfcp_adapter *, u32);
 extern void zfcp_erp_clear_adapter_status(struct zfcp_adapter *, u32);
+extern void zfcp_erp_port_forced_no_port_dbf(char *id,
+                                             struct zfcp_adapter *adapter,
+                                             u64 port_name, u32 port_id);
 extern void zfcp_erp_adapter_reopen(struct zfcp_adapter *, int, char *);
 extern void zfcp_erp_adapter_shutdown(struct zfcp_adapter *, int, char *);
 extern void zfcp_erp_set_port_status(struct zfcp_port *, u32);
 extern void zfcp_erp_clear_port_status(struct zfcp_port *, u32);
-extern int  zfcp_erp_port_reopen(struct zfcp_port *, int, char *);
+extern void zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id);
 extern void zfcp_erp_port_shutdown(struct zfcp_port *, int, char *);
 extern void zfcp_erp_port_forced_reopen(struct zfcp_port *, int, char *);
 extern void zfcp_erp_set_lun_status(struct scsi_device *, u32);
@@ -73,6 +79,7 @@ extern void zfcp_erp_thread_kill(struct zfcp_adapter *);
 extern void zfcp_erp_wait(struct zfcp_adapter *);
 extern void zfcp_erp_notify(struct zfcp_erp_action *, unsigned long);
 extern void zfcp_erp_timeout_handler(struct timer_list *t);
+extern void zfcp_erp_adapter_reset_sync(struct zfcp_adapter *adapter, char *id);
 
 /* zfcp_fc.c */
 extern struct kmem_cache *zfcp_fc_req_cache;
@@ -120,7 +127,8 @@ extern int zfcp_fsf_send_els(struct zfcp_adapter *, u32,
                              struct zfcp_fsf_ct_els *, unsigned int);
 extern int zfcp_fsf_fcp_cmnd(struct scsi_cmnd *);
 extern void zfcp_fsf_req_free(struct zfcp_fsf_req *);
-extern struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_cmnd *, u8);
+extern struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_device *sdev,
+                                                   u8 tm_flags);
 extern struct zfcp_fsf_req *zfcp_fsf_abort_fcp_cmnd(struct scsi_cmnd *);
 extern void zfcp_fsf_reqid_check(struct zfcp_qdio *, int);
 
diff --git a/drivers/s390/scsi/zfcp_fc.c b/drivers/s390/scsi/zfcp_fc.c
index 6162cf57a20a..f6c415d6ef48 100644
--- a/drivers/s390/scsi/zfcp_fc.c
+++ b/drivers/s390/scsi/zfcp_fc.c
@@ -111,11 +111,10 @@ void zfcp_fc_post_event(struct work_struct *work)
 
         list_for_each_entry_safe(event, tmp, &tmp_lh, list) {
                 fc_host_post_event(adapter->scsi_host, fc_get_event_number(),
-                                event->code, event->data);
+                                   event->code, event->data);
                 list_del(&event->list);
                 kfree(event);
         }
-
 }
 
 /**
@@ -126,7 +125,7 @@ void zfcp_fc_post_event(struct work_struct *work)
  * @event_data: The event data (e.g. n_port page in case of els)
  */
 void zfcp_fc_enqueue_event(struct zfcp_adapter *adapter,
-                        enum fc_host_event_code event_code, u32 event_data)
+                           enum fc_host_event_code event_code, u32 event_data)
 {
         struct zfcp_fc_event *event;
 
@@ -425,6 +424,7 @@ void zfcp_fc_port_did_lookup(struct work_struct *work)
         struct zfcp_port *port = container_of(work, struct zfcp_port,
                                               gid_pn_work);
 
+        set_worker_desc("zgidpn%16llx", port->wwpn); /* < WORKER_DESC_LEN=24 */
         ret = zfcp_fc_ns_gid_pn(port);
         if (ret) {
                 /* could not issue gid_pn for some reason */
@@ -559,6 +559,7 @@ void zfcp_fc_link_test_work(struct work_struct *work)
                 container_of(work, struct zfcp_port, test_link_work);
         int retval;
 
+        set_worker_desc("zadisc%16llx", port->wwpn); /* < WORKER_DESC_LEN=24 */
         get_device(&port->dev);
         port->rport_task = RPORT_DEL;
         zfcp_scsi_rport_work(&port->rport_work);
@@ -596,7 +597,7 @@ void zfcp_fc_test_link(struct zfcp_port *port)
                 put_device(&port->dev);
 }
 
-static struct zfcp_fc_req *zfcp_alloc_sg_env(int buf_num)
+static struct zfcp_fc_req *zfcp_fc_alloc_sg_env(int buf_num)
 {
         struct zfcp_fc_req *fc_req;
 
@@ -748,7 +749,7 @@ void zfcp_fc_scan_ports(struct work_struct *work)
         if (zfcp_fc_wka_port_get(&adapter->gs->ds))
                 return;
 
-        fc_req = zfcp_alloc_sg_env(buf_num);
+        fc_req = zfcp_fc_alloc_sg_env(buf_num);
         if (!fc_req)
                 goto out;
 
diff --git a/drivers/s390/scsi/zfcp_fc.h b/drivers/s390/scsi/zfcp_fc.h
index 6a397ddaadf0..3cd74729cfb9 100644
--- a/drivers/s390/scsi/zfcp_fc.h
+++ b/drivers/s390/scsi/zfcp_fc.h
@@ -207,21 +207,14 @@ struct zfcp_fc_wka_ports {
  * zfcp_fc_scsi_to_fcp - setup FCP command with data from scsi_cmnd
  * @fcp: fcp_cmnd to setup
  * @scsi: scsi_cmnd where to get LUN, task attributes/flags and CDB
- * @tm: task management flags to setup task management command
  */
 static inline
-void zfcp_fc_scsi_to_fcp(struct fcp_cmnd *fcp, struct scsi_cmnd *scsi,
-                         u8 tm_flags)
+void zfcp_fc_scsi_to_fcp(struct fcp_cmnd *fcp, struct scsi_cmnd *scsi)
 {
         u32 datalen;
 
         int_to_scsilun(scsi->device->lun, (struct scsi_lun *) &fcp->fc_lun);
 
-        if (unlikely(tm_flags)) {
-                fcp->fc_tm_flags = tm_flags;
-                return;
-        }
-
         fcp->fc_pri_ta = FCP_PTA_SIMPLE;
 
         if (scsi->sc_data_direction == DMA_FROM_DEVICE)
@@ -241,6 +234,19 @@ void zfcp_fc_scsi_to_fcp(struct fcp_cmnd *fcp, struct scsi_cmnd *scsi,
 }
 
 /**
+ * zfcp_fc_fcp_tm() - Setup FCP command as task management command.
+ * @fcp: Pointer to FCP_CMND IU to set up.
+ * @dev: Pointer to SCSI_device where to send the task management command.
+ * @tm_flags: Task management flags to setup tm command.
+ */
+static inline
+void zfcp_fc_fcp_tm(struct fcp_cmnd *fcp, struct scsi_device *dev, u8 tm_flags)
+{
+        int_to_scsilun(dev->lun, (struct scsi_lun *) &fcp->fc_lun);
+        fcp->fc_tm_flags = tm_flags;
+}
+
+/**
  * zfcp_fc_evap_fcp_rsp - evaluate FCP RSP IU and update scsi_cmnd accordingly
  * @fcp_rsp: FCP RSP IU to evaluate
  * @scsi: SCSI command where to update status and sense buffer
diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c
index b12cb81ad8a2..3c86e27f094d 100644
--- a/drivers/s390/scsi/zfcp_fsf.c
+++ b/drivers/s390/scsi/zfcp_fsf.c
@@ -4,7 +4,7 @@
  *
  * Implementation of FSF commands.
  *
- * Copyright IBM Corp. 2002, 2017
+ * Copyright IBM Corp. 2002, 2018
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -437,6 +437,9 @@ void zfcp_fsf_req_dismiss_all(struct zfcp_adapter *adapter)
 #define ZFCP_FSF_PORTSPEED_10GBIT       (1 <<  3)
 #define ZFCP_FSF_PORTSPEED_8GBIT        (1 <<  4)
 #define ZFCP_FSF_PORTSPEED_16GBIT       (1 <<  5)
+#define ZFCP_FSF_PORTSPEED_32GBIT       (1 <<  6)
+#define ZFCP_FSF_PORTSPEED_64GBIT       (1 <<  7)
+#define ZFCP_FSF_PORTSPEED_128GBIT      (1 <<  8)
 #define ZFCP_FSF_PORTSPEED_NOT_NEGOTIATED (1 << 15)
 
 static u32 zfcp_fsf_convert_portspeed(u32 fsf_speed)
@@ -454,6 +457,12 @@ static u32 zfcp_fsf_convert_portspeed(u32 fsf_speed)
                 fdmi_speed |= FC_PORTSPEED_8GBIT;
         if (fsf_speed & ZFCP_FSF_PORTSPEED_16GBIT)
                 fdmi_speed |= FC_PORTSPEED_16GBIT;
+        if (fsf_speed & ZFCP_FSF_PORTSPEED_32GBIT)
+                fdmi_speed |= FC_PORTSPEED_32GBIT;
+        if (fsf_speed & ZFCP_FSF_PORTSPEED_64GBIT)
+                fdmi_speed |= FC_PORTSPEED_64GBIT;
+        if (fsf_speed & ZFCP_FSF_PORTSPEED_128GBIT)
+                fdmi_speed |= FC_PORTSPEED_128GBIT;
         if (fsf_speed & ZFCP_FSF_PORTSPEED_NOT_NEGOTIATED)
                 fdmi_speed |= FC_PORTSPEED_NOT_NEGOTIATED;
         return fdmi_speed;
@@ -662,7 +671,7 @@ static struct zfcp_fsf_req *zfcp_fsf_alloc(mempool_t *pool)
         return req;
 }
 
-static struct fsf_qtcb *zfcp_qtcb_alloc(mempool_t *pool)
+static struct fsf_qtcb *zfcp_fsf_qtcb_alloc(mempool_t *pool)
 {
         struct fsf_qtcb *qtcb;
 
@@ -701,9 +710,10 @@ static struct zfcp_fsf_req *zfcp_fsf_req_create(struct zfcp_qdio *qdio,
 
         if (likely(fsf_cmd != FSF_QTCB_UNSOLICITED_STATUS)) {
                 if (likely(pool))
-                        req->qtcb = zfcp_qtcb_alloc(adapter->pool.qtcb_pool);
+                        req->qtcb = zfcp_fsf_qtcb_alloc(
+                                adapter->pool.qtcb_pool);
                 else
-                        req->qtcb = zfcp_qtcb_alloc(NULL);
+                        req->qtcb = zfcp_fsf_qtcb_alloc(NULL);
 
                 if (unlikely(!req->qtcb)) {
                         zfcp_fsf_req_free(req);
@@ -2036,10 +2046,14 @@ static void zfcp_fsf_req_trace(struct zfcp_fsf_req *req, struct scsi_cmnd *scsi)
                             sizeof(blktrc));
 }
 
-static void zfcp_fsf_fcp_handler_common(struct zfcp_fsf_req *req)
+/**
+ * zfcp_fsf_fcp_handler_common() - FCP response handler common to I/O and TMF.
+ * @req: Pointer to FSF request.
+ * @sdev: Pointer to SCSI device as request context.
+ */
+static void zfcp_fsf_fcp_handler_common(struct zfcp_fsf_req *req,
+                                        struct scsi_device *sdev)
 {
-        struct scsi_cmnd *scmnd = req->data;
-        struct scsi_device *sdev = scmnd->device;
         struct zfcp_scsi_dev *zfcp_sdev;
         struct fsf_qtcb_header *header = &req->qtcb->header;
 
@@ -2051,7 +2065,7 @@ static void zfcp_fsf_fcp_handler_common(struct zfcp_fsf_req *req)
         switch (header->fsf_status) {
         case FSF_HANDLE_MISMATCH:
         case FSF_PORT_HANDLE_NOT_VALID:
-                zfcp_erp_adapter_reopen(zfcp_sdev->port->adapter, 0, "fssfch1");
+                zfcp_erp_adapter_reopen(req->adapter, 0, "fssfch1");
                 req->status |= ZFCP_STATUS_FSFREQ_ERROR;
                 break;
         case FSF_FCPLUN_NOT_VALID:
@@ -2069,8 +2083,7 @@ static void zfcp_fsf_fcp_handler_common(struct zfcp_fsf_req *req)
                         req->qtcb->bottom.io.data_direction,
                         (unsigned long long)zfcp_scsi_dev_lun(sdev),
                         (unsigned long long)zfcp_sdev->port->wwpn);
-                zfcp_erp_adapter_shutdown(zfcp_sdev->port->adapter, 0,
-                                          "fssfch3");
+                zfcp_erp_adapter_shutdown(req->adapter, 0, "fssfch3");
                 req->status |= ZFCP_STATUS_FSFREQ_ERROR;
                 break;
         case FSF_CMND_LENGTH_NOT_VALID:
@@ -2080,8 +2093,7 @@ static void zfcp_fsf_fcp_handler_common(struct zfcp_fsf_req *req)
                         req->qtcb->bottom.io.fcp_cmnd_length,
                         (unsigned long long)zfcp_scsi_dev_lun(sdev),
                         (unsigned long long)zfcp_sdev->port->wwpn);
-                zfcp_erp_adapter_shutdown(zfcp_sdev->port->adapter, 0,
-                                          "fssfch4");
+                zfcp_erp_adapter_shutdown(req->adapter, 0, "fssfch4");
                 req->status |= ZFCP_STATUS_FSFREQ_ERROR;
                 break;
         case FSF_PORT_BOXED:
@@ -2120,7 +2132,7 @@ static void zfcp_fsf_fcp_cmnd_handler(struct zfcp_fsf_req *req)
                 return;
         }
 
-        zfcp_fsf_fcp_handler_common(req);
+        zfcp_fsf_fcp_handler_common(req, scpnt->device);
 
         if (unlikely(req->status & ZFCP_STATUS_FSFREQ_ERROR)) {
                 set_host_byte(scpnt, DID_TRANSPORT_DISRUPTED);
@@ -2258,7 +2270,7 @@ int zfcp_fsf_fcp_cmnd(struct scsi_cmnd *scsi_cmnd)
 
         BUILD_BUG_ON(sizeof(struct fcp_cmnd) > FSF_FCP_CMND_SIZE);
         fcp_cmnd = &req->qtcb->bottom.io.fcp_cmnd.iu;
-        zfcp_fc_scsi_to_fcp(fcp_cmnd, scsi_cmnd, 0);
+        zfcp_fc_scsi_to_fcp(fcp_cmnd, scsi_cmnd);
 
         if ((scsi_get_prot_op(scsi_cmnd) != SCSI_PROT_NORMAL) &&
             scsi_prot_sg_count(scsi_cmnd)) {
@@ -2297,10 +2309,11 @@ out:
 
 static void zfcp_fsf_fcp_task_mgmt_handler(struct zfcp_fsf_req *req)
 {
+        struct scsi_device *sdev = req->data;
         struct fcp_resp_with_ext *fcp_rsp;
         struct fcp_resp_rsp_info *rsp_info;
 
-        zfcp_fsf_fcp_handler_common(req);
+        zfcp_fsf_fcp_handler_common(req, sdev);
 
         fcp_rsp = &req->qtcb->bottom.io.fcp_rsp.iu;
         rsp_info = (struct fcp_resp_rsp_info *) &fcp_rsp[1];
@@ -2311,17 +2324,18 @@ static void zfcp_fsf_fcp_task_mgmt_handler(struct zfcp_fsf_req *req)
 }
 
 /**
- * zfcp_fsf_fcp_task_mgmt - send SCSI task management command
- * @scmnd: SCSI command to send the task management command for
- * @tm_flags: unsigned byte for task management flags
- * Returns: on success pointer to struct fsf_req, NULL otherwise
+ * zfcp_fsf_fcp_task_mgmt() - Send SCSI task management command (TMF).
+ * @sdev: Pointer to SCSI device to send the task management command to.
+ * @tm_flags: Unsigned byte for task management flags.
+ *
+ * Return: On success pointer to struct zfcp_fsf_req, %NULL otherwise.
  */
-struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_cmnd *scmnd,
+struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_device *sdev,
                                             u8 tm_flags)
 {
         struct zfcp_fsf_req *req = NULL;
         struct fcp_cmnd *fcp_cmnd;
-        struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(scmnd->device);
+        struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev);
         struct zfcp_qdio *qdio = zfcp_sdev->port->adapter->qdio;
 
         if (unlikely(!(atomic_read(&zfcp_sdev->status) &
@@ -2341,7 +2355,8 @@ struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_cmnd *scmnd,
                 goto out;
         }
 
-        req->data = scmnd;
+        req->data = sdev;
+
         req->handler = zfcp_fsf_fcp_task_mgmt_handler;
         req->qtcb->header.lun_handle = zfcp_sdev->lun_handle;
         req->qtcb->header.port_handle = zfcp_sdev->port->handle;
@@ -2352,7 +2367,7 @@ struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_cmnd *scmnd,
         zfcp_qdio_set_sbale_last(qdio, &req->qdio_req);
 
         fcp_cmnd = &req->qtcb->bottom.io.fcp_cmnd.iu;
-        zfcp_fc_scsi_to_fcp(fcp_cmnd, scmnd, tm_flags);
+        zfcp_fc_fcp_tm(fcp_cmnd, sdev, tm_flags);
 
         zfcp_fsf_start_timer(req, ZFCP_SCSI_ER_TIMEOUT);
         if (!zfcp_fsf_req_send(req))
diff --git a/drivers/s390/scsi/zfcp_fsf.h b/drivers/s390/scsi/zfcp_fsf.h
index 4baca67aba6d..535628b92f0a 100644
--- a/drivers/s390/scsi/zfcp_fsf.h
+++ b/drivers/s390/scsi/zfcp_fsf.h
@@ -4,7 +4,7 @@
  *
  * Interface to the FSF support functions.
  *
- * Copyright IBM Corp. 2002, 2017
+ * Copyright IBM Corp. 2002, 2018
  */
 
 #ifndef FSF_H
@@ -356,7 +356,7 @@ struct fsf_qtcb_bottom_config {
         u32 adapter_features;
         u32 connection_features;
         u32 fc_topology;
-        u32 fc_link_speed;
+        u32 fc_link_speed;      /* one of ZFCP_FSF_PORTSPEED_* */
         u32 adapter_type;
         u8 res0;
         u8 peer_d_id[3];
@@ -382,7 +382,7 @@ struct fsf_qtcb_bottom_port {
         u32 class_of_service;   /* should be 0x00000006 for class 2 and 3 */
         u8 supported_fc4_types[32]; /* should be 0x00000100 for scsi fcp */
         u8 active_fc4_types[32];
-        u32 supported_speed;    /* 0x0001 for 1 GBit/s or 0x0002 for 2 GBit/s */
+        u32 supported_speed;    /* any combination of ZFCP_FSF_PORTSPEED_* */
         u32 maximum_frame_size; /* fixed value of 2112 */
         u64 seconds_since_last_reset;
         u64 tx_frames;
diff --git a/drivers/s390/scsi/zfcp_scsi.c b/drivers/s390/scsi/zfcp_scsi.c
index 22f9562f415c..a8efcb330bc1 100644
--- a/drivers/s390/scsi/zfcp_scsi.c
+++ b/drivers/s390/scsi/zfcp_scsi.c
@@ -181,6 +181,7 @@ static int zfcp_scsi_eh_abort_handler(struct scsi_cmnd *scpnt)
                 if (abrt_req)
                         break;
 
+                zfcp_dbf_scsi_abort("abrt_wt", scpnt, NULL);
                 zfcp_erp_wait(adapter);
                 ret = fc_block_scsi_eh(scpnt);
                 if (ret) {
@@ -264,44 +265,52 @@ static void zfcp_scsi_forget_cmnds(struct zfcp_scsi_dev *zsdev, u8 tm_flags)
         write_unlock_irqrestore(&adapter->abort_lock, flags);
 }
 
-static int zfcp_task_mgmt_function(struct scsi_cmnd *scpnt, u8 tm_flags)
+/**
+ * zfcp_scsi_task_mgmt_function() - Send a task management function (sync).
+ * @sdev: Pointer to SCSI device to send the task management command to.
+ * @tm_flags: Task management flags,
+ *            here we only handle %FCP_TMF_TGT_RESET or %FCP_TMF_LUN_RESET.
+ */
+static int zfcp_scsi_task_mgmt_function(struct scsi_device *sdev, u8 tm_flags)
 {
-        struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(scpnt->device);
+        struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev);
         struct zfcp_adapter *adapter = zfcp_sdev->port->adapter;
+        struct fc_rport *rport = starget_to_rport(scsi_target(sdev));
         struct zfcp_fsf_req *fsf_req = NULL;
         int retval = SUCCESS, ret;
         int retry = 3;
 
         while (retry--) {
-                fsf_req = zfcp_fsf_fcp_task_mgmt(scpnt, tm_flags);
+                fsf_req = zfcp_fsf_fcp_task_mgmt(sdev, tm_flags);
                 if (fsf_req)
                         break;
 
+                zfcp_dbf_scsi_devreset("wait", sdev, tm_flags, NULL);
                 zfcp_erp_wait(adapter);
-                ret = fc_block_scsi_eh(scpnt);
+                ret = fc_block_rport(rport);
                 if (ret) {
-                        zfcp_dbf_scsi_devreset("fiof", scpnt, tm_flags, NULL);
+                        zfcp_dbf_scsi_devreset("fiof", sdev, tm_flags, NULL);
                         return ret;
                 }
 
                 if (!(atomic_read(&adapter->status) &
                       ZFCP_STATUS_COMMON_RUNNING)) {
-                        zfcp_dbf_scsi_devreset("nres", scpnt, tm_flags, NULL);
+                        zfcp_dbf_scsi_devreset("nres", sdev, tm_flags, NULL);
                         return SUCCESS;
                 }
         }
         if (!fsf_req) {
-                zfcp_dbf_scsi_devreset("reqf", scpnt, tm_flags, NULL);
+                zfcp_dbf_scsi_devreset("reqf", sdev, tm_flags, NULL);
                 return FAILED;
         }
 
         wait_for_completion(&fsf_req->completion);
 
         if (fsf_req->status & ZFCP_STATUS_FSFREQ_TMFUNCFAILED) {
-                zfcp_dbf_scsi_devreset("fail", scpnt, tm_flags, fsf_req);
+                zfcp_dbf_scsi_devreset("fail", sdev, tm_flags, fsf_req);
                 retval = FAILED;
         } else {
-                zfcp_dbf_scsi_devreset("okay", scpnt, tm_flags, fsf_req);
+                zfcp_dbf_scsi_devreset("okay", sdev, tm_flags, fsf_req);
                 zfcp_scsi_forget_cmnds(zfcp_sdev, tm_flags);
         }
 
@@ -311,27 +320,81 @@ static int zfcp_task_mgmt_function(struct scsi_cmnd *scpnt, u8 tm_flags)
 
 static int zfcp_scsi_eh_device_reset_handler(struct scsi_cmnd *scpnt)
 {
-        return zfcp_task_mgmt_function(scpnt, FCP_TMF_LUN_RESET);
+        struct scsi_device *sdev = scpnt->device;
+
+        return zfcp_scsi_task_mgmt_function(sdev, FCP_TMF_LUN_RESET);
 }
 
 static int zfcp_scsi_eh_target_reset_handler(struct scsi_cmnd *scpnt)
 {
-        return zfcp_task_mgmt_function(scpnt, FCP_TMF_TGT_RESET);
+        struct scsi_target *starget = scsi_target(scpnt->device);
+        struct fc_rport *rport = starget_to_rport(starget);
+        struct Scsi_Host *shost = rport_to_shost(rport);
+        struct scsi_device *sdev = NULL, *tmp_sdev;
+        struct zfcp_adapter *adapter =
+                (struct zfcp_adapter *)shost->hostdata[0];
+        int ret;
+
+        shost_for_each_device(tmp_sdev, shost) {
+                if (tmp_sdev->id == starget->id) {
+                        sdev = tmp_sdev;
+                        break;
+                }
+        }
+        if (!sdev) {
+                ret = FAILED;
+                zfcp_dbf_scsi_eh("tr_nosd", adapter, starget->id, ret);
+                return ret;
+        }
+
+        ret = zfcp_scsi_task_mgmt_function(sdev, FCP_TMF_TGT_RESET);
+
+        /* release reference from above shost_for_each_device */
+        if (sdev)
+                scsi_device_put(tmp_sdev);
+
+        return ret;
 }
 
 static int zfcp_scsi_eh_host_reset_handler(struct scsi_cmnd *scpnt)
 {
         struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(scpnt->device);
         struct zfcp_adapter *adapter = zfcp_sdev->port->adapter;
-        int ret;
+        int ret = SUCCESS, fc_ret;
 
         zfcp_erp_adapter_reopen(adapter, 0, "schrh_1");
         zfcp_erp_wait(adapter);
-        ret = fc_block_scsi_eh(scpnt);
-        if (ret)
+        fc_ret = fc_block_scsi_eh(scpnt);
+        if (fc_ret)
+                ret = fc_ret;
+
+        zfcp_dbf_scsi_eh("schrh_r", adapter, ~0, ret);
+        return ret;
+}
+
+/**
+ * zfcp_scsi_sysfs_host_reset() - Support scsi_host sysfs attribute host_reset.
+ * @shost: Pointer to Scsi_Host to perform action on.
+ * @reset_type: We support %SCSI_ADAPTER_RESET but not %SCSI_FIRMWARE_RESET.
+ *
+ * Return: 0 on %SCSI_ADAPTER_RESET, -%EOPNOTSUPP otherwise.
+ *
+ * This is similar to zfcp_sysfs_adapter_failed_store().
+ */
+static int zfcp_scsi_sysfs_host_reset(struct Scsi_Host *shost, int reset_type)
+{
+        struct zfcp_adapter *adapter =
+                (struct zfcp_adapter *)shost->hostdata[0];
+        int ret = 0;
+
+        if (reset_type != SCSI_ADAPTER_RESET) {
+                ret = -EOPNOTSUPP;
+                zfcp_dbf_scsi_eh("scshr_n", adapter, ~0, ret);
                 return ret;
+        }
 
-        return SUCCESS;
+        zfcp_erp_adapter_reset_sync(adapter, "scshr_y");
+        return ret;
 }
 
 struct scsi_transport_template *zfcp_scsi_transport_template;
@@ -349,6 +412,7 @@ static struct scsi_host_template zfcp_scsi_host_template = {
         .slave_configure         = zfcp_scsi_slave_configure,
         .slave_destroy           = zfcp_scsi_slave_destroy,
         .change_queue_depth      = scsi_change_queue_depth,
+        .host_reset              = zfcp_scsi_sysfs_host_reset,
         .proc_name               = "zfcp",
         .can_queue               = 4096,
         .this_id                 = -1,
@@ -363,6 +427,7 @@ static struct scsi_host_template zfcp_scsi_host_template = {
         .shost_attrs             = zfcp_sysfs_shost_attrs,
         .sdev_attrs              = zfcp_sysfs_sdev_attrs,
         .track_queue_depth       = 1,
+        .supported_mode          = MODE_INITIATOR,
 };
 
 /**
@@ -430,7 +495,7 @@ void zfcp_scsi_adapter_unregister(struct zfcp_adapter *adapter)
 }
 
 static struct fc_host_statistics*
-zfcp_init_fc_host_stats(struct zfcp_adapter *adapter)
+zfcp_scsi_init_fc_host_stats(struct zfcp_adapter *adapter)
 {
         struct fc_host_statistics *fc_stats;
 
@@ -444,9 +509,9 @@ zfcp_init_fc_host_stats(struct zfcp_adapter *adapter)
         return adapter->fc_stats;
 }
 
-static void zfcp_adjust_fc_host_stats(struct fc_host_statistics *fc_stats,
-                                      struct fsf_qtcb_bottom_port *data,
-                                      struct fsf_qtcb_bottom_port *old)
+static void zfcp_scsi_adjust_fc_host_stats(struct fc_host_statistics *fc_stats,
+                                           struct fsf_qtcb_bottom_port *data,
+                                           struct fsf_qtcb_bottom_port *old)
 {
         fc_stats->seconds_since_last_reset =
                 data->seconds_since_last_reset - old->seconds_since_last_reset;
@@ -477,8 +542,8 @@ static void zfcp_adjust_fc_host_stats(struct fc_host_statistics *fc_stats,
         fc_stats->fcp_output_megabytes = data->output_mb - old->output_mb;
 }
 
-static void zfcp_set_fc_host_stats(struct fc_host_statistics *fc_stats,
-                                   struct fsf_qtcb_bottom_port *data)
+static void zfcp_scsi_set_fc_host_stats(struct fc_host_statistics *fc_stats,
+                                        struct fsf_qtcb_bottom_port *data)
 {
         fc_stats->seconds_since_last_reset = data->seconds_since_last_reset;
         fc_stats->tx_frames = data->tx_frames;
@@ -502,7 +567,8 @@ static void zfcp_set_fc_host_stats(struct fc_host_statistics *fc_stats,
         fc_stats->fcp_output_megabytes = data->output_mb;
 }
 
-static struct fc_host_statistics *zfcp_get_fc_host_stats(struct Scsi_Host *host)
+static struct fc_host_statistics *
+zfcp_scsi_get_fc_host_stats(struct Scsi_Host *host)
 {
         struct zfcp_adapter *adapter;
         struct fc_host_statistics *fc_stats;
@@ -510,7 +576,7 @@ static struct fc_host_statistics *zfcp_get_fc_host_stats(struct Scsi_Host *host)
         int ret;
 
         adapter = (struct zfcp_adapter *)host->hostdata[0];
-        fc_stats = zfcp_init_fc_host_stats(adapter);
+        fc_stats = zfcp_scsi_init_fc_host_stats(adapter);
         if (!fc_stats)
                 return NULL;
 
@@ -527,16 +593,16 @@ static struct fc_host_statistics *zfcp_get_fc_host_stats(struct Scsi_Host *host)
         if (adapter->stats_reset &&
             ((jiffies/HZ - adapter->stats_reset) <
              data->seconds_since_last_reset))
-                zfcp_adjust_fc_host_stats(fc_stats, data,
-                                          adapter->stats_reset_data);
+                zfcp_scsi_adjust_fc_host_stats(fc_stats, data,
+                                               adapter->stats_reset_data);
         else
-                zfcp_set_fc_host_stats(fc_stats, data);
+                zfcp_scsi_set_fc_host_stats(fc_stats, data);
 
         kfree(data);
         return fc_stats;
 }
 
-static void zfcp_reset_fc_host_stats(struct Scsi_Host *shost)
+static void zfcp_scsi_reset_fc_host_stats(struct Scsi_Host *shost)
 {
         struct zfcp_adapter *adapter;
         struct fsf_qtcb_bottom_port *data;
@@ -558,7 +624,7 @@ static void zfcp_reset_fc_host_stats(struct Scsi_Host *shost)
         }
 }
 
-static void zfcp_get_host_port_state(struct Scsi_Host *shost)
+static void zfcp_scsi_get_host_port_state(struct Scsi_Host *shost)
 {
         struct zfcp_adapter *adapter =
                 (struct zfcp_adapter *)shost->hostdata[0];
@@ -575,7 +641,8 @@ static void zfcp_get_host_port_state(struct Scsi_Host *shost)
                 fc_host_port_state(shost) = FC_PORTSTATE_UNKNOWN;
 }
 
-static void zfcp_set_rport_dev_loss_tmo(struct fc_rport *rport, u32 timeout)
+static void zfcp_scsi_set_rport_dev_loss_tmo(struct fc_rport *rport,
+                                             u32 timeout)
 {
         rport->dev_loss_tmo = timeout;
 }
@@ -602,6 +669,11 @@ static void zfcp_scsi_terminate_rport_io(struct fc_rport *rport)
         if (port) {
                 zfcp_erp_port_forced_reopen(port, 0, "sctrpi1");
                 put_device(&port->dev);
+        } else {
+                zfcp_erp_port_forced_no_port_dbf(
+                        "sctrpin", adapter,
+                        rport->port_name /* zfcp_scsi_rport_register */,
+                        rport->port_id /* zfcp_scsi_rport_register */);
         }
 }
 
@@ -687,6 +759,9 @@ void zfcp_scsi_rport_work(struct work_struct *work)
         struct zfcp_port *port = container_of(work, struct zfcp_port,
                                               rport_work);
 
+        set_worker_desc("zrp%c-%16llx",
+                        (port->rport_task == RPORT_ADD) ? 'a' : 'd',
+                        port->wwpn); /* < WORKER_DESC_LEN=24 */
         while (port->rport_task) {
                 if (port->rport_task == RPORT_ADD) {
                         port->rport_task = RPORT_NONE;
@@ -761,10 +836,10 @@ struct fc_function_template zfcp_transport_functions = {
         .show_host_supported_speeds = 1,
         .show_host_maxframe_size = 1,
         .show_host_serial_number = 1,
-        .get_fc_host_stats = zfcp_get_fc_host_stats,
-        .reset_fc_host_stats = zfcp_reset_fc_host_stats,
-        .set_rport_dev_loss_tmo = zfcp_set_rport_dev_loss_tmo,
-        .get_host_port_state = zfcp_get_host_port_state,
+        .get_fc_host_stats = zfcp_scsi_get_fc_host_stats,
+        .reset_fc_host_stats = zfcp_scsi_reset_fc_host_stats,
+        .set_rport_dev_loss_tmo = zfcp_scsi_set_rport_dev_loss_tmo,
+        .get_host_port_state = zfcp_scsi_get_host_port_state,
         .terminate_rport_io = zfcp_scsi_terminate_rport_io,
         .show_host_port_state = 1,
         .show_host_active_fc4s = 1,
diff --git a/drivers/s390/scsi/zfcp_sysfs.c b/drivers/s390/scsi/zfcp_sysfs.c
index 3ac823f2540f..b277be6f7611 100644
--- a/drivers/s390/scsi/zfcp_sysfs.c
+++ b/drivers/s390/scsi/zfcp_sysfs.c
@@ -200,10 +200,7 @@ static ssize_t zfcp_sysfs_adapter_failed_store(struct device *dev,
                 goto out;
         }
 
-        zfcp_erp_set_adapter_status(adapter, ZFCP_STATUS_COMMON_RUNNING);
-        zfcp_erp_adapter_reopen(adapter, ZFCP_STATUS_COMMON_ERP_FAILED,
-                                "syafai2");
-        zfcp_erp_wait(adapter);
+        zfcp_erp_adapter_reset_sync(adapter, "syafai2");
 out:
         zfcp_ccw_adapter_put(adapter);
         return retval ? retval : (ssize_t) count;
diff --git a/drivers/scsi/3w-9xxx.c b/drivers/scsi/3w-9xxx.c
index b42c9c479d4b..99ba4a770406 100644
--- a/drivers/scsi/3w-9xxx.c
+++ b/drivers/scsi/3w-9xxx.c
@@ -882,6 +882,11 @@ static int twa_chrdev_open(struct inode *inode, struct file *file)
         unsigned int minor_number;
         int retval = TW_IOCTL_ERROR_OS_ENODEV;
 
+        if (!capable(CAP_SYS_ADMIN)) {
+                retval = -EACCES;
+                goto out;
+        }
+
         minor_number = iminor(inode);
         if (minor_number >= twa_device_extension_count)
                 goto out;
diff --git a/drivers/scsi/3w-xxxx.c b/drivers/scsi/3w-xxxx.c
index 33261b690774..f6179e3d6953 100644
--- a/drivers/scsi/3w-xxxx.c
+++ b/drivers/scsi/3w-xxxx.c
@@ -1033,6 +1033,9 @@ static int tw_chrdev_open(struct inode *inode, struct file *file)
 
         dprintk(KERN_WARNING "3w-xxxx: tw_ioctl_open()\n");
 
+        if (!capable(CAP_SYS_ADMIN))
+                return -EACCES;
+
         minor_number = iminor(inode);
         if (minor_number >= tw_device_extension_count)
                 return -ENODEV;
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index 11e89e56b865..35c909bbf8ba 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -1351,6 +1351,20 @@ config SCSI_ZORRO7XX
               accelerator card for the Amiga 1200,
             - the SCSI controller on the GVP Turbo 040/060 accelerator.
 
+config SCSI_ZORRO_ESP
+        tristate "Zorro ESP SCSI support"
+        depends on ZORRO && SCSI
+        select SCSI_SPI_ATTRS
+        help
+          Support for various NCR53C9x (ESP) based SCSI controllers on Zorro
+          expansion boards for the Amiga.
+          This includes:
+            - the Phase5 Blizzard 1230 II and IV SCSI controllers,
+            - the Phase5 Blizzard 2060 SCSI controller,
+            - the Phase5 Blizzard Cyberstorm and Cyberstorm II SCSI
+              controllers,
+            - the Fastlane Zorro III SCSI controller.
+
 config ATARI_SCSI
         tristate "Atari native SCSI support"
         depends on ATARI && SCSI
diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile
index 56c940394729..80aca2456353 100644
--- a/drivers/scsi/Makefile
+++ b/drivers/scsi/Makefile
@@ -48,6 +48,7 @@ obj-$(CONFIG_INFINIBAND_ISER) 	+= libiscsi.o
 obj-$(CONFIG_ISCSI_BOOT_SYSFS)  += iscsi_boot_sysfs.o
 obj-$(CONFIG_SCSI_A4000T)       += 53c700.o     a4000t.o
 obj-$(CONFIG_SCSI_ZORRO7XX)     += 53c700.o     zorro7xx.o
+obj-$(CONFIG_SCSI_ZORRO_ESP)    += esp_scsi.o   zorro_esp.o
 obj-$(CONFIG_A3000_SCSI)        += a3000.o      wd33c93.o
 obj-$(CONFIG_A2091_SCSI)        += a2091.o      wd33c93.o
 obj-$(CONFIG_GVP11_SCSI)        += gvp11.o      wd33c93.o
@@ -189,7 +190,7 @@ $(obj)/53c700.o $(MODVERDIR)/$(obj)/53c700.ver: $(obj)/53c700_d.h
 $(obj)/scsi_sysfs.o: $(obj)/scsi_devinfo_tbl.c
 
 quiet_cmd_bflags = GEN     $@
-        cmd_bflags = sed -n 's/.*BLIST_\([A-Z0-9_]*\) *.*/BLIST_FLAG_NAME(\1),/p' $< > $@
+        cmd_bflags = sed -n 's/.*define *BLIST_\([A-Z0-9_]*\) *.*/BLIST_FLAG_NAME(\1),/p' $< > $@
 
 $(obj)/scsi_devinfo_tbl.c: include/scsi/scsi_devinfo.h
         $(call if_changed,bflags)
diff --git a/drivers/scsi/a100u2w.c b/drivers/scsi/a100u2w.c
index 8086bd0ac9fd..b2942ec3d455 100644
--- a/drivers/scsi/a100u2w.c
+++ b/drivers/scsi/a100u2w.c
@@ -1222,19 +1222,8 @@ static struct pci_driver inia100_pci_driver = {
         .remove         = inia100_remove_one,
 };
 
-static int __init inia100_init(void)
-{
-        return pci_register_driver(&inia100_pci_driver);
-}
-
-static void __exit inia100_exit(void)
-{
-        pci_unregister_driver(&inia100_pci_driver);
-}
+module_pci_driver(inia100_pci_driver);
 
 MODULE_DESCRIPTION("Initio A100U2W SCSI driver");
 MODULE_AUTHOR("Initio Corporation");
 MODULE_LICENSE("Dual BSD/GPL");
-
-module_init(inia100_init);
-module_exit(inia100_exit);
diff --git a/drivers/scsi/am53c974.c b/drivers/scsi/am53c974.c
index beea30e5a34a..d81ca66e24d6 100644
--- a/drivers/scsi/am53c974.c
+++ b/drivers/scsi/am53c974.c
@@ -556,15 +556,7 @@ static struct pci_driver am53c974_driver = {
         .remove         = pci_esp_remove_one,
 };
 
-static int __init am53c974_module_init(void)
-{
-        return pci_register_driver(&am53c974_driver);
-}
-
-static void __exit am53c974_module_exit(void)
-{
-        pci_unregister_driver(&am53c974_driver);
-}
+module_pci_driver(am53c974_driver);
 
 MODULE_DESCRIPTION("AM53C974 SCSI driver");
 MODULE_AUTHOR("Hannes Reinecke <hare@suse.de>");
@@ -577,6 +569,3 @@ MODULE_PARM_DESC(am53c974_debug, "Enable debugging");
 
 module_param(am53c974_fenab, bool, 0444);
 MODULE_PARM_DESC(am53c974_fenab, "Enable 24-bit DMA transfer sizes");
-
-module_init(am53c974_module_init);
-module_exit(am53c974_module_exit);
diff --git a/drivers/scsi/cxlflash/Kconfig b/drivers/scsi/cxlflash/Kconfig
index a011c5dbf214..f1b17e3efb3f 100644
--- a/drivers/scsi/cxlflash/Kconfig
+++ b/drivers/scsi/cxlflash/Kconfig
@@ -4,7 +4,7 @@
 
 config CXLFLASH
         tristate "Support for IBM CAPI Flash"
-        depends on PCI && SCSI && CXL && EEH
+        depends on PCI && SCSI && (CXL || OCXL) && EEH
         select IRQ_POLL
         default m
         help
diff --git a/drivers/scsi/cxlflash/Makefile b/drivers/scsi/cxlflash/Makefile
index 7ec3f6b55dde..283377d8f6fb 100644
--- a/drivers/scsi/cxlflash/Makefile
+++ b/drivers/scsi/cxlflash/Makefile
@@ -1,2 +1,4 @@
 obj-$(CONFIG_CXLFLASH) += cxlflash.o
-cxlflash-y += main.o superpipe.o lunmgt.o vlun.o cxl_hw.o
+cxlflash-y += main.o superpipe.o lunmgt.o vlun.o
+cxlflash-$(CONFIG_CXL) += cxl_hw.o
+cxlflash-$(CONFIG_OCXL) += ocxl_hw.o
diff --git a/drivers/scsi/cxlflash/backend.h b/drivers/scsi/cxlflash/backend.h
index 339e42b03c49..55638d19c2fd 100644
--- a/drivers/scsi/cxlflash/backend.h
+++ b/drivers/scsi/cxlflash/backend.h
@@ -12,30 +12,41 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#ifndef _CXLFLASH_BACKEND_H
+#define _CXLFLASH_BACKEND_H
+
 extern const struct cxlflash_backend_ops cxlflash_cxl_ops;
+extern const struct cxlflash_backend_ops cxlflash_ocxl_ops;
 
 struct cxlflash_backend_ops {
         struct module *module;
-        void __iomem * (*psa_map)(void *);
-        void (*psa_unmap)(void __iomem *);
-        int (*process_element)(void *);
-        int (*map_afu_irq)(void *, int, irq_handler_t, void *, char *);
-        void (*unmap_afu_irq)(void *, int, void *);
-        int (*start_context)(void *);
-        int (*stop_context)(void *);
-        int (*afu_reset)(void *);
-        void (*set_master)(void *);
-        void * (*get_context)(struct pci_dev *, void *);
-        void * (*dev_context_init)(struct pci_dev *, void *);
-        int (*release_context)(void *);
-        void (*perst_reloads_same_image)(void *, bool);
-        ssize_t (*read_adapter_vpd)(struct pci_dev *, void *, size_t);
-        int (*allocate_afu_irqs)(void *, int);
-        void (*free_afu_irqs)(void *);
-        void * (*create_afu)(struct pci_dev *);
-        struct file * (*get_fd)(void *, struct file_operations *, int *);
-        void * (*fops_get_context)(struct file *);
-        int (*start_work)(void *, u64);
-        int (*fd_mmap)(struct file *, struct vm_area_struct *);
-        int (*fd_release)(struct inode *, struct file *);
+        void __iomem * (*psa_map)(void *ctx_cookie);
+        void (*psa_unmap)(void __iomem *addr);
+        int (*process_element)(void *ctx_cookie);
+        int (*map_afu_irq)(void *ctx_cookie, int num, irq_handler_t handler,
+                           void *cookie, char *name);
+        void (*unmap_afu_irq)(void *ctx_cookie, int num, void *cookie);
+        u64 (*get_irq_objhndl)(void *ctx_cookie, int irq);
+        int (*start_context)(void *ctx_cookie);
+        int (*stop_context)(void *ctx_cookie);
+        int (*afu_reset)(void *ctx_cookie);
+        void (*set_master)(void *ctx_cookie);
+        void * (*get_context)(struct pci_dev *dev, void *afu_cookie);
+        void * (*dev_context_init)(struct pci_dev *dev, void *afu_cookie);
+        int (*release_context)(void *ctx_cookie);
+        void (*perst_reloads_same_image)(void *afu_cookie, bool image);
+        ssize_t (*read_adapter_vpd)(struct pci_dev *dev, void *buf,
+                                    size_t count);
+        int (*allocate_afu_irqs)(void *ctx_cookie, int num);
+        void (*free_afu_irqs)(void *ctx_cookie);
+        void * (*create_afu)(struct pci_dev *dev);
+        void (*destroy_afu)(void *afu_cookie);
+        struct file * (*get_fd)(void *ctx_cookie, struct file_operations *fops,
+                                int *fd);
+        void * (*fops_get_context)(struct file *file);
+        int (*start_work)(void *ctx_cookie, u64 irqs);
+        int (*fd_mmap)(struct file *file, struct vm_area_struct *vm);
+        int (*fd_release)(struct inode *inode, struct file *file);
 };
+
+#endif /* _CXLFLASH_BACKEND_H */
diff --git a/drivers/scsi/cxlflash/common.h b/drivers/scsi/cxlflash/common.h
index 102fd26ca886..8908a20065c8 100644
--- a/drivers/scsi/cxlflash/common.h
+++ b/drivers/scsi/cxlflash/common.h
@@ -211,6 +211,7 @@ struct hwq {
         struct sisl_ctrl_map __iomem *ctrl_map;         /* MC control map */
         ctx_hndl_t ctx_hndl;    /* master's context handle */
         u32 index;              /* Index of this hwq */
+        int num_irqs;           /* Number of interrupts requested for context */
         struct list_head pending_cmds;  /* Commands pending completion */
 
         atomic_t hsq_credits;
@@ -223,6 +224,7 @@ struct hwq {
         u64 *hrrq_end;
         u64 *hrrq_curr;
         bool toggle;
+        bool hrrq_online;
 
         s64 room;
 
@@ -231,13 +233,14 @@ struct hwq {
 
 struct afu {
         struct hwq hwqs[CXLFLASH_MAX_HWQS];
-        int (*send_cmd)(struct afu *, struct afu_cmd *);
-        int (*context_reset)(struct hwq *);
+        int (*send_cmd)(struct afu *afu, struct afu_cmd *cmd);
+        int (*context_reset)(struct hwq *hwq);
 
         /* AFU HW */
         struct cxlflash_afu_map __iomem *afu_map;       /* entire MMIO map */
 
         atomic_t cmds_active;   /* Number of currently active AFU commands */
+        struct mutex sync_active;       /* Mutex to serialize AFU commands */
         u64 hb;
         u32 internal_lun;       /* User-desired LUN mode for this AFU */
 
@@ -272,6 +275,11 @@ static inline bool afu_has_cap(struct afu *afu, u64 cap)
         return afu_cap & cap;
 }
 
+static inline bool afu_is_ocxl_lisn(struct afu *afu)
+{
+        return afu_has_cap(afu, SISL_INTVER_CAP_OCXL_LISN);
+}
+
 static inline bool afu_is_afu_debug(struct afu *afu)
 {
         return afu_has_cap(afu, SISL_INTVER_CAP_AFU_DEBUG);
diff --git a/drivers/scsi/cxlflash/cxl_hw.c b/drivers/scsi/cxlflash/cxl_hw.c
index db1cadad5c5d..b42da88386bd 100644
--- a/drivers/scsi/cxlflash/cxl_hw.c
+++ b/drivers/scsi/cxlflash/cxl_hw.c
@@ -49,6 +49,12 @@ static void cxlflash_unmap_afu_irq(void *ctx_cookie, int num, void *cookie)
         cxl_unmap_afu_irq(ctx_cookie, num, cookie);
 }
 
+static u64 cxlflash_get_irq_objhndl(void *ctx_cookie, int irq)
+{
+        /* Dummy fop for cxl */
+        return 0;
+}
+
 static int cxlflash_start_context(void *ctx_cookie)
 {
         return cxl_start_context(ctx_cookie, 0, NULL);
@@ -110,6 +116,11 @@ static void *cxlflash_create_afu(struct pci_dev *dev)
         return cxl_pci_to_afu(dev);
 }
 
+static void cxlflash_destroy_afu(void *afu)
+{
+        /* Dummy fop for cxl */
+}
+
 static struct file *cxlflash_get_fd(void *ctx_cookie,
                                     struct file_operations *fops, int *fd)
 {
@@ -148,6 +159,7 @@ const struct cxlflash_backend_ops cxlflash_cxl_ops = {
         .process_element        = cxlflash_process_element,
         .map_afu_irq            = cxlflash_map_afu_irq,
         .unmap_afu_irq          = cxlflash_unmap_afu_irq,
+        .get_irq_objhndl        = cxlflash_get_irq_objhndl,
         .start_context          = cxlflash_start_context,
         .stop_context           = cxlflash_stop_context,
         .afu_reset              = cxlflash_afu_reset,
@@ -160,6 +172,7 @@ const struct cxlflash_backend_ops cxlflash_cxl_ops = {
         .allocate_afu_irqs      = cxlflash_allocate_afu_irqs,
         .free_afu_irqs          = cxlflash_free_afu_irqs,
         .create_afu             = cxlflash_create_afu,
+        .destroy_afu            = cxlflash_destroy_afu,
         .get_fd                 = cxlflash_get_fd,
         .fops_get_context       = cxlflash_fops_get_context,
         .start_work             = cxlflash_start_work,
diff --git a/drivers/scsi/cxlflash/lunmgt.c b/drivers/scsi/cxlflash/lunmgt.c
index 4d232e271af6..edea1255fdab 100644
--- a/drivers/scsi/cxlflash/lunmgt.c
+++ b/drivers/scsi/cxlflash/lunmgt.c
@@ -12,9 +12,11 @@
  * 2 of the License, or (at your option) any later version.
  */
 
-#include <misc/cxl.h>
 #include <asm/unaligned.h>
 
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+
 #include <scsi/scsi_host.h>
 #include <uapi/scsi/cxlflash_ioctl.h>
 
diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c
index d8fe7ab870b8..6637116529aa 100644
--- a/drivers/scsi/cxlflash/main.c
+++ b/drivers/scsi/cxlflash/main.c
@@ -19,8 +19,6 @@
 
 #include <asm/unaligned.h>
 
-#include <misc/cxl.h>
-
 #include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_host.h>
 #include <uapi/scsi/cxlflash_ioctl.h>
@@ -339,8 +337,8 @@ static int send_cmd_ioarrin(struct afu *afu, struct afu_cmd *cmd)
         writeq_be((u64)&cmd->rcb, &hwq->host_map->ioarrin);
 out:
         spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
-        dev_dbg(dev, "%s: cmd=%p len=%u ea=%016llx rc=%d\n", __func__,
-                cmd, cmd->rcb.data_len, cmd->rcb.data_ea, rc);
+        dev_dbg_ratelimited(dev, "%s: cmd=%p len=%u ea=%016llx rc=%d\n",
+                __func__, cmd, cmd->rcb.data_len, cmd->rcb.data_ea, rc);
         return rc;
 }
 
@@ -473,6 +471,7 @@ static int send_tmf(struct cxlflash_cfg *cfg, struct scsi_device *sdev,
         struct afu_cmd *cmd = NULL;
         struct device *dev = &cfg->dev->dev;
         struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
+        bool needs_deletion = false;
         char *buf = NULL;
         ulong lock_flags;
         int rc = 0;
@@ -527,6 +526,7 @@ static int send_tmf(struct cxlflash_cfg *cfg, struct scsi_device *sdev,
         if (!to) {
                 dev_err(dev, "%s: TMF timed out\n", __func__);
                 rc = -ETIMEDOUT;
+                needs_deletion = true;
         } else if (cmd->cmd_aborted) {
                 dev_err(dev, "%s: TMF aborted\n", __func__);
                 rc = -EAGAIN;
@@ -537,6 +537,12 @@ static int send_tmf(struct cxlflash_cfg *cfg, struct scsi_device *sdev,
         }
         cfg->tmf_active = false;
         spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags);
+
+        if (needs_deletion) {
+                spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
+                list_del(&cmd->list);
+                spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
+        }
 out:
         kfree(buf);
         return rc;
@@ -608,6 +614,7 @@ static int cxlflash_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scp)
                 rc = 0;
                 goto out;
         default:
+                atomic_inc(&afu->cmds_active);
                 break;
         }
 
@@ -633,6 +640,7 @@ static int cxlflash_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scp)
         memcpy(cmd->rcb.cdb, scp->cmnd, sizeof(cmd->rcb.cdb));
 
         rc = afu->send_cmd(afu, cmd);
+        atomic_dec(&afu->cmds_active);
 out:
         return rc;
 }
@@ -793,6 +801,10 @@ static void term_mc(struct cxlflash_cfg *cfg, u32 index)
                 WARN_ON(cfg->ops->release_context(hwq->ctx_cookie));
         hwq->ctx_cookie = NULL;
 
+        spin_lock_irqsave(&hwq->hrrq_slock, lock_flags);
+        hwq->hrrq_online = false;
+        spin_unlock_irqrestore(&hwq->hrrq_slock, lock_flags);
+
         spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
         flush_pending_cmds(hwq);
         spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
@@ -946,9 +958,9 @@ static void cxlflash_remove(struct pci_dev *pdev)
                 return;
         }
 
-        /* If a Task Management Function is active, wait for it to complete
-         * before continuing with remove.
-         */
+        /* Yield to running recovery threads before continuing with remove */
+        wait_event(cfg->reset_waitq, cfg->state != STATE_RESET &&
+                                     cfg->state != STATE_PROBING);
         spin_lock_irqsave(&cfg->tmf_slock, lock_flags);
         if (cfg->tmf_active)
                 wait_event_interruptible_lock_irq(cfg->tmf_waitq,
@@ -971,6 +983,7 @@ static void cxlflash_remove(struct pci_dev *pdev)
         case INIT_STATE_AFU:
                 term_afu(cfg);
         case INIT_STATE_PCI:
+                cfg->ops->destroy_afu(cfg->afu_cookie);
                 pci_disable_device(pdev);
         case INIT_STATE_NONE:
                 free_mem(cfg);
@@ -1303,7 +1316,10 @@ static void afu_err_intr_init(struct afu *afu)
         for (i = 0; i < afu->num_hwqs; i++) {
                 hwq = get_hwq(afu, i);
 
-                writeq_be(SISL_MSI_SYNC_ERROR, &hwq->host_map->ctx_ctrl);
+                reg = readq_be(&hwq->host_map->ctx_ctrl);
+                WARN_ON((reg & SISL_CTX_CTRL_LISN_MASK) != 0);
+                reg |= SISL_MSI_SYNC_ERROR;
+                writeq_be(reg, &hwq->host_map->ctx_ctrl);
                 writeq_be(SISL_ISTATUS_MASK, &hwq->host_map->intr_mask);
         }
 }
@@ -1463,6 +1479,12 @@ static irqreturn_t cxlflash_rrq_irq(int irq, void *data)
 
         spin_lock_irqsave(&hwq->hrrq_slock, hrrq_flags);
 
+        /* Silently drop spurious interrupts when queue is not online */
+        if (!hwq->hrrq_online) {
+                spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags);
+                return IRQ_HANDLED;
+        }
+
         if (afu_is_irqpoll_enabled(afu)) {
                 irq_poll_sched(&hwq->irqpoll);
                 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags);
@@ -1752,6 +1774,8 @@ static int init_global(struct cxlflash_cfg *cfg)
         u64 wwpn[MAX_FC_PORTS]; /* wwpn of AFU ports */
         int i = 0, num_ports = 0;
         int rc = 0;
+        int j;
+        void *ctx;
         u64 reg;
 
         rc = read_vpd(cfg, &wwpn[0]);
@@ -1767,6 +1791,7 @@ static int init_global(struct cxlflash_cfg *cfg)
 
                 writeq_be((u64) hwq->hrrq_start, &hmap->rrq_start);
                 writeq_be((u64) hwq->hrrq_end, &hmap->rrq_end);
+                hwq->hrrq_online = true;
 
                 if (afu_is_sq_cmd_mode(afu)) {
                         writeq_be((u64)hwq->hsq_start, &hmap->sq_start);
@@ -1812,6 +1837,25 @@ static int init_global(struct cxlflash_cfg *cfg)
                 msleep(100);
         }
 
+        if (afu_is_ocxl_lisn(afu)) {
+                /* Set up the LISN effective address for each master */
+                for (i = 0; i < afu->num_hwqs; i++) {
+                        hwq = get_hwq(afu, i);
+                        ctx = hwq->ctx_cookie;
+
+                        for (j = 0; j < hwq->num_irqs; j++) {
+                                reg = cfg->ops->get_irq_objhndl(ctx, j);
+                                writeq_be(reg, &hwq->ctrl_map->lisn_ea[j]);
+                        }
+
+                        reg = hwq->ctx_hndl;
+                        writeq_be(SISL_LISN_PASID(reg, reg),
+                                  &hwq->ctrl_map->lisn_pasid[0]);
+                        writeq_be(SISL_LISN_PASID(0UL, reg),
+                                  &hwq->ctrl_map->lisn_pasid[1]);
+                }
+        }
+
         /* Set up master's own CTX_CAP to allow real mode, host translation */
         /* tables, afu cmds and read/write GSCSI cmds. */
         /* First, unlock ctx_cap write by reading mbox */
@@ -1911,7 +1955,7 @@ static enum undo_level init_intr(struct cxlflash_cfg *cfg,
         int rc = 0;
         enum undo_level level = UNDO_NOOP;
         bool is_primary_hwq = (hwq->index == PRIMARY_HWQ);
-        int num_irqs = is_primary_hwq ? 3 : 2;
+        int num_irqs = hwq->num_irqs;
 
         rc = cfg->ops->allocate_afu_irqs(ctx, num_irqs);
         if (unlikely(rc)) {
@@ -1965,16 +2009,20 @@ static int init_mc(struct cxlflash_cfg *cfg, u32 index)
         struct device *dev = &cfg->dev->dev;
         struct hwq *hwq = get_hwq(cfg->afu, index);
         int rc = 0;
+        int num_irqs;
         enum undo_level level;
 
         hwq->afu = cfg->afu;
         hwq->index = index;
         INIT_LIST_HEAD(&hwq->pending_cmds);
 
-        if (index == PRIMARY_HWQ)
+        if (index == PRIMARY_HWQ) {
                 ctx = cfg->ops->get_context(cfg->dev, cfg->afu_cookie);
-        else
+                num_irqs = 3;
+        } else {
                 ctx = cfg->ops->dev_context_init(cfg->dev, cfg->afu_cookie);
+                num_irqs = 2;
+        }
         if (IS_ERR_OR_NULL(ctx)) {
                 rc = -ENOMEM;
                 goto err1;
@@ -1982,6 +2030,7 @@ static int init_mc(struct cxlflash_cfg *cfg, u32 index)
 
         WARN_ON(hwq->ctx_cookie);
         hwq->ctx_cookie = ctx;
+        hwq->num_irqs = num_irqs;
 
         /* Set it up as a master with the CXL */
         cfg->ops->set_master(ctx);
@@ -2075,6 +2124,7 @@ static int init_afu(struct cxlflash_cfg *cfg)
 
         cfg->ops->perst_reloads_same_image(cfg->afu_cookie, true);
 
+        mutex_init(&afu->sync_active);
         afu->num_hwqs = afu->desired_hwqs;
         for (i = 0; i < afu->num_hwqs; i++) {
                 rc = init_mc(cfg, i);
@@ -2254,10 +2304,10 @@ static int send_afu_cmd(struct afu *afu, struct sisl_ioarcb *rcb)
         struct device *dev = &cfg->dev->dev;
         struct afu_cmd *cmd = NULL;
         struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
+        ulong lock_flags;
         char *buf = NULL;
         int rc = 0;
         int nretry = 0;
-        static DEFINE_MUTEX(sync_active);
 
         if (cfg->state != STATE_NORMAL) {
                 dev_dbg(dev, "%s: Sync not required state=%u\n",
@@ -2265,7 +2315,7 @@ static int send_afu_cmd(struct afu *afu, struct sisl_ioarcb *rcb)
                 return 0;
         }
 
-        mutex_lock(&sync_active);
+        mutex_lock(&afu->sync_active);
         atomic_inc(&afu->cmds_active);
         buf = kmalloc(sizeof(*cmd) + __alignof__(*cmd) - 1, GFP_KERNEL);
         if (unlikely(!buf)) {
@@ -2299,6 +2349,11 @@ retry:
         case -ETIMEDOUT:
                 rc = afu->context_reset(hwq);
                 if (rc) {
+                        /* Delete the command from pending_cmds list */
+                        spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
+                        list_del(&cmd->list);
+                        spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
+
                         cxlflash_schedule_async_reset(cfg);
                         break;
                 }
@@ -2315,7 +2370,7 @@ retry:
                 *rcb->ioasa = cmd->sa;
 out:
         atomic_dec(&afu->cmds_active);
-        mutex_unlock(&sync_active);
+        mutex_unlock(&afu->sync_active);
         kfree(buf);
         dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
         return rc;
@@ -3138,7 +3193,8 @@ static struct dev_dependent_vals dev_corsa_vals = { CXLFLASH_MAX_SECTORS,
 static struct dev_dependent_vals dev_flash_gt_vals = { CXLFLASH_MAX_SECTORS,
                                         CXLFLASH_NOTIFY_SHUTDOWN };
 static struct dev_dependent_vals dev_briard_vals = { CXLFLASH_MAX_SECTORS,
-                                        CXLFLASH_NOTIFY_SHUTDOWN };
+                                        (CXLFLASH_NOTIFY_SHUTDOWN |
+                                        CXLFLASH_OCXL_DEV) };
 
 /*
  * PCI device binding table
@@ -3649,8 +3705,9 @@ static int cxlflash_probe(struct pci_dev *pdev,
 
         cfg->init_state = INIT_STATE_NONE;
         cfg->dev = pdev;
-        cfg->ops = &cxlflash_cxl_ops;
         cfg->cxl_fops = cxlflash_cxl_fops;
+        cfg->ops = cxlflash_assign_ops(ddv);
+        WARN_ON_ONCE(!cfg->ops);
 
         /*
          * Promoted LUNs move to the top of the LUN table. The rest stay on
@@ -3681,8 +3738,6 @@ static int cxlflash_probe(struct pci_dev *pdev,
 
         pci_set_drvdata(pdev, cfg);
 
-        cfg->afu_cookie = cfg->ops->create_afu(pdev);
-
         rc = init_pci(cfg);
         if (rc) {
                 dev_err(dev, "%s: init_pci failed rc=%d\n", __func__, rc);
@@ -3690,6 +3745,12 @@ static int cxlflash_probe(struct pci_dev *pdev,
         }
         cfg->init_state = INIT_STATE_PCI;
 
+        cfg->afu_cookie = cfg->ops->create_afu(pdev);
+        if (unlikely(!cfg->afu_cookie)) {
+                dev_err(dev, "%s: create_afu failed\n", __func__);
+                goto out_remove;
+        }
+
         rc = init_afu(cfg);
         if (rc && !wq_has_sleeper(&cfg->reset_waitq)) {
                 dev_err(dev, "%s: init_afu failed rc=%d\n", __func__, rc);
diff --git a/drivers/scsi/cxlflash/main.h b/drivers/scsi/cxlflash/main.h
index ba0108a7a9c2..2a3977823812 100644
--- a/drivers/scsi/cxlflash/main.h
+++ b/drivers/scsi/cxlflash/main.h
@@ -20,6 +20,8 @@
 #include <scsi/scsi.h>
 #include <scsi/scsi_device.h>
 
+#include "backend.h"
+
 #define CXLFLASH_NAME           "cxlflash"
 #define CXLFLASH_ADAPTER_NAME   "IBM POWER CXL Flash Adapter"
 #define CXLFLASH_MAX_ADAPTERS   32
@@ -97,8 +99,27 @@ struct dev_dependent_vals {
         u64 flags;
 #define CXLFLASH_NOTIFY_SHUTDOWN        0x0000000000000001ULL
 #define CXLFLASH_WWPN_VPD_REQUIRED      0x0000000000000002ULL
+#define CXLFLASH_OCXL_DEV               0x0000000000000004ULL
 };
 
+static inline const struct cxlflash_backend_ops *
+cxlflash_assign_ops(struct dev_dependent_vals *ddv)
+{
+        const struct cxlflash_backend_ops *ops = NULL;
+
+#ifdef CONFIG_OCXL
+        if (ddv->flags & CXLFLASH_OCXL_DEV)
+                ops = &cxlflash_ocxl_ops;
+#endif
+
+#ifdef CONFIG_CXL
+        if (!(ddv->flags & CXLFLASH_OCXL_DEV))
+                ops = &cxlflash_cxl_ops;
+#endif
+
+        return ops;
+}
+
 struct asyc_intr_info {
         u64 status;
         char *desc;
diff --git a/drivers/scsi/cxlflash/ocxl_hw.c b/drivers/scsi/cxlflash/ocxl_hw.c
new file mode 100644
index 000000000000..0a95b5f25380
--- /dev/null
+++ b/drivers/scsi/cxlflash/ocxl_hw.c
@@ -0,0 +1,1436 @@
+/*
+ * CXL Flash Device Driver
+ *
+ * Written by: Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
+ *             Uma Krishnan <ukrishn@linux.vnet.ibm.com>, IBM Corporation
+ *
+ * Copyright (C) 2018 IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include <linux/file.h>
+#include <linux/idr.h>
+#include <linux/module.h>
+#include <linux/mount.h>
+#include <linux/poll.h>
+#include <linux/sched/signal.h>
+
+#include <misc/ocxl.h>
+
+#include <uapi/misc/cxl.h>
+
+#include "backend.h"
+#include "ocxl_hw.h"
+
+/*
+ * Pseudo-filesystem to allocate inodes.
+ */
+
+#define OCXLFLASH_FS_MAGIC      0x1697698f
+
+static int ocxlflash_fs_cnt;
+static struct vfsmount *ocxlflash_vfs_mount;
+
+static const struct dentry_operations ocxlflash_fs_dops = {
+        .d_dname        = simple_dname,
+};
+
+/*
+ * ocxlflash_fs_mount() - mount the pseudo-filesystem
+ * @fs_type:    File system type.
+ * @flags:      Flags for the filesystem.
+ * @dev_name:   Device name associated with the filesystem.
+ * @data:       Data pointer.
+ *
+ * Return: pointer to the directory entry structure
+ */
+static struct dentry *ocxlflash_fs_mount(struct file_system_type *fs_type,
+                                         int flags, const char *dev_name,
+                                         void *data)
+{
+        return mount_pseudo(fs_type, "ocxlflash:", NULL, &ocxlflash_fs_dops,
+                            OCXLFLASH_FS_MAGIC);
+}
+
+static struct file_system_type ocxlflash_fs_type = {
+        .name           = "ocxlflash",
+        .owner          = THIS_MODULE,
+        .mount          = ocxlflash_fs_mount,
+        .kill_sb        = kill_anon_super,
+};
+
+/*
+ * ocxlflash_release_mapping() - release the memory mapping
+ * @ctx:        Context whose mapping is to be released.
+ */
+static void ocxlflash_release_mapping(struct ocxlflash_context *ctx)
+{
+        if (ctx->mapping)
+                simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt);
+        ctx->mapping = NULL;
+}
+
+/*
+ * ocxlflash_getfile() - allocate pseudo filesystem, inode, and the file
+ * @dev:        Generic device of the host.
+ * @name:       Name of the pseudo filesystem.
+ * @fops:       File operations.
+ * @priv:       Private data.
+ * @flags:      Flags for the file.
+ *
+ * Return: pointer to the file on success, ERR_PTR on failure
+ */
+static struct file *ocxlflash_getfile(struct device *dev, const char *name,
+                                      const struct file_operations *fops,
+                                      void *priv, int flags)
+{
+        struct qstr this;
+        struct path path;
+        struct file *file;
+        struct inode *inode = NULL;
+        int rc;
+
+        if (fops->owner && !try_module_get(fops->owner)) {
+                dev_err(dev, "%s: Owner does not exist\n", __func__);
+                rc = -ENOENT;
+                goto err1;
+        }
+
+        rc = simple_pin_fs(&ocxlflash_fs_type, &ocxlflash_vfs_mount,
+                           &ocxlflash_fs_cnt);
+        if (unlikely(rc < 0)) {
+                dev_err(dev, "%s: Cannot mount ocxlflash pseudofs rc=%d\n",
+                        __func__, rc);
+                goto err2;
+        }
+
+        inode = alloc_anon_inode(ocxlflash_vfs_mount->mnt_sb);
+        if (IS_ERR(inode)) {
+                rc = PTR_ERR(inode);
+                dev_err(dev, "%s: alloc_anon_inode failed rc=%d\n",
+                        __func__, rc);
+                goto err3;
+        }
+
+        this.name = name;
+        this.len = strlen(name);
+        this.hash = 0;
+        path.dentry = d_alloc_pseudo(ocxlflash_vfs_mount->mnt_sb, &this);
+        if (!path.dentry) {
+                dev_err(dev, "%s: d_alloc_pseudo failed\n", __func__);
+                rc = -ENOMEM;
+                goto err4;
+        }
+
+        path.mnt = mntget(ocxlflash_vfs_mount);
+        d_instantiate(path.dentry, inode);
+
+        file = alloc_file(&path, OPEN_FMODE(flags), fops);
+        if (IS_ERR(file)) {
+                rc = PTR_ERR(file);
+                dev_err(dev, "%s: alloc_file failed rc=%d\n",
+                        __func__, rc);
+                goto err5;
+        }
+
+        file->f_flags = flags & (O_ACCMODE | O_NONBLOCK);
+        file->private_data = priv;
+out:
+        return file;
+err5:
+        path_put(&path);
+err4:
+        iput(inode);
+err3:
+        simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt);
+err2:
+        module_put(fops->owner);
+err1:
+        file = ERR_PTR(rc);
+        goto out;
+}
+
+/**
+ * ocxlflash_psa_map() - map the process specific MMIO space
+ * @ctx_cookie: Adapter context for which the mapping needs to be done.
+ *
+ * Return: MMIO pointer of the mapped region
+ */
+static void __iomem *ocxlflash_psa_map(void *ctx_cookie)
+{
+        struct ocxlflash_context *ctx = ctx_cookie;
+        struct device *dev = ctx->hw_afu->dev;
+
+        mutex_lock(&ctx->state_mutex);
+        if (ctx->state != STARTED) {
+                dev_err(dev, "%s: Context not started, state=%d\n", __func__,
+                        ctx->state);
+                mutex_unlock(&ctx->state_mutex);
+                return NULL;
+        }
+        mutex_unlock(&ctx->state_mutex);
+
+        return ioremap(ctx->psn_phys, ctx->psn_size);
+}
+
+/**
+ * ocxlflash_psa_unmap() - unmap the process specific MMIO space
+ * @addr:       MMIO pointer to unmap.
+ */
+static void ocxlflash_psa_unmap(void __iomem *addr)
+{
+        iounmap(addr);
+}
+
+/**
+ * ocxlflash_process_element() - get process element of the adapter context
+ * @ctx_cookie: Adapter context associated with the process element.
+ *
+ * Return: process element of the adapter context
+ */
+static int ocxlflash_process_element(void *ctx_cookie)
+{
+        struct ocxlflash_context *ctx = ctx_cookie;
+
+        return ctx->pe;
+}
+
+/**
+ * afu_map_irq() - map the interrupt of the adapter context
+ * @flags:      Flags.
+ * @ctx:        Adapter context.
+ * @num:        Per-context AFU interrupt number.
+ * @handler:    Interrupt handler to register.
+ * @cookie:     Interrupt handler private data.
+ * @name:       Name of the interrupt.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num,
+                       irq_handler_t handler, void *cookie, char *name)
+{
+        struct ocxl_hw_afu *afu = ctx->hw_afu;
+        struct device *dev = afu->dev;
+        struct ocxlflash_irqs *irq;
+        void __iomem *vtrig;
+        u32 virq;
+        int rc = 0;
+
+        if (num < 0 || num >= ctx->num_irqs) {
+                dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num);
+                rc = -ENOENT;
+                goto out;
+        }
+
+        irq = &ctx->irqs[num];
+        virq = irq_create_mapping(NULL, irq->hwirq);
+        if (unlikely(!virq)) {
+                dev_err(dev, "%s: irq_create_mapping failed\n", __func__);
+                rc = -ENOMEM;
+                goto out;
+        }
+
+        rc = request_irq(virq, handler, 0, name, cookie);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: request_irq failed rc=%d\n", __func__, rc);
+                goto err1;
+        }
+
+        vtrig = ioremap(irq->ptrig, PAGE_SIZE);
+        if (unlikely(!vtrig)) {
+                dev_err(dev, "%s: Trigger page mapping failed\n", __func__);
+                rc = -ENOMEM;
+                goto err2;
+        }
+
+        irq->virq = virq;
+        irq->vtrig = vtrig;
+out:
+        return rc;
+err2:
+        free_irq(virq, cookie);
+err1:
+        irq_dispose_mapping(virq);
+        goto out;
+}
+
+/**
+ * ocxlflash_map_afu_irq() - map the interrupt of the adapter context
+ * @ctx_cookie: Adapter context.
+ * @num:        Per-context AFU interrupt number.
+ * @handler:    Interrupt handler to register.
+ * @cookie:     Interrupt handler private data.
+ * @name:       Name of the interrupt.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_map_afu_irq(void *ctx_cookie, int num,
+                                 irq_handler_t handler, void *cookie,
+                                 char *name)
+{
+        return afu_map_irq(0, ctx_cookie, num, handler, cookie, name);
+}
+
+/**
+ * afu_unmap_irq() - unmap the interrupt
+ * @flags:      Flags.
+ * @ctx:        Adapter context.
+ * @num:        Per-context AFU interrupt number.
+ * @cookie:     Interrupt handler private data.
+ */
+static void afu_unmap_irq(u64 flags, struct ocxlflash_context *ctx, int num,
+                          void *cookie)
+{
+        struct ocxl_hw_afu *afu = ctx->hw_afu;
+        struct device *dev = afu->dev;
+        struct ocxlflash_irqs *irq;
+
+        if (num < 0 || num >= ctx->num_irqs) {
+                dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num);
+                return;
+        }
+
+        irq = &ctx->irqs[num];
+        if (irq->vtrig)
+                iounmap(irq->vtrig);
+
+        if (irq_find_mapping(NULL, irq->hwirq)) {
+                free_irq(irq->virq, cookie);
+                irq_dispose_mapping(irq->virq);
+        }
+
+        memset(irq, 0, sizeof(*irq));
+}
+
+/**
+ * ocxlflash_unmap_afu_irq() - unmap the interrupt
+ * @ctx_cookie: Adapter context.
+ * @num:        Per-context AFU interrupt number.
+ * @cookie:     Interrupt handler private data.
+ */
+static void ocxlflash_unmap_afu_irq(void *ctx_cookie, int num, void *cookie)
+{
+        return afu_unmap_irq(0, ctx_cookie, num, cookie);
+}
+
+/**
+ * ocxlflash_get_irq_objhndl() - get the object handle for an interrupt
+ * @ctx_cookie: Context associated with the interrupt.
+ * @irq:        Interrupt number.
+ *
+ * Return: effective address of the mapped region
+ */
+static u64 ocxlflash_get_irq_objhndl(void *ctx_cookie, int irq)
+{
+        struct ocxlflash_context *ctx = ctx_cookie;
+
+        if (irq < 0 || irq >= ctx->num_irqs)
+                return 0;
+
+        return (__force u64)ctx->irqs[irq].vtrig;
+}
+
+/**
+ * ocxlflash_xsl_fault() - callback when translation error is triggered
+ * @data:       Private data provided at callback registration, the context.
+ * @addr:       Address that triggered the error.
+ * @dsisr:      Value of dsisr register.
+ */
+static void ocxlflash_xsl_fault(void *data, u64 addr, u64 dsisr)
+{
+        struct ocxlflash_context *ctx = data;
+
+        spin_lock(&ctx->slock);
+        ctx->fault_addr = addr;
+        ctx->fault_dsisr = dsisr;
+        ctx->pending_fault = true;
+        spin_unlock(&ctx->slock);
+
+        wake_up_all(&ctx->wq);
+}
+
+/**
+ * start_context() - local routine to start a context
+ * @ctx:        Adapter context to be started.
+ *
+ * Assign the context specific MMIO space, add and enable the PE.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int start_context(struct ocxlflash_context *ctx)
+{
+        struct ocxl_hw_afu *afu = ctx->hw_afu;
+        struct ocxl_afu_config *acfg = &afu->acfg;
+        void *link_token = afu->link_token;
+        struct device *dev = afu->dev;
+        bool master = ctx->master;
+        struct mm_struct *mm;
+        int rc = 0;
+        u32 pid;
+
+        mutex_lock(&ctx->state_mutex);
+        if (ctx->state != OPENED) {
+                dev_err(dev, "%s: Context state invalid, state=%d\n",
+                        __func__, ctx->state);
+                rc = -EINVAL;
+                goto out;
+        }
+
+        if (master) {
+                ctx->psn_size = acfg->global_mmio_size;
+                ctx->psn_phys = afu->gmmio_phys;
+        } else {
+                ctx->psn_size = acfg->pp_mmio_stride;
+                ctx->psn_phys = afu->ppmmio_phys + (ctx->pe * ctx->psn_size);
+        }
+
+        /* pid and mm not set for master contexts */
+        if (master) {
+                pid = 0;
+                mm = NULL;
+        } else {
+                pid = current->mm->context.id;
+                mm = current->mm;
+        }
+
+        rc = ocxl_link_add_pe(link_token, ctx->pe, pid, 0, 0, mm,
+                              ocxlflash_xsl_fault, ctx);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: ocxl_link_add_pe failed rc=%d\n",
+                        __func__, rc);
+                goto out;
+        }
+
+        ctx->state = STARTED;
+out:
+        mutex_unlock(&ctx->state_mutex);
+        return rc;
+}
+
+/**
+ * ocxlflash_start_context() - start a kernel context
+ * @ctx_cookie: Adapter context to be started.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_start_context(void *ctx_cookie)
+{
+        struct ocxlflash_context *ctx = ctx_cookie;
+
+        return start_context(ctx);
+}
+
+/**
+ * ocxlflash_stop_context() - stop a context
+ * @ctx_cookie: Adapter context to be stopped.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_stop_context(void *ctx_cookie)
+{
+        struct ocxlflash_context *ctx = ctx_cookie;
+        struct ocxl_hw_afu *afu = ctx->hw_afu;
+        struct ocxl_afu_config *acfg = &afu->acfg;
+        struct pci_dev *pdev = afu->pdev;
+        struct device *dev = afu->dev;
+        enum ocxlflash_ctx_state state;
+        int rc = 0;
+
+        mutex_lock(&ctx->state_mutex);
+        state = ctx->state;
+        ctx->state = CLOSED;
+        mutex_unlock(&ctx->state_mutex);
+        if (state != STARTED)
+                goto out;
+
+        rc = ocxl_config_terminate_pasid(pdev, acfg->dvsec_afu_control_pos,
+                                         ctx->pe);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: ocxl_config_terminate_pasid failed rc=%d\n",
+                        __func__, rc);
+                /* If EBUSY, PE could be referenced in future by the AFU */
+                if (rc == -EBUSY)
+                        goto out;
+        }
+
+        rc = ocxl_link_remove_pe(afu->link_token, ctx->pe);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: ocxl_link_remove_pe failed rc=%d\n",
+                        __func__, rc);
+                goto out;
+        }
+out:
+        return rc;
+}
+
+/**
+ * ocxlflash_afu_reset() - reset the AFU
+ * @ctx_cookie: Adapter context.
+ */
+static int ocxlflash_afu_reset(void *ctx_cookie)
+{
+        struct ocxlflash_context *ctx = ctx_cookie;
+        struct device *dev = ctx->hw_afu->dev;
+
+        /* Pending implementation from OCXL transport services */
+        dev_err_once(dev, "%s: afu_reset() fop not supported\n", __func__);
+
+        /* Silently return success until it is implemented */
+        return 0;
+}
+
+/**
+ * ocxlflash_set_master() - sets the context as master
+ * @ctx_cookie: Adapter context to set as master.
+ */
+static void ocxlflash_set_master(void *ctx_cookie)
+{
+        struct ocxlflash_context *ctx = ctx_cookie;
+
+        ctx->master = true;
+}
+
+/**
+ * ocxlflash_get_context() - obtains the context associated with the host
+ * @pdev:       PCI device associated with the host.
+ * @afu_cookie: Hardware AFU associated with the host.
+ *
+ * Return: returns the pointer to host adapter context
+ */
+static void *ocxlflash_get_context(struct pci_dev *pdev, void *afu_cookie)
+{
+        struct ocxl_hw_afu *afu = afu_cookie;
+
+        return afu->ocxl_ctx;
+}
+
+/**
+ * ocxlflash_dev_context_init() - allocate and initialize an adapter context
+ * @pdev:       PCI device associated with the host.
+ * @afu_cookie: Hardware AFU associated with the host.
+ *
+ * Return: returns the adapter context on success, ERR_PTR on failure
+ */
+static void *ocxlflash_dev_context_init(struct pci_dev *pdev, void *afu_cookie)
+{
+        struct ocxl_hw_afu *afu = afu_cookie;
+        struct device *dev = afu->dev;
+        struct ocxlflash_context *ctx;
+        int rc;
+
+        ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+        if (unlikely(!ctx)) {
+                dev_err(dev, "%s: Context allocation failed\n", __func__);
+                rc = -ENOMEM;
+                goto err1;
+        }
+
+        idr_preload(GFP_KERNEL);
+        rc = idr_alloc(&afu->idr, ctx, 0, afu->max_pasid, GFP_NOWAIT);
+        idr_preload_end();
+        if (unlikely(rc < 0)) {
+                dev_err(dev, "%s: idr_alloc failed rc=%d\n", __func__, rc);
+                goto err2;
+        }
+
+        spin_lock_init(&ctx->slock);
+        init_waitqueue_head(&ctx->wq);
+        mutex_init(&ctx->state_mutex);
+
+        ctx->state = OPENED;
+        ctx->pe = rc;
+        ctx->master = false;
+        ctx->mapping = NULL;
+        ctx->hw_afu = afu;
+        ctx->irq_bitmap = 0;
+        ctx->pending_irq = false;
+        ctx->pending_fault = false;
+out:
+        return ctx;
+err2:
+        kfree(ctx);
+err1:
+        ctx = ERR_PTR(rc);
+        goto out;
+}
+
+/**
+ * ocxlflash_release_context() - releases an adapter context
+ * @ctx_cookie: Adapter context to be released.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_release_context(void *ctx_cookie)
+{
+        struct ocxlflash_context *ctx = ctx_cookie;
+        struct device *dev;
+        int rc = 0;
+
+        if (!ctx)
+                goto out;
+
+        dev = ctx->hw_afu->dev;
+        mutex_lock(&ctx->state_mutex);
+        if (ctx->state >= STARTED) {
+                dev_err(dev, "%s: Context in use, state=%d\n", __func__,
+                        ctx->state);
+                mutex_unlock(&ctx->state_mutex);
+                rc = -EBUSY;
+                goto out;
+        }
+        mutex_unlock(&ctx->state_mutex);
+
+        idr_remove(&ctx->hw_afu->idr, ctx->pe);
+        ocxlflash_release_mapping(ctx);
+        kfree(ctx);
+out:
+        return rc;
+}
+
+/**
+ * ocxlflash_perst_reloads_same_image() - sets the image reload policy
+ * @afu_cookie: Hardware AFU associated with the host.
+ * @image:      Whether to load the same image on PERST.
+ */
+static void ocxlflash_perst_reloads_same_image(void *afu_cookie, bool image)
+{
+        struct ocxl_hw_afu *afu = afu_cookie;
+
+        afu->perst_same_image = image;
+}
+
+/**
+ * ocxlflash_read_adapter_vpd() - reads the adapter VPD
+ * @pdev:       PCI device associated with the host.
+ * @buf:        Buffer to get the VPD data.
+ * @count:      Size of buffer (maximum bytes that can be read).
+ *
+ * Return: size of VPD on success, -errno on failure
+ */
+static ssize_t ocxlflash_read_adapter_vpd(struct pci_dev *pdev, void *buf,
+                                          size_t count)
+{
+        return pci_read_vpd(pdev, 0, count, buf);
+}
+
+/**
+ * free_afu_irqs() - internal service to free interrupts
+ * @ctx:        Adapter context.
+ */
+static void free_afu_irqs(struct ocxlflash_context *ctx)
+{
+        struct ocxl_hw_afu *afu = ctx->hw_afu;
+        struct device *dev = afu->dev;
+        int i;
+
+        if (!ctx->irqs) {
+                dev_err(dev, "%s: Interrupts not allocated\n", __func__);
+                return;
+        }
+
+        for (i = ctx->num_irqs; i >= 0; i--)
+                ocxl_link_free_irq(afu->link_token, ctx->irqs[i].hwirq);
+
+        kfree(ctx->irqs);
+        ctx->irqs = NULL;
+}
+
+/**
+ * alloc_afu_irqs() - internal service to allocate interrupts
+ * @ctx:        Context associated with the request.
+ * @num:        Number of interrupts requested.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num)
+{
+        struct ocxl_hw_afu *afu = ctx->hw_afu;
+        struct device *dev = afu->dev;
+        struct ocxlflash_irqs *irqs;
+        u64 addr;
+        int rc = 0;
+        int hwirq;
+        int i;
+
+        if (ctx->irqs) {
+                dev_err(dev, "%s: Interrupts already allocated\n", __func__);
+                rc = -EEXIST;
+                goto out;
+        }
+
+        if (num > OCXL_MAX_IRQS) {
+                dev_err(dev, "%s: Too many interrupts num=%d\n", __func__, num);
+                rc = -EINVAL;
+                goto out;
+        }
+
+        irqs = kcalloc(num, sizeof(*irqs), GFP_KERNEL);
+        if (unlikely(!irqs)) {
+                dev_err(dev, "%s: Context irqs allocation failed\n", __func__);
+                rc = -ENOMEM;
+                goto out;
+        }
+
+        for (i = 0; i < num; i++) {
+                rc = ocxl_link_irq_alloc(afu->link_token, &hwirq, &addr);
+                if (unlikely(rc)) {
+                        dev_err(dev, "%s: ocxl_link_irq_alloc failed rc=%d\n",
+                                __func__, rc);
+                        goto err;
+                }
+
+                irqs[i].hwirq = hwirq;
+                irqs[i].ptrig = addr;
+        }
+
+        ctx->irqs = irqs;
+        ctx->num_irqs = num;
+out:
+        return rc;
+err:
+        for (i = i-1; i >= 0; i--)
+                ocxl_link_free_irq(afu->link_token, irqs[i].hwirq);
+        kfree(irqs);
+        goto out;
+}
+
+/**
+ * ocxlflash_allocate_afu_irqs() - allocates the requested number of interrupts
+ * @ctx_cookie: Context associated with the request.
+ * @num:        Number of interrupts requested.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_allocate_afu_irqs(void *ctx_cookie, int num)
+{
+        return alloc_afu_irqs(ctx_cookie, num);
+}
+
+/**
+ * ocxlflash_free_afu_irqs() - frees the interrupts of an adapter context
+ * @ctx_cookie: Adapter context.
+ */
+static void ocxlflash_free_afu_irqs(void *ctx_cookie)
+{
+        free_afu_irqs(ctx_cookie);
+}
+
+/**
+ * ocxlflash_unconfig_afu() - unconfigure the AFU
+ * @afu: AFU associated with the host.
+ */
+static void ocxlflash_unconfig_afu(struct ocxl_hw_afu *afu)
+{
+        if (afu->gmmio_virt) {
+                iounmap(afu->gmmio_virt);
+                afu->gmmio_virt = NULL;
+        }
+}
+
+/**
+ * ocxlflash_destroy_afu() - destroy the AFU structure
+ * @afu_cookie: AFU to be freed.
+ */
+static void ocxlflash_destroy_afu(void *afu_cookie)
+{
+        struct ocxl_hw_afu *afu = afu_cookie;
+        int pos;
+
+        if (!afu)
+                return;
+
+        ocxlflash_release_context(afu->ocxl_ctx);
+        idr_destroy(&afu->idr);
+
+        /* Disable the AFU */
+        pos = afu->acfg.dvsec_afu_control_pos;
+        ocxl_config_set_afu_state(afu->pdev, pos, 0);
+
+        ocxlflash_unconfig_afu(afu);
+        kfree(afu);
+}
+
+/**
+ * ocxlflash_config_fn() - configure the host function
+ * @pdev:       PCI device associated with the host.
+ * @afu:        AFU associated with the host.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_config_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
+{
+        struct ocxl_fn_config *fcfg = &afu->fcfg;
+        struct device *dev = &pdev->dev;
+        u16 base, enabled, supported;
+        int rc = 0;
+
+        /* Read DVSEC config of the function */
+        rc = ocxl_config_read_function(pdev, fcfg);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: ocxl_config_read_function failed rc=%d\n",
+                        __func__, rc);
+                goto out;
+        }
+
+        /* Check if function has AFUs defined, only 1 per function supported */
+        if (fcfg->max_afu_index >= 0) {
+                afu->is_present = true;
+                if (fcfg->max_afu_index != 0)
+                        dev_warn(dev, "%s: Unexpected AFU index value %d\n",
+                                 __func__, fcfg->max_afu_index);
+        }
+
+        rc = ocxl_config_get_actag_info(pdev, &base, &enabled, &supported);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: ocxl_config_get_actag_info failed rc=%d\n",
+                        __func__, rc);
+                goto out;
+        }
+
+        afu->fn_actag_base = base;
+        afu->fn_actag_enabled = enabled;
+
+        ocxl_config_set_actag(pdev, fcfg->dvsec_function_pos, base, enabled);
+        dev_dbg(dev, "%s: Function acTag range base=%u enabled=%u\n",
+                __func__, base, enabled);
+
+        rc = ocxl_link_setup(pdev, 0, &afu->link_token);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: ocxl_link_setup failed rc=%d\n",
+                        __func__, rc);
+                goto out;
+        }
+
+        rc = ocxl_config_set_TL(pdev, fcfg->dvsec_tl_pos);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: ocxl_config_set_TL failed rc=%d\n",
+                        __func__, rc);
+                goto err;
+        }
+out:
+        return rc;
+err:
+        ocxl_link_release(pdev, afu->link_token);
+        goto out;
+}
+
+/**
+ * ocxlflash_unconfig_fn() - unconfigure the host function
+ * @pdev:       PCI device associated with the host.
+ * @afu:        AFU associated with the host.
+ */
+static void ocxlflash_unconfig_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
+{
+        ocxl_link_release(pdev, afu->link_token);
+}
+
+/**
+ * ocxlflash_map_mmio() - map the AFU MMIO space
+ * @afu: AFU associated with the host.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_map_mmio(struct ocxl_hw_afu *afu)
+{
+        struct ocxl_afu_config *acfg = &afu->acfg;
+        struct pci_dev *pdev = afu->pdev;
+        struct device *dev = afu->dev;
+        phys_addr_t gmmio, ppmmio;
+        int rc = 0;
+
+        rc = pci_request_region(pdev, acfg->global_mmio_bar, "ocxlflash");
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: pci_request_region for global failed rc=%d\n",
+                        __func__, rc);
+                goto out;
+        }
+        gmmio = pci_resource_start(pdev, acfg->global_mmio_bar);
+        gmmio += acfg->global_mmio_offset;
+
+        rc = pci_request_region(pdev, acfg->pp_mmio_bar, "ocxlflash");
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: pci_request_region for pp bar failed rc=%d\n",
+                        __func__, rc);
+                goto err1;
+        }
+        ppmmio = pci_resource_start(pdev, acfg->pp_mmio_bar);
+        ppmmio += acfg->pp_mmio_offset;
+
+        afu->gmmio_virt = ioremap(gmmio, acfg->global_mmio_size);
+        if (unlikely(!afu->gmmio_virt)) {
+                dev_err(dev, "%s: MMIO mapping failed\n", __func__);
+                rc = -ENOMEM;
+                goto err2;
+        }
+
+        afu->gmmio_phys = gmmio;
+        afu->ppmmio_phys = ppmmio;
+out:
+        return rc;
+err2:
+        pci_release_region(pdev, acfg->pp_mmio_bar);
+err1:
+        pci_release_region(pdev, acfg->global_mmio_bar);
+        goto out;
+}
+
+/**
+ * ocxlflash_config_afu() - configure the host AFU
+ * @pdev:       PCI device associated with the host.
+ * @afu:        AFU associated with the host.
+ *
+ * Must be called _after_ host function configuration.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_config_afu(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
+{
+        struct ocxl_afu_config *acfg = &afu->acfg;
+        struct ocxl_fn_config *fcfg = &afu->fcfg;
+        struct device *dev = &pdev->dev;
+        int count;
+        int base;
+        int pos;
+        int rc = 0;
+
+        /* This HW AFU function does not have any AFUs defined */
+        if (!afu->is_present)
+                goto out;
+
+        /* Read AFU config at index 0 */
+        rc = ocxl_config_read_afu(pdev, fcfg, acfg, 0);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: ocxl_config_read_afu failed rc=%d\n",
+                        __func__, rc);
+                goto out;
+        }
+
+        /* Only one AFU per function is supported, so actag_base is same */
+        base = afu->fn_actag_base;
+        count = min_t(int, acfg->actag_supported, afu->fn_actag_enabled);
+        pos = acfg->dvsec_afu_control_pos;
+
+        ocxl_config_set_afu_actag(pdev, pos, base, count);
+        dev_dbg(dev, "%s: acTag base=%d enabled=%d\n", __func__, base, count);
+        afu->afu_actag_base = base;
+        afu->afu_actag_enabled = count;
+        afu->max_pasid = 1 << acfg->pasid_supported_log;
+
+        ocxl_config_set_afu_pasid(pdev, pos, 0, acfg->pasid_supported_log);
+
+        rc = ocxlflash_map_mmio(afu);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: ocxlflash_map_mmio failed rc=%d\n",
+                        __func__, rc);
+                goto out;
+        }
+
+        /* Enable the AFU */
+        ocxl_config_set_afu_state(pdev, acfg->dvsec_afu_control_pos, 1);
+out:
+        return rc;
+}
+
+/**
+ * ocxlflash_create_afu() - create the AFU for OCXL
+ * @pdev:       PCI device associated with the host.
+ *
+ * Return: AFU on success, NULL on failure
+ */
+static void *ocxlflash_create_afu(struct pci_dev *pdev)
+{
+        struct device *dev = &pdev->dev;
+        struct ocxlflash_context *ctx;
+        struct ocxl_hw_afu *afu;
+        int rc;
+
+        afu = kzalloc(sizeof(*afu), GFP_KERNEL);
+        if (unlikely(!afu)) {
+                dev_err(dev, "%s: HW AFU allocation failed\n", __func__);
+                goto out;
+        }
+
+        afu->pdev = pdev;
+        afu->dev = dev;
+        idr_init(&afu->idr);
+
+        rc = ocxlflash_config_fn(pdev, afu);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: Function configuration failed rc=%d\n",
+                        __func__, rc);
+                goto err1;
+        }
+
+        rc = ocxlflash_config_afu(pdev, afu);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: AFU configuration failed rc=%d\n",
+                        __func__, rc);
+                goto err2;
+        }
+
+        ctx = ocxlflash_dev_context_init(pdev, afu);
+        if (IS_ERR(ctx)) {
+                rc = PTR_ERR(ctx);
+                dev_err(dev, "%s: ocxlflash_dev_context_init failed rc=%d\n",
+                        __func__, rc);
+                goto err3;
+        }
+
+        afu->ocxl_ctx = ctx;
+out:
+        return afu;
+err3:
+        ocxlflash_unconfig_afu(afu);
+err2:
+        ocxlflash_unconfig_fn(pdev, afu);
+err1:
+        idr_destroy(&afu->idr);
+        kfree(afu);
+        afu = NULL;
+        goto out;
+}
+
+/**
+ * ctx_event_pending() - check for any event pending on the context
+ * @ctx:        Context to be checked.
+ *
+ * Return: true if there is an event pending, false if none pending
+ */
+static inline bool ctx_event_pending(struct ocxlflash_context *ctx)
+{
+        if (ctx->pending_irq || ctx->pending_fault)
+                return true;
+
+        return false;
+}
+
+/**
+ * afu_poll() - poll the AFU for events on the context
+ * @file:       File associated with the adapter context.
+ * @poll:       Poll structure from the user.
+ *
+ * Return: poll mask
+ */
+static unsigned int afu_poll(struct file *file, struct poll_table_struct *poll)
+{
+        struct ocxlflash_context *ctx = file->private_data;
+        struct device *dev = ctx->hw_afu->dev;
+        ulong lock_flags;
+        int mask = 0;
+
+        poll_wait(file, &ctx->wq, poll);
+
+        spin_lock_irqsave(&ctx->slock, lock_flags);
+        if (ctx_event_pending(ctx))
+                mask |= POLLIN | POLLRDNORM;
+        else if (ctx->state == CLOSED)
+                mask |= POLLERR;
+        spin_unlock_irqrestore(&ctx->slock, lock_flags);
+
+        dev_dbg(dev, "%s: Poll wait completed for pe %i mask %i\n",
+                __func__, ctx->pe, mask);
+
+        return mask;
+}
+
+/**
+ * afu_read() - perform a read on the context for any event
+ * @file:       File associated with the adapter context.
+ * @buf:        Buffer to receive the data.
+ * @count:      Size of buffer (maximum bytes that can be read).
+ * @off:        Offset.
+ *
+ * Return: size of the data read on success, -errno on failure
+ */
+static ssize_t afu_read(struct file *file, char __user *buf, size_t count,
+                        loff_t *off)
+{
+        struct ocxlflash_context *ctx = file->private_data;
+        struct device *dev = ctx->hw_afu->dev;
+        struct cxl_event event;
+        ulong lock_flags;
+        ssize_t esize;
+        ssize_t rc;
+        int bit;
+        DEFINE_WAIT(event_wait);
+
+        if (*off != 0) {
+                dev_err(dev, "%s: Non-zero offset not supported, off=%lld\n",
+                        __func__, *off);
+                rc = -EINVAL;
+                goto out;
+        }
+
+        spin_lock_irqsave(&ctx->slock, lock_flags);
+
+        for (;;) {
+                prepare_to_wait(&ctx->wq, &event_wait, TASK_INTERRUPTIBLE);
+
+                if (ctx_event_pending(ctx) || (ctx->state == CLOSED))
+                        break;
+
+                if (file->f_flags & O_NONBLOCK) {
+                        dev_err(dev, "%s: File cannot be blocked on I/O\n",
+                                __func__);
+                        rc = -EAGAIN;
+                        goto err;
+                }
+
+                if (signal_pending(current)) {
+                        dev_err(dev, "%s: Signal pending on the process\n",
+                                __func__);
+                        rc = -ERESTARTSYS;
+                        goto err;
+                }
+
+                spin_unlock_irqrestore(&ctx->slock, lock_flags);
+                schedule();
+                spin_lock_irqsave(&ctx->slock, lock_flags);
+        }
+
+        finish_wait(&ctx->wq, &event_wait);
+
+        memset(&event, 0, sizeof(event));
+        event.header.process_element = ctx->pe;
+        event.header.size = sizeof(struct cxl_event_header);
+        if (ctx->pending_irq) {
+                esize = sizeof(struct cxl_event_afu_interrupt);
+                event.header.size += esize;
+                event.header.type = CXL_EVENT_AFU_INTERRUPT;
+
+                bit = find_first_bit(&ctx->irq_bitmap, ctx->num_irqs);
+                clear_bit(bit, &ctx->irq_bitmap);
+                event.irq.irq = bit + 1;
+                if (bitmap_empty(&ctx->irq_bitmap, ctx->num_irqs))
+                        ctx->pending_irq = false;
+        } else if (ctx->pending_fault) {
+                event.header.size += sizeof(struct cxl_event_data_storage);
+                event.header.type = CXL_EVENT_DATA_STORAGE;
+                event.fault.addr = ctx->fault_addr;
+                event.fault.dsisr = ctx->fault_dsisr;
+                ctx->pending_fault = false;
+        }
+
+        spin_unlock_irqrestore(&ctx->slock, lock_flags);
+
+        if (copy_to_user(buf, &event, event.header.size)) {
+                dev_err(dev, "%s: copy_to_user failed\n", __func__);
+                rc = -EFAULT;
+                goto out;
+        }
+
+        rc = event.header.size;
+out:
+        return rc;
+err:
+        finish_wait(&ctx->wq, &event_wait);
+        spin_unlock_irqrestore(&ctx->slock, lock_flags);
+        goto out;
+}
+
+/**
+ * afu_release() - release and free the context
+ * @inode:      File inode pointer.
+ * @file:       File associated with the context.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int afu_release(struct inode *inode, struct file *file)
+{
+        struct ocxlflash_context *ctx = file->private_data;
+        int i;
+
+        /* Unmap and free the interrupts associated with the context */
+        for (i = ctx->num_irqs; i >= 0; i--)
+                afu_unmap_irq(0, ctx, i, ctx);
+        free_afu_irqs(ctx);
+
+        return ocxlflash_release_context(ctx);
+}
+
+/**
+ * ocxlflash_mmap_fault() - mmap fault handler
+ * @vmf:        VM fault associated with current fault.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_mmap_fault(struct vm_fault *vmf)
+{
+        struct vm_area_struct *vma = vmf->vma;
+        struct ocxlflash_context *ctx = vma->vm_file->private_data;
+        struct device *dev = ctx->hw_afu->dev;
+        u64 mmio_area, offset;
+
+        offset = vmf->pgoff << PAGE_SHIFT;
+        if (offset >= ctx->psn_size)
+                return VM_FAULT_SIGBUS;
+
+        mutex_lock(&ctx->state_mutex);
+        if (ctx->state != STARTED) {
+                dev_err(dev, "%s: Context not started, state=%d\n",
+                        __func__, ctx->state);
+                mutex_unlock(&ctx->state_mutex);
+                return VM_FAULT_SIGBUS;
+        }
+        mutex_unlock(&ctx->state_mutex);
+
+        mmio_area = ctx->psn_phys;
+        mmio_area += offset;
+
+        vm_insert_pfn(vma, vmf->address, mmio_area >> PAGE_SHIFT);
+        return VM_FAULT_NOPAGE;
+}
+
+static const struct vm_operations_struct ocxlflash_vmops = {
+        .fault = ocxlflash_mmap_fault,
+};
+
+/**
+ * afu_mmap() - map the fault handler operations
+ * @file:       File associated with the context.
+ * @vma:        VM area associated with mapping.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int afu_mmap(struct file *file, struct vm_area_struct *vma)
+{
+        struct ocxlflash_context *ctx = file->private_data;
+
+        if ((vma_pages(vma) + vma->vm_pgoff) >
+            (ctx->psn_size >> PAGE_SHIFT))
+                return -EINVAL;
+
+        vma->vm_flags |= VM_IO | VM_PFNMAP;
+        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+        vma->vm_ops = &ocxlflash_vmops;
+        return 0;
+}
+
+static const struct file_operations ocxl_afu_fops = {
+        .owner          = THIS_MODULE,
+        .poll           = afu_poll,
+        .read           = afu_read,
+        .release        = afu_release,
+        .mmap           = afu_mmap,
+};
+
+#define PATCH_FOPS(NAME)                                                \
+        do { if (!fops->NAME) fops->NAME = ocxl_afu_fops.NAME; } while (0)
+
+/**
+ * ocxlflash_get_fd() - get file descriptor for an adapter context
+ * @ctx_cookie: Adapter context.
+ * @fops:       File operations to be associated.
+ * @fd:         File descriptor to be returned back.
+ *
+ * Return: pointer to the file on success, ERR_PTR on failure
+ */
+static struct file *ocxlflash_get_fd(void *ctx_cookie,
+                                     struct file_operations *fops, int *fd)
+{
+        struct ocxlflash_context *ctx = ctx_cookie;
+        struct device *dev = ctx->hw_afu->dev;
+        struct file *file;
+        int flags, fdtmp;
+        int rc = 0;
+        char *name = NULL;
+
+        /* Only allow one fd per context */
+        if (ctx->mapping) {
+                dev_err(dev, "%s: Context is already mapped to an fd\n",
+                        __func__);
+                rc = -EEXIST;
+                goto err1;
+        }
+
+        flags = O_RDWR | O_CLOEXEC;
+
+        /* This code is similar to anon_inode_getfd() */
+        rc = get_unused_fd_flags(flags);
+        if (unlikely(rc < 0)) {
+                dev_err(dev, "%s: get_unused_fd_flags failed rc=%d\n",
+                        __func__, rc);
+                goto err1;
+        }
+        fdtmp = rc;
+
+        /* Patch the file ops that are not defined */
+        if (fops) {
+                PATCH_FOPS(poll);
+                PATCH_FOPS(read);
+                PATCH_FOPS(release);
+                PATCH_FOPS(mmap);
+        } else /* Use default ops */
+                fops = (struct file_operations *)&ocxl_afu_fops;
+
+        name = kasprintf(GFP_KERNEL, "ocxlflash:%d", ctx->pe);
+        file = ocxlflash_getfile(dev, name, fops, ctx, flags);
+        kfree(name);
+        if (IS_ERR(file)) {
+                rc = PTR_ERR(file);
+                dev_err(dev, "%s: ocxlflash_getfile failed rc=%d\n",
+                        __func__, rc);
+                goto err2;
+        }
+
+        ctx->mapping = file->f_mapping;
+        *fd = fdtmp;
+out:
+        return file;
+err2:
+        put_unused_fd(fdtmp);
+err1:
+        file = ERR_PTR(rc);
+        goto out;
+}
+
+/**
+ * ocxlflash_fops_get_context() - get the context associated with the file
+ * @file:       File associated with the adapter context.
+ *
+ * Return: pointer to the context
+ */
+static void *ocxlflash_fops_get_context(struct file *file)
+{
+        return file->private_data;
+}
+
+/**
+ * ocxlflash_afu_irq() - interrupt handler for user contexts
+ * @irq:        Interrupt number.
+ * @data:       Private data provided at interrupt registration, the context.
+ *
+ * Return: Always return IRQ_HANDLED.
+ */
+static irqreturn_t ocxlflash_afu_irq(int irq, void *data)
+{
+        struct ocxlflash_context *ctx = data;
+        struct device *dev = ctx->hw_afu->dev;
+        int i;
+
+        dev_dbg(dev, "%s: Interrupt raised for pe %i virq %i\n",
+                __func__, ctx->pe, irq);
+
+        for (i = 0; i < ctx->num_irqs; i++) {
+                if (ctx->irqs[i].virq == irq)
+                        break;
+        }
+        if (unlikely(i >= ctx->num_irqs)) {
+                dev_err(dev, "%s: Received AFU IRQ out of range\n", __func__);
+                goto out;
+        }
+
+        spin_lock(&ctx->slock);
+        set_bit(i - 1, &ctx->irq_bitmap);
+        ctx->pending_irq = true;
+        spin_unlock(&ctx->slock);
+
+        wake_up_all(&ctx->wq);
+out:
+        return IRQ_HANDLED;
+}
+
+/**
+ * ocxlflash_start_work() - start a user context
+ * @ctx_cookie: Context to be started.
+ * @num_irqs:   Number of interrupts requested.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_start_work(void *ctx_cookie, u64 num_irqs)
+{
+        struct ocxlflash_context *ctx = ctx_cookie;
+        struct ocxl_hw_afu *afu = ctx->hw_afu;
+        struct device *dev = afu->dev;
+        char *name;
+        int rc = 0;
+        int i;
+
+        rc = alloc_afu_irqs(ctx, num_irqs);
+        if (unlikely(rc < 0)) {
+                dev_err(dev, "%s: alloc_afu_irqs failed rc=%d\n", __func__, rc);
+                goto out;
+        }
+
+        for (i = 0; i < num_irqs; i++) {
+                name = kasprintf(GFP_KERNEL, "ocxlflash-%s-pe%i-%i",
+                                 dev_name(dev), ctx->pe, i);
+                rc = afu_map_irq(0, ctx, i, ocxlflash_afu_irq, ctx, name);
+                kfree(name);
+                if (unlikely(rc < 0)) {
+                        dev_err(dev, "%s: afu_map_irq failed rc=%d\n",
+                                __func__, rc);
+                        goto err;
+                }
+        }
+
+        rc = start_context(ctx);
+        if (unlikely(rc)) {
+                dev_err(dev, "%s: start_context failed rc=%d\n", __func__, rc);
+                goto err;
+        }
+out:
+        return rc;
+err:
+        for (i = i-1; i >= 0; i--)
+                afu_unmap_irq(0, ctx, i, ctx);
+        free_afu_irqs(ctx);
+        goto out;
+};
+
+/**
+ * ocxlflash_fd_mmap() - mmap handler for adapter file descriptor
+ * @file:       File installed with adapter file descriptor.
+ * @vma:        VM area associated with mapping.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_fd_mmap(struct file *file, struct vm_area_struct *vma)
+{
+        return afu_mmap(file, vma);
+}
+
+/**
+ * ocxlflash_fd_release() - release the context associated with the file
+ * @inode:      File inode pointer.
+ * @file:       File associated with the adapter context.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_fd_release(struct inode *inode, struct file *file)
+{
+        return afu_release(inode, file);
+}
+
+/* Backend ops to ocxlflash services */
+const struct cxlflash_backend_ops cxlflash_ocxl_ops = {
+        .module                 = THIS_MODULE,
+        .psa_map                = ocxlflash_psa_map,
+        .psa_unmap              = ocxlflash_psa_unmap,
+        .process_element        = ocxlflash_process_element,
+        .map_afu_irq            = ocxlflash_map_afu_irq,
+        .unmap_afu_irq          = ocxlflash_unmap_afu_irq,
+        .get_irq_objhndl        = ocxlflash_get_irq_objhndl,
+        .start_context          = ocxlflash_start_context,
+        .stop_context           = ocxlflash_stop_context,
+        .afu_reset              = ocxlflash_afu_reset,
+        .set_master             = ocxlflash_set_master,
+        .get_context            = ocxlflash_get_context,
+        .dev_context_init       = ocxlflash_dev_context_init,
+        .release_context        = ocxlflash_release_context,
+        .perst_reloads_same_image = ocxlflash_perst_reloads_same_image,
+        .read_adapter_vpd       = ocxlflash_read_adapter_vpd,
+        .allocate_afu_irqs      = ocxlflash_allocate_afu_irqs,
+        .free_afu_irqs          = ocxlflash_free_afu_irqs,
+        .create_afu             = ocxlflash_create_afu,
+        .destroy_afu            = ocxlflash_destroy_afu,
+        .get_fd                 = ocxlflash_get_fd,
+        .fops_get_context       = ocxlflash_fops_get_context,
+        .start_work             = ocxlflash_start_work,
+        .fd_mmap                = ocxlflash_fd_mmap,
+        .fd_release             = ocxlflash_fd_release,
+};
diff --git a/drivers/scsi/cxlflash/ocxl_hw.h b/drivers/scsi/cxlflash/ocxl_hw.h
new file mode 100644
index 000000000000..9270d35c4620
--- /dev/null
+++ b/drivers/scsi/cxlflash/ocxl_hw.h
@@ -0,0 +1,77 @@
+/*
+ * CXL Flash Device Driver
+ *
+ * Written by: Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
+ *             Uma Krishnan <ukrishn@linux.vnet.ibm.com>, IBM Corporation
+ *
+ * Copyright (C) 2018 IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#define OCXL_MAX_IRQS   4       /* Max interrupts per process */
+
+struct ocxlflash_irqs {
+        int hwirq;
+        u32 virq;
+        u64 ptrig;
+        void __iomem *vtrig;
+};
+
+/* OCXL hardware AFU associated with the host */
+struct ocxl_hw_afu {
+        struct ocxlflash_context *ocxl_ctx; /* Host context */
+        struct pci_dev *pdev;           /* PCI device */
+        struct device *dev;             /* Generic device */
+        bool perst_same_image;          /* Same image loaded on perst */
+
+        struct ocxl_fn_config fcfg;     /* DVSEC config of the function */
+        struct ocxl_afu_config acfg;    /* AFU configuration data */
+
+        int fn_actag_base;              /* Function acTag base */
+        int fn_actag_enabled;           /* Function acTag number enabled */
+        int afu_actag_base;             /* AFU acTag base */
+        int afu_actag_enabled;          /* AFU acTag number enabled */
+
+        phys_addr_t ppmmio_phys;        /* Per process MMIO space */
+        phys_addr_t gmmio_phys;         /* Global AFU MMIO space */
+        void __iomem *gmmio_virt;       /* Global MMIO map */
+
+        void *link_token;               /* Link token for the SPA */
+        struct idr idr;                 /* IDR to manage contexts */
+        int max_pasid;                  /* Maximum number of contexts */
+        bool is_present;                /* Function has AFUs defined */
+};
+
+enum ocxlflash_ctx_state {
+        CLOSED,
+        OPENED,
+        STARTED
+};
+
+struct ocxlflash_context {
+        struct ocxl_hw_afu *hw_afu;     /* HW AFU back pointer */
+        struct address_space *mapping;  /* Mapping for pseudo filesystem */
+        bool master;                    /* Whether this is a master context */
+        int pe;                         /* Process element */
+
+        phys_addr_t psn_phys;           /* Process mapping */
+        u64 psn_size;                   /* Process mapping size */
+
+        spinlock_t slock;               /* Protects irq/fault/event updates */
+        wait_queue_head_t wq;           /* Wait queue for poll and interrupts */
+        struct mutex state_mutex;       /* Mutex to update context state */
+        enum ocxlflash_ctx_state state; /* Context state */
+
+        struct ocxlflash_irqs *irqs;    /* Pointer to array of structures */
+        int num_irqs;                   /* Number of interrupts */
+        bool pending_irq;               /* Pending interrupt on the context */
+        ulong irq_bitmap;               /* Bits indicating pending irq num */
+
+        u64 fault_addr;                 /* Address that triggered the fault */
+        u64 fault_dsisr;                /* Value of dsisr register at fault */
+        bool pending_fault;             /* Pending translation fault */
+};
diff --git a/drivers/scsi/cxlflash/sislite.h b/drivers/scsi/cxlflash/sislite.h
index bedf1ce2f33c..874abce35ab4 100644
--- a/drivers/scsi/cxlflash/sislite.h
+++ b/drivers/scsi/cxlflash/sislite.h
@@ -258,23 +258,30 @@ struct sisl_host_map {
                                  * exit since there is no way to tell which
                                  * command caused the error.
                                  */
-#define SISL_ISTATUS_PERM_ERR_CMDROOM    0x0010ULL      /* b59, user error */
-#define SISL_ISTATUS_PERM_ERR_RCB_READ   0x0008ULL      /* b60, user error */
-#define SISL_ISTATUS_PERM_ERR_SA_WRITE   0x0004ULL      /* b61, user error */
-#define SISL_ISTATUS_PERM_ERR_RRQ_WRITE  0x0002ULL      /* b62, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_3_EA         0x0400ULL /* b53, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_2_EA         0x0200ULL /* b54, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_1_EA         0x0100ULL /* b55, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_3_PASID      0x0080ULL /* b56, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_2_PASID      0x0040ULL /* b57, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_1_PASID      0x0020ULL /* b58, user error */
+#define SISL_ISTATUS_PERM_ERR_CMDROOM           0x0010ULL /* b59, user error */
+#define SISL_ISTATUS_PERM_ERR_RCB_READ          0x0008ULL /* b60, user error */
+#define SISL_ISTATUS_PERM_ERR_SA_WRITE          0x0004ULL /* b61, user error */
+#define SISL_ISTATUS_PERM_ERR_RRQ_WRITE         0x0002ULL /* b62, user error */
         /* Page in wait accessing RCB/IOASA/RRQ is reported in b63.
          * Same error in data/LXT/RHT access is reported via IOASA.
          */
-#define SISL_ISTATUS_TEMP_ERR_PAGEIN     0x0001ULL      /* b63, can be generated
-                                                         * only when AFU auto
-                                                         * retry is disabled.
-                                                         * If user can determine
-                                                         * the command that
-                                                         * caused the error, it
-                                                         * can be retried.
-                                                         */
-#define SISL_ISTATUS_UNMASK  (0x001FULL)        /* 1 means unmasked */
-#define SISL_ISTATUS_MASK    ~(SISL_ISTATUS_UNMASK)     /* 1 means masked */
+#define SISL_ISTATUS_TEMP_ERR_PAGEIN            0x0001ULL /* b63, can only be
+                                                           * generated when AFU
+                                                           * auto retry is
+                                                           * disabled. If user
+                                                           * can determine the
+                                                           * command that caused
+                                                           * the error, it can
+                                                           * be retried.
+                                                           */
+#define SISL_ISTATUS_UNMASK     (0x07FFULL)             /* 1 means unmasked */
+#define SISL_ISTATUS_MASK       ~(SISL_ISTATUS_UNMASK)  /* 1 means masked */
 
         __be64 intr_clear;
         __be64 intr_mask;
@@ -284,6 +291,7 @@ struct sisl_host_map {
         __be64 cmd_room;
         __be64 ctx_ctrl;        /* least significant byte or b56:63 is LISN# */
 #define SISL_CTX_CTRL_UNMAP_SECTOR      0x8000000000000000ULL /* b0 */
+#define SISL_CTX_CTRL_LISN_MASK         (0xFFULL)
         __be64 mbox_w;          /* restricted use */
         __be64 sq_start;        /* Submission Queue (R/W): write sequence and */
         __be64 sq_end;          /* inclusion semantics are the same as RRQ    */
@@ -309,6 +317,10 @@ struct sisl_ctrl_map {
 #define SISL_CTX_CAP_WRITE_CMD         0x0000000000000002ULL /* afu_rc 0x21 */
 #define SISL_CTX_CAP_READ_CMD          0x0000000000000001ULL /* afu_rc 0x21 */
         __be64 mbox_r;
+        __be64 lisn_pasid[2];
+        /* pasid _a arg must be ULL */
+#define SISL_LISN_PASID(_a, _b) (((_a) << 32) | (_b))
+        __be64 lisn_ea[3];
 };
 
 /* single copy global regs */
@@ -415,6 +427,7 @@ struct sisl_global_regs {
 #define SISL_INTVER_CAP_RESERVED_CMD_MODE_B     0x100000000000ULL
 #define SISL_INTVER_CAP_LUN_PROVISION           0x080000000000ULL
 #define SISL_INTVER_CAP_AFU_DEBUG               0x040000000000ULL
+#define SISL_INTVER_CAP_OCXL_LISN               0x020000000000ULL
 };
 
 #define CXLFLASH_NUM_FC_PORTS_PER_BANK  2       /* fixed # of ports per bank */
diff --git a/drivers/scsi/cxlflash/superpipe.c b/drivers/scsi/cxlflash/superpipe.c
index 2fe79df5c73c..e489d89cbb45 100644
--- a/drivers/scsi/cxlflash/superpipe.c
+++ b/drivers/scsi/cxlflash/superpipe.c
@@ -14,8 +14,9 @@
 
 #include <linux/delay.h>
 #include <linux/file.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
 #include <linux/syscalls.h>
-#include <misc/cxl.h>
 #include <asm/unaligned.h>
 
 #include <scsi/scsi.h>
@@ -269,6 +270,7 @@ static int afu_attach(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)
         int rc = 0;
         struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
         u64 val;
+        int i;
 
         /* Unlock cap and restrict user to read/write cmds in translated mode */
         readq_be(&ctrl_map->mbox_r);
@@ -282,6 +284,19 @@ static int afu_attach(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)
                 goto out;
         }
 
+        if (afu_is_ocxl_lisn(afu)) {
+                /* Set up the LISN effective address for each interrupt */
+                for (i = 0; i < ctxi->irqs; i++) {
+                        val = cfg->ops->get_irq_objhndl(ctxi->ctx, i);
+                        writeq_be(val, &ctrl_map->lisn_ea[i]);
+                }
+
+                /* Use primary HWQ PASID as identifier for all interrupts */
+                val = hwq->ctx_hndl;
+                writeq_be(SISL_LISN_PASID(val, val), &ctrl_map->lisn_pasid[0]);
+                writeq_be(SISL_LISN_PASID(0UL, val), &ctrl_map->lisn_pasid[1]);
+        }
+
         /* Set up MMIO registers pointing to the RHT */
         writeq_be((u64)ctxi->rht_start, &ctrl_map->rht_start);
         val = SISL_RHT_CNT_ID((u64)MAX_RHT_PER_CONTEXT, (u64)(hwq->ctx_hndl));
@@ -974,6 +989,10 @@ static int cxlflash_disk_detach(struct scsi_device *sdev,
  * theoretically never occur), every call into this routine results
  * in a complete freeing of a context.
  *
+ * Detaching the LUN is typically an ioctl() operation and the underlying
+ * code assumes that ioctl_rwsem has been acquired as a reader. To support
+ * that design point, the semaphore is acquired and released around detach.
+ *
  * Return: 0 on success
  */
 static int cxlflash_cxl_release(struct inode *inode, struct file *file)
@@ -1012,9 +1031,11 @@ static int cxlflash_cxl_release(struct inode *inode, struct file *file)
 
         dev_dbg(dev, "%s: close for ctxid=%d\n", __func__, ctxid);
 
+        down_read(&cfg->ioctl_rwsem);
         detach.context_id = ctxi->ctxid;
         list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)
                 _cxlflash_disk_detach(lun_access->sdev, ctxi, &detach);
+        up_read(&cfg->ioctl_rwsem);
 out_release:
         cfg->ops->fd_release(inode, file);
 out:
diff --git a/drivers/scsi/cxlflash/vlun.c b/drivers/scsi/cxlflash/vlun.c
index 5deef57a7834..66e445a17d6c 100644
--- a/drivers/scsi/cxlflash/vlun.c
+++ b/drivers/scsi/cxlflash/vlun.c
@@ -12,8 +12,9 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#include <linux/interrupt.h>
+#include <linux/pci.h>
 #include <linux/syscalls.h>
-#include <misc/cxl.h>
 #include <asm/unaligned.h>
 #include <asm/bitsperlong.h>
 
diff --git a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c
index 5ceea8da7bb6..37de8fb186d7 100644
--- a/drivers/scsi/dpt_i2o.c
+++ b/drivers/scsi/dpt_i2o.c
@@ -1706,7 +1706,7 @@ static int adpt_i2o_passthru(adpt_hba* pHba, u32 __user *arg)
         u32 reply_size = 0;
         u32 __user *user_msg = arg;
         u32 __user * user_reply = NULL;
-        void *sg_list[pHba->sg_tablesize];
+        void **sg_list = NULL;
         u32 sg_offset = 0;
         u32 sg_count = 0;
         int sg_index = 0;
@@ -1748,19 +1748,23 @@ static int adpt_i2o_passthru(adpt_hba* pHba, u32 __user *arg)
         msg[2] = 0x40000000; // IOCTL context
         msg[3] = adpt_ioctl_to_context(pHba, reply);
         if (msg[3] == (u32)-1) {
-                kfree(reply);
-                return -EBUSY;
+                rcode = -EBUSY;
+                goto free;
         }
 
-        memset(sg_list,0, sizeof(sg_list[0])*pHba->sg_tablesize);
+        sg_list = kcalloc(pHba->sg_tablesize, sizeof(*sg_list), GFP_KERNEL);
+        if (!sg_list) {
+                rcode = -ENOMEM;
+                goto free;
+        }
         if(sg_offset) {
                 // TODO add 64 bit API
                 struct sg_simple_element *sg =  (struct sg_simple_element*) (msg+sg_offset);
                 sg_count = (size - sg_offset*4) / sizeof(struct sg_simple_element);
                 if (sg_count > pHba->sg_tablesize){
                         printk(KERN_DEBUG"%s:IOCTL SG List too large (%u)\n", pHba->name,sg_count);
-                        kfree (reply);
-                        return -EINVAL;
+                        rcode = -EINVAL;
+                        goto free;
                 }
 
                 for(i = 0; i < sg_count; i++) {
@@ -1879,7 +1883,6 @@ cleanup:
         if (rcode != -ETIME && rcode != -EINTR) {
                 struct sg_simple_element *sg =
                                 (struct sg_simple_element*) (msg +sg_offset);
-                kfree (reply);
                 while(sg_index) {
                         if(sg_list[--sg_index]) {
                                 dma_free_coherent(&pHba->pDev->dev,
@@ -1889,6 +1892,10 @@ cleanup:
                         }
                 }
         }
+
+free:
+        kfree(sg_list);
+        kfree(reply);
         return rcode;
 }
 
diff --git a/drivers/scsi/esas2r/esas2r_init.c b/drivers/scsi/esas2r/esas2r_init.c
index 9dffcb28c9b7..9db645dde35e 100644
--- a/drivers/scsi/esas2r/esas2r_init.c
+++ b/drivers/scsi/esas2r/esas2r_init.c
@@ -1202,8 +1202,6 @@ static bool esas2r_format_init_msg(struct esas2r_adapter *a,
         case ESAS2R_INIT_MSG_START:
         case ESAS2R_INIT_MSG_REINIT:
         {
-                struct timeval now;
-                do_gettimeofday(&now);
                 esas2r_hdebug("CFG init");
                 esas2r_build_cfg_req(a,
                                      rq,
@@ -1212,7 +1210,8 @@ static bool esas2r_format_init_msg(struct esas2r_adapter *a,
                                      NULL);
                 ci = (struct atto_vda_cfg_init *)&rq->vrq->cfg.data.init;
                 ci->sgl_page_size = cpu_to_le32(sgl_page_size);
-                ci->epoch_time = cpu_to_le32(now.tv_sec);
+                /* firmware interface overflows in y2106 */
+                ci->epoch_time = cpu_to_le32(ktime_get_real_seconds());
                 rq->flags |= RF_FAILURE_OK;
                 a->init_msg = ESAS2R_INIT_MSG_INIT;
                 break;
diff --git a/drivers/scsi/esas2r/esas2r_ioctl.c b/drivers/scsi/esas2r/esas2r_ioctl.c
index 97623002908f..34bcc8c04ff4 100644
--- a/drivers/scsi/esas2r/esas2r_ioctl.c
+++ b/drivers/scsi/esas2r/esas2r_ioctl.c
@@ -1849,7 +1849,7 @@ int esas2r_read_vda(struct esas2r_adapter *a, char *buf, long off, int count)
                 /* allocate a request */
                 rq = esas2r_alloc_request(a);
                 if (rq == NULL) {
-                        esas2r_debug("esas2r_read_vda: out of requestss");
+                        esas2r_debug("esas2r_read_vda: out of requests");
                         return -EBUSY;
                 }
 
diff --git a/drivers/scsi/esas2r/esas2r_main.c b/drivers/scsi/esas2r/esas2r_main.c
index e07eac5be087..c07118617d89 100644
--- a/drivers/scsi/esas2r/esas2r_main.c
+++ b/drivers/scsi/esas2r/esas2r_main.c
@@ -283,7 +283,7 @@ MODULE_PARM_DESC(num_requests,
 int num_ae_requests = 4;
 module_param(num_ae_requests, int, 0);
 MODULE_PARM_DESC(num_ae_requests,
-                 "Number of VDA asynchromous event requests.  Default 4.");
+                 "Number of VDA asynchronous event requests.  Default 4.");
 
 int cmd_per_lun = ESAS2R_DEFAULT_CMD_PER_LUN;
 module_param(cmd_per_lun, int, 0);
diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h
index d1153e8e846b..7052a5d45f7f 100644
--- a/drivers/scsi/hisi_sas/hisi_sas.h
+++ b/drivers/scsi/hisi_sas/hisi_sas.h
@@ -136,12 +136,14 @@ struct hisi_sas_phy {
         struct hisi_sas_port    *port;
         struct asd_sas_phy      sas_phy;
         struct sas_identify     identify;
+        struct completion *reset_completion;
+        spinlock_t lock;
         u64             port_id; /* from hw */
-        u64             dev_sas_addr;
         u64             frame_rcvd_size;
         u8              frame_rcvd[32];
         u8              phy_attached;
-        u8              reserved[3];
+        u8              in_reset;
+        u8              reserved[2];
         u32             phy_type;
         enum sas_linkrate       minimum_linkrate;
         enum sas_linkrate       maximum_linkrate;
@@ -162,7 +164,7 @@ struct hisi_sas_cq {
 
 struct hisi_sas_dq {
         struct hisi_hba *hisi_hba;
-        struct hisi_sas_slot    *slot_prep;
+        struct list_head list;
         spinlock_t lock;
         int     wr_point;
         int     id;
@@ -174,15 +176,22 @@ struct hisi_sas_device {
         struct completion *completion;
         struct hisi_sas_dq      *dq;
         struct list_head        list;
-        u64 attached_phy;
         enum sas_device_type    dev_type;
         int device_id;
         int sata_idx;
         u8 dev_status;
 };
 
+struct hisi_sas_tmf_task {
+        int force_phy;
+        int phy_id;
+        u8 tmf;
+        u16 tag_of_task_to_be_managed;
+};
+
 struct hisi_sas_slot {
         struct list_head entry;
+        struct list_head delivery;
         struct sas_task *task;
         struct hisi_sas_port    *port;
         u64     n_elem;
@@ -192,17 +201,15 @@ struct hisi_sas_slot {
         int     cmplt_queue_slot;
         int     idx;
         int     abort;
+        int     ready;
         void    *buf;
         dma_addr_t buf_dma;
         void    *cmd_hdr;
         dma_addr_t cmd_hdr_dma;
         struct work_struct abort_slot;
         struct timer_list internal_abort_timer;
-};
-
-struct hisi_sas_tmf_task {
-        u8 tmf;
-        u16 tag_of_task_to_be_managed;
+        bool is_internal;
+        struct hisi_sas_tmf_task *tmf;
 };
 
 struct hisi_sas_hw {
@@ -215,14 +222,13 @@ struct hisi_sas_hw {
         void (*sl_notify)(struct hisi_hba *hisi_hba, int phy_no);
         int (*get_free_slot)(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq);
         void (*start_delivery)(struct hisi_sas_dq *dq);
-        int (*prep_ssp)(struct hisi_hba *hisi_hba,
-                        struct hisi_sas_slot *slot, int is_tmf,
-                        struct hisi_sas_tmf_task *tmf);
-        int (*prep_smp)(struct hisi_hba *hisi_hba,
+        void (*prep_ssp)(struct hisi_hba *hisi_hba,
+                        struct hisi_sas_slot *slot);
+        void (*prep_smp)(struct hisi_hba *hisi_hba,
                         struct hisi_sas_slot *slot);
-        int (*prep_stp)(struct hisi_hba *hisi_hba,
+        void (*prep_stp)(struct hisi_hba *hisi_hba,
                         struct hisi_sas_slot *slot);
-        int (*prep_abort)(struct hisi_hba *hisi_hba,
+        void (*prep_abort)(struct hisi_hba *hisi_hba,
                           struct hisi_sas_slot *slot,
                           int device_id, int abort_flag, int tag_to_abort);
         int (*slot_complete)(struct hisi_hba *hisi_hba,
@@ -245,8 +251,11 @@ struct hisi_sas_hw {
         u32 (*get_phys_state)(struct hisi_hba *hisi_hba);
         int (*write_gpio)(struct hisi_hba *hisi_hba, u8 reg_type,
                                 u8 reg_index, u8 reg_count, u8 *write_data);
+        void (*wait_cmds_complete_timeout)(struct hisi_hba *hisi_hba,
+                                           int delay_ms, int timeout_ms);
         int max_command_entries;
         int complete_hdr_size;
+        struct scsi_host_template *sht;
 };
 
 struct hisi_hba {
@@ -273,6 +282,8 @@ struct hisi_hba {
         struct workqueue_struct *wq;
 
         int slot_index_count;
+        int last_slot_index;
+        int last_dev_id;
         unsigned long *slot_index_tags;
         unsigned long reject_stp_links_msk;
 
@@ -411,7 +422,7 @@ struct hisi_sas_command_table_ssp {
         union {
                 struct {
                         struct ssp_command_iu task;
-                        u32 prot[6];
+                        u32 prot[7];
                 };
                 struct ssp_tmf_iu ssp_task;
                 struct xfer_rdy_iu xfer_rdy;
@@ -437,10 +448,7 @@ struct hisi_sas_slot_buf_table {
 };
 
 extern struct scsi_transport_template *hisi_sas_stt;
-extern struct scsi_host_template *hisi_sas_sht;
-
 extern void hisi_sas_stop_phys(struct hisi_hba *hisi_hba);
-extern void hisi_sas_init_add(struct hisi_hba *hisi_hba);
 extern int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost);
 extern void hisi_sas_free(struct hisi_hba *hisi_hba);
 extern u8 hisi_sas_get_ata_protocol(struct host_to_dev_fis *fis,
@@ -454,6 +462,11 @@ extern int hisi_sas_probe(struct platform_device *pdev,
                           const struct hisi_sas_hw *ops);
 extern int hisi_sas_remove(struct platform_device *pdev);
 
+extern int hisi_sas_slave_configure(struct scsi_device *sdev);
+extern int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time);
+extern void hisi_sas_scan_start(struct Scsi_Host *shost);
+extern struct device_attribute *host_attrs[];
+extern int hisi_sas_host_reset(struct Scsi_Host *shost, int reset_type);
 extern void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy);
 extern void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba,
                                     struct sas_task *task,
@@ -465,4 +478,5 @@ extern void hisi_sas_kill_tasklets(struct hisi_hba *hisi_hba);
 extern bool hisi_sas_notify_phy_event(struct hisi_sas_phy *phy,
                                 enum hisi_sas_phy_event event);
 extern void hisi_sas_release_tasks(struct hisi_hba *hisi_hba);
+extern u8 hisi_sas_get_prog_phy_linkrate_mask(enum sas_linkrate max);
 #endif
diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c
index 49c1fa643803..6f562974f8f6 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_main.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_main.c
@@ -24,6 +24,9 @@ hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,
 static int hisi_sas_softreset_ata_disk(struct domain_device *device);
 static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
                                 void *funcdata);
+static void hisi_sas_release_task(struct hisi_hba *hisi_hba,
+                                  struct domain_device *device);
+static void hisi_sas_dev_gone(struct domain_device *device);
 
 u8 hisi_sas_get_ata_protocol(struct host_to_dev_fis *fis, int direction)
 {
@@ -78,22 +81,23 @@ u8 hisi_sas_get_ata_protocol(struct host_to_dev_fis *fis, int direction)
         case ATA_CMD_STANDBYNOW1:
         case ATA_CMD_ZAC_MGMT_OUT:
                 return HISI_SAS_SATA_PROTOCOL_NONDATA;
+
+        case ATA_CMD_SET_MAX:
+                switch (fis->features) {
+                case ATA_SET_MAX_PASSWD:
+                case ATA_SET_MAX_LOCK:
+                        return HISI_SAS_SATA_PROTOCOL_PIO;
+
+                case ATA_SET_MAX_PASSWD_DMA:
+                case ATA_SET_MAX_UNLOCK_DMA:
+                        return HISI_SAS_SATA_PROTOCOL_DMA;
+
+                default:
+                        return HISI_SAS_SATA_PROTOCOL_NONDATA;
+                }
+
         default:
         {
-                if (fis->command == ATA_CMD_SET_MAX) {
-                        switch (fis->features) {
-                        case ATA_SET_MAX_PASSWD:
-                        case ATA_SET_MAX_LOCK:
-                                return HISI_SAS_SATA_PROTOCOL_PIO;
-
-                        case ATA_SET_MAX_PASSWD_DMA:
-                        case ATA_SET_MAX_UNLOCK_DMA:
-                                return HISI_SAS_SATA_PROTOCOL_DMA;
-
-                        default:
-                                return HISI_SAS_SATA_PROTOCOL_NONDATA;
-                        }
-                }
                 if (direction == DMA_NONE)
                         return HISI_SAS_SATA_PROTOCOL_NONDATA;
                 return HISI_SAS_SATA_PROTOCOL_PIO;
@@ -134,6 +138,22 @@ int hisi_sas_get_ncq_tag(struct sas_task *task, u32 *tag)
 }
 EXPORT_SYMBOL_GPL(hisi_sas_get_ncq_tag);
 
+/*
+ * This function assumes linkrate mask fits in 8 bits, which it
+ * does for all HW versions supported.
+ */
+u8 hisi_sas_get_prog_phy_linkrate_mask(enum sas_linkrate max)
+{
+        u16 rate = 0;
+        int i;
+
+        max -= SAS_LINK_RATE_1_5_GBPS;
+        for (i = 0; i <= max; i++)
+                rate |= 1 << (i * 2);
+        return rate;
+}
+EXPORT_SYMBOL_GPL(hisi_sas_get_prog_phy_linkrate_mask);
+
 static struct hisi_hba *dev_to_hisi_hba(struct domain_device *device)
 {
         return device->port->ha->lldd_ha;
@@ -178,11 +198,18 @@ static int hisi_sas_slot_index_alloc(struct hisi_hba *hisi_hba, int *slot_idx)
         unsigned int index;
         void *bitmap = hisi_hba->slot_index_tags;
 
-        index = find_first_zero_bit(bitmap, hisi_hba->slot_index_count);
-        if (index >= hisi_hba->slot_index_count)
-                return -SAS_QUEUE_FULL;
+        index = find_next_zero_bit(bitmap, hisi_hba->slot_index_count,
+                        hisi_hba->last_slot_index + 1);
+        if (index >= hisi_hba->slot_index_count) {
+                index = find_next_zero_bit(bitmap, hisi_hba->slot_index_count,
+                                           0);
+                if (index >= hisi_hba->slot_index_count)
+                        return -SAS_QUEUE_FULL;
+        }
         hisi_sas_slot_index_set(hisi_hba, index);
         *slot_idx = index;
+        hisi_hba->last_slot_index = index;
+
         return 0;
 }
 
@@ -197,6 +224,8 @@ static void hisi_sas_slot_index_init(struct hisi_hba *hisi_hba)
 void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task,
                              struct hisi_sas_slot *slot)
 {
+        struct hisi_sas_dq *dq = &hisi_hba->dq[slot->dlvry_queue];
+        unsigned long flags;
 
         if (task) {
                 struct device *dev = hisi_hba->dev;
@@ -216,40 +245,43 @@ void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task,
         if (slot->buf)
                 dma_pool_free(hisi_hba->buffer_pool, slot->buf, slot->buf_dma);
 
+        spin_lock_irqsave(&dq->lock, flags);
         list_del_init(&slot->entry);
+        spin_unlock_irqrestore(&dq->lock, flags);
         slot->buf = NULL;
         slot->task = NULL;
         slot->port = NULL;
+        spin_lock_irqsave(&hisi_hba->lock, flags);
         hisi_sas_slot_index_free(hisi_hba, slot->idx);
+        spin_unlock_irqrestore(&hisi_hba->lock, flags);
 
         /* slot memory is fully zeroed when it is reused */
 }
 EXPORT_SYMBOL_GPL(hisi_sas_slot_task_free);
 
-static int hisi_sas_task_prep_smp(struct hisi_hba *hisi_hba,
+static void hisi_sas_task_prep_smp(struct hisi_hba *hisi_hba,
                                   struct hisi_sas_slot *slot)
 {
-        return hisi_hba->hw->prep_smp(hisi_hba, slot);
+        hisi_hba->hw->prep_smp(hisi_hba, slot);
 }
 
-static int hisi_sas_task_prep_ssp(struct hisi_hba *hisi_hba,
-                                  struct hisi_sas_slot *slot, int is_tmf,
-                                  struct hisi_sas_tmf_task *tmf)
+static void hisi_sas_task_prep_ssp(struct hisi_hba *hisi_hba,
+                                  struct hisi_sas_slot *slot)
 {
-        return hisi_hba->hw->prep_ssp(hisi_hba, slot, is_tmf, tmf);
+        hisi_hba->hw->prep_ssp(hisi_hba, slot);
 }
 
-static int hisi_sas_task_prep_ata(struct hisi_hba *hisi_hba,
+static void hisi_sas_task_prep_ata(struct hisi_hba *hisi_hba,
                                   struct hisi_sas_slot *slot)
 {
-        return hisi_hba->hw->prep_stp(hisi_hba, slot);
+        hisi_hba->hw->prep_stp(hisi_hba, slot);
 }
 
-static int hisi_sas_task_prep_abort(struct hisi_hba *hisi_hba,
+static void hisi_sas_task_prep_abort(struct hisi_hba *hisi_hba,
                 struct hisi_sas_slot *slot,
                 int device_id, int abort_flag, int tag_to_abort)
 {
-        return hisi_hba->hw->prep_abort(hisi_hba, slot,
+        hisi_hba->hw->prep_abort(hisi_hba, slot,
                         device_id, abort_flag, tag_to_abort);
 }
 
@@ -269,7 +301,6 @@ static void hisi_sas_slot_abort(struct work_struct *work)
         struct scsi_lun lun;
         struct device *dev = hisi_hba->dev;
         int tag = abort_slot->idx;
-        unsigned long flags;
 
         if (!(task->task_proto & SAS_PROTOCOL_SSP)) {
                 dev_err(dev, "cannot abort slot for non-ssp task\n");
@@ -283,27 +314,29 @@ static void hisi_sas_slot_abort(struct work_struct *work)
         hisi_sas_debug_issue_ssp_tmf(task->dev, lun.scsi_lun, &tmf_task);
 out:
         /* Do cleanup for this task */
-        spin_lock_irqsave(&hisi_hba->lock, flags);
         hisi_sas_slot_task_free(hisi_hba, task, abort_slot);
-        spin_unlock_irqrestore(&hisi_hba->lock, flags);
         if (task->task_done)
                 task->task_done(task);
 }
 
-static int hisi_sas_task_prep(struct sas_task *task, struct hisi_sas_dq
-                *dq, int is_tmf, struct hisi_sas_tmf_task *tmf,
-                int *pass)
+static int hisi_sas_task_prep(struct sas_task *task,
+                              struct hisi_sas_dq **dq_pointer,
+                              bool is_tmf, struct hisi_sas_tmf_task *tmf,
+                              int *pass)
 {
-        struct hisi_hba *hisi_hba = dq->hisi_hba;
         struct domain_device *device = task->dev;
+        struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
         struct hisi_sas_device *sas_dev = device->lldd_dev;
         struct hisi_sas_port *port;
         struct hisi_sas_slot *slot;
         struct hisi_sas_cmd_hdr *cmd_hdr_base;
         struct asd_sas_port *sas_port = device->port;
         struct device *dev = hisi_hba->dev;
-        int dlvry_queue_slot, dlvry_queue, n_elem = 0, rc, slot_idx;
-        unsigned long flags;
+        int dlvry_queue_slot, dlvry_queue, rc, slot_idx;
+        int n_elem = 0, n_elem_req = 0, n_elem_resp = 0;
+        unsigned long flags, flags_dq;
+        struct hisi_sas_dq *dq;
+        int wr_q_index;
 
         if (!sas_port) {
                 struct task_status_struct *ts = &task->task_status;
@@ -330,6 +363,8 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_sas_dq
                 return -ECOMM;
         }
 
+        *dq_pointer = dq = sas_dev->dq;
+
         port = to_hisi_sas_port(sas_port);
         if (port && !port->port_attached) {
                 dev_info(dev, "task prep: %s port%d not attach device\n",
@@ -341,6 +376,8 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_sas_dq
         }
 
         if (!sas_protocol_ata(task->task_proto)) {
+                unsigned int req_len, resp_len;
+
                 if (task->num_scatter) {
                         n_elem = dma_map_sg(dev, task->scatter,
                                             task->num_scatter, task->data_dir);
@@ -348,31 +385,74 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_sas_dq
                                 rc = -ENOMEM;
                                 goto prep_out;
                         }
+                } else if (task->task_proto & SAS_PROTOCOL_SMP) {
+                        n_elem_req = dma_map_sg(dev, &task->smp_task.smp_req,
+                                                1, DMA_TO_DEVICE);
+                        if (!n_elem_req) {
+                                rc = -ENOMEM;
+                                goto prep_out;
+                        }
+                        req_len = sg_dma_len(&task->smp_task.smp_req);
+                        if (req_len & 0x3) {
+                                rc = -EINVAL;
+                                goto err_out_dma_unmap;
+                        }
+                        n_elem_resp = dma_map_sg(dev, &task->smp_task.smp_resp,
+                                                 1, DMA_FROM_DEVICE);
+                        if (!n_elem_resp) {
+                                rc = -ENOMEM;
+                                goto err_out_dma_unmap;
+                        }
+                        resp_len = sg_dma_len(&task->smp_task.smp_resp);
+                        if (resp_len & 0x3) {
+                                rc = -EINVAL;
+                                goto err_out_dma_unmap;
+                        }
                 }
         } else
                 n_elem = task->num_scatter;
 
+        if (n_elem > HISI_SAS_SGE_PAGE_CNT) {
+                dev_err(dev, "task prep: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT",
+                        n_elem);
+                rc = -EINVAL;
+                goto err_out_dma_unmap;
+        }
+
         spin_lock_irqsave(&hisi_hba->lock, flags);
         if (hisi_hba->hw->slot_index_alloc)
                 rc = hisi_hba->hw->slot_index_alloc(hisi_hba, &slot_idx,
                                                     device);
         else
                 rc = hisi_sas_slot_index_alloc(hisi_hba, &slot_idx);
-        if (rc) {
-                spin_unlock_irqrestore(&hisi_hba->lock, flags);
-                goto err_out;
-        }
         spin_unlock_irqrestore(&hisi_hba->lock, flags);
-
-        rc = hisi_hba->hw->get_free_slot(hisi_hba, dq);
         if (rc)
-                goto err_out_tag;
+                goto err_out_dma_unmap;
 
-        dlvry_queue = dq->id;
-        dlvry_queue_slot = dq->wr_point;
         slot = &hisi_hba->slot_info[slot_idx];
         memset(slot, 0, sizeof(struct hisi_sas_slot));
 
+        slot->buf = dma_pool_alloc(hisi_hba->buffer_pool,
+                                   GFP_ATOMIC, &slot->buf_dma);
+        if (!slot->buf) {
+                rc = -ENOMEM;
+                goto err_out_tag;
+        }
+
+        spin_lock_irqsave(&dq->lock, flags_dq);
+        wr_q_index = hisi_hba->hw->get_free_slot(hisi_hba, dq);
+        if (wr_q_index < 0) {
+                spin_unlock_irqrestore(&dq->lock, flags_dq);
+                rc = -EAGAIN;
+                goto err_out_buf;
+        }
+
+        list_add_tail(&slot->delivery, &dq->list);
+        spin_unlock_irqrestore(&dq->lock, flags_dq);
+
+        dlvry_queue = dq->id;
+        dlvry_queue_slot = wr_q_index;
+
         slot->idx = slot_idx;
         slot->n_elem = n_elem;
         slot->dlvry_queue = dlvry_queue;
@@ -381,99 +461,94 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_sas_dq
         slot->cmd_hdr = &cmd_hdr_base[dlvry_queue_slot];
         slot->task = task;
         slot->port = port;
+        slot->tmf = tmf;
+        slot->is_internal = is_tmf;
         task->lldd_task = slot;
         INIT_WORK(&slot->abort_slot, hisi_sas_slot_abort);
 
-        slot->buf = dma_pool_alloc(hisi_hba->buffer_pool,
-                                   GFP_ATOMIC, &slot->buf_dma);
-        if (!slot->buf) {
-                rc = -ENOMEM;
-                goto err_out_slot_buf;
-        }
         memset(slot->cmd_hdr, 0, sizeof(struct hisi_sas_cmd_hdr));
         memset(hisi_sas_cmd_hdr_addr_mem(slot), 0, HISI_SAS_COMMAND_TABLE_SZ);
         memset(hisi_sas_status_buf_addr_mem(slot), 0, HISI_SAS_STATUS_BUF_SZ);
 
         switch (task->task_proto) {
         case SAS_PROTOCOL_SMP:
-                rc = hisi_sas_task_prep_smp(hisi_hba, slot);
+                hisi_sas_task_prep_smp(hisi_hba, slot);
                 break;
         case SAS_PROTOCOL_SSP:
-                rc = hisi_sas_task_prep_ssp(hisi_hba, slot, is_tmf, tmf);
+                hisi_sas_task_prep_ssp(hisi_hba, slot);
                 break;
         case SAS_PROTOCOL_SATA:
         case SAS_PROTOCOL_STP:
         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
-                rc = hisi_sas_task_prep_ata(hisi_hba, slot);
+                hisi_sas_task_prep_ata(hisi_hba, slot);
                 break;
         default:
                 dev_err(dev, "task prep: unknown/unsupported proto (0x%x)\n",
                         task->task_proto);
-                rc = -EINVAL;
                 break;
         }
 
-        if (rc) {
-                dev_err(dev, "task prep: rc = 0x%x\n", rc);
-                goto err_out_buf;
-        }
-
-        spin_lock_irqsave(&hisi_hba->lock, flags);
+        spin_lock_irqsave(&dq->lock, flags);
         list_add_tail(&slot->entry, &sas_dev->list);
-        spin_unlock_irqrestore(&hisi_hba->lock, flags);
+        spin_unlock_irqrestore(&dq->lock, flags);
         spin_lock_irqsave(&task->task_state_lock, flags);
         task->task_state_flags |= SAS_TASK_AT_INITIATOR;
         spin_unlock_irqrestore(&task->task_state_lock, flags);
 
-        dq->slot_prep = slot;
         ++(*pass);
+        slot->ready = 1;
 
         return 0;
 
 err_out_buf:
         dma_pool_free(hisi_hba->buffer_pool, slot->buf,
-                slot->buf_dma);
-err_out_slot_buf:
-        /* Nothing to be done */
+                      slot->buf_dma);
 err_out_tag:
         spin_lock_irqsave(&hisi_hba->lock, flags);
         hisi_sas_slot_index_free(hisi_hba, slot_idx);
         spin_unlock_irqrestore(&hisi_hba->lock, flags);
-err_out:
-        dev_err(dev, "task prep: failed[%d]!\n", rc);
-        if (!sas_protocol_ata(task->task_proto))
-                if (n_elem)
-                        dma_unmap_sg(dev, task->scatter,
-                                     task->num_scatter,
-                                     task->data_dir);
+err_out_dma_unmap:
+        if (!sas_protocol_ata(task->task_proto)) {
+                if (task->num_scatter) {
+                        dma_unmap_sg(dev, task->scatter, task->num_scatter,
+                             task->data_dir);
+                } else if (task->task_proto & SAS_PROTOCOL_SMP) {
+                        if (n_elem_req)
+                                dma_unmap_sg(dev, &task->smp_task.smp_req,
+                                             1, DMA_TO_DEVICE);
+                        if (n_elem_resp)
+                                dma_unmap_sg(dev, &task->smp_task.smp_resp,
+                                             1, DMA_FROM_DEVICE);
+                }
+        }
 prep_out:
+        dev_err(dev, "task prep: failed[%d]!\n", rc);
         return rc;
 }
 
 static int hisi_sas_task_exec(struct sas_task *task, gfp_t gfp_flags,
-                              int is_tmf, struct hisi_sas_tmf_task *tmf)
+                              bool is_tmf, struct hisi_sas_tmf_task *tmf)
 {
         u32 rc;
         u32 pass = 0;
         unsigned long flags;
         struct hisi_hba *hisi_hba = dev_to_hisi_hba(task->dev);
         struct device *dev = hisi_hba->dev;
-        struct domain_device *device = task->dev;
-        struct hisi_sas_device *sas_dev = device->lldd_dev;
-        struct hisi_sas_dq *dq = sas_dev->dq;
+        struct hisi_sas_dq *dq = NULL;
 
         if (unlikely(test_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags)))
                 return -EINVAL;
 
         /* protect task_prep and start_delivery sequence */
-        spin_lock_irqsave(&dq->lock, flags);
-        rc = hisi_sas_task_prep(task, dq, is_tmf, tmf, &pass);
+        rc = hisi_sas_task_prep(task, &dq, is_tmf, tmf, &pass);
         if (rc)
                 dev_err(dev, "task exec: failed[%d]!\n", rc);
 
-        if (likely(pass))
+        if (likely(pass)) {
+                spin_lock_irqsave(&dq->lock, flags);
                 hisi_hba->hw->start_delivery(dq);
-        spin_unlock_irqrestore(&dq->lock, flags);
+                spin_unlock_irqrestore(&dq->lock, flags);
+        }
 
         return rc;
 }
@@ -524,10 +599,12 @@ static struct hisi_sas_device *hisi_sas_alloc_dev(struct domain_device *device)
         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
         struct hisi_sas_device *sas_dev = NULL;
         unsigned long flags;
+        int last = hisi_hba->last_dev_id;
+        int first = (hisi_hba->last_dev_id + 1) % HISI_SAS_MAX_DEVICES;
         int i;
 
         spin_lock_irqsave(&hisi_hba->lock, flags);
-        for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
+        for (i = first; i != last; i %= HISI_SAS_MAX_DEVICES) {
                 if (hisi_hba->devices[i].dev_type == SAS_PHY_UNUSED) {
                         int queue = i % hisi_hba->queue_count;
                         struct hisi_sas_dq *dq = &hisi_hba->dq[queue];
@@ -542,18 +619,57 @@ static struct hisi_sas_device *hisi_sas_alloc_dev(struct domain_device *device)
                         INIT_LIST_HEAD(&hisi_hba->devices[i].list);
                         break;
                 }
+                i++;
         }
+        hisi_hba->last_dev_id = i;
         spin_unlock_irqrestore(&hisi_hba->lock, flags);
 
         return sas_dev;
 }
 
+#define HISI_SAS_SRST_ATA_DISK_CNT 3
+static int hisi_sas_init_device(struct domain_device *device)
+{
+        int rc = TMF_RESP_FUNC_COMPLETE;
+        struct scsi_lun lun;
+        struct hisi_sas_tmf_task tmf_task;
+        int retry = HISI_SAS_SRST_ATA_DISK_CNT;
+        struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
+
+        switch (device->dev_type) {
+        case SAS_END_DEVICE:
+                int_to_scsilun(0, &lun);
+
+                tmf_task.tmf = TMF_CLEAR_TASK_SET;
+                rc = hisi_sas_debug_issue_ssp_tmf(device, lun.scsi_lun,
+                                                  &tmf_task);
+                if (rc == TMF_RESP_FUNC_COMPLETE)
+                        hisi_sas_release_task(hisi_hba, device);
+                break;
+        case SAS_SATA_DEV:
+        case SAS_SATA_PM:
+        case SAS_SATA_PM_PORT:
+        case SAS_SATA_PENDING:
+                while (retry-- > 0) {
+                        rc = hisi_sas_softreset_ata_disk(device);
+                        if (!rc)
+                                break;
+                }
+                break;
+        default:
+                break;
+        }
+
+        return rc;
+}
+
 static int hisi_sas_dev_found(struct domain_device *device)
 {
         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
         struct domain_device *parent_dev = device->parent;
         struct hisi_sas_device *sas_dev;
         struct device *dev = hisi_hba->dev;
+        int rc;
 
         if (hisi_hba->hw->alloc_dev)
                 sas_dev = hisi_hba->hw->alloc_dev(device);
@@ -576,10 +692,8 @@ static int hisi_sas_dev_found(struct domain_device *device)
                 for (phy_no = 0; phy_no < phy_num; phy_no++) {
                         phy = &parent_dev->ex_dev.ex_phy[phy_no];
                         if (SAS_ADDR(phy->attached_sas_addr) ==
-                                SAS_ADDR(device->sas_addr)) {
-                                sas_dev->attached_phy = phy_no;
+                                SAS_ADDR(device->sas_addr))
                                 break;
-                        }
                 }
 
                 if (phy_no == phy_num) {
@@ -587,17 +701,25 @@ static int hisi_sas_dev_found(struct domain_device *device)
                                  "dev:%016llx at ex:%016llx\n",
                                  SAS_ADDR(device->sas_addr),
                                  SAS_ADDR(parent_dev->sas_addr));
-                        return -EINVAL;
+                        rc = -EINVAL;
+                        goto err_out;
                 }
         }
 
         dev_info(dev, "dev[%d:%x] found\n",
                 sas_dev->device_id, sas_dev->dev_type);
 
+        rc = hisi_sas_init_device(device);
+        if (rc)
+                goto err_out;
         return 0;
+
+err_out:
+        hisi_sas_dev_gone(device);
+        return rc;
 }
 
-static int hisi_sas_slave_configure(struct scsi_device *sdev)
+int hisi_sas_slave_configure(struct scsi_device *sdev)
 {
         struct domain_device *dev = sdev_to_domain_dev(sdev);
         int ret = sas_slave_configure(sdev);
@@ -609,15 +731,17 @@ static int hisi_sas_slave_configure(struct scsi_device *sdev)
 
         return 0;
 }
+EXPORT_SYMBOL_GPL(hisi_sas_slave_configure);
 
-static void hisi_sas_scan_start(struct Scsi_Host *shost)
+void hisi_sas_scan_start(struct Scsi_Host *shost)
 {
         struct hisi_hba *hisi_hba = shost_priv(shost);
 
         hisi_hba->hw->phys_init(hisi_hba);
 }
+EXPORT_SYMBOL_GPL(hisi_sas_scan_start);
 
-static int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time)
+int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time)
 {
         struct hisi_hba *hisi_hba = shost_priv(shost);
         struct sas_ha_struct *sha = &hisi_hba->sha;
@@ -629,6 +753,7 @@ static int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time)
         sas_drain_work(sha);
         return 1;
 }
+EXPORT_SYMBOL_GPL(hisi_sas_scan_finished);
 
 static void hisi_sas_phyup_work(struct work_struct *work)
 {
@@ -803,6 +928,33 @@ static int hisi_sas_queue_command(struct sas_task *task, gfp_t gfp_flags)
         return hisi_sas_task_exec(task, gfp_flags, 0, NULL);
 }
 
+static void hisi_sas_phy_set_linkrate(struct hisi_hba *hisi_hba, int phy_no,
+                        struct sas_phy_linkrates *r)
+{
+        struct sas_phy_linkrates _r;
+
+        struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
+        struct asd_sas_phy *sas_phy = &phy->sas_phy;
+        enum sas_linkrate min, max;
+
+        if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
+                max = sas_phy->phy->maximum_linkrate;
+                min = r->minimum_linkrate;
+        } else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
+                max = r->maximum_linkrate;
+                min = sas_phy->phy->minimum_linkrate;
+        } else
+                return;
+
+        _r.maximum_linkrate = max;
+        _r.minimum_linkrate = min;
+
+        hisi_hba->hw->phy_disable(hisi_hba, phy_no);
+        msleep(100);
+        hisi_hba->hw->phy_set_linkrate(hisi_hba, phy_no, &_r);
+        hisi_hba->hw->phy_start(hisi_hba, phy_no);
+}
+
 static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
                                 void *funcdata)
 {
@@ -826,7 +978,7 @@ static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
                 break;
 
         case PHY_FUNC_SET_LINK_RATE:
-                hisi_hba->hw->phy_set_linkrate(hisi_hba, phy_no, funcdata);
+                hisi_sas_phy_set_linkrate(hisi_hba, phy_no, funcdata);
                 break;
         case PHY_FUNC_GET_EVENTS:
                 if (hisi_hba->hw->get_events) {
@@ -990,7 +1142,6 @@ static int hisi_sas_softreset_ata_disk(struct domain_device *device)
         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
         struct device *dev = hisi_hba->dev;
         int s = sizeof(struct host_to_dev_fis);
-        unsigned long flags;
 
         ata_for_each_link(link, ap, EDGE) {
                 int pmp = sata_srst_pmp(link);
@@ -1015,11 +1166,8 @@ static int hisi_sas_softreset_ata_disk(struct domain_device *device)
                 dev_err(dev, "ata disk reset failed\n");
         }
 
-        if (rc == TMF_RESP_FUNC_COMPLETE) {
-                spin_lock_irqsave(&hisi_hba->lock, flags);
+        if (rc == TMF_RESP_FUNC_COMPLETE)
                 hisi_sas_release_task(hisi_hba, device);
-                spin_unlock_irqrestore(&hisi_hba->lock, flags);
-        }
 
         return rc;
 }
@@ -1111,12 +1259,103 @@ static void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 old_state,
         }
 }
 
+static void hisi_sas_reset_init_all_devices(struct hisi_hba *hisi_hba)
+{
+        struct hisi_sas_device *sas_dev;
+        struct domain_device *device;
+        int i;
+
+        for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
+                sas_dev = &hisi_hba->devices[i];
+                device = sas_dev->sas_device;
+
+                if ((sas_dev->dev_type == SAS_PHY_UNUSED) || !device)
+                        continue;
+
+                hisi_sas_init_device(device);
+        }
+}
+
+static void hisi_sas_send_ata_reset_each_phy(struct hisi_hba *hisi_hba,
+                                             struct asd_sas_port *sas_port,
+                                             struct domain_device *device)
+{
+        struct hisi_sas_tmf_task tmf_task = { .force_phy = 1 };
+        struct ata_port *ap = device->sata_dev.ap;
+        struct device *dev = hisi_hba->dev;
+        int s = sizeof(struct host_to_dev_fis);
+        int rc = TMF_RESP_FUNC_FAILED;
+        struct asd_sas_phy *sas_phy;
+        struct ata_link *link;
+        u8 fis[20] = {0};
+        u32 state;
+
+        state = hisi_hba->hw->get_phys_state(hisi_hba);
+        list_for_each_entry(sas_phy, &sas_port->phy_list, port_phy_el) {
+                if (!(state & BIT(sas_phy->id)))
+                        continue;
+
+                ata_for_each_link(link, ap, EDGE) {
+                        int pmp = sata_srst_pmp(link);
+
+                        tmf_task.phy_id = sas_phy->id;
+                        hisi_sas_fill_ata_reset_cmd(link->device, 1, pmp, fis);
+                        rc = hisi_sas_exec_internal_tmf_task(device, fis, s,
+                                                             &tmf_task);
+                        if (rc != TMF_RESP_FUNC_COMPLETE) {
+                                dev_err(dev, "phy%d ata reset failed rc=%d\n",
+                                        sas_phy->id, rc);
+                                break;
+                        }
+                }
+        }
+}
+
+static void hisi_sas_terminate_stp_reject(struct hisi_hba *hisi_hba)
+{
+        struct device *dev = hisi_hba->dev;
+        int port_no, rc, i;
+
+        for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
+                struct hisi_sas_device *sas_dev = &hisi_hba->devices[i];
+                struct domain_device *device = sas_dev->sas_device;
+
+                if ((sas_dev->dev_type == SAS_PHY_UNUSED) || !device)
+                        continue;
+
+                rc = hisi_sas_internal_task_abort(hisi_hba, device,
+                                                  HISI_SAS_INT_ABT_DEV, 0);
+                if (rc < 0)
+                        dev_err(dev, "STP reject: abort dev failed %d\n", rc);
+        }
+
+        for (port_no = 0; port_no < hisi_hba->n_phy; port_no++) {
+                struct hisi_sas_port *port = &hisi_hba->port[port_no];
+                struct asd_sas_port *sas_port = &port->sas_port;
+                struct domain_device *port_dev = sas_port->port_dev;
+                struct domain_device *device;
+
+                if (!port_dev || !DEV_IS_EXPANDER(port_dev->dev_type))
+                        continue;
+
+                /* Try to find a SATA device */
+                list_for_each_entry(device, &sas_port->dev_list,
+                                    dev_list_node) {
+                        if (dev_is_sata(device)) {
+                                hisi_sas_send_ata_reset_each_phy(hisi_hba,
+                                                                 sas_port,
+                                                                 device);
+                                break;
+                        }
+                }
+        }
+}
+
 static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba)
 {
         struct device *dev = hisi_hba->dev;
         struct Scsi_Host *shost = hisi_hba->shost;
         u32 old_state, state;
-        unsigned long flags;
         int rc;
 
         if (!hisi_hba->hw->soft_reset)
@@ -1129,6 +1368,11 @@ static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba)
         old_state = hisi_hba->hw->get_phys_state(hisi_hba);
 
         scsi_block_requests(shost);
+        hisi_hba->hw->wait_cmds_complete_timeout(hisi_hba, 100, 5000);
+
+        if (timer_pending(&hisi_hba->timer))
+                del_timer_sync(&hisi_hba->timer);
+
         set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
         rc = hisi_hba->hw->soft_reset(hisi_hba);
         if (rc) {
@@ -1137,9 +1381,6 @@ static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba)
                 scsi_unblock_requests(shost);
                 goto out;
         }
-        spin_lock_irqsave(&hisi_hba->lock, flags);
-        hisi_sas_release_tasks(hisi_hba);
-        spin_unlock_irqrestore(&hisi_hba->lock, flags);
 
         clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
 
@@ -1147,6 +1388,10 @@ static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba)
         hisi_hba->hw->phys_init(hisi_hba);
         msleep(1000);
         hisi_sas_refresh_port_id(hisi_hba);
+
+        if (hisi_hba->reject_stp_links_msk)
+                hisi_sas_terminate_stp_reject(hisi_hba);
+        hisi_sas_reset_init_all_devices(hisi_hba);
         scsi_unblock_requests(shost);
 
         state = hisi_hba->hw->get_phys_state(hisi_hba);
@@ -1165,20 +1410,25 @@ static int hisi_sas_abort_task(struct sas_task *task)
         struct hisi_sas_tmf_task tmf_task;
         struct domain_device *device = task->dev;
         struct hisi_sas_device *sas_dev = device->lldd_dev;
-        struct hisi_hba *hisi_hba = dev_to_hisi_hba(task->dev);
-        struct device *dev = hisi_hba->dev;
+        struct hisi_hba *hisi_hba;
+        struct device *dev;
         int rc = TMF_RESP_FUNC_FAILED;
         unsigned long flags;
 
-        if (!sas_dev) {
-                dev_warn(dev, "Device has been removed\n");
+        if (!sas_dev)
                 return TMF_RESP_FUNC_FAILED;
-        }
 
+        hisi_hba = dev_to_hisi_hba(task->dev);
+        dev = hisi_hba->dev;
+
+        spin_lock_irqsave(&task->task_state_lock, flags);
         if (task->task_state_flags & SAS_TASK_STATE_DONE) {
+                spin_unlock_irqrestore(&task->task_state_lock, flags);
                 rc = TMF_RESP_FUNC_COMPLETE;
                 goto out;
         }
+        task->task_state_flags |= SAS_TASK_STATE_ABORTED;
+        spin_unlock_irqrestore(&task->task_state_lock, flags);
 
         sas_dev->dev_status = HISI_SAS_DEV_EH;
         if (task->lldd_task && task->task_proto & SAS_PROTOCOL_SSP) {
@@ -1209,11 +1459,8 @@ static int hisi_sas_abort_task(struct sas_task *task)
                  * will have already been completed
                  */
                 if (rc == TMF_RESP_FUNC_COMPLETE && rc2 != TMF_RESP_FUNC_SUCC) {
-                        if (task->lldd_task) {
-                                spin_lock_irqsave(&hisi_hba->lock, flags);
+                        if (task->lldd_task)
                                 hisi_sas_do_release_task(hisi_hba, task, slot);
-                                spin_unlock_irqrestore(&hisi_hba->lock, flags);
-                        }
                 }
         } else if (task->task_proto & SAS_PROTOCOL_SATA ||
                 task->task_proto & SAS_PROTOCOL_STP) {
@@ -1235,11 +1482,8 @@ static int hisi_sas_abort_task(struct sas_task *task)
                 rc = hisi_sas_internal_task_abort(hisi_hba, device,
                              HISI_SAS_INT_ABT_CMD, tag);
                 if (((rc < 0) || (rc == TMF_RESP_FUNC_FAILED)) &&
-                                        task->lldd_task) {
-                        spin_lock_irqsave(&hisi_hba->lock, flags);
+                                        task->lldd_task)
                         hisi_sas_do_release_task(hisi_hba, task, slot);
-                        spin_unlock_irqrestore(&hisi_hba->lock, flags);
-                }
         }
 
 out:
@@ -1254,7 +1498,6 @@ static int hisi_sas_abort_task_set(struct domain_device *device, u8 *lun)
         struct device *dev = hisi_hba->dev;
         struct hisi_sas_tmf_task tmf_task;
         int rc = TMF_RESP_FUNC_FAILED;
-        unsigned long flags;
 
         rc = hisi_sas_internal_task_abort(hisi_hba, device,
                                         HISI_SAS_INT_ABT_DEV, 0);
@@ -1267,11 +1510,8 @@ static int hisi_sas_abort_task_set(struct domain_device *device, u8 *lun)
         tmf_task.tmf = TMF_ABORT_TASK_SET;
         rc = hisi_sas_debug_issue_ssp_tmf(device, lun, &tmf_task);
 
-        if (rc == TMF_RESP_FUNC_COMPLETE) {
-                spin_lock_irqsave(&hisi_hba->lock, flags);
+        if (rc == TMF_RESP_FUNC_COMPLETE)
                 hisi_sas_release_task(hisi_hba, device);
-                spin_unlock_irqrestore(&hisi_hba->lock, flags);
-        }
 
         return rc;
 }
@@ -1289,12 +1529,39 @@ static int hisi_sas_clear_aca(struct domain_device *device, u8 *lun)
 
 static int hisi_sas_debug_I_T_nexus_reset(struct domain_device *device)
 {
-        struct sas_phy *phy = sas_get_local_phy(device);
+        struct sas_phy *local_phy = sas_get_local_phy(device);
         int rc, reset_type = (device->dev_type == SAS_SATA_DEV ||
                         (device->tproto & SAS_PROTOCOL_STP)) ? 0 : 1;
-        rc = sas_phy_reset(phy, reset_type);
-        sas_put_local_phy(phy);
-        msleep(2000);
+        struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
+        struct sas_ha_struct *sas_ha = &hisi_hba->sha;
+        struct asd_sas_phy *sas_phy = sas_ha->sas_phy[local_phy->number];
+        struct hisi_sas_phy *phy = container_of(sas_phy,
+                        struct hisi_sas_phy, sas_phy);
+        DECLARE_COMPLETION_ONSTACK(phyreset);
+
+        if (scsi_is_sas_phy_local(local_phy)) {
+                phy->in_reset = 1;
+                phy->reset_completion = &phyreset;
+        }
+
+        rc = sas_phy_reset(local_phy, reset_type);
+        sas_put_local_phy(local_phy);
+
+        if (scsi_is_sas_phy_local(local_phy)) {
+                int ret = wait_for_completion_timeout(&phyreset, 2 * HZ);
+                unsigned long flags;
+
+                spin_lock_irqsave(&phy->lock, flags);
+                phy->reset_completion = NULL;
+                phy->in_reset = 0;
+                spin_unlock_irqrestore(&phy->lock, flags);
+
+                /* report PHY down if timed out */
+                if (!ret)
+                        hisi_sas_phy_down(hisi_hba, sas_phy->id, 0);
+        } else
+                msleep(2000);
+
         return rc;
 }
 
@@ -1304,7 +1571,6 @@ static int hisi_sas_I_T_nexus_reset(struct domain_device *device)
         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
         struct device *dev = hisi_hba->dev;
         int rc = TMF_RESP_FUNC_FAILED;
-        unsigned long flags;
 
         if (sas_dev->dev_status != HISI_SAS_DEV_EH)
                 return TMF_RESP_FUNC_FAILED;
@@ -1320,11 +1586,9 @@ static int hisi_sas_I_T_nexus_reset(struct domain_device *device)
 
         rc = hisi_sas_debug_I_T_nexus_reset(device);
 
-        if ((rc == TMF_RESP_FUNC_COMPLETE) || (rc == -ENODEV)) {
-                spin_lock_irqsave(&hisi_hba->lock, flags);
+        if ((rc == TMF_RESP_FUNC_COMPLETE) || (rc == -ENODEV))
                 hisi_sas_release_task(hisi_hba, device);
-                spin_unlock_irqrestore(&hisi_hba->lock, flags);
-        }
+
         return rc;
 }
 
@@ -1333,7 +1597,6 @@ static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun)
         struct hisi_sas_device *sas_dev = device->lldd_dev;
         struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
         struct device *dev = hisi_hba->dev;
-        unsigned long flags;
         int rc = TMF_RESP_FUNC_FAILED;
 
         sas_dev->dev_status = HISI_SAS_DEV_EH;
@@ -1353,11 +1616,8 @@ static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun)
 
                 rc = sas_phy_reset(phy, 1);
 
-                if (rc == 0) {
-                        spin_lock_irqsave(&hisi_hba->lock, flags);
+                if (rc == 0)
                         hisi_sas_release_task(hisi_hba, device);
-                        spin_unlock_irqrestore(&hisi_hba->lock, flags);
-                }
                 sas_put_local_phy(phy);
         } else {
                 struct hisi_sas_tmf_task tmf_task = { .tmf =  TMF_LU_RESET };
@@ -1371,11 +1631,8 @@ static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun)
                 hisi_sas_dereg_device(hisi_hba, device);
 
                 rc = hisi_sas_debug_issue_ssp_tmf(device, lun, &tmf_task);
-                if (rc == TMF_RESP_FUNC_COMPLETE) {
-                        spin_lock_irqsave(&hisi_hba->lock, flags);
+                if (rc == TMF_RESP_FUNC_COMPLETE)
                         hisi_sas_release_task(hisi_hba, device);
-                        spin_unlock_irqrestore(&hisi_hba->lock, flags);
-                }
         }
 out:
         if (rc != TMF_RESP_FUNC_COMPLETE)
@@ -1445,7 +1702,8 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id,
         struct hisi_sas_cmd_hdr *cmd_hdr_base;
         struct hisi_sas_dq *dq = sas_dev->dq;
         int dlvry_queue_slot, dlvry_queue, n_elem = 0, rc, slot_idx;
-        unsigned long flags, flags_dq;
+        unsigned long flags, flags_dq = 0;
+        int wr_q_index;
 
         if (unlikely(test_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags)))
                 return -EINVAL;
@@ -1464,16 +1722,28 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id,
         }
         spin_unlock_irqrestore(&hisi_hba->lock, flags);
 
-        spin_lock_irqsave(&dq->lock, flags_dq);
-        rc = hisi_hba->hw->get_free_slot(hisi_hba, dq);
-        if (rc)
+        slot = &hisi_hba->slot_info[slot_idx];
+        memset(slot, 0, sizeof(struct hisi_sas_slot));
+
+        slot->buf = dma_pool_alloc(hisi_hba->buffer_pool,
+                        GFP_ATOMIC, &slot->buf_dma);
+        if (!slot->buf) {
+                rc = -ENOMEM;
                 goto err_out_tag;
+        }
 
-        dlvry_queue = dq->id;
-        dlvry_queue_slot = dq->wr_point;
+        spin_lock_irqsave(&dq->lock, flags_dq);
+        wr_q_index = hisi_hba->hw->get_free_slot(hisi_hba, dq);
+        if (wr_q_index < 0) {
+                spin_unlock_irqrestore(&dq->lock, flags_dq);
+                rc = -EAGAIN;
+                goto err_out_buf;
+        }
+        list_add_tail(&slot->delivery, &dq->list);
+        spin_unlock_irqrestore(&dq->lock, flags_dq);
 
-        slot = &hisi_hba->slot_info[slot_idx];
-        memset(slot, 0, sizeof(struct hisi_sas_slot));
+        dlvry_queue = dq->id;
+        dlvry_queue_slot = wr_q_index;
 
         slot->idx = slot_idx;
         slot->n_elem = n_elem;
@@ -1483,47 +1753,36 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id,
         slot->cmd_hdr = &cmd_hdr_base[dlvry_queue_slot];
         slot->task = task;
         slot->port = port;
+        slot->is_internal = true;
         task->lldd_task = slot;
 
-        slot->buf = dma_pool_alloc(hisi_hba->buffer_pool,
-                        GFP_ATOMIC, &slot->buf_dma);
-        if (!slot->buf) {
-                rc = -ENOMEM;
-                goto err_out_tag;
-        }
-
         memset(slot->cmd_hdr, 0, sizeof(struct hisi_sas_cmd_hdr));
         memset(hisi_sas_cmd_hdr_addr_mem(slot), 0, HISI_SAS_COMMAND_TABLE_SZ);
         memset(hisi_sas_status_buf_addr_mem(slot), 0, HISI_SAS_STATUS_BUF_SZ);
 
-        rc = hisi_sas_task_prep_abort(hisi_hba, slot, device_id,
+        hisi_sas_task_prep_abort(hisi_hba, slot, device_id,
                                       abort_flag, task_tag);
-        if (rc)
-                goto err_out_buf;
 
-        spin_lock_irqsave(&hisi_hba->lock, flags);
-        list_add_tail(&slot->entry, &sas_dev->list);
-        spin_unlock_irqrestore(&hisi_hba->lock, flags);
         spin_lock_irqsave(&task->task_state_lock, flags);
         task->task_state_flags |= SAS_TASK_AT_INITIATOR;
         spin_unlock_irqrestore(&task->task_state_lock, flags);
 
-        dq->slot_prep = slot;
-
+        slot->ready = 1;
         /* send abort command to the chip */
+        spin_lock_irqsave(&dq->lock, flags);
+        list_add_tail(&slot->entry, &sas_dev->list);
         hisi_hba->hw->start_delivery(dq);
-        spin_unlock_irqrestore(&dq->lock, flags_dq);
+        spin_unlock_irqrestore(&dq->lock, flags);
 
         return 0;
 
 err_out_buf:
         dma_pool_free(hisi_hba->buffer_pool, slot->buf,
-                slot->buf_dma);
+                      slot->buf_dma);
 err_out_tag:
         spin_lock_irqsave(&hisi_hba->lock, flags);
         hisi_sas_slot_index_free(hisi_hba, slot_idx);
         spin_unlock_irqrestore(&hisi_hba->lock, flags);
-        spin_unlock_irqrestore(&dq->lock, flags_dq);
 err_out:
         dev_err(dev, "internal abort task prep: failed[%d]!\n", rc);
 
@@ -1651,6 +1910,7 @@ void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy)
         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
         struct asd_sas_phy *sas_phy = &phy->sas_phy;
         struct sas_ha_struct *sas_ha = &hisi_hba->sha;
+        struct device *dev = hisi_hba->dev;
 
         if (rdy) {
                 /* Phy down but ready */
@@ -1659,6 +1919,10 @@ void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy)
         } else {
                 struct hisi_sas_port *port  = phy->port;
 
+                if (phy->in_reset) {
+                        dev_info(dev, "ignore flutter phy%d down\n", phy_no);
+                        return;
+                }
                 /* Phy down and not ready */
                 sas_ha->notify_phy_event(sas_phy, PHYE_LOSS_OF_SIGNAL);
                 sas_phy_disconnected(sas_phy);
@@ -1693,34 +1957,11 @@ EXPORT_SYMBOL_GPL(hisi_sas_kill_tasklets);
 struct scsi_transport_template *hisi_sas_stt;
 EXPORT_SYMBOL_GPL(hisi_sas_stt);
 
-static struct device_attribute *host_attrs[] = {
+struct device_attribute *host_attrs[] = {
         &dev_attr_phy_event_threshold,
         NULL,
 };
-
-static struct scsi_host_template _hisi_sas_sht = {
-        .module                 = THIS_MODULE,
-        .name                   = DRV_NAME,
-        .queuecommand           = sas_queuecommand,
-        .target_alloc           = sas_target_alloc,
-        .slave_configure        = hisi_sas_slave_configure,
-        .scan_finished          = hisi_sas_scan_finished,
-        .scan_start             = hisi_sas_scan_start,
-        .change_queue_depth     = sas_change_queue_depth,
-        .bios_param             = sas_bios_param,
-        .can_queue              = 1,
-        .this_id                = -1,
-        .sg_tablesize           = SG_ALL,
-        .max_sectors            = SCSI_DEFAULT_MAX_SECTORS,
-        .use_clustering         = ENABLE_CLUSTERING,
-        .eh_device_reset_handler = sas_eh_device_reset_handler,
-        .eh_target_reset_handler = sas_eh_target_reset_handler,
-        .target_destroy         = sas_target_destroy,
-        .ioctl                  = sas_ioctl,
-        .shost_attrs            = host_attrs,
-};
-struct scsi_host_template *hisi_sas_sht = &_hisi_sas_sht;
-EXPORT_SYMBOL_GPL(hisi_sas_sht);
+EXPORT_SYMBOL_GPL(host_attrs);
 
 static struct sas_domain_function_template hisi_sas_transport_ops = {
         .lldd_dev_found         = hisi_sas_dev_found,
@@ -1798,6 +2039,7 @@ int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost)
 
                 /* Delivery queue structure */
                 spin_lock_init(&dq->lock);
+                INIT_LIST_HEAD(&dq->list);
                 dq->id = i;
                 dq->hisi_hba = hisi_hba;
 
@@ -1822,13 +2064,11 @@ int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost)
                 goto err_out;
 
         s = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_itct);
-        hisi_hba->itct = dma_alloc_coherent(dev, s, &hisi_hba->itct_dma,
+        hisi_hba->itct = dma_zalloc_coherent(dev, s, &hisi_hba->itct_dma,
                                             GFP_KERNEL);
         if (!hisi_hba->itct)
                 goto err_out;
 
-        memset(hisi_hba->itct, 0, s);
-
         hisi_hba->slot_info = devm_kcalloc(dev, max_command_entries,
                                            sizeof(struct hisi_sas_slot),
                                            GFP_KERNEL);
@@ -2031,7 +2271,7 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev,
         struct hisi_hba *hisi_hba;
         struct device *dev = &pdev->dev;
 
-        shost = scsi_host_alloc(hisi_sas_sht, sizeof(*hisi_hba));
+        shost = scsi_host_alloc(hw->sht, sizeof(*hisi_hba));
         if (!shost) {
                 dev_err(dev, "scsi host alloc failed\n");
                 return NULL;
@@ -2080,19 +2320,8 @@ err_out:
         return NULL;
 }
 
-void hisi_sas_init_add(struct hisi_hba *hisi_hba)
-{
-        int i;
-
-        for (i = 0; i < hisi_hba->n_phy; i++)
-                memcpy(&hisi_hba->phy[i].dev_sas_addr,
-                       hisi_hba->sas_addr,
-                       SAS_ADDR_SIZE);
-}
-EXPORT_SYMBOL_GPL(hisi_sas_init_add);
-
 int hisi_sas_probe(struct platform_device *pdev,
-                         const struct hisi_sas_hw *hw)
+                   const struct hisi_sas_hw *hw)
 {
         struct Scsi_Host *shost;
         struct hisi_hba *hisi_hba;
@@ -2144,8 +2373,6 @@ int hisi_sas_probe(struct platform_device *pdev,
                 sha->sas_port[i] = &hisi_hba->port[i].sas_port;
         }
 
-        hisi_sas_init_add(hisi_hba);
-
         rc = scsi_add_host(shost, &pdev->dev);
         if (rc)
                 goto err_out_ha;
@@ -2177,6 +2404,9 @@ int hisi_sas_remove(struct platform_device *pdev)
         struct hisi_hba *hisi_hba = sha->lldd_ha;
         struct Scsi_Host *shost = sha->core.shost;
 
+        if (timer_pending(&hisi_hba->timer))
+                del_timer(&hisi_hba->timer);
+
         sas_unregister_ha(sha);
         sas_remove_host(sha->core.shost);
 
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
index 84a0ccc4daf5..89ab18c1959c 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
@@ -855,39 +855,12 @@ static enum sas_linkrate phy_get_max_linkrate_v1_hw(void)
 static void phy_set_linkrate_v1_hw(struct hisi_hba *hisi_hba, int phy_no,
                 struct sas_phy_linkrates *r)
 {
-        u32 prog_phy_link_rate =
-                hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
-        struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
-        struct asd_sas_phy *sas_phy = &phy->sas_phy;
-        int i;
-        enum sas_linkrate min, max;
-        u32 rate_mask = 0;
-
-        if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-                max = sas_phy->phy->maximum_linkrate;
-                min = r->minimum_linkrate;
-        } else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-                max = r->maximum_linkrate;
-                min = sas_phy->phy->minimum_linkrate;
-        } else
-                return;
-
-        sas_phy->phy->maximum_linkrate = max;
-        sas_phy->phy->minimum_linkrate = min;
-
-        max -= SAS_LINK_RATE_1_5_GBPS;
+        enum sas_linkrate max = r->maximum_linkrate;
+        u32 prog_phy_link_rate = 0x800;
 
-        for (i = 0; i <= max; i++)
-                rate_mask |= 1 << (i * 2);
-
-        prog_phy_link_rate &= ~0xff;
-        prog_phy_link_rate |= rate_mask;
-
-        disable_phy_v1_hw(hisi_hba, phy_no);
-        msleep(100);
+        prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
         hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
-                        prog_phy_link_rate);
-        start_phy_v1_hw(hisi_hba, phy_no);
+                             prog_phy_link_rate);
 }
 
 static int get_wideport_bitmap_v1_hw(struct hisi_hba *hisi_hba, int port_id)
@@ -921,37 +894,45 @@ get_free_slot_v1_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
                 return -EAGAIN;
         }
 
-        return 0;
+        dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS;
+
+        return w;
 }
 
+/* DQ lock must be taken here */
 static void start_delivery_v1_hw(struct hisi_sas_dq *dq)
 {
         struct hisi_hba *hisi_hba = dq->hisi_hba;
-        int dlvry_queue = dq->slot_prep->dlvry_queue;
-        int dlvry_queue_slot = dq->slot_prep->dlvry_queue_slot;
+        struct hisi_sas_slot *s, *s1;
+        struct list_head *dq_list;
+        int dlvry_queue = dq->id;
+        int wp, count = 0;
+
+        dq_list = &dq->list;
+        list_for_each_entry_safe(s, s1, &dq->list, delivery) {
+                if (!s->ready)
+                        break;
+                count++;
+                wp = (s->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
+                list_del(&s->delivery);
+        }
 
-        dq->wr_point = ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS;
-        hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14),
-                         dq->wr_point);
+        if (!count)
+                return;
+
+        hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), wp);
 }
 
-static int prep_prd_sge_v1_hw(struct hisi_hba *hisi_hba,
+static void prep_prd_sge_v1_hw(struct hisi_hba *hisi_hba,
                               struct hisi_sas_slot *slot,
                               struct hisi_sas_cmd_hdr *hdr,
                               struct scatterlist *scatter,
                               int n_elem)
 {
         struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
-        struct device *dev = hisi_hba->dev;
         struct scatterlist *sg;
         int i;
 
-        if (n_elem > HISI_SAS_SGE_PAGE_CNT) {
-                dev_err(dev, "prd err: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT",
-                        n_elem);
-                return -EINVAL;
-        }
-
         for_each_sg(scatter, sg, n_elem, i) {
                 struct hisi_sas_sge *entry = &sge_page->sge[i];
 
@@ -964,48 +945,25 @@ static int prep_prd_sge_v1_hw(struct hisi_hba *hisi_hba,
         hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
 
         hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
-
-        return 0;
 }
 
-static int prep_smp_v1_hw(struct hisi_hba *hisi_hba,
+static void prep_smp_v1_hw(struct hisi_hba *hisi_hba,
                           struct hisi_sas_slot *slot)
 {
         struct sas_task *task = slot->task;
         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
         struct domain_device *device = task->dev;
-        struct device *dev = hisi_hba->dev;
         struct hisi_sas_port *port = slot->port;
-        struct scatterlist *sg_req, *sg_resp;
+        struct scatterlist *sg_req;
         struct hisi_sas_device *sas_dev = device->lldd_dev;
         dma_addr_t req_dma_addr;
-        unsigned int req_len, resp_len;
-        int elem, rc;
+        unsigned int req_len;
 
-        /*
-        * DMA-map SMP request, response buffers
-        */
         /* req */
         sg_req = &task->smp_task.smp_req;
-        elem = dma_map_sg(dev, sg_req, 1, DMA_TO_DEVICE);
-        if (!elem)
-                return -ENOMEM;
         req_len = sg_dma_len(sg_req);
         req_dma_addr = sg_dma_address(sg_req);
 
-        /* resp */
-        sg_resp = &task->smp_task.smp_resp;
-        elem = dma_map_sg(dev, sg_resp, 1, DMA_FROM_DEVICE);
-        if (!elem) {
-                rc = -ENOMEM;
-                goto err_out_req;
-        }
-        resp_len = sg_dma_len(sg_resp);
-        if ((req_len & 0x3) || (resp_len & 0x3)) {
-                rc = -EINVAL;
-                goto err_out_resp;
-        }
-
         /* create header */
         /* dw0 */
         hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
@@ -1025,21 +983,10 @@ static int prep_smp_v1_hw(struct hisi_hba *hisi_hba,
 
         hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
-
-        return 0;
-
-err_out_resp:
-        dma_unmap_sg(dev, &slot->task->smp_task.smp_resp, 1,
-                     DMA_FROM_DEVICE);
-err_out_req:
-        dma_unmap_sg(dev, &slot->task->smp_task.smp_req, 1,
-                     DMA_TO_DEVICE);
-        return rc;
 }
 
-static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
-                          struct hisi_sas_slot *slot, int is_tmf,
-                          struct hisi_sas_tmf_task *tmf)
+static void prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
+                          struct hisi_sas_slot *slot)
 {
         struct sas_task *task = slot->task;
         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
@@ -1048,7 +995,8 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
         struct hisi_sas_port *port = slot->port;
         struct sas_ssp_task *ssp_task = &task->ssp_task;
         struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
-        int has_data = 0, rc, priority = is_tmf;
+        struct hisi_sas_tmf_task *tmf = slot->tmf;
+        int has_data = 0, priority = !!tmf;
         u8 *buf_cmd, fburst = 0;
         u32 dw1, dw2;
 
@@ -1062,7 +1010,7 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
 
         dw1 = 1 << CMD_HDR_VERIFY_DTL_OFF;
 
-        if (is_tmf) {
+        if (tmf) {
                 dw1 |= 3 << CMD_HDR_SSP_FRAME_TYPE_OFF;
         } else {
                 switch (scsi_cmnd->sc_data_direction) {
@@ -1083,7 +1031,7 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
         dw1 |= sas_dev->device_id << CMD_HDR_DEVICE_ID_OFF;
         hdr->dw1 = cpu_to_le32(dw1);
 
-        if (is_tmf) {
+        if (tmf) {
                 dw2 = ((sizeof(struct ssp_tmf_iu) +
                         sizeof(struct ssp_frame_hdr)+3)/4) <<
                         CMD_HDR_CFL_OFF;
@@ -1097,12 +1045,9 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
 
         hdr->transfer_tags = cpu_to_le32(slot->idx << CMD_HDR_IPTT_OFF);
 
-        if (has_data) {
-                rc = prep_prd_sge_v1_hw(hisi_hba, slot, hdr, task->scatter,
+        if (has_data)
+                prep_prd_sge_v1_hw(hisi_hba, slot, hdr, task->scatter,
                                         slot->n_elem);
-                if (rc)
-                        return rc;
-        }
 
         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
@@ -1117,7 +1062,7 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
         hdr->dw2 = cpu_to_le32(dw2);
 
         memcpy(buf_cmd, &task->ssp_task.LUN, 8);
-        if (!is_tmf) {
+        if (!tmf) {
                 buf_cmd[9] = fburst | task->ssp_task.task_attr |
                                 (task->ssp_task.task_prio << 3);
                 memcpy(buf_cmd + 12, task->ssp_task.cmd->cmnd,
@@ -1136,8 +1081,6 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
                         break;
                 }
         }
-
-        return 0;
 }
 
 /* by default, task resp is complete */
@@ -1430,6 +1373,7 @@ static irqreturn_t int_phyup_v1_hw(int irq_no, void *p)
         u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
         struct sas_identify_frame *id = (struct sas_identify_frame *)frame_rcvd;
         irqreturn_t res = IRQ_HANDLED;
+        unsigned long flags;
 
         irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2);
         if (!(irq_value & CHL_INT2_SL_PHY_ENA_MSK)) {
@@ -1483,6 +1427,13 @@ static irqreturn_t int_phyup_v1_hw(int irq_no, void *p)
                         SAS_PROTOCOL_SMP;
         hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
 
+        spin_lock_irqsave(&phy->lock, flags);
+        if (phy->reset_completion) {
+                phy->in_reset = 0;
+                complete(phy->reset_completion);
+        }
+        spin_unlock_irqrestore(&phy->lock, flags);
+
 end:
         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2,
                              CHL_INT2_SL_PHY_ENA_MSK);
@@ -1845,6 +1796,28 @@ static int hisi_sas_v1_init(struct hisi_hba *hisi_hba)
         return 0;
 }
 
+static struct scsi_host_template sht_v1_hw = {
+        .name                   = DRV_NAME,
+        .module                 = THIS_MODULE,
+        .queuecommand           = sas_queuecommand,
+        .target_alloc           = sas_target_alloc,
+        .slave_configure        = hisi_sas_slave_configure,
+        .scan_finished          = hisi_sas_scan_finished,
+        .scan_start             = hisi_sas_scan_start,
+        .change_queue_depth     = sas_change_queue_depth,
+        .bios_param             = sas_bios_param,
+        .can_queue              = 1,
+        .this_id                = -1,
+        .sg_tablesize           = SG_ALL,
+        .max_sectors            = SCSI_DEFAULT_MAX_SECTORS,
+        .use_clustering         = ENABLE_CLUSTERING,
+        .eh_device_reset_handler = sas_eh_device_reset_handler,
+        .eh_target_reset_handler = sas_eh_target_reset_handler,
+        .target_destroy         = sas_target_destroy,
+        .ioctl                  = sas_ioctl,
+        .shost_attrs            = host_attrs,
+};
+
 static const struct hisi_sas_hw hisi_sas_v1_hw = {
         .hw_init = hisi_sas_v1_init,
         .setup_itct = setup_itct_v1_hw,
@@ -1864,6 +1837,7 @@ static const struct hisi_sas_hw hisi_sas_v1_hw = {
         .get_wideport_bitmap = get_wideport_bitmap_v1_hw,
         .max_command_entries = HISI_SAS_COMMAND_ENTRIES_V1_HW,
         .complete_hdr_size = sizeof(struct hisi_sas_complete_v1_hdr),
+        .sht = &sht_v1_hw,
 };
 
 static int hisi_sas_v1_probe(struct platform_device *pdev)
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
index f89fb9a49ea9..213c530e63f2 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
@@ -144,6 +144,7 @@
 #define SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF        19
 #define SAS_ECC_INTR_MSK                0x1ec
 #define HGC_ERR_STAT_EN                 0x238
+#define CQE_SEND_CNT                    0x248
 #define DLVRY_Q_0_BASE_ADDR_LO          0x260
 #define DLVRY_Q_0_BASE_ADDR_HI          0x264
 #define DLVRY_Q_0_DEPTH                 0x268
@@ -295,6 +296,10 @@
 #define CMD_HDR_RESP_REPORT_MSK         (0x1 << CMD_HDR_RESP_REPORT_OFF)
 #define CMD_HDR_TLR_CTRL_OFF            6
 #define CMD_HDR_TLR_CTRL_MSK            (0x3 << CMD_HDR_TLR_CTRL_OFF)
+#define CMD_HDR_PHY_ID_OFF              8
+#define CMD_HDR_PHY_ID_MSK              (0x1ff << CMD_HDR_PHY_ID_OFF)
+#define CMD_HDR_FORCE_PHY_OFF           17
+#define CMD_HDR_FORCE_PHY_MSK           (0x1 << CMD_HDR_FORCE_PHY_OFF)
 #define CMD_HDR_PORT_OFF                18
 #define CMD_HDR_PORT_MSK                (0xf << CMD_HDR_PORT_OFF)
 #define CMD_HDR_PRIORITY_OFF            27
@@ -1216,7 +1221,22 @@ static void init_reg_v2_hw(struct hisi_hba *hisi_hba)
         }
 
         for (i = 0; i < hisi_hba->n_phy; i++) {
-                hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE, 0x855);
+                struct hisi_sas_phy *phy = &hisi_hba->phy[i];
+                struct asd_sas_phy *sas_phy = &phy->sas_phy;
+                u32 prog_phy_link_rate = 0x800;
+
+                if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate <
+                                SAS_LINK_RATE_1_5_GBPS)) {
+                        prog_phy_link_rate = 0x855;
+                } else {
+                        enum sas_linkrate max = sas_phy->phy->maximum_linkrate;
+
+                        prog_phy_link_rate =
+                                hisi_sas_get_prog_phy_linkrate_mask(max) |
+                                0x800;
+                }
+                hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE,
+                        prog_phy_link_rate);
                 hisi_sas_phy_write32(hisi_hba, i, SAS_PHY_CTRL, sas_phy_ctrl);
                 hisi_sas_phy_write32(hisi_hba, i, SL_TOUT_CFG, 0x7d7d7d7d);
                 hisi_sas_phy_write32(hisi_hba, i, SL_CONTROL, 0x0);
@@ -1585,39 +1605,12 @@ static enum sas_linkrate phy_get_max_linkrate_v2_hw(void)
 static void phy_set_linkrate_v2_hw(struct hisi_hba *hisi_hba, int phy_no,
                 struct sas_phy_linkrates *r)
 {
-        u32 prog_phy_link_rate =
-                hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
-        struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
-        struct asd_sas_phy *sas_phy = &phy->sas_phy;
-        int i;
-        enum sas_linkrate min, max;
-        u32 rate_mask = 0;
-
-        if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-                max = sas_phy->phy->maximum_linkrate;
-                min = r->minimum_linkrate;
-        } else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-                max = r->maximum_linkrate;
-                min = sas_phy->phy->minimum_linkrate;
-        } else
-                return;
-
-        sas_phy->phy->maximum_linkrate = max;
-        sas_phy->phy->minimum_linkrate = min;
-
-        max -= SAS_LINK_RATE_1_5_GBPS;
-
-        for (i = 0; i <= max; i++)
-                rate_mask |= 1 << (i * 2);
-
-        prog_phy_link_rate &= ~0xff;
-        prog_phy_link_rate |= rate_mask;
+        enum sas_linkrate max = r->maximum_linkrate;
+        u32 prog_phy_link_rate = 0x800;
 
-        disable_phy_v2_hw(hisi_hba, phy_no);
-        msleep(100);
+        prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
         hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
-                        prog_phy_link_rate);
-        start_phy_v2_hw(hisi_hba, phy_no);
+                             prog_phy_link_rate);
 }
 
 static int get_wideport_bitmap_v2_hw(struct hisi_hba *hisi_hba, int port_id)
@@ -1658,42 +1651,50 @@ get_free_slot_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
         r = hisi_sas_read32_relaxed(hisi_hba,
                                 DLVRY_Q_0_RD_PTR + (queue * 0x14));
         if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
-                dev_warn(dev, "full queue=%d r=%d w=%d\n\n",
+                dev_warn(dev, "full queue=%d r=%d w=%d\n",
                                 queue, r, w);
                 return -EAGAIN;
         }
 
-        return 0;
+        dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS;
+
+        return w;
 }
 
+/* DQ lock must be taken here */
 static void start_delivery_v2_hw(struct hisi_sas_dq *dq)
 {
         struct hisi_hba *hisi_hba = dq->hisi_hba;
-        int dlvry_queue = dq->slot_prep->dlvry_queue;
-        int dlvry_queue_slot = dq->slot_prep->dlvry_queue_slot;
+        struct hisi_sas_slot *s, *s1;
+        struct list_head *dq_list;
+        int dlvry_queue = dq->id;
+        int wp, count = 0;
+
+        dq_list = &dq->list;
+        list_for_each_entry_safe(s, s1, &dq->list, delivery) {
+                if (!s->ready)
+                        break;
+                count++;
+                wp = (s->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
+                list_del(&s->delivery);
+        }
+
+        if (!count)
+                return;
 
-        dq->wr_point = ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS;
-        hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14),
-                         dq->wr_point);
+        hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), wp);
 }
 
-static int prep_prd_sge_v2_hw(struct hisi_hba *hisi_hba,
+static void prep_prd_sge_v2_hw(struct hisi_hba *hisi_hba,
                               struct hisi_sas_slot *slot,
                               struct hisi_sas_cmd_hdr *hdr,
                               struct scatterlist *scatter,
                               int n_elem)
 {
         struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
-        struct device *dev = hisi_hba->dev;
         struct scatterlist *sg;
         int i;
 
-        if (n_elem > HISI_SAS_SGE_PAGE_CNT) {
-                dev_err(dev, "prd err: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT",
-                        n_elem);
-                return -EINVAL;
-        }
-
         for_each_sg(scatter, sg, n_elem, i) {
                 struct hisi_sas_sge *entry = &sge_page->sge[i];
 
@@ -1706,47 +1707,24 @@ static int prep_prd_sge_v2_hw(struct hisi_hba *hisi_hba,
         hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
 
         hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
-
-        return 0;
 }
 
-static int prep_smp_v2_hw(struct hisi_hba *hisi_hba,
+static void prep_smp_v2_hw(struct hisi_hba *hisi_hba,
                           struct hisi_sas_slot *slot)
 {
         struct sas_task *task = slot->task;
         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
         struct domain_device *device = task->dev;
-        struct device *dev = hisi_hba->dev;
         struct hisi_sas_port *port = slot->port;
-        struct scatterlist *sg_req, *sg_resp;
+        struct scatterlist *sg_req;
         struct hisi_sas_device *sas_dev = device->lldd_dev;
         dma_addr_t req_dma_addr;
-        unsigned int req_len, resp_len;
-        int elem, rc;
+        unsigned int req_len;
 
-        /*
-        * DMA-map SMP request, response buffers
-        */
         /* req */
         sg_req = &task->smp_task.smp_req;
-        elem = dma_map_sg(dev, sg_req, 1, DMA_TO_DEVICE);
-        if (!elem)
-                return -ENOMEM;
-        req_len = sg_dma_len(sg_req);
         req_dma_addr = sg_dma_address(sg_req);
-
-        /* resp */
-        sg_resp = &task->smp_task.smp_resp;
-        elem = dma_map_sg(dev, sg_resp, 1, DMA_FROM_DEVICE);
-        if (!elem) {
-                rc = -ENOMEM;
-                goto err_out_req;
-        }
-        resp_len = sg_dma_len(sg_resp);
-        if ((req_len & 0x3) || (resp_len & 0x3)) {
-                rc = -EINVAL;
-                goto err_out_resp;
-        }
+        req_len = sg_dma_len(&task->smp_task.smp_req);
 
         /* create header */
         /* dw0 */
@@ -1768,21 +1746,10 @@ static int prep_smp_v2_hw(struct hisi_hba *hisi_hba,
 
         hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
-
-        return 0;
-
-err_out_resp:
-        dma_unmap_sg(dev, &slot->task->smp_task.smp_resp, 1,
-                     DMA_FROM_DEVICE);
-err_out_req:
-        dma_unmap_sg(dev, &slot->task->smp_task.smp_req, 1,
-                     DMA_TO_DEVICE);
-        return rc;
 }
 
-static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
-                          struct hisi_sas_slot *slot, int is_tmf,
-                          struct hisi_sas_tmf_task *tmf)
+static void prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
+                          struct hisi_sas_slot *slot)
 {
         struct sas_task *task = slot->task;
         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
@@ -1791,7 +1758,8 @@ static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
         struct hisi_sas_port *port = slot->port;
         struct sas_ssp_task *ssp_task = &task->ssp_task;
         struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
-        int has_data = 0, rc, priority = is_tmf;
+        struct hisi_sas_tmf_task *tmf = slot->tmf;
+        int has_data = 0, priority = !!tmf;
         u8 *buf_cmd;
         u32 dw1 = 0, dw2 = 0;
 
@@ -1802,7 +1770,7 @@ static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
                                (1 << CMD_HDR_CMD_OFF)); /* ssp */
 
         dw1 = 1 << CMD_HDR_VDTL_OFF;
-        if (is_tmf) {
+        if (tmf) {
                 dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
                 dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
         } else {
@@ -1833,12 +1801,9 @@ static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
 
         hdr->transfer_tags = cpu_to_le32(slot->idx);
 
-        if (has_data) {
-                rc = prep_prd_sge_v2_hw(hisi_hba, slot, hdr, task->scatter,
+        if (has_data)
+                prep_prd_sge_v2_hw(hisi_hba, slot, hdr, task->scatter,
                                         slot->n_elem);
-                if (rc)
-                        return rc;
-        }
 
         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
@@ -1848,7 +1813,7 @@ static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
                 sizeof(struct ssp_frame_hdr);
 
         memcpy(buf_cmd, &task->ssp_task.LUN, 8);
-        if (!is_tmf) {
+        if (!tmf) {
                 buf_cmd[9] = task->ssp_task.task_attr |
                                 (task->ssp_task.task_prio << 3);
                 memcpy(buf_cmd + 12, task->ssp_task.cmd->cmnd,
@@ -1867,8 +1832,6 @@ static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
                         break;
                 }
         }
-
-        return 0;
 }
 
 #define TRANS_TX_ERR    0
@@ -2380,23 +2343,24 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
         struct device *dev = hisi_hba->dev;
         struct task_status_struct *ts;
         struct domain_device *device;
+        struct sas_ha_struct *ha;
         enum exec_status sts;
         struct hisi_sas_complete_v2_hdr *complete_queue =
                         hisi_hba->complete_hdr[slot->cmplt_queue];
         struct hisi_sas_complete_v2_hdr *complete_hdr =
                         &complete_queue[slot->cmplt_queue_slot];
         unsigned long flags;
-        int aborted;
+        bool is_internal = slot->is_internal;
 
         if (unlikely(!task || !task->lldd_task || !task->dev))
                 return -EINVAL;
 
         ts = &task->task_status;
         device = task->dev;
+        ha = device->port->ha;
         sas_dev = device->lldd_dev;
 
         spin_lock_irqsave(&task->task_state_lock, flags);
-        aborted = task->task_state_flags & SAS_TASK_STATE_ABORTED;
         task->task_state_flags &=
                 ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
         spin_unlock_irqrestore(&task->task_state_lock, flags);
@@ -2404,15 +2368,6 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
         memset(ts, 0, sizeof(*ts));
         ts->resp = SAS_TASK_COMPLETE;
 
-        if (unlikely(aborted)) {
-                dev_dbg(dev, "slot_complete: task(%p) aborted\n", task);
-                ts->stat = SAS_ABORTED_TASK;
-                spin_lock_irqsave(&hisi_hba->lock, flags);
-                hisi_sas_slot_task_free(hisi_hba, task, slot);
-                spin_unlock_irqrestore(&hisi_hba->lock, flags);
-                return ts->stat;
-        }
-
         if (unlikely(!sas_dev)) {
                 dev_dbg(dev, "slot complete: port has no device\n");
                 ts->stat = SAS_PHY_DOWN;
@@ -2459,10 +2414,10 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
                         slot_err_v2_hw(hisi_hba, task, slot, 2);
 
                 if (ts->stat != SAS_DATA_UNDERRUN)
-                        dev_info(dev, "erroneous completion iptt=%d task=%p "
+                        dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d "
                                 "CQ hdr: 0x%x 0x%x 0x%x 0x%x "
                                 "Error info: 0x%x 0x%x 0x%x 0x%x\n",
-                                slot->idx, task,
+                                slot->idx, task, sas_dev->device_id,
                                 complete_hdr->dw0, complete_hdr->dw1,
                                 complete_hdr->act, complete_hdr->dw3,
                                 error_info[0], error_info[1],
@@ -2523,13 +2478,27 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
         }
 
 out:
+        hisi_sas_slot_task_free(hisi_hba, task, slot);
+        sts = ts->stat;
         spin_lock_irqsave(&task->task_state_lock, flags);
+        if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
+                spin_unlock_irqrestore(&task->task_state_lock, flags);
+                dev_info(dev, "slot complete: task(%p) aborted\n", task);
+                return SAS_ABORTED_TASK;
+        }
         task->task_state_flags |= SAS_TASK_STATE_DONE;
         spin_unlock_irqrestore(&task->task_state_lock, flags);
-        spin_lock_irqsave(&hisi_hba->lock, flags);
-        hisi_sas_slot_task_free(hisi_hba, task, slot);
-        spin_unlock_irqrestore(&hisi_hba->lock, flags);
-        sts = ts->stat;
+
+        if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
+                spin_lock_irqsave(&device->done_lock, flags);
+                if (test_bit(SAS_HA_FROZEN, &ha->state)) {
+                        spin_unlock_irqrestore(&device->done_lock, flags);
+                        dev_info(dev, "slot complete: task(%p) ignored\n ",
+                                 task);
+                        return sts;
+                }
+                spin_unlock_irqrestore(&device->done_lock, flags);
+        }
 
         if (task->task_done)
                 task->task_done(task);
@@ -2537,7 +2506,7 @@ out:
         return sts;
 }
 
-static int prep_ata_v2_hw(struct hisi_hba *hisi_hba,
+static void prep_ata_v2_hw(struct hisi_hba *hisi_hba,
                           struct hisi_sas_slot *slot)
 {
         struct sas_task *task = slot->task;
@@ -2547,8 +2516,9 @@ static int prep_ata_v2_hw(struct hisi_hba *hisi_hba,
         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
         struct asd_sas_port *sas_port = device->port;
         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
+        struct hisi_sas_tmf_task *tmf = slot->tmf;
         u8 *buf_cmd;
-        int has_data = 0, rc = 0, hdr_tag = 0;
+        int has_data = 0, hdr_tag = 0;
         u32 dw1 = 0, dw2 = 0;
 
         /* create header */
@@ -2559,6 +2529,12 @@ static int prep_ata_v2_hw(struct hisi_hba *hisi_hba,
         else
                 hdr->dw0 |= cpu_to_le32(4 << CMD_HDR_CMD_OFF);
 
+        if (tmf && tmf->force_phy) {
+                hdr->dw0 |= CMD_HDR_FORCE_PHY_MSK;
+                hdr->dw0 |= cpu_to_le32((1 << tmf->phy_id)
+                                << CMD_HDR_PHY_ID_OFF);
+        }
+
         /* dw1 */
         switch (task->data_dir) {
         case DMA_TO_DEVICE:
@@ -2596,12 +2572,9 @@ static int prep_ata_v2_hw(struct hisi_hba *hisi_hba,
         /* dw3 */
         hdr->transfer_tags = cpu_to_le32(slot->idx);
 
-        if (has_data) {
-                rc = prep_prd_sge_v2_hw(hisi_hba, slot, hdr, task->scatter,
+        if (has_data)
+                prep_prd_sge_v2_hw(hisi_hba, slot, hdr, task->scatter,
                                         slot->n_elem);
-                if (rc)
-                        return rc;
-        }
 
         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
@@ -2613,8 +2586,6 @@ static int prep_ata_v2_hw(struct hisi_hba *hisi_hba,
                 task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
         /* fill in command FIS */
         memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
-
-        return 0;
 }
 
 static void hisi_sas_internal_abort_quirk_timeout(struct timer_list *t)
@@ -2651,7 +2622,7 @@ static void hisi_sas_internal_abort_quirk_timeout(struct timer_list *t)
         }
 }
 
-static int prep_abort_v2_hw(struct hisi_hba *hisi_hba,
+static void prep_abort_v2_hw(struct hisi_hba *hisi_hba,
                 struct hisi_sas_slot *slot,
                 int device_id, int abort_flag, int tag_to_abort)
 {
@@ -2679,8 +2650,6 @@ static int prep_abort_v2_hw(struct hisi_hba *hisi_hba,
         /* dw7 */
         hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF);
         hdr->transfer_tags = cpu_to_le32(slot->idx);
-
-        return 0;
 }
 
 static int phy_up_v2_hw(int phy_no, struct hisi_hba *hisi_hba)
@@ -2692,6 +2661,7 @@ static int phy_up_v2_hw(int phy_no, struct hisi_hba *hisi_hba)
         struct device *dev = hisi_hba->dev;
         u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
         struct sas_identify_frame *id = (struct sas_identify_frame *)frame_rcvd;
+        unsigned long flags;
 
         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1);
 
@@ -2744,6 +2714,12 @@ static int phy_up_v2_hw(int phy_no, struct hisi_hba *hisi_hba)
                         set_link_timer_quirk(hisi_hba);
         }
         hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
+        spin_lock_irqsave(&phy->lock, flags);
+        if (phy->reset_completion) {
+                phy->in_reset = 0;
+                complete(phy->reset_completion);
+        }
+        spin_unlock_irqrestore(&phy->lock, flags);
 
 end:
         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
@@ -3151,14 +3127,12 @@ static void cq_tasklet_v2_hw(unsigned long val)
         struct hisi_sas_complete_v2_hdr *complete_queue;
         u32 rd_point = cq->rd_point, wr_point, dev_id;
         int queue = cq->id;
-        struct hisi_sas_dq *dq = &hisi_hba->dq[queue];
 
         if (unlikely(hisi_hba->reject_stp_links_msk))
                 phys_try_accept_stp_links_v2_hw(hisi_hba);
 
         complete_queue = hisi_hba->complete_hdr[queue];
 
-        spin_lock(&dq->lock);
         wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
                                    (0x14 * queue));
 
@@ -3208,7 +3182,6 @@ static void cq_tasklet_v2_hw(unsigned long val)
         /* update rd_point */
         cq->rd_point = rd_point;
         hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
-        spin_unlock(&dq->lock);
 }
 
 static irqreturn_t cq_interrupt_v2_hw(int irq_no, void *p)
@@ -3235,6 +3208,7 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p)
         u32 ent_tmp, ent_msk, ent_int, port_id, link_rate, hard_phy_linkrate;
         irqreturn_t res = IRQ_HANDLED;
         u8 attached_sas_addr[SAS_ADDR_SIZE] = {0};
+        unsigned long flags;
         int phy_no, offset;
 
         phy_no = sas_phy->id;
@@ -3295,6 +3269,7 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p)
         sas_phy->oob_mode = SATA_OOB_MODE;
         /* Make up some unique SAS address */
         attached_sas_addr[0] = 0x50;
+        attached_sas_addr[6] = hisi_hba->shost->host_no;
         attached_sas_addr[7] = phy_no;
         memcpy(sas_phy->attached_sas_addr, attached_sas_addr, SAS_ADDR_SIZE);
         memcpy(sas_phy->frame_rcvd, fis, sizeof(struct dev_to_host_fis));
@@ -3308,6 +3283,12 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p)
         phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
         hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
 
+        spin_lock_irqsave(&phy->lock, flags);
+        if (phy->reset_completion) {
+                phy->in_reset = 0;
+                complete(phy->reset_completion);
+        }
+        spin_unlock_irqrestore(&phy->lock, flags);
 end:
         hisi_sas_write32(hisi_hba, ENT_INT_SRC1 + offset, ent_tmp);
         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1 + offset, ent_msk);
@@ -3546,6 +3527,46 @@ static int write_gpio_v2_hw(struct hisi_hba *hisi_hba, u8 reg_type,
         return 0;
 }
 
+static void wait_cmds_complete_timeout_v2_hw(struct hisi_hba *hisi_hba,
+                                             int delay_ms, int timeout_ms)
+{
+        struct device *dev = hisi_hba->dev;
+        int entries, entries_old = 0, time;
+
+        for (time = 0; time < timeout_ms; time += delay_ms) {
+                entries = hisi_sas_read32(hisi_hba, CQE_SEND_CNT);
+                if (entries == entries_old)
+                        break;
+
+                entries_old = entries;
+                msleep(delay_ms);
+        }
+
+        dev_dbg(dev, "wait commands complete %dms\n", time);
+}
+
+static struct scsi_host_template sht_v2_hw = {
+        .name                   = DRV_NAME,
+        .module                 = THIS_MODULE,
+        .queuecommand           = sas_queuecommand,
+        .target_alloc           = sas_target_alloc,
+        .slave_configure        = hisi_sas_slave_configure,
+        .scan_finished          = hisi_sas_scan_finished,
+        .scan_start             = hisi_sas_scan_start,
+        .change_queue_depth     = sas_change_queue_depth,
+        .bios_param             = sas_bios_param,
+        .can_queue              = 1,
+        .this_id                = -1,
+        .sg_tablesize           = SG_ALL,
+        .max_sectors            = SCSI_DEFAULT_MAX_SECTORS,
+        .use_clustering         = ENABLE_CLUSTERING,
+        .eh_device_reset_handler = sas_eh_device_reset_handler,
+        .eh_target_reset_handler = sas_eh_target_reset_handler,
+        .target_destroy         = sas_target_destroy,
+        .ioctl                  = sas_ioctl,
+        .shost_attrs            = host_attrs,
+};
+
 static const struct hisi_sas_hw hisi_sas_v2_hw = {
         .hw_init = hisi_sas_v2_init,
         .setup_itct = setup_itct_v2_hw,
@@ -3574,6 +3595,8 @@ static const struct hisi_sas_hw hisi_sas_v2_hw = {
         .soft_reset = soft_reset_v2_hw,
         .get_phys_state = get_phys_state_v2_hw,
         .write_gpio = write_gpio_v2_hw,
+        .wait_cmds_complete_timeout = wait_cmds_complete_timeout_v2_hw,
+        .sht = &sht_v2_hw,
 };
 
 static int hisi_sas_v2_probe(struct platform_device *pdev)
@@ -3598,9 +3621,6 @@ static int hisi_sas_v2_remove(struct platform_device *pdev)
         struct sas_ha_struct *sha = platform_get_drvdata(pdev);
         struct hisi_hba *hisi_hba = sha->lldd_ha;
 
-        if (timer_pending(&hisi_hba->timer))
-                del_timer(&hisi_hba->timer);
-
         hisi_sas_kill_tasklets(hisi_hba);
 
         return hisi_sas_remove(pdev);
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
index 6f3e5ba6b472..9f1e2d03f914 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
@@ -92,6 +92,7 @@
 #define SAS_ECC_INTR                    0x1e8
 #define SAS_ECC_INTR_MSK                0x1ec
 #define HGC_ERR_STAT_EN                 0x238
+#define CQE_SEND_CNT                    0x248
 #define DLVRY_Q_0_BASE_ADDR_LO          0x260
 #define DLVRY_Q_0_BASE_ADDR_HI          0x264
 #define DLVRY_Q_0_DEPTH                 0x268
@@ -106,6 +107,11 @@
 #define COMPL_Q_0_RD_PTR                0x4f0
 #define AWQOS_AWCACHE_CFG       0xc84
 #define ARQOS_ARCACHE_CFG       0xc88
+#define HILINK_ERR_DFX          0xe04
+#define SAS_GPIO_CFG_0          0x1000
+#define SAS_GPIO_CFG_1          0x1004
+#define SAS_GPIO_TX_0_1 0x1040
+#define SAS_CFG_DRIVE_VLD       0x1070
 
 /* phy registers requiring init */
 #define PORT_BASE                       (0x2000)
@@ -167,6 +173,7 @@
 #define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF 22
 #define CHL_INT2                        (PORT_BASE + 0x1bc)
 #define CHL_INT2_SL_IDAF_TOUT_CONF_OFF  0
+#define CHL_INT2_RX_INVLD_DW_OFF        30
 #define CHL_INT2_STP_LINK_TIMEOUT_OFF   31
 #define CHL_INT0_MSK                    (PORT_BASE + 0x1c0)
 #define CHL_INT1_MSK                    (PORT_BASE + 0x1c4)
@@ -216,6 +223,9 @@
 #define SAS_RAS_INTR1                   (RAS_BASE + 0x04)
 #define SAS_RAS_INTR0_MASK              (RAS_BASE + 0x08)
 #define SAS_RAS_INTR1_MASK              (RAS_BASE + 0x0c)
+#define CFG_SAS_RAS_INTR_MASK           (RAS_BASE + 0x1c)
+#define SAS_RAS_INTR2                   (RAS_BASE + 0x20)
+#define SAS_RAS_INTR2_MASK              (RAS_BASE + 0x24)
 
 /* HW dma structures */
 /* Delivery queue header */
@@ -348,10 +358,11 @@ struct hisi_sas_err_record_v3 {
 #define DIR_TO_DEVICE 2
 #define DIR_RESERVED 3
 
-#define CMD_IS_UNCONSTRAINT(cmd) \
-        ((cmd == ATA_CMD_READ_LOG_EXT) || \
-        (cmd == ATA_CMD_READ_LOG_DMA_EXT) || \
-        (cmd == ATA_CMD_DEV_RESET))
+#define FIS_CMD_IS_UNCONSTRAINED(fis) \
+        ((fis.command == ATA_CMD_READ_LOG_EXT) || \
+        (fis.command == ATA_CMD_READ_LOG_DMA_EXT) || \
+        ((fis.command == ATA_CMD_DEV_RESET) && \
+        ((fis.control & ATA_SRST) != 0)))
 
 static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off)
 {
@@ -390,8 +401,23 @@ static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
         return readl(regs);
 }
 
+#define hisi_sas_read32_poll_timeout(off, val, cond, delay_us,          \
+                                     timeout_us)                        \
+({                                                                      \
+        void __iomem *regs = hisi_hba->regs + off;                      \
+        readl_poll_timeout(regs, val, cond, delay_us, timeout_us);      \
+})
+
+#define hisi_sas_read32_poll_timeout_atomic(off, val, cond, delay_us,   \
+                                            timeout_us)                 \
+({                                                                      \
+        void __iomem *regs = hisi_hba->regs + off;                      \
+        readl_poll_timeout_atomic(regs, val, cond, delay_us, timeout_us);\
+})
+
 static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 {
+        struct pci_dev *pdev = hisi_hba->pci_dev;
         int i;
 
         /* Global registers init */
@@ -409,7 +435,10 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
-        hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xfffe20ff);
+        if (pdev->revision >= 0x21)
+                hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffff7fff);
+        else
+                hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xfffe20ff);
         hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
         hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
         hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
@@ -422,13 +451,33 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
         hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1);
 
         for (i = 0; i < hisi_hba->n_phy; i++) {
-                hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE, 0x855);
+                struct hisi_sas_phy *phy = &hisi_hba->phy[i];
+                struct asd_sas_phy *sas_phy = &phy->sas_phy;
+                u32 prog_phy_link_rate = 0x800;
+
+                if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate <
+                                SAS_LINK_RATE_1_5_GBPS)) {
+                        prog_phy_link_rate = 0x855;
+                } else {
+                        enum sas_linkrate max = sas_phy->phy->maximum_linkrate;
+
+                        prog_phy_link_rate =
+                                hisi_sas_get_prog_phy_linkrate_mask(max) |
+                                0x800;
+                }
+                hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE,
+                        prog_phy_link_rate);
                 hisi_sas_phy_write32(hisi_hba, i, SAS_RX_TRAIN_TIMER, 0x13e80);
                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff);
                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
                 hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
-                hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xff87ffff);
+                if (pdev->revision >= 0x21)
+                        hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
+                                        0xffffffff);
+                else
+                        hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
+                                        0xff87ffff);
                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
                 hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
@@ -503,6 +552,16 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
         /* RAS registers init */
         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0_MASK, 0x0);
         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1_MASK, 0x0);
+        hisi_sas_write32(hisi_hba, SAS_RAS_INTR2_MASK, 0x0);
+        hisi_sas_write32(hisi_hba, CFG_SAS_RAS_INTR_MASK, 0x0);
+
+        /* LED registers init */
+        hisi_sas_write32(hisi_hba, SAS_CFG_DRIVE_VLD, 0x80000ff);
+        hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1, 0x80808080);
+        hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1 + 0x4, 0x80808080);
+        /* Configure blink generator rate A to 1Hz and B to 4Hz */
+        hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_1, 0x121700);
+        hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_0, 0x800000);
 }
 
 static void config_phy_opt_mode_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
@@ -654,8 +713,8 @@ static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
         udelay(50);
 
         /* Ensure axi bus idle */
-        ret = readl_poll_timeout(hisi_hba->regs + AXI_CFG, val, !val,
-                        20000, 1000000);
+        ret = hisi_sas_read32_poll_timeout(AXI_CFG, val, !val,
+                                           20000, 1000000);
         if (ret) {
                 dev_err(dev, "axi bus is not idle, ret = %d!\n", ret);
                 return -EIO;
@@ -794,42 +853,49 @@ get_free_slot_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
         r = hisi_sas_read32_relaxed(hisi_hba,
                                 DLVRY_Q_0_RD_PTR + (queue * 0x14));
         if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
-                dev_warn(dev, "full queue=%d r=%d w=%d\n\n",
+                dev_warn(dev, "full queue=%d r=%d w=%d\n",
                                 queue, r, w);
                 return -EAGAIN;
         }
 
-        return 0;
+        dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS;
+
+        return w;
 }
 
 static void start_delivery_v3_hw(struct hisi_sas_dq *dq)
 {
         struct hisi_hba *hisi_hba = dq->hisi_hba;
-        int dlvry_queue = dq->slot_prep->dlvry_queue;
-        int dlvry_queue_slot = dq->slot_prep->dlvry_queue_slot;
+        struct hisi_sas_slot *s, *s1;
+        struct list_head *dq_list;
+        int dlvry_queue = dq->id;
+        int wp, count = 0;
+
+        dq_list = &dq->list;
+        list_for_each_entry_safe(s, s1, &dq->list, delivery) {
+                if (!s->ready)
+                        break;
+                count++;
+                wp = (s->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
+                list_del(&s->delivery);
+        }
+
+        if (!count)
+                return;
 
-        dq->wr_point = ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS;
-        hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14),
-                         dq->wr_point);
+        hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), wp);
 }
 
-static int prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
+static void prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
                               struct hisi_sas_slot *slot,
                               struct hisi_sas_cmd_hdr *hdr,
                               struct scatterlist *scatter,
                               int n_elem)
 {
         struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
-        struct device *dev = hisi_hba->dev;
         struct scatterlist *sg;
         int i;
 
-        if (n_elem > HISI_SAS_SGE_PAGE_CNT) {
-                dev_err(dev, "prd err: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT",
-                        n_elem);
-                return -EINVAL;
-        }
-
         for_each_sg(scatter, sg, n_elem, i) {
                 struct hisi_sas_sge *entry = &sge_page->sge[i];
 
@@ -842,13 +908,10 @@ static int prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
         hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
 
         hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
-
-        return 0;
 }
 
-static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
-                          struct hisi_sas_slot *slot, int is_tmf,
-                          struct hisi_sas_tmf_task *tmf)
+static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
+                          struct hisi_sas_slot *slot)
 {
         struct sas_task *task = slot->task;
         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
@@ -857,7 +920,8 @@ static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
         struct hisi_sas_port *port = slot->port;
         struct sas_ssp_task *ssp_task = &task->ssp_task;
         struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
-        int has_data = 0, rc, priority = is_tmf;
+        struct hisi_sas_tmf_task *tmf = slot->tmf;
+        int has_data = 0, priority = !!tmf;
         u8 *buf_cmd;
         u32 dw1 = 0, dw2 = 0;
 
@@ -868,7 +932,7 @@ static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
                                (1 << CMD_HDR_CMD_OFF)); /* ssp */
 
         dw1 = 1 << CMD_HDR_VDTL_OFF;
-        if (is_tmf) {
+        if (tmf) {
                 dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
                 dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
         } else {
@@ -898,12 +962,9 @@ static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
         hdr->dw2 = cpu_to_le32(dw2);
         hdr->transfer_tags = cpu_to_le32(slot->idx);
 
-        if (has_data) {
-                rc = prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
+        if (has_data)
+                prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
                                         slot->n_elem);
-                if (rc)
-                        return rc;
-        }
 
         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
@@ -913,7 +974,7 @@ static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
                 sizeof(struct ssp_frame_hdr);
 
         memcpy(buf_cmd, &task->ssp_task.LUN, 8);
-        if (!is_tmf) {
+        if (!tmf) {
                 buf_cmd[9] = ssp_task->task_attr | (ssp_task->task_prio << 3);
                 memcpy(buf_cmd + 12, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
         } else {
@@ -930,48 +991,25 @@ static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
                         break;
                 }
         }
-
-        return 0;
 }
 
-static int prep_smp_v3_hw(struct hisi_hba *hisi_hba,
+static void prep_smp_v3_hw(struct hisi_hba *hisi_hba,
                           struct hisi_sas_slot *slot)
 {
         struct sas_task *task = slot->task;
         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
         struct domain_device *device = task->dev;
-        struct device *dev = hisi_hba->dev;
         struct hisi_sas_port *port = slot->port;
-        struct scatterlist *sg_req, *sg_resp;
+        struct scatterlist *sg_req;
         struct hisi_sas_device *sas_dev = device->lldd_dev;
         dma_addr_t req_dma_addr;
-        unsigned int req_len, resp_len;
-        int elem, rc;
+        unsigned int req_len;
 
-        /*
-         * DMA-map SMP request, response buffers
-         */
         /* req */
         sg_req = &task->smp_task.smp_req;
-        elem = dma_map_sg(dev, sg_req, 1, DMA_TO_DEVICE);
-        if (!elem)
-                return -ENOMEM;
         req_len = sg_dma_len(sg_req);
         req_dma_addr = sg_dma_address(sg_req);
 
-        /* resp */
-        sg_resp = &task->smp_task.smp_resp;
-        elem = dma_map_sg(dev, sg_resp, 1, DMA_FROM_DEVICE);
-        if (!elem) {
-                rc = -ENOMEM;
-                goto err_out_req;
-        }
-        resp_len = sg_dma_len(sg_resp);
-        if ((req_len & 0x3) || (resp_len & 0x3)) {
-                rc = -EINVAL;
-                goto err_out_resp;
-        }
-
         /* create header */
         /* dw0 */
         hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
@@ -993,18 +1031,9 @@ static int prep_smp_v3_hw(struct hisi_hba *hisi_hba,
         hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
 
-        return 0;
-
-err_out_resp:
-        dma_unmap_sg(dev, &slot->task->smp_task.smp_resp, 1,
-                     DMA_FROM_DEVICE);
-err_out_req:
-        dma_unmap_sg(dev, &slot->task->smp_task.smp_req, 1,
-                     DMA_TO_DEVICE);
-        return rc;
 }
 
-static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
+static void prep_ata_v3_hw(struct hisi_hba *hisi_hba,
                           struct hisi_sas_slot *slot)
 {
         struct sas_task *task = slot->task;
@@ -1015,7 +1044,7 @@ static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
         struct asd_sas_port *sas_port = device->port;
         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
         u8 *buf_cmd;
-        int has_data = 0, rc = 0, hdr_tag = 0;
+        int has_data = 0, hdr_tag = 0;
         u32 dw1 = 0, dw2 = 0;
 
         hdr->dw0 = cpu_to_le32(port->id << CMD_HDR_PORT_OFF);
@@ -1046,7 +1075,7 @@ static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
                 << CMD_HDR_FRAME_TYPE_OFF;
         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
 
-        if (CMD_IS_UNCONSTRAINT(task->ata_task.fis.command))
+        if (FIS_CMD_IS_UNCONSTRAINED(task->ata_task.fis))
                 dw1 |= 1 << CMD_HDR_UNCON_CMD_OFF;
 
         hdr->dw1 = cpu_to_le32(dw1);
@@ -1064,12 +1093,9 @@ static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
         /* dw3 */
         hdr->transfer_tags = cpu_to_le32(slot->idx);
 
-        if (has_data) {
-                rc = prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
+        if (has_data)
+                prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
                                         slot->n_elem);
-                if (rc)
-                        return rc;
-        }
 
         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
@@ -1081,11 +1107,9 @@ static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
                 task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
         /* fill in command FIS */
         memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
-
-        return 0;
 }
 
-static int prep_abort_v3_hw(struct hisi_hba *hisi_hba,
+static void prep_abort_v3_hw(struct hisi_hba *hisi_hba,
                 struct hisi_sas_slot *slot,
                 int device_id, int abort_flag, int tag_to_abort)
 {
@@ -1110,7 +1134,6 @@ static int prep_abort_v3_hw(struct hisi_hba *hisi_hba,
         hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF);
         hdr->transfer_tags = cpu_to_le32(slot->idx);
 
-        return 0;
 }
 
 static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
@@ -1120,6 +1143,7 @@ static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
         struct asd_sas_phy *sas_phy = &phy->sas_phy;
         struct device *dev = hisi_hba->dev;
+        unsigned long flags;
 
         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1);
 
@@ -1188,6 +1212,12 @@ static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
         phy->phy_attached = 1;
         hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
         res = IRQ_HANDLED;
+        spin_lock_irqsave(&phy->lock, flags);
+        if (phy->reset_completion) {
+                phy->in_reset = 0;
+                complete(phy->reset_completion);
+        }
+        spin_unlock_irqrestore(&phy->lock, flags);
 end:
         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
                              CHL_INT0_SL_PHY_ENABLE_MSK);
@@ -1301,14 +1331,10 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
 {
         struct hisi_hba *hisi_hba = p;
         struct device *dev = hisi_hba->dev;
-        u32 ent_msk, ent_tmp, irq_msk;
+        struct pci_dev *pci_dev = hisi_hba->pci_dev;
+        u32 irq_msk;
         int phy_no = 0;
 
-        ent_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
-        ent_tmp = ent_msk;
-        ent_msk |= ENT_INT_SRC_MSK3_ENT95_MSK_MSK;
-        hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, ent_msk);
-
         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
                                 & 0xeeeeeeee;
 
@@ -1319,6 +1345,13 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
                                                      CHL_INT1);
                 u32 irq_value2 = hisi_sas_phy_read32(hisi_hba, phy_no,
                                                      CHL_INT2);
+                u32 irq_msk1 = hisi_sas_phy_read32(hisi_hba, phy_no,
+                                                        CHL_INT1_MSK);
+                u32 irq_msk2 = hisi_sas_phy_read32(hisi_hba, phy_no,
+                                                        CHL_INT2_MSK);
+
+                irq_value1 &= ~irq_msk1;
+                irq_value2 &= ~irq_msk2;
 
                 if ((irq_msk & (4 << (phy_no * 4))) &&
                                                 irq_value1) {
@@ -1364,8 +1397,28 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
 
                         hisi_sas_phy_write32(hisi_hba, phy_no,
                                              CHL_INT2, irq_value2);
-                }
 
+                        if ((irq_value2 & BIT(CHL_INT2_RX_INVLD_DW_OFF)) &&
+                            (pci_dev->revision == 0x20)) {
+                                u32 reg_value;
+                                int rc;
+
+                                rc = hisi_sas_read32_poll_timeout_atomic(
+                                        HILINK_ERR_DFX, reg_value,
+                                        !((reg_value >> 8) & BIT(phy_no)),
+                                        1000, 10000);
+                                if (rc) {
+                                        disable_phy_v3_hw(hisi_hba, phy_no);
+                                        hisi_sas_phy_write32(hisi_hba, phy_no,
+                                                CHL_INT2,
+                                                BIT(CHL_INT2_RX_INVLD_DW_OFF));
+                                        hisi_sas_phy_read32(hisi_hba, phy_no,
+                                                ERR_CNT_INVLD_DW);
+                                        mdelay(1);
+                                        enable_phy_v3_hw(hisi_hba, phy_no);
+                                }
+                        }
+                }
 
                 if (irq_msk & (2 << (phy_no * 4)) && irq_value0) {
                         hisi_sas_phy_write32(hisi_hba, phy_no,
@@ -1378,8 +1431,6 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
                 phy_no++;
         }
 
-        hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, ent_tmp);
-
         return IRQ_HANDLED;
 }
 
@@ -1448,6 +1499,7 @@ static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk | 0x1df00);
 
         irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
+        irq_value &= ~irq_msk;
 
         for (i = 0; i < ARRAY_SIZE(fatal_axi_error); i++) {
                 const struct hisi_sas_hw_error *error = &fatal_axi_error[i];
@@ -1549,37 +1601,30 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
         struct device *dev = hisi_hba->dev;
         struct task_status_struct *ts;
         struct domain_device *device;
+        struct sas_ha_struct *ha;
         enum exec_status sts;
         struct hisi_sas_complete_v3_hdr *complete_queue =
                         hisi_hba->complete_hdr[slot->cmplt_queue];
         struct hisi_sas_complete_v3_hdr *complete_hdr =
                         &complete_queue[slot->cmplt_queue_slot];
-        int aborted;
         unsigned long flags;
+        bool is_internal = slot->is_internal;
 
         if (unlikely(!task || !task->lldd_task || !task->dev))
                 return -EINVAL;
 
         ts = &task->task_status;
         device = task->dev;
+        ha = device->port->ha;
         sas_dev = device->lldd_dev;
 
         spin_lock_irqsave(&task->task_state_lock, flags);
-        aborted = task->task_state_flags & SAS_TASK_STATE_ABORTED;
         task->task_state_flags &=
                 ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
         spin_unlock_irqrestore(&task->task_state_lock, flags);
 
         memset(ts, 0, sizeof(*ts));
         ts->resp = SAS_TASK_COMPLETE;
-        if (unlikely(aborted)) {
-                dev_dbg(dev, "slot complete: task(%p) aborted\n", task);
-                ts->stat = SAS_ABORTED_TASK;
-                spin_lock_irqsave(&hisi_hba->lock, flags);
-                hisi_sas_slot_task_free(hisi_hba, task, slot);
-                spin_unlock_irqrestore(&hisi_hba->lock, flags);
-                return ts->stat;
-        }
 
         if (unlikely(!sas_dev)) {
                 dev_dbg(dev, "slot complete: port has not device\n");
@@ -1619,10 +1664,10 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 
                 slot_err_v3_hw(hisi_hba, task, slot);
                 if (ts->stat != SAS_DATA_UNDERRUN)
-                        dev_info(dev, "erroneous completion iptt=%d task=%p "
+                        dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d "
                                 "CQ hdr: 0x%x 0x%x 0x%x 0x%x "
                                 "Error info: 0x%x 0x%x 0x%x 0x%x\n",
-                                slot->idx, task,
+                                slot->idx, task, sas_dev->device_id,
                                 complete_hdr->dw0, complete_hdr->dw1,
                                 complete_hdr->act, complete_hdr->dw3,
                                 error_info[0], error_info[1],
@@ -1677,13 +1722,27 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
         }
 
 out:
+        hisi_sas_slot_task_free(hisi_hba, task, slot);
+        sts = ts->stat;
         spin_lock_irqsave(&task->task_state_lock, flags);
+        if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
+                spin_unlock_irqrestore(&task->task_state_lock, flags);
+                dev_info(dev, "slot complete: task(%p) aborted\n", task);
+                return SAS_ABORTED_TASK;
+        }
         task->task_state_flags |= SAS_TASK_STATE_DONE;
         spin_unlock_irqrestore(&task->task_state_lock, flags);
-        spin_lock_irqsave(&hisi_hba->lock, flags);
-        hisi_sas_slot_task_free(hisi_hba, task, slot);
-        spin_unlock_irqrestore(&hisi_hba->lock, flags);
-        sts = ts->stat;
+
+        if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
+                spin_lock_irqsave(&device->done_lock, flags);
+                if (test_bit(SAS_HA_FROZEN, &ha->state)) {
+                        spin_unlock_irqrestore(&device->done_lock, flags);
+                        dev_info(dev, "slot complete: task(%p) ignored\n ",
+                                 task);
+                        return sts;
+                }
+                spin_unlock_irqrestore(&device->done_lock, flags);
+        }
 
         if (task->task_done)
                 task->task_done(task);
@@ -1699,25 +1758,27 @@ static void cq_tasklet_v3_hw(unsigned long val)
         struct hisi_sas_complete_v3_hdr *complete_queue;
         u32 rd_point = cq->rd_point, wr_point;
         int queue = cq->id;
-        struct hisi_sas_dq *dq = &hisi_hba->dq[queue];
 
         complete_queue = hisi_hba->complete_hdr[queue];
 
-        spin_lock(&dq->lock);
         wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
                                    (0x14 * queue));
 
         while (rd_point != wr_point) {
                 struct hisi_sas_complete_v3_hdr *complete_hdr;
+                struct device *dev = hisi_hba->dev;
                 int iptt;
 
                 complete_hdr = &complete_queue[rd_point];
 
                 iptt = (complete_hdr->dw1) & CMPLT_HDR_IPTT_MSK;
-                slot = &hisi_hba->slot_info[iptt];
-                slot->cmplt_queue_slot = rd_point;
-                slot->cmplt_queue = queue;
-                slot_complete_v3_hw(hisi_hba, slot);
+                if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) {
+                        slot = &hisi_hba->slot_info[iptt];
+                        slot->cmplt_queue_slot = rd_point;
+                        slot->cmplt_queue = queue;
+                        slot_complete_v3_hw(hisi_hba, slot);
+                } else
+                        dev_err(dev, "IPTT %d is invalid, discard it.\n", iptt);
 
                 if (++rd_point >= HISI_SAS_QUEUE_SLOTS)
                         rd_point = 0;
@@ -1726,7 +1787,6 @@ static void cq_tasklet_v3_hw(unsigned long val)
         /* update rd_point */
         cq->rd_point = rd_point;
         hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
-        spin_unlock(&dq->lock);
 }
 
 static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
@@ -1839,39 +1899,12 @@ static int hisi_sas_v3_init(struct hisi_hba *hisi_hba)
 static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no,
                 struct sas_phy_linkrates *r)
 {
-        u32 prog_phy_link_rate =
-                hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
-        struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
-        struct asd_sas_phy *sas_phy = &phy->sas_phy;
-        int i;
-        enum sas_linkrate min, max;
-        u32 rate_mask = 0;
-
-        if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-                max = sas_phy->phy->maximum_linkrate;
-                min = r->minimum_linkrate;
-        } else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-                max = r->maximum_linkrate;
-                min = sas_phy->phy->minimum_linkrate;
-        } else
-                return;
-
-        sas_phy->phy->maximum_linkrate = max;
-        sas_phy->phy->minimum_linkrate = min;
-
-        max -= SAS_LINK_RATE_1_5_GBPS;
-
-        for (i = 0; i <= max; i++)
-                rate_mask |= 1 << (i * 2);
-
-        prog_phy_link_rate &= ~0xff;
-        prog_phy_link_rate |= rate_mask;
+        enum sas_linkrate max = r->maximum_linkrate;
+        u32 prog_phy_link_rate = 0x800;
 
-        disable_phy_v3_hw(hisi_hba, phy_no);
-        msleep(100);
+        prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
         hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
-                        prog_phy_link_rate);
-        start_phy_v3_hw(hisi_hba, phy_no);
+                             prog_phy_link_rate);
 }
 
 static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba)
@@ -1948,8 +1981,9 @@ static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE + AM_CTRL_GLOBAL, 0x1);
 
         /* wait until bus idle */
-        rc = readl_poll_timeout(hisi_hba->regs + AXI_MASTER_CFG_BASE +
-                AM_CURR_TRANS_RETURN, status, status == 0x3, 10, 100);
+        rc = hisi_sas_read32_poll_timeout(AXI_MASTER_CFG_BASE +
+                                          AM_CURR_TRANS_RETURN, status,
+                                          status == 0x3, 10, 100);
         if (rc) {
                 dev_err(dev, "axi bus is not idle, rc = %d\n", rc);
                 return rc;
@@ -1960,6 +1994,75 @@ static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
         return hw_init_v3_hw(hisi_hba);
 }
 
+static int write_gpio_v3_hw(struct hisi_hba *hisi_hba, u8 reg_type,
+                        u8 reg_index, u8 reg_count, u8 *write_data)
+{
+        struct device *dev = hisi_hba->dev;
+        u32 *data = (u32 *)write_data;
+        int i;
+
+        switch (reg_type) {
+        case SAS_GPIO_REG_TX:
+                if ((reg_index + reg_count) > ((hisi_hba->n_phy + 3) / 4)) {
+                        dev_err(dev, "write gpio: invalid reg range[%d, %d]\n",
+                                reg_index, reg_index + reg_count - 1);
+                        return -EINVAL;
+                }
+
+                for (i = 0; i < reg_count; i++)
+                        hisi_sas_write32(hisi_hba,
+                                         SAS_GPIO_TX_0_1 + (reg_index + i) * 4,
+                                         data[i]);
+                break;
+        default:
+                dev_err(dev, "write gpio: unsupported or bad reg type %d\n",
+                                reg_type);
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static void wait_cmds_complete_timeout_v3_hw(struct hisi_hba *hisi_hba,
+                                             int delay_ms, int timeout_ms)
+{
+        struct device *dev = hisi_hba->dev;
+        int entries, entries_old = 0, time;
+
+        for (time = 0; time < timeout_ms; time += delay_ms) {
+                entries = hisi_sas_read32(hisi_hba, CQE_SEND_CNT);
+                if (entries == entries_old)
+                        break;
+
+                entries_old = entries;
+                msleep(delay_ms);
+        }
+
+        dev_dbg(dev, "wait commands complete %dms\n", time);
+}
+
+static struct scsi_host_template sht_v3_hw = {
+        .name                   = DRV_NAME,
+        .module                 = THIS_MODULE,
+        .queuecommand           = sas_queuecommand,
+        .target_alloc           = sas_target_alloc,
+        .slave_configure        = hisi_sas_slave_configure,
+        .scan_finished          = hisi_sas_scan_finished,
+        .scan_start             = hisi_sas_scan_start,
+        .change_queue_depth     = sas_change_queue_depth,
+        .bios_param             = sas_bios_param,
+        .can_queue              = 1,
+        .this_id                = -1,
+        .sg_tablesize           = SG_ALL,
+        .max_sectors            = SCSI_DEFAULT_MAX_SECTORS,
+        .use_clustering         = ENABLE_CLUSTERING,
+        .eh_device_reset_handler = sas_eh_device_reset_handler,
+        .eh_target_reset_handler = sas_eh_target_reset_handler,
+        .target_destroy         = sas_target_destroy,
+        .ioctl                  = sas_ioctl,
+        .shost_attrs            = host_attrs,
+};
+
 static const struct hisi_sas_hw hisi_sas_v3_hw = {
         .hw_init = hisi_sas_v3_init,
         .setup_itct = setup_itct_v3_hw,
@@ -1985,6 +2088,8 @@ static const struct hisi_sas_hw hisi_sas_v3_hw = {
         .soft_reset = soft_reset_v3_hw,
         .get_phys_state = get_phys_state_v3_hw,
         .get_events = phy_get_events_v3_hw,
+        .write_gpio = write_gpio_v3_hw,
+        .wait_cmds_complete_timeout = wait_cmds_complete_timeout_v3_hw,
 };
 
 static struct Scsi_Host *
@@ -1994,7 +2099,7 @@ hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
         struct hisi_hba *hisi_hba;
         struct device *dev = &pdev->dev;
 
-        shost = scsi_host_alloc(hisi_sas_sht, sizeof(*hisi_hba));
+        shost = scsi_host_alloc(&sht_v3_hw, sizeof(*hisi_hba));
         if (!shost) {
                 dev_err(dev, "shost alloc failed\n");
                 return NULL;
@@ -2108,8 +2213,6 @@ hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
                 sha->sas_port[i] = &hisi_hba->port[i].sas_port;
         }
 
-        hisi_sas_init_add(hisi_hba);
-
         rc = scsi_add_host(shost, dev);
         if (rc)
                 goto err_out_ha;
@@ -2161,6 +2264,9 @@ static void hisi_sas_v3_remove(struct pci_dev *pdev)
         struct hisi_hba *hisi_hba = sha->lldd_ha;
         struct Scsi_Host *shost = sha->core.shost;
 
+        if (timer_pending(&hisi_hba->timer))
+                del_timer(&hisi_hba->timer);
+
         sas_unregister_ha(sha);
         sas_remove_host(sha->core.shost);
 
@@ -2222,6 +2328,29 @@ static const struct hisi_sas_hw_error sas_ras_intr1_nfe[] = {
         { .irq_msk = BIT(31), .msg = "DMAC7_RX_POISON" },
 };
 
+static const struct hisi_sas_hw_error sas_ras_intr2_nfe[] = {
+        { .irq_msk = BIT(0), .msg = "DMAC0_AXI_BUS_ERR" },
+        { .irq_msk = BIT(1), .msg = "DMAC1_AXI_BUS_ERR" },
+        { .irq_msk = BIT(2), .msg = "DMAC2_AXI_BUS_ERR" },
+        { .irq_msk = BIT(3), .msg = "DMAC3_AXI_BUS_ERR" },
+        { .irq_msk = BIT(4), .msg = "DMAC4_AXI_BUS_ERR" },
+        { .irq_msk = BIT(5), .msg = "DMAC5_AXI_BUS_ERR" },
+        { .irq_msk = BIT(6), .msg = "DMAC6_AXI_BUS_ERR" },
+        { .irq_msk = BIT(7), .msg = "DMAC7_AXI_BUS_ERR" },
+        { .irq_msk = BIT(8), .msg = "DMAC0_FIFO_OMIT_ERR" },
+        { .irq_msk = BIT(9), .msg = "DMAC1_FIFO_OMIT_ERR" },
+        { .irq_msk = BIT(10), .msg = "DMAC2_FIFO_OMIT_ERR" },
+        { .irq_msk = BIT(11), .msg = "DMAC3_FIFO_OMIT_ERR" },
+        { .irq_msk = BIT(12), .msg = "DMAC4_FIFO_OMIT_ERR" },
+        { .irq_msk = BIT(13), .msg = "DMAC5_FIFO_OMIT_ERR" },
+        { .irq_msk = BIT(14), .msg = "DMAC6_FIFO_OMIT_ERR" },
+        { .irq_msk = BIT(15), .msg = "DMAC7_FIFO_OMIT_ERR" },
+        { .irq_msk = BIT(16), .msg = "HGC_RLSE_SLOT_UNMATCH" },
+        { .irq_msk = BIT(17), .msg = "HGC_LM_ADD_FCH_LIST_ERR" },
+        { .irq_msk = BIT(18), .msg = "HGC_AXI_BUS_ERR" },
+        { .irq_msk = BIT(19), .msg = "HGC_FIFO_OMIT_ERR" },
+};
+
 static bool process_non_fatal_error_v3_hw(struct hisi_hba *hisi_hba)
 {
         struct device *dev = hisi_hba->dev;
@@ -2252,6 +2381,17 @@ static bool process_non_fatal_error_v3_hw(struct hisi_hba *hisi_hba)
         }
         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1, irq_value);
 
+        irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR2);
+        for (i = 0; i < ARRAY_SIZE(sas_ras_intr2_nfe); i++) {
+                ras_error = &sas_ras_intr2_nfe[i];
+                if (ras_error->irq_msk & irq_value) {
+                        dev_warn(dev, "SAS_RAS_INTR2: %s(irq_value=0x%x) found.\n",
+                                        ras_error->msg, irq_value);
+                        need_reset = true;
+                }
+        }
+        hisi_sas_write32(hisi_hba, SAS_RAS_INTR2, irq_value);
+
         return need_reset;
 }
 
@@ -2307,7 +2447,6 @@ static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
         u32 device_state, status;
         int rc;
         u32 reg_val;
-        unsigned long flags;
 
         if (!pdev->pm_cap) {
                 dev_err(dev, "PCI PM not supported\n");
@@ -2332,8 +2471,9 @@ static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
                 AM_CTRL_GLOBAL, reg_val);
 
         /* wait until bus idle */
-        rc = readl_poll_timeout(hisi_hba->regs + AXI_MASTER_CFG_BASE +
-                AM_CURR_TRANS_RETURN, status, status == 0x3, 10, 100);
+        rc = hisi_sas_read32_poll_timeout(AXI_MASTER_CFG_BASE +
+                                          AM_CURR_TRANS_RETURN, status,
+                                          status == 0x3, 10, 100);
         if (rc) {
                 dev_err(dev, "axi bus is not idle, rc = %d\n", rc);
                 clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
@@ -2351,9 +2491,7 @@ static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
         pci_disable_device(pdev);
         pci_set_power_state(pdev, device_state);
 
-        spin_lock_irqsave(&hisi_hba->lock, flags);
         hisi_sas_release_tasks(hisi_hba);
-        spin_unlock_irqrestore(&hisi_hba->lock, flags);
 
         sas_suspend_ha(sha);
         return 0;
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index dda1a64ab89c..6615ad8754b8 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -435,6 +435,8 @@ struct ipr_error_table_t ipr_error_table[] = {
         "4080: IOA exceeded maximum operating temperature"},
         {0x060B8000, 0, IPR_DEFAULT_LOG_LEVEL,
         "4085: Service required"},
+        {0x060B8100, 0, IPR_DEFAULT_LOG_LEVEL,
+        "4086: SAS Adapter Hardware Configuration Error"},
         {0x06288000, 0, IPR_DEFAULT_LOG_LEVEL,
         "3140: Device bus not ready to ready transition"},
         {0x06290000, 0, IPR_DEFAULT_LOG_LEVEL,
diff --git a/drivers/scsi/ips.c b/drivers/scsi/ips.c
index e3c8857741a1..bd6ac6b5980a 100644
--- a/drivers/scsi/ips.c
+++ b/drivers/scsi/ips.c
@@ -291,7 +291,7 @@ static void ips_freescb(ips_ha_t *, ips_scb_t *);
 static void ips_setup_funclist(ips_ha_t *);
 static void ips_statinit(ips_ha_t *);
 static void ips_statinit_memio(ips_ha_t *);
-static void ips_fix_ffdc_time(ips_ha_t *, ips_scb_t *, time_t);
+static void ips_fix_ffdc_time(ips_ha_t *, ips_scb_t *, time64_t);
 static void ips_ffdc_reset(ips_ha_t *, int);
 static void ips_ffdc_time(ips_ha_t *);
 static uint32_t ips_statupd_copperhead(ips_ha_t *);
@@ -985,10 +985,7 @@ static int __ips_eh_reset(struct scsi_cmnd *SC)
 
         /* FFDC */
         if (le32_to_cpu(ha->subsys->param[3]) & 0x300000) {
-                struct timeval tv;
-
-                do_gettimeofday(&tv);
-                ha->last_ffdc = tv.tv_sec;
+                ha->last_ffdc = ktime_get_real_seconds();
                 ha->reset_count++;
                 ips_ffdc_reset(ha, IPS_INTR_IORL);
         }
@@ -2392,7 +2389,6 @@ static int
 ips_hainit(ips_ha_t * ha)
 {
         int i;
-        struct timeval tv;
 
         METHOD_TRACE("ips_hainit", 1);
 
@@ -2407,8 +2403,7 @@ ips_hainit(ips_ha_t * ha)
 
         /* Send FFDC */
         ha->reset_count = 1;
-        do_gettimeofday(&tv);
-        ha->last_ffdc = tv.tv_sec;
+        ha->last_ffdc = ktime_get_real_seconds();
         ips_ffdc_reset(ha, IPS_INTR_IORL);
 
         if (!ips_read_config(ha, IPS_INTR_IORL)) {
@@ -2548,12 +2543,9 @@ ips_next(ips_ha_t * ha, int intr)
 
         if ((ha->subsys->param[3] & 0x300000)
             && (ha->scb_activelist.count == 0)) {
-                struct timeval tv;
-
-                do_gettimeofday(&tv);
-
-                if (tv.tv_sec - ha->last_ffdc > IPS_SECS_8HOURS) {
-                        ha->last_ffdc = tv.tv_sec;
+                time64_t now = ktime_get_real_seconds();
+                if (now - ha->last_ffdc > IPS_SECS_8HOURS) {
+                        ha->last_ffdc = now;
                         ips_ffdc_time(ha);
                 }
         }
@@ -5988,59 +5980,21 @@ ips_ffdc_time(ips_ha_t * ha)
 /*                                                                          */
 /****************************************************************************/
 static void
-ips_fix_ffdc_time(ips_ha_t * ha, ips_scb_t * scb, time_t current_time)
+ips_fix_ffdc_time(ips_ha_t * ha, ips_scb_t * scb, time64_t current_time)
 {
-        long days;
-        long rem;
-        int i;
-        int year;
-        int yleap;
-        int year_lengths[2] = { IPS_DAYS_NORMAL_YEAR, IPS_DAYS_LEAP_YEAR };
-        int month_lengths[12][2] = { {31, 31},
-        {28, 29},
-        {31, 31},
-        {30, 30},
-        {31, 31},
-        {30, 30},
-        {31, 31},
-        {31, 31},
-        {30, 30},
-        {31, 31},
-        {30, 30},
-        {31, 31}
-        };
+        struct tm tm;
 
         METHOD_TRACE("ips_fix_ffdc_time", 1);
 
-        days = current_time / IPS_SECS_DAY;
-        rem = current_time % IPS_SECS_DAY;
-
-        scb->cmd.ffdc.hour = (rem / IPS_SECS_HOUR);
-        rem = rem % IPS_SECS_HOUR;
-        scb->cmd.ffdc.minute = (rem / IPS_SECS_MIN);
-        scb->cmd.ffdc.second = (rem % IPS_SECS_MIN);
-
-        year = IPS_EPOCH_YEAR;
-        while (days < 0 || days >= year_lengths[yleap = IPS_IS_LEAP_YEAR(year)]) {
-                int newy;
-
-                newy = year + (days / IPS_DAYS_NORMAL_YEAR);
-                if (days < 0)
-                        --newy;
-                days -= (newy - year) * IPS_DAYS_NORMAL_YEAR +
-                    IPS_NUM_LEAP_YEARS_THROUGH(newy - 1) -
-                    IPS_NUM_LEAP_YEARS_THROUGH(year - 1);
-                year = newy;
-        }
-
-        scb->cmd.ffdc.yearH = year / 100;
-        scb->cmd.ffdc.yearL = year % 100;
-
-        for (i = 0; days >= month_lengths[i][yleap]; ++i)
-                days -= month_lengths[i][yleap];
+        time64_to_tm(current_time, 0, &tm);
 
-        scb->cmd.ffdc.month = i + 1;
-        scb->cmd.ffdc.day = days + 1;
+        scb->cmd.ffdc.hour   = tm.tm_hour;
+        scb->cmd.ffdc.minute = tm.tm_min;
+        scb->cmd.ffdc.second = tm.tm_sec;
+        scb->cmd.ffdc.yearH  = (tm.tm_year + 1900) / 100;
+        scb->cmd.ffdc.yearL  = tm.tm_year % 100;
+        scb->cmd.ffdc.month  = tm.tm_mon + 1;
+        scb->cmd.ffdc.day    = tm.tm_mday;
 }
 
 /****************************************************************************
diff --git a/drivers/scsi/ips.h b/drivers/scsi/ips.h
index 366be3b2f9b4..db546171e97f 100644
--- a/drivers/scsi/ips.h
+++ b/drivers/scsi/ips.h
@@ -402,16 +402,7 @@
    #define IPS_BIOS_HEADER             0xC0
 
    /* time oriented stuff */
-   #define IPS_IS_LEAP_YEAR(y)           (((y % 4 == 0) && ((y % 100 != 0) || (y % 400 == 0))) ? 1 : 0)
-   #define IPS_NUM_LEAP_YEARS_THROUGH(y) ((y) / 4 - (y) / 100 + (y) / 400)
-
-   #define IPS_SECS_MIN                 60
-   #define IPS_SECS_HOUR                3600
    #define IPS_SECS_8HOURS              28800
-   #define IPS_SECS_DAY                 86400
-   #define IPS_DAYS_NORMAL_YEAR         365
-   #define IPS_DAYS_LEAP_YEAR           366
-   #define IPS_EPOCH_YEAR               1970
 
    /*
     * Scsi_Host Template
@@ -1054,7 +1045,7 @@ typedef struct ips_ha {
    uint8_t            active;
    int                ioctl_reset;        /* IOCTL Requested Reset Flag */
    uint16_t           reset_count;        /* number of resets           */
-   time_t             last_ffdc;          /* last time we sent ffdc info*/
+   time64_t           last_ffdc;          /* last time we sent ffdc info*/
    uint8_t            slot_num;           /* PCI Slot Number            */
    int                ioctl_len;          /* size of ioctl buffer       */
    dma_addr_t         ioctl_busaddr;      /* dma address of ioctl buffer*/
diff --git a/drivers/scsi/isci/init.c b/drivers/scsi/isci/init.c
index 922e3e56c90d..05cf4daf8788 100644
--- a/drivers/scsi/isci/init.c
+++ b/drivers/scsi/isci/init.c
@@ -433,9 +433,6 @@ static enum sci_status sci_user_parameters_set(struct isci_host *ihost,
                       (u->max_speed_generation > SCIC_SDS_PARM_NO_SPEED)))
                         return SCI_FAILURE_INVALID_PARAMETER_VALUE;
 
-                if (u->in_connection_align_insertion_frequency < 3)
-                        return SCI_FAILURE_INVALID_PARAMETER_VALUE;
-
                 if ((u->in_connection_align_insertion_frequency < 3) ||
                     (u->align_insertion_frequency == 0) ||
                     (u->notify_enable_spin_up_insertion_frequency == 0))
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index 2ba4b68fdb73..b025a0b74341 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -962,7 +962,6 @@ static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev)
         if (conn->datadgst_en)
                 sdev->request_queue->backing_dev_info->capabilities
                         |= BDI_CAP_STABLE_WRITES;
-        blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
         blk_queue_dma_alignment(sdev->request_queue, 0);
         return 0;
 }
diff --git a/drivers/scsi/libsas/sas_ata.c b/drivers/scsi/libsas/sas_ata.c
index 0cc1567eacc1..ff1d612f6fb9 100644
--- a/drivers/scsi/libsas/sas_ata.c
+++ b/drivers/scsi/libsas/sas_ata.c
@@ -577,6 +577,11 @@ int sas_ata_init(struct domain_device *found_dev)
                 ata_sas_port_destroy(ap);
                 return rc;
         }
+        rc = ata_sas_tport_add(found_dev->sata_dev.ata_host.dev, ap);
+        if (rc) {
+                ata_sas_port_destroy(ap);
+                return rc;
+        }
         found_dev->sata_dev.ap = ap;
 
         return 0;
diff --git a/drivers/scsi/libsas/sas_discover.c b/drivers/scsi/libsas/sas_discover.c
index a0fa7ef3a071..1ffca28fe6a8 100644
--- a/drivers/scsi/libsas/sas_discover.c
+++ b/drivers/scsi/libsas/sas_discover.c
@@ -314,6 +314,7 @@ void sas_free_device(struct kref *kref)
                 kfree(dev->ex_dev.ex_phy);
 
         if (dev_is_sata(dev) && dev->sata_dev.ap) {
+                ata_sas_tport_delete(dev->sata_dev.ap);
                 ata_sas_port_destroy(dev->sata_dev.ap);
                 dev->sata_dev.ap = NULL;
         }
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index 6c0d351c0d0d..20b249a649dd 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -64,8 +64,6 @@ struct lpfc_sli2_slim;
 #define LPFC_IOCB_LIST_CNT      2250    /* list of IOCBs for fast-path usage. */
 #define LPFC_Q_RAMP_UP_INTERVAL 120     /* lun q_depth ramp up interval */
 #define LPFC_VNAME_LEN          100     /* vport symbolic name length */
-#define LPFC_TGTQ_INTERVAL      40000   /* Min amount of time between tgt
-                                           queue depth change in millisecs */
 #define LPFC_TGTQ_RAMPUP_PCENT  5       /* Target queue rampup in percentage */
 #define LPFC_MIN_TGT_QDEPTH     10
 #define LPFC_MAX_TGT_QDEPTH     0xFFFF
@@ -784,6 +782,7 @@ struct lpfc_hba {
         uint32_t cfg_nvme_oas;
         uint32_t cfg_nvme_embed_cmd;
         uint32_t cfg_nvme_io_channel;
+        uint32_t cfg_nvmet_mrq_post;
         uint32_t cfg_nvmet_mrq;
         uint32_t cfg_enable_nvmet;
         uint32_t cfg_nvme_enable_fb;
@@ -922,12 +921,6 @@ struct lpfc_hba {
         atomic_t fc4ScsiOutputRequests;
         atomic_t fc4ScsiControlRequests;
         atomic_t fc4ScsiIoCmpls;
-        atomic_t fc4NvmeInputRequests;
-        atomic_t fc4NvmeOutputRequests;
-        atomic_t fc4NvmeControlRequests;
-        atomic_t fc4NvmeIoCmpls;
-        atomic_t fc4NvmeLsRequests;
-        atomic_t fc4NvmeLsCmpls;
 
         uint64_t bg_guard_err_cnt;
         uint64_t bg_apptag_err_cnt;
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index 2ac1d21c553f..729d343861f4 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -149,10 +149,14 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
         struct lpfc_nvmet_tgtport *tgtp;
         struct nvme_fc_local_port *localport;
         struct lpfc_nvme_lport *lport;
+        struct lpfc_nvme_rport *rport;
         struct lpfc_nodelist *ndlp;
         struct nvme_fc_remote_port *nrport;
-        uint64_t data1, data2, data3, tot;
+        struct lpfc_nvme_ctrl_stat *cstat;
+        uint64_t data1, data2, data3;
+        uint64_t totin, totout, tot;
         char *statep;
+        int i;
         int len = 0;
 
         if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) {
@@ -293,6 +297,13 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
         len = snprintf(buf, PAGE_SIZE, "NVME Initiator Enabled\n");
 
         spin_lock_irq(shost->host_lock);
+        len += snprintf(buf + len, PAGE_SIZE - len,
+                        "XRI Dist lpfc%d Total %d NVME %d SCSI %d ELS %d\n",
+                        phba->brd_no,
+                        phba->sli4_hba.max_cfg_param.max_xri,
+                        phba->sli4_hba.nvme_xri_max,
+                        phba->sli4_hba.scsi_xri_max,
+                        lpfc_sli4_get_els_iocb_cnt(phba));
 
         /* Port state is only one of two values for now. */
         if (localport->port_id)
@@ -309,11 +320,14 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
                         localport->port_id, statep);
 
         list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) {
-                if (!ndlp->nrport)
+                rport = lpfc_ndlp_get_nrport(ndlp);
+                if (!rport)
                         continue;
 
                 /* local short-hand pointer. */
-                nrport = ndlp->nrport->remoteport;
+                nrport = rport->remoteport;
+                if (!nrport)
+                        continue;
 
                 /* Port state is only one of two values for now. */
                 switch (nrport->port_state) {
@@ -364,11 +378,14 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
         }
         spin_unlock_irq(shost->host_lock);
 
+        if (!lport)
+                return len;
+
         len += snprintf(buf + len, PAGE_SIZE - len, "\nNVME Statistics\n");
         len += snprintf(buf+len, PAGE_SIZE-len,
                         "LS: Xmt %010x Cmpl %010x Abort %08x\n",
-                        atomic_read(&phba->fc4NvmeLsRequests),
-                        atomic_read(&phba->fc4NvmeLsCmpls),
+                        atomic_read(&lport->fc4NvmeLsRequests),
+                        atomic_read(&lport->fc4NvmeLsCmpls),
                         atomic_read(&lport->xmt_ls_abort));
 
         len += snprintf(buf + len, PAGE_SIZE - len,
@@ -377,28 +394,33 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
                         atomic_read(&lport->cmpl_ls_xb),
                         atomic_read(&lport->cmpl_ls_err));
 
-        tot = atomic_read(&phba->fc4NvmeIoCmpls);
-        data1 = atomic_read(&phba->fc4NvmeInputRequests);
-        data2 = atomic_read(&phba->fc4NvmeOutputRequests);
-        data3 = atomic_read(&phba->fc4NvmeControlRequests);
+        totin = 0;
+        totout = 0;
+        for (i = 0; i < phba->cfg_nvme_io_channel; i++) {
+                cstat = &lport->cstat[i];
+                tot = atomic_read(&cstat->fc4NvmeIoCmpls);
+                totin += tot;
+                data1 = atomic_read(&cstat->fc4NvmeInputRequests);
+                data2 = atomic_read(&cstat->fc4NvmeOutputRequests);
+                data3 = atomic_read(&cstat->fc4NvmeControlRequests);
+                totout += (data1 + data2 + data3);
+        }
         len += snprintf(buf+len, PAGE_SIZE-len,
-                        "FCP: Rd %016llx Wr %016llx IO %016llx\n",
-                        data1, data2, data3);
+                        "Total FCP Cmpl %016llx Issue %016llx "
+                        "OutIO %016llx\n",
+                        totin, totout, totout - totin);
 
         len += snprintf(buf+len, PAGE_SIZE-len,
-                        "    noxri %08x nondlp %08x qdepth %08x "
-                        "wqerr %08x\n",
+                        "      abort %08x noxri %08x nondlp %08x qdepth %08x "
+                        "wqerr %08x err %08x\n",
+                        atomic_read(&lport->xmt_fcp_abort),
                         atomic_read(&lport->xmt_fcp_noxri),
                         atomic_read(&lport->xmt_fcp_bad_ndlp),
                         atomic_read(&lport->xmt_fcp_qdepth),
+                        atomic_read(&lport->xmt_fcp_err),
                         atomic_read(&lport->xmt_fcp_wqerr));
 
         len += snprintf(buf + len, PAGE_SIZE - len,
-                        "    Cmpl %016llx Outstanding %016llx Abort %08x\n",
-                        tot, ((data1 + data2 + data3) - tot),
-                        atomic_read(&lport->xmt_fcp_abort));
-
-        len += snprintf(buf + len, PAGE_SIZE - len,
                         "FCP CMPL: xb %08x Err %08x\n",
                         atomic_read(&lport->cmpl_fcp_xb),
                         atomic_read(&lport->cmpl_fcp_err));
@@ -3280,6 +3302,9 @@ lpfc_update_rport_devloss_tmo(struct lpfc_vport *vport)
 {
         struct Scsi_Host  *shost;
         struct lpfc_nodelist  *ndlp;
+#if (IS_ENABLED(CONFIG_NVME_FC))
+        struct lpfc_nvme_rport *rport;
+#endif
 
         shost = lpfc_shost_from_vport(vport);
         spin_lock_irq(shost->host_lock);
@@ -3289,8 +3314,9 @@ lpfc_update_rport_devloss_tmo(struct lpfc_vport *vport)
                 if (ndlp->rport)
                         ndlp->rport->dev_loss_tmo = vport->cfg_devloss_tmo;
 #if (IS_ENABLED(CONFIG_NVME_FC))
-                if (ndlp->nrport)
-                        nvme_fc_set_remoteport_devloss(ndlp->nrport->remoteport,
+                rport = lpfc_ndlp_get_nrport(ndlp);
+                if (rport)
+                        nvme_fc_set_remoteport_devloss(rport->remoteport,
                                                        vport->cfg_devloss_tmo);
 #endif
         }
@@ -3414,6 +3440,15 @@ LPFC_ATTR_R(nvmet_mrq,
             "Specify number of RQ pairs for processing NVMET cmds");
 
 /*
+ * lpfc_nvmet_mrq_post: Specify number of RQ buffer to initially post
+ * to each NVMET RQ. Range 64 to 2048, default is 512.
+ */
+LPFC_ATTR_R(nvmet_mrq_post,
+            LPFC_NVMET_RQE_DEF_POST, LPFC_NVMET_RQE_MIN_POST,
+            LPFC_NVMET_RQE_DEF_COUNT,
+            "Specify number of RQ buffers to initially post");
+
+/*
  * lpfc_enable_fc4_type: Defines what FC4 types are supported.
  * Supported Values:  1 - register just FCP
  *                    3 - register both FCP and NVME
@@ -3469,8 +3504,49 @@ LPFC_VPORT_ATTR_R(lun_queue_depth, 30, 1, 512,
 # tgt_queue_depth:  This parameter is used to limit the number of outstanding
 # commands per target port. Value range is [10,65535]. Default value is 65535.
 */
-LPFC_VPORT_ATTR_RW(tgt_queue_depth, 65535, 10, 65535,
-                   "Max number of FCP commands we can queue to a specific target port");
+static uint lpfc_tgt_queue_depth = LPFC_MAX_TGT_QDEPTH;
+module_param(lpfc_tgt_queue_depth, uint, 0444);
+MODULE_PARM_DESC(lpfc_tgt_queue_depth, "Set max Target queue depth");
+lpfc_vport_param_show(tgt_queue_depth);
+lpfc_vport_param_init(tgt_queue_depth, LPFC_MAX_TGT_QDEPTH,
+                      LPFC_MIN_TGT_QDEPTH, LPFC_MAX_TGT_QDEPTH);
+
+/**
+ * lpfc_tgt_queue_depth_store: Sets an attribute value.
+ * @phba: pointer the the adapter structure.
+ * @val: integer attribute value.
+ *
+ * Description: Sets the parameter to the new value.
+ *
+ * Returns:
+ * zero on success
+ * -EINVAL if val is invalid
+ */
+static int
+lpfc_tgt_queue_depth_set(struct lpfc_vport *vport, uint val)
+{
+        struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
+        struct lpfc_nodelist *ndlp;
+
+        if (!lpfc_rangecheck(val, LPFC_MIN_TGT_QDEPTH, LPFC_MAX_TGT_QDEPTH))
+                return -EINVAL;
+
+        if (val == vport->cfg_tgt_queue_depth)
+                return 0;
+
+        spin_lock_irq(shost->host_lock);
+        vport->cfg_tgt_queue_depth = val;
+
+        /* Next loop thru nodelist and change cmd_qdepth */
+        list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp)
+                ndlp->cmd_qdepth = vport->cfg_tgt_queue_depth;
+
+        spin_unlock_irq(shost->host_lock);
+        return 0;
+}
+
+lpfc_vport_param_store(tgt_queue_depth);
+static DEVICE_ATTR_RW(lpfc_tgt_queue_depth);
 
 /*
 # hba_queue_depth:  This parameter is used to limit the number of outstanding
@@ -5302,6 +5378,7 @@ struct device_attribute *lpfc_hba_attrs[] = {
         &dev_attr_lpfc_suppress_rsp,
         &dev_attr_lpfc_nvme_io_channel,
         &dev_attr_lpfc_nvmet_mrq,
+        &dev_attr_lpfc_nvmet_mrq_post,
         &dev_attr_lpfc_nvme_enable_fb,
         &dev_attr_lpfc_nvmet_fb_size,
         &dev_attr_lpfc_enable_bg,
@@ -6352,6 +6429,7 @@ lpfc_get_cfgparam(struct lpfc_hba *phba)
 
         lpfc_enable_fc4_type_init(phba, lpfc_enable_fc4_type);
         lpfc_nvmet_mrq_init(phba, lpfc_nvmet_mrq);
+        lpfc_nvmet_mrq_post_init(phba, lpfc_nvmet_mrq_post);
 
         /* Initialize first burst. Target vs Initiator are different. */
         lpfc_nvme_enable_fb_init(phba, lpfc_nvme_enable_fb);
diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c
index 0f174ca80f67..edb1a18a6414 100644
--- a/drivers/scsi/lpfc/lpfc_bsg.c
+++ b/drivers/scsi/lpfc/lpfc_bsg.c
@@ -3621,7 +3621,7 @@ lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
                 bsg_reply->result = 0;
 
                 lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
-                                "2937 SLI_CONFIG ext-buffer maibox command "
+                                "2937 SLI_CONFIG ext-buffer mailbox command "
                                 "(x%x/x%x) complete bsg job done, bsize:%d\n",
                                 phba->mbox_ext_buf_ctx.nembType,
                                 phba->mbox_ext_buf_ctx.mboxType, size);
@@ -3632,7 +3632,7 @@ lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
                                         phba->mbox_ext_buf_ctx.mbx_dmabuf, 0);
         } else {
                 lpfc_printf_log(phba, KERN_ERR, LOG_LIBDFC,
-                                "2938 SLI_CONFIG ext-buffer maibox "
+                                "2938 SLI_CONFIG ext-buffer mailbox "
                                 "command (x%x/x%x) failure, rc:x%x\n",
                                 phba->mbox_ext_buf_ctx.nembType,
                                 phba->mbox_ext_buf_ctx.mboxType, rc);
@@ -3666,7 +3666,7 @@ lpfc_bsg_issue_read_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
                 pmboxq->u.mb.mbxStatus = MBXERR_ERROR;
 
         lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
-                        "2939 SLI_CONFIG ext-buffer rd maibox command "
+                        "2939 SLI_CONFIG ext-buffer rd mailbox command "
                         "complete, ctxState:x%x, mbxStatus:x%x\n",
                         phba->mbox_ext_buf_ctx.state, pmboxq->u.mb.mbxStatus);
 
@@ -3706,7 +3706,7 @@ lpfc_bsg_issue_write_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
                 pmboxq->u.mb.mbxStatus = MBXERR_ERROR;
 
         lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
-                        "2940 SLI_CONFIG ext-buffer wr maibox command "
+                        "2940 SLI_CONFIG ext-buffer wr mailbox command "
                         "complete, ctxState:x%x, mbxStatus:x%x\n",
                         phba->mbox_ext_buf_ctx.state, pmboxq->u.mb.mbxStatus);
 
@@ -3988,12 +3988,12 @@ lpfc_bsg_sli_cfg_read_cmd_ext(struct lpfc_hba *phba, struct bsg_job *job,
         if ((rc == MBX_SUCCESS) || (rc == MBX_BUSY)) {
                 lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
                                 "2947 Issued SLI_CONFIG ext-buffer "
-                                "maibox command, rc:x%x\n", rc);
+                                "mailbox command, rc:x%x\n", rc);
                 return SLI_CONFIG_HANDLED;
         }
         lpfc_printf_log(phba, KERN_ERR, LOG_LIBDFC,
                         "2948 Failed to issue SLI_CONFIG ext-buffer "
-                        "maibox command, rc:x%x\n", rc);
+                        "mailbox command, rc:x%x\n", rc);
         rc = -EPIPE;
 
 job_error:
@@ -4147,12 +4147,12 @@ lpfc_bsg_sli_cfg_write_cmd_ext(struct lpfc_hba *phba, struct bsg_job *job,
                 if ((rc == MBX_SUCCESS) || (rc == MBX_BUSY)) {
                         lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
                                         "2955 Issued SLI_CONFIG ext-buffer "
-                                        "maibox command, rc:x%x\n", rc);
+                                        "mailbox command, rc:x%x\n", rc);
                         return SLI_CONFIG_HANDLED;
                 }
                 lpfc_printf_log(phba, KERN_ERR, LOG_LIBDFC,
                                 "2956 Failed to issue SLI_CONFIG ext-buffer "
-                                "maibox command, rc:x%x\n", rc);
+                                "mailbox command, rc:x%x\n", rc);
                 rc = -EPIPE;
                 goto job_error;
         }
@@ -4492,12 +4492,12 @@ lpfc_bsg_write_ebuf_set(struct lpfc_hba *phba, struct bsg_job *job,
                 if ((rc == MBX_SUCCESS) || (rc == MBX_BUSY)) {
                         lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
                                         "2969 Issued SLI_CONFIG ext-buffer "
-                                        "maibox command, rc:x%x\n", rc);
+                                        "mailbox command, rc:x%x\n", rc);
                         return SLI_CONFIG_HANDLED;
                 }
                 lpfc_printf_log(phba, KERN_ERR, LOG_LIBDFC,
                                 "2970 Failed to issue SLI_CONFIG ext-buffer "
-                                "maibox command, rc:x%x\n", rc);
+                                "mailbox command, rc:x%x\n", rc);
                 rc = -EPIPE;
                 goto job_error;
         }
diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c
index 0617c8ea88c6..d4a200ae5a6f 100644
--- a/drivers/scsi/lpfc/lpfc_ct.c
+++ b/drivers/scsi/lpfc/lpfc_ct.c
@@ -471,6 +471,11 @@ lpfc_prep_node_fc4type(struct lpfc_vport *vport, uint32_t Did, uint8_t fc4_type)
                                 "Parse GID_FTrsp: did:x%x flg:x%x x%x",
                                 Did, ndlp->nlp_flag, vport->fc_flag);
 
+                        /* Don't assume the rport is always the previous
+                         * FC4 type.
+                         */
+                        ndlp->nlp_fc4_type &= ~(NLP_FC4_FCP | NLP_FC4_NVME);
+
                         /* By default, the driver expects to support FCP FC4 */
                         if (fc4_type == FC_TYPE_FCP)
                                 ndlp->nlp_fc4_type |= NLP_FC4_FCP;
@@ -691,6 +696,11 @@ lpfc_cmpl_ct_cmd_gid_ft(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
                 vport->fc_flag &= ~FC_RSCN_DEFERRED;
                 spin_unlock_irq(shost->host_lock);
 
+                /* This is a GID_FT completing so the gidft_inp counter was
+                 * incremented before the GID_FT was issued to the wire.
+                 */
+                vport->gidft_inp--;
+
                 /*
                  * Skip processing the NS response
                  * Re-issue the NS cmd
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c
index fb0dc2aeed91..9df0c051349f 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.c
+++ b/drivers/scsi/lpfc/lpfc_debugfs.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2007-2015 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -544,7 +544,7 @@ static int
 lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 {
         int len = 0;
-        int cnt;
+        int i, iocnt, outio, cnt;
         struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
         struct lpfc_hba  *phba = vport->phba;
         struct lpfc_nodelist *ndlp;
@@ -552,12 +552,15 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
         struct nvme_fc_local_port *localport;
         struct lpfc_nvmet_tgtport *tgtp;
         struct nvme_fc_remote_port *nrport;
+        struct lpfc_nvme_rport *rport;
 
         cnt = (LPFC_NODELIST_SIZE / LPFC_NODELIST_ENTRY_SIZE);
+        outio = 0;
 
         len += snprintf(buf+len, size-len, "\nFCP Nodelist Entries ...\n");
         spin_lock_irq(shost->host_lock);
         list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) {
+                iocnt = 0;
                 if (!cnt) {
                         len +=  snprintf(buf+len, size-len,
                                 "Missing Nodelist Entries\n");
@@ -585,9 +588,11 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
                         break;
                 case NLP_STE_UNMAPPED_NODE:
                         statep = "UNMAP ";
+                        iocnt = 1;
                         break;
                 case NLP_STE_MAPPED_NODE:
                         statep = "MAPPED";
+                        iocnt = 1;
                         break;
                 case NLP_STE_NPR_NODE:
                         statep = "NPR   ";
@@ -614,8 +619,10 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
                         len += snprintf(buf+len, size-len, "UNKNOWN_TYPE ");
                 if (ndlp->nlp_type & NLP_FC_NODE)
                         len += snprintf(buf+len, size-len, "FC_NODE ");
-                if (ndlp->nlp_type & NLP_FABRIC)
+                if (ndlp->nlp_type & NLP_FABRIC) {
                         len += snprintf(buf+len, size-len, "FABRIC ");
+                        iocnt = 0;
+                }
                 if (ndlp->nlp_type & NLP_FCP_TARGET)
                         len += snprintf(buf+len, size-len, "FCP_TGT sid:%d ",
                                 ndlp->nlp_sid);
@@ -632,10 +639,20 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
                         ndlp->nlp_usg_map);
                 len += snprintf(buf+len, size-len, "refcnt:%x",
                         kref_read(&ndlp->kref));
+                if (iocnt) {
+                        i = atomic_read(&ndlp->cmd_pending);
+                        len += snprintf(buf + len, size - len,
+                                        " OutIO:x%x Qdepth x%x",
+                                        i, ndlp->cmd_qdepth);
+                        outio += i;
+                }
                 len +=  snprintf(buf+len, size-len, "\n");
         }
         spin_unlock_irq(shost->host_lock);
 
+        len += snprintf(buf + len, size - len,
+                        "\nOutstanding IO x%x\n",  outio);
+
         if (phba->nvmet_support && phba->targetport && (vport == phba->pport)) {
                 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
                 len += snprintf(buf + len, size - len,
@@ -679,10 +696,13 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
         len += snprintf(buf + len, size - len, "\tRport List:\n");
         list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) {
                 /* local short-hand pointer. */
-                if (!ndlp->nrport)
+                rport = lpfc_ndlp_get_nrport(ndlp);
+                if (!rport)
                         continue;
 
-                nrport = ndlp->nrport->remoteport;
+                nrport = rport->remoteport;
+                if (!nrport)
+                        continue;
 
                 /* Port state is only one of two values for now. */
                 switch (nrport->port_state) {
@@ -751,10 +771,12 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
         struct lpfc_nvmet_tgtport *tgtp;
         struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
         struct nvme_fc_local_port *localport;
+        struct lpfc_nvme_ctrl_stat *cstat;
         struct lpfc_nvme_lport *lport;
-        uint64_t tot, data1, data2, data3;
+        uint64_t data1, data2, data3;
+        uint64_t tot, totin, totout;
+        int cnt, i, maxch;
         int len = 0;
-        int cnt;
 
         if (phba->nvmet_support) {
                 if (!phba->targetport)
@@ -880,33 +902,52 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
                 if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
                         return len;
 
+                localport = vport->localport;
+                if (!localport)
+                        return len;
+                lport = (struct lpfc_nvme_lport *)localport->private;
+                if (!lport)
+                        return len;
+
                 len += snprintf(buf + len, size - len,
                                 "\nNVME Lport Statistics\n");
 
                 len += snprintf(buf + len, size - len,
                                 "LS: Xmt %016x Cmpl %016x\n",
-                                atomic_read(&phba->fc4NvmeLsRequests),
-                                atomic_read(&phba->fc4NvmeLsCmpls));
-
-                tot = atomic_read(&phba->fc4NvmeIoCmpls);
-                data1 = atomic_read(&phba->fc4NvmeInputRequests);
-                data2 = atomic_read(&phba->fc4NvmeOutputRequests);
-                data3 = atomic_read(&phba->fc4NvmeControlRequests);
+                                atomic_read(&lport->fc4NvmeLsRequests),
+                                atomic_read(&lport->fc4NvmeLsCmpls));
 
-                len += snprintf(buf + len, size - len,
-                                "FCP: Rd %016llx Wr %016llx IO %016llx\n",
-                                data1, data2, data3);
-
-                len += snprintf(buf + len, size - len,
-                                "   Cmpl %016llx Outstanding %016llx\n",
-                                tot, (data1 + data2 + data3) - tot);
+                if (phba->cfg_nvme_io_channel < 32)
+                        maxch = phba->cfg_nvme_io_channel;
+                else
+                        maxch = 32;
+                totin = 0;
+                totout = 0;
+                for (i = 0; i < phba->cfg_nvme_io_channel; i++) {
+                        cstat = &lport->cstat[i];
+                        tot = atomic_read(&cstat->fc4NvmeIoCmpls);
+                        totin += tot;
+                        data1 = atomic_read(&cstat->fc4NvmeInputRequests);
+                        data2 = atomic_read(&cstat->fc4NvmeOutputRequests);
+                        data3 = atomic_read(&cstat->fc4NvmeControlRequests);
+                        totout += (data1 + data2 + data3);
+
+                        /* Limit to 32, debugfs display buffer limitation */
+                        if (i >= 32)
+                                continue;
 
-                localport = vport->localport;
-                if (!localport)
-                        return len;
-                lport = (struct lpfc_nvme_lport *)localport->private;
-                if (!lport)
-                        return len;
+                        len += snprintf(buf + len, PAGE_SIZE - len,
+                                        "FCP (%d): Rd %016llx Wr %016llx "
+                                        "IO %016llx ",
+                                        i, data1, data2, data3);
+                        len += snprintf(buf + len, PAGE_SIZE - len,
+                                        "Cmpl %016llx OutIO %016llx\n",
+                                        tot, ((data1 + data2 + data3) - tot));
+                }
+                len += snprintf(buf + len, PAGE_SIZE - len,
+                                "Total FCP Cmpl %016llx Issue %016llx "
+                                "OutIO %016llx\n",
+                                totin, totout, totout - totin);
 
                 len += snprintf(buf + len, size - len,
                                 "LS Xmt Err: Abrt %08x Err %08x  "
@@ -918,11 +959,12 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 
                 len += snprintf(buf + len, size - len,
                                 "FCP Xmt Err: noxri %06x nondlp %06x "
-                                "qdepth %06x wqerr %06x Abrt %06x\n",
+                                "qdepth %06x wqerr %06x err %06x Abrt %06x\n",
                                 atomic_read(&lport->xmt_fcp_noxri),
                                 atomic_read(&lport->xmt_fcp_bad_ndlp),
                                 atomic_read(&lport->xmt_fcp_qdepth),
                                 atomic_read(&lport->xmt_fcp_wqerr),
+                                atomic_read(&lport->xmt_fcp_err),
                                 atomic_read(&lport->xmt_fcp_abort));
 
                 len += snprintf(buf + len, size - len,
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index 74895e62aaea..6d84a10fef07 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -6268,7 +6268,6 @@ lpfc_els_handle_rscn(struct lpfc_vport *vport)
                  * flush the RSCN.  Otherwise, the outstanding requests
                  * need to complete.
                  */
-                vport->gidft_inp = 0;
                 if (lpfc_issue_gidft(vport) > 0)
                         return 1;
         } else {
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index 3e7712cd6c9a..2fef54fab86d 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -708,8 +708,7 @@ lpfc_work_done(struct lpfc_hba *phba)
                                                                 HA_RXMASK));
                         }
                 }
-                if ((phba->sli_rev == LPFC_SLI_REV4) &&
-                                 (!list_empty(&pring->txq)))
+                if (phba->sli_rev == LPFC_SLI_REV4)
                         lpfc_drain_txq(phba);
                 /*
                  * Turn on Ring interrupts
@@ -3876,10 +3875,6 @@ int
 lpfc_issue_gidft(struct lpfc_vport *vport)
 {
         struct lpfc_hba *phba = vport->phba;
-        struct lpfc_nodelist *ndlp;
-
-        list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp)
-                ndlp->nlp_fc4_type &= ~(NLP_FC4_FCP | NLP_FC4_NVME);
 
         /* Good status, issue CT Request to NameServer */
         if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h
index 98b80559c215..f43f0bacb77a 100644
--- a/drivers/scsi/lpfc/lpfc_hw4.h
+++ b/drivers/scsi/lpfc/lpfc_hw4.h
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2009-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -566,6 +566,7 @@ struct lpfc_register {
 
 /* The following BAR0 register sets are defined for if_type 0 and 2 UCNAs. */
 #define LPFC_SLI_INTF                   0x0058
+#define LPFC_SLI_ASIC_VER               0x009C
 
 #define LPFC_CTL_PORT_SEM_OFFSET        0x400
 #define lpfc_port_smphr_perr_SHIFT      31
@@ -3912,6 +3913,7 @@ struct lpfc_acqe_link {
 #define LPFC_ASYNC_LINK_FAULT_NONE      0x0
 #define LPFC_ASYNC_LINK_FAULT_LOCAL     0x1
 #define LPFC_ASYNC_LINK_FAULT_REMOTE    0x2
+#define LPFC_ASYNC_LINK_FAULT_LR_LRR    0x3
 #define lpfc_acqe_logical_link_speed_SHIFT      16
 #define lpfc_acqe_logical_link_speed_MASK       0x0000FFFF
 #define lpfc_acqe_logical_link_speed_WORD       word1
@@ -4616,6 +4618,9 @@ union lpfc_wqe128 {
         struct send_frame_wqe send_frame;
 };
 
+#define MAGIC_NUMER_G6 0xFEAA0003
+#define MAGIC_NUMER_G7 0xFEAA0005
+
 struct lpfc_grp_hdr {
         uint32_t size;
         uint32_t magic_number;
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 7887468c71b4..7ae343b14630 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -1266,6 +1266,9 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba)
         uint64_t tot, data1, data2, data3;
         struct lpfc_nvmet_tgtport *tgtp;
         struct lpfc_register reg_data;
+        struct nvme_fc_local_port *localport;
+        struct lpfc_nvme_lport *lport;
+        struct lpfc_nvme_ctrl_stat *cstat;
         void __iomem *eqdreg = phba->sli4_hba.u.if_type2.EQDregaddr;
 
         vports = lpfc_create_vport_work_array(phba);
@@ -1299,14 +1302,25 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba)
                                 tot += atomic_read(&tgtp->xmt_fcp_release);
                                 tot = atomic_read(&tgtp->rcv_fcp_cmd_in) - tot;
                         } else {
-                                tot = atomic_read(&phba->fc4NvmeIoCmpls);
-                                data1 = atomic_read(
-                                        &phba->fc4NvmeInputRequests);
-                                data2 = atomic_read(
-                                        &phba->fc4NvmeOutputRequests);
-                                data3 = atomic_read(
-                                        &phba->fc4NvmeControlRequests);
-                                tot =  (data1 + data2 + data3) - tot;
+                                localport = phba->pport->localport;
+                                if (!localport || !localport->private)
+                                        goto skip_eqdelay;
+                                lport = (struct lpfc_nvme_lport *)
+                                        localport->private;
+                                tot = 0;
+                                for (i = 0;
+                                        i < phba->cfg_nvme_io_channel; i++) {
+                                        cstat = &lport->cstat[i];
+                                        data1 = atomic_read(
+                                                &cstat->fc4NvmeInputRequests);
+                                        data2 = atomic_read(
+                                                &cstat->fc4NvmeOutputRequests);
+                                        data3 = atomic_read(
+                                                &cstat->fc4NvmeControlRequests);
+                                        tot += (data1 + data2 + data3);
+                                        tot -= atomic_read(
+                                                &cstat->fc4NvmeIoCmpls);
+                                }
                         }
                 }
 
@@ -4265,32 +4279,24 @@ lpfc_sli4_fcf_redisc_wait_tmo(struct timer_list *t)
  * @phba: pointer to lpfc hba data structure.
  * @acqe_link: pointer to the async link completion queue entry.
  *
- * This routine is to parse the SLI4 link-attention link fault code and
- * translate it into the base driver's read link attention mailbox command
- * status.
- *
- * Return: Link-attention status in terms of base driver's coding.
+ * This routine is to parse the SLI4 link-attention link fault code.
  **/
-static uint16_t
+static void
 lpfc_sli4_parse_latt_fault(struct lpfc_hba *phba,
                            struct lpfc_acqe_link *acqe_link)
 {
-        uint16_t latt_fault;
-
         switch (bf_get(lpfc_acqe_link_fault, acqe_link)) {
         case LPFC_ASYNC_LINK_FAULT_NONE:
         case LPFC_ASYNC_LINK_FAULT_LOCAL:
         case LPFC_ASYNC_LINK_FAULT_REMOTE:
-                latt_fault = 0;
+        case LPFC_ASYNC_LINK_FAULT_LR_LRR:
                 break;
         default:
                 lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-                                "0398 Invalid link fault code: x%x\n",
+                                "0398 Unknown link fault code: x%x\n",
                                 bf_get(lpfc_acqe_link_fault, acqe_link));
-                latt_fault = MBXERR_ERROR;
                 break;
         }
-        return latt_fault;
 }
 
 /**
@@ -4565,9 +4571,12 @@ lpfc_sli4_async_link_evt(struct lpfc_hba *phba,
          * the READ_TOPOLOGY completion routine to continue without actually
          * sending the READ_TOPOLOGY mailbox command to the port.
          */
-        /* Parse and translate status field */
+        /* Initialize completion status */
         mb = &pmb->u.mb;
-        mb->mbxStatus = lpfc_sli4_parse_latt_fault(phba, acqe_link);
+        mb->mbxStatus = MBX_SUCCESS;
+
+        /* Parse port fault information field */
+        lpfc_sli4_parse_latt_fault(phba, acqe_link);
 
         /* Parse and translate link attention fields */
         la = (struct lpfc_mbx_read_top *) &pmb->u.mb.un.varReadTop;
@@ -4695,10 +4704,12 @@ lpfc_sli4_async_fc_evt(struct lpfc_hba *phba, struct lpfc_acqe_fc_la *acqe_fc)
                         break;
                 }
 
-                /* Parse and translate status field */
+                /* Initialize completion status */
                 mb = &pmb->u.mb;
-                mb->mbxStatus = lpfc_sli4_parse_latt_fault(phba,
-                                                           (void *)acqe_fc);
+                mb->mbxStatus = MBX_SUCCESS;
+
+                /* Parse port fault information field */
+                lpfc_sli4_parse_latt_fault(phba, (void *)acqe_fc);
 
                 /* Parse and translate link attention fields */
                 la = (struct lpfc_mbx_read_top *)&pmb->u.mb.un.varReadTop;
@@ -5103,7 +5114,7 @@ lpfc_sli4_async_fip_evt(struct lpfc_hba *phba,
                 if (rc) {
                         lpfc_printf_log(phba, KERN_ERR, LOG_FIP |
                                         LOG_DISCOVERY,
-                                        "2772 Issue FCF rediscover mabilbox "
+                                        "2772 Issue FCF rediscover mailbox "
                                         "command failed, fail through to FCF "
                                         "dead event\n");
                         spin_lock_irq(&phba->hbalock);
@@ -5195,7 +5206,7 @@ lpfc_sli4_async_fip_evt(struct lpfc_hba *phba,
                                 lpfc_printf_log(phba, KERN_ERR, LOG_FIP |
                                                 LOG_DISCOVERY,
                                                 "2774 Issue FCF rediscover "
-                                                "mabilbox command failed, "
+                                                "mailbox command failed, "
                                                 "through to CVL event\n");
                                 spin_lock_irq(&phba->hbalock);
                                 phba->fcf.fcf_flag &= ~FCF_ACVL_DISC;
@@ -5839,6 +5850,8 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
         int fof_vectors = 0;
         int extra;
         uint64_t wwn;
+        u32 if_type;
+        u32 if_fam;
 
         phba->sli4_hba.num_online_cpu = num_online_cpus();
         phba->sli4_hba.num_present_cpu = lpfc_present_cpu;
@@ -6160,15 +6173,28 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
          */
         rc = lpfc_get_sli4_parameters(phba, mboxq);
         if (rc) {
+                if_type = bf_get(lpfc_sli_intf_if_type,
+                                 &phba->sli4_hba.sli_intf);
+                if_fam = bf_get(lpfc_sli_intf_sli_family,
+                                &phba->sli4_hba.sli_intf);
                 if (phba->sli4_hba.extents_in_use &&
                     phba->sli4_hba.rpi_hdrs_in_use) {
                         lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
                                 "2999 Unsupported SLI4 Parameters "
                                 "Extents and RPI headers enabled.\n");
+                        if (if_type == LPFC_SLI_INTF_IF_TYPE_0 &&
+                            if_fam ==  LPFC_SLI_INTF_FAMILY_BE2) {
+                                mempool_free(mboxq, phba->mbox_mem_pool);
+                                rc = -EIO;
+                                goto out_free_bsmbx;
+                        }
+                }
+                if (!(if_type == LPFC_SLI_INTF_IF_TYPE_0 &&
+                      if_fam == LPFC_SLI_INTF_FAMILY_BE2)) {
+                        mempool_free(mboxq, phba->mbox_mem_pool);
+                        rc = -EIO;
+                        goto out_free_bsmbx;
                 }
-                mempool_free(mboxq, phba->mbox_mem_pool);
-                rc = -EIO;
-                goto out_free_bsmbx;
         }
 
         mempool_free(mboxq, phba->mbox_mem_pool);
@@ -6406,8 +6432,11 @@ lpfc_setup_driver_resource_phase2(struct lpfc_hba *phba)
                 return error;
         }
 
-        /* workqueue for deferred irq use */
-        phba->wq = alloc_workqueue("lpfc_wq", WQ_MEM_RECLAIM, 0);
+        /* The lpfc_wq workqueue for deferred irq use, is only used for SLI4 */
+        if (phba->sli_rev == LPFC_SLI_REV4)
+                phba->wq = alloc_workqueue("lpfc_wq", WQ_MEM_RECLAIM, 0);
+        else
+                phba->wq = NULL;
 
         return 0;
 }
@@ -6430,7 +6459,8 @@ lpfc_unset_driver_resource_phase2(struct lpfc_hba *phba)
         }
 
         /* Stop kernel worker thread */
-        kthread_stop(phba->worker_thread);
+        if (phba->worker_thread)
+                kthread_stop(phba->worker_thread);
 }
 
 /**
@@ -6895,12 +6925,6 @@ lpfc_create_shost(struct lpfc_hba *phba)
         atomic_set(&phba->fc4ScsiOutputRequests, 0);
         atomic_set(&phba->fc4ScsiControlRequests, 0);
         atomic_set(&phba->fc4ScsiIoCmpls, 0);
-        atomic_set(&phba->fc4NvmeInputRequests, 0);
-        atomic_set(&phba->fc4NvmeOutputRequests, 0);
-        atomic_set(&phba->fc4NvmeControlRequests, 0);
-        atomic_set(&phba->fc4NvmeIoCmpls, 0);
-        atomic_set(&phba->fc4NvmeLsRequests, 0);
-        atomic_set(&phba->fc4NvmeLsCmpls, 0);
         vport = lpfc_create_port(phba, phba->brd_no, &phba->pcidev->dev);
         if (!vport)
                 return -ENODEV;
@@ -7781,6 +7805,40 @@ lpfc_sli4_read_config(struct lpfc_hba *phba)
                                 phba->sli4_hba.max_cfg_param.max_wq,
                                 phba->sli4_hba.max_cfg_param.max_rq);
 
+                /*
+                 * Calculate NVME queue resources based on how
+                 * many WQ/CQs are available.
+                 */
+                if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
+                        length = phba->sli4_hba.max_cfg_param.max_wq;
+                        if (phba->sli4_hba.max_cfg_param.max_cq <
+                            phba->sli4_hba.max_cfg_param.max_wq)
+                                length = phba->sli4_hba.max_cfg_param.max_cq;
+
+                        /*
+                         * Whats left after this can go toward NVME.
+                         * The minus 6 accounts for ELS, NVME LS, MBOX
+                         * fof plus a couple extra. When configured for
+                         * NVMET, FCP io channel WQs are not created.
+                         */
+                        length -= 6;
+                        if (!phba->nvmet_support)
+                                length -= phba->cfg_fcp_io_channel;
+
+                        if (phba->cfg_nvme_io_channel > length) {
+                                lpfc_printf_log(
+                                        phba, KERN_ERR, LOG_SLI,
+                                        "2005 Reducing NVME IO channel to %d: "
+                                        "WQ %d CQ %d NVMEIO %d FCPIO %d\n",
+                                        length,
+                                        phba->sli4_hba.max_cfg_param.max_wq,
+                                        phba->sli4_hba.max_cfg_param.max_cq,
+                                        phba->cfg_nvme_io_channel,
+                                        phba->cfg_fcp_io_channel);
+
+                                phba->cfg_nvme_io_channel = length;
+                        }
+                }
         }
 
         if (rc)
@@ -10533,6 +10591,7 @@ lpfc_get_sli4_parameters(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
         struct lpfc_pc_sli4_params *sli4_params;
         uint32_t mbox_tmo;
         int length;
+        bool exp_wqcq_pages = true;
         struct lpfc_sli4_parameters *mbx_sli4_parameters;
 
         /*
@@ -10659,8 +10718,15 @@ lpfc_get_sli4_parameters(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
                         phba->nvme_support, phba->nvme_embed_pbde,
                         phba->cfg_nvme_embed_cmd, phba->cfg_suppress_rsp);
 
+        if ((bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) ==
+            LPFC_SLI_INTF_IF_TYPE_2) &&
+            (bf_get(lpfc_sli_intf_sli_family, &phba->sli4_hba.sli_intf) ==
+                 LPFC_SLI_INTF_FAMILY_LNCR_A0))
+                exp_wqcq_pages = false;
+
         if ((bf_get(cfg_cqpsize, mbx_sli4_parameters) & LPFC_CQ_16K_PAGE_SZ) &&
             (bf_get(cfg_wqpsize, mbx_sli4_parameters) & LPFC_WQ_16K_PAGE_SZ) &&
+            exp_wqcq_pages &&
             (sli4_params->wqsize & LPFC_WQ_SZ128_SUPPORT))
                 phba->enab_exp_wqcq_pages = 1;
         else
@@ -11322,7 +11388,11 @@ lpfc_log_write_firmware_error(struct lpfc_hba *phba, uint32_t offset,
         uint32_t magic_number, uint32_t ftype, uint32_t fid, uint32_t fsize,
         const struct firmware *fw)
 {
-        if (offset == ADD_STATUS_FW_NOT_SUPPORTED)
+        if ((offset == ADD_STATUS_FW_NOT_SUPPORTED) ||
+            (phba->pcidev->device == PCI_DEVICE_ID_LANCER_G6_FC &&
+             magic_number != MAGIC_NUMER_G6) ||
+            (phba->pcidev->device == PCI_DEVICE_ID_LANCER_G7_FC &&
+             magic_number != MAGIC_NUMER_G7))
                 lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
                         "3030 This firmware version is not supported on "
                         "this HBA model. Device:%x Magic:%x Type:%x "
@@ -11719,6 +11789,7 @@ lpfc_pci_remove_one_s4(struct pci_dev *pdev)
         lpfc_nvme_free(phba);
         lpfc_free_iocb_list(phba);
 
+        lpfc_unset_driver_resource_phase2(phba);
         lpfc_sli4_driver_resource_unset(phba);
 
         /* Unmap adapter Control and Doorbell registers */
diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c
index 022060636ae1..1a803975bcbc 100644
--- a/drivers/scsi/lpfc/lpfc_nportdisc.c
+++ b/drivers/scsi/lpfc/lpfc_nportdisc.c
@@ -1936,31 +1936,14 @@ lpfc_cmpl_prli_prli_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
                         goto out;
                 }
 
-                /* When the rport rejected the FCP PRLI as unsupported.
-                 * This should only happen in Pt2Pt so an NVME PRLI
-                 * should be outstanding still.
-                 */
-                if (npr && ndlp->nlp_flag & NLP_FCP_PRLI_RJT) {
+                /* Adjust the nlp_type accordingly if the PRLI failed */
+                if (npr)
                         ndlp->nlp_fc4_type &= ~NLP_FC4_FCP;
-                        goto out_err;
-                }
-
-                /* The LS Req had some error.  Don't let this be a
-                 * target.
-                 */
-                if ((ndlp->fc4_prli_sent == 1) &&
-                    (ndlp->nlp_state == NLP_STE_PRLI_ISSUE) &&
-                    (ndlp->nlp_type & (NLP_FCP_TARGET | NLP_FCP_INITIATOR)))
-                        /* The FCP PRLI completed successfully but
-                         * the NVME PRLI failed.  Since they are sent in
-                         * succession, allow the FCP to complete.
-                         */
-                        goto out_err;
+                if (nvpr)
+                        ndlp->nlp_fc4_type &= ~NLP_FC4_NVME;
 
-                ndlp->nlp_prev_state = NLP_STE_PRLI_ISSUE;
-                ndlp->nlp_type |= NLP_FCP_INITIATOR;
-                lpfc_nlp_set_state(vport, ndlp, NLP_STE_UNMAPPED_NODE);
-                return ndlp->nlp_state;
+                /* We can't set the DSM state till BOTH PRLIs complete */
+                goto out_err;
         }
 
         if (npr && (npr->acceptRspCode == PRLI_REQ_EXECUTED) &&
@@ -1999,6 +1982,12 @@ lpfc_cmpl_prli_prli_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
                         if (bf_get_be32(prli_disc, nvpr))
                                 ndlp->nlp_type |= NLP_NVME_DISCOVERY;
 
+                        /* This node is an NVME target.  Adjust the command
+                         * queue depth on this node to not exceed the available
+                         * xris.
+                         */
+                        ndlp->cmd_qdepth = phba->sli4_hba.nvme_xri_max;
+
                         /*
                          * If prli_fba is set, the Target supports FirstBurst.
                          * If prli_fb_sz is 0, the FirstBurst size is unlimited,
diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c
index 378dca40ca20..76a5a99605aa 100644
--- a/drivers/scsi/lpfc/lpfc_nvme.c
+++ b/drivers/scsi/lpfc/lpfc_nvme.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -334,7 +334,14 @@ lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
                         "6146 remoteport delete of remoteport %p\n",
                         remoteport);
         spin_lock_irq(&vport->phba->hbalock);
-        ndlp->nrport = NULL;
+
+        /* The register rebind might have occurred before the delete
+         * downcall.  Guard against this race.
+         */
+        if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) {
+                ndlp->nrport = NULL;
+                ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
+        }
         spin_unlock_irq(&vport->phba->hbalock);
 
         /* Remove original register reference. The host transport
@@ -357,15 +364,19 @@ lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
         struct lpfc_dmabuf *buf_ptr;
         struct lpfc_nodelist *ndlp;
 
-        atomic_inc(&vport->phba->fc4NvmeLsCmpls);
-
         pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2;
         status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
-        if (status) {
+
+        if (vport->localport) {
                 lport = (struct lpfc_nvme_lport *)vport->localport->private;
-                if (bf_get(lpfc_wcqe_c_xb, wcqe))
-                        atomic_inc(&lport->cmpl_ls_xb);
-                atomic_inc(&lport->cmpl_ls_err);
+                if (lport) {
+                        atomic_inc(&lport->fc4NvmeLsCmpls);
+                        if (status) {
+                                if (bf_get(lpfc_wcqe_c_xb, wcqe))
+                                        atomic_inc(&lport->cmpl_ls_xb);
+                                atomic_inc(&lport->cmpl_ls_err);
+                        }
+                }
         }
 
         ndlp = (struct lpfc_nodelist *)cmdwqe->context1;
@@ -570,6 +581,9 @@ lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
 
         lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
         rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
+        if (unlikely(!lport) || unlikely(!rport))
+                return -EINVAL;
+
         vport = lport->vport;
 
         if (vport->load_flag & FC_UNLOADING)
@@ -639,7 +653,7 @@ lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
                          pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
                          &pnvme_lsreq->rspdma);
 
-        atomic_inc(&vport->phba->fc4NvmeLsRequests);
+        atomic_inc(&lport->fc4NvmeLsRequests);
 
         /* Hardcode the wait to 30 seconds.  Connections are failing otherwise.
          * This code allows it all to work.
@@ -690,6 +704,8 @@ lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
         struct lpfc_iocbq *wqe, *next_wqe;
 
         lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
+        if (unlikely(!lport))
+                return;
         vport = lport->vport;
         phba = vport->phba;
 
@@ -949,28 +965,48 @@ lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
         struct lpfc_nodelist *ndlp;
         struct lpfc_nvme_fcpreq_priv *freqpriv;
         struct lpfc_nvme_lport *lport;
+        struct lpfc_nvme_ctrl_stat *cstat;
         unsigned long flags;
-        uint32_t code, status;
+        uint32_t code, status, idx;
         uint16_t cid, sqhd, data;
         uint32_t *ptr;
 
         /* Sanity check on return of outstanding command */
         if (!lpfc_ncmd || !lpfc_ncmd->nvmeCmd || !lpfc_ncmd->nrport) {
+                if (!lpfc_ncmd) {
+                        lpfc_printf_vlog(vport, KERN_ERR,
+                                         LOG_NODE | LOG_NVME_IOERR,
+                                         "6071 Null lpfc_ncmd pointer. No "
+                                         "release, skip completion\n");
+                        return;
+                }
+
                 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
-                                 "6071 Completion pointers bad on wqe %p.\n",
-                                 wcqe);
+                                 "6066 Missing cmpl ptrs: lpfc_ncmd %p, "
+                                 "nvmeCmd %p nrport %p\n",
+                                 lpfc_ncmd, lpfc_ncmd->nvmeCmd,
+                                 lpfc_ncmd->nrport);
+
+                /* Release the lpfc_ncmd regardless of the missing elements. */
+                lpfc_release_nvme_buf(phba, lpfc_ncmd);
                 return;
         }
-        atomic_inc(&phba->fc4NvmeIoCmpls);
-
         nCmd = lpfc_ncmd->nvmeCmd;
         rport = lpfc_ncmd->nrport;
         status = bf_get(lpfc_wcqe_c_status, wcqe);
-        if (status) {
+
+        if (vport->localport) {
                 lport = (struct lpfc_nvme_lport *)vport->localport->private;
-                if (bf_get(lpfc_wcqe_c_xb, wcqe))
-                        atomic_inc(&lport->cmpl_fcp_xb);
-                atomic_inc(&lport->cmpl_fcp_err);
+                if (lport) {
+                        idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
+                        cstat = &lport->cstat[idx];
+                        atomic_inc(&cstat->fc4NvmeIoCmpls);
+                        if (status) {
+                                if (bf_get(lpfc_wcqe_c_xb, wcqe))
+                                        atomic_inc(&lport->cmpl_fcp_xb);
+                                atomic_inc(&lport->cmpl_fcp_err);
+                        }
+                }
         }
 
         lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
@@ -1163,7 +1199,8 @@ out_err:
 static int
 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
                       struct lpfc_nvme_buf *lpfc_ncmd,
-                      struct lpfc_nodelist *pnode)
+                      struct lpfc_nodelist *pnode,
+                      struct lpfc_nvme_ctrl_stat *cstat)
 {
         struct lpfc_hba *phba = vport->phba;
         struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
@@ -1201,7 +1238,7 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
                         } else {
                                 wqe->fcp_iwrite.initial_xfer_len = 0;
                         }
-                        atomic_inc(&phba->fc4NvmeOutputRequests);
+                        atomic_inc(&cstat->fc4NvmeOutputRequests);
                 } else {
                         /* From the iread template, initialize words 7 - 11 */
                         memcpy(&wqe->words[7],
@@ -1214,13 +1251,13 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
                         /* Word 5 */
                         wqe->fcp_iread.rsrvd5 = 0;
 
-                        atomic_inc(&phba->fc4NvmeInputRequests);
+                        atomic_inc(&cstat->fc4NvmeInputRequests);
                 }
         } else {
                 /* From the icmnd template, initialize words 4 - 11 */
                 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
                        sizeof(uint32_t) * 8);
-                atomic_inc(&phba->fc4NvmeControlRequests);
+                atomic_inc(&cstat->fc4NvmeControlRequests);
         }
         /*
          * Finish initializing those WQE fields that are independent
@@ -1400,7 +1437,9 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 {
         int ret = 0;
         int expedite = 0;
+        int idx;
         struct lpfc_nvme_lport *lport;
+        struct lpfc_nvme_ctrl_stat *cstat;
         struct lpfc_vport *vport;
         struct lpfc_hba *phba;
         struct lpfc_nodelist *ndlp;
@@ -1425,9 +1464,10 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
         vport = lport->vport;
 
         if (unlikely(!hw_queue_handle)) {
-                lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
-                                 "6129 Fail Abort, NULL hw_queue_handle\n");
-                ret = -EINVAL;
+                lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
+                                 "6117 Fail IO, NULL hw_queue_handle\n");
+                atomic_inc(&lport->xmt_fcp_err);
+                ret = -EBUSY;
                 goto out_fail;
         }
 
@@ -1439,12 +1479,18 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
         }
 
         if (vport->load_flag & FC_UNLOADING) {
+                lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
+                                 "6124 Fail IO, Driver unload\n");
+                atomic_inc(&lport->xmt_fcp_err);
                 ret = -ENODEV;
                 goto out_fail;
         }
 
         freqpriv = pnvme_fcreq->private;
         if (unlikely(!freqpriv)) {
+                lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
+                                 "6158 Fail IO, NULL request data\n");
+                atomic_inc(&lport->xmt_fcp_err);
                 ret = -EINVAL;
                 goto out_fail;
         }
@@ -1462,32 +1508,26 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
          */
         ndlp = rport->ndlp;
         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
-                lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
-                                 "6053 rport %p, ndlp %p, DID x%06x "
-                                 "ndlp not ready.\n",
+                lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
+                                 "6053 Fail IO, ndlp not ready: rport %p "
+                                  "ndlp %p, DID x%06x\n",
                                  rport, ndlp, pnvme_rport->port_id);
-
-                ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
-                if (!ndlp) {
-                        lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
-                                         "6066 Missing node for DID %x\n",
-                                         pnvme_rport->port_id);
-                        atomic_inc(&lport->xmt_fcp_bad_ndlp);
-                        ret = -ENODEV;
-                        goto out_fail;
-                }
+                atomic_inc(&lport->xmt_fcp_err);
+                ret = -EBUSY;
+                goto out_fail;
         }
 
         /* The remote node has to be a mapped target or it's an error. */
         if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
             (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
-                lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
-                                 "6036 rport %p, DID x%06x not ready for "
-                                 "IO. State x%x, Type x%x\n",
-                                 rport, pnvme_rport->port_id,
-                                 ndlp->nlp_state, ndlp->nlp_type);
+                lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
+                                 "6036 Fail IO, DID x%06x not ready for "
+                                 "IO. State x%x, Type x%x Flg x%x\n",
+                                 pnvme_rport->port_id,
+                                 ndlp->nlp_state, ndlp->nlp_type,
+                                 ndlp->upcall_flags);
                 atomic_inc(&lport->xmt_fcp_bad_ndlp);
-                ret = -ENODEV;
+                ret = -EBUSY;
                 goto out_fail;
 
         }
@@ -1508,6 +1548,12 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
          */
         if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
             !expedite) {
+                lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
+                                 "6174 Fail IO, ndlp qdepth exceeded: "
+                                 "idx %d DID %x pend %d qdepth %d\n",
+                                 lpfc_queue_info->index, ndlp->nlp_DID,
+                                 atomic_read(&ndlp->cmd_pending),
+                                 ndlp->cmd_qdepth);
                 atomic_inc(&lport->xmt_fcp_qdepth);
                 ret = -EBUSY;
                 goto out_fail;
@@ -1517,8 +1563,9 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
         if (lpfc_ncmd == NULL) {
                 atomic_inc(&lport->xmt_fcp_noxri);
                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
-                                 "6065 driver's buffer pool is empty, "
-                                 "IO failed\n");
+                                 "6065 Fail IO, driver buffer pool is empty: "
+                                 "idx %d DID %x\n",
+                                 lpfc_queue_info->index, ndlp->nlp_DID);
                 ret = -EBUSY;
                 goto out_fail;
         }
@@ -1543,15 +1590,6 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
         lpfc_ncmd->ndlp = ndlp;
         lpfc_ncmd->start_time = jiffies;
 
-        lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp);
-        ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
-        if (ret) {
-                ret = -ENOMEM;
-                goto out_free_nvme_buf;
-        }
-
-        atomic_inc(&ndlp->cmd_pending);
-
         /*
          * Issue the IO on the WQ indicated by index in the hw_queue_handle.
          * This identfier was create in our hardware queue create callback
@@ -1560,7 +1598,23 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
          * index to use and that they have affinitized a CPU to this hardware
          * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
          */
-        lpfc_ncmd->cur_iocbq.hba_wqidx = lpfc_queue_info->index;
+        idx = lpfc_queue_info->index;
+        lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
+        cstat = &lport->cstat[idx];
+
+        lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
+        ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
+        if (ret) {
+                lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
+                                 "6175 Fail IO, Prep DMA: "
+                                 "idx %d DID %x\n",
+                                 lpfc_queue_info->index, ndlp->nlp_DID);
+                atomic_inc(&lport->xmt_fcp_err);
+                ret = -ENOMEM;
+                goto out_free_nvme_buf;
+        }
+
+        atomic_inc(&ndlp->cmd_pending);
 
         lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
                          lpfc_ncmd->cur_iocbq.sli4_xritag,
@@ -1571,7 +1625,7 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
                 atomic_inc(&lport->xmt_fcp_wqerr);
                 atomic_dec(&ndlp->cmd_pending);
                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
-                                 "6113 FCP could not issue WQE err %x "
+                                 "6113 Fail IO, Could not issue WQE err %x "
                                  "sid: x%x did: x%x oxid: x%x\n",
                                  ret, vport->fc_myDID, ndlp->nlp_DID,
                                  lpfc_ncmd->cur_iocbq.sli4_xritag);
@@ -1605,11 +1659,11 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
  out_free_nvme_buf:
         if (lpfc_ncmd->nvmeCmd->sg_cnt) {
                 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
-                        atomic_dec(&phba->fc4NvmeOutputRequests);
+                        atomic_dec(&cstat->fc4NvmeOutputRequests);
                 else
-                        atomic_dec(&phba->fc4NvmeInputRequests);
+                        atomic_dec(&cstat->fc4NvmeInputRequests);
         } else
-                atomic_dec(&phba->fc4NvmeControlRequests);
+                atomic_dec(&cstat->fc4NvmeControlRequests);
         lpfc_release_nvme_buf(phba, lpfc_ncmd);
  out_fail:
         return ret;
@@ -2390,7 +2444,8 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
         struct nvme_fc_port_info nfcp_info;
         struct nvme_fc_local_port *localport;
         struct lpfc_nvme_lport *lport;
-        int len;
+        struct lpfc_nvme_ctrl_stat *cstat;
+        int len, i;
 
         /* Initialize this localport instance.  The vport wwn usage ensures
          * that NPIV is accounted for.
@@ -2414,6 +2469,11 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
         lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
         lpfc_nvme_template.max_hw_queues = phba->cfg_nvme_io_channel;
 
+        cstat = kmalloc((sizeof(struct lpfc_nvme_ctrl_stat) *
+                        phba->cfg_nvme_io_channel), GFP_KERNEL);
+        if (!cstat)
+                return -ENOMEM;
+
         /* localport is allocated from the stack, but the registration
          * call allocates heap memory as well as the private area.
          */
@@ -2436,11 +2496,13 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
                 lport = (struct lpfc_nvme_lport *)localport->private;
                 vport->localport = localport;
                 lport->vport = vport;
+                lport->cstat = cstat;
                 vport->nvmei_support = 1;
 
                 atomic_set(&lport->xmt_fcp_noxri, 0);
                 atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
                 atomic_set(&lport->xmt_fcp_qdepth, 0);
+                atomic_set(&lport->xmt_fcp_err, 0);
                 atomic_set(&lport->xmt_fcp_wqerr, 0);
                 atomic_set(&lport->xmt_fcp_abort, 0);
                 atomic_set(&lport->xmt_ls_abort, 0);
@@ -2449,6 +2511,16 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
                 atomic_set(&lport->cmpl_fcp_err, 0);
                 atomic_set(&lport->cmpl_ls_xb, 0);
                 atomic_set(&lport->cmpl_ls_err, 0);
+                atomic_set(&lport->fc4NvmeLsRequests, 0);
+                atomic_set(&lport->fc4NvmeLsCmpls, 0);
+
+                for (i = 0; i < phba->cfg_nvme_io_channel; i++) {
+                        cstat = &lport->cstat[i];
+                        atomic_set(&cstat->fc4NvmeInputRequests, 0);
+                        atomic_set(&cstat->fc4NvmeOutputRequests, 0);
+                        atomic_set(&cstat->fc4NvmeControlRequests, 0);
+                        atomic_set(&cstat->fc4NvmeIoCmpls, 0);
+                }
 
                 /* Don't post more new bufs if repost already recovered
                  * the nvme sgls.
@@ -2458,6 +2530,8 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
                                                  phba->sli4_hba.nvme_xri_max);
                         vport->phba->total_nvme_bufs += len;
                 }
+        } else {
+                kfree(cstat);
         }
 
         return ret;
@@ -2520,6 +2594,7 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
 #if (IS_ENABLED(CONFIG_NVME_FC))
         struct nvme_fc_local_port *localport;
         struct lpfc_nvme_lport *lport;
+        struct lpfc_nvme_ctrl_stat *cstat;
         int ret;
 
         if (vport->nvmei_support == 0)
@@ -2528,6 +2603,7 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
         localport = vport->localport;
         vport->localport = NULL;
         lport = (struct lpfc_nvme_lport *)localport->private;
+        cstat = lport->cstat;
 
         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
                          "6011 Destroying NVME localport %p\n",
@@ -2543,6 +2619,7 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
          * indefinitely or succeeds
          */
         lpfc_nvme_lport_unreg_wait(vport, lport);
+        kfree(cstat);
 
         /* Regardless of the unregister upcall response, clear
          * nvmei_support.  All rports are unregistered and the
@@ -2607,6 +2684,7 @@ lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
         struct nvme_fc_local_port *localport;
         struct lpfc_nvme_lport *lport;
         struct lpfc_nvme_rport *rport;
+        struct lpfc_nvme_rport *oldrport;
         struct nvme_fc_remote_port *remote_port;
         struct nvme_fc_port_info rpinfo;
         struct lpfc_nodelist *prev_ndlp;
@@ -2639,7 +2717,9 @@ lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 
         rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
         rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
-        if (!ndlp->nrport)
+
+        oldrport = lpfc_ndlp_get_nrport(ndlp);
+        if (!oldrport)
                 lpfc_nlp_get(ndlp);
 
         ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
@@ -2648,9 +2728,15 @@ lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
                  * a resume of the existing rport.  Else this is a
                  * new rport.
                  */
+                /* Guard against an unregister/reregister
+                 * race that leaves the WAIT flag set.
+                 */
+                spin_lock_irq(&vport->phba->hbalock);
+                ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
+                spin_unlock_irq(&vport->phba->hbalock);
                 rport = remote_port->private;
-                if (ndlp->nrport) {
-                        if (ndlp->nrport == remote_port->private) {
+                if (oldrport) {
+                        if (oldrport == remote_port->private) {
                                 /* Same remoteport.  Just reuse. */
                                 lpfc_printf_vlog(ndlp->vport, KERN_INFO,
                                                  LOG_NVME_DISC,
@@ -2674,11 +2760,20 @@ lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
                          */
                         spin_lock_irq(&vport->phba->hbalock);
                         ndlp->nrport = NULL;
+                        ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
                         spin_unlock_irq(&vport->phba->hbalock);
                         rport->ndlp = NULL;
                         rport->remoteport = NULL;
-                        if (prev_ndlp)
-                                lpfc_nlp_put(ndlp);
+
+                        /* Reference only removed if previous NDLP is no longer
+                         * active. It might be just a swap and removing the
+                         * reference would cause a premature cleanup.
+                         */
+                        if (prev_ndlp && prev_ndlp != ndlp) {
+                                if ((!NLP_CHK_NODE_ACT(prev_ndlp)) ||
+                                    (!prev_ndlp->nrport))
+                                        lpfc_nlp_put(prev_ndlp);
+                        }
                 }
 
                 /* Clean bind the rport to the ndlp. */
@@ -2746,7 +2841,7 @@ lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
         if (!lport)
                 goto input_err;
 
-        rport = ndlp->nrport;
+        rport = lpfc_ndlp_get_nrport(ndlp);
         if (!rport)
                 goto input_err;
 
@@ -2767,6 +2862,15 @@ lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
                  * The transport will update it.
                  */
                 ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG;
+
+                /* Don't let the host nvme transport keep sending keep-alives
+                 * on this remoteport. Vport is unloading, no recovery. The
+                 * return values is ignored.  The upcall is a courtesy to the
+                 * transport.
+                 */
+                if (vport->load_flag & FC_UNLOADING)
+                        (void)nvme_fc_set_remoteport_devloss(remoteport, 0);
+
                 ret = nvme_fc_unregister_remoteport(remoteport);
                 if (ret != 0) {
                         lpfc_nlp_put(ndlp);
diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h
index 9216653e0441..04bd463dd043 100644
--- a/drivers/scsi/lpfc/lpfc_nvme.h
+++ b/drivers/scsi/lpfc/lpfc_nvme.h
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -30,21 +30,36 @@
 #define LPFC_NVME_FB_SHIFT              9
 #define LPFC_NVME_MAX_FB                (1 << 20)       /* 1M */
 
+#define lpfc_ndlp_get_nrport(ndlp)                                      \
+        ((!ndlp->nrport || (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG))   \
+        ? NULL : ndlp->nrport)
+
 struct lpfc_nvme_qhandle {
         uint32_t index;         /* WQ index to use */
         uint32_t qidx;          /* queue index passed to create */
         uint32_t cpu_id;        /* current cpu id at time of create */
 };
 
+struct lpfc_nvme_ctrl_stat {
+        atomic_t fc4NvmeInputRequests;
+        atomic_t fc4NvmeOutputRequests;
+        atomic_t fc4NvmeControlRequests;
+        atomic_t fc4NvmeIoCmpls;
+};
+
 /* Declare nvme-based local and remote port definitions. */
 struct lpfc_nvme_lport {
         struct lpfc_vport *vport;
         struct completion lport_unreg_done;
         /* Add stats counters here */
+        struct lpfc_nvme_ctrl_stat *cstat;
+        atomic_t fc4NvmeLsRequests;
+        atomic_t fc4NvmeLsCmpls;
         atomic_t xmt_fcp_noxri;
         atomic_t xmt_fcp_bad_ndlp;
         atomic_t xmt_fcp_qdepth;
         atomic_t xmt_fcp_wqerr;
+        atomic_t xmt_fcp_err;
         atomic_t xmt_fcp_abort;
         atomic_t xmt_ls_abort;
         atomic_t xmt_ls_err;
diff --git a/drivers/scsi/lpfc/lpfc_nvmet.h b/drivers/scsi/lpfc/lpfc_nvmet.h
index c1bcef3f103c..81f520abfd64 100644
--- a/drivers/scsi/lpfc/lpfc_nvmet.h
+++ b/drivers/scsi/lpfc/lpfc_nvmet.h
@@ -22,8 +22,10 @@
  ********************************************************************/
 
 #define LPFC_NVMET_DEFAULT_SEGS         (64 + 1)        /* 256K IOs */
-#define LPFC_NVMET_RQE_DEF_COUNT        512
-#define LPFC_NVMET_SUCCESS_LEN  12
+#define LPFC_NVMET_RQE_MIN_POST         128
+#define LPFC_NVMET_RQE_DEF_POST         512
+#define LPFC_NVMET_RQE_DEF_COUNT        2048
+#define LPFC_NVMET_SUCCESS_LEN          12
 
 #define LPFC_NVMET_MRQ_OFF              0xffff
 #define LPFC_NVMET_MRQ_AUTO             0
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index 050f04418f5f..a94fb9f8bb44 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -1021,7 +1021,7 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
                 if (lpfc_test_rrq_active(phba, ndlp,
                                          lpfc_cmd->cur_iocbq.sli4_lxritag))
                         continue;
-                list_del(&lpfc_cmd->list);
+                list_del_init(&lpfc_cmd->list);
                 found = 1;
                 break;
         }
@@ -1036,7 +1036,7 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
                         if (lpfc_test_rrq_active(
                                 phba, ndlp, lpfc_cmd->cur_iocbq.sli4_lxritag))
                                 continue;
-                        list_del(&lpfc_cmd->list);
+                        list_del_init(&lpfc_cmd->list);
                         found = 1;
                         break;
                 }
@@ -3983,9 +3983,6 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
         }
 #endif
 
-        if (pnode && NLP_CHK_NODE_ACT(pnode))
-                atomic_dec(&pnode->cmd_pending);
-
         if (lpfc_cmd->status) {
                 if (lpfc_cmd->status == IOSTAT_LOCAL_REJECT &&
                     (lpfc_cmd->result & IOERR_DRVR_MASK))
@@ -4125,6 +4122,7 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
                 msecs_to_jiffies(vport->cfg_max_scsicmpl_time))) {
                 spin_lock_irqsave(shost->host_lock, flags);
                 if (pnode && NLP_CHK_NODE_ACT(pnode)) {
+                        atomic_dec(&pnode->cmd_pending);
                         if (pnode->cmd_qdepth >
                                 atomic_read(&pnode->cmd_pending) &&
                                 (atomic_read(&pnode->cmd_pending) >
@@ -4138,16 +4136,8 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
                 }
                 spin_unlock_irqrestore(shost->host_lock, flags);
         } else if (pnode && NLP_CHK_NODE_ACT(pnode)) {
-                if ((pnode->cmd_qdepth != vport->cfg_tgt_queue_depth) &&
-                    time_after(jiffies, pnode->last_change_time +
-                              msecs_to_jiffies(LPFC_TGTQ_INTERVAL))) {
-                        spin_lock_irqsave(shost->host_lock, flags);
-                        pnode->cmd_qdepth = vport->cfg_tgt_queue_depth;
-                        pnode->last_change_time = jiffies;
-                        spin_unlock_irqrestore(shost->host_lock, flags);
-                }
+                atomic_dec(&pnode->cmd_pending);
         }
-
         lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
 
         spin_lock_irqsave(&phba->hbalock, flags);
@@ -4591,6 +4581,8 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
                                  ndlp->nlp_portname.u.wwn[7]);
                 goto out_tgt_busy;
         }
+        atomic_inc(&ndlp->cmd_pending);
+
         lpfc_cmd = lpfc_get_scsi_buf(phba, ndlp);
         if (lpfc_cmd == NULL) {
                 lpfc_rampdown_queue_depth(phba);
@@ -4643,11 +4635,9 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
 
         lpfc_scsi_prep_cmnd(vport, lpfc_cmd, ndlp);
 
-        atomic_inc(&ndlp->cmd_pending);
         err = lpfc_sli_issue_iocb(phba, LPFC_FCP_RING,
                                   &lpfc_cmd->cur_iocbq, SLI_IOCB_RET_IOCB);
         if (err) {
-                atomic_dec(&ndlp->cmd_pending);
                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
                                  "3376 FCP could not issue IOCB err %x"
                                  "FCP cmd x%x <%d/%llu> "
@@ -4691,6 +4681,7 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
         lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
         lpfc_release_scsi_buf(phba, lpfc_cmd);
  out_host_busy:
+        atomic_dec(&ndlp->cmd_pending);
         return SCSI_MLQUEUE_HOST_BUSY;
 
  out_tgt_busy:
@@ -4725,7 +4716,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd)
         int ret = SUCCESS, status = 0;
         struct lpfc_sli_ring *pring_s4;
         int ret_val;
-        unsigned long flags, iflags;
+        unsigned long flags;
         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(waitq);
 
         status = fc_block_scsi_eh(cmnd);
@@ -4825,16 +4816,16 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd)
         abtsiocb->iocb_cmpl = lpfc_sli_abort_fcp_cmpl;
         abtsiocb->vport = vport;
         if (phba->sli_rev == LPFC_SLI_REV4) {
-                pring_s4 = lpfc_sli4_calc_ring(phba, iocb);
+                pring_s4 = lpfc_sli4_calc_ring(phba, abtsiocb);
                 if (pring_s4 == NULL) {
                         ret = FAILED;
                         goto out_unlock;
                 }
                 /* Note: both hbalock and ring_lock must be set here */
-                spin_lock_irqsave(&pring_s4->ring_lock, iflags);
+                spin_lock(&pring_s4->ring_lock);
                 ret_val = __lpfc_sli_issue_iocb(phba, pring_s4->ringno,
                                                 abtsiocb, 0);
-                spin_unlock_irqrestore(&pring_s4->ring_lock, iflags);
+                spin_unlock(&pring_s4->ring_lock);
         } else {
                 ret_val = __lpfc_sli_issue_iocb(phba, LPFC_FCP_RING,
                                                 abtsiocb, 0);
diff --git a/drivers/scsi/lpfc/lpfc_scsi.h b/drivers/scsi/lpfc/lpfc_scsi.h
index 8e38e0204c47..c38e4da71f5f 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.h
+++ b/drivers/scsi/lpfc/lpfc_scsi.h
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index cb17e2b2be81..4b70d53acb72 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -96,6 +96,34 @@ lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq)
         return &iocbq->iocb;
 }
 
+#if defined(CONFIG_64BIT) && defined(__LITTLE_ENDIAN)
+/**
+ * lpfc_sli4_pcimem_bcopy - SLI4 memory copy function
+ * @srcp: Source memory pointer.
+ * @destp: Destination memory pointer.
+ * @cnt: Number of words required to be copied.
+ *       Must be a multiple of sizeof(uint64_t)
+ *
+ * This function is used for copying data between driver memory
+ * and the SLI WQ. This function also changes the endianness
+ * of each word if native endianness is different from SLI
+ * endianness. This function can be called with or without
+ * lock.
+ **/
+void
+lpfc_sli4_pcimem_bcopy(void *srcp, void *destp, uint32_t cnt)
+{
+        uint64_t *src = srcp;
+        uint64_t *dest = destp;
+        int i;
+
+        for (i = 0; i < (int)cnt; i += sizeof(uint64_t))
+                *dest++ = *src++;
+}
+#else
+#define lpfc_sli4_pcimem_bcopy(a, b, c) lpfc_sli_pcimem_bcopy(a, b, c)
+#endif
+
 /**
  * lpfc_sli4_wq_put - Put a Work Queue Entry on an Work Queue
  * @q: The Work Queue to operate on.
@@ -137,7 +165,7 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe)
                 bf_set(wqe_wqec, &wqe->generic.wqe_com, 0);
         if (q->phba->sli3_options & LPFC_SLI4_PHWQ_ENABLED)
                 bf_set(wqe_wqid, &wqe->generic.wqe_com, q->queue_id);
-        lpfc_sli_pcimem_bcopy(wqe, temp_wqe, q->entry_size);
+        lpfc_sli4_pcimem_bcopy(wqe, temp_wqe, q->entry_size);
         if (q->dpp_enable && q->phba->cfg_enable_dpp) {
                 /* write to DPP aperture taking advatage of Combined Writes */
                 tmp = (uint8_t *)temp_wqe;
@@ -240,7 +268,7 @@ lpfc_sli4_mq_put(struct lpfc_queue *q, struct lpfc_mqe *mqe)
         /* If the host has not yet processed the next entry then we are done */
         if (((q->host_index + 1) % q->entry_count) == q->hba_index)
                 return -ENOMEM;
-        lpfc_sli_pcimem_bcopy(mqe, temp_mqe, q->entry_size);
+        lpfc_sli4_pcimem_bcopy(mqe, temp_mqe, q->entry_size);
         /* Save off the mailbox pointer for completion */
         q->phba->mbox = (MAILBOX_t *)temp_mqe;
 
@@ -663,8 +691,8 @@ lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq,
         /* If the host has not yet processed the next entry then we are done */
         if (((hq_put_index + 1) % hq->entry_count) == hq->hba_index)
                 return -EBUSY;
-        lpfc_sli_pcimem_bcopy(hrqe, temp_hrqe, hq->entry_size);
-        lpfc_sli_pcimem_bcopy(drqe, temp_drqe, dq->entry_size);
+        lpfc_sli4_pcimem_bcopy(hrqe, temp_hrqe, hq->entry_size);
+        lpfc_sli4_pcimem_bcopy(drqe, temp_drqe, dq->entry_size);
 
         /* Update the host index to point to the next slot */
         hq->host_index = ((hq_put_index + 1) % hq->entry_count);
@@ -7199,7 +7227,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)
                         lpfc_post_rq_buffer(
                                 phba, phba->sli4_hba.nvmet_mrq_hdr[i],
                                 phba->sli4_hba.nvmet_mrq_data[i],
-                                LPFC_NVMET_RQE_DEF_COUNT, i);
+                                phba->cfg_nvmet_mrq_post, i);
                 }
         }
 
@@ -8185,8 +8213,8 @@ lpfc_sli4_post_sync_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
          */
         mbx_cmnd = bf_get(lpfc_mqe_command, mb);
         memset(phba->sli4_hba.bmbx.avirt, 0, sizeof(struct lpfc_bmbx_create));
-        lpfc_sli_pcimem_bcopy(mb, phba->sli4_hba.bmbx.avirt,
-                              sizeof(struct lpfc_mqe));
+        lpfc_sli4_pcimem_bcopy(mb, phba->sli4_hba.bmbx.avirt,
+                               sizeof(struct lpfc_mqe));
 
         /* Post the high mailbox dma address to the port and wait for ready. */
         dma_address = &phba->sli4_hba.bmbx.dma_address;
@@ -8210,11 +8238,11 @@ lpfc_sli4_post_sync_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
          * If so, update the mailbox status so that the upper layers
          * can complete the request normally.
          */
-        lpfc_sli_pcimem_bcopy(phba->sli4_hba.bmbx.avirt, mb,
-                              sizeof(struct lpfc_mqe));
+        lpfc_sli4_pcimem_bcopy(phba->sli4_hba.bmbx.avirt, mb,
+                               sizeof(struct lpfc_mqe));
         mbox_rgn = (struct lpfc_bmbx_create *) phba->sli4_hba.bmbx.avirt;
-        lpfc_sli_pcimem_bcopy(&mbox_rgn->mcqe, &mboxq->mcqe,
-                              sizeof(struct lpfc_mcqe));
+        lpfc_sli4_pcimem_bcopy(&mbox_rgn->mcqe, &mboxq->mcqe,
+                               sizeof(struct lpfc_mcqe));
         mcqe_status = bf_get(lpfc_mcqe_status, &mbox_rgn->mcqe);
         /*
          * When the CQE status indicates a failure and the mailbox status
@@ -11300,11 +11328,11 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring,
         unsigned long iflags;
         struct lpfc_sli_ring *pring_s4;
 
-        spin_lock_irq(&phba->hbalock);
+        spin_lock_irqsave(&phba->hbalock, iflags);
 
         /* all I/Os are in process of being flushed */
         if (phba->hba_flag & HBA_FCP_IOQ_FLUSH) {
-                spin_unlock_irq(&phba->hbalock);
+                spin_unlock_irqrestore(&phba->hbalock, iflags);
                 return 0;
         }
         sum = 0;
@@ -11366,14 +11394,14 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring,
                 iocbq->iocb_flag |= LPFC_DRIVER_ABORTED;
 
                 if (phba->sli_rev == LPFC_SLI_REV4) {
-                        pring_s4 = lpfc_sli4_calc_ring(phba, iocbq);
-                        if (pring_s4 == NULL)
+                        pring_s4 = lpfc_sli4_calc_ring(phba, abtsiocbq);
+                        if (!pring_s4)
                                 continue;
                         /* Note: both hbalock and ring_lock must be set here */
-                        spin_lock_irqsave(&pring_s4->ring_lock, iflags);
+                        spin_lock(&pring_s4->ring_lock);
                         ret_val = __lpfc_sli_issue_iocb(phba, pring_s4->ringno,
                                                         abtsiocbq, 0);
-                        spin_unlock_irqrestore(&pring_s4->ring_lock, iflags);
+                        spin_unlock(&pring_s4->ring_lock);
                 } else {
                         ret_val = __lpfc_sli_issue_iocb(phba, pring->ringno,
                                                         abtsiocbq, 0);
@@ -11385,7 +11413,7 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring,
                 else
                         sum++;
         }
-        spin_unlock_irq(&phba->hbalock);
+        spin_unlock_irqrestore(&phba->hbalock, iflags);
         return sum;
 }
 
@@ -12830,7 +12858,7 @@ lpfc_sli4_sp_handle_mbox_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe)
 
         /* Move mbox data to caller's mailbox region, do endian swapping */
         if (pmb->mbox_cmpl && mbox)
-                lpfc_sli_pcimem_bcopy(mbox, mqe, sizeof(struct lpfc_mqe));
+                lpfc_sli4_pcimem_bcopy(mbox, mqe, sizeof(struct lpfc_mqe));
 
         /*
          * For mcqe errors, conditionally move a modified error code to
@@ -12913,7 +12941,7 @@ lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_cqe *cqe)
         bool workposted;
 
         /* Copy the mailbox MCQE and convert endian order as needed */
-        lpfc_sli_pcimem_bcopy(cqe, &mcqe, sizeof(struct lpfc_mcqe));
+        lpfc_sli4_pcimem_bcopy(cqe, &mcqe, sizeof(struct lpfc_mcqe));
 
         /* Invoke the proper event handling routine */
         if (!bf_get(lpfc_trailer_async, &mcqe))
@@ -12944,6 +12972,17 @@ lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
         int txcmplq_cnt = 0;
         int fcp_txcmplq_cnt = 0;
 
+        /* Check for response status */
+        if (unlikely(bf_get(lpfc_wcqe_c_status, wcqe))) {
+                /* Log the error status */
+                lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
+                                "0357 ELS CQE error: status=x%x: "
+                                "CQE: %08x %08x %08x %08x\n",
+                                bf_get(lpfc_wcqe_c_status, wcqe),
+                                wcqe->word0, wcqe->total_data_placed,
+                                wcqe->parameter, wcqe->word3);
+        }
+
         /* Get an irspiocbq for later ELS response processing use */
         irspiocbq = lpfc_sli_get_iocbq(phba);
         if (!irspiocbq) {
@@ -13173,7 +13212,7 @@ lpfc_sli4_sp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
         bool workposted = false;
 
         /* Copy the work queue CQE and convert endian order if needed */
-        lpfc_sli_pcimem_bcopy(cqe, &cqevt, sizeof(struct lpfc_cqe));
+        lpfc_sli4_pcimem_bcopy(cqe, &cqevt, sizeof(struct lpfc_cqe));
 
         /* Check and process for different type of WCQE and dispatch */
         switch (bf_get(lpfc_cqe_code, &cqevt)) {
@@ -13364,14 +13403,12 @@ lpfc_sli4_fp_handle_fcp_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
                         phba->lpfc_rampdown_queue_depth(phba);
 
                 /* Log the error status */
-                lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
-                                "0373 FCP complete error: status=x%x, "
-                                "hw_status=x%x, total_data_specified=%d, "
-                                "parameter=x%x, word3=x%x\n",
+                lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
+                                "0373 FCP CQE error: status=x%x: "
+                                "CQE: %08x %08x %08x %08x\n",
                                 bf_get(lpfc_wcqe_c_status, wcqe),
-                                bf_get(lpfc_wcqe_c_hw_status, wcqe),
-                                wcqe->total_data_placed, wcqe->parameter,
-                                wcqe->word3);
+                                wcqe->word0, wcqe->total_data_placed,
+                                wcqe->parameter, wcqe->word3);
         }
 
         /* Look up the FCP command IOCB and create pseudo response IOCB */
@@ -13581,7 +13618,7 @@ lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
         bool workposted = false;
 
         /* Copy the work queue CQE and convert endian order if needed */
-        lpfc_sli_pcimem_bcopy(cqe, &wcqe, sizeof(struct lpfc_cqe));
+        lpfc_sli4_pcimem_bcopy(cqe, &wcqe, sizeof(struct lpfc_cqe));
 
         /* Check and process for different type of WCQE and dispatch */
         switch (bf_get(lpfc_wcqe_c_code, &wcqe)) {
@@ -19032,9 +19069,22 @@ lpfc_drain_txq(struct lpfc_hba *phba)
         struct lpfc_sglq *sglq;
         union lpfc_wqe128 wqe;
         uint32_t txq_cnt = 0;
+        struct lpfc_queue *wq;
 
-        pring = lpfc_phba_elsring(phba);
-        if (unlikely(!pring))
+        if (phba->link_flag & LS_MDS_LOOPBACK) {
+                /* MDS WQE are posted only to first WQ*/
+                wq = phba->sli4_hba.fcp_wq[0];
+                if (unlikely(!wq))
+                        return 0;
+                pring = wq->pring;
+        } else {
+                wq = phba->sli4_hba.els_wq;
+                if (unlikely(!wq))
+                        return 0;
+                pring = lpfc_phba_elsring(phba);
+        }
+
+        if (unlikely(!pring) || list_empty(&pring->txq))
                 return 0;
 
         spin_lock_irqsave(&pring->ring_lock, iflags);
@@ -19075,7 +19125,7 @@ lpfc_drain_txq(struct lpfc_hba *phba)
                         fail_msg = "to convert bpl to sgl";
                 else if (lpfc_sli4_iocb2wqe(phba, piocbq, &wqe))
                         fail_msg = "to convert iocb to wqe";
-                else if (lpfc_sli4_wq_put(phba->sli4_hba.els_wq, &wqe))
+                else if (lpfc_sli4_wq_put(wq, &wqe))
                         fail_msg = " - Wq is full";
                 else
                         lpfc_sli_ringtxcmpl_put(phba, pring, piocbq);
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index e8b089abbfb3..18c23afcf46b 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -20,7 +20,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "12.0.0.1"
+#define LPFC_DRIVER_VERSION "12.0.0.4"
 #define LPFC_DRIVER_NAME                "lpfc"
 
 /* Used for SLI 2/3 */
diff --git a/drivers/scsi/megaraid.c b/drivers/scsi/megaraid.c
index 91f5e2c68dbc..3b3767e240d8 100644
--- a/drivers/scsi/megaraid.c
+++ b/drivers/scsi/megaraid.c
@@ -4166,6 +4166,9 @@ megaraid_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
         int irq, i, j;
         int error = -ENODEV;
 
+        if (hba_count >= MAX_CONTROLLERS)
+                goto out;
+
         if (pci_enable_device(pdev))
                 goto out;
         pci_set_master(pdev);
diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h
index 27fab8235ea5..75dc25f78336 100644
--- a/drivers/scsi/megaraid/megaraid_sas.h
+++ b/drivers/scsi/megaraid/megaraid_sas.h
@@ -35,8 +35,8 @@
 /*
  * MegaRAID SAS Driver meta data
  */
-#define MEGASAS_VERSION                         "07.704.04.00-rc1"
-#define MEGASAS_RELDATE                         "December 7, 2017"
+#define MEGASAS_VERSION                         "07.705.02.00-rc1"
+#define MEGASAS_RELDATE                         "April 4, 2018"
 
 /*
  * Device IDs
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index ce656c466ca9..c5d0c4bd71d2 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -92,7 +92,7 @@ MODULE_PARM_DESC(resetwaittime, "Wait time in seconds after I/O timeout "
 
 int smp_affinity_enable = 1;
 module_param(smp_affinity_enable, int, S_IRUGO);
-MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disbale Default: enable(1)");
+MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)");
 
 int rdpq_enable = 1;
 module_param(rdpq_enable, int, S_IRUGO);
@@ -2224,9 +2224,9 @@ static int megasas_get_ld_vf_affiliation_111(struct megasas_instance *instance,
                                sizeof(struct MR_LD_VF_AFFILIATION_111));
         else {
                 new_affiliation_111 =
-                        pci_alloc_consistent(instance->pdev,
-                                             sizeof(struct MR_LD_VF_AFFILIATION_111),
-                                             &new_affiliation_111_h);
+                        pci_zalloc_consistent(instance->pdev,
+                                              sizeof(struct MR_LD_VF_AFFILIATION_111),
+                                              &new_affiliation_111_h);
                 if (!new_affiliation_111) {
                         dev_printk(KERN_DEBUG, &instance->pdev->dev, "SR-IOV: Couldn't allocate "
                                "memory for new affiliation for scsi%d\n",
@@ -2234,8 +2234,6 @@ static int megasas_get_ld_vf_affiliation_111(struct megasas_instance *instance,
                         megasas_return_cmd(instance, cmd);
                         return -ENOMEM;
                 }
-                memset(new_affiliation_111, 0,
-                       sizeof(struct MR_LD_VF_AFFILIATION_111));
         }
 
         memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
@@ -2333,10 +2331,10 @@ static int megasas_get_ld_vf_affiliation_12(struct megasas_instance *instance,
                        sizeof(struct MR_LD_VF_AFFILIATION));
         else {
                 new_affiliation =
-                        pci_alloc_consistent(instance->pdev,
-                                             (MAX_LOGICAL_DRIVES + 1) *
-                                             sizeof(struct MR_LD_VF_AFFILIATION),
-                                             &new_affiliation_h);
+                        pci_zalloc_consistent(instance->pdev,
+                                              (MAX_LOGICAL_DRIVES + 1) *
+                                              sizeof(struct MR_LD_VF_AFFILIATION),
+                                              &new_affiliation_h);
                 if (!new_affiliation) {
                         dev_printk(KERN_DEBUG, &instance->pdev->dev, "SR-IOV: Couldn't allocate "
                                "memory for new affiliation for scsi%d\n",
@@ -2344,8 +2342,6 @@ static int megasas_get_ld_vf_affiliation_12(struct megasas_instance *instance,
                         megasas_return_cmd(instance, cmd);
                         return -ENOMEM;
                 }
-                memset(new_affiliation, 0, (MAX_LOGICAL_DRIVES + 1) *
-                       sizeof(struct MR_LD_VF_AFFILIATION));
         }
 
         memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
@@ -5636,16 +5632,15 @@ megasas_get_seq_num(struct megasas_instance *instance,
         }
 
         dcmd = &cmd->frame->dcmd;
-        el_info = pci_alloc_consistent(instance->pdev,
-                                       sizeof(struct megasas_evt_log_info),
-                                       &el_info_h);
+        el_info = pci_zalloc_consistent(instance->pdev,
+                                        sizeof(struct megasas_evt_log_info),
+                                        &el_info_h);
 
         if (!el_info) {
                 megasas_return_cmd(instance, cmd);
                 return -ENOMEM;
         }
 
-        memset(el_info, 0, sizeof(*el_info));
         memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
 
         dcmd->cmd = MFI_CMD_DCMD;
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index f4d988dd1e9d..98a7a090b75e 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -684,15 +684,14 @@ megasas_alloc_rdpq_fusion(struct megasas_instance *instance)
         array_size = sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) *
                      MAX_MSIX_QUEUES_FUSION;
 
-        fusion->rdpq_virt = pci_alloc_consistent(instance->pdev, array_size,
-                                                 &fusion->rdpq_phys);
+        fusion->rdpq_virt = pci_zalloc_consistent(instance->pdev, array_size,
+                                                  &fusion->rdpq_phys);
         if (!fusion->rdpq_virt) {
                 dev_err(&instance->pdev->dev,
                         "Failed from %s %d\n",  __func__, __LINE__);
                 return -ENOMEM;
         }
 
-        memset(fusion->rdpq_virt, 0, array_size);
         msix_count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
 
         fusion->reply_frames_desc_pool = dma_pool_create("mr_rdpq",
@@ -2981,6 +2980,9 @@ megasas_build_syspd_fusion(struct megasas_instance *instance,
                 pRAID_Context->timeout_value = cpu_to_le16(os_timeout_value);
                 pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
         } else {
+                if (os_timeout_value)
+                        os_timeout_value++;
+
                 /* system pd Fast Path */
                 io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
                 timeout_limit = (scmd->device->type == TYPE_DISK) ?
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2.h b/drivers/scsi/mpt3sas/mpi/mpi2.h
index b015c30d2c32..1e45268a78fc 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2.h
@@ -9,7 +9,7 @@
  *                 scatter/gather formats.
  * Creation Date:  June 21, 2006
  *
- * mpi2.h Version:  02.00.48
+ *  mpi2.h Version:  02.00.50
  *
  * NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
  *       prefix are for use only on MPI v2.5 products, and must not be used
@@ -114,6 +114,8 @@
  * 09-02-16  02.00.46  Bumped MPI2_HEADER_VERSION_UNIT.
  * 11-23-16  02.00.47  Bumped MPI2_HEADER_VERSION_UNIT.
  * 02-03-17  02.00.48  Bumped MPI2_HEADER_VERSION_UNIT.
+ * 06-13-17  02.00.49  Bumped MPI2_HEADER_VERSION_UNIT.
+ * 09-29-17  02.00.50  Bumped MPI2_HEADER_VERSION_UNIT.
  * --------------------------------------------------------------------------
  */
 
@@ -152,8 +154,9 @@
                                         MPI26_VERSION_MINOR)
 #define MPI2_VERSION_02_06                  (0x0206)
 
-/*Unit and Dev versioning for this MPI header set */
-#define MPI2_HEADER_VERSION_UNIT            (0x30)
+
+/* Unit and Dev versioning for this MPI header set */
+#define MPI2_HEADER_VERSION_UNIT            (0x32)
 #define MPI2_HEADER_VERSION_DEV             (0x00)
 #define MPI2_HEADER_VERSION_UNIT_MASK       (0xFF00)
 #define MPI2_HEADER_VERSION_UNIT_SHIFT      (8)
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
index 0ad88deb3176..5122920a961a 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
@@ -7,7 +7,7 @@
  *         Title:  MPI Configuration messages and pages
  * Creation Date:  November 10, 2006
  *
- *   mpi2_cnfg.h Version:  02.00.40
+ *    mpi2_cnfg.h Version:  02.00.42
  *
  * NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
  *       prefix are for use only on MPI v2.5 products, and must not be used
@@ -219,6 +219,18 @@
  *                     Added ChassisSlot field to SAS Enclosure Page 0.
  *                     Added ChassisSlot Valid bit (bit 5) to the Flags field
  *                     in SAS Enclosure Page 0.
+ * 06-13-17  02.00.41  Added MPI26_MFGPAGE_DEVID_SAS3816 and
+ *                     MPI26_MFGPAGE_DEVID_SAS3916 defines.
+ *                     Removed MPI26_MFGPAGE_DEVID_SAS4008 define.
+ *                     Added MPI26_PCIEIOUNIT1_LINKFLAGS_SRNS_EN define.
+ *                     Renamed PI26_PCIEIOUNIT1_LINKFLAGS_EN_SRIS to
+ *                     PI26_PCIEIOUNIT1_LINKFLAGS_SRIS_EN.
+ *                     Renamed MPI26_PCIEIOUNIT1_LINKFLAGS_DIS_SRIS to
+ *                     MPI26_PCIEIOUNIT1_LINKFLAGS_DIS_SEPARATE_REFCLK.
+ * 09-29-17  02.00.42  Added ControllerResetTO field to PCIe Device Page 2.
+ *                     Added NOIOB field to PCIe Device Page 2.
+ *                     Added MPI26_PCIEDEV2_CAP_DATA_BLK_ALIGN_AND_GRAN to
+ *                     the Capabilities field of PCIe Device Page 2.
  * --------------------------------------------------------------------------
  */
 
@@ -556,7 +568,8 @@ typedef struct _MPI2_CONFIG_REPLY {
 #define MPI26_MFGPAGE_DEVID_SAS3616                 (0x00D1)
 #define MPI26_MFGPAGE_DEVID_SAS3708                 (0x00D2)
 
-#define MPI26_MFGPAGE_DEVID_SAS4008                 (0x00A1)
+#define MPI26_MFGPAGE_DEVID_SAS3816                 (0x00A1)
+#define MPI26_MFGPAGE_DEVID_SAS3916                 (0x00A0)
 
 
 /*Manufacturing Page 0 */
@@ -3864,20 +3877,25 @@ typedef struct _MPI26_CONFIG_PAGE_PCIEDEV_0 {
 typedef struct _MPI26_CONFIG_PAGE_PCIEDEV_2 {
         MPI2_CONFIG_EXTENDED_PAGE_HEADER        Header; /*0x00 */
         U16     DevHandle;              /*0x08 */
-        U16     Reserved1;              /*0x0A */
-        U32     MaximumDataTransferSize;/*0x0C */
+        U8      ControllerResetTO;              /* 0x0A */
+        U8      Reserved1;              /* 0x0B */
+        U32     MaximumDataTransferSize;        /*0x0C */
         U32     Capabilities;           /*0x10 */
-        U32     Reserved2;              /*0x14 */
+        U16     NOIOB;          /* 0x14 */
+        U16     Reserved2;              /* 0x16 */
 } MPI26_CONFIG_PAGE_PCIEDEV_2, *PTR_MPI26_CONFIG_PAGE_PCIEDEV_2,
         Mpi26PCIeDevicePage2_t, *pMpi26PCIeDevicePage2_t;
 
-#define MPI26_PCIEDEVICE2_PAGEVERSION       (0x00)
+#define MPI26_PCIEDEVICE2_PAGEVERSION       (0x01)
 
 /*defines for PCIe Device Page 2 Capabilities field */
+#define MPI26_PCIEDEV2_CAP_DATA_BLK_ALIGN_AND_GRAN     (0x00000008)
 #define MPI26_PCIEDEV2_CAP_SGL_FORMAT                  (0x00000004)
 #define MPI26_PCIEDEV2_CAP_BIT_BUCKET_SUPPORT          (0x00000002)
 #define MPI26_PCIEDEV2_CAP_SGL_SUPPORT                 (0x00000001)
 
+/* Defines for the NOIOB field */
+#define MPI26_PCIEDEV2_NOIOB_UNSUPPORTED                (0x0000)
 
 /****************************************************************************
 *  PCIe Link Config Pages (MPI v2.6 and later)
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_init.h b/drivers/scsi/mpt3sas/mpi/mpi2_init.h
index 948a3ba682d7..6213ce6791ac 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2_init.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2_init.h
@@ -75,7 +75,7 @@
 
 typedef struct _MPI2_SCSI_IO_CDB_EEDP32 {
         U8 CDB[20];             /*0x00 */
-        U32 PrimaryReferenceTag;        /*0x14 */
+        __be32 PrimaryReferenceTag;     /*0x14 */
         U16 PrimaryApplicationTag;      /*0x18 */
         U16 PrimaryApplicationTagMask;  /*0x1A */
         U32 TransferLength;     /*0x1C */
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h b/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h
index cc2aff7aa67b..1faec3a93e69 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h
@@ -7,7 +7,7 @@
  *         Title:  MPI IOC, Port, Event, FW Download, and FW Upload messages
  * Creation Date:  October 11, 2006
  *
- * mpi2_ioc.h Version:  02.00.32
+ * mpi2_ioc.h Version:  02.00.34
  *
  * NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
  *       prefix are for use only on MPI v2.5 products, and must not be used
@@ -167,6 +167,10 @@
  * 02-02-17   02.00.32 Added MPI2_FW_DOWNLOAD_ITYPE_CBB_BACKUP.
  *                     Added MPI25_EVENT_DATA_ACTIVE_CABLE_EXCEPT and related
  *                     defines for the ReasonCode field.
+ * 06-13-17   02.00.33 Added MPI2_FW_DOWNLOAD_ITYPE_CPLD.
+ * 09-29-17   02.00.34 Added MPI26_EVENT_PCIDEV_STAT_RC_PCIE_HOT_RESET_FAILED
+ *                     to the ReasonCode field in PCIe Device Status Change
+ *                     Event Data.
  * --------------------------------------------------------------------------
  */
 
@@ -1182,6 +1186,7 @@ typedef struct _MPI26_EVENT_DATA_PCIE_DEVICE_STATUS_CHANGE {
 #define MPI26_EVENT_PCIDEV_STAT_RC_CMP_INTERNAL_DEV_RESET               (0x0E)
 #define MPI26_EVENT_PCIDEV_STAT_RC_CMP_TASK_ABORT_INTERNAL              (0x0F)
 #define MPI26_EVENT_PCIDEV_STAT_RC_DEV_INIT_FAILURE                     (0x10)
+#define MPI26_EVENT_PCIDEV_STAT_RC_PCIE_HOT_RESET_FAILED                (0x11)
 
 
 /*PCIe Enumeration Event data (MPI v2.6 and later) */
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index 61f93a134956..bf04fa90f433 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -87,7 +87,7 @@ MODULE_PARM_DESC(msix_disable, " disable msix routed interrupts (default=0)");
 
 static int smp_affinity_enable = 1;
 module_param(smp_affinity_enable, int, S_IRUGO);
-MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disbale Default: enable(1)");
+MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)");
 
 static int max_msix_vectors = -1;
 module_param(max_msix_vectors, int, 0);
@@ -297,12 +297,15 @@ static void *
 _base_get_chain_buffer_dma_to_chain_buffer(struct MPT3SAS_ADAPTER *ioc,
                 dma_addr_t chain_buffer_dma)
 {
-        u16 index;
-
-        for (index = 0; index < ioc->chain_depth; index++) {
-                if (ioc->chain_lookup[index].chain_buffer_dma ==
-                                chain_buffer_dma)
-                        return ioc->chain_lookup[index].chain_buffer;
+        u16 index, j;
+        struct chain_tracker *ct;
+
+        for (index = 0; index < ioc->scsiio_depth; index++) {
+                for (j = 0; j < ioc->chains_needed_per_io; j++) {
+                        ct = &ioc->chain_lookup[index].chains_per_smid[j];
+                        if (ct && ct->chain_buffer_dma == chain_buffer_dma)
+                                return ct->chain_buffer;
+                }
         }
         pr_info(MPT3SAS_FMT
             "Provided chain_buffer_dma address is not in the lookup list\n",
@@ -394,13 +397,14 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
         buff_ptr_phys = buffer_iomem_phys;
         WARN_ON(buff_ptr_phys > U32_MAX);
 
-        if (sgel->FlagsLength &
+        if (le32_to_cpu(sgel->FlagsLength) &
                         (MPI2_SGE_FLAGS_HOST_TO_IOC << MPI2_SGE_FLAGS_SHIFT))
                 is_write = 1;
 
         for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) {
 
-                sgl_flags = (sgel->FlagsLength >> MPI2_SGE_FLAGS_SHIFT);
+                sgl_flags =
+                    (le32_to_cpu(sgel->FlagsLength) >> MPI2_SGE_FLAGS_SHIFT);
 
                 switch (sgl_flags & MPI2_SGE_FLAGS_ELEMENT_MASK) {
                 case MPI2_SGE_FLAGS_CHAIN_ELEMENT:
@@ -411,7 +415,7 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
                          */
                         sgel_next =
                                 _base_get_chain_buffer_dma_to_chain_buffer(ioc,
-                                                sgel->Address);
+                                                le32_to_cpu(sgel->Address));
                         if (sgel_next == NULL)
                                 return;
                         /*
@@ -426,7 +430,8 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
                         dst_addr_phys = _base_get_chain_phys(ioc,
                                                 smid, sge_chain_count);
                         WARN_ON(dst_addr_phys > U32_MAX);
-                        sgel->Address = (u32)dst_addr_phys;
+                        sgel->Address =
+                                cpu_to_le32(lower_32_bits(dst_addr_phys));
                         sgel = sgel_next;
                         sge_chain_count++;
                         break;
@@ -435,22 +440,28 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
                                 if (is_scsiio_req) {
                                         _base_clone_to_sys_mem(buff_ptr,
                                             sg_virt(sg_scmd),
-                                            (sgel->FlagsLength & 0x00ffffff));
+                                            (le32_to_cpu(sgel->FlagsLength) &
+                                            0x00ffffff));
                                         /*
                                          * FIXME: this relies on a a zero
                                          * PCI mem_offset.
                                          */
-                                        sgel->Address = (u32)buff_ptr_phys;
+                                        sgel->Address =
+                                            cpu_to_le32((u32)buff_ptr_phys);
                                 } else {
                                         _base_clone_to_sys_mem(buff_ptr,
                                             ioc->config_vaddr,
-                                            (sgel->FlagsLength & 0x00ffffff));
-                                        sgel->Address = (u32)buff_ptr_phys;
+                                            (le32_to_cpu(sgel->FlagsLength) &
+                                            0x00ffffff));
+                                        sgel->Address =
+                                            cpu_to_le32((u32)buff_ptr_phys);
                                 }
                         }
-                        buff_ptr += (sgel->FlagsLength & 0x00ffffff);
-                        buff_ptr_phys += (sgel->FlagsLength & 0x00ffffff);
-                        if ((sgel->FlagsLength &
+                        buff_ptr += (le32_to_cpu(sgel->FlagsLength) &
+                            0x00ffffff);
+                        buff_ptr_phys += (le32_to_cpu(sgel->FlagsLength) &
+                            0x00ffffff);
+                        if ((le32_to_cpu(sgel->FlagsLength) &
                             (MPI2_SGE_FLAGS_END_OF_BUFFER
                                         << MPI2_SGE_FLAGS_SHIFT)))
                                 goto eob_clone_chain;
@@ -1019,6 +1030,9 @@ _base_display_event_data(struct MPT3SAS_ADAPTER *ioc,
         case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION:
                 desc = "Cable Event";
                 break;
+        case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
+                desc = "SAS Device Discovery Error";
+                break;
         case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
                 desc = "PCIE Device Status Change";
                 break;
@@ -1433,7 +1447,7 @@ _base_interrupt(int irq, void *bus_id)
                                     cpu_to_le32(reply);
                                 if (ioc->is_mcpu_endpoint)
                                         _base_clone_reply_to_sys_mem(ioc,
-                                                cpu_to_le32(reply),
+                                                reply,
                                                 ioc->reply_free_host_index);
                                 writel(ioc->reply_free_host_index,
                                     &ioc->chip->ReplyFreeHostIndex);
@@ -1671,7 +1685,8 @@ _base_add_sg_single_64(void *paddr, u32 flags_length, dma_addr_t dma_addr)
  * @ioc: per adapter object
  * @scmd: SCSI commands of the IO request
  *
- * Returns chain tracker(from ioc->free_chain_list)
+ * Returns chain tracker from chain_lookup table using key as
+ * smid and smid's chain_offset.
  */
 static struct chain_tracker *
 _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc,
@@ -1679,20 +1694,15 @@ _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc,
 {
         struct chain_tracker *chain_req;
         struct scsiio_tracker *st = scsi_cmd_priv(scmd);
-        unsigned long flags;
+        u16 smid = st->smid;
+        u8 chain_offset =
+           atomic_read(&ioc->chain_lookup[smid - 1].chain_offset);
 
-        spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
-        if (list_empty(&ioc->free_chain_list)) {
-                spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
-                dfailprintk(ioc, pr_warn(MPT3SAS_FMT
-                        "chain buffers not available\n", ioc->name));
+        if (chain_offset == ioc->chains_needed_per_io)
                 return NULL;
-        }
-        chain_req = list_entry(ioc->free_chain_list.next,
-            struct chain_tracker, tracker_list);
-        list_del_init(&chain_req->tracker_list);
-        list_add_tail(&chain_req->tracker_list, &st->chain_list);
-        spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
+
+        chain_req = &ioc->chain_lookup[smid - 1].chains_per_smid[chain_offset];
+        atomic_inc(&ioc->chain_lookup[smid - 1].chain_offset);
         return chain_req;
 }
 
@@ -3044,7 +3054,7 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
 
                 for (i = 0; i < ioc->combined_reply_index_count; i++) {
                         ioc->replyPostRegisterIndex[i] = (resource_size_t *)
-                             ((u8 *)&ioc->chip->Doorbell +
+                             ((u8 __force *)&ioc->chip->Doorbell +
                              MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET +
                              (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET));
                 }
@@ -3273,13 +3283,7 @@ void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc,
                 return;
         st->cb_idx = 0xFF;
         st->direct_io = 0;
-        if (!list_empty(&st->chain_list)) {
-                unsigned long flags;
-
-                spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
-                list_splice_init(&st->chain_list, &ioc->free_chain_list);
-                spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
-        }
+        atomic_set(&ioc->chain_lookup[st->smid - 1].chain_offset, 0);
 }
 
 /**
@@ -3339,7 +3343,7 @@ _base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr,
                                         spinlock_t *writeq_lock)
 {
         unsigned long flags;
-        __u64 data_out = cpu_to_le64(b);
+        __u64 data_out = b;
 
         spin_lock_irqsave(writeq_lock, flags);
         writel((u32)(data_out), addr);
@@ -3362,7 +3366,7 @@ _base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr,
 static inline void
 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
 {
-        writeq(cpu_to_le64(b), addr);
+        writeq(b, addr);
 }
 #else
 static inline void
@@ -3389,7 +3393,7 @@ _base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
         __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
 
         _clone_sg_entries(ioc, (void *) mfp, smid);
-        mpi_req_iomem = (void *)ioc->chip +
+        mpi_req_iomem = (void __force *)ioc->chip +
                         MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
         _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
                                         ioc->request_sz);
@@ -3473,7 +3477,8 @@ mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 
                 request_hdr = (MPI2RequestHeader_t *)mfp;
                 /* TBD 256 is offset within sys register. */
-                mpi_req_iomem = (void *)ioc->chip + MPI_FRAME_START_OFFSET
+                mpi_req_iomem = (void __force *)ioc->chip
+                                        + MPI_FRAME_START_OFFSET
                                         + (smid * ioc->request_sz);
                 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
                                                         ioc->request_sz);
@@ -3542,7 +3547,7 @@ mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)
 
                 _clone_sg_entries(ioc, (void *) mfp, smid);
                 /* TBD 256 is offset within sys register */
-                mpi_req_iomem = (void *)ioc->chip +
+                mpi_req_iomem = (void __force *)ioc->chip +
                         MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
                 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
                                                         ioc->request_sz);
@@ -3823,6 +3828,105 @@ _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc)
 }
 
 /**
+ * _base_display_fwpkg_version - sends FWUpload request to pull FWPkg
+ *                              version from FW Image Header.
+ * @ioc: per adapter object
+ *
+ * Returns 0 for success, non-zero for failure.
+ */
+        static int
+_base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc)
+{
+        Mpi2FWImageHeader_t *FWImgHdr;
+        Mpi25FWUploadRequest_t *mpi_request;
+        Mpi2FWUploadReply_t mpi_reply;
+        int r = 0;
+        void *fwpkg_data = NULL;
+        dma_addr_t fwpkg_data_dma;
+        u16 smid, ioc_status;
+        size_t data_length;
+
+        dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
+                                __func__));
+
+        if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
+                pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
+                                ioc->name, __func__);
+                return -EAGAIN;
+        }
+
+        data_length = sizeof(Mpi2FWImageHeader_t);
+        fwpkg_data = pci_alloc_consistent(ioc->pdev, data_length,
+                        &fwpkg_data_dma);
+        if (!fwpkg_data) {
+                pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
+                                ioc->name, __FILE__, __LINE__, __func__);
+                return -ENOMEM;
+        }
+
+        smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
+        if (!smid) {
+                pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
+                                ioc->name, __func__);
+                r = -EAGAIN;
+                goto out;
+        }
+
+        ioc->base_cmds.status = MPT3_CMD_PENDING;
+        mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
+        ioc->base_cmds.smid = smid;
+        memset(mpi_request, 0, sizeof(Mpi25FWUploadRequest_t));
+        mpi_request->Function = MPI2_FUNCTION_FW_UPLOAD;
+        mpi_request->ImageType = MPI2_FW_UPLOAD_ITYPE_FW_FLASH;
+        mpi_request->ImageSize = cpu_to_le32(data_length);
+        ioc->build_sg(ioc, &mpi_request->SGL, 0, 0, fwpkg_data_dma,
+                        data_length);
+        init_completion(&ioc->base_cmds.done);
+        mpt3sas_base_put_smid_default(ioc, smid);
+        /* Wait for 15 seconds */
+        wait_for_completion_timeout(&ioc->base_cmds.done,
+                        FW_IMG_HDR_READ_TIMEOUT*HZ);
+        pr_info(MPT3SAS_FMT "%s: complete\n",
+                        ioc->name, __func__);
+        if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
+                pr_err(MPT3SAS_FMT "%s: timeout\n",
+                                ioc->name, __func__);
+                _debug_dump_mf(mpi_request,
+                                sizeof(Mpi25FWUploadRequest_t)/4);
+                r = -ETIME;
+        } else {
+                memset(&mpi_reply, 0, sizeof(Mpi2FWUploadReply_t));
+                if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) {
+                        memcpy(&mpi_reply, ioc->base_cmds.reply,
+                                        sizeof(Mpi2FWUploadReply_t));
+                        ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
+                                                MPI2_IOCSTATUS_MASK;
+                        if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
+                                FWImgHdr = (Mpi2FWImageHeader_t *)fwpkg_data;
+                                if (FWImgHdr->PackageVersion.Word) {
+                                        pr_info(MPT3SAS_FMT "FW Package Version"
+                                        "(%02d.%02d.%02d.%02d)\n",
+                                        ioc->name,
+                                        FWImgHdr->PackageVersion.Struct.Major,
+                                        FWImgHdr->PackageVersion.Struct.Minor,
+                                        FWImgHdr->PackageVersion.Struct.Unit,
+                                        FWImgHdr->PackageVersion.Struct.Dev);
+                                }
+                        } else {
+                                _debug_dump_mf(&mpi_reply,
+                                                sizeof(Mpi2FWUploadReply_t)/4);
+                        }
+                }
+        }
+        ioc->base_cmds.status = MPT3_CMD_NOT_USED;
+out:
+        if (fwpkg_data)
+                pci_free_consistent(ioc->pdev, data_length, fwpkg_data,
+                                fwpkg_data_dma);
+        return r;
+}
+
+/**
  * _base_display_ioc_capabilities - Disply IOC's capabilities.
  * @ioc: per adapter object
  *
@@ -4038,6 +4142,7 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
         Mpi2ConfigReply_t mpi_reply;
         u32 iounit_pg1_flags;
 
+        ioc->nvme_abort_timeout = 30;
         mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply, &ioc->manu_pg0);
         if (ioc->ir_firmware)
                 mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply,
@@ -4056,6 +4161,18 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
                 mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply,
                     &ioc->manu_pg11);
         }
+        if (ioc->manu_pg11.AddlFlags2 & NVME_TASK_MNGT_CUSTOM_MASK)
+                ioc->tm_custom_handling = 1;
+        else {
+                ioc->tm_custom_handling = 0;
+                if (ioc->manu_pg11.NVMeAbortTO < NVME_TASK_ABORT_MIN_TIMEOUT)
+                        ioc->nvme_abort_timeout = NVME_TASK_ABORT_MIN_TIMEOUT;
+                else if (ioc->manu_pg11.NVMeAbortTO >
+                                        NVME_TASK_ABORT_MAX_TIMEOUT)
+                        ioc->nvme_abort_timeout = NVME_TASK_ABORT_MAX_TIMEOUT;
+                else
+                        ioc->nvme_abort_timeout = ioc->manu_pg11.NVMeAbortTO;
+        }
 
         mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2);
         mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3);
@@ -4085,6 +4202,27 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
 }
 
 /**
+ * mpt3sas_free_enclosure_list - release memory
+ * @ioc: per adapter object
+ *
+ * Free memory allocated during encloure add.
+ *
+ * Return nothing.
+ */
+void
+mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc)
+{
+        struct _enclosure_node *enclosure_dev, *enclosure_dev_next;
+
+        /* Free enclosure list */
+        list_for_each_entry_safe(enclosure_dev,
+                        enclosure_dev_next, &ioc->enclosure_list, list) {
+                list_del(&enclosure_dev->list);
+                kfree(enclosure_dev);
+        }
+}
+
+/**
  * _base_release_memory_pools - release memory
  * @ioc: per adapter object
  *
@@ -4096,6 +4234,8 @@ static void
 _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 {
         int i = 0;
+        int j = 0;
+        struct chain_tracker *ct;
         struct reply_post_struct *rps;
 
         dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
@@ -4153,7 +4293,14 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
                         }
                 } while (ioc->rdpq_array_enable &&
                            (++i < ioc->reply_queue_count));
-
+                if (ioc->reply_post_free_array &&
+                        ioc->rdpq_array_enable) {
+                        dma_pool_free(ioc->reply_post_free_array_dma_pool,
+                                ioc->reply_post_free_array,
+                                ioc->reply_post_free_array_dma);
+                        ioc->reply_post_free_array = NULL;
+                }
+                dma_pool_destroy(ioc->reply_post_free_array_dma_pool);
                 dma_pool_destroy(ioc->reply_post_free_dma_pool);
                 kfree(ioc->reply_post);
         }
@@ -4179,19 +4326,49 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
         kfree(ioc->hpr_lookup);
         kfree(ioc->internal_lookup);
         if (ioc->chain_lookup) {
-                for (i = 0; i < ioc->chain_depth; i++) {
-                        if (ioc->chain_lookup[i].chain_buffer)
-                                dma_pool_free(ioc->chain_dma_pool,
-                                    ioc->chain_lookup[i].chain_buffer,
-                                    ioc->chain_lookup[i].chain_buffer_dma);
+                for (i = 0; i < ioc->scsiio_depth; i++) {
+                        for (j = ioc->chains_per_prp_buffer;
+                            j < ioc->chains_needed_per_io; j++) {
+                                ct = &ioc->chain_lookup[i].chains_per_smid[j];
+                                if (ct && ct->chain_buffer)
+                                        dma_pool_free(ioc->chain_dma_pool,
+                                                ct->chain_buffer,
+                                                ct->chain_buffer_dma);
+                        }
+                        kfree(ioc->chain_lookup[i].chains_per_smid);
                 }
                 dma_pool_destroy(ioc->chain_dma_pool);
-                free_pages((ulong)ioc->chain_lookup, ioc->chain_pages);
+                kfree(ioc->chain_lookup);
                 ioc->chain_lookup = NULL;
         }
 }
 
 /**
+ * is_MSB_are_same - checks whether all reply queues in a set are
+ *      having same upper 32bits in their base memory address.
+ * @reply_pool_start_address: Base address of a reply queue set
+ * @pool_sz: Size of single Reply Descriptor Post Queues pool size
+ *
+ * Returns 1 if reply queues in a set have a same upper 32bits
+ * in their base memory address,
+ * else 0
+ */
+
+static int
+is_MSB_are_same(long reply_pool_start_address, u32 pool_sz)
+{
+        long reply_pool_end_address;
+
+        reply_pool_end_address = reply_pool_start_address + pool_sz;
+
+        if (upper_32_bits(reply_pool_start_address) ==
+                upper_32_bits(reply_pool_end_address))
+                return 1;
+        else
+                return 0;
+}
+
+/**
  * _base_allocate_memory_pools - allocate start of day memory pools
  * @ioc: per adapter object
  *
@@ -4203,12 +4380,13 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
         struct mpt3sas_facts *facts;
         u16 max_sge_elements;
         u16 chains_needed_per_io;
-        u32 sz, total_sz, reply_post_free_sz;
+        u32 sz, total_sz, reply_post_free_sz, reply_post_free_array_sz;
         u32 retry_sz;
         u16 max_request_credit, nvme_blocks_needed;
         unsigned short sg_tablesize;
         u16 sge_size;
-        int i;
+        int i, j;
+        struct chain_tracker *ct;
 
         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
             __func__));
@@ -4489,37 +4667,23 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
                 ioc->name, ioc->request, ioc->scsiio_depth));
 
         ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH);
-        sz = ioc->chain_depth * sizeof(struct chain_tracker);
-        ioc->chain_pages = get_order(sz);
-        ioc->chain_lookup = (struct chain_tracker *)__get_free_pages(
-            GFP_KERNEL, ioc->chain_pages);
+        sz = ioc->scsiio_depth * sizeof(struct chain_lookup);
+        ioc->chain_lookup = kzalloc(sz, GFP_KERNEL);
         if (!ioc->chain_lookup) {
-                pr_err(MPT3SAS_FMT "chain_lookup: __get_free_pages failed\n",
-                        ioc->name);
+                pr_err(MPT3SAS_FMT "chain_lookup: __get_free_pages "
+                                "failed\n", ioc->name);
                 goto out;
         }
-        ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
-            ioc->chain_segment_sz, 16, 0);
-        if (!ioc->chain_dma_pool) {
-                pr_err(MPT3SAS_FMT "chain_dma_pool: dma_pool_create failed\n",
-                        ioc->name);
-                goto out;
-        }
-        for (i = 0; i < ioc->chain_depth; i++) {
-                ioc->chain_lookup[i].chain_buffer = dma_pool_alloc(
-                    ioc->chain_dma_pool , GFP_KERNEL,
-                    &ioc->chain_lookup[i].chain_buffer_dma);
-                if (!ioc->chain_lookup[i].chain_buffer) {
-                        ioc->chain_depth = i;
-                        goto chain_done;
+
+        sz = ioc->chains_needed_per_io * sizeof(struct chain_tracker);
+        for (i = 0; i < ioc->scsiio_depth; i++) {
+                ioc->chain_lookup[i].chains_per_smid = kzalloc(sz, GFP_KERNEL);
+                if (!ioc->chain_lookup[i].chains_per_smid) {
+                        pr_err(MPT3SAS_FMT "chain_lookup: "
+                                        " kzalloc failed\n", ioc->name);
+                        goto out;
                 }
-                total_sz += ioc->chain_segment_sz;
         }
- chain_done:
-        dinitprintk(ioc, pr_info(MPT3SAS_FMT
-                "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
-                ioc->name, ioc->chain_depth, ioc->chain_segment_sz,
-                ((ioc->chain_depth *  ioc->chain_segment_sz))/1024));
 
         /* initialize hi-priority queue smid's */
         ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth,
@@ -4561,6 +4725,7 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
          * be required for NVMe PRP's, only each set of NVMe blocks will be
          * contiguous, so a new set is allocated for each possible I/O.
          */
+        ioc->chains_per_prp_buffer = 0;
         if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
                 nvme_blocks_needed =
                         (ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1;
@@ -4583,6 +4748,11 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
                             ioc->name);
                         goto out;
                 }
+
+                ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz;
+                ioc->chains_per_prp_buffer = min(ioc->chains_per_prp_buffer,
+                                                ioc->chains_needed_per_io);
+
                 for (i = 0; i < ioc->scsiio_depth; i++) {
                         ioc->pcie_sg_lookup[i].pcie_sgl = dma_pool_alloc(
                                 ioc->pcie_sgl_dma_pool, GFP_KERNEL,
@@ -4593,13 +4763,55 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
                                     ioc->name);
                                 goto out;
                         }
+                        for (j = 0; j < ioc->chains_per_prp_buffer; j++) {
+                                ct = &ioc->chain_lookup[i].chains_per_smid[j];
+                                ct->chain_buffer =
+                                    ioc->pcie_sg_lookup[i].pcie_sgl +
+                                    (j * ioc->chain_segment_sz);
+                                ct->chain_buffer_dma =
+                                    ioc->pcie_sg_lookup[i].pcie_sgl_dma +
+                                    (j * ioc->chain_segment_sz);
+                        }
                 }
 
                 dinitprintk(ioc, pr_info(MPT3SAS_FMT "PCIe sgl pool depth(%d), "
                         "element_size(%d), pool_size(%d kB)\n", ioc->name,
                         ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024));
+                dinitprintk(ioc, pr_info(MPT3SAS_FMT "Number of chains can "
+                    "fit in a PRP page(%d)\n", ioc->name,
+                    ioc->chains_per_prp_buffer));
                 total_sz += sz * ioc->scsiio_depth;
         }
+
+        ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
+            ioc->chain_segment_sz, 16, 0);
+        if (!ioc->chain_dma_pool) {
+                pr_err(MPT3SAS_FMT "chain_dma_pool: dma_pool_create failed\n",
+                        ioc->name);
+                goto out;
+        }
+        for (i = 0; i < ioc->scsiio_depth; i++) {
+                for (j = ioc->chains_per_prp_buffer;
+                                j < ioc->chains_needed_per_io; j++) {
+                        ct = &ioc->chain_lookup[i].chains_per_smid[j];
+                        ct->chain_buffer = dma_pool_alloc(
+                                        ioc->chain_dma_pool, GFP_KERNEL,
+                                        &ct->chain_buffer_dma);
+                        if (!ct->chain_buffer) {
+                                pr_err(MPT3SAS_FMT "chain_lookup: "
+                                " pci_pool_alloc failed\n", ioc->name);
+                                _base_release_memory_pools(ioc);
+                                goto out;
+                        }
+                }
+                total_sz += ioc->chain_segment_sz;
+        }
+
+        dinitprintk(ioc, pr_info(MPT3SAS_FMT
+                "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
+                ioc->name, ioc->chain_depth, ioc->chain_segment_sz,
+                ((ioc->chain_depth *  ioc->chain_segment_sz))/1024));
+
         /* sense buffers, 4 byte align */
         sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;
         ioc->sense_dma_pool = dma_pool_create("sense pool", &ioc->pdev->dev, sz,
@@ -4616,6 +4828,37 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
                     ioc->name);
                 goto out;
         }
+        /* sense buffer requires to be in same 4 gb region.
+         * Below function will check the same.
+         * In case of failure, new pci pool will be created with updated
+         * alignment. Older allocation and pool will be destroyed.
+         * Alignment will be used such a way that next allocation if
+         * success, will always meet same 4gb region requirement.
+         * Actual requirement is not alignment, but we need start and end of
+         * DMA address must have same upper 32 bit address.
+         */
+        if (!is_MSB_are_same((long)ioc->sense, sz)) {
+                //Release Sense pool & Reallocate
+                dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
+                dma_pool_destroy(ioc->sense_dma_pool);
+                ioc->sense = NULL;
+
+                ioc->sense_dma_pool =
+                        dma_pool_create("sense pool", &ioc->pdev->dev, sz,
+                                                roundup_pow_of_two(sz), 0);
+                if (!ioc->sense_dma_pool) {
+                        pr_err(MPT3SAS_FMT "sense pool: pci_pool_create failed\n",
+                                        ioc->name);
+                        goto out;
+                }
+                ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, GFP_KERNEL,
+                                &ioc->sense_dma);
+                if (!ioc->sense) {
+                        pr_err(MPT3SAS_FMT "sense pool: pci_pool_alloc failed\n",
+                                        ioc->name);
+                        goto out;
+                }
+        }
         dinitprintk(ioc, pr_info(MPT3SAS_FMT
             "sense pool(0x%p): depth(%d), element_size(%d), pool_size"
             "(%d kB)\n", ioc->name, ioc->sense, ioc->scsiio_depth,
@@ -4675,6 +4918,28 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
                 ioc->name, (unsigned long long)ioc->reply_free_dma));
         total_sz += sz;
 
+        if (ioc->rdpq_array_enable) {
+                reply_post_free_array_sz = ioc->reply_queue_count *
+                    sizeof(Mpi2IOCInitRDPQArrayEntry);
+                ioc->reply_post_free_array_dma_pool =
+                    dma_pool_create("reply_post_free_array pool",
+                    &ioc->pdev->dev, reply_post_free_array_sz, 16, 0);
+                if (!ioc->reply_post_free_array_dma_pool) {
+                        dinitprintk(ioc,
+                            pr_info(MPT3SAS_FMT "reply_post_free_array pool: "
+                            "dma_pool_create failed\n", ioc->name));
+                        goto out;
+                }
+                ioc->reply_post_free_array =
+                    dma_pool_alloc(ioc->reply_post_free_array_dma_pool,
+                    GFP_KERNEL, &ioc->reply_post_free_array_dma);
+                if (!ioc->reply_post_free_array) {
+                        dinitprintk(ioc,
+                            pr_info(MPT3SAS_FMT "reply_post_free_array pool: "
+                            "dma_pool_alloc failed\n", ioc->name));
+                        goto out;
+                }
+        }
         ioc->config_page_sz = 512;
         ioc->config_page = pci_alloc_consistent(ioc->pdev,
             ioc->config_page_sz, &ioc->config_page_dma);
@@ -5002,7 +5267,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
 
         /* send message 32-bits at a time */
         for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) {
-                writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell);
+                writel((u32)(request[i]), &ioc->chip->Doorbell);
                 if ((_base_wait_for_doorbell_ack(ioc, 5)))
                         failed = 1;
         }
@@ -5023,7 +5288,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
         }
 
         /* read the first two 16-bits, it gives the total length of the reply */
-        reply[0] = le16_to_cpu(readl(&ioc->chip->Doorbell)
+        reply[0] = (u16)(readl(&ioc->chip->Doorbell)
             & MPI2_DOORBELL_DATA_MASK);
         writel(0, &ioc->chip->HostInterruptStatus);
         if ((_base_wait_for_doorbell_int(ioc, 5))) {
@@ -5032,7 +5297,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
                         ioc->name, __LINE__);
                 return -EFAULT;
         }
-        reply[1] = le16_to_cpu(readl(&ioc->chip->Doorbell)
+        reply[1] = (u16)(readl(&ioc->chip->Doorbell)
             & MPI2_DOORBELL_DATA_MASK);
         writel(0, &ioc->chip->HostInterruptStatus);
 
@@ -5046,7 +5311,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
                 if (i >=  reply_bytes/2) /* overflow case */
                         readl(&ioc->chip->Doorbell);
                 else
-                        reply[i] = le16_to_cpu(readl(&ioc->chip->Doorbell)
+                        reply[i] = (u16)(readl(&ioc->chip->Doorbell)
                             & MPI2_DOORBELL_DATA_MASK);
                 writel(0, &ioc->chip->HostInterruptStatus);
         }
@@ -5481,8 +5746,6 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
         ktime_t current_time;
         u16 ioc_status;
         u32 reply_post_free_array_sz = 0;
-        Mpi2IOCInitRDPQArrayEntry *reply_post_free_array = NULL;
-        dma_addr_t reply_post_free_array_dma;
 
         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
             __func__));
@@ -5516,23 +5779,14 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
         if (ioc->rdpq_array_enable) {
                 reply_post_free_array_sz = ioc->reply_queue_count *
                     sizeof(Mpi2IOCInitRDPQArrayEntry);
-                reply_post_free_array = pci_alloc_consistent(ioc->pdev,
-                        reply_post_free_array_sz, &reply_post_free_array_dma);
-                if (!reply_post_free_array) {
-                        pr_err(MPT3SAS_FMT
-                        "reply_post_free_array: pci_alloc_consistent failed\n",
-                        ioc->name);
-                        r = -ENOMEM;
-                        goto out;
-                }
-                memset(reply_post_free_array, 0, reply_post_free_array_sz);
+                memset(ioc->reply_post_free_array, 0, reply_post_free_array_sz);
                 for (i = 0; i < ioc->reply_queue_count; i++)
-                        reply_post_free_array[i].RDPQBaseAddress =
+                        ioc->reply_post_free_array[i].RDPQBaseAddress =
                             cpu_to_le64(
                                 (u64)ioc->reply_post[i].reply_post_free_dma);
                 mpi_request.MsgFlags = MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE;
                 mpi_request.ReplyDescriptorPostQueueAddress =
-                    cpu_to_le64((u64)reply_post_free_array_dma);
+                    cpu_to_le64((u64)ioc->reply_post_free_array_dma);
         } else {
                 mpi_request.ReplyDescriptorPostQueueAddress =
                     cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma);
@@ -5562,7 +5816,7 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
         if (r != 0) {
                 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
                     ioc->name, __func__, r);
-                goto out;
+                return r;
         }
 
         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
@@ -5572,11 +5826,6 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
                 r = -EIO;
         }
 
-out:
-        if (reply_post_free_array)
-                pci_free_consistent(ioc->pdev, reply_post_free_array_sz,
-                                    reply_post_free_array,
-                                    reply_post_free_array_dma);
         return r;
 }
 
@@ -6157,12 +6406,6 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
                     &ioc->internal_free_list);
         }
 
-        /* chain pool */
-        INIT_LIST_HEAD(&ioc->free_chain_list);
-        for (i = 0; i < ioc->chain_depth; i++)
-                list_add_tail(&ioc->chain_lookup[i].tracker_list,
-                    &ioc->free_chain_list);
-
         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
 
         /* initialize Reply Free Queue */
@@ -6172,7 +6415,7 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
                 ioc->reply_free[i] = cpu_to_le32(reply_address);
                 if (ioc->is_mcpu_endpoint)
                         _base_clone_reply_to_sys_mem(ioc,
-                                        (__le32)reply_address, i);
+                                        reply_address, i);
         }
 
         /* initialize reply queues */
@@ -6230,12 +6473,18 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
  skip_init_reply_post_host_index:
 
         _base_unmask_interrupts(ioc);
+
+        if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
+                r = _base_display_fwpkg_version(ioc);
+                if (r)
+                        return r;
+        }
+
+        _base_static_config_pages(ioc);
         r = _base_event_notification(ioc);
         if (r)
                 return r;
 
-        _base_static_config_pages(ioc);
-
         if (ioc->is_driver_loading) {
 
                 if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier
@@ -6492,6 +6741,7 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
         _base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED);
         _base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD);
         _base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
+        _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
         if (ioc->hba_mpi_version_belonged == MPI26_VERSION) {
                 if (ioc->is_gen35_ioc) {
                         _base_unmask_events(ioc,
@@ -6558,6 +6808,7 @@ mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc)
         mpt3sas_base_stop_watchdog(ioc);
         mpt3sas_base_free_resources(ioc);
         _base_release_memory_pools(ioc);
+        mpt3sas_free_enclosure_list(ioc);
         pci_set_drvdata(ioc->pdev, NULL);
         kfree(ioc->cpu_msix_table);
         if (ioc->is_warpdrive)
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
index ae36d8fb2f2b..f02974c0be4a 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
@@ -74,8 +74,8 @@
 #define MPT3SAS_DRIVER_NAME             "mpt3sas"
 #define MPT3SAS_AUTHOR "Avago Technologies <MPT-FusionLinux.pdl@avagotech.com>"
 #define MPT3SAS_DESCRIPTION     "LSI MPT Fusion SAS 3.0 Device Driver"
-#define MPT3SAS_DRIVER_VERSION          "17.100.00.00"
-#define MPT3SAS_MAJOR_VERSION           17
+#define MPT3SAS_DRIVER_VERSION          "25.100.00.00"
+#define MPT3SAS_MAJOR_VERSION           25
 #define MPT3SAS_MINOR_VERSION           100
 #define MPT3SAS_BUILD_VERSION           0
 #define MPT3SAS_RELEASE_VERSION 00
@@ -138,6 +138,7 @@
 #define MAX_CHAIN_ELEMT_SZ              16
 #define DEFAULT_NUM_FWCHAIN_ELEMTS      8
 
+#define FW_IMG_HDR_READ_TIMEOUT 15
 /*
  * NVMe defines
  */
@@ -145,8 +146,12 @@
 #define NVME_CMD_PRP1_OFFSET            24      /* PRP1 offset in NVMe cmd */
 #define NVME_CMD_PRP2_OFFSET            32      /* PRP2 offset in NVMe cmd */
 #define NVME_ERROR_RESPONSE_SIZE        16      /* Max NVME Error Response */
+#define NVME_TASK_ABORT_MIN_TIMEOUT     6
+#define NVME_TASK_ABORT_MAX_TIMEOUT     60
+#define NVME_TASK_MNGT_CUSTOM_MASK      (0x0010)
 #define NVME_PRP_PAGE_SIZE              4096    /* Page size */
 
+
 /*
  * reset phases
  */
@@ -362,7 +367,15 @@ struct Mpi2ManufacturingPage11_t {
         u8      EEDPTagMode;                    /* 09h */
         u8      Reserved3;                      /* 0Ah */
         u8      Reserved4;                      /* 0Bh */
-        __le32  Reserved5[23];                  /* 0Ch-60h*/
+        __le32  Reserved5[8];                   /* 0Ch-2Ch */
+        u16     AddlFlags2;                     /* 2Ch */
+        u8      AddlFlags3;                     /* 2Eh */
+        u8      Reserved6;                      /* 2Fh */
+        __le32  Reserved7[7];                   /* 30h - 4Bh */
+        u8      NVMeAbortTO;                    /* 4Ch */
+        u8      Reserved8;                      /* 4Dh */
+        u16     Reserved9;                      /* 4Eh */
+        __le32  Reserved10[4];                  /* 50h - 60h */
 };
 
 /**
@@ -572,6 +585,7 @@ struct _pcie_device {
         u8      enclosure_level;
         u8      connector_name[4];
         u8      *serial_number;
+        u8      reset_timeout;
         struct kref refcount;
 };
 /**
@@ -741,6 +755,17 @@ struct _sas_node {
         struct list_head sas_port_list;
 };
 
+
+/**
+ * struct _enclosure_node - enclosure information
+ * @list: list of enclosures
+ * @pg0: enclosure pg0;
+ */
+struct _enclosure_node {
+        struct list_head list;
+        Mpi2SasEnclosurePage0_t pg0;
+};
+
 /**
  * enum reset_type - reset state
  * @FORCE_BIG_HAMMER: issue diagnostic reset
@@ -770,7 +795,11 @@ struct pcie_sg_list {
 struct chain_tracker {
         void *chain_buffer;
         dma_addr_t chain_buffer_dma;
-        struct list_head tracker_list;
+};
+
+struct chain_lookup {
+        struct chain_tracker *chains_per_smid;
+        atomic_t        chain_offset;
 };
 
 /**
@@ -829,8 +858,8 @@ struct _sc_list {
  */
 struct _event_ack_list {
         struct list_head list;
-        u16     Event;
-        u32     EventContext;
+        U16     Event;
+        U32     EventContext;
 };
 
 /**
@@ -1009,6 +1038,7 @@ typedef void (*MPT3SAS_FLUSH_RUNNING_CMDS)(struct MPT3SAS_ADAPTER *ioc);
  * @iounit_pg8: static iounit page 8
  * @sas_hba: sas host object
  * @sas_expander_list: expander object list
+ * @enclosure_list: enclosure object list
  * @sas_node_lock:
  * @sas_device_list: sas device object list
  * @sas_device_init_list: sas device object list (used only at init time)
@@ -1194,6 +1224,10 @@ struct MPT3SAS_ADAPTER {
         void            *event_log;
         u32             event_masks[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];
 
+        u8              tm_custom_handling;
+        u8              nvme_abort_timeout;
+
+
         /* static config pages */
         struct mpt3sas_facts facts;
         struct mpt3sas_port_facts *pfacts;
@@ -1214,6 +1248,7 @@ struct MPT3SAS_ADAPTER {
         /* sas hba, expander, and device list */
         struct _sas_node sas_hba;
         struct list_head sas_expander_list;
+        struct list_head enclosure_list;
         spinlock_t      sas_node_lock;
         struct list_head sas_device_list;
         struct list_head sas_device_init_list;
@@ -1261,7 +1296,7 @@ struct MPT3SAS_ADAPTER {
         u32             page_size;
 
         /* chain */
-        struct chain_tracker *chain_lookup;
+        struct chain_lookup *chain_lookup;
         struct list_head free_chain_list;
         struct dma_pool *chain_dma_pool;
         ulong           chain_pages;
@@ -1315,6 +1350,9 @@ struct MPT3SAS_ADAPTER {
         u8              rdpq_array_enable;
         u8              rdpq_array_enable_assigned;
         struct dma_pool *reply_post_free_dma_pool;
+        struct dma_pool *reply_post_free_array_dma_pool;
+        Mpi2IOCInitRDPQArrayEntry *reply_post_free_array;
+        dma_addr_t reply_post_free_array_dma;
         u8              reply_queue_count;
         struct list_head reply_queue_list;
 
@@ -1384,6 +1422,7 @@ int mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc);
 void mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc);
 int mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc);
 void mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc);
+void mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc);
 int mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc,
         enum reset_type type);
 
@@ -1451,10 +1490,11 @@ u8 mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,
         u32 reply);
 void mpt3sas_scsih_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase);
 
-int mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
-        u64 lun, u8 type, u16 smid_task, u16 msix_task, ulong timeout);
+int mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, u64 lun,
+        u8 type, u16 smid_task, u16 msix_task, u8 timeout, u8 tr_method);
 int mpt3sas_scsih_issue_locked_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
-        u64 lun, u8 type, u16 smid_task, u16 msix_task, ulong timeout);
+        u64 lun, u8 type, u16 smid_task, u16 msix_task,
+        u8 timeout, u8 tr_method);
 
 void mpt3sas_scsih_set_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle);
 void mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle);
diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.c b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
index d3cb387ba9f4..3269ef43f07e 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_ctl.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
@@ -297,7 +297,7 @@ mpt3sas_ctl_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
                         nvme_error_reply =
                             (Mpi26NVMeEncapsulatedErrorReply_t *)mpi_reply;
                         sz = min_t(u32, NVME_ERROR_RESPONSE_SIZE,
-                            le32_to_cpu(nvme_error_reply->ErrorResponseCount));
+                            le16_to_cpu(nvme_error_reply->ErrorResponseCount));
                         sense_data = mpt3sas_base_get_sense_buffer(ioc, smid);
                         memcpy(ioc->ctl_cmds.sense, sense_data, sz);
                 }
@@ -644,9 +644,10 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
         MPI2RequestHeader_t *mpi_request = NULL, *request;
         MPI2DefaultReply_t *mpi_reply;
         Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request = NULL;
+        struct _pcie_device *pcie_device = NULL;
         u32 ioc_state;
         u16 smid;
-        unsigned long timeout;
+        u8 timeout;
         u8 issue_reset;
         u32 sz, sz_arg;
         void *psge;
@@ -659,6 +660,7 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
         long ret;
         u16 wait_state_count;
         u16 device_handle = MPT3SAS_INVALID_DEVICE_HANDLE;
+        u8 tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
 
         issue_reset = 0;
 
@@ -803,12 +805,13 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
                  * Build the PRPs and set direction bits.
                  * Send the request.
                  */
-                nvme_encap_request->ErrorResponseBaseAddress = ioc->sense_dma &
-                    0xFFFFFFFF00000000;
+                nvme_encap_request->ErrorResponseBaseAddress =
+                    cpu_to_le64(ioc->sense_dma & 0xFFFFFFFF00000000UL);
                 nvme_encap_request->ErrorResponseBaseAddress |=
-                    (U64)mpt3sas_base_get_sense_buffer_dma(ioc, smid);
+                   cpu_to_le64(le32_to_cpu(
+                   mpt3sas_base_get_sense_buffer_dma(ioc, smid)));
                 nvme_encap_request->ErrorResponseAllocationLength =
-                                                NVME_ERROR_RESPONSE_SIZE;
+                                        cpu_to_le16(NVME_ERROR_RESPONSE_SIZE);
                 memset(ioc->ctl_cmds.sense, 0, NVME_ERROR_RESPONSE_SIZE);
                 ioc->build_nvme_prp(ioc, smid, nvme_encap_request,
                     data_out_dma, data_out_sz, data_in_dma, data_in_sz);
@@ -1073,14 +1076,26 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
                                 ioc->name,
                                 le16_to_cpu(mpi_request->FunctionDependent1));
                         mpt3sas_halt_firmware(ioc);
-                        mpt3sas_scsih_issue_locked_tm(ioc,
-                            le16_to_cpu(mpi_request->FunctionDependent1), 0,
-                            MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 0, 30);
+                        pcie_device = mpt3sas_get_pdev_by_handle(ioc,
+                                le16_to_cpu(mpi_request->FunctionDependent1));
+                        if (pcie_device && (!ioc->tm_custom_handling))
+                                mpt3sas_scsih_issue_locked_tm(ioc,
+                                  le16_to_cpu(mpi_request->FunctionDependent1),
+                                  0, MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0,
+                                  0, pcie_device->reset_timeout,
+                                  tr_method);
+                        else
+                                mpt3sas_scsih_issue_locked_tm(ioc,
+                                  le16_to_cpu(mpi_request->FunctionDependent1),
+                                  0, MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0,
+                                  0, 30, MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET);
                 } else
                         mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
         }
 
  out:
+        if (pcie_device)
+                pcie_device_put(pcie_device);
 
         /* free memory associated with sg buffers */
         if (data_in)
diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.h b/drivers/scsi/mpt3sas/mpt3sas_ctl.h
index a44046cff0f3..18b46faef6f1 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_ctl.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.h
@@ -184,7 +184,7 @@ struct mpt3_ioctl_iocinfo {
 
 
 /* number of event log entries */
-#define MPT3SAS_CTL_EVENT_LOG_SIZE (50)
+#define MPT3SAS_CTL_EVENT_LOG_SIZE (200)
 
 /**
  * struct mpt3_ioctl_eventquery - query event count and type
diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
index 8cd3782fab49..b8d131a455d0 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
@@ -157,8 +157,8 @@ MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=7 ");
 
 
 /* raid transport support */
-struct raid_template *mpt3sas_raid_template;
-struct raid_template *mpt2sas_raid_template;
+static struct raid_template *mpt3sas_raid_template;
+static struct raid_template *mpt2sas_raid_template;
 
 
 /**
@@ -1088,7 +1088,7 @@ _scsih_pcie_device_remove(struct MPT3SAS_ADAPTER *ioc,
                 pcie_device->slot);
         if (pcie_device->connector_name[0] != '\0')
                 pr_info(MPT3SAS_FMT
-                        "removing enclosure level(0x%04x), connector name( %s)\n",
+                    "removing enclosure level(0x%04x), connector name( %s)\n",
                         ioc->name, pcie_device->enclosure_level,
                         pcie_device->connector_name);
 
@@ -1362,6 +1362,30 @@ mpt3sas_scsih_expander_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 }
 
 /**
+ * mpt3sas_scsih_enclosure_find_by_handle - exclosure device search
+ * @ioc: per adapter object
+ * @handle: enclosure handle (assigned by firmware)
+ * Context: Calling function should acquire ioc->sas_device_lock
+ *
+ * This searches for enclosure device based on handle, then returns the
+ * enclosure object.
+ */
+static struct _enclosure_node *
+mpt3sas_scsih_enclosure_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
+{
+        struct _enclosure_node *enclosure_dev, *r;
+
+        r = NULL;
+        list_for_each_entry(enclosure_dev, &ioc->enclosure_list, list) {
+                if (le16_to_cpu(enclosure_dev->pg0.EnclosureHandle) != handle)
+                        continue;
+                r = enclosure_dev;
+                goto out;
+        }
+out:
+        return r;
+}
+/**
  * mpt3sas_scsih_expander_find_by_sas_address - expander device search
  * @ioc: per adapter object
  * @sas_address: sas address
@@ -2608,6 +2632,7 @@ mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
  * @smid_task: smid assigned to the task
  * @msix_task: MSIX table index supplied by the OS
  * @timeout: timeout in seconds
+ * @tr_method: Target Reset Method
  * Context: user
  *
  * A generic API for sending task management requests to firmware.
@@ -2618,8 +2643,8 @@ mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
  * Return SUCCESS or FAILED.
  */
 int
-mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
-        u64 lun, u8 type, u16 smid_task, u16 msix_task, ulong timeout)
+mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, u64 lun,
+        u8 type, u16 smid_task, u16 msix_task, u8 timeout, u8 tr_method)
 {
         Mpi2SCSITaskManagementRequest_t *mpi_request;
         Mpi2SCSITaskManagementReply_t *mpi_reply;
@@ -2665,8 +2690,8 @@ mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
         }
 
         dtmprintk(ioc, pr_info(MPT3SAS_FMT
-                "sending tm: handle(0x%04x), task_type(0x%02x), smid(%d)\n",
-                ioc->name, handle, type, smid_task));
+                "sending tm: handle(0x%04x), task_type(0x%02x), smid(%d), timeout(%d), tr_method(0x%x)\n",
+                ioc->name, handle, type, smid_task, timeout, tr_method));
         ioc->tm_cmds.status = MPT3_CMD_PENDING;
         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
         ioc->tm_cmds.smid = smid;
@@ -2675,6 +2700,7 @@ mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
         mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
         mpi_request->DevHandle = cpu_to_le16(handle);
         mpi_request->TaskType = type;
+        mpi_request->MsgFlags = tr_method;
         mpi_request->TaskMID = cpu_to_le16(smid_task);
         int_to_scsilun(lun, (struct scsi_lun *)mpi_request->LUN);
         mpt3sas_scsih_set_tm_flag(ioc, handle);
@@ -2721,13 +2747,14 @@ out:
 }
 
 int mpt3sas_scsih_issue_locked_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
-        u64 lun, u8 type, u16 smid_task, u16 msix_task, ulong timeout)
+                u64 lun, u8 type, u16 smid_task, u16 msix_task,
+                u8 timeout, u8 tr_method)
 {
         int ret;
 
         mutex_lock(&ioc->tm_cmds.mutex);
         ret = mpt3sas_scsih_issue_tm(ioc, handle, lun, type, smid_task,
-                        msix_task, timeout);
+                        msix_task, timeout, tr_method);
         mutex_unlock(&ioc->tm_cmds.mutex);
 
         return ret;
@@ -2830,6 +2857,8 @@ scsih_abort(struct scsi_cmnd *scmd)
         u16 handle;
         int r;
 
+        u8 timeout = 30;
+        struct _pcie_device *pcie_device = NULL;
         sdev_printk(KERN_INFO, scmd->device,
                 "attempting task abort! scmd(%p)\n", scmd);
         _scsih_tm_display_info(ioc, scmd);
@@ -2864,15 +2893,20 @@ scsih_abort(struct scsi_cmnd *scmd)
         mpt3sas_halt_firmware(ioc);
 
         handle = sas_device_priv_data->sas_target->handle;
+        pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
+        if (pcie_device && (!ioc->tm_custom_handling))
+                timeout = ioc->nvme_abort_timeout;
         r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->lun,
                 MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
-                st->smid, st->msix_io, 30);
+                st->smid, st->msix_io, timeout, 0);
         /* Command must be cleared after abort */
         if (r == SUCCESS && st->cb_idx != 0xFF)
                 r = FAILED;
  out:
         sdev_printk(KERN_INFO, scmd->device, "task abort: %s scmd(%p)\n",
             ((r == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
+        if (pcie_device)
+                pcie_device_put(pcie_device);
         return r;
 }
 
@@ -2888,7 +2922,10 @@ scsih_dev_reset(struct scsi_cmnd *scmd)
         struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
         struct MPT3SAS_DEVICE *sas_device_priv_data;
         struct _sas_device *sas_device = NULL;
+        struct _pcie_device *pcie_device = NULL;
         u16     handle;
+        u8      tr_method = 0;
+        u8      tr_timeout = 30;
         int r;
 
         struct scsi_target *starget = scmd->device->sdev_target;
@@ -2926,8 +2963,16 @@ scsih_dev_reset(struct scsi_cmnd *scmd)
                 goto out;
         }
 
+        pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
+
+        if (pcie_device && (!ioc->tm_custom_handling)) {
+                tr_timeout = pcie_device->reset_timeout;
+                tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
+        } else
+                tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
         r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->lun,
-                MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET, 0, 0, 30);
+                MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET, 0, 0,
+                tr_timeout, tr_method);
         /* Check for busy commands after reset */
         if (r == SUCCESS && atomic_read(&scmd->device->device_busy))
                 r = FAILED;
@@ -2937,6 +2982,8 @@ scsih_dev_reset(struct scsi_cmnd *scmd)
 
         if (sas_device)
                 sas_device_put(sas_device);
+        if (pcie_device)
+                pcie_device_put(pcie_device);
 
         return r;
 }
@@ -2953,7 +3000,10 @@ scsih_target_reset(struct scsi_cmnd *scmd)
         struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
         struct MPT3SAS_DEVICE *sas_device_priv_data;
         struct _sas_device *sas_device = NULL;
+        struct _pcie_device *pcie_device = NULL;
         u16     handle;
+        u8      tr_method = 0;
+        u8      tr_timeout = 30;
         int r;
         struct scsi_target *starget = scmd->device->sdev_target;
         struct MPT3SAS_TARGET *target_priv_data = starget->hostdata;
@@ -2990,8 +3040,16 @@ scsih_target_reset(struct scsi_cmnd *scmd)
                 goto out;
         }
 
+        pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
+
+        if (pcie_device && (!ioc->tm_custom_handling)) {
+                tr_timeout = pcie_device->reset_timeout;
+                tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
+        } else
+                tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
         r = mpt3sas_scsih_issue_locked_tm(ioc, handle, 0,
-                MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 0, 30);
+                MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 0,
+            tr_timeout, tr_method);
         /* Check for busy commands after reset */
         if (r == SUCCESS && atomic_read(&starget->target_busy))
                 r = FAILED;
@@ -3001,7 +3059,8 @@ scsih_target_reset(struct scsi_cmnd *scmd)
 
         if (sas_device)
                 sas_device_put(sas_device);
-
+        if (pcie_device)
+                pcie_device_put(pcie_device);
         return r;
 }
 
@@ -3535,6 +3594,7 @@ _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
         unsigned long flags;
         struct _tr_list *delayed_tr;
         u32 ioc_state;
+        u8 tr_method = 0;
 
         if (ioc->pci_error_recovery) {
                 dewtprintk(ioc, pr_info(MPT3SAS_FMT
@@ -3577,6 +3637,11 @@ _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
                         sas_address = pcie_device->wwid;
                 }
                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
+                if (pcie_device && (!ioc->tm_custom_handling))
+                        tr_method =
+                            MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
+                else
+                        tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
         }
         if (sas_target_priv_data) {
                 dewtprintk(ioc, pr_info(MPT3SAS_FMT
@@ -3640,6 +3705,7 @@ _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
         mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
         mpi_request->DevHandle = cpu_to_le16(handle);
         mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
+        mpi_request->MsgFlags = tr_method;
         set_bit(handle, ioc->device_remove_in_progress);
         mpt3sas_base_put_smid_hi_priority(ioc, smid, 0);
         mpt3sas_trigger_master(ioc, MASTER_TRIGGER_DEVICE_REMOVAL);
@@ -3680,11 +3746,7 @@ _scsih_tm_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
         u32 ioc_state;
         struct _sc_list *delayed_sc;
 
-        if (ioc->remove_host) {
-                dewtprintk(ioc, pr_info(MPT3SAS_FMT
-                        "%s: host has been removed\n", __func__, ioc->name));
-                return 1;
-        } else if (ioc->pci_error_recovery) {
+        if (ioc->pci_error_recovery) {
                 dewtprintk(ioc, pr_info(MPT3SAS_FMT
                         "%s: host in pci error recovery\n", __func__,
                         ioc->name));
@@ -3725,7 +3787,7 @@ _scsih_tm_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
                 if (!delayed_sc)
                         return _scsih_check_for_pending_tm(ioc, smid);
                 INIT_LIST_HEAD(&delayed_sc->list);
-                delayed_sc->handle = mpi_request_tm->DevHandle;
+                delayed_sc->handle = le16_to_cpu(mpi_request_tm->DevHandle);
                 list_add_tail(&delayed_sc->list, &ioc->delayed_sc_list);
                 dewtprintk(ioc, pr_info(MPT3SAS_FMT
                     "DELAYED:sc:handle(0x%04x), (open)\n",
@@ -3806,8 +3868,7 @@ _scsih_tm_tr_volume_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
         u16 smid;
         struct _tr_list *delayed_tr;
 
-        if (ioc->shost_recovery || ioc->remove_host ||
-            ioc->pci_error_recovery) {
+        if (ioc->pci_error_recovery) {
                 dewtprintk(ioc, pr_info(MPT3SAS_FMT
                         "%s: host reset in progress!\n",
                         __func__, ioc->name));
@@ -3860,8 +3921,7 @@ _scsih_tm_volume_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
         Mpi2SCSITaskManagementReply_t *mpi_reply =
             mpt3sas_base_get_reply_virt_addr(ioc, reply);
 
-        if (ioc->shost_recovery || ioc->remove_host ||
-            ioc->pci_error_recovery) {
+        if (ioc->shost_recovery || ioc->pci_error_recovery) {
                 dewtprintk(ioc, pr_info(MPT3SAS_FMT
                         "%s: host reset in progress!\n",
                         __func__, ioc->name));
@@ -3903,8 +3963,8 @@ _scsih_tm_volume_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
  * Context - processed in interrupt context.
  */
 static void
-_scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 event,
-                                u32 event_context)
+_scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, U16 event,
+                                U32 event_context)
 {
         Mpi2EventAckRequest_t *ack_request;
         int i = smid - ioc->internal_smid;
@@ -3979,13 +4039,13 @@ _scsih_issue_delayed_sas_io_unit_ctrl(struct MPT3SAS_ADAPTER *ioc,
 
         dewtprintk(ioc, pr_info(MPT3SAS_FMT
             "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
-            ioc->name, le16_to_cpu(handle), smid,
+            ioc->name, handle, smid,
             ioc->tm_sas_control_cb_idx));
         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
         memset(mpi_request, 0, sizeof(Mpi2SasIoUnitControlRequest_t));
         mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
         mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
-        mpi_request->DevHandle = handle;
+        mpi_request->DevHandle = cpu_to_le16(handle);
         mpt3sas_base_put_smid_default(ioc, smid);
 }
 
@@ -5618,10 +5678,10 @@ static int
 _scsih_expander_add(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 {
         struct _sas_node *sas_expander;
+        struct _enclosure_node *enclosure_dev;
         Mpi2ConfigReply_t mpi_reply;
         Mpi2ExpanderPage0_t expander_pg0;
         Mpi2ExpanderPage1_t expander_pg1;
-        Mpi2SasEnclosurePage0_t enclosure_pg0;
         u32 ioc_status;
         u16 parent_handle;
         u64 sas_address, sas_address_parent = 0;
@@ -5743,11 +5803,12 @@ _scsih_expander_add(struct MPT3SAS_ADAPTER *ioc, u16 handle)
         }
 
         if (sas_expander->enclosure_handle) {
-                if (!(mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
-                    &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
-                   sas_expander->enclosure_handle)))
+                enclosure_dev =
+                        mpt3sas_scsih_enclosure_find_by_handle(ioc,
+                                                sas_expander->enclosure_handle);
+                if (enclosure_dev)
                         sas_expander->enclosure_logical_id =
-                            le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
+                            le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
         }
 
         _scsih_expander_node_add(ioc, sas_expander);
@@ -5891,52 +5952,6 @@ _scsih_check_access_status(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
 }
 
 /**
- * _scsih_get_enclosure_logicalid_chassis_slot - get device's
- *                      EnclosureLogicalID and ChassisSlot information.
- * @ioc: per adapter object
- * @sas_device_pg0: SAS device page0
- * @sas_device: per sas device object
- *
- * Returns nothing.
- */
-static void
-_scsih_get_enclosure_logicalid_chassis_slot(struct MPT3SAS_ADAPTER *ioc,
-        Mpi2SasDevicePage0_t *sas_device_pg0, struct _sas_device *sas_device)
-{
-        Mpi2ConfigReply_t mpi_reply;
-        Mpi2SasEnclosurePage0_t enclosure_pg0;
-
-        if (!sas_device_pg0 || !sas_device)
-                return;
-
-        sas_device->enclosure_handle =
-            le16_to_cpu(sas_device_pg0->EnclosureHandle);
-        sas_device->is_chassis_slot_valid = 0;
-
-        if (!le16_to_cpu(sas_device_pg0->EnclosureHandle))
-                return;
-
-        if (mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
-            &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
-            le16_to_cpu(sas_device_pg0->EnclosureHandle))) {
-                pr_err(MPT3SAS_FMT
-                    "Enclosure Pg0 read failed for handle(0x%04x)\n",
-                    ioc->name, le16_to_cpu(sas_device_pg0->EnclosureHandle));
-                return;
-        }
-
-        sas_device->enclosure_logical_id =
-            le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
-
-        if (le16_to_cpu(enclosure_pg0.Flags) &
-            MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
-                sas_device->is_chassis_slot_valid = 1;
-                sas_device->chassis_slot = enclosure_pg0.ChassisSlot;
-        }
-}
-
-
-/**
  * _scsih_check_device - checking device responsiveness
  * @ioc: per adapter object
  * @parent_sas_address: sas address of parent expander or sas host
@@ -5953,6 +5968,7 @@ _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
         Mpi2ConfigReply_t mpi_reply;
         Mpi2SasDevicePage0_t sas_device_pg0;
         struct _sas_device *sas_device;
+        struct _enclosure_node *enclosure_dev = NULL;
         u32 ioc_status;
         unsigned long flags;
         u64 sas_address;
@@ -6007,8 +6023,21 @@ _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
                         sas_device->connector_name[0] = '\0';
                 }
 
-                _scsih_get_enclosure_logicalid_chassis_slot(ioc,
-                    &sas_device_pg0, sas_device);
+                sas_device->enclosure_handle =
+                                le16_to_cpu(sas_device_pg0.EnclosureHandle);
+                sas_device->is_chassis_slot_valid = 0;
+                enclosure_dev = mpt3sas_scsih_enclosure_find_by_handle(ioc,
+                                                sas_device->enclosure_handle);
+                if (enclosure_dev) {
+                        sas_device->enclosure_logical_id =
+                            le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
+                        if (le16_to_cpu(enclosure_dev->pg0.Flags) &
+                            MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
+                                sas_device->is_chassis_slot_valid = 1;
+                                sas_device->chassis_slot =
+                                        enclosure_dev->pg0.ChassisSlot;
+                        }
+                }
         }
 
         /* check if device is present */
@@ -6055,12 +6084,11 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
 {
         Mpi2ConfigReply_t mpi_reply;
         Mpi2SasDevicePage0_t sas_device_pg0;
-        Mpi2SasEnclosurePage0_t enclosure_pg0;
         struct _sas_device *sas_device;
+        struct _enclosure_node *enclosure_dev = NULL;
         u32 ioc_status;
         u64 sas_address;
         u32 device_info;
-        int encl_pg0_rc = -1;
 
         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
@@ -6106,12 +6134,12 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
         }
 
         if (sas_device_pg0.EnclosureHandle) {
-                encl_pg0_rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
-                    &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
-                    sas_device_pg0.EnclosureHandle);
-                if (encl_pg0_rc)
-                        pr_info(MPT3SAS_FMT
-                            "Enclosure Pg0 read failed for handle(0x%04x)\n",
+                enclosure_dev =
+                        mpt3sas_scsih_enclosure_find_by_handle(ioc,
+                            le16_to_cpu(sas_device_pg0.EnclosureHandle));
+                if (enclosure_dev == NULL)
+                        pr_info(MPT3SAS_FMT "Enclosure handle(0x%04x)"
+                            "doesn't match with enclosure device!\n",
                             ioc->name, sas_device_pg0.EnclosureHandle);
         }
 
@@ -6152,18 +6180,16 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
                 sas_device->enclosure_level = 0;
                 sas_device->connector_name[0] = '\0';
         }
-
-        /* get enclosure_logical_id & chassis_slot */
+        /* get enclosure_logical_id & chassis_slot*/
         sas_device->is_chassis_slot_valid = 0;
-        if (encl_pg0_rc == 0) {
+        if (enclosure_dev) {
                 sas_device->enclosure_logical_id =
-                    le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
-
-                if (le16_to_cpu(enclosure_pg0.Flags) &
+                    le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
+                if (le16_to_cpu(enclosure_dev->pg0.Flags) &
                     MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
                         sas_device->is_chassis_slot_valid = 1;
                         sas_device->chassis_slot =
-                            enclosure_pg0.ChassisSlot;
+                                        enclosure_dev->pg0.ChassisSlot;
                 }
         }
 
@@ -6845,8 +6871,8 @@ _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
         Mpi26PCIeDevicePage0_t pcie_device_pg0;
         Mpi26PCIeDevicePage2_t pcie_device_pg2;
         Mpi2ConfigReply_t mpi_reply;
-        Mpi2SasEnclosurePage0_t enclosure_pg0;
         struct _pcie_device *pcie_device;
+        struct _enclosure_node *enclosure_dev;
         u32 pcie_device_type;
         u32 ioc_status;
         u64 wwid;
@@ -6917,7 +6943,7 @@ _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
         if (pcie_device->enclosure_handle != 0)
                 pcie_device->slot = le16_to_cpu(pcie_device_pg0.Slot);
 
-        if (le16_to_cpu(pcie_device_pg0.Flags) &
+        if (le32_to_cpu(pcie_device_pg0.Flags) &
             MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) {
                 pcie_device->enclosure_level = pcie_device_pg0.EnclosureLevel;
                 memcpy(&pcie_device->connector_name[0],
@@ -6928,13 +6954,14 @@ _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
         }
 
         /* get enclosure_logical_id */
-        if (pcie_device->enclosure_handle &&
-                !(mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
-                        &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
-                        pcie_device->enclosure_handle)))
-                pcie_device->enclosure_logical_id =
-                        le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
-
+        if (pcie_device->enclosure_handle) {
+                enclosure_dev =
+                        mpt3sas_scsih_enclosure_find_by_handle(ioc,
+                                                pcie_device->enclosure_handle);
+                if (enclosure_dev)
+                        pcie_device->enclosure_logical_id =
+                            le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
+        }
         /* TODO -- Add device name once FW supports it */
         if (mpt3sas_config_get_pcie_device_pg2(ioc, &mpi_reply,
                 &pcie_device_pg2, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle)) {
@@ -6953,6 +6980,11 @@ _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
         }
         pcie_device->nvme_mdts =
                 le32_to_cpu(pcie_device_pg2.MaximumDataTransferSize);
+        if (pcie_device_pg2.ControllerResetTO)
+                pcie_device->reset_timeout =
+                        pcie_device_pg2.ControllerResetTO;
+        else
+                pcie_device->reset_timeout = 30;
 
         if (ioc->wait_for_discovery_to_complete)
                 _scsih_pcie_device_init_add(ioc, pcie_device);
@@ -7205,6 +7237,9 @@ _scsih_pcie_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
         case MPI26_EVENT_PCIDEV_STAT_RC_ASYNC_NOTIFICATION:
                 reason_str = "internal async notification";
                 break;
+        case MPI26_EVENT_PCIDEV_STAT_RC_PCIE_HOT_RESET_FAILED:
+                reason_str = "pcie hot reset failed";
+                break;
         default:
                 reason_str = "unknown reason";
                 break;
@@ -7320,10 +7355,60 @@ static void
 _scsih_sas_enclosure_dev_status_change_event(struct MPT3SAS_ADAPTER *ioc,
         struct fw_event_work *fw_event)
 {
+        Mpi2ConfigReply_t mpi_reply;
+        struct _enclosure_node *enclosure_dev = NULL;
+        Mpi2EventDataSasEnclDevStatusChange_t *event_data =
+                (Mpi2EventDataSasEnclDevStatusChange_t *)fw_event->event_data;
+        int rc;
+        u16 enclosure_handle = le16_to_cpu(event_data->EnclosureHandle);
+
         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
                 _scsih_sas_enclosure_dev_status_change_event_debug(ioc,
                      (Mpi2EventDataSasEnclDevStatusChange_t *)
                      fw_event->event_data);
+        if (ioc->shost_recovery)
+                return;
+
+        if (enclosure_handle)
+                enclosure_dev =
+                        mpt3sas_scsih_enclosure_find_by_handle(ioc,
+                                                enclosure_handle);
+        switch (event_data->ReasonCode) {
+        case MPI2_EVENT_SAS_ENCL_RC_ADDED:
+                if (!enclosure_dev) {
+                        enclosure_dev =
+                                kzalloc(sizeof(struct _enclosure_node),
+                                        GFP_KERNEL);
+                        if (!enclosure_dev) {
+                                pr_info(MPT3SAS_FMT
+                                        "failure at %s:%d/%s()!\n", ioc->name,
+                                        __FILE__, __LINE__, __func__);
+                                return;
+                        }
+                        rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
+                                &enclosure_dev->pg0,
+                                MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
+                                enclosure_handle);
+
+                        if (rc || (le16_to_cpu(mpi_reply.IOCStatus) &
+                                                MPI2_IOCSTATUS_MASK)) {
+                                kfree(enclosure_dev);
+                                return;
+                        }
+
+                        list_add_tail(&enclosure_dev->list,
+                                                        &ioc->enclosure_list);
+                }
+                break;
+        case MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING:
+                if (enclosure_dev) {
+                        list_del(&enclosure_dev->list);
+                        kfree(enclosure_dev);
+                }
+                break;
+        default:
+                break;
+        }
 }
 
 /**
@@ -7409,7 +7494,7 @@ _scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,
                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
                 r = mpt3sas_scsih_issue_tm(ioc, handle, lun,
                         MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK, st->smid,
-                        st->msix_io, 30);
+                        st->msix_io, 30, 0);
                 if (r == FAILED) {
                         sdev_printk(KERN_WARNING, sdev,
                             "mpt3sas_scsih_issue_tm: FAILED when sending "
@@ -7450,7 +7535,7 @@ _scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,
 
                 r = mpt3sas_scsih_issue_tm(ioc, handle, sdev->lun,
                         MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK, st->smid,
-                        st->msix_io, 30);
+                        st->msix_io, 30, 0);
                 if (r == FAILED || st->cb_idx != 0xFF) {
                         sdev_printk(KERN_WARNING, sdev,
                             "mpt3sas_scsih_issue_tm: ABORT_TASK: FAILED : "
@@ -7527,6 +7612,44 @@ _scsih_sas_discovery_event(struct MPT3SAS_ADAPTER *ioc,
 }
 
 /**
+ * _scsih_sas_device_discovery_error_event - display SAS device discovery error
+ *                                              events
+ * @ioc: per adapter object
+ * @fw_event: The fw_event_work object
+ * Context: user.
+ *
+ * Return nothing.
+ */
+static void
+_scsih_sas_device_discovery_error_event(struct MPT3SAS_ADAPTER *ioc,
+        struct fw_event_work *fw_event)
+{
+        Mpi25EventDataSasDeviceDiscoveryError_t *event_data =
+                (Mpi25EventDataSasDeviceDiscoveryError_t *)fw_event->event_data;
+
+        switch (event_data->ReasonCode) {
+        case MPI25_EVENT_SAS_DISC_ERR_SMP_FAILED:
+                pr_warn(MPT3SAS_FMT "SMP command sent to the expander"
+                        "(handle:0x%04x, sas_address:0x%016llx,"
+                        "physical_port:0x%02x) has failed",
+                        ioc->name, le16_to_cpu(event_data->DevHandle),
+                        (unsigned long long)le64_to_cpu(event_data->SASAddress),
+                        event_data->PhysicalPort);
+                break;
+        case MPI25_EVENT_SAS_DISC_ERR_SMP_TIMEOUT:
+                pr_warn(MPT3SAS_FMT "SMP command sent to the expander"
+                        "(handle:0x%04x, sas_address:0x%016llx,"
+                        "physical_port:0x%02x) has timed out",
+                        ioc->name, le16_to_cpu(event_data->DevHandle),
+                        (unsigned long long)le64_to_cpu(event_data->SASAddress),
+                        event_data->PhysicalPort);
+                break;
+        default:
+                break;
+        }
+}
+
+/**
  * _scsih_pcie_enumeration_event - handle enumeration events
  * @ioc: per adapter object
  * @fw_event: The fw_event_work object
@@ -8360,12 +8483,23 @@ Mpi2SasDevicePage0_t *sas_device_pg0)
         struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
         struct scsi_target *starget;
         struct _sas_device *sas_device = NULL;
+        struct _enclosure_node *enclosure_dev = NULL;
         unsigned long flags;
 
+        if (sas_device_pg0->EnclosureHandle) {
+                enclosure_dev =
+                        mpt3sas_scsih_enclosure_find_by_handle(ioc,
+                                le16_to_cpu(sas_device_pg0->EnclosureHandle));
+                if (enclosure_dev == NULL)
+                        pr_info(MPT3SAS_FMT "Enclosure handle(0x%04x)"
+                            "doesn't match with enclosure device!\n",
+                            ioc->name, sas_device_pg0->EnclosureHandle);
+        }
         spin_lock_irqsave(&ioc->sas_device_lock, flags);
         list_for_each_entry(sas_device, &ioc->sas_device_list, list) {
-                if ((sas_device->sas_address == sas_device_pg0->SASAddress) &&
-                        (sas_device->slot == sas_device_pg0->Slot)) {
+                if ((sas_device->sas_address == le64_to_cpu(
+                    sas_device_pg0->SASAddress)) && (sas_device->slot ==
+                    le16_to_cpu(sas_device_pg0->Slot))) {
                         sas_device->responding = 1;
                         starget = sas_device->starget;
                         if (starget && starget->hostdata) {
@@ -8377,7 +8511,7 @@ Mpi2SasDevicePage0_t *sas_device_pg0)
                         if (starget) {
                                 starget_printk(KERN_INFO, starget,
                                     "handle(0x%04x), sas_addr(0x%016llx)\n",
-                                    sas_device_pg0->DevHandle,
+                                    le16_to_cpu(sas_device_pg0->DevHandle),
                                     (unsigned long long)
                                     sas_device->sas_address);
 
@@ -8389,7 +8523,7 @@ Mpi2SasDevicePage0_t *sas_device_pg0)
                                          sas_device->enclosure_logical_id,
                                          sas_device->slot);
                         }
-                        if (sas_device_pg0->Flags &
+                        if (le16_to_cpu(sas_device_pg0->Flags) &
                               MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
                                 sas_device->enclosure_level =
                                    sas_device_pg0->EnclosureLevel;
@@ -8400,17 +8534,30 @@ Mpi2SasDevicePage0_t *sas_device_pg0)
                                 sas_device->connector_name[0] = '\0';
                         }
 
-                        _scsih_get_enclosure_logicalid_chassis_slot(ioc,
-                            sas_device_pg0, sas_device);
+                        sas_device->enclosure_handle =
+                                le16_to_cpu(sas_device_pg0->EnclosureHandle);
+                        sas_device->is_chassis_slot_valid = 0;
+                        if (enclosure_dev) {
+                                sas_device->enclosure_logical_id = le64_to_cpu(
+                                        enclosure_dev->pg0.EnclosureLogicalID);
+                                if (le16_to_cpu(enclosure_dev->pg0.Flags) &
+                                    MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
+                                        sas_device->is_chassis_slot_valid = 1;
+                                        sas_device->chassis_slot =
+                                                enclosure_dev->pg0.ChassisSlot;
+                                }
+                        }
 
-                        if (sas_device->handle == sas_device_pg0->DevHandle)
+                        if (sas_device->handle == le16_to_cpu(
+                            sas_device_pg0->DevHandle))
                                 goto out;
                         pr_info("\thandle changed from(0x%04x)!!!\n",
                             sas_device->handle);
-                        sas_device->handle = sas_device_pg0->DevHandle;
+                        sas_device->handle = le16_to_cpu(
+                            sas_device_pg0->DevHandle);
                         if (sas_target_priv_data)
                                 sas_target_priv_data->handle =
-                                        sas_device_pg0->DevHandle;
+                                    le16_to_cpu(sas_device_pg0->DevHandle);
                         goto out;
                 }
         }
@@ -8419,6 +8566,52 @@ Mpi2SasDevicePage0_t *sas_device_pg0)
 }
 
 /**
+ * _scsih_create_enclosure_list_after_reset - Free Existing list,
+ *      And create enclosure list by scanning all Enclosure Page(0)s
+ * @ioc: per adapter object
+ *
+ * Return nothing.
+ */
+static void
+_scsih_create_enclosure_list_after_reset(struct MPT3SAS_ADAPTER *ioc)
+{
+        struct _enclosure_node *enclosure_dev;
+        Mpi2ConfigReply_t mpi_reply;
+        u16 enclosure_handle;
+        int rc;
+
+        /* Free existing enclosure list */
+        mpt3sas_free_enclosure_list(ioc);
+
+        /* Re constructing enclosure list after reset*/
+        enclosure_handle = 0xFFFF;
+        do {
+                enclosure_dev =
+                        kzalloc(sizeof(struct _enclosure_node), GFP_KERNEL);
+                if (!enclosure_dev) {
+                        pr_err(MPT3SAS_FMT
+                                "failure at %s:%d/%s()!\n", ioc->name,
+                                __FILE__, __LINE__, __func__);
+                        return;
+                }
+                rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
+                                &enclosure_dev->pg0,
+                                MPI2_SAS_ENCLOS_PGAD_FORM_GET_NEXT_HANDLE,
+                                enclosure_handle);
+
+                if (rc || (le16_to_cpu(mpi_reply.IOCStatus) &
+                                                MPI2_IOCSTATUS_MASK)) {
+                        kfree(enclosure_dev);
+                        return;
+                }
+                list_add_tail(&enclosure_dev->list,
+                                                &ioc->enclosure_list);
+                enclosure_handle =
+                        le16_to_cpu(enclosure_dev->pg0.EnclosureHandle);
+        } while (1);
+}
+
+/**
  * _scsih_search_responding_sas_devices -
  * @ioc: per adapter object
  *
@@ -8449,15 +8642,10 @@ _scsih_search_responding_sas_devices(struct MPT3SAS_ADAPTER *ioc)
                     MPI2_IOCSTATUS_MASK;
                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
                         break;
-                handle = sas_device_pg0.DevHandle =
-                                le16_to_cpu(sas_device_pg0.DevHandle);
+                handle = le16_to_cpu(sas_device_pg0.DevHandle);
                 device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
                 if (!(_scsih_is_end_device(device_info)))
                         continue;
-                sas_device_pg0.SASAddress =
-                                le64_to_cpu(sas_device_pg0.SASAddress);
-                sas_device_pg0.Slot = le16_to_cpu(sas_device_pg0.Slot);
-                sas_device_pg0.Flags = le16_to_cpu(sas_device_pg0.Flags);
                 _scsih_mark_responding_sas_device(ioc, &sas_device_pg0);
         }
 
@@ -8487,8 +8675,9 @@ _scsih_mark_responding_pcie_device(struct MPT3SAS_ADAPTER *ioc,
 
         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
         list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {
-                if ((pcie_device->wwid == pcie_device_pg0->WWID) &&
-                    (pcie_device->slot == pcie_device_pg0->Slot)) {
+                if ((pcie_device->wwid == le64_to_cpu(pcie_device_pg0->WWID))
+                    && (pcie_device->slot == le16_to_cpu(
+                    pcie_device_pg0->Slot))) {
                         pcie_device->responding = 1;
                         starget = pcie_device->starget;
                         if (starget && starget->hostdata) {
@@ -8523,14 +8712,16 @@ _scsih_mark_responding_pcie_device(struct MPT3SAS_ADAPTER *ioc,
                                 pcie_device->connector_name[0] = '\0';
                         }
 
-                        if (pcie_device->handle == pcie_device_pg0->DevHandle)
+                        if (pcie_device->handle == le16_to_cpu(
+                            pcie_device_pg0->DevHandle))
                                 goto out;
                         pr_info("\thandle changed from(0x%04x)!!!\n",
                             pcie_device->handle);
-                        pcie_device->handle = pcie_device_pg0->DevHandle;
+                        pcie_device->handle = le16_to_cpu(
+                            pcie_device_pg0->DevHandle);
                         if (sas_target_priv_data)
                                 sas_target_priv_data->handle =
-                                    pcie_device_pg0->DevHandle;
+                                    le16_to_cpu(pcie_device_pg0->DevHandle);
                         goto out;
                 }
         }
@@ -8579,10 +8770,6 @@ _scsih_search_responding_pcie_devices(struct MPT3SAS_ADAPTER *ioc)
                 device_info = le32_to_cpu(pcie_device_pg0.DeviceInfo);
                 if (!(_scsih_is_nvme_device(device_info)))
                         continue;
-                pcie_device_pg0.WWID = le64_to_cpu(pcie_device_pg0.WWID),
-                pcie_device_pg0.Slot = le16_to_cpu(pcie_device_pg0.Slot);
-                pcie_device_pg0.Flags = le32_to_cpu(pcie_device_pg0.Flags);
-                pcie_device_pg0.DevHandle = handle;
                 _scsih_mark_responding_pcie_device(ioc, &pcie_device_pg0);
         }
 out:
@@ -8736,22 +8923,16 @@ _scsih_mark_responding_expander(struct MPT3SAS_ADAPTER *ioc,
 {
         struct _sas_node *sas_expander = NULL;
         unsigned long flags;
-        int i, encl_pg0_rc = -1;
-        Mpi2ConfigReply_t mpi_reply;
-        Mpi2SasEnclosurePage0_t enclosure_pg0;
+        int i;
+        struct _enclosure_node *enclosure_dev = NULL;
         u16 handle = le16_to_cpu(expander_pg0->DevHandle);
+        u16 enclosure_handle = le16_to_cpu(expander_pg0->EnclosureHandle);
         u64 sas_address = le64_to_cpu(expander_pg0->SASAddress);
 
-        if (le16_to_cpu(expander_pg0->EnclosureHandle)) {
-                encl_pg0_rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
-                    &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
-                    le16_to_cpu(expander_pg0->EnclosureHandle));
-                if (encl_pg0_rc)
-                        pr_info(MPT3SAS_FMT
-                            "Enclosure Pg0 read failed for handle(0x%04x)\n",
-                            ioc->name,
-                            le16_to_cpu(expander_pg0->EnclosureHandle));
-        }
+        if (enclosure_handle)
+                enclosure_dev =
+                        mpt3sas_scsih_enclosure_find_by_handle(ioc,
+                                                        enclosure_handle);
 
         spin_lock_irqsave(&ioc->sas_node_lock, flags);
         list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {
@@ -8759,12 +8940,12 @@ _scsih_mark_responding_expander(struct MPT3SAS_ADAPTER *ioc,
                         continue;
                 sas_expander->responding = 1;
 
-                if (!encl_pg0_rc)
+                if (enclosure_dev) {
                         sas_expander->enclosure_logical_id =
-                            le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
-
-                sas_expander->enclosure_handle =
-                    le16_to_cpu(expander_pg0->EnclosureHandle);
+                            le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
+                        sas_expander->enclosure_handle =
+                            le16_to_cpu(expander_pg0->EnclosureHandle);
+                }
 
                 if (sas_expander->handle == handle)
                         goto out;
@@ -9286,6 +9467,7 @@ mpt3sas_scsih_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase)
                 if ((!ioc->is_driver_loading) && !(disable_discovery > 0 &&
                     !ioc->sas_hba.num_phys)) {
                         _scsih_prep_device_scan(ioc);
+                        _scsih_create_enclosure_list_after_reset(ioc);
                         _scsih_search_responding_sas_devices(ioc);
                         _scsih_search_responding_pcie_devices(ioc);
                         _scsih_search_responding_raid_devices(ioc);
@@ -9356,6 +9538,9 @@ _mpt3sas_fw_work(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)
         case MPI2_EVENT_SAS_DISCOVERY:
                 _scsih_sas_discovery_event(ioc, fw_event);
                 break;
+        case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
+                _scsih_sas_device_discovery_error_event(ioc, fw_event);
+                break;
         case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
                 _scsih_sas_broadcast_primitive_event(ioc, fw_event);
                 break;
@@ -9433,8 +9618,8 @@ mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,
         u16 sz;
         Mpi26EventDataActiveCableExcept_t *ActiveCableEventData;
 
-        /* events turned off due to host reset or driver unloading */
-        if (ioc->remove_host || ioc->pci_error_recovery)
+        /* events turned off due to host reset */
+        if (ioc->pci_error_recovery)
                 return 1;
 
         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
@@ -9540,6 +9725,7 @@ mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,
         case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
         case MPI2_EVENT_IR_OPERATION_STATUS:
         case MPI2_EVENT_SAS_DISCOVERY:
+        case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
         case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
         case MPI2_EVENT_IR_PHYSICAL_DISK:
         case MPI2_EVENT_PCIE_ENUMERATION:
@@ -10513,6 +10699,7 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
         INIT_LIST_HEAD(&ioc->sas_device_list);
         INIT_LIST_HEAD(&ioc->sas_device_init_list);
         INIT_LIST_HEAD(&ioc->sas_expander_list);
+        INIT_LIST_HEAD(&ioc->enclosure_list);
         INIT_LIST_HEAD(&ioc->pcie_device_list);
         INIT_LIST_HEAD(&ioc->pcie_device_init_list);
         INIT_LIST_HEAD(&ioc->fw_event_list);
@@ -11100,10 +11287,10 @@ _mpt3sas_exit(void)
         pr_info("mpt3sas version %s unloading\n",
                                 MPT3SAS_DRIVER_VERSION);
 
-        pci_unregister_driver(&mpt3sas_driver);
-
         mpt3sas_ctl_exit(hbas_to_enumerate);
 
+        pci_unregister_driver(&mpt3sas_driver);
+
         scsih_exit();
 }
 
diff --git a/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c b/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c
index 6bfcee4757e0..45aa94915cbf 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c
@@ -177,7 +177,8 @@ mpt3sas_init_warpdrive_properties(struct MPT3SAS_ADAPTER *ioc,
                 if (mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,
                     &pd_pg0, MPI2_PHYSDISK_PGAD_FORM_PHYSDISKNUM,
                     vol_pg0->PhysDisk[count].PhysDiskNum) ||
-                    pd_pg0.DevHandle == MPT3SAS_INVALID_DEVICE_HANDLE) {
+                    le16_to_cpu(pd_pg0.DevHandle) ==
+                    MPT3SAS_INVALID_DEVICE_HANDLE) {
                         pr_info(MPT3SAS_FMT "WarpDrive : Direct IO is "
                             "disabled for the drive with handle(0x%04x) member"
                             "handle retrieval failed for member number=%d\n",
diff --git a/drivers/scsi/mvumi.c b/drivers/scsi/mvumi.c
index afd27165cd93..b3cd9a6b1d30 100644
--- a/drivers/scsi/mvumi.c
+++ b/drivers/scsi/mvumi.c
@@ -2693,22 +2693,4 @@ static struct pci_driver mvumi_pci_driver = {
 #endif
 };
 
-/**
- * mvumi_init - Driver load entry point
- */
-static int __init mvumi_init(void)
-{
-        return pci_register_driver(&mvumi_pci_driver);
-}
-
-/**
- * mvumi_exit - Driver unload entry point
- */
-static void __exit mvumi_exit(void)
-{
-
-        pci_unregister_driver(&mvumi_pci_driver);
-}
-
-module_init(mvumi_init);
-module_exit(mvumi_exit);
+module_pci_driver(mvumi_pci_driver);
diff --git a/drivers/scsi/osd/osd_initiator.c b/drivers/scsi/osd/osd_initiator.c
index 5a33e1ad9881..67b14576fff2 100644
--- a/drivers/scsi/osd/osd_initiator.c
+++ b/drivers/scsi/osd/osd_initiator.c
@@ -1840,14 +1840,14 @@ int osd_req_decode_sense_full(struct osd_request *or,
                 case osd_sense_response_integrity_check:
                 {
                         struct osd_sense_response_integrity_check_descriptor
-                                *osricd = cur_descriptor;
-                        const unsigned len =
-                                          sizeof(osricd->integrity_check_value);
-                        char key_dump[len*4 + 2]; /* 2nibbles+space+ASCII */
-
-                        hex_dump_to_buffer(osricd->integrity_check_value, len,
-                                       32, 1, key_dump, sizeof(key_dump), true);
-                        OSD_SENSE_PRINT2("response_integrity [%s]\n", key_dump);
+                                *d = cur_descriptor;
+                        /* 2nibbles+space+ASCII */
+                        char dump[sizeof(d->integrity_check_value) * 4 + 2];
+
+                        hex_dump_to_buffer(d->integrity_check_value,
+                                        sizeof(d->integrity_check_value),
+                                        32, 1, dump, sizeof(dump), true);
+                        OSD_SENSE_PRINT2("response_integrity [%s]\n", dump);
                 }
                 case osd_sense_attribute_identification:
                 {
diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c
index db88a8e7ee0e..4dd6cad330e8 100644
--- a/drivers/scsi/pm8001/pm8001_hwi.c
+++ b/drivers/scsi/pm8001/pm8001_hwi.c
@@ -3607,7 +3607,7 @@ int pm8001_mpi_reg_resp(struct pm8001_hba_info *pm8001_ha, void *piomb)
                 break;
         default:
                 PM8001_MSG_DBG(pm8001_ha,
-                 pm8001_printk("DEVREG_FAILURE_DEVICE_TYPE_NOT_UNSORPORTED\n"));
+                        pm8001_printk("DEVREG_FAILURE_DEVICE_TYPE_NOT_SUPPORTED\n"));
                 break;
         }
         complete(pm8001_dev->dcompletion);
diff --git a/drivers/scsi/qedf/drv_fcoe_fw_funcs.c b/drivers/scsi/qedf/drv_fcoe_fw_funcs.c
index a980ef756a67..5bd10b534c99 100644
--- a/drivers/scsi/qedf/drv_fcoe_fw_funcs.c
+++ b/drivers/scsi/qedf/drv_fcoe_fw_funcs.c
@@ -1,5 +1,5 @@
 /* QLogic FCoE Offload Driver
- * Copyright (c) 2016-2017 Cavium Inc.
+ * Copyright (c) 2016-2018 Cavium Inc.
  *
  * This software is available under the terms of the GNU General Public License
  * (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/drv_fcoe_fw_funcs.h b/drivers/scsi/qedf/drv_fcoe_fw_funcs.h
index b5c236efd465..42fde55ac735 100644
--- a/drivers/scsi/qedf/drv_fcoe_fw_funcs.h
+++ b/drivers/scsi/qedf/drv_fcoe_fw_funcs.h
@@ -1,5 +1,5 @@
 /* QLogic FCoE Offload Driver
- * Copyright (c) 2016-2017 Cavium Inc.
+ * Copyright (c) 2016-2018 Cavium Inc.
  *
  * This software is available under the terms of the GNU General Public License
  * (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/drv_scsi_fw_funcs.c b/drivers/scsi/qedf/drv_scsi_fw_funcs.c
index 5d5095e3d96d..29a55257224f 100644
--- a/drivers/scsi/qedf/drv_scsi_fw_funcs.c
+++ b/drivers/scsi/qedf/drv_scsi_fw_funcs.c
@@ -1,5 +1,5 @@
 /* QLogic FCoE Offload Driver
- * Copyright (c) 2016-2017 Cavium Inc.
+ * Copyright (c) 2016-2018 Cavium Inc.
  *
  * This software is available under the terms of the GNU General Public License
  * (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/drv_scsi_fw_funcs.h b/drivers/scsi/qedf/drv_scsi_fw_funcs.h
index 8fbe6e4d0b4f..bf102204fe56 100644
--- a/drivers/scsi/qedf/drv_scsi_fw_funcs.h
+++ b/drivers/scsi/qedf/drv_scsi_fw_funcs.h
@@ -1,5 +1,5 @@
 /* QLogic FCoE Offload Driver
- * Copyright (c) 2016-2017 Cavium Inc.
+ * Copyright (c) 2016-2018 Cavium Inc.
  *
  * This software is available under the terms of the GNU General Public License
  * (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/qedf.h b/drivers/scsi/qedf/qedf.h
index cabb6af60fb8..2c78d8fb9122 100644
--- a/drivers/scsi/qedf/qedf.h
+++ b/drivers/scsi/qedf/qedf.h
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -180,6 +180,7 @@ struct qedf_rport {
         spinlock_t rport_lock;
 #define QEDF_RPORT_SESSION_READY 1
 #define QEDF_RPORT_UPLOADING_CONNECTION 2
+#define QEDF_RPORT_IN_RESET 3
         unsigned long flags;
         unsigned long retry_delay_timestamp;
         struct fc_rport *rport;
@@ -300,6 +301,7 @@ struct qedf_ctx {
 #define QEDF_FALLBACK_VLAN      1002
 #define QEDF_DEFAULT_PRIO       3
         int vlan_id;
+        u8 prio;
         struct qed_dev *cdev;
         struct qed_dev_fcoe_info dev_info;
         struct qed_int_info int_info;
@@ -365,6 +367,7 @@ struct qedf_ctx {
 #define QEDF_IO_WORK_MIN                64
         mempool_t *io_mempool;
         struct workqueue_struct *dpc_wq;
+        struct delayed_work grcdump_work;
 
         u32 slow_sge_ios;
         u32 fast_sge_ios;
@@ -504,6 +507,7 @@ extern int qedf_send_flogi(struct qedf_ctx *qedf);
 extern void qedf_get_protocol_tlv_data(void *dev, void *data);
 extern void qedf_fp_io_handler(struct work_struct *work);
 extern void qedf_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data);
+extern void qedf_wq_grcdump(struct work_struct *work);
 
 #define FCOE_WORD_TO_BYTE  4
 #define QEDF_MAX_TASK_NUM       0xFFFF
diff --git a/drivers/scsi/qedf/qedf_attr.c b/drivers/scsi/qedf/qedf_attr.c
index fa6727685627..0487b7237104 100644
--- a/drivers/scsi/qedf/qedf_attr.c
+++ b/drivers/scsi/qedf/qedf_attr.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/qedf_dbg.c b/drivers/scsi/qedf/qedf_dbg.c
index bd1cef25a900..f2397ee9ba69 100644
--- a/drivers/scsi/qedf/qedf_dbg.c
+++ b/drivers/scsi/qedf/qedf_dbg.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -147,7 +147,7 @@ qedf_get_grc_dump(struct qed_dev *cdev, const struct qed_common_ops *common,
         if (!*buf)
                 return -EINVAL;
 
-        return common->dbg_grc(cdev, *buf, grcsize);
+        return common->dbg_all_data(cdev, *buf);
 }
 
 void
diff --git a/drivers/scsi/qedf/qedf_dbg.h b/drivers/scsi/qedf/qedf_dbg.h
index 77c27e888969..dd0109653aa3 100644
--- a/drivers/scsi/qedf/qedf_dbg.h
+++ b/drivers/scsi/qedf/qedf_dbg.h
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/qedf_debugfs.c b/drivers/scsi/qedf/qedf_debugfs.c
index 5789ce185923..c29c162a494f 100644
--- a/drivers/scsi/qedf/qedf_debugfs.c
+++ b/drivers/scsi/qedf/qedf_debugfs.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 QLogic Corporation
+ *  Copyright (c) 2016-2018 QLogic Corporation
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/qedf_els.c b/drivers/scsi/qedf/qedf_els.c
index aa22b11436ba..04f0c4d2e256 100644
--- a/drivers/scsi/qedf/qedf_els.c
+++ b/drivers/scsi/qedf/qedf_els.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -14,8 +14,8 @@ static int qedf_initiate_els(struct qedf_rport *fcport, unsigned int op,
         void (*cb_func)(struct qedf_els_cb_arg *cb_arg),
         struct qedf_els_cb_arg *cb_arg, uint32_t timer_msec)
 {
-        struct qedf_ctx *qedf = fcport->qedf;
-        struct fc_lport *lport = qedf->lport;
+        struct qedf_ctx *qedf;
+        struct fc_lport *lport;
         struct qedf_ioreq *els_req;
         struct qedf_mp_req *mp_req;
         struct fc_frame_header *fc_hdr;
@@ -29,6 +29,15 @@ static int qedf_initiate_els(struct qedf_rport *fcport, unsigned int op,
         unsigned long flags;
         u16 sqe_idx;
 
+        if (!fcport) {
+                QEDF_ERR(NULL, "fcport is NULL");
+                rc = -EINVAL;
+                goto els_err;
+        }
+
+        qedf = fcport->qedf;
+        lport = qedf->lport;
+
         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Sending ELS\n");
 
         rc = fc_remote_port_chkready(fcport->rport);
@@ -201,6 +210,14 @@ static void qedf_rrq_compl(struct qedf_els_cb_arg *cb_arg)
                 kref_put(&orig_io_req->refcount, qedf_release_cmd);
 
 out_free:
+        /*
+         * Release a reference to the rrq request if we timed out as the
+         * rrq completion handler is called directly from the timeout handler
+         * and not from els_compl where the reference would have normally been
+         * released.
+         */
+        if (rrq_req->event == QEDF_IOREQ_EV_ELS_TMO)
+                kref_put(&rrq_req->refcount, qedf_release_cmd);
         kfree(cb_arg);
 }
 
@@ -322,6 +339,17 @@ void qedf_restart_rport(struct qedf_rport *fcport)
         if (!fcport)
                 return;
 
+        if (test_bit(QEDF_RPORT_IN_RESET, &fcport->flags) ||
+            !test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) ||
+            test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
+                QEDF_ERR(&(fcport->qedf->dbg_ctx), "fcport %p already in reset or not offloaded.\n",
+                    fcport);
+                return;
+        }
+
+        /* Set that we are now in reset */
+        set_bit(QEDF_RPORT_IN_RESET, &fcport->flags);
+
         rdata = fcport->rdata;
         if (rdata) {
                 lport = fcport->qedf->lport;
@@ -334,6 +362,7 @@ void qedf_restart_rport(struct qedf_rport *fcport)
                 if (rdata)
                         fc_rport_login(rdata);
         }
+        clear_bit(QEDF_RPORT_IN_RESET, &fcport->flags);
 }
 
 static void qedf_l2_els_compl(struct qedf_els_cb_arg *cb_arg)
diff --git a/drivers/scsi/qedf/qedf_fip.c b/drivers/scsi/qedf/qedf_fip.c
index 16d1a21cdff9..3fd3af799b3d 100644
--- a/drivers/scsi/qedf/qedf_fip.c
+++ b/drivers/scsi/qedf/qedf_fip.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -137,7 +137,7 @@ void qedf_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb)
 
         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2, "FIP frame send: "
             "dest=%pM op=%x sub=%x vlan=%04x.", eth_hdr->h_dest, op, sub,
-            ntohs(vlan_tci));
+            vlan_tci);
         if (qedf_dump_frames)
                 print_hex_dump(KERN_WARNING, "fip ", DUMP_PREFIX_OFFSET, 16, 1,
                     skb->data, skb->len, false);
@@ -184,6 +184,7 @@ void qedf_fip_recv(struct qedf_ctx *qedf, struct sk_buff *skb)
                         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
                             "Dropping CVL since FCF has not been selected "
                             "yet.");
+                        kfree_skb(skb);
                         return;
                 }
 
diff --git a/drivers/scsi/qedf/qedf_hsi.h b/drivers/scsi/qedf/qedf_hsi.h
index 503c1ae3ccd0..f6f634e48d69 100644
--- a/drivers/scsi/qedf/qedf_hsi.h
+++ b/drivers/scsi/qedf/qedf_hsi.h
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/qedf_io.c b/drivers/scsi/qedf/qedf_io.c
index 3fe579d0f1a8..6bbc38b1b465 100644
--- a/drivers/scsi/qedf/qedf_io.c
+++ b/drivers/scsi/qedf/qedf_io.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -23,12 +23,31 @@ static void qedf_cmd_timeout(struct work_struct *work)
 
         struct qedf_ioreq *io_req =
             container_of(work, struct qedf_ioreq, timeout_work.work);
-        struct qedf_ctx *qedf = io_req->fcport->qedf;
-        struct qedf_rport *fcport = io_req->fcport;
+        struct qedf_ctx *qedf;
+        struct qedf_rport *fcport;
         u8 op = 0;
 
+        if (io_req == NULL) {
+                QEDF_INFO(NULL, QEDF_LOG_IO, "io_req is NULL.\n");
+                return;
+        }
+
+        fcport = io_req->fcport;
+        if (io_req->fcport == NULL) {
+                QEDF_INFO(NULL, QEDF_LOG_IO,  "fcport is NULL.\n");
+                return;
+        }
+
+        qedf = fcport->qedf;
+
         switch (io_req->cmd_type) {
         case QEDF_ABTS:
+                if (qedf == NULL) {
+                        QEDF_INFO(NULL, QEDF_LOG_IO, "qedf is NULL for xid=0x%x.\n",
+                            io_req->xid);
+                        return;
+                }
+
                 QEDF_ERR((&qedf->dbg_ctx), "ABTS timeout, xid=0x%x.\n",
                     io_req->xid);
                 /* Cleanup timed out ABTS */
@@ -931,6 +950,15 @@ qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
                 return 0;
         }
 
+        if (!qedf->pdev->msix_enabled) {
+                QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+                    "Completing sc_cmd=%p DID_NO_CONNECT as MSI-X is not enabled.\n",
+                    sc_cmd);
+                sc_cmd->result = DID_NO_CONNECT << 16;
+                sc_cmd->scsi_done(sc_cmd);
+                return 0;
+        }
+
         rval = fc_remote_port_chkready(rport);
         if (rval) {
                 sc_cmd->result = rval;
@@ -1420,6 +1448,12 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
         if (!fcport)
                 return;
 
+        /* Check that fcport is still offloaded */
+        if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+                QEDF_ERR(NULL, "fcport is no longer offloaded.\n");
+                return;
+        }
+
         qedf = fcport->qedf;
         cmd_mgr = qedf->cmd_mgr;
 
@@ -1436,8 +1470,8 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
                         rc = kref_get_unless_zero(&io_req->refcount);
                         if (!rc) {
                                 QEDF_ERR(&(qedf->dbg_ctx),
-                                    "Could not get kref for io_req=0x%p.\n",
-                                    io_req);
+                                    "Could not get kref for ELS io_req=0x%p xid=0x%x.\n",
+                                    io_req, io_req->xid);
                                 continue;
                         }
                         qedf_flush_els_req(qedf, io_req);
@@ -1448,6 +1482,31 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
                         goto free_cmd;
                 }
 
+                if (io_req->cmd_type == QEDF_ABTS) {
+                        rc = kref_get_unless_zero(&io_req->refcount);
+                        if (!rc) {
+                                QEDF_ERR(&(qedf->dbg_ctx),
+                                    "Could not get kref for abort io_req=0x%p xid=0x%x.\n",
+                                    io_req, io_req->xid);
+                                continue;
+                        }
+                        QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+                            "Flushing abort xid=0x%x.\n", io_req->xid);
+
+                        clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
+
+                        if (io_req->sc_cmd) {
+                                if (io_req->return_scsi_cmd_on_abts)
+                                        qedf_scsi_done(qedf, io_req, DID_ERROR);
+                        }
+
+                        /* Notify eh_abort handler that ABTS is complete */
+                        complete(&io_req->abts_done);
+                        kref_put(&io_req->refcount, qedf_release_cmd);
+
+                        goto free_cmd;
+                }
+
                 if (!io_req->sc_cmd)
                         continue;
                 if (lun > 0) {
@@ -1463,7 +1522,7 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
                 rc = kref_get_unless_zero(&io_req->refcount);
                 if (!rc) {
                         QEDF_ERR(&(qedf->dbg_ctx), "Could not get kref for "
-                            "io_req=0x%p\n", io_req);
+                            "io_req=0x%p xid=0x%x\n", io_req, io_req->xid);
                         continue;
                 }
                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
@@ -1525,6 +1584,21 @@ int qedf_initiate_abts(struct qedf_ioreq *io_req, bool return_scsi_cmd_on_abts)
                 goto abts_err;
         }
 
+        if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
+                QEDF_ERR(&qedf->dbg_ctx, "fcport is uploading.\n");
+                rc = 1;
+                goto out;
+        }
+
+        if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
+            test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
+            test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
+                QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
+                          "cleanup or abort processing or already "
+                          "completed.\n", io_req->xid);
+                rc = 1;
+                goto out;
+        }
 
         kref_get(&io_req->refcount);
 
@@ -1564,6 +1638,7 @@ abts_err:
          * task at the firmware.
          */
         qedf_initiate_cleanup(io_req, return_scsi_cmd_on_abts);
+out:
         return rc;
 }
 
diff --git a/drivers/scsi/qedf/qedf_main.c b/drivers/scsi/qedf/qedf_main.c
index d3f73d8d7738..90394cef0f41 100644
--- a/drivers/scsi/qedf/qedf_main.c
+++ b/drivers/scsi/qedf/qedf_main.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -44,20 +44,20 @@ module_param_named(debug, qedf_debug, uint, S_IRUGO);
 MODULE_PARM_DESC(debug, " Debug mask. Pass '1' to enable default debugging"
         " mask");
 
-static uint qedf_fipvlan_retries = 30;
+static uint qedf_fipvlan_retries = 60;
 module_param_named(fipvlan_retries, qedf_fipvlan_retries, int, S_IRUGO);
 MODULE_PARM_DESC(fipvlan_retries, " Number of FIP VLAN requests to attempt "
-        "before giving up (default 30)");
+        "before giving up (default 60)");
 
 static uint qedf_fallback_vlan = QEDF_FALLBACK_VLAN;
 module_param_named(fallback_vlan, qedf_fallback_vlan, int, S_IRUGO);
 MODULE_PARM_DESC(fallback_vlan, " VLAN ID to try if fip vlan request fails "
         "(default 1002).");
 
-static uint qedf_default_prio = QEDF_DEFAULT_PRIO;
+static int qedf_default_prio = -1;
 module_param_named(default_prio, qedf_default_prio, int, S_IRUGO);
-MODULE_PARM_DESC(default_prio, " Default 802.1q priority for FIP and FCoE"
-        " traffic (default 3).");
+MODULE_PARM_DESC(default_prio, " Override 802.1q priority for FIP and FCoE"
+        " traffic (value between 0 and 7, default 3).");
 
 uint qedf_dump_frames;
 module_param_named(dump_frames, qedf_dump_frames, int, S_IRUGO | S_IWUSR);
@@ -89,6 +89,11 @@ module_param_named(retry_delay, qedf_retry_delay, bool, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(retry_delay, " Enable/disable handling of FCP_RSP IU retry "
         "delay handling (default off).");
 
+static bool qedf_dcbx_no_wait;
+module_param_named(dcbx_no_wait, qedf_dcbx_no_wait, bool, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(dcbx_no_wait, " Do not wait for DCBX convergence to start "
+        "sending FIP VLAN requests on link up (Default: off).");
+
 static uint qedf_dp_module;
 module_param_named(dp_module, qedf_dp_module, uint, S_IRUGO);
 MODULE_PARM_DESC(dp_module, " bit flags control for verbose printk passed "
@@ -109,9 +114,9 @@ static struct kmem_cache *qedf_io_work_cache;
 void qedf_set_vlan_id(struct qedf_ctx *qedf, int vlan_id)
 {
         qedf->vlan_id = vlan_id;
-        qedf->vlan_id |= qedf_default_prio << VLAN_PRIO_SHIFT;
+        qedf->vlan_id |= qedf->prio << VLAN_PRIO_SHIFT;
         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Setting vlan_id=%04x "
-                   "prio=%d.\n", vlan_id, qedf_default_prio);
+                   "prio=%d.\n", vlan_id, qedf->prio);
 }
 
 /* Returns true if we have a valid vlan, false otherwise */
@@ -480,6 +485,11 @@ static void qedf_link_update(void *dev, struct qed_link_output *link)
         struct qedf_ctx *qedf = (struct qedf_ctx *)dev;
 
         if (link->link_up) {
+                if (atomic_read(&qedf->link_state) == QEDF_LINK_UP) {
+                        QEDF_INFO((&qedf->dbg_ctx), QEDF_LOG_DISC,
+                            "Ignoring link up event as link is already up.\n");
+                        return;
+                }
                 QEDF_ERR(&(qedf->dbg_ctx), "LINK UP (%d GB/s).\n",
                     link->speed / 1000);
 
@@ -489,7 +499,8 @@ static void qedf_link_update(void *dev, struct qed_link_output *link)
                 atomic_set(&qedf->link_state, QEDF_LINK_UP);
                 qedf_update_link_speed(qedf, link);
 
-                if (atomic_read(&qedf->dcbx) == QEDF_DCBX_DONE) {
+                if (atomic_read(&qedf->dcbx) == QEDF_DCBX_DONE ||
+                    qedf_dcbx_no_wait) {
                         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
                              "DCBx done.\n");
                         if (atomic_read(&qedf->link_down_tmo_valid) > 0)
@@ -515,7 +526,7 @@ static void qedf_link_update(void *dev, struct qed_link_output *link)
                             "Starting link down tmo.\n");
                         atomic_set(&qedf->link_down_tmo_valid, 1);
                 }
-                qedf->vlan_id  = 0;
+                qedf->vlan_id = 0;
                 qedf_update_link_speed(qedf, link);
                 queue_delayed_work(qedf->link_update_wq, &qedf->link_update,
                     qedf_link_down_tmo * HZ);
@@ -526,6 +537,7 @@ static void qedf_link_update(void *dev, struct qed_link_output *link)
 static void qedf_dcbx_handler(void *dev, struct qed_dcbx_get *get, u32 mib_type)
 {
         struct qedf_ctx *qedf = (struct qedf_ctx *)dev;
+        u8 tmp_prio;
 
         QEDF_ERR(&(qedf->dbg_ctx), "DCBx event valid=%d enabled=%d fcoe "
             "prio=%d.\n", get->operational.valid, get->operational.enabled,
@@ -541,7 +553,26 @@ static void qedf_dcbx_handler(void *dev, struct qed_dcbx_get *get, u32 mib_type)
 
                 atomic_set(&qedf->dcbx, QEDF_DCBX_DONE);
 
-                if (atomic_read(&qedf->link_state) == QEDF_LINK_UP) {
+                /*
+                 * Set the 8021q priority in the following manner:
+                 *
+                 * 1. If a modparam is set use that
+                 * 2. If the value is not between 0..7 use the default
+                 * 3. Use the priority we get from the DCBX app tag
+                 */
+                tmp_prio = get->operational.app_prio.fcoe;
+                if (qedf_default_prio > -1)
+                        qedf->prio = qedf_default_prio;
+                else if (tmp_prio < 0 || tmp_prio > 7) {
+                        QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+                            "FIP/FCoE prio %d out of range, setting to %d.\n",
+                            tmp_prio, QEDF_DEFAULT_PRIO);
+                        qedf->prio = QEDF_DEFAULT_PRIO;
+                } else
+                        qedf->prio = tmp_prio;
+
+                if (atomic_read(&qedf->link_state) == QEDF_LINK_UP &&
+                    !qedf_dcbx_no_wait) {
                         if (atomic_read(&qedf->link_down_tmo_valid) > 0)
                                 queue_delayed_work(qedf->link_update_wq,
                                     &qedf->link_recovery, 0);
@@ -614,16 +645,6 @@ static int qedf_eh_abort(struct scsi_cmnd *sc_cmd)
                 goto out;
         }
 
-        if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
-            test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
-            test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
-                QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
-                          "cleanup or abort processing or already "
-                          "completed.\n", io_req->xid);
-                rc = SUCCESS;
-                goto out;
-        }
-
         QEDF_ERR(&(qedf->dbg_ctx), "Aborting io_req sc_cmd=%p xid=0x%x "
                   "fp_idx=%d.\n", sc_cmd, io_req->xid, io_req->fp_idx);
 
@@ -705,7 +726,6 @@ static void qedf_ctx_soft_reset(struct fc_lport *lport)
 
         /* For host reset, essentially do a soft link up/down */
         atomic_set(&qedf->link_state, QEDF_LINK_DOWN);
-        atomic_set(&qedf->dcbx, QEDF_DCBX_PENDING);
         queue_delayed_work(qedf->link_update_wq, &qedf->link_update,
             0);
         qedf_wait_for_upload(qedf);
@@ -720,6 +740,22 @@ static int qedf_eh_host_reset(struct scsi_cmnd *sc_cmd)
 {
         struct fc_lport *lport;
         struct qedf_ctx *qedf;
+        struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
+        struct fc_rport_libfc_priv *rp = rport->dd_data;
+        struct qedf_rport *fcport = (struct qedf_rport *)&rp[1];
+        int rval;
+
+        rval = fc_remote_port_chkready(rport);
+
+        if (rval) {
+                QEDF_ERR(NULL, "device_reset rport not ready\n");
+                return FAILED;
+        }
+
+        if (fcport == NULL) {
+                QEDF_ERR(NULL, "device_reset: rport is NULL\n");
+                return FAILED;
+        }
 
         lport = shost_priv(sc_cmd->device->host);
         qedf = lport_priv(lport);
@@ -1109,7 +1145,7 @@ static int qedf_offload_connection(struct qedf_ctx *qedf,
         conn_info.vlan_tag = qedf->vlan_id <<
             FCOE_CONN_OFFLOAD_RAMROD_DATA_VLAN_ID_SHIFT;
         conn_info.vlan_tag |=
-            qedf_default_prio << FCOE_CONN_OFFLOAD_RAMROD_DATA_PRIORITY_SHIFT;
+            qedf->prio << FCOE_CONN_OFFLOAD_RAMROD_DATA_PRIORITY_SHIFT;
         conn_info.flags |= (FCOE_CONN_OFFLOAD_RAMROD_DATA_B_VLAN_FLAG_MASK <<
             FCOE_CONN_OFFLOAD_RAMROD_DATA_B_VLAN_FLAG_SHIFT);
 
@@ -1649,6 +1685,15 @@ static int qedf_vport_destroy(struct fc_vport *vport)
         struct Scsi_Host *shost = vport_to_shost(vport);
         struct fc_lport *n_port = shost_priv(shost);
         struct fc_lport *vn_port = vport->dd_data;
+        struct qedf_ctx *qedf = lport_priv(vn_port);
+
+        if (!qedf) {
+                QEDF_ERR(NULL, "qedf is NULL.\n");
+                goto out;
+        }
+
+        /* Set unloading bit on vport qedf_ctx to prevent more I/O */
+        set_bit(QEDF_UNLOADING, &qedf->flags);
 
         mutex_lock(&n_port->lp_mutex);
         list_del(&vn_port->list);
@@ -1675,6 +1720,7 @@ static int qedf_vport_destroy(struct fc_vport *vport)
         if (vn_port->host)
                 scsi_host_put(vn_port->host);
 
+out:
         return 0;
 }
 
@@ -2109,7 +2155,8 @@ static int qedf_setup_int(struct qedf_ctx *qedf)
             QEDF_SIMD_HANDLER_NUM, qedf_simd_int_handler);
         qedf->int_info.used_cnt = 1;
 
-        return 0;
+        QEDF_ERR(&qedf->dbg_ctx, "Only MSI-X supported. Failing probe.\n");
+        return -EINVAL;
 }
 
 /* Main function for libfc frame reception */
@@ -2195,6 +2242,7 @@ static void qedf_recv_frame(struct qedf_ctx *qedf,
         if (ntoh24(&dest_mac[3]) != ntoh24(fh->fh_d_id)) {
                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
                     "FC frame d_id mismatch with MAC %pM.\n", dest_mac);
+                kfree_skb(skb);
                 return;
         }
 
@@ -2983,8 +3031,17 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
         qedf->link_update_wq = create_workqueue(host_buf);
         INIT_DELAYED_WORK(&qedf->link_update, qedf_handle_link_update);
         INIT_DELAYED_WORK(&qedf->link_recovery, qedf_link_recovery);
-
+        INIT_DELAYED_WORK(&qedf->grcdump_work, qedf_wq_grcdump);
         qedf->fipvlan_retries = qedf_fipvlan_retries;
+        /* Set a default prio in case DCBX doesn't converge */
+        if (qedf_default_prio > -1) {
+                /*
+                 * This is the case where we pass a modparam in so we want to
+                 * honor it even if dcbx doesn't converge.
+                 */
+                qedf->prio = qedf_default_prio;
+        } else
+                qedf->prio = QEDF_DEFAULT_PRIO;
 
         /*
          * Common probe. Takes care of basic hardware init and pci_*
@@ -3214,7 +3271,8 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
          * unload process.
          */
         if (mode != QEDF_MODE_RECOVERY) {
-                qedf->grcdump_size = qed_ops->common->dbg_grc_size(qedf->cdev);
+                qedf->grcdump_size =
+                    qed_ops->common->dbg_all_data_size(qedf->cdev);
                 if (qedf->grcdump_size) {
                         rc = qedf_alloc_grc_dump_buf(&qedf->grcdump,
                             qedf->grcdump_size);
@@ -3398,6 +3456,15 @@ static void qedf_remove(struct pci_dev *pdev)
         __qedf_remove(pdev, QEDF_MODE_NORMAL);
 }
 
+void qedf_wq_grcdump(struct work_struct *work)
+{
+        struct qedf_ctx *qedf =
+            container_of(work, struct qedf_ctx, grcdump_work.work);
+
+        QEDF_ERR(&(qedf->dbg_ctx), "Collecting GRC dump.\n");
+        qedf_capture_grc_dump(qedf);
+}
+
 /*
  * Protocol TLV handler
  */
@@ -3508,6 +3575,17 @@ static int __init qedf_init(void)
         if (qedf_debug == QEDF_LOG_DEFAULT)
                 qedf_debug = QEDF_DEFAULT_LOG_MASK;
 
+        /*
+         * Check that default prio for FIP/FCoE traffic is between 0..7 if a
+         * value has been set
+         */
+        if (qedf_default_prio > -1)
+                if (qedf_default_prio > 7) {
+                        qedf_default_prio = QEDF_DEFAULT_PRIO;
+                        QEDF_ERR(NULL, "FCoE/FIP priority out of range, resetting to %d.\n",
+                            QEDF_DEFAULT_PRIO);
+                }
+
         /* Print driver banner */
         QEDF_INFO(NULL, QEDF_LOG_INFO, "%s v%s.\n", QEDF_DESCR,
                    QEDF_VERSION);
diff --git a/drivers/scsi/qedf/qedf_version.h b/drivers/scsi/qedf/qedf_version.h
index c2478056356a..9455faacd5de 100644
--- a/drivers/scsi/qedf/qedf_version.h
+++ b/drivers/scsi/qedf/qedf_version.h
@@ -1,15 +1,15 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
  *  this source tree.
  */
 
-#define QEDF_VERSION            "8.33.0.20"
+#define QEDF_VERSION            "8.33.16.20"
 #define QEDF_DRIVER_MAJOR_VER           8
 #define QEDF_DRIVER_MINOR_VER           33
-#define QEDF_DRIVER_REV_VER             0
+#define QEDF_DRIVER_REV_VER             16
 #define QEDF_DRIVER_ENG_VER             20
 
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index eb2ec1fb07cb..9442e18aef6f 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -2279,8 +2279,6 @@ enum discovery_state {
         DSC_LOGIN_PEND,
         DSC_LOGIN_FAILED,
         DSC_GPDB,
-        DSC_GFPN_ID,
-        DSC_GPSC,
         DSC_UPD_FCPORT,
         DSC_LOGIN_COMPLETE,
         DSC_ADISC,
@@ -2346,6 +2344,7 @@ typedef struct fc_port {
         unsigned int login_succ:1;
         unsigned int query:1;
         unsigned int id_changed:1;
+        unsigned int rscn_rcvd:1;
 
         struct work_struct nvme_del_work;
         struct completion nvme_del_done;
@@ -3226,6 +3225,7 @@ enum qla_work_type {
         QLA_EVT_GNNID,
         QLA_EVT_GFPNID,
         QLA_EVT_SP_RETRY,
+        QLA_EVT_IIDMA,
 };
 
 
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 3c4c84ed0f0f..f68eb6096559 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -116,7 +116,8 @@ extern int qla2x00_post_async_prlo_work(struct scsi_qla_host *, fc_port_t *,
     uint16_t *);
 extern int qla2x00_post_async_prlo_done_work(struct scsi_qla_host *,
     fc_port_t *, uint16_t *);
-
+int qla_post_iidma_work(struct scsi_qla_host *vha, fc_port_t *fcport);
+void qla_do_iidma_work(struct scsi_qla_host *vha, fc_port_t *fcport);
 /*
  * Global Data in qla_os.c source file.
  */
diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c
index 9e914f9c3ffb..4bc2b66b299f 100644
--- a/drivers/scsi/qla2xxx/qla_gs.c
+++ b/drivers/scsi/qla2xxx/qla_gs.c
@@ -3175,7 +3175,6 @@ int qla24xx_async_gidpn(scsi_qla_host_t *vha, fc_port_t *fcport)
 
 done_free_sp:
         sp->free(sp);
-        fcport->flags &= ~FCF_ASYNC_SENT;
 done:
         fcport->flags &= ~FCF_ASYNC_ACTIVE;
         return rval;
@@ -3239,7 +3238,7 @@ void qla24xx_handle_gpsc_event(scsi_qla_host_t *vha, struct event_arg *ea)
                 return;
         }
 
-        qla24xx_post_upd_fcport_work(vha, ea->fcport);
+        qla_post_iidma_work(vha, fcport);
 }
 
 static void qla24xx_async_gpsc_sp_done(void *s, int res)
@@ -3257,8 +3256,6 @@ static void qla24xx_async_gpsc_sp_done(void *s, int res)
             "Async done-%s res %x, WWPN %8phC \n",
             sp->name, res, fcport->port_name);
 
-        fcport->flags &= ~(FCF_ASYNC_SENT | FCF_ASYNC_ACTIVE);
-
         if (res == (DID_ERROR << 16)) {
                 /* entry status error */
                 goto done;
@@ -3327,7 +3324,6 @@ int qla24xx_async_gpsc(scsi_qla_host_t *vha, fc_port_t *fcport)
         if (!sp)
                 goto done;
 
-        fcport->flags |= FCF_ASYNC_SENT;
         sp->type = SRB_CT_PTHRU_CMD;
         sp->name = "gpsc";
         sp->gen1 = fcport->rscn_gen;
@@ -3862,6 +3858,7 @@ void qla24xx_async_gnnft_done(scsi_qla_host_t *vha, srb_t *sp)
         bool found;
         struct fab_scan_rp *rp;
         unsigned long flags;
+        u8 recheck = 0;
 
         ql_dbg(ql_dbg_disc, vha, 0xffff,
             "%s enter\n", __func__);
@@ -3914,8 +3911,8 @@ void qla24xx_async_gnnft_done(scsi_qla_host_t *vha, srb_t *sp)
                 list_for_each_entry(fcport, &vha->vp_fcports, list) {
                         if (memcmp(rp->port_name, fcport->port_name, WWN_SIZE))
                                 continue;
+                        fcport->rscn_rcvd = 0;
                         fcport->scan_state = QLA_FCPORT_FOUND;
-                        fcport->d_id.b24 = rp->id.b24;
                         found = true;
                         /*
                          * If device was not a fabric device before.
@@ -3923,7 +3920,10 @@ void qla24xx_async_gnnft_done(scsi_qla_host_t *vha, srb_t *sp)
                         if ((fcport->flags & FCF_FABRIC_DEVICE) == 0) {
                                 qla2x00_clear_loop_id(fcport);
                                 fcport->flags |= FCF_FABRIC_DEVICE;
+                        } else if (fcport->d_id.b24 != rp->id.b24) {
+                                qlt_schedule_sess_for_deletion(fcport);
                         }
+                        fcport->d_id.b24 = rp->id.b24;
                         break;
                 }
 
@@ -3940,10 +3940,13 @@ void qla24xx_async_gnnft_done(scsi_qla_host_t *vha, srb_t *sp)
          * Logout all previous fabric dev marked lost, except FCP2 devices.
          */
         list_for_each_entry(fcport, &vha->vp_fcports, list) {
-                if ((fcport->flags & FCF_FABRIC_DEVICE) == 0)
+                if ((fcport->flags & FCF_FABRIC_DEVICE) == 0) {
+                        fcport->rscn_rcvd = 0;
                         continue;
+                }
 
                 if (fcport->scan_state != QLA_FCPORT_FOUND) {
+                        fcport->rscn_rcvd = 0;
                         if ((qla_dual_mode_enabled(vha) ||
                                 qla_ini_mode_enabled(vha)) &&
                             atomic_read(&fcport->state) == FCS_ONLINE) {
@@ -3961,15 +3964,31 @@ void qla24xx_async_gnnft_done(scsi_qla_host_t *vha, srb_t *sp)
                                         continue;
                                 }
                         }
-                } else
-                        qla24xx_fcport_handle_login(vha, fcport);
+                } else {
+                        if (fcport->rscn_rcvd ||
+                            fcport->disc_state != DSC_LOGIN_COMPLETE) {
+                                fcport->rscn_rcvd = 0;
+                                qla24xx_fcport_handle_login(vha, fcport);
+                        }
+                }
         }
 
+        recheck = 1;
 out:
         qla24xx_sp_unmap(vha, sp);
         spin_lock_irqsave(&vha->work_lock, flags);
         vha->scan.scan_flags &= ~SF_SCANNING;
         spin_unlock_irqrestore(&vha->work_lock, flags);
+
+        if (recheck) {
+                list_for_each_entry(fcport, &vha->vp_fcports, list) {
+                        if (fcport->rscn_rcvd) {
+                                set_bit(LOCAL_LOOP_UPDATE, &vha->dpc_flags);
+                                set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
+                                break;
+                        }
+                }
+        }
 }
 
 static void qla2x00_find_free_fcp_nvme_slot(struct scsi_qla_host *vha,
@@ -4532,7 +4551,6 @@ int qla24xx_async_gnnid(scsi_qla_host_t *vha, fc_port_t *fcport)
 
 done_free_sp:
         sp->free(sp);
-        fcport->flags &= ~FCF_ASYNC_SENT;
 done:
         return rval;
 }
@@ -4594,7 +4612,6 @@ static void qla2x00_async_gfpnid_sp_done(void *s, int res)
         struct event_arg ea;
         u64 wwn;
 
-        fcport->flags &= ~FCF_ASYNC_SENT;
         wwn = wwn_to_u64(fpn);
         if (wwn)
                 memcpy(fcport->fabric_port_name, fpn, WWN_SIZE);
@@ -4623,12 +4640,10 @@ int qla24xx_async_gfpnid(scsi_qla_host_t *vha, fc_port_t *fcport)
         if (!vha->flags.online || (fcport->flags & FCF_ASYNC_SENT))
                 return rval;
 
-        fcport->disc_state = DSC_GFPN_ID;
         sp = qla2x00_get_sp(vha, fcport, GFP_ATOMIC);
         if (!sp)
                 goto done;
 
-        fcport->flags |= FCF_ASYNC_SENT;
         sp->type = SRB_CT_PTHRU_CMD;
         sp->name = "gfpnid";
         sp->gen1 = fcport->rscn_gen;
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index 8f55dd44adae..1aa3720ea2ed 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -1021,30 +1021,11 @@ void __qla24xx_handle_gpdb_event(scsi_qla_host_t *vha, struct event_arg *ea)
                 vha->fcport_count++;
                 ea->fcport->login_succ = 1;
 
-                if (!IS_IIDMA_CAPABLE(vha->hw) ||
-                    !vha->hw->flags.gpsc_supported) {
-                        ql_dbg(ql_dbg_disc, vha, 0x20d6,
-                            "%s %d %8phC post upd_fcport fcp_cnt %d\n",
-                            __func__, __LINE__,  ea->fcport->port_name,
-                            vha->fcport_count);
-
-                        qla24xx_post_upd_fcport_work(vha, ea->fcport);
-                } else {
-                        if (ea->fcport->id_changed) {
-                                ea->fcport->id_changed = 0;
-                                ql_dbg(ql_dbg_disc, vha, 0x20d7,
-                                    "%s %d %8phC post gfpnid fcp_cnt %d\n",
-                                    __func__, __LINE__, ea->fcport->port_name,
-                                    vha->fcport_count);
-                                qla24xx_post_gfpnid_work(vha, ea->fcport);
-                        } else {
-                                ql_dbg(ql_dbg_disc, vha, 0x20d7,
-                                    "%s %d %8phC post gpsc fcp_cnt %d\n",
-                                    __func__, __LINE__, ea->fcport->port_name,
-                                    vha->fcport_count);
-                                qla24xx_post_gpsc_work(vha, ea->fcport);
-                        }
-                }
+                ql_dbg(ql_dbg_disc, vha, 0x20d6,
+                    "%s %d %8phC post upd_fcport fcp_cnt %d\n",
+                    __func__, __LINE__,  ea->fcport->port_name,
+                    vha->fcport_count);
+                qla24xx_post_upd_fcport_work(vha, ea->fcport);
         } else if (ea->fcport->login_succ) {
                 /*
                  * We have an existing session. A late RSCN delivery
@@ -1167,9 +1148,6 @@ int qla24xx_fcport_handle_login(struct scsi_qla_host *vha, fc_port_t *fcport)
             fcport->login_gen, fcport->login_retry,
             fcport->loop_id, fcport->scan_state);
 
-        if (fcport->login_retry == 0)
-                return 0;
-
         if (fcport->scan_state != QLA_FCPORT_FOUND)
                 return 0;
 
@@ -1194,7 +1172,8 @@ int qla24xx_fcport_handle_login(struct scsi_qla_host *vha, fc_port_t *fcport)
                 return 0;
         }
 
-        fcport->login_retry--;
+        if (fcport->login_retry > 0)
+                fcport->login_retry--;
 
         switch (fcport->disc_state) {
         case DSC_DELETED:
@@ -1350,20 +1329,7 @@ void qla2x00_fcport_event_handler(scsi_qla_host_t *vha, struct event_arg *ea)
         fc_port_t *f, *tf;
         uint32_t id = 0, mask, rid;
         unsigned long flags;
-
-        switch (ea->event) {
-        case FCME_RSCN:
-        case FCME_GIDPN_DONE:
-        case FCME_GPSC_DONE:
-        case FCME_GPNID_DONE:
-        case FCME_GNNID_DONE:
-                if (test_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags) ||
-                    test_bit(LOOP_RESYNC_ACTIVE, &vha->dpc_flags))
-                        return;
-                break;
-        default:
-                break;
-        }
+        fc_port_t *fcport;
 
         switch (ea->event) {
         case FCME_RELOGIN:
@@ -1377,6 +1343,11 @@ void qla2x00_fcport_event_handler(scsi_qla_host_t *vha, struct event_arg *ea)
                         return;
                 switch (ea->id.b.rsvd_1) {
                 case RSCN_PORT_ADDR:
+                        fcport = qla2x00_find_fcport_by_nportid
+                                (vha, &ea->id, 1);
+                        if (fcport)
+                                fcport->rscn_rcvd = 1;
+
                         spin_lock_irqsave(&vha->work_lock, flags);
                         if (vha->scan.scan_flags == 0) {
                                 ql_dbg(ql_dbg_disc, vha, 0xffff,
@@ -4532,7 +4503,6 @@ qla2x00_alloc_fcport(scsi_qla_host_t *vha, gfp_t flags)
         fcport->fw_login_state = DSC_LS_PORT_UNAVAIL;
         fcport->deleted = QLA_SESS_DELETED;
         fcport->login_retry = vha->hw->login_retry_count;
-        fcport->login_retry = 5;
         fcport->logout_on_delete = 1;
 
         if (!fcport->ct_desc.ct_sns) {
@@ -5054,6 +5024,24 @@ qla2x00_iidma_fcport(scsi_qla_host_t *vha, fc_port_t *fcport)
         }
 }
 
+void qla_do_iidma_work(struct scsi_qla_host *vha, fc_port_t *fcport)
+{
+        qla2x00_iidma_fcport(vha, fcport);
+        qla24xx_update_fcport_fcp_prio(vha, fcport);
+}
+
+int qla_post_iidma_work(struct scsi_qla_host *vha, fc_port_t *fcport)
+{
+        struct qla_work_evt *e;
+
+        e = qla2x00_alloc_work(vha, QLA_EVT_IIDMA);
+        if (!e)
+                return QLA_FUNCTION_FAILED;
+
+        e->u.fcport.fcport = fcport;
+        return qla2x00_post_work(vha, e);
+}
+
 /* qla2x00_reg_remote_port is reserved for Initiator Mode only.*/
 static void
 qla2x00_reg_remote_port(scsi_qla_host_t *vha, fc_port_t *fcport)
@@ -5122,13 +5110,14 @@ qla2x00_update_fcport(scsi_qla_host_t *vha, fc_port_t *fcport)
 
         if (IS_QLAFX00(vha->hw)) {
                 qla2x00_set_fcport_state(fcport, FCS_ONLINE);
-                goto reg_port;
+        } else {
+                fcport->login_retry = 0;
+                fcport->flags &= ~(FCF_LOGIN_NEEDED | FCF_ASYNC_SENT);
+                fcport->disc_state = DSC_LOGIN_COMPLETE;
+                fcport->deleted = 0;
+                fcport->logout_on_delete = 1;
+                qla2x00_set_fcport_state(fcport, FCS_ONLINE);
         }
-        fcport->login_retry = 0;
-        fcport->flags &= ~(FCF_LOGIN_NEEDED | FCF_ASYNC_SENT);
-        fcport->disc_state = DSC_LOGIN_COMPLETE;
-        fcport->deleted = 0;
-        fcport->logout_on_delete = 1;
 
         qla2x00_set_fcport_state(fcport, FCS_ONLINE);
         qla2x00_iidma_fcport(vha, fcport);
@@ -5140,7 +5129,6 @@ qla2x00_update_fcport(scsi_qla_host_t *vha, fc_port_t *fcport)
 
         qla24xx_update_fcport_fcp_prio(vha, fcport);
 
-reg_port:
         switch (vha->host->active_mode) {
         case MODE_INITIATOR:
                 qla2x00_reg_remote_port(vha, fcport);
@@ -5159,6 +5147,23 @@ reg_port:
         default:
                 break;
         }
+
+        if (IS_IIDMA_CAPABLE(vha->hw) && vha->hw->flags.gpsc_supported) {
+                if (fcport->id_changed) {
+                        fcport->id_changed = 0;
+                        ql_dbg(ql_dbg_disc, vha, 0x20d7,
+                            "%s %d %8phC post gfpnid fcp_cnt %d\n",
+                            __func__, __LINE__, fcport->port_name,
+                            vha->fcport_count);
+                        qla24xx_post_gfpnid_work(vha, fcport);
+                } else {
+                        ql_dbg(ql_dbg_disc, vha, 0x20d7,
+                            "%s %d %8phC post gpsc fcp_cnt %d\n",
+                            __func__, __LINE__, fcport->port_name,
+                            vha->fcport_count);
+                        qla24xx_post_gpsc_work(vha, fcport);
+                }
+        }
 }
 
 /*
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 15eaa6dded04..817c18a8e84d 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -5063,6 +5063,10 @@ qla2x00_do_work(struct scsi_qla_host *vha)
                         break;
                 case QLA_EVT_SP_RETRY:
                         qla_sp_retry(vha, e);
+                        break;
+                case QLA_EVT_IIDMA:
+                        qla_do_iidma_work(vha, e->u.fcport.fcport);
+                        break;
                 }
                 if (e->flags & QLA_EVT_FLAG_FREE)
                         kfree(e);
diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c
index 025dc2d3f3de..b85c833099ff 100644
--- a/drivers/scsi/qla2xxx/qla_target.c
+++ b/drivers/scsi/qla2xxx/qla_target.c
@@ -601,24 +601,18 @@ void qla2x00_async_nack_sp_done(void *s, int res)
 
                         vha->fcport_count++;
 
-                        if (!IS_IIDMA_CAPABLE(vha->hw) ||
-                            !vha->hw->flags.gpsc_supported) {
-                                ql_dbg(ql_dbg_disc, vha, 0x20f3,
-                                    "%s %d %8phC post upd_fcport fcp_cnt %d\n",
-                                    __func__, __LINE__,
-                                    sp->fcport->port_name,
-                                    vha->fcport_count);
-                                sp->fcport->disc_state = DSC_UPD_FCPORT;
-                                qla24xx_post_upd_fcport_work(vha, sp->fcport);
-                        } else {
-                                ql_dbg(ql_dbg_disc, vha, 0x20f5,
-                                    "%s %d %8phC post gpsc fcp_cnt %d\n",
-                                    __func__, __LINE__,
-                                    sp->fcport->port_name,
-                                    vha->fcport_count);
-
-                                qla24xx_post_gpsc_work(vha, sp->fcport);
-                        }
+                        ql_dbg(ql_dbg_disc, vha, 0x20f3,
+                            "%s %d %8phC post upd_fcport fcp_cnt %d\n",
+                            __func__, __LINE__,
+                            sp->fcport->port_name,
+                            vha->fcport_count);
+                        sp->fcport->disc_state = DSC_UPD_FCPORT;
+                        qla24xx_post_upd_fcport_work(vha, sp->fcport);
+                } else {
+                        sp->fcport->login_retry = 0;
+                        sp->fcport->disc_state = DSC_LOGIN_COMPLETE;
+                        sp->fcport->deleted = 0;
+                        sp->fcport->logout_on_delete = 1;
                 }
                 break;
 
@@ -1930,13 +1924,84 @@ static void abort_cmds_for_lun(struct scsi_qla_host *vha,
         spin_unlock_irqrestore(&vha->cmd_list_lock, flags);
 }
 
+static struct qla_qpair_hint *qlt_find_qphint(struct scsi_qla_host *vha,
+    uint64_t unpacked_lun)
+{
+        struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
+        struct qla_qpair_hint *h = NULL;
+
+        if (vha->flags.qpairs_available) {
+                h = btree_lookup64(&tgt->lun_qpair_map, unpacked_lun);
+                if (!h)
+                        h = &tgt->qphints[0];
+        } else {
+                h = &tgt->qphints[0];
+        }
+
+        return h;
+}
+
+static void qlt_do_tmr_work(struct work_struct *work)
+{
+        struct qla_tgt_mgmt_cmd *mcmd =
+                container_of(work, struct qla_tgt_mgmt_cmd, work);
+        struct qla_hw_data *ha = mcmd->vha->hw;
+        int rc = EIO;
+        uint32_t tag;
+        unsigned long flags;
+
+        switch (mcmd->tmr_func) {
+        case QLA_TGT_ABTS:
+                tag = mcmd->orig_iocb.abts.exchange_addr_to_abort;
+                break;
+        default:
+                tag = 0;
+                break;
+        }
+
+        rc = ha->tgt.tgt_ops->handle_tmr(mcmd, mcmd->unpacked_lun,
+            mcmd->tmr_func, tag);
+
+        if (rc != 0) {
+                spin_lock_irqsave(mcmd->qpair->qp_lock_ptr, flags);
+                switch (mcmd->tmr_func) {
+                case QLA_TGT_ABTS:
+                        qlt_24xx_send_abts_resp(mcmd->qpair,
+                            &mcmd->orig_iocb.abts,
+                            FCP_TMF_REJECTED, false);
+                        break;
+                case QLA_TGT_LUN_RESET:
+                case QLA_TGT_CLEAR_TS:
+                case QLA_TGT_ABORT_TS:
+                case QLA_TGT_CLEAR_ACA:
+                case QLA_TGT_TARGET_RESET:
+                        qlt_send_busy(mcmd->qpair, &mcmd->orig_iocb.atio,
+                            qla_sam_status);
+                        break;
+
+                case QLA_TGT_ABORT_ALL:
+                case QLA_TGT_NEXUS_LOSS_SESS:
+                case QLA_TGT_NEXUS_LOSS:
+                        qlt_send_notify_ack(mcmd->qpair,
+                            &mcmd->orig_iocb.imm_ntfy, 0, 0, 0, 0, 0, 0);
+                        break;
+                }
+                spin_unlock_irqrestore(mcmd->qpair->qp_lock_ptr, flags);
+
+                ql_dbg(ql_dbg_tgt_mgt, mcmd->vha, 0xf052,
+                    "qla_target(%d):  tgt_ops->handle_tmr() failed: %d\n",
+                    mcmd->vha->vp_idx, rc);
+                mempool_free(mcmd, qla_tgt_mgmt_cmd_mempool);
+        }
+}
+
 /* ha->hardware_lock supposed to be held on entry */
 static int __qlt_24xx_handle_abts(struct scsi_qla_host *vha,
         struct abts_recv_from_24xx *abts, struct fc_port *sess)
 {
         struct qla_hw_data *ha = vha->hw;
         struct qla_tgt_mgmt_cmd *mcmd;
-        int rc;
+        struct qla_qpair_hint *h = &vha->vha_tgt.qla_tgt->qphints[0];
 
         if (abort_cmd_for_tag(vha, abts->exchange_addr_to_abort)) {
                 /* send TASK_ABORT response immediately */
@@ -1961,23 +2026,29 @@ static int __qlt_24xx_handle_abts(struct scsi_qla_host *vha,
         memcpy(&mcmd->orig_iocb.abts, abts, sizeof(mcmd->orig_iocb.abts));
         mcmd->reset_count = ha->base_qpair->chip_reset;
         mcmd->tmr_func = QLA_TGT_ABTS;
-        mcmd->qpair = ha->base_qpair;
+        mcmd->qpair = h->qpair;
         mcmd->vha = vha;
 
         /*
          * LUN is looked up by target-core internally based on the passed
          * abts->exchange_addr_to_abort tag.
          */
-        rc = ha->tgt.tgt_ops->handle_tmr(mcmd, 0, mcmd->tmr_func,
-            abts->exchange_addr_to_abort);
-        if (rc != 0) {
-                ql_dbg(ql_dbg_tgt_mgt, vha, 0xf052,
-                    "qla_target(%d):  tgt_ops->handle_tmr()"
-                    " failed: %d", vha->vp_idx, rc);
-                mempool_free(mcmd, qla_tgt_mgmt_cmd_mempool);
-                return -EFAULT;
+        mcmd->se_cmd.cpuid = h->cpuid;
+
+        if (ha->tgt.tgt_ops->find_cmd_by_tag) {
+                struct qla_tgt_cmd *abort_cmd;
+
+                abort_cmd = ha->tgt.tgt_ops->find_cmd_by_tag(sess,
+                    abts->exchange_addr_to_abort);
+                if (abort_cmd && abort_cmd->qpair) {
+                        mcmd->qpair = abort_cmd->qpair;
+                        mcmd->se_cmd.cpuid = abort_cmd->se_cmd.cpuid;
+                }
         }
 
+        INIT_WORK(&mcmd->work, qlt_do_tmr_work);
+        queue_work_on(mcmd->se_cmd.cpuid, qla_tgt_wq, &mcmd->work);
+
         return 0;
 }
 
@@ -3556,13 +3627,6 @@ static int __qlt_send_term_exchange(struct qla_qpair *qpair,
         temp = be16_to_cpu(atio->u.isp24.fcp_hdr.ox_id);
         ctio24->u.status1.ox_id = cpu_to_le16(temp);
 
-        /* Most likely, it isn't needed */
-        ctio24->u.status1.residual = get_unaligned((uint32_t *)
-            &atio->u.isp24.fcp_cmnd.add_cdb[
-            atio->u.isp24.fcp_cmnd.add_cdb_len]);
-        if (ctio24->u.status1.residual != 0)
-                ctio24->u.status1.scsi_status |= SS_RESIDUAL_UNDER;
-
         /* Memory Barrier */
         wmb();
         if (qpair->reqq_start_iocbs)
@@ -4057,9 +4121,7 @@ static void __qlt_do_work(struct qla_tgt_cmd *cmd)
 
         fcp_task_attr = qlt_get_fcp_task_attr(vha,
             atio->u.isp24.fcp_cmnd.task_attr);
-        data_length = be32_to_cpu(get_unaligned((uint32_t *)
-            &atio->u.isp24.fcp_cmnd.add_cdb[
-            atio->u.isp24.fcp_cmnd.add_cdb_len]));
+        data_length = get_datalen_for_atio(atio);
 
         ret = ha->tgt.tgt_ops->handle_cmd(vha, cmd, cdb, data_length,
                                           fcp_task_attr, data_dir, bidi);
@@ -4335,7 +4397,7 @@ static int qlt_issue_task_mgmt(struct fc_port *sess, u64 lun,
         struct qla_hw_data *ha = vha->hw;
         struct qla_tgt_mgmt_cmd *mcmd;
         struct atio_from_isp *a = (struct atio_from_isp *)iocb;
-        int res;
+        struct qla_qpair_hint *h = &vha->vha_tgt.qla_tgt->qphints[0];
 
         mcmd = mempool_alloc(qla_tgt_mgmt_cmd_mempool, GFP_ATOMIC);
         if (!mcmd) {
@@ -4355,24 +4417,36 @@ static int qlt_issue_task_mgmt(struct fc_port *sess, u64 lun,
         mcmd->tmr_func = fn;
         mcmd->flags = flags;
         mcmd->reset_count = ha->base_qpair->chip_reset;
-        mcmd->qpair = ha->base_qpair;
+        mcmd->qpair = h->qpair;
         mcmd->vha = vha;
+        mcmd->se_cmd.cpuid = h->cpuid;
+        mcmd->unpacked_lun = lun;
 
         switch (fn) {
         case QLA_TGT_LUN_RESET:
-            abort_cmds_for_lun(vha, lun, a->u.isp24.fcp_hdr.s_id);
-            break;
-        }
+        case QLA_TGT_CLEAR_TS:
+        case QLA_TGT_ABORT_TS:
+                abort_cmds_for_lun(vha, lun, a->u.isp24.fcp_hdr.s_id);
+                /* drop through */
+        case QLA_TGT_CLEAR_ACA:
+                h = qlt_find_qphint(vha, mcmd->unpacked_lun);
+                mcmd->qpair = h->qpair;
+                mcmd->se_cmd.cpuid = h->cpuid;
+                break;
 
-        res = ha->tgt.tgt_ops->handle_tmr(mcmd, lun, mcmd->tmr_func, 0);
-        if (res != 0) {
-                ql_dbg(ql_dbg_tgt_tmr, vha, 0x1000b,
-                    "qla_target(%d): tgt.tgt_ops->handle_tmr() failed: %d\n",
-                    sess->vha->vp_idx, res);
-                mempool_free(mcmd, qla_tgt_mgmt_cmd_mempool);
-                return -EFAULT;
+        case QLA_TGT_TARGET_RESET:
+        case QLA_TGT_NEXUS_LOSS_SESS:
+        case QLA_TGT_NEXUS_LOSS:
+        case QLA_TGT_ABORT_ALL:
+        default:
+                /* no-op */
+                break;
         }
 
+        INIT_WORK(&mcmd->work, qlt_do_tmr_work);
+        queue_work_on(mcmd->se_cmd.cpuid, qla_tgt_wq,
+            &mcmd->work);
+
         return 0;
 }
 
@@ -4841,7 +4915,6 @@ static int qlt_24xx_handle_els(struct scsi_qla_host *vha,
                         switch (sess->disc_state) {
                         case DSC_LOGIN_PEND:
                         case DSC_GPDB:
-                        case DSC_GPSC:
                         case DSC_UPD_FCPORT:
                         case DSC_LOGIN_COMPLETE:
                         case DSC_ADISC:
@@ -5113,8 +5186,6 @@ static void qlt_handle_imm_notify(struct scsi_qla_host *vha,
                 ql_dbg(ql_dbg_tgt_mgt, vha, 0xf038,
                     "qla_target(%d): Immediate notify task %x\n",
                     vha->vp_idx, iocb->u.isp2x.task_flags);
-                if (qlt_handle_task_mgmt(vha, iocb) == 0)
-                        send_notify_ack = 0;
                 break;
 
         case IMM_NTFY_ELS:
@@ -5147,10 +5218,15 @@ static int __qlt_send_busy(struct qla_qpair *qpair,
         struct fc_port *sess = NULL;
         unsigned long flags;
         u16 temp;
+        port_id_t id;
+
+        id.b.al_pa = atio->u.isp24.fcp_hdr.s_id[2];
+        id.b.area = atio->u.isp24.fcp_hdr.s_id[1];
+        id.b.domain = atio->u.isp24.fcp_hdr.s_id[0];
+        id.b.rsvd_1 = 0;
 
         spin_lock_irqsave(&ha->tgt.sess_lock, flags);
-        sess = ha->tgt.tgt_ops->find_sess_by_s_id(vha,
-            atio->u.isp24.fcp_hdr.s_id);
+        sess = qla2x00_find_fcport_by_nportid(vha, &id, 1);
         spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
         if (!sess) {
                 qlt_send_term_exchange(qpair, NULL, atio, 1, 0);
@@ -5189,6 +5265,12 @@ static int __qlt_send_busy(struct qla_qpair *qpair,
          */
         ctio24->u.status1.ox_id = swab16(atio->u.isp24.fcp_hdr.ox_id);
         ctio24->u.status1.scsi_status = cpu_to_le16(status);
+
+        ctio24->u.status1.residual = get_datalen_for_atio(atio);
+
+        if (ctio24->u.status1.residual != 0)
+                ctio24->u.status1.scsi_status |= SS_RESIDUAL_UNDER;
+
         /* Memory Barrier */
         wmb();
         if (qpair->reqq_start_iocbs)
diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h
index 728ce74358e7..fecf96f0225c 100644
--- a/drivers/scsi/qla2xxx/qla_target.h
+++ b/drivers/scsi/qla2xxx/qla_target.h
@@ -682,7 +682,7 @@ struct qla_tgt_cmd;
  * target module (tcm_qla2xxx).
  */
 struct qla_tgt_func_tmpl {
-
+        struct qla_tgt_cmd *(*find_cmd_by_tag)(struct fc_port *, uint64_t);
         int (*handle_cmd)(struct scsi_qla_host *, struct qla_tgt_cmd *,
                         unsigned char *, uint32_t, int, int, int);
         void (*handle_data)(struct qla_tgt_cmd *);
@@ -966,6 +966,8 @@ struct qla_tgt_mgmt_cmd {
         unsigned int flags;
         uint32_t reset_count;
 #define QLA24XX_MGMT_SEND_NACK  1
+        struct work_struct work;
+        uint64_t unpacked_lun;
         union {
                 struct atio_from_isp atio;
                 struct imm_ntfy_from_isp imm_ntfy;
diff --git a/drivers/scsi/qla2xxx/qla_version.h b/drivers/scsi/qla2xxx/qla_version.h
index 0c55d7057280..1ad7582220c3 100644
--- a/drivers/scsi/qla2xxx/qla_version.h
+++ b/drivers/scsi/qla2xxx/qla_version.h
@@ -7,7 +7,7 @@
 /*
  * Driver version
  */
-#define QLA2XXX_VERSION      "10.00.00.06-k"
+#define QLA2XXX_VERSION      "10.00.00.07-k"
 
 #define QLA_DRIVER_MAJOR_VER    10
 #define QLA_DRIVER_MINOR_VER    0
diff --git a/drivers/scsi/qla2xxx/tcm_qla2xxx.c b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
index aadfeaac3898..0c2e82af9c0a 100644
--- a/drivers/scsi/qla2xxx/tcm_qla2xxx.c
+++ b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
@@ -48,7 +48,6 @@
 #include "tcm_qla2xxx.h"
 
 static struct workqueue_struct *tcm_qla2xxx_free_wq;
-static struct workqueue_struct *tcm_qla2xxx_cmd_wq;
 
 /*
  * Parse WWN.
@@ -630,6 +629,32 @@ static int tcm_qla2xxx_handle_tmr(struct qla_tgt_mgmt_cmd *mcmd, u64 lun,
             transl_tmr_func, GFP_ATOMIC, tag, flags);
 }
 
+static struct qla_tgt_cmd *tcm_qla2xxx_find_cmd_by_tag(struct fc_port *sess,
+    uint64_t tag)
+{
+        struct qla_tgt_cmd *cmd = NULL;
+        struct se_cmd *secmd;
+        unsigned long flags;
+
+        if (!sess->se_sess)
+                return NULL;
+
+        spin_lock_irqsave(&sess->se_sess->sess_cmd_lock, flags);
+        list_for_each_entry(secmd, &sess->se_sess->sess_cmd_list, se_cmd_list) {
+                /* skip task management functions, including tmr->task_cmd */
+                if (secmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
+                        continue;
+
+                if (secmd->tag == tag) {
+                        cmd = container_of(secmd, struct qla_tgt_cmd, se_cmd);
+                        break;
+                }
+        }
+        spin_unlock_irqrestore(&sess->se_sess->sess_cmd_lock, flags);
+
+        return cmd;
+}
+
 static int tcm_qla2xxx_queue_data_in(struct se_cmd *se_cmd)
 {
         struct qla_tgt_cmd *cmd = container_of(se_cmd,
@@ -1608,6 +1633,7 @@ static void tcm_qla2xxx_update_sess(struct fc_port *sess, port_id_t s_id,
  * Calls into tcm_qla2xxx used by qla2xxx LLD I/O path.
  */
 static struct qla_tgt_func_tmpl tcm_qla2xxx_template = {
+        .find_cmd_by_tag        = tcm_qla2xxx_find_cmd_by_tag,
         .handle_cmd             = tcm_qla2xxx_handle_cmd,
         .handle_data            = tcm_qla2xxx_handle_data,
         .handle_tmr             = tcm_qla2xxx_handle_tmr,
@@ -1976,16 +2002,8 @@ static int tcm_qla2xxx_register_configfs(void)
                 goto out_fabric_npiv;
         }
 
-        tcm_qla2xxx_cmd_wq = alloc_workqueue("tcm_qla2xxx_cmd", 0, 0);
-        if (!tcm_qla2xxx_cmd_wq) {
-                ret = -ENOMEM;
-                goto out_free_wq;
-        }
-
         return 0;
 
-out_free_wq:
-        destroy_workqueue(tcm_qla2xxx_free_wq);
 out_fabric_npiv:
         target_unregister_template(&tcm_qla2xxx_npiv_ops);
 out_fabric:
@@ -1995,7 +2013,6 @@ out_fabric:
 
 static void tcm_qla2xxx_deregister_configfs(void)
 {
-        destroy_workqueue(tcm_qla2xxx_cmd_wq);
         destroy_workqueue(tcm_qla2xxx_free_wq);
 
         target_unregister_template(&tcm_qla2xxx_ops);
diff --git a/drivers/scsi/qlogicpti.c b/drivers/scsi/qlogicpti.c
index cec9a14982e6..8578e566ab41 100644
--- a/drivers/scsi/qlogicpti.c
+++ b/drivers/scsi/qlogicpti.c
@@ -1385,6 +1385,9 @@ fail_unmap_queues:
                           qpti->req_cpu, qpti->req_dvma);
 #undef QSIZE
 
+fail_free_irq:
+        free_irq(qpti->irq, qpti);
+
 fail_unmap_regs:
         of_iounmap(&op->resource[0], qpti->qregs,
                    resource_size(&op->resource[0]));
@@ -1392,9 +1395,6 @@ fail_unmap_regs:
                 of_iounmap(&op->resource[0], qpti->sreg,
                            sizeof(unsigned char));
 
-fail_free_irq:
-        free_irq(qpti->irq, qpti);
-
 fail_unlink:
         scsi_host_put(host);
 
diff --git a/drivers/scsi/scsi_debugfs.c b/drivers/scsi/scsi_debugfs.c
index b784002ef0bd..c5a8756384bc 100644
--- a/drivers/scsi/scsi_debugfs.c
+++ b/drivers/scsi/scsi_debugfs.c
@@ -4,7 +4,7 @@
 #include <scsi/scsi_dbg.h>
 #include "scsi_debugfs.h"
 
-#define SCSI_CMD_FLAG_NAME(name) [ilog2(SCMD_##name)] = #name
+#define SCSI_CMD_FLAG_NAME(name)[const_ilog2(SCMD_##name)] = #name
 static const char *const scsi_cmd_flags[] = {
         SCSI_CMD_FLAG_NAME(TAGGED),
         SCSI_CMD_FLAG_NAME(UNCHECKED_ISA_DMA),
diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c
index dd107dc4db0e..c4cbfd07b916 100644
--- a/drivers/scsi/scsi_devinfo.c
+++ b/drivers/scsi/scsi_devinfo.c
@@ -161,15 +161,16 @@ static struct {
         {"DGC", "RAID", NULL, BLIST_SPARSELUN}, /* EMC CLARiiON, storage on LUN 0 */
         {"DGC", "DISK", NULL, BLIST_SPARSELUN}, /* EMC CLARiiON, no storage on LUN 0 */
         {"EMC",  "Invista", "*", BLIST_SPARSELUN | BLIST_LARGELUN},
-        {"EMC", "SYMMETRIX", NULL, BLIST_SPARSELUN | BLIST_LARGELUN | BLIST_REPORTLUN2},
+        {"EMC", "SYMMETRIX", NULL, BLIST_SPARSELUN | BLIST_LARGELUN |
+         BLIST_REPORTLUN2 | BLIST_RETRY_ITF},
         {"EMULEX", "MD21/S2     ESDI", NULL, BLIST_SINGLELUN},
         {"easyRAID", "16P", NULL, BLIST_NOREPORTLUN},
         {"easyRAID", "X6P", NULL, BLIST_NOREPORTLUN},
         {"easyRAID", "F8", NULL, BLIST_NOREPORTLUN},
         {"FSC", "CentricStor", "*", BLIST_SPARSELUN | BLIST_LARGELUN},
+        {"FUJITSU", "ETERNUS_DXM", "*", BLIST_RETRY_ASC_C1},
         {"Generic", "USB SD Reader", "1.00", BLIST_FORCELUN | BLIST_INQUIRY_36},
-        {"Generic", "USB Storage-SMC", "0180", BLIST_FORCELUN | BLIST_INQUIRY_36},
-        {"Generic", "USB Storage-SMC", "0207", BLIST_FORCELUN | BLIST_INQUIRY_36},
+        {"Generic", "USB Storage-SMC", NULL, BLIST_FORCELUN | BLIST_INQUIRY_36}, /* FW: 0180 and 0207 */
         {"HITACHI", "DF400", "*", BLIST_REPORTLUN2},
         {"HITACHI", "DF500", "*", BLIST_REPORTLUN2},
         {"HITACHI", "DISK-SUBSYSTEM", "*", BLIST_REPORTLUN2},
@@ -361,8 +362,22 @@ int scsi_dev_info_list_add_keyed(int compatible, char *vendor, char *model,
         scsi_strcpy_devinfo("model", devinfo->model, sizeof(devinfo->model),
                             model, compatible);
 
-        if (strflags)
-                flags = (__force blist_flags_t)simple_strtoul(strflags, NULL, 0);
+        if (strflags) {
+                unsigned long long val;
+                int ret = kstrtoull(strflags, 0, &val);
+
+                if (ret != 0) {
+                        kfree(devinfo);
+                        return ret;
+                }
+                flags = (__force blist_flags_t)val;
+        }
+        if (flags & __BLIST_UNUSED_MASK) {
+                pr_err("scsi_devinfo (%s:%s): unsupported flags 0x%llx",
+                       vendor, model, flags & __BLIST_UNUSED_MASK);
+                kfree(devinfo);
+                return -EINVAL;
+        }
         devinfo->flags = flags;
         devinfo->compatible = compatible;
 
@@ -615,7 +630,7 @@ static int devinfo_seq_show(struct seq_file *m, void *v)
             devinfo_table->name)
                 seq_printf(m, "[%s]:\n", devinfo_table->name);
 
-        seq_printf(m, "'%.8s' '%.16s' 0x%x\n",
+        seq_printf(m, "'%.8s' '%.16s' 0x%llx\n",
                    devinfo->vendor, devinfo->model, devinfo->flags);
         return 0;
 }
@@ -734,9 +749,9 @@ MODULE_PARM_DESC(dev_flags,
          " list entries for vendor and model with an integer value of flags"
          " to the scsi device info list");
 
-module_param_named(default_dev_flags, scsi_default_dev_flags, int, S_IRUGO|S_IWUSR);
+module_param_named(default_dev_flags, scsi_default_dev_flags, ullong, 0644);
 MODULE_PARM_DESC(default_dev_flags,
-                 "scsi default device flag integer value");
+                 "scsi default device flag uint64_t value");
 
 /**
  * scsi_exit_devinfo - remove /proc/scsi/device_info & the scsi_dev_info_list
diff --git a/drivers/scsi/scsi_dh.c b/drivers/scsi/scsi_dh.c
index 188f30572aa1..5a58cbf3a75d 100644
--- a/drivers/scsi/scsi_dh.c
+++ b/drivers/scsi/scsi_dh.c
@@ -58,7 +58,10 @@ static const struct scsi_dh_blist scsi_dh_blist[] = {
         {"IBM", "3526",                 "rdac", },
         {"IBM", "3542",                 "rdac", },
         {"IBM", "3552",                 "rdac", },
-        {"SGI", "TP9",                  "rdac", },
+        {"SGI", "TP9300",               "rdac", },
+        {"SGI", "TP9400",               "rdac", },
+        {"SGI", "TP9500",               "rdac", },
+        {"SGI", "TP9700",               "rdac", },
         {"SGI", "IS",                   "rdac", },
         {"STK", "OPENstorage",          "rdac", },
         {"STK", "FLEXLINE 380",         "rdac", },
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index 9c02ba2e7ef3..8932ae81a15a 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -38,6 +38,7 @@
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_ioctl.h>
 #include <scsi/scsi_dh.h>
+#include <scsi/scsi_devinfo.h>
 #include <scsi/sg.h>
 
 #include "scsi_priv.h"
@@ -525,6 +526,12 @@ int scsi_check_sense(struct scsi_cmnd *scmd)
                 if (sshdr.asc == 0x10) /* DIF */
                         return SUCCESS;
 
+                if (sshdr.asc == 0x44 && sdev->sdev_bflags & BLIST_RETRY_ITF)
+                        return ADD_TO_MLQUEUE;
+                if (sshdr.asc == 0xc1 && sshdr.ascq == 0x01 &&
+                    sdev->sdev_bflags & BLIST_RETRY_ASC_C1)
+                        return ADD_TO_MLQUEUE;
+
                 return NEEDS_RETRY;
         case NOT_READY:
         case UNIT_ATTENTION:
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index fb38aeff9dbd..41e9ac9fc138 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -985,6 +985,10 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
                                 case 0x08: /* Long write in progress */
                                 case 0x09: /* self test in progress */
                                 case 0x14: /* space allocation in progress */
+                                case 0x1a: /* start stop unit in progress */
+                                case 0x1b: /* sanitize in progress */
+                                case 0x1d: /* configuration in progress */
+                                case 0x24: /* depopulation in progress */
                                         action = ACTION_DELAYED_RETRY;
                                         break;
                                 default:
diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c
index 1e36c9a9ad17..7943b762c12d 100644
--- a/drivers/scsi/scsi_sysfs.c
+++ b/drivers/scsi/scsi_sysfs.c
@@ -968,7 +968,7 @@ sdev_show_wwid(struct device *dev, struct device_attribute *attr,
 static DEVICE_ATTR(wwid, S_IRUGO, sdev_show_wwid, NULL);
 
 #define BLIST_FLAG_NAME(name)                                   \
-        [ilog2((__force unsigned int)BLIST_##name)] = #name
+        [const_ilog2((__force __u64)BLIST_##name)] = #name
 static const char *const sdev_bflags_name[] = {
 #include "scsi_devinfo_tbl.c"
 };
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index e2953b416746..0cd16e80b019 100644
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -213,10 +213,6 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy)
                 to_sas_host_attrs(shost)->q = q;
         }
 
-        /*
-         * by default assume old behaviour and bounce for any highmem page
-         */
-        blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH);
         blk_queue_flag_set(QUEUE_FLAG_BIDI, q);
         return 0;
 }
diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h
index 0d663b5e45bb..392c7d078ae3 100644
--- a/drivers/scsi/sd.h
+++ b/drivers/scsi/sd.h
@@ -74,12 +74,12 @@ struct scsi_disk {
         struct gendisk  *disk;
         struct opal_dev *opal_dev;
 #ifdef CONFIG_BLK_DEV_ZONED
-        unsigned int    nr_zones;
-        unsigned int    zone_blocks;
-        unsigned int    zone_shift;
-        unsigned int    zones_optimal_open;
-        unsigned int    zones_optimal_nonseq;
-        unsigned int    zones_max_open;
+        u32             nr_zones;
+        u32             zone_blocks;
+        u32             zone_shift;
+        u32             zones_optimal_open;
+        u32             zones_optimal_nonseq;
+        u32             zones_max_open;
 #endif
         atomic_t        openers;
         sector_t        capacity;       /* size in logical blocks */
diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c
index 210407cd2341..323e3dc4bc59 100644
--- a/drivers/scsi/sd_zbc.c
+++ b/drivers/scsi/sd_zbc.c
@@ -299,16 +299,6 @@ void sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes,
         case REQ_OP_WRITE:
         case REQ_OP_WRITE_ZEROES:
         case REQ_OP_WRITE_SAME:
-
-                if (result &&
-                    sshdr->sense_key == ILLEGAL_REQUEST &&
-                    sshdr->asc == 0x21)
-                        /*
-                         * INVALID ADDRESS FOR WRITE error: It is unlikely that
-                         * retrying write requests failed with any kind of
-                         * alignement error will result in success. So don't.
-                         */
-                        cmd->allowed = 0;
                 break;
 
         case REQ_OP_ZONE_REPORT:
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index 6fc58e2c99d3..573763908562 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -1191,7 +1191,7 @@ sg_fasync(int fd, struct file *filp, int mode)
         return fasync_helper(fd, filp, mode, &sfp->async_qp);
 }
 
-static int
+static vm_fault_t
 sg_vma_fault(struct vm_fault *vmf)
 {
         struct vm_area_struct *vma = vmf->vma;
diff --git a/drivers/scsi/snic/snic_scsi.c b/drivers/scsi/snic/snic_scsi.c
index d8a376b7882d..d9b2e46424aa 100644
--- a/drivers/scsi/snic/snic_scsi.c
+++ b/drivers/scsi/snic/snic_scsi.c
@@ -47,10 +47,10 @@ static const char * const snic_req_state_str[] = {
         [SNIC_IOREQ_NOT_INITED] = "SNIC_IOREQ_NOT_INITED",
         [SNIC_IOREQ_PENDING]    = "SNIC_IOREQ_PENDING",
         [SNIC_IOREQ_ABTS_PENDING] = "SNIC_IOREQ_ABTS_PENDING",
-        [SNIC_IOREQ_ABTS_COMPLETE] = "SNIC_IOREQ_ABTS_COMPELTE",
+        [SNIC_IOREQ_ABTS_COMPLETE] = "SNIC_IOREQ_ABTS_COMPLETE",
         [SNIC_IOREQ_LR_PENDING] = "SNIC_IOREQ_LR_PENDING",
-        [SNIC_IOREQ_LR_COMPLETE] = "SNIC_IOREQ_LR_COMPELTE",
-        [SNIC_IOREQ_COMPLETE]   = "SNIC_IOREQ_CMD_COMPELTE",
+        [SNIC_IOREQ_LR_COMPLETE] = "SNIC_IOREQ_LR_COMPLETE",
+        [SNIC_IOREQ_COMPLETE]   = "SNIC_IOREQ_CMD_COMPLETE",
 };
 
 /* snic cmd status strings */
diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c
index a427ce9497be..c9e27e752c25 100644
--- a/drivers/scsi/st.c
+++ b/drivers/scsi/st.c
@@ -3878,7 +3878,7 @@ static struct st_buffer *new_tape_buffer(int need_dma, int max_sg)
 {
         struct st_buffer *tb;
 
-        tb = kzalloc(sizeof(struct st_buffer), GFP_ATOMIC);
+        tb = kzalloc(sizeof(struct st_buffer), GFP_KERNEL);
         if (!tb) {
                 printk(KERN_NOTICE "st: Can't allocate new tape buffer.\n");
                 return NULL;
@@ -3889,7 +3889,7 @@ static struct st_buffer *new_tape_buffer(int need_dma, int max_sg)
         tb->buffer_size = 0;
 
         tb->reserved_pages = kzalloc(max_sg * sizeof(struct page *),
-                                     GFP_ATOMIC);
+                                     GFP_KERNEL);
         if (!tb->reserved_pages) {
                 kfree(tb);
                 return NULL;
@@ -4290,7 +4290,7 @@ static int st_probe(struct device *dev)
                 goto out_buffer_free;
         }
 
-        tpnt = kzalloc(sizeof(struct scsi_tape), GFP_ATOMIC);
+        tpnt = kzalloc(sizeof(struct scsi_tape), GFP_KERNEL);
         if (tpnt == NULL) {
                 sdev_printk(KERN_ERR, SDp,
                             "st: Can't allocate device descriptor.\n");
diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c
index a2ec0bc9e9fa..33a4a4dad324 100644
--- a/drivers/scsi/storvsc_drv.c
+++ b/drivers/scsi/storvsc_drv.c
@@ -395,6 +395,12 @@ MODULE_PARM_DESC(storvsc_ringbuffer_size, "Ring buffer size (bytes)");
 
 module_param(storvsc_vcpus_per_sub_channel, int, S_IRUGO);
 MODULE_PARM_DESC(storvsc_vcpus_per_sub_channel, "Ratio of VCPUs to subchannels");
+
+static int ring_avail_percent_lowater = 10;
+module_param(ring_avail_percent_lowater, int, S_IRUGO);
+MODULE_PARM_DESC(ring_avail_percent_lowater,
+                "Select a channel if available ring size > this in percent");
+
 /*
  * Timeout in seconds for all devices managed by this driver.
  */
@@ -1241,7 +1247,7 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device,
 {
         u16 slot = 0;
         u16 hash_qnum;
-        struct cpumask alloced_mask;
+        const struct cpumask *node_mask;
         int num_channels, tgt_cpu;
 
         if (stor_device->num_sc == 0)
@@ -1257,10 +1263,13 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device,
          * III. Mapping is persistent.
          */
 
-        cpumask_and(&alloced_mask, &stor_device->alloced_cpus,
-                    cpumask_of_node(cpu_to_node(q_num)));
+        node_mask = cpumask_of_node(cpu_to_node(q_num));
 
-        num_channels = cpumask_weight(&alloced_mask);
+        num_channels = 0;
+        for_each_cpu(tgt_cpu, &stor_device->alloced_cpus) {
+                if (cpumask_test_cpu(tgt_cpu, node_mask))
+                        num_channels++;
+        }
         if (num_channels == 0)
                 return stor_device->device->channel;
 
@@ -1268,7 +1277,9 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device,
         while (hash_qnum >= num_channels)
                 hash_qnum -= num_channels;
 
-        for_each_cpu(tgt_cpu, &alloced_mask) {
+        for_each_cpu(tgt_cpu, &stor_device->alloced_cpus) {
+                if (!cpumask_test_cpu(tgt_cpu, node_mask))
+                        continue;
                 if (slot == hash_qnum)
                         break;
                 slot++;
@@ -1285,9 +1296,9 @@ static int storvsc_do_io(struct hv_device *device,
 {
         struct storvsc_device *stor_device;
         struct vstor_packet *vstor_packet;
-        struct vmbus_channel *outgoing_channel;
+        struct vmbus_channel *outgoing_channel, *channel;
         int ret = 0;
-        struct cpumask alloced_mask;
+        const struct cpumask *node_mask;
         int tgt_cpu;
 
         vstor_packet = &request->vstor_packet;
@@ -1301,22 +1312,52 @@ static int storvsc_do_io(struct hv_device *device,
         /*
          * Select an an appropriate channel to send the request out.
          */
-
         if (stor_device->stor_chns[q_num] != NULL) {
                 outgoing_channel = stor_device->stor_chns[q_num];
-                if (outgoing_channel->target_cpu == smp_processor_id()) {
+                if (outgoing_channel->target_cpu == q_num) {
                         /*
                          * Ideally, we want to pick a different channel if
                          * available on the same NUMA node.
                          */
-                        cpumask_and(&alloced_mask, &stor_device->alloced_cpus,
-                                    cpumask_of_node(cpu_to_node(q_num)));
-                        for_each_cpu_wrap(tgt_cpu, &alloced_mask,
-                                        outgoing_channel->target_cpu + 1) {
-                                if (tgt_cpu != outgoing_channel->target_cpu) {
-                                        outgoing_channel =
-                                        stor_device->stor_chns[tgt_cpu];
-                                        break;
+                        node_mask = cpumask_of_node(cpu_to_node(q_num));
+                        for_each_cpu_wrap(tgt_cpu,
+                                 &stor_device->alloced_cpus, q_num + 1) {
+                                if (!cpumask_test_cpu(tgt_cpu, node_mask))
+                                        continue;
+                                if (tgt_cpu == q_num)
+                                        continue;
+                                channel = stor_device->stor_chns[tgt_cpu];
+                                if (hv_get_avail_to_write_percent(
+                                                        &channel->outbound)
+                                                > ring_avail_percent_lowater) {
+                                        outgoing_channel = channel;
+                                        goto found_channel;
+                                }
+                        }
+
+                        /*
+                         * All the other channels on the same NUMA node are
+                         * busy. Try to use the channel on the current CPU
+                         */
+                        if (hv_get_avail_to_write_percent(
+                                                &outgoing_channel->outbound)
+                                        > ring_avail_percent_lowater)
+                                goto found_channel;
+
+                        /*
+                         * If we reach here, all the channels on the current
+                         * NUMA node are busy. Try to find a channel in
+                         * other NUMA nodes
+                         */
+                        for_each_cpu(tgt_cpu, &stor_device->alloced_cpus) {
+                                if (cpumask_test_cpu(tgt_cpu, node_mask))
+                                        continue;
+                                channel = stor_device->stor_chns[tgt_cpu];
+                                if (hv_get_avail_to_write_percent(
+                                                        &channel->outbound)
+                                                > ring_avail_percent_lowater) {
+                                        outgoing_channel = channel;
+                                        goto found_channel;
                                 }
                         }
                 }
@@ -1324,7 +1365,7 @@ static int storvsc_do_io(struct hv_device *device,
                 outgoing_channel = get_og_chn(stor_device, q_num);
         }
 
-
+found_channel:
         vstor_packet->flags |= REQUEST_COMPLETION_FLAG;
 
         vstor_packet->vm_srb.length = (sizeof(struct vmscsi_request) -
@@ -1382,9 +1423,6 @@ static int storvsc_device_alloc(struct scsi_device *sdevice)
 
 static int storvsc_device_configure(struct scsi_device *sdevice)
 {
-
-        blk_queue_bounce_limit(sdevice->request_queue, BLK_BOUNCE_ANY);
-
         blk_queue_rq_timeout(sdevice->request_queue, (storvsc_timeout * HZ));
 
         /* Ensure there are no gaps in presented sgls */
@@ -1732,8 +1770,9 @@ static int storvsc_probe(struct hv_device *device,
                                 (num_cpus - 1) / storvsc_vcpus_per_sub_channel;
         }
 
-        scsi_driver.can_queue = (max_outstanding_req_per_channel *
-                                 (max_sub_channels + 1));
+        scsi_driver.can_queue = max_outstanding_req_per_channel *
+                                (max_sub_channels + 1) *
+                                (100 - ring_avail_percent_lowater) / 100;
 
         host = scsi_host_alloc(&scsi_driver,
                                sizeof(struct hv_host_device));
diff --git a/drivers/scsi/ufs/ufs-qcom.c b/drivers/scsi/ufs/ufs-qcom.c
index 2b38db2eeafa..221820a7c78b 100644
--- a/drivers/scsi/ufs/ufs-qcom.c
+++ b/drivers/scsi/ufs/ufs-qcom.c
@@ -1098,7 +1098,7 @@ static void ufs_qcom_advertise_quirks(struct ufs_hba *hba)
                 hba->quirks |= UFSHCD_QUIRK_BROKEN_LCC;
         }
 
-        if (host->hw_ver.major >= 0x2) {
+        if (host->hw_ver.major == 0x2) {
                 hba->quirks |= UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION;
 
                 if (!ufs_qcom_cap_qunipro(host))
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
index d0a1674915a1..3a811c5f70ba 100644
--- a/drivers/scsi/ufs/ufshcd.c
+++ b/drivers/scsi/ufs/ufshcd.c
@@ -233,8 +233,6 @@ static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
 static irqreturn_t ufshcd_intr(int irq, void *__hba);
-static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
-                struct ufs_pa_layer_attr *desired_pwr_mode);
 static int ufshcd_change_power_mode(struct ufs_hba *hba,
                              struct ufs_pa_layer_attr *pwr_mode);
 static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
@@ -266,6 +264,18 @@ static inline void ufshcd_disable_irq(struct ufs_hba *hba)
         }
 }
 
+static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
+{
+        if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
+                scsi_unblock_requests(hba->host);
+}
+
+static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
+{
+        if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
+                scsi_block_requests(hba->host);
+}
+
 /* replace non-printable or non-ASCII characters with spaces */
 static inline void ufshcd_remove_non_printable(char *val)
 {
@@ -675,7 +685,24 @@ static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot)
  */
 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
 {
-        ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
+        if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
+                ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
+        else
+                ufshcd_writel(hba, ~(1 << pos),
+                                REG_UTP_TRANSFER_REQ_LIST_CLEAR);
+}
+
+/**
+ * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
+ * @hba: per adapter instance
+ * @pos: position of the bit to be cleared
+ */
+static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
+{
+        if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
+                ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
+        else
+                ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 }
 
 /**
@@ -1091,12 +1118,12 @@ static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
          * make sure that there are no outstanding requests when
          * clock scaling is in progress
          */
-        scsi_block_requests(hba->host);
+        ufshcd_scsi_block_requests(hba);
         down_write(&hba->clk_scaling_lock);
         if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
                 ret = -EBUSY;
                 up_write(&hba->clk_scaling_lock);
-                scsi_unblock_requests(hba->host);
+                ufshcd_scsi_unblock_requests(hba);
         }
 
         return ret;
@@ -1105,7 +1132,7 @@ static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
 {
         up_write(&hba->clk_scaling_lock);
-        scsi_unblock_requests(hba->host);
+        ufshcd_scsi_unblock_requests(hba);
 }
 
 /**
@@ -1200,16 +1227,13 @@ static int ufshcd_devfreq_target(struct device *dev,
         struct ufs_hba *hba = dev_get_drvdata(dev);
         ktime_t start;
         bool scale_up, sched_clk_scaling_suspend_work = false;
+        struct list_head *clk_list = &hba->clk_list_head;
+        struct ufs_clk_info *clki;
         unsigned long irq_flags;
 
         if (!ufshcd_is_clkscaling_supported(hba))
                 return -EINVAL;
 
-        if ((*freq > 0) && (*freq < UINT_MAX)) {
-                dev_err(hba->dev, "%s: invalid freq = %lu\n", __func__, *freq);
-                return -EINVAL;
-        }
-
         spin_lock_irqsave(hba->host->host_lock, irq_flags);
         if (ufshcd_eh_in_progress(hba)) {
                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
@@ -1219,7 +1243,13 @@ static int ufshcd_devfreq_target(struct device *dev,
         if (!hba->clk_scaling.active_reqs)
                 sched_clk_scaling_suspend_work = true;
 
-        scale_up = (*freq == UINT_MAX) ? true : false;
+        if (list_empty(clk_list)) {
+                spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
+                goto out;
+        }
+
+        clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
+        scale_up = (*freq == clki->max_freq) ? true : false;
         if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
                 ret = 0;
@@ -1287,6 +1317,55 @@ static struct devfreq_dev_profile ufs_devfreq_profile = {
         .get_dev_status = ufshcd_devfreq_get_dev_status,
 };
 
+static int ufshcd_devfreq_init(struct ufs_hba *hba)
+{
+        struct list_head *clk_list = &hba->clk_list_head;
+        struct ufs_clk_info *clki;
+        struct devfreq *devfreq;
+        int ret;
+
+        /* Skip devfreq if we don't have any clocks in the list */
+        if (list_empty(clk_list))
+                return 0;
+
+        clki = list_first_entry(clk_list, struct ufs_clk_info, list);
+        dev_pm_opp_add(hba->dev, clki->min_freq, 0);
+        dev_pm_opp_add(hba->dev, clki->max_freq, 0);
+
+        devfreq = devfreq_add_device(hba->dev,
+                        &ufs_devfreq_profile,
+                        DEVFREQ_GOV_SIMPLE_ONDEMAND,
+                        NULL);
+        if (IS_ERR(devfreq)) {
+                ret = PTR_ERR(devfreq);
+                dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
+
+                dev_pm_opp_remove(hba->dev, clki->min_freq);
+                dev_pm_opp_remove(hba->dev, clki->max_freq);
+                return ret;
+        }
+
+        hba->devfreq = devfreq;
+
+        return 0;
+}
+
+static void ufshcd_devfreq_remove(struct ufs_hba *hba)
+{
+        struct list_head *clk_list = &hba->clk_list_head;
+        struct ufs_clk_info *clki;
+
+        if (!hba->devfreq)
+                return;
+
+        devfreq_remove_device(hba->devfreq);
+        hba->devfreq = NULL;
+
+        clki = list_first_entry(clk_list, struct ufs_clk_info, list);
+        dev_pm_opp_remove(hba->dev, clki->min_freq);
+        dev_pm_opp_remove(hba->dev, clki->max_freq);
+}
+
 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
 {
         unsigned long flags;
@@ -1425,7 +1504,7 @@ static void ufshcd_ungate_work(struct work_struct *work)
                 hba->clk_gating.is_suspended = false;
         }
 unblock_reqs:
-        scsi_unblock_requests(hba->host);
+        ufshcd_scsi_unblock_requests(hba);
 }
 
 /**
@@ -1481,11 +1560,12 @@ start:
                  * work and to enable clocks.
                  */
         case CLKS_OFF:
-                scsi_block_requests(hba->host);
+                ufshcd_scsi_block_requests(hba);
                 hba->clk_gating.state = REQ_CLKS_ON;
                 trace_ufshcd_clk_gating(dev_name(hba->dev),
                                         hba->clk_gating.state);
-                schedule_work(&hba->clk_gating.ungate_work);
+                queue_work(hba->clk_gating.clk_gating_workq,
+                           &hba->clk_gating.ungate_work);
                 /*
                  * fall through to check if we should wait for this
                  * work to be done or not.
@@ -1671,6 +1751,8 @@ out:
 
 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
 {
+        char wq_name[sizeof("ufs_clk_gating_00")];
+
         if (!ufshcd_is_clkgating_allowed(hba))
                 return;
 
@@ -1678,6 +1760,11 @@ static void ufshcd_init_clk_gating(struct ufs_hba *hba)
         INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
         INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
 
+        snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
+                 hba->host->host_no);
+        hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
+                                                           WQ_MEM_RECLAIM);
+
         hba->clk_gating.is_enabled = true;
 
         hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
@@ -1705,6 +1792,7 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
         device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
         cancel_work_sync(&hba->clk_gating.ungate_work);
         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
+        destroy_workqueue(hba->clk_gating.clk_gating_workq);
 }
 
 /* Must be called with host lock acquired */
@@ -3383,6 +3471,52 @@ static int ufshcd_dme_link_startup(struct ufs_hba *hba)
                         "dme-link-startup: error code %d\n", ret);
         return ret;
 }
+/**
+ * ufshcd_dme_reset - UIC command for DME_RESET
+ * @hba: per adapter instance
+ *
+ * DME_RESET command is issued in order to reset UniPro stack.
+ * This function now deal with cold reset.
+ *
+ * Returns 0 on success, non-zero value on failure
+ */
+static int ufshcd_dme_reset(struct ufs_hba *hba)
+{
+        struct uic_command uic_cmd = {0};
+        int ret;
+
+        uic_cmd.command = UIC_CMD_DME_RESET;
+
+        ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
+        if (ret)
+                dev_err(hba->dev,
+                        "dme-reset: error code %d\n", ret);
+
+        return ret;
+}
+
+/**
+ * ufshcd_dme_enable - UIC command for DME_ENABLE
+ * @hba: per adapter instance
+ *
+ * DME_ENABLE command is issued in order to enable UniPro stack.
+ *
+ * Returns 0 on success, non-zero value on failure
+ */
+static int ufshcd_dme_enable(struct ufs_hba *hba)
+{
+        struct uic_command uic_cmd = {0};
+        int ret;
+
+        uic_cmd.command = UIC_CMD_DME_ENABLE;
+
+        ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
+        if (ret)
+                dev_err(hba->dev,
+                        "dme-reset: error code %d\n", ret);
+
+        return ret;
+}
 
 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
 {
@@ -3906,7 +4040,7 @@ static int ufshcd_change_power_mode(struct ufs_hba *hba,
  * @hba: per-adapter instance
  * @desired_pwr_mode: desired power configuration
  */
-static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
+int ufshcd_config_pwr_mode(struct ufs_hba *hba,
                 struct ufs_pa_layer_attr *desired_pwr_mode)
 {
         struct ufs_pa_layer_attr final_params = { 0 };
@@ -3924,6 +4058,7 @@ static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
 
         return ret;
 }
+EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
 
 /**
  * ufshcd_complete_dev_init() - checks device readiness
@@ -4041,7 +4176,7 @@ static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
 }
 
 /**
- * ufshcd_hba_enable - initialize the controller
+ * ufshcd_hba_execute_hce - initialize the controller
  * @hba: per adapter instance
  *
  * The controller resets itself and controller firmware initialization
@@ -4050,7 +4185,7 @@ static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
  *
  * Returns 0 on success, non-zero value on failure
  */
-static int ufshcd_hba_enable(struct ufs_hba *hba)
+static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
 {
         int retry;
 
@@ -4105,6 +4240,31 @@ static int ufshcd_hba_enable(struct ufs_hba *hba)
         return 0;
 }
 
+static int ufshcd_hba_enable(struct ufs_hba *hba)
+{
+        int ret;
+
+        if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
+                ufshcd_set_link_off(hba);
+                ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
+
+                /* enable UIC related interrupts */
+                ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
+                ret = ufshcd_dme_reset(hba);
+                if (!ret) {
+                        ret = ufshcd_dme_enable(hba);
+                        if (!ret)
+                                ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
+                        if (ret)
+                                dev_err(hba->dev,
+                                        "Host controller enable failed with non-hce\n");
+                }
+        } else {
+                ret = ufshcd_hba_execute_hce(hba);
+        }
+
+        return ret;
+}
 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
 {
         int tx_lanes, i, err = 0;
@@ -4678,7 +4838,8 @@ static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
          * false interrupt if device completes another request after resetting
          * aggregation and before reading the DB.
          */
-        if (ufshcd_is_intr_aggr_allowed(hba))
+        if (ufshcd_is_intr_aggr_allowed(hba) &&
+            !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
                 ufshcd_reset_intr_aggr(hba);
 
         tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
@@ -4969,6 +5130,7 @@ static void ufshcd_exception_event_handler(struct work_struct *work)
         hba = container_of(work, struct ufs_hba, eeh_work);
 
         pm_runtime_get_sync(hba->dev);
+        scsi_block_requests(hba->host);
         err = ufshcd_get_ee_status(hba, &status);
         if (err) {
                 dev_err(hba->dev, "%s: failed to get exception status %d\n",
@@ -4982,6 +5144,7 @@ static void ufshcd_exception_event_handler(struct work_struct *work)
                 ufshcd_bkops_exception_event_handler(hba);
 
 out:
+        scsi_unblock_requests(hba->host);
         pm_runtime_put_sync(hba->dev);
         return;
 }
@@ -5192,7 +5355,7 @@ skip_err_handling:
 
 out:
         spin_unlock_irqrestore(hba->host->host_lock, flags);
-        scsi_unblock_requests(hba->host);
+        ufshcd_scsi_unblock_requests(hba);
         ufshcd_release(hba);
         pm_runtime_put_sync(hba->dev);
 }
@@ -5294,7 +5457,7 @@ static void ufshcd_check_errors(struct ufs_hba *hba)
                 /* handle fatal errors only when link is functional */
                 if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
                         /* block commands from scsi mid-layer */
-                        scsi_block_requests(hba->host);
+                        ufshcd_scsi_block_requests(hba);
 
                         hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED;
 
@@ -5371,19 +5534,30 @@ static irqreturn_t ufshcd_intr(int irq, void *__hba)
         u32 intr_status, enabled_intr_status;
         irqreturn_t retval = IRQ_NONE;
         struct ufs_hba *hba = __hba;
+        int retries = hba->nutrs;
 
         spin_lock(hba->host->host_lock);
         intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
-        enabled_intr_status =
-                intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
 
-        if (intr_status)
-                ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
+        /*
+         * There could be max of hba->nutrs reqs in flight and in worst case
+         * if the reqs get finished 1 by 1 after the interrupt status is
+         * read, make sure we handle them by checking the interrupt status
+         * again in a loop until we process all of the reqs before returning.
+         */
+        do {
+                enabled_intr_status =
+                        intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
+                if (intr_status)
+                        ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
+                if (enabled_intr_status) {
+                        ufshcd_sl_intr(hba, enabled_intr_status);
+                        retval = IRQ_HANDLED;
+                }
+
+                intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
+        } while (intr_status && --retries);
 
-        if (enabled_intr_status) {
-                ufshcd_sl_intr(hba, enabled_intr_status);
-                retval = IRQ_HANDLED;
-        }
         spin_unlock(hba->host->host_lock);
         return retval;
 }
@@ -5398,7 +5572,7 @@ static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
                 goto out;
 
         spin_lock_irqsave(hba->host->host_lock, flags);
-        ufshcd_writel(hba, ~(1 << tag), REG_UTP_TASK_REQ_LIST_CLEAR);
+        ufshcd_utmrl_clear(hba, tag);
         spin_unlock_irqrestore(hba->host->host_lock, flags);
 
         /* poll for max. 1 sec to clear door bell register by h/w */
@@ -5958,14 +6132,18 @@ static void ufshcd_init_icc_levels(struct ufs_hba *hba)
 {
         int ret;
         int buff_len = hba->desc_size.pwr_desc;
-        u8 desc_buf[hba->desc_size.pwr_desc];
+        u8 *desc_buf;
+
+        desc_buf = kmalloc(buff_len, GFP_KERNEL);
+        if (!desc_buf)
+                return;
 
         ret = ufshcd_read_power_desc(hba, desc_buf, buff_len);
         if (ret) {
                 dev_err(hba->dev,
                         "%s: Failed reading power descriptor.len = %d ret = %d",
                         __func__, buff_len, ret);
-                return;
+                goto out;
         }
 
         hba->init_prefetch_data.icc_level =
@@ -5983,6 +6161,8 @@ static void ufshcd_init_icc_levels(struct ufs_hba *hba)
                         "%s: Failed configuring bActiveICCLevel = %d ret = %d",
                         __func__, hba->init_prefetch_data.icc_level , ret);
 
+out:
+        kfree(desc_buf);
 }
 
 /**
@@ -6052,9 +6232,17 @@ static int ufs_get_device_desc(struct ufs_hba *hba,
                                struct ufs_dev_desc *dev_desc)
 {
         int err;
+        size_t buff_len;
         u8 model_index;
-        u8 str_desc_buf[QUERY_DESC_MAX_SIZE + 1] = {0};
-        u8 desc_buf[hba->desc_size.dev_desc];
+        u8 *desc_buf;
+
+        buff_len = max_t(size_t, hba->desc_size.dev_desc,
+                         QUERY_DESC_MAX_SIZE + 1);
+        desc_buf = kmalloc(buff_len, GFP_KERNEL);
+        if (!desc_buf) {
+                err = -ENOMEM;
+                goto out;
+        }
 
         err = ufshcd_read_device_desc(hba, desc_buf, hba->desc_size.dev_desc);
         if (err) {
@@ -6072,7 +6260,10 @@ static int ufs_get_device_desc(struct ufs_hba *hba,
 
         model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
 
-        err = ufshcd_read_string_desc(hba, model_index, str_desc_buf,
+        /* Zero-pad entire buffer for string termination. */
+        memset(desc_buf, 0, buff_len);
+
+        err = ufshcd_read_string_desc(hba, model_index, desc_buf,
                                       QUERY_DESC_MAX_SIZE, true/*ASCII*/);
         if (err) {
                 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
@@ -6080,15 +6271,16 @@ static int ufs_get_device_desc(struct ufs_hba *hba,
                 goto out;
         }
 
-        str_desc_buf[QUERY_DESC_MAX_SIZE] = '\0';
-        strlcpy(dev_desc->model, (str_desc_buf + QUERY_DESC_HDR_SIZE),
-                min_t(u8, str_desc_buf[QUERY_DESC_LENGTH_OFFSET],
+        desc_buf[QUERY_DESC_MAX_SIZE] = '\0';
+        strlcpy(dev_desc->model, (desc_buf + QUERY_DESC_HDR_SIZE),
+                min_t(u8, desc_buf[QUERY_DESC_LENGTH_OFFSET],
                       MAX_MODEL_LEN));
 
         /* Null terminate the model string */
         dev_desc->model[MAX_MODEL_LEN] = '\0';
 
 out:
+        kfree(desc_buf);
         return err;
 }
 
@@ -6439,16 +6631,9 @@ static int ufshcd_probe_hba(struct ufs_hba *hba)
                                 sizeof(struct ufs_pa_layer_attr));
                         hba->clk_scaling.saved_pwr_info.is_valid = true;
                         if (!hba->devfreq) {
-                                hba->devfreq = devm_devfreq_add_device(hba->dev,
-                                                        &ufs_devfreq_profile,
-                                                        "simple_ondemand",
-                                                        NULL);
-                                if (IS_ERR(hba->devfreq)) {
-                                        ret = PTR_ERR(hba->devfreq);
-                                        dev_err(hba->dev, "Unable to register with devfreq %d\n",
-                                                        ret);
+                                ret = ufshcd_devfreq_init(hba);
+                                if (ret)
                                         goto out;
-                                }
                         }
                         hba->clk_scaling.is_allowed = true;
                 }
@@ -6799,9 +6984,16 @@ static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
         if (list_empty(head))
                 goto out;
 
-        ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
-        if (ret)
-                return ret;
+        /*
+         * vendor specific setup_clocks ops may depend on clocks managed by
+         * this standard driver hence call the vendor specific setup_clocks
+         * before disabling the clocks managed here.
+         */
+        if (!on) {
+                ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
+                if (ret)
+                        return ret;
+        }
 
         list_for_each_entry(clki, head, list) {
                 if (!IS_ERR_OR_NULL(clki->clk)) {
@@ -6825,9 +7017,16 @@ static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
                 }
         }
 
-        ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
-        if (ret)
-                return ret;
+        /*
+         * vendor specific setup_clocks ops may depend on clocks managed by
+         * this standard driver hence call the vendor specific setup_clocks
+         * after enabling the clocks managed here.
+         */
+        if (on) {
+                ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
+                if (ret)
+                        return ret;
+        }
 
 out:
         if (ret) {
@@ -6992,6 +7191,7 @@ static void ufshcd_hba_exit(struct ufs_hba *hba)
                         if (hba->devfreq)
                                 ufshcd_suspend_clkscaling(hba);
                         destroy_workqueue(hba->clk_scaling.workq);
+                        ufshcd_devfreq_remove(hba);
                 }
                 ufshcd_setup_clocks(hba, false);
                 ufshcd_setup_hba_vreg(hba, false);
@@ -7904,7 +8104,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
 
         /* Hold auto suspend until async scan completes */
         pm_runtime_get_sync(dev);
-
+        atomic_set(&hba->scsi_block_reqs_cnt, 0);
         /*
          * We are assuming that device wasn't put in sleep/power-down
          * state exclusively during the boot stage before kernel.
diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h
index 8110dcd04d22..f51758f1e5cc 100644
--- a/drivers/scsi/ufs/ufshcd.h
+++ b/drivers/scsi/ufs/ufshcd.h
@@ -362,6 +362,7 @@ struct ufs_clk_gating {
         struct device_attribute enable_attr;
         bool is_enabled;
         int active_reqs;
+        struct workqueue_struct *clk_gating_workq;
 };
 
 struct ufs_saved_pwr_info {
@@ -499,6 +500,7 @@ struct ufs_stats {
  * @urgent_bkops_lvl: keeps track of urgent bkops level for device
  * @is_urgent_bkops_lvl_checked: keeps track if the urgent bkops level for
  *  device is known or not.
+ * @scsi_block_reqs_cnt: reference counting for scsi block requests
  */
 struct ufs_hba {
         void __iomem *mmio_base;
@@ -595,6 +597,22 @@ struct ufs_hba {
          */
         #define UFSHCD_QUIRK_PRDT_BYTE_GRAN                     0x80
 
+        /*
+         * Clear handling for transfer/task request list is just opposite.
+         */
+        #define UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR                0x100
+
+        /*
+         * This quirk needs to be enabled if host controller doesn't allow
+         * that the interrupt aggregation timer and counter are reset by s/w.
+         */
+        #define UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR               0x200
+
+        /*
+         * This quirks needs to be enabled if host controller cannot be
+         * enabled via HCE register.
+         */
+        #define UFSHCI_QUIRK_BROKEN_HCE                         0x400
         unsigned int quirks;    /* Deviations from standard UFSHCI spec. */
 
         /* Device deviations from standard UFS device spec. */
@@ -683,6 +701,7 @@ struct ufs_hba {
 
         struct rw_semaphore clk_scaling_lock;
         struct ufs_desc_size desc_size;
+        atomic_t scsi_block_reqs_cnt;
 };
 
 /* Returns true if clocks can be gated. Otherwise false */
@@ -789,6 +808,8 @@ extern int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
                                u8 attr_set, u32 mib_val, u8 peer);
 extern int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
                                u32 *mib_val, u8 peer);
+extern int ufshcd_config_pwr_mode(struct ufs_hba *hba,
+                        struct ufs_pa_layer_attr *desired_pwr_mode);
 
 /* UIC command interfaces for DME primitives */
 #define DME_LOCAL       0
diff --git a/drivers/scsi/wd719x.c b/drivers/scsi/wd719x.c
index 2ba2b7b47f41..974bfb3f30f4 100644
--- a/drivers/scsi/wd719x.c
+++ b/drivers/scsi/wd719x.c
@@ -978,18 +978,7 @@ static struct pci_driver wd719x_pci_driver = {
         .remove =       wd719x_pci_remove,
 };
 
-static int __init wd719x_init(void)
-{
-        return pci_register_driver(&wd719x_pci_driver);
-}
-
-static void __exit wd719x_exit(void)
-{
-        pci_unregister_driver(&wd719x_pci_driver);
-}
-
-module_init(wd719x_init);
-module_exit(wd719x_exit);
+module_pci_driver(wd719x_pci_driver);
 
 MODULE_DESCRIPTION("Western Digital WD7193/7197/7296 SCSI driver");
 MODULE_AUTHOR("Ondrej Zary, Aaron Dewell, Juergen Gaertner");
diff --git a/drivers/scsi/zorro_esp.c b/drivers/scsi/zorro_esp.c
new file mode 100644
index 000000000000..bb70882e6b56
--- /dev/null
+++ b/drivers/scsi/zorro_esp.c
@@ -0,0 +1,1172 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESP front-end for Amiga ZORRO SCSI systems.
+ *
+ * Copyright (C) 1996 Jesper Skov (jskov@cygnus.co.uk)
+ *
+ * Copyright (C) 2011,2018 Michael Schmitz (schmitz@debian.org) for
+ *               migration to ESP SCSI core
+ *
+ * Copyright (C) 2013 Tuomas Vainikka (tuomas.vainikka@aalto.fi) for
+ *               Blizzard 1230 DMA and probe function fixes
+ *
+ * Copyright (C) 2017 Finn Thain for PIO code from Mac ESP driver adapted here
+ */
+/*
+ * ZORRO bus code from:
+ */
+/*
+ * Detection routine for the NCR53c710 based Amiga SCSI Controllers for Linux.
+ *              Amiga MacroSystemUS WarpEngine SCSI controller.
+ *              Amiga Technologies/DKB A4091 SCSI controller.
+ *
+ * Written 1997 by Alan Hourihane <alanh@fairlite.demon.co.uk>
+ * plus modifications of the 53c7xx.c driver to support the Amiga.
+ *
+ * Rewritten to use 53c700.c by Kars de Jong <jongk@linux-m68k.org>
+ */
+
+#define pr_fmt(fmt)        KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/dma-mapping.h>
+#include <linux/scatterlist.h>
+#include <linux/delay.h>
+#include <linux/zorro.h>
+#include <linux/slab.h>
+
+#include <asm/page.h>
+#include <asm/pgtable.h>
+#include <asm/cacheflush.h>
+#include <asm/amigahw.h>
+#include <asm/amigaints.h>
+
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_transport_spi.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_tcq.h>
+
+#include "esp_scsi.h"
+
+MODULE_AUTHOR("Michael Schmitz <schmitz@debian.org>");
+MODULE_DESCRIPTION("Amiga Zorro NCR5C9x (ESP) driver");
+MODULE_LICENSE("GPL");
+
+/* per-board register layout definitions */
+
+/* Blizzard 1230 DMA interface */
+
+struct blz1230_dma_registers {
+        unsigned char dma_addr;         /* DMA address      [0x0000] */
+        unsigned char dmapad2[0x7fff];
+        unsigned char dma_latch;        /* DMA latch        [0x8000] */
+};
+
+/* Blizzard 1230II DMA interface */
+
+struct blz1230II_dma_registers {
+        unsigned char dma_addr;         /* DMA address      [0x0000] */
+        unsigned char dmapad2[0xf];
+        unsigned char dma_latch;        /* DMA latch        [0x0010] */
+};
+
+/* Blizzard 2060 DMA interface */
+
+struct blz2060_dma_registers {
+        unsigned char dma_led_ctrl;     /* DMA led control   [0x000] */
+        unsigned char dmapad1[0x0f];
+        unsigned char dma_addr0;        /* DMA address (MSB) [0x010] */
+        unsigned char dmapad2[0x03];
+        unsigned char dma_addr1;        /* DMA address       [0x014] */
+        unsigned char dmapad3[0x03];
+        unsigned char dma_addr2;        /* DMA address       [0x018] */
+        unsigned char dmapad4[0x03];
+        unsigned char dma_addr3;        /* DMA address (LSB) [0x01c] */
+};
+
+/* DMA control bits */
+#define DMA_WRITE 0x80000000
+
+/* Cyberstorm DMA interface */
+
+struct cyber_dma_registers {
+        unsigned char dma_addr0;        /* DMA address (MSB) [0x000] */
+        unsigned char dmapad1[1];
+        unsigned char dma_addr1;        /* DMA address       [0x002] */
+        unsigned char dmapad2[1];
+        unsigned char dma_addr2;        /* DMA address       [0x004] */
+        unsigned char dmapad3[1];
+        unsigned char dma_addr3;        /* DMA address (LSB) [0x006] */
+        unsigned char dmapad4[0x3fb];
+        unsigned char cond_reg;         /* DMA cond    (ro)  [0x402] */
+#define ctrl_reg  cond_reg              /* DMA control (wo)  [0x402] */
+};
+
+/* DMA control bits */
+#define CYBER_DMA_WRITE  0x40   /* DMA direction. 1 = write */
+#define CYBER_DMA_Z3     0x20   /* 16 (Z2) or 32 (CHIP/Z3) bit DMA transfer */
+
+/* DMA status bits */
+#define CYBER_DMA_HNDL_INTR 0x80        /* DMA IRQ pending? */
+
+/* The CyberStorm II DMA interface */
+struct cyberII_dma_registers {
+        unsigned char cond_reg;         /* DMA cond    (ro)  [0x000] */
+#define ctrl_reg  cond_reg              /* DMA control (wo)  [0x000] */
+        unsigned char dmapad4[0x3f];
+        unsigned char dma_addr0;        /* DMA address (MSB) [0x040] */
+        unsigned char dmapad1[3];
+        unsigned char dma_addr1;        /* DMA address       [0x044] */
+        unsigned char dmapad2[3];
+        unsigned char dma_addr2;        /* DMA address       [0x048] */
+        unsigned char dmapad3[3];
+        unsigned char dma_addr3;        /* DMA address (LSB) [0x04c] */
+};
+
+/* Fastlane DMA interface */
+
+struct fastlane_dma_registers {
+        unsigned char cond_reg;         /* DMA status  (ro) [0x0000] */
+#define ctrl_reg  cond_reg              /* DMA control (wo) [0x0000] */
+        char dmapad1[0x3f];
+        unsigned char clear_strobe;     /* DMA clear   (wo) [0x0040] */
+};
+
+/*
+ * The controller registers can be found in the Z2 config area at these
+ * offsets:
+ */
+#define FASTLANE_ESP_ADDR       0x1000001
+
+/* DMA status bits */
+#define FASTLANE_DMA_MINT       0x80
+#define FASTLANE_DMA_IACT       0x40
+#define FASTLANE_DMA_CREQ       0x20
+
+/* DMA control bits */
+#define FASTLANE_DMA_FCODE      0xa0
+#define FASTLANE_DMA_MASK       0xf3
+#define FASTLANE_DMA_WRITE      0x08    /* 1 = write */
+#define FASTLANE_DMA_ENABLE     0x04    /* Enable DMA */
+#define FASTLANE_DMA_EDI        0x02    /* Enable DMA IRQ ? */
+#define FASTLANE_DMA_ESI        0x01    /* Enable SCSI IRQ */
+
+/*
+ * private data used for driver
+ */
+struct zorro_esp_priv {
+        struct esp *esp;                /* our ESP instance - for Scsi_host* */
+        void __iomem *board_base;       /* virtual address (Zorro III board) */
+        int error;                      /* PIO error flag */
+        int zorro3;                     /* board is Zorro III */
+        unsigned char ctrl_data;        /* shadow copy of ctrl_reg */
+};
+
+/*
+ * On all implementations except for the Oktagon, padding between ESP
+ * registers is three bytes.
+ * On Oktagon, it is one byte - use a different accessor there.
+ *
+ * Oktagon needs PDMA - currently unsupported!
+ */
+
+static void zorro_esp_write8(struct esp *esp, u8 val, unsigned long reg)
+{
+        writeb(val, esp->regs + (reg * 4UL));
+}
+
+static u8 zorro_esp_read8(struct esp *esp, unsigned long reg)
+{
+        return readb(esp->regs + (reg * 4UL));
+}
+
+static dma_addr_t zorro_esp_map_single(struct esp *esp, void *buf,
+                                      size_t sz, int dir)
+{
+        return dma_map_single(esp->dev, buf, sz, dir);
+}
+
+static int zorro_esp_map_sg(struct esp *esp, struct scatterlist *sg,
+                                  int num_sg, int dir)
+{
+        return dma_map_sg(esp->dev, sg, num_sg, dir);
+}
+
+static void zorro_esp_unmap_single(struct esp *esp, dma_addr_t addr,
+                                  size_t sz, int dir)
+{
+        dma_unmap_single(esp->dev, addr, sz, dir);
+}
+
+static void zorro_esp_unmap_sg(struct esp *esp, struct scatterlist *sg,
+                              int num_sg, int dir)
+{
+        dma_unmap_sg(esp->dev, sg, num_sg, dir);
+}
+
+static int zorro_esp_irq_pending(struct esp *esp)
+{
+        /* check ESP status register; DMA has no status reg. */
+        if (zorro_esp_read8(esp, ESP_STATUS) & ESP_STAT_INTR)
+                return 1;
+
+        return 0;
+}
+
+static int cyber_esp_irq_pending(struct esp *esp)
+{
+        struct cyber_dma_registers __iomem *dregs = esp->dma_regs;
+        unsigned char dma_status = readb(&dregs->cond_reg);
+
+        /* It's important to check the DMA IRQ bit in the correct way! */
+        return ((zorro_esp_read8(esp, ESP_STATUS) & ESP_STAT_INTR) &&
+                (dma_status & CYBER_DMA_HNDL_INTR));
+}
+
+static int fastlane_esp_irq_pending(struct esp *esp)
+{
+        struct fastlane_dma_registers __iomem *dregs = esp->dma_regs;
+        unsigned char dma_status;
+
+        dma_status = readb(&dregs->cond_reg);
+
+        if (dma_status & FASTLANE_DMA_IACT)
+                return 0;       /* not our IRQ */
+
+        /* Return non-zero if ESP requested IRQ */
+        return (
+           (dma_status & FASTLANE_DMA_CREQ) &&
+           (!(dma_status & FASTLANE_DMA_MINT)) &&
+           (zorro_esp_read8(esp, ESP_STATUS) & ESP_STAT_INTR));
+}
+
+static u32 zorro_esp_dma_length_limit(struct esp *esp, u32 dma_addr,
+                                        u32 dma_len)
+{
+        return dma_len > 0xFFFFFF ? 0xFFFFFF : dma_len;
+}
+
+static void zorro_esp_reset_dma(struct esp *esp)
+{
+        /* nothing to do here */
+}
+
+static void zorro_esp_dma_drain(struct esp *esp)
+{
+        /* nothing to do here */
+}
+
+static void zorro_esp_dma_invalidate(struct esp *esp)
+{
+        /* nothing to do here */
+}
+
+static void fastlane_esp_dma_invalidate(struct esp *esp)
+{
+        struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+        struct fastlane_dma_registers __iomem *dregs = esp->dma_regs;
+        unsigned char *ctrl_data = &zep->ctrl_data;
+
+        *ctrl_data = (*ctrl_data & FASTLANE_DMA_MASK);
+        writeb(0, &dregs->clear_strobe);
+        z_writel(0, zep->board_base);
+}
+
+/*
+ * Programmed IO routines follow.
+ */
+
+static inline unsigned int zorro_esp_wait_for_fifo(struct esp *esp)
+{
+        int i = 500000;
+
+        do {
+                unsigned int fbytes = zorro_esp_read8(esp, ESP_FFLAGS)
+                                                        & ESP_FF_FBYTES;
+
+                if (fbytes)
+                        return fbytes;
+
+                udelay(2);
+        } while (--i);
+
+        pr_err("FIFO is empty (sreg %02x)\n",
+               zorro_esp_read8(esp, ESP_STATUS));
+        return 0;
+}
+
+static inline int zorro_esp_wait_for_intr(struct esp *esp)
+{
+        struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+        int i = 500000;
+
+        do {
+                esp->sreg = zorro_esp_read8(esp, ESP_STATUS);
+                if (esp->sreg & ESP_STAT_INTR)
+                        return 0;
+
+                udelay(2);
+        } while (--i);
+
+        pr_err("IRQ timeout (sreg %02x)\n", esp->sreg);
+        zep->error = 1;
+        return 1;
+}
+
+/*
+ * PIO macros as used in mac_esp.c.
+ * Note that addr and fifo arguments are local-scope variables declared
+ * in zorro_esp_send_pio_cmd(), the macros are only used in that function,
+ * and addr and fifo are referenced in each use of the macros so there
+ * is no need to pass them as macro parameters.
+ */
+#define ZORRO_ESP_PIO_LOOP(operands, reg1) \
+        asm volatile ( \
+             "1:     moveb " operands "\n" \
+             "       subqw #1,%1       \n" \
+             "       jbne 1b           \n" \
+             : "+a" (addr), "+r" (reg1) \
+             : "a" (fifo));
+
+#define ZORRO_ESP_PIO_FILL(operands, reg1) \
+        asm volatile ( \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       moveb " operands "\n" \
+             "       subqw #8,%1       \n" \
+             "       subqw #8,%1       \n" \
+             : "+a" (addr), "+r" (reg1) \
+             : "a" (fifo));
+
+#define ZORRO_ESP_FIFO_SIZE 16
+
+static void zorro_esp_send_pio_cmd(struct esp *esp, u32 addr, u32 esp_count,
+                                 u32 dma_count, int write, u8 cmd)
+{
+        struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+        u8 __iomem *fifo = esp->regs + ESP_FDATA * 16;
+        u8 phase = esp->sreg & ESP_STAT_PMASK;
+
+        cmd &= ~ESP_CMD_DMA;
+
+        if (write) {
+                u8 *dst = (u8 *)addr;
+                u8 mask = ~(phase == ESP_MIP ? ESP_INTR_FDONE : ESP_INTR_BSERV);
+
+                scsi_esp_cmd(esp, cmd);
+
+                while (1) {
+                        if (!zorro_esp_wait_for_fifo(esp))
+                                break;
+
+                        *dst++ = zorro_esp_read8(esp, ESP_FDATA);
+                        --esp_count;
+
+                        if (!esp_count)
+                                break;
+
+                        if (zorro_esp_wait_for_intr(esp))
+                                break;
+
+                        if ((esp->sreg & ESP_STAT_PMASK) != phase)
+                                break;
+
+                        esp->ireg = zorro_esp_read8(esp, ESP_INTRPT);
+                        if (esp->ireg & mask) {
+                                zep->error = 1;
+                                break;
+                        }
+
+                        if (phase == ESP_MIP)
+                                scsi_esp_cmd(esp, ESP_CMD_MOK);
+
+                        scsi_esp_cmd(esp, ESP_CMD_TI);
+                }
+        } else {        /* unused, as long as we only handle MIP here */
+                scsi_esp_cmd(esp, ESP_CMD_FLUSH);
+
+                if (esp_count >= ZORRO_ESP_FIFO_SIZE)
+                        ZORRO_ESP_PIO_FILL("%0@+,%2@", esp_count)
+                else
+                        ZORRO_ESP_PIO_LOOP("%0@+,%2@", esp_count)
+
+                scsi_esp_cmd(esp, cmd);
+
+                while (esp_count) {
+                        unsigned int n;
+
+                        if (zorro_esp_wait_for_intr(esp))
+                                break;
+
+                        if ((esp->sreg & ESP_STAT_PMASK) != phase)
+                                break;
+
+                        esp->ireg = zorro_esp_read8(esp, ESP_INTRPT);
+                        if (esp->ireg & ~ESP_INTR_BSERV) {
+                                zep->error = 1;
+                                break;
+                        }
+
+                        n = ZORRO_ESP_FIFO_SIZE -
+                            (zorro_esp_read8(esp, ESP_FFLAGS) & ESP_FF_FBYTES);
+                        if (n > esp_count)
+                                n = esp_count;
+
+                        if (n == ZORRO_ESP_FIFO_SIZE)
+                                ZORRO_ESP_PIO_FILL("%0@+,%2@", esp_count)
+                        else {
+                                esp_count -= n;
+                                ZORRO_ESP_PIO_LOOP("%0@+,%2@", n)
+                        }
+
+                        scsi_esp_cmd(esp, ESP_CMD_TI);
+                }
+        }
+}
+
+/* Blizzard 1230/60 SCSI-IV DMA */
+
+static void zorro_esp_send_blz1230_dma_cmd(struct esp *esp, u32 addr,
+                        u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+        struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+        struct blz1230_dma_registers __iomem *dregs = esp->dma_regs;
+        u8 phase = esp->sreg & ESP_STAT_PMASK;
+
+        zep->error = 0;
+        /*
+         * Use PIO if transferring message bytes to esp->command_block_dma.
+         * PIO requires a virtual address, so substitute esp->command_block
+         * for addr.
+         */
+        if (phase == ESP_MIP && addr == esp->command_block_dma) {
+                zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+                                        esp_count, dma_count, write, cmd);
+                return;
+        }
+
+        if (write)
+                /* DMA receive */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_FROM_DEVICE);
+        else
+                /* DMA send */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_TO_DEVICE);
+
+        addr >>= 1;
+        if (write)
+                addr &= ~(DMA_WRITE);
+        else
+                addr |= DMA_WRITE;
+
+        writeb((addr >> 24) & 0xff, &dregs->dma_latch);
+        writeb((addr >> 24) & 0xff, &dregs->dma_addr);
+        writeb((addr >> 16) & 0xff, &dregs->dma_addr);
+        writeb((addr >>  8) & 0xff, &dregs->dma_addr);
+        writeb(addr & 0xff, &dregs->dma_addr);
+
+        scsi_esp_cmd(esp, ESP_CMD_DMA);
+        zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+        zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+        zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+        scsi_esp_cmd(esp, cmd);
+}
+
+/* Blizzard 1230-II DMA */
+
+static void zorro_esp_send_blz1230II_dma_cmd(struct esp *esp, u32 addr,
+                        u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+        struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+        struct blz1230II_dma_registers __iomem *dregs = esp->dma_regs;
+        u8 phase = esp->sreg & ESP_STAT_PMASK;
+
+        zep->error = 0;
+        /* Use PIO if transferring message bytes to esp->command_block_dma */
+        if (phase == ESP_MIP && addr == esp->command_block_dma) {
+                zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+                                        esp_count, dma_count, write, cmd);
+                return;
+        }
+
+        if (write)
+                /* DMA receive */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_FROM_DEVICE);
+        else
+                /* DMA send */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_TO_DEVICE);
+
+        addr >>= 1;
+        if (write)
+                addr &= ~(DMA_WRITE);
+        else
+                addr |= DMA_WRITE;
+
+        writeb((addr >> 24) & 0xff, &dregs->dma_latch);
+        writeb((addr >> 16) & 0xff, &dregs->dma_addr);
+        writeb((addr >>  8) & 0xff, &dregs->dma_addr);
+        writeb(addr & 0xff, &dregs->dma_addr);
+
+        scsi_esp_cmd(esp, ESP_CMD_DMA);
+        zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+        zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+        zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+        scsi_esp_cmd(esp, cmd);
+}
+
+/* Blizzard 2060 DMA */
+
+static void zorro_esp_send_blz2060_dma_cmd(struct esp *esp, u32 addr,
+                        u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+        struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+        struct blz2060_dma_registers __iomem *dregs = esp->dma_regs;
+        u8 phase = esp->sreg & ESP_STAT_PMASK;
+
+        zep->error = 0;
+        /* Use PIO if transferring message bytes to esp->command_block_dma */
+        if (phase == ESP_MIP && addr == esp->command_block_dma) {
+                zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+                                        esp_count, dma_count, write, cmd);
+                return;
+        }
+
+        if (write)
+                /* DMA receive */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_FROM_DEVICE);
+        else
+                /* DMA send */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_TO_DEVICE);
+
+        addr >>= 1;
+        if (write)
+                addr &= ~(DMA_WRITE);
+        else
+                addr |= DMA_WRITE;
+
+        writeb(addr & 0xff, &dregs->dma_addr3);
+        writeb((addr >>  8) & 0xff, &dregs->dma_addr2);
+        writeb((addr >> 16) & 0xff, &dregs->dma_addr1);
+        writeb((addr >> 24) & 0xff, &dregs->dma_addr0);
+
+        scsi_esp_cmd(esp, ESP_CMD_DMA);
+        zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+        zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+        zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+        scsi_esp_cmd(esp, cmd);
+}
+
+/* Cyberstorm I DMA */
+
+static void zorro_esp_send_cyber_dma_cmd(struct esp *esp, u32 addr,
+                        u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+        struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+        struct cyber_dma_registers __iomem *dregs = esp->dma_regs;
+        u8 phase = esp->sreg & ESP_STAT_PMASK;
+        unsigned char *ctrl_data = &zep->ctrl_data;
+
+        zep->error = 0;
+        /* Use PIO if transferring message bytes to esp->command_block_dma */
+        if (phase == ESP_MIP && addr == esp->command_block_dma) {
+                zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+                                        esp_count, dma_count, write, cmd);
+                return;
+        }
+
+        zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+        zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+        zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+        if (write) {
+                /* DMA receive */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_FROM_DEVICE);
+                addr &= ~(1);
+        } else {
+                /* DMA send */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_TO_DEVICE);
+                addr |= 1;
+        }
+
+        writeb((addr >> 24) & 0xff, &dregs->dma_addr0);
+        writeb((addr >> 16) & 0xff, &dregs->dma_addr1);
+        writeb((addr >>  8) & 0xff, &dregs->dma_addr2);
+        writeb(addr & 0xff, &dregs->dma_addr3);
+
+        if (write)
+                *ctrl_data &= ~(CYBER_DMA_WRITE);
+        else
+                *ctrl_data |= CYBER_DMA_WRITE;
+
+        *ctrl_data &= ~(CYBER_DMA_Z3);  /* Z2, do 16 bit DMA */
+
+        writeb(*ctrl_data, &dregs->ctrl_reg);
+
+        scsi_esp_cmd(esp, cmd);
+}
+
+/* Cyberstorm II DMA */
+
+static void zorro_esp_send_cyberII_dma_cmd(struct esp *esp, u32 addr,
+                        u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+        struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+        struct cyberII_dma_registers __iomem *dregs = esp->dma_regs;
+        u8 phase = esp->sreg & ESP_STAT_PMASK;
+
+        zep->error = 0;
+        /* Use PIO if transferring message bytes to esp->command_block_dma */
+        if (phase == ESP_MIP && addr == esp->command_block_dma) {
+                zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+                                        esp_count, dma_count, write, cmd);
+                return;
+        }
+
+        zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+        zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+        zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+        if (write) {
+                /* DMA receive */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_FROM_DEVICE);
+                addr &= ~(1);
+        } else {
+                /* DMA send */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_TO_DEVICE);
+                addr |= 1;
+        }
+
+        writeb((addr >> 24) & 0xff, &dregs->dma_addr0);
+        writeb((addr >> 16) & 0xff, &dregs->dma_addr1);
+        writeb((addr >>  8) & 0xff, &dregs->dma_addr2);
+        writeb(addr & 0xff, &dregs->dma_addr3);
+
+        scsi_esp_cmd(esp, cmd);
+}
+
+/* Fastlane DMA */
+
+static void zorro_esp_send_fastlane_dma_cmd(struct esp *esp, u32 addr,
+                        u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+        struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+        struct fastlane_dma_registers __iomem *dregs = esp->dma_regs;
+        u8 phase = esp->sreg & ESP_STAT_PMASK;
+        unsigned char *ctrl_data = &zep->ctrl_data;
+
+        zep->error = 0;
+        /* Use PIO if transferring message bytes to esp->command_block_dma */
+        if (phase == ESP_MIP && addr == esp->command_block_dma) {
+                zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+                                        esp_count, dma_count, write, cmd);
+                return;
+        }
+
+        zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+        zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+        zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+        if (write) {
+                /* DMA receive */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_FROM_DEVICE);
+                addr &= ~(1);
+        } else {
+                /* DMA send */
+                dma_sync_single_for_device(esp->dev, addr, esp_count,
+                                DMA_TO_DEVICE);
+                addr |= 1;
+        }
+
+        writeb(0, &dregs->clear_strobe);
+        z_writel(addr, ((addr & 0x00ffffff) + zep->board_base));
+
+        if (write) {
+                *ctrl_data = (*ctrl_data & FASTLANE_DMA_MASK) |
+                                FASTLANE_DMA_ENABLE;
+        } else {
+                *ctrl_data = ((*ctrl_data & FASTLANE_DMA_MASK) |
+                                FASTLANE_DMA_ENABLE |
+                                FASTLANE_DMA_WRITE);
+        }
+
+        writeb(*ctrl_data, &dregs->ctrl_reg);
+
+        scsi_esp_cmd(esp, cmd);
+}
+
+static int zorro_esp_dma_error(struct esp *esp)
+{
+        struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+
+        /* check for error in case we've been doing PIO */
+        if (zep->error == 1)
+                return 1;
+
+        /* do nothing - there seems to be no way to check for DMA errors */
+        return 0;
+}
+
+/* per-board ESP driver ops */
+
+static const struct esp_driver_ops blz1230_esp_ops = {
+        .esp_write8             = zorro_esp_write8,
+        .esp_read8              = zorro_esp_read8,
+        .map_single             = zorro_esp_map_single,
+        .map_sg                 = zorro_esp_map_sg,
+        .unmap_single           = zorro_esp_unmap_single,
+        .unmap_sg               = zorro_esp_unmap_sg,
+        .irq_pending            = zorro_esp_irq_pending,
+        .dma_length_limit       = zorro_esp_dma_length_limit,
+        .reset_dma              = zorro_esp_reset_dma,
+        .dma_drain              = zorro_esp_dma_drain,
+        .dma_invalidate         = zorro_esp_dma_invalidate,
+        .send_dma_cmd           = zorro_esp_send_blz1230_dma_cmd,
+        .dma_error              = zorro_esp_dma_error,
+};
+
+static const struct esp_driver_ops blz1230II_esp_ops = {
+        .esp_write8             = zorro_esp_write8,
+        .esp_read8              = zorro_esp_read8,
+        .map_single             = zorro_esp_map_single,
+        .map_sg                 = zorro_esp_map_sg,
+        .unmap_single           = zorro_esp_unmap_single,
+        .unmap_sg               = zorro_esp_unmap_sg,
+        .irq_pending            = zorro_esp_irq_pending,
+        .dma_length_limit       = zorro_esp_dma_length_limit,
+        .reset_dma              = zorro_esp_reset_dma,
+        .dma_drain              = zorro_esp_dma_drain,
+        .dma_invalidate         = zorro_esp_dma_invalidate,
+        .send_dma_cmd           = zorro_esp_send_blz1230II_dma_cmd,
+        .dma_error              = zorro_esp_dma_error,
+};
+
+static const struct esp_driver_ops blz2060_esp_ops = {
+        .esp_write8             = zorro_esp_write8,
+        .esp_read8              = zorro_esp_read8,
+        .map_single             = zorro_esp_map_single,
+        .map_sg                 = zorro_esp_map_sg,
+        .unmap_single           = zorro_esp_unmap_single,
+        .unmap_sg               = zorro_esp_unmap_sg,
+        .irq_pending            = zorro_esp_irq_pending,
+        .dma_length_limit       = zorro_esp_dma_length_limit,
+        .reset_dma              = zorro_esp_reset_dma,
+        .dma_drain              = zorro_esp_dma_drain,
+        .dma_invalidate         = zorro_esp_dma_invalidate,
+        .send_dma_cmd           = zorro_esp_send_blz2060_dma_cmd,
+        .dma_error              = zorro_esp_dma_error,
+};
+
+static const struct esp_driver_ops cyber_esp_ops = {
+        .esp_write8             = zorro_esp_write8,
+        .esp_read8              = zorro_esp_read8,
+        .map_single             = zorro_esp_map_single,
+        .map_sg                 = zorro_esp_map_sg,
+        .unmap_single           = zorro_esp_unmap_single,
+        .unmap_sg               = zorro_esp_unmap_sg,
+        .irq_pending            = cyber_esp_irq_pending,
+        .dma_length_limit       = zorro_esp_dma_length_limit,
+        .reset_dma              = zorro_esp_reset_dma,
+        .dma_drain              = zorro_esp_dma_drain,
+        .dma_invalidate         = zorro_esp_dma_invalidate,
+        .send_dma_cmd           = zorro_esp_send_cyber_dma_cmd,
+        .dma_error              = zorro_esp_dma_error,
+};
+
+static const struct esp_driver_ops cyberII_esp_ops = {
+        .esp_write8             = zorro_esp_write8,
+        .esp_read8              = zorro_esp_read8,
+        .map_single             = zorro_esp_map_single,
+        .map_sg                 = zorro_esp_map_sg,
+        .unmap_single           = zorro_esp_unmap_single,
+        .unmap_sg               = zorro_esp_unmap_sg,
+        .irq_pending            = zorro_esp_irq_pending,
+        .dma_length_limit       = zorro_esp_dma_length_limit,
+        .reset_dma              = zorro_esp_reset_dma,
+        .dma_drain              = zorro_esp_dma_drain,
+        .dma_invalidate         = zorro_esp_dma_invalidate,
+        .send_dma_cmd           = zorro_esp_send_cyberII_dma_cmd,
+        .dma_error              = zorro_esp_dma_error,
+};
+
+static const struct esp_driver_ops fastlane_esp_ops = {
+        .esp_write8             = zorro_esp_write8,
+        .esp_read8              = zorro_esp_read8,
+        .map_single             = zorro_esp_map_single,
+        .map_sg                 = zorro_esp_map_sg,
+        .unmap_single           = zorro_esp_unmap_single,
+        .unmap_sg               = zorro_esp_unmap_sg,
+        .irq_pending            = fastlane_esp_irq_pending,
+        .dma_length_limit       = zorro_esp_dma_length_limit,
+        .reset_dma              = zorro_esp_reset_dma,
+        .dma_drain              = zorro_esp_dma_drain,
+        .dma_invalidate         = fastlane_esp_dma_invalidate,
+        .send_dma_cmd           = zorro_esp_send_fastlane_dma_cmd,
+        .dma_error              = zorro_esp_dma_error,
+};
+
+/* Zorro driver config data */
+
+struct zorro_driver_data {
+        const char *name;
+        unsigned long offset;
+        unsigned long dma_offset;
+        int absolute;   /* offset is absolute address */
+        int scsi_option;
+        const struct esp_driver_ops *esp_ops;
+};
+
+/* board types */
+
+enum {
+        ZORRO_BLZ1230,
+        ZORRO_BLZ1230II,
+        ZORRO_BLZ2060,
+        ZORRO_CYBER,
+        ZORRO_CYBERII,
+        ZORRO_FASTLANE,
+};
+
+/* per-board config data */
+
+static const struct zorro_driver_data zorro_esp_boards[] = {
+        [ZORRO_BLZ1230] = {
+                                .name           = "Blizzard 1230",
+                                .offset         = 0x8000,
+                                .dma_offset     = 0x10000,
+                                .scsi_option    = 1,
+                                .esp_ops        = &blz1230_esp_ops,
+        },
+        [ZORRO_BLZ1230II] = {
+                                .name           = "Blizzard 1230II",
+                                .offset         = 0x10000,
+                                .dma_offset     = 0x10021,
+                                .scsi_option    = 1,
+                                .esp_ops        = &blz1230II_esp_ops,
+        },
+        [ZORRO_BLZ2060] = {
+                                .name           = "Blizzard 2060",
+                                .offset         = 0x1ff00,
+                                .dma_offset     = 0x1ffe0,
+                                .esp_ops        = &blz2060_esp_ops,
+        },
+        [ZORRO_CYBER] = {
+                                .name           = "CyberStormI",
+                                .offset         = 0xf400,
+                                .dma_offset     = 0xf800,
+                                .esp_ops        = &cyber_esp_ops,
+        },
+        [ZORRO_CYBERII] = {
+                                .name           = "CyberStormII",
+                                .offset         = 0x1ff03,
+                                .dma_offset     = 0x1ff43,
+                                .scsi_option    = 1,
+                                .esp_ops        = &cyberII_esp_ops,
+        },
+        [ZORRO_FASTLANE] = {
+                                .name           = "Fastlane",
+                                .offset         = 0x1000001,
+                                .dma_offset     = 0x1000041,
+                                .esp_ops        = &fastlane_esp_ops,
+        },
+};
+
+static const struct zorro_device_id zorro_esp_zorro_tbl[] = {
+        {       /* Blizzard 1230 IV */
+                .id = ZORRO_ID(PHASE5, 0x11, 0),
+                .driver_data = ZORRO_BLZ1230,
+        },
+        {       /* Blizzard 1230 II (Zorro II) or Fastlane (Zorro III) */
+                .id = ZORRO_ID(PHASE5, 0x0B, 0),
+                .driver_data = ZORRO_BLZ1230II,
+        },
+        {       /* Blizzard 2060 */
+                .id = ZORRO_ID(PHASE5, 0x18, 0),
+                .driver_data = ZORRO_BLZ2060,
+        },
+        {       /* Cyberstorm */
+                .id = ZORRO_ID(PHASE5, 0x0C, 0),
+                .driver_data = ZORRO_CYBER,
+        },
+        {       /* Cyberstorm II */
+                .id = ZORRO_ID(PHASE5, 0x19, 0),
+                .driver_data = ZORRO_CYBERII,
+        },
+        { 0 }
+};
+MODULE_DEVICE_TABLE(zorro, zorro_esp_zorro_tbl);
+
+static int zorro_esp_probe(struct zorro_dev *z,
+                                       const struct zorro_device_id *ent)
+{
+        struct scsi_host_template *tpnt = &scsi_esp_template;
+        struct Scsi_Host *host;
+        struct esp *esp;
+        const struct zorro_driver_data *zdd;
+        struct zorro_esp_priv *zep;
+        unsigned long board, ioaddr, dmaaddr;
+        int err;
+
+        board = zorro_resource_start(z);
+        zdd = &zorro_esp_boards[ent->driver_data];
+
+        pr_info("%s found at address 0x%lx.\n", zdd->name, board);
+
+        zep = kzalloc(sizeof(*zep), GFP_KERNEL);
+        if (!zep) {
+                pr_err("Can't allocate device private data!\n");
+                return -ENOMEM;
+        }
+
+        /* let's figure out whether we have a Zorro II or Zorro III board */
+        if ((z->rom.er_Type & ERT_TYPEMASK) == ERT_ZORROIII) {
+                if (board > 0xffffff)
+                        zep->zorro3 = 1;
+        } else {
+                /*
+                 * Even though most of these boards identify as Zorro II,
+                 * they are in fact CPU expansion slot boards and have full
+                 * access to all of memory. Fix up DMA bitmask here.
+                 */
+                z->dev.coherent_dma_mask = DMA_BIT_MASK(32);
+        }
+
+        /*
+         * If Zorro III and ID matches Fastlane, our device table entry
+         * contains data for the Blizzard 1230 II board which does share the
+         * same ID. Fix up device table entry here.
+         * TODO: Some Cyberstom060 boards also share this ID but would need
+         * to use the Cyberstorm I driver data ... we catch this by checking
+         * for presence of ESP chip later, but don't try to fix up yet.
+         */
+        if (zep->zorro3 && ent->driver_data == ZORRO_BLZ1230II) {
+                pr_info("%s at address 0x%lx is Fastlane Z3, fixing data!\n",
+                        zdd->name, board);
+                zdd = &zorro_esp_boards[ZORRO_FASTLANE];
+        }
+
+        if (zdd->absolute) {
+                ioaddr  = zdd->offset;
+                dmaaddr = zdd->dma_offset;
+        } else {
+                ioaddr  = board + zdd->offset;
+                dmaaddr = board + zdd->dma_offset;
+        }
+
+        if (!zorro_request_device(z, zdd->name)) {
+                pr_err("cannot reserve region 0x%lx, abort\n",
+                       board);
+                err = -EBUSY;
+                goto fail_free_zep;
+        }
+
+        host = scsi_host_alloc(tpnt, sizeof(struct esp));
+
+        if (!host) {
+                pr_err("No host detected; board configuration problem?\n");
+                err = -ENOMEM;
+                goto fail_release_device;
+        }
+
+        host->base              = ioaddr;
+        host->this_id           = 7;
+
+        esp                     = shost_priv(host);
+        esp->host               = host;
+        esp->dev                = &z->dev;
+
+        esp->scsi_id            = host->this_id;
+        esp->scsi_id_mask       = (1 << esp->scsi_id);
+
+        esp->cfreq = 40000000;
+
+        zep->esp = esp;
+
+        dev_set_drvdata(esp->dev, zep);
+
+        /* additional setup required for Fastlane */
+        if (zep->zorro3 && ent->driver_data == ZORRO_BLZ1230II) {
+                /* map full address space up to ESP base for DMA */
+                zep->board_base = ioremap_nocache(board,
+                                                FASTLANE_ESP_ADDR-1);
+                if (!zep->board_base) {
+                        pr_err("Cannot allocate board address space\n");
+                        err = -ENOMEM;
+                        goto fail_free_host;
+                }
+                /* initialize DMA control shadow register */
+                zep->ctrl_data = (FASTLANE_DMA_FCODE |
+                                  FASTLANE_DMA_EDI | FASTLANE_DMA_ESI);
+        }
+
+        esp->ops = zdd->esp_ops;
+
+        if (ioaddr > 0xffffff)
+                esp->regs = ioremap_nocache(ioaddr, 0x20);
+        else
+                /* ZorroII address space remapped nocache by early startup */
+                esp->regs = ZTWO_VADDR(ioaddr);
+
+        if (!esp->regs) {
+                err = -ENOMEM;
+                goto fail_unmap_fastlane;
+        }
+
+        /* Check whether a Blizzard 12x0 or CyberstormII really has SCSI */
+        if (zdd->scsi_option) {
+                zorro_esp_write8(esp, (ESP_CONFIG1_PENABLE | 7), ESP_CFG1);
+                if (zorro_esp_read8(esp, ESP_CFG1) != (ESP_CONFIG1_PENABLE|7)) {
+                        err = -ENODEV;
+                        goto fail_unmap_regs;
+                }
+        }
+
+        if (zep->zorro3) {
+                /*
+                 * Only Fastlane Z3 for now - add switch for correct struct
+                 * dma_registers size if adding any more
+                 */
+                esp->dma_regs = ioremap_nocache(dmaaddr,
+                                sizeof(struct fastlane_dma_registers));
+        } else
+                /* ZorroII address space remapped nocache by early startup */
+                esp->dma_regs = ZTWO_VADDR(dmaaddr);
+
+        if (!esp->dma_regs) {
+                err = -ENOMEM;
+                goto fail_unmap_regs;
+        }
+
+        esp->command_block = dma_alloc_coherent(esp->dev, 16,
+                                                &esp->command_block_dma,
+                                                GFP_KERNEL);
+
+        if (!esp->command_block) {
+                err = -ENOMEM;
+                goto fail_unmap_dma_regs;
+        }
+
+        host->irq = IRQ_AMIGA_PORTS;
+        err = request_irq(host->irq, scsi_esp_intr, IRQF_SHARED,
+                          "Amiga Zorro ESP", esp);
+        if (err < 0) {
+                err = -ENODEV;
+                goto fail_free_command_block;
+        }
+
+        /* register the chip */
+        err = scsi_esp_register(esp, &z->dev);
+
+        if (err) {
+                err = -ENOMEM;
+                goto fail_free_irq;
+        }
+
+        return 0;
+
+fail_free_irq:
+        free_irq(host->irq, esp);
+
+fail_free_command_block:
+        dma_free_coherent(esp->dev, 16,
+                          esp->command_block,
+                          esp->command_block_dma);
+
+fail_unmap_dma_regs:
+        if (zep->zorro3)
+                iounmap(esp->dma_regs);
+
+fail_unmap_regs:
+        if (ioaddr > 0xffffff)
+                iounmap(esp->regs);
+
+fail_unmap_fastlane:
+        if (zep->zorro3)
+                iounmap(zep->board_base);
+
+fail_free_host:
+        scsi_host_put(host);
+
+fail_release_device:
+        zorro_release_device(z);
+
+fail_free_zep:
+        kfree(zep);
+
+        return err;
+}
+
+static void zorro_esp_remove(struct zorro_dev *z)
+{
+        struct zorro_esp_priv *zep = dev_get_drvdata(&z->dev);
+        struct esp *esp = zep->esp;
+        struct Scsi_Host *host = esp->host;
+
+        scsi_esp_unregister(esp);
+
+        free_irq(host->irq, esp);
+        dma_free_coherent(esp->dev, 16,
+                          esp->command_block,
+                          esp->command_block_dma);
+
+        if (zep->zorro3) {
+                iounmap(zep->board_base);
+                iounmap(esp->dma_regs);
+        }
+
+        if (host->base > 0xffffff)
+                iounmap(esp->regs);
+
+        scsi_host_put(host);
+
+        zorro_release_device(z);
+
+        kfree(zep);
+}
+
+static struct zorro_driver zorro_esp_driver = {
+        .name     = KBUILD_MODNAME,
+        .id_table = zorro_esp_zorro_tbl,
+        .probe    = zorro_esp_probe,
+        .remove   = zorro_esp_remove,
+};
+
+static int __init zorro_esp_scsi_init(void)
+{
+        return zorro_register_driver(&zorro_esp_driver);
+}
+
+static void __exit zorro_esp_scsi_exit(void)
+{
+        zorro_unregister_driver(&zorro_esp_driver);
+}
+
+module_init(zorro_esp_scsi_init);
+module_exit(zorro_esp_scsi_exit);
diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c
index 3f4bf126eed0..5ccef7d597fa 100644
--- a/drivers/target/target_core_configfs.c
+++ b/drivers/target/target_core_configfs.c
@@ -155,6 +155,8 @@ static ssize_t target_core_item_dbroot_store(struct config_item *item,
 
         mutex_unlock(&g_tf_lock);
 
+        pr_debug("Target_Core_ConfigFS: db_root set to %s\n", db_root);
+
         return read_bytes;
 }
 
@@ -3213,6 +3215,27 @@ void target_setup_backend_cits(struct target_backend *tb)
         target_core_setup_dev_stat_cit(tb);
 }
 
+static void target_init_dbroot(void)
+{
+        struct file *fp;
+
+        snprintf(db_root_stage, DB_ROOT_LEN, DB_ROOT_PREFERRED);
+        fp = filp_open(db_root_stage, O_RDONLY, 0);
+        if (IS_ERR(fp)) {
+                pr_err("db_root: cannot open: %s\n", db_root_stage);
+                return;
+        }
+        if (!S_ISDIR(file_inode(fp)->i_mode)) {
+                filp_close(fp, NULL);
+                pr_err("db_root: not a valid directory: %s\n", db_root_stage);
+                return;
+        }
+        filp_close(fp, NULL);
+
+        strncpy(db_root, db_root_stage, DB_ROOT_LEN);
+        pr_debug("Target_Core_ConfigFS: db_root set to %s\n", db_root);
+}
+
 static int __init target_core_init_configfs(void)
 {
         struct configfs_subsystem *subsys = &target_core_fabrics;
@@ -3293,6 +3316,8 @@ static int __init target_core_init_configfs(void)
         if (ret < 0)
                 goto out;
 
+        target_init_dbroot();
+
         return 0;
 
 out:
diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c
index 9b2c0c773022..16751ae55d7b 100644
--- a/drivers/target/target_core_file.c
+++ b/drivers/target/target_core_file.c
@@ -250,6 +250,84 @@ static void fd_destroy_device(struct se_device *dev)
         }
 }
 
+struct target_core_file_cmd {
+        unsigned long   len;
+        struct se_cmd   *cmd;
+        struct kiocb    iocb;
+};
+
+static void cmd_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
+{
+        struct target_core_file_cmd *cmd;
+
+        cmd = container_of(iocb, struct target_core_file_cmd, iocb);
+
+        if (ret != cmd->len)
+                target_complete_cmd(cmd->cmd, SAM_STAT_CHECK_CONDITION);
+        else
+                target_complete_cmd(cmd->cmd, SAM_STAT_GOOD);
+
+        kfree(cmd);
+}
+
+static sense_reason_t
+fd_execute_rw_aio(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
+              enum dma_data_direction data_direction)
+{
+        int is_write = !(data_direction == DMA_FROM_DEVICE);
+        struct se_device *dev = cmd->se_dev;
+        struct fd_dev *fd_dev = FD_DEV(dev);
+        struct file *file = fd_dev->fd_file;
+        struct target_core_file_cmd *aio_cmd;
+        struct iov_iter iter = {};
+        struct scatterlist *sg;
+        struct bio_vec *bvec;
+        ssize_t len = 0;
+        int ret = 0, i;
+
+        aio_cmd = kmalloc(sizeof(struct target_core_file_cmd), GFP_KERNEL);
+        if (!aio_cmd)
+                return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+
+        bvec = kcalloc(sgl_nents, sizeof(struct bio_vec), GFP_KERNEL);
+        if (!bvec) {
+                kfree(aio_cmd);
+                return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+        }
+
+        for_each_sg(sgl, sg, sgl_nents, i) {
+                bvec[i].bv_page = sg_page(sg);
+                bvec[i].bv_len = sg->length;
+                bvec[i].bv_offset = sg->offset;
+
+                len += sg->length;
+        }
+
+        iov_iter_bvec(&iter, ITER_BVEC | is_write, bvec, sgl_nents, len);
+
+        aio_cmd->cmd = cmd;
+        aio_cmd->len = len;
+        aio_cmd->iocb.ki_pos = cmd->t_task_lba * dev->dev_attrib.block_size;
+        aio_cmd->iocb.ki_filp = file;
+        aio_cmd->iocb.ki_complete = cmd_rw_aio_complete;
+        aio_cmd->iocb.ki_flags = IOCB_DIRECT;
+
+        if (is_write && (cmd->se_cmd_flags & SCF_FUA))
+                aio_cmd->iocb.ki_flags |= IOCB_DSYNC;
+
+        if (is_write)
+                ret = call_write_iter(file, &aio_cmd->iocb, &iter);
+        else
+                ret = call_read_iter(file, &aio_cmd->iocb, &iter);
+
+        kfree(bvec);
+
+        if (ret != -EIOCBQUEUED)
+                cmd_rw_aio_complete(&aio_cmd->iocb, ret, 0);
+
+        return 0;
+}
+
 static int fd_do_rw(struct se_cmd *cmd, struct file *fd,
                     u32 block_size, struct scatterlist *sgl,
                     u32 sgl_nents, u32 data_length, int is_write)
@@ -527,7 +605,7 @@ fd_execute_unmap(struct se_cmd *cmd, sector_t lba, sector_t nolb)
 }
 
 static sense_reason_t
-fd_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
+fd_execute_rw_buffered(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
               enum dma_data_direction data_direction)
 {
         struct se_device *dev = cmd->se_dev;
@@ -537,16 +615,6 @@ fd_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
         sense_reason_t rc;
         int ret = 0;
         /*
-         * We are currently limited by the number of iovecs (2048) per
-         * single vfs_[writev,readv] call.
-         */
-        if (cmd->data_length > FD_MAX_BYTES) {
-                pr_err("FILEIO: Not able to process I/O of %u bytes due to"
-                       "FD_MAX_BYTES: %u iovec count limitation\n",
-                        cmd->data_length, FD_MAX_BYTES);
-                return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
-        }
-        /*
          * Call vectorized fileio functions to map struct scatterlist
          * physical memory addresses to struct iovec virtual memory.
          */
@@ -620,14 +688,39 @@ fd_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
         return 0;
 }
 
+static sense_reason_t
+fd_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
+              enum dma_data_direction data_direction)
+{
+        struct se_device *dev = cmd->se_dev;
+        struct fd_dev *fd_dev = FD_DEV(dev);
+
+        /*
+         * We are currently limited by the number of iovecs (2048) per
+         * single vfs_[writev,readv] call.
+         */
+        if (cmd->data_length > FD_MAX_BYTES) {
+                pr_err("FILEIO: Not able to process I/O of %u bytes due to"
+                       "FD_MAX_BYTES: %u iovec count limitation\n",
+                        cmd->data_length, FD_MAX_BYTES);
+                return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+        }
+
+        if (fd_dev->fbd_flags & FDBD_HAS_ASYNC_IO)
+                return fd_execute_rw_aio(cmd, sgl, sgl_nents, data_direction);
+        return fd_execute_rw_buffered(cmd, sgl, sgl_nents, data_direction);
+}
+
 enum {
-        Opt_fd_dev_name, Opt_fd_dev_size, Opt_fd_buffered_io, Opt_err
+        Opt_fd_dev_name, Opt_fd_dev_size, Opt_fd_buffered_io,
+        Opt_fd_async_io, Opt_err
 };
 
 static match_table_t tokens = {
         {Opt_fd_dev_name, "fd_dev_name=%s"},
         {Opt_fd_dev_size, "fd_dev_size=%s"},
         {Opt_fd_buffered_io, "fd_buffered_io=%d"},
+        {Opt_fd_async_io, "fd_async_io=%d"},
         {Opt_err, NULL}
 };
 
@@ -693,6 +786,21 @@ static ssize_t fd_set_configfs_dev_params(struct se_device *dev,
 
                         fd_dev->fbd_flags |= FDBD_HAS_BUFFERED_IO_WCE;
                         break;
+                case Opt_fd_async_io:
+                        ret = match_int(args, &arg);
+                        if (ret)
+                                goto out;
+                        if (arg != 1) {
+                                pr_err("bogus fd_async_io=%d value\n", arg);
+                                ret = -EINVAL;
+                                goto out;
+                        }
+
+                        pr_debug("FILEIO: Using async I/O"
+                                " operations for struct fd_dev\n");
+
+                        fd_dev->fbd_flags |= FDBD_HAS_ASYNC_IO;
+                        break;
                 default:
                         break;
                 }
@@ -709,10 +817,11 @@ static ssize_t fd_show_configfs_dev_params(struct se_device *dev, char *b)
         ssize_t bl = 0;
 
         bl = sprintf(b + bl, "TCM FILEIO ID: %u", fd_dev->fd_dev_id);
-        bl += sprintf(b + bl, "        File: %s  Size: %llu  Mode: %s\n",
+        bl += sprintf(b + bl, "        File: %s  Size: %llu  Mode: %s Async: %d\n",
                 fd_dev->fd_dev_name, fd_dev->fd_dev_size,
                 (fd_dev->fbd_flags & FDBD_HAS_BUFFERED_IO_WCE) ?
-                "Buffered-WCE" : "O_DSYNC");
+                "Buffered-WCE" : "O_DSYNC",
+                !!(fd_dev->fbd_flags & FDBD_HAS_ASYNC_IO));
         return bl;
 }
 
diff --git a/drivers/target/target_core_file.h b/drivers/target/target_core_file.h
index 53be5ffd3261..929b1ecd544e 100644
--- a/drivers/target/target_core_file.h
+++ b/drivers/target/target_core_file.h
@@ -22,6 +22,7 @@
 #define FBDF_HAS_PATH           0x01
 #define FBDF_HAS_SIZE           0x02
 #define FDBD_HAS_BUFFERED_IO_WCE 0x04
+#define FDBD_HAS_ASYNC_IO        0x08
 #define FDBD_FORMAT_UNIT_SIZE   2048
 
 struct fd_dev {
diff --git a/drivers/target/target_core_internal.h b/drivers/target/target_core_internal.h
index 1d5afc3ae017..dead30b1d32c 100644
--- a/drivers/target/target_core_internal.h
+++ b/drivers/target/target_core_internal.h
@@ -166,6 +166,7 @@ extern struct se_portal_group xcopy_pt_tpg;
 /* target_core_configfs.c */
 #define DB_ROOT_LEN             4096
 #define DB_ROOT_DEFAULT         "/var/target"
+#define DB_ROOT_PREFERRED       "/etc/target"
 
 extern char db_root[];
 
diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c
index 668934ea74cb..47d76c862014 100644
--- a/drivers/target/target_core_pscsi.c
+++ b/drivers/target/target_core_pscsi.c
@@ -689,8 +689,29 @@ after_mode_sense:
         }
 after_mode_select:
 
-        if (scsi_status == SAM_STAT_CHECK_CONDITION)
+        if (scsi_status == SAM_STAT_CHECK_CONDITION) {
                 transport_copy_sense_to_cmd(cmd, req_sense);
+
+                /*
+                 * check for TAPE device reads with
+                 * FM/EOM/ILI set, so that we can get data
+                 * back despite framework assumption that a
+                 * check condition means there is no data
+                 */
+                if (sd->type == TYPE_TAPE &&
+                    cmd->data_direction == DMA_FROM_DEVICE) {
+                        /*
+                         * is sense data valid, fixed format,
+                         * and have FM, EOM, or ILI set?
+                         */
+                        if (req_sense[0] == 0xf0 &&     /* valid, fixed format */
+                            req_sense[2] & 0xe0 &&      /* FM, EOM, or ILI */
+                            (req_sense[2] & 0xf) == 0) { /* key==NO_SENSE */
+                                pr_debug("Tape FM/EOM/ILI status detected. Treat as normal read.\n");
+                                cmd->se_cmd_flags |= SCF_TREAT_READ_AS_NORMAL;
+                        }
+                }
+        }
 }
 
 enum {
@@ -1062,7 +1083,8 @@ static void pscsi_req_done(struct request *req, blk_status_t status)
 
         switch (host_byte(result)) {
         case DID_OK:
-                target_complete_cmd(cmd, scsi_status);
+                target_complete_cmd_with_length(cmd, scsi_status,
+                        cmd->data_length - scsi_req(req)->resid_len);
                 break;
         default:
                 pr_debug("PSCSI Host Byte exception at cmd: %p CDB:"
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index 4558f2e1fe1b..f0e8f0f4ccb4 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -779,7 +779,9 @@ EXPORT_SYMBOL(target_complete_cmd);
 
 void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length)
 {
-        if (scsi_status == SAM_STAT_GOOD && length < cmd->data_length) {
+        if ((scsi_status == SAM_STAT_GOOD ||
+             cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) &&
+            length < cmd->data_length) {
                 if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
                         cmd->residual_count += cmd->data_length - length;
                 } else {
@@ -1431,7 +1433,7 @@ transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl,
         return 0;
 }
 
-/*
+/**
  * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized
  *                       se_cmd + use pre-allocated SGL memory.
  *
@@ -1441,7 +1443,7 @@ transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl,
  * @sense: pointer to SCSI sense buffer
  * @unpacked_lun: unpacked LUN to reference for struct se_lun
  * @data_length: fabric expected data transfer length
- * @task_addr: SAM task attribute
+ * @task_attr: SAM task attribute
  * @data_dir: DMA data direction
  * @flags: flags for command submission from target_sc_flags_tables
  * @sgl: struct scatterlist memory for unidirectional mapping
@@ -1578,7 +1580,7 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess
 }
 EXPORT_SYMBOL(target_submit_cmd_map_sgls);
 
-/*
+/**
  * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd
  *
  * @se_cmd: command descriptor to submit
@@ -1587,7 +1589,7 @@ EXPORT_SYMBOL(target_submit_cmd_map_sgls);
  * @sense: pointer to SCSI sense buffer
  * @unpacked_lun: unpacked LUN to reference for struct se_lun
  * @data_length: fabric expected data transfer length
- * @task_addr: SAM task attribute
+ * @task_attr: SAM task attribute
  * @data_dir: DMA data direction
  * @flags: flags for command submission from target_sc_flags_tables
  *
@@ -1654,7 +1656,7 @@ static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag,
  * @se_sess: associated se_sess for endpoint
  * @sense: pointer to SCSI sense buffer
  * @unpacked_lun: unpacked LUN to reference for struct se_lun
- * @fabric_context: fabric context for TMR req
+ * @fabric_tmr_ptr: fabric context for TMR req
  * @tm_type: Type of TM request
  * @gfp: gfp type for caller
  * @tag: referenced task tag for TMR_ABORT_TASK
@@ -2084,12 +2086,24 @@ static void transport_complete_qf(struct se_cmd *cmd)
                 goto queue_status;
         }
 
-        if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
+        /*
+         * Check if we need to send a sense buffer from
+         * the struct se_cmd in question. We do NOT want
+         * to take this path of the IO has been marked as
+         * needing to be treated like a "normal read". This
+         * is the case if it's a tape read, and either the
+         * FM, EOM, or ILI bits are set, but there is no
+         * sense data.
+         */
+        if (!(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) &&
+            cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
                 goto queue_status;
 
         switch (cmd->data_direction) {
         case DMA_FROM_DEVICE:
-                if (cmd->scsi_status)
+                /* queue status if not treating this as a normal read */
+                if (cmd->scsi_status &&
+                    !(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL))
                         goto queue_status;
 
                 trace_target_cmd_complete(cmd);
@@ -2194,9 +2208,15 @@ static void target_complete_ok_work(struct work_struct *work)
 
         /*
          * Check if we need to send a sense buffer from
-         * the struct se_cmd in question.
+         * the struct se_cmd in question. We do NOT want
+         * to take this path of the IO has been marked as
+         * needing to be treated like a "normal read". This
+         * is the case if it's a tape read, and either the
+         * FM, EOM, or ILI bits are set, but there is no
+         * sense data.
          */
-        if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
+        if (!(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) &&
+            cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
                 WARN_ON(!cmd->scsi_status);
                 ret = transport_send_check_condition_and_sense(
                                         cmd, 0, 1);
@@ -2238,7 +2258,18 @@ static void target_complete_ok_work(struct work_struct *work)
 queue_rsp:
         switch (cmd->data_direction) {
         case DMA_FROM_DEVICE:
-                if (cmd->scsi_status)
+                /*
+                 * if this is a READ-type IO, but SCSI status
+                 * is set, then skip returning data and just
+                 * return the status -- unless this IO is marked
+                 * as needing to be treated as a normal read,
+                 * in which case we want to go ahead and return
+                 * the data. This happens, for example, for tape
+                 * reads with the FM, EOM, or ILI bits set, with
+                 * no sense data.
+                 */
+                if (cmd->scsi_status &&
+                    !(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL))
                         goto queue_status;
 
                 atomic_long_add(cmd->data_length,
@@ -2606,7 +2637,8 @@ int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
 }
 EXPORT_SYMBOL(transport_generic_free_cmd);
 
-/* target_get_sess_cmd - Add command to active ->sess_cmd_list
+/**
+ * target_get_sess_cmd - Add command to active ->sess_cmd_list
  * @se_cmd:     command descriptor to add
  * @ack_kref:   Signal that fabric will perform an ack target_put_sess_cmd()
  */
@@ -2800,7 +2832,8 @@ void target_show_cmd(const char *pfx, struct se_cmd *cmd)
 }
 EXPORT_SYMBOL(target_show_cmd);
 
-/* target_sess_cmd_list_set_waiting - Flag all commands in
+/**
+ * target_sess_cmd_list_set_waiting - Flag all commands in
  *         sess_cmd_list to complete cmd_wait_comp.  Set
  *         sess_tearing_down so no more commands are queued.
  * @se_sess:    session to flag
@@ -2835,7 +2868,8 @@ void target_sess_cmd_list_set_waiting(struct se_session *se_sess)
 }
 EXPORT_SYMBOL(target_sess_cmd_list_set_waiting);
 
-/* target_wait_for_sess_cmds - Wait for outstanding descriptors
+/**
+ * target_wait_for_sess_cmds - Wait for outstanding descriptors
  * @se_sess:    session to wait for active I/O
  */
 void target_wait_for_sess_cmds(struct se_session *se_sess)
@@ -3332,7 +3366,7 @@ static void target_tmr_work(struct work_struct *work)
                 tmr->response = TMR_FUNCTION_REJECTED;
                 break;
         default:
-                pr_err("Uknown TMR function: 0x%02x.\n",
+                pr_err("Unknown TMR function: 0x%02x.\n",
                                 tmr->function);
                 tmr->response = TMR_FUNCTION_REJECTED;
                 break;
diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c
index 4f26bdc3d1dc..94b183efd236 100644
--- a/drivers/target/target_core_user.c
+++ b/drivers/target/target_core_user.c
@@ -42,7 +42,11 @@
 
 #include <linux/target_core_user.h>
 
-/*
+/**
+ * DOC: Userspace I/O
+ * Userspace I/O
+ * -------------
+ *
  * Define a shared-memory interface for LIO to pass SCSI commands and
  * data to userspace for processing. This is to allow backends that
  * are too complex for in-kernel support to be possible.
@@ -53,7 +57,7 @@
  * See the .h file for how the ring is laid out. Note that while the
  * command ring is defined, the particulars of the data area are
  * not. Offset values in the command entry point to other locations
- * internal to the mmap()ed area. There is separate space outside the
+ * internal to the mmap-ed area. There is separate space outside the
  * command ring for data buffers. This leaves maximum flexibility for
  * moving buffer allocations, or even page flipping or other
  * allocation techniques, without altering the command ring layout.
@@ -1382,7 +1386,7 @@ static struct page *tcmu_try_get_block_page(struct tcmu_dev *udev, uint32_t dbi)
         return page;
 }
 
-static int tcmu_vma_fault(struct vm_fault *vmf)
+static vm_fault_t tcmu_vma_fault(struct vm_fault *vmf)
 {
         struct tcmu_dev *udev = vmf->vma->vm_private_data;
         struct uio_info *info = &udev->uio_info;
@@ -1586,8 +1590,9 @@ static int tcmu_wait_genl_cmd_reply(struct tcmu_dev *udev)
         return ret;
 }
 
-static int tcmu_netlink_event(struct tcmu_dev *udev, enum tcmu_genl_cmd cmd,
-                              int reconfig_attr, const void *reconfig_data)
+static int tcmu_netlink_event_init(struct tcmu_dev *udev,
+                                   enum tcmu_genl_cmd cmd,
+                                   struct sk_buff **buf, void **hdr)
 {
         struct sk_buff *skb;
         void *msg_header;
@@ -1613,46 +1618,66 @@ static int tcmu_netlink_event(struct tcmu_dev *udev, enum tcmu_genl_cmd cmd,
         if (ret < 0)
                 goto free_skb;
 
-        if (cmd == TCMU_CMD_RECONFIG_DEVICE) {
-                switch (reconfig_attr) {
-                case TCMU_ATTR_DEV_CFG:
-                        ret = nla_put_string(skb, reconfig_attr, reconfig_data);
-                        break;
-                case TCMU_ATTR_DEV_SIZE:
-                        ret = nla_put_u64_64bit(skb, reconfig_attr,
-                                                *((u64 *)reconfig_data),
-                                                TCMU_ATTR_PAD);
-                        break;
-                case TCMU_ATTR_WRITECACHE:
-                        ret = nla_put_u8(skb, reconfig_attr,
-                                          *((u8 *)reconfig_data));
-                        break;
-                default:
-                        BUG();
-                }
+        *buf = skb;
+        *hdr = msg_header;
+        return ret;
 
-                if (ret < 0)
-                        goto free_skb;
-        }
+free_skb:
+        nlmsg_free(skb);
+        return ret;
+}
+
+static int tcmu_netlink_event_send(struct tcmu_dev *udev,
+                                   enum tcmu_genl_cmd cmd,
+                                   struct sk_buff **buf, void **hdr)
+{
+        int ret = 0;
+        struct sk_buff *skb = *buf;
+        void *msg_header = *hdr;
 
         genlmsg_end(skb, msg_header);
 
         tcmu_init_genl_cmd_reply(udev, cmd);
 
         ret = genlmsg_multicast_allns(&tcmu_genl_family, skb, 0,
-                                TCMU_MCGRP_CONFIG, GFP_KERNEL);
-        /* We don't care if no one is listening */
+                                      TCMU_MCGRP_CONFIG, GFP_KERNEL);
+       /* We don't care if no one is listening */
         if (ret == -ESRCH)
                 ret = 0;
         if (!ret)
                 ret = tcmu_wait_genl_cmd_reply(udev);
-
-        return ret;
-free_skb:
-        nlmsg_free(skb);
         return ret;
 }
 
+static int tcmu_send_dev_add_event(struct tcmu_dev *udev)
+{
+        struct sk_buff *skb = NULL;
+        void *msg_header = NULL;
+        int ret = 0;
+
+        ret = tcmu_netlink_event_init(udev, TCMU_CMD_ADDED_DEVICE, &skb,
+                                      &msg_header);
+        if (ret < 0)
+                return ret;
+        return tcmu_netlink_event_send(udev, TCMU_CMD_ADDED_DEVICE, &skb,
+                                       &msg_header);
+
+}
+
+static int tcmu_send_dev_remove_event(struct tcmu_dev *udev)
+{
+        struct sk_buff *skb = NULL;
+        void *msg_header = NULL;
+        int ret = 0;
+
+        ret = tcmu_netlink_event_init(udev, TCMU_CMD_REMOVED_DEVICE,
+                                      &skb, &msg_header);
+        if (ret < 0)
+                return ret;
+        return tcmu_netlink_event_send(udev, TCMU_CMD_REMOVED_DEVICE,
+                                       &skb, &msg_header);
+}
+
 static int tcmu_update_uio_info(struct tcmu_dev *udev)
 {
         struct tcmu_hba *hba = udev->hba->hba_ptr;
@@ -1762,7 +1787,7 @@ static int tcmu_configure_device(struct se_device *dev)
          */
         kref_get(&udev->kref);
 
-        ret = tcmu_netlink_event(udev, TCMU_CMD_ADDED_DEVICE, 0, NULL);
+        ret = tcmu_send_dev_add_event(udev);
         if (ret)
                 goto err_netlink;
 
@@ -1812,7 +1837,7 @@ static void tcmu_destroy_device(struct se_device *dev)
         list_del(&udev->node);
         mutex_unlock(&root_udev_mutex);
 
-        tcmu_netlink_event(udev, TCMU_CMD_REMOVED_DEVICE, 0, NULL);
+        tcmu_send_dev_remove_event(udev);
 
         uio_unregister_device(&udev->uio_info);
 
@@ -2151,6 +2176,27 @@ static ssize_t tcmu_dev_config_show(struct config_item *item, char *page)
         return snprintf(page, PAGE_SIZE, "%s\n", udev->dev_config);
 }
 
+static int tcmu_send_dev_config_event(struct tcmu_dev *udev,
+                                      const char *reconfig_data)
+{
+        struct sk_buff *skb = NULL;
+        void *msg_header = NULL;
+        int ret = 0;
+
+        ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
+                                      &skb, &msg_header);
+        if (ret < 0)
+                return ret;
+        ret = nla_put_string(skb, TCMU_ATTR_DEV_CFG, reconfig_data);
+        if (ret < 0) {
+                nlmsg_free(skb);
+                return ret;
+        }
+        return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
+                                       &skb, &msg_header);
+}
+
+
 static ssize_t tcmu_dev_config_store(struct config_item *item, const char *page,
                                      size_t count)
 {
@@ -2165,8 +2211,7 @@ static ssize_t tcmu_dev_config_store(struct config_item *item, const char *page,
 
         /* Check if device has been configured before */
         if (tcmu_dev_configured(udev)) {
-                ret = tcmu_netlink_event(udev, TCMU_CMD_RECONFIG_DEVICE,
-                                         TCMU_ATTR_DEV_CFG, page);
+                ret = tcmu_send_dev_config_event(udev, page);
                 if (ret) {
                         pr_err("Unable to reconfigure device\n");
                         return ret;
@@ -2193,6 +2238,26 @@ static ssize_t tcmu_dev_size_show(struct config_item *item, char *page)
         return snprintf(page, PAGE_SIZE, "%zu\n", udev->dev_size);
 }
 
+static int tcmu_send_dev_size_event(struct tcmu_dev *udev, u64 size)
+{
+        struct sk_buff *skb = NULL;
+        void *msg_header = NULL;
+        int ret = 0;
+
+        ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
+                                      &skb, &msg_header);
+        if (ret < 0)
+                return ret;
+        ret = nla_put_u64_64bit(skb, TCMU_ATTR_DEV_SIZE,
+                                size, TCMU_ATTR_PAD);
+        if (ret < 0) {
+                nlmsg_free(skb);
+                return ret;
+        }
+        return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
+                                       &skb, &msg_header);
+}
+
 static ssize_t tcmu_dev_size_store(struct config_item *item, const char *page,
                                    size_t count)
 {
@@ -2208,8 +2273,7 @@ static ssize_t tcmu_dev_size_store(struct config_item *item, const char *page,
 
         /* Check if device has been configured before */
         if (tcmu_dev_configured(udev)) {
-                ret = tcmu_netlink_event(udev, TCMU_CMD_RECONFIG_DEVICE,
-                                         TCMU_ATTR_DEV_SIZE, &val);
+                ret = tcmu_send_dev_size_event(udev, val);
                 if (ret) {
                         pr_err("Unable to reconfigure device\n");
                         return ret;
@@ -2257,6 +2321,25 @@ static ssize_t tcmu_emulate_write_cache_show(struct config_item *item,
         return snprintf(page, PAGE_SIZE, "%i\n", da->emulate_write_cache);
 }
 
+static int tcmu_send_emulate_write_cache(struct tcmu_dev *udev, u8 val)
+{
+        struct sk_buff *skb = NULL;
+        void *msg_header = NULL;
+        int ret = 0;
+
+        ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
+                                      &skb, &msg_header);
+        if (ret < 0)
+                return ret;
+        ret = nla_put_u8(skb, TCMU_ATTR_WRITECACHE, val);
+        if (ret < 0) {
+                nlmsg_free(skb);
+                return ret;
+        }
+        return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
+                                       &skb, &msg_header);
+}
+
 static ssize_t tcmu_emulate_write_cache_store(struct config_item *item,
                                               const char *page, size_t count)
 {
@@ -2272,8 +2355,7 @@ static ssize_t tcmu_emulate_write_cache_store(struct config_item *item,
 
         /* Check if device has been configured before */
         if (tcmu_dev_configured(udev)) {
-                ret = tcmu_netlink_event(udev, TCMU_CMD_RECONFIG_DEVICE,
-                                         TCMU_ATTR_WRITECACHE, &val);
+                ret = tcmu_send_emulate_write_cache(udev, val);
                 if (ret) {
                         pr_err("Unable to reconfigure device\n");
                         return ret;