[SCSI] megaraid_sas: Add multiple MSI-X vector/multiple reply queue support

Signed-off-by: Adam Radford <aradford@gmail.com>
Signed-off-by: James Bottomley <JBottomley@Parallels.com>

4 files changed, 176 insertions, 59 deletions

diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h
index 6b3562554937..9ffb92daa58b 100644
--- a/drivers/scsi/megaraid/megaraid_sas.h
+++ b/drivers/scsi/megaraid/megaraid_sas.h
@@ -757,6 +757,7 @@ struct megasas_ctrl_info {
 #define MEGASAS_INT_CMDS                        32
 #define MEGASAS_SKINNY_INT_CMDS                 5
 
+#define MEGASAS_MAX_MSIX_QUEUES                 16
 /*
  * FW can accept both 32 and 64 bit SGLs. We want to allocate 32/64 bit
  * SGLs based on the size of dma_addr_t
@@ -1278,6 +1279,11 @@ struct megasas_aen_event {
         struct megasas_instance *instance;
 };
 
+struct megasas_irq_context {
+        struct megasas_instance *instance;
+        u32 MSIxIndex;
+};
+
 struct megasas_instance {
 
         u32 *producer;
@@ -1351,8 +1357,9 @@ struct megasas_instance {
 
         /* Ptr to hba specific information */
         void *ctrl_context;
-        u8      msi_flag;
-        struct msix_entry msixentry;
+        unsigned int msix_vectors;
+        struct msix_entry msixentry[MEGASAS_MAX_MSIX_QUEUES];
+        struct megasas_irq_context irq_context[MEGASAS_MAX_MSIX_QUEUES];
         u64 map_id;
         struct megasas_cmd *map_update_cmd;
         unsigned long bar;
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index f2cf768c896d..ec09d5c2ed90 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -2536,7 +2536,7 @@ megasas_deplete_reply_queue(struct megasas_instance *instance,
                                                 instance->reg_set)
                                                 ) == 0) {
                 /* Hardware may not set outbound_intr_status in MSI-X mode */
-                if (!instance->msi_flag)
+                if (!instance->msix_vectors)
                         return IRQ_NONE;
         }
 
@@ -2594,16 +2594,14 @@ megasas_deplete_reply_queue(struct megasas_instance *instance,
  */
 static irqreturn_t megasas_isr(int irq, void *devp)
 {
-        struct megasas_instance *instance;
+        struct megasas_irq_context *irq_context = devp;
+        struct megasas_instance *instance = irq_context->instance;
         unsigned long flags;
         irqreturn_t     rc;
 
-        if (atomic_read(
-                &(((struct megasas_instance *)devp)->fw_reset_no_pci_access)))
+        if (atomic_read(&instance->fw_reset_no_pci_access))
                 return IRQ_HANDLED;
 
-        instance = (struct megasas_instance *)devp;
-
         spin_lock_irqsave(&instance->hba_lock, flags);
         rc =  megasas_deplete_reply_queue(instance, DID_OK);
         spin_unlock_irqrestore(&instance->hba_lock, flags);
@@ -3488,6 +3486,7 @@ static int megasas_init_fw(struct megasas_instance *instance)
         struct megasas_register_set __iomem *reg_set;
         struct megasas_ctrl_info *ctrl_info;
         unsigned long bar_list;
+        int i;
 
         /* Find first memory bar */
         bar_list = pci_select_bars(instance->pdev, IORESOURCE_MEM);
@@ -3541,9 +3540,33 @@ static int megasas_init_fw(struct megasas_instance *instance)
         /* Check if MSI-X is supported while in ready state */
         msix_enable = (instance->instancet->read_fw_status_reg(reg_set) &
                        0x4000000) >> 0x1a;
-        if (msix_enable && !msix_disable &&
-            !pci_enable_msix(instance->pdev, &instance->msixentry, 1))
-                instance->msi_flag = 1;
+        if (msix_enable && !msix_disable) {
+                /* Check max MSI-X vectors */
+                if ((instance->pdev->device == PCI_DEVICE_ID_LSI_FUSION) ||
+                    (instance->pdev->device == PCI_DEVICE_ID_LSI_INVADER)) {
+                        instance->msix_vectors = (readl(&instance->reg_set->
+                                                        outbound_scratch_pad_2
+                                                          ) & 0x1F) + 1;
+                } else
+                        instance->msix_vectors = 1;
+                /* Don't bother allocating more MSI-X vectors than cpus */
+                instance->msix_vectors = min(instance->msix_vectors,
+                                             (unsigned int)num_online_cpus());
+                for (i = 0; i < instance->msix_vectors; i++)
+                        instance->msixentry[i].entry = i;
+                i = pci_enable_msix(instance->pdev, instance->msixentry,
+                                    instance->msix_vectors);
+                if (i >= 0) {
+                        if (i) {
+                                if (!pci_enable_msix(instance->pdev,
+                                                     instance->msixentry, i))
+                                        instance->msix_vectors = i;
+                                else
+                                        instance->msix_vectors = 0;
+                        }
+                } else
+                        instance->msix_vectors = 0;
+        }
 
         /* Get operational params, sge flags, send init cmd to controller */
         if (instance->instancet->init_adapter(instance))
@@ -3958,7 +3981,7 @@ fail_set_dma_mask:
 static int __devinit
 megasas_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 {
-        int rval, pos;
+        int rval, pos, i, j;
         struct Scsi_Host *host;
         struct megasas_instance *instance;
         u16 control = 0;
@@ -4126,11 +4149,32 @@ megasas_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
         /*
          * Register IRQ
          */
-        if (request_irq(instance->msi_flag ? instance->msixentry.vector :
-                        pdev->irq, instance->instancet->service_isr,
-                        IRQF_SHARED, "megasas", instance)) {
-                printk(KERN_DEBUG "megasas: Failed to register IRQ\n");
-                goto fail_irq;
+        if (instance->msix_vectors) {
+                for (i = 0 ; i < instance->msix_vectors; i++) {
+                        instance->irq_context[i].instance = instance;
+                        instance->irq_context[i].MSIxIndex = i;
+                        if (request_irq(instance->msixentry[i].vector,
+                                        instance->instancet->service_isr, 0,
+                                        "megasas",
+                                        &instance->irq_context[i])) {
+                                printk(KERN_DEBUG "megasas: Failed to "
+                                       "register IRQ for vector %d.\n", i);
+                                for (j = 0 ; j < i ; j++)
+                                        free_irq(
+                                                instance->msixentry[j].vector,
+                                                &instance->irq_context[j]);
+                                goto fail_irq;
+                        }
+                }
+        } else {
+                instance->irq_context[0].instance = instance;
+                instance->irq_context[0].MSIxIndex = 0;
+                if (request_irq(pdev->irq, instance->instancet->service_isr,
+                                IRQF_SHARED, "megasas",
+                                &instance->irq_context[0])) {
+                        printk(KERN_DEBUG "megasas: Failed to register IRQ\n");
+                        goto fail_irq;
+                }
         }
 
         instance->instancet->enable_intr(instance->reg_set);
@@ -4174,8 +4218,12 @@ megasas_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 
         pci_set_drvdata(pdev, NULL);
         instance->instancet->disable_intr(instance->reg_set);
-        free_irq(instance->msi_flag ? instance->msixentry.vector :
-                 instance->pdev->irq, instance);
+        if (instance->msix_vectors)
+                for (i = 0 ; i < instance->msix_vectors; i++)
+                        free_irq(instance->msixentry[i].vector,
+                                 &instance->irq_context[i]);
+        else
+                free_irq(instance->pdev->irq, &instance->irq_context[0]);
 fail_irq:
         if ((instance->pdev->device == PCI_DEVICE_ID_LSI_FUSION) ||
             (instance->pdev->device == PCI_DEVICE_ID_LSI_INVADER))
@@ -4183,7 +4231,7 @@ fail_irq:
         else
                 megasas_release_mfi(instance);
       fail_init_mfi:
-        if (instance->msi_flag)
+        if (instance->msix_vectors)
                 pci_disable_msix(instance->pdev);
       fail_alloc_dma_buf:
         if (instance->evt_detail)
@@ -4299,6 +4347,7 @@ megasas_suspend(struct pci_dev *pdev, pm_message_t state)
 {
         struct Scsi_Host *host;
         struct megasas_instance *instance;
+        int i;
 
         instance = pci_get_drvdata(pdev);
         host = instance->host;
@@ -4322,9 +4371,14 @@ megasas_suspend(struct pci_dev *pdev, pm_message_t state)
 
         pci_set_drvdata(instance->pdev, instance);
         instance->instancet->disable_intr(instance->reg_set);
-        free_irq(instance->msi_flag ? instance->msixentry.vector :
-                 instance->pdev->irq, instance);
-        if (instance->msi_flag)
+
+        if (instance->msix_vectors)
+                for (i = 0 ; i < instance->msix_vectors; i++)
+                        free_irq(instance->msixentry[i].vector,
+                                 &instance->irq_context[i]);
+        else
+                free_irq(instance->pdev->irq, &instance->irq_context[0]);
+        if (instance->msix_vectors)
                 pci_disable_msix(instance->pdev);
 
         pci_save_state(pdev);
@@ -4342,7 +4396,7 @@ megasas_suspend(struct pci_dev *pdev, pm_message_t state)
 static int
 megasas_resume(struct pci_dev *pdev)
 {
-        int rval;
+        int rval, i, j;
         struct Scsi_Host *host;
         struct megasas_instance *instance;
 
@@ -4380,8 +4434,9 @@ megasas_resume(struct pci_dev *pdev)
                 goto fail_ready_state;
 
         /* Now re-enable MSI-X */
-        if (instance->msi_flag)
-                pci_enable_msix(instance->pdev, &instance->msixentry, 1);
+        if (instance->msix_vectors)
+                pci_enable_msix(instance->pdev, instance->msixentry,
+                                instance->msix_vectors);
 
         switch (instance->pdev->device) {
         case PCI_DEVICE_ID_LSI_FUSION:
@@ -4411,11 +4466,32 @@ megasas_resume(struct pci_dev *pdev)
         /*
          * Register IRQ
          */
-        if (request_irq(instance->msi_flag ? instance->msixentry.vector :
-                        pdev->irq, instance->instancet->service_isr,
-                        IRQF_SHARED, "megasas", instance)) {
-                printk(KERN_ERR "megasas: Failed to register IRQ\n");
-                goto fail_irq;
+        if (instance->msix_vectors) {
+                for (i = 0 ; i < instance->msix_vectors; i++) {
+                        instance->irq_context[i].instance = instance;
+                        instance->irq_context[i].MSIxIndex = i;
+                        if (request_irq(instance->msixentry[i].vector,
+                                        instance->instancet->service_isr, 0,
+                                        "megasas",
+                                        &instance->irq_context[i])) {
+                                printk(KERN_DEBUG "megasas: Failed to "
+                                       "register IRQ for vector %d.\n", i);
+                                for (j = 0 ; j < i ; j++)
+                                        free_irq(
+                                                instance->msixentry[j].vector,
+                                                &instance->irq_context[j]);
+                                goto fail_irq;
+                        }
+                }
+        } else {
+                instance->irq_context[0].instance = instance;
+                instance->irq_context[0].MSIxIndex = 0;
+                if (request_irq(pdev->irq, instance->instancet->service_isr,
+                                IRQF_SHARED, "megasas",
+                                &instance->irq_context[0])) {
+                        printk(KERN_DEBUG "megasas: Failed to register IRQ\n");
+                        goto fail_irq;
+                }
         }
 
         instance->instancet->enable_intr(instance->reg_set);
@@ -4512,9 +4588,13 @@ static void __devexit megasas_detach_one(struct pci_dev *pdev)
 
         instance->instancet->disable_intr(instance->reg_set);
 
-        free_irq(instance->msi_flag ? instance->msixentry.vector :
-                 instance->pdev->irq, instance);
-        if (instance->msi_flag)
+        if (instance->msix_vectors)
+                for (i = 0 ; i < instance->msix_vectors; i++)
+                        free_irq(instance->msixentry[i].vector,
+                                 &instance->irq_context[i]);
+        else
+                free_irq(instance->pdev->irq, &instance->irq_context[0]);
+        if (instance->msix_vectors)
                 pci_disable_msix(instance->pdev);
 
         switch (instance->pdev->device) {
@@ -4560,14 +4640,20 @@ static void __devexit megasas_detach_one(struct pci_dev *pdev)
  */
 static void megasas_shutdown(struct pci_dev *pdev)
 {
+        int i;
         struct megasas_instance *instance = pci_get_drvdata(pdev);
+
         instance->unload = 1;
         megasas_flush_cache(instance);
         megasas_shutdown_controller(instance, MR_DCMD_CTRL_SHUTDOWN);
         instance->instancet->disable_intr(instance->reg_set);
-        free_irq(instance->msi_flag ? instance->msixentry.vector :
-                 instance->pdev->irq, instance);
-        if (instance->msi_flag)
+        if (instance->msix_vectors)
+                for (i = 0 ; i < instance->msix_vectors; i++)
+                        free_irq(instance->msixentry[i].vector,
+                                 &instance->irq_context[i]);
+        else
+                free_irq(instance->pdev->irq, &instance->irq_context[0]);
+        if (instance->msix_vectors)
                 pci_disable_msix(instance->pdev);
 }
 
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index ad6bd061a0f8..bfd87fab39aa 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -385,7 +385,7 @@ static int megasas_create_frame_pool_fusion(struct megasas_instance *instance)
 int
 megasas_alloc_cmds_fusion(struct megasas_instance *instance)
 {
-        int i, j;
+        int i, j, count;
         u32 max_cmd, io_frames_sz;
         struct fusion_context *fusion;
         struct megasas_cmd_fusion *cmd;
@@ -409,9 +409,10 @@ megasas_alloc_cmds_fusion(struct megasas_instance *instance)
                 goto fail_req_desc;
         }
 
+        count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
         fusion->reply_frames_desc_pool =
                 pci_pool_create("reply_frames pool", instance->pdev,
-                                fusion->reply_alloc_sz, 16, 0);
+                                fusion->reply_alloc_sz * count, 16, 0);
 
         if (!fusion->reply_frames_desc_pool) {
                 printk(KERN_ERR "megasas; Could not allocate memory for "
@@ -430,7 +431,7 @@ megasas_alloc_cmds_fusion(struct megasas_instance *instance)
         }
 
         reply_desc = fusion->reply_frames_desc;
-        for (i = 0; i < fusion->reply_q_depth; i++, reply_desc++)
+        for (i = 0; i < fusion->reply_q_depth * count; i++, reply_desc++)
                 reply_desc->Words = ULLONG_MAX;
 
         io_frames_sz = fusion->io_frames_alloc_sz;
@@ -633,7 +634,9 @@ megasas_ioc_init_fusion(struct megasas_instance *instance)
                 fusion->reply_frames_desc_phys;
         IOCInitMessage->SystemRequestFrameBaseAddress =
                 fusion->io_request_frames_phys;
-
+        /* Set to 0 for none or 1 MSI-X vectors */
+        IOCInitMessage->HostMSIxVectors = (instance->msix_vectors > 0 ?
+                                           instance->msix_vectors : 0);
         init_frame = (struct megasas_init_frame *)cmd->frame;
         memset(init_frame, 0, MEGAMFI_FRAME_SIZE);
 
@@ -877,7 +880,7 @@ megasas_init_adapter_fusion(struct megasas_instance *instance)
         struct megasas_register_set __iomem *reg_set;
         struct fusion_context *fusion;
         u32 max_cmd;
-        int i = 0;
+        int i = 0, count;
 
         fusion = instance->ctrl_context;
 
@@ -929,7 +932,9 @@ megasas_init_adapter_fusion(struct megasas_instance *instance)
                 (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE -
                  sizeof(union MPI2_SGE_IO_UNION))/16;
 
-        fusion->last_reply_idx = 0;
+        count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
+        for (i = 0 ; i < count; i++)
+                fusion->last_reply_idx[i] = 0;
 
         /*
          * Allocate memory for descriptors
@@ -1421,6 +1426,12 @@ megasas_build_ldio_fusion(struct megasas_instance *instance,
                         fp_possible = io_info.fpOkForIo;
         }
 
+        /* Use smp_processor_id() for now until cmd->request->cpu is CPU
+           id by default, not CPU group id, otherwise all MSI-X queues won't
+           be utilized */
+        cmd->request_desc->SCSIIO.MSIxIndex = instance->msix_vectors ?
+                smp_processor_id() % instance->msix_vectors : 0;
+
         if (fp_possible) {
                 megasas_set_pd_lba(io_request, scp->cmd_len, &io_info, scp,
                                    local_map_ptr, start_lba_lo);
@@ -1691,7 +1702,7 @@ megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance,
  * Completes all commands that is in reply descriptor queue
  */
 int
-complete_cmd_fusion(struct megasas_instance *instance)
+complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex)
 {
         union MPI2_REPLY_DESCRIPTORS_UNION *desc;
         struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *reply_desc;
@@ -1711,7 +1722,9 @@ complete_cmd_fusion(struct megasas_instance *instance)
                 return IRQ_HANDLED;
 
         desc = fusion->reply_frames_desc;
-        desc += fusion->last_reply_idx;
+        desc += ((MSIxIndex * fusion->reply_alloc_sz)/
+                 sizeof(union MPI2_REPLY_DESCRIPTORS_UNION)) +
+                fusion->last_reply_idx[MSIxIndex];
 
         reply_desc = (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
 
@@ -1784,16 +1797,19 @@ complete_cmd_fusion(struct megasas_instance *instance)
                         break;
                 }
 
-                fusion->last_reply_idx++;
-                if (fusion->last_reply_idx >= fusion->reply_q_depth)
-                        fusion->last_reply_idx = 0;
+                fusion->last_reply_idx[MSIxIndex]++;
+                if (fusion->last_reply_idx[MSIxIndex] >=
+                    fusion->reply_q_depth)
+                        fusion->last_reply_idx[MSIxIndex] = 0;
 
                 desc->Words = ULLONG_MAX;
                 num_completed++;
 
                 /* Get the next reply descriptor */
-                if (!fusion->last_reply_idx)
-                        desc = fusion->reply_frames_desc;
+                if (!fusion->last_reply_idx[MSIxIndex])
+                        desc = fusion->reply_frames_desc +
+                                ((MSIxIndex * fusion->reply_alloc_sz)/
+                                 sizeof(union MPI2_REPLY_DESCRIPTORS_UNION));
                 else
                         desc++;
 
@@ -1813,7 +1829,7 @@ complete_cmd_fusion(struct megasas_instance *instance)
                 return IRQ_NONE;
 
         wmb();
-        writel(fusion->last_reply_idx,
+        writel((MSIxIndex << 24) | fusion->last_reply_idx[MSIxIndex],
                &instance->reg_set->reply_post_host_index);
         megasas_check_and_restore_queue_depth(instance);
         return IRQ_HANDLED;
@@ -1831,6 +1847,9 @@ megasas_complete_cmd_dpc_fusion(unsigned long instance_addr)
         struct megasas_instance *instance =
                 (struct megasas_instance *)instance_addr;
         unsigned long flags;
+        u32 count, MSIxIndex;
+
+        count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
 
         /* If we have already declared adapter dead, donot complete cmds */
         spin_lock_irqsave(&instance->hba_lock, flags);
@@ -1841,7 +1860,8 @@ megasas_complete_cmd_dpc_fusion(unsigned long instance_addr)
         spin_unlock_irqrestore(&instance->hba_lock, flags);
 
         spin_lock_irqsave(&instance->completion_lock, flags);
-        complete_cmd_fusion(instance);
+        for (MSIxIndex = 0 ; MSIxIndex < count; MSIxIndex++)
+                complete_cmd_fusion(instance, MSIxIndex);
         spin_unlock_irqrestore(&instance->completion_lock, flags);
 }
 
@@ -1850,10 +1870,11 @@ megasas_complete_cmd_dpc_fusion(unsigned long instance_addr)
  */
 irqreturn_t megasas_isr_fusion(int irq, void *devp)
 {
-        struct megasas_instance *instance = (struct megasas_instance *)devp;
+        struct megasas_irq_context *irq_context = devp;
+        struct megasas_instance *instance = irq_context->instance;
         u32 mfiStatus, fw_state;
 
-        if (!instance->msi_flag) {
+        if (!instance->msix_vectors) {
                 mfiStatus = instance->instancet->clear_intr(instance->reg_set);
                 if (!mfiStatus)
                         return IRQ_NONE;
@@ -1865,7 +1886,7 @@ irqreturn_t megasas_isr_fusion(int irq, void *devp)
                 return IRQ_HANDLED;
         }
 
-        if (!complete_cmd_fusion(instance)) {
+        if (!complete_cmd_fusion(instance, irq_context->MSIxIndex)) {
                 instance->instancet->clear_intr(instance->reg_set);
                 /* If we didn't complete any commands, check for FW fault */
                 fw_state = instance->instancet->read_fw_status_reg(
@@ -2081,14 +2102,16 @@ out:
 
 void  megasas_reset_reply_desc(struct megasas_instance *instance)
 {
-        int i;
+        int i, count;
         struct fusion_context *fusion;
         union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
 
         fusion = instance->ctrl_context;
-        fusion->last_reply_idx = 0;
+        count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
+        for (i = 0 ; i < count ; i++)
+                fusion->last_reply_idx[i] = 0;
         reply_desc = fusion->reply_frames_desc;
-        for (i = 0 ; i < fusion->reply_q_depth; i++, reply_desc++)
+        for (i = 0 ; i < fusion->reply_q_depth * count; i++, reply_desc++)
                 reply_desc->Words = ULLONG_MAX;
 }
 
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.h b/drivers/scsi/megaraid/megaraid_sas_fusion.h
index 9556c08490b2..088c9f91da95 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.h
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.h
@@ -43,6 +43,7 @@
 #define HOST_DIAG_WRITE_ENABLE                      0x80
 #define HOST_DIAG_RESET_ADAPTER                     0x4
 #define MEGASAS_FUSION_MAX_RESET_TRIES              3
+#define MAX_MSIX_QUEUES_FUSION                      16
 
 /* Invader defines */
 #define MPI2_TYPE_CUDA                              0x2
@@ -673,7 +674,7 @@ struct fusion_context {
         union MPI2_REPLY_DESCRIPTORS_UNION *reply_frames_desc;
         struct dma_pool *reply_frames_desc_pool;
 
-        u16 last_reply_idx;
+        u16 last_reply_idx[MAX_MSIX_QUEUES_FUSION];
 
         u32 reply_q_depth;
         u32 request_alloc_sz;