scsi: lpfc: Resize cpu maps structures based on possible cpus

The work done to date utilized the number of present cpus when sizing
per-cpu structures. Structures should have been sized based on the max
possible cpu count.

Convert the driver over to possible cpu count for sizing allocation.

Signed-off-by: Dick Kennedy <dick.kennedy@broadcom.com>
Signed-off-by: James Smart <jsmart2021@gmail.com>
Reviewed-by: Hannes Reinecke <hare@suse.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

4 files changed, 51 insertions, 41 deletions

diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index 2864cb53b1e8..a114965a376c 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -5176,16 +5176,22 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr,
         case 1:
                 len += snprintf(buf + len, PAGE_SIZE-len,
                                 "fcp_cpu_map: HBA centric mapping (%d): "
-                                "%d online CPUs\n",
-                                phba->cfg_fcp_cpu_map,
-                                phba->sli4_hba.num_online_cpu);
+                                "%d of %d CPUs online from %d possible CPUs\n",
+                                phba->cfg_fcp_cpu_map, num_online_cpus(),
+                                num_present_cpus(),
+                                phba->sli4_hba.num_possible_cpu);
                 break;
         }
 
-        while (phba->sli4_hba.curr_disp_cpu < phba->sli4_hba.num_present_cpu) {
+        while (phba->sli4_hba.curr_disp_cpu <
+               phba->sli4_hba.num_possible_cpu) {
                 cpup = &phba->sli4_hba.cpu_map[phba->sli4_hba.curr_disp_cpu];
 
-                if (cpup->irq == LPFC_VECTOR_MAP_EMPTY) {
+                if (!cpu_present(phba->sli4_hba.curr_disp_cpu))
+                        len += snprintf(buf + len, PAGE_SIZE - len,
+                                        "CPU %02d not present\n",
+                                        phba->sli4_hba.curr_disp_cpu);
+                else if (cpup->irq == LPFC_VECTOR_MAP_EMPTY) {
                         if (cpup->hdwq == LPFC_VECTOR_MAP_EMPTY)
                                 len += snprintf(
                                         buf + len, PAGE_SIZE - len,
@@ -5225,14 +5231,15 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr,
 
                 /* display max number of CPUs keeping some margin */
                 if (phba->sli4_hba.curr_disp_cpu <
-                                phba->sli4_hba.num_present_cpu &&
+                                phba->sli4_hba.num_possible_cpu &&
                                 (len >= (PAGE_SIZE - 64))) {
-                        len += snprintf(buf + len, PAGE_SIZE-len, "more...\n");
+                        len += snprintf(buf + len,
+                                        PAGE_SIZE - len, "more...\n");
                         break;
                 }
         }
 
-        if (phba->sli4_hba.curr_disp_cpu == phba->sli4_hba.num_present_cpu)
+        if (phba->sli4_hba.curr_disp_cpu == phba->sli4_hba.num_possible_cpu)
                 phba->sli4_hba.curr_disp_cpu = 0;
 
         return len;
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 05919480e430..8ba2861db7b6 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -6373,8 +6373,8 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
         u32 if_type;
         u32 if_fam;
 
-        phba->sli4_hba.num_online_cpu = num_online_cpus();
         phba->sli4_hba.num_present_cpu = lpfc_present_cpu;
+        phba->sli4_hba.num_possible_cpu = num_possible_cpus();
         phba->sli4_hba.curr_disp_cpu = 0;
 
         /* Get all the module params for configuring this host */
@@ -6796,7 +6796,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
                 goto out_free_fcf_rr_bmask;
         }
 
-        phba->sli4_hba.cpu_map = kcalloc(phba->sli4_hba.num_present_cpu,
+        phba->sli4_hba.cpu_map = kcalloc(phba->sli4_hba.num_possible_cpu,
                                         sizeof(struct lpfc_vector_map_info),
                                         GFP_KERNEL);
         if (!phba->sli4_hba.cpu_map) {
@@ -6868,8 +6868,8 @@ lpfc_sli4_driver_resource_unset(struct lpfc_hba *phba)
 
         /* Free memory allocated for msi-x interrupt vector to CPU mapping */
         kfree(phba->sli4_hba.cpu_map);
+        phba->sli4_hba.num_possible_cpu = 0;
         phba->sli4_hba.num_present_cpu = 0;
-        phba->sli4_hba.num_online_cpu = 0;
         phba->sli4_hba.curr_disp_cpu = 0;
 
         /* Free memory allocated for fast-path work queue handles */
@@ -10519,15 +10519,14 @@ lpfc_find_cpu_handle(struct lpfc_hba *phba, uint16_t id, int match)
         int cpu;
 
         /* Find the desired phys_id for the specified EQ */
-        cpup = phba->sli4_hba.cpu_map;
-        for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) {
+        for_each_present_cpu(cpu) {
+                cpup = &phba->sli4_hba.cpu_map[cpu];
                 if ((match == LPFC_FIND_BY_EQ) &&
                     (cpup->irq != LPFC_VECTOR_MAP_EMPTY) &&
                     (cpup->eq == id))
                         return cpu;
                 if ((match == LPFC_FIND_BY_HDWQ) && (cpup->hdwq == id))
                         return cpu;
-                cpup++;
         }
         return 0;
 }
@@ -10545,11 +10544,10 @@ lpfc_find_eq_handle(struct lpfc_hba *phba, uint16_t hdwq)
         int cpu;
 
         /* Find the desired phys_id for the specified EQ */
-        cpup = phba->sli4_hba.cpu_map;
-        for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) {
+        for_each_present_cpu(cpu) {
+                cpup = &phba->sli4_hba.cpu_map[cpu];
                 if (cpup->hdwq == hdwq)
                         return cpup->eq;
-                cpup++;
         }
         return 0;
 }
@@ -10569,15 +10567,13 @@ lpfc_find_hyper(struct lpfc_hba *phba, int cpu,
         struct lpfc_vector_map_info *cpup;
         int idx;
 
-        cpup = phba->sli4_hba.cpu_map;
-        for (idx = 0; idx < phba->sli4_hba.num_present_cpu; idx++) {
+        for_each_present_cpu(idx) {
+                cpup = &phba->sli4_hba.cpu_map[idx];
                 /* Does the cpup match the one we are looking for */
                 if ((cpup->phys_id == phys_id) &&
                     (cpup->core_id == core_id) &&
-                    (cpu != idx)) {
+                    (cpu != idx))
                         return 1;
-                }
-                cpup++;
         }
         return 0;
 }
@@ -10608,7 +10604,7 @@ lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors)
         /* Init cpu_map array */
         memset(phba->sli4_hba.cpu_map, 0xff,
                (sizeof(struct lpfc_vector_map_info) *
-               phba->sli4_hba.num_present_cpu));
+               phba->sli4_hba.num_possible_cpu));
 
         max_phys_id = 0;
         min_phys_id = 0xffff;
@@ -10617,8 +10613,8 @@ lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors)
         phys_id = 0;
 
         /* Update CPU map with physical id and core id of each CPU */
-        cpup = phba->sli4_hba.cpu_map;
-        for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) {
+        for_each_present_cpu(cpu) {
+                cpup = &phba->sli4_hba.cpu_map[cpu];
 #ifdef CONFIG_X86
                 cpuinfo = &cpu_data(cpu);
                 cpup->phys_id = cpuinfo->phys_proc_id;
@@ -10645,8 +10641,6 @@ lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors)
                         max_core_id = cpup->core_id;
                 if (cpup->core_id < min_core_id)
                         min_core_id = cpup->core_id;
-
-                cpup++;
         }
 
         for_each_possible_cpu(i) {
diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c
index 0d296aee2d82..0b27e8c5ae32 100644
--- a/drivers/scsi/lpfc/lpfc_nvmet.c
+++ b/drivers/scsi/lpfc/lpfc_nvmet.c
@@ -1194,9 +1194,9 @@ lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba)
 
         /* Cycle the the entire CPU context list for every MRQ */
         for (i = 0; i < phba->cfg_nvmet_mrq; i++) {
-                for (j = 0; j < phba->sli4_hba.num_present_cpu; j++) {
+                for_each_present_cpu(j) {
+                        infop = lpfc_get_ctx_list(phba, j, i);
                         __lpfc_nvmet_clean_io_for_cpu(phba, infop);
-                        infop++; /* next */
                 }
         }
         kfree(phba->sli4_hba.nvmet_ctx_info);
@@ -1211,14 +1211,14 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
         union lpfc_wqe128 *wqe;
         struct lpfc_nvmet_ctx_info *last_infop;
         struct lpfc_nvmet_ctx_info *infop;
-        int i, j, idx;
+        int i, j, idx, cpu;
 
         lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
                         "6403 Allocate NVMET resources for %d XRIs\n",
                         phba->sli4_hba.nvmet_xri_cnt);
 
         phba->sli4_hba.nvmet_ctx_info = kcalloc(
-                phba->sli4_hba.num_present_cpu * phba->cfg_nvmet_mrq,
+                phba->sli4_hba.num_possible_cpu * phba->cfg_nvmet_mrq,
                 sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL);
         if (!phba->sli4_hba.nvmet_ctx_info) {
                 lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
@@ -1246,13 +1246,12 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
          * of the IO completion. Thus a context that was allocated for MRQ A
          * whose IO completed on CPU B will be freed to cpuB/mrqA.
          */
-        infop = phba->sli4_hba.nvmet_ctx_info;
-        for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
+        for_each_possible_cpu(i) {
                 for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
+                        infop = lpfc_get_ctx_list(phba, i, j);
                         INIT_LIST_HEAD(&infop->nvmet_ctx_list);
                         spin_lock_init(&infop->nvmet_ctx_list_lock);
                         infop->nvmet_ctx_list_cnt = 0;
-                        infop++;
                 }
         }
 
@@ -1262,8 +1261,10 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
          * MRQ 1 cycling thru CPUs 0 - X, and so on.
          */
         for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
-                last_infop = lpfc_get_ctx_list(phba, 0, j);
-                for (i = phba->sli4_hba.num_present_cpu - 1;  i >= 0; i--) {
+                last_infop = lpfc_get_ctx_list(phba,
+                                               cpumask_first(cpu_present_mask),
+                                               j);
+                for (i = phba->sli4_hba.num_possible_cpu - 1;  i >= 0; i--) {
                         infop = lpfc_get_ctx_list(phba, i, j);
                         infop->nvmet_ctx_next_cpu = last_infop;
                         last_infop = infop;
@@ -1274,6 +1275,7 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
          * received command on a per xri basis.
          */
         idx = 0;
+        cpu = cpumask_first(cpu_present_mask);
         for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) {
                 ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL);
                 if (!ctx_buf) {
@@ -1327,7 +1329,7 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
                  * is MRQidx will be associated with CPUidx. This association
                  * can change on the fly.
                  */
-                infop = lpfc_get_ctx_list(phba, idx, idx);
+                infop = lpfc_get_ctx_list(phba, cpu, idx);
                 spin_lock(&infop->nvmet_ctx_list_lock);
                 list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list);
                 infop->nvmet_ctx_list_cnt++;
@@ -1335,11 +1337,18 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
 
                 /* Spread ctx structures evenly across all MRQs */
                 idx++;
-                if (idx >= phba->cfg_nvmet_mrq)
+                if (idx >= phba->cfg_nvmet_mrq) {
                         idx = 0;
+                        cpu = cpumask_first(cpu_present_mask);
+                        continue;
+                }
+                cpu = cpumask_next(cpu, cpu_present_mask);
+                if (cpu == nr_cpu_ids)
+                        cpu = cpumask_first(cpu_present_mask);
+
         }
 
-        for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
+        for_each_present_cpu(i) {
                 for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
                         infop = lpfc_get_ctx_list(phba, i, j);
                         lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT,
@@ -1839,7 +1848,7 @@ lpfc_nvmet_replenish_context(struct lpfc_hba *phba,
         else
                 get_infop = current_infop->nvmet_ctx_next_cpu;
 
-        for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
+        for (i = 0; i < phba->sli4_hba.num_possible_cpu; i++) {
                 if (get_infop == current_infop) {
                         get_infop = get_infop->nvmet_ctx_next_cpu;
                         continue;
diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h
index 20566c506e5f..1e3d7f534eaa 100644
--- a/drivers/scsi/lpfc/lpfc_sli4.h
+++ b/drivers/scsi/lpfc/lpfc_sli4.h
@@ -890,7 +890,7 @@ struct lpfc_sli4_hba {
 
         /* CPU to vector mapping information */
         struct lpfc_vector_map_info *cpu_map;
-        uint16_t num_online_cpu;
+        uint16_t num_possible_cpu;
         uint16_t num_present_cpu;
         uint16_t curr_disp_cpu;
         struct lpfc_eq_intr_info __percpu *eq_info;