[SCSI] bfa: DMA memory allocation enhancement.

- Modified the design such that each BFA sub-module will provide
  the amount of DMA and KVA memory needed by it and queues the
  same request to the global dma and kva info queues.
- During the memory allocation we iterate over this queue to allocate
  the dma and kva memory requested by sub-modules.
- The change is needed to avoid requesting the aggregate amount of memory
  needed by all the BFA sub-modules as one contiguous chunk.

Signed-off-by: Krishna Gudipati <kgudipat@brocade.com>
Signed-off-by: James Bottomley <JBottomley@Parallels.com>

1 files changed, 75 insertions, 44 deletions

diff --git a/drivers/scsi/bfa/bfa_fcpim.c b/drivers/scsi/bfa/bfa_fcpim.c
index eb14fd6193a0..27eab36f89a5 100644
--- a/drivers/scsi/bfa/bfa_fcpim.c
+++ b/drivers/scsi/bfa/bfa_fcpim.c
@@ -286,10 +286,9 @@ static void     bfa_tskim_sm_hcb(struct bfa_tskim_s *tskim,
  * Compute and return memory needed by FCP(im) module.
  */
 static void
-bfa_fcpim_meminfo(struct bfa_iocfc_cfg_s *cfg, u32 *km_len,
-                u32 *dm_len)
+bfa_fcpim_meminfo(struct bfa_iocfc_cfg_s *cfg, u32 *km_len)
 {
-        bfa_itnim_meminfo(cfg, km_len, dm_len);
+        bfa_itnim_meminfo(cfg, km_len);
 
         /*
          * IO memory
@@ -308,8 +307,7 @@ bfa_fcpim_meminfo(struct bfa_iocfc_cfg_s *cfg, u32 *km_len,
 
 static void
 bfa_fcpim_attach(struct bfa_fcp_mod_s *fcp, void *bfad,
-                struct bfa_iocfc_cfg_s *cfg, struct bfa_meminfo_s *meminfo,
-                struct bfa_pcidev_s *pcidev)
+                struct bfa_iocfc_cfg_s *cfg, struct bfa_pcidev_s *pcidev)
 {
         struct bfa_fcpim_s *fcpim = &fcp->fcpim;
         struct bfa_s *bfa = fcp->bfa;
@@ -328,9 +326,9 @@ bfa_fcpim_attach(struct bfa_fcp_mod_s *fcp, void *bfad,
         fcpim->profile_comp = NULL;
         fcpim->profile_start = NULL;
 
-        bfa_itnim_attach(fcpim, meminfo);
-        bfa_tskim_attach(fcpim, meminfo);
-        bfa_ioim_attach(fcpim, meminfo);
+        bfa_itnim_attach(fcpim);
+        bfa_tskim_attach(fcpim);
+        bfa_ioim_attach(fcpim);
 }
 
 static void
@@ -972,8 +970,7 @@ bfa_itnim_tskdone(struct bfa_itnim_s *itnim)
 }
 
 void
-bfa_itnim_meminfo(struct bfa_iocfc_cfg_s *cfg, u32 *km_len,
-                u32 *dm_len)
+bfa_itnim_meminfo(struct bfa_iocfc_cfg_s *cfg, u32 *km_len)
 {
         /*
          * ITN memory
@@ -982,15 +979,16 @@ bfa_itnim_meminfo(struct bfa_iocfc_cfg_s *cfg, u32 *km_len,
 }
 
 void
-bfa_itnim_attach(struct bfa_fcpim_s *fcpim, struct bfa_meminfo_s *minfo)
+bfa_itnim_attach(struct bfa_fcpim_s *fcpim)
 {
         struct bfa_s    *bfa = fcpim->bfa;
+        struct bfa_fcp_mod_s    *fcp = fcpim->fcp;
         struct bfa_itnim_s *itnim;
         int     i, j;
 
         INIT_LIST_HEAD(&fcpim->itnim_q);
 
-        itnim = (struct bfa_itnim_s *) bfa_meminfo_kva(minfo);
+        itnim = (struct bfa_itnim_s *) bfa_mem_kva_curp(fcp);
         fcpim->itnim_arr = itnim;
 
         for (i = 0; i < fcpim->num_itnims; i++, itnim++) {
@@ -1012,7 +1010,7 @@ bfa_itnim_attach(struct bfa_fcpim_s *fcpim, struct bfa_meminfo_s *minfo)
                 bfa_sm_set_state(itnim, bfa_itnim_sm_uninit);
         }
 
-        bfa_meminfo_kva(minfo) = (u8 *) itnim;
+        bfa_mem_kva_curp(fcp) = (u8 *) itnim;
 }
 
 void
@@ -2345,22 +2343,23 @@ bfa_ioim_delayed_comp(struct bfa_ioim_s *ioim, bfa_boolean_t iotov)
  * Memory allocation and initialization.
  */
 void
-bfa_ioim_attach(struct bfa_fcpim_s *fcpim, struct bfa_meminfo_s *minfo)
+bfa_ioim_attach(struct bfa_fcpim_s *fcpim)
 {
         struct bfa_ioim_s               *ioim;
+        struct bfa_fcp_mod_s    *fcp = fcpim->fcp;
         struct bfa_ioim_sp_s    *iosp;
         u16             i;
 
         /*
          * claim memory first
          */
-        ioim = (struct bfa_ioim_s *) bfa_meminfo_kva(minfo);
+        ioim = (struct bfa_ioim_s *) bfa_mem_kva_curp(fcp);
         fcpim->ioim_arr = ioim;
-        bfa_meminfo_kva(minfo) = (u8 *) (ioim + fcpim->fcp->num_ioim_reqs);
+        bfa_mem_kva_curp(fcp) = (u8 *) (ioim + fcpim->fcp->num_ioim_reqs);
 
-        iosp = (struct bfa_ioim_sp_s *) bfa_meminfo_kva(minfo);
+        iosp = (struct bfa_ioim_sp_s *) bfa_mem_kva_curp(fcp);
         fcpim->ioim_sp_arr = iosp;
-        bfa_meminfo_kva(minfo) = (u8 *) (iosp + fcpim->fcp->num_ioim_reqs);
+        bfa_mem_kva_curp(fcp) = (u8 *) (iosp + fcpim->fcp->num_ioim_reqs);
 
         /*
          * Initialize ioim free queues
@@ -3109,15 +3108,16 @@ bfa_tskim_cleanup(struct bfa_tskim_s *tskim)
  * Memory allocation and initialization.
  */
 void
-bfa_tskim_attach(struct bfa_fcpim_s *fcpim, struct bfa_meminfo_s *minfo)
+bfa_tskim_attach(struct bfa_fcpim_s *fcpim)
 {
         struct bfa_tskim_s *tskim;
+        struct bfa_fcp_mod_s    *fcp = fcpim->fcp;
         u16     i;
 
         INIT_LIST_HEAD(&fcpim->tskim_free_q);
         INIT_LIST_HEAD(&fcpim->tskim_unused_q);
 
-        tskim = (struct bfa_tskim_s *) bfa_meminfo_kva(minfo);
+        tskim = (struct bfa_tskim_s *) bfa_mem_kva_curp(fcp);
         fcpim->tskim_arr = tskim;
 
         for (i = 0; i < fcpim->num_tskim_reqs; i++, tskim++) {
@@ -3136,7 +3136,7 @@ bfa_tskim_attach(struct bfa_fcpim_s *fcpim, struct bfa_meminfo_s *minfo)
                 list_add_tail(&tskim->qe, &fcpim->tskim_free_q);
         }
 
-        bfa_meminfo_kva(minfo) = (u8 *) tskim;
+        bfa_mem_kva_curp(fcp) = (u8 *) tskim;
 }
 
 void
@@ -3233,9 +3233,14 @@ bfa_tskim_res_recfg(struct bfa_s *bfa, u16 num_tskim_fw)
 BFA_MODULE(fcp);
 
 static void
-bfa_fcp_meminfo(struct bfa_iocfc_cfg_s *cfg, u32 *km_len, u32 *dm_len)
+bfa_fcp_meminfo(struct bfa_iocfc_cfg_s *cfg, struct bfa_meminfo_s *minfo,
+                struct bfa_s *bfa)
 {
-        u16     num_io_req;
+        struct bfa_fcp_mod_s *fcp = BFA_FCP_MOD(bfa);
+        struct bfa_mem_kva_s *fcp_kva = BFA_MEM_FCP_KVA(bfa);
+        struct bfa_mem_dma_s *seg_ptr;
+        u16     nsegs, idx, per_seg_ios, num_io_req;
+        u32     km_len = 0;
 
         /*
          * ZERO for num_ioim_reqs and num_fwtio_reqs is allowed config value.
@@ -3261,43 +3266,69 @@ bfa_fcp_meminfo(struct bfa_iocfc_cfg_s *cfg, u32 *km_len, u32 *dm_len)
                         cfg->fwcfg.num_ioim_reqs = BFA_IOIM_MAX;
         }
 
-        bfa_fcpim_meminfo(cfg, km_len, dm_len);
+        bfa_fcpim_meminfo(cfg, &km_len);
 
         num_io_req = (cfg->fwcfg.num_ioim_reqs + cfg->fwcfg.num_fwtio_reqs);
-        *km_len += num_io_req * sizeof(struct bfa_iotag_s);
-        *km_len += cfg->fwcfg.num_rports * sizeof(struct bfa_itn_s);
-        *dm_len += num_io_req * BFI_IOIM_SNSLEN;
+        km_len += num_io_req * sizeof(struct bfa_iotag_s);
+        km_len += cfg->fwcfg.num_rports * sizeof(struct bfa_itn_s);
+
+        /* dma memory */
+        nsegs = BFI_MEM_DMA_NSEGS(num_io_req, BFI_IOIM_SNSLEN);
+        per_seg_ios = BFI_MEM_NREQS_SEG(BFI_IOIM_SNSLEN);
+
+        bfa_mem_dma_seg_iter(fcp, seg_ptr, nsegs, idx) {
+                if (num_io_req >= per_seg_ios) {
+                        num_io_req -= per_seg_ios;
+                        bfa_mem_dma_setup(minfo, seg_ptr,
+                                per_seg_ios * BFI_IOIM_SNSLEN);
+                } else
+                        bfa_mem_dma_setup(minfo, seg_ptr,
+                                num_io_req * BFI_IOIM_SNSLEN);
+        }
+
+        /* kva memory */
+        bfa_mem_kva_setup(minfo, fcp_kva, km_len);
 }
 
 static void
 bfa_fcp_attach(struct bfa_s *bfa, void *bfad, struct bfa_iocfc_cfg_s *cfg,
-                struct bfa_meminfo_s *meminfo, struct bfa_pcidev_s *pcidev)
+                struct bfa_pcidev_s *pcidev)
 {
         struct bfa_fcp_mod_s *fcp = BFA_FCP_MOD(bfa);
-        u32     snsbufsz;
+        struct bfa_mem_dma_s *seg_ptr;
+        u16     idx, nsegs, num_io_req;
 
         fcp->num_ioim_reqs = cfg->fwcfg.num_ioim_reqs;
         fcp->num_fwtio_reqs  = cfg->fwcfg.num_fwtio_reqs;
-        fcp->num_itns   = cfg->fwcfg.num_rports;
+        fcp->num_itns   = cfg->fwcfg.num_rports;
         fcp->bfa = bfa;
 
-        snsbufsz = (fcp->num_ioim_reqs + fcp->num_fwtio_reqs) * BFI_IOIM_SNSLEN;
-        fcp->snsbase.pa = bfa_meminfo_dma_phys(meminfo);
-        bfa_meminfo_dma_phys(meminfo) += snsbufsz;
+        /*
+         * Setup the pool of snsbase addr's, that is passed to fw as
+         * part of bfi_iocfc_cfg_s.
+         */
+        num_io_req = (cfg->fwcfg.num_ioim_reqs + cfg->fwcfg.num_fwtio_reqs);
+        nsegs = BFI_MEM_DMA_NSEGS(num_io_req, BFI_IOIM_SNSLEN);
+
+        bfa_mem_dma_seg_iter(fcp, seg_ptr, nsegs, idx) {
+
+                if (!bfa_mem_dma_virt(seg_ptr))
+                        break;
+
+                fcp->snsbase[idx].pa = bfa_mem_dma_phys(seg_ptr);
+                fcp->snsbase[idx].kva = bfa_mem_dma_virt(seg_ptr);
+                bfa_iocfc_set_snsbase(bfa, idx, fcp->snsbase[idx].pa);
+        }
 
-        fcp->snsbase.kva = bfa_meminfo_dma_virt(meminfo);
-        bfa_meminfo_dma_virt(meminfo) += snsbufsz;
-        bfa_iocfc_set_snsbase(bfa, fcp->snsbase.pa);
+        bfa_fcpim_attach(fcp, bfad, cfg, pcidev);
 
-        bfa_fcpim_attach(fcp, bfad, cfg, meminfo, pcidev);
+        bfa_iotag_attach(fcp);
 
-        fcp->itn_arr = (struct bfa_itn_s *) bfa_meminfo_kva(meminfo);
-        bfa_meminfo_kva(meminfo) = (u8 *)fcp->itn_arr +
+        fcp->itn_arr = (struct bfa_itn_s *) bfa_mem_kva_curp(fcp);
+        bfa_mem_kva_curp(fcp) = (u8 *)fcp->itn_arr +
                         (fcp->num_itns * sizeof(struct bfa_itn_s));
         memset(fcp->itn_arr, 0,
                         (fcp->num_itns * sizeof(struct bfa_itn_s)));
-
-        bfa_iotag_attach(fcp, meminfo);
 }
 
 static void
@@ -3370,12 +3401,12 @@ bfa_itn_isr(struct bfa_s *bfa, struct bfi_msg_s *m)
 }
 
 void
-bfa_iotag_attach(struct bfa_fcp_mod_s *fcp, struct bfa_meminfo_s *minfo)
+bfa_iotag_attach(struct bfa_fcp_mod_s *fcp)
 {
         struct bfa_iotag_s *iotag;
         u16     num_io_req, i;
 
-        iotag = (struct bfa_iotag_s *) bfa_meminfo_kva(minfo);
+        iotag = (struct bfa_iotag_s *) bfa_mem_kva_curp(fcp);
         fcp->iotag_arr = iotag;
 
         INIT_LIST_HEAD(&fcp->iotag_ioim_free_q);
@@ -3392,5 +3423,5 @@ bfa_iotag_attach(struct bfa_fcp_mod_s *fcp, struct bfa_meminfo_s *minfo)
                         list_add_tail(&iotag->qe, &fcp->iotag_tio_free_q);
         }
 
-        bfa_meminfo_kva(minfo) = (u8 *) iotag;
+        bfa_mem_kva_curp(fcp) = (u8 *) iotag;
 }