gpu: nvgpu: add platform specific get_iova_addr()

Add platform specific API pointer (*get_iova_addr)()
which can be used to get iova/physical address from
given scatterlist and flags

Use this API with g->ops.mm.get_iova_addr() instead
of calling API gk20a_mm_iova_addr() which makes it
platform specific

Bug 1605653

Change-Id: I798763db1501bd0b16e84daab68f6093a83caac2
Signed-off-by: Deepak Nibade <dnibade@nvidia.com>
Reviewed-on: http://git-master/r/713089
GVS: Gerrit_Virtual_Submit
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>

9 files changed, 48 insertions, 30 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index 88e3f60a..9d323803 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -519,7 +519,7 @@ static int gk20a_init_fifo_setup_sw(struct gk20a *g)
                 f->channel[chid].userd_cpu_va =
                         f->userd.cpu_va + chid * f->userd_entry_size;
                 f->channel[chid].userd_iova =
-                        gk20a_mm_iova_addr(g, f->userd.sgt->sgl)
+                        g->ops.mm.get_iova_addr(g, f->userd.sgt->sgl, 0)
                                 + chid * f->userd_entry_size;
                 f->channel[chid].userd_gpu_va =
                         f->userd.gpu_va + chid * f->userd_entry_size;
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index 015c8274..19c9e786 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -349,6 +349,8 @@ struct gpu_ops {
                 const struct gk20a_mmu_level *
                         (*get_mmu_levels)(struct gk20a *g, u32 big_page_size);
                 void (*init_pdb)(struct gk20a *g, void *inst_ptr, u64 pdb_addr);
+                u64 (*get_iova_addr)(struct gk20a *g, struct scatterlist *sgl,
+                                         u32 flags);
         } mm;
         struct {
                 int (*prepare_ucode)(struct gk20a *g);
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
index 1e6cca6d..e4024f66 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
@@ -4142,7 +4142,7 @@ static int gk20a_init_gr_setup_hw(struct gk20a *g)
         gk20a_dbg_fn("");
 
         /* init mmu debug buffer */
-        addr = gk20a_mm_iova_addr(g, gr->mmu_wr_mem.sgt->sgl);
+        addr = g->ops.mm.get_iova_addr(g, gr->mmu_wr_mem.sgt->sgl, 0);
         addr >>= fb_mmu_debug_wr_addr_alignment_v();
 
         gk20a_writel(g, fb_mmu_debug_wr_r(),
@@ -4150,7 +4150,7 @@ static int gk20a_init_gr_setup_hw(struct gk20a *g)
                      fb_mmu_debug_wr_vol_false_f() |
                      fb_mmu_debug_wr_addr_f(addr));
 
-        addr = gk20a_mm_iova_addr(g, gr->mmu_rd_mem.sgt->sgl);
+        addr = g->ops.mm.get_iova_addr(g, gr->mmu_rd_mem.sgt->sgl, 0);
         addr >>= fb_mmu_debug_rd_addr_alignment_v();
 
         gk20a_writel(g, fb_mmu_debug_rd_r(),
diff --git a/drivers/gpu/nvgpu/gk20a/ltc_common.c b/drivers/gpu/nvgpu/gk20a/ltc_common.c
index f48f8298..6fbd0c2d 100644
--- a/drivers/gpu/nvgpu/gk20a/ltc_common.c
+++ b/drivers/gpu/nvgpu/gk20a/ltc_common.c
@@ -106,8 +106,8 @@ static void gk20a_ltc_init_cbc(struct gk20a *g, struct gr_gk20a *gr)
         if (tegra_platform_is_linsim())
                 compbit_store_iova = gk20a_mem_phys(&gr->compbit_store.mem);
         else
-                compbit_store_iova = gk20a_mm_iova_addr(g,
-                                gr->compbit_store.mem.sgt->sgl);
+                compbit_store_iova = g->ops.mm.get_iova_addr(g,
+                                gr->compbit_store.mem.sgt->sgl, 0);
 
         compbit_base_post_divide64 = compbit_store_iova >>
                 ltc_ltcs_ltss_cbc_base_alignment_shift_v();
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
index 9a61a779..3d2e5450 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
@@ -99,7 +99,8 @@ static int update_gmmu_ptes_locked(struct vm_gk20a *vm,
                                    u64 first_vaddr, u64 last_vaddr,
                                    u8 kind_v, u32 ctag_offset, bool cacheable,
                                    bool umapped_pte, int rw_flag,
-                                   bool sparse);
+                                   bool sparse,
+                                   u32 flags);
 static int __must_check gk20a_init_system_vm(struct mm_gk20a *mm);
 static int __must_check gk20a_init_bar1_vm(struct mm_gk20a *mm);
 static int __must_check gk20a_init_hwpm(struct mm_gk20a *mm);
@@ -610,6 +611,7 @@ static int gk20a_zalloc_gmmu_page_table(struct vm_gk20a *vm,
 {
         int err;
         int order;
+        struct gk20a *g = gk20a_from_vm(vm);
 
         gk20a_dbg_fn("");
 
@@ -621,7 +623,8 @@ static int gk20a_zalloc_gmmu_page_table(struct vm_gk20a *vm,
 
         err = alloc_gmmu_pages(vm, order, entry);
         gk20a_dbg(gpu_dbg_pte, "entry = 0x%p, addr=%08llx, size %d",
-                  entry, gk20a_mm_iova_addr(vm->mm->g, entry->sgt->sgl), order);
+                  entry, g->ops.mm.get_iova_addr(g, entry->sgt->sgl, 0),
+                  order);
         if (err)
                 return err;
         entry->pgsz = pgsz_idx;
@@ -1118,7 +1121,8 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
                                       flags &
                                       NVGPU_GPU_FLAGS_SUPPORT_UNMAPPED_PTE,
                                       rw_flag,
-                                      sparse);
+                                      sparse,
+                                      flags);
         if (err) {
                 gk20a_err(d, "failed to update ptes on map");
                 goto fail_validate;
@@ -1164,7 +1168,7 @@ void gk20a_locked_gmmu_unmap(struct vm_gk20a *vm,
                                 vaddr + size,
                                 0, 0, false /* n/a for unmap */,
                                 false, rw_flag,
-                                sparse);
+                                sparse, 0);
         if (err)
                 dev_err(dev_from_vm(vm),
                         "failed to update gmmu ptes on unmap");
@@ -1645,11 +1649,13 @@ dma_addr_t gk20a_mm_gpuva_to_iova_base(struct vm_gk20a *vm, u64 gpu_vaddr)
 {
         struct mapped_buffer_node *buffer;
         dma_addr_t addr = 0;
+        struct gk20a *g = gk20a_from_vm(vm);
 
         mutex_lock(&vm->update_gmmu_lock);
         buffer = find_mapped_buffer_locked(&vm->mapped_buffers, gpu_vaddr);
         if (buffer)
-                addr = gk20a_mm_iova_addr(vm->mm->g, buffer->sgt->sgl);
+                addr = g->ops.mm.get_iova_addr(g, buffer->sgt->sgl,
+                                buffer->flags);
         mutex_unlock(&vm->update_gmmu_lock);
 
         return addr;
@@ -1761,7 +1767,8 @@ u64 gk20a_mm_smmu_vaddr_translate(struct gk20a *g, dma_addr_t iova)
                 return iova | 1ULL << g->ops.mm.get_physical_addr_bits(g);
 }
 
-u64 gk20a_mm_iova_addr(struct gk20a *g, struct scatterlist *sgl)
+u64 gk20a_mm_iova_addr(struct gk20a *g, struct scatterlist *sgl,
+                u32 flags)
 {
         if (!device_is_iommuable(dev_from_gk20a(g)))
                 return sg_phys(sgl);
@@ -1807,8 +1814,9 @@ static int update_gmmu_pde_locked(struct vm_gk20a *vm,
                            u64 iova,
                            u32 kind_v, u32 *ctag,
                            bool cacheable, bool unammped_pte,
-                           int rw_flag, bool sparse)
+                           int rw_flag, bool sparse, u32 flags)
 {
+        struct gk20a *g = gk20a_from_vm(vm);
         bool small_valid, big_valid;
         u64 pte_addr_small = 0, pte_addr_big = 0;
         struct gk20a_mm_entry *entry = vm->pdb.entries + i;
@@ -1821,10 +1829,10 @@ static int update_gmmu_pde_locked(struct vm_gk20a *vm,
         big_valid   = entry->size && entry->pgsz == gmmu_page_size_big;
 
         if (small_valid)
-                pte_addr_small = gk20a_mm_iova_addr(vm->mm->g, entry->sgt->sgl);
+                pte_addr_small = g->ops.mm.get_iova_addr(g, entry->sgt->sgl, 0);
 
         if (big_valid)
-                pte_addr_big = gk20a_mm_iova_addr(vm->mm->g, entry->sgt->sgl);
+                pte_addr_big = g->ops.mm.get_iova_addr(g, entry->sgt->sgl, 0);
 
         pde_v[0] = gmmu_pde_size_full_f();
         pde_v[0] |= big_valid ? big_valid_pde0_bits(pte_addr_big) :
@@ -1854,7 +1862,7 @@ static int update_gmmu_pte_locked(struct vm_gk20a *vm,
                            u64 iova,
                            u32 kind_v, u32 *ctag,
                            bool cacheable, bool unmapped_pte,
-                           int rw_flag, bool sparse)
+                           int rw_flag, bool sparse, u32 flags)
 {
         struct gk20a *g = gk20a_from_vm(vm);
         u32 ctag_granularity = g->ops.fb.compression_page_size(g);
@@ -1927,7 +1935,8 @@ static int update_gmmu_level_locked(struct vm_gk20a *vm,
                                     bool cacheable, bool unmapped_pte,
                                     int rw_flag,
                                     bool sparse,
-                                    int lvl)
+                                    int lvl,
+                                    u32 flags)
 {
         const struct gk20a_mmu_level *l = &vm->mmu_levels[lvl];
         const struct gk20a_mmu_level *next_l = &vm->mmu_levels[lvl+1];
@@ -1973,7 +1982,7 @@ static int update_gmmu_level_locked(struct vm_gk20a *vm,
 
                 err = l->update_entry(vm, pte, pde_i, pgsz_idx,
                                 iova, kind_v, ctag, cacheable, unmapped_pte,
-                                rw_flag, sparse);
+                                rw_flag, sparse, flags);
                 if (err)
                         return err;
 
@@ -1992,7 +2001,7 @@ static int update_gmmu_level_locked(struct vm_gk20a *vm,
                                 gpu_va,
                                 next,
                                 kind_v, ctag, cacheable, unmapped_pte,
-                                rw_flag, sparse, lvl+1);
+                                rw_flag, sparse, lvl+1, flags);
                         unmap_gmmu_pages(next_pte);
 
                         if (err)
@@ -2018,7 +2027,8 @@ static int update_gmmu_ptes_locked(struct vm_gk20a *vm,
                                    u8 kind_v, u32 ctag_offset,
                                    bool cacheable, bool unmapped_pte,
                                    int rw_flag,
-                                   bool sparse)
+                                   bool sparse,
+                                   u32 flags)
 {
         struct gk20a *g = gk20a_from_vm(vm);
         int ctag_granularity = g->ops.fb.compression_page_size(g);
@@ -2030,13 +2040,15 @@ static int update_gmmu_ptes_locked(struct vm_gk20a *vm,
 
         gk20a_dbg(gpu_dbg_pte, "size_idx=%d, iova=%llx",
                    pgsz_idx,
-                   sgt ? gk20a_mm_iova_addr(vm->mm->g, sgt->sgl) : 0ULL);
+                   sgt ? g->ops.mm.get_iova_addr(vm->mm->g, sgt->sgl, flags)
+                       : 0ULL);
 
         if (space_to_skip & (page_size - 1))
                 return -EINVAL;
 
         if (sgt)
-                iova = gk20a_mm_iova_addr(vm->mm->g, sgt->sgl) + space_to_skip;
+                iova = g->ops.mm.get_iova_addr(vm->mm->g, sgt->sgl, flags)
+                                + space_to_skip;
 
         gk20a_dbg(gpu_dbg_map, "size_idx=%d, gpu_va=[%llx,%llx], iova=%llx",
                         pgsz_idx, gpu_va, gpu_end-1, iova);
@@ -2051,7 +2063,7 @@ static int update_gmmu_ptes_locked(struct vm_gk20a *vm,
                         iova,
                         gpu_va, gpu_end,
                         kind_v, &ctag,
-                        cacheable, unmapped_pte, rw_flag, sparse, 0);
+                        cacheable, unmapped_pte, rw_flag, sparse, 0, flags);
         unmap_gmmu_pages(&vm->pdb);
 
         smp_mb();
@@ -2824,7 +2836,7 @@ void gk20a_init_inst_block(struct mem_desc *inst_block, struct vm_gk20a *vm,
                 u32 big_page_size)
 {
         struct gk20a *g = gk20a_from_vm(vm);
-        u64 pde_addr = gk20a_mm_iova_addr(g, vm->pdb.sgt->sgl);
+        u64 pde_addr = g->ops.mm.get_iova_addr(g, vm->pdb.sgt->sgl, 0);
         phys_addr_t inst_pa = gk20a_mem_phys(inst_block);
         void *inst_ptr = inst_block->cpu_va;
 
@@ -3019,8 +3031,8 @@ int gk20a_vm_find_buffer(struct vm_gk20a *vm, u64 gpu_va,
 void gk20a_mm_tlb_invalidate(struct vm_gk20a *vm)
 {
         struct gk20a *g = gk20a_from_vm(vm);
-        u32 addr_lo = u64_lo32(gk20a_mm_iova_addr(vm->mm->g,
-                                                  vm->pdb.sgt->sgl) >> 12);
+        u32 addr_lo = u64_lo32(g->ops.mm.get_iova_addr(vm->mm->g,
+                                                  vm->pdb.sgt->sgl, 0) >> 12);
         u32 data;
         s32 retry = 2000;
         static DEFINE_MUTEX(tlb_lock);
@@ -3122,6 +3134,7 @@ void gk20a_init_mm(struct gpu_ops *gops)
         gops->mm.l2_invalidate = gk20a_mm_l2_invalidate;
         gops->mm.l2_flush = gk20a_mm_l2_flush;
         gops->mm.tlb_invalidate = gk20a_mm_tlb_invalidate;
+        gops->mm.get_iova_addr = gk20a_mm_iova_addr;
         gops->mm.get_physical_addr_bits = gk20a_mm_get_physical_addr_bits;
         gops->mm.get_mmu_levels = gk20a_mm_get_mmu_levels;
         gops->mm.init_pdb = gk20a_mm_init_pdb;
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
index ea3a2052..b510b472 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
@@ -223,7 +223,7 @@ struct gk20a_mmu_level {
                            u64 iova,
                            u32 kind_v, u32 *ctag,
                            bool cacheable, bool unmapped_pte,
-                           int rw_flag, bool sparse);
+                           int rw_flag, bool sparse, u32 flags);
         size_t entry_size;
 };
 
@@ -375,7 +375,8 @@ int gk20a_get_sgtable_from_pages(struct device *d, struct sg_table **sgt,
 
 void gk20a_free_sgtable(struct sg_table **sgt);
 
-u64 gk20a_mm_iova_addr(struct gk20a *g, struct scatterlist *sgl);
+u64 gk20a_mm_iova_addr(struct gk20a *g, struct scatterlist *sgl,
+                u32 flags);
 u64 gk20a_mm_smmu_vaddr_translate(struct gk20a *g, dma_addr_t iova);
 
 void gk20a_mm_ltc_isr(struct gk20a *g);
diff --git a/drivers/gpu/nvgpu/gm20b/mm_gm20b.c b/drivers/gpu/nvgpu/gm20b/mm_gm20b.c
index 6b9c6da8..cf7ae46b 100644
--- a/drivers/gpu/nvgpu/gm20b/mm_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/mm_gm20b.c
@@ -123,6 +123,7 @@ void gm20b_init_mm(struct gpu_ops *gops)
         gops->mm.tlb_invalidate = gk20a_mm_tlb_invalidate;
         gops->mm.set_big_page_size = gm20b_mm_set_big_page_size;
         gops->mm.get_big_page_sizes = gm20b_mm_get_big_page_sizes;
+        gops->mm.get_iova_addr = gk20a_mm_iova_addr;
         gops->mm.get_physical_addr_bits = gk20a_mm_get_physical_addr_bits;
         gops->mm.get_mmu_levels = gk20a_mm_get_mmu_levels;
         gops->mm.init_pdb = gk20a_mm_init_pdb;
diff --git a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c
index 25e6e4c7..45e7c336 100644
--- a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c
+++ b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c
@@ -287,7 +287,7 @@ static int vgpu_init_fifo_setup_sw(struct gk20a *g)
                 f->channel[chid].userd_cpu_va =
                         f->userd.cpu_va + chid * f->userd_entry_size;
                 f->channel[chid].userd_iova =
-                        gk20a_mm_iova_addr(g, f->userd.sgt->sgl)
+                        g->ops.mm.get_iova_addr(g, f->userd.sgt->sgl, 0)
                                 + chid * f->userd_entry_size;
                 f->channel[chid].userd_gpu_va =
                         f->userd.gpu_va + chid * f->userd_entry_size;
diff --git a/drivers/gpu/nvgpu/vgpu/mm_vgpu.c b/drivers/gpu/nvgpu/vgpu/mm_vgpu.c
index b3a8a885..94e4602f 100644
--- a/drivers/gpu/nvgpu/vgpu/mm_vgpu.c
+++ b/drivers/gpu/nvgpu/vgpu/mm_vgpu.c
@@ -75,7 +75,7 @@ static u64 vgpu_locked_gmmu_map(struct vm_gk20a *vm,
         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(d);
         struct tegra_vgpu_cmd_msg msg;
         struct tegra_vgpu_as_map_params *p = &msg.params.as_map;
-        u64 addr = gk20a_mm_iova_addr(g, sgt->sgl);
+        u64 addr = g->ops.mm.get_iova_addr(g, sgt->sgl, flags);
         u8 prot;
 
         gk20a_dbg_fn("");
@@ -213,7 +213,7 @@ u64 vgpu_bar1_map(struct gk20a *g, struct sg_table **sgt, u64 size)
         struct gk20a_platform *platform = gk20a_get_platform(g->dev);
         struct dma_iommu_mapping *mapping =
                         to_dma_iommu_mapping(dev_from_gk20a(g));
-        u64 addr = gk20a_mm_iova_addr(g, (*sgt)->sgl);
+        u64 addr = g->ops.mm.get_iova_addr(g, (*sgt)->sgl, 0);
         struct tegra_vgpu_cmd_msg msg;
         struct tegra_vgpu_as_map_params *p = &msg.params.as_map;
         int err;
@@ -450,4 +450,5 @@ void vgpu_init_mm_ops(struct gpu_ops *gops)
         gops->mm.l2_flush = vgpu_mm_l2_flush;
         gops->mm.tlb_invalidate = vgpu_mm_tlb_invalidate;
         gops->mm.get_physical_addr_bits = gk20a_mm_get_physical_addr_bits;
+        gops->mm.get_iova_addr = gk20a_mm_iova_addr;
 }