gpu: nvgpu: Move Linux nvgpu_mem fields

Hide the Linux specific nvgpu_mem fields so that in subsequent patches
core code can instead of using struct sg_table it can use mem_desc.
Routines for accessing system specific fields will be added as needed.

This is the first step in a fairly major overhaul of the GMMU mapping
routines. There are numerous issues with the current design (or lack
there of): massively coupled code, system dependencies, disorganization,
etc.

JIRA NVGPU-12
JIRA NVGPU-30

Change-Id: I2e7d3ae3a07468cfc17c1c642d28ed1b0952474d
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: http://git-master/r/1464076
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

7 files changed, 54 insertions, 49 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/cde_gk20a.c b/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
index 18432c55..391f6612 100644
--- a/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
@@ -1239,7 +1239,7 @@ static int gk20a_cde_load(struct gk20a_cde_ctx *cde_ctx)
         }
 
         /* map backing store to gpu virtual space */
-        vaddr = gk20a_gmmu_map(ch->vm, &gr->compbit_store.mem.sgt,
+        vaddr = gk20a_gmmu_map(ch->vm, &gr->compbit_store.mem.priv.sgt,
                                g->gr.compbit_store.mem.size,
                                NVGPU_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
                                gk20a_mem_flag_read_only,
diff --git a/drivers/gpu/nvgpu/gk20a/fb_gk20a.c b/drivers/gpu/nvgpu/gk20a/fb_gk20a.c
index 214014ce..4a76bd6b 100644
--- a/drivers/gpu/nvgpu/gk20a/fb_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fb_gk20a.c
@@ -44,9 +44,10 @@ void fb_gk20a_reset(struct gk20a *g)
 
 void gk20a_fb_init_hw(struct gk20a *g)
 {
-        gk20a_writel(g, fb_niso_flush_sysmem_addr_r(),
-                     g->ops.mm.get_iova_addr(g, g->mm.sysmem_flush.sgt->sgl, 0)
-                     >> 8);
+        u32 addr = g->ops.mm.get_iova_addr(g,
+                        g->mm.sysmem_flush.priv.sgt->sgl, 0) >> 8;
+
+        gk20a_writel(g, fb_niso_flush_sysmem_addr_r(), addr);
 }
 
 static void gk20a_fb_set_mmu_page_size(struct gk20a *g)
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index 12bb3688..314d4551 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -954,7 +954,7 @@ static int gk20a_init_fifo_setup_sw(struct gk20a *g)
 
         for (chid = 0; chid < f->num_channels; chid++) {
                 f->channel[chid].userd_iova =
-                        g->ops.mm.get_iova_addr(g, f->userd.sgt->sgl, 0)
+                        g->ops.mm.get_iova_addr(g, f->userd.priv.sgt->sgl, 0)
                         + chid * f->userd_entry_size;
                 f->channel[chid].userd_gpu_va =
                         f->userd.gpu_va + chid * f->userd_entry_size;
@@ -3148,7 +3148,7 @@ static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
         new_buf = !runlist->cur_buffer;
 
         runlist_iova = g->ops.mm.get_iova_addr(
-                        g, runlist->mem[new_buf].sgt->sgl, 0);
+                        g, runlist->mem[new_buf].priv.sgt->sgl, 0);
 
         gk20a_dbg_info("runlist_id : %d, switch to new buffer 0x%16llx",
                 runlist_id, (u64)runlist_iova);
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
index 22093a34..f47d3b12 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
@@ -1943,7 +1943,7 @@ int gr_gk20a_update_hwpm_ctxsw_mode(struct gk20a *g,
                         }
 
                         pm_ctx->mem.gpu_va = gk20a_gmmu_map(c->vm,
-                                                        &pm_ctx->mem.sgt,
+                                                        &pm_ctx->mem.priv.sgt,
                                                         pm_ctx->mem.size,
                                                         NVGPU_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
                                                         gk20a_mem_flag_none, true,
@@ -2205,7 +2205,7 @@ static int gr_gk20a_init_ctxsw_ucode_vaspace(struct gk20a *g)
 
         /* Map ucode surface to GMMU */
         ucode_info->surface_desc.gpu_va = gk20a_gmmu_map(vm,
-                                        &ucode_info->surface_desc.sgt,
+                                        &ucode_info->surface_desc.priv.sgt,
                                         ucode_info->surface_desc.size,
                                         0, /* flags */
                                         gk20a_mem_flag_read_only,
@@ -2823,13 +2823,14 @@ static int gr_gk20a_map_global_ctx_buffers(struct gk20a *g,
         gk20a_dbg_fn("");
 
         /* Circular Buffer */
-        if (!c->vpr || (gr->global_ctx_buffer[CIRCULAR_VPR].mem.sgt == NULL)) {
+        if (!c->vpr ||
+            (gr->global_ctx_buffer[CIRCULAR_VPR].mem.priv.sgt == NULL)) {
                 mem = &gr->global_ctx_buffer[CIRCULAR].mem;
         } else {
                 mem = &gr->global_ctx_buffer[CIRCULAR_VPR].mem;
         }
 
-        gpu_va = gk20a_gmmu_map(ch_vm, &mem->sgt, mem->size,
+        gpu_va = gk20a_gmmu_map(ch_vm, &mem->priv.sgt, mem->size,
                                 NVGPU_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
                                 gk20a_mem_flag_none, true, mem->aperture);
         if (!gpu_va)
@@ -2838,13 +2839,14 @@ static int gr_gk20a_map_global_ctx_buffers(struct gk20a *g,
         g_bfr_size[CIRCULAR_VA] = mem->size;
 
         /* Attribute Buffer */
-        if (!c->vpr || (gr->global_ctx_buffer[ATTRIBUTE_VPR].mem.sgt == NULL)) {
+        if (!c->vpr ||
+            (gr->global_ctx_buffer[ATTRIBUTE_VPR].mem.priv.sgt == NULL)) {
                 mem = &gr->global_ctx_buffer[ATTRIBUTE].mem;
         } else {
                 mem = &gr->global_ctx_buffer[ATTRIBUTE_VPR].mem;
         }
 
-        gpu_va = gk20a_gmmu_map(ch_vm, &mem->sgt, mem->size,
+        gpu_va = gk20a_gmmu_map(ch_vm, &mem->priv.sgt, mem->size,
                                 NVGPU_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
                                 gk20a_mem_flag_none, false, mem->aperture);
         if (!gpu_va)
@@ -2853,13 +2855,14 @@ static int gr_gk20a_map_global_ctx_buffers(struct gk20a *g,
         g_bfr_size[ATTRIBUTE_VA] = mem->size;
 
         /* Page Pool */
-        if (!c->vpr || (gr->global_ctx_buffer[PAGEPOOL_VPR].mem.sgt == NULL)) {
+        if (!c->vpr ||
+            (gr->global_ctx_buffer[PAGEPOOL_VPR].mem.priv.sgt == NULL)) {
                 mem = &gr->global_ctx_buffer[PAGEPOOL].mem;
         } else {
                 mem = &gr->global_ctx_buffer[PAGEPOOL_VPR].mem;
         }
 
-        gpu_va = gk20a_gmmu_map(ch_vm, &mem->sgt, mem->size,
+        gpu_va = gk20a_gmmu_map(ch_vm, &mem->priv.sgt, mem->size,
                                 NVGPU_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
                                 gk20a_mem_flag_none, true, mem->aperture);
         if (!gpu_va)
@@ -2869,7 +2872,7 @@ static int gr_gk20a_map_global_ctx_buffers(struct gk20a *g,
 
         /* Golden Image */
         mem = &gr->global_ctx_buffer[GOLDEN_CTX].mem;
-        gpu_va = gk20a_gmmu_map(ch_vm, &mem->sgt, mem->size, 0,
+        gpu_va = gk20a_gmmu_map(ch_vm, &mem->priv.sgt, mem->size, 0,
                                 gk20a_mem_flag_none, true, mem->aperture);
         if (!gpu_va)
                 goto clean_up;
@@ -2878,7 +2881,7 @@ static int gr_gk20a_map_global_ctx_buffers(struct gk20a *g,
 
         /* Priv register Access Map */
         mem = &gr->global_ctx_buffer[PRIV_ACCESS_MAP].mem;
-        gpu_va = gk20a_gmmu_map(ch_vm, &mem->sgt, mem->size, 0,
+        gpu_va = gk20a_gmmu_map(ch_vm, &mem->priv.sgt, mem->size, 0,
                                 gk20a_mem_flag_none, true, mem->aperture);
         if (!gpu_va)
                 goto clean_up;
@@ -2950,7 +2953,7 @@ int gr_gk20a_alloc_gr_ctx(struct gk20a *g,
                 goto err_free_ctx;
 
         gr_ctx->mem.gpu_va = gk20a_gmmu_map(vm,
-                                        &gr_ctx->mem.sgt,
+                                        &gr_ctx->mem.priv.sgt,
                                         gr_ctx->mem.size,
                                         NVGPU_MAP_BUFFER_FLAGS_CACHEABLE_FALSE,
                                         gk20a_mem_flag_none, true,
@@ -3196,7 +3199,7 @@ int gk20a_alloc_obj_ctx(struct channel_gk20a  *c,
         }
 
         /* allocate patch buffer */
-        if (ch_ctx->patch_ctx.mem.sgt == NULL) {
+        if (ch_ctx->patch_ctx.mem.priv.sgt == NULL) {
                 err = gr_gk20a_alloc_channel_patch_ctx(g, c);
                 if (err) {
                         nvgpu_err(g,
@@ -4735,7 +4738,7 @@ static int gk20a_init_gr_setup_hw(struct gk20a *g)
         gk20a_dbg_fn("");
 
         /* init mmu debug buffer */
-        addr = g->ops.mm.get_iova_addr(g, gr->mmu_wr_mem.sgt->sgl, 0);
+        addr = g->ops.mm.get_iova_addr(g, gr->mmu_wr_mem.priv.sgt->sgl, 0);
         addr >>= fb_mmu_debug_wr_addr_alignment_v();
 
         gk20a_writel(g, fb_mmu_debug_wr_r(),
@@ -4745,7 +4748,7 @@ static int gk20a_init_gr_setup_hw(struct gk20a *g)
                      fb_mmu_debug_wr_vol_false_f() |
                      fb_mmu_debug_wr_addr_f(addr));
 
-        addr = g->ops.mm.get_iova_addr(g, gr->mmu_rd_mem.sgt->sgl, 0);
+        addr = g->ops.mm.get_iova_addr(g, gr->mmu_rd_mem.priv.sgt->sgl, 0);
         addr >>= fb_mmu_debug_rd_addr_alignment_v();
 
         gk20a_writel(g, fb_mmu_debug_rd_r(),
@@ -8405,7 +8408,7 @@ int gr_gk20a_exec_ctx_ops(struct channel_gk20a *ch,
                                 }
                                 if (!pm_ctx_ready) {
                                         /* Make sure ctx buffer was initialized */
-                                        if (!ch_ctx->pm_ctx.mem.pages) {
+                                        if (!ch_ctx->pm_ctx.mem.priv.pages) {
                                                 nvgpu_err(g,
                                                         "Invalid ctx buffer");
                                                 err = -EINVAL;
diff --git a/drivers/gpu/nvgpu/gk20a/ltc_common.c b/drivers/gpu/nvgpu/gk20a/ltc_common.c
index b92dda6d..1958c11c 100644
--- a/drivers/gpu/nvgpu/gk20a/ltc_common.c
+++ b/drivers/gpu/nvgpu/gk20a/ltc_common.c
@@ -96,7 +96,7 @@ static void gk20a_ltc_init_cbc(struct gk20a *g, struct gr_gk20a *gr)
                 compbit_store_iova = gk20a_mem_phys(&gr->compbit_store.mem);
         else
                 compbit_store_iova = g->ops.mm.get_iova_addr(g,
-                                gr->compbit_store.mem.sgt->sgl, 0);
+                                gr->compbit_store.mem.priv.sgt->sgl, 0);
 
         compbit_base_post_divide64 = compbit_store_iova >>
                 ltc_ltcs_ltss_cbc_base_alignment_shift_v();
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
index 1db52c85..69e00c5e 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
@@ -817,27 +817,28 @@ static int alloc_gmmu_phys_pages(struct vm_gk20a *vm, u32 order,
                 gk20a_dbg(gpu_dbg_pte, "alloc_pages failed");
                 goto err_out;
         }
-        entry->mem.sgt = nvgpu_kzalloc(g, sizeof(*entry->mem.sgt));
-        if (!entry->mem.sgt) {
+        entry->mem.priv.sgt = nvgpu_kzalloc(g, sizeof(*entry->mem.priv.sgt));
+        if (!entry->mem.priv.sgt) {
                 gk20a_dbg(gpu_dbg_pte, "cannot allocate sg table");
                 goto err_alloced;
         }
-        err = sg_alloc_table(entry->mem.sgt, 1, GFP_KERNEL);
+        err = sg_alloc_table(entry->mem.priv.sgt, 1, GFP_KERNEL);
         if (err) {
                 gk20a_dbg(gpu_dbg_pte, "sg_alloc_table failed");
                 goto err_sg_table;
         }
-        sg_set_page(entry->mem.sgt->sgl, pages, len, 0);
+        sg_set_page(entry->mem.priv.sgt->sgl, pages, len, 0);
         entry->mem.cpu_va = page_address(pages);
         memset(entry->mem.cpu_va, 0, len);
         entry->mem.size = len;
         entry->mem.aperture = APERTURE_SYSMEM;
-        FLUSH_CPU_DCACHE(entry->mem.cpu_va, sg_phys(entry->mem.sgt->sgl), len);
+        FLUSH_CPU_DCACHE(entry->mem.cpu_va,
+                         sg_phys(entry->mem.priv.sgt->sgl), len);
 
         return 0;
 
 err_sg_table:
-        nvgpu_kfree(vm->mm->g, entry->mem.sgt);
+        nvgpu_kfree(vm->mm->g, entry->mem.priv.sgt);
 err_alloced:
         __free_pages(pages, order);
 err_out:
@@ -854,9 +855,9 @@ static void free_gmmu_phys_pages(struct vm_gk20a *vm,
         free_pages((unsigned long)entry->mem.cpu_va, get_order(entry->mem.size));
         entry->mem.cpu_va = NULL;
 
-        sg_free_table(entry->mem.sgt);
-        nvgpu_kfree(vm->mm->g, entry->mem.sgt);
-        entry->mem.sgt = NULL;
+        sg_free_table(entry->mem.priv.sgt);
+        nvgpu_kfree(vm->mm->g, entry->mem.priv.sgt);
+        entry->mem.priv.sgt = NULL;
         entry->mem.size = 0;
         entry->mem.aperture = APERTURE_INVALID;
 }
@@ -864,16 +865,16 @@ static void free_gmmu_phys_pages(struct vm_gk20a *vm,
 static int map_gmmu_phys_pages(struct gk20a_mm_entry *entry)
 {
         FLUSH_CPU_DCACHE(entry->mem.cpu_va,
-                         sg_phys(entry->mem.sgt->sgl),
-                         entry->mem.sgt->sgl->length);
+                         sg_phys(entry->mem.priv.sgt->sgl),
+                         entry->mem.priv.sgt->sgl->length);
         return 0;
 }
 
 static void unmap_gmmu_phys_pages(struct gk20a_mm_entry *entry)
 {
         FLUSH_CPU_DCACHE(entry->mem.cpu_va,
-                         sg_phys(entry->mem.sgt->sgl),
-                         entry->mem.sgt->sgl->length);
+                         sg_phys(entry->mem.priv.sgt->sgl),
+                         entry->mem.priv.sgt->sgl->length);
 }
 
 static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order,
@@ -941,7 +942,7 @@ int map_gmmu_pages(struct gk20a *g, struct gk20a_mm_entry *entry)
                         return 0;
 
                 FLUSH_CPU_DCACHE(entry->mem.cpu_va,
-                                 sg_phys(entry->mem.sgt->sgl),
+                                 sg_phys(entry->mem.priv.sgt->sgl),
                                  entry->mem.size);
         } else {
                 int err = nvgpu_mem_begin(g, &entry->mem);
@@ -967,7 +968,7 @@ void unmap_gmmu_pages(struct gk20a *g, struct gk20a_mm_entry *entry)
                         return;
 
                 FLUSH_CPU_DCACHE(entry->mem.cpu_va,
-                                 sg_phys(entry->mem.sgt->sgl),
+                                 sg_phys(entry->mem.priv.sgt->sgl),
                                  entry->mem.size);
         } else {
                 nvgpu_mem_end(g, &entry->mem);
@@ -1028,9 +1029,9 @@ static int gk20a_zalloc_gmmu_page_table(struct vm_gk20a *vm,
 
         gk20a_dbg(gpu_dbg_pte, "entry = 0x%p, addr=%08llx, size %d, woff %x",
                   entry,
-                  (entry->mem.sgt && entry->mem.aperture == APERTURE_SYSMEM) ?
-                    g->ops.mm.get_iova_addr(g, entry->mem.sgt->sgl, 0)
-                    : 0,
+                  (entry->mem.priv.sgt &&
+                   entry->mem.aperture == APERTURE_SYSMEM) ?
+                  g->ops.mm.get_iova_addr(g, entry->mem.priv.sgt->sgl, 0) : 0,
                   order, entry->woffset);
         if (err)
                 return err;
@@ -1726,7 +1727,7 @@ static struct sg_table *gk20a_vidbuf_map_dma_buf(
 {
         struct gk20a_vidmem_buf *buf = attach->dmabuf->priv;
 
-        return buf->mem->sgt;
+        return buf->mem->priv.sgt;
 }
 
 static void gk20a_vidbuf_unmap_dma_buf(struct dma_buf_attachment *attach,
@@ -2398,7 +2399,7 @@ int gk20a_vm_map_compbits(struct vm_gk20a *vm,
                         g->ops.mm.gmmu_map(
                                 vm,
                                 !fixed_mapping ? 0 : *compbits_win_gva, /* va */
-                                g->gr.compbit_store.mem.sgt,
+                                g->gr.compbit_store.mem.priv.sgt,
                                 cacheline_offset_start, /* sg offset */
                                 mapped_buffer->ctag_map_win_size, /* size */
                                 small_pgsz_index,
@@ -2518,7 +2519,7 @@ static int gk20a_gmmu_clear_vidmem_mem(struct gk20a *g, struct nvgpu_mem *mem)
         if (g->mm.vidmem.ce_ctx_id == (u32)~0)
                 return -EINVAL;
 
-        alloc = get_vidmem_page_alloc(mem->sgt->sgl);
+        alloc = get_vidmem_page_alloc(mem->priv.sgt->sgl);
 
         nvgpu_list_for_each_entry(chunk, &alloc->alloc_chunks,
                                   page_alloc_chunk, list_entry) {
@@ -2580,14 +2581,14 @@ u64 gk20a_mem_get_base_addr(struct gk20a *g, struct nvgpu_mem *mem,
         u64 addr;
 
         if (mem->aperture == APERTURE_VIDMEM) {
-                alloc = get_vidmem_page_alloc(mem->sgt->sgl);
+                alloc = get_vidmem_page_alloc(mem->priv.sgt->sgl);
 
                 /* This API should not be used with > 1 chunks */
                 WARN_ON(alloc->nr_chunks != 1);
 
                 addr = alloc->base;
         } else {
-                addr = g->ops.mm.get_iova_addr(g, mem->sgt->sgl, flags);
+                addr = g->ops.mm.get_iova_addr(g, mem->priv.sgt->sgl, flags);
         }
 
         return addr;
@@ -2619,8 +2620,8 @@ static void gk20a_vidmem_clear_mem_worker(struct work_struct *work)
         while ((mem = get_pending_mem_desc(mm)) != NULL) {
                 gk20a_gmmu_clear_vidmem_mem(g, mem);
                 nvgpu_free(mem->allocator,
-                           (u64)get_vidmem_page_alloc(mem->sgt->sgl));
-                gk20a_free_sgtable(g, &mem->sgt);
+                           (u64)get_vidmem_page_alloc(mem->priv.sgt->sgl));
+                gk20a_free_sgtable(g, &mem->priv.sgt);
 
                 WARN_ON(atomic64_sub_return(mem->size,
                                         &g->mm.vidmem.bytes_pending) < 0);
@@ -2774,7 +2775,7 @@ u64 gk20a_pde_addr(struct gk20a *g, struct gk20a_mm_entry *entry)
         u64 base;
 
         if (g->mm.has_physical_mode)
-                base = sg_phys(entry->mem.sgt->sgl);
+                base = sg_phys(entry->mem.priv.sgt->sgl);
         else
                 base = gk20a_mem_get_base_addr(g, &entry->mem, 0);
 
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
index 7fac811e..94dc0b6f 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
@@ -539,7 +539,7 @@ static inline phys_addr_t gk20a_mem_phys(struct nvgpu_mem *mem)
 {
         /* FIXME: the sgt/sgl may get null if this is accessed e.g. in an isr
          * during channel deletion - attempt to fix at least null derefs */
-        struct sg_table *sgt = mem->sgt;
+        struct sg_table *sgt = mem->priv.sgt;
 
         if (sgt) {
                 struct scatterlist *sgl = sgt->sgl;