3 files changed, 145 insertions, 83 deletions

diff --git a/drivers/gpu/nvgpu/common/mm/gmmu.c b/drivers/gpu/nvgpu/common/mm/gmmu.c
index 41f5acdd..66bce8f0 100644
--- a/drivers/gpu/nvgpu/common/mm/gmmu.c
+++ b/drivers/gpu/nvgpu/common/mm/gmmu.c
@@ -65,14 +65,14 @@ static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm,
         struct gk20a *g = gk20a_from_vm(vm);
         u64 vaddr;
 
-        struct nvgpu_mem_sgl *sgl = nvgpu_mem_sgl_create_from_mem(g, mem);
+        struct nvgpu_sgt *sgt = nvgpu_sgt_create_from_mem(g, mem);
 
-        if (!sgl)
+        if (!sgt)
                 return -ENOMEM;
 
         nvgpu_mutex_acquire(&vm->update_gmmu_lock);
         vaddr = g->ops.mm.gmmu_map(vm, addr,
-                                   sgl,    /* sg list */
+                                   sgt,    /* sg list */
                                    0,      /* sg offset */
                                    size,
                                    gmmu_page_size_kernel,
@@ -86,7 +86,7 @@ static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm,
                                    aperture);
         nvgpu_mutex_release(&vm->update_gmmu_lock);
 
-        nvgpu_mem_sgl_free(g, sgl);
+        nvgpu_sgt_free(sgt, g);
 
         if (!vaddr) {
                 nvgpu_err(g, "failed to map buffer!");
@@ -464,7 +464,7 @@ static int __set_pd_level(struct vm_gk20a *vm,
  * VIDMEM version of the update_ptes logic.
  */
 static int __nvgpu_gmmu_update_page_table_vidmem(struct vm_gk20a *vm,
-                                                 struct nvgpu_mem_sgl *sgl,
+                                                 struct nvgpu_sgt *sgt,
                                                  u64 space_to_skip,
                                                  u64 virt_addr,
                                                  u64 length,
@@ -472,8 +472,9 @@ static int __nvgpu_gmmu_update_page_table_vidmem(struct vm_gk20a *vm,
 {
         u64 phys_addr, chunk_length;
         int err = 0;
+        void *sgl;
 
-        if (!sgl) {
+        if (!sgt) {
                 /*
                  * This is considered an unmap. Just pass in 0 as the physical
                  * address for the entire GPU range.
@@ -490,16 +491,17 @@ static int __nvgpu_gmmu_update_page_table_vidmem(struct vm_gk20a *vm,
          * Otherwise iterate across all the chunks in this allocation and
          * map them.
          */
+        sgl = sgt->sgl;
         while (sgl) {
                 if (space_to_skip &&
-                    space_to_skip >= nvgpu_mem_sgl_length(sgl)) {
-                        space_to_skip -= nvgpu_mem_sgl_length(sgl);
-                        sgl = nvgpu_mem_sgl_next(sgl);
+                    space_to_skip >= nvgpu_sgt_get_length(sgt, sgl)) {
+                        space_to_skip -= nvgpu_sgt_get_length(sgt, sgl);
+                        sgl = nvgpu_sgt_get_next(sgt, sgl);
                         continue;
                 }
 
-                phys_addr = nvgpu_mem_sgl_phys(sgl) + space_to_skip;
-                chunk_length = min(length, (nvgpu_mem_sgl_length(sgl) -
+                phys_addr = nvgpu_sgt_get_phys(sgt, sgl) + space_to_skip;
+                chunk_length = min(length, (nvgpu_sgt_get_length(sgt, sgl) -
                                             space_to_skip));
 
                 err = __set_pd_level(vm, &vm->pdb,
@@ -518,27 +520,27 @@ static int __nvgpu_gmmu_update_page_table_vidmem(struct vm_gk20a *vm,
                  */
                 virt_addr += chunk_length;
                 length    -= chunk_length;
+                sgl = nvgpu_sgt_get_next(sgt, sgl);
 
                 if (length == 0)
                         break;
-
-                sgl = nvgpu_mem_sgl_next(sgl);
         }
 
         return err;
 }
 
 static int __nvgpu_gmmu_update_page_table_sysmem(struct vm_gk20a *vm,
-                                                 struct nvgpu_mem_sgl *sgl,
+                                                 struct nvgpu_sgt *sgt,
                                                  u64 space_to_skip,
                                                  u64 virt_addr,
                                                  u64 length,
                                                  struct nvgpu_gmmu_attrs *attrs)
 {
-        int err;
         struct gk20a *g = gk20a_from_vm(vm);
+        void *sgl;
+        int err;
 
-        if (!sgl) {
+        if (!sgt) {
                 /*
                  * This is considered an unmap. Just pass in 0 as the physical
                  * address for the entire GPU range.
@@ -559,8 +561,10 @@ static int __nvgpu_gmmu_update_page_table_sysmem(struct vm_gk20a *vm,
          * mapping is simple since the "physical" address is actually a virtual
          * IO address and will be contiguous.
          */
+        sgl = sgt->sgl;
+
         if (!g->mm.bypass_smmu) {
-                u64 io_addr = nvgpu_mem_sgl_gpu_addr(g, sgl, attrs);
+                u64 io_addr = nvgpu_sgt_get_gpu_addr(sgt, g, sgl, attrs);
 
                 io_addr += space_to_skip;
 
@@ -586,15 +590,15 @@ static int __nvgpu_gmmu_update_page_table_sysmem(struct vm_gk20a *vm,
                  * Cut out sgl ents for space_to_skip.
                  */
                 if (space_to_skip &&
-                    space_to_skip >= nvgpu_mem_sgl_length(sgl)) {
-                        space_to_skip -= nvgpu_mem_sgl_length(sgl);
-                        sgl = nvgpu_mem_sgl_next(sgl);
+                    space_to_skip >= nvgpu_sgt_get_length(sgt, sgl)) {
+                        space_to_skip -= nvgpu_sgt_get_length(sgt, sgl);
+                        sgl = nvgpu_sgt_get_next(sgt, sgl);
                         continue;
                 }
 
-                phys_addr = nvgpu_mem_sgl_phys(sgl) + space_to_skip;
+                phys_addr = nvgpu_sgt_get_phys(sgt, sgl) + space_to_skip;
                 chunk_length = min(length,
-                                   nvgpu_mem_sgl_length(sgl) - space_to_skip);
+                        nvgpu_sgt_get_length(sgt, sgl) - space_to_skip);
 
                 err = __set_pd_level(vm, &vm->pdb,
                                      0,
@@ -606,7 +610,7 @@ static int __nvgpu_gmmu_update_page_table_sysmem(struct vm_gk20a *vm,
                 space_to_skip = 0;
                 virt_addr += chunk_length;
                 length    -= chunk_length;
-                sgl        = nvgpu_mem_sgl_next(sgl);
+                sgl        = nvgpu_sgt_get_next(sgt, sgl);
 
                 if (length == 0)
                         break;
@@ -631,7 +635,7 @@ static int __nvgpu_gmmu_update_page_table_sysmem(struct vm_gk20a *vm,
  *     case of SMMU usage.
  */
 static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm,
-                                          struct nvgpu_mem_sgl *sgl,
+                                          struct nvgpu_sgt *sgt,
                                           u64 space_to_skip,
                                           u64 virt_addr,
                                           u64 length,
@@ -669,10 +673,10 @@ static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm,
                    "phys offset: %#-4llx;  pgsz: %3dkb perm=%-2s | "
                    "kind=%#02x APT=%-6s %c%c%c%c%c",
                    vm->name,
-                   sgl ? "MAP" : "UNMAP",
+                   sgt ? "MAP" : "UNMAP",
                    virt_addr,
                    length,
-                   sgl ? nvgpu_mem_sgl_phys(sgl) : 0,
+                   sgt ? nvgpu_sgt_get_phys(sgt, sgt->sgl) : 0,
                    space_to_skip,
                    page_size >> 10,
                    nvgpu_gmmu_perm_str(attrs->rw_flag),
@@ -690,14 +694,14 @@ static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm,
          */
         if (attrs->aperture == APERTURE_VIDMEM)
                 err = __nvgpu_gmmu_update_page_table_vidmem(vm,
-                                                            sgl,
+                                                            sgt,
                                                             space_to_skip,
                                                             virt_addr,
                                                             length,
                                                             attrs);
         else
                 err = __nvgpu_gmmu_update_page_table_sysmem(vm,
-                                                            sgl,
+                                                            sgt,
                                                             space_to_skip,
                                                             virt_addr,
                                                             length,
@@ -706,7 +710,7 @@ static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm,
         unmap_gmmu_pages(g, &vm->pdb);
         nvgpu_smp_mb();
 
-        __gmmu_dbg(g, attrs, "%-5s Done!", sgl ? "MAP" : "UNMAP");
+        __gmmu_dbg(g, attrs, "%-5s Done!", sgt ? "MAP" : "UNMAP");
 
         return err;
 }
@@ -725,7 +729,7 @@ static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm,
  */
 u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
                           u64 vaddr,
-                          struct nvgpu_mem_sgl *sgl,
+                          struct nvgpu_sgt *sgt,
                           u64 buffer_offset,
                           u64 size,
                           int pgsz_idx,
@@ -774,7 +778,7 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
                 allocated = true;
         }
 
-        err = __nvgpu_gmmu_update_page_table(vm, sgl, buffer_offset,
+        err = __nvgpu_gmmu_update_page_table(vm, sgt, buffer_offset,
                                              vaddr, size, &attrs);
         if (err) {
                 nvgpu_err(g, "failed to update ptes on map");
@@ -787,6 +791,7 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
                 batch->need_tlb_invalidate = true;
 
         return vaddr;
+
 fail_validate:
         if (allocated)
                 __nvgpu_vm_free_va(vm, vaddr, pgsz_idx);
diff --git a/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
index 7296c673..6decec24 100644
--- a/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
+++ b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
@@ -19,55 +19,34 @@
 
 #include "gk20a/gk20a.h"
 
-struct nvgpu_mem_sgl *nvgpu_mem_sgl_next(struct nvgpu_mem_sgl *sgl)
+void *nvgpu_sgt_get_next(struct nvgpu_sgt *sgt, void *sgl)
 {
-        return sgl->next;
+        return sgt->ops->sgl_next(sgl);
 }
 
-u64 nvgpu_mem_sgl_phys(struct nvgpu_mem_sgl *sgl)
+u64 nvgpu_sgt_get_phys(struct nvgpu_sgt *sgt, void *sgl)
 {
-        return sgl->phys;
+        return sgt->ops->sgl_phys(sgl);
 }
 
-u64 nvgpu_mem_sgl_dma(struct nvgpu_mem_sgl *sgl)
+u64 nvgpu_sgt_get_dma(struct nvgpu_sgt *sgt, void *sgl)
 {
-        return sgl->dma;
+        return sgt->ops->sgl_dma(sgl);
 }
 
-u64 nvgpu_mem_sgl_length(struct nvgpu_mem_sgl *sgl)
+u64 nvgpu_sgt_get_length(struct nvgpu_sgt *sgt, void *sgl)
 {
-        return sgl->length;
+        return sgt->ops->sgl_length(sgl);
 }
 
-/*
- * This builds a GPU address for the %sgl based on whether an IOMMU is present
- * or not. It also handles turning the physical address into the true GPU
- * physical address that should be programmed into the page tables.
- */
-u64 nvgpu_mem_sgl_gpu_addr(struct gk20a *g, struct nvgpu_mem_sgl *sgl,
+u64 nvgpu_sgt_get_gpu_addr(struct nvgpu_sgt *sgt, struct gk20a *g, void *sgl,
                            struct nvgpu_gmmu_attrs *attrs)
 {
-        if (nvgpu_mem_sgl_dma(sgl) == 0)
-                return g->ops.mm.gpu_phys_addr(g, attrs,
-                                               nvgpu_mem_sgl_phys(sgl));
-
-        if (nvgpu_mem_sgl_dma(sgl) == DMA_ERROR_CODE)
-                return 0;
-
-        return gk20a_mm_smmu_vaddr_translate(g, nvgpu_mem_sgl_dma(sgl));
+        return sgt->ops->sgl_gpu_addr(g, sgl, attrs);
 }
 
-void nvgpu_mem_sgl_free(struct gk20a *g, struct nvgpu_mem_sgl *sgl)
+void nvgpu_sgt_free(struct nvgpu_sgt *sgt, struct gk20a *g)
 {
-        struct nvgpu_mem_sgl *next;
-
-        /*
-         * Free each of the elements. We expect each element to have been
-         * nvgpu_k[mz]alloc()ed.
-         */
-        while (sgl) {
-                next = nvgpu_mem_sgl_next(sgl);
-                nvgpu_kfree(g, sgl);
-                sgl = next;
-        }
+        if (sgt && sgt->ops->sgt_free)
+                sgt->ops->sgt_free(g, sgt);
 }
diff --git a/drivers/gpu/nvgpu/common/mm/page_allocator.c b/drivers/gpu/nvgpu/common/mm/page_allocator.c
index 6d92b457..9c35f528 100644
--- a/drivers/gpu/nvgpu/common/mm/page_allocator.c
+++ b/drivers/gpu/nvgpu/common/mm/page_allocator.c
@@ -143,20 +143,93 @@ static void nvgpu_page_release_co(struct nvgpu_allocator *a,
         nvgpu_alloc_release_carveout(&va->source_allocator, co);
 }
 
+static void *nvgpu_page_alloc_sgl_next(void *sgl)
+{
+        struct nvgpu_mem_sgl *nvgpu_sgl = sgl;
+
+        return nvgpu_sgl->next;
+}
+
+static u64 nvgpu_page_alloc_sgl_phys(void *sgl)
+{
+        struct nvgpu_mem_sgl *nvgpu_sgl = sgl;
+
+        return nvgpu_sgl->phys;
+}
+
+static u64 nvgpu_page_alloc_sgl_dma(void *sgl)
+{
+        struct nvgpu_mem_sgl *nvgpu_sgl = sgl;
+
+        return nvgpu_sgl->dma;
+}
+
+static u64 nvgpu_page_alloc_sgl_length(void *sgl)
+{
+        struct nvgpu_mem_sgl *nvgpu_sgl = sgl;
+
+        return nvgpu_sgl->length;
+}
+
+static u64 nvgpu_page_alloc_sgl_gpu_addr(struct gk20a *g, void *sgl,
+                                         struct nvgpu_gmmu_attrs *attrs)
+{
+        struct nvgpu_mem_sgl *nvgpu_sgl = sgl;
+
+        return nvgpu_sgl->phys;
+}
+
+static void nvgpu_page_alloc_sgt_free(struct gk20a *g, struct nvgpu_sgt *sgt)
+{
+        /*
+         * No-op here. The free is handled by the page_alloc free() functions.
+         */
+}
+
+/*
+ * These implement the generic scatter gather ops for pages allocated
+ * by the page allocator. however, the primary aim for this, is of course,
+ * vidmem.
+ */
+static const struct nvgpu_sgt_ops page_alloc_sgl_ops = {
+        .sgl_next = nvgpu_page_alloc_sgl_next,
+        .sgl_phys = nvgpu_page_alloc_sgl_phys,
+        .sgl_dma = nvgpu_page_alloc_sgl_dma,
+        .sgl_length = nvgpu_page_alloc_sgl_length,
+        .sgl_gpu_addr = nvgpu_page_alloc_sgl_gpu_addr,
+        .sgt_free = nvgpu_page_alloc_sgt_free,
+};
+
+/*
+ * This actually frees the sgl memory. Used by the page_alloc free() functions.
+ */
+static void nvgpu_page_alloc_sgl_proper_free(struct gk20a *g,
+                                             struct nvgpu_mem_sgl *sgl)
+{
+        struct nvgpu_mem_sgl *next;
+
+        while (sgl) {
+                next = sgl->next;
+                nvgpu_kfree(g, sgl);
+                sgl = next;
+        }
+}
+
 static void __nvgpu_free_pages(struct nvgpu_page_allocator *a,
                                struct nvgpu_page_alloc *alloc,
                                bool free_buddy_alloc)
 {
-        struct nvgpu_mem_sgl *sgl = alloc->sgl;
+        struct nvgpu_mem_sgl *sgl = alloc->sgt.sgl;
 
         if (free_buddy_alloc) {
                 while (sgl) {
-                        nvgpu_free(&a->source_allocator, sgl->phys);
-                        sgl = nvgpu_mem_sgl_next(sgl);
+                        nvgpu_free(&a->source_allocator,
+                                   nvgpu_sgt_get_phys(&alloc->sgt, sgl));
+                        sgl = nvgpu_sgt_get_next(&alloc->sgt, sgl);
                 }
         }
 
-        nvgpu_mem_sgl_free(a->owner->g, alloc->sgl);
+        nvgpu_page_alloc_sgl_proper_free(a->owner->g, sgl);
         nvgpu_kmem_cache_free(a->alloc_cache, alloc);
 }
 
@@ -306,7 +379,7 @@ static int __do_slab_alloc(struct nvgpu_page_allocator *a,
         alloc->length = slab_page->slab_size;
         alloc->base = slab_page->page_addr + (offs * slab_page->slab_size);
 
-        sgl         = alloc->sgl;
+        sgl         = alloc->sgt.sgl;
         sgl->phys   = alloc->base;
         sgl->dma    = alloc->base;
         sgl->length = alloc->length;
@@ -338,13 +411,16 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_slab(
                 palloc_dbg(a, "OOM: could not alloc page_alloc struct!\n");
                 goto fail;
         }
+
+        alloc->sgt.ops = &page_alloc_sgl_ops;
+
         sgl = nvgpu_kzalloc(a->owner->g, sizeof(*sgl));
         if (!sgl) {
                 palloc_dbg(a, "OOM: could not alloc sgl struct!\n");
                 goto fail;
         }
 
-        alloc->sgl = sgl;
+        alloc->sgt.sgl = sgl;
         err = __do_slab_alloc(a, slab, alloc);
         if (err)
                 goto fail;
@@ -432,6 +508,7 @@ static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages(
         memset(alloc, 0, sizeof(*alloc));
 
         alloc->length = pages << a->page_shift;
+        alloc->sgt.ops = &page_alloc_sgl_ops;
 
         while (pages) {
                 u64 chunk_addr = 0;
@@ -495,7 +572,7 @@ static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages(
                 if (prev_sgl)
                         prev_sgl->next = sgl;
                 else
-                        alloc->sgl = sgl;
+                        alloc->sgt.sgl = sgl;
 
                 prev_sgl = sgl;
 
@@ -503,12 +580,12 @@ static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages(
         }
 
         alloc->nr_chunks = i;
-        alloc->base = alloc->sgl->phys;
+        alloc->base = ((struct nvgpu_mem_sgl *)alloc->sgt.sgl)->phys;
 
         return alloc;
 
 fail_cleanup:
-        sgl = alloc->sgl;
+        sgl = alloc->sgt.sgl;
         while (sgl) {
                 struct nvgpu_mem_sgl *next = sgl->next;
 
@@ -542,13 +619,13 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_pages(
 
         palloc_dbg(a, "Alloc 0x%llx (%llu) id=0x%010llx\n",
                    pages << a->page_shift, pages, alloc->base);
-        sgl = alloc->sgl;
+        sgl = alloc->sgt.sgl;
         while (sgl) {
                 palloc_dbg(a, "  Chunk %2d: 0x%010llx + 0x%llx\n",
                            i++,
-                           nvgpu_mem_sgl_phys(sgl),
-                           nvgpu_mem_sgl_length(sgl));
-                sgl = sgl->next;
+                           nvgpu_sgt_get_phys(&alloc->sgt, sgl),
+                           nvgpu_sgt_get_length(&alloc->sgt, sgl));
+                sgl = nvgpu_sgt_get_next(&alloc->sgt, sgl);
         }
         palloc_dbg(a, "Alloc done\n");
 
@@ -655,6 +732,7 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_pages_fixed(
         if (!alloc || !sgl)
                 goto fail;
 
+        alloc->sgt.ops = &page_alloc_sgl_ops;
         alloc->base = nvgpu_alloc_fixed(&a->source_allocator, base, length, 0);
         if (!alloc->base) {
                 WARN(1, "nvgpu: failed to fixed alloc pages @ 0x%010llx", base);
@@ -663,7 +741,7 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_pages_fixed(
 
         alloc->nr_chunks = 1;
         alloc->length = length;
-        alloc->sgl = sgl;
+        alloc->sgt.sgl = sgl;
 
         sgl->phys   = alloc->base;
         sgl->dma    = alloc->base;
@@ -708,13 +786,13 @@ static u64 nvgpu_page_alloc_fixed(struct nvgpu_allocator *__a,
 
         palloc_dbg(a, "Alloc [fixed] @ 0x%010llx + 0x%llx (%llu)\n",
                    alloc->base, aligned_len, pages);
-        sgl = alloc->sgl;
+        sgl = alloc->sgt.sgl;
         while (sgl) {
                 palloc_dbg(a, "  Chunk %2d: 0x%010llx + 0x%llx\n",
                            i++,
-                           nvgpu_mem_sgl_phys(sgl),
-                           nvgpu_mem_sgl_length(sgl));
-                sgl = sgl->next;
+                           nvgpu_sgt_get_phys(&alloc->sgt, sgl),
+                           nvgpu_sgt_get_length(&alloc->sgt, sgl));
+                sgl = nvgpu_sgt_get_next(&alloc->sgt, sgl);
         }
 
         a->nr_fixed_allocs++;