gpu: nvgpu: Use new kmem API functions (common/*)

Use the new kmem API functions in common/* and common/mm/*.

Add a struct gk20a pointer to struct nvgpu_allocator in order
to store the gk20a pointer used for allocating memory.

Bug 1799159
Bug 1823380

Change-Id: I881ea9545e8a8f0b75d77a1e35dd1812e0bb654e
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: http://git-master/r/1318315
Reviewed-by: svccoveritychecker <svccoveritychecker@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>

9 files changed, 71 insertions, 53 deletions

diff --git a/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c b/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c
index 6fc508d6..6e3bad6f 100644
--- a/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c
+++ b/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c
@@ -19,6 +19,7 @@
 #include <linux/bitops.h>
 
 #include <nvgpu/allocator.h>
+#include <nvgpu/kmem.h>
 
 #include "bitmap_allocator_priv.h"
 
@@ -248,12 +249,11 @@ static u64 nvgpu_bitmap_alloc(struct nvgpu_allocator *__a, u64 len)
 
         /*
          * Only do meta-data storage if we are allowed to allocate storage for
-         * that meta-data. The issue with using kmalloc() and friends is that
+         * that meta-data. The issue with using malloc and friends is that
          * in latency and success critical paths an alloc_page() call can either
-         * sleep for potentially a long time or, assuming GFP_ATOMIC, fail.
-         * Since we might not want either of these possibilities assume that the
-         * caller will keep what data it needs around to successfully free this
-         * allocation.
+         * sleep for potentially a long time or fail. Since we might not want
+         * either of these possibilities assume that the caller will keep what
+         * data it needs around to successfully free this allocation.
          */
         if (!(a->flags & GPU_ALLOC_NO_ALLOC_PAGE) &&
             __nvgpu_bitmap_store_alloc(a, addr, blks * a->blk_size))
@@ -332,8 +332,8 @@ static void nvgpu_bitmap_alloc_destroy(struct nvgpu_allocator *__a)
         }
 
         nvgpu_kmem_cache_destroy(a->meta_data_cache);
-        kfree(a->bitmap);
-        kfree(a);
+        nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a->bitmap);
+        nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a);
 }
 
 static void nvgpu_bitmap_print_stats(struct nvgpu_allocator *__a,
@@ -397,11 +397,11 @@ int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
                 length -= blk_size;
         }
 
-        a = kzalloc(sizeof(struct nvgpu_bitmap_allocator), GFP_KERNEL);
+        a = nvgpu_kzalloc(g, sizeof(struct nvgpu_bitmap_allocator));
         if (!a)
                 return -ENOMEM;
 
-        err = __nvgpu_alloc_common_init(__a, name, a, false, &bitmap_ops);
+        err = __nvgpu_alloc_common_init(__a, g, name, a, false, &bitmap_ops);
         if (err)
                 goto fail;
 
@@ -422,8 +422,8 @@ int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
         a->bit_offs = a->base >> a->blk_shift;
         a->flags = flags;
 
-        a->bitmap = kcalloc(BITS_TO_LONGS(a->num_bits), sizeof(*a->bitmap),
-                            GFP_KERNEL);
+        a->bitmap = nvgpu_kcalloc(g, BITS_TO_LONGS(a->num_bits),
+                                  sizeof(*a->bitmap));
         if (!a->bitmap) {
                 err = -ENOMEM;
                 goto fail;
@@ -445,6 +445,6 @@ int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
 fail:
         if (a->meta_data_cache)
                 nvgpu_kmem_cache_destroy(a->meta_data_cache);
-        kfree(a);
+        nvgpu_kfree(g, a);
         return err;
 }
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
index 6f4c670a..246be974 100644
--- a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
+++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
@@ -18,6 +18,7 @@
 #include <linux/slab.h>
 
 #include <nvgpu/allocator.h>
+#include <nvgpu/kmem.h>
 
 #include "gk20a/mm_gk20a.h"
 #include "gk20a/platform_gk20a.h"
@@ -304,7 +305,7 @@ static void nvgpu_buddy_allocator_destroy(struct nvgpu_allocator *__a)
         }
 
         nvgpu_kmem_cache_destroy(a->buddy_cache);
-        kfree(a);
+        nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a);
 
         alloc_unlock(__a);
 }
@@ -809,7 +810,7 @@ static void __balloc_do_free_fixed(struct nvgpu_buddy_allocator *a,
                 balloc_coalesce(a, bud);
         }
 
-        kfree(falloc);
+        nvgpu_kfree(nvgpu_alloc_to_gpu(a->owner), falloc);
 }
 
 /*
@@ -893,7 +894,7 @@ static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a,
                         goto fail;
         }
 
-        falloc = kmalloc(sizeof(*falloc), GFP_KERNEL);
+        falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc));
         if (!falloc)
                 goto fail;
 
@@ -932,7 +933,7 @@ static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a,
 fail_unlock:
         alloc_unlock(__a);
 fail:
-        kfree(falloc);
+        nvgpu_kfree(nvgpu_alloc_to_gpu(__a), falloc);
         nvgpu_alloc_trace_func_done();
         return 0;
 }
@@ -1261,11 +1262,11 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
         if (flags & GPU_ALLOC_GVA_SPACE && !vm)
                 return -EINVAL;
 
-        a = kzalloc(sizeof(struct nvgpu_buddy_allocator), GFP_KERNEL);
+        a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator));
         if (!a)
                 return -ENOMEM;
 
-        err = __nvgpu_alloc_common_init(__a, name, a, false, &buddy_ops);
+        err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops);
         if (err)
                 goto fail;
 
@@ -1339,7 +1340,7 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
 fail:
         if (a->buddy_cache)
                 nvgpu_kmem_cache_destroy(a->buddy_cache);
-        kfree(a);
+        nvgpu_kfree(g, a);
         return err;
 }
 
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h
index 56aaea62..935b3f1c 100644
--- a/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h
+++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h
@@ -20,8 +20,7 @@
 #include <linux/list.h>
 #include <linux/rbtree.h>
 
-#include <nvgpu/kmem.h>
-
+struct nvgpu_kmem_cache;
 struct nvgpu_allocator;
 struct vm_gk20a;
 
diff --git a/drivers/gpu/nvgpu/common/mm/lockless_allocator.c b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c
index e3063a42..6fd9bc48 100644
--- a/drivers/gpu/nvgpu/common/mm/lockless_allocator.c
+++ b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c
@@ -20,6 +20,7 @@
 #include <linux/atomic.h>
 
 #include <nvgpu/allocator.h>
+#include <nvgpu/kmem.h>
 
 #include "lockless_allocator_priv.h"
 
@@ -106,7 +107,7 @@ static void nvgpu_lockless_alloc_destroy(struct nvgpu_allocator *a)
         nvgpu_fini_alloc_debug(a);
 
         vfree(pa->next);
-        kfree(pa);
+        nvgpu_kfree(nvgpu_alloc_to_gpu(a), pa);
 }
 
 static void nvgpu_lockless_print_stats(struct nvgpu_allocator *a,
@@ -154,7 +155,7 @@ int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
                 return -EINVAL;
 
         /*
-         * Ensure we have space for atleast one node & there's no overflow.
+         * Ensure we have space for at least one node & there's no overflow.
          * In order to control memory footprint, we require count < INT_MAX
          */
         count = length;
@@ -162,11 +163,11 @@ int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
         if (!base || !count || count > INT_MAX)
                 return -EINVAL;
 
-        a = kzalloc(sizeof(struct nvgpu_lockless_allocator), GFP_KERNEL);
+        a = nvgpu_kzalloc(g, sizeof(struct nvgpu_lockless_allocator));
         if (!a)
                 return -ENOMEM;
 
-        err = __nvgpu_alloc_common_init(__a, name, a, false, &pool_ops);
+        err = __nvgpu_alloc_common_init(__a, g, name, a, false, &pool_ops);
         if (err)
                 goto fail;
 
@@ -202,6 +203,6 @@ int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
         return 0;
 
 fail:
-        kfree(a);
+        nvgpu_kfree(g, a);
         return err;
 }
diff --git a/drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c b/drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c
index 24a76a0b..02b7b48d 100644
--- a/drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c
+++ b/drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c
@@ -123,7 +123,7 @@ void nvgpu_alloc_destroy(struct nvgpu_allocator *a)
 /*
  * Handle the common init stuff for a nvgpu_allocator.
  */
-int __nvgpu_alloc_common_init(struct nvgpu_allocator *a,
+int __nvgpu_alloc_common_init(struct nvgpu_allocator *a, struct gk20a *g,
                               const char *name, void *priv, bool dbg,
                               const struct nvgpu_allocator_ops *ops)
 {
@@ -143,6 +143,7 @@ int __nvgpu_alloc_common_init(struct nvgpu_allocator *a,
         if (err)
                 return err;
 
+        a->g = g;
         a->ops = ops;
         a->priv = priv;
         a->debug = dbg;
diff --git a/drivers/gpu/nvgpu/common/mm/page_allocator.c b/drivers/gpu/nvgpu/common/mm/page_allocator.c
index 193decc9..7d2cedc9 100644
--- a/drivers/gpu/nvgpu/common/mm/page_allocator.c
+++ b/drivers/gpu/nvgpu/common/mm/page_allocator.c
@@ -21,6 +21,7 @@
 
 #include <nvgpu/allocator.h>
 #include <nvgpu/page_allocator.h>
+#include <nvgpu/kmem.h>
 
 #include "buddy_allocator_priv.h"
 
@@ -760,7 +761,7 @@ static void nvgpu_page_allocator_destroy(struct nvgpu_allocator *__a)
         struct nvgpu_page_allocator *a = page_allocator(__a);
 
         alloc_lock(__a);
-        kfree(a);
+        nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a);
         __a->priv = NULL;
         alloc_unlock(__a);
 }
@@ -848,9 +849,9 @@ static int nvgpu_page_alloc_init_slabs(struct nvgpu_page_allocator *a)
         size_t nr_slabs = ilog2(a->page_size >> 12);
         unsigned int i;
 
-        a->slabs = kcalloc(nr_slabs,
-                           sizeof(struct page_alloc_slab),
-                           GFP_KERNEL);
+        a->slabs = nvgpu_kcalloc(nvgpu_alloc_to_gpu(a->owner),
+                                 nr_slabs,
+                                 sizeof(struct page_alloc_slab));
         if (!a->slabs)
                 return -ENOMEM;
         a->nr_slabs = nr_slabs;
@@ -881,11 +882,11 @@ int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
         if (blk_size < SZ_4K)
                 return -EINVAL;
 
-        a = kzalloc(sizeof(struct nvgpu_page_allocator), GFP_KERNEL);
+        a = nvgpu_kzalloc(g, sizeof(struct nvgpu_page_allocator));
         if (!a)
                 return -ENOMEM;
 
-        err = __nvgpu_alloc_common_init(__a, name, a, false, &page_ops);
+        err = __nvgpu_alloc_common_init(__a, g, name, a, false, &page_ops);
         if (err)
                 goto fail;
 
@@ -938,6 +939,6 @@ fail:
                 nvgpu_kmem_cache_destroy(a->chunk_cache);
         if (a->slab_page_cache)
                 nvgpu_kmem_cache_destroy(a->slab_page_cache);
-        kfree(a);
+        nvgpu_kfree(g, a);
         return err;
 }
diff --git a/drivers/gpu/nvgpu/common/nvgpu_common.c b/drivers/gpu/nvgpu/common/nvgpu_common.c
index d7ff4841..771d4121 100644
--- a/drivers/gpu/nvgpu/common/nvgpu_common.c
+++ b/drivers/gpu/nvgpu/common/nvgpu_common.c
@@ -17,11 +17,12 @@
 #include <linux/dma-mapping.h>
 #include <linux/firmware.h>
 
+#include <nvgpu/kmem.h>
+#include <nvgpu/nvgpu_common.h>
+
 #include "gk20a/gk20a_scale.h"
 #include "gk20a/gk20a.h"
 
-#include <nvgpu/nvgpu_common.h>
-
 #define EMC3D_DEFAULT_RATIO 750
 
 static void nvgpu_init_vars(struct gk20a *g)
@@ -164,7 +165,7 @@ int nvgpu_probe(struct gk20a *g,
         gk20a_create_sysfs(g->dev);
         gk20a_debug_init(g->dev, debugfs_symlink);
 
-        g->dbg_regops_tmp_buf = kzalloc(SZ_4K, GFP_KERNEL);
+        g->dbg_regops_tmp_buf = nvgpu_kzalloc(g, SZ_4K);
         if (!g->dbg_regops_tmp_buf) {
                 dev_err(g->dev, "couldn't allocate regops tmp buf");
                 return -ENOMEM;
@@ -190,7 +191,8 @@ static const struct firmware *do_request_firmware(struct device *dev,
                 path_len = strlen(prefix) + strlen(fw_name);
                 path_len += 2; /* for the path separator and zero terminator*/
 
-                fw_path = kzalloc(sizeof(*fw_path) * path_len, GFP_KERNEL);
+                fw_path = nvgpu_kzalloc(get_gk20a(dev),
+                                        sizeof(*fw_path) * path_len);
                 if (!fw_path)
                         return NULL;
 
@@ -207,7 +209,7 @@ static const struct firmware *do_request_firmware(struct device *dev,
                 err = request_firmware(&fw, fw_name, dev);
 #endif
 
-        kfree(fw_path);
+        nvgpu_kfree(get_gk20a(dev), fw_path);
         if (err)
                 return NULL;
         return fw;
diff --git a/drivers/gpu/nvgpu/common/semaphore.c b/drivers/gpu/nvgpu/common/semaphore.c
index ff86ada9..675794d1 100644
--- a/drivers/gpu/nvgpu/common/semaphore.c
+++ b/drivers/gpu/nvgpu/common/semaphore.c
@@ -20,6 +20,10 @@
 #include <linux/slab.h>
 
 #include <nvgpu/semaphore.h>
+#include <nvgpu/kmem.h>
+
+#include "gk20a/gk20a.h"
+#include "gk20a/mm_gk20a.h"
 
 #define __lock_sema_sea(s)                                              \
         do {                                                            \
@@ -83,7 +87,7 @@ struct nvgpu_semaphore_sea *nvgpu_semaphore_sea_create(struct gk20a *g)
         if (g->sema_sea)
                 return g->sema_sea;
 
-        g->sema_sea = kzalloc(sizeof(*g->sema_sea), GFP_KERNEL);
+        g->sema_sea = nvgpu_kzalloc(g, sizeof(*g->sema_sea));
         if (!g->sema_sea)
                 return NULL;
 
@@ -103,7 +107,7 @@ struct nvgpu_semaphore_sea *nvgpu_semaphore_sea_create(struct gk20a *g)
 cleanup_destroy:
         nvgpu_mutex_destroy(&g->sema_sea->sea_lock);
 cleanup_free:
-        kfree(g->sema_sea);
+        nvgpu_kfree(g, g->sema_sea);
         g->sema_sea = NULL;
         gpu_sema_dbg("Failed to creat semaphore sea!");
         return NULL;
@@ -131,7 +135,7 @@ struct nvgpu_semaphore_pool *nvgpu_semaphore_pool_alloc(
         unsigned long page_idx;
         int ret, err = 0;
 
-        p = kzalloc(sizeof(*p), GFP_KERNEL);
+        p = nvgpu_kzalloc(sea->gk20a, sizeof(*p));
         if (!p)
                 return ERR_PTR(-ENOMEM);
 
@@ -168,7 +172,7 @@ fail_alloc:
         nvgpu_mutex_destroy(&p->pool_lock);
 fail:
         __unlock_sema_sea(sea);
-        kfree(p);
+        nvgpu_kfree(sea->gk20a, p);
         gpu_sema_dbg("Failed to allocate semaphore pool!");
         return ERR_PTR(err);
 }
@@ -191,7 +195,8 @@ int nvgpu_semaphore_pool_map(struct nvgpu_semaphore_pool *p,
         gpu_sema_dbg("  %d: CPU VA = 0x%p!", p->page_idx, p->cpu_va);
 
         /* First do the RW mapping. */
-        p->rw_sg_table = kzalloc(sizeof(*p->rw_sg_table), GFP_KERNEL);
+        p->rw_sg_table = nvgpu_kzalloc(p->sema_sea->gk20a,
+                                       sizeof(*p->rw_sg_table));
         if (!p->rw_sg_table)
                 return -ENOMEM;
 
@@ -261,7 +266,7 @@ fail_unmap_sgt:
 fail_free_sgt:
         sg_free_table(p->rw_sg_table);
 fail:
-        kfree(p->rw_sg_table);
+        nvgpu_kfree(p->sema_sea->gk20a, p->rw_sg_table);
         p->rw_sg_table = NULL;
         gpu_sema_dbg("  %d: Failed to map semaphore pool!", p->page_idx);
         return err;
@@ -292,7 +297,7 @@ void nvgpu_semaphore_pool_unmap(struct nvgpu_semaphore_pool *p,
                      DMA_BIDIRECTIONAL);
 
         sg_free_table(p->rw_sg_table);
-        kfree(p->rw_sg_table);
+        nvgpu_kfree(p->sema_sea->gk20a, p->rw_sg_table);
         p->rw_sg_table = NULL;
 
         list_for_each_entry(hw_sema, &p->hw_semas, hw_sema_list)
@@ -325,12 +330,12 @@ static void nvgpu_semaphore_pool_free(struct kref *ref)
         __unlock_sema_sea(s);
 
         list_for_each_entry_safe(hw_sema, tmp, &p->hw_semas, hw_sema_list)
-                kfree(hw_sema);
+                nvgpu_kfree(p->sema_sea->gk20a, hw_sema);
 
         nvgpu_mutex_destroy(&p->pool_lock);
 
         gpu_sema_dbg("Freed semaphore pool! (idx=%d)", p->page_idx);
-        kfree(p);
+        nvgpu_kfree(p->sema_sea->gk20a, p);
 }
 
 void nvgpu_semaphore_pool_get(struct nvgpu_semaphore_pool *p)
@@ -374,7 +379,7 @@ static int __nvgpu_init_hw_sema(struct channel_gk20a *ch)
                 goto fail;
         }
 
-        hw_sema = kzalloc(sizeof(struct nvgpu_semaphore_int), GFP_KERNEL);
+        hw_sema = nvgpu_kzalloc(ch->g, sizeof(struct nvgpu_semaphore_int));
         if (!hw_sema) {
                 ret = -ENOMEM;
                 goto fail_free_idx;
@@ -417,7 +422,7 @@ void nvgpu_semaphore_free_hw_sema(struct channel_gk20a *ch)
 
         /* Make sure that when the ch is re-opened it will get a new HW sema. */
         list_del(&ch->hw_sema->hw_sema_list);
-        kfree(ch->hw_sema);
+        nvgpu_kfree(ch->g, ch->hw_sema);
         ch->hw_sema = NULL;
 
         nvgpu_mutex_release(&p->pool_lock);
@@ -440,7 +445,7 @@ struct nvgpu_semaphore *nvgpu_semaphore_alloc(struct channel_gk20a *ch)
                         return NULL;
         }
 
-        s = kzalloc(sizeof(*s), GFP_KERNEL);
+        s = nvgpu_kzalloc(ch->g, sizeof(*s));
         if (!s)
                 return NULL;
 
@@ -466,7 +471,7 @@ static void nvgpu_semaphore_free(struct kref *ref)
 
         nvgpu_semaphore_pool_put(s->hw_sema->p);
 
-        kfree(s);
+        nvgpu_kfree(s->hw_sema->ch->g, s);
 }
 
 void nvgpu_semaphore_put(struct nvgpu_semaphore *s)
diff --git a/drivers/gpu/nvgpu/include/nvgpu/allocator.h b/drivers/gpu/nvgpu/include/nvgpu/allocator.h
index 16fe2641..1bde290f 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/allocator.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/allocator.h
@@ -20,6 +20,7 @@
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
 #include <linux/platform_device.h>
+
 #include <nvgpu/lock.h>
 
 /* #define ALLOCATOR_DEBUG */
@@ -78,6 +79,8 @@ struct nvgpu_allocator_ops {
 };
 
 struct nvgpu_allocator {
+        struct gk20a *g;
+
         char name[32];
         struct nvgpu_mutex lock;
 
@@ -238,13 +241,18 @@ void nvgpu_alloc_destroy(struct nvgpu_allocator *allocator);
 void nvgpu_alloc_print_stats(struct nvgpu_allocator *a,
                              struct seq_file *s, int lock);
 
+static inline struct gk20a *nvgpu_alloc_to_gpu(struct nvgpu_allocator *a)
+{
+        return a->g;
+}
+
 /*
  * Common functionality for the internals of the allocators.
  */
 void nvgpu_init_alloc_debug(struct gk20a *g, struct nvgpu_allocator *a);
 void nvgpu_fini_alloc_debug(struct nvgpu_allocator *a);
 
-int  __nvgpu_alloc_common_init(struct nvgpu_allocator *a,
+int  __nvgpu_alloc_common_init(struct nvgpu_allocator *a, struct gk20a *g,
                                const char *name, void *priv, bool dbg,
                                const struct nvgpu_allocator_ops *ops);