gpu: nvgpu: kmem abstraction and tracking

Implement kmem abstraction and tracking in nvgpu. The abstraction
helps move nvgpu's core code away from being Linux dependent and
allows kmem allocation tracking to be done for Linux and any other
OS supported by nvgpu.

Bug 1799159
Bug 1823380

Change-Id: Ieaae4ca1bbd1d4db4a1546616ab8b9fc53a4079d
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: http://git-master/r/1283828
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

2 files changed, 324 insertions, 22 deletions

diff --git a/drivers/gpu/nvgpu/include/nvgpu/kmem.h b/drivers/gpu/nvgpu/include/nvgpu/kmem.h
index c08e40a6..59192525 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/kmem.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/kmem.h
@@ -14,18 +14,21 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef NVGPU_KMEM_H
-#define NVGPU_KMEM_H
+#ifndef __NVGPU_KMEM_H__
+#define __NVGPU_KMEM_H__
 
-#include <linux/mm.h>
-#include <linux/slab.h>
-#include <linux/vmalloc.h>
-
-#include <asm/page.h>
+/*
+ * Incase this isn't defined already.
+ */
+#ifndef _THIS_IP_
+#define _THIS_IP_  ({ __label__ __here; __here: (unsigned long)&&__here; })
+#endif
 
 struct gk20a;
 
-/*
+/**
+ * DOC: Kmem cache support
+ *
  * In Linux there is support for the notion of a kmem_cache. It gives better
  * memory usage characteristics for lots of allocations of the same size. Think
  * structs that get allocated over and over. Normal kmalloc() type routines
@@ -37,26 +40,200 @@ struct gk20a;
  */
 struct nvgpu_kmem_cache;
 
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+/*
+ * Uncomment this if you want to enable stack traces in the memory profiling.
+ * Since this is a fairly high overhead operation and is only necessary for
+ * debugging actual bugs it's left here for developers to enable.
+ */
+/* #define __NVGPU_SAVE_KALLOC_STACK_TRACES */
+
+/*
+ * Defined per-OS.
+ */
+struct nvgpu_mem_alloc_tracker;
+#endif
+
+
+/**
+ * nvgpu_kmem_cache_create - create an nvgpu kernel memory cache.
+ *
+ * @g           The GPU driver struct using this cache.
+ * @size        Size of the object allocated by the cache.
+ *
+ * This cache can be used to allocate objects of size @size. Common usage would
+ * be for a struct that gets allocated a lot. In that case @size should be
+ * sizeof(struct my_struct).
+ *
+ * A given implementation of this need not do anything special. The allocation
+ * routines can simply be passed on to nvgpu_kzalloc() if desired so packing
+ * and alignment of the structs cannot be assumed.
+ */
 struct nvgpu_kmem_cache *nvgpu_kmem_cache_create(struct gk20a *g, size_t size);
+
+/**
+ * nvgpu_kmem_cache_destroy - destroy a cache created by
+ *                            nvgpu_kmem_cache_create().
+ *
+ * @cache       The cache to destroy.
+ */
 void nvgpu_kmem_cache_destroy(struct nvgpu_kmem_cache *cache);
 
+/**
+ * nvgpu_kmem_cache_alloc - Allocate an object from the cache
+ *
+ * @cache       The cache to alloc from.
+ */
 void *nvgpu_kmem_cache_alloc(struct nvgpu_kmem_cache *cache);
+
+/**
+ * nvgpu_kmem_cache_free - Free an object back to a cache
+ *
+ * @cache       The cache to return the object to.
+ * @ptr         Pointer to the object to free.
+ */
 void nvgpu_kmem_cache_free(struct nvgpu_kmem_cache *cache, void *ptr);
 
-static inline void *__nvgpu_big_alloc(size_t size, bool clear)
+/**
+ * nvgpu_kmalloc - Allocate from the kernel's allocator.
+ *
+ * @g:          Current GPU.
+ * @size:       Size of the allocation.
+ *
+ * Allocate a chunk of system memory from the kernel. Allocations larger than 1
+ * page may fail even when there may appear to be enough memory.
+ *
+ * This function may sleep so cannot be used in IRQs.
+ */
+#define nvgpu_kmalloc(g, size)          __nvgpu_kmalloc(g, size, _THIS_IP_)
+
+/**
+ * nvgpu_kzalloc - Allocate from the kernel's allocator.
+ *
+ * @g:          Current GPU.
+ * @size:       Size of the allocation.
+ *
+ * Identical to nvgpu_kalloc() except the memory will be zeroed before being
+ * returned.
+ */
+#define nvgpu_kzalloc(g, size)          __nvgpu_kzalloc(g, size, _THIS_IP_)
+
+/**
+ * nvgpu_kcalloc - Allocate from the kernel's allocator.
+ *
+ * @g:          Current GPU.
+ * @n:          Number of objects.
+ * @size:       Size of each object.
+ *
+ * Identical to nvgpu_kalloc() except the size of the memory chunk returned is
+ * @n * @size.
+ */
+#define nvgpu_kcalloc(g, n, size)       __nvgpu_kcalloc(g, n, size, _THIS_IP_)
+
+/**
+ * nvgpu_vmalloc - Allocate memory and return a map to it.
+ *
+ * @g:          Current GPU.
+ * @size:       Size of the allocation.
+ *
+ * Allocate some memory and return a pointer to a virtual memory mapping of
+ * that memory in the kernel's virtual address space. The underlying physical
+ * memory is not guaranteed to be contiguous (and indeed likely isn't). This
+ * allows for much larger allocations to be done without worrying about as much
+ * about physical memory fragmentation.
+ *
+ * This function may sleep.
+ */
+#define nvgpu_vmalloc(g, size)          __nvgpu_vmalloc(g, size, _THIS_IP_)
+
+/**
+ * nvgpu_vzalloc - Allocate memory and return a map to it.
+ *
+ * @g:          Current GPU.
+ * @size:       Size of the allocation.
+ *
+ * Identical to nvgpu_vmalloc() except this will return zero'ed memory.
+ */
+#define nvgpu_vzalloc(g, size)          __nvgpu_vzalloc(g, size, _THIS_IP_)
+
+/**
+ * nvgpu_kfree - Frees an alloc from nvgpu_kmalloc, nvgpu_kzalloc,
+ *               nvgpu_kcalloc.
+ *
+ * @g:          Current GPU.
+ * @addr:       Address of object to free.
+ */
+#define nvgpu_kfree(g, addr)            __nvgpu_kfree(g, addr)
+
+/**
+ * nvgpu_vfree - Frees an alloc from nvgpu_vmalloc, nvgpu_vzalloc.
+ *
+ * @g:          Current GPU.
+ * @addr:       Address of object to free.
+ */
+#define nvgpu_vfree(g, addr)            __nvgpu_vfree(g, addr)
+
+#define kmem_dbg(fmt, args...)                  \
+        gk20a_dbg(gpu_dbg_kmem, fmt, ##args)
+
+/**
+ * nvgpu_kmem_init - Initialize the kmem tracking stuff.
+ *
+ *@g: The driver to init.
+ *
+ * Returns non-zero on failure.
+ */
+int nvgpu_kmem_init(struct gk20a *g);
+
+/**
+ * nvgpu_kmem_fini - Finalize the kmem tracking code
+ *
+ * @g     - The GPU.
+ * @flags - Flags that control operation of this finalization.
+ *
+ * Cleanup resources used by nvgpu_kmem. Available flags for cleanup are:
+ *
+ *   %NVGPU_KMEM_FINI_DO_NOTHING
+ *   %NVGPU_KMEM_FINI_FORCE_CLEANUP
+ *   %NVGPU_KMEM_FINI_DUMP_ALLOCS
+ *   %NVGPU_KMEM_FINI_WARN
+ *   %NVGPU_KMEM_FINI_BUG
+ *
+ * %NVGPU_KMEM_FINI_DO_NOTHING will be overridden by anything else specified.
+ * Put another way don't just add %NVGPU_KMEM_FINI_DO_NOTHING and expect that
+ * to suppress other flags from doing anything.
+ */
+void nvgpu_kmem_fini(struct gk20a *g, int flags);
+
+/*
+ * These will simply be ignored if CONFIG_NVGPU_TRACK_MEM_USAGE is not defined.
+ */
+#define NVGPU_KMEM_FINI_DO_NOTHING              0
+#define NVGPU_KMEM_FINI_FORCE_CLEANUP           (1 << 0)
+#define NVGPU_KMEM_FINI_DUMP_ALLOCS             (1 << 1)
+#define NVGPU_KMEM_FINI_WARN                    (1 << 2)
+#define NVGPU_KMEM_FINI_BUG                     (1 << 3)
+
+/*
+ * When there's other implementations make sure they are included instead of
+ * Linux when not compiling on Linux!
+ */
+#include <nvgpu/kmem_linux.h>
+
+static inline void *__nvgpu_big_alloc(struct gk20a *g, size_t size, bool clear)
 {
         void *p;
 
         if (size > PAGE_SIZE) {
                 if (clear)
-                        p = vzalloc(size);
+                        p = nvgpu_vzalloc(g, size);
                 else
-                        p = vmalloc(size);
+                        p = nvgpu_vmalloc(g, size);
         } else {
                 if (clear)
-                        p = kzalloc(size, GFP_KERNEL);
+                        p = nvgpu_kzalloc(g, size);
                 else
-                        p = kmalloc(size, GFP_KERNEL);
+                        p = nvgpu_kmalloc(g, size);
         }
 
         return p;
@@ -65,6 +242,7 @@ static inline void *__nvgpu_big_alloc(size_t size, bool clear)
 /**
  * nvgpu_big_malloc - Pick virtual or physical alloc based on @size
  *
+ * @g - The GPU.
  * @size - Size of the allocation.
  *
  * On some platforms (i.e Linux) it is possible to allocate memory directly
@@ -83,30 +261,31 @@ static inline void *__nvgpu_big_alloc(size_t size, bool clear)
  * Returns a pointer to a virtual address range that the kernel can access or
  * %NULL on failure.
  */
-static inline void *nvgpu_big_malloc(size_t size)
+static inline void *nvgpu_big_malloc(struct gk20a *g, size_t size)
 {
-        return __nvgpu_big_alloc(size, false);
+        return __nvgpu_big_alloc(g, size, false);
 }
 
 /**
  * nvgpu_big_malloc - Pick virtual or physical alloc based on @size
  *
+ * @g - The GPU.
  * @size - Size of the allocation.
  *
  * Zeroed memory version of nvgpu_big_malloc().
  */
-static inline void *nvgpu_big_zalloc(size_t size)
+static inline void *nvgpu_big_zalloc(struct gk20a *g, size_t size)
 {
-        return __nvgpu_big_alloc(size, true);
+        return __nvgpu_big_alloc(g, size, true);
 }
 
 /**
  * nvgpu_big_free - Free and alloc from nvgpu_big_zalloc() or
  *                  nvgpu_big_malloc().
- *
+ * @g - The GPU.
  * @p - A pointer allocated by nvgpu_big_zalloc() or nvgpu_big_malloc().
  */
-static inline void nvgpu_big_free(void *p)
+static inline void nvgpu_big_free(struct gk20a *g, void *p)
 {
         /*
          * This will have to be fixed eventually. Allocs that use
@@ -114,9 +293,9 @@ static inline void nvgpu_big_free(void *p)
          * when freeing.
          */
         if (virt_addr_valid(p))
-                kfree(p);
+                nvgpu_kfree(g, p);
         else
-                vfree(p);
+                nvgpu_vfree(g, p);
 }
 
-#endif
+#endif /* __NVGPU_KMEM_H__ */
diff --git a/drivers/gpu/nvgpu/include/nvgpu/kmem_linux.h b/drivers/gpu/nvgpu/include/nvgpu/kmem_linux.h
new file mode 100644
index 00000000..d1cd27f3
--- /dev/null
+++ b/drivers/gpu/nvgpu/include/nvgpu/kmem_linux.h
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __NVGPU_KMEM_LINUX_H__
+#define __NVGPU_KMEM_LINUX_H__
+
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+#include <asm/page.h>
+
+struct gk20a;
+struct device;
+
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+void *__nvgpu_track_vmalloc(struct gk20a *g, unsigned long size,
+                            unsigned long ip);
+void *__nvgpu_track_vzalloc(struct gk20a *g, unsigned long size,
+                            unsigned long ip);
+void *__nvgpu_track_kmalloc(struct gk20a *g, size_t size, unsigned long ip);
+void *__nvgpu_track_kzalloc(struct gk20a *g, size_t size, unsigned long ip);
+void *__nvgpu_track_kcalloc(struct gk20a *g, size_t n, size_t size,
+                            unsigned long ip);
+void  __nvgpu_track_vfree(struct gk20a *g, void *addr);
+void  __nvgpu_track_kfree(struct gk20a *g, void *addr);
+
+void nvgpu_kmem_debugfs_init(struct device *dev);
+#else
+static inline void nvgpu_kmem_debugfs_init(struct device *dev)
+{
+}
+#endif
+
+/**
+ * DOC: Linux pass through kmem implementation.
+ *
+ * These are the Linux implementations of the various kmem functions defined by
+ * nvgpu. This should not be included directly - instead include <nvgpu/kmem.h>.
+ */
+
+static inline void *__nvgpu_kmalloc(struct gk20a *g, unsigned long size,
+                                    unsigned long ip)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        return __nvgpu_track_vmalloc(g, size, ip);
+#else
+        return kmalloc(size, GFP_KERNEL);
+#endif
+}
+
+static inline void *__nvgpu_kzalloc(struct gk20a *g, size_t size,
+                                    unsigned long ip)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        return __nvgpu_track_kzalloc(g, size, ip);
+#else
+        return kzalloc(size, GFP_KERNEL);
+#endif
+}
+
+static inline void *__nvgpu_kcalloc(struct gk20a *g, size_t n, size_t size,
+                                    unsigned long ip)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        return __nvgpu_track_kcalloc(g, n, size, ip);
+#else
+        return kcalloc(n, size, GFP_KERNEL);
+#endif
+}
+
+static inline void *__nvgpu_vmalloc(struct gk20a *g, unsigned long size,
+                                    unsigned long ip)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        return __nvgpu_track_vmalloc(g, size, ip);
+#else
+        return vmalloc(size);
+#endif
+}
+
+static inline void *__nvgpu_vzalloc(struct gk20a *g, unsigned long size,
+                                    unsigned long ip)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        return __nvgpu_track_vzalloc(g, size, ip);
+#else
+        return vzalloc(size);
+#endif
+}
+
+static inline void __nvgpu_kfree(struct gk20a *g, void *addr)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        __nvgpu_track_kfree(g, addr);
+#else
+        kfree(addr);
+#endif
+}
+
+static inline void __nvgpu_vfree(struct gk20a *g, void *addr)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        __nvgpu_track_vfree(g, addr);
+#else
+        vfree(addr);
+#endif
+}
+
+#endif