2 files changed, 324 insertions, 22 deletions

diff --git a/drivers/gpu/nvgpu/include/nvgpu/kmem.h b/drivers/gpu/nvgpu/include/nvgpu/kmem.h
index c08e40a6..59192525 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/kmem.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/kmem.h
@@ -14,18 +14,21 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef NVGPU_KMEM_H
-#define NVGPU_KMEM_H
+#ifndef __NVGPU_KMEM_H__
+#define __NVGPU_KMEM_H__
 
-#include <linux/mm.h>
-#include <linux/slab.h>
-#include <linux/vmalloc.h>
-
-#include <asm/page.h>
+/*
+ * Incase this isn't defined already.
+ */
+#ifndef _THIS_IP_
+#define _THIS_IP_  ({ __label__ __here; __here: (unsigned long)&&__here; })
+#endif
 
 struct gk20a;
 
-/*
+/**
+ * DOC: Kmem cache support
+ *
  * In Linux there is support for the notion of a kmem_cache. It gives better
  * memory usage characteristics for lots of allocations of the same size. Think
  * structs that get allocated over and over. Normal kmalloc() type routines
@@ -37,26 +40,200 @@ struct gk20a;
  */
 struct nvgpu_kmem_cache;
 
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+/*
+ * Uncomment this if you want to enable stack traces in the memory profiling.
+ * Since this is a fairly high overhead operation and is only necessary for
+ * debugging actual bugs it's left here for developers to enable.
+ */
+/* #define __NVGPU_SAVE_KALLOC_STACK_TRACES */
+
+/*
+ * Defined per-OS.
+ */
+struct nvgpu_mem_alloc_tracker;
+#endif
+
+
+/**
+ * nvgpu_kmem_cache_create - create an nvgpu kernel memory cache.
+ *
+ * @g           The GPU driver struct using this cache.
+ * @size        Size of the object allocated by the cache.
+ *
+ * This cache can be used to allocate objects of size @size. Common usage would
+ * be for a struct that gets allocated a lot. In that case @size should be
+ * sizeof(struct my_struct).
+ *
+ * A given implementation of this need not do anything special. The allocation
+ * routines can simply be passed on to nvgpu_kzalloc() if desired so packing
+ * and alignment of the structs cannot be assumed.
+ */
 struct nvgpu_kmem_cache *nvgpu_kmem_cache_create(struct gk20a *g, size_t size);
+
+/**
+ * nvgpu_kmem_cache_destroy - destroy a cache created by
+ *                            nvgpu_kmem_cache_create().
+ *
+ * @cache       The cache to destroy.
+ */
 void nvgpu_kmem_cache_destroy(struct nvgpu_kmem_cache *cache);
 
+/**
+ * nvgpu_kmem_cache_alloc - Allocate an object from the cache
+ *
+ * @cache       The cache to alloc from.
+ */
 void *nvgpu_kmem_cache_alloc(struct nvgpu_kmem_cache *cache);
+
+/**
+ * nvgpu_kmem_cache_free - Free an object back to a cache
+ *
+ * @cache       The cache to return the object to.
+ * @ptr         Pointer to the object to free.
+ */
 void nvgpu_kmem_cache_free(struct nvgpu_kmem_cache *cache, void *ptr);
 
-static inline void *__nvgpu_big_alloc(size_t size, bool clear)
+/**
+ * nvgpu_kmalloc - Allocate from the kernel's allocator.
+ *
+ * @g:          Current GPU.
+ * @size:       Size of the allocation.
+ *
+ * Allocate a chunk of system memory from the kernel. Allocations larger than 1
+ * page may fail even when there may appear to be enough memory.
+ *
+ * This function may sleep so cannot be used in IRQs.
+ */
+#define nvgpu_kmalloc(g, size)          __nvgpu_kmalloc(g, size, _THIS_IP_)
+
+/**
+ * nvgpu_kzalloc - Allocate from the kernel's allocator.
+ *
+ * @g:          Current GPU.
+ * @size:       Size of the allocation.
+ *
+ * Identical to nvgpu_kalloc() except the memory will be zeroed before being
+ * returned.
+ */
+#define nvgpu_kzalloc(g, size)          __nvgpu_kzalloc(g, size, _THIS_IP_)
+
+/**
+ * nvgpu_kcalloc - Allocate from the kernel's allocator.
+ *
+ * @g:          Current GPU.
+ * @n:          Number of objects.
+ * @size:       Size of each object.
+ *
+ * Identical to nvgpu_kalloc() except the size of the memory chunk returned is
+ * @n * @size.
+ */
+#define nvgpu_kcalloc(g, n, size)       __nvgpu_kcalloc(g, n, size, _THIS_IP_)
+
+/**
+ * nvgpu_vmalloc - Allocate memory and return a map to it.
+ *
+ * @g:          Current GPU.
+ * @size:       Size of the allocation.
+ *
+ * Allocate some memory and return a pointer to a virtual memory mapping of
+ * that memory in the kernel's virtual address space. The underlying physical
+ * memory is not guaranteed to be contiguous (and indeed likely isn't). This
+ * allows for much larger allocations to be done without worrying about as much
+ * about physical memory fragmentation.
+ *
+ * This function may sleep.
+ */
+#define nvgpu_vmalloc(g, size)          __nvgpu_vmalloc(g, size, _THIS_IP_)
+
+/**
+ * nvgpu_vzalloc - Allocate memory and return a map to it.
+ *
+ * @g:          Current GPU.
+ * @size:       Size of the allocation.
+ *
+ * Identical to nvgpu_vmalloc() except this will return zero'ed memory.
+ */
+#define nvgpu_vzalloc(g, size)          __nvgpu_vzalloc(g, size, _THIS_IP_)
+
+/**
+ * nvgpu_kfree - Frees an alloc from nvgpu_kmalloc, nvgpu_kzalloc,
+ *               nvgpu_kcalloc.
+ *
+ * @g:          Current GPU.
+ * @addr:       Address of object to free.
+ */
+#define nvgpu_kfree(g, addr)            __nvgpu_kfree(g, addr)
+
+/**
+ * nvgpu_vfree - Frees an alloc from nvgpu_vmalloc, nvgpu_vzalloc.
+ *
+ * @g:          Current GPU.
+ * @addr:       Address of object to free.
+ */
+#define nvgpu_vfree(g, addr)            __nvgpu_vfree(g, addr)
+
+#define kmem_dbg(fmt, args...)                  \
+        gk20a_dbg(gpu_dbg_kmem, fmt, ##args)
+
+/**
+ * nvgpu_kmem_init - Initialize the kmem tracking stuff.
+ *
+ *@g: The driver to init.
+ *
+ * Returns non-zero on failure.
+ */
+int nvgpu_kmem_init(struct gk20a *g);
+
+/**
+ * nvgpu_kmem_fini - Finalize the kmem tracking code
+ *
+ * @g     - The GPU.
+ * @flags - Flags that control operation of this finalization.
+ *
+ * Cleanup resources used by nvgpu_kmem. Available flags for cleanup are:
+ *
+ *   %NVGPU_KMEM_FINI_DO_NOTHING
+ *   %NVGPU_KMEM_FINI_FORCE_CLEANUP
+ *   %NVGPU_KMEM_FINI_DUMP_ALLOCS
+ *   %NVGPU_KMEM_FINI_WARN
+ *   %NVGPU_KMEM_FINI_BUG
+ *
+ * %NVGPU_KMEM_FINI_DO_NOTHING will be overridden by anything else specified.
+ * Put another way don't just add %NVGPU_KMEM_FINI_DO_NOTHING and expect that
+ * to suppress other flags from doing anything.
+ */
+void nvgpu_kmem_fini(struct gk20a *g, int flags);
+
+/*
+ * These will simply be ignored if CONFIG_NVGPU_TRACK_MEM_USAGE is not defined.
+ */
+#define NVGPU_KMEM_FINI_DO_NOTHING              0
+#define NVGPU_KMEM_FINI_FORCE_CLEANUP           (1 << 0)
+#define NVGPU_KMEM_FINI_DUMP_ALLOCS             (1 << 1)
+#define NVGPU_KMEM_FINI_WARN                    (1 << 2)
+#define NVGPU_KMEM_FINI_BUG                     (1 << 3)
+
+/*
+ * When there's other implementations make sure they are included instead of
+ * Linux when not compiling on Linux!
+ */
+#include <nvgpu/kmem_linux.h>
+
+static inline void *__nvgpu_big_alloc(struct gk20a *g, size_t size, bool clear)
 {
         void *p;
 
         if (size > PAGE_SIZE) {
                 if (clear)
-                        p = vzalloc(size);
+                        p = nvgpu_vzalloc(g, size);
                 else
-                        p = vmalloc(size);
+                        p = nvgpu_vmalloc(g, size);
         } else {
                 if (clear)
-                        p = kzalloc(size, GFP_KERNEL);
+                        p = nvgpu_kzalloc(g, size);
                 else
-                        p = kmalloc(size, GFP_KERNEL);
+                        p = nvgpu_kmalloc(g, size);
         }
 
         return p;
@@ -65,6 +242,7 @@ static inline void *__nvgpu_big_alloc(size_t size, bool clear)
 /**
  * nvgpu_big_malloc - Pick virtual or physical alloc based on @size
  *
+ * @g - The GPU.
  * @size - Size of the allocation.
  *
  * On some platforms (i.e Linux) it is possible to allocate memory directly
@@ -83,30 +261,31 @@ static inline void *__nvgpu_big_alloc(size_t size, bool clear)
  * Returns a pointer to a virtual address range that the kernel can access or
  * %NULL on failure.
  */
-static inline void *nvgpu_big_malloc(size_t size)
+static inline void *nvgpu_big_malloc(struct gk20a *g, size_t size)
 {
-        return __nvgpu_big_alloc(size, false);
+        return __nvgpu_big_alloc(g, size, false);
 }
 
 /**
  * nvgpu_big_malloc - Pick virtual or physical alloc based on @size
  *
+ * @g - The GPU.
  * @size - Size of the allocation.
  *
  * Zeroed memory version of nvgpu_big_malloc().
  */
-static inline void *nvgpu_big_zalloc(size_t size)
+static inline void *nvgpu_big_zalloc(struct gk20a *g, size_t size)
 {
-        return __nvgpu_big_alloc(size, true);
+        return __nvgpu_big_alloc(g, size, true);
 }
 
 /**
  * nvgpu_big_free - Free and alloc from nvgpu_big_zalloc() or
  *                  nvgpu_big_malloc().
- *
+ * @g - The GPU.
  * @p - A pointer allocated by nvgpu_big_zalloc() or nvgpu_big_malloc().
  */
-static inline void nvgpu_big_free(void *p)
+static inline void nvgpu_big_free(struct gk20a *g, void *p)
 {
         /*
          * This will have to be fixed eventually. Allocs that use
@@ -114,9 +293,9 @@ static inline void nvgpu_big_free(void *p)
          * when freeing.
          */
         if (virt_addr_valid(p))
-                kfree(p);
+                nvgpu_kfree(g, p);
         else
-                vfree(p);
+                nvgpu_vfree(g, p);
 }
 
-#endif
+#endif /* __NVGPU_KMEM_H__ */
diff --git a/drivers/gpu/nvgpu/include/nvgpu/kmem_linux.h b/drivers/gpu/nvgpu/include/nvgpu/kmem_linux.h
new file mode 100644
index 00000000..d1cd27f3
--- /dev/null
+++ b/drivers/gpu/nvgpu/include/nvgpu/kmem_linux.h
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __NVGPU_KMEM_LINUX_H__
+#define __NVGPU_KMEM_LINUX_H__
+
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+#include <asm/page.h>
+
+struct gk20a;
+struct device;
+
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+void *__nvgpu_track_vmalloc(struct gk20a *g, unsigned long size,
+                            unsigned long ip);
+void *__nvgpu_track_vzalloc(struct gk20a *g, unsigned long size,
+                            unsigned long ip);
+void *__nvgpu_track_kmalloc(struct gk20a *g, size_t size, unsigned long ip);
+void *__nvgpu_track_kzalloc(struct gk20a *g, size_t size, unsigned long ip);
+void *__nvgpu_track_kcalloc(struct gk20a *g, size_t n, size_t size,
+                            unsigned long ip);
+void  __nvgpu_track_vfree(struct gk20a *g, void *addr);
+void  __nvgpu_track_kfree(struct gk20a *g, void *addr);
+
+void nvgpu_kmem_debugfs_init(struct device *dev);
+#else
+static inline void nvgpu_kmem_debugfs_init(struct device *dev)
+{
+}
+#endif
+
+/**
+ * DOC: Linux pass through kmem implementation.
+ *
+ * These are the Linux implementations of the various kmem functions defined by
+ * nvgpu. This should not be included directly - instead include <nvgpu/kmem.h>.
+ */
+
+static inline void *__nvgpu_kmalloc(struct gk20a *g, unsigned long size,
+                                    unsigned long ip)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        return __nvgpu_track_vmalloc(g, size, ip);
+#else
+        return kmalloc(size, GFP_KERNEL);
+#endif
+}
+
+static inline void *__nvgpu_kzalloc(struct gk20a *g, size_t size,
+                                    unsigned long ip)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        return __nvgpu_track_kzalloc(g, size, ip);
+#else
+        return kzalloc(size, GFP_KERNEL);
+#endif
+}
+
+static inline void *__nvgpu_kcalloc(struct gk20a *g, size_t n, size_t size,
+                                    unsigned long ip)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        return __nvgpu_track_kcalloc(g, n, size, ip);
+#else
+        return kcalloc(n, size, GFP_KERNEL);
+#endif
+}
+
+static inline void *__nvgpu_vmalloc(struct gk20a *g, unsigned long size,
+                                    unsigned long ip)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        return __nvgpu_track_vmalloc(g, size, ip);
+#else
+        return vmalloc(size);
+#endif
+}
+
+static inline void *__nvgpu_vzalloc(struct gk20a *g, unsigned long size,
+                                    unsigned long ip)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        return __nvgpu_track_vzalloc(g, size, ip);
+#else
+        return vzalloc(size);
+#endif
+}
+
+static inline void __nvgpu_kfree(struct gk20a *g, void *addr)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        __nvgpu_track_kfree(g, addr);
+#else
+        kfree(addr);
+#endif
+}
+
+static inline void __nvgpu_vfree(struct gk20a *g, void *addr)
+{
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        __nvgpu_track_vfree(g, addr);
+#else
+        vfree(addr);
+#endif
+}
+
+#endif