1 files changed, 653 insertions, 0 deletions

diff --git a/include/os/linux/kmem.c b/include/os/linux/kmem.c
new file mode 100644
index 0000000..395cc45
--- /dev/null
+++ b/include/os/linux/kmem.c
@@ -0,0 +1,653 @@
+/*
+ * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/vmalloc.h>
+#include <linux/stacktrace.h>
+
+#include <nvgpu/lock.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/atomic.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/gk20a.h>
+
+#include "kmem_priv.h"
+
+/*
+ * Statically declared because this needs to be shared across all nvgpu driver
+ * instances. This makes sure that all kmem caches are _definitely_ uniquely
+ * named.
+ */
+static atomic_t kmem_cache_id;
+
+void *__nvgpu_big_alloc(struct gk20a *g, size_t size, bool clear)
+{
+        void *p;
+
+        if (size > PAGE_SIZE) {
+                if (clear)
+                        p = nvgpu_vzalloc(g, size);
+                else
+                        p = nvgpu_vmalloc(g, size);
+        } else {
+                if (clear)
+                        p = nvgpu_kzalloc(g, size);
+                else
+                        p = nvgpu_kmalloc(g, size);
+        }
+
+        return p;
+}
+
+void nvgpu_big_free(struct gk20a *g, void *p)
+{
+        /*
+         * This will have to be fixed eventually. Allocs that use
+         * nvgpu_big_[mz]alloc() will need to remember the size of the alloc
+         * when freeing.
+         */
+        if (is_vmalloc_addr(p))
+                nvgpu_vfree(g, p);
+        else
+                nvgpu_kfree(g, p);
+}
+
+void *__nvgpu_kmalloc(struct gk20a *g, size_t size, void *ip)
+{
+        void *alloc;
+
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        alloc = __nvgpu_track_kmalloc(g, size, ip);
+#else
+        alloc = kmalloc(size, GFP_KERNEL);
+#endif
+
+        kmem_dbg(g, "kmalloc: size=%-6ld addr=0x%p gfp=0x%08x",
+                 size, alloc, GFP_KERNEL);
+
+        return alloc;
+}
+
+void *__nvgpu_kzalloc(struct gk20a *g, size_t size, void *ip)
+{
+        void *alloc;
+
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        alloc = __nvgpu_track_kzalloc(g, size, ip);
+#else
+        alloc = kzalloc(size, GFP_KERNEL);
+#endif
+
+        kmem_dbg(g, "kzalloc: size=%-6ld addr=0x%p gfp=0x%08x",
+                 size, alloc, GFP_KERNEL);
+
+        return alloc;
+}
+
+void *__nvgpu_kcalloc(struct gk20a *g, size_t n, size_t size, void *ip)
+{
+        void *alloc;
+
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        alloc = __nvgpu_track_kcalloc(g, n, size, ip);
+#else
+        alloc = kcalloc(n, size, GFP_KERNEL);
+#endif
+
+        kmem_dbg(g, "kcalloc: size=%-6ld addr=0x%p gfp=0x%08x",
+                 n * size, alloc, GFP_KERNEL);
+
+        return alloc;
+}
+
+void *__nvgpu_vmalloc(struct gk20a *g, unsigned long size, void *ip)
+{
+        void *alloc;
+
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        alloc = __nvgpu_track_vmalloc(g, size, ip);
+#else
+        alloc = vmalloc(size);
+#endif
+
+        kmem_dbg(g, "vmalloc: size=%-6ld addr=0x%p", size, alloc);
+
+        return alloc;
+}
+
+void *__nvgpu_vzalloc(struct gk20a *g, unsigned long size, void *ip)
+{
+        void *alloc;
+
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        alloc = __nvgpu_track_vzalloc(g, size, ip);
+#else
+        alloc = vzalloc(size);
+#endif
+
+        kmem_dbg(g, "vzalloc: size=%-6ld addr=0x%p", size, alloc);
+
+        return alloc;
+}
+
+void __nvgpu_kfree(struct gk20a *g, void *addr)
+{
+        kmem_dbg(g, "kfree: addr=0x%p", addr);
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        __nvgpu_track_kfree(g, addr);
+#else
+        kfree(addr);
+#endif
+}
+
+void __nvgpu_vfree(struct gk20a *g, void *addr)
+{
+        kmem_dbg(g, "vfree: addr=0x%p", addr);
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+        __nvgpu_track_vfree(g, addr);
+#else
+        vfree(addr);
+#endif
+}
+
+#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
+
+void nvgpu_lock_tracker(struct nvgpu_mem_alloc_tracker *tracker)
+{
+        nvgpu_mutex_acquire(&tracker->lock);
+}
+
+void nvgpu_unlock_tracker(struct nvgpu_mem_alloc_tracker *tracker)
+{
+        nvgpu_mutex_release(&tracker->lock);
+}
+
+void kmem_print_mem_alloc(struct gk20a *g,
+                         struct nvgpu_mem_alloc *alloc,
+                         struct seq_file *s)
+{
+#ifdef __NVGPU_SAVE_KALLOC_STACK_TRACES
+        int i;
+
+        __pstat(s, "nvgpu-alloc: addr=0x%llx size=%ld\n",
+                alloc->addr, alloc->size);
+        for (i = 0; i < alloc->stack_length; i++)
+                __pstat(s, "  %3d [<%p>] %pS\n", i,
+                        (void *)alloc->stack[i],
+                        (void *)alloc->stack[i]);
+        __pstat(s, "\n");
+#else
+        __pstat(s, "nvgpu-alloc: addr=0x%llx size=%ld src=%pF\n",
+                alloc->addr, alloc->size, alloc->ip);
+#endif
+}
+
+static int nvgpu_add_alloc(struct nvgpu_mem_alloc_tracker *tracker,
+                           struct nvgpu_mem_alloc *alloc)
+{
+        alloc->allocs_entry.key_start = alloc->addr;
+        alloc->allocs_entry.key_end = alloc->addr + alloc->size;
+
+        nvgpu_rbtree_insert(&alloc->allocs_entry, &tracker->allocs);
+        return 0;
+}
+
+static struct nvgpu_mem_alloc *nvgpu_rem_alloc(
+        struct nvgpu_mem_alloc_tracker *tracker, u64 alloc_addr)
+{
+        struct nvgpu_mem_alloc *alloc;
+        struct nvgpu_rbtree_node *node = NULL;
+
+        nvgpu_rbtree_search(alloc_addr, &node, tracker->allocs);
+        if (!node)
+                return NULL;
+
+        alloc = nvgpu_mem_alloc_from_rbtree_node(node);
+
+        nvgpu_rbtree_unlink(node, &tracker->allocs);
+
+        return alloc;
+}
+
+static int __nvgpu_save_kmem_alloc(struct nvgpu_mem_alloc_tracker *tracker,
+                                   unsigned long size, unsigned long real_size,
+                                   u64 addr, void *ip)
+{
+        int ret;
+        struct nvgpu_mem_alloc *alloc;
+#ifdef __NVGPU_SAVE_KALLOC_STACK_TRACES
+        struct stack_trace stack_trace;
+#endif
+
+        alloc = kzalloc(sizeof(*alloc), GFP_KERNEL);
+        if (!alloc)
+                return -ENOMEM;
+
+        alloc->owner = tracker;
+        alloc->size = size;
+        alloc->real_size = real_size;
+        alloc->addr = addr;
+        alloc->ip = ip;
+
+#ifdef __NVGPU_SAVE_KALLOC_STACK_TRACES
+        stack_trace.max_entries = MAX_STACK_TRACE;
+        stack_trace.nr_entries = 0;
+        stack_trace.entries = alloc->stack;
+        /*
+         * This 4 here skips the 2 function calls that happen for all traced
+         * allocs due to nvgpu:
+         *
+         *   __nvgpu_save_kmem_alloc+0x7c/0x128
+         *   __nvgpu_track_kzalloc+0xcc/0xf8
+         *
+         * And the function calls that get made by the stack trace code itself.
+         * If the trace savings code changes this will likely have to change
+         * as well.
+         */
+        stack_trace.skip = 4;
+        save_stack_trace(&stack_trace);
+        alloc->stack_length = stack_trace.nr_entries;
+#endif
+
+        nvgpu_lock_tracker(tracker);
+        tracker->bytes_alloced += size;
+        tracker->bytes_alloced_real += real_size;
+        tracker->nr_allocs++;
+
+        /* Keep track of this for building a histogram later on. */
+        if (tracker->max_alloc < size)
+                tracker->max_alloc = size;
+        if (tracker->min_alloc > size)
+                tracker->min_alloc = size;
+
+        ret = nvgpu_add_alloc(tracker, alloc);
+        if (ret) {
+                WARN(1, "Duplicate alloc??? 0x%llx\n", addr);
+                kfree(alloc);
+                nvgpu_unlock_tracker(tracker);
+                return ret;
+        }
+        nvgpu_unlock_tracker(tracker);
+
+        return 0;
+}
+
+static int __nvgpu_free_kmem_alloc(struct nvgpu_mem_alloc_tracker *tracker,
+                                   u64 addr)
+{
+        struct nvgpu_mem_alloc *alloc;
+
+        nvgpu_lock_tracker(tracker);
+        alloc = nvgpu_rem_alloc(tracker, addr);
+        if (WARN(!alloc, "Possible double-free detected: 0x%llx!", addr)) {
+                nvgpu_unlock_tracker(tracker);
+                return -EINVAL;
+        }
+
+        memset((void *)alloc->addr, 0, alloc->size);
+
+        tracker->nr_frees++;
+        tracker->bytes_freed += alloc->size;
+        tracker->bytes_freed_real += alloc->real_size;
+        nvgpu_unlock_tracker(tracker);
+
+        return 0;
+}
+
+static void __nvgpu_check_valloc_size(unsigned long size)
+{
+        WARN(size < PAGE_SIZE, "Alloc smaller than page size! (%lu)!\n", size);
+}
+
+static void __nvgpu_check_kalloc_size(size_t size)
+{
+        WARN(size > PAGE_SIZE, "Alloc larger than page size! (%zu)!\n", size);
+}
+
+void *__nvgpu_track_vmalloc(struct gk20a *g, unsigned long size,
+                            void *ip)
+{
+        void *alloc = vmalloc(size);
+
+        if (!alloc)
+                return NULL;
+
+        __nvgpu_check_valloc_size(size);
+
+        /*
+         * Ignore the return message. If this fails let's not cause any issues
+         * for the rest of the driver.
+         */
+        __nvgpu_save_kmem_alloc(g->vmallocs, size, roundup_pow_of_two(size),
+                                (u64)(uintptr_t)alloc, ip);
+
+        return alloc;
+}
+
+void *__nvgpu_track_vzalloc(struct gk20a *g, unsigned long size,
+                            void *ip)
+{
+        void *alloc = vzalloc(size);
+
+        if (!alloc)
+                return NULL;
+
+        __nvgpu_check_valloc_size(size);
+
+        /*
+         * Ignore the return message. If this fails let's not cause any issues
+         * for the rest of the driver.
+         */
+        __nvgpu_save_kmem_alloc(g->vmallocs, size, roundup_pow_of_two(size),
+                                (u64)(uintptr_t)alloc, ip);
+
+        return alloc;
+}
+
+void *__nvgpu_track_kmalloc(struct gk20a *g, size_t size, void *ip)
+{
+        void *alloc = kmalloc(size, GFP_KERNEL);
+
+        if (!alloc)
+                return NULL;
+
+        __nvgpu_check_kalloc_size(size);
+
+        __nvgpu_save_kmem_alloc(g->kmallocs, size, roundup_pow_of_two(size),
+                                (u64)(uintptr_t)alloc, ip);
+
+        return alloc;
+}
+
+void *__nvgpu_track_kzalloc(struct gk20a *g, size_t size, void *ip)
+{
+        void *alloc = kzalloc(size, GFP_KERNEL);
+
+        if (!alloc)
+                return NULL;
+
+        __nvgpu_check_kalloc_size(size);
+
+        __nvgpu_save_kmem_alloc(g->kmallocs, size, roundup_pow_of_two(size),
+                                (u64)(uintptr_t)alloc, ip);
+
+        return alloc;
+}
+
+void *__nvgpu_track_kcalloc(struct gk20a *g, size_t n, size_t size,
+                            void *ip)
+{
+        void *alloc = kcalloc(n, size, GFP_KERNEL);
+
+        if (!alloc)
+                return NULL;
+
+        __nvgpu_check_kalloc_size(n * size);
+
+        __nvgpu_save_kmem_alloc(g->kmallocs, n * size,
+                                roundup_pow_of_two(n * size),
+                                (u64)(uintptr_t)alloc, ip);
+
+        return alloc;
+}
+
+void __nvgpu_track_vfree(struct gk20a *g, void *addr)
+{
+        /*
+         * Often it is accepted practice to pass NULL pointers into free
+         * functions to save code.
+         */
+        if (!addr)
+                return;
+
+        __nvgpu_free_kmem_alloc(g->vmallocs, (u64)(uintptr_t)addr);
+
+        vfree(addr);
+}
+
+void __nvgpu_track_kfree(struct gk20a *g, void *addr)
+{
+        if (!addr)
+                return;
+
+        __nvgpu_free_kmem_alloc(g->kmallocs, (u64)(uintptr_t)addr);
+
+        kfree(addr);
+}
+
+static int __do_check_for_outstanding_allocs(
+        struct gk20a *g,
+        struct nvgpu_mem_alloc_tracker *tracker,
+        const char *type, bool silent)
+{
+        struct nvgpu_rbtree_node *node;
+        int count = 0;
+
+        nvgpu_rbtree_enum_start(0, &node, tracker->allocs);
+        while (node) {
+                struct nvgpu_mem_alloc *alloc =
+                        nvgpu_mem_alloc_from_rbtree_node(node);
+
+                if (!silent)
+                        kmem_print_mem_alloc(g, alloc, NULL);
+
+                count++;
+                nvgpu_rbtree_enum_next(&node, node);
+        }
+
+        return count;
+}
+
+/**
+ * check_for_outstanding_allocs - Count and display outstanding allocs
+ *
+ * @g      - The GPU.
+ * @silent - If set don't print anything about the allocs.
+ *
+ * Dump (or just count) the number of allocations left outstanding.
+ */
+static int check_for_outstanding_allocs(struct gk20a *g, bool silent)
+{
+        int count = 0;
+
+        count += __do_check_for_outstanding_allocs(g, g->kmallocs, "kmalloc",
+                                                   silent);
+        count += __do_check_for_outstanding_allocs(g, g->vmallocs, "vmalloc",
+                                                   silent);
+
+        return count;
+}
+
+static void do_nvgpu_kmem_cleanup(struct nvgpu_mem_alloc_tracker *tracker,
+                                  void (*force_free_func)(const void *))
+{
+        struct nvgpu_rbtree_node *node;
+
+        nvgpu_rbtree_enum_start(0, &node, tracker->allocs);
+        while (node) {
+                struct nvgpu_mem_alloc *alloc =
+                        nvgpu_mem_alloc_from_rbtree_node(node);
+
+                if (force_free_func)
+                        force_free_func((void *)alloc->addr);
+
+                nvgpu_rbtree_unlink(node, &tracker->allocs);
+                kfree(alloc);
+
+                nvgpu_rbtree_enum_start(0, &node, tracker->allocs);
+        }
+}
+
+/**
+ * nvgpu_kmem_cleanup - Cleanup the kmem tracking
+ *
+ * @g          - The GPU.
+ * @force_free - If set will also free leaked objects if possible.
+ *
+ * Cleanup all of the allocs made by nvgpu_kmem tracking code. If @force_free
+ * is non-zero then the allocation made by nvgpu is also freed. This is risky,
+ * though, as it is possible that the memory is still in use by other parts of
+ * the GPU driver not aware that this has happened.
+ *
+ * In theory it should be fine if the GPU driver has been deinitialized and
+ * there are no bugs in that code. However, if there are any bugs in that code
+ * then they could likely manifest as odd crashes indeterminate amounts of time
+ * in the future. So use @force_free at your own risk.
+ */
+static void nvgpu_kmem_cleanup(struct gk20a *g, bool force_free)
+{
+        do_nvgpu_kmem_cleanup(g->kmallocs, force_free ? kfree : NULL);
+        do_nvgpu_kmem_cleanup(g->vmallocs, force_free ? vfree : NULL);
+}
+
+void nvgpu_kmem_fini(struct gk20a *g, int flags)
+{
+        int count;
+        bool silent, force_free;
+
+        if (!flags)
+                return;
+
+        silent = !(flags & NVGPU_KMEM_FINI_DUMP_ALLOCS);
+        force_free = !!(flags & NVGPU_KMEM_FINI_FORCE_CLEANUP);
+
+        count = check_for_outstanding_allocs(g, silent);
+        nvgpu_kmem_cleanup(g, force_free);
+
+        /*
+         * If we leak objects we can either BUG() out or just WARN(). In general
+         * it doesn't make sense to BUG() on here since leaking a few objects
+         * won't crash the kernel but it can be helpful for development.
+         *
+         * If neither flag is set then we just silently do nothing.
+         */
+        if (count > 0) {
+                if (flags & NVGPU_KMEM_FINI_WARN) {
+                        WARN(1, "Letting %d allocs leak!!\n", count);
+                } else if (flags & NVGPU_KMEM_FINI_BUG) {
+                        nvgpu_err(g, "Letting %d allocs leak!!", count);
+                        BUG();
+                }
+        }
+}
+
+int nvgpu_kmem_init(struct gk20a *g)
+{
+        int err;
+
+        g->vmallocs = kzalloc(sizeof(*g->vmallocs), GFP_KERNEL);
+        g->kmallocs = kzalloc(sizeof(*g->kmallocs), GFP_KERNEL);
+
+        if (!g->vmallocs || !g->kmallocs) {
+                err = -ENOMEM;
+                goto fail;
+        }
+
+        g->vmallocs->name = "vmalloc";
+        g->kmallocs->name = "kmalloc";
+
+        g->vmallocs->allocs = NULL;
+        g->kmallocs->allocs = NULL;
+
+        nvgpu_mutex_init(&g->vmallocs->lock);
+        nvgpu_mutex_init(&g->kmallocs->lock);
+
+        g->vmallocs->min_alloc = PAGE_SIZE;
+        g->kmallocs->min_alloc = KMALLOC_MIN_SIZE;
+
+        /*
+         * This needs to go after all the other initialization since they use
+         * the nvgpu_kzalloc() API.
+         */
+        g->vmallocs->allocs_cache = nvgpu_kmem_cache_create(g,
+                                                sizeof(struct nvgpu_mem_alloc));
+        g->kmallocs->allocs_cache = nvgpu_kmem_cache_create(g,
+                                                sizeof(struct nvgpu_mem_alloc));
+
+        if (!g->vmallocs->allocs_cache || !g->kmallocs->allocs_cache) {
+                err = -ENOMEM;
+                if (g->vmallocs->allocs_cache)
+                        nvgpu_kmem_cache_destroy(g->vmallocs->allocs_cache);
+                if (g->kmallocs->allocs_cache)
+                        nvgpu_kmem_cache_destroy(g->kmallocs->allocs_cache);
+                goto fail;
+        }
+
+        return 0;
+
+fail:
+        if (g->vmallocs)
+                kfree(g->vmallocs);
+        if (g->kmallocs)
+                kfree(g->kmallocs);
+        return err;
+}
+
+#else /* !CONFIG_NVGPU_TRACK_MEM_USAGE */
+
+int nvgpu_kmem_init(struct gk20a *g)
+{
+        return 0;
+}
+
+void nvgpu_kmem_fini(struct gk20a *g, int flags)
+{
+}
+#endif /* CONFIG_NVGPU_TRACK_MEM_USAGE */
+
+struct nvgpu_kmem_cache *nvgpu_kmem_cache_create(struct gk20a *g, size_t size)
+{
+        struct nvgpu_kmem_cache *cache =
+                nvgpu_kzalloc(g, sizeof(struct nvgpu_kmem_cache));
+
+        if (!cache)
+                return NULL;
+
+        cache->g = g;
+
+        snprintf(cache->name, sizeof(cache->name),
+                 "nvgpu-cache-0x%p-%d-%d", g, (int)size,
+                 atomic_inc_return(&kmem_cache_id));
+        cache->cache = kmem_cache_create(cache->name,
+                                         size, size, 0, NULL);
+        if (!cache->cache) {
+                nvgpu_kfree(g, cache);
+                return NULL;
+        }
+
+        return cache;
+}
+
+void nvgpu_kmem_cache_destroy(struct nvgpu_kmem_cache *cache)
+{
+        struct gk20a *g = cache->g;
+
+        kmem_cache_destroy(cache->cache);
+        nvgpu_kfree(g, cache);
+}
+
+void *nvgpu_kmem_cache_alloc(struct nvgpu_kmem_cache *cache)
+{
+        return kmem_cache_alloc(cache->cache, GFP_KERNEL);
+}
+
+void nvgpu_kmem_cache_free(struct nvgpu_kmem_cache *cache, void *ptr)
+{
+        kmem_cache_free(cache->cache, ptr);
+}