litmus-rt-tegra.git - LITMUS^RT and MC^2 V0 support for the NVIDIA Tegra 3 SoC

diff options

author	Jonathan Herman <hermanjl@cs.unc.edu>	2013-04-19 17:31:52 -0400
committer	Jonathan Herman <hermanjl@cs.unc.edu>	2013-04-19 17:31:52 -0400
commit	f70a290e8a889caa905ab7650c696f2bb299be1a (patch)
tree	56f0886d839499e9f522f189999024b3e86f9be2 /drivers/net/cxgb4
parent	fcc9d2e5a6c89d22b8b773a64fb4ad21ac318446 (diff)
parent	7ef4a793a624c6e66c16ca1051847f75161f5bec (diff)

Merge branch 'wip-nested-locking' into tegra-nested-lockingwip-nested-locking

Conflicts: Makefile include/linux/fs.h

Diffstat (limited to 'drivers/net/cxgb4')

0 files changed, 0 insertions, 0 deletions

/* * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ #include <linux/mm.h> #include <linux/slab.h> #include <linux/debugfs.h> #include <linux/seq_file.h> #include <linux/vmalloc.h> #include <linux/stacktrace.h> #include <nvgpu/lock.h> #include <nvgpu/kmem.h> #include <nvgpu/atomic.h> #include <nvgpu/bug.h> #include <nvgpu/gk20a.h> #include "kmem_priv.h" /* * Statically declared because this needs to be shared across all nvgpu driver * instances. This makes sure that all kmem caches are _definitely_ uniquely * named. */ static atomic_t kmem_cache_id; void *__nvgpu_big_alloc(struct gk20a *g, size_t size, bool clear) { void *p; if (size > PAGE_SIZE) { if (clear) p = nvgpu_vzalloc(g, size); else p = nvgpu_vmalloc(g, size); } else { if (clear) p = nvgpu_kzalloc(g, size); else p = nvgpu_kmalloc(g, size); } return p; } void nvgpu_big_free(struct gk20a *g, void *p) { /* * This will have to be fixed eventually. Allocs that use * nvgpu_big_[mz]alloc() will need to remember the size of the alloc * when freeing. */ if (is_vmalloc_addr(p)) nvgpu_vfree(g, p); else nvgpu_kfree(g, p); } void *__nvgpu_kmalloc(struct gk20a *g, size_t size, void *ip) { void *alloc; #ifdef CONFIG_NVGPU_TRACK_MEM_USAGE alloc = __nvgpu_track_kmalloc(g, size, ip); #else alloc = kmalloc(size, GFP_KERNEL); #endif kmem_dbg(g, "kmalloc: size=%-6ld addr=0x%p gfp=0x%08x", size, alloc, GFP_KERNEL); return alloc; } void *__nvgpu_kzalloc(struct gk20a *g, size_t size, void *ip) { void *alloc; #ifdef CONFIG_NVGPU_TRACK_MEM_USAGE alloc = __nvgpu_track_kzalloc(g, size, ip); #else alloc = kzalloc(size, GFP_KERNEL); #endif kmem_dbg(g, "kzalloc: size=%-6ld addr=0x%p gfp=0x%08x", size, alloc, GFP_KERNEL); return alloc; } void *__nvgpu_kcalloc(struct gk20a *g, size_t n, size_t size, void *ip) { void *alloc; #ifdef CONFIG_NVGPU_TRACK_MEM_USAGE alloc = __nvgpu_track_kcalloc(g, n, size, ip); #else alloc = kcalloc(n, size, GFP_KERNEL); #endif kmem_dbg(g, "kcalloc: size=%-6ld addr=0x%p gfp=0x%08x", n * size, alloc, GFP_KERNEL); return alloc; } void *__nvgpu_vmalloc(struct gk20a *g, unsigned long size, void *ip) { void *alloc; #ifdef CONFIG_NVGPU_TRACK_MEM_USAGE alloc = __nvgpu_track_vmalloc(g, size, ip); #else alloc = vmalloc(size); #endif kmem_dbg(g, "vmalloc: size=%-6ld addr=0x%p", size, alloc); return alloc; } void *__nvgpu_vzalloc(struct gk20a *g, unsigned long size, void *ip) { void *alloc; #ifdef CONFIG_NVGPU_TRACK_MEM_USAGE alloc = __nvgpu_track_vzalloc(g, size, ip); #else alloc = vzalloc(size); #endif kmem_dbg(g, "vzalloc: size=%-6ld addr=0x%p", size, alloc); return alloc; } void __nvgpu_kfree(struct gk20a *g, void *addr) { kmem_dbg(g, "kfree: addr=0x%p", addr); #ifdef CONFIG_NVGPU_TRACK_MEM_USAGE __nvgpu_track_kfree(g, addr); #else kfree(addr); #endif } void __nvgpu_vfree(struct gk20a *g, void *addr) { kmem_dbg(g, "vfree: addr=0x%p", addr); #ifdef CONFIG_NVGPU_TRACK_MEM_USAGE __nvgpu_track_vfree(g, addr); #else vfree(addr); #endif } #ifdef CONFIG_NVGPU_TRACK_MEM_USAGE void nvgpu_lock_tracker(struct nvgpu_mem_alloc_tracker *tracker) { nvgpu_mutex_acquire(&tracker->lock); } void nvgpu_unlock_tracker(struct nvgpu_mem_alloc_tracker *tracker) { nvgpu_mutex_release(&tracker->lock); } void kmem_print_mem_alloc(struct gk20a *g, struct nvgpu_mem_alloc *alloc, struct seq_file *s) { #ifdef __NVGPU_SAVE_KALLOC_STACK_TRACES int i; __pstat(s, "nvgpu-alloc: addr=0x%llx size=%ld\n", alloc->addr, alloc->size); for (i = 0; i < alloc->stack_length; i++) __pstat(s, " %3d [<%p>] %pS\n", i, (void *)alloc->stack[i], (void *)alloc->stack[i]); __pstat(s, "\n"); #else __pstat(s, "nvgpu-alloc: addr=0x%llx size=%ld src=%pF\n", alloc->addr, alloc->size, alloc->ip); #endif } static int nvgpu_add_alloc(struct nvgpu_mem_alloc_tracker *tracker, struct nvgpu_mem_alloc *alloc) { alloc->allocs_entry.key_start = alloc->addr; alloc->allocs_entry.key_end = alloc->addr + alloc->size; nvgpu_rbtree_insert(&alloc->allocs_entry, &tracker->allocs); return 0; } static struct nvgpu_mem_alloc *nvgpu_rem_alloc( struct nvgpu_mem_alloc_tracker *tracker, u64 alloc_addr) { struct nvgpu_mem_alloc *alloc; struct nvgpu_rbtree_node *node = NULL; nvgpu_rbtree_search(alloc_addr, &node, tracker->allocs); if (!node) return NULL; alloc = nvgpu_mem_alloc_from_rbtree_node(node); nvgpu_rbtree_unlink(node, &tracker->allocs); return alloc; } static int __nvgpu_save_kmem_alloc(struct nvgpu_mem_alloc_tracker *tracker, unsigned long size, unsigned long real_size, u64 addr, void *ip) { int ret; struct nvgpu_mem_alloc *alloc; #ifdef __NVGPU_SAVE_KALLOC_STACK_TRACES struct stack_trace stack_trace; #endif alloc = kzalloc(sizeof(*alloc), GFP_KERNEL); if (!alloc) return -ENOMEM; alloc->owner = tracker; alloc->size = size; alloc->real_size = real_size; alloc->addr = addr; alloc->ip = ip; #ifdef __NVGPU_SAVE_KALLOC_STACK_TRACES stack_trace.max_entries = MAX_STACK_TRACE; stack_trace.nr_entries = 0; stack_trace.entries = alloc->stack; /* * This 4 here skips the 2 function calls that happen for all traced * allocs due to nvgpu: * * __nvgpu_save_kmem_alloc+0x7c/0x128 * __nvgpu_track_kzalloc+0xcc/0xf8 * * And the function calls that get made by the stack trace code itself. * If the trace savings code changes this will likely have to change * as well. */ stack_trace.skip = 4; save_stack_trace(&stack_trace); alloc->stack_length = stack_trace.nr_entries; #endif nvgpu_lock_tracker(tracker); tracker->bytes_alloced += size; tracker->bytes_alloced_real += real_size; tracker->nr_allocs++; /* Keep track of this for building a histogram later on. */ if (tracker->max_alloc < size) tracker->max_alloc = size; if (tracker->min_alloc > size) tracker->min_alloc = size; ret = nvgpu_add_alloc(tracker, alloc); if (ret) { WARN(1, "Duplicate alloc??? 0x%llx\n", addr); kfree(alloc); nvgpu_unlock_tracker(tracker); return ret; } nvgpu_unlock_tracker(tracker); return 0; } static int __nvgpu_free_kmem_alloc(struct nvgpu_mem_alloc_tracker *tracker, u64 addr) { struct nvgpu_mem_alloc *alloc; nvgpu_lock_tracker(tracker); alloc = nvgpu_rem_alloc(tracker, addr); if (WARN(!alloc, "Possible double-free detected: 0x%llx!", addr)) { nvgpu_unlock_tracker(tracker); return -EINVAL; } memset((void *)alloc->addr, 0, alloc->size); tracker->nr_frees++; tracker->bytes_freed += alloc->size; tracker->bytes_freed_real += alloc->real_size; nvgpu_unlock_tracker(tracker); return 0; } static void __nvgpu_check_valloc_size(unsigned long size) { WARN(size < PAGE_SIZE, "Alloc smaller than page size! (%lu)!\n", size); } static void __nvgpu_check_kalloc_size(size_t size) { WARN(size > PAGE_SIZE, "Alloc larger than page size! (%zu)!\n", size); } void *__nvgpu_track_vmalloc(struct gk20a *g, unsigned long size, void *ip) { void *alloc = vmalloc(size); if (!alloc) return NULL; __nvgpu_check_valloc_size(size); /* * Ignore the return message. If this fails let's not cause any issues * for the rest of the driver. */ __nvgpu_save_kmem_alloc(g->vmallocs, size, roundup_pow_of_two(size), (u64)(uintptr_t)alloc, ip); return alloc; } void *__nvgpu_track_vzalloc(struct gk20a *g, unsigned long size, void *ip) { void *alloc = vzalloc(size); if (!alloc) return NULL; __nvgpu_check_valloc_size(size); /* * Ignore the return message. If this fails let's not cause any issues * for the rest of the driver. */ __nvgpu_save_kmem_alloc(g->vmallocs, size, roundup_pow_of_two(size), (u64)(uintptr_t)alloc, ip); return alloc; } void *__nvgpu_track_kmalloc(struct gk20a *g, size_t size, void *ip) { void *alloc = kmalloc(size, GFP_KERNEL); if (!alloc) return NULL; __nvgpu_check_kalloc_size(size); __nvgpu_save_kmem_alloc(g->kmallocs, size, roundup_pow_of_two(size), (u64)(uintptr_t)alloc, ip); return alloc; } void *__nvgpu_track_kzalloc(struct gk20a *g, size_t size, void *ip) { void *alloc = kzalloc(size, GFP_KERNEL); if (!alloc) return NULL; __nvgpu_check_kalloc_size(size); __nvgpu_save_kmem_alloc(g->kmallocs, size, roundup_pow_of_two(size), (u64)(uintptr_t)alloc, ip); return alloc; } void *__nvgpu_track_kcalloc(struct gk20a *g, size_t n, size_t size, void *ip) { void *alloc = kcalloc(n, size, GFP_KERNEL); if (!alloc) return NULL; __nvgpu_check_kalloc_size(n * size); __nvgpu_save_kmem_alloc(g->kmallocs, n * size, roundup_pow_of_two(n * size), (u64)(uintptr_t)alloc, ip); return alloc; } void __nvgpu_track_vfree(struct gk20a *g, void *addr) { /* * Often it is accepted practice to pass NULL pointers into free * functions to save code. */ if (!addr) return; __nvgpu_free_kmem_alloc(g->vmallocs, (u64)(uintptr_t)addr); vfree(addr); } void __nvgpu_track_kfree(struct gk20a *g, void *addr) { if (!addr) return; __nvgpu_free_kmem_alloc(g->kmallocs, (u64)(uintptr_t)addr); kfree(addr); } static int __do_check_for_outstanding_allocs( struct gk20a *g, struct nvgpu_mem_alloc_tracker *tracker, const char *type, bool silent) { struct nvgpu_rbtree_node *node; int count = 0; nvgpu_rbtree_enum_start(0, &node, tracker->allocs); while (node) { struct nvgpu_mem_alloc *alloc = nvgpu_mem_alloc_from_rbtree_node(node); if (!silent) kmem_print_mem_alloc(g, alloc, NULL); count++; nvgpu_rbtree_enum_next(&node, node); } return count; } /** * check_for_outstanding_allocs - Count and display outstanding allocs * * @g - The GPU. * @silent - If set don't print anything about the allocs. * * Dump (or just count) the number of allocations left outstanding. */ static int check_for_outstanding_allocs(struct gk20a *g, bool silent) { int count = 0; count += __do_check_for_outstanding_allocs(g, g->kmallocs, "kmalloc", silent); count += __do_check_for_outstanding_allocs(g, g->vmallocs, "vmalloc", silent); return count; } static void do_nvgpu_kmem_cleanup(struct nvgpu_mem_alloc_tracker *tracker, void (*force_free_func)(const void *)) { struct nvgpu_rbtree_node *node; nvgpu_rbtree_enum_start(0, &node, tracker->allocs); while (node) { struct nvgpu_mem_alloc *alloc = nvgpu_mem_alloc_from_rbtree_node(node); if (force_free_func) force_free_func((void *)alloc->addr); nvgpu_rbtree_unlink(node, &tracker->allocs); kfree(alloc); nvgpu_rbtree_enum_start(0, &node, tracker->allocs); } } /** * nvgpu_kmem_cleanup - Cleanup the kmem tracking * * @g - The GPU. * @force_free - If set will also free leaked objects if possible. * * Cleanup all of the allocs made by nvgpu_kmem tracking code. If @force_free * is non-zero then the allocation made by nvgpu is also freed. This is risky, * though, as it is possible that the memory is still in use by other parts of * the GPU driver not aware that this has happened. * * In theory it should be fine if the GPU driver has been deinitialized and * there are no bugs in that code. However, if there are any bugs in that code * then they could likely manifest as odd crashes indeterminate amounts of time * in the future. So use @force_free at your own risk. */ static void nvgpu_kmem_cleanup(struct gk20a *g, bool force_free) { do_nvgpu_kmem_cleanup(g->kmallocs, force_free ? kfree : NULL); do_nvgpu_kmem_cleanup(g->vmallocs, force_free ? vfree : NULL); } void nvgpu_kmem_fini(struct gk20a *g, int flags) { int count; bool silent, force_free; if (!flags) return; silent = !(flags & NVGPU_KMEM_FINI_DUMP_ALLOCS); force_free = !!(flags & NVGPU_KMEM_FINI_FORCE_CLEANUP); count = check_for_outstanding_allocs(g, silent); nvgpu_kmem_cleanup(g, force_free); /* * If we leak objects we can either BUG() out or just WARN(). In general * it doesn't make sense to BUG() on here since leaking a few objects * won't crash the kernel but it can be helpful for development. * * If neither flag is set then we just silently do nothing. */ if (count > 0) { if (flags & NVGPU_KMEM_FINI_WARN) { WARN(1, "Letting %d allocs leak!!\n", count); } else if (flags & NVGPU_KMEM_FINI_BUG) { nvgpu_err(g, "Letting %d allocs leak!!", count); BUG(); } } } int nvgpu_kmem_init(struct gk20a *g) { int err; g->vmallocs = kzalloc(sizeof(*g->vmallocs), GFP_KERNEL); g->kmallocs = kzalloc(sizeof(*g->kmallocs), GFP_KERNEL); if (!g->vmallocs || !g->kmallocs) { err = -ENOMEM; goto fail; } g->vmallocs->name = "vmalloc"; g->kmallocs->name = "kmalloc"; g->vmallocs->allocs = NULL; g->kmallocs->allocs = NULL; nvgpu_mutex_init(&g->vmallocs->lock); nvgpu_mutex_init(&g->kmallocs->lock); g->vmallocs->min_alloc = PAGE_SIZE; g->kmallocs->min_alloc = KMALLOC_MIN_SIZE; /* * This needs to go after all the other initialization since they use * the nvgpu_kzalloc() API. */ g->vmallocs->allocs_cache = nvgpu_kmem_cache_create(g, sizeof(struct nvgpu_mem_alloc)); g->kmallocs->allocs_cache = nvgpu_kmem_cache_create(g, sizeof(struct nvgpu_mem_alloc)); if (!g->vmallocs->allocs_cache || !g->kmallocs->allocs_cache) { err = -ENOMEM; if (g->vmallocs->allocs_cache) nvgpu_kmem_cache_destroy(g->vmallocs->allocs_cache); if (g->kmallocs->allocs_cache) nvgpu_kmem_cache_destroy(g->kmallocs->allocs_cache); goto fail; } return 0; fail: if (g->vmallocs) kfree(g->vmallocs); if (g->kmallocs) kfree(g->kmallocs); return err; } #else /* !CONFIG_NVGPU_TRACK_MEM_USAGE */ int nvgpu_kmem_init(struct gk20a *g) { return 0; } void nvgpu_kmem_fini(struct gk20a *g, int flags) { } #endif /* CONFIG_NVGPU_TRACK_MEM_USAGE */ struct nvgpu_kmem_cache *nvgpu_kmem_cache_create(struct gk20a *g, size_t size) { struct nvgpu_kmem_cache *cache = nvgpu_kzalloc(g, sizeof(struct nvgpu_kmem_cache)); if (!cache) return NULL; cache->g = g; snprintf(cache->name, sizeof(cache->name), "nvgpu-cache-0x%p-%d-%d", g, (int)size, atomic_inc_return(&kmem_cache_id)); cache->cache = kmem_cache_create(cache->name, size, size, 0, NULL); if (!cache->cache) { nvgpu_kfree(g, cache); return NULL; } return cache; } void nvgpu_kmem_cache_destroy(struct nvgpu_kmem_cache *cache) { struct gk20a *g = cache->g; kmem_cache_destroy(cache->cache); nvgpu_kfree(g, cache); } void *nvgpu_kmem_cache_alloc(struct nvgpu_kmem_cache *cache) { return kmem_cache_alloc(cache->cache, GFP_KERNEL); } void nvgpu_kmem_cache_free(struct nvgpu_kmem_cache *cache, void *ptr) { kmem_cache_free(cache->cache, ptr); }


context:
space:
mode: