From 6df3992b60959d32c7113cb77e131a2547174f3a Mon Sep 17 00:00:00 2001 From: Alex Waterman Date: Tue, 20 Dec 2016 13:55:48 -0800 Subject: gpu: nvgpu: Move allocators to common/mm/ Move the GPU allocators to common/mm/ since the allocators are common code across all GPUs. Also rename the allocator code to move away from gk20a_ prefixed structs and functions. This caused one issue with the nvgpu_alloc() and nvgpu_free() functions. There was a function for allocating either with kmalloc() or vmalloc() depending on the size of the allocation. Those have now been renamed to nvgpu_kalloc() and nvgpu_kfree(). Bug 1799159 Change-Id: Iddda92c013612bcb209847084ec85b8953002fa5 Signed-off-by: Alex Waterman Reviewed-on: http://git-master/r/1274400 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/include/nvgpu/allocator.h | 302 +++++++++++++++++++++++ drivers/gpu/nvgpu/include/nvgpu/page_allocator.h | 164 ++++++++++++ 2 files changed, 466 insertions(+) create mode 100644 drivers/gpu/nvgpu/include/nvgpu/allocator.h create mode 100644 drivers/gpu/nvgpu/include/nvgpu/page_allocator.h (limited to 'drivers/gpu/nvgpu/include') diff --git a/drivers/gpu/nvgpu/include/nvgpu/allocator.h b/drivers/gpu/nvgpu/include/nvgpu/allocator.h new file mode 100644 index 00000000..dee9b562 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/allocator.h @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2011-2016, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef NVGPU_ALLOCATOR_H +#define NVGPU_ALLOCATOR_H + +#include +#include +#include + +/* #define ALLOCATOR_DEBUG */ + +struct nvgpu_allocator; +struct nvgpu_alloc_carveout; +struct vm_gk20a; +struct gk20a; + +/* + * Operations for an allocator to implement. + */ +struct nvgpu_allocator_ops { + u64 (*alloc)(struct nvgpu_allocator *allocator, u64 len); + void (*free)(struct nvgpu_allocator *allocator, u64 addr); + + /* + * Special interface to allocate a memory region with a specific + * starting address. Yikes. Note: if free() works for freeing both + * regular and fixed allocations then free_fixed() does not need to + * be implemented. This behavior exists for legacy reasons and should + * not be propagated to new allocators. + */ + u64 (*alloc_fixed)(struct nvgpu_allocator *allocator, + u64 base, u64 len); + void (*free_fixed)(struct nvgpu_allocator *allocator, + u64 base, u64 len); + + /* + * Allow allocators to reserve space for carveouts. + */ + int (*reserve_carveout)(struct nvgpu_allocator *allocator, + struct nvgpu_alloc_carveout *co); + void (*release_carveout)(struct nvgpu_allocator *allocator, + struct nvgpu_alloc_carveout *co); + + /* + * Returns info about the allocator. + */ + u64 (*base)(struct nvgpu_allocator *allocator); + u64 (*length)(struct nvgpu_allocator *allocator); + u64 (*end)(struct nvgpu_allocator *allocator); + int (*inited)(struct nvgpu_allocator *allocator); + u64 (*space)(struct nvgpu_allocator *allocator); + + /* Destructor. */ + void (*fini)(struct nvgpu_allocator *allocator); + + /* Debugging. */ + void (*print_stats)(struct nvgpu_allocator *allocator, + struct seq_file *s, int lock); +}; + +struct nvgpu_allocator { + char name[32]; + struct mutex lock; + + void *priv; + const struct nvgpu_allocator_ops *ops; + + struct dentry *debugfs_entry; + bool debug; /* Control for debug msgs. */ +}; + +struct nvgpu_alloc_carveout { + const char *name; + u64 base; + u64 length; + + struct nvgpu_allocator *allocator; + + /* + * For usage by the allocator implementation. + */ + struct list_head co_entry; +}; + +#define NVGPU_CARVEOUT(__name, __base, __length) \ + { \ + .name = (__name), \ + .base = (__base), \ + .length = (__length) \ + } + +/* + * These are the available allocator flags. + * + * GPU_ALLOC_GVA_SPACE + * + * This flag makes sense for the buddy allocator only. It specifies that the + * allocator will be used for managing a GVA space. When managing GVA spaces + * special care has to be taken to ensure that allocations of similar PTE + * sizes are placed in the same PDE block. This allows the higher level + * code to skip defining both small and large PTE tables for every PDE. That + * can save considerable memory for address spaces that have a lot of + * allocations. + * + * GPU_ALLOC_NO_ALLOC_PAGE + * + * For any allocator that needs to manage a resource in a latency critical + * path this flag specifies that the allocator should not use any kmalloc() + * or similar functions during normal operation. Initialization routines + * may still use kmalloc(). This prevents the possibility of long waits for + * pages when using alloc_page(). Currently only the bitmap allocator + * implements this functionality. + * + * Also note that if you accept this flag then you must also define the + * free_fixed() function. Since no meta-data is allocated to help free + * allocations you need to keep track of the meta-data yourself (in this + * case the base and length of the allocation as opposed to just the base + * of the allocation). + * + * GPU_ALLOC_4K_VIDMEM_PAGES + * + * We manage vidmem pages at a large page granularity for performance + * reasons; however, this can lead to wasting memory. For page allocators + * setting this flag will tell the allocator to manage pools of 4K pages + * inside internally allocated large pages. + * + * Currently this flag is ignored since the only usage of the page allocator + * uses a 4K block size already. However, this flag has been reserved since + * it will be necessary in the future. + * + * GPU_ALLOC_FORCE_CONTIG + * + * Force allocations to be contiguous. Currently only relevant for page + * allocators since all other allocators are naturally contiguous. + * + * GPU_ALLOC_NO_SCATTER_GATHER + * + * The page allocator normally returns a scatter gather data structure for + * allocations (to handle discontiguous pages). However, at times that can + * be annoying so this flag forces the page allocator to return a u64 + * pointing to the allocation base (requires GPU_ALLOC_FORCE_CONTIG to be + * set as well). + */ +#define GPU_ALLOC_GVA_SPACE 0x1 +#define GPU_ALLOC_NO_ALLOC_PAGE 0x2 +#define GPU_ALLOC_4K_VIDMEM_PAGES 0x4 +#define GPU_ALLOC_FORCE_CONTIG 0x8 +#define GPU_ALLOC_NO_SCATTER_GATHER 0x10 + +static inline void alloc_lock(struct nvgpu_allocator *a) +{ + mutex_lock(&a->lock); +} + +static inline void alloc_unlock(struct nvgpu_allocator *a) +{ + mutex_unlock(&a->lock); +} + +/* + * Buddy allocator specific initializers. + */ +int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, + struct vm_gk20a *vm, const char *name, + u64 base, u64 size, u64 blk_size, + u64 max_order, u64 flags); +int nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, + const char *name, u64 base, u64 size, + u64 blk_size, u64 flags); + +/* + * Bitmap initializers. + */ +int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, + const char *name, u64 base, u64 length, + u64 blk_size, u64 flags); + +/* + * Page allocator initializers. + */ +int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, + const char *name, u64 base, u64 length, + u64 blk_size, u64 flags); + +/* + * Lockless allocatior initializers. + * Note: This allocator can only allocate fixed-size structures of a + * pre-defined size. + */ +int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, + const char *name, u64 base, u64 length, + u64 struct_size, u64 flags); + +#define GPU_BALLOC_MAX_ORDER 31 + +/* + * Allocator APIs. + */ +u64 nvgpu_alloc(struct nvgpu_allocator *allocator, u64 len); +void nvgpu_free(struct nvgpu_allocator *allocator, u64 addr); + +u64 nvgpu_alloc_fixed(struct nvgpu_allocator *allocator, u64 base, u64 len); +void nvgpu_free_fixed(struct nvgpu_allocator *allocator, u64 base, u64 len); + +int nvgpu_alloc_reserve_carveout(struct nvgpu_allocator *a, + struct nvgpu_alloc_carveout *co); +void nvgpu_alloc_release_carveout(struct nvgpu_allocator *a, + struct nvgpu_alloc_carveout *co); + +u64 nvgpu_alloc_base(struct nvgpu_allocator *a); +u64 nvgpu_alloc_length(struct nvgpu_allocator *a); +u64 nvgpu_alloc_end(struct nvgpu_allocator *a); +u64 nvgpu_alloc_initialized(struct nvgpu_allocator *a); +u64 nvgpu_alloc_space(struct nvgpu_allocator *a); + +void nvgpu_alloc_destroy(struct nvgpu_allocator *allocator); + +void nvgpu_alloc_print_stats(struct nvgpu_allocator *a, + struct seq_file *s, int lock); + +/* + * Common functionality for the internals of the allocators. + */ +void nvgpu_init_alloc_debug(struct gk20a *g, struct nvgpu_allocator *a); +void nvgpu_fini_alloc_debug(struct nvgpu_allocator *a); + +int __nvgpu_alloc_common_init(struct nvgpu_allocator *a, + const char *name, void *priv, bool dbg, + const struct nvgpu_allocator_ops *ops); + +static inline void nvgpu_alloc_enable_dbg(struct nvgpu_allocator *a) +{ + a->debug = true; +} + +static inline void nvgpu_alloc_disable_dbg(struct nvgpu_allocator *a) +{ + a->debug = false; +} + +/* + * Debug stuff. + */ +extern u32 nvgpu_alloc_tracing_on; + +void nvgpu_alloc_debugfs_init(struct device *dev); + +#define nvgpu_alloc_trace_func() \ + do { \ + if (nvgpu_alloc_tracing_on) \ + trace_printk("%s\n", __func__); \ + } while (0) + +#define nvgpu_alloc_trace_func_done() \ + do { \ + if (nvgpu_alloc_tracing_on) \ + trace_printk("%s_done\n", __func__); \ + } while (0) + +#define __alloc_pstat(seq, allocator, fmt, arg...) \ + do { \ + if (s) \ + seq_printf(seq, fmt, ##arg); \ + else \ + alloc_dbg(allocator, fmt, ##arg); \ + } while (0) + +#define __alloc_dbg(a, fmt, arg...) \ + pr_info("%-25s %25s() " fmt, (a)->name, __func__, ##arg) + +#if defined(ALLOCATOR_DEBUG) +/* + * Always print the debug messages... + */ +#define alloc_dbg(a, fmt, arg...) __alloc_dbg(a, fmt, ##arg) +#else +/* + * Only print debug messages if debug is enabled for a given allocator. + */ +#define alloc_dbg(a, fmt, arg...) \ + do { \ + if ((a)->debug) \ + __alloc_dbg((a), fmt, ##arg); \ + } while (0) + +#endif + +#endif /* NVGPU_ALLOCATOR_H */ diff --git a/drivers/gpu/nvgpu/include/nvgpu/page_allocator.h b/drivers/gpu/nvgpu/include/nvgpu/page_allocator.h new file mode 100644 index 00000000..7c21c117 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/page_allocator.h @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef PAGE_ALLOCATOR_PRIV_H +#define PAGE_ALLOCATOR_PRIV_H + +#include +#include + +#include + +struct nvgpu_allocator; + +/* + * This allocator implements the ability to do SLAB style allocation since the + * GPU has two page sizes available - 4k and 64k/128k. When the default + * granularity is the large page size (64k/128k) small allocations become very + * space inefficient. This is most notable in PDE and PTE blocks which are 4k + * in size. + * + * Thus we need the ability to suballocate in 64k pages. The way we do this for + * the GPU is as follows. We have several buckets for sub-64K allocations: + * + * B0 - 4k + * B1 - 8k + * B3 - 16k + * B4 - 32k + * B5 - 64k (for when large pages are 128k) + * + * When an allocation comes in for less than the large page size (from now on + * assumed to be 64k) the allocation is satisfied by one of the buckets. + */ +struct page_alloc_slab { + struct list_head empty; + struct list_head partial; + struct list_head full; + + int nr_empty; + int nr_partial; + int nr_full; + + u32 slab_size; +}; + +enum slab_page_state { + SP_EMPTY, + SP_PARTIAL, + SP_FULL, + SP_NONE +}; + +struct page_alloc_slab_page { + unsigned long bitmap; + u64 page_addr; + u32 slab_size; + + u32 nr_objects; + u32 nr_objects_alloced; + + enum slab_page_state state; + + struct page_alloc_slab *owner; + struct list_head list_entry; +}; + +struct page_alloc_chunk { + struct list_head list_entry; + + u64 base; + u64 length; +}; + +/* + * Struct to handle internal management of page allocation. It holds a list + * of the chunks of pages that make up the overall allocation - much like a + * scatter gather table. + */ +struct nvgpu_page_alloc { + struct list_head alloc_chunks; + + int nr_chunks; + u64 length; + + /* + * Only useful for the RB tree - since the alloc may have discontiguous + * pages the base is essentially irrelevant except for the fact that it + * is guarenteed to be unique. + */ + u64 base; + + struct rb_node tree_entry; + + /* + * Set if this is a slab alloc. Points back to the slab page that owns + * this particular allocation. nr_chunks will always be 1 if this is + * set. + */ + struct page_alloc_slab_page *slab_page; +}; + +struct nvgpu_page_allocator { + struct nvgpu_allocator *owner; /* Owner of this allocator. */ + + /* + * Use a buddy allocator to manage the allocation of the underlying + * pages. This lets us abstract the discontiguous allocation handling + * out of the annoyingly complicated buddy allocator. + */ + struct nvgpu_allocator source_allocator; + + /* + * Page params. + */ + u64 base; + u64 length; + u64 page_size; + u32 page_shift; + + struct rb_root allocs; /* Outstanding allocations. */ + + struct page_alloc_slab *slabs; + int nr_slabs; + + u64 flags; + + /* + * Stat tracking. + */ + u64 nr_allocs; + u64 nr_frees; + u64 nr_fixed_allocs; + u64 nr_fixed_frees; + u64 nr_slab_allocs; + u64 nr_slab_frees; + u64 pages_alloced; + u64 pages_freed; +}; + +static inline struct nvgpu_page_allocator *page_allocator( + struct nvgpu_allocator *a) +{ + return (struct nvgpu_page_allocator *)(a)->priv; +} + +static inline struct nvgpu_allocator *palloc_owner( + struct nvgpu_page_allocator *a) +{ + return a->owner; +} + +#endif -- cgit v1.2.2