gpu: nvgpu: Move allocators to common/mm/

Move the GPU allocators to common/mm/ since the allocators are common
code across all GPUs. Also rename the allocator code to move away from
gk20a_ prefixed structs and functions.

This caused one issue with the nvgpu_alloc() and nvgpu_free() functions.
There was a function for allocating either with kmalloc() or vmalloc()
depending on the size of the allocation. Those have now been renamed to
nvgpu_kalloc() and nvgpu_kfree().

Bug 1799159

Change-Id: Iddda92c013612bcb209847084ec85b8953002fa5
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: http://git-master/r/1274400
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

2 files changed, 466 insertions, 0 deletions

diff --git a/drivers/gpu/nvgpu/include/nvgpu/allocator.h b/drivers/gpu/nvgpu/include/nvgpu/allocator.h
new file mode 100644
index 00000000..dee9b562
--- /dev/null
+++ b/drivers/gpu/nvgpu/include/nvgpu/allocator.h
@@ -0,0 +1,302 @@
+/*
+ * Copyright (c) 2011-2016, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef NVGPU_ALLOCATOR_H
+#define NVGPU_ALLOCATOR_H
+
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/platform_device.h>
+
+/* #define ALLOCATOR_DEBUG */
+
+struct nvgpu_allocator;
+struct nvgpu_alloc_carveout;
+struct vm_gk20a;
+struct gk20a;
+
+/*
+ * Operations for an allocator to implement.
+ */
+struct nvgpu_allocator_ops {
+        u64  (*alloc)(struct nvgpu_allocator *allocator, u64 len);
+        void (*free)(struct nvgpu_allocator *allocator, u64 addr);
+
+        /*
+         * Special interface to allocate a memory region with a specific
+         * starting address. Yikes. Note: if free() works for freeing both
+         * regular and fixed allocations then free_fixed() does not need to
+         * be implemented. This behavior exists for legacy reasons and should
+         * not be propagated to new allocators.
+         */
+        u64  (*alloc_fixed)(struct nvgpu_allocator *allocator,
+                             u64 base, u64 len);
+        void (*free_fixed)(struct nvgpu_allocator *allocator,
+                            u64 base, u64 len);
+
+        /*
+         * Allow allocators to reserve space for carveouts.
+         */
+        int  (*reserve_carveout)(struct nvgpu_allocator *allocator,
+                                 struct nvgpu_alloc_carveout *co);
+        void (*release_carveout)(struct nvgpu_allocator *allocator,
+                                 struct nvgpu_alloc_carveout *co);
+
+        /*
+         * Returns info about the allocator.
+         */
+        u64  (*base)(struct nvgpu_allocator *allocator);
+        u64  (*length)(struct nvgpu_allocator *allocator);
+        u64  (*end)(struct nvgpu_allocator *allocator);
+        int  (*inited)(struct nvgpu_allocator *allocator);
+        u64  (*space)(struct nvgpu_allocator *allocator);
+
+        /* Destructor. */
+        void (*fini)(struct nvgpu_allocator *allocator);
+
+        /* Debugging. */
+        void (*print_stats)(struct nvgpu_allocator *allocator,
+                            struct seq_file *s, int lock);
+};
+
+struct nvgpu_allocator {
+        char name[32];
+        struct mutex lock;
+
+        void *priv;
+        const struct nvgpu_allocator_ops *ops;
+
+        struct dentry *debugfs_entry;
+        bool debug;                             /* Control for debug msgs. */
+};
+
+struct nvgpu_alloc_carveout {
+        const char *name;
+        u64 base;
+        u64 length;
+
+        struct nvgpu_allocator *allocator;
+
+        /*
+         * For usage by the allocator implementation.
+         */
+        struct list_head co_entry;
+};
+
+#define NVGPU_CARVEOUT(__name, __base, __length)        \
+        {                                               \
+                .name = (__name),                       \
+                .base = (__base),                       \
+                .length = (__length)                    \
+        }
+
+/*
+ * These are the available allocator flags.
+ *
+ *   GPU_ALLOC_GVA_SPACE
+ *
+ *     This flag makes sense for the buddy allocator only. It specifies that the
+ *     allocator will be used for managing a GVA space. When managing GVA spaces
+ *     special care has to be taken to ensure that allocations of similar PTE
+ *     sizes are placed in the same PDE block. This allows the higher level
+ *     code to skip defining both small and large PTE tables for every PDE. That
+ *     can save considerable memory for address spaces that have a lot of
+ *     allocations.
+ *
+ *   GPU_ALLOC_NO_ALLOC_PAGE
+ *
+ *     For any allocator that needs to manage a resource in a latency critical
+ *     path this flag specifies that the allocator should not use any kmalloc()
+ *     or similar functions during normal operation. Initialization routines
+ *     may still use kmalloc(). This prevents the possibility of long waits for
+ *     pages when using alloc_page(). Currently only the bitmap allocator
+ *     implements this functionality.
+ *
+ *     Also note that if you accept this flag then you must also define the
+ *     free_fixed() function. Since no meta-data is allocated to help free
+ *     allocations you need to keep track of the meta-data yourself (in this
+ *     case the base and length of the allocation as opposed to just the base
+ *     of the allocation).
+ *
+ *   GPU_ALLOC_4K_VIDMEM_PAGES
+ *
+ *     We manage vidmem pages at a large page granularity for performance
+ *     reasons; however, this can lead to wasting memory. For page allocators
+ *     setting this flag will tell the allocator to manage pools of 4K pages
+ *     inside internally allocated large pages.
+ *
+ *     Currently this flag is ignored since the only usage of the page allocator
+ *     uses a 4K block size already. However, this flag has been reserved since
+ *     it will be necessary in the future.
+ *
+ *   GPU_ALLOC_FORCE_CONTIG
+ *
+ *     Force allocations to be contiguous. Currently only relevant for page
+ *     allocators since all other allocators are naturally contiguous.
+ *
+ *   GPU_ALLOC_NO_SCATTER_GATHER
+ *
+ *     The page allocator normally returns a scatter gather data structure for
+ *     allocations (to handle discontiguous pages). However, at times that can
+ *     be annoying so this flag forces the page allocator to return a u64
+ *     pointing to the allocation base (requires GPU_ALLOC_FORCE_CONTIG to be
+ *     set as well).
+ */
+#define GPU_ALLOC_GVA_SPACE             0x1
+#define GPU_ALLOC_NO_ALLOC_PAGE         0x2
+#define GPU_ALLOC_4K_VIDMEM_PAGES       0x4
+#define GPU_ALLOC_FORCE_CONTIG          0x8
+#define GPU_ALLOC_NO_SCATTER_GATHER     0x10
+
+static inline void alloc_lock(struct nvgpu_allocator *a)
+{
+        mutex_lock(&a->lock);
+}
+
+static inline void alloc_unlock(struct nvgpu_allocator *a)
+{
+        mutex_unlock(&a->lock);
+}
+
+/*
+ * Buddy allocator specific initializers.
+ */
+int  __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *a,
+                                  struct vm_gk20a *vm, const char *name,
+                                  u64 base, u64 size, u64 blk_size,
+                                  u64 max_order, u64 flags);
+int  nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *a,
+                                const char *name, u64 base, u64 size,
+                                u64 blk_size, u64 flags);
+
+/*
+ * Bitmap initializers.
+ */
+int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *a,
+                                const char *name, u64 base, u64 length,
+                                u64 blk_size, u64 flags);
+
+/*
+ * Page allocator initializers.
+ */
+int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *a,
+                              const char *name, u64 base, u64 length,
+                              u64 blk_size, u64 flags);
+
+/*
+ * Lockless allocatior initializers.
+ * Note: This allocator can only allocate fixed-size structures of a
+ * pre-defined size.
+ */
+int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *a,
+                                  const char *name, u64 base, u64 length,
+                                  u64 struct_size, u64 flags);
+
+#define GPU_BALLOC_MAX_ORDER            31
+
+/*
+ * Allocator APIs.
+ */
+u64  nvgpu_alloc(struct nvgpu_allocator *allocator, u64 len);
+void nvgpu_free(struct nvgpu_allocator *allocator, u64 addr);
+
+u64  nvgpu_alloc_fixed(struct nvgpu_allocator *allocator, u64 base, u64 len);
+void nvgpu_free_fixed(struct nvgpu_allocator *allocator, u64 base, u64 len);
+
+int  nvgpu_alloc_reserve_carveout(struct nvgpu_allocator *a,
+                                  struct nvgpu_alloc_carveout *co);
+void nvgpu_alloc_release_carveout(struct nvgpu_allocator *a,
+                                  struct nvgpu_alloc_carveout *co);
+
+u64  nvgpu_alloc_base(struct nvgpu_allocator *a);
+u64  nvgpu_alloc_length(struct nvgpu_allocator *a);
+u64  nvgpu_alloc_end(struct nvgpu_allocator *a);
+u64  nvgpu_alloc_initialized(struct nvgpu_allocator *a);
+u64  nvgpu_alloc_space(struct nvgpu_allocator *a);
+
+void nvgpu_alloc_destroy(struct nvgpu_allocator *allocator);
+
+void nvgpu_alloc_print_stats(struct nvgpu_allocator *a,
+                             struct seq_file *s, int lock);
+
+/*
+ * Common functionality for the internals of the allocators.
+ */
+void nvgpu_init_alloc_debug(struct gk20a *g, struct nvgpu_allocator *a);
+void nvgpu_fini_alloc_debug(struct nvgpu_allocator *a);
+
+int  __nvgpu_alloc_common_init(struct nvgpu_allocator *a,
+                               const char *name, void *priv, bool dbg,
+                               const struct nvgpu_allocator_ops *ops);
+
+static inline void nvgpu_alloc_enable_dbg(struct nvgpu_allocator *a)
+{
+        a->debug = true;
+}
+
+static inline void nvgpu_alloc_disable_dbg(struct nvgpu_allocator *a)
+{
+        a->debug = false;
+}
+
+/*
+ * Debug stuff.
+ */
+extern u32 nvgpu_alloc_tracing_on;
+
+void nvgpu_alloc_debugfs_init(struct device *dev);
+
+#define nvgpu_alloc_trace_func()                        \
+        do {                                            \
+                if (nvgpu_alloc_tracing_on)             \
+                        trace_printk("%s\n", __func__); \
+        } while (0)
+
+#define nvgpu_alloc_trace_func_done()                           \
+        do {                                                    \
+                if (nvgpu_alloc_tracing_on)                     \
+                        trace_printk("%s_done\n", __func__);    \
+        } while (0)
+
+#define __alloc_pstat(seq, allocator, fmt, arg...)              \
+        do {                                                    \
+                if (s)                                          \
+                        seq_printf(seq, fmt, ##arg);            \
+                else                                            \
+                        alloc_dbg(allocator, fmt, ##arg);       \
+        } while (0)
+
+#define __alloc_dbg(a, fmt, arg...)                                     \
+        pr_info("%-25s %25s() " fmt, (a)->name, __func__, ##arg)
+
+#if defined(ALLOCATOR_DEBUG)
+/*
+ * Always print the debug messages...
+ */
+#define alloc_dbg(a, fmt, arg...) __alloc_dbg(a, fmt, ##arg)
+#else
+/*
+ * Only print debug messages if debug is enabled for a given allocator.
+ */
+#define alloc_dbg(a, fmt, arg...)                       \
+        do {                                            \
+                if ((a)->debug)                         \
+                        __alloc_dbg((a), fmt, ##arg);   \
+        } while (0)
+
+#endif
+
+#endif /* NVGPU_ALLOCATOR_H */
diff --git a/drivers/gpu/nvgpu/include/nvgpu/page_allocator.h b/drivers/gpu/nvgpu/include/nvgpu/page_allocator.h
new file mode 100644
index 00000000..7c21c117
--- /dev/null
+++ b/drivers/gpu/nvgpu/include/nvgpu/page_allocator.h
@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef PAGE_ALLOCATOR_PRIV_H
+#define PAGE_ALLOCATOR_PRIV_H
+
+#include <linux/list.h>
+#include <linux/rbtree.h>
+
+#include <nvgpu/allocator.h>
+
+struct nvgpu_allocator;
+
+/*
+ * This allocator implements the ability to do SLAB style allocation since the
+ * GPU has two page sizes available - 4k and 64k/128k. When the default
+ * granularity is the large page size (64k/128k) small allocations become very
+ * space inefficient. This is most notable in PDE and PTE blocks which are 4k
+ * in size.
+ *
+ * Thus we need the ability to suballocate in 64k pages. The way we do this for
+ * the GPU is as follows. We have several buckets for sub-64K allocations:
+ *
+ *   B0 - 4k
+ *   B1 - 8k
+ *   B3 - 16k
+ *   B4 - 32k
+ *   B5 - 64k (for when large pages are 128k)
+ *
+ * When an allocation comes in for less than the large page size (from now on
+ * assumed to be 64k) the allocation is satisfied by one of the buckets.
+ */
+struct page_alloc_slab {
+        struct list_head empty;
+        struct list_head partial;
+        struct list_head full;
+
+        int nr_empty;
+        int nr_partial;
+        int nr_full;
+
+        u32 slab_size;
+};
+
+enum slab_page_state {
+        SP_EMPTY,
+        SP_PARTIAL,
+        SP_FULL,
+        SP_NONE
+};
+
+struct page_alloc_slab_page {
+        unsigned long bitmap;
+        u64 page_addr;
+        u32 slab_size;
+
+        u32 nr_objects;
+        u32 nr_objects_alloced;
+
+        enum slab_page_state state;
+
+        struct page_alloc_slab *owner;
+        struct list_head list_entry;
+};
+
+struct page_alloc_chunk {
+        struct list_head list_entry;
+
+        u64 base;
+        u64 length;
+};
+
+/*
+ * Struct to handle internal management of page allocation. It holds a list
+ * of the chunks of pages that make up the overall allocation - much like a
+ * scatter gather table.
+ */
+struct nvgpu_page_alloc {
+        struct list_head alloc_chunks;
+
+        int nr_chunks;
+        u64 length;
+
+        /*
+         * Only useful for the RB tree - since the alloc may have discontiguous
+         * pages the base is essentially irrelevant except for the fact that it
+         * is guarenteed to be unique.
+         */
+        u64 base;
+
+        struct rb_node tree_entry;
+
+        /*
+         * Set if this is a slab alloc. Points back to the slab page that owns
+         * this particular allocation. nr_chunks will always be 1 if this is
+         * set.
+         */
+        struct page_alloc_slab_page *slab_page;
+};
+
+struct nvgpu_page_allocator {
+        struct nvgpu_allocator *owner;  /* Owner of this allocator. */
+
+        /*
+         * Use a buddy allocator to manage the allocation of the underlying
+         * pages. This lets us abstract the discontiguous allocation handling
+         * out of the annoyingly complicated buddy allocator.
+         */
+        struct nvgpu_allocator source_allocator;
+
+        /*
+         * Page params.
+         */
+        u64 base;
+        u64 length;
+        u64 page_size;
+        u32 page_shift;
+
+        struct rb_root allocs;          /* Outstanding allocations. */
+
+        struct page_alloc_slab *slabs;
+        int nr_slabs;
+
+        u64 flags;
+
+        /*
+         * Stat tracking.
+         */
+        u64 nr_allocs;
+        u64 nr_frees;
+        u64 nr_fixed_allocs;
+        u64 nr_fixed_frees;
+        u64 nr_slab_allocs;
+        u64 nr_slab_frees;
+        u64 pages_alloced;
+        u64 pages_freed;
+};
+
+static inline struct nvgpu_page_allocator *page_allocator(
+        struct nvgpu_allocator *a)
+{
+        return (struct nvgpu_page_allocator *)(a)->priv;
+}
+
+static inline struct nvgpu_allocator *palloc_owner(
+        struct nvgpu_page_allocator *a)
+{
+        return a->owner;
+}
+
+#endif