1 files changed, 369 insertions, 0 deletions

diff --git a/include/nvgpu/gmmu.h b/include/nvgpu/gmmu.h
new file mode 100644
index 0000000..2fc0d44
--- /dev/null
+++ b/include/nvgpu/gmmu.h
@@ -0,0 +1,369 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_GMMU_H
+#define NVGPU_GMMU_H
+
+#include <nvgpu/types.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/list.h>
+#include <nvgpu/rbtree.h>
+#include <nvgpu/lock.h>
+#include <nvgpu/bitops.h>
+#include <nvgpu/mm.h>
+
+/*
+ * This is the GMMU API visible to blocks outside of the GMMU. Basically this
+ * API supports all the different types of mappings that might be done in the
+ * GMMU.
+ */
+
+struct vm_gk20a;
+struct nvgpu_mem;
+
+#define GMMU_PAGE_SIZE_SMALL    0U
+#define GMMU_PAGE_SIZE_BIG      1U
+#define GMMU_PAGE_SIZE_KERNEL   2U
+#define GMMU_NR_PAGE_SIZES      3U
+
+enum gk20a_mem_rw_flag {
+        gk20a_mem_flag_none = 0,        /* RW */
+        gk20a_mem_flag_read_only = 1,   /* RO */
+        gk20a_mem_flag_write_only = 2,  /* WO */
+};
+
+/*
+ * Minimum size of a cache. The number of different caches in the nvgpu_pd_cache
+ * structure is of course depending on this. The MIN_SHIFT define is the right
+ * number of bits to shift to determine which list to use in the array of lists.
+ *
+ * For Linux, limit the use of the cache to entries less than the page size, to
+ * avoid potential problems with running out of CMA memory when allocating large,
+ * contiguous slabs, as would be required for non-iommmuable chips.
+ */
+#define NVGPU_PD_CACHE_MIN              256U
+#define NVGPU_PD_CACHE_MIN_SHIFT        9U
+
+#ifdef __KERNEL__
+
+#if PAGE_SIZE == 4096
+#define NVGPU_PD_CACHE_COUNT            4U
+#elif PAGE_SIZE == 65536
+#define NVGPU_PD_CACHE_COUNT            8U
+#else
+#error "Unsupported page size."
+#endif
+
+#else
+#define NVGPU_PD_CACHE_COUNT            8U
+#endif
+
+#define NVGPU_PD_CACHE_SIZE             (NVGPU_PD_CACHE_MIN * (1U << NVGPU_PD_CACHE_COUNT))
+
+struct nvgpu_pd_mem_entry {
+        struct nvgpu_mem                mem;
+
+        /*
+         * Size of the page directories (not the mem). alloc_map is a bitmap
+         * showing which PDs have been allocated.
+         *
+         * The size of mem will be NVGPU_PD_CACHE_SIZE
+         * and pd_size will always be a power of 2.
+         *
+         */
+        u32                             pd_size;
+        DECLARE_BITMAP(alloc_map, NVGPU_PD_CACHE_SIZE / NVGPU_PD_CACHE_MIN);
+
+        /* Total number of allocations in this PD. */
+        u32                             allocs;
+
+        struct nvgpu_list_node          list_entry;
+        struct nvgpu_rbtree_node        tree_entry;
+};
+
+static inline struct nvgpu_pd_mem_entry *
+nvgpu_pd_mem_entry_from_list_entry(struct nvgpu_list_node *node)
+{
+        return (struct nvgpu_pd_mem_entry *)
+                ((uintptr_t)node -
+                 offsetof(struct nvgpu_pd_mem_entry, list_entry));
+};
+
+static inline struct nvgpu_pd_mem_entry *
+nvgpu_pd_mem_entry_from_tree_entry(struct nvgpu_rbtree_node *node)
+{
+        return (struct nvgpu_pd_mem_entry *)
+                ((uintptr_t)node -
+                 offsetof(struct nvgpu_pd_mem_entry, tree_entry));
+};
+
+/*
+ * A cache for allocating PD memory from. This enables smaller PDs to be packed
+ * into single pages.
+ *
+ * This is fairly complex so see the documentation in pd_cache.c for a full
+ * description of how this is organized.
+ */
+struct nvgpu_pd_cache {
+        /*
+         * Array of lists of full nvgpu_pd_mem_entries and partially full (or
+         * empty) nvgpu_pd_mem_entries.
+         */
+        struct nvgpu_list_node           full[NVGPU_PD_CACHE_COUNT];
+        struct nvgpu_list_node           partial[NVGPU_PD_CACHE_COUNT];
+
+        /*
+         * Tree of all allocated struct nvgpu_mem's for fast look up.
+         */
+        struct nvgpu_rbtree_node        *mem_tree;
+
+        /*
+         * All access to the cache much be locked. This protects the lists and
+         * the rb tree.
+         */
+        struct nvgpu_mutex               lock;
+};
+
+/*
+ * GMMU page directory. This is the kernel's tracking of a list of PDEs or PTEs
+ * in the GMMU.
+ */
+struct nvgpu_gmmu_pd {
+        /*
+         * DMA memory describing the PTEs or PDEs. @mem_offs describes the
+         * offset of the PDE table in @mem. @cached specifies if this PD is
+         * using pd_cache memory.
+         */
+        struct nvgpu_mem        *mem;
+        u32                      mem_offs;
+        bool                     cached;
+
+        /*
+         * List of pointers to the next level of page tables. Does not
+         * need to be populated when this PD is pointing to PTEs.
+         */
+        struct nvgpu_gmmu_pd    *entries;
+        int                      num_entries;
+};
+
+/*
+ * Reduce the number of arguments getting passed through the various levels of
+ * GMMU mapping functions.
+ *
+ * The following fields are set statically and do not change throughout the
+ * mapping call:
+ *
+ *   pgsz:        Index into the page size table.
+ *   kind_v:      Kind attributes for mapping.
+ *   cacheable:   Cacheability of the mapping.
+ *   rw_flag:     Flag from enum gk20a_mem_rw_flag
+ *   sparse:      Set if the mapping should be sparse.
+ *   priv:        Privilidged mapping.
+ *   coherent:    Set if the mapping should be IO coherent.
+ *   valid:       Set if the PTE should be marked valid.
+ *   aperture:    VIDMEM or SYSMEM.
+ *   debug:       When set print debugging info.
+ *   platform_atomic: True if platform_atomic flag is valid.
+ *
+ * These fields are dynamically updated as necessary during the map:
+ *
+ *   ctag:        Comptag line in the comptag cache;
+ *                updated every time we write a PTE.
+ */
+struct nvgpu_gmmu_attrs {
+        u32                      pgsz;
+        u32                      kind_v;
+        u64                      ctag;
+        bool                     cacheable;
+        enum gk20a_mem_rw_flag   rw_flag;
+        bool                     sparse;
+        bool                     priv;
+        bool                     valid;
+        enum nvgpu_aperture      aperture;
+        bool                     debug;
+        bool                     l3_alloc;
+        bool                     platform_atomic;
+};
+
+struct gk20a_mmu_level {
+        int hi_bit[2];
+        int lo_bit[2];
+
+        /*
+         * Build map from virt_addr -> phys_addr.
+         */
+        void (*update_entry)(struct vm_gk20a *vm,
+                             const struct gk20a_mmu_level *l,
+                             struct nvgpu_gmmu_pd *pd,
+                             u32 pd_idx,
+                             u64 phys_addr,
+                             u64 virt_addr,
+                             struct nvgpu_gmmu_attrs *attrs);
+        u32 entry_size;
+        /*
+         * Get pde page size
+         */
+        u32 (*get_pgsz)(struct gk20a *g, const struct gk20a_mmu_level *l,
+                                struct nvgpu_gmmu_pd *pd, u32 pd_idx);
+};
+
+static inline const char *nvgpu_gmmu_perm_str(enum gk20a_mem_rw_flag p)
+{
+        switch (p) {
+        case gk20a_mem_flag_none:
+                return "RW";
+        case gk20a_mem_flag_write_only:
+                return "WO";
+        case gk20a_mem_flag_read_only:
+                return "RO";
+        default:
+                return "??";
+        }
+}
+
+int nvgpu_gmmu_init_page_table(struct vm_gk20a *vm);
+
+/**
+ * nvgpu_gmmu_map - Map memory into the GMMU.
+ *
+ * Kernel space.
+ */
+u64 nvgpu_gmmu_map(struct vm_gk20a *vm,
+                   struct nvgpu_mem *mem,
+                   u64 size,
+                   u32 flags,
+                   enum gk20a_mem_rw_flag rw_flag,
+                   bool priv,
+                   enum nvgpu_aperture aperture);
+
+/**
+ * nvgpu_gmmu_map_fixed - Map memory into the GMMU.
+ *
+ * Kernel space.
+ */
+u64 nvgpu_gmmu_map_fixed(struct vm_gk20a *vm,
+                         struct nvgpu_mem *mem,
+                         u64 addr,
+                         u64 size,
+                         u32 flags,
+                         enum gk20a_mem_rw_flag rw_flag,
+                         bool priv,
+                         enum nvgpu_aperture aperture);
+
+/**
+ * nvgpu_gmmu_unmap - Unmap a buffer.
+ *
+ * Kernel space.
+ */
+void nvgpu_gmmu_unmap(struct vm_gk20a *vm,
+                      struct nvgpu_mem *mem,
+                      u64 gpu_va);
+
+int nvgpu_pd_alloc(struct vm_gk20a *vm,
+                   struct nvgpu_gmmu_pd *pd,
+                   u32 bytes);
+
+void nvgpu_pd_free(struct vm_gk20a *vm, struct nvgpu_gmmu_pd *pd);
+int nvgpu_pd_cache_alloc_direct(struct gk20a *g,
+                                  struct nvgpu_gmmu_pd *pd, u32 bytes);
+void nvgpu_pd_cache_free_direct(struct gk20a *g, struct nvgpu_gmmu_pd *pd);
+int nvgpu_pd_cache_init(struct gk20a *g);
+void nvgpu_pd_cache_fini(struct gk20a *g);
+
+/*
+ * Some useful routines that are shared across chips.
+ */
+static inline u32 pd_offset_from_index(const struct gk20a_mmu_level *l,
+                                       u32 pd_idx)
+{
+        return (pd_idx * l->entry_size) / sizeof(u32);
+}
+
+static inline void pd_write(struct gk20a *g, struct nvgpu_gmmu_pd *pd,
+                            size_t w, size_t data)
+{
+        nvgpu_mem_wr32(g, pd->mem, (pd->mem_offs / sizeof(u32)) + w, data);
+}
+
+/**
+ * __nvgpu_pte_words - Compute number of words in a PTE.
+ *
+ * @g  - The GPU.
+ *
+ * This computes and returns the size of a PTE for the passed chip.
+ */
+u32 __nvgpu_pte_words(struct gk20a *g);
+
+/**
+ * __nvgpu_get_pte - Get the contents of a PTE by virtual address
+ *
+ * @g     - The GPU.
+ * @vm    - VM to look in.
+ * @vaddr - GPU virtual address.
+ * @pte   - [out] Set to the contents of the PTE.
+ *
+ * Find a PTE in the passed VM based on the passed GPU virtual address. This
+ * will @pte with a copy of the contents of the PTE. @pte must be an array of
+ * u32s large enough to contain the PTE. This can be computed using
+ * __nvgpu_pte_words().
+ *
+ * If you wish to write to this PTE then you may modify @pte and then use the
+ * __nvgpu_set_pte().
+ *
+ * This function returns 0 if the PTE is found and -EINVAL otherwise.
+ */
+int __nvgpu_get_pte(struct gk20a *g, struct vm_gk20a *vm, u64 vaddr, u32 *pte);
+
+/**
+ * __nvgpu_set_pte - Set a PTE based on virtual address
+ *
+ * @g     - The GPU.
+ * @vm    - VM to look in.
+ * @vaddr - GPU virtual address.
+ * @pte   - The contents of the PTE to write.
+ *
+ * Find a PTE and overwrite the contents of that PTE with the passed in data
+ * located in @pte. If the PTE does not exist then no writing will happen. That
+ * is this function will not fill out the page tables for you. The expectation
+ * is that the passed @vaddr has already been mapped and this is just modifying
+ * the mapping (for instance changing invalid to valid).
+ *
+ * @pte must contain at least the required words for the PTE. See
+ * __nvgpu_pte_words().
+ *
+ * This function returns 0 on success and -EINVAL otherwise.
+ */
+int __nvgpu_set_pte(struct gk20a *g, struct vm_gk20a *vm, u64 vaddr, u32 *pte);
+
+
+/*
+ * Internal debugging routines. Probably not something you want to use.
+ */
+#define pte_dbg(g, attrs, fmt, args...)                                 \
+        do {                                                            \
+                if ((attrs != NULL) && (attrs->debug))                  \
+                        nvgpu_info(g, fmt, ##args);                     \
+                else                                                    \
+                        nvgpu_log(g, gpu_dbg_pte, fmt, ##args);         \
+        } while (0)
+
+#endif /* NVGPU_GMMU_H */