1 files changed, 330 insertions, 0 deletions

diff --git a/include/nvgpu/vm.h b/include/nvgpu/vm.h
new file mode 100644
index 0000000..3867c74
--- /dev/null
+++ b/include/nvgpu/vm.h
@@ -0,0 +1,330 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_VM_H
+#define NVGPU_VM_H
+
+#include <nvgpu/kref.h>
+#include <nvgpu/list.h>
+#include <nvgpu/rbtree.h>
+#include <nvgpu/types.h>
+#include <nvgpu/gmmu.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/allocator.h>
+
+struct vm_gk20a;
+struct nvgpu_vm_area;
+struct gk20a_comptag_allocator;
+
+/*
+ * Defined by each OS. Allows the common VM code do things to the OS specific
+ * buffer structures.
+ */
+struct nvgpu_os_buffer;
+
+#ifdef __KERNEL__
+#include <nvgpu/linux/vm.h>
+#elif defined(__NVGPU_POSIX__)
+#include <nvgpu/posix/vm.h>
+#else
+/* QNX include goes here. */
+#include <nvgpu_rmos/include/vm.h>
+#endif
+
+/**
+ * This header contains the OS agnostic APIs for dealing with VMs. Most of the
+ * VM implementation is system specific - it must translate from a platform's
+ * representation of DMA'able memory to our nvgpu_mem notion.
+ *
+ * However, some stuff is platform agnostic. VM ref-counting and the VM struct
+ * itself are platform agnostic. Also, the initialization and destruction of
+ * VMs is the same across all platforms (for now).
+ *
+ * VM Architecture:
+ * ----------------
+ *
+ *   The VM managment in nvgpu is split up as follows: a vm_gk20a struct which
+ * defines an address space. Each address space is a set of page tables and a
+ * GPU Virtual Address (GVA) allocator. Any number of channels may bind to a VM.
+ *
+ *     +----+  +----+     +----+     +-----+     +-----+
+ *     | C1 |  | C2 | ... | Cn |     | VM1 | ... | VMn |
+ *     +-+--+  +-+--+     +-+--+     +--+--+     +--+--+
+ *       |       |          |           |           |
+ *       |       |          +----->-----+           |
+ *       |       +---------------->-----+           |
+ *       +------------------------>-----------------+
+ *
+ *   Each VM also manages a set of mapped buffers (struct nvgpu_mapped_buf)
+ * which corresponds to _user space_ buffers which have been mapped into this VM.
+ * Kernel space mappings (created by nvgpu_gmmu_map()) are not tracked by VMs.
+ * This may be an architectural bug, but for now it seems to be OK. VMs can be
+ * closed in various ways - refs counts hitting zero, direct calls to the remove
+ * routine, etc. Note: this is going to change. VM cleanup is going to be
+ * homogonized around ref-counts. When a VM is closed all mapped buffers in the
+ * VM are unmapped from the GMMU. This means that those mappings will no longer
+ * be valid and any subsequent access by the GPU will fault. That means one must
+ * ensure the VM is not in use before closing it.
+ *
+ *   VMs may also contain VM areas (struct nvgpu_vm_area) which are created for
+ * the purpose of sparse and/or fixed mappings. If userspace wishes to create a
+ * fixed mapping it must first create a VM area - either with a fixed address or
+ * not. VM areas are reserved - other mapping operations will not use the space.
+ * Userspace may then create fixed mappings within that VM area.
+ */
+
+/* map/unmap batch state */
+struct vm_gk20a_mapping_batch {
+        bool gpu_l2_flushed;
+        bool need_tlb_invalidate;
+};
+
+struct nvgpu_mapped_buf {
+        struct vm_gk20a *vm;
+        struct nvgpu_vm_area *vm_area;
+
+        struct nvgpu_ref ref;
+
+        struct nvgpu_rbtree_node node;
+        struct nvgpu_list_node buffer_list;
+        u64 addr;
+        u64 size;
+
+        u32 pgsz_idx;
+
+        u32 flags;
+        u32 kind;
+        bool va_allocated;
+
+        /*
+         * Separate from the nvgpu_os_buffer struct to clearly distinguish
+         * lifetime. A nvgpu_mapped_buf_priv will _always_ be wrapped by a
+         * struct nvgpu_mapped_buf; however, there are times when a struct
+         * nvgpu_os_buffer would be separate. This aims to prevent dangerous
+         * usage of container_of() or the like in OS code.
+         */
+        struct nvgpu_mapped_buf_priv os_priv;
+};
+
+static inline struct nvgpu_mapped_buf *
+nvgpu_mapped_buf_from_buffer_list(struct nvgpu_list_node *node)
+{
+        return (struct nvgpu_mapped_buf *)
+                ((uintptr_t)node - offsetof(struct nvgpu_mapped_buf,
+                                            buffer_list));
+}
+
+static inline struct nvgpu_mapped_buf *
+mapped_buffer_from_rbtree_node(struct nvgpu_rbtree_node *node)
+{
+        return (struct nvgpu_mapped_buf *)
+                  ((uintptr_t)node - offsetof(struct nvgpu_mapped_buf, node));
+}
+
+struct vm_gk20a {
+        struct mm_gk20a *mm;
+        struct gk20a_as_share *as_share; /* as_share this represents */
+        char name[20];
+
+        u64 va_start;
+        u64 va_limit;
+
+        int num_user_mapped_buffers;
+
+        bool big_pages;   /* enable large page support */
+        bool enable_ctag;
+        bool guest_managed; /* whether the vm addr space is managed by guest */
+
+        u32 big_page_size;
+
+        bool userspace_managed;
+
+        const struct gk20a_mmu_level *mmu_levels;
+
+        struct nvgpu_ref ref;
+
+        struct nvgpu_mutex update_gmmu_lock;
+
+        struct nvgpu_gmmu_pd pdb;
+
+        /*
+         * These structs define the address spaces. In some cases it's possible
+         * to merge address spaces (user and user_lp) and in other cases it's
+         * not. vma[] allows the code to be agnostic to this by always using
+         * address spaces through this pointer array.
+         */
+        struct nvgpu_allocator *vma[GMMU_NR_PAGE_SIZES];
+        struct nvgpu_allocator kernel;
+        struct nvgpu_allocator user;
+        struct nvgpu_allocator user_lp;
+
+        struct nvgpu_rbtree_node *mapped_buffers;
+
+        struct nvgpu_list_node vm_area_list;
+
+#ifdef CONFIG_TEGRA_GR_VIRTUALIZATION
+        u64 handle;
+#endif
+        u32 gmmu_page_sizes[GMMU_NR_PAGE_SIZES];
+
+        /* if non-NULL, kref_put will use this batch when
+           unmapping. Must hold vm->update_gmmu_lock. */
+        struct vm_gk20a_mapping_batch *kref_put_batch;
+
+        /*
+         * Each address space needs to have a semaphore pool.
+         */
+        struct nvgpu_semaphore_pool *sema_pool;
+
+        /*
+         * Create sync point read only map for sync point range.
+         * Channels sharing same vm will also share same sync point ro map
+         */
+        u64 syncpt_ro_map_gpu_va;
+        /* Protect allocation of sync point map */
+        struct nvgpu_mutex syncpt_ro_map_lock;
+};
+
+/*
+ * Mapping flags.
+ */
+#define NVGPU_VM_MAP_FIXED_OFFSET                       BIT32(0)
+#define NVGPU_VM_MAP_CACHEABLE                          BIT32(1)
+#define NVGPU_VM_MAP_IO_COHERENT                        BIT32(2)
+#define NVGPU_VM_MAP_UNMAPPED_PTE                       BIT32(3)
+#define NVGPU_VM_MAP_DIRECT_KIND_CTRL                   BIT32(4)
+#define NVGPU_VM_MAP_L3_ALLOC                           BIT32(5)
+#define NVGPU_VM_MAP_PLATFORM_ATOMIC                    BIT32(6)
+
+#define NVGPU_KIND_INVALID                              -1
+
+void nvgpu_vm_get(struct vm_gk20a *vm);
+void nvgpu_vm_put(struct vm_gk20a *vm);
+
+int vm_aspace_id(struct vm_gk20a *vm);
+bool nvgpu_big_pages_possible(struct vm_gk20a *vm, u64 base, u64 size);
+
+int nvgpu_vm_pde_coverage_bit_count(struct vm_gk20a *vm);
+
+/* batching eliminates redundant cache flushes and invalidates */
+void nvgpu_vm_mapping_batch_start(struct vm_gk20a_mapping_batch *batch);
+void nvgpu_vm_mapping_batch_finish(
+        struct vm_gk20a *vm, struct vm_gk20a_mapping_batch *batch);
+/* called when holding vm->update_gmmu_lock */
+void nvgpu_vm_mapping_batch_finish_locked(
+        struct vm_gk20a *vm, struct vm_gk20a_mapping_batch *batch);
+
+/* get reference to all currently mapped buffers */
+int nvgpu_vm_get_buffers(struct vm_gk20a *vm,
+                         struct nvgpu_mapped_buf ***mapped_buffers,
+                         int *num_buffers);
+/* put references on the given buffers */
+void nvgpu_vm_put_buffers(struct vm_gk20a *vm,
+                          struct nvgpu_mapped_buf **mapped_buffers,
+                          int num_buffers);
+
+struct nvgpu_mapped_buf *nvgpu_vm_find_mapping(struct vm_gk20a *vm,
+                                               struct nvgpu_os_buffer *os_buf,
+                                               u64 map_addr,
+                                               u32 flags,
+                                               int kind);
+
+struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
+                                      struct nvgpu_os_buffer *os_buf,
+                                      struct nvgpu_sgt *sgt,
+                                      u64 map_addr,
+                                      u64 map_size,
+                                      u64 phys_offset,
+                                      int rw,
+                                      u32 flags,
+                                      s16 compr_kind,
+                                      s16 incompr_kind,
+                                      struct vm_gk20a_mapping_batch *batch,
+                                      enum nvgpu_aperture aperture);
+
+void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset,
+                    struct vm_gk20a_mapping_batch *batch);
+
+/*
+ * Implemented by each OS. Called from within the core VM code to handle OS
+ * specific components of an nvgpu_mapped_buf.
+ */
+void nvgpu_vm_unmap_system(struct nvgpu_mapped_buf *mapped_buffer);
+
+/*
+ * Don't use this outside of the core VM code!
+ */
+void __nvgpu_vm_unmap_ref(struct nvgpu_ref *ref);
+
+u64 nvgpu_os_buf_get_size(struct nvgpu_os_buffer *os_buf);
+
+/*
+ * These all require the VM update lock to be held.
+ */
+struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf(
+        struct vm_gk20a *vm, u64 addr);
+struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_range(
+        struct vm_gk20a *vm, u64 addr);
+struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_less_than(
+        struct vm_gk20a *vm, u64 addr);
+
+int nvgpu_insert_mapped_buf(struct vm_gk20a *vm,
+                            struct nvgpu_mapped_buf *mapped_buffer);
+void nvgpu_remove_mapped_buf(struct vm_gk20a *vm,
+                             struct nvgpu_mapped_buf *mapped_buffer);
+
+/*
+ * Initialize a preallocated vm
+ */
+int __nvgpu_vm_init(struct mm_gk20a *mm,
+                           struct vm_gk20a *vm,
+                           u32 big_page_size,
+                           u64 low_hole,
+                           u64 kernel_reserved,
+                           u64 aperture_size,
+                           bool big_pages,
+                           bool userspace_managed,
+                           char *name);
+
+struct vm_gk20a *nvgpu_vm_init(struct gk20a *g,
+                               u32 big_page_size,
+                               u64 low_hole,
+                               u64 kernel_reserved,
+                               u64 aperture_size,
+                               bool big_pages,
+                               bool userspace_managed,
+                               char *name);
+
+/*
+ * These are private to the VM code but are unfortunately used by the vgpu code.
+ * It appears to be used for an optimization in reducing the number of server
+ * requests to the vgpu server. Basically the vgpu implementation of
+ * map_global_ctx_buffers() sends a bunch of VA ranges over to the RM server.
+ * Ideally the RM server can just batch mappings but until such a time this
+ * will be used by the vgpu code.
+ */
+u64 __nvgpu_vm_alloc_va(struct vm_gk20a *vm, u64 size,
+                                u32 pgsz_idx);
+int __nvgpu_vm_free_va(struct vm_gk20a *vm, u64 addr,
+                                u32 pgsz_idx);
+
+#endif /* NVGPU_VM_H */