/*
* Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include <nvgpu/bug.h>
#include <nvgpu/log.h>
#include <nvgpu/dma.h>
#include <nvgpu/vm.h>
#include <nvgpu/vm_area.h>
#include <nvgpu/gmmu.h>
#include <nvgpu/lock.h>
#include <nvgpu/list.h>
#include <nvgpu/rbtree.h>
#include <nvgpu/semaphore.h>
#include <nvgpu/enabled.h>
#include <nvgpu/sizes.h>
#include <nvgpu/vgpu/vm.h>
#include "gk20a/gk20a.h"
#include "gk20a/mm_gk20a.h"
struct nvgpu_ctag_buffer_info {
u64 size;
enum gmmu_pgsz_gk20a pgsz_idx;
u32 flags;
s16 compr_kind;
s16 incompr_kind;
u32 ctag_lines;
};
static int nvgpu_vm_compute_compression(struct vm_gk20a *vm,
struct nvgpu_ctag_buffer_info *binfo);
static void __nvgpu_vm_unmap(struct nvgpu_mapped_buf *mapped_buffer,
struct vm_gk20a_mapping_batch *batch);
int vm_aspace_id(struct vm_gk20a *vm)
{
return vm->as_share ? vm->as_share->id : -1;
}
/*
* Determine how many bits of the address space each last level PDE covers. For
* example, for gp10b, with a last level address bit PDE range of 28 to 21 the
* amount of memory each last level PDE addresses is 21 bits - i.e 2MB.
*/
int nvgpu_vm_pde_coverage_bit_count(struct vm_gk20a *vm)
{
int final_pde_level = 0;
/*
* Find the second to last level of the page table programming
* heirarchy: the last level is PTEs so we really want the level
* before that which is the last level of PDEs.
*/
while (vm->mmu_levels[final_pde_level + 2].update_entry)
final_pde_level++;
return vm->mmu_levels[final_pde_level].lo_bit[0];
}
static void __nvgpu_vm_free_entries(struct vm_gk20a *vm,
struct nvgpu_gmmu_pd *pd,
int level)
{
int i;
if (pd->mem) {
__nvgpu_pd_free(vm, pd);
pd->mem = NULL;
}
if (pd->entries) {
for (i = 0; i < pd->num_entries; i++)
__nvgpu_vm_free_entries(vm, &pd->entries[i],
level + 1);
nvgpu_vfree(vm->mm->g, pd->entries);
pd->entries = NULL;
}
}
static void nvgpu_vm_free_entries(struct vm_gk20a *vm,
struct nvgpu_gmmu_pd *pdb)
{
struct gk20a *g = vm->mm->g;
int i;
__nvgpu_pd_cache_free_direct(g, pdb);
if (!pdb->entries)
return;
for (i = 0; i < pdb->num_entries; i++)
__nvgpu_vm_free_entries(vm, &pdb->entries[i], 1);
nvgpu_vfree(g, pdb->entries);
pdb->entries = NULL;
}
u64 __nvgpu_vm_alloc_va(struct vm_gk20a *vm, u64 size,
enum gmmu_pgsz_gk20a pgsz_idx)
{
struct gk20a *g = vm->mm->g;
struct nvgpu_allocator *vma = NULL;
u64 addr;
u64 page_size = vm->gmmu_page_sizes[pgsz_idx];
vma = vm->vma[pgsz_idx];
if (vm->guest_managed) {
nvgpu_err(g, "Illegal GPU allocation on behalf of guest OS");
return 0;
}
if (pgsz_idx >= gmmu_nr_page_sizes) {
nvgpu_err(g, "(%s) invalid page size requested", vma->name);
return 0;
}
if ((pgsz_idx == gmmu_page_size_big) && !vm->big_pages) {
nvgpu_err(g, "(%s) unsupportd page size requested", vma->name);
return 0;
}
/* Be certain we round up to page_size if needed */
size = (size + ((u64)page_size - 1)) & ~((u64)page_size - 1);
addr = nvgpu_alloc(vma, size);
if (!addr) {
nvgpu_err(g, "(%s) oom: sz=0x%llx", vma->name, size);
return 0;
}
return addr;
}
int __nvgpu_vm_free_va(struct vm_gk20a *vm, u64 addr,
enum gmmu_pgsz_gk20a pgsz_idx)
{
struct nvgpu_allocator *vma = vm->vma[pgsz_idx];
nvgpu_free(vma, addr);
return 0;
}
void nvgpu_vm_mapping_batch_start(struct vm_gk20a_mapping_batch *mapping_batch)
{
memset(mapping_batch, 0, sizeof(*mapping_batch));
mapping_batch->gpu_l2_flushed = false;
mapping_batch->need_tlb_invalidate = false;
}
void nvgpu_vm_mapping_batch_finish_locked(
struct vm_gk20a *vm, struct vm_gk20a_mapping_batch *mapping_batch)
{
/* hanging kref_put batch pointer? */
WARN_ON(vm->kref_put_batch == mapping_batch);
if (mapping_batch->need_tlb_invalidate) {
struct gk20a *g = gk20a_from_vm(vm);
g->ops.fb.tlb_invalidate(g, vm->pdb.mem);
}
}
void nvgpu_vm_mapping_batch_finish(struct vm_gk20a *vm,
struct vm_gk20a_mapping_batch *mapping_batch)
{
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
nvgpu_vm_mapping_batch_finish_locked(vm, mapping_batch);
nvgpu_mutex_release(&vm->update_gmmu_lock);
}
/*
* Determine if the passed address space can support big pages or not.
*/
int nvgpu_big_pages_possible(struct vm_gk20a *vm, u64 base, u64 size)
{
u64 mask = ((u64)vm->big_page_size << 10) - 1;
if (base & mask || size & mask)
return 0;
return 1;
}
/*
* Initialize a semaphore pool. Just return successfully if we do not need
* semaphores (i.e when sync-pts are active).
*/
static int nvgpu_init_sema_pool(struct vm_gk20a *vm)
{
struct nvgpu_semaphore_sea *sema_sea;
struct mm_gk20a *mm = vm->mm;
struct gk20a *g = mm->g;
int err;
/*
* Don't waste the memory on semaphores if we don't need them.
*/
if (nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS))
return 0;
if (vm->sema_pool)
return 0;
sema_sea = nvgpu_semaphore_sea_create(g);
if (!sema_sea)
return -ENOMEM;
vm->sema_pool = nvgpu_semaphore_pool_alloc(sema_sea);
if (!vm->sema_pool)
return -ENOMEM;
/*
* Allocate a chunk of GPU VA space for mapping the semaphores. We will
* do a fixed alloc in the kernel VM so that all channels have the same
* RO address range for the semaphores.
*
* !!! TODO: cleanup.
*/
sema_sea->gpu_va = nvgpu_alloc_fixed(&vm->kernel,
vm->va_limit -
mm->channel.kernel_size,
512 * PAGE_SIZE,
SZ_4K);
if (!sema_sea->gpu_va) {
nvgpu_free(&vm->kernel, sema_sea->gpu_va);
nvgpu_vm_put(vm);
return -ENOMEM;
}
err = nvgpu_semaphore_pool_map(vm->sema_pool, vm);
if (err) {
nvgpu_semaphore_pool_unmap(vm->sema_pool, vm);
nvgpu_free(vm->vma[gmmu_page_size_small],
vm->sema_pool->gpu_va);
return err;
}
return 0;
}
/*
* Initialize a preallocated vm
*/
int __nvgpu_vm_init(struct mm_gk20a *mm,
struct vm_gk20a *vm,
u32 big_page_size,
u64 low_hole,
u64 kernel_reserved,
u64 aperture_size,
bool big_pages,
bool userspace_managed,
char *name)
{
int err;
char alloc_name[32];
u64 kernel_vma_flags;
u64 user_vma_start, user_vma_limit;
u64 user_lp_vma_start, user_lp_vma_limit;
u64 kernel_vma_start, kernel_vma_limit;
struct gk20a *g = gk20a_from_mm(mm);
if (WARN_ON(kernel_reserved + low_hole > aperture_size))
return -ENOMEM;
if (WARN_ON(vm->guest_managed && kernel_reserved != 0))
return -EINVAL;
nvgpu_log_info(g, "Init space for %s: valimit=0x%llx, "
"LP size=0x%x lowhole=0x%llx",
name, aperture_size,
(unsigned int)big_page_size, low_hole);
vm->mm = mm;
vm->gmmu_page_sizes[gmmu_page_size_small] = SZ_4K;
vm->gmmu_page_sizes[gmmu_page_size_big] = big_page_size;
vm->gmmu_page_sizes[gmmu_page_size_kernel] = SZ_4K;
/* Set up vma pointers. */
vm->vma[gmmu_page_size_small] = &vm->user;
vm->vma[gmmu_page_size_big] = &vm->user;
vm->vma[gmmu_page_size_kernel] = &vm->kernel;
if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES))
vm->vma[gmmu_page_size_big] = &vm->user_lp;
vm->va_start = low_hole;
vm->va_limit = aperture_size;
vm->big_page_size = vm->gmmu_page_sizes[gmmu_page_size_big];
vm->userspace_managed = userspace_managed;
vm->mmu_levels = g->ops.mm.get_mmu_levels(g, vm->big_page_size);
#ifdef CONFIG_TEGRA_GR_VIRTUALIZATION
if (g->is_virtual && userspace_managed) {
nvgpu_err(g, "vGPU: no userspace managed addr space support");
return -ENOSYS;
}
if (g->is_virtual && vgpu_vm_init(g, vm)) {
nvgpu_err(g, "Failed to init vGPU VM!");
return -ENOMEM;
}
#endif
/* Initialize the page table data structures. */
strncpy(vm->name, name, min(strlen(name), sizeof(vm->name)));
err = nvgpu_gmmu_init_page_table(vm);
if (err)
goto clean_up_vgpu_vm;
/* Setup vma limits. */
if (kernel_reserved + low_hole < aperture_size) {
/*
* If big_pages are disabled for this VM then it only makes
* sense to make one VM, same as if the unified address flag
* is set.
*/
if (!big_pages ||
nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
user_vma_start = low_hole;
user_vma_limit = vm->va_limit - kernel_reserved;
user_lp_vma_start = user_vma_limit;
user_lp_vma_limit = user_vma_limit;
} else {
user_vma_start = low_hole;
user_vma_limit = __nv_gmmu_va_small_page_limit();
user_lp_vma_start = __nv_gmmu_va_small_page_limit();
user_lp_vma_limit = vm->va_limit - kernel_reserved;
}
} else {
user_vma_start = 0;
user_vma_limit = 0;
user_lp_vma_start = 0;
user_lp_vma_limit = 0;
}
kernel_vma_start = vm->va_limit - kernel_reserved;
kernel_vma_limit = vm->va_limit;
nvgpu_log_info(g, "user_vma [0x%llx,0x%llx)",
user_vma_start, user_vma_limit);
nvgpu_log_info(g, "user_lp_vma [0x%llx,0x%llx)",
user_lp_vma_start, user_lp_vma_limit);
nvgpu_log_info(g, "kernel_vma [0x%llx,0x%llx)",
kernel_vma_start, kernel_vma_limit);
if (WARN_ON(user_vma_start > user_vma_limit) ||
WARN_ON(user_lp_vma_start > user_lp_vma_limit) ||
WARN_ON(!vm->guest_managed && kernel_vma_start >= kernel_vma_limit)) {
err = -EINVAL;
goto clean_up_page_tables;
}
kernel_vma_flags = (kernel_reserved + low_hole) == aperture_size ?
0 : GPU_ALLOC_GVA_SPACE;
/*
* A "user" area only makes sense for the GVA spaces. For VMs where
* there is no "user" area user_vma_start will be equal to
* user_vma_limit (i.e a 0 sized space). In such a situation the kernel
* area must be non-zero in length.
*/
if (user_vma_start >= user_vma_limit &&
kernel_vma_start >= kernel_vma_limit) {
err = -EINVAL;
goto clean_up_page_tables;
}
/*
* Determine if big pages are possible in this VM. If a split address
* space is used then check the user_lp vma instead of the user vma.
*/
if (nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES))
vm->big_pages = big_pages &&
nvgpu_big_pages_possible(vm, user_vma_start,
user_vma_limit - user_vma_start);
else
vm->big_pages = big_pages &&
nvgpu_big_pages_possible(vm, user_lp_vma_start,
user_lp_vma_limit - user_lp_vma_start);
/*
* User VMA.
*/
if (user_vma_start < user_vma_limit) {
snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s", name);
err = __nvgpu_buddy_allocator_init(g, &vm->user,
vm, alloc_name,
user_vma_start,
user_vma_limit -
user_vma_start,
SZ_4K,
GPU_BALLOC_MAX_ORDER,
GPU_ALLOC_GVA_SPACE);
if (err)
goto clean_up_page_tables;
} else {
/*
* Make these allocator pointers point to the kernel allocator
* since we still use the legacy notion of page size to choose
* the allocator.
*/
vm->vma[0] = &vm->kernel;
vm->vma[1] = &vm->kernel;
}
/*
* User VMA for large pages when a split address range is used.
*/
if (user_lp_vma_start < user_lp_vma_limit) {
snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s_lp", name);
err = __nvgpu_buddy_allocator_init(g, &vm->user_lp,
vm, alloc_name,
user_lp_vma_start,
user_lp_vma_limit -
user_lp_vma_start,
vm->big_page_size,
GPU_BALLOC_MAX_ORDER,
GPU_ALLOC_GVA_SPACE);
if (err)
goto clean_up_allocators;
}
/*
* Kernel VMA. Must always exist for an address space.
*/
snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-sys", name);
err = __nvgpu_buddy_allocator_init(g, &vm->kernel,
vm, alloc_name,
kernel_vma_start,
kernel_vma_limit - kernel_vma_start,
SZ_4K,
GPU_BALLOC_MAX_ORDER,
kernel_vma_flags);
if (err)
goto clean_up_allocators;
vm->mapped_buffers = NULL;
nvgpu_mutex_init(&vm->update_gmmu_lock);
nvgpu_ref_init(&vm->ref);
nvgpu_init_list_node(&vm->vm_area_list);
/*
* This is only necessary for channel address spaces. The best way to
* distinguish channel address spaces from other address spaces is by
* size - if the address space is 4GB or less, it's not a channel.
*/
if (vm->va_limit > 4ULL * SZ_1G) {
err = nvgpu_init_sema_pool(vm);
if (err)
goto clean_up_allocators;
}
return 0;
clean_up_allocators:
if (nvgpu_alloc_initialized(&vm->kernel))
nvgpu_alloc_destroy(&vm->kernel);
if (nvgpu_alloc_initialized(&vm->user))
nvgpu_alloc_destroy(&vm->user);
if (nvgpu_alloc_initialized(&vm->user_lp))
nvgpu_alloc_destroy(&vm->user_lp);
clean_up_page_tables:
/* Cleans up nvgpu_gmmu_init_page_table() */
__nvgpu_pd_cache_free_direct(g, &vm->pdb);
clean_up_vgpu_vm:
#ifdef CONFIG_TEGRA_GR_VIRTUALIZATION
if (g->is_virtual)
vgpu_vm_remove(vm);
#endif
return err;
}
/**
* nvgpu_init_vm() - Initialize an address space.
*
* @mm - Parent MM.
* @vm - The VM to init.
* @big_page_size - Size of big pages associated with this VM.
* @low_hole - The size of the low hole (unaddressable memory at the bottom of
* the address space).
* @kernel_reserved - Space reserved for kernel only allocations.
* @aperture_size - Total size of the aperture.
* @big_pages - If true then big pages are possible in the VM. Note this does
* not guarantee that big pages will be possible.
* @name - Name of the address space.
*
* This function initializes an address space according to the following map:
*
* +--+ 0x0
* | |
* +--+ @low_hole
* | |
* ~ ~ This is the "user" section.
* | |
* +--+ @aperture_size - @kernel_reserved
* | |
* ~ ~ This is the "kernel" section.
* | |
* +--+ @aperture_size
*
* The user section is therefor what ever is left over after the @low_hole and
* @kernel_reserved memory have been portioned out. The @kernel_reserved is
* always persent at the top of the memory space and the @low_hole is always at
* the bottom.
*
* For certain address spaces a "user" section makes no sense (bar1, etc) so in
* such cases the @kernel_reserved and @low_hole should sum to exactly
* @aperture_size.
*/
struct vm_gk20a *nvgpu_vm_init(struct gk20a *g,
u32 big_page_size,
u64 low_hole,
u64 kernel_reserved,
u64 aperture_size,
bool big_pages,
bool userspace_managed,
char *name)
{
struct vm_gk20a *vm = nvgpu_kzalloc(g, sizeof(*vm));
if (!vm)
return NULL;
if (__nvgpu_vm_init(&g->mm, vm, big_page_size, low_hole,
kernel_reserved, aperture_size, big_pages,
userspace_managed, name)) {
nvgpu_kfree(g, vm);
return NULL;
}
return vm;
}
/*
* Cleanup the VM!
*/
static void __nvgpu_vm_remove(struct vm_gk20a *vm)
{
struct nvgpu_mapped_buf *mapped_buffer;
struct nvgpu_vm_area *vm_area, *vm_area_tmp;
struct nvgpu_rbtree_node *node = NULL;
struct gk20a *g = vm->mm->g;
/*
* Do this outside of the update_gmmu_lock since unmapping the semaphore
* pool involves unmapping a GMMU mapping which means aquiring the
* update_gmmu_lock.
*/
if (!nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS)) {
if (vm->sema_pool) {
nvgpu_semaphore_pool_unmap(vm->sema_pool, vm);
nvgpu_semaphore_pool_put(vm->sema_pool);
}
}
#if defined(CONFIG_TEGRA_GK20A_NVHOST) && defined(CONFIG_TEGRA_19x_GPU)
if (nvgpu_mem_is_valid(&g->syncpt_mem) && vm->syncpt_ro_map_gpu_va)
nvgpu_gmmu_unmap(vm, &g->syncpt_mem,
vm->syncpt_ro_map_gpu_va);
#endif
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
nvgpu_rbtree_enum_start(0, &node, vm->mapped_buffers);
while (node) {
mapped_buffer = mapped_buffer_from_rbtree_node(node);
__nvgpu_vm_unmap(mapped_buffer, NULL);
nvgpu_rbtree_enum_start(0, &node, vm->mapped_buffers);
}
/* destroy remaining reserved memory areas */
nvgpu_list_for_each_entry_safe(vm_area, vm_area_tmp,
&vm->vm_area_list,
nvgpu_vm_area, vm_area_list) {
nvgpu_list_del(&vm_area->vm_area_list);
nvgpu_kfree(vm->mm->g, vm_area);
}
if (nvgpu_alloc_initialized(&vm->kernel))
nvgpu_alloc_destroy(&vm->kernel);
if (nvgpu_alloc_initialized(&vm->user))
nvgpu_alloc_destroy(&vm->user);
if (nvgpu_alloc_initialized(&vm->user_lp))
nvgpu_alloc_destroy(&vm->user_lp);
nvgpu_vm_free_entries(vm, &vm->pdb);
#ifdef CONFIG_TEGRA_GR_VIRTUALIZATION
if (g->is_virtual)
vgpu_vm_remove(vm);
#endif
nvgpu_mutex_release(&vm->update_gmmu_lock);
nvgpu_kfree(g, vm);
}
static void __nvgpu_vm_remove_ref(struct nvgpu_ref *ref)
{
struct vm_gk20a *vm = container_of(ref, struct vm_gk20a, ref);
__nvgpu_vm_remove(vm);
}
void nvgpu_vm_get(struct vm_gk20a *vm)
{
nvgpu_ref_get(&vm->ref);
}
void nvgpu_vm_put(struct vm_gk20a *vm)
{
nvgpu_ref_put(&vm->ref, __nvgpu_vm_remove_ref);
}
int nvgpu_insert_mapped_buf(struct vm_gk20a *vm,
struct nvgpu_mapped_buf *mapped_buffer)
{
mapped_buffer->node.key_start = mapped_buffer->addr;
mapped_buffer->node.key_end = mapped_buffer->addr + mapped_buffer->size;
nvgpu_rbtree_insert(&mapped_buffer->node, &vm->mapped_buffers);
return 0;
}
void nvgpu_remove_mapped_buf(struct vm_gk20a *vm,
struct nvgpu_mapped_buf *mapped_buffer)
{
nvgpu_rbtree_unlink(&mapped_buffer->node, &vm->mapped_buffers);
}
struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf(
struct vm_gk20a *vm, u64 addr)
{
struct nvgpu_rbtree_node *node = NULL;
struct nvgpu_rbtree_node *root = vm->mapped_buffers;
nvgpu_rbtree_search(addr, &node, root);
if (!node)
return NULL;
return mapped_buffer_from_rbtree_node(node);
}
struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_range(
struct vm_gk20a *vm, u64 addr)
{
struct nvgpu_rbtree_node *node = NULL;
struct nvgpu_rbtree_node *root = vm->mapped_buffers;
nvgpu_rbtree_range_search(addr, &node, root);
if (!node)
return NULL;
return mapped_buffer_from_rbtree_node(node);
}
struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_less_than(
struct vm_gk20a *vm, u64 addr)
{
struct nvgpu_rbtree_node *node = NULL;
struct nvgpu_rbtree_node *root = vm->mapped_buffers;
nvgpu_rbtree_less_than_search(addr, &node, root);
if (!node)
return NULL;
return mapped_buffer_from_rbtree_node(node);
}
int nvgpu_vm_get_buffers(struct vm_gk20a *vm,
struct nvgpu_mapped_buf ***mapped_buffers,
int *num_buffers)
{
struct nvgpu_mapped_buf *mapped_buffer;
struct nvgpu_mapped_buf **buffer_list;
struct nvgpu_rbtree_node *node = NULL;
int i = 0;
if (vm->userspace_managed) {
*mapped_buffers = NULL;
*num_buffers = 0;
return 0;
}
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
buffer_list = nvgpu_big_zalloc(vm->mm->g, sizeof(*buffer_list) *
vm->num_user_mapped_buffers);
if (!buffer_list) {
nvgpu_mutex_release(&vm->update_gmmu_lock);
return -ENOMEM;
}
nvgpu_rbtree_enum_start(0, &node, vm->mapped_buffers);
while (node) {
mapped_buffer = mapped_buffer_from_rbtree_node(node);
buffer_list[i] = mapped_buffer;
nvgpu_ref_get(&mapped_buffer->ref);
i++;
nvgpu_rbtree_enum_next(&node, node);
}
BUG_ON(i != vm->num_user_mapped_buffers);
*num_buffers = vm->num_user_mapped_buffers;
*mapped_buffers = buffer_list;
nvgpu_mutex_release(&vm->update_gmmu_lock);
return 0;
}
void nvgpu_vm_put_buffers(struct vm_gk20a *vm,
struct nvgpu_mapped_buf **mapped_buffers,
int num_buffers)
{
int i;
struct vm_gk20a_mapping_batch batch;
if (num_buffers == 0)
return;
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
nvgpu_vm_mapping_batch_start(&batch);
vm->kref_put_batch = &batch;
for (i = 0; i < num_buffers; ++i)
nvgpu_ref_put(&mapped_buffers[i]->ref, __nvgpu_vm_unmap_ref);
vm->kref_put_batch = NULL;
nvgpu_vm_mapping_batch_finish_locked(vm, &batch);
nvgpu_mutex_release(&vm->update_gmmu_lock);
nvgpu_big_free(vm->mm->g, mapped_buffers);
}
struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
struct nvgpu_os_buffer *os_buf,
struct nvgpu_sgt *sgt,
u64 map_addr,
u64 map_size,
u64 phys_offset,
int rw,
u32 flags,
s16 compr_kind,
s16 incompr_kind,
struct vm_gk20a_mapping_batch *batch,
enum nvgpu_aperture aperture)
{
struct gk20a *g = gk20a_from_vm(vm);
struct nvgpu_mapped_buf *mapped_buffer = NULL;
struct nvgpu_ctag_buffer_info binfo = { 0 };
struct nvgpu_vm_area *vm_area = NULL;
int err = 0;
u64 align;
u32 ctag_offset = 0;
bool clear_ctags = false;
bool va_allocated = true;
/*
* The kind used as part of the key for map caching. HW may
* actually be programmed with the fallback kind in case the
* key kind is compressible but we're out of comptags.
*/
s16 map_key_kind;
/*
* The actual GMMU PTE kind
*/
u8 pte_kind;
if (vm->userspace_managed &&
!(flags & NVGPU_VM_MAP_FIXED_OFFSET)) {
nvgpu_err(g,
"non-fixed-offset mapping not available on "
"userspace managed address spaces");
return ERR_PTR(-EINVAL);
}
binfo.flags = flags;
binfo.size = nvgpu_os_buf_get_size(os_buf);
binfo.compr_kind =
(vm->enable_ctag && compr_kind != NVGPU_KIND_INVALID ?
compr_kind : NVGPU_KIND_INVALID);
binfo.incompr_kind = incompr_kind;
if (compr_kind != NVGPU_KIND_INVALID)
map_key_kind = compr_kind;
else
map_key_kind = incompr_kind;
/*
* Check if this buffer is already mapped.
*/
if (!vm->userspace_managed) {
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
mapped_buffer = nvgpu_vm_find_mapping(vm,
os_buf,
map_addr,
flags,
map_key_kind);
nvgpu_mutex_release(&vm->update_gmmu_lock);
if (mapped_buffer) {
nvgpu_ref_get(&mapped_buffer->ref);
return mapped_buffer;
}
}
/*
* Generate a new mapping!
*/
mapped_buffer = nvgpu_kzalloc(g, sizeof(*mapped_buffer));
if (!mapped_buffer) {
nvgpu_warn(g, "oom allocating tracking buffer");
return ERR_PTR(-ENOMEM);
}
align = nvgpu_sgt_alignment(g, sgt);
if (g->mm.disable_bigpage)
binfo.pgsz_idx = gmmu_page_size_small;
else
binfo.pgsz_idx = __get_pte_size(vm, map_addr,
min_t(u64, binfo.size, align));
map_size = map_size ? map_size : binfo.size;
map_size = ALIGN(map_size, SZ_4K);
if ((map_size > binfo.size) ||
(phys_offset > (binfo.size - map_size))) {
err = -EINVAL;
goto clean_up_nolock;
}
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
/*
* Check if we should use a fixed offset for mapping this buffer.
*/
if (flags & NVGPU_VM_MAP_FIXED_OFFSET) {
err = nvgpu_vm_area_validate_buffer(vm,
map_addr,
map_size,
binfo.pgsz_idx,
&vm_area);
if (err)
goto clean_up;
va_allocated = false;
}
err = nvgpu_vm_compute_compression(vm, &binfo);
if (err) {
nvgpu_err(g, "failure setting up compression");
goto clean_up;
}
if ((binfo.compr_kind != NVGPU_KIND_INVALID) &&
(flags & NVGPU_VM_MAP_FIXED_OFFSET)) {
/*
* Fixed-address compressible mapping is
* requested. Make sure we're respecting the alignment
* requirement for virtual addresses and buffer
* offsets.
*
* This check must be done before we may fall back to
* the incompressible kind.
*/
const u64 offset_mask = g->ops.fb.compression_align_mask(g);
if ((map_addr & offset_mask) != (phys_offset & offset_mask)) {
nvgpu_log(g, gpu_dbg_map,
"Misaligned compressible-kind fixed-address "
"mapping");
err = -EINVAL;
goto clean_up;
}
}
if (binfo.compr_kind != NVGPU_KIND_INVALID) {
struct gk20a_comptags comptags = { 0 };
/*
* Get the comptags state, alloc if necessary
*/
err = gk20a_alloc_or_get_comptags(g, os_buf,
&g->gr.comp_tags,
&comptags);
if (err) {
/*
* This is an irrecoverable failure and we need to
* abort. In particular, it is not safe to proceed with
* the incompressible fallback, since we cannot not mark
* our alloc failure anywere. Later we would retry
* allocation and break compressible map aliasing.
*/
nvgpu_err(g, "Error %d setting up comptags", err);
goto clean_up;
}
/*
* Newly allocated comptags needs to be cleared
*/
if (comptags.needs_clear) {
if (g->ops.ltc.cbc_ctrl) {
if (gk20a_comptags_start_clear(os_buf)) {
err = g->ops.ltc.cbc_ctrl(
g, gk20a_cbc_op_clear,
comptags.offset,
(comptags.offset +
comptags.lines - 1));
gk20a_comptags_finish_clear(
os_buf, err == 0);
if (err)
goto clean_up;
}
} else {
/*
* Cleared as part of gmmu map
*/
clear_ctags = true;
}
}
/*
* Store the ctag offset for later use if we got the comptags
*/
if (comptags.lines)
ctag_offset = comptags.offset;
}
/*
* Figure out the kind and ctag offset for the GMMU page tables
*/
if (binfo.compr_kind != NVGPU_KIND_INVALID && ctag_offset) {
/*
* Adjust the ctag_offset as per the buffer map offset
*/
ctag_offset += phys_offset >>
ilog2(g->ops.fb.compression_page_size(g));
pte_kind = binfo.compr_kind;
} else if (binfo.incompr_kind != NVGPU_KIND_INVALID) {
/*
* Incompressible kind, ctag offset will not be programmed
*/
ctag_offset = 0;
pte_kind = binfo.incompr_kind;
} else {
/*
* Caller required compression, but we cannot provide it
*/
nvgpu_err(g, "No comptags and no incompressible fallback kind");
err = -ENOMEM;
goto clean_up;
}
if (clear_ctags)
clear_ctags = gk20a_comptags_start_clear(os_buf);
map_addr = g->ops.mm.gmmu_map(vm,
map_addr,
sgt,
phys_offset,
map_size,
binfo.pgsz_idx,
pte_kind,
ctag_offset,
flags,
rw,
clear_ctags,
false,
false,
batch,
aperture);
if (clear_ctags)
gk20a_comptags_finish_clear(os_buf, map_addr != 0);
if (!map_addr) {
err = -ENOMEM;
goto clean_up;
}
nvgpu_init_list_node(&mapped_buffer->buffer_list);
nvgpu_ref_init(&mapped_buffer->ref);
mapped_buffer->addr = map_addr;
mapped_buffer->size = map_size;
mapped_buffer->pgsz_idx = binfo.pgsz_idx;
mapped_buffer->vm = vm;
mapped_buffer->flags = flags;
mapped_buffer->kind = map_key_kind;
mapped_buffer->va_allocated = va_allocated;
mapped_buffer->vm_area = vm_area;
err = nvgpu_insert_mapped_buf(vm, mapped_buffer);
if (err) {
nvgpu_err(g, "failed to insert into mapped buffer tree");
goto clean_up;
}
vm->num_user_mapped_buffers++;
if (vm_area) {
nvgpu_list_add_tail(&mapped_buffer->buffer_list,
&vm_area->buffer_list_head);
mapped_buffer->vm_area = vm_area;
}
nvgpu_mutex_release(&vm->update_gmmu_lock);
return mapped_buffer;
clean_up:
if (mapped_buffer->addr)
g->ops.mm.gmmu_unmap(vm,
mapped_buffer->addr,
mapped_buffer->size,
mapped_buffer->pgsz_idx,
mapped_buffer->va_allocated,
gk20a_mem_flag_none,
mapped_buffer->vm_area ?
mapped_buffer->vm_area->sparse : false,
NULL);
nvgpu_mutex_release(&vm->update_gmmu_lock);
clean_up_nolock:
nvgpu_kfree(g, mapped_buffer);
return ERR_PTR(err);
}
/*
* Really unmap. This does the real GMMU unmap and removes the mapping from the
* VM map tracking tree (and vm_area list if necessary).
*/
static void __nvgpu_vm_unmap(struct nvgpu_mapped_buf *mapped_buffer,
struct vm_gk20a_mapping_batch *batch)
{
struct vm_gk20a *vm = mapped_buffer->vm;
struct gk20a *g = vm->mm->g;
vm->num_user_mapped_buffers--;
g->ops.mm.gmmu_unmap(vm,
mapped_buffer->addr,
mapped_buffer->size,
mapped_buffer->pgsz_idx,
mapped_buffer->va_allocated,
gk20a_mem_flag_none,
mapped_buffer->vm_area ?
mapped_buffer->vm_area->sparse : false,
batch);
/*
* Remove from mapped buffer tree. Then delete the buffer from the
* linked list of mapped buffers; though note: not all mapped buffers
* are part of a vm_area.
*/
nvgpu_remove_mapped_buf(vm, mapped_buffer);
nvgpu_list_del(&mapped_buffer->buffer_list);
/*
* OS specific freeing. This is after the generic freeing incase the
* generic freeing relies on some component of the OS specific
* nvgpu_mapped_buf in some abstraction or the like.
*/
nvgpu_vm_unmap_system(mapped_buffer);
nvgpu_kfree(g, mapped_buffer);
}
void __nvgpu_vm_unmap_ref(struct nvgpu_ref *ref)
{
struct nvgpu_mapped_buf *mapped_buffer =
container_of(ref, struct nvgpu_mapped_buf, ref);
__nvgpu_vm_unmap(mapped_buffer, mapped_buffer->vm->kref_put_batch);
}
/*
* For fixed-offset buffers we must sync the buffer. That means we wait for the
* buffer to hit a ref-count of 1 before proceeding.
*
* Note: this requires the update_gmmu_lock to be held since we release it and
* re-aquire it in this function.
*/
static int nvgpu_vm_unmap_sync_buffer(struct vm_gk20a *vm,
struct nvgpu_mapped_buf *mapped_buffer)
{
struct nvgpu_timeout timeout;
int ret = 0;
nvgpu_mutex_release(&vm->update_gmmu_lock);
/*
* 500ms second timer.
*/
nvgpu_timeout_init(vm->mm->g, &timeout, 50, NVGPU_TIMER_CPU_TIMER);
do {
if (nvgpu_atomic_read(&mapped_buffer->ref.refcount) == 1)
break;
nvgpu_msleep(10);
} while (!nvgpu_timeout_expired_msg(&timeout,
"sync-unmap failed on 0x%llx"));
if (nvgpu_timeout_expired(&timeout))
ret = -ETIMEDOUT;
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
return ret;
}
void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset,
struct vm_gk20a_mapping_batch *batch)
{
struct nvgpu_mapped_buf *mapped_buffer;
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, offset);
if (!mapped_buffer)
goto done;
if (mapped_buffer->flags & NVGPU_VM_MAP_FIXED_OFFSET) {
if (nvgpu_vm_unmap_sync_buffer(vm, mapped_buffer))
/*
* Looks like we have failed... Better not continue in
* case the buffer is in use.
*/
goto done;
}
/*
* Make sure we have access to the batch if we end up calling through to
* the unmap_ref function.
*/
vm->kref_put_batch = batch;
nvgpu_ref_put(&mapped_buffer->ref, __nvgpu_vm_unmap_ref);
vm->kref_put_batch = NULL;
done:
nvgpu_mutex_release(&vm->update_gmmu_lock);
return;
}
static int nvgpu_vm_compute_compression(struct vm_gk20a *vm,
struct nvgpu_ctag_buffer_info *binfo)
{
bool kind_compressible = (binfo->compr_kind != NVGPU_KIND_INVALID);
struct gk20a *g = gk20a_from_vm(vm);
if (kind_compressible &&
vm->gmmu_page_sizes[binfo->pgsz_idx] <
g->ops.fb.compressible_page_size(g)) {
/*
* Let's double check that there is a fallback kind
*/
if (binfo->incompr_kind == NVGPU_KIND_INVALID) {
nvgpu_err(g,
"Unsupported page size for compressible "
"kind, but no fallback kind");
return -EINVAL;
} else {
nvgpu_log(g, gpu_dbg_map,
"Unsupported page size for compressible "
"kind, demoting to incompressible");
binfo->compr_kind = NVGPU_KIND_INVALID;
kind_compressible = false;
}
}
return 0;
}