/*
* Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <nvgpu/vm.h>
#include <nvgpu/vm_area.h>
#include "gk20a/gk20a.h"
#include "gk20a/mm_gk20a.h"
struct nvgpu_vm_area *nvgpu_vm_area_find(struct vm_gk20a *vm, u64 addr)
{
struct nvgpu_vm_area *vm_area;
nvgpu_list_for_each_entry(vm_area, &vm->vm_area_list,
nvgpu_vm_area, vm_area_list) {
if (addr >= vm_area->addr &&
addr < (u64)vm_area->addr + (u64)vm_area->size)
return vm_area;
}
return NULL;
}
int nvgpu_vm_area_validate_buffer(struct vm_gk20a *vm,
u64 map_addr, u64 map_size, int pgsz_idx,
struct nvgpu_vm_area **pvm_area)
{
struct gk20a *g = vm->mm->g;
struct nvgpu_vm_area *vm_area;
struct nvgpu_mapped_buf *buffer;
u64 map_end = map_addr + map_size;
/* can wrap around with insane map_size; zero is disallowed too */
if (map_end <= map_addr) {
nvgpu_warn(g, "fixed offset mapping with invalid map_size");
return -EINVAL;
}
if (map_addr & (vm->gmmu_page_sizes[pgsz_idx] - 1)) {
nvgpu_err(g, "map offset must be buffer page size aligned 0x%llx",
map_addr);
return -EINVAL;
}
/* Find the space reservation, but it's ok to have none for
* userspace-managed address spaces */
vm_area = nvgpu_vm_area_find(vm, map_addr);
if (!vm_area && !vm->userspace_managed) {
nvgpu_warn(g, "fixed offset mapping without space allocation");
return -EINVAL;
}
/* Mapped area should fit inside va, if there's one */
if (vm_area && map_end > vm_area->addr + vm_area->size) {
nvgpu_warn(g, "fixed offset mapping size overflows va node");
return -EINVAL;
}
/* check that this mapping does not collide with existing
* mappings by checking the buffer with the highest GPU VA
* that is less than our buffer end */
buffer = __nvgpu_vm_find_mapped_buf_less_than(
vm, map_addr + map_size);
if (buffer && buffer->addr + buffer->size > map_addr) {
nvgpu_warn(g, "overlapping buffer map requested");
return -EINVAL;
}
*pvm_area = vm_area;
return 0;
}
int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u32 pages, u32 page_size,
u64 *addr, u32 flags)
{
struct gk20a *g = vm->mm->g;
struct nvgpu_allocator *vma;
struct nvgpu_vm_area *vm_area;
u64 vaddr_start = 0;
int pgsz_idx = gmmu_page_size_small;
nvgpu_log(g, gpu_dbg_map,
"ADD vm_area: pgsz=%#-8x pages=%-9u addr=%#-14llx flags=0x%x",
page_size, pages, *addr, flags);
for (; pgsz_idx < gmmu_nr_page_sizes; pgsz_idx++) {
if (vm->gmmu_page_sizes[pgsz_idx] == page_size)
break;
}
if (pgsz_idx > gmmu_page_size_big)
return -EINVAL;
if (!vm->big_pages && pgsz_idx == gmmu_page_size_big)
return -EINVAL;
vm_area = nvgpu_kzalloc(g, sizeof(*vm_area));
if (!vm_area)
goto clean_up_err;
vma = vm->vma[pgsz_idx];
if (flags & NVGPU_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET)
vaddr_start = nvgpu_alloc_fixed(vma, *addr,
(u64)pages *
(u64)page_size,
page_size);
else
vaddr_start = nvgpu_alloc(vma,
(u64)pages *
(u64)page_size);
if (!vaddr_start)
goto clean_up_err;
vm_area->flags = flags;
vm_area->addr = vaddr_start;
vm_area->size = (u64)page_size * (u64)pages;
vm_area->pgsz_idx = pgsz_idx;
nvgpu_init_list_node(&vm_area->buffer_list_head);
nvgpu_init_list_node(&vm_area->vm_area_list);
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
if (flags & NVGPU_AS_ALLOC_SPACE_FLAGS_SPARSE) {
u64 map_addr = g->ops.mm.gmmu_map(vm, vaddr_start,
NULL,
0,
vm_area->size,
pgsz_idx,
0,
0,
flags,
gk20a_mem_flag_none,
false,
true,
false,
NULL,
APERTURE_INVALID);
if (!map_addr) {
nvgpu_mutex_release(&vm->update_gmmu_lock);
goto clean_up_err;
}
vm_area->sparse = true;
}
nvgpu_list_add_tail(&vm_area->vm_area_list, &vm->vm_area_list);
nvgpu_mutex_release(&vm->update_gmmu_lock);
*addr = vaddr_start;
return 0;
clean_up_err:
if (vaddr_start)
nvgpu_free(vma, vaddr_start);
if (vm_area)
nvgpu_kfree(g, vm_area);
return -ENOMEM;
}
int nvgpu_vm_area_free(struct vm_gk20a *vm, u64 addr)
{
struct gk20a *g = gk20a_from_vm(vm);
struct nvgpu_mapped_buf *buffer, *n;
struct nvgpu_vm_area *vm_area;
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
vm_area = nvgpu_vm_area_find(vm, addr);
if (!vm_area) {
nvgpu_mutex_release(&vm->update_gmmu_lock);
return 0;
}
nvgpu_list_del(&vm_area->vm_area_list);
nvgpu_mutex_release(&vm->update_gmmu_lock);
nvgpu_log(g, gpu_dbg_map,
"DEL vm_area: pgsz=%#-8x pages=%-9llu "
"addr=%#-14llx flags=0x%x",
vm->gmmu_page_sizes[vm_area->pgsz_idx],
vm_area->size / vm->gmmu_page_sizes[vm_area->pgsz_idx],
vm_area->addr,
vm_area->flags);
/* Decrement the ref count on all buffers in this vm_area. This
* allows userspace to let the kernel free mappings that are
* only used by this vm_area. */
nvgpu_list_for_each_entry_safe(buffer, n,
&vm_area->buffer_list_head,
nvgpu_mapped_buf, buffer_list) {
nvgpu_list_del(&buffer->buffer_list);
kref_put(&buffer->ref, gk20a_vm_unmap_locked_kref);
}
/* if this was a sparse mapping, free the va */
if (vm_area->sparse)
g->ops.mm.gmmu_unmap(vm,
vm_area->addr,
vm_area->size,
vm_area->pgsz_idx,
true,
gk20a_mem_flag_none,
true,
NULL);
nvgpu_free(vm->vma[vm_area->pgsz_idx], vm_area->addr);
nvgpu_kfree(g, vm_area);
return 0;
}