From b44c7fdb114a63ab98fffc0f246776b56399ff64 Mon Sep 17 00:00:00 2001 From: Alex Waterman Date: Tue, 14 Aug 2018 11:30:48 -0700 Subject: gpu: nvgpu: Move common DMA code to common/mm This migrates the common DMA code (os agnostic) to the common directory. This new unit will be the common DMA allocator that lets users allocate SYSMEM, VIDMEM, or either. Other units will be responsible for actually handling the mechanics of allocating VIDMEM or SYSMEM. Also update the names of the DMA related files so that tmake doesn't complain about duplicate C file names. To do this call the common DMA file dma.c and prepend the OS to the other DMA files. So now we have: common/mm/dma.c os/posix/posix-dma.c os/linux/linux-dma.c JIRA NVGPU-990 Change-Id: I22d2d41803ad89be7d9c28f87864ce4fedf10836 Signed-off-by: Alex Waterman Reviewed-on: https://git-master.nvidia.com/r/1799807 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/Makefile | 3 +- drivers/gpu/nvgpu/Makefile.sources | 3 +- drivers/gpu/nvgpu/common/mm/dma.c | 197 +++++++++ drivers/gpu/nvgpu/include/nvgpu/dma.h | 6 + drivers/gpu/nvgpu/os/linux/dma.c | 702 --------------------------------- drivers/gpu/nvgpu/os/linux/linux-dma.c | 533 +++++++++++++++++++++++++ drivers/gpu/nvgpu/os/posix/dma.c | 228 ----------- drivers/gpu/nvgpu/os/posix/posix-dma.c | 88 +++++ 8 files changed, 828 insertions(+), 932 deletions(-) create mode 100644 drivers/gpu/nvgpu/common/mm/dma.c delete mode 100644 drivers/gpu/nvgpu/os/linux/dma.c create mode 100644 drivers/gpu/nvgpu/os/linux/linux-dma.c delete mode 100644 drivers/gpu/nvgpu/os/posix/dma.c create mode 100644 drivers/gpu/nvgpu/os/posix/posix-dma.c (limited to 'drivers/gpu') diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 43a36221..a3372cef 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -65,7 +65,7 @@ nvgpu-y += \ os/linux/log.o \ os/linux/cond.o \ os/linux/nvgpu_mem.o \ - os/linux/dma.o \ + os/linux/linux-dma.o \ os/linux/driver_common.o \ os/linux/firmware.o \ os/linux/thread.o \ @@ -180,6 +180,7 @@ nvgpu-y += \ common/mm/nvgpu_mem.o \ common/mm/comptags.o \ common/mm/mm.o \ + common/mm/dma.o \ common/enabled.o \ common/nvlink.o \ common/pramin.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index f09ebf78..7dc0251b 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -27,7 +27,7 @@ srcs := os/posix/nvgpu.c \ os/posix/kmem.c \ os/posix/io.c \ os/posix/posix-nvgpu_mem.c \ - os/posix/dma.c \ + os/posix/posix-dma.c \ os/posix/posix-vm.c \ os/posix/timers.c \ os/posix/posix-comptags.c \ @@ -56,6 +56,7 @@ srcs := os/posix/nvgpu.c \ common/mm/nvgpu_mem.c \ common/mm/comptags.c \ common/mm/mm.c \ + common/mm/dma.c \ common/bus/bus_gk20a.c \ common/bus/bus_gm20b.c \ common/bus/bus_gp10b.c \ diff --git a/drivers/gpu/nvgpu/common/mm/dma.c b/drivers/gpu/nvgpu/common/mm/dma.c new file mode 100644 index 00000000..f7331f8e --- /dev/null +++ b/drivers/gpu/nvgpu/common/mm/dma.c @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +int nvgpu_dma_alloc(struct gk20a *g, size_t size, struct nvgpu_mem *mem) +{ + return nvgpu_dma_alloc_flags(g, 0, size, mem); +} + +int nvgpu_dma_alloc_flags(struct gk20a *g, unsigned long flags, size_t size, + struct nvgpu_mem *mem) +{ + if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) { + /* + * Force the no-kernel-mapping flag on because we don't support + * the lack of it for vidmem - the user should not care when + * using nvgpu_gmmu_alloc_map and it's vidmem, or if there's a + * difference, the user should use the flag explicitly anyway. + * + * Incoming flags are ignored here, since bits other than the + * no-kernel-mapping flag are ignored by the vidmem mapping + * functions anyway. + */ + int err = nvgpu_dma_alloc_flags_vid(g, + NVGPU_DMA_NO_KERNEL_MAPPING, + size, mem); + + if (!err) + return 0; + /* + * Fall back to sysmem (which may then also fail) in case + * vidmem is exhausted. + */ + } + + return nvgpu_dma_alloc_flags_sys(g, flags, size, mem); +} + +int nvgpu_dma_alloc_sys(struct gk20a *g, size_t size, struct nvgpu_mem *mem) +{ + return nvgpu_dma_alloc_flags_sys(g, 0, size, mem); +} + +int nvgpu_dma_alloc_vid(struct gk20a *g, size_t size, struct nvgpu_mem *mem) +{ + return nvgpu_dma_alloc_flags_vid(g, + NVGPU_DMA_NO_KERNEL_MAPPING, size, mem); +} + +int nvgpu_dma_alloc_flags_vid(struct gk20a *g, unsigned long flags, + size_t size, struct nvgpu_mem *mem) +{ + return nvgpu_dma_alloc_flags_vid_at(g, flags, size, mem, 0); +} + +int nvgpu_dma_alloc_vid_at(struct gk20a *g, + size_t size, struct nvgpu_mem *mem, u64 at) +{ + return nvgpu_dma_alloc_flags_vid_at(g, + NVGPU_DMA_NO_KERNEL_MAPPING, size, mem, at); +} + +int nvgpu_dma_alloc_map(struct vm_gk20a *vm, size_t size, + struct nvgpu_mem *mem) +{ + return nvgpu_dma_alloc_map_flags(vm, 0, size, mem); +} + +int nvgpu_dma_alloc_map_flags(struct vm_gk20a *vm, unsigned long flags, + size_t size, struct nvgpu_mem *mem) +{ + if (!nvgpu_is_enabled(gk20a_from_vm(vm), NVGPU_MM_UNIFIED_MEMORY)) { + /* + * Force the no-kernel-mapping flag on because we don't support + * the lack of it for vidmem - the user should not care when + * using nvgpu_dma_alloc_map and it's vidmem, or if there's a + * difference, the user should use the flag explicitly anyway. + */ + int err = nvgpu_dma_alloc_map_flags_vid(vm, + flags | NVGPU_DMA_NO_KERNEL_MAPPING, + size, mem); + + if (!err) + return 0; + /* + * Fall back to sysmem (which may then also fail) in case + * vidmem is exhausted. + */ + } + + return nvgpu_dma_alloc_map_flags_sys(vm, flags, size, mem); +} + +int nvgpu_dma_alloc_map_sys(struct vm_gk20a *vm, size_t size, + struct nvgpu_mem *mem) +{ + return nvgpu_dma_alloc_map_flags_sys(vm, 0, size, mem); +} + +int nvgpu_dma_alloc_map_flags_sys(struct vm_gk20a *vm, unsigned long flags, + size_t size, struct nvgpu_mem *mem) +{ + int err = nvgpu_dma_alloc_flags_sys(vm->mm->g, flags, size, mem); + + if (err) + return err; + + mem->gpu_va = nvgpu_gmmu_map(vm, mem, size, 0, + gk20a_mem_flag_none, false, + mem->aperture); + if (!mem->gpu_va) { + err = -ENOMEM; + goto fail_free; + } + + return 0; + +fail_free: + nvgpu_dma_free(vm->mm->g, mem); + return err; +} + +int nvgpu_dma_alloc_map_vid(struct vm_gk20a *vm, size_t size, + struct nvgpu_mem *mem) +{ + return nvgpu_dma_alloc_map_flags_vid(vm, + NVGPU_DMA_NO_KERNEL_MAPPING, size, mem); +} + +int nvgpu_dma_alloc_map_flags_vid(struct vm_gk20a *vm, unsigned long flags, + size_t size, struct nvgpu_mem *mem) +{ + int err = nvgpu_dma_alloc_flags_vid(vm->mm->g, flags, size, mem); + + if (err) + return err; + + mem->gpu_va = nvgpu_gmmu_map(vm, mem, size, 0, + gk20a_mem_flag_none, false, + mem->aperture); + if (!mem->gpu_va) { + err = -ENOMEM; + goto fail_free; + } + + return 0; + +fail_free: + nvgpu_dma_free(vm->mm->g, mem); + return err; +} + +void nvgpu_dma_free(struct gk20a *g, struct nvgpu_mem *mem) +{ + switch (mem->aperture) { + case APERTURE_SYSMEM: + return nvgpu_dma_free_sys(g, mem); + case APERTURE_VIDMEM: + return nvgpu_dma_free_vid(g, mem); + default: + break; /* like free() on "null" memory */ + } +} + +void nvgpu_dma_unmap_free(struct vm_gk20a *vm, struct nvgpu_mem *mem) +{ + if (mem->gpu_va) + nvgpu_gmmu_unmap(vm, mem, mem->gpu_va); + mem->gpu_va = 0; + + nvgpu_dma_free(vm->mm->g, mem); +} diff --git a/drivers/gpu/nvgpu/include/nvgpu/dma.h b/drivers/gpu/nvgpu/include/nvgpu/dma.h index d83346a3..f6f264cf 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/dma.h +++ b/drivers/gpu/nvgpu/include/nvgpu/dma.h @@ -352,4 +352,10 @@ int nvgpu_dma_alloc_map_flags_vid(struct vm_gk20a *vm, unsigned long flags, */ void nvgpu_dma_unmap_free(struct vm_gk20a *vm, struct nvgpu_mem *mem); +/* + * Don't use these directly. Instead use nvgpu_dma_free(). + */ +void nvgpu_dma_free_sys(struct gk20a *g, struct nvgpu_mem *mem); +void nvgpu_dma_free_vid(struct gk20a *g, struct nvgpu_mem *mem); + #endif diff --git a/drivers/gpu/nvgpu/os/linux/dma.c b/drivers/gpu/nvgpu/os/linux/dma.c deleted file mode 100644 index 77669493..00000000 --- a/drivers/gpu/nvgpu/os/linux/dma.c +++ /dev/null @@ -1,702 +0,0 @@ -/* - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "gk20a/gk20a.h" - -#include "platform_gk20a.h" -#include "os_linux.h" -#include "dmabuf_vidmem.h" - -#ifdef __DMA_ATTRS_LONGS -#define NVGPU_DEFINE_DMA_ATTRS(x) \ - struct dma_attrs x = { \ - .flags = { [0 ... __DMA_ATTRS_LONGS-1] = 0 }, \ - } -#define NVGPU_DMA_ATTR(attrs) &attrs -#else -#define NVGPU_DEFINE_DMA_ATTRS(attrs) unsigned long attrs = 0 -#define NVGPU_DMA_ATTR(attrs) attrs -#endif - -/* - * Enough to hold all the possible flags in string form. When a new flag is - * added it must be added here as well!! - */ -#define NVGPU_DMA_STR_SIZE \ - sizeof("NO_KERNEL_MAPPING FORCE_CONTIGUOUS") - -/* - * The returned string is kmalloc()ed here but must be freed by the caller. - */ -static char *nvgpu_dma_flags_to_str(struct gk20a *g, unsigned long flags) -{ - char *buf = nvgpu_kzalloc(g, NVGPU_DMA_STR_SIZE); - int bytes_available = NVGPU_DMA_STR_SIZE; - - /* - * Return the empty buffer if there's no flags. Makes it easier on the - * calling code to just print it instead of any if (NULL) type logic. - */ - if (!flags) - return buf; - -#define APPEND_FLAG(flag, str_flag) \ - do { \ - if (flags & flag) { \ - strncat(buf, str_flag, bytes_available); \ - bytes_available -= strlen(str_flag); \ - } \ - } while (0) - - APPEND_FLAG(NVGPU_DMA_NO_KERNEL_MAPPING, "NO_KERNEL_MAPPING "); - APPEND_FLAG(NVGPU_DMA_FORCE_CONTIGUOUS, "FORCE_CONTIGUOUS "); -#undef APPEND_FLAG - - return buf; -} - -/** - * __dma_dbg - Debug print for DMA allocs and frees. - * - * @g - The GPU. - * @size - The requested size of the alloc (size_t). - * @flags - The flags (unsigned long). - * @type - A string describing the type (i.e: sysmem or vidmem). - * @what - A string with 'alloc' or 'free'. - * - * @flags is the DMA flags. If there are none or it doesn't make sense to print - * flags just pass 0. - * - * Please use dma_dbg_alloc() and dma_dbg_free() instead of this function. - */ -static void __dma_dbg(struct gk20a *g, size_t size, unsigned long flags, - const char *type, const char *what, - const char *func, int line) -{ - char *flags_str = NULL; - - /* - * Don't bother making the flags_str if debugging is - * not enabled. This saves a malloc and a free. - */ - if (!nvgpu_log_mask_enabled(g, gpu_dbg_dma)) - return; - - flags_str = nvgpu_dma_flags_to_str(g, flags); - - __nvgpu_log_dbg(g, gpu_dbg_dma, - func, line, - "DMA %s: [%s] size=%-7zu " - "aligned=%-7zu total=%-10llukB %s", - what, type, - size, PAGE_ALIGN(size), - g->dma_memory_used >> 10, - flags_str); - - if (flags_str) - nvgpu_kfree(g, flags_str); -} - -#define dma_dbg_alloc(g, size, flags, type) \ - __dma_dbg(g, size, flags, type, "alloc", __func__, __LINE__) -#define dma_dbg_free(g, size, flags, type) \ - __dma_dbg(g, size, flags, type, "free", __func__, __LINE__) - -/* - * For after the DMA alloc is done. - */ -#define __dma_dbg_done(g, size, type, what) \ - nvgpu_log(g, gpu_dbg_dma, \ - "DMA %s: [%s] size=%-7zu Done!", \ - what, type, size); \ - -#define dma_dbg_alloc_done(g, size, type) \ - __dma_dbg_done(g, size, type, "alloc") -#define dma_dbg_free_done(g, size, type) \ - __dma_dbg_done(g, size, type, "free") - -#if defined(CONFIG_GK20A_VIDMEM) -static u64 __nvgpu_dma_alloc(struct nvgpu_allocator *allocator, u64 at, - size_t size) -{ - u64 addr = 0; - - if (at) - addr = nvgpu_alloc_fixed(allocator, at, size, 0); - else - addr = nvgpu_alloc(allocator, size); - - return addr; -} -#endif - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) -static void nvgpu_dma_flags_to_attrs(unsigned long *attrs, - unsigned long flags) -#define ATTR_ARG(x) *x -#else -static void nvgpu_dma_flags_to_attrs(struct dma_attrs *attrs, - unsigned long flags) -#define ATTR_ARG(x) x -#endif -{ - if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) - dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, ATTR_ARG(attrs)); - if (flags & NVGPU_DMA_FORCE_CONTIGUOUS) - dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, ATTR_ARG(attrs)); -#undef ATTR_ARG -} - -int nvgpu_dma_alloc(struct gk20a *g, size_t size, struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_flags(g, 0, size, mem); -} - -int nvgpu_dma_alloc_flags(struct gk20a *g, unsigned long flags, size_t size, - struct nvgpu_mem *mem) -{ - if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) { - /* - * Force the no-kernel-mapping flag on because we don't support - * the lack of it for vidmem - the user should not care when - * using nvgpu_gmmu_alloc_map and it's vidmem, or if there's a - * difference, the user should use the flag explicitly anyway. - * - * Incoming flags are ignored here, since bits other than the - * no-kernel-mapping flag are ignored by the vidmem mapping - * functions anyway. - */ - int err = nvgpu_dma_alloc_flags_vid(g, - NVGPU_DMA_NO_KERNEL_MAPPING, - size, mem); - - if (!err) - return 0; - /* - * Fall back to sysmem (which may then also fail) in case - * vidmem is exhausted. - */ - } - - return nvgpu_dma_alloc_flags_sys(g, flags, size, mem); -} - -int nvgpu_dma_alloc_sys(struct gk20a *g, size_t size, struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_flags_sys(g, 0, size, mem); -} - -int nvgpu_dma_alloc_flags_sys(struct gk20a *g, unsigned long flags, - size_t size, struct nvgpu_mem *mem) -{ - struct device *d = dev_from_gk20a(g); - int err; - dma_addr_t iova; - NVGPU_DEFINE_DMA_ATTRS(dma_attrs); - void *alloc_ret; - - if (nvgpu_mem_is_valid(mem)) { - nvgpu_warn(g, "memory leak !!"); - WARN_ON(1); - } - - /* - * WAR for IO coherent chips: the DMA API does not seem to generate - * mappings that work correctly. Unclear why - Bug ID: 2040115. - * - * Basically we just tell the DMA API not to map with NO_KERNEL_MAPPING - * and then make a vmap() ourselves. - */ - if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) - flags |= NVGPU_DMA_NO_KERNEL_MAPPING; - - /* - * Before the debug print so we see this in the total. But during - * cleanup in the fail path this has to be subtracted. - */ - g->dma_memory_used += PAGE_ALIGN(size); - - dma_dbg_alloc(g, size, flags, "sysmem"); - - /* - * Save the old size but for actual allocation purposes the size is - * going to be page aligned. - */ - mem->size = size; - size = PAGE_ALIGN(size); - - nvgpu_dma_flags_to_attrs(&dma_attrs, flags); - - alloc_ret = dma_alloc_attrs(d, size, &iova, - GFP_KERNEL|__GFP_ZERO, - NVGPU_DMA_ATTR(dma_attrs)); - if (!alloc_ret) - return -ENOMEM; - - if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) { - mem->priv.pages = alloc_ret; - err = nvgpu_get_sgtable_from_pages(g, &mem->priv.sgt, - mem->priv.pages, - iova, size); - } else { - mem->cpu_va = alloc_ret; - err = nvgpu_get_sgtable_attrs(g, &mem->priv.sgt, mem->cpu_va, - iova, size, flags); - } - if (err) - goto fail_free_dma; - - if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) { - mem->cpu_va = vmap(mem->priv.pages, - size >> PAGE_SHIFT, - 0, PAGE_KERNEL); - if (!mem->cpu_va) { - err = -ENOMEM; - goto fail_free_sgt; - } - } - - mem->aligned_size = size; - mem->aperture = APERTURE_SYSMEM; - mem->priv.flags = flags; - - dma_dbg_alloc_done(g, mem->size, "sysmem"); - - return 0; - -fail_free_sgt: - nvgpu_free_sgtable(g, &mem->priv.sgt); -fail_free_dma: - dma_free_attrs(d, size, alloc_ret, iova, NVGPU_DMA_ATTR(dma_attrs)); - mem->cpu_va = NULL; - mem->priv.sgt = NULL; - mem->size = 0; - g->dma_memory_used -= mem->aligned_size; - return err; -} - -int nvgpu_dma_alloc_vid(struct gk20a *g, size_t size, struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_flags_vid(g, - NVGPU_DMA_NO_KERNEL_MAPPING, size, mem); -} - -int nvgpu_dma_alloc_flags_vid(struct gk20a *g, unsigned long flags, - size_t size, struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_flags_vid_at(g, flags, size, mem, 0); -} - -int nvgpu_dma_alloc_vid_at(struct gk20a *g, - size_t size, struct nvgpu_mem *mem, u64 at) -{ - return nvgpu_dma_alloc_flags_vid_at(g, - NVGPU_DMA_NO_KERNEL_MAPPING, size, mem, at); -} - -int nvgpu_dma_alloc_flags_vid_at(struct gk20a *g, unsigned long flags, - size_t size, struct nvgpu_mem *mem, u64 at) -{ -#if defined(CONFIG_GK20A_VIDMEM) - u64 addr; - int err; - struct nvgpu_allocator *vidmem_alloc = g->mm.vidmem.cleared ? - &g->mm.vidmem.allocator : - &g->mm.vidmem.bootstrap_allocator; - u64 before_pending; - - if (nvgpu_mem_is_valid(mem)) { - nvgpu_warn(g, "memory leak !!"); - WARN_ON(1); - } - - dma_dbg_alloc(g, size, flags, "vidmem"); - - mem->size = size; - size = PAGE_ALIGN(size); - - if (!nvgpu_alloc_initialized(&g->mm.vidmem.allocator)) - return -ENOSYS; - - /* - * Our own allocator doesn't have any flags yet, and we can't - * kernel-map these, so require explicit flags. - */ - WARN_ON(flags != NVGPU_DMA_NO_KERNEL_MAPPING); - - nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex); - before_pending = atomic64_read(&g->mm.vidmem.bytes_pending.atomic_var); - addr = __nvgpu_dma_alloc(vidmem_alloc, at, size); - nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); - if (!addr) { - /* - * If memory is known to be freed soon, let the user know that - * it may be available after a while. - */ - if (before_pending) - return -EAGAIN; - else - return -ENOMEM; - } - - if (at) - mem->mem_flags |= NVGPU_MEM_FLAG_FIXED; - - mem->priv.sgt = nvgpu_kzalloc(g, sizeof(struct sg_table)); - if (!mem->priv.sgt) { - err = -ENOMEM; - goto fail_physfree; - } - - err = sg_alloc_table(mem->priv.sgt, 1, GFP_KERNEL); - if (err) - goto fail_kfree; - - nvgpu_vidmem_set_page_alloc(mem->priv.sgt->sgl, addr); - sg_set_page(mem->priv.sgt->sgl, NULL, size, 0); - - mem->aligned_size = size; - mem->aperture = APERTURE_VIDMEM; - mem->vidmem_alloc = (struct nvgpu_page_alloc *)(uintptr_t)addr; - mem->allocator = vidmem_alloc; - mem->priv.flags = flags; - - nvgpu_init_list_node(&mem->clear_list_entry); - - dma_dbg_alloc_done(g, mem->size, "vidmem"); - - return 0; - -fail_kfree: - nvgpu_kfree(g, mem->priv.sgt); -fail_physfree: - nvgpu_free(&g->mm.vidmem.allocator, addr); - mem->size = 0; - return err; -#else - return -ENOSYS; -#endif -} - -int nvgpu_dma_alloc_map(struct vm_gk20a *vm, size_t size, - struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_map_flags(vm, 0, size, mem); -} - -int nvgpu_dma_alloc_map_flags(struct vm_gk20a *vm, unsigned long flags, - size_t size, struct nvgpu_mem *mem) -{ - if (!nvgpu_is_enabled(gk20a_from_vm(vm), NVGPU_MM_UNIFIED_MEMORY)) { - /* - * Force the no-kernel-mapping flag on because we don't support - * the lack of it for vidmem - the user should not care when - * using nvgpu_dma_alloc_map and it's vidmem, or if there's a - * difference, the user should use the flag explicitly anyway. - */ - int err = nvgpu_dma_alloc_map_flags_vid(vm, - flags | NVGPU_DMA_NO_KERNEL_MAPPING, - size, mem); - - if (!err) - return 0; - /* - * Fall back to sysmem (which may then also fail) in case - * vidmem is exhausted. - */ - } - - return nvgpu_dma_alloc_map_flags_sys(vm, flags, size, mem); -} - -int nvgpu_dma_alloc_map_sys(struct vm_gk20a *vm, size_t size, - struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_map_flags_sys(vm, 0, size, mem); -} - -int nvgpu_dma_alloc_map_flags_sys(struct vm_gk20a *vm, unsigned long flags, - size_t size, struct nvgpu_mem *mem) -{ - int err = nvgpu_dma_alloc_flags_sys(vm->mm->g, flags, size, mem); - - if (err) - return err; - - mem->gpu_va = nvgpu_gmmu_map(vm, mem, size, 0, - gk20a_mem_flag_none, false, - mem->aperture); - if (!mem->gpu_va) { - err = -ENOMEM; - goto fail_free; - } - - return 0; - -fail_free: - nvgpu_dma_free(vm->mm->g, mem); - return err; -} - -int nvgpu_dma_alloc_map_vid(struct vm_gk20a *vm, size_t size, - struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_map_flags_vid(vm, - NVGPU_DMA_NO_KERNEL_MAPPING, size, mem); -} - -int nvgpu_dma_alloc_map_flags_vid(struct vm_gk20a *vm, unsigned long flags, - size_t size, struct nvgpu_mem *mem) -{ - int err = nvgpu_dma_alloc_flags_vid(vm->mm->g, flags, size, mem); - - if (err) - return err; - - mem->gpu_va = nvgpu_gmmu_map(vm, mem, size, 0, - gk20a_mem_flag_none, false, - mem->aperture); - if (!mem->gpu_va) { - err = -ENOMEM; - goto fail_free; - } - - return 0; - -fail_free: - nvgpu_dma_free(vm->mm->g, mem); - return err; -} - -static void nvgpu_dma_free_sys(struct gk20a *g, struct nvgpu_mem *mem) -{ - struct device *d = dev_from_gk20a(g); - - g->dma_memory_used -= mem->aligned_size; - - dma_dbg_free(g, mem->size, mem->priv.flags, "sysmem"); - - if (!(mem->mem_flags & NVGPU_MEM_FLAG_SHADOW_COPY) && - !(mem->mem_flags & __NVGPU_MEM_FLAG_NO_DMA) && - (mem->cpu_va || mem->priv.pages)) { - /* - * Free side of WAR for bug 2040115. - */ - if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) - vunmap(mem->cpu_va); - - if (mem->priv.flags) { - NVGPU_DEFINE_DMA_ATTRS(dma_attrs); - - nvgpu_dma_flags_to_attrs(&dma_attrs, mem->priv.flags); - - if (mem->priv.flags & NVGPU_DMA_NO_KERNEL_MAPPING) { - dma_free_attrs(d, mem->aligned_size, mem->priv.pages, - sg_dma_address(mem->priv.sgt->sgl), - NVGPU_DMA_ATTR(dma_attrs)); - } else { - dma_free_attrs(d, mem->aligned_size, mem->cpu_va, - sg_dma_address(mem->priv.sgt->sgl), - NVGPU_DMA_ATTR(dma_attrs)); - } - } else { - dma_free_coherent(d, mem->aligned_size, mem->cpu_va, - sg_dma_address(mem->priv.sgt->sgl)); - } - mem->cpu_va = NULL; - mem->priv.pages = NULL; - } - - /* - * When this flag is set we expect that pages is still populated but not - * by the DMA API. - */ - if (mem->mem_flags & __NVGPU_MEM_FLAG_NO_DMA) - nvgpu_kfree(g, mem->priv.pages); - - if (mem->priv.sgt) - nvgpu_free_sgtable(g, &mem->priv.sgt); - - dma_dbg_free_done(g, mem->size, "sysmem"); - - mem->size = 0; - mem->aligned_size = 0; - mem->aperture = APERTURE_INVALID; -} - -static void nvgpu_dma_free_vid(struct gk20a *g, struct nvgpu_mem *mem) -{ -#if defined(CONFIG_GK20A_VIDMEM) - size_t mem_size = mem->size; - - dma_dbg_free(g, mem->size, mem->priv.flags, "vidmem"); - - /* Sanity check - only this supported when allocating. */ - WARN_ON(mem->priv.flags != NVGPU_DMA_NO_KERNEL_MAPPING); - - if (mem->mem_flags & NVGPU_MEM_FLAG_USER_MEM) { - int err = nvgpu_vidmem_clear_list_enqueue(g, mem); - - /* - * If there's an error here then that means we can't clear the - * vidmem. That's too bad; however, we still own the nvgpu_mem - * buf so we have to free that. - * - * We don't need to worry about the vidmem allocator itself - * since when that gets cleaned up in the driver shutdown path - * all the outstanding allocs are force freed. - */ - if (err) - nvgpu_kfree(g, mem); - } else { - nvgpu_memset(g, mem, 0, 0, mem->aligned_size); - nvgpu_free(mem->allocator, - (u64)nvgpu_vidmem_get_page_alloc(mem->priv.sgt->sgl)); - nvgpu_free_sgtable(g, &mem->priv.sgt); - - mem->size = 0; - mem->aligned_size = 0; - mem->aperture = APERTURE_INVALID; - } - - dma_dbg_free_done(g, mem_size, "vidmem"); -#endif -} - -void nvgpu_dma_free(struct gk20a *g, struct nvgpu_mem *mem) -{ - switch (mem->aperture) { - case APERTURE_SYSMEM: - return nvgpu_dma_free_sys(g, mem); - case APERTURE_VIDMEM: - return nvgpu_dma_free_vid(g, mem); - default: - break; /* like free() on "null" memory */ - } -} - -void nvgpu_dma_unmap_free(struct vm_gk20a *vm, struct nvgpu_mem *mem) -{ - if (mem->gpu_va) - nvgpu_gmmu_unmap(vm, mem, mem->gpu_va); - mem->gpu_va = 0; - - nvgpu_dma_free(vm->mm->g, mem); -} - -int nvgpu_get_sgtable_attrs(struct gk20a *g, struct sg_table **sgt, - void *cpuva, u64 iova, size_t size, unsigned long flags) -{ - int err = 0; - struct sg_table *tbl; - NVGPU_DEFINE_DMA_ATTRS(dma_attrs); - - tbl = nvgpu_kzalloc(g, sizeof(struct sg_table)); - if (!tbl) { - err = -ENOMEM; - goto fail; - } - - nvgpu_dma_flags_to_attrs(&dma_attrs, flags); - err = dma_get_sgtable_attrs(dev_from_gk20a(g), tbl, cpuva, iova, - size, NVGPU_DMA_ATTR(dma_attrs)); - if (err) - goto fail; - - sg_dma_address(tbl->sgl) = iova; - *sgt = tbl; - - return 0; - -fail: - if (tbl) - nvgpu_kfree(g, tbl); - - return err; -} - -int nvgpu_get_sgtable(struct gk20a *g, struct sg_table **sgt, - void *cpuva, u64 iova, size_t size) -{ - return nvgpu_get_sgtable_attrs(g, sgt, cpuva, iova, size, 0); -} - -int nvgpu_get_sgtable_from_pages(struct gk20a *g, struct sg_table **sgt, - struct page **pages, u64 iova, size_t size) -{ - int err = 0; - struct sg_table *tbl; - - tbl = nvgpu_kzalloc(g, sizeof(struct sg_table)); - if (!tbl) { - err = -ENOMEM; - goto fail; - } - - err = sg_alloc_table_from_pages(tbl, pages, - DIV_ROUND_UP(size, PAGE_SIZE), - 0, size, GFP_KERNEL); - if (err) - goto fail; - - sg_dma_address(tbl->sgl) = iova; - *sgt = tbl; - - return 0; - -fail: - if (tbl) - nvgpu_kfree(g, tbl); - - return err; -} - -void nvgpu_free_sgtable(struct gk20a *g, struct sg_table **sgt) -{ - sg_free_table(*sgt); - nvgpu_kfree(g, *sgt); - *sgt = NULL; -} - -bool nvgpu_iommuable(struct gk20a *g) -{ -#ifdef CONFIG_TEGRA_GK20A - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - - /* - * Check against the nvgpu device to see if it's been marked as - * IOMMU'able. - */ - if (!device_is_iommuable(l->dev)) - return false; -#endif - - return true; -} diff --git a/drivers/gpu/nvgpu/os/linux/linux-dma.c b/drivers/gpu/nvgpu/os/linux/linux-dma.c new file mode 100644 index 00000000..a42e7cb5 --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/linux-dma.c @@ -0,0 +1,533 @@ +/* + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "gk20a/gk20a.h" + +#include "platform_gk20a.h" +#include "os_linux.h" +#include "dmabuf_vidmem.h" + +#ifdef __DMA_ATTRS_LONGS +#define NVGPU_DEFINE_DMA_ATTRS(x) \ + struct dma_attrs x = { \ + .flags = { [0 ... __DMA_ATTRS_LONGS-1] = 0 }, \ + } +#define NVGPU_DMA_ATTR(attrs) &attrs +#else +#define NVGPU_DEFINE_DMA_ATTRS(attrs) unsigned long attrs = 0 +#define NVGPU_DMA_ATTR(attrs) attrs +#endif + +/* + * Enough to hold all the possible flags in string form. When a new flag is + * added it must be added here as well!! + */ +#define NVGPU_DMA_STR_SIZE \ + sizeof("NO_KERNEL_MAPPING FORCE_CONTIGUOUS") + +/* + * The returned string is kmalloc()ed here but must be freed by the caller. + */ +static char *nvgpu_dma_flags_to_str(struct gk20a *g, unsigned long flags) +{ + char *buf = nvgpu_kzalloc(g, NVGPU_DMA_STR_SIZE); + int bytes_available = NVGPU_DMA_STR_SIZE; + + /* + * Return the empty buffer if there's no flags. Makes it easier on the + * calling code to just print it instead of any if (NULL) type logic. + */ + if (!flags) + return buf; + +#define APPEND_FLAG(flag, str_flag) \ + do { \ + if (flags & flag) { \ + strncat(buf, str_flag, bytes_available); \ + bytes_available -= strlen(str_flag); \ + } \ + } while (0) + + APPEND_FLAG(NVGPU_DMA_NO_KERNEL_MAPPING, "NO_KERNEL_MAPPING "); + APPEND_FLAG(NVGPU_DMA_FORCE_CONTIGUOUS, "FORCE_CONTIGUOUS "); +#undef APPEND_FLAG + + return buf; +} + +/** + * __dma_dbg - Debug print for DMA allocs and frees. + * + * @g - The GPU. + * @size - The requested size of the alloc (size_t). + * @flags - The flags (unsigned long). + * @type - A string describing the type (i.e: sysmem or vidmem). + * @what - A string with 'alloc' or 'free'. + * + * @flags is the DMA flags. If there are none or it doesn't make sense to print + * flags just pass 0. + * + * Please use dma_dbg_alloc() and dma_dbg_free() instead of this function. + */ +static void __dma_dbg(struct gk20a *g, size_t size, unsigned long flags, + const char *type, const char *what, + const char *func, int line) +{ + char *flags_str = NULL; + + /* + * Don't bother making the flags_str if debugging is + * not enabled. This saves a malloc and a free. + */ + if (!nvgpu_log_mask_enabled(g, gpu_dbg_dma)) + return; + + flags_str = nvgpu_dma_flags_to_str(g, flags); + + __nvgpu_log_dbg(g, gpu_dbg_dma, + func, line, + "DMA %s: [%s] size=%-7zu " + "aligned=%-7zu total=%-10llukB %s", + what, type, + size, PAGE_ALIGN(size), + g->dma_memory_used >> 10, + flags_str); + + if (flags_str) + nvgpu_kfree(g, flags_str); +} + +#define dma_dbg_alloc(g, size, flags, type) \ + __dma_dbg(g, size, flags, type, "alloc", __func__, __LINE__) +#define dma_dbg_free(g, size, flags, type) \ + __dma_dbg(g, size, flags, type, "free", __func__, __LINE__) + +/* + * For after the DMA alloc is done. + */ +#define __dma_dbg_done(g, size, type, what) \ + nvgpu_log(g, gpu_dbg_dma, \ + "DMA %s: [%s] size=%-7zu Done!", \ + what, type, size); \ + +#define dma_dbg_alloc_done(g, size, type) \ + __dma_dbg_done(g, size, type, "alloc") +#define dma_dbg_free_done(g, size, type) \ + __dma_dbg_done(g, size, type, "free") + +#if defined(CONFIG_GK20A_VIDMEM) +static u64 __nvgpu_dma_alloc(struct nvgpu_allocator *allocator, u64 at, + size_t size) +{ + u64 addr = 0; + + if (at) + addr = nvgpu_alloc_fixed(allocator, at, size, 0); + else + addr = nvgpu_alloc(allocator, size); + + return addr; +} +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) +static void nvgpu_dma_flags_to_attrs(unsigned long *attrs, + unsigned long flags) +#define ATTR_ARG(x) *x +#else +static void nvgpu_dma_flags_to_attrs(struct dma_attrs *attrs, + unsigned long flags) +#define ATTR_ARG(x) x +#endif +{ + if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) + dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, ATTR_ARG(attrs)); + if (flags & NVGPU_DMA_FORCE_CONTIGUOUS) + dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, ATTR_ARG(attrs)); +#undef ATTR_ARG +} + +int nvgpu_dma_alloc_flags_sys(struct gk20a *g, unsigned long flags, + size_t size, struct nvgpu_mem *mem) +{ + struct device *d = dev_from_gk20a(g); + int err; + dma_addr_t iova; + NVGPU_DEFINE_DMA_ATTRS(dma_attrs); + void *alloc_ret; + + if (nvgpu_mem_is_valid(mem)) { + nvgpu_warn(g, "memory leak !!"); + WARN_ON(1); + } + + /* + * WAR for IO coherent chips: the DMA API does not seem to generate + * mappings that work correctly. Unclear why - Bug ID: 2040115. + * + * Basically we just tell the DMA API not to map with NO_KERNEL_MAPPING + * and then make a vmap() ourselves. + */ + if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) + flags |= NVGPU_DMA_NO_KERNEL_MAPPING; + + /* + * Before the debug print so we see this in the total. But during + * cleanup in the fail path this has to be subtracted. + */ + g->dma_memory_used += PAGE_ALIGN(size); + + dma_dbg_alloc(g, size, flags, "sysmem"); + + /* + * Save the old size but for actual allocation purposes the size is + * going to be page aligned. + */ + mem->size = size; + size = PAGE_ALIGN(size); + + nvgpu_dma_flags_to_attrs(&dma_attrs, flags); + + alloc_ret = dma_alloc_attrs(d, size, &iova, + GFP_KERNEL|__GFP_ZERO, + NVGPU_DMA_ATTR(dma_attrs)); + if (!alloc_ret) + return -ENOMEM; + + if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) { + mem->priv.pages = alloc_ret; + err = nvgpu_get_sgtable_from_pages(g, &mem->priv.sgt, + mem->priv.pages, + iova, size); + } else { + mem->cpu_va = alloc_ret; + err = nvgpu_get_sgtable_attrs(g, &mem->priv.sgt, mem->cpu_va, + iova, size, flags); + } + if (err) + goto fail_free_dma; + + if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) { + mem->cpu_va = vmap(mem->priv.pages, + size >> PAGE_SHIFT, + 0, PAGE_KERNEL); + if (!mem->cpu_va) { + err = -ENOMEM; + goto fail_free_sgt; + } + } + + mem->aligned_size = size; + mem->aperture = APERTURE_SYSMEM; + mem->priv.flags = flags; + + dma_dbg_alloc_done(g, mem->size, "sysmem"); + + return 0; + +fail_free_sgt: + nvgpu_free_sgtable(g, &mem->priv.sgt); +fail_free_dma: + dma_free_attrs(d, size, alloc_ret, iova, NVGPU_DMA_ATTR(dma_attrs)); + mem->cpu_va = NULL; + mem->priv.sgt = NULL; + mem->size = 0; + g->dma_memory_used -= mem->aligned_size; + return err; +} + +int nvgpu_dma_alloc_flags_vid_at(struct gk20a *g, unsigned long flags, + size_t size, struct nvgpu_mem *mem, u64 at) +{ +#if defined(CONFIG_GK20A_VIDMEM) + u64 addr; + int err; + struct nvgpu_allocator *vidmem_alloc = g->mm.vidmem.cleared ? + &g->mm.vidmem.allocator : + &g->mm.vidmem.bootstrap_allocator; + u64 before_pending; + + if (nvgpu_mem_is_valid(mem)) { + nvgpu_warn(g, "memory leak !!"); + WARN_ON(1); + } + + dma_dbg_alloc(g, size, flags, "vidmem"); + + mem->size = size; + size = PAGE_ALIGN(size); + + if (!nvgpu_alloc_initialized(&g->mm.vidmem.allocator)) + return -ENOSYS; + + /* + * Our own allocator doesn't have any flags yet, and we can't + * kernel-map these, so require explicit flags. + */ + WARN_ON(flags != NVGPU_DMA_NO_KERNEL_MAPPING); + + nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex); + before_pending = atomic64_read(&g->mm.vidmem.bytes_pending.atomic_var); + addr = __nvgpu_dma_alloc(vidmem_alloc, at, size); + nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); + if (!addr) { + /* + * If memory is known to be freed soon, let the user know that + * it may be available after a while. + */ + if (before_pending) + return -EAGAIN; + else + return -ENOMEM; + } + + if (at) + mem->mem_flags |= NVGPU_MEM_FLAG_FIXED; + + mem->priv.sgt = nvgpu_kzalloc(g, sizeof(struct sg_table)); + if (!mem->priv.sgt) { + err = -ENOMEM; + goto fail_physfree; + } + + err = sg_alloc_table(mem->priv.sgt, 1, GFP_KERNEL); + if (err) + goto fail_kfree; + + nvgpu_vidmem_set_page_alloc(mem->priv.sgt->sgl, addr); + sg_set_page(mem->priv.sgt->sgl, NULL, size, 0); + + mem->aligned_size = size; + mem->aperture = APERTURE_VIDMEM; + mem->vidmem_alloc = (struct nvgpu_page_alloc *)(uintptr_t)addr; + mem->allocator = vidmem_alloc; + mem->priv.flags = flags; + + nvgpu_init_list_node(&mem->clear_list_entry); + + dma_dbg_alloc_done(g, mem->size, "vidmem"); + + return 0; + +fail_kfree: + nvgpu_kfree(g, mem->priv.sgt); +fail_physfree: + nvgpu_free(&g->mm.vidmem.allocator, addr); + mem->size = 0; + return err; +#else + return -ENOSYS; +#endif +} + +void nvgpu_dma_free_sys(struct gk20a *g, struct nvgpu_mem *mem) +{ + struct device *d = dev_from_gk20a(g); + + g->dma_memory_used -= mem->aligned_size; + + dma_dbg_free(g, mem->size, mem->priv.flags, "sysmem"); + + if (!(mem->mem_flags & NVGPU_MEM_FLAG_SHADOW_COPY) && + !(mem->mem_flags & __NVGPU_MEM_FLAG_NO_DMA) && + (mem->cpu_va || mem->priv.pages)) { + /* + * Free side of WAR for bug 2040115. + */ + if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) + vunmap(mem->cpu_va); + + if (mem->priv.flags) { + NVGPU_DEFINE_DMA_ATTRS(dma_attrs); + + nvgpu_dma_flags_to_attrs(&dma_attrs, mem->priv.flags); + + if (mem->priv.flags & NVGPU_DMA_NO_KERNEL_MAPPING) { + dma_free_attrs(d, mem->aligned_size, mem->priv.pages, + sg_dma_address(mem->priv.sgt->sgl), + NVGPU_DMA_ATTR(dma_attrs)); + } else { + dma_free_attrs(d, mem->aligned_size, mem->cpu_va, + sg_dma_address(mem->priv.sgt->sgl), + NVGPU_DMA_ATTR(dma_attrs)); + } + } else { + dma_free_coherent(d, mem->aligned_size, mem->cpu_va, + sg_dma_address(mem->priv.sgt->sgl)); + } + mem->cpu_va = NULL; + mem->priv.pages = NULL; + } + + /* + * When this flag is set we expect that pages is still populated but not + * by the DMA API. + */ + if (mem->mem_flags & __NVGPU_MEM_FLAG_NO_DMA) + nvgpu_kfree(g, mem->priv.pages); + + if (mem->priv.sgt) + nvgpu_free_sgtable(g, &mem->priv.sgt); + + dma_dbg_free_done(g, mem->size, "sysmem"); + + mem->size = 0; + mem->aligned_size = 0; + mem->aperture = APERTURE_INVALID; +} + +void nvgpu_dma_free_vid(struct gk20a *g, struct nvgpu_mem *mem) +{ +#if defined(CONFIG_GK20A_VIDMEM) + size_t mem_size = mem->size; + + dma_dbg_free(g, mem->size, mem->priv.flags, "vidmem"); + + /* Sanity check - only this supported when allocating. */ + WARN_ON(mem->priv.flags != NVGPU_DMA_NO_KERNEL_MAPPING); + + if (mem->mem_flags & NVGPU_MEM_FLAG_USER_MEM) { + int err = nvgpu_vidmem_clear_list_enqueue(g, mem); + + /* + * If there's an error here then that means we can't clear the + * vidmem. That's too bad; however, we still own the nvgpu_mem + * buf so we have to free that. + * + * We don't need to worry about the vidmem allocator itself + * since when that gets cleaned up in the driver shutdown path + * all the outstanding allocs are force freed. + */ + if (err) + nvgpu_kfree(g, mem); + } else { + nvgpu_memset(g, mem, 0, 0, mem->aligned_size); + nvgpu_free(mem->allocator, + (u64)nvgpu_vidmem_get_page_alloc(mem->priv.sgt->sgl)); + nvgpu_free_sgtable(g, &mem->priv.sgt); + + mem->size = 0; + mem->aligned_size = 0; + mem->aperture = APERTURE_INVALID; + } + + dma_dbg_free_done(g, mem_size, "vidmem"); +#endif +} + +int nvgpu_get_sgtable_attrs(struct gk20a *g, struct sg_table **sgt, + void *cpuva, u64 iova, size_t size, unsigned long flags) +{ + int err = 0; + struct sg_table *tbl; + NVGPU_DEFINE_DMA_ATTRS(dma_attrs); + + tbl = nvgpu_kzalloc(g, sizeof(struct sg_table)); + if (!tbl) { + err = -ENOMEM; + goto fail; + } + + nvgpu_dma_flags_to_attrs(&dma_attrs, flags); + err = dma_get_sgtable_attrs(dev_from_gk20a(g), tbl, cpuva, iova, + size, NVGPU_DMA_ATTR(dma_attrs)); + if (err) + goto fail; + + sg_dma_address(tbl->sgl) = iova; + *sgt = tbl; + + return 0; + +fail: + if (tbl) + nvgpu_kfree(g, tbl); + + return err; +} + +int nvgpu_get_sgtable(struct gk20a *g, struct sg_table **sgt, + void *cpuva, u64 iova, size_t size) +{ + return nvgpu_get_sgtable_attrs(g, sgt, cpuva, iova, size, 0); +} + +int nvgpu_get_sgtable_from_pages(struct gk20a *g, struct sg_table **sgt, + struct page **pages, u64 iova, size_t size) +{ + int err = 0; + struct sg_table *tbl; + + tbl = nvgpu_kzalloc(g, sizeof(struct sg_table)); + if (!tbl) { + err = -ENOMEM; + goto fail; + } + + err = sg_alloc_table_from_pages(tbl, pages, + DIV_ROUND_UP(size, PAGE_SIZE), + 0, size, GFP_KERNEL); + if (err) + goto fail; + + sg_dma_address(tbl->sgl) = iova; + *sgt = tbl; + + return 0; + +fail: + if (tbl) + nvgpu_kfree(g, tbl); + + return err; +} + +void nvgpu_free_sgtable(struct gk20a *g, struct sg_table **sgt) +{ + sg_free_table(*sgt); + nvgpu_kfree(g, *sgt); + *sgt = NULL; +} + +bool nvgpu_iommuable(struct gk20a *g) +{ +#ifdef CONFIG_TEGRA_GK20A + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + + /* + * Check against the nvgpu device to see if it's been marked as + * IOMMU'able. + */ + if (!device_is_iommuable(l->dev)) + return false; +#endif + + return true; +} diff --git a/drivers/gpu/nvgpu/os/posix/dma.c b/drivers/gpu/nvgpu/os/posix/dma.c deleted file mode 100644 index 95bb1a75..00000000 --- a/drivers/gpu/nvgpu/os/posix/dma.c +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include - -/* - * In userspace vidmem vs sysmem is just a difference in what is placed in the - * aperture field. - */ -static int __nvgpu_do_dma_alloc(struct gk20a *g, unsigned long flags, - size_t size, struct nvgpu_mem *mem, - enum nvgpu_aperture ap) -{ - void *memory = malloc(mem->aligned_size); - - if (memory == NULL) - return -ENOMEM; - - mem->cpu_va = memory; - mem->aperture = ap; - mem->size = size; - mem->aligned_size = PAGE_ALIGN(size); - mem->gpu_va = 0ULL; - mem->skip_wmb = true; - mem->vidmem_alloc = NULL; - mem->allocator = NULL; - - return 0; -} - -bool nvgpu_iommuable(struct gk20a *g) -{ - return false; -} - -int nvgpu_dma_alloc(struct gk20a *g, size_t size, struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_flags(g, 0, size, mem); -} - -int nvgpu_dma_alloc_flags(struct gk20a *g, unsigned long flags, size_t size, - struct nvgpu_mem *mem) -{ - if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) { - /* - * First try vidmem. Obviously in userspace there's no such - * thing as vidmem per se but we will mark the aperture as - * vidmem. - */ - int err = nvgpu_dma_alloc_flags_vid(g, 0, size, mem); - - if (!err) - return 0; - /* - * Fall back to sysmem (which may then also fail) in case - * vidmem is exhausted. - */ - } - - return nvgpu_dma_alloc_flags_sys(g, flags, size, mem); - -} - -int nvgpu_dma_alloc_sys(struct gk20a *g, size_t size, struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_flags_sys(g, 0, size, mem); -} - -int nvgpu_dma_alloc_flags_sys(struct gk20a *g, unsigned long flags, - size_t size, struct nvgpu_mem *mem) -{ - return __nvgpu_do_dma_alloc(g, flags, size, mem, APERTURE_SYSMEM); -} - -int nvgpu_dma_alloc_vid(struct gk20a *g, size_t size, struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_flags_vid(g, 0, size, mem); -} - -int nvgpu_dma_alloc_flags_vid(struct gk20a *g, unsigned long flags, - size_t size, struct nvgpu_mem *mem) -{ - return __nvgpu_do_dma_alloc(g, flags, size, mem, APERTURE_VIDMEM); -} - -int nvgpu_dma_alloc_vid_at(struct gk20a *g, - size_t size, struct nvgpu_mem *mem, u64 at) -{ - BUG(); - - return 0; -} - -int nvgpu_dma_alloc_flags_vid_at(struct gk20a *g, unsigned long flags, - size_t size, struct nvgpu_mem *mem, u64 at) -{ - BUG(); - - return 0; -} - -void nvgpu_dma_free(struct gk20a *g, struct nvgpu_mem *mem) -{ - if (!(mem->mem_flags & NVGPU_MEM_FLAG_SHADOW_COPY)) - free(mem->cpu_va); - - memset(mem, 0, sizeof(*mem)); -} - -int nvgpu_dma_alloc_map(struct vm_gk20a *vm, size_t size, - struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_map_flags(vm, 0, size, mem); -} - -int nvgpu_dma_alloc_map_flags(struct vm_gk20a *vm, unsigned long flags, - size_t size, struct nvgpu_mem *mem) -{ - if (!nvgpu_is_enabled(gk20a_from_vm(vm), NVGPU_MM_UNIFIED_MEMORY)) { - int err = nvgpu_dma_alloc_map_flags_vid(vm, - flags | NVGPU_DMA_NO_KERNEL_MAPPING, - size, mem); - - if (!err) - return 0; - /* - * Fall back to sysmem (which may then also fail) in case - * vidmem is exhausted. - */ - } - - return nvgpu_dma_alloc_map_flags_sys(vm, flags, size, mem); -} - -int nvgpu_dma_alloc_map_sys(struct vm_gk20a *vm, size_t size, - struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_map_flags_sys(vm, 0, size, mem); -} - -int nvgpu_dma_alloc_map_flags_sys(struct vm_gk20a *vm, unsigned long flags, - size_t size, struct nvgpu_mem *mem) -{ - int err = nvgpu_dma_alloc_flags_sys(vm->mm->g, flags, size, mem); - - if (err) - return err; - - mem->gpu_va = nvgpu_gmmu_map(vm, mem, size, 0, - gk20a_mem_flag_none, false, - mem->aperture); - if (!mem->gpu_va) { - err = -ENOMEM; - goto fail_free; - } - - return 0; - -fail_free: - nvgpu_dma_free(vm->mm->g, mem); - return err; -} - -int nvgpu_dma_alloc_map_vid(struct vm_gk20a *vm, size_t size, - struct nvgpu_mem *mem) -{ - return nvgpu_dma_alloc_map_flags_vid(vm, - NVGPU_DMA_NO_KERNEL_MAPPING, size, mem); -} - -int nvgpu_dma_alloc_map_flags_vid(struct vm_gk20a *vm, unsigned long flags, - size_t size, struct nvgpu_mem *mem) -{ - int err = nvgpu_dma_alloc_flags_vid(vm->mm->g, flags, size, mem); - - if (err) - return err; - - mem->gpu_va = nvgpu_gmmu_map(vm, mem, size, 0, - gk20a_mem_flag_none, false, - mem->aperture); - if (!mem->gpu_va) { - err = -ENOMEM; - goto fail_free; - } - - return 0; - -fail_free: - nvgpu_dma_free(vm->mm->g, mem); - return err; -} - -void nvgpu_dma_unmap_free(struct vm_gk20a *vm, struct nvgpu_mem *mem) -{ - if (mem->gpu_va) - nvgpu_gmmu_unmap(vm, mem, mem->gpu_va); - mem->gpu_va = 0; - - nvgpu_dma_free(vm->mm->g, mem); -} diff --git a/drivers/gpu/nvgpu/os/posix/posix-dma.c b/drivers/gpu/nvgpu/os/posix/posix-dma.c new file mode 100644 index 00000000..e8c5c9d4 --- /dev/null +++ b/drivers/gpu/nvgpu/os/posix/posix-dma.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +/* + * In userspace vidmem vs sysmem is just a difference in what is placed in the + * aperture field. + */ +static int __nvgpu_do_dma_alloc(struct gk20a *g, unsigned long flags, + size_t size, struct nvgpu_mem *mem, + enum nvgpu_aperture ap) +{ + void *memory = malloc(mem->aligned_size); + + if (memory == NULL) + return -ENOMEM; + + mem->cpu_va = memory; + mem->aperture = ap; + mem->size = size; + mem->aligned_size = PAGE_ALIGN(size); + mem->gpu_va = 0ULL; + mem->skip_wmb = true; + mem->vidmem_alloc = NULL; + mem->allocator = NULL; + + return 0; +} + +bool nvgpu_iommuable(struct gk20a *g) +{ + return false; +} + +int nvgpu_dma_alloc_flags_sys(struct gk20a *g, unsigned long flags, + size_t size, struct nvgpu_mem *mem) +{ + return __nvgpu_do_dma_alloc(g, flags, size, mem, APERTURE_SYSMEM); +} + +int nvgpu_dma_alloc_flags_vid_at(struct gk20a *g, unsigned long flags, + size_t size, struct nvgpu_mem *mem, u64 at) +{ + BUG(); + + return 0; +} + +void nvgpu_dma_free_sys(struct gk20a *g, struct nvgpu_mem *mem) +{ + if (!(mem->mem_flags & NVGPU_MEM_FLAG_SHADOW_COPY)) + free(mem->cpu_va); + + memset(mem, 0, sizeof(*mem)); +} + +void nvgpu_dma_free_vid(struct gk20a *g, struct nvgpu_mem *mem) +{ + BUG(); +} -- cgit v1.2.2