1 files changed, 534 insertions, 0 deletions

diff --git a/include/os/linux/linux-dma.c b/include/os/linux/linux-dma.c
new file mode 100644
index 0000000..d704b2a
--- /dev/null
+++ b/include/os/linux/linux-dma.c
@@ -0,0 +1,534 @@
+/*
+ * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/version.h>
+
+#include <nvgpu/log.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/lock.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/gmmu.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/vidmem.h>
+#include <nvgpu/gk20a.h>
+
+#include <nvgpu/linux/dma.h>
+
+#include "platform_gk20a.h"
+#include "os_linux.h"
+#include "dmabuf_vidmem.h"
+
+#ifdef __DMA_ATTRS_LONGS
+#define NVGPU_DEFINE_DMA_ATTRS(x)                                     \
+        struct dma_attrs x = {                                  \
+                .flags = { [0 ... __DMA_ATTRS_LONGS-1] = 0 },   \
+        }
+#define NVGPU_DMA_ATTR(attrs) &attrs
+#else
+#define NVGPU_DEFINE_DMA_ATTRS(attrs) unsigned long attrs = 0
+#define NVGPU_DMA_ATTR(attrs) attrs
+#endif
+
+/*
+ * Enough to hold all the possible flags in string form. When a new flag is
+ * added it must be added here as well!!
+ */
+#define NVGPU_DMA_STR_SIZE                                      \
+        sizeof("NO_KERNEL_MAPPING FORCE_CONTIGUOUS")
+
+/*
+ * The returned string is kmalloc()ed here but must be freed by the caller.
+ */
+static char *nvgpu_dma_flags_to_str(struct gk20a *g, unsigned long flags)
+{
+        char *buf = nvgpu_kzalloc(g, NVGPU_DMA_STR_SIZE);
+        int bytes_available = NVGPU_DMA_STR_SIZE;
+
+        /*
+         * Return the empty buffer if there's no flags. Makes it easier on the
+         * calling code to just print it instead of any if (NULL) type logic.
+         */
+        if (!flags)
+                return buf;
+
+#define APPEND_FLAG(flag, str_flag)                                     \
+        do {                                                            \
+                if (flags & flag) {                                     \
+                        strncat(buf, str_flag, bytes_available);        \
+                        bytes_available -= strlen(str_flag);            \
+                }                                                       \
+        } while (0)
+
+        APPEND_FLAG(NVGPU_DMA_NO_KERNEL_MAPPING, "NO_KERNEL_MAPPING ");
+        APPEND_FLAG(NVGPU_DMA_FORCE_CONTIGUOUS,  "FORCE_CONTIGUOUS ");
+#undef APPEND_FLAG
+
+        return buf;
+}
+
+/**
+ * __dma_dbg - Debug print for DMA allocs and frees.
+ *
+ * @g     - The GPU.
+ * @size  - The requested size of the alloc (size_t).
+ * @flags - The flags (unsigned long).
+ * @type  - A string describing the type (i.e: sysmem or vidmem).
+ * @what  - A string with 'alloc' or 'free'.
+ *
+ * @flags is the DMA flags. If there are none or it doesn't make sense to print
+ * flags just pass 0.
+ *
+ * Please use dma_dbg_alloc() and dma_dbg_free() instead of this function.
+ */
+static void __dma_dbg(struct gk20a *g, size_t size, unsigned long flags,
+                      const char *type, const char *what,
+                      const char *func, int line)
+{
+        char *flags_str = NULL;
+
+        /*
+         * Don't bother making the flags_str if debugging is
+         * not enabled. This saves a malloc and a free.
+         */
+        if (!nvgpu_log_mask_enabled(g, gpu_dbg_dma))
+                return;
+
+        flags_str = nvgpu_dma_flags_to_str(g, flags);
+
+        __nvgpu_log_dbg(g, gpu_dbg_dma,
+                        func, line,
+                        "DMA %s: [%s] size=%-7zu "
+                        "aligned=%-7zu total=%-10llukB %s",
+                        what, type,
+                        size, PAGE_ALIGN(size),
+                        g->dma_memory_used >> 10,
+                        flags_str);
+
+        if (flags_str)
+                nvgpu_kfree(g, flags_str);
+}
+
+#define dma_dbg_alloc(g, size, flags, type)                             \
+        __dma_dbg(g, size, flags, type, "alloc", __func__, __LINE__)
+#define dma_dbg_free(g, size, flags, type)                              \
+        __dma_dbg(g, size, flags, type, "free", __func__, __LINE__)
+
+/*
+ * For after the DMA alloc is done.
+ */
+#define __dma_dbg_done(g, size, type, what)                             \
+        nvgpu_log(g, gpu_dbg_dma,                                       \
+                  "DMA %s: [%s] size=%-7zu Done!",                      \
+                  what, type, size);                                    \
+
+#define dma_dbg_alloc_done(g, size, type)                               \
+        __dma_dbg_done(g, size, type, "alloc")
+#define dma_dbg_free_done(g, size, type)                                \
+        __dma_dbg_done(g, size, type, "free")
+
+#if defined(CONFIG_GK20A_VIDMEM)
+static u64 __nvgpu_dma_alloc(struct nvgpu_allocator *allocator, u64 at,
+                                size_t size)
+{
+        u64 addr = 0;
+
+        if (at)
+                addr = nvgpu_alloc_fixed(allocator, at, size, 0);
+        else
+                addr = nvgpu_alloc(allocator, size);
+
+        return addr;
+}
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)
+static void nvgpu_dma_flags_to_attrs(unsigned long *attrs,
+                unsigned long flags)
+#define ATTR_ARG(x) *x
+#else
+static void nvgpu_dma_flags_to_attrs(struct dma_attrs *attrs,
+                unsigned long flags)
+#define ATTR_ARG(x) x
+#endif
+{
+        if (flags & NVGPU_DMA_NO_KERNEL_MAPPING)
+                dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, ATTR_ARG(attrs));
+        if (flags & NVGPU_DMA_FORCE_CONTIGUOUS)
+                dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, ATTR_ARG(attrs));
+#undef ATTR_ARG
+}
+
+int nvgpu_dma_alloc_flags_sys(struct gk20a *g, unsigned long flags,
+                size_t size, struct nvgpu_mem *mem)
+{
+        struct device *d = dev_from_gk20a(g);
+        int err;
+        dma_addr_t iova;
+        NVGPU_DEFINE_DMA_ATTRS(dma_attrs);
+        void *alloc_ret;
+
+        if (nvgpu_mem_is_valid(mem)) {
+                nvgpu_warn(g, "memory leak !!");
+                WARN_ON(1);
+        }
+
+        /*
+         * WAR for IO coherent chips: the DMA API does not seem to generate
+         * mappings that work correctly. Unclear why - Bug ID: 2040115.
+         *
+         * Basically we just tell the DMA API not to map with NO_KERNEL_MAPPING
+         * and then make a vmap() ourselves.
+         */
+        if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM))
+                flags |= NVGPU_DMA_NO_KERNEL_MAPPING;
+
+        /*
+         * Before the debug print so we see this in the total. But during
+         * cleanup in the fail path this has to be subtracted.
+         */
+        g->dma_memory_used += PAGE_ALIGN(size);
+
+        dma_dbg_alloc(g, size, flags, "sysmem");
+
+        /*
+         * Save the old size but for actual allocation purposes the size is
+         * going to be page aligned.
+         */
+        mem->size = size;
+        size = PAGE_ALIGN(size);
+
+        nvgpu_dma_flags_to_attrs(&dma_attrs, flags);
+
+        alloc_ret = dma_alloc_attrs(d, size, &iova,
+                                    GFP_KERNEL|__GFP_ZERO,
+                                    NVGPU_DMA_ATTR(dma_attrs));
+        if (!alloc_ret)
+                return -ENOMEM;
+
+        if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) {
+                mem->priv.pages = alloc_ret;
+                err = nvgpu_get_sgtable_from_pages(g, &mem->priv.sgt,
+                                                   mem->priv.pages,
+                                                   iova, size);
+        } else {
+                mem->cpu_va = alloc_ret;
+                err = nvgpu_get_sgtable_attrs(g, &mem->priv.sgt, mem->cpu_va,
+                                        iova, size, flags);
+        }
+        if (err)
+                goto fail_free_dma;
+
+        if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) {
+                mem->cpu_va = vmap(mem->priv.pages,
+                                   size >> PAGE_SHIFT,
+                                   0, PAGE_KERNEL);
+                if (!mem->cpu_va) {
+                        err = -ENOMEM;
+                        goto fail_free_sgt;
+                }
+        }
+
+        mem->aligned_size = size;
+        mem->aperture = APERTURE_SYSMEM;
+        mem->priv.flags = flags;
+
+        dma_dbg_alloc_done(g, mem->size, "sysmem");
+
+        return 0;
+
+fail_free_sgt:
+        nvgpu_free_sgtable(g, &mem->priv.sgt);
+fail_free_dma:
+        dma_free_attrs(d, size, alloc_ret, iova, NVGPU_DMA_ATTR(dma_attrs));
+        mem->cpu_va = NULL;
+        mem->priv.sgt = NULL;
+        mem->size = 0;
+        g->dma_memory_used -= mem->aligned_size;
+        return err;
+}
+
+int nvgpu_dma_alloc_flags_vid_at(struct gk20a *g, unsigned long flags,
+                size_t size, struct nvgpu_mem *mem, u64 at)
+{
+#if defined(CONFIG_GK20A_VIDMEM)
+        u64 addr;
+        int err;
+        struct nvgpu_allocator *vidmem_alloc = g->mm.vidmem.cleared ?
+                &g->mm.vidmem.allocator :
+                &g->mm.vidmem.bootstrap_allocator;
+        u64 before_pending;
+
+        if (nvgpu_mem_is_valid(mem)) {
+                nvgpu_warn(g, "memory leak !!");
+                WARN_ON(1);
+        }
+
+        dma_dbg_alloc(g, size, flags, "vidmem");
+
+        mem->size = size;
+        size = PAGE_ALIGN(size);
+
+        if (!nvgpu_alloc_initialized(&g->mm.vidmem.allocator))
+                return -ENOSYS;
+
+        /*
+         * Our own allocator doesn't have any flags yet, and we can't
+         * kernel-map these, so require explicit flags.
+         */
+        WARN_ON(flags != NVGPU_DMA_NO_KERNEL_MAPPING);
+
+        nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex);
+        before_pending = atomic64_read(&g->mm.vidmem.bytes_pending.atomic_var);
+        addr = __nvgpu_dma_alloc(vidmem_alloc, at, size);
+        nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex);
+        if (!addr) {
+                /*
+                 * If memory is known to be freed soon, let the user know that
+                 * it may be available after a while.
+                 */
+                if (before_pending)
+                        return -EAGAIN;
+                else
+                        return -ENOMEM;
+        }
+
+        if (at)
+                mem->mem_flags |= NVGPU_MEM_FLAG_FIXED;
+
+        mem->priv.sgt = nvgpu_kzalloc(g, sizeof(struct sg_table));
+        if (!mem->priv.sgt) {
+                err = -ENOMEM;
+                goto fail_physfree;
+        }
+
+        err = sg_alloc_table(mem->priv.sgt, 1, GFP_KERNEL);
+        if (err)
+                goto fail_kfree;
+
+        nvgpu_vidmem_set_page_alloc(mem->priv.sgt->sgl, addr);
+        sg_set_page(mem->priv.sgt->sgl, NULL, size, 0);
+
+        mem->aligned_size = size;
+        mem->aperture = APERTURE_VIDMEM;
+        mem->vidmem_alloc = (struct nvgpu_page_alloc *)(uintptr_t)addr;
+        mem->allocator = vidmem_alloc;
+        mem->priv.flags = flags;
+
+        nvgpu_init_list_node(&mem->clear_list_entry);
+
+        dma_dbg_alloc_done(g, mem->size, "vidmem");
+
+        return 0;
+
+fail_kfree:
+        nvgpu_kfree(g, mem->priv.sgt);
+fail_physfree:
+        nvgpu_free(&g->mm.vidmem.allocator, addr);
+        mem->size = 0;
+        return err;
+#else
+        return -ENOSYS;
+#endif
+}
+
+void nvgpu_dma_free_sys(struct gk20a *g, struct nvgpu_mem *mem)
+{
+        struct device *d = dev_from_gk20a(g);
+
+        g->dma_memory_used -= mem->aligned_size;
+
+        dma_dbg_free(g, mem->size, mem->priv.flags, "sysmem");
+
+        if (!(mem->mem_flags & NVGPU_MEM_FLAG_SHADOW_COPY) &&
+            !(mem->mem_flags & __NVGPU_MEM_FLAG_NO_DMA) &&
+            (mem->cpu_va || mem->priv.pages)) {
+                /*
+                 * Free side of WAR for bug 2040115.
+                 */
+                if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM))
+                        vunmap(mem->cpu_va);
+
+                if (mem->priv.flags) {
+                        NVGPU_DEFINE_DMA_ATTRS(dma_attrs);
+
+                        nvgpu_dma_flags_to_attrs(&dma_attrs, mem->priv.flags);
+
+                        if (mem->priv.flags & NVGPU_DMA_NO_KERNEL_MAPPING) {
+                                dma_free_attrs(d, mem->aligned_size, mem->priv.pages,
+                                        sg_dma_address(mem->priv.sgt->sgl),
+                                        NVGPU_DMA_ATTR(dma_attrs));
+                        } else {
+                                dma_free_attrs(d, mem->aligned_size, mem->cpu_va,
+                                        sg_dma_address(mem->priv.sgt->sgl),
+                                        NVGPU_DMA_ATTR(dma_attrs));
+                        }
+                } else {
+                        dma_free_coherent(d, mem->aligned_size, mem->cpu_va,
+                                        sg_dma_address(mem->priv.sgt->sgl));
+                }
+                mem->cpu_va = NULL;
+                mem->priv.pages = NULL;
+        }
+
+        /*
+         * When this flag is set we expect that pages is still populated but not
+         * by the DMA API.
+         */
+        if (mem->mem_flags & __NVGPU_MEM_FLAG_NO_DMA)
+                nvgpu_kfree(g, mem->priv.pages);
+
+        if ((mem->mem_flags & NVGPU_MEM_FLAG_FOREIGN_SGT) == 0 &&
+                        mem->priv.sgt != NULL) {
+                nvgpu_free_sgtable(g, &mem->priv.sgt);
+        }
+
+        dma_dbg_free_done(g, mem->size, "sysmem");
+
+        mem->size = 0;
+        mem->aligned_size = 0;
+        mem->aperture = APERTURE_INVALID;
+}
+
+void nvgpu_dma_free_vid(struct gk20a *g, struct nvgpu_mem *mem)
+{
+#if defined(CONFIG_GK20A_VIDMEM)
+        size_t mem_size = mem->size;
+
+        dma_dbg_free(g, mem->size, mem->priv.flags, "vidmem");
+
+        /* Sanity check - only this supported when allocating. */
+        WARN_ON(mem->priv.flags != NVGPU_DMA_NO_KERNEL_MAPPING);
+
+        if (mem->mem_flags & NVGPU_MEM_FLAG_USER_MEM) {
+                int err = nvgpu_vidmem_clear_list_enqueue(g, mem);
+
+                /*
+                 * If there's an error here then that means we can't clear the
+                 * vidmem. That's too bad; however, we still own the nvgpu_mem
+                 * buf so we have to free that.
+                 *
+                 * We don't need to worry about the vidmem allocator itself
+                 * since when that gets cleaned up in the driver shutdown path
+                 * all the outstanding allocs are force freed.
+                 */
+                if (err)
+                        nvgpu_kfree(g, mem);
+        } else {
+                nvgpu_memset(g, mem, 0, 0, mem->aligned_size);
+                nvgpu_free(mem->allocator,
+                           (u64)nvgpu_vidmem_get_page_alloc(mem->priv.sgt->sgl));
+                nvgpu_free_sgtable(g, &mem->priv.sgt);
+
+                mem->size = 0;
+                mem->aligned_size = 0;
+                mem->aperture = APERTURE_INVALID;
+        }
+
+        dma_dbg_free_done(g, mem_size, "vidmem");
+#endif
+}
+
+int nvgpu_get_sgtable_attrs(struct gk20a *g, struct sg_table **sgt,
+                      void *cpuva, u64 iova, size_t size, unsigned long flags)
+{
+        int err = 0;
+        struct sg_table *tbl;
+        NVGPU_DEFINE_DMA_ATTRS(dma_attrs);
+
+        tbl = nvgpu_kzalloc(g, sizeof(struct sg_table));
+        if (!tbl) {
+                err = -ENOMEM;
+                goto fail;
+        }
+
+        nvgpu_dma_flags_to_attrs(&dma_attrs, flags);
+        err = dma_get_sgtable_attrs(dev_from_gk20a(g), tbl, cpuva, iova,
+                                        size, NVGPU_DMA_ATTR(dma_attrs));
+        if (err)
+                goto fail;
+
+        sg_dma_address(tbl->sgl) = iova;
+        *sgt = tbl;
+
+        return 0;
+
+fail:
+        if (tbl)
+                nvgpu_kfree(g, tbl);
+
+        return err;
+}
+
+int nvgpu_get_sgtable(struct gk20a *g, struct sg_table **sgt,
+                      void *cpuva, u64 iova, size_t size)
+{
+        return nvgpu_get_sgtable_attrs(g, sgt, cpuva, iova, size, 0);
+}
+
+int nvgpu_get_sgtable_from_pages(struct gk20a *g, struct sg_table **sgt,
+                                 struct page **pages, u64 iova, size_t size)
+{
+        int err = 0;
+        struct sg_table *tbl;
+
+        tbl = nvgpu_kzalloc(g, sizeof(struct sg_table));
+        if (!tbl) {
+                err = -ENOMEM;
+                goto fail;
+        }
+
+        err = sg_alloc_table_from_pages(tbl, pages,
+                                        DIV_ROUND_UP(size, PAGE_SIZE),
+                                        0, size, GFP_KERNEL);
+        if (err)
+                goto fail;
+
+        sg_dma_address(tbl->sgl) = iova;
+        *sgt = tbl;
+
+        return 0;
+
+fail:
+        if (tbl)
+                nvgpu_kfree(g, tbl);
+
+        return err;
+}
+
+void nvgpu_free_sgtable(struct gk20a *g, struct sg_table **sgt)
+{
+        sg_free_table(*sgt);
+        nvgpu_kfree(g, *sgt);
+        *sgt = NULL;
+}
+
+bool nvgpu_iommuable(struct gk20a *g)
+{
+#ifdef CONFIG_TEGRA_GK20A
+        struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+
+        /*
+         * Check against the nvgpu device to see if it's been marked as
+         * IOMMU'able.
+         */
+        if (!device_is_iommuable(l->dev))
+                return false;
+#endif
+
+        return true;
+}