From 01e6fac4d61fdd7fff5433942ec93fc2ea1e4df1 Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Wed, 28 Jun 2023 18:24:25 -0400 Subject: Include nvgpu headers These are needed to build on NVIDIA's Jetson boards for the time being. Only a couple structs are required, so it should be fairly easy to remove this dependency at some point in the future. --- include/os/linux/cde.c | 1794 ++++++++++++++++ include/os/linux/cde.h | 326 +++ include/os/linux/cde_gm20b.c | 59 + include/os/linux/cde_gm20b.h | 33 + include/os/linux/cde_gp10b.c | 153 ++ include/os/linux/cde_gp10b.h | 40 + include/os/linux/channel.h | 102 + include/os/linux/clk.c | 286 +++ include/os/linux/clk.h | 22 + include/os/linux/comptags.c | 140 ++ include/os/linux/cond.c | 73 + include/os/linux/ctxsw_trace.c | 792 +++++++ include/os/linux/ctxsw_trace.h | 39 + include/os/linux/debug.c | 457 ++++ include/os/linux/debug_allocator.c | 69 + include/os/linux/debug_allocator.h | 21 + include/os/linux/debug_bios.c | 60 + include/os/linux/debug_bios.h | 21 + include/os/linux/debug_cde.c | 53 + include/os/linux/debug_cde.h | 21 + include/os/linux/debug_ce.c | 30 + include/os/linux/debug_ce.h | 21 + include/os/linux/debug_clk_gm20b.c | 280 +++ include/os/linux/debug_clk_gm20b.h | 29 + include/os/linux/debug_clk_gp106.c | 193 ++ include/os/linux/debug_clk_gp106.h | 29 + include/os/linux/debug_clk_gv100.c | 193 ++ include/os/linux/debug_clk_gv100.h | 29 + include/os/linux/debug_fecs_trace.c | 151 ++ include/os/linux/debug_fecs_trace.h | 30 + include/os/linux/debug_fifo.c | 376 ++++ include/os/linux/debug_fifo.h | 22 + include/os/linux/debug_gr.c | 31 + include/os/linux/debug_gr.h | 21 + include/os/linux/debug_hal.c | 95 + include/os/linux/debug_hal.h | 22 + include/os/linux/debug_kmem.c | 312 +++ include/os/linux/debug_kmem.h | 23 + include/os/linux/debug_ltc.c | 94 + include/os/linux/debug_ltc.h | 21 + include/os/linux/debug_pmgr.c | 104 + include/os/linux/debug_pmgr.h | 28 + include/os/linux/debug_pmu.c | 484 +++++ include/os/linux/debug_pmu.h | 21 + include/os/linux/debug_sched.c | 79 + include/os/linux/debug_sched.h | 21 + include/os/linux/debug_therm_gp106.c | 49 + include/os/linux/debug_therm_gp106.h | 29 + include/os/linux/debug_xve.c | 177 ++ include/os/linux/debug_xve.h | 21 + include/os/linux/dmabuf.c | 219 ++ include/os/linux/dmabuf.h | 62 + include/os/linux/dmabuf_vidmem.c | 269 +++ include/os/linux/dmabuf_vidmem.h | 78 + include/os/linux/driver_common.c | 351 ++++ include/os/linux/driver_common.h | 22 + include/os/linux/dt.c | 29 + include/os/linux/ecc_sysfs.c | 80 + include/os/linux/firmware.c | 117 ++ include/os/linux/fuse.c | 55 + include/os/linux/intr.c | 136 ++ include/os/linux/intr.h | 22 + include/os/linux/io.c | 130 ++ include/os/linux/io_usermode.c | 29 + include/os/linux/ioctl.c | 297 +++ include/os/linux/ioctl.h | 23 + include/os/linux/ioctl_as.c | 427 ++++ include/os/linux/ioctl_as.h | 30 + include/os/linux/ioctl_channel.c | 1388 ++++++++++++ include/os/linux/ioctl_channel.h | 57 + include/os/linux/ioctl_clk_arb.c | 574 +++++ include/os/linux/ioctl_ctrl.c | 2106 +++++++++++++++++++ include/os/linux/ioctl_ctrl.h | 27 + include/os/linux/ioctl_dbg.c | 2210 ++++++++++++++++++++ include/os/linux/ioctl_dbg.h | 38 + include/os/linux/ioctl_tsg.c | 750 +++++++ include/os/linux/ioctl_tsg.h | 28 + include/os/linux/kmem.c | 653 ++++++ include/os/linux/kmem_priv.h | 105 + include/os/linux/linux-channel.c | 657 ++++++ include/os/linux/linux-dma.c | 534 +++++ include/os/linux/log.c | 132 ++ include/os/linux/ltc.c | 60 + include/os/linux/module.c | 1529 ++++++++++++++ include/os/linux/module.h | 35 + include/os/linux/module_usermode.c | 62 + include/os/linux/module_usermode.h | 27 + include/os/linux/nvgpu_mem.c | 348 +++ include/os/linux/nvhost.c | 295 +++ include/os/linux/nvhost_priv.h | 24 + include/os/linux/nvidia_p2p.c | 299 +++ include/os/linux/nvlink.c | 132 ++ include/os/linux/nvlink.h | 22 + include/os/linux/os_fence_android.c | 79 + include/os/linux/os_fence_android_sema.c | 112 + include/os/linux/os_fence_android_syncpt.c | 121 ++ include/os/linux/os_linux.h | 187 ++ include/os/linux/os_ops.c | 61 + include/os/linux/os_ops.h | 22 + include/os/linux/os_ops_gm20b.c | 47 + include/os/linux/os_ops_gm20b.h | 22 + include/os/linux/os_ops_gp106.c | 40 + include/os/linux/os_ops_gp106.h | 22 + include/os/linux/os_ops_gp10b.c | 41 + include/os/linux/os_ops_gp10b.h | 22 + include/os/linux/os_ops_gv100.c | 40 + include/os/linux/os_ops_gv100.h | 22 + include/os/linux/os_ops_gv11b.c | 30 + include/os/linux/os_ops_gv11b.h | 24 + include/os/linux/os_sched.c | 32 + include/os/linux/pci.c | 854 ++++++++ include/os/linux/pci.h | 27 + include/os/linux/pci_usermode.c | 24 + include/os/linux/pci_usermode.h | 23 + include/os/linux/platform_gk20a.h | 329 +++ include/os/linux/platform_gk20a_tegra.c | 966 +++++++++ include/os/linux/platform_gk20a_tegra.h | 23 + include/os/linux/platform_gp10b.h | 39 + include/os/linux/platform_gp10b_tegra.c | 510 +++++ include/os/linux/platform_gp10b_tegra.h | 22 + include/os/linux/platform_gv11b_tegra.c | 331 +++ include/os/linux/rwsem.c | 39 + include/os/linux/scale.c | 435 ++++ include/os/linux/scale.h | 66 + include/os/linux/sched.c | 666 ++++++ include/os/linux/sched.h | 36 + include/os/linux/sim.c | 96 + include/os/linux/sim_pci.c | 93 + include/os/linux/soc.c | 122 ++ include/os/linux/sync_sema_android.c | 418 ++++ include/os/linux/sync_sema_android.h | 51 + include/os/linux/sysfs.c | 1275 +++++++++++ include/os/linux/sysfs.h | 24 + include/os/linux/thread.c | 70 + include/os/linux/timers.c | 269 +++ include/os/linux/vgpu/fecs_trace_vgpu.c | 225 ++ .../linux/vgpu/gv11b/platform_gv11b_vgpu_tegra.c | 103 + include/os/linux/vgpu/platform_vgpu_tegra.c | 97 + include/os/linux/vgpu/platform_vgpu_tegra.h | 24 + include/os/linux/vgpu/sysfs_vgpu.c | 143 ++ include/os/linux/vgpu/vgpu_ivc.c | 77 + include/os/linux/vgpu/vgpu_ivm.c | 53 + include/os/linux/vgpu/vgpu_linux.c | 525 +++++ include/os/linux/vgpu/vgpu_linux.h | 68 + include/os/linux/vm.c | 356 ++++ include/os/linux/vpr.c | 22 + 146 files changed, 31469 insertions(+) create mode 100644 include/os/linux/cde.c create mode 100644 include/os/linux/cde.h create mode 100644 include/os/linux/cde_gm20b.c create mode 100644 include/os/linux/cde_gm20b.h create mode 100644 include/os/linux/cde_gp10b.c create mode 100644 include/os/linux/cde_gp10b.h create mode 100644 include/os/linux/channel.h create mode 100644 include/os/linux/clk.c create mode 100644 include/os/linux/clk.h create mode 100644 include/os/linux/comptags.c create mode 100644 include/os/linux/cond.c create mode 100644 include/os/linux/ctxsw_trace.c create mode 100644 include/os/linux/ctxsw_trace.h create mode 100644 include/os/linux/debug.c create mode 100644 include/os/linux/debug_allocator.c create mode 100644 include/os/linux/debug_allocator.h create mode 100644 include/os/linux/debug_bios.c create mode 100644 include/os/linux/debug_bios.h create mode 100644 include/os/linux/debug_cde.c create mode 100644 include/os/linux/debug_cde.h create mode 100644 include/os/linux/debug_ce.c create mode 100644 include/os/linux/debug_ce.h create mode 100644 include/os/linux/debug_clk_gm20b.c create mode 100644 include/os/linux/debug_clk_gm20b.h create mode 100644 include/os/linux/debug_clk_gp106.c create mode 100644 include/os/linux/debug_clk_gp106.h create mode 100644 include/os/linux/debug_clk_gv100.c create mode 100644 include/os/linux/debug_clk_gv100.h create mode 100644 include/os/linux/debug_fecs_trace.c create mode 100644 include/os/linux/debug_fecs_trace.h create mode 100644 include/os/linux/debug_fifo.c create mode 100644 include/os/linux/debug_fifo.h create mode 100644 include/os/linux/debug_gr.c create mode 100644 include/os/linux/debug_gr.h create mode 100644 include/os/linux/debug_hal.c create mode 100644 include/os/linux/debug_hal.h create mode 100644 include/os/linux/debug_kmem.c create mode 100644 include/os/linux/debug_kmem.h create mode 100644 include/os/linux/debug_ltc.c create mode 100644 include/os/linux/debug_ltc.h create mode 100644 include/os/linux/debug_pmgr.c create mode 100644 include/os/linux/debug_pmgr.h create mode 100644 include/os/linux/debug_pmu.c create mode 100644 include/os/linux/debug_pmu.h create mode 100644 include/os/linux/debug_sched.c create mode 100644 include/os/linux/debug_sched.h create mode 100644 include/os/linux/debug_therm_gp106.c create mode 100644 include/os/linux/debug_therm_gp106.h create mode 100644 include/os/linux/debug_xve.c create mode 100644 include/os/linux/debug_xve.h create mode 100644 include/os/linux/dmabuf.c create mode 100644 include/os/linux/dmabuf.h create mode 100644 include/os/linux/dmabuf_vidmem.c create mode 100644 include/os/linux/dmabuf_vidmem.h create mode 100644 include/os/linux/driver_common.c create mode 100644 include/os/linux/driver_common.h create mode 100644 include/os/linux/dt.c create mode 100644 include/os/linux/ecc_sysfs.c create mode 100644 include/os/linux/firmware.c create mode 100644 include/os/linux/fuse.c create mode 100644 include/os/linux/intr.c create mode 100644 include/os/linux/intr.h create mode 100644 include/os/linux/io.c create mode 100644 include/os/linux/io_usermode.c create mode 100644 include/os/linux/ioctl.c create mode 100644 include/os/linux/ioctl.h create mode 100644 include/os/linux/ioctl_as.c create mode 100644 include/os/linux/ioctl_as.h create mode 100644 include/os/linux/ioctl_channel.c create mode 100644 include/os/linux/ioctl_channel.h create mode 100644 include/os/linux/ioctl_clk_arb.c create mode 100644 include/os/linux/ioctl_ctrl.c create mode 100644 include/os/linux/ioctl_ctrl.h create mode 100644 include/os/linux/ioctl_dbg.c create mode 100644 include/os/linux/ioctl_dbg.h create mode 100644 include/os/linux/ioctl_tsg.c create mode 100644 include/os/linux/ioctl_tsg.h create mode 100644 include/os/linux/kmem.c create mode 100644 include/os/linux/kmem_priv.h create mode 100644 include/os/linux/linux-channel.c create mode 100644 include/os/linux/linux-dma.c create mode 100644 include/os/linux/log.c create mode 100644 include/os/linux/ltc.c create mode 100644 include/os/linux/module.c create mode 100644 include/os/linux/module.h create mode 100644 include/os/linux/module_usermode.c create mode 100644 include/os/linux/module_usermode.h create mode 100644 include/os/linux/nvgpu_mem.c create mode 100644 include/os/linux/nvhost.c create mode 100644 include/os/linux/nvhost_priv.h create mode 100644 include/os/linux/nvidia_p2p.c create mode 100644 include/os/linux/nvlink.c create mode 100644 include/os/linux/nvlink.h create mode 100644 include/os/linux/os_fence_android.c create mode 100644 include/os/linux/os_fence_android_sema.c create mode 100644 include/os/linux/os_fence_android_syncpt.c create mode 100644 include/os/linux/os_linux.h create mode 100644 include/os/linux/os_ops.c create mode 100644 include/os/linux/os_ops.h create mode 100644 include/os/linux/os_ops_gm20b.c create mode 100644 include/os/linux/os_ops_gm20b.h create mode 100644 include/os/linux/os_ops_gp106.c create mode 100644 include/os/linux/os_ops_gp106.h create mode 100644 include/os/linux/os_ops_gp10b.c create mode 100644 include/os/linux/os_ops_gp10b.h create mode 100644 include/os/linux/os_ops_gv100.c create mode 100644 include/os/linux/os_ops_gv100.h create mode 100644 include/os/linux/os_ops_gv11b.c create mode 100644 include/os/linux/os_ops_gv11b.h create mode 100644 include/os/linux/os_sched.c create mode 100644 include/os/linux/pci.c create mode 100644 include/os/linux/pci.h create mode 100644 include/os/linux/pci_usermode.c create mode 100644 include/os/linux/pci_usermode.h create mode 100644 include/os/linux/platform_gk20a.h create mode 100644 include/os/linux/platform_gk20a_tegra.c create mode 100644 include/os/linux/platform_gk20a_tegra.h create mode 100644 include/os/linux/platform_gp10b.h create mode 100644 include/os/linux/platform_gp10b_tegra.c create mode 100644 include/os/linux/platform_gp10b_tegra.h create mode 100644 include/os/linux/platform_gv11b_tegra.c create mode 100644 include/os/linux/rwsem.c create mode 100644 include/os/linux/scale.c create mode 100644 include/os/linux/scale.h create mode 100644 include/os/linux/sched.c create mode 100644 include/os/linux/sched.h create mode 100644 include/os/linux/sim.c create mode 100644 include/os/linux/sim_pci.c create mode 100644 include/os/linux/soc.c create mode 100644 include/os/linux/sync_sema_android.c create mode 100644 include/os/linux/sync_sema_android.h create mode 100644 include/os/linux/sysfs.c create mode 100644 include/os/linux/sysfs.h create mode 100644 include/os/linux/thread.c create mode 100644 include/os/linux/timers.c create mode 100644 include/os/linux/vgpu/fecs_trace_vgpu.c create mode 100644 include/os/linux/vgpu/gv11b/platform_gv11b_vgpu_tegra.c create mode 100644 include/os/linux/vgpu/platform_vgpu_tegra.c create mode 100644 include/os/linux/vgpu/platform_vgpu_tegra.h create mode 100644 include/os/linux/vgpu/sysfs_vgpu.c create mode 100644 include/os/linux/vgpu/vgpu_ivc.c create mode 100644 include/os/linux/vgpu/vgpu_ivm.c create mode 100644 include/os/linux/vgpu/vgpu_linux.c create mode 100644 include/os/linux/vgpu/vgpu_linux.h create mode 100644 include/os/linux/vm.c create mode 100644 include/os/linux/vpr.c (limited to 'include/os/linux') diff --git a/include/os/linux/cde.c b/include/os/linux/cde.c new file mode 100644 index 0000000..715513c --- /dev/null +++ b/include/os/linux/cde.c @@ -0,0 +1,1794 @@ +/* + * Color decompression engine support + * + * Copyright (c) 2014-2018, NVIDIA Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "gk20a/mm_gk20a.h" +#include "gk20a/fence_gk20a.h" +#include "gk20a/gr_gk20a.h" + +#include "cde.h" +#include "os_linux.h" +#include "dmabuf.h" +#include "channel.h" +#include "cde_gm20b.h" +#include "cde_gp10b.h" + +#include +#include + +static int gk20a_cde_load(struct gk20a_cde_ctx *cde_ctx); +static struct gk20a_cde_ctx *gk20a_cde_allocate_context(struct nvgpu_os_linux *l); + +#define CTX_DELETE_TIME 1000 + +#define MAX_CTX_USE_COUNT 42 +#define MAX_CTX_RETRY_TIME 2000 + +static dma_addr_t gpuva_to_iova_base(struct vm_gk20a *vm, u64 gpu_vaddr) +{ + struct nvgpu_mapped_buf *buffer; + dma_addr_t addr = 0; + struct gk20a *g = gk20a_from_vm(vm); + + nvgpu_mutex_acquire(&vm->update_gmmu_lock); + buffer = __nvgpu_vm_find_mapped_buf(vm, gpu_vaddr); + if (buffer) + addr = nvgpu_mem_get_addr_sgl(g, buffer->os_priv.sgt->sgl); + nvgpu_mutex_release(&vm->update_gmmu_lock); + + return addr; +} + +static void gk20a_deinit_cde_img(struct gk20a_cde_ctx *cde_ctx) +{ + unsigned int i; + + for (i = 0; i < cde_ctx->num_bufs; i++) { + struct nvgpu_mem *mem = cde_ctx->mem + i; + nvgpu_dma_unmap_free(cde_ctx->vm, mem); + } + + nvgpu_kfree(&cde_ctx->l->g, cde_ctx->init_convert_cmd); + + cde_ctx->convert_cmd = NULL; + cde_ctx->init_convert_cmd = NULL; + cde_ctx->num_bufs = 0; + cde_ctx->num_params = 0; + cde_ctx->init_cmd_num_entries = 0; + cde_ctx->convert_cmd_num_entries = 0; + cde_ctx->init_cmd_executed = false; +} + +static void gk20a_cde_remove_ctx(struct gk20a_cde_ctx *cde_ctx) +__must_hold(&cde_app->mutex) +{ + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + struct channel_gk20a *ch = cde_ctx->ch; + struct vm_gk20a *vm = ch->vm; + + trace_gk20a_cde_remove_ctx(cde_ctx); + + /* release mapped memory */ + gk20a_deinit_cde_img(cde_ctx); + nvgpu_gmmu_unmap(vm, &g->gr.compbit_store.mem, + cde_ctx->backing_store_vaddr); + + /* + * free the channel + * gk20a_channel_close() will also unbind the channel from TSG + */ + gk20a_channel_close(ch); + nvgpu_ref_put(&cde_ctx->tsg->refcount, gk20a_tsg_release); + + /* housekeeping on app */ + nvgpu_list_del(&cde_ctx->list); + l->cde_app.ctx_count--; + nvgpu_kfree(g, cde_ctx); +} + +static void gk20a_cde_cancel_deleter(struct gk20a_cde_ctx *cde_ctx, + bool wait_finish) +__releases(&cde_app->mutex) +__acquires(&cde_app->mutex) +{ + struct gk20a_cde_app *cde_app = &cde_ctx->l->cde_app; + + /* permanent contexts do not have deleter works */ + if (!cde_ctx->is_temporary) + return; + + if (wait_finish) { + nvgpu_mutex_release(&cde_app->mutex); + cancel_delayed_work_sync(&cde_ctx->ctx_deleter_work); + nvgpu_mutex_acquire(&cde_app->mutex); + } else { + cancel_delayed_work(&cde_ctx->ctx_deleter_work); + } +} + +static void gk20a_cde_remove_contexts(struct nvgpu_os_linux *l) +__must_hold(&l->cde_app->mutex) +{ + struct gk20a_cde_app *cde_app = &l->cde_app; + struct gk20a_cde_ctx *cde_ctx, *cde_ctx_save; + + /* safe to go off the mutex in cancel_deleter since app is + * deinitialised; no new jobs are started. deleter works may be only at + * waiting for the mutex or before, going to abort */ + + nvgpu_list_for_each_entry_safe(cde_ctx, cde_ctx_save, + &cde_app->free_contexts, gk20a_cde_ctx, list) { + gk20a_cde_cancel_deleter(cde_ctx, true); + gk20a_cde_remove_ctx(cde_ctx); + } + + nvgpu_list_for_each_entry_safe(cde_ctx, cde_ctx_save, + &cde_app->used_contexts, gk20a_cde_ctx, list) { + gk20a_cde_cancel_deleter(cde_ctx, true); + gk20a_cde_remove_ctx(cde_ctx); + } +} + +static void gk20a_cde_stop(struct nvgpu_os_linux *l) +__must_hold(&l->cde_app->mutex) +{ + struct gk20a_cde_app *cde_app = &l->cde_app; + + /* prevent further conversions and delayed works from working */ + cde_app->initialised = false; + /* free all data, empty the list */ + gk20a_cde_remove_contexts(l); +} + +void gk20a_cde_destroy(struct nvgpu_os_linux *l) +__acquires(&l->cde_app->mutex) +__releases(&l->cde_app->mutex) +{ + struct gk20a_cde_app *cde_app = &l->cde_app; + + if (!cde_app->initialised) + return; + + nvgpu_mutex_acquire(&cde_app->mutex); + gk20a_cde_stop(l); + nvgpu_mutex_release(&cde_app->mutex); + + nvgpu_mutex_destroy(&cde_app->mutex); +} + +void gk20a_cde_suspend(struct nvgpu_os_linux *l) +__acquires(&l->cde_app->mutex) +__releases(&l->cde_app->mutex) +{ + struct gk20a_cde_app *cde_app = &l->cde_app; + struct gk20a_cde_ctx *cde_ctx, *cde_ctx_save; + + if (!cde_app->initialised) + return; + + nvgpu_mutex_acquire(&cde_app->mutex); + + nvgpu_list_for_each_entry_safe(cde_ctx, cde_ctx_save, + &cde_app->free_contexts, gk20a_cde_ctx, list) { + gk20a_cde_cancel_deleter(cde_ctx, false); + } + + nvgpu_list_for_each_entry_safe(cde_ctx, cde_ctx_save, + &cde_app->used_contexts, gk20a_cde_ctx, list) { + gk20a_cde_cancel_deleter(cde_ctx, false); + } + + nvgpu_mutex_release(&cde_app->mutex); + +} + +static int gk20a_cde_create_context(struct nvgpu_os_linux *l) +__must_hold(&l->cde_app->mutex) +{ + struct gk20a_cde_app *cde_app = &l->cde_app; + struct gk20a_cde_ctx *cde_ctx; + + cde_ctx = gk20a_cde_allocate_context(l); + if (IS_ERR(cde_ctx)) + return PTR_ERR(cde_ctx); + + nvgpu_list_add(&cde_ctx->list, &cde_app->free_contexts); + cde_app->ctx_count++; + if (cde_app->ctx_count > cde_app->ctx_count_top) + cde_app->ctx_count_top = cde_app->ctx_count; + + return 0; +} + +static int gk20a_cde_create_contexts(struct nvgpu_os_linux *l) +__must_hold(&l->cde_app->mutex) +{ + int err; + int i; + + for (i = 0; i < NUM_CDE_CONTEXTS; i++) { + err = gk20a_cde_create_context(l); + if (err) + goto out; + } + + return 0; +out: + gk20a_cde_remove_contexts(l); + return err; +} + +static int gk20a_init_cde_buf(struct gk20a_cde_ctx *cde_ctx, + struct nvgpu_firmware *img, + struct gk20a_cde_hdr_buf *buf) +{ + struct nvgpu_mem *mem; + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + int err; + + /* check that the file can hold the buf */ + if (buf->data_byte_offset != 0 && + buf->data_byte_offset + buf->num_bytes > img->size) { + nvgpu_warn(g, "cde: invalid data section. buffer idx = %d", + cde_ctx->num_bufs); + return -EINVAL; + } + + /* check that we have enough buf elems available */ + if (cde_ctx->num_bufs >= MAX_CDE_BUFS) { + nvgpu_warn(g, "cde: invalid data section. buffer idx = %d", + cde_ctx->num_bufs); + return -ENOMEM; + } + + /* allocate buf */ + mem = cde_ctx->mem + cde_ctx->num_bufs; + err = nvgpu_dma_alloc_map_sys(cde_ctx->vm, buf->num_bytes, mem); + if (err) { + nvgpu_warn(g, "cde: could not allocate device memory. buffer idx = %d", + cde_ctx->num_bufs); + return -ENOMEM; + } + + /* copy the content */ + if (buf->data_byte_offset != 0) + memcpy(mem->cpu_va, img->data + buf->data_byte_offset, + buf->num_bytes); + + cde_ctx->num_bufs++; + + return 0; +} + +static int gk20a_replace_data(struct gk20a_cde_ctx *cde_ctx, void *target, + int type, s32 shift, u64 mask, u64 value) +{ + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + u32 *target_mem_ptr = target; + u64 *target_mem_ptr_u64 = target; + u64 current_value, new_value; + + value = (shift >= 0) ? value << shift : value >> -shift; + value &= mask; + + /* read current data from the location */ + current_value = 0; + if (type == TYPE_PARAM_TYPE_U32) { + if (mask != 0xfffffffful) + current_value = *target_mem_ptr; + } else if (type == TYPE_PARAM_TYPE_U64_LITTLE) { + if (mask != ~0ul) + current_value = *target_mem_ptr_u64; + } else if (type == TYPE_PARAM_TYPE_U64_BIG) { + current_value = *target_mem_ptr_u64; + current_value = (u64)(current_value >> 32) | + (u64)(current_value << 32); + } else { + nvgpu_warn(g, "cde: unknown type. type=%d", + type); + return -EINVAL; + } + + current_value &= ~mask; + new_value = current_value | value; + + /* store the element data back */ + if (type == TYPE_PARAM_TYPE_U32) + *target_mem_ptr = (u32)new_value; + else if (type == TYPE_PARAM_TYPE_U64_LITTLE) + *target_mem_ptr_u64 = new_value; + else { + new_value = (u64)(new_value >> 32) | + (u64)(new_value << 32); + *target_mem_ptr_u64 = new_value; + } + + return 0; +} + +static int gk20a_init_cde_replace(struct gk20a_cde_ctx *cde_ctx, + struct nvgpu_firmware *img, + struct gk20a_cde_hdr_replace *replace) +{ + struct nvgpu_mem *source_mem; + struct nvgpu_mem *target_mem; + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + u32 *target_mem_ptr; + u64 vaddr; + int err; + + if (replace->target_buf >= cde_ctx->num_bufs || + replace->source_buf >= cde_ctx->num_bufs) { + nvgpu_warn(g, "cde: invalid buffer. target_buf=%u, source_buf=%u, num_bufs=%d", + replace->target_buf, replace->source_buf, + cde_ctx->num_bufs); + return -EINVAL; + } + + source_mem = cde_ctx->mem + replace->source_buf; + target_mem = cde_ctx->mem + replace->target_buf; + target_mem_ptr = target_mem->cpu_va; + + if (source_mem->size < (replace->source_byte_offset + 3) || + target_mem->size < (replace->target_byte_offset + 3)) { + nvgpu_warn(g, "cde: invalid buffer offsets. target_buf_offs=%lld, source_buf_offs=%lld, source_buf_size=%zu, dest_buf_size=%zu", + replace->target_byte_offset, + replace->source_byte_offset, + source_mem->size, + target_mem->size); + return -EINVAL; + } + + /* calculate the target pointer */ + target_mem_ptr += (replace->target_byte_offset / sizeof(u32)); + + /* determine patch value */ + vaddr = source_mem->gpu_va + replace->source_byte_offset; + err = gk20a_replace_data(cde_ctx, target_mem_ptr, replace->type, + replace->shift, replace->mask, + vaddr); + if (err) { + nvgpu_warn(g, "cde: replace failed. err=%d, target_buf=%u, target_buf_offs=%lld, source_buf=%u, source_buf_offs=%lld", + err, replace->target_buf, + replace->target_byte_offset, + replace->source_buf, + replace->source_byte_offset); + } + + return err; +} + +static int gk20a_cde_patch_params(struct gk20a_cde_ctx *cde_ctx) +{ + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + struct nvgpu_mem *target_mem; + u32 *target_mem_ptr; + u64 new_data; + int user_id = 0, err; + unsigned int i; + + for (i = 0; i < cde_ctx->num_params; i++) { + struct gk20a_cde_hdr_param *param = cde_ctx->params + i; + target_mem = cde_ctx->mem + param->target_buf; + target_mem_ptr = target_mem->cpu_va; + target_mem_ptr += (param->target_byte_offset / sizeof(u32)); + + switch (param->id) { + case TYPE_PARAM_COMPTAGS_PER_CACHELINE: + new_data = g->gr.comptags_per_cacheline; + break; + case TYPE_PARAM_GPU_CONFIGURATION: + new_data = (u64)g->ltc_count * g->gr.slices_per_ltc * + g->gr.cacheline_size; + break; + case TYPE_PARAM_FIRSTPAGEOFFSET: + new_data = cde_ctx->surf_param_offset; + break; + case TYPE_PARAM_NUMPAGES: + new_data = cde_ctx->surf_param_lines; + break; + case TYPE_PARAM_BACKINGSTORE: + new_data = cde_ctx->backing_store_vaddr; + break; + case TYPE_PARAM_DESTINATION: + new_data = cde_ctx->compbit_vaddr; + break; + case TYPE_PARAM_DESTINATION_SIZE: + new_data = cde_ctx->compbit_size; + break; + case TYPE_PARAM_BACKINGSTORE_SIZE: + new_data = g->gr.compbit_store.mem.size; + break; + case TYPE_PARAM_SOURCE_SMMU_ADDR: + new_data = gpuva_to_iova_base(cde_ctx->vm, + cde_ctx->surf_vaddr); + if (new_data == 0) { + nvgpu_warn(g, "cde: failed to find 0x%llx", + cde_ctx->surf_vaddr); + return -EINVAL; + } + break; + case TYPE_PARAM_BACKINGSTORE_BASE_HW: + new_data = g->gr.compbit_store.base_hw; + break; + case TYPE_PARAM_GOBS_PER_COMPTAGLINE_PER_SLICE: + new_data = g->gr.gobs_per_comptagline_per_slice; + break; + case TYPE_PARAM_SCATTERBUFFER: + new_data = cde_ctx->scatterbuffer_vaddr; + break; + case TYPE_PARAM_SCATTERBUFFER_SIZE: + new_data = cde_ctx->scatterbuffer_size; + break; + default: + user_id = param->id - NUM_RESERVED_PARAMS; + if (user_id < 0 || user_id >= MAX_CDE_USER_PARAMS) + continue; + new_data = cde_ctx->user_param_values[user_id]; + } + + nvgpu_log(g, gpu_dbg_cde, "cde: patch: idx_in_file=%d param_id=%d target_buf=%u target_byte_offset=%lld data_value=0x%llx data_offset/data_diff=%lld data_type=%d data_shift=%d data_mask=0x%llx", + i, param->id, param->target_buf, + param->target_byte_offset, new_data, + param->data_offset, param->type, param->shift, + param->mask); + + new_data += param->data_offset; + + err = gk20a_replace_data(cde_ctx, target_mem_ptr, param->type, + param->shift, param->mask, new_data); + + if (err) { + nvgpu_warn(g, "cde: patch failed. err=%d, idx=%d, id=%d, target_buf=%u, target_buf_offs=%lld, patch_value=%llu", + err, i, param->id, param->target_buf, + param->target_byte_offset, new_data); + return err; + } + } + + return 0; +} + +static int gk20a_init_cde_param(struct gk20a_cde_ctx *cde_ctx, + struct nvgpu_firmware *img, + struct gk20a_cde_hdr_param *param) +{ + struct nvgpu_mem *target_mem; + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + + if (param->target_buf >= cde_ctx->num_bufs) { + nvgpu_warn(g, "cde: invalid buffer parameter. param idx = %d, target_buf=%u, num_bufs=%u", + cde_ctx->num_params, param->target_buf, + cde_ctx->num_bufs); + return -EINVAL; + } + + target_mem = cde_ctx->mem + param->target_buf; + if (target_mem->size < (param->target_byte_offset + 3)) { + nvgpu_warn(g, "cde: invalid buffer parameter. param idx = %d, target_buf_offs=%lld, target_buf_size=%zu", + cde_ctx->num_params, param->target_byte_offset, + target_mem->size); + return -EINVAL; + } + + /* does this parameter fit into our parameter structure */ + if (cde_ctx->num_params >= MAX_CDE_PARAMS) { + nvgpu_warn(g, "cde: no room for new parameters param idx = %d", + cde_ctx->num_params); + return -ENOMEM; + } + + /* is the given id valid? */ + if (param->id >= NUM_RESERVED_PARAMS + MAX_CDE_USER_PARAMS) { + nvgpu_warn(g, "cde: parameter id is not valid. param idx = %d, id=%u, max=%u", + param->id, cde_ctx->num_params, + NUM_RESERVED_PARAMS + MAX_CDE_USER_PARAMS); + return -EINVAL; + } + + cde_ctx->params[cde_ctx->num_params] = *param; + cde_ctx->num_params++; + + return 0; +} + +static int gk20a_init_cde_required_class(struct gk20a_cde_ctx *cde_ctx, + struct nvgpu_firmware *img, + u32 required_class) +{ + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + int err; + + /* CDE enabled */ + cde_ctx->ch->cde = true; + + err = gk20a_alloc_obj_ctx(cde_ctx->ch, required_class, 0); + if (err) { + nvgpu_warn(g, "cde: failed to allocate ctx. err=%d", + err); + return err; + } + + return 0; +} + +static int gk20a_init_cde_command(struct gk20a_cde_ctx *cde_ctx, + struct nvgpu_firmware *img, + u32 op, + struct gk20a_cde_cmd_elem *cmd_elem, + u32 num_elems) +{ + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + struct nvgpu_gpfifo_entry **gpfifo, *gpfifo_elem; + u32 *num_entries; + unsigned int i; + + /* check command type */ + if (op == TYPE_BUF_COMMAND_INIT) { + gpfifo = &cde_ctx->init_convert_cmd; + num_entries = &cde_ctx->init_cmd_num_entries; + } else if (op == TYPE_BUF_COMMAND_CONVERT) { + gpfifo = &cde_ctx->convert_cmd; + num_entries = &cde_ctx->convert_cmd_num_entries; + } else { + nvgpu_warn(g, "cde: unknown command. op=%u", + op); + return -EINVAL; + } + + /* allocate gpfifo entries to be pushed */ + *gpfifo = nvgpu_kzalloc(g, + sizeof(struct nvgpu_gpfifo_entry) * num_elems); + if (!*gpfifo) { + nvgpu_warn(g, "cde: could not allocate memory for gpfifo entries"); + return -ENOMEM; + } + + gpfifo_elem = *gpfifo; + for (i = 0; i < num_elems; i++, cmd_elem++, gpfifo_elem++) { + struct nvgpu_mem *target_mem; + + /* validate the current entry */ + if (cmd_elem->target_buf >= cde_ctx->num_bufs) { + nvgpu_warn(g, "cde: target buffer is not available (target=%u, num_bufs=%u)", + cmd_elem->target_buf, cde_ctx->num_bufs); + return -EINVAL; + } + + target_mem = cde_ctx->mem + cmd_elem->target_buf; + if (target_mem->size< + cmd_elem->target_byte_offset + cmd_elem->num_bytes) { + nvgpu_warn(g, "cde: target buffer cannot hold all entries (target_size=%zu, target_byte_offset=%lld, num_bytes=%llu)", + target_mem->size, + cmd_elem->target_byte_offset, + cmd_elem->num_bytes); + return -EINVAL; + } + + /* store the element into gpfifo */ + gpfifo_elem->entry0 = + u64_lo32(target_mem->gpu_va + + cmd_elem->target_byte_offset); + gpfifo_elem->entry1 = + u64_hi32(target_mem->gpu_va + + cmd_elem->target_byte_offset) | + pbdma_gp_entry1_length_f(cmd_elem->num_bytes / + sizeof(u32)); + } + + *num_entries = num_elems; + return 0; +} + +static int gk20a_cde_pack_cmdbufs(struct gk20a_cde_ctx *cde_ctx) +{ + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + unsigned long init_bytes = cde_ctx->init_cmd_num_entries * + sizeof(struct nvgpu_gpfifo_entry); + unsigned long conv_bytes = cde_ctx->convert_cmd_num_entries * + sizeof(struct nvgpu_gpfifo_entry); + unsigned long total_bytes = init_bytes + conv_bytes; + struct nvgpu_gpfifo_entry *combined_cmd; + + /* allocate buffer that has space for both */ + combined_cmd = nvgpu_kzalloc(g, total_bytes); + if (!combined_cmd) { + nvgpu_warn(g, + "cde: could not allocate memory for gpfifo entries"); + return -ENOMEM; + } + + /* move the original init here and append convert */ + memcpy(combined_cmd, cde_ctx->init_convert_cmd, init_bytes); + memcpy(combined_cmd + cde_ctx->init_cmd_num_entries, + cde_ctx->convert_cmd, conv_bytes); + + nvgpu_kfree(g, cde_ctx->init_convert_cmd); + nvgpu_kfree(g, cde_ctx->convert_cmd); + + cde_ctx->init_convert_cmd = combined_cmd; + cde_ctx->convert_cmd = combined_cmd + + cde_ctx->init_cmd_num_entries; + + return 0; +} + +static int gk20a_init_cde_img(struct gk20a_cde_ctx *cde_ctx, + struct nvgpu_firmware *img) +{ + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + struct gk20a_cde_app *cde_app = &l->cde_app; + u32 *data = (u32 *)img->data; + u32 num_of_elems; + struct gk20a_cde_hdr_elem *elem; + u32 min_size = 0; + int err = 0; + unsigned int i; + + min_size += 2 * sizeof(u32); + if (img->size < min_size) { + nvgpu_warn(g, "cde: invalid image header"); + return -EINVAL; + } + + cde_app->firmware_version = data[0]; + num_of_elems = data[1]; + + min_size += num_of_elems * sizeof(*elem); + if (img->size < min_size) { + nvgpu_warn(g, "cde: bad image"); + return -EINVAL; + } + + elem = (struct gk20a_cde_hdr_elem *)&data[2]; + for (i = 0; i < num_of_elems; i++) { + int err = 0; + switch (elem->type) { + case TYPE_BUF: + err = gk20a_init_cde_buf(cde_ctx, img, &elem->buf); + break; + case TYPE_REPLACE: + err = gk20a_init_cde_replace(cde_ctx, img, + &elem->replace); + break; + case TYPE_PARAM: + err = gk20a_init_cde_param(cde_ctx, img, &elem->param); + break; + case TYPE_REQUIRED_CLASS: + err = gk20a_init_cde_required_class(cde_ctx, img, + elem->required_class); + break; + case TYPE_COMMAND: + { + struct gk20a_cde_cmd_elem *cmd = (void *) + &img->data[elem->command.data_byte_offset]; + err = gk20a_init_cde_command(cde_ctx, img, + elem->command.op, cmd, + elem->command.num_entries); + break; + } + case TYPE_ARRAY: + memcpy(&cde_app->arrays[elem->array.id][0], + elem->array.data, + MAX_CDE_ARRAY_ENTRIES*sizeof(u32)); + break; + default: + nvgpu_warn(g, "cde: unknown header element"); + err = -EINVAL; + } + + if (err) + goto deinit_image; + + elem++; + } + + if (!cde_ctx->init_convert_cmd || !cde_ctx->init_cmd_num_entries) { + nvgpu_warn(g, "cde: convert command not defined"); + err = -EINVAL; + goto deinit_image; + } + + if (!cde_ctx->convert_cmd || !cde_ctx->convert_cmd_num_entries) { + nvgpu_warn(g, "cde: convert command not defined"); + err = -EINVAL; + goto deinit_image; + } + + err = gk20a_cde_pack_cmdbufs(cde_ctx); + if (err) + goto deinit_image; + + return 0; + +deinit_image: + gk20a_deinit_cde_img(cde_ctx); + return err; +} + +static int gk20a_cde_execute_buffer(struct gk20a_cde_ctx *cde_ctx, + u32 op, struct nvgpu_channel_fence *fence, + u32 flags, struct gk20a_fence **fence_out) +{ + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + struct nvgpu_gpfifo_entry *gpfifo = NULL; + int num_entries = 0; + + /* check command type */ + if (op == TYPE_BUF_COMMAND_INIT) { + /* both init and convert combined */ + gpfifo = cde_ctx->init_convert_cmd; + num_entries = cde_ctx->init_cmd_num_entries + + cde_ctx->convert_cmd_num_entries; + } else if (op == TYPE_BUF_COMMAND_CONVERT) { + gpfifo = cde_ctx->convert_cmd; + num_entries = cde_ctx->convert_cmd_num_entries; + } else if (op == TYPE_BUF_COMMAND_NOOP) { + /* Any non-null gpfifo will suffice with 0 num_entries */ + gpfifo = cde_ctx->init_convert_cmd; + num_entries = 0; + } else { + nvgpu_warn(g, "cde: unknown buffer"); + return -EINVAL; + } + + if (gpfifo == NULL) { + nvgpu_warn(g, "cde: buffer not available"); + return -ENOSYS; + } + + return nvgpu_submit_channel_gpfifo_kernel(cde_ctx->ch, gpfifo, + num_entries, flags, fence, fence_out); +} + +static void gk20a_cde_ctx_release(struct gk20a_cde_ctx *cde_ctx) +__acquires(&cde_app->mutex) +__releases(&cde_app->mutex) +{ + struct gk20a_cde_app *cde_app = &cde_ctx->l->cde_app; + struct gk20a *g = &cde_ctx->l->g; + + nvgpu_log(g, gpu_dbg_cde_ctx, "releasing use on %p", cde_ctx); + trace_gk20a_cde_release(cde_ctx); + + nvgpu_mutex_acquire(&cde_app->mutex); + + if (cde_ctx->in_use) { + cde_ctx->in_use = false; + nvgpu_list_move(&cde_ctx->list, &cde_app->free_contexts); + cde_app->ctx_usecount--; + } else { + nvgpu_log_info(g, "double release cde context %p", cde_ctx); + } + + nvgpu_mutex_release(&cde_app->mutex); +} + +static void gk20a_cde_ctx_deleter_fn(struct work_struct *work) +__acquires(&cde_app->mutex) +__releases(&cde_app->mutex) +{ + struct delayed_work *delay_work = to_delayed_work(work); + struct gk20a_cde_ctx *cde_ctx = container_of(delay_work, + struct gk20a_cde_ctx, ctx_deleter_work); + struct gk20a_cde_app *cde_app = &cde_ctx->l->cde_app; + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + int err; + + /* someone has just taken it? engine deletion started? */ + if (cde_ctx->in_use || !cde_app->initialised) + return; + + nvgpu_log(g, gpu_dbg_fn | gpu_dbg_cde_ctx, + "cde: attempting to delete temporary %p", cde_ctx); + + err = gk20a_busy(g); + if (err) { + /* this context would find new use anyway later, so not freeing + * here does not leak anything */ + nvgpu_warn(g, "cde: cannot set gk20a on, postponing" + " temp ctx deletion"); + return; + } + + nvgpu_mutex_acquire(&cde_app->mutex); + if (cde_ctx->in_use || !cde_app->initialised) { + nvgpu_log(g, gpu_dbg_cde_ctx, + "cde: context use raced, not deleting %p", + cde_ctx); + goto out; + } + + WARN(delayed_work_pending(&cde_ctx->ctx_deleter_work), + "double pending %p", cde_ctx); + + gk20a_cde_remove_ctx(cde_ctx); + nvgpu_log(g, gpu_dbg_fn | gpu_dbg_cde_ctx, + "cde: destroyed %p count=%d use=%d max=%d", + cde_ctx, cde_app->ctx_count, cde_app->ctx_usecount, + cde_app->ctx_count_top); + +out: + nvgpu_mutex_release(&cde_app->mutex); + gk20a_idle(g); +} + +static struct gk20a_cde_ctx *gk20a_cde_do_get_context(struct nvgpu_os_linux *l) +__must_hold(&cde_app->mutex) +{ + struct gk20a *g = &l->g; + struct gk20a_cde_app *cde_app = &l->cde_app; + struct gk20a_cde_ctx *cde_ctx; + + /* exhausted? */ + + if (cde_app->ctx_usecount >= MAX_CTX_USE_COUNT) + return ERR_PTR(-EAGAIN); + + /* idle context available? */ + + if (!nvgpu_list_empty(&cde_app->free_contexts)) { + cde_ctx = nvgpu_list_first_entry(&cde_app->free_contexts, + gk20a_cde_ctx, list); + nvgpu_log(g, gpu_dbg_fn | gpu_dbg_cde_ctx, + "cde: got free %p count=%d use=%d max=%d", + cde_ctx, cde_app->ctx_count, + cde_app->ctx_usecount, + cde_app->ctx_count_top); + trace_gk20a_cde_get_context(cde_ctx); + + /* deleter work may be scheduled, but in_use prevents it */ + cde_ctx->in_use = true; + nvgpu_list_move(&cde_ctx->list, &cde_app->used_contexts); + cde_app->ctx_usecount++; + + /* cancel any deletions now that ctx is in use */ + gk20a_cde_cancel_deleter(cde_ctx, true); + return cde_ctx; + } + + /* no free contexts, get a temporary one */ + + nvgpu_log(g, gpu_dbg_fn | gpu_dbg_cde_ctx, + "cde: no free contexts, count=%d", + cde_app->ctx_count); + + cde_ctx = gk20a_cde_allocate_context(l); + if (IS_ERR(cde_ctx)) { + nvgpu_warn(g, "cde: cannot allocate context: %ld", + PTR_ERR(cde_ctx)); + return cde_ctx; + } + + trace_gk20a_cde_get_context(cde_ctx); + cde_ctx->in_use = true; + cde_ctx->is_temporary = true; + cde_app->ctx_usecount++; + cde_app->ctx_count++; + if (cde_app->ctx_count > cde_app->ctx_count_top) + cde_app->ctx_count_top = cde_app->ctx_count; + nvgpu_list_add(&cde_ctx->list, &cde_app->used_contexts); + + return cde_ctx; +} + +static struct gk20a_cde_ctx *gk20a_cde_get_context(struct nvgpu_os_linux *l) +__releases(&cde_app->mutex) +__acquires(&cde_app->mutex) +{ + struct gk20a *g = &l->g; + struct gk20a_cde_app *cde_app = &l->cde_app; + struct gk20a_cde_ctx *cde_ctx = NULL; + struct nvgpu_timeout timeout; + + nvgpu_timeout_init(g, &timeout, MAX_CTX_RETRY_TIME, + NVGPU_TIMER_CPU_TIMER); + + do { + cde_ctx = gk20a_cde_do_get_context(l); + if (PTR_ERR(cde_ctx) != -EAGAIN) + break; + + /* exhausted, retry */ + nvgpu_mutex_release(&cde_app->mutex); + cond_resched(); + nvgpu_mutex_acquire(&cde_app->mutex); + } while (!nvgpu_timeout_expired(&timeout)); + + return cde_ctx; +} + +static struct gk20a_cde_ctx *gk20a_cde_allocate_context(struct nvgpu_os_linux *l) +{ + struct gk20a *g = &l->g; + struct gk20a_cde_ctx *cde_ctx; + int ret; + + cde_ctx = nvgpu_kzalloc(g, sizeof(*cde_ctx)); + if (!cde_ctx) + return ERR_PTR(-ENOMEM); + + cde_ctx->l = l; + cde_ctx->dev = dev_from_gk20a(g); + + ret = gk20a_cde_load(cde_ctx); + if (ret) { + nvgpu_kfree(g, cde_ctx); + return ERR_PTR(ret); + } + + nvgpu_init_list_node(&cde_ctx->list); + cde_ctx->is_temporary = false; + cde_ctx->in_use = false; + INIT_DELAYED_WORK(&cde_ctx->ctx_deleter_work, + gk20a_cde_ctx_deleter_fn); + + nvgpu_log(g, gpu_dbg_fn | gpu_dbg_cde_ctx, "cde: allocated %p", cde_ctx); + trace_gk20a_cde_allocate_context(cde_ctx); + return cde_ctx; +} + +static u32 gk20a_cde_mapping_page_size(struct vm_gk20a *vm, + u32 map_offset, u32 map_size) +{ + struct gk20a *g = gk20a_from_vm(vm); + + /* + * To be simple we will just make the map size depend on the + * iommu'ability of the driver. If there's an IOMMU we can rely on + * buffers being contiguous. If not, then we'll use 4k pages since we + * know that will work for any buffer. + */ + if (!nvgpu_iommuable(g)) + return SZ_4K; + + /* + * If map size or offset is not 64K aligned then use small pages. + */ + if (map_size & (vm->big_page_size - 1) || + map_offset & (vm->big_page_size - 1)) + return SZ_4K; + + return vm->big_page_size; +} + +int gk20a_cde_convert(struct nvgpu_os_linux *l, + struct dma_buf *compbits_scatter_buf, + u64 compbits_byte_offset, + u64 scatterbuffer_byte_offset, + struct nvgpu_channel_fence *fence, + u32 __flags, struct gk20a_cde_param *params, + int num_params, struct gk20a_fence **fence_out) +__acquires(&l->cde_app->mutex) +__releases(&l->cde_app->mutex) +{ + struct gk20a *g = &l->g; + struct gk20a_cde_ctx *cde_ctx = NULL; + struct gk20a_comptags comptags; + struct nvgpu_os_buffer os_buf = { + compbits_scatter_buf, + NULL, + dev_from_gk20a(g) + }; + u64 mapped_compbits_offset = 0; + u64 compbits_size = 0; + u64 mapped_scatterbuffer_offset = 0; + u64 scatterbuffer_size = 0; + u64 map_vaddr = 0; + u64 map_offset = 0; + u64 map_size = 0; + u8 *surface = NULL; + u64 big_page_mask = 0; + u32 flags; + int err, i; + const s16 compbits_kind = 0; + u32 submit_op; + struct dma_buf_attachment *attachment; + + nvgpu_log(g, gpu_dbg_cde, "compbits_byte_offset=%llu scatterbuffer_byte_offset=%llu", + compbits_byte_offset, scatterbuffer_byte_offset); + + /* scatter buffer must be after compbits buffer */ + if (scatterbuffer_byte_offset && + scatterbuffer_byte_offset < compbits_byte_offset) + return -EINVAL; + + err = gk20a_busy(g); + if (err) + return err; + + nvgpu_mutex_acquire(&l->cde_app.mutex); + cde_ctx = gk20a_cde_get_context(l); + nvgpu_mutex_release(&l->cde_app.mutex); + if (IS_ERR(cde_ctx)) { + err = PTR_ERR(cde_ctx); + goto exit_idle; + } + + /* First, map the buffer to local va */ + + /* ensure that the compbits buffer has drvdata */ + err = gk20a_dmabuf_alloc_drvdata(compbits_scatter_buf, + dev_from_gk20a(g)); + if (err) + goto exit_idle; + + /* compbits don't start at page aligned offset, so we need to align + the region to be mapped */ + big_page_mask = cde_ctx->vm->big_page_size - 1; + map_offset = compbits_byte_offset & ~big_page_mask; + map_size = compbits_scatter_buf->size - map_offset; + + + /* compute compbit start offset from the beginning of the mapped + area */ + mapped_compbits_offset = compbits_byte_offset - map_offset; + if (scatterbuffer_byte_offset) { + compbits_size = scatterbuffer_byte_offset - + compbits_byte_offset; + mapped_scatterbuffer_offset = scatterbuffer_byte_offset - + map_offset; + scatterbuffer_size = compbits_scatter_buf->size - + scatterbuffer_byte_offset; + } else { + compbits_size = compbits_scatter_buf->size - + compbits_byte_offset; + } + + nvgpu_log(g, gpu_dbg_cde, "map_offset=%llu map_size=%llu", + map_offset, map_size); + nvgpu_log(g, gpu_dbg_cde, "mapped_compbits_offset=%llu compbits_size=%llu", + mapped_compbits_offset, compbits_size); + nvgpu_log(g, gpu_dbg_cde, "mapped_scatterbuffer_offset=%llu scatterbuffer_size=%llu", + mapped_scatterbuffer_offset, scatterbuffer_size); + + + /* map the destination buffer */ + get_dma_buf(compbits_scatter_buf); /* a ref for nvgpu_vm_map_linux */ + err = nvgpu_vm_map_linux(cde_ctx->vm, compbits_scatter_buf, 0, + NVGPU_VM_MAP_CACHEABLE | + NVGPU_VM_MAP_DIRECT_KIND_CTRL, + gk20a_cde_mapping_page_size(cde_ctx->vm, + map_offset, + map_size), + NV_KIND_INVALID, + compbits_kind, /* incompressible kind */ + gk20a_mem_flag_none, + map_offset, map_size, + NULL, + &map_vaddr); + if (err) { + nvgpu_warn(g, "cde: failed to map compbits scatter buf at %lld size %lld", + map_offset, map_size); + dma_buf_put(compbits_scatter_buf); + err = -EINVAL; + goto exit_idle; + } + + if (scatterbuffer_byte_offset && + l->ops.cde.need_scatter_buffer && + l->ops.cde.need_scatter_buffer(g)) { + struct sg_table *sgt; + void *scatter_buffer; + + surface = dma_buf_vmap(compbits_scatter_buf); + if (IS_ERR(surface)) { + nvgpu_warn(g, + "dma_buf_vmap failed"); + err = -EINVAL; + goto exit_unmap_vaddr; + } + + scatter_buffer = surface + scatterbuffer_byte_offset; + + nvgpu_log(g, gpu_dbg_cde, "surface=0x%p scatterBuffer=0x%p", + surface, scatter_buffer); + sgt = gk20a_mm_pin(dev_from_gk20a(g), compbits_scatter_buf, + &attachment); + if (IS_ERR(sgt)) { + nvgpu_warn(g, + "mm_pin failed"); + err = -EINVAL; + goto exit_unmap_surface; + } else { + err = l->ops.cde.populate_scatter_buffer(g, sgt, + compbits_byte_offset, scatter_buffer, + scatterbuffer_size); + WARN_ON(err); + + gk20a_mm_unpin(dev_from_gk20a(g), compbits_scatter_buf, + attachment, sgt); + if (err) + goto exit_unmap_surface; + } + + __cpuc_flush_dcache_area(scatter_buffer, scatterbuffer_size); + dma_buf_vunmap(compbits_scatter_buf, surface); + surface = NULL; + } + + /* store source buffer compression tags */ + gk20a_get_comptags(&os_buf, &comptags); + cde_ctx->surf_param_offset = comptags.offset; + cde_ctx->surf_param_lines = comptags.lines; + + /* store surface vaddr. This is actually compbit vaddr, but since + compbits live in the same surface, and we can get the alloc base + address by using gpuva_to_iova_base, this will do */ + cde_ctx->surf_vaddr = map_vaddr; + + /* store information about destination */ + cde_ctx->compbit_vaddr = map_vaddr + mapped_compbits_offset; + cde_ctx->compbit_size = compbits_size; + + cde_ctx->scatterbuffer_vaddr = map_vaddr + mapped_scatterbuffer_offset; + cde_ctx->scatterbuffer_size = scatterbuffer_size; + + /* remove existing argument data */ + memset(cde_ctx->user_param_values, 0, + sizeof(cde_ctx->user_param_values)); + + /* read user space arguments for the conversion */ + for (i = 0; i < num_params; i++) { + struct gk20a_cde_param *param = params + i; + int id = param->id - NUM_RESERVED_PARAMS; + + if (id < 0 || id >= MAX_CDE_USER_PARAMS) { + nvgpu_warn(g, "cde: unknown user parameter"); + err = -EINVAL; + goto exit_unmap_surface; + } + cde_ctx->user_param_values[id] = param->value; + } + + /* patch data */ + err = gk20a_cde_patch_params(cde_ctx); + if (err) { + nvgpu_warn(g, "cde: failed to patch parameters"); + goto exit_unmap_surface; + } + + nvgpu_log(g, gpu_dbg_cde, "cde: buffer=cbc, size=%zu, gpuva=%llx\n", + g->gr.compbit_store.mem.size, cde_ctx->backing_store_vaddr); + nvgpu_log(g, gpu_dbg_cde, "cde: buffer=compbits, size=%llu, gpuva=%llx\n", + cde_ctx->compbit_size, cde_ctx->compbit_vaddr); + nvgpu_log(g, gpu_dbg_cde, "cde: buffer=scatterbuffer, size=%llu, gpuva=%llx\n", + cde_ctx->scatterbuffer_size, cde_ctx->scatterbuffer_vaddr); + + /* take always the postfence as it is needed for protecting the + * cde context */ + flags = __flags | NVGPU_SUBMIT_FLAGS_FENCE_GET; + + /* gk20a_cde_execute_buffer() will grab a power reference of it's own */ + gk20a_idle(g); + + if (comptags.lines == 0) { + /* + * Nothing to do on the buffer, but do a null kickoff for + * managing the pre and post fences. + */ + submit_op = TYPE_BUF_COMMAND_NOOP; + } else if (!cde_ctx->init_cmd_executed) { + /* + * First time, so include the init pushbuf too in addition to + * the conversion code. + */ + submit_op = TYPE_BUF_COMMAND_INIT; + } else { + /* + * The usual condition: execute just the conversion. + */ + submit_op = TYPE_BUF_COMMAND_CONVERT; + } + err = gk20a_cde_execute_buffer(cde_ctx, submit_op, + fence, flags, fence_out); + + if (comptags.lines != 0 && !err) + cde_ctx->init_cmd_executed = true; + + /* unmap the buffers - channel holds references to them now */ + nvgpu_vm_unmap(cde_ctx->vm, map_vaddr, NULL); + + return err; + +exit_unmap_surface: + if (surface) + dma_buf_vunmap(compbits_scatter_buf, surface); +exit_unmap_vaddr: + nvgpu_vm_unmap(cde_ctx->vm, map_vaddr, NULL); +exit_idle: + gk20a_idle(g); + return err; +} + +static void gk20a_cde_finished_ctx_cb(struct channel_gk20a *ch, void *data) +__acquires(&cde_app->mutex) +__releases(&cde_app->mutex) +{ + struct gk20a_cde_ctx *cde_ctx = data; + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + struct gk20a_cde_app *cde_app = &l->cde_app; + bool channel_idle; + + channel_gk20a_joblist_lock(ch); + channel_idle = channel_gk20a_joblist_is_empty(ch); + channel_gk20a_joblist_unlock(ch); + + if (!channel_idle) + return; + + trace_gk20a_cde_finished_ctx_cb(cde_ctx); + nvgpu_log(g, gpu_dbg_fn | gpu_dbg_cde_ctx, "cde: finished %p", cde_ctx); + if (!cde_ctx->in_use) + nvgpu_log_info(g, "double finish cde context %p on channel %p", + cde_ctx, ch); + + if (gk20a_channel_check_timedout(ch)) { + if (cde_ctx->is_temporary) { + nvgpu_warn(g, + "cde: channel had timed out" + " (temporary channel)"); + /* going to be deleted anyway */ + } else { + nvgpu_warn(g, + "cde: channel had timed out" + ", reloading"); + /* mark it to be deleted, replace with a new one */ + nvgpu_mutex_acquire(&cde_app->mutex); + cde_ctx->is_temporary = true; + if (gk20a_cde_create_context(l)) { + nvgpu_err(g, "cde: can't replace context"); + } + nvgpu_mutex_release(&cde_app->mutex); + } + } + + /* delete temporary contexts later (watch for doubles) */ + if (cde_ctx->is_temporary && cde_ctx->in_use) { + WARN_ON(delayed_work_pending(&cde_ctx->ctx_deleter_work)); + schedule_delayed_work(&cde_ctx->ctx_deleter_work, + msecs_to_jiffies(CTX_DELETE_TIME)); + } + + if (!gk20a_channel_check_timedout(ch)) { + gk20a_cde_ctx_release(cde_ctx); + } +} + +static int gk20a_cde_load(struct gk20a_cde_ctx *cde_ctx) +{ + struct nvgpu_os_linux *l = cde_ctx->l; + struct gk20a *g = &l->g; + struct nvgpu_firmware *img; + struct channel_gk20a *ch; + struct tsg_gk20a *tsg; + struct gr_gk20a *gr = &g->gr; + struct nvgpu_setup_bind_args setup_bind_args; + int err = 0; + u64 vaddr; + + img = nvgpu_request_firmware(g, "gpu2cde.bin", 0); + if (!img) { + nvgpu_err(g, "cde: could not fetch the firmware"); + return -ENOSYS; + } + + tsg = gk20a_tsg_open(g, nvgpu_current_pid(g)); + if (!tsg) { + nvgpu_err(g, "cde: could not create TSG"); + err = -ENOMEM; + goto err_get_gk20a_channel; + } + + ch = gk20a_open_new_channel_with_cb(g, gk20a_cde_finished_ctx_cb, + cde_ctx, + -1, + false); + if (!ch) { + nvgpu_warn(g, "cde: gk20a channel not available"); + err = -ENOMEM; + goto err_get_gk20a_channel; + } + + ch->timeout.enabled = false; + + /* bind the channel to the vm */ + err = g->ops.mm.vm_bind_channel(g->mm.cde.vm, ch); + if (err) { + nvgpu_warn(g, "cde: could not bind vm"); + goto err_commit_va; + } + + err = gk20a_tsg_bind_channel(tsg, ch); + if (err) { + nvgpu_err(g, "cde: unable to bind to tsg"); + goto err_setup_bind; + } + + setup_bind_args.num_gpfifo_entries = 1024; + setup_bind_args.num_inflight_jobs = 0; + setup_bind_args.flags = 0; + err = nvgpu_channel_setup_bind(ch, &setup_bind_args); + if (err) { + nvgpu_warn(g, "cde: unable to setup channel"); + goto err_setup_bind; + } + + /* map backing store to gpu virtual space */ + vaddr = nvgpu_gmmu_map(ch->vm, &gr->compbit_store.mem, + g->gr.compbit_store.mem.size, + NVGPU_VM_MAP_CACHEABLE, + gk20a_mem_flag_read_only, + false, + gr->compbit_store.mem.aperture); + + if (!vaddr) { + nvgpu_warn(g, "cde: cannot map compression bit backing store"); + err = -ENOMEM; + goto err_map_backingstore; + } + + /* store initialisation data */ + cde_ctx->ch = ch; + cde_ctx->tsg = tsg; + cde_ctx->vm = ch->vm; + cde_ctx->backing_store_vaddr = vaddr; + + /* initialise the firmware */ + err = gk20a_init_cde_img(cde_ctx, img); + if (err) { + nvgpu_warn(g, "cde: image initialisation failed"); + goto err_init_cde_img; + } + + /* initialisation done */ + nvgpu_release_firmware(g, img); + + return 0; + +err_init_cde_img: + nvgpu_gmmu_unmap(ch->vm, &g->gr.compbit_store.mem, vaddr); +err_map_backingstore: +err_setup_bind: + nvgpu_vm_put(ch->vm); +err_commit_va: +err_get_gk20a_channel: + nvgpu_release_firmware(g, img); + nvgpu_err(g, "cde: couldn't initialise buffer converter: %d", err); + return err; +} + +int gk20a_cde_reload(struct nvgpu_os_linux *l) +__acquires(&l->cde_app->mutex) +__releases(&l->cde_app->mutex) +{ + struct gk20a *g = &l->g; + struct gk20a_cde_app *cde_app = &l->cde_app; + int err; + + if (!cde_app->initialised) + return -ENOSYS; + + err = gk20a_busy(g); + if (err) + return err; + + nvgpu_mutex_acquire(&cde_app->mutex); + + gk20a_cde_stop(l); + + err = gk20a_cde_create_contexts(l); + if (!err) + cde_app->initialised = true; + + nvgpu_mutex_release(&cde_app->mutex); + + gk20a_idle(g); + return err; +} + +int gk20a_init_cde_support(struct nvgpu_os_linux *l) +__acquires(&cde_app->mutex) +__releases(&cde_app->mutex) +{ + struct gk20a_cde_app *cde_app = &l->cde_app; + struct gk20a *g = &l->g; + int err; + + if (cde_app->initialised) + return 0; + + nvgpu_log(g, gpu_dbg_fn | gpu_dbg_cde_ctx, "cde: init"); + + err = nvgpu_mutex_init(&cde_app->mutex); + if (err) + return err; + + nvgpu_mutex_acquire(&cde_app->mutex); + + nvgpu_init_list_node(&cde_app->free_contexts); + nvgpu_init_list_node(&cde_app->used_contexts); + cde_app->ctx_count = 0; + cde_app->ctx_count_top = 0; + cde_app->ctx_usecount = 0; + + err = gk20a_cde_create_contexts(l); + if (!err) + cde_app->initialised = true; + + nvgpu_mutex_release(&cde_app->mutex); + nvgpu_log(g, gpu_dbg_cde_ctx, "cde: init finished: %d", err); + + if (err) + nvgpu_mutex_destroy(&cde_app->mutex); + + return err; +} + +enum cde_launch_patch_id { + PATCH_H_QMD_CTA_RASTER_WIDTH_ID = 1024, + PATCH_H_QMD_CTA_RASTER_HEIGHT_ID = 1025, + PATCH_QMD_CTA_RASTER_DEPTH_ID = 1026, /* for firmware v0 only */ + PATCH_QMD_CTA_THREAD_DIMENSION0_ID = 1027, + PATCH_QMD_CTA_THREAD_DIMENSION1_ID = 1028, + PATCH_QMD_CTA_THREAD_DIMENSION2_ID = 1029, /* for firmware v0 only */ + PATCH_USER_CONST_XTILES_ID = 1030, /* for firmware v0 only */ + PATCH_USER_CONST_YTILES_ID = 1031, /* for firmware v0 only */ + PATCH_USER_CONST_BLOCKHEIGHTLOG2_ID = 1032, + PATCH_USER_CONST_DSTPITCH_ID = 1033, /* for firmware v0 only */ + PATCH_H_USER_CONST_FLAGS_ID = 1034, /* for firmware v0 only */ + PATCH_H_VPC_CURRENT_GRID_SIZE_X_ID = 1035, + PATCH_H_VPC_CURRENT_GRID_SIZE_Y_ID = 1036, + PATCH_H_VPC_CURRENT_GRID_SIZE_Z_ID = 1037, + PATCH_VPC_CURRENT_GROUP_SIZE_X_ID = 1038, + PATCH_VPC_CURRENT_GROUP_SIZE_Y_ID = 1039, + PATCH_VPC_CURRENT_GROUP_SIZE_Z_ID = 1040, + PATCH_USER_CONST_XBLOCKS_ID = 1041, + PATCH_H_USER_CONST_DSTOFFSET_ID = 1042, + PATCH_V_QMD_CTA_RASTER_WIDTH_ID = 1043, + PATCH_V_QMD_CTA_RASTER_HEIGHT_ID = 1044, + PATCH_V_USER_CONST_DSTOFFSET_ID = 1045, + PATCH_V_VPC_CURRENT_GRID_SIZE_X_ID = 1046, + PATCH_V_VPC_CURRENT_GRID_SIZE_Y_ID = 1047, + PATCH_V_VPC_CURRENT_GRID_SIZE_Z_ID = 1048, + PATCH_H_LAUNCH_WORD1_ID = 1049, + PATCH_H_LAUNCH_WORD2_ID = 1050, + PATCH_V_LAUNCH_WORD1_ID = 1051, + PATCH_V_LAUNCH_WORD2_ID = 1052, + PATCH_H_QMD_PROGRAM_OFFSET_ID = 1053, + PATCH_H_QMD_REGISTER_COUNT_ID = 1054, + PATCH_V_QMD_PROGRAM_OFFSET_ID = 1055, + PATCH_V_QMD_REGISTER_COUNT_ID = 1056, +}; + +/* maximum number of WRITE_PATCHes in the below function */ +#define MAX_CDE_LAUNCH_PATCHES 32 + +static int gk20a_buffer_convert_gpu_to_cde_v1( + struct nvgpu_os_linux *l, + struct dma_buf *dmabuf, u32 consumer, + u64 offset, u64 compbits_hoffset, u64 compbits_voffset, + u64 scatterbuffer_offset, + u32 width, u32 height, u32 block_height_log2, + u32 submit_flags, struct nvgpu_channel_fence *fence_in, + struct gk20a_buffer_state *state) +{ + struct gk20a *g = &l->g; + struct gk20a_cde_param params[MAX_CDE_LAUNCH_PATCHES]; + int param = 0; + int err = 0; + struct gk20a_fence *new_fence = NULL; + const int wgx = 8; + const int wgy = 8; + const int compbits_per_byte = 4; /* one byte stores 4 compbit pairs */ + const int xalign = compbits_per_byte * wgx; + const int yalign = wgy; + + /* Compute per launch parameters */ + const int xtiles = (width + 7) >> 3; + const int ytiles = (height + 7) >> 3; + const int gridw_h = roundup(xtiles, xalign) / xalign; + const int gridh_h = roundup(ytiles, yalign) / yalign; + const int gridw_v = roundup(ytiles, xalign) / xalign; + const int gridh_v = roundup(xtiles, yalign) / yalign; + const int xblocks = (xtiles + 1) >> 1; + const int voffset = compbits_voffset - compbits_hoffset; + + int hprog = -1; + int vprog = -1; + + if (l->ops.cde.get_program_numbers) + l->ops.cde.get_program_numbers(g, block_height_log2, + l->cde_app.shader_parameter, + &hprog, &vprog); + else { + nvgpu_warn(g, "cde: chip not supported"); + return -ENOSYS; + } + + if (hprog < 0 || vprog < 0) { + nvgpu_warn(g, "cde: could not determine programs"); + return -ENOSYS; + } + + if (xtiles > 8192 / 8 || ytiles > 8192 / 8) + nvgpu_warn(g, "cde: surface is exceptionally large (xtiles=%d, ytiles=%d)", + xtiles, ytiles); + + nvgpu_log(g, gpu_dbg_cde, "w=%d, h=%d, bh_log2=%d, compbits_hoffset=0x%llx, compbits_voffset=0x%llx, scatterbuffer_offset=0x%llx", + width, height, block_height_log2, + compbits_hoffset, compbits_voffset, scatterbuffer_offset); + nvgpu_log(g, gpu_dbg_cde, "resolution (%d, %d) tiles (%d, %d)", + width, height, xtiles, ytiles); + nvgpu_log(g, gpu_dbg_cde, "group (%d, %d) gridH (%d, %d) gridV (%d, %d)", + wgx, wgy, gridw_h, gridh_h, gridw_v, gridh_v); + nvgpu_log(g, gpu_dbg_cde, "hprog=%d, offset=0x%x, regs=%d, vprog=%d, offset=0x%x, regs=%d", + hprog, + l->cde_app.arrays[ARRAY_PROGRAM_OFFSET][hprog], + l->cde_app.arrays[ARRAY_REGISTER_COUNT][hprog], + vprog, + l->cde_app.arrays[ARRAY_PROGRAM_OFFSET][vprog], + l->cde_app.arrays[ARRAY_REGISTER_COUNT][vprog]); + + /* Write parameters */ +#define WRITE_PATCH(NAME, VALUE) \ + params[param++] = (struct gk20a_cde_param){NAME##_ID, 0, VALUE} + WRITE_PATCH(PATCH_USER_CONST_XBLOCKS, xblocks); + WRITE_PATCH(PATCH_USER_CONST_BLOCKHEIGHTLOG2, + block_height_log2); + WRITE_PATCH(PATCH_QMD_CTA_THREAD_DIMENSION0, wgx); + WRITE_PATCH(PATCH_QMD_CTA_THREAD_DIMENSION1, wgy); + WRITE_PATCH(PATCH_VPC_CURRENT_GROUP_SIZE_X, wgx); + WRITE_PATCH(PATCH_VPC_CURRENT_GROUP_SIZE_Y, wgy); + WRITE_PATCH(PATCH_VPC_CURRENT_GROUP_SIZE_Z, 1); + + WRITE_PATCH(PATCH_H_QMD_CTA_RASTER_WIDTH, gridw_h); + WRITE_PATCH(PATCH_H_QMD_CTA_RASTER_HEIGHT, gridh_h); + WRITE_PATCH(PATCH_H_USER_CONST_DSTOFFSET, 0); + WRITE_PATCH(PATCH_H_VPC_CURRENT_GRID_SIZE_X, gridw_h); + WRITE_PATCH(PATCH_H_VPC_CURRENT_GRID_SIZE_Y, gridh_h); + WRITE_PATCH(PATCH_H_VPC_CURRENT_GRID_SIZE_Z, 1); + + WRITE_PATCH(PATCH_V_QMD_CTA_RASTER_WIDTH, gridw_v); + WRITE_PATCH(PATCH_V_QMD_CTA_RASTER_HEIGHT, gridh_v); + WRITE_PATCH(PATCH_V_USER_CONST_DSTOFFSET, voffset); + WRITE_PATCH(PATCH_V_VPC_CURRENT_GRID_SIZE_X, gridw_v); + WRITE_PATCH(PATCH_V_VPC_CURRENT_GRID_SIZE_Y, gridh_v); + WRITE_PATCH(PATCH_V_VPC_CURRENT_GRID_SIZE_Z, 1); + + WRITE_PATCH(PATCH_H_QMD_PROGRAM_OFFSET, + l->cde_app.arrays[ARRAY_PROGRAM_OFFSET][hprog]); + WRITE_PATCH(PATCH_H_QMD_REGISTER_COUNT, + l->cde_app.arrays[ARRAY_REGISTER_COUNT][hprog]); + WRITE_PATCH(PATCH_V_QMD_PROGRAM_OFFSET, + l->cde_app.arrays[ARRAY_PROGRAM_OFFSET][vprog]); + WRITE_PATCH(PATCH_V_QMD_REGISTER_COUNT, + l->cde_app.arrays[ARRAY_REGISTER_COUNT][vprog]); + + if (consumer & NVGPU_GPU_COMPBITS_CDEH) { + WRITE_PATCH(PATCH_H_LAUNCH_WORD1, + l->cde_app.arrays[ARRAY_LAUNCH_COMMAND][0]); + WRITE_PATCH(PATCH_H_LAUNCH_WORD2, + l->cde_app.arrays[ARRAY_LAUNCH_COMMAND][1]); + } else { + WRITE_PATCH(PATCH_H_LAUNCH_WORD1, + l->cde_app.arrays[ARRAY_LAUNCH_COMMAND][2]); + WRITE_PATCH(PATCH_H_LAUNCH_WORD2, + l->cde_app.arrays[ARRAY_LAUNCH_COMMAND][3]); + } + + if (consumer & NVGPU_GPU_COMPBITS_CDEV) { + WRITE_PATCH(PATCH_V_LAUNCH_WORD1, + l->cde_app.arrays[ARRAY_LAUNCH_COMMAND][0]); + WRITE_PATCH(PATCH_V_LAUNCH_WORD2, + l->cde_app.arrays[ARRAY_LAUNCH_COMMAND][1]); + } e