From 01e6fac4d61fdd7fff5433942ec93fc2ea1e4df1 Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Wed, 28 Jun 2023 18:24:25 -0400 Subject: Include nvgpu headers These are needed to build on NVIDIA's Jetson boards for the time being. Only a couple structs are required, so it should be fairly easy to remove this dependency at some point in the future. --- include/gk20a/ce2_gk20a.c | 576 ++ include/gk20a/ce2_gk20a.h | 156 + include/gk20a/clk_gk20a.h | 134 + include/gk20a/css_gr_gk20a.c | 636 ++ include/gk20a/css_gr_gk20a.h | 151 + include/gk20a/dbg_gpu_gk20a.c | 388 ++ include/gk20a/dbg_gpu_gk20a.h | 147 + include/gk20a/fecs_trace_gk20a.c | 744 +++ include/gk20a/fecs_trace_gk20a.h | 45 + include/gk20a/fence_gk20a.c | 319 + include/gk20a/fence_gk20a.h | 100 + include/gk20a/fifo_gk20a.c | 4649 +++++++++++++ include/gk20a/fifo_gk20a.h | 471 ++ include/gk20a/flcn_gk20a.c | 759 +++ include/gk20a/flcn_gk20a.h | 29 + include/gk20a/gk20a.c | 590 ++ include/gk20a/gk20a.h | 33 + include/gk20a/gr_ctx_gk20a.c | 486 ++ include/gk20a/gr_ctx_gk20a.h | 206 + include/gk20a/gr_ctx_gk20a_sim.c | 356 + include/gk20a/gr_gk20a.c | 8998 ++++++++++++++++++++++++++ include/gk20a/gr_gk20a.h | 851 +++ include/gk20a/gr_pri_gk20a.h | 261 + include/gk20a/hw_bus_gk20a.h | 171 + include/gk20a/hw_ccsr_gk20a.h | 163 + include/gk20a/hw_ce2_gk20a.h | 87 + include/gk20a/hw_ctxsw_prog_gk20a.h | 447 ++ include/gk20a/hw_falcon_gk20a.h | 559 ++ include/gk20a/hw_fb_gk20a.h | 263 + include/gk20a/hw_fifo_gk20a.h | 619 ++ include/gk20a/hw_flush_gk20a.h | 187 + include/gk20a/hw_gmmu_gk20a.h | 283 + include/gk20a/hw_gr_gk20a.h | 3807 +++++++++++ include/gk20a/hw_ltc_gk20a.h | 455 ++ include/gk20a/hw_mc_gk20a.h | 291 + include/gk20a/hw_pbdma_gk20a.h | 575 ++ include/gk20a/hw_perf_gk20a.h | 211 + include/gk20a/hw_pram_gk20a.h | 63 + include/gk20a/hw_pri_ringmaster_gk20a.h | 159 + include/gk20a/hw_pri_ringstation_fbp_gk20a.h | 231 + include/gk20a/hw_pri_ringstation_gpc_gk20a.h | 79 + include/gk20a/hw_pri_ringstation_sys_gk20a.h | 91 + include/gk20a/hw_proj_gk20a.h | 167 + include/gk20a/hw_pwr_gk20a.h | 823 +++ include/gk20a/hw_ram_gk20a.h | 443 ++ include/gk20a/hw_therm_gk20a.h | 367 ++ include/gk20a/hw_timer_gk20a.h | 127 + include/gk20a/hw_top_gk20a.h | 211 + include/gk20a/hw_trim_gk20a.h | 315 + include/gk20a/mm_gk20a.c | 654 ++ include/gk20a/mm_gk20a.h | 155 + include/gk20a/pmu_gk20a.c | 879 +++ include/gk20a/pmu_gk20a.h | 80 + include/gk20a/regops_gk20a.c | 472 ++ include/gk20a/regops_gk20a.h | 90 + 55 files changed, 34609 insertions(+) create mode 100644 include/gk20a/ce2_gk20a.c create mode 100644 include/gk20a/ce2_gk20a.h create mode 100644 include/gk20a/clk_gk20a.h create mode 100644 include/gk20a/css_gr_gk20a.c create mode 100644 include/gk20a/css_gr_gk20a.h create mode 100644 include/gk20a/dbg_gpu_gk20a.c create mode 100644 include/gk20a/dbg_gpu_gk20a.h create mode 100644 include/gk20a/fecs_trace_gk20a.c create mode 100644 include/gk20a/fecs_trace_gk20a.h create mode 100644 include/gk20a/fence_gk20a.c create mode 100644 include/gk20a/fence_gk20a.h create mode 100644 include/gk20a/fifo_gk20a.c create mode 100644 include/gk20a/fifo_gk20a.h create mode 100644 include/gk20a/flcn_gk20a.c create mode 100644 include/gk20a/flcn_gk20a.h create mode 100644 include/gk20a/gk20a.c create mode 100644 include/gk20a/gk20a.h create mode 100644 include/gk20a/gr_ctx_gk20a.c create mode 100644 include/gk20a/gr_ctx_gk20a.h create mode 100644 include/gk20a/gr_ctx_gk20a_sim.c create mode 100644 include/gk20a/gr_gk20a.c create mode 100644 include/gk20a/gr_gk20a.h create mode 100644 include/gk20a/gr_pri_gk20a.h create mode 100644 include/gk20a/hw_bus_gk20a.h create mode 100644 include/gk20a/hw_ccsr_gk20a.h create mode 100644 include/gk20a/hw_ce2_gk20a.h create mode 100644 include/gk20a/hw_ctxsw_prog_gk20a.h create mode 100644 include/gk20a/hw_falcon_gk20a.h create mode 100644 include/gk20a/hw_fb_gk20a.h create mode 100644 include/gk20a/hw_fifo_gk20a.h create mode 100644 include/gk20a/hw_flush_gk20a.h create mode 100644 include/gk20a/hw_gmmu_gk20a.h create mode 100644 include/gk20a/hw_gr_gk20a.h create mode 100644 include/gk20a/hw_ltc_gk20a.h create mode 100644 include/gk20a/hw_mc_gk20a.h create mode 100644 include/gk20a/hw_pbdma_gk20a.h create mode 100644 include/gk20a/hw_perf_gk20a.h create mode 100644 include/gk20a/hw_pram_gk20a.h create mode 100644 include/gk20a/hw_pri_ringmaster_gk20a.h create mode 100644 include/gk20a/hw_pri_ringstation_fbp_gk20a.h create mode 100644 include/gk20a/hw_pri_ringstation_gpc_gk20a.h create mode 100644 include/gk20a/hw_pri_ringstation_sys_gk20a.h create mode 100644 include/gk20a/hw_proj_gk20a.h create mode 100644 include/gk20a/hw_pwr_gk20a.h create mode 100644 include/gk20a/hw_ram_gk20a.h create mode 100644 include/gk20a/hw_therm_gk20a.h create mode 100644 include/gk20a/hw_timer_gk20a.h create mode 100644 include/gk20a/hw_top_gk20a.h create mode 100644 include/gk20a/hw_trim_gk20a.h create mode 100644 include/gk20a/mm_gk20a.c create mode 100644 include/gk20a/mm_gk20a.h create mode 100644 include/gk20a/pmu_gk20a.c create mode 100644 include/gk20a/pmu_gk20a.h create mode 100644 include/gk20a/regops_gk20a.c create mode 100644 include/gk20a/regops_gk20a.h (limited to 'include/gk20a') diff --git a/include/gk20a/ce2_gk20a.c b/include/gk20a/ce2_gk20a.c new file mode 100644 index 0000000..2a40b08 --- /dev/null +++ b/include/gk20a/ce2_gk20a.c @@ -0,0 +1,576 @@ +/* + * GK20A Graphics Copy Engine (gr host) + * + * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gk20a.h" +#include "gk20a/fence_gk20a.h" + +#include +#include +#include +#include +#include +#include +#include + +/* + * Copy engine defines line size in pixels + */ +#define MAX_CE_SHIFT 31 /* 4Gpixels -1 */ +#define MAX_CE_MASK ((u32) (~(~0U << MAX_CE_SHIFT))) +#define MAX_CE_ALIGN(a) (a & MAX_CE_MASK) + + +static u32 ce2_nonblockpipe_isr(struct gk20a *g, u32 fifo_intr) +{ + nvgpu_log(g, gpu_dbg_intr, "ce2 non-blocking pipe interrupt\n"); + + return ce2_intr_status_nonblockpipe_pending_f(); +} + +static u32 ce2_blockpipe_isr(struct gk20a *g, u32 fifo_intr) +{ + nvgpu_log(g, gpu_dbg_intr, "ce2 blocking pipe interrupt\n"); + + return ce2_intr_status_blockpipe_pending_f(); +} + +static u32 ce2_launcherr_isr(struct gk20a *g, u32 fifo_intr) +{ + nvgpu_log(g, gpu_dbg_intr, "ce2 launch error interrupt\n"); + + return ce2_intr_status_launcherr_pending_f(); +} + +void gk20a_ce2_isr(struct gk20a *g, u32 inst_id, u32 pri_base) +{ + u32 ce2_intr = gk20a_readl(g, ce2_intr_status_r()); + u32 clear_intr = 0; + + nvgpu_log(g, gpu_dbg_intr, "ce2 isr %08x\n", ce2_intr); + + /* clear blocking interrupts: they exibit broken behavior */ + if (ce2_intr & ce2_intr_status_blockpipe_pending_f()) { + clear_intr |= ce2_blockpipe_isr(g, ce2_intr); + } + + if (ce2_intr & ce2_intr_status_launcherr_pending_f()) { + clear_intr |= ce2_launcherr_isr(g, ce2_intr); + } + + gk20a_writel(g, ce2_intr_status_r(), clear_intr); + return; +} + +u32 gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base) +{ + u32 ops = 0; + u32 ce2_intr = gk20a_readl(g, ce2_intr_status_r()); + + nvgpu_log(g, gpu_dbg_intr, "ce2 nonstall isr %08x\n", ce2_intr); + + if (ce2_intr & ce2_intr_status_nonblockpipe_pending_f()) { + gk20a_writel(g, ce2_intr_status_r(), + ce2_nonblockpipe_isr(g, ce2_intr)); + ops |= (GK20A_NONSTALL_OPS_WAKEUP_SEMAPHORE | + GK20A_NONSTALL_OPS_POST_EVENTS); + } + return ops; +} + +/* static CE app api */ +static void gk20a_ce_put_fences(struct gk20a_gpu_ctx *ce_ctx) +{ + u32 i; + + for (i = 0; i < NVGPU_CE_MAX_INFLIGHT_JOBS; i++) { + struct gk20a_fence **fence = &ce_ctx->postfences[i]; + if (*fence) { + gk20a_fence_put(*fence); + } + *fence = NULL; + } +} + +/* assume this api should need to call under nvgpu_mutex_acquire(&ce_app->app_mutex) */ +static void gk20a_ce_delete_gpu_context(struct gk20a_gpu_ctx *ce_ctx) +{ + struct nvgpu_list_node *list = &ce_ctx->list; + + ce_ctx->gpu_ctx_state = NVGPU_CE_GPU_CTX_DELETED; + + nvgpu_mutex_acquire(&ce_ctx->gpu_ctx_mutex); + + if (nvgpu_mem_is_valid(&ce_ctx->cmd_buf_mem)) { + gk20a_ce_put_fences(ce_ctx); + nvgpu_dma_unmap_free(ce_ctx->vm, &ce_ctx->cmd_buf_mem); + } + + /* + * free the channel + * gk20a_channel_close() will also unbind the channel from TSG + */ + gk20a_channel_close(ce_ctx->ch); + nvgpu_ref_put(&ce_ctx->tsg->refcount, gk20a_tsg_release); + + /* housekeeping on app */ + if (list->prev && list->next) { + nvgpu_list_del(list); + } + + nvgpu_mutex_release(&ce_ctx->gpu_ctx_mutex); + nvgpu_mutex_destroy(&ce_ctx->gpu_ctx_mutex); + + nvgpu_kfree(ce_ctx->g, ce_ctx); +} + +static inline unsigned int gk20a_ce_get_method_size(int request_operation, + u64 size) +{ + /* failure size */ + unsigned int methodsize = UINT_MAX; + unsigned int iterations = 0; + u32 shift; + u64 chunk = size; + u32 height, width; + + while (chunk) { + iterations++; + + shift = MAX_CE_ALIGN(chunk) ? __ffs(MAX_CE_ALIGN(chunk)) : + MAX_CE_SHIFT; + width = chunk >> shift; + height = 1 << shift; + width = MAX_CE_ALIGN(width); + + chunk -= (u64) height * width; + } + + if (request_operation & NVGPU_CE_PHYS_MODE_TRANSFER) { + methodsize = (2 + (16 * iterations)) * sizeof(u32); + } else if (request_operation & NVGPU_CE_MEMSET) { + methodsize = (2 + (15 * iterations)) * sizeof(u32); + } + + return methodsize; +} + +int gk20a_ce_prepare_submit(u64 src_buf, + u64 dst_buf, + u64 size, + u32 *cmd_buf_cpu_va, + u32 max_cmd_buf_size, + unsigned int payload, + int launch_flags, + int request_operation, + u32 dma_copy_class) +{ + u32 launch = 0; + u32 methodSize = 0; + u64 offset = 0; + u64 chunk_size = 0; + u64 chunk = size; + + /* failure case handling */ + if ((gk20a_ce_get_method_size(request_operation, size) > + max_cmd_buf_size) || (!size) || + (request_operation > NVGPU_CE_MEMSET)) { + return 0; + } + + /* set the channel object */ + cmd_buf_cpu_va[methodSize++] = 0x20018000; + cmd_buf_cpu_va[methodSize++] = dma_copy_class; + + /* + * The purpose clear the memory in 2D rectangles. We get the ffs to + * determine the number of lines to copy. The only constraint is that + * maximum number of pixels per line is 4Gpix - 1, which is awkward for + * calculation, so we settle to 2Gpix per line to make calculatione + * more agreable + */ + + /* The copy engine in 2D mode can have (2^32 - 1) x (2^32 - 1) pixels in + * a single submit, we are going to try to clear a range of up to 2Gpix + * multiple lines. Because we want to copy byte aligned we will be + * setting 1 byte pixels */ + + /* + * per iteration + * <------------------------- 40 bits ------------------------------> + * 1 <------ ffs -------> + * <-----------up to 30 bits-----------> + */ + while (chunk) { + u32 width, height, shift; + + /* + * We will be aligning to bytes, making the maximum number of + * pix per line 2Gb + */ + + shift = MAX_CE_ALIGN(chunk) ? __ffs(MAX_CE_ALIGN(chunk)) : + MAX_CE_SHIFT; + height = chunk >> shift; + width = 1 << shift; + height = MAX_CE_ALIGN(height); + + chunk_size = (u64) height * width; + + /* reset launch flag */ + launch = 0; + + if (request_operation & NVGPU_CE_PHYS_MODE_TRANSFER) { + /* setup the source */ + cmd_buf_cpu_va[methodSize++] = 0x20028100; + cmd_buf_cpu_va[methodSize++] = (u64_hi32(src_buf + + offset) & NVGPU_CE_UPPER_ADDRESS_OFFSET_MASK); + cmd_buf_cpu_va[methodSize++] = (u64_lo32(src_buf + + offset) & NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK); + + cmd_buf_cpu_va[methodSize++] = 0x20018098; + if (launch_flags & NVGPU_CE_SRC_LOCATION_LOCAL_FB) { + cmd_buf_cpu_va[methodSize++] = 0x00000000; + } else if (launch_flags & + NVGPU_CE_SRC_LOCATION_NONCOHERENT_SYSMEM) { + cmd_buf_cpu_va[methodSize++] = 0x00000002; + } else { + cmd_buf_cpu_va[methodSize++] = 0x00000001; + } + + launch |= 0x00001000; + } else if (request_operation & NVGPU_CE_MEMSET) { + /* Remap from component A on 1 byte wide pixels */ + cmd_buf_cpu_va[methodSize++] = 0x200181c2; + cmd_buf_cpu_va[methodSize++] = 0x00000004; + + cmd_buf_cpu_va[methodSize++] = 0x200181c0; + cmd_buf_cpu_va[methodSize++] = payload; + + launch |= 0x00000400; + } else { + /* Illegal size */ + return 0; + } + + /* setup the destination/output */ + cmd_buf_cpu_va[methodSize++] = 0x20068102; + cmd_buf_cpu_va[methodSize++] = (u64_hi32(dst_buf + + offset) & NVGPU_CE_UPPER_ADDRESS_OFFSET_MASK); + cmd_buf_cpu_va[methodSize++] = (u64_lo32(dst_buf + + offset) & NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK); + /* Pitch in/out */ + cmd_buf_cpu_va[methodSize++] = width; + cmd_buf_cpu_va[methodSize++] = width; + /* width and line count */ + cmd_buf_cpu_va[methodSize++] = width; + cmd_buf_cpu_va[methodSize++] = height; + + cmd_buf_cpu_va[methodSize++] = 0x20018099; + if (launch_flags & NVGPU_CE_DST_LOCATION_LOCAL_FB) { + cmd_buf_cpu_va[methodSize++] = 0x00000000; + } else if (launch_flags & + NVGPU_CE_DST_LOCATION_NONCOHERENT_SYSMEM) { + cmd_buf_cpu_va[methodSize++] = 0x00000002; + } else { + cmd_buf_cpu_va[methodSize++] = 0x00000001; + } + + launch |= 0x00002005; + + if (launch_flags & NVGPU_CE_SRC_MEMORY_LAYOUT_BLOCKLINEAR) { + launch |= 0x00000000; + } else { + launch |= 0x00000080; + } + + if (launch_flags & NVGPU_CE_DST_MEMORY_LAYOUT_BLOCKLINEAR) { + launch |= 0x00000000; + } else { + launch |= 0x00000100; + } + + cmd_buf_cpu_va[methodSize++] = 0x200180c0; + cmd_buf_cpu_va[methodSize++] = launch; + offset += chunk_size; + chunk -= chunk_size; + } + + return methodSize; +} + +/* global CE app related apis */ +int gk20a_init_ce_support(struct gk20a *g) +{ + struct gk20a_ce_app *ce_app = &g->ce_app; + int err; + u32 ce_reset_mask; + + ce_reset_mask = gk20a_fifo_get_all_ce_engine_reset_mask(g); + + g->ops.mc.reset(g, ce_reset_mask); + + nvgpu_cg_slcg_ce2_load_enable(g); + + nvgpu_cg_blcg_ce_load_enable(g); + + if (ce_app->initialised) { + /* assume this happen during poweron/poweroff GPU sequence */ + ce_app->app_state = NVGPU_CE_ACTIVE; + return 0; + } + + nvgpu_log(g, gpu_dbg_fn, "ce: init"); + + err = nvgpu_mutex_init(&ce_app->app_mutex); + if (err) { + return err; + } + + nvgpu_mutex_acquire(&ce_app->app_mutex); + + nvgpu_init_list_node(&ce_app->allocated_contexts); + ce_app->ctx_count = 0; + ce_app->next_ctx_id = 0; + ce_app->initialised = true; + ce_app->app_state = NVGPU_CE_ACTIVE; + + nvgpu_mutex_release(&ce_app->app_mutex); + + if (g->ops.ce2.init_prod_values != NULL) { + g->ops.ce2.init_prod_values(g); + } + + nvgpu_log(g, gpu_dbg_cde_ctx, "ce: init finished"); + + return 0; +} + +void gk20a_ce_destroy(struct gk20a *g) +{ + struct gk20a_ce_app *ce_app = &g->ce_app; + struct gk20a_gpu_ctx *ce_ctx, *ce_ctx_save; + + if (!ce_app->initialised) { + return; + } + + ce_app->app_state = NVGPU_CE_SUSPEND; + ce_app->initialised = false; + + nvgpu_mutex_acquire(&ce_app->app_mutex); + + nvgpu_list_for_each_entry_safe(ce_ctx, ce_ctx_save, + &ce_app->allocated_contexts, gk20a_gpu_ctx, list) { + gk20a_ce_delete_gpu_context(ce_ctx); + } + + nvgpu_init_list_node(&ce_app->allocated_contexts); + ce_app->ctx_count = 0; + ce_app->next_ctx_id = 0; + + nvgpu_mutex_release(&ce_app->app_mutex); + + nvgpu_mutex_destroy(&ce_app->app_mutex); +} + +void gk20a_ce_suspend(struct gk20a *g) +{ + struct gk20a_ce_app *ce_app = &g->ce_app; + + if (!ce_app->initialised) { + return; + } + + ce_app->app_state = NVGPU_CE_SUSPEND; + + return; +} + +/* CE app utility functions */ +u32 gk20a_ce_create_context(struct gk20a *g, + int runlist_id, + int timeslice, + int runlist_level) +{ + struct gk20a_gpu_ctx *ce_ctx; + struct gk20a_ce_app *ce_app = &g->ce_app; + struct nvgpu_setup_bind_args setup_bind_args; + u32 ctx_id = ~0; + int err = 0; + + if (!ce_app->initialised || ce_app->app_state != NVGPU_CE_ACTIVE) { + return ctx_id; + } + + ce_ctx = nvgpu_kzalloc(g, sizeof(*ce_ctx)); + if (!ce_ctx) { + return ctx_id; + } + + err = nvgpu_mutex_init(&ce_ctx->gpu_ctx_mutex); + if (err) { + nvgpu_kfree(g, ce_ctx); + return ctx_id; + } + + ce_ctx->g = g; + + ce_ctx->cmd_buf_read_queue_offset = 0; + + ce_ctx->vm = g->mm.ce.vm; + + /* allocate a tsg if needed */ + ce_ctx->tsg = gk20a_tsg_open(g, nvgpu_current_pid(g)); + if (!ce_ctx->tsg) { + nvgpu_err(g, "ce: gk20a tsg not available"); + err = -ENOMEM; + goto end; + } + + /* always kernel client needs privileged channel */ + ce_ctx->ch = gk20a_open_new_channel(g, runlist_id, true, + nvgpu_current_pid(g), nvgpu_current_tid(g)); + if (!ce_ctx->ch) { + nvgpu_err(g, "ce: gk20a channel not available"); + err = -ENOMEM; + goto end; + } + ce_ctx->ch->timeout.enabled = false; + + /* bind the channel to the vm */ + err = g->ops.mm.vm_bind_channel(g->mm.ce.vm, ce_ctx->ch); + if (err) { + nvgpu_err(g, "ce: could not bind vm"); + goto end; + } + + err = gk20a_tsg_bind_channel(ce_ctx->tsg, ce_ctx->ch); + if (err) { + nvgpu_err(g, "ce: unable to bind to tsg"); + goto end; + } + + setup_bind_args.num_gpfifo_entries = 1024; + setup_bind_args.num_inflight_jobs = 0; + setup_bind_args.flags = 0; + /* allocate gpfifo (1024 should be more than enough) */ + err = nvgpu_channel_setup_bind(ce_ctx->ch, &setup_bind_args); + if (err) { + nvgpu_err(g, "ce: unable to setup and bind channel"); + goto end; + } + + /* allocate command buffer from sysmem */ + err = nvgpu_dma_alloc_map_sys(ce_ctx->vm, + NVGPU_CE_MAX_INFLIGHT_JOBS * + NVGPU_CE_MAX_COMMAND_BUFF_BYTES_PER_KICKOFF, + &ce_ctx->cmd_buf_mem); + if (err) { + nvgpu_err(g, + "ce: could not allocate command buffer for CE context"); + goto end; + } + + memset(ce_ctx->cmd_buf_mem.cpu_va, 0x00, ce_ctx->cmd_buf_mem.size); + + /* -1 means default channel timeslice value */ + if (timeslice != -1) { + err = gk20a_fifo_tsg_set_timeslice(ce_ctx->tsg, timeslice); + if (err) { + nvgpu_err(g, + "ce: could not set the channel timeslice value for CE context"); + goto end; + } + } + + /* -1 means default channel runlist level */ + if (runlist_level != -1) { + err = gk20a_tsg_set_runlist_interleave(ce_ctx->tsg, + runlist_level); + if (err) { + nvgpu_err(g, + "ce: could not set the runlist interleave for CE context"); + goto end; + } + } + + nvgpu_mutex_acquire(&ce_app->app_mutex); + ctx_id = ce_ctx->ctx_id = ce_app->next_ctx_id; + nvgpu_list_add(&ce_ctx->list, &ce_app->allocated_contexts); + ++ce_app->next_ctx_id; + ++ce_app->ctx_count; + nvgpu_mutex_release(&ce_app->app_mutex); + + ce_ctx->gpu_ctx_state = NVGPU_CE_GPU_CTX_ALLOCATED; + +end: + if (ctx_id == (u32)~0) { + nvgpu_mutex_acquire(&ce_app->app_mutex); + gk20a_ce_delete_gpu_context(ce_ctx); + nvgpu_mutex_release(&ce_app->app_mutex); + } + return ctx_id; + +} + +void gk20a_ce_delete_context(struct gk20a *g, + u32 ce_ctx_id) +{ + gk20a_ce_delete_context_priv(g, ce_ctx_id); +} + +void gk20a_ce_delete_context_priv(struct gk20a *g, + u32 ce_ctx_id) +{ + struct gk20a_ce_app *ce_app = &g->ce_app; + struct gk20a_gpu_ctx *ce_ctx, *ce_ctx_save; + + if (!ce_app->initialised || ce_app->app_state != NVGPU_CE_ACTIVE) { + return; + } + + nvgpu_mutex_acquire(&ce_app->app_mutex); + + nvgpu_list_for_each_entry_safe(ce_ctx, ce_ctx_save, + &ce_app->allocated_contexts, gk20a_gpu_ctx, list) { + if (ce_ctx->ctx_id == ce_ctx_id) { + gk20a_ce_delete_gpu_context(ce_ctx); + --ce_app->ctx_count; + break; + } + } + + nvgpu_mutex_release(&ce_app->app_mutex); + return; +} diff --git a/include/gk20a/ce2_gk20a.h b/include/gk20a/ce2_gk20a.h new file mode 100644 index 0000000..df3a0e8 --- /dev/null +++ b/include/gk20a/ce2_gk20a.h @@ -0,0 +1,156 @@ +/* + * drivers/video/tegra/host/gk20a/fifo_gk20a.h + * + * GK20A graphics copy engine (gr host) + * + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef NVGPU_GK20A_CE2_GK20A_H +#define NVGPU_GK20A_CE2_GK20A_H + +struct channel_gk20a; +struct tsg_gk20a; + +void gk20a_ce2_isr(struct gk20a *g, u32 inst_id, u32 pri_base); +u32 gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base); + +/* CE command utility macros */ +#define NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK 0xffffffff +#define NVGPU_CE_UPPER_ADDRESS_OFFSET_MASK 0xff + +#define NVGPU_CE_MAX_INFLIGHT_JOBS 32 +#define NVGPU_CE_MAX_COMMAND_BUFF_BYTES_PER_KICKOFF 256 + +/* dma launch_flags */ +enum { + /* location */ + NVGPU_CE_SRC_LOCATION_COHERENT_SYSMEM = (1 << 0), + NVGPU_CE_SRC_LOCATION_NONCOHERENT_SYSMEM = (1 << 1), + NVGPU_CE_SRC_LOCATION_LOCAL_FB = (1 << 2), + NVGPU_CE_DST_LOCATION_COHERENT_SYSMEM = (1 << 3), + NVGPU_CE_DST_LOCATION_NONCOHERENT_SYSMEM = (1 << 4), + NVGPU_CE_DST_LOCATION_LOCAL_FB = (1 << 5), + + /* memory layout */ + NVGPU_CE_SRC_MEMORY_LAYOUT_PITCH = (1 << 6), + NVGPU_CE_SRC_MEMORY_LAYOUT_BLOCKLINEAR = (1 << 7), + NVGPU_CE_DST_MEMORY_LAYOUT_PITCH = (1 << 8), + NVGPU_CE_DST_MEMORY_LAYOUT_BLOCKLINEAR = (1 << 9), + + /* transfer type */ + NVGPU_CE_DATA_TRANSFER_TYPE_PIPELINED = (1 << 10), + NVGPU_CE_DATA_TRANSFER_TYPE_NON_PIPELINED = (1 << 11), +}; + +/* CE operation mode */ +enum { + NVGPU_CE_PHYS_MODE_TRANSFER = (1 << 0), + NVGPU_CE_MEMSET = (1 << 1), +}; + +/* CE app state machine flags */ +enum { + NVGPU_CE_ACTIVE = (1 << 0), + NVGPU_CE_SUSPEND = (1 << 1), +}; + +/* gpu context state machine flags */ +enum { + NVGPU_CE_GPU_CTX_ALLOCATED = (1 << 0), + NVGPU_CE_GPU_CTX_DELETED = (1 << 1), +}; + +/* global ce app db */ +struct gk20a_ce_app { + bool initialised; + struct nvgpu_mutex app_mutex; + int app_state; + + struct nvgpu_list_node allocated_contexts; + u32 ctx_count; + u32 next_ctx_id; +}; + +/* ce context db */ +struct gk20a_gpu_ctx { + struct gk20a *g; + u32 ctx_id; + struct nvgpu_mutex gpu_ctx_mutex; + int gpu_ctx_state; + + /* tsg related data */ + struct tsg_gk20a *tsg; + + /* channel related data */ + struct channel_gk20a *ch; + struct vm_gk20a *vm; + + /* cmd buf mem_desc */ + struct nvgpu_mem cmd_buf_mem; + struct gk20a_fence *postfences[NVGPU_CE_MAX_INFLIGHT_JOBS]; + + struct nvgpu_list_node list; + + u32 cmd_buf_read_queue_offset; +}; + +static inline struct gk20a_gpu_ctx * +gk20a_gpu_ctx_from_list(struct nvgpu_list_node *node) +{ + return (struct gk20a_gpu_ctx *) + ((uintptr_t)node - offsetof(struct gk20a_gpu_ctx, list)); +}; + +/* global CE app related apis */ +int gk20a_init_ce_support(struct gk20a *g); +void gk20a_ce_suspend(struct gk20a *g); +void gk20a_ce_destroy(struct gk20a *g); + +/* CE app utility functions */ +u32 gk20a_ce_create_context(struct gk20a *g, + int runlist_id, + int timeslice, + int runlist_level); +int gk20a_ce_execute_ops(struct gk20a *g, + u32 ce_ctx_id, + u64 src_buf, + u64 dst_buf, + u64 size, + unsigned int payload, + int launch_flags, + int request_operation, + u32 submit_flags, + struct gk20a_fence **gk20a_fence_out); +void gk20a_ce_delete_context_priv(struct gk20a *g, + u32 ce_ctx_id); +void gk20a_ce_delete_context(struct gk20a *g, + u32 ce_ctx_id); +int gk20a_ce_prepare_submit(u64 src_buf, + u64 dst_buf, + u64 size, + u32 *cmd_buf_cpu_va, + u32 max_cmd_buf_size, + unsigned int payload, + int launch_flags, + int request_operation, + u32 dma_copy_class); + +#endif /*NVGPU_GK20A_CE2_GK20A_H*/ diff --git a/include/gk20a/clk_gk20a.h b/include/gk20a/clk_gk20a.h new file mode 100644 index 0000000..b8ec942 --- /dev/null +++ b/include/gk20a/clk_gk20a.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2011 - 2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef CLK_GK20A_H +#define CLK_GK20A_H + +#include + +#if defined(CONFIG_COMMON_CLK) +#include +#endif + +#define GPUFREQ_TABLE_END ~(u32)1 +enum { + /* only one PLL for gk20a */ + GK20A_GPC_PLL = 0, + /* 2 PLL revisions for gm20b */ + GM20B_GPC_PLL_B1, + GM20B_GPC_PLL_C1, +}; + +enum gpc_pll_mode { + GPC_PLL_MODE_F = 0, /* fixed frequency mode a.k.a legacy mode */ + GPC_PLL_MODE_DVFS, /* DVFS mode a.k.a NA mode */ +}; + +struct na_dvfs { + u32 n_int; + u32 sdm_din; + int dfs_coeff; + int dfs_det_max; + int dfs_ext_cal; + int uv_cal; + int mv; +}; + +struct pll { + u32 id; + u32 clk_in; /* KHz */ + u32 M; + u32 N; + u32 PL; + u32 freq; /* KHz */ + bool enabled; + enum gpc_pll_mode mode; + struct na_dvfs dvfs; +}; + +struct pll_parms { + u32 min_freq, max_freq; /* KHz */ + u32 min_vco, max_vco; /* KHz */ + u32 min_u, max_u; /* KHz */ + u32 min_M, max_M; + u32 min_N, max_N; + u32 min_PL, max_PL; + /* NA mode parameters*/ + int coeff_slope, coeff_offs; /* coeff = slope * V + offs */ + int uvdet_slope, uvdet_offs; /* uV = slope * det + offs */ + u32 vco_ctrl; + /* + * Timing parameters in us. Lock timeout is applied to locking in fixed + * frequency mode and to dynamic ramp in any mode; does not affect lock + * latency, since lock/ramp done status bit is polled. NA mode lock and + * and IDDQ exit delays set the time of the respective opertaions with + * no status polling. + */ + u32 lock_timeout; + u32 na_lock_delay; + u32 iddq_exit_delay; + /* NA mode DFS control */ + u32 dfs_ctrl; +}; + +struct namemap_cfg; + +struct clk_gk20a { + struct gk20a *g; +#if defined(CONFIG_COMMON_CLK) + struct clk *tegra_clk; + struct clk *tegra_clk_parent; + struct clk_hw hw; +#endif + struct pll gpc_pll; + struct pll gpc_pll_last; + struct nvgpu_mutex clk_mutex; + struct namemap_cfg *clk_namemap; + u32 namemap_num; + u32 *namemap_xlat_table; + bool sw_ready; + bool clk_hw_on; + bool debugfs_set; + int pll_poweron_uv; + unsigned long dvfs_safe_max_freq; +}; + +#if defined(CONFIG_COMMON_CLK) +#define to_clk_gk20a(_hw) container_of(_hw, struct clk_gk20a, hw) +#endif + +struct gpu_ops; + +#define KHZ 1000 +#define MHZ 1000000 + +static inline unsigned long rate_gpc2clk_to_gpu(unsigned long rate) +{ + /* convert the kHz gpc2clk frequency to Hz gpcpll frequency */ + return (rate * KHZ) / 2; +} +static inline unsigned long rate_gpu_to_gpc2clk(unsigned long rate) +{ + /* convert the Hz gpcpll frequency to kHz gpc2clk frequency */ + return (rate * 2) / KHZ; +} + +#endif /* CLK_GK20A_H */ diff --git a/include/gk20a/css_gr_gk20a.c b/include/gk20a/css_gr_gk20a.c new file mode 100644 index 0000000..28a3d49 --- /dev/null +++ b/include/gk20a/css_gr_gk20a.c @@ -0,0 +1,636 @@ +/* + * GK20A Cycle stats snapshots support (subsystem for gr_gk20a). + * + * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gk20a.h" +#include "css_gr_gk20a.h" + +#include + +/* check client for pointed perfmon ownership */ +#define CONTAINS_PERFMON(cl, pm) \ + ((cl)->perfmon_start <= (pm) && \ + ((pm) - (cl)->perfmon_start) < (cl)->perfmon_count) + +/* address of fifo entry by offset */ +#define CSS_FIFO_ENTRY(fifo, offs) \ + ((struct gk20a_cs_snapshot_fifo_entry *)(((char *)(fifo)) + (offs))) + +/* calculate area capacity in number of fifo entries */ +#define CSS_FIFO_ENTRY_CAPACITY(s) \ + (((s) - sizeof(struct gk20a_cs_snapshot_fifo)) \ + / sizeof(struct gk20a_cs_snapshot_fifo_entry)) + +/* reserved to indicate failures with data */ +#define CSS_FIRST_PERFMON_ID 32 +/* should correlate with size of gk20a_cs_snapshot_fifo_entry::perfmon_id */ +#define CSS_MAX_PERFMON_IDS 256 + +/* reports whether the hw queue overflowed */ +bool css_hw_get_overflow_status(struct gk20a *g) +{ + const u32 st = perf_pmasys_control_membuf_status_overflowed_f(); + return st == (gk20a_readl(g, perf_pmasys_control_r()) & st); +} + +/* returns how many pending snapshot entries are pending */ +u32 css_hw_get_pending_snapshots(struct gk20a *g) +{ + return gk20a_readl(g, perf_pmasys_mem_bytes_r()) / + sizeof(struct gk20a_cs_snapshot_fifo_entry); +} + +/* informs hw how many snapshots have been processed (frees up fifo space) */ +void css_hw_set_handled_snapshots(struct gk20a *g, u32 done) +{ + if (done > 0) { + gk20a_writel(g, perf_pmasys_mem_bump_r(), + done * sizeof(struct gk20a_cs_snapshot_fifo_entry)); + } +} + +/* disable streaming to memory */ +static void css_hw_reset_streaming(struct gk20a *g) +{ + u32 engine_status; + + /* reset the perfmon */ + g->ops.mc.reset(g, g->ops.mc.reset_mask(g, NVGPU_UNIT_PERFMON)); + + /* RBUFEMPTY must be set -- otherwise we'll pick up */ + /* snapshot that have been queued up from earlier */ + engine_status = gk20a_readl(g, perf_pmasys_enginestatus_r()); + WARN_ON(0 == (engine_status + & perf_pmasys_enginestatus_rbufempty_empty_f())); + + /* turn off writes */ + gk20a_writel(g, perf_pmasys_control_r(), + perf_pmasys_control_membuf_clear_status_doit_f()); + + /* pointing all pending snapshots as handled */ + css_hw_set_handled_snapshots(g, css_hw_get_pending_snapshots(g)); +} + +/* + * WARNING: all css_gr_XXX functions are local and expected to be called + * from locked context (protected by cs_lock) + */ + +static int css_gr_create_shared_data(struct gr_gk20a *gr) +{ + struct gk20a_cs_snapshot *data; + + if (gr->cs_data) + return 0; + + data = nvgpu_kzalloc(gr->g, sizeof(*data)); + if (!data) + return -ENOMEM; + + nvgpu_init_list_node(&data->clients); + gr->cs_data = data; + + return 0; +} + +int css_hw_enable_snapshot(struct channel_gk20a *ch, + struct gk20a_cs_snapshot_client *cs_client) +{ + struct gk20a *g = ch->g; + struct mm_gk20a *mm = &g->mm; + struct gr_gk20a *gr = &g->gr; + struct gk20a_cs_snapshot *data = gr->cs_data; + u32 snapshot_size = cs_client->snapshot_size; + int ret; + + u32 virt_addr_lo; + u32 virt_addr_hi; + u32 inst_pa_page; + + if (data->hw_snapshot) + return 0; + + if (snapshot_size < CSS_MIN_HW_SNAPSHOT_SIZE) + snapshot_size = CSS_MIN_HW_SNAPSHOT_SIZE; + + ret = nvgpu_dma_alloc_map_sys(g->mm.pmu.vm, snapshot_size, + &data->hw_memdesc); + if (ret) + return ret; + + /* perf output buffer may not cross a 4GB boundary - with a separate */ + /* va smaller than that, it won't but check anyway */ + if (!data->hw_memdesc.cpu_va || + data->hw_memdesc.size < snapshot_size || + data->hw_memdesc.gpu_va + u64_lo32(snapshot_size) > SZ_4G) { + ret = -EFAULT; + goto failed_allocation; + } + + data->hw_snapshot = + (struct gk20a_cs_snapshot_fifo_entry *)data->hw_memdesc.cpu_va; + data->hw_end = data->hw_snapshot + + snapshot_size / sizeof(struct gk20a_cs_snapshot_fifo_entry); + data->hw_get = data->hw_snapshot; + memset(data->hw_snapshot, 0xff, snapshot_size); + + /* address and size are aligned to 32 bytes, the lowest bits read back + * as zeros */ + virt_addr_lo = u64_lo32(data->hw_memdesc.gpu_va); + virt_addr_hi = u64_hi32(data->hw_memdesc.gpu_va); + + css_hw_reset_streaming(g); + + gk20a_writel(g, perf_pmasys_outbase_r(), virt_addr_lo); + gk20a_writel(g, perf_pmasys_outbaseupper_r(), + perf_pmasys_outbaseupper_ptr_f(virt_addr_hi)); + gk20a_writel(g, perf_pmasys_outsize_r(), snapshot_size); + + /* this field is aligned to 4K */ + inst_pa_page = nvgpu_inst_block_addr(g, &g->mm.hwpm.inst_block) >> 12; + + /* A write to MEM_BLOCK triggers the block bind operation. MEM_BLOCK + * should be written last */ + gk20a_writel(g, perf_pmasys_mem_block_r(), + perf_pmasys_mem_block_base_f(inst_pa_page) | + nvgpu_aperture_mask(g, &mm->hwpm.inst_block, + perf_pmasys_mem_block_target_sys_ncoh_f(), + perf_pmasys_mem_block_target_sys_coh_f(), + perf_pmasys_mem_block_target_lfb_f()) | + perf_pmasys_mem_block_valid_true_f()); + + nvgpu_log_info(g, "cyclestats: buffer for hardware snapshots enabled\n"); + + return 0; + +failed_allocation: + if (data->hw_memdesc.size) { + nvgpu_dma_unmap_free(g->mm.pmu.vm, &data->hw_memdesc); + memset(&data->hw_memdesc, 0, sizeof(data->hw_memdesc)); + } + data->hw_snapshot = NULL; + + return ret; +} + +void css_hw_disable_snapshot(struct gr_gk20a *gr) +{ + struct gk20a *g = gr->g; + struct gk20a_cs_snapshot *data = gr->cs_data; + + if (!data->hw_snapshot) + return; + + css_hw_reset_streaming(g); + + gk20a_writel(g, perf_pmasys_outbase_r(), 0); + gk20a_writel(g, perf_pmasys_outbaseupper_r(), + perf_pmasys_outbaseupper_ptr_f(0)); + gk20a_writel(g, perf_pmasys_outsize_r(), 0); + + gk20a_writel(g, perf_pmasys_mem_block_r(), + perf_pmasys_mem_block_base_f(0) | + perf_pmasys_mem_block_valid_false_f() | + perf_pmasys_mem_block_target_f(0)); + + nvgpu_dma_unmap_free(g->mm.pmu.vm, &data->hw_memdesc); + memset(&data->hw_memdesc, 0, sizeof(data->hw_memdesc)); + data->hw_snapshot = NULL; + + nvgpu_log_info(g, "cyclestats: buffer for hardware snapshots disabled\n"); +} + +static void css_gr_free_shared_data(struct gr_gk20a *gr) +{ + struct gk20a *g = gr->g; + + if (gr->cs_data) { + /* the clients list is expected to be empty */ + g->ops.css.disable_snapshot(gr); + + /* release the objects */ + nvgpu_kfree(gr->g, gr->cs_data); + gr->cs_data = NULL; + } +} + + +struct gk20a_cs_snapshot_client* +css_gr_search_client(struct nvgpu_list_node *clients, u32 perfmon) +{ + struct gk20a_cs_snapshot_client *client; + + nvgpu_list_for_each_entry(client, clients, + gk20a_cs_snapshot_client, list) { + if (CONTAINS_PERFMON(client, perfmon)) + return client; + } + + return NULL; +} + +static int css_gr_flush_snapshots(struct channel_gk20a *ch) +{ + struct gk20a *g = ch->g; + struct gr_gk20a *gr = &g->gr; + struct gk20a_cs_snapshot *css = gr->cs_data; + struct gk20a_cs_snapshot_client *cur; + u32 pending, completed; + bool hw_overflow; + int err; + + /* variables for iterating over HW entries */ + u32 sid; + struct gk20a_cs_snapshot_fifo_entry *src; + + /* due to data sharing with userspace we allowed update only */ + /* overflows and put field in the fifo header */ + struct gk20a_cs_snapshot_fifo *dst; + struct gk20a_cs_snapshot_fifo_entry *dst_get; + struct gk20a_cs_snapshot_fifo_entry *dst_put; + struct gk20a_cs_snapshot_fifo_entry *dst_nxt; + struct gk20a_cs_snapshot_fifo_entry *dst_head; + struct gk20a_cs_snapshot_fifo_entry *dst_tail; + + if (!css) + return -EINVAL; + + if (nvgpu_list_empty(&css->clients)) + return -EBADF; + + /* check data available */ + err = g->ops.css.check_data_available(ch, &pending, &hw_overflow); + if (err) + return err; + + if (!pending) + return 0; + + if (hw_overflow) { + nvgpu_list_for_each_entry(cur, &css->clients, + gk20a_cs_snapshot_client, list) { + cur->snapshot->hw_overflow_events_occured++; + } + + nvgpu_warn(g, "cyclestats: hardware overflow detected"); + } + + /* process all items in HW buffer */ + sid = 0; + completed = 0; + cur = NULL; + dst = NULL; + dst_put = NULL; + src = css->hw_get; + + /* proceed all completed records */ + while (sid < pending && 0 == src->zero0) { + /* we may have a new perfmon_id which required to */ + /* switch to a new client -> let's forget current */ + if (cur && !CONTAINS_PERFMON(cur, src->perfmon_id)) { + dst->put = (char *)dst_put - (char *)dst; + dst = NULL; + cur = NULL; + } + + /* now we have to select a new current client */ + /* the client selection rate depends from experiment */ + /* activity but on Android usually happened 1-2 times */ + if (!cur) { + cur = css_gr_search_client(&css->clients, + src->perfmon_id); + if (cur) { + /* found - setup all required data */ + dst = cur->snapshot; + dst_get = CSS_FIFO_ENTRY(dst, dst->get); + dst_put = CSS_FIFO_ENTRY(dst, dst->put); + dst_head = CSS_FIFO_ENTRY(dst, dst->start); + dst_tail = CSS_FIFO_ENTRY(dst, dst->end); + + dst_nxt = dst_put + 1; + if (dst_nxt == dst_tail) + dst_nxt = dst_head; + } else { + /* client not found - skipping this entry */ + nvgpu_warn(g, "cyclestats: orphaned perfmon %u", + src->perfmon_id); + goto next_hw_fifo_entry; + } + } + + /* check for software overflows */ + if (dst_nxt == dst_get) { + /* no data copy, no pointer updates */ + dst->sw_overflow_events_occured++; + nvgpu_warn(g, "cyclestats: perfmon %u soft overflow", + src->perfmon_id); + } else { + *dst_put = *src; + completed++; + + dst_put = dst_nxt++; + + if (dst_nxt == dst_tail) + dst_nxt = dst_head; + } + +next_hw_fifo_entry: + sid++; + if (++src >= css->hw_end) + src = css->hw_snapshot; + } + + /* update client put pointer if necessary */ + if (cur && dst) + dst->put = (char *)dst_put - (char *)dst; + + /* re-set HW buffer after processing taking wrapping into account */ + if (css->hw_get < src) { + memset(css->hw_get, 0xff, (src - css->hw_get) * sizeof(*src)); + } else { + memset(css->hw_snapshot, 0xff, + (src - css->hw_snapshot) * sizeof(*src)); + memset(css->hw_get, 0xff, + (css->hw_end - css->hw_get) * sizeof(*src)); + } + gr->cs_data->hw_get = src; + + if (g->ops.css.set_handled_snapshots) + g->ops.css.set_handled_snapshots(g, sid); + + if (completed != sid) { + /* not all entries proceed correctly. some of problems */ + /* reported as overflows, some as orphaned perfmons, */ + /* but it will be better notify with summary about it */ + nvgpu_warn(g, "cyclestats: completed %u from %u entries", + completed, pending); + } + + return 0; +} + +u32 css_gr_allocate_perfmon_ids(struct gk20a_cs_snapshot *data, + u32 count) +{ + unsigned long *pids = data->perfmon_ids; + unsigned int f; + + f = bitmap_find_next_zero_area(pids, CSS_MAX_PERFMON_IDS, + CSS_FIRST_PERFMON_ID, count, 0); + if (f > CSS_MAX_PERFMON_IDS) + f = 0; + else + bitmap_set(pids, f, count); + + return f; +} + +u32 css_gr_release_perfmon_ids(struct gk20a_cs_snapshot *data, + u32 start, + u32 count) +{ + unsigned long *pids = data->perfmon_ids; + u32 end = start + count; + u32 cnt = 0; + + if (start >= CSS_FIRST_PERFMON_ID && end <= CSS_MAX_PERFMON_IDS) { + bitmap_clear(pids, start, count); + cnt = count; + } + + return cnt; +} + + +static int css_gr_free_client_data(struct gk20a *g, + struct gk20a_cs_snapshot *data, + struct gk20a_cs_snapshot_client *client) +{ + int ret = 0; + + if (client->list.next && client->list.prev) + nvgpu_list_del(&client->list); + + if (client->perfmon_start && client->perfmon_count + && g->ops.css.release_perfmon_ids) { + if (client->perfmon_count != g->ops.css.release_perfmon_ids(data, + client->perfmon_start, client->perfmon_count)) + ret = -EINVAL; + } + + return ret; +} + +static int css_gr_create_client_data(struct gk20a *g, + struct gk20a_cs_snapshot *data, + u32 perfmon_count, + struct gk20a_cs_snapshot_client *cur) +{ + /* + * Special handling in-case of rm-server + * + * client snapshot buffer will not be mapped + * in-case of rm-server its only mapped in + * guest side + */ + if (cur->snapshot) { + memset(cur->snapshot, 0, sizeof(*cur->snapshot)); + cur->snapshot->start = sizeof(*cur->snapshot); + /* we should be ensure that can fit all fifo entries here */ + cur->snapshot->end = + CSS_FIFO_ENTRY_CAPACITY(cur->snapshot_size) + * sizeof(struct gk20a_cs_snapshot_fifo_entry) + + sizeof(struct gk20a_cs_snapshot_fifo); + cur->snapshot->get = cur->snapshot->start; + cur->snapshot->put = cur->snapshot->start; + } + + cur->perfmon_count = perfmon_count; + + /* In virtual case, perfmon ID allocation is handled by the server + * at the time of the attach (allocate_perfmon_ids is NULL in this case) + */ + if (cur->perfmon_count && g->ops.css.allocate_perfmon_ids) { + cur->perfmon_start = g->ops.css.allocate_perfmon_ids(data, + cur->perfmon_count); + if (!cur->perfmon_start) + return -ENOENT; + } + + nvgpu_list_add_tail(&cur->list, &data->clients); + + return 0; +} + + +int gr_gk20a_css_attach(struct channel_gk20a *ch, + u32 perfmon_count, + u32 *perfmon_start, + struct gk20a_cs_snapshot_client *cs_client) +{ + int ret = 0; + struct gk20a *g = ch->g; + struct gr_gk20a *gr; + + /* we must have a placeholder to store pointer to client structure */ + if (!cs_client) + return -EINVAL; + + if (!perfmon_count || + perfmon_count > CSS_MAX_PERFMON_IDS - CSS_FIRST_PERFMON_ID) + return -EINVAL; + + nvgpu_speculation_barrier(); + + gr = &g->gr; + + nvgpu_mutex_acquire(&gr->cs_lock); + + ret = css_gr_create_shared_data(gr); + if (ret) + goto failed; + + ret = css_gr_create_client_data(g, gr->cs_data, + perfmon_count, + cs_client); + if (ret) + goto failed; + + ret = g->ops.css.enable_snapshot(ch, cs_client); + if (ret) + goto failed; + + if (perfmon_start) + *perfmon_start = cs_client->perfmon_start; + + nvgpu_mutex_release(&gr->cs_lock); + + return 0; + +failed: + if (gr->cs_data) { + if (cs_client) { + css_gr_free_client_data(g, gr->cs_data, cs_client); + cs_client = NULL; + } + + if (nvgpu_list_empty(&gr->cs_data->clients)) + css_gr_free_shared_data(gr); + } + nvgpu_mutex_release(&gr->cs_lock); + + if (perfmon_start) + *perfmon_start = 0; + + return ret; +} + +int gr_gk20a_css_detach(struct channel_gk20a *ch, + struct gk20a_cs_snapshot_client *cs_client) +{ + int ret = 0; + struct gk20a *g = ch->g; + struct gr_gk20a *gr; + + if (!cs_client) + return -EINVAL; + + gr = &g->gr; + nvgpu_mutex_acquire(&gr->cs_lock); + if (gr->cs_data) { + struct gk20a_cs_snapshot *data = gr->cs_data; + + if (g->ops.css.detach_snapshot) + g->ops.css.detach_snapshot(ch, cs_client); + + ret = css_gr_free_client_data(g, data, cs_client); + if (nvgpu_list_empty(&data->clients)) + css_gr_free_shared_data(gr); + } else { + ret = -EBADF; + } + nvgpu_mutex_release(&gr->cs_lock); + + return ret; +} + +int gr_gk20a_css_flush(struct channel_gk20a *ch, + struct gk20a_cs_snapshot_client *cs_client) +{ + int ret = 0; + struct gk20a *g = ch->g; + struct gr_gk20a *gr; + + if (!cs_client) + return -EINVAL; + + gr = &g->gr; + nvgpu_mutex_acquire(&gr->cs_lock); + ret = css_gr_flush_snapshots(ch); + nvgpu_mutex_release(&gr->cs_lock); + + return ret; +} + +/* helper function with locking to cleanup snapshot code code in gr_gk20a.c */ +void gr_gk20a_free_cyclestats_snapshot_data(struct gk20a *g) +{ + struct gr_gk20a *gr = &g->gr; + + nvgpu_mutex_acquire(&gr->cs_lock); + css_gr_free_shared_data(gr); + nvgpu_mutex_release(&gr->cs_lock); + nvgpu_mutex_destroy(&gr->cs_lock); +} + +int css_hw_check_data_available(struct channel_gk20a *ch, u32 *pending, + bool *hw_overflow) +{ + struct gk20a *g = ch->g; + struct gr_gk20a *gr = &g->gr; + struct gk20a_cs_snapshot *css = gr->cs_data; + + if (!css->hw_snapshot) + return -EINVAL; + + *pending = css_hw_get_pending_snapshots(g); + if (!*pending) + return 0; + + *hw_overflow = css_hw_get_overflow_status(g); + return 0; +} diff --git a/include/gk20a/css_gr_gk20a.h b/include/gk20a/css_gr_gk20a.h new file mode 100644 index 0000000..bf8890b --- /dev/null +++ b/include/gk20a/css_gr_gk20a.h @@ -0,0 +1,151 @@ +/* + * GK20A Cycle stats snapshots support (subsystem for gr_gk20a). + * + * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef CSS_GR_GK20A_H +#define CSS_GR_GK20A_H + +#include +#include + +/* the minimal size of HW buffer - should be enough to avoid HW overflows */ +#define CSS_MIN_HW_SNAPSHOT_SIZE (8 * 1024 * 1024) + +struct gk20a; +struct gr_gk20a; +struct channel_gk20a; + +/* cycle stats fifo header (must match NvSnapshotBufferFifo) */ +struct gk20a_cs_snapshot_fifo { + /* layout description of the buffer */ + u32 start; + u32 end; + + /* snafu bits */ + u32 hw_overflow_events_occured; + u32 sw_overflow_events_occured; + + /* the kernel copies new entries to put and + * increment the put++. if put == get then + * overflowEventsOccured++ + */ + u32 put; + u32 _reserved10; + u32 _reserved11; + u32 _reserved12; + + /* the driver/client reads from get until + * put==get, get++ */ + u32 get; + u32 _reserved20; + u32 _reserved21; + u32 _reserved22; + + /* unused */ + u32 _reserved30; + u32 _reserved31; + u32 _reserved32; + u32 _reserved33; +}; + +/* cycle stats fifo entry (must match NvSnapshotBufferFifoEntry) */ +struct gk20a_cs_snapshot_fifo_entry { + /* global 48 timestamp */ + u32 timestamp31_00:32; + u32 timestamp39_32:8; + + /* id of perfmon, should correlate with CSS_MAX_PERFMON_IDS */ + u32 perfmon_id:8; + + /* typically samples_counter is wired to #pmtrigger count */ + u32 samples_counter:12; + + /* DS=Delay Sample, SZ=Size (0=32B, 1=16B) */ + u32 ds:1; + u32 sz:1; + u32 zero0:1; + u32 zero1:1; + + /* counter results */ + u32 event_cnt:32; + u32 trigger0_cnt:32; + u32 trigger1_cnt:32; + u32 sample_cnt:32; + + /* Local PmTrigger results for Maxwell+ or padding otherwise */ + u16 local_trigger_b_count:16; + u16 book_mark_b:16; + u16 local_trigger_a_count:16; + u16 book_mark_a:16; +}; + +/* cycle stats snapshot client data (e.g. associated with channel) */ +struct gk20a_cs_snapshot_client { + struct nvgpu_list_node list; + struct gk20a_cs_snapshot_fifo *snapshot; + u32 snapshot_size; + u32 perfmon_start; + u32 perfmon_count; +}; + +static inline struct gk20a_cs_snapshot_client * +gk20a_cs_snapshot_client_from_list(struct nvgpu_list_node *node) +{ + return (struct gk20a_cs_snapshot_client *) + ((uintptr_t)node - offsetof(struct gk20a_cs_snapshot_client, list)); +}; + +/* should correlate with size of gk20a_cs_snapshot_fifo_entry::perfmon_id */ +#define CSS_MAX_PERFMON_IDS 256 + +/* local definitions to avoid hardcodes sizes and shifts */ +#define PM_BITMAP_SIZE DIV_ROUND_UP(CSS_MAX_PERFMON_IDS, BITS_PER_LONG) + +/* cycle stats snapshot control structure for one HW entry and many clients */ +struct gk20a_cs_snapshot { + unsigned long perfmon_ids[PM_BITMAP_SIZE]; + struct nvgpu_list_node clients; + struct nvgpu_mem hw_memdesc; + /* pointer to allocated cpu_va memory where GPU place data */ + struct gk20a_cs_snapshot_fifo_entry *hw_snapshot; + struct gk20a_cs_snapshot_fifo_entry *hw_end; + struct gk20a_cs_snapshot_fifo_entry *hw_get; +}; + +bool css_hw_get_overflow_status(struct gk20a *g); +u32 css_hw_get_pending_snapshots(struct gk20a *g); +void css_hw_set_handled_snapshots(struct gk20a *g, u32 done); +int css_hw_enable_snapshot(struct channel_gk20a *ch, + struct gk20a_cs_snapshot_client *cs_client); +void css_hw_disable_snapshot(struct gr_gk20a *gr); +u32 css_gr_allocate_perfmon_ids(struct gk20a_cs_snapshot *data, + u32 count); +u32 css_gr_release_perfmon_ids(struct gk20a_cs_snapshot *data, + u32 start, + u32 count); +int css_hw_check_data_available(struct channel_gk20a *ch, u32 *pending, + bool *hw_overflow); +struct gk20a_cs_snapshot_client* +css_gr_search_client(struct nvgpu_list_node *clients, u32 perfmon); + +#endif /* CSS_GR_GK20A_H */ diff --git a/include/gk20a/dbg_gpu_gk20a.c b/include/gk20a/dbg_gpu_gk20a.c new file mode 100644 index 0000000..1686d01 --- /dev/null +++ b/include/gk20a/dbg_gpu_gk20a.c @@ -0,0 +1,388 @@ +/* + * Tegra GK20A GPU Debugger/Profiler Driver + * + * Copyright (c) 2013-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gk20a.h" +#include "gr_gk20a.h" +#include "dbg_gpu_gk20a.h" +#include "regops_gk20a.h" + +#include +#include + +static void gk20a_perfbuf_reset_streaming(struct gk20a *g) +{ + u32 engine_status; + u32 num_unread_bytes; + + g->ops.mc.reset(g, g->ops.mc.reset_mask(g, NVGPU_UNIT_PERFMON)); + + engine_status = gk20a_readl(g, perf_pmasys_enginestatus_r()); + WARN_ON(0u == + (engine_status & perf_pmasys_enginestatus_rbufempty_empty_f())); + + gk20a_writel(g, perf_pmasys_control_r(), + perf_pmasys_control_membuf_clear_status_doit_f()); + + num_unread_bytes = gk20a_readl(g, perf_pmasys_mem_bytes_r()); + if (num_unread_bytes != 0u) { + gk20a_writel(g, perf_pmasys_mem_bump_r(), num_unread_bytes); + } +} + +/* + * API to get first channel from the list of all channels + * bound to the debug session + */ +struct channel_gk20a * +nvgpu_dbg_gpu_get_session_channel(struct dbg_session_gk20a *dbg_s) +{ + struct dbg_session_channel_data *ch_data; + struct channel_gk20a *ch; + struct gk20a *g = dbg_s->g; + + nvgpu_mutex_acquire(&dbg_s->ch_list_lock); + if (nvgpu_list_empty(&dbg_s->ch_list)) { + nvgpu_mutex_release(&dbg_s->ch_list_lock); + return NULL; + } + + ch_data = nvgpu_list_first_entry(&dbg_s->ch_list, + dbg_session_channel_data, + ch_entry); + ch = g->fifo.channel + ch_data->chid; + + nvgpu_mutex_release(&dbg_s->ch_list_lock); + + return ch; +} + +void gk20a_dbg_gpu_post_events(struct channel_gk20a *ch) +{ + struct dbg_session_data *session_data; + struct dbg_session_gk20a *dbg_s; + struct gk20a *g = ch->g; + + nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " "); + + /* guard against the session list being modified */ + nvgpu_mutex_acquire(&ch->dbg_s_lock); + + nvgpu_list_for_each_entry(session_data, &ch->dbg_s_list, + dbg_session_data, dbg_s_entry) { + dbg_s = session_data->dbg_s; + if (dbg_s->dbg_events.events_enabled) { + nvgpu_log(g, gpu_dbg_gpu_dbg, "posting event on session id %d", + dbg_s->id); + nvgpu_log(g, gpu_dbg_gpu_dbg, "%d events pending", + dbg_s->dbg_events.num_pending_events); + + dbg_s->dbg_events.num_pending_events++; + + nvgpu_dbg_session_post_event(dbg_s); + } + } + + nvgpu_mutex_release(&ch->dbg_s_lock); +} + +bool gk20a_dbg_gpu_broadcast_stop_trigger(struct channel_gk20a *ch) +{ + struct dbg_session_data *session_data; + struct dbg_session_gk20a *dbg_s; + bool broadcast = false; + struct gk20a *g = ch->g; + + nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg | gpu_dbg_intr, " "); + + /* guard against the session list being modified */ + nvgpu_mutex_acquire(&ch->dbg_s_lock); + + nvgpu_list_for_each_entry(session_data, &ch->dbg_s_list, + dbg_session_data, dbg_s_entry) { + dbg_s = session_data->dbg_s; + if (dbg_s->broadcast_stop_trigger) { + nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn | gpu_dbg_intr, + "stop trigger broadcast enabled"); + broadcast = true; + break; + } + } + + nvgpu_mutex_release(&ch->dbg_s_lock); + + return broadcast; +} + +int gk20a_dbg_gpu_clear_broadcast_stop_trigger(struct channel_gk20a *ch) +{ + struct dbg_session_data *session_data; + struct dbg_session_gk20a *dbg_s; + struct gk20a *g = ch->g; + + nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg | gpu_dbg_intr, " "); + + /* guard against the session list being modified */ + nvgpu_mutex_acquire(&ch->dbg_s_lock); + + nvgpu_list_for_each_entry(session_data, &ch->dbg_s_list, + dbg_session_data, dbg_s_entry) { + dbg_s = session_data->dbg_s; + if (dbg_s->broadcast_stop_trigger) { + nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn | gpu_dbg_intr, + "stop trigger broadcast disabled"); + dbg_s->broadcast_stop_trigger = false; + } + } + + nvgpu_mutex_release(&ch->dbg_s_lock); + + return 0; +} + +u32 nvgpu_set_powergate_locked(struct dbg_session_gk20a *dbg_s, + bool mode) +{ + u32 err = 0U; + struct gk20a *g = dbg_s->g; + + if (dbg_s->is_pg_disabled != mode) { + if (mode == false) { + g->dbg_powergating_disabled_refcount--; + } + + /* + * Allow powergate disable or enable only if + * the global pg disabled refcount is zero + */ + if (g->dbg_powergating_disabled_refcount == 0) { + err = g->ops.dbg_session_ops.dbg_set_powergate(dbg_s, + mode); + } + + if (mode) { + g->dbg_powergating_disabled_refcount++; + } + + dbg_s->is_pg_disabled = mode; + } + + return err; +} + +int dbg_set_powergate(struct dbg_session_gk20a *dbg_s, bool disable_powergate) +{ + int err = 0; + struct gk20a *g = dbg_s->g; + + /* This function must be called with g->dbg_sessions_lock held */ + + nvgpu_log(g, gpu_dbg_fn|gpu_dbg_gpu_dbg, "%s powergate mode = %s", + g->name, disable_powergate ? "disable" : "enable"); + + /* + * Powergate mode here refers to railgate+powergate+clockgate + * so in case slcg/blcg/elcg are disabled and railgating is enabled, + * disable railgating and then set is_pg_disabled = true + * Similarly re-enable railgating and not other features if they are not + * enabled when powermode=MODE_ENABLE + */ + if (disable_powergate) { + /* save off current powergate, clk state. + * set gpu module's can_powergate = 0. + * set gpu module's clk to max. + * while *a* debug session is active there will be no power or + * clocking state changes allowed from mainline code (but they + * should be saved). + */ + + nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, + "module busy"); + err = gk20a_busy(g); + if (err) { + return err; + } + + err = nvgpu_cg_pg_disable(g); + + if (err == 0) { + dbg_s->is_pg_disabled = true; + nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, + "pg disabled"); + } + } else { + /* restore (can) powergate, clk state */ + /* release pending exceptions to fault/be handled as usual */ + /*TBD: ordering of these? */ + + err = nvgpu_cg_pg_enable(g); + + nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, "module idle"); + gk20a_idle(g); + + if (err == 0) { + dbg_s->is_pg_disabled = false; + nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, + "pg enabled"); + } + } + + nvgpu_log(g, gpu_dbg_fn|gpu_dbg_gpu_dbg, "%s powergate mode = %s done", + g->name, disable_powergate ? "disable" : "enable"); + return err; +} + +bool nvgpu_check_and_set_global_reservation( + struct dbg_session_gk20a *dbg_s, + struct dbg_profiler_object_data *prof_obj) +{ + struct gk20a *g = dbg_s->g; + + if (g->profiler_reservation_count == 0) { + g->global_profiler_reservation_held = true; + g->profiler_reservation_count = 1; + dbg_s->has_profiler_reservation = true; + prof_obj->has_reservation = true; + return true; + } + return false; +} + +bool nvgpu_check_and_set_context_reservation( + struct dbg_session_gk20a *dbg_s, + struct dbg_profiler_object_data *prof_obj) +{ + struct gk20a *g = dbg_s->g; + + /* Assumes that we've already checked that no global reservation + * is in effect. + */ + g->profiler_reservation_count++; + dbg_s->has_profiler_reservation = true; + prof_obj->has_reservation = true; + return true; +} + +void nvgpu_release_profiler_reservation(struct dbg_session_gk20a *dbg_s, + struct dbg_profiler_object_data *prof_obj) +{ + struct gk20a *g = dbg_s->g; + + g->profiler_reservation_count--; + if (g->profiler_reservation_count < 0) { + nvgpu_err(g, "Negative reservation count!"); + } + dbg_s->has_profiler_reservation = false; + prof_obj->has_reservation = false; + if (prof_obj->ch == NULL) { + g->global_profiler_reservation_held = false; + } +} + +int gk20a_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size) +{ + struct mm_gk20a *mm = &g->mm; + u32 virt_addr_lo; + u32 virt_addr_hi; + u32 inst_pa_page; + int err; + + err = gk20a_busy(g); + if (err) { + nvgpu_err(g, "failed to poweron"); + return err; + } + + err = g->ops.mm.alloc_inst_block(g, &mm->perfbuf.inst_block); + if (err) { + return err; + } + + g->ops.mm.init_inst_block(&mm->perfbuf.inst_block, mm->perfbuf.vm, 0); + + gk20a_perfbuf_reset_streaming(g); + + virt_addr_lo = u64_lo32(offset); + virt_addr_hi = u64_hi32(offset); + + /* address and size are aligned to 32 bytes, the lowest bits read back + * as zeros */ + gk20a_writel(g, perf_pmasys_outbase_r(), virt_addr_lo); + gk20a_writel(g, perf_pmasys_outbaseupper_r(), + perf_pmasys_outbaseupper_ptr_f(virt_addr_hi)); + gk20a_writel(g, perf_pmasys_outsize_r(), size); + + /* this field is aligned to 4K */ + inst_pa_page = nvgpu_inst_block_addr(g, &mm->perfbuf.inst_block) >> 12; + + /* A write to MEM_BLOCK triggers the block bind operation. MEM_BLOCK + * should be written last */ + gk20a_writel(g, perf_pmasys_mem_block_r(), + perf_pmasys_mem_block_base_f(inst_pa_page) | + nvgpu_aperture_mask(g, &mm->perfbuf.inst_block, + perf_pmasys_mem_block_target_sys_ncoh_f(), + perf_pmasys_mem_block_target_sys_coh_f(), + perf_pmasys_mem_block_target_lfb_f()) | + perf_pmasys_mem_block_valid_true_f()); + + gk20a_idle(g); + return 0; +} + +/* must be called with dbg_sessions_lock held */ +int gk20a_perfbuf_disable_locked(struct gk20a *g) +{ + int err = gk20a_busy(g); + if (err) { + nvgpu_err(g, "failed to poweron"); + return err; + } + + gk20a_perfbuf_reset_streaming(g); + + gk20a_writel(g, perf_pmasys_outbase_r(), 0); + gk20a_writel(g, perf_pmasys_outbaseupper_r(), + perf_pmasys_outbaseupper_ptr_f(0)); + gk20a_writel(g, perf_pmasys_outsize_r(), 0); + + gk20a_writel(g, perf_pmasys_mem_block_r(), + perf_pmasys_mem_block_base_f(0) | + perf_pmasys_mem_block_valid_false_f() | + perf_pmasys_mem_block_target_f(0)); + + gk20a_idle(g); + + return 0; +} diff --git a/include/gk20a/dbg_gpu_gk20a.h b/include/gk20a/dbg_gpu_gk20a.h new f