/* * GK20A Graphics * * Copyright (c) 2011-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gk20a.h" #include "gr_gk20a.h" #include "gk20a/fecs_trace_gk20a.h" #include "gr_ctx_gk20a.h" #include "gr_pri_gk20a.h" #include "regops_gk20a.h" #include "dbg_gpu_gk20a.h" #include #include #include #include #include #include #include #include #include #include #define BLK_SIZE (256) #define NV_PERF_PMM_FBP_ROUTER_STRIDE 0x0200 #define NV_PERF_PMMGPCROUTER_STRIDE 0x0200 #define NV_PCFG_BASE 0x00088000 #define NV_XBAR_MXBAR_PRI_GPC_GNIC_STRIDE 0x0020 #define FE_PWR_MODE_TIMEOUT_MAX 2000 #define FE_PWR_MODE_TIMEOUT_DEFAULT 10 #define CTXSW_MEM_SCRUBBING_TIMEOUT_MAX 1000 #define CTXSW_MEM_SCRUBBING_TIMEOUT_DEFAULT 10 #define FECS_ARB_CMD_TIMEOUT_MAX 40 #define FECS_ARB_CMD_TIMEOUT_DEFAULT 2 static int gk20a_init_gr_bind_fecs_elpg(struct gk20a *g); static void gr_gk20a_free_channel_pm_ctx(struct gk20a *g, struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx); /* channel patch ctx buffer */ static int gr_gk20a_alloc_channel_patch_ctx(struct gk20a *g, struct channel_gk20a *c); static void gr_gk20a_free_channel_patch_ctx(struct gk20a *g, struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx); /* golden ctx image */ static int gr_gk20a_init_golden_ctx_image(struct gk20a *g, struct channel_gk20a *c); int gr_gk20a_get_ctx_id(struct gk20a *g, struct channel_gk20a *c, u32 *ctx_id) { struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx = NULL; struct nvgpu_mem *mem = NULL; tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; mem = &gr_ctx->mem; /* Channel gr_ctx buffer is gpu cacheable. Flush and invalidate before cpu update. */ g->ops.mm.l2_flush(g, true); *ctx_id = nvgpu_mem_rd(g, mem, ctxsw_prog_main_image_context_id_o()); nvgpu_log(g, gpu_dbg_fn | gpu_dbg_intr, "ctx_id: 0x%x", *ctx_id); return 0; } void gk20a_fecs_dump_falcon_stats(struct gk20a *g) { unsigned int i; nvgpu_err(g, "gr_fecs_os_r : %d", gk20a_readl(g, gr_fecs_os_r())); nvgpu_err(g, "gr_fecs_cpuctl_r : 0x%x", gk20a_readl(g, gr_fecs_cpuctl_r())); nvgpu_err(g, "gr_fecs_idlestate_r : 0x%x", gk20a_readl(g, gr_fecs_idlestate_r())); nvgpu_err(g, "gr_fecs_mailbox0_r : 0x%x", gk20a_readl(g, gr_fecs_mailbox0_r())); nvgpu_err(g, "gr_fecs_mailbox1_r : 0x%x", gk20a_readl(g, gr_fecs_mailbox1_r())); nvgpu_err(g, "gr_fecs_irqstat_r : 0x%x", gk20a_readl(g, gr_fecs_irqstat_r())); nvgpu_err(g, "gr_fecs_irqmode_r : 0x%x", gk20a_readl(g, gr_fecs_irqmode_r())); nvgpu_err(g, "gr_fecs_irqmask_r : 0x%x", gk20a_readl(g, gr_fecs_irqmask_r())); nvgpu_err(g, "gr_fecs_irqdest_r : 0x%x", gk20a_readl(g, gr_fecs_irqdest_r())); nvgpu_err(g, "gr_fecs_debug1_r : 0x%x", gk20a_readl(g, gr_fecs_debug1_r())); nvgpu_err(g, "gr_fecs_debuginfo_r : 0x%x", gk20a_readl(g, gr_fecs_debuginfo_r())); nvgpu_err(g, "gr_fecs_ctxsw_status_1_r : 0x%x", gk20a_readl(g, gr_fecs_ctxsw_status_1_r())); for (i = 0; i < g->ops.gr.fecs_ctxsw_mailbox_size(); i++) { nvgpu_err(g, "gr_fecs_ctxsw_mailbox_r(%d) : 0x%x", i, gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(i))); } nvgpu_err(g, "gr_fecs_engctl_r : 0x%x", gk20a_readl(g, gr_fecs_engctl_r())); nvgpu_err(g, "gr_fecs_curctx_r : 0x%x", gk20a_readl(g, gr_fecs_curctx_r())); nvgpu_err(g, "gr_fecs_nxtctx_r : 0x%x", gk20a_readl(g, gr_fecs_nxtctx_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_IMB)); nvgpu_err(g, "FECS_FALCON_REG_IMB : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_DMB)); nvgpu_err(g, "FECS_FALCON_REG_DMB : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_CSW)); nvgpu_err(g, "FECS_FALCON_REG_CSW : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_CTX)); nvgpu_err(g, "FECS_FALCON_REG_CTX : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_EXCI)); nvgpu_err(g, "FECS_FALCON_REG_EXCI : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); for (i = 0; i < 4; i++) { gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_PC)); nvgpu_err(g, "FECS_FALCON_REG_PC : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_SP)); nvgpu_err(g, "FECS_FALCON_REG_SP : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); } } static void gr_gk20a_load_falcon_dmem(struct gk20a *g) { u32 i, ucode_u32_size; const u32 *ucode_u32_data; u32 checksum; nvgpu_log_fn(g, " "); gk20a_writel(g, gr_gpccs_dmemc_r(0), (gr_gpccs_dmemc_offs_f(0) | gr_gpccs_dmemc_blk_f(0) | gr_gpccs_dmemc_aincw_f(1))); ucode_u32_size = g->gr.ctx_vars.ucode.gpccs.data.count; ucode_u32_data = (const u32 *)g->gr.ctx_vars.ucode.gpccs.data.l; for (i = 0, checksum = 0; i < ucode_u32_size; i++) { gk20a_writel(g, gr_gpccs_dmemd_r(0), ucode_u32_data[i]); checksum += ucode_u32_data[i]; } gk20a_writel(g, gr_fecs_dmemc_r(0), (gr_fecs_dmemc_offs_f(0) | gr_fecs_dmemc_blk_f(0) | gr_fecs_dmemc_aincw_f(1))); ucode_u32_size = g->gr.ctx_vars.ucode.fecs.data.count; ucode_u32_data = (const u32 *)g->gr.ctx_vars.ucode.fecs.data.l; for (i = 0, checksum = 0; i < ucode_u32_size; i++) { gk20a_writel(g, gr_fecs_dmemd_r(0), ucode_u32_data[i]); checksum += ucode_u32_data[i]; } nvgpu_log_fn(g, "done"); } static void gr_gk20a_load_falcon_imem(struct gk20a *g) { u32 cfg, fecs_imem_size, gpccs_imem_size, ucode_u32_size; const u32 *ucode_u32_data; u32 tag, i, pad_start, pad_end; u32 checksum; nvgpu_log_fn(g, " "); cfg = gk20a_readl(g, gr_fecs_cfg_r()); fecs_imem_size = gr_fecs_cfg_imem_sz_v(cfg); cfg = gk20a_readl(g, gr_gpc0_cfg_r()); gpccs_imem_size = gr_gpc0_cfg_imem_sz_v(cfg); /* Use the broadcast address to access all of the GPCCS units. */ gk20a_writel(g, gr_gpccs_imemc_r(0), (gr_gpccs_imemc_offs_f(0) | gr_gpccs_imemc_blk_f(0) | gr_gpccs_imemc_aincw_f(1))); /* Setup the tags for the instruction memory. */ tag = 0; gk20a_writel(g, gr_gpccs_imemt_r(0), gr_gpccs_imemt_tag_f(tag)); ucode_u32_size = g->gr.ctx_vars.ucode.gpccs.inst.count; ucode_u32_data = (const u32 *)g->gr.ctx_vars.ucode.gpccs.inst.l; for (i = 0, checksum = 0; i < ucode_u32_size; i++) { if ((i != 0U) && ((i % (256U/sizeof(u32))) == 0U)) { tag++; gk20a_writel(g, gr_gpccs_imemt_r(0), gr_gpccs_imemt_tag_f(tag)); } gk20a_writel(g, gr_gpccs_imemd_r(0), ucode_u32_data[i]); checksum += ucode_u32_data[i]; } pad_start = i * 4U; pad_end = pad_start + (256U - pad_start % 256U) + 256U; for (i = pad_start; (i < gpccs_imem_size * 256U) && (i < pad_end); i += 4U) { if ((i != 0U) && ((i % 256U) == 0U)) { tag++; gk20a_writel(g, gr_gpccs_imemt_r(0), gr_gpccs_imemt_tag_f(tag)); } gk20a_writel(g, gr_gpccs_imemd_r(0), 0); } gk20a_writel(g, gr_fecs_imemc_r(0), (gr_fecs_imemc_offs_f(0) | gr_fecs_imemc_blk_f(0) | gr_fecs_imemc_aincw_f(1))); /* Setup the tags for the instruction memory. */ tag = 0; gk20a_writel(g, gr_fecs_imemt_r(0), gr_fecs_imemt_tag_f(tag)); ucode_u32_size = g->gr.ctx_vars.ucode.fecs.inst.count; ucode_u32_data = (const u32 *)g->gr.ctx_vars.ucode.fecs.inst.l; for (i = 0, checksum = 0; i < ucode_u32_size; i++) { if ((i != 0U) && ((i % (256U/sizeof(u32))) == 0U)) { tag++; gk20a_writel(g, gr_fecs_imemt_r(0), gr_fecs_imemt_tag_f(tag)); } gk20a_writel(g, gr_fecs_imemd_r(0), ucode_u32_data[i]); checksum += ucode_u32_data[i]; } pad_start = i * 4U; pad_end = pad_start + (256U - pad_start % 256U) + 256U; for (i = pad_start; (i < fecs_imem_size * 256U) && i < pad_end; i += 4U) { if ((i != 0U) && ((i % 256U) == 0U)) { tag++; gk20a_writel(g, gr_fecs_imemt_r(0), gr_fecs_imemt_tag_f(tag)); } gk20a_writel(g, gr_fecs_imemd_r(0), 0); } } int gr_gk20a_wait_idle(struct gk20a *g, unsigned long duration_ms, u32 expect_delay) { u32 delay = expect_delay; bool ctxsw_active; bool gr_busy; u32 gr_engine_id; u32 engine_status; bool ctx_status_invalid; struct nvgpu_timeout timeout; nvgpu_log_fn(g, " "); gr_engine_id = gk20a_fifo_get_gr_engine_id(g); nvgpu_timeout_init(g, &timeout, duration_ms, NVGPU_TIMER_CPU_TIMER); do { /* fmodel: host gets fifo_engine_status(gr) from gr only when gr_status is read */ (void) gk20a_readl(g, gr_status_r()); engine_status = gk20a_readl(g, fifo_engine_status_r(gr_engine_id)); ctxsw_active = engine_status & fifo_engine_status_ctxsw_in_progress_f(); ctx_status_invalid = (fifo_engine_status_ctx_status_v(engine_status) == fifo_engine_status_ctx_status_invalid_v()); gr_busy = gk20a_readl(g, gr_engine_status_r()) & gr_engine_status_value_busy_f(); if (ctx_status_invalid || (!gr_busy && !ctxsw_active)) { nvgpu_log_fn(g, "done"); return 0; } nvgpu_usleep_range(delay, delay * 2); delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); } while (nvgpu_timeout_expired(&timeout) == 0); nvgpu_err(g, "timeout, ctxsw busy : %d, gr busy : %d", ctxsw_active, gr_busy); return -EAGAIN; } int gr_gk20a_wait_fe_idle(struct gk20a *g, unsigned long duration_ms, u32 expect_delay) { u32 val; u32 delay = expect_delay; struct nvgpu_timeout timeout; if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) { return 0; } nvgpu_log_fn(g, " "); nvgpu_timeout_init(g, &timeout, duration_ms, NVGPU_TIMER_CPU_TIMER); do { val = gk20a_readl(g, gr_status_r()); if (gr_status_fe_method_lower_v(val) == 0U) { nvgpu_log_fn(g, "done"); return 0; } nvgpu_usleep_range(delay, delay * 2); delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); } while (nvgpu_timeout_expired(&timeout) == 0); nvgpu_err(g, "timeout, fe busy : %x", val); return -EAGAIN; } int gr_gk20a_ctx_wait_ucode(struct gk20a *g, u32 mailbox_id, u32 *mailbox_ret, u32 opc_success, u32 mailbox_ok, u32 opc_fail, u32 mailbox_fail, bool sleepduringwait) { struct nvgpu_timeout timeout; u32 delay = GR_FECS_POLL_INTERVAL; u32 check = WAIT_UCODE_LOOP; u32 reg; nvgpu_log_fn(g, " "); if (sleepduringwait) { delay = GR_IDLE_CHECK_DEFAULT; } nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), NVGPU_TIMER_CPU_TIMER); while (check == WAIT_UCODE_LOOP) { if (nvgpu_timeout_expired(&timeout)) { check = WAIT_UCODE_TIMEOUT; } reg = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(mailbox_id)); if (mailbox_ret) { *mailbox_ret = reg; } switch (opc_success) { case GR_IS_UCODE_OP_EQUAL: if (reg == mailbox_ok) { check = WAIT_UCODE_OK; } break; case GR_IS_UCODE_OP_NOT_EQUAL: if (reg != mailbox_ok) { check = WAIT_UCODE_OK; } break; case GR_IS_UCODE_OP_AND: if (reg & mailbox_ok) { check = WAIT_UCODE_OK; } break; case GR_IS_UCODE_OP_LESSER: if (reg < mailbox_ok) { check = WAIT_UCODE_OK; } break; case GR_IS_UCODE_OP_LESSER_EQUAL: if (reg <= mailbox_ok) { check = WAIT_UCODE_OK; } break; case GR_IS_UCODE_OP_SKIP: /* do no success check */ break; default: nvgpu_err(g, "invalid success opcode 0x%x", opc_success); check = WAIT_UCODE_ERROR; break; } switch (opc_fail) { case GR_IS_UCODE_OP_EQUAL: if (reg == mailbox_fail) { check = WAIT_UCODE_ERROR; } break; case GR_IS_UCODE_OP_NOT_EQUAL: if (reg != mailbox_fail) { check = WAIT_UCODE_ERROR; } break; case GR_IS_UCODE_OP_AND: if (reg & mailbox_fail) { check = WAIT_UCODE_ERROR; } break; case GR_IS_UCODE_OP_LESSER: if (reg < mailbox_fail) { check = WAIT_UCODE_ERROR; } break; case GR_IS_UCODE_OP_LESSER_EQUAL: if (reg <= mailbox_fail) { check = WAIT_UCODE_ERROR; } break; case GR_IS_UCODE_OP_SKIP: /* do no check on fail*/ break; default: nvgpu_err(g, "invalid fail opcode 0x%x", opc_fail); check = WAIT_UCODE_ERROR; break; } if (sleepduringwait) { nvgpu_usleep_range(delay, delay * 2); delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); } else { nvgpu_udelay(delay); } } if (check == WAIT_UCODE_TIMEOUT) { nvgpu_err(g, "timeout waiting on mailbox=%d value=0x%08x", mailbox_id, reg); gk20a_fecs_dump_falcon_stats(g); gk20a_gr_debug_dump(g); return -1; } else if (check == WAIT_UCODE_ERROR) { nvgpu_err(g, "ucode method failed on mailbox=%d value=0x%08x", mailbox_id, reg); gk20a_fecs_dump_falcon_stats(g); return -1; } nvgpu_log_fn(g, "done"); return 0; } /* The following is a less brittle way to call gr_gk20a_submit_fecs_method(...) * We should replace most, if not all, fecs method calls to this instead. */ int gr_gk20a_submit_fecs_method_op(struct gk20a *g, struct fecs_method_op_gk20a op, bool sleepduringwait) { struct gr_gk20a *gr = &g->gr; int ret; nvgpu_mutex_acquire(&gr->fecs_mutex); if (op.mailbox.id != 0) { gk20a_writel(g, gr_fecs_ctxsw_mailbox_r(op.mailbox.id), op.mailbox.data); } gk20a_writel(g, gr_fecs_ctxsw_mailbox_clear_r(0), gr_fecs_ctxsw_mailbox_clear_value_f(op.mailbox.clr)); gk20a_writel(g, gr_fecs_method_data_r(), op.method.data); gk20a_writel(g, gr_fecs_method_push_r(), gr_fecs_method_push_adr_f(op.method.addr)); /* op.mailbox.id == 4 cases require waiting for completion on * for op.mailbox.id == 0 */ if (op.mailbox.id == 4) { op.mailbox.id = 0; } ret = gr_gk20a_ctx_wait_ucode(g, op.mailbox.id, op.mailbox.ret, op.cond.ok, op.mailbox.ok, op.cond.fail, op.mailbox.fail, sleepduringwait); if (ret) { nvgpu_err(g,"fecs method: data=0x%08x push adr=0x%08x", op.method.data, op.method.addr); } nvgpu_mutex_release(&gr->fecs_mutex); return ret; } /* Sideband mailbox writes are done a bit differently */ int gr_gk20a_submit_fecs_sideband_method_op(struct gk20a *g, struct fecs_method_op_gk20a op) { struct gr_gk20a *gr = &g->gr; int ret; nvgpu_mutex_acquire(&gr->fecs_mutex); gk20a_writel(g, gr_fecs_ctxsw_mailbox_clear_r(op.mailbox.id), gr_fecs_ctxsw_mailbox_clear_value_f(op.mailbox.clr)); gk20a_writel(g, gr_fecs_method_data_r(), op.method.data); gk20a_writel(g, gr_fecs_method_push_r(), gr_fecs_method_push_adr_f(op.method.addr)); ret = gr_gk20a_ctx_wait_ucode(g, op.mailbox.id, op.mailbox.ret, op.cond.ok, op.mailbox.ok, op.cond.fail, op.mailbox.fail, false); if (ret) { nvgpu_err(g,"fecs method: data=0x%08x push adr=0x%08x", op.method.data, op.method.addr); } nvgpu_mutex_release(&gr->fecs_mutex); return ret; } static int gr_gk20a_ctrl_ctxsw(struct gk20a *g, u32 fecs_method, u32 *ret) { return gr_gk20a_submit_fecs_method_op(g, (struct fecs_method_op_gk20a) { .method.addr = fecs_method, .method.data = ~0, .mailbox = { .id = 1, /*sideband?*/ .data = ~0, .clr = ~0, .ret = ret, .ok = gr_fecs_ctxsw_mailbox_value_pass_v(), .fail = gr_fecs_ctxsw_mailbox_value_fail_v(), }, .cond.ok = GR_IS_UCODE_OP_EQUAL, .cond.fail = GR_IS_UCODE_OP_EQUAL }, true); } /** * Stop processing (stall) context switches at FECS:- * If fecs is sent stop_ctxsw method, elpg entry/exit cannot happen * and may timeout. It could manifest as different error signatures * depending on when stop_ctxsw fecs method gets sent with respect * to pmu elpg sequence. It could come as pmu halt or abort or * maybe ext error too. */ int gr_gk20a_disable_ctxsw(struct gk20a *g) { int err = 0; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " "); nvgpu_mutex_acquire(&g->ctxsw_disable_lock); g->ctxsw_disable_count++; if (g->ctxsw_disable_count == 1) { err = nvgpu_pg_elpg_disable(g); if (err != 0) { nvgpu_err(g, "failed to disable elpg. not safe to " "stop_ctxsw"); /* stop ctxsw command is not sent */ g->ctxsw_disable_count--; } else { err = gr_gk20a_ctrl_ctxsw(g, gr_fecs_method_push_adr_stop_ctxsw_v(), NULL); if (err != 0) { nvgpu_err(g, "failed to stop fecs ctxsw"); /* stop ctxsw failed */ g->ctxsw_disable_count--; } } } else { nvgpu_log_info(g, "ctxsw disabled, ctxsw_disable_count: %d", g->ctxsw_disable_count); } nvgpu_mutex_release(&g->ctxsw_disable_lock); return err; } /* Start processing (continue) context switches at FECS */ int gr_gk20a_enable_ctxsw(struct gk20a *g) { int err = 0; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " "); nvgpu_mutex_acquire(&g->ctxsw_disable_lock); if (g->ctxsw_disable_count == 0) { goto ctxsw_already_enabled; } g->ctxsw_disable_count--; WARN_ON(g->ctxsw_disable_count < 0); if (g->ctxsw_disable_count == 0) { err = gr_gk20a_ctrl_ctxsw(g, gr_fecs_method_push_adr_start_ctxsw_v(), NULL); if (err != 0) { nvgpu_err(g, "failed to start fecs ctxsw"); } else { if (nvgpu_pg_elpg_enable(g) != 0) { nvgpu_err(g, "failed to enable elpg " "after start_ctxsw"); } } } else { nvgpu_log_info(g, "ctxsw_disable_count: %d is not 0 yet", g->ctxsw_disable_count); } ctxsw_already_enabled: nvgpu_mutex_release(&g->ctxsw_disable_lock); return err; } int gr_gk20a_halt_pipe(struct gk20a *g) { return gr_gk20a_submit_fecs_method_op(g, (struct fecs_method_op_gk20a) { .method.addr = gr_fecs_method_push_adr_halt_pipeline_v(), .method.data = ~0, .mailbox = { .id = 1, /*sideband?*/ .data = ~0, .clr = ~0, .ret = NULL, .ok = gr_fecs_ctxsw_mailbox_value_pass_v(), .fail = gr_fecs_ctxsw_mailbox_value_fail_v(), }, .cond.ok = GR_IS_UCODE_OP_EQUAL, .cond.fail = GR_IS_UCODE_OP_EQUAL }, false); } int gr_gk20a_commit_inst(struct channel_gk20a *c, u64 gpu_va) { u32 addr_lo; u32 addr_hi; nvgpu_log_fn(c->g, " "); addr_lo = u64_lo32(gpu_va) >> 12; addr_hi = u64_hi32(gpu_va); nvgpu_mem_wr32(c->g, &c->inst_block, ram_in_gr_wfi_target_w(), ram_in_gr_cs_wfi_f() | ram_in_gr_wfi_mode_virtual_f() | ram_in_gr_wfi_ptr_lo_f(addr_lo)); nvgpu_mem_wr32(c->g, &c->inst_block, ram_in_gr_wfi_ptr_hi_w(), ram_in_gr_wfi_ptr_hi_f(addr_hi)); return 0; } /* * Context state can be written directly, or "patched" at times. So that code * can be used in either situation it is written using a series of * _ctx_patch_write(..., patch) statements. However any necessary map overhead * should be minimized; thus, bundle the sequence of these writes together, and * set them up and close with _ctx_patch_write_begin/_ctx_patch_write_end. */ int gr_gk20a_ctx_patch_write_begin(struct gk20a *g, struct nvgpu_gr_ctx *gr_ctx, bool update_patch_count) { if (update_patch_count) { /* reset patch count if ucode has already processed it */ gr_ctx->patch_ctx.data_count = nvgpu_mem_rd(g, &gr_ctx->mem, ctxsw_prog_main_image_patch_count_o()); nvgpu_log(g, gpu_dbg_info, "patch count reset to %d", gr_ctx->patch_ctx.data_count); } return 0; } void gr_gk20a_ctx_patch_write_end(struct gk20a *g, struct nvgpu_gr_ctx *gr_ctx, bool update_patch_count) { /* Write context count to context image if it is mapped */ if (update_patch_count) { nvgpu_mem_wr(g, &gr_ctx->mem, ctxsw_prog_main_image_patch_count_o(), gr_ctx->patch_ctx.data_count); nvgpu_log(g, gpu_dbg_info, "write patch count %d", gr_ctx->patch_ctx.data_count); } } void gr_gk20a_ctx_patch_write(struct gk20a *g, struct nvgpu_gr_ctx *gr_ctx, u32 addr, u32 data, bool patch) { if (patch) { u32 patch_slot = gr_ctx->patch_ctx.data_count * PATCH_CTX_SLOTS_REQUIRED_PER_ENTRY; if (patch_slot > (PATCH_CTX_ENTRIES_FROM_SIZE( gr_ctx->patch_ctx.mem.size) - PATCH_CTX_SLOTS_REQUIRED_PER_ENTRY)) { nvgpu_err(g, "failed to access patch_slot %d", patch_slot); return; } nvgpu_mem_wr32(g, &gr_ctx->patch_ctx.mem, patch_slot, addr); nvgpu_mem_wr32(g, &gr_ctx->patch_ctx.mem, patch_slot + 1, data); gr_ctx->patch_ctx.data_count++; nvgpu_log(g, gpu_dbg_info, "patch addr = 0x%x data = 0x%x data_count %d", addr, data, gr_ctx->patch_ctx.data_count); } else { gk20a_writel(g, addr, data); } } static u32 fecs_current_ctx_data(struct gk20a *g, struct nvgpu_mem *inst_block) { u64 ptr = nvgpu_inst_block_addr(g, inst_block) >> ram_in_base_shift_v(); u32 aperture = nvgpu_aperture_mask(g, inst_block, gr_fecs_current_ctx_target_sys_mem_ncoh_f(), gr_fecs_current_ctx_target_sys_mem_coh_f(), gr_fecs_current_ctx_target_vid_mem_f()); return gr_fecs_current_ctx_ptr_f(u64_lo32(ptr)) | aperture | gr_fecs_current_ctx_valid_f(1); } int gr_gk20a_fecs_ctx_bind_channel(struct gk20a *g, struct channel_gk20a *c) { u32 inst_base_ptr = u64_lo32(nvgpu_inst_block_addr(g, &c->inst_block) >> ram_in_base_shift_v()); u32 data = fecs_current_ctx_data(g, &c->inst_block); u32 ret; nvgpu_log_info(g, "bind channel %d inst ptr 0x%08x", c->chid, inst_base_ptr); ret = gr_gk20a_submit_fecs_method_op(g, (struct fecs_method_op_gk20a) { .method.addr = gr_fecs_method_push_adr_bind_pointer_v(), .method.data = data, .mailbox = { .id = 0, .data = 0, .clr = 0x30, .ret = NULL, .ok = 0x10, .fail = 0x20, }, .cond.ok = GR_IS_UCODE_OP_AND, .cond.fail = GR_IS_UCODE_OP_AND}, true); if (ret) { nvgpu_err(g, "bind channel instance failed"); } return ret; } void gr_gk20a_write_zcull_ptr(struct gk20a *g, struct nvgpu_mem *mem, u64 gpu_va) { u32 va = u64_lo32(gpu_va >> 8); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_zcull_ptr_o(), va); } void gr_gk20a_write_pm_ptr(struct gk20a *g, struct nvgpu_mem *mem, u64 gpu_va) { u32 va = u64_lo32(gpu_va >> 8); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_pm_ptr_o(), va); } static int gr_gk20a_ctx_zcull_setup(struct gk20a *g, struct channel_gk20a *c) { struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx = NULL; struct nvgpu_mem *mem = NULL; struct nvgpu_mem *ctxheader = &c->ctx_header; int ret = 0; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; mem = &gr_ctx->mem; if (gr_ctx->zcull_ctx.gpu_va == 0 && gr_ctx->zcull_ctx.ctx_sw_mode == ctxsw_prog_main_image_zcull_mode_separate_buffer_v()) { return -EINVAL; } ret = gk20a_disable_channel_tsg(g, c); if (ret) { nvgpu_err(g, "failed to disable channel/TSG"); return ret; } ret = gk20a_fifo_preempt(g, c); if (ret) { gk20a_enable_channel_tsg(g, c); nvgpu_err(g, "failed to preempt channel/TSG"); return ret; } nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_zcull_o(), gr_ctx->zcull_ctx.ctx_sw_mode); if (ctxheader->gpu_va) { g->ops.gr.write_zcull_ptr(g, ctxheader, gr_ctx->zcull_ctx.gpu_va); } else { g->ops.gr.write_zcull_ptr(g, mem, gr_ctx->zcull_ctx.gpu_va); } gk20a_enable_channel_tsg(g, c); return ret; } u32 gk20a_gr_gpc_offset(struct gk20a *g, u32 gpc) { u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); u32 gpc_offset = gpc_stride * gpc; return gpc_offset; } u32 gk20a_gr_tpc_offset(struct gk20a *g, u32 tpc) { u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); u32 tpc_offset = tpc_in_gpc_stride * tpc; return tpc_offset; } int gr_gk20a_commit_global_ctx_buffers(struct gk20a *g, struct channel_gk20a *c, bool patch) { struct gr_gk20a *gr = &g->gr; struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx = NULL; u64 addr; u32 size; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; if (patch) { int err; err = gr_gk20a_ctx_patch_write_begin(g, gr_ctx, false); if (err != 0) { return err; } } /* global pagepool buffer */ addr = (u64_lo32(gr_ctx->global_ctx_buffer_va[PAGEPOOL_VA]) >> gr_scc_pagepool_base_addr_39_8_align_bits_v()) | (u64_hi32(gr_ctx->global_ctx_buffer_va[PAGEPOOL_VA]) << (32 - gr_scc_pagepool_base_addr_39_8_align_bits_v())); size = gr->global_ctx_buffer[PAGEPOOL].mem.size / gr_scc_pagepool_total_pages_byte_granularity_v(); if (size == g->ops.gr.pagepool_default_size(g)) { size = gr_scc_pagepool_total_pages_hwmax_v(); } nvgpu_log_info(g, "pagepool buffer addr : 0x%016llx, size : %d", addr, size); g->ops.gr.commit_global_pagepool(g, gr_ctx, addr, size, patch); /* global bundle cb */ addr = (u64_lo32(gr_ctx->global_ctx_buffer_va[CIRCULAR_VA]) >> gr_scc_bundle_cb_base_addr_39_8_align_bits_v()) | (u64_hi32(gr_ctx->global_ctx_buffer_va[CIRCULAR_VA]) << (32 - gr_scc_bundle_cb_base_addr_39_8_align_bits_v())); size = gr->bundle_cb_default_size; nvgpu_log_info(g, "bundle cb addr : 0x%016llx, size : %d", addr, size); g->ops.gr.commit_global_bundle_cb(g, gr_ctx, addr, size, patch); /* global attrib cb */ addr = (u64_lo32(gr_ctx->global_ctx_buffer_va[ATTRIBUTE_VA]) >> gr_gpcs_setup_attrib_cb_base_addr_39_12_align_bits_v()) | (u64_hi32(gr_ctx->global_ctx_buffer_va[ATTRIBUTE_VA]) << (32 - gr_gpcs_setup_attrib_cb_base_addr_39_12_align_bits_v())); nvgpu_log_info(g, "attrib cb addr : 0x%016llx", addr); g->ops.gr.commit_global_attrib_cb(g, gr_ctx, addr, patch); g->ops.gr.commit_global_cb_manager(g, c, patch); if (patch) { gr_gk20a_ctx_patch_write_end(g, gr_ctx, false); } return 0; } int gr_gk20a_commit_global_timeslice(struct gk20a *g, struct channel_gk20a *c) { struct gr_gk20a *gr = &g->gr; struct nvgpu_gr_ctx *gr_ctx = NULL; u32 gpm_pd_cfg; u32 pd_ab_dist_cfg0; u32 ds_debug; u32 mpc_vtg_debug; u32 pe_vaf; u32 pe_vsc_vpc; nvgpu_log_fn(g, " "); gpm_pd_cfg = gk20a_readl(g, gr_gpcs_gpm_pd_cfg_r()); pd_ab_dist_cfg0 = gk20a_readl(g, gr_pd_ab_dist_cfg0_r()); ds_debug = gk20a_readl(g, gr_ds_debug_r()); mpc_vtg_debug = gk20a_readl(g, gr_gpcs_tpcs_mpc_vtg_debug_r()); if (gr->timeslice_mode == gr_gpcs_ppcs_cbm_cfg_timeslice_mode_enable_v()) { pe_vaf = gk20a_readl(g, gr_gpcs_tpcs_pe_vaf_r()); pe_vsc_vpc = gk20a_readl(g, gr_gpcs_tpcs_pes_vsc_vpc_r()); gpm_pd_cfg = gr_gpcs_gpm_pd_cfg_timeslice_mode_enable_f() | gpm_pd_cfg; pe_vaf = gr_gpcs_tpcs_pe_vaf_fast_mode_switch_true_f() | pe_vaf; pe_vsc_vpc = gr_gpcs_tpcs_pes_vsc_vpc_fast_mode_switch_true_f() | pe_vsc_vpc; pd_ab_dist_cfg0 = gr_pd_ab_dist_cfg0_timeslice_enable_en_f() | pd_ab_dist_cfg0; ds_debug = gr_ds_debug_timeslice_mode_enable_f() | ds_debug; mpc_vtg_debug = gr_gpcs_tpcs_mpc_vtg_debug_timeslice_mode_enabled_f() | mpc_vtg_debug; gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_gpm_pd_cfg_r(), gpm_pd_cfg, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_tpcs_pe_vaf_r(), pe_vaf, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_tpcs_pes_vsc_vpc_r(), pe_vsc_vpc, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_pd_ab_dist_cfg0_r(), pd_ab_dist_cfg0, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_ds_debug_r(), ds_debug, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_tpcs_mpc_vtg_debug_r(), mpc_vtg_debug, false); } else { gpm_pd_cfg = gr_gpcs_gpm_pd_cfg_timeslice_mode_disable_f() | gpm_pd_cfg; pd_ab_dist_cfg0 = gr_pd_ab_dist_cfg0_timeslice_enable_dis_f() | pd_ab_dist_cfg0; ds_debug = gr_ds_debug_timeslice_mode_disable_f() | ds_debug; mpc_vtg_debug = gr_gpcs_tpcs_mpc_vtg_debug_timeslice_mode_disabled_f() | mpc_vtg_debug; gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_gpm_pd_cfg_r(), gpm_pd_cfg, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_pd_ab_dist_cfg0_r(), pd_ab_dist_cfg0, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_ds_debug_r(), ds_debug, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_tpcs_mpc_vtg_debug_r(), mpc_vtg_debug, false); } return 0; } /* * Return map tiles count for given index * Return 0 if index is out-of-bounds */ static u32 gr_gk20a_get_map_tile_count(struct gr_gk20a *gr, u32 index) { if (index >= gr->map_tile_count) { return 0; } return gr->map_tiles[index]; } int gr_gk20a_setup_rop_mapping(struct gk20a *g, struct gr_gk20a *gr) { u32 norm_entries, norm_shift; u32 coeff5_mod, coeff6_mod, coeff7_mod, coeff8_mod, coeff9_mod, coeff10_mod, coeff11_mod; u32 map0, map1, map2, map3, map4, map5; if (gr->map_tiles == NULL) { return -1; } nvgpu_log_fn(g, " "); gk20a_writel(g, gr_crstr_map_table_cfg_r(), gr_crstr_map_table_cfg_row_offset_f(gr->map_row_offset) | gr_crstr_map_table_cfg_num_entries_f(gr->tpc_count)); map0 = gr_crstr_gpc_map0_tile0_f(gr_gk20a_get_map_tile_count(gr, 0)) | gr_crstr_gpc_map0_tile1_f(gr_gk20a_get_map_tile_count(gr, 1)) | gr_crstr_gpc_map0_tile2_f(gr_gk20a_get_map_tile_count(gr, 2)) | gr_crstr_gpc_map0_tile3_f(gr_gk20a_get_map_tile_count(gr, 3)) | gr_crstr_gpc_map0_tile4_f(gr_gk20a_get_map_tile_count(gr, 4)) | gr_crstr_gpc_map0_tile5_f(gr_gk20a_get_map_tile_count(gr, 5)); map1 = gr_crstr_gpc_map1_tile6_f(gr_gk20a_get_map_tile_count(gr, 6)) | gr_crstr_gpc_map1_tile7_f(gr_gk20a_get_map_tile_count(gr, 7)) | gr_crstr_gpc_map1_tile8_f(gr_gk20a_get_map_tile_count(gr, 8)) | gr_crstr_gpc_map1_tile9_f(gr_gk20a_get_map_tile_count(gr, 9)) | gr_crstr_gpc_map1_tile10_f(gr_gk20a_get_map_tile_count(gr, 10)) | gr_crstr_gpc_map1_tile11_f(gr_gk20a_get_map_tile_count(gr, 11)); map2 = gr_crstr_gpc_map2_tile12_f(gr_gk20a_get_map_tile_count(gr, 12)) | gr_crstr_gpc_map2_tile13_f(gr_gk20a_get_map_tile_count(gr, 13)) | gr_crstr_gpc_map2_tile14_f(gr_gk20a_get_map_tile_count(gr, 14)) | gr_crstr_gpc_map2_tile15_f(gr_gk20a_get_map_tile_count(gr, 15)) | gr_crstr_gpc_map2_tile16_f(gr_gk20a_get_map_tile_count(gr, 16)) | gr_crstr_gpc_map2_tile17_f(gr_gk20a_get_map_tile_count(gr, 17)); map3 = gr_crstr_gpc_map3_tile18_f(gr_gk20a_get_map_tile_count(gr, 18)) | gr_crstr_gpc_map3_tile19_f(gr_gk20a_get_map_tile_count(gr, 19)) | gr_crstr_gpc_map3_tile20_f(gr_gk20a_get_map_tile_count(gr, 20)) | gr_crstr_gpc_map3_tile21_f(gr_gk20a_get_map_tile_count(gr, 21)) | gr_crstr_gpc_map3_tile22_f(gr_gk20a_get_map_tile_count(gr, 22)) | gr_crstr_gpc_map3_tile23_f(gr_gk20a_get_map_tile_count(gr, 23)); map4 = gr_crstr_gpc_map4_tile24_f(gr_gk20a_get_map_tile_count(gr, 24)) | gr_crstr_gpc_map4_tile25_f(gr_gk20a_get_map_tile_count(gr, 25)) | gr_crstr_gpc_map4_tile26_f(gr_gk20a_get_map_tile_count(gr, 26)) | gr_crstr_gpc_map4_tile27_f(gr_gk20a_get_map_tile_count(gr, 27)) | gr_crstr_gpc_map4_tile28_f(gr_gk20a_get_map_tile_count(gr, 28)) | gr_crstr_gpc_map4_tile29_f(gr_gk20a_get_map_tile_count(gr, 29)); map5 = gr_crstr_gpc_map5_tile30_f(gr_gk20a_get_map_tile_count(gr, 30)) | gr_crstr_gpc_map5_tile31_f(gr_gk20a_get_map_tile_count(gr, 31)) | gr_crstr_gpc_map5_tile32_f(0) | gr_crstr_gpc_map5_tile33_f(0) | gr_crstr_gpc_map5_tile34_f(0) | gr_crstr_gpc_map5_tile35_f(0); gk20a_writel(g, gr_crstr_gpc_map0_r(), map0); gk20a_writel(g, gr_crstr_gpc_map1_r(), map1); gk20a_writel(g, gr_crstr_gpc_map2_r(), map2); gk20a_writel(g, gr_crstr_gpc_map3_r(), map3); gk20a_writel(g, gr_crstr_gpc_map4_r(), map4); gk20a_writel(g, gr_crstr_gpc_map5_r(), map5); switch (gr->tpc_count) { case 1: norm_shift = 4; break; case 2: case 3: norm_shift = 3; break; case 4: case 5: case 6: case 7: norm_shift = 2; break; case 8: case 9: case 10: case 11: case 12: case 13: case 14: case 15: norm_shift = 1; break; default: norm_shift = 0; break; } norm_entries = gr->tpc_count << norm_shift; coeff5_mod = (1 << 5) % norm_entries; coeff6_mod = (1 << 6) % norm_entries; coeff7_mod = (1 << 7) % norm_entries; coeff8_mod = (1 << 8) % norm_entries; coeff9_mod = (1 << 9) % norm_entries; coeff10_mod = (1 << 10) % norm_entries; coeff11_mod = (1 << 11) % norm_entries; gk20a_writel(g, gr_ppcs_wwdx_map_table_cfg_r(), gr_ppcs_wwdx_map_table_cfg_row_offset_f(gr->map_row_offset) | gr_ppcs_wwdx_map_table_cfg_normalized_num_entries_f(norm_entries) | gr_ppcs_wwdx_map_table_cfg_normalized_shift_value_f(norm_shift) | gr_ppcs_wwdx_map_table_cfg_coeff5_mod_value_f(coeff5_mod) | gr_ppcs_wwdx_map_table_cfg_num_entries_f(gr->tpc_count)); gk20a_writel(g, gr_ppcs_wwdx_map_table_cfg2_r(), gr_ppcs_wwdx_map_table_cfg2_coeff6_mod_value_f(coeff6_mod) | gr_ppcs_wwdx_map_table_cfg2_coeff7_mod_value_f(coeff7_mod) | gr_ppcs_wwdx_map_table_cfg2_coeff8_mod_value_f(coeff8_mod) | gr_ppcs_wwdx_map_table_cfg2_coeff9_mod_value_f(coeff9_mod) | gr_ppcs_wwdx_map_table_cfg2_coeff10_mod_value_f(coeff10_mod) | gr_ppcs_wwdx_map_table_cfg2_coeff11_mod_value_f(coeff11_mod)); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map0_r(), map0); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map1_r(), map1); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map2_r(), map2); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map3_r(), map3); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map4_r(), map4); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map5_r(), map5); gk20a_writel(g, gr_rstr2d_map_table_cfg_r(), gr_rstr2d_map_table_cfg_row_offset_f(gr->map_row_offset) | gr_rstr2d_map_table_cfg_num_entries_f(gr->tpc_count)); gk20a_writel(g, gr_rstr2d_gpc_map0_r(), map0); gk20a_writel(g, gr_rstr2d_gpc_map1_r(), map1); gk20a_writel(g, gr_rstr2d_gpc_map2_r(), map2); gk20a_writel(g, gr_rstr2d_gpc_map3_r(), map3); gk20a_writel(g, gr_rstr2d_gpc_map4_r(), map4); gk20a_writel(g, gr_rstr2d_gpc_map5_r(), map5); return 0; } static inline u32 count_bits(u32 mask) { u32 temp = mask; u32 count; for (count = 0; temp != 0; count++) { temp &= temp - 1; } return count; } int gr_gk20a_init_sm_id_table(struct gk20a *g) { u32 gpc, tpc; u32 sm_id = 0; for (tpc = 0; tpc < g->gr.max_tpc_per_gpc_count; tpc++) { for (gpc = 0; gpc < g->gr.gpc_count; gpc++) { if (tpc < g->gr.gpc_tpc_count[gpc]) { g->gr.sm_to_cluster[sm_id].tpc_index = tpc; g->gr.sm_to_cluster[sm_id].gpc_index = gpc; g->gr.sm_to_cluster[sm_id].sm_index = 0; g->gr.sm_to_cluster[sm_id].global_tpc_index = sm_id; sm_id++; } } } g->gr.no_of_sm = sm_id; return 0; } /* * Return number of TPCs in a GPC * Return 0 if GPC index is invalid i.e. GPC is disabled */ u32 gr_gk20a_get_tpc_count(struct gr_gk20a *gr, u32 gpc_index) { if (gpc_index >= gr->gpc_count) { return 0; } return gr->gpc_tpc_count[gpc_index]; } int gr_gk20a_init_fs_state(struct gk20a *g) { struct gr_gk20a *gr = &g->gr; u32 tpc_index, gpc_index; u32 sm_id = 0, gpc_id = 0; u32 tpc_per_gpc; u32 fuse_tpc_mask; u32 reg_index; int err; nvgpu_log_fn(g, " "); if (g->ops.gr.init_sm_id_table) { err = g->ops.gr.init_sm_id_table(g); if (err != 0) { return err; } /* Is table empty ? */ if (g->gr.no_of_sm == 0) { return -EINVAL; } } for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) { tpc_index = g->gr.sm_to_cluster[sm_id].tpc_index; gpc_index = g->gr.sm_to_cluster[sm_id].gpc_index; g->ops.gr.program_sm_id_numbering(g, gpc_index, tpc_index, sm_id); if (g->ops.gr.program_active_tpc_counts) { g->ops.gr.program_active_tpc_counts(g, gpc_index); } } for (reg_index = 0, gpc_id = 0; reg_index < gr_pd_num_tpc_per_gpc__size_1_v(); reg_index++, gpc_id += 8) { tpc_per_gpc = gr_pd_num_tpc_per_gpc_count0_f(gr_gk20a_get_tpc_count(gr, gpc_id + 0)) | gr_pd_num_tpc_per_gpc_count1_f(gr_gk20a_get_tpc_count(gr, gpc_id + 1)) | gr_pd_num_tpc_per_gpc_count2_f(gr_gk20a_get_tpc_count(gr, gpc_id + 2)) | gr_pd_num_tpc_per_gpc_count3_f(gr_gk20a_get_tpc_count(gr, gpc_id + 3)) | gr_pd_num_tpc_per_gpc_count4_f(gr_gk20a_get_tpc_count(gr, gpc_id + 4)) | gr_pd_num_tpc_per_gpc_count5_f(gr_gk20a_get_tpc_count(gr, gpc_id + 5)) | gr_pd_num_tpc_per_gpc_count6_f(gr_gk20a_get_tpc_count(gr, gpc_id + 6)) | gr_pd_num_tpc_per_gpc_count7_f(gr_gk20a_get_tpc_count(gr, gpc_id + 7)); gk20a_writel(g, gr_pd_num_tpc_per_gpc_r(reg_index), tpc_per_gpc); gk20a_writel(g, gr_ds_num_tpc_per_gpc_r(reg_index), tpc_per_gpc); } /* gr__setup_pd_mapping stubbed for gk20a */ g->ops.gr.setup_rop_mapping(g, gr); if (g->ops.gr.setup_alpha_beta_tables) { g->ops.gr.setup_alpha_beta_tables(g, gr); } for (gpc_index = 0; gpc_index < gr_pd_dist_skip_table__size_1_v() * 4; gpc_index += 4) { gk20a_writel(g, gr_pd_dist_skip_table_r(gpc_index/4), (gr_pd_dist_skip_table_gpc_4n0_mask_f(gr->gpc_skip_mask[gpc_index]) != 0U) || (gr_pd_dist_skip_table_gpc_4n1_mask_f(gr->gpc_skip_mask[gpc_index + 1]) != 0U) || (gr_pd_dist_skip_table_gpc_4n2_mask_f(gr->gpc_skip_mask[gpc_index + 2]) != 0U) || (gr_pd_dist_skip_table_gpc_4n3_mask_f(gr->gpc_skip_mask[gpc_index + 3]) != 0U)); } fuse_tpc_mask = g->ops.gr.get_gpc_tpc_mask(g, 0); if ((g->tpc_fs_mask_user != 0U) && (fuse_tpc_mask == BIT32(gr->max_tpc_count) - 1U)) { u32 val = g->tpc_fs_mask_user; val &= (0x1U << gr->max_tpc_count) - 1U; gk20a_writel(g, gr_cwd_fs_r(), gr_cwd_fs_num_gpcs_f(gr->gpc_count) | gr_cwd_fs_num_tpcs_f(hweight32(val))); } else { gk20a_writel(g, gr_cwd_fs_r(), gr_cwd_fs_num_gpcs_f(gr->gpc_count) | gr_cwd_fs_num_tpcs_f(gr->tpc_count)); } gk20a_writel(g, gr_bes_zrop_settings_r(), gr_bes_zrop_settings_num_active_fbps_f(gr->num_fbps)); gk20a_writel(g, gr_bes_crop_settings_r(), gr_bes_crop_settings_num_active_fbps_f(gr->num_fbps)); return 0; } int gr_gk20a_fecs_ctx_image_save(struct channel_gk20a *c, u32 save_type) { struct gk20a *g = c->g; int ret; nvgpu_log_fn(g, " "); ret = gr_gk20a_submit_fecs_method_op(g, (struct fecs_method_op_gk20a) { .method.addr = save_type, .method.data = fecs_current_ctx_data(g, &c->inst_block), .mailbox = {.id = 0, .data = 0, .clr = 3, .ret = NULL, .ok = 1, .fail = 2, }, .cond.ok = GR_IS_UCODE_OP_AND, .cond.fail = GR_IS_UCODE_OP_AND, }, true); if (ret) { nvgpu_err(g, "save context image failed"); } return ret; } u32 gk20a_init_sw_bundle(struct gk20a *g) { struct av_list_gk20a *sw_bundle_init = &g->gr.ctx_vars.sw_bundle_init; u32 last_bundle_data = 0; u32 err = 0; unsigned int i; /* disable fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_disabled_f()); /* enable pipe mode override */ gk20a_writel(g, gr_pipe_bundle_config_r(), gr_pipe_bundle_config_override_pipe_mode_enabled_f()); /* load bundle init */ for (i = 0; i < sw_bundle_init->count; i++) { if (i == 0 || last_bundle_data != sw_bundle_init->l[i].value) { gk20a_writel(g, gr_pipe_bundle_data_r(), sw_bundle_init->l[i].value); last_bundle_data = sw_bundle_init->l[i].value; } gk20a_writel(g, gr_pipe_bundle_address_r(), sw_bundle_init->l[i].addr); if (gr_pipe_bundle_address_value_v(sw_bundle_init->l[i].addr) == GR_GO_IDLE_BUNDLE) { err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto error; } } err = gr_gk20a_wait_fe_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto error; } } if ((err == 0U) && (g->ops.gr.init_sw_veid_bundle != NULL)) { err = g->ops.gr.init_sw_veid_bundle(g); if (err != 0U) { goto error; } } if (g->ops.gr.init_sw_bundle64) { err = g->ops.gr.init_sw_bundle64(g); if (err != 0U) { goto error; } } /* disable pipe mode override */ gk20a_writel(g, gr_pipe_bundle_config_r(), gr_pipe_bundle_config_override_pipe_mode_disabled_f()); err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); /* restore fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_prod_f()); return err; error: /* in case of error skip waiting for GR idle - just restore state */ gk20a_writel(g, gr_pipe_bundle_config_r(), gr_pipe_bundle_config_override_pipe_mode_disabled_f()); /* restore fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_prod_f()); return err; } /* init global golden image from a fresh gr_ctx in channel ctx. save a copy in local_golden_image in ctx_vars */ static int gr_gk20a_init_golden_ctx_image(struct gk20a *g, struct channel_gk20a *c) { struct gr_gk20a *gr = &g->gr; struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx = NULL; u32 ctx_header_bytes = ctxsw_prog_fecs_header_v(); u32 ctx_header_words; u32 i; u32 data; struct nvgpu_mem *gold_mem = &gr->global_ctx_buffer[GOLDEN_CTX].mem; struct nvgpu_mem *gr_mem; u32 err = 0; struct aiv_list_gk20a *sw_ctx_load = &g->gr.ctx_vars.sw_ctx_load; struct av_list_gk20a *sw_method_init = &g->gr.ctx_vars.sw_method_init; u32 last_method_data = 0; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; gr_mem = &gr_ctx->mem; /* golden ctx is global to all channels. Although only the first channel initializes golden image, driver needs to prevent multiple channels from initializing golden ctx at the same time */ nvgpu_mutex_acquire(&gr->ctx_mutex); if (gr->ctx_vars.golden_image_initialized) { goto clean_up; } if (!nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) { struct nvgpu_timeout timeout; nvgpu_timeout_init(g, &timeout, FE_PWR_MODE_TIMEOUT_MAX / FE_PWR_MODE_TIMEOUT_DEFAULT, NVGPU_TIMER_RETRY_TIMER); gk20a_writel(g, gr_fe_pwr_mode_r(), gr_fe_pwr_mode_req_send_f() | gr_fe_pwr_mode_mode_force_on_f()); do { u32 req = gr_fe_pwr_mode_req_v(gk20a_readl(g, gr_fe_pwr_mode_r())); if (req == gr_fe_pwr_mode_req_done_v()) { break; } nvgpu_udelay(FE_PWR_MODE_TIMEOUT_DEFAULT); } while (nvgpu_timeout_expired_msg(&timeout, "timeout forcing FE on") == 0); } gk20a_writel(g, gr_fecs_ctxsw_reset_ctl_r(), gr_fecs_ctxsw_reset_ctl_sys_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_be_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_sys_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_be_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_sys_context_reset_enabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_context_reset_enabled_f() | gr_fecs_ctxsw_reset_ctl_be_context_reset_enabled_f()); (void) gk20a_readl(g, gr_fecs_ctxsw_reset_ctl_r()); nvgpu_udelay(10); gk20a_writel(g, gr_fecs_ctxsw_reset_ctl_r(), gr_fecs_ctxsw_reset_ctl_sys_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_be_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_sys_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_be_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_sys_context_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_context_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_be_context_reset_disabled_f()); (void) gk20a_readl(g, gr_fecs_ctxsw_reset_ctl_r()); nvgpu_udelay(10); if (!nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) { struct nvgpu_timeout timeout; nvgpu_timeout_init(g, &timeout, FE_PWR_MODE_TIMEOUT_MAX / FE_PWR_MODE_TIMEOUT_DEFAULT, NVGPU_TIMER_RETRY_TIMER); gk20a_writel(g, gr_fe_pwr_mode_r(), gr_fe_pwr_mode_req_send_f() | gr_fe_pwr_mode_mode_auto_f()); do { u32 req = gr_fe_pwr_mode_req_v(gk20a_readl(g, gr_fe_pwr_mode_r())); if (req == gr_fe_pwr_mode_req_done_v()) { break; } nvgpu_udelay(FE_PWR_MODE_TIMEOUT_DEFAULT); } while (nvgpu_timeout_expired_msg(&timeout, "timeout setting FE power to auto") == 0); } /* clear scc ram */ gk20a_writel(g, gr_scc_init_r(), gr_scc_init_ram_trigger_f()); err = gr_gk20a_fecs_ctx_bind_channel(g, c); if (err != 0U) { goto clean_up; } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); /* load ctx init */ for (i = 0; i < sw_ctx_load->count; i++) { gk20a_writel(g, sw_ctx_load->l[i].addr, sw_ctx_load->l[i].value); } if (g->ops.gr.init_preemption_state) { g->ops.gr.init_preemption_state(g); } if (g->ops.clock_gating.blcg_gr_load_gating_prod) { g->ops.clock_gating.blcg_gr_load_gating_prod(g, g->blcg_enabled); } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto clean_up; } /* disable fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_disabled_f()); err = g->ops.gr.commit_global_ctx_buffers(g, c, false); if (err != 0U) { goto clean_up; } /* override a few ctx state registers */ g->ops.gr.commit_global_timeslice(g, c); /* floorsweep anything left */ err = g->ops.gr.init_fs_state(g); if (err != 0U) { goto clean_up; } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto restore_fe_go_idle; } err = gk20a_init_sw_bundle(g); if (err != 0U) { goto clean_up; } restore_fe_go_idle: /* restore fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_prod_f()); if ((err != 0U) || (gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT) != 0)) { goto clean_up; } /* load method init */ if (sw_method_init->count) { gk20a_writel(g, gr_pri_mme_shadow_raw_data_r(), sw_method_init->l[0].value); gk20a_writel(g, gr_pri_mme_shadow_raw_index_r(), gr_pri_mme_shadow_raw_index_write_trigger_f() | sw_method_init->l[0].addr); last_method_data = sw_method_init->l[0].value; } for (i = 1; i < sw_method_init->count; i++) { if (sw_method_init->l[i].value != last_method_data) { gk20a_writel(g, gr_pri_mme_shadow_raw_data_r(), sw_method_init->l[i].value); last_method_data = sw_method_init->l[i].value; } gk20a_writel(g, gr_pri_mme_shadow_raw_index_r(), gr_pri_mme_shadow_raw_index_write_trigger_f() | sw_method_init->l[i].addr); } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto clean_up; } ctx_header_words = roundup(ctx_header_bytes, sizeof(u32)); ctx_header_words >>= 2; g->ops.mm.l2_flush(g, true); for (i = 0; i < ctx_header_words; i++) { data = nvgpu_mem_rd32(g, gr_mem, i); nvgpu_mem_wr32(g, gold_mem, i, data); } nvgpu_mem_wr(g, gold_mem, ctxsw_prog_main_image_zcull_o(), ctxsw_prog_main_image_zcull_mode_no_ctxsw_v()); g->ops.gr.write_zcull_ptr(g, gold_mem, 0); err = g->ops.gr.commit_inst(c, gr_ctx->global_ctx_buffer_va[GOLDEN_CTX_VA]); if (err != 0U) { goto clean_up; } gr_gk20a_fecs_ctx_image_save(c, gr_fecs_method_push_adr_wfi_golden_save_v()); if (gr->ctx_vars.local_golden_image == NULL) { gr->ctx_vars.local_golden_image = nvgpu_vzalloc(g, gr->ctx_vars.golden_image_size); if (gr->ctx_vars.local_golden_image == NULL) { err = -ENOMEM; goto clean_up; } nvgpu_mem_rd_n(g, gold_mem, 0, gr->ctx_vars.local_golden_image, gr->ctx_vars.golden_image_size); } err = g->ops.gr.commit_inst(c, gr_mem->gpu_va); if (err != 0U) { goto clean_up; } gr->ctx_vars.golden_image_initialized = true; gk20a_writel(g, gr_fecs_current_ctx_r(), gr_fecs_current_ctx_valid_false_f()); clean_up: if (err != 0U) { nvgpu_err(g, "fail"); } else { nvgpu_log_fn(g, "done"); } nvgpu_mutex_release(&gr->ctx_mutex); return err; } int gr_gk20a_update_smpc_ctxsw_mode(struct gk20a *g, struct channel_gk20a *c, bool enable_smpc_ctxsw) { struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx = NULL; struct nvgpu_mem *mem = NULL; u32 data; int ret; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; mem = &gr_ctx->mem; if (!nvgpu_mem_is_valid(mem)) { nvgpu_err(g, "no graphics context allocated"); return -EFAULT; } ret = gk20a_disable_channel_tsg(g, c); if (ret) { nvgpu_err(g, "failed to disable channel/TSG"); goto out; } ret = gk20a_fifo_preempt(g, c); if (ret) { gk20a_enable_channel_tsg(g, c); nvgpu_err(g, "failed to preempt channel/TSG"); goto out; } /* Channel gr_ctx buffer is gpu cacheable. Flush and invalidate before cpu update. */ g->ops.mm.l2_flush(g, true); data = nvgpu_mem_rd(g, mem, ctxsw_prog_main_image_pm_o()); data = data & ~ctxsw_prog_main_image_pm_smpc_mode_m(); data |= enable_smpc_ctxsw ? ctxsw_prog_main_image_pm_smpc_mode_ctxsw_f() : ctxsw_prog_main_image_pm_smpc_mode_no_ctxsw_f(); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_pm_o(), data); out: gk20a_enable_channel_tsg(g, c); return ret; } int gr_gk20a_update_hwpm_ctxsw_mode(struct gk20a *g, struct channel_gk20a *c, u64 gpu_va, u32 mode) { struct tsg_gk20a *tsg; struct nvgpu_mem *gr_mem = NULL; struct nvgpu_gr_ctx *gr_ctx; struct pm_ctx_desc *pm_ctx; u32 data; u64 virt_addr = 0; struct nvgpu_mem *ctxheader = &c->ctx_header; int ret; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; pm_ctx = &gr_ctx->pm_ctx; gr_mem = &gr_ctx->mem; if (!nvgpu_mem_is_valid(gr_mem)) { nvgpu_err(g, "no graphics context allocated"); return -EFAULT; } if ((mode == NVGPU_DBG_HWPM_CTXSW_MODE_STREAM_OUT_CTXSW) && (g->ops.gr.get_hw_accessor_stream_out_mode == NULL)) { nvgpu_err(g, "Mode-E hwpm context switch mode is not supported"); return -EINVAL; } switch (mode) { case NVGPU_DBG_HWPM_CTXSW_MODE_CTXSW: if (pm_ctx->pm_mode == ctxsw_prog_main_image_pm_mode_ctxsw_f()) { return 0; } break; case NVGPU_DBG_HWPM_CTXSW_MODE_NO_CTXSW: if (pm_ctx->pm_mode == ctxsw_prog_main_image_pm_mode_no_ctxsw_f()) { return 0; } break; case NVGPU_DBG_HWPM_CTXSW_MODE_STREAM_OUT_CTXSW: if (pm_ctx->pm_mode == g->ops.gr.get_hw_accessor_stream_out_mode()) { return 0; } break; default: nvgpu_err(g, "invalid hwpm context switch mode"); return -EINVAL; } ret = gk20a_disable_channel_tsg(g, c); if (ret) { nvgpu_err(g, "failed to disable channel/TSG"); return ret; } ret = gk20a_fifo_preempt(g, c); if (ret) { gk20a_enable_channel_tsg(g, c); nvgpu_err(g, "failed to preempt channel/TSG"); return ret; } /* Channel gr_ctx buffer is gpu cacheable. Flush and invalidate before cpu update. */ g->ops.mm.l2_flush(g, true); if (mode != NVGPU_DBG_HWPM_CTXSW_MODE_NO_CTXSW) { /* Allocate buffer if necessary */ if (pm_ctx->mem.gpu_va == 0) { ret = nvgpu_dma_alloc_sys(g, g->gr.ctx_vars.pm_ctxsw_image_size, &pm_ctx->mem); if (ret) { c->g->ops.fifo.enable_channel(c); nvgpu_err(g, "failed to allocate pm ctxt buffer"); return ret; } pm_ctx->mem.gpu_va = nvgpu_gmmu_map_fixed(c->vm, &pm_ctx->mem, gpu_va, pm_ctx->mem.size, NVGPU_VM_MAP_CACHEABLE, gk20a_mem_flag_none, true, pm_ctx->mem.aperture); if (pm_ctx->mem.gpu_va == 0ULL) { nvgpu_err(g, "failed to map pm ctxt buffer"); nvgpu_dma_free(g, &pm_ctx->mem); c->g->ops.fifo.enable_channel(c); return -ENOMEM; } } if ((mode == NVGPU_DBG_HWPM_CTXSW_MODE_STREAM_OUT_CTXSW) && (g->ops.gr.init_hwpm_pmm_register != NULL)) { g->ops.gr.init_hwpm_pmm_register(g); } } data = nvgpu_mem_rd(g, gr_mem, ctxsw_prog_main_image_pm_o()); data = data & ~ctxsw_prog_main_image_pm_mode_m(); switch (mode) { case NVGPU_DBG_HWPM_CTXSW_MODE_CTXSW: pm_ctx->pm_mode = ctxsw_prog_main_image_pm_mode_ctxsw_f(); virt_addr = pm_ctx->mem.gpu_va; break; case NVGPU_DBG_HWPM_CTXSW_MODE_STREAM_OUT_CTXSW: pm_ctx->pm_mode = g->ops.gr.get_hw_accessor_stream_out_mode(); virt_addr = pm_ctx->mem.gpu_va; break; case NVGPU_DBG_HWPM_CTXSW_MODE_NO_CTXSW: pm_ctx->pm_mode = ctxsw_prog_main_image_pm_mode_no_ctxsw_f(); virt_addr = 0; } data |= pm_ctx->pm_mode; nvgpu_mem_wr(g, gr_mem, ctxsw_prog_main_image_pm_o(), data); if (ctxheader->gpu_va) { struct channel_gk20a *ch; nvgpu_rwsem_down_read(&tsg->ch_list_lock); nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { g->ops.gr.write_pm_ptr(g, &ch->ctx_header, virt_addr); } nvgpu_rwsem_up_read(&tsg->ch_list_lock); } else { g->ops.gr.write_pm_ptr(g, gr_mem, virt_addr); } /* enable channel */ gk20a_enable_channel_tsg(g, c); return 0; } void gk20a_gr_init_ctxsw_hdr_data(struct gk20a *g, struct nvgpu_mem *mem) { nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_num_save_ops_o(), 0); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_num_restore_ops_o(), 0); } /* load saved fresh copy of gloden image into channel gr_ctx */ int gr_gk20a_load_golden_ctx_image(struct gk20a *g, struct channel_gk20a *c) { struct gr_gk20a *gr = &g->gr; struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx; u32 virt_addr_lo; u32 virt_addr_hi; u64 virt_addr = 0; u32 v, data; int ret = 0; struct nvgpu_mem *mem; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; mem = &gr_ctx->mem; if (gr->ctx_vars.local_golden_image == NULL) { return -EINVAL; } /* Channel gr_ctx buffer is gpu cacheable. Flush and invalidate before cpu update. */ g->ops.mm.l2_flush(g, true); nvgpu_mem_wr_n(g, mem, 0, gr->ctx_vars.local_golden_image, gr->ctx_vars.golden_image_size); if (g->ops.gr.init_ctxsw_hdr_data) { g->ops.gr.init_ctxsw_hdr_data(g, mem); } if ((g->ops.gr.enable_cde_in_fecs != NULL) && c->cde) { g->ops.gr.enable_cde_in_fecs(g, mem); } /* set priv access map */ virt_addr_lo = u64_lo32(gr_ctx->global_ctx_buffer_va[PRIV_ACCESS_MAP_VA]); virt_addr_hi = u64_hi32(gr_ctx->global_ctx_buffer_va[PRIV_ACCESS_MAP_VA]); if (g->allow_all) { data = ctxsw_prog_main_image_priv_access_map_config_mode_allow_all_f(); } else { data = ctxsw_prog_main_image_priv_access_map_config_mode_use_map_f(); } nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_priv_access_map_config_o(), data); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_priv_access_map_addr_lo_o(), virt_addr_lo); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_priv_access_map_addr_hi_o(), virt_addr_hi); /* disable verif features */ v = nvgpu_mem_rd(g, mem, ctxsw_prog_main_image_misc_options_o()); v = v & ~(ctxsw_prog_main_image_misc_options_verif_features_m()); v = v | ctxsw_prog_main_image_misc_options_verif_features_disabled_f(); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_misc_options_o(), v); if (g->ops.gr.update_ctxsw_preemption_mode) { g->ops.gr.update_ctxsw_preemption_mode(g, c, mem); } if (g->ops.gr.update_boosted_ctx) { g->ops.gr.update_boosted_ctx(g, mem, gr_ctx); } virt_addr_lo = u64_lo32(gr_ctx->patch_ctx.mem.gpu_va); virt_addr_hi = u64_hi32(gr_ctx->patch_ctx.mem.gpu_va); nvgpu_log(g, gpu_dbg_info, "write patch count = %d", gr_ctx->patch_ctx.data_count); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_patch_count_o(), gr_ctx->patch_ctx.data_count); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_patch_adr_lo_o(), virt_addr_lo); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_patch_adr_hi_o(), virt_addr_hi); /* Update main header region of the context buffer with the info needed * for PM context switching, including mode and possibly a pointer to * the PM backing store. */ if (gr_ctx->pm_ctx.pm_mode != ctxsw_prog_main_image_pm_mode_no_ctxsw_f()) { if (gr_ctx->pm_ctx.mem.gpu_va == 0) { nvgpu_err(g, "context switched pm with no pm buffer!"); return -EFAULT; } virt_addr = gr_ctx->pm_ctx.mem.gpu_va; } else { virt_addr = 0; } data = nvgpu_mem_rd(g, mem, ctxsw_prog_main_image_pm_o()); data = data & ~ctxsw_prog_main_image_pm_mode_m(); data |= gr_ctx->pm_ctx.pm_mode; nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_pm_o(), data); g->ops.gr.write_pm_ptr(g, mem, virt_addr); return ret; } static void gr_gk20a_start_falcon_ucode(struct gk20a *g) { nvgpu_log_fn(g, " "); gk20a_writel(g, gr_fecs_ctxsw_mailbox_clear_r(0), gr_fecs_ctxsw_mailbox_clear_value_f(~0)); gk20a_writel(g, gr_gpccs_dmactl_r(), gr_gpccs_dmactl_require_ctx_f(0)); gk20a_writel(g, gr_fecs_dmactl_r(), gr_fecs_dmactl_require_ctx_f(0)); gk20a_writel(g, gr_gpccs_cpuctl_r(), gr_gpccs_cpuctl_startcpu_f(1)); gk20a_writel(g, gr_fecs_cpuctl_r(), gr_fecs_cpuctl_startcpu_f(1)); nvgpu_log_fn(g, "done"); } static int gr_gk20a_init_ctxsw_ucode_vaspace(struct gk20a *g) { struct mm_gk20a *mm = &g->mm; struct vm_gk20a *vm = mm->pmu.vm; struct gk20a_ctxsw_ucode_info *ucode_info = &g->ctxsw_ucode_info; int err; err = g->ops.mm.alloc_inst_block(g, &ucode_info->inst_blk_desc); if (err != 0) { return err; } g->ops.mm.init_inst_block(&ucode_info->inst_blk_desc, vm, 0); /* Map ucode surface to GMMU */ ucode_info->surface_desc.gpu_va = nvgpu_gmmu_map(vm, &ucode_info->surface_desc, ucode_info->surface_desc.size, 0, /* flags */ gk20a_mem_flag_read_only, false, ucode_info->surface_desc.aperture); if (ucode_info->surface_desc.gpu_va == 0ULL) { nvgpu_err(g, "failed to update gmmu ptes"); return -ENOMEM; } return 0; } static void gr_gk20a_init_ctxsw_ucode_segment( struct gk20a_ctxsw_ucode_segment *p_seg, u32 *offset, u32 size) { p_seg->offset = *offset; p_seg->size = size; *offset = ALIGN(*offset + size, BLK_SIZE); } static void gr_gk20a_init_ctxsw_ucode_segments( struct gk20a_ctxsw_ucode_segments *segments, u32 *offset, struct gk20a_ctxsw_bootloader_desc *bootdesc, u32 code_size, u32 data_size) { u32 boot_size = ALIGN(bootdesc->size, sizeof(u32)); segments->boot_entry = bootdesc->entry_point; segments->boot_imem_offset = bootdesc->imem_offset; gr_gk20a_init_ctxsw_ucode_segment(&segments->boot, offset, boot_size); gr_gk20a_init_ctxsw_ucode_segment(&segments->code, offset, code_size); gr_gk20a_init_ctxsw_ucode_segment(&segments->data, offset, data_size); } static int gr_gk20a_copy_ctxsw_ucode_segments( struct gk20a *g, struct nvgpu_mem *dst, struct gk20a_ctxsw_ucode_segments *segments, u32 *bootimage, u32 *code, u32 *data) { unsigned int i; nvgpu_mem_wr_n(g, dst, segments->boot.offset, bootimage, segments->boot.size); nvgpu_mem_wr_n(g, dst, segments->code.offset, code, segments->code.size); nvgpu_mem_wr_n(g, dst, segments->data.offset, data, segments->data.size); /* compute a "checksum" for the boot binary to detect its version */ segments->boot_signature = 0; for (i = 0; i < segments->boot.size / sizeof(u32); i++) { segments->boot_signature += bootimage[i]; } return 0; } int gr_gk20a_init_ctxsw_ucode(struct gk20a *g) { struct mm_gk20a *mm = &g->mm; struct vm_gk20a *vm = mm->pmu.vm; struct gk20a_ctxsw_bootloader_desc *fecs_boot_desc; struct gk20a_ctxsw_bootloader_desc *gpccs_boot_desc; struct nvgpu_firmware *fecs_fw; struct nvgpu_firmware *gpccs_fw; u32 *fecs_boot_image; u32 *gpccs_boot_image; struct gk20a_ctxsw_ucode_info *ucode_info = &g->ctxsw_ucode_info; u32 ucode_size; int err = 0; fecs_fw = nvgpu_request_firmware(g, GK20A_FECS_UCODE_IMAGE, 0); if (fecs_fw == NULL) { nvgpu_err(g, "failed to load fecs ucode!!"); return -ENOENT; } fecs_boot_desc = (void *)fecs_fw->data; fecs_boot_image = (void *)(fecs_fw->data + sizeof(struct gk20a_ctxsw_bootloader_desc)); gpccs_fw = nvgpu_request_firmware(g, GK20A_GPCCS_UCODE_IMAGE, 0); if (gpccs_fw == NULL) { nvgpu_release_firmware(g, fecs_fw); nvgpu_err(g, "failed to load gpccs ucode!!"); return -ENOENT; } gpccs_boot_desc = (void *)gpccs_fw->data; gpccs_boot_image = (void *)(gpccs_fw->data + sizeof(struct gk20a_ctxsw_bootloader_desc)); ucode_size = 0; gr_gk20a_init_ctxsw_ucode_segments(&ucode_info->fecs, &ucode_size, fecs_boot_desc, g->gr.ctx_vars.ucode.fecs.inst.count * sizeof(u32), g->gr.ctx_vars.ucode.fecs.data.count * sizeof(u32)); gr_gk20a_init_ctxsw_ucode_segments(&ucode_info->gpccs, &ucode_size, gpccs_boot_desc, g->gr.ctx_vars.ucode.gpccs.inst.count * sizeof(u32), g->gr.ctx_vars.ucode.gpccs.data.count * sizeof(u32)); err = nvgpu_dma_alloc_sys(g, ucode_size, &ucode_info->surface_desc); if (err != 0) { goto clean_up; } gr_gk20a_copy_ctxsw_ucode_segments(g, &ucode_info->surface_desc, &ucode_info->fecs, fecs_boot_image, g->gr.ctx_vars.ucode.fecs.inst.l, g->gr.ctx_vars.ucode.fecs.data.l); nvgpu_release_firmware(g, fecs_fw); fecs_fw = NULL; gr_gk20a_copy_ctxsw_ucode_segments(g, &ucode_info->surface_desc, &ucode_info->gpccs, gpccs_boot_image, g->gr.ctx_vars.ucode.gpccs.inst.l, g->gr.ctx_vars.ucode.gpccs.data.l); nvgpu_release_firmware(g, gpccs_fw); gpccs_fw = NULL; err = gr_gk20a_init_ctxsw_ucode_vaspace(g); if (err != 0) { goto clean_up; } return 0; clean_up: if (ucode_info->surface_desc.gpu_va) { nvgpu_gmmu_unmap(vm, &ucode_info->surface_desc, ucode_info->surface_desc.gpu_va); } nvgpu_dma_free(g, &ucode_info->surface_desc); nvgpu_release_firmware(g, gpccs_fw); gpccs_fw = NULL; nvgpu_release_firmware(g, fecs_fw); fecs_fw = NULL; return err; } static void gr_gk20a_wait_for_fecs_arb_idle(struct gk20a *g) { int retries = FECS_ARB_CMD_TIMEOUT_MAX / FECS_ARB_CMD_TIMEOUT_DEFAULT; u32 val; val = gk20a_readl(g, gr_fecs_arb_ctx_cmd_r()); while ((gr_fecs_arb_ctx_cmd_cmd_v(val) != 0U) && (retries != 0)) { nvgpu_udelay(FECS_ARB_CMD_TIMEOUT_DEFAULT); retries--; val = gk20a_readl(g, gr_fecs_arb_ctx_cmd_r()); } if (retries == 0) { nvgpu_err(g, "arbiter cmd timeout, fecs arb ctx cmd: 0x%08x", gk20a_readl(g, gr_fecs_arb_ctx_cmd_r())); } retries = FECS_ARB_CMD_TIMEOUT_MAX / FECS_ARB_CMD_TIMEOUT_DEFAULT; while (((gk20a_readl(g, gr_fecs_ctxsw_status_1_r()) & gr_fecs_ctxsw_status_1_arb_busy_m()) != 0U) && (retries != 0)) { nvgpu_udelay(FECS_ARB_CMD_TIMEOUT_DEFAULT); retries--; } if (retries == 0) { nvgpu_err(g, "arbiter idle timeout, fecs ctxsw status: 0x%08x", gk20a_readl(g, gr_fecs_ctxsw_status_1_r())); } } void gr_gk20a_load_falcon_bind_instblk(struct gk20a *g) { struct gk20a_ctxsw_ucode_info *ucode_info = &g->ctxsw_ucode_info; int retries = FECS_ARB_CMD_TIMEOUT_MAX / FECS_ARB_CMD_TIMEOUT_DEFAULT; u64 inst_ptr; while (((gk20a_readl(g, gr_fecs_ctxsw_status_1_r()) & gr_fecs_ctxsw_status_1_arb_busy_m()) != 0U) && (retries != 0)) { nvgpu_udelay(FECS_ARB_CMD_TIMEOUT_DEFAULT); retries--; } if (retries == 0) { nvgpu_err(g, "arbiter idle timeout, status: %08x", gk20a_readl(g, gr_fecs_ctxsw_status_1_r())); } gk20a_writel(g, gr_fecs_arb_ctx_adr_r(), 0x0); inst_ptr = nvgpu_inst_block_addr(g, &ucode_info->inst_blk_desc); gk20a_writel(g, gr_fecs_new_ctx_r(), gr_fecs_new_ctx_ptr_f(inst_ptr >> 12) | nvgpu_aperture_mask(g, &ucode_info->inst_blk_desc, gr_fecs_new_ctx_target_sys_mem_ncoh_f(), gr_fecs_new_ctx_target_sys_mem_coh_f(), gr_fecs_new_ctx_target_vid_mem_f()) | gr_fecs_new_ctx_valid_m()); gk20a_writel(g, gr_fecs_arb_ctx_ptr_r(), gr_fecs_arb_ctx_ptr_ptr_f(inst_ptr >> 12) | nvgpu_aperture_mask(g, &ucode_info->inst_blk_desc, gr_fecs_arb_ctx_ptr_target_sys_mem_ncoh_f(), gr_fecs_arb_ctx_ptr_target_sys_mem_coh_f(), gr_fecs_arb_ctx_ptr_target_vid_mem_f())); gk20a_writel(g, gr_fecs_arb_ctx_cmd_r(), 0x7); /* Wait for arbiter command to complete */ gr_gk20a_wait_for_fecs_arb_idle(g); gk20a_writel(g, gr_fecs_current_ctx_r(), gr_fecs_current_ctx_ptr_f(inst_ptr >> 12) | gr_fecs_current_ctx_target_m() | gr_fecs_current_ctx_valid_m()); /* Send command to arbiter to flush */ gk20a_writel(g, gr_fecs_arb_ctx_cmd_r(), gr_fecs_arb_ctx_cmd_cmd_s()); gr_gk20a_wait_for_fecs_arb_idle(g); } void gr_gk20a_load_ctxsw_ucode_header(struct gk20a *g, u64 addr_base, struct gk20a_ctxsw_ucode_segments *segments, u32 reg_offset) { u32 addr_code32; u32 addr_data32; addr_code32 = u64_lo32((addr_base + segments->code.offset) >> 8); addr_data32 = u64_lo32((addr_base + segments->data.offset) >> 8); /* * Copy falcon bootloader header into dmem at offset 0. * Configure dmem port 0 for auto-incrementing writes starting at dmem * offset 0. */ gk20a_writel(g, reg_offset + gr_fecs_dmemc_r(0), gr_fecs_dmemc_offs_f(0) | gr_fecs_dmemc_blk_f(0) | gr_fecs_dmemc_aincw_f(1)); /* Write out the actual data */ switch (segments->boot_signature) { case FALCON_UCODE_SIG_T18X_GPCCS_WITH_RESERVED: case FALCON_UCODE_SIG_T21X_FECS_WITH_DMEM_SIZE: case FALCON_UCODE_SIG_T21X_FECS_WITH_RESERVED: case FALCON_UCODE_SIG_T21X_GPCCS_WITH_RESERVED: case FALCON_UCODE_SIG_T12X_FECS_WITH_RESERVED: case FALCON_UCODE_SIG_T12X_GPCCS_WITH_RESERVED: gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); /* fallthrough */ case FALCON_UCODE_SIG_T12X_FECS_WITHOUT_RESERVED: case FALCON_UCODE_SIG_T12X_GPCCS_WITHOUT_RESERVED: case FALCON_UCODE_SIG_T21X_FECS_WITHOUT_RESERVED: case FALCON_UCODE_SIG_T21X_FECS_WITHOUT_RESERVED2: case FALCON_UCODE_SIG_T21X_GPCCS_WITHOUT_RESERVED: gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 4); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), addr_code32); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), segments->code.size); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), addr_data32); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), segments->data.size); break; case FALCON_UCODE_SIG_T12X_FECS_OLDER: case FALCON_UCODE_SIG_T12X_GPCCS_OLDER: gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), addr_code32); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), segments->code.size); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), addr_data32); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), segments->data.size); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), addr_code32); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); gk20a_writel(g, reg_offset + gr_fecs_dmemd_r(0), 0); break; default: nvgpu_err(g, "unknown falcon ucode boot signature 0x%08x" " with reg_offset 0x%08x", segments->boot_signature, reg_offset); BUG(); } } void gr_gk20a_load_ctxsw_ucode_boot(struct gk20a *g, u64 addr_base, struct gk20a_ctxsw_ucode_segments *segments, u32 reg_offset) { u32 addr_load32; u32 blocks; u32 b; u32 dst; addr_load32 = u64_lo32((addr_base + segments->boot.offset) >> 8); blocks = ((segments->boot.size + 0xFF) & ~0xFF) >> 8; /* * Set the base FB address for the DMA transfer. Subtract off the 256 * byte IMEM block offset such that the relative FB and IMEM offsets * match, allowing the IMEM tags to be properly created. */ dst = segments->boot_imem_offset; gk20a_writel(g, reg_offset + gr_fecs_dmatrfbase_r(), (addr_load32 - (dst >> 8))); for (b = 0; b < blocks; b++) { /* Setup destination IMEM offset */ gk20a_writel(g, reg_offset + gr_fecs_dmatrfmoffs_r(), dst + (b << 8)); /* Setup source offset (relative to BASE) */ gk20a_writel(g, reg_offset + gr_fecs_dmatrffboffs_r(), dst + (b << 8)); gk20a_writel(g, reg_offset + gr_fecs_dmatrfcmd_r(), gr_fecs_dmatrfcmd_imem_f(0x01) | gr_fecs_dmatrfcmd_write_f(0x00) | gr_fecs_dmatrfcmd_size_f(0x06) | gr_fecs_dmatrfcmd_ctxdma_f(0)); } /* Specify the falcon boot vector */ gk20a_writel(g, reg_offset + gr_fecs_bootvec_r(), gr_fecs_bootvec_vec_f(segments->boot_entry)); } static void gr_gk20a_load_falcon_with_bootloader(struct gk20a *g) { struct gk20a_ctxsw_ucode_info *ucode_info = &g->ctxsw_ucode_info; u64 addr_base = ucode_info->surface_desc.gpu_va; gk20a_writel(g, gr_fecs_ctxsw_mailbox_clear_r(0), 0x0); gr_gk20a_load_falcon_bind_instblk(g); g->ops.gr.falcon_load_ucode(g, addr_base, &g->ctxsw_ucode_info.fecs, 0); g->ops.gr.falcon_load_ucode(g, addr_base, &g->ctxsw_ucode_info.gpccs, gr_gpcs_gpccs_falcon_hwcfg_r() - gr_fecs_falcon_hwcfg_r()); } int gr_gk20a_load_ctxsw_ucode(struct gk20a *g) { int err; nvgpu_log_fn(g, " "); if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) { gk20a_writel(g, gr_fecs_ctxsw_mailbox_r(7), gr_fecs_ctxsw_mailbox_value_f(0xc0de7777)); gk20a_writel(g, gr_gpccs_ctxsw_mailbox_r(7), gr_gpccs_ctxsw_mailbox_value_f(0xc0de7777)); } /* * In case bootloader is not supported, revert to the old way of * loading gr ucode, without the faster bootstrap routine. */ if (!nvgpu_is_enabled(g, NVGPU_GR_USE_DMA_FOR_FW_BOOTSTRAP)) { gr_gk20a_load_falcon_dmem(g); gr_gk20a_load_falcon_imem(g); gr_gk20a_start_falcon_ucode(g); } else { if (!g->gr.skip_ucode_init) { err = gr_gk20a_init_ctxsw_ucode(g); if (err != 0) { return err; } } gr_gk20a_load_falcon_with_bootloader(g); g->gr.skip_ucode_init = true; } nvgpu_log_fn(g, "done"); return 0; } static int gr_gk20a_wait_ctxsw_ready(struct gk20a *g) { u32 ret; nvgpu_log_fn(g, " "); ret = gr_gk20a_ctx_wait_ucode(g, 0, NULL, GR_IS_UCODE_OP_EQUAL, eUcodeHandshakeInitComplete, GR_IS_UCODE_OP_SKIP, 0, false); if (ret) { nvgpu_err(g, "falcon ucode init timeout"); return ret; } if (nvgpu_is_enabled(g, NVGPU_GR_USE_DMA_FOR_FW_BOOTSTRAP) || nvgpu_is_enabled(g, NVGPU_SEC_SECUREGPCCS)) { gk20a_writel(g, gr_fecs_current_ctx_r(), gr_fecs_current_ctx_valid_false_f()); } gk20a_writel(g, gr_fecs_ctxsw_mailbox_clear_r(0), 0xffffffff); gk20a_writel(g, gr_fecs_method_data_r(), 0x7fffffff); gk20a_writel(g, gr_fecs_method_push_r(), gr_fecs_method_push_adr_set_watchdog_timeout_f()); nvgpu_log_fn(g, "done"); return 0; } int gr_gk20a_init_ctx_state(struct gk20a *g) { u32 ret; struct fecs_method_op_gk20a op = { .mailbox = { .id = 0, .data = 0, .clr = ~0, .ok = 0, .fail = 0}, .method.data = 0, .cond.ok = GR_IS_UCODE_OP_NOT_EQUAL, .cond.fail = GR_IS_UCODE_OP_SKIP, }; nvgpu_log_fn(g, " "); /* query ctxsw image sizes, if golden context is not created */ if (!g->gr.ctx_vars.golden_image_initialized) { op.method.addr = gr_fecs_method_push_adr_discover_image_size_v(); op.mailbox.ret = &g->gr.ctx_vars.golden_image_size; ret = gr_gk20a_submit_fecs_method_op(g, op, false); if (ret) { nvgpu_err(g, "query golden image size failed"); return ret; } op.method.addr = gr_fecs_method_push_adr_discover_zcull_image_size_v(); op.mailbox.ret = &g->gr.ctx_vars.zcull_ctxsw_image_size; ret = gr_gk20a_submit_fecs_method_op(g, op, false); if (ret) { nvgpu_err(g, "query zcull ctx image size failed"); return ret; } op.method.addr = gr_fecs_method_push_adr_discover_pm_image_size_v(); op.mailbox.ret = &g->gr.ctx_vars.pm_ctxsw_image_size; ret = gr_gk20a_submit_fecs_method_op(g, op, false); if (ret) { nvgpu_err(g, "query pm ctx image size failed"); return ret; } g->gr.ctx_vars.priv_access_map_size = 512 * 1024; #ifdef CONFIG_GK20A_CTXSW_TRACE g->gr.ctx_vars.fecs_trace_buffer_size = gk20a_fecs_trace_buffer_size(g); #endif } nvgpu_log_fn(g, "done"); return 0; } void gk20a_gr_destroy_ctx_buffer(struct gk20a *g, struct gr_ctx_buffer_desc *desc) { if (desc == NULL) { return; } nvgpu_dma_free(g, &desc->mem); desc->destroy = NULL; } int gk20a_gr_alloc_ctx_buffer(struct gk20a *g, struct gr_ctx_buffer_desc *desc, size_t size) { int err = 0; nvgpu_log_fn(g, " "); if (nvgpu_mem_is_valid(&desc->mem)) { return 0; } err = nvgpu_dma_alloc_sys(g, size, &desc->mem); if (err != 0) { return err; } desc->destroy = gk20a_gr_destroy_ctx_buffer; return err; } static void gr_gk20a_free_global_ctx_buffers(struct gk20a *g) { struct gr_gk20a *gr = &g->gr; u32 i; for (i = 0; i < NR_GLOBAL_CTX_BUF; i++) { /* destroy exists iff buffer is allocated */ if (gr->global_ctx_buffer[i].destroy) { gr->global_ctx_buffer[i].destroy(g, &gr->global_ctx_buffer[i]); } } nvgpu_log_fn(g, "done"); } int gr_gk20a_alloc_global_ctx_buffers(struct gk20a *g) { struct gr_gk20a *gr = &g->gr; int attr_buffer_size, err; u32 cb_buffer_size = gr->bundle_cb_default_size * gr_scc_bundle_cb_size_div_256b_byte_granularity_v(); u32 pagepool_buffer_size = g->ops.gr.pagepool_default_size(g) * gr_scc_pagepool_total_pages_byte_granularity_v(); nvgpu_log_fn(g, " "); attr_buffer_size = g->ops.gr.calc_global_ctx_buffer_size(g); nvgpu_log_info(g, "cb_buffer_size : %d", cb_buffer_size); err = gk20a_gr_alloc_ctx_buffer(g, &gr->global_ctx_buffer[CIRCULAR], cb_buffer_size); if (err != 0) { goto clean_up; } if (g->ops.secure_alloc) { err = g->ops.secure_alloc(g, &gr->global_ctx_buffer[CIRCULAR_VPR], cb_buffer_size); if (err != 0) { goto clean_up; } } nvgpu_log_info(g, "pagepool_buffer_size : %d", pagepool_buffer_size); err = gk20a_gr_alloc_ctx_buffer(g, &gr->global_ctx_buffer[PAGEPOOL], pagepool_buffer_size); if (err != 0) { goto clean_up; } if (g->ops.secure_alloc) { err = g->ops.secure_alloc(g, &gr->global_ctx_buffer[PAGEPOOL_VPR], pagepool_buffer_size); if (err != 0) { goto clean_up; } } nvgpu_log_info(g, "attr_buffer_size : %d", attr_buffer_size); err = gk20a_gr_alloc_ctx_buffer(g, &gr->global_ctx_buffer[ATTRIBUTE], attr_buffer_size); if (err != 0) { goto clean_up; } if (g->ops.secure_alloc) { err = g->ops.secure_alloc(g, &gr->global_ctx_buffer[ATTRIBUTE_VPR], attr_buffer_size); if (err != 0) { goto clean_up; } } nvgpu_log_info(g, "golden_image_size : %d", gr->ctx_vars.golden_image_size); err = gk20a_gr_alloc_ctx_buffer(g, &gr->global_ctx_buffer[GOLDEN_CTX], gr->ctx_vars.golden_image_size); if (err != 0) { goto clean_up; } nvgpu_log_info(g, "priv_access_map_size : %d", gr->ctx_vars.priv_access_map_size); err = gk20a_gr_alloc_ctx_buffer(g, &gr->global_ctx_buffer[PRIV_ACCESS_MAP], gr->ctx_vars.priv_access_map_size); if (err != 0) { goto clean_up; } #ifdef CONFIG_GK20A_CTXSW_TRACE nvgpu_log_info(g, "fecs_trace_buffer_size : %d", gr->ctx_vars.fecs_trace_buffer_size); err = nvgpu_dma_alloc_sys(g, gr->ctx_vars.fecs_trace_buffer_size, &gr->global_ctx_buffer[FECS_TRACE_BUFFER].mem); if (err != 0) { goto clean_up; } gr->global_ctx_buffer[FECS_TRACE_BUFFER].destroy = gk20a_gr_destroy_ctx_buffer; #endif nvgpu_log_fn(g, "done"); return 0; clean_up: nvgpu_err(g, "fail"); gr_gk20a_free_global_ctx_buffers(g); return -ENOMEM; } static void gr_gk20a_unmap_global_ctx_buffers(struct gk20a *g, struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx) { u64 *g_bfr_va = gr_ctx->global_ctx_buffer_va; u64 *g_bfr_size = gr_ctx->global_ctx_buffer_size; int *g_bfr_index = gr_ctx->global_ctx_buffer_index; u32 i; nvgpu_log_fn(g, " "); for (i = 0; i < NR_GLOBAL_CTX_BUF_VA; i++) { if (g_bfr_index[i]) { struct nvgpu_mem *mem; /* * Translate from VA index to buffer index to determine * the correct struct nvgpu_mem to use. Handles the VPR * vs non-VPR difference in context images. */ mem = &g->gr.global_ctx_buffer[g_bfr_index[i]].mem; nvgpu_gmmu_unmap(vm, mem, g_bfr_va[i]); } } memset(g_bfr_va, 0, sizeof(gr_ctx->global_ctx_buffer_va)); memset(g_bfr_size, 0, sizeof(gr_ctx->global_ctx_buffer_size)); memset(g_bfr_index, 0, sizeof(gr_ctx->global_ctx_buffer_index)); gr_ctx->global_ctx_buffer_mapped = false; } int gr_gk20a_map_global_ctx_buffers(struct gk20a *g, struct channel_gk20a *c) { struct tsg_gk20a *tsg; struct vm_gk20a *ch_vm = c->vm; u64 *g_bfr_va; u64 *g_bfr_size; int *g_bfr_index; struct gr_gk20a *gr = &g->gr; struct nvgpu_mem *mem; u64 gpu_va; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } g_bfr_va = tsg->gr_ctx.global_ctx_buffer_va; g_bfr_size = tsg->gr_ctx.global_ctx_buffer_size; g_bfr_index = tsg->gr_ctx.global_ctx_buffer_index; /* Circular Buffer */ if (c->vpr && nvgpu_mem_is_valid(&gr->global_ctx_buffer[CIRCULAR_VPR].mem)) { mem = &gr->global_ctx_buffer[CIRCULAR_VPR].mem; g_bfr_index[CIRCULAR_VA] = CIRCULAR_VPR; } else { mem = &gr->global_ctx_buffer[CIRCULAR].mem; g_bfr_index[CIRCULAR_VA] = CIRCULAR; } gpu_va = nvgpu_gmmu_map(ch_vm, mem, mem->size, NVGPU_VM_MAP_CACHEABLE, gk20a_mem_flag_none, true, mem->aperture); if (gpu_va == 0ULL) { goto clean_up; } g_bfr_va[CIRCULAR_VA] = gpu_va; g_bfr_size[CIRCULAR_VA] = mem->size; /* Attribute Buffer */ if (c->vpr && nvgpu_mem_is_valid(&gr->global_ctx_buffer[ATTRIBUTE_VPR].mem)) { mem = &gr->global_ctx_buffer[ATTRIBUTE_VPR].mem; g_bfr_index[ATTRIBUTE_VA] = ATTRIBUTE_VPR; } else { mem = &gr->global_ctx_buffer[ATTRIBUTE].mem; g_bfr_index[ATTRIBUTE_VA] = ATTRIBUTE; } gpu_va = nvgpu_gmmu_map(ch_vm, mem, mem->size, NVGPU_VM_MAP_CACHEABLE, gk20a_mem_flag_none, false, mem->aperture); if (gpu_va == 0ULL) { goto clean_up; } g_bfr_va[ATTRIBUTE_VA] = gpu_va; g_bfr_size[ATTRIBUTE_VA] = mem->size; /* Page Pool */ if (c->vpr && nvgpu_mem_is_valid(&gr->global_ctx_buffer[PAGEPOOL_VPR].mem)) { mem = &gr->global_ctx_buffer[PAGEPOOL_VPR].mem; g_bfr_index[PAGEPOOL_VA] = PAGEPOOL_VPR; } else { mem = &gr->global_ctx_buffer[PAGEPOOL].mem; g_bfr_index[PAGEPOOL_VA] = PAGEPOOL; } gpu_va = nvgpu_gmmu_map(ch_vm, mem, mem->size, NVGPU_VM_MAP_CACHEABLE, gk20a_mem_flag_none, true, mem->aperture); if (gpu_va == 0ULL) { goto clean_up; } g_bfr_va[PAGEPOOL_VA] = gpu_va; g_bfr_size[PAGEPOOL_VA] = mem->size; /* Golden Image */ mem = &gr->global_ctx_buffer[GOLDEN_CTX].mem; gpu_va = nvgpu_gmmu_map(ch_vm, mem, mem->size, 0, gk20a_mem_flag_none, true, mem->aperture); if (gpu_va == 0ULL) { goto clean_up; } g_bfr_va[GOLDEN_CTX_VA] = gpu_va; g_bfr_size[GOLDEN_CTX_VA] = mem->size; g_bfr_index[GOLDEN_CTX_VA] = GOLDEN_CTX; /* Priv register Access Map */ mem = &gr->global_ctx_buffer[PRIV_ACCESS_MAP].mem; gpu_va = nvgpu_gmmu_map(ch_vm, mem, mem->size, 0, gk20a_mem_flag_none, true, mem->aperture); if (gpu_va == 0ULL) { goto clean_up; } g_bfr_va[PRIV_ACCESS_MAP_VA] = gpu_va; g_bfr_size[PRIV_ACCESS_MAP_VA] = mem->size; g_bfr_index[PRIV_ACCESS_MAP_VA] = PRIV_ACCESS_MAP; tsg->gr_ctx.global_ctx_buffer_mapped = true; #ifdef CONFIG_GK20A_CTXSW_TRACE /* FECS trace buffer */ if (nvgpu_is_enabled(g, NVGPU_FECS_TRACE_VA)) { mem = &gr->global_ctx_buffer[FECS_TRACE_BUFFER].mem; gpu_va = nvgpu_gmmu_map(ch_vm, mem, mem->size, 0, gk20a_mem_flag_none, true, mem->aperture); if (!gpu_va) goto clean_up; g_bfr_va[FECS_TRACE_BUFFER_VA] = gpu_va; g_bfr_size[FECS_TRACE_BUFFER_VA] = mem->size; g_bfr_index[FECS_TRACE_BUFFER_VA] = FECS_TRACE_BUFFER; } #endif return 0; clean_up: gr_gk20a_unmap_global_ctx_buffers(g, ch_vm, &tsg->gr_ctx); return -ENOMEM; } int gr_gk20a_alloc_gr_ctx(struct gk20a *g, struct nvgpu_gr_ctx *gr_ctx, struct vm_gk20a *vm, u32 class, u32 padding) { struct gr_gk20a *gr = &g->gr; int err = 0; nvgpu_log_fn(g, " "); if (gr->ctx_vars.buffer_size == 0) { return 0; } /* alloc channel gr ctx buffer */ gr->ctx_vars.buffer_size = gr->ctx_vars.golden_image_size; gr->ctx_vars.buffer_total_size = gr->ctx_vars.golden_image_size; err = nvgpu_dma_alloc(g, gr->ctx_vars.buffer_total_size, &gr_ctx->mem); if (err != 0) { return err; } gr_ctx->mem.gpu_va = nvgpu_gmmu_map(vm, &gr_ctx->mem, gr_ctx->mem.size, 0, /* not GPU-cacheable */ gk20a_mem_flag_none, true, gr_ctx->mem.aperture); if (gr_ctx->mem.gpu_va == 0ULL) { goto err_free_mem; } return 0; err_free_mem: nvgpu_dma_free(g, &gr_ctx->mem); return err; } static int gr_gk20a_alloc_tsg_gr_ctx(struct gk20a *g, struct tsg_gk20a *tsg, u32 class, u32 padding) { struct nvgpu_gr_ctx *gr_ctx = &tsg->gr_ctx; int err; if (tsg->vm == NULL) { nvgpu_err(tsg->g, "No address space bound"); return -ENOMEM; } err = g->ops.gr.alloc_gr_ctx(g, gr_ctx, tsg->vm, class, padding); if (err != 0) { return err; } gr_ctx->tsgid = tsg->tsgid; return 0; } void gr_gk20a_free_gr_ctx(struct gk20a *g, struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx) { nvgpu_log_fn(g, " "); if (gr_ctx->mem.gpu_va) { gr_gk20a_unmap_global_ctx_buffers(g, vm, gr_ctx); gr_gk20a_free_channel_patch_ctx(g, vm, gr_ctx); gr_gk20a_free_channel_pm_ctx(g, vm, gr_ctx); if ((g->ops.gr.dump_ctxsw_stats != NULL) && g->gr.ctx_vars.dump_ctxsw_stats_on_channel_close) { g->ops.gr.dump_ctxsw_stats(g, vm, gr_ctx); } nvgpu_dma_unmap_free(vm, &gr_ctx->pagepool_ctxsw_buffer); nvgpu_dma_unmap_free(vm, &gr_ctx->betacb_ctxsw_buffer); nvgpu_dma_unmap_free(vm, &gr_ctx->spill_ctxsw_buffer); nvgpu_dma_unmap_free(vm, &gr_ctx->preempt_ctxsw_buffer); nvgpu_dma_unmap_free(vm, &gr_ctx->mem); memset(gr_ctx, 0, sizeof(*gr_ctx)); } } void gr_gk20a_free_tsg_gr_ctx(struct tsg_gk20a *tsg) { struct gk20a *g = tsg->g; if (tsg->vm == NULL) { nvgpu_err(g, "No address space bound"); return; } tsg->g->ops.gr.free_gr_ctx(g, tsg->vm, &tsg->gr_ctx); } u32 gr_gk20a_get_patch_slots(struct gk20a *g) { return PATCH_CTX_SLOTS_PER_PAGE; } static int gr_gk20a_alloc_channel_patch_ctx(struct gk20a *g, struct channel_gk20a *c) { struct tsg_gk20a *tsg; struct patch_desc *patch_ctx; struct vm_gk20a *ch_vm = c->vm; u32 alloc_size; int err = 0; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } patch_ctx = &tsg->gr_ctx.patch_ctx; alloc_size = g->ops.gr.get_patch_slots(g) * PATCH_CTX_SLOTS_REQUIRED_PER_ENTRY; nvgpu_log(g, gpu_dbg_info, "patch buffer size in entries: %d", alloc_size); err = nvgpu_dma_alloc_map_sys(ch_vm, alloc_size * sizeof(u32), &patch_ctx->mem); if (err != 0) { return err; } nvgpu_log_fn(g, "done"); return 0; } static void gr_gk20a_free_channel_patch_ctx(struct gk20a *g, struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx) { struct patch_desc *patch_ctx = &gr_ctx->patch_ctx; nvgpu_log_fn(g, " "); if (patch_ctx->mem.gpu_va) { nvgpu_gmmu_unmap(vm, &patch_ctx->mem, patch_ctx->mem.gpu_va); } nvgpu_dma_free(g, &patch_ctx->mem); patch_ctx->data_count = 0; } static void gr_gk20a_free_channel_pm_ctx(struct gk20a *g, struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx) { struct pm_ctx_desc *pm_ctx = &gr_ctx->pm_ctx; nvgpu_log_fn(g, " "); if (pm_ctx->mem.gpu_va) { nvgpu_gmmu_unmap(vm, &pm_ctx->mem, pm_ctx->mem.gpu_va); nvgpu_dma_free(g, &pm_ctx->mem); } } int gk20a_alloc_obj_ctx(struct channel_gk20a *c, u32 class_num, u32 flags) { struct gk20a *g = c->g; struct nvgpu_gr_ctx *gr_ctx; struct tsg_gk20a *tsg = NULL; int err = 0; nvgpu_log_fn(g, " "); /* an address space needs to have been bound at this point.*/ if (!gk20a_channel_as_bound(c) && (c->vm == NULL)) { nvgpu_err(g, "not bound to address space at time" " of grctx allocation"); return -EINVAL; } if (!g->ops.gr.is_valid_class(g, class_num)) { nvgpu_err(g, "invalid obj class 0x%x", class_num); err = -EINVAL; goto out; } c->obj_class = class_num; tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; if (!nvgpu_mem_is_valid(&gr_ctx->mem)) { tsg->vm = c->vm; nvgpu_vm_get(tsg->vm); err = gr_gk20a_alloc_tsg_gr_ctx(g, tsg, class_num, flags); if (err != 0) { nvgpu_err(g, "fail to allocate TSG gr ctx buffer"); nvgpu_vm_put(tsg->vm); tsg->vm = NULL; goto out; } /* allocate patch buffer */ if (!nvgpu_mem_is_valid(&gr_ctx->patch_ctx.mem)) { gr_ctx->patch_ctx.data_count = 0; err = gr_gk20a_alloc_channel_patch_ctx(g, c); if (err != 0) { nvgpu_err(g, "fail to allocate patch buffer"); goto out; } } /* map global buffer to channel gpu_va and commit */ err = g->ops.gr.map_global_ctx_buffers(g, c); if (err != 0) { nvgpu_err(g, "fail to map global ctx buffer"); goto out; } g->ops.gr.commit_global_ctx_buffers(g, c, true); /* commit gr ctx buffer */ err = g->ops.gr.commit_inst(c, gr_ctx->mem.gpu_va); if (err != 0) { nvgpu_err(g, "fail to commit gr ctx buffer"); goto out; } /* init golden image */ err = gr_gk20a_init_golden_ctx_image(g, c); if (err != 0) { nvgpu_err(g, "fail to init golden ctx image"); goto out; } /* Re-enable ELPG now that golden image has been initialized. * The PMU PG init code may already have tried to enable elpg, but * would not have been able to complete this action since the golden * image hadn't been initialized yet, so do this now. */ err = nvgpu_pmu_reenable_elpg(g); if (err != 0) { nvgpu_err(g, "fail to re-enable elpg"); goto out; } /* load golden image */ gr_gk20a_load_golden_ctx_image(g, c); if (err != 0) { nvgpu_err(g, "fail to load golden ctx image"); goto out; } #ifdef CONFIG_GK20A_CTXSW_TRACE if (g->ops.fecs_trace.bind_channel && !c->vpr) { err = g->ops.fecs_trace.bind_channel(g, c); if (err != 0) { nvgpu_warn(g, "fail to bind channel for ctxsw trace"); } } #endif if (g->ops.gr.set_czf_bypass) { g->ops.gr.set_czf_bypass(g, c); } /* PM ctxt switch is off by default */ gr_ctx->pm_ctx.pm_mode = ctxsw_prog_main_image_pm_mode_no_ctxsw_f(); } else { /* commit gr ctx buffer */ err = g->ops.gr.commit_inst(c, gr_ctx->mem.gpu_va); if (err != 0) { nvgpu_err(g, "fail to commit gr ctx buffer"); goto out; } #ifdef CONFIG_GK20A_CTXSW_TRACE if (g->ops.fecs_trace.bind_channel && !c->vpr) { err = g->ops.fecs_trace.bind_channel(g, c); if (err != 0) { nvgpu_warn(g, "fail to bind channel for ctxsw trace"); } } #endif } nvgpu_log_fn(g, "done"); return 0; out: /* 1. gr_ctx, patch_ctx and global ctx buffer mapping can be reused so no need to release them. 2. golden image init and load is a one time thing so if they pass, no need to undo. */ nvgpu_err(g, "fail"); return err; } static void gk20a_remove_gr_support(struct gr_gk20a *gr) { struct gk20a *g = gr->g; nvgpu_log_fn(g, " "); gr_gk20a_free_cyclestats_snapshot_data(g); gr_gk20a_free_global_ctx_buffers(g); nvgpu_dma_free(g, &gr->compbit_store.mem); memset(&gr->compbit_store, 0, sizeof(struct compbit_store_desc)); nvgpu_kfree(g, gr->gpc_tpc_count); nvgpu_kfree(g, gr->gpc_zcb_count); nvgpu_kfree(g, gr->gpc_ppc_count); nvgpu_kfree(g, gr->pes_tpc_count[0]); nvgpu_kfree(g, gr->pes_tpc_count[1]); nvgpu_kfree(g, gr->pes_tpc_mask[0]); nvgpu_kfree(g, gr->pes_tpc_mask[1]); nvgpu_kfree(g, gr->sm_to_cluster); nvgpu_kfree(g, gr->gpc_skip_mask); nvgpu_kfree(g, gr->map_tiles); nvgpu_kfree(g, gr->fbp_rop_l2_en_mask); gr->gpc_tpc_count = NULL; gr->gpc_zcb_count = NULL; gr->gpc_ppc_count = NULL; gr->pes_tpc_count[0] = NULL; gr->pes_tpc_count[1] = NULL; gr->pes_tpc_mask[0] = NULL; gr->pes_tpc_mask[1] = NULL; gr->gpc_skip_mask = NULL; gr->map_tiles = NULL; gr->fbp_rop_l2_en_mask = NULL; gr->ctx_vars.valid = false; nvgpu_kfree(g, gr->ctx_vars.ucode.fecs.inst.l); nvgpu_kfree(g, gr->ctx_vars.ucode.fecs.data.l); nvgpu_kfree(g, gr->ctx_vars.ucode.gpccs.inst.l); nvgpu_kfree(g, gr->ctx_vars.ucode.gpccs.data.l); nvgpu_kfree(g, gr->ctx_vars.sw_bundle_init.l); nvgpu_kfree(g, gr->ctx_vars.sw_veid_bundle_init.l); nvgpu_kfree(g, gr->ctx_vars.sw_method_init.l); nvgpu_kfree(g, gr->ctx_vars.sw_ctx_load.l); nvgpu_kfree(g, gr->ctx_vars.sw_non_ctx_load.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.sys.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.gpc.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.tpc.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.zcull_gpc.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.ppc.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.pm_sys.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.pm_gpc.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.pm_tpc.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.pm_ppc.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.perf_sys.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.fbp.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.perf_gpc.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.fbp_router.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.gpc_router.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.pm_ltc.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.pm_fbpa.l); nvgpu_kfree(g, gr->ctx_vars.sw_bundle64_init.l); nvgpu_kfree(g, gr->ctx_vars.ctxsw_regs.pm_cau.l); nvgpu_vfree(g, gr->ctx_vars.local_golden_image); gr->ctx_vars.local_golden_image = NULL; if (gr->ctx_vars.hwpm_ctxsw_buffer_offset_map) { nvgpu_big_free(g, gr->ctx_vars.hwpm_ctxsw_buffer_offset_map); } gr->ctx_vars.hwpm_ctxsw_buffer_offset_map = NULL; gk20a_comptag_allocator_destroy(g, &gr->comp_tags); nvgpu_ecc_remove_support(g); } static int gr_gk20a_init_gr_config(struct gk20a *g, struct gr_gk20a *gr) { u32 gpc_index, pes_index; u32 pes_tpc_mask; u32 pes_tpc_count; u32 pes_heavy_index; u32 gpc_new_skip_mask; u32 tmp; u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); u32 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); tmp = gk20a_readl(g, pri_ringmaster_enum_fbp_r()); gr->num_fbps = pri_ringmaster_enum_fbp_count_v(tmp); tmp = gk20a_readl(g, top_num_gpcs_r()); gr->max_gpc_count = top_num_gpcs_value_v(tmp); tmp = gk20a_readl(g, top_num_fbps_r()); gr->max_fbps_count = top_num_fbps_value_v(tmp); gr->fbp_en_mask = g->ops.gr.get_fbp_en_mask(g); if (gr->fbp_rop_l2_en_mask == NULL) { gr->fbp_rop_l2_en_mask = nvgpu_kzalloc(g, gr->max_fbps_count * sizeof(u32)); if (gr->fbp_rop_l2_en_mask == NULL) { goto clean_up; } } else { memset(gr->fbp_rop_l2_en_mask, 0, gr->max_fbps_count * sizeof(u32)); } tmp = gk20a_readl(g, top_tpc_per_gpc_r()); gr->max_tpc_per_gpc_count = top_tpc_per_gpc_value_v(tmp); gr->max_tpc_count = gr->max_gpc_count * gr->max_tpc_per_gpc_count; tmp = gk20a_readl(g, top_num_fbps_r()); gr->sys_count = top_num_fbps_value_v(tmp); tmp = gk20a_readl(g, pri_ringmaster_enum_gpc_r()); gr->gpc_count = pri_ringmaster_enum_gpc_count_v(tmp); gr->pe_count_per_gpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_PES_PER_GPC); if (WARN(gr->pe_count_per_gpc > GK20A_GR_MAX_PES_PER_GPC, "too many pes per gpc\n")) { goto clean_up; } gr->max_zcull_per_gpc_count = nvgpu_get_litter_value(g, GPU_LIT_NUM_ZCULL_BANKS); if (gr->gpc_count == 0U) { nvgpu_err(g, "gpc_count==0!"); goto clean_up; } if (gr->gpc_tpc_count == NULL) { gr->gpc_tpc_count = nvgpu_kzalloc(g, gr->gpc_count * sizeof(u32)); } else { memset(gr->gpc_tpc_count, 0, gr->gpc_count * sizeof(u32)); } if (gr->gpc_tpc_mask == NULL) { gr->gpc_tpc_mask = nvgpu_kzalloc(g, gr->max_gpc_count * sizeof(u32)); } else { memset(gr->gpc_tpc_mask, 0, gr->max_gpc_count * sizeof(u32)); } if (gr->gpc_zcb_count == NULL) { gr->gpc_zcb_count = nvgpu_kzalloc(g, gr->gpc_count * sizeof(u32)); } else { memset(gr->gpc_zcb_count, 0, gr->gpc_count * sizeof(u32)); } if (gr->gpc_ppc_count == NULL) { gr->gpc_ppc_count = nvgpu_kzalloc(g, gr->gpc_count * sizeof(u32)); } else { memset(gr->gpc_ppc_count, 0, gr->gpc_count * sizeof(u32)); } if (gr->gpc_skip_mask == NULL) { gr->gpc_skip_mask = nvgpu_kzalloc(g, gr_pd_dist_skip_table__size_1_v() * 4 * sizeof(u32)); } else { memset(gr->gpc_skip_mask, 0, gr_pd_dist_skip_table__size_1_v() * 4 * sizeof(u32)); } if ((gr->gpc_tpc_count == NULL) || (gr->gpc_tpc_mask == NULL) || (gr->gpc_zcb_count == NULL) || (gr->gpc_ppc_count == NULL) || (gr->gpc_skip_mask == NULL)) { goto clean_up; } for (gpc_index = 0; gpc_index < gr->max_gpc_count; gpc_index++) { if (g->ops.gr.get_gpc_tpc_mask) { gr->gpc_tpc_mask[gpc_index] = g->ops.gr.get_gpc_tpc_mask(g, gpc_index); } } gr->ppc_count = 0; gr->tpc_count = 0; gr->zcb_count = 0; for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { tmp = gk20a_readl(g, gr_gpc0_fs_gpc_r() + gpc_stride * gpc_index); gr->gpc_tpc_count[gpc_index] = gr_gpc0_fs_gpc_num_available_tpcs_v(tmp); gr->tpc_count += gr->gpc_tpc_count[gpc_index]; gr->gpc_zcb_count[gpc_index] = gr_gpc0_fs_gpc_num_available_zculls_v(tmp); gr->zcb_count += gr->gpc_zcb_count[gpc_index]; for (pes_index = 0; pes_index < gr->pe_count_per_gpc; pes_index++) { if (gr->pes_tpc_count[pes_index] == NULL) { gr->pes_tpc_count[pes_index] = nvgpu_kzalloc(g, gr->gpc_count * sizeof(u32)); gr->pes_tpc_mask[pes_index] = nvgpu_kzalloc(g, gr->gpc_count * sizeof(u32)); if ((gr->pes_tpc_count[pes_index] == NULL) || (gr->pes_tpc_mask[pes_index] == NULL)) { goto clean_up; } } tmp = gk20a_readl(g, gr_gpc0_gpm_pd_pes_tpc_id_mask_r(pes_index) + gpc_index * gpc_stride); pes_tpc_mask = gr_gpc0_gpm_pd_pes_tpc_id_mask_mask_v(tmp); pes_tpc_count = count_bits(pes_tpc_mask); /* detect PES presence by seeing if there are * TPCs connected to it. */ if (pes_tpc_count != 0) { gr->gpc_ppc_count[gpc_index]++; } gr->pes_tpc_count[pes_index][gpc_index] = pes_tpc_count; gr->pes_tpc_mask[pes_index][gpc_index] = pes_tpc_mask; } gr->ppc_count += gr->gpc_ppc_count[gpc_index]; gpc_new_skip_mask = 0; if (gr->pe_count_per_gpc > 1 && gr->pes_tpc_count[0][gpc_index] + gr->pes_tpc_count[1][gpc_index] == 5) { pes_heavy_index = gr->pes_tpc_count[0][gpc_index] > gr->pes_tpc_count[1][gpc_index] ? 0 : 1; gpc_new_skip_mask = gr->pes_tpc_mask[pes_heavy_index][gpc_index] ^ (gr->pes_tpc_mask[pes_heavy_index][gpc_index] & (gr->pes_tpc_mask[pes_heavy_index][gpc_index] - 1)); } else if (gr->pe_count_per_gpc > 1 && (gr->pes_tpc_count[0][gpc_index] + gr->pes_tpc_count[1][gpc_index] == 4) && (gr->pes_tpc_count[0][gpc_index] != gr->pes_tpc_count[1][gpc_index])) { pes_heavy_index = gr->pes_tpc_count[0][gpc_index] > gr->pes_tpc_count[1][gpc_index] ? 0 : 1; gpc_new_skip_mask = gr->pes_tpc_mask[pes_heavy_index][gpc_index] ^ (gr->pes_tpc_mask[pes_heavy_index][gpc_index] & (gr->pes_tpc_mask[pes_heavy_index][gpc_index] - 1)); } gr->gpc_skip_mask[gpc_index] = gpc_new_skip_mask; } /* allocate for max tpc per gpc */ if (gr->sm_to_cluster == NULL) { gr->sm_to_cluster = nvgpu_kzalloc(g, gr->gpc_count * gr->max_tpc_per_gpc_count * sm_per_tpc * sizeof(struct sm_info)); if (!gr->sm_to_cluster) goto clean_up; } else { memset(gr->sm_to_cluster, 0, gr->gpc_count * gr->max_tpc_per_gpc_count * sm_per_tpc * sizeof(struct sm_info)); } gr->no_of_sm = 0; nvgpu_log_info(g, "fbps: %d", gr->num_fbps); nvgpu_log_info(g, "max_gpc_count: %d", gr->max_gpc_count); nvgpu_log_info(g, "max_fbps_count: %d", gr->max_fbps_count); nvgpu_log_info(g, "max_tpc_per_gpc_count: %d", gr->max_tpc_per_gpc_count); nvgpu_log_info(g, "max_zcull_per_gpc_count: %d", gr->max_zcull_per_gpc_count); nvgpu_log_info(g, "max_tpc_count: %d", gr->max_tpc_count); nvgpu_log_info(g, "sys_count: %d", gr->sys_count); nvgpu_log_info(g, "gpc_count: %d", gr->gpc_count); nvgpu_log_info(g, "pe_count_per_gpc: %d", gr->pe_count_per_gpc); nvgpu_log_info(g, "tpc_count: %d", gr->tpc_count); nvgpu_log_info(g, "ppc_count: %d", gr->ppc_count); for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { nvgpu_log_info(g, "gpc_tpc_count[%d] : %d", gpc_index, gr->gpc_tpc_count[gpc_index]); } for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { nvgpu_log_info(g, "gpc_zcb_count[%d] : %d", gpc_index, gr->gpc_zcb_count[gpc_index]); } for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { nvgpu_log_info(g, "gpc_ppc_count[%d] : %d", gpc_index, gr->gpc_ppc_count[gpc_index]); } for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { nvgpu_log_info(g, "gpc_skip_mask[%d] : %d", gpc_index, gr->gpc_skip_mask[gpc_index]); } for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { for (pes_index = 0; pes_index < gr->pe_count_per_gpc; pes_index++) { nvgpu_log_info(g, "pes_tpc_count[%d][%d] : %d", pes_index, gpc_index, gr->pes_tpc_count[pes_index][gpc_index]); } } for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { for (pes_index = 0; pes_index < gr->pe_count_per_gpc; pes_index++) { nvgpu_log_info(g, "pes_tpc_mask[%d][%d] : %d", pes_index, gpc_index, gr->pes_tpc_mask[pes_index][gpc_index]); } } g->ops.gr.bundle_cb_defaults(g); g->ops.gr.cb_size_default(g); g->ops.gr.calc_global_ctx_buffer_size(g); gr->timeslice_mode = gr_gpcs_ppcs_cbm_cfg_timeslice_mode_enable_v(); nvgpu_log_info(g, "bundle_cb_default_size: %d", gr->bundle_cb_default_size); nvgpu_log_info(g, "min_gpm_fifo_depth: %d", gr->min_gpm_fifo_depth); nvgpu_log_info(g, "bundle_cb_token_limit: %d", gr->bundle_cb_token_limit); nvgpu_log_info(g, "attrib_cb_default_size: %d", gr->attrib_cb_default_size); nvgpu_log_info(g, "attrib_cb_size: %d", gr->attrib_cb_size); nvgpu_log_info(g, "alpha_cb_default_size: %d", gr->alpha_cb_default_size); nvgpu_log_info(g, "alpha_cb_size: %d", gr->alpha_cb_size); nvgpu_log_info(g, "timeslice_mode: %d", gr->timeslice_mode); return 0; clean_up: return -ENOMEM; } static u32 prime_set[18] = { 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61 }; static int gr_gk20a_init_map_tiles(struct gk20a *g, struct gr_gk20a *gr) { s32 comm_denom; s32 mul_factor; s32 *init_frac = NULL; s32 *init_err = NULL; s32 *run_err = NULL; s32 *sorted_num_tpcs = NULL; s32 *sorted_to_unsorted_gpc_map = NULL; u32 gpc_index; u32 gpc_mark = 0; u32 num_tpc; u32 max_tpc_count = 0; u32 swap; u32 tile_count; u32 index; bool delete_map = false; bool gpc_sorted; int ret = 0; int num_gpcs = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS); int num_tpc_per_gpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_TPC_PER_GPC); int map_tile_count = num_gpcs * num_tpc_per_gpc; init_frac = nvgpu_kzalloc(g, num_gpcs * sizeof(s32)); init_err = nvgpu_kzalloc(g, num_gpcs * sizeof(s32)); run_err = nvgpu_kzalloc(g, num_gpcs * sizeof(s32)); sorted_num_tpcs = nvgpu_kzalloc(g, num_gpcs * num_tpc_per_gpc * sizeof(s32)); sorted_to_unsorted_gpc_map = nvgpu_kzalloc(g, num_gpcs * sizeof(s32)); if (!((init_frac != NULL) && (init_err != NULL) && (run_err != NULL) && (sorted_num_tpcs != NULL) && (sorted_to_unsorted_gpc_map != NULL))) { ret = -ENOMEM; goto clean_up; } gr->map_row_offset = INVALID_SCREEN_TILE_ROW_OFFSET; if (gr->tpc_count == 3) { gr->map_row_offset = 2; } else if (gr->tpc_count < 3) { gr->map_row_offset = 1; } else { gr->map_row_offset = 3; for (index = 1; index < 18; index++) { u32 prime = prime_set[index]; if ((gr->tpc_count % prime) != 0) { gr->map_row_offset = prime; break; } } } switch (gr->tpc_count) { case 15: gr->map_row_offset = 6; break; case 14: gr->map_row_offset = 5; break; case 13: gr->map_row_offset = 2; break; case 11: gr->map_row_offset = 7; break; case 10: gr->map_row_offset = 6; break; case 7: case 5: gr->map_row_offset = 1; break; default: break; } if (gr->map_tiles) { if (gr->map_tile_count != gr->tpc_count) { delete_map = true; } for (tile_count = 0; tile_count < gr->map_tile_count; tile_count++) { if (gr_gk20a_get_map_tile_count(gr, tile_count) >= gr->tpc_count) { delete_map = true; } } if (delete_map) { nvgpu_kfree(g, gr->map_tiles); gr->map_tiles = NULL; gr->map_tile_count = 0; } } if (gr->map_tiles == NULL) { gr->map_tiles = nvgpu_kzalloc(g, map_tile_count * sizeof(u8)); if (gr->map_tiles == NULL) { ret = -ENOMEM; goto clean_up; } gr->map_tile_count = map_tile_count; for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { sorted_num_tpcs[gpc_index] = gr->gpc_tpc_count[gpc_index]; sorted_to_unsorted_gpc_map[gpc_index] = gpc_index; } gpc_sorted = false; while (!gpc_sorted) { gpc_sorted = true; for (gpc_index = 0; gpc_index < gr->gpc_count - 1; gpc_index++) { if (sorted_num_tpcs[gpc_index + 1] > sorted_num_tpcs[gpc_index]) { gpc_sorted = false; swap = sorted_num_tpcs[gpc_index]; sorted_num_tpcs[gpc_index] = sorted_num_tpcs[gpc_index + 1]; sorted_num_tpcs[gpc_index + 1] = swap; swap = sorted_to_unsorted_gpc_map[gpc_index]; sorted_to_unsorted_gpc_map[gpc_index] = sorted_to_unsorted_gpc_map[gpc_index + 1]; sorted_to_unsorted_gpc_map[gpc_index + 1] = swap; } } } for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { if (gr->gpc_tpc_count[gpc_index] > max_tpc_count) { max_tpc_count = gr->gpc_tpc_count[gpc_index]; } } mul_factor = gr->gpc_count * max_tpc_count; if (mul_factor & 0x1) { mul_factor = 2; } else { mul_factor = 1; } comm_denom = gr->gpc_count * max_tpc_count * mul_factor; for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { num_tpc = sorted_num_tpcs[gpc_index]; init_frac[gpc_index] = num_tpc * gr->gpc_count * mul_factor; if (num_tpc != 0) { init_err[gpc_index] = gpc_index * max_tpc_count * mul_factor - comm_denom/2; } else { init_err[gpc_index] = 0; } run_err[gpc_index] = init_frac[gpc_index] + init_err[gpc_index]; } while (gpc_mark < gr->tpc_count) { for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { if ((run_err[gpc_index] * 2) >= comm_denom) { gr->map_tiles[gpc_mark++] = (u8)sorted_to_unsorted_gpc_map[gpc_index]; run_err[gpc_index] += init_frac[gpc_index] - comm_denom; } else { run_err[gpc_index] += init_frac[gpc_index]; } } } } clean_up: nvgpu_kfree(g, init_frac); nvgpu_kfree(g, init_err); nvgpu_kfree(g, run_err); nvgpu_kfree(g, sorted_num_tpcs); nvgpu_kfree(g, sorted_to_unsorted_gpc_map); if (ret) { nvgpu_err(g, "fail"); } else { nvgpu_log_fn(g, "done"); } return ret; } static int gr_gk20a_init_zcull(struct gk20a *g, struct gr_gk20a *gr) { struct gr_zcull_gk20a *zcull = &gr->zcull; zcull->aliquot_width = gr->tpc_count * 16; zcull->aliquot_height = 16; zcull->width_align_pixels = gr->tpc_count * 16; zcull->height_align_pixels = 32; zcull->aliquot_size = zcull->aliquot_width * zcull->aliquot_height; /* assume no floor sweeping since we only have 1 tpc in 1 gpc */ zcull->pixel_squares_by_aliquots = gr->zcb_count * 16 * 16 * gr->tpc_count / (gr->gpc_count * gr->gpc_tpc_count[0]); zcull->total_aliquots = gr_gpc0_zcull_total_ram_size_num_aliquots_f( gk20a_readl(g, gr_gpc0_zcull_total_ram_size_r())); return 0; } u32 gr_gk20a_get_ctxsw_zcull_size(struct gk20a *g, struct gr_gk20a *gr) { /* assuming gr has already been initialized */ return gr->ctx_vars.zcull_ctxsw_image_size; } int gr_gk20a_bind_ctxsw_zcull(struct gk20a *g, struct gr_gk20a *gr, struct channel_gk20a *c, u64 zcull_va, u32 mode) { struct tsg_gk20a *tsg; struct zcull_ctx_desc *zcull_ctx; tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } zcull_ctx = &tsg->gr_ctx.zcull_ctx; zcull_ctx->ctx_sw_mode = mode; zcull_ctx->gpu_va = zcull_va; /* TBD: don't disable channel in sw method processing */ return gr_gk20a_ctx_zcull_setup(g, c); } int gr_gk20a_get_zcull_info(struct gk20a *g, struct gr_gk20a *gr, struct gr_zcull_info *zcull_params) { struct gr_zcull_gk20a *zcull = &gr->zcull; zcull_params->width_align_pixels = zcull->width_align_pixels; zcull_params->height_align_pixels = zcull->height_align_pixels; zcull_params->pixel_squares_by_aliquots = zcull->pixel_squares_by_aliquots; zcull_params->aliquot_total = zcull->total_aliquots; zcull_params->region_byte_multiplier = gr->gpc_count * gr_zcull_bytes_per_aliquot_per_gpu_v(); zcull_params->region_header_size = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS) * gr_zcull_save_restore_header_bytes_per_gpc_v(); zcull_params->subregion_header_size = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS) * gr_zcull_save_restore_subregion_header_bytes_per_gpc_v(); zcull_params->subregion_width_align_pixels = gr->tpc_count * gr_gpc0_zcull_zcsize_width_subregion__multiple_v(); zcull_params->subregion_height_align_pixels = gr_gpc0_zcull_zcsize_height_subregion__multiple_v(); zcull_params->subregion_count = gr_zcull_subregion_qty_v(); return 0; } int gr_gk20a_add_zbc_color(struct gk20a *g, struct gr_gk20a *gr, struct zbc_entry *color_val, u32 index) { u32 i; /* update l2 table */ g->ops.ltc.set_zbc_color_entry(g, color_val, index); /* update ds table */ gk20a_writel(g, gr_ds_zbc_color_r_r(), gr_ds_zbc_color_r_val_f(color_val->color_ds[0])); gk20a_writel(g, gr_ds_zbc_color_g_r(), gr_ds_zbc_color_g_val_f(color_val->color_ds[1])); gk20a_writel(g, gr_ds_zbc_color_b_r(), gr_ds_zbc_color_b_val_f(color_val->color_ds[2])); gk20a_writel(g, gr_ds_zbc_color_a_r(), gr_ds_zbc_color_a_val_f(color_val->color_ds[3])); gk20a_writel(g, gr_ds_zbc_color_fmt_r(), gr_ds_zbc_color_fmt_val_f(color_val->format)); gk20a_writel(g, gr_ds_zbc_tbl_index_r(), gr_ds_zbc_tbl_index_val_f(index + GK20A_STARTOF_ZBC_TABLE)); /* trigger the write */ gk20a_writel(g, gr_ds_zbc_tbl_ld_r(), gr_ds_zbc_tbl_ld_select_c_f() | gr_ds_zbc_tbl_ld_action_write_f() | gr_ds_zbc_tbl_ld_trigger_active_f()); /* update local copy */ for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) { gr->zbc_col_tbl[index].color_l2[i] = color_val->color_l2[i]; gr->zbc_col_tbl[index].color_ds[i] = color_val->color_ds[i]; } gr->zbc_col_tbl[index].format = color_val->format; gr->zbc_col_tbl[index].ref_cnt++; return 0; } int gr_gk20a_add_zbc_depth(struct gk20a *g, struct gr_gk20a *gr, struct zbc_entry *depth_val, u32 index) { /* update l2 table */ g->ops.ltc.set_zbc_depth_entry(g, depth_val, index); /* update ds table */ gk20a_writel(g, gr_ds_zbc_z_r(), gr_ds_zbc_z_val_f(depth_val->depth)); gk20a_writel(g, gr_ds_zbc_z_fmt_r(), gr_ds_zbc_z_fmt_val_f(depth_val->format)); gk20a_writel(g, gr_ds_zbc_tbl_index_r(), gr_ds_zbc_tbl_index_val_f(index + GK20A_STARTOF_ZBC_TABLE)); /* trigger the write */ gk20a_writel(g, gr_ds_zbc_tbl_ld_r(), gr_ds_zbc_tbl_ld_select_z_f() | gr_ds_zbc_tbl_ld_action_write_f() | gr_ds_zbc_tbl_ld_trigger_active_f()); /* update local copy */ gr->zbc_dep_tbl[index].depth = depth_val->depth; gr->zbc_dep_tbl[index].format = depth_val->format; gr->zbc_dep_tbl[index].ref_cnt++; return 0; } void gr_gk20a_pmu_save_zbc(struct gk20a *g, u32 entries) { struct fifo_gk20a *f = &g->fifo; struct fifo_engine_info_gk20a *gr_info = NULL; u32 ret; u32 engine_id; engine_id = gk20a_fifo_get_gr_engine_id(g); gr_info = (f->engine_info + engine_id); ret = gk20a_fifo_disable_engine_activity(g, gr_info, true); if (ret) { nvgpu_err(g, "failed to disable gr engine activity"); return; } ret = g->ops.gr.wait_empty(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (ret) { nvgpu_err(g, "failed to idle graphics"); goto clean_up; } /* update zbc */ g->ops.gr.pmu_save_zbc(g, entries); clean_up: ret = gk20a_fifo_enable_engine_activity(g, gr_info); if (ret) { nvgpu_err(g, "failed to enable gr engine activity"); } } int gr_gk20a_add_zbc(struct gk20a *g, struct gr_gk20a *gr, struct zbc_entry *zbc_val) { struct zbc_color_table *c_tbl; struct zbc_depth_table *d_tbl; u32 i; int ret = -ENOSPC; bool added = false; u32 entries; /* no endian swap ? */ nvgpu_mutex_acquire(&gr->zbc_lock); nvgpu_speculation_barrier(); switch (zbc_val->type) { case GK20A_ZBC_TYPE_COLOR: /* search existing tables */ for (i = 0; i < gr->max_used_color_index; i++) { c_tbl = &gr->zbc_col_tbl[i]; if ((c_tbl->ref_cnt != 0U) && (c_tbl->format == zbc_val->format) && (memcmp(c_tbl->color_ds, zbc_val->color_ds, sizeof(zbc_val->color_ds)) == 0) && (memcmp(c_tbl->color_l2, zbc_val->color_l2, sizeof(zbc_val->color_l2)) == 0)) { added = true; c_tbl->ref_cnt++; ret = 0; break; } } /* add new table */ if (!added && gr->max_used_color_index < GK20A_ZBC_TABLE_SIZE) { c_tbl = &gr->zbc_col_tbl[gr->max_used_color_index]; WARN_ON(c_tbl->ref_cnt != 0); ret = g->ops.gr.add_zbc_color(g, gr, zbc_val, gr->max_used_color_index); if (ret == 0) { gr->max_used_color_index++; } } break; case GK20A_ZBC_TYPE_DEPTH: /* search existing tables */ for (i = 0; i < gr->max_used_depth_index; i++) { d_tbl = &gr->zbc_dep_tbl[i]; if ((d_tbl->ref_cnt != 0U) && (d_tbl->depth == zbc_val->depth) && (d_tbl->format == zbc_val->format)) { added = true; d_tbl->ref_cnt++; ret = 0; break; } } /* add new table */ if (!added && gr->max_used_depth_index < GK20A_ZBC_TABLE_SIZE) { d_tbl = &gr->zbc_dep_tbl[gr->max_used_depth_index]; WARN_ON(d_tbl->ref_cnt != 0); ret = g->ops.gr.add_zbc_depth(g, gr, zbc_val, gr->max_used_depth_index); if (ret == 0) { gr->max_used_depth_index++; } } break; case T19X_ZBC: if (g->ops.gr.add_zbc_type_s) { added = g->ops.gr.add_zbc_type_s(g, gr, zbc_val, &ret); } else { nvgpu_err(g, "invalid zbc table type %d", zbc_val->type); ret = -EINVAL; goto err_mutex; } break; default: nvgpu_err(g, "invalid zbc table type %d", zbc_val->type); ret = -EINVAL; goto err_mutex; } if (!added && ret == 0) { /* update zbc for elpg only when new entry is added */ entries = max(gr->max_used_color_index, gr->max_used_depth_index); g->ops.gr.pmu_save_zbc(g, entries); } err_mutex: nvgpu_mutex_release(&gr->zbc_lock); return ret; } /* get a zbc table entry specified by index * return table size when type is invalid */ int gr_gk20a_query_zbc(struct gk20a *g, struct gr_gk20a *gr, struct zbc_query_params *query_params) { u32 index = query_params->index_size; u32 i; nvgpu_speculation_barrier(); switch (query_params->type) { case GK20A_ZBC_TYPE_INVALID: query_params->index_size = GK20A_ZBC_TABLE_SIZE; break; case GK20A_ZBC_TYPE_COLOR: if (index >= GK20A_ZBC_TABLE_SIZE) { nvgpu_err(g, "invalid zbc color table index"); return -EINVAL; } nvgpu_speculation_barrier(); for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) { query_params->color_l2[i] = gr->zbc_col_tbl[index].color_l2[i]; query_params->color_ds[i] = gr->zbc_col_tbl[index].color_ds[i]; } query_params->format = gr->zbc_col_tbl[index].format; query_params->ref_cnt = gr->zbc_col_tbl[index].ref_cnt; break; case GK20A_ZBC_TYPE_DEPTH: if (index >= GK20A_ZBC_TABLE_SIZE) { nvgpu_err(g, "invalid zbc depth table index"); return -EINVAL; } nvgpu_speculation_barrier(); query_params->depth = gr->zbc_dep_tbl[index].depth; query_params->format = gr->zbc_dep_tbl[index].format; query_params->ref_cnt = gr->zbc_dep_tbl[index].ref_cnt; break; case T19X_ZBC: if (g->ops.gr.zbc_s_query_table) { return g->ops.gr.zbc_s_query_table(g, gr, query_params); } else { nvgpu_err(g, "invalid zbc table type"); return -EINVAL; } break; default: nvgpu_err(g, "invalid zbc table type"); return -EINVAL; } return 0; } static int gr_gk20a_load_zbc_table(struct gk20a *g, struct gr_gk20a *gr) { unsigned int i; int ret; for (i = 0; i < gr->max_used_color_index; i++) { struct zbc_color_table *c_tbl = &gr->zbc_col_tbl[i]; struct zbc_entry zbc_val; zbc_val.type = GK20A_ZBC_TYPE_COLOR; memcpy(zbc_val.color_ds, c_tbl->color_ds, sizeof(zbc_val.color_ds)); memcpy(zbc_val.color_l2, c_tbl->color_l2, sizeof(zbc_val.color_l2)); zbc_val.format = c_tbl->format; ret = g->ops.gr.add_zbc_color(g, gr, &zbc_val, i); if (ret) { return ret; } } for (i = 0; i < gr->max_used_depth_index; i++) { struct zbc_depth_table *d_tbl = &gr->zbc_dep_tbl[i]; struct zbc_entry zbc_val; zbc_val.type = GK20A_ZBC_TYPE_DEPTH; zbc_val.depth = d_tbl->depth; zbc_val.format = d_tbl->format; ret = g->ops.gr.add_zbc_depth(g, gr, &zbc_val, i); if (ret) { return ret; } } if (g->ops.gr.load_zbc_s_tbl) { ret = g->ops.gr.load_zbc_s_tbl(g, gr); if (ret) { return ret; } } return 0; } int gr_gk20a_load_zbc_default_table(struct gk20a *g, struct gr_gk20a *gr) { struct zbc_entry zbc_val; u32 i = 0; int err = 0; err = nvgpu_mutex_init(&gr->zbc_lock); if (err != 0) { nvgpu_err(g, "Error in zbc_lock mutex initialization"); return err; } /* load default color table */ zbc_val.type = GK20A_ZBC_TYPE_COLOR; /* Opaque black (i.e. solid black, fmt 0x28 = A8B8G8R8) */ zbc_val.format = gr_ds_zbc_color_fmt_val_a8_b8_g8_r8_v(); for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) { zbc_val.color_ds[i] = 0; zbc_val.color_l2[i] = 0; } zbc_val.color_l2[0] = 0xff000000; zbc_val.color_ds[3] = 0x3f800000; err = gr_gk20a_add_zbc(g, gr, &zbc_val); if (err != 0) { goto color_fail; } /* Transparent black = (fmt 1 = zero) */ zbc_val.format = gr_ds_zbc_color_fmt_val_zero_v(); for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) { zbc_val.color_ds[i] = 0; zbc_val.color_l2[i] = 0; } err = gr_gk20a_add_zbc(g, gr, &zbc_val); if (err != 0) { goto color_fail; } /* Opaque white (i.e. solid white) = (fmt 2 = uniform 1) */ zbc_val.format = gr_ds_zbc_color_fmt_val_unorm_one_v(); for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) { zbc_val.color_ds[i] = 0x3f800000; zbc_val.color_l2[i] = 0xffffffff; } err = gr_gk20a_add_zbc(g, gr, &zbc_val); if (err != 0) { goto color_fail; } gr->max_default_color_index = 3; /* load default depth table */ zbc_val.type = GK20A_ZBC_TYPE_DEPTH; zbc_val.format = gr_ds_zbc_z_fmt_val_fp32_v(); zbc_val.depth = 0x3f800000; err = gr_gk20a_add_zbc(g, gr, &zbc_val); if (err != 0) { goto depth_fail; } zbc_val.format = gr_ds_zbc_z_fmt_val_fp32_v(); zbc_val.depth = 0; err = gr_gk20a_add_zbc(g, gr, &zbc_val); if (err != 0) { goto depth_fail; } gr->max_default_depth_index = 2; if (g->ops.gr.load_zbc_s_default_tbl) { err = g->ops.gr.load_zbc_s_default_tbl(g, gr); if (err != 0) { return err; } } return 0; color_fail: nvgpu_err(g, "fail to load default zbc color table"); return err; depth_fail: nvgpu_err(g, "fail to load default zbc depth table"); return err; } int _gk20a_gr_zbc_set_table(struct gk20a *g, struct gr_gk20a *gr, struct zbc_entry *zbc_val) { struct fifo_gk20a *f = &g->fifo; struct fifo_engine_info_gk20a *gr_info = NULL; int ret; u32 engine_id; engine_id = gk20a_fifo_get_gr_engine_id(g); gr_info = (f->engine_info + engine_id); ret = gk20a_fifo_disable_engine_activity(g, gr_info, true); if (ret) { nvgpu_err(g, "failed to disable gr engine activity"); return ret; } ret = g->ops.gr.wait_empty(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (ret) { nvgpu_err(g, "failed to idle graphics"); goto clean_up; } ret = gr_gk20a_add_zbc(g, gr, zbc_val); clean_up: if (gk20a_fifo_enable_engine_activity(g, gr_info)) { nvgpu_err(g, "failed to enable gr engine activity"); } return ret; } int gk20a_gr_zbc_set_table(struct gk20a *g, struct gr_gk20a *gr, struct zbc_entry *zbc_val) { nvgpu_log_fn(g, " "); return gr_gk20a_elpg_protected_call(g, gr_gk20a_add_zbc(g, gr, zbc_val)); } void gr_gk20a_program_zcull_mapping(struct gk20a *g, u32 zcull_num_entries, u32 *zcull_map_tiles) { u32 val; nvgpu_log_fn(g, " "); if (zcull_num_entries >= 8) { nvgpu_log_fn(g, "map0"); val = gr_gpcs_zcull_sm_in_gpc_number_map0_tile_0_f( zcull_map_tiles[0]) | gr_gpcs_zcull_sm_in_gpc_number_map0_tile_1_f( zcull_map_tiles[1]) | gr_gpcs_zcull_sm_in_gpc_number_map0_tile_2_f( zcull_map_tiles[2]) | gr_gpcs_zcull_sm_in_gpc_number_map0_tile_3_f( zcull_map_tiles[3]) | gr_gpcs_zcull_sm_in_gpc_number_map0_tile_4_f( zcull_map_tiles[4]) | gr_gpcs_zcull_sm_in_gpc_number_map0_tile_5_f( zcull_map_tiles[5]) | gr_gpcs_zcull_sm_in_gpc_number_map0_tile_6_f( zcull_map_tiles[6]) | gr_gpcs_zcull_sm_in_gpc_number_map0_tile_7_f( zcull_map_tiles[7]); gk20a_writel(g, gr_gpcs_zcull_sm_in_gpc_number_map0_r(), val); } if (zcull_num_entries >= 16) { nvgpu_log_fn(g, "map1"); val = gr_gpcs_zcull_sm_in_gpc_number_map1_tile_8_f( zcull_map_tiles[8]) | gr_gpcs_zcull_sm_in_gpc_number_map1_tile_9_f( zcull_map_tiles[9]) | gr_gpcs_zcull_sm_in_gpc_number_map1_tile_10_f( zcull_map_tiles[10]) | gr_gpcs_zcull_sm_in_gpc_number_map1_tile_11_f( zcull_map_tiles[11]) | gr_gpcs_zcull_sm_in_gpc_number_map1_tile_12_f( zcull_map_tiles[12]) | gr_gpcs_zcull_sm_in_gpc_number_map1_tile_13_f( zcull_map_tiles[13]) | gr_gpcs_zcull_sm_in_gpc_number_map1_tile_14_f( zcull_map_tiles[14]) | gr_gpcs_zcull_sm_in_gpc_number_map1_tile_15_f( zcull_map_tiles[15]); gk20a_writel(g, gr_gpcs_zcull_sm_in_gpc_number_map1_r(), val); } if (zcull_num_entries >= 24) { nvgpu_log_fn(g, "map2"); val = gr_gpcs_zcull_sm_in_gpc_number_map2_tile_16_f( zcull_map_tiles[16]) | gr_gpcs_zcull_sm_in_gpc_number_map2_tile_17_f( zcull_map_tiles[17]) | gr_gpcs_zcull_sm_in_gpc_number_map2_tile_18_f( zcull_map_tiles[18]) | gr_gpcs_zcull_sm_in_gpc_number_map2_tile_19_f( zcull_map_tiles[19]) | gr_gpcs_zcull_sm_in_gpc_number_map2_tile_20_f( zcull_map_tiles[20]) | gr_gpcs_zcull_sm_in_gpc_number_map2_tile_21_f( zcull_map_tiles[21]) | gr_gpcs_zcull_sm_in_gpc_number_map2_tile_22_f( zcull_map_tiles[22]) | gr_gpcs_zcull_sm_in_gpc_number_map2_tile_23_f( zcull_map_tiles[23]); gk20a_writel(g, gr_gpcs_zcull_sm_in_gpc_number_map2_r(), val); } if (zcull_num_entries >= 32) { nvgpu_log_fn(g, "map3"); val = gr_gpcs_zcull_sm_in_gpc_number_map3_tile_24_f( zcull_map_tiles[24]) | gr_gpcs_zcull_sm_in_gpc_number_map3_tile_25_f( zcull_map_tiles[25]) | gr_gpcs_zcull_sm_in_gpc_number_map3_tile_26_f( zcull_map_tiles[26]) | gr_gpcs_zcull_sm_in_gpc_number_map3_tile_27_f( zcull_map_tiles[27]) | gr_gpcs_zcull_sm_in_gpc_number_map3_tile_28_f( zcull_map_tiles[28]) | gr_gpcs_zcull_sm_in_gpc_number_map3_tile_29_f( zcull_map_tiles[29]) | gr_gpcs_zcull_sm_in_gpc_number_map3_tile_30_f( zcull_map_tiles[30]) | gr_gpcs_zcull_sm_in_gpc_number_map3_tile_31_f( zcull_map_tiles[31]); gk20a_writel(g, gr_gpcs_zcull_sm_in_gpc_number_map3_r(), val); } } static int gr_gk20a_zcull_init_hw(struct gk20a *g, struct gr_gk20a *gr) { u32 gpc_index, gpc_tpc_count, gpc_zcull_count; u32 *zcull_map_tiles, *zcull_bank_counters; u32 map_counter; u32 rcp_conserv; u32 offset; bool floorsweep = false; u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); u32 num_gpcs = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS); u32 num_tpc_per_gpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_TPC_PER_GPC); u32 zcull_alloc_num = num_gpcs * num_tpc_per_gpc; u32 map_tile_count; if (gr->map_tiles == NULL) { return -1; } if (zcull_alloc_num % 8 != 0) { /* Total 8 fields per map reg i.e. tile_0 to tile_7*/ zcull_alloc_num += (zcull_alloc_num % 8); } zcull_map_tiles = nvgpu_kzalloc(g, zcull_alloc_num * sizeof(u32)); if (zcull_map_tiles == NULL) { nvgpu_err(g, "failed to allocate zcull map titles"); return -ENOMEM; } zcull_bank_counters = nvgpu_kzalloc(g, zcull_alloc_num * sizeof(u32)); if (zcull_bank_counters == NULL) { nvgpu_err(g, "failed to allocate zcull bank counters"); nvgpu_kfree(g, zcull_map_tiles); return -ENOMEM; } for (map_counter = 0; map_counter < gr->tpc_count; map_counter++) { map_tile_count = gr_gk20a_get_map_tile_count(gr, map_counter); zcull_map_tiles[map_counter] = zcull_bank_counters[map_tile_count]; zcull_bank_counters[map_tile_count]++; } if (g->ops.gr.program_zcull_mapping != NULL) { g->ops.gr.program_zcull_mapping(g, zcull_alloc_num, zcull_map_tiles); } nvgpu_kfree(g, zcull_map_tiles); nvgpu_kfree(g, zcull_bank_counters); for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { gpc_tpc_count = gr->gpc_tpc_count[gpc_index]; gpc_zcull_count = gr->gpc_zcb_count[gpc_index]; if (gpc_zcull_count != gr->max_zcull_per_gpc_count && gpc_zcull_count < gpc_tpc_count) { nvgpu_err(g, "zcull_banks (%d) less than tpcs (%d) for gpc (%d)", gpc_zcull_count, gpc_tpc_count, gpc_index); return -EINVAL; } if (gpc_zcull_count != gr->max_zcull_per_gpc_count && gpc_zcull_count != 0) { floorsweep = true; } } /* ceil(1.0f / SM_NUM * gr_gpc0_zcull_sm_num_rcp_conservative__max_v()) */ rcp_conserv = DIV_ROUND_UP(gr_gpc0_zcull_sm_num_rcp_conservative__max_v(), gr->gpc_tpc_count[0]); for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { offset = gpc_index * gpc_stride; if (floorsweep) { gk20a_writel(g, gr_gpc0_zcull_ram_addr_r() + offset, gr_gpc0_zcull_ram_addr_row_offset_f(gr->map_row_offset) | gr_gpc0_zcull_ram_addr_tiles_per_hypertile_row_per_gpc_f( gr->max_zcull_per_gpc_count)); } else { gk20a_writel(g, gr_gpc0_zcull_ram_addr_r() + offset, gr_gpc0_zcull_ram_addr_row_offset_f(gr->map_row_offset) | gr_gpc0_zcull_ram_addr_tiles_per_hypertile_row_per_gpc_f( gr->gpc_tpc_count[gpc_index])); } gk20a_writel(g, gr_gpc0_zcull_fs_r() + offset, gr_gpc0_zcull_fs_num_active_banks_f(gr->gpc_zcb_count[gpc_index]) | gr_gpc0_zcull_fs_num_sms_f(gr->tpc_count)); gk20a_writel(g, gr_gpc0_zcull_sm_num_rcp_r() + offset, gr_gpc0_zcull_sm_num_rcp_conservative_f(rcp_conserv)); } gk20a_writel(g, gr_gpcs_ppcs_wwdx_sm_num_rcp_r(), gr_gpcs_ppcs_wwdx_sm_num_rcp_conservative_f(rcp_conserv)); return 0; } void gk20a_gr_enable_exceptions(struct gk20a *g) { gk20a_writel(g, gr_exception_r(), 0xFFFFFFFF); gk20a_writel(g, gr_exception_en_r(), 0xFFFFFFFF); gk20a_writel(g, gr_exception1_r(), 0xFFFFFFFF); gk20a_writel(g, gr_exception1_en_r(), 0xFFFFFFFF); gk20a_writel(g, gr_exception2_r(), 0xFFFFFFFF); gk20a_writel(g, gr_exception2_en_r(), 0xFFFFFFFF); } void gk20a_gr_enable_gpc_exceptions(struct gk20a *g) { struct gr_gk20a *gr = &g->gr; u32 tpc_mask; gk20a_writel(g, gr_gpcs_tpcs_tpccs_tpc_exception_en_r(), gr_gpcs_tpcs_tpccs_tpc_exception_en_tex_enabled_f() | gr_gpcs_tpcs_tpccs_tpc_exception_en_sm_enabled_f()); tpc_mask = gr_gpcs_gpccs_gpc_exception_en_tpc_f((1 << gr->max_tpc_per_gpc_count) - 1); gk20a_writel(g, gr_gpcs_gpccs_gpc_exception_en_r(), tpc_mask); } void gr_gk20a_enable_hww_exceptions(struct gk20a *g) { /* enable exceptions */ gk20a_writel(g, gr_fe_hww_esr_r(), gr_fe_hww_esr_en_enable_f() | gr_fe_hww_esr_reset_active_f()); gk20a_writel(g, gr_memfmt_hww_esr_r(), gr_memfmt_hww_esr_en_enable_f() | gr_memfmt_hww_esr_reset_active_f()); } void gr_gk20a_fecs_host_int_enable(struct gk20a *g) { gk20a_writel(g, gr_fecs_host_int_enable_r(), gr_fecs_host_int_enable_ctxsw_intr1_enable_f() | gr_fecs_host_int_enable_fault_during_ctxsw_enable_f() | gr_fecs_host_int_enable_umimp_firmware_method_enable_f() | gr_fecs_host_int_enable_umimp_illegal_method_enable_f() | gr_fecs_host_int_enable_watchdog_enable_f()); } static int gk20a_init_gr_setup_hw(struct gk20a *g) { struct gr_gk20a *gr = &g->gr; struct aiv_list_gk20a *sw_ctx_load = &g->gr.ctx_vars.sw_ctx_load; struct av_list_gk20a *sw_method_init = &g->gr.ctx_vars.sw_method_init; u32 data; u32 last_method_data = 0; u32 i, err; nvgpu_log_fn(g, " "); if (g->ops.gr.init_gpc_mmu) { g->ops.gr.init_gpc_mmu(g); } /* load gr floorsweeping registers */ data = gk20a_readl(g, gr_gpc0_ppc0_pes_vsc_strem_r()); data = set_field(data, gr_gpc0_ppc0_pes_vsc_strem_master_pe_m(), gr_gpc0_ppc0_pes_vsc_strem_master_pe_true_f()); gk20a_writel(g, gr_gpc0_ppc0_pes_vsc_strem_r(), data); gr_gk20a_zcull_init_hw(g, gr); if (g->ops.priv_ring.set_ppriv_timeout_settings != NULL) { g->ops.priv_ring.set_ppriv_timeout_settings(g); } /* enable fifo access */ gk20a_writel(g, gr_gpfifo_ctl_r(), gr_gpfifo_ctl_access_enabled_f() | gr_gpfifo_ctl_semaphore_access_enabled_f()); /* TBD: reload gr ucode when needed */ /* enable interrupts */ gk20a_writel(g, gr_intr_r(), 0xFFFFFFFF); gk20a_writel(g, gr_intr_en_r(), 0xFFFFFFFF); /* enable fecs error interrupts */ g->ops.gr.fecs_host_int_enable(g); g->ops.gr.enable_hww_exceptions(g); g->ops.gr.set_hww_esr_report_mask(g); /* enable TPC exceptions per GPC */ if (g->ops.gr.enable_gpc_exceptions) { g->ops.gr.enable_gpc_exceptions(g); } /* enable ECC for L1/SM */ if (g->ops.gr.ecc_init_scrub_reg) { g->ops.gr.ecc_init_scrub_reg(g); } /* TBD: enable per BE exceptions */ /* reset and enable exceptions */ g->ops.gr.enable_exceptions(g); gr_gk20a_load_zbc_table(g, gr); if (g->ops.ltc.init_cbc) { g->ops.ltc.init_cbc(g, gr); } if (g->ops.fb.init_cbc) { g->ops.fb.init_cbc(g, gr); } if (g->ops.gr.disable_rd_coalesce) { g->ops.gr.disable_rd_coalesce(g); } /* load ctx init */ for (i = 0; i < sw_ctx_load->count; i++) { gk20a_writel(g, sw_ctx_load->l[i].addr, sw_ctx_load->l[i].value); } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto out; } if (g->ops.gr.init_preemption_state) { err = g->ops.gr.init_preemption_state(g); if (err != 0U) { goto out; } } /* disable fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_disabled_f()); /* override a few ctx state registers */ g->ops.gr.commit_global_timeslice(g, NULL); /* floorsweep anything left */ err = g->ops.gr.init_fs_state(g); if (err != 0U) { goto out; } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto restore_fe_go_idle; } restore_fe_go_idle: /* restore fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_prod_f()); if ((err != 0U) || (gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT) != 0)) { goto out; } /* load method init */ if (sw_method_init->count) { gk20a_writel(g, gr_pri_mme_shadow_raw_data_r(), sw_method_init->l[0].value); gk20a_writel(g, gr_pri_mme_shadow_raw_index_r(), gr_pri_mme_shadow_raw_index_write_trigger_f() | sw_method_init->l[0].addr); last_method_data = sw_method_init->l[0].value; } for (i = 1; i < sw_method_init->count; i++) { if (sw_method_init->l[i].value != last_method_data) { gk20a_writel(g, gr_pri_mme_shadow_raw_data_r(), sw_method_init->l[i].value); last_method_data = sw_method_init->l[i].value; } gk20a_writel(g, gr_pri_mme_shadow_raw_index_r(), gr_pri_mme_shadow_raw_index_write_trigger_f() | sw_method_init->l[i].addr); } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); out: nvgpu_log_fn(g, "done"); return err; } static int gk20a_init_gr_prepare(struct gk20a *g) { u32 err = 0; /* reset gr engine */ g->ops.mc.reset(g, g->ops.mc.reset_mask(g, NVGPU_UNIT_GRAPH) | g->ops.mc.reset_mask(g, NVGPU_UNIT_BLG) | g->ops.mc.reset_mask(g, NVGPU_UNIT_PERFMON)); nvgpu_cg_init_gr_load_gating_prod(g); /* Disable elcg until it gets enabled later in the init*/ nvgpu_cg_elcg_disable_no_wait(g); /* enable fifo access */ gk20a_writel(g, gr_gpfifo_ctl_r(), gr_gpfifo_ctl_access_enabled_f() | gr_gpfifo_ctl_semaphore_access_enabled_f()); if (!g->gr.ctx_vars.valid) { err = gr_gk20a_init_ctx_vars(g, &g->gr); if (err != 0U) { nvgpu_err(g, "fail to load gr init ctx"); } } return err; } static int gr_gk20a_wait_mem_scrubbing(struct gk20a *g) { struct nvgpu_timeout timeout; bool fecs_scrubbing; bool gpccs_scrubbing; nvgpu_log_fn(g, " "); nvgpu_timeout_init(g, &timeout, CTXSW_MEM_SCRUBBING_TIMEOUT_MAX / CTXSW_MEM_SCRUBBING_TIMEOUT_DEFAULT, NVGPU_TIMER_RETRY_TIMER); do { fecs_scrubbing = gk20a_readl(g, gr_fecs_dmactl_r()) & (gr_fecs_dmactl_imem_scrubbing_m() | gr_fecs_dmactl_dmem_scrubbing_m()); gpccs_scrubbing = gk20a_readl(g, gr_gpccs_dmactl_r()) & (gr_gpccs_dmactl_imem_scrubbing_m() | gr_gpccs_dmactl_imem_scrubbing_m()); if (!fecs_scrubbing && !gpccs_scrubbing) { nvgpu_log_fn(g, "done"); return 0; } nvgpu_udelay(CTXSW_MEM_SCRUBBING_TIMEOUT_DEFAULT); } while (nvgpu_timeout_expired(&timeout) == 0); nvgpu_err(g, "Falcon mem scrubbing timeout"); return -ETIMEDOUT; } static int gr_gk20a_init_ctxsw(struct gk20a *g) { u32 err = 0; err = g->ops.gr.load_ctxsw_ucode(g); if (err != 0U) { goto out; } err = gr_gk20a_wait_ctxsw_ready(g); if (err != 0U) { goto out; } out: if (err != 0U) { nvgpu_err(g, "fail"); } else { nvgpu_log_fn(g, "done"); } return err; } static int gk20a_init_gr_reset_enable_hw(struct gk20a *g) { struct av_list_gk20a *sw_non_ctx_load = &g->gr.ctx_vars.sw_non_ctx_load; u32 i, err = 0; nvgpu_log_fn(g, " "); /* enable interrupts */ gk20a_writel(g, gr_intr_r(), ~0); gk20a_writel(g, gr_intr_en_r(), ~0); /* load non_ctx init */ for (i = 0; i < sw_non_ctx_load->count; i++) { gk20a_writel(g, sw_non_ctx_load->l[i].addr, sw_non_ctx_load->l[i].value); } err = gr_gk20a_wait_mem_scrubbing(g); if (err != 0U) { goto out; } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto out; } out: if (err != 0U) { nvgpu_err(g, "fail"); } else { nvgpu_log_fn(g, "done"); } return 0; } static int gr_gk20a_init_access_map(struct gk20a *g) { struct gr_gk20a *gr = &g->gr; struct nvgpu_mem *mem = &gr->global_ctx_buffer[PRIV_ACCESS_MAP].mem; u32 nr_pages = DIV_ROUND_UP(gr->ctx_vars.priv_access_map_size, PAGE_SIZE); u32 *whitelist = NULL; int w, num_entries = 0; nvgpu_memset(g, mem, 0, 0, PAGE_SIZE * nr_pages); g->ops.gr.get_access_map(g, &whitelist, &num_entries); for (w = 0; w < num_entries; w++) { u32 map_bit, map_byte, map_shift, x; map_bit = whitelist[w] >> 2; map_byte = map_bit >> 3; map_shift = map_bit & 0x7; /* i.e. 0-7 */ nvgpu_log_info(g, "access map addr:0x%x byte:0x%x bit:%d", whitelist[w], map_byte, map_shift); x = nvgpu_mem_rd32(g, mem, map_byte / sizeof(u32)); x |= 1 << ( (map_byte % sizeof(u32) * BITS_PER_BYTE) + map_shift); nvgpu_mem_wr32(g, mem, map_byte / sizeof(u32), x); } return 0; } static int gk20a_init_gr_setup_sw(struct gk20a *g) { struct gr_gk20a *gr = &g->gr; int err = 0; nvgpu_log_fn(g, " "); if (gr->sw_ready) { nvgpu_log_fn(g, "skip init"); return 0; } gr->g = g; #if defined(CONFIG_GK20A_CYCLE_STATS) err = nvgpu_mutex_init(&g->gr.cs_lock); if (err != 0) { nvgpu_err(g, "Error in gr.cs_lock mutex initialization"); return err; } #endif err = gr_gk20a_init_gr_config(g, gr); if (err != 0) { goto clean_up; } err = gr_gk20a_init_map_tiles(g, gr); if (err != 0) { goto clean_up; } if (g->ops.ltc.init_comptags) { err = g->ops.ltc.init_comptags(g, gr); if (err != 0) { goto clean_up; } } err = gr_gk20a_init_zcull(g, gr); if (err != 0) { goto clean_up; } err = g->ops.gr.alloc_global_ctx_buffers(g); if (err != 0) { goto clean_up; } err = gr_gk20a_init_access_map(g); if (err != 0) { goto clean_up; } gr_gk20a_load_zbc_default_table(g, gr); if (g->ops.gr.init_czf_bypass) { g->ops.gr.init_czf_bypass(g); } if (g->ops.gr.init_gfxp_wfi_timeout_count) { g->ops.gr.init_gfxp_wfi_timeout_count(g); } err = nvgpu_mutex_init(&gr->ctx_mutex); if (err != 0) { nvgpu_err(g, "Error in gr.ctx_mutex initialization"); goto clean_up; } nvgpu_spinlock_init(&gr->ch_tlb_lock); gr->remove_support = gk20a_remove_gr_support; gr->sw_ready = true; err = nvgpu_ecc_init_support(g); if (err != 0) { goto clean_up; } nvgpu_log_fn(g, "done"); return 0; clean_up: nvgpu_err(g, "fail"); gk20a_remove_gr_support(gr); return err; } static int gk20a_init_gr_bind_fecs_elpg(struct gk20a *g) { struct nvgpu_pmu *pmu = &g->pmu; struct mm_gk20a *mm = &g->mm; struct vm_gk20a *vm = mm->pmu.vm; int err = 0; u32 size; nvgpu_log_fn(g, " "); size = 0; err = gr_gk20a_fecs_get_reglist_img_size(g, &size); if (err != 0) { nvgpu_err(g, "fail to query fecs pg buffer size"); return err; } if (pmu->pg_buf.cpu_va == NULL) { err = nvgpu_dma_alloc_map_sys(vm, size, &pmu->pg_buf); if (err != 0) { nvgpu_err(g, "failed to allocate memory"); return -ENOMEM; } } err = gr_gk20a_fecs_set_reglist_bind_inst(g, &mm->pmu.inst_block); if (err != 0) { nvgpu_err(g, "fail to bind pmu inst to gr"); return err; } err = gr_gk20a_fecs_set_reglist_virtual_addr(g, pmu->pg_buf.gpu_va); if (err != 0) { nvgpu_err(g, "fail to set pg buffer pmu va"); return err; } return err; } int gk20a_init_gr_support(struct gk20a *g) { int err = 0; nvgpu_log_fn(g, " "); g->gr.initialized = false; /* this is required before gr_gk20a_init_ctx_state */ err = nvgpu_mutex_init(&g->gr.fecs_mutex); if (err != 0) { nvgpu_err(g, "Error in gr.fecs_mutex initialization"); return err; } err = gr_gk20a_init_ctxsw(g); if (err != 0) { return err; } /* this appears query for sw states but fecs actually init ramchain, etc so this is hw init */ err = g->ops.gr.init_ctx_state(g); if (err != 0) { return err; } err = gk20a_init_gr_setup_sw(g); if (err != 0) { return err; } err = gk20a_init_gr_setup_hw(g); if (err != 0) { return err; } if (g->can_elpg) { err = gk20a_init_gr_bind_fecs_elpg(g); if (err != 0) { return err; } } /* GR is inialized, signal possible waiters */ g->gr.initialized = true; nvgpu_cond_signal(&g->gr.init_wq); return 0; } /* Wait until GR is initialized */ void gk20a_gr_wait_initialized(struct gk20a *g) { NVGPU_COND_WAIT(&g->gr.init_wq, g->gr.initialized, 0); } #define NVA297_SET_ALPHA_CIRCULAR_BUFFER_SIZE 0x02dc #define NVA297_SET_CIRCULAR_BUFFER_SIZE 0x1280 #define NVA297_SET_SHADER_EXCEPTIONS 0x1528 #define NVA0C0_SET_SHADER_EXCEPTIONS 0x1528 #define NVA297_SET_SHADER_EXCEPTIONS_ENABLE_FALSE 0 void gk20a_gr_set_shader_exceptions(struct gk20a *g, u32 data) { nvgpu_log_fn(g, " "); if (data == NVA297_SET_SHADER_EXCEPTIONS_ENABLE_FALSE) { gk20a_writel(g, gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_r(), 0); gk20a_writel(g, gr_gpcs_tpcs_sm_hww_global_esr_report_mask_r(), 0); } else { /* setup sm warp esr report masks */ gk20a_writel(g, gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_r(), gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_stack_error_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_api_stack_error_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_ret_empty_stack_error_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_pc_wrap_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_misaligned_pc_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_pc_overflow_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_misaligned_immc_addr_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_misaligned_reg_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_illegal_instr_encoding_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_illegal_sph_instr_combo_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_illegal_instr_param_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_invalid_const_addr_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_oor_reg_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_oor_addr_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_misaligned_addr_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_invalid_addr_space_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_illegal_instr_param2_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_invalid_const_addr_ldc_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_geometry_sm_error_report_f() | gr_gpcs_tpcs_sm_hww_warp_esr_report_mask_divergent_report_f()); /* setup sm global esr report mask */ gk20a_writel(g, gr_gpcs_tpcs_sm_hww_global_esr_report_mask_r(), gr_gpcs_tpcs_sm_hww_global_esr_report_mask_sm_to_sm_fault_report_f() | gr_gpcs_tpcs_sm_hww_global_esr_report_mask_l1_error_report_f() | gr_gpcs_tpcs_sm_hww_global_esr_report_mask_multiple_warp_errors_report_f() | gr_gpcs_tpcs_sm_hww_global_esr_report_mask_physical_stack_overflow_error_report_f() | gr_gpcs_tpcs_sm_hww_global_esr_report_mask_bpt_int_report_f() | gr_gpcs_tpcs_sm_hww_global_esr_report_mask_bpt_pause_report_f() | gr_gpcs_tpcs_sm_hww_global_esr_report_mask_single_step_complete_report_f()); } } int gk20a_enable_gr_hw(struct gk20a *g) { int err; nvgpu_log_fn(g, " "); err = gk20a_init_gr_prepare(g); if (err != 0) { return err; } err = gk20a_init_gr_reset_enable_hw(g); if (err != 0) { return err; } nvgpu_log_fn(g, "done"); return 0; } int gk20a_gr_reset(struct gk20a *g) { int err; u32 size; g->gr.initialized = false; nvgpu_mutex_acquire(&g->gr.fecs_mutex); err = gk20a_enable_gr_hw(g); if (err != 0) { nvgpu_mutex_release(&g->gr.fecs_mutex); return err; } err = gk20a_init_gr_setup_hw(g); if (err != 0) { nvgpu_mutex_release(&g->gr.fecs_mutex); return err; } err = gr_gk20a_init_ctxsw(g); if (err != 0) { nvgpu_mutex_release(&g->gr.fecs_mutex); return err; } nvgpu_mutex_release(&g->gr.fecs_mutex); /* this appears query for sw states but fecs actually init ramchain, etc so this is hw init */ err = g->ops.gr.init_ctx_state(g); if (err != 0) { return err; } size = 0; err = gr_gk20a_fecs_get_reglist_img_size(g, &size); if (err != 0) { nvgpu_err(g, "fail to query fecs pg buffer size"); return err; } err = gr_gk20a_fecs_set_reglist_bind_inst(g, &g->mm.pmu.inst_block); if (err != 0) { nvgpu_err(g, "fail to bind pmu inst to gr"); return err; } err = gr_gk20a_fecs_set_reglist_virtual_addr(g, g->pmu.pg_buf.gpu_va); if (err != 0) { nvgpu_err(g, "fail to set pg buffer pmu va"); return err; } nvgpu_cg_init_gr_load_gating_prod(g); nvgpu_cg_elcg_enable_no_wait(g); /* GR is inialized, signal possible waiters */ g->gr.initialized = true; nvgpu_cond_signal(&g->gr.init_wq); return err; } static void gk20a_gr_set_error_notifier(struct gk20a *g, struct gr_gk20a_isr_data *isr_data, u32 error_notifier) { struct channel_gk20a *ch; struct tsg_gk20a *tsg; struct channel_gk20a *ch_tsg; ch = isr_data->ch; if (ch == NULL) { return; } tsg = tsg_gk20a_from_ch(ch); if (tsg != NULL) { nvgpu_rwsem_down_read(&tsg->ch_list_lock); nvgpu_list_for_each_entry(ch_tsg, &tsg->ch_list, channel_gk20a, ch_entry) { if (gk20a_channel_get(ch_tsg)) { g->ops.fifo.set_error_notifier(ch_tsg, error_notifier); gk20a_channel_put(ch_tsg); } } nvgpu_rwsem_up_read(&tsg->ch_list_lock); } else { nvgpu_err(g, "chid: %d is not bound to tsg", ch->chid); } } static int gk20a_gr_handle_semaphore_timeout_pending(struct gk20a *g, struct gr_gk20a_isr_data *isr_data) { nvgpu_log_fn(g, " "); gk20a_gr_set_error_notifier(g, isr_data, NVGPU_ERR_NOTIFIER_GR_SEMAPHORE_TIMEOUT); nvgpu_err(g, "gr semaphore timeout"); return -EINVAL; } static int gk20a_gr_intr_illegal_notify_pending(struct gk20a *g, struct gr_gk20a_isr_data *isr_data) { nvgpu_log_fn(g, " "); gk20a_gr_set_error_notifier(g, isr_data, NVGPU_ERR_NOTIFIER_GR_ILLEGAL_NOTIFY); /* This is an unrecoverable error, reset is needed */ nvgpu_err(g, "gr semaphore timeout"); return -EINVAL; } static int gk20a_gr_handle_illegal_method(struct gk20a *g, struct gr_gk20a_isr_data *isr_data) { int ret = g->ops.gr.handle_sw_method(g, isr_data->addr, isr_data->class_num, isr_data->offset, isr_data->data_lo); if (ret) { gk20a_gr_set_error_notifier(g, isr_data, NVGPU_ERR_NOTIFIER_GR_ILLEGAL_NOTIFY); nvgpu_err(g, "invalid method class 0x%08x" ", offset 0x%08x address 0x%08x", isr_data->class_num, isr_data->offset, isr_data->addr); } return ret; } static int gk20a_gr_handle_illegal_class(struct gk20a *g, struct gr_gk20a_isr_data *isr_data) { nvgpu_log_fn(g, " "); gk20a_gr_set_error_notifier(g, isr_data, NVGPU_ERR_NOTIFIER_GR_ERROR_SW_NOTIFY); nvgpu_err(g, "invalid class 0x%08x, offset 0x%08x", isr_data->class_num, isr_data->offset); return -EINVAL; } int gk20a_gr_handle_fecs_error(struct gk20a *g, struct channel_gk20a *ch, struct gr_gk20a_isr_data *isr_data) { u32 gr_fecs_intr = gk20a_readl(g, gr_fecs_host_int_status_r()); int ret = 0; u32 chid = isr_data->ch != NULL ? isr_data->ch->chid : FIFO_INVAL_CHANNEL_ID; if (gr_fecs_intr == 0U) { return 0; } if (gr_fecs_intr & gr_fecs_host_int_status_umimp_firmware_method_f(1)) { gk20a_gr_set_error_notifier(g, isr_data, NVGPU_ERR_NOTIFIER_FECS_ERR_UNIMP_FIRMWARE_METHOD); nvgpu_err(g, "firmware method error 0x%08x for offset 0x%04x", gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(6)), isr_data->data_lo); ret = -1; } else if ((gr_fecs_intr & gr_fecs_host_int_status_watchdog_active_f()) != 0U) { /* currently, recovery is not initiated */ nvgpu_err(g, "fecs watchdog triggered for channel %u, " "cannot ctxsw anymore !!", chid); gk20a_fecs_dump_falcon_stats(g); } else if ((gr_fecs_intr & gr_fecs_host_int_status_ctxsw_intr_f(CTXSW_INTR0)) != 0U) { u32 mailbox_value = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(6)); if (mailbox_value == MAILBOX_VALUE_TIMESTAMP_BUFFER_FULL) { nvgpu_info(g, "ctxsw intr0 set by ucode, " "timestamp buffer full"); #ifdef CONFIG_GK20A_CTXSW_TRACE gk20a_fecs_trace_reset_buffer(g); #else ret = -1; #endif } else { nvgpu_err(g, "ctxsw intr0 set by ucode, error_code: 0x%08x", mailbox_value); ret = -1; } } else { nvgpu_err(g, "unhandled fecs error interrupt 0x%08x for channel %u", gr_fecs_intr, ch->chid); gk20a_fecs_dump_falcon_stats(g); } gk20a_writel(g, gr_fecs_host_int_clear_r(), gr_fecs_intr); return ret; } static int gk20a_gr_handle_class_error(struct gk20a *g, struct gr_gk20a_isr_data *isr_data) { u32 gr_class_error; u32 chid = isr_data->ch != NULL ? isr_data->ch->chid : FIFO_INVAL_CHANNEL_ID; nvgpu_log_fn(g, " "); gr_class_error = gr_class_error_code_v(gk20a_readl(g, gr_class_error_r())); gk20a_gr_set_error_notifier(g, isr_data, NVGPU_ERR_NOTIFIER_GR_ERROR_SW_NOTIFY); nvgpu_err(g, "class error 0x%08x, offset 0x%08x," "sub channel 0x%08x mme generated %d," " mme pc 0x%08xdata high %d priv status %d" " unhandled intr 0x%08x for channel %u", isr_data->class_num, (isr_data->offset << 2), gr_trapped_addr_subch_v(isr_data->addr), gr_trapped_addr_mme_generated_v(isr_data->addr), gr_trapped_data_mme_pc_v( gk20a_readl(g, gr_trapped_data_mme_r())), gr_trapped_addr_datahigh_v(isr_data->addr), gr_trapped_addr_priv_v(isr_data->addr), gr_class_error, chid); nvgpu_err(g, "trapped data low 0x%08x", gk20a_readl(g, gr_trapped_data_lo_r())); if (gr_trapped_addr_datahigh_v(isr_data->addr)) { nvgpu_err(g, "trapped data high 0x%08x", gk20a_readl(g, gr_trapped_data_hi_r())); } return -EINVAL; } static int gk20a_gr_handle_firmware_method(struct gk20a *g, struct gr_gk20a_isr_data *isr_data) { u32 chid = isr_data->ch != NULL ? isr_data->ch->chid : FIFO_INVAL_CHANNEL_ID; nvgpu_log_fn(g, " "); gk20a_gr_set_error_notifier(g, isr_data, NVGPU_ERR_NOTIFIER_GR_ERROR_SW_NOTIFY); nvgpu_err(g, "firmware method 0x%08x, offset 0x%08x for channel %u", isr_data->class_num, isr_data->offset, chid); return -EINVAL; } int gk20a_gr_handle_semaphore_pending(struct gk20a *g, struct gr_gk20a_isr_data *isr_data) { struct channel_gk20a *ch = isr_data->ch; struct tsg_gk20a *tsg; if (ch == NULL) { return 0; } tsg = tsg_gk20a_from_ch(ch); if (tsg != NULL) { g->ops.fifo.post_event_id(tsg, NVGPU_EVENT_ID_GR_SEMAPHORE_WRITE_AWAKEN); nvgpu_cond_broadcast(&ch->semaphore_wq); } else { nvgpu_err(g, "chid: %d is not bound to tsg", ch->chid); } return 0; } #if defined(CONFIG_GK20A_CYCLE_STATS) static inline bool is_valid_cyclestats_bar0_offset_gk20a(struct gk20a *g, u32 offset) { /* support only 24-bit 4-byte aligned offsets */ bool valid = !(offset & 0xFF000003); if (g->allow_all) return true; /* whitelist check */ valid = valid && is_bar0_global_offset_whitelisted_gk20a(g, offset); /* resource size check in case there was a problem * with allocating the assumed size of bar0 */ valid = valid && gk20a_io_valid_reg(g, offset); return valid; } #endif int gk20a_gr_handle_notify_pending(struct gk20a *g, struct gr_gk20a_isr_data *isr_data) { struct channel_gk20a *ch = isr_data->ch; #if defined(CONFIG_GK20A_CYCLE_STATS) void *virtual_address; u32 buffer_size; u32 offset; bool exit; #endif if (ch == NULL || tsg_gk20a_from_ch(ch) == NULL) { return 0; } #if defined(CONFIG_GK20A_CYCLE_STATS) /* GL will never use payload 0 for cycle state */ if ((ch->cyclestate.cyclestate_buffer == NULL) || (isr_data->data_lo == 0)) return 0; nvgpu_mutex_acquire(&ch->cyclestate.cyclestate_buffer_mutex); virtual_address = ch->cyclestate.cyclestate_buffer; buffer_size = ch->cyclestate.cyclestate_buffer_size; offset = isr_data->data_lo; exit = false; while (!exit) { struct share_buffer_head *sh_hdr; u32 min_element_size; /* validate offset */ if (offset + sizeof(struct share_buffer_head) > buffer_size || offset + sizeof(struct share_buffer_head) < offset) { nvgpu_err(g, "cyclestats buffer overrun at offset 0x%x", offset); break; } sh_hdr = (struct share_buffer_head *) ((char *)virtual_address + offset); min_element_size = (sh_hdr->operation == OP_END ? sizeof(struct share_buffer_head) : sizeof(struct gk20a_cyclestate_buffer_elem)); /* validate sh_hdr->size */ if (sh_hdr->size < min_element_size || offset + sh_hdr->size > buffer_size || offset + sh_hdr->size < offset) { nvgpu_err(g, "bad cyclestate buffer header size at offset 0x%x", offset); sh_hdr->failed = true; break; } switch (sh_hdr->operation) { case OP_END: exit = true; break; case BAR0_READ32: case BAR0_WRITE32: { struct gk20a_cyclestate_buffer_elem *op_elem = (struct gk20a_cyclestate_buffer_elem *)sh_hdr; bool valid = is_valid_cyclestats_bar0_offset_gk20a( g, op_elem->offset_bar0); u32 raw_reg; u64 mask_orig; u64 v; if (!valid) { nvgpu_err(g, "invalid cycletstats op offset: 0x%x", op_elem->offset_bar0); sh_hdr->failed = exit = true; break; } mask_orig = ((1ULL << (op_elem->last_bit + 1)) -1)&~((1ULL << op_elem->first_bit)-1); raw_reg = gk20a_readl(g, op_elem->offset_bar0); switch (sh_hdr->operation) { case BAR0_READ32: op_elem->data = (raw_reg & mask_orig) >> op_elem->first_bit; break; case BAR0_WRITE32: v = 0; if ((unsigned int)mask_orig != (unsigned int)~0) { v = (unsigned int) (raw_reg & ~mask_orig); } v |= ((op_elem->data << op_elem->first_bit) & mask_orig); gk20a_writel(g, op_elem->offset_bar0, (unsigned int)v); break; default: /* nop ok?*/ break; } } break; default: /* no operation content case */ exit = true; break; } sh_hdr->completed = true; offset += sh_hdr->size; } nvgpu_mutex_release(&ch->cyclestate.cyclestate_buffer_mutex); #endif nvgpu_log_fn(g, " "); nvgpu_cond_broadcast_interruptible(&ch->notifier_wq); return 0; } /* Used by sw interrupt thread to translate current ctx to chid. * Also used by regops to translate current ctx to chid and tsgid. * For performance, we don't want to go through 128 channels every time. * curr_ctx should be the value read from gr_fecs_current_ctx_r(). * A small tlb is used here to cache translation. * * Returned channel must be freed with gk20a_channel_put() */ static struct channel_gk20a *gk20a_gr_get_channel_from_ctx( struct gk20a *g, u32 curr_ctx, u32 *curr_tsgid) { struct fifo_gk20a *f = &g->fifo; struct gr_gk20a *gr = &g->gr; u32 chid = -1; u32 tsgid = NVGPU_INVALID_TSG_ID; u32 i; struct channel_gk20a *ret = NULL; /* when contexts are unloaded from GR, the valid bit is reset * but the instance pointer information remains intact. * This might be called from gr_isr where contexts might be * unloaded. No need to check ctx_valid bit */ nvgpu_spinlock_acquire(&gr->ch_tlb_lock); /* check cache first */ for (i = 0; i < GR_CHANNEL_MAP_TLB_SIZE; i++) { if (gr->chid_tlb[i].curr_ctx == curr_ctx) { chid = gr->chid_tlb[i].chid; tsgid = gr->chid_tlb[i].tsgid; ret = gk20a_channel_from_id(g, chid); goto unlock; } } /* slow path */ for (chid = 0; chid < f->num_channels; chid++) { struct channel_gk20a *ch = gk20a_channel_from_id(g, chid); if (ch == NULL) { continue; } if ((u32)(nvgpu_inst_block_addr(g, &ch->inst_block) >> ram_in_base_shift_v()) == gr_fecs_current_ctx_ptr_v(curr_ctx)) { tsgid = ch->tsgid; /* found it */ ret = ch; break; } gk20a_channel_put(ch); } if (ret == NULL) { goto unlock; } /* add to free tlb entry */ for (i = 0; i < GR_CHANNEL_MAP_TLB_SIZE; i++) { if (gr->chid_tlb[i].curr_ctx == 0) { gr->chid_tlb[i].curr_ctx = curr_ctx; gr->chid_tlb[i].chid = chid; gr->chid_tlb[i].tsgid = tsgid; goto unlock; } } /* no free entry, flush one */ gr->chid_tlb[gr->channel_tlb_flush_index].curr_ctx = curr_ctx; gr->chid_tlb[gr->channel_tlb_flush_index].chid = chid; gr->chid_tlb[gr->channel_tlb_flush_index].tsgid = tsgid; gr->channel_tlb_flush_index = (gr->channel_tlb_flush_index + 1) & (GR_CHANNEL_MAP_TLB_SIZE - 1); unlock: nvgpu_spinlock_release(&gr->ch_tlb_lock); if (curr_tsgid) { *curr_tsgid = tsgid; } return ret; } int gk20a_gr_lock_down_sm(struct gk20a *g, u32 gpc, u32 tpc, u32 sm, u32 global_esr_mask, bool check_errors) { u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc); u32 dbgr_control0; nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "GPC%d TPC%d SM%d: assert stop trigger", gpc, tpc, sm); /* assert stop trigger */ dbgr_control0 = gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_control0_r() + offset); dbgr_control0 |= gr_gpc0_tpc0_sm_dbgr_control0_stop_trigger_enable_f(); gk20a_writel(g, gr_gpc0_tpc0_sm_dbgr_control0_r() + offset, dbgr_control0); return g->ops.gr.wait_for_sm_lock_down(g, gpc, tpc, sm, global_esr_mask, check_errors); } bool gk20a_gr_sm_debugger_attached(struct gk20a *g) { u32 dbgr_control0 = gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_control0_r()); /* check if an sm debugger is attached. * assumption: all SMs will have debug mode enabled/disabled * uniformly. */ if (gr_gpc0_tpc0_sm_dbgr_control0_debugger_mode_v(dbgr_control0) == gr_gpc0_tpc0_sm_dbgr_control0_debugger_mode_on_v()) { return true; } return false; } int gr_gk20a_handle_sm_exception(struct gk20a *g, u32 gpc, u32 tpc, u32 sm, bool *post_event, struct channel_gk20a *fault_ch, u32 *hww_global_esr) { int ret = 0; bool do_warp_sync = false, early_exit = false, ignore_debugger = false; bool disable_sm_exceptions = true; u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc); bool sm_debugger_attached; u32 global_esr, warp_esr, global_mask; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " "); sm_debugger_attached = g->ops.gr.sm_debugger_attached(g); global_esr = g->ops.gr.get_sm_hww_global_esr(g, gpc, tpc, sm); *hww_global_esr = global_esr; warp_esr = g->ops.gr.get_sm_hww_warp_esr(g, gpc, tpc, sm); global_mask = g->ops.gr.get_sm_no_lock_down_hww_global_esr_mask(g); if (!sm_debugger_attached) { nvgpu_err(g, "sm hww global 0x%08x warp 0x%08x", global_esr, warp_esr); return -EFAULT; } nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "sm hww global 0x%08x warp 0x%08x", global_esr, warp_esr); gr_gk20a_elpg_protected_call(g, g->ops.gr.record_sm_error_state(g, gpc, tpc, sm, fault_ch)); if (g->ops.gr.pre_process_sm_exception) { ret = g->ops.gr.pre_process_sm_exception(g, gpc, tpc, sm, global_esr, warp_esr, sm_debugger_attached, fault_ch, &early_exit, &ignore_debugger); if (ret) { nvgpu_err(g, "could not pre-process sm error!"); return ret; } } if (early_exit) { nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "returning early"); return ret; } /* * Disable forwarding of tpc exceptions, * the debugger will reenable exceptions after servicing them. * * Do not disable exceptions if the only SM exception is BPT_INT */ if ((global_esr == gr_gpc0_tpc0_sm_hww_global_esr_bpt_int_pending_f()) && (warp_esr == 0)) { disable_sm_exceptions = false; } if (!ignore_debugger && disable_sm_exceptions) { u32 tpc_exception_en = gk20a_readl(g, gr_gpc0_tpc0_tpccs_tpc_exception_en_r() + offset); tpc_exception_en &= ~gr_gpc0_tpc0_tpccs_tpc_exception_en_sm_enabled_f(); gk20a_writel(g, gr_gpc0_tpc0_tpccs_tpc_exception_en_r() + offset, tpc_exception_en); nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "SM Exceptions disabled"); } /* if a debugger is present and an error has occurred, do a warp sync */ if (!ignore_debugger && ((warp_esr != 0) || ((global_esr & ~global_mask) != 0))) { nvgpu_log(g, gpu_dbg_intr, "warp sync needed"); do_warp_sync = true; } if (do_warp_sync) { ret = g->ops.gr.lock_down_sm(g, gpc, tpc, sm, global_mask, true); if (ret) { nvgpu_err(g, "sm did not lock down!"); return ret; } } if (ignore_debugger) { nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "ignore_debugger set, skipping event posting"); } else { *post_event = true; } return ret; } int gr_gk20a_handle_tex_exception(struct gk20a *g, u32 gpc, u32 tpc, bool *post_event) { int ret = 0; u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); u32 offset = gpc_stride * gpc + tpc_in_gpc_stride * tpc; u32 esr; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " "); esr = gk20a_readl(g, gr_gpc0_tpc0_tex_m_hww_esr_r() + offset); nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "0x%08x", esr); gk20a_writel(g, gr_gpc0_tpc0_tex_m_hww_esr_r() + offset, esr); return ret; } void gk20a_gr_get_esr_sm_sel(struct gk20a *g, u32 gpc, u32 tpc, u32 *esr_sm_sel) { *esr_sm_sel = 1; } static int gk20a_gr_handle_tpc_exception(struct gk20a *g, u32 gpc, u32 tpc, bool *post_event, struct channel_gk20a *fault_ch, u32 *hww_global_esr) { int ret = 0; u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc); u32 tpc_exception = gk20a_readl(g, gr_gpc0_tpc0_tpccs_tpc_exception_r() + offset); u32 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "GPC%d TPC%d: pending exception 0x%x", gpc, tpc, tpc_exception); /* check if an sm exeption is pending */ if (gr_gpc0_tpc0_tpccs_tpc_exception_sm_v(tpc_exception) == gr_gpc0_tpc0_tpccs_tpc_exception_sm_pending_v()) { u32 esr_sm_sel, sm; nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "GPC%d TPC%d: SM exception pending", gpc, tpc); if (g->ops.gr.handle_tpc_sm_ecc_exception) { g->ops.gr.handle_tpc_sm_ecc_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr); } g->ops.gr.get_esr_sm_sel(g, gpc, tpc, &esr_sm_sel); for (sm = 0; sm < sm_per_tpc; sm++) { if ((esr_sm_sel & BIT32(sm)) == 0U) { continue; } nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "GPC%d TPC%d: SM%d exception pending", gpc, tpc, sm); ret |= g->ops.gr.handle_sm_exception(g, gpc, tpc, sm, post_event, fault_ch, hww_global_esr); /* clear the hwws, also causes tpc and gpc * exceptions to be cleared. Should be cleared * only if SM is locked down or empty. */ g->ops.gr.clear_sm_hww(g, gpc, tpc, sm, *hww_global_esr); } } /* check if a tex exeption is pending */ if (gr_gpc0_tpc0_tpccs_tpc_exception_tex_v(tpc_exception) == gr_gpc0_tpc0_tpccs_tpc_exception_tex_pending_v()) { nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "GPC%d TPC%d: TEX exception pending", gpc, tpc); ret |= g->ops.gr.handle_tex_exception(g, gpc, tpc, post_event); } if (g->ops.gr.handle_tpc_mpc_exception) { ret |= g->ops.gr.handle_tpc_mpc_exception(g, gpc, tpc, post_event); } return ret; } static int gk20a_gr_handle_gpc_exception(struct gk20a *g, bool *post_event, struct channel_gk20a *fault_ch, u32 *hww_global_esr) { int ret = 0; u32 gpc_offset, gpc, tpc; struct gr_gk20a *gr = &g->gr; u32 exception1 = gk20a_readl(g, gr_exception1_r()); u32 gpc_exception; nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, " "); for (gpc = 0; gpc < gr->gpc_count; gpc++) { if ((exception1 & (1 << gpc)) == 0) { continue; } nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "GPC%d exception pending", gpc); gpc_offset = gk20a_gr_gpc_offset(g, gpc); gpc_exception = gk20a_readl(g, gr_gpc0_gpccs_gpc_exception_r() + gpc_offset); /* check if any tpc has an exception */ for (tpc = 0; tpc < gr->gpc_tpc_count[gpc]; tpc++) { if ((gr_gpc0_gpccs_gpc_exception_tpc_v(gpc_exception) & (1 << tpc)) == 0) { continue; } nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "GPC%d: TPC%d exception pending", gpc, tpc); ret |= gk20a_gr_handle_tpc_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr); } /* Handle GCC exception */ if ((gr_gpc0_gpccs_gpc_exception_gcc_v(gpc_exception) != 0U) && (g->ops.gr.handle_gcc_exception != NULL)) { int gcc_ret = 0; gcc_ret = g->ops.gr.handle_gcc_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr); ret |= (ret != 0) ? ret : gcc_ret; } /* Handle GPCCS exceptions */ if (g->ops.gr.handle_gpc_gpccs_exception) { int ret_ecc = 0; ret_ecc = g->ops.gr.handle_gpc_gpccs_exception(g, gpc, gpc_exception); ret |= (ret != 0) ? ret : ret_ecc; } /* Handle GPCMMU exceptions */ if (g->ops.gr.handle_gpc_gpcmmu_exception) { int ret_mmu = 0; ret_mmu = g->ops.gr.handle_gpc_gpcmmu_exception(g, gpc, gpc_exception); ret |= (ret != 0) ? ret : ret_mmu; } } return ret; } static int gk20a_gr_post_bpt_events(struct gk20a *g, struct tsg_gk20a *tsg, u32 global_esr) { if (global_esr & gr_gpc0_tpc0_sm_hww_global_esr_bpt_int_pending_f()) { g->ops.fifo.post_event_id(tsg, NVGPU_EVENT_ID_BPT_INT); } if (global_esr & gr_gpc0_tpc0_sm_hww_global_esr_bpt_pause_pending_f()) { g->ops.fifo.post_event_id(tsg, NVGPU_EVENT_ID_BPT_PAUSE); } return 0; } int gk20a_gr_isr(struct gk20a *g) { struct gr_gk20a_isr_data isr_data; u32 grfifo_ctl; u32 obj_table; bool need_reset = false; u32 gr_intr = gk20a_readl(g, gr_intr_r()); struct channel_gk20a *ch = NULL; struct channel_gk20a *fault_ch = NULL; u32 tsgid = NVGPU_INVALID_TSG_ID; struct tsg_gk20a *tsg = NULL; u32 gr_engine_id; u32 global_esr = 0; u32 chid; nvgpu_log_fn(g, " "); nvgpu_log(g, gpu_dbg_intr, "pgraph intr 0x%08x", gr_intr); if (gr_intr == 0U) { return 0; } gr_engine_id = gk20a_fifo_get_gr_engine_id(g); if (gr_engine_id != FIFO_INVAL_ENGINE_ID) { gr_engine_id = BIT(gr_engine_id); } grfifo_ctl = gk20a_readl(g, gr_gpfifo_ctl_r()); grfifo_ctl &= ~gr_gpfifo_ctl_semaphore_access_f(1); grfifo_ctl &= ~gr_gpfifo_ctl_access_f(1); gk20a_writel(g, gr_gpfifo_ctl_r(), grfifo_ctl | gr_gpfifo_ctl_access_f(0) | gr_gpfifo_ctl_semaphore_access_f(0)); isr_data.addr = gk20a_readl(g, gr_trapped_addr_r()); isr_data.data_lo = gk20a_readl(g, gr_trapped_data_lo_r()); isr_data.data_hi = gk20a_readl(g, gr_trapped_data_hi_r()); isr_data.curr_ctx = gk20a_readl(g, gr_fecs_current_ctx_r()); isr_data.offset = gr_trapped_addr_mthd_v(isr_data.addr); isr_data.sub_chan = gr_trapped_addr_subch_v(isr_data.addr); obj_table = (isr_data.sub_chan < 4) ? gk20a_readl(g, gr_fe_object_table_r(isr_data.sub_chan)) : 0; isr_data.class_num = gr_fe_object_table_nvclass_v(obj_table); ch = gk20a_gr_get_channel_from_ctx(g, isr_data.curr_ctx, &tsgid); isr_data.ch = ch; chid = ch != NULL ? ch->chid : FIFO_INVAL_CHANNEL_ID; if (ch == NULL) { nvgpu_err(g, "pgraph intr: 0x%08x, chid: INVALID", gr_intr); } else { tsg = tsg_gk20a_from_ch(ch); if (tsg == NULL) { nvgpu_err(g, "pgraph intr: 0x%08x, chid: %d " "not bound to tsg", gr_intr, chid); } } nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "channel %d: addr 0x%08x, " "data 0x%08x 0x%08x," "ctx 0x%08x, offset 0x%08x, " "subchannel 0x%08x, class 0x%08x", chid, isr_data.addr, isr_data.data_hi, isr_data.data_lo, isr_data.curr_ctx, isr_data.offset, isr_data.sub_chan, isr_data.class_num); if (gr_intr & gr_intr_notify_pending_f()) { g->ops.gr.handle_notify_pending(g, &isr_data); gk20a_writel(g, gr_intr_r(), gr_intr_notify_reset_f()); gr_intr &= ~gr_intr_notify_pending_f(); } if (gr_intr & gr_intr_semaphore_pending_f()) { g->ops.gr.handle_semaphore_pending(g, &isr_data); gk20a_writel(g, gr_intr_r(), gr_intr_semaphore_reset_f()); gr_intr &= ~gr_intr_semaphore_pending_f(); } if (gr_intr & gr_intr_semaphore_timeout_pending_f()) { if (gk20a_gr_handle_semaphore_timeout_pending(g, &isr_data) != 0) { need_reset = true; } gk20a_writel(g, gr_intr_r(), gr_intr_semaphore_reset_f()); gr_intr &= ~gr_intr_semaphore_pending_f(); } if (gr_intr & gr_intr_illegal_notify_pending_f()) { if (gk20a_gr_intr_illegal_notify_pending(g, &isr_data) != 0) { need_reset = true; } gk20a_writel(g, gr_intr_r(), gr_intr_illegal_notify_reset_f()); gr_intr &= ~gr_intr_illegal_notify_pending_f(); } if (gr_intr & gr_intr_illegal_method_pending_f()) { if (gk20a_gr_handle_illegal_method(g, &isr_data) != 0) { need_reset = true; } gk20a_writel(g, gr_intr_r(), gr_intr_illegal_method_reset_f()); gr_intr &= ~gr_intr_illegal_method_pending_f(); } if (gr_intr & gr_intr_illegal_class_pending_f()) { if (gk20a_gr_handle_illegal_class(g, &isr_data) != 0) { need_reset = true; } gk20a_writel(g, gr_intr_r(), gr_intr_illegal_class_reset_f()); gr_intr &= ~gr_intr_illegal_class_pending_f(); } if (gr_intr & gr_intr_fecs_error_pending_f()) { if (g->ops.gr.handle_fecs_error(g, ch, &isr_data) != 0) { need_reset = true; } gk20a_writel(g, gr_intr_r(), gr_intr_fecs_error_reset_f()); gr_intr &= ~gr_intr_fecs_error_pending_f(); } if (gr_intr & gr_intr_class_error_pending_f()) { if (gk20a_gr_handle_class_error(g, &isr_data) != 0) { need_reset = true; } gk20a_writel(g, gr_intr_r(), gr_intr_class_error_reset_f()); gr_intr &= ~gr_intr_class_error_pending_f(); } /* this one happens if someone tries to hit a non-whitelisted * register using set_falcon[4] */ if (gr_intr & gr_intr_firmware_method_pending_f()) { if (gk20a_gr_handle_firmware_method(g, &isr_data) != 0) { need_reset = true; } nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "firmware method intr pending\n"); gk20a_writel(g, gr_intr_r(), gr_intr_firmware_method_reset_f()); gr_intr &= ~gr_intr_firmware_method_pending_f(); } if (gr_intr & gr_intr_exception_pending_f()) { u32 exception = gk20a_readl(g, gr_exception_r()); nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "exception %08x\n", exception); if (exception & gr_exception_fe_m()) { u32 fe = gk20a_readl(g, gr_fe_hww_esr_r()); u32 info = gk20a_readl(g, gr_fe_hww_esr_info_r()); nvgpu_err(g, "fe exception: esr 0x%08x, info 0x%08x", fe, info); gk20a_writel(g, gr_fe_hww_esr_r(), gr_fe_hww_esr_reset_active_f()); need_reset = true; } if (exception & gr_exception_memfmt_m()) { u32 memfmt = gk20a_readl(g, gr_memfmt_hww_esr_r()); nvgpu_err(g, "memfmt exception: esr %08x", memfmt); gk20a_writel(g, gr_memfmt_hww_esr_r(), gr_memfmt_hww_esr_reset_active_f()); need_reset = true; } if (exception & gr_exception_pd_m()) { u32 pd = gk20a_readl(g, gr_pd_hww_esr_r()); nvgpu_err(g, "pd exception: esr 0x%08x", pd); gk20a_writel(g, gr_pd_hww_esr_r(), gr_pd_hww_esr_reset_active_f()); need_reset = true; } if (exception & gr_exception_scc_m()) { u32 scc = gk20a_readl(g, gr_scc_hww_esr_r()); nvgpu_err(g, "scc exception: esr 0x%08x", scc); gk20a_writel(g, gr_scc_hww_esr_r(), gr_scc_hww_esr_reset_active_f()); need_reset = true; } if (exception & gr_exception_ds_m()) { u32 ds = gk20a_readl(g, gr_ds_hww_esr_r()); nvgpu_err(g, "ds exception: esr: 0x%08x", ds); gk20a_writel(g, gr_ds_hww_esr_r(), gr_ds_hww_esr_reset_task_f()); need_reset = true; } if (exception & gr_exception_ssync_m()) { if (g->ops.gr.handle_ssync_hww) { if (g->ops.gr.handle_ssync_hww(g) != 0) { need_reset = true; } } else { nvgpu_err(g, "unhandled ssync exception"); } } if (exception & gr_exception_mme_m()) { u32 mme = gk20a_readl(g, gr_mme_hww_esr_r()); u32 info = gk20a_readl(g, gr_mme_hww_esr_info_r()); nvgpu_err(g, "mme exception: esr 0x%08x info:0x%08x", mme, info); gk20a_writel(g, gr_mme_hww_esr_r(), gr_mme_hww_esr_reset_active_f()); need_reset = true; } if (exception & gr_exception_sked_m()) { u32 sked = gk20a_readl(g, gr_sked_hww_esr_r()); nvgpu_err(g, "sked exception: esr 0x%08x", sked); gk20a_writel(g, gr_sked_hww_esr_r(), gr_sked_hww_esr_reset_active_f()); need_reset = true; } /* check if a gpc exception has occurred */ if (((exception & gr_exception_gpc_m()) != 0U) && !need_reset) { bool post_event = false; nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "GPC exception pending"); if (tsg != NULL) { fault_ch = isr_data.ch; } /* fault_ch can be NULL */ /* check if any gpc has an exception */ if (gk20a_gr_handle_gpc_exception(g, &post_event, fault_ch, &global_esr) != 0) { need_reset = true; } /* signal clients waiting on an event */ if (g->ops.gr.sm_debugger_attached(g) && post_event && (fault_ch != NULL)) { g->ops.debugger.post_events(fault_ch); } } gk20a_writel(g, gr_intr_r(), gr_intr_exception_reset_f()); gr_intr &= ~gr_intr_exception_pending_f(); if (need_reset) { nvgpu_err(g, "set gr exception notifier"); gk20a_gr_set_error_notifier(g, &isr_data, NVGPU_ERR_NOTIFIER_GR_EXCEPTION); } } if (need_reset) { if (tsg != NULL) { gk20a_fifo_recover(g, gr_engine_id, tsgid, true, true, true, RC_TYPE_GR_FAULT); } else { if (ch != NULL) { nvgpu_err(g, "chid: %d referenceable but not " "bound to tsg", chid); } gk20a_fifo_recover(g, gr_engine_id, 0, false, false, true, RC_TYPE_GR_FAULT); } } if (gr_intr != 0U) { /* clear unhandled interrupts */ if (ch == NULL) { /* * This is probably an interrupt during * gk20a_free_channel() */ nvgpu_err(g, "unhandled gr intr 0x%08x for " "unreferenceable channel, clearing", gr_intr); } else { nvgpu_err(g, "unhandled gr intr 0x%08x for chid: %d", gr_intr, chid); } gk20a_writel(g, gr_intr_r(), gr_intr); } gk20a_writel(g, gr_gpfifo_ctl_r(), grfifo_ctl | gr_gpfifo_ctl_access_f(1) | gr_gpfifo_ctl_semaphore_access_f(1)); /* Posting of BPT events should be the last thing in this function */ if ((global_esr != 0U) && (tsg != NULL)) { gk20a_gr_post_bpt_events(g, tsg, global_esr); } if (ch) { gk20a_channel_put(ch); } return 0; } u32 gk20a_gr_nonstall_isr(struct gk20a *g) { u32 ops = 0; u32 gr_intr = gk20a_readl(g, gr_intr_nonstall_r()); nvgpu_log(g, gpu_dbg_intr, "pgraph nonstall intr %08x", gr_intr); if ((gr_intr & gr_intr_nonstall_trap_pending_f()) != 0U) { /* Clear the interrupt */ gk20a_writel(g, gr_intr_nonstall_r(), gr_intr_nonstall_trap_pending_f()); ops |= (GK20A_NONSTALL_OPS_WAKEUP_SEMAPHORE | GK20A_NONSTALL_OPS_POST_EVENTS); } return ops; } int gr_gk20a_fecs_get_reglist_img_size(struct gk20a *g, u32 *size) { BUG_ON(size == NULL); return gr_gk20a_submit_fecs_method_op(g, (struct fecs_method_op_gk20a) { .mailbox.id = 0, .mailbox.data = 0, .mailbox.clr = ~0, .method.data = 1, .method.addr = gr_fecs_method_push_adr_discover_reglist_image_size_v(), .mailbox.ret = size, .cond.ok = GR_IS_UCODE_OP_NOT_EQUAL, .mailbox.ok = 0, .cond.fail = GR_IS_UCODE_OP_SKIP, .mailbox.fail = 0}, false); } int gr_gk20a_fecs_set_reglist_bind_inst(struct gk20a *g, struct nvgpu_mem *inst_block) { u32 data = fecs_current_ctx_data(g, inst_block); return gr_gk20a_submit_fecs_method_op(g, (struct fecs_method_op_gk20a){ .mailbox.id = 4, .mailbox.data = data, .mailbox.clr = ~0, .method.data = 1, .method.addr = gr_fecs_method_push_adr_set_reglist_bind_instance_v(), .mailbox.ret = NULL, .cond.ok = GR_IS_UCODE_OP_EQUAL, .mailbox.ok = 1, .cond.fail = GR_IS_UCODE_OP_SKIP, .mailbox.fail = 0}, false); } int gr_gk20a_fecs_set_reglist_virtual_addr(struct gk20a *g, u64 pmu_va) { return gr_gk20a_submit_fecs_method_op(g, (struct fecs_method_op_gk20a) { .mailbox.id = 4, .mailbox.data = u64_lo32(pmu_va >> 8), .mailbox.clr = ~0, .method.data = 1, .method.addr = gr_fecs_method_push_adr_set_reglist_virtual_address_v(), .mailbox.ret = NULL, .cond.ok = GR_IS_UCODE_OP_EQUAL, .mailbox.ok = 1, .cond.fail = GR_IS_UCODE_OP_SKIP, .mailbox.fail = 0}, false); } int gk20a_gr_suspend(struct gk20a *g) { u32 ret = 0; nvgpu_log_fn(g, " "); ret = g->ops.gr.wait_empty(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (ret) { return ret; } gk20a_writel(g, gr_gpfifo_ctl_r(), gr_gpfifo_ctl_access_disabled_f()); /* disable gr intr */ gk20a_writel(g, gr_intr_r(), 0); gk20a_writel(g, gr_intr_en_r(), 0); /* disable all exceptions */ gk20a_writel(g, gr_exception_r(), 0); gk20a_writel(g, gr_exception_en_r(), 0); gk20a_writel(g, gr_exception1_r(), 0); gk20a_writel(g, gr_exception1_en_r(), 0); gk20a_writel(g, gr_exception2_r(), 0); gk20a_writel(g, gr_exception2_en_r(), 0); gk20a_gr_flush_channel_tlb(&g->gr); g->gr.initialized = false; nvgpu_log_fn(g, "done"); return ret; } static int gr_gk20a_find_priv_offset_in_buffer(struct gk20a *g, u32 addr, bool is_quad, u32 quad, u32 *context_buffer, u32 context_buffer_size, u32 *priv_offset); static int gr_gk20a_find_priv_offset_in_pm_buffer(struct gk20a *g, u32 addr, u32 *priv_offset); /* This function will decode a priv address and return the partition type and numbers. */ int gr_gk20a_decode_priv_addr(struct gk20a *g, u32 addr, enum ctxsw_addr_type *addr_type, u32 *gpc_num, u32 *tpc_num, u32 *ppc_num, u32 *be_num, u32 *broadcast_flags) { u32 gpc_addr; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "addr=0x%x", addr); /* setup defaults */ *addr_type = CTXSW_ADDR_TYPE_SYS; *broadcast_flags = PRI_BROADCAST_FLAGS_NONE; *gpc_num = 0; *tpc_num = 0; *ppc_num = 0; *be_num = 0; if (pri_is_gpc_addr(g, addr)) { *addr_type = CTXSW_ADDR_TYPE_GPC; gpc_addr = pri_gpccs_addr_mask(addr); if (pri_is_gpc_addr_shared(g, addr)) { *addr_type = CTXSW_ADDR_TYPE_GPC; *broadcast_flags |= PRI_BROADCAST_FLAGS_GPC; } else { *gpc_num = pri_get_gpc_num(g, addr); } if (pri_is_ppc_addr(g, gpc_addr)) { *addr_type = CTXSW_ADDR_TYPE_PPC; if (pri_is_ppc_addr_shared(g, gpc_addr)) { *broadcast_flags |= PRI_BROADCAST_FLAGS_PPC; return 0; } } if (g->ops.gr.is_tpc_addr(g, gpc_addr)) { *addr_type = CTXSW_ADDR_TYPE_TPC; if (pri_is_tpc_addr_shared(g, gpc_addr)) { *broadcast_flags |= PRI_BROADCAST_FLAGS_TPC; return 0; } *tpc_num = g->ops.gr.get_tpc_num(g, gpc_addr); } return 0; } else if (pri_is_be_addr(g, addr)) { *addr_type = CTXSW_ADDR_TYPE_BE; if (pri_is_be_addr_shared(g, addr)) { *broadcast_flags |= PRI_BROADCAST_FLAGS_BE; return 0; } *be_num = pri_get_be_num(g, addr); return 0; } else if (g->ops.ltc.pri_is_ltc_addr(g, addr)) { *addr_type = CTXSW_ADDR_TYPE_LTCS; if (g->ops.ltc.is_ltcs_ltss_addr(g, addr)) { *broadcast_flags |= PRI_BROADCAST_FLAGS_LTCS; } else if (g->ops.ltc.is_ltcn_ltss_addr(g, addr)) { *broadcast_flags |= PRI_BROADCAST_FLAGS_LTSS; } return 0; } else if (pri_is_fbpa_addr(g, addr)) { *addr_type = CTXSW_ADDR_TYPE_FBPA; if (pri_is_fbpa_addr_shared(g, addr)) { *broadcast_flags |= PRI_BROADCAST_FLAGS_FBPA; return 0; } return 0; } else if ((g->ops.gr.is_egpc_addr != NULL) && g->ops.gr.is_egpc_addr(g, addr)) { return g->ops.gr.decode_egpc_addr(g, addr, addr_type, gpc_num, tpc_num, broadcast_flags); } else { *addr_type = CTXSW_ADDR_TYPE_SYS; return 0; } /* PPC!?!?!?! */ /*NOTREACHED*/ return -EINVAL; } void gr_gk20a_split_fbpa_broadcast_addr(struct gk20a *g, u32 addr, u32 num_fbpas, u32 *priv_addr_table, u32 *t) { u32 fbpa_id; for (fbpa_id = 0; fbpa_id < num_fbpas; fbpa_id++) { priv_addr_table[(*t)++] = pri_fbpa_addr(g, pri_fbpa_addr_mask(g, addr), fbpa_id); } } int gr_gk20a_split_ppc_broadcast_addr(struct gk20a *g, u32 addr, u32 gpc_num, u32 *priv_addr_table, u32 *t) { u32 ppc_num; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "addr=0x%x", addr); for (ppc_num = 0; ppc_num < g->gr.gpc_ppc_count[gpc_num]; ppc_num++) { priv_addr_table[(*t)++] = pri_ppc_addr(g, pri_ppccs_addr_mask(addr), gpc_num, ppc_num); } return 0; } /* * The context buffer is indexed using BE broadcast addresses and GPC/TPC * unicast addresses. This function will convert a BE unicast address to a BE * broadcast address and split a GPC/TPC broadcast address into a table of * GPC/TPC addresses. The addresses generated by this function can be * successfully processed by gr_gk20a_find_priv_offset_in_buffer */ int gr_gk20a_create_priv_addr_table(struct gk20a *g, u32 addr, u32 *priv_addr_table, u32 *num_registers) { enum ctxsw_addr_type addr_type; u32 gpc_num, tpc_num, ppc_num, be_num; u32 priv_addr, gpc_addr; u32 broadcast_flags; u32 t; int err; t = 0; *num_registers = 0; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "addr=0x%x", addr); err = g->ops.gr.decode_priv_addr(g, addr, &addr_type, &gpc_num, &tpc_num, &ppc_num, &be_num, &broadcast_flags); nvgpu_log(g, gpu_dbg_gpu_dbg, "addr_type = %d", addr_type); if (err != 0) { return err; } if ((addr_type == CTXSW_ADDR_TYPE_SYS) || (addr_type == CTXSW_ADDR_TYPE_BE)) { /* The BE broadcast registers are included in the compressed PRI * table. Convert a BE unicast address to a broadcast address * so that we can look up the offset. */ if ((addr_type == CTXSW_ADDR_TYPE_BE) && ((broadcast_flags & PRI_BROADCAST_FLAGS_BE) == 0U)) { priv_addr_table[t++] = pri_be_shared_addr(g, addr); } else { priv_addr_table[t++] = addr; } *num_registers = t; return 0; } /* The GPC/TPC unicast registers are included in the compressed PRI * tables. Convert a GPC/TPC broadcast address to unicast addresses so * that we can look up the offsets. */ if (broadcast_flags & PRI_BROADCAST_FLAGS_GPC) { for (gpc_num = 0; gpc_num < g->gr.gpc_count; gpc_num++) { if (broadcast_flags & PRI_BROADCAST_FLAGS_TPC) { for (tpc_num = 0; tpc_num < g->gr.gpc_tpc_count[gpc_num]; tpc_num++) { priv_addr_table[t++] = pri_tpc_addr(g, pri_tpccs_addr_mask(addr), gpc_num, tpc_num); } } else if (broadcast_flags & PRI_BROADCAST_FLAGS_PPC) { err = gr_gk20a_split_ppc_broadcast_addr(g, addr, gpc_num, priv_addr_table, &t); if (err != 0) { return err; } } else { priv_addr = pri_gpc_addr(g, pri_gpccs_addr_mask(addr), gpc_num); gpc_addr = pri_gpccs_addr_mask(priv_addr); tpc_num = g->ops.gr.get_tpc_num(g, gpc_addr); if (tpc_num >= g->gr.gpc_tpc_count[gpc_num]) { continue; } priv_addr_table[t++] = priv_addr; } } } else if (((addr_type == CTXSW_ADDR_TYPE_EGPC) || (addr_type == CTXSW_ADDR_TYPE_ETPC)) && (g->ops.gr.egpc_etpc_priv_addr_table != NULL)) { nvgpu_log(g, gpu_dbg_gpu_dbg, "addr_type : EGPC/ETPC"); g->ops.gr.egpc_etpc_priv_addr_table(g, addr, gpc_num, tpc_num, broadcast_flags, priv_addr_table, &t); } else if (broadcast_flags & PRI_BROADCAST_FLAGS_LTSS) { g->ops.ltc.split_lts_broadcast_addr(g, addr, priv_addr_table, &t); } else if (broadcast_flags & PRI_BROADCAST_FLAGS_LTCS) { g->ops.ltc.split_ltc_broadcast_addr(g, addr, priv_addr_table, &t); } else if (broadcast_flags & PRI_BROADCAST_FLAGS_FBPA) { g->ops.gr.split_fbpa_broadcast_addr(g, addr, nvgpu_get_litter_value(g, GPU_LIT_NUM_FBPAS), priv_addr_table, &t); } else if ((broadcast_flags & PRI_BROADCAST_FLAGS_GPC) == 0U) { if (broadcast_flags & PRI_BROADCAST_FLAGS_TPC) { for (tpc_num = 0; tpc_num < g->gr.gpc_tpc_count[gpc_num]; tpc_num++) { priv_addr_table[t++] = pri_tpc_addr(g, pri_tpccs_addr_mask(addr), gpc_num, tpc_num); } } else if (broadcast_flags & PRI_BROADCAST_FLAGS_PPC) { err = gr_gk20a_split_ppc_broadcast_addr(g, addr, gpc_num, priv_addr_table, &t); } else { priv_addr_table[t++] = addr; } } *num_registers = t; return 0; } int gr_gk20a_get_ctx_buffer_offsets(struct gk20a *g, u32 addr, u32 max_offsets, u32 *offsets, u32 *offset_addrs, u32 *num_offsets, bool is_quad, u32 quad) { u32 i; u32 priv_offset = 0; u32 *priv_registers; u32 num_registers = 0; int err = 0; struct gr_gk20a *gr = &g->gr; u32 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); u32 potential_offsets = gr->max_gpc_count * gr->max_tpc_per_gpc_count * sm_per_tpc; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "addr=0x%x", addr); /* implementation is crossed-up if either of these happen */ if (max_offsets > potential_offsets) { nvgpu_log_fn(g, "max_offsets > potential_offsets"); return -EINVAL; } if (!g->gr.ctx_vars.golden_image_initialized) { return -ENODEV; } priv_registers = nvgpu_kzalloc(g, sizeof(u32) * potential_offsets); if (priv_registers == NULL) { nvgpu_log_fn(g, "failed alloc for potential_offsets=%d", potential_offsets); err = PTR_ERR(priv_registers); goto cleanup; } memset(offsets, 0, sizeof(u32) * max_offsets); memset(offset_addrs, 0, sizeof(u32) * max_offsets); *num_offsets = 0; g->ops.gr.create_priv_addr_table(g, addr, &priv_registers[0], &num_registers); if ((max_offsets > 1) && (num_registers > max_offsets)) { nvgpu_log_fn(g, "max_offsets = %d, num_registers = %d", max_offsets, num_registers); err = -EINVAL; goto cleanup; } if ((max_offsets == 1) && (num_registers > 1)) { num_registers = 1; } if (g->gr.ctx_vars.local_golden_image == NULL) { nvgpu_log_fn(g, "no context switch header info to work with"); err = -EINVAL; goto cleanup; } for (i = 0; i < num_registers; i++) { err = gr_gk20a_find_priv_offset_in_buffer(g, priv_registers[i], is_quad, quad, g->gr.ctx_vars.local_golden_image, g->gr.ctx_vars.golden_image_size, &priv_offset); if (err != 0) { nvgpu_log_fn(g, "Could not determine priv_offset for addr:0x%x", addr); /*, grPriRegStr(addr)));*/ goto cleanup; } offsets[i] = priv_offset; offset_addrs[i] = priv_registers[i]; } *num_offsets = num_registers; cleanup: if (!IS_ERR_OR_NULL(priv_registers)) { nvgpu_kfree(g, priv_registers); } return err; } int gr_gk20a_get_pm_ctx_buffer_offsets(struct gk20a *g, u32 addr, u32 max_offsets, u32 *offsets, u32 *offset_addrs, u32 *num_offsets) { u32 i; u32 priv_offset = 0; u32 *priv_registers; u32 num_registers = 0; int err = 0; struct gr_gk20a *gr = &g->gr; u32 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); u32 potential_offsets = gr->max_gpc_count * gr->max_tpc_per_gpc_count * sm_per_tpc; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "addr=0x%x", addr); /* implementation is crossed-up if either of these happen */ if (max_offsets > potential_offsets) { return -EINVAL; } if (!g->gr.ctx_vars.golden_image_initialized) { return -ENODEV; } priv_registers = nvgpu_kzalloc(g, sizeof(u32) * potential_offsets); if (priv_registers == NULL) { nvgpu_log_fn(g, "failed alloc for potential_offsets=%d", potential_offsets); return -ENOMEM; } memset(offsets, 0, sizeof(u32) * max_offsets); memset(offset_addrs, 0, sizeof(u32) * max_offsets); *num_offsets = 0; g->ops.gr.create_priv_addr_table(g, addr, priv_registers, &num_registers); if ((max_offsets > 1) && (num_registers > max_offsets)) { err = -EINVAL; goto cleanup; } if ((max_offsets == 1) && (num_registers > 1)) { num_registers = 1; } if (g->gr.ctx_vars.local_golden_image == NULL) { nvgpu_log_fn(g, "no context switch header info to work with"); err = -EINVAL; goto cleanup; } for (i = 0; i < num_registers; i++) { err = gr_gk20a_find_priv_offset_in_pm_buffer(g, priv_registers[i], &priv_offset); if (err != 0) { nvgpu_log_fn(g, "Could not determine priv_offset for addr:0x%x", addr); /*, grPriRegStr(addr)));*/ goto cleanup; } offsets[i] = priv_offset; offset_addrs[i] = priv_registers[i]; } *num_offsets = num_registers; cleanup: nvgpu_kfree(g, priv_registers); return err; } /* Setup some register tables. This looks hacky; our * register/offset functions are just that, functions. * So they can't be used as initializers... TBD: fix to * generate consts at least on an as-needed basis. */ static const u32 _num_ovr_perf_regs = 17; static u32 _ovr_perf_regs[17] = { 0, }; /* Following are the blocks of registers that the ucode stores in the extended region.*/ void gk20a_gr_init_ovr_sm_dsm_perf(void) { if (_ovr_perf_regs[0] != 0) { return; } _ovr_perf_regs[0] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter_control_sel0_r(); _ovr_perf_regs[1] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter_control_sel1_r(); _ovr_perf_regs[2] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter_control0_r(); _ovr_perf_regs[3] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter_control5_r(); _ovr_perf_regs[4] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter_status1_r(); _ovr_perf_regs[5] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter0_control_r(); _ovr_perf_regs[6] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter1_control_r(); _ovr_perf_regs[7] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter2_control_r(); _ovr_perf_regs[8] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter3_control_r(); _ovr_perf_regs[9] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter4_control_r(); _ovr_perf_regs[10] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter5_control_r(); _ovr_perf_regs[11] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter6_control_r(); _ovr_perf_regs[12] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter7_control_r(); _ovr_perf_regs[13] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter4_r(); _ovr_perf_regs[14] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter5_r(); _ovr_perf_regs[15] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter6_r(); _ovr_perf_regs[16] = gr_pri_gpc0_tpc0_sm_dsm_perf_counter7_r(); } /* TBD: would like to handle this elsewhere, at a higher level. * these are currently constructed in a "test-then-write" style * which makes it impossible to know externally whether a ctx * write will actually occur. so later we should put a lazy, * map-and-hold system in the patch write state */ static int gr_gk20a_ctx_patch_smpc(struct gk20a *g, struct channel_gk20a *ch, u32 addr, u32 data, struct nvgpu_mem *mem) { u32 num_gpc = g->gr.gpc_count; u32 num_tpc; u32 tpc, gpc, reg; u32 chk_addr; u32 vaddr_lo; u32 vaddr_hi; u32 tmp; u32 num_ovr_perf_regs = 0; u32 *ovr_perf_regs = NULL; u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx; struct nvgpu_mem *ctxheader = &ch->ctx_header; tsg = tsg_gk20a_from_ch(ch); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; g->ops.gr.init_ovr_sm_dsm_perf(); g->ops.gr.init_sm_dsm_reg_info(); g->ops.gr.get_ovr_perf_regs(g, &num_ovr_perf_regs, &ovr_perf_regs); nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "addr=0x%x", addr); for (reg = 0; reg < num_ovr_perf_regs; reg++) { for (gpc = 0; gpc < num_gpc; gpc++) { num_tpc = g->gr.gpc_tpc_count[gpc]; for (tpc = 0; tpc < num_tpc; tpc++) { chk_addr = ((gpc_stride * gpc) + (tpc_in_gpc_stride * tpc) + ovr_perf_regs[reg]); if (chk_addr != addr) { continue; } /* reset the patch count from previous runs,if ucode has already processed it */ tmp = nvgpu_mem_rd(g, mem, ctxsw_prog_main_image_patch_count_o()); if (tmp == 0U) { gr_ctx->patch_ctx.data_count = 0; } gr_gk20a_ctx_patch_write(g, gr_ctx, addr, data, true); vaddr_lo = u64_lo32(gr_ctx->patch_ctx.mem.gpu_va); vaddr_hi = u64_hi32(gr_ctx->patch_ctx.mem.gpu_va); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_patch_count_o(), gr_ctx->patch_ctx.data_count); if (ctxheader->gpu_va) { nvgpu_mem_wr(g, ctxheader, ctxsw_prog_main_image_patch_adr_lo_o(), vaddr_lo); nvgpu_mem_wr(g, ctxheader, ctxsw_prog_main_image_patch_adr_hi_o(), vaddr_hi); } else { nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_patch_adr_lo_o(), vaddr_lo); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_patch_adr_hi_o(), vaddr_hi); } /* we're not caching these on cpu side, but later watch for it */ return 0; } } } return 0; } #define ILLEGAL_ID ((u32)~0) static inline bool check_main_image_header_magic(u8 *context) { u32 magic = *(u32 *)(context + ctxsw_prog_main_image_magic_value_o()); return magic == ctxsw_prog_main_image_magic_value_v_value_v(); } static inline bool check_local_header_magic(u8 *context) { u32 magic = *(u32 *)(context + ctxsw_prog_local_magic_value_o()); return magic == ctxsw_prog_local_magic_value_v_value_v(); } /* most likely dupe of ctxsw_gpccs_header__size_1_v() */ static inline int ctxsw_prog_ucode_header_size_in_bytes(void) { return 256; } void gk20a_gr_get_ovr_perf_regs(struct gk20a *g, u32 *num_ovr_perf_regs, u32 **ovr_perf_regs) { *num_ovr_perf_regs = _num_ovr_perf_regs; *ovr_perf_regs = _ovr_perf_regs; } static int gr_gk20a_find_priv_offset_in_ext_buffer(struct gk20a *g, u32 addr, bool is_quad, u32 quad, u32 *context_buffer, u32 context_buffer_size, u32 *priv_offset) { u32 i, data32; u32 gpc_num, tpc_num; u32 num_gpcs, num_tpcs; u32 chk_addr; u32 ext_priv_offset, ext_priv_size; u8 *context; u32 offset_to_segment, offset_to_segment_end; u32 sm_dsm_perf_reg_id = ILLEGAL_ID; u32 sm_dsm_perf_ctrl_reg_id = ILLEGAL_ID; u32 num_ext_gpccs_ext_buffer_segments; u32 inter_seg_offset; u32 max_tpc_count; u32 *sm_dsm_perf_ctrl_regs = NULL; u32 num_sm_dsm_perf_ctrl_regs = 0; u32 *sm_dsm_perf_regs = NULL; u32 num_sm_dsm_perf_regs = 0; u32 buffer_segments_size = 0; u32 marker_size = 0; u32 control_register_stride = 0; u32 perf_register_stride = 0; struct gr_gk20a *gr = &g->gr; u32 gpc_base = nvgpu_get_litter_value(g, GPU_LIT_GPC_BASE); u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); u32 tpc_in_gpc_base = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_BASE); u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); u32 tpc_gpc_mask = (tpc_in_gpc_stride - 1); /* Only have TPC registers in extended region, so if not a TPC reg, then return error so caller can look elsewhere. */ if (pri_is_gpc_addr(g, addr)) { u32 gpc_addr = 0; gpc_num = pri_get_gpc_num(g, addr); gpc_addr = pri_gpccs_addr_mask(addr); if (g->ops.gr.is_tpc_addr(g, gpc_addr)) { tpc_num = g->ops.gr.get_tpc_num(g, gpc_addr); } else { return -EINVAL; } nvgpu_log_info(g, " gpc = %d tpc = %d", gpc_num, tpc_num); } else if ((g->ops.gr.is_etpc_addr != NULL) && g->ops.gr.is_etpc_addr(g, addr)) { g->ops.gr.get_egpc_etpc_num(g, addr, &gpc_num, &tpc_num); gpc_base = g->ops.gr.get_egpc_base(g); } else { nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "does not exist in extended region"); return -EINVAL; } buffer_segments_size = ctxsw_prog_extended_buffer_segments_size_in_bytes_v(); /* note below is in words/num_registers */ marker_size = ctxsw_prog_extended_marker_size_in_bytes_v() >> 2; context = (u8 *)context_buffer; /* sanity check main header */ if (!check_main_image_header_magic(context)) { nvgpu_err(g, "Invalid main header: magic value"); return -EINVAL; } num_gpcs = *(u32 *)(context + ctxsw_prog_main_image_num_gpcs_o()); if (gpc_num >= num_gpcs) { nvgpu_err(g, "GPC 0x%08x is greater than total count 0x%08x!", gpc_num, num_gpcs); return -EINVAL; } data32 = *(u32 *)(context + ctxsw_prog_main_extended_buffer_ctl_o()); ext_priv_size = ctxsw_prog_main_extended_buffer_ctl_size_v(data32); if (0 == ext_priv_size) { nvgpu_log_info(g, " No extended memory in context buffer"); return -EINVAL; } ext_priv_offset = ctxsw_prog_main_extended_buffer_ctl_offset_v(data32); offset_to_segment = ext_priv_offset * ctxsw_prog_ucode_header_size_in_bytes(); offset_to_segment_end = offset_to_segment + (ext_priv_size * buffer_segments_size); /* check local header magic */ context += ctxsw_prog_ucode_header_size_in_bytes(); if (!check_local_header_magic(context)) { nvgpu_err(g, "Invalid local header: magic value"); return -EINVAL; } /* * See if the incoming register address is in the first table of * registers. We check this by decoding only the TPC addr portion. * If we get a hit on the TPC bit, we then double check the address * by computing it from the base gpc/tpc strides. Then make sure * it is a real match. */ g->ops.gr.get_sm_dsm_perf_regs(g, &num_sm_dsm_perf_regs, &sm_dsm_perf_regs, &perf_register_stride); g->ops.gr.init_sm_dsm_reg_info(); for (i = 0; i < num_sm_dsm_perf_regs; i++) { if ((addr & tpc_gpc_mask) == (sm_dsm_perf_regs[i] & tpc_gpc_mask)) { sm_dsm_perf_reg_id = i; nvgpu_log_info(g, "register match: 0x%08x", sm_dsm_perf_regs[i]); chk_addr = (gpc_base + gpc_stride * gpc_num) + tpc_in_gpc_base + (tpc_in_gpc_stride * tpc_num) + (sm_dsm_perf_regs[sm_dsm_perf_reg_id] & tpc_gpc_mask); if (chk_addr != addr) { nvgpu_err(g, "Oops addr miss-match! : 0x%08x != 0x%08x", addr, chk_addr); return -EINVAL; } break; } } /* Didn't find reg in supported group 1. * so try the second group now */ g->ops.gr.get_sm_dsm_perf_ctrl_regs(g, &num_sm_dsm_perf_ctrl_regs, &sm_dsm_perf_ctrl_regs, &control_register_stride); if (ILLEGAL_ID == sm_dsm_perf_reg_id) { for (i = 0; i < num_sm_dsm_perf_ctrl_regs; i++) { if ((addr & tpc_gpc_mask) == (sm_dsm_perf_ctrl_regs[i] & tpc_gpc_mask)) { sm_dsm_perf_ctrl_reg_id = i; nvgpu_log_info(g, "register match: 0x%08x", sm_dsm_perf_ctrl_regs[i]); chk_addr = (gpc_base + gpc_stride * gpc_num) + tpc_in_gpc_base + tpc_in_gpc_stride * tpc_num + (sm_dsm_perf_ctrl_regs[sm_dsm_perf_ctrl_reg_id] & tpc_gpc_mask); if (chk_addr != addr) { nvgpu_err(g, "Oops addr miss-match! : 0x%08x != 0x%08x", addr, chk_addr); return -EINVAL; } break; } } } if ((ILLEGAL_ID == sm_dsm_perf_ctrl_reg_id) && (ILLEGAL_ID == sm_dsm_perf_reg_id)) { return -EINVAL; } /* Skip the FECS extended header, nothing there for us now. */ offset_to_segment += buffer_segments_size; /* skip through the GPCCS extended headers until we get to the data for * our GPC. The size of each gpc extended segment is enough to hold the * max tpc count for the gpcs,in 256b chunks. */ max_tpc_count = gr->max_tpc_per_gpc_count; num_ext_gpccs_ext_buffer_segments = (u32)((max_tpc_count + 1) / 2); offset_to_segment += (num_ext_gpccs_ext_buffer_segments * buffer_segments_size * gpc_num); num_tpcs = g->gr.gpc_tpc_count[gpc_num]; /* skip the head marker to start with */ inter_seg_offset = marker_size; if (ILLEGAL_ID != sm_dsm_perf_ctrl_reg_id) { /* skip over control regs of TPC's before the one we want. * then skip to the register in this tpc */ inter_seg_offset = inter_seg_offset + (tpc_num * control_register_stride) + sm_dsm_perf_ctrl_reg_id; } else { /* skip all the control registers */ inter_seg_offset = inter_seg_offset + (num_tpcs * control_register_stride); /* skip the marker between control and counter segments */ inter_seg_offset += marker_size; /* skip over counter regs of TPCs before the one we want */ inter_seg_offset = inter_seg_offset + (tpc_num * perf_register_stride) * ctxsw_prog_extended_num_smpc_quadrants_v(); /* skip over the register for the quadrants we do not want. * then skip to the register in this tpc */ inter_seg_offset = inter_seg_offset + (perf_register_stride * quad) + sm_dsm_perf_reg_id; } /* set the offset to the segment offset plus the inter segment offset to * our register */ offset_to_segment += (inter_seg_offset * 4); /* last sanity check: did we somehow compute an offset outside the * extended buffer? */ if (offset_to_segment > offset_to_segment_end) { nvgpu_err(g, "Overflow ctxsw buffer! 0x%08x > 0x%08x", offset_to_segment, offset_to_segment_end); return -EINVAL; } *priv_offset = offset_to_segment; return 0; } static int gr_gk20a_process_context_buffer_priv_segment(struct gk20a *g, enum ctxsw_addr_type addr_type, u32 pri_addr, u32 gpc_num, u32 num_tpcs, u32 num_ppcs, u32 ppc_mask, u32 *priv_offset) { u32 i; u32 address, base_address; u32 sys_offset, gpc_offset, tpc_offset, ppc_offset; u32 ppc_num, tpc_num, tpc_addr, gpc_addr, ppc_addr; struct aiv_gk20a *reg; u32 gpc_base = nvgpu_get_litter_value(g, GPU_LIT_GPC_BASE); u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); u32 ppc_in_gpc_base = nvgpu_get_litter_value(g, GPU_LIT_PPC_IN_GPC_BASE); u32 ppc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_PPC_IN_GPC_STRIDE); u32 tpc_in_gpc_base = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_BASE); u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "pri_addr=0x%x", pri_addr); if (!g->gr.ctx_vars.valid) { return -EINVAL; } /* Process the SYS/BE segment. */ if ((addr_type == CTXSW_ADDR_TYPE_SYS) || (addr_type == CTXSW_ADDR_TYPE_BE)) { for (i = 0; i < g->gr.ctx_vars.ctxsw_regs.sys.count; i++) { reg = &g->gr.ctx_vars.ctxsw_regs.sys.l[i]; address = reg->addr; sys_offset = reg->index; if (pri_addr == address) { *priv_offset = sys_offset; return 0; } } } /* Process the TPC segment. */ if (addr_type == CTXSW_ADDR_TYPE_TPC) { for (tpc_num = 0; tpc_num < num_tpcs; tpc_num++) { for (i = 0; i < g->gr.ctx_vars.ctxsw_regs.tpc.count; i++) { reg = &g->gr.ctx_vars.ctxsw_regs.tpc.l[i]; address = reg->addr; tpc_addr = pri_tpccs_addr_mask(address); base_address = gpc_base + (gpc_num * gpc_stride) + tpc_in_gpc_base + (tpc_num * tpc_in_gpc_stride); address = base_address + tpc_addr; /* * The data for the TPCs is interleaved in the context buffer. * Example with num_tpcs = 2 * 0 1 2 3 4 5 6 7 8 9 10 11 ... * 0-0 1-0 0-1 1-1 0-2 1-2 0-3 1-3 0-4 1-4 0-5 1-5 ... */ tpc_offset = (reg->index * num_tpcs) + (tpc_num * 4); if (pri_addr == address) { *priv_offset = tpc_offset; return 0; } } } } else if ((addr_type == CTXSW_ADDR_TYPE_EGPC) || (addr_type == CTXSW_ADDR_TYPE_ETPC)) { if (g->ops.gr.get_egpc_base == NULL) { return -EINVAL; } for (tpc_num = 0; tpc_num < num_tpcs; tpc_num++) { for (i = 0; i < g->gr.ctx_vars.ctxsw_regs.etpc.count; i++) { reg = &g->gr.ctx_vars.ctxsw_regs.etpc.l[i]; address = reg->addr; tpc_addr = pri_tpccs_addr_mask(address); base_address = g->ops.gr.get_egpc_base(g) + (gpc_num * gpc_stride) + tpc_in_gpc_base + (tpc_num * tpc_in_gpc_stride); address = base_address + tpc_addr; /* * The data for the TPCs is interleaved in the context buffer. * Example with num_tpcs = 2 * 0 1 2 3 4 5 6 7 8 9 10 11 ... * 0-0 1-0 0-1 1-1 0-2 1-2 0-3 1-3 0-4 1-4 0-5 1-5 ... */ tpc_offset = (reg->index * num_tpcs) + (tpc_num * 4); if (pri_addr == address) { *priv_offset = tpc_offset; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "egpc/etpc priv_offset=0x%#08x", *priv_offset); return 0; } } } } /* Process the PPC segment. */ if (addr_type == CTXSW_ADDR_TYPE_PPC) { for (ppc_num = 0; ppc_num < num_ppcs; ppc_num++) { for (i = 0; i < g->gr.ctx_vars.ctxsw_regs.ppc.count; i++) { reg = &g->gr.ctx_vars.ctxsw_regs.ppc.l[i]; address = reg->addr; ppc_addr = pri_ppccs_addr_mask(address); base_address = gpc_base + (gpc_num * gpc_stride) + ppc_in_gpc_base + (ppc_num * ppc_in_gpc_stride); address = base_address + ppc_addr; /* * The data for the PPCs is interleaved in the context buffer. * Example with numPpcs = 2 * 0 1 2 3 4 5 6 7 8 9 10 11 ... * 0-0 1-0 0-1 1-1 0-2 1-2 0-3 1-3 0-4 1-4 0-5 1-5 ... */ ppc_offset = (reg->index * num_ppcs) + (ppc_num * 4); if (pri_addr == address) { *priv_offset = ppc_offset; return 0; } } } } /* Process the GPC segment. */ if (addr_type == CTXSW_ADDR_TYPE_GPC) { for (i = 0; i < g->gr.ctx_vars.ctxsw_regs.gpc.count; i++) { reg = &g->gr.ctx_vars.ctxsw_regs.gpc.l[i]; address = reg->addr; gpc_addr = pri_gpccs_addr_mask(address); gpc_offset = reg->index; base_address = gpc_base + (gpc_num * gpc_stride); address = base_address + gpc_addr; if (pri_addr == address) { *priv_offset = gpc_offset; return 0; } } } return -EINVAL; } static int gr_gk20a_determine_ppc_configuration(struct gk20a *g, u8 *context, u32 *num_ppcs, u32 *ppc_mask, u32 *reg_ppc_count) { u32 data32; u32 num_pes_per_gpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_PES_PER_GPC); /* * if there is only 1 PES_PER_GPC, then we put the PES registers * in the GPC reglist, so we can't error out if ppc.count == 0 */ if ((!g->gr.ctx_vars.valid) || ((g->gr.ctx_vars.ctxsw_regs.ppc.count == 0) && (num_pes_per_gpc > 1))) { return -EINVAL; } data32 = *(u32 *)(context + ctxsw_prog_local_image_ppc_info_o()); *num_ppcs = ctxsw_prog_local_image_ppc_info_num_ppcs_v(data32); *ppc_mask = ctxsw_prog_local_image_ppc_info_ppc_mask_v(data32); *reg_ppc_count = g->gr.ctx_vars.ctxsw_regs.ppc.count; return 0; } int gr_gk20a_get_offset_in_gpccs_segment(struct gk20a *g, enum ctxsw_addr_type addr_type, u32 num_tpcs, u32 num_ppcs, u32 reg_list_ppc_count, u32 *__offset_in_segment) { u32 offset_in_segment = 0; struct gr_gk20a *gr = &g->gr; if (addr_type == CTXSW_ADDR_TYPE_TPC) { /* * reg = gr->ctx_vars.ctxsw_regs.tpc.l; * offset_in_segment = 0; */ } else if ((addr_type == CTXSW_ADDR_TYPE_EGPC) || (addr_type == CTXSW_ADDR_TYPE_ETPC)) { offset_in_segment = ((gr->ctx_vars.ctxsw_regs.tpc.count * num_tpcs) << 2); nvgpu_log(g, gpu_dbg_info | gpu_dbg_gpu_dbg, "egpc etpc offset_in_segment 0x%#08x", offset_in_segment); } else if (addr_type == CTXSW_ADDR_TYPE_PPC) { /* * The ucode stores TPC data before PPC data. * Advance offset past TPC data to PPC data. */ offset_in_segment = (((gr->ctx_vars.ctxsw_regs.tpc.count + gr->ctx_vars.ctxsw_regs.etpc.count) * num_tpcs) << 2); } else if (addr_type == CTXSW_ADDR_TYPE_GPC) { /* * The ucode stores TPC/PPC data before GPC data. * Advance offset past TPC/PPC data to GPC data. * * Note 1 PES_PER_GPC case */ u32 num_pes_per_gpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_PES_PER_GPC); if (num_pes_per_gpc > 1) { offset_in_segment = ((((gr->ctx_vars.ctxsw_regs.tpc.count + gr->ctx_vars.ctxsw_regs.etpc.count) * num_tpcs) << 2) + ((reg_list_ppc_count * num_ppcs) << 2)); } else { offset_in_segment = (((gr->ctx_vars.ctxsw_regs.tpc.count + gr->ctx_vars.ctxsw_regs.etpc.count) * num_tpcs) << 2); } } else { nvgpu_log_fn(g, "Unknown address type."); return -EINVAL; } *__offset_in_segment = offset_in_segment; return 0; } /* * This function will return the 32 bit offset for a priv register if it is * present in the context buffer. The context buffer is in CPU memory. */ static int gr_gk20a_find_priv_offset_in_buffer(struct gk20a *g, u32 addr, bool is_quad, u32 quad, u32 *context_buffer, u32 context_buffer_size, u32 *priv_offset) { u32 i, data32; int err; enum ctxsw_addr_type addr_type; u32 broadcast_flags; u32 gpc_num, tpc_num, ppc_num, be_num; u32 num_gpcs, num_tpcs, num_ppcs; u32 offset; u32 sys_priv_offset, gpc_priv_offset; u32 ppc_mask, reg_list_ppc_count; u8 *context; u32 offset_to_segment, offset_in_segment = 0; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "addr=0x%x", addr); err = g->ops.gr.decode_priv_addr(g, addr, &addr_type, &gpc_num, &tpc_num, &ppc_num, &be_num, &broadcast_flags); nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "addr_type = %d, broadcast_flags: %08x", addr_type, broadcast_flags); if (err != 0) { return err; } context = (u8 *)context_buffer; if (!check_main_image_header_magic(context)) { nvgpu_err(g, "Invalid main header: magic value"); return -EINVAL; } num_gpcs = *(u32 *)(context + ctxsw_prog_main_image_num_gpcs_o()); /* Parse the FECS local header. */ context += ctxsw_prog_ucode_header_size_in_bytes(); if (!check_local_header_magic(context)) { nvgpu_err(g, "Invalid FECS local header: magic value"); return -EINVAL; } data32 = *(u32 *)(context + ctxsw_prog_local_priv_register_ctl_o()); sys_priv_offset = ctxsw_prog_local_priv_register_ctl_offset_v(data32); nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "sys_priv_offset=0x%x", sys_priv_offset); /* If found in Ext buffer, ok. * If it failed and we expected to find it there (quad offset) * then return the error. Otherwise continue on. */ err = gr_gk20a_find_priv_offset_in_ext_buffer(g, addr, is_quad, quad, context_buffer, context_buffer_size, priv_offset); if ((err == 0) || ((err != 0) && is_quad)) { nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "err = %d, is_quad = %s", err, is_quad ? "true" : "false"); return err; } if ((addr_type == CTXSW_ADDR_TYPE_SYS) || (addr_type == CTXSW_ADDR_TYPE_BE)) { /* Find the offset in the FECS segment. */ offset_to_segment = sys_priv_offset * ctxsw_prog_ucode_header_size_in_bytes(); err = gr_gk20a_process_context_buffer_priv_segment(g, addr_type, addr, 0, 0, 0, 0, &offset); if (err != 0) { return err; } *priv_offset = (offset_to_segment + offset); return 0; } if ((gpc_num + 1) > num_gpcs) { nvgpu_err(g, "GPC %d not in this context buffer.", gpc_num); return -EINVAL; } /* Parse the GPCCS local header(s).*/ for (i = 0; i < num_gpcs; i++) { context += ctxsw_prog_ucode_header_size_in_bytes(); if (!check_local_header_magic(context)) { nvgpu_err(g, "Invalid GPCCS local header: magic value"); return -EINVAL; } data32 = *(u32 *)(context + ctxsw_prog_local_priv_register_ctl_o()); gpc_priv_offset = ctxsw_prog_local_priv_register_ctl_offset_v(data32); err = gr_gk20a_determine_ppc_configuration(g, context, &num_ppcs, &ppc_mask, ®_list_ppc_count); if (err != 0) { nvgpu_err(g, "determine ppc configuration failed"); return err; } num_tpcs = *(u32 *)(context + ctxsw_prog_local_image_num_tpcs_o()); if ((i == gpc_num) && ((tpc_num + 1) > num_tpcs)) { nvgpu_err(g, "GPC %d TPC %d not in this context buffer.", gpc_num, tpc_num); return -EINVAL; } /* Find the offset in the GPCCS segment.*/ if (i == gpc_num) { nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "gpc_priv_offset 0x%#08x", gpc_priv_offset); offset_to_segment = gpc_priv_offset * ctxsw_prog_ucode_header_size_in_bytes(); err = g->ops.gr.get_offset_in_gpccs_segment(g, addr_type, num_tpcs, num_ppcs, reg_list_ppc_count, &offset_in_segment); if (err != 0) { return -EINVAL; } offset_to_segment += offset_in_segment; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "offset_to_segment 0x%#08x", offset_to_segment); err = gr_gk20a_process_context_buffer_priv_segment(g, addr_type, addr, i, num_tpcs, num_ppcs, ppc_mask, &offset); if (err != 0) { return -EINVAL; } *priv_offset = offset_to_segment + offset; return 0; } } return -EINVAL; } static int map_cmp(const void *a, const void *b) { struct ctxsw_buf_offset_map_entry *e1 = (struct ctxsw_buf_offset_map_entry *)a; struct ctxsw_buf_offset_map_entry *e2 = (struct ctxsw_buf_offset_map_entry *)b; if (e1->addr < e2->addr) { return -1; } if (e1->addr > e2->addr) { return 1; } return 0; } static int add_ctxsw_buffer_map_entries_pmsys(struct ctxsw_buf_offset_map_entry *map, struct aiv_list_gk20a *regs, u32 *count, u32 *offset, u32 max_cnt, u32 base, u32 mask) { u32 idx; u32 cnt = *count; u32 off = *offset; if ((cnt + regs->count) > max_cnt) { return -EINVAL; } for (idx = 0; idx < regs->count; idx++) { if ((base + (regs->l[idx].addr & mask)) < 0xFFF) { map[cnt].addr = base + (regs->l[idx].addr & mask) + NV_PCFG_BASE; } else { map[cnt].addr = base + (regs->l[idx].addr & mask); } map[cnt++].offset = off; off += 4; } *count = cnt; *offset = off; return 0; } static int add_ctxsw_buffer_map_entries_pmgpc(struct gk20a *g, struct ctxsw_buf_offset_map_entry *map, struct aiv_list_gk20a *regs, u32 *count, u32 *offset, u32 max_cnt, u32 base, u32 mask) { u32 idx; u32 cnt = *count; u32 off = *offset; if ((cnt + regs->count) > max_cnt) { return -EINVAL; } /* NOTE: The PPC offsets get added to the pm_gpc list if numPpc <= 1 * To handle the case of PPC registers getting added into GPC, the below * code specifically checks for any PPC offsets and adds them using * proper mask */ for (idx = 0; idx < regs->count; idx++) { /* Check if the address is PPC address */ if (pri_is_ppc_addr_shared(g, regs->l[idx].addr & mask)) { u32 ppc_in_gpc_base = nvgpu_get_litter_value(g, GPU_LIT_PPC_IN_GPC_BASE); u32 ppc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_PPC_IN_GPC_STRIDE); /* Use PPC mask instead of the GPC mask provided */ u32 ppcmask = ppc_in_gpc_stride - 1; map[cnt].addr = base + ppc_in_gpc_base + (regs->l[idx].addr & ppcmask); } else { map[cnt].addr = base + (regs->l[idx].addr & mask); } map[cnt++].offset = off; off += 4; } *count = cnt; *offset = off; return 0; } static int add_ctxsw_buffer_map_entries(struct ctxsw_buf_offset_map_entry *map, struct aiv_list_gk20a *regs, u32 *count, u32 *offset, u32 max_cnt, u32 base, u32 mask) { u32 idx; u32 cnt = *count; u32 off = *offset; if ((cnt + regs->count) > max_cnt) { return -EINVAL; } for (idx = 0; idx < regs->count; idx++) { map[cnt].addr = base + (regs->l[idx].addr & mask); map[cnt++].offset = off; off += 4; } *count = cnt; *offset = off; return 0; } /* Helper function to add register entries to the register map for all * subunits */ static int add_ctxsw_buffer_map_entries_subunits( struct ctxsw_buf_offset_map_entry *map, struct aiv_list_gk20a *regs, u32 *count, u32 *offset, u32 max_cnt, u32 base, u32 num_units, u32 stride, u32 mask) { u32 unit; u32 idx; u32 cnt = *count; u32 off = *offset; if ((cnt + (regs->count * num_units)) > max_cnt) { return -EINVAL; } /* Data is interleaved for units in ctxsw buffer */ for (idx = 0; idx < regs->count; idx++) { for (unit = 0; unit < num_units; unit++) { map[cnt].addr = base + (regs->l[idx].addr & mask) + (unit * stride); map[cnt++].offset = off; off += 4; } } *count = cnt; *offset = off; return 0; } int gr_gk20a_add_ctxsw_reg_pm_fbpa(struct gk20a *g, struct ctxsw_buf_offset_map_entry *map, struct aiv_list_gk20a *regs, u32 *count, u32 *offset, u32 max_cnt, u32 base, u32 num_fbpas, u32 stride, u32 mask) { return add_ctxsw_buffer_map_entries_subunits(map, regs, count, offset, max_cnt, base, num_fbpas, stride, mask); } static int add_ctxsw_buffer_map_entries_gpcs(struct gk20a *g, struct ctxsw_buf_offset_map_entry *map, u32 *count, u32 *offset, u32 max_cnt) { u32 num_gpcs = g->gr.gpc_count; u32 num_ppcs, num_tpcs, gpc_num, base; u32 gpc_base = nvgpu_get_litter_value(g, GPU_LIT_GPC_BASE); u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); u32 ppc_in_gpc_base = nvgpu_get_litter_value(g, GPU_LIT_PPC_IN_GPC_BASE); u32 ppc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_PPC_IN_GPC_STRIDE); u32 tpc_in_gpc_base = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_BASE); u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); for (gpc_num = 0; gpc_num < num_gpcs; gpc_num++) { num_tpcs = g->gr.gpc_tpc_count[gpc_num]; base = gpc_base + (gpc_stride * gpc_num) + tpc_in_gpc_base; if (add_ctxsw_buffer_map_entries_subunits(map, &g->gr.ctx_vars.ctxsw_regs.pm_tpc, count, offset, max_cnt, base, num_tpcs, tpc_in_gpc_stride, (tpc_in_gpc_stride - 1))) { return -EINVAL; } num_ppcs = g->gr.gpc_ppc_count[gpc_num]; base = gpc_base + (gpc_stride * gpc_num) + ppc_in_gpc_base; if (add_ctxsw_buffer_map_entries_subunits(map, &g->gr.ctx_vars.ctxsw_regs.pm_ppc, count, offset, max_cnt, base, num_ppcs, ppc_in_gpc_stride, (ppc_in_gpc_stride - 1))) { return -EINVAL; } base = gpc_base + (gpc_stride * gpc_num); if (add_ctxsw_buffer_map_entries_pmgpc(g, map, &g->gr.ctx_vars.ctxsw_regs.pm_gpc, count, offset, max_cnt, base, (gpc_stride - 1))) { return -EINVAL; } base = NV_XBAR_MXBAR_PRI_GPC_GNIC_STRIDE * gpc_num; if (add_ctxsw_buffer_map_entries(map, &g->gr.ctx_vars.ctxsw_regs.pm_ucgpc, count, offset, max_cnt, base, ~0)) { return -EINVAL; } base = (g->ops.gr.get_pmm_per_chiplet_offset() * gpc_num); if (add_ctxsw_buffer_map_entries(map, &g->gr.ctx_vars.ctxsw_regs.perf_gpc, count, offset, max_cnt, base, ~0)) { return -EINVAL; } base = (NV_PERF_PMMGPCROUTER_STRIDE * gpc_num); if (add_ctxsw_buffer_map_entries(map, &g->gr.ctx_vars.ctxsw_regs.gpc_router, count, offset, max_cnt, base, ~0)) { return -EINVAL; } /* Counter Aggregation Unit, if available */ if (g->gr.ctx_vars.ctxsw_regs.pm_cau.count) { base = gpc_base + (gpc_stride * gpc_num) + tpc_in_gpc_base; if (add_ctxsw_buffer_map_entries_subunits(map, &g->gr.ctx_vars.ctxsw_regs.pm_cau, count, offset, max_cnt, base, num_tpcs, tpc_in_gpc_stride, (tpc_in_gpc_stride - 1))) { return -EINVAL; } } *offset = ALIGN(*offset, 256); } return 0; } int gr_gk20a_add_ctxsw_reg_perf_pma(struct ctxsw_buf_offset_map_entry *map, struct aiv_list_gk20a *regs, u32 *count, u32 *offset, u32 max_cnt, u32 base, u32 mask) { return add_ctxsw_buffer_map_entries(map, regs, count, offset, max_cnt, base, mask); } /* * PM CTXSW BUFFER LAYOUT : *|---------------------------------------------|0x00 <----PM CTXSW BUFFER BASE *| | *| LIST_compressed_pm_ctx_reg_SYS |Space allocated: numRegs words *|---------------------------------------------| *| | *| LIST_compressed_nv_perf_ctx_reg_SYS |Space allocated: numRegs words *|---------------------------------------------| *| | *| LIST_compressed_nv_perf_ctx_reg_sysrouter|Space allocated: numRegs words *|---------------------------------------------| *| | *| LIST_compressed_nv_perf_ctx_reg_PMA |Space allocated: numRegs words *|---------------------------------------------| *| PADDING for 256 byte alignment | *|---------------------------------------------|<----256 byte aligned *| LIST_compressed_nv_perf_fbp_ctx_regs | *| |Space allocated: numRegs * n words (for n FB units) *|---------------------------------------------| *| LIST_compressed_nv_perf_fbprouter_ctx_regs | *| |Space allocated: numRegs * n words (for n FB units) *|---------------------------------------------| *| LIST_compressed_pm_fbpa_ctx_regs | *| |Space allocated: numRegs * n words (for n FB units) *|---------------------------------------------| *| LIST_compressed_pm_rop_ctx_regs | *|---------------------------------------------| *| LIST_compressed_pm_ltc_ctx_regs | *| LTC0 LTS0 | *| LTC1 LTS0 |Space allocated: numRegs * n words (for n LTC units) *| LTCn LTS0 | *| LTC0 LTS1 | *| LTC1 LTS1 | *| LTCn LTS1 | *| LTC0 LTSn | *| LTC1 LTSn | *| LTCn LTSn | *|---------------------------------------------| *| PADDING for 256 byte alignment | *|---------------------------------------------|<----256 byte aligned *| GPC0 REG0 TPC0 |Each GPC has space allocated to accommodate *| REG0 TPC1 | all the GPC/TPC register lists *| Lists in each GPC region: REG0 TPCn |Per GPC allocated space is always 256 byte aligned *| LIST_pm_ctx_reg_TPC REG1 TPC0 | *| * numTpcs REG1 TPC1 | *| LIST_pm_ctx_reg_PPC REG1 TPCn | *| * numPpcs REGn TPC0 | *| LIST_pm_ctx_reg_GPC REGn TPC1 | *| List_pm_ctx_reg_uc_GPC REGn TPCn | *| LIST_nv_perf_ctx_reg_GPC | *| LIST_nv_perf_gpcrouter_ctx_reg | *| LIST_nv_perf_ctx_reg_CAU | *| ---- |-- *| GPC1 . | *| . |<---- *|---------------------------------------------| *= = *| GPCn | *= = *|---------------------------------------------| */ static int gr_gk20a_create_hwpm_ctxsw_buffer_offset_map(struct gk20a *g) { u32 hwpm_ctxsw_buffer_size = g->gr.ctx_vars.pm_ctxsw_image_size; u32 hwpm_ctxsw_reg_count_max; u32 map_size; u32 i, count = 0; u32 offset = 0; struct ctxsw_buf_offset_map_entry *map; u32 ltc_stride = nvgpu_get_litter_value(g, GPU_LIT_LTC_STRIDE); u32 num_fbpas = nvgpu_get_litter_value(g, GPU_LIT_NUM_FBPAS); u32 fbpa_stride = nvgpu_get_litter_value(g, GPU_LIT_FBPA_STRIDE); u32 num_ltc = g->ops.gr.get_max_ltc_per_fbp(g) * g->gr.num_fbps; if (hwpm_ctxsw_buffer_size == 0) { nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "no PM Ctxsw buffer memory in context buffer"); return -EINVAL; } hwpm_ctxsw_reg_count_max = hwpm_ctxsw_buffer_size >> 2; map_size = hwpm_ctxsw_reg_count_max * sizeof(*map); map = nvgpu_big_zalloc(g, map_size); if (map == NULL) { return -ENOMEM; } /* Add entries from _LIST_pm_ctx_reg_SYS */ if (add_ctxsw_buffer_map_entries_pmsys(map, &g->gr.ctx_vars.ctxsw_regs.pm_sys, &count, &offset, hwpm_ctxsw_reg_count_max, 0, ~0)) { goto cleanup; } /* Add entries from _LIST_nv_perf_ctx_reg_SYS */ if (add_ctxsw_buffer_map_entries(map, &g->gr.ctx_vars.ctxsw_regs.perf_sys, &count, &offset, hwpm_ctxsw_reg_count_max, 0, ~0)) { goto cleanup; } /* Add entries from _LIST_nv_perf_sysrouter_ctx_reg*/ if (add_ctxsw_buffer_map_entries(map, &g->gr.ctx_vars.ctxsw_regs.perf_sys_router, &count, &offset, hwpm_ctxsw_reg_count_max, 0, ~0)) { goto cleanup; } /* Add entries from _LIST_nv_perf_pma_ctx_reg*/ if (g->ops.gr.add_ctxsw_reg_perf_pma(map, &g->gr.ctx_vars.ctxsw_regs.perf_pma, &count, &offset, hwpm_ctxsw_reg_count_max, 0, ~0)) { goto cleanup; } offset = ALIGN(offset, 256); /* Add entries from _LIST_nv_perf_fbp_ctx_regs */ if (add_ctxsw_buffer_map_entries_subunits(map, &g->gr.ctx_vars.ctxsw_regs.fbp, &count, &offset, hwpm_ctxsw_reg_count_max, 0, g->gr.num_fbps, g->ops.gr.get_pmm_per_chiplet_offset(), ~0)) { goto cleanup; } /* Add entries from _LIST_nv_perf_fbprouter_ctx_regs */ if (add_ctxsw_buffer_map_entries_subunits(map, &g->gr.ctx_vars.ctxsw_regs.fbp_router, &count, &offset, hwpm_ctxsw_reg_count_max, 0, g->gr.num_fbps, NV_PERF_PMM_FBP_ROUTER_STRIDE, ~0)) { goto cleanup; } /* Add entries from _LIST_nv_pm_fbpa_ctx_regs */ if (g->ops.gr.add_ctxsw_reg_pm_fbpa(g, map, &g->gr.ctx_vars.ctxsw_regs.pm_fbpa, &count, &offset, hwpm_ctxsw_reg_count_max, 0, num_fbpas, fbpa_stride, ~0)) { goto cleanup; } /* Add entries from _LIST_nv_pm_rop_ctx_regs */ if (add_ctxsw_buffer_map_entries(map, &g->gr.ctx_vars.ctxsw_regs.pm_rop, &count, &offset, hwpm_ctxsw_reg_count_max, 0, ~0)) { goto cleanup; } /* Add entries from _LIST_compressed_nv_pm_ltc_ctx_regs */ if (add_ctxsw_buffer_map_entries_subunits(map, &g->gr.ctx_vars.ctxsw_regs.pm_ltc, &count, &offset, hwpm_ctxsw_reg_count_max, 0, num_ltc, ltc_stride, ~0)) { goto cleanup; } offset = ALIGN(offset, 256); /* Add GPC entries */ if (add_ctxsw_buffer_map_entries_gpcs(g, map, &count, &offset, hwpm_ctxsw_reg_count_max)) { goto cleanup; } if (offset > hwpm_ctxsw_buffer_size) { nvgpu_err(g, "offset > buffer size"); goto cleanup; } sort(map, count, sizeof(*map), map_cmp, NULL); g->gr.ctx_vars.hwpm_ctxsw_buffer_offset_map = map; g->gr.ctx_vars.hwpm_ctxsw_buffer_offset_map_count = count; nvgpu_log_info(g, "Reg Addr => HWPM Ctxt switch buffer offset"); for (i = 0; i < count; i++) { nvgpu_log_info(g, "%08x => %08x", map[i].addr, map[i].offset); } return 0; cleanup: nvgpu_err(g, "Failed to create HWPM buffer offset map"); nvgpu_big_free(g, map); return -EINVAL; } /* * This function will return the 32 bit offset for a priv register if it is * present in the PM context buffer. */ static int gr_gk20a_find_priv_offset_in_pm_buffer(struct gk20a *g, u32 addr, u32 *priv_offset) { struct gr_gk20a *gr = &g->gr; int err = 0; u32 count; struct ctxsw_buf_offset_map_entry *map, *result, map_key; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "addr=0x%x", addr); /* Create map of pri address and pm offset if necessary */ if (gr->ctx_vars.hwpm_ctxsw_buffer_offset_map == NULL) { err = gr_gk20a_create_hwpm_ctxsw_buffer_offset_map(g); if (err != 0) { return err; } } *priv_offset = 0; map = gr->ctx_vars.hwpm_ctxsw_buffer_offset_map; count = gr->ctx_vars.hwpm_ctxsw_buffer_offset_map_count; map_key.addr = addr; result = bsearch(&map_key, map, count, sizeof(*map), map_cmp); if (result) { *priv_offset = result->offset; } else { nvgpu_err(g, "Lookup failed for address 0x%x", addr); err = -EINVAL; } return err; } bool gk20a_is_channel_ctx_resident(struct channel_gk20a *ch) { int curr_gr_ctx; u32 curr_gr_tsgid; struct gk20a *g = ch->g; struct channel_gk20a *curr_ch; bool ret = false; struct tsg_gk20a *tsg; curr_gr_ctx = gk20a_readl(g, gr_fecs_current_ctx_r()); /* when contexts are unloaded from GR, the valid bit is reset * but the instance pointer information remains intact. So the * valid bit must be checked to be absolutely certain that a * valid context is currently resident. */ if (gr_fecs_current_ctx_valid_v(curr_gr_ctx) == 0U) { return NULL; } curr_ch = gk20a_gr_get_channel_from_ctx(g, curr_gr_ctx, &curr_gr_tsgid); nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "curr_gr_chid=%d curr_tsgid=%d, ch->tsgid=%d" " ch->chid=%d", (curr_ch != NULL) ? curr_ch->chid : U32_MAX, curr_gr_tsgid, ch->tsgid, ch->chid); if (curr_ch == NULL) { return false; } if (ch->chid == curr_ch->chid) { ret = true; } tsg = tsg_gk20a_from_ch(ch); if ((tsg != NULL) && (tsg->tsgid == curr_gr_tsgid)) { ret = true; } gk20a_channel_put(curr_ch); return ret; } int __gr_gk20a_exec_ctx_ops(struct channel_gk20a *ch, struct nvgpu_dbg_reg_op *ctx_ops, u32 num_ops, u32 num_ctx_wr_ops, u32 num_ctx_rd_ops, bool ch_is_curr_ctx) { struct gk20a *g = ch->g; struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx; bool gr_ctx_ready = false; bool pm_ctx_ready = false; struct nvgpu_mem *current_mem = NULL; u32 i, j, offset, v; struct gr_gk20a *gr = &g->gr; u32 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); u32 max_offsets = gr->max_gpc_count * gr->max_tpc_per_gpc_count * sm_per_tpc; u32 *offsets = NULL; u32 *offset_addrs = NULL; u32 ctx_op_nr, num_ctx_ops[2] = {num_ctx_wr_ops, num_ctx_rd_ops}; int err = 0, pass; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "wr_ops=%d rd_ops=%d", num_ctx_wr_ops, num_ctx_rd_ops); tsg = tsg_gk20a_from_ch(ch); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; if (ch_is_curr_ctx) { for (pass = 0; pass < 2; pass++) { ctx_op_nr = 0; for (i = 0; (ctx_op_nr < num_ctx_ops[pass]) && (i < num_ops); ++i) { /* only do ctx ops and only on the right pass */ if ((ctx_ops[i].type == REGOP(TYPE_GLOBAL)) || (((pass == 0) && reg_op_is_read(ctx_ops[i].op)) || ((pass == 1) && !reg_op_is_read(ctx_ops[i].op)))) { continue; } /* if this is a quad access, setup for special access*/ if ((ctx_ops[i].type == REGOP(TYPE_GR_CTX_QUAD)) && (g->ops.gr.access_smpc_reg != NULL)) { g->ops.gr.access_smpc_reg(g, ctx_ops[i].quad, ctx_ops[i].offset); } offset = ctx_ops[i].offset; if (pass == 0) { /* write pass */ v = gk20a_readl(g, offset); v &= ~ctx_ops[i].and_n_mask_lo; v |= ctx_ops[i].value_lo; gk20a_writel(g, offset, v); nvgpu_log(g, gpu_dbg_gpu_dbg, "direct wr: offset=0x%x v=0x%x", offset, v); if (ctx_ops[i].op == REGOP(WRITE_64)) { v = gk20a_readl(g, offset + 4); v &= ~ctx_ops[i].and_n_mask_hi; v |= ctx_ops[i].value_hi; gk20a_writel(g, offset + 4, v); nvgpu_log(g, gpu_dbg_gpu_dbg, "direct wr: offset=0x%x v=0x%x", offset + 4, v); } } else { /* read pass */ ctx_ops[i].value_lo = gk20a_readl(g, offset); nvgpu_log(g, gpu_dbg_gpu_dbg, "direct rd: offset=0x%x v=0x%x", offset, ctx_ops[i].value_lo); if (ctx_ops[i].op == REGOP(READ_64)) { ctx_ops[i].value_hi = gk20a_readl(g, offset + 4); nvgpu_log(g, gpu_dbg_gpu_dbg, "direct rd: offset=0x%x v=0x%x", offset, ctx_ops[i].value_lo); } else { ctx_ops[i].value_hi = 0; } } ctx_op_nr++; } } goto cleanup; } /* they're the same size, so just use one alloc for both */ offsets = nvgpu_kzalloc(g, 2 * sizeof(u32) * max_offsets); if (offsets == NULL) { err = -ENOMEM; goto cleanup; } offset_addrs = offsets + max_offsets; err = gr_gk20a_ctx_patch_write_begin(g, gr_ctx, false); if (err != 0) { goto cleanup; } g->ops.mm.l2_flush(g, true); /* write to appropriate place in context image, * first have to figure out where that really is */ /* first pass is writes, second reads */ for (pass = 0; pass < 2; pass++) { ctx_op_nr = 0; for (i = 0; (ctx_op_nr < num_ctx_ops[pass]) && (i < num_ops); ++i) { u32 num_offsets; /* only do ctx ops and only on the right pass */ if ((ctx_ops[i].type == REGOP(TYPE_GLOBAL)) || (((pass == 0) && reg_op_is_read(ctx_ops[i].op)) || ((pass == 1) && !reg_op_is_read(ctx_ops[i].op)))) { continue; } err = gr_gk20a_get_ctx_buffer_offsets(g, ctx_ops[i].offset, max_offsets, offsets, offset_addrs, &num_offsets, ctx_ops[i].type == REGOP(TYPE_GR_CTX_QUAD), ctx_ops[i].quad); if (err == 0) { if (!gr_ctx_ready) { gr_ctx_ready = true; } current_mem = &gr_ctx->mem; } else { err = gr_gk20a_get_pm_ctx_buffer_offsets(g, ctx_ops[i].offset, max_offsets, offsets, offset_addrs, &num_offsets); if (err != 0) { nvgpu_log(g, gpu_dbg_gpu_dbg, "ctx op invalid offset: offset=0x%x", ctx_ops[i].offset); ctx_ops[i].status = REGOP(STATUS_INVALID_OFFSET); continue; } if (!pm_ctx_ready) { /* Make sure ctx buffer was initialized */ if (!nvgpu_mem_is_valid(&gr_ctx->pm_ctx.mem)) { nvgpu_err(g, "Invalid ctx buffer"); err = -EINVAL; goto cleanup; } pm_ctx_ready = true; } current_mem = &gr_ctx->pm_ctx.mem; } /* if this is a quad access, setup for special access*/ if ((ctx_ops[i].type == REGOP(TYPE_GR_CTX_QUAD)) && (g->ops.gr.access_smpc_reg != NULL)) { g->ops.gr.access_smpc_reg(g, ctx_ops[i].quad, ctx_ops[i].offset); } for (j = 0; j < num_offsets; j++) { /* sanity check gr ctxt offsets, * don't write outside, worst case */ if ((current_mem == &gr_ctx->mem) && (offsets[j] >= g->gr.ctx_vars.golden_image_size)) { continue; } if (pass == 0) { /* write pass */ v = nvgpu_mem_rd(g, current_mem, offsets[j]); v &= ~ctx_ops[i].and_n_mask_lo; v |= ctx_ops[i].value_lo; nvgpu_mem_wr(g, current_mem, offsets[j], v); nvgpu_log(g, gpu_dbg_gpu_dbg, "context wr: offset=0x%x v=0x%x", offsets[j], v); if (ctx_ops[i].op == REGOP(WRITE_64)) { v = nvgpu_mem_rd(g, current_mem, offsets[j] + 4); v &= ~ctx_ops[i].and_n_mask_hi; v |= ctx_ops[i].value_hi; nvgpu_mem_wr(g, current_mem, offsets[j] + 4, v); nvgpu_log(g, gpu_dbg_gpu_dbg, "context wr: offset=0x%x v=0x%x", offsets[j] + 4, v); } /* check to see if we need to add a special WAR for some of the SMPC perf regs */ gr_gk20a_ctx_patch_smpc(g, ch, offset_addrs[j], v, current_mem); } else { /* read pass */ ctx_ops[i].value_lo = nvgpu_mem_rd(g, current_mem, offsets[0]); nvgpu_log(g, gpu_dbg_gpu_dbg, "context rd: offset=0x%x v=0x%x", offsets[0], ctx_ops[i].value_lo); if (ctx_ops[i].op == REGOP(READ_64)) { ctx_ops[i].value_hi = nvgpu_mem_rd(g, current_mem, offsets[0] + 4); nvgpu_log(g, gpu_dbg_gpu_dbg, "context rd: offset=0x%x v=0x%x", offsets[0] + 4, ctx_ops[i].value_hi); } else { ctx_ops[i].value_hi = 0; } } } ctx_op_nr++; } } cleanup: if (offsets) { nvgpu_kfree(g, offsets); } if (gr_ctx->patch_ctx.mem.cpu_va) { gr_gk20a_ctx_patch_write_end(g, gr_ctx, gr_ctx_ready); } return err; } int gr_gk20a_exec_ctx_ops(struct channel_gk20a *ch, struct nvgpu_dbg_reg_op *ctx_ops, u32 num_ops, u32 num_ctx_wr_ops, u32 num_ctx_rd_ops, bool *is_curr_ctx) { struct gk20a *g = ch->g; int err, tmp_err; bool ch_is_curr_ctx; /* disable channel switching. * at that point the hardware state can be inspected to * determine if the context we're interested in is current. */ err = gr_gk20a_disable_ctxsw(g); if (err != 0) { nvgpu_err(g, "unable to stop gr ctxsw"); /* this should probably be ctx-fatal... */ return err; } ch_is_curr_ctx = gk20a_is_channel_ctx_resident(ch); if (is_curr_ctx != NULL) { *is_curr_ctx = ch_is_curr_ctx; } nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "is curr ctx=%d", ch_is_curr_ctx); err = __gr_gk20a_exec_ctx_ops(ch, ctx_ops, num_ops, num_ctx_wr_ops, num_ctx_rd_ops, ch_is_curr_ctx); tmp_err = gr_gk20a_enable_ctxsw(g); if (tmp_err) { nvgpu_err(g, "unable to restart ctxsw!"); err = tmp_err; } return err; } void gr_gk20a_commit_global_pagepool(struct gk20a *g, struct nvgpu_gr_ctx *gr_ctx, u64 addr, u32 size, bool patch) { gr_gk20a_ctx_patch_write(g, gr_ctx, gr_scc_pagepool_base_r(), gr_scc_pagepool_base_addr_39_8_f(addr), patch); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_scc_pagepool_r(), gr_scc_pagepool_total_pages_f(size) | gr_scc_pagepool_valid_true_f(), patch); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_gcc_pagepool_base_r(), gr_gpcs_gcc_pagepool_base_addr_39_8_f(addr), patch); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_gcc_pagepool_r(), gr_gpcs_gcc_pagepool_total_pages_f(size), patch); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_pd_pagepool_r(), gr_pd_pagepool_total_pages_f(size) | gr_pd_pagepool_valid_true_f(), patch); } void gk20a_init_gr(struct gk20a *g) { nvgpu_cond_init(&g->gr.init_wq); } int gk20a_gr_wait_for_sm_lock_down(struct gk20a *g, u32 gpc, u32 tpc, u32 sm, u32 global_esr_mask, bool check_errors) { bool locked_down; bool no_error_pending; u32 delay = GR_IDLE_CHECK_DEFAULT; bool mmu_debug_mode_enabled = g->ops.fb.is_debug_mode_enabled(g); u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc); u32 dbgr_status0 = 0, dbgr_control0 = 0; u64 warps_valid = 0, warps_paused = 0, warps_trapped = 0; struct nvgpu_timeout timeout; u32 warp_esr; nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "GPC%d TPC%d SM%d: locking down SM", gpc, tpc, sm); nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), NVGPU_TIMER_CPU_TIMER); /* wait for the sm to lock down */ do { u32 global_esr = g->ops.gr.get_sm_hww_global_esr(g, gpc, tpc, sm); dbgr_status0 = gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_status0_r() + offset); warp_esr = g->ops.gr.get_sm_hww_warp_esr(g, gpc, tpc, sm); locked_down = (gr_gpc0_tpc0_sm_dbgr_status0_locked_down_v(dbgr_status0) == gr_gpc0_tpc0_sm_dbgr_status0_locked_down_true_v()); no_error_pending = check_errors && (gr_gpc0_tpc0_sm_hww_warp_esr_error_v(warp_esr) == gr_gpc0_tpc0_sm_hww_warp_esr_error_none_v()) && ((global_esr & ~global_esr_mask) == 0); if (locked_down || no_error_pending) { nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "GPC%d TPC%d SM%d: locked down SM", gpc, tpc, sm); return 0; } /* if an mmu fault is pending and mmu debug mode is not * enabled, the sm will never lock down. */ if (!mmu_debug_mode_enabled && (g->ops.mm.mmu_fault_pending(g))) { nvgpu_err(g, "GPC%d TPC%d: mmu fault pending," " SM%d will never lock down!", gpc, tpc, sm); return -EFAULT; } nvgpu_usleep_range(delay, delay * 2); delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); } while (nvgpu_timeout_expired(&timeout) == 0); dbgr_control0 = gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_control0_r() + offset); /* 64 bit read */ warps_valid = (u64)gk20a_readl(g, gr_gpc0_tpc0_sm_warp_valid_mask_1_r() + offset) << 32; warps_valid |= gk20a_readl(g, gr_gpc0_tpc0_sm_warp_valid_mask_r() + offset); /* 64 bit read */ warps_paused = (u64)gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_bpt_pause_mask_1_r() + offset) << 32; warps_paused |= gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_bpt_pause_mask_r() + offset); /* 64 bit read */ warps_trapped = (u64)gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_bpt_trap_mask_1_r() + offset) << 32; warps_trapped |= gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_bpt_trap_mask_r() + offset); nvgpu_err(g, "GPC%d TPC%d: timed out while trying to lock down SM", gpc, tpc); nvgpu_err(g, "STATUS0(0x%x)=0x%x CONTROL0=0x%x VALID_MASK=0x%llx PAUSE_MASK=0x%llx TRAP_MASK=0x%llx", gr_gpc0_tpc0_sm_dbgr_status0_r() + offset, dbgr_status0, dbgr_control0, warps_valid, warps_paused, warps_trapped); return -ETIMEDOUT; } void gk20a_gr_suspend_single_sm(struct gk20a *g, u32 gpc, u32 tpc, u32 sm, u32 global_esr_mask, bool check_errors) { int err; u32 dbgr_control0; u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc); /* if an SM debugger isn't attached, skip suspend */ if (!g->ops.gr.sm_debugger_attached(g)) { nvgpu_err(g, "SM debugger not attached, skipping suspend!"); return; } nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "suspending gpc:%d, tpc:%d, sm%d", gpc, tpc, sm); /* assert stop trigger. */ dbgr_control0 = gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_control0_r() + offset); dbgr_control0 |= gr_gpcs_tpcs_sm_dbgr_control0_stop_trigger_enable_f(); gk20a_writel(g, gr_gpc0_tpc0_sm_dbgr_control0_r() + offset, dbgr_control0); err = g->ops.gr.wait_for_sm_lock_down(g, gpc, tpc, sm, global_esr_mask, check_errors); if (err != 0) { nvgpu_err(g, "SuspendSm failed"); return; } } void gk20a_gr_suspend_all_sms(struct gk20a *g, u32 global_esr_mask, bool check_errors) { struct gr_gk20a *gr = &g->gr; u32 gpc, tpc, sm; int err; u32 dbgr_control0; u32 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); /* if an SM debugger isn't attached, skip suspend */ if (!g->ops.gr.sm_debugger_attached(g)) { nvgpu_err(g, "SM debugger not attached, skipping suspend!"); return; } nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "suspending all sms"); /* assert stop trigger. uniformity assumption: all SMs will have * the same state in dbg_control0. */ dbgr_control0 = gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_control0_r()); dbgr_control0 |= gr_gpcs_tpcs_sm_dbgr_control0_stop_trigger_enable_f(); /* broadcast write */ gk20a_writel(g, gr_gpcs_tpcs_sm_dbgr_control0_r(), dbgr_control0); for (gpc = 0; gpc < gr->gpc_count; gpc++) { for (tpc = 0; tpc < gr_gk20a_get_tpc_count(gr, gpc); tpc++) { for (sm = 0; sm < sm_per_tpc; sm++) { err = g->ops.gr.wait_for_sm_lock_down(g, gpc, tpc, sm, global_esr_mask, check_errors); if (err != 0) { nvgpu_err(g, "SuspendAllSms failed"); return; } } } } } void gk20a_gr_resume_single_sm(struct gk20a *g, u32 gpc, u32 tpc, u32 sm) { u32 dbgr_control0; u32 offset; /* * The following requires some clarification. Despite the fact that both * RUN_TRIGGER and STOP_TRIGGER have the word "TRIGGER" in their * names, only one is actually a trigger, and that is the STOP_TRIGGER. * Merely writing a 1(_TASK) to the RUN_TRIGGER is not sufficient to * resume the gpu - the _STOP_TRIGGER must explicitly be set to 0 * (_DISABLE) as well. * Advice from the arch group: Disable the stop trigger first, as a * separate operation, in order to ensure that the trigger has taken * effect, before enabling the run trigger. */ offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc); /*De-assert stop trigger */ dbgr_control0 = gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_control0_r() + offset); dbgr_control0 = set_field(dbgr_control0, gr_gpcs_tpcs_sm_dbgr_control0_stop_trigger_m(), gr_gpcs_tpcs_sm_dbgr_control0_stop_trigger_disable_f()); gk20a_writel(g, gr_gpc0_tpc0_sm_dbgr_control0_r() + offset, dbgr_control0); /* Run trigger */ dbgr_control0 |= gr_gpcs_tpcs_sm_dbgr_control0_run_trigger_task_f(); gk20a_writel(g, gr_gpc0_tpc0_sm_dbgr_control0_r() + offset, dbgr_control0); } void gk20a_gr_resume_all_sms(struct gk20a *g) { u32 dbgr_control0; /* * The following requires some clarification. Despite the fact that both * RUN_TRIGGER and STOP_TRIGGER have the word "TRIGGER" in their * names, only one is actually a trigger, and that is the STOP_TRIGGER. * Merely writing a 1(_TASK) to the RUN_TRIGGER is not sufficient to * resume the gpu - the _STOP_TRIGGER must explicitly be set to 0 * (_DISABLE) as well. * Advice from the arch group: Disable the stop trigger first, as a * separate operation, in order to ensure that the trigger has taken * effect, before enabling the run trigger. */ /*De-assert stop trigger */ dbgr_control0 = gk20a_readl(g, gr_gpcs_tpcs_sm_dbgr_control0_r()); dbgr_control0 &= ~gr_gpcs_tpcs_sm_dbgr_control0_stop_trigger_enable_f(); gk20a_writel(g, gr_gpcs_tpcs_sm_dbgr_control0_r(), dbgr_control0); /* Run trigger */ dbgr_control0 |= gr_gpcs_tpcs_sm_dbgr_control0_run_trigger_task_f(); gk20a_writel(g, gr_gpcs_tpcs_sm_dbgr_control0_r(), dbgr_control0); } int gr_gk20a_set_sm_debug_mode(struct gk20a *g, struct channel_gk20a *ch, u64 sms, bool enable) { struct nvgpu_dbg_reg_op *ops; unsigned int i = 0, sm_id; int err; u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); ops = nvgpu_kcalloc(g, g->gr.no_of_sm, sizeof(*ops)); if (ops == NULL) { return -ENOMEM; } for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) { int gpc, tpc; u32 tpc_offset, gpc_offset, reg_offset, reg_mask, reg_val; if ((sms & BIT64(sm_id)) == 0ULL) { continue; } gpc = g->gr.sm_to_cluster[sm_id].gpc_index; tpc = g->gr.sm_to_cluster[sm_id].tpc_index; tpc_offset = tpc_in_gpc_stride * tpc; gpc_offset = gpc_stride * gpc; reg_offset = tpc_offset + gpc_offset; ops[i].op = REGOP(WRITE_32); ops[i].type = REGOP(TYPE_GR_CTX); ops[i].offset = gr_gpc0_tpc0_sm_dbgr_control0_r() + reg_offset; reg_mask = 0; reg_val = 0; if (enable) { reg_mask |= gr_gpc0_tpc0_sm_dbgr_control0_debugger_mode_m(); reg_val |= gr_gpc0_tpc0_sm_dbgr_control0_debugger_mode_on_f(); reg_mask |= gr_gpc0_tpc0_sm_dbgr_control0_stop_on_any_warp_m(); reg_val |= gr_gpc0_tpc0_sm_dbgr_control0_stop_on_any_warp_disable_f(); reg_mask |= gr_gpc0_tpc0_sm_dbgr_control0_stop_on_any_sm_m(); reg_val |= gr_gpc0_tpc0_sm_dbgr_control0_stop_on_any_sm_disable_f(); } else { reg_mask |= gr_gpc0_tpc0_sm_dbgr_control0_debugger_mode_m(); reg_val |= gr_gpc0_tpc0_sm_dbgr_control0_debugger_mode_off_f(); } ops[i].and_n_mask_lo = reg_mask; ops[i].value_lo = reg_val; i++; } err = gr_gk20a_exec_ctx_ops(ch, ops, i, i, 0, NULL); if (err != 0) { nvgpu_err(g, "Failed to access register"); } nvgpu_kfree(g, ops); return err; } /* * gr_gk20a_suspend_context() * This API should be called with dbg_session lock held * and ctxsw disabled * Returns bool value indicating if context was resident * or not */ bool gr_gk20a_suspend_context(struct channel_gk20a *ch) { struct gk20a *g = ch->g; bool ctx_resident = false; if (gk20a_is_channel_ctx_resident(ch)) { g->ops.gr.suspend_all_sms(g, 0, false); ctx_resident = true; } else { gk20a_disable_channel_tsg(g, ch); } return ctx_resident; } bool gr_gk20a_resume_context(struct channel_gk20a *ch) { struct gk20a *g = ch->g; bool ctx_resident = false; if (gk20a_is_channel_ctx_resident(ch)) { g->ops.gr.resume_all_sms(g); ctx_resident = true; } else { gk20a_enable_channel_tsg(g, ch); } return ctx_resident; } int gr_gk20a_suspend_contexts(struct gk20a *g, struct dbg_session_gk20a *dbg_s, int *ctx_resident_ch_fd) { int local_ctx_resident_ch_fd = -1; bool ctx_resident; struct channel_gk20a *ch; struct dbg_session_channel_data *ch_data; int err = 0; nvgpu_mutex_acquire(&g->dbg_sessions_lock); err = gr_gk20a_disable_ctxsw(g); if (err != 0) { nvgpu_err(g, "unable to stop gr ctxsw"); goto clean_up; } nvgpu_mutex_acquire(&dbg_s->ch_list_lock); nvgpu_list_for_each_entry(ch_data, &dbg_s->ch_list, dbg_session_channel_data, ch_entry) { ch = g->fifo.channel + ch_data->chid; ctx_resident = gr_gk20a_suspend_context(ch); if (ctx_resident) { local_ctx_resident_ch_fd = ch_data->channel_fd; } } nvgpu_mutex_release(&dbg_s->ch_list_lock); err = gr_gk20a_enable_ctxsw(g); if (err != 0) { nvgpu_err(g, "unable to restart ctxsw!"); } *ctx_resident_ch_fd = local_ctx_resident_ch_fd; clean_up: nvgpu_mutex_release(&g->dbg_sessions_lock); return err; } int gr_gk20a_resume_contexts(struct gk20a *g, struct dbg_session_gk20a *dbg_s, int *ctx_resident_ch_fd) { int local_ctx_resident_ch_fd = -1; bool ctx_resident; struct channel_gk20a *ch; int err = 0; struct dbg_session_channel_data *ch_data; nvgpu_mutex_acquire(&g->dbg_sessions_lock); err = gr_gk20a_disable_ctxsw(g); if (err != 0) { nvgpu_err(g, "unable to stop gr ctxsw"); goto clean_up; } nvgpu_list_for_each_entry(ch_data, &dbg_s->ch_list, dbg_session_channel_data, ch_entry) { ch = g->fifo.channel + ch_data->chid; ctx_resident = gr_gk20a_resume_context(ch); if (ctx_resident) { local_ctx_resident_ch_fd = ch_data->channel_fd; } } err = gr_gk20a_enable_ctxsw(g); if (err != 0) { nvgpu_err(g, "unable to restart ctxsw!"); } *ctx_resident_ch_fd = local_ctx_resident_ch_fd; clean_up: nvgpu_mutex_release(&g->dbg_sessions_lock); return err; } int gr_gk20a_trigger_suspend(struct gk20a *g) { int err = 0; u32 dbgr_control0; /* assert stop trigger. uniformity assumption: all SMs will have * the same state in dbg_control0. */ dbgr_control0 = gk20a_readl(g, gr_gpc0_tpc0_sm_dbgr_control0_r()); dbgr_control0 |= gr_gpcs_tpcs_sm_dbgr_control0_stop_trigger_enable_f(); /* broadcast write */ gk20a_writel(g, gr_gpcs_tpcs_sm_dbgr_control0_r(), dbgr_control0); return err; } int gr_gk20a_wait_for_pause(struct gk20a *g, struct nvgpu_warpstate *w_state) { int err = 0; struct gr_gk20a *gr = &g->gr; u32 gpc, tpc, sm, sm_id; u32 global_mask; /* Wait for the SMs to reach full stop. This condition is: * 1) All SMs with valid warps must be in the trap handler (SM_IN_TRAP_MODE) * 2) All SMs in the trap handler must have equivalent VALID and PAUSED warp * masks. */ global_mask = g->ops.gr.get_sm_no_lock_down_hww_global_esr_mask(g); /* Lock down all SMs */ for (sm_id = 0; sm_id < gr->no_of_sm; sm_id++) { gpc = g->gr.sm_to_cluster[sm_id].gpc_index; tpc = g->gr.sm_to_cluster[sm_id].tpc_index; sm = g->gr.sm_to_cluster[sm_id].sm_index; err = g->ops.gr.lock_down_sm(g, gpc, tpc, sm, global_mask, false); if (err != 0) { nvgpu_err(g, "sm did not lock down!"); return err; } } /* Read the warp status */ g->ops.gr.bpt_reg_info(g, w_state); return 0; } int gr_gk20a_resume_from_pause(struct gk20a *g) { int err = 0; u32 reg_val; /* Clear the pause mask to tell the GPU we want to resume everyone */ gk20a_writel(g, gr_gpcs_tpcs_sm_dbgr_bpt_pause_mask_r(), 0); /* explicitly re-enable forwarding of SM interrupts upon any resume */ reg_val = gk20a_readl(g, gr_gpc0_tpc0_tpccs_tpc_exception_en_r()); reg_val |= gr_gpc0_tpc0_tpccs_tpc_exception_en_sm_enabled_f(); gk20a_writel(g, gr_gpcs_tpcs_tpccs_tpc_exception_en_r(), reg_val); /* Now resume all sms, write a 0 to the stop trigger * then a 1 to the run trigger */ g->ops.gr.resume_all_sms(g); return err; } int gr_gk20a_clear_sm_errors(struct gk20a *g) { int ret = 0; u32 gpc, tpc, sm; struct gr_gk20a *gr = &g->gr; u32 global_esr; u32 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); for (gpc = 0; gpc < gr->gpc_count; gpc++) { /* check if any tpc has an exception */ for (tpc = 0; tpc < gr->gpc_tpc_count[gpc]; tpc++) { for (sm = 0; sm < sm_per_tpc; sm++) { global_esr = g->ops.gr.get_sm_hww_global_esr(g, gpc, tpc, sm); /* clearing hwws, also causes tpc and gpc * exceptions to be cleared */ g->ops.gr.clear_sm_hww(g, gpc, tpc, sm, global_esr); } } } return ret; } u32 gr_gk20a_tpc_enabled_exceptions(struct gk20a *g) { struct gr_gk20a *gr = &g->gr; u32 sm_id, tpc_exception_en = 0; u32 offset, regval, tpc_offset, gpc_offset; u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); for (sm_id = 0; sm_id < gr->no_of_sm; sm_id++) { tpc_offset = tpc_in_gpc_stride * g->gr.sm_to_cluster[sm_id].tpc_index; gpc_offset = gpc_stride * g->gr.sm_to_cluster[sm_id].gpc_index; offset = tpc_offset + gpc_offset; regval = gk20a_readl(g, gr_gpc0_tpc0_tpccs_tpc_exception_en_r() + offset); /* Each bit represents corresponding enablement state, bit 0 corrsponds to SM0 */ tpc_exception_en |= gr_gpc0_tpc0_tpccs_tpc_exception_en_sm_v(regval) << sm_id; } return tpc_exception_en; } u32 gk20a_gr_get_sm_hww_warp_esr(struct gk20a *g, u32 gpc, u32 tpc, u32 sm) { u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc); u32 hww_warp_esr = gk20a_readl(g, gr_gpc0_tpc0_sm_hww_warp_esr_r() + offset); return hww_warp_esr; } u32 gk20a_gr_get_sm_hww_global_esr(struct gk20a *g, u32 gpc, u32 tpc, u32 sm) { u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc); u32 hww_global_esr = gk20a_readl(g, gr_gpc0_tpc0_sm_hww_global_esr_r() + offset); return hww_global_esr; } u32 gk20a_gr_get_sm_no_lock_down_hww_global_esr_mask(struct gk20a *g) { /* * These three interrupts don't require locking down the SM. They can * be handled by usermode clients as they aren't fatal. Additionally, * usermode clients may wish to allow some warps to execute while others * are at breakpoints, as opposed to fatal errors where all warps should * halt. */ u32 global_esr_mask = gr_gpc0_tpc0_sm_hww_global_esr_bpt_int_pending_f() | gr_gpc0_tpc0_sm_hww_global_esr_bpt_pause_pending_f() | gr_gpc0_tpc0_sm_hww_global_esr_single_step_complete_pending_f(); return global_esr_mask; } /* invalidate channel lookup tlb */ void gk20a_gr_flush_channel_tlb(struct gr_gk20a *gr) { nvgpu_spinlock_acquire(&gr->ch_tlb_lock); memset(gr->chid_tlb, 0, sizeof(struct gr_channel_map_tlb_entry) * GR_CHANNEL_MAP_TLB_SIZE); nvgpu_spinlock_release(&gr->ch_tlb_lock); }