/* * GK20A Graphics * * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gk20a.h" #include "gr_gk20a.h" #include "gk20a/fecs_trace_gk20a.h" #include "gr_ctx_gk20a.h" #include "gr_pri_gk20a.h" #include "regops_gk20a.h" #include "dbg_gpu_gk20a.h" #include #include #include #include #include #include #include #include #include #include #define BLK_SIZE (256) #define NV_PERF_PMM_FBP_ROUTER_STRIDE 0x0200 #define NV_PERF_PMMGPCROUTER_STRIDE 0x0200 #define NV_PCFG_BASE 0x00088000 #define NV_XBAR_MXBAR_PRI_GPC_GNIC_STRIDE 0x0020 #define FE_PWR_MODE_TIMEOUT_MAX 2000 #define FE_PWR_MODE_TIMEOUT_DEFAULT 10 #define CTXSW_MEM_SCRUBBING_TIMEOUT_MAX 1000 #define CTXSW_MEM_SCRUBBING_TIMEOUT_DEFAULT 10 #define FECS_ARB_CMD_TIMEOUT_MAX 40 #define FECS_ARB_CMD_TIMEOUT_DEFAULT 2 static int gk20a_init_gr_bind_fecs_elpg(struct gk20a *g); static void gr_gk20a_free_channel_pm_ctx(struct gk20a *g, struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx); /* channel patch ctx buffer */ static int gr_gk20a_alloc_channel_patch_ctx(struct gk20a *g, struct channel_gk20a *c); static void gr_gk20a_free_channel_patch_ctx(struct gk20a *g, struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx); /* golden ctx image */ static int gr_gk20a_init_golden_ctx_image(struct gk20a *g, struct channel_gk20a *c); int gr_gk20a_get_ctx_id(struct gk20a *g, struct channel_gk20a *c, u32 *ctx_id) { struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx = NULL; struct nvgpu_mem *mem = NULL; tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; mem = &gr_ctx->mem; /* Channel gr_ctx buffer is gpu cacheable. Flush and invalidate before cpu update. */ g->ops.mm.l2_flush(g, true); *ctx_id = nvgpu_mem_rd(g, mem, ctxsw_prog_main_image_context_id_o()); nvgpu_log(g, gpu_dbg_fn | gpu_dbg_intr, "ctx_id: 0x%x", *ctx_id); return 0; } void gk20a_gpccs_dump_falcon_stats(struct gk20a *g) { unsigned int i; nvgpu_err(g, "gr_gpc0_gpccs_falcon_irqstat : %d", gk20a_readl(g, gr_gpc0_gpccs_falcon_irqstat_r())); nvgpu_err(g, "gr_gpc0_gpccs_falcon_irqmode : %d", gk20a_readl(g, gr_gpc0_gpccs_falcon_irqmode_r())); nvgpu_err(g, "gr_gpc0_gpccs_falcon_irqmask : %d", gk20a_readl(g, gr_gpc0_gpccs_falcon_irqmask_r())); nvgpu_err(g, "gr_gpc0_gpccs_falcon_irqdest : %d", gk20a_readl(g, gr_gpc0_gpccs_falcon_irqdest_r())); nvgpu_err(g, "gr_gpc0_gpccs_falcon_debug1 : %d", gk20a_readl(g, gr_gpc0_gpccs_falcon_debug1_r())); nvgpu_err(g, "gr_gpc0_gpccs_falcon_debuginfo : %d", gk20a_readl(g, gr_gpc0_gpccs_falcon_debuginfo_r())); nvgpu_err(g, "gr_gpc0_gpccs_falcon_engctl : %d", gk20a_readl(g, gr_gpc0_gpccs_falcon_engctl_r())); nvgpu_err(g, "gr_gpc0_gpccs_falcon_curctx : %d", gk20a_readl(g, gr_gpc0_gpccs_falcon_curctx_r())); nvgpu_err(g, "gr_gpc0_gpccs_falcon_nxtctx : %d", gk20a_readl(g, gr_gpc0_gpccs_falcon_nxtctx_r())); nvgpu_err(g, "gr_gpc0_gpccs_ctxsw_status_1 : %d", gk20a_readl(g, gr_gpc0_gpccs_ctxsw_status_1_r())); for (i = 0; i < g->ops.gr.gpc0_gpccs_ctxsw_mailbox_size(); i++) { nvgpu_err(g, "gr_gpc0_gpccs_ctxsw_mailbox_r(%d) : 0x%x", i, gk20a_readl(g, gr_gpc0_gpccs_ctxsw_mailbox_r(i))); } gk20a_writel(g, gr_gpc0_gpccs_falcon_icd_cmd_r(), gr_gpc0_gpccs_falcon_icd_cmd_opc_rreg_f() | gr_gpc0_gpccs_falcon_icd_cmd_idx_f(PMU_FALCON_REG_IMB)); nvgpu_err(g, "GPC0_GPCCS_FALCON_REG_IMB : 0x%x", gk20a_readl(g, gr_gpc_gpccs_falcon_icd_rdata_r())); gk20a_writel(g, gr_gpc0_gpccs_falcon_icd_cmd_r(), gr_gpc0_gpccs_falcon_icd_cmd_opc_rreg_f() | gr_gpc0_gpccs_falcon_icd_cmd_idx_f(PMU_FALCON_REG_DMB)); nvgpu_err(g, "GPC0_GPCCS_FALCON_REG_DMB : 0x%x", gk20a_readl(g, gr_gpc_gpccs_falcon_icd_rdata_r())); gk20a_writel(g, gr_gpc0_gpccs_falcon_icd_cmd_r(), gr_gpc0_gpccs_falcon_icd_cmd_opc_rreg_f() | gr_gpc0_gpccs_falcon_icd_cmd_idx_f(PMU_FALCON_REG_CSW)); nvgpu_err(g, "GPC0_GPCCS_FALCON_REG_CSW : 0x%x", gk20a_readl(g, gr_gpc_gpccs_falcon_icd_rdata_r())); gk20a_writel(g, gr_gpc0_gpccs_falcon_icd_cmd_r(), gr_gpc0_gpccs_falcon_icd_cmd_opc_rreg_f() | gr_gpc0_gpccs_falcon_icd_cmd_idx_f(PMU_FALCON_REG_CTX)); nvgpu_err(g, "GPC0_GPCCS_FALCON_REG_CTX : 0x%x", gk20a_readl(g, gr_gpc_gpccs_falcon_icd_rdata_r())); gk20a_writel(g, gr_gpc0_gpccs_falcon_icd_cmd_r(), gr_gpc0_gpccs_falcon_icd_cmd_opc_rreg_f() | gr_gpc0_gpccs_falcon_icd_cmd_idx_f(PMU_FALCON_REG_EXCI)); nvgpu_err(g, "GPC0_GPCCS_FALCON_REG_EXCI : 0x%x", gk20a_readl(g, gr_gpc_gpccs_falcon_icd_rdata_r())); for (i = 0; i < 4U; i++) { gk20a_writel(g, gr_gpc0_gpccs_falcon_icd_cmd_r(), gr_gpc0_gpccs_falcon_icd_cmd_opc_rreg_f() | gr_gpc0_gpccs_falcon_icd_cmd_idx_f(PMU_FALCON_REG_PC)); nvgpu_err(g, "GPC0_GPCCS_FALCON_REG_PC : 0x%x", gk20a_readl(g, gr_gpc_gpccs_falcon_icd_rdata_r())); gk20a_writel(g, gr_gpc0_gpccs_falcon_icd_cmd_r(), gr_gpc0_gpccs_falcon_icd_cmd_opc_rreg_f() | gr_gpc0_gpccs_falcon_icd_cmd_idx_f(PMU_FALCON_REG_SP)); nvgpu_err(g, "GPC0_GPCCS_FALCON_REG_SP : 0x%x", gk20a_readl(g, gr_gpc_gpccs_falcon_icd_rdata_r())); } } void gk20a_fecs_dump_falcon_stats(struct gk20a *g) { unsigned int i; nvgpu_err(g, "gr_fecs_os_r : %d", gk20a_readl(g, gr_fecs_os_r())); nvgpu_err(g, "gr_fecs_cpuctl_r : 0x%x", gk20a_readl(g, gr_fecs_cpuctl_r())); nvgpu_err(g, "gr_fecs_idlestate_r : 0x%x", gk20a_readl(g, gr_fecs_idlestate_r())); nvgpu_err(g, "gr_fecs_mailbox0_r : 0x%x", gk20a_readl(g, gr_fecs_mailbox0_r())); nvgpu_err(g, "gr_fecs_mailbox1_r : 0x%x", gk20a_readl(g, gr_fecs_mailbox1_r())); nvgpu_err(g, "gr_fecs_irqstat_r : 0x%x", gk20a_readl(g, gr_fecs_irqstat_r())); nvgpu_err(g, "gr_fecs_irqmode_r : 0x%x", gk20a_readl(g, gr_fecs_irqmode_r())); nvgpu_err(g, "gr_fecs_irqmask_r : 0x%x", gk20a_readl(g, gr_fecs_irqmask_r())); nvgpu_err(g, "gr_fecs_irqdest_r : 0x%x", gk20a_readl(g, gr_fecs_irqdest_r())); nvgpu_err(g, "gr_fecs_debug1_r : 0x%x", gk20a_readl(g, gr_fecs_debug1_r())); nvgpu_err(g, "gr_fecs_debuginfo_r : 0x%x", gk20a_readl(g, gr_fecs_debuginfo_r())); nvgpu_err(g, "gr_fecs_ctxsw_status_1_r : 0x%x", gk20a_readl(g, gr_fecs_ctxsw_status_1_r())); for (i = 0; i < g->ops.gr.fecs_ctxsw_mailbox_size(); i++) { nvgpu_err(g, "gr_fecs_ctxsw_mailbox_r(%d) : 0x%x", i, gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(i))); } nvgpu_err(g, "gr_fecs_engctl_r : 0x%x", gk20a_readl(g, gr_fecs_engctl_r())); nvgpu_err(g, "gr_fecs_curctx_r : 0x%x", gk20a_readl(g, gr_fecs_curctx_r())); nvgpu_err(g, "gr_fecs_nxtctx_r : 0x%x", gk20a_readl(g, gr_fecs_nxtctx_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_IMB)); nvgpu_err(g, "FECS_FALCON_REG_IMB : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_DMB)); nvgpu_err(g, "FECS_FALCON_REG_DMB : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_CSW)); nvgpu_err(g, "FECS_FALCON_REG_CSW : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_CTX)); nvgpu_err(g, "FECS_FALCON_REG_CTX : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_EXCI)); nvgpu_err(g, "FECS_FALCON_REG_EXCI : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); for (i = 0; i < 4; i++) { gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_PC)); nvgpu_err(g, "FECS_FALCON_REG_PC : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); gk20a_writel(g, gr_fecs_icd_cmd_r(), gr_fecs_icd_cmd_opc_rreg_f() | gr_fecs_icd_cmd_idx_f(PMU_FALCON_REG_SP)); nvgpu_err(g, "FECS_FALCON_REG_SP : 0x%x", gk20a_readl(g, gr_fecs_icd_rdata_r())); } } static void gr_gk20a_load_falcon_dmem(struct gk20a *g) { u32 i, ucode_u32_size; const u32 *ucode_u32_data; u32 checksum; nvgpu_log_fn(g, " "); gk20a_writel(g, gr_gpccs_dmemc_r(0), (gr_gpccs_dmemc_offs_f(0) | gr_gpccs_dmemc_blk_f(0) | gr_gpccs_dmemc_aincw_f(1))); ucode_u32_size = g->gr.ctx_vars.ucode.gpccs.data.count; ucode_u32_data = (const u32 *)g->gr.ctx_vars.ucode.gpccs.data.l; for (i = 0, checksum = 0; i < ucode_u32_size; i++) { gk20a_writel(g, gr_gpccs_dmemd_r(0), ucode_u32_data[i]); checksum += ucode_u32_data[i]; } gk20a_writel(g, gr_fecs_dmemc_r(0), (gr_fecs_dmemc_offs_f(0) | gr_fecs_dmemc_blk_f(0) | gr_fecs_dmemc_aincw_f(1))); ucode_u32_size = g->gr.ctx_vars.ucode.fecs.data.count; ucode_u32_data = (const u32 *)g->gr.ctx_vars.ucode.fecs.data.l; for (i = 0, checksum = 0; i < ucode_u32_size; i++) { gk20a_writel(g, gr_fecs_dmemd_r(0), ucode_u32_data[i]); checksum += ucode_u32_data[i]; } nvgpu_log_fn(g, "done"); } static void gr_gk20a_load_falcon_imem(struct gk20a *g) { u32 cfg, fecs_imem_size, gpccs_imem_size, ucode_u32_size; const u32 *ucode_u32_data; u32 tag, i, pad_start, pad_end; u32 checksum; nvgpu_log_fn(g, " "); cfg = gk20a_readl(g, gr_fecs_cfg_r()); fecs_imem_size = gr_fecs_cfg_imem_sz_v(cfg); cfg = gk20a_readl(g, gr_gpc0_cfg_r()); gpccs_imem_size = gr_gpc0_cfg_imem_sz_v(cfg); /* Use the broadcast address to access all of the GPCCS units. */ gk20a_writel(g, gr_gpccs_imemc_r(0), (gr_gpccs_imemc_offs_f(0) | gr_gpccs_imemc_blk_f(0) | gr_gpccs_imemc_aincw_f(1))); /* Setup the tags for the instruction memory. */ tag = 0; gk20a_writel(g, gr_gpccs_imemt_r(0), gr_gpccs_imemt_tag_f(tag)); ucode_u32_size = g->gr.ctx_vars.ucode.gpccs.inst.count; ucode_u32_data = (const u32 *)g->gr.ctx_vars.ucode.gpccs.inst.l; for (i = 0, checksum = 0; i < ucode_u32_size; i++) { if ((i != 0U) && ((i % (256U/sizeof(u32))) == 0U)) { tag++; gk20a_writel(g, gr_gpccs_imemt_r(0), gr_gpccs_imemt_tag_f(tag)); } gk20a_writel(g, gr_gpccs_imemd_r(0), ucode_u32_data[i]); checksum += ucode_u32_data[i]; } pad_start = i * 4U; pad_end = pad_start + (256U - pad_start % 256U) + 256U; for (i = pad_start; (i < gpccs_imem_size * 256U) && (i < pad_end); i += 4U) { if ((i != 0U) && ((i % 256U) == 0U)) { tag++; gk20a_writel(g, gr_gpccs_imemt_r(0), gr_gpccs_imemt_tag_f(tag)); } gk20a_writel(g, gr_gpccs_imemd_r(0), 0); } gk20a_writel(g, gr_fecs_imemc_r(0), (gr_fecs_imemc_offs_f(0) | gr_fecs_imemc_blk_f(0) | gr_fecs_imemc_aincw_f(1))); /* Setup the tags for the instruction memory. */ tag = 0; gk20a_writel(g, gr_fecs_imemt_r(0), gr_fecs_imemt_tag_f(tag)); ucode_u32_size = g->gr.ctx_vars.ucode.fecs.inst.count; ucode_u32_data = (const u32 *)g->gr.ctx_vars.ucode.fecs.inst.l; for (i = 0, checksum = 0; i < ucode_u32_size; i++) { if ((i != 0U) && ((i % (256U/sizeof(u32))) == 0U)) { tag++; gk20a_writel(g, gr_fecs_imemt_r(0), gr_fecs_imemt_tag_f(tag)); } gk20a_writel(g, gr_fecs_imemd_r(0), ucode_u32_data[i]); checksum += ucode_u32_data[i]; } pad_start = i * 4U; pad_end = pad_start + (256U - pad_start % 256U) + 256U; for (i = pad_start; (i < fecs_imem_size * 256U) && i < pad_end; i += 4U) { if ((i != 0U) && ((i % 256U) == 0U)) { tag++; gk20a_writel(g, gr_fecs_imemt_r(0), gr_fecs_imemt_tag_f(tag)); } gk20a_writel(g, gr_fecs_imemd_r(0), 0); } } int gr_gk20a_wait_idle(struct gk20a *g, unsigned long duration_ms, u32 expect_delay) { u32 delay = expect_delay; bool ctxsw_active; bool gr_busy; u32 gr_engine_id; u32 engine_status; bool ctx_status_invalid; struct nvgpu_timeout timeout; nvgpu_log_fn(g, " "); gr_engine_id = gk20a_fifo_get_gr_engine_id(g); nvgpu_timeout_init(g, &timeout, duration_ms, NVGPU_TIMER_CPU_TIMER); do { /* fmodel: host gets fifo_engine_status(gr) from gr only when gr_status is read */ (void) gk20a_readl(g, gr_status_r()); engine_status = gk20a_readl(g, fifo_engine_status_r(gr_engine_id)); ctxsw_active = engine_status & fifo_engine_status_ctxsw_in_progress_f(); ctx_status_invalid = (fifo_engine_status_ctx_status_v(engine_status) == fifo_engine_status_ctx_status_invalid_v()); gr_busy = gk20a_readl(g, gr_engine_status_r()) & gr_engine_status_value_busy_f(); if (ctx_status_invalid || (!gr_busy && !ctxsw_active)) { nvgpu_log_fn(g, "done"); return 0; } nvgpu_usleep_range(delay, delay * 2); delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); } while (nvgpu_timeout_expired(&timeout) == 0); nvgpu_err(g, "timeout, ctxsw busy : %d, gr busy : %d", ctxsw_active, gr_busy); return -EAGAIN; } int gr_gk20a_wait_fe_idle(struct gk20a *g, unsigned long duration_ms, u32 expect_delay) { u32 val; u32 delay = expect_delay; struct nvgpu_timeout timeout; if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) { return 0; } nvgpu_log_fn(g, " "); nvgpu_timeout_init(g, &timeout, duration_ms, NVGPU_TIMER_CPU_TIMER); do { val = gk20a_readl(g, gr_status_r()); if (gr_status_fe_method_lower_v(val) == 0U) { nvgpu_log_fn(g, "done"); return 0; } nvgpu_usleep_range(delay, delay * 2); delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); } while (nvgpu_timeout_expired(&timeout) == 0); nvgpu_err(g, "timeout, fe busy : %x", val); return -EAGAIN; } int gr_gk20a_ctx_wait_ucode(struct gk20a *g, u32 mailbox_id, u32 *mailbox_ret, u32 opc_success, u32 mailbox_ok, u32 opc_fail, u32 mailbox_fail, bool sleepduringwait) { struct nvgpu_timeout timeout; u32 delay = GR_FECS_POLL_INTERVAL; u32 check = WAIT_UCODE_LOOP; u32 reg; nvgpu_log_fn(g, " "); if (sleepduringwait) { delay = GR_IDLE_CHECK_DEFAULT; } nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), NVGPU_TIMER_CPU_TIMER); while (check == WAIT_UCODE_LOOP) { if (nvgpu_timeout_expired(&timeout)) { check = WAIT_UCODE_TIMEOUT; } reg = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(mailbox_id)); if (mailbox_ret) { *mailbox_ret = reg; } switch (opc_success) { case GR_IS_UCODE_OP_EQUAL: if (reg == mailbox_ok) { check = WAIT_UCODE_OK; } break; case GR_IS_UCODE_OP_NOT_EQUAL: if (reg != mailbox_ok) { check = WAIT_UCODE_OK; } break; case GR_IS_UCODE_OP_AND: if (reg & mailbox_ok) { check = WAIT_UCODE_OK; } break; case GR_IS_UCODE_OP_LESSER: if (reg < mailbox_ok) { check = WAIT_UCODE_OK; } break; case GR_IS_UCODE_OP_LESSER_EQUAL: if (reg <= mailbox_ok) { check = WAIT_UCODE_OK; } break; case GR_IS_UCODE_OP_SKIP: /* do no success check */ break; default: nvgpu_err(g, "invalid success opcode 0x%x", opc_success); check = WAIT_UCODE_ERROR; break; } switch (opc_fail) { case GR_IS_UCODE_OP_EQUAL: if (reg == mailbox_fail) { check = WAIT_UCODE_ERROR; } break; case GR_IS_UCODE_OP_NOT_EQUAL: if (reg != mailbox_fail) { check = WAIT_UCODE_ERROR; } break; case GR_IS_UCODE_OP_AND: if (reg & mailbox_fail) { check = WAIT_UCODE_ERROR; } break; case GR_IS_UCODE_OP_LESSER: if (reg < mailbox_fail) { check = WAIT_UCODE_ERROR; } break; case GR_IS_UCODE_OP_LESSER_EQUAL: if (reg <= mailbox_fail) { check = WAIT_UCODE_ERROR; } break; case GR_IS_UCODE_OP_SKIP: /* do no check on fail*/ break; default: nvgpu_err(g, "invalid fail opcode 0x%x", opc_fail); check = WAIT_UCODE_ERROR; break; } if (sleepduringwait) { nvgpu_usleep_range(delay, delay * 2); delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); } else { nvgpu_udelay(delay); } } if (check == WAIT_UCODE_TIMEOUT) { nvgpu_err(g, "timeout waiting on mailbox=%d value=0x%08x", mailbox_id, reg); gk20a_fecs_dump_falcon_stats(g); gk20a_gpccs_dump_falcon_stats(g); gk20a_gr_debug_dump(g); return -1; } else if (check == WAIT_UCODE_ERROR) { nvgpu_err(g, "ucode method failed on mailbox=%d value=0x%08x", mailbox_id, reg); gk20a_fecs_dump_falcon_stats(g); gk20a_gpccs_dump_falcon_stats(g); return -1; } nvgpu_log_fn(g, "done"); return 0; } int gr_gk20a_submit_fecs_method_op_locked(struct gk20a *g, struct fecs_method_op_gk20a op, bool sleepduringwait) { int ret; if (op.mailbox.id != 0) { gk20a_writel(g, gr_fecs_ctxsw_mailbox_r(op.mailbox.id), op.mailbox.data); } gk20a_writel(g, gr_fecs_ctxsw_mailbox_clear_r(0), gr_fecs_ctxsw_mailbox_clear_value_f(op.mailbox.clr)); gk20a_writel(g, gr_fecs_method_data_r(), op.method.data); gk20a_writel(g, gr_fecs_method_push_r(), gr_fecs_method_push_adr_f(op.method.addr)); /* op.mailbox.id == 4 cases require waiting for completion on * for op.mailbox.id == 0 */ if (op.mailbox.id == 4) { op.mailbox.id = 0; } ret = gr_gk20a_ctx_wait_ucode(g, op.mailbox.id, op.mailbox.ret, op.cond.ok, op.mailbox.ok, op.cond.fail, op.mailbox.fail, sleepduringwait); if (ret) { nvgpu_err(g,"fecs method: data=0x%08x push adr=0x%08x", op.method.data, op.method.addr); } return ret; } /* The following is a less brittle way to call gr_gk20a_submit_fecs_method(...) * We should replace most, if not all, fecs method calls to this instead. */ int gr_gk20a_submit_fecs_method_op(struct gk20a *g, struct fecs_method_op_gk20a op, bool sleepduringwait) { struct gr_gk20a *gr = &g->gr; int ret; nvgpu_mutex_acquire(&gr->fecs_mutex); ret = gr_gk20a_submit_fecs_method_op_locked(g, op, sleepduringwait); nvgpu_mutex_release(&gr->fecs_mutex); return ret; } /* Sideband mailbox writes are done a bit differently */ int gr_gk20a_submit_fecs_sideband_method_op(struct gk20a *g, struct fecs_method_op_gk20a op) { struct gr_gk20a *gr = &g->gr; int ret; nvgpu_mutex_acquire(&gr->fecs_mutex); gk20a_writel(g, gr_fecs_ctxsw_mailbox_clear_r(op.mailbox.id), gr_fecs_ctxsw_mailbox_clear_value_f(op.mailbox.clr)); gk20a_writel(g, gr_fecs_method_data_r(), op.method.data); gk20a_writel(g, gr_fecs_method_push_r(), gr_fecs_method_push_adr_f(op.method.addr)); ret = gr_gk20a_ctx_wait_ucode(g, op.mailbox.id, op.mailbox.ret, op.cond.ok, op.mailbox.ok, op.cond.fail, op.mailbox.fail, false); if (ret) { nvgpu_err(g,"fecs method: data=0x%08x push adr=0x%08x", op.method.data, op.method.addr); } nvgpu_mutex_release(&gr->fecs_mutex); return ret; } static int gr_gk20a_ctrl_ctxsw(struct gk20a *g, u32 fecs_method, u32 *ret) { return gr_gk20a_submit_fecs_method_op(g, (struct fecs_method_op_gk20a) { .method.addr = fecs_method, .method.data = ~0, .mailbox = { .id = 1, /*sideband?*/ .data = ~0, .clr = ~0, .ret = ret, .ok = gr_fecs_ctxsw_mailbox_value_pass_v(), .fail = gr_fecs_ctxsw_mailbox_value_fail_v(), }, .cond.ok = GR_IS_UCODE_OP_EQUAL, .cond.fail = GR_IS_UCODE_OP_EQUAL }, true); } /** * Stop processing (stall) context switches at FECS:- * If fecs is sent stop_ctxsw method, elpg entry/exit cannot happen * and may timeout. It could manifest as different error signatures * depending on when stop_ctxsw fecs method gets sent with respect * to pmu elpg sequence. It could come as pmu halt or abort or * maybe ext error too. */ int gr_gk20a_disable_ctxsw(struct gk20a *g) { int err = 0; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " "); nvgpu_mutex_acquire(&g->ctxsw_disable_lock); g->ctxsw_disable_count++; if (g->ctxsw_disable_count == 1) { err = nvgpu_pg_elpg_disable(g); if (err != 0) { nvgpu_err(g, "failed to disable elpg. not safe to " "stop_ctxsw"); /* stop ctxsw command is not sent */ g->ctxsw_disable_count--; } else { err = gr_gk20a_ctrl_ctxsw(g, gr_fecs_method_push_adr_stop_ctxsw_v(), NULL); if (err != 0) { nvgpu_err(g, "failed to stop fecs ctxsw"); /* stop ctxsw failed */ g->ctxsw_disable_count--; } } } else { nvgpu_log_info(g, "ctxsw disabled, ctxsw_disable_count: %d", g->ctxsw_disable_count); } nvgpu_mutex_release(&g->ctxsw_disable_lock); return err; } /* Start processing (continue) context switches at FECS */ int gr_gk20a_enable_ctxsw(struct gk20a *g) { int err = 0; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " "); nvgpu_mutex_acquire(&g->ctxsw_disable_lock); if (g->ctxsw_disable_count == 0) { goto ctxsw_already_enabled; } g->ctxsw_disable_count--; WARN_ON(g->ctxsw_disable_count < 0); if (g->ctxsw_disable_count == 0) { err = gr_gk20a_ctrl_ctxsw(g, gr_fecs_method_push_adr_start_ctxsw_v(), NULL); if (err != 0) { nvgpu_err(g, "failed to start fecs ctxsw"); } else { if (nvgpu_pg_elpg_enable(g) != 0) { nvgpu_err(g, "failed to enable elpg " "after start_ctxsw"); } } } else { nvgpu_log_info(g, "ctxsw_disable_count: %d is not 0 yet", g->ctxsw_disable_count); } ctxsw_already_enabled: nvgpu_mutex_release(&g->ctxsw_disable_lock); return err; } int gr_gk20a_halt_pipe(struct gk20a *g) { return gr_gk20a_submit_fecs_method_op(g, (struct fecs_method_op_gk20a) { .method.addr = gr_fecs_method_push_adr_halt_pipeline_v(), .method.data = ~0, .mailbox = { .id = 1, /*sideband?*/ .data = ~0, .clr = ~0, .ret = NULL, .ok = gr_fecs_ctxsw_mailbox_value_pass_v(), .fail = gr_fecs_ctxsw_mailbox_value_fail_v(), }, .cond.ok = GR_IS_UCODE_OP_EQUAL, .cond.fail = GR_IS_UCODE_OP_EQUAL }, false); } int gr_gk20a_commit_inst(struct channel_gk20a *c, u64 gpu_va) { u32 addr_lo; u32 addr_hi; nvgpu_log_fn(c->g, " "); addr_lo = u64_lo32(gpu_va) >> 12; addr_hi = u64_hi32(gpu_va); nvgpu_mem_wr32(c->g, &c->inst_block, ram_in_gr_wfi_target_w(), ram_in_gr_cs_wfi_f() | ram_in_gr_wfi_mode_virtual_f() | ram_in_gr_wfi_ptr_lo_f(addr_lo)); nvgpu_mem_wr32(c->g, &c->inst_block, ram_in_gr_wfi_ptr_hi_w(), ram_in_gr_wfi_ptr_hi_f(addr_hi)); return 0; } /* * Context state can be written directly, or "patched" at times. So that code * can be used in either situation it is written using a series of * _ctx_patch_write(..., patch) statements. However any necessary map overhead * should be minimized; thus, bundle the sequence of these writes together, and * set them up and close with _ctx_patch_write_begin/_ctx_patch_write_end. */ int gr_gk20a_ctx_patch_write_begin(struct gk20a *g, struct nvgpu_gr_ctx *gr_ctx, bool update_patch_count) { if (update_patch_count) { /* reset patch count if ucode has already processed it */ gr_ctx->patch_ctx.data_count = nvgpu_mem_rd(g, &gr_ctx->mem, ctxsw_prog_main_image_patch_count_o()); nvgpu_log(g, gpu_dbg_info, "patch count reset to %d", gr_ctx->patch_ctx.data_count); } return 0; } void gr_gk20a_ctx_patch_write_end(struct gk20a *g, struct nvgpu_gr_ctx *gr_ctx, bool update_patch_count) { /* Write context count to context image if it is mapped */ if (update_patch_count) { nvgpu_mem_wr(g, &gr_ctx->mem, ctxsw_prog_main_image_patch_count_o(), gr_ctx->patch_ctx.data_count); nvgpu_log(g, gpu_dbg_info, "write patch count %d", gr_ctx->patch_ctx.data_count); } } void gr_gk20a_ctx_patch_write(struct gk20a *g, struct nvgpu_gr_ctx *gr_ctx, u32 addr, u32 data, bool patch) { if (patch) { u32 patch_slot = gr_ctx->patch_ctx.data_count * PATCH_CTX_SLOTS_REQUIRED_PER_ENTRY; if (patch_slot > (PATCH_CTX_ENTRIES_FROM_SIZE( gr_ctx->patch_ctx.mem.size) - PATCH_CTX_SLOTS_REQUIRED_PER_ENTRY)) { nvgpu_err(g, "failed to access patch_slot %d", patch_slot); return; } nvgpu_mem_wr32(g, &gr_ctx->patch_ctx.mem, patch_slot, addr); nvgpu_mem_wr32(g, &gr_ctx->patch_ctx.mem, patch_slot + 1, data); gr_ctx->patch_ctx.data_count++; nvgpu_log(g, gpu_dbg_info, "patch addr = 0x%x data = 0x%x data_count %d", addr, data, gr_ctx->patch_ctx.data_count); } else { gk20a_writel(g, addr, data); } } static u32 fecs_current_ctx_data(struct gk20a *g, struct nvgpu_mem *inst_block) { u64 ptr = nvgpu_inst_block_addr(g, inst_block) >> ram_in_base_shift_v(); u32 aperture = nvgpu_aperture_mask(g, inst_block, gr_fecs_current_ctx_target_sys_mem_ncoh_f(), gr_fecs_current_ctx_target_sys_mem_coh_f(), gr_fecs_current_ctx_target_vid_mem_f()); return gr_fecs_current_ctx_ptr_f(u64_lo32(ptr)) | aperture | gr_fecs_current_ctx_valid_f(1); } int gr_gk20a_fecs_ctx_bind_channel(struct gk20a *g, struct channel_gk20a *c) { u32 inst_base_ptr = u64_lo32(nvgpu_inst_block_addr(g, &c->inst_block) >> ram_in_base_shift_v()); u32 data = fecs_current_ctx_data(g, &c->inst_block); u32 ret; nvgpu_log_info(g, "bind channel %d inst ptr 0x%08x", c->chid, inst_base_ptr); ret = gr_gk20a_submit_fecs_method_op(g, (struct fecs_method_op_gk20a) { .method.addr = gr_fecs_method_push_adr_bind_pointer_v(), .method.data = data, .mailbox = { .id = 0, .data = 0, .clr = 0x30, .ret = NULL, .ok = 0x10, .fail = 0x20, }, .cond.ok = GR_IS_UCODE_OP_AND, .cond.fail = GR_IS_UCODE_OP_AND}, true); if (ret) { nvgpu_err(g, "bind channel instance failed"); } return ret; } void gr_gk20a_write_zcull_ptr(struct gk20a *g, struct nvgpu_mem *mem, u64 gpu_va) { u32 va = u64_lo32(gpu_va >> 8); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_zcull_ptr_o(), va); } void gr_gk20a_write_pm_ptr(struct gk20a *g, struct nvgpu_mem *mem, u64 gpu_va) { u32 va = u64_lo32(gpu_va >> 8); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_pm_ptr_o(), va); } static int gr_gk20a_ctx_zcull_setup(struct gk20a *g, struct channel_gk20a *c) { struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx = NULL; struct nvgpu_mem *mem = NULL; struct nvgpu_mem *ctxheader = &c->ctx_header; int ret = 0; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; mem = &gr_ctx->mem; if (gr_ctx->zcull_ctx.gpu_va == 0 && gr_ctx->zcull_ctx.ctx_sw_mode == ctxsw_prog_main_image_zcull_mode_separate_buffer_v()) { return -EINVAL; } ret = gk20a_disable_channel_tsg(g, c); if (ret) { nvgpu_err(g, "failed to disable channel/TSG"); return ret; } ret = gk20a_fifo_preempt(g, c); if (ret) { gk20a_enable_channel_tsg(g, c); nvgpu_err(g, "failed to preempt channel/TSG"); return ret; } nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_zcull_o(), gr_ctx->zcull_ctx.ctx_sw_mode); if (ctxheader->gpu_va) { g->ops.gr.write_zcull_ptr(g, ctxheader, gr_ctx->zcull_ctx.gpu_va); } else { g->ops.gr.write_zcull_ptr(g, mem, gr_ctx->zcull_ctx.gpu_va); } gk20a_enable_channel_tsg(g, c); return ret; } u32 gk20a_gr_gpc_offset(struct gk20a *g, u32 gpc) { u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); u32 gpc_offset = gpc_stride * gpc; return gpc_offset; } u32 gk20a_gr_tpc_offset(struct gk20a *g, u32 tpc) { u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); u32 tpc_offset = tpc_in_gpc_stride * tpc; return tpc_offset; } int gr_gk20a_commit_global_ctx_buffers(struct gk20a *g, struct channel_gk20a *c, bool patch) { struct gr_gk20a *gr = &g->gr; struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx = NULL; u64 addr; u32 size; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; if (patch) { int err; err = gr_gk20a_ctx_patch_write_begin(g, gr_ctx, false); if (err != 0) { return err; } } /* global pagepool buffer */ addr = (u64_lo32(gr_ctx->global_ctx_buffer_va[PAGEPOOL_VA]) >> gr_scc_pagepool_base_addr_39_8_align_bits_v()) | (u64_hi32(gr_ctx->global_ctx_buffer_va[PAGEPOOL_VA]) << (32 - gr_scc_pagepool_base_addr_39_8_align_bits_v())); size = gr->global_ctx_buffer[PAGEPOOL].mem.size / gr_scc_pagepool_total_pages_byte_granularity_v(); if (size == g->ops.gr.pagepool_default_size(g)) { size = gr_scc_pagepool_total_pages_hwmax_v(); } nvgpu_log_info(g, "pagepool buffer addr : 0x%016llx, size : %d", addr, size); g->ops.gr.commit_global_pagepool(g, gr_ctx, addr, size, patch); /* global bundle cb */ addr = (u64_lo32(gr_ctx->global_ctx_buffer_va[CIRCULAR_VA]) >> gr_scc_bundle_cb_base_addr_39_8_align_bits_v()) | (u64_hi32(gr_ctx->global_ctx_buffer_va[CIRCULAR_VA]) << (32 - gr_scc_bundle_cb_base_addr_39_8_align_bits_v())); size = gr->bundle_cb_default_size; nvgpu_log_info(g, "bundle cb addr : 0x%016llx, size : %d", addr, size); g->ops.gr.commit_global_bundle_cb(g, gr_ctx, addr, size, patch); /* global attrib cb */ addr = (u64_lo32(gr_ctx->global_ctx_buffer_va[ATTRIBUTE_VA]) >> gr_gpcs_setup_attrib_cb_base_addr_39_12_align_bits_v()) | (u64_hi32(gr_ctx->global_ctx_buffer_va[ATTRIBUTE_VA]) << (32 - gr_gpcs_setup_attrib_cb_base_addr_39_12_align_bits_v())); nvgpu_log_info(g, "attrib cb addr : 0x%016llx", addr); g->ops.gr.commit_global_attrib_cb(g, gr_ctx, addr, patch); g->ops.gr.commit_global_cb_manager(g, c, patch); if (patch) { gr_gk20a_ctx_patch_write_end(g, gr_ctx, false); } return 0; } int gr_gk20a_commit_global_timeslice(struct gk20a *g, struct channel_gk20a *c) { struct gr_gk20a *gr = &g->gr; struct nvgpu_gr_ctx *gr_ctx = NULL; u32 gpm_pd_cfg; u32 pd_ab_dist_cfg0; u32 ds_debug; u32 mpc_vtg_debug; u32 pe_vaf; u32 pe_vsc_vpc; nvgpu_log_fn(g, " "); gpm_pd_cfg = gk20a_readl(g, gr_gpcs_gpm_pd_cfg_r()); pd_ab_dist_cfg0 = gk20a_readl(g, gr_pd_ab_dist_cfg0_r()); ds_debug = gk20a_readl(g, gr_ds_debug_r()); mpc_vtg_debug = gk20a_readl(g, gr_gpcs_tpcs_mpc_vtg_debug_r()); if (gr->timeslice_mode == gr_gpcs_ppcs_cbm_cfg_timeslice_mode_enable_v()) { pe_vaf = gk20a_readl(g, gr_gpcs_tpcs_pe_vaf_r()); pe_vsc_vpc = gk20a_readl(g, gr_gpcs_tpcs_pes_vsc_vpc_r()); gpm_pd_cfg = gr_gpcs_gpm_pd_cfg_timeslice_mode_enable_f() | gpm_pd_cfg; pe_vaf = gr_gpcs_tpcs_pe_vaf_fast_mode_switch_true_f() | pe_vaf; pe_vsc_vpc = gr_gpcs_tpcs_pes_vsc_vpc_fast_mode_switch_true_f() | pe_vsc_vpc; pd_ab_dist_cfg0 = gr_pd_ab_dist_cfg0_timeslice_enable_en_f() | pd_ab_dist_cfg0; ds_debug = gr_ds_debug_timeslice_mode_enable_f() | ds_debug; mpc_vtg_debug = gr_gpcs_tpcs_mpc_vtg_debug_timeslice_mode_enabled_f() | mpc_vtg_debug; gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_gpm_pd_cfg_r(), gpm_pd_cfg, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_tpcs_pe_vaf_r(), pe_vaf, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_tpcs_pes_vsc_vpc_r(), pe_vsc_vpc, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_pd_ab_dist_cfg0_r(), pd_ab_dist_cfg0, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_ds_debug_r(), ds_debug, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_tpcs_mpc_vtg_debug_r(), mpc_vtg_debug, false); } else { gpm_pd_cfg = gr_gpcs_gpm_pd_cfg_timeslice_mode_disable_f() | gpm_pd_cfg; pd_ab_dist_cfg0 = gr_pd_ab_dist_cfg0_timeslice_enable_dis_f() | pd_ab_dist_cfg0; ds_debug = gr_ds_debug_timeslice_mode_disable_f() | ds_debug; mpc_vtg_debug = gr_gpcs_tpcs_mpc_vtg_debug_timeslice_mode_disabled_f() | mpc_vtg_debug; gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_gpm_pd_cfg_r(), gpm_pd_cfg, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_pd_ab_dist_cfg0_r(), pd_ab_dist_cfg0, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_ds_debug_r(), ds_debug, false); gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_tpcs_mpc_vtg_debug_r(), mpc_vtg_debug, false); } return 0; } /* * Return map tiles count for given index * Return 0 if index is out-of-bounds */ static u32 gr_gk20a_get_map_tile_count(struct gr_gk20a *gr, u32 index) { if (index >= gr->map_tile_count) { return 0; } return gr->map_tiles[index]; } int gr_gk20a_setup_rop_mapping(struct gk20a *g, struct gr_gk20a *gr) { u32 norm_entries, norm_shift; u32 coeff5_mod, coeff6_mod, coeff7_mod, coeff8_mod, coeff9_mod, coeff10_mod, coeff11_mod; u32 map0, map1, map2, map3, map4, map5; if (gr->map_tiles == NULL) { return -1; } nvgpu_log_fn(g, " "); gk20a_writel(g, gr_crstr_map_table_cfg_r(), gr_crstr_map_table_cfg_row_offset_f(gr->map_row_offset) | gr_crstr_map_table_cfg_num_entries_f(gr->tpc_count)); map0 = gr_crstr_gpc_map0_tile0_f(gr_gk20a_get_map_tile_count(gr, 0)) | gr_crstr_gpc_map0_tile1_f(gr_gk20a_get_map_tile_count(gr, 1)) | gr_crstr_gpc_map0_tile2_f(gr_gk20a_get_map_tile_count(gr, 2)) | gr_crstr_gpc_map0_tile3_f(gr_gk20a_get_map_tile_count(gr, 3)) | gr_crstr_gpc_map0_tile4_f(gr_gk20a_get_map_tile_count(gr, 4)) | gr_crstr_gpc_map0_tile5_f(gr_gk20a_get_map_tile_count(gr, 5)); map1 = gr_crstr_gpc_map1_tile6_f(gr_gk20a_get_map_tile_count(gr, 6)) | gr_crstr_gpc_map1_tile7_f(gr_gk20a_get_map_tile_count(gr, 7)) | gr_crstr_gpc_map1_tile8_f(gr_gk20a_get_map_tile_count(gr, 8)) | gr_crstr_gpc_map1_tile9_f(gr_gk20a_get_map_tile_count(gr, 9)) | gr_crstr_gpc_map1_tile10_f(gr_gk20a_get_map_tile_count(gr, 10)) | gr_crstr_gpc_map1_tile11_f(gr_gk20a_get_map_tile_count(gr, 11)); map2 = gr_crstr_gpc_map2_tile12_f(gr_gk20a_get_map_tile_count(gr, 12)) | gr_crstr_gpc_map2_tile13_f(gr_gk20a_get_map_tile_count(gr, 13)) | gr_crstr_gpc_map2_tile14_f(gr_gk20a_get_map_tile_count(gr, 14)) | gr_crstr_gpc_map2_tile15_f(gr_gk20a_get_map_tile_count(gr, 15)) | gr_crstr_gpc_map2_tile16_f(gr_gk20a_get_map_tile_count(gr, 16)) | gr_crstr_gpc_map2_tile17_f(gr_gk20a_get_map_tile_count(gr, 17)); map3 = gr_crstr_gpc_map3_tile18_f(gr_gk20a_get_map_tile_count(gr, 18)) | gr_crstr_gpc_map3_tile19_f(gr_gk20a_get_map_tile_count(gr, 19)) | gr_crstr_gpc_map3_tile20_f(gr_gk20a_get_map_tile_count(gr, 20)) | gr_crstr_gpc_map3_tile21_f(gr_gk20a_get_map_tile_count(gr, 21)) | gr_crstr_gpc_map3_tile22_f(gr_gk20a_get_map_tile_count(gr, 22)) | gr_crstr_gpc_map3_tile23_f(gr_gk20a_get_map_tile_count(gr, 23)); map4 = gr_crstr_gpc_map4_tile24_f(gr_gk20a_get_map_tile_count(gr, 24)) | gr_crstr_gpc_map4_tile25_f(gr_gk20a_get_map_tile_count(gr, 25)) | gr_crstr_gpc_map4_tile26_f(gr_gk20a_get_map_tile_count(gr, 26)) | gr_crstr_gpc_map4_tile27_f(gr_gk20a_get_map_tile_count(gr, 27)) | gr_crstr_gpc_map4_tile28_f(gr_gk20a_get_map_tile_count(gr, 28)) | gr_crstr_gpc_map4_tile29_f(gr_gk20a_get_map_tile_count(gr, 29)); map5 = gr_crstr_gpc_map5_tile30_f(gr_gk20a_get_map_tile_count(gr, 30)) | gr_crstr_gpc_map5_tile31_f(gr_gk20a_get_map_tile_count(gr, 31)) | gr_crstr_gpc_map5_tile32_f(0) | gr_crstr_gpc_map5_tile33_f(0) | gr_crstr_gpc_map5_tile34_f(0) | gr_crstr_gpc_map5_tile35_f(0); gk20a_writel(g, gr_crstr_gpc_map0_r(), map0); gk20a_writel(g, gr_crstr_gpc_map1_r(), map1); gk20a_writel(g, gr_crstr_gpc_map2_r(), map2); gk20a_writel(g, gr_crstr_gpc_map3_r(), map3); gk20a_writel(g, gr_crstr_gpc_map4_r(), map4); gk20a_writel(g, gr_crstr_gpc_map5_r(), map5); switch (gr->tpc_count) { case 1: norm_shift = 4; break; case 2: case 3: norm_shift = 3; break; case 4: case 5: case 6: case 7: norm_shift = 2; break; case 8: case 9: case 10: case 11: case 12: case 13: case 14: case 15: norm_shift = 1; break; default: norm_shift = 0; break; } norm_entries = gr->tpc_count << norm_shift; coeff5_mod = (1 << 5) % norm_entries; coeff6_mod = (1 << 6) % norm_entries; coeff7_mod = (1 << 7) % norm_entries; coeff8_mod = (1 << 8) % norm_entries; coeff9_mod = (1 << 9) % norm_entries; coeff10_mod = (1 << 10) % norm_entries; coeff11_mod = (1 << 11) % norm_entries; gk20a_writel(g, gr_ppcs_wwdx_map_table_cfg_r(), gr_ppcs_wwdx_map_table_cfg_row_offset_f(gr->map_row_offset) | gr_ppcs_wwdx_map_table_cfg_normalized_num_entries_f(norm_entries) | gr_ppcs_wwdx_map_table_cfg_normalized_shift_value_f(norm_shift) | gr_ppcs_wwdx_map_table_cfg_coeff5_mod_value_f(coeff5_mod) | gr_ppcs_wwdx_map_table_cfg_num_entries_f(gr->tpc_count)); gk20a_writel(g, gr_ppcs_wwdx_map_table_cfg2_r(), gr_ppcs_wwdx_map_table_cfg2_coeff6_mod_value_f(coeff6_mod) | gr_ppcs_wwdx_map_table_cfg2_coeff7_mod_value_f(coeff7_mod) | gr_ppcs_wwdx_map_table_cfg2_coeff8_mod_value_f(coeff8_mod) | gr_ppcs_wwdx_map_table_cfg2_coeff9_mod_value_f(coeff9_mod) | gr_ppcs_wwdx_map_table_cfg2_coeff10_mod_value_f(coeff10_mod) | gr_ppcs_wwdx_map_table_cfg2_coeff11_mod_value_f(coeff11_mod)); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map0_r(), map0); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map1_r(), map1); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map2_r(), map2); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map3_r(), map3); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map4_r(), map4); gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map5_r(), map5); gk20a_writel(g, gr_rstr2d_map_table_cfg_r(), gr_rstr2d_map_table_cfg_row_offset_f(gr->map_row_offset) | gr_rstr2d_map_table_cfg_num_entries_f(gr->tpc_count)); gk20a_writel(g, gr_rstr2d_gpc_map0_r(), map0); gk20a_writel(g, gr_rstr2d_gpc_map1_r(), map1); gk20a_writel(g, gr_rstr2d_gpc_map2_r(), map2); gk20a_writel(g, gr_rstr2d_gpc_map3_r(), map3); gk20a_writel(g, gr_rstr2d_gpc_map4_r(), map4); gk20a_writel(g, gr_rstr2d_gpc_map5_r(), map5); return 0; } static inline u32 count_bits(u32 mask) { u32 temp = mask; u32 count; for (count = 0; temp != 0; count++) { temp &= temp - 1; } return count; } int gr_gk20a_init_sm_id_table(struct gk20a *g) { u32 gpc, tpc; u32 sm_id = 0; for (tpc = 0; tpc < g->gr.max_tpc_per_gpc_count; tpc++) { for (gpc = 0; gpc < g->gr.gpc_count; gpc++) { if (tpc < g->gr.gpc_tpc_count[gpc]) { g->gr.sm_to_cluster[sm_id].tpc_index = tpc; g->gr.sm_to_cluster[sm_id].gpc_index = gpc; g->gr.sm_to_cluster[sm_id].sm_index = 0; g->gr.sm_to_cluster[sm_id].global_tpc_index = sm_id; sm_id++; } } } g->gr.no_of_sm = sm_id; return 0; } /* * Return number of TPCs in a GPC * Return 0 if GPC index is invalid i.e. GPC is disabled */ u32 gr_gk20a_get_tpc_count(struct gr_gk20a *gr, u32 gpc_index) { if (gpc_index >= gr->gpc_count) { return 0; } return gr->gpc_tpc_count[gpc_index]; } int gr_gk20a_init_fs_state(struct gk20a *g) { struct gr_gk20a *gr = &g->gr; u32 tpc_index, gpc_index; u32 sm_id = 0, gpc_id = 0; u32 tpc_per_gpc; u32 fuse_tpc_mask; u32 reg_index; int err; nvgpu_log_fn(g, " "); if (g->ops.gr.init_sm_id_table) { err = g->ops.gr.init_sm_id_table(g); if (err != 0) { return err; } /* Is table empty ? */ if (g->gr.no_of_sm == 0) { return -EINVAL; } } for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) { tpc_index = g->gr.sm_to_cluster[sm_id].tpc_index; gpc_index = g->gr.sm_to_cluster[sm_id].gpc_index; g->ops.gr.program_sm_id_numbering(g, gpc_index, tpc_index, sm_id); if (g->ops.gr.program_active_tpc_counts) { g->ops.gr.program_active_tpc_counts(g, gpc_index); } } for (reg_index = 0, gpc_id = 0; reg_index < gr_pd_num_tpc_per_gpc__size_1_v(); reg_index++, gpc_id += 8) { tpc_per_gpc = gr_pd_num_tpc_per_gpc_count0_f(gr_gk20a_get_tpc_count(gr, gpc_id + 0)) | gr_pd_num_tpc_per_gpc_count1_f(gr_gk20a_get_tpc_count(gr, gpc_id + 1)) | gr_pd_num_tpc_per_gpc_count2_f(gr_gk20a_get_tpc_count(gr, gpc_id + 2)) | gr_pd_num_tpc_per_gpc_count3_f(gr_gk20a_get_tpc_count(gr, gpc_id + 3)) | gr_pd_num_tpc_per_gpc_count4_f(gr_gk20a_get_tpc_count(gr, gpc_id + 4)) | gr_pd_num_tpc_per_gpc_count5_f(gr_gk20a_get_tpc_count(gr, gpc_id + 5)) | gr_pd_num_tpc_per_gpc_count6_f(gr_gk20a_get_tpc_count(gr, gpc_id + 6)) | gr_pd_num_tpc_per_gpc_count7_f(gr_gk20a_get_tpc_count(gr, gpc_id + 7)); gk20a_writel(g, gr_pd_num_tpc_per_gpc_r(reg_index), tpc_per_gpc); gk20a_writel(g, gr_ds_num_tpc_per_gpc_r(reg_index), tpc_per_gpc); } /* gr__setup_pd_mapping stubbed for gk20a */ g->ops.gr.setup_rop_mapping(g, gr); if (g->ops.gr.setup_alpha_beta_tables) { g->ops.gr.setup_alpha_beta_tables(g, gr); } for (gpc_index = 0; gpc_index < gr_pd_dist_skip_table__size_1_v() * 4; gpc_index += 4) { gk20a_writel(g, gr_pd_dist_skip_table_r(gpc_index/4), (gr_pd_dist_skip_table_gpc_4n0_mask_f(gr->gpc_skip_mask[gpc_index]) != 0U) || (gr_pd_dist_skip_table_gpc_4n1_mask_f(gr->gpc_skip_mask[gpc_index + 1]) != 0U) || (gr_pd_dist_skip_table_gpc_4n2_mask_f(gr->gpc_skip_mask[gpc_index + 2]) != 0U) || (gr_pd_dist_skip_table_gpc_4n3_mask_f(gr->gpc_skip_mask[gpc_index + 3]) != 0U)); } fuse_tpc_mask = g->ops.gr.get_gpc_tpc_mask(g, 0); if ((g->tpc_fs_mask_user != 0U) && (fuse_tpc_mask == BIT32(gr->max_tpc_count) - 1U)) { u32 val = g->tpc_fs_mask_user; val &= (0x1U << gr->max_tpc_count) - 1U; gk20a_writel(g, gr_cwd_fs_r(), gr_cwd_fs_num_gpcs_f(gr->gpc_count) | gr_cwd_fs_num_tpcs_f(hweight32(val))); } else { gk20a_writel(g, gr_cwd_fs_r(), gr_cwd_fs_num_gpcs_f(gr->gpc_count) | gr_cwd_fs_num_tpcs_f(gr->tpc_count)); } gk20a_writel(g, gr_bes_zrop_settings_r(), gr_bes_zrop_settings_num_active_fbps_f(gr->num_fbps)); gk20a_writel(g, gr_bes_crop_settings_r(), gr_bes_crop_settings_num_active_fbps_f(gr->num_fbps)); return 0; } int gr_gk20a_fecs_ctx_image_save(struct channel_gk20a *c, u32 save_type) { struct gk20a *g = c->g; int ret; nvgpu_log_fn(g, " "); ret = gr_gk20a_submit_fecs_method_op(g, (struct fecs_method_op_gk20a) { .method.addr = save_type, .method.data = fecs_current_ctx_data(g, &c->inst_block), .mailbox = {.id = 0, .data = 0, .clr = 3, .ret = NULL, .ok = 1, .fail = 2, }, .cond.ok = GR_IS_UCODE_OP_AND, .cond.fail = GR_IS_UCODE_OP_AND, }, true); if (ret) { nvgpu_err(g, "save context image failed"); } return ret; } u32 gk20a_init_sw_bundle(struct gk20a *g) { struct av_list_gk20a *sw_bundle_init = &g->gr.ctx_vars.sw_bundle_init; u32 last_bundle_data = 0; u32 err = 0; unsigned int i; /* disable fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_disabled_f()); /* enable pipe mode override */ gk20a_writel(g, gr_pipe_bundle_config_r(), gr_pipe_bundle_config_override_pipe_mode_enabled_f()); /* load bundle init */ for (i = 0; i < sw_bundle_init->count; i++) { if (i == 0 || last_bundle_data != sw_bundle_init->l[i].value) { gk20a_writel(g, gr_pipe_bundle_data_r(), sw_bundle_init->l[i].value); last_bundle_data = sw_bundle_init->l[i].value; } gk20a_writel(g, gr_pipe_bundle_address_r(), sw_bundle_init->l[i].addr); if (gr_pipe_bundle_address_value_v(sw_bundle_init->l[i].addr) == GR_GO_IDLE_BUNDLE) { err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto error; } } err = gr_gk20a_wait_fe_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto error; } } if ((err == 0U) && (g->ops.gr.init_sw_veid_bundle != NULL)) { err = g->ops.gr.init_sw_veid_bundle(g); if (err != 0U) { goto error; } } if (g->ops.gr.init_sw_bundle64) { err = g->ops.gr.init_sw_bundle64(g); if (err != 0U) { goto error; } } /* disable pipe mode override */ gk20a_writel(g, gr_pipe_bundle_config_r(), gr_pipe_bundle_config_override_pipe_mode_disabled_f()); err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); /* restore fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_prod_f()); return err; error: /* in case of error skip waiting for GR idle - just restore state */ gk20a_writel(g, gr_pipe_bundle_config_r(), gr_pipe_bundle_config_override_pipe_mode_disabled_f()); /* restore fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_prod_f()); return err; } /* init global golden image from a fresh gr_ctx in channel ctx. save a copy in local_golden_image in ctx_vars */ static int gr_gk20a_init_golden_ctx_image(struct gk20a *g, struct channel_gk20a *c) { struct gr_gk20a *gr = &g->gr; struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx = NULL; u32 ctx_header_bytes = ctxsw_prog_fecs_header_v(); u32 ctx_header_words; u32 i; u32 data; struct nvgpu_mem *gold_mem = &gr->global_ctx_buffer[GOLDEN_CTX].mem; struct nvgpu_mem *gr_mem; u32 err = 0; struct aiv_list_gk20a *sw_ctx_load = &g->gr.ctx_vars.sw_ctx_load; struct av_list_gk20a *sw_method_init = &g->gr.ctx_vars.sw_method_init; u32 last_method_data = 0; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; gr_mem = &gr_ctx->mem; /* golden ctx is global to all channels. Although only the first channel initializes golden image, driver needs to prevent multiple channels from initializing golden ctx at the same time */ nvgpu_mutex_acquire(&gr->ctx_mutex); if (gr->ctx_vars.golden_image_initialized) { goto clean_up; } if (!nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) { struct nvgpu_timeout timeout; nvgpu_timeout_init(g, &timeout, FE_PWR_MODE_TIMEOUT_MAX / FE_PWR_MODE_TIMEOUT_DEFAULT, NVGPU_TIMER_RETRY_TIMER); gk20a_writel(g, gr_fe_pwr_mode_r(), gr_fe_pwr_mode_req_send_f() | gr_fe_pwr_mode_mode_force_on_f()); do { u32 req = gr_fe_pwr_mode_req_v(gk20a_readl(g, gr_fe_pwr_mode_r())); if (req == gr_fe_pwr_mode_req_done_v()) { break; } nvgpu_udelay(FE_PWR_MODE_TIMEOUT_DEFAULT); } while (nvgpu_timeout_expired_msg(&timeout, "timeout forcing FE on") == 0); } gk20a_writel(g, gr_fecs_ctxsw_reset_ctl_r(), gr_fecs_ctxsw_reset_ctl_sys_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_be_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_sys_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_be_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_sys_context_reset_enabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_context_reset_enabled_f() | gr_fecs_ctxsw_reset_ctl_be_context_reset_enabled_f()); (void) gk20a_readl(g, gr_fecs_ctxsw_reset_ctl_r()); nvgpu_udelay(10); gk20a_writel(g, gr_fecs_ctxsw_reset_ctl_r(), gr_fecs_ctxsw_reset_ctl_sys_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_be_halt_disabled_f() | gr_fecs_ctxsw_reset_ctl_sys_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_be_engine_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_sys_context_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_gpc_context_reset_disabled_f() | gr_fecs_ctxsw_reset_ctl_be_context_reset_disabled_f()); (void) gk20a_readl(g, gr_fecs_ctxsw_reset_ctl_r()); nvgpu_udelay(10); if (!nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) { struct nvgpu_timeout timeout; nvgpu_timeout_init(g, &timeout, FE_PWR_MODE_TIMEOUT_MAX / FE_PWR_MODE_TIMEOUT_DEFAULT, NVGPU_TIMER_RETRY_TIMER); gk20a_writel(g, gr_fe_pwr_mode_r(), gr_fe_pwr_mode_req_send_f() | gr_fe_pwr_mode_mode_auto_f()); do { u32 req = gr_fe_pwr_mode_req_v(gk20a_readl(g, gr_fe_pwr_mode_r())); if (req == gr_fe_pwr_mode_req_done_v()) { break; } nvgpu_udelay(FE_PWR_MODE_TIMEOUT_DEFAULT); } while (nvgpu_timeout_expired_msg(&timeout, "timeout setting FE power to auto") == 0); } /* clear scc ram */ gk20a_writel(g, gr_scc_init_r(), gr_scc_init_ram_trigger_f()); err = gr_gk20a_fecs_ctx_bind_channel(g, c); if (err != 0U) { goto clean_up; } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); /* load ctx init */ for (i = 0; i < sw_ctx_load->count; i++) { gk20a_writel(g, sw_ctx_load->l[i].addr, sw_ctx_load->l[i].value); } if (g->ops.gr.disable_rd_coalesce) { g->ops.gr.disable_rd_coalesce(g); } if (g->ops.gr.init_preemption_state) { g->ops.gr.init_preemption_state(g); } if (g->ops.clock_gating.blcg_gr_load_gating_prod) { g->ops.clock_gating.blcg_gr_load_gating_prod(g, g->blcg_enabled); } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto clean_up; } /* disable fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_disabled_f()); err = g->ops.gr.commit_global_ctx_buffers(g, c, false); if (err != 0U) { goto clean_up; } /* override a few ctx state registers */ g->ops.gr.commit_global_timeslice(g, c); /* floorsweep anything left */ err = g->ops.gr.init_fs_state(g); if (err != 0U) { goto clean_up; } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto restore_fe_go_idle; } err = gk20a_init_sw_bundle(g); if (err != 0U) { goto clean_up; } restore_fe_go_idle: /* restore fe_go_idle */ gk20a_writel(g, gr_fe_go_idle_timeout_r(), gr_fe_go_idle_timeout_count_prod_f()); if ((err != 0U) || (gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT) != 0)) { goto clean_up; } /* load method init */ if (sw_method_init->count) { gk20a_writel(g, gr_pri_mme_shadow_raw_data_r(), sw_method_init->l[0].value); gk20a_writel(g, gr_pri_mme_shadow_raw_index_r(), gr_pri_mme_shadow_raw_index_write_trigger_f() | sw_method_init->l[0].addr); last_method_data = sw_method_init->l[0].value; } for (i = 1; i < sw_method_init->count; i++) { if (sw_method_init->l[i].value != last_method_data) { gk20a_writel(g, gr_pri_mme_shadow_raw_data_r(), sw_method_init->l[i].value); last_method_data = sw_method_init->l[i].value; } gk20a_writel(g, gr_pri_mme_shadow_raw_index_r(), gr_pri_mme_shadow_raw_index_write_trigger_f() | sw_method_init->l[i].addr); } err = gr_gk20a_wait_idle(g, gk20a_get_gr_idle_timeout(g), GR_IDLE_CHECK_DEFAULT); if (err != 0U) { goto clean_up; } ctx_header_words = roundup(ctx_header_bytes, sizeof(u32)); ctx_header_words >>= 2; g->ops.mm.l2_flush(g, true); for (i = 0; i < ctx_header_words; i++) { data = nvgpu_mem_rd32(g, gr_mem, i); nvgpu_mem_wr32(g, gold_mem, i, data); } nvgpu_mem_wr(g, gold_mem, ctxsw_prog_main_image_zcull_o(), ctxsw_prog_main_image_zcull_mode_no_ctxsw_v()); g->ops.gr.write_zcull_ptr(g, gold_mem, 0); err = g->ops.gr.commit_inst(c, gr_ctx->global_ctx_buffer_va[GOLDEN_CTX_VA]); if (err != 0U) { goto clean_up; } gr_gk20a_fecs_ctx_image_save(c, gr_fecs_method_push_adr_wfi_golden_save_v()); if (gr->ctx_vars.local_golden_image == NULL) { gr->ctx_vars.local_golden_image = nvgpu_vzalloc(g, gr->ctx_vars.golden_image_size); if (gr->ctx_vars.local_golden_image == NULL) { err = -ENOMEM; goto clean_up; } nvgpu_mem_rd_n(g, gold_mem, 0, gr->ctx_vars.local_golden_image, gr->ctx_vars.golden_image_size); } err = g->ops.gr.commit_inst(c, gr_mem->gpu_va); if (err != 0U) { goto clean_up; } gr->ctx_vars.golden_image_initialized = true; gk20a_writel(g, gr_fecs_current_ctx_r(), gr_fecs_current_ctx_valid_false_f()); clean_up: if (err != 0U) { nvgpu_err(g, "fail"); } else { nvgpu_log_fn(g, "done"); } nvgpu_mutex_release(&gr->ctx_mutex); return err; } int gr_gk20a_update_smpc_ctxsw_mode(struct gk20a *g, struct channel_gk20a *c, bool enable_smpc_ctxsw) { struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx = NULL; struct nvgpu_mem *mem = NULL; u32 data; int ret; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; mem = &gr_ctx->mem; if (!nvgpu_mem_is_valid(mem)) { nvgpu_err(g, "no graphics context allocated"); return -EFAULT; } ret = gk20a_disable_channel_tsg(g, c); if (ret) { nvgpu_err(g, "failed to disable channel/TSG"); goto out; } ret = gk20a_fifo_preempt(g, c); if (ret) { gk20a_enable_channel_tsg(g, c); nvgpu_err(g, "failed to preempt channel/TSG"); goto out; } /* Channel gr_ctx buffer is gpu cacheable. Flush and invalidate before cpu update. */ g->ops.mm.l2_flush(g, true); data = nvgpu_mem_rd(g, mem, ctxsw_prog_main_image_pm_o()); data = data & ~ctxsw_prog_main_image_pm_smpc_mode_m(); data |= enable_smpc_ctxsw ? ctxsw_prog_main_image_pm_smpc_mode_ctxsw_f() : ctxsw_prog_main_image_pm_smpc_mode_no_ctxsw_f(); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_pm_o(), data); out: gk20a_enable_channel_tsg(g, c); return ret; } int gr_gk20a_update_hwpm_ctxsw_mode(struct gk20a *g, struct channel_gk20a *c, u64 gpu_va, u32 mode) { struct tsg_gk20a *tsg; struct nvgpu_mem *gr_mem = NULL; struct nvgpu_gr_ctx *gr_ctx; struct pm_ctx_desc *pm_ctx; u32 data; u64 virt_addr = 0; struct nvgpu_mem *ctxheader = &c->ctx_header; int ret; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; pm_ctx = &gr_ctx->pm_ctx; gr_mem = &gr_ctx->mem; if (!nvgpu_mem_is_valid(gr_mem)) { nvgpu_err(g, "no graphics context allocated"); return -EFAULT; } if ((mode == NVGPU_DBG_HWPM_CTXSW_MODE_STREAM_OUT_CTXSW) && (g->ops.gr.get_hw_accessor_stream_out_mode == NULL)) { nvgpu_err(g, "Mode-E hwpm context switch mode is not supported"); return -EINVAL; } switch (mode) { case NVGPU_DBG_HWPM_CTXSW_MODE_CTXSW: if (pm_ctx->pm_mode == ctxsw_prog_main_image_pm_mode_ctxsw_f()) { return 0; } break; case NVGPU_DBG_HWPM_CTXSW_MODE_NO_CTXSW: if (pm_ctx->pm_mode == ctxsw_prog_main_image_pm_mode_no_ctxsw_f()) { return 0; } break; case NVGPU_DBG_HWPM_CTXSW_MODE_STREAM_OUT_CTXSW: if (pm_ctx->pm_mode == g->ops.gr.get_hw_accessor_stream_out_mode()) { return 0; } break; default: nvgpu_err(g, "invalid hwpm context switch mode"); return -EINVAL; } ret = gk20a_disable_channel_tsg(g, c); if (ret) { nvgpu_err(g, "failed to disable channel/TSG"); return ret; } ret = gk20a_fifo_preempt(g, c); if (ret) { gk20a_enable_channel_tsg(g, c); nvgpu_err(g, "failed to preempt channel/TSG"); return ret; } /* Channel gr_ctx buffer is gpu cacheable. Flush and invalidate before cpu update. */ g->ops.mm.l2_flush(g, true); if (mode != NVGPU_DBG_HWPM_CTXSW_MODE_NO_CTXSW) { /* Allocate buffer if necessary */ if (pm_ctx->mem.gpu_va == 0) { ret = nvgpu_dma_alloc_sys(g, g->gr.ctx_vars.pm_ctxsw_image_size, &pm_ctx->mem); if (ret) { c->g->ops.fifo.enable_channel(c); nvgpu_err(g, "failed to allocate pm ctxt buffer"); return ret; } pm_ctx->mem.gpu_va = nvgpu_gmmu_map_fixed(c->vm, &pm_ctx->mem, gpu_va, pm_ctx->mem.size, NVGPU_VM_MAP_CACHEABLE, gk20a_mem_flag_none, true, pm_ctx->mem.aperture); if (pm_ctx->mem.gpu_va == 0ULL) { nvgpu_err(g, "failed to map pm ctxt buffer"); nvgpu_dma_free(g, &pm_ctx->mem); c->g->ops.fifo.enable_channel(c); return -ENOMEM; } } if ((mode == NVGPU_DBG_HWPM_CTXSW_MODE_STREAM_OUT_CTXSW) && (g->ops.gr.init_hwpm_pmm_register != NULL)) { g->ops.gr.init_hwpm_pmm_register(g); } } data = nvgpu_mem_rd(g, gr_mem, ctxsw_prog_main_image_pm_o()); data = data & ~ctxsw_prog_main_image_pm_mode_m(); switch (mode) { case NVGPU_DBG_HWPM_CTXSW_MODE_CTXSW: pm_ctx->pm_mode = ctxsw_prog_main_image_pm_mode_ctxsw_f(); virt_addr = pm_ctx->mem.gpu_va; break; case NVGPU_DBG_HWPM_CTXSW_MODE_STREAM_OUT_CTXSW: pm_ctx->pm_mode = g->ops.gr.get_hw_accessor_stream_out_mode(); virt_addr = pm_ctx->mem.gpu_va; break; case NVGPU_DBG_HWPM_CTXSW_MODE_NO_CTXSW: pm_ctx->pm_mode = ctxsw_prog_main_image_pm_mode_no_ctxsw_f(); virt_addr = 0; } data |= pm_ctx->pm_mode; nvgpu_mem_wr(g, gr_mem, ctxsw_prog_main_image_pm_o(), data); if (ctxheader->gpu_va) { struct channel_gk20a *ch; nvgpu_rwsem_down_read(&tsg->ch_list_lock); nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { g->ops.gr.write_pm_ptr(g, &ch->ctx_header, virt_addr); } nvgpu_rwsem_up_read(&tsg->ch_list_lock); } else { g->ops.gr.write_pm_ptr(g, gr_mem, virt_addr); } /* enable channel */ gk20a_enable_channel_tsg(g, c); return 0; } void gk20a_gr_init_ctxsw_hdr_data(struct gk20a *g, struct nvgpu_mem *mem) { nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_num_save_ops_o(), 0); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_num_restore_ops_o(), 0); } /* load saved fresh copy of gloden image into channel gr_ctx */ int gr_gk20a_load_golden_ctx_image(struct gk20a *g, struct channel_gk20a *c) { struct gr_gk20a *gr = &g->gr; struct tsg_gk20a *tsg; struct nvgpu_gr_ctx *gr_ctx; u32 virt_addr_lo; u32 virt_addr_hi; u64 virt_addr = 0; u32 v, data; int ret = 0; struct nvgpu_mem *mem; nvgpu_log_fn(g, " "); tsg = tsg_gk20a_from_ch(c); if (tsg == NULL) { return -EINVAL; } gr_ctx = &tsg->gr_ctx; mem = &gr_ctx->mem; if (gr->ctx_vars.local_golden_image == NULL) { return -EINVAL; } /* Channel gr_ctx buffer is gpu cacheable. Flush and invalidate before cpu update. */ g->ops.mm.l2_flush(g, true); nvgpu_mem_wr_n(g, mem, 0, gr->ctx_vars.local_golden_image, gr->ctx_vars.golden_image_size); if (g->ops.gr.init_ctxsw_hdr_data) { g->ops.gr.init_ctxsw_hdr_data(g, mem); } if ((g->ops.gr.enable_cde_in_fecs != NULL) && c->cde) { g->ops.gr.enable_cde_in_fecs(g, mem); } /* set priv access map */ virt_addr_lo = u64_lo32(gr_ctx->global_ctx_buffer_va[PRIV_ACCESS_MAP_VA]); virt_addr_hi = u64_hi32(gr_ctx->global_ctx_buffer_va[PRIV_ACCESS_MAP_VA]); if (g->allow_all) { data = ctxsw_prog_main_image_priv_access_map_config_mode_allow_all_f(); } else { data = ctxsw_prog_main_image_priv_access_map_config_mode_use_map_f(); } nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_priv_access_map_config_o(), data); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_priv_access_map_addr_lo_o(), virt_addr_lo); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_priv_access_map_addr_hi_o(), virt_addr_hi); /* disable verif features */ v = nvgpu_mem_rd(g, mem, ctxsw_prog_main_image_misc_options_o()); v = v & ~(ctxsw_prog_main_image_misc_options_verif_features_m()); v = v | ctxsw_prog_main_image_misc_options_verif_features_disabled_f(); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_misc_options_o(), v); if (g->ops.gr.update_ctxsw_preemption_mode) { g->ops.gr.update_ctxsw_preemption_mode(g, c, mem); } if (g->ops.gr.update_boosted_ctx) { g->ops.gr.update_boosted_ctx(g, mem, gr_ctx); } virt_addr_lo = u64_lo32(gr_ctx->patch_ctx.mem.gpu_va); virt_addr_hi = u64_hi32(gr_ctx->patch_ctx.mem.gpu_va); nvgpu_log(g, gpu_dbg_info, "write patch count = %d", gr_ctx->patch_ctx.data_count); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_patch_count_o(), gr_ctx->patch_ctx.data_count); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_patch_adr_lo_o(), virt_addr_lo); nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_patch_adr_hi_o(), virt_addr_hi); /* Update main header region of the context buffer with the info needed * for PM context switching, including mode and possibly a pointer to * the PM backing store. */ if (gr_ctx->pm_ctx.pm_mode != ctxsw_prog_main_image_pm_mode_no_ctxsw_f()) { if (gr_ctx->pm_ctx.mem.gpu_va == 0) { nvgpu_err(g, "context switched pm with no pm buffer!"); return -EFAULT; } virt_addr = gr_ctx->pm_ctx.mem.gpu_va; } else { virt_addr = 0; } data = nvgpu_mem_rd(g, mem, ctxsw_prog_main_image_pm_o()); data = data & ~ctxsw_prog_main_image_pm_mode_m(); data |= gr_ctx->pm_ctx.pm_mode; nvgpu_mem_wr(g, mem, ctxsw_prog_main_image_pm_o(), data); g->ops.gr.write_pm_ptr(g, mem, virt_addr); return ret; } static void gr_gk20a_start_falcon_ucode(struct gk20a *g) { nvgpu_log_fn(g, " "); gk20a_writel(g, gr_fecs_ctxsw_mailbox_clear_r(0), gr_fecs_ctxsw_mailbox_clear_value_f(~0)); gk20a_writel(g, gr_gpccs_dmactl_r(), gr_gpccs_dmactl_require_ctx_f(0)); gk20a_writel(g, gr_fecs_dmactl_r(), gr_fecs_dmactl_require_ctx_f(0)); gk20a_writel(g, gr_gpccs_cpuctl_r(), gr_gpccs_cpuctl_startcpu_f(1)); gk20a_writel(g, gr_fecs_cpuctl_r(), gr_fecs_cpuctl_startcpu_f(1)); nvgpu_log_fn(g, "done"); } static int gr_gk20a_init_ctxsw_ucode_vaspace(struct gk20a *g) { struct mm_gk20a *mm = &g->mm; struct vm_gk20a *vm = mm->pmu.vm; struct gk20a_ctxsw_ucode_info *ucode_info = &g->ctxsw_ucode_info; int err; err = g->ops.mm.alloc_inst_block(g, &ucode_info->inst_blk_desc); if (err != 0) { return err; } g->ops.mm.init_inst_block(&ucode_info->inst_blk_desc, vm, 0); /* Map ucode surface to GMMU */ ucode_info->surface_desc.gpu_va = nvgpu_gmmu_map(vm, &ucode_info->surface_desc, ucode_info->surface_desc.size, 0, /* flags */ gk20a_mem_flag_read_only, false, ucode_info->surface_desc.aperture); if (ucode_info->surface_desc.gpu_va == 0ULL) { nvgpu_err(g, "failed to update gmmu ptes"); return -ENOMEM; } return 0; } static void gr_gk20a_init_ctxsw_ucode_segment( struct gk20a_ctxsw_ucode_segment *p_seg, u32 *offset, u32 size) { p_seg->offset = *offset; p_seg->size = size; *offset = ALIGN(*offset + size, BLK_SIZE); } static void gr_gk20a_init_ctxsw_ucode_segments( struct gk20a_ctxsw_ucode_segments *segments, u32 *offset, struct gk20a_ctxsw_bootloader_desc *bootdesc, u32 code_size, u32 data_size) { u32 boot_size = ALIGN(bootdesc->size, sizeof(u32)); segments->boot_entry = bootdesc->entry_point; segments->boot_imem_offset = bootdesc->imem_offset; gr_gk20a_init_ctxsw_ucode_segment(&segments->boot, offset, boot_size); gr_gk20a_init_ctxsw_ucode_segment(&segments->code, offset, code_size); gr_gk20a_init_ctxsw_ucode_segment(&segments->data, offset, data_size); } static int gr_gk20a_copy_ctxsw_ucode_segments( struct gk20a *g, struct nvgpu_mem *dst, struct gk20a_ctxsw_ucode_segments *segments, u32 *bootimage, u32 *code, u32 *data) { unsigned int i; nvgpu_mem_wr_n(g, dst, segments->boot.offset, bootimage, segments->boot.size); nvgpu_mem_wr_n(g, dst, segments->code.offset, code, segments->code.size); nvgpu_mem_wr_n(g, dst, segments->data.offset, data, segments->data.size); /* compute a "checksum" for the boot binary to detect its version */ segments->boot_signature = 0; for (i = 0; i < segments->boot.size / sizeof(u32); i++) { segments->boot_signature += bootimage[i]; } return 0; } int gr_gk20a_init_ctxsw_ucode(struct gk20a *g) { struct mm_gk20a *mm = &g->mm; struct vm_gk20a *vm = mm->pmu.vm; struct gk20a_ctxsw_bootloader_desc *fecs_boot_desc; struct gk20a_ctxsw_bootloader_desc *gpccs_boot_desc; struct nvgpu_firmware *fecs_fw; struct nvgpu_firmware *gpccs_fw; u32 *fecs_boot_image; u32 *gpccs_boot_image; struct gk20a_ctxsw_ucode_info *ucode_info = &g->ctxsw_ucode_info; u32 ucode_size; int err = 0; fecs_fw = nvgpu_request_firmware(g, GK20A_FECS_UCODE_IMAGE, 0); if (fecs_fw == NULL) { nvgpu_err(g, "failed to load fecs ucode!!"); return -ENOENT; } fecs_boot_desc = (void *)fecs_fw->data; fecs_boot_image = (void *)(fecs_fw->data + sizeof(struct gk20a_ctxsw_bootloader_desc)); gpccs_fw = nvgpu_request_firmware(g, GK20A_GPCCS_UCODE_IMAGE, 0); if