From b1d0d8ece83ba0aa7b1e7ea9062eedc5cd9e4e33 Mon Sep 17 00:00:00 2001 From: Seema Khowala Date: Tue, 26 Jun 2018 22:57:02 -0700 Subject: Revert "Revert: GV11B runlist preemption patches" This reverts commit 0b02c8589dcc507865a8fd398431c45fbda2ba9c. Originally change was reverted as it was making ap_compute test on embedded-qnx-hv e3550-t194 fail. With fixes related to replacing tsg preempt with runlist preempt during teardown, preempt timeout set to 100 ms (earlier this was set to 1000ms for t194 and 3000ms for legacy chips) and not issuing preempt timeout recovery if preempt fails, helped resolve the issue. Bug 200426402 Change-Id: If9a68d028a155075444cc1bdf411057e3388d48e Signed-off-by: Seema Khowala Reviewed-on: https://git-master.nvidia.com/r/1762563 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/gv11b/fifo_gv11b.c | 469 +++++++++++++++++++++++------------ 1 file changed, 305 insertions(+), 164 deletions(-) (limited to 'drivers/gpu/nvgpu/gv11b/fifo_gv11b.c') diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c index 4edaaac1..f30f2ae1 100644 --- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c @@ -387,17 +387,24 @@ u32 gv11b_fifo_intr_0_error_mask(struct gk20a *g) u32 gv11b_fifo_get_preempt_timeout(struct gk20a *g) { - return gk20a_get_gr_idle_timeout(g); + /* if timeouts are enabled, using 3000ms timeout + * for polling pdma/eng/runlist might kick in + * timeout handler in the cases where preempt + * is stuck. Use 1000ms timeout for polling when + * timeouts are enabled */ + return nvgpu_is_timeouts_enabled(g) ? PREEMPT_TIMEOUT_1000_MS : + g->gr_idle_timeout_default; } static int gv11b_fifo_poll_pbdma_chan_status(struct gk20a *g, u32 id, - u32 pbdma_id, unsigned int timeout_rc_type) + u32 pbdma_id) { struct nvgpu_timeout timeout; unsigned long delay = GR_IDLE_CHECK_DEFAULT; /* in micro seconds */ u32 pbdma_stat; u32 chan_stat; int ret = -EBUSY; + unsigned int loop_count = 0; /* timeout in milli seconds */ nvgpu_timeout_init(g, &timeout, g->ops.fifo.get_preempt_timeout(g), @@ -406,6 +413,14 @@ static int gv11b_fifo_poll_pbdma_chan_status(struct gk20a *g, u32 id, nvgpu_log(g, gpu_dbg_info, "wait preempt pbdma %d", pbdma_id); /* Verify that ch/tsg is no longer on the pbdma */ do { + if (!nvgpu_platform_is_silicon(g)) { + if (loop_count >= MAX_PRE_SI_RETRIES) { + nvgpu_err(g, "preempt pbdma retries: %u", + loop_count); + break; + } + loop_count++; + } /* * If the PBDMA has a stalling interrupt and receives a NACK, * the PBDMA won't save out until the STALLING interrupt is @@ -458,21 +473,24 @@ static int gv11b_fifo_poll_pbdma_chan_status(struct gk20a *g, u32 id, nvgpu_usleep_range(delay, delay * 2); delay = min_t(unsigned long, delay << 1, GR_IDLE_CHECK_MAX); - } while (!nvgpu_timeout_expired_msg(&timeout, - "preempt timeout pbdma")); + } while (!nvgpu_timeout_expired(&timeout)); + + if (ret) + nvgpu_err(g, "preempt timeout pbdma: %u pbdma_stat: %u " + "tsgid: %u", pbdma_id, pbdma_stat, id); return ret; } static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id, - u32 act_eng_id, u32 *reset_eng_bitmask, - unsigned int timeout_rc_type) + u32 act_eng_id, u32 *reset_eng_bitmask) { struct nvgpu_timeout timeout; unsigned long delay = GR_IDLE_CHECK_DEFAULT; /* in micro seconds */ u32 eng_stat; u32 ctx_stat; int ret = -EBUSY; - bool stall_intr = false; + unsigned int loop_count = 0; + u32 eng_intr_pending; /* timeout in milli seconds */ nvgpu_timeout_init(g, &timeout, g->ops.fifo.get_preempt_timeout(g), @@ -482,20 +500,56 @@ static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id, act_eng_id); /* Check if ch/tsg has saved off the engine or if ctxsw is hung */ do { + if (!nvgpu_platform_is_silicon(g)) { + if (loop_count >= MAX_PRE_SI_RETRIES) { + nvgpu_err(g, "preempt eng retries: %u", + loop_count); + break; + } + loop_count++; + } eng_stat = gk20a_readl(g, fifo_engine_status_r(act_eng_id)); ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); - if (g->ops.mc.is_stall_and_eng_intr_pending(g, act_eng_id)) { - stall_intr = true; + if (g->ops.mc.is_stall_and_eng_intr_pending(g, act_eng_id, + &eng_intr_pending)) { + /* From h/w team + * Engine save can be blocked by eng stalling interrupts. + * FIFO interrupts shouldn’t block an engine save from + * finishing, but could block FIFO from reporting preempt done. + * No immediate reason to reset the engine if FIFO interrupt is + * pending. + * The hub, priv_ring, and ltc interrupts could block context + * switch (or memory), but doesn’t necessarily have to. + * For Hub interrupts they just report access counters and page + * faults. Neither of these necessarily block context switch + * or preemption, but they could. + * For example a page fault for graphics would prevent graphics + * from saving out. An access counter interrupt is a + * notification and has no effect. + * SW should handle page faults though for preempt to complete. + * PRI interrupt (due to a failed PRI transaction) will result + * in ctxsw failure reported to HOST. + * LTC interrupts are generally ECC related and if so, + * certainly don’t block preemption/ctxsw but they could. + * Bus interrupts shouldn’t have anything to do with preemption + * state as they are part of the Host EXT pipe, though they may + * exhibit a symptom that indicates that GPU is in a bad state. + * To be completely fair, when an engine is preempting SW + * really should just handle other interrupts as they come in. + * It’s generally bad to just poll and wait on a preempt + * to complete since there are many things in the GPU which may + * cause a system to hang/stop responding. + */ nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr, "stall intr set, " - "preemption will not finish"); + "preemption might not finish"); } if (ctx_stat == fifo_engine_status_ctx_status_ctxsw_switch_v()) { /* Eng save hasn't started yet. Continue polling */ - if (stall_intr) { - /* if stall intr stop polling */ + if (eng_intr_pending) { + /* if eng intr, stop polling */ *reset_eng_bitmask |= BIT(act_eng_id); ret = 0; break; @@ -507,8 +561,7 @@ static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id, fifo_engine_status_ctx_status_ctxsw_save_v()) { if (id == fifo_engine_status_id_v(eng_stat)) { - if (stall_intr || - timeout_rc_type == PREEMPT_TIMEOUT_NORC) { + if (eng_intr_pending) { /* preemption will not finish */ *reset_eng_bitmask |= BIT(act_eng_id); ret = 0; @@ -524,9 +577,7 @@ static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id, fifo_engine_status_ctx_status_ctxsw_load_v()) { if (id == fifo_engine_status_next_id_v(eng_stat)) { - - if (stall_intr || - timeout_rc_type == PREEMPT_TIMEOUT_NORC) { + if (eng_intr_pending) { /* preemption will not finish */ *reset_eng_bitmask |= BIT(act_eng_id); ret = 0; @@ -546,8 +597,21 @@ static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id, nvgpu_usleep_range(delay, delay * 2); delay = min_t(unsigned long, delay << 1, GR_IDLE_CHECK_MAX); - } while (!nvgpu_timeout_expired_msg(&timeout, - "preempt timeout eng")); + } while (!nvgpu_timeout_expired(&timeout)); + + if (ret) { + /* + * The reasons a preempt can fail are: + * 1.Some other stalling interrupt is asserted preventing + * channel or context save. + * 2.The memory system hangs. + * 3.The engine hangs during CTXSW. + */ + nvgpu_err(g, "preempt timeout eng: %u ctx_stat: %u tsgid: %u", + act_eng_id, ctx_stat, id); + *reset_eng_bitmask |= BIT(act_eng_id); + } + return ret; } @@ -594,29 +658,19 @@ static void gv11b_reset_pbdma_faulted_tsg(struct tsg_gk20a *tsg) } void gv11b_fifo_reset_pbdma_and_eng_faulted(struct gk20a *g, - struct channel_gk20a *refch, + struct tsg_gk20a *tsg, u32 faulted_pbdma, u32 faulted_engine) { - struct tsg_gk20a *tsg; + if (!tsg) + return; nvgpu_log(g, gpu_dbg_intr, "reset faulted pbdma:0x%x eng:0x%x", faulted_pbdma, faulted_engine); - if (!refch) - return; - - if (gk20a_is_channel_marked_as_tsg(refch)) { - tsg = &g->fifo.tsg[refch->tsgid]; - if (faulted_pbdma != FIFO_INVAL_PBDMA_ID) - gv11b_reset_pbdma_faulted_tsg(tsg); - if (faulted_engine != FIFO_INVAL_ENGINE_ID) - gv11b_reset_eng_faulted_tsg(tsg); - } else { - if (faulted_pbdma != FIFO_INVAL_PBDMA_ID) - gv11b_reset_pbdma_faulted_ch(g, refch->chid); - if (faulted_engine != FIFO_INVAL_ENGINE_ID) - gv11b_reset_eng_faulted_ch(g, refch->chid); - } + if (faulted_pbdma != FIFO_INVAL_PBDMA_ID) + gv11b_reset_pbdma_faulted_tsg(tsg); + if (faulted_engine != FIFO_INVAL_ENGINE_ID) + gv11b_reset_eng_faulted_tsg(tsg); } static u32 gv11b_fifo_get_runlists_mask(struct gk20a *g, u32 act_eng_bitmask, @@ -626,7 +680,7 @@ static u32 gv11b_fifo_get_runlists_mask(struct gk20a *g, u32 act_eng_bitmask, u32 runlists_mask = 0; struct fifo_gk20a *f = &g->fifo; struct fifo_runlist_info_gk20a *runlist; - u32 pbdma_bitmask = 0; + u32 rlid, pbdma_bitmask = 0; if (id_type != ID_TYPE_UNKNOWN) { if (id_type == ID_TYPE_TSG) @@ -641,31 +695,31 @@ static u32 gv11b_fifo_get_runlists_mask(struct gk20a *g, u32 act_eng_bitmask, if (mmfault->faulted_pbdma != FIFO_INVAL_PBDMA_ID) pbdma_bitmask = BIT(mmfault->faulted_pbdma); - for (id = 0; id < f->max_runlists; id++) { + for (rlid = 0; rlid < f->max_runlists; rlid++) { - runlist = &f->runlist_info[id]; + runlist = &f->runlist_info[rlid]; if (runlist->eng_bitmask & act_eng_bitmask) runlists_mask |= - fifo_sched_disable_runlist_m(id); + fifo_sched_disable_runlist_m(rlid); if (runlist->pbdma_bitmask & pbdma_bitmask) runlists_mask |= - fifo_sched_disable_runlist_m(id); + fifo_sched_disable_runlist_m(rlid); } } if (id_type == ID_TYPE_UNKNOWN) { - for (id = 0; id < f->max_runlists; id++) { + for (rlid = 0; rlid < f->max_runlists; rlid++) { if (act_eng_bitmask) { /* eng ids are known */ - runlist = &f->runlist_info[id]; + runlist = &f->runlist_info[rlid]; if (runlist->eng_bitmask & act_eng_bitmask) runlists_mask |= - fifo_sched_disable_runlist_m(id); + fifo_sched_disable_runlist_m(rlid); } else { runlists_mask |= - fifo_sched_disable_runlist_m(id); + fifo_sched_disable_runlist_m(rlid); } } } @@ -697,10 +751,20 @@ static int gv11b_fifo_poll_runlist_preempt_pending(struct gk20a *g, struct nvgpu_timeout timeout; u32 delay = GR_IDLE_CHECK_DEFAULT; int ret = -EBUSY; + unsigned int loop_count = 0; nvgpu_timeout_init(g, &timeout, g->ops.fifo.get_preempt_timeout(g), NVGPU_TIMER_CPU_TIMER); do { + if (!nvgpu_platform_is_silicon(g)) { + if (loop_count >= MAX_PRE_SI_RETRIES) { + nvgpu_err(g, "preempt runlist retries: %u", + loop_count); + break; + } + loop_count++; + } + if (!((gk20a_readl(g, fifo_runlist_preempt_r())) & runlists_mask)) { ret = 0; @@ -710,13 +774,16 @@ static int gv11b_fifo_poll_runlist_preempt_pending(struct gk20a *g, nvgpu_usleep_range(delay, delay * 2); delay = min_t(unsigned long, delay << 1, GR_IDLE_CHECK_MAX); - } while (!nvgpu_timeout_expired_msg(&timeout, - "runlist preempt timeout")); + } while (!nvgpu_timeout_expired(&timeout)); + + if (ret) + nvgpu_err(g, "preempt runlist timeout, runlists_mask:0x%08x", + runlists_mask); return ret; } int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, - unsigned int id_type, unsigned int timeout_rc_type) + unsigned int id_type) { struct fifo_gk20a *f = &g->fifo; unsigned long runlist_served_pbdmas; @@ -724,7 +791,6 @@ int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, u32 pbdma_id; u32 act_eng_id; u32 runlist_id; - int func_ret; int ret = 0; u32 tsgid; @@ -741,30 +807,14 @@ int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, runlist_served_pbdmas = f->runlist_info[runlist_id].pbdma_bitmask; runlist_served_engines = f->runlist_info[runlist_id].eng_bitmask; - for_each_set_bit(pbdma_id, &runlist_served_pbdmas, f->num_pbdma) { - - func_ret = gv11b_fifo_poll_pbdma_chan_status(g, tsgid, pbdma_id, - timeout_rc_type); - if (func_ret != 0) { - nvgpu_log_info(g, "preempt timeout pbdma %d", pbdma_id); - ret |= func_ret; - } - } + for_each_set_bit(pbdma_id, &runlist_served_pbdmas, f->num_pbdma) + ret |= gv11b_fifo_poll_pbdma_chan_status(g, tsgid, pbdma_id); f->runlist_info[runlist_id].reset_eng_bitmask = 0; - for_each_set_bit(act_eng_id, &runlist_served_engines, f->max_engines) { - - func_ret = gv11b_fifo_poll_eng_ctx_status(g, tsgid, act_eng_id, - &f->runlist_info[runlist_id].reset_eng_bitmask, - timeout_rc_type); - - if (func_ret != 0) { - nvgpu_log_info(g, "preempt timeout engine %d", act_eng_id); - ret |= func_ret; - } - } - + for_each_set_bit(act_eng_id, &runlist_served_engines, f->max_engines) + ret |= gv11b_fifo_poll_eng_ctx_status(g, tsgid, act_eng_id, + &f->runlist_info[runlist_id].reset_eng_bitmask); return ret; } @@ -848,6 +898,9 @@ int gv11b_fifo_preempt_tsg(struct gk20a *g, u32 tsgid) nvgpu_mutex_acquire(&f->runlist_info[runlist_id].runlist_lock); + /* WAR for Bug 2065990 */ + gk20a_fifo_disable_tsg_sched(g, &f->tsg[tsgid]); + mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); ret = __locked_fifo_preempt(g, tsgid, true); @@ -855,6 +908,9 @@ int gv11b_fifo_preempt_tsg(struct gk20a *g, u32 tsgid) if (!mutex_ret) nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + /* WAR for Bug 2065990 */ + gk20a_fifo_enable_tsg_sched(g, &f->tsg[tsgid]); + nvgpu_mutex_release(&f->runlist_info[runlist_id].runlist_lock); if (ret) @@ -863,44 +919,36 @@ int gv11b_fifo_preempt_tsg(struct gk20a *g, u32 tsgid) return ret; } -static int gv11b_fifo_preempt_runlists(struct gk20a *g, u32 runlists_mask) +static void gv11b_fifo_locked_preempt_runlists(struct gk20a *g, u32 runlists_mask) { int ret = 0; u32 token = PMU_INVALID_MUTEX_OWNER_ID; u32 mutex_ret = 0; - u32 runlist_id; - - nvgpu_log_fn(g, " "); + u32 rlid; - for (runlist_id = 0; runlist_id < g->fifo.max_runlists; runlist_id++) { - if (runlists_mask & fifo_runlist_preempt_runlist_m(runlist_id)) - nvgpu_mutex_acquire(&g->fifo. - runlist_info[runlist_id].runlist_lock); - } + /* runlist_lock are locked by teardown and sched are disabled too */ + nvgpu_log_fn(g, "preempt runlists_mask:0x%08x", runlists_mask); mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); ret = __locked_fifo_preempt_runlists(g, runlists_mask); - if (!mutex_ret) - nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); - - for (runlist_id = 0; runlist_id < g->fifo.max_runlists; runlist_id++) { - if (runlists_mask & - fifo_runlist_preempt_runlist_m(runlist_id)) { - /* during recovery reset engs served by this runlist */ - g->fifo.runlist_info[runlist_id].reset_eng_bitmask = - g->fifo.runlist_info[runlist_id].eng_bitmask; - nvgpu_mutex_release(&g->fifo. - runlist_info[runlist_id].runlist_lock); + if (ret) { + /* if preempt timed out, reset engs served by runlists */ + for (rlid = 0; rlid < g->fifo.max_runlists; rlid++) { + if (runlists_mask & + fifo_runlist_preempt_runlist_m(rlid)) + g->fifo.runlist_info[rlid].reset_eng_bitmask = + g->fifo.runlist_info[rlid].eng_bitmask; } } - return ret; + if (!mutex_ret) + nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); } static int __locked_fifo_preempt_ch_tsg(struct gk20a *g, u32 id, - unsigned int id_type, unsigned int timeout_rc_type) + unsigned int id_type) { int ret; struct fifo_gk20a *f = &g->fifo; @@ -914,52 +962,97 @@ static int __locked_fifo_preempt_ch_tsg(struct gk20a *g, u32 id, gk20a_fifo_issue_preempt(g, id, true); /* wait for preempt */ - ret = g->ops.fifo.is_preempt_pending(g, id, id_type, - timeout_rc_type); + ret = g->ops.fifo.is_preempt_pending(g, id, id_type); - if (ret && (timeout_rc_type == PREEMPT_TIMEOUT_RC)) - gk20a_fifo_preempt_timeout_rc(g, id, id_type); + /* No recovery even if preempt timed out since + * this is called from recovery path + */ return ret; } int gv11b_fifo_preempt_ch_tsg(struct gk20a *g, u32 id, - unsigned int id_type, unsigned int timeout_rc_type) + unsigned int id_type) { - struct fifo_gk20a *f = &g->fifo; u32 ret = 0; u32 token = PMU_INVALID_MUTEX_OWNER_ID; u32 mutex_ret = 0; - u32 runlist_id; - if (id_type == ID_TYPE_TSG) - runlist_id = f->tsg[id].runlist_id; - else if (id_type == ID_TYPE_CHANNEL) - runlist_id = f->channel[id].runlist_id; - else - return -EINVAL; + mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + /* + * This is called from teardown path only. runlist_lock + * is already acquired before calling this function. + */ + ret = __locked_fifo_preempt_ch_tsg(g, id, id_type); - if (runlist_id >= g->fifo.max_runlists) { - nvgpu_log_info(g, "runlist_id = %d", runlist_id); - return -EINVAL; - } + if (!mutex_ret) + nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); - nvgpu_log_fn(g, "preempt id = %d, runlist_id = %d", id, runlist_id); + return ret; - nvgpu_mutex_acquire(&f->runlist_info[runlist_id].runlist_lock); +} + +static void gv11b_fifo_locked_abort_runlist_active_tsgs(struct gk20a *g, + unsigned int rc_type, + u32 runlists_mask) +{ + struct tsg_gk20a *tsg = NULL; + u32 rlid, tsgid; + struct fifo_runlist_info_gk20a *runlist = NULL; + u32 token = PMU_INVALID_MUTEX_OWNER_ID; + u32 mutex_ret = 0; + bool add = false, wait_for_finish = false; + int err; + nvgpu_err(g, "runlist id unknown, abort active tsgs in runlists"); + + /* runlist_lock are locked by teardown */ mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); - ret = __locked_fifo_preempt_ch_tsg(g, id, id_type, timeout_rc_type); + for (rlid = 0; rlid < g->fifo.max_runlists; + rlid++) { + if (!(runlists_mask & BIT(rlid))) + continue; + nvgpu_log(g, gpu_dbg_info, "abort runlist id %d", + rlid); + runlist = &g->fifo.runlist_info[rlid]; + + for_each_set_bit(tsgid, runlist->active_tsgs, + g->fifo.num_channels) { + nvgpu_log(g, gpu_dbg_info, "abort tsg id %d", tsgid); + tsg = &g->fifo.tsg[tsgid]; + gk20a_disable_tsg(tsg); - if (!mutex_ret) - nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + /* assume all pbdma and eng faulted are set */ + nvgpu_log(g, gpu_dbg_info, "reset pbdma and eng faulted"); + gv11b_reset_pbdma_faulted_tsg(tsg); + gv11b_reset_eng_faulted_tsg(tsg); - nvgpu_mutex_release(&f->runlist_info[runlist_id].runlist_lock); +#ifdef CONFIG_GK20A_CTXSW_TRACE + gk20a_ctxsw_trace_tsg_reset(g, tsg); +#endif + if (!g->fifo.deferred_reset_pending) { + if (rc_type == RC_TYPE_MMU_FAULT) { + gk20a_fifo_set_ctx_mmu_error_tsg(g, tsg); + gk20a_fifo_error_tsg(g, tsg); + } + } - return ret; + /* (chid == ~0 && !add) remove all act ch from runlist*/ + err = gk20a_fifo_update_runlist_locked(g, rlid, + FIFO_INVAL_CHANNEL_ID, add, wait_for_finish); + if (err) + nvgpu_err(g, "runlist id %d is not cleaned up", + rlid); + gk20a_fifo_abort_tsg(g, tsg->tsgid, false); + + nvgpu_log(g, gpu_dbg_info, "aborted tsg id %d", tsgid); + } + } + if (!mutex_ret) + nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); } void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, @@ -967,10 +1060,66 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, struct mmu_fault_info *mmfault) { struct tsg_gk20a *tsg = NULL; - struct channel_gk20a *refch = NULL; - u32 runlists_mask, runlist_id; + u32 runlists_mask, rlid; struct fifo_runlist_info_gk20a *runlist = NULL; u32 engine_id, client_type = ~0; + struct fifo_gk20a *f = &g->fifo; + u32 runlist_id = FIFO_INVAL_RUNLIST_ID; + u32 num_runlists = 0; + + nvgpu_log_fn(g, "acquire runlist_lock for all runlists"); + for (rlid = 0; rlid < g->fifo.max_runlists; rlid++) + nvgpu_mutex_acquire(&f->runlist_info[rlid]. + runlist_lock); + + /* get runlist id and tsg */ + if (id_type == ID_TYPE_TSG) { + if (id != FIFO_INVAL_TSG_ID) { + tsg = &g->fifo.tsg[id]; + runlist_id = tsg->runlist_id; + if (runlist_id != FIFO_INVAL_RUNLIST_ID) + num_runlists++; + else + nvgpu_log_fn(g, "tsg runlist id is invalid"); + } else { + nvgpu_log_fn(g, "id type is tsg but tsg id is inval"); + } + } else { + /* + * id type is unknown, get runlist_id if eng mask is such that + * it corresponds to single runlist id. If eng mask corresponds + * to multiple runlists, then abort all runlists + */ + for (rlid = 0; rlid < f->max_runlists; rlid++) { + if (act_eng_bitmask) { + /* eng ids are known */ + runlist = &f->runlist_info[rlid]; + if (runlist->eng_bitmask & act_eng_bitmask) { + runlist_id = rlid; + num_runlists++; + } + } else { + break; + } + } + if (num_runlists > 1 ) /* abort all runlists */ + runlist_id = FIFO_INVAL_RUNLIST_ID; + } + + /* if runlist_id is valid and there is only single runlist to be + * aborted, release runlist lock that are not + * needed for this recovery + */ + if (runlist_id != FIFO_INVAL_RUNLIST_ID && num_runlists == 1) { + for (rlid = 0; rlid < g->fifo.max_runlists; rlid++) { + if (rlid != runlist_id) { + nvgpu_log_fn(g, "release runlist_lock for " + "unused runlist id: %d", rlid); + nvgpu_mutex_release(&f->runlist_info[rlid]. + runlist_lock); + } + } + } nvgpu_log(g, gpu_dbg_info, "id = %d, id_type = %d, rc_type = %d, " "act_eng_bitmask = 0x%x, mmfault ptr = 0x%p", @@ -979,6 +1128,7 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, runlists_mask = gv11b_fifo_get_runlists_mask(g, act_eng_bitmask, id, id_type, rc_type, mmfault); + /* Disable runlist scheduler */ gk20a_fifo_set_runlist_state(g, runlists_mask, RUNLIST_DISABLED); g->fifo.deferred_reset_pending = false; @@ -1000,41 +1150,41 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); - /* Get tsg/ch */ if (rc_type == RC_TYPE_MMU_FAULT) { gk20a_debug_dump(g); - refch = mmfault->refch; client_type = mmfault->client_type; - gv11b_fifo_reset_pbdma_and_eng_faulted(g, refch, - mmfault->faulted_pbdma, - mmfault->faulted_engine); + gv11b_fifo_reset_pbdma_and_eng_faulted(g, tsg, + mmfault->faulted_pbdma, + mmfault->faulted_engine); } - if (id_type == ID_TYPE_TSG) { - tsg = &g->fifo.tsg[id]; - } else if (id_type == ID_TYPE_CHANNEL) { - if (refch == NULL) - refch = gk20a_channel_get(&g->fifo.channel[id]); - } - /* Disable tsg/ch */ if (tsg) gk20a_disable_tsg(tsg); - else if (refch) - g->ops.fifo.disable_channel(refch); - /* Preempt tsg/ch */ - if (id_type == ID_TYPE_TSG || id_type == ID_TYPE_CHANNEL) { - g->ops.fifo.preempt_ch_tsg(g, id, id_type, - PREEMPT_TIMEOUT_NORC); + /* + * Even though TSG preempt timed out, the RC sequence would by design + * require s/w to issue another preempt. + * If recovery includes an ENGINE_RESET, to not have race conditions, + * use RUNLIST_PREEMPT to kick all work off, and cancel any context + * load which may be pending. This is also needed to make sure + * that all PBDMAs serving the engine are not loaded when engine is + * reset. + */ + if (tsg) { + int preempt_failed; + + preempt_failed = g->ops.fifo.preempt_ch_tsg(g, id, id_type); + if (preempt_failed) + gv11b_fifo_locked_preempt_runlists(g, runlists_mask); } else { - gv11b_fifo_preempt_runlists(g, runlists_mask); + gv11b_fifo_locked_preempt_runlists(g, runlists_mask); } /* check if engine reset should be deferred */ - for (runlist_id = 0; runlist_id < g->fifo.max_runlists; runlist_id++) { + for (rlid = 0; rlid < g->fifo.max_runlists; rlid++) { - runlist = &g->fifo.runlist_info[runlist_id]; - if ((runlists_mask & BIT(runlist_id)) && + runlist = &g->fifo.runlist_info[rlid]; + if ((runlists_mask & BIT(rlid)) && runlist->reset_eng_bitmask) { unsigned long __reset_eng_bitmask = @@ -1042,7 +1192,7 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, for_each_set_bit(engine_id, &__reset_eng_bitmask, g->fifo.max_engines) { - if ((refch || tsg) && + if (tsg && gk20a_fifo_should_defer_engine_reset(g, engine_id, client_type, false)) { @@ -1074,13 +1224,9 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, } #ifdef CONFIG_GK20A_CTXSW_TRACE - /* tsg and refch both could be valid for mmu fault. Check tsg first */ if (tsg) gk20a_ctxsw_trace_tsg_reset(g, tsg); - else if (refch) - gk20a_ctxsw_trace_channel_reset(g, refch); #endif - if (tsg) { if (g->fifo.deferred_reset_pending) { gk20a_disable_tsg(tsg); @@ -1090,26 +1236,9 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, gk20a_fifo_abort_tsg(g, tsg->tsgid, false); } - if (refch) - gk20a_channel_put(refch); - } else if (refch) { - if (g->fifo.deferred_reset_pending) { - g->ops.fifo.disable_channel(refch); - } else { - if (rc_type == RC_TYPE_MMU_FAULT) - gk20a_fifo_set_ctx_mmu_error_ch(g, refch); - - gk20a_channel_abort(refch, false); - } - gk20a_channel_put(refch); } else { - nvgpu_err(g, "id unknown, abort runlist"); - for (runlist_id = 0; runlist_id < g->fifo.max_runlists; - runlist_id++) { - if (runlists_mask & BIT(runlist_id)) - g->ops.fifo.update_runlist(g, runlist_id, - FIFO_INVAL_CHANNEL_ID, false, true); - } + gv11b_fifo_locked_abort_runlist_active_tsgs(g, rc_type, + runlists_mask); } gk20a_fifo_set_runlist_state(g, runlists_mask, RUNLIST_ENABLED); @@ -1117,6 +1246,18 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, /* It is safe to enable ELPG again. */ if (g->support_pmu && g->elpg_enabled) nvgpu_pmu_enable_elpg(g); + + /* release runlist_lock */ + if (runlist_id != FIFO_INVAL_RUNLIST_ID) { + nvgpu_log_fn(g, "release runlist_lock runlist_id = %d", + runlist_id); + nvgpu_mutex_release(&f->runlist_info[runlist_id].runlist_lock); + } else { + nvgpu_log_fn(g, "release runlist_lock for all runlists"); + for (rlid = 0; rlid < g->fifo.max_runlists; rlid++) + nvgpu_mutex_release(&f->runlist_info[rlid]. + runlist_lock); + } } void gv11b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f) -- cgit v1.2.2