gpu: nvgpu: implement per-channel watchdog

Implement per-channel watchdog/timer as per below rules :
- start the timer while submitting first job on channel or if
  no timer is already running
- cancel the timer when job completes
- re-start the timer if there is any incomplete job left
  in the channel's queue
- trigger appropriate recovery method as part of timeout
  handling mechanism

Handle the timeout as per below :
- get timed out channel, and job data
- disable activity on all engines
- check if fence is really pending
- get information on failing engine
- if no engine is failing, just abort the channel
- if engine is failing, trigger the recovery

Also, add flag "ch_wdt_enabled" to enable/disable channel
watchdog mechanism. Watchdog can also be disabled using
global flag "timeouts_enabled"

Set the watchdog time to be 5s using macro
NVGPU_CHANNEL_WATCHDOG_DEFAULT_TIMEOUT_MS

Bug 200133289

Change-Id: I401cf14dd34a210bc429f31bd5216a361edf1237
Signed-off-by: Deepak Nibade <dnibade@nvidia.com>
Reviewed-on: http://git-master/r/797072
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Tested-by: Terje Bergstrom <tbergstrom@nvidia.com>

1 files changed, 3 insertions, 3 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index 069ea82a..f736fe8c 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -852,7 +852,7 @@ static bool gk20a_fifo_set_ctx_mmu_error(struct gk20a *g,
         return verbose;
 }
 
-static bool gk20a_fifo_set_ctx_mmu_error_ch(struct gk20a *g,
+bool gk20a_fifo_set_ctx_mmu_error_ch(struct gk20a *g,
                 struct channel_gk20a *ch)
 {
         gk20a_err(dev_from_gk20a(g),
@@ -861,7 +861,7 @@ static bool gk20a_fifo_set_ctx_mmu_error_ch(struct gk20a *g,
         return gk20a_fifo_set_ctx_mmu_error(g, ch);
 }
 
-static bool gk20a_fifo_set_ctx_mmu_error_tsg(struct gk20a *g,
+bool gk20a_fifo_set_ctx_mmu_error_tsg(struct gk20a *g,
                 struct tsg_gk20a *tsg)
 {
         bool ret = true;
@@ -883,7 +883,7 @@ static bool gk20a_fifo_set_ctx_mmu_error_tsg(struct gk20a *g,
         return ret;
 }
 
-static void gk20a_fifo_abort_tsg(struct gk20a *g, u32 tsgid)
+void gk20a_fifo_abort_tsg(struct gk20a *g, u32 tsgid)
 {
         struct tsg_gk20a *tsg = &g->fifo.tsg[tsgid];
         struct channel_gk20a *ch;