gpu: nvgpu: fix channel unbind sequence from TSG

We right now remove a channel from TSG list and disable all the channels in
TSG while removing a channel from TSG
With this sequence if any one channel in TSG is closed, rest of the channels
are set as timed out and cannot be used anymore

We need to fix this sequence as below to allow removing a channel from active
TSG so that rest of the channels can still be used

- disable all channels of TSG
- preempt TSG
- check if CTX_RELOAD is set if support is available
  if CTX_RELOAD is set on channel, it should be moved to some other channel
- check if FAULTED is set if support is available
- if NEXT is set on channel then it means channel is still active
  print out an error in this case for the time being until properly handled
- remove the channel from runlist
- remove channel from TSG list
- re-enable rest of the channels in TSG
- clean up the channel (same as regular channels)

Add below fifo operations to support checking channel status
g->ops.fifo.tsg_verify_status_ctx_reload
g->ops.fifo.tsg_verify_status_faulted

Define ops.fifo.tsg_verify_status_ctx_reload operation for gm20b/gp10b/gp106
as gm20b_fifo_tsg_verify_status_ctx_reload()
This API will check if channel to be released has CTX_RELOAD set, if yes
CTX_RELOAD needs to be moved to some other channel in TSG

Remove static from channel_gk20a_update_runlist() and export it

Bug 200327095

Change-Id: I0dd4be7c7e0b9b759389ec12c5a148a4b919d3e2
Signed-off-by: Deepak Nibade <dnibade@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1560637
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Seshendra Gadagottu <sgadagottu@nvidia.com>
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>

17 files changed, 136 insertions, 21 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
index d0d5c41f..0b8422a6 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
@@ -82,9 +82,6 @@ static void channel_gk20a_joblist_delete(struct channel_gk20a *c,
 static struct channel_gk20a_job *channel_gk20a_joblist_peek(
                 struct channel_gk20a *c);
 
-static int channel_gk20a_update_runlist(struct channel_gk20a *c,
-                                        bool add);
-
 static u32 gk20a_get_channel_watchdog_timeout(struct channel_gk20a *ch);
 
 static void gk20a_channel_clean_up_jobs(struct channel_gk20a *c,
@@ -189,7 +186,7 @@ int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g,
         return 0;
 }
 
-static int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add)
+int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add)
 {
         return c->g->ops.fifo.update_runlist(c->g, c->runlist_id, c->chid, add, true);
 }
@@ -459,6 +456,8 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
         struct dbg_session_gk20a *dbg_s;
         struct dbg_session_data *session_data, *tmp_s;
         struct dbg_session_channel_data *ch_data, *tmp;
+        bool was_tsg = false;
+        int err;
 
         gk20a_dbg_fn("");
 
@@ -467,7 +466,19 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
         trace_gk20a_free_channel(ch->chid);
 
         /* abort channel and remove from runlist */
-        gk20a_disable_channel(ch);
+        if (gk20a_is_channel_marked_as_tsg(ch)) {
+                err = g->ops.fifo.tsg_unbind_channel(ch);
+                if (err)
+                        nvgpu_err(g, "failed to unbind channel %d from TSG", ch->chid);
+                /*
+                 * Channel is not a part of TSG this point onwards
+                 * So stash its status and use it whenever necessary
+                 * e.g. while releasing gr_ctx in g->ops.gr.free_channel_ctx()
+                 */
+                was_tsg = true;
+        } else {
+                gk20a_disable_channel(ch);
+        }
 
         /* wait until there's only our ref to the channel */
         if (!force)
@@ -524,7 +535,7 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
                 g->ops.fecs_trace.unbind_channel(g, ch);
 
         /* release channel ctx */
-        g->ops.gr.free_channel_ctx(ch);
+        g->ops.gr.free_channel_ctx(ch, was_tsg);
 
         gk20a_gr_flush_channel_tlb(gr);
 
@@ -571,9 +582,6 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
         nvgpu_wait_for_deferred_interrupts(g);
 
 unbind:
-        if (gk20a_is_channel_marked_as_tsg(ch))
-                g->ops.fifo.tsg_unbind_channel(ch);
-
         g->ops.fifo.unbind_channel(ch);
         g->ops.fifo.free_inst(g, ch);
 
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
index f022e630..f6ac5780 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
@@ -398,6 +398,7 @@ void channel_gk20a_joblist_lock(struct channel_gk20a *c);
 void channel_gk20a_joblist_unlock(struct channel_gk20a *c);
 bool channel_gk20a_joblist_is_empty(struct channel_gk20a *c);
 
+int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add);
 u32 gk20a_channel_get_timeslice(struct channel_gk20a *ch);
 int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g,
                 int timeslice_period,
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index 2cc5e4cd..1815c15b 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -1899,6 +1899,62 @@ int gk20a_fifo_force_reset_ch(struct channel_gk20a *ch,
         return 0;
 }
 
+static int gk20a_fifo_tsg_unbind_channel_verify_status(struct channel_gk20a *ch)
+{
+        struct gk20a *g = ch->g;
+
+        if (g->ops.fifo.tsg_verify_status_ctx_reload)
+                g->ops.fifo.tsg_verify_status_ctx_reload(ch);
+
+        if (g->ops.fifo.tsg_verify_status_faulted)
+                g->ops.fifo.tsg_verify_status_faulted(ch);
+
+        if (gk20a_fifo_channel_status_is_next(g, ch->chid))
+                nvgpu_err(g, "Channel %d to be removed from TSG has NEXT set!",
+                        ch->chid);
+
+        return 0;
+}
+
+int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch)
+{
+        struct gk20a *g = ch->g;
+        struct fifo_gk20a *f = &g->fifo;
+        struct tsg_gk20a *tsg = &f->tsg[ch->tsgid];
+        int err;
+
+        /* Disable TSG and examine status before unbinding channel */
+        g->ops.fifo.disable_tsg(tsg);
+
+        err = g->ops.fifo.preempt_tsg(g, tsg->tsgid);
+        if (err)
+                goto fail_enable_tsg;
+
+        err = gk20a_fifo_tsg_unbind_channel_verify_status(ch);
+        if (err)
+                goto fail_enable_tsg;
+
+        /* Channel should be seen as TSG channel while updating runlist */
+        err = channel_gk20a_update_runlist(ch, false);
+        if (err)
+                goto fail_enable_tsg;
+
+        /* Remove channel from TSG and re-enable rest of the channels */
+        down_write(&tsg->ch_list_lock);
+        nvgpu_list_del(&ch->ch_entry);
+        up_write(&tsg->ch_list_lock);
+
+        g->ops.fifo.enable_tsg(tsg);
+
+        gk20a_channel_abort_clean_up(ch);
+
+        return 0;
+
+fail_enable_tsg:
+        g->ops.fifo.enable_tsg(tsg);
+        return err;
+}
+
 u32 gk20a_fifo_get_failing_engine_data(struct gk20a *g,
                         int *__id, bool *__is_tsg)
 {
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
index 70c70931..92dcc8e6 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
@@ -273,6 +273,7 @@ int gk20a_fifo_force_reset_ch(struct channel_gk20a *ch,
                                 u32 err_code, bool verbose);
 void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id);
 int gk20a_init_fifo_reset_enable_hw(struct gk20a *g);
+int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch);
 
 void fifo_gk20a_finish_mmu_fault_handling(struct gk20a *g,
                 unsigned long fault_id);
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index 4564b6e9..adc630e6 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -212,7 +212,7 @@ struct gpu_ops {
                 int (*load_ctxsw_ucode)(struct gk20a *g);
                 u32 (*get_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index);
                 void (*set_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index);
-                void (*free_channel_ctx)(struct channel_gk20a *c);
+                void (*free_channel_ctx)(struct channel_gk20a *c, bool is_tsg);
                 int (*alloc_obj_ctx)(struct channel_gk20a  *c,
                                 struct nvgpu_alloc_obj_ctx_args *args);
                 int (*bind_ctxsw_zcull)(struct gk20a *g, struct gr_gk20a *gr,
@@ -477,6 +477,8 @@ struct gpu_ops {
                 int (*preempt_tsg)(struct gk20a *g, u32 tsgid);
                 int (*enable_tsg)(struct tsg_gk20a *tsg);
                 int (*disable_tsg)(struct tsg_gk20a *tsg);
+                void (*tsg_verify_status_ctx_reload)(struct channel_gk20a *ch);
+                void (*tsg_verify_status_faulted)(struct channel_gk20a *ch);
                 int (*reschedule_runlist)(struct gk20a *g, u32 runlist_id);
                 int (*update_runlist)(struct gk20a *g, u32 runlist_id,
                                 u32 chid, bool add,
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
index 27442947..833a3ab9 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
@@ -2868,14 +2868,14 @@ static void gr_gk20a_free_channel_pm_ctx(struct channel_gk20a *c)
         }
 }
 
-void gk20a_free_channel_ctx(struct channel_gk20a *c)
+void gk20a_free_channel_ctx(struct channel_gk20a *c, bool is_tsg)
 {
         if(c->g->ops.fifo.free_channel_ctx_header)
                 c->g->ops.fifo.free_channel_ctx_header(c);
         gr_gk20a_unmap_global_ctx_buffers(c);
         gr_gk20a_free_channel_patch_ctx(c);
         gr_gk20a_free_channel_pm_ctx(c);
-        if (!gk20a_is_channel_marked_as_tsg(c))
+        if (!is_tsg)
                 gr_gk20a_free_channel_gr_ctx(c);
 
         /* zcull_ctx */
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.h b/drivers/gpu/nvgpu/gk20a/gr_gk20a.h
index 42296084..c69d9df9 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.h
@@ -506,7 +506,7 @@ int gk20a_alloc_obj_ctx(struct channel_gk20a *c,
                         struct nvgpu_alloc_obj_ctx_args *args);
 int gk20a_free_obj_ctx(struct channel_gk20a *c,
                         struct nvgpu_free_obj_ctx_args *args);
-void gk20a_free_channel_ctx(struct channel_gk20a *c);
+void gk20a_free_channel_ctx(struct channel_gk20a *c, bool is_tsg);
 
 int gk20a_gr_isr(struct gk20a *g);
 int gk20a_gr_nonstall_isr(struct gk20a *g);
diff --git a/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c b/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c
index eabb98ea..f8c8be5e 100644
--- a/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c
@@ -139,17 +139,20 @@ int gk20a_tsg_bind_channel(struct tsg_gk20a *tsg,
 
 int gk20a_tsg_unbind_channel(struct channel_gk20a *ch)
 {
-        struct fifo_gk20a *f = &ch->g->fifo;
-        struct tsg_gk20a *tsg = &f->tsg[ch->tsgid];
+        struct gk20a *g = ch->g;
+        struct tsg_gk20a *tsg = &g->fifo.tsg[ch->tsgid];
+        int err;
 
-        down_write(&tsg->ch_list_lock);
-        nvgpu_list_del(&ch->ch_entry);
-        up_write(&tsg->ch_list_lock);
+        err = gk20a_fifo_tsg_unbind_channel(ch);
+        if (err)
+                return err;
 
         nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release);
-
         ch->tsgid = NVGPU_INVALID_TSG_ID;
 
+        gk20a_dbg(gpu_dbg_fn, "UNBIND tsg:%d channel:%d\n",
+                                        tsg->tsgid, ch->chid);
+
         return 0;
 }
 
diff --git a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c
index 8e913f23..6b462acd 100644
--- a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c
@@ -183,3 +183,31 @@ void gm20b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f)
         f->intr.pbdma.restartable_0 =
                 pbdma_intr_0_device_pending_f();
 }
+
+static void gm20b_fifo_set_ctx_reload(struct channel_gk20a *ch)
+{
+        struct gk20a *g = ch->g;
+        u32 channel = gk20a_readl(g, ccsr_channel_r(ch->chid));
+
+        gk20a_writel(g, ccsr_channel_r(ch->chid),
+                channel | ccsr_channel_force_ctx_reload_true_f());
+}
+
+void gm20b_fifo_tsg_verify_status_ctx_reload(struct channel_gk20a *ch)
+{
+        struct gk20a *g = ch->g;
+        struct tsg_gk20a *tsg = &g->fifo.tsg[ch->tsgid];
+        struct channel_gk20a *temp_ch;
+
+        /* If CTX_RELOAD is set on a channel, move it to some other channel */
+        if (gk20a_fifo_channel_status_is_ctx_reload(ch->g, ch->chid)) {
+                down_read(&tsg->ch_list_lock);
+                nvgpu_list_for_each_entry(temp_ch, &tsg->ch_list, channel_gk20a, ch_entry) {
+                        if (temp_ch->chid != ch->chid) {
+                                gm20b_fifo_set_ctx_reload(temp_ch);
+                                break;
+                        }
+                }
+                up_read(&tsg->ch_list_lock);
+        }
+}
diff --git a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h
index 1b1b8cc1..f82ae09b 100644
--- a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h
+++ b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h
@@ -25,5 +25,6 @@ void gm20b_device_info_data_parse(struct gk20a *g,
                                                 u32 table_entry, u32 *inst_id,
                                                 u32 *pri_base, u32 *fault_id);
 void gm20b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f);
+void gm20b_fifo_tsg_verify_status_ctx_reload(struct channel_gk20a *ch);
 
 #endif
diff --git a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c
index 46c1e81f..9ff9fdd7 100644
--- a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c
@@ -368,6 +368,7 @@ static const struct gpu_ops gm20b_ops = {
                 .preempt_tsg = gk20a_fifo_preempt_tsg,
                 .enable_tsg = gk20a_enable_tsg,
                 .disable_tsg = gk20a_disable_tsg,
+                .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload,
                 .update_runlist = gk20a_fifo_update_runlist,
                 .trigger_mmu_fault = gm20b_fifo_trigger_mmu_fault,
                 .get_mmu_fault_info = gk20a_fifo_get_mmu_fault_info,
diff --git a/drivers/gpu/nvgpu/gp106/hal_gp106.c b/drivers/gpu/nvgpu/gp106/hal_gp106.c
index e3fa596a..10b26712 100644
--- a/drivers/gpu/nvgpu/gp106/hal_gp106.c
+++ b/drivers/gpu/nvgpu/gp106/hal_gp106.c
@@ -428,6 +428,7 @@ static const struct gpu_ops gp106_ops = {
                 .preempt_tsg = gk20a_fifo_preempt_tsg,
                 .enable_tsg = gk20a_enable_tsg,
                 .disable_tsg = gk20a_disable_tsg,
+                .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload,
                 .update_runlist = gk20a_fifo_update_runlist,
                 .trigger_mmu_fault = gm20b_fifo_trigger_mmu_fault,
                 .get_mmu_fault_info = gp10b_fifo_get_mmu_fault_info,
diff --git a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c
index ffb6fe24..4dae79e1 100644
--- a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c
@@ -389,6 +389,7 @@ static const struct gpu_ops gp10b_ops = {
                 .preempt_tsg = gk20a_fifo_preempt_tsg,
                 .enable_tsg = gk20a_enable_tsg,
                 .disable_tsg = gk20a_disable_tsg,
+                .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload,
                 .reschedule_runlist = gk20a_fifo_reschedule_runlist,
                 .update_runlist = gk20a_fifo_update_runlist,
                 .trigger_mmu_fault = gm20b_fifo_trigger_mmu_fault,
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h
index 3f5d312f..b00979ff 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h
@@ -146,6 +146,10 @@ static inline u32 ccsr_channel_next_true_v(void)
 {
         return 0x00000001;
 }
+static inline u32 ccsr_channel_force_ctx_reload_true_f(void)
+{
+        return 0x100;
+}
 static inline u32 ccsr_channel_busy_v(u32 r)
 {
         return (r >> 28) & 0x1;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h
index 13bd4251..5e67ede0 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h
@@ -146,6 +146,10 @@ static inline u32 ccsr_channel_next_true_v(void)
 {
         return 0x00000001;
 }
+static inline u32 ccsr_channel_force_ctx_reload_true_f(void)
+{
+        return 0x100;
+}
 static inline u32 ccsr_channel_busy_v(u32 r)
 {
         return (r >> 28) & 0x1;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h
index 33c83c80..6cbe44d4 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h
@@ -146,6 +146,10 @@ static inline u32 ccsr_channel_next_true_v(void)
 {
         return 0x00000001;
 }
+static inline u32 ccsr_channel_force_ctx_reload_true_f(void)
+{
+        return 0x100;
+}
 static inline u32 ccsr_channel_busy_v(u32 r)
 {
         return (r >> 28) & 0x1;
diff --git a/drivers/gpu/nvgpu/vgpu/gr_vgpu.c b/drivers/gpu/nvgpu/vgpu/gr_vgpu.c
index e3dfb874..bacd6ded 100644
--- a/drivers/gpu/nvgpu/vgpu/gr_vgpu.c
+++ b/drivers/gpu/nvgpu/vgpu/gr_vgpu.c
@@ -418,7 +418,7 @@ static void vgpu_gr_free_channel_pm_ctx(struct channel_gk20a *c)
         pm_ctx->mem.gpu_va = 0;
 }
 
-static void vgpu_gr_free_channel_ctx(struct channel_gk20a *c)
+static void vgpu_gr_free_channel_ctx(struct channel_gk20a *c, bool is_tsg)
 {
         gk20a_dbg_fn("");
 
@@ -427,7 +427,7 @@ static void vgpu_gr_free_channel_ctx(struct channel_gk20a *c)
         vgpu_gr_unmap_global_ctx_buffers(c);
         vgpu_gr_free_channel_patch_ctx(c);
         vgpu_gr_free_channel_pm_ctx(c);
-        if (!gk20a_is_channel_marked_as_tsg(c))
+        if (!is_tsg)
                 vgpu_gr_free_channel_gr_ctx(c);
 
         /* zcull_ctx, pm_ctx */