17 files changed, 136 insertions, 21 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
index d0d5c41f..0b8422a6 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
@@ -82,9 +82,6 @@ static void channel_gk20a_joblist_delete(struct channel_gk20a *c,
 static struct channel_gk20a_job *channel_gk20a_joblist_peek(
                 struct channel_gk20a *c);
 
-static int channel_gk20a_update_runlist(struct channel_gk20a *c,
-                                        bool add);
-
 static u32 gk20a_get_channel_watchdog_timeout(struct channel_gk20a *ch);
 
 static void gk20a_channel_clean_up_jobs(struct channel_gk20a *c,
@@ -189,7 +186,7 @@ int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g,
         return 0;
 }
 
-static int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add)
+int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add)
 {
         return c->g->ops.fifo.update_runlist(c->g, c->runlist_id, c->chid, add, true);
 }
@@ -459,6 +456,8 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
         struct dbg_session_gk20a *dbg_s;
         struct dbg_session_data *session_data, *tmp_s;
         struct dbg_session_channel_data *ch_data, *tmp;
+        bool was_tsg = false;
+        int err;
 
         gk20a_dbg_fn("");
 
@@ -467,7 +466,19 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
         trace_gk20a_free_channel(ch->chid);
 
         /* abort channel and remove from runlist */
-        gk20a_disable_channel(ch);
+        if (gk20a_is_channel_marked_as_tsg(ch)) {
+                err = g->ops.fifo.tsg_unbind_channel(ch);
+                if (err)
+                        nvgpu_err(g, "failed to unbind channel %d from TSG", ch->chid);
+                /*
+                 * Channel is not a part of TSG this point onwards
+                 * So stash its status and use it whenever necessary
+                 * e.g. while releasing gr_ctx in g->ops.gr.free_channel_ctx()
+                 */
+                was_tsg = true;
+        } else {
+                gk20a_disable_channel(ch);
+        }
 
         /* wait until there's only our ref to the channel */
         if (!force)
@@ -524,7 +535,7 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
                 g->ops.fecs_trace.unbind_channel(g, ch);
 
         /* release channel ctx */
-        g->ops.gr.free_channel_ctx(ch);
+        g->ops.gr.free_channel_ctx(ch, was_tsg);
 
         gk20a_gr_flush_channel_tlb(gr);
 
@@ -571,9 +582,6 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
         nvgpu_wait_for_deferred_interrupts(g);
 
 unbind:
-        if (gk20a_is_channel_marked_as_tsg(ch))
-                g->ops.fifo.tsg_unbind_channel(ch);
-
         g->ops.fifo.unbind_channel(ch);
         g->ops.fifo.free_inst(g, ch);
 
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
index f022e630..f6ac5780 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
@@ -398,6 +398,7 @@ void channel_gk20a_joblist_lock(struct channel_gk20a *c);
 void channel_gk20a_joblist_unlock(struct channel_gk20a *c);
 bool channel_gk20a_joblist_is_empty(struct channel_gk20a *c);
 
+int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add);
 u32 gk20a_channel_get_timeslice(struct channel_gk20a *ch);
 int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g,
                 int timeslice_period,
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index 2cc5e4cd..1815c15b 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -1899,6 +1899,62 @@ int gk20a_fifo_force_reset_ch(struct channel_gk20a *ch,
         return 0;
 }
 
+static int gk20a_fifo_tsg_unbind_channel_verify_status(struct channel_gk20a *ch)
+{
+        struct gk20a *g = ch->g;
+
+        if (g->ops.fifo.tsg_verify_status_ctx_reload)
+                g->ops.fifo.tsg_verify_status_ctx_reload(ch);
+
+        if (g->ops.fifo.tsg_verify_status_faulted)
+                g->ops.fifo.tsg_verify_status_faulted(ch);
+
+        if (gk20a_fifo_channel_status_is_next(g, ch->chid))
+                nvgpu_err(g, "Channel %d to be removed from TSG has NEXT set!",
+                        ch->chid);
+
+        return 0;
+}
+
+int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch)
+{
+        struct gk20a *g = ch->g;
+        struct fifo_gk20a *f = &g->fifo;
+        struct tsg_gk20a *tsg = &f->tsg[ch->tsgid];
+        int err;
+
+        /* Disable TSG and examine status before unbinding channel */
+        g->ops.fifo.disable_tsg(tsg);
+
+        err = g->ops.fifo.preempt_tsg(g, tsg->tsgid);
+        if (err)
+                goto fail_enable_tsg;
+
+        err = gk20a_fifo_tsg_unbind_channel_verify_status(ch);
+        if (err)
+                goto fail_enable_tsg;
+
+        /* Channel should be seen as TSG channel while updating runlist */
+        err = channel_gk20a_update_runlist(ch, false);
+        if (err)
+                goto fail_enable_tsg;
+
+        /* Remove channel from TSG and re-enable rest of the channels */
+        down_write(&tsg->ch_list_lock);
+        nvgpu_list_del(&ch->ch_entry);
+        up_write(&tsg->ch_list_lock);
+
+        g->ops.fifo.enable_tsg(tsg);
+
+        gk20a_channel_abort_clean_up(ch);
+
+        return 0;
+
+fail_enable_tsg:
+        g->ops.fifo.enable_tsg(tsg);
+        return err;
+}
+
 u32 gk20a_fifo_get_failing_engine_data(struct gk20a *g,
                         int *__id, bool *__is_tsg)
 {
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
index 70c70931..92dcc8e6 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
@@ -273,6 +273,7 @@ int gk20a_fifo_force_reset_ch(struct channel_gk20a *ch,
                                 u32 err_code, bool verbose);
 void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id);
 int gk20a_init_fifo_reset_enable_hw(struct gk20a *g);
+int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch);
 
 void fifo_gk20a_finish_mmu_fault_handling(struct gk20a *g,
                 unsigned long fault_id);
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index 4564b6e9..adc630e6 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -212,7 +212,7 @@ struct gpu_ops {
                 int (*load_ctxsw_ucode)(struct gk20a *g);
                 u32 (*get_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index);
                 void (*set_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index);
-                void (*free_channel_ctx)(struct channel_gk20a *c);
+                void (*free_channel_ctx)(struct channel_gk20a *c, bool is_tsg);
                 int (*alloc_obj_ctx)(struct channel_gk20a  *c,
                                 struct nvgpu_alloc_obj_ctx_args *args);
                 int (*bind_ctxsw_zcull)(struct gk20a *g, struct gr_gk20a *gr,
@@ -477,6 +477,8 @@ struct gpu_ops {
                 int (*preempt_tsg)(struct gk20a *g, u32 tsgid);
                 int (*enable_tsg)(struct tsg_gk20a *tsg);
                 int (*disable_tsg)(struct tsg_gk20a *tsg);
+                void (*tsg_verify_status_ctx_reload)(struct channel_gk20a *ch);
+                void (*tsg_verify_status_faulted)(struct channel_gk20a *ch);
                 int (*reschedule_runlist)(struct gk20a *g, u32 runlist_id);
                 int (*update_runlist)(struct gk20a *g, u32 runlist_id,
                                 u32 chid, bool add,
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
index 27442947..833a3ab9 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
@@ -2868,14 +2868,14 @@ static void gr_gk20a_free_channel_pm_ctx(struct channel_gk20a *c)
         }
 }
 
-void gk20a_free_channel_ctx(struct channel_gk20a *c)
+void gk20a_free_channel_ctx(struct channel_gk20a *c, bool is_tsg)
 {
         if(c->g->ops.fifo.free_channel_ctx_header)
                 c->g->ops.fifo.free_channel_ctx_header(c);
         gr_gk20a_unmap_global_ctx_buffers(c);
         gr_gk20a_free_channel_patch_ctx(c);
         gr_gk20a_free_channel_pm_ctx(c);
-        if (!gk20a_is_channel_marked_as_tsg(c))
+        if (!is_tsg)
                 gr_gk20a_free_channel_gr_ctx(c);
 
         /* zcull_ctx */
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.h b/drivers/gpu/nvgpu/gk20a/gr_gk20a.h
index 42296084..c69d9df9 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.h
@@ -506,7 +506,7 @@ int gk20a_alloc_obj_ctx(struct channel_gk20a *c,
                         struct nvgpu_alloc_obj_ctx_args *args);
 int gk20a_free_obj_ctx(struct channel_gk20a *c,
                         struct nvgpu_free_obj_ctx_args *args);
-void gk20a_free_channel_ctx(struct channel_gk20a *c);
+void gk20a_free_channel_ctx(struct channel_gk20a *c, bool is_tsg);
 
 int gk20a_gr_isr(struct gk20a *g);
 int gk20a_gr_nonstall_isr(struct gk20a *g);
diff --git a/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c b/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c
index eabb98ea..f8c8be5e 100644
--- a/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c
@@ -139,17 +139,20 @@ int gk20a_tsg_bind_channel(struct tsg_gk20a *tsg,
 
 int gk20a_tsg_unbind_channel(struct channel_gk20a *ch)
 {
-        struct fifo_gk20a *f = &ch->g->fifo;
-        struct tsg_gk20a *tsg = &f->tsg[ch->tsgid];
+        struct gk20a *g = ch->g;
+        struct tsg_gk20a *tsg = &g->fifo.tsg[ch->tsgid];
+        int err;
 
-        down_write(&tsg->ch_list_lock);
-        nvgpu_list_del(&ch->ch_entry);
-        up_write(&tsg->ch_list_lock);
+        err = gk20a_fifo_tsg_unbind_channel(ch);
+        if (err)
+                return err;
 
         nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release);
-
         ch->tsgid = NVGPU_INVALID_TSG_ID;
 
+        gk20a_dbg(gpu_dbg_fn, "UNBIND tsg:%d channel:%d\n",
+                                        tsg->tsgid, ch->chid);
+
         return 0;
 }
 
diff --git a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c
index 8e913f23..6b462acd 100644
--- a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c
@@ -183,3 +183,31 @@ void gm20b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f)
         f->intr.pbdma.restartable_0 =
                 pbdma_intr_0_device_pending_f();
 }
+
+static void gm20b_fifo_set_ctx_reload(struct channel_gk20a *ch)
+{
+        struct gk20a *g = ch->g;
+        u32 channel = gk20a_readl(g, ccsr_channel_r(ch->chid));
+
+        gk20a_writel(g, ccsr_channel_r(ch->chid),
+                channel | ccsr_channel_force_ctx_reload_true_f());
+}
+
+void gm20b_fifo_tsg_verify_status_ctx_reload(struct channel_gk20a *ch)
+{
+        struct gk20a *g = ch->g;
+        struct tsg_gk20a *tsg = &g->fifo.tsg[ch->tsgid];
+        struct channel_gk20a *temp_ch;
+
+        /* If CTX_RELOAD is set on a channel, move it to some other channel */
+        if (gk20a_fifo_channel_status_is_ctx_reload(ch->g, ch->chid)) {
+                down_read(&tsg->ch_list_lock);
+                nvgpu_list_for_each_entry(temp_ch, &tsg->ch_list, channel_gk20a, ch_entry) {
+                        if (temp_ch->chid != ch->chid) {
+                                gm20b_fifo_set_ctx_reload(temp_ch);
+                                break;
+                        }
+                }
+                up_read(&tsg->ch_list_lock);
+        }
+}
diff --git a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h
index 1b1b8cc1..f82ae09b 100644
--- a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h
+++ b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h
@@ -25,5 +25,6 @@ void gm20b_device_info_data_parse(struct gk20a *g,
                                                 u32 table_entry, u32 *inst_id,
                                                 u32 *pri_base, u32 *fault_id);
 void gm20b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f);
+void gm20b_fifo_tsg_verify_status_ctx_reload(struct channel_gk20a *ch);
 
 #endif
diff --git a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c
index 46c1e81f..9ff9fdd7 100644
--- a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c
@@ -368,6 +368,7 @@ static const struct gpu_ops gm20b_ops = {
                 .preempt_tsg = gk20a_fifo_preempt_tsg,
                 .enable_tsg = gk20a_enable_tsg,
                 .disable_tsg = gk20a_disable_tsg,
+                .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload,
                 .update_runlist = gk20a_fifo_update_runlist,
                 .trigger_mmu_fault = gm20b_fifo_trigger_mmu_fault,
                 .get_mmu_fault_info = gk20a_fifo_get_mmu_fault_info,
diff --git a/drivers/gpu/nvgpu/gp106/hal_gp106.c b/drivers/gpu/nvgpu/gp106/hal_gp106.c
index e3fa596a..10b26712 100644
--- a/drivers/gpu/nvgpu/gp106/hal_gp106.c
+++ b/drivers/gpu/nvgpu/gp106/hal_gp106.c
@@ -428,6 +428,7 @@ static const struct gpu_ops gp106_ops = {
                 .preempt_tsg = gk20a_fifo_preempt_tsg,
                 .enable_tsg = gk20a_enable_tsg,
                 .disable_tsg = gk20a_disable_tsg,
+                .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload,
                 .update_runlist = gk20a_fifo_update_runlist,
                 .trigger_mmu_fault = gm20b_fifo_trigger_mmu_fault,
                 .get_mmu_fault_info = gp10b_fifo_get_mmu_fault_info,
diff --git a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c
index ffb6fe24..4dae79e1 100644
--- a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c
@@ -389,6 +389,7 @@ static const struct gpu_ops gp10b_ops = {
                 .preempt_tsg = gk20a_fifo_preempt_tsg,
                 .enable_tsg = gk20a_enable_tsg,
                 .disable_tsg = gk20a_disable_tsg,
+                .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload,
                 .reschedule_runlist = gk20a_fifo_reschedule_runlist,
                 .update_runlist = gk20a_fifo_update_runlist,
                 .trigger_mmu_fault = gm20b_fifo_trigger_mmu_fault,
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h
index 3f5d312f..b00979ff 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h
@@ -146,6 +146,10 @@ static inline u32 ccsr_channel_next_true_v(void)
 {
         return 0x00000001;
 }
+static inline u32 ccsr_channel_force_ctx_reload_true_f(void)
+{
+        return 0x100;
+}
 static inline u32 ccsr_channel_busy_v(u32 r)
 {
         return (r >> 28) & 0x1;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h
index 13bd4251..5e67ede0 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h
@@ -146,6 +146,10 @@ static inline u32 ccsr_channel_next_true_v(void)
 {
         return 0x00000001;
 }
+static inline u32 ccsr_channel_force_ctx_reload_true_f(void)
+{
+        return 0x100;
+}
 static inline u32 ccsr_channel_busy_v(u32 r)
 {
         return (r >> 28) & 0x1;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h
index 33c83c80..6cbe44d4 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h
@@ -146,6 +146,10 @@ static inline u32 ccsr_channel_next_true_v(void)
 {
         return 0x00000001;
 }
+static inline u32 ccsr_channel_force_ctx_reload_true_f(void)
+{
+        return 0x100;
+}
 static inline u32 ccsr_channel_busy_v(u32 r)
 {
         return (r >> 28) & 0x1;
diff --git a/drivers/gpu/nvgpu/vgpu/gr_vgpu.c b/drivers/gpu/nvgpu/vgpu/gr_vgpu.c
index e3dfb874..bacd6ded 100644
--- a/drivers/gpu/nvgpu/vgpu/gr_vgpu.c
+++ b/drivers/gpu/nvgpu/vgpu/gr_vgpu.c
@@ -418,7 +418,7 @@ static void vgpu_gr_free_channel_pm_ctx(struct channel_gk20a *c)
         pm_ctx->mem.gpu_va = 0;
 }
 
-static void vgpu_gr_free_channel_ctx(struct channel_gk20a *c)
+static void vgpu_gr_free_channel_ctx(struct channel_gk20a *c, bool is_tsg)
 {
         gk20a_dbg_fn("");
 
@@ -427,7 +427,7 @@ static void vgpu_gr_free_channel_ctx(struct channel_gk20a *c)
         vgpu_gr_unmap_global_ctx_buffers(c);
         vgpu_gr_free_channel_patch_ctx(c);
         vgpu_gr_free_channel_pm_ctx(c);
-        if (!gk20a_is_channel_marked_as_tsg(c))
+        if (!is_tsg)
                 vgpu_gr_free_channel_gr_ctx(c);
 
         /* zcull_ctx, pm_ctx */