gpu: nvgpu: fix TSG enable sequence

Due to a h/w bug in Maxwell and Pascal we first need to enable all channels with
NEXT and CTX_RELOAD set in a TSG, and then rest of the channels should be
enabled
Add this sequence to gk20a_tsg_enable()

Add new APIs to enable/disable scheduling of TSG runlist
gk20a_fifo_enable_tsg_sched()
gk20a_fifo_disble_tsg_sched()

Add new APIs to check if channel has NEXT or CTX_RELOAD set
gk20a_fifo_channel_status_is_next()
gk20a_fifo_channel_status_is_ctx_reload()

Bug 1739362

Change-Id: I4891cbd7f22ebc1e0bf32c52801002cdc259dbe1
Signed-off-by: Deepak Nibade <dnibade@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1560636
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Seshendra Gadagottu <sgadagottu@nvidia.com>
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>

7 files changed, 208 insertions, 6 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index 88ce6a83..2cc5e4cd 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -2671,6 +2671,21 @@ void gk20a_fifo_set_runlist_state(struct gk20a *g, u32 runlists_mask,
         gk20a_dbg_fn("done");
 }
 
+void gk20a_fifo_enable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg)
+{
+        gk20a_fifo_set_runlist_state(g, fifo_sched_disable_runlist_m(
+                                        tsg->runlist_id), RUNLIST_ENABLED,
+                                        !RUNLIST_INFO_MUTEX_LOCKED);
+
+}
+
+void gk20a_fifo_disable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg)
+{
+        gk20a_fifo_set_runlist_state(g, fifo_sched_disable_runlist_m(
+                                        tsg->runlist_id), RUNLIST_DISABLED,
+                                        !RUNLIST_INFO_MUTEX_LOCKED);
+}
+
 int gk20a_fifo_enable_engine_activity(struct gk20a *g,
                                 struct fifo_engine_info_gk20a *eng_info)
 {
@@ -3413,6 +3428,27 @@ const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index)
                 return pbdma_chan_eng_ctx_status_str[index];
 }
 
+bool gk20a_fifo_channel_status_is_next(struct gk20a *g, u32 chid)
+{
+        u32 channel = gk20a_readl(g, ccsr_channel_r(chid));
+
+        return ccsr_channel_next_v(channel) == ccsr_channel_next_true_v();
+}
+
+bool gk20a_fifo_channel_status_is_ctx_reload(struct gk20a *g, u32 chid)
+{
+        u32 channel = gk20a_readl(g, ccsr_channel_r(chid));
+        u32 status = ccsr_channel_status_v(channel);
+
+        return (status == ccsr_channel_status_pending_ctx_reload_v() ||
+                status == ccsr_channel_status_pending_acq_ctx_reload_v() ||
+                status == ccsr_channel_status_on_pbdma_ctx_reload_v() ||
+                status == ccsr_channel_status_on_pbdma_and_eng_ctx_reload_v() ||
+                status == ccsr_channel_status_on_eng_ctx_reload_v() ||
+                status == ccsr_channel_status_on_eng_pending_ctx_reload_v() ||
+                status == ccsr_channel_status_on_eng_pending_acq_ctx_reload_v());
+}
+
 void gk20a_dump_channel_status_ramfc(struct gk20a *g,
                                      struct gk20a_debug_output *o,
                                      u32 chid,
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
index d5b686f0..70c70931 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
@@ -248,6 +248,9 @@ int gk20a_fifo_disable_engine_activity(struct gk20a *g,
                         bool wait_for_idle);
 int gk20a_fifo_disable_all_engine_activity(struct gk20a *g,
                                 bool wait_for_idle);
+void gk20a_fifo_enable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg);
+void gk20a_fifo_disable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg);
+
 u32 gk20a_fifo_engines_on_ch(struct gk20a *g, u32 chid);
 
 int gk20a_fifo_reschedule_runlist(struct gk20a *g, u32 runlist_id);
@@ -362,6 +365,9 @@ const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index);
 void gk20a_fifo_enable_channel(struct channel_gk20a *ch);
 void gk20a_fifo_disable_channel(struct channel_gk20a *ch);
 
+bool gk20a_fifo_channel_status_is_next(struct gk20a *g, u32 chid);
+bool gk20a_fifo_channel_status_is_ctx_reload(struct gk20a *g, u32 chid);
+
 struct channel_gk20a *gk20a_refch_from_inst_ptr(struct gk20a *g, u64 inst_ptr);
 void gk20a_fifo_channel_unbind(struct channel_gk20a *ch_gk20a);
 
diff --git a/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c b/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c
index f3e87a13..eabb98ea 100644
--- a/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c
@@ -29,13 +29,37 @@ int gk20a_enable_tsg(struct tsg_gk20a *tsg)
 {
         struct gk20a *g = tsg->g;
         struct channel_gk20a *ch;
+        bool is_next, is_ctx_reload;
 
+        gk20a_fifo_disable_tsg_sched(g, tsg);
+
+        /*
+         * Due to h/w bug that exists in Maxwell and Pascal,
+         * we first need to enable all channels with NEXT and CTX_RELOAD set,
+         * and then rest of the channels should be enabled
+         */
         down_read(&tsg->ch_list_lock);
         nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) {
+                is_next = gk20a_fifo_channel_status_is_next(g, ch->chid);
+                is_ctx_reload = gk20a_fifo_channel_status_is_ctx_reload(g, ch->chid);
+
+                if (is_next || is_ctx_reload)
+                        g->ops.fifo.enable_channel(ch);
+        }
+
+        nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) {
+                is_next = gk20a_fifo_channel_status_is_next(g, ch->chid);
+                is_ctx_reload = gk20a_fifo_channel_status_is_ctx_reload(g, ch->chid);
+
+                if (is_next || is_ctx_reload)
+                        continue;
+
                 g->ops.fifo.enable_channel(ch);
         }
         up_read(&tsg->ch_list_lock);
 
+        gk20a_fifo_enable_tsg_sched(g, tsg);
+
         return 0;
 }
 
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gk20a/hw_ccsr_gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gk20a/hw_ccsr_gk20a.h
index 4877e4a8..c6f792e7 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gk20a/hw_ccsr_gk20a.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gk20a/hw_ccsr_gk20a.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012-2016, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2012-2017, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -114,6 +114,42 @@ static inline u32 ccsr_channel_status_v(u32 r)
 {
         return (r >> 24) & 0xf;
 }
+static inline u32 ccsr_channel_status_pending_ctx_reload_v(void)
+{
+        return 0x00000002;
+}
+static inline u32 ccsr_channel_status_pending_acq_ctx_reload_v(void)
+{
+        return 0x00000004;
+}
+static inline u32 ccsr_channel_status_on_pbdma_ctx_reload_v(void)
+{
+        return 0x0000000a;
+}
+static inline u32 ccsr_channel_status_on_pbdma_and_eng_ctx_reload_v(void)
+{
+        return 0x0000000b;
+}
+static inline u32 ccsr_channel_status_on_eng_ctx_reload_v(void)
+{
+        return 0x0000000c;
+}
+static inline u32 ccsr_channel_status_on_eng_pending_ctx_reload_v(void)
+{
+        return 0x0000000d;
+}
+static inline u32 ccsr_channel_status_on_eng_pending_acq_ctx_reload_v(void)
+{
+        return 0x0000000e;
+}
+static inline u32 ccsr_channel_next_v(u32 r)
+{
+        return (r >> 1) & 0x1;
+}
+static inline u32 ccsr_channel_next_true_v(void)
+{
+        return 0x00000001;
+}
 static inline u32 ccsr_channel_busy_v(u32 r)
 {
         return (r >> 28) & 0x1;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h
index 2fdf73ae..3f5d312f 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2017, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -110,6 +110,42 @@ static inline u32 ccsr_channel_status_v(u32 r)
 {
         return (r >> 24) & 0xf;
 }
+static inline u32 ccsr_channel_status_pending_ctx_reload_v(void)
+{
+        return 0x00000002;
+}
+static inline u32 ccsr_channel_status_pending_acq_ctx_reload_v(void)
+{
+        return 0x00000004;
+}
+static inline u32 ccsr_channel_status_on_pbdma_ctx_reload_v(void)
+{
+        return 0x0000000a;
+}
+static inline u32 ccsr_channel_status_on_pbdma_and_eng_ctx_reload_v(void)
+{
+        return 0x0000000b;
+}
+static inline u32 ccsr_channel_status_on_eng_ctx_reload_v(void)
+{
+        return 0x0000000c;
+}
+static inline u32 ccsr_channel_status_on_eng_pending_ctx_reload_v(void)
+{
+        return 0x0000000d;
+}
+static inline u32 ccsr_channel_status_on_eng_pending_acq_ctx_reload_v(void)
+{
+        return 0x0000000e;
+}
+static inline u32 ccsr_channel_next_v(u32 r)
+{
+        return (r >> 1) & 0x1;
+}
+static inline u32 ccsr_channel_next_true_v(void)
+{
+        return 0x00000001;
+}
 static inline u32 ccsr_channel_busy_v(u32 r)
 {
         return (r >> 28) & 0x1;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h
index 65146d39..13bd4251 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_ccsr_gp106.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2016-2017, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -114,12 +114,40 @@ static inline u32 ccsr_channel_status_pending_ctx_reload_v(void)
 {
         return 0x00000002;
 }
-static inline u32 ccsr_channel_busy_v(u32 r)
+static inline u32 ccsr_channel_status_pending_acq_ctx_reload_v(void)
 {
-        return (r >> 28) & 0x1;
+        return 0x00000004;
+}
+static inline u32 ccsr_channel_status_on_pbdma_ctx_reload_v(void)
+{
+        return 0x0000000a;
+}
+static inline u32 ccsr_channel_status_on_pbdma_and_eng_ctx_reload_v(void)
+{
+        return 0x0000000b;
+}
+static inline u32 ccsr_channel_status_on_eng_ctx_reload_v(void)
+{
+        return 0x0000000c;
+}
+static inline u32 ccsr_channel_status_on_eng_pending_ctx_reload_v(void)
+{
+        return 0x0000000d;
+}
+static inline u32 ccsr_channel_status_on_eng_pending_acq_ctx_reload_v(void)
+{
+        return 0x0000000e;
 }
 static inline u32 ccsr_channel_next_v(u32 r)
 {
         return (r >> 1) & 0x1;
 }
+static inline u32 ccsr_channel_next_true_v(void)
+{
+        return 0x00000001;
+}
+static inline u32 ccsr_channel_busy_v(u32 r)
+{
+        return (r >> 28) & 0x1;
+}
 #endif
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h
index 99398961..33c83c80 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2017, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -110,6 +110,42 @@ static inline u32 ccsr_channel_status_v(u32 r)
 {
         return (r >> 24) & 0xf;
 }
+static inline u32 ccsr_channel_status_pending_ctx_reload_v(void)
+{
+        return 0x00000002;
+}
+static inline u32 ccsr_channel_status_pending_acq_ctx_reload_v(void)
+{
+        return 0x00000004;
+}
+static inline u32 ccsr_channel_status_on_pbdma_ctx_reload_v(void)
+{
+        return 0x0000000a;
+}
+static inline u32 ccsr_channel_status_on_pbdma_and_eng_ctx_reload_v(void)
+{
+        return 0x0000000b;
+}
+static inline u32 ccsr_channel_status_on_eng_ctx_reload_v(void)
+{
+        return 0x0000000c;
+}
+static inline u32 ccsr_channel_status_on_eng_pending_ctx_reload_v(void)
+{
+        return 0x0000000d;
+}
+static inline u32 ccsr_channel_status_on_eng_pending_acq_ctx_reload_v(void)
+{
+        return 0x0000000e;
+}
+static inline u32 ccsr_channel_next_v(u32 r)
+{
+        return (r >> 1) & 0x1;
+}
+static inline u32 ccsr_channel_next_true_v(void)
+{
+        return 0x00000001;
+}
 static inline u32 ccsr_channel_busy_v(u32 r)
 {
         return (r >> 28) & 0x1;