gpu: nvgpu: add NVGPU_SUBMIT_GPFIFO_FLAGS_RESCHEDULE_RUNLIST

NVGPU_SUBMIT_GPFIFO_FLAGS_RESCHEDULE_RUNLIST causes host to expire
current timeslice and reschedule from front of runlist.
This can be used with NVGPU_RUNLIST_INTERLEAVE_LEVEL_HIGH to make a
channel start sooner after submit rather than waiting for natural
timeslice expiration or block/finish of currently running channel.

Bug 1968813

Change-Id: I632e87c5f583a09ec8bf521dc73f595150abebb0
Signed-off-by: David Li <davli@nvidia.com>
Reviewed-on: http://git-master/r/#/c/1537198
Reviewed-on: https://git-master.nvidia.com/r/1537198
Reviewed-by: svccoveritychecker <svccoveritychecker@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>

4 files changed, 35 insertions, 0 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
index 0c1b06e9..89862e63 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
@@ -2668,6 +2668,10 @@ int gk20a_submit_channel_gpfifo(struct channel_gk20a *c,
 
         g->ops.fifo.userd_gp_put(g, c);
 
+        if ((NVGPU_SUBMIT_GPFIFO_FLAGS_RESCHEDULE_RUNLIST & flags) &&
+                g->ops.fifo.reschedule_runlist)
+                g->ops.fifo.reschedule_runlist(g, c->runlist_id);
+
         /* No hw access beyond this point */
         if (c->deterministic)
                 up_read(&g->deterministic_busy);
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index fd249bc9..7062a8a0 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -3211,6 +3211,34 @@ end:
         return ret;
 }
 
+/* trigger host to expire current timeslice and reschedule runlist from front */
+int gk20a_fifo_reschedule_runlist(struct gk20a *g, u32 runlist_id)
+{
+        struct fifo_runlist_info_gk20a *runlist;
+        u32 token = PMU_INVALID_MUTEX_OWNER_ID;
+        u32 mutex_ret;
+        int ret = 0;
+
+        runlist = &g->fifo.runlist_info[runlist_id];
+        if (nvgpu_mutex_tryacquire(&runlist->mutex)) {
+                mutex_ret = nvgpu_pmu_mutex_acquire(
+                        &g->pmu, PMU_MUTEX_ID_FIFO, &token);
+
+                gk20a_writel(g, fifo_runlist_r(),
+                        gk20a_readl(g, fifo_runlist_r()));
+                gk20a_fifo_runlist_wait_pending(g, runlist_id);
+
+                if (!mutex_ret)
+                        nvgpu_pmu_mutex_release(
+                                &g->pmu, PMU_MUTEX_ID_FIFO, &token);
+                nvgpu_mutex_release(&runlist->mutex);
+        } else {
+                /* someone else is writing fifo_runlist_r so not needed here */
+                ret = -EBUSY;
+        }
+        return ret;
+}
+
 /* add/remove a channel from runlist
    special cases below: runlist->active_channels will NOT be changed.
    (chid == ~0 && !add) means remove all active channels from runlist.
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
index fb4932c8..d5b686f0 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
@@ -250,6 +250,8 @@ int gk20a_fifo_disable_all_engine_activity(struct gk20a *g,
                                 bool wait_for_idle);
 u32 gk20a_fifo_engines_on_ch(struct gk20a *g, u32 chid);
 
+int gk20a_fifo_reschedule_runlist(struct gk20a *g, u32 runlist_id);
+
 int gk20a_fifo_update_runlist(struct gk20a *g, u32 engine_id, u32 chid,
                               bool add, bool wait_for_finish);
 
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index 121dd962..ab715bdc 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -476,6 +476,7 @@ struct gpu_ops {
                 int (*resetup_ramfc)(struct channel_gk20a *c);
                 int (*preempt_channel)(struct gk20a *g, u32 chid);
                 int (*preempt_tsg)(struct gk20a *g, u32 tsgid);
+                int (*reschedule_runlist)(struct gk20a *g, u32 runlist_id);
                 int (*update_runlist)(struct gk20a *g, u32 runlist_id,
                                 u32 chid, bool add,
                                 bool wait_for_finish);