gpu: nvgpu: add high priority channel interleave

Interleave all high priority channels between all other channels.
This reduces the latency for high priority work when there
are a lot of lower priority work present, imposing an upper
bound on the latency. Change the default high priority timeslice
from 5.2ms to 3.0 in the process, to prevent long running high priority
apps from hogging the GPU too much.

Introduce a new debugfs node to enable/disable high priority
channel interleaving. It is currently enabled by default.

Adds new runlist length max register, used for allocating
suitable sized runlist.

Limit the number of interleaved channels to 32.

This change reduces the maximum time a lower priority job
is running (one timeslice) before we check that high priority
jobs are running.

Tested with gles2_context_priority (still passes)
Basic sanity testing is done with graphics_submit
(one app is high priority)

Also more functional testing using lots of parallel runs with:
NVRM_GPU_CHANNEL_PRIORITY=3 ./gles2_expensive_draw
 –drawsperframe 20000 –triangles 50 –runtime 30 –finish
plus multiple:
NVRM_GPU_CHANNEL_PRIORITY=2 ./gles2_expensive_draw
–drawsperframe 20000 –triangles 50 –runtime 30 -finish

Previous to this change, the relative performance between
high priority work and normal priority work comes down
to timeslice value. This means that when there are many
low priority channels, the high priority work will still
drop quite a lot. But with this change, the high priority
work will roughly get about half the entire GPU time, meaning
that after the initial lower performance, it is less likely
to get lower in performance due to more apps running on the system.

This change makes a large step towards real priority levels.
It is not perfect and there are no guarantees on anything,
but it is a step forwards without any additional CPU overhead
or other complications. It will also serve as a baseline to
judge other algorithms against.

Support for priorities with TSG is future work.
Support for interleave mid + high priority channels,
instead of just high, is also future work.

Bug 1419900

Change-Id: I0f7d0ce83b6598fe86000577d72e14d312fdad98
Signed-off-by: Peter Pipkorn <ppipkorn@nvidia.com>
Reviewed-on: http://git-master/r/805961
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Tested-by: Terje Bergstrom <tbergstrom@nvidia.com>

1 files changed, 46 insertions, 4 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
index a5c2efb3..0421c0f6 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
@@ -175,7 +175,7 @@ int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g,
 }
 
 static int channel_gk20a_set_schedule_params(struct channel_gk20a *c,
-                                u32 timeslice_period)
+                                u32 timeslice_period, bool interleave)
 {
         void *inst_ptr;
         int shift = 0, value = 0;
@@ -203,6 +203,30 @@ static int channel_gk20a_set_schedule_params(struct channel_gk20a *c,
                 gk20a_readl(c->g, ccsr_channel_r(c->hw_chid)) |
                 ccsr_channel_enable_set_true_f());
 
+        if (c->interleave != interleave) {
+                mutex_lock(&c->g->interleave_lock);
+                c->interleave = interleave;
+                if (interleave)
+                        if (c->g->num_interleaved_channels >=
+                                        MAX_INTERLEAVED_CHANNELS) {
+                                gk20a_err(dev_from_gk20a(c->g),
+                                        "Change of priority would exceed runlist length, only changing timeslice\n");
+                                c->interleave = false;
+                        } else
+                                c->g->num_interleaved_channels += 1;
+                else
+                        c->g->num_interleaved_channels -= 1;
+
+                mutex_unlock(&c->g->interleave_lock);
+                gk20a_dbg_info("Set channel %d to interleave %d",
+                        c->hw_chid, c->interleave);
+
+                gk20a_fifo_set_channel_priority(
+                                c->g, 0, c->hw_chid, c->interleave);
+                c->g->ops.fifo.update_runlist(
+                                c->g, 0, ~0, true, false);
+        }
+
         return 0;
 }
 
@@ -836,6 +860,17 @@ static void gk20a_free_channel(struct channel_gk20a *ch)
         }
         mutex_unlock(&f->deferred_reset_mutex);
 
+        if (ch->interleave) {
+                ch->interleave = false;
+                gk20a_fifo_set_channel_priority(
+                                ch->g, 0, ch->hw_chid, ch->interleave);
+
+                mutex_lock(&f->g->interleave_lock);
+                WARN_ON(f->g->num_interleaved_channels == 0);
+                f->g->num_interleaved_channels -= 1;
+                mutex_unlock(&f->g->interleave_lock);
+        }
+
         if (!ch->bound)
                 goto release;
 
@@ -1079,6 +1114,10 @@ struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g)
         ch->timeout_debug_dump = true;
         ch->has_timedout = false;
         ch->obj_class = 0;
+        ch->interleave = false;
+        gk20a_fifo_set_channel_priority(
+                        ch->g, 0, ch->hw_chid, ch->interleave);
+
 
         /* The channel is *not* runnable at this point. It still needs to have
          * an address space bound and allocate a gpfifo and grctx. */
@@ -2458,6 +2497,7 @@ static int gk20a_channel_set_priority(struct channel_gk20a *ch,
                 u32 priority)
 {
         u32 timeslice_timeout;
+        bool interleave = false;
 
         if (gk20a_is_channel_marked_as_tsg(ch)) {
                 gk20a_err(dev_from_gk20a(ch->g),
@@ -2474,15 +2514,17 @@ static int gk20a_channel_set_priority(struct channel_gk20a *ch,
                 timeslice_timeout = ch->g->timeslice_medium_priority_us;
                 break;
         case NVGPU_PRIORITY_HIGH:
+                if (ch->g->interleave_high_priority)
+                        interleave = true;
                 timeslice_timeout = ch->g->timeslice_high_priority_us;
                 break;
         default:
                 pr_err("Unsupported priority");
                 return -EINVAL;
         }
-        channel_gk20a_set_schedule_params(ch,
-                        timeslice_timeout);
-        return 0;
+
+        return channel_gk20a_set_schedule_params(ch,
+                        timeslice_timeout, interleave);
 }
 
 static int gk20a_channel_zcull_bind(struct channel_gk20a *ch,