From 2b064ce65e0035a860d1bc3bcccfcf8aac1f31c7 Mon Sep 17 00:00:00 2001 From: Peter Pipkorn Date: Mon, 28 Sep 2015 13:49:53 +0200 Subject: gpu: nvgpu: add high priority channel interleave MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Interleave all high priority channels between all other channels. This reduces the latency for high priority work when there are a lot of lower priority work present, imposing an upper bound on the latency. Change the default high priority timeslice from 5.2ms to 3.0 in the process, to prevent long running high priority apps from hogging the GPU too much. Introduce a new debugfs node to enable/disable high priority channel interleaving. It is currently enabled by default. Adds new runlist length max register, used for allocating suitable sized runlist. Limit the number of interleaved channels to 32. This change reduces the maximum time a lower priority job is running (one timeslice) before we check that high priority jobs are running. Tested with gles2_context_priority (still passes) Basic sanity testing is done with graphics_submit (one app is high priority) Also more functional testing using lots of parallel runs with: NVRM_GPU_CHANNEL_PRIORITY=3 ./gles2_expensive_draw –drawsperframe 20000 –triangles 50 –runtime 30 –finish plus multiple: NVRM_GPU_CHANNEL_PRIORITY=2 ./gles2_expensive_draw –drawsperframe 20000 –triangles 50 –runtime 30 -finish Previous to this change, the relative performance between high priority work and normal priority work comes down to timeslice value. This means that when there are many low priority channels, the high priority work will still drop quite a lot. But with this change, the high priority work will roughly get about half the entire GPU time, meaning that after the initial lower performance, it is less likely to get lower in performance due to more apps running on the system. This change makes a large step towards real priority levels. It is not perfect and there are no guarantees on anything, but it is a step forwards without any additional CPU overhead or other complications. It will also serve as a baseline to judge other algorithms against. Support for priorities with TSG is future work. Support for interleave mid + high priority channels, instead of just high, is also future work. Bug 1419900 Change-Id: I0f7d0ce83b6598fe86000577d72e14d312fdad98 Signed-off-by: Peter Pipkorn Reviewed-on: http://git-master/r/805961 Reviewed-by: Terje Bergstrom Tested-by: Terje Bergstrom --- drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | 133 ++++++++++++++++++++++++++++++++--- 1 file changed, 123 insertions(+), 10 deletions(-) (limited to 'drivers/gpu/nvgpu/gk20a/fifo_gk20a.c') diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c index 5c99877b..ca5c0ee6 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c @@ -303,7 +303,13 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f) if (!runlist->active_tsgs) goto clean_up_runlist_info; - runlist_size = ram_rl_entry_size_v() * f->num_channels; + runlist->high_prio_channels = + kzalloc(DIV_ROUND_UP(f->num_channels, BITS_PER_BYTE), + GFP_KERNEL); + if (!runlist->high_prio_channels) + goto clean_up_runlist_info; + + runlist_size = ram_rl_entry_size_v() * f->num_runlist_entries; for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) { int err = gk20a_gmmu_alloc(g, runlist_size, &runlist->mem[i]); if (err) { @@ -324,10 +330,16 @@ clean_up_runlist: for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) gk20a_gmmu_free(g, &runlist->mem[i]); +clean_up_runlist_info: kfree(runlist->active_channels); runlist->active_channels = NULL; -clean_up_runlist_info: + kfree(runlist->active_tsgs); + runlist->active_tsgs = NULL; + + kfree(runlist->high_prio_channels); + runlist->high_prio_channels = NULL; + kfree(f->runlist_info); f->runlist_info = NULL; @@ -483,6 +495,7 @@ static int gk20a_init_fifo_setup_sw(struct gk20a *g) gk20a_init_fifo_pbdma_intr_descs(f); /* just filling in data/tables */ f->num_channels = g->ops.fifo.get_num_fifos(g); + f->num_runlist_entries = fifo_eng_runlist_length_max_v(); f->num_pbdma = proj_host_num_pbdma_v(); f->max_engines = ENGINE_INVAL_GK20A; @@ -2149,6 +2162,34 @@ static inline u32 gk20a_get_tsg_runlist_entry_0(struct tsg_gk20a *tsg) return runlist_entry_0; } +/* add all active high priority channels */ +static inline u32 gk20a_fifo_runlist_add_high_prio_entries( + struct fifo_gk20a *f, + struct fifo_runlist_info_gk20a *runlist, + u32 *runlist_entry) +{ + struct channel_gk20a *ch = NULL; + unsigned long high_prio_chid; + u32 count = 0; + + for_each_set_bit(high_prio_chid, + runlist->high_prio_channels, f->num_channels) { + ch = &f->channel[high_prio_chid]; + + if (!gk20a_is_channel_marked_as_tsg(ch) && + test_bit(high_prio_chid, runlist->active_channels) == 1) { + gk20a_dbg_info("add high prio channel %lu to runlist", + high_prio_chid); + runlist_entry[0] = ram_rl_entry_chid_f(high_prio_chid); + runlist_entry[1] = 0; + runlist_entry += 2; + count++; + } + } + + return count; +} + static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, u32 hw_chid, bool add, bool wait_for_finish) @@ -2158,7 +2199,7 @@ static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, struct fifo_runlist_info_gk20a *runlist = NULL; u32 *runlist_entry_base = NULL; u32 *runlist_entry = NULL; - phys_addr_t runlist_pa; + u64 runlist_iova; u32 old_buf, new_buf; u32 chid, tsgid; struct channel_gk20a *ch = NULL; @@ -2194,11 +2235,13 @@ static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, old_buf = runlist->cur_buffer; new_buf = !runlist->cur_buffer; + runlist_iova = g->ops.mm.get_iova_addr( + g, runlist->mem[new_buf].sgt->sgl, 0); + gk20a_dbg_info("runlist_id : %d, switch to new buffer 0x%16llx", - runlist_id, (u64)gk20a_mem_phys(&runlist->mem[new_buf])); + runlist_id, (u64)runlist_iova); - runlist_pa = gk20a_mem_phys(&runlist->mem[new_buf]); - if (!runlist_pa) { + if (!runlist_iova) { ret = -EINVAL; goto clean_up; } @@ -2213,25 +2256,52 @@ static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, add /* resume to add all channels back */) { runlist_entry = runlist_entry_base; - /* add non-TSG channels first */ + /* Runlist manipulation: + Insert an entry of all high priority channels inbetween + all lower priority channels. This ensure that the maximum + delay a runnable high priority channel has to wait is one + medium timeslice + any context switching overhead + + wait on other high priority channels. + add non-TSG channels first */ for_each_set_bit(chid, runlist->active_channels, f->num_channels) { ch = &f->channel[chid]; - if (!gk20a_is_channel_marked_as_tsg(ch)) { - gk20a_dbg_info("add channel %d to runlist", + if (!gk20a_is_channel_marked_as_tsg(ch) && + !ch->interleave) { + u32 added; + + gk20a_dbg_info("add normal prio channel %d to runlist", chid); runlist_entry[0] = ram_rl_entry_chid_f(chid); runlist_entry[1] = 0; runlist_entry += 2; count++; + + added = gk20a_fifo_runlist_add_high_prio_entries( + f, + runlist, + runlist_entry); + count += added; + runlist_entry += 2 * added; } } + /* if there were no lower priority channels, then just + * add the high priority channels once. */ + if (count == 0) { + count = gk20a_fifo_runlist_add_high_prio_entries( + f, + runlist, + runlist_entry); + runlist_entry += 2 * count; + } + /* now add TSG entries and channels bound to TSG */ mutex_lock(&f->tsg_inuse_mutex); for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) { + u32 added; tsg = &f->tsg[tsgid]; /* add TSG entry */ gk20a_dbg_info("add TSG %d to runlist", tsg->tsgid); @@ -2260,6 +2330,13 @@ static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, WARN_ON(tsg->num_active_channels != count_channels_in_tsg); + + added = gk20a_fifo_runlist_add_high_prio_entries( + f, + runlist, + runlist_entry); + count += added; + runlist_entry += 2 * added; } mutex_unlock(&f->tsg_inuse_mutex); } else /* suspend to remove all channels */ @@ -2267,7 +2344,7 @@ static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, if (count != 0) { gk20a_writel(g, fifo_runlist_base_r(), - fifo_runlist_base_ptr_f(u64_lo32(runlist_pa >> 12)) | + fifo_runlist_base_ptr_f(u64_lo32(runlist_iova >> 12)) | fifo_runlist_base_target_vid_mem_f()); } @@ -2416,6 +2493,42 @@ u32 gk20a_fifo_get_pbdma_signature(struct gk20a *g) return pbdma_signature_hw_valid_f() | pbdma_signature_sw_zero_f(); } +int gk20a_fifo_set_channel_priority( + struct gk20a *g, + u32 runlist_id, + u32 hw_chid, + bool interleave) +{ + struct fifo_runlist_info_gk20a *runlist = NULL; + struct fifo_gk20a *f = &g->fifo; + struct channel_gk20a *ch = NULL; + + if (hw_chid >= f->num_channels) + return -EINVAL; + + if (runlist_id >= f->max_runlists) + return -EINVAL; + + ch = &f->channel[hw_chid]; + + gk20a_dbg_fn(""); + + runlist = &f->runlist_info[runlist_id]; + + mutex_lock(&runlist->mutex); + + if (ch->interleave) + set_bit(hw_chid, runlist->high_prio_channels); + else + clear_bit(hw_chid, runlist->high_prio_channels); + + gk20a_dbg_fn("done"); + + mutex_unlock(&runlist->mutex); + + return 0; +} + void gk20a_init_fifo(struct gpu_ops *gops) { gk20a_init_channel(gops); -- cgit v1.2.2