1 files changed, 108 insertions, 50 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
index 1f63bbd8..20976992 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
@@ -28,6 +28,7 @@
 #include <linux/vmalloc.h>
 
 #include "debug_gk20a.h"
+#include "ctxsw_trace_gk20a.h"
 
 #include "gk20a.h"
 #include "dbg_gpu_gk20a.h"
@@ -44,6 +45,9 @@
 
 #define NVGPU_BEGIN_AGGRESSIVE_SYNC_DESTROY_LIMIT       64      /* channels */
 
+#define NVGPU_CHANNEL_MIN_TIMESLICE_US 1000
+#define NVGPU_CHANNEL_MAX_TIMESLICE_US 50000
+
 static struct channel_gk20a *allocate_channel(struct fifo_gk20a *f);
 static void free_channel(struct fifo_gk20a *f, struct channel_gk20a *c);
 
@@ -177,7 +181,7 @@ int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g,
 }
 
 static int channel_gk20a_set_schedule_params(struct channel_gk20a *c,
-                                u32 timeslice_period, bool interleave)
+                                u32 timeslice_period)
 {
         void *inst_ptr;
         int shift = 0, value = 0;
@@ -205,30 +209,6 @@ static int channel_gk20a_set_schedule_params(struct channel_gk20a *c,
                 gk20a_readl(c->g, ccsr_channel_r(c->hw_chid)) |
                 ccsr_channel_enable_set_true_f());
 
-        if (c->interleave != interleave) {
-                mutex_lock(&c->g->interleave_lock);
-                c->interleave = interleave;
-                if (interleave)
-                        if (c->g->num_interleaved_channels >=
-                                        MAX_INTERLEAVED_CHANNELS) {
-                                gk20a_err(dev_from_gk20a(c->g),
-                                        "Change of priority would exceed runlist length, only changing timeslice\n");
-                                c->interleave = false;
-                        } else
-                                c->g->num_interleaved_channels += 1;
-                else
-                        c->g->num_interleaved_channels -= 1;
-
-                mutex_unlock(&c->g->interleave_lock);
-                gk20a_dbg_info("Set channel %d to interleave %d",
-                        c->hw_chid, c->interleave);
-
-                gk20a_fifo_set_channel_priority(
-                                c->g, 0, c->hw_chid, c->interleave);
-                c->g->ops.fifo.update_runlist(
-                                c->g, 0, ~0, true, false);
-        }
-
         return 0;
 }
 
@@ -238,6 +218,12 @@ u32 channel_gk20a_pbdma_acquire_val(struct channel_gk20a *c)
         u64 timeout;
         int val_len;
 
+        val = pbdma_acquire_retry_man_2_f() |
+                pbdma_acquire_retry_exp_2_f();
+
+        if (!c->g->timeouts_enabled)
+                return val;
+
         timeout = gk20a_get_channel_watchdog_timeout(c);
         do_div(timeout, 2); /* set acquire timeout to half of channel wdt */
         timeout *= 1000000UL; /* ms -> ns */
@@ -256,11 +242,10 @@ u32 channel_gk20a_pbdma_acquire_val(struct channel_gk20a *c)
                 man = timeout;
         }
 
-        val = pbdma_acquire_retry_man_2_f() |
-                pbdma_acquire_retry_exp_2_f() |
-                pbdma_acquire_timeout_exp_f(exp) |
+        val |= pbdma_acquire_timeout_exp_f(exp) |
                 pbdma_acquire_timeout_man_f(man) |
                 pbdma_acquire_timeout_en_enable_f();
+
         return val;
 }
 
@@ -711,11 +696,39 @@ static int gk20a_channel_set_wdt_status(struct channel_gk20a *ch,
         return 0;
 }
 
-static int gk20a_init_error_notifier(struct channel_gk20a *ch,
-                struct nvgpu_set_error_notifier *args) {
-        void *va;
+static int gk20a_channel_set_runlist_interleave(struct channel_gk20a *ch,
+                                                u32 level)
+{
+        struct gk20a *g = ch->g;
+        int ret;
+
+        if (gk20a_is_channel_marked_as_tsg(ch)) {
+                gk20a_err(dev_from_gk20a(g), "invalid operation for TSG!\n");
+                return -EINVAL;
+        }
+
+        switch (level) {
+        case NVGPU_RUNLIST_INTERLEAVE_LEVEL_LOW:
+        case NVGPU_RUNLIST_INTERLEAVE_LEVEL_MEDIUM:
+        case NVGPU_RUNLIST_INTERLEAVE_LEVEL_HIGH:
+                ret = g->ops.fifo.set_runlist_interleave(g, ch->hw_chid,
+                                                        false, 0, level);
+                break;
+        default:
+                ret = -EINVAL;
+                break;
+        }
 
+        return ret ? ret : g->ops.fifo.update_runlist(g, 0, ~0, true, true);
+}
+
+static int gk20a_init_error_notifier(struct channel_gk20a *ch,
+                struct nvgpu_set_error_notifier *args)
+{
+        struct device *dev = dev_from_gk20a(ch->g);
         struct dma_buf *dmabuf;
+        void *va;
+        u64 end = args->offset + sizeof(struct nvgpu_notification);
 
         if (!args->mem) {
                 pr_err("gk20a_init_error_notifier: invalid memory handle\n");
@@ -731,6 +744,13 @@ static int gk20a_init_error_notifier(struct channel_gk20a *ch,
                 pr_err("Invalid handle: %d\n", args->mem);
                 return -EINVAL;
         }
+
+        if (end > dmabuf->size || end < sizeof(struct nvgpu_notification)) {
+                dma_buf_put(dmabuf);
+                gk20a_err(dev, "gk20a_init_error_notifier: invalid offset\n");
+                return -EINVAL;
+        }
+
         /* map handle */
         va = dma_buf_vmap(dmabuf);
         if (!va) {
@@ -890,17 +910,6 @@ static void gk20a_free_channel(struct channel_gk20a *ch)
         }
         mutex_unlock(&f->deferred_reset_mutex);
 
-        if (ch->interleave) {
-                ch->interleave = false;
-                gk20a_fifo_set_channel_priority(
-                                ch->g, 0, ch->hw_chid, ch->interleave);
-
-                mutex_lock(&f->g->interleave_lock);
-                WARN_ON(f->g->num_interleaved_channels == 0);
-                f->g->num_interleaved_channels -= 1;
-                mutex_unlock(&f->g->interleave_lock);
-        }
-
         if (!ch->bound)
                 goto release;
 
@@ -912,6 +921,9 @@ static void gk20a_free_channel(struct channel_gk20a *ch)
 
         gk20a_free_error_notifiers(ch);
 
+        if (g->ops.fecs_trace.unbind_channel)
+                g->ops.fecs_trace.unbind_channel(g, ch);
+
         /* release channel ctx */
         g->ops.gr.free_channel_ctx(ch);
 
@@ -1145,11 +1157,8 @@ struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g)
         ch->has_timedout = false;
         ch->wdt_enabled = true;
         ch->obj_class = 0;
-        ch->interleave = false;
         ch->clean_up.scheduled = false;
-        gk20a_fifo_set_channel_priority(
-                        ch->g, 0, ch->hw_chid, ch->interleave);
-
+        ch->interleave_level = NVGPU_RUNLIST_INTERLEAVE_LEVEL_LOW;
 
         /* The channel is *not* runnable at this point. It still needs to have
          * an address space bound and allocate a gpfifo and grctx. */
@@ -1697,6 +1706,10 @@ static void gk20a_channel_timeout_handler(struct work_struct *work)
         /* Need global lock since multiple channels can timeout at a time */
         mutex_lock(&g->ch_wdt_lock);
 
+        gk20a_debug_dump(g->dev);
+        gk20a_gr_debug_dump(g->dev);
+
+
         /* Get timed out job and reset the timer */
         mutex_lock(&ch->timeout.lock);
         job = ch->timeout.job;
@@ -2399,6 +2412,7 @@ static int gk20a_channel_wait(struct channel_gk20a *ch,
         u32 offset;
         unsigned long timeout;
         int remain, ret = 0;
+        u64 end;
 
         gk20a_dbg_fn("");
 
@@ -2414,6 +2428,7 @@ static int gk20a_channel_wait(struct channel_gk20a *ch,
         case NVGPU_WAIT_TYPE_NOTIFIER:
                 id = args->condition.notifier.dmabuf_fd;
                 offset = args->condition.notifier.offset;
+                end = offset + sizeof(struct notification);
 
                 dmabuf = dma_buf_get(id);
                 if (IS_ERR(dmabuf)) {
@@ -2422,6 +2437,12 @@ static int gk20a_channel_wait(struct channel_gk20a *ch,
                         return -EINVAL;
                 }
 
+                if (end > dmabuf->size || end < sizeof(struct notification)) {
+                        dma_buf_put(dmabuf);
+                        gk20a_err(d, "invalid notifier offset\n");
+                        return -EINVAL;
+                }
+
                 notif = dma_buf_vmap(dmabuf);
                 if (!notif) {
                         gk20a_err(d, "failed to map notifier memory");
@@ -2596,7 +2617,6 @@ unsigned int gk20a_channel_poll(struct file *filep, poll_table *wait)
 int gk20a_channel_set_priority(struct channel_gk20a *ch, u32 priority)
 {
         u32 timeslice_timeout;
-        bool interleave = false;
 
         if (gk20a_is_channel_marked_as_tsg(ch)) {
                 gk20a_err(dev_from_gk20a(ch->g),
@@ -2613,8 +2633,6 @@ int gk20a_channel_set_priority(struct channel_gk20a *ch, u32 priority)
                 timeslice_timeout = ch->g->timeslice_medium_priority_us;
                 break;
         case NVGPU_PRIORITY_HIGH:
-                if (ch->g->interleave_high_priority)
-                        interleave = true;
                 timeslice_timeout = ch->g->timeslice_high_priority_us;
                 break;
         default:
@@ -2623,7 +2641,22 @@ int gk20a_channel_set_priority(struct channel_gk20a *ch, u32 priority)
         }
 
         return channel_gk20a_set_schedule_params(ch,
-                        timeslice_timeout, interleave);
+                        timeslice_timeout);
+}
+
+int gk20a_channel_set_timeslice(struct channel_gk20a *ch, u32 timeslice)
+{
+        if (gk20a_is_channel_marked_as_tsg(ch)) {
+                gk20a_err(dev_from_gk20a(ch->g),
+                        "invalid operation for TSG!\n");
+                return -EINVAL;
+        }
+
+        if (timeslice < NVGPU_CHANNEL_MIN_TIMESLICE_US ||
+                timeslice > NVGPU_CHANNEL_MAX_TIMESLICE_US)
+                return -EINVAL;
+
+        return channel_gk20a_set_schedule_params(ch, timeslice);
 }
 
 static int gk20a_channel_zcull_bind(struct channel_gk20a *ch,
@@ -2778,6 +2811,7 @@ void gk20a_init_channel(struct gpu_ops *gops)
         gops->fifo.free_inst = channel_gk20a_free_inst;
         gops->fifo.setup_ramfc = channel_gk20a_setup_ramfc;
         gops->fifo.channel_set_priority = gk20a_channel_set_priority;
+        gops->fifo.channel_set_timeslice = gk20a_channel_set_timeslice;
 }
 
 long gk20a_channel_ioctl(struct file *filp,
@@ -3028,6 +3062,30 @@ long gk20a_channel_ioctl(struct file *filp,
                 err = gk20a_channel_set_wdt_status(ch,
                                 (struct nvgpu_channel_wdt_args *)buf);
                 break;
+        case NVGPU_IOCTL_CHANNEL_SET_RUNLIST_INTERLEAVE:
+                err = gk20a_busy(dev);
+                if (err) {
+                        dev_err(&dev->dev,
+                                "%s: failed to host gk20a for ioctl cmd: 0x%x",
+                                __func__, cmd);
+                        break;
+                }
+                err = gk20a_channel_set_runlist_interleave(ch,
+                        ((struct nvgpu_runlist_interleave_args *)buf)->level);
+                gk20a_idle(dev);
+                break;
+        case NVGPU_IOCTL_CHANNEL_SET_TIMESLICE:
+                err = gk20a_busy(dev);
+                if (err) {
+                        dev_err(&dev->dev,
+                                "%s: failed to host gk20a for ioctl cmd: 0x%x",
+                                __func__, cmd);
+                        break;
+                }
+                err = ch->g->ops.fifo.channel_set_timeslice(ch,
+                        ((struct nvgpu_timeslice_args *)buf)->timeslice_us);
+                gk20a_idle(dev);
+                break;
         default:
                 dev_dbg(&dev->dev, "unrecognized ioctl cmd: 0x%x", cmd);
                 err = -ENOTTY;