1 files changed, 85 insertions, 42 deletions

diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
index ea30d6ad4c13..3a4820e863ec 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
+++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
@@ -30,8 +30,7 @@
 #define CREATE_TRACE_POINTS
 #include "gpu_sched_trace.h"
 
-static struct amd_sched_job *
-amd_sched_entity_pop_job(struct amd_sched_entity *entity);
+static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity);
 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
 
 struct kmem_cache *sched_fence_slab;
@@ -64,36 +63,36 @@ static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
 }
 
 /**
- * Select next job from a specified run queue with round robin policy.
- * Return NULL if nothing available.
+ * Select an entity which could provide a job to run
+ *
+ * @rq          The run queue to check.
+ *
+ * Try to find a ready entity, returns NULL if none found.
  */
-static struct amd_sched_job *
-amd_sched_rq_select_job(struct amd_sched_rq *rq)
+static struct amd_sched_entity *
+amd_sched_rq_select_entity(struct amd_sched_rq *rq)
 {
         struct amd_sched_entity *entity;
-        struct amd_sched_job *sched_job;
 
         spin_lock(&rq->lock);
 
         entity = rq->current_entity;
         if (entity) {
                 list_for_each_entry_continue(entity, &rq->entities, list) {
-                        sched_job = amd_sched_entity_pop_job(entity);
-                        if (sched_job) {
+                        if (amd_sched_entity_is_ready(entity)) {
                                 rq->current_entity = entity;
                                 spin_unlock(&rq->lock);
-                                return sched_job;
+                                return entity;
                         }
                 }
         }
 
         list_for_each_entry(entity, &rq->entities, list) {
 
-                sched_job = amd_sched_entity_pop_job(entity);
-                if (sched_job) {
+                if (amd_sched_entity_is_ready(entity)) {
                         rq->current_entity = entity;
                         spin_unlock(&rq->lock);
-                        return sched_job;
+                        return entity;
                 }
 
                 if (entity == rq->current_entity)
@@ -177,6 +176,24 @@ static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity)
 }
 
 /**
+ * Check if entity is ready
+ *
+ * @entity      The pointer to a valid scheduler entity
+ *
+ * Return true if entity could provide a job.
+ */
+static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity)
+{
+        if (kfifo_is_empty(&entity->job_queue))
+                return false;
+
+        if (ACCESS_ONCE(entity->dependency))
+                return false;
+
+        return true;
+}
+
+/**
  * Destroy a context entity
  *
  * @sched       Pointer to scheduler instance
@@ -211,32 +228,53 @@ static void amd_sched_entity_wakeup(struct fence *f, struct fence_cb *cb)
         amd_sched_wakeup(entity->sched);
 }
 
+static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
+{
+        struct amd_gpu_scheduler *sched = entity->sched;
+        struct fence * fence = entity->dependency;
+        struct amd_sched_fence *s_fence;
+
+        if (fence->context == entity->fence_context) {
+                /* We can ignore fences from ourself */
+                fence_put(entity->dependency);
+                return false;
+        }
+
+        s_fence = to_amd_sched_fence(fence);
+        if (s_fence && s_fence->sched == sched) {
+                /* Fence is from the same scheduler */
+                if (test_bit(AMD_SCHED_FENCE_SCHEDULED_BIT, &fence->flags)) {
+                        /* Ignore it when it is already scheduled */
+                        fence_put(entity->dependency);
+                        return false;
+                }
+
+                /* Wait for fence to be scheduled */
+                entity->cb.func = amd_sched_entity_wakeup;
+                list_add_tail(&entity->cb.node, &s_fence->scheduled_cb);
+                return true;
+        }
+
+        if (!fence_add_callback(entity->dependency, &entity->cb,
+                                amd_sched_entity_wakeup))
+                return true;
+
+        fence_put(entity->dependency);
+        return false;
+}
+
 static struct amd_sched_job *
 amd_sched_entity_pop_job(struct amd_sched_entity *entity)
 {
         struct amd_gpu_scheduler *sched = entity->sched;
         struct amd_sched_job *sched_job;
 
-        if (ACCESS_ONCE(entity->dependency))
-                return NULL;
-
         if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job)))
                 return NULL;
 
-        while ((entity->dependency = sched->ops->dependency(sched_job))) {
-
-                if (entity->dependency->context == entity->fence_context) {
-                        /* We can ignore fences from ourself */
-                        fence_put(entity->dependency);
-                        continue;
-                }
-
-                if (fence_add_callback(entity->dependency, &entity->cb,
-                                       amd_sched_entity_wakeup))
-                        fence_put(entity->dependency);
-                else
+        while ((entity->dependency = sched->ops->dependency(sched_job)))
+                if (amd_sched_entity_add_dependency_cb(entity))
                         return NULL;
-        }
 
         return sched_job;
 }
@@ -250,6 +288,7 @@ amd_sched_entity_pop_job(struct amd_sched_entity *entity)
  */
 static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
 {
+        struct amd_gpu_scheduler *sched = sched_job->sched;
         struct amd_sched_entity *entity = sched_job->s_entity;
         bool added, first = false;
 
@@ -264,7 +303,7 @@ static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
 
         /* first job wakes up scheduler */
         if (first)
-                amd_sched_wakeup(sched_job->sched);
+                amd_sched_wakeup(sched);
 
         return added;
 }
@@ -280,9 +319,9 @@ void amd_sched_entity_push_job(struct amd_sched_job *sched_job)
 {
         struct amd_sched_entity *entity = sched_job->s_entity;
 
+        trace_amd_sched_job(sched_job);
         wait_event(entity->sched->job_scheduled,
                    amd_sched_entity_in(sched_job));
-        trace_amd_sched_job(sched_job);
 }
 
 /**
@@ -304,22 +343,22 @@ static void amd_sched_wakeup(struct amd_gpu_scheduler *sched)
 }
 
 /**
- * Select next to run
+ * Select next entity to process
 */
-static struct amd_sched_job *
-amd_sched_select_job(struct amd_gpu_scheduler *sched)
+static struct amd_sched_entity *
+amd_sched_select_entity(struct amd_gpu_scheduler *sched)
 {
-        struct amd_sched_job *sched_job;
+        struct amd_sched_entity *entity;
 
         if (!amd_sched_ready(sched))
                 return NULL;
 
         /* Kernel run queue has higher priority than normal run queue*/
-        sched_job = amd_sched_rq_select_job(&sched->kernel_rq);
-        if (sched_job == NULL)
-                sched_job = amd_sched_rq_select_job(&sched->sched_rq);
+        entity = amd_sched_rq_select_entity(&sched->kernel_rq);
+        if (entity == NULL)
+                entity = amd_sched_rq_select_entity(&sched->sched_rq);
 
-        return sched_job;
+        return entity;
 }
 
 static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
@@ -381,13 +420,16 @@ static int amd_sched_main(void *param)
                 unsigned long flags;
 
                 wait_event_interruptible(sched->wake_up_worker,
-                        kthread_should_stop() ||
-                        (sched_job = amd_sched_select_job(sched)));
+                        (entity = amd_sched_select_entity(sched)) ||
+                        kthread_should_stop());
 
+                if (!entity)
+                        continue;
+
+                sched_job = amd_sched_entity_pop_job(entity);
                 if (!sched_job)
                         continue;
 
-                entity = sched_job->s_entity;
                 s_fence = sched_job->s_fence;
 
                 if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
@@ -400,6 +442,7 @@ static int amd_sched_main(void *param)
 
                 atomic_inc(&sched->hw_rq_count);
                 fence = sched->ops->run_job(sched_job);
+                amd_sched_fence_scheduled(s_fence);
                 if (fence) {
                         r = fence_add_callback(fence, &s_fence->cb,
                                                amd_sched_process_job);