1 files changed, 97 insertions, 7 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 5ce65280b396..a98fbbb4739f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -136,7 +136,8 @@ void amdgpu_ring_commit(struct amdgpu_ring *ring)
         if (ring->funcs->end_use)
                 ring->funcs->end_use(ring);
 
-        amdgpu_ring_lru_touch(ring->adev, ring);
+        if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ)
+                amdgpu_ring_lru_touch(ring->adev, ring);
 }
 
 /**
@@ -155,6 +156,75 @@ void amdgpu_ring_undo(struct amdgpu_ring *ring)
 }
 
 /**
+ * amdgpu_ring_priority_put - restore a ring's priority
+ *
+ * @ring: amdgpu_ring structure holding the information
+ * @priority: target priority
+ *
+ * Release a request for executing at @priority
+ */
+void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
+                              enum amd_sched_priority priority)
+{
+        int i;
+
+        if (!ring->funcs->set_priority)
+                return;
+
+        if (atomic_dec_return(&ring->num_jobs[priority]) > 0)
+                return;
+
+        /* no need to restore if the job is already at the lowest priority */
+        if (priority == AMD_SCHED_PRIORITY_NORMAL)
+                return;
+
+        mutex_lock(&ring->priority_mutex);
+        /* something higher prio is executing, no need to decay */
+        if (ring->priority > priority)
+                goto out_unlock;
+
+        /* decay priority to the next level with a job available */
+        for (i = priority; i >= AMD_SCHED_PRIORITY_MIN; i--) {
+                if (i == AMD_SCHED_PRIORITY_NORMAL
+                                || atomic_read(&ring->num_jobs[i])) {
+                        ring->priority = i;
+                        ring->funcs->set_priority(ring, i);
+                        break;
+                }
+        }
+
+out_unlock:
+        mutex_unlock(&ring->priority_mutex);
+}
+
+/**
+ * amdgpu_ring_priority_get - change the ring's priority
+ *
+ * @ring: amdgpu_ring structure holding the information
+ * @priority: target priority
+ *
+ * Request a ring's priority to be raised to @priority (refcounted).
+ */
+void amdgpu_ring_priority_get(struct amdgpu_ring *ring,
+                              enum amd_sched_priority priority)
+{
+        if (!ring->funcs->set_priority)
+                return;
+
+        atomic_inc(&ring->num_jobs[priority]);
+
+        mutex_lock(&ring->priority_mutex);
+        if (priority <= ring->priority)
+                goto out_unlock;
+
+        ring->priority = priority;
+        ring->funcs->set_priority(ring, priority);
+
+out_unlock:
+        mutex_unlock(&ring->priority_mutex);
+}
+
+/**
  * amdgpu_ring_init - init driver ring struct.
  *
  * @adev: amdgpu_device pointer
@@ -169,7 +239,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
                      unsigned max_dw, struct amdgpu_irq_src *irq_src,
                      unsigned irq_type)
 {
-        int r;
+        int r, i;
         int sched_hw_submission = amdgpu_sched_hw_submission;
 
         /* Set the hw submission limit higher for KIQ because
@@ -247,9 +317,14 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
         }
 
         ring->max_dw = max_dw;
+        ring->priority = AMD_SCHED_PRIORITY_NORMAL;
+        mutex_init(&ring->priority_mutex);
         INIT_LIST_HEAD(&ring->lru_list);
         amdgpu_ring_lru_touch(adev, ring);
 
+        for (i = 0; i < AMD_SCHED_PRIORITY_MAX; ++i)
+                atomic_set(&ring->num_jobs[i], 0);
+
         if (amdgpu_debugfs_ring_init(adev, ring)) {
                 DRM_ERROR("Failed to register debugfs file for rings !\n");
         }
@@ -315,14 +390,16 @@ static bool amdgpu_ring_is_blacklisted(struct amdgpu_ring *ring,
  * @type: amdgpu_ring_type enum
  * @blacklist: blacklisted ring ids array
  * @num_blacklist: number of entries in @blacklist
+ * @lru_pipe_order: find a ring from the least recently used pipe
  * @ring: output ring
  *
  * Retrieve the amdgpu_ring structure for the least recently used ring of
  * a specific IP block (all asics).
  * Returns 0 on success, error on failure.
  */
-int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type, int *blacklist,
-                        int num_blacklist, struct amdgpu_ring **ring)
+int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type,
+                        int *blacklist, int num_blacklist,
+                        bool lru_pipe_order, struct amdgpu_ring **ring)
 {
         struct amdgpu_ring *entry;
 
@@ -337,10 +414,23 @@ int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type, int *blacklist,
                 if (amdgpu_ring_is_blacklisted(entry, blacklist, num_blacklist))
                         continue;
 
-                *ring = entry;
-                amdgpu_ring_lru_touch_locked(adev, *ring);
-                break;
+                if (!*ring) {
+                        *ring = entry;
+
+                        /* We are done for ring LRU */
+                        if (!lru_pipe_order)
+                                break;
+                }
+
+                /* Move all rings on the same pipe to the end of the list */
+                if (entry->pipe == (*ring)->pipe)
+                        amdgpu_ring_lru_touch_locked(adev, entry);
         }
+
+        /* Move the ring we found to the end of the list */
+        if (*ring)
+                amdgpu_ring_lru_touch_locked(adev, *ring);
+
         spin_unlock(&adev->ring_lru_list_lock);
 
         if (!*ring) {