drm/amdgpu/SRIOV:implement guilty job TDR for(V2)

1,TDR will kickout guilty job if it hang exceed the threshold
of the given one from kernel paramter "job_hang_limit", that
way a bad command stream will not infinitly cause GPU hang.

by default this threshold is 1 so a job will be kicked out
after it hang.

2,if a job timeout TDR routine will not reset all sched/ring,
instead if will only reset on the givn one which is indicated
by @job of amdgpu_sriov_gpu_reset, that way we don't need to
reset and recover each sched/ring if we already know which job
cause GPU hang.

3,unblock sriov_gpu_reset for AI family.

V2:
1:put kickout guilty job after sched parked.
2:since parking scheduler prior to kickout already occupies a
while, we can do last check on the in question job before
doing hw_reset.

TODO:
1:when a job is considered as guilty, we should mark some flag
in its fence status flag, and let UMD side aware that this
fence signaling is not due to job complete but job hang.

2:if gpu reset cause all video memory lost, we need introduce
a new policy to implement TDR, like drop all jobs not yet
signaled, and all IOCTL on this device will return ERROR
DEVICE_LOST.
this will be implemented later.

Signed-off-by: Monk Liu <Monk.Liu@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

1 files changed, 34 insertions, 9 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 41c18700e275..8b0f4864a885 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2617,7 +2617,7 @@ err:
  */
 int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
 {
-        int i, r = 0;
+        int i, j, r = 0;
         int resched;
         struct amdgpu_bo *bo, *tmp;
         struct amdgpu_ring *ring;
@@ -2630,19 +2630,36 @@ int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
         /* block TTM */
         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
 
-        /* block scheduler */
-        for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
-                ring = adev->rings[i];
+        /* we start from the ring trigger GPU hang */
+        j = job ? job->ring->idx : 0;
 
+        /* block scheduler */
+        for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
+                ring = adev->rings[i % AMDGPU_MAX_RINGS];
                 if (!ring || !ring->sched.thread)
                         continue;
 
                 kthread_park(ring->sched.thread);
+
+                if (job && j != i)
+                        continue;
+
+                /* here give the last chance to check if fence signaled
+                 * since we already pay some time on kthread_park */
+                if (job && dma_fence_is_signaled(&job->base.s_fence->finished)) {
+                        kthread_unpark(ring->sched.thread);
+                        goto give_up_reset;
+                }
+
+                if (amd_sched_invalidate_job(&job->base, amdgpu_job_hang_limit))
+                        amd_sched_job_kickout(&job->base);
+
+                /* only do job_reset on the hang ring if @job not NULL */
                 amd_sched_hw_job_reset(&ring->sched);
-        }
 
-        /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
-        amdgpu_fence_driver_force_completion(adev);
+                /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
+                amdgpu_fence_driver_force_completion_ring(ring);
+        }
 
         /* request to take full control of GPU before re-initialization  */
         if (job)
@@ -2695,20 +2712,28 @@ int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
         }
         dma_fence_put(fence);
 
-        for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
-                struct amdgpu_ring *ring = adev->rings[i];
+        for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
+                ring = adev->rings[i % AMDGPU_MAX_RINGS];
                 if (!ring || !ring->sched.thread)
                         continue;
 
+                if (job && j != i) {
+                        kthread_unpark(ring->sched.thread);
+                        continue;
+                }
+
                 amd_sched_job_recovery(&ring->sched);
                 kthread_unpark(ring->sched.thread);
         }
 
         drm_helper_resume_force_mode(adev->ddev);
+give_up_reset:
         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
         if (r) {
                 /* bad news, how to tell it to userspace ? */
                 dev_info(adev->dev, "GPU reset failed\n");
+        } else {
+                dev_info(adev->dev, "GPU reset successed!\n");
         }
 
         adev->gfx.in_reset = false;