drm/amdgpu: wait engine idle before vm flush for sdma

Signed-off-by: Chunming Zhou <David1.Zhou@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

3 files changed, 44 insertions, 0 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index e4e4b2ac77b7..6cf45b0ff91b 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -829,6 +829,20 @@ static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring,
 {
         u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) |
                           SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */
+        uint32_t seq = ring->fence_drv.sync_seq;
+        uint64_t addr = ring->fence_drv.gpu_addr;
+
+        /* wait for idle */
+        amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0,
+                                            SDMA_POLL_REG_MEM_EXTRA_OP(0) |
+                                            SDMA_POLL_REG_MEM_EXTRA_FUNC(3) | /* equal */
+                                            SDMA_POLL_REG_MEM_EXTRA_M));
+        amdgpu_ring_write(ring, addr & 0xfffffffc);
+        amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
+        amdgpu_ring_write(ring, seq); /* reference */
+        amdgpu_ring_write(ring, 0xfffffff); /* mask */
+        amdgpu_ring_write(ring, (0xfff << 16) | 4); /* retry count, poll interval */
+
 
         amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
         if (vm_id < 8) {
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index dddb8d6a81f3..0e246ff6734c 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -885,6 +885,21 @@ static void sdma_v2_4_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib
 static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring,
                                          unsigned vm_id, uint64_t pd_addr)
 {
+        uint32_t seq = ring->fence_drv.sync_seq;
+        uint64_t addr = ring->fence_drv.gpu_addr;
+
+        /* wait for idle */
+        amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
+                          SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
+                          SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
+                          SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
+        amdgpu_ring_write(ring, addr & 0xfffffffc);
+        amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
+        amdgpu_ring_write(ring, seq); /* reference */
+        amdgpu_ring_write(ring, 0xfffffff); /* mask */
+        amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
+                          SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
+
         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
                           SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
         if (vm_id < 8) {
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 19e02f7a06f3..67441f8844bb 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -1035,6 +1035,21 @@ static void sdma_v3_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib
 static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
                                          unsigned vm_id, uint64_t pd_addr)
 {
+        uint32_t seq = ring->fence_drv.sync_seq;
+        uint64_t addr = ring->fence_drv.gpu_addr;
+
+        /* wait for idle */
+        amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
+                          SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
+                          SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
+                          SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
+        amdgpu_ring_write(ring, addr & 0xfffffffc);
+        amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
+        amdgpu_ring_write(ring, seq); /* reference */
+        amdgpu_ring_write(ring, 0xfffffff); /* mask */
+        amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
+                          SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
+
         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
                           SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
         if (vm_id < 8) {