3 files changed, 40 insertions, 0 deletions

diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index 360de9f1f491..aea48c89b241 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -2516,6 +2516,16 @@ void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
         radeon_ring_write(ring, 1 << vm_id);
 
+        /* wait for the invalidate to complete */
+        radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
+        radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
+                                 WAIT_REG_MEM_ENGINE(0))); /* me */
+        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 0); /* ref */
+        radeon_ring_write(ring, 0); /* mask */
+        radeon_ring_write(ring, 0x20); /* poll interval */
+
         /* sync PFP to ME, otherwise we might get invalid PFP reads */
         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
         radeon_ring_write(ring, 0x0);
diff --git a/drivers/gpu/drm/radeon/ni_dma.c b/drivers/gpu/drm/radeon/ni_dma.c
index 50f88611ff60..4be2bb7cbef3 100644
--- a/drivers/gpu/drm/radeon/ni_dma.c
+++ b/drivers/gpu/drm/radeon/ni_dma.c
@@ -463,5 +463,11 @@ void cayman_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
         radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
         radeon_ring_write(ring, 1 << vm_id);
+
+        /* wait for invalidate to complete */
+        radeon_ring_write(ring, DMA_SRBM_READ_PACKET);
+        radeon_ring_write(ring, (0xff << 20) | (VM_INVALIDATE_REQUEST >> 2));
+        radeon_ring_write(ring, 0); /* mask */
+        radeon_ring_write(ring, 0); /* value */
 }
 
diff --git a/drivers/gpu/drm/radeon/nid.h b/drivers/gpu/drm/radeon/nid.h
index 2e12e4d69253..ad7125486894 100644
--- a/drivers/gpu/drm/radeon/nid.h
+++ b/drivers/gpu/drm/radeon/nid.h
@@ -1133,6 +1133,23 @@
 #define PACKET3_MEM_SEMAPHORE                           0x39
 #define PACKET3_MPEG_INDEX                              0x3A
 #define PACKET3_WAIT_REG_MEM                            0x3C
+#define         WAIT_REG_MEM_FUNCTION(x)                ((x) << 0)
+                /* 0 - always
+                 * 1 - <
+                 * 2 - <=
+                 * 3 - ==
+                 * 4 - !=
+                 * 5 - >=
+                 * 6 - >
+                 */
+#define         WAIT_REG_MEM_MEM_SPACE(x)               ((x) << 4)
+                /* 0 - reg
+                 * 1 - mem
+                 */
+#define         WAIT_REG_MEM_ENGINE(x)                  ((x) << 8)
+                /* 0 - me
+                 * 1 - pfp
+                 */
 #define PACKET3_MEM_WRITE                               0x3D
 #define PACKET3_PFP_SYNC_ME                             0x42
 #define PACKET3_SURFACE_SYNC                            0x43
@@ -1272,6 +1289,13 @@
                                          (1 << 21) |                    \
                                          (((n) & 0xFFFFF) << 0))
 
+#define DMA_SRBM_POLL_PACKET            ((9 << 28) |                    \
+                                         (1 << 27) |                    \
+                                         (1 << 26))
+
+#define DMA_SRBM_READ_PACKET            ((9 << 28) |                    \
+                                         (1 << 27))
+
 /* async DMA Packet types */
 #define DMA_PACKET_WRITE                                  0x2
 #define DMA_PACKET_COPY                                   0x3