drm/radeon: make VM flushs a ring operation

Move flushing the VMs as function into the rings.
First step to make VM operations async.

Signed-off-by: Christian König <deathsimple@vodafone.de>
Reviewed-by: Jerome Glisse <jglisse@redhat.com>

8 files changed, 42 insertions, 38 deletions

diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index 7786c4a60d6c..8e3d70c7c9b7 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -1502,24 +1502,9 @@ int cayman_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm, int id)
         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (id << 2), 0);
         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (id << 2), vm->last_pfn);
         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (id << 2), vm->pt_gpu_addr >> 12);
-        /* flush hdp cache */
-        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
-        /* bits 0-7 are the VM contexts0-7 */
-        WREG32(VM_INVALIDATE_REQUEST, 1 << id);
         return 0;
 }
 
-void cayman_vm_tlb_flush(struct radeon_device *rdev, struct radeon_vm *vm)
-{
-        if (vm->id == -1)
-                return;
-
-        /* flush hdp cache */
-        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
-        /* bits 0-7 are the VM contexts0-7 */
-        WREG32(VM_INVALIDATE_REQUEST, 1 << vm->id);
-}
-
 #define R600_PTE_VALID     (1 << 0)
 #define R600_PTE_SYSTEM    (1 << 1)
 #define R600_PTE_SNOOPED   (1 << 2)
@@ -1551,3 +1536,19 @@ void cayman_vm_set_page(struct radeon_device *rdev, struct radeon_vm *vm,
         addr |= flags;
         writeq(addr, ptr + (pfn * 8));
 }
+
+void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ib *ib)
+{
+        struct radeon_ring *ring = &rdev->ring[ib->ring];
+
+        if (!ib->vm || ib->vm->id == -1)
+                return;
+
+        /* flush hdp cache */
+        radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
+        radeon_ring_write(ring, 0x1);
+
+        /* bits 0-7 are the VM contexts0-7 */
+        radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
+        radeon_ring_write(ring, 1 << ib->vm->id);
+}
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 19a29db7c1f5..320355de9c54 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -639,6 +639,8 @@ struct radeon_vm {
         struct mutex                    mutex;
         /* last fence for cs using this vm */
         struct radeon_fence             *fence;
+        /* last flush or NULL if we still need to flush */
+        struct radeon_fence             *last_flush;
 };
 
 struct radeon_vm_manager {
@@ -1116,7 +1118,6 @@ struct radeon_asic {
                 int (*init)(struct radeon_device *rdev);
                 void (*fini)(struct radeon_device *rdev);
                 int (*bind)(struct radeon_device *rdev, struct radeon_vm *vm, int id);
-                void (*tlb_flush)(struct radeon_device *rdev, struct radeon_vm *vm);
                 uint32_t (*page_flags)(struct radeon_device *rdev,
                                        struct radeon_vm *vm,
                                        uint32_t flags);
@@ -1135,6 +1136,7 @@ struct radeon_asic {
                 int (*ring_test)(struct radeon_device *rdev, struct radeon_ring *cp);
                 int (*ib_test)(struct radeon_device *rdev, struct radeon_ring *cp);
                 bool (*is_lockup)(struct radeon_device *rdev, struct radeon_ring *cp);
+                void (*vm_flush)(struct radeon_device *rdev, struct radeon_ib *ib);
         } ring[RADEON_NUM_RINGS];
         /* irqs */
         struct {
@@ -1733,7 +1735,6 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v);
 #define radeon_asic_vm_init(rdev) (rdev)->asic->vm.init((rdev))
 #define radeon_asic_vm_fini(rdev) (rdev)->asic->vm.fini((rdev))
 #define radeon_asic_vm_bind(rdev, v, id) (rdev)->asic->vm.bind((rdev), (v), (id))
-#define radeon_asic_vm_tlb_flush(rdev, v) (rdev)->asic->vm.tlb_flush((rdev), (v))
 #define radeon_asic_vm_page_flags(rdev, v, flags) (rdev)->asic->vm.page_flags((rdev), (v), (flags))
 #define radeon_asic_vm_set_page(rdev, v, pfn, addr, flags) (rdev)->asic->vm.set_page((rdev), (v), (pfn), (addr), (flags))
 #define radeon_ring_start(rdev, r, cp) (rdev)->asic->ring[(r)].ring_start((rdev), (cp))
@@ -1742,6 +1743,7 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v);
 #define radeon_ring_ib_execute(rdev, r, ib) (rdev)->asic->ring[(r)].ib_execute((rdev), (ib))
 #define radeon_ring_ib_parse(rdev, r, ib) (rdev)->asic->ring[(r)].ib_parse((rdev), (ib))
 #define radeon_ring_is_lockup(rdev, r, cp) (rdev)->asic->ring[(r)].is_lockup((rdev), (cp))
+#define radeon_ring_vm_flush(rdev, r, ib) (rdev)->asic->ring[(r)].vm_flush((rdev), (ib))
 #define radeon_irq_set(rdev) (rdev)->asic->irq.set((rdev))
 #define radeon_irq_process(rdev) (rdev)->asic->irq.process((rdev))
 #define radeon_get_vblank_counter(rdev, crtc) (rdev)->asic->display.get_vblank_counter((rdev), (crtc))
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index f524735f05ad..4a6e39f7ffd1 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -1376,7 +1376,6 @@ static struct radeon_asic cayman_asic = {
                 .init = &cayman_vm_init,
                 .fini = &cayman_vm_fini,
                 .bind = &cayman_vm_bind,
-                .tlb_flush = &cayman_vm_tlb_flush,
                 .page_flags = &cayman_vm_page_flags,
                 .set_page = &cayman_vm_set_page,
         },
@@ -1390,6 +1389,7 @@ static struct radeon_asic cayman_asic = {
                         .ring_test = &r600_ring_test,
                         .ib_test = &r600_ib_test,
                         .is_lockup = &evergreen_gpu_is_lockup,
+                        .vm_flush = &cayman_vm_flush,
                 },
                 [CAYMAN_RING_TYPE_CP1_INDEX] = {
                         .ib_execute = &cayman_ring_ib_execute,
@@ -1400,6 +1400,7 @@ static struct radeon_asic cayman_asic = {
                         .ring_test = &r600_ring_test,
                         .ib_test = &r600_ib_test,
                         .is_lockup = &evergreen_gpu_is_lockup,
+                        .vm_flush = &cayman_vm_flush,
                 },
                 [CAYMAN_RING_TYPE_CP2_INDEX] = {
                         .ib_execute = &cayman_ring_ib_execute,
@@ -1410,6 +1411,7 @@ static struct radeon_asic cayman_asic = {
                         .ring_test = &r600_ring_test,
                         .ib_test = &r600_ib_test,
                         .is_lockup = &evergreen_gpu_is_lockup,
+                        .vm_flush = &cayman_vm_flush,
                 }
         },
         .irq = {
@@ -1479,7 +1481,6 @@ static struct radeon_asic trinity_asic = {
                 .init = &cayman_vm_init,
                 .fini = &cayman_vm_fini,
                 .bind = &cayman_vm_bind,
-                .tlb_flush = &cayman_vm_tlb_flush,
                 .page_flags = &cayman_vm_page_flags,
                 .set_page = &cayman_vm_set_page,
         },
@@ -1493,6 +1494,7 @@ static struct radeon_asic trinity_asic = {
                         .ring_test = &r600_ring_test,