5 files changed, 254 insertions, 5 deletions

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 38b6fa374053..5d68346b2c01 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -797,6 +797,10 @@ void r600_dma_stop(struct radeon_device *rdev);
 int r600_dma_resume(struct radeon_device *rdev);
 void r600_dma_fini(struct radeon_device *rdev);
 
+void cayman_dma_stop(struct radeon_device *rdev);
+int cayman_dma_resume(struct radeon_device *rdev);
+void cayman_dma_fini(struct radeon_device *rdev);
+
 /*
  * CS.
  */
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index 8cf8ae86973c..d455bcb655c2 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -1731,6 +1731,26 @@ static struct radeon_asic si_asic = {
                         .ib_test = &r600_ib_test,
                         .is_lockup = &si_gpu_is_lockup,
                         .vm_flush = &si_vm_flush,
+                },
+                [R600_RING_TYPE_DMA_INDEX] = {
+                        .ib_execute = &cayman_dma_ring_ib_execute,
+                        .emit_fence = &evergreen_dma_fence_ring_emit,
+                        .emit_semaphore = &r600_dma_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &r600_dma_ring_test,
+                        .ib_test = &r600_dma_ib_test,
+                        .is_lockup = &cayman_dma_is_lockup,
+                        .vm_flush = &si_dma_vm_flush,
+                },
+                [CAYMAN_RING_TYPE_DMA1_INDEX] = {
+                        .ib_execute = &cayman_dma_ring_ib_execute,
+                        .emit_fence = &evergreen_dma_fence_ring_emit,
+                        .emit_semaphore = &r600_dma_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &r600_dma_ring_test,
+                        .ib_test = &r600_dma_ib_test,
+                        .is_lockup = &cayman_dma_is_lockup,
+                        .vm_flush = &si_dma_vm_flush,
                 }
         },
         .irq = {
@@ -1747,8 +1767,8 @@ static struct radeon_asic si_asic = {
         .copy = {
                 .blit = NULL,
                 .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX,
-                .dma = NULL,
-                .dma_ring_index = RADEON_RING_TYPE_GFX_INDEX,
+                .dma = &si_copy_dma,
+                .dma_ring_index = R600_RING_TYPE_DMA_INDEX,
                 .copy = NULL,
                 .copy_ring_index = RADEON_RING_TYPE_GFX_INDEX,
         },
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index c2988f706524..ae56673d2410 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -501,5 +501,10 @@ void si_vm_set_page(struct radeon_device *rdev, uint64_t pe,
 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib);
 uint64_t si_get_gpu_clock(struct radeon_device *rdev);
+int si_copy_dma(struct radeon_device *rdev,
+                uint64_t src_offset, uint64_t dst_offset,
+                unsigned num_gpu_pages,
+                struct radeon_fence **fence);
+void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
 
 #endif
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index c4d9eb623ce5..93f7171e6bcd 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -1660,6 +1660,8 @@ static void si_gpu_init(struct radeon_device *rdev)
         WREG32(GB_ADDR_CONFIG, gb_addr_config);
         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
+        WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
+        WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
 
         si_tiling_mode_table_init(rdev);
 
@@ -1836,6 +1838,9 @@ static void si_cp_enable(struct radeon_device *rdev, bool enable)
                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
                 WREG32(SCRATCH_UMSK, 0);
+                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
+                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
+                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
         }
         udelay(50);
 }
@@ -2891,6 +2896,32 @@ void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
         radeon_ring_write(ring, 0x0);
 }
 
+void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
+{
+        struct radeon_ring *ring = &rdev->ring[ridx];
+
+        if (vm == NULL)
+                return;
+
+        radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
+        if (vm->id < 8) {
+                radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
+        } else {
+                radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
+        }
+        radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
+
+        /* flush hdp cache */
+        radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
+        radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
+        radeon_ring_write(ring, 1);
+
+        /* bits 0-7 are the VM contexts0-7 */
+        radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
+        radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
+        radeon_ring_write(ring, 1 << vm->id);
+}
+
 /*
  * RLC
  */
@@ -3059,6 +3090,10 @@ static void si_disable_interrupt_state(struct radeon_device *rdev)
         WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
         WREG32(CP_INT_CNTL_RING1, 0);
         WREG32(CP_INT_CNTL_RING2, 0);
+        tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
+        WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
+        tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
+        WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
         WREG32(GRBM_INT_CNTL, 0);
         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
@@ -3178,6 +3213,7 @@ int si_irq_set(struct radeon_device *rdev)
         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
         u32 grbm_int_cntl = 0;
         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
+        u32 dma_cntl, dma_cntl1;
 
         if (!rdev->irq.installed) {
                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
@@ -3198,6 +3234,9 @@ int si_irq_set(struct radeon_device *rdev)
         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
 
+        dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
+        dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
+
         /* enable CP interrupts on all rings */
         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
                 DRM_DEBUG("si_irq_set: sw int gfx\n");
@@ -3211,6 +3250,15 @@ int si_irq_set(struct radeon_device *rdev)
                 DRM_DEBUG("si_irq_set: sw int cp2\n");
                 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
         }
+        if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
+                DRM_DEBUG("si_irq_set: sw int dma\n");
+                dma_cntl |= TRAP_ENABLE;