1 files changed, 302 insertions, 93 deletions

diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index 0e5799857465..321137295400 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -934,7 +934,7 @@ void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
 
 int cayman_pcie_gart_enable(struct radeon_device *rdev)
 {
-        int r;
+        int i, r;
 
         if (rdev->gart.robj == NULL) {
                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
@@ -945,9 +945,12 @@ int cayman_pcie_gart_enable(struct radeon_device *rdev)
                 return r;
         radeon_gart_restore(rdev);
         /* Setup TLB control */
-        WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB |
+        WREG32(MC_VM_MX_L1_TLB_CNTL,
+               (0xA << 7) |
+               ENABLE_L1_TLB |
                ENABLE_L1_FRAGMENT_PROCESSING |
                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
+               ENABLE_ADVANCED_DRIVER_MODEL |
                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
         /* Setup L2 cache */
         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
@@ -967,9 +970,26 @@ int cayman_pcie_gart_enable(struct radeon_device *rdev)
         WREG32(VM_CONTEXT0_CNTL2, 0);
         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
-        /* disable context1-7 */
+
+        WREG32(0x15D4, 0);
+        WREG32(0x15D8, 0);
+        WREG32(0x15DC, 0);
+
+        /* empty context1-7 */
+        for (i = 1; i < 8; i++) {
+                WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
+                WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), 0);
+                WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
+                        rdev->gart.table_addr >> 12);
+        }
+
+        /* enable context1-7 */
+        WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
+               (u32)(rdev->dummy_page.addr >> 12));
         WREG32(VM_CONTEXT1_CNTL2, 0);
         WREG32(VM_CONTEXT1_CNTL, 0);
+        WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
+                                RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 
         cayman_pcie_gart_tlb_flush(rdev);
         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
@@ -1006,9 +1026,69 @@ void cayman_pcie_gart_fini(struct radeon_device *rdev)
         radeon_gart_fini(rdev);
 }
 
+void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
+                              int ring, u32 cp_int_cntl)
+{
+        u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
+
+        WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
+        WREG32(CP_INT_CNTL, cp_int_cntl);
+}
+
 /*
  * CP.
  */
+void cayman_fence_ring_emit(struct radeon_device *rdev,
+                            struct radeon_fence *fence)
+{
+        struct radeon_ring *ring = &rdev->ring[fence->ring];
+        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
+
+        /* flush read cache over gart for this vmid */
+        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+        radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
+        radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
+        radeon_ring_write(ring, 0xFFFFFFFF);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 10); /* poll interval */
+        /* EVENT_WRITE_EOP - flush caches, send int */
+        radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
+        radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
+        radeon_ring_write(ring, addr & 0xffffffff);
+        radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
+        radeon_ring_write(ring, fence->seq);
+        radeon_ring_write(ring, 0);
+}
+
+void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
+{
+        struct radeon_ring *ring = &rdev->ring[ib->fence->ring];
+
+        /* set to DX10/11 mode */
+        radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
+        radeon_ring_write(ring, 1);
+        radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+        radeon_ring_write(ring,
+#ifdef __BIG_ENDIAN
+                          (2 << 0) |
+#endif
+                          (ib->gpu_addr & 0xFFFFFFFC));
+        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
+        radeon_ring_write(ring, ib->length_dw | (ib->vm_id << 24));
+
+        /* flush read cache over gart for this vmid */
+        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+        radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
+        radeon_ring_write(ring, ib->vm_id);
+        radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
+        radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
+        radeon_ring_write(ring, 0xFFFFFFFF);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 10); /* poll interval */
+}
+
 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
 {
         if (enable)
@@ -1049,63 +1129,64 @@ static int cayman_cp_load_microcode(struct radeon_device *rdev)
 
 static int cayman_cp_start(struct radeon_device *rdev)
 {
+        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
         int r, i;
 
-        r = radeon_ring_lock(rdev, 7);
+        r = radeon_ring_lock(rdev, ring, 7);
         if (r) {
                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
                 return r;
         }
-        radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
-        radeon_ring_write(rdev, 0x1);
-        radeon_ring_write(rdev, 0x0);
-        radeon_ring_write(rdev, rdev->config.cayman.max_hw_contexts - 1);
-        radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
-        radeon_ring_write(rdev, 0);
-        radeon_ring_write(rdev, 0);
-        radeon_ring_unlock_commit(rdev);
+        radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
+        radeon_ring_write(ring, 0x1);
+        radeon_ring_write(ring, 0x0);
+        radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
+        radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 0);
+        radeon_ring_unlock_commit(rdev, ring);
 
         cayman_cp_enable(rdev, true);
 
-        r = radeon_ring_lock(rdev, cayman_default_size + 19);
+        r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
         if (r) {
                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
                 return r;
         }
 
         /* setup clear context state */
-        radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
-        radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
+        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
 
         for (i = 0; i < cayman_default_size; i++)
-                radeon_ring_write(rdev, cayman_default_state[i]);
+                radeon_ring_write(ring, cayman_default_state[i]);
 
-        radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
-        radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE);
+        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
 
         /* set clear context state */
-        radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
-        radeon_ring_write(rdev, 0);
+        radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
+        radeon_ring_write(ring, 0);
 
         /* SQ_VTX_BASE_VTX_LOC */
-        radeon_ring_write(rdev, 0xc0026f00);
-        radeon_ring_write(rdev, 0x00000000);
-        radeon_ring_write(rdev, 0x00000000);
-        radeon_ring_write(rdev, 0x00000000);
+        radeon_ring_write(ring, 0xc0026f00);
+        radeon_ring_write(ring, 0x00000000);
+        radeon_ring_write(ring, 0x00000000);
+        radeon_ring_write(ring, 0x00000000);
 
         /* Clear consts */
-        radeon_ring_write(rdev, 0xc0036f00);
-        radeon_ring_write(rdev, 0x00000bc4);
-        radeon_ring_write(rdev, 0xffffffff);
-        radeon_ring_write(rdev, 0xffffffff);
-        radeon_ring_write(rdev, 0xffffffff);
+        radeon_ring_write(ring, 0xc0036f00);
+        radeon_ring_write(ring, 0x00000bc4);
+        radeon_ring_write(ring, 0xffffffff);
+        radeon_ring_write(ring, 0xffffffff);
+        radeon_ring_write(ring, 0xffffffff);
 
-        radeon_ring_write(rdev, 0xc0026900);
-        radeon_ring_write(rdev, 0x00000316);
-        radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
-        radeon_ring_write(rdev, 0x00000010); /*  */
+        radeon_ring_write(ring, 0xc0026900);
+        radeon_ring_write(ring, 0x00000316);
+        radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
+        radeon_ring_write(ring, 0x00000010); /*  */
 
-        radeon_ring_unlock_commit(rdev);
+        radeon_ring_unlock_commit(rdev, ring);
 
         /* XXX init other rings */
 
@@ -1115,11 +1196,12 @@ static int cayman_cp_start(struct radeon_device *rdev)
 static void cayman_cp_fini(struct radeon_device *rdev)
 {
         cayman_cp_enable(rdev, false);
-        radeon_ring_fini(rdev);
+        radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
 }
 
 int cayman_cp_resume(struct radeon_device *rdev)
 {
+        struct radeon_ring *ring;
         u32 tmp;
         u32 rb_bufsz;
         int r;
@@ -1136,7 +1218,7 @@ int cayman_cp_resume(struct radeon_device *rdev)
         WREG32(GRBM_SOFT_RESET, 0);
         RREG32(GRBM_SOFT_RESET);
 
-        WREG32(CP_SEM_WAIT_TIMER, 0x4);
+        WREG32(CP_SEM_WAIT_TIMER, 0x0);
 
         /* Set the write pointer delay */
         WREG32(CP_RB_WPTR_DELAY, 0);
@@ -1145,7 +1227,8 @@ int cayman_cp_resume(struct radeon_device *rdev)
 
         /* ring 0 - compute and gfx */
         /* Set ring buffer size */
-        rb_bufsz = drm_order(rdev->cp.ring_size / 8);
+        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
+        rb_bufsz = drm_order(ring->ring_size / 8);
         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
 #ifdef __BIG_ENDIAN
         tmp |= BUF_SWAP_32BIT;
@@ -1154,8 +1237,8 @@ int cayman_cp_resume(struct radeon_device *rdev)
 
         /* Initialize the ring buffer's read and write pointers */
         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
-        rdev->cp.wptr = 0;
-        WREG32(CP_RB0_WPTR, rdev->cp.wptr);
+        ring->wptr = 0;
+        WREG32(CP_RB0_WPTR, ring->wptr);
 
         /* set the wb address wether it's enabled or not */
         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
@@ -1172,13 +1255,14 @@ int cayman_cp_resume(struct radeon_device *rdev)
         mdelay(1);
         WREG32(CP_RB0_CNTL, tmp);
 
-        WREG32(CP_RB0_BASE, rdev->cp.gpu_addr >> 8);
+        WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
 
-        rdev->cp.rptr = RREG32(CP_RB0_RPTR);
+        ring->rptr = RREG32(CP_RB0_RPTR);
 
         /* ring1  - compute only */
         /* Set ring buffer size */
-        rb_bufsz = drm_order(rdev->cp1.ring_size / 8);
+        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
+        rb_bufsz = drm_order(ring->ring_size / 8);
         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
 #ifdef __BIG_ENDIAN
         tmp |= BUF_SWAP_32BIT;
@@ -1187,8 +1271,8 @@ int cayman_cp_resume(struct radeon_device *rdev)
 
         /* Initialize the ring buffer's read and write pointers */
         WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
-        rdev->cp1.wptr = 0;
-        WREG32(CP_RB1_WPTR, rdev->cp1.wptr);
+        ring->wptr = 0;
+        WREG32(CP_RB1_WPTR, ring->wptr);
 
         /* set the wb address wether it's enabled or not */
         WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
@@ -1197,13 +1281,14 @@ int cayman_cp_resume(struct radeon_device *rdev)
         mdelay(1);
         WREG32(CP_RB1_CNTL, tmp);
 
-        WREG32(CP_RB1_BASE, rdev->cp1.gpu_addr >> 8);
+        WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
 
-        rdev->cp1.rptr = RREG32(CP_RB1_RPTR);
+        ring->rptr = RREG32(CP_RB1_RPTR);
 
         /* ring2 - compute only */
         /* Set ring buffer size */
-        rb_bufsz = drm_order(rdev->cp2.ring_size / 8);
+        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
+        rb_bufsz = drm_order(ring->ring_size / 8);
         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
 #ifdef __BIG_ENDIAN
         tmp |= BUF_SWAP_32BIT;
@@ -1212,8 +1297,8 @@ int cayman_cp_resume(struct radeon_device *rdev)
 
         /* Initialize the ring buffer's read and write pointers */
         WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
-        rdev->cp2.wptr = 0;
-        WREG32(CP_RB2_WPTR, rdev->cp2.wptr);
+        ring->wptr = 0;
+        WREG32(CP_RB2_WPTR, ring->wptr);
 
         /* set the wb address wether it's enabled or not */
         WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
@@ -1222,28 +1307,28 @@ int cayman_cp_resume(struct radeon_device *rdev)
         mdelay(1);
         WREG32(CP_RB2_CNTL, tmp);
 
-        WREG32(CP_RB2_BASE, rdev->cp2.gpu_addr >> 8);
+        WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
 
-        rdev->cp2.rptr = RREG32(CP_RB2_RPTR);
+        ring->rptr = RREG32(CP_RB2_RPTR);
 
         /* start the rings */
         cayman_cp_start(rdev);
-        rdev->cp.ready = true;
-        rdev->cp1.ready = true;
-        rdev->cp2.ready = true;
+        rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
+        rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
+        rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
         /* this only test cp0 */
-        r = radeon_ring_test(rdev);
+        r = radeon_ring_test(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
         if (r) {
-                rdev->cp.ready = false;
-                rdev->cp1.ready = false;
-                rdev->cp2.ready = false;
+                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
+                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
+                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
                 return r;
         }
 
         return 0;
 }
 
-bool cayman_gpu_is_lockup(struct radeon_device *rdev)
+bool cayman_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
 {
         u32 srbm_status;
         u32 grbm_status;
@@ -1256,20 +1341,20 @@ bool cayman_gpu_is_lockup(struct radeon_device *rdev)
         grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
         grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
         if (!(grbm_status & GUI_ACTIVE)) {
-                r100_gpu_lockup_update(lockup, &rdev->cp);
+                r100_gpu_lockup_update(lockup, ring);
                 return false;
         }
         /* force CP activities */
-        r = radeon_ring_lock(rdev, 2);
+        r = radeon_ring_lock(rdev, ring, 2);
         if (!r) {
                 /* PACKET2 NOP */
-                radeon_ring_write(rdev, 0x80000000);
-                radeon_ring_write(rdev, 0x80000000);
-                radeon_ring_unlock_commit(rdev);
+                radeon_ring_write(ring, 0x80000000);
+                radeon_ring_write(ring, 0x80000000);
+                radeon_ring_unlock_commit(rdev, ring);
         }
         /* XXX deal with CP0,1,2 */
-        rdev->cp.rptr = RREG32(CP_RB0_RPTR);
-        return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
+        ring->rptr = RREG32(ring->rptr_reg);
+        return r100_gpu_cp_is_lockup(rdev, lockup, ring);
 }
 
 static int cayman_gpu_soft_reset(struct radeon_device *rdev)
@@ -1289,6 +1374,15 @@ static int cayman_gpu_soft_reset(struct radeon_device *rdev)
                 RREG32(GRBM_STATUS_SE1));
         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
                 RREG32(SRBM_STATUS));
+        dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
+                 RREG32(0x14F8));
+        dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
+                 RREG32(0x14D8));
+        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+                 RREG32(0x14FC));
+        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+                 RREG32(0x14DC));
+
         evergreen_mc_stop(rdev, &save);
         if (evergreen_mc_wait_for_idle(rdev)) {
                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
@@ -1319,6 +1413,7 @@ static int cayman_gpu_soft_reset(struct radeon_device *rdev)
         (void)RREG32(GRBM_SOFT_RESET);
         /* Wait a little for things to settle down */
         udelay(50);
+
         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
                 RREG32(GRBM_STATUS));
         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
@@ -1338,6 +1433,7 @@ int cayman_asic_reset(struct radeon_device *rdev)
 
 static int cayman_startup(struct radeon_device *rdev)
 {
+        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
         int r;
 
         /* enable pcie gen2 link */
@@ -1378,6 +1474,24 @@ static int cayman_startup(struct radeon_device *rdev)
         if (r)
                 return r;
 
+        r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
+        if (r) {
+                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
+                return r;
+        }
+
+        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
+        if (r) {
+                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
+                return r;
+        }
+
+        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
+        if (r) {
+                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
+                return r;
+        }
+
         /* Enable IRQ */
         r = r600_irq_init(rdev);
         if (r) {
@@ -1387,7 +1501,9 @@ static int cayman_startup(struct radeon_device *rdev)
         }
         evergreen_irq_set(rdev);
 
-        r = radeon_ring_init(rdev, rdev->cp.ring_size);
+        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
+                             CP_RB0_RPTR, CP_RB0_WPTR,
+                             0, 0xfffff, RADEON_CP_PACKET2);
         if (r)
                 return r;
         r = cayman_cp_load_microcode(rdev);
@@ -1397,6 +1513,21 @@ static int cayman_startup(struct radeon_device *rdev)
         if (r)
                 return r;
 
+        r = radeon_ib_pool_start(rdev);
+        if (r)
+                return r;
+
+        r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
+        if (r) {
+                DRM_ERROR("radeon: failed testing IB (%d).\n", r);
+                rdev->accel_working = false;
+                return r;
+        }
+
+        r = radeon_vm_manager_start(rdev);
+        if (r)
+                return r;
+
         return 0;
 }
 
@@ -1411,32 +1542,26 @@ int cayman_resume(struct radeon_device *rdev)
         /* post card */
         atom_asic_init(rdev->mode_info.atom_context);
 
+        rdev->accel_working = true;
         r = cayman_startup(rdev);
         if (r) {
                 DRM_ERROR("cayman startup failed on resume\n");
                 return r;
         }
-
-        r = r600_ib_test(rdev);
-        if (r) {
-                DRM_ERROR("radeon: failled testing IB (%d).\n", r);
-                return r;
-        }
-
         return r;
-
 }
 
 int cayman_suspend(struct radeon_device *rdev)
 {
         /* FIXME: we should wait for ring to be empty */
+        radeon_ib_pool_suspend(rdev);
+        radeon_vm_manager_suspend(rdev);
+        r600_blit_suspend(rdev);
         cayman_cp_enable(rdev, false);
-        rdev->cp.ready = false;
+        rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
         evergreen_irq_suspend(rdev);
         radeon_wb_disable(rdev);
         cayman_pcie_gart_disable(rdev);
-        r600_blit_suspend(rdev);
-
         return 0;
 }
 
@@ -1448,6 +1573,7 @@ int cayman_suspend(struct radeon_device *rdev)
  */
 int cayman_init(struct radeon_device *rdev)
 {
+        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
         int r;
 
         /* This don't do much */
@@ -1500,8 +1626,8 @@ int cayman_init(struct radeon_device *rdev)
         if (r)
                 return r;
 
-        rdev->cp.ring_obj = NULL;
-        r600_ring_init(rdev, 1024 * 1024);
+        ring->ring_obj = NULL;
+        r600_ring_init(rdev, ring, 1024 * 1024);
 
         rdev->ih.ring_obj = NULL;
         r600_ih_ring_init(rdev, 64 * 1024);
@@ -1510,29 +1636,29 @@ int cayman_init(struct radeon_device *rdev)
         if (r)
                 return r;
 
+        r = radeon_ib_pool_init(rdev);
         rdev->accel_working = true;
+        if (r) {
+                dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
+                rdev->accel_working = false;
+        }
+        r = radeon_vm_manager_init(rdev);
+        if (r) {
+                dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
+        }
+
         r = cayman_startup(rdev);
         if (r) {
                 dev_err(rdev->dev, "disabling GPU acceleration\n");
                 cayman_cp_fini(rdev);
                 r600_irq_fini(rdev);
                 radeon_wb_fini(rdev);
+                r100_ib_fini(rdev);
+                radeon_vm_manager_fini(rdev);
                 radeon_irq_kms_fini(rdev);
                 cayman_pcie_gart_fini(rdev);
                 rdev->accel_working = false;
         }
-        if (rdev->accel_working) {
-                r = radeon_ib_pool_init(rdev);
-                if (r) {
-                        DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r);
-                        rdev->accel_working = false;
-                }
-                r = r600_ib_test(rdev);
-                if (r) {
-                        DRM_ERROR("radeon: failed testing IB (%d).\n", r);
-                        rdev->accel_working = false;
-                }
-        }
 
         /* Don't start up if the MC ucode is missing.
          * The default clocks and voltages before the MC ucode
@@ -1552,11 +1678,13 @@ void cayman_fini(struct radeon_device *rdev)
         cayman_cp_fini(rdev);
         r600_irq_fini(rdev);
         radeon_wb_fini(rdev);
-        radeon_ib_pool_fini(rdev);
+        radeon_vm_manager_fini(rdev);
+        r100_ib_fini(rdev);
         radeon_irq_kms_fini(rdev);
         cayman_pcie_gart_fini(rdev);
         r600_vram_scratch_fini(rdev);
         radeon_gem_fini(rdev);
+        radeon_semaphore_driver_fini(rdev);
         radeon_fence_driver_fini(rdev);
         radeon_bo_fini(rdev);
         radeon_atombios_fini(rdev);
@@ -1564,3 +1692,84 @@ void cayman_fini(struct radeon_device *rdev)
         rdev->bios = NULL;
 }
 
+/*
+ * vm
+ */
+int cayman_vm_init(struct radeon_device *rdev)
+{
+        /* number of VMs */
+        rdev->vm_manager.nvm = 8;
+        /* base offset of vram pages */
+        rdev->vm_manager.vram_base_offset = 0;
+        return 0;
+}
+
+void cayman_vm_fini(struct radeon_device *rdev)
+{
+}
+
+int cayman_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm, int id)
+{
+        WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (id << 2), 0);
+        WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (id << 2), vm->last_pfn);
+        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (id << 2), vm->pt_gpu_addr >> 12);
+        /* flush hdp cache */
+        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
+        /* bits 0-7 are the VM contexts0-7 */
+        WREG32(VM_INVALIDATE_REQUEST, 1 << id);
+        return 0;
+}
+
+void cayman_vm_unbind(struct radeon_device *rdev, struct radeon_vm *vm)
+{
+        WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (vm->id << 2), 0);
+        WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (vm->id << 2), 0);
+        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0);
+        /* flush hdp cache */
+        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
+        /* bits 0-7 are the VM contexts0-7 */
+        WREG32(VM_INVALIDATE_REQUEST, 1 << vm->id);
+}
+
+void cayman_vm_tlb_flush(struct radeon_device *rdev, struct radeon_vm *vm)
+{
+        if (vm->id == -1)
+                return;
+
+        /* flush hdp cache */
+        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
+        /* bits 0-7 are the VM contexts0-7 */
+        WREG32(VM_INVALIDATE_REQUEST, 1 << vm->id);
+}
+
+#define R600_PTE_VALID     (1 << 0)
+#define R600_PTE_SYSTEM    (1 << 1)
+#define R600_PTE_SNOOPED   (1 << 2)
+#define R600_PTE_READABLE  (1 << 5)
+#define R600_PTE_WRITEABLE (1 << 6)
+
+uint32_t cayman_vm_page_flags(struct radeon_device *rdev,
+                              struct radeon_vm *vm,
+                              uint32_t flags)
+{
+        uint32_t r600_flags = 0;
+
+        r600_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_PTE_VALID : 0;
+        r600_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
+        r600_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
+        if (flags & RADEON_VM_PAGE_SYSTEM) {
+                r600_flags |= R600_PTE_SYSTEM;
+                r600_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
+        }
+        return r600_flags;
+}
+
+void cayman_vm_set_page(struct radeon_device *rdev, struct radeon_vm *vm,
+                        unsigned pfn, uint64_t addr, uint32_t flags)
+{
+        void __iomem *ptr = (void *)vm->pt;
+
+        addr = addr & 0xFFFFFFFFFFFFF000ULL;
+        addr |= flags;
+        writeq(addr, ptr + (pfn * 8));
+}