aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--drivers/gpu/drm/radeon/evergreen.c36
-rw-r--r--drivers/gpu/drm/radeon/r100.c97
-rw-r--r--drivers/gpu/drm/radeon/r300.c15
-rw-r--r--drivers/gpu/drm/radeon/r420.c16
-rw-r--r--drivers/gpu/drm/radeon/r520.c11
-rw-r--r--drivers/gpu/drm/radeon/r600.c116
-rw-r--r--drivers/gpu/drm/radeon/radeon.h12
-rw-r--r--drivers/gpu/drm/radeon/radeon_asic.h3
-rw-r--r--drivers/gpu/drm/radeon/radeon_device.c77
-rw-r--r--drivers/gpu/drm/radeon/radeon_fence.c6
-rw-r--r--drivers/gpu/drm/radeon/radeon_ring.c12
-rw-r--r--drivers/gpu/drm/radeon/rs400.c15
-rw-r--r--drivers/gpu/drm/radeon/rs600.c15
-rw-r--r--drivers/gpu/drm/radeon/rs690.c15
-rw-r--r--drivers/gpu/drm/radeon/rv515.c15
-rw-r--r--drivers/gpu/drm/radeon/rv770.c15
16 files changed, 258 insertions, 218 deletions
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index 79082d4398ae..e47d221e24ac 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -731,7 +731,7 @@ int evergreen_cp_resume(struct radeon_device *rdev)
731 731
732 /* Set ring buffer size */ 732 /* Set ring buffer size */
733 rb_bufsz = drm_order(rdev->cp.ring_size / 8); 733 rb_bufsz = drm_order(rdev->cp.ring_size / 8);
734 tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 734 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
735#ifdef __BIG_ENDIAN 735#ifdef __BIG_ENDIAN
736 tmp |= BUF_SWAP_32BIT; 736 tmp |= BUF_SWAP_32BIT;
737#endif 737#endif
@@ -745,8 +745,19 @@ int evergreen_cp_resume(struct radeon_device *rdev)
745 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 745 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
746 WREG32(CP_RB_RPTR_WR, 0); 746 WREG32(CP_RB_RPTR_WR, 0);
747 WREG32(CP_RB_WPTR, 0); 747 WREG32(CP_RB_WPTR, 0);
748 WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF); 748
749 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr)); 749 /* set the wb address wether it's enabled or not */
750 WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
751 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
752 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
753
754 if (rdev->wb.enabled)
755 WREG32(SCRATCH_UMSK, 0xff);
756 else {
757 tmp |= RB_NO_UPDATE;
758 WREG32(SCRATCH_UMSK, 0);
759 }
760
750 mdelay(1); 761 mdelay(1);
751 WREG32(CP_RB_CNTL, tmp); 762 WREG32(CP_RB_CNTL, tmp);
752 763
@@ -1759,8 +1770,10 @@ static inline u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
1759{ 1770{
1760 u32 wptr, tmp; 1771 u32 wptr, tmp;
1761 1772
1762 /* XXX use writeback */ 1773 if (rdev->wb.enabled)
1763 wptr = RREG32(IH_RB_WPTR); 1774 wptr = rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4];
1775 else
1776 wptr = RREG32(IH_RB_WPTR);
1764 1777
1765 if (wptr & RB_OVERFLOW) { 1778 if (wptr & RB_OVERFLOW) {
1766 /* When a ring buffer overflow happen start parsing interrupt 1779 /* When a ring buffer overflow happen start parsing interrupt
@@ -2068,6 +2081,11 @@ static int evergreen_startup(struct radeon_device *rdev)
2068 } 2081 }
2069#endif 2082#endif
2070 2083
2084 /* allocate wb buffer */
2085 r = radeon_wb_init(rdev);
2086 if (r)
2087 return r;
2088
2071 /* Enable IRQ */ 2089 /* Enable IRQ */
2072 r = r600_irq_init(rdev); 2090 r = r600_irq_init(rdev);
2073 if (r) { 2091 if (r) {
@@ -2086,8 +2104,6 @@ static int evergreen_startup(struct radeon_device *rdev)
2086 r = evergreen_cp_resume(rdev); 2104 r = evergreen_cp_resume(rdev);
2087 if (r) 2105 if (r)
2088 return r; 2106 return r;
2089 /* write back buffer are not vital so don't worry about failure */
2090 r600_wb_enable(rdev);
2091 2107
2092 return 0; 2108 return 0;
2093} 2109}
@@ -2128,7 +2144,7 @@ int evergreen_suspend(struct radeon_device *rdev)
2128 r700_cp_stop(rdev); 2144 r700_cp_stop(rdev);
2129 rdev->cp.ready = false; 2145 rdev->cp.ready = false;
2130 evergreen_irq_suspend(rdev); 2146 evergreen_irq_suspend(rdev);
2131 r600_wb_disable(rdev); 2147 radeon_wb_disable(rdev);
2132 evergreen_pcie_gart_disable(rdev); 2148 evergreen_pcie_gart_disable(rdev);
2133#if 0 2149#if 0
2134 /* unpin shaders bo */ 2150 /* unpin shaders bo */
@@ -2245,8 +2261,8 @@ int evergreen_init(struct radeon_device *rdev)
2245 if (r) { 2261 if (r) {
2246 dev_err(rdev->dev, "disabling GPU acceleration\n"); 2262 dev_err(rdev->dev, "disabling GPU acceleration\n");
2247 r700_cp_fini(rdev); 2263 r700_cp_fini(rdev);
2248 r600_wb_fini(rdev);
2249 r600_irq_fini(rdev); 2264 r600_irq_fini(rdev);
2265 radeon_wb_fini(rdev);
2250 radeon_irq_kms_fini(rdev); 2266 radeon_irq_kms_fini(rdev);
2251 evergreen_pcie_gart_fini(rdev); 2267 evergreen_pcie_gart_fini(rdev);
2252 rdev->accel_working = false; 2268 rdev->accel_working = false;
@@ -2270,8 +2286,8 @@ void evergreen_fini(struct radeon_device *rdev)
2270{ 2286{
2271 /*r600_blit_fini(rdev);*/ 2287 /*r600_blit_fini(rdev);*/
2272 r700_cp_fini(rdev); 2288 r700_cp_fini(rdev);
2273 r600_wb_fini(rdev);
2274 r600_irq_fini(rdev); 2289 r600_irq_fini(rdev);
2290 radeon_wb_fini(rdev);
2275 radeon_irq_kms_fini(rdev); 2291 radeon_irq_kms_fini(rdev);
2276 evergreen_pcie_gart_fini(rdev); 2292 evergreen_pcie_gart_fini(rdev);
2277 radeon_gem_fini(rdev); 2293 radeon_gem_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index e151f16a8f86..7712c055b3e8 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -675,67 +675,6 @@ void r100_fence_ring_emit(struct radeon_device *rdev,
675 radeon_ring_write(rdev, RADEON_SW_INT_FIRE); 675 radeon_ring_write(rdev, RADEON_SW_INT_FIRE);
676} 676}
677 677
678int r100_wb_init(struct radeon_device *rdev)
679{
680 int r;
681
682 if (rdev->wb.wb_obj == NULL) {
683 r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true,
684 RADEON_GEM_DOMAIN_GTT,
685 &rdev->wb.wb_obj);
686 if (r) {
687 dev_err(rdev->dev, "(%d) create WB buffer failed\n", r);
688 return r;
689 }
690 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
691 if (unlikely(r != 0))
692 return r;
693 r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
694 &rdev->wb.gpu_addr);
695 if (r) {
696 dev_err(rdev->dev, "(%d) pin WB buffer failed\n", r);
697 radeon_bo_unreserve(rdev->wb.wb_obj);
698 return r;
699 }
700 r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
701 radeon_bo_unreserve(rdev->wb.wb_obj);
702 if (r) {
703 dev_err(rdev->dev, "(%d) map WB buffer failed\n", r);
704 return r;
705 }
706 }
707 WREG32(R_000774_SCRATCH_ADDR, rdev->wb.gpu_addr);
708 WREG32(R_00070C_CP_RB_RPTR_ADDR,
709 S_00070C_RB_RPTR_ADDR((rdev->wb.gpu_addr + 1024) >> 2));
710 WREG32(R_000770_SCRATCH_UMSK, 0xff);
711 return 0;
712}
713
714void r100_wb_disable(struct radeon_device *rdev)
715{
716 WREG32(R_000770_SCRATCH_UMSK, 0);
717}
718
719void r100_wb_fini(struct radeon_device *rdev)
720{
721 int r;
722
723 r100_wb_disable(rdev);
724 if (rdev->wb.wb_obj) {
725 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
726 if (unlikely(r != 0)) {
727 dev_err(rdev->dev, "(%d) can't finish WB\n", r);
728 return;
729 }
730 radeon_bo_kunmap(rdev->wb.wb_obj);
731 radeon_bo_unpin(rdev->wb.wb_obj);
732 radeon_bo_unreserve(rdev->wb.wb_obj);
733 radeon_bo_unref(&rdev->wb.wb_obj);
734 rdev->wb.wb = NULL;
735 rdev->wb.wb_obj = NULL;
736 }
737}
738
739int r100_copy_blit(struct radeon_device *rdev, 678int r100_copy_blit(struct radeon_device *rdev,
740 uint64_t src_offset, 679 uint64_t src_offset,
741 uint64_t dst_offset, 680 uint64_t dst_offset,
@@ -996,20 +935,32 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
996 WREG32(0x718, pre_write_timer | (pre_write_limit << 28)); 935 WREG32(0x718, pre_write_timer | (pre_write_limit << 28));
997 tmp = (REG_SET(RADEON_RB_BUFSZ, rb_bufsz) | 936 tmp = (REG_SET(RADEON_RB_BUFSZ, rb_bufsz) |
998 REG_SET(RADEON_RB_BLKSZ, rb_blksz) | 937 REG_SET(RADEON_RB_BLKSZ, rb_blksz) |
999 REG_SET(RADEON_MAX_FETCH, max_fetch) | 938 REG_SET(RADEON_MAX_FETCH, max_fetch));
1000 RADEON_RB_NO_UPDATE);
1001#ifdef __BIG_ENDIAN 939#ifdef __BIG_ENDIAN
1002 tmp |= RADEON_BUF_SWAP_32BIT; 940 tmp |= RADEON_BUF_SWAP_32BIT;
1003#endif 941#endif
1004 WREG32(RADEON_CP_RB_CNTL, tmp); 942 WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_NO_UPDATE);
1005 943
1006 /* Set ring address */ 944 /* Set ring address */
1007 DRM_INFO("radeon: ring at 0x%016lX\n", (unsigned long)rdev->cp.gpu_addr); 945 DRM_INFO("radeon: ring at 0x%016lX\n", (unsigned long)rdev->cp.gpu_addr);
1008 WREG32(RADEON_CP_RB_BASE, rdev->cp.gpu_addr); 946 WREG32(RADEON_CP_RB_BASE, rdev->cp.gpu_addr);
1009 /* Force read & write ptr to 0 */ 947 /* Force read & write ptr to 0 */
1010 WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_RPTR_WR_ENA); 948 WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_RPTR_WR_ENA | RADEON_RB_NO_UPDATE);
1011 WREG32(RADEON_CP_RB_RPTR_WR, 0); 949 WREG32(RADEON_CP_RB_RPTR_WR, 0);
1012 WREG32(RADEON_CP_RB_WPTR, 0); 950 WREG32(RADEON_CP_RB_WPTR, 0);
951
952 /* set the wb address whether it's enabled or not */
953 WREG32(R_00070C_CP_RB_RPTR_ADDR,
954 S_00070C_RB_RPTR_ADDR((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) >> 2));
955 WREG32(R_000774_SCRATCH_ADDR, rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET);
956
957 if (rdev->wb.enabled)
958 WREG32(R_000770_SCRATCH_UMSK, 0xff);
959 else {
960 tmp |= RADEON_RB_NO_UPDATE;
961 WREG32(R_000770_SCRATCH_UMSK, 0);
962 }
963
1013 WREG32(RADEON_CP_RB_CNTL, tmp); 964 WREG32(RADEON_CP_RB_CNTL, tmp);
1014 udelay(10); 965 udelay(10);
1015 rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR); 966 rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR);
@@ -1050,6 +1001,7 @@ void r100_cp_disable(struct radeon_device *rdev)
1050 rdev->cp.ready = false; 1001 rdev->cp.ready = false;
1051 WREG32(RADEON_CP_CSQ_MODE, 0); 1002 WREG32(RADEON_CP_CSQ_MODE, 0);
1052 WREG32(RADEON_CP_CSQ_CNTL, 0); 1003 WREG32(RADEON_CP_CSQ_CNTL, 0);
1004 WREG32(R_000770_SCRATCH_UMSK, 0);
1053 if (r100_gui_wait_for_idle(rdev)) { 1005 if (r100_gui_wait_for_idle(rdev)) {
1054 printk(KERN_WARNING "Failed to wait GUI idle while " 1006 printk(KERN_WARNING "Failed to wait GUI idle while "
1055 "programming pipes. Bad things might happen.\n"); 1007 "programming pipes. Bad things might happen.\n");
@@ -3734,6 +3686,12 @@ static int r100_startup(struct radeon_device *rdev)
3734 if (r) 3686 if (r)
3735 return r; 3687 return r;
3736 } 3688 }
3689
3690 /* allocate wb buffer */
3691 r = radeon_wb_init(rdev);
3692 if (r)
3693 return r;
3694
3737 /* Enable IRQ */ 3695 /* Enable IRQ */
3738 r100_irq_set(rdev); 3696 r100_irq_set(rdev);
3739 rdev->config.r100.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 3697 rdev->config.r100.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -3743,9 +3701,6 @@ static int r100_startup(struct radeon_device *rdev)
3743 dev_err(rdev->dev, "failled initializing CP (%d).\n", r); 3701 dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
3744 return r; 3702 return r;
3745 } 3703 }
3746 r = r100_wb_init(rdev);
3747 if (r)
3748 dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
3749 r = r100_ib_init(rdev); 3704 r = r100_ib_init(rdev);
3750 if (r) { 3705 if (r) {
3751 dev_err(rdev->dev, "failled initializing IB (%d).\n", r); 3706 dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
@@ -3779,7 +3734,7 @@ int r100_resume(struct radeon_device *rdev)
3779int r100_suspend(struct radeon_device *rdev) 3734int r100_suspend(struct radeon_device *rdev)
3780{ 3735{
3781 r100_cp_disable(rdev); 3736 r100_cp_disable(rdev);
3782 r100_wb_disable(rdev); 3737 radeon_wb_disable(rdev);
3783 r100_irq_disable(rdev); 3738 r100_irq_disable(rdev);
3784 if (rdev->flags & RADEON_IS_PCI) 3739 if (rdev->flags & RADEON_IS_PCI)
3785 r100_pci_gart_disable(rdev); 3740 r100_pci_gart_disable(rdev);
@@ -3789,7 +3744,7 @@ int r100_suspend(struct radeon_device *rdev)
3789void r100_fini(struct radeon_device *rdev) 3744void r100_fini(struct radeon_device *rdev)
3790{ 3745{
3791 r100_cp_fini(rdev); 3746 r100_cp_fini(rdev);
3792 r100_wb_fini(rdev); 3747 radeon_wb_fini(rdev);
3793 r100_ib_fini(rdev); 3748 r100_ib_fini(rdev);
3794 radeon_gem_fini(rdev); 3749 radeon_gem_fini(rdev);
3795 if (rdev->flags & RADEON_IS_PCI) 3750 if (rdev->flags & RADEON_IS_PCI)
@@ -3902,7 +3857,7 @@ int r100_init(struct radeon_device *rdev)
3902 /* Somethings want wront with the accel init stop accel */ 3857 /* Somethings want wront with the accel init stop accel */
3903 dev_err(rdev->dev, "Disabling GPU acceleration\n"); 3858 dev_err(rdev->dev, "Disabling GPU acceleration\n");
3904 r100_cp_fini(rdev); 3859 r100_cp_fini(rdev);
3905 r100_wb_fini(rdev); 3860 radeon_wb_fini(rdev);
3906 r100_ib_fini(rdev); 3861 r100_ib_fini(rdev);
3907 radeon_irq_kms_fini(rdev); 3862 radeon_irq_kms_fini(rdev);
3908 if (rdev->flags & RADEON_IS_PCI) 3863 if (rdev->flags & RADEON_IS_PCI)
diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c
index c827738ad7dd..34527e600fe9 100644
--- a/drivers/gpu/drm/radeon/r300.c
+++ b/drivers/gpu/drm/radeon/r300.c
@@ -1332,6 +1332,12 @@ static int r300_startup(struct radeon_device *rdev)
1332 if (r) 1332 if (r)
1333 return r; 1333 return r;
1334 } 1334 }
1335
1336 /* allocate wb buffer */
1337 r = radeon_wb_init(rdev);
1338 if (r)
1339 return r;
1340
1335 /* Enable IRQ */ 1341 /* Enable IRQ */
1336 r100_irq_set(rdev); 1342 r100_irq_set(rdev);
1337 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 1343 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -1341,9 +1347,6 @@ static int r300_startup(struct radeon_device *rdev)
1341 dev_err(rdev->dev, "failled initializing CP (%d).\n", r); 1347 dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
1342 return r; 1348 return r;
1343 } 1349 }
1344 r = r100_wb_init(rdev);
1345 if (r)
1346 dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
1347 r = r100_ib_init(rdev); 1350 r = r100_ib_init(rdev);
1348 if (r) { 1351 if (r) {
1349 dev_err(rdev->dev, "failled initializing IB (%d).\n", r); 1352 dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
@@ -1379,7 +1382,7 @@ int r300_resume(struct radeon_device *rdev)
1379int r300_suspend(struct radeon_device *rdev) 1382int r300_suspend(struct radeon_device *rdev)
1380{ 1383{
1381 r100_cp_disable(rdev); 1384 r100_cp_disable(rdev);
1382 r100_wb_disable(rdev); 1385 radeon_wb_disable(rdev);
1383 r100_irq_disable(rdev); 1386 r100_irq_disable(rdev);
1384 if (rdev->flags & RADEON_IS_PCIE) 1387 if (rdev->flags & RADEON_IS_PCIE)
1385 rv370_pcie_gart_disable(rdev); 1388 rv370_pcie_gart_disable(rdev);
@@ -1391,7 +1394,7 @@ int r300_suspend(struct radeon_device *rdev)
1391void r300_fini(struct radeon_device *rdev) 1394void r300_fini(struct radeon_device *rdev)
1392{ 1395{
1393 r100_cp_fini(rdev); 1396 r100_cp_fini(rdev);
1394 r100_wb_fini(rdev); 1397 radeon_wb_fini(rdev);
1395 r100_ib_fini(rdev); 1398 r100_ib_fini(rdev);
1396 radeon_gem_fini(rdev); 1399 radeon_gem_fini(rdev);
1397 if (rdev->flags & RADEON_IS_PCIE) 1400 if (rdev->flags & RADEON_IS_PCIE)
@@ -1484,7 +1487,7 @@ int r300_init(struct radeon_device *rdev)
1484 /* Somethings want wront with the accel init stop accel */ 1487 /* Somethings want wront with the accel init stop accel */
1485 dev_err(rdev->dev, "Disabling GPU acceleration\n"); 1488 dev_err(rdev->dev, "Disabling GPU acceleration\n");
1486 r100_cp_fini(rdev); 1489 r100_cp_fini(rdev);
1487 r100_wb_fini(rdev); 1490 radeon_wb_fini(rdev);
1488 r100_ib_fini(rdev); 1491 r100_ib_fini(rdev);
1489 radeon_irq_kms_fini(rdev); 1492 radeon_irq_kms_fini(rdev);
1490 if (rdev->flags & RADEON_IS_PCIE) 1493 if (rdev->flags & RADEON_IS_PCIE)
diff --git a/drivers/gpu/drm/radeon/r420.c b/drivers/gpu/drm/radeon/r420.c
index 59f7bccc5be0..c387346f93a9 100644
--- a/drivers/gpu/drm/radeon/r420.c
+++ b/drivers/gpu/drm/radeon/r420.c
@@ -248,6 +248,12 @@ static int r420_startup(struct radeon_device *rdev)
248 return r; 248 return r;
249 } 249 }
250 r420_pipes_init(rdev); 250 r420_pipes_init(rdev);
251
252 /* allocate wb buffer */
253 r = radeon_wb_init(rdev);
254 if (r)
255 return r;
256
251 /* Enable IRQ */ 257 /* Enable IRQ */
252 r100_irq_set(rdev); 258 r100_irq_set(rdev);
253 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 259 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -258,10 +264,6 @@ static int r420_startup(struct radeon_device *rdev)
258 return r; 264 return r;
259 } 265 }
260 r420_cp_errata_init(rdev); 266 r420_cp_errata_init(rdev);
261 r = r100_wb_init(rdev);
262 if (r) {
263 dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
264 }
265 r = r100_ib_init(rdev); 267 r = r100_ib_init(rdev);
266 if (r) { 268 if (r) {
267 dev_err(rdev->dev, "failled initializing IB (%d).\n", r); 269 dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
@@ -302,7 +304,7 @@ int r420_suspend(struct radeon_device *rdev)
302{ 304{
303 r420_cp_errata_fini(rdev); 305 r420_cp_errata_fini(rdev);
304 r100_cp_disable(rdev); 306 r100_cp_disable(rdev);
305 r100_wb_disable(rdev); 307 radeon_wb_disable(rdev);
306 r100_irq_disable(rdev); 308 r100_irq_disable(rdev);
307 if (rdev->flags & RADEON_IS_PCIE) 309 if (rdev->flags & RADEON_IS_PCIE)
308 rv370_pcie_gart_disable(rdev); 310 rv370_pcie_gart_disable(rdev);
@@ -314,7 +316,7 @@ int r420_suspend(struct radeon_device *rdev)
314void r420_fini(struct radeon_device *rdev) 316void r420_fini(struct radeon_device *rdev)
315{ 317{
316 r100_cp_fini(rdev); 318 r100_cp_fini(rdev);
317 r100_wb_fini(rdev); 319 radeon_wb_fini(rdev);
318 r100_ib_fini(rdev); 320 r100_ib_fini(rdev);
319 radeon_gem_fini(rdev); 321 radeon_gem_fini(rdev);
320 if (rdev->flags & RADEON_IS_PCIE) 322 if (rdev->flags & RADEON_IS_PCIE)
@@ -418,7 +420,7 @@ int r420_init(struct radeon_device *rdev)
418 /* Somethings want wront with the accel init stop accel */ 420 /* Somethings want wront with the accel init stop accel */
419 dev_err(rdev->dev, "Disabling GPU acceleration\n"); 421 dev_err(rdev->dev, "Disabling GPU acceleration\n");
420 r100_cp_fini(rdev); 422 r100_cp_fini(rdev);
421 r100_wb_fini(rdev); 423 radeon_wb_fini(rdev);
422 r100_ib_fini(rdev); 424 r100_ib_fini(rdev);
423 radeon_irq_kms_fini(rdev); 425 radeon_irq_kms_fini(rdev);
424 if (rdev->flags & RADEON_IS_PCIE) 426 if (rdev->flags & RADEON_IS_PCIE)
diff --git a/drivers/gpu/drm/radeon/r520.c b/drivers/gpu/drm/radeon/r520.c
index 1458dee902dd..3c8677f9e385 100644
--- a/drivers/gpu/drm/radeon/r520.c
+++ b/drivers/gpu/drm/radeon/r520.c
@@ -181,6 +181,12 @@ static int r520_startup(struct radeon_device *rdev)
181 if (r) 181 if (r)
182 return r; 182 return r;
183 } 183 }
184
185 /* allocate wb buffer */
186 r = radeon_wb_init(rdev);
187 if (r)
188 return r;
189
184 /* Enable IRQ */ 190 /* Enable IRQ */
185 rs600_irq_set(rdev); 191 rs600_irq_set(rdev);
186 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 192 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -190,9 +196,6 @@ static int r520_startup(struct radeon_device *rdev)
190 dev_err(rdev->dev, "failled initializing CP (%d).\n", r); 196 dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
191 return r; 197 return r;
192 } 198 }
193 r = r100_wb_init(rdev);
194 if (r)
195 dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
196 r = r100_ib_init(rdev); 199 r = r100_ib_init(rdev);
197 if (r) { 200 if (r) {
198 dev_err(rdev->dev, "failled initializing IB (%d).\n", r); 201 dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
@@ -295,7 +298,7 @@ int r520_init(struct radeon_device *rdev)
295 /* Somethings want wront with the accel init stop accel */ 298 /* Somethings want wront with the accel init stop accel */
296 dev_err(rdev->dev, "Disabling GPU acceleration\n"); 299 dev_err(rdev->dev, "Disabling GPU acceleration\n");
297 r100_cp_fini(rdev); 300 r100_cp_fini(rdev);
298 r100_wb_fini(rdev); 301 radeon_wb_fini(rdev);
299 r100_ib_fini(rdev); 302 r100_ib_fini(rdev);
300 radeon_irq_kms_fini(rdev); 303 radeon_irq_kms_fini(rdev);
301 rv370_pcie_gart_fini(rdev); 304 rv370_pcie_gart_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index 927509ff349a..fbce58b2cd04 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -1918,6 +1918,7 @@ void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1918void r600_cp_stop(struct radeon_device *rdev) 1918void r600_cp_stop(struct radeon_device *rdev)
1919{ 1919{
1920 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1)); 1920 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1921 WREG32(SCRATCH_UMSK, 0);
1921} 1922}
1922 1923
1923int r600_init_microcode(struct radeon_device *rdev) 1924int r600_init_microcode(struct radeon_device *rdev)
@@ -2150,7 +2151,7 @@ int r600_cp_resume(struct radeon_device *rdev)
2150 2151
2151 /* Set ring buffer size */ 2152 /* Set ring buffer size */
2152 rb_bufsz = drm_order(rdev->cp.ring_size / 8); 2153 rb_bufsz = drm_order(rdev->cp.ring_size / 8);
2153 tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 2154 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2154#ifdef __BIG_ENDIAN 2155#ifdef __BIG_ENDIAN
2155 tmp |= BUF_SWAP_32BIT; 2156 tmp |= BUF_SWAP_32BIT;
2156#endif 2157#endif
@@ -2164,8 +2165,19 @@ int r600_cp_resume(struct radeon_device *rdev)
2164 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 2165 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2165 WREG32(CP_RB_RPTR_WR, 0); 2166 WREG32(CP_RB_RPTR_WR, 0);
2166 WREG32(CP_RB_WPTR, 0); 2167 WREG32(CP_RB_WPTR, 0);
2167 WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF); 2168
2168 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr)); 2169 /* set the wb address whether it's enabled or not */
2170 WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
2171 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2172 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2173
2174 if (rdev->wb.enabled)
2175 WREG32(SCRATCH_UMSK, 0xff);
2176 else {
2177 tmp |= RB_NO_UPDATE;
2178 WREG32(SCRATCH_UMSK, 0);
2179 }
2180
2169 mdelay(1); 2181 mdelay(1);
2170 WREG32(CP_RB_CNTL, tmp); 2182 WREG32(CP_RB_CNTL, tmp);
2171 2183
@@ -2217,9 +2229,10 @@ void r600_scratch_init(struct radeon_device *rdev)
2217 int i; 2229 int i;
2218 2230
2219 rdev->scratch.num_reg = 7; 2231 rdev->scratch.num_reg = 7;
2232 rdev->scratch.reg_base = SCRATCH_REG0;
2220 for (i = 0; i < rdev->scratch.num_reg; i++) { 2233 for (i = 0; i < rdev->scratch.num_reg; i++) {
2221 rdev->scratch.free[i] = true; 2234 rdev->scratch.free[i] = true;
2222 rdev->scratch.reg[i] = SCRATCH_REG0 + (i * 4); 2235 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2223 } 2236 }
2224} 2237}
2225 2238
@@ -2263,70 +2276,6 @@ int r600_ring_test(struct radeon_device *rdev)
2263 return r; 2276 return r;
2264} 2277}
2265 2278
2266void r600_wb_disable(struct radeon_device *rdev)
2267{
2268 int r;
2269
2270 WREG32(SCRATCH_UMSK, 0);
2271 if (rdev->wb.wb_obj) {
2272 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
2273 if (unlikely(r != 0))
2274 return;
2275 radeon_bo_kunmap(rdev->wb.wb_obj);
2276 radeon_bo_unpin(rdev->wb.wb_obj);
2277 radeon_bo_unreserve(rdev->wb.wb_obj);
2278 }
2279}
2280
2281void r600_wb_fini(struct radeon_device *rdev)
2282{
2283 r600_wb_disable(rdev);
2284 if (rdev->wb.wb_obj) {
2285 radeon_bo_unref(&rdev->wb.wb_obj);
2286 rdev->wb.wb = NULL;
2287 rdev->wb.wb_obj = NULL;
2288 }
2289}
2290
2291int r600_wb_enable(struct radeon_device *rdev)
2292{
2293 int r;
2294
2295 if (rdev->wb.wb_obj == NULL) {
2296 r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true,
2297 RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
2298 if (r) {
2299 dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
2300 return r;
2301 }
2302 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
2303 if (unlikely(r != 0)) {
2304 r600_wb_fini(rdev);
2305 return r;
2306 }
2307 r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
2308 &rdev->wb.gpu_addr);
2309 if (r) {
2310 radeon_bo_unreserve(rdev->wb.wb_obj);
2311 dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
2312 r600_wb_fini(rdev);
2313 return r;
2314 }
2315 r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
2316 radeon_bo_unreserve(rdev->wb.wb_obj);
2317 if (r) {
2318 dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
2319 r600_wb_fini(rdev);
2320 return r;
2321 }
2322 }
2323 WREG32(SCRATCH_ADDR, (rdev->wb.gpu_addr >> 8) & 0xFFFFFFFF);
2324 WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + 1024) & 0xFFFFFFFC);
2325 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + 1024) & 0xFF);
2326 WREG32(SCRATCH_UMSK, 0xff);
2327 return 0;
2328}
2329
2330void r600_fence_ring_emit(struct radeon_device *rdev, 2279void r600_fence_ring_emit(struct radeon_device *rdev,
2331 struct radeon_fence *fence) 2280 struct radeon_fence *fence)
2332{ 2281{
@@ -2427,6 +2376,11 @@ int r600_startup(struct radeon_device *rdev)
2427 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); 2376 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
2428 } 2377 }
2429 2378
2379 /* allocate wb buffer */
2380 r = radeon_wb_init(rdev);
2381 if (r)
2382 return r;
2383
2430 /* Enable IRQ */ 2384 /* Enable IRQ */
2431 r = r600_irq_init(rdev); 2385 r = r600_irq_init(rdev);
2432 if (r) { 2386 if (r) {
@@ -2445,8 +2399,7 @@ int r600_startup(struct radeon_device *rdev)
2445 r = r600_cp_resume(rdev); 2399 r = r600_cp_resume(rdev);
2446 if (r) 2400 if (r)
2447 return r; 2401 return r;
2448 /* write back buffer are not vital so don't worry about failure */ 2402
2449 r600_wb_enable(rdev);
2450 return 0; 2403 return 0;
2451} 2404}
2452 2405
@@ -2505,7 +2458,7 @@ int r600_suspend(struct radeon_device *rdev)
2505 r600_cp_stop(rdev); 2458 r600_cp_stop(rdev);
2506 rdev->cp.ready = false; 2459 rdev->cp.ready = false;
2507 r600_irq_suspend(rdev); 2460 r600_irq_suspend(rdev);
2508 r600_wb_disable(rdev); 2461 radeon_wb_disable(rdev);
2509 r600_pcie_gart_disable(rdev); 2462 r600_pcie_gart_disable(rdev);
2510 /* unpin shaders bo */ 2463 /* unpin shaders bo */
2511 if (rdev->r600_blit.shader_obj) { 2464 if (rdev->r600_blit.shader_obj) {
@@ -2602,8 +2555,8 @@ int r600_init(struct radeon_device *rdev)
2602 if (r) { 2555 if (r) {
2603 dev_err(rdev->dev, "disabling GPU acceleration\n"); 2556 dev_err(rdev->dev, "disabling GPU acceleration\n");
2604 r600_cp_fini(rdev); 2557 r600_cp_fini(rdev);
2605 r600_wb_fini(rdev);
2606 r600_irq_fini(rdev); 2558 r600_irq_fini(rdev);
2559 radeon_wb_fini(rdev);
2607 radeon_irq_kms_fini(rdev); 2560 radeon_irq_kms_fini(rdev);
2608 r600_pcie_gart_fini(rdev); 2561 r600_pcie_gart_fini(rdev);
2609 rdev->accel_working = false; 2562 rdev->accel_working = false;
@@ -2633,8 +2586,8 @@ void r600_fini(struct radeon_device *rdev)
2633 r600_audio_fini(rdev); 2586 r600_audio_fini(rdev);
2634 r600_blit_fini(rdev); 2587 r600_blit_fini(rdev);
2635 r600_cp_fini(rdev); 2588 r600_cp_fini(rdev);
2636 r600_wb_fini(rdev);
2637 r600_irq_fini(rdev); 2589 r600_irq_fini(rdev);
2590 radeon_wb_fini(rdev);
2638 radeon_irq_kms_fini(rdev); 2591 radeon_irq_kms_fini(rdev);
2639 r600_pcie_gart_fini(rdev); 2592 r600_pcie_gart_fini(rdev);
2640 radeon_agp_fini(rdev); 2593 radeon_agp_fini(rdev);
@@ -2969,10 +2922,13 @@ int r600_irq_init(struct radeon_device *rdev)
2969 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 2922 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
2970 IH_WPTR_OVERFLOW_CLEAR | 2923 IH_WPTR_OVERFLOW_CLEAR |
2971 (rb_bufsz << 1)); 2924 (rb_bufsz << 1));
2972 /* WPTR writeback, not yet */ 2925
2973 /*ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;*/ 2926 if (rdev->wb.enabled)
2974 WREG32(IH_RB_WPTR_ADDR_LO, 0); 2927 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
2975 WREG32(IH_RB_WPTR_ADDR_HI, 0); 2928
2929 /* set the writeback address whether it's enabled or not */
2930 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
2931 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
2976 2932
2977 WREG32(IH_RB_CNTL, ih_rb_cntl); 2933 WREG32(IH_RB_CNTL, ih_rb_cntl);
2978 2934
@@ -3230,8 +3186,10 @@ static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
3230{ 3186{
3231 u32 wptr, tmp; 3187 u32 wptr, tmp;
3232 3188
3233 /* XXX use writeback */ 3189 if (rdev->wb.enabled)
3234 wptr = RREG32(IH_RB_WPTR); 3190 wptr = rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4];
3191 else
3192 wptr = RREG32(IH_RB_WPTR);
3235 3193
3236 if (wptr & RB_OVERFLOW) { 3194 if (wptr & RB_OVERFLOW) {
3237 /* When a ring buffer overflow happen start parsing interrupt 3195 /* When a ring buffer overflow happen start parsing interrupt
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index a168d644bf9e..4e10938d8dd1 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -365,6 +365,7 @@ bool radeon_atombios_sideport_present(struct radeon_device *rdev);
365 */ 365 */
366struct radeon_scratch { 366struct radeon_scratch {
367 unsigned num_reg; 367 unsigned num_reg;
368 uint32_t reg_base;
368 bool free[32]; 369 bool free[32];
369 uint32_t reg[32]; 370 uint32_t reg[32];
370}; 371};
@@ -593,8 +594,13 @@ struct radeon_wb {
593 struct radeon_bo *wb_obj; 594 struct radeon_bo *wb_obj;
594 volatile uint32_t *wb; 595 volatile uint32_t *wb;
595 uint64_t gpu_addr; 596 uint64_t gpu_addr;
597 bool enabled;
596}; 598};
597 599
600#define RADEON_WB_SCRATCH_OFFSET 0
601#define RADEON_WB_CP_RPTR_OFFSET 1024
602#define R600_WB_IH_WPTR_OFFSET 2048
603
598/** 604/**
599 * struct radeon_pm - power management datas 605 * struct radeon_pm - power management datas
600 * @max_bandwidth: maximum bandwidth the gpu has (MByte/s) 606 * @max_bandwidth: maximum bandwidth the gpu has (MByte/s)
@@ -1340,6 +1346,9 @@ extern void radeon_update_bandwidth_info(struct radeon_device *rdev);
1340extern void radeon_update_display_priority(struct radeon_device *rdev); 1346extern void radeon_update_display_priority(struct radeon_device *rdev);
1341extern bool radeon_boot_test_post_card(struct radeon_device *rdev); 1347extern bool radeon_boot_test_post_card(struct radeon_device *rdev);
1342extern void radeon_scratch_init(struct radeon_device *rdev); 1348extern void radeon_scratch_init(struct radeon_device *rdev);
1349extern void radeon_wb_fini(struct radeon_device *rdev);
1350extern int radeon_wb_init(struct radeon_device *rdev);
1351extern void radeon_wb_disable(struct radeon_device *rdev);
1343extern void radeon_surface_init(struct radeon_device *rdev); 1352extern void radeon_surface_init(struct radeon_device *rdev);
1344extern int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data); 1353extern int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data);
1345extern void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable); 1354extern void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable);
@@ -1424,9 +1433,6 @@ extern int r600_pcie_gart_init(struct radeon_device *rdev);
1424extern void r600_pcie_gart_tlb_flush(struct radeon_device *rdev); 1433extern void r600_pcie_gart_tlb_flush(struct radeon_device *rdev);
1425extern int r600_ib_test(struct radeon_device *rdev); 1434extern int r600_ib_test(struct radeon_device *rdev);
1426extern int r600_ring_test(struct radeon_device *rdev); 1435extern int r600_ring_test(struct radeon_device *rdev);
1427extern void r600_wb_fini(struct radeon_device *rdev);
1428extern int r600_wb_enable(struct radeon_device *rdev);
1429extern void r600_wb_disable(struct radeon_device *rdev);
1430extern void r600_scratch_init(struct radeon_device *rdev); 1436extern void r600_scratch_init(struct radeon_device *rdev);
1431extern int r600_blit_init(struct radeon_device *rdev); 1437extern int r600_blit_init(struct radeon_device *rdev);
1432extern void r600_blit_fini(struct radeon_device *rdev); 1438extern void r600_blit_fini(struct radeon_device *rdev);
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index a5aff755f0d2..6d3b055c02fd 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -108,9 +108,6 @@ void r100_irq_disable(struct radeon_device *rdev);
108void r100_mc_stop(struct radeon_device *rdev, struct r100_mc_save *save); 108void r100_mc_stop(struct radeon_device *rdev, struct r100_mc_save *save);
109void r100_mc_resume(struct radeon_device *rdev, struct r100_mc_save *save); 109void r100_mc_resume(struct radeon_device *rdev, struct r100_mc_save *save);
110void r100_vram_init_sizes(struct radeon_device *rdev); 110void r100_vram_init_sizes(struct radeon_device *rdev);
111void r100_wb_disable(struct radeon_device *rdev);
112void r100_wb_fini(struct radeon_device *rdev);
113int r100_wb_init(struct radeon_device *rdev);
114int r100_cp_reset(struct radeon_device *rdev); 111int r100_cp_reset(struct radeon_device *rdev);
115void r100_vga_render_disable(struct radeon_device *rdev); 112void r100_vga_render_disable(struct radeon_device *rdev);
116void r100_restore_sanity(struct radeon_device *rdev); 113void r100_restore_sanity(struct radeon_device *rdev);
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 256d204a6d24..cfc162d05010 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -117,9 +117,10 @@ void radeon_scratch_init(struct radeon_device *rdev)
117 } else { 117 } else {
118 rdev->scratch.num_reg = 7; 118 rdev->scratch.num_reg = 7;
119 } 119 }
120 rdev->scratch.reg_base = RADEON_SCRATCH_REG0;
120 for (i = 0; i < rdev->scratch.num_reg; i++) { 121 for (i = 0; i < rdev->scratch.num_reg; i++) {
121 rdev->scratch.free[i] = true; 122 rdev->scratch.free[i] = true;
122 rdev->scratch.reg[i] = RADEON_SCRATCH_REG0 + (i * 4); 123 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
123 } 124 }
124} 125}
125 126
@@ -149,6 +150,80 @@ void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg)
149 } 150 }
150} 151}
151 152
153void radeon_wb_disable(struct radeon_device *rdev)
154{
155 int r;
156
157 if (rdev->wb.wb_obj) {
158 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
159 if (unlikely(r != 0))
160 return;
161 radeon_bo_kunmap(rdev->wb.wb_obj);
162 radeon_bo_unpin(rdev->wb.wb_obj);
163 radeon_bo_unreserve(rdev->wb.wb_obj);
164 }
165 rdev->wb.enabled = false;
166}
167
168void radeon_wb_fini(struct radeon_device *rdev)
169{
170 radeon_wb_disable(rdev);
171 if (rdev->wb.wb_obj) {
172 radeon_bo_unref(&rdev->wb.wb_obj);
173 rdev->wb.wb = NULL;
174 rdev->wb.wb_obj = NULL;
175 }
176}
177
178int radeon_wb_init(struct radeon_device *rdev)
179{
180 int r;
181
182 if (rdev->wb.wb_obj == NULL) {
183 r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true,
184 RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
185 if (r) {
186 dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
187 return r;
188 }
189 }
190 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
191 if (unlikely(r != 0)) {
192 radeon_wb_fini(rdev);
193 return r;
194 }
195 r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
196 &rdev->wb.gpu_addr);
197 if (r) {
198 radeon_bo_unreserve(rdev->wb.wb_obj);
199 dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
200 radeon_wb_fini(rdev);
201 return r;
202 }
203 r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
204 radeon_bo_unreserve(rdev->wb.wb_obj);
205 if (r) {
206 dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
207 radeon_wb_fini(rdev);
208 return r;
209 }
210
211 /* disabled via module param */
212 if (radeon_no_wb == 1)
213 rdev->wb.enabled = false;
214 else {
215 /* often unreliable on AGP */
216 if (rdev->flags & RADEON_IS_AGP) {
217 rdev->wb.enabled = false;
218 } else
219 rdev->wb.enabled = true;
220 }
221
222 dev_info(rdev->dev, "WB %sabled\n", rdev->wb.enabled ? "en" : "dis");
223
224 return 0;
225}
226
152/** 227/**
153 * radeon_vram_location - try to find VRAM location 228 * radeon_vram_location - try to find VRAM location
154 * @rdev: radeon device structure holding all necessary informations 229 * @rdev: radeon device structure holding all necessary informations
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index b1f9a81b5d1d..698a7ed3a6a1 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -72,7 +72,11 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev)
72 bool wake = false; 72 bool wake = false;
73 unsigned long cjiffies; 73 unsigned long cjiffies;
74 74
75 seq = RREG32(rdev->fence_drv.scratch_reg); 75 if (rdev->wb.enabled) {
76 u32 scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
77 seq = rdev->wb.wb[scratch_index/4];
78 } else
79 seq = RREG32(rdev->fence_drv.scratch_reg);
76 if (seq != rdev->fence_drv.last_seq) { 80 if (seq != rdev->fence_drv.last_seq) {
77 rdev->fence_drv.last_seq = seq; 81 rdev->fence_drv.last_seq = seq;
78 rdev->fence_drv.last_jiffies = jiffies; 82 rdev->fence_drv.last_jiffies = jiffies;
diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c
index 261e98a276db..6ea798ce8218 100644
--- a/drivers/gpu/drm/radeon/radeon_ring.c
+++ b/drivers/gpu/drm/radeon/radeon_ring.c
@@ -247,10 +247,14 @@ void radeon_ib_pool_fini(struct radeon_device *rdev)
247 */ 247 */
248void radeon_ring_free_size(struct radeon_device *rdev) 248void radeon_ring_free_size(struct radeon_device *rdev)
249{ 249{
250 if (rdev->family >= CHIP_R600) 250 if (rdev->wb.enabled)
251 rdev->cp.rptr = RREG32(R600_CP_RB_RPTR); 251 rdev->cp.rptr = rdev->wb.wb[RADEON_WB_CP_RPTR_OFFSET/4];
252 else 252 else {
253 rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR); 253 if (rdev->family >= CHIP_R600)
254 rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
255 else
256 rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR);
257 }
254 /* This works because ring_size is a power of 2 */ 258 /* This works because ring_size is a power of 2 */
255 rdev->cp.ring_free_dw = (rdev->cp.rptr + (rdev->cp.ring_size / 4)); 259 rdev->cp.ring_free_dw = (rdev->cp.rptr + (rdev->cp.ring_size / 4));
256 rdev->cp.ring_free_dw -= rdev->cp.wptr; 260 rdev->cp.ring_free_dw -= rdev->cp.wptr;
diff --git a/drivers/gpu/drm/radeon/rs400.c b/drivers/gpu/drm/radeon/rs400.c
index ae2b76b9a388..f683e51a2a06 100644
--- a/drivers/gpu/drm/radeon/rs400.c
+++ b/drivers/gpu/drm/radeon/rs400.c
@@ -397,6 +397,12 @@ static int rs400_startup(struct radeon_device *rdev)
397 r = rs400_gart_enable(rdev); 397 r = rs400_gart_enable(rdev);
398 if (r) 398 if (r)
399 return r; 399 return r;
400
401 /* allocate wb buffer */
402 r = radeon_wb_init(rdev);
403 if (r)
404 return r;
405
400 /* Enable IRQ */ 406 /* Enable IRQ */
401 r100_irq_set(rdev); 407 r100_irq_set(rdev);
402 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 408 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -406,9 +412,6 @@ static int rs400_startup(struct radeon_device *rdev)
406 dev_err(rdev->dev, "failled initializing CP (%d).\n", r); 412 dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
407 return r; 413 return r;
408 } 414 }
409 r = r100_wb_init(rdev);
410 if (r)
411 dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
412 r = r100_ib_init(rdev); 415 r = r100_ib_init(rdev);
413 if (r) { 416 if (r) {
414 dev_err(rdev->dev, "failled initializing IB (%d).\n", r); 417 dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
@@ -443,7 +446,7 @@ int rs400_resume(struct radeon_device *rdev)
443int rs400_suspend(struct radeon_device *rdev) 446int rs400_suspend(struct radeon_device *rdev)
444{ 447{
445 r100_cp_disable(rdev); 448 r100_cp_disable(rdev);
446 r100_wb_disable(rdev); 449 radeon_wb_disable(rdev);
447 r100_irq_disable(rdev); 450 r100_irq_disable(rdev);
448 rs400_gart_disable(rdev); 451 rs400_gart_disable(rdev);
449 return 0; 452 return 0;
@@ -452,7 +455,7 @@ int rs400_suspend(struct radeon_device *rdev)
452void rs400_fini(struct radeon_device *rdev) 455void rs400_fini(struct radeon_device *rdev)
453{ 456{
454 r100_cp_fini(rdev); 457 r100_cp_fini(rdev);
455 r100_wb_fini(rdev); 458 radeon_wb_fini(rdev);
456 r100_ib_fini(rdev); 459 r100_ib_fini(rdev);
457 radeon_gem_fini(rdev); 460 radeon_gem_fini(rdev);
458 rs400_gart_fini(rdev); 461 rs400_gart_fini(rdev);
@@ -526,7 +529,7 @@ int rs400_init(struct radeon_device *rdev)
526 /* Somethings want wront with the accel init stop accel */ 529 /* Somethings want wront with the accel init stop accel */
527 dev_err(rdev->dev, "Disabling GPU acceleration\n"); 530 dev_err(rdev->dev, "Disabling GPU acceleration\n");
528 r100_cp_fini(rdev); 531 r100_cp_fini(rdev);
529 r100_wb_fini(rdev); 532 radeon_wb_fini(rdev);
530 r100_ib_fini(rdev); 533 r100_ib_fini(rdev);
531 rs400_gart_fini(rdev); 534 rs400_gart_fini(rdev);
532 radeon_irq_kms_fini(rdev); 535 radeon_irq_kms_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index cc05b230d7ef..8d8359a5d459 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -795,6 +795,12 @@ static int rs600_startup(struct radeon_device *rdev)
795 r = rs600_gart_enable(rdev); 795 r = rs600_gart_enable(rdev);
796 if (r) 796 if (r)
797 return r; 797 return r;
798
799 /* allocate wb buffer */
800 r = radeon_wb_init(rdev);
801 if (r)
802 return r;
803
798 /* Enable IRQ */ 804 /* Enable IRQ */
799 rs600_irq_set(rdev); 805 rs600_irq_set(rdev);
800 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 806 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -804,9 +810,6 @@ static int rs600_startup(struct radeon_device *rdev)
804 dev_err(rdev->dev, "failled initializing CP (%d).\n", r); 810 dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
805 return r; 811 return r;
806 } 812 }
807 r = r100_wb_init(rdev);
808 if (r)
809 dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
810 r = r100_ib_init(rdev); 813 r = r100_ib_init(rdev);
811 if (r) { 814 if (r) {
812 dev_err(rdev->dev, "failled initializing IB (%d).\n", r); 815 dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
@@ -847,7 +850,7 @@ int rs600_suspend(struct radeon_device *rdev)
847{ 850{
848 r600_audio_fini(rdev); 851 r600_audio_fini(rdev);
849 r100_cp_disable(rdev); 852 r100_cp_disable(rdev);
850 r100_wb_disable(rdev); 853 radeon_wb_disable(rdev);
851 rs600_irq_disable(rdev); 854 rs600_irq_disable(rdev);
852 rs600_gart_disable(rdev); 855 rs600_gart_disable(rdev);
853 return 0; 856 return 0;
@@ -857,7 +860,7 @@ void rs600_fini(struct radeon_device *rdev)
857{ 860{
858 r600_audio_fini(rdev); 861 r600_audio_fini(rdev);
859 r100_cp_fini(rdev); 862 r100_cp_fini(rdev);
860 r100_wb_fini(rdev); 863 radeon_wb_fini(rdev);
861 r100_ib_fini(rdev); 864 r100_ib_fini(rdev);
862 radeon_gem_fini(rdev); 865 radeon_gem_fini(rdev);
863 rs600_gart_fini(rdev); 866 rs600_gart_fini(rdev);
@@ -931,7 +934,7 @@ int rs600_init(struct radeon_device *rdev)
931 /* Somethings want wront with the accel init stop accel */ 934 /* Somethings want wront with the accel init stop accel */
932 dev_err(rdev->dev, "Disabling GPU acceleration\n"); 935 dev_err(rdev->dev, "Disabling GPU acceleration\n");
933 r100_cp_fini(rdev); 936 r100_cp_fini(rdev);
934 r100_wb_fini(rdev); 937 radeon_wb_fini(rdev);
935 r100_ib_fini(rdev); 938 r100_ib_fini(rdev);
936 rs600_gart_fini(rdev); 939 rs600_gart_fini(rdev);
937 radeon_irq_kms_fini(rdev); 940 radeon_irq_kms_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/rs690.c b/drivers/gpu/drm/radeon/rs690.c
index 3e3f75718be3..70ed66ef1ca8 100644
--- a/drivers/gpu/drm/radeon/rs690.c
+++ b/drivers/gpu/drm/radeon/rs690.c
@@ -615,6 +615,12 @@ static int rs690_startup(struct radeon_device *rdev)
615 r = rs400_gart_enable(rdev); 615 r = rs400_gart_enable(rdev);
616 if (r) 616 if (r)
617 return r; 617 return r;
618
619 /* allocate wb buffer */
620 r = radeon_wb_init(rdev);
621 if (r)
622 return r;
623
618 /* Enable IRQ */ 624 /* Enable IRQ */
619 rs600_irq_set(rdev); 625 rs600_irq_set(rdev);
620 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 626 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -624,9 +630,6 @@ static int rs690_startup(struct radeon_device *rdev)
624 dev_err(rdev->dev, "failled initializing CP (%d).\n", r); 630 dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
625 return r; 631 return r;
626 } 632 }
627 r = r100_wb_init(rdev);
628 if (r)
629 dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
630 r = r100_ib_init(rdev); 633 r = r100_ib_init(rdev);
631 if (r) { 634 if (r) {
632 dev_err(rdev->dev, "failled initializing IB (%d).\n", r); 635 dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
@@ -667,7 +670,7 @@ int rs690_suspend(struct radeon_device *rdev)
667{ 670{
668 r600_audio_fini(rdev); 671 r600_audio_fini(rdev);
669 r100_cp_disable(rdev); 672 r100_cp_disable(rdev);
670 r100_wb_disable(rdev); 673 radeon_wb_disable(rdev);
671 rs600_irq_disable(rdev); 674 rs600_irq_disable(rdev);
672 rs400_gart_disable(rdev); 675 rs400_gart_disable(rdev);
673 return 0; 676 return 0;
@@ -677,7 +680,7 @@ void rs690_fini(struct radeon_device *rdev)
677{ 680{
678 r600_audio_fini(rdev); 681 r600_audio_fini(rdev);
679 r100_cp_fini(rdev); 682 r100_cp_fini(rdev);
680 r100_wb_fini(rdev); 683 radeon_wb_fini(rdev);
681 r100_ib_fini(rdev); 684 r100_ib_fini(rdev);
682 radeon_gem_fini(rdev); 685 radeon_gem_fini(rdev);
683 rs400_gart_fini(rdev); 686 rs400_gart_fini(rdev);
@@ -752,7 +755,7 @@ int rs690_init(struct radeon_device *rdev)
752 /* Somethings want wront with the accel init stop accel */ 755 /* Somethings want wront with the accel init stop accel */
753 dev_err(rdev->dev, "Disabling GPU acceleration\n"); 756 dev_err(rdev->dev, "Disabling GPU acceleration\n");
754 r100_cp_fini(rdev); 757 r100_cp_fini(rdev);
755 r100_wb_fini(rdev); 758 radeon_wb_fini(rdev);
756 r100_ib_fini(rdev); 759 r100_ib_fini(rdev);
757 rs400_gart_fini(rdev); 760 rs400_gart_fini(rdev);
758 radeon_irq_kms_fini(rdev); 761 radeon_irq_kms_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/rv515.c b/drivers/gpu/drm/radeon/rv515.c
index 4d6e86041a9f..5d569f41f4ae 100644
--- a/drivers/gpu/drm/radeon/rv515.c
+++ b/drivers/gpu/drm/radeon/rv515.c
@@ -386,6 +386,12 @@ static int rv515_startup(struct radeon_device *rdev)
386 if (r) 386 if (r)
387 return r; 387 return r;
388 } 388 }
389
390 /* allocate wb buffer */
391 r = radeon_wb_init(rdev);
392 if (r)
393 return r;
394
389 /* Enable IRQ */ 395 /* Enable IRQ */
390 rs600_irq_set(rdev); 396 rs600_irq_set(rdev);
391 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 397 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -395,9 +401,6 @@ static int rv515_startup(struct radeon_device *rdev)
395 dev_err(rdev->dev, "failled initializing CP (%d).\n", r); 401 dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
396 return r; 402 return r;
397 } 403 }
398 r = r100_wb_init(rdev);
399 if (r)
400 dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
401 r = r100_ib_init(rdev); 404 r = r100_ib_init(rdev);
402 if (r) { 405 if (r) {
403 dev_err(rdev->dev, "failled initializing IB (%d).\n", r); 406 dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
@@ -431,7 +434,7 @@ int rv515_resume(struct radeon_device *rdev)
431int rv515_suspend(struct radeon_device *rdev) 434int rv515_suspend(struct radeon_device *rdev)
432{ 435{
433 r100_cp_disable(rdev); 436 r100_cp_disable(rdev);
434 r100_wb_disable(rdev); 437 radeon_wb_disable(rdev);
435 rs600_irq_disable(rdev); 438 rs600_irq_disable(rdev);
436 if (rdev->flags & RADEON_IS_PCIE) 439 if (rdev->flags & RADEON_IS_PCIE)
437 rv370_pcie_gart_disable(rdev); 440 rv370_pcie_gart_disable(rdev);
@@ -447,7 +450,7 @@ void rv515_set_safe_registers(struct radeon_device *rdev)
447void rv515_fini(struct radeon_device *rdev) 450void rv515_fini(struct radeon_device *rdev)
448{ 451{
449 r100_cp_fini(rdev); 452 r100_cp_fini(rdev);
450 r100_wb_fini(rdev); 453 radeon_wb_fini(rdev);
451 r100_ib_fini(rdev); 454 r100_ib_fini(rdev);
452 radeon_gem_fini(rdev); 455 radeon_gem_fini(rdev);
453 rv370_pcie_gart_fini(rdev); 456 rv370_pcie_gart_fini(rdev);
@@ -527,7 +530,7 @@ int rv515_init(struct radeon_device *rdev)
527 /* Somethings want wront with the accel init stop accel */ 530 /* Somethings want wront with the accel init stop accel */
528 dev_err(rdev->dev, "Disabling GPU acceleration\n"); 531 dev_err(rdev->dev, "Disabling GPU acceleration\n");
529 r100_cp_fini(rdev); 532 r100_cp_fini(rdev);
530 r100_wb_fini(rdev); 533 radeon_wb_fini(rdev);
531 r100_ib_fini(rdev); 534 r100_ib_fini(rdev);
532 radeon_irq_kms_fini(rdev); 535 radeon_irq_kms_fini(rdev);
533 rv370_pcie_gart_fini(rdev); 536 rv370_pcie_gart_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c
index bc2beb7d35e9..ff1cc58920c0 100644
--- a/drivers/gpu/drm/radeon/rv770.c
+++ b/drivers/gpu/drm/radeon/rv770.c
@@ -268,6 +268,7 @@ static void rv770_mc_program(struct radeon_device *rdev)
268void r700_cp_stop(struct radeon_device *rdev) 268void r700_cp_stop(struct radeon_device *rdev)
269{ 269{
270 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT)); 270 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
271 WREG32(SCRATCH_UMSK, 0);
271} 272}
272 273
273static int rv770_cp_load_microcode(struct radeon_device *rdev) 274static int rv770_cp_load_microcode(struct radeon_device *rdev)
@@ -1029,6 +1030,11 @@ static int rv770_startup(struct radeon_device *rdev)
1029 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); 1030 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
1030 } 1031 }
1031 1032
1033 /* allocate wb buffer */
1034 r = radeon_wb_init(rdev);
1035 if (r)
1036 return r;
1037
1032 /* Enable IRQ */ 1038 /* Enable IRQ */
1033 r = r600_irq_init(rdev); 1039 r = r600_irq_init(rdev);
1034 if (r) { 1040 if (r) {
@@ -1047,8 +1053,7 @@ static int rv770_startup(struct radeon_device *rdev)
1047 r = r600_cp_resume(rdev); 1053 r = r600_cp_resume(rdev);
1048 if (r) 1054 if (r)
1049 return r; 1055 return r;
1050 /* write back buffer are not vital so don't worry about failure */ 1056
1051 r600_wb_enable(rdev);
1052 return 0; 1057 return 0;
1053} 1058}
1054 1059
@@ -1094,7 +1099,7 @@ int rv770_suspend(struct radeon_device *rdev)
1094 r700_cp_stop(rdev); 1099 r700_cp_stop(rdev);
1095 rdev->cp.ready = false; 1100 rdev->cp.ready = false;
1096 r600_irq_suspend(rdev); 1101 r600_irq_suspend(rdev);
1097 r600_wb_disable(rdev); 1102 radeon_wb_disable(rdev);
1098 rv770_pcie_gart_disable(rdev); 1103 rv770_pcie_gart_disable(rdev);
1099 /* unpin shaders bo */ 1104 /* unpin shaders bo */
1100 if (rdev->r600_blit.shader_obj) { 1105 if (rdev->r600_blit.shader_obj) {
@@ -1189,8 +1194,8 @@ int rv770_init(struct radeon_device *rdev)
1189 if (r) { 1194 if (r) {
1190 dev_err(rdev->dev, "disabling GPU acceleration\n"); 1195 dev_err(rdev->dev, "disabling GPU acceleration\n");
1191 r700_cp_fini(rdev); 1196 r700_cp_fini(rdev);
1192 r600_wb_fini(rdev);
1193 r600_irq_fini(rdev); 1197 r600_irq_fini(rdev);
1198 radeon_wb_fini(rdev);
1194 radeon_irq_kms_fini(rdev); 1199 radeon_irq_kms_fini(rdev);
1195 rv770_pcie_gart_fini(rdev); 1200 rv770_pcie_gart_fini(rdev);
1196 rdev->accel_working = false; 1201 rdev->accel_working = false;
@@ -1222,8 +1227,8 @@ void rv770_fini(struct radeon_device *rdev)
1222{ 1227{
1223 r600_blit_fini(rdev); 1228 r600_blit_fini(rdev);
1224 r700_cp_fini(rdev); 1229 r700_cp_fini(rdev);
1225 r600_wb_fini(rdev);
1226 r600_irq_fini(rdev); 1230 r600_irq_fini(rdev);
1231 radeon_wb_fini(rdev);
1227 radeon_irq_kms_fini(rdev); 1232 radeon_irq_kms_fini(rdev);
1228 rv770_pcie_gart_fini(rdev); 1233 rv770_pcie_gart_fini(rdev);
1229 rv770_vram_scratch_fini(rdev); 1234 rv770_vram_scratch_fini(rdev);