1 files changed, 459 insertions, 226 deletions

diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index 7b65e4efe8af..bc54b26cb32f 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -47,6 +47,7 @@
 #define EVERGREEN_PFP_UCODE_SIZE 1120
 #define EVERGREEN_PM4_UCODE_SIZE 1376
 #define EVERGREEN_RLC_UCODE_SIZE 768
+#define CAYMAN_RLC_UCODE_SIZE 1024
 
 /* Firmware Names */
 MODULE_FIRMWARE("radeon/R600_pfp.bin");
@@ -83,6 +84,13 @@ MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
+MODULE_FIRMWARE("radeon/PALM_pfp.bin");
+MODULE_FIRMWARE("radeon/PALM_me.bin");
+MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
+MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
+MODULE_FIRMWARE("radeon/SUMO_me.bin");
+MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
+MODULE_FIRMWARE("radeon/SUMO2_me.bin");
 
 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
 
@@ -91,18 +99,17 @@ int r600_mc_wait_for_idle(struct radeon_device *rdev);
 void r600_gpu_init(struct radeon_device *rdev);
 void r600_fini(struct radeon_device *rdev);
 void r600_irq_disable(struct radeon_device *rdev);
+static void r600_pcie_gen2_enable(struct radeon_device *rdev);
 
 /* get temperature in millidegrees */
-u32 rv6xx_get_temp(struct radeon_device *rdev)
+int rv6xx_get_temp(struct radeon_device *rdev)
 {
         u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
                 ASIC_T_SHIFT;
-        u32 actual_temp = 0;
+        int actual_temp = temp & 0xff;
 
-        if ((temp >> 7) & 1)
-                actual_temp = 0;
-        else
-                actual_temp = (temp >> 1) & 0xff;
+        if (temp & 0x100)
+                actual_temp -= 256;
 
         return actual_temp * 1000;
 }
@@ -583,8 +590,11 @@ void r600_pm_misc(struct radeon_device *rdev)
         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 
         if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
+                /* 0xff01 is a flag rather then an actual voltage */
+                if (voltage->voltage == 0xff01)
+                        return;
                 if (voltage->voltage != rdev->pm.current_vddc) {
-                        radeon_atom_set_voltage(rdev, voltage->voltage);
+                        radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
                         rdev->pm.current_vddc = voltage->voltage;
                         DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
                 }
@@ -884,12 +894,15 @@ void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
         u32 tmp;
 
         /* flush hdp cache so updates hit vram */
-        if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740)) {
+        if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
+            !(rdev->flags & RADEON_IS_AGP)) {
                 void __iomem *ptr = (void *)rdev->gart.table.vram.ptr;
                 u32 tmp;
 
                 /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
                  * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
+                 * This seems to cause problems on some AGP cards. Just use the old
+                 * method for them.
                  */
                 WREG32(HDP_DEBUG1, 0);
                 tmp = readl((void __iomem *)ptr);
@@ -919,7 +932,7 @@ int r600_pcie_gart_init(struct radeon_device *rdev)
         int r;
 
         if (rdev->gart.table.vram.robj) {
-                WARN(1, "R600 PCIE GART already initialized.\n");
+                WARN(1, "R600 PCIE GART already initialized\n");
                 return 0;
         }
         /* Initialize common gart structure */
@@ -1167,7 +1180,7 @@ static void r600_mc_program(struct radeon_device *rdev)
  * Note: GTT start, end, size should be initialized before calling this
  * function on AGP platform.
  */
-void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
+static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
 {
         u64 size_bf, size_af;
 
@@ -1201,8 +1214,10 @@ void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
                                 mc->vram_end, mc->real_vram_size >> 20);
         } else {
                 u64 base = 0;
-                if (rdev->flags & RADEON_IS_IGP)
-                        base = (RREG32(MC_VM_FB_LOCATION) & 0xFFFF) << 24;
+                if (rdev->flags & RADEON_IS_IGP) {
+                        base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
+                        base <<= 24;
+                }
                 radeon_vram_location(rdev, &rdev->mc, base);
                 rdev->mc.gtt_base_align = 0;
                 radeon_gtt_location(rdev, mc);
@@ -1248,7 +1263,6 @@ int r600_mc_init(struct radeon_device *rdev)
         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
         rdev->mc.visible_vram_size = rdev->mc.aper_size;
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         r600_vram_gtt_location(rdev, &rdev->mc);
 
         if (rdev->flags & RADEON_IS_IGP) {
@@ -1284,6 +1298,9 @@ int r600_gpu_soft_reset(struct radeon_device *rdev)
                         S_008014_CB2_BUSY(1) | S_008014_CB3_BUSY(1);
         u32 tmp;
 
+        if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
+                return 0;
+
         dev_info(rdev->dev, "GPU softreset \n");
         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
                 RREG32(R_008010_GRBM_STATUS));
@@ -1343,13 +1360,19 @@ bool r600_gpu_is_lockup(struct radeon_device *rdev)
         u32 srbm_status;
         u32 grbm_status;
         u32 grbm_status2;
+        struct r100_gpu_lockup *lockup;
         int r;
 
+        if (rdev->family >= CHIP_RV770)
+                lockup = &rdev->config.rv770.lockup;
+        else
+                lockup = &rdev->config.r600.lockup;
+
         srbm_status = RREG32(R_000E50_SRBM_STATUS);
         grbm_status = RREG32(R_008010_GRBM_STATUS);
         grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
         if (!G_008010_GUI_ACTIVE(grbm_status)) {
-                r100_gpu_lockup_update(&rdev->config.r300.lockup, &rdev->cp);
+                r100_gpu_lockup_update(lockup, &rdev->cp);
                 return false;
         }
         /* force CP activities */
@@ -1361,7 +1384,7 @@ bool r600_gpu_is_lockup(struct radeon_device *rdev)
                 radeon_ring_unlock_commit(rdev);
         }
         rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
-        return r100_gpu_cp_is_lockup(rdev, &rdev->config.r300.lockup, &rdev->cp);
+        return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
 }
 
 int r600_asic_reset(struct radeon_device *rdev)
@@ -1608,8 +1631,11 @@ void r600_gpu_init(struct radeon_device *rdev)
         rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
         rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
         tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
-        tiling_config |= GROUP_SIZE(0);
-        rdev->config.r600.tiling_group_size = 256;
+        tiling_config |= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
+        if ((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT)
+                rdev->config.r600.tiling_group_size = 512;
+        else
+                rdev->config.r600.tiling_group_size = 256;
         tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
         if (tmp > 3) {
                 tiling_config |= ROW_TILING(3);
@@ -1918,8 +1944,9 @@ void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  */
 void r600_cp_stop(struct radeon_device *rdev)
 {
-        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
+        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
+        WREG32(SCRATCH_UMSK, 0);
 }
 
 int r600_init_microcode(struct radeon_device *rdev)
@@ -2000,6 +2027,18 @@ int r600_init_microcode(struct radeon_device *rdev)
                 chip_name = "CYPRESS";
                 rlc_chip_name = "CYPRESS";
                 break;
+        case CHIP_PALM:
+                chip_name = "PALM";
+                rlc_chip_name = "SUMO";
+                break;
+        case CHIP_SUMO:
+                chip_name = "SUMO";
+                rlc_chip_name = "SUMO";
+                break;
+        case CHIP_SUMO2:
+                chip_name = "SUMO2";
+                rlc_chip_name = "SUMO";
+                break;
         default: BUG();
         }
 
@@ -2081,7 +2120,11 @@ static int r600_cp_load_microcode(struct radeon_device *rdev)
 
         r600_cp_stop(rdev);
 
-        WREG32(CP_RB_CNTL, RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
+        WREG32(CP_RB_CNTL,
+#ifdef __BIG_ENDIAN
+               BUF_SWAP_32BIT |
+#endif
+               RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
 
         /* Reset cp */
         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
@@ -2152,7 +2195,7 @@ int r600_cp_resume(struct radeon_device *rdev)
 
         /* Set ring buffer size */
         rb_bufsz = drm_order(rdev->cp.ring_size / 8);
-        tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
+        tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
 #ifdef __BIG_ENDIAN
         tmp |= BUF_SWAP_32BIT;
 #endif
@@ -2166,8 +2209,23 @@ int r600_cp_resume(struct radeon_device *rdev)
         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
         WREG32(CP_RB_RPTR_WR, 0);
         WREG32(CP_RB_WPTR, 0);
-        WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF);
-        WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr));
+
+        /* set the wb address whether it's enabled or not */
+        WREG32(CP_RB_RPTR_ADDR,
+#ifdef __BIG_ENDIAN
+               RB_RPTR_SWAP(2) |
+#endif
+               ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
+        WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
+        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
+
+        if (rdev->wb.enabled)
+                WREG32(SCRATCH_UMSK, 0xff);
+        else {
+                tmp |= RB_NO_UPDATE;
+                WREG32(SCRATCH_UMSK, 0);
+        }
+
         mdelay(1);
         WREG32(CP_RB_CNTL, tmp);
 
@@ -2219,9 +2277,10 @@ void r600_scratch_init(struct radeon_device *rdev)
         int i;
 
         rdev->scratch.num_reg = 7;
+        rdev->scratch.reg_base = SCRATCH_REG0;
         for (i = 0; i < rdev->scratch.num_reg; i++) {
                 rdev->scratch.free[i] = true;
-                rdev->scratch.reg[i] = SCRATCH_REG0 + (i * 4);
+                rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
         }
 }
 
@@ -2265,88 +2324,34 @@ int r600_ring_test(struct radeon_device *rdev)
         return r;
 }
 
-void r600_wb_disable(struct radeon_device *rdev)
-{
-        int r;
-
-        WREG32(SCRATCH_UMSK, 0);
-        if (rdev->wb.wb_obj) {
-                r = radeon_bo_reserve(rdev->wb.wb_obj, false);
-                if (unlikely(r != 0))
-                        return;
-                radeon_bo_kunmap(rdev->wb.wb_obj);
-                radeon_bo_unpin(rdev->wb.wb_obj);
-                radeon_bo_unreserve(rdev->wb.wb_obj);
-        }
-}
-
-void r600_wb_fini(struct radeon_device *rdev)
-{
-        r600_wb_disable(rdev);
-        if (rdev->wb.wb_obj) {
-                radeon_bo_unref(&rdev->wb.wb_obj);
-                rdev->wb.wb = NULL;
-                rdev->wb.wb_obj = NULL;
-        }
-}
-
-int r600_wb_enable(struct radeon_device *rdev)
-{
-        int r;
-
-        if (rdev->wb.wb_obj == NULL) {
-                r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true,
-                                RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
-                if (r) {
-                        dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
-                        return r;
-                }
-                r = radeon_bo_reserve(rdev->wb.wb_obj, false);
-                if (unlikely(r != 0)) {
-                        r600_wb_fini(rdev);
-                        return r;
-                }
-                r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
-                                &rdev->wb.gpu_addr);
-                if (r) {
-                        radeon_bo_unreserve(rdev->wb.wb_obj);
-                        dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
-                        r600_wb_fini(rdev);
-                        return r;
-                }
-                r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
-                radeon_bo_unreserve(rdev->wb.wb_obj);
-                if (r) {
-                        dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
-                        r600_wb_fini(rdev);
-                        return r;
-                }
-        }
-        WREG32(SCRATCH_ADDR, (rdev->wb.gpu_addr >> 8) & 0xFFFFFFFF);
-        WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + 1024) & 0xFFFFFFFC);
-        WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + 1024) & 0xFF);
-        WREG32(SCRATCH_UMSK, 0xff);
-        return 0;
-}
-
 void r600_fence_ring_emit(struct radeon_device *rdev,
                           struct radeon_fence *fence)
 {
-        /* Also consider EVENT_WRITE_EOP.  it handles the interrupts + timestamps + events */
-
-        radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
-        radeon_ring_write(rdev, CACHE_FLUSH_AND_INV_EVENT);
-        /* wait for 3D idle clean */
-        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
-        radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
-        radeon_ring_write(rdev, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
-        /* Emit fence sequence & fire IRQ */
-        radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
-        radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
-        radeon_ring_write(rdev, fence->seq);
-        /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
-        radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
-        radeon_ring_write(rdev, RB_INT_STAT);
+        if (rdev->wb.use_event) {
+                u64 addr = rdev->wb.gpu_addr + R600_WB_EVENT_OFFSET +
+                        (u64)(rdev->fence_drv.scratch_reg - rdev->scratch.reg_base);
+                /* EVENT_WRITE_EOP - flush caches, send int */
+                radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
+                radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
+                radeon_ring_write(rdev, addr & 0xffffffff);
+                radeon_ring_write(rdev, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
+                radeon_ring_write(rdev, fence->seq);
+                radeon_ring_write(rdev, 0);
+        } else {
+                radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
+                radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
+                /* wait for 3D idle clean */
+                radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+                radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
+                radeon_ring_write(rdev, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
+                /* Emit fence sequence & fire IRQ */
+                radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+                radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
+                radeon_ring_write(rdev, fence->seq);
+                /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
+                radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
+                radeon_ring_write(rdev, RB_INT_STAT);
+        }
 }
 
 int r600_copy_blit(struct radeon_device *rdev,
@@ -2383,28 +2388,13 @@ void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
         /* FIXME: implement */
 }
 
-
-bool r600_card_posted(struct radeon_device *rdev)
-{
-        uint32_t reg;
-
-        /* first check CRTCs */
-        reg = RREG32(D1CRTC_CONTROL) |
-                RREG32(D2CRTC_CONTROL);
-        if (reg & CRTC_EN)
-                return true;
-
-        /* then check MEM_SIZE, in case the crtcs are off */
-        if (RREG32(CONFIG_MEMSIZE))
-                return true;
-
-        return false;
-}
-
 int r600_startup(struct radeon_device *rdev)
 {
         int r;
 
+        /* enable pcie gen2 link */
+        r600_pcie_gen2_enable(rdev);
+
         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
                 r = r600_init_microcode(rdev);
                 if (r) {
@@ -2428,19 +2418,12 @@ int r600_startup(struct radeon_device *rdev)
                 rdev->asic->copy = NULL;
                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
         }
-        /* pin copy shader into vram */
-        if (rdev->r600_blit.shader_obj) {
-                r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
-                if (unlikely(r != 0))
-                        return r;
-                r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM,
-                                &rdev->r600_blit.shader_gpu_addr);
-                radeon_bo_unreserve(rdev->r600_blit.shader_obj);
-                if (r) {
-                        dev_err(rdev->dev, "(%d) pin blit object failed\n", r);
-                        return r;
-                }
-        }
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
         /* Enable IRQ */
         r = r600_irq_init(rdev);
         if (r) {
@@ -2459,8 +2442,7 @@ int r600_startup(struct radeon_device *rdev)
         r = r600_cp_resume(rdev);
         if (r)
                 return r;
-        /* write back buffer are not vital so don't worry about failure */
-        r600_wb_enable(rdev);
+
         return 0;
 }
 
@@ -2497,7 +2479,7 @@ int r600_resume(struct radeon_device *rdev)
 
         r = r600_ib_test(rdev);
         if (r) {
-                DRM_ERROR("radeon: failled testing IB (%d).\n", r);
+                DRM_ERROR("radeon: failed testing IB (%d).\n", r);
                 return r;
         }
 
@@ -2519,7 +2501,7 @@ int r600_suspend(struct radeon_device *rdev)
         r600_cp_stop(rdev);
         rdev->cp.ready = false;
         r600_irq_suspend(rdev);
-        r600_wb_disable(rdev);
+        radeon_wb_disable(rdev);
         r600_pcie_gart_disable(rdev);
         /* unpin shaders bo */
         if (rdev->r600_blit.shader_obj) {
@@ -2542,9 +2524,6 @@ int r600_init(struct radeon_device *rdev)
 {
         int r;
 
-        r = radeon_dummy_page_init(rdev);
-        if (r)
-                return r;
         if (r600_debugfs_mc_info_init(rdev)) {
                 DRM_ERROR("Failed to register debugfs file for mc !\n");
         }
@@ -2566,7 +2545,7 @@ int r600_init(struct radeon_device *rdev)
         if (r)
                 return r;
         /* Post card if necessary */
-        if (!r600_card_posted(rdev)) {
+        if (!radeon_card_posted(rdev)) {
                 if (!rdev->bios) {
                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
                         return -EINVAL;
@@ -2616,8 +2595,8 @@ int r600_init(struct radeon_device *rdev)
         if (r) {
                 dev_err(rdev->dev, "disabling GPU acceleration\n");
                 r600_cp_fini(rdev);
-                r600_wb_fini(rdev);
                 r600_irq_fini(rdev);
+                radeon_wb_fini(rdev);
                 radeon_irq_kms_fini(rdev);
                 r600_pcie_gart_fini(rdev);
                 rdev->accel_working = false;
@@ -2647,8 +2626,9 @@ void r600_fini(struct radeon_device *rdev)
         r600_audio_fini(rdev);
         r600_blit_fini(rdev);
         r600_cp_fini(rdev);
-        r600_wb_fini(rdev);
         r600_irq_fini(rdev);
+        radeon_wb_fini(rdev);
+        radeon_ib_pool_fini(rdev);
         radeon_irq_kms_fini(rdev);
         r600_pcie_gart_fini(rdev);
         radeon_agp_fini(rdev);
@@ -2658,7 +2638,6 @@ void r600_fini(struct radeon_device *rdev)
         radeon_atombios_fini(rdev);
         kfree(rdev->bios);
         rdev->bios = NULL;
-        radeon_dummy_page_fini(rdev);
 }
 
 
@@ -2669,7 +2648,11 @@ void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
 {
         /* FIXME: implement */
         radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
-        radeon_ring_write(rdev, ib->gpu_addr & 0xFFFFFFFC);
+        radeon_ring_write(rdev,
+#ifdef __BIG_ENDIAN
+                          (2 << 0) |
+#endif
+                          (ib->gpu_addr & 0xFFFFFFFC));
         radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
         radeon_ring_write(rdev, ib->length_dw);
 }
@@ -2769,8 +2752,8 @@ static int r600_ih_ring_alloc(struct radeon_device *rdev)
 
         /* Allocate ring buffer */
         if (rdev->ih.ring_obj == NULL) {
-                r = radeon_bo_create(rdev, NULL, rdev->ih.ring_size,
-                                     true,
+                r = radeon_bo_create(rdev, rdev->ih.ring_size,
+                                     PAGE_SIZE, true,
                                      RADEON_GEM_DOMAIN_GTT,
                                      &rdev->ih.ring_obj);
                 if (r) {
@@ -2850,13 +2833,20 @@ static int r600_rlc_init(struct radeon_device *rdev)
         WREG32(RLC_HB_CNTL, 0);
         WREG32(RLC_HB_RPTR, 0);
         WREG32(RLC_HB_WPTR, 0);
-        WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
-        WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
+        if (rdev->family <= CHIP_CAICOS) {
+                WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
+                WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
+        }
         WREG32(RLC_MC_CNTL, 0);
         WREG32(RLC_UCODE_CNTL, 0);
 
         fw_data = (const __be32 *)rdev->rlc_fw->data;
-        if (rdev->family >= CHIP_CEDAR) {
+        if (rdev->family >= CHIP_CAYMAN) {
+                for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
+                        WREG32(RLC_UCODE_ADDR, i);
+                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
+                }
+        } else if (rdev->family >= CHIP_CEDAR) {
                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
                         WREG32(RLC_UCODE_ADDR, i);
                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
@@ -2915,6 +2905,8 @@ static void r600_disable_interrupt_state(struct radeon_device *rdev)
         WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
         WREG32(GRBM_INT_CNTL, 0);
         WREG32(DxMODE_INT_MASK, 0);
+        WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
+        WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
         if (ASIC_IS_DCE3(rdev)) {
                 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
                 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
@@ -2983,10 +2975,13 @@ int r600_irq_init(struct radeon_device *rdev)
         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
                       IH_WPTR_OVERFLOW_CLEAR |
                       (rb_bufsz << 1));
-        /* WPTR writeback, not yet */
-        /*ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;*/
-        WREG32(IH_RB_WPTR_ADDR_LO, 0);
-        WREG32(IH_RB_WPTR_ADDR_HI, 0);
+
+        if (rdev->wb.enabled)
+                ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
+
+        /* set the writeback address whether it's enabled or not */
+        WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
+        WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
 
         WREG32(IH_RB_CNTL, ih_rb_cntl);
 
@@ -3036,9 +3031,10 @@ int r600_irq_set(struct radeon_device *rdev)
         u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
         u32 grbm_int_cntl = 0;
         u32 hdmi1, hdmi2;
+        u32 d1grph = 0, d2grph = 0;
 
         if (!rdev->irq.installed) {
-                WARN(1, "Can't enable IRQ/MSI because no handler is installed.\n");
+                WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
                 return -EINVAL;
         }
         /* don't enable anything if the ih is disabled */
@@ -3070,12 +3066,15 @@ int r600_irq_set(struct radeon_device *rdev)
         if (rdev->irq.sw_int) {
                 DRM_DEBUG("r600_irq_set: sw int\n");
                 cp_int_cntl |= RB_INT_ENABLE;
+                cp_int_cntl |= TIME_STAMP_INT_ENABLE;
         }
-        if (rdev->irq.crtc_vblank_int[0]) {
+        if (rdev->irq.crtc_vblank_int[0] ||
+            rdev->irq.pflip[0]) {
                 DRM_DEBUG("r600_irq_set: vblank 0\n");
                 mode_int |= D1MODE_VBLANK_INT_MASK;
         }
-        if (rdev->irq.crtc_vblank_int[1]) {
+        if (rdev->irq.crtc_vblank_int[1] ||
+            rdev->irq.pflip[1]) {
                 DRM_DEBUG("r600_irq_set: vblank 1\n");
                 mode_int |= D2MODE_VBLANK_INT_MASK;
         }
@@ -3118,6 +3117,8 @@ int r600_irq_set(struct radeon_device *rdev)
 
         WREG32(CP_INT_CNTL, cp_int_cntl);
         WREG32(DxMODE_INT_MASK, mode_int);
+        WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
+        WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
         WREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, hdmi1);
         if (ASIC_IS_DCE3(rdev)) {
@@ -3140,32 +3141,35 @@ int r600_irq_set(struct radeon_device *rdev)
         return 0;
 }
 
-static inline void r600_irq_ack(struct radeon_device *rdev,
-                                u32 *disp_int,
-                                u32 *disp_int_cont,
-                                u32 *disp_int_cont2)
+static inline void r600_irq_ack(struct radeon_device *rdev)
 {
         u32 tmp;
 
         if (ASIC_IS_DCE3(rdev)) {
-                *disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
-                *disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
-                *disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
+                rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
+                rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
+                rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
         } else {
-                *disp_int = RREG32(DISP_INTERRUPT_STATUS);
-                *disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
-                *disp_int_cont2 = 0;
-        }
-
-        if (*disp_int & LB_D1_VBLANK_INTERRUPT)
+                rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
+                rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
+                rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
+        }
+        rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
+        rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
+
+        if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
+                WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
+        if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
+                WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
+        if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
                 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
-        if (*disp_int & LB_D1_VLINE_INTERRUPT)
+        if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
                 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
-        if (*disp_int & LB_D2_VBLANK_INTERRUPT)
+        if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
                 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
-        if (*disp_int & LB_D2_VLINE_INTERRUPT)
+        if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
                 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
-        if (*disp_int & DC_HPD1_INTERRUPT) {
+        if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
                 if (ASIC_IS_DCE3(rdev)) {
                         tmp = RREG32(DC_HPD1_INT_CONTROL);
                         tmp |= DC_HPDx_INT_ACK;
@@ -3176,7 +3180,7 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
                 }
         }
-        if (*disp_int & DC_HPD2_INTERRUPT) {
+        if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
                 if (ASIC_IS_DCE3(rdev)) {
                         tmp = RREG32(DC_HPD2_INT_CONTROL);
                         tmp |= DC_HPDx_INT_ACK;
@@ -3187,7 +3191,7 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
                 }
         }
-        if (*disp_int_cont & DC_HPD3_INTERRUPT) {
+        if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
                 if (ASIC_IS_DCE3(rdev)) {
                         tmp = RREG32(DC_HPD3_INT_CONTROL);
                         tmp |= DC_HPDx_INT_ACK;
@@ -3198,18 +3202,18 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
                 }
         }
-        if (*disp_int_cont & DC_HPD4_INTERRUPT) {
+        if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
                 tmp = RREG32(DC_HPD4_INT_CONTROL);
                 tmp |= DC_HPDx_INT_ACK;
                 WREG32(DC_HPD4_INT_CONTROL, tmp);
         }
         if (ASIC_IS_DCE32(rdev)) {
-                if (*disp_int_cont2 & DC_HPD5_INTERRUPT) {
+                if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
                         tmp = RREG32(DC_HPD5_INT_CONTROL);
                         tmp |= DC_HPDx_INT_ACK;
                         WREG32(DC_HPD5_INT_CONTROL, tmp);
                 }
-                if (*disp_int_cont2 & DC_HPD6_INTERRUPT) {
+                if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
                         tmp = RREG32(DC_HPD5_INT_CONTROL);
                         tmp |= DC_HPDx_INT_ACK;
                         WREG32(DC_HPD6_INT_CONTROL, tmp);
@@ -3231,12 +3235,10 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
 
 void r600_irq_disable(struct radeon_device *rdev)
 {
-        u32 disp_int, disp_int_cont, disp_int_cont2;
-
         r600_disable_interrupts(rdev);
         /* Wait and acknowledge irq */
         mdelay(1);
-        r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
+        r600_irq_ack(rdev);
         r600_disable_interrupt_state(rdev);
 }
 
@@ -3244,8 +3246,10 @@ static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
 {
         u32 wptr, tmp;
 
-        /* XXX use writeback */
-        wptr = RREG32(IH_RB_WPTR);
+        if (rdev->wb.enabled)
+                wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
+        else
+                wptr = RREG32(IH_RB_WPTR);
 
         if (wptr & RB_OVERFLOW) {
                 /* When a ring buffer overflow happen start parsing interrupt
@@ -3294,54 +3298,57 @@ static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
 
 int r600_irq_process(struct radeon_device *rdev)
 {
-        u32 wptr = r600_get_ih_wptr(rdev);
-        u32 rptr = rdev->ih.rptr;
+        u32 wptr;
+        u32 rptr;
         u32 src_id, src_data;
-        u32 ring_index, disp_int, disp_int_cont, disp_int_cont2;
+        u32 ring_index;
         unsigned long flags;
         bool queue_hotplug = false;
 
-        DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
-        if (!rdev->ih.enabled)
+        if (!rdev->ih.enabled || rdev->shutdown)
                 return IRQ_NONE;
 
+        wptr = r600_get_ih_wptr(rdev);
+        rptr = rdev->ih.rptr;
+        DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
+
         spin_lock_irqsave(&rdev->ih.lock, flags);
 
         if (rptr == wptr) {
                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
                 return IRQ_NONE;
         }
-        if (rdev->shutdown) {
-                spin_unlock_irqrestore(&rdev->ih.lock, flags);
-                return IRQ_NONE;
-        }
 
 restart_ih:
         /* display interrupts */
-        r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
+        r600_irq_ack(rdev);
 
         rdev->ih.wptr = wptr;
         while (rptr != wptr) {
                 /* wptr/rptr are in bytes! */
                 ring_index = rptr / 4;
-                src_id =  rdev->ih.ring[ring_index] & 0xff;
-                src_data = rdev->ih.ring[ring_index + 1] & 0xfffffff;
+                src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
+                src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
 
                 switch (src_id) {
                 case 1: /* D1 vblank/vline */
                         switch (src_data) {
                         case 0: /* D1 vblank */
-                                if (disp_int & LB_D1_VBLANK_INTERRUPT) {
-                                        drm_handle_vblank(rdev->ddev, 0);
-                                        rdev->pm.vblank_sync = true;
-                                        wake_up(&rdev->irq.vblank_queue);
-                                        disp_int &= ~LB_D1_VBLANK_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[0]) {
+                                                drm_handle_vblank(rdev->ddev, 0);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (rdev->irq.pflip[0])
+                                                radeon_crtc_handle_flip(rdev, 0);
+                                        rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
                                         DRM_DEBUG("IH: D1 vblank\n");
                                 }
                                 break;
                         case 1: /* D1 vline */
-                                if (disp_int & LB_D1_VLINE_INTERRUPT) {
-                                        disp_int &= ~LB_D1_VLINE_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
                                         DRM_DEBUG("IH: D1 vline\n");
                                 }
                                 break;
@@ -3353,17 +3360,21 @@ restart_ih:
                 case 5: /* D2 vblank/vline */
                         switch (src_data) {
                         case 0: /* D2 vblank */
-                                if (disp_int & LB_D2_VBLANK_INTERRUPT) {
-                                        drm_handle_vblank(rdev->ddev, 1);
-                                        rdev->pm.vblank_sync = true;
-                                        wake_up(&rdev->irq.vblank_queue);
-                                        disp_int &= ~LB_D2_VBLANK_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[1]) {
+                                                drm_handle_vblank(rdev->ddev, 1);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (rdev->irq.pflip[1])
+                                                radeon_crtc_handle_flip(rdev, 1);
+                                        rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
                                         DRM_DEBUG("IH: D2 vblank\n");
                                 }
                                 break;
                         case 1: /* D1 vline */
-                                if (disp_int & LB_D2_VLINE_INTERRUPT) {
-                                        disp_int &= ~LB_D2_VLINE_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
                                         DRM_DEBUG("IH: D2 vline\n");
                                 }
                                 break;
@@ -3375,43 +3386,43 @@ restart_ih:
                 case 19: /* HPD/DAC hotplug */
                         switch (src_data) {
                         case 0:
-                                if (disp_int & DC_HPD1_INTERRUPT) {
-                                        disp_int &= ~DC_HPD1_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD1\n");
                                 }
                                 break;
                         case 1:
-                                if (disp_int & DC_HPD2_INTERRUPT) {
-                                        disp_int &= ~DC_HPD2_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD2\n");
                                 }
                                 break;
                         case 4:
-                                if (disp_int_cont & DC_HPD3_INTERRUPT) {
-                                        disp_int_cont &= ~DC_HPD3_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD3\n");
                                 }
                                 break;
                         case 5:
-                                if (disp_int_cont & DC_HPD4_INTERRUPT) {
-                                        disp_int_cont &= ~DC_HPD4_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD4\n");
                                 }
                                 break;
                         case 10:
-                                if (disp_int_cont2 & DC_HPD5_INTERRUPT) {
-                                        disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD5\n");
                                 }
                                 break;
                         case 12:
-                                if (disp_int_cont2 & DC_HPD6_INTERRUPT) {
-                                        disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
+                                if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
+                                        rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
                                         queue_hotplug = true;
                                         DRM_DEBUG("IH: HPD6\n");
                                 }
@@ -3433,9 +3444,10 @@ restart_ih:
                         break;
                 case 181: /* CP EOP event */
                         DRM_DEBUG("IH: CP EOP\n");
+                        radeon_fence_process(rdev);
                         break;
                 case 233: /* GUI IDLE */
-                        DRM_DEBUG("IH: CP EOP\n");
+                        DRM_DEBUG("IH: GUI idle\n");
                         rdev->pm.gui_idle = true;
                         wake_up(&rdev->irq.idle_queue);
                         break;
@@ -3453,7 +3465,7 @@ restart_ih:
         if (wptr != rdev->ih.wptr)
                 goto restart_ih;
         if (queue_hotplug)
-                queue_work(rdev->wq, &rdev->hotplug_work);
+                schedule_work(&rdev->hotplug_work);
         rdev->ih.rptr = rptr;
         WREG32(IH_RB_RPTR, rdev->ih.rptr);
         spin_unlock_irqrestore(&rdev->ih.lock, flags);
@@ -3528,10 +3540,12 @@ int r600_debugfs_mc_info_init(struct radeon_device *rdev)
 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
 {
         /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
-         * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
+         * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
+         * This seems to cause problems on some AGP cards. Just use the old
+         * method for them.
          */
         if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
-            rdev->vram_scratch.ptr) {
+            rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
                 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
                 u32 tmp;
 
@@ -3540,3 +3554,222 @@ void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
         } else
                 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 }
+
+void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
+{
+        u32 link_width_cntl, mask, target_reg;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return;
+
+        /* FIXME wait for idle */
+
+        switch (lanes) {
+        case 0:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
+                break;
+        case 1:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
+                break;
+        case 2:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
+                break;
+        case 4:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
+                break;
+        case 8:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
+                break;
+        case 12:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
+                break;
+        case 16:
+        default:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
+                break;
+        }
+
+        link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
+
+        if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
+            (mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
+                return;
+
+        if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
+                return;
+
+        link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK |
+                             RADEON_PCIE_LC_RECONFIG_NOW |
+                             R600_PCIE_LC_RENEGOTIATE_EN |
+                             R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
+        link_width_cntl |= mask;
+
+        WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+
+        /* some northbridges can renegotiate the link rather than requiring                                  
+         * a complete re-config.                                                                             
+         * e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)                            
+         */
+        if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
+                link_width_cntl |= R600_PCIE_LC_RENEGOTIATE_EN | R600_PCIE_LC_UPCONFIGURE_SUPPORT;
+        else
+                link_width_cntl |= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
+
+        WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl |
+                                                       RADEON_PCIE_LC_RECONFIG_NOW));
+
+        if (rdev->family >= CHIP_RV770)
+                target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
+        else
+                target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
+
+        /* wait for lane set to complete */
+        link_width_cntl = RREG32(target_reg);
+        while (link_width_cntl == 0xffffffff)
+                link_width_cntl = RREG32(target_reg);
+
+}
+
+int r600_get_pcie_lanes(struct radeon_device *rdev)
+{
+        u32 link_width_cntl;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return 0;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return 0;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return 0;
+
+        /* FIXME wait for idle */
+
+        link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
+
+        switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
+        case RADEON_PCIE_LC_LINK_WIDTH_X0:
+                return 0;
+        case RADEON_PCIE_LC_LINK_WIDTH_X1:
+                return 1;
+        case RADEON_PCIE_LC_LINK_WIDTH_X2:
+                return 2;
+        case RADEON_PCIE_LC_LINK_WIDTH_X4:
+                return 4;
+        case RADEON_PCIE_LC_LINK_WIDTH_X8:
+                return 8;
+        case RADEON_PCIE_LC_LINK_WIDTH_X16:
+        default:
+                return 16;
+        }
+}
+
+static void r600_pcie_gen2_enable(struct radeon_device *rdev)
+{
+        u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
+        u16 link_cntl2;
+
+        if (radeon_pcie_gen2 == 0)
+                return;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return;
+
+        /* only RV6xx+ chips are supported */
+        if (rdev->family <= CHIP_R600)
+                return;
+
+        /* 55 nm r6xx asics */
+        if ((rdev->family == CHIP_RV670) ||
+            (rdev->family == CHIP_RV620) ||
+            (rdev->family == CHIP_RV635)) {
+                /* advertise upconfig capability */
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
+                        lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
+                        link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
+                                             LC_RECONFIG_ARC_MISSING_ESCAPE);
+                        link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
+                        WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+                } else {
+                        link_width_cntl |= LC_UPCONFIGURE_DIS;
+                        WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+                }
+        }
+
+        speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+        if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+            (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+
+                /* 55 nm r6xx asics */
+                if ((rdev->family == CHIP_RV670) ||
+                    (rdev->family == CHIP_RV620) ||
+                    (rdev->family == CHIP_RV635)) {
+                        WREG32(MM_CFGREGS_CNTL, 0x8);
+                        link_cntl2 = RREG32(0x4088);
+                        WREG32(MM_CFGREGS_CNTL, 0);
+                        /* not supported yet */
+                        if (link_cntl2 & SELECTABLE_DEEMPHASIS)
+                                return;
+                }
+
+                speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
+                speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
+                speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
+                speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
+                speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                tmp = RREG32(0x541c);
+                WREG32(0x541c, tmp | 0x8);
+                WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
+                link_cntl2 = RREG16(0x4088);
+                link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
+                link_cntl2 |= 0x2;
+                WREG16(0x4088, link_cntl2);
+                WREG32(MM_CFGREGS_CNTL, 0);
+
+                if ((rdev->family == CHIP_RV670) ||
+                    (rdev->family == CHIP_RV620) ||
+                    (rdev->family == CHIP_RV635)) {
+                        training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
+                        training_cntl &= ~LC_POINT_7_PLUS_EN;
+                        WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
+                } else {
+                        speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                        speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
+                        WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+                }
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl |= LC_GEN2_EN_STRAP;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+        } else {
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
+                if (1)
+                        link_width_cntl |= LC_UPCONFIGURE_DIS;
+                else
+                        link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        }
+}