18 files changed, 115 insertions, 46 deletions

diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index c74e4e8006d4..2dd2c93ebfa3 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -2231,6 +2231,9 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         dev_priv->mchdev_lock = &mchdev_lock;
         spin_unlock(&mchdev_lock);
 
+        /* XXX Prevent module unload due to memory corruption bugs. */
+        __module_get(THIS_MODULE);
+
         return 0;
 
 out_workqueue_free:
diff --git a/drivers/gpu/drm/i915/intel_fb.c b/drivers/gpu/drm/i915/intel_fb.c
index 56ad9df2ccb5..b61966c126d3 100644
--- a/drivers/gpu/drm/i915/intel_fb.c
+++ b/drivers/gpu/drm/i915/intel_fb.c
@@ -238,8 +238,8 @@ int intel_fbdev_destroy(struct drm_device *dev,
 
         drm_framebuffer_cleanup(&ifb->base);
         if (ifb->obj) {
-                drm_gem_object_handle_unreference(ifb->obj);
                 drm_gem_object_unreference(ifb->obj);
+                ifb->obj = NULL;
         }
 
         return 0;
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index d2047713dc59..dbd30b2e43fd 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -352,7 +352,6 @@ nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *nfbdev)
 
         if (nouveau_fb->nvbo) {
                 nouveau_bo_unmap(nouveau_fb->nvbo);
-                drm_gem_object_handle_unreference_unlocked(nouveau_fb->nvbo->gem);
                 drm_gem_object_unreference_unlocked(nouveau_fb->nvbo->gem);
                 nouveau_fb->nvbo = NULL;
         }
diff --git a/drivers/gpu/drm/nouveau/nouveau_notifier.c b/drivers/gpu/drm/nouveau/nouveau_notifier.c
index 3c9964a8fbad..3ec181ff50ce 100644
--- a/drivers/gpu/drm/nouveau/nouveau_notifier.c
+++ b/drivers/gpu/drm/nouveau/nouveau_notifier.c
@@ -79,7 +79,6 @@ nouveau_notifier_takedown_channel(struct nouveau_channel *chan)
         mutex_lock(&dev->struct_mutex);
         nouveau_bo_unpin(chan->notifier_bo);
         mutex_unlock(&dev->struct_mutex);
-        drm_gem_object_handle_unreference_unlocked(chan->notifier_bo->gem);
         drm_gem_object_unreference_unlocked(chan->notifier_bo->gem);
         drm_mm_takedown(&chan->notifier_heap);
 }
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index 79082d4398ae..2f93d46ae69a 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -1137,7 +1137,7 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
 
                 WREG32(RCU_IND_INDEX, 0x203);
                 efuse_straps_3 = RREG32(RCU_IND_DATA);
-                efuse_box_bit_127_124 = (u8)(efuse_straps_3 & 0xF0000000) >> 28;
+                efuse_box_bit_127_124 = (u8)((efuse_straps_3 & 0xF0000000) >> 28);
 
                 switch(efuse_box_bit_127_124) {
                 case 0x0:
@@ -1407,6 +1407,7 @@ int evergreen_mc_init(struct radeon_device *rdev)
         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
         rdev->mc.visible_vram_size = rdev->mc.aper_size;
+        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         r600_vram_gtt_location(rdev, &rdev->mc);
         radeon_update_bandwidth_info(rdev);
 
@@ -1520,7 +1521,7 @@ void evergreen_disable_interrupt_state(struct radeon_device *rdev)
 {
         u32 tmp;
 
-        WREG32(CP_INT_CNTL, 0);
+        WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
         WREG32(GRBM_INT_CNTL, 0);
         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index e151f16a8f86..e59422320bb6 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -1030,6 +1030,7 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
                 return r;
         }
         rdev->cp.ready = true;
+        rdev->mc.active_vram_size = rdev->mc.real_vram_size;
         return 0;
 }
 
@@ -1047,6 +1048,7 @@ void r100_cp_fini(struct radeon_device *rdev)
 void r100_cp_disable(struct radeon_device *rdev)
 {
         /* Disable ring */
+        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         rdev->cp.ready = false;
         WREG32(RADEON_CP_CSQ_MODE, 0);
         WREG32(RADEON_CP_CSQ_CNTL, 0);
@@ -2295,6 +2297,7 @@ void r100_vram_init_sizes(struct radeon_device *rdev)
         /* FIXME we don't use the second aperture yet when we could use it */
         if (rdev->mc.visible_vram_size > rdev->mc.aper_size)
                 rdev->mc.visible_vram_size = rdev->mc.aper_size;
+        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         config_aper_size = RREG32(RADEON_CONFIG_APER_SIZE);
         if (rdev->flags & RADEON_IS_IGP) {
                 uint32_t tom;
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index 7a04959ba0ee..7b65e4efe8af 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -1248,6 +1248,7 @@ int r600_mc_init(struct radeon_device *rdev)
         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
         rdev->mc.visible_vram_size = rdev->mc.aper_size;
+        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         r600_vram_gtt_location(rdev, &rdev->mc);
 
         if (rdev->flags & RADEON_IS_IGP) {
@@ -1917,6 +1918,7 @@ void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  */
 void r600_cp_stop(struct radeon_device *rdev)
 {
+        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
 }
 
@@ -2910,7 +2912,7 @@ static void r600_disable_interrupt_state(struct radeon_device *rdev)
 {
         u32 tmp;
 
-        WREG32(CP_INT_CNTL, 0);
+        WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
         WREG32(GRBM_INT_CNTL, 0);
         WREG32(DxMODE_INT_MASK, 0);
         if (ASIC_IS_DCE3(rdev)) {
diff --git a/drivers/gpu/drm/radeon/r600_blit_kms.c b/drivers/gpu/drm/radeon/r600_blit_kms.c
index 9ceb2a1ce799..3473c00781ff 100644
--- a/drivers/gpu/drm/radeon/r600_blit_kms.c
+++ b/drivers/gpu/drm/radeon/r600_blit_kms.c
@@ -532,6 +532,7 @@ int r600_blit_init(struct radeon_device *rdev)
         memcpy(ptr + rdev->r600_blit.ps_offset, r6xx_ps, r6xx_ps_size * 4);
         radeon_bo_kunmap(rdev->r600_blit.shader_obj);
         radeon_bo_unreserve(rdev->r600_blit.shader_obj);
+        rdev->mc.active_vram_size = rdev->mc.real_vram_size;
         return 0;
 }
 
@@ -539,6 +540,7 @@ void r600_blit_fini(struct radeon_device *rdev)
 {
         int r;
 
+        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         if (rdev->r600_blit.shader_obj == NULL)
                 return;
         /* If we can't reserve the bo, unref should be enough to destroy
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index a168d644bf9e..9ff38c99a6ea 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -344,6 +344,7 @@ struct radeon_mc {
          * about vram size near mc fb location */
         u64                     mc_vram_size;
         u64                     visible_vram_size;
+        u64                     active_vram_size;
         u64                     gtt_size;
         u64                     gtt_start;
         u64                     gtt_end;
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index 68932ba7b8a4..8e43ddae70cc 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -1558,39 +1558,39 @@ radeon_atombios_get_tv_info(struct radeon_device *rdev)
                 switch (tv_info->ucTV_BootUpDefaultStandard) {
                 case ATOM_TV_NTSC:
                         tv_std = TV_STD_NTSC;
-                        DRM_INFO("Default TV standard: NTSC\n");
+                        DRM_DEBUG_KMS("Default TV standard: NTSC\n");
                         break;
                 case ATOM_TV_NTSCJ:
                         tv_std = TV_STD_NTSC_J;
-                        DRM_INFO("Default TV standard: NTSC-J\n");
+                        DRM_DEBUG_KMS("Default TV standard: NTSC-J\n");
                         break;
                 case ATOM_TV_PAL:
                         tv_std = TV_STD_PAL;
-                        DRM_INFO("Default TV standard: PAL\n");
+                        DRM_DEBUG_KMS("Default TV standard: PAL\n");
                         break;
                 case ATOM_TV_PALM:
                         tv_std = TV_STD_PAL_M;
-                        DRM_INFO("Default TV standard: PAL-M\n");
+                        DRM_DEBUG_KMS("Default TV standard: PAL-M\n");
                         break;
                 case ATOM_TV_PALN:
                         tv_std = TV_STD_PAL_N;
-                        DRM_INFO("Default TV standard: PAL-N\n");
+                        DRM_DEBUG_KMS("Default TV standard: PAL-N\n");
                         break;
                 case ATOM_TV_PALCN:
                         tv_std = TV_STD_PAL_CN;
-                        DRM_INFO("Default TV standard: PAL-CN\n");
+                        DRM_DEBUG_KMS("Default TV standard: PAL-CN\n");
                         break;
                 case ATOM_TV_PAL60:
                         tv_std = TV_STD_PAL_60;
-                        DRM_INFO("Default TV standard: PAL-60\n");
+                        DRM_DEBUG_KMS("Default TV standard: PAL-60\n");
                         break;
                 case ATOM_TV_SECAM:
                         tv_std = TV_STD_SECAM;
-                        DRM_INFO("Default TV standard: SECAM\n");
+                        DRM_DEBUG_KMS("Default TV standard: SECAM\n");
                         break;
                 default:
                         tv_std = TV_STD_NTSC;
-                        DRM_INFO("Unknown TV standard; defaulting to NTSC\n");
+                        DRM_DEBUG_KMS("Unknown TV standard; defaulting to NTSC\n");
                         break;
                 }
         }
diff --git a/drivers/gpu/drm/radeon/radeon_combios.c b/drivers/gpu/drm/radeon/radeon_combios.c
index a04b7a6ad95f..7b7ea269549c 100644
--- a/drivers/gpu/drm/radeon/radeon_combios.c
+++ b/drivers/gpu/drm/radeon/radeon_combios.c
@@ -913,47 +913,47 @@ radeon_combios_get_tv_info(struct radeon_device *rdev)
                         switch (RBIOS8(tv_info + 7) & 0xf) {
                         case 1:
                                 tv_std = TV_STD_NTSC;
-                                DRM_INFO("Default TV standard: NTSC\n");
+                                DRM_DEBUG_KMS("Default TV standard: NTSC\n");
                                 break;
                         case 2:
                                 tv_std = TV_STD_PAL;
-                                DRM_INFO("Default TV standard: PAL\n");
+                                DRM_DEBUG_KMS("Default TV standard: PAL\n");
                                 break;
                         case 3:
                                 tv_std = TV_STD_PAL_M;
-                                DRM_INFO("Default TV standard: PAL-M\n");
+                                DRM_DEBUG_KMS("Default TV standard: PAL-M\n");
                                 break;
                         case 4:
                                 tv_std = TV_STD_PAL_60;
-                                DRM_INFO("Default TV standard: PAL-60\n");
+                                DRM_DEBUG_KMS("Default TV standard: PAL-60\n");
                                 break;
                         case 5:
                                 tv_std = TV_STD_NTSC_J;
-                                DRM_INFO("Default TV standard: NTSC-J\n");
+                                DRM_DEBUG_KMS("Default TV standard: NTSC-J\n");
                                 break;
                         case 6:
                                 tv_std = TV_STD_SCART_PAL;
-                                DRM_INFO("Default TV standard: SCART-PAL\n");
+                                DRM_DEBUG_KMS("Default TV standard: SCART-PAL\n");
                                 break;
                         default:
                                 tv_std = TV_STD_NTSC;
-                                DRM_INFO
+                                DRM_DEBUG_KMS
                                     ("Unknown TV standard; defaulting to NTSC\n");
                                 break;
                         }
 
                         switch ((RBIOS8(tv_info + 9) >> 2) & 0x3) {
                         case 0:
-                                DRM_INFO("29.498928713 MHz TV ref clk\n");
+                                DRM_DEBUG_KMS("29.498928713 MHz TV ref clk\n");
                                 break;
                         case 1:
-                                DRM_INFO("28.636360000 MHz TV ref clk\n");
+                                DRM_DEBUG_KMS("28.636360000 MHz TV ref clk\n");
                                 break;
                         case 2:
-                                DRM_INFO("14.318180000 MHz TV ref clk\n");
+                                DRM_DEBUG_KMS("14.318180000 MHz TV ref clk\n");
                                 break;
                         case 3:
-                                DRM_INFO("27.000000000 MHz TV ref clk\n");
+                                DRM_DEBUG_KMS("27.000000000 MHz TV ref clk\n");
                                 break;
                         default:
                                 break;
@@ -1324,7 +1324,7 @@ bool radeon_legacy_get_tmds_info_from_combios(struct radeon_encoder *encoder,
 
         if (tmds_info) {
                 ver = RBIOS8(tmds_info);
-                DRM_INFO("DFP table revision: %d\n", ver);
+                DRM_DEBUG_KMS("DFP table revision: %d\n", ver);
                 if (ver == 3) {
                         n = RBIOS8(tmds_info + 5) + 1;
                         if (n > 4)
@@ -1408,7 +1408,7 @@ bool radeon_legacy_get_ext_tmds_info_from_combios(struct radeon_encoder *encoder
                 offset = combios_get_table_offset(dev, COMBIOS_EXT_TMDS_INFO_TABLE);
                 if (offset) {
                         ver = RBIOS8(offset);
-                        DRM_INFO("External TMDS Table revision: %d\n", ver);
+                        DRM_DEBUG_KMS("External TMDS Table revision: %d\n", ver);
                         tmds->slave_addr = RBIOS8(offset + 4 + 2);
                         tmds->slave_addr >>= 1; /* 7 bit addressing */
                         gpio = RBIOS8(offset + 4 + 3);
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index 9cdf6a35bc2c..40b0c087b592 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -97,7 +97,6 @@ static void radeonfb_destroy_pinned_object(struct drm_gem_object *gobj)
                 radeon_bo_unpin(rbo);
                 radeon_bo_unreserve(rbo);
         }
-        drm_gem_object_handle_unreference(gobj);
         drm_gem_object_unreference_unlocked(gobj);
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index 0afd1e62347d..b3b5306bb578 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -69,7 +69,7 @@ void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain)
         u32 c = 0;
 
         rbo->placement.fpfn = 0;
-        rbo->placement.lpfn = 0;
+        rbo->placement.lpfn = rbo->rdev->mc.active_vram_size >> PAGE_SHIFT;
         rbo->placement.placement = rbo->placements;
         rbo->placement.busy_placement = rbo->placements;
         if (domain & RADEON_GEM_DOMAIN_VRAM)
diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h
index 353998dc2c03..3481bc7f6f58 100644
--- a/drivers/gpu/drm/radeon/radeon_object.h
+++ b/drivers/gpu/drm/radeon/radeon_object.h
@@ -124,11 +124,8 @@ static inline int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type,
         int r;
 
         r = ttm_bo_reserve(&bo->tbo, true, no_wait, false, 0);
-        if (unlikely(r != 0)) {
-                if (r != -ERESTARTSYS)
-                        dev_err(bo->rdev->dev, "%p reserve failed for wait\n", bo);
+        if (unlikely(r != 0))
                 return r;
-        }
         spin_lock(&bo->tbo.lock);
         if (mem_type)
                 *mem_type = bo->tbo.mem.mem_type;
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index cc05b230d7ef..51d5f7b5ab21 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -693,6 +693,7 @@ void rs600_mc_init(struct radeon_device *rdev)
         rdev->mc.real_vram_size = RREG32(RADEON_CONFIG_MEMSIZE);
         rdev->mc.mc_vram_size = rdev->mc.real_vram_size;
         rdev->mc.visible_vram_size = rdev->mc.aper_size;
+        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
         base = RREG32_MC(R_000004_MC_FB_LOCATION);
         base = G_000004_MC_FB_START(base) << 16;
diff --git a/drivers/gpu/drm/radeon/rs690.c b/drivers/gpu/drm/radeon/rs690.c
index 3e3f75718be3..4dc2a87ea680 100644
--- a/drivers/gpu/drm/radeon/rs690.c
+++ b/drivers/gpu/drm/radeon/rs690.c
@@ -157,6 +157,7 @@ void rs690_mc_init(struct radeon_device *rdev)
         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
         rdev->mc.visible_vram_size = rdev->mc.aper_size;
+        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         base = RREG32_MC(R_000100_MCCFG_FB_LOCATION);
         base = G_000100_MC_FB_START(base) << 16;
         rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c
index bfa59db374d2..9490da700749 100644
--- a/drivers/gpu/drm/radeon/rv770.c
+++ b/drivers/gpu/drm/radeon/rv770.c
@@ -267,6 +267,7 @@ static void rv770_mc_program(struct radeon_device *rdev)
  */
 void r700_cp_stop(struct radeon_device *rdev)
 {
+        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
 }
 
@@ -992,6 +993,7 @@ int rv770_mc_init(struct radeon_device *rdev)
         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
         rdev->mc.visible_vram_size = rdev->mc.aper_size;
+        rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
         r600_vram_gtt_location(rdev, &rdev->mc);
         radeon_update_bandwidth_info(rdev);
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index cb4cf7ef4d1e..db809e034cc4 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -442,6 +442,43 @@ out_err:
 }
 
 /**
+ * Call bo::reserved and with the lru lock held.
+ * Will release GPU memory type usage on destruction.
+ * This is the place to put in driver specific hooks.
+ * Will release the bo::reserved lock and the
+ * lru lock on exit.
+ */
+
+static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
+{
+        struct ttm_bo_global *glob = bo->glob;
+
+        if (bo->ttm) {
+
+                /**
+                 * Release the lru_lock, since we don't want to have
+                 * an atomic requirement on ttm_tt[unbind|destroy].
+                 */
+
+                spin_unlock(&glob->lru_lock);
+                ttm_tt_unbind(bo->ttm);
+                ttm_tt_destroy(bo->ttm);
+                bo->ttm = NULL;
+                spin_lock(&glob->lru_lock);
+        }
+
+        if (bo->mem.mm_node) {
+                drm_mm_put_block(bo->mem.mm_node);
+                bo->mem.mm_node = NULL;
+        }
+
+        atomic_set(&bo->reserved, 0);
+        wake_up_all(&bo->event_queue);
+        spin_unlock(&glob->lru_lock);
+}
+
+
+/**
  * If bo idle, remove from delayed- and lru lists, and unref.
  * If not idle, and already on delayed list, do nothing.
  * If not idle, and not on delayed list, put on delayed list,
@@ -456,6 +493,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all)
         int ret;
 
         spin_lock(&bo->lock);
+retry:
         (void) ttm_bo_wait(bo, false, false, !remove_all);
 
         if (!bo->sync_obj) {
@@ -464,31 +502,52 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all)
                 spin_unlock(&bo->lock);
 
                 spin_lock(&glob->lru_lock);
-                put_count = ttm_bo_del_from_lru(bo);
+                ret = ttm_bo_reserve_locked(bo, false, !remove_all, false, 0);
+
+                /**
+                 * Someone else has the object reserved. Bail and retry.
+                 */
 
-                ret = ttm_bo_reserve_locked(bo, false, false, false, 0);
-                BUG_ON(ret);
-                if (bo->ttm)
-                        ttm_tt_unbind(bo->ttm);
+                if (unlikely(ret == -EBUSY)) {
+                        spin_unlock(&glob->lru_lock);
+                        spin_lock(&bo->lock);
+                        goto requeue;
+                }
+
+                /**
+                 * We can re-check for sync object without taking
+                 * the bo::lock since setting the sync object requires
+                 * also bo::reserved. A busy object at this point may
+                 * be caused by another thread starting an accelerated
+                 * eviction.
+                 */
+
+                if (unlikely(bo->sync_obj)) {
+                        atomic_set(&bo->reserved, 0);
+                        wake_up_all(&bo->event_queue);
+                        spin_unlock(&glob->lru_lock);
+                        spin_lock(&bo->lock);
+                        if (remove_all)
+                                goto retry;
+                        else
+                                goto requeue;
+                }
+
+                put_count = ttm_bo_del_from_lru(bo);
 
                 if (!list_empty(&bo->ddestroy)) {
                         list_del_init(&bo->ddestroy);
                         ++put_count;
                 }
-                if (bo->mem.mm_node) {
-                        drm_mm_put_block(bo->mem.mm_node);
-                        bo->mem.mm_node = NULL;
-                }
-                spin_unlock(&glob->lru_lock);
 
-                atomic_set(&bo->reserved, 0);
+                ttm_bo_cleanup_memtype_use(bo);
 
                 while (put_count--)
                         kref_put(&bo->list_kref, ttm_bo_ref_bug);
 
                 return 0;
         }
-
+requeue:
         spin_lock(&glob->lru_lock);
         if (list_empty(&bo->ddestroy)) {
                 void *sync_obj = bo->sync_obj;