Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

10 files changed, 747 insertions, 376 deletions

diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile
index b256d4adfafe..f3cf6f02c997 100644
--- a/drivers/gpu/drm/ttm/Makefile
+++ b/drivers/gpu/drm/ttm/Makefile
@@ -4,6 +4,7 @@
 ccflags-y := -Iinclude/drm
 ttm-y := ttm_agp_backend.o ttm_memory.o ttm_tt.o ttm_bo.o \
         ttm_bo_util.o ttm_bo_vm.o ttm_module.o \
-        ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o
+        ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o \
+        ttm_bo_manager.o
 
 obj-$(CONFIG_DRM_TTM) += ttm.o
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c
index 4bf69c404491..1c4a72f681c1 100644
--- a/drivers/gpu/drm/ttm/ttm_agp_backend.c
+++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c
@@ -47,7 +47,8 @@ struct ttm_agp_backend {
 
 static int ttm_agp_populate(struct ttm_backend *backend,
                             unsigned long num_pages, struct page **pages,
-                            struct page *dummy_read_page)
+                            struct page *dummy_read_page,
+                            dma_addr_t *dma_addrs)
 {
         struct ttm_agp_backend *agp_be =
             container_of(backend, struct ttm_agp_backend, backend);
@@ -74,6 +75,7 @@ static int ttm_agp_bind(struct ttm_backend *backend, struct ttm_mem_reg *bo_mem)
 {
         struct ttm_agp_backend *agp_be =
             container_of(backend, struct ttm_agp_backend, backend);
+        struct drm_mm_node *node = bo_mem->mm_node;
         struct agp_memory *mem = agp_be->mem;
         int cached = (bo_mem->placement & TTM_PL_FLAG_CACHED);
         int ret;
@@ -81,7 +83,7 @@ static int ttm_agp_bind(struct ttm_backend *backend, struct ttm_mem_reg *bo_mem)
         mem->is_flushed = 1;
         mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY;
 
-        ret = agp_bind_memory(mem, bo_mem->mm_node->start);
+        ret = agp_bind_memory(mem, node->start);
         if (ret)
                 printk(KERN_ERR TTM_PFX "AGP Bind memory failed.\n");
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index db809e034cc4..2e618b5ac465 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -27,14 +27,6 @@
 /*
  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  */
-/* Notes:
- *
- * We store bo pointer in drm_mm_node struct so we know which bo own a
- * specific node. There is no protection on the pointer, thus to make
- * sure things don't go berserk you have to access this pointer while
- * holding the global lru lock and make sure anytime you free a node you
- * reset the pointer to NULL.
- */
 
 #include "ttm/ttm_module.h"
 #include "ttm/ttm_bo_driver.h"
@@ -45,6 +37,7 @@
 #include <linux/mm.h>
 #include <linux/file.h>
 #include <linux/module.h>
+#include <asm/atomic.h>
 
 #define TTM_ASSERT_LOCKED(param)
 #define TTM_DEBUG(fmt, arg...)
@@ -84,11 +77,8 @@ static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
                 man->available_caching);
         printk(KERN_ERR TTM_PFX "    default_caching: 0x%08X\n",
                 man->default_caching);
-        if (mem_type != TTM_PL_SYSTEM) {
-                spin_lock(&bdev->glob->lru_lock);
-                drm_mm_debug_table(&man->manager, TTM_PFX);
-                spin_unlock(&bdev->glob->lru_lock);
-        }
+        if (mem_type != TTM_PL_SYSTEM)
+                (*man->func->debug)(man, TTM_PFX);
 }
 
 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
@@ -169,22 +159,17 @@ static void ttm_bo_release_list(struct kref *list_kref)
 
 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)
 {
-
         if (interruptible) {
-                int ret = 0;
-
-                ret = wait_event_interruptible(bo->event_queue,
+                return wait_event_interruptible(bo->event_queue,
                                                atomic_read(&bo->reserved) == 0);
-                if (unlikely(ret != 0))
-                        return ret;
         } else {
                 wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0);
+                return 0;
         }
-        return 0;
 }
 EXPORT_SYMBOL(ttm_bo_wait_unreserved);
 
-static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
+void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 {
         struct ttm_bo_device *bdev = bo->bdev;
         struct ttm_mem_type_manager *man;
@@ -206,11 +191,7 @@ static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
         }
 }
 
-/**
- * Call with the lru_lock held.
- */
-
-static int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
+int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 {
         int put_count = 0;
 
@@ -239,9 +220,21 @@ int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
         int ret;
 
         while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) {
-                if (use_sequence && bo->seq_valid &&
-                        (sequence - bo->val_seq < (1 << 31))) {
-                        return -EAGAIN;
+                /**
+                 * Deadlock avoidance for multi-bo reserving.
+                 */
+                if (use_sequence && bo->seq_valid) {
+                        /**
+                         * We've already reserved this one.
+                         */
+                        if (unlikely(sequence == bo->val_seq))
+                                return -EDEADLK;
+                        /**
+                         * Already reserved by a thread that will not back
+                         * off for us. We need to back off.
+                         */
+                        if (unlikely(sequence - bo->val_seq < (1 << 31)))
+                                return -EAGAIN;
                 }
 
                 if (no_wait)
@@ -256,6 +249,14 @@ int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
         }
 
         if (use_sequence) {
+                /**
+                 * Wake up waiters that may need to recheck for deadlock,
+                 * if we decreased the sequence number.
+                 */
+                if (unlikely((bo->val_seq - sequence < (1 << 31))
+                             || !bo->seq_valid))
+                        wake_up_all(&bo->event_queue);
+
                 bo->val_seq = sequence;
                 bo->seq_valid = true;
         } else {
@@ -271,6 +272,13 @@ static void ttm_bo_ref_bug(struct kref *list_kref)
         BUG();
 }
 
+void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
+                         bool never_free)
+{
+        kref_sub(&bo->list_kref, count,
+                 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list);
+}
+
 int ttm_bo_reserve(struct ttm_buffer_object *bo,
                    bool interruptible,
                    bool no_wait, bool use_sequence, uint32_t sequence)
@@ -286,20 +294,24 @@ int ttm_bo_reserve(struct ttm_buffer_object *bo,
                 put_count = ttm_bo_del_from_lru(bo);
         spin_unlock(&glob->lru_lock);
 
-        while (put_count--)
-                kref_put(&bo->list_kref, ttm_bo_ref_bug);
+        ttm_bo_list_ref_sub(bo, put_count, true);
 
         return ret;
 }
 
+void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)
+{
+        ttm_bo_add_to_lru(bo);
+        atomic_set(&bo->reserved, 0);
+        wake_up_all(&bo->event_queue);
+}
+
 void ttm_bo_unreserve(struct ttm_buffer_object *bo)
 {
         struct ttm_bo_global *glob = bo->glob;
 
         spin_lock(&glob->lru_lock);
-        ttm_bo_add_to_lru(bo);
-        atomic_set(&bo->reserved, 0);
-        wake_up_all(&bo->event_queue);
+        ttm_bo_unreserve_locked(bo);
         spin_unlock(&glob->lru_lock);
 }
 EXPORT_SYMBOL(ttm_bo_unreserve);
@@ -366,8 +378,13 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
         int ret = 0;
 
         if (old_is_pci || new_is_pci ||
-            ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0))
-                ttm_bo_unmap_virtual(bo);
+            ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
+                ret = ttm_mem_io_lock(old_man, true);
+                if (unlikely(ret != 0))
+                        goto out_err;
+                ttm_bo_unmap_virtual_locked(bo);
+                ttm_mem_io_unlock(old_man);
+        }
 
         /*
          * Create and bind a ttm if required.
@@ -389,11 +406,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
                 }
 
                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
+                        if (bdev->driver->move_notify)
+                                bdev->driver->move_notify(bo, mem);
                         bo->mem = *mem;
                         mem->mm_node = NULL;
                         goto moved;
                 }
-
         }
 
         if (bdev->driver->move_notify)
@@ -420,11 +438,9 @@ moved:
         }
 
         if (bo->mem.mm_node) {
-                spin_lock(&bo->lock);
-                bo->offset = (bo->mem.mm_node->start << PAGE_SHIFT) +
+                bo->offset = (bo->mem.start << PAGE_SHIFT) +
                     bdev->man[bo->mem.mem_type].gpu_offset;
                 bo->cur_placement = bo->mem.placement;
-                spin_unlock(&bo->lock);
         } else
                 bo->offset = 0;
 
@@ -442,135 +458,150 @@ out_err:
 }
 
 /**
- * Call bo::reserved and with the lru lock held.
+ * Call bo::reserved.
  * Will release GPU memory type usage on destruction.
- * This is the place to put in driver specific hooks.
- * Will release the bo::reserved lock and the
- * lru lock on exit.
+ * This is the place to put in driver specific hooks to release
+ * driver private resources.
+ * Will release the bo::reserved lock.
  */
 
 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 {
-        struct ttm_bo_global *glob = bo->glob;
-
         if (bo->ttm) {
-
-                /**
-                 * Release the lru_lock, since we don't want to have
-                 * an atomic requirement on ttm_tt[unbind|destroy].
-                 */
-
-                spin_unlock(&glob->lru_lock);
                 ttm_tt_unbind(bo->ttm);
                 ttm_tt_destroy(bo->ttm);
                 bo->ttm = NULL;
-                spin_lock(&glob->lru_lock);
-        }
-
-        if (bo->mem.mm_node) {
-                drm_mm_put_block(bo->mem.mm_node);
-                bo->mem.mm_node = NULL;
         }
+        ttm_bo_mem_put(bo, &bo->mem);
 
         atomic_set(&bo->reserved, 0);
+
+        /*
+         * Make processes trying to reserve really pick it up.
+         */
+        smp_mb__after_atomic_dec();
         wake_up_all(&bo->event_queue);
-        spin_unlock(&glob->lru_lock);
 }
 
-
-/**
- * If bo idle, remove from delayed- and lru lists, and unref.
- * If not idle, and already on delayed list, do nothing.
- * If not idle, and not on delayed list, put on delayed list,
- *   up the list_kref and schedule a delayed list check.
- */
-
-static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all)
+static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 {
         struct ttm_bo_device *bdev = bo->bdev;
         struct ttm_bo_global *glob = bo->glob;
-        struct ttm_bo_driver *driver = bdev->driver;
+        struct ttm_bo_driver *driver;
+        void *sync_obj = NULL;
+        void *sync_obj_arg;
+        int put_count;
         int ret;
 
-        spin_lock(&bo->lock);
-retry:
-        (void) ttm_bo_wait(bo, false, false, !remove_all);
-
+        spin_lock(&bdev->fence_lock);
+        (void) ttm_bo_wait(bo, false, false, true);
         if (!bo->sync_obj) {
-                int put_count;
-
-                spin_unlock(&bo->lock);
 
                 spin_lock(&glob->lru_lock);
-                ret = ttm_bo_reserve_locked(bo, false, !remove_all, false, 0);
 
                 /**
-                 * Someone else has the object reserved. Bail and retry.
+                 * Lock inversion between bo:reserve and bdev::fence_lock here,
+                 * but that's OK, since we're only trylocking.
                  */
 
-                if (unlikely(ret == -EBUSY)) {
-                        spin_unlock(&glob->lru_lock);
-                        spin_lock(&bo->lock);
-                        goto requeue;
-                }
-
-                /**
-                 * We can re-check for sync object without taking
-                 * the bo::lock since setting the sync object requires
-                 * also bo::reserved. A busy object at this point may
-                 * be caused by another thread starting an accelerated
-                 * eviction.
-                 */
+                ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
 
-                if (unlikely(bo->sync_obj)) {
-                        atomic_set(&bo->reserved, 0);
-                        wake_up_all(&bo->event_queue);
-                        spin_unlock(&glob->lru_lock);
-                        spin_lock(&bo->lock);
-                        if (remove_all)
-                                goto retry;
-                        else
-                                goto requeue;
-                }
+                if (unlikely(ret == -EBUSY))
+                        goto queue;
 
+                spin_unlock(&bdev->fence_lock);
                 put_count = ttm_bo_del_from_lru(bo);
 
-                if (!list_empty(&bo->ddestroy)) {
-                        list_del_init(&bo->ddestroy);
-                        ++put_count;
-                }
-
+                spin_unlock(&glob->lru_lock);
                 ttm_bo_cleanup_memtype_use(bo);
 
-                while (put_count--)
-                        kref_put(&bo->list_kref, ttm_bo_ref_bug);
+                ttm_bo_list_ref_sub(bo, put_count, true);
 
-                return 0;
+                return;
+        } else {
+                spin_lock(&glob->lru_lock);
+        }
+queue:
+        driver = bdev->driver;
+        if (bo->sync_obj)
+                sync_obj = driver->sync_obj_ref(bo->sync_obj);
+        sync_obj_arg = bo->sync_obj_arg;
+
+        kref_get(&bo->list_kref);
+        list_add_tail(&bo->ddestroy, &bdev->ddestroy);
+        spin_unlock(&glob->lru_lock);
+        spin_unlock(&bdev->fence_lock);
+
+        if (sync_obj) {
+                driver->sync_obj_flush(sync_obj, sync_obj_arg);
+                driver->sync_obj_unref(&sync_obj);
         }
-requeue:
+        schedule_delayed_work(&bdev->wq,
+                              ((HZ / 100) < 1) ? 1 : HZ / 100);
+}
+
+/**
+ * function ttm_bo_cleanup_refs
+ * If bo idle, remove from delayed- and lru lists, and unref.
+ * If not idle, do nothing.
+ *
+ * @interruptible         Any sleeps should occur interruptibly.
+ * @no_wait_reserve       Never wait for reserve. Return -EBUSY instead.
+ * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
+ */
+
+static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
+                               bool interruptible,
+                               bool no_wait_reserve,
+                               bool no_wait_gpu)
+{
+        struct ttm_bo_device *bdev = bo->bdev;
+        struct ttm_bo_global *glob = bo->glob;
+        int put_count;
+        int ret = 0;
+
+retry:
+        spin_lock(&bdev->fence_lock);
+        ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
+        spin_unlock(&bdev->fence_lock);
+
+        if (unlikely(ret != 0))
+                return ret;
+
         spin_lock(&glob->lru_lock);
-        if (list_empty(&bo->ddestroy)) {
-                void *sync_obj = bo->sync_obj;
-                void *sync_obj_arg = bo->sync_obj_arg;
+        ret = ttm_bo_reserve_locked(bo, interruptible,
+                                    no_wait_reserve, false, 0);
 
-                kref_get(&bo->list_kref);
-                list_add_tail(&bo->ddestroy, &bdev->ddestroy);
+        if (unlikely(ret != 0) || list_empty(&bo->ddestroy)) {
                 spin_unlock(&glob->lru_lock);
-                spin_unlock(&bo->lock);
+                return ret;
+        }
 
-                if (sync_obj)
-                        driver->sync_obj_flush(sync_obj, sync_obj_arg);
-                schedule_delayed_work(&bdev->wq,
-                                      ((HZ / 100) < 1) ? 1 : HZ / 100);
-                ret = 0;
+        /**
+         * We can re-check for sync object without taking
+         * the bo::lock since setting the sync object requires
+         * also bo::reserved. A busy object at this point may
+         * be caused by another thread recently starting an accelerated
+         * eviction.
+         */
 
-        } else {
+        if (unlikely(bo->sync_obj)) {
+                atomic_set(&bo->reserved, 0);
+                wake_up_all(&bo->event_queue);
                 spin_unlock(&glob->lru_lock);
-                spin_unlock(&bo->lock);
-                ret = -EBUSY;
+                goto retry;
         }
 
-        return ret;
+        put_count = ttm_bo_del_from_lru(bo);
+        list_del_init(&bo->ddestroy);
+        ++put_count;
+
+        spin_unlock(&glob->lru_lock);
+        ttm_bo_cleanup_memtype_use(bo);
+
+        ttm_bo_list_ref_sub(bo, put_count, true);
+
+        return 0;
 }
 
 /**
@@ -602,7 +633,8 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
                 }
 
                 spin_unlock(&glob->lru_lock);
-                ret = ttm_bo_cleanup_refs(entry, remove_all);
+                ret = ttm_bo_cleanup_refs(entry, false, !remove_all,
+                                          !remove_all);
                 kref_put(&entry->list_kref, ttm_bo_release_list);
                 entry = nentry;
 
@@ -638,6 +670,7 @@ static void ttm_bo_release(struct kref *kref)
         struct ttm_buffer_object *bo =
             container_of(kref, struct ttm_buffer_object, kref);
         struct ttm_bo_device *bdev = bo->bdev;
+        struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 
         if (likely(bo->vm_node != NULL)) {
                 rb_erase(&bo->vm_rb, &bdev->addr_space_rb);
@@ -645,7 +678,10 @@ static void ttm_bo_release(struct kref *kref)
                 bo->vm_node = NULL;
         }
         write_unlock(&bdev->vm_lock);
-        ttm_bo_cleanup_refs(bo, false);
+        ttm_mem_io_lock(man, false);
+        ttm_mem_io_free_vm(bo);
+        ttm_mem_io_unlock(man);
+        ttm_bo_cleanup_refs_or_queue(bo);
         kref_put(&bo->list_kref, ttm_bo_release_list);
         write_lock(&bdev->vm_lock);
 }
@@ -680,14 +716,13 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
                         bool no_wait_reserve, bool no_wait_gpu)
 {
         struct ttm_bo_device *bdev = bo->bdev;
-        struct ttm_bo_global *glob = bo->glob;
         struct ttm_mem_reg evict_mem;
         struct ttm_placement placement;
         int ret = 0;
 
-        spin_lock(&bo->lock);
+        spin_lock(&bdev->fence_lock);
         ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
-        spin_unlock(&bo->lock);
+        spin_unlock(&bdev->fence_lock);
 
         if (unlikely(ret != 0)) {
                 if (ret != -ERESTARTSYS) {
@@ -702,7 +737,8 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
 
         evict_mem = bo->mem;
         evict_mem.mm_node = NULL;
-        evict_mem.bus.io_reserved = false;
+        evict_mem.bus.io_reserved_vm = false;
+        evict_mem.bus.io_reserved_count = 0;
 
         placement.fpfn = 0;
         placement.lpfn = 0;
@@ -726,12 +762,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
         if (ret) {
                 if (ret != -ERESTARTSYS)
                         printk(KERN_ERR TTM_PFX "Buffer eviction failed\n");
-                spin_lock(&glob->lru_lock);
-                if (evict_mem.mm_node) {
-                        drm_mm_put_block(evict_mem.mm_node);
-                        evict_mem.mm_node = NULL;
-                }
-                spin_unlock(&glob->lru_lock);
+                ttm_bo_mem_put(bo, &evict_mem);
                 goto out;
         }
         bo->evicted = true;
@@ -759,6 +790,18 @@ retry:
         bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru);
         kref_get(&bo->list_kref);
 
+        if (!list_empty(&bo->ddestroy)) {
+                spin_unlock(&glob->lru_lock);
+                ret = ttm_bo_cleanup_refs(bo, interruptible,
+                                          no_wait_reserve, no_wait_gpu);
+                kref_put(&bo->list_kref, ttm_bo_release_list);
+
+                if (likely(ret == 0 || ret == -ERESTARTSYS))
+                        return ret;
+
+                goto retry;
+        }
+
         ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0);
 
         if (unlikely(ret == -EBUSY)) {
@@ -782,8 +825,7 @@ retry:
 
         BUG_ON(ret != 0);
 
-        while (put_count--)
-                kref_put(&bo->list_kref, ttm_bo_ref_bug);
+        ttm_bo_list_ref_sub(bo, put_count, true);
 
         ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu);
         ttm_bo_unreserve(bo);
@@ -792,41 +834,14 @@ retry:
         return ret;
 }
 
-static int ttm_bo_man_get_node(struct ttm_buffer_object *bo,
-                                struct ttm_mem_type_manager *man,
-                                struct ttm_placement *placement,
-                                struct ttm_mem_reg *mem,
-                                struct drm_mm_node **node)
+void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
 {
-        struct ttm_bo_global *glob = bo->glob;
-        unsigned long lpfn;
-        int ret;
-
-        lpfn = placement->lpfn;
-        if (!lpfn)
-                lpfn = man->size;
-        *node = NULL;
-        do {
-                ret = drm_mm_pre_get(&man->manager);
-                if (unlikely(ret))
-                        return ret;
+        struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
 
-                spin_lock(&glob->lru_lock);
-                *node = drm_mm_search_free_in_range(&man->manager,
-                                        mem->num_pages, mem->page_alignment,
-                                        placement->fpfn, lpfn, 1);
-                if (unlikely(*node == NULL)) {
-                        spin_unlock(&glob->lru_lock);
-                        return 0;
-                }
-                *node = drm_mm_get_block_atomic_range(*node, mem->num_pages,
-                                                        mem->page_alignment,
-                                                        placement->fpfn,
-                                                        lpfn);
-                spin_unlock(&glob->lru_lock);
-        } while (*node == NULL);
-        return 0;
+        if (mem->mm_node)
+                (*man->func->put_node)(man, mem);
 }
+EXPORT_SYMBOL(ttm_bo_mem_put);
 
 /**
  * Repeatedly evict memory from the LRU for @mem_type until we create enough
@@ -841,31 +856,22 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
                                         bool no_wait_gpu)
 {
         struct ttm_bo_device *bdev = bo->bdev;
-        struct ttm_bo_global *glob = bdev->glob;
         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
-        struct drm_mm_node *node;
         int ret;
 
         do {
-                ret = ttm_bo_man_get_node(bo, man, placement, mem, &node);
+                ret = (*man->func->get_node)(man, bo, placement, mem);
                 if (unlikely(ret != 0))
                         return ret;
-                if (node)
-                        break;
-                spin_lock(&glob->lru_lock);
-                if (list_empty(&man->lru)) {
-                        spin_unlock(&glob->lru_lock);
+                if (mem->mm_node)
                         break;
-                }
-                spin_unlock(&glob->lru_lock);
                 ret = ttm_mem_evict_first(bdev, mem_type, interruptible,
                                                 no_wait_reserve, no_wait_gpu);
                 if (unlikely(ret != 0))
                         return ret;
         } while (1);
-        if (node == NULL)
+        if (mem->mm_node == NULL)
                 return -ENOMEM;
-        mem->mm_node = node;
         mem->mem_type = mem_type;
         return 0;
 }
@@ -939,7 +945,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
         bool type_found = false;
         bool type_ok = false;
         bool has_erestartsys = false;
-        struct drm_mm_node *node = NULL;
         int i, ret;
 
         mem->mm_node = NULL;
@@ -973,17 +978,15 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 
                 if (man->has_type && man->use_type) {
                         type_found = true;
-                        ret = ttm_bo_man_get_node(bo, man, placement, mem,
-                                                        &node);
+                        ret = (*man->func->get_node)(man, bo, placement, mem);
                         if (unlikely(ret))
                                 return ret;
                 }
-                if (node)
+                if (mem->mm_node)
                         break;
         }
 
-        if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || node) {
-                mem->mm_node = node;
+        if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
                 mem->mem_type = mem_type;
                 mem->placement = cur_flags;
                 return 0;
@@ -1053,9 +1056,9 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
                         bool interruptible, bool no_wait_reserve,
                         bool no_wait_gpu)
 {
-        struct ttm_bo_global *glob = bo->glob;
         int ret = 0;
         struct ttm_mem_reg mem;
+        struct ttm_bo_device *bdev = bo->bdev;
 
         BUG_ON(!atomic_read(&bo->reserved));
 
@@ -1064,15 +1067,16 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
          * Have the driver move function wait for idle when necessary,
          * instead of doing it here.
          */
-        spin_lock(&bo->lock);
+        spin_lock(&bdev->fence_lock);
         ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
-        spin_unlock(&bo->lock);
+        spin_unlock(&bdev->fence_lock);
         if (ret)
                 return ret;
         mem.num_pages = bo->num_pages;
         mem.size = mem.num_pages << PAGE_SHIFT;
         mem.page_alignment = bo->mem.page_alignment;
-        mem.bus.io_reserved = false;
+        mem.bus.io_reserved_vm = false;
+        mem.bus.io_reserved_count = 0;
         /*
          * Determine where to move the buffer.
          */
@@ -1081,11 +1085,8 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
                 goto out_unlock;
         ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu);
 out_unlock:
-        if (ret && mem.mm_node) {
-                spin_lock(&glob->lru_lock);
-                drm_mm_put_block(mem.mm_node);
-                spin_unlock(&glob->lru_lock);
-        }
+        if (ret && mem.mm_node)
+                ttm_bo_mem_put(bo, &mem);
         return ret;
 }
 
@@ -1093,11 +1094,10 @@ static int ttm_bo_mem_compat(struct ttm_placement *placement,
                              struct ttm_mem_reg *mem)
 {
         int i;
-        struct drm_mm_node *node = mem->mm_node;
 
-        if (node && placement->lpfn != 0 &&
-            (node->start < placement->fpfn ||
-             node->start + node->size > placement->lpfn))
+        if (mem->mm_node && placement->lpfn != 0 &&
+            (mem->start < placement->fpfn ||
+             mem->start + mem->num_pages > placement->lpfn))
                 return -1;
 
         for (i = 0; i < placement->num_placement; i++) {
@@ -1154,35 +1154,9 @@ EXPORT_SYMBOL(ttm_bo_validate);
 int ttm_bo_check_placement(struct ttm_buffer_object *bo,
                                 struct ttm_placement *placement)
 {
-        int i;
+        BUG_ON((placement->fpfn || placement->lpfn) &&
+               (bo->mem.num_pages > (placement->lpfn - placement->fpfn)));
 
-        if (placement->fpfn || placement->lpfn) {
-                if (bo->mem.num_pages > (placement->lpfn - placement->fpfn)) {
-                        printk(KERN_ERR TTM_PFX "Page number range to small "
-                                "Need %lu pages, range is [%u, %u]\n",
-                                bo->mem.num_pages, placement->fpfn,
-                                placement->lpfn);
-                        return -EINVAL;
-                }
-        }
-        for (i = 0; i < placement->num_placement; i++) {
-                if (!capable(CAP_SYS_ADMIN)) {
-                        if (placement->placement[i] & TTM_PL_FLAG_NO_EVICT) {
-                                printk(KERN_ERR TTM_PFX "Need to be root to "
-                                        "modify NO_EVICT status.\n");
-                                return -EINVAL;
-                        }
-                }
-        }
-        for (i = 0; i < placement->num_busy_placement; i++) {
-                if (!capable(CAP_SYS_ADMIN)) {
-                        if (placement->busy_placement[i] & TTM_PL_FLAG_NO_EVICT) {
-                                printk(KERN_ERR TTM_PFX "Need to be root to "
-                                        "modify NO_EVICT status.\n");
-                                return -EINVAL;
-                        }
-                }
-        }
         return 0;
 }
 
@@ -1194,7 +1168,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
                 uint32_t page_alignment,
                 unsigned long buffer_start,
                 bool interruptible,
-                struct file *persistant_swap_storage,
+                struct file *persistent_swap_storage,
                 size_t acc_size,
                 void (*destroy) (struct ttm_buffer_object *))
 {
@@ -1205,11 +1179,14 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
         if (num_pages == 0) {
                 printk(KERN_ERR TTM_PFX "Illegal buffer object size.\n");
+                if (destroy)
+                        (*destroy)(bo);
+                else
+                        kfree(bo);
                 return -EINVAL;
         }
         bo->destroy = destroy;
 
-        spin_lock_init(&bo->lock);
         kref_init(&bo->kref);
         kref_init(&bo->list_kref);
         atomic_set(&bo->cpu_writers, 0);
@@ -1218,6 +1195,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
         INIT_LIST_HEAD(&bo->lru);
         INIT_LIST_HEAD(&bo->ddestroy);
         INIT_LIST_HEAD(&bo->swap);
+        INIT_LIST_HEAD(&bo->io_reserve_lru);
         bo->bdev = bdev;
         bo->glob = bdev->glob;
         bo->type = type;
@@ -1227,12 +1205,13 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
         bo->mem.num_pages = bo->num_pages;
         bo->mem.mm_node = NULL;
         bo->mem.page_alignment = page_alignment;
-        bo->mem.bus.io_reserved = false;
+        bo->mem.bus.io_reserved_vm = false;
+        bo->mem.bus.io_reserved_count = 0;
         bo->buffer_start = buffer_start & PAGE_MASK;
         bo->priv_flags = 0;
         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
         bo->seq_valid = false;
-        bo->persistant_swap_storage = persistant_swap_storage;
+        bo->persistent_swap_storage = persistent_swap_storage;
         bo->acc_size = acc_size;
         atomic_inc(&bo->glob->bo_count);
 
@@ -1281,7 +1260,7 @@ int ttm_bo_create(struct ttm_bo_device *bdev,
                         uint32_t page_alignment,
                         unsigned long buffer_start,
                         bool interruptible,
-                        struct file *persistant_swap_storage,
+                        struct file *persistent_swap_storage,
                         struct ttm_buffer_object **p_bo)
 {
         struct ttm_buffer_object *bo;
@@ -1303,7 +1282,7 @@ int ttm_bo_create(struct ttm_bo_device *bdev,
 
         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
                                 buffer_start, interruptible,
-                                persistant_swap_storage, acc_size, NULL);
+                                persistent_swap_storage, acc_size, NULL);
         if (likely(ret == 0))
                 *p_bo = bo;
 
@@ -1341,7 +1320,6 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
 
 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
 {
-        struct ttm_bo_global *glob = bdev->glob;
         struct ttm_mem_type_manager *man;
         int ret = -EINVAL;
 
@@ -1364,13 +1342,7 @@ int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
         if (mem_type > 0) {
                 ttm_bo_force_list_clean(bdev, mem_type, false);
 
-                spin_lock(&glob->lru_lock);
-                if (drm_mm_clean(&man->manager))
-                        drm_mm_takedown(&man->manager);
-                else
-                        ret = -EBUSY;
-
-                spin_unlock(&glob->lru_lock);
+                ret = (*man->func->takedown)(man);
         }
 
         return ret;
@@ -1405,32 +1377,22 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
         int ret = -EINVAL;
         struct ttm_mem_type_manager *man;
 
-        if (type >= TTM_NUM_MEM_TYPES) {
-                printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", type);
-                return ret;
-        }
-
+        BUG_ON(type >= TTM_NUM_MEM_TYPES);
         man = &bdev->man[type];
-        if (man->has_type) {
-                printk(KERN_ERR TTM_PFX
-                       "Memory manager already initialized for type %d\n",
-                       type);
-                return ret;
-        }
+        BUG_ON(man->has_type);
+        man->io_reserve_fastpath = true;
+        man->use_io_reserve_lru = false;
+        mutex_init(&man->io_reserve_mutex);
+        INIT_LIST_HEAD(&man->io_reserve_lru);
 
         ret = bdev->driver->init_mem_type(bdev, type, man);
         if (ret)
                 return ret;
+        man->bdev = bdev;
 
         ret = 0;
         if (type != TTM_PL_SYSTEM) {
-                if (!p_size) {
-                        printk(KERN_ERR TTM_PFX
-                               "Zero size memory manager type %d\n",
-                               type);
-                        return ret;
-                }
-                ret = drm_mm_init(&man->manager, 0, p_size);
+                ret = (*man->func->init)(man, p_size);
                 if (ret)
                         return ret;
         }
@@ -1539,8 +1501,7 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev)
         list_del(&bdev->device_list);
         mutex_unlock(&glob->device_list_mutex);
 
-        if (!cancel_delayed_work(&bdev->wq))
-                flush_scheduled_work();
+        cancel_delayed_work_sync(&bdev->wq);
 
         while (ttm_bo_delayed_delete(bdev, true))
                 ;
@@ -1594,7 +1555,8 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,
         bdev->dev_mapping = NULL;
         bdev->glob = glob;
         bdev->need_dma32 = need_dma32;
-
+        bdev->val_seq = 0;
+        spin_lock_init(&bdev->fence_lock);
         mutex_lock(&glob->device_list_mutex);
         list_add_tail(&bdev->device_list, &glob->device_list);
         mutex_unlock(&glob->device_list_mutex);
@@ -1628,7 +1590,7 @@ bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
         return true;
 }
 
-void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
+void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
 {
         struct ttm_bo_device *bdev = bo->bdev;
         loff_t offset = (loff_t) bo->addr_space_offset;
@@ -1637,8 +1599,20 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
         if (!bdev->dev_mapping)
                 return;
         unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1);
-        ttm_mem_io_free(bdev, &bo->mem);
+        ttm_mem_io_free_vm(bo);
+}
+
+void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
+{
+        struct ttm_bo_device *bdev = bo->bdev;
+        struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
+
+        ttm_mem_io_lock(man, false);
+        ttm_bo_unmap_virtual_locked(bo);
+        ttm_mem_io_unlock(man);
 }
+
+
 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
 
 static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo)
@@ -1718,6 +1692,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
                 bool lazy, bool interruptible, bool no_wait)
 {
         struct ttm_bo_driver *driver = bo->bdev->driver;
+        struct ttm_bo_device *bdev = bo->bdev;
         void *sync_obj;
         void *sync_obj_arg;
         int ret = 0;
@@ -1731,9 +1706,9 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
                         void *tmp_obj = bo->sync_obj;
                         bo->sync_obj = NULL;
                         clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
-                        spin_unlock(&bo->lock);
+                        spin_unlock(&bdev->fence_lock);
                         driver->sync_obj_unref(&tmp_obj);
-                        spin_lock(&bo->lock);
+                        spin_lock(&bdev->fence_lock);
                         continue;
                 }
 
@@ -1742,29 +1717,29 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
 
                 sync_obj = driver->sync_obj_ref(bo->sync_obj);
                 sync_obj_arg = bo->sync_obj_arg;
-                spin_unlock(&bo->lock);
+                spin_unlock(&bdev->fence_lock);
                 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg,
                                             lazy, interruptible);
                 if (unlikely(ret != 0)) {
                         driver->sync_obj_unref(&sync_obj);
-                        spin_lock(&bo->lock);
+                        spin_lock(&bdev->fence_lock);
                         return ret;
                 }
-                spin_lock(&bo->lock);
+                spin_lock(&bdev->fence_lock);
                 if (likely(bo->sync_obj == sync_obj &&
                            bo->sync_obj_arg == sync_obj_arg)) {
                         void *tmp_obj = bo->sync_obj;
                         bo->sync_obj = NULL;
                         clear_bit(TTM_BO_PRIV_FLAG_MOVING,
                                   &bo->priv_flags);
-                        spin_unlock(&bo->lock);
+                        spin_unlock(&bdev->fence_lock);
                         driver->sync_obj_unref(&sync_obj);
                         driver->sync_obj_unref(&tmp_obj);
-                        spin_lock(&bo->lock);
+                        spin_lock(&bdev->fence_lock);
                 } else {
-                        spin_unlock(&bo->lock);
+                        spin_unlock(&bdev->fence_lock);
                         driver->sync_obj_unref(&sync_obj);
-                        spin_lock(&bo->lock);
+                        spin_lock(&bdev->fence_lock);
                 }
         }
         return 0;
@@ -1773,6 +1748,7 @@ EXPORT_SYMBOL(ttm_bo_wait);
 
 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
 {
+        struct ttm_bo_device *bdev = bo->bdev;
         int ret = 0;
 
         /*
@@ -1782,9 +1758,9 @@ int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
         ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
         if (unlikely(ret != 0))
                 return ret;
-        spin_lock(&bo->lock);
+        spin_lock(&bdev->fence_lock);
         ret = ttm_bo_wait(bo, false, true, no_wait);
-        spin_unlock(&bo->lock);
+        spin_unlock(&bdev->fence_lock);
         if (likely(ret == 0))
                 atomic_inc(&bo->cpu_writers);
         ttm_bo_unreserve(bo);
@@ -1824,6 +1800,13 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
                                       struct ttm_buffer_object, swap);
                 kref_get(&bo->list_kref);
 
+                if (!list_empty(&bo->ddestroy)) {
+                        spin_unlock(&glob->lru_lock);
+                        (void) ttm_bo_cleanup_refs(bo, false, false, false);
+                        kref_put(&bo->list_kref, ttm_bo_release_list);
+                        continue;
+                }
+
                 /**
                  * Reserve buffer. Since we unlock while sleeping, we need
                  * to re-check that nobody removed us from the swap-list while
@@ -1843,16 +1826,15 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
         put_count = ttm_bo_del_from_lru(bo);
         spin_unlock(&glob->lru_lock);
 
-        while (put_count--)
-                kref_put(&bo->list_kref, ttm_bo_ref_bug);
+        ttm_bo_list_ref_sub(bo, put_count, true);
 
         /**
          * Wait for GPU, then move to system cached.
          */
 
-        spin_lock(&bo->lock);
+        spin_lock(&bo->bdev->fence_lock);
         ret = ttm_bo_wait(bo, false, false, false);
-        spin_unlock(&bo->lock);
+        spin_unlock(&bo->bdev->fence_lock);
 
         if (unlikely(ret != 0))
                 goto out;
@@ -1881,7 +1863,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
         if (bo->bdev->driver->swap_notify)
                 bo->bdev->driver->swap_notify(bo);
 
-        ret = ttm_tt_swapout(bo->ttm, bo->persistant_swap_storage);
+        ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
 out:
 
         /**
diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c
new file mode 100644
index 000000000000..038e947d00f9
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c
@@ -0,0 +1,157 @@
+/**************************************************************************
+ *
+ * Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
+ */
+
+#include "ttm/ttm_module.h"
+#include "ttm/ttm_bo_driver.h"
+#include "ttm/ttm_placement.h"
+#include "drm_mm.h"
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/module.h>
+
+/**
+ * Currently we use a spinlock for the lock, but a mutex *may* be
+ * more appropriate to reduce scheduling latency if the range manager
+ * ends up with very fragmented allocation patterns.
+ */
+
+struct ttm_range_manager {
+        struct drm_mm mm;
+        spinlock_t lock;
+};
+
+static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man,
+                               struct ttm_buffer_object *bo,
+                               struct ttm_placement *placement,
+                               struct ttm_mem_reg *mem)
+{
+        struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
+        struct drm_mm *mm = &rman->mm;
+        struct drm_mm_node *node = NULL;
+        unsigned long lpfn;
+        int ret;
+
+        lpfn = placement->lpfn;
+        if (!lpfn)
+                lpfn = man->size;
+        do {
+                ret = drm_mm_pre_get(mm);
+                if (unlikely(ret))
+                        return ret;
+
+                spin_lock(&rman->lock);
+                node = drm_mm_search_free_in_range(mm,
+                                        mem->num_pages, mem->page_alignment,
+                                        placement->fpfn, lpfn, 1);
+                if (unlikely(node == NULL)) {
+                        spin_unlock(&rman->lock);
+                        return 0;
+                }
+                node = drm_mm_get_block_atomic_range(node, mem->num_pages,
+                                                     mem->page_alignment,
+                                                     placement->fpfn,
+                                                     lpfn);
+                spin_unlock(&rman->lock);
+        } while (node == NULL);
+
+        mem->mm_node = node;
+        mem->start = node->start;
+        return 0;
+}
+
+static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man,
+                                struct ttm_mem_reg *mem)
+{
+        struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
+
+        if (mem->mm_node) {
+                spin_lock(&rman->lock);
+                drm_mm_put_block(mem->mm_node);
+                spin_unlock(&rman->lock);
+                mem->mm_node = NULL;
+        }
+}
+
+static int ttm_bo_man_init(struct ttm_mem_type_manager *man,
+                           unsigned long p_size)
+{
+        struct ttm_range_manager *rman;
+        int ret;
+
+        rman = kzalloc(sizeof(*rman), GFP_KERNEL);
+        if (!rman)
+                return -ENOMEM;
+
+        ret = drm_mm_init(&rman->mm, 0, p_size);
+        if (ret) {
+                kfree(rman);
+                return ret;
+        }
+
+        spin_lock_init(&rman->lock);
+        man->priv = rman;
+        return 0;
+}
+
+static int ttm_bo_man_takedown(struct ttm_mem_type_manager *man)
+{
+        struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
+        struct drm_mm *mm = &rman->mm;
+
+        spin_lock(&rman->lock);
+        if (drm_mm_clean(mm)) {
+                drm_mm_takedown(mm);
+                spin_unlock(&rman->lock);
+                kfree(rman);
+                man->priv = NULL;
+                return 0;
+        }
+        spin_unlock(&rman->lock);
+        return -EBUSY;
+}
+
+static void ttm_bo_man_debug(struct ttm_mem_type_manager *man,
+                             const char *prefix)
+{
+        struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
+
+        spin_lock(&rman->lock);
+        drm_mm_debug_table(&rman->mm, prefix);
+        spin_unlock(&rman->lock);
+}
+
+const struct ttm_mem_type_manager_func ttm_bo_manager_func = {
+        ttm_bo_man_init,
+        ttm_bo_man_takedown,
+        ttm_bo_man_get_node,
+        ttm_bo_man_put_node,
+        ttm_bo_man_debug
+};
+EXPORT_SYMBOL(ttm_bo_manager_func);
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 3451a82adba7..77dbf408c0d0 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -39,14 +39,7 @@
 
 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
 {
-        struct ttm_mem_reg *old_mem = &bo->mem;
-
-        if (old_mem->mm_node) {
-                spin_lock(&bo->glob->lru_lock);
-                drm_mm_put_block(old_mem->mm_node);
-                spin_unlock(&bo->glob->lru_lock);
-        }
-        old_mem->mm_node = NULL;
+        ttm_bo_mem_put(bo, &bo->mem);
 }
 
 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
@@ -82,37 +75,123 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
 }
 EXPORT_SYMBOL(ttm_bo_move_ttm);
 
-int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
+int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
 {
-        int ret;
+        if (likely(man->io_reserve_fastpath))
+                return 0;
+
+        if (interruptible)
+                return mutex_lock_interruptible(&man->io_reserve_mutex);
+
+        mutex_lock(&man->io_reserve_mutex);
+        return 0;
+}
+
+void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
+{
+        if (likely(man->io_reserve_fastpath))
+                return;
+
+        mutex_unlock(&man->io_reserve_mutex);
+}
+
+static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
+{
+        struct ttm_buffer_object *bo;
+
+        if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
+                return -EAGAIN;
+
+        bo = list_first_entry(&man->io_reserve_lru,
+                              struct ttm_buffer_object,
+                              io_reserve_lru);
+        list_del_init(&bo->io_reserve_lru);
+        ttm_bo_unmap_virtual_locked(bo);
+
+        return 0;
+}
+
+static int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
+                              struct ttm_mem_reg *mem)
+{
+        struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
+        int ret = 0;
+
+        if (!bdev->driver->io_mem_reserve)
+                return 0;
+        if (likely(man->io_reserve_fastpath))
+                return bdev->driver->io_mem_reserve(bdev, mem);
 
-        if (!mem->bus.io_reserved) {
-                mem->bus.io_reserved = true;
+        if (bdev->driver->io_mem_reserve &&
+            mem->bus.io_reserved_count++ == 0) {
+retry:
                 ret = bdev->driver->io_mem_reserve(bdev, mem);
+                if (ret == -EAGAIN) {
+                        ret = ttm_mem_io_evict(man);
+                        if (ret == 0)
+                                goto retry;
+                }
+        }
+        return ret;
+}
+
+static void ttm_mem_io_free(struct ttm_bo_device *bdev,
+                            struct ttm_mem_reg *mem)
+{
+        struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
+
+        if (likely(man->io_reserve_fastpath))
+                return;
+
+        if (bdev->driver->io_mem_reserve &&
+            --mem->bus.io_reserved_count == 0 &&
+            bdev->driver->io_mem_free)
+                bdev->driver->io_mem_free(bdev, mem);
+
+}
+
+int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
+{
+        struct ttm_mem_reg *mem = &bo->mem;
+        int ret;
+
+        if (!mem->bus.io_reserved_vm) {
+                struct ttm_mem_type_manager *man =
+                        &bo->bdev->man[mem->mem_type];
+
+                ret = ttm_mem_io_reserve(bo->bdev, mem);
                 if (unlikely(ret != 0))
                         return ret;
+                mem->bus.io_reserved_vm = true;
+                if (man->use_io_reserve_lru)
+                        list_add_tail(&bo->io_reserve_lru,
+                                      &man->io_reserve_lru);
         }
         return 0;
 }
 
-void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
+void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
 {
-        if (bdev->driver->io_mem_reserve) {
-                if (mem->bus.io_reserved) {
-                        mem->bus.io_reserved = false;
-                        bdev->driver->io_mem_free(bdev, mem);
-                }
+        struct ttm_mem_reg *mem = &bo->mem;
+
+        if (mem->bus.io_reserved_vm) {
+                mem->bus.io_reserved_vm = false;
+                list_del_init(&bo->io_reserve_lru);
+                ttm_mem_io_free(bo->bdev, mem);
         }
 }
 
 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
                         void **virtual)
 {
+        struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
         int ret;
         void *addr;
 
         *virtual = NULL;
+        (void) ttm_mem_io_lock(man, false);
         ret = ttm_mem_io_reserve(bdev, mem);
+        ttm_mem_io_unlock(man);
         if (ret || !mem->bus.is_iomem)
                 return ret;
 
@@ -124,7 +203,9 @@ int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
                 else
                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
                 if (!addr) {
+                        (void) ttm_mem_io_lock(man, false);
                         ttm_mem_io_free(bdev, mem);
+                        ttm_mem_io_unlock(man);
                         return -ENOMEM;
                 }
         }
@@ -141,7 +222,9 @@ void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 
         if (virtual && mem->bus.addr == NULL)
                 iounmap(virtual);
+        (void) ttm_mem_io_lock(man, false);
         ttm_mem_io_free(bdev, mem);
+        ttm_mem_io_unlock(man);
 }
 
 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
@@ -170,7 +253,7 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 
 #ifdef CONFIG_X86
-        dst = kmap_atomic_prot(d, KM_USER0, prot);
+        dst = kmap_atomic_prot(d, prot);
 #else
         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
                 dst = vmap(&d, 1, 0, prot);
@@ -183,7 +266,7 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
         memcpy_fromio(dst, src, PAGE_SIZE);
 
 #ifdef CONFIG_X86
-        kunmap_atomic(dst, KM_USER0);
+        kunmap_atomic(dst);
 #else
         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
                 vunmap(dst);
@@ -206,7 +289,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 
         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 #ifdef CONFIG_X86
-        src = kmap_atomic_prot(s, KM_USER0, prot);
+        src = kmap_atomic_prot(s, prot);
 #else
         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
                 src = vmap(&s, 1, 0, prot);
@@ -219,7 +302,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
         memcpy_toio(dst, src, PAGE_SIZE);
 
 #ifdef CONFIG_X86
-        kunmap_atomic(src, KM_USER0);
+        kunmap_atomic(src);
 #else
         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
                 vunmap(src);
@@ -238,7 +321,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
         struct ttm_tt *ttm = bo->ttm;
         struct ttm_mem_reg *old_mem = &bo->mem;
-        struct ttm_mem_reg old_copy = *old_mem;
+        struct ttm_mem_reg old_copy;
         void *old_iomap;
         void *new_iomap;
         int ret;
@@ -263,8 +346,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
         dir = 1;
 
         if ((old_mem->mem_type == new_mem->mem_type) &&
-            (new_mem->mm_node->start <
-             old_mem->mm_node->start + old_mem->mm_node->size)) {
+            (new_mem->start < old_mem->start + old_mem->size)) {
                 dir = -1;
                 add = new_mem->num_pages - 1;
         }
@@ -288,8 +370,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
         }
         mb();
 out2:
-        ttm_bo_free_old_node(bo);
-
+        old_copy = *old_mem;
         *old_mem = *new_mem;
         new_mem->mm_node = NULL;
 
@@ -300,9 +381,10 @@ out2:
         }
 
 out1:
-        ttm_mem_reg_iounmap(bdev, new_mem, new_iomap);
+        ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
 out:
         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
+        ttm_bo_mem_put(bo, &old_copy);
         return ret;
 }
 EXPORT_SYMBOL(ttm_bo_move_memcpy);
@@ -345,11 +427,11 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
          * TODO: Explicit member copy would probably be better here.
          */
 
-        spin_lock_init(&fbo->lock);
         init_waitqueue_head(&fbo->event_queue);
         INIT_LIST_HEAD(&fbo->ddestroy);
         INIT_LIST_HEAD(&fbo->lru);
         INIT_LIST_HEAD(&fbo->swap);
+        INIT_LIST_HEAD(&fbo->io_reserve_lru);
         fbo->vm_node = NULL;
         atomic_set(&fbo->cpu_writers, 0);
 
@@ -461,6 +543,8 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo,
                 unsigned long start_page, unsigned long num_pages,
                 struct ttm_bo_kmap_obj *map)
 {
+        struct ttm_mem_type_manager *man =
+                &bo->bdev->man[bo->mem.mem_type];
         unsigned long offset, size;
         int ret;
 
@@ -475,7 +559,9 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo,
         if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
                 return -EPERM;
 #endif
+        (void) ttm_mem_io_lock(man, false);
         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
+        ttm_mem_io_unlock(man);
         if (ret)
                 return ret;
         if (!bo->mem.bus.is_iomem) {
@@ -490,12 +576,15 @@ EXPORT_SYMBOL(ttm_bo_kmap);
 
 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 {
+        struct ttm_buffer_object *bo = map->bo;
+        struct ttm_mem_type_manager *man =
+                &bo->bdev->man[bo->mem.mem_type];
+
         if (!map->virtual)
                 return;
         switch (map->bo_kmap_type) {
         case ttm_bo_map_iomap:
                 iounmap(map->virtual);
-                ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
                 break;
         case ttm_bo_map_vmap:
                 vunmap(map->virtual);
@@ -508,6 +597,9 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
         default:
                 BUG();
         }
+        (void) ttm_mem_io_lock(man, false);
+        ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
+        ttm_mem_io_unlock(man);
         map->virtual = NULL;
         map->page = NULL;
 }
@@ -528,7 +620,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
         struct ttm_buffer_object *ghost_obj;
         void *tmp_obj = NULL;
 
-        spin_lock(&bo->lock);
+        spin_lock(&bdev->fence_lock);
         if (bo->sync_obj) {
                 tmp_obj = bo->sync_obj;
                 bo->sync_obj = NULL;
@@ -537,7 +629,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
         bo->sync_obj_arg = sync_obj_arg;
         if (evict) {
                 ret = ttm_bo_wait(bo, false, false, false);
-                spin_unlock(&bo->lock);
+                spin_unlock(&bdev->fence_lock);
                 if (tmp_obj)
                         driver->sync_obj_unref(&tmp_obj);
                 if (ret)
@@ -560,7 +652,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
                  */
 
                 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
-                spin_unlock(&bo->lock);
+                spin_unlock(&bdev->fence_lock);
                 if (tmp_obj)
                         driver->sync_obj_unref(&tmp_obj);
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index fe6cb77899f4..221b924acebe 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -83,6 +83,8 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         int i;
         unsigned long address = (unsigned long)vmf->virtual_address;
         int retval = VM_FAULT_NOPAGE;
+        struct ttm_mem_type_manager *man =
+                &bdev->man[bo->mem.mem_type];
 
         /*
          * Work around locking order reversal in fault / nopfn
@@ -118,24 +120,28 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
          * move.
          */
 
-        spin_lock(&bo->lock);
+        spin_lock(&bdev->fence_lock);
         if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
                 ret = ttm_bo_wait(bo, false, true, false);
-                spin_unlock(&bo->lock);
+                spin_unlock(&bdev->fence_lock);
                 if (unlikely(ret != 0)) {
                         retval = (ret != -ERESTARTSYS) ?
                             VM_FAULT_SIGBUS : VM_FAULT_NOPAGE;
                         goto out_unlock;
                 }
         } else
-                spin_unlock(&bo->lock);
+                spin_unlock(&bdev->fence_lock);
 
-
-        ret = ttm_mem_io_reserve(bdev, &bo->mem);
-        if (ret) {
-                retval = VM_FAULT_SIGBUS;
+        ret = ttm_mem_io_lock(man, true);
+        if (unlikely(ret != 0)) {
+                retval = VM_FAULT_NOPAGE;
                 goto out_unlock;
         }
+        ret = ttm_mem_io_reserve_vm(bo);
+        if (unlikely(ret != 0)) {
+                retval = VM_FAULT_SIGBUS;
+                goto out_io_unlock;
+        }
 
         page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
             bo->vm_node->start - vma->vm_pgoff;
@@ -144,7 +150,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 
         if (unlikely(page_offset >= bo->num_pages)) {
                 retval = VM_FAULT_SIGBUS;
-                goto out_unlock;
+                goto out_io_unlock;
         }
 
         /*
@@ -182,7 +188,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
                         page = ttm_tt_get_page(ttm, page_offset);
                         if (unlikely(!page && i == 0)) {
                                 retval = VM_FAULT_OOM;
-                                goto out_unlock;
+                                goto out_io_unlock;
                         } else if (unlikely(!page)) {
                                 break;
                         }
@@ -200,14 +206,15 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
                 else if (unlikely(ret != 0)) {
                         retval =
                             (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
-                        goto out_unlock;
+                        goto out_io_unlock;
                 }
 
                 address += PAGE_SIZE;
                 if (unlikely(++page_offset >= page_last))
                         break;
         }
-
+out_io_unlock:
+        ttm_mem_io_unlock(man);
 out_unlock:
         ttm_bo_unreserve(bo);
         return retval;
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index c285c2902d15..3832fe10b4df 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -32,7 +32,7 @@
 #include <linux/sched.h>
 #include <linux/module.h>
 
-void ttm_eu_backoff_reservation(struct list_head *list)
+static void ttm_eu_backoff_reservation_locked(struct list_head *list)
 {
         struct ttm_validate_buffer *entry;
 
@@ -41,10 +41,77 @@ void ttm_eu_backoff_reservation(struct list_head *list)
                 if (!entry->reserved)
                         continue;
 
+                if (entry->removed) {
+                        ttm_bo_add_to_lru(bo);
+                        entry->removed = false;
+
+                }
                 entry->reserved = false;
-                ttm_bo_unreserve(bo);
+                atomic_set(&bo->reserved, 0);
+                wake_up_all(&bo->event_queue);
+        }
+}
+
+static void ttm_eu_del_from_lru_locked(struct list_head *list)
+{
+        struct ttm_validate_buffer *entry;
+
+        list_for_each_entry(entry, list, head) {
+                struct ttm_buffer_object *bo = entry->bo;
+                if (!entry->reserved)
+                        continue;
+
+                if (!entry->removed) {
+                        entry->put_count = ttm_bo_del_from_lru(bo);
+                        entry->removed = true;
+                }
         }
 }
+
+static void ttm_eu_list_ref_sub(struct list_head *list)
+{
+        struct ttm_validate_buffer *entry;
+
+        list_for_each_entry(entry, list, head) {
+                struct ttm_buffer_object *bo = entry->bo;
+
+                if (entry->put_count) {
+                        ttm_bo_list_ref_sub(bo, entry->put_count, true);
+                        entry->put_count = 0;
+                }
+        }
+}
+
+static int ttm_eu_wait_unreserved_locked(struct list_head *list,
+                                         struct ttm_buffer_object *bo)
+{
+        struct ttm_bo_global *glob = bo->glob;
+        int ret;
+
+        ttm_eu_del_from_lru_locked(list);
+        spin_unlock(&glob->lru_lock);
+        ret = ttm_bo_wait_unreserved(bo, true);
+        spin_lock(&glob->lru_lock);
+        if (unlikely(ret != 0))
+                ttm_eu_backoff_reservation_locked(list);
+        return ret;
+}
+
+
+void ttm_eu_backoff_reservation(struct list_head *list)
+{
+        struct ttm_validate_buffer *entry;
+        struct ttm_bo_global *glob;
+
+        if (list_empty(list))
+                return;
+
+        entry = list_first_entry(list, struct ttm_validate_buffer, head);
+        glob = entry->bo->glob;
+        spin_lock(&glob->lru_lock);
+        ttm_eu_backoff_reservation_locked(list);
+        spin_unlock(&glob->lru_lock);
+}
 EXPORT_SYMBOL(ttm_eu_backoff_reservation);
 
 /*
@@ -59,37 +126,76 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation);
  * buffers in different orders.
  */
 
-int ttm_eu_reserve_buffers(struct list_head *list, uint32_t val_seq)
+int ttm_eu_reserve_buffers(struct list_head *list)
 {
+        struct ttm_bo_global *glob;
         struct ttm_validate_buffer *entry;
         int ret;
+        uint32_t val_seq;
+
+        if (list_empty(list))
+                return 0;
+
+        list_for_each_entry(entry, list, head) {
+                entry->reserved = false;
+                entry->put_count = 0;
+                entry->removed = false;
+        }
+
+        entry = list_first_entry(list, struct ttm_validate_buffer, head);
+        glob = entry->bo->glob;
 
 retry:
+        spin_lock(&glob->lru_lock);
+        val_seq = entry->bo->bdev->val_seq++;
+
         list_for_each_entry(entry, list, head) {
                 struct ttm_buffer_object *bo = entry->bo;
 
-                entry->reserved = false;
-                ret = ttm_bo_reserve(bo, true, false, true, val_seq);
-                if (ret != 0) {
-                        ttm_eu_backoff_reservation(list);
-                        if (ret == -EAGAIN) {
-                                ret = ttm_bo_wait_unreserved(bo, true);
-                                if (unlikely(ret != 0))
-                                        return ret;
-                                goto retry;
-                        } else
+retry_this_bo:
+                ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq);
+                switch (ret) {
+                case 0:
+                        break;
+                case -EBUSY:
+                        ret = ttm_eu_wait_unreserved_locked(list, bo);
+                        if (unlikely(ret != 0)) {
+                                spin_unlock(&glob->lru_lock);
+                                ttm_eu_list_ref_sub(list);
                                 return ret;
+                        }
+                        goto retry_this_bo;
+                case -EAGAIN:
+                        ttm_eu_backoff_reservation_locked(list);
+                        spin_unlock(&glob->lru_lock);
+                        ttm_eu_list_ref_sub(list);
+                        ret = ttm_bo_wait_unreserved(bo, true);
+                        if (unlikely(ret != 0))
+                                return ret;
+                        goto retry;
+                default:
+                        ttm_eu_backoff_reservation_locked(list);
+                        spin_unlock(&glob->lru_lock);
+                        ttm_eu_list_ref_sub(list);
+                        return ret;
                 }
 
                 entry->reserved = true;
                 if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {
-                        ttm_eu_backoff_reservation(list);
+                        ttm_eu_backoff_reservation_locked(list);
+                        spin_unlock(&glob->lru_lock);
+                        ttm_eu_list_ref_sub(list);
                         ret = ttm_bo_wait_cpu(bo, false);
                         if (ret)
                                 return ret;
                         goto retry;
                 }
         }
+
+        ttm_eu_del_from_lru_locked(list);
+        spin_unlock(&glob->lru_lock);
+        ttm_eu_list_ref_sub(list);
+
         return 0;
 }
 EXPORT_SYMBOL(ttm_eu_reserve_buffers);
@@ -97,21 +203,36 @@ EXPORT_SYMBOL(ttm_eu_reserve_buffers);
 void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj)
 {
         struct ttm_validate_buffer *entry;
+        struct ttm_buffer_object *bo;
+        struct ttm_bo_global *glob;
+        struct ttm_bo_device *bdev;
+        struct ttm_bo_driver *driver;
 
-        list_for_each_entry(entry, list, head) {
-                struct ttm_buffer_object *bo = entry->bo;
-                struct ttm_bo_driver *driver = bo->bdev->driver;
-                void *old_sync_obj;
+        if (list_empty(list))
+                return;
+
+        bo = list_first_entry(list, struct ttm_validate_buffer, head)->bo;
+        bdev = bo->bdev;
+        driver = bdev->driver;
+        glob = bo->glob;
 
-                spin_lock(&bo->lock);
-                old_sync_obj = bo->sync_obj;
+        spin_lock(&bdev->fence_lock);
+        spin_lock(&glob->lru_lock);
+
+        list_for_each_entry(entry, list, head) {
+                bo = entry->bo;
+                entry->old_sync_obj = bo->sync_obj;
                 bo->sync_obj = driver->sync_obj_ref(sync_obj);
                 bo->sync_obj_arg = entry->new_sync_obj_arg;
-                spin_unlock(&bo->lock);
-                ttm_bo_unreserve(bo);
+                ttm_bo_unreserve_locked(bo);
                 entry->reserved = false;
-                if (old_sync_obj)
-                        driver->sync_obj_unref(&old_sync_obj);
+        }
+        spin_unlock(&glob->lru_lock);
+        spin_unlock(&bdev->fence_lock);
+
+        list_for_each_entry(entry, list, head) {
+                if (entry->old_sync_obj)
+                        driver->sync_obj_unref(&entry->old_sync_obj);
         }
 }
 EXPORT_SYMBOL(ttm_eu_fence_buffer_objects);
diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c
index 75e9d6f86ba4..ebddd443d91a 100644
--- a/drivers/gpu/drm/ttm/ttm_object.c
+++ b/drivers/gpu/drm/ttm/ttm_object.c
@@ -206,7 +206,7 @@ void ttm_base_object_unref(struct ttm_base_object **p_base)
          */
 
         write_lock(&tdev->object_lock);
-        (void)kref_put(&base->refcount, &ttm_release_base);
+        kref_put(&base->refcount, ttm_release_base);
         write_unlock(&tdev->object_lock);
 }
 EXPORT_SYMBOL(ttm_base_object_unref);
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index b1e02fffd3cc..d948575717bf 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -38,6 +38,7 @@
 #include <linux/mm.h>
 #include <linux/seq_file.h> /* for seq_printf */
 #include <linux/slab.h>
+#include <linux/dma-mapping.h>
 
 #include <asm/atomic.h>
 
@@ -394,12 +395,14 @@ static int ttm_pool_get_num_unused_pages(void)
 /**
  * Callback for mm to request pool to reduce number of page held.
  */
-static int ttm_pool_mm_shrink(struct shrinker *shrink, int shrink_pages, gfp_t gfp_mask)
+static int ttm_pool_mm_shrink(struct shrinker *shrink,
+                              struct shrink_control *sc)
 {
         static atomic_t start_pool = ATOMIC_INIT(0);
         unsigned i;
         unsigned pool_offset = atomic_add_return(1, &start_pool);
         struct ttm_page_pool *pool;
+        int shrink_pages = sc->nr_to_scan;
 
         pool_offset = pool_offset % NUM_POOLS;
         /* select start pool in round robin fashion */
@@ -662,7 +665,8 @@ out:
  * cached pages.
  */
 int ttm_get_pages(struct list_head *pages, int flags,
-                enum ttm_caching_state cstate, unsigned count)
+                  enum ttm_caching_state cstate, unsigned count,
+                  dma_addr_t *dma_address)
 {
         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
         struct page *p = NULL;
@@ -720,7 +724,7 @@ int ttm_get_pages(struct list_head *pages, int flags,
                         printk(KERN_ERR TTM_PFX
                                "Failed to allocate extra pages "
                                "for large request.");
-                        ttm_put_pages(pages, 0, flags, cstate);
+                        ttm_put_pages(pages, 0, flags, cstate, NULL);
                         return r;
                 }
         }
@@ -731,7 +735,7 @@ int ttm_get_pages(struct list_head *pages, int flags,
 
 /* Put all pages in pages list to correct pool to wait for reuse */
 void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags,
-                enum ttm_caching_state cstate)
+                   enum ttm_caching_state cstate, dma_addr_t *dma_address)
 {
         unsigned long irq_flags;
         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index a7bab87a548b..58c271ebc0f7 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -31,6 +31,7 @@
 #include <linux/sched.h>
 #include <linux/highmem.h>
 #include <linux/pagemap.h>
+#include <linux/shmem_fs.h>
 #include <linux/file.h>
 #include <linux/swap.h>
 #include <linux/slab.h>
@@ -49,12 +50,16 @@ static int ttm_tt_swapin(struct ttm_tt *ttm);
 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
 {
         ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(*ttm->pages));
+        ttm->dma_address = drm_calloc_large(ttm->num_pages,
+                                            sizeof(*ttm->dma_address));
 }
 
 static void ttm_tt_free_page_directory(struct ttm_tt *ttm)
 {
         drm_free_large(ttm->pages);
         ttm->pages = NULL;
+        drm_free_large(ttm->dma_address);
+        ttm->dma_address = NULL;
 }
 
 static void ttm_tt_free_user_pages(struct ttm_tt *ttm)
@@ -105,7 +110,8 @@ static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index)
 
                 INIT_LIST_HEAD(&h);
 
-                ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1);
+                ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1,
+                                    &ttm->dma_address[index]);
 
                 if (ret != 0)
                         return NULL;
@@ -164,7 +170,7 @@ int ttm_tt_populate(struct ttm_tt *ttm)
         }
 
         be->func->populate(be, ttm->num_pages, ttm->pages,
-                           ttm->dummy_read_page);
+                           ttm->dummy_read_page, ttm->dma_address);
         ttm->state = tt_unbound;
         return 0;
 }
@@ -298,7 +304,8 @@ static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm)
                         count++;
                 }
         }
-        ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state);
+        ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state,
+                      ttm->dma_address);
         ttm->state = tt_unpopulated;
         ttm->first_himem_page = ttm->num_pages;
         ttm->last_lomem_page = -1;
@@ -326,7 +333,7 @@ void ttm_tt_destroy(struct ttm_tt *ttm)
                 ttm_tt_free_page_directory(ttm);
         }
 
-        if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP) &&
+        if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) &&
             ttm->swap_storage)
                 fput(ttm->swap_storage);
 
@@ -440,10 +447,8 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
                 return ret;
 
         ret = be->func->bind(be, bo_mem);
-        if (ret) {
-                printk(KERN_ERR TTM_PFX "Couldn't bind backend.\n");
+        if (unlikely(ret != 0))
                 return ret;
-        }
 
         ttm->state = tt_bound;
 
@@ -480,7 +485,7 @@ static int ttm_tt_swapin(struct ttm_tt *ttm)
         swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;
 
         for (i = 0; i < ttm->num_pages; ++i) {
-                from_page = read_mapping_page(swap_space, i, NULL);
+                from_page = shmem_read_mapping_page(swap_space, i);
                 if (IS_ERR(from_page)) {
                         ret = PTR_ERR(from_page);
                         goto out_err;
@@ -499,7 +504,7 @@ static int ttm_tt_swapin(struct ttm_tt *ttm)
                 page_cache_release(from_page);
         }
 
-        if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP))
+        if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP))
                 fput(swap_storage);
         ttm->swap_storage = NULL;
         ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
@@ -510,7 +515,7 @@ out_err:
         return ret;
 }
 
-int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
+int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage)
 {
         struct address_space *swap_space;
         struct file *swap_storage;
@@ -536,7 +541,7 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
                 return 0;
         }
 
-        if (!persistant_swap_storage) {
+        if (!persistent_swap_storage) {
                 swap_storage = shmem_file_setup("ttm swap",
                                                 ttm->num_pages << PAGE_SHIFT,
                                                 0);
@@ -545,7 +550,7 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
                         return PTR_ERR(swap_storage);
                 }
         } else
-                swap_storage = persistant_swap_storage;
+                swap_storage = persistent_swap_storage;
 
         swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;
 
@@ -553,7 +558,7 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
                 from_page = ttm->pages[i];
                 if (unlikely(from_page == NULL))
                         continue;
-                to_page = read_mapping_page(swap_space, i, NULL);
+                to_page = shmem_read_mapping_page(swap_space, i);
                 if (unlikely(IS_ERR(to_page))) {
                         ret = PTR_ERR(to_page);
                         goto out_err;
@@ -573,12 +578,12 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
         ttm_tt_free_alloced_pages(ttm);
         ttm->swap_storage = swap_storage;
         ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
-        if (persistant_swap_storage)
-                ttm->page_flags |= TTM_PAGE_FLAG_PERSISTANT_SWAP;
+        if (persistent_swap_storage)
+                ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP;
 
         return 0;
 out_err:
-        if (!persistant_swap_storage)
+        if (!persistent_swap_storage)
                 fput(swap_storage);
 
         return ret;