drm/ttm: call ttm_bo_cleanup_refs with reservation and lru lock held, v3

By removing the unlocking of lru and retaking it immediately, a race is
removed where the bo is taken off the swap list or the lru list between
the unlock and relock. As such the cleanup_refs code can be simplified,
it will attempt to call ttm_bo_wait non-blockingly, and if it fails
it will drop the locks and perform a blocking wait, or return an error
if no_wait_gpu was set.

The need for looping is also eliminated, since swapout and evict_mem_first
will always follow the destruction path, no new fence is allowed
to be attached. As far as I can see this may already have been the case,
but the unlocking / relocking required a complicated loop to deal with
re-reservation.

Changes since v1:
 - Simplify no_wait_gpu case by folding it in with empty ddestroy.
 - Hold a reservation while calling ttm_bo_cleanup_memtype_use again.
Changes since v2:
 - Do not remove bo from lru list while waiting

Signed-off-by: Maarten Lankhorst <maarten.lankhorst@canonical.com>
Reviewed-by: Thomas Hellstrom <thellstrom@vmware.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>

1 files changed, 76 insertions, 47 deletions

diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index b7781453bfd1..ef223d581a70 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -488,12 +488,16 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
         ttm_bo_mem_put(bo, &bo->mem);
 
         atomic_set(&bo->reserved, 0);
+        wake_up_all(&bo->event_queue);
 
         /*
-         * Make processes trying to reserve really pick it up.
+         * Since the final reference to this bo may not be dropped by
+         * the current task we have to put a memory barrier here to make
+         * sure the changes done in this function are always visible.
+         *
+         * This function only needs protection against the final kref_put.
          */
-        smp_mb__after_atomic_dec();
-        wake_up_all(&bo->event_queue);
+        smp_mb__before_atomic_dec();
 }
 
 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
@@ -543,68 +547,84 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 }
 
 /**
- * function ttm_bo_cleanup_refs
+ * function ttm_bo_cleanup_refs_and_unlock
  * If bo idle, remove from delayed- and lru lists, and unref.
  * If not idle, do nothing.
  *
+ * Must be called with lru_lock and reservation held, this function
+ * will drop both before returning.
+ *
  * @interruptible         Any sleeps should occur interruptibly.
- * @no_wait_reserve       Never wait for reserve. Return -EBUSY instead.
  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
  */
 
-static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
-                               bool interruptible,
-                               bool no_wait_reserve,
-                               bool no_wait_gpu)
+static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
+                                          bool interruptible,
+                                          bool no_wait_gpu)
 {
         struct ttm_bo_device *bdev = bo->bdev;
+        struct ttm_bo_driver *driver = bdev->driver;
         struct ttm_bo_global *glob = bo->glob;
         int put_count;
-        int ret = 0;
+        int ret;
 
-retry:
         spin_lock(&bdev->fence_lock);
-        ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
-        spin_unlock(&bdev->fence_lock);
+        ret = ttm_bo_wait(bo, false, false, true);
 
-        if (unlikely(ret != 0))
-                return ret;
+        if (ret && !no_wait_gpu) {
+                void *sync_obj;
 
-retry_reserve:
-        spin_lock(&glob->lru_lock);
+                /*
+                 * Take a reference to the fence and unreserve,
+                 * at this point the buffer should be dead, so
+                 * no new sync objects can be attached.
+                 */
+                sync_obj = driver->sync_obj_ref(&bo->sync_obj);
+                spin_unlock(&bdev->fence_lock);
 
-        if (unlikely(list_empty(&bo->ddestroy))) {
+                atomic_set(&bo->reserved, 0);
+                wake_up_all(&bo->event_queue);
                 spin_unlock(&glob->lru_lock);
-                return 0;
-        }
 
-        ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
-
-        if (unlikely(ret == -EBUSY)) {
-                spin_unlock(&glob->lru_lock);
-                if (likely(!no_wait_reserve))
-                        ret = ttm_bo_wait_unreserved(bo, interruptible);
-                if (unlikely(ret != 0))
+                ret = driver->sync_obj_wait(sync_obj, false, interruptible);
+                driver->sync_obj_unref(&sync_obj);
+                if (ret)
                         return ret;
 
-                goto retry_reserve;
-        }
+                /*
+                 * remove sync_obj with ttm_bo_wait, the wait should be
+                 * finished, and no new wait object should have been added.
+                 */
+                spin_lock(&bdev->fence_lock);
+                ret = ttm_bo_wait(bo, false, false, true);
+                WARN_ON(ret);
+                spin_unlock(&bdev->fence_lock);
+                if (ret)
+                        return ret;
 
-        BUG_ON(ret != 0);
+                spin_lock(&glob->lru_lock);
+                ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
 
-        /**
-         * We can re-check for sync object without taking
-         * the bo::lock since setting the sync object requires
-         * also bo::reserved. A busy object at this point may
-         * be caused by another thread recently starting an accelerated
-         * eviction.
-         */
+                /*
+                 * We raced, and lost, someone else holds the reservation now,
+                 * and is probably busy in ttm_bo_cleanup_memtype_use.
+                 *
+                 * Even if it's not the case, because we finished waiting any
+                 * delayed destruction would succeed, so just return success
+                 * here.
+                 */
+                if (ret) {
+                        spin_unlock(&glob->lru_lock);
+                        return 0;
+                }
+        } else
+                spin_unlock(&bdev->fence_lock);
 
-        if (unlikely(bo->sync_obj)) {
+        if (ret || unlikely(list_empty(&bo->ddestroy))) {
                 atomic_set(&bo->reserved, 0);
                 wake_up_all(&bo->event_queue);
                 spin_unlock(&glob->lru_lock);
-                goto retry;
+                return ret;
         }
 
         put_count = ttm_bo_del_from_lru(bo);
@@ -647,9 +667,13 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
                         kref_get(&nentry->list_kref);
                 }
 
-                spin_unlock(&glob->lru_lock);
-                ret = ttm_bo_cleanup_refs(entry, false, !remove_all,
-                                          !remove_all);
+                ret = ttm_bo_reserve_locked(entry, false, !remove_all, false, 0);
+                if (!ret)
+                        ret = ttm_bo_cleanup_refs_and_unlock(entry, false,
+                                                             !remove_all);
+                else
+                        spin_unlock(&glob->lru_lock);
+
                 kref_put(&entry->list_kref, ttm_bo_release_list);
                 entry = nentry;
 
@@ -800,9 +824,13 @@ retry:
         kref_get(&bo->list_kref);
 
         if (!list_empty(&bo->ddestroy)) {
-                spin_unlock(&glob->lru_lock);
-                ret = ttm_bo_cleanup_refs(bo, interruptible,
-                                          no_wait_reserve, no_wait_gpu);
+                ret = ttm_bo_reserve_locked(bo, interruptible, no_wait_reserve, false, 0);
+                if (!ret)
+                        ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible,
+                                                             no_wait_gpu);
+                else
+                        spin_unlock(&glob->lru_lock);
+
                 kref_put(&bo->list_kref, ttm_bo_release_list);
 
                 return ret;
@@ -1796,8 +1824,9 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
                 kref_get(&bo->list_kref);
 
                 if (!list_empty(&bo->ddestroy)) {
-                        spin_unlock(&glob->lru_lock);
-                        (void) ttm_bo_cleanup_refs(bo, false, false, false);
+                        ttm_bo_reserve_locked(bo, false, false, false, 0);
+                        ttm_bo_cleanup_refs_and_unlock(bo, false, false);
+
                         kref_put(&bo->list_kref, ttm_bo_release_list);
                         spin_lock(&glob->lru_lock);
                         continue;