drm/ttm: Add a bo list reserve fastpath (v2)

Makes it possible to reserve a list of buffer objects with a single
spin lock / unlock if there is no contention.
Should improve cpu usage on SMP kernels.

v2: Initialize private list members on reserve and don't call
ttm_bo_list_ref_sub() with zero put_count.

Signed-off-by: Thomas Hellstrom <thellstrom@vmware.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>

1 files changed, 113 insertions, 11 deletions

diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index c285c2902d15..201a71d111ec 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -32,6 +32,72 @@
 #include <linux/sched.h>
 #include <linux/module.h>
 
+static void ttm_eu_backoff_reservation_locked(struct list_head *list)
+{
+        struct ttm_validate_buffer *entry;
+
+        list_for_each_entry(entry, list, head) {
+                struct ttm_buffer_object *bo = entry->bo;
+                if (!entry->reserved)
+                        continue;
+
+                if (entry->removed) {
+                        ttm_bo_add_to_lru(bo);
+                        entry->removed = false;
+
+                }
+                entry->reserved = false;
+                atomic_set(&bo->reserved, 0);
+                wake_up_all(&bo->event_queue);
+        }
+}
+
+static void ttm_eu_del_from_lru_locked(struct list_head *list)
+{
+        struct ttm_validate_buffer *entry;
+
+        list_for_each_entry(entry, list, head) {
+                struct ttm_buffer_object *bo = entry->bo;
+                if (!entry->reserved)
+                        continue;
+
+                if (!entry->removed) {
+                        entry->put_count = ttm_bo_del_from_lru(bo);
+                        entry->removed = true;
+                }
+        }
+}
+
+static void ttm_eu_list_ref_sub(struct list_head *list)
+{
+        struct ttm_validate_buffer *entry;
+
+        list_for_each_entry(entry, list, head) {
+                struct ttm_buffer_object *bo = entry->bo;
+
+                if (entry->put_count) {
+                        ttm_bo_list_ref_sub(bo, entry->put_count, true);
+                        entry->put_count = 0;
+                }
+        }
+}
+
+static int ttm_eu_wait_unreserved_locked(struct list_head *list,
+                                         struct ttm_buffer_object *bo)
+{
+        struct ttm_bo_global *glob = bo->glob;
+        int ret;
+
+        ttm_eu_del_from_lru_locked(list);
+        spin_unlock(&glob->lru_lock);
+        ret = ttm_bo_wait_unreserved(bo, true);
+        spin_lock(&glob->lru_lock);
+        if (unlikely(ret != 0))
+                ttm_eu_backoff_reservation_locked(list);
+        return ret;
+}
+
+
 void ttm_eu_backoff_reservation(struct list_head *list)
 {
         struct ttm_validate_buffer *entry;
@@ -61,35 +127,71 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation);
 
 int ttm_eu_reserve_buffers(struct list_head *list, uint32_t val_seq)
 {
+        struct ttm_bo_global *glob;
         struct ttm_validate_buffer *entry;
         int ret;
 
+        if (list_empty(list))
+                return 0;
+
+        list_for_each_entry(entry, list, head) {
+                entry->reserved = false;
+                entry->put_count = 0;
+                entry->removed = false;
+        }
+
+        entry = list_first_entry(list, struct ttm_validate_buffer, head);
+        glob = entry->bo->glob;
+
 retry:
+        spin_lock(&glob->lru_lock);
         list_for_each_entry(entry, list, head) {
                 struct ttm_buffer_object *bo = entry->bo;
 
-                entry->reserved = false;
-                ret = ttm_bo_reserve(bo, true, false, true, val_seq);
-                if (ret != 0) {
-                        ttm_eu_backoff_reservation(list);
-                        if (ret == -EAGAIN) {
-                                ret = ttm_bo_wait_unreserved(bo, true);
-                                if (unlikely(ret != 0))
-                                        return ret;
-                                goto retry;
-                        } else
+retry_this_bo:
+                ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq);
+                switch (ret) {
+                case 0:
+                        break;
+                case -EBUSY:
+                        ret = ttm_eu_wait_unreserved_locked(list, bo);
+                        if (unlikely(ret != 0)) {
+                                spin_unlock(&glob->lru_lock);
+                                ttm_eu_list_ref_sub(list);
+                                return ret;
+                        }
+                        goto retry_this_bo;
+                case -EAGAIN:
+                        ttm_eu_backoff_reservation_locked(list);
+                        spin_unlock(&glob->lru_lock);
+                        ttm_eu_list_ref_sub(list);
+                        ret = ttm_bo_wait_unreserved(bo, true);
+                        if (unlikely(ret != 0))
                                 return ret;
+                        goto retry;
+                default:
+                        ttm_eu_backoff_reservation_locked(list);
+                        spin_unlock(&glob->lru_lock);
+                        ttm_eu_list_ref_sub(list);
+                        return ret;
                 }
 
                 entry->reserved = true;
                 if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {
-                        ttm_eu_backoff_reservation(list);
+                        ttm_eu_backoff_reservation_locked(list);
+                        spin_unlock(&glob->lru_lock);
+                        ttm_eu_list_ref_sub(list);
                         ret = ttm_bo_wait_cpu(bo, false);
                         if (ret)
                                 return ret;
                         goto retry;
                 }
         }
+
+        ttm_eu_del_from_lru_locked(list);
+        spin_unlock(&glob->lru_lock);
+        ttm_eu_list_ref_sub(list);
+
         return 0;
 }
 EXPORT_SYMBOL(ttm_eu_reserve_buffers);