1 files changed, 81 insertions, 90 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 1a8f4e0dd023..ccba88cc8c54 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -119,9 +119,7 @@ static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
          * is currently evicted. add the bo to the evicted list to make sure it
          * is validated on next vm use to avoid fault.
          * */
-        spin_lock(&vm->status_lock);
         list_move_tail(&base->vm_status, &vm->evicted);
-        spin_unlock(&vm->status_lock);
 }
 
 /**
@@ -226,24 +224,16 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                               void *param)
 {
         struct ttm_bo_global *glob = adev->mman.bdev.glob;
-        int r;
-
-        spin_lock(&vm->status_lock);
-        while (!list_empty(&vm->evicted)) {
-                struct amdgpu_vm_bo_base *bo_base;
-                struct amdgpu_bo *bo;
+        struct amdgpu_vm_bo_base *bo_base, *tmp;
+        int r = 0;
 
-                bo_base = list_first_entry(&vm->evicted,
-                                           struct amdgpu_vm_bo_base,
-                                           vm_status);
-                spin_unlock(&vm->status_lock);
+        list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
+                struct amdgpu_bo *bo = bo_base->bo;
 
-                bo = bo_base->bo;
-                BUG_ON(!bo);
                 if (bo->parent) {
                         r = validate(param, bo);
                         if (r)
-                                return r;
+                                break;
 
                         spin_lock(&glob->lru_lock);
                         ttm_bo_move_to_lru_tail(&bo->tbo);
@@ -252,22 +242,29 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                         spin_unlock(&glob->lru_lock);
                 }
 
-                if (bo->tbo.type == ttm_bo_type_kernel &&
-                    vm->use_cpu_for_update) {
-                        r = amdgpu_bo_kmap(bo, NULL);
-                        if (r)
-                                return r;
-                }
-
-                spin_lock(&vm->status_lock);
-                if (bo->tbo.type != ttm_bo_type_kernel)
+                if (bo->tbo.type != ttm_bo_type_kernel) {
+                        spin_lock(&vm->moved_lock);
                         list_move(&bo_base->vm_status, &vm->moved);
-                else
+                        spin_unlock(&vm->moved_lock);
+                } else {
                         list_move(&bo_base->vm_status, &vm->relocated);
+                }
         }
-        spin_unlock(&vm->status_lock);
 
-        return 0;
+        spin_lock(&glob->lru_lock);
+        list_for_each_entry(bo_base, &vm->idle, vm_status) {
+                struct amdgpu_bo *bo = bo_base->bo;
+
+                if (!bo->parent)
+                        continue;
+
+                ttm_bo_move_to_lru_tail(&bo->tbo);
+                if (bo->shadow)
+                        ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
+        }
+        spin_unlock(&glob->lru_lock);
+
+        return r;
 }
 
 /**
@@ -279,13 +276,7 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  */
 bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 {
-        bool ready;
-
-        spin_lock(&vm->status_lock);
-        ready = list_empty(&vm->evicted);
-        spin_unlock(&vm->status_lock);
-
-        return ready;
+        return list_empty(&vm->evicted);
 }
 
 /**
@@ -477,9 +468,7 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
                         pt->parent = amdgpu_bo_ref(parent->base.bo);
 
                         amdgpu_vm_bo_base_init(&entry->base, vm, pt);
-                        spin_lock(&vm->status_lock);
                         list_move(&entry->base.vm_status, &vm->relocated);
-                        spin_unlock(&vm->status_lock);
                 }
 
                 if (level < AMDGPU_VM_PTB) {
@@ -926,10 +915,8 @@ static void amdgpu_vm_invalidate_level(struct amdgpu_device *adev,
                 if (!entry->base.bo)
                         continue;
 
-                spin_lock(&vm->status_lock);
-                if (list_empty(&entry->base.vm_status))
-                        list_add(&entry->base.vm_status, &vm->relocated);
-                spin_unlock(&vm->status_lock);
+                if (!entry->base.moved)
+                        list_move(&entry->base.vm_status, &vm->relocated);
                 amdgpu_vm_invalidate_level(adev, vm, entry, level + 1);
         }
 }
@@ -959,6 +946,14 @@ restart:
         params.adev = adev;
 
         if (vm->use_cpu_for_update) {
+                struct amdgpu_vm_bo_base *bo_base;
+
+                list_for_each_entry(bo_base, &vm->relocated, vm_status) {
+                        r = amdgpu_bo_kmap(bo_base->bo, NULL);
+                        if (unlikely(r))
+                                return r;
+                }
+
                 r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
                 if (unlikely(r))
                         return r;
@@ -974,7 +969,6 @@ restart:
                 params.func = amdgpu_vm_do_set_ptes;
         }
 
-        spin_lock(&vm->status_lock);
         while (!list_empty(&vm->relocated)) {
                 struct amdgpu_vm_bo_base *bo_base, *parent;
                 struct amdgpu_vm_pt *pt, *entry;
@@ -983,14 +977,12 @@ restart:
                 bo_base = list_first_entry(&vm->relocated,
                                            struct amdgpu_vm_bo_base,
                                            vm_status);
-                list_del_init(&bo_base->vm_status);
-                spin_unlock(&vm->status_lock);
+                bo_base->moved = false;
+                list_move(&bo_base->vm_status, &vm->idle);
 
                 bo = bo_base->bo->parent;
-                if (!bo) {
-                        spin_lock(&vm->status_lock);
+                if (!bo)
                         continue;
-                }
 
                 parent = list_first_entry(&bo->va, struct amdgpu_vm_bo_base,
                                           bo_list);
@@ -999,12 +991,10 @@ restart:
 
                 amdgpu_vm_update_pde(&params, vm, pt, entry);
 
-                spin_lock(&vm->status_lock);
                 if (!vm->use_cpu_for_update &&
                     (ndw - params.ib->length_dw) < 32)
                         break;
         }
-        spin_unlock(&vm->status_lock);
 
         if (vm->use_cpu_for_update) {
                 /* Flush HDP */
@@ -1107,9 +1097,7 @@ static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
                 if (entry->huge) {
                         /* Add the entry to the relocated list to update it. */
                         entry->huge = false;
-                        spin_lock(&p->vm->status_lock);
                         list_move(&entry->base.vm_status, &p->vm->relocated);
-                        spin_unlock(&p->vm->status_lock);
                 }
                 return;
         }
@@ -1588,18 +1576,22 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
                 amdgpu_asic_flush_hdp(adev, NULL);
         }
 
-        spin_lock(&vm->status_lock);
+        spin_lock(&vm->moved_lock);
         list_del_init(&bo_va->base.vm_status);
+        spin_unlock(&vm->moved_lock);
 
         /* If the BO is not in its preferred location add it back to
          * the evicted list so that it gets validated again on the
          * next command submission.
          */
-        if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
-            !(bo->preferred_domains &
-            amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type)))
-                list_add_tail(&bo_va->base.vm_status, &vm->evicted);
-        spin_unlock(&vm->status_lock);
+        if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
+                uint32_t mem_type = bo->tbo.mem.mem_type;
+
+                if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type)))
+                        list_add_tail(&bo_va->base.vm_status, &vm->evicted);
+                else
+                        list_add(&bo_va->base.vm_status, &vm->idle);
+        }
 
         list_splice_init(&bo_va->invalids, &bo_va->valids);
         bo_va->cleared = clear;
@@ -1808,19 +1800,18 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
                            struct amdgpu_vm *vm)
 {
+        struct amdgpu_bo_va *bo_va, *tmp;
+        struct list_head moved;
         bool clear;
-        int r = 0;
-
-        spin_lock(&vm->status_lock);
-        while (!list_empty(&vm->moved)) {
-                struct amdgpu_bo_va *bo_va;
-                struct reservation_object *resv;
+        int r;
 
-                bo_va = list_first_entry(&vm->moved,
-                        struct amdgpu_bo_va, base.vm_status);
-                spin_unlock(&vm->status_lock);
+        INIT_LIST_HEAD(&moved);
+        spin_lock(&vm->moved_lock);
+        list_splice_init(&vm->moved, &moved);
+        spin_unlock(&vm->moved_lock);
 
-                resv = bo_va->base.bo->tbo.resv;
+        list_for_each_entry_safe(bo_va, tmp, &moved, base.vm_status) {
+                struct reservation_object *resv = bo_va->base.bo->tbo.resv;
 
                 /* Per VM BOs never need to bo cleared in the page tables */
                 if (resv == vm->root.base.bo->tbo.resv)
@@ -1833,17 +1824,19 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
                         clear = true;
 
                 r = amdgpu_vm_bo_update(adev, bo_va, clear);
-                if (r)
+                if (r) {
+                        spin_lock(&vm->moved_lock);
+                        list_splice(&moved, &vm->moved);
+                        spin_unlock(&vm->moved_lock);
                         return r;
+                }
 
                 if (!clear && resv != vm->root.base.bo->tbo.resv)
                         reservation_object_unlock(resv);
 
-                spin_lock(&vm->status_lock);
         }
-        spin_unlock(&vm->status_lock);
 
-        return r;
+        return 0;
 }
 
 /**
@@ -1902,11 +1895,11 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
         if (mapping->flags & AMDGPU_PTE_PRT)
                 amdgpu_vm_prt_get(adev);
 
-        if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
-                spin_lock(&vm->status_lock);
-                if (list_empty(&bo_va->base.vm_status))
-                        list_add(&bo_va->base.vm_status, &vm->moved);
-                spin_unlock(&vm->status_lock);
+        if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
+            !bo_va->base.moved) {
+                spin_lock(&vm->moved_lock);
+                list_move(&bo_va->base.vm_status, &vm->moved);
+                spin_unlock(&vm->moved_lock);
         }
         trace_amdgpu_vm_bo_map(bo_va, mapping);
 }
@@ -2216,9 +2209,9 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
 
         list_del(&bo_va->base.bo_list);
 
-        spin_lock(&vm->status_lock);
+        spin_lock(&vm->moved_lock);
         list_del(&bo_va->base.vm_status);
-        spin_unlock(&vm->status_lock);
+        spin_unlock(&vm->moved_lock);
 
         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
                 list_del(&mapping->list);
@@ -2258,31 +2251,28 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
 
         list_for_each_entry(bo_base, &bo->va, bo_list) {
                 struct amdgpu_vm *vm = bo_base->vm;
+                bool was_moved = bo_base->moved;
 
                 bo_base->moved = true;
                 if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
-                        spin_lock(&bo_base->vm->status_lock);
                         if (bo->tbo.type == ttm_bo_type_kernel)
                                 list_move(&bo_base->vm_status, &vm->evicted);
                         else
                                 list_move_tail(&bo_base->vm_status,
                                                &vm->evicted);
-                        spin_unlock(&bo_base->vm->status_lock);
                         continue;
                 }
 
-                if (bo->tbo.type == ttm_bo_type_kernel) {
-                        spin_lock(&bo_base->vm->status_lock);
-                        if (list_empty(&bo_base->vm_status))
-                                list_add(&bo_base->vm_status, &vm->relocated);
-                        spin_unlock(&bo_base->vm->status_lock);
+                if (was_moved)
                         continue;
-                }
 
-                spin_lock(&bo_base->vm->status_lock);
-                if (list_empty(&bo_base->vm_status))
-                        list_add(&bo_base->vm_status, &vm->moved);
-                spin_unlock(&bo_base->vm->status_lock);
+                if (bo->tbo.type == ttm_bo_type_kernel) {
+                        list_move(&bo_base->vm_status, &vm->relocated);
+                } else {
+                        spin_lock(&bo_base->vm->moved_lock);
+                        list_move(&bo_base->vm_status, &vm->moved);
+                        spin_unlock(&bo_base->vm->moved_lock);
+                }
         }
 }
 
@@ -2391,10 +2381,11 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         vm->va = RB_ROOT_CACHED;
         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
                 vm->reserved_vmid[i] = NULL;
-        spin_lock_init(&vm->status_lock);
         INIT_LIST_HEAD(&vm->evicted);
         INIT_LIST_HEAD(&vm->relocated);
+        spin_lock_init(&vm->moved_lock);
         INIT_LIST_HEAD(&vm->moved);
+        INIT_LIST_HEAD(&vm->idle);
         INIT_LIST_HEAD(&vm->freed);
 
         /* create scheduler entity for page table updates */