Merge branch 'drm-next-4.15' of git://people.freedesktop.org/~agd5f/linux into drm-next

First feature pull for 4.15.  Highlights:
- Per VM BO support
- Lots of powerplay cleanups
- Powerplay support for CI
- pasid mgr for kfd
- interrupt infrastructure for recoverable page faults
- SR-IOV fixes
- initial GPU reset for vega10
- prime mmap support
- ttm page table debugging improvements
- lots of bug fixes

* 'drm-next-4.15' of git://people.freedesktop.org/~agd5f/linux: (232 commits)
  drm/amdgpu: clarify license in amdgpu_trace_points.c
  drm/amdgpu: Add gem_prime_mmap support
  drm/amd/powerplay: delete dead code in smumgr
  drm/amd/powerplay: delete SMUM_FIELD_MASK
  drm/amd/powerplay: delete SMUM_WAIT_INDIRECT_FIELD
  drm/amd/powerplay: delete SMUM_READ_FIELD
  drm/amd/powerplay: delete SMUM_SET_FIELD
  drm/amd/powerplay: delete SMUM_READ_VFPF_INDIRECT_FIELD
  drm/amd/powerplay: delete SMUM_WRITE_VFPF_INDIRECT_FIELD
  drm/amd/powerplay: delete SMUM_WRITE_FIELD
  drm/amd/powerplay: delete SMU_WRITE_INDIRECT_FIELD
  drm/amd/powerplay: move macros to hwmgr.h
  drm/amd/powerplay: move PHM_WAIT_VFPF_INDIRECT_FIELD to hwmgr.h
  drm/amd/powerplay: move SMUM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL to hwmgr.h
  drm/amd/powerplay: move SMUM_WAIT_INDIRECT_FIELD_UNEQUAL to hwmgr.h
  drm/amd/powerplay: add new helper functions in hwmgr.h
  drm/amd/powerplay: use SMU_IND_INDEX/DATA_11 pair
  drm/amd/powerplay: refine powerplay code.
  drm/amd/powerplay: delete dead code in hwmgr.h
  drm/amd/powerplay: refine interface in struct pp_smumgr_func
  ...

1 files changed, 382 insertions, 216 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index bd20ff018512..bbcc67038203 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -27,12 +27,59 @@
  */
 #include <linux/dma-fence-array.h>
 #include <linux/interval_tree_generic.h>
+#include <linux/idr.h>
 #include <drm/drmP.h>
 #include <drm/amdgpu_drm.h>
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
 /*
+ * PASID manager
+ *
+ * PASIDs are global address space identifiers that can be shared
+ * between the GPU, an IOMMU and the driver. VMs on different devices
+ * may use the same PASID if they share the same address
+ * space. Therefore PASIDs are allocated using a global IDA. VMs are
+ * looked up from the PASID per amdgpu_device.
+ */
+static DEFINE_IDA(amdgpu_vm_pasid_ida);
+
+/**
+ * amdgpu_vm_alloc_pasid - Allocate a PASID
+ * @bits: Maximum width of the PASID in bits, must be at least 1
+ *
+ * Allocates a PASID of the given width while keeping smaller PASIDs
+ * available if possible.
+ *
+ * Returns a positive integer on success. Returns %-EINVAL if bits==0.
+ * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on
+ * memory allocation failure.
+ */
+int amdgpu_vm_alloc_pasid(unsigned int bits)
+{
+        int pasid = -EINVAL;
+
+        for (bits = min(bits, 31U); bits > 0; bits--) {
+                pasid = ida_simple_get(&amdgpu_vm_pasid_ida,
+                                       1U << (bits - 1), 1U << bits,
+                                       GFP_KERNEL);
+                if (pasid != -ENOSPC)
+                        break;
+        }
+
+        return pasid;
+}
+
+/**
+ * amdgpu_vm_free_pasid - Free a PASID
+ * @pasid: PASID to free
+ */
+void amdgpu_vm_free_pasid(unsigned int pasid)
+{
+        ida_simple_remove(&amdgpu_vm_pasid_ida, pasid);
+}
+
+/*
  * GPUVM
  * GPUVM is similar to the legacy gart on older asics, however
  * rather than there being a single global gart table
@@ -140,7 +187,7 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
                          struct list_head *validated,
                          struct amdgpu_bo_list_entry *entry)
 {
-        entry->robj = vm->root.bo;
+        entry->robj = vm->root.base.bo;
         entry->priority = 0;
         entry->tv.bo = &entry->robj->tbo;
         entry->tv.shared = true;
@@ -149,86 +196,80 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 }
 
 /**
- * amdgpu_vm_validate_layer - validate a single page table level
+ * amdgpu_vm_validate_pt_bos - validate the page table BOs
  *
- * @parent: parent page table level
+ * @adev: amdgpu device pointer
+ * @vm: vm providing the BOs
  * @validate: callback to do the validation
  * @param: parameter for the validation callback
  *
  * Validate the page table BOs on command submission if neccessary.
  */
-static int amdgpu_vm_validate_level(struct amdgpu_vm_pt *parent,
-                                    int (*validate)(void *, struct amdgpu_bo *),
-                                    void *param, bool use_cpu_for_update,
-                                    struct ttm_bo_global *glob)
+int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+                              int (*validate)(void *p, struct amdgpu_bo *bo),
+                              void *param)
 {
-        unsigned i;
+        struct ttm_bo_global *glob = adev->mman.bdev.glob;
         int r;
 
-        if (use_cpu_for_update) {
-                r = amdgpu_bo_kmap(parent->bo, NULL);
-                if (r)
-                        return r;
-        }
-
-        if (!parent->entries)
-                return 0;
+        spin_lock(&vm->status_lock);
+        while (!list_empty(&vm->evicted)) {
+                struct amdgpu_vm_bo_base *bo_base;
+                struct amdgpu_bo *bo;
 
-        for (i = 0; i <= parent->last_entry_used; ++i) {
-                struct amdgpu_vm_pt *entry = &parent->entries[i];
+                bo_base = list_first_entry(&vm->evicted,
+                                           struct amdgpu_vm_bo_base,
+                                           vm_status);
+                spin_unlock(&vm->status_lock);
 
-                if (!entry->bo)
-                        continue;
+                bo = bo_base->bo;
+                BUG_ON(!bo);
+                if (bo->parent) {
+                        r = validate(param, bo);
+                        if (r)
+                                return r;
 
-                r = validate(param, entry->bo);
-                if (r)
-                        return r;
+                        spin_lock(&glob->lru_lock);
+                        ttm_bo_move_to_lru_tail(&bo->tbo);
+                        if (bo->shadow)
+                                ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
+                        spin_unlock(&glob->lru_lock);
+                }
 
-                spin_lock(&glob->lru_lock);
-                ttm_bo_move_to_lru_tail(&entry->bo->tbo);
-                if (entry->bo->shadow)
-                        ttm_bo_move_to_lru_tail(&entry->bo->shadow->tbo);
-                spin_unlock(&glob->lru_lock);
+                if (bo->tbo.type == ttm_bo_type_kernel &&
+                    vm->use_cpu_for_update) {
+                        r = amdgpu_bo_kmap(bo, NULL);
+                        if (r)
+                                return r;
+                }
 
-                /*
-                 * Recurse into the sub directory. This is harmless because we
-                 * have only a maximum of 5 layers.
-                 */
-                r = amdgpu_vm_validate_level(entry, validate, param,
-                                             use_cpu_for_update, glob);
-                if (r)
-                        return r;
+                spin_lock(&vm->status_lock);
+                if (bo->tbo.type != ttm_bo_type_kernel)
+                        list_move(&bo_base->vm_status, &vm->moved);
+                else
+                        list_move(&bo_base->vm_status, &vm->relocated);
         }
+        spin_unlock(&vm->status_lock);
 
-        return r;
+        return 0;
 }
 
 /**
- * amdgpu_vm_validate_pt_bos - validate the page table BOs
+ * amdgpu_vm_ready - check VM is ready for updates
  *
- * @adev: amdgpu device pointer
- * @vm: vm providing the BOs
- * @validate: callback to do the validation
- * @param: parameter for the validation callback
+ * @vm: VM to check
  *
- * Validate the page table BOs on command submission if neccessary.
+ * Check if all VM PDs/PTs are ready for updates
  */
-int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                              int (*validate)(void *p, struct amdgpu_bo *bo),
-                              void *param)
+bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 {
-        uint64_t num_evictions;
+        bool ready;
 
-        /* We only need to validate the page tables
-         * if they aren't already valid.
-         */
-        num_evictions = atomic64_read(&adev->num_evictions);
-        if (num_evictions == vm->last_eviction_counter)
-                return 0;
+        spin_lock(&vm->status_lock);
+        ready = list_empty(&vm->evicted);
+        spin_unlock(&vm->status_lock);
 
-        return amdgpu_vm_validate_level(&vm->root, validate, param,
-                                        vm->use_cpu_for_update,
-                                        adev->mman.bdev.glob);
+        return ready;
 }
 
 /**
@@ -294,11 +335,11 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 
         /* walk over the address space and allocate the page tables */
         for (pt_idx = from; pt_idx <= to; ++pt_idx) {
-                struct reservation_object *resv = vm->root.bo->tbo.resv;
+                struct reservation_object *resv = vm->root.base.bo->tbo.resv;
                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
                 struct amdgpu_bo *pt;
 
-                if (!entry->bo) {
+                if (!entry->base.bo) {
                         r = amdgpu_bo_create(adev,
                                              amdgpu_vm_bo_size(adev, level),
                                              AMDGPU_GPU_PAGE_SIZE, true,
@@ -319,9 +360,14 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
                         /* Keep a reference to the root directory to avoid
                         * freeing them up in the wrong order.
                         */
-                        pt->parent = amdgpu_bo_ref(vm->root.bo);
-
-                        entry->bo = pt;
+                        pt->parent = amdgpu_bo_ref(parent->base.bo);
+
+                        entry->base.vm = vm;
+                        entry->base.bo = pt;
+                        list_add_tail(&entry->base.bo_list, &pt->va);
+                        spin_lock(&vm->status_lock);
+                        list_add(&entry->base.vm_status, &vm->relocated);
+                        spin_unlock(&vm->status_lock);
                         entry->addr = 0;
                 }
 
@@ -988,7 +1034,7 @@ static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         int r;
 
         amdgpu_sync_create(&sync);
-        amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.resv, owner);
+        amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner);
         r = amdgpu_sync_wait(&sync, true);
         amdgpu_sync_free(&sync);
 
@@ -1007,18 +1053,17 @@ static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  */
 static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                                   struct amdgpu_vm *vm,
-                                  struct amdgpu_vm_pt *parent,
-                                  unsigned level)
+                                  struct amdgpu_vm_pt *parent)
 {
         struct amdgpu_bo *shadow;
         struct amdgpu_ring *ring = NULL;
         uint64_t pd_addr, shadow_addr = 0;
-        uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
         uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
         unsigned count = 0, pt_idx, ndw = 0;
         struct amdgpu_job *job;
         struct amdgpu_pte_update_params params;
         struct dma_fence *fence = NULL;
+        uint32_t incr;
 
         int r;
 
@@ -1027,10 +1072,10 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
         memset(&params, 0, sizeof(params));
         params.adev = adev;
-        shadow = parent->bo->shadow;
+        shadow = parent->base.bo->shadow;
 
         if (vm->use_cpu_for_update) {
-                pd_addr = (unsigned long)amdgpu_bo_kptr(parent->bo);
+                pd_addr = (unsigned long)amdgpu_bo_kptr(parent->base.bo);
                 r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
                 if (unlikely(r))
                         return r;
@@ -1046,7 +1091,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                 /* assume the worst case */
                 ndw += parent->last_entry_used * 6;
 
-                pd_addr = amdgpu_bo_gpu_offset(parent->bo);
+                pd_addr = amdgpu_bo_gpu_offset(parent->base.bo);
 
                 if (shadow) {
                         shadow_addr = amdgpu_bo_gpu_offset(shadow);
@@ -1066,12 +1111,17 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
         /* walk over the address space and update the directory */
         for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
-                struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
+                struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
+                struct amdgpu_bo *bo = entry->base.bo;
                 uint64_t pde, pt;
 
                 if (bo == NULL)
                         continue;
 
+                spin_lock(&vm->status_lock);
+                list_del_init(&entry->base.vm_status);
+                spin_unlock(&vm->status_lock);
+
                 pt = amdgpu_bo_gpu_offset(bo);
                 pt = amdgpu_gart_get_vm_pde(adev, pt);
                 /* Don't update huge pages here */
@@ -1082,6 +1132,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                 parent->entries[pt_idx].addr = pt | AMDGPU_PTE_VALID;
 
                 pde = pd_addr + pt_idx * 8;
+                incr = amdgpu_bo_size(bo);
                 if (((last_pde + 8 * count) != pde) ||
                     ((last_pt + incr * count) != pt) ||
                     (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
@@ -1109,7 +1160,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
         }
 
         if (count) {
-                if (vm->root.bo->shadow)
+                if (vm->root.base.bo->shadow)
                         params.func(&params, last_shadow, last_pt,
                                     count, incr, AMDGPU_PTE_VALID);
 
@@ -1122,7 +1173,8 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                         amdgpu_job_free(job);
                 } else {
                         amdgpu_ring_pad_ib(ring, params.ib);
-                        amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
+                        amdgpu_sync_resv(adev, &job->sync,
+                                         parent->base.bo->tbo.resv,
                                          AMDGPU_FENCE_OWNER_VM);
                         if (shadow)
                                 amdgpu_sync_resv(adev, &job->sync,
@@ -1135,26 +1187,11 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                         if (r)
                                 goto error_free;
 
-                        amdgpu_bo_fence(parent->bo, fence, true);
-                        dma_fence_put(vm->last_dir_update);
-                        vm->last_dir_update = dma_fence_get(fence);
-                        dma_fence_put(fence);
+                        amdgpu_bo_fence(parent->base.bo, fence, true);
+                        dma_fence_put(vm->last_update);
+                        vm->last_update = fence;
                 }
         }
-        /*
-         * Recurse into the subdirectories. This recursion is harmless because
-         * we only have a maximum of 5 layers.
-         */
-        for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
-                struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
-
-                if (!entry->bo)
-                        continue;
-
-                r = amdgpu_vm_update_level(adev, vm, entry, level + 1);
-                if (r)
-                        return r;
-        }
 
         return 0;
 
@@ -1170,7 +1207,8 @@ error_free:
  *
  * Mark all PD level as invalid after an error.
  */
-static void amdgpu_vm_invalidate_level(struct amdgpu_vm_pt *parent)
+static void amdgpu_vm_invalidate_level(struct amdgpu_vm *vm,
+                                       struct amdgpu_vm_pt *parent)
 {
         unsigned pt_idx;
 
@@ -1181,11 +1219,15 @@ static void amdgpu_vm_invalidate_level(struct amdgpu_vm_pt *parent)
         for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 
-                if (!entry->bo)
+                if (!entry->base.bo)
                         continue;
 
                 entry->addr = ~0ULL;
-                amdgpu_vm_invalidate_level(entry);
+                spin_lock(&vm->status_lock);
+                if (list_empty(&entry->base.vm_status))
+                        list_add(&entry->base.vm_status, &vm->relocated);
+                spin_unlock(&vm->status_lock);
+                amdgpu_vm_invalidate_level(vm, entry);
         }
 }
 
@@ -1203,9 +1245,38 @@ int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 {
         int r;
 
-        r = amdgpu_vm_update_level(adev, vm, &vm->root, 0);
-        if (r)
-                amdgpu_vm_invalidate_level(&vm->root);
+        spin_lock(&vm->status_lock);
+        while (!list_empty(&vm->relocated)) {
+                struct amdgpu_vm_bo_base *bo_base;
+                struct amdgpu_bo *bo;
+
+                bo_base = list_first_entry(&vm->relocated,
+                                           struct amdgpu_vm_bo_base,
+                                           vm_status);
+                spin_unlock(&vm->status_lock);
+
+                bo = bo_base->bo->parent;
+                if (bo) {
+                        struct amdgpu_vm_bo_base *parent;
+                        struct amdgpu_vm_pt *pt;
+
+                        parent = list_first_entry(&bo->va,
+                                                  struct amdgpu_vm_bo_base,
+                                                  bo_list);
+                        pt = container_of(parent, struct amdgpu_vm_pt, base);
+
+                        r = amdgpu_vm_update_level(adev, vm, pt);
+                        if (r) {
+                                amdgpu_vm_invalidate_level(vm, &vm->root);
+                                return r;
+                        }
+                        spin_lock(&vm->status_lock);
+                } else {
+                        spin_lock(&vm->status_lock);
+                        list_del_init(&bo_base->vm_status);
+                }
+        }
+        spin_unlock(&vm->status_lock);
 
         if (vm->use_cpu_for_update) {
                 /* Flush HDP */
@@ -1236,7 +1307,7 @@ void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
         *entry = &p->vm->root;
         while ((*entry)->entries) {
                 idx = addr >> (p->adev->vm_manager.block_size * level--);
-                idx %= amdgpu_bo_size((*entry)->bo) / 8;
+                idx %= amdgpu_bo_size((*entry)->base.bo) / 8;
                 *parent = *entry;
                 *entry = &(*entry)->entries[idx];
         }
@@ -1272,7 +1343,7 @@ static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
             p->src ||
             !(flags & AMDGPU_PTE_VALID)) {
 
-                dst = amdgpu_bo_gpu_offset(entry->bo);
+                dst = amdgpu_bo_gpu_offset(entry->base.bo);
                 dst = amdgpu_gart_get_vm_pde(p->adev, dst);
                 flags = AMDGPU_PTE_VALID;
         } else {
@@ -1298,18 +1369,18 @@ static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
                 tmp = p->pages_addr;
                 p->pages_addr = NULL;
 
-                pd_addr = (unsigned long)amdgpu_bo_kptr(parent->bo);
+                pd_addr = (unsigned long)amdgpu_bo_kptr(parent->base.bo);
                 pde = pd_addr + (entry - parent->entries) * 8;
                 amdgpu_vm_cpu_set_ptes(p, pde, dst, 1, 0, flags);
 
                 p->pages_addr = tmp;
         } else {
-                if (parent->bo->shadow) {
-                        pd_addr = amdgpu_bo_gpu_offset(parent->bo->shadow);
+                if (parent->base.bo->shadow) {
+                        pd_addr = amdgpu_bo_gpu_offset(parent->base.bo->shadow);
                         pde = pd_addr + (entry - parent->entries) * 8;
                         amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
                 }
-                pd_addr = amdgpu_bo_gpu_offset(parent->bo);
+                pd_addr = amdgpu_bo_gpu_offset(parent->base.bo);
                 pde = pd_addr + (entry - parent->entries) * 8;
                 amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
         }
@@ -1360,7 +1431,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
                 if (entry->addr & AMDGPU_PDE_PTE)
                         continue;
 
-                pt = entry->bo;
+                pt = entry->base.bo;
                 if (use_cpu_update) {
                         pe_start = (unsigned long)amdgpu_bo_kptr(pt);
                 } else {
@@ -1396,8 +1467,6 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
                                 uint64_t start, uint64_t end,
                                 uint64_t dst, uint64_t flags)
 {
-        int r;
-
         /**
          * The MC L1 TLB supports variable sized pages, based on a fragment
          * field in the PTE. When this field is set to a non-zero value, page
@@ -1416,39 +1485,38 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
          * Userspace can support this by aligning virtual base address and
          * allocation size to the fragment size.
          */
-        unsigned pages_per_frag = params->adev->vm_manager.fragment_size;
-        uint64_t frag_flags = AMDGPU_PTE_FRAG(pages_per_frag);
-        uint64_t frag_align = 1 << pages_per_frag;
-
-        uint64_t frag_start = ALIGN(start, frag_align);
-        uint64_t frag_end = end & ~(frag_align - 1);
+        unsigned max_frag = params->adev->vm_manager.fragment_size;
+        int r;
 
         /* system pages are non continuously */
-        if (params->src || !(flags & AMDGPU_PTE_VALID) ||
-            (frag_start >= frag_end))
+        if (params->src || !(flags & AMDGPU_PTE_VALID))
                 return amdgpu_vm_update_ptes(params, start, end, dst, flags);
 
-        /* handle the 4K area at the beginning */
-        if (start != frag_start) {
-                r = amdgpu_vm_update_ptes(params, start, frag_start,
-                                          dst, flags);
+        while (start != end) {
+                uint64_t frag_flags, frag_end;
+                unsigned frag;
+
+                /* This intentionally wraps around if no bit is set */
+                frag = min((unsigned)ffs(start) - 1,
+                           (unsigned)fls64(end - start) - 1);
+                if (frag >= max_frag) {
+                        frag_flags = AMDGPU_PTE_FRAG(max_frag);
+                        frag_end = end & ~((1ULL << max_frag) - 1);
+                } else {
+                        frag_flags = AMDGPU_PTE_FRAG(frag);
+                        frag_end = start + (1 << frag);
+                }
+
+                r = amdgpu_vm_update_ptes(params, start, frag_end, dst,
+                                          flags | frag_flags);
                 if (r)
                         return r;
-                dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
-        }
 
-        /* handle the area in the middle */
-        r = amdgpu_vm_update_ptes(params, frag_start, frag_end, dst,
-                                  flags | frag_flags);
-        if (r)
-                return r;
-
-        /* handle the 4K area at the end */
-        if (frag_end != end) {
-                dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
-                r = amdgpu_vm_update_ptes(params, frag_end, end, dst, flags);
+                dst += (frag_end - start) * AMDGPU_GPU_PAGE_SIZE;
+                start = frag_end;
         }
-        return r;
+
+        return 0;
 }
 
 /**
@@ -1456,7 +1524,6 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
  *
  * @adev: amdgpu_device pointer
  * @exclusive: fence we need to sync to
- * @src: address where to copy page table entries from
  * @pages_addr: DMA addresses to use for mapping
  * @vm: requested vm
  * @start: start of mapped range
@@ -1470,7 +1537,6 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
  */
 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
                                        struct dma_fence *exclusive,
-                                       uint64_t src,
                                        dma_addr_t *pages_addr,
                                        struct amdgpu_vm *vm,
                                        uint64_t start, uint64_t last,
@@ -1488,7 +1554,6 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         memset(&params, 0, sizeof(params));
         params.adev = adev;
         params.vm = vm;
-        params.src = src;
 
         /* sync to everything on unmapping */
         if (!(flags & AMDGPU_PTE_VALID))
@@ -1517,10 +1582,12 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         nptes = last - start + 1;
 
         /*
-         * reserve space for one command every (1 << BLOCK_SIZE)
+         * reserve space for two commands every (1 << BLOCK_SIZE)
          *  entries or 2k dwords (whatever is smaller)
+         *
+         * The second command is for the shadow pagetables.
          */
-        ncmds = (nptes >> min(adev->vm_manager.block_size, 11u)) + 1;
+        ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2;
 
         /* padding, etc. */
         ndw = 64;
@@ -1528,15 +1595,9 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         /* one PDE write for each huge page */
         ndw += ((nptes >> adev->vm_manager.block_size) + 1) * 6;
 
-        if (src) {
-                /* only copy commands needed */
-                ndw += ncmds * 7;
-
-                params.func = amdgpu_vm_do_copy_ptes;
-
-        } else if (pages_addr) {
+        if (pages_addr) {
                 /* copy commands needed */
-                ndw += ncmds * 7;
+                ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw;
 
                 /* and also PTEs */
                 ndw += nptes * 2;
@@ -1545,10 +1606,11 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 
         } else {
                 /* set page commands needed */
-                ndw += ncmds * 10;
+                ndw += ncmds * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw;
 
-                /* two extra commands for begin/end of fragment */
-                ndw += 2 * 10;
+                /* extra commands for begin/end fragments */
+                ndw += 2 * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw
+                                * adev->vm_manager.fragment_size;
 
                 params.func = amdgpu_vm_do_set_ptes;
         }
@@ -1559,7 +1621,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 
         params.ib = &job->ibs[0];
 
-        if (!src && pages_addr) {
+        if (pages_addr) {
                 uint64_t *pte;
                 unsigned i;
 
@@ -1580,12 +1642,12 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         if (r)
                 goto error_free;
 
-        r = amdgpu_sync_resv(adev, &job->sync, vm->root.bo->tbo.resv,
+        r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,
                              owner);
         if (r)
                 goto error_free;
 
-        r = reservation_object_reserve_shared(vm->root.bo->tbo.resv);
+        r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);
         if (r)
                 goto error_free;
 
@@ -1600,14 +1662,14 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         if (r)
                 goto error_free;
 
-        amdgpu_bo_fence(vm->root.bo, f, true);
+        amdgpu_bo_fence(vm->root.base.bo, f, true);
         dma_fence_put(*fence);
         *fence = f;
         return 0;
 
 error_free:
         amdgpu_job_free(job);
-        amdgpu_vm_invalidate_level(&vm->root);
+        amdgpu_vm_invalidate_level(vm, &vm->root);
         return r;
 }
 
@@ -1636,7 +1698,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
                                       struct drm_mm_node *nodes,
                                       struct dma_fence **fence)
 {
-        uint64_t pfn, src = 0, start = mapping->start;
+        uint64_t pfn, start = mapping->start;
         int r;
 
         /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
@@ -1691,8 +1753,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
                 addr += pfn << PAGE_SHIFT;
 
                 last = min((uint64_t)mapping->last, start + max_entries - 1);
-                r = amdgpu_vm_bo_update_mapping(adev, exclusive,
-                                                src, pages_addr, vm,
+                r = amdgpu_vm_bo_update_mapping(adev, exclusive, pages_addr, vm,
                                                 start, last, flags, addr,
                                                 fence);
                 if (r)
@@ -1730,7 +1791,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
         dma_addr_t *pages_addr = NULL;
         struct ttm_mem_reg *mem;
         struct drm_mm_node *nodes;
-        struct dma_fence *exclusive;
+        struct dma_fence *exclusive, **last_update;
         uint64_t flags;
         int r;
 
@@ -1756,38 +1817,43 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
         else
                 flags = 0x0;
 
-        spin_lock(&vm->status_lock);
-        if (!list_empty(&bo_va->base.vm_status))
+        if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
+                last_update = &vm->last_update;
+        else
+                last_update = &bo_va->last_pt_update;
+
+        if (!clear && bo_va->base.moved) {
+                bo_va->base.moved = false;
                 list_splice_init(&bo_va->valids, &bo_va->invalids);
-        spin_unlock(&vm->status_lock);
+
+        } else if (bo_va->cleared != clear) {
+                list_splice_init(&bo_va->valids, &bo_va->invalids);
+        }
 
         list_for_each_entry(mapping, &bo_va->invalids, list) {
                 r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
                                                mapping, flags, nodes,
-                                               &bo_va->last_pt_update);
+                                               last_update);
                 if (r)
                         return r;
         }
 
-        if (trace_amdgpu_vm_bo_mapping_enabled()) {
-                list_for_each_entry(mapping, &bo_va->valids, list)
-                        trace_amdgpu_vm_bo_mapping(mapping);
-
-                list_for_each_entry(mapping, &bo_va->invalids, list)
-                        trace_amdgpu_vm_bo_mapping(mapping);
+        if (vm->use_cpu_for_update) {
+                /* Flush HDP */
+                mb();
+                amdgpu_gart_flush_gpu_tlb(adev, 0);
         }
 
         spin_lock(&vm->status_lock);
-        list_splice_init(&bo_va->invalids, &bo_va->valids);
         list_del_init(&bo_va->base.vm_status);
-        if (clear)
-                list_add(&bo_va->base.vm_status, &vm->cleared);
         spin_unlock(&vm->status_lock);
 
-        if (vm->use_cpu_for_update) {
-                /* Flush HDP */
-                mb();
-                amdgpu_gart_flush_gpu_tlb(adev, 0);
+        list_splice_init(&bo_va->invalids, &bo_va->valids);
+        bo_va->cleared = clear;
+
+        if (trace_amdgpu_vm_bo_mapping_enabled()) {
+                list_for_each_entry(mapping, &bo_va->valids, list)
+                        trace_amdgpu_vm_bo_mapping(mapping);
         }
 
         return 0;
@@ -1895,7 +1961,7 @@ static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
  */
 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 {
-        struct reservation_object *resv = vm->root.bo->tbo.resv;
+        struct reservation_object *resv = vm->root.base.bo->tbo.resv;
         struct dma_fence *excl, **shared;
         unsigned i, shared_count;
         int r;
@@ -1953,7 +2019,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
                 if (vm->pte_support_ats)
                         init_pte_value = AMDGPU_PTE_SYSTEM;
 
-                r = amdgpu_vm_bo_update_mapping(adev, NULL, 0, NULL, vm,
+                r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
                                                 mapping->start, mapping->last,
                                                 init_pte_value, 0, &f);
                 amdgpu_vm_free_mapping(adev, vm, mapping, f);
@@ -1975,29 +2041,35 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 }
 
 /**
- * amdgpu_vm_clear_moved - clear moved BOs in the PT
+ * amdgpu_vm_handle_moved - handle moved BOs in the PT
  *
  * @adev: amdgpu_device pointer
  * @vm: requested vm
+ * @sync: sync object to add fences to
  *
- * Make sure all moved BOs are cleared in the PT.
+ * Make sure all BOs which are moved are updated in the PTs.
  * Returns 0 for success.
  *
- * PTs have to be reserved and mutex must be locked!
+ * PTs have to be reserved!
  */
-int amdgpu_vm_clear_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                            struct amdgpu_sync *sync)
+int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
+                           struct amdgpu_vm *vm)
 {
-        struct amdgpu_bo_va *bo_va = NULL;
+        bool clear;
         int r = 0;
 
         spin_lock(&vm->status_lock);
         while (!list_empty(&vm->moved)) {
+                struct amdgpu_bo_va *bo_va;
+
                 bo_va = list_first_entry(&vm->moved,
                         struct amdgpu_bo_va, base.vm_status);
                 spin_unlock(&vm->status_lock);
 
-                r = amdgpu_vm_bo_update(adev, bo_va, true);
+                /* Per VM BOs never need to bo cleared in the page tables */
+                clear = bo_va->base.bo->tbo.resv != vm->root.base.bo->tbo.resv;
+
+                r = amdgpu_vm_bo_update(adev, bo_va, clear);
                 if (r)
                         return r;
 
@@ -2005,9 +2077,6 @@ int amdgpu_vm_clear_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         }
         spin_unlock(&vm->status_lock);
 
-        if (bo_va)
-                r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
-
         return r;
 }
 
@@ -2049,6 +2118,39 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
         return bo_va;
 }
 
+
+/**
+ * amdgpu_vm_bo_insert_mapping - insert a new mapping
+ *
+ * @adev: amdgpu_device pointer
+ * @bo_va: bo_va to store the address
+ * @mapping: the mapping to insert
+ *
+ * Insert a new mapping into all structures.
+ */
+static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
+                                    struct amdgpu_bo_va *bo_va,
+                                    struct amdgpu_bo_va_mapping *mapping)
+{
+        struct amdgpu_vm *vm = bo_va->base.vm;
+        struct amdgpu_bo *bo = bo_va->base.bo;
+
+        mapping->bo_va = bo_va;
+        list_add(&mapping->list, &bo_va->invalids);
+        amdgpu_vm_it_insert(mapping, &vm->va);
+
+        if (mapping->flags & AMDGPU_PTE_PRT)
+                amdgpu_vm_prt_get(adev);
+
+        if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
+                spin_lock(&vm->status_lock);
+                if (list_empty(&bo_va->base.vm_status))
+                        list_add(&bo_va->base.vm_status, &vm->moved);
+                spin_unlock(&vm->status_lock);
+        }
+        trace_amdgpu_vm_bo_map(bo_va, mapping);
+}
+
 /**
  * amdgpu_vm_bo_map - map bo inside a vm
  *
@@ -2100,17 +2202,12 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
         if (!mapping)
                 return -ENOMEM;
 
-        INIT_LIST_HEAD(&mapping->list);
         mapping->start = saddr;
         mapping->last = eaddr;
         mapping->offset = offset;
         mapping->flags = flags;
 
-        list_add(&mapping->list, &bo_va->invalids);
-        amdgpu_vm_it_insert(mapping, &vm->va);
-
-        if (flags & AMDGPU_PTE_PRT)
-                amdgpu_vm_prt_get(adev);
+        amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
 
         return 0;
 }
@@ -2137,7 +2234,6 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
 {
         struct amdgpu_bo_va_mapping *mapping;
         struct amdgpu_bo *bo = bo_va->base.bo;
-        struct amdgpu_vm *vm = bo_va->base.vm;
         uint64_t eaddr;
         int r;
 
@@ -2171,11 +2267,7 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
         mapping->offset = offset;
         mapping->flags = flags;
 
-        list_add(&mapping->list, &bo_va->invalids);
-        amdgpu_vm_it_insert(mapping, &vm->va);
-
-        if (flags & AMDGPU_PTE_PRT)
-                amdgpu_vm_prt_get(adev);
+        amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
 
         return 0;
 }
@@ -2221,6 +2313,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
 
         list_del(&mapping->list);
         amdgpu_vm_it_remove(mapping, &vm->va);
+        mapping->bo_va = NULL;
         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
 
         if (valid)
@@ -2306,6 +2399,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
                 if (tmp->last > eaddr)
                     tmp->last = eaddr;
 
+                tmp->bo_va = NULL;
                 list_add(&tmp->list, &vm->freed);
                 trace_amdgpu_vm_bo_unmap(NULL, tmp);
         }
@@ -2332,6 +2426,19 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
 }
 
 /**
+ * amdgpu_vm_bo_lookup_mapping - find mapping by address
+ *
+ * @vm: the requested VM
+ *
+ * Find a mapping by it's address.
+ */
+struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
+                                                         uint64_t addr)
+{
+        return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
+}
+
+/**
  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
  *
  * @adev: amdgpu_device pointer
@@ -2356,6 +2463,7 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
                 list_del(&mapping->list);
                 amdgpu_vm_it_remove(mapping, &vm->va);
+                mapping->bo_va = NULL;
                 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
                 list_add(&mapping->list, &vm->freed);
         }
@@ -2380,15 +2488,36 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
  * Mark @bo as invalid.
  */
 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
-                             struct amdgpu_bo *bo)
+                             struct amdgpu_bo *bo, bool evicted)
 {
         struct amdgpu_vm_bo_base *bo_base;
 
         list_for_each_entry(bo_base, &bo->va, bo_list) {
+                struct amdgpu_vm *vm = bo_base->vm;
+
+                bo_base->moved = true;
+                if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
+                        spin_lock(&bo_base->vm->status_lock);
+                        if (bo->tbo.type == ttm_bo_type_kernel)
+                                list_move(&bo_base->vm_status, &vm->evicted);
+                        else
+                                list_move_tail(&bo_base->vm_status,
+                                               &vm->evicted);
+                        spin_unlock(&bo_base->vm->status_lock);
+                        continue;
+                }
+
+                if (bo->tbo.type == ttm_bo_type_kernel) {
+                        spin_lock(&bo_base->vm->status_lock);
+                        if (list_empty(&bo_base->vm_status))
+                                list_add(&bo_base->vm_status, &vm->relocated);
+                        spin_unlock(&bo_base->vm->status_lock);
+                        continue;
+                }
+
                 spin_lock(&bo_base->vm->status_lock);
                 if (list_empty(&bo_base->vm_status))
-                        list_add(&bo_base->vm_status,
-                                 &bo_base->vm->moved);
+                        list_add(&bo_base->vm_status, &vm->moved);
                 spin_unlock(&bo_base->vm->status_lock);
         }
 }
@@ -2458,7 +2587,7 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_
  * Init @vm fields.
  */
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                   int vm_context)
+                   int vm_context, unsigned int pasid)
 {
         const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
                 AMDGPU_VM_PTE_COUNT(adev) * 8);
@@ -2474,8 +2603,9 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
                 vm->reserved_vmid[i] = NULL;
         spin_lock_init(&vm->status_lock);
+        INIT_LIST_HEAD(&vm->evicted);
+        INIT_LIST_HEAD(&vm->relocated);
         INIT_LIST_HEAD(&vm->moved);
-        INIT_LIST_HEAD(&vm->cleared);
         INIT_LIST_HEAD(&vm->freed);
 
         /* create scheduler entity for page table updates */
@@ -2506,7 +2636,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                          vm->use_cpu_for_update ? "CPU" : "SDMA");
         WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
                   "CPU update of VM recommended only for large BAR system\n");
-        vm->last_dir_update = NULL;
+        vm->last_update = NULL;
 
         flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
                         AMDGPU_GEM_CREATE_VRAM_CLEARED;
@@ -2519,30 +2649,46 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, 0), align, true,
                              AMDGPU_GEM_DOMAIN_VRAM,
                              flags,
-                             NULL, NULL, init_pde_value, &vm->root.bo);
+                             NULL, NULL, init_pde_value, &vm->root.base.bo);
         if (r)
                 goto error_free_sched_entity;
 
-        r = amdgpu_bo_reserve(vm->root.bo, false);
-        if (r)
-                goto error_free_root;
-
-        vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
+        vm->root.base.vm = vm;
+        list_add_tail(&vm->root.base.bo_list, &vm->root.base.bo->va);
+        INIT_LIST_HEAD(&vm->root.base.vm_status);
 
         if (vm->use_cpu_for_update) {
-                r = amdgpu_bo_kmap(vm->root.bo, NULL);
+                r = amdgpu_bo_reserve(vm->root.base.bo, false);
                 if (r)
                         goto error_free_root;
+
+                r = amdgpu_bo_kmap(vm->root.base.bo, NULL);
+                amdgpu_bo_unreserve(vm->root.base.bo);
+                if (r)
+                        goto error_free_root;
+        }
+
+        if (pasid) {
+                unsigned long flags;
+
+                spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
+                r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
+                              GFP_ATOMIC);
+                spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
+                if (r < 0)
+                        goto error_free_root;
+
+                vm->pasid = pasid;
         }
 
-        amdgpu_bo_unreserve(vm->root.bo);
+        INIT_KFIFO(vm->faults);
 
         return 0;
 
 error_free_root:
-        amdgpu_bo_unref(&vm->root.bo->shadow);
-        amdgpu_bo_unref(&vm->root.bo);
-        vm->root.bo = NULL;
+        amdgpu_bo_unref(&vm->root.base.bo->shadow);
+        amdgpu_bo_unref(&vm->root.base.bo);
+        vm->root.base.bo = NULL;
 
 error_free_sched_entity:
         amd_sched_entity_fini(&ring->sched, &vm->entity);
@@ -2561,9 +2707,11 @@ static void amdgpu_vm_free_levels(struct amdgpu_vm_pt *level)
 {
         unsigned i;
 
-        if (level->bo) {
-                amdgpu_bo_unref(&level->bo->shadow);
-                amdgpu_bo_unref(&level->bo);
+        if (level->base.bo) {
+                list_del(&level->base.bo_list);
+                list_del(&level->base.vm_status);
+                amdgpu_bo_unref(&level->base.bo->shadow);
+                amdgpu_bo_unref(&level->base.bo);
         }
 
         if (level->entries)
@@ -2586,8 +2734,21 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 {
         struct amdgpu_bo_va_mapping *mapping, *tmp;
         bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
+        u64 fault;
         int i;
 
+        /* Clear pending page faults from IH when the VM is destroyed */
+        while (kfifo_get(&vm->faults, &fault))
+                amdgpu_ih_clear_fault(adev, fault);
+
+        if (vm->pasid) {
+                unsigned long flags;
+
+                spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
+                idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
+                spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
+        }
+
         amd_sched_entity_fini(vm->entity.sched, &vm->entity);
 
         if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
@@ -2610,7 +2771,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
         }
 
         amdgpu_vm_free_levels(&vm->root);
-        dma_fence_put(vm->last_dir_update);
+        dma_fence_put(vm->last_update);
         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
                 amdgpu_vm_free_reserved_vmid(adev, vm, i);
 }
@@ -2668,6 +2829,8 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
         adev->vm_manager.vm_update_mode = 0;
 #endif
 
+        idr_init(&adev->vm_manager.pasid_idr);
+        spin_lock_init(&adev->vm_manager.pasid_lock);
 }
 
 /**
@@ -2681,6 +2844,9 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
 {
         unsigned i, j;
 
+        WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
+        idr_destroy(&adev->vm_manager.pasid_idr);
+
         for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
                 struct amdgpu_vm_id_manager *id_mgr =
                         &adev->vm_manager.id_mgr[i];