1 files changed, 133 insertions, 91 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index fcf421263fd9..a44fc12ae1f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -47,6 +47,7 @@
 #include "amdgpu_object.h"
 #include "amdgpu_trace.h"
 #include "amdgpu_amdkfd.h"
+#include "amdgpu_sdma.h"
 #include "bif/bif_4_1_d.h"
 
 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
@@ -255,6 +256,13 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 
         abo = ttm_to_amdgpu_bo(bo);
         switch (bo->mem.mem_type) {
+        case AMDGPU_PL_GDS:
+        case AMDGPU_PL_GWS:
+        case AMDGPU_PL_OA:
+                placement->num_placement = 0;
+                placement->num_busy_placement = 0;
+                return;
+
         case TTM_PL_VRAM:
                 if (!adev->mman.buffer_funcs_enabled) {
                         /* Move to system memory */
@@ -282,6 +290,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
         case TTM_PL_TT:
         default:
                 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
+                break;
         }
         *placement = abo->placement;
 }
@@ -344,7 +353,7 @@ static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
 {
         uint64_t addr = 0;
 
-        if (mem->mem_type != TTM_PL_TT || amdgpu_gtt_mgr_has_gart_addr(mem)) {
+        if (mm_node->start != AMDGPU_BO_INVALID_OFFSET) {
                 addr = mm_node->start << PAGE_SHIFT;
                 addr += bo->bdev->man[mem->mem_type].gpu_offset;
         }
@@ -432,8 +441,7 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
                 /* Map only what needs to be accessed. Map src to window 0 and
                  * dst to window 1
                  */
-                if (src->mem->mem_type == TTM_PL_TT &&
-                    !amdgpu_gtt_mgr_has_gart_addr(src->mem)) {
+                if (src->mem->start == AMDGPU_BO_INVALID_OFFSET) {
                         r = amdgpu_map_buffer(src->bo, src->mem,
                                         PFN_UP(cur_size + src_page_offset),
                                         src_node_start, 0, ring,
@@ -446,8 +454,7 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
                         from += src_page_offset;
                 }
 
-                if (dst->mem->mem_type == TTM_PL_TT &&
-                    !amdgpu_gtt_mgr_has_gart_addr(dst->mem)) {
+                if (dst->mem->start == AMDGPU_BO_INVALID_OFFSET) {
                         r = amdgpu_map_buffer(dst->bo, dst->mem,
                                         PFN_UP(cur_size + dst_page_offset),
                                         dst_node_start, 1, ring,
@@ -525,7 +532,11 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
         if (r)
                 goto error;
 
-        r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
+        /* Always block for VM page tables before committing the new location */
+        if (bo->type == ttm_bo_type_kernel)
+                r = ttm_bo_move_accel_cleanup(bo, fence, true, new_mem);
+        else
+                r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
         dma_fence_put(fence);
         return r;
 
@@ -676,6 +687,16 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
                 amdgpu_move_null(bo, new_mem);
                 return 0;
         }
+        if (old_mem->mem_type == AMDGPU_PL_GDS ||
+            old_mem->mem_type == AMDGPU_PL_GWS ||
+            old_mem->mem_type == AMDGPU_PL_OA ||
+            new_mem->mem_type == AMDGPU_PL_GDS ||
+            new_mem->mem_type == AMDGPU_PL_GWS ||
+            new_mem->mem_type == AMDGPU_PL_OA) {
+                /* Nothing to save here */
+                amdgpu_move_null(bo, new_mem);
+                return 0;
+        }
 
         if (!adev->mman.buffer_funcs_enabled)
                 goto memcpy;
@@ -1082,42 +1103,48 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
         struct ttm_mem_reg tmp;
         struct ttm_placement placement;
         struct ttm_place placements;
-        uint64_t flags;
+        uint64_t addr, flags;
         int r;
 
-        if (bo->mem.mem_type != TTM_PL_TT ||
-            amdgpu_gtt_mgr_has_gart_addr(&bo->mem))
+        if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET)
                 return 0;
 
-        /* allocate GTT space */
-        tmp = bo->mem;
-        tmp.mm_node = NULL;
-        placement.num_placement = 1;
-        placement.placement = &placements;
-        placement.num_busy_placement = 1;
-        placement.busy_placement = &placements;
-        placements.fpfn = 0;
-        placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
-        placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
-                TTM_PL_FLAG_TT;
+        addr = amdgpu_gmc_agp_addr(bo);
+        if (addr != AMDGPU_BO_INVALID_OFFSET) {
+                bo->mem.start = addr >> PAGE_SHIFT;
+        } else {
 
-        r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
-        if (unlikely(r))
-                return r;
+                /* allocate GART space */
+                tmp = bo->mem;
+                tmp.mm_node = NULL;
+                placement.num_placement = 1;
+                placement.placement = &placements;
+                placement.num_busy_placement = 1;
+                placement.busy_placement = &placements;
+                placements.fpfn = 0;
+                placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
+                placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
+                        TTM_PL_FLAG_TT;
+
+                r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
+                if (unlikely(r))
+                        return r;
 
-        /* compute PTE flags for this buffer object */
-        flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
+                /* compute PTE flags for this buffer object */
+                flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
 
-        /* Bind pages */
-        gtt->offset = (u64)tmp.start << PAGE_SHIFT;
-        r = amdgpu_ttm_gart_bind(adev, bo, flags);
-        if (unlikely(r)) {
-                ttm_bo_mem_put(bo, &tmp);
-                return r;
+                /* Bind pages */
+                gtt->offset = (u64)tmp.start << PAGE_SHIFT;
+                r = amdgpu_ttm_gart_bind(adev, bo, flags);
+                if (unlikely(r)) {
+                        ttm_bo_mem_put(bo, &tmp);
+                        return r;
+                }
+
+                ttm_bo_mem_put(bo, &bo->mem);
+                bo->mem = tmp;
         }
 
-        ttm_bo_mem_put(bo, &bo->mem);
-        bo->mem = tmp;
         bo->offset = (bo->mem.start << PAGE_SHIFT) +
                 bo->bdev->man[bo->mem.mem_type].gpu_offset;
 
@@ -1427,13 +1454,14 @@ bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
 }
 
 /**
- * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
+ *