Merge airlied/drm-next into drm-misc-next

Just catching up with upstream.

Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>

87 files changed, 3549 insertions, 1904 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 658bac0cdc5e..25a95c95df14 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -133,5 +133,3 @@ include $(FULL_AMD_PATH)/powerplay/Makefile
 amdgpu-y += $(AMD_POWERPLAY_FILES)
 
 obj-$(CONFIG_DRM_AMDGPU)+= amdgpu.o
-
-CFLAGS_amdgpu_trace_points.o := -I$(src)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index a5427cf4b19d..ebfc267467ee 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -65,6 +65,7 @@
 #include "amdgpu_uvd.h"
 #include "amdgpu_vce.h"
 #include "amdgpu_vcn.h"
+#include "amdgpu_mn.h"
 
 #include "gpu_scheduler.h"
 #include "amdgpu_virt.h"
@@ -76,7 +77,7 @@
 extern int amdgpu_modeset;
 extern int amdgpu_vram_limit;
 extern int amdgpu_vis_vram_limit;
-extern unsigned amdgpu_gart_size;
+extern int amdgpu_gart_size;
 extern int amdgpu_gtt_size;
 extern int amdgpu_moverate;
 extern int amdgpu_benchmarking;
@@ -91,11 +92,12 @@ extern int amdgpu_dpm;
 extern int amdgpu_fw_load_type;
 extern int amdgpu_aspm;
 extern int amdgpu_runtime_pm;
-extern unsigned amdgpu_ip_block_mask;
+extern uint amdgpu_ip_block_mask;
 extern int amdgpu_bapm;
 extern int amdgpu_deep_color;
 extern int amdgpu_vm_size;
 extern int amdgpu_vm_block_size;
+extern int amdgpu_vm_fragment_size;
 extern int amdgpu_vm_fault_stop;
 extern int amdgpu_vm_debug;
 extern int amdgpu_vm_update_mode;
@@ -103,14 +105,14 @@ extern int amdgpu_sched_jobs;
 extern int amdgpu_sched_hw_submission;
 extern int amdgpu_no_evict;
 extern int amdgpu_direct_gma_size;
-extern unsigned amdgpu_pcie_gen_cap;
-extern unsigned amdgpu_pcie_lane_cap;
-extern unsigned amdgpu_cg_mask;
-extern unsigned amdgpu_pg_mask;
-extern unsigned amdgpu_sdma_phase_quantum;
+extern uint amdgpu_pcie_gen_cap;
+extern uint amdgpu_pcie_lane_cap;
+extern uint amdgpu_cg_mask;
+extern uint amdgpu_pg_mask;
+extern uint amdgpu_sdma_phase_quantum;
 extern char *amdgpu_disable_cu;
 extern char *amdgpu_virtual_display;
-extern unsigned amdgpu_pp_feature_mask;
+extern uint amdgpu_pp_feature_mask;
 extern int amdgpu_vram_page_split;
 extern int amdgpu_ngg;
 extern int amdgpu_prim_buf_per_se;
@@ -177,6 +179,7 @@ struct amdgpu_cs_parser;
 struct amdgpu_job;
 struct amdgpu_irq_src;
 struct amdgpu_fpriv;
+struct amdgpu_bo_va_mapping;
 
 enum amdgpu_cp_irq {
         AMDGPU_CP_IRQ_GFX_EOP = 0,
@@ -291,14 +294,25 @@ struct amdgpu_buffer_funcs {
 
 /* provided by hw blocks that can write ptes, e.g., sdma */
 struct amdgpu_vm_pte_funcs {
+        /* number of dw to reserve per operation */
+        unsigned        copy_pte_num_dw;
+
         /* copy pte entries from GART */
         void (*copy_pte)(struct amdgpu_ib *ib,
                          uint64_t pe, uint64_t src,
                          unsigned count);
+
         /* write pte one entry at a time with addr mapping */
         void (*write_pte)(struct amdgpu_ib *ib, uint64_t pe,
                           uint64_t value, unsigned count,
                           uint32_t incr);
+
+        /* maximum nums of PTEs/PDEs in a single operation */
+        uint32_t        set_max_nums_pte_pde;
+
+        /* number of dw to reserve per operation */
+        unsigned        set_pte_pde_num_dw;
+
         /* for linear pte/pde updates without addr mapping */
         void (*set_pte_pde)(struct amdgpu_ib *ib,
                             uint64_t pe,
@@ -331,6 +345,7 @@ struct amdgpu_gart_funcs {
 struct amdgpu_ih_funcs {
         /* ring read/write ptr handling, called from interrupt context */
         u32 (*get_wptr)(struct amdgpu_device *adev);
+        bool (*prescreen_iv)(struct amdgpu_device *adev);
         void (*decode_iv)(struct amdgpu_device *adev,
                           struct amdgpu_iv_entry *entry);
         void (*set_rptr)(struct amdgpu_device *adev);
@@ -398,6 +413,7 @@ void amdgpu_gem_prime_unpin(struct drm_gem_object *obj);
 struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *);
 void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj);
 void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
+int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
 int amdgpu_gem_debugfs_init(struct amdgpu_device *adev);
 
 /* sub-allocation manager, it has to be protected by another lock.
@@ -454,9 +470,10 @@ struct amdgpu_sa_bo {
  */
 void amdgpu_gem_force_release(struct amdgpu_device *adev);
 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
-                                int alignment, u32 initial_domain,
-                                u64 flags, bool kernel,
-                                struct drm_gem_object **obj);
+                             int alignment, u32 initial_domain,
+                             u64 flags, bool kernel,
+                             struct reservation_object *resv,
+                             struct drm_gem_object **obj);
 
 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
                             struct drm_device *dev,
@@ -730,8 +747,8 @@ struct amdgpu_ctx_mgr {
 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id);
 int amdgpu_ctx_put(struct amdgpu_ctx *ctx);
 
-uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
-                              struct dma_fence *fence);
+int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
+                              struct dma_fence *fence, uint64_t *seq);
 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
                                    struct amdgpu_ring *ring, uint64_t seq);
 
@@ -748,6 +765,7 @@ void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
 struct amdgpu_fpriv {
         struct amdgpu_vm        vm;
         struct amdgpu_bo_va     *prt_va;
+        struct amdgpu_bo_va     *csa_va;
         struct mutex            bo_list_lock;
         struct idr              bo_list_handles;
         struct amdgpu_ctx_mgr   ctx_mgr;
@@ -1012,7 +1030,6 @@ struct amdgpu_gfx {
         /* reset mask */
         uint32_t                        grbm_soft_reset;
         uint32_t                        srbm_soft_reset;
-        bool                            in_reset;
         /* s3/s4 mask */
         bool                            in_suspend;
         /* NGG */
@@ -1054,6 +1071,7 @@ struct amdgpu_cs_parser {
         /* buffer objects */
         struct ww_acquire_ctx           ticket;
         struct amdgpu_bo_list           *bo_list;
+        struct amdgpu_mn                *mn;
         struct amdgpu_bo_list_entry     vm_pd;
         struct list_head                validated;
         struct dma_fence                *fence;
@@ -1181,6 +1199,9 @@ struct amdgpu_firmware {
 
         /* gpu info firmware data pointer */
         const struct firmware *gpu_info_fw;
+
+        void *fw_buf_ptr;
+        uint64_t fw_buf_mc;
 };
 
 /*
@@ -1195,20 +1216,6 @@ void amdgpu_benchmark(struct amdgpu_device *adev, int test_number);
 void amdgpu_test_moves(struct amdgpu_device *adev);
 
 /*
- * MMU Notifier
- */
-#if defined(CONFIG_MMU_NOTIFIER)
-int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
-void amdgpu_mn_unregister(struct amdgpu_bo *bo);
-#else
-static inline int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
-{
-        return -ENODEV;
-}
-static inline void amdgpu_mn_unregister(struct amdgpu_bo *bo) {}
-#endif
-
-/*
  * Debugfs
  */
 struct amdgpu_debugfs {
@@ -1482,9 +1489,6 @@ struct amdgpu_device {
         struct amdgpu_mman              mman;
         struct amdgpu_vram_scratch      vram_scratch;
         struct amdgpu_wb                wb;
-        atomic64_t                      vram_usage;
-        atomic64_t                      vram_vis_usage;
-        atomic64_t                      gtt_usage;
         atomic64_t                      num_bytes_moved;
         atomic64_t                      num_evictions;
         atomic64_t                      num_vram_cpu_page_faults;
@@ -1593,6 +1597,7 @@ struct amdgpu_device {
 
         /* record last mm index being written through WREG32*/
         unsigned long last_mm_index;
+        bool                            in_sriov_reset;
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
@@ -1760,6 +1765,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
 #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
 #define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev))
+#define amdgpu_ih_prescreen_iv(adev) (adev)->irq.ih_funcs->prescreen_iv((adev))
 #define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv))
 #define amdgpu_ih_set_rptr(adev) (adev)->irq.ih_funcs->set_rptr((adev))
 #define amdgpu_display_vblank_get_counter(adev, crtc) (adev)->mode_info.funcs->vblank_get_counter((adev), (crtc))
@@ -1792,18 +1798,6 @@ void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
                                   u64 num_vis_bytes);
 void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
 bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
-int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
-int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
-                                     uint32_t flags);
-bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm);
-struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm);
-bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
-                                  unsigned long end);
-bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
-                                       int *last_invalidated);
-bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
-uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
-                                 struct ttm_mem_reg *mem);
 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base);
 void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);
 void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size);
@@ -1886,10 +1880,9 @@ static inline int amdgpu_acpi_init(struct amdgpu_device *adev) { return 0; }
 static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { }
 #endif
 
-struct amdgpu_bo_va_mapping *
-amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
-                       uint64_t addr, struct amdgpu_bo **bo);
-int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser);
+int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
+                           uint64_t addr, struct amdgpu_bo **bo,
+                           struct amdgpu_bo_va_mapping **mapping);
 
 #include "amdgpu_object.h"
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index c7bcf5207d79..5432af39a674 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -28,14 +28,14 @@
 #include <linux/module.h>
 
 const struct kgd2kfd_calls *kgd2kfd;
-bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);
+bool (*kgd2kfd_init_p)(unsigned int, const struct kgd2kfd_calls**);
 
 int amdgpu_amdkfd_init(void)
 {
         int ret;
 
 #if defined(CONFIG_HSA_AMD_MODULE)
-        int (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);
+        int (*kgd2kfd_init_p)(unsigned int, const struct kgd2kfd_calls**);
 
         kgd2kfd_init_p = symbol_request(kgd2kfd_init);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index b8802a561cbd..8d689ab7e429 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -26,6 +26,7 @@
 #define AMDGPU_AMDKFD_H_INCLUDED
 
 #include <linux/types.h>
+#include <linux/mmu_context.h>
 #include <kgd_kfd_interface.h>
 
 struct amdgpu_device;
@@ -60,4 +61,19 @@ uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
 
 uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
 
+#define read_user_wptr(mmptr, wptr, dst)                                \
+        ({                                                              \
+                bool valid = false;                                     \
+                if ((mmptr) && (wptr)) {                                \
+                        if ((mmptr) == current->mm) {                   \
+                                valid = !get_user((dst), (wptr));       \
+                        } else if (current->mm == NULL) {               \
+                                use_mm(mmptr);                          \
+                                valid = !get_user((dst), (wptr));       \
+                                unuse_mm(mmptr);                        \
+                        }                                               \
+                }                                                       \
+                valid;                                                  \
+        })
+
 #endif /* AMDGPU_AMDKFD_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index 5254562fd0f9..dc7e25cce741 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -39,6 +39,12 @@
 #include "gmc/gmc_7_1_sh_mask.h"
 #include "cik_structs.h"
 
+enum hqd_dequeue_request_type {
+        NO_ACTION = 0,
+        DRAIN_PIPE,
+        RESET_WAVES
+};
+
 enum {
         MAX_TRAPID = 8,         /* 3 bits in the bitfield. */
         MAX_WATCH_ADDRESSES = 4
@@ -96,12 +102,15 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
                                 uint32_t hpd_size, uint64_t hpd_gpu_addr);
 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
-                        uint32_t queue_id, uint32_t __user *wptr);
+                        uint32_t queue_id, uint32_t __user *wptr,
+                        uint32_t wptr_shift, uint32_t wptr_mask,
+                        struct mm_struct *mm);
 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
                                 uint32_t pipe_id, uint32_t queue_id);
 
-static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+                                enum kfd_preempt_type reset_type,
                                 unsigned int utimeout, uint32_t pipe_id,
                                 uint32_t queue_id);
 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
@@ -126,6 +135,33 @@ static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
 
 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
+static void set_scratch_backing_va(struct kgd_dev *kgd,
+                                        uint64_t va, uint32_t vmid);
+
+/* Because of REG_GET_FIELD() being used, we put this function in the
+ * asic specific file.
+ */
+static int get_tile_config(struct kgd_dev *kgd,
+                struct tile_config *config)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+        config->gb_addr_config = adev->gfx.config.gb_addr_config;
+        config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+                                MC_ARB_RAMCFG, NOOFBANK);
+        config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+                                MC_ARB_RAMCFG, NOOFRANKS);
+
+        config->tile_config_ptr = adev->gfx.config.tile_mode_array;
+        config->num_tile_configs =
+                        ARRAY_SIZE(adev->gfx.config.tile_mode_array);
+        config->macro_tile_config_ptr =
+                        adev->gfx.config.macrotile_mode_array;
+        config->num_macro_tile_configs =
+                        ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
+
+        return 0;
+}
 
 static const struct kfd2kgd_calls kfd2kgd = {
         .init_gtt_mem_allocation = alloc_gtt_mem,
@@ -133,6 +169,8 @@ static const struct kfd2kgd_calls kfd2kgd = {
         .get_vmem_size = get_vmem_size,
         .get_gpu_clock_counter = get_gpu_clock_counter,
         .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+        .alloc_pasid = amdgpu_vm_alloc_pasid,
+        .free_pasid = amdgpu_vm_free_pasid,
         .program_sh_mem_settings = kgd_program_sh_mem_settings,
         .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
         .init_pipeline = kgd_init_pipeline,
@@ -150,7 +188,9 @@ static const struct kfd2kgd_calls kfd2kgd = {
         .get_atc_vmid_pasid_mapping_pasid = get_atc_vmid_pasid_mapping_pasid,
         .get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid,
         .write_vmid_invalidate_request = write_vmid_invalidate_request,
-        .get_fw_version = get_fw_version
+        .get_fw_version = get_fw_version,
+        .set_scratch_backing_va = set_scratch_backing_va,
+        .get_tile_config = get_tile_config,
 };
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void)
@@ -186,7 +226,7 @@ static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
 {
         struct amdgpu_device *adev = get_amdgpu_device(kgd);
 
-        uint32_t mec = (++pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+        uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
         uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
 
         lock_srbm(kgd, mec, pipe, queue_id, 0);
@@ -290,20 +330,38 @@ static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
 }
 
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
-                        uint32_t queue_id, uint32_t __user *wptr)
+                        uint32_t queue_id, uint32_t __user *wptr,
+                        uint32_t wptr_shift, uint32_t wptr_mask,
+                        struct mm_struct *mm)
 {
         struct amdgpu_device *adev = get_amdgpu_device(kgd);
-        uint32_t wptr_shadow, is_wptr_shadow_valid;
         struct cik_mqd *m;
+        uint32_t *mqd_hqd;
+        uint32_t reg, wptr_val, data;
 
         m = get_mqd(mqd);
 
-        is_wptr_shadow_valid = !get_user(wptr_shadow, wptr);
-        if (is_wptr_shadow_valid)
-                m->cp_hqd_pq_wptr = wptr_shadow;
-
         acquire_queue(kgd, pipe_id, queue_id);
-        gfx_v7_0_mqd_commit(adev, m);
+
+        /* HQD registers extend from CP_MQD_BASE_ADDR to CP_MQD_CONTROL. */
+        mqd_hqd = &m->cp_mqd_base_addr_lo;
+
+        for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++)
+                WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]);
+
+        /* Copy userspace write pointer value to register.
+         * Activate doorbell logic to monitor subsequent changes.
+         */
+        data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
+                             CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
+        WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
+
+        if (read_user_wptr(mm, wptr, wptr_val))
+                WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
+
+        data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
+        WREG32(mmCP_HQD_ACTIVE, data);
+
         release_queue(kgd);
 
         return 0;
@@ -382,30 +440,99 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
         return false;
 }
 
-static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+                                enum kfd_preempt_type reset_type,
                                 unsigned int utimeout, uint32_t pipe_id,
                                 uint32_t queue_id)
 {
         struct amdgpu_device *adev = get_amdgpu_device(kgd);
         uint32_t temp;
-        int timeout = utimeout;
+        enum hqd_dequeue_request_type type;
+        unsigned long flags, end_jiffies;
+        int retry;
 
         acquire_queue(kgd, pipe_id, queue_id);
         WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
 
-        WREG32(mmCP_HQD_DEQUEUE_REQUEST, reset_type);
+        switch (reset_type) {
+        case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
+                type = DRAIN_PIPE;
+                break;
+        case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
+                type = RESET_WAVES;
+                break;
+        default:
+                type = DRAIN_PIPE;
+                break;
+        }
+
+        /* Workaround: If IQ timer is active and the wait time is close to or
+         * equal to 0, dequeueing is not safe. Wait until either the wait time
+         * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is
+         * cleared before continuing. Also, ensure wait times are set to at
+         * least 0x3.
+         */
+        local_irq_save(flags);
+        preempt_disable();
+        retry = 5000; /* wait for 500 usecs at maximum */
+        while (true) {
+                temp = RREG32(mmCP_HQD_IQ_TIMER);
+                if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) {
+                        pr_debug("HW is processing IQ\n");
+                        goto loop;
+                }
+                if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) {
+                        if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE)
+                                        == 3) /* SEM-rearm is safe */
+                                break;
+                        /* Wait time 3 is safe for CP, but our MMIO read/write
+                         * time is close to 1 microsecond, so check for 10 to
+                         * leave more buffer room
+                         */
+                        if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME)
+                                        >= 10)
+                                break;
+                        pr_debug("IQ timer is active\n");
+                } else
+                        break;
+loop:
+                if (!retry) {
+                        pr_err("CP HQD IQ timer status time out\n");
+                        break;
+                }
+                ndelay(100);
+                --retry;
+        }
+        retry = 1000;
+        while (true) {
+                temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
+                if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK))
+                        break;
+                pr_debug("Dequeue request is pending\n");
 
+                if (!retry) {
+                        pr_err("CP HQD dequeue request time out\n");
+                        break;
+                }
+                ndelay(100);
+                --retry;
+        }
+        local_irq_restore(flags);
+        preempt_enable();
+
+        WREG32(mmCP_HQD_DEQUEUE_REQUEST, type);
+
+        end_jiffies = (utimeout * HZ / 1000) + jiffies;
         while (true) {
                 temp = RREG32(mmCP_HQD_ACTIVE);
-                if (temp & CP_HQD_ACTIVE__ACTIVE_MASK)
+                if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
                         break;
-                if (timeout <= 0) {
-                        pr_err("kfd: cp queue preemption time out.\n");
+                if (time_after(jiffies, end_jiffies)) {
+                        pr_err("cp queue preemption time out\n");
                         release_queue(kgd);
                         return -ETIME;
                 }
-                msleep(20);
-                timeout -= 20;
+                usleep_range(500, 1000);
         }
 
         release_queue(kgd);
@@ -556,6 +683,16 @@ static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
         WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
 }
 
+static void set_scratch_backing_va(struct kgd_dev *kgd,
+                                        uint64_t va, uint32_t vmid)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+        lock_srbm(kgd, 0, 0, 0, vmid);
+        WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va);
+        unlock_srbm(kgd);
+}
+
 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
@@ -566,42 +703,42 @@ static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
         switch (type) {
         case KGD_ENGINE_PFP:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.pfp_fw->data;
+                                                adev->gfx.pfp_fw->data;
                 break;
 
         case KGD_ENGINE_ME:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.me_fw->data;
+                                                adev->gfx.me_fw->data;
                 break;
 
         case KGD_ENGINE_CE:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.ce_fw->data;
+                                                adev->gfx.ce_fw->data;
                 break;
 
         case KGD_ENGINE_MEC1:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.mec_fw->data;
+                                                adev->gfx.mec_fw->data;
                 break;
 
         case KGD_ENGINE_MEC2:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.mec2_fw->data;
+                                                adev->gfx.mec2_fw->data;
                 break;
 
         case KGD_ENGINE_RLC:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.rlc_fw->data;
+                                                adev->gfx.rlc_fw->data;
                 break;
 
         case KGD_ENGINE_SDMA1:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->sdma.instance[0].fw->data;
+                                                adev->sdma.instance[0].fw->data;
                 break;
 
         case KGD_ENGINE_SDMA2:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->sdma.instance[1].fw->data;
+                                                adev->sdma.instance[1].fw->data;
                 break;
 
         default:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index 133d06671e46..c678c69936a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -39,6 +39,12 @@
 #include "vi_structs.h"
 #include "vid.h"
 
+enum hqd_dequeue_request_type {
+        NO_ACTION = 0,
+        DRAIN_PIPE,
+        RESET_WAVES
+};
+
 struct cik_sdma_rlc_registers;
 
 /*
@@ -55,12 +61,15 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
                 uint32_t hpd_size, uint64_t hpd_gpu_addr);
 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
-                uint32_t queue_id, uint32_t __user *wptr);
+                        uint32_t queue_id, uint32_t __user *wptr,
+                        uint32_t wptr_shift, uint32_t wptr_mask,
+                        struct mm_struct *mm);
 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
                 uint32_t pipe_id, uint32_t queue_id);
 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
-static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+                                enum kfd_preempt_type reset_type,
                                 unsigned int utimeout, uint32_t pipe_id,
                                 uint32_t queue_id);
 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
@@ -85,6 +94,33 @@ static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
                 uint8_t vmid);
 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
+static void set_scratch_backing_va(struct kgd_dev *kgd,
+                                        uint64_t va, uint32_t vmid);
+
+/* Because of REG_GET_FIELD() being used, we put this function in the
+ * asic specific file.
+ */
+static int get_tile_config(struct kgd_dev *kgd,
+                struct tile_config *config)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+        config->gb_addr_config = adev->gfx.config.gb_addr_config;
+        config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+                                MC_ARB_RAMCFG, NOOFBANK);
+        config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+                                MC_ARB_RAMCFG, NOOFRANKS);
+
+        config->tile_config_ptr = adev->gfx.config.tile_mode_array;
+        config->num_tile_configs =
+                        ARRAY_SIZE(adev->gfx.config.tile_mode_array);
+        config->macro_tile_config_ptr =
+                        adev->gfx.config.macrotile_mode_array;
+        config->num_macro_tile_configs =
+                        ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
+
+        return 0;
+}
 
 static const struct kfd2kgd_calls kfd2kgd = {
         .init_gtt_mem_allocation = alloc_gtt_mem,
@@ -92,6 +128,8 @@ static const struct kfd2kgd_calls kfd2kgd = {
         .get_vmem_size = get_vmem_size,
         .get_gpu_clock_counter = get_gpu_clock_counter,
         .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+        .alloc_pasid = amdgpu_vm_alloc_pasid,
+        .free_pasid = amdgpu_vm_free_pasid,
         .program_sh_mem_settings = kgd_program_sh_mem_settings,
         .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
         .init_pipeline = kgd_init_pipeline,
@@ -111,7 +149,9 @@ static const struct kfd2kgd_calls kfd2kgd = {
         .get_atc_vmid_pasid_mapping_valid =
                         get_atc_vmid_pasid_mapping_valid,
         .write_vmid_invalidate_request = write_vmid_invalidate_request,
-        .get_fw_version = get_fw_version
+        .get_fw_version = get_fw_version,
+        .set_scratch_backing_va = set_scratch_backing_va,
+        .get_tile_config = get_tile_config,
 };
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void)
@@ -147,7 +187,7 @@ static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
 {
         struct amdgpu_device *adev = get_amdgpu_device(kgd);
 
-        uint32_t mec = (++pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+        uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
         uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
 
         lock_srbm(kgd, mec, pipe, queue_id, 0);
@@ -216,7 +256,7 @@ static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
         uint32_t mec;
         uint32_t pipe;
 
-        mec = (++pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+        mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
         pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
 
         lock_srbm(kgd, mec, pipe, 0, 0);
@@ -244,20 +284,67 @@ static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
 }
 
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
-                        uint32_t queue_id, uint32_t __user *wptr)
+                        uint32_t queue_id, uint32_t __user *wptr,
+                        uint32_t wptr_shift, uint32_t wptr_mask,
+                        struct mm_struct *mm)
 {
-        struct vi_mqd *m;
-        uint32_t shadow_wptr, valid_wptr;
         struct amdgpu_device *adev = get_amdgpu_device(kgd);
+        struct vi_mqd *m;
+        uint32_t *mqd_hqd;
+        uint32_t reg, wptr_val, data;
 
         m = get_mqd(mqd);
 
-        valid_wptr = copy_from_user(&shadow_wptr, wptr, sizeof(shadow_wptr));
-        if (valid_wptr == 0)
-                m->cp_hqd_pq_wptr = shadow_wptr;
-
         acquire_queue(kgd, pipe_id, queue_id);
-        gfx_v8_0_mqd_commit(adev, mqd);
+
+        /* HIQ is set during driver init period with vmid set to 0*/
+        if (m->cp_hqd_vmid == 0) {
+                uint32_t value, mec, pipe;
+
+                mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+                pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+
+                pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
+                        mec, pipe, queue_id);
+                value = RREG32(mmRLC_CP_SCHEDULERS);
+                value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
+                        ((mec << 5) | (pipe << 3) | queue_id | 0x80));
+                WREG32(mmRLC_CP_SCHEDULERS, value);
+        }
+
+        /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
+        mqd_hqd = &m->cp_mqd_base_addr_lo;
+
+        for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_HQD_EOP_CONTROL; reg++)
+                WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]);
+
+        /* Tonga errata: EOP RPTR/WPTR should be left unmodified.
+         * This is safe since EOP RPTR==WPTR for any inactive HQD
+         * on ASICs that do not support context-save.
+         * EOP writes/reads can start anywhere in the ring.
+         */
+        if (get_amdgpu_device(kgd)->asic_type != CHIP_TONGA) {
+                WREG32(mmCP_HQD_EOP_RPTR, m->cp_hqd_eop_rptr);
+                WREG32(mmCP_HQD_EOP_WPTR, m->cp_hqd_eop_wptr);
+                WREG32(mmCP_HQD_EOP_WPTR_MEM, m->cp_hqd_eop_wptr_mem);
+        }
+
+        for (reg = mmCP_HQD_EOP_EVENTS; reg <= mmCP_HQD_ERROR; reg++)
+                WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]);
+
+        /* Copy userspace write pointer value to register.
+         * Activate doorbell logic to monitor subsequent changes.
+         */
+        data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
+                             CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
+        WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
+
+        if (read_user_wptr(mm, wptr, wptr_val))
+                WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
+
+        data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
+        WREG32(mmCP_HQD_ACTIVE, data);
+
         release_queue(kgd);
 
         return 0;
@@ -308,29 +395,102 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
         return false;
 }
 
-static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+                                enum kfd_preempt_type reset_type,
                                 unsigned int utimeout, uint32_t pipe_id,
                                 uint32_t queue_id)
 {
         struct amdgpu_device *adev = get_amdgpu_device(kgd);
         uint32_t temp;
-        int timeout = utimeout;
+        enum hqd_dequeue_request_type type;
+        unsigned long flags, end_jiffies;
+        int retry;
+        struct vi_mqd *m = get_mqd(mqd);
 
         acquire_queue(kgd, pipe_id, queue_id);
 
-        WREG32(mmCP_HQD_DEQUEUE_REQUEST, reset_type);
+        if (m->cp_hqd_vmid == 0)
+                WREG32_FIELD(RLC_CP_SCHEDULERS, scheduler1, 0);
 
+        switch (reset_type) {
+        case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
+                type = DRAIN_PIPE;
+                break;
+        case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
+                type = RESET_WAVES;
+                break;
+        default:
+                type = DRAIN_PIPE;
+                break;
+        }
+
+        /* Workaround: If IQ timer is active and the wait time is close to or
+         * equal to 0, dequeueing is not safe. Wait until either the wait time
+         * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is
+         * cleared before continuing. Also, ensure wait times are set to at
+         * least 0x3.
+         */
+        local_irq_save(flags);
+        preempt_disable();
+        retry = 5000; /* wait for 500 usecs at maximum */
+        while (true) {
+                temp = RREG32(mmCP_HQD_IQ_TIMER);
+                if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) {
+                        pr_debug("HW is processing IQ\n");
+                        goto loop;
+                }
+                if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) {
+                        if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE)
+                                        == 3) /* SEM-rearm is safe */
+                                break;
+                        /* Wait time 3 is safe for CP, but our MMIO read/write
+                         * time is close to 1 microsecond, so check for 10 to
+                         * leave more buffer room
+                         */
+                        if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME)
+                                        >= 10)
+                                break;
+                        pr_debug("IQ timer is active\n");
+                } else
+                        break;
+loop:
+                if (!retry) {
+                        pr_err("CP HQD IQ timer status time out\n");
+                        break;
+                }
+                ndelay(100);
+                --retry;
+        }
+        retry = 1000;
+        while (true) {
+                temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
+                if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK))
+                        break;
+                pr_debug("Dequeue request is pending\n");
+
+                if (!retry) {
+                        pr_err("CP HQD dequeue request time out\n");
+                        break;
+                }
+                ndelay(100);
+                --retry;
+        }
+        local_irq_restore(flags);
+        preempt_enable();
+
+        WREG32(mmCP_HQD_DEQUEUE_REQUEST, type);
+
+        end_jiffies = (utimeout * HZ / 1000) + jiffies;
         while (true) {
                 temp = RREG32(mmCP_HQD_ACTIVE);
-                if (temp & CP_HQD_ACTIVE__ACTIVE_MASK)
+                if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
                         break;
-                if (timeout <= 0) {
-                        pr_err("kfd: cp queue preemption time out.\n");
+                if (time_after(jiffies, end_jiffies)) {
+                        pr_err("cp queue preemption time out.\n");
                         release_queue(kgd);
                         return -ETIME;
                 }
-                msleep(20);
-                timeout -= 20;
+                usleep_range(500, 1000);
         }
 
         release_queue(kgd);
@@ -444,6 +604,16 @@ static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
         return 0;
 }
 
+static void set_scratch_backing_va(struct kgd_dev *kgd,
+                                        uint64_t va, uint32_t vmid)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+        lock_srbm(kgd, 0, 0, 0, vmid);
+        WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va);
+        unlock_srbm(kgd);
+}
+
 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
@@ -454,42 +624,42 @@ static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
         switch (type) {
         case KGD_ENGINE_PFP:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.pfp_fw->data;
+                                                adev->gfx.pfp_fw->data;
                 break;
 
         case KGD_ENGINE_ME:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.me_fw->data;
+                                                adev->gfx.me_fw->data;
                 break;
 
         case KGD_ENGINE_CE:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.ce_fw->data;
+                                                adev->gfx.ce_fw->data;
                 break;
 
         case KGD_ENGINE_MEC1:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.mec_fw->data;
+                                                adev->gfx.mec_fw->data;
                 break;
 
         case KGD_ENGINE_MEC2:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.mec2_fw->data;
+                                                adev->gfx.mec2_fw->data;
                 break;
 
         case KGD_ENGINE_RLC:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->gfx.rlc_fw->data;
+                                                adev->gfx.rlc_fw->data;
                 break;
 
         case KGD_ENGINE_SDMA1:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->sdma.instance[0].fw->data;
+                                                adev->sdma.instance[0].fw->data;
                 break;
 
         case KGD_ENGINE_SDMA2:
                 hdr = (const union amdgpu_firmware_header *)
-                                                        adev->sdma.instance[1].fw->data;
+                                                adev->sdma.instance[1].fw->data;
                 break;
 
         default:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index fd435a96481c..383204e911a4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -45,7 +45,6 @@ struct amdgpu_cgs_device {
 static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
                                     enum cgs_gpu_mem_type type,
                                     uint64_t size, uint64_t align,
-                                    uint64_t min_offset, uint64_t max_offset,
                                     cgs_handle_t *handle)
 {
         CGS_FUNC_ADEV;
@@ -53,13 +52,6 @@ static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
         int ret = 0;
         uint32_t domain = 0;
         struct amdgpu_bo *obj;
-        struct ttm_placement placement;
-        struct ttm_place place;
-
-        if (min_offset > max_offset) {
-                BUG_ON(1);
-                return -EINVAL;
-        }
 
         /* fail if the alignment is not a power of 2 */
         if (((align != 1) && (align & (align - 1)))
@@ -73,41 +65,19 @@ static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
                 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
                         AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
                 domain = AMDGPU_GEM_DOMAIN_VRAM;
-                if (max_offset > adev->mc.real_vram_size)
-                        return -EINVAL;
-                place.fpfn = min_offset >> PAGE_SHIFT;
-                place.lpfn = max_offset >> PAGE_SHIFT;
-                place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
-                        TTM_PL_FLAG_VRAM;
                 break;
         case CGS_GPU_MEM_TYPE__INVISIBLE_CONTIG_FB:
         case CGS_GPU_MEM_TYPE__INVISIBLE_FB:
                 flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
                         AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
                 domain = AMDGPU_GEM_DOMAIN_VRAM;
-                if (adev->mc.visible_vram_size < adev->mc.real_vram_size) {
-                        place.fpfn =
-                                max(min_offset, adev->mc.visible_vram_size) >> PAGE_SHIFT;
-                        place.lpfn =
-                                min(max_offset, adev->mc.real_vram_size) >> PAGE_SHIFT;
-                        place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
-                                TTM_PL_FLAG_VRAM;
-                }
-
                 break;
         case CGS_GPU_MEM_TYPE__GART_CACHEABLE:
                 domain = AMDGPU_GEM_DOMAIN_GTT;
-                place.fpfn = min_offset >> PAGE_SHIFT;
-                place.lpfn = max_offset >> PAGE_SHIFT;
-                place.flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_TT;
                 break;
         case CGS_GPU_MEM_TYPE__GART_WRITECOMBINE:
                 flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
                 domain = AMDGPU_GEM_DOMAIN_GTT;
-                place.fpfn = min_offset >> PAGE_SHIFT;
-                place.lpfn = max_offset >> PAGE_SHIFT;
-                place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_TT |
-                        TTM_PL_FLAG_UNCACHED;
                 break;
         default:
                 return -EINVAL;
@@ -116,15 +86,8 @@ static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
 
         *handle = 0;
 
-        placement.placement = &place;
-        placement.num_placement = 1;
-        placement.busy_placement = &place;
-        placement.num_busy_placement = 1;
-
-        ret = amdgpu_bo_create_restricted(adev, size, PAGE_SIZE,
-                                          true, domain, flags,
-                                          NULL, &placement, NULL,
-                                          0, &obj);
+        ret = amdgpu_bo_create(adev, size, align, true, domain, flags,
+                               NULL, NULL, 0, &obj);
         if (ret) {
                 DRM_ERROR("(%d) bo create failed\n", ret);
                 return ret;
@@ -155,19 +118,14 @@ static int amdgpu_cgs_gmap_gpu_mem(struct cgs_device *cgs_device, cgs_handle_t h
                                    uint64_t *mcaddr)
 {
         int r;
-        u64 min_offset, max_offset;
         struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
 
         WARN_ON_ONCE(obj->placement.num_placement > 1);
 
-        min_offset = obj->placements[0].fpfn << PAGE_SHIFT;
-        max_offset = obj->placements[0].lpfn << PAGE_SHIFT;
-
         r = amdgpu_bo_reserve(obj, true);
         if (unlikely(r != 0))
                 return r;
-        r = amdgpu_bo_pin_restricted(obj, obj->preferred_domains,
-                                     min_offset, max_offset, mcaddr);
+        r = amdgpu_bo_pin(obj, obj->preferred_domains, mcaddr);
         amdgpu_bo_unreserve(obj);
         return r;
 }
@@ -675,6 +633,85 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 
                 if (!adev->pm.fw) {
                         switch (adev->asic_type) {
+                        case CHIP_TAHITI:
+                                strcpy(fw_name, "radeon/tahiti_smc.bin");
+                                break;
+                        case CHIP_PITCAIRN:
+                                if ((adev->pdev->revision == 0x81) &&
+                                    ((adev->pdev->device == 0x6810) ||
+                                    (adev->pdev->device == 0x6811))) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/pitcairn_k_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/pitcairn_smc.bin");
+                                }
+                                break;
+                        case CHIP_VERDE:
+                                if (((adev->pdev->device == 0x6820) &&
+                                        ((adev->pdev->revision == 0x81) ||
+                                        (adev->pdev->revision == 0x83))) ||
+                                    ((adev->pdev->device == 0x6821) &&
+                                        ((adev->pdev->revision == 0x83) ||
+                                        (adev->pdev->revision == 0x87))) ||
+                                    ((adev->pdev->revision == 0x87) &&
+                                        ((adev->pdev->device == 0x6823) ||
+                                        (adev->pdev->device == 0x682b)))) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/verde_k_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/verde_smc.bin");
+                                }
+                                break;
+                        case CHIP_OLAND:
+                                if (((adev->pdev->revision == 0x81) &&
+                                        ((adev->pdev->device == 0x6600) ||
+                                        (adev->pdev->device == 0x6604) ||
+                                        (adev->pdev->device == 0x6605) ||
+                                        (adev->pdev->device == 0x6610))) ||
+                                    ((adev->pdev->revision == 0x83) &&
+                                        (adev->pdev->device == 0x6610))) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/oland_k_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/oland_smc.bin");
+                                }
+                                break;
+                        case CHIP_HAINAN:
+                                if (((adev->pdev->revision == 0x81) &&
+                                        (adev->pdev->device == 0x6660)) ||
+                                    ((adev->pdev->revision == 0x83) &&
+                                        ((adev->pdev->device == 0x6660) ||
+                                        (adev->pdev->device == 0x6663) ||
+                                        (adev->pdev->device == 0x6665) ||
+                                         (adev->pdev->device == 0x6667)))) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/hainan_k_smc.bin");
+                                } else if ((adev->pdev->revision == 0xc3) &&
+                                         (adev->pdev->device == 0x6665)) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/banks_k_2_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/hainan_smc.bin");
+                                }
+                                break;
+                        case CHIP_BONAIRE:
+                                if ((adev->pdev->revision == 0x80) ||
+                                        (adev->pdev->revision == 0x81) ||
+                                        (adev->pdev->device == 0x665f)) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/bonaire_k_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/bonaire_smc.bin");
+                                }
+                                break;
+                        case CHIP_HAWAII:
+                                if (adev->pdev->revision == 0x80) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/hawaii_k_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/hawaii_smc.bin");
+                                }
+                                break;
                         case CHIP_TOPAZ:
                                 if (((adev->pdev->device == 0x6900) && (adev->pdev->revision == 0x81)) ||
                                     ((adev->pdev->device == 0x6900) && (adev->pdev->revision == 0x83)) ||
@@ -838,6 +875,9 @@ static int amdgpu_cgs_query_system_info(struct cgs_device *cgs_device,
         case CGS_SYSTEM_INFO_PCIE_SUB_SYS_VENDOR_ID:
                 sys_info->value = adev->pdev->subsystem_vendor;
                 break;
+        case CGS_SYSTEM_INFO_PCIE_BUS_DEVFN:
+                sys_info->value = adev->pdev->devfn;
+                break;
         default:
                 return -ENODEV;
         }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index 8d1cf2d3e663..f51b41f094ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -346,10 +346,8 @@ static void amdgpu_connector_free_edid(struct drm_connector *connector)
 {
         struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-        if (amdgpu_connector->edid) {
-                kfree(amdgpu_connector->edid);
-                amdgpu_connector->edid = NULL;
-        }
+        kfree(amdgpu_connector->edid);
+        amdgpu_connector->edid = NULL;
 }
 
 static int amdgpu_connector_ddc_get_modes(struct drm_connector *connector)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index c05479ec825a..c6a214f1e991 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -246,7 +246,7 @@ static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,
         }
 
         total_vram = adev->mc.real_vram_size - adev->vram_pin_size;
-        used_vram = atomic64_read(&adev->vram_usage);
+        used_vram = amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
         free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;
 
         spin_lock(&adev->mm_stats.lock);
@@ -292,7 +292,8 @@ static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,
         /* Do the same for visible VRAM if half of it is free */
         if (adev->mc.visible_vram_size < adev->mc.real_vram_size) {
                 u64 total_vis_vram = adev->mc.visible_vram_size;
-                u64 used_vis_vram = atomic64_read(&adev->vram_vis_usage);
+                u64 used_vis_vram =
+                        amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
 
                 if (used_vis_vram < total_vis_vram) {
                         u64 free_vis_vram = total_vis_vram - used_vis_vram;
@@ -472,11 +473,16 @@ static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
                         return -EPERM;
 
                 /* Check if we have user pages and nobody bound the BO already */
-                if (lobj->user_pages && bo->tbo.ttm->state != tt_bound) {
-                        size_t size = sizeof(struct page *);
-
-                        size *= bo->tbo.ttm->num_pages;
-                        memcpy(bo->tbo.ttm->pages, lobj->user_pages, size);
+                if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
+                    lobj->user_pages) {
+                        amdgpu_ttm_placement_from_domain(bo,
+                                                         AMDGPU_GEM_DOMAIN_CPU);
+                        r = ttm_bo_validate(&bo->tbo, &bo->placement, true,
+                                            false);
+                        if (r)
+                                return r;
+                        amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
+                                                     lobj->user_pages);
                         binding_userptr = true;
                 }
 
@@ -501,7 +507,6 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
         struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
         struct amdgpu_bo_list_entry *e;
         struct list_head duplicates;
-        bool need_mmap_lock = false;
         unsigned i, tries = 10;
         int r;
 
@@ -509,9 +514,9 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 
         p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
         if (p->bo_list) {
-                need_mmap_lock = p->bo_list->first_userptr !=
-                        p->bo_list->num_entries;
                 amdgpu_bo_list_get_list(p->bo_list, &p->validated);
+                if (p->bo_list->first_userptr != p->bo_list->num_entries)
+                        p->mn = amdgpu_mn_get(p->adev);
         }
 
         INIT_LIST_HEAD(&duplicates);
@@ -520,9 +525,6 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
         if (p->uf_entry.robj)
                 list_add(&p->uf_entry.tv.head, &p->validated);
 
-        if (need_mmap_lock)
-                down_read(&current->mm->mmap_sem);
-
         while (1) {
                 struct list_head need_pages;
                 unsigned i;
@@ -542,23 +544,25 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
                 INIT_LIST_HEAD(&need_pages);
                 for (i = p->bo_list->first_userptr;
                      i < p->bo_list->num_entries; ++i) {
+                        struct amdgpu_bo *bo;
 
                         e = &p->bo_list->array[i];
+                        bo = e->robj;
 
-                        if (amdgpu_ttm_tt_userptr_invalidated(e->robj->tbo.ttm,
+                        if (amdgpu_ttm_tt_userptr_invalidated(bo->tbo.ttm,
                                  &e->user_invalidated) && e->user_pages) {
 
                                 /* We acquired a page array, but somebody
                                  * invalidated it. Free it and try again
                                  */
                                 release_pages(e->user_pages,
-                                              e->robj->tbo.ttm->num_pages,
+                                              bo->tbo.ttm->num_pages,
                                               false);
                                 kvfree(e->user_pages);
                                 e->user_pages = NULL;
                         }
 
-                        if (e->robj->tbo.ttm->state != tt_bound &&
+                        if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
                             !e->user_pages) {
                                 list_del(&e->tv.head);
                                 list_add(&e->tv.head, &need_pages);
@@ -635,9 +639,6 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 
         amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
                                      p->bytes_moved_vis);
-        fpriv->vm.last_eviction_counter =
-                atomic64_read(&p->adev->num_evictions);
-
         if (p->bo_list) {
                 struct amdgpu_bo *gds = p->bo_list->gds_obj;
                 struct amdgpu_bo *gws = p->bo_list->gws_obj;
@@ -673,16 +674,11 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
         }
 
 error_validate:
-        if (r) {
-                amdgpu_vm_move_pt_bos_in_lru(p->adev, &fpriv->vm);
+        if (r)
                 ttm_eu_backoff_reservation(&p->ticket, &p->validated);
-        }
 
 error_free_pages:
 
-        if (need_mmap_lock)
-                up_read(&current->mm->mmap_sem);
-
         if (p->bo_list) {
                 for (i = p->bo_list->first_userptr;
                      i < p->bo_list->num_entries; ++i) {
@@ -724,21 +720,14 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
  * If error is set than unvalidate buffer, otherwise just free memory
  * used by parsing context.
  **/
-static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)
+static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
+                                  bool backoff)
 {
-        struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
         unsigned i;
 
-        if (!error) {
-                amdgpu_vm_move_pt_bos_in_lru(parser->adev, &fpriv->vm);
-
-                ttm_eu_fence_buffer_objects(&parser->ticket,
-                                            &parser->validated,
-                                            parser->fence);
-        } else if (backoff) {
+        if (error && backoff)
                 ttm_eu_backoff_reservation(&parser->ticket,
                                            &parser->validated);
-        }
 
         for (i = 0; i < parser->num_post_dep_syncobjs; i++)
                 drm_syncobj_put(parser->post_dep_syncobjs[i]);
@@ -772,10 +761,6 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
         if (r)
                 return r;
 
-        r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_dir_update);
-        if (r)
-                return r;
-
         r = amdgpu_vm_clear_freed(adev, vm, NULL);
         if (r)
                 return r;
@@ -791,7 +776,8 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 
         if (amdgpu_sriov_vf(adev)) {
                 struct dma_fence *f;
-                bo_va = vm->csa_bo_va;
+
+                bo_va = fpriv->csa_va;
                 BUG_ON(!bo_va);
                 r = amdgpu_vm_bo_update(adev, bo_va, false);
                 if (r)
@@ -828,7 +814,13 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 
         }
 
-        r = amdgpu_vm_clear_invalids(adev, vm, &p->job->sync);
+        r = amdgpu_vm_handle_moved(adev, vm);
+        if (r)
+                return r;
+
+        r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update);
+        if (r)
+                return r;
 
         if (amdgpu_vm_debug && p->bo_list) {
                 /* Invalidate all BOs to test for userspace bugs */
@@ -838,7 +830,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
                         if (!bo)
                                 continue;
 
-                        amdgpu_vm_bo_invalidate(adev, bo);
+                        amdgpu_vm_bo_invalidate(adev, bo, false);
                 }
         }
 
@@ -863,7 +855,7 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
         }
 
         if (p->job->vm) {
-                p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->root.bo);
+                p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->root.base.bo);
 
                 r = amdgpu_bo_vm_update_pte(p);
                 if (r)
@@ -931,11 +923,11 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
                         uint64_t offset;
                         uint8_t *kptr;
 
-                        m = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
-                                                   &aobj);
-                        if (!aobj) {
+                        r = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
+                                                   &aobj, &m);
+                        if (r) {
                                 DRM_ERROR("IB va_start is invalid\n");
-                                return -EINVAL;
+                                return r;
                         }
 
                         if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
@@ -1038,7 +1030,7 @@ static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p,
 {
         int r;
         struct dma_fence *fence;
-        r = drm_syncobj_fence_get(p->filp, handle, &fence);
+        r = drm_syncobj_find_fence(p->filp, handle, &fence);
         if (r)
                 return r;
 
@@ -1082,6 +1074,9 @@ static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p,
                                              GFP_KERNEL);
         p->num_post_dep_syncobjs = 0;
 
+        if (!p->post_dep_syncobjs)
+                return -ENOMEM;
+
         for (i = 0; i < num_deps; ++i) {
                 p->post_dep_syncobjs[i] = drm_syncobj_find(p->filp, deps[i].handle);
                 if (!p->post_dep_syncobjs[i])
@@ -1133,14 +1128,31 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
         struct amdgpu_ring *ring = p->job->ring;
         struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
         struct amdgpu_job *job;
+        unsigned i;
+        uint64_t seq;
+
         int r;
 
+        amdgpu_mn_lock(p->mn);
+        if (p->bo_list) {
+                for (i = p->bo_list->first_userptr;
+                     i < p->bo_list->num_entries; ++i) {
+                        struct amdgpu_bo *bo = p->bo_list->array[i].robj;
+
+                        if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm)) {
+                                amdgpu_mn_unlock(p->mn);
+                                return -ERESTARTSYS;
+                        }
+                }
+        }
+
         job = p->job;
         p->job = NULL;
 
         r = amd_sched_job_init(&job->base, &ring->sched, entity, p->filp);
         if (r) {
                 amdgpu_job_free(job);
+                amdgpu_mn_unlock(p->mn);
                 return r;
         }
 
@@ -1148,15 +1160,28 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
         job->fence_ctx = entity->fence_context;
         p->fence = dma_fence_get(&job->base.s_fence->finished);
 
+        r = amdgpu_ctx_add_fence(p->ctx, ring, p->fence, &seq);
+        if (r) {
+                dma_fence_put(p->fence);
+                dma_fence_put(&job->base.s_fence->finished);
+                amdgpu_job_free(job);
+                amdgpu_mn_unlock(p->mn);
+                return r;
+        }
+
         amdgpu_cs_post_dependencies(p);
 
-        cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence);
-        job->uf_sequence = cs->out.handle;
+        cs->out.handle = seq;
+        job->uf_sequence = seq;
+
         amdgpu_job_free_resources(job);
-        amdgpu_cs_parser_fini(p, 0, true);
 
         trace_amdgpu_cs_ioctl(job);
         amd_sched_entity_push_job(&job->base);
+
+        ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
+        amdgpu_mn_unlock(p->mn);
+
         return 0;
 }
 
@@ -1211,10 +1236,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                 goto out;
 
         r = amdgpu_cs_submit(&parser, cs);
-        if (r)
-                goto out;
 
-        return 0;
 out:
         amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
         return r;
@@ -1387,6 +1409,7 @@ static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,
                         array[i] = fence;
                 } else { /* NULL, the fence has been already signaled */
                         r = 1;
+                        first = i;
                         goto out;
                 }
         }
@@ -1466,78 +1489,36 @@ err_free_fences:
  * virtual memory address. Returns allocation structure when found, NULL
  * otherwise.
  */
-struct amdgpu_bo_va_mapping *
-amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
-                       uint64_t addr, struct amdgpu_bo **bo)
+int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
+                           uint64_t addr, struct amdgpu_bo **bo,
+                           struct amdgpu_bo_va_mapping **map)
 {
+        struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
+        struct amdgpu_vm *vm = &fpriv->vm;
         struct amdgpu_bo_va_mapping *mapping;
-        unsigned i;
-
-        if (!parser->bo_list)
-                return NULL;
+        int r;
 
         addr /= AMDGPU_GPU_PAGE_SIZE;
 
-        for (i = 0; i < parser->bo_list->num_entries; i++) {
-                struct amdgpu_bo_list_entry *lobj;
-
-                lobj = &parser->bo_list->array[i];
-                if (!lobj->bo_va)
-                        continue;
-
-                list_for_each_entry(mapping, &lobj->bo_va->valids, list) {
-                        if (mapping->start > addr ||
-                            addr > mapping->last)
-                                continue;
-
-                        *bo = lobj->bo_va->bo;
-                        return mapping;
-                }
-
-                list_for_each_entry(mapping, &lobj->bo_va->invalids, list) {
-                        if (mapping->start > addr ||
-                            addr > mapping->last)
-                                continue;
+        mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
+        if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
+                return -EINVAL;
 
-                        *bo = lobj->bo_va->bo;
-                        return mapping;
-                }
-        }
+        *bo = mapping->bo_va->base.bo;
+        *map = mapping;
 
-        return NULL;
-}
+        /* Double check that the BO is reserved by this CS */
+        if (READ_ONCE((*bo)->tbo.resv->lock.ctx) != &parser->ticket)
+                return -EINVAL;
 
-/**
- * amdgpu_cs_sysvm_access_required - make BOs accessible by the system VM
- *
- * @parser: command submission parser context
- *
- * Helper for UVD/VCE VM emulation, make sure BOs are accessible by the system VM.
- */
-int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser)
-{
-        unsigned i;
-        int r;
+        r = amdgpu_ttm_bind(&(*bo)->tbo, &(*bo)->tbo.mem);
+        if (unlikely(r))
+                return r;
 
-        if (!parser->bo_list)
+        if ((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
                 return 0;
 
-        for (i = 0; i < parser->bo_list->num_entries; i++) {
-                struct amdgpu_bo *bo = parser->bo_list->array[i].robj;
-
-                r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
-                if (unlikely(r))
-                        return r;
-
-                if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
-                        continue;
-
-                bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
-                amdgpu_ttm_placement_from_domain(bo, bo->allowed_domains);
-                r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
-                if (unlikely(r))
-                        return r;
-        }
-
-        return 0;
+        (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+        amdgpu_ttm_placement_from_domain(*bo, (*bo)->allowed_domains);
+        return ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, false, false);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index a11e44340b23..75c933b1a432 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -246,8 +246,8 @@ int amdgpu_ctx_put(struct amdgpu_ctx *ctx)
         return 0;
 }
 
-uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
-                              struct dma_fence *fence)
+int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
+                              struct dma_fence *fence, uint64_t* handler)
 {
         struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx];
         uint64_t seq = cring->sequence;
@@ -258,9 +258,9 @@ uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
         other = cring->fences[idx];
         if (other) {
                 signed long r;
-                r = dma_fence_wait_timeout(other, false, MAX_SCHEDULE_TIMEOUT);
+                r = dma_fence_wait_timeout(other, true, MAX_SCHEDULE_TIMEOUT);
                 if (r < 0)
-                        DRM_ERROR("Error (%ld) waiting for fence!\n", r);
+                        return r;
         }
 
         dma_fence_get(fence);
@@ -271,8 +271,10 @@ uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
         spin_unlock(&ctx->ring_lock);
 
         dma_fence_put(other);
+        if (handler)
+                *handler = seq;
 
-        return seq;
+        return 0;
 }
 
 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index a6f6cb0f2e02..3e84ddf9e3b5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -65,6 +65,7 @@ MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev);
+static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev);
 
 static const char *amdgpu_asic_name[] = {
         "TAHITI",
@@ -402,6 +403,15 @@ void amdgpu_pci_config_reset(struct amdgpu_device *adev)
  */
 static int amdgpu_doorbell_init(struct amdgpu_device *adev)
 {
+        /* No doorbell on SI hardware generation */
+        if (adev->asic_type < CHIP_BONAIRE) {
+                adev->doorbell.base = 0;
+                adev->doorbell.size = 0;
+                adev->doorbell.num_doorbells = 0;
+                adev->doorbell.ptr = NULL;
+                return 0;
+        }
+
         /* doorbell bar mapping */
         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
@@ -887,6 +897,20 @@ static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
         return r;
 }
 
+static ssize_t amdgpu_atombios_get_vbios_version(struct device *dev,
+                                                 struct device_attribute *attr,
+                                                 char *buf)
+{
+        struct drm_device *ddev = dev_get_drvdata(dev);
+        struct amdgpu_device *adev = ddev->dev_private;
+        struct atom_context *ctx = adev->mode_info.atom_context;
+
+        return snprintf(buf, PAGE_SIZE, "%s\n", ctx->vbios_version);
+}
+
+static DEVICE_ATTR(vbios_version, 0444, amdgpu_atombios_get_vbios_version,
+                   NULL);
+
 /**
  * amdgpu_atombios_fini - free the driver info and callbacks for atombios
  *
@@ -906,6 +930,7 @@ static void amdgpu_atombios_fini(struct amdgpu_device *adev)
         adev->mode_info.atom_context = NULL;
         kfree(adev->mode_info.atom_card_info);
         adev->mode_info.atom_card_info = NULL;
+        device_remove_file(adev->dev, &dev_attr_vbios_version);
 }
 
 /**
@@ -922,6 +947,7 @@ static int amdgpu_atombios_init(struct amdgpu_device *adev)
 {
         struct card_info *atom_card_info =
             kzalloc(sizeof(struct card_info), GFP_KERNEL);
+        int ret;
 
         if (!atom_card_info)
                 return -ENOMEM;
@@ -958,6 +984,13 @@ static int amdgpu_atombios_init(struct amdgpu_device *adev)
                 amdgpu_atombios_scratch_regs_init(adev);
                 amdgpu_atombios_allocate_fb_scratch(adev);
         }
+
+        ret = device_create_file(adev->dev, &dev_attr_vbios_version);
+        if (ret) {
+                DRM_ERROR("Failed to create device file for VBIOS version\n");
+                return ret;
+        }
+
         return 0;
 }
 
@@ -1062,11 +1095,11 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
         }
 
-        if (amdgpu_gart_size < 32) {
+        if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
                 /* gart size must be greater or equal to 32M */
                 dev_warn(adev->dev, "gart size (%d) too small\n",
                          amdgpu_gart_size);
-                amdgpu_gart_size = 32;
+                amdgpu_gart_size = -1;
         }
 
         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
@@ -1076,6 +1109,13 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
                 amdgpu_gtt_size = -1;
         }
 
+        /* valid range is between 4 and 9 inclusive */
+        if (amdgpu_vm_fragment_size != -1 &&
+            (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
+                dev_warn(adev->dev, "valid range is between 4 and 9\n");
+                amdgpu_vm_fragment_size = -1;
+        }
+
         amdgpu_check_vm_size(adev);
 
         amdgpu_check_block_size(adev);
@@ -1750,10 +1790,8 @@ static int amdgpu_fini(struct amdgpu_device *adev)
                 adev->ip_blocks[i].status.late_initialized = false;
         }
 
-        if (amdgpu_sriov_vf(adev)) {
-                amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
+        if (amdgpu_sriov_vf(adev))
                 amdgpu_virt_release_full_gpu(adev, false);
-        }
 
         return 0;
 }
@@ -2044,9 +2082,8 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
 
-        if (adev->asic_type >= CHIP_BONAIRE)
-                /* doorbell bar mapping */
-                amdgpu_doorbell_init(adev);
+        /* doorbell bar mapping */
+        amdgpu_doorbell_init(adev);
 
         /* io port mapping */
         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
@@ -2194,6 +2231,10 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         if (r)
                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
 
+        r = amdgpu_debugfs_vbios_dump_init(adev);
+        if (r)
+                DRM_ERROR("Creating vbios dump debugfs failed (%d).\n", r);
+
         if ((amdgpu_testing & 1)) {
                 if (adev->accel_working)
                         amdgpu_test_moves(adev);
@@ -2269,8 +2310,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
         adev->rio_mem = NULL;
         iounmap(adev->rmmio);
         adev->rmmio = NULL;
-        if (adev->asic_type >= CHIP_BONAIRE)
-                amdgpu_doorbell_fini(adev);
+        amdgpu_doorbell_fini(adev);
         amdgpu_debugfs_regs_cleanup(adev);
 }
 
@@ -2539,7 +2579,8 @@ static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
-                    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)) {
+                    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
+                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
                         if (adev->ip_blocks[i].status.hang) {
                                 DRM_INFO("Some block need full reset!\n");
                                 return true;
@@ -2615,12 +2656,6 @@ static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
                         goto err;
                 }
 
-                r = amdgpu_ttm_bind(&bo->shadow->tbo, &bo->shadow->tbo.mem);
-                if (r) {
-                        DRM_ERROR("%p bind failed\n", bo->shadow);
-                        goto err;
-                }
-
                 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
                                                  NULL, fence, true);
                 if (r) {
@@ -2653,7 +2688,7 @@ int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
 
         mutex_lock(&adev->virt.lock_reset);
         atomic_inc(&adev->gpu_reset_counter);
-        adev->gfx.in_reset = true;
+        adev->in_sriov_reset = true;
 
         /* block TTM */
         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
@@ -2764,7 +2799,7 @@ give_up_reset:
                 dev_info(adev->dev, "GPU reset successed!\n");
         }
 
-        adev->gfx.in_reset = false;
+        adev->in_sriov_reset = false;
         mutex_unlock(&adev->virt.lock_reset);
         return r;
 }
@@ -3462,10 +3497,7 @@ static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 
         valuesize = sizeof(values);
         if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
-                r = adev->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, idx, &values[0], &valuesize);
-        else if (adev->pm.funcs && adev->pm.funcs->read_sensor)
-                r = adev->pm.funcs->read_sensor(adev, idx, &values[0],
-                                                &valuesize);
+                r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
         else
                 return -EINVAL;
 
@@ -3753,6 +3785,28 @@ int amdgpu_debugfs_init(struct drm_minor *minor)
 {
         return 0;
 }
+
+static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct amdgpu_device *adev = dev->dev_private;
+
+        seq_write(m, adev->bios, adev->bios_size);
+        return 0;
+}
+
+static const struct drm_info_list amdgpu_vbios_dump_list[] = {
+                {"amdgpu_vbios",
+                 amdgpu_debugfs_get_vbios_dump,
+                 0, NULL},
+};
+
+static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
+{
+        return amdgpu_debugfs_add_files(adev,
+                                        amdgpu_vbios_dump_list, 1);
+}
 #else
 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
 {
@@ -3762,5 +3816,9 @@ static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
 {
         return 0;
 }
+static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
+{
+        return 0;
+}
 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
index 1cb52fd19060..e997ebbe43ea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
@@ -960,8 +960,10 @@ u8 amdgpu_encode_pci_lane_width(u32 lanes)
 }
 
 struct amd_vce_state*
-amdgpu_get_vce_clock_state(struct amdgpu_device *adev, unsigned idx)
+amdgpu_get_vce_clock_state(void *handle, u32 idx)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
         if (idx < adev->pm.dpm.num_of_vce_states)
                 return &adev->pm.dpm.vce_states[idx];
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index 8c96a4caa715..f79f9ea58b17 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -241,177 +241,119 @@ enum amdgpu_pcie_gen {
         AMDGPU_PCIE_GEN_INVALID = 0xffff
 };
 
-struct amdgpu_dpm_funcs {
-        int (*get_temperature)(struct amdgpu_device *adev);
-        int (*pre_set_power_state)(struct amdgpu_device *adev);
-        int (*set_power_state)(struct amdgpu_device *adev);
-        void (*post_set_power_state)(struct amdgpu_device *adev);
-        void (*display_configuration_changed)(struct amdgpu_device *adev);
-        u32 (*get_sclk)(struct amdgpu_device *adev, bool low);
-        u32 (*get_mclk)(struct amdgpu_device *adev, bool low);
-        void (*print_power_state)(struct amdgpu_device *adev, struct amdgpu_ps *ps);
-        void (*debugfs_print_current_performance_level)(struct amdgpu_device *adev, struct seq_file *m);
-        int (*force_performance_level)(struct amdgpu_device *adev, enum amd_dpm_forced_level level);
-        bool (*vblank_too_short)(struct amdgpu_device *adev);
-        void (*powergate_uvd)(struct amdgpu_device *adev, bool gate);
-        void (*powergate_vce)(struct amdgpu_device *adev, bool gate);
-        void (*enable_bapm)(struct amdgpu_device *adev, bool enable);
-        void (*set_fan_control_mode)(struct amdgpu_device *adev, u32 mode);
-        u32 (*get_fan_control_mode)(struct amdgpu_device *adev);
-        int (*set_fan_speed_percent)(struct amdgpu_device *adev, u32 speed);
-        int (*get_fan_speed_percent)(struct amdgpu_device *adev, u32 *speed);
-        int (*force_clock_level)(struct amdgpu_device *adev, enum pp_clock_type type, uint32_t mask);
-        int (*print_clock_levels)(struct amdgpu_device *adev, enum pp_clock_type type, char *buf);
-        int (*get_sclk_od)(struct amdgpu_device *adev);
-        int (*set_sclk_od)(struct amdgpu_device *adev, uint32_t value);
-        int (*get_mclk_od)(struct amdgpu_device *adev);
-        int (*set_mclk_od)(struct amdgpu_device *adev, uint32_t value);
-        int (*check_state_equal)(struct amdgpu_device *adev,
-                                struct amdgpu_ps *cps,
-                                struct amdgpu_ps *rps,
-                                bool *equal);
-        int (*read_sensor)(struct amdgpu_device *adev, int idx, void *value,
-                           int *size);
-
-        struct amd_vce_state* (*get_vce_clock_state)(struct amdgpu_device *adev, unsigned idx);
-        int (*reset_power_profile_state)(struct amdgpu_device *adev,
-                        struct amd_pp_profile *request);
-        int (*get_power_profile_state)(struct amdgpu_device *adev,
-                        struct amd_pp_profile *query);
-        int (*set_power_profile_state)(struct amdgpu_device *adev,
-                        struct amd_pp_profile *request);
-        int (*switch_power_profile)(struct amdgpu_device *adev,
-                        enum amd_pp_profile_type type);
-};
+#define amdgpu_dpm_pre_set_power_state(adev) \
+                ((adev)->powerplay.pp_funcs->pre_set_power_state((adev)->powerplay.pp_handle))
+
+#define amdgpu_dpm_set_power_state(adev) \
+                ((adev)->powerplay.pp_funcs->set_power_state((adev)->powerplay.pp_handle))
+
+#define amdgpu_dpm_post_set_power_state(adev) \
+                ((adev)->powerplay.pp_funcs->post_set_power_state((adev)->powerplay.pp_handle))
+
+#define amdgpu_dpm_display_configuration_changed(adev) \
+                ((adev)->powerplay.pp_funcs->display_configuration_changed((adev)->powerplay.pp_handle))
 
-#define amdgpu_dpm_pre_set_power_state(adev) (adev)->pm.funcs->pre_set_power_state((adev))
-#define amdgpu_dpm_set_power_state(adev) (adev)->pm.funcs->set_power_state((adev))
-#define amdgpu_dpm_post_set_power_state(adev) (adev)->pm.funcs->post_set_power_state((adev))
-#define amdgpu_dpm_display_configuration_changed(adev) (adev)->pm.funcs->display_configuration_changed((adev))
-#define amdgpu_dpm_print_power_state(adev, ps) (adev)->pm.funcs->print_power_state((adev), (ps))
-#define amdgpu_dpm_vblank_too_short(adev) (adev)->pm.funcs->vblank_too_short((adev))
-#define amdgpu_dpm_enable_bapm(adev, e) (adev)->pm.funcs->enable_bapm((adev), (e))
+#define amdgpu_dpm_print_power_state(adev, ps) \
+                ((adev)->powerplay.pp_funcs->print_power_state((adev)->powerplay.pp_handle, (ps)))
+
+#define amdgpu_dpm_vblank_too_short(adev) \
+                ((adev)->powerplay.pp_funcs->vblank_too_short((adev)->powerplay.pp_handle))
+
+#define amdgpu_dpm_enable_bapm(adev, e) \
+                ((adev)->powerplay.pp_funcs->enable_bapm((adev)->powerplay.pp_handle, (e)))
 
 #define amdgpu_dpm_read_sensor(adev, idx, value, size) \
-        ((adev)->pp_enabled ? \
-                (adev)->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, (idx), (value), (size)) : \
-                (adev)->pm.funcs->read_sensor((adev), (idx), (value), (size)))
+                ((adev)->powerplay.pp_funcs->read_sensor((adev)->powerplay.pp_handle, (idx), (value), (size)))
 
 #define amdgpu_dpm_get_temperature(adev) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_temperature((adev)->powerplay.pp_handle) : \
-              (adev)->pm.funcs->get_temperature((adev)))
+                ((adev)->powerplay.pp_funcs->get_temperature((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_set_fan_control_mode(adev, m) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->set_fan_control_mode((adev)->powerplay.pp_handle, (m)) : \
-              (adev)->pm.funcs->set_fan_control_mode((adev), (m)))
+                ((adev)->powerplay.pp_funcs->set_fan_control_mode((adev)->powerplay.pp_handle, (m)))
 
 #define amdgpu_dpm_get_fan_control_mode(adev) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_fan_control_mode((adev)->powerplay.pp_handle) : \
-              (adev)->pm.funcs->get_fan_control_mode((adev)))
+                ((adev)->powerplay.pp_funcs->get_fan_control_mode((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_set_fan_speed_percent(adev, s) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->set_fan_speed_percent((adev)->powerplay.pp_handle, (s)) : \
-              (adev)->pm.funcs->set_fan_speed_percent((adev), (s)))
+                ((adev)->powerplay.pp_funcs->set_fan_speed_percent((adev)->powerplay.pp_handle, (s)))
 
 #define amdgpu_dpm_get_fan_speed_percent(adev, s) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_fan_speed_percent((adev)->powerplay.pp_handle, (s)) : \
-              (adev)->pm.funcs->get_fan_speed_percent((adev), (s)))
+                ((adev)->powerplay.pp_funcs->get_fan_speed_percent((adev)->powerplay.pp_handle, (s)))
 
 #define amdgpu_dpm_get_fan_speed_rpm(adev, s) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_fan_speed_rpm((adev)->powerplay.pp_handle, (s)) : \
-              -EINVAL)
+                ((adev)->powerplay.pp_funcs->get_fan_speed_rpm)((adev)->powerplay.pp_handle, (s))
 
 #define amdgpu_dpm_get_sclk(adev, l) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_sclk((adev)->powerplay.pp_handle, (l)) : \
-                (adev)->pm.funcs->get_sclk((adev), (l)))
+                ((adev)->powerplay.pp_funcs->get_sclk((adev)->powerplay.pp_handle, (l)))
 
 #define amdgpu_dpm_get_mclk(adev, l)  \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_mclk((adev)->powerplay.pp_handle, (l)) : \
-              (adev)->pm.funcs->get_mclk((adev), (l)))
-
+                ((adev)->powerplay.pp_funcs->get_mclk((adev)->powerplay.pp_handle, (l)))
 
 #define amdgpu_dpm_force_performance_level(adev, l) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->force_performance_level((adev)->powerplay.pp_handle, (l)) : \
-              (adev)->pm.funcs->force_performance_level((adev), (l)))
+                ((adev)->powerplay.pp_funcs->force_performance_level((adev)->powerplay.pp_handle, (l)))
 
 #define amdgpu_dpm_powergate_uvd(adev, g) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->powergate_uvd((adev)->powerplay.pp_handle, (g)) : \
-              (adev)->pm.funcs->powergate_uvd((adev), (g)))
+                ((adev)->powerplay.pp_funcs->powergate_uvd((adev)->powerplay.pp_handle, (g)))
 
 #define amdgpu_dpm_powergate_vce(adev, g) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->powergate_vce((adev)->powerplay.pp_handle, (g)) : \
-              (adev)->pm.funcs->powergate_vce((adev), (g)))
+                ((adev)->powerplay.pp_funcs->powergate_vce((adev)->powerplay.pp_handle, (g)))
 
 #define amdgpu_dpm_get_current_power_state(adev) \
-        (adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle)
+                ((adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_get_pp_num_states(adev, data) \
-        (adev)->powerplay.pp_funcs->get_pp_num_states((adev)->powerplay.pp_handle, data)
+                ((adev)->powerplay.pp_funcs->get_pp_num_states((adev)->powerplay.pp_handle, data))
 
 #define amdgpu_dpm_get_pp_table(adev, table) \
-        (adev)->powerplay.pp_funcs->get_pp_table((adev)->powerplay.pp_handle, table)
+                ((adev)->powerplay.pp_funcs->get_pp_table((adev)->powerplay.pp_handle, table))
 
 #define amdgpu_dpm_set_pp_table(adev, buf, size) \
-        (adev)->powerplay.pp_funcs->set_pp_table((adev)->powerplay.pp_handle, buf, size)
+                ((adev)->powerplay.pp_funcs->set_pp_table((adev)->powerplay.pp_handle, buf, size))
 
 #define amdgpu_dpm_print_clock_levels(adev, type, buf) \
-        (adev)->powerplay.pp_funcs->print_clock_levels((adev)->powerplay.pp_handle, type, buf)
+                ((adev)->powerplay.pp_funcs->print_clock_levels((adev)->powerplay.pp_handle, type, buf))
 
 #define amdgpu_dpm_force_clock_level(adev, type, level) \
-                (adev)->powerplay.pp_funcs->force_clock_level((adev)->powerplay.pp_handle, type, level)
+                ((adev)->powerplay.pp_funcs->force_clock_level((adev)->powerplay.pp_handle, type, level))
 
 #define amdgpu_dpm_get_sclk_od(adev) \
-        (adev)->powerplay.pp_funcs->get_sclk_od((adev)->powerplay.pp_handle)
+                ((adev)->powerplay.pp_funcs->get_sclk_od((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_set_sclk_od(adev, value) \
-        (adev)->powerplay.pp_funcs->set_sclk_od((adev)->powerplay.pp_handle, value)
+                ((adev)->powerplay.pp_funcs->set_sclk_od((adev)->powerplay.pp_handle, value))
 
 #define amdgpu_dpm_get_mclk_od(adev) \
-        ((adev)->powerplay.pp_funcs->get_mclk_od((adev)->powerplay.pp_handle))
+                ((adev)->powerplay.pp_funcs->get_mclk_od((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_set_mclk_od(adev, value) \
-        ((adev)->powerplay.pp_funcs->set_mclk_od((adev)->powerplay.pp_handle, value))
+                ((adev)->powerplay.pp_funcs->set_mclk_od((adev)->powerplay.pp_handle, value))
 
-#define amdgpu_dpm_dispatch_task(adev, event_id, input, output)         \
-        (adev)->powerplay.pp_funcs->dispatch_tasks((adev)->powerplay.pp_handle, (event_id), (input), (output))
+#define amdgpu_dpm_dispatch_task(adev, task_id, input, output)          \
+                ((adev)->powerplay.pp_funcs->dispatch_tasks)((adev)->powerplay.pp_handle, (task_id), (input), (output))
 
-#define amgdpu_dpm_check_state_equal(adev, cps, rps, equal) (adev)->pm.funcs->check_state_equal((adev), (cps),(rps),(equal))
+#define amdgpu_dpm_check_state_equal(adev, cps, rps, equal) \
+                ((adev)->powerplay.pp_funcs->check_state_equal((adev)->powerplay.pp_handle, (cps), (rps), (equal)))
 
 #define amdgpu_dpm_get_vce_clock_state(adev, i)                         \
-        ((adev)->pp_enabled ?                                           \
-         (adev)->powerplay.pp_funcs->get_vce_clock_state((adev)->powerplay.pp_handle, (i)) : \
-         (adev)->pm.funcs->get_vce_clock_state((adev), (i)))
+                ((adev)->powerplay.pp_funcs->get_vce_clock_state((adev)->powerplay.pp_handle, (i)))
 
-#define amdgpu_dpm_get_performance_level(adev) \
-        ((adev)->pp_enabled ?                                           \
-        (adev)->powerplay.pp_funcs->get_performance_level((adev)->powerplay.pp_handle) : \
-        (adev)->pm.dpm.forced_level)
+#define amdgpu_dpm_get_performance_level(adev)                          \
+                ((adev)->powerplay.pp_funcs->get_performance_level((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_reset_power_profile_state(adev, request) \
-        ((adev)->powerplay.pp_funcs->reset_power_profile_state(\
+                ((adev)->powerplay.pp_funcs->reset_power_profile_state(\
                         (adev)->powerplay.pp_handle, request))
 
 #define amdgpu_dpm_get_power_profile_state(adev, query) \
-        ((adev)->powerplay.pp_funcs->get_power_profile_state(\
+                ((adev)->powerplay.pp_funcs->get_power_profile_state(\
                         (adev)->powerplay.pp_handle, query))
 
 #define amdgpu_dpm_set_power_profile_state(adev, request) \
-        ((adev)->powerplay.pp_funcs->set_power_profile_state(\
+                ((adev)->powerplay.pp_funcs->set_power_profile_state(\
                         (adev)->powerplay.pp_handle, request))
 
 #define amdgpu_dpm_switch_power_profile(adev, type) \
-        ((adev)->powerplay.pp_funcs->switch_power_profile(\
+                ((adev)->powerplay.pp_funcs->switch_power_profile(\
                         (adev)->powerplay.pp_handle, type))
 
 struct amdgpu_dpm {
@@ -485,7 +427,6 @@ struct amdgpu_pm {
         struct amdgpu_dpm       dpm;
         const struct firmware   *fw;    /* SMC firmware */
         uint32_t                fw_version;
-        const struct amdgpu_dpm_funcs *funcs;
         uint32_t                pcie_gen_mask;
         uint32_t                pcie_mlw_mask;
         struct amd_pp_display_configuration pm_display_cfg;/* set by DAL */
@@ -551,6 +492,6 @@ u16 amdgpu_get_pcie_lane_support(struct amdgpu_device *adev,
 u8 amdgpu_encode_pci_lane_width(u32 lanes);
 
 struct amd_vce_state*
-amdgpu_get_vce_clock_state(struct amdgpu_device *adev, unsigned idx);
+amdgpu_get_vce_clock_state(void *handle, u32 idx);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 5e9ce8a29669..4f98960e47f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -68,14 +68,16 @@
  * - 3.16.0 - Add reserved vmid support
  * - 3.17.0 - Add AMDGPU_NUM_VRAM_CPU_PAGE_FAULTS.
  * - 3.18.0 - Export gpu always on cu bitmap
+ * - 3.19.0 - Add support for UVD MJPEG decode
+ * - 3.20.0 - Add support for local BOs
  */
 #define KMS_DRIVER_MAJOR        3
-#define KMS_DRIVER_MINOR        18
+#define KMS_DRIVER_MINOR        20
 #define KMS_DRIVER_PATCHLEVEL   0
 
 int amdgpu_vram_limit = 0;
 int amdgpu_vis_vram_limit = 0;
-unsigned amdgpu_gart_size = 256;
+int amdgpu_gart_size = -1; /* auto */
 int amdgpu_gtt_size = -1; /* auto */
 int amdgpu_moverate = -1; /* auto */
 int amdgpu_benchmarking = 0;
@@ -90,10 +92,11 @@ int amdgpu_dpm = -1;
 int amdgpu_fw_load_type = -1;
 int amdgpu_aspm = -1;
 int amdgpu_runtime_pm = -1;
-unsigned amdgpu_ip_block_mask = 0xffffffff;
+uint amdgpu_ip_block_mask = 0xffffffff;
 int amdgpu_bapm = -1;
 int amdgpu_deep_color = 0;
 int amdgpu_vm_size = -1;
+int amdgpu_vm_fragment_size = -1;
 int amdgpu_vm_block_size = -1;
 int amdgpu_vm_fault_stop = 0;
 int amdgpu_vm_debug = 0;
@@ -104,14 +107,14 @@ int amdgpu_sched_jobs = 32;
 int amdgpu_sched_hw_submission = 2;
 int amdgpu_no_evict = 0;
 int amdgpu_direct_gma_size = 0;
-unsigned amdgpu_pcie_gen_cap = 0;
-unsigned amdgpu_pcie_lane_cap = 0;
-unsigned amdgpu_cg_mask = 0xffffffff;
-unsigned amdgpu_pg_mask = 0xffffffff;
-unsigned amdgpu_sdma_phase_quantum = 32;
+uint amdgpu_pcie_gen_cap = 0;
+uint amdgpu_pcie_lane_cap = 0;
+uint amdgpu_cg_mask = 0xffffffff;
+uint amdgpu_pg_mask = 0xffffffff;
+uint amdgpu_sdma_phase_quantum = 32;
 char *amdgpu_disable_cu = NULL;
 char *amdgpu_virtual_display = NULL;
-unsigned amdgpu_pp_feature_mask = 0xffffffff;
+uint amdgpu_pp_feature_mask = 0xffffffff;
 int amdgpu_ngg = 0;
 int amdgpu_prim_buf_per_se = 0;
 int amdgpu_pos_buf_per_se = 0;
@@ -126,7 +129,7 @@ module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
 MODULE_PARM_DESC(vis_vramlimit, "Restrict visible VRAM for testing, in megabytes");
 module_param_named(vis_vramlimit, amdgpu_vis_vram_limit, int, 0444);
 
-MODULE_PARM_DESC(gartsize, "Size of PCIE/IGP gart to setup in megabytes (32, 64, etc.)");
+MODULE_PARM_DESC(gartsize, "Size of GART to setup in megabytes (32, 64, etc., -1=auto)");
 module_param_named(gartsize, amdgpu_gart_size, uint, 0600);
 
 MODULE_PARM_DESC(gttsize, "Size of the GTT domain in megabytes (-1 = auto)");
@@ -183,6 +186,9 @@ module_param_named(deep_color, amdgpu_deep_color, int, 0444);
 MODULE_PARM_DESC(vm_size, "VM address space size in gigabytes (default 64GB)");
 module_param_named(vm_size, amdgpu_vm_size, int, 0444);
 
+MODULE_PARM_DESC(vm_fragment_size, "VM fragment size in bits (4, 5, etc. 4 = 64K (default), Max 9 = 2M)");
+module_param_named(vm_fragment_size, amdgpu_vm_fragment_size, int, 0444);
+
 MODULE_PARM_DESC(vm_block_size, "VM page table size in bits (default depending on vm_size)");
 module_param_named(vm_block_size, amdgpu_vm_block_size, int, 0444);
 
@@ -603,6 +609,8 @@ amdgpu_pci_remove(struct pci_dev *pdev)
 
         drm_dev_unregister(dev);
         drm_dev_unref(dev);
+        pci_disable_device(pdev);
+        pci_set_drvdata(pdev, NULL);
 }
 
 static void
@@ -847,6 +855,7 @@ static struct drm_driver kms_driver = {
         .gem_prime_import_sg_table = amdgpu_gem_prime_import_sg_table,
         .gem_prime_vmap = amdgpu_gem_prime_vmap,
         .gem_prime_vunmap = amdgpu_gem_prime_vunmap,
+        .gem_prime_mmap = amdgpu_gem_prime_mmap,
 
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index 9afa9c097e1f..562930b17a6d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -149,7 +149,7 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
                                        AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
                                        AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
                                        AMDGPU_GEM_CREATE_VRAM_CLEARED,
-                                       true, &gobj);
+                                       true, NULL, &gobj);
         if (ret) {
                 pr_err("failed to allocate framebuffer (%d)\n", aligned_size);
                 return -ENOMEM;
@@ -303,10 +303,10 @@ static int amdgpu_fbdev_destroy(struct drm_device *dev, struct amdgpu_fbdev *rfb
         if (rfb->obj) {
                 amdgpufb_destroy_pinned_object(rfb->obj);
                 rfb->obj = NULL;
+                drm_framebuffer_unregister_private(&rfb->base);
+                drm_framebuffer_cleanup(&rfb->base);
         }
         drm_fb_helper_fini(&rfbdev->helper);
-        drm_framebuffer_unregister_private(&rfb->base);
-        drm_framebuffer_cleanup(&rfb->base);
 
         return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index 94c1e2e8e34c..f4370081f6e6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -57,18 +57,6 @@
  */
 
 /**
- * amdgpu_gart_set_defaults - set the default gart_size
- *
- * @adev: amdgpu_device pointer
- *
- * Set the default gart_size based on parameters and available VRAM.
- */
-void amdgpu_gart_set_defaults(struct amdgpu_device *adev)
-{
-        adev->mc.gart_size = (uint64_t)amdgpu_gart_size << 20;
-}
-
-/**
  * amdgpu_gart_table_ram_alloc - allocate system ram for gart page table
  *
  * @adev: amdgpu_device pointer
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
index d4cce6936200..afbe803b1a13 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
@@ -56,7 +56,6 @@ struct amdgpu_gart {
         const struct amdgpu_gart_funcs *gart_funcs;
 };
 
-void amdgpu_gart_set_defaults(struct amdgpu_device *adev);
 int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev);
 void amdgpu_gart_table_ram_free(struct amdgpu_device *adev);
 int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 81127ffcefb2..b0d45c8e6bb3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -44,11 +44,12 @@ void amdgpu_gem_object_free(struct drm_gem_object *gobj)
 }
 
 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
-                                int alignment, u32 initial_domain,
-                                u64 flags, bool kernel,
-                                struct drm_gem_object **obj)
+                             int alignment, u32 initial_domain,
+                             u64 flags, bool kernel,
+                             struct reservation_object *resv,
+                             struct drm_gem_object **obj)
 {
-        struct amdgpu_bo *robj;
+        struct amdgpu_bo *bo;
         int r;
 
         *obj = NULL;
@@ -59,7 +60,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 
 retry:
         r = amdgpu_bo_create(adev, size, alignment, kernel, initial_domain,
-                             flags, NULL, NULL, 0, &robj);
+                             flags, NULL, resv, 0, &bo);
         if (r) {
                 if (r != -ERESTARTSYS) {
                         if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
@@ -71,7 +72,7 @@ retry:
                 }
                 return r;
         }
-        *obj = &robj->gem_base;
+        *obj = &bo->gem_base;
 
         return 0;
 }
@@ -112,7 +113,17 @@ int amdgpu_gem_object_open(struct drm_gem_object *obj,
         struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
         struct amdgpu_vm *vm = &fpriv->vm;
         struct amdgpu_bo_va *bo_va;
+        struct mm_struct *mm;
         int r;
+
+        mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
+        if (mm && mm != current->mm)
+                return -EPERM;
+
+        if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
+            abo->tbo.resv != vm->root.base.bo->tbo.resv)
+                return -EPERM;
+
         r = amdgpu_bo_reserve(abo, false);
         if (r)
                 return r;
@@ -127,35 +138,6 @@ int amdgpu_gem_object_open(struct drm_gem_object *obj,
         return 0;
 }
 
-static int amdgpu_gem_vm_check(void *param, struct amdgpu_bo *bo)
-{
-        /* if anything is swapped out don't swap it in here,
-           just abort and wait for the next CS */
-        if (!amdgpu_bo_gpu_accessible(bo))
-                return -ERESTARTSYS;
-
-        if (bo->shadow && !amdgpu_bo_gpu_accessible(bo->shadow))
-                return -ERESTARTSYS;
-
-        return 0;
-}
-
-static bool amdgpu_gem_vm_ready(struct amdgpu_device *adev,
-                                struct amdgpu_vm *vm,
-                                struct list_head *list)
-{
-        struct ttm_validate_buffer *entry;
-
-        list_for_each_entry(entry, list, head) {
-                struct amdgpu_bo *bo =
-                        container_of(entry->bo, struct amdgpu_bo, tbo);
-                if (amdgpu_gem_vm_check(NULL, bo))
-                        return false;
-        }
-
-        return !amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_gem_vm_check, NULL);
-}
-
 void amdgpu_gem_object_close(struct drm_gem_object *obj,
                              struct drm_file *file_priv)
 {
@@ -165,13 +147,14 @@ void amdgpu_gem_object_close(struct drm_gem_object *obj,
         struct amdgpu_vm *vm = &fpriv->vm;
 
         struct amdgpu_bo_list_entry vm_pd;
-        struct list_head list;
+        struct list_head list, duplicates;
         struct ttm_validate_buffer tv;
         struct ww_acquire_ctx ticket;
         struct amdgpu_bo_va *bo_va;
         int r;
 
         INIT_LIST_HEAD(&list);
+        INIT_LIST_HEAD(&duplicates);
 
         tv.bo = &bo->tbo;
         tv.shared = true;
@@ -179,7 +162,7 @@ void amdgpu_gem_object_close(struct drm_gem_object *obj,
 
         amdgpu_vm_get_pd_bo(vm, &list, &vm_pd);
 
-        r = ttm_eu_reserve_buffers(&ticket, &list, false, NULL);
+        r = ttm_eu_reserve_buffers(&ticket, &list, false, &duplicates);
         if (r) {
                 dev_err(adev->dev, "leaking bo va because "
                         "we fail to reserve bo (%d)\n", r);
@@ -189,7 +172,7 @@ void amdgpu_gem_object_close(struct drm_gem_object *obj,
         if (bo_va && --bo_va->ref_count == 0) {
                 amdgpu_vm_bo_rmv(adev, bo_va);
 
-                if (amdgpu_gem_vm_ready(adev, vm, &list)) {
+                if (amdgpu_vm_ready(vm)) {
                         struct dma_fence *fence = NULL;
 
                         r = amdgpu_vm_clear_freed(adev, vm, &fence);
@@ -214,20 +197,22 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
                             struct drm_file *filp)
 {
         struct amdgpu_device *adev = dev->dev_private;
+        struct amdgpu_fpriv *fpriv = filp->driver_priv;
+        struct amdgpu_vm *vm = &fpriv->vm;
         union drm_amdgpu_gem_create *args = data;
+        uint64_t flags = args->in.domain_flags;
         uint64_t size = args->in.bo_size;
+        struct reservation_object *resv = NULL;
         struct drm_gem_object *gobj;
         uint32_t handle;
-        bool kernel = false;
         int r;
 
         /* reject invalid gem flags */
-        if (args->in.domain_flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-                                      AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
-                                      AMDGPU_GEM_CREATE_CPU_GTT_USWC |
-                                      AMDGPU_GEM_CREATE_VRAM_CLEARED|
-                                      AMDGPU_GEM_CREATE_SHADOW |
-                                      AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS))
+        if (flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                      AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
+                      AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+                      AMDGPU_GEM_CREATE_VRAM_CLEARED |
+                      AMDGPU_GEM_CREATE_VM_ALWAYS_VALID))
                 return -EINVAL;
 
         /* reject invalid gem domains */
@@ -242,7 +227,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
         /* create a gem object to contain this object in */
         if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
             AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
-                kernel = true;
+                flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
                 if (args->in.domains == AMDGPU_GEM_DOMAIN_GDS)
                         size = size << AMDGPU_GDS_SHIFT;
                 else if (args->in.domains == AMDGPU_GEM_DOMAIN_GWS)
@@ -254,10 +239,25 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
         }
         size = roundup(size, PAGE_SIZE);
 
+        if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
+                r = amdgpu_bo_reserve(vm->root.base.bo, false);
+                if (r)
+                        return r;
+
+                resv = vm->root.base.bo->tbo.resv;
+        }
+
         r = amdgpu_gem_object_create(adev, size, args->in.alignment,
                                      (u32)(0xffffffff & args->in.domains),
-                                     args->in.domain_flags,
-                                     kernel, &gobj);
+                                     flags, false, resv, &gobj);
+        if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
+                if (!r) {
+                        struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
+
+                        abo->parent = amdgpu_bo_ref(vm->root.base.bo);
+                }
+                amdgpu_bo_unreserve(vm->root.base.bo);
+        }
         if (r)
                 return r;
 
@@ -299,9 +299,8 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
         }
 
         /* create a gem object to contain this object in */
-        r = amdgpu_gem_object_create(adev, args->size, 0,
-                                     AMDGPU_GEM_DOMAIN_CPU, 0,
-                                     0, &gobj);
+        r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
+                                     0, 0, NULL, &gobj);
         if (r)
                 return r;
 
@@ -319,8 +318,6 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
         }
 
         if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
-                down_read(&current->mm->mmap_sem);
-
                 r = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
                                                  bo->tbo.ttm->pages);
                 if (r)
@@ -335,8 +332,6 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
                 amdgpu_bo_unreserve(bo);
                 if (r)
                         goto free_pages;
-
-                up_read(&current->mm->mmap_sem);
         }
 
         r = drm_gem_handle_create(filp, gobj, &handle);
@@ -513,10 +508,10 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
                                     struct list_head *list,
                                     uint32_t operation)
 {
-        int r = -ERESTARTSYS;
+        int r;
 
-        if (!amdgpu_gem_vm_ready(adev, vm, list))
-                goto error;
+        if (!amdgpu_vm_ready(vm))
+                return;
 
         r = amdgpu_vm_update_directories(adev, vm);
         if (r)
@@ -553,7 +548,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
         struct amdgpu_bo_list_entry vm_pd;
         struct ttm_validate_buffer tv;
         struct ww_acquire_ctx ticket;
-        struct list_head list;
+        struct list_head list, duplicates;
         uint64_t va_flags;
         int r = 0;
 
@@ -589,6 +584,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
         }
 
         INIT_LIST_HEAD(&list);
+        INIT_LIST_HEAD(&duplicates);
         if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
             !(args->flags & AMDGPU_VM_PAGE_PRT)) {
                 gobj = drm_gem_object_lookup(filp, args->handle);
@@ -605,7 +601,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 
         amdgpu_vm_get_pd_bo(&fpriv->vm, &list, &vm_pd);
 
-        r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL);
+        r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates);
         if (r)
                 goto error_unref;
 
@@ -623,7 +619,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 
         switch (args->operation) {
         case AMDGPU_VA_OP_MAP:
-                r = amdgpu_vm_alloc_pts(adev, bo_va->vm, args->va_address,
+                r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
                                         args->map_size);
                 if (r)
                         goto error_backoff;
@@ -643,7 +639,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
                                                 args->map_size);
                 break;
         case AMDGPU_VA_OP_REPLACE:
-                r = amdgpu_vm_alloc_pts(adev, bo_va->vm, args->va_address,
+                r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
                                         args->map_size);
                 if (r)
                         goto error_backoff;
@@ -671,6 +667,7 @@ error_unref:
 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
                         struct drm_file *filp)
 {
+        struct amdgpu_device *adev = dev->dev_private;
         struct drm_amdgpu_gem_op *args = data;
         struct drm_gem_object *gobj;
         struct amdgpu_bo *robj;
@@ -718,6 +715,9 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
                 if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
                         robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
 
+                if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
+                        amdgpu_vm_bo_invalidate(adev, robj, true);
+
                 amdgpu_bo_unreserve(robj);
                 break;
         default:
@@ -747,8 +747,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
         r = amdgpu_gem_object_create(adev, args->size, 0,
                                      AMDGPU_GEM_DOMAIN_VRAM,
                                      AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
-                                     ttm_bo_type_device,
-                                     &gobj);
+                                     false, NULL, &gobj);
         if (r)
                 return -ENOMEM;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 4f6c68fc1dd9..4fcd98e65998 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -260,8 +260,13 @@ int amdgpu_gfx_compute_mqd_sw_init(struct amdgpu_device *adev,
         /* create MQD for KIQ */
         ring = &adev->gfx.kiq.ring;
         if (!ring->mqd_obj) {
+                /* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must
+                 * otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD
+                 * deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for
+                 * KIQ MQD no matter SRIOV or Bare-metal
+                 */
                 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
-                                            AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
+                                            AMDGPU_GEM_DOMAIN_VRAM, &ring->mqd_obj,
                                             &ring->mqd_gpu_addr, &ring->mqd_ptr);
                 if (r) {
                         dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index 5e6b90c6794f..0d15eb7d31d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -28,7 +28,7 @@
 struct amdgpu_gtt_mgr {
         struct drm_mm mm;
         spinlock_t lock;
-        uint64_t available;
+        atomic64_t available;
 };
 
 /**
@@ -54,7 +54,7 @@ static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man,
         size = (adev->mc.gart_size >> PAGE_SHIFT) - start;
         drm_mm_init(&mgr->mm, start, size);
         spin_lock_init(&mgr->lock);
-        mgr->available = p_size;
+        atomic64_set(&mgr->available, p_size);
         man->priv = mgr;
         return 0;
 }
@@ -108,10 +108,10 @@ bool amdgpu_gtt_mgr_is_allocated(struct ttm_mem_reg *mem)
  *
  * Allocate the address space for a node.
  */
-int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
-                         struct ttm_buffer_object *tbo,
-                         const struct ttm_place *place,
-                         struct ttm_mem_reg *mem)
+static int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
+                                struct ttm_buffer_object *tbo,
+                                const struct ttm_place *place,
+                                struct ttm_mem_reg *mem)
 {
         struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
         struct amdgpu_gtt_mgr *mgr = man->priv;
@@ -143,25 +143,12 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
                                         fpfn, lpfn, mode);
         spin_unlock(&mgr->lock);
 
-        if (!r) {
+        if (!r)
                 mem->start = node->start;
-                if (&tbo->mem == mem)
-                        tbo->offset = (tbo->mem.start << PAGE_SHIFT) +
-                            tbo->bdev->man[tbo->mem.mem_type].gpu_offset;
-        }
 
         return r;
 }
 
-void amdgpu_gtt_mgr_print(struct seq_file *m, struct ttm_mem_type_manager *man)
-{
-        struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
-        struct amdgpu_gtt_mgr *mgr = man->priv;
-
-        seq_printf(m, "man size:%llu pages, gtt available:%llu pages, usage:%lluMB\n",
-                   man->size, mgr->available, (u64)atomic64_read(&adev->gtt_usage) >> 20);
-
-}
 /**
  * amdgpu_gtt_mgr_new - allocate a new node
  *
@@ -182,11 +169,11 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
         int r;
 
         spin_lock(&mgr->lock);
-        if (mgr->available < mem->num_pages) {
+        if (atomic64_read(&mgr->available) < mem->num_pages) {
                 spin_unlock(&mgr->lock);
                 return 0;
         }
-        mgr->available -= mem->num_pages;
+        atomic64_sub(mem->num_pages, &mgr->available);
         spin_unlock(&mgr->lock);
 
         node = kzalloc(sizeof(*node), GFP_KERNEL);
@@ -213,9 +200,7 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
 
         return 0;
 err_out:
-        spin_lock(&mgr->lock);
-        mgr->available += mem->num_pages;
-        spin_unlock(&mgr->lock);
+        atomic64_add(mem->num_pages, &mgr->available);
 
         return r;
 }
@@ -242,30 +227,47 @@ static void amdgpu_gtt_mgr_del(struct ttm_mem_type_manager *man,
         spin_lock(&mgr->lock);
         if (node->start != AMDGPU_BO_INVALID_OFFSET)
                 drm_mm_remove_node(node);
-        mgr->available += mem->num_pages;
         spin_unlock(&mgr->lock);
+        atomic64_add(mem->num_pages, &mgr->available);
 
         kfree(node);
         mem->mm_node = NULL;
 }
 
 /**
+ * amdgpu_gtt_mgr_usage - return usage of GTT domain
+ *
+ * @man: TTM memory type manager
+ *
+ * Return how many bytes are used in the GTT domain
+ */
+uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man)
+{
+        struct amdgpu_gtt_mgr *mgr = man->priv;
+
+        return (u64)(man->size - atomic64_read(&mgr->available)) * PAGE_SIZE;
+}
+
+/**
  * amdgpu_gtt_mgr_debug - dump VRAM table
  *
  * @man: TTM memory type manager
- * @prefix: text prefix
+ * @printer: DRM printer to use
  *
  * Dump the table content using printk.
  */
 static void amdgpu_gtt_mgr_debug(struct ttm_mem_type_manager *man,
-                                  const char *prefix)
+                                 struct drm_printer *printer)
 {
         struct amdgpu_gtt_mgr *mgr = man->priv;
-        struct drm_printer p = drm_debug_printer(prefix);
 
         spin_lock(&mgr->lock);
-        drm_mm_print(&mgr->mm, &p);
+        drm_mm_print(&mgr->mm, printer);
         spin_unlock(&mgr->lock);
+
+        drm_printf(printer, "man size:%llu pages, gtt available:%llu pages, usage:%lluMB\n",
+                   man->size, (u64)atomic64_read(&mgr->available),
+                   amdgpu_gtt_mgr_usage(man) >> 20);
 }
 
 const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
index 3ab4c65ecc8b..f5f27e4f0f7f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
@@ -169,6 +169,12 @@ restart_ih:
         while (adev->irq.ih.rptr != wptr) {
                 u32 ring_index = adev->irq.ih.rptr >> 2;
 
+                /* Prescreening of high-frequency interrupts */
+                if (!amdgpu_ih_prescreen_iv(adev)) {
+                        adev->irq.ih.rptr &= adev->irq.ih.ptr_mask;
+                        continue;
+                }
+
                 /* Before dispatching irq to IP blocks, send it to amdkfd */
                 amdgpu_amdkfd_interrupt(adev,
                                 (const void *) &adev->irq.ih.ring[ring_index]);
@@ -190,3 +196,79 @@ restart_ih:
 
         return IRQ_HANDLED;
 }
+
+/**
+ * amdgpu_ih_add_fault - Add a page fault record
+ *
+ * @adev: amdgpu device pointer
+ * @key: 64-bit encoding of PASID and address
+ *
+ * This should be called when a retry page fault interrupt is
+ * received. If this is a new page fault, it will be added to a hash
+ * table. The return value indicates whether this is a new fault, or
+ * a fault that was already known and is already being handled.
+ *
+ * If there are too many pending page faults, this will fail. Retry
+ * interrupts should be ignored in this case until there is enough
+ * free space.
+ *
+ * Returns 0 if the fault was added, 1 if the fault was already known,
+ * -ENOSPC if there are too many pending faults.
+ */
+int amdgpu_ih_add_fault(struct amdgpu_device *adev, u64 key)
+{
+        unsigned long flags;
+        int r = -ENOSPC;
+
+        if (WARN_ON_ONCE(!adev->irq.ih.faults))
+                /* Should be allocated in <IP>_ih_sw_init on GPUs that
+                 * support retry faults and require retry filtering.
+                 */
+                return r;
+
+        spin_lock_irqsave(&adev->irq.ih.faults->lock, flags);
+
+        /* Only let the hash table fill up to 50% for best performance */
+        if (adev->irq.ih.faults->count >= (1 << (AMDGPU_PAGEFAULT_HASH_BITS-1)))
+                goto unlock_out;
+
+        r = chash_table_copy_in(&adev->irq.ih.faults->hash, key, NULL);
+        if (!r)
+                adev->irq.ih.faults->count++;
+
+        /* chash_table_copy_in should never fail unless we're losing count */
+        WARN_ON_ONCE(r < 0);
+
+unlock_out:
+        spin_unlock_irqrestore(&adev->irq.ih.faults->lock, flags);
+        return r;
+}
+
+/**
+ * amdgpu_ih_clear_fault - Remove a page fault record
+ *
+ * @adev: amdgpu device pointer
+ * @key: 64-bit encoding of PASID and address
+ *
+ * This should be called when a page fault has been handled. Any
+ * future interrupt with this key will be processed as a new
+ * page fault.
+ */
+void amdgpu_ih_clear_fault(struct amdgpu_device *adev, u64 key)
+{
+        unsigned long flags;
+        int r;
+
+        if (!adev->irq.ih.faults)
+                return;
+
+        spin_lock_irqsave(&adev->irq.ih.faults->lock, flags);
+
+        r = chash_table_remove(&adev->irq.ih.faults->hash, key, NULL);
+        if (!WARN_ON_ONCE(r < 0)) {
+                adev->irq.ih.faults->count--;
+                WARN_ON_ONCE(adev->irq.ih.faults->count < 0);
+        }
+
+        spin_unlock_irqrestore(&adev->irq.ih.faults->lock, flags);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
index 3de8e74e5b3a..ada89358e220 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
@@ -24,6 +24,8 @@
 #ifndef __AMDGPU_IH_H__
 #define __AMDGPU_IH_H__
 
+#include <linux/chash.h>
+
 struct amdgpu_device;
  /*
   * vega10+ IH clients
@@ -69,6 +71,13 @@ enum amdgpu_ih_clientid
 
 #define AMDGPU_IH_CLIENTID_LEGACY 0
 
+#define AMDGPU_PAGEFAULT_HASH_BITS 8
+struct amdgpu_retryfault_hashtable {
+        DECLARE_CHASH_TABLE(hash, AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
+        spinlock_t      lock;
+        int             count;
+};
+
 /*
  * R6xx+ IH ring
  */
@@ -87,6 +96,7 @@ struct amdgpu_ih_ring {
         bool                    use_doorbell;
         bool                    use_bus_addr;
         dma_addr_t              rb_dma_addr; /* only used when use_bus_addr = true */
+        struct amdgpu_retryfault_hashtable *faults;
 };
 
 #define AMDGPU_IH_SRC_DATA_MAX_SIZE_DW 4
@@ -109,5 +119,7 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, unsigned ring_size,
                         bool use_bus_addr);
 void amdgpu_ih_ring_fini(struct amdgpu_device *adev);
 int amdgpu_ih_process(struct amdgpu_device *adev);
+int amdgpu_ih_add_fault(struct amdgpu_device *adev, u64 key);
+void amdgpu_ih_clear_fault(struct amdgpu_device *adev, u64 key);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 4bdd851f56d0..538e5f27d120 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -221,8 +221,9 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
 
         spin_lock_init(&adev->irq.lock);
 
-        /* Disable vblank irqs aggressively for power-saving */
-        adev->ddev->vblank_disable_immediate = true;
+        if (!adev->enable_virtual_display)
+                /* Disable vblank irqs aggressively for power-saving */
+                adev->ddev->vblank_disable_immediate = true;
 
         r = drm_vblank_init(adev->ddev, adev->mode_info.num_crtc);
         if (r) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index c908f972283c..4fd06f8d9768 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -455,13 +455,13 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
                 ui64 = atomic64_read(&adev->num_vram_cpu_page_faults);
                 return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
         case AMDGPU_INFO_VRAM_USAGE:
-                ui64 = atomic64_read(&adev->vram_usage);
+                ui64 = amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
                 return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
         case AMDGPU_INFO_VIS_VRAM_USAGE:
-                ui64 = atomic64_read(&adev->vram_vis_usage);
+                ui64 = amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
                 return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
         case AMDGPU_INFO_GTT_USAGE:
-                ui64 = atomic64_read(&adev->gtt_usage);
+                ui64 = amdgpu_gtt_mgr_usage(&adev->mman.bdev.man[TTM_PL_TT]);
                 return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
         case AMDGPU_INFO_GDS_CONFIG: {
                 struct drm_amdgpu_info_gds gds_info;
@@ -497,7 +497,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
                 mem.vram.total_heap_size = adev->mc.real_vram_size;
                 mem.vram.usable_heap_size =
                         adev->mc.real_vram_size - adev->vram_pin_size;
-                mem.vram.heap_usage = atomic64_read(&adev->vram_usage);
+                mem.vram.heap_usage =
+                        amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
                 mem.vram.max_allocation = mem.vram.usable_heap_size * 3 / 4;
 
                 mem.cpu_accessible_vram.total_heap_size =
@@ -506,7 +507,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
                         adev->mc.visible_vram_size -
                         (adev->vram_pin_size - adev->invisible_pin_size);
                 mem.cpu_accessible_vram.heap_usage =
-                        atomic64_read(&adev->vram_vis_usage);
+                        amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
                 mem.cpu_accessible_vram.max_allocation =
                         mem.cpu_accessible_vram.usable_heap_size * 3 / 4;
 
@@ -514,7 +515,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
                 mem.gtt.total_heap_size *= PAGE_SIZE;
                 mem.gtt.usable_heap_size = mem.gtt.total_heap_size
                         - adev->gart_pin_size;
-                mem.gtt.heap_usage = atomic64_read(&adev->gtt_usage);
+                mem.gtt.heap_usage =
+                        amdgpu_gtt_mgr_usage(&adev->mman.bdev.man[TTM_PL_TT]);
                 mem.gtt.max_allocation = mem.gtt.usable_heap_size * 3 / 4;
 
                 return copy_to_user(out, &mem,
@@ -588,11 +590,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
                 dev_info.virtual_address_offset = AMDGPU_VA_RESERVED_SIZE;
                 dev_info.virtual_address_max = (uint64_t)adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
                 dev_info.virtual_address_alignment = max((int)PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE);
-                dev_info.pte_fragment_size =
-                        (1 << AMDGPU_LOG2_PAGES_PER_FRAG(adev)) *
-                        AMDGPU_GPU_PAGE_SIZE;
+                dev_info.pte_fragment_size = (1 << adev->vm_manager.fragment_size) * AMDGPU_GPU_PAGE_SIZE;
                 dev_info.gart_page_size = AMDGPU_GPU_PAGE_SIZE;
-
                 dev_info.cu_active_number = adev->gfx.cu_info.number;
                 dev_info.cu_ao_mask = adev->gfx.cu_info.ao_cu_mask;
                 dev_info.ce_ram_size = adev->gfx.ce_ram_size;
@@ -826,7 +825,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
         }
 
         r = amdgpu_vm_init(adev, &fpriv->vm,
-                           AMDGPU_VM_CONTEXT_GFX);
+                           AMDGPU_VM_CONTEXT_GFX, 0);
         if (r) {
                 kfree(fpriv);
                 goto out_suspend;
@@ -841,9 +840,12 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
         }
 
         if (amdgpu_sriov_vf(adev)) {
-                r = amdgpu_map_static_csa(adev, &fpriv->vm);
-                if (r)
+                r = amdgpu_map_static_csa(adev, &fpriv->vm, &fpriv->csa_va);
+                if (r) {
+                        amdgpu_vm_fini(adev, &fpriv->vm);
+                        kfree(fpriv);
                         goto out_suspend;
+                }
         }
 
         mutex_init(&fpriv->bo_list_lock);
@@ -894,8 +896,8 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
         if (amdgpu_sriov_vf(adev)) {
                 /* TODO: how to handle reserve failure */
                 BUG_ON(amdgpu_bo_reserve(adev->virt.csa_obj, true));
-                amdgpu_vm_bo_rmv(adev, fpriv->vm.csa_bo_va);
-                fpriv->vm.csa_bo_va = NULL;
+                amdgpu_vm_bo_rmv(adev, fpriv->csa_va);
+                fpriv->csa_va = NULL;
                 amdgpu_bo_unreserve(adev->virt.csa_obj);
         }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
index 6558a3ed57a7..bd67f4cb8e6c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
@@ -50,8 +50,10 @@ struct amdgpu_mn {
         struct hlist_node       node;
 
         /* objects protected by lock */
-        struct mutex            lock;
-        struct rb_root          objects;
+        struct rw_semaphore     lock;
+        struct rb_root_cached   objects;
+        struct mutex            read_lock;
+        atomic_t                recursion;
 };
 
 struct amdgpu_mn_node {
@@ -74,17 +76,17 @@ static void amdgpu_mn_destroy(struct work_struct *work)
         struct amdgpu_bo *bo, *next_bo;
 
         mutex_lock(&adev->mn_lock);
-        mutex_lock(&rmn->lock);
+        down_write(&rmn->lock);
         hash_del(&rmn->node);
-        rbtree_postorder_for_each_entry_safe(node, next_node, &rmn->objects,
-                                             it.rb) {
+        rbtree_postorder_for_each_entry_safe(node, next_node,
+                                             &rmn->objects.rb_root, it.rb) {
                 list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {
                         bo->mn = NULL;
                         list_del_init(&bo->mn_list);
                 }
                 kfree(node);
         }
-        mutex_unlock(&rmn->lock);
+        up_write(&rmn->lock);
         mutex_unlock(&adev->mn_lock);
         mmu_notifier_unregister_no_release(&rmn->mn, rmn->mm);
         kfree(rmn);
@@ -106,6 +108,53 @@ static void amdgpu_mn_release(struct mmu_notifier *mn,
         schedule_work(&rmn->work);
 }
 
+
+/**
+ * amdgpu_mn_lock - take the write side lock for this mn
+ */
+void amdgpu_mn_lock(struct amdgpu_mn *mn)
+{
+        if (mn)
+                down_write(&mn->lock);
+}
+
+/**
+ * amdgpu_mn_unlock - drop the write side lock for this mn
+ */
+void amdgpu_mn_unlock(struct amdgpu_mn *mn)
+{
+        if (mn)
+                up_write(&mn->lock);
+}
+
+/**
+ * amdgpu_mn_read_lock - take the rmn read lock
+ *
+ * @rmn: our notifier
+ *
+ * Take the rmn read side lock.
+ */
+static void amdgpu_mn_read_lock(struct amdgpu_mn *rmn)
+{
+        mutex_lock(&rmn->read_lock);
+        if (atomic_inc_return(&rmn->recursion) == 1)
+                down_read_non_owner(&rmn->lock);
+        mutex_unlock(&rmn->read_lock);
+}
+
+/**
+ * amdgpu_mn_read_unlock - drop the rmn read lock
+ *
+ * @rmn: our notifier
+ *
+ * Drop the rmn read side lock.
+ */
+static void amdgpu_mn_read_unlock(struct amdgpu_mn *rmn)
+{
+        if (atomic_dec_return(&rmn->recursion) == 0)
+                up_read_non_owner(&rmn->lock);
+}
+
 /**
  * amdgpu_mn_invalidate_node - unmap all BOs of a node
  *
@@ -126,54 +175,13 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
                 if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, start, end))
                         continue;
 
-                r = amdgpu_bo_reserve(bo, true);
-                if (r) {
-                        DRM_ERROR("(%ld) failed to reserve user bo\n", r);
-                        continue;
-                }
-
                 r = reservation_object_wait_timeout_rcu(bo->tbo.resv,
                         true, false, MAX_SCHEDULE_TIMEOUT);
                 if (r <= 0)
                         DRM_ERROR("(%ld) failed to wait for user bo\n", r);
 
-                amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
-                r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
-                if (r)
-                        DRM_ERROR("(%ld) failed to validate user bo\n", r);
-
-                amdgpu_bo_unreserve(bo);
-        }
-}
-
-/**
- * amdgpu_mn_invalidate_page - callback to notify about mm change
- *
- * @mn: our notifier
- * @mn: the mm this callback is about
- * @address: address of invalidate page
- *
- * Invalidation of a single page. Blocks for all BOs mapping it
- * and unmap them by move them into system domain again.
- */
-static void amdgpu_mn_invalidate_page(struct mmu_notifier *mn,
-                                      struct mm_struct *mm,
-                                      unsigned long address)
-{
-        struct amdgpu_mn *rmn = container_of(mn, struct amdgpu_mn, mn);
-        struct interval_tree_node *it;
-
-        mutex_lock(&rmn->lock);
-
-        it = interval_tree_iter_first(&rmn->objects, address, address);
-        if (it) {
-                struct amdgpu_mn_node *node;
-
-                node = container_of(it, struct amdgpu_mn_node, it);
-                amdgpu_mn_invalidate_node(node, address, address);
+                amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
         }
-
-        mutex_unlock(&rmn->lock);
 }
 
 /**
@@ -198,7 +206,7 @@ static void amdgpu_mn_invalidate_range_start(struct mmu_notifier *mn,
         /* notification is exclusive, but interval is inclusive */
         end -= 1;
 
-        mutex_lock(&rmn->lock);
+        amdgpu_mn_read_lock(rmn);
 
         it = interval_tree_iter_first(&rmn->objects, start, end);
         while (it) {
@@ -209,14 +217,32 @@ static void amdgpu_mn_invalidate_range_start(struct mmu_notifier *mn,
 
                 amdgpu_mn_invalidate_node(node, start, end);
         }
+}
 
-        mutex_unlock(&rmn->lock);
+/**
+ * amdgpu_mn_invalidate_range_end - callback to notify about mm change
+ *
+ * @mn: our notifier
+ * @mn: the mm this callback is about
+ * @start: start of updated range
+ * @end: end of updated range
+ *
+ * Release the lock again to allow new command submissions.
+ */
+static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
+                                           struct mm_struct *mm,
+                                           unsigned long start,
+                                           unsigned long end)
+{
+        struct amdgpu_mn *rmn = container_of(mn, struct amdgpu_mn, mn);
+
+        amdgpu_mn_read_unlock(rmn);
 }
 
 static const struct mmu_notifier_ops amdgpu_mn_ops = {
         .release = amdgpu_mn_release,
-        .invalidate_page = amdgpu_mn_invalidate_page,
         .invalidate_range_start = amdgpu_mn_invalidate_range_start,
+        .invalidate_range_end = amdgpu_mn_invalidate_range_end,
 };
 
 /**
@@ -226,7 +252,7 @@ static const struct mmu_notifier_ops amdgpu_mn_ops = {
  *
  * Creates a notifier context for current->mm.
  */
-static struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
+struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
 {
         struct mm_struct *mm = current->mm;
         struct amdgpu_mn *rmn;
@@ -251,8 +277,10 @@ static struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
         rmn->adev = adev;
         rmn->mm = mm;
         rmn->mn.ops = &amdgpu_mn_ops;
-        mutex_init(&rmn->lock);
-        rmn->objects = RB_ROOT;
+        init_rwsem(&rmn->lock);
+        rmn->objects = RB_ROOT_CACHED;
+        mutex_init(&rmn->read_lock);
+        atomic_set(&rmn->recursion, 0);
 
         r = __mmu_notifier_register(&rmn->mn, mm);
         if (r)
@@ -298,7 +326,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
 
         INIT_LIST_HEAD(&bos);
 
-        mutex_lock(&rmn->lock);
+        down_write(&rmn->lock);
 
         while ((it = interval_tree_iter_first(&rmn->objects, addr, end))) {
                 kfree(node);
@@ -312,7 +340,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
         if (!node) {
                 node = kmalloc(sizeof(struct amdgpu_mn_node), GFP_KERNEL);
                 if (!node) {
-                        mutex_unlock(&rmn->lock);
+                        up_write(&rmn->lock);
                         return -ENOMEM;
                 }
         }
@@ -327,7 +355,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
 
         interval_tree_insert(&node->it, &rmn->objects);
 
-        mutex_unlock(&rmn->lock);
+        up_write(&rmn->lock);
 
         return 0;
 }
@@ -353,7 +381,7 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
                 return;
         }
 
-        mutex_lock(&rmn->lock);
+        down_write(&rmn->lock);
 
         /* save the next list entry for later */
         head = bo->mn_list.next;
@@ -368,6 +396,7 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
                 kfree(node);
         }
 
-        mutex_unlock(&rmn->lock);
+        up_write(&rmn->lock);
         mutex_unlock(&adev->mn_lock);
 }
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
new file mode 100644
index 000000000000..d0095a3793b8
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Christian König
+ */
+#ifndef __AMDGPU_MN_H__
+#define __AMDGPU_MN_H__
+
+/*
+ * MMU Notifier
+ */
+struct amdgpu_mn;
+
+#if defined(CONFIG_MMU_NOTIFIER)
+void amdgpu_mn_lock(struct amdgpu_mn *mn);
+void amdgpu_mn_unlock(struct amdgpu_mn *mn);
+struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev);
+int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
+void amdgpu_mn_unregister(struct amdgpu_bo *bo);
+#else
+static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {}
+static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {}
+static inline struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
+{
+        return NULL;
+}
+static inline int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
+{
+        return -ENODEV;
+}
+static inline void amdgpu_mn_unregister(struct amdgpu_bo *bo) {}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 6e72fe7901ec..6982baeccd14 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -37,55 +37,6 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
-
-
-static u64 amdgpu_get_vis_part_size(struct amdgpu_device *adev,
-                                                struct ttm_mem_reg *mem)
-{
-        if (mem->start << PAGE_SHIFT >= adev->mc.visible_vram_size)
-                return 0;
-
-        return ((mem->start << PAGE_SHIFT) + mem->size) >
-                adev->mc.visible_vram_size ?
-                adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT) :
-                mem->size;
-}
-
-static void amdgpu_update_memory_usage(struct amdgpu_device *adev,
-                       struct ttm_mem_reg *old_mem,
-                       struct ttm_mem_reg *new_mem)
-{
-        u64 vis_size;
-        if (!adev)
-                return;
-
-        if (new_mem) {
-                switch (new_mem->mem_type) {
-                case TTM_PL_TT:
-                        atomic64_add(new_mem->size, &adev->gtt_usage);
-                        break;
-                case TTM_PL_VRAM:
-                        atomic64_add(new_mem->size, &adev->vram_usage);
-                        vis_size = amdgpu_get_vis_part_size(adev, new_mem);
-                        atomic64_add(vis_size, &adev->vram_vis_usage);
-                        break;
-                }
-        }
-
-        if (old_mem) {
-                switch (old_mem->mem_type) {
-                case TTM_PL_TT:
-                        atomic64_sub(old_mem->size, &adev->gtt_usage);
-                        break;
-                case TTM_PL_VRAM:
-                        atomic64_sub(old_mem->size, &adev->vram_usage);
-                        vis_size = amdgpu_get_vis_part_size(adev, old_mem);
-                        atomic64_sub(vis_size, &adev->vram_vis_usage);
-                        break;
-                }
-        }
-}
-
 static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 {
         struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
@@ -94,7 +45,6 @@ static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
         bo = container_of(tbo, struct amdgpu_bo, tbo);
 
         amdgpu_bo_kunmap(bo);
-        amdgpu_update_memory_usage(adev, &bo->tbo.mem, NULL);
 
         drm_gem_object_release(&bo->gem_base);
         amdgpu_bo_unref(&bo->parent);
@@ -114,11 +64,12 @@ bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
         return false;
 }
 
-static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
-                                      struct ttm_placement *placement,
-                                      struct ttm_place *places,
-                                      u32 domain, u64 flags)
+void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 {
+        struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
+        struct ttm_placement *placement = &abo->placement;
+        struct ttm_place *places = abo->placements;
+        u64 flags = abo->flags;
         u32 c = 0;
 
         if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
@@ -141,7 +92,10 @@ static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
 
         if (domain & AMDGPU_GEM_DOMAIN_GTT) {
                 places[c].fpfn = 0;
-                places[c].lpfn = 0;
+                if (flags & AMDGPU_GEM_CREATE_SHADOW)
+                        places[c].lpfn = adev->mc.gart_size >> PAGE_SHIFT;
+                else
+                        places[c].lpfn = 0;
                 places[c].flags = TTM_PL_FLAG_TT;
                 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
                         places[c].flags |= TTM_PL_FLAG_WC |
@@ -198,27 +152,6 @@ static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
         placement->busy_placement = places;
 }
 
-void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
-{
-        struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
-
-        amdgpu_ttm_placement_init(adev, &abo->placement, abo->placements,
-                                  domain, abo->flags);
-}
-
-static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
-                                        struct ttm_placement *placement)
-{
-        BUG_ON(placement->num_placement > (AMDGPU_GEM_DOMAIN_MAX + 1));
-
-        memcpy(bo->placements, placement->placement,
-               placement->num_placement * sizeof(struct ttm_place));
-        bo->placement.num_placement = placement->num_placement;
-        bo->placement.num_busy_placement = placement->num_busy_placement;
-        bo->placement.placement = bo->placements;
-        bo->placement.busy_placement = bo->placements;
-}
-
 /**
  * amdgpu_bo_create_reserved - create reserved BO for kernel use
  *
@@ -350,14 +283,13 @@ void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
                 *cpu_addr = NULL;
 }
 
-int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
-                                unsigned long size, int byte_align,
-                                bool kernel, u32 domain, u64 flags,
-                                struct sg_table *sg,
-                                struct ttm_placement *placement,
-                                struct reservation_object *resv,
-                                uint64_t init_value,
-                                struct amdgpu_bo **bo_ptr)
+static int amdgpu_bo_do_create(struct amdgpu_device *adev,
+                               unsigned long size, int byte_align,
+                               bool kernel, u32 domain, u64 flags,
+                               struct sg_table *sg,
+                               struct reservation_object *resv,
+                               uint64_t init_value,
+                               struct amdgpu_bo **bo_ptr)
 {
         struct amdgpu_bo *bo;
         enum ttm_bo_type type;
@@ -431,10 +363,11 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
                 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
 #endif
 
-        amdgpu_fill_placement_to_bo(bo, placement);
-        /* Kernel allocation are uninterruptible */
+        bo->tbo.bdev = &adev->mman.bdev;
+        amdgpu_ttm_placement_from_domain(bo, domain);
 
         initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
+        /* Kernel allocation are uninterruptible */
         r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
                                  &bo->placement, page_align, !kernel, NULL,
                                  acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
@@ -489,28 +422,17 @@ static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
                                    unsigned long size, int byte_align,
                                    struct amdgpu_bo *bo)
 {
-        struct ttm_placement placement = {0};
-        struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
         int r;
 
         if (bo->shadow)
                 return 0;
 
-        bo->flags |= AMDGPU_GEM_CREATE_SHADOW;
-        memset(&placements, 0,
-               (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));
-
-        amdgpu_ttm_placement_init(adev, &placement,
-                                  placements, AMDGPU_GEM_DOMAIN_GTT,
-                                  AMDGPU_GEM_CREATE_CPU_GTT_USWC);
-
-        r = amdgpu_bo_create_restricted(adev, size, byte_align, true,
-                                        AMDGPU_GEM_DOMAIN_GTT,
-                                        AMDGPU_GEM_CREATE_CPU_GTT_USWC,
-                                        NULL, &placement,
-                                        bo->tbo.resv,
-                                        0,
-                                        &bo->shadow);
+        r = amdgpu_bo_do_create(adev, size, byte_align, true,
+                                AMDGPU_GEM_DOMAIN_GTT,
+                                AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+                                AMDGPU_GEM_CREATE_SHADOW,
+                                NULL, bo->tbo.resv, 0,
+                                &bo->shadow);
         if (!r) {
                 bo->shadow->parent = amdgpu_bo_ref(bo);
                 mutex_lock(&adev->shadow_list_lock);
@@ -532,32 +454,23 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
                      uint64_t init_value,
                      struct amdgpu_bo **bo_ptr)
 {
-        struct ttm_placement placement = {0};
-        struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
+        uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW;
         int r;
 
-        memset(&placements, 0,
-               (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));
-
-        amdgpu_ttm_placement_init(adev, &placement,
-                                  placements, domain, flags);
-
-        r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel,
-                                        domain, flags, sg, &placement,
-                                        resv, init_value, bo_ptr);
+        r = amdgpu_bo_do_create(adev, size, byte_align, kernel, domain,
+                                parent_flags, sg, resv, init_value, bo_ptr);
         if (r)
                 return r;
 
-        if (amdgpu_need_backup(adev) && (flags & AMDGPU_GEM_CREATE_SHADOW)) {
-                if (!resv) {
-                        r = ww_mutex_lock(&(*bo_ptr)->tbo.resv->lock, NULL);
-                        WARN_ON(r != 0);
-                }
+        if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_need_backup(adev)) {
+                if (!resv)
+                        WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv,
+                                                        NULL));
 
                 r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
 
                 if (!resv)
-                        ww_mutex_unlock(&(*bo_ptr)->tbo.resv->lock);
+                        reservation_object_unlock((*bo_ptr)->tbo.resv);
 
                 if (r)
                         amdgpu_bo_unref(bo_ptr);
@@ -722,7 +635,6 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 {
         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
         int r, i;
-        unsigned fpfn, lpfn;
 
         if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
                 return -EPERM;
@@ -754,22 +666,16 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
         }
 
         bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+        /* force to pin into visible video ram */
+        if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
+                bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
         amdgpu_ttm_placement_from_domain(bo, domain);
         for (i = 0; i < bo->placement.num_placement; i++) {
-                /* force to pin into visible video ram */
-                if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
-                    !(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) &&
-                    (!max_offset || max_offset >
-                     adev->mc.visible_vram_size)) {
-                        if (WARN_ON_ONCE(min_offset >
-                                         adev->mc.visible_vram_size))
-                                return -EINVAL;
-                        fpfn = min_offset >> PAGE_SHIFT;
-                        lpfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
-                } else {
-                        fpfn = min_offset >> PAGE_SHIFT;
-                        lpfn = max_offset >> PAGE_SHIFT;
-                }
+                unsigned fpfn, lpfn;
+
+                fpfn = min_offset >> PAGE_SHIFT;
+                lpfn = max_offset >> PAGE_SHIFT;
+
                 if (fpfn > bo->placements[i].fpfn)
                         bo->placements[i].fpfn = fpfn;
                 if (!bo->placements[i].lpfn ||
@@ -979,7 +885,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
                 return;
 
         abo = container_of(bo, struct amdgpu_bo, tbo);
-        amdgpu_vm_bo_invalidate(adev, abo);
+        amdgpu_vm_bo_invalidate(adev, abo, evict);
 
         amdgpu_bo_kunmap(abo);
 
@@ -992,8 +898,6 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
                 return;
 
         /* move_notify is called before move happens */
-        amdgpu_update_memory_usage(adev, &bo->mem, new_mem);
-
         trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 9b7b4fcb047b..39b6bf6fb051 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -33,7 +33,9 @@
 
 #define AMDGPU_BO_INVALID_OFFSET        LONG_MAX
 
+/* bo virtual addresses in a vm */
 struct amdgpu_bo_va_mapping {
+        struct amdgpu_bo_va             *bo_va;
         struct list_head                list;
         struct rb_node                  rb;
         uint64_t                        start;
@@ -43,26 +45,24 @@ struct amdgpu_bo_va_mapping {
         uint64_t                        flags;
 };
 
-/* bo virtual addresses in a specific vm */
+/* User space allocated BO in a VM */
 struct amdgpu_bo_va {
+        struct amdgpu_vm_bo_base        base;
+
         /* protected by bo being reserved */
-        struct list_head                bo_list;
-        struct dma_fence                *last_pt_update;
         unsigned                        ref_count;
 
-        /* protected by vm mutex and spinlock */
-        struct list_head                vm_status;
+        /* all other members protected by the VM PD being reserved */
+        struct dma_fence                *last_pt_update;
 
         /* mappings for this bo_va */
         struct list_head                invalids;
         struct list_head                valids;
 
-        /* constant after initialization */
-        struct amdgpu_vm                *vm;
-        struct amdgpu_bo                *bo;
+        /* If the mappings are cleared or filled */
+        bool                            cleared;
 };
 
-
 struct amdgpu_bo {
         /* Protected by tbo.reserved */
         u32                             preferred_domains;
@@ -195,14 +195,6 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
                             struct reservation_object *resv,
                             uint64_t init_value,
                             struct amdgpu_bo **bo_ptr);
-int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
-                                unsigned long size, int byte_align,
-                                bool kernel, u32 domain, u64 flags,
-                                struct sg_table *sg,
-                                struct ttm_placement *placement,
-                                struct reservation_object *resv,
-                                uint64_t init_value,
-                                struct amdgpu_bo **bo_ptr);
 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
                               unsigned long size, int align,
                               u32 domain, struct amdgpu_bo **bo_ptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index 7df503aedb69..f6ce52956e6d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -74,7 +74,7 @@ void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev)
                         adev->pm.dpm.ac_power = true;
                 else
                         adev->pm.dpm.ac_power = false;
-                if (adev->pm.funcs->enable_bapm)
+                if (adev->powerplay.pp_funcs->enable_bapm)
                         amdgpu_dpm_enable_bapm(adev, adev->pm.dpm.ac_power);
                 mutex_unlock(&adev->pm.mutex);
         }
@@ -88,9 +88,9 @@ static ssize_t amdgpu_get_dpm_state(struct device *dev,
         struct amdgpu_device *adev = ddev->dev_private;
         enum amd_pm_state_type pm;
 
-        if (adev->pp_enabled) {
+        if (adev->powerplay.pp_funcs->get_current_power_state)
                 pm = amdgpu_dpm_get_current_power_state(adev);
-        } else
+        else
                 pm = adev->pm.dpm.user_state;
 
         return snprintf(buf, PAGE_SIZE, "%s\n",
@@ -119,7 +119,7 @@ static ssize_t amdgpu_set_dpm_state(struct device *dev,
         }
 
         if (adev->pp_enabled) {
-                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_ENABLE_USER_STATE, &state, NULL);
+                amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_ENABLE_USER_STATE, &state, NULL);
         } else {
                 mutex_lock(&adev->pm.mutex);
                 adev->pm.dpm.user_state = state;
@@ -140,13 +140,17 @@ static ssize_t amdgpu_get_dpm_forced_performance_level(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        enum amd_dpm_forced_level level;
+        enum amd_dpm_forced_level level = 0xff;
 
         if  ((adev->flags & AMD_IS_PX) &&
              (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
                 return snprintf(buf, PAGE_SIZE, "off\n");
 
-        level = amdgpu_dpm_get_performance_level(adev);
+        if (adev->powerplay.pp_funcs->get_performance_level)
+                level = amdgpu_dpm_get_performance_level(adev);
+        else
+                level = adev->pm.dpm.forced_level;
+
         return snprintf(buf, PAGE_SIZE, "%s\n",
                         (level == AMD_DPM_FORCED_LEVEL_AUTO) ? "auto" :
                         (level == AMD_DPM_FORCED_LEVEL_LOW) ? "low" :
@@ -167,7 +171,7 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
         enum amd_dpm_forced_level level;
-        enum amd_dpm_forced_level current_level;
+        enum amd_dpm_forced_level current_level = 0xff;
         int ret = 0;
 
         /* Can't force performance level when the card is off */
@@ -175,7 +179,8 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
              (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
                 return -EINVAL;
 
-        current_level = amdgpu_dpm_get_performance_level(adev);
+        if (adev->powerplay.pp_funcs->get_performance_level)
+                current_level = amdgpu_dpm_get_performance_level(adev);
 
         if (strncmp("low", buf, strlen("low")) == 0) {
                 level = AMD_DPM_FORCED_LEVEL_LOW;
@@ -203,9 +208,7 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
         if (current_level == level)
                 return count;
 
-        if (adev->pp_enabled)
-                amdgpu_dpm_force_performance_level(adev, level);
-        else {
+        if (adev->powerplay.pp_funcs->force_performance_level) {
                 mutex_lock(&adev->pm.mutex);
                 if (adev->pm.dpm.thermal_active) {
                         count = -EINVAL;
@@ -233,7 +236,7 @@ static ssize_t amdgpu_get_pp_num_states(struct device *dev,
         struct pp_states_info data;
         int i, buf_len;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->get_pp_num_states)
                 amdgpu_dpm_get_pp_num_states(adev, &data);
 
         buf_len = snprintf(buf, PAGE_SIZE, "states: %d\n", data.nums);
@@ -257,8 +260,8 @@ static ssize_t amdgpu_get_pp_cur_state(struct device *dev,
         enum amd_pm_state_type pm = 0;
         int i = 0;
 
-        if (adev->pp_enabled) {
-
+        if (adev->powerplay.pp_funcs->get_current_power_state
+                 && adev->powerplay.pp_funcs->get_pp_num_states) {
                 pm = amdgpu_dpm_get_current_power_state(adev);
                 amdgpu_dpm_get_pp_num_states(adev, &data);
 
@@ -280,25 +283,10 @@ static ssize_t amdgpu_get_pp_force_state(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        struct pp_states_info data;
-        enum amd_pm_state_type pm = 0;
-        int i;
-
-        if (adev->pp_force_state_enabled && adev->pp_enabled) {
-                pm = amdgpu_dpm_get_current_power_state(adev);
-                amdgpu_dpm_get_pp_num_states(adev, &data);
-
-                for (i = 0; i < data.nums; i++) {
-                        if (pm == data.states[i])
-                                break;
-                }
 
-                if (i == data.nums)
-                        i = -EINVAL;
-
-                return snprintf(buf, PAGE_SIZE, "%d\n", i);
-
-        } else
+        if (adev->pp_force_state_enabled)
+                return amdgpu_get_pp_cur_state(dev, attr, buf);
+        else
                 return snprintf(buf, PAGE_SIZE, "\n");
 }
 
@@ -330,7 +318,7 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev,
                 if (state != POWER_STATE_TYPE_INTERNAL_BOOT &&
                     state != POWER_STATE_TYPE_DEFAULT) {
                         amdgpu_dpm_dispatch_task(adev,
-                                        AMD_PP_EVENT_ENABLE_USER_STATE, &state, NULL);
+                                        AMD_PP_TASK_ENABLE_USER_STATE, &state, NULL);
                         adev->pp_force_state_enabled = true;
                 }
         }
@@ -347,7 +335,7 @@ static ssize_t amdgpu_get_pp_table(struct device *dev,
         char *table = NULL;
         int size;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->get_pp_table)
                 size = amdgpu_dpm_get_pp_table(adev, &table);
         else
                 return 0;
@@ -368,7 +356,7 @@ static ssize_t amdgpu_set_pp_table(struct device *dev,
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->set_pp_table)
                 amdgpu_dpm_set_pp_table(adev, buf, count);
 
         return count;
@@ -380,14 +368,11 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        ssize_t size = 0;
 
-        if (adev->pp_enabled)
-                size = amdgpu_dpm_print_clock_levels(adev, PP_SCLK, buf);
-        else if (adev->pm.funcs->print_clock_levels)
-                size = adev->pm.funcs->print_clock_levels(adev, PP_SCLK, buf);
-
-        return size;
+        if (adev->powerplay.pp_funcs->print_clock_levels)
+                return amdgpu_dpm_print_clock_levels(adev, PP_SCLK, buf);
+        else
+                return snprintf(buf, PAGE_SIZE, "\n");
 }
 
 static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
@@ -416,10 +401,9 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
                 mask |= 1 << level;
         }
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->force_clock_level)
                 amdgpu_dpm_force_clock_level(adev, PP_SCLK, mask);
-        else if (adev->pm.funcs->force_clock_level)
-                adev->pm.funcs->force_clock_level(adev, PP_SCLK, mask);
+
 fail:
         return count;
 }
@@ -430,14 +414,11 @@ static ssize_t amdgpu_get_pp_dpm_mclk(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        ssize_t size = 0;
-
-        if (adev->pp_enabled)
-                size = amdgpu_dpm_print_clock_levels(adev, PP_MCLK, buf);
-        else if (adev->pm.funcs->print_clock_levels)
-                size = adev->pm.funcs->print_clock_levels(adev, PP_MCLK, buf);
 
-        return size;
+        if (adev->powerplay.pp_funcs->print_clock_levels)
+                return amdgpu_dpm_print_clock_levels(adev, PP_MCLK, buf);
+        else
+                return snprintf(buf, PAGE_SIZE, "\n");
 }
 
 static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev,
@@ -465,11 +446,9 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev,
                 }
                 mask |= 1 << level;
         }
-
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->force_clock_level)
                 amdgpu_dpm_force_clock_level(adev, PP_MCLK, mask);
-        else if (adev->pm.funcs->force_clock_level)
-                adev->pm.funcs->force_clock_level(adev, PP_MCLK, mask);
+
 fail:
         return count;
 }
@@ -480,14 +459,11 @@ static ssize_t amdgpu_get_pp_dpm_pcie(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        ssize_t size = 0;
-
-        if (adev->pp_enabled)
-                size = amdgpu_dpm_print_clock_levels(adev, PP_PCIE, buf);
-        else if (adev->pm.funcs->print_clock_levels)
-                size = adev->pm.funcs->print_clock_levels(adev, PP_PCIE, buf);
 
-        return size;
+        if (adev->powerplay.pp_funcs->print_clock_levels)
+                return amdgpu_dpm_print_clock_levels(adev, PP_PCIE, buf);
+        else
+                return snprintf(buf, PAGE_SIZE, "\n");
 }
 
 static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev,
@@ -515,11 +491,9 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev,
                 }
                 mask |= 1 << level;
         }
-
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->force_clock_level)
                 amdgpu_dpm_force_clock_level(adev, PP_PCIE, mask);
-        else if (adev->pm.funcs->force_clock_level)
-                adev->pm.funcs->force_clock_level(adev, PP_PCIE, mask);
+
 fail:
         return count;
 }
@@ -532,10 +506,8 @@ static ssize_t amdgpu_get_pp_sclk_od(struct device *dev,
         struct amdgpu_device *adev = ddev->dev_private;
         uint32_t value = 0;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->get_sclk_od)
                 value = amdgpu_dpm_get_sclk_od(adev);
-        else if (adev->pm.funcs->get_sclk_od)
-                value = adev->pm.funcs->get_sclk_od(adev);
 
         return snprintf(buf, PAGE_SIZE, "%d\n", value);
 }
@@ -556,12 +528,12 @@ static ssize_t amdgpu_set_pp_sclk_od(struct device *dev,
                 count = -EINVAL;
                 goto fail;
         }
+        if (adev->powerplay.pp_funcs->set_sclk_od)
+                amdgpu_dpm_set_sclk_od(adev, (uint32_t)value);
 
         if (adev->pp_enabled) {
-                amdgpu_dpm_set_sclk_od(adev, (uint32_t)value);
-                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_READJUST_POWER_STATE, NULL, NULL);
-        } else if (adev->pm.funcs->set_sclk_od) {
-                adev->pm.funcs->set_sclk_od(adev, (uint32_t)value);
+                amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL, NULL);
+        } else {
                 adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
                 amdgpu_pm_compute_clocks(adev);
         }
@@ -578,10 +550,8 @@ static ssize_t amdgpu_get_pp_mclk_od(struct device *dev,
         struct amdgpu_device *adev = ddev->dev_private;
         uint32_t value = 0;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->get_mclk_od)
                 value = amdgpu_dpm_get_mclk_od(adev);
-        else if (adev->pm.funcs->get_mclk_od)
-                value = adev->pm.funcs->get_mclk_od(adev);
 
         return snprintf(buf, PAGE_SIZE, "%d\n", value);
 }
@@ -602,12 +572,12 @@ static ssize_t amdgpu_set_pp_mclk_od(struct device *dev,
                 count = -EINVAL;
                 goto fail;
         }
+        if (adev->powerplay.pp_funcs->set_mclk_od)
+                amdgpu_dpm_set_mclk_od(adev, (uint32_t)value);
 
         if (adev->pp_enabled) {
-                amdgpu_dpm_set_mclk_od(adev, (uint32_t)value);
-                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_READJUST_POWER_STATE, NULL, NULL);
-        } else if (adev->pm.funcs->set_mclk_od) {
-                adev->pm.funcs->set_mclk_od(adev, (uint32_t)value);
+                amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL, NULL);
+        } else {
                 adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
                 amdgpu_pm_compute_clocks(adev);
         }
@@ -621,14 +591,11 @@ static ssize_t amdgpu_get_pp_power_profile(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        int ret = 0;
+        int ret = 0xff;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->get_power_profile_state)
                 ret = amdgpu_dpm_get_power_profile_state(
                                 adev, query);
-        else if (adev->pm.funcs->get_power_profile_state)
-                ret = adev->pm.funcs->get_power_profile_state(
-                                adev, query);
 
         if (ret)
                 return ret;
@@ -675,15 +642,12 @@ static ssize_t amdgpu_set_pp_power_profile(struct device *dev,
         char *sub_str, buf_cpy[128], *tmp_str;
         const char delimiter[3] = {' ', '\n', '\0'};
         long int value;
-        int ret = 0;
+        int ret = 0xff;
 
         if (strncmp("reset", buf, strlen("reset")) == 0) {
-                if (adev->pp_enabled)
+                if (adev->powerplay.pp_funcs->reset_power_profile_state)
                         ret = amdgpu_dpm_reset_power_profile_state(
                                         adev, request);
-                else if (adev->pm.funcs->reset_power_profile_state)
-                        ret = adev->pm.funcs->reset_power_profile_state(
-                                        adev, request);
                 if (ret) {
                         count = -EINVAL;
                         goto fail;
@@ -692,12 +656,10 @@ static ssize_t amdgpu_set_pp_power_profile(struct device *dev,
         }
 
         if (strncmp("set", buf, strlen("set")) == 0) {
-                if (adev->pp_enabled)
+                if (adev->powerplay.pp_funcs->set_power_profile_state)
                         ret = amdgpu_dpm_set_power_profile_state(
                                         adev, request);
-                else if (adev->pm.funcs->set_power_profile_state)
-                        ret = adev->pm.funcs->set_power_profile_state(
-                                        adev, request);
+
                 if (ret) {
                         count = -EINVAL;
                         goto fail;
@@ -745,13 +707,8 @@ static ssize_t amdgpu_set_pp_power_profile(struct device *dev,
 
                 loop++;
         }
-
-        if (adev->pp_enabled)
-                ret = amdgpu_dpm_set_power_profile_state(
-                                adev, request);
-        else if (adev->pm.funcs->set_power_profile_state)
-                ret = adev->pm.funcs->set_power_profile_state(
-                                adev, request);
+        if (adev->powerplay.pp_funcs->set_power_profile_state)
+                ret = amdgpu_dpm_set_power_profile_state(adev, request);
 
         if (ret)
                 count = -EINVAL;
@@ -831,7 +788,7 @@ static ssize_t amdgpu_hwmon_show_temp(struct device *dev,
              (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
                 return -EINVAL;
 
-        if (!adev->pp_enabled && !adev->pm.funcs->get_temperature)
+        if (!adev->powerplay.pp_funcs->get_temperature)
                 temp = 0;
         else
                 temp = amdgpu_dpm_get_temperature(adev);
@@ -862,7 +819,7 @@ static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev,
         struct amdgpu_device *adev = dev_get_drvdata(dev);
         u32 pwm_mode = 0;
 
-        if (!adev->pp_enabled && !adev->pm.funcs->get_fan_control_mode)
+        if (!adev->powerplay.pp_funcs->get_fan_control_mode)
                 return -EINVAL;
 
         pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
@@ -879,7 +836,7 @@ static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev,
         int err;
         int value;
 
-        if (!adev->pp_enabled && !adev->pm.funcs->set_fan_control_mode)
+        if (!adev->powerplay.pp_funcs->set_fan_control_mode)
                 return -EINVAL;
 
         err = kstrtoint(buf, 10, &value);
@@ -919,9 +876,11 @@ static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev,
 
         value = (value * 100) / 255;
 
-        err = amdgpu_dpm_set_fan_speed_percent(adev, value);
-        if (err)
-                return err;
+        if (adev->powerplay.pp_funcs->set_fan_speed_percent) {
+                err = amdgpu_dpm_set_fan_speed_percent(adev, value);
+                if (err)
+                        return err;
+        }
 
         return count;
 }
@@ -932,11 +891,13 @@ static ssize_t amdgpu_hwmon_get_pwm1(struct device *dev,
 {
         struct amdgpu_device *adev = dev_get_drvdata(dev);
         int err;
-        u32 speed;
+        u32 speed = 0;
 
-        err = amdgpu_dpm_get_fan_speed_percent(adev, &speed);
-        if (err)
-                return err;
+        if (adev->powerplay.pp_funcs->get_fan_speed_percent) {
+                err = amdgpu_dpm_get_fan_speed_percent(adev, &speed);
+                if (err)
+                        return err;
+        }
 
         speed = (speed * 255) / 100;
 
@@ -949,11 +910,13 @@ static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev,
 {
         struct amdgpu_device *adev = dev_get_drvdata(dev);
         int err;
-        u32 speed;
+        u32 speed = 0;
 
-        err = amdgpu_dpm_get_fan_speed_rpm(adev, &speed);
-        if (err)
-                return err;
+        if (adev->powerplay.pp_funcs->get_fan_speed_rpm) {
+                err = amdgpu_dpm_get_fan_speed_rpm(adev, &speed);
+                if (err)
+                        return err;
+        }
 
         return sprintf(buf, "%i\n", speed);
 }
@@ -1008,21 +971,21 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
                 return 0;
 
         /* mask fan attributes if we have no bindings for this asic to expose */
-        if ((!adev->pm.funcs->get_fan_speed_percent &&
+        if ((!adev->powerplay.pp_funcs->get_fan_speed_percent &&
              attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't query fan */
-            (!adev->pm.funcs->get_fan_control_mode &&
+            (!adev->powerplay.pp_funcs->get_fan_control_mode &&
              attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't query state */
                 effective_mode &= ~S_IRUGO;
 
-        if ((!adev->pm.funcs->set_fan_speed_percent &&
+        if ((!adev->powerplay.pp_funcs->set_fan_speed_percent &&
              attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't manage fan */
-            (!adev->pm.funcs->set_fan_control_mode &&
+            (!adev->powerplay.pp_funcs->set_fan_control_mode &&
              attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't manage state */
                 effective_mode &= ~S_IWUSR;
 
         /* hide max/min values if we can't both query and manage the fan */
-        if ((!adev->pm.funcs->set_fan_speed_percent &&
-             !adev->pm.funcs->get_fan_speed_percent) &&
+        if ((!adev->powerplay.pp_funcs->set_fan_speed_percent &&
+             !adev->powerplay.pp_funcs->get_fan_speed_percent) &&
             (attr == &sensor_dev_attr_pwm1_max.dev_attr.attr ||
              attr == &sensor_dev_attr_pwm1_min.dev_attr.attr))
                 return 0;
@@ -1055,7 +1018,7 @@ void amdgpu_dpm_thermal_work_handler(struct work_struct *work)
         if (!adev->pm.dpm_enabled)
                 return;
 
-        if (adev->pm.funcs->get_temperature) {
+        if (adev->powerplay.pp_funcs->get_temperature) {
                 int temp = amdgpu_dpm_get_temperature(adev);
 
                 if (temp < adev->pm.dpm.thermal.min_temp)
@@ -1087,7 +1050,7 @@ static struct amdgpu_ps *amdgpu_dpm_pick_power_state(struct amdgpu_device *adev,
                 true : false;
 
         /* check if the vblank period is too short to adjust the mclk */
-        if (single_display && adev->pm.funcs->vblank_too_short) {
+        if (single_display && adev->powerplay.pp_funcs->vblank_too_short) {
                 if (amdgpu_dpm_vblank_too_short(adev))
                         single_display = false;
         }
@@ -1216,7 +1179,7 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
         struct amdgpu_ps *ps;
         enum amd_pm_state_type dpm_state;
         int ret;
-        bool equal;
+        bool equal = false;
 
         /* if dpm init failed */
         if (!adev->pm.dpm_enabled)
@@ -1236,7 +1199,7 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
         else
                 return;
 
-        if (amdgpu_dpm == 1) {
+        if (amdgpu_dpm == 1 && adev->powerplay.pp_funcs->print_power_state) {
                 printk("switching from power state:\n");
                 amdgpu_dpm_print_power_state(adev, adev->pm.dpm.current_ps);
                 printk("switching to power state:\n");
@@ -1245,15 +1208,17 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
 
         /* update whether vce is active */
         ps->vce_active = adev->pm.dpm.vce_active;
-
-        amdgpu_dpm_display_configuration_changed(adev);
+        if (adev->powerplay.pp_funcs->display_configuration_changed)
+                amdgpu_dpm_display_configuration_changed(adev);
 
         ret = amdgpu_dpm_pre_set_power_state(adev);
         if (ret)
                 return;
 
-        if ((0 != amgdpu_dpm_check_state_equal(adev, adev->pm.dpm.current_ps, adev->pm.dpm.requested_ps, &equal)))
-                equal = false;
+        if (adev->powerplay.pp_funcs->check_state_equal) {
+                if (0 != amdgpu_dpm_check_state_equal(adev, adev->pm.dpm.current_ps, adev->pm.dpm.requested_ps, &equal))
+                        equal = false;
+        }
 
         if (equal)
                 return;
@@ -1264,7 +1229,7 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
         adev->pm.dpm.current_active_crtcs = adev->pm.dpm.new_active_crtcs;
         adev->pm.dpm.current_active_crtc_count = adev->pm.dpm.new_active_crtc_count;
 
-        if (adev->pm.funcs->force_performance_level) {
+        if (adev->powerplay.pp_funcs->force_performance_level) {
                 if (adev->pm.dpm.thermal_active) {
                         enum amd_dpm_forced_level level = adev->pm.dpm.forced_level;
                         /* force low perf level for thermal */
@@ -1280,7 +1245,7 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
 
 void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable)
 {
-        if (adev->pp_enabled || adev->pm.funcs->powergate_uvd) {
+        if (adev->powerplay.pp_funcs->powergate_uvd) {
                 /* enable/disable UVD */
                 mutex_lock(&adev->pm.mutex);
                 amdgpu_dpm_powergate_uvd(adev, !enable);
@@ -1302,7 +1267,7 @@ void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable)
 
 void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable)
 {
-        if (adev->pp_enabled || adev->pm.funcs->powergate_vce) {
+        if (adev->powerplay.pp_funcs->powergate_vce) {
                 /* enable/disable VCE */
                 mutex_lock(&adev->pm.mutex);
                 amdgpu_dpm_powergate_vce(adev, !enable);
@@ -1337,8 +1302,7 @@ void amdgpu_pm_print_power_states(struct amdgpu_device *adev)
 {
         int i;
 
-        if (adev->pp_enabled)
-                /* TO DO */
+        if (adev->powerplay.pp_funcs->print_power_state == NULL)
                 return;
 
         for (i = 0; i < adev->pm.dpm.num_ps; i++)
@@ -1353,10 +1317,8 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
         if (adev->pm.sysfs_initialized)
                 return 0;
 
-        if (!adev->pp_enabled) {
-                if (adev->pm.funcs->get_temperature == NULL)
-                        return 0;
-        }
+        if (adev->powerplay.pp_funcs->get_temperature == NULL)
+                return 0;
 
         adev->pm.int_hwmon_dev = hwmon_device_register_with_groups(adev->dev,
                                                                    DRIVER_NAME, adev,
@@ -1496,7 +1458,7 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
         }
 
         if (adev->pp_enabled) {
-                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_DISPLAY_CONFIG_CHANGE, NULL, NULL);
+                amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, NULL, NULL);
         } else {
                 mutex_lock(&adev->pm.mutex);
                 adev->pm.dpm.new_active_crtcs = 0;
@@ -1634,8 +1596,8 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data)
                 return amdgpu_debugfs_pm_info_pp(m, adev);
         } else {
                 mutex_lock(&adev->pm.mutex);
-                if (adev->pm.funcs->debugfs_print_current_performance_level)
-                        adev->pm.funcs->debugfs_print_current_performance_level(adev, m);
+                if (adev->powerplay.pp_funcs->debugfs_print_current_performance_level)
+                        adev->powerplay.pp_funcs->debugfs_print_current_performance_level(adev, m);
                 else
                         seq_printf(m, "Debugfs support not implemented for this asic\n");
                 mutex_unlock(&adev->pm.mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
index b7e1c026c0c8..2d2f0960b025 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
@@ -87,17 +87,28 @@ static int amdgpu_pp_early_init(void *handle)
         case CHIP_OLAND:
         case CHIP_HAINAN:
                 amd_pp->ip_funcs = &si_dpm_ip_funcs;
+                amd_pp->pp_funcs = &si_dpm_funcs;
         break;
 #endif
 #ifdef CONFIG_DRM_AMDGPU_CIK
         case CHIP_BONAIRE:
         case CHIP_HAWAII:
-                amd_pp->ip_funcs = &ci_dpm_ip_funcs;
+                if (amdgpu_dpm == -1) {
+                        amd_pp->ip_funcs = &ci_dpm_ip_funcs;
+                        amd_pp->pp_funcs = &ci_dpm_funcs;
+                } else {
+                        adev->pp_enabled = true;
+                        if (amdgpu_create_pp_handle(adev))
+                                return -EINVAL;
+                        amd_pp->ip_funcs = &pp_ip_funcs;
+                        amd_pp->pp_funcs = &pp_dpm_funcs;
+                }
                 break;
         case CHIP_KABINI:
         case CHIP_MULLINS:
         case CHIP_KAVERI:
                 amd_pp->ip_funcs = &kv_dpm_ip_funcs;
+                amd_pp->pp_funcs = &kv_dpm_funcs;
                 break;
 #endif
         default:
@@ -128,7 +139,7 @@ static int amdgpu_pp_late_init(void *handle)
 
         if (adev->pp_enabled && adev->pm.dpm_enabled) {
                 amdgpu_pm_sysfs_init(adev);
-                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_COMPLETE_INIT, NULL, NULL);
+                amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_COMPLETE_INIT, NULL, NULL);
         }
 
         return ret;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
index 5b3f92891f89..90af8e82b16a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
@@ -57,6 +57,40 @@ void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
         ttm_bo_kunmap(&bo->dma_buf_vmap);
 }
 
+int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+        struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
+        struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+        unsigned asize = amdgpu_bo_size(bo);
+        int ret;
+
+        if (!vma->vm_file)
+                return -ENODEV;
+
+        if (adev == NULL)
+                return -ENODEV;
+
+        /* Check for valid size. */
+        if (asize < vma->vm_end - vma->vm_start)
+                return -EINVAL;
+
+        if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) ||
+            (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
+                return -EPERM;
+        }
+        vma->vm_pgoff += amdgpu_bo_mmap_offset(bo) >> PAGE_SHIFT;
+
+        /* prime mmap does not need to check access, so allow here */
+        ret = drm_vma_node_allow(&obj->vma_node, vma->vm_file->private_data);
+        if (ret)
+                return ret;
+
+        ret = ttm_bo_mmap(vma->vm_file, vma, &adev->mman.bdev);
+        drm_vma_node_revoke(&obj->vma_node, vma->vm_file->private_data);
+
+        return ret;
+}
+
 struct drm_gem_object *
 amdgpu_gem_prime_import_sg_table(struct drm_device *dev,
                                  struct dma_buf_attachment *attach,
@@ -136,7 +170,8 @@ struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev,
 {
         struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
 
-        if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
+        if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) ||
+            bo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
                 return ERR_PTR(-EPERM);
 
         return drm_gem_prime_export(dev, gobj, flags);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 8c2204c7b384..447d446b5015 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -57,21 +57,23 @@ static int psp_sw_init(void *handle)
                 psp->prep_cmd_buf = psp_v3_1_prep_cmd_buf;
                 psp->ring_init = psp_v3_1_ring_init;
                 psp->ring_create = psp_v3_1_ring_create;
+                psp->ring_stop = psp_v3_1_ring_stop;
                 psp->ring_destroy = psp_v3_1_ring_destroy;
                 psp->cmd_submit = psp_v3_1_cmd_submit;
                 psp->compare_sram_data = psp_v3_1_compare_sram_data;
                 psp->smu_reload_quirk = psp_v3_1_smu_reload_quirk;
+                psp->mode1_reset = psp_v3_1_mode1_reset;
                 break;
         case CHIP_RAVEN:
-#if 0
                 psp->init_microcode = psp_v10_0_init_microcode;
-#endif
                 psp->prep_cmd_buf = psp_v10_0_prep_cmd_buf;
                 psp->ring_init = psp_v10_0_ring_init;
                 psp->ring_create = psp_v10_0_ring_create;
+                psp->ring_stop = psp_v10_0_ring_stop;
                 psp->ring_destroy = psp_v10_0_ring_destroy;
                 psp->cmd_submit = psp_v10_0_cmd_submit;
                 psp->compare_sram_data = psp_v10_0_compare_sram_data;
+                psp->mode1_reset = psp_v10_0_mode1_reset;
                 break;
         default:
                 return -EINVAL;
@@ -90,6 +92,12 @@ static int psp_sw_init(void *handle)
 
 static int psp_sw_fini(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+        release_firmware(adev->psp.sos_fw);
+        adev->psp.sos_fw = NULL;
+        release_firmware(adev->psp.asd_fw);
+        adev->psp.asd_fw = NULL;
         return 0;
 }
 
@@ -253,15 +261,18 @@ static int psp_asd_load(struct psp_context *psp)
 
 static int psp_hw_start(struct psp_context *psp)
 {
+        struct amdgpu_device *adev = psp->adev;
         int ret;
 
-        ret = psp_bootloader_load_sysdrv(psp);
-        if (ret)
-                return ret;
+        if (!amdgpu_sriov_vf(adev) || !adev->in_sriov_reset) {
+                ret = psp_bootloader_load_sysdrv(psp);
+                if (ret)
+                        return ret;
 
-        ret = psp_bootloader_load_sos(psp);
-        if (ret)
-                return ret;
+                ret = psp_bootloader_load_sos(psp);
+                if (ret)
+                        return ret;
+        }
 
         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
         if (ret)
@@ -453,6 +464,16 @@ static int psp_hw_fini(void *handle)
 
 static int psp_suspend(void *handle)
 {
+        int ret;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        struct psp_context *psp = &adev->psp;
+
+        ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
+        if (ret) {
+                DRM_ERROR("PSP ring stop failed\n");
+                return ret;
+        }
+
         return 0;
 }
 
@@ -487,6 +508,22 @@ failed:
         return ret;
 }
 
+static bool psp_check_reset(void* handle)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+        if (adev->flags & AMD_IS_APU)
+                return true;
+
+        return false;
+}
+
+static int psp_reset(void* handle)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        return psp_mode1_reset(&adev->psp);
+}
+
 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
                                         enum AMDGPU_UCODE_ID ucode_type)
 {
@@ -530,8 +567,9 @@ const struct amd_ip_funcs psp_ip_funcs = {
         .suspend = psp_suspend,
         .resume = psp_resume,
         .is_idle = NULL,
+        .check_soft_reset = psp_check_reset,
         .wait_for_idle = NULL,
-        .soft_reset = NULL,
+        .soft_reset = psp_reset,
         .set_clockgating_state = psp_set_clockgating_state,
         .set_powergating_state = psp_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index 538fa9dbfb21..ce4654550416 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -66,6 +66,8 @@ struct psp_context
                             struct psp_gfx_cmd_resp *cmd);
         int (*ring_init)(struct psp_context *psp, enum psp_ring_type ring_type);
         int (*ring_create)(struct psp_context *psp, enum psp_ring_type ring_type);
+        int (*ring_stop)(struct psp_context *psp,
+                            enum psp_ring_type ring_type);
         int (*ring_destroy)(struct psp_context *psp,
                             enum psp_ring_type ring_type);
         int (*cmd_submit)(struct psp_context *psp, struct amdgpu_firmware_info *ucode,
@@ -74,6 +76,7 @@ struct psp_context
                                   struct amdgpu_firmware_info *ucode,
                                   enum AMDGPU_UCODE_ID ucode_type);
         bool (*smu_reload_quirk)(struct psp_context *psp);
+        int (*mode1_reset)(struct psp_context *psp);
 
         /* fence buffer */
         struct amdgpu_bo                *fw_pri_bo;
@@ -123,6 +126,7 @@ struct amdgpu_psp_funcs {
 #define psp_prep_cmd_buf(ucode, type) (psp)->prep_cmd_buf((ucode), (type))
 #define psp_ring_init(psp, type) (psp)->ring_init((psp), (type))
 #define psp_ring_create(psp, type) (psp)->ring_create((psp), (type))
+#define psp_ring_stop(psp, type) (psp)->ring_stop((psp), (type))
 #define psp_ring_destroy(psp, type) ((psp)->ring_destroy((psp), (type)))
 #define psp_cmd_submit(psp, ucode, cmd_mc, fence_mc, index) \
                 (psp)->cmd_submit((psp), (ucode), (cmd_mc), (fence_mc), (index))
@@ -136,6 +140,8 @@ struct amdgpu_psp_funcs {
                 ((psp)->bootloader_load_sos ? (psp)->bootloader_load_sos((psp)) : 0)
 #define psp_smu_reload_quirk(psp) \
                 ((psp)->smu_reload_quirk ? (psp)->smu_reload_quirk((psp)) : false)
+#define psp_mode1_reset(psp) \
+                ((psp)->mode1_reset ? (psp)->mode1_reset((psp)) : false)
 
 extern const struct amd_ip_funcs psp_ip_funcs;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 6c5646b48d1a..5ce65280b396 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -170,6 +170,16 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
                      unsigned irq_type)
 {
         int r;
+        int sched_hw_submission = amdgpu_sched_hw_submission;
+
+        /* Set the hw submission limit higher for KIQ because
+         * it's used for a number of gfx/compute tasks by both
+         * KFD and KGD which may have outstanding fences and
+         * it doesn't really use the gpu scheduler anyway;
+         * KIQ tasks get submitted directly to the ring.
+         */
+        if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
+                sched_hw_submission = max(sched_hw_submission, 256);
 
         if (ring->adev == NULL) {
                 if (adev->num_rings >= AMDGPU_MAX_RINGS)
@@ -178,8 +188,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
                 ring->adev = adev;
                 ring->idx = adev->num_rings++;
                 adev->rings[ring->idx] = ring;
-                r = amdgpu_fence_driver_init_ring(ring,
-                        amdgpu_sched_hw_submission);
+                r = amdgpu_fence_driver_init_ring(ring, sched_hw_submission);
                 if (r)
                         return r;
         }
@@ -218,8 +227,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
                 return r;
         }
 
-        ring->ring_size = roundup_pow_of_two(max_dw * 4 *
-                                             amdgpu_sched_hw_submission);
+        ring->ring_size = roundup_pow_of_two(max_dw * 4 * sched_hw_submission);
 
         ring->buf_mask = (ring->ring_size / 4) - 1;
         ring->ptr_mask = ring->funcs->support_64bit_ptrs ?
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index a6899180b265..c586f44312f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -244,6 +244,12 @@ struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
                 struct dma_fence *f = e->fence;
                 struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
 
+                if (dma_fence_is_signaled(f)) {
+                        hash_del(&e->node);
+                        dma_fence_put(f);
+                        kmem_cache_free(amdgpu_sync_slab, e);
+                        continue;
+                }
                 if (ring && s_fence) {
                         /* For fences from the same ring it is sufficient
                          * when they are scheduled.
@@ -256,13 +262,6 @@ struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
                         }
                 }
 
-                if (dma_fence_is_signaled(f)) {
-                        hash_del(&e->node);
-                        dma_fence_put(f);
-                        kmem_cache_free(amdgpu_sync_slab, e);
-                        continue;
-                }
-
                 return f;
         }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 9ab58245e518..213988f336ed 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -228,7 +228,7 @@ TRACE_EVENT(amdgpu_vm_bo_map,
                              ),
 
             TP_fast_assign(
-                           __entry->bo = bo_va ? bo_va->bo : NULL;
+                           __entry->bo = bo_va ? bo_va->base.bo : NULL;
                            __entry->start = mapping->start;
                            __entry->last = mapping->last;
                            __entry->offset = mapping->offset;
@@ -252,7 +252,7 @@ TRACE_EVENT(amdgpu_vm_bo_unmap,
                              ),
 
             TP_fast_assign(
-                           __entry->bo = bo_va->bo;
+                           __entry->bo = bo_va->base.bo;
                            __entry->start = mapping->start;
                            __entry->last = mapping->last;
                            __entry->offset = mapping->offset;
@@ -417,5 +417,5 @@ TRACE_EVENT(amdgpu_ttm_bo_move,
 
 /* This part must be outside protection */
 #undef TRACE_INCLUDE_PATH
-#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/amd/amdgpu
 #include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
index 385b7e1d72f9..9ec96b9e85d1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
@@ -1,4 +1,23 @@
 /* Copyright Red Hat Inc 2010.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
  * Author : Dave Airlie <airlied@redhat.com>
  */
 #include <drm/drmP.h>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index c803b082324d..15a28578d458 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -42,7 +42,9 @@
 #include <linux/swap.h>
 #include <linux/pagemap.h>
 #include <linux/debugfs.h>
+#include <linux/iommu.h>
 #include "amdgpu.h"
+#include "amdgpu_trace.h"
 #include "bif/bif_4_1_d.h"
 
 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
@@ -607,6 +609,7 @@ struct amdgpu_ttm_tt {
         spinlock_t              guptasklock;
         struct list_head        guptasks;
         atomic_t                mmu_invalidations;
+        uint32_t                last_set_pages;
         struct list_head        list;
 };
 
@@ -620,6 +623,8 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
         if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
                 flags |= FOLL_WRITE;
 
+        down_read(&current->mm->mmap_sem);
+
         if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
                 /* check that we only use anonymous memory
                    to prevent problems with writeback */
@@ -627,8 +632,10 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
                 struct vm_area_struct *vma;
 
                 vma = find_vma(gtt->usermm, gtt->userptr);
-                if (!vma || vma->vm_file || vma->vm_end < end)
+                if (!vma || vma->vm_file || vma->vm_end < end) {
+                        up_read(&current->mm->mmap_sem);
                         return -EPERM;
+                }
         }
 
         do {
@@ -655,13 +662,47 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
 
         } while (pinned < ttm->num_pages);
 
+        up_read(&current->mm->mmap_sem);
         return 0;
 
 release_pages:
         release_pages(pages, pinned, 0);
+        up_read(&current->mm->mmap_sem);
         return r;
 }
 
+void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
+{
+        struct amdgpu_ttm_tt *gtt = (void *)ttm;
+        unsigned i;
+
+        gtt->last_set_pages = atomic_read(&gtt->mmu_invalidations);
+        for (i = 0; i < ttm->num_pages; ++i) {
+                if (ttm->pages[i])
+                        put_page(ttm->pages[i]);
+
+                ttm->pages[i] = pages ? pages[i] : NULL;
+        }
+}
+
+void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm)
+{
+        struct amdgpu_ttm_tt *gtt = (void *)ttm;
+        unsigned i;
+
+        for (i = 0; i < ttm->num_pages; ++i) {
+                struct page *page = ttm->pages[i];
+
+                if (!page)
+                        continue;
+
+                if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
+                        set_page_dirty(page);
+
+                mark_page_accessed(page);
+        }
+}
+
 /* prepare the sg table with the user pages */
 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
 {
@@ -699,7 +740,6 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
 {
         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
         struct amdgpu_ttm_tt *gtt = (void *)ttm;
-        struct sg_page_iter sg_iter;
 
         int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
         enum dma_data_direction direction = write ?
@@ -712,47 +752,16 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
         /* free the sg table and pages again */
         dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
 
-        for_each_sg_page(ttm->sg->sgl, &sg_iter, ttm->sg->nents, 0) {
-                struct page *page = sg_page_iter_page(&sg_iter);
-                if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
-                        set_page_dirty(page);
-
-                mark_page_accessed(page);
-                put_page(page);
-        }
+        amdgpu_ttm_tt_mark_user_pages(ttm);
 
         sg_free_table(ttm->sg);
 }
 
-static int amdgpu_ttm_do_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
-{
-        struct amdgpu_ttm_tt *gtt = (void *)ttm;
-        uint64_t flags;
-        int r;
-
-        spin_lock(&gtt->adev->gtt_list_lock);
-        flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, mem);
-        gtt->offset = (u64)mem->start << PAGE_SHIFT;
-        r = amdgpu_gart_bind(gtt->adev, gtt->offset, ttm->num_pages,
-                ttm->pages, gtt->ttm.dma_address, flags);
-
-        if (r) {
-                DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
-                          ttm->num_pages, gtt->offset);
-                goto error_gart_bind;
-        }
-
-        list_add_tail(&gtt->list, &gtt->adev->gtt_list);
-error_gart_bind:
-        spin_unlock(&gtt->adev->gtt_list_lock);
-        return r;
-
-}
-
 static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
                                    struct ttm_mem_reg *bo_mem)
 {
         struct amdgpu_ttm_tt *gtt = (void*)ttm;
+        uint64_t flags;
         int r = 0;
 
         if (gtt->userptr) {
@@ -772,9 +781,24 @@ static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
             bo_mem->mem_type == AMDGPU_PL_OA)
                 return -EINVAL;
 
-        if (amdgpu_gtt_mgr_is_allocated(bo_mem))
-            r = amdgpu_ttm_do_bind(ttm, bo_mem);
+        if (!amdgpu_gtt_mgr_is_allocated(bo_mem))
+                return 0;
+
+        spin_lock(&gtt->adev->gtt_list_lock);
+        flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, bo_mem);
+        gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
+        r = amdgpu_gart_bind(gtt->adev, gtt->offset, ttm->num_pages,
+                ttm->pages, gtt->ttm.dma_address, flags);
 
+        if (r) {
+                DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
+                          ttm->num_pages, gtt->offset);
+                goto error_gart_bind;
+        }
+
+        list_add_tail(&gtt->list, &gtt->adev->gtt_list);
+error_gart_bind:
+        spin_unlock(&gtt->adev->gtt_list_lock);
         return r;
 }
 
@@ -787,20 +811,38 @@ bool amdgpu_ttm_is_bound(struct ttm_tt *ttm)
 
 int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
 {
+        struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
         struct ttm_tt *ttm = bo->ttm;
+        struct ttm_mem_reg tmp;
+        struct ttm_placement placement;
+        struct ttm_place placements;
         int r;
 
         if (!ttm || amdgpu_ttm_is_bound(ttm))
                 return 0;
 
-        r = amdgpu_gtt_mgr_alloc(&bo->bdev->man[TTM_PL_TT], bo,
-                                 NULL, bo_mem);
-        if (r) {
-                DRM_ERROR("Failed to allocate GTT address space (%d)\n", r);
+        tmp = bo->mem;
+        tmp.mm_node = NULL;
+        placement.num_placement = 1;
+        placement.placement = &placements;
+        placement.num_busy_placement = 1;
+        placement.busy_placement = &placements;
+        placements.fpfn = 0;
+        placements.lpfn = adev->mc.gart_size >> PAGE_SHIFT;
+        placements.flags = bo->mem.placement | TTM_PL_FLAG_TT;
+
+        r = ttm_bo_mem_space(bo, &placement, &tmp, true, false);
+        if (unlikely(r))
                 return r;
-        }
 
-        return amdgpu_ttm_do_bind(ttm, bo_mem);
+        r = ttm_bo_move_ttm(bo, true, false, &tmp);
+        if (unlikely(r))
+                ttm_bo_mem_put(bo, &tmp);
+        else
+                bo->offset = (bo->mem.start << PAGE_SHIFT) +
+                        bo->bdev->man[bo->mem.mem_type].gpu_offset;
+
+        return r;
 }
 
 int amdgpu_ttm_recover_gart(struct amdgpu_device *adev)
@@ -892,10 +934,8 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_bo_device *bdev,
 
 static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
 {
-        struct amdgpu_device *adev;
+        struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
         struct amdgpu_ttm_tt *gtt = (void *)ttm;
-        unsigned i;
-        int r;
         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
 
         if (ttm->state != tt_unpopulated)
@@ -918,44 +958,23 @@ static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
                 return 0;
         }
 
-        adev = amdgpu_ttm_adev(ttm->bdev);
-
 #ifdef CONFIG_SWIOTLB
         if (swiotlb_nr_tbl()) {
                 return ttm_dma_populate(&gtt->ttm, adev->dev);
         }
 #endif
 
-        r = ttm_pool_populate(ttm);
-        if (r) {
-                return r;
-        }
-
-        for (i = 0; i < ttm->num_pages; i++) {
-                gtt->ttm.dma_address[i] = pci_map_page(adev->pdev, ttm->pages[i],
-                                                       0, PAGE_SIZE,
-                                                       PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(adev->pdev, gtt->ttm.dma_address[i])) {
-                        while (i--) {
-                                pci_unmap_page(adev->pdev, gtt->ttm.dma_address[i],
-                                               PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-                                gtt->ttm.dma_address[i] = 0;
-                        }
-                        ttm_pool_unpopulate(ttm);
-                        return -EFAULT;
-                }
-        }
-        return 0;
+        return ttm_populate_and_map_pages(adev->dev, &gtt->ttm);
 }
 
 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
 {
         struct amdgpu_device *adev;
         struct amdgpu_ttm_tt *gtt = (void *)ttm;
-        unsigned i;
         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
 
         if (gtt && gtt->userptr) {
+                amdgpu_ttm_tt_set_user_pages(ttm, NULL);
                 kfree(ttm->sg);
                 ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
                 return;
@@ -973,14 +992,7 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
         }
 #endif
 
-        for (i = 0; i < ttm->num_pages; i++) {
-                if (gtt->ttm.dma_address[i]) {
-                        pci_unmap_page(adev->pdev, gtt->ttm.dma_address[i],
-                                       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-                }
-        }
-
-        ttm_pool_unpopulate(ttm);
+        ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);
 }
 
 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
@@ -997,6 +1009,7 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
         spin_lock_init(&gtt->guptasklock);
         INIT_LIST_HEAD(&gtt->guptasks);
         atomic_set(&gtt->mmu_invalidations, 0);
+        gtt->last_set_pages = 0;
 
         return 0;
 }
@@ -1049,6 +1062,16 @@ bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
         return prev_invalidated != *last_invalidated;
 }
 
+bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
+{
+        struct amdgpu_ttm_tt *gtt = (void *)ttm;
+
+        if (gtt == NULL || !gtt->userptr)
+                return false;
+
+        return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;
+}
+
 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
 {
         struct amdgpu_ttm_tt *gtt = (void *)ttm;
@@ -1148,14 +1171,14 @@ static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
                 }
 
                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
-                WREG32(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
-                WREG32(mmMM_INDEX_HI, aligned_pos >> 31);
+                WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
+                WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31);
                 if (!write || mask != 0xffffffff)
-                        value = RREG32(mmMM_DATA);
+                        value = RREG32_NO_KIQ(mmMM_DATA);
                 if (write) {
                         value &= ~mask;
                         value |= (*(uint32_t *)buf << shift) & mask;
-                        WREG32(mmMM_DATA, value);
+                        WREG32_NO_KIQ(mmMM_DATA, value);
                 }
                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
                 if (!write) {
@@ -1503,8 +1526,8 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
                        struct dma_fence **fence)
 {
         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
-        /* max_bytes applies to SDMA_OP_PTEPDE as well as SDMA_OP_CONST_FILL*/
-        uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
+        uint32_t max_bytes = 8 *
+                        adev->vm_manager.vm_pte_funcs->set_max_nums_pte_pde;
         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 
         struct drm_mm_node *mm_node;
@@ -1536,8 +1559,8 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
                 ++mm_node;
         }
 
-        /* 10 double words for each SDMA_OP_PTEPDE cmd */
-        num_dw = num_loops * 10;
+        /* num of dwords for each SDMA_OP_PTEPDE cmd */
+        num_dw = num_loops * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw;
 
         /* for IB padding */
         num_dw += 64;
@@ -1597,32 +1620,16 @@ error_free:
 
 #if defined(CONFIG_DEBUG_FS)
 
-extern void amdgpu_gtt_mgr_print(struct seq_file *m, struct ttm_mem_type_manager
-                                 *man);
 static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *)m->private;
         unsigned ttm_pl = *(int *)node->info_ent->data;
         struct drm_device *dev = node->minor->dev;
         struct amdgpu_device *adev = dev->dev_private;
-        struct drm_mm *mm = (struct drm_mm *)adev->mman.bdev.man[ttm_pl].priv;
-        struct ttm_bo_global *glob = adev->mman.bdev.glob;
+        struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl];
         struct drm_printer p = drm_seq_file_printer(m);
 
-        spin_lock(&glob->lru_lock);
-        drm_mm_print(mm, &p);
-        spin_unlock(&glob->lru_lock);
-        switch (ttm_pl) {
-        case TTM_PL_VRAM:
-                seq_printf(m, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n",
-                           adev->mman.bdev.man[ttm_pl].size,
-                           (u64)atomic64_read(&adev->vram_usage) >> 20,
-                           (u64)atomic64_read(&adev->vram_vis_usage) >> 20);
-                break;
-        case TTM_PL_TT:
-                amdgpu_gtt_mgr_print(m, &adev->mman.bdev.man[TTM_PL_TT]);
-                break;
-        }
+        man->func->debug(man, &p);
         return 0;
 }
 
@@ -1659,9 +1666,9 @@ static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
                         return result;
 
                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
-                WREG32(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
-                WREG32(mmMM_INDEX_HI, *pos >> 31);
-                value = RREG32(mmMM_DATA);
+                WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
+                WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
+                value = RREG32_NO_KIQ(mmMM_DATA);
                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 
                 r = put_user(value, (uint32_t *)buf);
@@ -1677,10 +1684,50 @@ static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
         return result;
 }
 
+static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
+                                    size_t size, loff_t *pos)
+{
+        struct amdgpu_device *adev = file_inode(f)->i_private;
+        ssize_t result = 0;
+        int r;
+
+        if (size & 0x3 || *pos & 0x3)
+                return -EINVAL;
+
+        if (*pos >= adev->mc.mc_vram_size)
+                return -ENXIO;
+
+        while (size) {
+                unsigned long flags;
+                uint32_t value;
+
+                if (*pos >= adev->mc.mc_vram_size)
+                        return result;
+
+                r = get_user(value, (uint32_t *)buf);
+                if (r)
+                        return r;
+
+                spin_lock_irqsave(&adev->mmio_idx_lock, flags);
+                WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
+                WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
+                WREG32_NO_KIQ(mmMM_DATA, value);
+                spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
+
+                result += 4;
+                buf += 4;
+                *pos += 4;
+                size -= 4;
+        }
+
+        return result;
+}
+
 static const struct file_operations amdgpu_ttm_vram_fops = {
         .owner = THIS_MODULE,
         .read = amdgpu_ttm_vram_read,
-        .llseek = default_llseek
+        .write = amdgpu_ttm_vram_write,
+        .llseek = default_llseek,
 };
 
 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
@@ -1732,6 +1779,53 @@ static const struct file_operations amdgpu_ttm_gtt_fops = {
 
 #endif
 
+static ssize_t amdgpu_iova_to_phys_read(struct file *f, char __user *buf,
+                                   size_t size, loff_t *pos)
+{
+        struct amdgpu_device *adev = file_inode(f)->i_private;
+        int r;
+        uint64_t phys;
+        struct iommu_domain *dom;
+
+        // always return 8 bytes
+        if (size != 8)
+                return -EINVAL;
+
+        // only accept page addresses
+        if (*pos & 0xFFF)
+                return -EINVAL;
+
+        dom = iommu_get_domain_for_dev(adev->dev);
+        if (dom)
+                phys = iommu_iova_to_phys(dom, *pos);
+        else
+                phys = *pos;
+
+        r = copy_to_user(buf, &phys, 8);
+        if (r)
+                return -EFAULT;
+
+        return 8;
+}
+
+static const struct file_operations amdgpu_ttm_iova_fops = {
+        .owner = THIS_MODULE,
+        .read = amdgpu_iova_to_phys_read,
+        .llseek = default_llseek
+};
+
+static const struct {
+        char *name;
+        const struct file_operations *fops;
+        int domain;
+} ttm_debugfs_entries[] = {
+        { "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM },
+#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
+        { "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT },
+#endif
+        { "amdgpu_iova", &amdgpu_ttm_iova_fops, TTM_PL_SYSTEM },
+};
+
 #endif
 
 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
@@ -1742,22 +1836,21 @@ static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
         struct drm_minor *minor = adev->ddev->primary;
         struct dentry *ent, *root = minor->debugfs_root;
 
-        ent = debugfs_create_file("amdgpu_vram", S_IFREG | S_IRUGO, root,
-                                  adev, &amdgpu_ttm_vram_fops);
-        if (IS_ERR(ent))
-                return PTR_ERR(ent);
-        i_size_write(ent->d_inode, adev->mc.mc_vram_size);
-        adev->mman.vram = ent;
-
-#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
-        ent = debugfs_create_file("amdgpu_gtt", S_IFREG | S_IRUGO, root,
-                                  adev, &amdgpu_ttm_gtt_fops);
-        if (IS_ERR(ent))
-                return PTR_ERR(ent);
-        i_size_write(ent->d_inode, adev->mc.gart_size);
-        adev->mman.gtt = ent;
+        for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) {
+                ent = debugfs_create_file(
+                                ttm_debugfs_entries[count].name,
+                                S_IFREG | S_IRUGO, root,
+                                adev,
+                                ttm_debugfs_entries[count].fops);
+                if (IS_ERR(ent))
+                        return PTR_ERR(ent);
+                if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM)
+                        i_size_write(ent->d_inode, adev->mc.mc_vram_size);
+                else if (ttm_debugfs_entries[count].domain == TTM_PL_TT)
+                        i_size_write(ent->d_inode, adev->mc.gart_size);
+                adev->mman.debugfs_entries[count] = ent;
+        }
 
-#endif
         count = ARRAY_SIZE(amdgpu_ttm_debugfs_list);
 
 #ifdef CONFIG_SWIOTLB
@@ -1767,7 +1860,6 @@ static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
 
         return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);
 #else
-
         return 0;
 #endif
 }
@@ -1775,14 +1867,9 @@ static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
 {
 #if defined(CONFIG_DEBUG_FS)
+        unsigned i;
 
-        debugfs_remove(adev->mman.vram);
-        adev->mman.vram = NULL;
-
-#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
-        debugfs_remove(adev->mman.gtt);
-        adev->mman.gtt = NULL;
-#endif
-
+        for (i = 0; i < ARRAY_SIZE(ttm_debugfs_entries); i++)
+                debugfs_remove(adev->mman.debugfs_entries[i]);
 #endif
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 0e2399f32de7..7abae6867339 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -24,6 +24,7 @@
 #ifndef __AMDGPU_TTM_H__
 #define __AMDGPU_TTM_H__
 
+#include "amdgpu.h"
 #include "gpu_scheduler.h"
 
 #define AMDGPU_PL_GDS           (TTM_PL_PRIV + 0)
@@ -45,8 +46,7 @@ struct amdgpu_mman {
         bool                            initialized;
 
 #if defined(CONFIG_DEBUG_FS)
-        struct dentry                   *vram;
-        struct dentry                   *gtt;
+        struct dentry                   *debugfs_entries[8];
 #endif
 
         /* buffer handling */
@@ -62,10 +62,10 @@ extern const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func;
 extern const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func;
 
 bool amdgpu_gtt_mgr_is_allocated(struct ttm_mem_reg *mem);
-int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
-                         struct ttm_buffer_object *tbo,
-                         const struct ttm_place *place,
-                         struct ttm_mem_reg *mem);
+uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man);
+
+uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man);
+uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man);
 
 int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
                        uint64_t dst_offset, uint32_t byte_count,
@@ -82,4 +82,20 @@ bool amdgpu_ttm_is_bound(struct ttm_tt *ttm);
 int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem);
 int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);
 
+int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
+void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);
+void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm);
+int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
+                                     uint32_t flags);
+bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm);
+struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm);
+bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
+                                  unsigned long end);
+bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
+                                       int *last_invalidated);
+bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm);
+bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
+uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
+                                 struct ttm_mem_reg *mem);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 36c763310df5..65649026b836 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -270,12 +270,8 @@ amdgpu_ucode_get_load_type(struct amdgpu_device *adev, int load_type)
                 else
                         return AMDGPU_FW_LOAD_SMU;
         case CHIP_VEGA10:
-                if (!load_type)
-                        return AMDGPU_FW_LOAD_DIRECT;
-                else
-                        return AMDGPU_FW_LOAD_PSP;
         case CHIP_RAVEN:
-                if (load_type != 2)
+                if (!load_type)
                         return AMDGPU_FW_LOAD_DIRECT;
                 else
                         return AMDGPU_FW_LOAD_PSP;
@@ -364,8 +360,6 @@ static int amdgpu_ucode_patch_jt(struct amdgpu_firmware_info *ucode,
 int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 {
         struct amdgpu_bo **bo = &adev->firmware.fw_buf;
-        uint64_t fw_mc_addr;
-        void *fw_buf_ptr = NULL;
         uint64_t fw_offset = 0;
         int i, err;
         struct amdgpu_firmware_info *ucode = NULL;
@@ -376,37 +370,39 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
                 return 0;
         }
 
-        err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true,
-                                amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
-                                AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-                                NULL, NULL, 0, bo);
-        if (err) {
-                dev_err(adev->dev, "(%d) Firmware buffer allocate failed\n", err);
-                goto failed;
-        }
+        if (!amdgpu_sriov_vf(adev) || !adev->in_sriov_reset) {
+                err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true,
+                                        amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
+                                        AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
+                                        NULL, NULL, 0, bo);
+                if (err) {
+                        dev_err(adev->dev, "(%d) Firmware buffer allocate failed\n", err);
+                        goto failed;
+                }
 
-        err = amdgpu_bo_reserve(*bo, false);
-        if (err) {
-                dev_err(adev->dev, "(%d) Firmware buffer reserve failed\n", err);
-                goto failed_reserve;
-        }
+                err = amdgpu_bo_reserve(*bo, false);
+                if (err) {
+                        dev_err(adev->dev, "(%d) Firmware buffer reserve failed\n", err);
+                        goto failed_reserve;
+                }
 
-        err = amdgpu_bo_pin(*bo, amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
-                                &fw_mc_addr);
-        if (err) {
-                dev_err(adev->dev, "(%d) Firmware buffer pin failed\n", err);
-                goto failed_pin;
-        }
+                err = amdgpu_bo_pin(*bo, amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
+                                        &adev->firmware.fw_buf_mc);
+                if (err) {
+                        dev_err(adev->dev, "(%d) Firmware buffer pin failed\n", err);
+                        goto failed_pin;
+                }
 
-        err = amdgpu_bo_kmap(*bo, &fw_buf_ptr);
-        if (err) {
-                dev_err(adev->dev, "(%d) Firmware buffer kmap failed\n", err);
-                goto failed_kmap;
-        }
+                err = amdgpu_bo_kmap(*bo, &adev->firmware.fw_buf_ptr);
+                if (err) {
+                        dev_err(adev->dev, "(%d) Firmware buffer kmap failed\n", err);
+                        goto failed_kmap;
+                }
 
-        amdgpu_bo_unreserve(*bo);
+                amdgpu_bo_unreserve(*bo);
+        }
 
-        memset(fw_buf_ptr, 0, adev->firmware.fw_size);
+        memset(adev->firmware.fw_buf_ptr, 0, adev->firmware.fw_size);
 
         /*
          * if SMU loaded firmware, it needn't add SMC, UVD, and VCE
@@ -425,14 +421,14 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
                 ucode = &adev->firmware.ucode[i];
                 if (ucode->fw) {
                         header = (const struct common_firmware_header *)ucode->fw->data;
-                        amdgpu_ucode_init_single_fw(adev, ucode, fw_mc_addr + fw_offset,
-                                                    (void *)((uint8_t *)fw_buf_ptr + fw_offset));
+                        amdgpu_ucode_init_single_fw(adev, ucode, adev->firmware.fw_buf_mc + fw_offset,
+                                                    adev->firmware.fw_buf_ptr + fw_offset);
                         if (i == AMDGPU_UCODE_ID_CP_MEC1 &&
                             adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
                                 const struct gfx_firmware_header_v1_0 *cp_hdr;
                                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data;
-                                amdgpu_ucode_patch_jt(ucode, fw_mc_addr + fw_offset,
-                                                    fw_buf_ptr + fw_offset);
+                                amdgpu_ucode_patch_jt(ucode,  adev->firmware.fw_buf_mc + fw_offset,
+                                                    adev->firmware.fw_buf_ptr + fw_offset);
                                 fw_offset += ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE);
                         }
                         fw_offset += ALIGN(ucode->ucode_size, PAGE_SIZE);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index aefecf6c1e7b..e8bd50cf9785 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -269,6 +269,7 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 
 int amdgpu_uvd_sw_fini(struct amdgpu_device *adev)
 {
+        int i;
         kfree(adev->uvd.saved_bo);
 
         amd_sched_entity_fini(&adev->uvd.ring.sched, &adev->uvd.entity);
@@ -279,6 +280,9 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev)
 
         amdgpu_ring_fini(&adev->uvd.ring);
 
+        for (i = 0; i < AMDGPU_MAX_UVD_ENC_RINGS; ++i)
+                amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
+
         release_firmware(adev->uvd.fw);
 
         return 0;
@@ -410,10 +414,10 @@ static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx)
         uint64_t addr = amdgpu_uvd_get_addr_from_ctx(ctx);
         int r = 0;
 
-        mapping = amdgpu_cs_find_mapping(ctx->parser, addr, &bo);
-        if (mapping == NULL) {
+        r = amdgpu_cs_find_mapping(ctx->parser, addr, &bo, &mapping);
+        if (r) {
                 DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
-                return -EINVAL;
+                return r;
         }
 
         if (!ctx->parser->adev->uvd.address_64_bit) {
@@ -588,6 +592,10 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg,
                 }
                 break;
 
+        case 8: /* MJPEG */
+                min_dpb_size = 0;
+                break;
+
         case 16: /* H265 */
                 image_size = (ALIGN(width, 16) * ALIGN(height, 16) * 3) / 2;
                 image_size = ALIGN(image_size, 256);
@@ -733,10 +741,10 @@ static int amdgpu_uvd_cs_pass2(struct amdgpu_uvd_cs_ctx *ctx)
         uint64_t addr = amdgpu_uvd_get_addr_from_ctx(ctx);
         int r;
 
-        mapping = amdgpu_cs_find_mapping(ctx->parser, addr, &bo);
-        if (mapping == NULL) {
+        r = amdgpu_cs_find_mapping(ctx->parser, addr, &bo, &mapping);
+        if (r) {
                 DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
-                return -EINVAL;
+                return r;
         }
 
         start = amdgpu_bo_gpu_offset(bo);
@@ -913,10 +921,6 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx)
                 return -EINVAL;
         }
 
-        r = amdgpu_cs_sysvm_access_required(parser);
-        if (r)
-                return r;
-
         ctx.parser = parser;
         ctx.buf_sizes = buf_sizes;
         ctx.ib_idx = ib_idx;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index c855366521ab..b46280c1279f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -559,6 +559,7 @@ static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx,
         struct amdgpu_bo_va_mapping *mapping;
         struct amdgpu_bo *bo;
         uint64_t addr;
+        int r;
 
         if (index == 0xffffffff)
                 index = 0;
@@ -567,11 +568,11 @@ static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx,
                ((uint64_t)amdgpu_get_ib_value(p, ib_idx, hi)) << 32;
         addr += ((uint64_t)size) * ((uint64_t)index);
 
-        mapping = amdgpu_cs_find_mapping(p, addr, &bo);
-        if (mapping == NULL) {
+        r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping);
+        if (r) {
                 DRM_ERROR("Can't find BO for addr 0x%010Lx %d %d %d %d\n",
                           addr, lo, hi, size, index);
-                return -EINVAL;
+                return r;
         }
 
         if ((addr + (uint64_t)size) >
@@ -652,10 +653,6 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
         p->job->vm = NULL;
         ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
 
-        r = amdgpu_cs_sysvm_access_required(p);
-        if (r)
-                return r;
-
         while (idx < ib->length_dw) {
                 uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
                 uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 8a081e162d13..ab05121b9272 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -46,14 +46,14 @@ int amdgpu_allocate_static_csa(struct amdgpu_device *adev)
  * address within META_DATA init package to support SRIOV gfx preemption.
  */
 
-int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm)
+int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+                          struct amdgpu_bo_va **bo_va)
 {
-        int r;
-        struct amdgpu_bo_va *bo_va;
         struct ww_acquire_ctx ticket;
         struct list_head list;
         struct amdgpu_bo_list_entry pd;
         struct ttm_validate_buffer csa_tv;
+        int r;
 
         INIT_LIST_HEAD(&list);
         INIT_LIST_HEAD(&csa_tv.head);
@@ -69,34 +69,33 @@ int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm)
                 return r;
         }
 
-        bo_va = amdgpu_vm_bo_add(adev, vm, adev->virt.csa_obj);
-        if (!bo_va) {
+        *bo_va = amdgpu_vm_bo_add(adev, vm, adev->virt.csa_obj);
+        if (!*bo_va) {
                 ttm_eu_backoff_reservation(&ticket, &list);
                 DRM_ERROR("failed to create bo_va for static CSA\n");
                 return -ENOMEM;
         }
 
-        r = amdgpu_vm_alloc_pts(adev, bo_va->vm, AMDGPU_CSA_VADDR,
-                                   AMDGPU_CSA_SIZE);
+        r = amdgpu_vm_alloc_pts(adev, (*bo_va)->base.vm, AMDGPU_CSA_VADDR,
+                                AMDGPU_CSA_SIZE);
         if (r) {
                 DRM_ERROR("failed to allocate pts for static CSA, err=%d\n", r);
-                amdgpu_vm_bo_rmv(adev, bo_va);
+                amdgpu_vm_bo_rmv(adev, *bo_va);
                 ttm_eu_backoff_reservation(&ticket, &list);
                 return r;
         }
 
-        r = amdgpu_vm_bo_map(adev, bo_va, AMDGPU_CSA_VADDR, 0,AMDGPU_CSA_SIZE,
-                                                AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
-                                                AMDGPU_PTE_EXECUTABLE);
+        r = amdgpu_vm_bo_map(adev, *bo_va, AMDGPU_CSA_VADDR, 0, AMDGPU_CSA_SIZE,
+                             AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
+                             AMDGPU_PTE_EXECUTABLE);
 
         if (r) {
                 DRM_ERROR("failed to do bo_map on static CSA, err=%d\n", r);
-                amdgpu_vm_bo_rmv(adev, bo_va);
+                amdgpu_vm_bo_rmv(adev, *bo_va);
                 ttm_eu_backoff_reservation(&ticket, &list);
                 return r;
         }
 
-        vm->csa_bo_va = bo_va;
         ttm_eu_backoff_reservation(&ticket, &list);
         return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index e5b1baf387c1..afcfb8bcfb65 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -90,7 +90,8 @@ static inline bool is_virtual_machine(void)
 
 struct amdgpu_vm;
 int amdgpu_allocate_static_csa(struct amdgpu_device *adev);
-int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+                          struct amdgpu_bo_va **bo_va);
 void amdgpu_virt_init_setting(struct amdgpu_device *adev);
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
 void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 9ce36652029e..bbcc67038203 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -27,12 +27,59 @@
  */
 #include <linux/dma-fence-array.h>
 #include <linux/interval_tree_generic.h>
+#include <linux/idr.h>
 #include <drm/drmP.h>
 #include <drm/amdgpu_drm.h>
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
 /*
+ * PASID manager
+ *
+ * PASIDs are global address space identifiers that can be shared
+ * between the GPU, an IOMMU and the driver. VMs on different devices
+ * may use the same PASID if they share the same address
+ * space. Therefore PASIDs are allocated using a global IDA. VMs are
+ * looked up from the PASID per amdgpu_device.
+ */
+static DEFINE_IDA(amdgpu_vm_pasid_ida);
+
+/**
+ * amdgpu_vm_alloc_pasid - Allocate a PASID
+ * @bits: Maximum width of the PASID in bits, must be at least 1
+ *
+ * Allocates a PASID of the given width while keeping smaller PASIDs
+ * available if possible.
+ *
+ * Returns a positive integer on success. Returns %-EINVAL if bits==0.
+ * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on
+ * memory allocation failure.
+ */
+int amdgpu_vm_alloc_pasid(unsigned int bits)
+{
+        int pasid = -EINVAL;
+
+        for (bits = min(bits, 31U); bits > 0; bits--) {
+                pasid = ida_simple_get(&amdgpu_vm_pasid_ida,
+                                       1U << (bits - 1), 1U << bits,
+                                       GFP_KERNEL);
+                if (pasid != -ENOSPC)
+                        break;
+        }
+
+        return pasid;
+}
+
+/**
+ * amdgpu_vm_free_pasid - Free a PASID
+ * @pasid: PASID to free
+ */
+void amdgpu_vm_free_pasid(unsigned int pasid)
+{
+        ida_simple_remove(&amdgpu_vm_pasid_ida, pasid);
+}
+
+/*
  * GPUVM
  * GPUVM is similar to the legacy gart on older asics, however
  * rather than there being a single global gart table
@@ -140,7 +187,7 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
                          struct list_head *validated,
                          struct amdgpu_bo_list_entry *entry)
 {
-        entry->robj = vm->root.bo;
+        entry->robj = vm->root.base.bo;
         entry->priority = 0;
         entry->tv.bo = &entry->robj->tbo;
         entry->tv.shared = true;
@@ -149,54 +196,6 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 }
 
 /**
- * amdgpu_vm_validate_layer - validate a single page table level
- *
- * @parent: parent page table level
- * @validate: callback to do the validation
- * @param: parameter for the validation callback
- *
- * Validate the page table BOs on command submission if neccessary.
- */
-static int amdgpu_vm_validate_level(struct amdgpu_vm_pt *parent,
-                                    int (*validate)(void *, struct amdgpu_bo *),
-                                    void *param, bool use_cpu_for_update)
-{
-        unsigned i;
-        int r;
-
-        if (use_cpu_for_update) {
-                r = amdgpu_bo_kmap(parent->bo, NULL);
-                if (r)
-                        return r;
-        }
-
-        if (!parent->entries)
-                return 0;
-
-        for (i = 0; i <= parent->last_entry_used; ++i) {
-                struct amdgpu_vm_pt *entry = &parent->entries[i];
-
-                if (!entry->bo)
-                        continue;
-
-                r = validate(param, entry->bo);
-                if (r)
-                        return r;
-
-                /*
-                 * Recurse into the sub directory. This is harmless because we
-                 * have only a maximum of 5 layers.
-                 */
-                r = amdgpu_vm_validate_level(entry, validate, param,
-                                             use_cpu_for_update);
-                if (r)
-                        return r;
-        }
-
-        return r;
-}
-
-/**
  * amdgpu_vm_validate_pt_bos - validate the page table BOs
  *
  * @adev: amdgpu device pointer
@@ -210,64 +209,70 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                               int (*validate)(void *p, struct amdgpu_bo *bo),
                               void *param)
 {
-        uint64_t num_evictions;
-
-        /* We only need to validate the page tables
-         * if they aren't already valid.
-         */
-        num_evictions = atomic64_read(&adev->num_evictions);
-        if (num_evictions == vm->last_eviction_counter)
-                return 0;
+        struct ttm_bo_global *glob = adev->mman.bdev.glob;
+        int r;
 
-        return amdgpu_vm_validate_level(&vm->root, validate, param,
-                                        vm->use_cpu_for_update);
-}
+        spin_lock(&vm->status_lock);
+        while (!list_empty(&vm->evicted)) {
+                struct amdgpu_vm_bo_base *bo_base;
+                struct amdgpu_bo *bo;
 
-/**
- * amdgpu_vm_move_level_in_lru - move one level of PT BOs to the LRU tail
- *
- * @adev: amdgpu device instance
- * @vm: vm providing the BOs
- *
- * Move the PT BOs to the tail of the LRU.
- */
-static void amdgpu_vm_move_level_in_lru(struct amdgpu_vm_pt *parent)
-{
-        unsigned i;
+                bo_base = list_first_entry(&vm->evicted,
+                                           struct amdgpu_vm_bo_base,
+                                           vm_status);
+                spin_unlock(&vm->status_lock);
 
-        if (!parent->entries)
-                return;
+                bo = bo_base->bo;
+                BUG_ON(!bo);
+                if (bo->parent) {
+                        r = validate(param, bo);
+                        if (r)
+                                return r;
 
-        for (i = 0; i <= parent->last_entry_used; ++i) {
-                struct amdgpu_vm_pt *entry = &parent->entries[i];
+                        spin_lock(&glob->lru_lock);
+                        ttm_bo_move_to_lru_tail(&bo->tbo);
+                        if (bo->shadow)
+                                ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
+                        spin_unlock(&glob->lru_lock);
+                }
 
-                if (!entry->bo)
-                        continue;
+                if (bo->tbo.type == ttm_bo_type_kernel &&
+                    vm->use_cpu_for_update) {
+                        r = amdgpu_bo_kmap(bo, NULL);
+                        if (r)
+                                return r;
+                }
 
-                ttm_bo_move_to_lru_tail(&entry->bo->tbo);
-                amdgpu_vm_move_level_in_lru(entry);
+                spin_lock(&vm->status_lock);
+                if (bo->tbo.type != ttm_bo_type_kernel)
+                        list_move(&bo_base->vm_status, &vm->moved);
+                else
+                        list_move(&bo_base->vm_status, &vm->relocated);
         }
+        spin_unlock(&vm->status_lock);
+
+        return 0;
 }
 
 /**
- * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
+ * amdgpu_vm_ready - check VM is ready for updates
  *
- * @adev: amdgpu device instance
- * @vm: vm providing the BOs
+ * @vm: VM to check
  *
- * Move the PT BOs to the tail of the LRU.
+ * Check if all VM PDs/PTs are ready for updates
  */
-void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
-                                  struct amdgpu_vm *vm)
+bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 {
-        struct ttm_bo_global *glob = adev->mman.bdev.glob;
+        bool ready;
 
-        spin_lock(&glob->lru_lock);
-        amdgpu_vm_move_level_in_lru(&vm->root);
-        spin_unlock(&glob->lru_lock);
+        spin_lock(&vm->status_lock);
+        ready = list_empty(&vm->evicted);
+        spin_unlock(&vm->status_lock);
+
+        return ready;
 }
 
- /**
+/**
  * amdgpu_vm_alloc_levels - allocate the PD/PT levels
  *
  * @adev: amdgpu_device pointer
@@ -330,11 +335,11 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 
         /* walk over the address space and allocate the page tables */
         for (pt_idx = from; pt_idx <= to; ++pt_idx) {
-                struct reservation_object *resv = vm->root.bo->tbo.resv;
+                struct reservation_object *resv = vm->root.base.bo->tbo.resv;
                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
                 struct amdgpu_bo *pt;
 
-                if (!entry->bo) {
+                if (!entry->base.bo) {
                         r = amdgpu_bo_create(adev,
                                              amdgpu_vm_bo_size(adev, level),
                                              AMDGPU_GPU_PAGE_SIZE, true,
@@ -355,11 +360,15 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
                         /* Keep a reference to the root directory to avoid
                         * freeing them up in the wrong order.
                         */
-                        pt->parent = amdgpu_bo_ref(vm->root.bo);
-
-                        entry->bo = pt;
+                        pt->parent = amdgpu_bo_ref(parent->base.bo);
+
+                        entry->base.vm = vm;
+                        entry->base.bo = pt;
+                        list_add_tail(&entry->base.bo_list, &pt->va);
+                        spin_lock(&vm->status_lock);
+                        list_add(&entry->base.vm_status, &vm->relocated);
+                        spin_unlock(&vm->status_lock);
                         entry->addr = 0;
-                        entry->huge_page = false;
                 }
 
                 if (level < adev->vm_manager.num_level) {
@@ -899,8 +908,8 @@ struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 {
         struct amdgpu_bo_va *bo_va;
 
-        list_for_each_entry(bo_va, &bo->va, bo_list) {
-                if (bo_va->vm == vm) {
+        list_for_each_entry(bo_va, &bo->va, base.bo_list) {
+                if (bo_va->base.vm == vm) {
                         return bo_va;
                 }
         }
@@ -1025,7 +1034,7 @@ static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         int r;
 
         amdgpu_sync_create(&sync);
-        amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.resv, owner);
+        amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner);
         r = amdgpu_sync_wait(&sync, true);
         amdgpu_sync_free(&sync);
 
@@ -1044,18 +1053,17 @@ static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  */
 static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                                   struct amdgpu_vm *vm,
-                                  struct amdgpu_vm_pt *parent,
-                                  unsigned level)
+                                  struct amdgpu_vm_pt *parent)
 {
         struct amdgpu_bo *shadow;
         struct amdgpu_ring *ring = NULL;
         uint64_t pd_addr, shadow_addr = 0;
-        uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
         uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
         unsigned count = 0, pt_idx, ndw = 0;
         struct amdgpu_job *job;
         struct amdgpu_pte_update_params params;
         struct dma_fence *fence = NULL;
+        uint32_t incr;
 
         int r;
 
@@ -1064,21 +1072,16 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
         memset(&params, 0, sizeof(params));
         params.adev = adev;
-        shadow = parent->bo->shadow;
+        shadow = parent->base.bo->shadow;
 
         if (vm->use_cpu_for_update) {
-                pd_addr = (unsigned long)amdgpu_bo_kptr(parent->bo);
+                pd_addr = (unsigned long)amdgpu_bo_kptr(parent->base.bo);
                 r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
                 if (unlikely(r))
                         return r;
 
                 params.func = amdgpu_vm_cpu_set_ptes;
         } else {
-                if (shadow) {
-                        r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
-                        if (r)
-                                return r;
-                }
                 ring = container_of(vm->entity.sched, struct amdgpu_ring,
                                     sched);
 
@@ -1088,7 +1091,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                 /* assume the worst case */
                 ndw += parent->last_entry_used * 6;
 
-                pd_addr = amdgpu_bo_gpu_offset(parent->bo);
+                pd_addr = amdgpu_bo_gpu_offset(parent->base.bo);
 
                 if (shadow) {
                         shadow_addr = amdgpu_bo_gpu_offset(shadow);
@@ -1108,30 +1111,28 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
         /* walk over the address space and update the directory */
         for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
-                struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
+                struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
+                struct amdgpu_bo *bo = entry->base.bo;
                 uint64_t pde, pt;
 
                 if (bo == NULL)
                         continue;
 
-                if (bo->shadow) {
-                        struct amdgpu_bo *pt_shadow = bo->shadow;
-
-                        r = amdgpu_ttm_bind(&pt_shadow->tbo,
-                                            &pt_shadow->tbo.mem);
-                        if (r)
-                                return r;
-                }
+                spin_lock(&vm->status_lock);
+                list_del_init(&entry->base.vm_status);
+                spin_unlock(&vm->status_lock);
 
                 pt = amdgpu_bo_gpu_offset(bo);
                 pt = amdgpu_gart_get_vm_pde(adev, pt);
-                if (parent->entries[pt_idx].addr == pt ||
-                    parent->entries[pt_idx].huge_page)
+                /* Don't update huge pages here */
+                if ((parent->entries[pt_idx].addr & AMDGPU_PDE_PTE) ||
+                    parent->entries[pt_idx].addr == (pt | AMDGPU_PTE_VALID))
                         continue;
 
-                parent->entries[pt_idx].addr = pt;
+                parent->entries[pt_idx].addr = pt | AMDGPU_PTE_VALID;
 
                 pde = pd_addr + pt_idx * 8;
+                incr = amdgpu_bo_size(bo);
                 if (((last_pde + 8 * count) != pde) ||
                     ((last_pt + incr * count) != pt) ||
                     (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
@@ -1159,7 +1160,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
         }
 
         if (count) {
-                if (vm->root.bo->shadow)
+                if (vm->root.base.bo->shadow)
                         params.func(&params, last_shadow, last_pt,
                                     count, incr, AMDGPU_PTE_VALID);
 
@@ -1172,7 +1173,8 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                         amdgpu_job_free(job);
                 } else {
                         amdgpu_ring_pad_ib(ring, params.ib);
-                        amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
+                        amdgpu_sync_resv(adev, &job->sync,
+                                         parent->base.bo->tbo.resv,
                                          AMDGPU_FENCE_OWNER_VM);
                         if (shadow)
                                 amdgpu_sync_resv(adev, &job->sync,
@@ -1185,26 +1187,11 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                         if (r)
                                 goto error_free;
 
-                        amdgpu_bo_fence(parent->bo, fence, true);
-                        dma_fence_put(vm->last_dir_update);
-                        vm->last_dir_update = dma_fence_get(fence);
-                        dma_fence_put(fence);
+                        amdgpu_bo_fence(parent->base.bo, fence, true);
+                        dma_fence_put(vm->last_update);
+                        vm->last_update = fence;
                 }
         }
-        /*
-         * Recurse into the subdirectories. This recursion is harmless because
-         * we only have a maximum of 5 layers.
-         */
-        for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
-                struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
-
-                if (!entry->bo)
-                        continue;
-
-                r = amdgpu_vm_update_level(adev, vm, entry, level + 1);
-                if (r)
-                        return r;
-        }
 
         return 0;
 
@@ -1220,7 +1207,8 @@ error_free:
  *
  * Mark all PD level as invalid after an error.
  */
-static void amdgpu_vm_invalidate_level(struct amdgpu_vm_pt *parent)
+static void amdgpu_vm_invalidate_level(struct amdgpu_vm *vm,
+                                       struct amdgpu_vm_pt *parent)
 {
         unsigned pt_idx;
 
@@ -1231,11 +1219,15 @@ static void amdgpu_vm_invalidate_level(struct amdgpu_vm_pt *parent)
         for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 
-                if (!entry->bo)
+                if (!entry->base.bo)
                         continue;
 
                 entry->addr = ~0ULL;
-                amdgpu_vm_invalidate_level(entry);
+                spin_lock(&vm->status_lock);
+                if (list_empty(&entry->base.vm_status))
+                        list_add(&entry->base.vm_status, &vm->relocated);
+                spin_unlock(&vm->status_lock);
+                amdgpu_vm_invalidate_level(vm, entry);
         }
 }
 
@@ -1253,9 +1245,38 @@ int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 {
         int r;
 
-        r = amdgpu_vm_update_level(adev, vm, &vm->root, 0);
-        if (r)
-                amdgpu_vm_invalidate_level(&vm->root);
+        spin_lock(&vm->status_lock);
+        while (!list_empty(&vm->relocated)) {
+                struct amdgpu_vm_bo_base *bo_base;
+                struct amdgpu_bo *bo;
+
+                bo_base = list_first_entry(&vm->relocated,
+                                           struct amdgpu_vm_bo_base,
+                                           vm_status);
+                spin_unlock(&vm->status_lock);
+
+                bo = bo_base->bo->parent;
+                if (bo) {
+                        struct amdgpu_vm_bo_base *parent;
+                        struct amdgpu_vm_pt *pt;
+
+                        parent = list_first_entry(&bo->va,
+                                                  struct amdgpu_vm_bo_base,
+                                                  bo_list);
+                        pt = container_of(parent, struct amdgpu_vm_pt, base);
+
+                        r = amdgpu_vm_update_level(adev, vm, pt);
+                        if (r) {
+                                amdgpu_vm_invalidate_level(vm, &vm->root);
+                                return r;
+                        }
+                        spin_lock(&vm->status_lock);
+                } else {
+                        spin_lock(&vm->status_lock);
+                        list_del_init(&bo_base->vm_status);
+                }
+        }
+        spin_unlock(&vm->status_lock);
 
         if (vm->use_cpu_for_update) {
                 /* Flush HDP */
@@ -1286,7 +1307,7 @@ void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
         *entry = &p->vm->root;
         while ((*entry)->entries) {
                 idx = addr >> (p->adev->vm_manager.block_size * level--);
-                idx %= amdgpu_bo_size((*entry)->bo) / 8;
+                idx %= amdgpu_bo_size((*entry)->base.bo) / 8;
                 *parent = *entry;
                 *entry = &(*entry)->entries[idx];
         }
@@ -1307,55 +1328,62 @@ void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
  *
  * Check if we can update the PD with a huge page.
  */
-static int amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
-                                       struct amdgpu_vm_pt *entry,
-                                       struct amdgpu_vm_pt *parent,
-                                       unsigned nptes, uint64_t dst,
-                                       uint64_t flags)
+static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
+                                        struct amdgpu_vm_pt *entry,
+                                        struct amdgpu_vm_pt *parent,
+                                        unsigned nptes, uint64_t dst,
+                                        uint64_t flags)
 {
         bool use_cpu_update = (p->func == amdgpu_vm_cpu_set_ptes);
         uint64_t pd_addr, pde;
-        int r;
 
         /* In the case of a mixed PT the PDE must point to it*/
         if (p->adev->asic_type < CHIP_VEGA10 ||
             nptes != AMDGPU_VM_PTE_COUNT(p->adev) ||
-            p->func == amdgpu_vm_do_copy_ptes ||
+            p->src ||
             !(flags & AMDGPU_PTE_VALID)) {
 
-                dst = amdgpu_bo_gpu_offset(entry->bo);
+                dst = amdgpu_bo_gpu_offset(entry->base.bo);
                 dst = amdgpu_gart_get_vm_pde(p->adev, dst);
                 flags = AMDGPU_PTE_VALID;
         } else {
+                /* Set the huge page flag to stop scanning at this PDE */
                 flags |= AMDGPU_PDE_PTE;
         }
 
-        if (entry->addr == dst &&
-            entry->huge_page == !!(flags & AMDGPU_PDE_PTE))
-                return 0;
+        if (entry->addr == (dst | flags))
+                return;
 
-        entry->addr = dst;
-        entry->huge_page = !!(flags & AMDGPU_PDE_PTE);
+        entry->addr = (dst | flags);
 
         if (use_cpu_update) {
-                r = amdgpu_bo_kmap(parent->bo, (void *)&pd_addr);
-                if (r)
-                        return r;
+                /* In case a huge page is replaced with a system
+                 * memory mapping, p->pages_addr != NULL and
+                 * amdgpu_vm_cpu_set_ptes would try to translate dst
+                 * through amdgpu_vm_map_gart. But dst is already a
+                 * GPU address (of the page table). Disable
+                 * amdgpu_vm_map_gart temporarily.
+                 */
+                dma_addr_t *tmp;
+
+                tmp = p->pages_addr;
+                p->pages_addr = NULL;
 
+                pd_addr = (unsigned long)amdgpu_bo_kptr(parent->base.bo);
                 pde = pd_addr + (entry - parent->entries) * 8;
                 amdgpu_vm_cpu_set_ptes(p, pde, dst, 1, 0, flags);
+
+                p->pages_addr = tmp;
         } else {
-                if (parent->bo->shadow) {
-                        pd_addr = amdgpu_bo_gpu_offset(parent->bo->shadow);
+                if (parent->base.bo->shadow) {
+                        pd_addr = amdgpu_bo_gpu_offset(parent->base.bo->shadow);
                         pde = pd_addr + (entry - parent->entries) * 8;
                         amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
                 }
-                pd_addr = amdgpu_bo_gpu_offset(parent->bo);
+                pd_addr = amdgpu_bo_gpu_offset(parent->base.bo);
                 pde = pd_addr + (entry - parent->entries) * 8;
                 amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
         }
-
-        return 0;
 }
 
 /**
@@ -1382,7 +1410,6 @@ static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
         struct amdgpu_bo *pt;
         unsigned nptes;
         bool use_cpu_update = (params->func == amdgpu_vm_cpu_set_ptes);
-        int r;
 
         /* walk over the address space and update the page tables */
         for (addr = start; addr < end; addr += nptes,
@@ -1398,15 +1425,13 @@ static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
                 else
                         nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
 
-                r = amdgpu_vm_handle_huge_pages(params, entry, parent,
-                                                nptes, dst, flags);
-                if (r)
-                        return r;
-
-                if (entry->huge_page)
+                amdgpu_vm_handle_huge_pages(params, entry, parent,
+                                            nptes, dst, flags);
+                /* We don't need to update PTEs for huge pages */
+                if (entry->addr & AMDGPU_PDE_PTE)
                         continue;
 
-                pt = entry->bo;
+                pt = entry->base.bo;
                 if (use_cpu_update) {
                         pe_start = (unsigned long)amdgpu_bo_kptr(pt);
                 } else {
@@ -1442,8 +1467,6 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
                                 uint64_t start, uint64_t end,
                                 uint64_t dst, uint64_t flags)
 {
-        int r;
-
         /**
          * The MC L1 TLB supports variable sized pages, based on a fragment
          * field in the PTE. When this field is set to a non-zero value, page
@@ -1462,41 +1485,38 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
          * Userspace can support this by aligning virtual base address and
          * allocation size to the fragment size.
          */
-
-        /* SI and newer are optimized for 64KB */
-        unsigned pages_per_frag = AMDGPU_LOG2_PAGES_PER_FRAG(params->adev);
-        uint64_t frag_flags = AMDGPU_PTE_FRAG(pages_per_frag);
-        uint64_t frag_align = 1 << pages_per_frag;
-
-        uint64_t frag_start = ALIGN(start, frag_align);
-        uint64_t frag_end = end & ~(frag_align - 1);
+        unsigned max_frag = params->adev->vm_manager.fragment_size;
+        int r;
 
         /* system pages are non continuously */
-        if (params->src || !(flags & AMDGPU_PTE_VALID) ||
-            (frag_start >= frag_end))
+        if (params->src || !(flags & AMDGPU_PTE_VALID))
                 return amdgpu_vm_update_ptes(params, start, end, dst, flags);
 
-        /* handle the 4K area at the beginning */
-        if (start != frag_start) {
-                r = amdgpu_vm_update_ptes(params, start, frag_start,
-                                          dst, flags);
+        while (start != end) {
+                uint64_t frag_flags, frag_end;
+                unsigned frag;
+
+                /* This intentionally wraps around if no bit is set */
+                frag = min((unsigned)ffs(start) - 1,
+                           (unsigned)fls64(end - start) - 1);
+                if (frag >= max_frag) {
+                        frag_flags = AMDGPU_PTE_FRAG(max_frag);
+                        frag_end = end & ~((1ULL << max_frag) - 1);
+                } else {
+                        frag_flags = AMDGPU_PTE_FRAG(frag);
+                        frag_end = start + (1 << frag);
+                }
+
+                r = amdgpu_vm_update_ptes(params, start, frag_end, dst,
+                                          flags | frag_flags);
                 if (r)
                         return r;
-                dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
-        }
-
-        /* handle the area in the middle */
-        r = amdgpu_vm_update_ptes(params, frag_start, frag_end, dst,
-                                  flags | frag_flags);
-        if (r)
-                return r;
 
-        /* handle the 4K area at the end */
-        if (frag_end != end) {
-                dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
-                r = amdgpu_vm_update_ptes(params, frag_end, end, dst, flags);
+                dst += (frag_end - start) * AMDGPU_GPU_PAGE_SIZE;
+                start = frag_end;
         }
-        return r;
+
+        return 0;
 }
 
 /**
@@ -1504,7 +1524,6 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
  *
  * @adev: amdgpu_device pointer
  * @exclusive: fence we need to sync to
- * @src: address where to copy page table entries from
  * @pages_addr: DMA addresses to use for mapping
  * @vm: requested vm
  * @start: start of mapped range
@@ -1518,7 +1537,6 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
  */
 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
                                        struct dma_fence *exclusive,
-                                       uint64_t src,
                                        dma_addr_t *pages_addr,
                                        struct amdgpu_vm *vm,
                                        uint64_t start, uint64_t last,
@@ -1536,7 +1554,6 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         memset(&params, 0, sizeof(params));
         params.adev = adev;
         params.vm = vm;
-        params.src = src;
 
         /* sync to everything on unmapping */
         if (!(flags & AMDGPU_PTE_VALID))
@@ -1565,10 +1582,12 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         nptes = last - start + 1;
 
         /*
-         * reserve space for one command every (1 << BLOCK_SIZE)
+         * reserve space for two commands every (1 << BLOCK_SIZE)
          *  entries or 2k dwords (whatever is smaller)
+         *
+         * The second command is for the shadow pagetables.
          */
-        ncmds = (nptes >> min(adev->vm_manager.block_size, 11u)) + 1;
+        ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2;
 
         /* padding, etc. */
         ndw = 64;
@@ -1576,15 +1595,9 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         /* one PDE write for each huge page */
         ndw += ((nptes >> adev->vm_manager.block_size) + 1) * 6;
 
-        if (src) {
-                /* only copy commands needed */
-                ndw += ncmds * 7;
-
-                params.func = amdgpu_vm_do_copy_ptes;
-
-        } else if (pages_addr) {
+        if (pages_addr) {
                 /* copy commands needed */
-                ndw += ncmds * 7;
+                ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw;
 
                 /* and also PTEs */
                 ndw += nptes * 2;
@@ -1593,10 +1606,11 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 
         } else {
                 /* set page commands needed */
-                ndw += ncmds * 10;
+                ndw += ncmds * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw;
 
-                /* two extra commands for begin/end of fragment */
-                ndw += 2 * 10;
+                /* extra commands for begin/end fragments */
+                ndw += 2 * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw
+                                * adev->vm_manager.fragment_size;
 
                 params.func = amdgpu_vm_do_set_ptes;
         }
@@ -1607,7 +1621,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 
         params.ib = &job->ibs[0];
 
-        if (!src && pages_addr) {
+        if (pages_addr) {
                 uint64_t *pte;
                 unsigned i;
 
@@ -1628,12 +1642,12 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         if (r)
                 goto error_free;
 
-        r = amdgpu_sync_resv(adev, &job->sync, vm->root.bo->tbo.resv,
+        r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,
                              owner);
         if (r)
                 goto error_free;
 
-        r = reservation_object_reserve_shared(vm->root.bo->tbo.resv);
+        r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);
         if (r)
                 goto error_free;
 
@@ -1648,14 +1662,14 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         if (r)
                 goto error_free;
 
-        amdgpu_bo_fence(vm->root.bo, f, true);
+        amdgpu_bo_fence(vm->root.base.bo, f, true);
         dma_fence_put(*fence);
         *fence = f;
         return 0;
 
 error_free:
         amdgpu_job_free(job);
-        amdgpu_vm_invalidate_level(&vm->root);
+        amdgpu_vm_invalidate_level(vm, &vm->root);
         return r;
 }
 
@@ -1664,7 +1678,6 @@ error_free:
  *
  * @adev: amdgpu_device pointer
  * @exclusive: fence we need to sync to
- * @gtt_flags: flags as they are used for GTT
  * @pages_addr: DMA addresses to use for mapping
  * @vm: requested vm
  * @mapping: mapped range and flags to use for the update
@@ -1678,7 +1691,6 @@ error_free:
  */
 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
                                       struct dma_fence *exclusive,
-                                      uint64_t gtt_flags,
                                       dma_addr_t *pages_addr,
                                       struct amdgpu_vm *vm,
                                       struct amdgpu_bo_va_mapping *mapping,
@@ -1686,7 +1698,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
                                       struct drm_mm_node *nodes,
                                       struct dma_fence **fence)
 {
-        uint64_t pfn, src = 0, start = mapping->start;
+        uint64_t pfn, start = mapping->start;
         int r;
 
         /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
@@ -1733,11 +1745,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
                 }
 
                 if (pages_addr) {
-                        if (flags == gtt_flags)
-                                src = adev->gart.table_addr +
-                                        (addr >> AMDGPU_GPU_PAGE_SHIFT) * 8;
-                        else
-                                max_entries = min(max_entries, 16ull * 1024ull);
+                        max_entries = min(max_entries, 16ull * 1024ull);
                         addr = 0;
                 } else if (flags & AMDGPU_PTE_VALID) {
                         addr += adev->vm_manager.vram_base_offset;
@@ -1745,8 +1753,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
                 addr += pfn << PAGE_SHIFT;
 
                 last = min((uint64_t)mapping->last, start + max_entries - 1);
-                r = amdgpu_vm_bo_update_mapping(adev, exclusive,
-                                                src, pages_addr, vm,
+                r = amdgpu_vm_bo_update_mapping(adev, exclusive, pages_addr, vm,
                                                 start, last, flags, addr,
                                                 fence);
                 if (r)
@@ -1778,75 +1785,75 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
                         struct amdgpu_bo_va *bo_va,
                         bool clear)
 {
-        struct amdgpu_vm *vm = bo_va->vm;
+        struct amdgpu_bo *bo = bo_va->base.bo;
+        struct amdgpu_vm *vm = bo_va->base.vm;
         struct amdgpu_bo_va_mapping *mapping;
         dma_addr_t *pages_addr = NULL;
-        uint64_t gtt_flags, flags;
         struct ttm_mem_reg *mem;
         struct drm_mm_node *nodes;
-        struct dma_fence *exclusive;
+        struct dma_fence *exclusive, **last_update;
+        uint64_t flags;
         int r;
 
-        if (clear || !bo_va->bo) {
+        if (clear || !bo_va->base.bo) {
                 mem = NULL;
                 nodes = NULL;
                 exclusive = NULL;
         } else {
                 struct ttm_dma_tt *ttm;
 
-                mem = &bo_va->bo->tbo.mem;
+                mem = &bo_va->base.bo->tbo.mem;
                 nodes = mem->mm_node;
                 if (mem->mem_type == TTM_PL_TT) {
-                        ttm = container_of(bo_va->bo->tbo.ttm, struct
-                                           ttm_dma_tt, ttm);
+                        ttm = container_of(bo_va->base.bo->tbo.ttm,
+                                           struct ttm_dma_tt, ttm);
                         pages_addr = ttm->dma_address;
                 }
-                exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
+                exclusive = reservation_object_get_excl(bo->tbo.resv);
         }
 
-        if (bo_va->bo) {
-                flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
-                gtt_flags = (amdgpu_ttm_is_bound(bo_va->bo->tbo.ttm) &&
-                        adev == amdgpu_ttm_adev(bo_va->bo->tbo.bdev)) ?
-                        flags : 0;
-        } else {
+        if (bo)
+                flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
+        else
                 flags = 0x0;
-                gtt_flags = ~0x0;
-        }
 
-        spin_lock(&vm->status_lock);
-        if (!list_empty(&bo_va->vm_status))
+        if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
+                last_update = &vm->last_update;
+        else
+                last_update = &bo_va->last_pt_update;
+
+        if (!clear && bo_va->base.moved) {
+                bo_va->base.moved = false;
                 list_splice_init(&bo_va->valids, &bo_va->invalids);
-        spin_unlock(&vm->status_lock);
+
+        } else if (bo_va->cleared != clear) {
+                list_splice_init(&bo_va->valids, &bo_va->invalids);
+        }
 
         list_for_each_entry(mapping, &bo_va->invalids, list) {
-                r = amdgpu_vm_bo_split_mapping(adev, exclusive,
-                                               gtt_flags, pages_addr, vm,
+                r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
                                                mapping, flags, nodes,
-                                               &bo_va->last_pt_update);
+                                               last_update);
                 if (r)
                         return r;
         }
 
-        if (trace_amdgpu_vm_bo_mapping_enabled()) {
-                list_for_each_entry(mapping, &bo_va->valids, list)
-                        trace_amdgpu_vm_bo_mapping(mapping);
-
-                list_for_each_entry(mapping, &bo_va->invalids, list)
-                        trace_amdgpu_vm_bo_mapping(mapping);
+        if (vm->use_cpu_for_update) {
+                /* Flush HDP */
+                mb();
+                amdgpu_gart_flush_gpu_tlb(adev, 0);
         }
 
         spin_lock(&vm->status_lock);
-        list_splice_init(&bo_va->invalids, &bo_va->valids);
-        list_del_init(&bo_va->vm_status);
-        if (clear)
-                list_add(&bo_va->vm_status, &vm->cleared);
+        list_del_init(&bo_va->base.vm_status);
         spin_unlock(&vm->status_lock);
 
-        if (vm->use_cpu_for_update) {
-                /* Flush HDP */
-                mb();
-                amdgpu_gart_flush_gpu_tlb(adev, 0);
+        list_splice_init(&bo_va->invalids, &bo_va->valids);
+        bo_va->cleared = clear;
+
+        if (trace_amdgpu_vm_bo_mapping_enabled()) {
+                list_for_each_entry(mapping, &bo_va->valids, list)
+                        trace_amdgpu_vm_bo_mapping(mapping);
         }
 
         return 0;
@@ -1954,7 +1961,7 @@ static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
  */
 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 {
-        struct reservation_object *resv = vm->root.bo->tbo.resv;
+        struct reservation_object *resv = vm->root.base.bo->tbo.resv;
         struct dma_fence *excl, **shared;
         unsigned i, shared_count;
         int r;
@@ -2012,7 +2019,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
                 if (vm->pte_support_ats)
                         init_pte_value = AMDGPU_PTE_SYSTEM;
 
-                r = amdgpu_vm_bo_update_mapping(adev, NULL, 0, NULL, vm,
+                r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
                                                 mapping->start, mapping->last,
                                                 init_pte_value, 0, &f);
                 amdgpu_vm_free_mapping(adev, vm, mapping, f);
@@ -2034,29 +2041,35 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 }
 
 /**
- * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
+ * amdgpu_vm_handle_moved - handle moved BOs in the PT
  *
  * @adev: amdgpu_device pointer
  * @vm: requested vm
+ * @sync: sync object to add fences to
  *
- * Make sure all invalidated BOs are cleared in the PT.
+ * Make sure all BOs which are moved are updated in the PTs.
  * Returns 0 for success.
  *
- * PTs have to be reserved and mutex must be locked!
+ * PTs have to be reserved!
  */
-int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
-                             struct amdgpu_vm *vm, struct amdgpu_sync *sync)
+int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
+                           struct amdgpu_vm *vm)
 {
-        struct amdgpu_bo_va *bo_va = NULL;
+        bool clear;
         int r = 0;
 
         spin_lock(&vm->status_lock);
-        while (!list_empty(&vm->invalidated)) {
-                bo_va = list_first_entry(&vm->invalidated,
-                        struct amdgpu_bo_va, vm_status);
+        while (!list_empty(&vm->moved)) {
+                struct amdgpu_bo_va *bo_va;
+
+                bo_va = list_first_entry(&vm->moved,
+                        struct amdgpu_bo_va, base.vm_status);
                 spin_unlock(&vm->status_lock);
 
-                r = amdgpu_vm_bo_update(adev, bo_va, true);
+                /* Per VM BOs never need to bo cleared in the page tables */
+                clear = bo_va->base.bo->tbo.resv != vm->root.base.bo->tbo.resv;
+
+                r = amdgpu_vm_bo_update(adev, bo_va, clear);
                 if (r)
                         return r;
 
@@ -2064,9 +2077,6 @@ int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
         }
         spin_unlock(&vm->status_lock);
 
-        if (bo_va)
-                r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
-
         return r;
 }
 
@@ -2093,20 +2103,54 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
         if (bo_va == NULL) {
                 return NULL;
         }
-        bo_va->vm = vm;
-        bo_va->bo = bo;
+        bo_va->base.vm = vm;
+        bo_va->base.bo = bo;
+        INIT_LIST_HEAD(&bo_va->base.bo_list);
+        INIT_LIST_HEAD(&bo_va->base.vm_status);
+
         bo_va->ref_count = 1;
-        INIT_LIST_HEAD(&bo_va->bo_list);
         INIT_LIST_HEAD(&bo_va->valids);
         INIT_LIST_HEAD(&bo_va->invalids);
-        INIT_LIST_HEAD(&bo_va->vm_status);
 
         if (bo)
-                list_add_tail(&bo_va->bo_list, &bo->va);
+                list_add_tail(&bo_va->base.bo_list, &bo->va);
 
         return bo_va;
 }
 
+
+/**
+ * amdgpu_vm_bo_insert_mapping - insert a new mapping
+ *
+ * @adev: amdgpu_device pointer
+ * @bo_va: bo_va to store the address
+ * @mapping: the mapping to insert
+ *
+ * Insert a new mapping into all structures.
+ */
+static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
+                                    struct amdgpu_bo_va *bo_va,
+                                    struct amdgpu_bo_va_mapping *mapping)
+{
+        struct amdgpu_vm *vm = bo_va->base.vm;
+        struct amdgpu_bo *bo = bo_va->base.bo;
+
+        mapping->bo_va = bo_va;
+        list_add(&mapping->list, &bo_va->invalids);
+        amdgpu_vm_it_insert(mapping, &vm->va);
+
+        if (mapping->flags & AMDGPU_PTE_PRT)
+                amdgpu_vm_prt_get(adev);
+
+        if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
+                spin_lock(&vm->status_lock);
+                if (list_empty(&bo_va->base.vm_status))
+                        list_add(&bo_va->base.vm_status, &vm->moved);
+                spin_unlock(&vm->status_lock);
+        }
+        trace_amdgpu_vm_bo_map(bo_va, mapping);
+}
+
 /**
  * amdgpu_vm_bo_map - map bo inside a vm
  *
@@ -2127,7 +2171,8 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
                      uint64_t size, uint64_t flags)
 {
         struct amdgpu_bo_va_mapping *mapping, *tmp;
-        struct amdgpu_vm *vm = bo_va->vm;
+        struct amdgpu_bo *bo = bo_va->base.bo;
+        struct amdgpu_vm *vm = bo_va->base.vm;
         uint64_t eaddr;
 
         /* validate the parameters */
@@ -2138,7 +2183,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
         /* make sure object fit at this offset */
         eaddr = saddr + size - 1;
         if (saddr >= eaddr ||
-            (bo_va->bo && offset + size > amdgpu_bo_size(bo_va->bo)))
+            (bo && offset + size > amdgpu_bo_size(bo)))
                 return -EINVAL;
 
         saddr /= AMDGPU_GPU_PAGE_SIZE;
@@ -2148,7 +2193,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
         if (tmp) {
                 /* bo and tmp overlap, invalid addr */
                 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
-                        "0x%010Lx-0x%010Lx\n", bo_va->bo, saddr, eaddr,
+                        "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
                         tmp->start, tmp->last + 1);
                 return -EINVAL;
         }
@@ -2157,17 +2202,12 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
         if (!mapping)
                 return -ENOMEM;
 
-        INIT_LIST_HEAD(&mapping->list);
         mapping->start = saddr;
         mapping->last = eaddr;
         mapping->offset = offset;
         mapping->flags = flags;
 
-        list_add(&mapping->list, &bo_va->invalids);
-        amdgpu_vm_it_insert(mapping, &vm->va);
-
-        if (flags & AMDGPU_PTE_PRT)
-                amdgpu_vm_prt_get(adev);
+        amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
 
         return 0;
 }
@@ -2193,7 +2233,7 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
                              uint64_t size, uint64_t flags)
 {
         struct amdgpu_bo_va_mapping *mapping;
-        struct amdgpu_vm *vm = bo_va->vm;
+        struct amdgpu_bo *bo = bo_va->base.bo;
         uint64_t eaddr;
         int r;
 
@@ -2205,7 +2245,7 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
         /* make sure object fit at this offset */
         eaddr = saddr + size - 1;
         if (saddr >= eaddr ||
-            (bo_va->bo && offset + size > amdgpu_bo_size(bo_va->bo)))
+            (bo && offset + size > amdgpu_bo_size(bo)))
                 return -EINVAL;
 
         /* Allocate all the needed memory */
@@ -2213,7 +2253,7 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
         if (!mapping)
                 return -ENOMEM;
 
-        r = amdgpu_vm_bo_clear_mappings(adev, bo_va->vm, saddr, size);
+        r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
         if (r) {
                 kfree(mapping);
                 return r;
@@ -2227,11 +2267,7 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
         mapping->offset = offset;
         mapping->flags = flags;
 
-        list_add(&mapping->list, &bo_va->invalids);
-        amdgpu_vm_it_insert(mapping, &vm->va);
-
-        if (flags & AMDGPU_PTE_PRT)
-                amdgpu_vm_prt_get(adev);
+        amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
 
         return 0;
 }
@@ -2253,7 +2289,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
                        uint64_t saddr)
 {
         struct amdgpu_bo_va_mapping *mapping;
-        struct amdgpu_vm *vm = bo_va->vm;
+        struct amdgpu_vm *vm = bo_va->base.vm;
         bool valid = true;
 
         saddr /= AMDGPU_GPU_PAGE_SIZE;
@@ -2277,6 +2313,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
 
         list_del(&mapping->list);
         amdgpu_vm_it_remove(mapping, &vm->va);
+        mapping->bo_va = NULL;
         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
 
         if (valid)
@@ -2362,6 +2399,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
                 if (tmp->last > eaddr)
                     tmp->last = eaddr;
 
+                tmp->bo_va = NULL;
                 list_add(&tmp->list, &vm->freed);
                 trace_amdgpu_vm_bo_unmap(NULL, tmp);
         }
@@ -2388,6 +2426,19 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
 }
 
 /**
+ * amdgpu_vm_bo_lookup_mapping - find mapping by address
+ *
+ * @vm: the requested VM
+ *
+ * Find a mapping by it's address.
+ */
+struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
+                                                         uint64_t addr)
+{
+        return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
+}
+
+/**
  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
  *
  * @adev: amdgpu_device pointer
@@ -2401,17 +2452,18 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
                       struct amdgpu_bo_va *bo_va)
 {
         struct amdgpu_bo_va_mapping *mapping, *next;
-        struct amdgpu_vm *vm = bo_va->vm;
+        struct amdgpu_vm *vm = bo_va->base.vm;
 
-        list_del(&bo_va->bo_list);
+        list_del(&bo_va->base.bo_list);
 
         spin_lock(&vm->status_lock);
-        list_del(&bo_va->vm_status);
+        list_del(&bo_va->base.vm_status);
         spin_unlock(&vm->status_lock);
 
         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
                 list_del(&mapping->list);
                 amdgpu_vm_it_remove(mapping, &vm->va);
+                mapping->bo_va = NULL;
                 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
                 list_add(&mapping->list, &vm->freed);
         }
@@ -2436,15 +2488,37 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
  * Mark @bo as invalid.
  */
 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
-                             struct amdgpu_bo *bo)
+                             struct amdgpu_bo *bo, bool evicted)
 {
-        struct amdgpu_bo_va *bo_va;
+        struct amdgpu_vm_bo_base *bo_base;
+
+        list_for_each_entry(bo_base, &bo->va, bo_list) {
+                struct amdgpu_vm *vm = bo_base->vm;
+
+                bo_base->moved = true;
+                if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
+                        spin_lock(&bo_base->vm->status_lock);
+                        if (bo->tbo.type == ttm_bo_type_kernel)
+                                list_move(&bo_base->vm_status, &vm->evicted);
+                        else
+                                list_move_tail(&bo_base->vm_status,
+                                               &vm->evicted);
+                        spin_unlock(&bo_base->vm->status_lock);
+                        continue;
+                }
+
+                if (bo->tbo.type == ttm_bo_type_kernel) {
+                        spin_lock(&bo_base->vm->status_lock);
+                        if (list_empty(&bo_base->vm_status))
+                                list_add(&bo_base->vm_status, &vm->relocated);
+                        spin_unlock(&bo_base->vm->status_lock);
+                        continue;
+                }
 
-        list_for_each_entry(bo_va, &bo->va, bo_list) {
-                spin_lock(&bo_va->vm->status_lock);
-                if (list_empty(&bo_va->vm_status))
-                        list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
-                spin_unlock(&bo_va->vm->status_lock);
+                spin_lock(&bo_base->vm->status_lock);
+                if (list_empty(&bo_base->vm_status))
+                        list_add(&bo_base->vm_status, &vm->moved);
+                spin_unlock(&bo_base->vm->status_lock);
         }
 }
 
@@ -2462,12 +2536,26 @@ static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
 }
 
 /**
- * amdgpu_vm_adjust_size - adjust vm size and block size
+ * amdgpu_vm_set_fragment_size - adjust fragment size in PTE
+ *
+ * @adev: amdgpu_device pointer
+ * @fragment_size_default: the default fragment size if it's set auto
+ */
+void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev, uint32_t fragment_size_default)
+{
+        if (amdgpu_vm_fragment_size == -1)
+                adev->vm_manager.fragment_size = fragment_size_default;
+        else
+                adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
+}
+
+/**
+ * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
  *
  * @adev: amdgpu_device pointer
  * @vm_size: the default vm size if it's set auto
  */
-void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size)
+void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_t fragment_size_default)
 {
         /* adjust vm size firstly */
         if (amdgpu_vm_size == -1)
@@ -2482,8 +2570,11 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size)
         else
                 adev->vm_manager.block_size = amdgpu_vm_block_size;
 
-        DRM_INFO("vm size is %llu GB, block size is %u-bit\n",
-                adev->vm_manager.vm_size, adev->vm_manager.block_size);
+        amdgpu_vm_set_fragment_size(adev, fragment_size_default);
+
+        DRM_INFO("vm size is %llu GB, block size is %u-bit, fragment size is %u-bit\n",
+                adev->vm_manager.vm_size, adev->vm_manager.block_size,
+                adev->vm_manager.fragment_size);
 }
 
 /**
@@ -2496,7 +2587,7 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size)
  * Init @vm fields.
  */
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                   int vm_context)
+                   int vm_context, unsigned int pasid)
 {
         const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
                 AMDGPU_VM_PTE_COUNT(adev) * 8);
@@ -2507,13 +2598,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         u64 flags;
         uint64_t init_pde_value = 0;
 
-        vm->va = RB_ROOT;
+        vm->va = RB_ROOT_CACHED;
         vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
                 vm->reserved_vmid[i] = NULL;
         spin_lock_init(&vm->status_lock);
-        INIT_LIST_HEAD(&vm->invalidated);
-        INIT_LIST_HEAD(&vm->cleared);
+        INIT_LIST_HEAD(&vm->evicted);
+        INIT_LIST_HEAD(&vm->relocated);
+        INIT_LIST_HEAD(&vm->moved);
         INIT_LIST_HEAD(&vm->freed);
 
         /* create scheduler entity for page table updates */
@@ -2544,7 +2636,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                          vm->use_cpu_for_update ? "CPU" : "SDMA");
         WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
                   "CPU update of VM recommended only for large BAR system\n");
-        vm->last_dir_update = NULL;
+        vm->last_update = NULL;
 
         flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
                         AMDGPU_GEM_CREATE_VRAM_CLEARED;
@@ -2557,30 +2649,46 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, 0), align, true,
                              AMDGPU_GEM_DOMAIN_VRAM,
                              flags,
-                             NULL, NULL, init_pde_value, &vm->root.bo);
+                             NULL, NULL, init_pde_value, &vm->root.base.bo);
         if (r)
                 goto error_free_sched_entity;
 
-        r = amdgpu_bo_reserve(vm->root.bo, false);
-        if (r)
-                goto error_free_root;
-
-        vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
+        vm->root.base.vm = vm;
+        list_add_tail(&vm->root.base.bo_list, &vm->root.base.bo->va);
+        INIT_LIST_HEAD(&vm->root.base.vm_status);
 
         if (vm->use_cpu_for_update) {
-                r = amdgpu_bo_kmap(vm->root.bo, NULL);
+                r = amdgpu_bo_reserve(vm->root.base.bo, false);
+                if (r)
+                        goto error_free_root;
+
+                r = amdgpu_bo_kmap(vm->root.base.bo, NULL);
+                amdgpu_bo_unreserve(vm->root.base.bo);
                 if (r)
                         goto error_free_root;
         }
 
-        amdgpu_bo_unreserve(vm->root.bo);
+        if (pasid) {
+                unsigned long flags;
+
+                spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
+                r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
+                              GFP_ATOMIC);
+                spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
+                if (r < 0)
+                        goto error_free_root;
+
+                vm->pasid = pasid;
+        }
+
+        INIT_KFIFO(vm->faults);
 
         return 0;
 
 error_free_root:
-        amdgpu_bo_unref(&vm->root.bo->shadow);
-        amdgpu_bo_unref(&vm->root.bo);
-        vm->root.bo = NULL;
+        amdgpu_bo_unref(&vm->root.base.bo->shadow);
+        amdgpu_bo_unref(&vm->root.base.bo);
+        vm->root.base.bo = NULL;
 
 error_free_sched_entity:
         amd_sched_entity_fini(&ring->sched, &vm->entity);
@@ -2599,9 +2707,11 @@ static void amdgpu_vm_free_levels(struct amdgpu_vm_pt *level)
 {
         unsigned i;
 
-        if (level->bo) {
-                amdgpu_bo_unref(&level->bo->shadow);
-                amdgpu_bo_unref(&level->bo);
+        if (level->base.bo) {
+                list_del(&level->base.bo_list);
+                list_del(&level->base.vm_status);
+                amdgpu_bo_unref(&level->base.bo->shadow);
+                amdgpu_bo_unref(&level->base.bo);
         }
 
         if (level->entries)
@@ -2624,14 +2734,28 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 {
         struct amdgpu_bo_va_mapping *mapping, *tmp;
         bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
+        u64 fault;
         int i;
 
+        /* Clear pending page faults from IH when the VM is destroyed */
+        while (kfifo_get(&vm->faults, &fault))
+                amdgpu_ih_clear_fault(adev, fault);
+
+        if (vm->pasid) {
+                unsigned long flags;
+
+                spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
+                idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
+                spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
+        }
+
         amd_sched_entity_fini(vm->entity.sched, &vm->entity);
 
-        if (!RB_EMPTY_ROOT(&vm->va)) {
+        if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
                 dev_err(adev->dev, "still active bo inside vm\n");
         }
-        rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, rb) {
+        rbtree_postorder_for_each_entry_safe(mapping, tmp,
+                                             &vm->va.rb_root, rb) {
                 list_del(&mapping->list);
                 amdgpu_vm_it_remove(mapping, &vm->va);
                 kfree(mapping);
@@ -2647,7 +2771,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
         }
 
         amdgpu_vm_free_levels(&vm->root);
-        dma_fence_put(vm->last_dir_update);
+        dma_fence_put(vm->last_update);
         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
                 amdgpu_vm_free_reserved_vmid(adev, vm, i);
 }
@@ -2705,6 +2829,8 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
         adev->vm_manager.vm_update_mode = 0;
 #endif
 
+        idr_init(&adev->vm_manager.pasid_idr);
+        spin_lock_init(&adev->vm_manager.pasid_lock);
 }
 
 /**
@@ -2718,6 +2844,9 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
 {
         unsigned i, j;
 
+        WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
+        idr_destroy(&adev->vm_manager.pasid_idr);
+
         for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
                 struct amdgpu_vm_id_manager *id_mgr =
                         &adev->vm_manager.id_mgr[i];
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 217ecba8f4cc..0af090667dfc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -25,6 +25,7 @@
 #define __AMDGPU_VM_H__
 
 #include <linux/rbtree.h>
+#include <linux/idr.h>
 
 #include "gpu_scheduler.h"
 #include "amdgpu_sync.h"
@@ -50,11 +51,6 @@ struct amdgpu_bo_list_entry;
 /* PTBs (Page Table Blocks) need to be aligned to 32K */
 #define AMDGPU_VM_PTB_ALIGN_SIZE   32768
 
-/* LOG2 number of continuous pages for the fragment field */
-#define AMDGPU_LOG2_PAGES_PER_FRAG(adev) \
-        ((adev)->asic_type < CHIP_VEGA10 ? 4 : \
-         (adev)->vm_manager.block_size)
-
 #define AMDGPU_PTE_VALID        (1ULL << 0)
 #define AMDGPU_PTE_SYSTEM       (1ULL << 1)
 #define AMDGPU_PTE_SNOOPED      (1ULL << 2)
@@ -99,37 +95,57 @@ struct amdgpu_bo_list_entry;
 #define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0)
 #define AMDGPU_VM_USE_CPU_FOR_COMPUTE (1 << 1)
 
+/* base structure for tracking BO usage in a VM */
+struct amdgpu_vm_bo_base {
+        /* constant after initialization */
+        struct amdgpu_vm                *vm;
+        struct amdgpu_bo                *bo;
+
+        /* protected by bo being reserved */
+        struct list_head                bo_list;
+
+        /* protected by spinlock */
+        struct list_head                vm_status;
+
+        /* protected by the BO being reserved */
+        bool                            moved;
+};
 
 struct amdgpu_vm_pt {
-        struct amdgpu_bo        *bo;
-        uint64_t                addr;
-        bool                    huge_page;
+        struct amdgpu_vm_bo_base        base;
+        uint64_t                        addr;
 
         /* array of page tables, one for each directory entry */
-        struct amdgpu_vm_pt     *entries;
-        unsigned                last_entry_used;
+        struct amdgpu_vm_pt             *entries;
+        unsigned                        last_entry_used;
 };
 
+#define AMDGPU_VM_FAULT(pasid, addr) (((u64)(pasid) << 48) | (addr))
+#define AMDGPU_VM_FAULT_PASID(fault) ((u64)(fault) >> 48)
+#define AMDGPU_VM_FAULT_ADDR(fault)  ((u64)(fault) & 0xfffffffff000ULL)
+
 struct amdgpu_vm {
         /* tree of virtual addresses mapped */
-        struct rb_root          va;
+        struct rb_root_cached   va;
 
         /* protecting invalidated */
         spinlock_t              status_lock;
 
-        /* BOs moved, but not yet updated in the PT */
-        struct list_head        invalidated;
+        /* BOs who needs a validation */
+        struct list_head        evicted;
+
+        /* PT BOs which relocated and their parent need an update */
+        struct list_head        relocated;
 
-        /* BOs cleared in the PT because of a move */
-        struct list_head        cleared;
+        /* BOs moved, but not yet updated in the PT */
+        struct list_head        moved;
 
         /* BO mappings freed, but not yet updated in the PT */
         struct list_head        freed;
 
         /* contains the page directory */
         struct amdgpu_vm_pt     root;
-        struct dma_fence        *last_dir_update;
-        uint64_t                last_eviction_counter;
+        struct dma_fence        *last_update;
 
         /* protecting freed */
         spinlock_t              freed_lock;
@@ -137,18 +153,20 @@ struct amdgpu_vm {
         /* Scheduler entity for page table updates */
         struct amd_sched_entity entity;
 
-        /* client id */
+        /* client id and PASID (TODO: replace client_id with PASID) */
         u64                     client_id;
+        unsigned int            pasid;
         /* dedicated to vm */
         struct amdgpu_vm_id     *reserved_vmid[AMDGPU_MAX_VMHUBS];
-        /* each VM will map on CSA */
-        struct amdgpu_bo_va *csa_bo_va;
 
         /* Flag to indicate if VM tables are updated by CPU or GPU (SDMA) */
         bool                    use_cpu_for_update;
 
         /* Flag to indicate ATS support from PTE for GFX9 */
         bool                    pte_support_ats;
+
+        /* Up to 128 pending page faults */
+        DECLARE_KFIFO(faults, u64, 128);
 };
 
 struct amdgpu_vm_id {
@@ -191,6 +209,7 @@ struct amdgpu_vm_manager {
         uint32_t                                num_level;
         uint64_t                                vm_size;
         uint32_t                                block_size;
+        uint32_t                                fragment_size;
         /* vram base address for page table entry  */
         u64                                     vram_base_offset;
         /* vm pte handling */
@@ -210,21 +229,28 @@ struct amdgpu_vm_manager {
          * BIT1[= 0] Compute updated by SDMA [= 1] by CPU
          */
         int                                     vm_update_mode;
+
+        /* PASID to VM mapping, will be used in interrupt context to
+         * look up VM of a page fault
+         */
+        struct idr                              pasid_idr;
+        spinlock_t                              pasid_lock;
 };
 
+int amdgpu_vm_alloc_pasid(unsigned int bits);
+void amdgpu_vm_free_pasid(unsigned int pasid);
 void amdgpu_vm_manager_init(struct amdgpu_device *adev);
 void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                   int vm_context);
+                   int vm_context, unsigned int pasid);
 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
                          struct list_head *validated,
                          struct amdgpu_bo_list_entry *entry);
+bool amdgpu_vm_ready(struct amdgpu_vm *vm);
 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                               int (*callback)(void *p, struct amdgpu_bo *bo),
                               void *param);
-void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
-                                  struct amdgpu_vm *vm);
 int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
                         struct amdgpu_vm *vm,
                         uint64_t saddr, uint64_t size);
@@ -240,13 +266,13 @@ int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
                           struct amdgpu_vm *vm,
                           struct dma_fence **fence);
-int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                             struct amdgpu_sync *sync);
+int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
+                           struct amdgpu_vm *vm);
 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
                         struct amdgpu_bo_va *bo_va,
                         bool clear);
 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
-                             struct amdgpu_bo *bo);
+                             struct amdgpu_bo *bo, bool evicted);
 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
                                        struct amdgpu_bo *bo);
 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
@@ -266,9 +292,14 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
                                 struct amdgpu_vm *vm,
                                 uint64_t saddr, uint64_t size);
+struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
+                                                         uint64_t addr);
 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
                       struct amdgpu_bo_va *bo_va);
-void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size);
+void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev,
+                                uint32_t fragment_size_default);
+void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size,
+                                uint32_t fragment_size_default);
 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
                                   struct amdgpu_job *job);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index a2c59a08b2bd..26e900627971 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -28,6 +28,8 @@
 struct amdgpu_vram_mgr {
         struct drm_mm mm;
         spinlock_t lock;
+        atomic64_t usage;
+        atomic64_t vis_usage;
 };
 
 /**
@@ -79,6 +81,27 @@ static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man)
 }
 
 /**
+ * amdgpu_vram_mgr_vis_size - Calculate visible node size
+ *
+ * @adev: amdgpu device structure
+ * @node: MM node structure
+ *
+ * Calculate how many bytes of the MM node are inside visible VRAM
+ */
+static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
+                                    struct drm_mm_node *node)
+{
+        uint64_t start = node->start << PAGE_SHIFT;
+        uint64_t end = (node->size + node->start) << PAGE_SHIFT;
+
+        if (start >= adev->mc.visible_vram_size)
+                return 0;
+
+        return (end > adev->mc.visible_vram_size ?
+                adev->mc.visible_vram_size : end) - start;
+}
+
+/**
  * amdgpu_vram_mgr_new - allocate new ranges
  *
  * @man: TTM memory type manager
@@ -93,11 +116,13 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
                                const struct ttm_place *place,
                                struct ttm_mem_reg *mem)
 {
+        struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
         struct amdgpu_vram_mgr *mgr = man->priv;
         struct drm_mm *mm = &mgr->mm;
         struct drm_mm_node *nodes;
         enum drm_mm_insert_mode mode;
         unsigned long lpfn, num_nodes, pages_per_node, pages_left;
+        uint64_t usage = 0, vis_usage = 0;
         unsigned i;
         int r;
 
@@ -142,6 +167,9 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
                 if (unlikely(r))
                         goto error;
 
+                usage += nodes[i].size << PAGE_SHIFT;
+                vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
+
                 /* Calculate a virtual BO start address to easily check if
                  * everything is CPU accessible.
                  */
@@ -155,6 +183,9 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
         }
         spin_unlock(&mgr->lock);
 
+        atomic64_add(usage, &mgr->usage);
+        atomic64_add(vis_usage, &mgr->vis_usage);
+
         mem->mm_node = nodes;
 
         return 0;
@@ -181,8 +212,10 @@ error:
 static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
                                 struct ttm_mem_reg *mem)
 {
+        struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
         struct amdgpu_vram_mgr *mgr = man->priv;
         struct drm_mm_node *nodes = mem->mm_node;
+        uint64_t usage = 0, vis_usage = 0;
         unsigned pages = mem->num_pages;
 
         if (!mem->mm_node)
@@ -192,31 +225,67 @@ static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
         while (pages) {
                 pages -= nodes->size;
                 drm_mm_remove_node(nodes);
+                usage += nodes->size << PAGE_SHIFT;
+                vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes);
                 ++nodes;
         }
         spin_unlock(&mgr->lock);
 
+        atomic64_sub(usage, &mgr->usage);
+        atomic64_sub(vis_usage, &mgr->vis_usage);
+
         kfree(mem->mm_node);
         mem->mm_node = NULL;
 }
 
 /**
+ * amdgpu_vram_mgr_usage - how many bytes are used in this domain
+ *
+ * @man: TTM memory type manager
+ *
+ * Returns how many bytes are used in this domain.
+ */
+uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man)
+{
+        struct amdgpu_vram_mgr *mgr = man->priv;
+
+        return atomic64_read(&mgr->usage);
+}
+
+/**
+ * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
+ *
+ * @man: TTM memory type manager
+ *
+ * Returns how many bytes are used in the visible part of VRAM
+ */
+uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man)
+{
+        struct amdgpu_vram_mgr *mgr = man->priv;
+
+        return atomic64_read(&mgr->vis_usage);
+}
+
+/**
  * amdgpu_vram_mgr_debug - dump VRAM table
  *
  * @man: TTM memory type manager
- * @prefix: text prefix
+ * @printer: DRM printer to use
  *
  * Dump the table content using printk.
  */
 static void amdgpu_vram_mgr_debug(struct ttm_mem_type_manager *man,
-                                  const char *prefix)
+                                  struct drm_printer *printer)
 {
         struct amdgpu_vram_mgr *mgr = man->priv;
-        struct drm_printer p = drm_debug_printer(prefix);
 
         spin_lock(&mgr->lock);
-        drm_mm_print(&mgr->mm, &p);
+        drm_mm_print(&mgr->mm, printer);
         spin_unlock(&mgr->lock);
+
+        drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n",
+                   man->size, amdgpu_vram_mgr_usage(man) >> 20,
+                   amdgpu_vram_mgr_vis_usage(man) >> 20);
 }
 
 const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func = {
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c
index d69aa2e179bb..69500a8b4e2d 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.c
+++ b/drivers/gpu/drm/amd/amdgpu/atom.c
@@ -1343,8 +1343,11 @@ struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios)
                 idx = 0x80;
 
         str = CSTR(idx);
-        if (*str != '\0')
+        if (*str != '\0') {
                 pr_info("ATOM BIOS: %s\n", str);
+                strlcpy(ctx->vbios_version, str, sizeof(ctx->vbios_version));
+        }
+
 
         return ctx;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.h b/drivers/gpu/drm/amd/amdgpu/atom.h
index ddd8045accf3..a39170991afe 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.h
+++ b/drivers/gpu/drm/amd/amdgpu/atom.h
@@ -140,6 +140,7 @@ struct atom_context {
         int io_mode;
         uint32_t *scratch;
         int scratch_size_bytes;
+        char vbios_version[20];
 };
 
 extern int amdgpu_atom_debug;
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index cb508a211b2f..68ce1bdaf2fc 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -307,7 +307,6 @@ static int ci_set_power_limit(struct amdgpu_device *adev, u32 n);
 static int ci_set_overdrive_target_tdp(struct amdgpu_device *adev,
                                        u32 target_tdp);
 static int ci_update_uvd_dpm(struct amdgpu_device *adev, bool gate);
-static void ci_dpm_set_dpm_funcs(struct amdgpu_device *adev);
 static void ci_dpm_set_irq_funcs(struct amdgpu_device *adev);
 
 static PPSMC_Result amdgpu_ci_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
@@ -883,8 +882,9 @@ static int ci_power_control_set_level(struct amdgpu_device *adev)
         return ret;
 }
 
-static void ci_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
+static void ci_dpm_powergate_uvd(void *handle, bool gate)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         pi->uvd_power_gated = gate;
@@ -901,8 +901,9 @@ static void ci_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
         }
 }
 
-static bool ci_dpm_vblank_too_short(struct amdgpu_device *adev)
+static bool ci_dpm_vblank_too_short(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 vblank_time = amdgpu_dpm_get_vblank_time(adev);
         u32 switch_limit = adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5 ? 450 : 300;
 
@@ -1210,11 +1211,12 @@ static int ci_fan_ctrl_stop_smc_fan_control(struct amdgpu_device *adev)
         }
 }
 
-static int ci_dpm_get_fan_speed_percent(struct amdgpu_device *adev,
+static int ci_dpm_get_fan_speed_percent(void *handle,
                                         u32 *speed)
 {
         u32 duty, duty100;
         u64 tmp64;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (adev->pm.no_fan)
                 return -ENOENT;
@@ -1237,12 +1239,13 @@ static int ci_dpm_get_fan_speed_percent(struct amdgpu_device *adev,
         return 0;
 }
 
-static int ci_dpm_set_fan_speed_percent(struct amdgpu_device *adev,
+static int ci_dpm_set_fan_speed_percent(void *handle,
                                         u32 speed)
 {
         u32 tmp;
         u32 duty, duty100;
         u64 tmp64;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         if (adev->pm.no_fan)
@@ -1271,8 +1274,10 @@ static int ci_dpm_set_fan_speed_percent(struct amdgpu_device *adev,
         return 0;
 }
 
-static void ci_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode)
+static void ci_dpm_set_fan_control_mode(void *handle, u32 mode)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
         switch (mode) {
         case AMD_FAN_CTRL_NONE:
                 if (adev->pm.dpm.fan.ucode_fan_control)
@@ -1292,8 +1297,9 @@ static void ci_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode)
         }
 }
 
-static u32 ci_dpm_get_fan_control_mode(struct amdgpu_device *adev)
+static u32 ci_dpm_get_fan_control_mode(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         if (pi->fan_is_controlled_by_smc)
@@ -4378,9 +4384,10 @@ static u32 ci_get_lowest_enabled_level(struct amdgpu_device *adev,
 }
 
 
-static int ci_dpm_force_performance_level(struct amdgpu_device *adev,
+static int ci_dpm_force_performance_level(void *handle,
                                           enum amd_dpm_forced_level level)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         u32 tmp, levels, i;
         int ret;
@@ -5291,8 +5298,9 @@ static void ci_update_requested_ps(struct amdgpu_device *adev,
         adev->pm.dpm.requested_ps = &pi->requested_rps;
 }
 
-static int ci_dpm_pre_set_power_state(struct amdgpu_device *adev)
+static int ci_dpm_pre_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct amdgpu_ps requested_ps = *adev->pm.dpm.requested_ps;
         struct amdgpu_ps *new_ps = &requested_ps;
@@ -5304,8 +5312,9 @@ static int ci_dpm_pre_set_power_state(struct amdgpu_device *adev)
         return 0;
 }
 
-static void ci_dpm_post_set_power_state(struct amdgpu_device *adev)
+static void ci_dpm_post_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct amdgpu_ps *new_ps = &pi->requested_rps;
 
@@ -5479,8 +5488,9 @@ static void ci_dpm_disable(struct amdgpu_device *adev)
         ci_update_current_ps(adev, boot_ps);
 }
 
-static int ci_dpm_set_power_state(struct amdgpu_device *adev)
+static int ci_dpm_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct amdgpu_ps *new_ps = &pi->requested_rps;
         struct amdgpu_ps *old_ps = &pi->current_rps;
@@ -5551,8 +5561,10 @@ static void ci_dpm_reset_asic(struct amdgpu_device *adev)
 }
 #endif
 
-static void ci_dpm_display_configuration_changed(struct amdgpu_device *adev)
+static void ci_dpm_display_configuration_changed(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
         ci_program_display_gap(adev);
 }
 
@@ -6105,9 +6117,10 @@ static int ci_dpm_init(struct amdgpu_device *adev)
 }
 
 static void
-ci_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
+ci_dpm_debugfs_print_current_performance_level(void *handle,
                                                struct seq_file *m)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct amdgpu_ps *rps = &pi->current_rps;
         u32 sclk = ci_get_average_sclk_freq(adev);
@@ -6131,12 +6144,13 @@ ci_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
         seq_printf(m, "GPU load: %u %%\n", activity_percent);
 }
 
-static void ci_dpm_print_power_state(struct amdgpu_device *adev,
-                                     struct amdgpu_ps *rps)
+static void ci_dpm_print_power_state(void *handle, void *current_ps)
 {
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)current_ps;
         struct ci_ps *ps = ci_get_ps(rps);
         struct ci_pl *pl;
         int i;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         amdgpu_dpm_print_class_info(rps->class, rps->class2);
         amdgpu_dpm_print_cap_info(rps->caps);
@@ -6158,20 +6172,23 @@ static inline bool ci_are_power_levels_equal(const struct ci_pl *ci_cpl1,
                   (ci_cpl1->pcie_lane == ci_cpl2->pcie_lane));
 }
 
-static int ci_check_state_equal(struct amdgpu_device *adev,
-                                struct amdgpu_ps *cps,
-                                struct amdgpu_ps *rps,
+static int ci_check_state_equal(void *handle,
+                                void *current_ps,
+                                void *request_ps,
                                 bool *equal)
 {
         struct ci_ps *ci_cps;
         struct ci_ps *ci_rps;
         int i;
+        struct amdgpu_ps *cps = (struct amdgpu_ps *)current_ps;
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)request_ps;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (adev == NULL || cps == NULL || rps == NULL || equal == NULL)
                 return -EINVAL;
 
-        ci_cps = ci_get_ps(cps);
-        ci_rps = ci_get_ps(rps);
+        ci_cps = ci_get_ps((struct amdgpu_ps *)cps);
+        ci_rps = ci_get_ps((struct amdgpu_ps *)rps);
 
         if (ci_cps == NULL) {
                 *equal = false;
@@ -6199,8 +6216,9 @@ static int ci_check_state_equal(struct amdgpu_device *adev,
         return 0;
 }
 
-static u32 ci_dpm_get_sclk(struct amdgpu_device *adev, bool low)
+static u32 ci_dpm_get_sclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps);
 
@@ -6210,8 +6228,9 @@ static u32 ci_dpm_get_sclk(struct amdgpu_device *adev, bool low)
                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
 }
 
-static u32 ci_dpm_get_mclk(struct amdgpu_device *adev, bool low)
+static u32 ci_dpm_get_mclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps);
 
@@ -6222,10 +6241,11 @@ static u32 ci_dpm_get_mclk(struct amdgpu_device *adev, bool low)
 }
 
 /* get temperature in millidegrees */
-static int ci_dpm_get_temp(struct amdgpu_device *adev)
+static int ci_dpm_get_temp(void *handle)
 {
         u32 temp;
         int actual_temp = 0;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         temp = (RREG32_SMC(ixCG_MULT_THERMAL_STATUS) & CG_MULT_THERMAL_STATUS__CTF_TEMP_MASK) >>
                 CG_MULT_THERMAL_STATUS__CTF_TEMP__SHIFT;
@@ -6261,7 +6281,6 @@ static int ci_dpm_early_init(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        ci_dpm_set_dpm_funcs(adev);
         ci_dpm_set_irq_funcs(adev);
 
         return 0;
@@ -6551,9 +6570,10 @@ static int ci_dpm_set_powergating_state(void *handle,
         return 0;
 }
 
-static int ci_dpm_print_clock_levels(struct amdgpu_device *adev,
+static int ci_dpm_print_clock_levels(void *handle,
                 enum pp_clock_type type, char *buf)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_single_dpm_table *sclk_table = &pi->dpm_table.sclk_table;
         struct ci_single_dpm_table *mclk_table = &pi->dpm_table.mclk_table;
@@ -6618,9 +6638,10 @@ static int ci_dpm_print_clock_levels(struct amdgpu_device *adev,
         return size;
 }
 
-static int ci_dpm_force_clock_level(struct amdgpu_device *adev,
+static int ci_dpm_force_clock_level(void *handle,
                 enum pp_clock_type type, uint32_t mask)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         if (adev->pm.dpm.forced_level & (AMD_DPM_FORCED_LEVEL_AUTO |
@@ -6664,8 +6685,9 @@ static int ci_dpm_force_clock_level(struct amdgpu_device *adev,
         return 0;
 }
 
-static int ci_dpm_get_sclk_od(struct amdgpu_device *adev)
+static int ci_dpm_get_sclk_od(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_single_dpm_table *sclk_table = &(pi->dpm_table.sclk_table);
         struct ci_single_dpm_table *golden_sclk_table =
@@ -6680,8 +6702,9 @@ static int ci_dpm_get_sclk_od(struct amdgpu_device *adev)
         return value;
 }
 
-static int ci_dpm_set_sclk_od(struct amdgpu_device *adev, uint32_t value)
+static int ci_dpm_set_sclk_od(void *handle, uint32_t value)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_ps *ps = ci_get_ps(adev->pm.dpm.requested_ps);
         struct ci_single_dpm_table *golden_sclk_table =
@@ -6698,8 +6721,9 @@ static int ci_dpm_set_sclk_od(struct amdgpu_device *adev, uint32_t value)
         return 0;
 }
 
-static int ci_dpm_get_mclk_od(struct amdgpu_device *adev)
+static int ci_dpm_get_mclk_od(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_single_dpm_table *mclk_table = &(pi->dpm_table.mclk_table);
         struct ci_single_dpm_table *golden_mclk_table =
@@ -6714,8 +6738,9 @@ static int ci_dpm_get_mclk_od(struct amdgpu_device *adev)
         return value;
 }
 
-static int ci_dpm_set_mclk_od(struct amdgpu_device *adev, uint32_t value)
+static int ci_dpm_set_mclk_od(void *handle, uint32_t value)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_ps *ps = ci_get_ps(adev->pm.dpm.requested_ps);
         struct ci_single_dpm_table *golden_mclk_table =
@@ -6732,9 +6757,10 @@ static int ci_dpm_set_mclk_od(struct amdgpu_device *adev, uint32_t value)
         return 0;
 }
 
-static int ci_dpm_get_power_profile_state(struct amdgpu_device *adev,
+static int ci_dpm_get_power_profile_state(void *handle,
                 struct amd_pp_profile *query)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         if (!pi || !query)
@@ -6851,9 +6877,10 @@ static int ci_set_power_profile_state(struct amdgpu_device *adev,
         return result;
 }
 
-static int ci_dpm_set_power_profile_state(struct amdgpu_device *adev,
+static int ci_dpm_set_power_profile_state(void *handle,
                 struct amd_pp_profile *request)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         int ret = -1;
 
@@ -6906,9 +6933,10 @@ static int ci_dpm_set_power_profile_state(struct amdgpu_device *adev,
         return 0;
 }
 
-static int ci_dpm_reset_power_profile_state(struct amdgpu_device *adev,
+static int ci_dpm_reset_power_profile_state(void *handle,
                 struct amd_pp_profile *request)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         if (!pi || !request)
@@ -6927,9 +6955,10 @@ static int ci_dpm_reset_power_profile_state(struct amdgpu_device *adev,
                 return -EINVAL;
 }
 
-static int ci_dpm_switch_power_profile(struct amdgpu_device *adev,
+static int ci_dpm_switch_power_profile(void *handle,
                 enum amd_pp_profile_type type)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct amd_pp_profile request = {0};
 
@@ -6944,11 +6973,12 @@ static int ci_dpm_switch_power_profile(struct amdgpu_device *adev,
         return 0;
 }
 
-static int ci_dpm_read_sensor(struct amdgpu_device *adev, int idx,
+static int ci_dpm_read_sensor(void *handle, int idx,
                               void *value, int *size)
 {
         u32 activity_percent = 50;
         int ret;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         /* size must be at least 4 bytes for all sensors */
         if (*size < 4)
@@ -7003,7 +7033,7 @@ const struct amd_ip_funcs ci_dpm_ip_funcs = {
         .set_powergating_state = ci_dpm_set_powergating_state,
 };
 
-static const struct amdgpu_dpm_funcs ci_dpm_funcs = {
+const struct amd_pm_funcs ci_dpm_funcs = {
         .get_temperature = &ci_dpm_get_temp,
         .pre_set_power_state = &ci_dpm_pre_set_power_state,
         .set_power_state = &ci_dpm_set_power_state,
@@ -7035,12 +7065,6 @@ static const struct amdgpu_dpm_funcs ci_dpm_funcs = {
         .read_sensor = ci_dpm_read_sensor,
 };
 
-static void ci_dpm_set_dpm_funcs(struct amdgpu_device *adev)
-{
-        if (adev->pm.funcs == NULL)
-                adev->pm.funcs = &ci_dpm_funcs;
-}
-
 static const struct amdgpu_irq_src_funcs ci_dpm_irq_funcs = {
         .set = ci_dpm_set_interrupt_state,
         .process = ci_dpm_process_interrupt,
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_dpm.h b/drivers/gpu/drm/amd/amdgpu/cik_dpm.h
index b1c8e7b446ea..c7b4349f6319 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/cik_dpm.h
@@ -26,5 +26,6 @@
 
 extern const struct amd_ip_funcs ci_dpm_ip_funcs;
 extern const struct amd_ip_funcs kv_dpm_ip_funcs;
-
+extern const struct amd_pm_funcs ci_dpm_funcs;
+extern const struct amd_pm_funcs kv_dpm_funcs;
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
index b8918432c572..07d3d895da10 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
@@ -228,6 +228,19 @@ static u32 cik_ih_get_wptr(struct amdgpu_device *adev)
  * [127:96] - reserved
  */
 
+/**
+ * cik_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool cik_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        /* Process all interrupts */
+        return true;
+}
+
  /**
  * cik_ih_decode_iv - decode an interrupt vector
  *
@@ -433,6 +446,7 @@ static const struct amd_ip_funcs cik_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs cik_ih_funcs = {
         .get_wptr = cik_ih_get_wptr,
+        .prescreen_iv = cik_ih_prescreen_iv,
         .decode_iv = cik_ih_decode_iv,
         .set_rptr = cik_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index f508f4d01e4a..60cecd117705 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -1387,8 +1387,13 @@ static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev)
 }
 
 static const struct amdgpu_vm_pte_funcs cik_sdma_vm_pte_funcs = {
+        .copy_pte_num_dw = 7,
         .copy_pte = cik_sdma_vm_copy_pte,
+
         .write_pte = cik_sdma_vm_write_pte,
+
+        .set_max_nums_pte_pde = 0x1fffff >> 3,
+        .set_pte_pde_num_dw = 10,
         .set_pte_pde = cik_sdma_vm_set_pte_pde,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
index 0c1209cdd1cb..b6cdf4afaf46 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
@@ -208,6 +208,19 @@ static u32 cz_ih_get_wptr(struct amdgpu_device *adev)
 }
 
 /**
+ * cz_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool cz_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        /* Process all interrupts */
+        return true;
+}
+
+/**
  * cz_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -414,6 +427,7 @@ static const struct amd_ip_funcs cz_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs cz_ih_funcs = {
         .get_wptr = cz_ih_get_wptr,
+        .prescreen_iv = cz_ih_prescreen_iv,
         .decode_iv = cz_ih_decode_iv,
         .set_rptr = cz_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index d228f5a99044..dbbe986f90f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -636,7 +636,194 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
                                 NUM_BANKS(ADDR_SURF_2_BANK);
                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
                         WREG32(mmGB_TILE_MODE0 + reg_offset, tilemode[reg_offset]);
-        } else if (adev->asic_type == CHIP_OLAND || adev->asic_type == CHIP_HAINAN) {
+        } else if (adev->asic_type == CHIP_OLAND) {
+                tilemode[0] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
+                tilemode[1] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
+                tilemode[2] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
+                tilemode[3] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
+                tilemode[4] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[5] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(split_equal_to_row_size) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[6] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(split_equal_to_row_size) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[7] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(split_equal_to_row_size) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
+                tilemode[8] =   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[9] =   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[10] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
+                tilemode[11] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[12] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[13] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[14] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[15] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[16] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[17] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                TILE_SPLIT(split_equal_to_row_size) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[21] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[22] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
+                tilemode[23] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[24] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+                                NUM_BANKS(ADDR_SURF_16_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+                tilemode[25] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+                                ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+                                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
+                                NUM_BANKS(ADDR_SURF_8_BANK) |
+                                BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1);
+                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
+                        WREG32(mmGB_TILE_MODE0 + reg_offset, tilemode[reg_offset]);
+        } else if (adev->asic_type == CHIP_HAINAN) {
                 tilemode[0] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
                                 ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
                                 PIPE_CONFIG(ADDR_SURF_P2) |
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 53a4af7596c1..00868764a0dd 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -1921,6 +1921,7 @@ static void gfx_v7_0_gpu_init(struct amdgpu_device *adev)
                                    ELEMENT_SIZE, 1);
         sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
                                    INDEX_STRIDE, 3);
+        WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
 
         mutex_lock(&adev->srbm_mutex);
         for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
@@ -1934,7 +1935,6 @@ static void gfx_v7_0_gpu_init(struct amdgpu_device *adev)
                 WREG32(mmSH_MEM_APE1_BASE, 1);
                 WREG32(mmSH_MEM_APE1_LIMIT, 0);
                 WREG32(mmSH_MEM_BASES, sh_mem_base);
-                WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
         }
         cik_srbm_select(adev, 0, 0, 0, 0);
         mutex_unlock(&adev->srbm_mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 0710b0b2e4b6..dfc10b1baea0 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -125,24 +125,39 @@ MODULE_FIRMWARE("amdgpu/fiji_mec2.bin");
 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin");
 
 MODULE_FIRMWARE("amdgpu/polaris11_ce.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_me.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_mec.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin");
 
 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_me.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_mec.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin");
 
 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin");
+MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin");
+MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_me.bin");
+MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_mec.bin");
+MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin");
+MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin");
 
 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
@@ -918,8 +933,17 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
                 BUG();
         }
 
-        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
-        err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
+        if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp_2.bin", chip_name);
+                err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
+                if (err == -ENOENT) {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
+                        err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
+                }
+        } else {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
+                err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
+        }
         if (err)
                 goto out;
         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
@@ -929,8 +953,17 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 
-        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
-        err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
+        if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me_2.bin", chip_name);
+                err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
+                if (err == -ENOENT) {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
+                        err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
+                }
+        } else {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
+                err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
+        }
         if (err)
                 goto out;
         err = amdgpu_ucode_validate(adev->gfx.me_fw);
@@ -941,8 +974,17 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
 
         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 
-        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
-        err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
+        if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce_2.bin", chip_name);
+                err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
+                if (err == -ENOENT) {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
+                        err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
+                }
+        } else {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
+                err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
+        }
         if (err)
                 goto out;
         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
@@ -1012,8 +1054,17 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
         for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
                 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
 
-        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
-        err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
+        if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec_2.bin", chip_name);
+                err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
+                if (err == -ENOENT) {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
+                        err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
+                }
+        } else {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
+                err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
+        }
         if (err)
                 goto out;
         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
@@ -1025,8 +1076,17 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
 
         if ((adev->asic_type != CHIP_STONEY) &&
             (adev->asic_type != CHIP_TOPAZ)) {
-                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
-                err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
+                if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2_2.bin", chip_name);
+                        err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
+                        if (err == -ENOENT) {
+                                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
+                                err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
+                        }
+                } else {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
+                        err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
+                }
                 if (!err) {
                         err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
                         if (err)
@@ -2053,6 +2113,7 @@ static int gfx_v8_0_sw_fini(void *handle)
         amdgpu_gfx_compute_mqd_sw_fini(adev);
         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
         amdgpu_gfx_kiq_fini(adev);
+        amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
 
         gfx_v8_0_mec_fini(adev);
         gfx_v8_0_rlc_fini(adev);
@@ -3707,6 +3768,8 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
                                    ELEMENT_SIZE, 1);
         sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
                                    INDEX_STRIDE, 3);
+        WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
+
         mutex_lock(&adev->srbm_mutex);
         for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
                 vi_srbm_select(adev, 0, 0, 0, i);
@@ -3730,7 +3793,6 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
 
                 WREG32(mmSH_MEM_APE1_BASE, 1);
                 WREG32(mmSH_MEM_APE1_LIMIT, 0);
-                WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
         }
         vi_srbm_select(adev, 0, 0, 0, 0);
         mutex_unlock(&adev->srbm_mutex);
@@ -4576,12 +4638,10 @@ static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
         mqd->compute_misc_reserved = 0x00000003;
-        if (!(adev->flags & AMD_IS_APU)) {
-                mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr
-                                             + offsetof(struct vi_mqd_allocation, dyamic_cu_mask));
-                mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr
-                                             + offsetof(struct vi_mqd_allocation, dyamic_cu_mask));
-        }
+        mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr
+                                                     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
+        mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr
+                                                     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
         eop_base_addr = ring->eop_gpu_addr >> 8;
         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
@@ -4752,7 +4812,7 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
 
         gfx_v8_0_kiq_setting(ring);
 
-        if (adev->gfx.in_reset) { /* for GPU_RESET case */
+        if (adev->in_sriov_reset) { /* for GPU_RESET case */
                 /* reset MQD to a clean status */
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
@@ -4767,8 +4827,8 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
                 mutex_unlock(&adev->srbm_mutex);
         } else {
                 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
-                ((struct vi_mqd_allocation *)mqd)->dyamic_cu_mask = 0xFFFFFFFF;
-                ((struct vi_mqd_allocation *)mqd)->dyamic_rb_mask = 0xFFFFFFFF;
+                ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
+                ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
                 mutex_lock(&adev->srbm_mutex);
                 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
                 gfx_v8_0_mqd_init(ring);
@@ -4789,10 +4849,10 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
         struct vi_mqd *mqd = ring->mqd_ptr;
         int mqd_idx = ring - &adev->gfx.compute_ring[0];
 
-        if (!adev->gfx.in_reset && !adev->gfx.in_suspend) {
+        if (!adev->in_sriov_reset && !adev->gfx.in_suspend) {
                 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
-                ((struct vi_mqd_allocation *)mqd)->dyamic_cu_mask = 0xFFFFFFFF;
-                ((struct vi_mqd_allocation *)mqd)->dyamic_rb_mask = 0xFFFFFFFF;
+                ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
+                ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
                 mutex_lock(&adev->srbm_mutex);
                 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
                 gfx_v8_0_mqd_init(ring);
@@ -4801,7 +4861,7 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
 
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
-        } else if (adev->gfx.in_reset) { /* for GPU_RESET case */
+        } else if (adev->in_sriov_reset) { /* for GPU_RESET case */
                 /* reset MQD to a clean status */
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
@@ -4974,12 +5034,69 @@ static int gfx_v8_0_hw_init(void *handle)
         return r;
 }
 
+static int gfx_v8_0_kcq_disable(struct amdgpu_ring *kiq_ring,struct amdgpu_ring *ring)
+{
+        struct amdgpu_device *adev = kiq_ring->adev;
+        uint32_t scratch, tmp = 0;
+        int r, i;
+
+        r = amdgpu_gfx_scratch_get(adev, &scratch);
+        if (r) {
+                DRM_ERROR("Failed to get scratch reg (%d).\n", r);
+                return r;
+        }
+        WREG32(scratch, 0xCAFEDEAD);
+
+        r = amdgpu_ring_alloc(kiq_ring, 10);
+        if (r) {
+                DRM_ERROR("Failed to lock KIQ (%d).\n", r);
+                amdgpu_gfx_scratch_free(adev, scratch);
+                return r;
+        }
+
+        /* unmap queues */
+        amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
+        amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
+                                                PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */
+                                                PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
+                                                PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) |
+                                                PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
+        amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
+        amdgpu_ring_write(kiq_ring, 0);
+        amdgpu_ring_write(kiq_ring, 0);
+        amdgpu_ring_write(kiq_ring, 0);
+        /* write to scratch for completion */
+        amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
+        amdgpu_ring_write(kiq_ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
+        amdgpu_ring_write(kiq_ring, 0xDEADBEEF);
+        amdgpu_ring_commit(kiq_ring);
+
+        for (i = 0; i < adev->usec_timeout; i++) {
+                tmp = RREG32(scratch);
+                if (tmp == 0xDEADBEEF)
+                        break;
+                DRM_UDELAY(1);
+        }
+        if (i >= adev->usec_timeout) {
+                DRM_ERROR("KCQ disabled failed (scratch(0x%04X)=0x%08X)\n", scratch, tmp);
+                r = -EINVAL;
+        }
+        amdgpu_gfx_scratch_free(adev, scratch);
+        return r;
+}
+
 static int gfx_v8_0_hw_fini(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int i;
 
         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+
+        /* disable KCQ to avoid CPC touch memory not valid anymore */
+        for (i = 0; i < adev->gfx.num_compute_rings; i++)
+                gfx_v8_0_kcq_disable(&adev->gfx.kiq.ring, &adev->gfx.compute_ring[i]);
+
         if (amdgpu_sriov_vf(adev)) {
                 pr_debug("For SRIOV client, shouldn't do anything.\n");
                 return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index b39f81dda847..deeaee1457ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -66,38 +66,70 @@ MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
 
 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
 {
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID1), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID1)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID2), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID2)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID3), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID3)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID4), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID4)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID5), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID5)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID6), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID6)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID7), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID7)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID8), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID8)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID9), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID9)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID10), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID10)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID11), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID11)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID12), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID12)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID13), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID13)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID14), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID14)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID15), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID15)}
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID1),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID1) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID2),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID2) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID3),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID3) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID4),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID4) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID5),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID5) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID6),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID6) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID7),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID7) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID8),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID8) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID9),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID9) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID10),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID10) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID11),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID11) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID12),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID12)},
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID13),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID13) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID14),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID14) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID15),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID15) }
 };
 
 static const u32 golden_settings_gc_9_0[] =
@@ -352,6 +384,25 @@ err1:
         return r;
 }
 
+
+static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
+{
+        release_firmware(adev->gfx.pfp_fw);
+        adev->gfx.pfp_fw = NULL;
+        release_firmware(adev->gfx.me_fw);
+        adev->gfx.me_fw = NULL;
+        release_firmware(adev->gfx.ce_fw);
+        adev->gfx.ce_fw = NULL;
+        release_firmware(adev->gfx.rlc_fw);
+        adev->gfx.rlc_fw = NULL;
+        release_firmware(adev->gfx.mec_fw);
+        adev->gfx.mec_fw = NULL;
+        release_firmware(adev->gfx.mec2_fw);
+        adev->gfx.mec2_fw = NULL;
+
+        kfree(adev->gfx.rlc.register_list_format);
+}
+
 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 {
         const char *chip_name;
@@ -1120,30 +1171,22 @@ static int gfx_v9_0_ngg_en(struct amdgpu_device *adev)
 {
         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
         int r;
-        u32 data;
-        u32 size;
-        u32 base;
+        u32 data, base;
 
         if (!amdgpu_ngg)
                 return 0;
 
         /* Program buffer size */
-        data = 0;
-        size = adev->gfx.ngg.buf[NGG_PRIM].size / 256;
-        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, INDEX_BUF_SIZE, size);
-
-        size = adev->gfx.ngg.buf[NGG_POS].size / 256;
-        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, POS_BUF_SIZE, size);
-
+        data = REG_SET_FIELD(0, WD_BUF_RESOURCE_1, INDEX_BUF_SIZE,
+                             adev->gfx.ngg.buf[NGG_PRIM].size >> 8);
+        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, POS_BUF_SIZE,
+                             adev->gfx.ngg.buf[NGG_POS].size >> 8);
         WREG32_SOC15(GC, 0, mmWD_BUF_RESOURCE_1, data);
 
-        data = 0;
-        size = adev->gfx.ngg.buf[NGG_CNTL].size / 256;
-        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, CNTL_SB_BUF_SIZE, size);
-
-        size = adev->gfx.ngg.buf[NGG_PARAM].size / 1024;
-        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, PARAM_BUF_SIZE, size);
-
+        data = REG_SET_FIELD(0, WD_BUF_RESOURCE_2, CNTL_SB_BUF_SIZE,
+                             adev->gfx.ngg.buf[NGG_CNTL].size >> 8);
+        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, PARAM_BUF_SIZE,
+                             adev->gfx.ngg.buf[NGG_PARAM].size >> 10);
         WREG32_SOC15(GC, 0, mmWD_BUF_RESOURCE_2, data);
 
         /* Program buffer base address */
@@ -1306,7 +1349,10 @@ static int gfx_v9_0_sw_init(void *handle)
         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
                 ring = &adev->gfx.gfx_ring[i];
                 ring->ring_obj = NULL;
-                sprintf(ring->name, "gfx");
+                if (!i)
+                        sprintf(ring->name, "gfx");
+                else
+                        sprintf(ring->name, "gfx_%d", i);
                 ring->use_doorbell = true;
                 ring->doorbell_index = AMDGPU_DOORBELL64_GFX_RING0 << 1;
                 r = amdgpu_ring_init(adev, ring, 1024,
@@ -1346,7 +1392,7 @@ static int gfx_v9_0_sw_init(void *handle)
                 return r;
 
         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
-        r = amdgpu_gfx_compute_mqd_sw_init(adev, sizeof(struct v9_mqd));
+        r = amdgpu_gfx_compute_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
         if (r)
                 return r;
 
@@ -1398,9 +1444,11 @@ static int gfx_v9_0_sw_fini(void *handle)
         amdgpu_gfx_compute_mqd_sw_fini(adev);
         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
         amdgpu_gfx_kiq_fini(adev);
+        amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
 
         gfx_v9_0_mec_fini(adev);
         gfx_v9_0_ngg_fini(adev);
+        gfx_v9_0_free_microcode(adev);
 
         return 0;
 }
@@ -1740,11 +1788,7 @@ static int gfx_v9_0_init_rlc_save_restore_list(struct amdgpu_device *adev)
 
 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
 {
-        u32 tmp = 0;
-
-        tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
-        tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
-        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
+        WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
 }
 
 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
@@ -1822,16 +1866,11 @@ static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev
         uint32_t default_data = 0;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-
-        if (enable == true) {
-                data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
-                if (default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        } else {
-                data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
-                if(default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        }
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
+                             enable ? 1 : 0);
+        if (default_data != data)
+                WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
 
 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
@@ -1841,16 +1880,11 @@ static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *ad
         uint32_t default_data = 0;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-
-        if (enable == true) {
-                data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
-                if(default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        } else {
-                data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
-                if(default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        }
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
+                             enable ? 1 : 0);
+        if(default_data != data)
+                WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
 
 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
@@ -1860,16 +1894,11 @@ static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
         uint32_t default_data = 0;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-
-        if (enable == true) {
-                data &= ~RLC_PG_CNTL__CP_PG_DISABLE_MASK;
-                if(default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        } else {
-                data |= RLC_PG_CNTL__CP_PG_DISABLE_MASK;
-                if(default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        }
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             CP_PG_DISABLE,
+                             enable ? 0 : 1);
+        if(default_data != data)
+                WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
 
 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
@@ -1878,10 +1907,9 @@ static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
         uint32_t data, default_data;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-        if (enable == true)
-                data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
-        else
-                data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             GFX_POWER_GATING_ENABLE,
+                             enable ? 1 : 0);
         if(default_data != data)
                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
@@ -1892,10 +1920,9 @@ static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
         uint32_t data, default_data;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-        if (enable == true)
-                data |= RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK;
-        else
-                data &= ~RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK;
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             GFX_PIPELINE_PG_ENABLE,
+                             enable ? 1 : 0);
         if(default_data != data)
                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 
@@ -1910,10 +1937,9 @@ static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *ade
         uint32_t data, default_data;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-        if (enable == true)
-                data |= RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
-        else
-                data &= ~RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             STATIC_PER_CU_PG_ENABLE,
+                             enable ? 1 : 0);
         if(default_data != data)
                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
@@ -1924,10 +1950,9 @@ static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *ad
         uint32_t data, default_data;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-        if (enable == true)
-                data |= RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
-        else
-                data &= ~RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             DYN_PER_CU_PG_ENABLE,
+                             enable ? 1 : 0);
         if(default_data != data)
                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
@@ -1967,13 +1992,8 @@ static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
 
 void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
 {
-        u32 tmp = RREG32_SOC15(GC, 0, mmRLC_CNTL);
-
-        tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
-        WREG32_SOC15(GC, 0, mmRLC_CNTL, tmp);
-
+        WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
-
         gfx_v9_0_wait_for_rlc_serdes(adev);
 }
 
@@ -2045,8 +2065,10 @@ static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
 {
         int r;
 
-        if (amdgpu_sriov_vf(adev))
+        if (amdgpu_sriov_vf(adev)) {
+                gfx_v9_0_init_csb(adev);
                 return 0;
+        }
 
         gfx_v9_0_rlc_stop(adev);
 
@@ -2157,7 +2179,7 @@ static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
         const struct cs_section_def *sect = NULL;
         const struct cs_extent_def *ext = NULL;
-        int r, i;
+        int r, i, tmp;
 
         /* init the CP */
         WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
@@ -2165,7 +2187,7 @@ static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
 
         gfx_v9_0_cp_gfx_enable(adev, true);
 
-        r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4);
+        r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
         if (r) {
                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
                 return r;
@@ -2203,6 +2225,12 @@ static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
         amdgpu_ring_write(ring, 0x8000);
         amdgpu_ring_write(ring, 0x8000);
 
+        amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
+        tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
+                (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
+        amdgpu_ring_write(ring, tmp);
+        amdgpu_ring_write(ring, 0);
+
         amdgpu_ring_commit(ring);
 
         return 0;
@@ -2457,6 +2485,13 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
         mqd->compute_misc_reserved = 0x00000003;
 
+        mqd->dynamic_cu_mask_addr_lo =
+                lower_32_bits(ring->mqd_gpu_addr
+                              + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
+        mqd->dynamic_cu_mask_addr_hi =
+                upper_32_bits(ring->mqd_gpu_addr
+                              + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
+
         eop_base_addr = ring->eop_gpu_addr >> 8;
         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
@@ -2480,10 +2515,10 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
                                     DOORBELL_SOURCE, 0);
                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
                                     DOORBELL_HIT, 0);
-        }
-        else
+        } else {
                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
                                          DOORBELL_EN, 0);
+        }
 
         mqd->cp_hqd_pq_doorbell_control = tmp;
 
@@ -2686,10 +2721,10 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
 
         gfx_v9_0_kiq_setting(ring);
 
-        if (adev->gfx.in_reset) { /* for GPU_RESET case */
+        if (adev->in_sriov_reset) { /* for GPU_RESET case */
                 /* reset MQD to a clean status */
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
-                        memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
+                        memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
 
                 /* reset ring buffer */
                 ring->wptr = 0;
@@ -2701,7 +2736,9 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
                 soc15_grbm_select(adev, 0, 0, 0, 0);
                 mutex_unlock(&adev->srbm_mutex);
         } else {
-                memset((void *)mqd, 0, sizeof(*mqd));
+                memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
+                ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
+                ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
                 mutex_lock(&adev->srbm_mutex);
                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
                 gfx_v9_0_mqd_init(ring);
@@ -2710,7 +2747,7 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
                 mutex_unlock(&adev->srbm_mutex);
 
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
-                        memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
+                        memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
         }
 
         return 0;
@@ -2722,8 +2759,10 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
         struct v9_mqd *mqd = ring->mqd_ptr;
         int mqd_idx = ring - &adev->gfx.compute_ring[0];
 
-        if (!adev->gfx.in_reset && !adev->gfx.in_suspend) {
-                memset((void *)mqd, 0, sizeof(*mqd));
+        if (!adev->in_sriov_reset && !adev->gfx.in_suspend) {
+                memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
+                ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
+                ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
                 mutex_lock(&adev->srbm_mutex);
                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
                 gfx_v9_0_mqd_init(ring);
@@ -2731,11 +2770,11 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
                 mutex_unlock(&adev->srbm_mutex);
 
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
-                        memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
-        } else if (adev->gfx.in_reset) { /* for GPU_RESET case */
+                        memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
+        } else if (adev->in_sriov_reset) { /* for GPU_RESET case */
                 /* reset MQD to a clean status */
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
-                        memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
+                        memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
 
                 /* reset ring buffer */
                 ring->wptr = 0;
@@ -2876,12 +2915,70 @@ static int gfx_v9_0_hw_init(void *handle)
         return r;
 }
 
+static int gfx_v9_0_kcq_disable(struct amdgpu_ring *kiq_ring,struct amdgpu_ring *ring)
+{
+        struct amdgpu_device *adev = kiq_ring->adev;
+        uint32_t scratch, tmp = 0;
+        int r, i;
+
+        r = amdgpu_gfx_scratch_get(adev, &scratch);
+        if (r) {
+                DRM_ERROR("Failed to get scratch reg (%d).\n", r);
+                return r;
+        }
+        WREG32(scratch, 0xCAFEDEAD);
+
+        r = amdgpu_ring_alloc(kiq_ring, 10);
+        if (r) {
+                DRM_ERROR("Failed to lock KIQ (%d).\n", r);
+                amdgpu_gfx_scratch_free(adev, scratch);
+                return r;
+        }
+
+        /* unmap queues */
+        amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
+        amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
+                                                PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */
+                                                PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
+                                                PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) |
+                                                PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
+        amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
+        amdgpu_ring_write(kiq_ring, 0);
+        amdgpu_ring_write(kiq_ring, 0);
+        amdgpu_ring_write(kiq_ring, 0);
+        /* write to scratch for completion */
+        amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
+        amdgpu_ring_write(kiq_ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
+        amdgpu_ring_write(kiq_ring, 0xDEADBEEF);
+        amdgpu_ring_commit(kiq_ring);
+
+        for (i = 0; i < adev->usec_timeout; i++) {
+                tmp = RREG32(scratch);
+                if (tmp == 0xDEADBEEF)
+                        break;
+                DRM_UDELAY(1);
+        }
+        if (i >= adev->usec_timeout) {
+                DRM_ERROR("KCQ disabled failed (scratch(0x%04X)=0x%08X)\n", scratch, tmp);
+                r = -EINVAL;
+        }
+        amdgpu_gfx_scratch_free(adev, scratch);
+        return r;
+}
+
+
 static int gfx_v9_0_hw_fini(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int i;
 
         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+
+        /* disable KCQ to avoid CPC touch memory not valid anymore */
+        for (i = 0; i < adev->gfx.num_compute_rings; i++)
+                gfx_v9_0_kcq_disable(&adev->gfx.kiq.ring, &adev->gfx.compute_ring[i]);
+
         if (amdgpu_sriov_vf(adev)) {
                 pr_debug("For SRIOV client, shouldn't do anything.\n");
                 return 0;
@@ -2924,15 +3021,10 @@ static bool gfx_v9_0_is_idle(void *handle)
 static int gfx_v9_0_wait_for_idle(void *handle)
 {
         unsigned i;
-        u32 tmp;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         for (i = 0; i < adev->usec_timeout; i++) {
-                /* read MC_STATUS */
-                tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS) &
-                        GRBM_STATUS__GUI_ACTIVE_MASK;
-
-                if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
+                if (gfx_v9_0_is_idle(handle))
                         return 0;
                 udelay(1);
         }
@@ -3493,7 +3585,9 @@ static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
         u32 ref_and_mask, reg_mem_engine;
         struct nbio_hdp_flush_reg *nbio_hf_reg;
 
-        if (ring->adev->asic_type == CHIP_VEGA10)
+        if (ring->adev->flags & AMD_IS_APU)
+                nbio_hf_reg = &nbio_v7_0_hdp_flush_reg;
+        else
                 nbio_hf_reg = &nbio_v6_1_hdp_flush_reg;
 
         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
@@ -3522,7 +3616,7 @@ static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 static void gfx_v9_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
         gfx_v9_0_write_data_to_reg(ring, 0, true,
-                                   SOC15_REG_OFFSET(HDP, 0, mmHDP_DEBUG0), 1);
+                                   SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1);
 }
 
 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
@@ -3751,6 +3845,12 @@ static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
         amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
 }
 
+static void gfx_v9_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start)
+{
+        amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
+        amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */
+}
+
 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
 {
         uint32_t dw2 = 0;
@@ -3758,6 +3858,8 @@ static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
         if (amdgpu_sriov_vf(ring->adev))
                 gfx_v9_0_ring_emit_ce_meta(ring);
 
+        gfx_v9_0_ring_emit_tmz(ring, true);
+
         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
                 /* set load_global_config & load_global_uconfig */
@@ -3808,12 +3910,6 @@ static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigne
                 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
 }
 
-static void gfx_v9_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start)
-{
-        amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
-        amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */
-}
-
 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
 {
         struct amdgpu_device *adev = ring->adev;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
index 408723ef157c..c17996e18086 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
@@ -144,8 +144,8 @@ static void gfxhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
         WREG32_SOC15(GC, 0, mmVM_L2_CNTL2, tmp);
 
         tmp = mmVM_L2_CNTL3_DEFAULT;
-        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12);
-        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
+        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
+        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
         WREG32_SOC15(GC, 0, mmVM_L2_CNTL3, tmp);
 
         tmp = mmVM_L2_CNTL4_DEFAULT;
@@ -319,6 +319,12 @@ void gfxhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev,
                         WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
                         EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+        if (!value) {
+                tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+                                CRASH_ON_NO_RETRY_FAULT, 1);
+                tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+                                CRASH_ON_RETRY_FAULT, 1);
+    }
         WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index 93c45f26b7c8..f4603a7c8ef3 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -332,7 +332,24 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
         adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
         adev->mc.visible_vram_size = adev->mc.aper_size;
 
-        amdgpu_gart_set_defaults(adev);
+        /* set the gart size */
+        if (amdgpu_gart_size == -1) {
+                switch (adev->asic_type) {
+                case CHIP_HAINAN:    /* no MM engines */
+                default:
+                        adev->mc.gart_size = 256ULL << 20;
+                        break;
+                case CHIP_VERDE:    /* UVD, VCE do not support GPUVM */
+                case CHIP_TAHITI:   /* UVD, VCE do not support GPUVM */
+                case CHIP_PITCAIRN: /* UVD, VCE do not support GPUVM */
+                case CHIP_OLAND:    /* UVD, VCE do not support GPUVM */
+                        adev->mc.gart_size = 1024ULL << 20;
+                        break;
+                }
+        } else {
+                adev->mc.gart_size = (u64)amdgpu_gart_size << 20;
+        }
+
         gmc_v6_0_vram_gtt_location(adev, &adev->mc);
 
         return 0;
@@ -461,6 +478,7 @@ static void gmc_v6_0_set_prt(struct amdgpu_device *adev, bool enable)
 static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
 {
         int r, i;
+        u32 field;
 
         if (adev->gart.robj == NULL) {
                 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
@@ -488,10 +506,12 @@ static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
         WREG32(mmVM_L2_CNTL2,
                VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK |
                VM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK);
+
+        field = adev->vm_manager.fragment_size;
         WREG32(mmVM_L2_CNTL3,
                VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
-               (4UL << VM_L2_CNTL3__BANK_SELECT__SHIFT) |
-               (4UL << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
+               (field << VM_L2_CNTL3__BANK_SELECT__SHIFT) |
+               (field << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
         /* setup context0 */
         WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gart_start >> 12);
         WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gart_end >> 12);
@@ -811,7 +831,7 @@ static int gmc_v6_0_sw_init(void *handle)
         if (r)
                 return r;
 
-        amdgpu_vm_adjust_size(adev, 64);
+        amdgpu_vm_adjust_size(adev, 64, 9);
         adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
 
         adev->mc.mc_mask = 0xffffffffffULL;
@@ -881,6 +901,8 @@ static int gmc_v6_0_sw_fini(void *handle)
         gmc_v6_0_gart_fini(adev);
         amdgpu_gem_force_release(adev);
         amdgpu_bo_fini(adev);
+        release_firmware(adev->mc.fw);
+        adev->mc.fw = NULL;
 
         return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 4a9e84062874..b0528ca9207b 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -386,7 +386,27 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
         if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
                 adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-        amdgpu_gart_set_defaults(adev);
+        /* set the gart size */
+        if (amdgpu_gart_size == -1) {
+                switch (adev->asic_type) {
+                case CHIP_TOPAZ:     /* no MM engines */
+                default:
+                        adev->mc.gart_size = 256ULL << 20;
+                        break;
+#ifdef CONFIG_DRM_AMDGPU_CIK
+                case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */
+                case CHIP_HAWAII:  /* UVD, VCE do not support GPUVM */
+                case CHIP_KAVERI:  /* UVD, VCE do not support GPUVM */
+                case CHIP_KABINI:  /* UVD, VCE do not support GPUVM */
+                case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */
+                        adev->mc.gart_size = 1024ULL << 20;
+                        break;
+#endif
+                }
+        } else {
+                adev->mc.gart_size = (u64)amdgpu_gart_size << 20;
+        }
+
         gmc_v7_0_vram_gtt_location(adev, &adev->mc);
 
         return 0;
@@ -562,7 +582,7 @@ static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable)
 static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
 {
         int r, i;
-        u32 tmp;
+        u32 tmp, field;
 
         if (adev->gart.robj == NULL) {
                 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
@@ -592,10 +612,12 @@ static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
         tmp = REG_SET_FIELD(0, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
         WREG32(mmVM_L2_CNTL2, tmp);
+
+        field = adev->vm_manager.fragment_size;
         tmp = RREG32(mmVM_L2_CNTL3);
         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
-        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 4);
-        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 4);
+        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, field);
+        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field);
         WREG32(mmVM_L2_CNTL3, tmp);
         /* setup context0 */
         WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gart_start >> 12);
@@ -948,7 +970,7 @@ static int gmc_v7_0_sw_init(void *handle)
          * Currently set to 4GB ((1 << 20) 4k pages).
          * Max GPUVM size for cayman and SI is 40 bits.
          */
-        amdgpu_vm_adjust_size(adev, 64);
+        amdgpu_vm_adjust_size(adev, 64, 9);
         adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
 
         /* Set the internal MC address mask
@@ -1028,6 +1050,8 @@ static int gmc_v7_0_sw_fini(void *handle)
         gmc_v7_0_gart_fini(adev);
         amdgpu_gem_force_release(adev);
         amdgpu_bo_fini(adev);
+        release_firmware(adev->mc.fw);
+        adev->mc.fw = NULL;
 
         return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 85c937b5e40b..f368cfe2f585 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -562,7 +562,26 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
         if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
                 adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-        amdgpu_gart_set_defaults(adev);
+        /* set the gart size */
+        if (amdgpu_gart_size == -1) {
+                switch (adev->asic_type) {
+                case CHIP_POLARIS11: /* all engines support GPUVM */
+                case CHIP_POLARIS10: /* all engines support GPUVM */
+                case CHIP_POLARIS12: /* all engines support GPUVM */
+                default:
+                        adev->mc.gart_size = 256ULL << 20;
+                        break;
+                case CHIP_TONGA:   /* UVD, VCE do not support GPUVM */
+                case CHIP_FIJI:    /* UVD, VCE do not support GPUVM */
+                case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */
+                case CHIP_STONEY:  /* UVD does not support GPUVM, DCE SG support */
+                        adev->mc.gart_size = 1024ULL << 20;
+                        break;
+                }
+        } else {
+                adev->mc.gart_size = (u64)amdgpu_gart_size << 20;
+        }
+
         gmc_v8_0_vram_gtt_location(adev, &adev->mc);
 
         return 0;
@@ -762,7 +781,7 @@ static void gmc_v8_0_set_prt(struct amdgpu_device *adev, bool enable)
 static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
 {
         int r, i;
-        u32 tmp;
+        u32 tmp, field;
 
         if (adev->gart.robj == NULL) {
                 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
@@ -793,10 +812,12 @@ static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
         WREG32(mmVM_L2_CNTL2, tmp);
+
+        field = adev->vm_manager.fragment_size;
         tmp = RREG32(mmVM_L2_CNTL3);
         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
-        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 4);
-        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 4);
+        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, field);
+        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field);
         WREG32(mmVM_L2_CNTL3, tmp);
         /* XXX: set to enable PTE/PDE in system memory */
         tmp = RREG32(mmVM_L2_CNTL4);
@@ -1046,7 +1067,7 @@ static int gmc_v8_0_sw_init(void *handle)
          * Currently set to 4GB ((1 << 20) 4k pages).
          * Max GPUVM size for cayman and SI is 40 bits.
          */
-        amdgpu_vm_adjust_size(adev, 64);
+        amdgpu_vm_adjust_size(adev, 64, 9);
         adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
 
         /* Set the internal MC address mask
@@ -1126,6 +1147,8 @@ static int gmc_v8_0_sw_fini(void *handle)
         gmc_v8_0_gart_fini(adev);
         amdgpu_gem_force_release(adev);
         amdgpu_bo_fini(adev);
+        release_firmware(adev->mc.fw);
+        adev->mc.fw = NULL;
 
         return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index c22899a08106..621699331e09 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -32,6 +32,8 @@
 #include "vega10/DC/dce_12_0_offset.h"
 #include "vega10/DC/dce_12_0_sh_mask.h"
 #include "vega10/vega10_enum.h"
+#include "vega10/MMHUB/mmhub_1_0_offset.h"
+#include "vega10/ATHUB/athub_1_0_offset.h"
 
 #include "soc15_common.h"
 
@@ -71,13 +73,25 @@ static const u32 golden_settings_vega10_hdp[] =
         0xf6e, 0x0fffffff, 0x00000000,
 };
 
+static const u32 golden_settings_mmhub_1_0_0[] =
+{
+        SOC15_REG_OFFSET(MMHUB, 0, mmDAGB1_WRCLI2), 0x00000007, 0xfe5fe0fa,
+        SOC15_REG_OFFSET(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0), 0x00000030, 0x55555565
+};
+
+static const u32 golden_settings_athub_1_0_0[] =
+{
+        SOC15_REG_OFFSET(ATHUB, 0, mmRPB_ARB_CNTL), 0x0000ff00, 0x00000800,
+        SOC15_REG_OFFSET(ATHUB, 0, mmRPB_ARB_CNTL2), 0x00ff00ff, 0x00080008
+};
+
 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
                                         struct amdgpu_irq_src *src,
                                         unsigned type,
                                         enum amdgpu_interrupt_state state)
 {
         struct amdgpu_vmhub *hub;
-        u32 tmp, reg, bits, i;
+        u32 tmp, reg, bits, i, j;
 
         bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
                 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
@@ -89,43 +103,26 @@ static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
 
         switch (state) {
         case AMDGPU_IRQ_STATE_DISABLE:
-                /* MM HUB */
-                hub = &adev->vmhub[AMDGPU_MMHUB];
-                for (i = 0; i< 16; i++) {
-                        reg = hub->vm_context0_cntl + i;
-                        tmp = RREG32(reg);
-                        tmp &= ~bits;
-                        WREG32(reg, tmp);
-                }
-
-                /* GFX HUB */
-                hub = &adev->vmhub[AMDGPU_GFXHUB];
-                for (i = 0; i < 16; i++) {
-                        reg = hub->vm_context0_cntl + i;
-                        tmp = RREG32(reg);
-                        tmp &= ~bits;
-                        WREG32(reg, tmp);
+                for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) {
+                        hub = &adev->vmhub[j];
+                        for (i = 0; i < 16; i++) {
+                                reg = hub->vm_context0_cntl + i;
+                                tmp = RREG32(reg);
+                                tmp &= ~bits;
+                                WREG32(reg, tmp);
+                        }
                 }
                 break;
         case AMDGPU_IRQ_STATE_ENABLE:
-                /* MM HUB */
-                hub = &adev->vmhub[AMDGPU_MMHUB];
-                for (i = 0; i< 16; i++) {
-                        reg = hub->vm_context0_cntl + i;
-                        tmp = RREG32(reg);
-                        tmp |= bits;
-                        WREG32(reg, tmp);
+                for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) {
+                        hub = &adev->vmhub[j];
+                        for (i = 0; i < 16; i++) {
+                                reg = hub->vm_context0_cntl + i;
+                                tmp = RREG32(reg);
+                                tmp |= bits;
+                                WREG32(reg, tmp);
+                        }
                 }
-
-                /* GFX HUB */
-                hub = &adev->vmhub[AMDGPU_GFXHUB];
-                for (i = 0; i < 16; i++) {
-                        reg = hub->vm_context0_cntl + i;
-                        tmp = RREG32(reg);
-                        tmp |= bits;
-                        WREG32(reg, tmp);
-                }
-                break;
         default:
                 break;
         }
@@ -499,7 +496,21 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
         if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
                 adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-        amdgpu_gart_set_defaults(adev);
+        /* set the gart size */
+        if (amdgpu_gart_size == -1) {
+                switch (adev->asic_type) {
+                case CHIP_VEGA10:  /* all engines support GPUVM */
+                default:
+                        adev->mc.gart_size = 256ULL << 20;
+                        break;
+                case CHIP_RAVEN:   /* DCE SG support */
+                        adev->mc.gart_size = 1024ULL << 20;
+                        break;
+                }
+        } else {
+                adev->mc.gart_size = (u64)amdgpu_gart_size << 20;
+        }
+
         gmc_v9_0_vram_gtt_location(adev, &adev->mc);
 
         return 0;
@@ -541,9 +552,10 @@ static int gmc_v9_0_sw_init(void *handle)
                         adev->vm_manager.vm_size = 1U << 18;
                         adev->vm_manager.block_size = 9;
                         adev->vm_manager.num_level = 3;
+                        amdgpu_vm_set_fragment_size(adev, 9);
                 } else {
-                        /* vm_size is 64GB for legacy 2-level page support*/
-                        amdgpu_vm_adjust_size(adev, 64);
+                        /* vm_size is 64GB for legacy 2-level page support */
+                        amdgpu_vm_adjust_size(adev, 64, 9);
                         adev->vm_manager.num_level = 1;
                 }
                 break;
@@ -558,14 +570,16 @@ static int gmc_v9_0_sw_init(void *handle)
                 adev->vm_manager.vm_size = 1U << 18;
                 adev->vm_manager.block_size = 9;
                 adev->vm_manager.num_level = 3;
+                amdgpu_vm_set_fragment_size(adev, 9);
                 break;
         default:
                 break;
         }
 
-        DRM_INFO("vm size is %llu GB, block size is %u-bit\n",
+        DRM_INFO("vm size is %llu GB, block size is %u-bit,fragment size is %u-bit\n",
                         adev->vm_manager.vm_size,
-                        adev->vm_manager.block_size);
+                        adev->vm_manager.block_size,
+                        adev->vm_manager.fragment_size);
 
         /* This interrupt is VMC page fault.*/
         r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_VMC, 0,
@@ -665,8 +679,17 @@ static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
 {
         switch (adev->asic_type) {
         case CHIP_VEGA10:
+                amdgpu_program_register_sequence(adev,
+                                                golden_settings_mmhub_1_0_0,
+                                                (const u32)ARRAY_SIZE(golden_settings_mmhub_1_0_0));
+                amdgpu_program_register_sequence(adev,
+                                                golden_settings_athub_1_0_0,
+                                                (const u32)ARRAY_SIZE(golden_settings_athub_1_0_0));
                 break;
         case CHIP_RAVEN:
+                amdgpu_program_register_sequence(adev,
+                                                golden_settings_athub_1_0_0,
+                                                (const u32)ARRAY_SIZE(golden_settings_athub_1_0_0));
                 break;
         default:
                 break;
@@ -696,12 +719,6 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
         if (r)
                 return r;
 
-        /* After HDP is initialized, flush HDP.*/
-        if (adev->flags & AMD_IS_APU)
-                nbio_v7_0_hdp_flush(adev);
-        else
-                nbio_v6_1_hdp_flush(adev);
-
         switch (adev->asic_type) {
         case CHIP_RAVEN:
                 mmhub_v1_0_initialize_power_gating(adev);
@@ -719,13 +736,16 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
         if (r)
                 return r;
 
-        tmp = RREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL);
-        tmp |= HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE_MASK;
-        WREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL, tmp);
+        WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1);
 
         tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
         WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
 
+        /* After HDP is initialized, flush HDP.*/
+        if (adev->flags & AMD_IS_APU)
+                nbio_v7_0_hdp_flush(adev);
+        else
+                nbio_v6_1_hdp_flush(adev);
 
         if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
                 value = false;
@@ -734,7 +754,6 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
 
         gfxhub_v1_0_set_fault_enable_default(adev, value);
         mmhub_v1_0_set_fault_enable_default(adev, value);
-
         gmc_v9_0_gart_flush_gpu_tlb(adev, 0);
 
         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
@@ -753,17 +772,11 @@ static int gmc_v9_0_hw_init(void *handle)
         gmc_v9_0_init_golden_registers(adev);
 
         if (adev->mode_info.num_crtc) {
-                u32 tmp;
-
                 /* Lockout access through VGA aperture*/
-                tmp = RREG32_SOC15(DCE, 0, mmVGA_HDP_CONTROL);
-                tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
-                WREG32_SOC15(DCE, 0, mmVGA_HDP_CONTROL, tmp);
+                WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
 
                 /* disable VGA render */
-                tmp = RREG32_SOC15(DCE, 0, mmVGA_RENDER_CONTROL);
-                tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
-                WREG32_SOC15(DCE, 0, mmVGA_RENDER_CONTROL, tmp);
+                WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
         }
 
         r = gmc_v9_0_gart_enable(adev);
@@ -805,9 +818,7 @@ static int gmc_v9_0_suspend(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        gmc_v9_0_hw_fini(adev);
-
-        return 0;
+        return gmc_v9_0_hw_fini(adev);
 }
 
 static int gmc_v9_0_resume(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
index 7a0ea27ac429..65ed6d3a8f05 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
@@ -208,6 +208,19 @@ static u32 iceland_ih_get_wptr(struct amdgpu_device *adev)
 }
 
 /**
+ * iceland_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool iceland_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        /* Process all interrupts */
+        return true;
+}
+
+/**
  * iceland_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -412,6 +425,7 @@ static const struct amd_ip_funcs iceland_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs iceland_ih_funcs = {
         .get_wptr = iceland_ih_get_wptr,
+        .prescreen_iv = iceland_ih_prescreen_iv,
         .decode_iv = iceland_ih_decode_iv,
         .set_rptr = iceland_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index 3bbf2ccfca89..b57399a462c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -42,7 +42,6 @@
 #define KV_MINIMUM_ENGINE_CLOCK         800
 #define SMC_RAM_END                     0x40000
 
-static void kv_dpm_set_dpm_funcs(struct amdgpu_device *adev);
 static void kv_dpm_set_irq_funcs(struct amdgpu_device *adev);
 static int kv_enable_nb_dpm(struct amdgpu_device *adev,
                             bool enable);
@@ -64,7 +63,7 @@ static int kv_set_thermal_temperature_range(struct amdgpu_device *adev,
                                             int min_temp, int max_temp);
 static int kv_init_fps_limits(struct amdgpu_device *adev);
 
-static void kv_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate);
+static void kv_dpm_powergate_uvd(void *handle, bool gate);
 static void kv_dpm_powergate_vce(struct amdgpu_device *adev, bool gate);
 static void kv_dpm_powergate_samu(struct amdgpu_device *adev, bool gate);
 static void kv_dpm_powergate_acp(struct amdgpu_device *adev, bool gate);
@@ -1245,8 +1244,9 @@ static void kv_update_requested_ps(struct amdgpu_device *adev,
         adev->pm.dpm.requested_ps = &pi->requested_rps;
 }
 
-static void kv_dpm_enable_bapm(struct amdgpu_device *adev, bool enable)
+static void kv_dpm_enable_bapm(void *handle, bool enable)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         int ret;
 
@@ -1672,8 +1672,9 @@ static int kv_update_acp_dpm(struct amdgpu_device *adev, bool gate)
         return kv_enable_acp_dpm(adev, !gate);
 }
 
-static void kv_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
+static void kv_dpm_powergate_uvd(void *handle, bool gate)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         int ret;
 
@@ -1868,10 +1869,11 @@ static int kv_enable_nb_dpm(struct amdgpu_device *adev,
         return ret;
 }
 
-static int kv_dpm_force_performance_level(struct amdgpu_device *adev,
+static int kv_dpm_force_performance_level(void *handle,
                                           enum amd_dpm_forced_level level)
 {
         int ret;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (level == AMD_DPM_FORCED_LEVEL_HIGH) {
                 ret = kv_force_dpm_highest(adev);
@@ -1892,8 +1894,9 @@ static int kv_dpm_force_performance_level(struct amdgpu_device *adev,
         return 0;
 }
 
-static int kv_dpm_pre_set_power_state(struct amdgpu_device *adev)
+static int kv_dpm_pre_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         struct amdgpu_ps requested_ps = *adev->pm.dpm.requested_ps;
         struct amdgpu_ps *new_ps = &requested_ps;
@@ -1907,8 +1910,9 @@ static int kv_dpm_pre_set_power_state(struct amdgpu_device *adev)
         return 0;
 }
 
-static int kv_dpm_set_power_state(struct amdgpu_device *adev)
+static int kv_dpm_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         struct amdgpu_ps *new_ps = &pi->requested_rps;
         struct amdgpu_ps *old_ps = &pi->current_rps;
@@ -1981,8 +1985,9 @@ static int kv_dpm_set_power_state(struct amdgpu_device *adev)
         return 0;
 }
 
-static void kv_dpm_post_set_power_state(struct amdgpu_device *adev)
+static void kv_dpm_post_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         struct amdgpu_ps *new_ps = &pi->requested_rps;
 
@@ -2848,9 +2853,10 @@ static int kv_dpm_init(struct amdgpu_device *adev)
 }
 
 static void
-kv_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
+kv_dpm_debugfs_print_current_performance_level(void *handle,
                                                struct seq_file *m)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         u32 current_index =
                 (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX) &
@@ -2875,11 +2881,12 @@ kv_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
 }
 
 static void
-kv_dpm_print_power_state(struct amdgpu_device *adev,
-                         struct amdgpu_ps *rps)
+kv_dpm_print_power_state(void *handle, void *request_ps)
 {
         int i;
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)request_ps;
         struct kv_ps *ps = kv_get_ps(rps);
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         amdgpu_dpm_print_class_info(rps->class, rps->class2);
         amdgpu_dpm_print_cap_info(rps->caps);
@@ -2905,13 +2912,14 @@ static void kv_dpm_fini(struct amdgpu_device *adev)
         amdgpu_free_extended_power_table(adev);
 }
 
-static void kv_dpm_display_configuration_changed(struct amdgpu_device *adev)
+static void kv_dpm_display_configuration_changed(void *handle)
 {
 
 }
 
-static u32 kv_dpm_get_sclk(struct amdgpu_device *adev, bool low)
+static u32 kv_dpm_get_sclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         struct kv_ps *requested_state = kv_get_ps(&pi->requested_rps);
 
@@ -2921,18 +2929,20 @@ static u32 kv_dpm_get_sclk(struct amdgpu_device *adev, bool low)
                 return requested_state->levels[requested_state->num_levels - 1].sclk;
 }
 
-static u32 kv_dpm_get_mclk(struct amdgpu_device *adev, bool low)
+static u32 kv_dpm_get_mclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
 
         return pi->sys_info.bootup_uma_clk;
 }
 
 /* get temperature in millidegrees */
-static int kv_dpm_get_temp(struct amdgpu_device *adev)
+static int kv_dpm_get_temp(void *handle)
 {
         u32 temp;
         int actual_temp = 0;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         temp = RREG32_SMC(0xC0300E0C);
 
@@ -2950,7 +2960,6 @@ static int kv_dpm_early_init(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        kv_dpm_set_dpm_funcs(adev);
         kv_dpm_set_irq_funcs(adev);
 
         return 0;
@@ -3222,14 +3231,17 @@ static inline bool kv_are_power_levels_equal(const struct kv_pl *kv_cpl1,
                   (kv_cpl1->force_nbp_state == kv_cpl2->force_nbp_state));
 }
 
-static int kv_check_state_equal(struct amdgpu_device *adev,
-                                struct amdgpu_ps *cps,
-                                struct amdgpu_ps *rps,
+static int kv_check_state_equal(void *handle,
+                                void *current_ps,
+                                void *request_ps,
                                 bool *equal)
 {
         struct kv_ps *kv_cps;
         struct kv_ps *kv_rps;
         int i;
+        struct amdgpu_ps *cps = (struct amdgpu_ps *)current_ps;
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)request_ps;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (adev == NULL || cps == NULL || rps == NULL || equal == NULL)
                 return -EINVAL;
@@ -3262,9 +3274,10 @@ static int kv_check_state_equal(struct amdgpu_device *adev,
         return 0;
 }
 
-static int kv_dpm_read_sensor(struct amdgpu_device *adev, int idx,
+static int kv_dpm_read_sensor(void *handle, int idx,
                               void *value, int *size)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         uint32_t sclk;
         u32 pl_index =
@@ -3312,7 +3325,7 @@ const struct amd_ip_funcs kv_dpm_ip_funcs = {
         .set_powergating_state = kv_dpm_set_powergating_state,
 };
 
-static const struct amdgpu_dpm_funcs kv_dpm_funcs = {
+const struct amd_pm_funcs kv_dpm_funcs = {
         .get_temperature = &kv_dpm_get_temp,
         .pre_set_power_state = &kv_dpm_pre_set_power_state,
         .set_power_state = &kv_dpm_set_power_state,
@@ -3330,12 +3343,6 @@ static const struct amdgpu_dpm_funcs kv_dpm_funcs = {
         .read_sensor = &kv_dpm_read_sensor,
 };
 
-static void kv_dpm_set_dpm_funcs(struct amdgpu_device *adev)
-{
-        if (adev->pm.funcs == NULL)
-                adev->pm.funcs = &kv_dpm_funcs;
-}
-
 static const struct amdgpu_irq_src_funcs kv_dpm_irq_funcs = {
         .set = kv_dpm_set_interrupt_state,
         .process = kv_dpm_process_interrupt,
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
index ad8def3cc343..cc21c4bdec27 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
@@ -158,8 +158,8 @@ static void mmhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL2, tmp);
 
         tmp = mmVM_L2_CNTL3_DEFAULT;
-        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12);
-        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
+        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
+        tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, tmp);
 
         tmp = mmVM_L2_CNTL4_DEFAULT;
@@ -273,7 +273,7 @@ static const struct pctl_data pctl0_data[] = {
         {0x135, 0x12a810},
         {0x149, 0x7a82c}
 };
-#define PCTL0_DATA_LEN (sizeof(pctl0_data)/sizeof(pctl0_data[0]))
+#define PCTL0_DATA_LEN (ARRAY_SIZE(pctl0_data))
 
 #define PCTL0_RENG_EXEC_END_PTR 0x151
 #define PCTL0_STCTRL_REG_SAVE_RANGE0_BASE  0xa640
@@ -309,7 +309,7 @@ static const struct pctl_data pctl1_data[] = {
         {0x1f0, 0x5000a7f6},
         {0x1f1, 0x5000a7e4}
 };
-#define PCTL1_DATA_LEN (sizeof(pctl1_data)/sizeof(pctl1_data[0]))
+#define PCTL1_DATA_LEN (ARRAY_SIZE(pctl1_data))
 
 #define PCTL1_RENG_EXEC_END_PTR 0x1f1
 #define PCTL1_STCTRL_REG_SAVE_RANGE0_BASE  0xa000
@@ -561,6 +561,13 @@ void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, bool value)
                         WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
                         EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+        if (!value) {
+                tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+                                CRASH_ON_NO_RETRY_FAULT, 1);
+                tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+                                CRASH_ON_RETRY_FAULT, 1);
+    }
+
         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
index 1e91b9a1c591..67e78576a9eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
@@ -24,7 +24,7 @@
 #ifndef __MXGPU_AI_H__
 #define __MXGPU_AI_H__
 
-#define AI_MAILBOX_TIMEDOUT     5000
+#define AI_MAILBOX_TIMEDOUT     12000
 
 enum idh_request {
         IDH_REQ_GPU_INIT_ACCESS = 1,
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
index c791d73d2d54..f13dc6cc158f 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
@@ -23,7 +23,7 @@
 #ifndef __MXGPU_VI_H__
 #define __MXGPU_VI_H__
 
-#define VI_MAILBOX_TIMEDOUT     5000
+#define VI_MAILBOX_TIMEDOUT     12000
 #define VI_MAILBOX_RESET_TIME   12
 
 /* VI mailbox messages request */
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
index f7cf994b1da2..dea7c909ca5f 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
@@ -35,6 +35,8 @@
 #include "raven1/GC/gc_9_1_offset.h"
 #include "raven1/SDMA0/sdma0_4_1_offset.h"
 
+MODULE_FIRMWARE("amdgpu/raven_asd.bin");
+
 static int
 psp_v10_0_get_fw_type(struct amdgpu_firmware_info *ucode, enum psp_gfx_fw_type *type)
 {
@@ -136,15 +138,13 @@ int psp_v10_0_prep_cmd_buf(struct amdgpu_firmware_info *ucode, struct psp_gfx_cm
 {
         int ret;
         uint64_t fw_mem_mc_addr = ucode->mc_addr;
-        struct  common_firmware_header *header;
 
         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
-        header = (struct common_firmware_header *)ucode->fw;
 
         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
-        cmd->cmd.cmd_load_ip_fw.fw_size = le32_to_cpu(header->ucode_size_bytes);
+        cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
 
         ret = psp_v10_0_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
         if (ret)
@@ -209,7 +209,7 @@ int psp_v10_0_ring_create(struct psp_context *psp, enum psp_ring_type ring_type)
         return ret;
 }
 
-int psp_v10_0_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
+int psp_v10_0_ring_stop(struct psp_context *psp, enum psp_ring_type ring_type)
 {
         int ret = 0;
         struct psp_ring *ring;
@@ -229,6 +229,19 @@ int psp_v10_0_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type
         ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
                            0x80000000, 0x80000000, false);
 
+        return ret;
+}
+
+int psp_v10_0_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
+{
+        int ret = 0;
+        struct psp_ring *ring = &psp->km_ring;
+        struct amdgpu_device *adev = psp->adev;
+
+        ret = psp_v10_0_ring_stop(psp, ring_type);
+        if (ret)
+                DRM_ERROR("Fail to stop psp ring\n");
+
         amdgpu_bo_free_kernel(&adev->firmware.rbuf,
                               &ring->ring_mem_mc_addr,
                               (void **)&ring->ring_mem);
@@ -245,15 +258,20 @@ int psp_v10_0_cmd_submit(struct psp_context *psp,
         struct psp_gfx_rb_frame * write_frame = psp->km_ring.ring_mem;
         struct psp_ring *ring = &psp->km_ring;
         struct amdgpu_device *adev = psp->adev;
+        uint32_t ring_size_dw = ring->ring_size / 4;
+        uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
 
         /* KM (GPCOM) prepare write pointer */
         psp_write_ptr_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_67);
 
         /* Update KM RB frame pointer to new frame */
-        if ((psp_write_ptr_reg % ring->ring_size) == 0)
+        if ((psp_write_ptr_reg % ring_size_dw) == 0)
                 write_frame = ring->ring_mem;
         else
-                write_frame = ring->ring_mem + (psp_write_ptr_reg / (sizeof(struct psp_gfx_rb_frame) / 4));
+                write_frame = ring->ring_mem + (psp_write_ptr_reg / rb_frame_size_dw);
+
+        /* Initialize KM RB frame */
+        memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
 
         /* Update KM RB frame */
         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
@@ -263,8 +281,7 @@ int psp_v10_0_cmd_submit(struct psp_context *psp,
         write_frame->fence_value = index;
 
         /* Update the write Pointer in DWORDs */
-        psp_write_ptr_reg += sizeof(struct psp_gfx_rb_frame) / 4;
-        psp_write_ptr_reg = (psp_write_ptr_reg >= ring->ring_size) ? 0 : psp_write_ptr_reg;
+        psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
         WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_67, psp_write_ptr_reg);
 
         return 0;
@@ -390,3 +407,10 @@ bool psp_v10_0_compare_sram_data(struct psp_context *psp,
 
         return true;
 }
+
+
+int psp_v10_0_mode1_reset(struct psp_context *psp)
+{
+        DRM_INFO("psp mode 1 reset not supported now! \n");
+        return -EINVAL;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h
index e76cde2f01f9..451e8308303f 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h
@@ -34,6 +34,8 @@ extern int psp_v10_0_ring_init(struct psp_context *psp,
                               enum psp_ring_type ring_type);
 extern int psp_v10_0_ring_create(struct psp_context *psp,
                                  enum psp_ring_type ring_type);
+extern int psp_v10_0_ring_stop(struct psp_context *psp,
+                                  enum psp_ring_type ring_type);
 extern int psp_v10_0_ring_destroy(struct psp_context *psp,
                                   enum psp_ring_type ring_type);
 extern int psp_v10_0_cmd_submit(struct psp_context *psp,
@@ -43,4 +45,6 @@ extern int psp_v10_0_cmd_submit(struct psp_context *psp,
 extern bool psp_v10_0_compare_sram_data(struct psp_context *psp,
                                        struct amdgpu_firmware_info *ucode,
                                        enum AMDGPU_UCODE_ID ucode_type);
+
+extern int psp_v10_0_mode1_reset(struct psp_context *psp);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index 2a535a4b8d5b..cee5c396b277 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -319,7 +319,7 @@ int psp_v3_1_ring_create(struct psp_context *psp, enum psp_ring_type ring_type)
         return ret;
 }
 
-int psp_v3_1_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
+int psp_v3_1_ring_stop(struct psp_context *psp, enum psp_ring_type ring_type)
 {
         int ret = 0;
         struct psp_ring *ring;
@@ -339,6 +339,19 @@ int psp_v3_1_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
         ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
                            0x80000000, 0x80000000, false);
 
+        return ret;
+}
+
+int psp_v3_1_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
+{
+        int ret = 0;
+        struct psp_ring *ring = &psp->km_ring;
+        struct amdgpu_device *adev = psp->adev;
+
+        ret = psp_v3_1_ring_stop(psp, ring_type);
+        if (ret)
+                DRM_ERROR("Fail to stop psp ring\n");
+
         amdgpu_bo_free_kernel(&adev->firmware.rbuf,
                               &ring->ring_mem_mc_addr,
                               (void **)&ring->ring_mem);
@@ -517,3 +530,37 @@ bool psp_v3_1_smu_reload_quirk(struct psp_context *psp)
         reg = RREG32_SOC15(NBIO, 0, mmPCIE_DATA2);
         return (reg & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) ? true : false;
 }
+
+int psp_v3_1_mode1_reset(struct psp_context *psp)
+{
+        int ret;
+        uint32_t offset;
+        struct amdgpu_device *adev = psp->adev;
+
+        offset = SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64);
+
+        ret = psp_wait_for(psp, offset, 0x80000000, 0x8000FFFF, false);
+
+        if (ret) {
+                DRM_INFO("psp is not working correctly before mode1 reset!\n");
+                return -EINVAL;
+        }
+
+        /*send the mode 1 reset command*/
+        WREG32(offset, 0x70000);
+
+        mdelay(1000);
+
+        offset = SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_33);
+
+        ret = psp_wait_for(psp, offset, 0x80000000, 0x80000000, false);
+
+        if (ret) {
+                DRM_INFO("psp mode 1 reset failed!\n");
+                return -EINVAL;
+        }
+
+        DRM_INFO("psp mode1 reset succeed \n");
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h
index 9dcd0b25c4c6..b05dbada7751 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h
@@ -41,6 +41,8 @@ extern int psp_v3_1_ring_init(struct psp_context *psp,
                               enum psp_ring_type ring_type);
 extern int psp_v3_1_ring_create(struct psp_context *psp,
                                 enum psp_ring_type ring_type);
+extern int psp_v3_1_ring_stop(struct psp_context *psp,
+                                enum psp_ring_type ring_type);
 extern int psp_v3_1_ring_destroy(struct psp_context *psp,
                                 enum psp_ring_type ring_type);
 extern int psp_v3_1_cmd_submit(struct psp_context *psp,
@@ -51,4 +53,5 @@ extern bool psp_v3_1_compare_sram_data(struct psp_context *psp,
                                        struct amdgpu_firmware_info *ucode,
                                        enum AMDGPU_UCODE_ID ucode_type);
 extern bool psp_v3_1_smu_reload_quirk(struct psp_context *psp);
+extern int psp_v3_1_mode1_reset(struct psp_context *psp);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index f2d0710258cb..acdee3a4602c 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -1324,8 +1324,13 @@ static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev)
 }
 
 static const struct amdgpu_vm_pte_funcs sdma_v2_4_vm_pte_funcs = {
+        .copy_pte_num_dw = 7,
         .copy_pte = sdma_v2_4_vm_copy_pte,
+
         .write_pte = sdma_v2_4_vm_write_pte,
+
+        .set_max_nums_pte_pde = 0x1fffff >> 3,
+        .set_pte_pde_num_dw = 10,
         .set_pte_pde = sdma_v2_4_vm_set_pte_pde,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index b1de44f22824..72f31cc7df00 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -379,8 +379,10 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring)
         struct amdgpu_device *adev = ring->adev;
 
         if (ring->use_doorbell) {
+                u32 *wb = (u32 *)&adev->wb.wb[ring->wptr_offs];
+
                 /* XXX check if swapping is necessary on BE */
-                adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr) << 2;
+                WRITE_ONCE(*wb, (lower_32_bits(ring->wptr) << 2));
                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr) << 2);
         } else {
                 int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
@@ -641,10 +643,11 @@ static void sdma_v3_0_enable(struct amdgpu_device *adev, bool enable)
 static int sdma_v3_0_gfx_resume(struct amdgpu_device *adev)
 {
         struct amdgpu_ring *ring;
-        u32 rb_cntl, ib_cntl;
+        u32 rb_cntl, ib_cntl, wptr_poll_cntl;
         u32 rb_bufsz;
         u32 wb_offset;
         u32 doorbell;
+        u64 wptr_gpu_addr;
         int i, j, r;
 
         for (i = 0; i < adev->sdma.num_instances; i++) {
@@ -707,6 +710,20 @@ static int sdma_v3_0_gfx_resume(struct amdgpu_device *adev)
                 }
                 WREG32(mmSDMA0_GFX_DOORBELL + sdma_offsets[i], doorbell);
 
+                /* setup the wptr shadow polling */
+                wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+
+                WREG32(mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO + sdma_offsets[i],
+                       lower_32_bits(wptr_gpu_addr));
+                WREG32(mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI + sdma_offsets[i],
+                       upper_32_bits(wptr_gpu_addr));
+                wptr_poll_cntl = RREG32(mmSDMA0_GFX_RB_WPTR_POLL_CNTL + sdma_offsets[i]);
+                if (amdgpu_sriov_vf(adev))
+                        wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 1);
+                else
+                        wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 0);
+                WREG32(mmSDMA0_GFX_RB_WPTR_POLL_CNTL + sdma_offsets[i], wptr_poll_cntl);
+
                 /* enable DMA RB */
                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1);
                 WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);
@@ -1713,11 +1730,11 @@ static void sdma_v3_0_emit_fill_buffer(struct amdgpu_ib *ib,
 }
 
 static const struct amdgpu_buffer_funcs sdma_v3_0_buffer_funcs = {
-        .copy_max_bytes = 0x1fffff,
+        .copy_max_bytes = 0x3fffe0, /* not 0x3fffff due to HW limitation */
         .copy_num_dw = 7,
         .emit_copy_buffer = sdma_v3_0_emit_copy_buffer,
 
-        .fill_max_bytes = 0x1fffff,
+        .fill_max_bytes = 0x3fffe0, /* not 0x3fffff due to HW limitation */
         .fill_num_dw = 5,
         .emit_fill_buffer = sdma_v3_0_emit_fill_buffer,
 };
@@ -1731,8 +1748,14 @@ static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev)
 }
 
 static const struct amdgpu_vm_pte_funcs sdma_v3_0_vm_pte_funcs = {
+        .copy_pte_num_dw = 7,
         .copy_pte = sdma_v3_0_vm_copy_pte,
+
         .write_pte = sdma_v3_0_vm_write_pte,
+
+        /* not 0x3fffff due to HW limitation */
+        .set_max_nums_pte_pde = 0x3fffe0 >> 3,
+        .set_pte_pde_num_dw = 10,
         .set_pte_pde = sdma_v3_0_vm_set_pte_pde,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index fd7c72aaafa6..c26d205ff3bf 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -54,7 +54,7 @@ static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev);
 
 static const u32 golden_settings_sdma_4[] = {
-        SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831f07,
+        SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831d07,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CLK_CTRL), 0xff000ff0, 0x3f000100,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_IB_CNTL), 0x800f0100, 0x00000100,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL), 0xfffffff7, 0x00403000,
@@ -89,7 +89,7 @@ static const u32 golden_settings_sdma_vg10[] = {
 
 static const u32 golden_settings_sdma_4_1[] =
 {
-        SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831f07,
+        SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831d07,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CLK_CTRL), 0xffffffff, 0x3f000100,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_IB_CNTL), 0x800f0111, 0x00000100,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL), 0xfffffff7, 0x00403000,
@@ -398,7 +398,7 @@ static void sdma_v4_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
                           SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
-        amdgpu_ring_write(ring, SOC15_REG_OFFSET(HDP, 0, mmHDP_DEBUG0));
+        amdgpu_ring_write(ring, SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE));
         amdgpu_ring_write(ring, 1);
 }
 
@@ -1264,6 +1264,11 @@ static int sdma_v4_0_sw_fini(void *handle)
         for (i = 0; i < adev->sdma.num_instances; i++)
                 amdgpu_ring_fini(&adev->sdma.instance[i].ring);
 
+        for (i = 0; i < adev->sdma.num_instances; i++) {
+                release_firmware(adev->sdma.instance[i].fw);
+                adev->sdma.instance[i].fw = NULL;
+        }
+
         return 0;
 }
 
@@ -1714,8 +1719,13 @@ static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev)
 }
 
 static const struct amdgpu_vm_pte_funcs sdma_v4_0_vm_pte_funcs = {
+        .copy_pte_num_dw = 7,
         .copy_pte = sdma_v4_0_vm_copy_pte,
+
         .write_pte = sdma_v4_0_vm_write_pte,
+
+        .set_max_nums_pte_pde = 0x400000 >> 3,
+        .set_pte_pde_num_dw = 10,
         .set_pte_pde = sdma_v4_0_vm_set_pte_pde,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 112969f3301a..3fa2fbf8c9a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -887,8 +887,13 @@ static void si_dma_set_buffer_funcs(struct amdgpu_device *adev)
 }
 
 static const struct amdgpu_vm_pte_funcs si_dma_vm_pte_funcs = {
+        .copy_pte_num_dw = 5,
         .copy_pte = si_dma_vm_copy_pte,
+
         .write_pte = si_dma_vm_write_pte,
+
+        .set_max_nums_pte_pde = 0xffff8 >> 3,
+        .set_pte_pde_num_dw = 9,
         .set_pte_pde = si_dma_vm_set_pte_pde,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index d63873f3f574..9b8db6046271 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -1847,7 +1847,6 @@ static int si_calculate_sclk_params(struct amdgpu_device *adev,
 
 static void si_thermal_start_smc_fan_control(struct amdgpu_device *adev);
 static void si_fan_ctrl_set_default_mode(struct amdgpu_device *adev);
-static void si_dpm_set_dpm_funcs(struct amdgpu_device *adev);
 static void si_dpm_set_irq_funcs(struct amdgpu_device *adev);
 
 static struct si_power_info *si_get_pi(struct amdgpu_device *adev)
@@ -3060,9 +3059,9 @@ static int si_get_vce_clock_voltage(struct amdgpu_device *adev,
         return ret;
 }
 
-static bool si_dpm_vblank_too_short(struct amdgpu_device *adev)
+static bool si_dpm_vblank_too_short(void *handle)
 {
-
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 vblank_time = amdgpu_dpm_get_vblank_time(adev);
         /* we never hit the non-gddr5 limit so disable it */
         u32 switch_limit = adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5 ? 450 : 0;
@@ -3871,9 +3870,10 @@ static int si_restrict_performance_levels_before_switch(struct amdgpu_device *ad
                 0 : -EINVAL;
 }
 
-static int si_dpm_force_performance_level(struct amdgpu_device *adev,
+static int si_dpm_force_performance_level(void *handle,
                                    enum amd_dpm_forced_level level)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct amdgpu_ps *rps = adev->pm.dpm.current_ps;
         struct  si_ps *ps = si_get_ps(rps);
         u32 levels = ps->performance_level_count;
@@ -6575,11 +6575,12 @@ static int si_fan_ctrl_stop_smc_fan_control(struct amdgpu_device *adev)
         }
 }
 
-static int si_dpm_get_fan_speed_percent(struct amdgpu_device *adev,
+static int si_dpm_get_fan_speed_percent(void *handle,
                                       u32 *speed)
 {
         u32 duty, duty100;
         u64 tmp64;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (adev->pm.no_fan)
                 return -ENOENT;
@@ -6600,9 +6601,10 @@ static int si_dpm_get_fan_speed_percent(struct amdgpu_device *adev,
         return 0;
 }
 
-static int si_dpm_set_fan_speed_percent(struct amdgpu_device *adev,
+static int si_dpm_set_fan_speed_percent(void *handle,
                                       u32 speed)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct si_power_info *si_pi = si_get_pi(adev);
         u32 tmp;
         u32 duty, duty100;
@@ -6633,8 +6635,10 @@ static int si_dpm_set_fan_speed_percent(struct amdgpu_device *adev,
         return 0;
 }
 
-static void si_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode)
+static void si_dpm_set_fan_control_mode(void *handle, u32 mode)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
         if (mode) {
                 /* stop auto-manage */
                 if (adev->pm.dpm.fan.ucode_fan_control)
@@ -6649,8 +6653,9 @@ static void si_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode)
         }
 }
 
-static u32 si_dpm_get_fan_control_mode(struct amdgpu_device *adev)
+static u32 si_dpm_get_fan_control_mode(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct si_power_info *si_pi = si_get_pi(adev);
         u32 tmp;
 
@@ -6946,8 +6951,9 @@ static void si_dpm_disable(struct amdgpu_device *adev)
         ni_update_current_ps(adev, boot_ps);
 }
 
-static int si_dpm_pre_set_power_state(struct amdgpu_device *adev)
+static int si_dpm_pre_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct amdgpu_ps requested_ps = *adev->pm.dpm.requested_ps;
         struct amdgpu_ps *new_ps = &requested_ps;
@@ -6984,8 +6990,9 @@ static int si_power_control_set_level(struct amdgpu_device *adev)
         return 0;
 }
 
-static int si_dpm_set_power_state(struct amdgpu_device *adev)
+static int si_dpm_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct amdgpu_ps *new_ps = &eg_pi->requested_rps;
         struct amdgpu_ps *old_ps = &eg_pi->current_rps;
@@ -7086,8 +7093,9 @@ static int si_dpm_set_power_state(struct amdgpu_device *adev)
         return 0;
 }
 
-static void si_dpm_post_set_power_state(struct amdgpu_device *adev)
+static void si_dpm_post_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct amdgpu_ps *new_ps = &eg_pi->requested_rps;
 
@@ -7103,8 +7111,10 @@ void si_dpm_reset_asic(struct amdgpu_device *adev)
 }
 #endif
 
-static void si_dpm_display_configuration_changed(struct amdgpu_device *adev)
+static void si_dpm_display_configuration_changed(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
         si_program_display_gap(adev);
 }
 
@@ -7486,9 +7496,10 @@ static void si_dpm_fini(struct amdgpu_device *adev)
         amdgpu_free_extended_power_table(adev);
 }
 
-static void si_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
+static void si_dpm_debugfs_print_current_performance_level(void *handle,
                                                     struct seq_file *m)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct amdgpu_ps *rps = &eg_pi->current_rps;
         struct  si_ps *ps = si_get_ps(rps);
@@ -7860,10 +7871,11 @@ static int si_dpm_set_powergating_state(void *handle,
 }
 
 /* get temperature in millidegrees */
-static int si_dpm_get_temp(struct amdgpu_device *adev)
+static int si_dpm_get_temp(void *handle)
 {
         u32 temp;
         int actual_temp = 0;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
                 CTF_TEMP_SHIFT;
@@ -7878,8 +7890,9 @@ static int si_dpm_get_temp(struct amdgpu_device *adev)
         return actual_temp;
 }
 
-static u32 si_dpm_get_sclk(struct amdgpu_device *adev, bool low)
+static u32 si_dpm_get_sclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct  si_ps *requested_state = si_get_ps(&eg_pi->requested_rps);
 
@@ -7889,8 +7902,9 @@ static u32 si_dpm_get_sclk(struct amdgpu_device *adev, bool low)
                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
 }
 
-static u32 si_dpm_get_mclk(struct amdgpu_device *adev, bool low)
+static u32 si_dpm_get_mclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct  si_ps *requested_state = si_get_ps(&eg_pi->requested_rps);
 
@@ -7900,9 +7914,11 @@ static u32 si_dpm_get_mclk(struct amdgpu_device *adev, bool low)
                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
 }
 
-static void si_dpm_print_power_state(struct amdgpu_device *adev,
-                                     struct amdgpu_ps *rps)
+static void si_dpm_print_power_state(void *handle,
+                                     void *current_ps)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)current_ps;
         struct  si_ps *ps = si_get_ps(rps);
         struct rv7xx_pl *pl;
         int i;
@@ -7927,7 +7943,6 @@ static int si_dpm_early_init(void *handle)
 
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        si_dpm_set_dpm_funcs(adev);
         si_dpm_set_irq_funcs(adev);
         return 0;
 }
@@ -7942,20 +7957,23 @@ static inline bool si_are_power_levels_equal(const struct rv7xx_pl  *si_cpl1,
                   (si_cpl1->vddci == si_cpl2->vddci));
 }
 
-static int si_check_state_equal(struct amdgpu_device *adev,
-                                struct amdgpu_ps *cps,
-                                struct amdgpu_ps *rps,
+static int si_check_state_equal(void *handle,
+                                void *current_ps,
+                                void *request_ps,
                                 bool *equal)
 {
         struct si_ps *si_cps;
         struct si_ps *si_rps;
         int i;
+        struct amdgpu_ps *cps = (struct amdgpu_ps *)current_ps;
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)request_ps;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (adev == NULL || cps == NULL || rps == NULL || equal == NULL)
                 return -EINVAL;
 
-        si_cps = si_get_ps(cps);
-        si_rps = si_get_ps(rps);
+        si_cps = si_get_ps((struct amdgpu_ps *)cps);
+        si_rps = si_get_ps((struct amdgpu_ps *)rps);
 
         if (si_cps == NULL) {
                 printk("si_cps is NULL\n");
@@ -7983,9 +8001,10 @@ static int si_check_state_equal(struct amdgpu_device *adev,
         return 0;
 }
 
-static int si_dpm_read_sensor(struct amdgpu_device *adev, int idx,
+static int si_dpm_read_sensor(void *handle, int idx,
                               void *value, int *size)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct amdgpu_ps *rps = &eg_pi->current_rps;
         struct  si_ps *ps = si_get_ps(rps);
@@ -8041,7 +8060,7 @@ const struct amd_ip_funcs si_dpm_ip_funcs = {
         .set_powergating_state = si_dpm_set_powergating_state,
 };
 
-static const struct amdgpu_dpm_funcs si_dpm_funcs = {
+const struct amd_pm_funcs si_dpm_funcs = {
         .get_temperature = &si_dpm_get_temp,
         .pre_set_power_state = &si_dpm_pre_set_power_state,
         .set_power_state = &si_dpm_set_power_state,
@@ -8062,12 +8081,6 @@ static const struct amdgpu_dpm_funcs si_dpm_funcs = {
         .read_sensor = &si_dpm_read_sensor,
 };
 
-static void si_dpm_set_dpm_funcs(struct amdgpu_device *adev)
-{
-        if (adev->pm.funcs == NULL)
-                adev->pm.funcs = &si_dpm_funcs;
-}
-
 static const struct amdgpu_irq_src_funcs si_dpm_irq_funcs = {
         .set = si_dpm_set_interrupt_state,
         .process = si_dpm_process_interrupt,
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.h b/drivers/gpu/drm/amd/amdgpu/si_dpm.h
index 51ce21c5f4fb..9fe343de3477 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.h
@@ -246,6 +246,7 @@ enum si_display_gap
 };
 
 extern const struct amd_ip_funcs si_dpm_ip_funcs;
+extern const struct amd_pm_funcs si_dpm_funcs;
 
 struct ni_leakage_coeffients
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c
index ce25e03a077d..d2c6b80309c8 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c
@@ -118,6 +118,19 @@ static u32 si_ih_get_wptr(struct amdgpu_device *adev)
         return (wptr & adev->irq.ih.ptr_mask);
 }
 
+/**
+ * si_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool si_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        /* Process all interrupts */
+        return true;
+}
+
 static void si_ih_decode_iv(struct amdgpu_device *adev,
                              struct amdgpu_iv_entry *entry)
 {
@@ -288,6 +301,7 @@ static const struct amd_ip_funcs si_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs si_ih_funcs = {
         .get_wptr = si_ih_get_wptr,
+        .prescreen_iv = si_ih_prescreen_iv,
         .decode_iv = si_ih_decode_iv,
         .set_rptr = si_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index f2c3a49f73a0..245a18aeb389 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -407,18 +407,27 @@ static int soc15_read_register(struct amdgpu_device *adev, u32 se_num,
         return -EINVAL;
 }
 
-static void soc15_gpu_pci_config_reset(struct amdgpu_device *adev)
+static int soc15_asic_reset(struct amdgpu_device *adev)
 {
         u32 i;
 
-        dev_info(adev->dev, "GPU pci config reset\n");
+        amdgpu_atombios_scratch_regs_engine_hung(adev, true);
+
+        dev_info(adev->dev, "GPU reset\n");
 
         /* disable BM */
         pci_clear_master(adev->pdev);
-        /* reset */
-        amdgpu_pci_config_reset(adev);
 
-        udelay(100);
+        pci_save_state(adev->pdev);
+
+        for (i = 0; i < AMDGPU_MAX_IP_NUM; i++) {
+                if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP){
+                        adev->ip_blocks[i].version->funcs->soft_reset((void *)adev);
+                        break;
+                }
+        }
+
+        pci_restore_state(adev->pdev);
 
         /* wait for asic to come out of reset */
         for (i = 0; i < adev->usec_timeout; i++) {
@@ -430,14 +439,6 @@ static void soc15_gpu_pci_config_reset(struct amdgpu_device *adev)
                 udelay(1);
         }
 
-}
-
-static int soc15_asic_reset(struct amdgpu_device *adev)
-{
-        amdgpu_atombios_scratch_regs_engine_hung(adev, true);
-
-        soc15_gpu_pci_config_reset(adev);
-
         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
 
         return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15d.h b/drivers/gpu/drm/amd/amdgpu/soc15d.h
index e79befd80eed..7f408f85fdb6 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15d.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15d.h
@@ -250,6 +250,7 @@
 #define PACKET3_SET_UCONFIG_REG                         0x79
 #define         PACKET3_SET_UCONFIG_REG_START                   0x0000c000
 #define         PACKET3_SET_UCONFIG_REG_END                     0x0000c400
+#define         PACKET3_SET_UCONFIG_REG_INDEX_TYPE              (2 << 28)
 #define PACKET3_SCRATCH_RAM_WRITE                       0x7D
 #define PACKET3_SCRATCH_RAM_READ                        0x7E
 #define PACKET3_LOAD_CONST_RAM                          0x80
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index 923df2c0e535..5ed00692618e 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -219,6 +219,19 @@ static u32 tonga_ih_get_wptr(struct amdgpu_device *adev)
 }
 
 /**
+ * tonga_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool tonga_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        /* Process all interrupts */
+        return true;
+}
+
+/**
  * tonga_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -478,6 +491,7 @@ static const struct amd_ip_funcs tonga_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs tonga_ih_funcs = {
         .get_wptr = tonga_ih_get_wptr,
+        .prescreen_iv = tonga_ih_prescreen_iv,
         .decode_iv = tonga_ih_decode_iv,
         .set_rptr = tonga_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index 23a85750edd6..b8ed8faf2003 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -1161,7 +1161,7 @@ static void uvd_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
  */
 static void uvd_v7_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
-        amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_DEBUG0), 0));
+        amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 0));
         amdgpu_ring_write(ring, 1);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index 11134d5f7443..75745544600a 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -1011,10 +1011,6 @@ static int vce_v4_0_process_interrupt(struct amdgpu_device *adev,
 {
         DRM_DEBUG("IH: VCE\n");
 
-        WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_STATUS),
-                        VCE_SYS_INT_STATUS__VCE_SYS_INT_TRAP_INTERRUPT_INT_MASK,
-                        ~VCE_SYS_INT_STATUS__VCE_SYS_INT_TRAP_INTERRUPT_INT_MASK);
-
         switch (entry->src_data[0]) {
         case 0:
         case 1:
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
index 21e7b88401e1..1eb4d79d6e30 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
@@ -812,7 +812,7 @@ static void vcn_v1_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64
  */
 static void vcn_v1_0_dec_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
-        amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_DEBUG0), 0));
+        amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 0));
         amdgpu_ring_write(ring, 1);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
index 56150e8d1ed2..a3b30d84dbb3 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
@@ -219,14 +219,92 @@ static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
                         wptr, adev->irq.ih.rptr, tmp);
                 adev->irq.ih.rptr = tmp;
 
-                tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
+                tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
                 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
-                WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
+                WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
         }
         return (wptr & adev->irq.ih.ptr_mask);
 }
 
 /**
+ * vega10_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        u32 ring_index = adev->irq.ih.rptr >> 2;
+        u32 dw0, dw3, dw4, dw5;
+        u16 pasid;
+        u64 addr, key;
+        struct amdgpu_vm *vm;
+        int r;
+
+        dw0 = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
+        dw3 = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
+        dw4 = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
+        dw5 = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
+
+        /* Filter retry page faults, let only the first one pass. If
+         * there are too many outstanding faults, ignore them until
+         * some faults get cleared.
+         */
+        switch (dw0 & 0xff) {
+        case AMDGPU_IH_CLIENTID_VMC:
+        case AMDGPU_IH_CLIENTID_UTCL2:
+                break;
+        default:
+                /* Not a VM fault */
+                return true;
+        }
+
+        /* Not a retry fault */
+        if (!(dw5 & 0x80))
+                return true;
+
+        pasid = dw3 & 0xffff;
+        /* No PASID, can't identify faulting process */
+        if (!pasid)
+                return true;
+
+        addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12);
+        key = AMDGPU_VM_FAULT(pasid, addr);
+        r = amdgpu_ih_add_fault(adev, key);
+
+        /* Hash table is full or the fault is already being processed,
+         * ignore further page faults
+         */
+        if (r != 0)
+                goto ignore_iv;
+
+        /* Track retry faults in per-VM fault FIFO. */
+        spin_lock(&adev->vm_manager.pasid_lock);
+        vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
+        spin_unlock(&adev->vm_manager.pasid_lock);
+        if (WARN_ON_ONCE(!vm)) {
+                /* VM not found, process it normally */
+                amdgpu_ih_clear_fault(adev, key);
+                return true;
+        }
+        /* No locking required with single writer and single reader */
+        r = kfifo_put(&vm->faults, key);
+        if (!r) {
+                /* FIFO is full. Ignore it until there is space */
+                amdgpu_ih_clear_fault(adev, key);
+                goto ignore_iv;
+        }
+
+        /* It's the first fault for this address, process it normally */
+        return true;
+
+ignore_iv:
+        adev->irq.ih.rptr += 32;
+        return false;
+}
+
+/**
  * vega10_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -310,6 +388,14 @@ static int vega10_ih_sw_init(void *handle)
         adev->irq.ih.use_doorbell = true;
         adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;
 
+        adev->irq.ih.faults = kmalloc(sizeof(*adev->irq.ih.faults), GFP_KERNEL);
+        if (!adev->irq.ih.faults)
+                return -ENOMEM;
+        INIT_CHASH_TABLE(adev->irq.ih.faults->hash,
+                         AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
+        spin_lock_init(&adev->irq.ih.faults->lock);
+        adev->irq.ih.faults->count = 0;
+
         r = amdgpu_irq_init(adev);
 
         return r;
@@ -322,6 +408,9 @@ static int vega10_ih_sw_fini(void *handle)
         amdgpu_irq_fini(adev);
         amdgpu_ih_ring_fini(adev);
 
+        kfree(adev->irq.ih.faults);
+        adev->irq.ih.faults = NULL;
+
         return 0;
 }
 
@@ -410,6 +499,7 @@ const struct amd_ip_funcs vega10_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs vega10_ih_funcs = {
         .get_wptr = vega10_ih_get_wptr,
+        .prescreen_iv = vega10_ih_prescreen_iv,
         .decode_iv = vega10_ih_decode_iv,
         .set_rptr = vega10_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 6cac291c96da..9ff69b90df36 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -1028,8 +1028,7 @@ static int vi_common_early_init(void *handle)
                 /* rev0 hardware requires workarounds to support PG */
                 adev->pg_flags = 0;
                 if (adev->rev_id != 0x00 || CZ_REV_BRISTOL(adev->pdev->revision)) {
-                        adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
-                                AMD_PG_SUPPORT_GFX_SMG |
+                        adev->pg_flags |= AMD_PG_SUPPORT_GFX_SMG |
                                 AMD_PG_SUPPORT_GFX_PIPELINE |
                                 AMD_PG_SUPPORT_CP |
                                 AMD_PG_SUPPORT_UVD |