108 files changed, 5447 insertions, 2306 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig
index 26682454a446..e8af1f5e8a79 100644
--- a/drivers/gpu/drm/amd/amdgpu/Kconfig
+++ b/drivers/gpu/drm/amd/amdgpu/Kconfig
@@ -41,3 +41,4 @@ config DRM_AMDGPU_GART_DEBUGFS
           pages. Uses more memory for housekeeping, enable only for debugging.
 
 source "drivers/gpu/drm/amd/acp/Kconfig"
+source "drivers/gpu/drm/amd/display/Kconfig"
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 567b0377e1e2..78d609123420 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -4,13 +4,19 @@
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
 FULL_AMD_PATH=$(src)/..
+DISPLAY_FOLDER_NAME=display
+FULL_AMD_DISPLAY_PATH = $(FULL_AMD_PATH)/$(DISPLAY_FOLDER_NAME)
 
 ccflags-y := -I$(FULL_AMD_PATH)/include/asic_reg \
         -I$(FULL_AMD_PATH)/include \
         -I$(FULL_AMD_PATH)/amdgpu \
         -I$(FULL_AMD_PATH)/scheduler \
         -I$(FULL_AMD_PATH)/powerplay/inc \
-        -I$(FULL_AMD_PATH)/acp/include
+        -I$(FULL_AMD_PATH)/acp/include \
+        -I$(FULL_AMD_DISPLAY_PATH) \
+        -I$(FULL_AMD_DISPLAY_PATH)/include \
+        -I$(FULL_AMD_DISPLAY_PATH)/dc \
+        -I$(FULL_AMD_DISPLAY_PATH)/amdgpu_dm
 
 amdgpu-y := amdgpu_drv.o
 
@@ -26,7 +32,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
         amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
         amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
         amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \
-        amdgpu_queue_mgr.o amdgpu_vf_error.o
+        amdgpu_queue_mgr.o amdgpu_vf_error.o amdgpu_sched.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
@@ -133,6 +139,13 @@ include $(FULL_AMD_PATH)/powerplay/Makefile
 
 amdgpu-y += $(AMD_POWERPLAY_FILES)
 
-obj-$(CONFIG_DRM_AMDGPU)+= amdgpu.o
+ifneq ($(CONFIG_DRM_AMD_DC),)
+
+RELATIVE_AMD_DISPLAY_PATH = ../$(DISPLAY_FOLDER_NAME)
+include $(FULL_AMD_DISPLAY_PATH)/Makefile
+
+amdgpu-y += $(AMD_DISPLAY_FILES)
 
-CFLAGS_amdgpu_trace_points.o := -I$(src)
+endif
+
+obj-$(CONFIG_DRM_AMDGPU)+= amdgpu.o
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 103635ab784c..5afaf6016b4a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -65,6 +65,8 @@
 #include "amdgpu_uvd.h"
 #include "amdgpu_vce.h"
 #include "amdgpu_vcn.h"
+#include "amdgpu_mn.h"
+#include "amdgpu_dm.h"
 
 #include "gpu_scheduler.h"
 #include "amdgpu_virt.h"
@@ -91,7 +93,7 @@ extern int amdgpu_dpm;
 extern int amdgpu_fw_load_type;
 extern int amdgpu_aspm;
 extern int amdgpu_runtime_pm;
-extern unsigned amdgpu_ip_block_mask;
+extern uint amdgpu_ip_block_mask;
 extern int amdgpu_bapm;
 extern int amdgpu_deep_color;
 extern int amdgpu_vm_size;
@@ -100,18 +102,20 @@ extern int amdgpu_vm_fragment_size;
 extern int amdgpu_vm_fault_stop;
 extern int amdgpu_vm_debug;
 extern int amdgpu_vm_update_mode;
+extern int amdgpu_dc;
+extern int amdgpu_dc_log;
 extern int amdgpu_sched_jobs;
 extern int amdgpu_sched_hw_submission;
 extern int amdgpu_no_evict;
 extern int amdgpu_direct_gma_size;
-extern unsigned amdgpu_pcie_gen_cap;
-extern unsigned amdgpu_pcie_lane_cap;
-extern unsigned amdgpu_cg_mask;
-extern unsigned amdgpu_pg_mask;
-extern unsigned amdgpu_sdma_phase_quantum;
+extern uint amdgpu_pcie_gen_cap;
+extern uint amdgpu_pcie_lane_cap;
+extern uint amdgpu_cg_mask;
+extern uint amdgpu_pg_mask;
+extern uint amdgpu_sdma_phase_quantum;
 extern char *amdgpu_disable_cu;
 extern char *amdgpu_virtual_display;
-extern unsigned amdgpu_pp_feature_mask;
+extern uint amdgpu_pp_feature_mask;
 extern int amdgpu_vram_page_split;
 extern int amdgpu_ngg;
 extern int amdgpu_prim_buf_per_se;
@@ -120,6 +124,7 @@ extern int amdgpu_cntl_sb_buf_per_se;
 extern int amdgpu_param_buf_per_se;
 extern int amdgpu_job_hang_limit;
 extern int amdgpu_lbpw;
+extern int amdgpu_compute_multipipe;
 
 #ifdef CONFIG_DRM_AMDGPU_SI
 extern int amdgpu_si_support;
@@ -178,6 +183,7 @@ struct amdgpu_cs_parser;
 struct amdgpu_job;
 struct amdgpu_irq_src;
 struct amdgpu_fpriv;
+struct amdgpu_bo_va_mapping;
 
 enum amdgpu_cp_irq {
         AMDGPU_CP_IRQ_GFX_EOP = 0,
@@ -292,14 +298,25 @@ struct amdgpu_buffer_funcs {
 
 /* provided by hw blocks that can write ptes, e.g., sdma */
 struct amdgpu_vm_pte_funcs {
+        /* number of dw to reserve per operation */
+        unsigned        copy_pte_num_dw;
+
         /* copy pte entries from GART */
         void (*copy_pte)(struct amdgpu_ib *ib,
                          uint64_t pe, uint64_t src,
                          unsigned count);
+
         /* write pte one entry at a time with addr mapping */
         void (*write_pte)(struct amdgpu_ib *ib, uint64_t pe,
                           uint64_t value, unsigned count,
                           uint32_t incr);
+
+        /* maximum nums of PTEs/PDEs in a single operation */
+        uint32_t        set_max_nums_pte_pde;
+
+        /* number of dw to reserve per operation */
+        unsigned        set_pte_pde_num_dw;
+
         /* for linear pte/pde updates without addr mapping */
         void (*set_pte_pde)(struct amdgpu_ib *ib,
                             uint64_t pe,
@@ -332,6 +349,7 @@ struct amdgpu_gart_funcs {
 struct amdgpu_ih_funcs {
         /* ring read/write ptr handling, called from interrupt context */
         u32 (*get_wptr)(struct amdgpu_device *adev);
+        bool (*prescreen_iv)(struct amdgpu_device *adev);
         void (*decode_iv)(struct amdgpu_device *adev,
                           struct amdgpu_iv_entry *entry);
         void (*set_rptr)(struct amdgpu_device *adev);
@@ -399,6 +417,7 @@ void amdgpu_gem_prime_unpin(struct drm_gem_object *obj);
 struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *);
 void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj);
 void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
+int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
 int amdgpu_gem_debugfs_init(struct amdgpu_device *adev);
 
 /* sub-allocation manager, it has to be protected by another lock.
@@ -455,9 +474,10 @@ struct amdgpu_sa_bo {
  */
 void amdgpu_gem_force_release(struct amdgpu_device *adev);
 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
-                                int alignment, u32 initial_domain,
-                                u64 flags, bool kernel,
-                                struct drm_gem_object **obj);
+                             int alignment, u32 initial_domain,
+                             u64 flags, bool kernel,
+                             struct reservation_object *resv,
+                             struct drm_gem_object **obj);
 
 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
                             struct drm_device *dev,
@@ -715,10 +735,14 @@ struct amdgpu_ctx {
         struct amdgpu_device    *adev;
         struct amdgpu_queue_mgr queue_mgr;
         unsigned                reset_counter;
+        uint32_t                vram_lost_counter;
         spinlock_t              ring_lock;
         struct dma_fence        **fences;
         struct amdgpu_ctx_ring  rings[AMDGPU_MAX_RINGS];
-        bool preamble_presented;
+        bool                    preamble_presented;
+        enum amd_sched_priority init_priority;
+        enum amd_sched_priority override_priority;
+        struct mutex            lock;
 };
 
 struct amdgpu_ctx_mgr {
@@ -731,17 +755,22 @@ struct amdgpu_ctx_mgr {
 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id);
 int amdgpu_ctx_put(struct amdgpu_ctx *ctx);
 
-uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
-                              struct dma_fence *fence);
+int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
+                              struct dma_fence *fence, uint64_t *seq);
 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
                                    struct amdgpu_ring *ring, uint64_t seq);
+void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
+                                  enum amd_sched_priority priority);
 
 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
                      struct drm_file *filp);
 
+int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, unsigned ring_id);
+
 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
 
+
 /*
  * file private structure
  */
@@ -753,7 +782,6 @@ struct amdgpu_fpriv {
         struct mutex            bo_list_lock;
         struct idr              bo_list_handles;
         struct amdgpu_ctx_mgr   ctx_mgr;
-        u32                     vram_lost_counter;
 };
 
 /*
@@ -854,7 +882,7 @@ struct amdgpu_mec {
 struct amdgpu_kiq {
         u64                     eop_gpu_addr;
         struct amdgpu_bo        *eop_obj;
-        struct mutex            ring_mutex;
+        spinlock_t              ring_lock;
         struct amdgpu_ring      ring;
         struct amdgpu_irq_src   irq;
 };
@@ -1014,11 +1042,14 @@ struct amdgpu_gfx {
         /* reset mask */
         uint32_t                        grbm_soft_reset;
         uint32_t                        srbm_soft_reset;
-        bool                            in_reset;
         /* s3/s4 mask */
         bool                            in_suspend;
         /* NGG */
         struct amdgpu_ngg               ngg;
+
+        /* pipe reservation */
+        struct mutex                    pipe_reserve_mutex;
+        DECLARE_BITMAP                  (pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 };
 
 int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
@@ -1056,6 +1087,7 @@ struct amdgpu_cs_parser {
         /* buffer objects */
         struct ww_acquire_ctx           ticket;
         struct amdgpu_bo_list           *bo_list;
+        struct amdgpu_mn                *mn;
         struct amdgpu_bo_list_entry     vm_pd;
         struct list_head                validated;
         struct dma_fence                *fence;
@@ -1096,6 +1128,7 @@ struct amdgpu_job {
         uint32_t                gds_base, gds_size;
         uint32_t                gws_base, gws_size;
         uint32_t                oa_base, oa_size;
+        uint32_t                vram_lost_counter;
 
         /* user fence handling */
         uint64_t                uf_addr;
@@ -1121,7 +1154,7 @@ static inline void amdgpu_set_ib_value(struct amdgpu_cs_parser *p,
 /*
  * Writeback
  */
-#define AMDGPU_MAX_WB 1024      /* Reserve at most 1024 WB slots for amdgpu-owned rings. */
+#define AMDGPU_MAX_WB 512       /* Reserve at most 512 WB slots for amdgpu-owned rings. */
 
 struct amdgpu_wb {
         struct amdgpu_bo        *wb_obj;
@@ -1183,6 +1216,9 @@ struct amdgpu_firmware {
 
         /* gpu info firmware data pointer */
         const struct firmware *gpu_info_fw;
+
+        void *fw_buf_ptr;
+        uint64_t fw_buf_mc;
 };
 
 /*
@@ -1197,20 +1233,6 @@ void amdgpu_benchmark(struct amdgpu_device *adev, int test_number);
 void amdgpu_test_moves(struct amdgpu_device *adev);
 
 /*
- * MMU Notifier
- */
-#if defined(CONFIG_MMU_NOTIFIER)
-int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
-void amdgpu_mn_unregister(struct amdgpu_bo *bo);
-#else
-static inline int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
-{
-        return -ENODEV;
-}
-static inline void amdgpu_mn_unregister(struct amdgpu_bo *bo) {}
-#endif
-
-/*
  * Debugfs
  */
 struct amdgpu_debugfs {
@@ -1305,6 +1327,8 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
                         struct drm_file *filp);
 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
+int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
+                                    struct drm_file *filp);
 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
                                 struct drm_file *filp);
@@ -1371,6 +1395,18 @@ struct amdgpu_atcs {
 };
 
 /*
+ * Firmware VRAM reservation
+ */
+struct amdgpu_fw_vram_usage {
+        u64 start_offset;
+        u64 size;
+        struct amdgpu_bo *reserved_bo;
+        void *va;
+};
+
+int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev);
+
+/*
  * CGS
  */
 struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev);
@@ -1502,6 +1538,7 @@ struct amdgpu_device {
         /* display */
         bool                            enable_virtual_display;
         struct amdgpu_mode_info         mode_info;
+        /* For pre-DCE11. DCE11 and later are in "struct amdgpu_device->dm" */
         struct work_struct              hotplug_work;
         struct amdgpu_irq_src           crtc_irq;
         struct amdgpu_irq_src           pageflip_irq;
@@ -1519,7 +1556,6 @@ struct amdgpu_device {
 
         /* powerplay */
         struct amd_powerplay            powerplay;
-        bool                            pp_enabled;
         bool                            pp_force_state_enabled;
 
         /* dpm */
@@ -1558,6 +1594,9 @@ struct amdgpu_device {
         /* GDS */
         struct amdgpu_gds               gds;
 
+        /* display related functionality */
+        struct amdgpu_display_manager dm;
+
         struct amdgpu_ip_block          ip_blocks[AMDGPU_MAX_IP_NUM];
         int                             num_ip_blocks;
         struct mutex    mn_lock;
@@ -1575,6 +1614,8 @@ struct amdgpu_device {
         struct delayed_work     late_init_work;
 
         struct amdgpu_virt      virt;
+        /* firmware VRAM reservation */
+        struct amdgpu_fw_vram_usage fw_vram_usage;
 
         /* link all shadow bo */
         struct list_head                shadow_list;
@@ -1592,6 +1633,7 @@ struct amdgpu_device {
 
         /* record last mm index being written through WREG32*/
         unsigned long last_mm_index;
+        bool                            in_sriov_reset;
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
@@ -1618,6 +1660,9 @@ void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v);
 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index);
 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);
 
+bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type);
+bool amdgpu_device_has_dc_support(struct amdgpu_device *adev);
+
 /*
  * Registers read & write functions.
  */
@@ -1759,6 +1804,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
 #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
 #define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev))
+#define amdgpu_ih_prescreen_iv(adev) (adev)->irq.ih_funcs->prescreen_iv((adev))
 #define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv))
 #define amdgpu_ih_set_rptr(adev) (adev)->irq.ih_funcs->set_rptr((adev))
 #define amdgpu_display_vblank_get_counter(adev, crtc) (adev)->mode_info.funcs->vblank_get_counter((adev), (crtc))
@@ -1791,18 +1837,6 @@ void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
                                   u64 num_vis_bytes);
 void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
 bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
-int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
-int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
-                                     uint32_t flags);
-bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm);
-struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm);
-bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
-                                  unsigned long end);
-bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
-                                       int *last_invalidated);
-bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
-uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
-                                 struct ttm_mem_reg *mem);
 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base);
 void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);
 void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size);
@@ -1836,8 +1870,6 @@ static inline bool amdgpu_has_atpx(void) { return false; }
 extern const struct drm_ioctl_desc amdgpu_ioctls_kms[];
 extern const int amdgpu_max_kms_ioctl;
 
-bool amdgpu_kms_vram_lost(struct amdgpu_device *adev,
-                          struct amdgpu_fpriv *fpriv);
 int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags);
 void amdgpu_driver_unload_kms(struct drm_device *dev);
 void amdgpu_driver_lastclose_kms(struct drm_device *dev);
@@ -1885,10 +1917,15 @@ static inline int amdgpu_acpi_init(struct amdgpu_device *adev) { return 0; }
 static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { }
 #endif
 
-struct amdgpu_bo_va_mapping *
-amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
-                       uint64_t addr, struct amdgpu_bo **bo);
-int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser);
+int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
+                           uint64_t addr, struct amdgpu_bo **bo,
+                           struct amdgpu_bo_va_mapping **mapping);
+
+#if defined(CONFIG_DRM_AMD_DC)
+int amdgpu_dm_display_resume(struct amdgpu_device *adev );
+#else
+static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return 0; }
+#endif
 
 #include "amdgpu_object.h"
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
index a52795d9b458..c04f44a90392 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
@@ -35,41 +35,50 @@
 
 #include "acp_gfx_if.h"
 
-#define ACP_TILE_ON_MASK                0x03
-#define ACP_TILE_OFF_MASK               0x02
-#define ACP_TILE_ON_RETAIN_REG_MASK     0x1f
-#define ACP_TILE_OFF_RETAIN_REG_MASK    0x20
-
-#define ACP_TILE_P1_MASK                0x3e
-#define ACP_TILE_P2_MASK                0x3d
-#define ACP_TILE_DSP0_MASK              0x3b
-#define ACP_TILE_DSP1_MASK              0x37
-
-#define ACP_TILE_DSP2_MASK              0x2f
-
-#define ACP_DMA_REGS_END                0x146c0
-#define ACP_I2S_PLAY_REGS_START         0x14840
-#define ACP_I2S_PLAY_REGS_END           0x148b4
-#define ACP_I2S_CAP_REGS_START          0x148b8
-#define ACP_I2S_CAP_REGS_END            0x1496c
-
-#define ACP_I2S_COMP1_CAP_REG_OFFSET    0xac
-#define ACP_I2S_COMP2_CAP_REG_OFFSET    0xa8
-#define ACP_I2S_COMP1_PLAY_REG_OFFSET   0x6c
-#define ACP_I2S_COMP2_PLAY_REG_OFFSET   0x68
-
-#define mmACP_PGFSM_RETAIN_REG          0x51c9
-#define mmACP_PGFSM_CONFIG_REG          0x51ca
-#define mmACP_PGFSM_READ_REG_0          0x51cc
-
-#define mmACP_MEM_SHUT_DOWN_REQ_LO      0x51f8
-#define mmACP_MEM_SHUT_DOWN_REQ_HI      0x51f9
-#define mmACP_MEM_SHUT_DOWN_STS_LO      0x51fa
-#define mmACP_MEM_SHUT_DOWN_STS_HI      0x51fb
-
-#define ACP_TIMEOUT_LOOP                0x000000FF
-#define ACP_DEVS                        3
-#define ACP_SRC_ID                      162
+#define ACP_TILE_ON_MASK                        0x03
+#define ACP_TILE_OFF_MASK                       0x02
+#define ACP_TILE_ON_RETAIN_REG_MASK             0x1f
+#define ACP_TILE_OFF_RETAIN_REG_MASK            0x20
+
+#define ACP_TILE_P1_MASK                        0x3e
+#define ACP_TILE_P2_MASK                        0x3d
+#define ACP_TILE_DSP0_MASK                      0x3b
+#define ACP_TILE_DSP1_MASK                      0x37
+
+#define ACP_TILE_DSP2_MASK                      0x2f
+
+#define ACP_DMA_REGS_END                        0x146c0
+#define ACP_I2S_PLAY_REGS_START                 0x14840
+#define ACP_I2S_PLAY_REGS_END                   0x148b4
+#define ACP_I2S_CAP_REGS_START                  0x148b8
+#define ACP_I2S_CAP_REGS_END                    0x1496c
+
+#define ACP_I2S_COMP1_CAP_REG_OFFSET            0xac
+#define ACP_I2S_COMP2_CAP_REG_OFFSET            0xa8
+#define ACP_I2S_COMP1_PLAY_REG_OFFSET           0x6c
+#define ACP_I2S_COMP2_PLAY_REG_OFFSET           0x68
+
+#define mmACP_PGFSM_RETAIN_REG                  0x51c9
+#define mmACP_PGFSM_CONFIG_REG                  0x51ca
+#define mmACP_PGFSM_READ_REG_0                  0x51cc
+
+#define mmACP_MEM_SHUT_DOWN_REQ_LO              0x51f8
+#define mmACP_MEM_SHUT_DOWN_REQ_HI              0x51f9
+#define mmACP_MEM_SHUT_DOWN_STS_LO              0x51fa
+#define mmACP_MEM_SHUT_DOWN_STS_HI              0x51fb
+
+#define mmACP_CONTROL                           0x5131
+#define mmACP_STATUS                            0x5133
+#define mmACP_SOFT_RESET                        0x5134
+#define ACP_CONTROL__ClkEn_MASK                 0x1
+#define ACP_SOFT_RESET__SoftResetAud_MASK       0x100
+#define ACP_SOFT_RESET__SoftResetAudDone_MASK   0x1000000
+#define ACP_CLOCK_EN_TIME_OUT_VALUE             0x000000FF
+#define ACP_SOFT_RESET_DONE_TIME_OUT_VALUE      0x000000FF
+
+#define ACP_TIMEOUT_LOOP                        0x000000FF
+#define ACP_DEVS                                3
+#define ACP_SRC_ID                              162
 
 enum {
         ACP_TILE_P1 = 0,
@@ -260,6 +269,8 @@ static int acp_hw_init(void *handle)
 {
         int r, i;
         uint64_t acp_base;
+        u32 val = 0;
+        u32 count = 0;
         struct device *dev;
         struct i2s_platform_data *i2s_pdata;
 
@@ -371,6 +382,8 @@ static int acp_hw_init(void *handle)
         adev->acp.acp_cell[0].name = "acp_audio_dma";
         adev->acp.acp_cell[0].num_resources = 4;
         adev->acp.acp_cell[0].resources = &adev->acp.acp_res[0];
+        adev->acp.acp_cell[0].platform_data = &adev->asic_type;
+        adev->acp.acp_cell[0].pdata_size = sizeof(adev->asic_type);
 
         adev->acp.acp_cell[1].name = "designware-i2s";
         adev->acp.acp_cell[1].num_resources = 1;
@@ -400,6 +413,46 @@ static int acp_hw_init(void *handle)
                 }
         }
 
+        /* Assert Soft reset of ACP */
+        val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET);
+
+        val |= ACP_SOFT_RESET__SoftResetAud_MASK;
+        cgs_write_register(adev->acp.cgs_device, mmACP_SOFT_RESET, val);
+
+        count = ACP_SOFT_RESET_DONE_TIME_OUT_VALUE;
+        while (true) {
+                val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET);
+                if (ACP_SOFT_RESET__SoftResetAudDone_MASK ==
+                    (val & ACP_SOFT_RESET__SoftResetAudDone_MASK))
+                        break;
+                if (--count == 0) {
+                        dev_err(&adev->pdev->dev, "Failed to reset ACP\n");
+                        return -ETIMEDOUT;
+                }
+                udelay(100);
+        }
+        /* Enable clock to ACP and wait until the clock is enabled */
+        val = cgs_read_register(adev->acp.cgs_device, mmACP_CONTROL);
+        val = val | ACP_CONTROL__ClkEn_MASK;
+        cgs_write_register(adev->acp.cgs_device, mmACP_CONTROL, val);
+
+        count = ACP_CLOCK_EN_TIME_OUT_VALUE;
+
+        while (true) {
+                val = cgs_read_register(adev->acp.cgs_device, mmACP_STATUS);
+                if (val & (u32) 0x1)
+                        break;
+                if (--count == 0) {
+                        dev_err(&adev->pdev->dev, "Failed to reset ACP\n");
+                        return -ETIMEDOUT;
+                }
+                udelay(100);
+        }
+        /* Deassert the SOFT RESET flags */
+        val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET);
+        val &= ~ACP_SOFT_RESET__SoftResetAud_MASK;
+        cgs_write_register(adev->acp.cgs_device, mmACP_SOFT_RESET, val);
+
         return 0;
 }
 
@@ -412,6 +465,8 @@ static int acp_hw_init(void *handle)
 static int acp_hw_fini(void *handle)
 {
         int i, ret;
+        u32 val = 0;
+        u32 count = 0;
         struct device *dev;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
@@ -419,6 +474,42 @@ static int acp_hw_fini(void *handle)
         if (!adev->acp.acp_cell)
                 return 0;
 
+        /* Assert Soft reset of ACP */
+        val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET);
+
+        val |= ACP_SOFT_RESET__SoftResetAud_MASK;
+        cgs_write_register(adev->acp.cgs_device, mmACP_SOFT_RESET, val);
+
+        count = ACP_SOFT_RESET_DONE_TIME_OUT_VALUE;
+        while (true) {
+                val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET);
+                if (ACP_SOFT_RESET__SoftResetAudDone_MASK ==
+                    (val & ACP_SOFT_RESET__SoftResetAudDone_MASK))
+                        break;
+                if (--count == 0) {
+                        dev_err(&adev->pdev->dev, "Failed to reset ACP\n");
+                        return -ETIMEDOUT;
+                }
+                udelay(100);
+        }
+        /* Disable ACP clock */
+        val = cgs_read_register(adev->acp.cgs_device, mmACP_CONTROL);
+        val &= ~ACP_CONTROL__ClkEn_MASK;
+        cgs_write_register(adev->acp.cgs_device, mmACP_CONTROL, val);
+
+        count = ACP_CLOCK_EN_TIME_OUT_VALUE;
+
+        while (true) {
+                val = cgs_read_register(adev->acp.cgs_device, mmACP_STATUS);
+                if (val & (u32) 0x1)
+                        break;
+                if (--count == 0) {
+                        dev_err(&adev->pdev->dev, "Failed to reset ACP\n");
+                        return -ETIMEDOUT;
+                }
+                udelay(100);
+        }
+
         if (adev->acp.acp_genpd) {
                 for (i = 0; i < ACP_DEVS ; i++) {
                         dev = get_mfd_cell_dev(adev->acp.acp_cell[i].name, i);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index b9dbbf9cb8b0..47d1c132ac40 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -169,6 +169,8 @@ static const struct kfd2kgd_calls kfd2kgd = {
         .get_vmem_size = get_vmem_size,
         .get_gpu_clock_counter = get_gpu_clock_counter,
         .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+        .alloc_pasid = amdgpu_vm_alloc_pasid,
+        .free_pasid = amdgpu_vm_free_pasid,
         .program_sh_mem_settings = kgd_program_sh_mem_settings,
         .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
         .init_pipeline = kgd_init_pipeline,
@@ -336,6 +338,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
         struct cik_mqd *m;
         uint32_t *mqd_hqd;
         uint32_t reg, wptr_val, data;
+        bool valid_wptr = false;
 
         m = get_mqd(mqd);
 
@@ -354,7 +357,14 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
                              CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
         WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
 
-        if (read_user_wptr(mm, wptr, wptr_val))
+        /* read_user_ptr may take the mm->mmap_sem.
+         * release srbm_mutex to avoid circular dependency between
+         * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
+         */
+        release_queue(kgd);
+        valid_wptr = read_user_wptr(mm, wptr, wptr_val);
+        acquire_queue(kgd, pipe_id, queue_id);
+        if (valid_wptr)
                 WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
 
         data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index 309f2419c6d8..056929b8ccd0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -128,6 +128,8 @@ static const struct kfd2kgd_calls kfd2kgd = {
         .get_vmem_size = get_vmem_size,
         .get_gpu_clock_counter = get_gpu_clock_counter,
         .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+        .alloc_pasid = amdgpu_vm_alloc_pasid,
+        .free_pasid = amdgpu_vm_free_pasid,
         .program_sh_mem_settings = kgd_program_sh_mem_settings,
         .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
         .init_pipeline = kgd_init_pipeline,
@@ -290,6 +292,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
         struct vi_mqd *m;
         uint32_t *mqd_hqd;
         uint32_t reg, wptr_val, data;
+        bool valid_wptr = false;
 
         m = get_mqd(mqd);
 
@@ -337,7 +340,14 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
                              CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
         WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
 
-        if (read_user_wptr(mm, wptr, wptr_val))
+        /* read_user_ptr may take the mm->mmap_sem.
+         * release srbm_mutex to avoid circular dependency between
+         * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
+         */
+        release_queue(kgd);
+        valid_wptr = read_user_wptr(mm, wptr, wptr_val);
+        acquire_queue(kgd, pipe_id, queue_id);
+        if (valid_wptr)
                 WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
 
         data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index ce443586a0c7..f450b69323fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -1766,34 +1766,32 @@ bool amdgpu_atombios_scratch_need_asic_init(struct amdgpu_device *adev)
                 return true;
 }
 
-/* Atom needs data in little endian format
- * so swap as appropriate when copying data to
- * or from atom. Note that atom operates on
- * dw units.
+/* Atom needs data in little endian format so swap as appropriate when copying
+ * data to or from atom. Note that atom operates on dw units.
+ *
+ * Use to_le=true when sending data to atom and provide at least
+ * ALIGN(num_bytes,4) bytes in the dst buffer.
+ *
+ * Use to_le=false when receiving data from atom and provide ALIGN(num_bytes,4)
+ * byes in the src buffer.
  */
 void amdgpu_atombios_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le)
 {
 #ifdef __BIG_ENDIAN
-        u8 src_tmp[20], dst_tmp[20]; /* used for byteswapping */
-        u32 *dst32, *src32;
+        u32 src_tmp[5], dst_tmp[5];
         int i;
+        u8 align_num_bytes = ALIGN(num_bytes, 4);
 
-        memcpy(src_tmp, src, num_bytes);
-        src32 = (u32 *)src_tmp;
-        dst32 = (u32 *)dst_tmp;
         if (to_le) {
-                for (i = 0; i < ((num_bytes + 3) / 4); i++)
-                        dst32[i] = cpu_to_le32(src32[i]);
-                memcpy(dst, dst_tmp, num_bytes);
+                memcpy(src_tmp, src, num_bytes);
+                for (i = 0; i < align_num_bytes / 4; i++)
+                        dst_tmp[i] = cpu_to_le32(src_tmp[i]);
+                memcpy(dst, dst_tmp, align_num_bytes);
         } else {
-                u8 dws = num_bytes & ~3;
-                for (i = 0; i < ((num_bytes + 3) / 4); i++)
-                        dst32[i] = le32_to_cpu(src32[i]);
-                memcpy(dst, dst_tmp, dws);
-                if (num_bytes % 4) {
-                        for (i = 0; i < (num_bytes % 4); i++)
-                                dst[dws+i] = dst_tmp[dws+i];
-                }
+                memcpy(src_tmp, src, align_num_bytes);
+                for (i = 0; i < align_num_bytes / 4; i++)
+                        dst_tmp[i] = le32_to_cpu(src_tmp[i]);
+                memcpy(dst, dst_tmp, num_bytes);
         }
 #else
         memcpy(dst, src, num_bytes);
@@ -1807,6 +1805,8 @@ int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev)
         uint16_t data_offset;
         int usage_bytes = 0;
         struct _ATOM_VRAM_USAGE_BY_FIRMWARE *firmware_usage;
+        u64 start_addr;
+        u64 size;
 
         if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
                 firmware_usage = (struct _ATOM_VRAM_USAGE_BY_FIRMWARE *)(ctx->bios + data_offset);
@@ -1815,7 +1815,21 @@ int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev)
                           le32_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].ulStartAddrUsedByFirmware),
                           le16_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb));
 
-                usage_bytes = le16_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb) * 1024;
+                start_addr = firmware_usage->asFirmwareVramReserveInfo[0].ulStartAddrUsedByFirmware;
+                size = firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb;
+
+                if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+                        (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+                        ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+                        /* Firmware request VRAM reservation for SR-IOV */
+                        adev->fw_vram_usage.start_offset = (start_addr &
+                                (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+                        adev->fw_vram_usage.size = size << 10;
+                        /* Use the default scratch size */
+                        usage_bytes = 0;
+                } else {
+                        usage_bytes = le16_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb) * 1024;
+                }
         }
         ctx->scratch_size_bytes = 0;
         if (usage_bytes == 0)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index f9ffe8ef0cd6..ff8efd0f8fd5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -71,19 +71,33 @@ int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
         struct atom_context *ctx = adev->mode_info.atom_context;
         int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
                                                 vram_usagebyfirmware);
+        struct vram_usagebyfirmware_v2_1 *      firmware_usage;
+        uint32_t start_addr, size;
         uint16_t data_offset;
         int usage_bytes = 0;
 
         if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
-                struct vram_usagebyfirmware_v2_1 *firmware_usage =
-                        (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
-
+                firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
                 DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
                           le32_to_cpu(firmware_usage->start_address_in_kb),
                           le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
                           le16_to_cpu(firmware_usage->used_by_driver_in_kb));
 
-                usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) * 1024;
+                start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
+                size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
+
+                if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+                        (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+                        ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+                        /* Firmware request VRAM reservation for SR-IOV */
+                        adev->fw_vram_usage.start_offset = (start_addr &
+                                (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+                        adev->fw_vram_usage.size = size << 10;
+                        /* Use the default scratch size */
+                        usage_bytes = 0;
+                } else {
+                        usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+                }
         }
         ctx->scratch_size_bytes = 0;
         if (usage_bytes == 0)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
index c21adf60a7f2..057e1ecd83ce 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
@@ -59,12 +59,6 @@ static bool check_atom_bios(uint8_t *bios, size_t size)
                 return false;
         }
 
-        tmp = bios[0x18] | (bios[0x19] << 8);
-        if (bios[tmp + 0x14] != 0x0) {
-                DRM_INFO("Not an x86 BIOS ROM\n");
-                return false;
-        }
-
         bios_header_start = bios[0x48] | (bios[0x49] << 8);
         if (!bios_header_start) {
                 DRM_INFO("Can't locate bios header\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index fd435a96481c..f2b72c7c6857 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -42,10 +42,31 @@ struct amdgpu_cgs_device {
         struct amdgpu_device *adev =                                    \
                 ((struct amdgpu_cgs_device *)cgs_device)->adev
 
+static void *amdgpu_cgs_register_pp_handle(struct cgs_device *cgs_device,
+                        int (*call_back_func)(struct amd_pp_init *, void **))
+{
+        CGS_FUNC_ADEV;
+        struct amd_pp_init pp_init;
+        struct amd_powerplay *amd_pp;
+
+        if (call_back_func == NULL)
+                return NULL;
+
+        amd_pp = &(adev->powerplay);
+        pp_init.chip_family = adev->family;
+        pp_init.chip_id = adev->asic_type;
+        pp_init.pm_en = (amdgpu_dpm != 0 && !amdgpu_sriov_vf(adev)) ? true : false;
+        pp_init.feature_mask = amdgpu_pp_feature_mask;
+        pp_init.device = cgs_device;
+        if (call_back_func(&pp_init, &(amd_pp->pp_handle)))
+                return NULL;
+
+        return adev->powerplay.pp_handle;
+}
+
 static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
                                     enum cgs_gpu_mem_type type,
                                     uint64_t size, uint64_t align,
-                                    uint64_t min_offset, uint64_t max_offset,
                                     cgs_handle_t *handle)
 {
         CGS_FUNC_ADEV;
@@ -53,13 +74,6 @@ static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
         int ret = 0;
         uint32_t domain = 0;
         struct amdgpu_bo *obj;
-        struct ttm_placement placement;
-        struct ttm_place place;
-
-        if (min_offset > max_offset) {
-                BUG_ON(1);
-                return -EINVAL;
-        }
 
         /* fail if the alignment is not a power of 2 */
         if (((align != 1) && (align & (align - 1)))
@@ -73,41 +87,19 @@ static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
                 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
                         AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
                 domain = AMDGPU_GEM_DOMAIN_VRAM;
-                if (max_offset > adev->mc.real_vram_size)
-                        return -EINVAL;
-                place.fpfn = min_offset >> PAGE_SHIFT;
-                place.lpfn = max_offset >> PAGE_SHIFT;
-                place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
-                        TTM_PL_FLAG_VRAM;
                 break;
         case CGS_GPU_MEM_TYPE__INVISIBLE_CONTIG_FB:
         case CGS_GPU_MEM_TYPE__INVISIBLE_FB:
                 flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
                         AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
                 domain = AMDGPU_GEM_DOMAIN_VRAM;
-                if (adev->mc.visible_vram_size < adev->mc.real_vram_size) {
-                        place.fpfn =
-                                max(min_offset, adev->mc.visible_vram_size) >> PAGE_SHIFT;
-                        place.lpfn =
-                                min(max_offset, adev->mc.real_vram_size) >> PAGE_SHIFT;
-                        place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
-                                TTM_PL_FLAG_VRAM;
-                }
-
                 break;
         case CGS_GPU_MEM_TYPE__GART_CACHEABLE:
                 domain = AMDGPU_GEM_DOMAIN_GTT;
-                place.fpfn = min_offset >> PAGE_SHIFT;
-                place.lpfn = max_offset >> PAGE_SHIFT;
-                place.flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_TT;
                 break;
         case CGS_GPU_MEM_TYPE__GART_WRITECOMBINE:
                 flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
                 domain = AMDGPU_GEM_DOMAIN_GTT;
-                place.fpfn = min_offset >> PAGE_SHIFT;
-                place.lpfn = max_offset >> PAGE_SHIFT;
-                place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_TT |
-                        TTM_PL_FLAG_UNCACHED;
                 break;
         default:
                 return -EINVAL;
@@ -116,15 +108,8 @@ static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
 
         *handle = 0;
 
-        placement.placement = &place;
-        placement.num_placement = 1;
-        placement.busy_placement = &place;
-        placement.num_busy_placement = 1;
-
-        ret = amdgpu_bo_create_restricted(adev, size, PAGE_SIZE,
-                                          true, domain, flags,
-                                          NULL, &placement, NULL,
-                                          0, &obj);
+        ret = amdgpu_bo_create(adev, size, align, true, domain, flags,
+                               NULL, NULL, 0, &obj);
         if (ret) {
                 DRM_ERROR("(%d) bo create failed\n", ret);
                 return ret;
@@ -155,19 +140,14 @@ static int amdgpu_cgs_gmap_gpu_mem(struct cgs_device *cgs_device, cgs_handle_t h
                                    uint64_t *mcaddr)
 {
         int r;
-        u64 min_offset, max_offset;
         struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
 
         WARN_ON_ONCE(obj->placement.num_placement > 1);
 
-        min_offset = obj->placements[0].fpfn << PAGE_SHIFT;
-        max_offset = obj->placements[0].lpfn << PAGE_SHIFT;
-
         r = amdgpu_bo_reserve(obj, true);
         if (unlikely(r != 0))
                 return r;
-        r = amdgpu_bo_pin_restricted(obj, obj->preferred_domains,
-                                     min_offset, max_offset, mcaddr);
+        r = amdgpu_bo_pin(obj, obj->preferred_domains, mcaddr);
         amdgpu_bo_unreserve(obj);
         return r;
 }
@@ -675,6 +655,85 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 
                 if (!adev->pm.fw) {
                         switch (adev->asic_type) {
+                        case CHIP_TAHITI:
+                                strcpy(fw_name, "radeon/tahiti_smc.bin");
+                                break;
+                        case CHIP_PITCAIRN:
+                                if ((adev->pdev->revision == 0x81) &&
+                                    ((adev->pdev->device == 0x6810) ||
+                                    (adev->pdev->device == 0x6811))) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/pitcairn_k_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/pitcairn_smc.bin");
+                                }
+                                break;
+                        case CHIP_VERDE:
+                                if (((adev->pdev->device == 0x6820) &&
+                                        ((adev->pdev->revision == 0x81) ||
+                                        (adev->pdev->revision == 0x83))) ||
+                                    ((adev->pdev->device == 0x6821) &&
+                                        ((adev->pdev->revision == 0x83) ||
+                                        (adev->pdev->revision == 0x87))) ||
+                                    ((adev->pdev->revision == 0x87) &&
+                                        ((adev->pdev->device == 0x6823) ||
+                                        (adev->pdev->device == 0x682b)))) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/verde_k_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/verde_smc.bin");
+                                }
+                                break;
+                        case CHIP_OLAND:
+                                if (((adev->pdev->revision == 0x81) &&
+                                        ((adev->pdev->device == 0x6600) ||
+                                        (adev->pdev->device == 0x6604) ||
+                                        (adev->pdev->device == 0x6605) ||
+                                        (adev->pdev->device == 0x6610))) ||
+                                    ((adev->pdev->revision == 0x83) &&
+                                        (adev->pdev->device == 0x6610))) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/oland_k_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/oland_smc.bin");
+                                }
+                                break;
+                        case CHIP_HAINAN:
+                                if (((adev->pdev->revision == 0x81) &&
+                                        (adev->pdev->device == 0x6660)) ||
+                                    ((adev->pdev->revision == 0x83) &&
+                                        ((adev->pdev->device == 0x6660) ||
+                                        (adev->pdev->device == 0x6663) ||
+                                        (adev->pdev->device == 0x6665) ||
+                                         (adev->pdev->device == 0x6667)))) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/hainan_k_smc.bin");
+                                } else if ((adev->pdev->revision == 0xc3) &&
+                                         (adev->pdev->device == 0x6665)) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/banks_k_2_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/hainan_smc.bin");
+                                }
+                                break;
+                        case CHIP_BONAIRE:
+                                if ((adev->pdev->revision == 0x80) ||
+                                        (adev->pdev->revision == 0x81) ||
+                                        (adev->pdev->device == 0x665f)) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/bonaire_k_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/bonaire_smc.bin");
+                                }
+                                break;
+                        case CHIP_HAWAII:
+                                if (adev->pdev->revision == 0x80) {
+                                        info->is_kicker = true;
+                                        strcpy(fw_name, "radeon/hawaii_k_smc.bin");
+                                } else {
+                                        strcpy(fw_name, "radeon/hawaii_smc.bin");
+                                }
+                                break;
                         case CHIP_TOPAZ:
                                 if (((adev->pdev->device == 0x6900) && (adev->pdev->revision == 0x81)) ||
                                     ((adev->pdev->device == 0x6900) && (adev->pdev->revision == 0x83)) ||
@@ -838,6 +897,9 @@ static int amdgpu_cgs_query_system_info(struct cgs_device *cgs_device,
         case CGS_SYSTEM_INFO_PCIE_SUB_SYS_VENDOR_ID:
                 sys_info->value = adev->pdev->subsystem_vendor;
                 break;
+        case CGS_SYSTEM_INFO_PCIE_BUS_DEVFN:
+                sys_info->value = adev->pdev->devfn;
+                break;
         default:
                 return -ENODEV;
         }
@@ -849,10 +911,6 @@ static int amdgpu_cgs_get_active_displays_info(struct cgs_device *cgs_device,
                                           struct cgs_display_info *info)
 {
         CGS_FUNC_ADEV;
-        struct amdgpu_crtc *amdgpu_crtc;
-        struct drm_device *ddev = adev->ddev;
-        struct drm_crtc *crtc;
-        uint32_t line_time_us, vblank_lines;
         struct cgs_mode_info *mode_info;
 
         if (info == NULL)
@@ -866,30 +924,43 @@ static int amdgpu_cgs_get_active_displays_info(struct cgs_device *cgs_device,
                 mode_info->ref_clock = adev->clock.spll.reference_freq;
         }
 
-        if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) {
-                list_for_each_entry(crtc,
-                                &ddev->mode_config.crtc_list, head) {
-                        amdgpu_crtc = to_amdgpu_crtc(crtc);
-                        if (crtc->enabled) {
-                                info->active_display_mask |= (1 << amdgpu_crtc->crtc_id);
-                                info->display_count++;
-                        }
-                        if (mode_info != NULL &&
-                                crtc->enabled && amdgpu_crtc->enabled &&
-                                amdgpu_crtc->hw_mode.clock) {
-                                line_time_us = (amdgpu_crtc->hw_mode.crtc_htotal * 1000) /
-                                                        amdgpu_crtc->hw_mode.clock;
-                                vblank_lines = amdgpu_crtc->hw_mode.crtc_vblank_end -
-                                                        amdgpu_crtc->hw_mode.crtc_vdisplay +
-                                                        (amdgpu_crtc->v_border * 2);
-                                mode_info->vblank_time_us = vblank_lines * line_time_us;
-                                mode_info->refresh_rate = drm_mode_vrefresh(&amdgpu_crtc->hw_mode);
-                                mode_info->ref_clock = adev->clock.spll.reference_freq;
-                                mode_info = NULL;
+        if (!amdgpu_device_has_dc_support(adev)) {
+                struct amdgpu_crtc *amdgpu_crtc;
+                struct drm_device *ddev = adev->ddev;
+                struct drm_crtc *crtc;
+                uint32_t line_time_us, vblank_lines;
+
+                if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) {
+                        list_for_each_entry(crtc,
+                                        &ddev->mode_config.crtc_list, head) {
+                                amdgpu_crtc = to_amdgpu_crtc(crtc);
+                                if (crtc->enabled) {
+                                        info->active_display_mask |= (1 << amdgpu_crtc->crtc_id);
+                                        info->display_count++;
+                                }
+                                if (mode_info != NULL &&
+                                        crtc->enabled && amdgpu_crtc->enabled &&
+                                        amdgpu_crtc->hw_mode.clock) {
+                                        line_time_us = (amdgpu_crtc->hw_mode.crtc_htotal * 1000) /
+                                                                amdgpu_crtc->hw_mode.clock;
+                                        vblank_lines = amdgpu_crtc->hw_mode.crtc_vblank_end -
+                                                                amdgpu_crtc->hw_mode.crtc_vdisplay +
+                                                                (amdgpu_crtc->v_border * 2);
+                                        mode_info->vblank_time_us = vblank_lines * line_time_us;
+                                        mode_info->refresh_rate = drm_mode_vrefresh(&amdgpu_crtc->hw_mode);
+                                        mode_info->ref_clock = adev->clock.spll.reference_freq;
+                                        mode_info = NULL;
+                                }
                         }
                 }
+        } else {
+                info->display_count = adev->pm.pm_display_cfg.num_display;
+                if (mode_info != NULL) {
+                        mode_info->vblank_time_us = adev->pm.pm_display_cfg.min_vblank_time;
+                        mode_info->refresh_rate = adev->pm.pm_display_cfg.vrefresh;
+                        mode_info->ref_clock = adev->clock.spll.reference_freq;
+                }
         }
-
         return 0;
 }
 
@@ -1139,6 +1210,7 @@ static const struct cgs_ops amdgpu_cgs_ops = {
         .is_virtualization_enabled = amdgpu_cgs_is_virtualization_enabled,
         .enter_safe_mode = amdgpu_cgs_enter_safe_mode,
         .lock_grbm_idx = amdgpu_cgs_lock_grbm_idx,
+        .register_pp_handle = amdgpu_cgs_register_pp_handle,
 };
 
 static const struct cgs_os_ops amdgpu_cgs_os_ops = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index 8d1cf2d3e663..df9cbc78e168 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -231,7 +231,7 @@ amdgpu_connector_update_scratch_regs(struct drm_connector *connector,
                 if (connector->encoder_ids[i] == 0)
                         break;
 
-                encoder = drm_encoder_find(connector->dev,
+                encoder = drm_encoder_find(connector->dev, NULL,
                                         connector->encoder_ids[i]);
                 if (!encoder)
                         continue;
@@ -256,7 +256,7 @@ amdgpu_connector_find_encoder(struct drm_connector *connector,
         for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
                 if (connector->encoder_ids[i] == 0)
                         break;
-                encoder = drm_encoder_find(connector->dev,
+                encoder = drm_encoder_find(connector->dev, NULL,
                                         connector->encoder_ids[i]);
                 if (!encoder)
                         continue;
@@ -346,10 +346,8 @@ static void amdgpu_connector_free_edid(struct drm_connector *connector)
 {
         struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-        if (amdgpu_connector->edid) {
-                kfree(amdgpu_connector->edid);
-                amdgpu_connector->edid = NULL;
-        }
+        kfree(amdgpu_connector->edid);
+        amdgpu_connector->edid = NULL;
 }
 
 static int amdgpu_connector_ddc_get_modes(struct drm_connector *connector)
@@ -374,7 +372,7 @@ amdgpu_connector_best_single_encoder(struct drm_connector *connector)
 
         /* pick the encoder ids */
         if (enc_id)
-                return drm_encoder_find(connector->dev, enc_id);
+                return drm_encoder_find(connector->dev, NULL, enc_id);
         return NULL;
 }
 
@@ -1079,7 +1077,7 @@ amdgpu_connector_dvi_detect(struct drm_connector *connector, bool force)
                         if (connector->encoder_ids[i] == 0)
                                 break;
 
-                        encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]);
+                        encoder = drm_encoder_find(connector->dev, NULL, connector->encoder_ids[i]);
                         if (!encoder)
                                 continue;
 
@@ -1136,7 +1134,7 @@ amdgpu_connector_dvi_encoder(struct drm_connector *connector)
                 if (connector->encoder_ids[i] == 0)
                         break;
 
-                encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]);
+                encoder = drm_encoder_find(connector->dev, NULL, connector->encoder_ids[i]);
                 if (!encoder)
                         continue;
 
@@ -1155,7 +1153,7 @@ amdgpu_connector_dvi_encoder(struct drm_connector *connector)
         /* then check use digitial */
         /* pick the first one */
         if (enc_id)
-                return drm_encoder_find(connector->dev, enc_id);
+                return drm_encoder_find(connector->dev, NULL, enc_id);
         return NULL;
 }
 
@@ -1296,7 +1294,7 @@ u16 amdgpu_connector_encoder_get_dp_bridge_encoder_id(struct drm_connector *conn
                 if (connector->encoder_ids[i] == 0)
                         break;
 
-                encoder = drm_encoder_find(connector->dev,
+                encoder = drm_encoder_find(connector->dev, NULL,
                                         connector->encoder_ids[i]);
                 if (!encoder)
                         continue;
@@ -1325,7 +1323,7 @@ static bool amdgpu_connector_encoder_is_hbr2(struct drm_connector *connector)
         for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
                 if (connector->encoder_ids[i] == 0)
                         break;
-                encoder = drm_encoder_find(connector->dev,
+                encoder = drm_encoder_find(connector->dev, NULL,
                                         connector->encoder_ids[i]);
                 if (!encoder)
                         continue;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 60d8bedb694d..a57cec737c18 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -25,6 +25,7 @@
  *    Jerome Glisse <glisse@freedesktop.org>
  */
 #include <linux/pagemap.h>
+#include <linux/sync_file.h>
 #include <drm/drmP.h>
 #include <drm/amdgpu_drm.h>
 #include <drm/drm_syncobj.h>
@@ -89,12 +90,14 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
                 goto free_chunk;
         }
 
+        mutex_lock(&p->ctx->lock);
+
         /* get chunks */
         chunk_array_user = u64_to_user_ptr(cs->in.chunks);
         if (copy_from_user(chunk_array, chunk_array_user,
                            sizeof(uint64_t)*cs->in.num_chunks)) {
                 ret = -EFAULT;
-                goto put_ctx;
+                goto free_chunk;
         }
 
         p->nchunks = cs->in.num_chunks;
@@ -102,7 +105,7 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
                             GFP_KERNEL);
         if (!p->chunks) {
                 ret = -ENOMEM;
-                goto put_ctx;
+                goto free_chunk;
         }
 
         for (i = 0; i < p->nchunks; i++) {
@@ -169,6 +172,11 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
         if (ret)
                 goto free_all_kdata;
 
+        if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) {
+                ret = -ECANCELED;
+                goto free_all_kdata;
+        }
+
         if (p->uf_entry.robj)
                 p->job->uf_addr = uf_offset;
         kfree(chunk_array);
@@ -182,8 +190,6 @@ free_partial_kdata:
         kfree(p->chunks);
         p->chunks = NULL;
         p->nchunks = 0;
-put_ctx:
-        amdgpu_ctx_put(p->ctx);
 free_chunk:
         kfree(chunk_array);
 
@@ -473,11 +479,16 @@ static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
                         return -EPERM;
 
                 /* Check if we have user pages and nobody bound the BO already */
-                if (lobj->user_pages && bo->tbo.ttm->state != tt_bound) {
-                        size_t size = sizeof(struct page *);
-
-                        size *= bo->tbo.ttm->num_pages;
-                        memcpy(bo->tbo.ttm->pages, lobj->user_pages, size);
+                if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
+                    lobj->user_pages) {
+                        amdgpu_ttm_placement_from_domain(bo,
+                                                         AMDGPU_GEM_DOMAIN_CPU);
+                        r = ttm_bo_validate(&bo->tbo, &bo->placement, true,
+                                            false);
+                        if (r)
+                                return r;
+                        amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
+                                                     lobj->user_pages);
                         binding_userptr = true;
                 }
 
@@ -502,7 +513,6 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
         struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
         struct amdgpu_bo_list_entry *e;
         struct list_head duplicates;
-        bool need_mmap_lock = false;
         unsigned i, tries = 10;
         int r;
 
@@ -510,9 +520,9 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 
         p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
         if (p->bo_list) {
-                need_mmap_lock = p->bo_list->first_userptr !=
-                        p->bo_list->num_entries;
                 amdgpu_bo_list_get_list(p->bo_list, &p->validated);
+                if (p->bo_list->first_userptr != p->bo_list->num_entries)
+                        p->mn = amdgpu_mn_get(p->adev);
         }
 
         INIT_LIST_HEAD(&duplicates);
@@ -521,9 +531,6 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
         if (p->uf_entry.robj)
                 list_add(&p->uf_entry.tv.head, &p->validated);
 
-        if (need_mmap_lock)
-                down_read(&current->mm->mmap_sem);
-
         while (1) {
                 struct list_head need_pages;
                 unsigned i;
@@ -543,23 +550,24 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
                 INIT_LIST_HEAD(&need_pages);
                 for (i = p->bo_list->first_userptr;
                      i < p->bo_list->num_entries; ++i) {
+                        struct amdgpu_bo *bo;
 
                         e = &p->bo_list->array[i];
+                        bo = e->robj;
 
-                        if (amdgpu_ttm_tt_userptr_invalidated(e->robj->tbo.ttm,
+                        if (amdgpu_ttm_tt_userptr_invalidated(bo->tbo.ttm,
                                  &e->user_invalidated) && e->user_pages) {
 
                                 /* We acquired a page array, but somebody
                                  * invalidated it. Free it and try again
                                  */
                                 release_pages(e->user_pages,
-                                              e->robj->tbo.ttm->num_pages,
-                                              false);
+                                              bo->tbo.ttm->num_pages);
                                 kvfree(e->user_pages);
                                 e->user_pages = NULL;
                         }
 
-                        if (e->robj->tbo.ttm->state != tt_bound &&
+                        if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
                             !e->user_pages) {
                                 list_del(&e->tv.head);
                                 list_add(&e->tv.head, &need_pages);
@@ -636,9 +644,6 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 
         amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
                                      p->bytes_moved_vis);
-        fpriv->vm.last_eviction_counter =
-                atomic64_read(&p->adev->num_evictions);
-
         if (p->bo_list) {
                 struct amdgpu_bo *gds = p->bo_list->gds_obj;
                 struct amdgpu_bo *gws = p->bo_list->gws_obj;
@@ -679,9 +684,6 @@ error_validate:
 
 error_free_pages:
 
-        if (need_mmap_lock)
-                up_read(&current->mm->mmap_sem);
-
         if (p->bo_list) {
                 for (i = p->bo_list->first_userptr;
                      i < p->bo_list->num_entries; ++i) {
@@ -691,8 +693,7 @@ error_free_pages:
                                 continue;
 
                         release_pages(e->user_pages,
-                                      e->robj->tbo.ttm->num_pages,
-                                      false);
+                                      e->robj->tbo.ttm->num_pages);
                         kvfree(e->user_pages);
                 }
         }
@@ -707,7 +708,8 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
 
         list_for_each_entry(e, &p->validated, tv.head) {
                 struct reservation_object *resv = e->robj->tbo.resv;
-                r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp);
+                r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp,
+                                     amdgpu_bo_explicit_sync(e->robj));
 
                 if (r)
                         return r;
@@ -728,11 +730,7 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
 {
         unsigned i;
 
-        if (!error)
-                ttm_eu_fence_buffer_objects(&parser->ticket,
-                                            &parser->validated,
-                                            parser->fence);
-        else if (backoff)
+        if (error && backoff)
                 ttm_eu_backoff_reservation(&parser->ticket,
                                            &parser->validated);
 
@@ -742,8 +740,10 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
 
         dma_fence_put(parser->fence);
 
-        if (parser->ctx)
+        if (parser->ctx) {
+                mutex_unlock(&parser->ctx->lock);
                 amdgpu_ctx_put(parser->ctx);
+        }
         if (parser->bo_list)
                 amdgpu_bo_list_put(parser->bo_list);
 
@@ -768,10 +768,6 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
         if (r)
                 return r;
 
-        r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_dir_update);
-        if (r)
-                return r;
-
         r = amdgpu_vm_clear_freed(adev, vm, NULL);
         if (r)
                 return r;
@@ -825,7 +821,13 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 
         }
 
-        r = amdgpu_vm_clear_moved(adev, vm, &p->job->sync);
+        r = amdgpu_vm_handle_moved(adev, vm);
+        if (r)
+                return r;
+
+        r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update);
+        if (r)
+                return r;
 
         if (amdgpu_vm_debug && p->bo_list) {
                 /* Invalidate all BOs to test for userspace bugs */
@@ -835,7 +837,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
                         if (!bo)
                                 continue;
 
-                        amdgpu_vm_bo_invalidate(adev, bo);
+                        amdgpu_vm_bo_invalidate(adev, bo, false);
                 }
         }
 
@@ -848,19 +850,63 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
         struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
         struct amdgpu_vm *vm = &fpriv->vm;
         struct amdgpu_ring *ring = p->job->ring;
-        int i, r;
+        int r;
 
         /* Only for UVD/VCE VM emulation */
-        if (ring->funcs->parse_cs) {
-                for (i = 0; i < p->job->num_ibs; i++) {
-                        r = amdgpu_ring_parse_cs(ring, p, i);
+        if (p->job->ring->funcs->parse_cs) {
+                unsigned i, j;
+
+                for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) {
+                        struct drm_amdgpu_cs_chunk_ib *chunk_ib;
+                        struct amdgpu_bo_va_mapping *m;
+                        struct amdgpu_bo *aobj = NULL;
+                        struct amdgpu_cs_chunk *chunk;
+                        struct amdgpu_ib *ib;
+                        uint64_t offset;
+                        uint8_t *kptr;
+
+                        chunk = &p->chunks[i];
+                        ib = &p->job->ibs[j];
+                        chunk_ib = chunk->kdata;
+
+                        if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
+                                continue;
+
+                        r = amdgpu_cs_find_mapping(p, chunk_ib->va_start,
+                                                   &aobj, &m);
+                        if (r) {
+                                DRM_ERROR("IB va_start is invalid\n");
+                                return r;
+                        }
+
+                        if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
+                            (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
+                                DRM_ERROR("IB va_start+ib_bytes is invalid\n");
+                                return -EINVAL;
+                        }
+
+                        /* the IB should be reserved at this point */
+                        r = amdgpu_bo_kmap(aobj, (void **)&kptr);
+                        if (r) {
+                                return r;
+                        }
+
+                        offset = m->start * AMDGPU_GPU_PAGE_SIZE;
+                        kptr += chunk_ib->va_start - offset;
+
+                        memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
+                        amdgpu_bo_kunmap(aobj);
+
+                        r = amdgpu_ring_parse_cs(ring, p, j);
                         if (r)
                                 return r;
+
+                        j++;
                 }
         }
 
         if (p->job->vm) {
-                p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->root.bo);
+                p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->root.base.bo);
 
                 r = amdgpu_bo_vm_update_pte(p);
                 if (r)
@@ -922,54 +968,18 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
 
                 parser->job->ring = ring;
 
-                if (ring->funcs->parse_cs) {
-                        struct amdgpu_bo_va_mapping *m;
-                        struct amdgpu_bo *aobj = NULL;
-                        uint64_t offset;
-                        uint8_t *kptr;
-
-                        m = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
-                                                   &aobj);
-                        if (!aobj) {
-                                DRM_ERROR("IB va_start is invalid\n");
-                                return -EINVAL;
-                        }
-
-                        if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
-                            (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
-                                DRM_ERROR("IB va_start+ib_bytes is invalid\n");
-                                return -EINVAL;
-                        }
-
-                        /* the IB should be reserved at this point */
-                        r = amdgpu_bo_kmap(aobj, (void **)&kptr);
-                        if (r) {
-                                return r;
-                        }
-
-                        offset = m->start * AMDGPU_GPU_PAGE_SIZE;
-                        kptr += chunk_ib->va_start - offset;
-
-                        r =  amdgpu_ib_get(adev, vm, chunk_ib->ib_bytes, ib);
-                        if (r) {
-                                DRM_ERROR("Failed to get ib !\n");
-                                return r;
-                        }
-
-                        memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
-                        amdgpu_bo_kunmap(aobj);
-                } else {
-                        r =  amdgpu_ib_get(adev, vm, 0, ib);
-                        if (r) {
-                                DRM_ERROR("Failed to get ib !\n");
-                                return r;
-                        }
-
+                r =  amdgpu_ib_get(adev, vm,
+                                        ring->funcs->parse_cs ? chunk_ib->ib_bytes : 0,
+                                        ib);
+                if (r) {
+                        DRM_ERROR("Failed to get ib !\n");
+                        return r;
                 }
 
                 ib->gpu_addr = chunk_ib->va_start;
                 ib->length_dw = chunk_ib->ib_bytes / 4;
                 ib->flags = chunk_ib->flags;
+
                 j++;
         }
 
@@ -979,7 +989,7 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
             parser->job->ring->funcs->type == AMDGPU_RING_TYPE_VCE))
                 return -EINVAL;
 
-        return 0;
+        return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->job->ring->idx);
 }
 
 static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
@@ -1133,14 +1143,31 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
         struct amdgpu_ring *ring = p->job->ring;
         struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
         struct amdgpu_job *job;
+        unsigned i;
+        uint64_t seq;
+
         int r;
 
+        amdgpu_mn_lock(p->mn);
+        if (p->bo_list) {
+                for (i = p->bo_list->first_userptr;
+                     i < p->bo_list->num_entries; ++i) {
+                        struct amdgpu_bo *bo = p->bo_list->array[i].robj;
+
+                        if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm)) {
+                                amdgpu_mn_unlock(p->mn);
+                                return -ERESTARTSYS;
+                        }
+                }
+        }
+
         job = p->job;
         p->job = NULL;
 
         r = amd_sched_job_init(&job->base, &ring->sched, entity, p->filp);
         if (r) {
                 amdgpu_job_free(job);
+                amdgpu_mn_unlock(p->mn);
                 return r;
         }
 
@@ -1148,21 +1175,36 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
         job->fence_ctx = entity->fence_context;
         p->fence = dma_fence_get(&job->base.s_fence->finished);
 
+        r = amdgpu_ctx_add_fence(p->ctx, ring, p->fence, &seq);
+        if (r) {
+                dma_fence_put(p->fence);
+                dma_fence_put(&job->base.s_fence->finished);
+                amdgpu_job_free(job);
+                amdgpu_mn_unlock(p->mn);
+                return r;
+        }
+
         amdgpu_cs_post_dependencies(p);
 
-        cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence);
-        job->uf_sequence = cs->out.handle;
+        cs->out.handle = seq;
+        job->uf_sequence = seq;
+
         amdgpu_job_free_resources(job);
+        amdgpu_ring_priority_get(job->ring,
+                                 amd_sched_get_job_priority(&job->base));
 
         trace_amdgpu_cs_ioctl(job);
         amd_sched_entity_push_job(&job->base);
+
+        ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
+        amdgpu_mn_unlock(p->mn);
+
         return 0;
 }
 
 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 {
         struct amdgpu_device *adev = dev->dev_private;
-        struct amdgpu_fpriv *fpriv = filp->driver_priv;
         union drm_amdgpu_cs *cs = data;
         struct amdgpu_cs_parser parser = {};
         bool reserved_buffers = false;
@@ -1170,8 +1212,6 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 
         if (!adev->accel_working)
                 return -EBUSY;
-        if (amdgpu_kms_vram_lost(adev, fpriv))
-                return -ENODEV;
 
         parser.adev = adev;
         parser.filp = filp;
@@ -1182,6 +1222,10 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                 goto out;
         }
 
+        r = amdgpu_cs_ib_fill(adev, &parser);
+        if (r)
+                goto out;
+
         r = amdgpu_cs_parser_bos(&parser, data);
         if (r) {
                 if (r == -ENOMEM)
@@ -1192,9 +1236,6 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
         }
 
         reserved_buffers = true;
-        r = amdgpu_cs_ib_fill(adev, &parser);
-        if (r)
-                goto out;
 
         r = amdgpu_cs_dependencies(adev, &parser);
         if (r) {
@@ -1230,16 +1271,12 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
 {
         union drm_amdgpu_wait_cs *wait = data;
         struct amdgpu_device *adev = dev->dev_private;
-        struct amdgpu_fpriv *fpriv = filp->driver_priv;
         unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
         struct amdgpu_ring *ring = NULL;
         struct amdgpu_ctx *ctx;
         struct dma_fence *fence;
         long r;
 
-        if (amdgpu_kms_vram_lost(adev, fpriv))
-                return -ENODEV;
-
         ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
         if (ctx == NULL)
                 return -EINVAL;
@@ -1257,6 +1294,8 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
                 r = PTR_ERR(fence);
         else if (fence) {
                 r = dma_fence_wait_timeout(fence, true, timeout);
+                if (r > 0 && fence->error)
+                        r = fence->error;
                 dma_fence_put(fence);
         } else
                 r = 1;
@@ -1304,6 +1343,62 @@ static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,
         return fence;
 }
 
+int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
+                                    struct drm_file *filp)
+{
+        struct amdgpu_device *adev = dev->dev_private;
+        union drm_amdgpu_fence_to_handle *info = data;
+        struct dma_fence *fence;
+        struct drm_syncobj *syncobj;
+        struct sync_file *sync_file;
+        int fd, r;
+
+        fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence);
+        if (IS_ERR(fence))
+                return PTR_ERR(fence);
+
+        switch (info->in.what) {
+        case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ:
+                r = drm_syncobj_create(&syncobj, 0, fence);
+                dma_fence_put(fence);
+                if (r)
+                        return r;
+                r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle);
+                drm_syncobj_put(syncobj);
+                return r;
+
+        case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD:
+                r = drm_syncobj_create(&syncobj, 0, fence);
+                dma_fence_put(fence);
+                if (r)
+                        return r;
+                r = drm_syncobj_get_fd(syncobj, (int*)&info->out.handle);
+                drm_syncobj_put(syncobj);
+                return r;
+
+        case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD:
+                fd = get_unused_fd_flags(O_CLOEXEC);
+                if (fd < 0) {
+                        dma_fence_put(fence);
+                        return fd;
+                }
+
+                sync_file = sync_file_create(fence);
+                dma_fence_put(fence);
+                if (!sync_file) {
+                        put_unused_fd(fd);
+                        return -ENOMEM;
+                }
+
+                fd_install(fd, sync_file->file);
+                info->out.handle = fd;
+                return 0;
+
+        default:
+                return -EINVAL;
+        }
+}
+
 /**
  * amdgpu_cs_wait_all_fence - wait on all fences to signal
  *
@@ -1338,6 +1433,9 @@ static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,
 
                 if (r == 0)
                         break;
+
+                if (fence->error)
+                        return fence->error;
         }
 
         memset(wait, 0, sizeof(*wait));
@@ -1383,6 +1481,7 @@ static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,
                         array[i] = fence;
                 } else { /* NULL, the fence has been already signaled */
                         r = 1;
+                        first = i;
                         goto out;
                 }
         }
@@ -1396,8 +1495,11 @@ out:
         memset(wait, 0, sizeof(*wait));
         wait->out.status = (r > 0);
         wait->out.first_signaled = first;
-        /* set return value 0 to indicate success */
-        r = 0;
+
+        if (first < fence_count && array[first])
+                r = array[first]->error;
+        else
+                r = 0;
 
 err_free_fence_array:
         for (i = 0; i < fence_count; i++)
@@ -1418,15 +1520,12 @@ int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
                                 struct drm_file *filp)
 {
         struct amdgpu_device *adev = dev->dev_private;
-        struct amdgpu_fpriv *fpriv = filp->driver_priv;
         union drm_amdgpu_wait_fences *wait = data;
         uint32_t fence_count = wait->in.fence_count;
         struct drm_amdgpu_fence *fences_user;
         struct drm_amdgpu_fence *fences;
         int r;
 
-        if (amdgpu_kms_vram_lost(adev, fpriv))
-                return -ENODEV;
         /* Get the fences from userspace */
         fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence),
                         GFP_KERNEL);
@@ -1462,78 +1561,36 @@ err_free_fences:
  * virtual memory address. Returns allocation structure when found, NULL
  * otherwise.
  */
-struct amdgpu_bo_va_mapping *
-amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
-                       uint64_t addr, struct amdgpu_bo **bo)
+int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
+                           uint64_t addr, struct amdgpu_bo **bo,
+                           struct amdgpu_bo_va_mapping **map)
 {
+        struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
+        struct amdgpu_vm *vm = &fpriv->vm;
         struct amdgpu_bo_va_mapping *mapping;
-        unsigned i;
-
-        if (!parser->bo_list)
-                return NULL;
-
-        addr /= AMDGPU_GPU_PAGE_SIZE;
-
-        for (i = 0; i < parser->bo_list->num_entries; i++) {
-                struct amdgpu_bo_list_entry *lobj;
-
-                lobj = &parser->bo_list->array[i];
-                if (!lobj->bo_va)
-                        continue;
-
-                list_for_each_entry(mapping, &lobj->bo_va->valids, list) {
-                        if (mapping->start > addr ||
-                            addr > mapping->last)
-                                continue;
-
-                        *bo = lobj->bo_va->base.bo;
-                        return mapping;
-                }
-
-                list_for_each_entry(mapping, &lobj->bo_va->invalids, list) {
-                        if (mapping->start > addr ||
-                            addr > mapping->last)
-                                continue;
-
-                        *bo = lobj->bo_va->base.bo;
-                        return mapping;
-                }
-        }
-
-        return NULL;
-}
-
-/**
- * amdgpu_cs_sysvm_access_required - make BOs accessible by the system VM
- *
- * @parser: command submission parser context
- *
- * Helper for UVD/VCE VM emulation, make sure BOs are accessible by the system VM.
- */
-int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser)
-{
-        unsigned i;
         int r;
 
-        if (!parser->bo_list)
-                return 0;
+        addr /= AMDGPU_GPU_PAGE_SIZE;
 
-        for (i = 0; i < parser->bo_list->num_entries; i++) {
-                struct amdgpu_bo *bo = parser->bo_list->array[i].robj;
+        mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
+        if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
+                return -EINVAL;
 
-                r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
-                if (unlikely(r))
-                        return r;
+        *bo = mapping->bo_va->base.bo;
+        *map = mapping;
 
-                if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
-                        continue;
+        /* Double check that the BO is reserved by this CS */
+        if (READ_ONCE((*bo)->tbo.resv->lock.ctx) != &parser->ticket)
+                return -EINVAL;
 
-                bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
-                amdgpu_ttm_placement_from_domain(bo, bo->allowed_domains);
-                r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
-                if (unlikely(r))
+        if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
+                (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+                amdgpu_ttm_placement_from_domain(*bo, (*bo)->allowed_domains);
+                r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, false,
+                                    false);
+                if (r)
                         return r;
         }
 
-        return 0;
+        return amdgpu_ttm_bind(&(*bo)->tbo, &(*bo)->tbo.mem);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index a11e44340b23..c184468e2b2b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -23,13 +23,41 @@
  */
 
 #include <drm/drmP.h>
+#include <drm/drm_auth.h>
 #include "amdgpu.h"
+#include "amdgpu_sched.h"
 
-static int amdgpu_ctx_init(struct amdgpu_device *adev, struct amdgpu_ctx *ctx)
+static int amdgpu_ctx_priority_permit(struct drm_file *filp,
+                                      enum amd_sched_priority priority)
+{
+        /* NORMAL and below are accessible by everyone */
+        if (priority <= AMD_SCHED_PRIORITY_NORMAL)
+                return 0;
+
+        if (capable(CAP_SYS_NICE))
+                return 0;
+
+        if (drm_is_current_master(filp))
+                return 0;
+
+        return -EACCES;
+}
+
+static int amdgpu_ctx_init(struct amdgpu_device *adev,
+                           enum amd_sched_priority priority,
+                           struct drm_file *filp,
+                           struct amdgpu_ctx *ctx)
 {
         unsigned i, j;
         int r;
 
+        if (priority < 0 || priority >= AMD_SCHED_PRIORITY_MAX)
+                return -EINVAL;
+
+        r = amdgpu_ctx_priority_permit(filp, priority);
+        if (r)
+                return r;
+
         memset(ctx, 0, sizeof(*ctx));
         ctx->adev = adev;
         kref_init(&ctx->refcount);
@@ -39,19 +67,24 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, struct amdgpu_ctx *ctx)
         if (!ctx->fences)
                 return -ENOMEM;
 
+        mutex_init(&ctx->lock);
+
         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
                 ctx->rings[i].sequence = 1;
                 ctx->rings[i].fences = &ctx->fences[amdgpu_sched_jobs * i];
         }
 
         ctx->reset_counter = atomic_read(&adev->gpu_reset_counter);
+        ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
+        ctx->init_priority = priority;
+        ctx->override_priority = AMD_SCHED_PRIORITY_UNSET;
 
         /* create context entity for each ring */
         for (i = 0; i < adev->num_rings; i++) {
                 struct amdgpu_ring *ring = adev->rings[i];
                 struct amd_sched_rq *rq;
 
-                rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
+                rq = &ring->sched.sched_rq[priority];
 
                 if (ring == &adev->gfx.kiq.ring)
                         continue;
@@ -96,10 +129,14 @@ static void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
                                       &ctx->rings[i].entity);
 
         amdgpu_queue_mgr_fini(adev, &ctx->queue_mgr);
+
+        mutex_destroy(&ctx->lock);
 }
 
 static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
                             struct amdgpu_fpriv *fpriv,
+                            struct drm_file *filp,
+                            enum amd_sched_priority priority,
                             uint32_t *id)
 {
         struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
@@ -117,8 +154,9 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
                 kfree(ctx);
                 return r;
         }
+
         *id = (uint32_t)r;
-        r = amdgpu_ctx_init(adev, ctx);
+        r = amdgpu_ctx_init(adev, priority, filp, ctx);
         if (r) {
                 idr_remove(&mgr->ctx_handles, *id);
                 *id = 0;
@@ -193,6 +231,7 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 {
         int r;
         uint32_t id;
+        enum amd_sched_priority priority;
 
         union drm_amdgpu_ctx *args = data;
         struct amdgpu_device *adev = dev->dev_private;
@@ -200,10 +239,16 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 
         r = 0;
         id = args->in.ctx_id;
+        priority = amdgpu_to_sched_priority(args->in.priority);
+
+        /* For backwards compatibility reasons, we need to accept
+         * ioctls with garbage in the priority field */
+        if (priority == AMD_SCHED_PRIORITY_INVALID)
+                priority = AMD_SCHED_PRIORITY_NORMAL;
 
         switch (args->in.op) {
         case AMDGPU_CTX_OP_ALLOC_CTX:
-                r = amdgpu_ctx_alloc(adev, fpriv, &id);
+                r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id);
                 args->out.alloc.ctx_id = id;
                 break;
         case AMDGPU_CTX_OP_FREE_CTX:
@@ -246,8 +291,8 @@ int amdgpu_ctx_put(struct amdgpu_ctx *ctx)
         return 0;
 }
 
-uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
-                              struct dma_fence *fence)
+int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
+                              struct dma_fence *fence, uint64_t* handler)
 {
         struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx];
         uint64_t seq = cring->sequence;
@@ -256,12 +301,8 @@ uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
 
         idx = seq & (amdgpu_sched_jobs - 1);
         other = cring->fences[idx];
-        if (other) {
-                signed long r;
-                r = dma_fence_wait_timeout(other, false, MAX_SCHEDULE_TIMEOUT);
-                if (r < 0)
-                        DRM_ERROR("Error (%ld) waiting for fence!\n", r);
-        }
+        if (other)
+                BUG_ON(!dma_fence_is_signaled(other));
 
         dma_fence_get(fence);
 
@@ -271,8 +312,10 @@ uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
         spin_unlock(&ctx->ring_lock);
 
         dma_fence_put(other);
+        if (handler)
+                *handler = seq;
 
-        return seq;
+        return 0;
 }
 
 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
@@ -303,6 +346,51 @@ struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
         return fence;
 }
 
+void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
+                                  enum amd_sched_priority priority)
+{
+        int i;
+        struct amdgpu_device *adev = ctx->adev;
+        struct amd_sched_rq *rq;
+        struct amd_sched_entity *entity;
+        struct amdgpu_ring *ring;
+        enum amd_sched_priority ctx_prio;
+
+        ctx->override_priority = priority;
+
+        ctx_prio = (ctx->override_priority == AMD_SCHED_PRIORITY_UNSET) ?
+                        ctx->init_priority : ctx->override_priority;
+
+        for (i = 0; i < adev->num_rings; i++) {
+                ring = adev->rings[i];
+                entity = &ctx->rings[i].entity;
+                rq = &ring->sched.sched_rq[ctx_prio];
+
+                if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
+                        continue;
+
+                amd_sched_entity_set_rq(entity, rq);
+        }
+}
+
+int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, unsigned ring_id)
+{
+        struct amdgpu_ctx_ring *cring = &ctx->rings[ring_id];
+        unsigned idx = cring->sequence & (amdgpu_sched_jobs - 1);
+        struct dma_fence *other = cring->fences[idx];
+
+        if (other) {
+                signed long r;
+                r = dma_fence_wait_timeout(other, false, MAX_SCHEDULE_TIMEOUT);
+                if (r < 0) {
+                        DRM_ERROR("Error (%ld) waiting for fence!\n", r);
+                        return r;
+                }
+        }
+
+        return 0;
+}
+
 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
 {
         mutex_init(&mgr->lock);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e630d918fefc..2c574374d9b6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -31,6 +31,7 @@
 #include <linux/debugfs.h>
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/amdgpu_drm.h>
 #include <linux/vgaarb.h>
 #include <linux/vga_switcheroo.h>
@@ -56,6 +57,7 @@
 #include "amdgpu_vf_error.h"
 
 #include "amdgpu_amdkfd.h"
+#include "amdgpu_pm.h"
 
 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
@@ -65,6 +67,7 @@ MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev);
+static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev);
 
 static const char *amdgpu_asic_name[] = {
         "TAHITI",
@@ -107,10 +110,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
 {
         uint32_t ret;
 
-        if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
-                BUG_ON(in_interrupt());
+        if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
                 return amdgpu_virt_kiq_rreg(adev, reg);
-        }
 
         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
                 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
@@ -135,10 +136,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
                 adev->last_mm_index = v;
         }
 
-        if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
-                BUG_ON(in_interrupt());
+        if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
                 return amdgpu_virt_kiq_wreg(adev, reg, v);
-        }
 
         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
@@ -402,6 +401,15 @@ void amdgpu_pci_config_reset(struct amdgpu_device *adev)
  */
 static int amdgpu_doorbell_init(struct amdgpu_device *adev)
 {
+        /* No doorbell on SI hardware generation */
+        if (adev->asic_type < CHIP_BONAIRE) {
+                adev->doorbell.base = 0;
+                adev->doorbell.size = 0;
+                adev->doorbell.num_doorbells = 0;
+                adev->doorbell.ptr = NULL;
+                return 0;
+        }
+
         /* doorbell bar mapping */
         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
@@ -539,7 +547,7 @@ int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
 
         if (offset < adev->wb.num_wb) {
                 __set_bit(offset, adev->wb.used);
-                *wb = offset * 8; /* convert to dw offset */
+                *wb = offset << 3; /* convert to dw offset */
                 return 0;
         } else {
                 return -EINVAL;
@@ -557,7 +565,7 @@ int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
 {
         if (wb < adev->wb.num_wb)
-                __clear_bit(wb, adev->wb.used);
+                __clear_bit(wb >> 3, adev->wb.used);
 }
 
 /**
@@ -647,42 +655,96 @@ void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
 }
 
 /*
- * GPU helpers function.
+ * Firmware Reservation functions
  */
 /**
- * amdgpu_need_post - check if the hw need post or not
+ * amdgpu_fw_reserve_vram_fini - free fw reserved vram
  *
  * @adev: amdgpu_device pointer
  *
- * Check if the asic has been initialized (all asics) at driver startup
- * or post is needed if  hw reset is performed.
- * Returns true if need or false if not.
+ * free fw reserved vram if it has been reserved.
  */
-bool amdgpu_need_post(struct amdgpu_device *adev)
+void amdgpu_fw_reserve_vram_fini(struct amdgpu_device *adev)
 {
-        uint32_t reg;
+        amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
+                NULL, &adev->fw_vram_usage.va);
+}
 
-        if (adev->has_hw_reset) {
-                adev->has_hw_reset = false;
-                return true;
-        }
+/**
+ * amdgpu_fw_reserve_vram_init - create bo vram reservation from fw
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from fw.
+ */
+int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev)
+{
+        int r = 0;
+        u64 gpu_addr;
+        u64 vram_size = adev->mc.visible_vram_size;
 
-        /* bios scratch used on CIK+ */
-        if (adev->asic_type >= CHIP_BONAIRE)
-                return amdgpu_atombios_scratch_need_asic_init(adev);
+        adev->fw_vram_usage.va = NULL;
+        adev->fw_vram_usage.reserved_bo = NULL;
 
-        /* check MEM_SIZE for older asics */
-        reg = amdgpu_asic_get_config_memsize(adev);
+        if (adev->fw_vram_usage.size > 0 &&
+                adev->fw_vram_usage.size <= vram_size) {
 
-        if ((reg != 0) && (reg != 0xffffffff))
-                return false;
+                r = amdgpu_bo_create(adev, adev->fw_vram_usage.size,
+                        PAGE_SIZE, true, 0,
+                        AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                        AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, NULL, NULL, 0,
+                        &adev->fw_vram_usage.reserved_bo);
+                if (r)
+                        goto error_create;
 
-        return true;
+                r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
+                if (r)
+                        goto error_reserve;
+                r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
+                        AMDGPU_GEM_DOMAIN_VRAM,
+                        adev->fw_vram_usage.start_offset,
+                        (adev->fw_vram_usage.start_offset +
+                        adev->fw_vram_usage.size), &gpu_addr);
+                if (r)
+                        goto error_pin;
+                r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
+                        &adev->fw_vram_usage.va);
+                if (r)
+                        goto error_kmap;
+
+                amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
+        }
+        return r;
 
+error_kmap:
+        amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
+error_pin:
+        amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
+error_reserve:
+        amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
+error_create:
+        adev->fw_vram_usage.va = NULL;
+        adev->fw_vram_usage.reserved_bo = NULL;
+        return r;
 }
 
-static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
+
+/*
+ * GPU helpers function.
+ */
+/**
+ * amdgpu_need_post - check if the hw need post or not
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Check if the asic has been initialized (all asics) at driver startup
+ * or post is needed if  hw reset is performed.
+ * Returns true if need or false if not.
+ */
+bool amdgpu_need_post(struct amdgpu_device *adev)
 {
+        uint32_t reg;
+
         if (amdgpu_sriov_vf(adev))
                 return false;
 
@@ -705,7 +767,23 @@ static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
                                 return true;
                 }
         }
-        return amdgpu_need_post(adev);
+
+        if (adev->has_hw_reset) {
+                adev->has_hw_reset = false;
+                return true;
+        }
+
+        /* bios scratch used on CIK+ */
+        if (adev->asic_type >= CHIP_BONAIRE)
+                return amdgpu_atombios_scratch_need_asic_init(adev);
+
+        /* check MEM_SIZE for older asics */
+        reg = amdgpu_asic_get_config_memsize(adev);
+
+        if ((reg != 0) && (reg != 0xffffffff))
+                return false;
+
+        return true;
 }
 
 /**
@@ -887,6 +965,20 @@ static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
         return r;
 }
 
+static ssize_t amdgpu_atombios_get_vbios_version(struct device *dev,
+                                                 struct device_attribute *attr,
+                                                 char *buf)
+{
+        struct drm_device *ddev = dev_get_drvdata(dev);
+        struct amdgpu_device *adev = ddev->dev_private;
+        struct atom_context *ctx = adev->mode_info.atom_context;
+
+        return snprintf(buf, PAGE_SIZE, "%s\n", ctx->vbios_version);
+}
+
+static DEVICE_ATTR(vbios_version, 0444, amdgpu_atombios_get_vbios_version,
+                   NULL);
+
 /**
  * amdgpu_atombios_fini - free the driver info and callbacks for atombios
  *
@@ -906,6 +998,7 @@ static void amdgpu_atombios_fini(struct amdgpu_device *adev)
         adev->mode_info.atom_context = NULL;
         kfree(adev->mode_info.atom_card_info);
         adev->mode_info.atom_card_info = NULL;
+        device_remove_file(adev->dev, &dev_attr_vbios_version);
 }
 
 /**
@@ -922,6 +1015,7 @@ static int amdgpu_atombios_init(struct amdgpu_device *adev)
 {
         struct card_info *atom_card_info =
             kzalloc(sizeof(struct card_info), GFP_KERNEL);
+        int ret;
 
         if (!atom_card_info)
                 return -ENOMEM;
@@ -958,6 +1052,13 @@ static int amdgpu_atombios_init(struct amdgpu_device *adev)
                 amdgpu_atombios_scratch_regs_init(adev);
                 amdgpu_atombios_allocate_fb_scratch(adev);
         }
+
+        ret = device_create_file(adev->dev, &dev_attr_vbios_version);
+        if (ret) {
+                DRM_ERROR("Failed to create device file for VBIOS version\n");
+                return ret;
+        }
+
         return 0;
 }
 
@@ -1736,6 +1837,9 @@ static int amdgpu_fini(struct amdgpu_device *adev)
                 adev->ip_blocks[i].status.hw = false;
         }
 
+        if (adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
+                amdgpu_ucode_fini_bo(adev);
+
         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
                 if (!adev->ip_blocks[i].status.sw)
                         continue;
@@ -1757,10 +1861,8 @@ static int amdgpu_fini(struct amdgpu_device *adev)
                 adev->ip_blocks[i].status.late_initialized = false;
         }
 
-        if (amdgpu_sriov_vf(adev)) {
-                amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
+        if (amdgpu_sriov_vf(adev))
                 amdgpu_virt_release_full_gpu(adev, false);
-        }
 
         return 0;
 }
@@ -1848,6 +1950,7 @@ static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
 
         static enum amd_ip_block_type ip_order[] = {
                 AMD_IP_BLOCK_TYPE_SMC,
+                AMD_IP_BLOCK_TYPE_PSP,
                 AMD_IP_BLOCK_TYPE_DCE,
                 AMD_IP_BLOCK_TYPE_GFX,
                 AMD_IP_BLOCK_TYPE_SDMA,
@@ -1933,16 +2036,67 @@ static int amdgpu_resume(struct amdgpu_device *adev)
 
 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
 {
-        if (adev->is_atom_fw) {
-                if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
-                        adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
-        } else {
-                if (amdgpu_atombios_has_gpu_virtualization_table(adev))
-                        adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
+        if (amdgpu_sriov_vf(adev)) {
+                if (adev->is_atom_fw) {
+                        if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
+                                adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
+                } else {
+                        if (amdgpu_atombios_has_gpu_virtualization_table(adev))
+                                adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
+                }
+
+                if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
+                        amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
+        }
+}
+
+bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
+{
+        switch (asic_type) {
+#if defined(CONFIG_DRM_AMD_DC)
+        case CHIP_BONAIRE:
+        case CHIP_HAWAII:
+        case CHIP_KAVERI:
+        case CHIP_CARRIZO:
+        case CHIP_STONEY:
+        case CHIP_POLARIS11:
+        case CHIP_POLARIS10:
+        case CHIP_POLARIS12:
+        case CHIP_TONGA:
+        case CHIP_FIJI:
+#if defined(CONFIG_DRM_AMD_DC_PRE_VEGA)
+                return amdgpu_dc != 0;
+#endif
+        case CHIP_KABINI:
+        case CHIP_MULLINS:
+                return amdgpu_dc > 0;
+        case CHIP_VEGA10:
+#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
+        case CHIP_RAVEN:
+#endif
+                return amdgpu_dc != 0;
+#endif
+        default:
+                return false;
         }
 }
 
 /**
+ * amdgpu_device_has_dc_support - check if dc is supported
+ *
+ * @adev: amdgpu_device_pointer
+ *
+ * Returns true for supported, false for not supported
+ */
+bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
+{
+        if (amdgpu_sriov_vf(adev))
+                return false;
+
+        return amdgpu_device_asic_has_dc_support(adev->asic_type);
+}
+
+/**
  * amdgpu_device_init - initialize the driver
  *
  * @adev: amdgpu_device pointer
@@ -1979,6 +2133,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         adev->vm_manager.vm_pte_num_rings = 0;
         adev->gart.gart_funcs = NULL;
         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
+        bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 
         adev->smc_rreg = &amdgpu_invalid_rreg;
         adev->smc_wreg = &amdgpu_invalid_wreg;
@@ -1995,7 +2150,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
 
-
         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
@@ -2007,8 +2161,10 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         mutex_init(&adev->pm.mutex);
         mutex_init(&adev->gfx.gpu_clock_mutex);
         mutex_init(&adev->srbm_mutex);
+        mutex_init(&adev->gfx.pipe_reserve_mutex);
         mutex_init(&adev->grbm_idx_mutex);
         mutex_init(&adev->mn_lock);
+        mutex_init(&adev->virt.vf_errors.lock);
         hash_init(adev->mn_hash);
 
         amdgpu_check_arguments(adev);
@@ -2051,9 +2207,8 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
 
-        if (adev->asic_type >= CHIP_BONAIRE)
-                /* doorbell bar mapping */
-                amdgpu_doorbell_init(adev);
+        /* doorbell bar mapping */
+        amdgpu_doorbell_init(adev);
 
         /* io port mapping */
         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
@@ -2095,7 +2250,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         r = amdgpu_atombios_init(adev);
         if (r) {
                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
-                amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
+                amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
                 goto failed;
         }
 
@@ -2103,10 +2258,9 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         amdgpu_device_detect_sriov_bios(adev);
 
         /* Post card if necessary */
-        if (amdgpu_vpost_needed(adev)) {
+        if (amdgpu_need_post(adev)) {
                 if (!adev->bios) {
                         dev_err(adev->dev, "no vBIOS found\n");
-                        amdgpu_vf_error_put(AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
                         r = -EINVAL;
                         goto failed;
                 }
@@ -2114,7 +2268,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
                 if (r) {
                         dev_err(adev->dev, "gpu post error!\n");
-                        amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_POST_ERROR, 0, 0);
                         goto failed;
                 }
         } else {
@@ -2126,7 +2279,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
                 r = amdgpu_atomfirmware_get_clock_info(adev);
                 if (r) {
                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
-                        amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
+                        amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
                         goto failed;
                 }
         } else {
@@ -2134,18 +2287,19 @@ int amdgpu_device_init(struct amdgpu_device *adev,
                 r = amdgpu_atombios_get_clock_info(adev);
                 if (r) {
                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
-                        amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
+                        amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
                         goto failed;
                 }
                 /* init i2c buses */
-                amdgpu_atombios_i2c_init(adev);
+                if (!amdgpu_device_has_dc_support(adev))
+                        amdgpu_atombios_i2c_init(adev);
         }
 
         /* Fence driver */
         r = amdgpu_fence_driver_init(adev);
         if (r) {
                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
-                amdgpu_vf_error_put(AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
+                amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
                 goto failed;
         }
 
@@ -2155,7 +2309,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         r = amdgpu_init(adev);
         if (r) {
                 dev_err(adev->dev, "amdgpu_init failed\n");
-                amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
+                amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
                 amdgpu_fini(adev);
                 goto failed;
         }
@@ -2175,7 +2329,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         r = amdgpu_ib_pool_init(adev);
         if (r) {
                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
-                amdgpu_vf_error_put(AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
+                amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
                 goto failed;
         }
 
@@ -2183,8 +2337,15 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         if (r)
                 DRM_ERROR("ib ring test failed (%d).\n", r);
 
+        if (amdgpu_sriov_vf(adev))
+                amdgpu_virt_init_data_exchange(adev);
+
         amdgpu_fbdev_init(adev);
 
+        r = amdgpu_pm_sysfs_init(adev);
+        if (r)
+                DRM_ERROR("registering pm debugfs failed (%d).\n", r);
+
         r = amdgpu_gem_debugfs_init(adev);
         if (r)
                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
@@ -2201,6 +2362,10 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         if (r)
                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
 
+        r = amdgpu_debugfs_vbios_dump_init(adev);
+        if (r)
+                DRM_ERROR("Creating vbios dump debugfs failed (%d).\n", r);
+
         if ((amdgpu_testing & 1)) {
                 if (adev->accel_working)
                         amdgpu_test_moves(adev);
@@ -2220,7 +2385,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         r = amdgpu_late_init(adev);
         if (r) {
                 dev_err(adev->dev, "amdgpu_late_init failed\n");
-                amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
+                amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
                 goto failed;
         }
 
@@ -2252,6 +2417,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
         /* evict vram memory */
         amdgpu_bo_evict_vram(adev);
         amdgpu_ib_pool_fini(adev);
+        amdgpu_fw_reserve_vram_fini(adev);
         amdgpu_fence_driver_fini(adev);
         amdgpu_fbdev_fini(adev);
         r = amdgpu_fini(adev);
@@ -2262,7 +2428,8 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
         adev->accel_working = false;
         cancel_delayed_work_sync(&adev->late_init_work);
         /* free i2c buses */
-        amdgpu_i2c_fini(adev);
+        if (!amdgpu_device_has_dc_support(adev))
+                amdgpu_i2c_fini(adev);
         amdgpu_atombios_fini(adev);
         kfree(adev->bios);
         adev->bios = NULL;
@@ -2276,8 +2443,8 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
         adev->rio_mem = NULL;
         iounmap(adev->rmmio);
         adev->rmmio = NULL;
-        if (adev->asic_type >= CHIP_BONAIRE)
-                amdgpu_doorbell_fini(adev);
+        amdgpu_doorbell_fini(adev);
+        amdgpu_pm_sysfs_fini(adev);
         amdgpu_debugfs_regs_cleanup(adev);
 }
 
@@ -2313,12 +2480,14 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 
         drm_kms_helper_poll_disable(dev);
 
-        /* turn off display hw */
-        drm_modeset_lock_all(dev);
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
+        if (!amdgpu_device_has_dc_support(adev)) {
+                /* turn off display hw */
+                drm_modeset_lock_all(dev);
+                list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                        drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
+                }
+                drm_modeset_unlock_all(dev);
         }
-        drm_modeset_unlock_all(dev);
 
         amdgpu_amdkfd_suspend(adev);
 
@@ -2461,13 +2630,25 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 
         /* blat the mode back in */
         if (fbcon) {
-                drm_helper_resume_force_mode(dev);
-                /* turn on display hw */
-                drm_modeset_lock_all(dev);
-                list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                        drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
+                if (!amdgpu_device_has_dc_support(adev)) {
+                        /* pre DCE11 */
+                        drm_helper_resume_force_mode(dev);
+
+                        /* turn on display hw */
+                        drm_modeset_lock_all(dev);
+                        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                                drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
+                        }
+                        drm_modeset_unlock_all(dev);
+                } else {
+                        /*
+                         * There is no equivalent atomic helper to turn on
+                         * display, so we defined our own function for this,
+                         * once suspend resume is supported by the atomic
+                         * framework this will be reworked
+                         */
+                        amdgpu_dm_display_resume(adev);
                 }
-                drm_modeset_unlock_all(dev);
         }
 
         drm_kms_helper_poll_enable(dev);
@@ -2484,7 +2665,10 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 #ifdef CONFIG_PM
         dev->dev->power.disable_depth++;
 #endif
-        drm_helper_hpd_irq_event(dev);
+        if (!amdgpu_device_has_dc_support(adev))
+                drm_helper_hpd_irq_event(dev);
+        else
+                drm_kms_helper_hotplug_event(dev);
 #ifdef CONFIG_PM
         dev->dev->power.disable_depth--;
 #endif
@@ -2504,6 +2688,9 @@ static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
         int i;
         bool asic_hang = false;
 
+        if (amdgpu_sriov_vf(adev))
+                return true;
+
         for (i = 0; i < adev->num_ip_blocks; i++) {
                 if (!adev->ip_blocks[i].status.valid)
                         continue;
@@ -2546,7 +2733,8 @@ static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
-                    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)) {
+                    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
+                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
                         if (adev->ip_blocks[i].status.hang) {
                                 DRM_INFO("Some block need full reset!\n");
                                 return true;
@@ -2654,7 +2842,7 @@ int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
 
         mutex_lock(&adev->virt.lock_reset);
         atomic_inc(&adev->gpu_reset_counter);
-        adev->gfx.in_reset = true;
+        adev->in_sriov_reset = true;
 
         /* block TTM */
         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
@@ -2765,7 +2953,7 @@ give_up_reset:
                 dev_info(adev->dev, "GPU reset successed!\n");
         }
 
-        adev->gfx.in_reset = false;
+        adev->in_sriov_reset = false;
         mutex_unlock(&adev->virt.lock_reset);
         return r;
 }
@@ -2780,6 +2968,7 @@ give_up_reset:
  */
 int amdgpu_gpu_reset(struct amdgpu_device *adev)
 {
+        struct drm_atomic_state *state = NULL;
         int i, r;
         int resched;
         bool need_full_reset, vram_lost = false;
@@ -2793,6 +2982,9 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
 
         /* block TTM */
         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
+        /* store modesetting */
+        if (amdgpu_device_has_dc_support(adev))
+                state = drm_atomic_helper_suspend(adev->ddev);
 
         /* block scheduler */
         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
@@ -2902,7 +3094,6 @@ out:
                 }
         } else {
                 dev_err(adev->dev, "asic resume failed (%d).\n", r);
-                amdgpu_vf_error_put(AMDGIM_ERROR_VF_ASIC_RESUME_FAIL, 0, r);
                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
                         if (adev->rings[i] && adev->rings[i]->sched.thread) {
                                 kthread_unpark(adev->rings[i]->sched.thread);
@@ -2910,13 +3101,16 @@ out:
                 }
         }
 
-        drm_helper_resume_force_mode(adev->ddev);
+        if (amdgpu_device_has_dc_support(adev)) {
+                r = drm_atomic_helper_resume(adev->ddev, state);
+                amdgpu_dm_display_resume(adev);
+        } else
+                drm_helper_resume_force_mode(adev->ddev);
 
         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
         if (r) {
                 /* bad news, how to tell it to userspace ? */
                 dev_info(adev->dev, "GPU reset failed\n");
-                amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
         }
         else {
                 dev_info(adev->dev, "GPU reset successed!\n");
@@ -3070,9 +3264,9 @@ static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
         pm_pg_lock = (*pos >> 23) & 1;
 
         if (*pos & (1ULL << 62)) {
-                se_bank = (*pos >> 24) & 0x3FF;
-                sh_bank = (*pos >> 34) & 0x3FF;
-                instance_bank = (*pos >> 44) & 0x3FF;
+                se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
+                sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
+                instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
 
                 if (se_bank == 0x3FF)
                         se_bank = 0xFFFFFFFF;
@@ -3146,9 +3340,9 @@ static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
         pm_pg_lock = (*pos >> 23) & 1;
 
         if (*pos & (1ULL << 62)) {
-                se_bank = (*pos >> 24) & 0x3FF;
-                sh_bank = (*pos >> 34) & 0x3FF;
-                instance_bank = (*pos >> 44) & 0x3FF;
+                se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
+                sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
+                instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
 
                 if (se_bank == 0x3FF)
                         se_bank = 0xFFFFFFFF;
@@ -3463,10 +3657,7 @@ static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 
         valuesize = sizeof(values);
         if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
-                r = adev->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, idx, &values[0], &valuesize);
-        else if (adev->pm.funcs && adev->pm.funcs->read_sensor)
-                r = adev->pm.funcs->read_sensor(adev, idx, &values[0],
-                                                &valuesize);
+                r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
         else
                 return -EINVAL;
 
@@ -3499,12 +3690,12 @@ static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
                 return -EINVAL;
 
         /* decode offset */
-        offset = (*pos & 0x7F);
-        se = ((*pos >> 7) & 0xFF);
-        sh = ((*pos >> 15) & 0xFF);
-        cu = ((*pos >> 23) & 0xFF);
-        wave = ((*pos >> 31) & 0xFF);
-        simd = ((*pos >> 37) & 0xFF);
+        offset = (*pos & GENMASK_ULL(6, 0));
+        se = (*pos & GENMASK_ULL(14, 7)) >> 7;
+        sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
+        cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
+        wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
+        simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
 
         /* switch to the specific se/sh/cu */
         mutex_lock(&adev->grbm_idx_mutex);
@@ -3549,14 +3740,14 @@ static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
                 return -EINVAL;
 
         /* decode offset */
-        offset = (*pos & 0xFFF);       /* in dwords */
-        se = ((*pos >> 12) & 0xFF);
-        sh = ((*pos >> 20) & 0xFF);
-        cu = ((*pos >> 28) & 0xFF);
-        wave = ((*pos >> 36) & 0xFF);
-        simd = ((*pos >> 44) & 0xFF);
-        thread = ((*pos >> 52) & 0xFF);
-        bank = ((*pos >> 60) & 1);
+        offset = *pos & GENMASK_ULL(11, 0);
+        se = (*pos & GENMASK_ULL(19, 12)) >> 12;
+        sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
+        cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
+        wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
+        simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
+        thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
+        bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
 
         data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
         if (!data)
@@ -3754,6 +3945,28 @@ int amdgpu_debugfs_init(struct drm_minor *minor)
 {
         return 0;
 }
+
+static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct amdgpu_device *adev = dev->dev_private;
+
+        seq_write(m, adev->bios, adev->bios_size);
+        return 0;
+}
+
+static const struct drm_info_list amdgpu_vbios_dump_list[] = {
+                {"amdgpu_vbios",
+                 amdgpu_debugfs_get_vbios_dump,
+                 0, NULL},
+};
+
+static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
+{
+        return amdgpu_debugfs_add_files(adev,
+                                        amdgpu_vbios_dump_list, 1);
+}
 #else
 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
 {
@@ -3763,5 +3976,9 @@ static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
 {
         return 0;
 }
+static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
+{
+        return 0;
+}
 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index 6ad243293a78..138beb550a58 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -518,7 +518,7 @@ amdgpu_framebuffer_init(struct drm_device *dev,
         return 0;
 }
 
-static struct drm_framebuffer *
+struct drm_framebuffer *
 amdgpu_user_framebuffer_create(struct drm_device *dev,
                                struct drm_file *file_priv,
                                const struct drm_mode_fb_cmd2 *mode_cmd)
@@ -556,7 +556,7 @@ amdgpu_user_framebuffer_create(struct drm_device *dev,
         return &amdgpu_fb->base;
 }
 
-static void amdgpu_output_poll_changed(struct drm_device *dev)
+void amdgpu_output_poll_changed(struct drm_device *dev)
 {
         struct amdgpu_device *adev = dev->dev_private;
         amdgpu_fb_output_poll_changed(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h
new file mode 100644
index 000000000000..3cc0ef0c055e
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __AMDGPU_DISPLAY_H__
+#define __AMDGPU_DISPLAY_H__
+
+struct drm_framebuffer *
+amdgpu_user_framebuffer_create(struct drm_device *dev,
+                                                       struct drm_file *file_priv,
+                                                           const struct drm_mode_fb_cmd2 *mode_cmd);
+
+void amdgpu_output_poll_changed(struct drm_device *dev);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
index 1cb52fd19060..e997ebbe43ea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
@@ -960,8 +960,10 @@ u8 amdgpu_encode_pci_lane_width(u32 lanes)
 }
 
 struct amd_vce_state*
-amdgpu_get_vce_clock_state(struct amdgpu_device *adev, unsigned idx)
+amdgpu_get_vce_clock_state(void *handle, u32 idx)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
         if (idx < adev->pm.dpm.num_of_vce_states)
                 return &adev->pm.dpm.vce_states[idx];
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index 8c96a4caa715..56caaeee6fea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -241,179 +241,125 @@ enum amdgpu_pcie_gen {
         AMDGPU_PCIE_GEN_INVALID = 0xffff
 };
 
-struct amdgpu_dpm_funcs {
-        int (*get_temperature)(struct amdgpu_device *adev);
-        int (*pre_set_power_state)(struct amdgpu_device *adev);
-        int (*set_power_state)(struct amdgpu_device *adev);
-        void (*post_set_power_state)(struct amdgpu_device *adev);
-        void (*display_configuration_changed)(struct amdgpu_device *adev);
-        u32 (*get_sclk)(struct amdgpu_device *adev, bool low);
-        u32 (*get_mclk)(struct amdgpu_device *adev, bool low);
-        void (*print_power_state)(struct amdgpu_device *adev, struct amdgpu_ps *ps);
-        void (*debugfs_print_current_performance_level)(struct amdgpu_device *adev, struct seq_file *m);
-        int (*force_performance_level)(struct amdgpu_device *adev, enum amd_dpm_forced_level level);
-        bool (*vblank_too_short)(struct amdgpu_device *adev);
-        void (*powergate_uvd)(struct amdgpu_device *adev, bool gate);
-        void (*powergate_vce)(struct amdgpu_device *adev, bool gate);
-        void (*enable_bapm)(struct amdgpu_device *adev, bool enable);
-        void (*set_fan_control_mode)(struct amdgpu_device *adev, u32 mode);
-        u32 (*get_fan_control_mode)(struct amdgpu_device *adev);
-        int (*set_fan_speed_percent)(struct amdgpu_device *adev, u32 speed);
-        int (*get_fan_speed_percent)(struct amdgpu_device *adev, u32 *speed);
-        int (*force_clock_level)(struct amdgpu_device *adev, enum pp_clock_type type, uint32_t mask);
-        int (*print_clock_levels)(struct amdgpu_device *adev, enum pp_clock_type type, char *buf);
-        int (*get_sclk_od)(struct amdgpu_device *adev);
-        int (*set_sclk_od)(struct amdgpu_device *adev, uint32_t value);
-        int (*get_mclk_od)(struct amdgpu_device *adev);
-        int (*set_mclk_od)(struct amdgpu_device *adev, uint32_t value);
-        int (*check_state_equal)(struct amdgpu_device *adev,
-                                struct amdgpu_ps *cps,
-                                struct amdgpu_ps *rps,
-                                bool *equal);
-        int (*read_sensor)(struct amdgpu_device *adev, int idx, void *value,
-                           int *size);
-
-        struct amd_vce_state* (*get_vce_clock_state)(struct amdgpu_device *adev, unsigned idx);
-        int (*reset_power_profile_state)(struct amdgpu_device *adev,
-                        struct amd_pp_profile *request);
-        int (*get_power_profile_state)(struct amdgpu_device *adev,
-                        struct amd_pp_profile *query);
-        int (*set_power_profile_state)(struct amdgpu_device *adev,
-                        struct amd_pp_profile *request);
-        int (*switch_power_profile)(struct amdgpu_device *adev,
-                        enum amd_pp_profile_type type);
-};
+#define amdgpu_dpm_pre_set_power_state(adev) \
+                ((adev)->powerplay.pp_funcs->pre_set_power_state((adev)->powerplay.pp_handle))
+
+#define amdgpu_dpm_set_power_state(adev) \
+                ((adev)->powerplay.pp_funcs->set_power_state((adev)->powerplay.pp_handle))
+
+#define amdgpu_dpm_post_set_power_state(adev) \
+                ((adev)->powerplay.pp_funcs->post_set_power_state((adev)->powerplay.pp_handle))
+
+#define amdgpu_dpm_display_configuration_changed(adev) \
+                ((adev)->powerplay.pp_funcs->display_configuration_changed((adev)->powerplay.pp_handle))
 
-#define amdgpu_dpm_pre_set_power_state(adev) (adev)->pm.funcs->pre_set_power_state((adev))
-#define amdgpu_dpm_set_power_state(adev) (adev)->pm.funcs->set_power_state((adev))
-#define amdgpu_dpm_post_set_power_state(adev) (adev)->pm.funcs->post_set_power_state((adev))
-#define amdgpu_dpm_display_configuration_changed(adev) (adev)->pm.funcs->display_configuration_changed((adev))
-#define amdgpu_dpm_print_power_state(adev, ps) (adev)->pm.funcs->print_power_state((adev), (ps))
-#define amdgpu_dpm_vblank_too_short(adev) (adev)->pm.funcs->vblank_too_short((adev))
-#define amdgpu_dpm_enable_bapm(adev, e) (adev)->pm.funcs->enable_bapm((adev), (e))
+#define amdgpu_dpm_print_power_state(adev, ps) \
+                ((adev)->powerplay.pp_funcs->print_power_state((adev)->powerplay.pp_handle, (ps)))
+
+#define amdgpu_dpm_vblank_too_short(adev) \
+                ((adev)->powerplay.pp_funcs->vblank_too_short((adev)->powerplay.pp_handle))
+
+#define amdgpu_dpm_enable_bapm(adev, e) \
+                ((adev)->powerplay.pp_funcs->enable_bapm((adev)->powerplay.pp_handle, (e)))
 
 #define amdgpu_dpm_read_sensor(adev, idx, value, size) \
-        ((adev)->pp_enabled ? \
-                (adev)->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, (idx), (value), (size)) : \
-                (adev)->pm.funcs->read_sensor((adev), (idx), (value), (size)))
+                ((adev)->powerplay.pp_funcs->read_sensor((adev)->powerplay.pp_handle, (idx), (value), (size)))
 
 #define amdgpu_dpm_get_temperature(adev) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_temperature((adev)->powerplay.pp_handle) : \
-              (adev)->pm.funcs->get_temperature((adev)))
+                ((adev)->powerplay.pp_funcs->get_temperature((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_set_fan_control_mode(adev, m) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->set_fan_control_mode((adev)->powerplay.pp_handle, (m)) : \
-              (adev)->pm.funcs->set_fan_control_mode((adev), (m)))
+                ((adev)->powerplay.pp_funcs->set_fan_control_mode((adev)->powerplay.pp_handle, (m)))
 
 #define amdgpu_dpm_get_fan_control_mode(adev) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_fan_control_mode((adev)->powerplay.pp_handle) : \
-              (adev)->pm.funcs->get_fan_control_mode((adev)))
+                ((adev)->powerplay.pp_funcs->get_fan_control_mode((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_set_fan_speed_percent(adev, s) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->set_fan_speed_percent((adev)->powerplay.pp_handle, (s)) : \
-              (adev)->pm.funcs->set_fan_speed_percent((adev), (s)))
+                ((adev)->powerplay.pp_funcs->set_fan_speed_percent((adev)->powerplay.pp_handle, (s)))
 
 #define amdgpu_dpm_get_fan_speed_percent(adev, s) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_fan_speed_percent((adev)->powerplay.pp_handle, (s)) : \
-              (adev)->pm.funcs->get_fan_speed_percent((adev), (s)))
+                ((adev)->powerplay.pp_funcs->get_fan_speed_percent((adev)->powerplay.pp_handle, (s)))
 
 #define amdgpu_dpm_get_fan_speed_rpm(adev, s) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_fan_speed_rpm((adev)->powerplay.pp_handle, (s)) : \
-              -EINVAL)
+                ((adev)->powerplay.pp_funcs->get_fan_speed_rpm)((adev)->powerplay.pp_handle, (s))
 
 #define amdgpu_dpm_get_sclk(adev, l) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_sclk((adev)->powerplay.pp_handle, (l)) : \
-                (adev)->pm.funcs->get_sclk((adev), (l)))
+                ((adev)->powerplay.pp_funcs->get_sclk((adev)->powerplay.pp_handle, (l)))
 
 #define amdgpu_dpm_get_mclk(adev, l)  \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->get_mclk((adev)->powerplay.pp_handle, (l)) : \
-              (adev)->pm.funcs->get_mclk((adev), (l)))
-
+                ((adev)->powerplay.pp_funcs->get_mclk((adev)->powerplay.pp_handle, (l)))
 
 #define amdgpu_dpm_force_performance_level(adev, l) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->force_performance_level((adev)->powerplay.pp_handle, (l)) : \
-              (adev)->pm.funcs->force_performance_level((adev), (l)))
+                ((adev)->powerplay.pp_funcs->force_performance_level((adev)->powerplay.pp_handle, (l)))
 
 #define amdgpu_dpm_powergate_uvd(adev, g) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->powergate_uvd((adev)->powerplay.pp_handle, (g)) : \
-              (adev)->pm.funcs->powergate_uvd((adev), (g)))
+                ((adev)->powerplay.pp_funcs->powergate_uvd((adev)->powerplay.pp_handle, (g)))
 
 #define amdgpu_dpm_powergate_vce(adev, g) \
-        ((adev)->pp_enabled ?                                           \
-              (adev)->powerplay.pp_funcs->powergate_vce((adev)->powerplay.pp_handle, (g)) : \
-              (adev)->pm.funcs->powergate_vce((adev), (g)))
+                ((adev)->powerplay.pp_funcs->powergate_vce((adev)->powerplay.pp_handle, (g)))
 
 #define amdgpu_dpm_get_current_power_state(adev) \
-        (adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle)
+                ((adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_get_pp_num_states(adev, data) \
-        (adev)->powerplay.pp_funcs->get_pp_num_states((adev)->powerplay.pp_handle, data)
+                ((adev)->powerplay.pp_funcs->get_pp_num_states((adev)->powerplay.pp_handle, data))
 
 #define amdgpu_dpm_get_pp_table(adev, table) \
-        (adev)->powerplay.pp_funcs->get_pp_table((adev)->powerplay.pp_handle, table)
+                ((adev)->powerplay.pp_funcs->get_pp_table((adev)->powerplay.pp_handle, table))
 
 #define amdgpu_dpm_set_pp_table(adev, buf, size) \
-        (adev)->powerplay.pp_funcs->set_pp_table((adev)->powerplay.pp_handle, buf, size)
+                ((adev)->powerplay.pp_funcs->set_pp_table((adev)->powerplay.pp_handle, buf, size))
 
 #define amdgpu_dpm_print_clock_levels(adev, type, buf) \
-        (adev)->powerplay.pp_funcs->print_clock_levels((adev)->powerplay.pp_handle, type, buf)
+                ((adev)->powerplay.pp_funcs->print_clock_levels((adev)->powerplay.pp_handle, type, buf))
 
 #define amdgpu_dpm_force_clock_level(adev, type, level) \
-                (adev)->powerplay.pp_funcs->force_clock_level((adev)->powerplay.pp_handle, type, level)
+                ((adev)->powerplay.pp_funcs->force_clock_level((adev)->powerplay.pp_handle, type, level))
 
 #define amdgpu_dpm_get_sclk_od(adev) \
-        (adev)->powerplay.pp_funcs->get_sclk_od((adev)->powerplay.pp_handle)
+                ((adev)->powerplay.pp_funcs->get_sclk_od((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_set_sclk_od(adev, value) \
-        (adev)->powerplay.pp_funcs->set_sclk_od((adev)->powerplay.pp_handle, value)
+                ((adev)->powerplay.pp_funcs->set_sclk_od((adev)->powerplay.pp_handle, value))
 
 #define amdgpu_dpm_get_mclk_od(adev) \
-        ((adev)->powerplay.pp_funcs->get_mclk_od((adev)->powerplay.pp_handle))
+                ((adev)->powerplay.pp_funcs->get_mclk_od((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_set_mclk_od(adev, value) \
-        ((adev)->powerplay.pp_funcs->set_mclk_od((adev)->powerplay.pp_handle, value))
+                ((adev)->powerplay.pp_funcs->set_mclk_od((adev)->powerplay.pp_handle, value))
 
-#define amdgpu_dpm_dispatch_task(adev, event_id, input, output)         \
-        (adev)->powerplay.pp_funcs->dispatch_tasks((adev)->powerplay.pp_handle, (event_id), (input), (output))
+#define amdgpu_dpm_dispatch_task(adev, task_id, input, output)          \
+                ((adev)->powerplay.pp_funcs->dispatch_tasks)((adev)->powerplay.pp_handle, (task_id), (input), (output))
 
-#define amgdpu_dpm_check_state_equal(adev, cps, rps, equal) (adev)->pm.funcs->check_state_equal((adev), (cps),(rps),(equal))
+#define amdgpu_dpm_check_state_equal(adev, cps, rps, equal) \
+                ((adev)->powerplay.pp_funcs->check_state_equal((adev)->powerplay.pp_handle, (cps), (rps), (equal)))
 
 #define amdgpu_dpm_get_vce_clock_state(adev, i)                         \
-        ((adev)->pp_enabled ?                                           \
-         (adev)->powerplay.pp_funcs->get_vce_clock_state((adev)->powerplay.pp_handle, (i)) : \
-         (adev)->pm.funcs->get_vce_clock_state((adev), (i)))
+                ((adev)->powerplay.pp_funcs->get_vce_clock_state((adev)->powerplay.pp_handle, (i)))
 
-#define amdgpu_dpm_get_performance_level(adev) \
-        ((adev)->pp_enabled ?                                           \
-        (adev)->powerplay.pp_funcs->get_performance_level((adev)->powerplay.pp_handle) : \
-        (adev)->pm.dpm.forced_level)
+#define amdgpu_dpm_get_performance_level(adev)                          \
+                ((adev)->powerplay.pp_funcs->get_performance_level((adev)->powerplay.pp_handle))
 
 #define amdgpu_dpm_reset_power_profile_state(adev, request) \
-        ((adev)->powerplay.pp_funcs->reset_power_profile_state(\
+                ((adev)->powerplay.pp_funcs->reset_power_profile_state(\
                         (adev)->powerplay.pp_handle, request))
 
 #define amdgpu_dpm_get_power_profile_state(adev, query) \
-        ((adev)->powerplay.pp_funcs->get_power_profile_state(\
+                ((adev)->powerplay.pp_funcs->get_power_profile_state(\
                         (adev)->powerplay.pp_handle, query))
 
 #define amdgpu_dpm_set_power_profile_state(adev, request) \
-        ((adev)->powerplay.pp_funcs->set_power_profile_state(\
+                ((adev)->powerplay.pp_funcs->set_power_profile_state(\
                         (adev)->powerplay.pp_handle, request))
 
 #define amdgpu_dpm_switch_power_profile(adev, type) \
-        ((adev)->powerplay.pp_funcs->switch_power_profile(\
+                ((adev)->powerplay.pp_funcs->switch_power_profile(\
                         (adev)->powerplay.pp_handle, type))
 
+#define amdgpu_dpm_set_clockgating_by_smu(adev, msg_id) \
+                ((adev)->powerplay.pp_funcs->set_clockgating_by_smu(\
+                        (adev)->powerplay.pp_handle, msg_id))
+
 struct amdgpu_dpm {
         struct amdgpu_ps        *ps;
         /* number of valid power states */
@@ -485,10 +431,9 @@ struct amdgpu_pm {
         struct amdgpu_dpm       dpm;
         const struct firmware   *fw;    /* SMC firmware */
         uint32_t                fw_version;
-        const struct amdgpu_dpm_funcs *funcs;
         uint32_t                pcie_gen_mask;
         uint32_t                pcie_mlw_mask;
-        struct amd_pp_display_configuration pm_display_cfg;/* set by DAL */
+        struct amd_pp_display_configuration pm_display_cfg;/* set by dc */
 };
 
 #define R600_SSTU_DFLT                               0
@@ -551,6 +496,6 @@ u16 amdgpu_get_pcie_lane_support(struct amdgpu_device *adev,
 u8 amdgpu_encode_pci_lane_width(u32 lanes);
 
 struct amd_vce_state*
-amdgpu_get_vce_clock_state(struct amdgpu_device *adev, unsigned idx);
+amdgpu_get_vce_clock_state(void *handle, u32 idx);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 0f16986ec5bc..ec96bb1f9eaf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -69,9 +69,13 @@
  * - 3.17.0 - Add AMDGPU_NUM_VRAM_CPU_PAGE_FAULTS.
  * - 3.18.0 - Export gpu always on cu bitmap
  * - 3.19.0 - Add support for UVD MJPEG decode
+ * - 3.20.0 - Add support for local BOs
+ * - 3.21.0 - Add DRM_AMDGPU_FENCE_TO_HANDLE ioctl
+ * - 3.22.0 - Add DRM_AMDGPU_SCHED ioctl
+ * - 3.23.0 - Add query for VRAM lost counter
  */
 #define KMS_DRIVER_MAJOR        3
-#define KMS_DRIVER_MINOR        19
+#define KMS_DRIVER_MINOR        23
 #define KMS_DRIVER_PATCHLEVEL   0
 
 int amdgpu_vram_limit = 0;
@@ -91,7 +95,7 @@ int amdgpu_dpm = -1;
 int amdgpu_fw_load_type = -1;
 int amdgpu_aspm = -1;
 int amdgpu_runtime_pm = -1;
-unsigned amdgpu_ip_block_mask = 0xffffffff;
+uint amdgpu_ip_block_mask = 0xffffffff;
 int amdgpu_bapm = -1;
 int amdgpu_deep_color = 0;
 int amdgpu_vm_size = -1;
@@ -102,18 +106,20 @@ int amdgpu_vm_debug = 0;
 int amdgpu_vram_page_split = 512;
 int amdgpu_vm_update_mode = -1;
 int amdgpu_exp_hw_support = 0;
+int amdgpu_dc = -1;
+int amdgpu_dc_log = 0;
 int amdgpu_sched_jobs = 32;
 int amdgpu_sched_hw_submission = 2;
 int amdgpu_no_evict = 0;
 int amdgpu_direct_gma_size = 0;
-unsigned amdgpu_pcie_gen_cap = 0;
-unsigned amdgpu_pcie_lane_cap = 0;
-unsigned amdgpu_cg_mask = 0xffffffff;
-unsigned amdgpu_pg_mask = 0xffffffff;
-unsigned amdgpu_sdma_phase_quantum = 32;
+uint amdgpu_pcie_gen_cap = 0;
+uint amdgpu_pcie_lane_cap = 0;
+uint amdgpu_cg_mask = 0xffffffff;
+uint amdgpu_pg_mask = 0xffffffff;
+uint amdgpu_sdma_phase_quantum = 32;
 char *amdgpu_disable_cu = NULL;
 char *amdgpu_virtual_display = NULL;
-unsigned amdgpu_pp_feature_mask = 0xffffffff;
+uint amdgpu_pp_feature_mask = 0xffffffff;
 int amdgpu_ngg = 0;
 int amdgpu_prim_buf_per_se = 0;
 int amdgpu_pos_buf_per_se = 0;
@@ -121,6 +127,7 @@ int amdgpu_cntl_sb_buf_per_se = 0;
 int amdgpu_param_buf_per_se = 0;
 int amdgpu_job_hang_limit = 0;
 int amdgpu_lbpw = -1;
+int amdgpu_compute_multipipe = -1;
 
 MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
 module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -206,6 +213,12 @@ module_param_named(vram_page_split, amdgpu_vram_page_split, int, 0444);
 MODULE_PARM_DESC(exp_hw_support, "experimental hw support (1 = enable, 0 = disable (default))");
 module_param_named(exp_hw_support, amdgpu_exp_hw_support, int, 0444);
 
+MODULE_PARM_DESC(dc, "Display Core driver (1 = enable, 0 = disable, -1 = auto (default))");
+module_param_named(dc, amdgpu_dc, int, 0444);
+
+MODULE_PARM_DESC(dc, "Display Core Log Level (0 = minimal (default), 1 = chatty");
+module_param_named(dc_log, amdgpu_dc_log, int, 0444);
+
 MODULE_PARM_DESC(sched_jobs, "the max number of jobs supported in the sw queue (default 32)");
 module_param_named(sched_jobs, amdgpu_sched_jobs, int, 0444);
 
@@ -264,6 +277,9 @@ module_param_named(job_hang_limit, amdgpu_job_hang_limit, int ,0444);
 MODULE_PARM_DESC(lbpw, "Load Balancing Per Watt (LBPW) support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(lbpw, amdgpu_lbpw, int, 0444);
 
+MODULE_PARM_DESC(compute_multipipe, "Force compute queues to be spread across pipes (1 = enable, 0 = disable, -1 = auto)");
+module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444);
+
 #ifdef CONFIG_DRM_AMDGPU_SI
 
 #if defined(CONFIG_DRM_RADEON) || defined(CONFIG_DRM_RADEON_MODULE)
@@ -510,15 +526,15 @@ static const struct pci_device_id pciidlist[] = {
         {0x1002, 0x6997, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
         {0x1002, 0x699F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
         /* Vega 10 */
-        {0x1002, 0x6860, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
-        {0x1002, 0x6861, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
-        {0x1002, 0x6862, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
-        {0x1002, 0x6863, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
-        {0x1002, 0x6864, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
-        {0x1002, 0x6867, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
-        {0x1002, 0x6868, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
-        {0x1002, 0x686c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
-        {0x1002, 0x687f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
+        {0x1002, 0x6860, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10},
+        {0x1002, 0x6861, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10},
+        {0x1002, 0x6862, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10},
+        {0x1002, 0x6863, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10},
+        {0x1002, 0x6864, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10},
+        {0x1002, 0x6867, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10},
+        {0x1002, 0x6868, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10},
+        {0x1002, 0x686c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10},
+        {0x1002, 0x687f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10},
         /* Raven */
         {0x1002, 0x15dd, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RAVEN|AMD_IS_APU|AMD_EXP_HW_SUPPORT},
 
@@ -608,6 +624,8 @@ amdgpu_pci_remove(struct pci_dev *pdev)
 
         drm_dev_unregister(dev);
         drm_dev_unref(dev);
+        pci_disable_device(pdev);
+        pci_set_drvdata(pdev, NULL);
 }
 
 static void
@@ -852,6 +870,7 @@ static struct drm_driver kms_driver = {
         .gem_prime_import_sg_table = amdgpu_gem_prime_import_sg_table,
         .gem_prime_vmap = amdgpu_gem_prime_vmap,
         .gem_prime_vunmap = amdgpu_gem_prime_vunmap,
+        .gem_prime_mmap = amdgpu_gem_prime_mmap,
 
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index 9afa9c097e1f..90fa8e8bc6fb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -42,11 +42,6 @@
    this contains a helper + a amdgpu fb
    the helper contains a pointer to amdgpu framebuffer baseclass.
 */
-struct amdgpu_fbdev {
-        struct drm_fb_helper helper;
-        struct amdgpu_framebuffer rfb;
-        struct amdgpu_device *adev;
-};
 
 static int
 amdgpufb_open(struct fb_info *info, int user)
@@ -149,7 +144,7 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
                                        AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
                                        AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
                                        AMDGPU_GEM_CREATE_VRAM_CLEARED,
-                                       true, &gobj);
+                                       true, NULL, &gobj);
         if (ret) {
                 pr_err("failed to allocate framebuffer (%d)\n", aligned_size);
                 return -ENOMEM;
@@ -303,10 +298,10 @@ static int amdgpu_fbdev_destroy(struct drm_device *dev, struct amdgpu_fbdev *rfb
         if (rfb->obj) {
                 amdgpufb_destroy_pinned_object(rfb->obj);
                 rfb->obj = NULL;
+                drm_framebuffer_unregister_private(&rfb->base);
+                drm_framebuffer_cleanup(&rfb->base);
         }
         drm_fb_helper_fini(&rfbdev->helper);
-        drm_framebuffer_unregister_private(&rfb->base);
-        drm_framebuffer_cleanup(&rfb->base);
 
         return 0;
 }
@@ -353,7 +348,8 @@ int amdgpu_fbdev_init(struct amdgpu_device *adev)
         drm_fb_helper_single_add_all_connectors(&rfbdev->helper);
 
         /* disable all the possible outputs/crtcs before entering KMS mode */
-        drm_helper_disable_unused_functions(adev->ddev);
+        if (!amdgpu_device_has_dc_support(adev))
+                drm_helper_disable_unused_functions(adev->ddev);
 
         drm_fb_helper_initial_config(&rfbdev->helper, bpp_sel);
         return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 303b5e099a98..2fa95aef74d5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -169,6 +169,32 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f)
 }
 
 /**
+ * amdgpu_fence_emit_polling - emit a fence on the requeste ring
+ *
+ * @ring: ring the fence is associated with
+ * @s: resulting sequence number
+ *
+ * Emits a fence command on the requested ring (all asics).
+ * Used For polling fence.
+ * Returns 0 on success, -ENOMEM on failure.
+ */
+int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s)
+{
+        uint32_t seq;
+
+        if (!s)
+                return -EINVAL;
+
+        seq = ++ring->fence_drv.sync_seq;
+        amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
+                               seq, AMDGPU_FENCE_FLAG_INT);
+
+        *s = seq;
+
+        return 0;
+}
+
+/**
  * amdgpu_fence_schedule_fallback - schedule fallback check
  *
  * @ring: pointer to struct amdgpu_ring
@@ -242,9 +268,10 @@ void amdgpu_fence_process(struct amdgpu_ring *ring)
  *
  * Checks for fence activity.
  */
-static void amdgpu_fence_fallback(unsigned long arg)
+static void amdgpu_fence_fallback(struct timer_list *t)
 {
-        struct amdgpu_ring *ring = (void *)arg;
+        struct amdgpu_ring *ring = from_timer(ring, t,
+                                              fence_drv.fallback_timer);
 
         amdgpu_fence_process(ring);
 }
@@ -282,6 +309,30 @@ int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
 }
 
 /**
+ * amdgpu_fence_wait_polling - busy wait for givn sequence number
+ *
+ * @ring: ring index the fence is associated with
+ * @wait_seq: sequence number to wait
+ * @timeout: the timeout for waiting in usecs
+ *
+ * Wait for all fences on the requested ring to signal (all asics).
+ * Returns left time if no timeout, 0 or minus if timeout.
+ */
+signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
+                                      uint32_t wait_seq,
+                                      signed long timeout)
+{
+        uint32_t seq;
+
+        do {
+                seq = amdgpu_fence_read(ring);
+                udelay(5);
+                timeout -= 5;
+        } while ((int32_t)(wait_seq - seq) > 0 && timeout > 0);
+
+        return timeout > 0 ? timeout : 0;
+}
+/**
  * amdgpu_fence_count_emitted - get the count of emitted fences
  *
  * @ring: ring the fence is associated with
@@ -372,8 +423,7 @@ int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
         atomic_set(&ring->fence_drv.last_seq, 0);
         ring->fence_drv.initialized = false;
 
-        setup_timer(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback,
-                    (unsigned long)ring);
+        timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);
 
         ring->fence_drv.num_fences_mask = num_hw_submission * 2 - 1;
         spin_lock_init(&ring->fence_drv.lock);
@@ -641,6 +691,19 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data)
                            atomic_read(&ring->fence_drv.last_seq));
                 seq_printf(m, "Last emitted        0x%08x\n",
                            ring->fence_drv.sync_seq);
+
+                if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
+                        continue;
+
+                /* set in CP_VMID_PREEMPT and preemption occurred */
+                seq_printf(m, "Last preempted      0x%08x\n",
+                           le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
+                /* set in CP_VMID_RESET and reset occurred */
+                seq_printf(m, "Last reset          0x%08x\n",
+                           le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));
+                /* Both preemption and reset occurred */
+                seq_printf(m, "Last both           0x%08x\n",
+                           le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));
         }
         return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index f4370081f6e6..fe818501c520 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -332,12 +332,13 @@ int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset,
                 adev->gart.pages[p] = pagelist[i];
 #endif
 
-        if (adev->gart.ptr) {
-                r = amdgpu_gart_map(adev, offset, pages, dma_addr, flags,
-                            adev->gart.ptr);
-                if (r)
-                        return r;
-        }
+        if (!adev->gart.ptr)
+                return 0;
+
+        r = amdgpu_gart_map(adev, offset, pages, dma_addr, flags,
+                    adev->gart.ptr);
+        if (r)
+                return r;
 
         mb();
         amdgpu_gart_flush_gpu_tlb(adev, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 6149a47fe63d..e87eedcc0da9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -44,11 +44,12 @@ void amdgpu_gem_object_free(struct drm_gem_object *gobj)
 }
 
 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
-                                int alignment, u32 initial_domain,
-                                u64 flags, bool kernel,
-                                struct drm_gem_object **obj)
+                             int alignment, u32 initial_domain,
+                             u64 flags, bool kernel,
+                             struct reservation_object *resv,
+                             struct drm_gem_object **obj)
 {
-        struct amdgpu_bo *robj;
+        struct amdgpu_bo *bo;
         int r;
 
         *obj = NULL;
@@ -59,9 +60,14 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 
 retry:
         r = amdgpu_bo_create(adev, size, alignment, kernel, initial_domain,
-                             flags, NULL, NULL, 0, &robj);
+                             flags, NULL, resv, 0, &bo);
         if (r) {
                 if (r != -ERESTARTSYS) {
+                        if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
+                                flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+                                goto retry;
+                        }
+
                         if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
                                 initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
                                 goto retry;
@@ -71,7 +77,7 @@ retry:
                 }
                 return r;
         }
-        *obj = &robj->gem_base;
+        *obj = &bo->gem_base;
 
         return 0;
 }
@@ -112,7 +118,17 @@ int amdgpu_gem_object_open(struct drm_gem_object *obj,
         struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
         struct amdgpu_vm *vm = &fpriv->vm;
         struct amdgpu_bo_va *bo_va;
+        struct mm_struct *mm;
         int r;
+
+        mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
+        if (mm && mm != current->mm)
+                return -EPERM;
+
+        if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
+            abo->tbo.resv != vm->root.base.bo->tbo.resv)
+                return -EPERM;
+
         r = amdgpu_bo_reserve(abo, false);
         if (r)
                 return r;
@@ -127,35 +143,6 @@ int amdgpu_gem_object_open(struct drm_gem_object *obj,
         return 0;
 }
 
-static int amdgpu_gem_vm_check(void *param, struct amdgpu_bo *bo)
-{
-        /* if anything is swapped out don't swap it in here,
-           just abort and wait for the next CS */
-        if (!amdgpu_bo_gpu_accessible(bo))
-                return -ERESTARTSYS;
-
-        if (bo->shadow && !amdgpu_bo_gpu_accessible(bo->shadow))
-                return -ERESTARTSYS;
-
-        return 0;
-}
-
-static bool amdgpu_gem_vm_ready(struct amdgpu_device *adev,
-                                struct amdgpu_vm *vm,
-                                struct list_head *list)
-{
-        struct ttm_validate_buffer *entry;
-
-        list_for_each_entry(entry, list, head) {
-                struct amdgpu_bo *bo =
-                        container_of(entry->bo, struct amdgpu_bo, tbo);
-                if (amdgpu_gem_vm_check(NULL, bo))
-                        return false;
-        }
-
-        return !amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_gem_vm_check, NULL);
-}
-
 void amdgpu_gem_object_close(struct drm_gem_object *obj,
                              struct drm_file *file_priv)
 {
@@ -165,13 +152,14 @@ void amdgpu_gem_object_close(struct drm_gem_object *obj,
         struct amdgpu_vm *vm = &fpriv->vm;
 
         struct amdgpu_bo_list_entry vm_pd;
-        struct list_head list;
+        struct list_head list, duplicates;
         struct ttm_validate_buffer tv;
         struct ww_acquire_ctx ticket;
         struct amdgpu_bo_va *bo_va;
         int r;
 
         INIT_LIST_HEAD(&list);
+        INIT_LIST_HEAD(&duplicates);
 
         tv.bo = &bo->tbo;
         tv.shared = true;
@@ -179,7 +167,7 @@ void amdgpu_gem_object_close(struct drm_gem_object *obj,
 
         amdgpu_vm_get_pd_bo(vm, &list, &vm_pd);
 
-        r = ttm_eu_reserve_buffers(&ticket, &list, false, NULL);
+        r = ttm_eu_reserve_buffers(&ticket, &list, false, &duplicates);
         if (r) {
                 dev_err(adev->dev, "leaking bo va because "
                         "we fail to reserve bo (%d)\n", r);
@@ -189,7 +177,7 @@ void amdgpu_gem_object_close(struct drm_gem_object *obj,
         if (bo_va && --bo_va->ref_count == 0) {
                 amdgpu_vm_bo_rmv(adev, bo_va);
 
-                if (amdgpu_gem_vm_ready(adev, vm, &list)) {
+                if (amdgpu_vm_ready(vm)) {
                         struct dma_fence *fence = NULL;
 
                         r = amdgpu_vm_clear_freed(adev, vm, &fence);
@@ -214,18 +202,24 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
                             struct drm_file *filp)
 {
         struct amdgpu_device *adev = dev->dev_private;
+        struct amdgpu_fpriv *fpriv = filp->driver_priv;
+        struct amdgpu_vm *vm = &fpriv->vm;
         union drm_amdgpu_gem_create *args = data;
+        uint64_t flags = args->in.domain_flags;
         uint64_t size = args->in.bo_size;
+        struct reservation_object *resv = NULL;
         struct drm_gem_object *gobj;
         uint32_t handle;
-        bool kernel = false;
         int r;
 
         /* reject invalid gem flags */
-        if (args->in.domain_flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-                                      AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
-                                      AMDGPU_GEM_CREATE_CPU_GTT_USWC |
-                                      AMDGPU_GEM_CREATE_VRAM_CLEARED))
+        if (flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                      AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
+                      AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+                      AMDGPU_GEM_CREATE_VRAM_CLEARED |
+                      AMDGPU_GEM_CREATE_VM_ALWAYS_VALID |
+                      AMDGPU_GEM_CREATE_EXPLICIT_SYNC))
+
                 return -EINVAL;
 
         /* reject invalid gem domains */
@@ -240,7 +234,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
         /* create a gem object to contain this object in */
         if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
             AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
-                kernel = true;
+                flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
                 if (args->in.domains == AMDGPU_GEM_DOMAIN_GDS)
                         size = size << AMDGPU_GDS_SHIFT;
                 else if (args->in.domains == AMDGPU_GEM_DOMAIN_GWS)
@@ -252,10 +246,25 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
         }
         size = roundup(size, PAGE_SIZE);
 
+        if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
+                r = amdgpu_bo_reserve(vm->root.base.bo, false);
+                if (r)
+                        return r;
+
+                resv = vm->root.base.bo->tbo.resv;
+        }
+
         r = amdgpu_gem_object_create(adev, size, args->in.alignment,
                                      (u32)(0xffffffff & args->in.domains),
-                                     args->in.domain_flags,
-                                     kernel, &gobj);
+                                     flags, false, resv, &gobj);
+        if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
+                if (!r) {
+                        struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
+
+                        abo->parent = amdgpu_bo_ref(vm->root.base.bo);
+                }
+                amdgpu_bo_unreserve(vm->root.base.bo);
+        }
         if (r)
                 return r;
 
@@ -297,9 +306,8 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
         }
 
         /* create a gem object to contain this object in */
-        r = amdgpu_gem_object_create(adev, args->size, 0,
-                                     AMDGPU_GEM_DOMAIN_CPU, 0,
-                                     0, &gobj);
+        r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
+                                     0, 0, NULL, &gobj);
         if (r)
                 return r;
 
@@ -317,12 +325,10 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
         }
 
         if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
-                down_read(&current->mm->mmap_sem);
-
                 r = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
                                                  bo->tbo.ttm->pages);
                 if (r)
-                        goto unlock_mmap_sem;
+                        goto release_object;
 
                 r = amdgpu_bo_reserve(bo, true);
                 if (r)
@@ -333,8 +339,6 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
                 amdgpu_bo_unreserve(bo);
                 if (r)
                         goto free_pages;
-
-                up_read(&current->mm->mmap_sem);
         }
 
         r = drm_gem_handle_create(filp, gobj, &handle);
@@ -347,10 +351,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
         return 0;
 
 free_pages:
-        release_pages(bo->tbo.ttm->pages, bo->tbo.ttm->num_pages, false);
-
-unlock_mmap_sem:
-        up_read(&current->mm->mmap_sem);
+        release_pages(bo->tbo.ttm->pages, bo->tbo.ttm->num_pages);
 
 release_object:
         drm_gem_object_put_unlocked(gobj);
@@ -511,10 +512,10 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
                                     struct list_head *list,
                                     uint32_t operation)
 {
-        int r = -ERESTARTSYS;
+        int r;
 
-        if (!amdgpu_gem_vm_ready(adev, vm, list))
-                goto error;
+        if (!amdgpu_vm_ready(vm))
+                return;
 
         r = amdgpu_vm_update_directories(adev, vm);
         if (r)
@@ -551,15 +552,14 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
         struct amdgpu_bo_list_entry vm_pd;
         struct ttm_validate_buffer tv;
         struct ww_acquire_ctx ticket;
-        struct list_head list;
+        struct list_head list, duplicates;
         uint64_t va_flags;
         int r = 0;
 
         if (args->va_address < AMDGPU_VA_RESERVED_SIZE) {
                 dev_err(&dev->pdev->dev,
-                        "va_address 0x%lX is in reserved area 0x%X\n",
-                        (unsigned long)args->va_address,
-                        AMDGPU_VA_RESERVED_SIZE);
+                        "va_address 0x%LX is in reserved area 0x%LX\n",
+                        args->va_address, AMDGPU_VA_RESERVED_SIZE);
                 return -EINVAL;
         }
 
@@ -580,13 +580,9 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
                         args->operation);
                 return -EINVAL;
         }
-        if ((args->operation == AMDGPU_VA_OP_MAP) ||
-            (args->operation == AMDGPU_VA_OP_REPLACE)) {
-                if (amdgpu_kms_vram_lost(adev, fpriv))
-                        return -ENODEV;
-        }
 
         INIT_LIST_HEAD(&list);
+        INIT_LIST_HEAD(&duplicates);
         if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
             !(args->flags & AMDGPU_VM_PAGE_PRT)) {
                 gobj = drm_gem_object_lookup(filp, args->handle);
@@ -603,7 +599,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 
         amdgpu_vm_get_pd_bo(&fpriv->vm, &list, &vm_pd);
 
-        r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL);
+        r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates);
         if (r)
                 goto error_unref;
 
@@ -669,6 +665,7 @@ error_unref:
 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
                         struct drm_file *filp)
 {
+        struct amdgpu_device *adev = dev->dev_private;
         struct drm_amdgpu_gem_op *args = data;
         struct drm_gem_object *gobj;
         struct amdgpu_bo *robj;
@@ -716,6 +713,9 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
                 if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
                         robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
 
+                if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
+                        amdgpu_vm_bo_invalidate(adev, robj, true);
+
                 amdgpu_bo_unreserve(robj);
                 break;
         default:
@@ -745,8 +745,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
         r = amdgpu_gem_object_create(adev, args->size, 0,
                                      AMDGPU_GEM_DOMAIN_VRAM,
                                      AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
-                                     ttm_bo_type_device,
-                                     &gobj);
+                                     false, NULL, &gobj);
         if (r)
                 return -ENOMEM;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 4f6c68fc1dd9..ef043361009f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -109,9 +109,26 @@ void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_s
         }
 }
 
+static bool amdgpu_gfx_is_multipipe_capable(struct amdgpu_device *adev)
+{
+        if (amdgpu_compute_multipipe != -1) {
+                DRM_INFO("amdgpu: forcing compute pipe policy %d\n",
+                         amdgpu_compute_multipipe);
+                return amdgpu_compute_multipipe == 1;
+        }
+
+        /* FIXME: spreading the queues across pipes causes perf regressions
+         * on POLARIS11 compute workloads */
+        if (adev->asic_type == CHIP_POLARIS11)
+                return false;
+
+        return adev->gfx.mec.num_mec > 1;
+}
+
 void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
 {
         int i, queue, pipe, mec;
+        bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev);
 
         /* policy for amdgpu compute queue ownership */
         for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
@@ -125,8 +142,7 @@ void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
                 if (mec >= adev->gfx.mec.num_mec)
                         break;
 
-                /* FIXME: spreading the queues across pipes causes perf regressions */
-                if (0) {
+                if (multipipe_policy) {
                         /* policy: amdgpu owns the first two queues of the first MEC */
                         if (mec == 0 && queue < 2)
                                 set_bit(i, adev->gfx.mec.queue_bitmap);
@@ -185,7 +201,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,
         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
         int r = 0;
 
-        mutex_init(&kiq->ring_mutex);
+        spin_lock_init(&kiq->ring_lock);
 
         r = amdgpu_wb_get(adev, &adev->virt.reg_val_offs);
         if (r)
@@ -260,8 +276,13 @@ int amdgpu_gfx_compute_mqd_sw_init(struct amdgpu_device *adev,
         /* create MQD for KIQ */
         ring = &adev->gfx.kiq.ring;
         if (!ring->mqd_obj) {
+                /* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must
+                 * otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD
+                 * deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for
+                 * KIQ MQD no matter SRIOV or Bare-metal
+                 */
                 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
-                                            AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
+                                            AMDGPU_GEM_DOMAIN_VRAM, &ring->mqd_obj,
                                             &ring->mqd_gpu_addr, &ring->mqd_ptr);
                 if (r) {
                         dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index 0d15eb7d31d7..00e0ce10862f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -71,12 +71,6 @@ static int amdgpu_gtt_mgr_fini(struct ttm_mem_type_manager *man)
 {
         struct amdgpu_gtt_mgr *mgr = man->priv;
 
-        spin_lock(&mgr->lock);
-        if (!drm_mm_clean(&mgr->mm)) {
-                spin_unlock(&mgr->lock);
-                return -EBUSY;
-        }
-
         drm_mm_takedown(&mgr->mm);
         spin_unlock(&mgr->lock);
         kfree(mgr);
@@ -169,7 +163,8 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
         int r;
 
         spin_lock(&mgr->lock);
-        if (atomic64_read(&mgr->available) < mem->num_pages) {
+        if ((&tbo->mem == mem || tbo->mem.mem_type != TTM_PL_TT) &&
+            atomic64_read(&mgr->available) < mem->num_pages) {
                 spin_unlock(&mgr->lock);
                 return 0;
         }
@@ -244,8 +239,9 @@ static void amdgpu_gtt_mgr_del(struct ttm_mem_type_manager *man,
 uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man)
 {
         struct amdgpu_gtt_mgr *mgr = man->priv;
+        s64 result = man->size - atomic64_read(&mgr->available);
 
-        return (u64)(man->size - atomic64_read(&mgr->available)) * PAGE_SIZE;
+        return (result > 0 ? result : 0) * PAGE_SIZE;
 }
 
 /**
@@ -265,7 +261,7 @@ static void amdgpu_gtt_mgr_debug(struct ttm_mem_type_manager *man,
         drm_mm_print(&mgr->mm, printer);
         spin_unlock(&mgr->lock);
 
-        drm_printf(printer, "man size:%llu pages, gtt available:%llu pages, usage:%lluMB\n",
+        drm_printf(printer, "man size:%llu pages, gtt available:%lld pages, usage:%lluMB\n",
                    man->size, (u64)atomic64_read(&mgr->available),
                    amdgpu_gtt_mgr_usage(man) >> 20);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
index 3ab4c65ecc8b..f5f27e4f0f7f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
@@ -169,6 +169,12 @@ restart_ih:
         while (adev->irq.ih.rptr != wptr) {
                 u32 ring_index = adev->irq.ih.rptr >> 2;
 
+                /* Prescreening of high-frequency interrupts */
+                if (!amdgpu_ih_prescreen_iv(adev)) {
+                        adev->irq.ih.rptr &= adev->irq.ih.ptr_mask;
+                        continue;
+                }
+
                 /* Before dispatching irq to IP blocks, send it to amdkfd */
                 amdgpu_amdkfd_interrupt(adev,
                                 (const void *) &adev->irq.ih.ring[ring_index]);
@@ -190,3 +196,79 @@ restart_ih:
 
         return IRQ_HANDLED;
 }
+
+/**
+ * amdgpu_ih_add_fault - Add a page fault record
+ *
+ * @adev: amdgpu device pointer
+ * @key: 64-bit encoding of PASID and address
+ *
+ * This should be called when a retry page fault interrupt is
+ * received. If this is a new page fault, it will be added to a hash
+ * table. The return value indicates whether this is a new fault, or
+ * a fault that was already known and is already being handled.
+ *
+ * If there are too many pending page faults, this will fail. Retry
+ * interrupts should be ignored in this case until there is enough
+ * free space.
+ *
+ * Returns 0 if the fault was added, 1 if the fault was already known,
+ * -ENOSPC if there are too many pending faults.
+ */
+int amdgpu_ih_add_fault(struct amdgpu_device *adev, u64 key)
+{
+        unsigned long flags;
+        int r = -ENOSPC;
+
+        if (WARN_ON_ONCE(!adev->irq.ih.faults))
+                /* Should be allocated in <IP>_ih_sw_init on GPUs that
+                 * support retry faults and require retry filtering.
+                 */
+                return r;
+
+        spin_lock_irqsave(&adev->irq.ih.faults->lock, flags);
+
+        /* Only let the hash table fill up to 50% for best performance */
+        if (adev->irq.ih.faults->count >= (1 << (AMDGPU_PAGEFAULT_HASH_BITS-1)))
+                goto unlock_out;
+
+        r = chash_table_copy_in(&adev->irq.ih.faults->hash, key, NULL);
+        if (!r)
+                adev->irq.ih.faults->count++;
+
+        /* chash_table_copy_in should never fail unless we're losing count */
+        WARN_ON_ONCE(r < 0);
+
+unlock_out:
+        spin_unlock_irqrestore(&adev->irq.ih.faults->lock, flags);
+        return r;
+}
+
+/**
+ * amdgpu_ih_clear_fault - Remove a page fault record
+ *
+ * @adev: amdgpu device pointer
+ * @key: 64-bit encoding of PASID and address
+ *
+ * This should be called when a page fault has been handled. Any
+ * future interrupt with this key will be processed as a new
+ * page fault.
+ */
+void amdgpu_ih_clear_fault(struct amdgpu_device *adev, u64 key)
+{
+        unsigned long flags;
+        int r;
+
+        if (!adev->irq.ih.faults)
+                return;
+
+        spin_lock_irqsave(&adev->irq.ih.faults->lock, flags);
+
+        r = chash_table_remove(&adev->irq.ih.faults->hash, key, NULL);
+        if (!WARN_ON_ONCE(r < 0)) {
+                adev->irq.ih.faults->count--;
+                WARN_ON_ONCE(adev->irq.ih.faults->count < 0);
+        }
+
+        spin_unlock_irqrestore(&adev->irq.ih.faults->lock, flags);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
index 3de8e74e5b3a..ada89358e220 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
@@ -24,6 +24,8 @@
 #ifndef __AMDGPU_IH_H__
 #define __AMDGPU_IH_H__
 
+#include <linux/chash.h>
+
 struct amdgpu_device;
  /*
   * vega10+ IH clients
@@ -69,6 +71,13 @@ enum amdgpu_ih_clientid
 
 #define AMDGPU_IH_CLIENTID_LEGACY 0
 
+#define AMDGPU_PAGEFAULT_HASH_BITS 8
+struct amdgpu_retryfault_hashtable {
+        DECLARE_CHASH_TABLE(hash, AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
+        spinlock_t      lock;
+        int             count;
+};
+
 /*
  * R6xx+ IH ring
  */
@@ -87,6 +96,7 @@ struct amdgpu_ih_ring {
         bool                    use_doorbell;
         bool                    use_bus_addr;
         dma_addr_t              rb_dma_addr; /* only used when use_bus_addr = true */
+        struct amdgpu_retryfault_hashtable *faults;
 };
 
 #define AMDGPU_IH_SRC_DATA_MAX_SIZE_DW 4
@@ -109,5 +119,7 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, unsigned ring_size,
                         bool use_bus_addr);
 void amdgpu_ih_ring_fini(struct amdgpu_device *adev);
 int amdgpu_ih_process(struct amdgpu_device *adev);
+int amdgpu_ih_add_fault(struct amdgpu_device *adev, u64 key);
+void amdgpu_ih_clear_fault(struct amdgpu_device *adev, u64 key);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 538e5f27d120..47c5ce9807db 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -37,6 +37,10 @@
 
 #include <linux/pm_runtime.h>
 
+#ifdef CONFIG_DRM_AMD_DC
+#include "amdgpu_dm_irq.h"
+#endif
+
 #define AMDGPU_WAIT_IDLE_TIMEOUT 200
 
 /*
@@ -221,15 +225,6 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
 
         spin_lock_init(&adev->irq.lock);
 
-        if (!adev->enable_virtual_display)
-                /* Disable vblank irqs aggressively for power-saving */
-                adev->ddev->vblank_disable_immediate = true;
-
-        r = drm_vblank_init(adev->ddev, adev->mode_info.num_crtc);
-        if (r) {
-                return r;
-        }
-
         /* enable msi */
         adev->irq.msi_enabled = false;
 
@@ -241,7 +236,21 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
                 }
         }
 
-        INIT_WORK(&adev->hotplug_work, amdgpu_hotplug_work_func);
+        if (!amdgpu_device_has_dc_support(adev)) {
+                if (!adev->enable_virtual_display)
+                        /* Disable vblank irqs aggressively for power-saving */
+                        /* XXX: can this be enabled for DC? */
+                        adev->ddev->vblank_disable_immediate = true;
+
+                r = drm_vblank_init(adev->ddev, adev->mode_info.num_crtc);
+                if (r)
+                        return r;
+
+                /* pre DCE11 */
+                INIT_WORK(&adev->hotplug_work,
+                                amdgpu_hotplug_work_func);
+        }
+
         INIT_WORK(&adev->reset_work, amdgpu_irq_reset_work_func);
 
         adev->irq.installed = true;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 4510627ae83e..0cfc68db575b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -65,6 +65,7 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
         amdgpu_sync_create(&(*job)->sync);
         amdgpu_sync_create(&(*job)->dep_sync);
         amdgpu_sync_create(&(*job)->sched_sync);
+        (*job)->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
 
         return 0;
 }
@@ -103,6 +104,7 @@ static void amdgpu_job_free_cb(struct amd_sched_job *s_job)
 {
         struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base);
 
+        amdgpu_ring_priority_put(job->ring, amd_sched_get_job_priority(s_job));
         dma_fence_put(job->fence);
         amdgpu_sync_free(&job->sync);
         amdgpu_sync_free(&job->dep_sync);
@@ -139,6 +141,8 @@ int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
         job->fence_ctx = entity->fence_context;
         *f = dma_fence_get(&job->base.s_fence->finished);
         amdgpu_job_free_resources(job);
+        amdgpu_ring_priority_get(job->ring,
+                                 amd_sched_get_job_priority(&job->base));
         amd_sched_entity_push_job(&job->base);
 
         return 0;
@@ -177,8 +181,8 @@ static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
 static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
 {
         struct dma_fence *fence = NULL;
+        struct amdgpu_device *adev;
         struct amdgpu_job *job;
-        struct amdgpu_fpriv *fpriv = NULL;
         int r;
 
         if (!sched_job) {
@@ -186,23 +190,25 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
                 return NULL;
         }
         job = to_amdgpu_job(sched_job);
+        adev = job->adev;
 
         BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
 
         trace_amdgpu_sched_run_job(job);
-        if (job->vm)
-                fpriv = container_of(job->vm, struct amdgpu_fpriv, vm);
         /* skip ib schedule when vram is lost */
-        if (fpriv && amdgpu_kms_vram_lost(job->adev, fpriv))
+        if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) {
+                dma_fence_set_error(&job->base.s_fence->finished, -ECANCELED);
                 DRM_ERROR("Skip scheduling IBs!\n");
-        else {
-                r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job, &fence);
+        } else {
+                r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job,
+                                       &fence);
                 if (r)
                         DRM_ERROR("Error scheduling IBs (%d)\n", r);
         }
         /* if gpu reset, hw fence will be replaced here */
         dma_fence_put(job->fence);
         job->fence = dma_fence_get(fence);
+
         amdgpu_job_free_resources(job);
         return fence;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index e16229000a98..720139e182a3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -28,6 +28,7 @@
 #include <drm/drmP.h>
 #include "amdgpu.h"
 #include <drm/amdgpu_drm.h>
+#include "amdgpu_sched.h"
 #include "amdgpu_uvd.h"
 #include "amdgpu_vce.h"
 
@@ -269,7 +270,6 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info,
 static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 {
         struct amdgpu_device *adev = dev->dev_private;
-        struct amdgpu_fpriv *fpriv = filp->driver_priv;
         struct drm_amdgpu_info *info = data;
         struct amdgpu_mode_info *minfo = &adev->mode_info;
         void __user *out = (void __user *)(uintptr_t)info->return_pointer;
@@ -282,8 +282,6 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 
         if (!info->return_size || !info->return_pointer)
                 return -EINVAL;
-        if (amdgpu_kms_vram_lost(adev, fpriv))
-                return -ENODEV;
 
         switch (info->query) {
         case AMDGPU_INFO_ACCEL_WORKING:
@@ -765,6 +763,9 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
                 }
                 return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0;
         }
+        case AMDGPU_INFO_VRAM_LOST_COUNTER:
+                ui32 = atomic_read(&adev->vram_lost_counter);
+                return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0;
         default:
                 DRM_DEBUG_KMS("Invalid request %d\n", info->query);
                 return -EINVAL;
@@ -791,12 +792,6 @@ void amdgpu_driver_lastclose_kms(struct drm_device *dev)
         vga_switcheroo_process_delayed_switch();
 }
 
-bool amdgpu_kms_vram_lost(struct amdgpu_device *adev,
-                          struct amdgpu_fpriv *fpriv)
-{
-        return fpriv->vram_lost_counter != atomic_read(&adev->vram_lost_counter);
-}
-
 /**
  * amdgpu_driver_open_kms - drm callback for open
  *
@@ -825,7 +820,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
         }
 
         r = amdgpu_vm_init(adev, &fpriv->vm,
-                           AMDGPU_VM_CONTEXT_GFX);
+                           AMDGPU_VM_CONTEXT_GFX, 0);
         if (r) {
                 kfree(fpriv);
                 goto out_suspend;
@@ -841,8 +836,11 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 
         if (amdgpu_sriov_vf(adev)) {
                 r = amdgpu_map_static_csa(adev, &fpriv->vm, &fpriv->csa_va);
-                if (r)
+                if (r) {
+                        amdgpu_vm_fini(adev, &fpriv->vm);
+                        kfree(fpriv);
                         goto out_suspend;
+                }
         }
 
         mutex_init(&fpriv->bo_list_lock);
@@ -850,7 +848,6 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 
         amdgpu_ctx_mgr_init(&fpriv->ctx_mgr);
 
-        fpriv->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
         file_priv->driver_priv = fpriv;
 
 out_suspend:
@@ -1020,7 +1017,9 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
         DRM_IOCTL_DEF_DRV(AMDGPU_GEM_CREATE, amdgpu_gem_create_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
         DRM_IOCTL_DEF_DRV(AMDGPU_CTX, amdgpu_ctx_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
         DRM_IOCTL_DEF_DRV(AMDGPU_VM, amdgpu_vm_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(AMDGPU_SCHED, amdgpu_sched_ioctl, DRM_MASTER),
         DRM_IOCTL_DEF_DRV(AMDGPU_BO_LIST, amdgpu_bo_list_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(AMDGPU_FENCE_TO_HANDLE, amdgpu_cs_fence_to_handle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
         /* KMS */
         DRM_IOCTL_DEF_DRV(AMDGPU_GEM_MMAP, amdgpu_gem_mmap_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
         DRM_IOCTL_DEF_DRV(AMDGPU_GEM_WAIT_IDLE, amdgpu_gem_wait_idle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
@@ -1031,7 +1030,7 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
         DRM_IOCTL_DEF_DRV(AMDGPU_GEM_METADATA, amdgpu_gem_metadata_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
         DRM_IOCTL_DEF_DRV(AMDGPU_GEM_VA, amdgpu_gem_va_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
         DRM_IOCTL_DEF_DRV(AMDGPU_GEM_OP, amdgpu_gem_op_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(AMDGPU_GEM_USERPTR, amdgpu_gem_userptr_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(AMDGPU_GEM_USERPTR, amdgpu_gem_userptr_ioctl, DRM_AUTH|DRM_RENDER_ALLOW)
 };
 const int amdgpu_max_kms_ioctl = ARRAY_SIZE(amdgpu_ioctls_kms);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
index 3b0f2ec6eec7..bd67f4cb8e6c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
@@ -50,8 +50,10 @@ struct amdgpu_mn {
         struct hlist_node       node;
 
         /* objects protected by lock */
-        struct mutex            lock;
+        struct rw_semaphore     lock;
         struct rb_root_cached   objects;
+        struct mutex            read_lock;
+        atomic_t                recursion;
 };
 
 struct amdgpu_mn_node {
@@ -74,7 +76,7 @@ static void amdgpu_mn_destroy(struct work_struct *work)
         struct amdgpu_bo *bo, *next_bo;
 
         mutex_lock(&adev->mn_lock);
-        mutex_lock(&rmn->lock);
+        down_write(&rmn->lock);
         hash_del(&rmn->node);
         rbtree_postorder_for_each_entry_safe(node, next_node,
                                              &rmn->objects.rb_root, it.rb) {
@@ -84,7 +86,7 @@ static void amdgpu_mn_destroy(struct work_struct *work)
                 }
                 kfree(node);
         }
-        mutex_unlock(&rmn->lock);
+        up_write(&rmn->lock);
         mutex_unlock(&adev->mn_lock);
         mmu_notifier_unregister_no_release(&rmn->mn, rmn->mm);
         kfree(rmn);
@@ -106,6 +108,53 @@ static void amdgpu_mn_release(struct mmu_notifier *mn,
         schedule_work(&rmn->work);
 }
 
+
+/**
+ * amdgpu_mn_lock - take the write side lock for this mn
+ */
+void amdgpu_mn_lock(struct amdgpu_mn *mn)
+{
+        if (mn)
+                down_write(&mn->lock);
+}
+
+/**
+ * amdgpu_mn_unlock - drop the write side lock for this mn
+ */
+void amdgpu_mn_unlock(struct amdgpu_mn *mn)
+{
+        if (mn)
+                up_write(&mn->lock);
+}
+
+/**
+ * amdgpu_mn_read_lock - take the rmn read lock
+ *
+ * @rmn: our notifier
+ *
+ * Take the rmn read side lock.
+ */
+static void amdgpu_mn_read_lock(struct amdgpu_mn *rmn)
+{
+        mutex_lock(&rmn->read_lock);
+        if (atomic_inc_return(&rmn->recursion) == 1)
+                down_read_non_owner(&rmn->lock);
+        mutex_unlock(&rmn->read_lock);
+}
+
+/**
+ * amdgpu_mn_read_unlock - drop the rmn read lock
+ *
+ * @rmn: our notifier
+ *
+ * Drop the rmn read side lock.
+ */
+static void amdgpu_mn_read_unlock(struct amdgpu_mn *rmn)
+{
+        if (atomic_dec_return(&rmn->recursion) == 0)
+                up_read_non_owner(&rmn->lock);
+}
+
 /**
  * amdgpu_mn_invalidate_node - unmap all BOs of a node
  *
@@ -126,23 +175,12 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
                 if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, start, end))
                         continue;
 
-                r = amdgpu_bo_reserve(bo, true);
-                if (r) {
-                        DRM_ERROR("(%ld) failed to reserve user bo\n", r);
-                        continue;
-                }
-
                 r = reservation_object_wait_timeout_rcu(bo->tbo.resv,
                         true, false, MAX_SCHEDULE_TIMEOUT);
                 if (r <= 0)
                         DRM_ERROR("(%ld) failed to wait for user bo\n", r);
 
-                amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
-                r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
-                if (r)
-                        DRM_ERROR("(%ld) failed to validate user bo\n", r);
-
-                amdgpu_bo_unreserve(bo);
+                amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
         }
 }
 
@@ -168,7 +206,7 @@ static void amdgpu_mn_invalidate_range_start(struct mmu_notifier *mn,
         /* notification is exclusive, but interval is inclusive */
         end -= 1;
 
-        mutex_lock(&rmn->lock);
+        amdgpu_mn_read_lock(rmn);
 
         it = interval_tree_iter_first(&rmn->objects, start, end);
         while (it) {
@@ -179,13 +217,32 @@ static void amdgpu_mn_invalidate_range_start(struct mmu_notifier *mn,
 
                 amdgpu_mn_invalidate_node(node, start, end);
         }
+}
 
-        mutex_unlock(&rmn->lock);
+/**
+ * amdgpu_mn_invalidate_range_end - callback to notify about mm change
+ *
+ * @mn: our notifier
+ * @mn: the mm this callback is about
+ * @start: start of updated range
+ * @end: end of updated range
+ *
+ * Release the lock again to allow new command submissions.
+ */
+static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
+                                           struct mm_struct *mm,
+                                           unsigned long start,
+                                           unsigned long end)
+{
+        struct amdgpu_mn *rmn = container_of(mn, struct amdgpu_mn, mn);
+
+        amdgpu_mn_read_unlock(rmn);
 }
 
 static const struct mmu_notifier_ops amdgpu_mn_ops = {
         .release = amdgpu_mn_release,
         .invalidate_range_start = amdgpu_mn_invalidate_range_start,
+        .invalidate_range_end = amdgpu_mn_invalidate_range_end,
 };
 
 /**
@@ -195,7 +252,7 @@ static const struct mmu_notifier_ops amdgpu_mn_ops = {
  *
  * Creates a notifier context for current->mm.
  */
-static struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
+struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
 {
         struct mm_struct *mm = current->mm;
         struct amdgpu_mn *rmn;
@@ -220,8 +277,10 @@ static struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
         rmn->adev = adev;
         rmn->mm = mm;
         rmn->mn.ops = &amdgpu_mn_ops;
-        mutex_init(&rmn->lock);
+        init_rwsem(&rmn->lock);
         rmn->objects = RB_ROOT_CACHED;
+        mutex_init(&rmn->read_lock);
+        atomic_set(&rmn->recursion, 0);
 
         r = __mmu_notifier_register(&rmn->mn, mm);
         if (r)
@@ -267,7 +326,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
 
         INIT_LIST_HEAD(&bos);
 
-        mutex_lock(&rmn->lock);
+        down_write(&rmn->lock);
 
         while ((it = interval_tree_iter_first(&rmn->objects, addr, end))) {
                 kfree(node);
@@ -281,7 +340,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
         if (!node) {
                 node = kmalloc(sizeof(struct amdgpu_mn_node), GFP_KERNEL);
                 if (!node) {
-                        mutex_unlock(&rmn->lock);
+                        up_write(&rmn->lock);
                         return -ENOMEM;
                 }
         }
@@ -296,7 +355,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
 
         interval_tree_insert(&node->it, &rmn->objects);
 
-        mutex_unlock(&rmn->lock);
+        up_write(&rmn->lock);
 
         return 0;
 }
@@ -322,7 +381,7 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
                 return;
         }
 
-        mutex_lock(&rmn->lock);
+        down_write(&rmn->lock);
 
         /* save the next list entry for later */
         head = bo->mn_list.next;
@@ -337,6 +396,7 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
                 kfree(node);
         }
 
-        mutex_unlock(&rmn->lock);
+        up_write(&rmn->lock);
         mutex_unlock(&adev->mn_lock);
 }
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
new file mode 100644
index 000000000000..d0095a3793b8
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Christian König
+ */
+#ifndef __AMDGPU_MN_H__
+#define __AMDGPU_MN_H__
+
+/*
+ * MMU Notifier
+ */
+struct amdgpu_mn;
+
+#if defined(CONFIG_MMU_NOTIFIER)
+void amdgpu_mn_lock(struct amdgpu_mn *mn);
+void amdgpu_mn_unlock(struct amdgpu_mn *mn);
+struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev);
+int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
+void amdgpu_mn_unregister(struct amdgpu_bo *bo);
+#else
+static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {}
+static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {}
+static inline struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
+{
+        return NULL;
+}
+static inline int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
+{
+        return -ENODEV;
+}
+static inline void amdgpu_mn_unregister(struct amdgpu_bo *bo) {}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index 2af2678ddaf6..ffde1e9666e8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -38,11 +38,15 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_plane_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
 #include <linux/hrtimer.h>
 #include "amdgpu_irq.h"
 
+#include <drm/drm_dp_mst_helper.h>
+#include "modules/inc/mod_freesync.h"
+
 struct amdgpu_bo;
 struct amdgpu_device;
 struct amdgpu_encoder;
@@ -53,9 +57,13 @@ struct amdgpu_hpd;
 #define to_amdgpu_connector(x) container_of(x, struct amdgpu_connector, base)
 #define to_amdgpu_encoder(x) container_of(x, struct amdgpu_encoder, base)
 #define to_amdgpu_framebuffer(x) container_of(x, struct amdgpu_framebuffer, base)
+#define to_amdgpu_plane(x)      container_of(x, struct amdgpu_plane, base)
+
+#define to_dm_plane_state(x)    container_of(x, struct dm_plane_state, base);
 
 #define AMDGPU_MAX_HPD_PINS 6
 #define AMDGPU_MAX_CRTCS 6
+#define AMDGPU_MAX_PLANES 6
 #define AMDGPU_MAX_AFMT_BLOCKS 9
 
 enum amdgpu_rmx_type {
@@ -292,6 +300,30 @@ struct amdgpu_display_funcs {
                               uint16_t connector_object_id,
                               struct amdgpu_hpd *hpd,
                               struct amdgpu_router *router);
+        /* it is used to enter or exit into free sync mode */
+        int (*notify_freesync)(struct drm_device *dev, void *data,
+                               struct drm_file *filp);
+        /* it is used to allow enablement of freesync mode */
+        int (*set_freesync_property)(struct drm_connector *connector,
+                                     struct drm_property *property,
+                                     uint64_t val);
+
+
+};
+
+struct amdgpu_framebuffer {
+        struct drm_framebuffer base;
+        struct drm_gem_object *obj;
+
+        /* caching for later use */
+        uint64_t address;
+};
+
+struct amdgpu_fbdev {
+        struct drm_fb_helper helper;
+        struct amdgpu_framebuffer rfb;
+        struct list_head fbdev_list;
+        struct amdgpu_device *adev;
 };
 
 struct amdgpu_mode_info {
@@ -299,6 +331,7 @@ struct amdgpu_mode_info {
         struct card_info *atom_card_info;
         bool mode_config_initialized;
         struct amdgpu_crtc *crtcs[AMDGPU_MAX_CRTCS];
+        struct amdgpu_plane *planes[AMDGPU_MAX_PLANES];
         struct amdgpu_afmt *afmt[AMDGPU_MAX_AFMT_BLOCKS];
         /* DVI-I properties */
         struct drm_property *coherent_mode_property;
@@ -328,6 +361,7 @@ struct amdgpu_mode_info {
         int                     num_dig; /* number of dig blocks */
         int                     disp_priority;
         const struct amdgpu_display_funcs *funcs;
+        const enum drm_plane_type *plane_type;
 };
 
 #define AMDGPU_MAX_BL_LEVEL 0xFF
@@ -400,6 +434,14 @@ struct amdgpu_crtc {
         /* for virtual dce */
         struct hrtimer vblank_timer;
         enum amdgpu_interrupt_state vsync_timer_enabled;
+
+        int otg_inst;
+        struct drm_pending_vblank_event *event;
+};
+
+struct amdgpu_plane {
+        struct drm_plane base;
+        enum drm_plane_type plane_type;
 };
 
 struct amdgpu_encoder_atom_dig {
@@ -489,6 +531,19 @@ enum amdgpu_connector_dither {
         AMDGPU_FMT_DITHER_ENABLE = 1,
 };
 
+struct amdgpu_dm_dp_aux {
+        struct drm_dp_aux aux;
+        struct ddc_service *ddc_service;
+};
+
+struct amdgpu_i2c_adapter {
+        struct i2c_adapter base;
+
+        struct ddc_service *ddc_service;
+};
+
+#define TO_DM_AUX(x) container_of((x), struct amdgpu_dm_dp_aux, aux)
+
 struct amdgpu_connector {
         struct drm_connector base;
         uint32_t connector_id;
@@ -500,6 +555,14 @@ struct amdgpu_connector {
         /* we need to mind the EDID between detect
            and get modes due to analog/digital/tvencoder */
         struct edid *edid;
+        /* number of modes generated from EDID at 'dc_sink' */
+        int num_modes;
+        /* The 'old' sink - before an HPD.
+         * The 'current' sink is in dc_link->sink. */
+        struct dc_sink *dc_sink;
+        struct dc_link *dc_link;
+        struct dc_sink *dc_em_sink;
+        const struct dc_stream *stream;
         void *con_priv;
         bool dac_load_detect;
         bool detected_by_load; /* if the connection status was determined by load */
@@ -510,11 +573,39 @@ struct amdgpu_connector {
         enum amdgpu_connector_audio audio;
         enum amdgpu_connector_dither dither;
         unsigned pixelclock_for_modeset;
+
+        struct drm_dp_mst_topology_mgr mst_mgr;
+        struct amdgpu_dm_dp_aux dm_dp_aux;
+        struct drm_dp_mst_port *port;
+        struct amdgpu_connector *mst_port;
+        struct amdgpu_encoder *mst_encoder;
+        struct semaphore mst_sem;
+
+        /* TODO see if we can merge with ddc_bus or make a dm_connector */
+        struct amdgpu_i2c_adapter *i2c;
+
+        /* Monitor range limits */
+        int min_vfreq ;
+        int max_vfreq ;
+        int pixel_clock_mhz;
+
+        /*freesync caps*/
+        struct mod_freesync_caps caps;
+
+        struct mutex hpd_lock;
+
 };
 
-struct amdgpu_framebuffer {
-        struct drm_framebuffer base;
-        struct drm_gem_object *obj;
+/* TODO: start to use this struct and remove same field from base one */
+struct amdgpu_mst_connector {
+        struct amdgpu_connector base;
+
+        struct drm_dp_mst_topology_mgr mst_mgr;
+        struct amdgpu_dm_dp_aux dm_dp_aux;
+        struct drm_dp_mst_port *port;
+        struct amdgpu_connector *mst_port;
+        bool is_mst_connector;
+        struct amdgpu_encoder *mst_encoder;
 };
 
 #define ENCODER_MODE_IS_DP(em) (((em) == ATOM_ENCODER_MODE_DP) || \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 9e495da0bb03..ea25164e7f4b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -40,9 +40,7 @@
 static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 {
         struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
-        struct amdgpu_bo *bo;
-
-        bo = container_of(tbo, struct amdgpu_bo, tbo);
+        struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
 
         amdgpu_bo_kunmap(bo);
 
@@ -64,11 +62,12 @@ bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
         return false;
 }
 
-static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
-                                      struct ttm_placement *placement,
-                                      struct ttm_place *places,
-                                      u32 domain, u64 flags)
+void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 {
+        struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
+        struct ttm_placement *placement = &abo->placement;
+        struct ttm_place *places = abo->placements;
+        u64 flags = abo->flags;
         u32 c = 0;
 
         if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
@@ -151,27 +150,6 @@ static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
         placement->busy_placement = places;
 }
 
-void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
-{
-        struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
-
-        amdgpu_ttm_placement_init(adev, &abo->placement, abo->placements,
-                                  domain, abo->flags);
-}
-
-static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
-                                        struct ttm_placement *placement)
-{
-        BUG_ON(placement->num_placement > (AMDGPU_GEM_DOMAIN_MAX + 1));
-
-        memcpy(bo->placements, placement->placement,
-               placement->num_placement * sizeof(struct ttm_place));
-        bo->placement.num_placement = placement->num_placement;
-        bo->placement.num_busy_placement = placement->num_busy_placement;
-        bo->placement.placement = bo->placements;
-        bo->placement.busy_placement = bo->placements;
-}
-
 /**
  * amdgpu_bo_create_reserved - create reserved BO for kernel use
  *
@@ -303,14 +281,13 @@ void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
                 *cpu_addr = NULL;
 }
 
-int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
-                                unsigned long size, int byte_align,
-                                bool kernel, u32 domain, u64 flags,
-                                struct sg_table *sg,
-                                struct ttm_placement *placement,
-                                struct reservation_object *resv,
-                                uint64_t init_value,
-                                struct amdgpu_bo **bo_ptr)
+static int amdgpu_bo_do_create(struct amdgpu_device *adev,
+                               unsigned long size, int byte_align,
+                               bool kernel, u32 domain, u64 flags,
+                               struct sg_table *sg,
+                               struct reservation_object *resv,
+                               uint64_t init_value,
+                               struct amdgpu_bo **bo_ptr)
 {
         struct amdgpu_bo *bo;
         enum ttm_bo_type type;
@@ -384,13 +361,17 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
                 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
 #endif
 
-        amdgpu_fill_placement_to_bo(bo, placement);
-        /* Kernel allocation are uninterruptible */
+        bo->tbo.bdev = &adev->mman.bdev;
+        amdgpu_ttm_placement_from_domain(bo, domain);
 
         initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
+        /* Kernel allocation are uninterruptible */
         r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
                                  &bo->placement, page_align, !kernel, NULL,
                                  acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
+        if (unlikely(r != 0))
+                return r;
+
         bytes_moved = atomic64_read(&adev->num_bytes_moved) -
                       initial_bytes_moved;
         if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
@@ -400,9 +381,6 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
         else
                 amdgpu_cs_report_moved_bytes(adev, bytes_moved, 0);
 
-        if (unlikely(r != 0))
-                return r;
-
         if (kernel)
                 bo->tbo.priority = 1;
 
@@ -442,27 +420,17 @@ static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
                                    unsigned long size, int byte_align,
                                    struct amdgpu_bo *bo)
 {
-        struct ttm_placement placement = {0};
-        struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
         int r;
 
         if (bo->shadow)
                 return 0;
 
-        memset(&placements, 0, sizeof(placements));
-        amdgpu_ttm_placement_init(adev, &placement, placements,
-                                  AMDGPU_GEM_DOMAIN_GTT,
-                                  AMDGPU_GEM_CREATE_CPU_GTT_USWC |
-                                  AMDGPU_GEM_CREATE_SHADOW);
-
-        r = amdgpu_bo_create_restricted(adev, size, byte_align, true,
-                                        AMDGPU_GEM_DOMAIN_GTT,
-                                        AMDGPU_GEM_CREATE_CPU_GTT_USWC |
-                                        AMDGPU_GEM_CREATE_SHADOW,
-                                        NULL, &placement,
-                                        bo->tbo.resv,
-                                        0,
-                                        &bo->shadow);
+        r = amdgpu_bo_do_create(adev, size, byte_align, true,
+                                AMDGPU_GEM_DOMAIN_GTT,
+                                AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+                                AMDGPU_GEM_CREATE_SHADOW,
+                                NULL, bo->tbo.resv, 0,
+                                &bo->shadow);
         if (!r) {
                 bo->shadow->parent = amdgpu_bo_ref(bo);
                 mutex_lock(&adev->shadow_list_lock);
@@ -484,18 +452,11 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
                      uint64_t init_value,
                      struct amdgpu_bo **bo_ptr)
 {
-        struct ttm_placement placement = {0};
-        struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
         uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW;
         int r;
 
-        memset(&placements, 0, sizeof(placements));
-        amdgpu_ttm_placement_init(adev, &placement, placements,
-                                  domain, parent_flags);
-
-        r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel, domain,
-                                        parent_flags, sg, &placement, resv,
-                                        init_value, bo_ptr);
+        r = amdgpu_bo_do_create(adev, size, byte_align, kernel, domain,
+                                parent_flags, sg, resv, init_value, bo_ptr);
         if (r)
                 return r;
 
@@ -672,7 +633,6 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 {
         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
         int r, i;
-        unsigned fpfn, lpfn;
 
         if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
                 return -EPERM;
@@ -704,22 +664,16 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
         }
 
         bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+        /* force to pin into visible video ram */
+        if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
+                bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
         amdgpu_ttm_placement_from_domain(bo, domain);
         for (i = 0; i < bo->placement.num_placement; i++) {
-                /* force to pin into visible video ram */
-                if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
-                    !(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) &&
-                    (!max_offset || max_offset >
-                     adev->mc.visible_vram_size)) {
-                        if (WARN_ON_ONCE(min_offset >
-                                         adev->mc.visible_vram_size))
-                                return -EINVAL;
-                        fpfn = min_offset >> PAGE_SHIFT;
-                        lpfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
-                } else {
-                        fpfn = min_offset >> PAGE_SHIFT;
-                        lpfn = max_offset >> PAGE_SHIFT;
-                }
+                unsigned fpfn, lpfn;
+
+                fpfn = min_offset >> PAGE_SHIFT;
+                lpfn = max_offset >> PAGE_SHIFT;
+
                 if (fpfn > bo->placements[i].fpfn)
                         bo->placements[i].fpfn = fpfn;
                 if (!bo->placements[i].lpfn ||
@@ -928,8 +882,8 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
         if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
                 return;
 
-        abo = container_of(bo, struct amdgpu_bo, tbo);
-        amdgpu_vm_bo_invalidate(adev, abo);
+        abo = ttm_to_amdgpu_bo(bo);
+        amdgpu_vm_bo_invalidate(adev, abo, evict);
 
         amdgpu_bo_kunmap(abo);
 
@@ -955,7 +909,7 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
         if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
                 return 0;
 
-        abo = container_of(bo, struct amdgpu_bo, tbo);
+        abo = ttm_to_amdgpu_bo(bo);
 
         /* Remember that this BO was accessed by the CPU */
         abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index a288fa6d72c8..428aae048f4b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -35,6 +35,7 @@
 
 /* bo virtual addresses in a vm */
 struct amdgpu_bo_va_mapping {
+        struct amdgpu_bo_va             *bo_va;
         struct list_head                list;
         struct rb_node                  rb;
         uint64_t                        start;
@@ -49,12 +50,17 @@ struct amdgpu_bo_va {
         struct amdgpu_vm_bo_base        base;
 
         /* protected by bo being reserved */
-        struct dma_fence                *last_pt_update;
         unsigned                        ref_count;
 
+        /* all other members protected by the VM PD being reserved */
+        struct dma_fence                *last_pt_update;
+
         /* mappings for this bo_va */
         struct list_head                invalids;
         struct list_head                valids;
+
+        /* If the mappings are cleared or filled */
+        bool                            cleared;
 };
 
 struct amdgpu_bo {
@@ -88,6 +94,11 @@ struct amdgpu_bo {
         };
 };
 
+static inline struct amdgpu_bo *ttm_to_amdgpu_bo(struct ttm_buffer_object *tbo)
+{
+        return container_of(tbo, struct amdgpu_bo, tbo);
+}
+
 /**
  * amdgpu_mem_type_to_domain - return domain corresponding to mem_type
  * @mem_type:   ttm memory type
@@ -182,6 +193,14 @@ static inline bool amdgpu_bo_gpu_accessible(struct amdgpu_bo *bo)
         }
 }
 
+/**
+ * amdgpu_bo_explicit_sync - return whether the bo is explicitly synced
+ */
+static inline bool amdgpu_bo_explicit_sync(struct amdgpu_bo *bo)
+{
+        return bo->flags & AMDGPU_GEM_CREATE_EXPLICIT_SYNC;
+}
+
 int amdgpu_bo_create(struct amdgpu_device *adev,
                             unsigned long size, int byte_align,
                             bool kernel, u32 domain, u64 flags,
@@ -189,14 +208,6 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
                             struct reservation_object *resv,
                             uint64_t init_value,
                             struct amdgpu_bo **bo_ptr);
-int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
-                                unsigned long size, int byte_align,
-                                bool kernel, u32 domain, u64 flags,
-                                struct sg_table *sg,
-                                struct ttm_placement *placement,
-                                struct reservation_object *resv,
-                                uint64_t init_value,
-                                struct amdgpu_bo **bo_ptr);
 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
                               unsigned long size, int align,
                               u32 domain, struct amdgpu_bo **bo_ptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index 7df503aedb69..6c570d4e4516 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -64,17 +64,13 @@ static const struct cg_flag_name clocks[] = {
 
 void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev)
 {
-        if (adev->pp_enabled)
-                /* TODO */
-                return;
-
         if (adev->pm.dpm_enabled) {
                 mutex_lock(&adev->pm.mutex);
                 if (power_supply_is_system_supplied() > 0)
                         adev->pm.dpm.ac_power = true;
                 else
                         adev->pm.dpm.ac_power = false;
-                if (adev->pm.funcs->enable_bapm)
+                if (adev->powerplay.pp_funcs->enable_bapm)
                         amdgpu_dpm_enable_bapm(adev, adev->pm.dpm.ac_power);
                 mutex_unlock(&adev->pm.mutex);
         }
@@ -88,9 +84,9 @@ static ssize_t amdgpu_get_dpm_state(struct device *dev,
         struct amdgpu_device *adev = ddev->dev_private;
         enum amd_pm_state_type pm;
 
-        if (adev->pp_enabled) {
+        if (adev->powerplay.pp_funcs->get_current_power_state)
                 pm = amdgpu_dpm_get_current_power_state(adev);
-        } else
+        else
                 pm = adev->pm.dpm.user_state;
 
         return snprintf(buf, PAGE_SIZE, "%s\n",
@@ -118,8 +114,8 @@ static ssize_t amdgpu_set_dpm_state(struct device *dev,
                 goto fail;
         }
 
-        if (adev->pp_enabled) {
-                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_ENABLE_USER_STATE, &state, NULL);
+        if (adev->powerplay.pp_funcs->dispatch_tasks) {
+                amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_ENABLE_USER_STATE, &state, NULL);
         } else {
                 mutex_lock(&adev->pm.mutex);
                 adev->pm.dpm.user_state = state;
@@ -140,13 +136,17 @@ static ssize_t amdgpu_get_dpm_forced_performance_level(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        enum amd_dpm_forced_level level;
+        enum amd_dpm_forced_level level = 0xff;
 
         if  ((adev->flags & AMD_IS_PX) &&
              (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
                 return snprintf(buf, PAGE_SIZE, "off\n");
 
-        level = amdgpu_dpm_get_performance_level(adev);
+        if (adev->powerplay.pp_funcs->get_performance_level)
+                level = amdgpu_dpm_get_performance_level(adev);
+        else
+                level = adev->pm.dpm.forced_level;
+
         return snprintf(buf, PAGE_SIZE, "%s\n",
                         (level == AMD_DPM_FORCED_LEVEL_AUTO) ? "auto" :
                         (level == AMD_DPM_FORCED_LEVEL_LOW) ? "low" :
@@ -167,7 +167,7 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
         enum amd_dpm_forced_level level;
-        enum amd_dpm_forced_level current_level;
+        enum amd_dpm_forced_level current_level = 0xff;
         int ret = 0;
 
         /* Can't force performance level when the card is off */
@@ -175,7 +175,8 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
              (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
                 return -EINVAL;
 
-        current_level = amdgpu_dpm_get_performance_level(adev);
+        if (adev->powerplay.pp_funcs->get_performance_level)
+                current_level = amdgpu_dpm_get_performance_level(adev);
 
         if (strncmp("low", buf, strlen("low")) == 0) {
                 level = AMD_DPM_FORCED_LEVEL_LOW;
@@ -203,9 +204,7 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
         if (current_level == level)
                 return count;
 
-        if (adev->pp_enabled)
-                amdgpu_dpm_force_performance_level(adev, level);
-        else {
+        if (adev->powerplay.pp_funcs->force_performance_level) {
                 mutex_lock(&adev->pm.mutex);
                 if (adev->pm.dpm.thermal_active) {
                         count = -EINVAL;
@@ -233,7 +232,7 @@ static ssize_t amdgpu_get_pp_num_states(struct device *dev,
         struct pp_states_info data;
         int i, buf_len;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->get_pp_num_states)
                 amdgpu_dpm_get_pp_num_states(adev, &data);
 
         buf_len = snprintf(buf, PAGE_SIZE, "states: %d\n", data.nums);
@@ -257,8 +256,8 @@ static ssize_t amdgpu_get_pp_cur_state(struct device *dev,
         enum amd_pm_state_type pm = 0;
         int i = 0;
 
-        if (adev->pp_enabled) {
-
+        if (adev->powerplay.pp_funcs->get_current_power_state
+                 && adev->powerplay.pp_funcs->get_pp_num_states) {
                 pm = amdgpu_dpm_get_current_power_state(adev);
                 amdgpu_dpm_get_pp_num_states(adev, &data);
 
@@ -280,25 +279,10 @@ static ssize_t amdgpu_get_pp_force_state(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        struct pp_states_info data;
-        enum amd_pm_state_type pm = 0;
-        int i;
-
-        if (adev->pp_force_state_enabled && adev->pp_enabled) {
-                pm = amdgpu_dpm_get_current_power_state(adev);
-                amdgpu_dpm_get_pp_num_states(adev, &data);
 
-                for (i = 0; i < data.nums; i++) {
-                        if (pm == data.states[i])
-                                break;
-                }
-
-                if (i == data.nums)
-                        i = -EINVAL;
-
-                return snprintf(buf, PAGE_SIZE, "%d\n", i);
-
-        } else
+        if (adev->pp_force_state_enabled)
+                return amdgpu_get_pp_cur_state(dev, attr, buf);
+        else
                 return snprintf(buf, PAGE_SIZE, "\n");
 }
 
@@ -315,7 +299,8 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev,
 
         if (strlen(buf) == 1)
                 adev->pp_force_state_enabled = false;
-        else if (adev->pp_enabled) {
+        else if (adev->powerplay.pp_funcs->dispatch_tasks &&
+                        adev->powerplay.pp_funcs->get_pp_num_states) {
                 struct pp_states_info data;
 
                 ret = kstrtoul(buf, 0, &idx);
@@ -330,7 +315,7 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev,
                 if (state != POWER_STATE_TYPE_INTERNAL_BOOT &&
                     state != POWER_STATE_TYPE_DEFAULT) {
                         amdgpu_dpm_dispatch_task(adev,
-                                        AMD_PP_EVENT_ENABLE_USER_STATE, &state, NULL);
+                                        AMD_PP_TASK_ENABLE_USER_STATE, &state, NULL);
                         adev->pp_force_state_enabled = true;
                 }
         }
@@ -347,7 +332,7 @@ static ssize_t amdgpu_get_pp_table(struct device *dev,
         char *table = NULL;
         int size;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->get_pp_table)
                 size = amdgpu_dpm_get_pp_table(adev, &table);
         else
                 return 0;
@@ -368,7 +353,7 @@ static ssize_t amdgpu_set_pp_table(struct device *dev,
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->set_pp_table)
                 amdgpu_dpm_set_pp_table(adev, buf, count);
 
         return count;
@@ -380,14 +365,11 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        ssize_t size = 0;
-
-        if (adev->pp_enabled)
-                size = amdgpu_dpm_print_clock_levels(adev, PP_SCLK, buf);
-        else if (adev->pm.funcs->print_clock_levels)
-                size = adev->pm.funcs->print_clock_levels(adev, PP_SCLK, buf);
 
-        return size;
+        if (adev->powerplay.pp_funcs->print_clock_levels)
+                return amdgpu_dpm_print_clock_levels(adev, PP_SCLK, buf);
+        else
+                return snprintf(buf, PAGE_SIZE, "\n");
 }
 
 static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
@@ -416,10 +398,9 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
                 mask |= 1 << level;
         }
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->force_clock_level)
                 amdgpu_dpm_force_clock_level(adev, PP_SCLK, mask);
-        else if (adev->pm.funcs->force_clock_level)
-                adev->pm.funcs->force_clock_level(adev, PP_SCLK, mask);
+
 fail:
         return count;
 }
@@ -430,14 +411,11 @@ static ssize_t amdgpu_get_pp_dpm_mclk(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        ssize_t size = 0;
-
-        if (adev->pp_enabled)
-                size = amdgpu_dpm_print_clock_levels(adev, PP_MCLK, buf);
-        else if (adev->pm.funcs->print_clock_levels)
-                size = adev->pm.funcs->print_clock_levels(adev, PP_MCLK, buf);
 
-        return size;
+        if (adev->powerplay.pp_funcs->print_clock_levels)
+                return amdgpu_dpm_print_clock_levels(adev, PP_MCLK, buf);
+        else
+                return snprintf(buf, PAGE_SIZE, "\n");
 }
 
 static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev,
@@ -465,11 +443,9 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev,
                 }
                 mask |= 1 << level;
         }
-
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->force_clock_level)
                 amdgpu_dpm_force_clock_level(adev, PP_MCLK, mask);
-        else if (adev->pm.funcs->force_clock_level)
-                adev->pm.funcs->force_clock_level(adev, PP_MCLK, mask);
+
 fail:
         return count;
 }
@@ -480,14 +456,11 @@ static ssize_t amdgpu_get_pp_dpm_pcie(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        ssize_t size = 0;
-
-        if (adev->pp_enabled)
-                size = amdgpu_dpm_print_clock_levels(adev, PP_PCIE, buf);
-        else if (adev->pm.funcs->print_clock_levels)
-                size = adev->pm.funcs->print_clock_levels(adev, PP_PCIE, buf);
 
-        return size;
+        if (adev->powerplay.pp_funcs->print_clock_levels)
+                return amdgpu_dpm_print_clock_levels(adev, PP_PCIE, buf);
+        else
+                return snprintf(buf, PAGE_SIZE, "\n");
 }
 
 static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev,
@@ -515,11 +488,9 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev,
                 }
                 mask |= 1 << level;
         }
-
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->force_clock_level)
                 amdgpu_dpm_force_clock_level(adev, PP_PCIE, mask);
-        else if (adev->pm.funcs->force_clock_level)
-                adev->pm.funcs->force_clock_level(adev, PP_PCIE, mask);
+
 fail:
         return count;
 }
@@ -532,10 +503,8 @@ static ssize_t amdgpu_get_pp_sclk_od(struct device *dev,
         struct amdgpu_device *adev = ddev->dev_private;
         uint32_t value = 0;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->get_sclk_od)
                 value = amdgpu_dpm_get_sclk_od(adev);
-        else if (adev->pm.funcs->get_sclk_od)
-                value = adev->pm.funcs->get_sclk_od(adev);
 
         return snprintf(buf, PAGE_SIZE, "%d\n", value);
 }
@@ -556,12 +525,12 @@ static ssize_t amdgpu_set_pp_sclk_od(struct device *dev,
                 count = -EINVAL;
                 goto fail;
         }
-
-        if (adev->pp_enabled) {
+        if (adev->powerplay.pp_funcs->set_sclk_od)
                 amdgpu_dpm_set_sclk_od(adev, (uint32_t)value);
-                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_READJUST_POWER_STATE, NULL, NULL);
-        } else if (adev->pm.funcs->set_sclk_od) {
-                adev->pm.funcs->set_sclk_od(adev, (uint32_t)value);
+
+        if (adev->powerplay.pp_funcs->dispatch_tasks) {
+                amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL, NULL);
+        } else {
                 adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
                 amdgpu_pm_compute_clocks(adev);
         }
@@ -578,10 +547,8 @@ static ssize_t amdgpu_get_pp_mclk_od(struct device *dev,
         struct amdgpu_device *adev = ddev->dev_private;
         uint32_t value = 0;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->get_mclk_od)
                 value = amdgpu_dpm_get_mclk_od(adev);
-        else if (adev->pm.funcs->get_mclk_od)
-                value = adev->pm.funcs->get_mclk_od(adev);
 
         return snprintf(buf, PAGE_SIZE, "%d\n", value);
 }
@@ -602,12 +569,12 @@ static ssize_t amdgpu_set_pp_mclk_od(struct device *dev,
                 count = -EINVAL;
                 goto fail;
         }
-
-        if (adev->pp_enabled) {
+        if (adev->powerplay.pp_funcs->set_mclk_od)
                 amdgpu_dpm_set_mclk_od(adev, (uint32_t)value);
-                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_READJUST_POWER_STATE, NULL, NULL);
-        } else if (adev->pm.funcs->set_mclk_od) {
-                adev->pm.funcs->set_mclk_od(adev, (uint32_t)value);
+
+        if (adev->powerplay.pp_funcs->dispatch_tasks) {
+                amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL, NULL);
+        } else {
                 adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
                 amdgpu_pm_compute_clocks(adev);
         }
@@ -621,14 +588,11 @@ static ssize_t amdgpu_get_pp_power_profile(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        int ret = 0;
+        int ret = 0xff;
 
-        if (adev->pp_enabled)
+        if (adev->powerplay.pp_funcs->get_power_profile_state)
                 ret = amdgpu_dpm_get_power_profile_state(
                                 adev, query);
-        else if (adev->pm.funcs->get_power_profile_state)
-                ret = adev->pm.funcs->get_power_profile_state(
-                                adev, query);
 
         if (ret)
                 return ret;
@@ -675,15 +639,12 @@ static ssize_t amdgpu_set_pp_power_profile(struct device *dev,
         char *sub_str, buf_cpy[128], *tmp_str;
         const char delimiter[3] = {' ', '\n', '\0'};
         long int value;
-        int ret = 0;
+        int ret = 0xff;
 
         if (strncmp("reset", buf, strlen("reset")) == 0) {
-                if (adev->pp_enabled)
+                if (adev->powerplay.pp_funcs->reset_power_profile_state)
                         ret = amdgpu_dpm_reset_power_profile_state(
                                         adev, request);
-                else if (adev->pm.funcs->reset_power_profile_state)
-                        ret = adev->pm.funcs->reset_power_profile_state(
-                                        adev, request);
                 if (ret) {
                         count = -EINVAL;
                         goto fail;
@@ -692,12 +653,10 @@ static ssize_t amdgpu_set_pp_power_profile(struct device *dev,
         }
 
         if (strncmp("set", buf, strlen("set")) == 0) {
-                if (adev->pp_enabled)
+                if (adev->powerplay.pp_funcs->set_power_profile_state)
                         ret = amdgpu_dpm_set_power_profile_state(
                                         adev, request);
-                else if (adev->pm.funcs->set_power_profile_state)
-                        ret = adev->pm.funcs->set_power_profile_state(
-                                        adev, request);
+
                 if (ret) {
                         count = -EINVAL;
                         goto fail;
@@ -745,13 +704,8 @@ static ssize_t amdgpu_set_pp_power_profile(struct device *dev,
 
                 loop++;
         }
-
-        if (adev->pp_enabled)
-                ret = amdgpu_dpm_set_power_profile_state(
-                                adev, request);
-        else if (adev->pm.funcs->set_power_profile_state)
-                ret = adev->pm.funcs->set_power_profile_state(
-                                adev, request);
+        if (adev->powerplay.pp_funcs->set_power_profile_state)
+                ret = amdgpu_dpm_set_power_profile_state(adev, request);
 
         if (ret)
                 count = -EINVAL;
@@ -831,7 +785,7 @@ static ssize_t amdgpu_hwmon_show_temp(struct device *dev,
              (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
                 return -EINVAL;
 
-        if (!adev->pp_enabled && !adev->pm.funcs->get_temperature)
+        if (!adev->powerplay.pp_funcs->get_temperature)
                 temp = 0;
         else
                 temp = amdgpu_dpm_get_temperature(adev);
@@ -862,7 +816,7 @@ static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev,
         struct amdgpu_device *adev = dev_get_drvdata(dev);
         u32 pwm_mode = 0;
 
-        if (!adev->pp_enabled && !adev->pm.funcs->get_fan_control_mode)
+        if (!adev->powerplay.pp_funcs->get_fan_control_mode)
                 return -EINVAL;
 
         pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
@@ -879,7 +833,7 @@ static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev,
         int err;
         int value;
 
-        if (!adev->pp_enabled && !adev->pm.funcs->set_fan_control_mode)
+        if (!adev->powerplay.pp_funcs->set_fan_control_mode)
                 return -EINVAL;
 
         err = kstrtoint(buf, 10, &value);
@@ -919,9 +873,11 @@ static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev,
 
         value = (value * 100) / 255;
 
-        err = amdgpu_dpm_set_fan_speed_percent(adev, value);
-        if (err)
-                return err;
+        if (adev->powerplay.pp_funcs->set_fan_speed_percent) {
+                err = amdgpu_dpm_set_fan_speed_percent(adev, value);
+                if (err)
+                        return err;
+        }
 
         return count;
 }
@@ -932,11 +888,13 @@ static ssize_t amdgpu_hwmon_get_pwm1(struct device *dev,
 {
         struct amdgpu_device *adev = dev_get_drvdata(dev);
         int err;
-        u32 speed;
+        u32 speed = 0;
 
-        err = amdgpu_dpm_get_fan_speed_percent(adev, &speed);
-        if (err)
-                return err;
+        if (adev->powerplay.pp_funcs->get_fan_speed_percent) {
+                err = amdgpu_dpm_get_fan_speed_percent(adev, &speed);
+                if (err)
+                        return err;
+        }
 
         speed = (speed * 255) / 100;
 
@@ -949,11 +907,13 @@ static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev,
 {
         struct amdgpu_device *adev = dev_get_drvdata(dev);
         int err;
-        u32 speed;
+        u32 speed = 0;
 
-        err = amdgpu_dpm_get_fan_speed_rpm(adev, &speed);
-        if (err)
-                return err;
+        if (adev->powerplay.pp_funcs->get_fan_speed_rpm) {
+                err = amdgpu_dpm_get_fan_speed_rpm(adev, &speed);
+                if (err)
+                        return err;
+        }
 
         return sprintf(buf, "%i\n", speed);
 }
@@ -986,6 +946,10 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
         struct amdgpu_device *adev = dev_get_drvdata(dev);
         umode_t effective_mode = attr->mode;
 
+        /* no skipping for powerplay */
+        if (adev->powerplay.cgs_device)
+                return effective_mode;
+
         /* Skip limit attributes if DPM is not enabled */
         if (!adev->pm.dpm_enabled &&
             (attr == &sensor_dev_attr_temp1_crit.dev_attr.attr ||
@@ -996,9 +960,6 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
              attr == &sensor_dev_attr_pwm1_min.dev_attr.attr))
                 return 0;
 
-        if (adev->pp_enabled)
-                return effective_mode;
-
         /* Skip fan attributes if fan is not present */
         if (adev->pm.no_fan &&
             (attr == &sensor_dev_attr_pwm1.dev_attr.attr ||
@@ -1008,21 +969,21 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
                 return 0;
 
         /* mask fan attributes if we have no bindings for this asic to expose */
-        if ((!adev->pm.funcs->get_fan_speed_percent &&
+        if ((!adev->powerplay.pp_funcs->get_fan_speed_percent &&
              attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't query fan */
-            (!adev->pm.funcs->get_fan_control_mode &&
+            (!adev->powerplay.pp_funcs->get_fan_control_mode &&
              attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't query state */
                 effective_mode &= ~S_IRUGO;
 
-        if ((!adev->pm.funcs->set_fan_speed_percent &&
+        if ((!adev->powerplay.pp_funcs->set_fan_speed_percent &&
              attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't manage fan */
-            (!adev->pm.funcs->set_fan_control_mode &&
+            (!adev->powerplay.pp_funcs->set_fan_control_mode &&
              attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't manage state */
                 effective_mode &= ~S_IWUSR;
 
         /* hide max/min values if we can't both query and manage the fan */
-        if ((!adev->pm.funcs->set_fan_speed_percent &&
-             !adev->pm.funcs->get_fan_speed_percent) &&
+        if ((!adev->powerplay.pp_funcs->set_fan_speed_percent &&
+             !adev->powerplay.pp_funcs->get_fan_speed_percent) &&
             (attr == &sensor_dev_attr_pwm1_max.dev_attr.attr ||
              attr == &sensor_dev_attr_pwm1_min.dev_attr.attr))
                 return 0;
@@ -1055,7 +1016,7 @@ void amdgpu_dpm_thermal_work_handler(struct work_struct *work)
         if (!adev->pm.dpm_enabled)
                 return;
 
-        if (adev->pm.funcs->get_temperature) {
+        if (adev->powerplay.pp_funcs->get_temperature) {
                 int temp = amdgpu_dpm_get_temperature(adev);
 
                 if (temp < adev->pm.dpm.thermal.min_temp)
@@ -1087,7 +1048,7 @@ static struct amdgpu_ps *amdgpu_dpm_pick_power_state(struct amdgpu_device *adev,
                 true : false;
 
         /* check if the vblank period is too short to adjust the mclk */
-        if (single_display && adev->pm.funcs->vblank_too_short) {
+        if (single_display && adev->powerplay.pp_funcs->vblank_too_short) {
                 if (amdgpu_dpm_vblank_too_short(adev))
                         single_display = false;
         }
@@ -1216,7 +1177,7 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
         struct amdgpu_ps *ps;
         enum amd_pm_state_type dpm_state;
         int ret;
-        bool equal;
+        bool equal = false;
 
         /* if dpm init failed */
         if (!adev->pm.dpm_enabled)
@@ -1236,7 +1197,7 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
         else
                 return;
 
-        if (amdgpu_dpm == 1) {
+        if (amdgpu_dpm == 1 && adev->powerplay.pp_funcs->print_power_state) {
                 printk("switching from power state:\n");
                 amdgpu_dpm_print_power_state(adev, adev->pm.dpm.current_ps);
                 printk("switching to power state:\n");
@@ -1245,15 +1206,17 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
 
         /* update whether vce is active */
         ps->vce_active = adev->pm.dpm.vce_active;
-
-        amdgpu_dpm_display_configuration_changed(adev);
+        if (adev->powerplay.pp_funcs->display_configuration_changed)
+                amdgpu_dpm_display_configuration_changed(adev);
 
         ret = amdgpu_dpm_pre_set_power_state(adev);
         if (ret)
                 return;
 
-        if ((0 != amgdpu_dpm_check_state_equal(adev, adev->pm.dpm.current_ps, adev->pm.dpm.requested_ps, &equal)))
-                equal = false;
+        if (adev->powerplay.pp_funcs->check_state_equal) {
+                if (0 != amdgpu_dpm_check_state_equal(adev, adev->pm.dpm.current_ps, adev->pm.dpm.requested_ps, &equal))
+                        equal = false;
+        }
 
         if (equal)
                 return;
@@ -1264,7 +1227,7 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
         adev->pm.dpm.current_active_crtcs = adev->pm.dpm.new_active_crtcs;
         adev->pm.dpm.current_active_crtc_count = adev->pm.dpm.new_active_crtc_count;
 
-        if (adev->pm.funcs->force_performance_level) {
+        if (adev->powerplay.pp_funcs->force_performance_level) {
                 if (adev->pm.dpm.thermal_active) {
                         enum amd_dpm_forced_level level = adev->pm.dpm.forced_level;
                         /* force low perf level for thermal */
@@ -1280,7 +1243,7 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
 
 void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable)
 {
-        if (adev->pp_enabled || adev->pm.funcs->powergate_uvd) {
+        if (adev->powerplay.pp_funcs->powergate_uvd) {
                 /* enable/disable UVD */
                 mutex_lock(&adev->pm.mutex);
                 amdgpu_dpm_powergate_uvd(adev, !enable);
@@ -1302,7 +1265,7 @@ void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable)
 
 void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable)
 {
-        if (adev->pp_enabled || adev->pm.funcs->powergate_vce) {
+        if (adev->powerplay.pp_funcs->powergate_vce) {
                 /* enable/disable VCE */
                 mutex_lock(&adev->pm.mutex);
                 amdgpu_dpm_powergate_vce(adev, !enable);
@@ -1337,8 +1300,7 @@ void amdgpu_pm_print_power_states(struct amdgpu_device *adev)
 {
         int i;
 
-        if (adev->pp_enabled)
-                /* TO DO */
+        if (adev->powerplay.pp_funcs->print_power_state == NULL)
                 return;
 
         for (i = 0; i < adev->pm.dpm.num_ps; i++)
@@ -1353,10 +1315,11 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
         if (adev->pm.sysfs_initialized)
                 return 0;
 
-        if (!adev->pp_enabled) {
-                if (adev->pm.funcs->get_temperature == NULL)
-                        return 0;
-        }
+        if (adev->pm.dpm_enabled == 0)
+                return 0;
+
+        if (adev->powerplay.pp_funcs->get_temperature == NULL)
+                return 0;
 
         adev->pm.int_hwmon_dev = hwmon_device_register_with_groups(adev->dev,
                                                                    DRIVER_NAME, adev,
@@ -1379,27 +1342,26 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
                 return ret;
         }
 
-        if (adev->pp_enabled) {
-                ret = device_create_file(adev->dev, &dev_attr_pp_num_states);
-                if (ret) {
-                        DRM_ERROR("failed to create device file pp_num_states\n");
-                        return ret;
-                }
-                ret = device_create_file(adev->dev, &dev_attr_pp_cur_state);
-                if (ret) {
-                        DRM_ERROR("failed to create device file pp_cur_state\n");
-                        return ret;
-                }
-                ret = device_create_file(adev->dev, &dev_attr_pp_force_state);
-                if (ret) {
-                        DRM_ERROR("failed to create device file pp_force_state\n");
-                        return ret;
-                }
-                ret = device_create_file(adev->dev, &dev_attr_pp_table);
-                if (ret) {
-                        DRM_ERROR("failed to create device file pp_table\n");
-                        return ret;
-                }
+
+        ret = device_create_file(adev->dev, &dev_attr_pp_num_states);
+        if (ret) {
+                DRM_ERROR("failed to create device file pp_num_states\n");
+                return ret;
+        }
+        ret = device_create_file(adev->dev, &dev_attr_pp_cur_state);
+        if (ret) {
+                DRM_ERROR("failed to create device file pp_cur_state\n");
+                return ret;
+        }
+        ret = device_create_file(adev->dev, &dev_attr_pp_force_state);
+        if (ret) {
+                DRM_ERROR("failed to create device file pp_force_state\n");
+                return ret;
+        }
+        ret = device_create_file(adev->dev, &dev_attr_pp_table);
+        if (ret) {
+                DRM_ERROR("failed to create device file pp_table\n");
+                return ret;
         }
 
         ret = device_create_file(adev->dev, &dev_attr_pp_dpm_sclk);
@@ -1455,16 +1417,19 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
 
 void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev)
 {
+        if (adev->pm.dpm_enabled == 0)
+                return;
+
         if (adev->pm.int_hwmon_dev)
                 hwmon_device_unregister(adev->pm.int_hwmon_dev);
         device_remove_file(adev->dev, &dev_attr_power_dpm_state);
         device_remove_file(adev->dev, &dev_attr_power_dpm_force_performance_level);
-        if (adev->pp_enabled) {
-                device_remove_file(adev->dev, &dev_attr_pp_num_states);
-                device_remove_file(adev->dev, &dev_attr_pp_cur_state);
-                device_remove_file(adev->dev, &dev_attr_pp_force_state);
-                device_remove_file(adev->dev, &dev_attr_pp_table);
-        }
+
+        device_remove_file(adev->dev, &dev_attr_pp_num_states);
+        device_remove_file(adev->dev, &dev_attr_pp_cur_state);
+        device_remove_file(adev->dev, &dev_attr_pp_force_state);
+        device_remove_file(adev->dev, &dev_attr_pp_table);
+
         device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk);
         device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk);
         device_remove_file(adev->dev, &dev_attr_pp_dpm_pcie);
@@ -1495,8 +1460,8 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
                         amdgpu_fence_wait_empty(ring);
         }
 
-        if (adev->pp_enabled) {
-                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_DISPLAY_CONFIG_CHANGE, NULL, NULL);
+        if (adev->powerplay.pp_funcs->dispatch_tasks) {
+                amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, NULL, NULL);
         } else {
                 mutex_lock(&adev->pm.mutex);
                 adev->pm.dpm.new_active_crtcs = 0;
@@ -1505,7 +1470,7 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
                         list_for_each_entry(crtc,
                                             &ddev->mode_config.crtc_list, head) {
                                 amdgpu_crtc = to_amdgpu_crtc(crtc);
-                                if (crtc->enabled) {
+                                if (amdgpu_crtc->enabled) {
                                         adev->pm.dpm.new_active_crtcs |= (1 << amdgpu_crtc->crtc_id);
                                         adev->pm.dpm.new_active_crtc_count++;
                                 }
@@ -1630,15 +1595,15 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data)
         if  ((adev->flags & AMD_IS_PX) &&
              (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) {
                 seq_printf(m, "PX asic powered off\n");
-        } else if (adev->pp_enabled) {
-                return amdgpu_debugfs_pm_info_pp(m, adev);
-        } else {
+        } else if (adev->powerplay.pp_funcs->debugfs_print_current_performance_level) {
                 mutex_lock(&adev->pm.mutex);
-                if (adev->pm.funcs->debugfs_print_current_performance_level)
-                        adev->pm.funcs->debugfs_print_current_performance_level(adev, m);
+                if (adev->powerplay.pp_funcs->debugfs_print_current_performance_level)
+                        adev->powerplay.pp_funcs->debugfs_print_current_performance_level(adev, m);
                 else
                         seq_printf(m, "Debugfs support not implemented for this asic\n");
                 mutex_unlock(&adev->pm.mutex);
+        } else {
+                return amdgpu_debugfs_pm_info_pp(m, adev);
         }
 
         return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
index b7e1c026c0c8..033fba2def6f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
@@ -34,24 +34,6 @@
 #include "cik_dpm.h"
 #include "vi_dpm.h"
 
-static int amdgpu_create_pp_handle(struct amdgpu_device *adev)
-{
-        struct amd_pp_init pp_init;
-        struct amd_powerplay *amd_pp;
-        int ret;
-
-        amd_pp = &(adev->powerplay);
-        pp_init.chip_family = adev->family;
-        pp_init.chip_id = adev->asic_type;
-        pp_init.pm_en = (amdgpu_dpm != 0 && !amdgpu_sriov_vf(adev)) ? true : false;
-        pp_init.feature_mask = amdgpu_pp_feature_mask;
-        pp_init.device = amdgpu_cgs_create_device(adev);
-        ret = amd_powerplay_create(&pp_init, &(amd_pp->pp_handle));
-        if (ret)
-                return -EINVAL;
-        return 0;
-}
-
 static int amdgpu_pp_early_init(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -59,7 +41,6 @@ static int amdgpu_pp_early_init(void *handle)
         int ret = 0;
 
         amd_pp = &(adev->powerplay);
-        adev->pp_enabled = false;
         amd_pp->pp_handle = (void *)adev;
 
         switch (adev->asic_type) {
@@ -73,9 +54,7 @@ static int amdgpu_pp_early_init(void *handle)
         case CHIP_STONEY:
         case CHIP_VEGA10:
         case CHIP_RAVEN:
-                adev->pp_enabled = true;
-                if (amdgpu_create_pp_handle(adev))
-                        return -EINVAL;
+                amd_pp->cgs_device = amdgpu_cgs_create_device(adev);
                 amd_pp->ip_funcs = &pp_ip_funcs;
                 amd_pp->pp_funcs = &pp_dpm_funcs;
                 break;
@@ -87,17 +66,26 @@ static int amdgpu_pp_early_init(void *handle)
         case CHIP_OLAND:
         case CHIP_HAINAN:
                 amd_pp->ip_funcs = &si_dpm_ip_funcs;
+                amd_pp->pp_funcs = &si_dpm_funcs;
         break;
 #endif
 #ifdef CONFIG_DRM_AMDGPU_CIK
         case CHIP_BONAIRE:
         case CHIP_HAWAII:
-                amd_pp->ip_funcs = &ci_dpm_ip_funcs;
+                if (amdgpu_dpm == -1) {
+                        amd_pp->ip_funcs = &ci_dpm_ip_funcs;
+                        amd_pp->pp_funcs = &ci_dpm_funcs;
+                } else {
+                        amd_pp->cgs_device = amdgpu_cgs_create_device(adev);
+                        amd_pp->ip_funcs = &pp_ip_funcs;
+                        amd_pp->pp_funcs = &pp_dpm_funcs;
+                }
                 break;
         case CHIP_KABINI:
         case CHIP_MULLINS:
         case CHIP_KAVERI:
                 amd_pp->ip_funcs = &kv_dpm_ip_funcs;
+                amd_pp->pp_funcs = &kv_dpm_funcs;
                 break;
 #endif
         default:
@@ -107,12 +95,9 @@ static int amdgpu_pp_early_init(void *handle)
 
         if (adev->powerplay.ip_funcs->early_init)
                 ret = adev->powerplay.ip_funcs->early_init(
-                                        adev->powerplay.pp_handle);
+                                        amd_pp->cgs_device ? amd_pp->cgs_device :
+                                        amd_pp->pp_handle);
 
-        if (ret == PP_DPM_DISABLED) {
-                adev->pm.dpm_enabled = false;
-                return 0;
-        }
         return ret;
 }
 
@@ -126,11 +111,6 @@ static int amdgpu_pp_late_init(void *handle)
                 ret = adev->powerplay.ip_funcs->late_init(
                                         adev->powerplay.pp_handle);
 
-        if (adev->pp_enabled && adev->pm.dpm_enabled) {
-                amdgpu_pm_sysfs_init(adev);
-                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_COMPLETE_INIT, NULL, NULL);
-        }
-
         return ret;
 }
 
@@ -165,21 +145,13 @@ static int amdgpu_pp_hw_init(void *handle)
         int ret = 0;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (adev->pp_enabled && adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
+        if (adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
                 amdgpu_ucode_init_bo(adev);
 
         if (adev->powerplay.ip_funcs->hw_init)
                 ret = adev->powerplay.ip_funcs->hw_init(
                                         adev->powerplay.pp_handle);
 
-        if (ret == PP_DPM_DISABLED) {
-                adev->pm.dpm_enabled = false;
-                return 0;
-        }
-
-        if ((amdgpu_dpm != 0) && !amdgpu_sriov_vf(adev))
-                adev->pm.dpm_enabled = true;
-
         return ret;
 }
 
@@ -188,16 +160,10 @@ static int amdgpu_pp_hw_fini(void *handle)
         int ret = 0;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (adev->pp_enabled && adev->pm.dpm_enabled)
-                amdgpu_pm_sysfs_fini(adev);
-
         if (adev->powerplay.ip_funcs->hw_fini)
                 ret = adev->powerplay.ip_funcs->hw_fini(
                                         adev->powerplay.pp_handle);
 
-        if (adev->pp_enabled && adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
-                amdgpu_ucode_fini_bo(adev);
-
         return ret;
 }
 
@@ -209,9 +175,8 @@ static void amdgpu_pp_late_fini(void *handle)
                 adev->powerplay.ip_funcs->late_fini(
                           adev->powerplay.pp_handle);
 
-
-        if (adev->pp_enabled)
-                amd_powerplay_destroy(adev->powerplay.pp_handle);
+        if (adev->powerplay.cgs_device)
+                amdgpu_cgs_destroy_device(adev->powerplay.cgs_device);
 }
 
 static int amdgpu_pp_suspend(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
index 5b3f92891f89..ae9c106979d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
@@ -57,6 +57,40 @@ void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
         ttm_bo_kunmap(&bo->dma_buf_vmap);
 }
 
+int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+        struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
+        struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+        unsigned asize = amdgpu_bo_size(bo);
+        int ret;
+
+        if (!vma->vm_file)
+                return -ENODEV;
+
+        if (adev == NULL)
+                return -ENODEV;
+
+        /* Check for valid size. */
+        if (asize < vma->vm_end - vma->vm_start)
+                return -EINVAL;
+
+        if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) ||
+            (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
+                return -EPERM;
+        }
+        vma->vm_pgoff += amdgpu_bo_mmap_offset(bo) >> PAGE_SHIFT;
+
+        /* prime mmap does not need to check access, so allow here */
+        ret = drm_vma_node_allow(&obj->vma_node, vma->vm_file->private_data);
+        if (ret)
+                return ret;
+
+        ret = ttm_bo_mmap(vma->vm_file, vma, &adev->mman.bdev);
+        drm_vma_node_revoke(&obj->vma_node, vma->vm_file->private_data);
+
+        return ret;
+}
+
 struct drm_gem_object *
 amdgpu_gem_prime_import_sg_table(struct drm_device *dev,
                                  struct dma_buf_attachment *attach,
@@ -135,9 +169,14 @@ struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev,
                                         int flags)
 {
         struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
+        struct dma_buf *buf;
 
-        if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
+        if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) ||
+            bo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
                 return ERR_PTR(-EPERM);
 
-        return drm_gem_prime_export(dev, gobj, flags);
+        buf = drm_gem_prime_export(dev, gobj, flags);
+        if (!IS_ERR(buf))
+                buf->file->f_mapping = dev->anon_inode->i_mapping;
+        return buf;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 8c2204c7b384..7714f4a6c8b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -57,21 +57,23 @@ static int psp_sw_init(void *handle)
                 psp->prep_cmd_buf = psp_v3_1_prep_cmd_buf;
                 psp->ring_init = psp_v3_1_ring_init;
                 psp->ring_create = psp_v3_1_ring_create;
+                psp->ring_stop = psp_v3_1_ring_stop;
                 psp->ring_destroy = psp_v3_1_ring_destroy;
                 psp->cmd_submit = psp_v3_1_cmd_submit;
                 psp->compare_sram_data = psp_v3_1_compare_sram_data;
                 psp->smu_reload_quirk = psp_v3_1_smu_reload_quirk;
+                psp->mode1_reset = psp_v3_1_mode1_reset;
                 break;
         case CHIP_RAVEN:
-#if 0
                 psp->init_microcode = psp_v10_0_init_microcode;
-#endif
                 psp->prep_cmd_buf = psp_v10_0_prep_cmd_buf;
                 psp->ring_init = psp_v10_0_ring_init;
                 psp->ring_create = psp_v10_0_ring_create;
+                psp->ring_stop = psp_v10_0_ring_stop;
                 psp->ring_destroy = psp_v10_0_ring_destroy;
                 psp->cmd_submit = psp_v10_0_cmd_submit;
                 psp->compare_sram_data = psp_v10_0_compare_sram_data;
+                psp->mode1_reset = psp_v10_0_mode1_reset;
                 break;
         default:
                 return -EINVAL;
@@ -90,6 +92,12 @@ static int psp_sw_init(void *handle)
 
 static int psp_sw_fini(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+        release_firmware(adev->psp.sos_fw);
+        adev->psp.sos_fw = NULL;
+        release_firmware(adev->psp.asd_fw);
+        adev->psp.asd_fw = NULL;
         return 0;
 }
 
@@ -253,15 +261,18 @@ static int psp_asd_load(struct psp_context *psp)
 
 static int psp_hw_start(struct psp_context *psp)
 {
+        struct amdgpu_device *adev = psp->adev;
         int ret;
 
-        ret = psp_bootloader_load_sysdrv(psp);
-        if (ret)
-                return ret;
+        if (!amdgpu_sriov_vf(adev) || !adev->in_sriov_reset) {
+                ret = psp_bootloader_load_sysdrv(psp);
+                if (ret)
+                        return ret;
 
-        ret = psp_bootloader_load_sos(psp);
-        if (ret)
-                return ret;
+                ret = psp_bootloader_load_sos(psp);
+                if (ret)
+                        return ret;
+        }
 
         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
         if (ret)
@@ -431,8 +442,6 @@ static int psp_hw_fini(void *handle)
         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
                 return 0;
 
-        amdgpu_ucode_fini_bo(adev);
-
         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
 
         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
@@ -453,6 +462,16 @@ static int psp_hw_fini(void *handle)
 
 static int psp_suspend(void *handle)
 {
+        int ret;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        struct psp_context *psp = &adev->psp;
+
+        ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
+        if (ret) {
+                DRM_ERROR("PSP ring stop failed\n");
+                return ret;
+        }
+
         return 0;
 }
 
@@ -487,6 +506,22 @@ failed:
         return ret;
 }
 
+static bool psp_check_reset(void* handle)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+        if (adev->flags & AMD_IS_APU)
+                return true;
+
+        return false;
+}
+
+static int psp_reset(void* handle)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        return psp_mode1_reset(&adev->psp);
+}
+
 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
                                         enum AMDGPU_UCODE_ID ucode_type)
 {
@@ -530,8 +565,9 @@ const struct amd_ip_funcs psp_ip_funcs = {
         .suspend = psp_suspend,
         .resume = psp_resume,
         .is_idle = NULL,
+        .check_soft_reset = psp_check_reset,
         .wait_for_idle = NULL,
-        .soft_reset = NULL,
+        .soft_reset = psp_reset,
         .set_clockgating_state = psp_set_clockgating_state,
         .set_powergating_state = psp_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index 538fa9dbfb21..ce4654550416 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -66,6 +66,8 @@ struct psp_context
                             struct psp_gfx_cmd_resp *cmd);
         int (*ring_init)(struct psp_context *psp, enum psp_ring_type ring_type);
         int (*ring_create)(struct psp_context *psp, enum psp_ring_type ring_type);
+        int (*ring_stop)(struct psp_context *psp,
+                            enum psp_ring_type ring_type);
         int (*ring_destroy)(struct psp_context *psp,
                             enum psp_ring_type ring_type);
         int (*cmd_submit)(struct psp_context *psp, struct amdgpu_firmware_info *ucode,
@@ -74,6 +76,7 @@ struct psp_context
                                   struct amdgpu_firmware_info *ucode,
                                   enum AMDGPU_UCODE_ID ucode_type);
         bool (*smu_reload_quirk)(struct psp_context *psp);
+        int (*mode1_reset)(struct psp_context *psp);
 
         /* fence buffer */
         struct amdgpu_bo                *fw_pri_bo;
@@ -123,6 +126,7 @@ struct amdgpu_psp_funcs {
 #define psp_prep_cmd_buf(ucode, type) (psp)->prep_cmd_buf((ucode), (type))
 #define psp_ring_init(psp, type) (psp)->ring_init((psp), (type))
 #define psp_ring_create(psp, type) (psp)->ring_create((psp), (type))
+#define psp_ring_stop(psp, type) (psp)->ring_stop((psp), (type))
 #define psp_ring_destroy(psp, type) ((psp)->ring_destroy((psp), (type)))
 #define psp_cmd_submit(psp, ucode, cmd_mc, fence_mc, index) \
                 (psp)->cmd_submit((psp), (ucode), (cmd_mc), (fence_mc), (index))
@@ -136,6 +140,8 @@ struct amdgpu_psp_funcs {
                 ((psp)->bootloader_load_sos ? (psp)->bootloader_load_sos((psp)) : 0)
 #define psp_smu_reload_quirk(psp) \
                 ((psp)->smu_reload_quirk ? (psp)->smu_reload_quirk((psp)) : false)
+#define psp_mode1_reset(psp) \
+                ((psp)->mode1_reset ? (psp)->mode1_reset((psp)) : false)
 
 extern const struct amd_ip_funcs psp_ip_funcs;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
index befc09b68543..190e28cb827e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
@@ -121,7 +121,7 @@ static enum amdgpu_ring_type amdgpu_hw_ip_to_ring_type(int hw_ip)
 
 static int amdgpu_lru_map(struct amdgpu_device *adev,
                           struct amdgpu_queue_mapper *mapper,
-                          int user_ring,
+                          int user_ring, bool lru_pipe_order,
                           struct amdgpu_ring **out_ring)
 {
         int r, i, j;
@@ -139,7 +139,7 @@ static int amdgpu_lru_map(struct amdgpu_device *adev,
         }
 
         r = amdgpu_ring_lru_get(adev, ring_type, ring_blacklist,
-                                j, out_ring);
+                                j, lru_pipe_order, out_ring);
         if (r)
                 return r;
 
@@ -284,8 +284,10 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
                 r = amdgpu_identity_map(adev, mapper, ring, out_ring);
                 break;
         case AMDGPU_HW_IP_DMA:
+                r = amdgpu_lru_map(adev, mapper, ring, false, out_ring);
+                break;
         case AMDGPU_HW_IP_COMPUTE:
-                r = amdgpu_lru_map(adev, mapper, ring, out_ring);
+                r = amdgpu_lru_map(adev, mapper, ring, true, out_ring);
                 break;
         default:
                 *out_ring = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 5ce65280b396..a98fbbb4739f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -136,7 +136,8 @@ void amdgpu_ring_commit(struct amdgpu_ring *ring)
         if (ring->funcs->end_use)
                 ring->funcs->end_use(ring);
 
-        amdgpu_ring_lru_touch(ring->adev, ring);
+        if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ)
+                amdgpu_ring_lru_touch(ring->adev, ring);
 }
 
 /**
@@ -155,6 +156,75 @@ void amdgpu_ring_undo(struct amdgpu_ring *ring)
 }
 
 /**
+ * amdgpu_ring_priority_put - restore a ring's priority
+ *
+ * @ring: amdgpu_ring structure holding the information
+ * @priority: target priority
+ *
+ * Release a request for executing at @priority
+ */
+void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
+                              enum amd_sched_priority priority)
+{
+        int i;
+
+        if (!ring->funcs->set_priority)
+                return;
+
+        if (atomic_dec_return(&ring->num_jobs[priority]) > 0)
+                return;
+
+        /* no need to restore if the job is already at the lowest priority */
+        if (priority == AMD_SCHED_PRIORITY_NORMAL)
+                return;
+
+        mutex_lock(&ring->priority_mutex);
+        /* something higher prio is executing, no need to decay */
+        if (ring->priority > priority)
+                goto out_unlock;
+
+        /* decay priority to the next level with a job available */
+        for (i = priority; i >= AMD_SCHED_PRIORITY_MIN; i--) {
+                if (i == AMD_SCHED_PRIORITY_NORMAL
+                                || atomic_read(&ring->num_jobs[i])) {
+                        ring->priority = i;
+                        ring->funcs->set_priority(ring, i);
+                        break;
+                }
+        }
+
+out_unlock:
+        mutex_unlock(&ring->priority_mutex);
+}
+
+/**
+ * amdgpu_ring_priority_get - change the ring's priority
+ *
+ * @ring: amdgpu_ring structure holding the information
+ * @priority: target priority
+ *
+ * Request a ring's priority to be raised to @priority (refcounted).
+ */
+void amdgpu_ring_priority_get(struct amdgpu_ring *ring,
+                              enum amd_sched_priority priority)
+{
+        if (!ring->funcs->set_priority)
+                return;
+
+        atomic_inc(&ring->num_jobs[priority]);
+
+        mutex_lock(&ring->priority_mutex);
+        if (priority <= ring->priority)
+                goto out_unlock;
+
+        ring->priority = priority;
+        ring->funcs->set_priority(ring, priority);
+
+out_unlock:
+        mutex_unlock(&ring->priority_mutex);
+}
+
+/**
  * amdgpu_ring_init - init driver ring struct.
  *
  * @adev: amdgpu_device pointer
@@ -169,7 +239,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
                      unsigned max_dw, struct amdgpu_irq_src *irq_src,
                      unsigned irq_type)
 {
-        int r;
+        int r, i;
         int sched_hw_submission = amdgpu_sched_hw_submission;
 
         /* Set the hw submission limit higher for KIQ because
@@ -247,9 +317,14 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
         }
 
         ring->max_dw = max_dw;
+        ring->priority = AMD_SCHED_PRIORITY_NORMAL;
+        mutex_init(&ring->priority_mutex);
         INIT_LIST_HEAD(&ring->lru_list);
         amdgpu_ring_lru_touch(adev, ring);
 
+        for (i = 0; i < AMD_SCHED_PRIORITY_MAX; ++i)
+                atomic_set(&ring->num_jobs[i], 0);
+
         if (amdgpu_debugfs_ring_init(adev, ring)) {
                 DRM_ERROR("Failed to register debugfs file for rings !\n");
         }
@@ -315,14 +390,16 @@ static bool amdgpu_ring_is_blacklisted(struct amdgpu_ring *ring,
  * @type: amdgpu_ring_type enum
  * @blacklist: blacklisted ring ids array
  * @num_blacklist: number of entries in @blacklist
+ * @lru_pipe_order: find a ring from the least recently used pipe
  * @ring: output ring
  *
  * Retrieve the amdgpu_ring structure for the least recently used ring of
  * a specific IP block (all asics).
  * Returns 0 on success, error on failure.
  */
-int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type, int *blacklist,
-                        int num_blacklist, struct amdgpu_ring **ring)
+int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type,
+                        int *blacklist, int num_blacklist,
+                        bool lru_pipe_order, struct amdgpu_ring **ring)
 {
         struct amdgpu_ring *entry;
 
@@ -337,10 +414,23 @@ int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type, int *blacklist,
                 if (amdgpu_ring_is_blacklisted(entry, blacklist, num_blacklist))
                         continue;
 
-                *ring = entry;
-                amdgpu_ring_lru_touch_locked(adev, *ring);
-                break;
+                if (!*ring) {
+                        *ring = entry;
+
+                        /* We are done for ring LRU */
+                        if (!lru_pipe_order)
+                                break;
+                }
+
+                /* Move all rings on the same pipe to the end of the list */
+                if (entry->pipe == (*ring)->pipe)
+                        amdgpu_ring_lru_touch_locked(adev, entry);
         }
+
+        /* Move the ring we found to the end of the list */
+        if (*ring)
+                amdgpu_ring_lru_touch_locked(adev, *ring);
+
         spin_unlock(&adev->ring_lru_list_lock);
 
         if (!*ring) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 322d25299a00..b18c2b96691f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -24,6 +24,7 @@
 #ifndef __AMDGPU_RING_H__
 #define __AMDGPU_RING_H__
 
+#include <drm/amdgpu_drm.h>
 #include "gpu_scheduler.h"
 
 /* max number of rings */
@@ -56,6 +57,7 @@ struct amdgpu_device;
 struct amdgpu_ring;
 struct amdgpu_ib;
 struct amdgpu_cs_parser;
+struct amdgpu_job;
 
 /*
  * Fences.
@@ -88,8 +90,12 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
 void amdgpu_fence_driver_suspend(struct amdgpu_device *adev);
 void amdgpu_fence_driver_resume(struct amdgpu_device *adev);
 int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence);
+int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s);
 void amdgpu_fence_process(struct amdgpu_ring *ring);
 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring);
+signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
+                                      uint32_t wait_seq,
+                                      signed long timeout);
 unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring);
 
 /*
@@ -147,6 +153,9 @@ struct amdgpu_ring_funcs {
         void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg);
         void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
         void (*emit_tmz)(struct amdgpu_ring *ring, bool start);
+        /* priority functions */
+        void (*set_priority) (struct amdgpu_ring *ring,
+                              enum amd_sched_priority priority);
 };
 
 struct amdgpu_ring {
@@ -187,6 +196,12 @@ struct amdgpu_ring {
         volatile u32            *cond_exe_cpu_addr;
         unsigned                vm_inv_eng;
         bool                    has_compute_vm_bug;
+
+        atomic_t                num_jobs[AMD_SCHED_PRIORITY_MAX];
+        struct mutex            priority_mutex;
+        /* protected by priority_mutex */
+        int                     priority;
+
 #if defined(CONFIG_DEBUG_FS)
         struct dentry *ent;
 #endif
@@ -197,12 +212,17 @@ void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count);
 void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
 void amdgpu_ring_commit(struct amdgpu_ring *ring);
 void amdgpu_ring_undo(struct amdgpu_ring *ring);
+void amdgpu_ring_priority_get(struct amdgpu_ring *ring,
+                              enum amd_sched_priority priority);
+void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
+                              enum amd_sched_priority priority);
 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
                      unsigned ring_size, struct amdgpu_irq_src *irq_src,
                      unsigned irq_type);
 void amdgpu_ring_fini(struct amdgpu_ring *ring);
-int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type, int *blacklist,
-                        int num_blacklist, struct amdgpu_ring **ring);
+int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type,
+                        int *blacklist, int num_blacklist,
+                        bool lru_pipe_order, struct amdgpu_ring **ring);
 void amdgpu_ring_lru_touch(struct amdgpu_device *adev, struct amdgpu_ring *ring);
 static inline void amdgpu_ring_clear_ring(struct amdgpu_ring *ring)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
new file mode 100644
index 000000000000..290cc3f9c433
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2017 Valve Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Andres Rodriguez <andresx7@gmail.com>
+ */
+
+#include <linux/fdtable.h>
+#include <linux/pid.h>
+#include <drm/amdgpu_drm.h>
+#include "amdgpu.h"
+
+#include "amdgpu_vm.h"
+
+enum amd_sched_priority amdgpu_to_sched_priority(int amdgpu_priority)
+{
+        switch (amdgpu_priority) {
+        case AMDGPU_CTX_PRIORITY_VERY_HIGH:
+                return AMD_SCHED_PRIORITY_HIGH_HW;
+        case AMDGPU_CTX_PRIORITY_HIGH:
+                return AMD_SCHED_PRIORITY_HIGH_SW;
+        case AMDGPU_CTX_PRIORITY_NORMAL:
+                return AMD_SCHED_PRIORITY_NORMAL;
+        case AMDGPU_CTX_PRIORITY_LOW:
+        case AMDGPU_CTX_PRIORITY_VERY_LOW:
+                return AMD_SCHED_PRIORITY_LOW;
+        case AMDGPU_CTX_PRIORITY_UNSET:
+                return AMD_SCHED_PRIORITY_UNSET;
+        default:
+                WARN(1, "Invalid context priority %d\n", amdgpu_priority);
+                return AMD_SCHED_PRIORITY_INVALID;
+        }
+}
+
+static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev,
+                                                  int fd,
+                                                  enum amd_sched_priority priority)
+{
+        struct file *filp = fcheck(fd);
+        struct drm_file *file;
+        struct pid *pid;
+        struct amdgpu_fpriv *fpriv;
+        struct amdgpu_ctx *ctx;
+        uint32_t id;
+
+        if (!filp)
+                return -EINVAL;
+
+        pid = get_pid(((struct drm_file *)filp->private_data)->pid);
+
+        mutex_lock(&adev->ddev->filelist_mutex);
+        list_for_each_entry(file, &adev->ddev->filelist, lhead) {
+                if (file->pid != pid)
+                        continue;
+
+                fpriv = file->driver_priv;
+                idr_for_each_entry(&fpriv->ctx_mgr.ctx_handles, ctx, id)
+                                amdgpu_ctx_priority_override(ctx, priority);
+        }
+        mutex_unlock(&adev->ddev->filelist_mutex);
+
+        put_pid(pid);
+
+        return 0;
+}
+
+int amdgpu_sched_ioctl(struct drm_device *dev, void *data,
+                       struct drm_file *filp)
+{
+        union drm_amdgpu_sched *args = data;
+        struct amdgpu_device *adev = dev->dev_private;
+        enum amd_sched_priority priority;
+        int r;
+
+        priority = amdgpu_to_sched_priority(args->in.priority);
+        if (args->in.flags || priority == AMD_SCHED_PRIORITY_INVALID)
+                return -EINVAL;
+
+        switch (args->in.op) {
+        case AMDGPU_SCHED_OP_PROCESS_PRIORITY_OVERRIDE:
+                r = amdgpu_sched_process_priority_override(adev,
+                                                           args->in.fd,
+                                                           priority);
+                break;
+        default:
+                DRM_ERROR("Invalid sched op specified: %d\n", args->in.op);
+                r = -EINVAL;
+                break;
+        }
+
+        return r;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h
new file mode 100644
index 000000000000..b28c067d3822
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2017 Valve Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Andres Rodriguez <andresx7@gmail.com>
+ */
+
+#ifndef __AMDGPU_SCHED_H__
+#define __AMDGPU_SCHED_H__
+
+#include <drm/drmP.h>
+
+enum amd_sched_priority amdgpu_to_sched_priority(int amdgpu_priority);
+int amdgpu_sched_ioctl(struct drm_device *dev, void *data,
+                       struct drm_file *filp);
+
+#endif // __AMDGPU_SCHED_H__
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index c586f44312f9..a4bf21f8f1c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -169,14 +169,14 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
  *
  * @sync: sync object to add fences from reservation object to
  * @resv: reservation object with embedded fence
- * @shared: true if we should only sync to the exclusive fence
+ * @explicit_sync: true if we should only sync to the exclusive fence
  *
  * Sync to the fence
  */
 int amdgpu_sync_resv(struct amdgpu_device *adev,
                      struct amdgpu_sync *sync,
                      struct reservation_object *resv,
-                     void *owner)
+                     void *owner, bool explicit_sync)
 {
         struct reservation_object_list *flist;
         struct dma_fence *f;
@@ -191,6 +191,9 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
         f = reservation_object_get_excl(resv);
         r = amdgpu_sync_fence(adev, sync, f);
 
+        if (explicit_sync)
+                return r;
+
         flist = reservation_object_get_list(resv);
         if (!flist || r)
                 return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
index dc7687993317..70d7e3a279a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
@@ -45,7 +45,8 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
 int amdgpu_sync_resv(struct amdgpu_device *adev,
                      struct amdgpu_sync *sync,
                      struct reservation_object *resv,
-                     void *owner);
+                     void *owner,
+                     bool explicit_sync);
 struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
                                      struct amdgpu_ring *ring);
 struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 34c99a3c8d2d..f337c316ec2c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -15,62 +15,6 @@
 #define AMDGPU_JOB_GET_TIMELINE_NAME(job) \
          job->base.s_fence->finished.ops->get_timeline_name(&job->base.s_fence->finished)
 
-TRACE_EVENT(amdgpu_ttm_tt_populate,
-            TP_PROTO(struct amdgpu_device *adev, uint64_t dma_address, uint64_t phys_address),
-            TP_ARGS(adev, dma_address, phys_address),
-            TP_STRUCT__entry(
-                                __field(uint16_t, domain)
-                                __field(uint8_t, bus)
-                                __field(uint8_t, slot)
-                                __field(uint8_t, func)
-                                __field(uint64_t, dma)
-                                __field(uint64_t, phys)
-                            ),
-            TP_fast_assign(
-                           __entry->domain = pci_domain_nr(adev->pdev->bus);
-                           __entry->bus = adev->pdev->bus->number;
-                           __entry->slot = PCI_SLOT(adev->pdev->devfn);
-                           __entry->func = PCI_FUNC(adev->pdev->devfn);
-                           __entry->dma = dma_address;
-                           __entry->phys = phys_address;
-                           ),
-            TP_printk("%04x:%02x:%02x.%x: 0x%llx => 0x%llx",
-                      (unsigned)__entry->domain,
-                      (unsigned)__entry->bus,
-                      (unsigned)__entry->slot,
-                      (unsigned)__entry->func,
-                      (unsigned long long)__entry->dma,
-                      (unsigned long long)__entry->phys)
-);
-
-TRACE_EVENT(amdgpu_ttm_tt_unpopulate,
-            TP_PROTO(struct amdgpu_device *adev, uint64_t dma_address, uint64_t phys_address),
-            TP_ARGS(adev, dma_address, phys_address),
-            TP_STRUCT__entry(
-                                __field(uint16_t, domain)
-                                __field(uint8_t, bus)
-                                __field(uint8_t, slot)
-                                __field(uint8_t, func)
-                                __field(uint64_t, dma)
-                                __field(uint64_t, phys)
-                            ),
-            TP_fast_assign(
-                           __entry->domain = pci_domain_nr(adev->pdev->bus);
-                           __entry->bus = adev->pdev->bus->number;
-                           __entry->slot = PCI_SLOT(adev->pdev->devfn);
-                           __entry->func = PCI_FUNC(adev->pdev->devfn);
-                           __entry->dma = dma_address;
-                           __entry->phys = phys_address;
-                           ),
-            TP_printk("%04x:%02x:%02x.%x: 0x%llx => 0x%llx",
-                      (unsigned)__entry->domain,
-                      (unsigned)__entry->bus,
-                      (unsigned)__entry->slot,
-                      (unsigned)__entry->func,
-                      (unsigned long long)__entry->dma,
-                      (unsigned long long)__entry->phys)
-);
-
 TRACE_EVENT(amdgpu_mm_rreg,
             TP_PROTO(unsigned did, uint32_t reg, uint32_t value),
             TP_ARGS(did, reg, value),
@@ -474,5 +418,5 @@ TRACE_EVENT(amdgpu_ttm_bo_move,
 
 /* This part must be outside protection */
 #undef TRACE_INCLUDE_PATH
-#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/amd/amdgpu
 #include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
index 89680d554ed8..b160b958e5fe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
@@ -1,5 +1,24 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright Red Hat Inc 2010.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
  * Author : Dave Airlie <airlied@redhat.com>
  */
 #include <drm/drmP.h>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index bc746131987f..ad5bf86ee8a3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -42,7 +42,9 @@
 #include <linux/swap.h>
 #include <linux/pagemap.h>
 #include <linux/debugfs.h>
+#include <linux/iommu.h>
 #include "amdgpu.h"
+#include "amdgpu_object.h"
 #include "amdgpu_trace.h"
 #include "bif/bif_4_1_d.h"
 
@@ -208,7 +210,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
                 placement->num_busy_placement = 1;
                 return;
         }
-        abo = container_of(bo, struct amdgpu_bo, tbo);
+        abo = ttm_to_amdgpu_bo(bo);
         switch (bo->mem.mem_type) {
         case TTM_PL_VRAM:
                 if (adev->mman.buffer_funcs &&
@@ -256,7 +258,7 @@ gtt:
 
 static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 {
-        struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo);
+        struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
 
         if (amdgpu_ttm_tt_get_usermm(bo->ttm))
                 return -EPERM;
@@ -288,97 +290,177 @@ static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
         return addr;
 }
 
-static int amdgpu_move_blit(struct ttm_buffer_object *bo,
-                            bool evict, bool no_wait_gpu,
-                            struct ttm_mem_reg *new_mem,
-                            struct ttm_mem_reg *old_mem)
+/**
+ * amdgpu_find_mm_node - Helper function finds the drm_mm_node
+ *  corresponding to @offset. It also modifies the offset to be
+ *  within the drm_mm_node returned
+ */
+static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
+                                               unsigned long *offset)
 {
-        struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
-        struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+        struct drm_mm_node *mm_node = mem->mm_node;
 
-        struct drm_mm_node *old_mm, *new_mm;
-        uint64_t old_start, old_size, new_start, new_size;
-        unsigned long num_pages;
-        struct dma_fence *fence = NULL;
-        int r;
+        while (*offset >= (mm_node->size << PAGE_SHIFT)) {
+                *offset -= (mm_node->size << PAGE_SHIFT);
+                ++mm_node;
+        }
+        return mm_node;
+}
 
-        BUILD_BUG_ON((PAGE_SIZE % AMDGPU_GPU_PAGE_SIZE) != 0);
+/**
+ * amdgpu_copy_ttm_mem_to_mem - Helper function for copy
+ *
+ * The function copies @size bytes from {src->mem + src->offset} to
+ * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
+ * move and different for a BO to BO copy.
+ *
+ * @f: Returns the last fence if multiple jobs are submitted.
+ */
+int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
+                               struct amdgpu_copy_mem *src,
+                               struct amdgpu_copy_mem *dst,
+                               uint64_t size,
+                               struct reservation_object *resv,
+                               struct dma_fence **f)
+{
+        struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+        struct drm_mm_node *src_mm, *dst_mm;
+        uint64_t src_node_start, dst_node_start, src_node_size,
+                 dst_node_size, src_page_offset, dst_page_offset;
+        struct dma_fence *fence = NULL;
+        int r = 0;
+        const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE *
+                                        AMDGPU_GPU_PAGE_SIZE);
 
         if (!ring->ready) {
                 DRM_ERROR("Trying to move memory with ring turned off.\n");
                 return -EINVAL;
         }
 
-        old_mm = old_mem->mm_node;
-        old_size = old_mm->size;
-        old_start = amdgpu_mm_node_addr(bo, old_mm, old_mem);
+        src_mm = amdgpu_find_mm_node(src->mem, &src->offset);
+        src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) +
+                                             src->offset;
+        src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset;
+        src_page_offset = src_node_start & (PAGE_SIZE - 1);
 
-        new_mm = new_mem->mm_node;
-        new_size = new_mm->size;
-        new_start = amdgpu_mm_node_addr(bo, new_mm, new_mem);
+        dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset);
+        dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) +
+                                             dst->offset;
+        dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset;
+        dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
 
-        num_pages = new_mem->num_pages;
         mutex_lock(&adev->mman.gtt_window_lock);
-        while (num_pages) {
-                unsigned long cur_pages = min(min(old_size, new_size),
-                                              (u64)AMDGPU_GTT_MAX_TRANSFER_SIZE);
-                uint64_t from = old_start, to = new_start;
+
+        while (size) {
+                unsigned long cur_size;
+                uint64_t from = src_node_start, to = dst_node_start;
                 struct dma_fence *next;
 
-                if (old_mem->mem_type == TTM_PL_TT &&
-                    !amdgpu_gtt_mgr_is_allocated(old_mem)) {
-                        r = amdgpu_map_buffer(bo, old_mem, cur_pages,
-                                              old_start, 0, ring, &from);
+                /* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst
+                 * begins at an offset, then adjust the size accordingly
+                 */
+                cur_size = min3(min(src_node_size, dst_node_size), size,
+                                GTT_MAX_BYTES);
+                if (cur_size + src_page_offset > GTT_MAX_BYTES ||
+                    cur_size + dst_page_offset > GTT_MAX_BYTES)
+                        cur_size -= max(src_page_offset, dst_page_offset);
+
+                /* Map only what needs to be accessed. Map src to window 0 and
+                 * dst to window 1
+                 */
+                if (src->mem->mem_type == TTM_PL_TT &&
+                    !amdgpu_gtt_mgr_is_allocated(src->mem)) {
+                        r = amdgpu_map_buffer(src->bo, src->mem,
+                                        PFN_UP(cur_size + src_page_offset),
+                                        src_node_start, 0, ring,
+                                        &from);
                         if (r)
                                 goto error;
+                        /* Adjust the offset because amdgpu_map_buffer returns
+                         * start of mapped page
+                         */
+                        from += src_page_offset;
                 }
 
-                if (new_mem->mem_type == TTM_PL_TT &&
-                    !amdgpu_gtt_mgr_is_allocated(new_mem)) {
-                        r = amdgpu_map_buffer(bo, new_mem, cur_pages,
-                                              new_start, 1, ring, &to);
+                if (dst->mem->mem_type == TTM_PL_TT &&
+                    !amdgpu_gtt_mgr_is_allocated(dst->mem)) {
+                        r = amdgpu_map_buffer(dst->bo, dst->mem,
+                                        PFN_UP(cur_size + dst_page_offset),
+                                        dst_node_start, 1, ring,
+                                        &to);
                         if (r)
                                 goto error;
+                        to += dst_page_offset;
                 }
 
-                r = amdgpu_copy_buffer(ring, from, to,
-                                       cur_pages * PAGE_SIZE,
-                                       bo->resv, &next, false, true);
+                r = amdgpu_copy_buffer(ring, from, to, cur_size,
+                                       resv, &next, false, true);
                 if (r)
                         goto error;
 
                 dma_fence_put(fence);
                 fence = next;
 
-                num_pages -= cur_pages;
-                if (!num_pages)
+                size -= cur_size;
+                if (!size)
                         break;
 
-                old_size -= cur_pages;
-                if (!old_size) {
-                        old_start = amdgpu_mm_node_addr(bo, ++old_mm, old_mem);
-                        old_size = old_mm->size;
+                src_node_size -= cur_size;
+                if (!src_node_size) {
+                        src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm,
+                                                             src->mem);
+                        src_node_size = (src_mm->size << PAGE_SHIFT);
                 } else {
-                        old_start += cur_pages * PAGE_SIZE;
+                        src_node_start += cur_size;
+                        src_page_offset = src_node_start & (PAGE_SIZE - 1);
                 }
-
-                new_size -= cur_pages;
-                if (!new_size) {
-                        new_start = amdgpu_mm_node_addr(bo, ++new_mm, new_mem);
-                        new_size = new_mm->size;
+                dst_node_size -= cur_size;
+                if (!dst_node_size) {
+                        dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm,
+                                                             dst->mem);
+                        dst_node_size = (dst_mm->size << PAGE_SHIFT);
                 } else {
-                        new_start += cur_pages * PAGE_SIZE;
+                        dst_node_start += cur_size;
+                        dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
                 }
         }
+error:
         mutex_unlock(&adev->mman.gtt_window_lock);
+        if (f)
+                *f = dma_fence_get(fence);
+        dma_fence_put(fence);
+        return r;
+}
+
+
+static int amdgpu_move_blit(struct ttm_buffer_object *bo,
+                            bool evict, bool no_wait_gpu,
+                            struct ttm_mem_reg *new_mem,
+                            struct ttm_mem_reg *old_mem)
+{
+        struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
+        struct amdgpu_copy_mem src, dst;
+        struct dma_fence *fence = NULL;
+        int r;
+
+        src.bo = bo;
+        dst.bo = bo;
+        src.mem = old_mem;
+        dst.mem = new_mem;
+        src.offset = 0;
+        dst.offset = 0;
+
+        r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,
+                                       new_mem->num_pages << PAGE_SHIFT,
+                                       bo->resv, &fence);
+        if (r)
+                goto error;
 
         r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
         dma_fence_put(fence);
         return r;
 
 error:
-        mutex_unlock(&adev->mman.gtt_window_lock);
-
         if (fence)
                 dma_fence_wait(fence, false);
         dma_fence_put(fence);
@@ -483,7 +565,7 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo,
         int r;
 
         /* Can't move a pinned BO */
-        abo = container_of(bo, struct amdgpu_bo, tbo);
+        abo = ttm_to_amdgpu_bo(bo);
         if (WARN_ON_ONCE(abo->pin_count > 0))
                 return -EINVAL;
 
@@ -581,13 +663,12 @@ static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_re
 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
                                            unsigned long page_offset)
 {
-        struct drm_mm_node *mm = bo->mem.mm_node;
-        uint64_t size = mm->size;
-        uint64_t offset = page_offset;
+        struct drm_mm_node *mm;
+        unsigned long offset = (page_offset << PAGE_SHIFT);
 
-        page_offset = do_div(offset, size);
-        mm += offset;
-        return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start + page_offset;
+        mm = amdgpu_find_mm_node(&bo->mem, &offset);
+        return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start +
+                (offset >> PAGE_SHIFT);
 }
 
 /*
@@ -608,6 +689,7 @@ struct amdgpu_ttm_tt {
         spinlock_t              guptasklock;
         struct list_head        guptasks;
         atomic_t                mmu_invalidations;
+        uint32_t                last_set_pages;
         struct list_head        list;
 };
 
@@ -621,6 +703,8 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
         if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
                 flags |= FOLL_WRITE;
 
+        down_read(&current->mm->mmap_sem);
+
         if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
                 /* check that we only use anonymous memory
                    to prevent problems with writeback */
@@ -628,8 +712,10 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
                 struct vm_area_struct *vma;
 
                 vma = find_vma(gtt->usermm, gtt->userptr);
-                if (!vma || vma->vm_file || vma->vm_end < end)
+                if (!vma || vma->vm_file || vma->vm_end < end) {
+                        up_read(&current->mm->mmap_sem);
                         return -EPERM;
+                }
         }
 
         do {
@@ -656,42 +742,44 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
 
         } while (pinned < ttm->num_pages);
 
+        up_read(&current->mm->mmap_sem);
         return 0;
 
 release_pages:
-        release_pages(pages, pinned, 0);
+        release_pages(pages, pinned);
+        up_read(&current->mm->mmap_sem);
         return r;
 }
 
-static void amdgpu_trace_dma_map(struct ttm_tt *ttm)
+void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
 {
-        struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
         struct amdgpu_ttm_tt *gtt = (void *)ttm;
         unsigned i;
 
-        if (unlikely(trace_amdgpu_ttm_tt_populate_enabled())) {
-                for (i = 0; i < ttm->num_pages; i++) {
-                        trace_amdgpu_ttm_tt_populate(
-                                adev,
-                                gtt->ttm.dma_address[i],
-                                page_to_phys(ttm->pages[i]));
-                }
+        gtt->last_set_pages = atomic_read(&gtt->mmu_invalidations);
+        for (i = 0; i < ttm->num_pages; ++i) {
+                if (ttm->pages[i])
+                        put_page(ttm->pages[i]);
+
+                ttm->pages[i] = pages ? pages[i] : NULL;
         }
 }
 
-static void amdgpu_trace_dma_unmap(struct ttm_tt *ttm)
+void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm)
 {
-        struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
         struct amdgpu_ttm_tt *gtt = (void *)ttm;
         unsigned i;
 
-        if (unlikely(trace_amdgpu_ttm_tt_unpopulate_enabled())) {
-                for (i = 0; i < ttm->num_pages; i++) {
-                        trace_amdgpu_ttm_tt_unpopulate(
-                                adev,
-                                gtt->ttm.dma_address[i],
-                                page_to_phys(ttm->pages[i]));
-                }
+        for (i = 0; i < ttm->num_pages; ++i) {
+                struct page *page = ttm->pages[i];
+
+                if (!page)
+                        continue;
+
+                if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
+                        set_page_dirty(page);
+
+                mark_page_accessed(page);
         }
 }
 
@@ -721,8 +809,6 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
         drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
                                          gtt->ttm.dma_address, ttm->num_pages);
 
-        amdgpu_trace_dma_map(ttm);
-
         return 0;
 
 release_sg:
@@ -734,7 +820,6 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
 {
         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
         struct amdgpu_ttm_tt *gtt = (void *)ttm;
-        struct sg_page_iter sg_iter;
 
         int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
         enum dma_data_direction direction = write ?
@@ -747,16 +832,7 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
         /* free the sg table and pages again */
         dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
 
-        for_each_sg_page(ttm->sg->sgl, &sg_iter, ttm->sg->nents, 0) {
-                struct page *page = sg_page_iter_page(&sg_iter);
-                if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
-                        set_page_dirty(page);
-
-                mark_page_accessed(page);
-                put_page(page);
-        }
-
-        amdgpu_trace_dma_unmap(ttm);
+        amdgpu_ttm_tt_mark_user_pages(ttm);
 
         sg_free_table(ttm->sg);
 }
@@ -818,7 +894,6 @@ int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
         struct ttm_tt *ttm = bo->ttm;
         struct ttm_mem_reg tmp;
-
         struct ttm_placement placement;
         struct ttm_place placements;
         int r;
@@ -834,7 +909,8 @@ int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
         placement.busy_placement = &placements;
         placements.fpfn = 0;
         placements.lpfn = adev->mc.gart_size >> PAGE_SHIFT;
-        placements.flags = bo->mem.placement | TTM_PL_FLAG_TT;
+        placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
+                TTM_PL_FLAG_TT;
 
         r = ttm_bo_mem_space(bo, &placement, &tmp, true, false);
         if (unlikely(r))
@@ -941,8 +1017,6 @@ static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
 {
         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
         struct amdgpu_ttm_tt *gtt = (void *)ttm;
-        unsigned i;
-        int r;
         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
 
         if (ttm->state != tt_unpopulated)
@@ -962,52 +1036,26 @@ static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
                 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
                                                  gtt->ttm.dma_address, ttm->num_pages);
                 ttm->state = tt_unbound;
-                r = 0;
-                goto trace_mappings;
+                return 0;
         }
 
 #ifdef CONFIG_SWIOTLB
         if (swiotlb_nr_tbl()) {
-                r = ttm_dma_populate(&gtt->ttm, adev->dev);
-                goto trace_mappings;
+                return ttm_dma_populate(&gtt->ttm, adev->dev);
         }
 #endif
 
-        r = ttm_pool_populate(ttm);
-        if (r) {
-                return r;
-        }
-
-        for (i = 0; i < ttm->num_pages; i++) {
-                gtt->ttm.dma_address[i] = pci_map_page(adev->pdev, ttm->pages[i],
-                                                       0, PAGE_SIZE,
-                                                       PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(adev->pdev, gtt->ttm.dma_address[i])) {
-                        while (i--) {
-                                pci_unmap_page(adev->pdev, gtt->ttm.dma_address[i],
-                                               PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-                                gtt->ttm.dma_address[i] = 0;
-                        }
-                        ttm_pool_unpopulate(ttm);
-                        return -EFAULT;
-                }
-        }
-
-        r = 0;
-trace_mappings:
-        if (likely(!r))
-                amdgpu_trace_dma_map(ttm);
-        return r;
+        return ttm_populate_and_map_pages(adev->dev, &gtt->ttm);
 }
 
 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
 {
         struct amdgpu_device *adev;
         struct amdgpu_ttm_tt *gtt = (void *)ttm;
-        unsigned i;
         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
 
         if (gtt && gtt->userptr) {
+                amdgpu_ttm_tt_set_user_pages(ttm, NULL);
                 kfree(ttm->sg);
                 ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
                 return;
@@ -1018,8 +1066,6 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
 
         adev = amdgpu_ttm_adev(ttm->bdev);
 
-        amdgpu_trace_dma_unmap(ttm);
-
 #ifdef CONFIG_SWIOTLB
         if (swiotlb_nr_tbl()) {
                 ttm_dma_unpopulate(&gtt->ttm, adev->dev);
@@ -1027,14 +1073,7 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
         }
 #endif
 
-        for (i = 0; i < ttm->num_pages; i++) {
-                if (gtt->ttm.dma_address[i]) {
-                        pci_unmap_page(adev->pdev, gtt->ttm.dma_address[i],
-                                       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-                }
-        }
-
-        ttm_pool_unpopulate(ttm);
+        ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);
 }
 
 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
@@ -1051,6 +1090,7 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
         spin_lock_init(&gtt->guptasklock);
         INIT_LIST_HEAD(&gtt->guptasks);
         atomic_set(&gtt->mmu_invalidations, 0);
+        gtt->last_set_pages = 0;
 
         return 0;
 }
@@ -1103,6 +1143,16 @@ bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
         return prev_invalidated != *last_invalidated;
 }
 
+bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
+{
+        struct amdgpu_ttm_tt *gtt = (void *)ttm;
+
+        if (gtt == NULL || !gtt->userptr)
+                return false;
+
+        return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;
+}
+
 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
 {
         struct amdgpu_ttm_tt *gtt = (void *)ttm;
@@ -1143,9 +1193,6 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
         unsigned long num_pages = bo->mem.num_pages;
         struct drm_mm_node *node = bo->mem.mm_node;
 
-        if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET)
-                return ttm_bo_eviction_valuable(bo, place);
-
         switch (bo->mem.mem_type) {
         case TTM_PL_TT:
                 return true;
@@ -1160,7 +1207,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
                         num_pages -= node->size;
                         ++node;
                 }
-                break;
+                return false;
 
         default:
                 break;
@@ -1173,9 +1220,9 @@ static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
                                     unsigned long offset,
                                     void *buf, int len, int write)
 {
-        struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo);
+        struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
         struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
-        struct drm_mm_node *nodes = abo->tbo.mem.mm_node;
+        struct drm_mm_node *nodes;
         uint32_t value = 0;
         int ret = 0;
         uint64_t pos;
@@ -1184,10 +1231,7 @@ static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
         if (bo->mem.mem_type != TTM_PL_VRAM)
                 return -EIO;
 
-        while (offset >= (nodes->size << PAGE_SHIFT)) {
-                offset -= nodes->size << PAGE_SHIFT;
-                ++nodes;
-        }
+        nodes = amdgpu_find_mm_node(&abo->tbo.mem, &offset);
         pos = (nodes->start << PAGE_SHIFT) + offset;
 
         while (len && pos < adev->mc.mc_vram_size) {
@@ -1202,14 +1246,14 @@ static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
                 }
 
                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
-                WREG32(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
-                WREG32(mmMM_INDEX_HI, aligned_pos >> 31);
+                WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
+                WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31);
                 if (!write || mask != 0xffffffff)
-                        value = RREG32(mmMM_DATA);
+                        value = RREG32_NO_KIQ(mmMM_DATA);
                 if (write) {
                         value &= ~mask;
                         value |= (*(uint32_t *)buf << shift) & mask;
-                        WREG32(mmMM_DATA, value);
+                        WREG32_NO_KIQ(mmMM_DATA, value);
                 }
                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
                 if (!write) {
@@ -1286,6 +1330,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
         /* Change the size here instead of the init above so only lpfn is affected */
         amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);
 
+        /*
+         *The reserved vram for firmware must be pinned to the specified
+         *place on the VRAM, so reserve it early.
+         */
+        r = amdgpu_fw_reserve_vram_init(adev);
+        if (r) {
+                return r;
+        }
+
         r = amdgpu_bo_create_kernel(adev, adev->mc.stolen_size, PAGE_SIZE,
                                     AMDGPU_GEM_DOMAIN_VRAM,
                                     &adev->stolen_vga_memory,
@@ -1510,7 +1563,8 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
         job->vm_needs_flush = vm_needs_flush;
         if (resv) {
                 r = amdgpu_sync_resv(adev, &job->sync, resv,
-                                     AMDGPU_FENCE_OWNER_UNDEFINED);
+                                     AMDGPU_FENCE_OWNER_UNDEFINED,
+                                     false);
                 if (r) {
                         DRM_ERROR("sync failed (%d).\n", r);
                         goto error_free;
@@ -1557,8 +1611,8 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
                        struct dma_fence **fence)
 {
         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
-        /* max_bytes applies to SDMA_OP_PTEPDE as well as SDMA_OP_CONST_FILL*/
-        uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
+        uint32_t max_bytes = 8 *
+                        adev->vm_manager.vm_pte_funcs->set_max_nums_pte_pde;
         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 
         struct drm_mm_node *mm_node;
@@ -1590,8 +1644,8 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
                 ++mm_node;
         }
 
-        /* 10 double words for each SDMA_OP_PTEPDE cmd */
-        num_dw = num_loops * 10;
+        /* num of dwords for each SDMA_OP_PTEPDE cmd */
+        num_dw = num_loops * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw;
 
         /* for IB padding */
         num_dw += 64;
@@ -1602,7 +1656,7 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 
         if (resv) {
                 r = amdgpu_sync_resv(adev, &job->sync, resv,
-                                     AMDGPU_FENCE_OWNER_UNDEFINED);
+                                     AMDGPU_FENCE_OWNER_UNDEFINED, false);
                 if (r) {
                         DRM_ERROR("sync failed (%d).\n", r);
                         goto error_free;
@@ -1697,9 +1751,9 @@ static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
                         return result;
 
                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
-                WREG32(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
-                WREG32(mmMM_INDEX_HI, *pos >> 31);
-                value = RREG32(mmMM_DATA);
+                WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
+                WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
+                value = RREG32_NO_KIQ(mmMM_DATA);
                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 
                 r = put_user(value, (uint32_t *)buf);
@@ -1715,10 +1769,50 @@ static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
         return result;
 }
 
+static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
+                                    size_t size, loff_t *pos)
+{
+        struct amdgpu_device *adev = file_inode(f)->i_private;
+        ssize_t result = 0;
+        int r;
+
+        if (size & 0x3 || *pos & 0x3)
+                return -EINVAL;
+
+        if (*pos >= adev->mc.mc_vram_size)
+                return -ENXIO;
+
+        while (size) {
+                unsigned long flags;
+                uint32_t value;
+
+                if (*pos >= adev->mc.mc_vram_size)
+                        return result;
+
+                r = get_user(value, (uint32_t *)buf);
+                if (r)
+                        return r;
+
+                spin_lock_irqsave(&adev->mmio_idx_lock, flags);
+                WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
+                WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
+                WREG32_NO_KIQ(mmMM_DATA, value);
+                spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
+
+                result += 4;
+                buf += 4;
+                *pos += 4;
+                size -= 4;
+        }
+
+        return result;
+}
+
 static const struct file_operations amdgpu_ttm_vram_fops = {
         .owner = THIS_MODULE,
         .read = amdgpu_ttm_vram_read,
-        .llseek = default_llseek
+        .write = amdgpu_ttm_vram_write,
+        .llseek = default_llseek,
 };
 
 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
@@ -1770,6 +1864,53 @@ static const struct file_operations amdgpu_ttm_gtt_fops = {
 
 #endif
 
+static ssize_t amdgpu_iova_to_phys_read(struct file *f, char __user *buf,
+                                   size_t size, loff_t *pos)
+{
+        struct amdgpu_device *adev = file_inode(f)->i_private;
+        int r;
+        uint64_t phys;
+        struct iommu_domain *dom;
+
+        // always return 8 bytes
+        if (size != 8)
+                return -EINVAL;
+
+        // only accept page addresses
+        if (*pos & 0xFFF)
+                return -EINVAL;
+
+        dom = iommu_get_domain_for_dev(adev->dev);
+        if (dom)
+                phys = iommu_iova_to_phys(dom, *pos);
+        else
+                phys = *pos;
+
+        r = copy_to_user(buf, &phys, 8);
+        if (r)
+                return -EFAULT;
+
+        return 8;
+}
+
+static const struct file_operations amdgpu_ttm_iova_fops = {
+        .owner = THIS_MODULE,
+        .read = amdgpu_iova_to_phys_read,
+        .llseek = default_llseek
+};
+
+static const struct {
+        char *name;
+        const struct file_operations *fops;
+        int domain;
+} ttm_debugfs_entries[] = {
+        { "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM },
+#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
+        { "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT },
+#endif
+        { "amdgpu_iova", &amdgpu_ttm_iova_fops, TTM_PL_SYSTEM },
+};
+
 #endif
 
 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
@@ -1780,22 +1921,21 @@ static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
         struct drm_minor *minor = adev->ddev->primary;
         struct dentry *ent, *root = minor->debugfs_root;
 
-        ent = debugfs_create_file("amdgpu_vram", S_IFREG | S_IRUGO, root,
-                                  adev, &amdgpu_ttm_vram_fops);
-        if (IS_ERR(ent))
-                return PTR_ERR(ent);
-        i_size_write(ent->d_inode, adev->mc.mc_vram_size);
-        adev->mman.vram = ent;
-
-#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
-        ent = debugfs_create_file("amdgpu_gtt", S_IFREG | S_IRUGO, root,
-                                  adev, &amdgpu_ttm_gtt_fops);
-        if (IS_ERR(ent))
-                return PTR_ERR(ent);
-        i_size_write(ent->d_inode, adev->mc.gart_size);
-        adev->mman.gtt = ent;
+        for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) {
+                ent = debugfs_create_file(
+                                ttm_debugfs_entries[count].name,
+                                S_IFREG | S_IRUGO, root,
+                                adev,
+                                ttm_debugfs_entries[count].fops);
+                if (IS_ERR(ent))
+                        return PTR_ERR(ent);
+                if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM)
+                        i_size_write(ent->d_inode, adev->mc.mc_vram_size);
+                else if (ttm_debugfs_entries[count].domain == TTM_PL_TT)
+                        i_size_write(ent->d_inode, adev->mc.gart_size);
+                adev->mman.debugfs_entries[count] = ent;
+        }
 
-#endif
         count = ARRAY_SIZE(amdgpu_ttm_debugfs_list);
 
 #ifdef CONFIG_SWIOTLB
@@ -1805,7 +1945,6 @@ static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
 
         return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);
 #else
-
         return 0;
 #endif
 }
@@ -1813,14 +1952,9 @@ static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
 {
 #if defined(CONFIG_DEBUG_FS)
+        unsigned i;
 
-        debugfs_remove(adev->mman.vram);
-        adev->mman.vram = NULL;
-
-#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
-        debugfs_remove(adev->mman.gtt);
-        adev->mman.gtt = NULL;
-#endif
-
+        for (i = 0; i < ARRAY_SIZE(ttm_debugfs_entries); i++)
+                debugfs_remove(adev->mman.debugfs_entries[i]);
 #endif
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 43093bffa2cf..abd4084982a3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -24,6 +24,7 @@
 #ifndef __AMDGPU_TTM_H__
 #define __AMDGPU_TTM_H__
 
+#include "amdgpu.h"
 #include "gpu_scheduler.h"
 
 #define AMDGPU_PL_GDS           (TTM_PL_PRIV + 0)
@@ -45,8 +46,7 @@ struct amdgpu_mman {
         bool                            initialized;
 
 #if defined(CONFIG_DEBUG_FS)
-        struct dentry                   *vram;
-        struct dentry                   *gtt;
+        struct dentry                   *debugfs_entries[8];
 #endif
 
         /* buffer handling */
@@ -58,6 +58,12 @@ struct amdgpu_mman {
         struct amd_sched_entity                 entity;
 };
 
+struct amdgpu_copy_mem {
+        struct ttm_buffer_object        *bo;
+        struct ttm_mem_reg              *mem;
+        unsigned long                   offset;
+};
+
 extern const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func;
 extern const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func;
 
@@ -72,6 +78,12 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
                        struct reservation_object *resv,
                        struct dma_fence **fence, bool direct_submit,
                        bool vm_needs_flush);
+int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
+                               struct amdgpu_copy_mem *src,
+                               struct amdgpu_copy_mem *dst,
+                               uint64_t size,
+                               struct reservation_object *resv,
+                               struct dma_fence **f);
 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
                         uint64_t src_data,
                         struct reservation_object *resv,
@@ -82,4 +94,20 @@ bool amdgpu_ttm_is_bound(struct ttm_tt *ttm);
 int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem);
 int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);
 
+int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
+void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);
+void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm);
+int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
+                                     uint32_t flags);
+bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm);
+struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm);
+bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
+                                  unsigned long end);
+bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
+                                       int *last_invalidated);
+bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm);
+bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
+uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
+                                 struct ttm_mem_reg *mem);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 36c763310df5..65649026b836 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -270,12 +270,8 @@ amdgpu_ucode_get_load_type(struct amdgpu_device *adev, int load_type)
                 else
                         return AMDGPU_FW_LOAD_SMU;
         case CHIP_VEGA10:
-                if (!load_type)
-                        return AMDGPU_FW_LOAD_DIRECT;
-                else
-                        return AMDGPU_FW_LOAD_PSP;
         case CHIP_RAVEN:
-                if (load_type != 2)
+                if (!load_type)
                         return AMDGPU_FW_LOAD_DIRECT;
                 else
                         return AMDGPU_FW_LOAD_PSP;
@@ -364,8 +360,6 @@ static int amdgpu_ucode_patch_jt(struct amdgpu_firmware_info *ucode,
 int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 {
         struct amdgpu_bo **bo = &adev->firmware.fw_buf;
-        uint64_t fw_mc_addr;
-        void *fw_buf_ptr = NULL;
         uint64_t fw_offset = 0;
         int i, err;
         struct amdgpu_firmware_info *ucode = NULL;
@@ -376,37 +370,39 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
                 return 0;
         }
 
-        err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true,
-                                amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
-                                AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-                                NULL, NULL, 0, bo);
-        if (err) {
-                dev_err(adev->dev, "(%d) Firmware buffer allocate failed\n", err);
-                goto failed;
-        }
+        if (!amdgpu_sriov_vf(adev) || !adev->in_sriov_reset) {
+                err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true,
+                                        amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
+                                        AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
+                                        NULL, NULL, 0, bo);
+                if (err) {
+                        dev_err(adev->dev, "(%d) Firmware buffer allocate failed\n", err);
+                        goto failed;
+                }
 
-        err = amdgpu_bo_reserve(*bo, false);
-        if (err) {
-                dev_err(adev->dev, "(%d) Firmware buffer reserve failed\n", err);
-                goto failed_reserve;
-        }
+                err = amdgpu_bo_reserve(*bo, false);
+                if (err) {
+                        dev_err(adev->dev, "(%d) Firmware buffer reserve failed\n", err);
+                        goto failed_reserve;
+                }
 
-        err = amdgpu_bo_pin(*bo, amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
-                                &fw_mc_addr);
-        if (err) {
-                dev_err(adev->dev, "(%d) Firmware buffer pin failed\n", err);
-                goto failed_pin;
-        }
+                err = amdgpu_bo_pin(*bo, amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
+                                        &adev->firmware.fw_buf_mc);
+                if (err) {
+                        dev_err(adev->dev, "(%d) Firmware buffer pin failed\n", err);
+                        goto failed_pin;
+                }
 
-        err = amdgpu_bo_kmap(*bo, &fw_buf_ptr);
-        if (err) {
-                dev_err(adev->dev, "(%d) Firmware buffer kmap failed\n", err);
-                goto failed_kmap;
-        }
+                err = amdgpu_bo_kmap(*bo, &adev->firmware.fw_buf_ptr);
+                if (err) {
+                        dev_err(adev->dev, "(%d) Firmware buffer kmap failed\n", err);
+                        goto failed_kmap;
+                }
 
-        amdgpu_bo_unreserve(*bo);
+                amdgpu_bo_unreserve(*bo);
+        }
 
-        memset(fw_buf_ptr, 0, adev->firmware.fw_size);
+        memset(adev->firmware.fw_buf_ptr, 0, adev->firmware.fw_size);
 
         /*
          * if SMU loaded firmware, it needn't add SMC, UVD, and VCE
@@ -425,14 +421,14 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
                 ucode = &adev->firmware.ucode[i];
                 if (ucode->fw) {
                         header = (const struct common_firmware_header *)ucode->fw->data;
-                        amdgpu_ucode_init_single_fw(adev, ucode, fw_mc_addr + fw_offset,
-                                                    (void *)((uint8_t *)fw_buf_ptr + fw_offset));
+                        amdgpu_ucode_init_single_fw(adev, ucode, adev->firmware.fw_buf_mc + fw_offset,
+                                                    adev->firmware.fw_buf_ptr + fw_offset);
                         if (i == AMDGPU_UCODE_ID_CP_MEC1 &&
                             adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
                                 const struct gfx_firmware_header_v1_0 *cp_hdr;
                                 cp_hdr = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data;
-                                amdgpu_ucode_patch_jt(ucode, fw_mc_addr + fw_offset,
-                                                    fw_buf_ptr + fw_offset);
+                                amdgpu_ucode_patch_jt(ucode,  adev->firmware.fw_buf_mc + fw_offset,
+                                                    adev->firmware.fw_buf_ptr + fw_offset);
                                 fw_offset += ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE);
                         }
                         fw_offset += ALIGN(ucode->ucode_size, PAGE_SIZE);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index e19928dae8e3..e8bd50cf9785 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -269,6 +269,7 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 
 int amdgpu_uvd_sw_fini(struct amdgpu_device *adev)
 {
+        int i;
         kfree(adev->uvd.saved_bo);
 
         amd_sched_entity_fini(&adev->uvd.ring.sched, &adev->uvd.entity);
@@ -279,6 +280,9 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev)
 
         amdgpu_ring_fini(&adev->uvd.ring);
 
+        for (i = 0; i < AMDGPU_MAX_UVD_ENC_RINGS; ++i)
+                amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
+
         release_firmware(adev->uvd.fw);
 
         return 0;
@@ -410,10 +414,10 @@ static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx)
         uint64_t addr = amdgpu_uvd_get_addr_from_ctx(ctx);
         int r = 0;
 
-        mapping = amdgpu_cs_find_mapping(ctx->parser, addr, &bo);
-        if (mapping == NULL) {
+        r = amdgpu_cs_find_mapping(ctx->parser, addr, &bo, &mapping);
+        if (r) {
                 DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
-                return -EINVAL;
+                return r;
         }
 
         if (!ctx->parser->adev->uvd.address_64_bit) {
@@ -737,10 +741,10 @@ static int amdgpu_uvd_cs_pass2(struct amdgpu_uvd_cs_ctx *ctx)
         uint64_t addr = amdgpu_uvd_get_addr_from_ctx(ctx);
         int r;
 
-        mapping = amdgpu_cs_find_mapping(ctx->parser, addr, &bo);
-        if (mapping == NULL) {
+        r = amdgpu_cs_find_mapping(ctx->parser, addr, &bo, &mapping);
+        if (r) {
                 DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
-                return -EINVAL;
+                return r;
         }
 
         start = amdgpu_bo_gpu_offset(bo);
@@ -917,10 +921,6 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx)
                 return -EINVAL;
         }
 
-        r = amdgpu_cs_sysvm_access_required(parser);
-        if (r)
-                return r;
-
         ctx.parser = parser;
         ctx.buf_sizes = buf_sizes;
         ctx.ib_idx = ib_idx;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index c855366521ab..2918de2f39ec 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -559,6 +559,7 @@ static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx,
         struct amdgpu_bo_va_mapping *mapping;
         struct amdgpu_bo *bo;
         uint64_t addr;
+        int r;
 
         if (index == 0xffffffff)
                 index = 0;
@@ -567,11 +568,11 @@ static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx,
                ((uint64_t)amdgpu_get_ib_value(p, ib_idx, hi)) << 32;
         addr += ((uint64_t)size) * ((uint64_t)index);
 
-        mapping = amdgpu_cs_find_mapping(p, addr, &bo);
-        if (mapping == NULL) {
+        r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping);
+        if (r) {
                 DRM_ERROR("Can't find BO for addr 0x%010Lx %d %d %d %d\n",
                           addr, lo, hi, size, index);
-                return -EINVAL;
+                return r;
         }
 
         if ((addr + (uint64_t)size) >
@@ -647,15 +648,11 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
         uint32_t allocated = 0;
         uint32_t tmp, handle = 0;
         uint32_t *size = &tmp;
-        int i, r, idx = 0;
+        int i, r = 0, idx = 0;
 
         p->job->vm = NULL;
         ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
 
-        r = amdgpu_cs_sysvm_access_required(p);
-        if (r)
-                return r;
-
         while (idx < ib->length_dw) {
                 uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
                 uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c
index 45ac91861965..7f7097931c6f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c
@@ -25,30 +25,26 @@
 #include "amdgpu_vf_error.h"
 #include "mxgpu_ai.h"
 
-#define AMDGPU_VF_ERROR_ENTRY_SIZE    16 
-
-/* struct error_entry - amdgpu VF error information. */
-struct amdgpu_vf_error_buffer {
-        int read_count;
-        int write_count;
-        uint16_t code[AMDGPU_VF_ERROR_ENTRY_SIZE];
-        uint16_t flags[AMDGPU_VF_ERROR_ENTRY_SIZE];
-        uint64_t data[AMDGPU_VF_ERROR_ENTRY_SIZE];
-};
-
-struct amdgpu_vf_error_buffer admgpu_vf_errors;
-
-
-void amdgpu_vf_error_put(uint16_t sub_error_code, uint16_t error_flags, uint64_t error_data)
+void amdgpu_vf_error_put(struct amdgpu_device *adev,
+                         uint16_t sub_error_code,
+                         uint16_t error_flags,
+                         uint64_t error_data)
 {
         int index;
-        uint16_t error_code = AMDGIM_ERROR_CODE(AMDGIM_ERROR_CATEGORY_VF, sub_error_code);
+        uint16_t error_code;
 
-        index = admgpu_vf_errors.write_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
-        admgpu_vf_errors.code [index] = error_code;
-        admgpu_vf_errors.flags [index] = error_flags;
-        admgpu_vf_errors.data [index] = error_data;
-        admgpu_vf_errors.write_count ++;
+        if (!amdgpu_sriov_vf(adev))
+                return;
+
+        error_code = AMDGIM_ERROR_CODE(AMDGIM_ERROR_CATEGORY_VF, sub_error_code);
+
+        mutex_lock(&adev->virt.vf_errors.lock);
+        index = adev->virt.vf_errors.write_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
+        adev->virt.vf_errors.code [index] = error_code;
+        adev->virt.vf_errors.flags [index] = error_flags;
+        adev->virt.vf_errors.data [index] = error_data;
+        adev->virt.vf_errors.write_count ++;
+        mutex_unlock(&adev->virt.vf_errors.lock);
 }
 
 
@@ -58,7 +54,8 @@ void amdgpu_vf_error_trans_all(struct amdgpu_device *adev)
         u32 data1, data2, data3;
         int index;
 
-        if ((NULL == adev) || (!amdgpu_sriov_vf(adev)) || (!adev->virt.ops) || (!adev->virt.ops->trans_msg)) {
+        if ((NULL == adev) || (!amdgpu_sriov_vf(adev)) ||
+            (!adev->virt.ops) || (!adev->virt.ops->trans_msg)) {
                 return;
         }
 /*
@@ -68,18 +65,22 @@ void amdgpu_vf_error_trans_all(struct amdgpu_device *adev)
                 return;
         }
 */
+
+        mutex_lock(&adev->virt.vf_errors.lock);
         /* The errors are overlay of array, correct read_count as full. */
-        if (admgpu_vf_errors.write_count - admgpu_vf_errors.read_count > AMDGPU_VF_ERROR_ENTRY_SIZE) {
-                admgpu_vf_errors.read_count = admgpu_vf_errors.write_count - AMDGPU_VF_ERROR_ENTRY_SIZE;
+        if (adev->virt.vf_errors.write_count - adev->virt.vf_errors.read_count > AMDGPU_VF_ERROR_ENTRY_SIZE) {
+                adev->virt.vf_errors.read_count = adev->virt.vf_errors.write_count - AMDGPU_VF_ERROR_ENTRY_SIZE;
         }
 
-        while (admgpu_vf_errors.read_count < admgpu_vf_errors.write_count) {
-                index =admgpu_vf_errors.read_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
-                data1 = AMDGIM_ERROR_CODE_FLAGS_TO_MAILBOX (admgpu_vf_errors.code[index], admgpu_vf_errors.flags[index]);
-                data2 = admgpu_vf_errors.data[index] & 0xFFFFFFFF;
-                data3 = (admgpu_vf_errors.data[index] >> 32) & 0xFFFFFFFF;
+        while (adev->virt.vf_errors.read_count < adev->virt.vf_errors.write_count) {
+                index =adev->virt.vf_errors.read_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
+                data1 = AMDGIM_ERROR_CODE_FLAGS_TO_MAILBOX(adev->virt.vf_errors.code[index],
+                                                           adev->virt.vf_errors.flags[index]);
+                data2 = adev->virt.vf_errors.data[index] & 0xFFFFFFFF;
+                data3 = (adev->virt.vf_errors.data[index] >> 32) & 0xFFFFFFFF;
 
                 adev->virt.ops->trans_msg(adev, IDH_LOG_VF_ERROR, data1, data2, data3);
-                admgpu_vf_errors.read_count ++;
+                adev->virt.vf_errors.read_count ++;
         }
+        mutex_unlock(&adev->virt.vf_errors.lock);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h
index 2a3278ec76ba..6436bd053325 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h
@@ -56,7 +56,10 @@ enum AMDGIM_ERROR_CATEGORY {
         AMDGIM_ERROR_CATEGORY_MAX
 };
 
-void amdgpu_vf_error_put(uint16_t sub_error_code, uint16_t error_flags, uint64_t error_data);
+void amdgpu_vf_error_put(struct amdgpu_device *adev,
+                         uint16_t sub_error_code,
+                         uint16_t error_flags,
+                         uint64_t error_data);
 void amdgpu_vf_error_trans_all (struct amdgpu_device *adev);
 
 #endif /* __VF_ERROR_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index ab05121b9272..6738df836a70 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -22,7 +22,7 @@
  */
 
 #include "amdgpu.h"
-#define MAX_KIQ_REG_WAIT        100000
+#define MAX_KIQ_REG_WAIT        100000000 /* in usecs */
 
 int amdgpu_allocate_static_csa(struct amdgpu_device *adev)
 {
@@ -114,27 +114,25 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
 {
         signed long r;
-        uint32_t val;
-        struct dma_fence *f;
+        unsigned long flags;
+        uint32_t val, seq;
         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
         struct amdgpu_ring *ring = &kiq->ring;
 
         BUG_ON(!ring->funcs->emit_rreg);
 
-        mutex_lock(&kiq->ring_mutex);
+        spin_lock_irqsave(&kiq->ring_lock, flags);
         amdgpu_ring_alloc(ring, 32);
         amdgpu_ring_emit_rreg(ring, reg);
-        amdgpu_fence_emit(ring, &f);
+        amdgpu_fence_emit_polling(ring, &seq);
         amdgpu_ring_commit(ring);
-        mutex_unlock(&kiq->ring_mutex);
+        spin_unlock_irqrestore(&kiq->ring_lock, flags);
 
-        r = dma_fence_wait_timeout(f, false, msecs_to_jiffies(MAX_KIQ_REG_WAIT));
-        dma_fence_put(f);
+        r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
         if (r < 1) {
-                DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+                DRM_ERROR("wait for kiq fence error: %ld\n", r);
                 return ~0;
         }
-
         val = adev->wb.wb[adev->virt.reg_val_offs];
 
         return val;
@@ -143,23 +141,23 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
 void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 {
         signed long r;
-        struct dma_fence *f;
+        unsigned long flags;
+        uint32_t seq;
         struct amdgpu_kiq *kiq = &adev->gfx.kiq;
         struct amdgpu_ring *ring = &kiq->ring;
 
         BUG_ON(!ring->funcs->emit_wreg);
 
-        mutex_lock(&kiq->ring_mutex);
+        spin_lock_irqsave(&kiq->ring_lock, flags);
         amdgpu_ring_alloc(ring, 32);
         amdgpu_ring_emit_wreg(ring, reg, v);
-        amdgpu_fence_emit(ring, &f);
+        amdgpu_fence_emit_polling(ring, &seq);
         amdgpu_ring_commit(ring);
-        mutex_unlock(&kiq->ring_mutex);
+        spin_unlock_irqrestore(&kiq->ring_lock, flags);
 
-        r = dma_fence_wait_timeout(f, false, msecs_to_jiffies(MAX_KIQ_REG_WAIT));
+        r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
         if (r < 1)
-                DRM_ERROR("wait for kiq fence error: %ld.\n", r);
-        dma_fence_put(f);
+                DRM_ERROR("wait for kiq fence error: %ld\n", r);
 }
 
 /**
@@ -274,3 +272,80 @@ void amdgpu_virt_free_mm_table(struct amdgpu_device *adev)
                               (void *)&adev->virt.mm_table.cpu_addr);
         adev->virt.mm_table.gpu_addr = 0;
 }
+
+
+int amdgpu_virt_fw_reserve_get_checksum(void *obj,
+                                        unsigned long obj_size,
+                                        unsigned int key,
+                                        unsigned int chksum)
+{
+        unsigned int ret = key;
+        unsigned long i = 0;
+        unsigned char *pos;
+
+        pos = (char *)obj;
+        /* calculate checksum */
+        for (i = 0; i < obj_size; ++i)
+                ret += *(pos + i);
+        /* minus the chksum itself */
+        pos = (char *)&chksum;
+        for (i = 0; i < sizeof(chksum); ++i)
+                ret -= *(pos + i);
+        return ret;
+}
+
+void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
+{
+        uint32_t pf2vf_ver = 0;
+        uint32_t pf2vf_size = 0;
+        uint32_t checksum = 0;
+        uint32_t checkval;
+        char *str;
+
+        adev->virt.fw_reserve.p_pf2vf = NULL;
+        adev->virt.fw_reserve.p_vf2pf = NULL;
+
+        if (adev->fw_vram_usage.va != NULL) {
+                adev->virt.fw_reserve.p_pf2vf =
+                        (struct amdgim_pf2vf_info_header *)(
+                        adev->fw_vram_usage.va + AMDGIM_DATAEXCHANGE_OFFSET);
+                pf2vf_ver = adev->virt.fw_reserve.p_pf2vf->version;
+                AMDGPU_FW_VRAM_PF2VF_READ(adev, header.size, &pf2vf_size);
+                AMDGPU_FW_VRAM_PF2VF_READ(adev, checksum, &checksum);
+
+                /* pf2vf message must be in 4K */
+                if (pf2vf_size > 0 && pf2vf_size < 4096) {
+                        checkval = amdgpu_virt_fw_reserve_get_checksum(
+                                adev->virt.fw_reserve.p_pf2vf, pf2vf_size,
+                                adev->virt.fw_reserve.checksum_key, checksum);
+                        if (checkval == checksum) {
+                                adev->virt.fw_reserve.p_vf2pf =
+                                        ((void *)adev->virt.fw_reserve.p_pf2vf +
+                                        pf2vf_size);
+                                memset((void *)adev->virt.fw_reserve.p_vf2pf, 0,
+                                        sizeof(amdgim_vf2pf_info));
+                                AMDGPU_FW_VRAM_VF2PF_WRITE(adev, header.version,
+                                        AMDGPU_FW_VRAM_VF2PF_VER);
+                                AMDGPU_FW_VRAM_VF2PF_WRITE(adev, header.size,
+                                        sizeof(amdgim_vf2pf_info));
+                                AMDGPU_FW_VRAM_VF2PF_READ(adev, driver_version,
+                                        &str);
+#ifdef MODULE
+                                if (THIS_MODULE->version != NULL)
+                                        strcpy(str, THIS_MODULE->version);
+                                else
+#endif
+                                        strcpy(str, "N/A");
+                                AMDGPU_FW_VRAM_VF2PF_WRITE(adev, driver_cert,
+                                        0);
+                                AMDGPU_FW_VRAM_VF2PF_WRITE(adev, checksum,
+                                        amdgpu_virt_fw_reserve_get_checksum(
+                                        adev->virt.fw_reserve.p_vf2pf,
+                                        pf2vf_size,
+                                        adev->virt.fw_reserve.checksum_key, 0));
+                        }
+                }
+        }
+}
+
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index afcfb8bcfb65..b89d37fc406f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -36,6 +36,18 @@ struct amdgpu_mm_table {
         uint64_t                gpu_addr;
 };
 
+#define AMDGPU_VF_ERROR_ENTRY_SIZE    16
+
+/* struct error_entry - amdgpu VF error information. */
+struct amdgpu_vf_error_buffer {
+        struct mutex lock;
+        int read_count;
+        int write_count;
+        uint16_t code[AMDGPU_VF_ERROR_ENTRY_SIZE];
+        uint16_t flags[AMDGPU_VF_ERROR_ENTRY_SIZE];
+        uint64_t data[AMDGPU_VF_ERROR_ENTRY_SIZE];
+};
+
 /**
  * struct amdgpu_virt_ops - amdgpu device virt operations
  */
@@ -46,6 +58,179 @@ struct amdgpu_virt_ops {
         void (*trans_msg)(struct amdgpu_device *adev, u32 req, u32 data1, u32 data2, u32 data3);
 };
 
+/*
+ * Firmware Reserve Frame buffer
+ */
+struct amdgpu_virt_fw_reserve {
+        struct amdgim_pf2vf_info_header *p_pf2vf;
+        struct amdgim_vf2pf_info_header *p_vf2pf;
+        unsigned int checksum_key;
+};
+/*
+ * Defination between PF and VF
+ * Structures forcibly aligned to 4 to keep the same style as PF.
+ */
+#define AMDGIM_DATAEXCHANGE_OFFSET              (64 * 1024)
+
+#define AMDGIM_GET_STRUCTURE_RESERVED_SIZE(total, u8, u16, u32, u64) \
+                (total - (((u8)+3) / 4 + ((u16)+1) / 2 + (u32) + (u64)*2))
+
+enum AMDGIM_FEATURE_FLAG {
+        /* GIM supports feature of Error log collecting */
+        AMDGIM_FEATURE_ERROR_LOG_COLLECT = 0x1,
+        /* GIM supports feature of loading uCodes */
+        AMDGIM_FEATURE_GIM_LOAD_UCODES   = 0x2,
+};
+
+struct amdgim_pf2vf_info_header {
+        /* the total structure size in byte. */
+        uint32_t size;
+        /* version of this structure, written by the GIM */
+        uint32_t version;
+} __aligned(4);
+struct  amdgim_pf2vf_info_v1 {
+        /* header contains size and version */
+        struct amdgim_pf2vf_info_header header;
+        /* max_width * max_height */
+        unsigned int uvd_enc_max_pixels_count;
+        /* 16x16 pixels/sec, codec independent */
+        unsigned int uvd_enc_max_bandwidth;
+        /* max_width * max_height */
+        unsigned int vce_enc_max_pixels_count;
+        /* 16x16 pixels/sec, codec independent */
+        unsigned int vce_enc_max_bandwidth;
+        /* MEC FW position in kb from the start of visible frame buffer */
+        unsigned int mecfw_kboffset;
+        /* The features flags of the GIM driver supports. */
+        unsigned int feature_flags;
+        /* use private key from mailbox 2 to create chueksum */
+        unsigned int checksum;
+} __aligned(4);
+
+struct  amdgim_pf2vf_info_v2 {
+        /* header contains size and version */
+        struct amdgim_pf2vf_info_header header;
+        /* use private key from mailbox 2 to create chueksum */
+        uint32_t checksum;
+        /* The features flags of the GIM driver supports. */
+        uint32_t feature_flags;
+        /* max_width * max_height */
+        uint32_t uvd_enc_max_pixels_count;
+        /* 16x16 pixels/sec, codec independent */
+        uint32_t uvd_enc_max_bandwidth;
+        /* max_width * max_height */
+        uint32_t vce_enc_max_pixels_count;
+        /* 16x16 pixels/sec, codec independent */
+        uint32_t vce_enc_max_bandwidth;
+        /* MEC FW position in kb from the start of VF visible frame buffer */
+        uint64_t mecfw_kboffset;
+        /* MEC FW size in KB */
+        uint32_t mecfw_ksize;
+        /* UVD FW position in kb from the start of VF visible frame buffer */
+        uint64_t uvdfw_kboffset;
+        /* UVD FW size in KB */
+        uint32_t uvdfw_ksize;
+        /* VCE FW position in kb from the start of VF visible frame buffer */
+        uint64_t vcefw_kboffset;
+        /* VCE FW size in KB */
+        uint32_t vcefw_ksize;
+        uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 0, 0, (9 + sizeof(struct amdgim_pf2vf_info_header)/sizeof(uint32_t)), 3)];
+} __aligned(4);
+
+
+struct amdgim_vf2pf_info_header {
+        /* the total structure size in byte. */
+        uint32_t size;
+        /*version of this structure, written by the guest */
+        uint32_t version;
+} __aligned(4);
+
+struct amdgim_vf2pf_info_v1 {
+        /* header contains size and version */
+        struct amdgim_vf2pf_info_header header;
+        /* driver version */
+        char driver_version[64];
+        /* driver certification, 1=WHQL, 0=None */
+        unsigned int driver_cert;
+        /* guest OS type and version: need a define */
+        unsigned int os_info;
+        /* in the unit of 1M */
+        unsigned int fb_usage;
+        /* guest gfx engine usage percentage */
+        unsigned int gfx_usage;
+        /* guest gfx engine health percentage */
+        unsigned int gfx_health;
+        /* guest compute engine usage percentage */
+        unsigned int compute_usage;
+        /* guest compute engine health percentage */
+        unsigned int compute_health;
+        /* guest vce engine usage percentage. 0xffff means N/A. */
+        unsigned int vce_enc_usage;
+        /* guest vce engine health percentage. 0xffff means N/A. */
+        unsigned int vce_enc_health;
+        /* guest uvd engine usage percentage. 0xffff means N/A. */
+        unsigned int uvd_enc_usage;
+        /* guest uvd engine usage percentage. 0xffff means N/A. */
+        unsigned int uvd_enc_health;
+        unsigned int checksum;
+} __aligned(4);
+
+struct amdgim_vf2pf_info_v2 {
+        /* header contains size and version */
+        struct amdgim_vf2pf_info_header header;
+        uint32_t checksum;
+        /* driver version */
+        uint8_t driver_version[64];
+        /* driver certification, 1=WHQL, 0=None */
+        uint32_t driver_cert;
+        /* guest OS type and version: need a define */
+        uint32_t os_info;
+        /* in the unit of 1M */
+        uint32_t fb_usage;
+        /* guest gfx engine usage percentage */
+        uint32_t gfx_usage;
+        /* guest gfx engine health percentage */
+        uint32_t gfx_health;
+        /* guest compute engine usage percentage */
+        uint32_t compute_usage;
+        /* guest compute engine health percentage */
+        uint32_t compute_health;
+        /* guest vce engine usage percentage. 0xffff means N/A. */
+        uint32_t vce_enc_usage;
+        /* guest vce engine health percentage. 0xffff means N/A. */
+        uint32_t vce_enc_health;
+        /* guest uvd engine usage percentage. 0xffff means N/A. */
+        uint32_t uvd_enc_usage;
+        /* guest uvd engine usage percentage. 0xffff means N/A. */
+        uint32_t uvd_enc_health;
+        uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 64, 0, (12 + sizeof(struct amdgim_vf2pf_info_header)/sizeof(uint32_t)), 0)];
+} __aligned(4);
+
+#define AMDGPU_FW_VRAM_VF2PF_VER 2
+typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ;
+
+#define AMDGPU_FW_VRAM_VF2PF_WRITE(adev, field, val) \
+        do { \
+                ((amdgim_vf2pf_info *)adev->virt.fw_reserve.p_vf2pf)->field = (val); \
+        } while (0)
+
+#define AMDGPU_FW_VRAM_VF2PF_READ(adev, field, val) \
+        do { \
+                (*val) = ((amdgim_vf2pf_info *)adev->virt.fw_reserve.p_vf2pf)->field; \
+        } while (0)
+
+#define AMDGPU_FW_VRAM_PF2VF_READ(adev, field, val) \
+        do { \
+                if (!adev->virt.fw_reserve.p_pf2vf) \
+                        *(val) = 0; \
+                else { \
+                        if (adev->virt.fw_reserve.p_pf2vf->version == 1) \
+                                *(val) = ((struct amdgim_pf2vf_info_v1 *)adev->virt.fw_reserve.p_pf2vf)->field; \
+                        if (adev->virt.fw_reserve.p_pf2vf->version == 2) \
+                                *(val) = ((struct amdgim_pf2vf_info_v2 *)adev->virt.fw_reserve.p_pf2vf)->field; \
+                } \
+        } while (0)
+
 /* GPU virtualization */
 struct amdgpu_virt {
         uint32_t                        caps;
@@ -59,6 +244,8 @@ struct amdgpu_virt {
         struct work_struct              flr_work;
         struct amdgpu_mm_table          mm_table;
         const struct amdgpu_virt_ops    *ops;
+        struct amdgpu_vf_error_buffer   vf_errors;
+        struct amdgpu_virt_fw_reserve   fw_reserve;
 };
 
 #define AMDGPU_CSA_SIZE    (8 * 1024)
@@ -101,5 +288,9 @@ int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
 int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job);
 int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev);
 void amdgpu_virt_free_mm_table(struct amdgpu_device *adev);
+int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned long obj_size,
+                                        unsigned int key,
+                                        unsigned int chksum);
+void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index bd20ff018512..c8c26f21993c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -27,12 +27,59 @@
  */
 #include <linux/dma-fence-array.h>
 #include <linux/interval_tree_generic.h>
+#include <linux/idr.h>
 #include <drm/drmP.h>
 #include <drm/amdgpu_drm.h>
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
 /*
+ * PASID manager
+ *
+ * PASIDs are global address space identifiers that can be shared
+ * between the GPU, an IOMMU and the driver. VMs on different devices
+ * may use the same PASID if they share the same address
+ * space. Therefore PASIDs are allocated using a global IDA. VMs are
+ * looked up from the PASID per amdgpu_device.
+ */
+static DEFINE_IDA(amdgpu_vm_pasid_ida);
+
+/**
+ * amdgpu_vm_alloc_pasid - Allocate a PASID
+ * @bits: Maximum width of the PASID in bits, must be at least 1
+ *
+ * Allocates a PASID of the given width while keeping smaller PASIDs
+ * available if possible.
+ *
+ * Returns a positive integer on success. Returns %-EINVAL if bits==0.
+ * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on
+ * memory allocation failure.
+ */
+int amdgpu_vm_alloc_pasid(unsigned int bits)
+{
+        int pasid = -EINVAL;
+
+        for (bits = min(bits, 31U); bits > 0; bits--) {
+                pasid = ida_simple_get(&amdgpu_vm_pasid_ida,
+                                       1U << (bits - 1), 1U << bits,
+                                       GFP_KERNEL);
+                if (pasid != -ENOSPC)
+                        break;
+        }
+
+        return pasid;
+}
+
+/**
+ * amdgpu_vm_free_pasid - Free a PASID
+ * @pasid: PASID to free
+ */
+void amdgpu_vm_free_pasid(unsigned int pasid)
+{
+        ida_simple_remove(&amdgpu_vm_pasid_ida, pasid);
+}
+
+/*
  * GPUVM
  * GPUVM is similar to the legacy gart on older asics, however
  * rather than there being a single global gart table
@@ -140,7 +187,7 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
                          struct list_head *validated,
                          struct amdgpu_bo_list_entry *entry)
 {
-        entry->robj = vm->root.bo;
+        entry->robj = vm->root.base.bo;
         entry->priority = 0;
         entry->tv.bo = &entry->robj->tbo;
         entry->tv.shared = true;
@@ -149,86 +196,80 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 }
 
 /**
- * amdgpu_vm_validate_layer - validate a single page table level
+ * amdgpu_vm_validate_pt_bos - validate the page table BOs
  *
- * @parent: parent page table level
+ * @adev: amdgpu device pointer
+ * @vm: vm providing the BOs
  * @validate: callback to do the validation
  * @param: parameter for the validation callback
  *
  * Validate the page table BOs on command submission if neccessary.
  */
-static int amdgpu_vm_validate_level(struct amdgpu_vm_pt *parent,
-                                    int (*validate)(void *, struct amdgpu_bo *),
-                                    void *param, bool use_cpu_for_update,
-                                    struct ttm_bo_global *glob)
+int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+                              int (*validate)(void *p, struct amdgpu_bo *bo),
+                              void *param)
 {
-        unsigned i;
+        struct ttm_bo_global *glob = adev->mman.bdev.glob;
         int r;
 
-        if (use_cpu_for_update) {
-                r = amdgpu_bo_kmap(parent->bo, NULL);
-                if (r)
-                        return r;
-        }
-
-        if (!parent->entries)
-                return 0;
+        spin_lock(&vm->status_lock);
+        while (!list_empty(&vm->evicted)) {
+                struct amdgpu_vm_bo_base *bo_base;
+                struct amdgpu_bo *bo;
 
-        for (i = 0; i <= parent->last_entry_used; ++i) {
-                struct amdgpu_vm_pt *entry = &parent->entries[i];
+                bo_base = list_first_entry(&vm->evicted,
+                                           struct amdgpu_vm_bo_base,
+                                           vm_status);
+                spin_unlock(&vm->status_lock);
 
-                if (!entry->bo)
-                        continue;
+                bo = bo_base->bo;
+                BUG_ON(!bo);
+                if (bo->parent) {
+                        r = validate(param, bo);
+                        if (r)
+                                return r;
 
-                r = validate(param, entry->bo);
-                if (r)
-                        return r;
+                        spin_lock(&glob->lru_lock);
+                        ttm_bo_move_to_lru_tail(&bo->tbo);
+                        if (bo->shadow)
+                                ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
+                        spin_unlock(&glob->lru_lock);
+                }
 
-                spin_lock(&glob->lru_lock);
-                ttm_bo_move_to_lru_tail(&entry->bo->tbo);
-                if (entry->bo->shadow)
-                        ttm_bo_move_to_lru_tail(&entry->bo->shadow->tbo);
-                spin_unlock(&glob->lru_lock);
+                if (bo->tbo.type == ttm_bo_type_kernel &&
+                    vm->use_cpu_for_update) {
+                        r = amdgpu_bo_kmap(bo, NULL);
+                        if (r)
+                                return r;
+                }
 
-                /*
-                 * Recurse into the sub directory. This is harmless because we
-                 * have only a maximum of 5 layers.
-                 */
-                r = amdgpu_vm_validate_level(entry, validate, param,
-                                             use_cpu_for_update, glob);
-                if (r)
-                        return r;
+                spin_lock(&vm->status_lock);
+                if (bo->tbo.type != ttm_bo_type_kernel)
+                        list_move(&bo_base->vm_status, &vm->moved);
+                else
+                        list_move(&bo_base->vm_status, &vm->relocated);
         }
+        spin_unlock(&vm->status_lock);
 
-        return r;
+        return 0;
 }
 
 /**
- * amdgpu_vm_validate_pt_bos - validate the page table BOs
+ * amdgpu_vm_ready - check VM is ready for updates
  *
- * @adev: amdgpu device pointer
- * @vm: vm providing the BOs
- * @validate: callback to do the validation
- * @param: parameter for the validation callback
+ * @vm: VM to check
  *
- * Validate the page table BOs on command submission if neccessary.
+ * Check if all VM PDs/PTs are ready for updates
  */
-int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                              int (*validate)(void *p, struct amdgpu_bo *bo),
-                              void *param)
+bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 {
-        uint64_t num_evictions;
+        bool ready;
 
-        /* We only need to validate the page tables
-         * if they aren't already valid.
-         */
-        num_evictions = atomic64_read(&adev->num_evictions);
-        if (num_evictions == vm->last_eviction_counter)
-                return 0;
+        spin_lock(&vm->status_lock);
+        ready = list_empty(&vm->evicted);
+        spin_unlock(&vm->status_lock);
 
-        return amdgpu_vm_validate_level(&vm->root, validate, param,
-                                        vm->use_cpu_for_update,
-                                        adev->mman.bdev.glob);
+        return ready;
 }
 
 /**
@@ -287,18 +328,19 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
                                 AMDGPU_GEM_CREATE_SHADOW);
 
         if (vm->pte_support_ats) {
-                init_value = AMDGPU_PTE_SYSTEM;
+                init_value = AMDGPU_PTE_DEFAULT_ATC;
                 if (level != adev->vm_manager.num_level - 1)
                         init_value |= AMDGPU_PDE_PTE;
+
         }
 
         /* walk over the address space and allocate the page tables */
         for (pt_idx = from; pt_idx <= to; ++pt_idx) {
-                struct reservation_object *resv = vm->root.bo->tbo.resv;
+                struct reservation_object *resv = vm->root.base.bo->tbo.resv;
                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
                 struct amdgpu_bo *pt;
 
-                if (!entry->bo) {
+                if (!entry->base.bo) {
                         r = amdgpu_bo_create(adev,
                                              amdgpu_vm_bo_size(adev, level),
                                              AMDGPU_GPU_PAGE_SIZE, true,
@@ -319,9 +361,14 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
                         /* Keep a reference to the root directory to avoid
                         * freeing them up in the wrong order.
                         */
-                        pt->parent = amdgpu_bo_ref(vm->root.bo);
-
-                        entry->bo = pt;
+                        pt->parent = amdgpu_bo_ref(parent->base.bo);
+
+                        entry->base.vm = vm;
+                        entry->base.bo = pt;
+                        list_add_tail(&entry->base.bo_list, &pt->va);
+                        spin_lock(&vm->status_lock);
+                        list_add(&entry->base.vm_status, &vm->relocated);
+                        spin_unlock(&vm->status_lock);
                         entry->addr = 0;
                 }
 
@@ -988,7 +1035,7 @@ static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         int r;
 
         amdgpu_sync_create(&sync);
-        amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.resv, owner);
+        amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false);
         r = amdgpu_sync_wait(&sync, true);
         amdgpu_sync_free(&sync);
 
@@ -1007,18 +1054,17 @@ static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  */
 static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                                   struct amdgpu_vm *vm,
-                                  struct amdgpu_vm_pt *parent,
-                                  unsigned level)
+                                  struct amdgpu_vm_pt *parent)
 {
         struct amdgpu_bo *shadow;
         struct amdgpu_ring *ring = NULL;
         uint64_t pd_addr, shadow_addr = 0;
-        uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
         uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
         unsigned count = 0, pt_idx, ndw = 0;
         struct amdgpu_job *job;
         struct amdgpu_pte_update_params params;
         struct dma_fence *fence = NULL;
+        uint32_t incr;
 
         int r;
 
@@ -1027,10 +1073,10 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
         memset(&params, 0, sizeof(params));
         params.adev = adev;
-        shadow = parent->bo->shadow;
+        shadow = parent->base.bo->shadow;
 
         if (vm->use_cpu_for_update) {
-                pd_addr = (unsigned long)amdgpu_bo_kptr(parent->bo);
+                pd_addr = (unsigned long)amdgpu_bo_kptr(parent->base.bo);
                 r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
                 if (unlikely(r))
                         return r;
@@ -1046,7 +1092,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                 /* assume the worst case */
                 ndw += parent->last_entry_used * 6;
 
-                pd_addr = amdgpu_bo_gpu_offset(parent->bo);
+                pd_addr = amdgpu_bo_gpu_offset(parent->base.bo);
 
                 if (shadow) {
                         shadow_addr = amdgpu_bo_gpu_offset(shadow);
@@ -1066,12 +1112,17 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
         /* walk over the address space and update the directory */
         for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
-                struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
+                struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
+                struct amdgpu_bo *bo = entry->base.bo;
                 uint64_t pde, pt;
 
                 if (bo == NULL)
                         continue;
 
+                spin_lock(&vm->status_lock);
+                list_del_init(&entry->base.vm_status);
+                spin_unlock(&vm->status_lock);
+
                 pt = amdgpu_bo_gpu_offset(bo);
                 pt = amdgpu_gart_get_vm_pde(adev, pt);
                 /* Don't update huge pages here */
@@ -1082,6 +1133,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                 parent->entries[pt_idx].addr = pt | AMDGPU_PTE_VALID;
 
                 pde = pd_addr + pt_idx * 8;
+                incr = amdgpu_bo_size(bo);
                 if (((last_pde + 8 * count) != pde) ||
                     ((last_pt + incr * count) != pt) ||
                     (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
@@ -1109,7 +1161,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
         }
 
         if (count) {
-                if (vm->root.bo->shadow)
+                if (vm->root.base.bo->shadow)
                         params.func(&params, last_shadow, last_pt,
                                     count, incr, AMDGPU_PTE_VALID);
 
@@ -1122,12 +1174,13 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                         amdgpu_job_free(job);
                 } else {
                         amdgpu_ring_pad_ib(ring, params.ib);
-                        amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
-                                         AMDGPU_FENCE_OWNER_VM);
+                        amdgpu_sync_resv(adev, &job->sync,
+                                         parent->base.bo->tbo.resv,
+                                         AMDGPU_FENCE_OWNER_VM, false);
                         if (shadow)
                                 amdgpu_sync_resv(adev, &job->sync,
                                                  shadow->tbo.resv,
-                                                 AMDGPU_FENCE_OWNER_VM);
+                                                 AMDGPU_FENCE_OWNER_VM, false);
 
                         WARN_ON(params.ib->length_dw > ndw);
                         r = amdgpu_job_submit(job, ring, &vm->entity,
@@ -1135,26 +1188,11 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
                         if (r)
                                 goto error_free;
 
-                        amdgpu_bo_fence(parent->bo, fence, true);
-                        dma_fence_put(vm->last_dir_update);
-                        vm->last_dir_update = dma_fence_get(fence);
-                        dma_fence_put(fence);
+                        amdgpu_bo_fence(parent->base.bo, fence, true);
+                        dma_fence_put(vm->last_update);
+                        vm->last_update = fence;
                 }
         }
-        /*
-         * Recurse into the subdirectories. This recursion is harmless because
-         * we only have a maximum of 5 layers.
-         */
-        for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
-                struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
-
-                if (!entry->bo)
-                        continue;
-
-                r = amdgpu_vm_update_level(adev, vm, entry, level + 1);
-                if (r)
-                        return r;
-        }
 
         return 0;
 
@@ -1170,7 +1208,8 @@ error_free:
  *
  * Mark all PD level as invalid after an error.
  */
-static void amdgpu_vm_invalidate_level(struct amdgpu_vm_pt *parent)
+static void amdgpu_vm_invalidate_level(struct amdgpu_vm *vm,
+                                       struct amdgpu_vm_pt *parent)
 {
         unsigned pt_idx;
 
@@ -1181,11 +1220,15 @@ static void amdgpu_vm_invalidate_level(struct amdgpu_vm_pt *parent)
         for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 
-                if (!entry->bo)
+                if (!entry->base.bo)
                         continue;
 
                 entry->addr = ~0ULL;
-                amdgpu_vm_invalidate_level(entry);
+                spin_lock(&vm->status_lock);
+                if (list_empty(&entry->base.vm_status))
+                        list_add(&entry->base.vm_status, &vm->relocated);
+                spin_unlock(&vm->status_lock);
+                amdgpu_vm_invalidate_level(vm, entry);
         }
 }
 
@@ -1201,11 +1244,40 @@ static void amdgpu_vm_invalidate_level(struct amdgpu_vm_pt *parent)
 int amdgpu_vm_update_directories(struct amdgpu_device *adev,
                                  struct amdgpu_vm *vm)
 {
-        int r;
+        int r = 0;
 
-        r = amdgpu_vm_update_level(adev, vm, &vm->root, 0);
-        if (r)
-                amdgpu_vm_invalidate_level(&vm->root);
+        spin_lock(&vm->status_lock);
+        while (!list_empty(&vm->relocated)) {
+                struct amdgpu_vm_bo_base *bo_base;
+                struct amdgpu_bo *bo;
+
+                bo_base = list_first_entry(&vm->relocated,
+                                           struct amdgpu_vm_bo_base,
+                                           vm_status);
+                spin_unlock(&vm->status_lock);
+
+                bo = bo_base->bo->parent;
+                if (bo) {
+                        struct amdgpu_vm_bo_base *parent;
+                        struct amdgpu_vm_pt *pt;
+
+                        parent = list_first_entry(&bo->va,
+                                                  struct amdgpu_vm_bo_base,
+                                                  bo_list);
+                        pt = container_of(parent, struct amdgpu_vm_pt, base);
+
+                        r = amdgpu_vm_update_level(adev, vm, pt);
+                        if (r) {
+                                amdgpu_vm_invalidate_level(vm, &vm->root);
+                                return r;
+                        }
+                        spin_lock(&vm->status_lock);
+                } else {
+                        spin_lock(&vm->status_lock);
+                        list_del_init(&bo_base->vm_status);
+                }
+        }
+        spin_unlock(&vm->status_lock);
 
         if (vm->use_cpu_for_update) {
                 /* Flush HDP */
@@ -1236,7 +1308,7 @@ void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
         *entry = &p->vm->root;
         while ((*entry)->entries) {
                 idx = addr >> (p->adev->vm_manager.block_size * level--);
-                idx %= amdgpu_bo_size((*entry)->bo) / 8;
+                idx %= amdgpu_bo_size((*entry)->base.bo) / 8;
                 *parent = *entry;
                 *entry = &(*entry)->entries[idx];
         }
@@ -1272,7 +1344,7 @@ static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
             p->src ||
             !(flags & AMDGPU_PTE_VALID)) {
 
-                dst = amdgpu_bo_gpu_offset(entry->bo);
+                dst = amdgpu_bo_gpu_offset(entry->base.bo);
                 dst = amdgpu_gart_get_vm_pde(p->adev, dst);
                 flags = AMDGPU_PTE_VALID;
         } else {
@@ -1298,18 +1370,18 @@ static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
                 tmp = p->pages_addr;
                 p->pages_addr = NULL;
 
-                pd_addr = (unsigned long)amdgpu_bo_kptr(parent->bo);
+                pd_addr = (unsigned long)amdgpu_bo_kptr(parent->base.bo);
                 pde = pd_addr + (entry - parent->entries) * 8;
                 amdgpu_vm_cpu_set_ptes(p, pde, dst, 1, 0, flags);
 
                 p->pages_addr = tmp;
         } else {
-                if (parent->bo->shadow) {
-                        pd_addr = amdgpu_bo_gpu_offset(parent->bo->shadow);
+                if (parent->base.bo->shadow) {
+                        pd_addr = amdgpu_bo_gpu_offset(parent->base.bo->shadow);
                         pde = pd_addr + (entry - parent->entries) * 8;
                         amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
                 }
-                pd_addr = amdgpu_bo_gpu_offset(parent->bo);
+                pd_addr = amdgpu_bo_gpu_offset(parent->base.bo);
                 pde = pd_addr + (entry - parent->entries) * 8;
                 amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
         }
@@ -1360,7 +1432,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
                 if (entry->addr & AMDGPU_PDE_PTE)
                         continue;
 
-                pt = entry->bo;
+                pt = entry->base.bo;
                 if (use_cpu_update) {
                         pe_start = (unsigned long)amdgpu_bo_kptr(pt);
                 } else {
@@ -1396,8 +1468,6 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
                                 uint64_t start, uint64_t end,
                                 uint64_t dst, uint64_t flags)
 {
-        int r;
-
         /**
          * The MC L1 TLB supports variable sized pages, based on a fragment
          * field in the PTE. When this field is set to a non-zero value, page
@@ -1416,39 +1486,38 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
          * Userspace can support this by aligning virtual base address and
          * allocation size to the fragment size.
          */
-        unsigned pages_per_frag = params->adev->vm_manager.fragment_size;
-        uint64_t frag_flags = AMDGPU_PTE_FRAG(pages_per_frag);
-        uint64_t frag_align = 1 << pages_per_frag;
-
-        uint64_t frag_start = ALIGN(start, frag_align);
-        uint64_t frag_end = end & ~(frag_align - 1);
+        unsigned max_frag = params->adev->vm_manager.fragment_size;
+        int r;
 
         /* system pages are non continuously */
-        if (params->src || !(flags & AMDGPU_PTE_VALID) ||
-            (frag_start >= frag_end))
+        if (params->src || !(flags & AMDGPU_PTE_VALID))
                 return amdgpu_vm_update_ptes(params, start, end, dst, flags);
 
-        /* handle the 4K area at the beginning */
-        if (start != frag_start) {
-                r = amdgpu_vm_update_ptes(params, start, frag_start,
-                                          dst, flags);
+        while (start != end) {
+                uint64_t frag_flags, frag_end;
+                unsigned frag;
+
+                /* This intentionally wraps around if no bit is set */
+                frag = min((unsigned)ffs(start) - 1,
+                           (unsigned)fls64(end - start) - 1);
+                if (frag >= max_frag) {
+                        frag_flags = AMDGPU_PTE_FRAG(max_frag);
+                        frag_end = end & ~((1ULL << max_frag) - 1);
+                } else {
+                        frag_flags = AMDGPU_PTE_FRAG(frag);
+                        frag_end = start + (1 << frag);
+                }
+
+                r = amdgpu_vm_update_ptes(params, start, frag_end, dst,
+                                          flags | frag_flags);
                 if (r)
                         return r;
-                dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
-        }
 
-        /* handle the area in the middle */
-        r = amdgpu_vm_update_ptes(params, frag_start, frag_end, dst,
-                                  flags | frag_flags);
-        if (r)
-                return r;
-
-        /* handle the 4K area at the end */
-        if (frag_end != end) {
-                dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
-                r = amdgpu_vm_update_ptes(params, frag_end, end, dst, flags);
+                dst += (frag_end - start) * AMDGPU_GPU_PAGE_SIZE;
+                start = frag_end;
         }
-        return r;
+
+        return 0;
 }
 
 /**
@@ -1456,7 +1525,6 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
  *
  * @adev: amdgpu_device pointer
  * @exclusive: fence we need to sync to
- * @src: address where to copy page table entries from
  * @pages_addr: DMA addresses to use for mapping
  * @vm: requested vm
  * @start: start of mapped range
@@ -1470,7 +1538,6 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
  */
 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
                                        struct dma_fence *exclusive,
-                                       uint64_t src,
                                        dma_addr_t *pages_addr,
                                        struct amdgpu_vm *vm,
                                        uint64_t start, uint64_t last,
@@ -1488,7 +1555,6 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         memset(&params, 0, sizeof(params));
         params.adev = adev;
         params.vm = vm;
-        params.src = src;
 
         /* sync to everything on unmapping */
         if (!(flags & AMDGPU_PTE_VALID))
@@ -1517,10 +1583,12 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         nptes = last - start + 1;
 
         /*
-         * reserve space for one command every (1 << BLOCK_SIZE)
+         * reserve space for two commands every (1 << BLOCK_SIZE)
          *  entries or 2k dwords (whatever is smaller)
+         *
+         * The second command is for the shadow pagetables.
          */
-        ncmds = (nptes >> min(adev->vm_manager.block_size, 11u)) + 1;
+        ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2;
 
         /* padding, etc. */
         ndw = 64;
@@ -1528,15 +1596,9 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         /* one PDE write for each huge page */
         ndw += ((nptes >> adev->vm_manager.block_size) + 1) * 6;
 
-        if (src) {
-                /* only copy commands needed */
-                ndw += ncmds * 7;
-
-                params.func = amdgpu_vm_do_copy_ptes;
-
-        } else if (pages_addr) {
+        if (pages_addr) {
                 /* copy commands needed */
-                ndw += ncmds * 7;
+                ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw;
 
                 /* and also PTEs */
                 ndw += nptes * 2;
@@ -1545,10 +1607,11 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 
         } else {
                 /* set page commands needed */
-                ndw += ncmds * 10;
+                ndw += ncmds * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw;
 
-                /* two extra commands for begin/end of fragment */
-                ndw += 2 * 10;
+                /* extra commands for begin/end fragments */
+                ndw += 2 * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw
+                                * adev->vm_manager.fragment_size;
 
                 params.func = amdgpu_vm_do_set_ptes;
         }
@@ -1559,7 +1622,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 
         params.ib = &job->ibs[0];
 
-        if (!src && pages_addr) {
+        if (pages_addr) {
                 uint64_t *pte;
                 unsigned i;
 
@@ -1580,12 +1643,12 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         if (r)
                 goto error_free;
 
-        r = amdgpu_sync_resv(adev, &job->sync, vm->root.bo->tbo.resv,
-                             owner);
+        r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,
+                             owner, false);
         if (r)
                 goto error_free;
 
-        r = reservation_object_reserve_shared(vm->root.bo->tbo.resv);
+        r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);
         if (r)
                 goto error_free;
 
@@ -1600,14 +1663,14 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
         if (r)
                 goto error_free;
 
-        amdgpu_bo_fence(vm->root.bo, f, true);
+        amdgpu_bo_fence(vm->root.base.bo, f, true);
         dma_fence_put(*fence);
         *fence = f;
         return 0;
 
 error_free:
         amdgpu_job_free(job);
-        amdgpu_vm_invalidate_level(&vm->root);
+        amdgpu_vm_invalidate_level(vm, &vm->root);
         return r;
 }
 
@@ -1636,7 +1699,8 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
                                       struct drm_mm_node *nodes,
                                       struct dma_fence **fence)
 {
-        uint64_t pfn, src = 0, start = mapping->start;
+        unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size;
+        uint64_t pfn, start = mapping->start;
         int r;
 
         /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
@@ -1670,6 +1734,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
         }
 
         do {
+                dma_addr_t *dma_addr = NULL;
                 uint64_t max_entries;
                 uint64_t addr, last;
 
@@ -1683,16 +1748,32 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
                 }
 
                 if (pages_addr) {
+                        uint64_t count;
+
                         max_entries = min(max_entries, 16ull * 1024ull);
-                        addr = 0;
+                        for (count = 1; count < max_entries; ++count) {
+                                uint64_t idx = pfn + count;
+
+                                if (pages_addr[idx] !=
+                                    (pages_addr[idx - 1] + PAGE_SIZE))
+                                        break;
+                        }
+
+                        if (count < min_linear_pages) {
+                                addr = pfn << PAGE_SHIFT;
+                                dma_addr = pages_addr;
+                        } else {
+                                addr = pages_addr[pfn];
+                                max_entries = count;
+                        }
+
                 } else if (flags & AMDGPU_PTE_VALID) {
                         addr += adev->vm_manager.vram_base_offset;
+                        addr += pfn << PAGE_SHIFT;
                 }
-                addr += pfn << PAGE_SHIFT;
 
                 last = min((uint64_t)mapping->last, start + max_entries - 1);
-                r = amdgpu_vm_bo_update_mapping(adev, exclusive,
-                                                src, pages_addr, vm,
+                r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,
                                                 start, last, flags, addr,
                                                 fence);
                 if (r)
@@ -1730,7 +1811,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
         dma_addr_t *pages_addr = NULL;
         struct ttm_mem_reg *mem;
         struct drm_mm_node *nodes;
-        struct dma_fence *exclusive;
+        struct dma_fence *exclusive, **last_update;
         uint64_t flags;
         int r;
 
@@ -1756,38 +1837,43 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
         else
                 flags = 0x0;
 
-        spin_lock(&vm->status_lock);
-        if (!list_empty(&bo_va->base.vm_status))
+        if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
+                last_update = &vm->last_update;
+        else
+                last_update = &bo_va->last_pt_update;
+
+        if (!clear && bo_va->base.moved) {
+                bo_va->base.moved = false;
                 list_splice_init(&bo_va->valids, &bo_va->invalids);
-        spin_unlock(&vm->status_lock);
+
+        } else if (bo_va->cleared != clear) {
+                list_splice_init(&bo_va->valids, &bo_va->invalids);
+        }
 
         list_for_each_entry(mapping, &bo_va->invalids, list) {
                 r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
                                                mapping, flags, nodes,
-                                               &bo_va->last_pt_update);
+                                               last_update);
                 if (r)
                         return r;
         }
 
-        if (trace_amdgpu_vm_bo_mapping_enabled()) {
-                list_for_each_entry(mapping, &bo_va->valids, list)
-                        trace_amdgpu_vm_bo_mapping(mapping);
-
-                list_for_each_entry(mapping, &bo_va->invalids, list)
-                        trace_amdgpu_vm_bo_mapping(mapping);
+        if (vm->use_cpu_for_update) {
+                /* Flush HDP */
+                mb();
+                amdgpu_gart_flush_gpu_tlb(adev, 0);
         }
 
         spin_lock(&vm->status_lock);
-        list_splice_init(&bo_va->invalids, &bo_va->valids);
         list_del_init(&bo_va->base.vm_status);
-        if (clear)
-                list_add(&bo_va->base.vm_status, &vm->cleared);
         spin_unlock(&vm->status_lock);
 
-        if (vm->use_cpu_for_update) {
-                /* Flush HDP */
-                mb();
-                amdgpu_gart_flush_gpu_tlb(adev, 0);
+        list_splice_init(&bo_va->invalids, &bo_va->valids);
+        bo_va->cleared = clear;
+
+        if (trace_amdgpu_vm_bo_mapping_enabled()) {
+                list_for_each_entry(mapping, &bo_va->valids, list)
+                        trace_amdgpu_vm_bo_mapping(mapping);
         }
 
         return 0;
@@ -1895,7 +1981,7 @@ static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
  */
 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 {
-        struct reservation_object *resv = vm->root.bo->tbo.resv;
+        struct reservation_object *resv = vm->root.base.bo->tbo.resv;
         struct dma_fence *excl, **shared;
         unsigned i, shared_count;
         int r;
@@ -1951,9 +2037,9 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
                 list_del(&mapping->list);
 
                 if (vm->pte_support_ats)
-                        init_pte_value = AMDGPU_PTE_SYSTEM;
+                        init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
 
-                r = amdgpu_vm_bo_update_mapping(adev, NULL, 0, NULL, vm,
+                r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
                                                 mapping->start, mapping->last,
                                                 init_pte_value, 0, &f);
                 amdgpu_vm_free_mapping(adev, vm, mapping, f);
@@ -1975,29 +2061,35 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 }
 
 /**
- * amdgpu_vm_clear_moved - clear moved BOs in the PT
+ * amdgpu_vm_handle_moved - handle moved BOs in the PT
  *
  * @adev: amdgpu_device pointer
  * @vm: requested vm
+ * @sync: sync object to add fences to
  *
- * Make sure all moved BOs are cleared in the PT.
+ * Make sure all BOs which are moved are updated in the PTs.
  * Returns 0 for success.
  *
- * PTs have to be reserved and mutex must be locked!
+ * PTs have to be reserved!
  */
-int amdgpu_vm_clear_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                            struct amdgpu_sync *sync)
+int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
+                           struct amdgpu_vm *vm)
 {
-        struct amdgpu_bo_va *bo_va = NULL;
+        bool clear;
         int r = 0;
 
         spin_lock(&vm->status_lock);
         while (!list_empty(&vm->moved)) {
+                struct amdgpu_bo_va *bo_va;
+
                 bo_va = list_first_entry(&vm->moved,
                         struct amdgpu_bo_va, base.vm_status);
                 spin_unlock(&vm->status_lock);
 
-                r = amdgpu_vm_bo_update(adev, bo_va, true);
+                /* Per VM BOs never need to bo cleared in the page tables */
+                clear = bo_va->base.bo->tbo.resv != vm->root.base.bo->tbo.resv;
+
+                r = amdgpu_vm_bo_update(adev, bo_va, clear);
                 if (r)
                         return r;
 
@@ -2005,9 +2097,6 @@ int amdgpu_vm_clear_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         }
         spin_unlock(&vm->status_lock);
 
-        if (bo_va)
-                r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
-
         return r;
 }
 
@@ -2049,6 +2138,39 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
         return bo_va;
 }
 
+
+/**
+ * amdgpu_vm_bo_insert_mapping - insert a new mapping
+ *
+ * @adev: amdgpu_device pointer
+ * @bo_va: bo_va to store the address
+ * @mapping: the mapping to insert
+ *
+ * Insert a new mapping into all structures.
+ */
+static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
+                                    struct amdgpu_bo_va *bo_va,
+                                    struct amdgpu_bo_va_mapping *mapping)
+{
+        struct amdgpu_vm *vm = bo_va->base.vm;
+        struct amdgpu_bo *bo = bo_va->base.bo;
+
+        mapping->bo_va = bo_va;
+        list_add(&mapping->list, &bo_va->invalids);
+        amdgpu_vm_it_insert(mapping, &vm->va);
+
+        if (mapping->flags & AMDGPU_PTE_PRT)
+                amdgpu_vm_prt_get(adev);
+
+        if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
+                spin_lock(&vm->status_lock);
+                if (list_empty(&bo_va->base.vm_status))
+                        list_add(&bo_va->base.vm_status, &vm->moved);
+                spin_unlock(&vm->status_lock);
+        }
+        trace_amdgpu_vm_bo_map(bo_va, mapping);
+}
+
 /**
  * amdgpu_vm_bo_map - map bo inside a vm
  *
@@ -2100,17 +2222,12 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
         if (!mapping)
                 return -ENOMEM;
 
-        INIT_LIST_HEAD(&mapping->list);
         mapping->start = saddr;
         mapping->last = eaddr;
         mapping->offset = offset;
         mapping->flags = flags;
 
-        list_add(&mapping->list, &bo_va->invalids);
-        amdgpu_vm_it_insert(mapping, &vm->va);
-
-        if (flags & AMDGPU_PTE_PRT)
-                amdgpu_vm_prt_get(adev);
+        amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
 
         return 0;
 }
@@ -2137,7 +2254,6 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
 {
         struct amdgpu_bo_va_mapping *mapping;
         struct amdgpu_bo *bo = bo_va->base.bo;
-        struct amdgpu_vm *vm = bo_va->base.vm;
         uint64_t eaddr;
         int r;
 
@@ -2171,11 +2287,7 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
         mapping->offset = offset;
         mapping->flags = flags;
 
-        list_add(&mapping->list, &bo_va->invalids);
-        amdgpu_vm_it_insert(mapping, &vm->va);
-
-        if (flags & AMDGPU_PTE_PRT)
-                amdgpu_vm_prt_get(adev);
+        amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
 
         return 0;
 }
@@ -2221,6 +2333,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
 
         list_del(&mapping->list);
         amdgpu_vm_it_remove(mapping, &vm->va);
+        mapping->bo_va = NULL;
         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
 
         if (valid)
@@ -2306,6 +2419,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
                 if (tmp->last > eaddr)
                     tmp->last = eaddr;
 
+                tmp->bo_va = NULL;
                 list_add(&tmp->list, &vm->freed);
                 trace_amdgpu_vm_bo_unmap(NULL, tmp);
         }
@@ -2332,6 +2446,19 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
 }
 
 /**
+ * amdgpu_vm_bo_lookup_mapping - find mapping by address
+ *
+ * @vm: the requested VM
+ *
+ * Find a mapping by it's address.
+ */
+struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
+                                                         uint64_t addr)
+{
+        return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
+}
+
+/**
  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
  *
  * @adev: amdgpu_device pointer
@@ -2356,6 +2483,7 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
                 list_del(&mapping->list);
                 amdgpu_vm_it_remove(mapping, &vm->va);
+                mapping->bo_va = NULL;
                 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
                 list_add(&mapping->list, &vm->freed);
         }
@@ -2380,15 +2508,36 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
  * Mark @bo as invalid.
  */
 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
-                             struct amdgpu_bo *bo)
+                             struct amdgpu_bo *bo, bool evicted)
 {
         struct amdgpu_vm_bo_base *bo_base;
 
         list_for_each_entry(bo_base, &bo->va, bo_list) {
+                struct amdgpu_vm *vm = bo_base->vm;
+
+                bo_base->moved = true;
+                if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
+                        spin_lock(&bo_base->vm->status_lock);
+                        if (bo->tbo.type == ttm_bo_type_kernel)
+                                list_move(&bo_base->vm_status, &vm->evicted);
+                        else
+                                list_move_tail(&bo_base->vm_status,
+                                               &vm->evicted);
+                        spin_unlock(&bo_base->vm->status_lock);
+                        continue;
+                }
+
+                if (bo->tbo.type == ttm_bo_type_kernel) {
+                        spin_lock(&bo_base->vm->status_lock);
+                        if (list_empty(&bo_base->vm_status))
+                                list_add(&bo_base->vm_status, &vm->relocated);
+                        spin_unlock(&bo_base->vm->status_lock);
+                        continue;
+                }
+
                 spin_lock(&bo_base->vm->status_lock);
                 if (list_empty(&bo_base->vm_status))
-                        list_add(&bo_base->vm_status,
-                                 &bo_base->vm->moved);
+                        list_add(&bo_base->vm_status, &vm->moved);
                 spin_unlock(&bo_base->vm->status_lock);
         }
 }
@@ -2412,7 +2561,8 @@ static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
  * @adev: amdgpu_device pointer
  * @fragment_size_default: the default fragment size if it's set auto
  */
-void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev, uint32_t fragment_size_default)
+void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev,
+                                 uint32_t fragment_size_default)
 {
         if (amdgpu_vm_fragment_size == -1)
                 adev->vm_manager.fragment_size = fragment_size_default;
@@ -2426,7 +2576,8 @@ void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev, uint32_t fragment_s
  * @adev: amdgpu_device pointer
  * @vm_size: the default vm size if it's set auto
  */
-void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_t fragment_size_default)
+void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size,
+                           uint32_t fragment_size_default)
 {
         /* adjust vm size firstly */
         if (amdgpu_vm_size == -1)
@@ -2458,7 +2609,7 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_
  * Init @vm fields.
  */
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                   int vm_context)
+                   int vm_context, unsigned int pasid)
 {
         const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
                 AMDGPU_VM_PTE_COUNT(adev) * 8);
@@ -2474,8 +2625,9 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
                 vm->reserved_vmid[i] = NULL;
         spin_lock_init(&vm->status_lock);
+        INIT_LIST_HEAD(&vm->evicted);
+        INIT_LIST_HEAD(&vm->relocated);
         INIT_LIST_HEAD(&vm->moved);
-        INIT_LIST_HEAD(&vm->cleared);
         INIT_LIST_HEAD(&vm->freed);
 
         /* create scheduler entity for page table updates */
@@ -2497,7 +2649,9 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
                 if (adev->asic_type == CHIP_RAVEN) {
                         vm->pte_support_ats = true;
-                        init_pde_value = AMDGPU_PTE_SYSTEM | AMDGPU_PDE_PTE;
+                        init_pde_value = AMDGPU_PTE_DEFAULT_ATC
+                                        | AMDGPU_PDE_PTE;
+
                 }
         } else
                 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
@@ -2506,7 +2660,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                          vm->use_cpu_for_update ? "CPU" : "SDMA");
         WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
                   "CPU update of VM recommended only for large BAR system\n");
-        vm->last_dir_update = NULL;
+        vm->last_update = NULL;
 
         flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
                         AMDGPU_GEM_CREATE_VRAM_CLEARED;
@@ -2519,30 +2673,47 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
         r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, 0), align, true,
                              AMDGPU_GEM_DOMAIN_VRAM,
                              flags,
-                             NULL, NULL, init_pde_value, &vm->root.bo);
+                             NULL, NULL, init_pde_value, &vm->root.base.bo);
         if (r)
                 goto error_free_sched_entity;
 
-        r = amdgpu_bo_reserve(vm->root.bo, false);
-        if (r)
-                goto error_free_root;
-
-        vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
+        vm->root.base.vm = vm;
+        list_add_tail(&vm->root.base.bo_list, &vm->root.base.bo->va);
+        INIT_LIST_HEAD(&vm->root.base.vm_status);
 
         if (vm->use_cpu_for_update) {
-                r = amdgpu_bo_kmap(vm->root.bo, NULL);
+                r = amdgpu_bo_reserve(vm->root.base.bo, false);
+                if (r)
+                        goto error_free_root;
+
+                r = amdgpu_bo_kmap(vm->root.base.bo, NULL);
+                amdgpu_bo_unreserve(vm->root.base.bo);
                 if (r)
                         goto error_free_root;
         }
 
-        amdgpu_bo_unreserve(vm->root.bo);
+        if (pasid) {
+                unsigned long flags;
+
+                spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
+                r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
+                              GFP_ATOMIC);
+                spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
+                if (r < 0)
+                        goto error_free_root;
+
+                vm->pasid = pasid;
+        }
+
+        INIT_KFIFO(vm->faults);
+        vm->fault_credit = 16;
 
         return 0;
 
 error_free_root:
-        amdgpu_bo_unref(&vm->root.bo->shadow);
-        amdgpu_bo_unref(&vm->root.bo);
-        vm->root.bo = NULL;
+        amdgpu_bo_unref(&vm->root.base.bo->shadow);
+        amdgpu_bo_unref(&vm->root.base.bo);
+        vm->root.base.bo = NULL;
 
 error_free_sched_entity:
         amd_sched_entity_fini(&ring->sched, &vm->entity);
@@ -2561,9 +2732,11 @@ static void amdgpu_vm_free_levels(struct amdgpu_vm_pt *level)
 {
         unsigned i;
 
-        if (level->bo) {
-                amdgpu_bo_unref(&level->bo->shadow);
-                amdgpu_bo_unref(&level->bo);
+        if (level->base.bo) {
+                list_del(&level->base.bo_list);
+                list_del(&level->base.vm_status);
+                amdgpu_bo_unref(&level->base.bo->shadow);
+                amdgpu_bo_unref(&level->base.bo);
         }
 
         if (level->entries)
@@ -2586,7 +2759,21 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 {
         struct amdgpu_bo_va_mapping *mapping, *tmp;
         bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
-        int i;
+        struct amdgpu_bo *root;
+        u64 fault;
+        int i, r;
+
+        /* Clear pending page faults from IH when the VM is destroyed */
+        while (kfifo_get(&vm->faults, &fault))
+                amdgpu_ih_clear_fault(adev, fault);
+
+        if (vm->pasid) {
+                unsigned long flags;
+
+                spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
+                idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
+                spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
+        }
 
         amd_sched_entity_fini(vm->entity.sched, &vm->entity);
 
@@ -2609,13 +2796,51 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
                 amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
         }
 
-        amdgpu_vm_free_levels(&vm->root);
-        dma_fence_put(vm->last_dir_update);
+        root = amdgpu_bo_ref(vm->root.base.bo);
+        r = amdgpu_bo_reserve(root, true);
+        if (r) {
+                dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
+        } else {
+                amdgpu_vm_free_levels(&vm->root);
+                amdgpu_bo_unreserve(root);
+        }
+        amdgpu_bo_unref(&root);
+        dma_fence_put(vm->last_update);
         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
                 amdgpu_vm_free_reserved_vmid(adev, vm, i);
 }
 
 /**
+ * amdgpu_vm_pasid_fault_credit - Check fault credit for given PASID
+ *
+ * @adev: amdgpu_device pointer
+ * @pasid: PASID do identify the VM
+ *
+ * This function is expected to be called in interrupt context. Returns
+ * true if there was fault credit, false otherwise
+ */
+bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
+                                  unsigned int pasid)
+{
+        struct amdgpu_vm *vm;
+
+        spin_lock(&adev->vm_manager.pasid_lock);
+        vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
+        spin_unlock(&adev->vm_manager.pasid_lock);
+        if (!vm)
+                /* VM not found, can't track fault credit */
+                return true;
+
+        /* No lock needed. only accessed by IRQ handler */
+        if (!vm->fault_credit)
+                /* Too many faults in this VM */
+                return false;
+
+        vm->fault_credit--;
+        return true;
+}
+
+/**
  * amdgpu_vm_manager_init - init the VM manager
  *
  * @adev: amdgpu_device pointer
@@ -2668,6 +2893,8 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
         adev->vm_manager.vm_update_mode = 0;
 #endif
 
+        idr_init(&adev->vm_manager.pasid_idr);
+        spin_lock_init(&adev->vm_manager.pasid_lock);
 }
 
 /**
@@ -2681,6 +2908,9 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
 {
         unsigned i, j;
 
+        WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
+        idr_destroy(&adev->vm_manager.pasid_idr);
+
         for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
                 struct amdgpu_vm_id_manager *id_mgr =
                         &adev->vm_manager.id_mgr[i];
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 6716355403ec..bae77353447b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -25,6 +25,7 @@
 #define __AMDGPU_VM_H__
 
 #include <linux/rbtree.h>
+#include <linux/idr.h>
 
 #include "gpu_scheduler.h"
 #include "amdgpu_sync.h"
@@ -72,6 +73,16 @@ struct amdgpu_bo_list_entry;
 #define AMDGPU_PTE_MTYPE(a)    ((uint64_t)a << 57)
 #define AMDGPU_PTE_MTYPE_MASK   AMDGPU_PTE_MTYPE(3ULL)
 
+/* For Raven */
+#define AMDGPU_MTYPE_CC 2
+
+#define AMDGPU_PTE_DEFAULT_ATC  (AMDGPU_PTE_SYSTEM      \
+                                | AMDGPU_PTE_SNOOPED    \
+                                | AMDGPU_PTE_EXECUTABLE \
+                                | AMDGPU_PTE_READABLE   \
+                                | AMDGPU_PTE_WRITEABLE  \
+                                | AMDGPU_PTE_MTYPE(AMDGPU_MTYPE_CC))
+
 /* How to programm VM fault handling */
 #define AMDGPU_VM_FAULT_STOP_NEVER      0
 #define AMDGPU_VM_FAULT_STOP_FIRST      1
@@ -83,7 +94,8 @@ struct amdgpu_bo_list_entry;
 #define AMDGPU_MMHUB                            1
 
 /* hardcode that limit for now */
-#define AMDGPU_VA_RESERVED_SIZE                 (8 << 20)
+#define AMDGPU_VA_RESERVED_SIZE                 (8ULL << 20)
+
 /* max vmids dedicated for process */
 #define AMDGPU_VM_MAX_RESERVED_VMID     1
 
@@ -105,17 +117,24 @@ struct amdgpu_vm_bo_base {
 
         /* protected by spinlock */
         struct list_head                vm_status;
+
+        /* protected by the BO being reserved */
+        bool                            moved;
 };
 
 struct amdgpu_vm_pt {
-        struct amdgpu_bo        *bo;
-        uint64_t                addr;
+        struct amdgpu_vm_bo_base        base;
+        uint64_t                        addr;
 
         /* array of page tables, one for each directory entry */
-        struct amdgpu_vm_pt     *entries;
-        unsigned                last_entry_used;
+        struct amdgpu_vm_pt             *entries;
+        unsigned                        last_entry_used;
 };
 
+#define AMDGPU_VM_FAULT(pasid, addr) (((u64)(pasid) << 48) | (addr))
+#define AMDGPU_VM_FAULT_PASID(fault) ((u64)(fault) >> 48)
+#define AMDGPU_VM_FAULT_ADDR(fault)  ((u64)(fault) & 0xfffffffff000ULL)
+
 struct amdgpu_vm {
         /* tree of virtual addresses mapped */
         struct rb_root_cached   va;
@@ -123,19 +142,21 @@ struct amdgpu_vm {
         /* protecting invalidated */
         spinlock_t              status_lock;
 
+        /* BOs who needs a validation */
+        struct list_head        evicted;
+
+        /* PT BOs which relocated and their parent need an update */
+        struct list_head        relocated;
+
         /* BOs moved, but not yet updated in the PT */
         struct list_head        moved;
 
-        /* BOs cleared in the PT because of a move */
-        struct list_head        cleared;
-
         /* BO mappings freed, but not yet updated in the PT */
         struct list_head        freed;
 
         /* contains the page directory */
         struct amdgpu_vm_pt     root;
-        struct dma_fence        *last_dir_update;
-        uint64_t                last_eviction_counter;
+        struct dma_fence        *last_update;
 
         /* protecting freed */
         spinlock_t              freed_lock;
@@ -143,8 +164,9 @@ struct amdgpu_vm {
         /* Scheduler entity for page table updates */
         struct amd_sched_entity entity;
 
-        /* client id */
+        /* client id and PASID (TODO: replace client_id with PASID) */
         u64                     client_id;
+        unsigned int            pasid;
         /* dedicated to vm */
         struct amdgpu_vm_id     *reserved_vmid[AMDGPU_MAX_VMHUBS];
 
@@ -153,6 +175,12 @@ struct amdgpu_vm {
 
         /* Flag to indicate ATS support from PTE for GFX9 */
         bool                    pte_support_ats;
+
+        /* Up to 128 pending retry page faults */
+        DECLARE_KFIFO(faults, u64, 128);
+
+        /* Limit non-retry fault storms */
+        unsigned int            fault_credit;
 };
 
 struct amdgpu_vm_id {
@@ -215,16 +243,27 @@ struct amdgpu_vm_manager {
          * BIT1[= 0] Compute updated by SDMA [= 1] by CPU
          */
         int                                     vm_update_mode;
+
+        /* PASID to VM mapping, will be used in interrupt context to
+         * look up VM of a page fault
+         */
+        struct idr                              pasid_idr;
+        spinlock_t                              pasid_lock;
 };
 
+int amdgpu_vm_alloc_pasid(unsigned int bits);
+void amdgpu_vm_free_pasid(unsigned int pasid);
 void amdgpu_vm_manager_init(struct amdgpu_device *adev);
 void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                   int vm_context);
+                   int vm_context, unsigned int pasid);
 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
+                                  unsigned int pasid);
 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
                          struct list_head *validated,
                          struct amdgpu_bo_list_entry *entry);
+bool amdgpu_vm_ready(struct amdgpu_vm *vm);
 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                               int (*callback)(void *p, struct amdgpu_bo *bo),
                               void *param);
@@ -243,13 +282,13 @@ int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
                           struct amdgpu_vm *vm,
                           struct dma_fence **fence);
-int amdgpu_vm_clear_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                          struct amdgpu_sync *sync);
+int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
+                           struct amdgpu_vm *vm);
 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
                         struct amdgpu_bo_va *bo_va,
                         bool clear);
 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
-                             struct amdgpu_bo *bo);
+                             struct amdgpu_bo *bo, bool evicted);
 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
                                        struct amdgpu_bo *bo);
 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
@@ -269,6 +308,8 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
                                 struct amdgpu_vm *vm,
                                 uint64_t saddr, uint64_t size);
+struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
+                                                         uint64_t addr);
 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
                       struct amdgpu_bo_va *bo_va);
 void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 26e900627971..4acca92f6a52 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -68,11 +68,6 @@ static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man)
         struct amdgpu_vram_mgr *mgr = man->priv;
 
         spin_lock(&mgr->lock);
-        if (!drm_mm_clean(&mgr->mm)) {
-                spin_unlock(&mgr->lock);
-                return -EBUSY;
-        }
-
         drm_mm_takedown(&mgr->mm);
         spin_unlock(&mgr->lock);
         kfree(mgr);
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c
index d69aa2e179bb..69500a8b4e2d 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.c
+++ b/drivers/gpu/drm/amd/amdgpu/atom.c
@@ -1343,8 +1343,11 @@ struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios)
                 idx = 0x80;
 
         str = CSTR(idx);
-        if (*str != '\0')
+        if (*str != '\0') {
                 pr_info("ATOM BIOS: %s\n", str);
+                strlcpy(ctx->vbios_version, str, sizeof(ctx->vbios_version));
+        }
+
 
         return ctx;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.h b/drivers/gpu/drm/amd/amdgpu/atom.h
index ddd8045accf3..a39170991afe 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.h
+++ b/drivers/gpu/drm/amd/amdgpu/atom.h
@@ -140,6 +140,7 @@ struct atom_context {
         int io_mode;
         uint32_t *scratch;
         int scratch_size_bytes;
+        char vbios_version[20];
 };
 
 extern int amdgpu_atom_debug;
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index cb508a211b2f..68b505c768ad 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -307,7 +307,6 @@ static int ci_set_power_limit(struct amdgpu_device *adev, u32 n);
 static int ci_set_overdrive_target_tdp(struct amdgpu_device *adev,
                                        u32 target_tdp);
 static int ci_update_uvd_dpm(struct amdgpu_device *adev, bool gate);
-static void ci_dpm_set_dpm_funcs(struct amdgpu_device *adev);
 static void ci_dpm_set_irq_funcs(struct amdgpu_device *adev);
 
 static PPSMC_Result amdgpu_ci_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
@@ -883,8 +882,9 @@ static int ci_power_control_set_level(struct amdgpu_device *adev)
         return ret;
 }
 
-static void ci_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
+static void ci_dpm_powergate_uvd(void *handle, bool gate)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         pi->uvd_power_gated = gate;
@@ -901,8 +901,9 @@ static void ci_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
         }
 }
 
-static bool ci_dpm_vblank_too_short(struct amdgpu_device *adev)
+static bool ci_dpm_vblank_too_short(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 vblank_time = amdgpu_dpm_get_vblank_time(adev);
         u32 switch_limit = adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5 ? 450 : 300;
 
@@ -1210,11 +1211,12 @@ static int ci_fan_ctrl_stop_smc_fan_control(struct amdgpu_device *adev)
         }
 }
 
-static int ci_dpm_get_fan_speed_percent(struct amdgpu_device *adev,
+static int ci_dpm_get_fan_speed_percent(void *handle,
                                         u32 *speed)
 {
         u32 duty, duty100;
         u64 tmp64;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (adev->pm.no_fan)
                 return -ENOENT;
@@ -1237,12 +1239,13 @@ static int ci_dpm_get_fan_speed_percent(struct amdgpu_device *adev,
         return 0;
 }
 
-static int ci_dpm_set_fan_speed_percent(struct amdgpu_device *adev,
+static int ci_dpm_set_fan_speed_percent(void *handle,
                                         u32 speed)
 {
         u32 tmp;
         u32 duty, duty100;
         u64 tmp64;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         if (adev->pm.no_fan)
@@ -1271,8 +1274,10 @@ static int ci_dpm_set_fan_speed_percent(struct amdgpu_device *adev,
         return 0;
 }
 
-static void ci_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode)
+static void ci_dpm_set_fan_control_mode(void *handle, u32 mode)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
         switch (mode) {
         case AMD_FAN_CTRL_NONE:
                 if (adev->pm.dpm.fan.ucode_fan_control)
@@ -1292,8 +1297,9 @@ static void ci_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode)
         }
 }
 
-static u32 ci_dpm_get_fan_control_mode(struct amdgpu_device *adev)
+static u32 ci_dpm_get_fan_control_mode(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         if (pi->fan_is_controlled_by_smc)
@@ -4378,9 +4384,10 @@ static u32 ci_get_lowest_enabled_level(struct amdgpu_device *adev,
 }
 
 
-static int ci_dpm_force_performance_level(struct amdgpu_device *adev,
+static int ci_dpm_force_performance_level(void *handle,
                                           enum amd_dpm_forced_level level)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         u32 tmp, levels, i;
         int ret;
@@ -5291,8 +5298,9 @@ static void ci_update_requested_ps(struct amdgpu_device *adev,
         adev->pm.dpm.requested_ps = &pi->requested_rps;
 }
 
-static int ci_dpm_pre_set_power_state(struct amdgpu_device *adev)
+static int ci_dpm_pre_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct amdgpu_ps requested_ps = *adev->pm.dpm.requested_ps;
         struct amdgpu_ps *new_ps = &requested_ps;
@@ -5304,8 +5312,9 @@ static int ci_dpm_pre_set_power_state(struct amdgpu_device *adev)
         return 0;
 }
 
-static void ci_dpm_post_set_power_state(struct amdgpu_device *adev)
+static void ci_dpm_post_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct amdgpu_ps *new_ps = &pi->requested_rps;
 
@@ -5479,8 +5488,9 @@ static void ci_dpm_disable(struct amdgpu_device *adev)
         ci_update_current_ps(adev, boot_ps);
 }
 
-static int ci_dpm_set_power_state(struct amdgpu_device *adev)
+static int ci_dpm_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct amdgpu_ps *new_ps = &pi->requested_rps;
         struct amdgpu_ps *old_ps = &pi->current_rps;
@@ -5551,8 +5561,10 @@ static void ci_dpm_reset_asic(struct amdgpu_device *adev)
 }
 #endif
 
-static void ci_dpm_display_configuration_changed(struct amdgpu_device *adev)
+static void ci_dpm_display_configuration_changed(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
         ci_program_display_gap(adev);
 }
 
@@ -6105,9 +6117,10 @@ static int ci_dpm_init(struct amdgpu_device *adev)
 }
 
 static void
-ci_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
+ci_dpm_debugfs_print_current_performance_level(void *handle,
                                                struct seq_file *m)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct amdgpu_ps *rps = &pi->current_rps;
         u32 sclk = ci_get_average_sclk_freq(adev);
@@ -6131,12 +6144,13 @@ ci_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
         seq_printf(m, "GPU load: %u %%\n", activity_percent);
 }
 
-static void ci_dpm_print_power_state(struct amdgpu_device *adev,
-                                     struct amdgpu_ps *rps)
+static void ci_dpm_print_power_state(void *handle, void *current_ps)
 {
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)current_ps;
         struct ci_ps *ps = ci_get_ps(rps);
         struct ci_pl *pl;
         int i;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         amdgpu_dpm_print_class_info(rps->class, rps->class2);
         amdgpu_dpm_print_cap_info(rps->caps);
@@ -6158,20 +6172,23 @@ static inline bool ci_are_power_levels_equal(const struct ci_pl *ci_cpl1,
                   (ci_cpl1->pcie_lane == ci_cpl2->pcie_lane));
 }
 
-static int ci_check_state_equal(struct amdgpu_device *adev,
-                                struct amdgpu_ps *cps,
-                                struct amdgpu_ps *rps,
+static int ci_check_state_equal(void *handle,
+                                void *current_ps,
+                                void *request_ps,
                                 bool *equal)
 {
         struct ci_ps *ci_cps;
         struct ci_ps *ci_rps;
         int i;
+        struct amdgpu_ps *cps = (struct amdgpu_ps *)current_ps;
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)request_ps;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (adev == NULL || cps == NULL || rps == NULL || equal == NULL)
                 return -EINVAL;
 
-        ci_cps = ci_get_ps(cps);
-        ci_rps = ci_get_ps(rps);
+        ci_cps = ci_get_ps((struct amdgpu_ps *)cps);
+        ci_rps = ci_get_ps((struct amdgpu_ps *)rps);
 
         if (ci_cps == NULL) {
                 *equal = false;
@@ -6199,8 +6216,9 @@ static int ci_check_state_equal(struct amdgpu_device *adev,
         return 0;
 }
 
-static u32 ci_dpm_get_sclk(struct amdgpu_device *adev, bool low)
+static u32 ci_dpm_get_sclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps);
 
@@ -6210,8 +6228,9 @@ static u32 ci_dpm_get_sclk(struct amdgpu_device *adev, bool low)
                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
 }
 
-static u32 ci_dpm_get_mclk(struct amdgpu_device *adev, bool low)
+static u32 ci_dpm_get_mclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps);
 
@@ -6222,10 +6241,11 @@ static u32 ci_dpm_get_mclk(struct amdgpu_device *adev, bool low)
 }
 
 /* get temperature in millidegrees */
-static int ci_dpm_get_temp(struct amdgpu_device *adev)
+static int ci_dpm_get_temp(void *handle)
 {
         u32 temp;
         int actual_temp = 0;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         temp = (RREG32_SMC(ixCG_MULT_THERMAL_STATUS) & CG_MULT_THERMAL_STATUS__CTF_TEMP_MASK) >>
                 CG_MULT_THERMAL_STATUS__CTF_TEMP__SHIFT;
@@ -6261,7 +6281,6 @@ static int ci_dpm_early_init(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        ci_dpm_set_dpm_funcs(adev);
         ci_dpm_set_irq_funcs(adev);
 
         return 0;
@@ -6346,7 +6365,6 @@ static int ci_dpm_sw_fini(void *handle)
         flush_work(&adev->pm.dpm.thermal.work);
 
         mutex_lock(&adev->pm.mutex);
-        amdgpu_pm_sysfs_fini(adev);
         ci_dpm_fini(adev);
         mutex_unlock(&adev->pm.mutex);
 
@@ -6551,9 +6569,10 @@ static int ci_dpm_set_powergating_state(void *handle,
         return 0;
 }
 
-static int ci_dpm_print_clock_levels(struct amdgpu_device *adev,
+static int ci_dpm_print_clock_levels(void *handle,
                 enum pp_clock_type type, char *buf)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_single_dpm_table *sclk_table = &pi->dpm_table.sclk_table;
         struct ci_single_dpm_table *mclk_table = &pi->dpm_table.mclk_table;
@@ -6618,9 +6637,10 @@ static int ci_dpm_print_clock_levels(struct amdgpu_device *adev,
         return size;
 }
 
-static int ci_dpm_force_clock_level(struct amdgpu_device *adev,
+static int ci_dpm_force_clock_level(void *handle,
                 enum pp_clock_type type, uint32_t mask)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         if (adev->pm.dpm.forced_level & (AMD_DPM_FORCED_LEVEL_AUTO |
@@ -6664,8 +6684,9 @@ static int ci_dpm_force_clock_level(struct amdgpu_device *adev,
         return 0;
 }
 
-static int ci_dpm_get_sclk_od(struct amdgpu_device *adev)
+static int ci_dpm_get_sclk_od(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_single_dpm_table *sclk_table = &(pi->dpm_table.sclk_table);
         struct ci_single_dpm_table *golden_sclk_table =
@@ -6680,8 +6701,9 @@ static int ci_dpm_get_sclk_od(struct amdgpu_device *adev)
         return value;
 }
 
-static int ci_dpm_set_sclk_od(struct amdgpu_device *adev, uint32_t value)
+static int ci_dpm_set_sclk_od(void *handle, uint32_t value)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_ps *ps = ci_get_ps(adev->pm.dpm.requested_ps);
         struct ci_single_dpm_table *golden_sclk_table =
@@ -6698,8 +6720,9 @@ static int ci_dpm_set_sclk_od(struct amdgpu_device *adev, uint32_t value)
         return 0;
 }
 
-static int ci_dpm_get_mclk_od(struct amdgpu_device *adev)
+static int ci_dpm_get_mclk_od(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_single_dpm_table *mclk_table = &(pi->dpm_table.mclk_table);
         struct ci_single_dpm_table *golden_mclk_table =
@@ -6714,8 +6737,9 @@ static int ci_dpm_get_mclk_od(struct amdgpu_device *adev)
         return value;
 }
 
-static int ci_dpm_set_mclk_od(struct amdgpu_device *adev, uint32_t value)
+static int ci_dpm_set_mclk_od(void *handle, uint32_t value)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct ci_ps *ps = ci_get_ps(adev->pm.dpm.requested_ps);
         struct ci_single_dpm_table *golden_mclk_table =
@@ -6732,9 +6756,10 @@ static int ci_dpm_set_mclk_od(struct amdgpu_device *adev, uint32_t value)
         return 0;
 }
 
-static int ci_dpm_get_power_profile_state(struct amdgpu_device *adev,
+static int ci_dpm_get_power_profile_state(void *handle,
                 struct amd_pp_profile *query)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         if (!pi || !query)
@@ -6851,9 +6876,10 @@ static int ci_set_power_profile_state(struct amdgpu_device *adev,
         return result;
 }
 
-static int ci_dpm_set_power_profile_state(struct amdgpu_device *adev,
+static int ci_dpm_set_power_profile_state(void *handle,
                 struct amd_pp_profile *request)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         int ret = -1;
 
@@ -6906,9 +6932,10 @@ static int ci_dpm_set_power_profile_state(struct amdgpu_device *adev,
         return 0;
 }
 
-static int ci_dpm_reset_power_profile_state(struct amdgpu_device *adev,
+static int ci_dpm_reset_power_profile_state(void *handle,
                 struct amd_pp_profile *request)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
 
         if (!pi || !request)
@@ -6927,9 +6954,10 @@ static int ci_dpm_reset_power_profile_state(struct amdgpu_device *adev,
                 return -EINVAL;
 }
 
-static int ci_dpm_switch_power_profile(struct amdgpu_device *adev,
+static int ci_dpm_switch_power_profile(void *handle,
                 enum amd_pp_profile_type type)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct ci_power_info *pi = ci_get_pi(adev);
         struct amd_pp_profile request = {0};
 
@@ -6944,11 +6972,12 @@ static int ci_dpm_switch_power_profile(struct amdgpu_device *adev,
         return 0;
 }
 
-static int ci_dpm_read_sensor(struct amdgpu_device *adev, int idx,
+static int ci_dpm_read_sensor(void *handle, int idx,
                               void *value, int *size)
 {
         u32 activity_percent = 50;
         int ret;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         /* size must be at least 4 bytes for all sensors */
         if (*size < 4)
@@ -7003,7 +7032,7 @@ const struct amd_ip_funcs ci_dpm_ip_funcs = {
         .set_powergating_state = ci_dpm_set_powergating_state,
 };
 
-static const struct amdgpu_dpm_funcs ci_dpm_funcs = {
+const struct amd_pm_funcs ci_dpm_funcs = {
         .get_temperature = &ci_dpm_get_temp,
         .pre_set_power_state = &ci_dpm_pre_set_power_state,
         .set_power_state = &ci_dpm_set_power_state,
@@ -7035,12 +7064,6 @@ static const struct amdgpu_dpm_funcs ci_dpm_funcs = {
         .read_sensor = ci_dpm_read_sensor,
 };
 
-static void ci_dpm_set_dpm_funcs(struct amdgpu_device *adev)
-{
-        if (adev->pm.funcs == NULL)
-                adev->pm.funcs = &ci_dpm_funcs;
-}
-
 static const struct amdgpu_irq_src_funcs ci_dpm_irq_funcs = {
         .set = ci_dpm_set_interrupt_state,
         .process = ci_dpm_process_interrupt,
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 567c4a5cf90c..793b1470284d 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -65,6 +65,7 @@
 #include "oss/oss_2_0_d.h"
 #include "oss/oss_2_0_sh_mask.h"
 
+#include "amdgpu_dm.h"
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_powerplay.h"
 #include "dce_virtual.h"
@@ -1900,6 +1901,10 @@ int cik_set_ip_blocks(struct amdgpu_device *adev)
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
                 if (adev->enable_virtual_display)
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+                else if (amdgpu_device_has_dc_support(adev))
+                        amdgpu_ip_block_add(adev, &dm_ip_block);
+#endif
                 else
                         amdgpu_ip_block_add(adev, &dce_v8_2_ip_block);
                 amdgpu_ip_block_add(adev, &gfx_v7_2_ip_block);
@@ -1914,6 +1919,10 @@ int cik_set_ip_blocks(struct amdgpu_device *adev)
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
                 if (adev->enable_virtual_display)
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+                else if (amdgpu_device_has_dc_support(adev))
+                        amdgpu_ip_block_add(adev, &dm_ip_block);
+#endif
                 else
                         amdgpu_ip_block_add(adev, &dce_v8_5_ip_block);
                 amdgpu_ip_block_add(adev, &gfx_v7_3_ip_block);
@@ -1928,6 +1937,10 @@ int cik_set_ip_blocks(struct amdgpu_device *adev)
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
                 if (adev->enable_virtual_display)
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+                else if (amdgpu_device_has_dc_support(adev))
+                        amdgpu_ip_block_add(adev, &dm_ip_block);
+#endif
                 else
                         amdgpu_ip_block_add(adev, &dce_v8_1_ip_block);
                 amdgpu_ip_block_add(adev, &gfx_v7_1_ip_block);
@@ -1943,6 +1956,10 @@ int cik_set_ip_blocks(struct amdgpu_device *adev)
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
                 if (adev->enable_virtual_display)
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+                else if (amdgpu_device_has_dc_support(adev))
+                        amdgpu_ip_block_add(adev, &dm_ip_block);
+#endif
                 else
                         amdgpu_ip_block_add(adev, &dce_v8_3_ip_block);
                 amdgpu_ip_block_add(adev, &gfx_v7_2_ip_block);
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_dpm.h b/drivers/gpu/drm/amd/amdgpu/cik_dpm.h
index b1c8e7b446ea..c7b4349f6319 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/cik_dpm.h
@@ -26,5 +26,6 @@
 
 extern const struct amd_ip_funcs ci_dpm_ip_funcs;
 extern const struct amd_ip_funcs kv_dpm_ip_funcs;
-
+extern const struct amd_pm_funcs ci_dpm_funcs;
+extern const struct amd_pm_funcs kv_dpm_funcs;
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
index b8918432c572..a870b354e3f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
@@ -228,6 +228,34 @@ static u32 cik_ih_get_wptr(struct amdgpu_device *adev)
  * [127:96] - reserved
  */
 
+/**
+ * cik_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool cik_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        u32 ring_index = adev->irq.ih.rptr >> 2;
+        u16 pasid;
+
+        switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
+        case 146:
+        case 147:
+                pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
+                if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
+                        return true;
+                break;
+        default:
+                /* Not a VM fault */
+                return true;
+        }
+
+        adev->irq.ih.rptr += 16;
+        return false;
+}
+
  /**
  * cik_ih_decode_iv - decode an interrupt vector
  *
@@ -433,6 +461,7 @@ static const struct amd_ip_funcs cik_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs cik_ih_funcs = {
         .get_wptr = cik_ih_get_wptr,
+        .prescreen_iv = cik_ih_prescreen_iv,
         .decode_iv = cik_ih_decode_iv,
         .set_rptr = cik_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index f508f4d01e4a..60cecd117705 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -1387,8 +1387,13 @@ static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev)
 }
 
 static const struct amdgpu_vm_pte_funcs cik_sdma_vm_pte_funcs = {
+        .copy_pte_num_dw = 7,
         .copy_pte = cik_sdma_vm_copy_pte,
+
         .write_pte = cik_sdma_vm_write_pte,
+
+        .set_max_nums_pte_pde = 0x1fffff >> 3,
+        .set_pte_pde_num_dw = 10,
         .set_pte_pde = cik_sdma_vm_set_pte_pde,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
index 0c1209cdd1cb..fa61d649bb44 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
@@ -208,6 +208,34 @@ static u32 cz_ih_get_wptr(struct amdgpu_device *adev)
 }
 
 /**
+ * cz_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool cz_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        u32 ring_index = adev->irq.ih.rptr >> 2;
+        u16 pasid;
+
+        switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
+        case 146:
+        case 147:
+                pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
+                if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
+                        return true;
+                break;
+        default:
+                /* Not a VM fault */
+                return true;
+        }
+
+        adev->irq.ih.rptr += 16;
+        return false;
+}
+
+/**
  * cz_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -414,6 +442,7 @@ static const struct amd_ip_funcs cz_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs cz_ih_funcs = {
         .get_wptr = cz_ih_get_wptr,
+        .prescreen_iv = cz_ih_prescreen_iv,
         .decode_iv = cz_ih_decode_iv,
         .set_rptr = cz_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
index b9ee9073cb0d..a8829af120c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -288,7 +288,7 @@ dce_virtual_encoder(struct drm_connector *connector)
                 if (connector->encoder_ids[i] == 0)
                         break;
 
-                encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]);
+                encoder = drm_encoder_find(connector->dev, NULL, connector->encoder_ids[i]);
                 if (!encoder)
                         continue;
 
@@ -298,7 +298,7 @@ dce_virtual_encoder(struct drm_connector *connector)
 
         /* pick the first one */
         if (enc_id)
-                return drm_encoder_find(connector->dev, enc_id);
+                return drm_encoder_find(connector->dev, NULL, enc_id);
         return NULL;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 00868764a0dd..5c8a7a48a4ad 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -4670,6 +4670,14 @@ static int gfx_v7_0_sw_fini(void *handle)
         gfx_v7_0_cp_compute_fini(adev);
         gfx_v7_0_rlc_fini(adev);
         gfx_v7_0_mec_fini(adev);
+        amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
+                                &adev->gfx.rlc.clear_state_gpu_addr,
+                                (void **)&adev->gfx.rlc.cs_ptr);
+        if (adev->gfx.rlc.cp_table_size) {
+                amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
+                                &adev->gfx.rlc.cp_table_gpu_addr,
+                                (void **)&adev->gfx.rlc.cp_table_ptr);
+        }
         gfx_v7_0_free_microcode(adev);
 
         return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index fc260c13b1da..9ecdf621a74a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -20,6 +20,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include <linux/kernel.h>
 #include <linux/firmware.h>
 #include <drm/drmP.h>
 #include "amdgpu.h"
@@ -125,24 +126,39 @@ MODULE_FIRMWARE("amdgpu/fiji_mec2.bin");
 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin");
 
 MODULE_FIRMWARE("amdgpu/polaris11_ce.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_me.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_mec.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin");
 
 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_me.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_mec.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin");
 
 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin");
+MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin");
+MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_me.bin");
+MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_mec.bin");
+MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin");
+MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin");
 
 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
@@ -918,8 +934,17 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
                 BUG();
         }
 
-        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
-        err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
+        if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp_2.bin", chip_name);
+                err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
+                if (err == -ENOENT) {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
+                        err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
+                }
+        } else {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
+                err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
+        }
         if (err)
                 goto out;
         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
@@ -929,8 +954,17 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
         adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
         adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 
-        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
-        err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
+        if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me_2.bin", chip_name);
+                err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
+                if (err == -ENOENT) {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
+                        err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
+                }
+        } else {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
+                err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
+        }
         if (err)
                 goto out;
         err = amdgpu_ucode_validate(adev->gfx.me_fw);
@@ -941,8 +975,17 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
 
         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 
-        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
-        err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
+        if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce_2.bin", chip_name);
+                err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
+                if (err == -ENOENT) {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
+                        err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
+                }
+        } else {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
+                err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
+        }
         if (err)
                 goto out;
         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
@@ -1012,8 +1055,17 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
         for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
                 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
 
-        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
-        err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
+        if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec_2.bin", chip_name);
+                err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
+                if (err == -ENOENT) {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
+                        err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
+                }
+        } else {
+                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
+                err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
+        }
         if (err)
                 goto out;
         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
@@ -1025,8 +1077,17 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
 
         if ((adev->asic_type != CHIP_STONEY) &&
             (adev->asic_type != CHIP_TOPAZ)) {
-                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
-                err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
+                if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2_2.bin", chip_name);
+                        err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
+                        if (err == -ENOENT) {
+                                snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
+                                err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
+                        }
+                } else {
+                        snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
+                        err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
+                }
                 if (!err) {
                         err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
                         if (err)
@@ -2053,9 +2114,19 @@ static int gfx_v8_0_sw_fini(void *handle)
         amdgpu_gfx_compute_mqd_sw_fini(adev);
         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
         amdgpu_gfx_kiq_fini(adev);
+        amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
 
         gfx_v8_0_mec_fini(adev);
         gfx_v8_0_rlc_fini(adev);
+        amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
+                                &adev->gfx.rlc.clear_state_gpu_addr,
+                                (void **)&adev->gfx.rlc.cs_ptr);
+        if ((adev->asic_type == CHIP_CARRIZO) ||
+            (adev->asic_type == CHIP_STONEY)) {
+                amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
+                                &adev->gfx.rlc.cp_table_gpu_addr,
+                                (void **)&adev->gfx.rlc.cp_table_ptr);
+        }
         gfx_v8_0_free_microcode(adev);
 
         return 0;
@@ -3891,10 +3962,10 @@ static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
                                 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
                                 unique_indices,
                                 &indices_count,
-                                sizeof(unique_indices) / sizeof(int),
+                                ARRAY_SIZE(unique_indices),
                                 indirect_start_offsets,
                                 &offset_count,
-                                sizeof(indirect_start_offsets)/sizeof(int));
+                                ARRAY_SIZE(indirect_start_offsets));
 
         /* save and restore list */
         WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1);
@@ -3916,14 +3987,14 @@ static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
         /* starting offsets starts */
         WREG32(mmRLC_GPM_SCRATCH_ADDR,
                 adev->gfx.rlc.starting_offsets_start);
-        for (i = 0; i < sizeof(indirect_start_offsets)/sizeof(int); i++)
+        for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
                 WREG32(mmRLC_GPM_SCRATCH_DATA,
                                 indirect_start_offsets[i]);
 
         /* unique indices */
         temp = mmRLC_SRM_INDEX_CNTL_ADDR_0;
         data = mmRLC_SRM_INDEX_CNTL_DATA_0;
-        for (i = 0; i < sizeof(unique_indices) / sizeof(int); i++) {
+        for (i = 0; i < ARRAY_SIZE(unique_indices); i++) {
                 if (unique_indices[i] != 0) {
                         WREG32(temp + i, unique_indices[i] & 0x3FFFF);
                         WREG32(data + i, unique_indices[i] >> 20);
@@ -4071,18 +4142,12 @@ static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
         gfx_v8_0_rlc_reset(adev);
         gfx_v8_0_init_pg(adev);
 
-        if (!adev->pp_enabled) {
-                if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) {
-                        /* legacy rlc firmware loading */
-                        r = gfx_v8_0_rlc_load_microcode(adev);
-                        if (r)
-                                return r;
-                } else {
-                        r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-                                                        AMDGPU_UCODE_ID_RLC_G);
-                        if (r)
-                                return -EINVAL;
-                }
+
+        if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+                /* legacy rlc firmware loading */
+                r = gfx_v8_0_rlc_load_microcode(adev);
+                if (r)
+                        return r;
         }
 
         gfx_v8_0_rlc_start(adev);
@@ -4577,12 +4642,10 @@ static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
         mqd->compute_misc_reserved = 0x00000003;
-        if (!(adev->flags & AMD_IS_APU)) {
-                mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr
-                                             + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
-                mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr
-                                             + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
-        }
+        mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr
+                                                     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
+        mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr
+                                                     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
         eop_base_addr = ring->eop_gpu_addr >> 8;
         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
@@ -4753,7 +4816,7 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
 
         gfx_v8_0_kiq_setting(ring);
 
-        if (adev->gfx.in_reset) { /* for GPU_RESET case */
+        if (adev->in_sriov_reset) { /* for GPU_RESET case */
                 /* reset MQD to a clean status */
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
@@ -4790,7 +4853,7 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
         struct vi_mqd *mqd = ring->mqd_ptr;
         int mqd_idx = ring - &adev->gfx.compute_ring[0];
 
-        if (!adev->gfx.in_reset && !adev->gfx.in_suspend) {
+        if (!adev->in_sriov_reset && !adev->gfx.in_suspend) {
                 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
                 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
                 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
@@ -4802,7 +4865,7 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
 
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
                         memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
-        } else if (adev->gfx.in_reset) { /* for GPU_RESET case */
+        } else if (adev->in_sriov_reset) { /* for GPU_RESET case */
                 /* reset MQD to a clean status */
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
                         memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
@@ -4900,43 +4963,15 @@ static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
         if (!(adev->flags & AMD_IS_APU))
                 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
 
-        if (!adev->pp_enabled) {
-                if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) {
+        if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
                         /* legacy firmware loading */
-                        r = gfx_v8_0_cp_gfx_load_microcode(adev);
-                        if (r)
-                                return r;
+                r = gfx_v8_0_cp_gfx_load_microcode(adev);
+                if (r)
+                        return r;
 
-                        r = gfx_v8_0_cp_compute_load_microcode(adev);
-                        if (r)
-                                return r;
-                } else {
-                        r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-                                                        AMDGPU_UCODE_ID_CP_CE);
-                        if (r)
-                                return -EINVAL;
-
-                        r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-                                                        AMDGPU_UCODE_ID_CP_PFP);
-                        if (r)
-                                return -EINVAL;
-
-                        r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-                                                        AMDGPU_UCODE_ID_CP_ME);
-                        if (r)
-                                return -EINVAL;
-
-                        if (adev->asic_type == CHIP_TOPAZ) {
-                                r = gfx_v8_0_cp_compute_load_microcode(adev);
-                                if (r)
-                                        return r;
-                        } else {
-                                r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-                                                                                 AMDGPU_UCODE_ID_CP_MEC1);
-                                if (r)
-                                        return -EINVAL;
-                        }
-                }
+                r = gfx_v8_0_cp_compute_load_microcode(adev);
+                if (r)
+                        return r;
         }
 
         r = gfx_v8_0_cp_gfx_resume(adev);
@@ -4975,12 +5010,69 @@ static int gfx_v8_0_hw_init(void *handle)
         return r;
 }
 
+static int gfx_v8_0_kcq_disable(struct amdgpu_ring *kiq_ring,struct amdgpu_ring *ring)
+{
+        struct amdgpu_device *adev = kiq_ring->adev;
+        uint32_t scratch, tmp = 0;
+        int r, i;
+
+        r = amdgpu_gfx_scratch_get(adev, &scratch);
+        if (r) {
+                DRM_ERROR("Failed to get scratch reg (%d).\n", r);
+                return r;
+        }
+        WREG32(scratch, 0xCAFEDEAD);
+
+        r = amdgpu_ring_alloc(kiq_ring, 10);
+        if (r) {
+                DRM_ERROR("Failed to lock KIQ (%d).\n", r);
+                amdgpu_gfx_scratch_free(adev, scratch);
+                return r;
+        }
+
+        /* unmap queues */
+        amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
+        amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
+                                                PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */
+                                                PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
+                                                PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) |
+                                                PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
+        amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
+        amdgpu_ring_write(kiq_ring, 0);
+        amdgpu_ring_write(kiq_ring, 0);
+        amdgpu_ring_write(kiq_ring, 0);
+        /* write to scratch for completion */
+        amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
+        amdgpu_ring_write(kiq_ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
+        amdgpu_ring_write(kiq_ring, 0xDEADBEEF);
+        amdgpu_ring_commit(kiq_ring);
+
+        for (i = 0; i < adev->usec_timeout; i++) {
+                tmp = RREG32(scratch);
+                if (tmp == 0xDEADBEEF)
+                        break;
+                DRM_UDELAY(1);
+        }
+        if (i >= adev->usec_timeout) {
+                DRM_ERROR("KCQ disabled failed (scratch(0x%04X)=0x%08X)\n", scratch, tmp);
+                r = -EINVAL;
+        }
+        amdgpu_gfx_scratch_free(adev, scratch);
+        return r;
+}
+
 static int gfx_v8_0_hw_fini(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int i;
 
         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+
+        /* disable KCQ to avoid CPC touch memory not valid anymore */
+        for (i = 0; i < adev->gfx.num_compute_rings; i++)
+                gfx_v8_0_kcq_disable(&adev->gfx.kiq.ring, &adev->gfx.compute_ring[i]);
+
         if (amdgpu_sriov_vf(adev)) {
                 pr_debug("For SRIOV client, shouldn't do anything.\n");
                 return 0;
@@ -5902,7 +5994,6 @@ static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
 {
         uint32_t msg_id, pp_state = 0;
         uint32_t pp_support_state = 0;
-        void *pp_handle = adev->powerplay.pp_handle;
 
         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
@@ -5920,7 +6011,8 @@ static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
                                 PP_BLOCK_GFX_CG,
                                 pp_support_state,
                                 pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
@@ -5941,7 +6033,8 @@ static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
                                 PP_BLOCK_GFX_MG,
                                 pp_support_state,
                                 pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
         return 0;
@@ -5953,7 +6046,6 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
 
         uint32_t msg_id, pp_state = 0;
         uint32_t pp_support_state = 0;
-        void *pp_handle = adev->powerplay.pp_handle;
 
         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
@@ -5971,7 +6063,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
                                 PP_BLOCK_GFX_CG,
                                 pp_support_state,
                                 pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) {
@@ -5990,7 +6083,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
                                 PP_BLOCK_GFX_3D,
                                 pp_support_state,
                                 pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
         if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
@@ -6011,7 +6105,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
                                 PP_BLOCK_GFX_MG,
                                 pp_support_state,
                                 pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
@@ -6026,7 +6121,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
                                 PP_BLOCK_GFX_RLC,
                                 pp_support_state,
                                 pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
@@ -6040,7 +6136,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
                         PP_BLOCK_GFX_CP,
                         pp_support_state,
                         pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
         return 0;
@@ -6307,6 +6404,104 @@ static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
 }
 
+static void gfx_v8_0_ring_set_pipe_percent(struct amdgpu_ring *ring,
+                                           bool acquire)
+{
+        struct amdgpu_device *adev = ring->adev;
+        int pipe_num, tmp, reg;
+        int pipe_percent = acquire ? SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK : 0x1;
+
+        pipe_num = ring->me * adev->gfx.mec.num_pipe_per_mec + ring->pipe;
+
+        /* first me only has 2 entries, GFX and HP3D */
+        if (ring->me > 0)
+                pipe_num -= 2;
+
+        reg = mmSPI_WCL_PIPE_PERCENT_GFX + pipe_num;
+        tmp = RREG32(reg);
+        tmp = REG_SET_FIELD(tmp, SPI_WCL_PIPE_PERCENT_GFX, VALUE, pipe_percent);
+        WREG32(reg, tmp);
+}
+
+static void gfx_v8_0_pipe_reserve_resources(struct amdgpu_device *adev,
+                                            struct amdgpu_ring *ring,
+                                            bool acquire)
+{
+        int i, pipe;
+        bool reserve;
+        struct amdgpu_ring *iring;
+
+        mutex_lock(&adev->gfx.pipe_reserve_mutex);
+        pipe = amdgpu_gfx_queue_to_bit(adev, ring->me, ring->pipe, 0);
+        if (acquire)
+                set_bit(pipe, adev->gfx.pipe_reserve_bitmap);
+        else
+                clear_bit(pipe, adev->gfx.pipe_reserve_bitmap);
+
+        if (!bitmap_weight(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)) {
+                /* Clear all reservations - everyone reacquires all resources */
+                for (i = 0; i < adev->gfx.num_gfx_rings; ++i)
+                        gfx_v8_0_ring_set_pipe_percent(&adev->gfx.gfx_ring[i],
+                                                       true);
+
+                for (i = 0; i < adev->gfx.num_compute_rings; ++i)
+                        gfx_v8_0_ring_set_pipe_percent(&adev->gfx.compute_ring[i],
+                                                       true);
+        } else {
+                /* Lower all pipes without a current reservation */
+                for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
+                        iring = &adev->gfx.gfx_ring[i];
+                        pipe = amdgpu_gfx_queue_to_bit(adev,
+                                                       iring->me,
+                                                       iring->pipe,
+                                                       0);
+                        reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
+                        gfx_v8_0_ring_set_pipe_percent(iring, reserve);
+                }
+
+                for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
+                        iring = &adev->gfx.compute_ring[i];
+                        pipe = amdgpu_gfx_queue_to_bit(adev,
+                                                       iring->me,
+                                                       iring->pipe,
+                                                       0);
+                        reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
+                        gfx_v8_0_ring_set_pipe_percent(iring, reserve);
+                }
+        }
+
+        mutex_unlock(&adev->gfx.pipe_reserve_mutex);
+}
+
+static void gfx_v8_0_hqd_set_priority(struct amdgpu_device *adev,
+                                      struct amdgpu_ring *ring,
+                                      bool acquire)
+{
+        uint32_t pipe_priority = acquire ? 0x2 : 0x0;
+        uint32_t queue_priority = acquire ? 0xf : 0x0;
+
+        mutex_lock(&adev->srbm_mutex);
+        vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
+
+        WREG32(mmCP_HQD_PIPE_PRIORITY, pipe_priority);
+        WREG32(mmCP_HQD_QUEUE_PRIORITY, queue_priority);
+
+        vi_srbm_select(adev, 0, 0, 0, 0);
+        mutex_unlock(&adev->srbm_mutex);
+}
+static void gfx_v8_0_ring_set_priority_compute(struct amdgpu_ring *ring,
+                                               enum amd_sched_priority priority)
+{
+        struct amdgpu_device *adev = ring->adev;
+        bool acquire = priority == AMD_SCHED_PRIORITY_HIGH_HW;
+
+        if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
+                return;
+
+        gfx_v8_0_hqd_set_priority(adev, ring, acquire);
+        gfx_v8_0_pipe_reserve_resources(adev, ring, acquire);
+}
+
 static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
                                              u64 addr, u64 seq,
                                              unsigned flags)
@@ -6752,6 +6947,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
         .test_ib = gfx_v8_0_ring_test_ib,
         .insert_nop = amdgpu_ring_insert_nop,
         .pad_ib = amdgpu_ring_generic_pad_ib,
+        .set_priority = gfx_v8_0_ring_set_priority_compute,
 };
 
 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
@@ -6960,7 +7156,7 @@ static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
 {
         uint64_t ce_payload_addr;
         int cnt_ce;
-        static union {
+        union {
                 struct vi_ce_ib_state regular;
                 struct vi_ce_ib_state_chained_ib chained;
         } ce_payload = {};
@@ -6989,7 +7185,7 @@ static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
 {
         uint64_t de_payload_addr, gds_addr, csa_addr;
         int cnt_de;
-        static union {
+        union {
                 struct vi_de_ib_state regular;
                 struct vi_de_ib_state_chained_ib chained;
         } de_payload = {};
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 69182eeca264..da43813d67a4 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -20,6 +20,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include <linux/kernel.h>
 #include <linux/firmware.h>
 #include <drm/drmP.h>
 #include "amdgpu.h"
@@ -66,38 +67,70 @@ MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
 
 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
 {
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID1), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID1)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID2), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID2)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID3), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID3)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID4), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID4)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID5), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID5)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID6), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID6)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID7), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID7)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID8), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID8)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID9), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID9)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID10), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID10)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID11), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID11)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID12), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID12)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID13), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID13)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID14), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID14)},
-        {SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_BASE), SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_SIZE),
-                SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID15), SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID15)}
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID1),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID1) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID2),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID2) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID3),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID3) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID4),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID4) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID5),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID5) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID6),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID6) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID7),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID7) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID8),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID8) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID9),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID9) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID10),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID10) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID11),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID11) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID12),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID12)},
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID13),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID13) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID14),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID14) },
+        { SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_BASE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_SIZE),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID15),
+          SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID15) }
 };
 
 static const u32 golden_settings_gc_9_0[] =
@@ -174,6 +207,12 @@ static const u32 golden_settings_gc_9_1_rv1[] =
         SOC15_REG_OFFSET(GC, 0, mmTD_CNTL), 0x01bd9f33, 0x00000800
 };
 
+static const u32 golden_settings_gc_9_x_common[] =
+{
+        SOC15_REG_OFFSET(GC, 0, mmGRBM_CAM_INDEX), 0xffffffff, 0x00000000,
+        SOC15_REG_OFFSET(GC, 0, mmGRBM_CAM_DATA), 0xffffffff, 0x2544c382
+};
+
 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
 
@@ -209,6 +248,9 @@ static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
         default:
                 break;
         }
+
+        amdgpu_program_register_sequence(adev, golden_settings_gc_9_x_common,
+                                        (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
 }
 
 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
@@ -352,6 +394,25 @@ err1:
         return r;
 }
 
+
+static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
+{
+        release_firmware(adev->gfx.pfp_fw);
+        adev->gfx.pfp_fw = NULL;
+        release_firmware(adev->gfx.me_fw);
+        adev->gfx.me_fw = NULL;
+        release_firmware(adev->gfx.ce_fw);
+        adev->gfx.ce_fw = NULL;
+        release_firmware(adev->gfx.rlc_fw);
+        adev->gfx.rlc_fw = NULL;
+        release_firmware(adev->gfx.mec_fw);
+        adev->gfx.mec_fw = NULL;
+        release_firmware(adev->gfx.mec2_fw);
+        adev->gfx.mec2_fw = NULL;
+
+        kfree(adev->gfx.rlc.register_list_format);
+}
+
 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 {
         const char *chip_name;
@@ -936,12 +997,22 @@ static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
 }
 
+static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
+                                     uint32_t wave, uint32_t thread,
+                                     uint32_t start, uint32_t size,
+                                     uint32_t *dst)
+{
+        wave_read_regs(
+                adev, simd, wave, thread,
+                start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
+}
 
 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
         .select_se_sh = &gfx_v9_0_select_se_sh,
         .read_wave_data = &gfx_v9_0_read_wave_data,
         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
+        .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
 };
 
 static void gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
@@ -1120,30 +1191,22 @@ static int gfx_v9_0_ngg_en(struct amdgpu_device *adev)
 {
         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
         int r;
-        u32 data;
-        u32 size;
-        u32 base;
+        u32 data, base;
 
         if (!amdgpu_ngg)
                 return 0;
 
         /* Program buffer size */
-        data = 0;
-        size = adev->gfx.ngg.buf[NGG_PRIM].size / 256;
-        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, INDEX_BUF_SIZE, size);
-
-        size = adev->gfx.ngg.buf[NGG_POS].size / 256;
-        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, POS_BUF_SIZE, size);
-
+        data = REG_SET_FIELD(0, WD_BUF_RESOURCE_1, INDEX_BUF_SIZE,
+                             adev->gfx.ngg.buf[NGG_PRIM].size >> 8);
+        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, POS_BUF_SIZE,
+                             adev->gfx.ngg.buf[NGG_POS].size >> 8);
         WREG32_SOC15(GC, 0, mmWD_BUF_RESOURCE_1, data);
 
-        data = 0;
-        size = adev->gfx.ngg.buf[NGG_CNTL].size / 256;
-        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, CNTL_SB_BUF_SIZE, size);
-
-        size = adev->gfx.ngg.buf[NGG_PARAM].size / 1024;
-        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, PARAM_BUF_SIZE, size);
-
+        data = REG_SET_FIELD(0, WD_BUF_RESOURCE_2, CNTL_SB_BUF_SIZE,
+                             adev->gfx.ngg.buf[NGG_CNTL].size >> 8);
+        data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, PARAM_BUF_SIZE,
+                             adev->gfx.ngg.buf[NGG_PARAM].size >> 10);
         WREG32_SOC15(GC, 0, mmWD_BUF_RESOURCE_2, data);
 
         /* Program buffer base address */
@@ -1306,7 +1369,10 @@ static int gfx_v9_0_sw_init(void *handle)
         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
                 ring = &adev->gfx.gfx_ring[i];
                 ring->ring_obj = NULL;
-                sprintf(ring->name, "gfx");
+                if (!i)
+                        sprintf(ring->name, "gfx");
+                else
+                        sprintf(ring->name, "gfx_%d", i);
                 ring->use_doorbell = true;
                 ring->doorbell_index = AMDGPU_DOORBELL64_GFX_RING0 << 1;
                 r = amdgpu_ring_init(adev, ring, 1024,
@@ -1346,7 +1412,7 @@ static int gfx_v9_0_sw_init(void *handle)
                 return r;
 
         /* create MQD for all compute queues as wel as KIQ for SRIOV case */
-        r = amdgpu_gfx_compute_mqd_sw_init(adev, sizeof(struct v9_mqd));
+        r = amdgpu_gfx_compute_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
         if (r)
                 return r;
 
@@ -1398,9 +1464,19 @@ static int gfx_v9_0_sw_fini(void *handle)
         amdgpu_gfx_compute_mqd_sw_fini(adev);
         amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
         amdgpu_gfx_kiq_fini(adev);
+        amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
 
         gfx_v9_0_mec_fini(adev);
         gfx_v9_0_ngg_fini(adev);
+        amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
+                                &adev->gfx.rlc.clear_state_gpu_addr,
+                                (void **)&adev->gfx.rlc.cs_ptr);
+        if (adev->asic_type == CHIP_RAVEN) {
+                amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
+                                &adev->gfx.rlc.cp_table_gpu_addr,
+                                (void **)&adev->gfx.rlc.cp_table_ptr);
+        }
+        gfx_v9_0_free_microcode(adev);
 
         return 0;
 }
@@ -1682,10 +1758,10 @@ static int gfx_v9_0_init_rlc_save_restore_list(struct amdgpu_device *adev)
                                 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
                                 unique_indirect_regs,
                                 &unique_indirect_reg_count,
-                                sizeof(unique_indirect_regs)/sizeof(int),
+                                ARRAY_SIZE(unique_indirect_regs),
                                 indirect_start_offsets,
                                 &indirect_start_offsets_count,
-                                sizeof(indirect_start_offsets)/sizeof(int));
+                                ARRAY_SIZE(indirect_start_offsets));
 
         /* enable auto inc in case it is disabled */
         tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
@@ -1722,12 +1798,12 @@ static int gfx_v9_0_init_rlc_save_restore_list(struct amdgpu_device *adev)
         /* write the starting offsets to RLC scratch ram */
         WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
                 adev->gfx.rlc.starting_offsets_start);
-        for (i = 0; i < sizeof(indirect_start_offsets)/sizeof(int); i++)
+        for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
                         indirect_start_offsets[i]);
 
         /* load unique indirect regs*/
-        for (i = 0; i < sizeof(unique_indirect_regs)/sizeof(int); i++) {
+        for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) + i,
                         unique_indirect_regs[i] & 0x3FFFF);
                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) + i,
@@ -1740,11 +1816,7 @@ static int gfx_v9_0_init_rlc_save_restore_list(struct amdgpu_device *adev)
 
 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
 {
-        u32 tmp = 0;
-
-        tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
-        tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
-        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
+        WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
 }
 
 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
@@ -1822,16 +1894,11 @@ static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev
         uint32_t default_data = 0;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-
-        if (enable == true) {
-                data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
-                if (default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        } else {
-                data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
-                if(default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        }
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
+                             enable ? 1 : 0);
+        if (default_data != data)
+                WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
 
 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
@@ -1841,16 +1908,11 @@ static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *ad
         uint32_t default_data = 0;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-
-        if (enable == true) {
-                data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
-                if(default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        } else {
-                data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
-                if(default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        }
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
+                             enable ? 1 : 0);
+        if(default_data != data)
+                WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
 
 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
@@ -1860,16 +1922,11 @@ static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
         uint32_t default_data = 0;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-
-        if (enable == true) {
-                data &= ~RLC_PG_CNTL__CP_PG_DISABLE_MASK;
-                if(default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        } else {
-                data |= RLC_PG_CNTL__CP_PG_DISABLE_MASK;
-                if(default_data != data)
-                        WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
-        }
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             CP_PG_DISABLE,
+                             enable ? 0 : 1);
+        if(default_data != data)
+                WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
 
 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
@@ -1878,10 +1935,9 @@ static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
         uint32_t data, default_data;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-        if (enable == true)
-                data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
-        else
-                data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             GFX_POWER_GATING_ENABLE,
+                             enable ? 1 : 0);
         if(default_data != data)
                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
@@ -1892,10 +1948,9 @@ static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
         uint32_t data, default_data;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-        if (enable == true)
-                data |= RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK;
-        else
-                data &= ~RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK;
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             GFX_PIPELINE_PG_ENABLE,
+                             enable ? 1 : 0);
         if(default_data != data)
                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 
@@ -1910,10 +1965,9 @@ static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *ade
         uint32_t data, default_data;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-        if (enable == true)
-                data |= RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
-        else
-                data &= ~RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             STATIC_PER_CU_PG_ENABLE,
+                             enable ? 1 : 0);
         if(default_data != data)
                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
@@ -1924,10 +1978,9 @@ static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *ad
         uint32_t data, default_data;
 
         default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
-        if (enable == true)
-                data |= RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
-        else
-                data &= ~RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
+        data = REG_SET_FIELD(data, RLC_PG_CNTL,
+                             DYN_PER_CU_PG_ENABLE,
+                             enable ? 1 : 0);
         if(default_data != data)
                 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
 }
@@ -1967,13 +2020,8 @@ static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
 
 void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
 {
-        u32 tmp = RREG32_SOC15(GC, 0, mmRLC_CNTL);
-
-        tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
-        WREG32_SOC15(GC, 0, mmRLC_CNTL, tmp);
-
+        WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
         gfx_v9_0_enable_gui_idle_interrupt(adev, false);
-
         gfx_v9_0_wait_for_rlc_serdes(adev);
 }
 
@@ -2045,8 +2093,10 @@ static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
 {
         int r;
 
-        if (amdgpu_sriov_vf(adev))
+        if (amdgpu_sriov_vf(adev)) {
+                gfx_v9_0_init_csb(adev);
                 return 0;
+        }
 
         gfx_v9_0_rlc_stop(adev);
 
@@ -2463,6 +2513,13 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
         mqd->compute_misc_reserved = 0x00000003;
 
+        mqd->dynamic_cu_mask_addr_lo =
+                lower_32_bits(ring->mqd_gpu_addr
+                              + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
+        mqd->dynamic_cu_mask_addr_hi =
+                upper_32_bits(ring->mqd_gpu_addr
+                              + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
+
         eop_base_addr = ring->eop_gpu_addr >> 8;
         mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
@@ -2486,10 +2543,10 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
                                     DOORBELL_SOURCE, 0);
                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
                                     DOORBELL_HIT, 0);
-        }
-        else
+        } else {
                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
                                          DOORBELL_EN, 0);
+        }
 
         mqd->cp_hqd_pq_doorbell_control = tmp;
 
@@ -2692,10 +2749,10 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
 
         gfx_v9_0_kiq_setting(ring);
 
-        if (adev->gfx.in_reset) { /* for GPU_RESET case */
+        if (adev->in_sriov_reset) { /* for GPU_RESET case */
                 /* reset MQD to a clean status */
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
-                        memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
+                        memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
 
                 /* reset ring buffer */
                 ring->wptr = 0;
@@ -2707,7 +2764,9 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
                 soc15_grbm_select(adev, 0, 0, 0, 0);
                 mutex_unlock(&adev->srbm_mutex);
         } else {
-                memset((void *)mqd, 0, sizeof(*mqd));
+                memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
+                ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
+                ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
                 mutex_lock(&adev->srbm_mutex);
                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
                 gfx_v9_0_mqd_init(ring);
@@ -2716,7 +2775,7 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
                 mutex_unlock(&adev->srbm_mutex);
 
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
-                        memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
+                        memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
         }
 
         return 0;
@@ -2728,8 +2787,10 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
         struct v9_mqd *mqd = ring->mqd_ptr;
         int mqd_idx = ring - &adev->gfx.compute_ring[0];
 
-        if (!adev->gfx.in_reset && !adev->gfx.in_suspend) {
-                memset((void *)mqd, 0, sizeof(*mqd));
+        if (!adev->in_sriov_reset && !adev->gfx.in_suspend) {
+                memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
+                ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
+                ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
                 mutex_lock(&adev->srbm_mutex);
                 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
                 gfx_v9_0_mqd_init(ring);
@@ -2737,11 +2798,11 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
                 mutex_unlock(&adev->srbm_mutex);
 
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
-                        memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
-        } else if (adev->gfx.in_reset) { /* for GPU_RESET case */
+                        memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
+        } else if (adev->in_sriov_reset) { /* for GPU_RESET case */
                 /* reset MQD to a clean status */
                 if (adev->gfx.mec.mqd_backup[mqd_idx])
-                        memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
+                        memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
 
                 /* reset ring buffer */
                 ring->wptr = 0;
@@ -2882,12 +2943,70 @@ static int gfx_v9_0_hw_init(void *handle)
         return r;
 }
 
+static int gfx_v9_0_kcq_disable(struct amdgpu_ring *kiq_ring,struct amdgpu_ring *ring)
+{
+        struct amdgpu_device *adev = kiq_ring->adev;
+        uint32_t scratch, tmp = 0;
+        int r, i;
+
+        r = amdgpu_gfx_scratch_get(adev, &scratch);
+        if (r) {
+                DRM_ERROR("Failed to get scratch reg (%d).\n", r);
+                return r;
+        }
+        WREG32(scratch, 0xCAFEDEAD);
+
+        r = amdgpu_ring_alloc(kiq_ring, 10);
+        if (r) {
+                DRM_ERROR("Failed to lock KIQ (%d).\n", r);
+                amdgpu_gfx_scratch_free(adev, scratch);
+                return r;
+        }
+
+        /* unmap queues */
+        amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
+        amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
+                                                PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */
+                                                PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
+                                                PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) |
+                                                PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
+        amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
+        amdgpu_ring_write(kiq_ring, 0);
+        amdgpu_ring_write(kiq_ring, 0);
+        amdgpu_ring_write(kiq_ring, 0);
+        /* write to scratch for completion */
+        amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
+        amdgpu_ring_write(kiq_ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
+        amdgpu_ring_write(kiq_ring, 0xDEADBEEF);
+        amdgpu_ring_commit(kiq_ring);
+
+        for (i = 0; i < adev->usec_timeout; i++) {
+                tmp = RREG32(scratch);
+                if (tmp == 0xDEADBEEF)
+                        break;
+                DRM_UDELAY(1);
+        }
+        if (i >= adev->usec_timeout) {
+                DRM_ERROR("KCQ disabled failed (scratch(0x%04X)=0x%08X)\n", scratch, tmp);
+                r = -EINVAL;
+        }
+        amdgpu_gfx_scratch_free(adev, scratch);
+        return r;
+}
+
+
 static int gfx_v9_0_hw_fini(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int i;
 
         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+
+        /* disable KCQ to avoid CPC touch memory not valid anymore */
+        for (i = 0; i < adev->gfx.num_compute_rings; i++)
+                gfx_v9_0_kcq_disable(&adev->gfx.kiq.ring, &adev->gfx.compute_ring[i]);
+
         if (amdgpu_sriov_vf(adev)) {
                 pr_debug("For SRIOV client, shouldn't do anything.\n");
                 return 0;
@@ -2930,15 +3049,10 @@ static bool gfx_v9_0_is_idle(void *handle)
 static int gfx_v9_0_wait_for_idle(void *handle)
 {
         unsigned i;
-        u32 tmp;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         for (i = 0; i < adev->usec_timeout; i++) {
-                /* read MC_STATUS */
-                tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS) &
-                        GRBM_STATUS__GUI_ACTIVE_MASK;
-
-                if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
+                if (gfx_v9_0_is_idle(handle))
                         return 0;
                 udelay(1);
         }
@@ -3497,9 +3611,11 @@ static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
         u32 ref_and_mask, reg_mem_engine;
-        struct nbio_hdp_flush_reg *nbio_hf_reg;
+        const struct nbio_hdp_flush_reg *nbio_hf_reg;
 
-        if (ring->adev->asic_type == CHIP_VEGA10)
+        if (ring->adev->flags & AMD_IS_APU)
+                nbio_hf_reg = &nbio_v7_0_hdp_flush_reg;
+        else
                 nbio_hf_reg = &nbio_v6_1_hdp_flush_reg;
 
         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
@@ -3528,7 +3644,7 @@ static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 static void gfx_v9_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
         gfx_v9_0_write_data_to_reg(ring, 0, true,
-                                   SOC15_REG_OFFSET(HDP, 0, mmHDP_DEBUG0), 1);
+                                   SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1);
 }
 
 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
@@ -3718,7 +3834,7 @@ static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
 
 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
 {
-        static struct v9_ce_ib_state ce_payload = {0};
+        struct v9_ce_ib_state ce_payload = {0};
         uint64_t csa_addr;
         int cnt;
 
@@ -3737,7 +3853,7 @@ static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
 
 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
 {
-        static struct v9_de_ib_state de_payload = {0};
+        struct v9_de_ib_state de_payload = {0};
         uint64_t csa_addr, gds_addr;
         int cnt;
 
@@ -3757,6 +3873,12 @@ static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
         amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
 }
 
+static void gfx_v9_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start)
+{
+        amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
+        amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */
+}
+
 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
 {
         uint32_t dw2 = 0;
@@ -3764,6 +3886,8 @@ static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
         if (amdgpu_sriov_vf(ring->adev))
                 gfx_v9_0_ring_emit_ce_meta(ring);
 
+        gfx_v9_0_ring_emit_tmz(ring, true);
+
         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
                 /* set load_global_config & load_global_uconfig */
@@ -3814,12 +3938,6 @@ static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigne
                 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
 }
 
-static void gfx_v9_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start)
-{
-        amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
-        amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */
-}
-
 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
 {
         struct amdgpu_device *adev = ring->adev;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
index 6c8040e616c4..c17996e18086 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
@@ -319,6 +319,12 @@ void gfxhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev,
                         WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
                         EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+        if (!value) {
+                tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+                                CRASH_ON_NO_RETRY_FAULT, 1);
+                tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+                                CRASH_ON_RETRY_FAULT, 1);
+    }
         WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index 5be9c83dfcf7..f4603a7c8ef3 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -831,7 +831,7 @@ static int gmc_v6_0_sw_init(void *handle)
         if (r)
                 return r;
 
-        amdgpu_vm_adjust_size(adev, 64, 4);
+        amdgpu_vm_adjust_size(adev, 64, 9);
         adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
 
         adev->mc.mc_mask = 0xffffffffffULL;
@@ -901,6 +901,8 @@ static int gmc_v6_0_sw_fini(void *handle)
         gmc_v6_0_gart_fini(adev);
         amdgpu_gem_force_release(adev);
         amdgpu_bo_fini(adev);
+        release_firmware(adev->mc.fw);
+        adev->mc.fw = NULL;
 
         return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index eace9e7182c8..b0528ca9207b 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -970,7 +970,7 @@ static int gmc_v7_0_sw_init(void *handle)
          * Currently set to 4GB ((1 << 20) 4k pages).
          * Max GPUVM size for cayman and SI is 40 bits.
          */
-        amdgpu_vm_adjust_size(adev, 64, 4);
+        amdgpu_vm_adjust_size(adev, 64, 9);
         adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
 
         /* Set the internal MC address mask
@@ -1050,6 +1050,8 @@ static int gmc_v7_0_sw_fini(void *handle)
         gmc_v7_0_gart_fini(adev);
         amdgpu_gem_force_release(adev);
         amdgpu_bo_fini(adev);
+        release_firmware(adev->mc.fw);
+        adev->mc.fw = NULL;
 
         return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 3b3326daf32b..f368cfe2f585 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -1067,7 +1067,7 @@ static int gmc_v8_0_sw_init(void *handle)
          * Currently set to 4GB ((1 << 20) 4k pages).
          * Max GPUVM size for cayman and SI is 40 bits.
          */
-        amdgpu_vm_adjust_size(adev, 64, 4);
+        amdgpu_vm_adjust_size(adev, 64, 9);
         adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
 
         /* Set the internal MC address mask
@@ -1147,6 +1147,8 @@ static int gmc_v8_0_sw_fini(void *handle)
         gmc_v8_0_gart_fini(adev);
         amdgpu_gem_force_release(adev);
         amdgpu_bo_fini(adev);
+        release_firmware(adev->mc.fw);
+        adev->mc.fw = NULL;
 
         return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index d04d0b123212..c8f1aebeac7a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -32,6 +32,8 @@
 #include "vega10/DC/dce_12_0_offset.h"
 #include "vega10/DC/dce_12_0_sh_mask.h"
 #include "vega10/vega10_enum.h"
+#include "vega10/MMHUB/mmhub_1_0_offset.h"
+#include "vega10/ATHUB/athub_1_0_offset.h"
 
 #include "soc15_common.h"
 
@@ -71,13 +73,25 @@ static const u32 golden_settings_vega10_hdp[] =
         0xf6e, 0x0fffffff, 0x00000000,
 };
 
+static const u32 golden_settings_mmhub_1_0_0[] =
+{
+        SOC15_REG_OFFSET(MMHUB, 0, mmDAGB1_WRCLI2), 0x00000007, 0xfe5fe0fa,
+        SOC15_REG_OFFSET(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0), 0x00000030, 0x55555565
+};
+
+static const u32 golden_settings_athub_1_0_0[] =
+{
+        SOC15_REG_OFFSET(ATHUB, 0, mmRPB_ARB_CNTL), 0x0000ff00, 0x00000800,
+        SOC15_REG_OFFSET(ATHUB, 0, mmRPB_ARB_CNTL2), 0x00ff00ff, 0x00080008
+};
+
 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
                                         struct amdgpu_irq_src *src,
                                         unsigned type,
                                         enum amdgpu_interrupt_state state)
 {
         struct amdgpu_vmhub *hub;
-        u32 tmp, reg, bits, i;
+        u32 tmp, reg, bits, i, j;
 
         bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
                 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
@@ -89,43 +103,26 @@ static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
 
         switch (state) {
         case AMDGPU_IRQ_STATE_DISABLE:
-                /* MM HUB */
-                hub = &adev->vmhub[AMDGPU_MMHUB];
-                for (i = 0; i< 16; i++) {
-                        reg = hub->vm_context0_cntl + i;
-                        tmp = RREG32(reg);
-                        tmp &= ~bits;
-                        WREG32(reg, tmp);
-                }
-
-                /* GFX HUB */
-                hub = &adev->vmhub[AMDGPU_GFXHUB];
-                for (i = 0; i < 16; i++) {
-                        reg = hub->vm_context0_cntl + i;
-                        tmp = RREG32(reg);
-                        tmp &= ~bits;
-                        WREG32(reg, tmp);
+                for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) {
+                        hub = &adev->vmhub[j];
+                        for (i = 0; i < 16; i++) {
+                                reg = hub->vm_context0_cntl + i;
+                                tmp = RREG32(reg);
+                                tmp &= ~bits;
+                                WREG32(reg, tmp);
+                        }
                 }
                 break;
         case AMDGPU_IRQ_STATE_ENABLE:
-                /* MM HUB */
-                hub = &adev->vmhub[AMDGPU_MMHUB];
-                for (i = 0; i< 16; i++) {
-                        reg = hub->vm_context0_cntl + i;
-                        tmp = RREG32(reg);
-                        tmp |= bits;
-                        WREG32(reg, tmp);
-                }
-
-                /* GFX HUB */
-                hub = &adev->vmhub[AMDGPU_GFXHUB];
-                for (i = 0; i < 16; i++) {
-                        reg = hub->vm_context0_cntl + i;
-                        tmp = RREG32(reg);
-                        tmp |= bits;
-                        WREG32(reg, tmp);
+                for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) {
+                        hub = &adev->vmhub[j];
+                        for (i = 0; i < 16; i++) {
+                                reg = hub->vm_context0_cntl + i;
+                                tmp = RREG32(reg);
+                                tmp |= bits;
+                                WREG32(reg, tmp);
+                        }
                 }
-                break;
         default:
                 break;
         }
@@ -395,7 +392,16 @@ static int gmc_v9_0_early_init(void *handle)
 static int gmc_v9_0_late_init(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-        unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 3, 3 };
+        /*
+         * The latest engine allocation on gfx9 is:
+         * Engine 0, 1: idle
+         * Engine 2, 3: firmware
+         * Engine 4~13: amdgpu ring, subject to change when ring number changes
+         * Engine 14~15: idle
+         * Engine 16: kfd tlb invalidation
+         * Engine 17: Gart flushes
+         */
+        unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 };
         unsigned i;
 
         for(i = 0; i < adev->num_rings; ++i) {
@@ -408,9 +414,9 @@ static int gmc_v9_0_late_init(void *handle)
                          ring->funcs->vmhub);
         }
 
-        /* Engine 17 is used for GART flushes */
+        /* Engine 16 is used for KFD and 17 for GART flushes */
         for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i)
-                BUG_ON(vm_inv_eng[i] > 17);
+                BUG_ON(vm_inv_eng[i] > 16);
 
         return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
 }
@@ -682,8 +688,17 @@ static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
 {
         switch (adev->asic_type) {
         case CHIP_VEGA10:
+                amdgpu_program_register_sequence(adev,
+                                                golden_settings_mmhub_1_0_0,
+                                                (const u32)ARRAY_SIZE(golden_settings_mmhub_1_0_0));
+                amdgpu_program_register_sequence(adev,
+                                                golden_settings_athub_1_0_0,
+                                                (const u32)ARRAY_SIZE(golden_settings_athub_1_0_0));
                 break;
         case CHIP_RAVEN:
+                amdgpu_program_register_sequence(adev,
+                                                golden_settings_athub_1_0_0,
+                                                (const u32)ARRAY_SIZE(golden_settings_athub_1_0_0));
                 break;
         default:
                 break;
@@ -713,12 +728,6 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
         if (r)
                 return r;
 
-        /* After HDP is initialized, flush HDP.*/
-        if (adev->flags & AMD_IS_APU)
-                nbio_v7_0_hdp_flush(adev);
-        else
-                nbio_v6_1_hdp_flush(adev);
-
         switch (adev->asic_type) {
         case CHIP_RAVEN:
                 mmhub_v1_0_initialize_power_gating(adev);
@@ -736,13 +745,16 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
         if (r)
                 return r;
 
-        tmp = RREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL);
-        tmp |= HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE_MASK;
-        WREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL, tmp);
+        WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1);
 
         tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
         WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
 
+        /* After HDP is initialized, flush HDP.*/
+        if (adev->flags & AMD_IS_APU)
+                nbio_v7_0_hdp_flush(adev);
+        else
+                nbio_v6_1_hdp_flush(adev);
 
         if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
                 value = false;
@@ -751,7 +763,6 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
 
         gfxhub_v1_0_set_fault_enable_default(adev, value);
         mmhub_v1_0_set_fault_enable_default(adev, value);
-
         gmc_v9_0_gart_flush_gpu_tlb(adev, 0);
 
         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
@@ -770,17 +781,11 @@ static int gmc_v9_0_hw_init(void *handle)
         gmc_v9_0_init_golden_registers(adev);
 
         if (adev->mode_info.num_crtc) {
-                u32 tmp;
-
                 /* Lockout access through VGA aperture*/
-                tmp = RREG32_SOC15(DCE, 0, mmVGA_HDP_CONTROL);
-                tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
-                WREG32_SOC15(DCE, 0, mmVGA_HDP_CONTROL, tmp);
+                WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
 
                 /* disable VGA render */
-                tmp = RREG32_SOC15(DCE, 0, mmVGA_RENDER_CONTROL);
-                tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
-                WREG32_SOC15(DCE, 0, mmVGA_RENDER_CONTROL, tmp);
+                WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
         }
 
         r = gmc_v9_0_gart_enable(adev);
@@ -822,9 +827,7 @@ static int gmc_v9_0_suspend(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        gmc_v9_0_hw_fini(adev);
-
-        return 0;
+        return gmc_v9_0_hw_fini(adev);
 }
 
 static int gmc_v9_0_resume(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
index 7a0ea27ac429..bd592cb39f37 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
@@ -208,6 +208,34 @@ static u32 iceland_ih_get_wptr(struct amdgpu_device *adev)
 }
 
 /**
+ * iceland_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool iceland_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        u32 ring_index = adev->irq.ih.rptr >> 2;
+        u16 pasid;
+
+        switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
+        case 146:
+        case 147:
+                pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
+                if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
+                        return true;
+                break;
+        default:
+                /* Not a VM fault */
+                return true;
+        }
+
+        adev->irq.ih.rptr += 16;
+        return false;
+}
+
+/**
  * iceland_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -412,6 +440,7 @@ static const struct amd_ip_funcs iceland_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs iceland_ih_funcs = {
         .get_wptr = iceland_ih_get_wptr,
+        .prescreen_iv = iceland_ih_prescreen_iv,
         .decode_iv = iceland_ih_decode_iv,
         .set_rptr = iceland_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index 3bbf2ccfca89..f33d1ffdb20b 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -42,7 +42,6 @@
 #define KV_MINIMUM_ENGINE_CLOCK         800
 #define SMC_RAM_END                     0x40000
 
-static void kv_dpm_set_dpm_funcs(struct amdgpu_device *adev);
 static void kv_dpm_set_irq_funcs(struct amdgpu_device *adev);
 static int kv_enable_nb_dpm(struct amdgpu_device *adev,
                             bool enable);
@@ -64,7 +63,7 @@ static int kv_set_thermal_temperature_range(struct amdgpu_device *adev,
                                             int min_temp, int max_temp);
 static int kv_init_fps_limits(struct amdgpu_device *adev);
 
-static void kv_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate);
+static void kv_dpm_powergate_uvd(void *handle, bool gate);
 static void kv_dpm_powergate_vce(struct amdgpu_device *adev, bool gate);
 static void kv_dpm_powergate_samu(struct amdgpu_device *adev, bool gate);
 static void kv_dpm_powergate_acp(struct amdgpu_device *adev, bool gate);
@@ -1245,8 +1244,9 @@ static void kv_update_requested_ps(struct amdgpu_device *adev,
         adev->pm.dpm.requested_ps = &pi->requested_rps;
 }
 
-static void kv_dpm_enable_bapm(struct amdgpu_device *adev, bool enable)
+static void kv_dpm_enable_bapm(void *handle, bool enable)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         int ret;
 
@@ -1672,8 +1672,9 @@ static int kv_update_acp_dpm(struct amdgpu_device *adev, bool gate)
         return kv_enable_acp_dpm(adev, !gate);
 }
 
-static void kv_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
+static void kv_dpm_powergate_uvd(void *handle, bool gate)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         int ret;
 
@@ -1868,10 +1869,11 @@ static int kv_enable_nb_dpm(struct amdgpu_device *adev,
         return ret;
 }
 
-static int kv_dpm_force_performance_level(struct amdgpu_device *adev,
+static int kv_dpm_force_performance_level(void *handle,
                                           enum amd_dpm_forced_level level)
 {
         int ret;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (level == AMD_DPM_FORCED_LEVEL_HIGH) {
                 ret = kv_force_dpm_highest(adev);
@@ -1892,8 +1894,9 @@ static int kv_dpm_force_performance_level(struct amdgpu_device *adev,
         return 0;
 }
 
-static int kv_dpm_pre_set_power_state(struct amdgpu_device *adev)
+static int kv_dpm_pre_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         struct amdgpu_ps requested_ps = *adev->pm.dpm.requested_ps;
         struct amdgpu_ps *new_ps = &requested_ps;
@@ -1907,8 +1910,9 @@ static int kv_dpm_pre_set_power_state(struct amdgpu_device *adev)
         return 0;
 }
 
-static int kv_dpm_set_power_state(struct amdgpu_device *adev)
+static int kv_dpm_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         struct amdgpu_ps *new_ps = &pi->requested_rps;
         struct amdgpu_ps *old_ps = &pi->current_rps;
@@ -1981,8 +1985,9 @@ static int kv_dpm_set_power_state(struct amdgpu_device *adev)
         return 0;
 }
 
-static void kv_dpm_post_set_power_state(struct amdgpu_device *adev)
+static void kv_dpm_post_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         struct amdgpu_ps *new_ps = &pi->requested_rps;
 
@@ -2848,9 +2853,10 @@ static int kv_dpm_init(struct amdgpu_device *adev)
 }
 
 static void
-kv_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
+kv_dpm_debugfs_print_current_performance_level(void *handle,
                                                struct seq_file *m)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         u32 current_index =
                 (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX) &
@@ -2875,11 +2881,12 @@ kv_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
 }
 
 static void
-kv_dpm_print_power_state(struct amdgpu_device *adev,
-                         struct amdgpu_ps *rps)
+kv_dpm_print_power_state(void *handle, void *request_ps)
 {
         int i;
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)request_ps;
         struct kv_ps *ps = kv_get_ps(rps);
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         amdgpu_dpm_print_class_info(rps->class, rps->class2);
         amdgpu_dpm_print_cap_info(rps->caps);
@@ -2905,13 +2912,14 @@ static void kv_dpm_fini(struct amdgpu_device *adev)
         amdgpu_free_extended_power_table(adev);
 }
 
-static void kv_dpm_display_configuration_changed(struct amdgpu_device *adev)
+static void kv_dpm_display_configuration_changed(void *handle)
 {
 
 }
 
-static u32 kv_dpm_get_sclk(struct amdgpu_device *adev, bool low)
+static u32 kv_dpm_get_sclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         struct kv_ps *requested_state = kv_get_ps(&pi->requested_rps);
 
@@ -2921,18 +2929,20 @@ static u32 kv_dpm_get_sclk(struct amdgpu_device *adev, bool low)
                 return requested_state->levels[requested_state->num_levels - 1].sclk;
 }
 
-static u32 kv_dpm_get_mclk(struct amdgpu_device *adev, bool low)
+static u32 kv_dpm_get_mclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
 
         return pi->sys_info.bootup_uma_clk;
 }
 
 /* get temperature in millidegrees */
-static int kv_dpm_get_temp(struct amdgpu_device *adev)
+static int kv_dpm_get_temp(void *handle)
 {
         u32 temp;
         int actual_temp = 0;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         temp = RREG32_SMC(0xC0300E0C);
 
@@ -2950,7 +2960,6 @@ static int kv_dpm_early_init(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        kv_dpm_set_dpm_funcs(adev);
         kv_dpm_set_irq_funcs(adev);
 
         return 0;
@@ -2960,16 +2969,10 @@ static int kv_dpm_late_init(void *handle)
 {
         /* powerdown unused blocks for now */
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-        int ret;
 
         if (!amdgpu_dpm)
                 return 0;
 
-        /* init the sysfs and debugfs files late */
-        ret = amdgpu_pm_sysfs_init(adev);
-        if (ret)
-                return ret;
-
         kv_dpm_powergate_acp(adev, true);
         kv_dpm_powergate_samu(adev, true);
 
@@ -3031,7 +3034,6 @@ static int kv_dpm_sw_fini(void *handle)
         flush_work(&adev->pm.dpm.thermal.work);
 
         mutex_lock(&adev->pm.mutex);
-        amdgpu_pm_sysfs_fini(adev);
         kv_dpm_fini(adev);
         mutex_unlock(&adev->pm.mutex);
 
@@ -3222,14 +3224,17 @@ static inline bool kv_are_power_levels_equal(const struct kv_pl *kv_cpl1,
                   (kv_cpl1->force_nbp_state == kv_cpl2->force_nbp_state));
 }
 
-static int kv_check_state_equal(struct amdgpu_device *adev,
-                                struct amdgpu_ps *cps,
-                                struct amdgpu_ps *rps,
+static int kv_check_state_equal(void *handle,
+                                void *current_ps,
+                                void *request_ps,
                                 bool *equal)
 {
         struct kv_ps *kv_cps;
         struct kv_ps *kv_rps;
         int i;
+        struct amdgpu_ps *cps = (struct amdgpu_ps *)current_ps;
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)request_ps;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (adev == NULL || cps == NULL || rps == NULL || equal == NULL)
                 return -EINVAL;
@@ -3262,9 +3267,10 @@ static int kv_check_state_equal(struct amdgpu_device *adev,
         return 0;
 }
 
-static int kv_dpm_read_sensor(struct amdgpu_device *adev, int idx,
+static int kv_dpm_read_sensor(void *handle, int idx,
                               void *value, int *size)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct kv_power_info *pi = kv_get_pi(adev);
         uint32_t sclk;
         u32 pl_index =
@@ -3312,7 +3318,7 @@ const struct amd_ip_funcs kv_dpm_ip_funcs = {
         .set_powergating_state = kv_dpm_set_powergating_state,
 };
 
-static const struct amdgpu_dpm_funcs kv_dpm_funcs = {
+const struct amd_pm_funcs kv_dpm_funcs = {
         .get_temperature = &kv_dpm_get_temp,
         .pre_set_power_state = &kv_dpm_pre_set_power_state,
         .set_power_state = &kv_dpm_set_power_state,
@@ -3330,12 +3336,6 @@ static const struct amdgpu_dpm_funcs kv_dpm_funcs = {
         .read_sensor = &kv_dpm_read_sensor,
 };
 
-static void kv_dpm_set_dpm_funcs(struct amdgpu_device *adev)
-{
-        if (adev->pm.funcs == NULL)
-                adev->pm.funcs = &kv_dpm_funcs;
-}
-
 static const struct amdgpu_irq_src_funcs kv_dpm_irq_funcs = {
         .set = kv_dpm_set_interrupt_state,
         .process = kv_dpm_process_interrupt,
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
index 74cb647da30e..cc21c4bdec27 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
@@ -273,7 +273,7 @@ static const struct pctl_data pctl0_data[] = {
         {0x135, 0x12a810},
         {0x149, 0x7a82c}
 };
-#define PCTL0_DATA_LEN (sizeof(pctl0_data)/sizeof(pctl0_data[0]))
+#define PCTL0_DATA_LEN (ARRAY_SIZE(pctl0_data))
 
 #define PCTL0_RENG_EXEC_END_PTR 0x151
 #define PCTL0_STCTRL_REG_SAVE_RANGE0_BASE  0xa640
@@ -309,7 +309,7 @@ static const struct pctl_data pctl1_data[] = {
         {0x1f0, 0x5000a7f6},
         {0x1f1, 0x5000a7e4}
 };
-#define PCTL1_DATA_LEN (sizeof(pctl1_data)/sizeof(pctl1_data[0]))
+#define PCTL1_DATA_LEN (ARRAY_SIZE(pctl1_data))
 
 #define PCTL1_RENG_EXEC_END_PTR 0x1f1
 #define PCTL1_STCTRL_REG_SAVE_RANGE0_BASE  0xa000
@@ -561,6 +561,13 @@ void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, bool value)
                         WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
                         EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+        if (!value) {
+                tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+                                CRASH_ON_NO_RETRY_FAULT, 1);
+                tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
+                                CRASH_ON_RETRY_FAULT, 1);
+    }
+
         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index 2812d88a8bdd..b4906d2f30d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -183,6 +183,12 @@ static int xgpu_ai_send_access_requests(struct amdgpu_device *adev,
                         pr_err("Doesn't get READY_TO_ACCESS_GPU from pf, give up\n");
                         return r;
                 }
+                /* Retrieve checksum from mailbox2 */
+                if (req == IDH_REQ_GPU_INIT_ACCESS) {
+                        adev->virt.fw_reserve.checksum_key =
+                                RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
+                                        mmBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2));
+                }
         }
 
         return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
index 1e91b9a1c591..67e78576a9eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
@@ -24,7 +24,7 @@
 #ifndef __MXGPU_AI_H__
 #define __MXGPU_AI_H__
 
-#define AI_MAILBOX_TIMEDOUT     5000
+#define AI_MAILBOX_TIMEDOUT     12000
 
 enum idh_request {
         IDH_REQ_GPU_INIT_ACCESS = 1,
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
index c791d73d2d54..f13dc6cc158f 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
@@ -23,7 +23,7 @@
 #ifndef __MXGPU_VI_H__
 #define __MXGPU_VI_H__
 
-#define VI_MAILBOX_TIMEDOUT     5000
+#define VI_MAILBOX_TIMEDOUT     12000
 #define VI_MAILBOX_RESET_TIME   12
 
 /* VI mailbox messages request */
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
index 045988b18bc3..904a1bab9b9f 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
@@ -215,31 +215,27 @@ void nbio_v6_1_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
                 *flags |= AMD_CG_SUPPORT_BIF_LS;
 }
 
-struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg;
-struct nbio_pcie_index_data nbio_v6_1_pcie_index_data;
+const struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg = {
+        .hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_REQ),
+        .hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_DONE),
+        .ref_and_mask_cp0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK,
+        .ref_and_mask_cp1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK,
+        .ref_and_mask_cp2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK,
+        .ref_and_mask_cp3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK,
+        .ref_and_mask_cp4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK,
+        .ref_and_mask_cp5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK,
+        .ref_and_mask_cp6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK,
+        .ref_and_mask_cp7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK,
+        .ref_and_mask_cp8 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK,
+        .ref_and_mask_cp9 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK,
+        .ref_and_mask_sdma0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK,
+        .ref_and_mask_sdma1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK
+};
 
-int nbio_v6_1_init(struct amdgpu_device *adev)
-{
-        nbio_v6_1_hdp_flush_reg.hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_REQ);
-        nbio_v6_1_hdp_flush_reg.hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_DONE);
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_cp0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK;
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_cp1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK;
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_cp2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK;
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_cp3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK;
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_cp4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK;
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_cp5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK;
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_cp6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK;
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_cp7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK;
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_cp8 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK;
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_cp9 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK;
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_sdma0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK;
-        nbio_v6_1_hdp_flush_reg.ref_and_mask_sdma1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK;
-
-        nbio_v6_1_pcie_index_data.index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX);
-        nbio_v6_1_pcie_index_data.data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA);
-
-        return 0;
-}
+const struct nbio_pcie_index_data nbio_v6_1_pcie_index_data = {
+        .index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX),
+        .data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA),
+};
 
 void nbio_v6_1_detect_hw_virt(struct amdgpu_device *adev)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
index 686e4b4d296a..14ca8d45a46c 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
@@ -26,8 +26,8 @@
 
 #include "soc15_common.h"
 
-extern struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg;
-extern struct nbio_pcie_index_data nbio_v6_1_pcie_index_data;
+extern const struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg;
+extern const struct nbio_pcie_index_data nbio_v6_1_pcie_index_data;
 int nbio_v6_1_init(struct amdgpu_device *adev);
 u32 nbio_v6_1_get_atombios_scratch_regs(struct amdgpu_device *adev,
                                         uint32_t idx);
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
index 11b70d601922..f802b973410a 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
@@ -185,28 +185,24 @@ void nbio_v7_0_ih_control(struct amdgpu_device *adev)
         WREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL, interrupt_cntl);
 }
 
-struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg;
-struct nbio_pcie_index_data nbio_v7_0_pcie_index_data;
+const struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg = {
+        .hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_REQ),
+        .hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_DONE),
+        .ref_and_mask_cp0 = GPU_HDP_FLUSH_DONE__CP0_MASK,
+        .ref_and_mask_cp1 = GPU_HDP_FLUSH_DONE__CP1_MASK,
+        .ref_and_mask_cp2 = GPU_HDP_FLUSH_DONE__CP2_MASK,
+        .ref_and_mask_cp3 = GPU_HDP_FLUSH_DONE__CP3_MASK,
+        .ref_and_mask_cp4 = GPU_HDP_FLUSH_DONE__CP4_MASK,
+        .ref_and_mask_cp5 = GPU_HDP_FLUSH_DONE__CP5_MASK,
+        .ref_and_mask_cp6 = GPU_HDP_FLUSH_DONE__CP6_MASK,
+        .ref_and_mask_cp7 = GPU_HDP_FLUSH_DONE__CP7_MASK,
+        .ref_and_mask_cp8 = GPU_HDP_FLUSH_DONE__CP8_MASK,
+        .ref_and_mask_cp9 = GPU_HDP_FLUSH_DONE__CP9_MASK,
+        .ref_and_mask_sdma0 = GPU_HDP_FLUSH_DONE__SDMA0_MASK,
+        .ref_and_mask_sdma1 = GPU_HDP_FLUSH_DONE__SDMA1_MASK,
+};
 
-int nbio_v7_0_init(struct amdgpu_device *adev)
-{
-        nbio_v7_0_hdp_flush_reg.hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_REQ);
-        nbio_v7_0_hdp_flush_reg.hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_DONE);
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_cp0 = GPU_HDP_FLUSH_DONE__CP0_MASK;
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_cp1 = GPU_HDP_FLUSH_DONE__CP1_MASK;
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_cp2 = GPU_HDP_FLUSH_DONE__CP2_MASK;
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_cp3 = GPU_HDP_FLUSH_DONE__CP3_MASK;
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_cp4 = GPU_HDP_FLUSH_DONE__CP4_MASK;
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_cp5 = GPU_HDP_FLUSH_DONE__CP5_MASK;
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_cp6 = GPU_HDP_FLUSH_DONE__CP6_MASK;
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_cp7 = GPU_HDP_FLUSH_DONE__CP7_MASK;
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_cp8 = GPU_HDP_FLUSH_DONE__CP8_MASK;
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_cp9 = GPU_HDP_FLUSH_DONE__CP9_MASK;
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_sdma0 = GPU_HDP_FLUSH_DONE__SDMA0_MASK;
-        nbio_v7_0_hdp_flush_reg.ref_and_mask_sdma1 = GPU_HDP_FLUSH_DONE__SDMA1_MASK;
-
-        nbio_v7_0_pcie_index_data.index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2);
-        nbio_v7_0_pcie_index_data.data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2);
-
-        return 0;
-}
+const struct nbio_pcie_index_data nbio_v7_0_pcie_index_data = {
+        .index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2),
+        .data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2)
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
index 054ff49427e6..df8fa90f40d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
@@ -26,8 +26,8 @@
 
 #include "soc15_common.h"
 
-extern struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg;
-extern struct nbio_pcie_index_data nbio_v7_0_pcie_index_data;
+extern const struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg;
+extern const struct nbio_pcie_index_data nbio_v7_0_pcie_index_data;
 int nbio_v7_0_init(struct amdgpu_device *adev);
 u32 nbio_v7_0_get_atombios_scratch_regs(struct amdgpu_device *adev,
                                         uint32_t idx);
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
index f7cf994b1da2..4e20d91d5d50 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
@@ -35,6 +35,8 @@
 #include "raven1/GC/gc_9_1_offset.h"
 #include "raven1/SDMA0/sdma0_4_1_offset.h"
 
+MODULE_FIRMWARE("amdgpu/raven_asd.bin");
+
 static int
 psp_v10_0_get_fw_type(struct amdgpu_firmware_info *ucode, enum psp_gfx_fw_type *type)
 {
@@ -136,15 +138,13 @@ int psp_v10_0_prep_cmd_buf(struct amdgpu_firmware_info *ucode, struct psp_gfx_cm
 {
         int ret;
         uint64_t fw_mem_mc_addr = ucode->mc_addr;
-        struct  common_firmware_header *header;
 
         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
-        header = (struct common_firmware_header *)ucode->fw;
 
         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
-        cmd->cmd.cmd_load_ip_fw.fw_size = le32_to_cpu(header->ucode_size_bytes);
+        cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
 
         ret = psp_v10_0_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
         if (ret)
@@ -209,7 +209,7 @@ int psp_v10_0_ring_create(struct psp_context *psp, enum psp_ring_type ring_type)
         return ret;
 }
 
-int psp_v10_0_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
+int psp_v10_0_ring_stop(struct psp_context *psp, enum psp_ring_type ring_type)
 {
         int ret = 0;
         struct psp_ring *ring;
@@ -229,6 +229,19 @@ int psp_v10_0_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type
         ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
                            0x80000000, 0x80000000, false);
 
+        return ret;
+}
+
+int psp_v10_0_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
+{
+        int ret = 0;
+        struct psp_ring *ring = &psp->km_ring;
+        struct amdgpu_device *adev = psp->adev;
+
+        ret = psp_v10_0_ring_stop(psp, ring_type);
+        if (ret)
+                DRM_ERROR("Fail to stop psp ring\n");
+
         amdgpu_bo_free_kernel(&adev->firmware.rbuf,
                               &ring->ring_mem_mc_addr,
                               (void **)&ring->ring_mem);
@@ -244,16 +257,31 @@ int psp_v10_0_cmd_submit(struct psp_context *psp,
         unsigned int psp_write_ptr_reg = 0;
         struct psp_gfx_rb_frame * write_frame = psp->km_ring.ring_mem;
         struct psp_ring *ring = &psp->km_ring;
+        struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
+        struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
+                ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
         struct amdgpu_device *adev = psp->adev;
+        uint32_t ring_size_dw = ring->ring_size / 4;
+        uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
 
         /* KM (GPCOM) prepare write pointer */
         psp_write_ptr_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_67);
 
         /* Update KM RB frame pointer to new frame */
-        if ((psp_write_ptr_reg % ring->ring_size) == 0)
-                write_frame = ring->ring_mem;
+        if ((psp_write_ptr_reg % ring_size_dw) == 0)
+                write_frame = ring_buffer_start;
         else
-                write_frame = ring->ring_mem + (psp_write_ptr_reg / (sizeof(struct psp_gfx_rb_frame) / 4));
+                write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
+        /* Check invalid write_frame ptr address */
+        if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
+                DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
+                          ring_buffer_start, ring_buffer_end, write_frame);
+                DRM_ERROR("write_frame is pointing to address out of bounds\n");
+                return -EINVAL;
+        }
+
+        /* Initialize KM RB frame */
+        memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
 
         /* Update KM RB frame */
         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
@@ -263,8 +291,7 @@ int psp_v10_0_cmd_submit(struct psp_context *psp,
         write_frame->fence_value = index;
 
         /* Update the write Pointer in DWORDs */
-        psp_write_ptr_reg += sizeof(struct psp_gfx_rb_frame) / 4;
-        psp_write_ptr_reg = (psp_write_ptr_reg >= ring->ring_size) ? 0 : psp_write_ptr_reg;
+        psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
         WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_67, psp_write_ptr_reg);
 
         return 0;
@@ -390,3 +417,10 @@ bool psp_v10_0_compare_sram_data(struct psp_context *psp,
 
         return true;
 }
+
+
+int psp_v10_0_mode1_reset(struct psp_context *psp)
+{
+        DRM_INFO("psp mode 1 reset not supported now! \n");
+        return -EINVAL;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h
index e76cde2f01f9..451e8308303f 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h
@@ -34,6 +34,8 @@ extern int psp_v10_0_ring_init(struct psp_context *psp,
                               enum psp_ring_type ring_type);
 extern int psp_v10_0_ring_create(struct psp_context *psp,
                                  enum psp_ring_type ring_type);
+extern int psp_v10_0_ring_stop(struct psp_context *psp,
+                                  enum psp_ring_type ring_type);
 extern int psp_v10_0_ring_destroy(struct psp_context *psp,
                                   enum psp_ring_type ring_type);
 extern int psp_v10_0_cmd_submit(struct psp_context *psp,
@@ -43,4 +45,6 @@ extern int psp_v10_0_cmd_submit(struct psp_context *psp,
 extern bool psp_v10_0_compare_sram_data(struct psp_context *psp,
                                        struct amdgpu_firmware_info *ucode,
                                        enum AMDGPU_UCODE_ID ucode_type);
+
+extern int psp_v10_0_mode1_reset(struct psp_context *psp);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index 2a535a4b8d5b..c7bcfe8e286c 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -319,7 +319,7 @@ int psp_v3_1_ring_create(struct psp_context *psp, enum psp_ring_type ring_type)
         return ret;
 }
 
-int psp_v3_1_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
+int psp_v3_1_ring_stop(struct psp_context *psp, enum psp_ring_type ring_type)
 {
         int ret = 0;
         struct psp_ring *ring;
@@ -339,6 +339,19 @@ int psp_v3_1_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
         ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
                            0x80000000, 0x80000000, false);
 
+        return ret;
+}
+
+int psp_v3_1_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
+{
+        int ret = 0;
+        struct psp_ring *ring = &psp->km_ring;
+        struct amdgpu_device *adev = psp->adev;
+
+        ret = psp_v3_1_ring_stop(psp, ring_type);
+        if (ret)
+                DRM_ERROR("Fail to stop psp ring\n");
+
         amdgpu_bo_free_kernel(&adev->firmware.rbuf,
                               &ring->ring_mem_mc_addr,
                               (void **)&ring->ring_mem);
@@ -354,6 +367,9 @@ int psp_v3_1_cmd_submit(struct psp_context *psp,
         unsigned int psp_write_ptr_reg = 0;
         struct psp_gfx_rb_frame * write_frame = psp->km_ring.ring_mem;
         struct psp_ring *ring = &psp->km_ring;
+        struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
+        struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
+                ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
         struct amdgpu_device *adev = psp->adev;
         uint32_t ring_size_dw = ring->ring_size / 4;
         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
@@ -365,9 +381,16 @@ int psp_v3_1_cmd_submit(struct psp_context *psp,
         /* write_frame ptr increments by size of rb_frame in bytes */
         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
         if ((psp_write_ptr_reg % ring_size_dw) == 0)
-                write_frame = ring->ring_mem;
+                write_frame = ring_buffer_start;
         else
-                write_frame = ring->ring_mem + (psp_write_ptr_reg / rb_frame_size_dw);
+                write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
+        /* Check invalid write_frame ptr address */
+        if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
+                DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
+                          ring_buffer_start, ring_buffer_end, write_frame);
+                DRM_ERROR("write_frame is pointing to address out of bounds\n");
+                return -EINVAL;
+        }
 
         /* Initialize KM RB frame */
         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
@@ -517,3 +540,37 @@ bool psp_v3_1_smu_reload_quirk(struct psp_context *psp)
         reg = RREG32_SOC15(NBIO, 0, mmPCIE_DATA2);
         return (reg & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) ? true : false;
 }
+
+int psp_v3_1_mode1_reset(struct psp_context *psp)
+{
+        int ret;
+        uint32_t offset;
+        struct amdgpu_device *adev = psp->adev;
+
+        offset = SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64);
+
+        ret = psp_wait_for(psp, offset, 0x80000000, 0x8000FFFF, false);
+
+        if (ret) {
+                DRM_INFO("psp is not working correctly before mode1 reset!\n");
+                return -EINVAL;
+        }
+
+        /*send the mode 1 reset command*/
+        WREG32(offset, 0x70000);
+
+        mdelay(1000);
+
+        offset = SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_33);
+
+        ret = psp_wait_for(psp, offset, 0x80000000, 0x80000000, false);
+
+        if (ret) {
+                DRM_INFO("psp mode 1 reset failed!\n");
+                return -EINVAL;
+        }
+
+        DRM_INFO("psp mode1 reset succeed \n");
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h
index 9dcd0b25c4c6..b05dbada7751 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h
@@ -41,6 +41,8 @@ extern int psp_v3_1_ring_init(struct psp_context *psp,
                               enum psp_ring_type ring_type);
 extern int psp_v3_1_ring_create(struct psp_context *psp,
                                 enum psp_ring_type ring_type);
+extern int psp_v3_1_ring_stop(struct psp_context *psp,
+                                enum psp_ring_type ring_type);
 extern int psp_v3_1_ring_destroy(struct psp_context *psp,
                                 enum psp_ring_type ring_type);
 extern int psp_v3_1_cmd_submit(struct psp_context *psp,
@@ -51,4 +53,5 @@ extern bool psp_v3_1_compare_sram_data(struct psp_context *psp,
                                        struct amdgpu_firmware_info *ucode,
                                        enum AMDGPU_UCODE_ID ucode_type);
 extern bool psp_v3_1_smu_reload_quirk(struct psp_context *psp);
+extern int psp_v3_1_mode1_reset(struct psp_context *psp);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index f2d0710258cb..67f375bfe452 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -561,21 +561,11 @@ static int sdma_v2_4_start(struct amdgpu_device *adev)
 {
         int r;
 
-        if (!adev->pp_enabled) {
-                if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) {
-                        r = sdma_v2_4_load_microcode(adev);
-                        if (r)
-                                return r;
-                } else {
-                        r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-                                                        AMDGPU_UCODE_ID_SDMA0);
-                        if (r)
-                                return -EINVAL;
-                        r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-                                                        AMDGPU_UCODE_ID_SDMA1);
-                        if (r)
-                                return -EINVAL;
-                }
+
+        if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+                r = sdma_v2_4_load_microcode(adev);
+                if (r)
+                        return r;
         }
 
         /* halt the engine before programing */
@@ -1324,8 +1314,13 @@ static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev)
 }
 
 static const struct amdgpu_vm_pte_funcs sdma_v2_4_vm_pte_funcs = {
+        .copy_pte_num_dw = 7,
         .copy_pte = sdma_v2_4_vm_copy_pte,
+
         .write_pte = sdma_v2_4_vm_write_pte,
+
+        .set_max_nums_pte_pde = 0x1fffff >> 3,
+        .set_pte_pde_num_dw = 10,
         .set_pte_pde = sdma_v2_4_vm_set_pte_pde,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index b1de44f22824..6d06f8eb659f 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -379,8 +379,10 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring)
         struct amdgpu_device *adev = ring->adev;
 
         if (ring->use_doorbell) {
+                u32 *wb = (u32 *)&adev->wb.wb[ring->wptr_offs];
+
                 /* XXX check if swapping is necessary on BE */
-                adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr) << 2;
+                WRITE_ONCE(*wb, (lower_32_bits(ring->wptr) << 2));
                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr) << 2);
         } else {
                 int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
@@ -641,10 +643,11 @@ static void sdma_v3_0_enable(struct amdgpu_device *adev, bool enable)
 static int sdma_v3_0_gfx_resume(struct amdgpu_device *adev)
 {
         struct amdgpu_ring *ring;
-        u32 rb_cntl, ib_cntl;
+        u32 rb_cntl, ib_cntl, wptr_poll_cntl;
         u32 rb_bufsz;
         u32 wb_offset;
         u32 doorbell;
+        u64 wptr_gpu_addr;
         int i, j, r;
 
         for (i = 0; i < adev->sdma.num_instances; i++) {
@@ -707,6 +710,20 @@ static int sdma_v3_0_gfx_resume(struct amdgpu_device *adev)
                 }
                 WREG32(mmSDMA0_GFX_DOORBELL + sdma_offsets[i], doorbell);
 
+                /* setup the wptr shadow polling */
+                wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+
+                WREG32(mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO + sdma_offsets[i],
+                       lower_32_bits(wptr_gpu_addr));
+                WREG32(mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI + sdma_offsets[i],
+                       upper_32_bits(wptr_gpu_addr));
+                wptr_poll_cntl = RREG32(mmSDMA0_GFX_RB_WPTR_POLL_CNTL + sdma_offsets[i]);
+                if (amdgpu_sriov_vf(adev))
+                        wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 1);
+                else
+                        wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 0);
+                WREG32(mmSDMA0_GFX_RB_WPTR_POLL_CNTL + sdma_offsets[i], wptr_poll_cntl);
+
                 /* enable DMA RB */
                 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1);
                 WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);
@@ -802,23 +819,12 @@ static int sdma_v3_0_load_microcode(struct amdgpu_device *adev)
  */
 static int sdma_v3_0_start(struct amdgpu_device *adev)
 {
-        int r, i;
+        int r;
 
-        if (!adev->pp_enabled) {
-                if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) {
-                        r = sdma_v3_0_load_microcode(adev);
-                        if (r)
-                                return r;
-                } else {
-                        for (i = 0; i < adev->sdma.num_instances; i++) {
-                                r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-                                                                                 (i == 0) ?
-                                                                                 AMDGPU_UCODE_ID_SDMA0 :
-                                                                                 AMDGPU_UCODE_ID_SDMA1);
-                                if (r)
-                                        return -EINVAL;
-                        }
-                }
+        if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+                r = sdma_v3_0_load_microcode(adev);
+                if (r)
+                        return r;
         }
 
         /* disable sdma engine before programing it */
@@ -1713,11 +1719,11 @@ static void sdma_v3_0_emit_fill_buffer(struct amdgpu_ib *ib,
 }
 
 static const struct amdgpu_buffer_funcs sdma_v3_0_buffer_funcs = {
-        .copy_max_bytes = 0x1fffff,
+        .copy_max_bytes = 0x3fffe0, /* not 0x3fffff due to HW limitation */
         .copy_num_dw = 7,
         .emit_copy_buffer = sdma_v3_0_emit_copy_buffer,
 
-        .fill_max_bytes = 0x1fffff,
+        .fill_max_bytes = 0x3fffe0, /* not 0x3fffff due to HW limitation */
         .fill_num_dw = 5,
         .emit_fill_buffer = sdma_v3_0_emit_fill_buffer,
 };
@@ -1731,8 +1737,14 @@ static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev)
 }
 
 static const struct amdgpu_vm_pte_funcs sdma_v3_0_vm_pte_funcs = {
+        .copy_pte_num_dw = 7,
         .copy_pte = sdma_v3_0_vm_copy_pte,
+
         .write_pte = sdma_v3_0_vm_write_pte,
+
+        /* not 0x3fffff due to HW limitation */
+        .set_max_nums_pte_pde = 0x3fffe0 >> 3,
+        .set_pte_pde_num_dw = 10,
         .set_pte_pde = sdma_v3_0_vm_set_pte_pde,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index fd7c72aaafa6..46009db3d195 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -54,7 +54,7 @@ static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev);
 
 static const u32 golden_settings_sdma_4[] = {
-        SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831f07,
+        SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831d07,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CLK_CTRL), 0xff000ff0, 0x3f000100,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_IB_CNTL), 0x800f0100, 0x00000100,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL), 0xfffffff7, 0x00403000,
@@ -89,7 +89,7 @@ static const u32 golden_settings_sdma_vg10[] = {
 
 static const u32 golden_settings_sdma_4_1[] =
 {
-        SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831f07,
+        SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831d07,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CLK_CTRL), 0xffffffff, 0x3f000100,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_IB_CNTL), 0x800f0111, 0x00000100,
         SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL), 0xfffffff7, 0x00403000,
@@ -371,7 +371,7 @@ static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring,
 static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
         u32 ref_and_mask = 0;
-        struct nbio_hdp_flush_reg *nbio_hf_reg;
+        const struct nbio_hdp_flush_reg *nbio_hf_reg;
 
         if (ring->adev->flags & AMD_IS_APU)
                 nbio_hf_reg = &nbio_v7_0_hdp_flush_reg;
@@ -398,7 +398,7 @@ static void sdma_v4_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
         amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
                           SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
-        amdgpu_ring_write(ring, SOC15_REG_OFFSET(HDP, 0, mmHDP_DEBUG0));
+        amdgpu_ring_write(ring, SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE));
         amdgpu_ring_write(ring, 1);
 }
 
@@ -1264,6 +1264,11 @@ static int sdma_v4_0_sw_fini(void *handle)
         for (i = 0; i < adev->sdma.num_instances; i++)
                 amdgpu_ring_fini(&adev->sdma.instance[i].ring);
 
+        for (i = 0; i < adev->sdma.num_instances; i++) {
+                release_firmware(adev->sdma.instance[i].fw);
+                adev->sdma.instance[i].fw = NULL;
+        }
+
         return 0;
 }
 
@@ -1714,8 +1719,13 @@ static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev)
 }
 
 static const struct amdgpu_vm_pte_funcs sdma_v4_0_vm_pte_funcs = {
+        .copy_pte_num_dw = 7,
         .copy_pte = sdma_v4_0_vm_copy_pte,
+
         .write_pte = sdma_v4_0_vm_write_pte,
+
+        .set_max_nums_pte_pde = 0x400000 >> 3,
+        .set_pte_pde_num_dw = 10,
         .set_pte_pde = sdma_v4_0_vm_set_pte_pde,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 112969f3301a..3fa2fbf8c9a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -887,8 +887,13 @@ static void si_dma_set_buffer_funcs(struct amdgpu_device *adev)
 }
 
 static const struct amdgpu_vm_pte_funcs si_dma_vm_pte_funcs = {
+        .copy_pte_num_dw = 5,
         .copy_pte = si_dma_vm_copy_pte,
+
         .write_pte = si_dma_vm_write_pte,
+
+        .set_max_nums_pte_pde = 0xffff8 >> 3,
+        .set_pte_pde_num_dw = 9,
         .set_pte_pde = si_dma_vm_set_pte_pde,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index d63873f3f574..51fd0c9a20a5 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -1847,7 +1847,6 @@ static int si_calculate_sclk_params(struct amdgpu_device *adev,
 
 static void si_thermal_start_smc_fan_control(struct amdgpu_device *adev);
 static void si_fan_ctrl_set_default_mode(struct amdgpu_device *adev);
-static void si_dpm_set_dpm_funcs(struct amdgpu_device *adev);
 static void si_dpm_set_irq_funcs(struct amdgpu_device *adev);
 
 static struct si_power_info *si_get_pi(struct amdgpu_device *adev)
@@ -3060,9 +3059,9 @@ static int si_get_vce_clock_voltage(struct amdgpu_device *adev,
         return ret;
 }
 
-static bool si_dpm_vblank_too_short(struct amdgpu_device *adev)
+static bool si_dpm_vblank_too_short(void *handle)
 {
-
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         u32 vblank_time = amdgpu_dpm_get_vblank_time(adev);
         /* we never hit the non-gddr5 limit so disable it */
         u32 switch_limit = adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5 ? 450 : 0;
@@ -3871,9 +3870,10 @@ static int si_restrict_performance_levels_before_switch(struct amdgpu_device *ad
                 0 : -EINVAL;
 }
 
-static int si_dpm_force_performance_level(struct amdgpu_device *adev,
+static int si_dpm_force_performance_level(void *handle,
                                    enum amd_dpm_forced_level level)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct amdgpu_ps *rps = adev->pm.dpm.current_ps;
         struct  si_ps *ps = si_get_ps(rps);
         u32 levels = ps->performance_level_count;
@@ -6575,11 +6575,12 @@ static int si_fan_ctrl_stop_smc_fan_control(struct amdgpu_device *adev)
         }
 }
 
-static int si_dpm_get_fan_speed_percent(struct amdgpu_device *adev,
+static int si_dpm_get_fan_speed_percent(void *handle,
                                       u32 *speed)
 {
         u32 duty, duty100;
         u64 tmp64;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (adev->pm.no_fan)
                 return -ENOENT;
@@ -6600,9 +6601,10 @@ static int si_dpm_get_fan_speed_percent(struct amdgpu_device *adev,
         return 0;
 }
 
-static int si_dpm_set_fan_speed_percent(struct amdgpu_device *adev,
+static int si_dpm_set_fan_speed_percent(void *handle,
                                       u32 speed)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct si_power_info *si_pi = si_get_pi(adev);
         u32 tmp;
         u32 duty, duty100;
@@ -6633,8 +6635,10 @@ static int si_dpm_set_fan_speed_percent(struct amdgpu_device *adev,
         return 0;
 }
 
-static void si_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode)
+static void si_dpm_set_fan_control_mode(void *handle, u32 mode)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
         if (mode) {
                 /* stop auto-manage */
                 if (adev->pm.dpm.fan.ucode_fan_control)
@@ -6649,8 +6653,9 @@ static void si_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode)
         }
 }
 
-static u32 si_dpm_get_fan_control_mode(struct amdgpu_device *adev)
+static u32 si_dpm_get_fan_control_mode(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct si_power_info *si_pi = si_get_pi(adev);
         u32 tmp;
 
@@ -6946,8 +6951,9 @@ static void si_dpm_disable(struct amdgpu_device *adev)
         ni_update_current_ps(adev, boot_ps);
 }
 
-static int si_dpm_pre_set_power_state(struct amdgpu_device *adev)
+static int si_dpm_pre_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct amdgpu_ps requested_ps = *adev->pm.dpm.requested_ps;
         struct amdgpu_ps *new_ps = &requested_ps;
@@ -6984,8 +6990,9 @@ static int si_power_control_set_level(struct amdgpu_device *adev)
         return 0;
 }
 
-static int si_dpm_set_power_state(struct amdgpu_device *adev)
+static int si_dpm_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct amdgpu_ps *new_ps = &eg_pi->requested_rps;
         struct amdgpu_ps *old_ps = &eg_pi->current_rps;
@@ -7086,8 +7093,9 @@ static int si_dpm_set_power_state(struct amdgpu_device *adev)
         return 0;
 }
 
-static void si_dpm_post_set_power_state(struct amdgpu_device *adev)
+static void si_dpm_post_set_power_state(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct amdgpu_ps *new_ps = &eg_pi->requested_rps;
 
@@ -7103,8 +7111,10 @@ void si_dpm_reset_asic(struct amdgpu_device *adev)
 }
 #endif
 
-static void si_dpm_display_configuration_changed(struct amdgpu_device *adev)
+static void si_dpm_display_configuration_changed(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
         si_program_display_gap(adev);
 }
 
@@ -7486,9 +7496,10 @@ static void si_dpm_fini(struct amdgpu_device *adev)
         amdgpu_free_extended_power_table(adev);
 }
 
-static void si_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
+static void si_dpm_debugfs_print_current_performance_level(void *handle,
                                                     struct seq_file *m)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct amdgpu_ps *rps = &eg_pi->current_rps;
         struct  si_ps *ps = si_get_ps(rps);
@@ -7593,11 +7604,6 @@ static int si_dpm_late_init(void *handle)
         if (!amdgpu_dpm)
                 return 0;
 
-        /* init the sysfs and debugfs files late */
-        ret = amdgpu_pm_sysfs_init(adev);
-        if (ret)
-                return ret;
-
         ret = si_set_temperature_range(adev);
         if (ret)
                 return ret;
@@ -7753,7 +7759,6 @@ static int si_dpm_sw_fini(void *handle)
         flush_work(&adev->pm.dpm.thermal.work);
 
         mutex_lock(&adev->pm.mutex);
-        amdgpu_pm_sysfs_fini(adev);
         si_dpm_fini(adev);
         mutex_unlock(&adev->pm.mutex);
 
@@ -7860,10 +7865,11 @@ static int si_dpm_set_powergating_state(void *handle,
 }
 
 /* get temperature in millidegrees */
-static int si_dpm_get_temp(struct amdgpu_device *adev)
+static int si_dpm_get_temp(void *handle)
 {
         u32 temp;
         int actual_temp = 0;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
                 CTF_TEMP_SHIFT;
@@ -7878,8 +7884,9 @@ static int si_dpm_get_temp(struct amdgpu_device *adev)
         return actual_temp;
 }
 
-static u32 si_dpm_get_sclk(struct amdgpu_device *adev, bool low)
+static u32 si_dpm_get_sclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct  si_ps *requested_state = si_get_ps(&eg_pi->requested_rps);
 
@@ -7889,8 +7896,9 @@ static u32 si_dpm_get_sclk(struct amdgpu_device *adev, bool low)
                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
 }
 
-static u32 si_dpm_get_mclk(struct amdgpu_device *adev, bool low)
+static u32 si_dpm_get_mclk(void *handle, bool low)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct  si_ps *requested_state = si_get_ps(&eg_pi->requested_rps);
 
@@ -7900,9 +7908,11 @@ static u32 si_dpm_get_mclk(struct amdgpu_device *adev, bool low)
                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
 }
 
-static void si_dpm_print_power_state(struct amdgpu_device *adev,
-                                     struct amdgpu_ps *rps)
+static void si_dpm_print_power_state(void *handle,
+                                     void *current_ps)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)current_ps;
         struct  si_ps *ps = si_get_ps(rps);
         struct rv7xx_pl *pl;
         int i;
@@ -7927,7 +7937,6 @@ static int si_dpm_early_init(void *handle)
 
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        si_dpm_set_dpm_funcs(adev);
         si_dpm_set_irq_funcs(adev);
         return 0;
 }
@@ -7942,20 +7951,23 @@ static inline bool si_are_power_levels_equal(const struct rv7xx_pl  *si_cpl1,
                   (si_cpl1->vddci == si_cpl2->vddci));
 }
 
-static int si_check_state_equal(struct amdgpu_device *adev,
-                                struct amdgpu_ps *cps,
-                                struct amdgpu_ps *rps,
+static int si_check_state_equal(void *handle,
+                                void *current_ps,
+                                void *request_ps,
                                 bool *equal)
 {
         struct si_ps *si_cps;
         struct si_ps *si_rps;
         int i;
+        struct amdgpu_ps *cps = (struct amdgpu_ps *)current_ps;
+        struct amdgpu_ps *rps = (struct amdgpu_ps *)request_ps;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         if (adev == NULL || cps == NULL || rps == NULL || equal == NULL)
                 return -EINVAL;
 
-        si_cps = si_get_ps(cps);
-        si_rps = si_get_ps(rps);
+        si_cps = si_get_ps((struct amdgpu_ps *)cps);
+        si_rps = si_get_ps((struct amdgpu_ps *)rps);
 
         if (si_cps == NULL) {
                 printk("si_cps is NULL\n");
@@ -7983,9 +7995,10 @@ static int si_check_state_equal(struct amdgpu_device *adev,
         return 0;
 }
 
-static int si_dpm_read_sensor(struct amdgpu_device *adev, int idx,
+static int si_dpm_read_sensor(void *handle, int idx,
                               void *value, int *size)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
         struct amdgpu_ps *rps = &eg_pi->current_rps;
         struct  si_ps *ps = si_get_ps(rps);
@@ -8041,7 +8054,7 @@ const struct amd_ip_funcs si_dpm_ip_funcs = {
         .set_powergating_state = si_dpm_set_powergating_state,
 };
 
-static const struct amdgpu_dpm_funcs si_dpm_funcs = {
+const struct amd_pm_funcs si_dpm_funcs = {
         .get_temperature = &si_dpm_get_temp,
         .pre_set_power_state = &si_dpm_pre_set_power_state,
         .set_power_state = &si_dpm_set_power_state,
@@ -8062,12 +8075,6 @@ static const struct amdgpu_dpm_funcs si_dpm_funcs = {
         .read_sensor = &si_dpm_read_sensor,
 };
 
-static void si_dpm_set_dpm_funcs(struct amdgpu_device *adev)
-{
-        if (adev->pm.funcs == NULL)
-                adev->pm.funcs = &si_dpm_funcs;
-}
-
 static const struct amdgpu_irq_src_funcs si_dpm_irq_funcs = {
         .set = si_dpm_set_interrupt_state,
         .process = si_dpm_process_interrupt,
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.h b/drivers/gpu/drm/amd/amdgpu/si_dpm.h
index 51ce21c5f4fb..9fe343de3477 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.h
@@ -246,6 +246,7 @@ enum si_display_gap
 };
 
 extern const struct amd_ip_funcs si_dpm_ip_funcs;
+extern const struct amd_pm_funcs si_dpm_funcs;
 
 struct ni_leakage_coeffients
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c
index ce25e03a077d..d2c6b80309c8 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c
@@ -118,6 +118,19 @@ static u32 si_ih_get_wptr(struct amdgpu_device *adev)
         return (wptr & adev->irq.ih.ptr_mask);
 }
 
+/**
+ * si_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool si_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        /* Process all interrupts */
+        return true;
+}
+
 static void si_ih_decode_iv(struct amdgpu_device *adev,
                              struct amdgpu_iv_entry *entry)
 {
@@ -288,6 +301,7 @@ static const struct amd_ip_funcs si_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs si_ih_funcs = {
         .get_wptr = si_ih_get_wptr,
+        .prescreen_iv = si_ih_prescreen_iv,
         .decode_iv = si_ih_decode_iv,
         .set_rptr = si_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index f2c3a49f73a0..4e67fe1e7955 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -101,7 +101,7 @@ static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 {
         unsigned long flags, address, data;
         u32 r;
-        struct nbio_pcie_index_data *nbio_pcie_id;
+        const struct nbio_pcie_index_data *nbio_pcie_id;
 
         if (adev->flags & AMD_IS_APU)
                 nbio_pcie_id = &nbio_v7_0_pcie_index_data;
@@ -122,7 +122,7 @@ static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 static void soc15_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
         unsigned long flags, address, data;
-        struct nbio_pcie_index_data *nbio_pcie_id;
+        const struct nbio_pcie_index_data *nbio_pcie_id;
 
         if (adev->flags & AMD_IS_APU)
                 nbio_pcie_id = &nbio_v7_0_pcie_index_data;
@@ -279,10 +279,7 @@ static void soc15_init_golden_registers(struct amdgpu_device *adev)
 }
 static u32 soc15_get_xclk(struct amdgpu_device *adev)
 {
-        if (adev->asic_type == CHIP_VEGA10)
-                return adev->clock.spll.reference_freq/4;
-        else
-                return adev->clock.spll.reference_freq;
+        return adev->clock.spll.reference_freq;
 }
 
 
@@ -407,18 +404,27 @@ static int soc15_read_register(struct amdgpu_device *adev, u32 se_num,
         return -EINVAL;
 }
 
-static void soc15_gpu_pci_config_reset(struct amdgpu_device *adev)
+static int soc15_asic_reset(struct amdgpu_device *adev)
 {
         u32 i;
 
-        dev_info(adev->dev, "GPU pci config reset\n");
+        amdgpu_atombios_scratch_regs_engine_hung(adev, true);
+
+        dev_info(adev->dev, "GPU reset\n");
 
         /* disable BM */
         pci_clear_master(adev->pdev);
-        /* reset */
-        amdgpu_pci_config_reset(adev);
 
-        udelay(100);
+        pci_save_state(adev->pdev);
+
+        for (i = 0; i < AMDGPU_MAX_IP_NUM; i++) {
+                if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP){
+                        adev->ip_blocks[i].version->funcs->soft_reset((void *)adev);
+                        break;
+                }
+        }
+
+        pci_restore_state(adev->pdev);
 
         /* wait for asic to come out of reset */
         for (i = 0; i < adev->usec_timeout; i++) {
@@ -430,14 +436,6 @@ static void soc15_gpu_pci_config_reset(struct amdgpu_device *adev)
                 udelay(1);
         }
 
-}
-
-static int soc15_asic_reset(struct amdgpu_device *adev)
-{
-        amdgpu_atombios_scratch_regs_engine_hung(adev, true);
-
-        soc15_gpu_pci_config_reset(adev);
-
         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
 
         return 0;
@@ -534,6 +532,12 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev)
                         amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
                 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+                else if (amdgpu_device_has_dc_support(adev))
+                        amdgpu_ip_block_add(adev, &dm_ip_block);
+#else
+#       warning "Enable CONFIG_DRM_AMD_DC for display support on SOC15."
+#endif
                 amdgpu_ip_block_add(adev, &gfx_v9_0_ip_block);
                 amdgpu_ip_block_add(adev, &sdma_v4_0_ip_block);
                 amdgpu_ip_block_add(adev, &uvd_v7_0_ip_block);
@@ -547,6 +551,12 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev)
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
                 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+                else if (amdgpu_device_has_dc_support(adev))
+                        amdgpu_ip_block_add(adev, &dm_ip_block);
+#else
+#       warning "Enable CONFIG_DRM_AMD_DC for display support on SOC15."
+#endif
                 amdgpu_ip_block_add(adev, &gfx_v9_0_ip_block);
                 amdgpu_ip_block_add(adev, &sdma_v4_0_ip_block);
                 amdgpu_ip_block_add(adev, &vcn_v1_0_ip_block);
@@ -603,21 +613,6 @@ static int soc15_common_early_init(void *handle)
                 (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_PSP)))
                 psp_enabled = true;
 
-        /*
-         * nbio need be used for both sdma and gfx9, but only
-         * initializes once
-         */
-        switch(adev->asic_type) {
-        case CHIP_VEGA10:
-                nbio_v6_1_init(adev);
-                break;
-        case CHIP_RAVEN:
-                nbio_v7_0_init(adev);
-                break;
-        default:
-                return -EINVAL;
-        }
-
         adev->rev_id = soc15_get_rev_id(adev);
         adev->external_rev_id = 0xFF;
         switch (adev->asic_type) {
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index 923df2c0e535..aa4e320e31f8 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -219,6 +219,34 @@ static u32 tonga_ih_get_wptr(struct amdgpu_device *adev)
 }
 
 /**
+ * tonga_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool tonga_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        u32 ring_index = adev->irq.ih.rptr >> 2;
+        u16 pasid;
+
+        switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
+        case 146:
+        case 147:
+                pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
+                if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
+                        return true;
+                break;
+        default:
+                /* Not a VM fault */
+                return true;
+        }
+
+        adev->irq.ih.rptr += 16;
+        return false;
+}
+
+/**
  * tonga_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -478,6 +506,7 @@ static const struct amd_ip_funcs tonga_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs tonga_ih_funcs = {
         .get_wptr = tonga_ih_get_wptr,
+        .prescreen_iv = tonga_ih_prescreen_iv,
         .decode_iv = tonga_ih_decode_iv,
         .set_rptr = tonga_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index 62cd16a23921..920910ac8663 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -38,6 +38,8 @@
 #include "vi.h"
 
 static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev);
+static void uvd_v6_0_set_enc_ring_funcs(struct amdgpu_device *adev);
+
 static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev);
 static int uvd_v6_0_start(struct amdgpu_device *adev);
 static void uvd_v6_0_stop(struct amdgpu_device *adev);
@@ -48,6 +50,18 @@ static void uvd_v6_0_enable_mgcg(struct amdgpu_device *adev,
                                  bool enable);
 
 /**
+* uvd_v6_0_enc_support - get encode support status
+*
+* @adev: amdgpu_device pointer
+*
+* Returns the current hardware encode support status
+*/
+static inline bool uvd_v6_0_enc_support(struct amdgpu_device *adev)
+{
+        return ((adev->asic_type >= CHIP_POLARIS10) && (adev->asic_type <= CHIP_POLARIS12));
+}
+
+/**
  * uvd_v6_0_ring_get_rptr - get read pointer
  *
  * @ring: amdgpu_ring pointer
@@ -62,6 +76,22 @@ static uint64_t uvd_v6_0_ring_get_rptr(struct amdgpu_ring *ring)
 }
 
 /**
+ * uvd_v6_0_enc_ring_get_rptr - get enc read pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware enc read pointer
+ */
+static uint64_t uvd_v6_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
+{
+        struct amdgpu_device *adev = ring->adev;
+
+        if (ring == &adev->uvd.ring_enc[0])
+                return RREG32(mmUVD_RB_RPTR);
+        else
+                return RREG32(mmUVD_RB_RPTR2);
+}
+/**
  * uvd_v6_0_ring_get_wptr - get write pointer
  *
  * @ring: amdgpu_ring pointer
@@ -76,6 +106,23 @@ static uint64_t uvd_v6_0_ring_get_wptr(struct amdgpu_ring *ring)
 }
 
 /**
+ * uvd_v6_0_enc_ring_get_wptr - get enc write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware enc write pointer
+ */
+static uint64_t uvd_v6_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
+{
+        struct amdgpu_device *adev = ring->adev;
+
+        if (ring == &adev->uvd.ring_enc[0])
+                return RREG32(mmUVD_RB_WPTR);
+        else
+                return RREG32(mmUVD_RB_WPTR2);
+}
+
+/**
  * uvd_v6_0_ring_set_wptr - set write pointer
  *
  * @ring: amdgpu_ring pointer
@@ -89,6 +136,237 @@ static void uvd_v6_0_ring_set_wptr(struct amdgpu_ring *ring)
         WREG32(mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
 }
 
+/**
+ * uvd_v6_0_enc_ring_set_wptr - set enc write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Commits the enc write pointer to the hardware
+ */
+static void uvd_v6_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
+{
+        struct amdgpu_device *adev = ring->adev;
+
+        if (ring == &adev->uvd.ring_enc[0])
+                WREG32(mmUVD_RB_WPTR,
+                        lower_32_bits(ring->wptr));
+        else
+                WREG32(mmUVD_RB_WPTR2,
+                        lower_32_bits(ring->wptr));
+}
+
+/**
+ * uvd_v6_0_enc_ring_test_ring - test if UVD ENC ring is working
+ *
+ * @ring: the engine to test on
+ *
+ */
+static int uvd_v6_0_enc_ring_test_ring(struct amdgpu_ring *ring)
+{
+        struct amdgpu_device *adev = ring->adev;
+        uint32_t rptr = amdgpu_ring_get_rptr(ring);
+        unsigned i;
+        int r;
+
+        r = amdgpu_ring_alloc(ring, 16);
+        if (r) {
+                DRM_ERROR("amdgpu: uvd enc failed to lock ring %d (%d).\n",
+                          ring->idx, r);
+                return r;
+        }
+        amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
+        amdgpu_ring_commit(ring);
+
+        for (i = 0; i < adev->usec_timeout; i++) {
+                if (amdgpu_ring_get_rptr(ring) != rptr)
+                        break;
+                DRM_UDELAY(1);
+        }
+
+        if (i < adev->usec_timeout) {
+                DRM_INFO("ring test on %d succeeded in %d usecs\n",
+                         ring->idx, i);
+        } else {
+                DRM_ERROR("amdgpu: ring %d test failed\n",
+                          ring->idx);
+                r = -ETIMEDOUT;
+        }
+
+        return r;
+}
+
+/**
+ * uvd_v6_0_enc_get_create_msg - generate a UVD ENC create msg
+ *
+ * @adev: amdgpu_device pointer
+ * @ring: ring we should submit the msg to
+ * @handle: session handle to use
+ * @fence: optional fence to return
+ *
+ * Open up a stream for HW test
+ */
+static int uvd_v6_0_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
+                                       struct dma_fence **fence)
+{
+        const unsigned ib_size_dw = 16;
+        struct amdgpu_job *job;
+        struct amdgpu_ib *ib;
+        struct dma_fence *f = NULL;
+        uint64_t dummy;
+        int i, r;
+
+        r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
+        if (r)
+                return r;
+
+        ib = &job->ibs[0];
+        dummy = ib->gpu_addr + 1024;
+
+        ib->length_dw = 0;
+        ib->ptr[ib->length_dw++] = 0x00000018;
+        ib->ptr[ib->length_dw++] = 0x00000001; /* session info */
+        ib->ptr[ib->length_dw++] = handle;
+        ib->ptr[ib->length_dw++] = 0x00010000;
+        ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
+        ib->ptr[ib->length_dw++] = dummy;
+
+        ib->ptr[ib->length_dw++] = 0x00000014;
+        ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
+        ib->ptr[ib->length_dw++] = 0x0000001c;
+        ib->ptr[ib->length_dw++] = 0x00000001;
+        ib->ptr[ib->length_dw++] = 0x00000000;
+
+        ib->ptr[ib->length_dw++] = 0x00000008;
+        ib->ptr[ib->length_dw++] = 0x08000001; /* op initialize */
+
+        for (i = ib->length_dw; i < ib_size_dw; ++i)
+                ib->ptr[i] = 0x0;
+
+        r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
+        job->fence = dma_fence_get(f);
+        if (r)
+                goto err;
+
+        amdgpu_job_free(job);
+        if (fence)
+                *fence = dma_fence_get(f);
+        dma_fence_put(f);
+        return 0;
+
+err:
+        amdgpu_job_free(job);
+        return r;
+}
+
+/**
+ * uvd_v6_0_enc_get_destroy_msg - generate a UVD ENC destroy msg
+ *
+ * @adev: amdgpu_device pointer
+ * @ring: ring we should submit the msg to
+ * @handle: session handle to use
+ * @fence: optional fence to return
+ *
+ * Close up a stream for HW test or if userspace failed to do so
+ */
+static int uvd_v6_0_enc_get_destroy_msg(struct amdgpu_ring *ring,
+                                        uint32_t handle,
+                                        bool direct, struct dma_fence **fence)
+{
+        const unsigned ib_size_dw = 16;
+        struct amdgpu_job *job;
+        struct amdgpu_ib *ib;
+        struct dma_fence *f = NULL;
+        uint64_t dummy;
+        int i, r;
+
+        r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
+        if (r)
+                return r;
+
+        ib = &job->ibs[0];
+        dummy = ib->gpu_addr + 1024;
+
+        ib->length_dw = 0;
+        ib->ptr[ib->length_dw++] = 0x00000018;
+        ib->ptr[ib->length_dw++] = 0x00000001; /* session info */
+        ib->ptr[ib->length_dw++] = handle;
+        ib->ptr[ib->length_dw++] = 0x00010000;
+        ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
+        ib->ptr[ib->length_dw++] = dummy;
+
+        ib->ptr[ib->length_dw++] = 0x00000014;
+        ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
+        ib->ptr[ib->length_dw++] = 0x0000001c;
+        ib->ptr[ib->length_dw++] = 0x00000001;
+        ib->ptr[ib->length_dw++] = 0x00000000;
+
+        ib->ptr[ib->length_dw++] = 0x00000008;
+        ib->ptr[ib->length_dw++] = 0x08000002; /* op close session */
+
+        for (i = ib->length_dw; i < ib_size_dw; ++i)
+                ib->ptr[i] = 0x0;
+
+        if (direct) {
+                r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
+                job->fence = dma_fence_get(f);
+                if (r)
+                        goto err;
+
+                amdgpu_job_free(job);
+        } else {
+                r = amdgpu_job_submit(job, ring, &ring->adev->vce.entity,
+                                      AMDGPU_FENCE_OWNER_UNDEFINED, &f);
+                if (r)
+                        goto err;
+        }
+
+        if (fence)
+                *fence = dma_fence_get(f);
+        dma_fence_put(f);
+        return 0;
+
+err:
+        amdgpu_job_free(job);
+        return r;
+}
+
+/**
+ * uvd_v6_0_enc_ring_test_ib - test if UVD ENC IBs are working
+ *
+ * @ring: the engine to test on
+ *
+ */
+static int uvd_v6_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
+{
+        struct dma_fence *fence = NULL;
+        long r;
+
+        r = uvd_v6_0_enc_get_create_msg(ring, 1, NULL);
+        if (r) {
+                DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r);
+                goto error;
+        }
+
+        r = uvd_v6_0_enc_get_destroy_msg(ring, 1, true, &fence);
+        if (r) {
+                DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r);
+                goto error;
+        }
+
+        r = dma_fence_wait_timeout(fence, false, timeout);
+        if (r == 0) {
+                DRM_ERROR("amdgpu: IB test timed out.\n");
+                r = -ETIMEDOUT;
+        } else if (r < 0) {
+                DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
+        } else {
+                DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+                r = 0;
+        }
+error:
+        dma_fence_put(fence);
+        return r;
+}
 static int uvd_v6_0_early_init(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -98,6 +376,12 @@ static int uvd_v6_0_early_init(void *handle)
                 return -ENOENT;
 
         uvd_v6_0_set_ring_funcs(adev);
+
+        if (uvd_v6_0_enc_support(adev)) {
+                adev->uvd.num_enc_rings = 2;
+                uvd_v6_0_set_enc_ring_funcs(adev);
+        }
+
         uvd_v6_0_set_irq_funcs(adev);
 
         return 0;
@@ -106,7 +390,7 @@ static int uvd_v6_0_early_init(void *handle)
 static int uvd_v6_0_sw_init(void *handle)
 {
         struct amdgpu_ring *ring;
-        int r;
+        int i, r;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         /* UVD TRAP */
@@ -114,10 +398,31 @@ static int uvd_v6_0_sw_init(void *handle)
         if (r)
                 return r;
 
+        /* UVD ENC TRAP */
+        if (uvd_v6_0_enc_support(adev)) {
+                for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
+                        r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, i + 119, &adev->uvd.irq);
+                        if (r)
+                                return r;
+                }
+        }
+
         r = amdgpu_uvd_sw_init(adev);
         if (r)
                 return r;
 
+        if (uvd_v6_0_enc_support(adev)) {
+                struct amd_sched_rq *rq;
+                ring = &adev->uvd.ring_enc[0];
+                rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
+                r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
+                                          rq, amdgpu_sched_jobs);
+                if (r) {
+                        DRM_ERROR("Failed setting up UVD ENC run queue.\n");
+                        return r;
+                }
+        }
+
         r = amdgpu_uvd_resume(adev);
         if (r)
                 return r;
@@ -125,19 +430,38 @@ static int uvd_v6_0_sw_init(void *handle)
         ring = &adev->uvd.ring;
         sprintf(ring->name, "uvd");
         r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
+        if (r)
+                return r;
+
+        if (uvd_v6_0_enc_support(adev)) {
+                for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
+                        ring = &adev->uvd.ring_enc[i];
+                        sprintf(ring->name, "uvd_enc%d", i);
+                        r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
+                        if (r)
+                                return r;
+                }
+        }
 
         return r;
 }
 
 static int uvd_v6_0_sw_fini(void *handle)
 {
-        int r;
+        int i, r;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         r = amdgpu_uvd_suspend(adev);
         if (r)
                 return r;
 
+        if (uvd_v6_0_enc_support(adev)) {
+                amd_sched_entity_fini(&adev->uvd.ring_enc[0].sched, &adev->uvd.entity_enc);
+
+                for (i = 0; i < adev->uvd.num_enc_rings; ++i)
+                        amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
+        }
+
         return amdgpu_uvd_sw_fini(adev);
 }
 
@@ -153,7 +477,7 @@ static int uvd_v6_0_hw_init(void *handle)
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
         struct amdgpu_ring *ring = &adev->uvd.ring;
         uint32_t tmp;
-        int r;
+        int i, r;
 
         amdgpu_asic_set_uvd_clocks(adev, 10000, 10000);
         uvd_v6_0_set_clockgating_state(adev, AMD_CG_STATE_UNGATE);
@@ -193,9 +517,25 @@ static int uvd_v6_0_hw_init(void *handle)
 
         amdgpu_ring_commit(ring);
 
+        if (uvd_v6_0_enc_support(adev)) {
+                for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
+                        ring = &adev->uvd.ring_enc[i];
+                        ring->ready = true;
+                        r = amdgpu_ring_test_ring(ring);
+                        if (r) {
+                                ring->ready = false;
+                                goto done;
+                        }
+                }
+        }
+
 done:
-        if (!r)
-                DRM_INFO("UVD initialized successfully.\n");
+        if (!r) {
+                if (uvd_v6_0_enc_support(adev))
+                        DRM_INFO("UVD and UVD ENC initialized successfully.\n");
+                else
+                        DRM_INFO("UVD initialized successfully.\n");
+        }
 
         return r;
 }
@@ -512,6 +852,22 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
 
         WREG32_FIELD(UVD_RBC_RB_CNTL, RB_NO_FETCH, 0);
 
+        if (uvd_v6_0_enc_support(adev)) {
+                ring = &adev->uvd.ring_enc[0];
+                WREG32(mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
+                WREG32(mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
+                WREG32(mmUVD_RB_BASE_LO, ring->gpu_addr);
+                WREG32(mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
+                WREG32(mmUVD_RB_SIZE, ring->ring_size / 4);
+
+                ring = &adev->uvd.ring_enc[1];
+                WREG32(mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
+                WREG32(mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
+                WREG32(mmUVD_RB_BASE_LO2, ring->gpu_addr);
+                WREG32(mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
+                WREG32(mmUVD_RB_SIZE2, ring->ring_size / 4);
+        }
+
         return 0;
 }
 
@@ -575,6 +931,26 @@ static void uvd_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq
 }
 
 /**
+ * uvd_v6_0_enc_ring_emit_fence - emit an enc fence & trap command
+ *
+ * @ring: amdgpu_ring pointer
+ * @fence: fence to emit
+ *
+ * Write enc a fence and a trap command to the ring.
+ */
+static void uvd_v6_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
+                        u64 seq, unsigned flags)
+{
+        WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+
+        amdgpu_ring_write(ring, HEVC_ENC_CMD_FENCE);
+        amdgpu_ring_write(ring, addr);
+        amdgpu_ring_write(ring, upper_32_bits(addr));
+        amdgpu_ring_write(ring, seq);
+        amdgpu_ring_write(ring, HEVC_ENC_CMD_TRAP);
+}
+
+/**
  * uvd_v6_0_ring_emit_hdp_flush - emit an hdp flush
  *
  * @ring: amdgpu_ring pointer
@@ -665,6 +1041,24 @@ static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
         amdgpu_ring_write(ring, ib->length_dw);
 }
 
+/**
+ * uvd_v6_0_enc_ring_emit_ib - enc execute indirect buffer
+ *
+ * @ring: amdgpu_ring pointer
+ * @ib: indirect buffer to execute
+ *
+ * Write enc ring commands to execute the indirect buffer
+ */
+static void uvd_v6_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
+                struct amdgpu_ib *ib, unsigned int vm_id, bool ctx_switch)
+{
+        amdgpu_ring_write(ring, HEVC_ENC_CMD_IB_VM);
+        amdgpu_ring_write(ring, vm_id);
+        amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
+        amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+        amdgpu_ring_write(ring, ib->length_dw);
+}
+
 static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
                                          unsigned vm_id, uint64_t pd_addr)
 {
@@ -716,6 +1110,33 @@ static void uvd_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
         amdgpu_ring_write(ring, 0xE);
 }
 
+static void uvd_v6_0_enc_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
+{
+        uint32_t seq = ring->fence_drv.sync_seq;
+        uint64_t addr = ring->fence_drv.gpu_addr;
+
+        amdgpu_ring_write(ring, HEVC_ENC_CMD_WAIT_GE);
+        amdgpu_ring_write(ring, lower_32_bits(addr));
+        amdgpu_ring_write(ring, upper_32_bits(addr));
+        amdgpu_ring_write(ring, seq);
+}
+
+static void uvd_v6_0_enc_ring_insert_end(struct amdgpu_ring *ring)
+{
+        amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
+}
+
+static void uvd_v6_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
+        unsigned int vm_id, uint64_t pd_addr)
+{
+        amdgpu_ring_write(ring, HEVC_ENC_CMD_UPDATE_PTB);
+        amdgpu_ring_write(ring, vm_id);
+        amdgpu_ring_write(ring, pd_addr >> 12);
+
+        amdgpu_ring_write(ring, HEVC_ENC_CMD_FLUSH_TLB);
+        amdgpu_ring_write(ring, vm_id);
+}
+
 static bool uvd_v6_0_is_idle(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -823,8 +1244,31 @@ static int uvd_v6_0_process_interrupt(struct amdgpu_device *adev,
                                       struct amdgpu_irq_src *source,
                                       struct amdgpu_iv_entry *entry)
 {
+        bool int_handled = true;
         DRM_DEBUG("IH: UVD TRAP\n");
-        amdgpu_fence_process(&adev->uvd.ring);
+
+        switch (entry->src_id) {
+        case 124:
+                amdgpu_fence_process(&adev->uvd.ring);
+                break;
+        case 119:
+                if (likely(uvd_v6_0_enc_support(adev)))
+                        amdgpu_fence_process(&adev->uvd.ring_enc[0]);
+                else
+                        int_handled = false;
+                break;
+        case 120:
+                if (likely(uvd_v6_0_enc_support(adev)))
+                        amdgpu_fence_process(&adev->uvd.ring_enc[1]);
+                else
+                        int_handled = false;
+                break;
+        }
+
+        if (false == int_handled)
+                        DRM_ERROR("Unhandled interrupt: %d %d\n",
+                          entry->src_id, entry->src_data[0]);
+
         return 0;
 }
 
@@ -1151,6 +1595,33 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
         .end_use = amdgpu_uvd_ring_end_use,
 };
 
+static const struct amdgpu_ring_funcs uvd_v6_0_enc_ring_vm_funcs = {
+        .type = AMDGPU_RING_TYPE_UVD_ENC,
+        .align_mask = 0x3f,
+        .nop = HEVC_ENC_CMD_NO_OP,
+        .support_64bit_ptrs = false,
+        .get_rptr = uvd_v6_0_enc_ring_get_rptr,
+        .get_wptr = uvd_v6_0_enc_ring_get_wptr,
+        .set_wptr = uvd_v6_0_enc_ring_set_wptr,
+        .emit_frame_size =
+                4 + /* uvd_v6_0_enc_ring_emit_pipeline_sync */
+                6 + /* uvd_v6_0_enc_ring_emit_vm_flush */
+                5 + 5 + /* uvd_v6_0_enc_ring_emit_fence x2 vm fence */
+                1, /* uvd_v6_0_enc_ring_insert_end */
+        .emit_ib_size = 5, /* uvd_v6_0_enc_ring_emit_ib */
+        .emit_ib = uvd_v6_0_enc_ring_emit_ib,
+        .emit_fence = uvd_v6_0_enc_ring_emit_fence,
+        .emit_vm_flush = uvd_v6_0_enc_ring_emit_vm_flush,
+        .emit_pipeline_sync = uvd_v6_0_enc_ring_emit_pipeline_sync,
+        .test_ring = uvd_v6_0_enc_ring_test_ring,
+        .test_ib = uvd_v6_0_enc_ring_test_ib,
+        .insert_nop = amdgpu_ring_insert_nop,
+        .insert_end = uvd_v6_0_enc_ring_insert_end,
+        .pad_ib = amdgpu_ring_generic_pad_ib,
+        .begin_use = amdgpu_uvd_ring_begin_use,
+        .end_use = amdgpu_uvd_ring_end_use,
+};
+
 static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
 {
         if (adev->asic_type >= CHIP_POLARIS10) {
@@ -1162,6 +1633,16 @@ static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
         }
 }
 
+static void uvd_v6_0_set_enc_ring_funcs(struct amdgpu_device *adev)
+{
+        int i;
+
+        for (i = 0; i < adev->uvd.num_enc_rings; ++i)
+                adev->uvd.ring_enc[i].funcs = &uvd_v6_0_enc_ring_vm_funcs;
+
+        DRM_INFO("UVD ENC is enabled in VM mode\n");
+}
+
 static const struct amdgpu_irq_src_funcs uvd_v6_0_irq_funcs = {
         .set = uvd_v6_0_set_interrupt_state,
         .process = uvd_v6_0_process_interrupt,
@@ -1169,7 +1650,11 @@ static const struct amdgpu_irq_src_funcs uvd_v6_0_irq_funcs = {
 
 static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev)
 {
-        adev->uvd.irq.num_types = 1;
+        if (uvd_v6_0_enc_support(adev))
+                adev->uvd.irq.num_types = adev->uvd.num_enc_rings + 1;
+        else
+                adev->uvd.irq.num_types = 1;
+
         adev->uvd.irq.funcs = &uvd_v6_0_irq_funcs;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index 23a85750edd6..6634545060fd 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -592,11 +592,7 @@ static int uvd_v7_0_suspend(void *handle)
         if (r)
                 return r;
 
-        /* Skip this for APU for now */
-        if (!(adev->flags & AMD_IS_APU))
-                r = amdgpu_uvd_suspend(adev);
-
-        return r;
+        return amdgpu_uvd_suspend(adev);
 }
 
 static int uvd_v7_0_resume(void *handle)
@@ -604,12 +600,10 @@ static int uvd_v7_0_resume(void *handle)
         int r;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        /* Skip this for APU for now */
-        if (!(adev->flags & AMD_IS_APU)) {
-                r = amdgpu_uvd_resume(adev);
-                if (r)
-                        return r;
-        }
+        r = amdgpu_uvd_resume(adev);
+        if (r)
+                return r;
+
         return uvd_v7_0_hw_init(adev);
 }
 
@@ -1161,7 +1155,7 @@ static void uvd_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
  */
 static void uvd_v7_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
-        amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_DEBUG0), 0));
+        amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 0));
         amdgpu_ring_write(ring, 1);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index 11134d5f7443..75745544600a 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -1011,10 +1011,6 @@ static int vce_v4_0_process_interrupt(struct amdgpu_device *adev,
 {
         DRM_DEBUG("IH: VCE\n");
 
-        WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_STATUS),
-                        VCE_SYS_INT_STATUS__VCE_SYS_INT_TRAP_INTERRUPT_INT_MASK,
-                        ~VCE_SYS_INT_STATUS__VCE_SYS_INT_TRAP_INTERRUPT_INT_MASK);
-
         switch (entry->src_data[0]) {
         case 0:
         case 1:
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
index 21e7b88401e1..1eb4d79d6e30 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
@@ -812,7 +812,7 @@ static void vcn_v1_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64
  */
 static void vcn_v1_0_dec_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
-        amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_DEBUG0), 0));
+        amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 0));
         amdgpu_ring_write(ring, 1);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
index 56150e8d1ed2..697325737ba8 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
@@ -219,14 +219,95 @@ static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
                         wptr, adev->irq.ih.rptr, tmp);
                 adev->irq.ih.rptr = tmp;
 
-                tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
+                tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
                 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
-                WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
+                WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
         }
         return (wptr & adev->irq.ih.ptr_mask);
 }
 
 /**
+ * vega10_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
+{
+        u32 ring_index = adev->irq.ih.rptr >> 2;
+        u32 dw0, dw3, dw4, dw5;
+        u16 pasid;
+        u64 addr, key;
+        struct amdgpu_vm *vm;
+        int r;
+
+        dw0 = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
+        dw3 = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
+        dw4 = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
+        dw5 = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
+
+        /* Filter retry page faults, let only the first one pass. If
+         * there are too many outstanding faults, ignore them until
+         * some faults get cleared.
+         */
+        switch (dw0 & 0xff) {
+        case AMDGPU_IH_CLIENTID_VMC:
+        case AMDGPU_IH_CLIENTID_UTCL2:
+                break;
+        default:
+                /* Not a VM fault */
+                return true;
+        }
+
+        pasid = dw3 & 0xffff;
+        /* No PASID, can't identify faulting process */
+        if (!pasid)
+                return true;
+
+        /* Not a retry fault, check fault credit */
+        if (!(dw5 & 0x80)) {
+                if (!amdgpu_vm_pasid_fault_credit(adev, pasid))
+                        goto ignore_iv;
+                return true;
+        }
+
+        addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12);
+        key = AMDGPU_VM_FAULT(pasid, addr);
+        r = amdgpu_ih_add_fault(adev, key);
+
+        /* Hash table is full or the fault is already being processed,
+         * ignore further page faults
+         */
+        if (r != 0)
+                goto ignore_iv;
+
+        /* Track retry faults in per-VM fault FIFO. */
+        spin_lock(&adev->vm_manager.pasid_lock);
+        vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
+        spin_unlock(&adev->vm_manager.pasid_lock);
+        if (WARN_ON_ONCE(!vm)) {
+                /* VM not found, process it normally */
+                amdgpu_ih_clear_fault(adev, key);
+                return true;
+        }
+        /* No locking required with single writer and single reader */
+        r = kfifo_put(&vm->faults, key);
+        if (!r) {
+                /* FIFO is full. Ignore it until there is space */
+                amdgpu_ih_clear_fault(adev, key);
+                goto ignore_iv;
+        }
+
+        /* It's the first fault for this address, process it normally */
+        return true;
+
+ignore_iv:
+        adev->irq.ih.rptr += 32;
+        return false;
+}
+
+/**
  * vega10_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -310,6 +391,14 @@ static int vega10_ih_sw_init(void *handle)
         adev->irq.ih.use_doorbell = true;
         adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;
 
+        adev->irq.ih.faults = kmalloc(sizeof(*adev->irq.ih.faults), GFP_KERNEL);
+        if (!adev->irq.ih.faults)
+                return -ENOMEM;
+        INIT_CHASH_TABLE(adev->irq.ih.faults->hash,
+                         AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
+        spin_lock_init(&adev->irq.ih.faults->lock);
+        adev->irq.ih.faults->count = 0;
+
         r = amdgpu_irq_init(adev);
 
         return r;
@@ -322,6 +411,9 @@ static int vega10_ih_sw_fini(void *handle)
         amdgpu_irq_fini(adev);
         amdgpu_ih_ring_fini(adev);
 
+        kfree(adev->irq.ih.faults);
+        adev->irq.ih.faults = NULL;
+
         return 0;
 }
 
@@ -410,6 +502,7 @@ const struct amd_ip_funcs vega10_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs vega10_ih_funcs = {
         .get_wptr = vega10_ih_get_wptr,
+        .prescreen_iv = vega10_ih_prescreen_iv,
         .decode_iv = vega10_ih_decode_iv,
         .set_rptr = vega10_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 9ff69b90df36..3a4c2fa7e36d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -77,6 +77,7 @@
 #endif
 #include "dce_virtual.h"
 #include "mxgpu_vi.h"
+#include "amdgpu_dm.h"
 
 /*
  * Indirect registers accessor
@@ -1254,7 +1255,6 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
         uint32_t msg_id, pp_state = 0;
         uint32_t pp_support_state = 0;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-        void *pp_handle = adev->powerplay.pp_handle;
 
         if (adev->cg_flags & (AMD_CG_SUPPORT_MC_LS | AMD_CG_SUPPORT_MC_MGCG)) {
                 if (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) {
@@ -1271,7 +1271,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
                                PP_BLOCK_SYS_MC,
                                pp_support_state,
                                pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
         if (adev->cg_flags & (AMD_CG_SUPPORT_SDMA_LS | AMD_CG_SUPPORT_SDMA_MGCG)) {
@@ -1289,7 +1290,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
                                PP_BLOCK_SYS_SDMA,
                                pp_support_state,
                                pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
         if (adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_MGCG)) {
@@ -1307,7 +1309,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
                                PP_BLOCK_SYS_HDP,
                                pp_support_state,
                                pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
 
@@ -1321,7 +1324,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
                                PP_BLOCK_SYS_BIF,
                                PP_STATE_SUPPORT_LS,
                                 pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
         if (adev->cg_flags & AMD_CG_SUPPORT_BIF_MGCG) {
                 if (state == AMD_CG_STATE_UNGATE)
@@ -1333,7 +1337,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
                                PP_BLOCK_SYS_BIF,
                                PP_STATE_SUPPORT_CG,
                                pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
         if (adev->cg_flags & AMD_CG_SUPPORT_DRM_LS) {
@@ -1347,7 +1352,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
                                PP_BLOCK_SYS_DRM,
                                PP_STATE_SUPPORT_LS,
                                pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
 
         if (adev->cg_flags & AMD_CG_SUPPORT_ROM_MGCG) {
@@ -1361,7 +1367,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
                                PP_BLOCK_SYS_ROM,
                                PP_STATE_SUPPORT_CG,
                                pp_state);
-                amd_set_clockgating_by_smu(pp_handle, msg_id);
+                if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
+                        amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
         }
         return 0;
 }
@@ -1496,6 +1503,10 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
                 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+                else if (amdgpu_device_has_dc_support(adev))
+                        amdgpu_ip_block_add(adev, &dm_ip_block);
+#endif
                 else
                         amdgpu_ip_block_add(adev, &dce_v10_1_ip_block);
                 amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
@@ -1512,6 +1523,10 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
                 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+                else if (amdgpu_device_has_dc_support(adev))
+                        amdgpu_ip_block_add(adev, &dm_ip_block);
+#endif
                 else
                         amdgpu_ip_block_add(adev, &dce_v10_0_ip_block);
                 amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
@@ -1530,6 +1545,10 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
                 if (adev->enable_virtual_display)
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+                else if (amdgpu_device_has_dc_support(adev))
+                        amdgpu_ip_block_add(adev, &dm_ip_block);
+#endif
                 else
                         amdgpu_ip_block_add(adev, &dce_v11_2_ip_block);
                 amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
@@ -1544,6 +1563,10 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
                 if (adev->enable_virtual_display)
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+                else if (amdgpu_device_has_dc_support(adev))
+                        amdgpu_ip_block_add(adev, &dm_ip_block);
+#endif
                 else
                         amdgpu_ip_block_add(adev, &dce_v11_0_ip_block);
                 amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
@@ -1561,6 +1584,10 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
                 if (adev->enable_virtual_display)
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+                else if (amdgpu_device_has_dc_support(adev))
+                        amdgpu_ip_block_add(adev, &dm_ip_block);
+#endif
                 else
                         amdgpu_ip_block_add(adev, &dce_v11_0_ip_block);
                 amdgpu_ip_block_add(adev, &gfx_v8_1_ip_block);
diff --git a/drivers/gpu/drm/amd/amdgpu/vid.h b/drivers/gpu/drm/amd/amdgpu/vid.h
index a6485254a169..dbf3703cbd1b 100644
--- a/drivers/gpu/drm/amd/amdgpu/vid.h
+++ b/drivers/gpu/drm/amd/amdgpu/vid.h
@@ -465,6 +465,16 @@
 #define VCE_CMD_UPDATE_PTB      0x00000107
 #define VCE_CMD_FLUSH_TLB       0x00000108
 
+/* HEVC ENC */
+#define HEVC_ENC_CMD_NO_OP         0x00000000
+#define HEVC_ENC_CMD_END           0x00000001
+#define HEVC_ENC_CMD_FENCE         0x00000003
+#define HEVC_ENC_CMD_TRAP          0x00000004
+#define HEVC_ENC_CMD_IB_VM         0x00000102
+#define HEVC_ENC_CMD_WAIT_GE       0x00000106
+#define HEVC_ENC_CMD_UPDATE_PTB    0x00000107
+#define HEVC_ENC_CMD_FLUSH_TLB     0x00000108
+
 /* mmPA_SC_RASTER_CONFIG mask */
 #define RB_MAP_PKR0(x)                          ((x) << 0)
 #define RB_MAP_PKR0_MASK                        (0x3 << 0)