Merge branch 'drm-next-4.11' of git://people.freedesktop.org/~agd5f/linux into drm-next

This is the main feature pull for radeon and amdgpu for 4.11.  Highlights:
- Power and clockgating improvements
- Preliminary SR-IOV support
- ttm buffer priority support
- ttm eviction fixes
- Removal of the ttm lru callbacks
- Remove SI DPM quirks due to MC firmware issues
- Handle VFCT with multiple vbioses
- Powerplay improvements
- Lots of driver cleanups

* 'drm-next-4.11' of git://people.freedesktop.org/~agd5f/linux: (120 commits)
  drm/amdgpu: fix amdgpu_bo_va_mapping flags
  drm/amdgpu: access stolen VRAM directly on CZ (v2)
  drm/amdgpu: access stolen VRAM directly on KV/KB (v2)
  drm/amdgpu: fix kernel panic when dpm disabled on Kv.
  drm/amdgpu: fix dpm bug on Kv.
  drm/amd/powerplay: fix regresstion issue can't set manual dpm mode.
  drm/amdgpu: handle vfct with multiple vbios images
  drm/radeon: handle vfct with multiple vbios images
  drm/amdgpu: move misc si headers into amdgpu
  drm/amdgpu: remove unused header si_reg.h
  drm/radeon: drop pitcairn dpm quirks
  drm/amdgpu: drop pitcairn dpm quirks
  drm: radeon: radeon_ttm: Handle return NULL error from ioremap_nocache
  drm/amd/amdgpu/amdgpu_ttm: Handle return NULL error from ioremap_nocache
  drm/amdgpu: add new virtual display ID
  drm/amd/amdgpu: remove the uncessary parameter for ib scheduler
  drm/amdgpu: Bring bo creation in line with radeon driver (v2)
  drm/amd/powerplay: fix misspelling in header guard
  drm/ttm: revert "add optional LRU removal callback v2"
  drm/ttm: revert "implement LRU add callbacks v2"
  ...

134 files changed, 4838 insertions, 6122 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 41bd2bf28f4c..2814aad81752 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -24,7 +24,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
         atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
         amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
         amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-        amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+        amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
@@ -34,7 +34,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
 amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
 
 amdgpu-y += \
-        vi.o
+        vi.o mxgpu_vi.o
 
 # add GMC block
 amdgpu-y += \
@@ -52,8 +52,7 @@ amdgpu-y += \
 # add SMC block
 amdgpu-y += \
         amdgpu_dpm.o \
-        amdgpu_powerplay.o \
-        cz_smc.o cz_dpm.o
+        amdgpu_powerplay.o
 
 # add DCE block
 amdgpu-y += \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 73863d465f98..94a64e3bc682 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -91,7 +91,6 @@ extern int amdgpu_vm_fault_stop;
 extern int amdgpu_vm_debug;
 extern int amdgpu_sched_jobs;
 extern int amdgpu_sched_hw_submission;
-extern int amdgpu_powerplay;
 extern int amdgpu_no_evict;
 extern int amdgpu_direct_gma_size;
 extern unsigned amdgpu_pcie_gen_cap;
@@ -184,12 +183,18 @@ enum amdgpu_thermal_irq {
         AMDGPU_THERMAL_IRQ_LAST
 };
 
+enum amdgpu_kiq_irq {
+        AMDGPU_CP_KIQ_IRQ_DRIVER0 = 0,
+        AMDGPU_CP_KIQ_IRQ_LAST
+};
+
 int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
                                   enum amd_ip_block_type block_type,
                                   enum amd_clockgating_state state);
 int amdgpu_set_powergating_state(struct amdgpu_device *adev,
                                   enum amd_ip_block_type block_type,
                                   enum amd_powergating_state state);
+void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags);
 int amdgpu_wait_for_idle(struct amdgpu_device *adev,
                          enum amd_ip_block_type block_type);
 bool amdgpu_is_idle(struct amdgpu_device *adev,
@@ -352,7 +357,7 @@ struct amdgpu_bo_va_mapping {
         struct list_head                list;
         struct interval_tree_node       it;
         uint64_t                        offset;
-        uint32_t                        flags;
+        uint64_t                        flags;
 };
 
 /* bo virtual addresses in a specific vm */
@@ -776,14 +781,20 @@ struct amdgpu_mec {
         u32 num_queue;
 };
 
+struct amdgpu_kiq {
+        u64                     eop_gpu_addr;
+        struct amdgpu_bo        *eop_obj;
+        struct amdgpu_ring      ring;
+        struct amdgpu_irq_src   irq;
+};
+
 /*
  * GPU scratch registers structures, functions & helpers
  */
 struct amdgpu_scratch {
         unsigned                num_reg;
         uint32_t                reg_base;
-        bool                    free[32];
-        uint32_t                reg[32];
+        uint32_t                free_mask;
 };
 
 /*
@@ -851,6 +862,7 @@ struct amdgpu_gfx {
         struct amdgpu_gca_config        config;
         struct amdgpu_rlc               rlc;
         struct amdgpu_mec               mec;
+        struct amdgpu_kiq               kiq;
         struct amdgpu_scratch           scratch;
         const struct firmware           *me_fw; /* ME firmware */
         uint32_t                        me_fw_version;
@@ -894,8 +906,8 @@ int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib,
                     struct dma_fence *f);
 int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
-                       struct amdgpu_ib *ib, struct dma_fence *last_vm_update,
-                       struct amdgpu_job *job, struct dma_fence **f);
+                       struct amdgpu_ib *ibs, struct amdgpu_job *job,
+                       struct dma_fence **f);
 int amdgpu_ib_pool_init(struct amdgpu_device *adev);
 void amdgpu_ib_pool_fini(struct amdgpu_device *adev);
 int amdgpu_ib_ring_tests(struct amdgpu_device *adev);
@@ -938,6 +950,7 @@ struct amdgpu_cs_parser {
 #define AMDGPU_PREAMBLE_IB_PRESENT          (1 << 0) /* bit set means command submit involves a preamble IB */
 #define AMDGPU_PREAMBLE_IB_PRESENT_FIRST    (1 << 1) /* bit set means preamble IB is first presented in belonging context */
 #define AMDGPU_HAVE_CTX_SWITCH              (1 << 2) /* bit set means context switch occured */
+#define AMDGPU_VM_DOMAIN                    (1 << 3) /* bit set means in virtual memory context */
 
 struct amdgpu_job {
         struct amd_sched_job    base;
@@ -1024,6 +1037,7 @@ struct amdgpu_uvd {
         bool                    use_ctx_buf;
         struct amd_sched_entity entity;
         uint32_t                srbm_soft_reset;
+        bool                    is_powergated;
 };
 
 /*
@@ -1052,6 +1066,7 @@ struct amdgpu_vce {
         struct amd_sched_entity entity;
         uint32_t                srbm_soft_reset;
         unsigned                num_rings;
+        bool                    is_powergated;
 };
 
 /*
@@ -1177,7 +1192,6 @@ struct amdgpu_asic_funcs {
         bool (*read_disabled_bios)(struct amdgpu_device *adev);
         bool (*read_bios_from_rom)(struct amdgpu_device *adev,
                                    u8 *bios, u32 length_bytes);
-        void (*detect_hw_virtualization) (struct amdgpu_device *adev);
         int (*read_register)(struct amdgpu_device *adev, u32 se_num,
                              u32 sh_num, u32 reg_offset, u32 *value);
         void (*set_vga_state)(struct amdgpu_device *adev, bool state);
@@ -1332,7 +1346,6 @@ struct amdgpu_device {
         /* BIOS */
         uint8_t                         *bios;
         uint32_t                        bios_size;
-        bool                            is_atom_bios;
         struct amdgpu_bo                *stollen_vga_memory;
         uint32_t                        bios_scratch[AMDGPU_BIOS_NUM_SCRATCH];
 
@@ -1462,7 +1475,7 @@ struct amdgpu_device {
         /* amdkfd interface */
         struct kfd_dev          *kfd;
 
-        struct amdgpu_virtualization virtualization;
+        struct amdgpu_virt      virt;
 
         /* link all shadow bo */
         struct list_head                shadow_list;
@@ -1575,6 +1588,37 @@ static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v)
         ring->count_dw--;
 }
 
+static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring, void *src, int count_dw)
+{
+        unsigned occupied, chunk1, chunk2;
+        void *dst;
+
+        if (ring->count_dw < count_dw) {
+                DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
+        } else {
+                occupied = ring->wptr & ring->ptr_mask;
+                dst = (void *)&ring->ring[occupied];
+                chunk1 = ring->ptr_mask + 1 - occupied;
+                chunk1 = (chunk1 >= count_dw) ? count_dw: chunk1;
+                chunk2 = count_dw - chunk1;
+                chunk1 <<= 2;
+                chunk2 <<= 2;
+
+                if (chunk1)
+                        memcpy(dst, src, chunk1);
+
+                if (chunk2) {
+                        src += chunk1;
+                        dst = (void *)ring->ring;
+                        memcpy(dst, src, chunk2);
+                }
+
+                ring->wptr += count_dw;
+                ring->wptr &= ring->ptr_mask;
+                ring->count_dw -= count_dw;
+        }
+}
+
 static inline struct amdgpu_sdma_instance *
 amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 {
@@ -1604,7 +1648,6 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_asic_get_gpu_clock_counter(adev) (adev)->asic_funcs->get_gpu_clock_counter((adev))
 #define amdgpu_asic_read_disabled_bios(adev) (adev)->asic_funcs->read_disabled_bios((adev))
 #define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
-#define amdgpu_asic_detect_hw_virtualization(adev) (adev)->asic_funcs->detect_hw_virtualization((adev))
 #define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v)))
 #define amdgpu_gart_flush_gpu_tlb(adev, vmid) (adev)->gart.gart_funcs->flush_gpu_tlb((adev), (vmid))
 #define amdgpu_gart_set_pte_pde(adev, pt, idx, addr, flags) (adev)->gart.gart_funcs->set_pte_pde((adev), (pt), (idx), (addr), (flags))
@@ -1626,6 +1669,8 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_ring_emit_hdp_invalidate(r) (r)->funcs->emit_hdp_invalidate((r))
 #define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r))
 #define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))
+#define amdgpu_ring_emit_rreg(r, d) (r)->funcs->emit_rreg((r), (d))
+#define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v))
 #define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
 #define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
 #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
index 5796539a0bcb..ef79551b4cb7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
@@ -672,12 +672,10 @@ int amdgpu_acpi_init(struct amdgpu_device *adev)
 
                         if ((enc->devices & (ATOM_DEVICE_LCD_SUPPORT)) &&
                             enc->enc_priv) {
-                                if (adev->is_atom_bios) {
-                                        struct amdgpu_encoder_atom_dig *dig = enc->enc_priv;
-                                        if (dig->bl_dev) {
-                                                atif->encoder_for_bl = enc;
-                                                break;
-                                        }
+                                struct amdgpu_encoder_atom_dig *dig = enc->enc_priv;
+                                if (dig->bl_dev) {
+                                        atif->encoder_for_bl = enc;
+                                        break;
                                 }
                         }
                 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
index 8ec1967a850b..d9def01f276e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
@@ -42,6 +42,51 @@
 #define AMD_IS_VALID_VBIOS(p) ((p)[0] == 0x55 && (p)[1] == 0xAA)
 #define AMD_VBIOS_LENGTH(p) ((p)[2] << 9)
 
+/* Check if current bios is an ATOM BIOS.
+ * Return true if it is ATOM BIOS. Otherwise, return false.
+ */
+static bool check_atom_bios(uint8_t *bios, size_t size)
+{
+        uint16_t tmp, bios_header_start;
+
+        if (!bios || size < 0x49) {
+                DRM_INFO("vbios mem is null or mem size is wrong\n");
+                return false;
+        }
+
+        if (!AMD_IS_VALID_VBIOS(bios)) {
+                DRM_INFO("BIOS signature incorrect %x %x\n", bios[0], bios[1]);
+                return false;
+        }
+
+        tmp = bios[0x18] | (bios[0x19] << 8);
+        if (bios[tmp + 0x14] != 0x0) {
+                DRM_INFO("Not an x86 BIOS ROM\n");
+                return false;
+        }
+
+        bios_header_start = bios[0x48] | (bios[0x49] << 8);
+        if (!bios_header_start) {
+                DRM_INFO("Can't locate bios header\n");
+                return false;
+        }
+
+        tmp = bios_header_start + 4;
+        if (size < tmp) {
+                DRM_INFO("BIOS header is broken\n");
+                return false;
+        }
+
+        if (!memcmp(bios + tmp, "ATOM", 4) ||
+            !memcmp(bios + tmp, "MOTA", 4)) {
+                DRM_DEBUG("ATOMBIOS detected\n");
+                return true;
+        }
+
+        return false;
+}
+
+
 /* If you boot an IGP board with a discrete card as the primary,
  * the IGP rom is not accessible via the rom bar as the IGP rom is
  * part of the system bios.  On boot, the system bios puts a
@@ -65,10 +110,6 @@ static bool igp_read_bios_from_vram(struct amdgpu_device *adev)
                 return false;
         }
 
-        if (size == 0 || !AMD_IS_VALID_VBIOS(bios)) {
-                iounmap(bios);
-                return false;
-        }
         adev->bios = kmalloc(size, GFP_KERNEL);
         if (!adev->bios) {
                 iounmap(bios);
@@ -77,12 +118,18 @@ static bool igp_read_bios_from_vram(struct amdgpu_device *adev)
         adev->bios_size = size;
         memcpy_fromio(adev->bios, bios, size);
         iounmap(bios);
+
+        if (!check_atom_bios(adev->bios, size)) {
+                kfree(adev->bios);
+                return false;
+        }
+
         return true;
 }
 
 bool amdgpu_read_bios(struct amdgpu_device *adev)
 {
-        uint8_t __iomem *bios, val[2];
+        uint8_t __iomem *bios;
         size_t size;
 
         adev->bios = NULL;
@@ -92,13 +139,6 @@ bool amdgpu_read_bios(struct amdgpu_device *adev)
                 return false;
         }
 
-        val[0] = readb(&bios[0]);
-        val[1] = readb(&bios[1]);
-
-        if (size == 0 || !AMD_IS_VALID_VBIOS(val)) {
-                pci_unmap_rom(adev->pdev, bios);
-                return false;
-        }
         adev->bios = kzalloc(size, GFP_KERNEL);
         if (adev->bios == NULL) {
                 pci_unmap_rom(adev->pdev, bios);
@@ -107,6 +147,12 @@ bool amdgpu_read_bios(struct amdgpu_device *adev)
         adev->bios_size = size;
         memcpy_fromio(adev->bios, bios, size);
         pci_unmap_rom(adev->pdev, bios);
+
+        if (!check_atom_bios(adev->bios, size)) {
+                kfree(adev->bios);
+                return false;
+        }
+
         return true;
 }
 
@@ -140,7 +186,14 @@ static bool amdgpu_read_bios_from_rom(struct amdgpu_device *adev)
         adev->bios_size = len;
 
         /* read complete BIOS */
-        return amdgpu_asic_read_bios_from_rom(adev, adev->bios, len);
+        amdgpu_asic_read_bios_from_rom(adev, adev->bios, len);
+
+        if (!check_atom_bios(adev->bios, len)) {
+                kfree(adev->bios);
+                return false;
+        }
+
+        return true;
 }
 
 static bool amdgpu_read_platform_bios(struct amdgpu_device *adev)
@@ -155,13 +208,17 @@ static bool amdgpu_read_platform_bios(struct amdgpu_device *adev)
                 return false;
         }
 
-        if (size == 0 || !AMD_IS_VALID_VBIOS(bios)) {
+        adev->bios = kzalloc(size, GFP_KERNEL);
+        if (adev->bios == NULL)
                 return false;
-        }
-        adev->bios = kmemdup(bios, size, GFP_KERNEL);
-        if (adev->bios == NULL) {
+
+        memcpy_fromio(adev->bios, bios, size);
+
+        if (!check_atom_bios(adev->bios, size)) {
+                kfree(adev->bios);
                 return false;
         }
+
         adev->bios_size = size;
 
         return true;
@@ -273,7 +330,7 @@ static bool amdgpu_atrm_get_bios(struct amdgpu_device *adev)
                         break;
         }
 
-        if (i == 0 || !AMD_IS_VALID_VBIOS(adev->bios)) {
+        if (!check_atom_bios(adev->bios, size)) {
                 kfree(adev->bios);
                 return false;
         }
@@ -298,53 +355,59 @@ static bool amdgpu_read_disabled_bios(struct amdgpu_device *adev)
 #ifdef CONFIG_ACPI
 static bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev)
 {
-        bool ret = false;
         struct acpi_table_header *hdr;
         acpi_size tbl_size;
         UEFI_ACPI_VFCT *vfct;
-        GOP_VBIOS_CONTENT *vbios;
-        VFCT_IMAGE_HEADER *vhdr;
+        unsigned offset;
 
         if (!ACPI_SUCCESS(acpi_get_table("VFCT", 1, &hdr)))
                 return false;
         tbl_size = hdr->length;
         if (tbl_size < sizeof(UEFI_ACPI_VFCT)) {
                 DRM_ERROR("ACPI VFCT table present but broken (too short #1)\n");
-                goto out_unmap;
+                return false;
         }
 
         vfct = (UEFI_ACPI_VFCT *)hdr;
-        if (vfct->VBIOSImageOffset + sizeof(VFCT_IMAGE_HEADER) > tbl_size) {
-                DRM_ERROR("ACPI VFCT table present but broken (too short #2)\n");
-                goto out_unmap;
-        }
+        offset = vfct->VBIOSImageOffset;
 
-        vbios = (GOP_VBIOS_CONTENT *)((char *)hdr + vfct->VBIOSImageOffset);
-        vhdr = &vbios->VbiosHeader;
-        DRM_INFO("ACPI VFCT contains a BIOS for %02x:%02x.%d %04x:%04x, size %d\n",
-                        vhdr->PCIBus, vhdr->PCIDevice, vhdr->PCIFunction,
-                        vhdr->VendorID, vhdr->DeviceID, vhdr->ImageLength);
-
-        if (vhdr->PCIBus != adev->pdev->bus->number ||
-            vhdr->PCIDevice != PCI_SLOT(adev->pdev->devfn) ||
-            vhdr->PCIFunction != PCI_FUNC(adev->pdev->devfn) ||
-            vhdr->VendorID != adev->pdev->vendor ||
-            vhdr->DeviceID != adev->pdev->device) {
-                DRM_INFO("ACPI VFCT table is not for this card\n");
-                goto out_unmap;
-        }
+        while (offset < tbl_size) {
+                GOP_VBIOS_CONTENT *vbios = (GOP_VBIOS_CONTENT *)((char *)hdr + offset);
+                VFCT_IMAGE_HEADER *vhdr = &vbios->VbiosHeader;
 
-        if (vfct->VBIOSImageOffset + sizeof(VFCT_IMAGE_HEADER) + vhdr->ImageLength > tbl_size) {
-                DRM_ERROR("ACPI VFCT image truncated\n");
-                goto out_unmap;
-        }
+                offset += sizeof(VFCT_IMAGE_HEADER);
+                if (offset > tbl_size) {
+                        DRM_ERROR("ACPI VFCT image header truncated\n");
+                        return false;
+                }
+
+                offset += vhdr->ImageLength;
+                if (offset > tbl_size) {
+                        DRM_ERROR("ACPI VFCT image truncated\n");
+                        return false;
+                }
 
-        adev->bios = kmemdup(&vbios->VbiosContent, vhdr->ImageLength, GFP_KERNEL);
-        adev->bios_size = vhdr->ImageLength;
-        ret = !!adev->bios;
+                if (vhdr->ImageLength &&
+                    vhdr->PCIBus == adev->pdev->bus->number &&
+                    vhdr->PCIDevice == PCI_SLOT(adev->pdev->devfn) &&
+                    vhdr->PCIFunction == PCI_FUNC(adev->pdev->devfn) &&
+                    vhdr->VendorID == adev->pdev->vendor &&
+                    vhdr->DeviceID == adev->pdev->device) {
+                        adev->bios = kmemdup(&vbios->VbiosContent,
+                                             vhdr->ImageLength,
+                                             GFP_KERNEL);
+
+                        if (!check_atom_bios(adev->bios, vhdr->ImageLength)) {
+                                kfree(adev->bios);
+                                return false;
+                        }
+                        adev->bios_size = vhdr->ImageLength;
+                        return true;
+                }
+        }
 
-out_unmap:
-        return ret;
+        DRM_ERROR("ACPI VFCT table present but broken (too short #2)\n");
+        return false;
 }
 #else
 static inline bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev)
@@ -355,57 +418,27 @@ static inline bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev)
 
 bool amdgpu_get_bios(struct amdgpu_device *adev)
 {
-        bool r;
-        uint16_t tmp, bios_header_start;
+        if (amdgpu_atrm_get_bios(adev))
+                return true;
 
-        r = amdgpu_atrm_get_bios(adev);
-        if (!r)
-                r = amdgpu_acpi_vfct_bios(adev);
-        if (!r)
-                r = igp_read_bios_from_vram(adev);
-        if (!r)
-                r = amdgpu_read_bios(adev);
-        if (!r) {
-                r = amdgpu_read_bios_from_rom(adev);
-        }
-        if (!r) {
-                r = amdgpu_read_disabled_bios(adev);
-        }
-        if (!r) {
-                r = amdgpu_read_platform_bios(adev);
-        }
-        if (!r || adev->bios == NULL) {
-                DRM_ERROR("Unable to locate a BIOS ROM\n");
-                adev->bios = NULL;
-                return false;
-        }
-        if (!AMD_IS_VALID_VBIOS(adev->bios)) {
-                printk("BIOS signature incorrect %x %x\n", adev->bios[0], adev->bios[1]);
-                goto free_bios;
-        }
+        if (amdgpu_acpi_vfct_bios(adev))
+                return true;
 
-        tmp = RBIOS16(0x18);
-        if (RBIOS8(tmp + 0x14) != 0x0) {
-                DRM_INFO("Not an x86 BIOS ROM, not using.\n");
-                goto free_bios;
-        }
+        if (igp_read_bios_from_vram(adev))
+                return true;
 
-        bios_header_start = RBIOS16(0x48);
-        if (!bios_header_start) {
-                goto free_bios;
-        }
-        tmp = bios_header_start + 4;
-        if (!memcmp(adev->bios + tmp, "ATOM", 4) ||
-            !memcmp(adev->bios + tmp, "MOTA", 4)) {
-                adev->is_atom_bios = true;
-        } else {
-                adev->is_atom_bios = false;
-        }
+        if (amdgpu_read_bios(adev))
+                return true;
 
-        DRM_DEBUG("%sBIOS detected\n", adev->is_atom_bios ? "ATOM" : "COM");
-        return true;
-free_bios:
-        kfree(adev->bios);
-        adev->bios = NULL;
+        if (amdgpu_read_bios_from_rom(adev))
+                return true;
+
+        if (amdgpu_read_disabled_bios(adev))
+                return true;
+
+        if (amdgpu_read_platform_bios(adev))
+                return true;
+
+        DRM_ERROR("Unable to locate a BIOS ROM\n");
         return false;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index 4c851fde1e82..a5df1ef306d9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -713,6 +713,7 @@ static int amdgpu_cgs_rel_firmware(struct cgs_device *cgs_device, enum cgs_ucode
         CGS_FUNC_ADEV;
         if ((CGS_UCODE_ID_SMU == type) || (CGS_UCODE_ID_SMU_SK == type)) {
                 release_firmware(adev->pm.fw);
+                adev->pm.fw = NULL;
                 return 0;
         }
         /* cannot release other firmware because they are not created by cgs */
@@ -762,6 +763,23 @@ static uint16_t amdgpu_get_firmware_version(struct cgs_device *cgs_device,
         return fw_version;
 }
 
+static int amdgpu_cgs_enter_safe_mode(struct cgs_device *cgs_device,
+                                        bool en)
+{
+        CGS_FUNC_ADEV;
+
+        if (adev->gfx.rlc.funcs->enter_safe_mode == NULL ||
+                adev->gfx.rlc.funcs->exit_safe_mode == NULL)
+                return 0;
+
+        if (en)
+                adev->gfx.rlc.funcs->enter_safe_mode(adev);
+        else
+                adev->gfx.rlc.funcs->exit_safe_mode(adev);
+
+        return 0;
+}
+
 static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
                                         enum cgs_ucode_id type,
                                         struct cgs_firmware_info *info)
@@ -808,6 +826,9 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
                 const uint8_t *src;
                 const struct smc_firmware_header_v1_0 *hdr;
 
+                if (CGS_UCODE_ID_SMU_SK == type)
+                        amdgpu_cgs_rel_firmware(cgs_device, CGS_UCODE_ID_SMU);
+
                 if (!adev->pm.fw) {
                         switch (adev->asic_type) {
                         case CHIP_TOPAZ:
@@ -1200,51 +1221,52 @@ static int amdgpu_cgs_call_acpi_method(struct cgs_device *cgs_device,
 }
 
 static const struct cgs_ops amdgpu_cgs_ops = {
-        amdgpu_cgs_gpu_mem_info,
-        amdgpu_cgs_gmap_kmem,
-        amdgpu_cgs_gunmap_kmem,
-        amdgpu_cgs_alloc_gpu_mem,
-        amdgpu_cgs_free_gpu_mem,
-        amdgpu_cgs_gmap_gpu_mem,
-        amdgpu_cgs_gunmap_gpu_mem,
-        amdgpu_cgs_kmap_gpu_mem,
-        amdgpu_cgs_kunmap_gpu_mem,
-        amdgpu_cgs_read_register,
-        amdgpu_cgs_write_register,
-        amdgpu_cgs_read_ind_register,
-        amdgpu_cgs_write_ind_register,
-        amdgpu_cgs_read_pci_config_byte,
-        amdgpu_cgs_read_pci_config_word,
-        amdgpu_cgs_read_pci_config_dword,
-        amdgpu_cgs_write_pci_config_byte,
-        amdgpu_cgs_write_pci_config_word,
-        amdgpu_cgs_write_pci_config_dword,
-        amdgpu_cgs_get_pci_resource,
-        amdgpu_cgs_atom_get_data_table,
-        amdgpu_cgs_atom_get_cmd_table_revs,
-        amdgpu_cgs_atom_exec_cmd_table,
-        amdgpu_cgs_create_pm_request,
-        amdgpu_cgs_destroy_pm_request,
-        amdgpu_cgs_set_pm_request,
-        amdgpu_cgs_pm_request_clock,
-        amdgpu_cgs_pm_request_engine,
-        amdgpu_cgs_pm_query_clock_limits,
-        amdgpu_cgs_set_camera_voltages,
-        amdgpu_cgs_get_firmware_info,
-        amdgpu_cgs_rel_firmware,
-        amdgpu_cgs_set_powergating_state,
-        amdgpu_cgs_set_clockgating_state,
-        amdgpu_cgs_get_active_displays_info,
-        amdgpu_cgs_notify_dpm_enabled,
-        amdgpu_cgs_call_acpi_method,
-        amdgpu_cgs_query_system_info,
-        amdgpu_cgs_is_virtualization_enabled
+        .gpu_mem_info = amdgpu_cgs_gpu_mem_info,
+        .gmap_kmem = amdgpu_cgs_gmap_kmem,
+        .gunmap_kmem = amdgpu_cgs_gunmap_kmem,
+        .alloc_gpu_mem = amdgpu_cgs_alloc_gpu_mem,
+        .free_gpu_mem = amdgpu_cgs_free_gpu_mem,
+        .gmap_gpu_mem = amdgpu_cgs_gmap_gpu_mem,
+        .gunmap_gpu_mem = amdgpu_cgs_gunmap_gpu_mem,
+        .kmap_gpu_mem = amdgpu_cgs_kmap_gpu_mem,
+        .kunmap_gpu_mem = amdgpu_cgs_kunmap_gpu_mem,
+        .read_register = amdgpu_cgs_read_register,
+        .write_register = amdgpu_cgs_write_register,
+        .read_ind_register = amdgpu_cgs_read_ind_register,
+        .write_ind_register = amdgpu_cgs_write_ind_register,
+        .read_pci_config_byte = amdgpu_cgs_read_pci_config_byte,
+        .read_pci_config_word = amdgpu_cgs_read_pci_config_word,
+        .read_pci_config_dword = amdgpu_cgs_read_pci_config_dword,
+        .write_pci_config_byte = amdgpu_cgs_write_pci_config_byte,
+        .write_pci_config_word = amdgpu_cgs_write_pci_config_word,
+        .write_pci_config_dword = amdgpu_cgs_write_pci_config_dword,
+        .get_pci_resource = amdgpu_cgs_get_pci_resource,
+        .atom_get_data_table = amdgpu_cgs_atom_get_data_table,
+        .atom_get_cmd_table_revs = amdgpu_cgs_atom_get_cmd_table_revs,
+        .atom_exec_cmd_table = amdgpu_cgs_atom_exec_cmd_table,
+        .create_pm_request = amdgpu_cgs_create_pm_request,
+        .destroy_pm_request = amdgpu_cgs_destroy_pm_request,
+        .set_pm_request = amdgpu_cgs_set_pm_request,
+        .pm_request_clock = amdgpu_cgs_pm_request_clock,
+        .pm_request_engine = amdgpu_cgs_pm_request_engine,
+        .pm_query_clock_limits = amdgpu_cgs_pm_query_clock_limits,
+        .set_camera_voltages = amdgpu_cgs_set_camera_voltages,
+        .get_firmware_info = amdgpu_cgs_get_firmware_info,
+        .rel_firmware = amdgpu_cgs_rel_firmware,
+        .set_powergating_state = amdgpu_cgs_set_powergating_state,
+        .set_clockgating_state = amdgpu_cgs_set_clockgating_state,
+        .get_active_displays_info = amdgpu_cgs_get_active_displays_info,
+        .notify_dpm_enabled = amdgpu_cgs_notify_dpm_enabled,
+        .call_acpi_method = amdgpu_cgs_call_acpi_method,
+        .query_system_info = amdgpu_cgs_query_system_info,
+        .is_virtualization_enabled = amdgpu_cgs_is_virtualization_enabled,
+        .enter_safe_mode = amdgpu_cgs_enter_safe_mode,
 };
 
 static const struct cgs_os_ops amdgpu_cgs_os_ops = {
-        amdgpu_cgs_add_irq_source,
-        amdgpu_cgs_irq_get,
-        amdgpu_cgs_irq_put
+        .add_irq_source = amdgpu_cgs_add_irq_source,
+        .irq_get = amdgpu_cgs_irq_get,
+        .irq_put = amdgpu_cgs_irq_put
 };
 
 struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 29d6d84d1c28..cf2e8c4e9b8b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -75,10 +75,10 @@ int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
                 *out_ring = &adev->uvd.ring;
                 break;
         case AMDGPU_HW_IP_VCE:
-                if (ring < 2){
+                if (ring < adev->vce.num_rings){
                         *out_ring = &adev->vce.ring[ring];
                 } else {
-                        DRM_ERROR("only two VCE rings are supported\n");
+                        DRM_ERROR("only %d VCE rings are supported\n", adev->vce.num_rings);
                         return -EINVAL;
                 }
                 break;
@@ -771,6 +771,20 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
         if (r)
                 return r;
 
+        if (amdgpu_sriov_vf(adev)) {
+                struct dma_fence *f;
+                bo_va = vm->csa_bo_va;
+                BUG_ON(!bo_va);
+                r = amdgpu_vm_bo_update(adev, bo_va, false);
+                if (r)
+                        return r;
+
+                f = bo_va->last_pt_update;
+                r = amdgpu_sync_fence(adev, &p->job->sync, f);
+                if (r)
+                        return r;
+        }
+
         if (p->bo_list) {
                 for (i = 0; i < p->bo_list->num_entries; i++) {
                         struct dma_fence *f;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 2201303b9262..944ba0d3874a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -94,6 +94,11 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
 {
         uint32_t ret;
 
+        if (amdgpu_sriov_runtime(adev)) {
+                BUG_ON(in_interrupt());
+                return amdgpu_virt_kiq_rreg(adev, reg);
+        }
+
         if ((reg * 4) < adev->rmmio_size && !always_indirect)
                 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
         else {
@@ -113,6 +118,11 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 {
         trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+        if (amdgpu_sriov_runtime(adev)) {
+                BUG_ON(in_interrupt());
+                return amdgpu_virt_kiq_wreg(adev, reg, v);
+        }
+
         if ((reg * 4) < adev->rmmio_size && !always_indirect)
                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
         else {
@@ -885,7 +895,7 @@ static int amdgpu_atombios_init(struct amdgpu_device *adev)
                 atom_card_info->ioreg_read = cail_ioreg_read;
                 atom_card_info->ioreg_write = cail_ioreg_write;
         } else {
-                DRM_ERROR("Unable to find PCI I/O BAR; using MMIO for ATOM IIO\n");
+                DRM_INFO("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
                 atom_card_info->ioreg_read = cail_reg_read;
                 atom_card_info->ioreg_write = cail_reg_write;
         }
@@ -1131,6 +1141,18 @@ int amdgpu_set_powergating_state(struct amdgpu_device *adev,
         return r;
 }
 
+void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
+{
+        int i;
+
+        for (i = 0; i < adev->num_ip_blocks; i++) {
+                if (!adev->ip_blocks[i].status.valid)
+                        continue;
+                if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
+                        adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
+        }
+}
+
 int amdgpu_wait_for_idle(struct amdgpu_device *adev,
                          enum amd_ip_block_type block_type)
 {
@@ -1235,7 +1257,8 @@ static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
                 pciaddstr_tmp = pciaddstr;
                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
                         pciaddname = strsep(&pciaddname_tmp, ",");
-                        if (!strcmp(pci_address_name, pciaddname)) {
+                        if (!strcmp("all", pciaddname)
+                            || !strcmp(pci_address_name, pciaddname)) {
                                 long num_crtc;
                                 int res = -1;
 
@@ -1323,6 +1346,12 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
                 return -EINVAL;
         }
 
+        if (amdgpu_sriov_vf(adev)) {
+                r = amdgpu_virt_request_full_gpu(adev, true);
+                if (r)
+                        return r;
+        }
+
         for (i = 0; i < adev->num_ip_blocks; i++) {
                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
                         DRM_ERROR("disabled ip block: %d\n", i);
@@ -1383,6 +1412,15 @@ static int amdgpu_init(struct amdgpu_device *adev)
                                 return r;
                         }
                         adev->ip_blocks[i].status.hw = true;
+
+                        /* right after GMC hw init, we create CSA */
+                        if (amdgpu_sriov_vf(adev)) {
+                                r = amdgpu_allocate_static_csa(adev);
+                                if (r) {
+                                        DRM_ERROR("allocate CSA failed %d\n", r);
+                                        return r;
+                                }
+                        }
                 }
         }
 
@@ -1516,6 +1554,11 @@ static int amdgpu_fini(struct amdgpu_device *adev)
                 adev->ip_blocks[i].status.late_initialized = false;
         }
 
+        if (amdgpu_sriov_vf(adev)) {
+                amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
+                amdgpu_virt_release_full_gpu(adev, false);
+        }
+
         return 0;
 }
 
@@ -1523,6 +1566,9 @@ int amdgpu_suspend(struct amdgpu_device *adev)
 {
         int i, r;
 
+        if (amdgpu_sriov_vf(adev))
+                amdgpu_virt_request_full_gpu(adev, false);
+
         /* ungate SMC block first */
         r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
                                          AMD_CG_STATE_UNGATE);
@@ -1551,6 +1597,9 @@ int amdgpu_suspend(struct amdgpu_device *adev)
                 }
         }
 
+        if (amdgpu_sriov_vf(adev))
+                amdgpu_virt_release_full_gpu(adev, false);
+
         return 0;
 }
 
@@ -1575,7 +1624,7 @@ static int amdgpu_resume(struct amdgpu_device *adev)
 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
 {
         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
-                adev->virtualization.virtual_caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
+                adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
 }
 
 /**
@@ -1605,7 +1654,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         adev->pdev = pdev;
         adev->flags = flags;
         adev->asic_type = flags & AMD_ASIC_MASK;
-        adev->is_atom_bios = false;
         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
         adev->mc.gtt_size = 512 * 1024 * 1024;
         adev->accel_working = false;
@@ -1695,7 +1743,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
                 }
         }
         if (adev->rio_mem == NULL)
-                DRM_ERROR("Unable to find PCI I/O BAR\n");
+                DRM_INFO("PCI I/O BAR is not found.\n");
 
         /* early init functions */
         r = amdgpu_early_init(adev);
@@ -1720,12 +1768,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
                 r = -EINVAL;
                 goto failed;
         }
-        /* Must be an ATOMBIOS */
-        if (!adev->is_atom_bios) {
-                dev_err(adev->dev, "Expecting atombios for GPU\n");
-                r = -EINVAL;
-                goto failed;
-        }
+
         r = amdgpu_atombios_init(adev);
         if (r) {
                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
@@ -2249,6 +2292,9 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
         int resched;
         bool need_full_reset;
 
+        if (amdgpu_sriov_vf(adev))
+                return 0;
+
         if (!amdgpu_check_soft_reset(adev)) {
                 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
                 return 0;
@@ -2837,7 +2883,7 @@ static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
                 return -ENOMEM;
 
         /* version, increment each time something is added */
-        config[no_regs++] = 2;
+        config[no_regs++] = 3;
         config[no_regs++] = adev->gfx.config.max_shader_engines;
         config[no_regs++] = adev->gfx.config.max_tile_pipes;
         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
@@ -2871,6 +2917,12 @@ static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
         config[no_regs++] = adev->family;
         config[no_regs++] = adev->external_rev_id;
 
+        /* rev==3 */
+        config[no_regs++] = adev->pdev->device;
+        config[no_regs++] = adev->pdev->revision;
+        config[no_regs++] = adev->pdev->subsystem_device;
+        config[no_regs++] = adev->pdev->subsystem_vendor;
+
         while (size && (*pos < no_regs * 4)) {
                 uint32_t value;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index d2036df145b3..39fc388f222a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -138,10 +138,52 @@ static void amdgpu_unpin_work_func(struct work_struct *__work)
         kfree(work);
 }
 
-int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
-                                 struct drm_framebuffer *fb,
-                                 struct drm_pending_vblank_event *event,
-                                 uint32_t page_flip_flags, uint32_t target)
+
+static void amdgpu_flip_work_cleanup(struct amdgpu_flip_work *work)
+{
+        int i;
+
+        amdgpu_bo_unref(&work->old_abo);
+        dma_fence_put(work->excl);
+        for (i = 0; i < work->shared_count; ++i)
+                dma_fence_put(work->shared[i]);
+        kfree(work->shared);
+        kfree(work);
+}
+
+static void amdgpu_flip_cleanup_unreserve(struct amdgpu_flip_work *work,
+                                          struct amdgpu_bo *new_abo)
+{
+        amdgpu_bo_unreserve(new_abo);
+        amdgpu_flip_work_cleanup(work);
+}
+
+static void amdgpu_flip_cleanup_unpin(struct amdgpu_flip_work *work,
+                                      struct amdgpu_bo *new_abo)
+{
+        if (unlikely(amdgpu_bo_unpin(new_abo) != 0))
+                DRM_ERROR("failed to unpin new abo in error path\n");
+        amdgpu_flip_cleanup_unreserve(work, new_abo);
+}
+
+void amdgpu_crtc_cleanup_flip_ctx(struct amdgpu_flip_work *work,
+                                  struct amdgpu_bo *new_abo)
+{
+        if (unlikely(amdgpu_bo_reserve(new_abo, false) != 0)) {
+                DRM_ERROR("failed to reserve new abo in error path\n");
+                amdgpu_flip_work_cleanup(work);
+                return;
+        }
+        amdgpu_flip_cleanup_unpin(work, new_abo);
+}
+
+int amdgpu_crtc_prepare_flip(struct drm_crtc *crtc,
+                             struct drm_framebuffer *fb,
+                             struct drm_pending_vblank_event *event,
+                             uint32_t page_flip_flags,
+                             uint32_t target,
+                             struct amdgpu_flip_work **work_p,
+                             struct amdgpu_bo **new_abo_p)
 {
         struct drm_device *dev = crtc->dev;
         struct amdgpu_device *adev = dev->dev_private;
@@ -154,7 +196,7 @@ int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
         unsigned long flags;
         u64 tiling_flags;
         u64 base;
-        int i, r;
+        int r;
 
         work = kzalloc(sizeof *work, GFP_KERNEL);
         if (work == NULL)
@@ -189,7 +231,6 @@ int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
 
         r = amdgpu_bo_pin(new_abo, AMDGPU_GEM_DOMAIN_VRAM, &base);
         if (unlikely(r != 0)) {
-                r = -EINVAL;
                 DRM_ERROR("failed to pin new abo buffer before flip\n");
                 goto unreserve;
         }
@@ -216,41 +257,79 @@ int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
                 spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
                 r = -EBUSY;
                 goto pflip_cleanup;
+
         }
+        spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+
+        *work_p = work;
+        *new_abo_p = new_abo;
+
+        return 0;
+
+pflip_cleanup:
+        amdgpu_crtc_cleanup_flip_ctx(work, new_abo);
+        return r;
 
+unpin:
+        amdgpu_flip_cleanup_unpin(work, new_abo);
+        return r;
+
+unreserve:
+        amdgpu_flip_cleanup_unreserve(work, new_abo);
+        return r;
+
+cleanup:
+        amdgpu_flip_work_cleanup(work);
+        return r;
+
+}
+
+void amdgpu_crtc_submit_flip(struct drm_crtc *crtc,
+                             struct drm_framebuffer *fb,
+                             struct amdgpu_flip_work *work,
+                             struct amdgpu_bo *new_abo)
+{
+        unsigned long flags;
+        struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+        spin_lock_irqsave(&crtc->dev->event_lock, flags);
         amdgpu_crtc->pflip_status = AMDGPU_FLIP_PENDING;
         amdgpu_crtc->pflip_works = work;
 
-
-        DRM_DEBUG_DRIVER("crtc:%d[%p], pflip_stat:AMDGPU_FLIP_PENDING, work: %p,\n",
-                                         amdgpu_crtc->crtc_id, amdgpu_crtc, work);
         /* update crtc fb */
         crtc->primary->fb = fb;
         spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+
+        DRM_DEBUG_DRIVER(
+                        "crtc:%d[%p], pflip_stat:AMDGPU_FLIP_PENDING, work: %p,\n",
+                        amdgpu_crtc->crtc_id, amdgpu_crtc, work);
+
         amdgpu_flip_work_func(&work->flip_work.work);
-        return 0;
+}
 
-pflip_cleanup:
-        if (unlikely(amdgpu_bo_reserve(new_abo, false) != 0)) {
-                DRM_ERROR("failed to reserve new abo in error path\n");
-                goto cleanup;
-        }
-unpin:
-        if (unlikely(amdgpu_bo_unpin(new_abo) != 0)) {
-                DRM_ERROR("failed to unpin new abo in error path\n");
-        }
-unreserve:
-        amdgpu_bo_unreserve(new_abo);
+int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
+                                 struct drm_framebuffer *fb,
+                                 struct drm_pending_vblank_event *event,
+                                 uint32_t page_flip_flags,
+                                 uint32_t target)
+{
+        struct amdgpu_bo *new_abo;
+        struct amdgpu_flip_work *work;
+        int r;
 
-cleanup:
-        amdgpu_bo_unref(&work->old_abo);
-        dma_fence_put(work->excl);
-        for (i = 0; i < work->shared_count; ++i)
-                dma_fence_put(work->shared[i]);
-        kfree(work->shared);
-        kfree(work);
+        r = amdgpu_crtc_prepare_flip(crtc,
+                                     fb,
+                                     event,
+                                     page_flip_flags,
+                                     target,
+                                     &work,
+                                     &new_abo);
+        if (r)
+                return r;
 
-        return r;
+        amdgpu_crtc_submit_flip(crtc, fb, work, new_abo);
+
+        return 0;
 }
 
 int amdgpu_crtc_set_config(struct drm_mode_set *set)
@@ -582,12 +661,10 @@ int amdgpu_modeset_create_props(struct amdgpu_device *adev)
 {
         int sz;
 
-        if (adev->is_atom_bios) {
-                adev->mode_info.coherent_mode_property =
-                        drm_property_create_range(adev->ddev, 0 , "coherent", 0, 1);
-                if (!adev->mode_info.coherent_mode_property)
-                        return -ENOMEM;
-        }
+        adev->mode_info.coherent_mode_property =
+                drm_property_create_range(adev->ddev, 0 , "coherent", 0, 1);
+        if (!adev->mode_info.coherent_mode_property)
+                return -ENOMEM;
 
         adev->mode_info.load_detect_property =
                 drm_property_create_range(adev->ddev, 0, "load detection", 0, 1);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index 955d6f21e2b3..fa2b55681422 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -241,13 +241,6 @@ enum amdgpu_pcie_gen {
         AMDGPU_PCIE_GEN_INVALID = 0xffff
 };
 
-enum amdgpu_dpm_forced_level {
-        AMDGPU_DPM_FORCED_LEVEL_AUTO = 0,
-        AMDGPU_DPM_FORCED_LEVEL_LOW = 1,
-        AMDGPU_DPM_FORCED_LEVEL_HIGH = 2,
-        AMDGPU_DPM_FORCED_LEVEL_MANUAL = 3,
-};
-
 struct amdgpu_dpm_funcs {
         int (*get_temperature)(struct amdgpu_device *adev);
         int (*pre_set_power_state)(struct amdgpu_device *adev);
@@ -258,7 +251,7 @@ struct amdgpu_dpm_funcs {
         u32 (*get_mclk)(struct amdgpu_device *adev, bool low);
         void (*print_power_state)(struct amdgpu_device *adev, struct amdgpu_ps *ps);
         void (*debugfs_print_current_performance_level)(struct amdgpu_device *adev, struct seq_file *m);
-        int (*force_performance_level)(struct amdgpu_device *adev, enum amdgpu_dpm_forced_level level);
+        int (*force_performance_level)(struct amdgpu_device *adev, enum amd_dpm_forced_level level);
         bool (*vblank_too_short)(struct amdgpu_device *adev);
         void (*powergate_uvd)(struct amdgpu_device *adev, bool gate);
         void (*powergate_vce)(struct amdgpu_device *adev, bool gate);
@@ -353,9 +346,6 @@ struct amdgpu_dpm_funcs {
 #define amdgpu_dpm_get_current_power_state(adev) \
         (adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle)
 
-#define amdgpu_dpm_get_performance_level(adev) \
-        (adev)->powerplay.pp_funcs->get_performance_level((adev)->powerplay.pp_handle)
-
 #define amdgpu_dpm_get_pp_num_states(adev, data) \
         (adev)->powerplay.pp_funcs->get_pp_num_states((adev)->powerplay.pp_handle, data)
 
@@ -393,6 +383,11 @@ struct amdgpu_dpm_funcs {
          (adev)->powerplay.pp_funcs->get_vce_clock_state((adev)->powerplay.pp_handle, (i)) : \
          (adev)->pm.funcs->get_vce_clock_state((adev), (i)))
 
+#define amdgpu_dpm_get_performance_level(adev) \
+        ((adev)->pp_enabled ?                                           \
+        (adev)->powerplay.pp_funcs->get_performance_level((adev)->powerplay.pp_handle) : \
+        (adev)->pm.dpm.forced_level)
+
 struct amdgpu_dpm {
         struct amdgpu_ps        *ps;
         /* number of valid power states */
@@ -440,7 +435,7 @@ struct amdgpu_dpm {
         /* thermal handling */
         struct amdgpu_dpm_thermal thermal;
         /* forced levels */
-        enum amdgpu_dpm_forced_level forced_level;
+        enum amd_dpm_forced_level forced_level;
 };
 
 struct amdgpu_pm {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index ba2816b3cd07..75fc376ba735 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -90,7 +90,6 @@ int amdgpu_vram_page_split = 1024;
 int amdgpu_exp_hw_support = 0;
 int amdgpu_sched_jobs = 32;
 int amdgpu_sched_hw_submission = 2;
-int amdgpu_powerplay = -1;
 int amdgpu_no_evict = 0;
 int amdgpu_direct_gma_size = 0;
 unsigned amdgpu_pcie_gen_cap = 0;
@@ -179,9 +178,6 @@ module_param_named(sched_jobs, amdgpu_sched_jobs, int, 0444);
 MODULE_PARM_DESC(sched_hw_submission, "the max number of HW submissions (default 2)");
 module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444);
 
-MODULE_PARM_DESC(powerplay, "Powerplay component (1 = enable, 0 = disable, -1 = auto (default))");
-module_param_named(powerplay, amdgpu_powerplay, int, 0444);
-
 MODULE_PARM_DESC(ppfeaturemask, "all power features enabled (default))");
 module_param_named(ppfeaturemask, amdgpu_pp_feature_mask, int, 0444);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index cd62f6ffde2a..9bd1b4eae32e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -471,12 +471,15 @@ out:
 
 static int amdgpu_gem_va_check(void *param, struct amdgpu_bo *bo)
 {
-        unsigned domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
-
         /* if anything is swapped out don't swap it in here,
            just abort and wait for the next CS */
+        if (!amdgpu_bo_gpu_accessible(bo))
+                return -ERESTARTSYS;
+
+        if (bo->shadow && !amdgpu_bo_gpu_accessible(bo->shadow))
+                return -ERESTARTSYS;
 
-        return domain == AMDGPU_GEM_DOMAIN_CPU ? -ERESTARTSYS : 0;
+        return 0;
 }
 
 /**
@@ -496,7 +499,6 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
         struct amdgpu_bo_list_entry vm_pd;
         struct ww_acquire_ctx ticket;
         struct list_head list, duplicates;
-        unsigned domain;
         int r;
 
         INIT_LIST_HEAD(&list);
@@ -514,12 +516,18 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
                 goto error_print;
 
         list_for_each_entry(entry, &list, head) {
-                domain = amdgpu_mem_type_to_domain(entry->bo->mem.mem_type);
+                struct amdgpu_bo *bo =
+                        container_of(entry->bo, struct amdgpu_bo, tbo);
+
                 /* if anything is swapped out don't swap it in here,
                    just abort and wait for the next CS */
-                if (domain == AMDGPU_GEM_DOMAIN_CPU)
+                if (!amdgpu_bo_gpu_accessible(bo))
+                        goto error_unreserve;
+
+                if (bo->shadow && !amdgpu_bo_gpu_accessible(bo->shadow))
                         goto error_unreserve;
         }
+
         r = amdgpu_vm_validate_pt_bos(adev, bo_va->vm, amdgpu_gem_va_check,
                                       NULL);
         if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 01a42b6a69a4..19943356cca7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -42,12 +42,12 @@ int amdgpu_gfx_scratch_get(struct amdgpu_device *adev, uint32_t *reg)
 {
         int i;
 
-        for (i = 0; i < adev->gfx.scratch.num_reg; i++) {
-                if (adev->gfx.scratch.free[i]) {
-                        adev->gfx.scratch.free[i] = false;
-                        *reg = adev->gfx.scratch.reg[i];
-                        return 0;
-                }
+        i = ffs(adev->gfx.scratch.free_mask);
+        if (i != 0 && i <= adev->gfx.scratch.num_reg) {
+                i--;
+                adev->gfx.scratch.free_mask &= ~(1u << i);
+                *reg = adev->gfx.scratch.reg_base + i;
+                return 0;
         }
         return -EINVAL;
 }
@@ -62,14 +62,7 @@ int amdgpu_gfx_scratch_get(struct amdgpu_device *adev, uint32_t *reg)
  */
 void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg)
 {
-        int i;
-
-        for (i = 0; i < adev->gfx.scratch.num_reg; i++) {
-                if (adev->gfx.scratch.reg[i] == reg) {
-                        adev->gfx.scratch.free[i] = true;
-                        return;
-                }
-        }
+        adev->gfx.scratch.free_mask |= 1u << (reg - adev->gfx.scratch.reg_base);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index c6c125d31161..e4eb6dd3798a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -243,9 +243,9 @@ static void amdgpu_gtt_mgr_debug(struct ttm_mem_type_manager *man,
 }
 
 const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func = {
-        amdgpu_gtt_mgr_init,
-        amdgpu_gtt_mgr_fini,
-        amdgpu_gtt_mgr_new,
-        amdgpu_gtt_mgr_del,
-        amdgpu_gtt_mgr_debug
+        .init = amdgpu_gtt_mgr_init,
+        .takedown = amdgpu_gtt_mgr_fini,
+        .get_node = amdgpu_gtt_mgr_new,
+        .put_node = amdgpu_gtt_mgr_del,
+        .debug = amdgpu_gtt_mgr_debug
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
index 91d367399956..f2739995c335 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
@@ -231,8 +231,7 @@ void amdgpu_i2c_init(struct amdgpu_device *adev)
         if (amdgpu_hw_i2c)
                 DRM_INFO("hw_i2c forced on, you may experience display detection problems!\n");
 
-        if (adev->is_atom_bios)
-                amdgpu_atombios_i2c_init(adev);
+        amdgpu_atombios_i2c_init(adev);
 }
 
 /* remove all the buses */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 216a9572d946..e02a70dd37b5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -116,8 +116,8 @@ void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib,
  * to SI there was just a DE IB.
  */
 int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
-                       struct amdgpu_ib *ibs, struct dma_fence *last_vm_update,
-                       struct amdgpu_job *job, struct dma_fence **f)
+                       struct amdgpu_ib *ibs, struct amdgpu_job *job,
+                       struct dma_fence **f)
 {
         struct amdgpu_device *adev = ring->adev;
         struct amdgpu_ib *ib = &ibs[0];
@@ -175,15 +175,15 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
         if (ring->funcs->emit_hdp_flush)
                 amdgpu_ring_emit_hdp_flush(ring);
 
-        /* always set cond_exec_polling to CONTINUE */
-        *ring->cond_exe_cpu_addr = 1;
-
         skip_preamble = ring->current_ctx == fence_ctx;
         need_ctx_switch = ring->current_ctx != fence_ctx;
         if (job && ring->funcs->emit_cntxcntl) {
                 if (need_ctx_switch)
                         status |= AMDGPU_HAVE_CTX_SWITCH;
                 status |= job->preamble_status;
+
+                if (vm)
+                        status |= AMDGPU_VM_DOMAIN;
                 amdgpu_ring_emit_cntxcntl(ring, status);
         }
 
@@ -193,7 +193,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
                 /* drop preamble IBs if we don't have a context switch */
                 if ((ib->flags & AMDGPU_IB_FLAG_PREAMBLE) &&
                         skip_preamble &&
-                        !(status & AMDGPU_PREAMBLE_IB_PRESENT_FIRST))
+                        !(status & AMDGPU_PREAMBLE_IB_PRESENT_FIRST) &&
+                        !amdgpu_sriov_vf(adev)) /* for SRIOV preemption, Preamble CE ib must be inserted anyway */
                         continue;
 
                 amdgpu_ring_emit_ib(ring, ib, job ? job->vm_id : 0,
@@ -223,7 +224,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
                 amdgpu_ring_patch_cond_exec(ring, patch_offset);
 
         ring->current_ctx = fence_ctx;
-        if (ring->funcs->emit_switch_buffer)
+        if (vm && ring->funcs->emit_switch_buffer)
                 amdgpu_ring_emit_switch_buffer(ring);
         amdgpu_ring_commit(ring);
         return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index a0de6286c453..86a12424c162 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -170,8 +170,7 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
         BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
 
         trace_amdgpu_sched_run_job(job);
-        r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs,
-                               job->sync.last_vm_update, job, &fence);
+        r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job, &fence);
         if (r)
                 DRM_ERROR("Error scheduling IBs (%d)\n", r);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 8aef25828888..61d94c745672 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -60,6 +60,9 @@ void amdgpu_driver_unload_kms(struct drm_device *dev)
         if (adev->rmmio == NULL)
                 goto done_free;
 
+        if (amdgpu_sriov_vf(adev))
+                amdgpu_virt_request_full_gpu(adev, false);
+
         if (amdgpu_device_is_px(dev)) {
                 pm_runtime_get_sync(dev->dev);
                 pm_runtime_forbid(dev->dev);
@@ -138,6 +141,9 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
                 pm_runtime_put_autosuspend(dev->dev);
         }
 
+        if (amdgpu_sriov_vf(adev))
+                amdgpu_virt_release_full_gpu(adev, true);
+
 out:
         if (r) {
                 /* balance pm_runtime_get_sync in amdgpu_driver_unload_kms */
@@ -569,6 +575,27 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
                         return -EINVAL;
                 }
         }
+        case AMDGPU_INFO_NUM_HANDLES: {
+                struct drm_amdgpu_info_num_handles handle;
+
+                switch (info->query_hw_ip.type) {
+                case AMDGPU_HW_IP_UVD:
+                        /* Starting Polaris, we support unlimited UVD handles */
+                        if (adev->asic_type < CHIP_POLARIS10) {
+                                handle.uvd_max_handles = adev->uvd.max_handles;
+                                handle.uvd_used_handles = amdgpu_uvd_used_handles(adev);
+
+                                return copy_to_user(out, &handle,
+                                        min((size_t)size, sizeof(handle))) ? -EFAULT : 0;
+                        } else {
+                                return -ENODATA;
+                        }
+
+                        break;
+                default:
+                        return -EINVAL;
+                }
+        }
         default:
                 DRM_DEBUG_KMS("Invalid request %d\n", info->query);
                 return -EINVAL;
@@ -628,6 +655,12 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
                 goto out_suspend;
         }
 
+        if (amdgpu_sriov_vf(adev)) {
+                r = amdgpu_map_static_csa(adev, &fpriv->vm);
+                if (r)
+                        goto out_suspend;
+        }
+
         mutex_init(&fpriv->bo_list_lock);
         idr_init(&fpriv->bo_list_handles);
 
@@ -666,6 +699,14 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
         amdgpu_uvd_free_handles(adev, file_priv);
         amdgpu_vce_free_handles(adev, file_priv);
 
+        if (amdgpu_sriov_vf(adev)) {
+                /* TODO: how to handle reserve failure */
+                BUG_ON(amdgpu_bo_reserve(adev->virt.csa_obj, false));
+                amdgpu_vm_bo_rmv(adev, fpriv->vm.csa_bo_va);
+                fpriv->vm.csa_bo_va = NULL;
+                amdgpu_bo_unreserve(adev->virt.csa_obj);
+        }
+
         amdgpu_vm_fini(adev, &fpriv->vm);
 
         idr_for_each_entry(&fpriv->bo_list_handles, list, handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index b60346792bf8..c12497bd3889 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -595,6 +595,21 @@ int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
                                  struct drm_framebuffer *fb,
                                  struct drm_pending_vblank_event *event,
                                  uint32_t page_flip_flags, uint32_t target);
+void amdgpu_crtc_cleanup_flip_ctx(struct amdgpu_flip_work *work,
+                                  struct amdgpu_bo *new_abo);
+int amdgpu_crtc_prepare_flip(struct drm_crtc *crtc,
+                             struct drm_framebuffer *fb,
+                             struct drm_pending_vblank_event *event,
+                             uint32_t page_flip_flags,
+                             uint32_t target,
+                             struct amdgpu_flip_work **work,
+                             struct amdgpu_bo **new_abo);
+
+void amdgpu_crtc_submit_flip(struct drm_crtc *crtc,
+                             struct drm_framebuffer *fb,
+                             struct amdgpu_flip_work *work,
+                             struct amdgpu_bo *new_abo);
+
 extern const struct drm_mode_config_funcs amdgpu_mode_funcs;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index bf79b73e1538..d1aa291b2638 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -363,11 +363,31 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 
         bo->flags = flags;
 
+#ifdef CONFIG_X86_32
+        /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
+         * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
+         */
+        bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
+#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
+        /* Don't try to enable write-combining when it can't work, or things
+         * may be slow
+         * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
+         */
+
+#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
+         thanks to write-combining
+
+        if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
+                DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
+                              "better performance thanks to write-combining\n");
+        bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
+#else
         /* For architectures that don't support WC memory,
          * mask out the WC flag from the BO
          */
         if (!drm_arch_can_wc_memory())
                 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
+#endif
 
         amdgpu_fill_placement_to_bo(bo, placement);
         /* Kernel allocation are uninterruptible */
@@ -386,6 +406,11 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
         if (unlikely(r != 0))
                 return r;
 
+        bo->tbo.priority = ilog2(bo->tbo.num_pages);
+        if (kernel)
+                bo->tbo.priority *= 2;
+        bo->tbo.priority = min(bo->tbo.priority, (unsigned)(TTM_MAX_BO_PRIORITY - 1));
+
         if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
             bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
                 struct dma_fence *fence;
@@ -408,7 +433,8 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
         return 0;
 
 fail_unreserve:
-        ww_mutex_unlock(&bo->tbo.resv->lock);
+        if (!resv)
+                ww_mutex_unlock(&bo->tbo.resv->lock);
         amdgpu_bo_unref(&bo);
         return r;
 }
@@ -472,7 +498,16 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
                 return r;
 
         if (amdgpu_need_backup(adev) && (flags & AMDGPU_GEM_CREATE_SHADOW)) {
+                if (!resv) {
+                        r = ww_mutex_lock(&(*bo_ptr)->tbo.resv->lock, NULL);
+                        WARN_ON(r != 0);
+                }
+
                 r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
+
+                if (!resv)
+                        ww_mutex_unlock(&(*bo_ptr)->tbo.resv->lock);
+
                 if (r)
                         amdgpu_bo_unref(bo_ptr);
         }
@@ -849,6 +884,7 @@ int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
 }
 
 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
+                           bool evict,
                            struct ttm_mem_reg *new_mem)
 {
         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
@@ -861,6 +897,10 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
         abo = container_of(bo, struct amdgpu_bo, tbo);
         amdgpu_vm_bo_invalidate(adev, abo);
 
+        /* remember the eviction */
+        if (evict)
+                atomic64_inc(&adev->num_evictions);
+
         /* update statistics */
         if (!new_mem)
                 return;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 5cbf59ec0f68..15a723adca76 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -114,6 +114,15 @@ static inline u64 amdgpu_bo_mmap_offset(struct amdgpu_bo *bo)
         return drm_vma_node_offset_addr(&bo->tbo.vma_node);
 }
 
+/**
+ * amdgpu_bo_gpu_accessible - return whether the bo is currently in memory that
+ * is accessible to the GPU.
+ */
+static inline bool amdgpu_bo_gpu_accessible(struct amdgpu_bo *bo)
+{
+        return bo->tbo.mem.mem_type != TTM_PL_SYSTEM;
+}
+
 int amdgpu_bo_create(struct amdgpu_device *adev,
                             unsigned long size, int byte_align,
                             bool kernel, u32 domain, u64 flags,
@@ -155,7 +164,8 @@ int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
                            size_t buffer_size, uint32_t *metadata_size,
                            uint64_t *flags);
 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
-                                  struct ttm_mem_reg *new_mem);
+                           bool evict,
+                           struct ttm_mem_reg *new_mem);
 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo);
 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
                      bool shared);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index 723ae682bf25..a61882ddc804 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -34,6 +34,28 @@
 
 static int amdgpu_debugfs_pm_init(struct amdgpu_device *adev);
 
+static const struct cg_flag_name clocks[] = {
+        {AMD_CG_SUPPORT_GFX_MGCG, "Graphics Medium Grain Clock Gating"},
+        {AMD_CG_SUPPORT_GFX_MGLS, "Graphics Medium Grain memory Light Sleep"},
+        {AMD_CG_SUPPORT_GFX_CGCG, "Graphics Coarse Grain Clock Gating"},
+        {AMD_CG_SUPPORT_GFX_CGLS, "Graphics Coarse Grain memory Light Sleep"},
+        {AMD_CG_SUPPORT_GFX_CGTS, "Graphics Coarse Grain Tree Shader Clock Gating"},
+        {AMD_CG_SUPPORT_GFX_CGTS_LS, "Graphics Coarse Grain Tree Shader Light Sleep"},
+        {AMD_CG_SUPPORT_GFX_CP_LS, "Graphics Command Processor Light Sleep"},
+        {AMD_CG_SUPPORT_GFX_RLC_LS, "Graphics Run List Controller Light Sleep"},
+        {AMD_CG_SUPPORT_MC_LS, "Memory Controller Light Sleep"},
+        {AMD_CG_SUPPORT_MC_MGCG, "Memory Controller Medium Grain Clock Gating"},
+        {AMD_CG_SUPPORT_SDMA_LS, "System Direct Memory Access Light Sleep"},
+        {AMD_CG_SUPPORT_SDMA_MGCG, "System Direct Memory Access Medium Grain Clock Gating"},
+        {AMD_CG_SUPPORT_BIF_LS, "Bus Interface Light Sleep"},
+        {AMD_CG_SUPPORT_UVD_MGCG, "Unified Video Decoder Medium Grain Clock Gating"},
+        {AMD_CG_SUPPORT_VCE_MGCG, "Video Compression Engine Medium Grain Clock Gating"},
+        {AMD_CG_SUPPORT_HDP_LS, "Host Data Path Light Sleep"},
+        {AMD_CG_SUPPORT_HDP_MGCG, "Host Data Path Medium Grain Clock Gating"},
+        {AMD_CG_SUPPORT_ROM_MGCG, "Rom Medium Grain Clock Gating"},
+        {0, NULL},
+};
+
 void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev)
 {
         if (adev->pp_enabled)
@@ -112,28 +134,23 @@ static ssize_t amdgpu_get_dpm_forced_performance_level(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
+        enum amd_dpm_forced_level level;
 
         if  ((adev->flags & AMD_IS_PX) &&
              (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
                 return snprintf(buf, PAGE_SIZE, "off\n");
 
-        if (adev->pp_enabled) {
-                enum amd_dpm_forced_level level;
-
-                level = amdgpu_dpm_get_performance_level(adev);
-                return snprintf(buf, PAGE_SIZE, "%s\n",
-                                (level == AMD_DPM_FORCED_LEVEL_AUTO) ? "auto" :
-                                (level == AMD_DPM_FORCED_LEVEL_LOW) ? "low" :
-                                (level == AMD_DPM_FORCED_LEVEL_HIGH) ? "high" :
-                                (level == AMD_DPM_FORCED_LEVEL_MANUAL) ? "manual" : "unknown");
-        } else {
-                enum amdgpu_dpm_forced_level level;
-
-                level = adev->pm.dpm.forced_level;
-                return snprintf(buf, PAGE_SIZE, "%s\n",
-                                (level == AMDGPU_DPM_FORCED_LEVEL_AUTO) ? "auto" :
-                                (level == AMDGPU_DPM_FORCED_LEVEL_LOW) ? "low" : "high");
-        }
+        level = amdgpu_dpm_get_performance_level(adev);
+        return snprintf(buf, PAGE_SIZE, "%s\n",
+                        (level == AMD_DPM_FORCED_LEVEL_AUTO) ? "auto" :
+                        (level == AMD_DPM_FORCED_LEVEL_LOW) ? "low" :
+                        (level == AMD_DPM_FORCED_LEVEL_HIGH) ? "high" :
+                        (level == AMD_DPM_FORCED_LEVEL_MANUAL) ? "manual" :
+                        (level == AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD) ? "profile_standard" :
+                        (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK) ? "profile_min_sclk" :
+                        (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK) ? "profile_min_mclk" :
+                        (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) ? "profile_peak" :
+                        "unknown");
 }
 
 static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
@@ -143,7 +160,8 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
 {
         struct drm_device *ddev = dev_get_drvdata(dev);
         struct amdgpu_device *adev = ddev->dev_private;
-        enum amdgpu_dpm_forced_level level;
+        enum amd_dpm_forced_level level;
+        enum amd_dpm_forced_level current_level;
         int ret = 0;
 
         /* Can't force performance level when the card is off */
@@ -151,19 +169,34 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
              (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
                 return -EINVAL;
 
+        current_level = amdgpu_dpm_get_performance_level(adev);
+
         if (strncmp("low", buf, strlen("low")) == 0) {
-                level = AMDGPU_DPM_FORCED_LEVEL_LOW;
+                level = AMD_DPM_FORCED_LEVEL_LOW;
         } else if (strncmp("high", buf, strlen("high")) == 0) {
-                level = AMDGPU_DPM_FORCED_LEVEL_HIGH;
+                level = AMD_DPM_FORCED_LEVEL_HIGH;
         } else if (strncmp("auto", buf, strlen("auto")) == 0) {
-                level = AMDGPU_DPM_FORCED_LEVEL_AUTO;
+                level = AMD_DPM_FORCED_LEVEL_AUTO;
         } else if (strncmp("manual", buf, strlen("manual")) == 0) {
-                level = AMDGPU_DPM_FORCED_LEVEL_MANUAL;
-        } else {
+                level = AMD_DPM_FORCED_LEVEL_MANUAL;
+        } else if (strncmp("profile_exit", buf, strlen("profile_exit")) == 0) {
+                level = AMD_DPM_FORCED_LEVEL_PROFILE_EXIT;
+        } else if (strncmp("profile_standard", buf, strlen("profile_standard")) == 0) {
+                level = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD;
+        } else if (strncmp("profile_min_sclk", buf, strlen("profile_min_sclk")) == 0) {
+                level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK;
+        } else if (strncmp("profile_min_mclk", buf, strlen("profile_min_mclk")) == 0) {
+                level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK;
+        } else if (strncmp("profile_peak", buf, strlen("profile_peak")) == 0) {
+                level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK;
+        }  else {
                 count = -EINVAL;
                 goto fail;
         }
 
+        if (current_level == level)
+                return count;
+
         if (adev->pp_enabled)
                 amdgpu_dpm_force_performance_level(adev, level);
         else {
@@ -180,6 +213,7 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
                         adev->pm.dpm.forced_level = level;
                 mutex_unlock(&adev->pm.mutex);
         }
+
 fail:
         return count;
 }
@@ -1060,9 +1094,9 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
 
         if (adev->pm.funcs->force_performance_level) {
                 if (adev->pm.dpm.thermal_active) {
-                        enum amdgpu_dpm_forced_level level = adev->pm.dpm.forced_level;
+                        enum amd_dpm_forced_level level = adev->pm.dpm.forced_level;
                         /* force low perf level for thermal */
-                        amdgpu_dpm_force_performance_level(adev, AMDGPU_DPM_FORCED_LEVEL_LOW);
+                        amdgpu_dpm_force_performance_level(adev, AMD_DPM_FORCED_LEVEL_LOW);
                         /* save the user's level */
                         adev->pm.dpm.forced_level = level;
                 } else {
@@ -1351,12 +1385,27 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a
         return 0;
 }
 
+static void amdgpu_parse_cg_state(struct seq_file *m, u32 flags)
+{
+        int i;
+
+        for (i = 0; clocks[i].flag; i++)
+                seq_printf(m, "\t%s: %s\n", clocks[i].name,
+                           (flags & clocks[i].flag) ? "On" : "Off");
+}
+
 static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         struct amdgpu_device *adev = dev->dev_private;
         struct drm_device *ddev = adev->ddev;
+        u32 flags = 0;
+
+        amdgpu_get_clockgating_state(adev, &flags);
+        seq_printf(m, "Clock Gating Flags Mask: 0x%x\n", flags);
+        amdgpu_parse_cg_state(m, flags);
+        seq_printf(m, "\n");
 
         if (!adev->pm.dpm_enabled) {
                 seq_printf(m, "dpm not enabled\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h
index 5fd7734f15ca..c19c4d138751 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h
@@ -24,6 +24,12 @@
 #ifndef __AMDGPU_PM_H__
 #define __AMDGPU_PM_H__
 
+struct cg_flag_name
+{
+        u32 flag;
+        const char *name;
+};
+
 int amdgpu_pm_sysfs_init(struct amdgpu_device *adev);
 void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev);
 void amdgpu_pm_print_power_states(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
index 95a568df8551..8856eccc37fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
@@ -34,67 +34,34 @@
 #include "cik_dpm.h"
 #include "vi_dpm.h"
 
-static int amdgpu_powerplay_init(struct amdgpu_device *adev)
+static int amdgpu_create_pp_handle(struct amdgpu_device *adev)
 {
-        int ret = 0;
+        struct amd_pp_init pp_init;
         struct amd_powerplay *amd_pp;
+        int ret;
 
         amd_pp = &(adev->powerplay);
-
-        if (adev->pp_enabled) {
-                struct amd_pp_init *pp_init;
-
-                pp_init = kzalloc(sizeof(struct amd_pp_init), GFP_KERNEL);
-
-                if (pp_init == NULL)
-                        return -ENOMEM;
-
-                pp_init->chip_family = adev->family;
-                pp_init->chip_id = adev->asic_type;
-                pp_init->device = amdgpu_cgs_create_device(adev);
-                ret = amd_powerplay_init(pp_init, amd_pp);
-                kfree(pp_init);
-        } else {
-                amd_pp->pp_handle = (void *)adev;
-
-                switch (adev->asic_type) {
-#ifdef CONFIG_DRM_AMDGPU_SI
-                case CHIP_TAHITI:
-                case CHIP_PITCAIRN:
-                case CHIP_VERDE:
-                case CHIP_OLAND:
-                case CHIP_HAINAN:
-                        amd_pp->ip_funcs = &si_dpm_ip_funcs;
-                break;
-#endif
-#ifdef CONFIG_DRM_AMDGPU_CIK
-                case CHIP_BONAIRE:
-                case CHIP_HAWAII:
-                        amd_pp->ip_funcs = &ci_dpm_ip_funcs;
-                        break;
-                case CHIP_KABINI:
-                case CHIP_MULLINS:
-                case CHIP_KAVERI:
-                        amd_pp->ip_funcs = &kv_dpm_ip_funcs;
-                        break;
-#endif
-                case CHIP_CARRIZO:
-                case CHIP_STONEY:
-                        amd_pp->ip_funcs = &cz_dpm_ip_funcs;
-                        break;
-                default:
-                        ret = -EINVAL;
-                        break;
-                }
-        }
-        return ret;
+        pp_init.chip_family = adev->family;
+        pp_init.chip_id = adev->asic_type;
+        pp_init.pm_en = amdgpu_dpm != 0 ? true : false;
+        pp_init.feature_mask = amdgpu_pp_feature_mask;
+        pp_init.device = amdgpu_cgs_create_device(adev);
+        ret = amd_powerplay_create(&pp_init, &(amd_pp->pp_handle));
+        if (ret)
+                return -EINVAL;
+        return 0;
 }
 
 static int amdgpu_pp_early_init(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        struct amd_powerplay *amd_pp;
         int ret = 0;
 
+        amd_pp = &(adev->powerplay);
+        adev->pp_enabled = false;
+        amd_pp->pp_handle = (void *)adev;
+
         switch (adev->asic_type) {
         case CHIP_POLARIS11:
         case CHIP_POLARIS10:
@@ -102,30 +69,48 @@ static int amdgpu_pp_early_init(void *handle)
         case CHIP_TONGA:
         case CHIP_FIJI:
         case CHIP_TOPAZ:
-                adev->pp_enabled = true;
-                break;
         case CHIP_CARRIZO:
         case CHIP_STONEY:
-                adev->pp_enabled = (amdgpu_powerplay == 0) ? false : true;
+                adev->pp_enabled = true;
+                if (amdgpu_create_pp_handle(adev))
+                        return -EINVAL;
+                amd_pp->ip_funcs = &pp_ip_funcs;
+                amd_pp->pp_funcs = &pp_dpm_funcs;
                 break;
         /* These chips don't have powerplay implemenations */
+#ifdef CONFIG_DRM_AMDGPU_SI
+        case CHIP_TAHITI:
+        case CHIP_PITCAIRN:
+        case CHIP_VERDE:
+        case CHIP_OLAND:
+        case CHIP_HAINAN:
+                amd_pp->ip_funcs = &si_dpm_ip_funcs;
+        break;
+#endif
+#ifdef CONFIG_DRM_AMDGPU_CIK
         case CHIP_BONAIRE:
         case CHIP_HAWAII:
+                amd_pp->ip_funcs = &ci_dpm_ip_funcs;
+                break;
         case CHIP_KABINI:
         case CHIP_MULLINS:
         case CHIP_KAVERI:
+                amd_pp->ip_funcs = &kv_dpm_ip_funcs;
+                break;
+#endif
         default:
-                adev->pp_enabled = false;
+                ret = -EINVAL;
                 break;
         }
 
-        ret = amdgpu_powerplay_init(adev);
-        if (ret)
-                return ret;
-
         if (adev->powerplay.ip_funcs->early_init)
                 ret = adev->powerplay.ip_funcs->early_init(
                                         adev->powerplay.pp_handle);
+
+        if (ret == PP_DPM_DISABLED) {
+                adev->pm.dpm_enabled = false;
+                return 0;
+        }
         return ret;
 }
 
@@ -185,6 +170,11 @@ static int amdgpu_pp_hw_init(void *handle)
                 ret = adev->powerplay.ip_funcs->hw_init(
                                         adev->powerplay.pp_handle);
 
+        if (ret == PP_DPM_DISABLED) {
+                adev->pm.dpm_enabled = false;
+                return 0;
+        }
+
         if ((amdgpu_dpm != 0) && !amdgpu_sriov_vf(adev))
                 adev->pm.dpm_enabled = true;
 
@@ -210,14 +200,14 @@ static void amdgpu_pp_late_fini(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-        if (adev->pp_enabled) {
-                amdgpu_pm_sysfs_fini(adev);
-                amd_powerplay_fini(adev->powerplay.pp_handle);
-        }
-
         if (adev->powerplay.ip_funcs->late_fini)
                 adev->powerplay.ip_funcs->late_fini(
                           adev->powerplay.pp_handle);
+
+        if (adev->pp_enabled && adev->pm.dpm_enabled)
+                amdgpu_pm_sysfs_fini(adev);
+
+        amd_powerplay_destroy(adev->powerplay.pp_handle);
 }
 
 static int amdgpu_pp_suspend(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index a47628395914..7c842b7f1004 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -207,6 +207,8 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
         }
         ring->cond_exe_gpu_addr = adev->wb.gpu_addr + (ring->cond_exe_offs * 4);
         ring->cond_exe_cpu_addr = &adev->wb.wb[ring->cond_exe_offs];
+        /* always set cond_exec_polling to CONTINUE */
+        *ring->cond_exe_cpu_addr = 1;
 
         r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type);
         if (r) {
@@ -307,7 +309,7 @@ static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf,
         while (size) {
                 if (*pos >= (ring->ring_size + 12))
                         return result;
-                        
+
                 value = ring->ring[(*pos - 12)/4];
                 r = put_user(value, (uint32_t*)buf);
                 if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 574f0b79c690..2345b39878c6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -135,6 +135,8 @@ struct amdgpu_ring_funcs {
         void (*end_use)(struct amdgpu_ring *ring);
         void (*emit_switch_buffer) (struct amdgpu_ring *ring);
         void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
+        void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg);
+        void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
 };
 
 struct amdgpu_ring {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index bb964a8ff938..a18ae1e97860 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -24,7 +24,7 @@ TRACE_EVENT(amdgpu_mm_rreg,
                            __entry->reg = reg;
                            __entry->value = value;
                            ),
-            TP_printk("0x%04lx, 0x%04lx, 0x%08lx",
+            TP_printk("0x%04lx, 0x%08lx, 0x%08lx",
                       (unsigned long)__entry->did,
                       (unsigned long)__entry->reg,
                       (unsigned long)__entry->value)
@@ -43,7 +43,7 @@ TRACE_EVENT(amdgpu_mm_wreg,
                            __entry->reg = reg;
                            __entry->value = value;
                            ),
-            TP_printk("0x%04lx, 0x%04lx, 0x%08lx",
+            TP_printk("0x%04lx, 0x%08lx, 0x%08lx",
                       (unsigned long)__entry->did,
                       (unsigned long)__entry->reg,
                       (unsigned long)__entry->value)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index c695b6c55361..1154b0a8881d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -466,10 +466,6 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo,
 
         adev = amdgpu_ttm_adev(bo->bdev);
 
-        /* remember the eviction */
-        if (evict)
-                atomic64_inc(&adev->num_evictions);
-
         if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
                 amdgpu_move_null(bo, new_mem);
                 return 0;
@@ -552,6 +548,8 @@ static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_
                         mem->bus.addr =
                                 ioremap_nocache(mem->bus.base + mem->bus.offset,
                                                 mem->bus.size);
+                if (!mem->bus.addr)
+                        return -ENOMEM;
 
                 /*
                  * Alpha: Use just the bus offset plus
@@ -1052,56 +1050,6 @@ uint32_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
         return flags;
 }
 
-static void amdgpu_ttm_lru_removal(struct ttm_buffer_object *tbo)
-{
-        struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
-        unsigned i, j;
-
-        for (i = 0; i < AMDGPU_TTM_LRU_SIZE; ++i) {
-                struct amdgpu_mman_lru *lru = &adev->mman.log2_size[i];
-
-                for (j = 0; j < TTM_NUM_MEM_TYPES; ++j)
-                        if (&tbo->lru == lru->lru[j])
-                                lru->lru[j] = tbo->lru.prev;
-
-                if (&tbo->swap == lru->swap_lru)
-                        lru->swap_lru = tbo->swap.prev;
-        }
-}
-
-static struct amdgpu_mman_lru *amdgpu_ttm_lru(struct ttm_buffer_object *tbo)
-{
-        struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
-        unsigned log2_size = min(ilog2(tbo->num_pages),
-                                 AMDGPU_TTM_LRU_SIZE - 1);
-
-        return &adev->mman.log2_size[log2_size];
-}
-
-static struct list_head *amdgpu_ttm_lru_tail(struct ttm_buffer_object *tbo)
-{
-        struct amdgpu_mman_lru *lru = amdgpu_ttm_lru(tbo);
-        struct list_head *res = lru->lru[tbo->mem.mem_type];
-
-        lru->lru[tbo->mem.mem_type] = &tbo->lru;
-        while ((++lru)->lru[tbo->mem.mem_type] == res)
-                lru->lru[tbo->mem.mem_type] = &tbo->lru;
-
-        return res;
-}
-
-static struct list_head *amdgpu_ttm_swap_lru_tail(struct ttm_buffer_object *tbo)
-{
-        struct amdgpu_mman_lru *lru = amdgpu_ttm_lru(tbo);
-        struct list_head *res = lru->swap_lru;
-
-        lru->swap_lru = &tbo->swap;
-        while ((++lru)->swap_lru == res)
-                lru->swap_lru = &tbo->swap;
-
-        return res;
-}
-
 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
                                             const struct ttm_place *place)
 {
@@ -1140,14 +1088,10 @@ static struct ttm_bo_driver amdgpu_bo_driver = {
         .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
         .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
         .io_mem_free = &amdgpu_ttm_io_mem_free,
-        .lru_removal = &amdgpu_ttm_lru_removal,
-        .lru_tail = &amdgpu_ttm_lru_tail,
-        .swap_lru_tail = &amdgpu_ttm_swap_lru_tail,
 };
 
 int amdgpu_ttm_init(struct amdgpu_device *adev)
 {
-        unsigned i, j;
         int r;
 
         r = amdgpu_ttm_global_init(adev);
@@ -1165,19 +1109,6 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
                 DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
                 return r;
         }
-
-        for (i = 0; i < AMDGPU_TTM_LRU_SIZE; ++i) {
-                struct amdgpu_mman_lru *lru = &adev->mman.log2_size[i];
-
-                for (j = 0; j < TTM_NUM_MEM_TYPES; ++j)
-                        lru->lru[j] = &adev->mman.bdev.man[j].lru;
-                lru->swap_lru = &adev->mman.bdev.glob->swap_lru;
-        }
-
-        for (j = 0; j < TTM_NUM_MEM_TYPES; ++j)
-                adev->mman.guard.lru[j] = NULL;
-        adev->mman.guard.swap_lru = NULL;
-
         adev->mman.initialized = true;
         r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
                                 adev->mc.real_vram_size >> PAGE_SHIFT);
@@ -1365,7 +1296,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring,
         WARN_ON(job->ibs[0].length_dw > num_dw);
         if (direct_submit) {
                 r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs,
-                                       NULL, NULL, fence);
+                                       NULL, fence);
                 job->fence = dma_fence_get(*fence);
                 if (r)
                         DRM_ERROR("Error scheduling IBs (%d)\n", r);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 98ee384f0fca..6bdede8ff12b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -34,13 +34,6 @@
 #define AMDGPU_PL_FLAG_GWS              (TTM_PL_FLAG_PRIV << 1)
 #define AMDGPU_PL_FLAG_OA               (TTM_PL_FLAG_PRIV << 2)
 
-#define AMDGPU_TTM_LRU_SIZE     20
-
-struct amdgpu_mman_lru {
-        struct list_head                *lru[TTM_NUM_MEM_TYPES];
-        struct list_head                *swap_lru;
-};
-
 struct amdgpu_mman {
         struct ttm_bo_global_ref        bo_global_ref;
         struct drm_global_reference     mem_global_ref;
@@ -58,11 +51,6 @@ struct amdgpu_mman {
         struct amdgpu_ring                      *buffer_funcs_ring;
         /* Scheduler entity for buffer moves */
         struct amd_sched_entity                 entity;
-
-        /* custom LRU management */
-        struct amdgpu_mman_lru                  log2_size[AMDGPU_TTM_LRU_SIZE];
-        /* guard for log2_size array, don't add anything in between */
-        struct amdgpu_mman_lru                  guard;
 };
 
 extern const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index 1d564beb0fde..6f62ac473064 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -976,7 +976,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
         ib->length_dw = 16;
 
         if (direct) {
-                r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f);
+                r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
                 job->fence = dma_fence_get(f);
                 if (r)
                         goto err_free;
@@ -1178,3 +1178,28 @@ int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 error:
         return r;
 }
+
+/**
+ * amdgpu_uvd_used_handles - returns used UVD handles
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns the number of UVD handles in use
+ */
+uint32_t amdgpu_uvd_used_handles(struct amdgpu_device *adev)
+{
+        unsigned i;
+        uint32_t used_handles = 0;
+
+        for (i = 0; i < adev->uvd.max_handles; ++i) {
+                /*
+                 * Handles can be freed in any order, and not
+                 * necessarily linear. So we need to count
+                 * all non-zero handles.
+                 */
+                if (atomic_read(&adev->uvd.handles[i]))
+                        used_handles++;
+        }
+
+        return used_handles;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
index 6249ba1bde2a..c10682baccae 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
@@ -38,5 +38,6 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx);
 void amdgpu_uvd_ring_begin_use(struct amdgpu_ring *ring);
 void amdgpu_uvd_ring_end_use(struct amdgpu_ring *ring);
 int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout);
+uint32_t amdgpu_uvd_used_handles(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 8fec802d3908..79bc9c7aad45 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -455,7 +455,7 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
         for (i = ib->length_dw; i < ib_size_dw; ++i)
                 ib->ptr[i] = 0x0;
 
-        r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f);
+        r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
         job->fence = dma_fence_get(f);
         if (r)
                 goto err;
@@ -518,7 +518,7 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
                 ib->ptr[i] = 0x0;
 
         if (direct) {
-                r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f);
+                r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
                 job->fence = dma_fence_get(f);
                 if (r)
                         goto err;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
new file mode 100644
index 000000000000..3fd951c71d1b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -0,0 +1,222 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+
+int amdgpu_allocate_static_csa(struct amdgpu_device *adev)
+{
+        int r;
+        void *ptr;
+
+        r = amdgpu_bo_create_kernel(adev, AMDGPU_CSA_SIZE, PAGE_SIZE,
+                                AMDGPU_GEM_DOMAIN_VRAM, &adev->virt.csa_obj,
+                                &adev->virt.csa_vmid0_addr, &ptr);
+        if (r)
+                return r;
+
+        memset(ptr, 0, AMDGPU_CSA_SIZE);
+        return 0;
+}
+
+/*
+ * amdgpu_map_static_csa should be called during amdgpu_vm_init
+ * it maps virtual address "AMDGPU_VA_RESERVED_SIZE - AMDGPU_CSA_SIZE"
+ * to this VM, and each command submission of GFX should use this virtual
+ * address within META_DATA init package to support SRIOV gfx preemption.
+ */
+
+int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm)
+{
+        int r;
+        struct amdgpu_bo_va *bo_va;
+        struct ww_acquire_ctx ticket;
+        struct list_head list;
+        struct amdgpu_bo_list_entry pd;
+        struct ttm_validate_buffer csa_tv;
+
+        INIT_LIST_HEAD(&list);
+        INIT_LIST_HEAD(&csa_tv.head);
+        csa_tv.bo = &adev->virt.csa_obj->tbo;
+        csa_tv.shared = true;
+
+        list_add(&csa_tv.head, &list);
+        amdgpu_vm_get_pd_bo(vm, &list, &pd);
+
+        r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL);
+        if (r) {
+                DRM_ERROR("failed to reserve CSA,PD BOs: err=%d\n", r);
+                return r;
+        }
+
+        bo_va = amdgpu_vm_bo_add(adev, vm, adev->virt.csa_obj);
+        if (!bo_va) {
+                ttm_eu_backoff_reservation(&ticket, &list);
+                DRM_ERROR("failed to create bo_va for static CSA\n");
+                return -ENOMEM;
+        }
+
+        r = amdgpu_vm_bo_map(adev, bo_va, AMDGPU_CSA_VADDR, 0,AMDGPU_CSA_SIZE,
+                                                AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
+                                                AMDGPU_PTE_EXECUTABLE);
+
+        if (r) {
+                DRM_ERROR("failed to do bo_map on static CSA, err=%d\n", r);
+                amdgpu_vm_bo_rmv(adev, bo_va);
+                ttm_eu_backoff_reservation(&ticket, &list);
+                kfree(bo_va);
+                return r;
+        }
+
+        vm->csa_bo_va = bo_va;
+        ttm_eu_backoff_reservation(&ticket, &list);
+        return 0;
+}
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev)
+{
+        /* enable virtual display */
+        adev->mode_info.num_crtc = 1;
+        adev->enable_virtual_display = true;
+
+        mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
+{
+        signed long r;
+        uint32_t val;
+        struct dma_fence *f;
+        struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+        struct amdgpu_ring *ring = &kiq->ring;
+
+        BUG_ON(!ring->funcs->emit_rreg);
+
+        mutex_lock(&adev->virt.lock);
+        amdgpu_ring_alloc(ring, 32);
+        amdgpu_ring_emit_hdp_flush(ring);
+        amdgpu_ring_emit_rreg(ring, reg);
+        amdgpu_ring_emit_hdp_invalidate(ring);
+        amdgpu_fence_emit(ring, &f);
+        amdgpu_ring_commit(ring);
+        mutex_unlock(&adev->virt.lock);
+
+        r = dma_fence_wait(f, false);
+        if (r)
+                DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+        dma_fence_put(f);
+
+        val = adev->wb.wb[adev->virt.reg_val_offs];
+
+        return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
+{
+        signed long r;
+        struct dma_fence *f;
+        struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+        struct amdgpu_ring *ring = &kiq->ring;
+
+        BUG_ON(!ring->funcs->emit_wreg);
+
+        mutex_lock(&adev->virt.lock);
+        amdgpu_ring_alloc(ring, 32);
+        amdgpu_ring_emit_hdp_flush(ring);
+        amdgpu_ring_emit_wreg(ring, reg, v);
+        amdgpu_ring_emit_hdp_invalidate(ring);
+        amdgpu_fence_emit(ring, &f);
+        amdgpu_ring_commit(ring);
+        mutex_unlock(&adev->virt.lock);
+
+        r = dma_fence_wait(f, false);
+        if (r)
+                DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+        dma_fence_put(f);
+}
+
+/**
+ * amdgpu_virt_request_full_gpu() - request full gpu access
+ * @amdgpu:     amdgpu device.
+ * @init:       is driver init time.
+ * When start to init/fini driver, first need to request full gpu access.
+ * Return: Zero if request success, otherwise will return error.
+ */
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init)
+{
+        struct amdgpu_virt *virt = &adev->virt;
+        int r;
+
+        if (virt->ops && virt->ops->req_full_gpu) {
+                r = virt->ops->req_full_gpu(adev, init);
+                if (r)
+                        return r;
+
+                adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+        }
+
+        return 0;
+}
+
+/**
+ * amdgpu_virt_release_full_gpu() - release full gpu access
+ * @amdgpu:     amdgpu device.
+ * @init:       is driver init time.
+ * When finishing driver init/fini, need to release full gpu access.
+ * Return: Zero if release success, otherwise will returen error.
+ */
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init)
+{
+        struct amdgpu_virt *virt = &adev->virt;
+        int r;
+
+        if (virt->ops && virt->ops->rel_full_gpu) {
+                r = virt->ops->rel_full_gpu(adev, init);
+                if (r)
+                        return r;
+
+                adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
+        }
+        return 0;
+}
+
+/**
+ * amdgpu_virt_reset_gpu() - reset gpu
+ * @amdgpu:     amdgpu device.
+ * Send reset command to GPU hypervisor to reset GPU that VM is using
+ * Return: Zero if reset success, otherwise will return error.
+ */
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev)
+{
+        struct amdgpu_virt *virt = &adev->virt;
+        int r;
+
+        if (virt->ops && virt->ops->reset_gpu) {
+                r = virt->ops->reset_gpu(adev);
+                if (r)
+                        return r;
+
+                adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+        }
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 2c37a374917f..675e12c42532 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -28,22 +28,48 @@
 #define AMDGPU_SRIOV_CAPS_ENABLE_IOV   (1 << 1) /* sr-iov is enabled on this GPU */
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual function */
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
+#define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access mode */
+
+/**
+ * struct amdgpu_virt_ops - amdgpu device virt operations
+ */
+struct amdgpu_virt_ops {
+        int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
+        int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
+        int (*reset_gpu)(struct amdgpu_device *adev);
+};
+
 /* GPU virtualization */
-struct amdgpu_virtualization {
-        uint32_t virtual_caps;
+struct amdgpu_virt {
+        uint32_t                        caps;
+        struct amdgpu_bo                *csa_obj;
+        uint64_t                        csa_vmid0_addr;
+        bool chained_ib_support;
+        uint32_t                        reg_val_offs;
+        struct mutex                    lock;
+        struct amdgpu_irq_src           ack_irq;
+        struct amdgpu_irq_src           rcv_irq;
+        struct delayed_work             flr_work;
+        const struct amdgpu_virt_ops    *ops;
 };
 
+#define AMDGPU_CSA_SIZE    (8 * 1024)
+#define AMDGPU_CSA_VADDR   (AMDGPU_VA_RESERVED_SIZE - AMDGPU_CSA_SIZE)
+
 #define amdgpu_sriov_enabled(adev) \
-((adev)->virtualization.virtual_caps & AMDGPU_SRIOV_CAPS_ENABLE_IOV)
+((adev)->virt.caps & AMDGPU_SRIOV_CAPS_ENABLE_IOV)
 
 #define amdgpu_sriov_vf(adev) \
-((adev)->virtualization.virtual_caps & AMDGPU_SRIOV_CAPS_IS_VF)
+((adev)->virt.caps & AMDGPU_SRIOV_CAPS_IS_VF)
 
 #define amdgpu_sriov_bios(adev) \
-((adev)->virtualization.virtual_caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)
+((adev)->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)
+
+#define amdgpu_sriov_runtime(adev) \
+((adev)->virt.caps & AMDGPU_SRIOV_CAPS_RUNTIME)
 
 #define amdgpu_passthrough(adev) \
-((adev)->virtualization.virtual_caps & AMDGPU_PASSTHROUGH_MODE)
+((adev)->virt.caps & AMDGPU_PASSTHROUGH_MODE)
 
 static inline bool is_virtual_machine(void)
 {
@@ -54,4 +80,14 @@ static inline bool is_virtual_machine(void)
 #endif
 }
 
-#endif
\ No newline at end of file
+struct amdgpu_vm;
+int amdgpu_allocate_static_csa(struct amdgpu_device *adev);
+int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+void amdgpu_virt_init_setting(struct amdgpu_device *adev);
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 1dda9321bd5a..bd0d33125c18 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1293,7 +1293,7 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
                      struct amdgpu_bo_va *bo_va,
                      uint64_t saddr, uint64_t offset,
-                     uint64_t size, uint32_t flags)
+                     uint64_t size, uint64_t flags)
 {
         struct amdgpu_bo_va_mapping *mapping;
         struct amdgpu_vm *vm = bo_va->vm;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index adbc2f5e5c7f..18c72c0b478d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -111,6 +111,8 @@ struct amdgpu_vm {
 
         /* client id */
         u64                     client_id;
+        /* each VM will map on CSA */
+        struct amdgpu_bo_va *csa_bo_va;
 };
 
 struct amdgpu_vm_id {
@@ -195,7 +197,7 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
                      struct amdgpu_bo_va *bo_va,
                      uint64_t addr, uint64_t offset,
-                     uint64_t size, uint32_t flags);
+                     uint64_t size, uint64_t flags);
 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
                        struct amdgpu_bo_va *bo_va,
                        uint64_t addr);
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
index c32eca26155c..2af26d2da127 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
@@ -181,9 +181,6 @@ void amdgpu_atombios_encoder_init_backlight(struct amdgpu_encoder *amdgpu_encode
         if (!amdgpu_encoder->enc_priv)
                 return;
 
-        if (!adev->is_atom_bios)
-                return;
-
         if (!(adev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU))
                 return;
 
@@ -236,9 +233,6 @@ amdgpu_atombios_encoder_fini_backlight(struct amdgpu_encoder *amdgpu_encoder)
         if (!amdgpu_encoder->enc_priv)
                 return;
 
-        if (!adev->is_atom_bios)
-                return;
-
         if (!(adev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU))
                 return;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index bda9e3de191e..9498e78b90d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -889,7 +889,16 @@ static void ci_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
 
         pi->uvd_power_gated = gate;
 
-        ci_update_uvd_dpm(adev, gate);
+        if (gate) {
+                /* stop the UVD block */
+                amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
+                                                        AMD_PG_STATE_GATE);
+                ci_update_uvd_dpm(adev, gate);
+        } else {
+                amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
+                                                        AMD_PG_STATE_UNGATE);
+                ci_update_uvd_dpm(adev, gate);
+        }
 }
 
 static bool ci_dpm_vblank_too_short(struct amdgpu_device *adev)
@@ -4336,13 +4345,13 @@ static u32 ci_get_lowest_enabled_level(struct amdgpu_device *adev,
 
 
 static int ci_dpm_force_performance_level(struct amdgpu_device *adev,
-                                          enum amdgpu_dpm_forced_level level)
+                                          enum amd_dpm_forced_level level)
 {
         struct ci_power_info *pi = ci_get_pi(adev);
         u32 tmp, levels, i;
         int ret;
 
-        if (level == AMDGPU_DPM_FORCED_LEVEL_HIGH) {
+        if (level == AMD_DPM_FORCED_LEVEL_HIGH) {
                 if ((!pi->pcie_dpm_key_disabled) &&
                     pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
                         levels = 0;
@@ -4403,7 +4412,7 @@ static int ci_dpm_force_performance_level(struct amdgpu_device *adev,
                                 }
                         }
                 }
-        } else if (level == AMDGPU_DPM_FORCED_LEVEL_LOW) {
+        } else if (level == AMD_DPM_FORCED_LEVEL_LOW) {
                 if ((!pi->sclk_dpm_key_disabled) &&
                     pi->dpm_level_enable_mask.sclk_dpm_enable_mask) {
                         levels = ci_get_lowest_enabled_level(adev,
@@ -4452,7 +4461,7 @@ static int ci_dpm_force_performance_level(struct amdgpu_device *adev,
                                 udelay(1);
                         }
                 }
-        } else if (level == AMDGPU_DPM_FORCED_LEVEL_AUTO) {
+        } else if (level == AMD_DPM_FORCED_LEVEL_AUTO) {
                 if (!pi->pcie_dpm_key_disabled) {
                         PPSMC_Result smc_result;
 
@@ -6262,7 +6271,7 @@ static int ci_dpm_sw_init(void *handle)
         /* default to balanced state */
         adev->pm.dpm.state = POWER_STATE_TYPE_BALANCED;
         adev->pm.dpm.user_state = POWER_STATE_TYPE_BALANCED;
-        adev->pm.dpm.forced_level = AMDGPU_DPM_FORCED_LEVEL_AUTO;
+        adev->pm.dpm.forced_level = AMD_DPM_FORCED_LEVEL_AUTO;
         adev->pm.default_sclk = adev->clock.default_sclk;
         adev->pm.default_mclk = adev->clock.default_mclk;
         adev->pm.current_sclk = adev->clock.default_sclk;
@@ -6571,8 +6580,9 @@ static int ci_dpm_force_clock_level(struct amdgpu_device *adev,
 {
         struct ci_power_info *pi = ci_get_pi(adev);
 
-        if (adev->pm.dpm.forced_level
-                        != AMDGPU_DPM_FORCED_LEVEL_MANUAL)
+        if (adev->pm.dpm.forced_level & (AMD_DPM_FORCED_LEVEL_AUTO |
+                                AMD_DPM_FORCED_LEVEL_LOW |
+                                AMD_DPM_FORCED_LEVEL_HIGH))
                 return -EINVAL;
 
         switch (type) {
@@ -6739,12 +6749,3 @@ static void ci_dpm_set_irq_funcs(struct amdgpu_device *adev)
         adev->pm.dpm.thermal.irq.num_types = AMDGPU_THERMAL_IRQ_LAST;
         adev->pm.dpm.thermal.irq.funcs = &ci_dpm_irq_funcs;
 }
-
-const struct amdgpu_ip_block_version ci_dpm_ip_block =
-{
-        .type = AMD_IP_BLOCK_TYPE_SMC,
-        .major = 7,
-        .minor = 0,
-        .rev = 0,
-        .funcs = &ci_dpm_ip_funcs,
-};
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 302df85893ab..7da688b0d27d 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -1627,14 +1627,13 @@ static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
 static void cik_detect_hw_virtualization(struct amdgpu_device *adev)
 {
         if (is_virtual_machine()) /* passthrough mode */
-                adev->virtualization.virtual_caps |= AMDGPU_PASSTHROUGH_MODE;
+                adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
 }
 
 static const struct amdgpu_asic_funcs cik_asic_funcs =
 {
         .read_disabled_bios = &cik_read_disabled_bios,
         .read_bios_from_rom = &cik_read_bios_from_rom,
-        .detect_hw_virtualization = cik_detect_hw_virtualization,
         .read_register = &cik_read_register,
         .reset = &cik_asic_reset,
         .set_vga_state = &cik_vga_set_state,
@@ -1890,6 +1889,8 @@ static const struct amdgpu_ip_block_version cik_common_ip_block =
 
 int cik_set_ip_blocks(struct amdgpu_device *adev)
 {
+        cik_detect_hw_virtualization(adev);
+
         switch (adev->asic_type) {
         case CHIP_BONAIRE:
                 amdgpu_ip_block_add(adev, &cik_common_ip_block);
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index 4c34dbc7a254..810bba533975 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -651,7 +651,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
         ib.ptr[3] = 1;
         ib.ptr[4] = 0xDEADBEEF;
         ib.length_dw = 5;
-        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, NULL, &f);
+        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
         if (r)
                 goto err1;
 
diff --git a/drivers/gpu/drm/amd/include/asic_reg/si/clearstate_si.h b/drivers/gpu/drm/amd/amdgpu/clearstate_si.h
index 66e39cdb5cb0..66e39cdb5cb0 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/si/clearstate_si.h
+++ b/drivers/gpu/drm/amd/amdgpu/clearstate_si.h
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
deleted file mode 100644
index ba2b66be9022..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
+++ /dev/null
@@ -1,2320 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include <linux/firmware.h>
-#include <linux/seq_file.h>
-#include "drmP.h"
-#include "amdgpu.h"
-#include "amdgpu_pm.h"
-#include "amdgpu_atombios.h"
-#include "vid.h"
-#include "vi_dpm.h"
-#include "amdgpu_dpm.h"
-#include "cz_dpm.h"
-#include "cz_ppsmc.h"
-#include "atom.h"
-
-#include "smu/smu_8_0_d.h"
-#include "smu/smu_8_0_sh_mask.h"
-#include "gca/gfx_8_0_d.h"
-#include "gca/gfx_8_0_sh_mask.h"
-#include "gmc/gmc_8_1_d.h"
-#include "bif/bif_5_1_d.h"
-#include "gfx_v8_0.h"
-
-static void cz_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate);
-static void cz_dpm_powergate_vce(struct amdgpu_device *adev, bool gate);
-static void cz_dpm_fini(struct amdgpu_device *adev);
-
-static struct cz_ps *cz_get_ps(struct amdgpu_ps *rps)
-{
-        struct cz_ps *ps = rps->ps_priv;
-
-        return ps;
-}
-
-static struct cz_power_info *cz_get_pi(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = adev->pm.dpm.priv;
-
-        return pi;
-}
-
-static uint16_t cz_convert_8bit_index_to_voltage(struct amdgpu_device *adev,
-                                                        uint16_t voltage)
-{
-        uint16_t tmp = 6200 - voltage * 25;
-
-        return tmp;
-}
-
-static void cz_construct_max_power_limits_table(struct amdgpu_device *adev,
-                                struct amdgpu_clock_and_voltage_limits *table)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_clock_voltage_dependency_table *dep_table =
-                &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
-
-        if (dep_table->count > 0) {
-                table->sclk = dep_table->entries[dep_table->count - 1].clk;
-                table->vddc = cz_convert_8bit_index_to_voltage(adev,
-                                dep_table->entries[dep_table->count - 1].v);
-        }
-
-        table->mclk = pi->sys_info.nbp_memory_clock[0];
-
-}
-
-union igp_info {
-        struct _ATOM_INTEGRATED_SYSTEM_INFO info;
-        struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7;
-        struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_8 info_8;
-        struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_9 info_9;
-};
-
-static int cz_parse_sys_info_table(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_mode_info *mode_info = &adev->mode_info;
-        int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
-        union igp_info *igp_info;
-        u8 frev, crev;
-        u16 data_offset;
-        int i = 0;
-
-        if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,
-                                   &frev, &crev, &data_offset)) {
-                igp_info = (union igp_info *)(mode_info->atom_context->bios +
-                                              data_offset);
-
-                if (crev != 9) {
-                        DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev);
-                        return -EINVAL;
-                }
-                pi->sys_info.bootup_sclk =
-                        le32_to_cpu(igp_info->info_9.ulBootUpEngineClock);
-                pi->sys_info.bootup_uma_clk =
-                        le32_to_cpu(igp_info->info_9.ulBootUpUMAClock);
-                pi->sys_info.dentist_vco_freq =
-                        le32_to_cpu(igp_info->info_9.ulDentistVCOFreq);
-                pi->sys_info.bootup_nb_voltage_index =
-                        le16_to_cpu(igp_info->info_9.usBootUpNBVoltage);
-
-                if (igp_info->info_9.ucHtcTmpLmt == 0)
-                        pi->sys_info.htc_tmp_lmt = 203;
-                else
-                        pi->sys_info.htc_tmp_lmt = igp_info->info_9.ucHtcTmpLmt;
-
-                if (igp_info->info_9.ucHtcHystLmt == 0)
-                        pi->sys_info.htc_hyst_lmt = 5;
-                else
-                        pi->sys_info.htc_hyst_lmt = igp_info->info_9.ucHtcHystLmt;
-
-                if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) {
-                        DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n");
-                        return -EINVAL;
-                }
-
-                if (le32_to_cpu(igp_info->info_9.ulSystemConfig) & (1 << 3) &&
-                                pi->enable_nb_ps_policy)
-                        pi->sys_info.nb_dpm_enable = true;
-                else
-                        pi->sys_info.nb_dpm_enable = false;
-
-                for (i = 0; i < CZ_NUM_NBPSTATES; i++) {
-                        if (i < CZ_NUM_NBPMEMORY_CLOCK)
-                                pi->sys_info.nbp_memory_clock[i] =
-                                le32_to_cpu(igp_info->info_9.ulNbpStateMemclkFreq[i]);
-                        pi->sys_info.nbp_n_clock[i] =
-                        le32_to_cpu(igp_info->info_9.ulNbpStateNClkFreq[i]);
-                }
-
-                for (i = 0; i < CZ_MAX_DISPLAY_CLOCK_LEVEL; i++)
-                        pi->sys_info.display_clock[i] =
-                        le32_to_cpu(igp_info->info_9.sDispClkVoltageMapping[i].ulMaximumSupportedCLK);
-
-                for (i = 0; i < CZ_NUM_NBPSTATES; i++)
-                        pi->sys_info.nbp_voltage_index[i] =
-                                le32_to_cpu(igp_info->info_9.usNBPStateVoltage[i]);
-
-                if (le32_to_cpu(igp_info->info_9.ulGPUCapInfo) &
-                        SYS_INFO_GPUCAPS__ENABEL_DFS_BYPASS)
-                        pi->caps_enable_dfs_bypass = true;
-
-                pi->sys_info.uma_channel_number =
-                        igp_info->info_9.ucUMAChannelNumber;
-
-                cz_construct_max_power_limits_table(adev,
-                        &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac);
-        }
-
-        return 0;
-}
-
-static void cz_patch_voltage_values(struct amdgpu_device *adev)
-{
-        int i;
-        struct amdgpu_uvd_clock_voltage_dependency_table *uvd_table =
-                &adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table;
-        struct amdgpu_vce_clock_voltage_dependency_table *vce_table =
-                &adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
-        struct amdgpu_clock_voltage_dependency_table *acp_table =
-                &adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table;
-
-        if (uvd_table->count) {
-                for (i = 0; i < uvd_table->count; i++)
-                        uvd_table->entries[i].v =
-                                cz_convert_8bit_index_to_voltage(adev,
-                                                uvd_table->entries[i].v);
-        }
-
-        if (vce_table->count) {
-                for (i = 0; i < vce_table->count; i++)
-                        vce_table->entries[i].v =
-                                cz_convert_8bit_index_to_voltage(adev,
-                                                vce_table->entries[i].v);
-        }
-
-        if (acp_table->count) {
-                for (i = 0; i < acp_table->count; i++)
-                        acp_table->entries[i].v =
-                                cz_convert_8bit_index_to_voltage(adev,
-                                                acp_table->entries[i].v);
-        }
-
-}
-
-static void cz_construct_boot_state(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        pi->boot_pl.sclk = pi->sys_info.bootup_sclk;
-        pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index;
-        pi->boot_pl.ds_divider_index = 0;
-        pi->boot_pl.ss_divider_index = 0;
-        pi->boot_pl.allow_gnb_slow = 1;
-        pi->boot_pl.force_nbp_state = 0;
-        pi->boot_pl.display_wm = 0;
-        pi->boot_pl.vce_wm = 0;
-
-}
-
-static void cz_patch_boot_state(struct amdgpu_device *adev,
-                                struct cz_ps *ps)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        ps->num_levels = 1;
-        ps->levels[0] = pi->boot_pl;
-}
-
-union pplib_clock_info {
-        struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
-        struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
-        struct _ATOM_PPLIB_CZ_CLOCK_INFO carrizo;
-};
-
-static void cz_parse_pplib_clock_info(struct amdgpu_device *adev,
-                                        struct amdgpu_ps *rps, int index,
-                                        union pplib_clock_info *clock_info)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct cz_ps *ps = cz_get_ps(rps);
-        struct cz_pl *pl = &ps->levels[index];
-        struct amdgpu_clock_voltage_dependency_table *table =
-                        &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
-
-        pl->sclk = table->entries[clock_info->carrizo.index].clk;
-        pl->vddc_index = table->entries[clock_info->carrizo.index].v;
-
-        ps->num_levels = index + 1;
-
-        if (pi->caps_sclk_ds) {
-                pl->ds_divider_index = 5;
-                pl->ss_divider_index = 5;
-        }
-
-}
-
-static void cz_parse_pplib_non_clock_info(struct amdgpu_device *adev,
-                        struct amdgpu_ps *rps,
-                        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
-                        u8 table_rev)
-{
-        struct cz_ps *ps = cz_get_ps(rps);
-
-        rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
-        rps->class = le16_to_cpu(non_clock_info->usClassification);
-        rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
-
-        if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
-                rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
-                rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
-        } else {
-                rps->vclk = 0;
-                rps->dclk = 0;
-        }
-
-        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
-                adev->pm.dpm.boot_ps = rps;
-                cz_patch_boot_state(adev, ps);
-        }
-        if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
-                adev->pm.dpm.uvd_ps = rps;
-
-}
-
-union power_info {
-        struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
-        struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
-        struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
-        struct _ATOM_PPLIB_POWERPLAYTABLE4 pplib4;
-        struct _ATOM_PPLIB_POWERPLAYTABLE5 pplib5;
-};
-
-union pplib_power_state {
-        struct _ATOM_PPLIB_STATE v1;
-        struct _ATOM_PPLIB_STATE_V2 v2;
-};
-
-static int cz_parse_power_table(struct amdgpu_device *adev)
-{
-        struct amdgpu_mode_info *mode_info = &adev->mode_info;
-        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
-        union pplib_power_state *power_state;
-        int i, j, k, non_clock_array_index, clock_array_index;
-        union pplib_clock_info *clock_info;
-        struct _StateArray *state_array;
-        struct _ClockInfoArray *clock_info_array;
-        struct _NonClockInfoArray *non_clock_info_array;
-        union power_info *power_info;
-        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
-        u16 data_offset;
-        u8 frev, crev;
-        u8 *power_state_offset;
-        struct cz_ps *ps;
-
-        if (!amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,
-                                    &frev, &crev, &data_offset))
-                return -EINVAL;
-        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
-
-        state_array = (struct _StateArray *)
-                (mode_info->atom_context->bios + data_offset +
-                le16_to_cpu(power_info->pplib.usStateArrayOffset));
-        clock_info_array = (struct _ClockInfoArray *)
-                (mode_info->atom_context->bios + data_offset +
-                le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
-        non_clock_info_array = (struct _NonClockInfoArray *)
-                (mode_info->atom_context->bios + data_offset +
-                le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
-
-        adev->pm.dpm.ps = kzalloc(sizeof(struct amdgpu_ps) *
-                                        state_array->ucNumEntries, GFP_KERNEL);
-
-        if (!adev->pm.dpm.ps)
-                return -ENOMEM;
-
-        power_state_offset = (u8 *)state_array->states;
-        adev->pm.dpm.platform_caps =
-                        le32_to_cpu(power_info->pplib.ulPlatformCaps);
-        adev->pm.dpm.backbias_response_time =
-                        le16_to_cpu(power_info->pplib.usBackbiasTime);
-        adev->pm.dpm.voltage_response_time =
-                        le16_to_cpu(power_info->pplib.usVoltageTime);
-
-        for (i = 0; i < state_array->ucNumEntries; i++) {
-                power_state = (union pplib_power_state *)power_state_offset;
-                non_clock_array_index = power_state->v2.nonClockInfoIndex;
-                non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
-                        &non_clock_info_array->nonClockInfo[non_clock_array_index];
-
-                ps = kzalloc(sizeof(struct cz_ps), GFP_KERNEL);
-                if (ps == NULL) {
-                        for (j = 0; j < i; j++)
-                                kfree(adev->pm.dpm.ps[j].ps_priv);
-                        kfree(adev->pm.dpm.ps);
-                        return -ENOMEM;
-                }
-
-                adev->pm.dpm.ps[i].ps_priv = ps;
-                k = 0;
-                for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
-                        clock_array_index = power_state->v2.clockInfoIndex[j];
-                        if (clock_array_index >= clock_info_array->ucNumEntries)
-                                continue;
-                        if (k >= CZ_MAX_HARDWARE_POWERLEVELS)
-                                break;
-                        clock_info = (union pplib_clock_info *)
-                                &clock_info_array->clockInfo[clock_array_index *
-                                clock_info_array->ucEntrySize];
-                        cz_parse_pplib_clock_info(adev, &adev->pm.dpm.ps[i],
-                                k, clock_info);
-                        k++;
-                }
-                cz_parse_pplib_non_clock_info(adev, &adev->pm.dpm.ps[i],
-                                        non_clock_info,
-                                        non_clock_info_array->ucEntrySize);
-                power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
-        }
-        adev->pm.dpm.num_ps = state_array->ucNumEntries;
-
-        return 0;
-}
-
-static int cz_process_firmware_header(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        u32 tmp;
-        int ret;
-
-        ret = cz_read_smc_sram_dword(adev, SMU8_FIRMWARE_HEADER_LOCATION +
-                                     offsetof(struct SMU8_Firmware_Header,
-                                     DpmTable),
-                                     &tmp, pi->sram_end);
-
-        if (ret == 0)
-                pi->dpm_table_start = tmp;
-
-        return ret;
-}
-
-static int cz_dpm_init(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi;
-        int ret, i;
-
-        pi = kzalloc(sizeof(struct cz_power_info), GFP_KERNEL);
-        if (NULL == pi)
-                return -ENOMEM;
-
-        adev->pm.dpm.priv = pi;
-
-        ret = amdgpu_get_platform_caps(adev);
-        if (ret)
-                goto err;
-
-        ret = amdgpu_parse_extended_power_table(adev);
-        if (ret)
-                goto err;
-
-        pi->sram_end = SMC_RAM_END;
-
-        /* set up DPM defaults */
-        for (i = 0; i < CZ_MAX_HARDWARE_POWERLEVELS; i++)
-                pi->active_target[i] = CZ_AT_DFLT;
-
-        pi->mgcg_cgtt_local0 = 0x0;
-        pi->mgcg_cgtt_local1 = 0x0;
-        pi->clock_slow_down_step = 25000;
-        pi->skip_clock_slow_down = 1;
-        pi->enable_nb_ps_policy = false;
-        pi->caps_power_containment = true;
-        pi->caps_cac = true;
-        pi->didt_enabled = false;
-        if (pi->didt_enabled) {
-                pi->caps_sq_ramping = true;
-                pi->caps_db_ramping = true;
-                pi->caps_td_ramping = true;
-                pi->caps_tcp_ramping = true;
-        }
-        if (amdgpu_pp_feature_mask & SCLK_DEEP_SLEEP_MASK)
-                pi->caps_sclk_ds = true;
-        else
-                pi->caps_sclk_ds = false;
-
-        pi->voting_clients = 0x00c00033;
-        pi->auto_thermal_throttling_enabled = true;
-        pi->bapm_enabled = false;
-        pi->disable_nb_ps3_in_battery = false;
-        pi->voltage_drop_threshold = 0;
-        pi->caps_sclk_throttle_low_notification = false;
-        pi->gfx_pg_threshold = 500;
-        pi->caps_fps = true;
-        /* uvd */
-        pi->caps_uvd_pg = (adev->pg_flags & AMD_PG_SUPPORT_UVD) ? true : false;
-        pi->caps_uvd_dpm = true;
-        /* vce */
-        pi->caps_vce_pg = (adev->pg_flags & AMD_PG_SUPPORT_VCE) ? true : false;
-        pi->caps_vce_dpm = true;
-        /* acp */
-        pi->caps_acp_pg = (adev->pg_flags & AMD_PG_SUPPORT_ACP) ? true : false;
-        pi->caps_acp_dpm = true;
-
-        pi->caps_stable_power_state = false;
-        pi->nb_dpm_enabled_by_driver = true;
-        pi->nb_dpm_enabled = false;
-        pi->caps_voltage_island = false;
-        /* flags which indicate need to upload pptable */
-        pi->need_pptable_upload = true;
-
-        ret = cz_parse_sys_info_table(adev);
-        if (ret)
-                goto err;
-
-        cz_patch_voltage_values(adev);
-        cz_construct_boot_state(adev);
-
-        ret = cz_parse_power_table(adev);
-        if (ret)
-                goto err;
-
-        ret = cz_process_firmware_header(adev);
-        if (ret)
-                goto err;
-
-        pi->dpm_enabled = true;
-        pi->uvd_dynamic_pg = false;
-
-        return 0;
-err:
-        cz_dpm_fini(adev);
-        return ret;
-}
-
-static void cz_dpm_fini(struct amdgpu_device *adev)
-{
-        int i;
-
-        for (i = 0; i < adev->pm.dpm.num_ps; i++)
-                kfree(adev->pm.dpm.ps[i].ps_priv);
-
-        kfree(adev->pm.dpm.ps);
-        kfree(adev->pm.dpm.priv);
-        amdgpu_free_extended_power_table(adev);
-}
-
-#define ixSMUSVI_NB_CURRENTVID 0xD8230044
-#define CURRENT_NB_VID_MASK 0xff000000
-#define CURRENT_NB_VID__SHIFT 24
-#define ixSMUSVI_GFX_CURRENTVID  0xD8230048
-#define CURRENT_GFX_VID_MASK 0xff000000
-#define CURRENT_GFX_VID__SHIFT 24
-
-static void
-cz_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
-                                               struct seq_file *m)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_clock_voltage_dependency_table *table =
-                &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
-        struct amdgpu_uvd_clock_voltage_dependency_table *uvd_table =
-                &adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table;
-        struct amdgpu_vce_clock_voltage_dependency_table *vce_table =
-                &adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
-        u32 sclk_index = REG_GET_FIELD(RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX),
-                                       TARGET_AND_CURRENT_PROFILE_INDEX, CURR_SCLK_INDEX);
-        u32 uvd_index = REG_GET_FIELD(RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX_2),
-                                      TARGET_AND_CURRENT_PROFILE_INDEX_2, CURR_UVD_INDEX);
-        u32 vce_index = REG_GET_FIELD(RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX_2),
-                                      TARGET_AND_CURRENT_PROFILE_INDEX_2, CURR_VCE_INDEX);
-        u32 sclk, vclk, dclk, ecclk, tmp;
-        u16 vddnb, vddgfx;
-
-        if (sclk_index >= NUM_SCLK_LEVELS) {
-                seq_printf(m, "invalid sclk dpm profile %d\n", sclk_index);
-        } else {
-                sclk = table->entries[sclk_index].clk;
-                seq_printf(m, "%u sclk: %u\n", sclk_index, sclk);
-        }
-
-        tmp = (RREG32_SMC(ixSMUSVI_NB_CURRENTVID) &
-               CURRENT_NB_VID_MASK) >> CURRENT_NB_VID__SHIFT;
-        vddnb = cz_convert_8bit_index_to_voltage(adev, (u16)tmp);
-        tmp = (RREG32_SMC(ixSMUSVI_GFX_CURRENTVID) &
-               CURRENT_GFX_VID_MASK) >> CURRENT_GFX_VID__SHIFT;
-        vddgfx = cz_convert_8bit_index_to_voltage(adev, (u16)tmp);
-        seq_printf(m, "vddnb: %u vddgfx: %u\n", vddnb, vddgfx);
-
-        seq_printf(m, "uvd    %sabled\n", pi->uvd_power_gated ? "dis" : "en");
-        if (!pi->uvd_power_gated) {
-                if (uvd_index >= CZ_MAX_HARDWARE_POWERLEVELS) {
-                        seq_printf(m, "invalid uvd dpm level %d\n", uvd_index);
-                } else {
-                        vclk = uvd_table->entries[uvd_index].vclk;
-                        dclk = uvd_table->entries[uvd_index].dclk;
-                        seq_printf(m, "%u uvd vclk: %u dclk: %u\n", uvd_index, vclk, dclk);
-                }
-        }
-
-        seq_printf(m, "vce    %sabled\n", pi->vce_power_gated ? "dis" : "en");
-        if (!pi->vce_power_gated) {
-                if (vce_index >= CZ_MAX_HARDWARE_POWERLEVELS) {
-                        seq_printf(m, "invalid vce dpm level %d\n", vce_index);
-                } else {
-                        ecclk = vce_table->entries[vce_index].ecclk;
-                        seq_printf(m, "%u vce ecclk: %u\n", vce_index, ecclk);
-                }
-        }
-}
-
-static void cz_dpm_print_power_state(struct amdgpu_device *adev,
-                                        struct amdgpu_ps *rps)
-{
-        int i;
-        struct cz_ps *ps = cz_get_ps(rps);
-
-        amdgpu_dpm_print_class_info(rps->class, rps->class2);
-        amdgpu_dpm_print_cap_info(rps->caps);
-
-        DRM_INFO("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
-        for (i = 0; i < ps->num_levels; i++) {
-                struct cz_pl *pl = &ps->levels[i];
-
-                DRM_INFO("\t\tpower level %d    sclk: %u vddc: %u\n",
-                       i, pl->sclk,
-                       cz_convert_8bit_index_to_voltage(adev, pl->vddc_index));
-        }
-
-        amdgpu_dpm_print_ps_status(adev, rps);
-}
-
-static void cz_dpm_set_funcs(struct amdgpu_device *adev);
-
-static int cz_dpm_early_init(void *handle)
-{
-        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-        cz_dpm_set_funcs(adev);
-
-        return 0;
-}
-
-
-static int cz_dpm_late_init(void *handle)
-{
-        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-        if (amdgpu_dpm) {
-                int ret;
-                /* init the sysfs and debugfs files late */
-                ret = amdgpu_pm_sysfs_init(adev);
-                if (ret)
-                        return ret;
-
-                /* powerdown unused blocks for now */
-                cz_dpm_powergate_uvd(adev, true);
-                cz_dpm_powergate_vce(adev, true);
-        }
-
-        return 0;
-}
-
-static int cz_dpm_sw_init(void *handle)
-{
-        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-        int ret = 0;
-        /* fix me to add thermal support TODO */
-
-        /* default to balanced state */
-        adev->pm.dpm.state = POWER_STATE_TYPE_BALANCED;
-        adev->pm.dpm.user_state = POWER_STATE_TYPE_BALANCED;
-        adev->pm.dpm.forced_level = AMDGPU_DPM_FORCED_LEVEL_AUTO;
-        adev->pm.default_sclk = adev->clock.default_sclk;
-        adev->pm.default_mclk = adev->clock.default_mclk;
-        adev->pm.current_sclk = adev->clock.default_sclk;
-        adev->pm.current_mclk = adev->clock.default_mclk;
-        adev->pm.int_thermal_type = THERMAL_TYPE_NONE;
-
-        if (amdgpu_dpm == 0)
-                return 0;
-
-        mutex_lock(&adev->pm.mutex);
-        ret = cz_dpm_init(adev);
-        if (ret)
-                goto dpm_init_failed;
-
-        adev->pm.dpm.current_ps = adev->pm.dpm.requested_ps = adev->pm.dpm.boot_ps;
-        if (amdgpu_dpm == 1)
-                amdgpu_pm_print_power_states(adev);
-
-        mutex_unlock(&adev->pm.mutex);
-        DRM_INFO("amdgpu: dpm initialized\n");
-
-        return 0;
-
-dpm_init_failed:
-        cz_dpm_fini(adev);
-        mutex_unlock(&adev->pm.mutex);
-        DRM_ERROR("amdgpu: dpm initialization failed\n");
-
-        return ret;
-}
-
-static int cz_dpm_sw_fini(void *handle)
-{
-        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-        mutex_lock(&adev->pm.mutex);
-        amdgpu_pm_sysfs_fini(adev);
-        cz_dpm_fini(adev);
-        mutex_unlock(&adev->pm.mutex);
-
-        return 0;
-}
-
-static void cz_reset_ap_mask(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        pi->active_process_mask = 0;
-}
-
-static int cz_dpm_download_pptable_from_smu(struct amdgpu_device *adev,
-                                                        void **table)
-{
-        return cz_smu_download_pptable(adev, table);
-}
-
-static int cz_dpm_upload_pptable_to_smu(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct SMU8_Fusion_ClkTable *clock_table;
-        struct atom_clock_dividers dividers;
-        void *table = NULL;
-        uint8_t i = 0;
-        int ret = 0;
-
-        struct amdgpu_clock_voltage_dependency_table *vddc_table =
-                &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
-        struct amdgpu_clock_voltage_dependency_table *vddgfx_table =
-                &adev->pm.dpm.dyn_state.vddgfx_dependency_on_sclk;
-        struct amdgpu_uvd_clock_voltage_dependency_table *uvd_table =
-                &adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table;
-        struct amdgpu_vce_clock_voltage_dependency_table *vce_table =
-                &adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
-        struct amdgpu_clock_voltage_dependency_table *acp_table =
-                &adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table;
-
-        if (!pi->need_pptable_upload)
-                return 0;
-
-        ret = cz_dpm_download_pptable_from_smu(adev, &table);
-        if (ret) {
-                DRM_ERROR("amdgpu: Failed to get power play table from SMU!\n");
-                return -EINVAL;
-        }
-
-        clock_table = (struct SMU8_Fusion_ClkTable *)table;
-        /* patch clock table */
-        if (vddc_table->count > CZ_MAX_HARDWARE_POWERLEVELS ||
-                        vddgfx_table->count > CZ_MAX_HARDWARE_POWERLEVELS ||
-                        uvd_table->count > CZ_MAX_HARDWARE_POWERLEVELS ||
-                        vce_table->count > CZ_MAX_HARDWARE_POWERLEVELS ||
-                        acp_table->count > CZ_MAX_HARDWARE_POWERLEVELS) {
-                DRM_ERROR("amdgpu: Invalid Clock Voltage Dependency Table!\n");
-                return -EINVAL;
-        }
-
-        for (i = 0; i < CZ_MAX_HARDWARE_POWERLEVELS; i++) {
-
-                /* vddc sclk */
-                clock_table->SclkBreakdownTable.ClkLevel[i].GnbVid =
-                        (i < vddc_table->count) ? (uint8_t)vddc_table->entries[i].v : 0;
-                clock_table->SclkBreakdownTable.ClkLevel[i].Frequency =
-                        (i < vddc_table->count) ? vddc_table->entries[i].clk : 0;
-                ret = amdgpu_atombios_get_clock_dividers(adev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
-                                clock_table->SclkBreakdownTable.ClkLevel[i].Frequency,
-                                false, &dividers);
-                if (ret)
-                        return ret;
-                clock_table->SclkBreakdownTable.ClkLevel[i].DfsDid =
-                                                (uint8_t)dividers.post_divider;
-
-                /* vddgfx sclk */
-                clock_table->SclkBreakdownTable.ClkLevel[i].GfxVid =
-                        (i < vddgfx_table->count) ? (uint8_t)vddgfx_table->entries[i].v : 0;
-
-                /* acp breakdown */
-                clock_table->AclkBreakdownTable.ClkLevel[i].GfxVid =
-                        (i < acp_table->count) ? (uint8_t)acp_table->entries[i].v : 0;
-                clock_table->AclkBreakdownTable.ClkLevel[i].Frequency =
-                        (i < acp_table->count) ? acp_table->entries[i].clk : 0;
-                ret = amdgpu_atombios_get_clock_dividers(adev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
-                                clock_table->SclkBreakdownTable.ClkLevel[i].Frequency,
-                                false, &dividers);
-                if (ret)
-                        return ret;
-                clock_table->AclkBreakdownTable.ClkLevel[i].DfsDid =
-                                                (uint8_t)dividers.post_divider;
-
-                /* uvd breakdown */
-                clock_table->VclkBreakdownTable.ClkLevel[i].GfxVid =
-                        (i < uvd_table->count) ? (uint8_t)uvd_table->entries[i].v : 0;
-                clock_table->VclkBreakdownTable.ClkLevel[i].Frequency =
-                        (i < uvd_table->count) ? uvd_table->entries[i].vclk : 0;
-                ret = amdgpu_atombios_get_clock_dividers(adev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
-                                clock_table->VclkBreakdownTable.ClkLevel[i].Frequency,
-                                false, &dividers);
-                if (ret)
-                        return ret;
-                clock_table->VclkBreakdownTable.ClkLevel[i].DfsDid =
-                                                (uint8_t)dividers.post_divider;
-
-                clock_table->DclkBreakdownTable.ClkLevel[i].GfxVid =
-                        (i < uvd_table->count) ? (uint8_t)uvd_table->entries[i].v : 0;
-                clock_table->DclkBreakdownTable.ClkLevel[i].Frequency =
-                        (i < uvd_table->count) ? uvd_table->entries[i].dclk : 0;
-                ret = amdgpu_atombios_get_clock_dividers(adev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
-                                clock_table->DclkBreakdownTable.ClkLevel[i].Frequency,
-                                false, &dividers);
-                if (ret)
-                        return ret;
-                clock_table->DclkBreakdownTable.ClkLevel[i].DfsDid =
-                                                (uint8_t)dividers.post_divider;
-
-                /* vce breakdown */
-                clock_table->EclkBreakdownTable.ClkLevel[i].GfxVid =
-                        (i < vce_table->count) ? (uint8_t)vce_table->entries[i].v : 0;
-                clock_table->EclkBreakdownTable.ClkLevel[i].Frequency =
-                        (i < vce_table->count) ? vce_table->entries[i].ecclk : 0;
-                ret = amdgpu_atombios_get_clock_dividers(adev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
-                                clock_table->EclkBreakdownTable.ClkLevel[i].Frequency,
-                                false, &dividers);
-                if (ret)
-                        return ret;
-                clock_table->EclkBreakdownTable.ClkLevel[i].DfsDid =
-                                                (uint8_t)dividers.post_divider;
-        }
-
-        /* its time to upload to SMU */
-        ret = cz_smu_upload_pptable(adev);
-        if (ret) {
-                DRM_ERROR("amdgpu: Failed to put power play table to SMU!\n");
-                return ret;
-        }
-
-        return 0;
-}
-
-static void cz_init_sclk_limit(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_clock_voltage_dependency_table *table =
-                &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
-        uint32_t clock = 0, level;
-
-        if (!table || !table->count) {
-                DRM_ERROR("Invalid Voltage Dependency table.\n");
-                return;
-        }
-
-        pi->sclk_dpm.soft_min_clk = 0;
-        pi->sclk_dpm.hard_min_clk = 0;
-        cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxSclkLevel);
-        level = cz_get_argument(adev);
-        if (level < table->count) {
-                clock = table->entries[level].clk;
-        } else {
-                DRM_ERROR("Invalid SLCK Voltage Dependency table entry.\n");
-                clock = table->entries[table->count - 1].clk;
-        }
-
-        pi->sclk_dpm.soft_max_clk = clock;
-        pi->sclk_dpm.hard_max_clk = clock;
-
-}
-
-static void cz_init_uvd_limit(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_uvd_clock_voltage_dependency_table *table =
-                &adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table;
-        uint32_t clock = 0, level;
-
-        if (!table || !table->count) {
-                DRM_ERROR("Invalid Voltage Dependency table.\n");
-                return;
-        }
-
-        pi->uvd_dpm.soft_min_clk = 0;
-        pi->uvd_dpm.hard_min_clk = 0;
-        cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxUvdLevel);
-        level = cz_get_argument(adev);
-        if (level < table->count) {
-                clock = table->entries[level].vclk;
-        } else {
-                DRM_ERROR("Invalid UVD Voltage Dependency table entry.\n");
-                clock = table->entries[table->count - 1].vclk;
-        }
-
-        pi->uvd_dpm.soft_max_clk = clock;
-        pi->uvd_dpm.hard_max_clk = clock;
-
-}
-
-static void cz_init_vce_limit(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_vce_clock_voltage_dependency_table *table =
-                &adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
-        uint32_t clock = 0, level;
-
-        if (!table || !table->count) {
-                DRM_ERROR("Invalid Voltage Dependency table.\n");
-                return;
-        }
-
-        pi->vce_dpm.soft_min_clk = table->entries[0].ecclk;
-        pi->vce_dpm.hard_min_clk = table->entries[0].ecclk;
-        cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxEclkLevel);
-        level = cz_get_argument(adev);
-        if (level < table->count) {
-                clock = table->entries[level].ecclk;
-        } else {
-                /* future BIOS would fix this error */
-                DRM_ERROR("Invalid VCE Voltage Dependency table entry.\n");
-                clock = table->entries[table->count - 1].ecclk;
-        }
-
-        pi->vce_dpm.soft_max_clk = clock;
-        pi->vce_dpm.hard_max_clk = clock;
-
-}
-
-static void cz_init_acp_limit(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_clock_voltage_dependency_table *table =
-                &adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table;
-        uint32_t clock = 0, level;
-
-        if (!table || !table->count) {
-                DRM_ERROR("Invalid Voltage Dependency table.\n");
-                return;
-        }
-
-        pi->acp_dpm.soft_min_clk = 0;
-        pi->acp_dpm.hard_min_clk = 0;
-        cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxAclkLevel);
-        level = cz_get_argument(adev);
-        if (level < table->count) {
-                clock = table->entries[level].clk;
-        } else {
-                DRM_ERROR("Invalid ACP Voltage Dependency table entry.\n");
-                clock = table->entries[table->count - 1].clk;
-        }
-
-        pi->acp_dpm.soft_max_clk = clock;
-        pi->acp_dpm.hard_max_clk = clock;
-
-}
-
-static void cz_init_pg_state(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        pi->uvd_power_gated = false;
-        pi->vce_power_gated = false;
-        pi->acp_power_gated = false;
-
-}
-
-static void cz_init_sclk_threshold(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        pi->low_sclk_interrupt_threshold = 0;
-}
-
-static void cz_dpm_setup_asic(struct amdgpu_device *adev)
-{
-        cz_reset_ap_mask(adev);
-        cz_dpm_upload_pptable_to_smu(adev);
-        cz_init_sclk_limit(adev);
-        cz_init_uvd_limit(adev);
-        cz_init_vce_limit(adev);
-        cz_init_acp_limit(adev);
-        cz_init_pg_state(adev);
-        cz_init_sclk_threshold(adev);
-
-}
-
-static bool cz_check_smu_feature(struct amdgpu_device *adev,
-                                uint32_t feature)
-{
-        uint32_t smu_feature = 0;
-        int ret;
-
-        ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_GetFeatureStatus, 0);
-        if (ret) {
-                DRM_ERROR("Failed to get SMU features from SMC.\n");
-                return false;
-        } else {
-                smu_feature = cz_get_argument(adev);
-                if (feature & smu_feature)
-                        return true;
-        }
-
-        return false;
-}
-
-static bool cz_check_for_dpm_enabled(struct amdgpu_device *adev)
-{
-        if (cz_check_smu_feature(adev,
-                                SMU_EnabledFeatureScoreboard_SclkDpmOn))
-                return true;
-
-        return false;
-}
-
-static void cz_program_voting_clients(struct amdgpu_device *adev)
-{
-        WREG32_SMC(ixCG_FREQ_TRAN_VOTING_0, PPCZ_VOTINGRIGHTSCLIENTS_DFLT0);
-}
-
-static void cz_clear_voting_clients(struct amdgpu_device *adev)
-{
-        WREG32_SMC(ixCG_FREQ_TRAN_VOTING_0, 0);
-}
-
-static int cz_start_dpm(struct amdgpu_device *adev)
-{
-        int ret = 0;
-
-        if (amdgpu_dpm) {
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_EnableAllSmuFeatures, SCLK_DPM_MASK);
-                if (ret) {
-                        DRM_ERROR("SMU feature: SCLK_DPM enable failed\n");
-                        return -EINVAL;
-                }
-        }
-
-        return 0;
-}
-
-static int cz_stop_dpm(struct amdgpu_device *adev)
-{
-        int ret = 0;
-
-        if (amdgpu_dpm && adev->pm.dpm_enabled) {
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_DisableAllSmuFeatures, SCLK_DPM_MASK);
-                if (ret) {
-                        DRM_ERROR("SMU feature: SCLK_DPM disable failed\n");
-                        return -EINVAL;
-                }
-        }
-
-        return 0;
-}
-
-static uint32_t cz_get_sclk_level(struct amdgpu_device *adev,
-                                uint32_t clock, uint16_t msg)
-{
-        int i = 0;
-        struct amdgpu_clock_voltage_dependency_table *table =
-                &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
-
-        switch (msg) {
-        case PPSMC_MSG_SetSclkSoftMin:
-        case PPSMC_MSG_SetSclkHardMin:
-                for (i = 0; i < table->count; i++)
-                        if (clock <= table->entries[i].clk)
-                                break;
-                if (i == table->count)
-                        i = table->count - 1;
-                break;
-        case PPSMC_MSG_SetSclkSoftMax:
-        case PPSMC_MSG_SetSclkHardMax:
-                for (i = table->count - 1; i >= 0; i--)
-                        if (clock >= table->entries[i].clk)
-                                break;
-                if (i < 0)
-                        i = 0;
-                break;
-        default:
-                break;
-        }
-
-        return i;
-}
-
-static uint32_t cz_get_eclk_level(struct amdgpu_device *adev,
-                                uint32_t clock, uint16_t msg)
-{
-        int i = 0;
-        struct amdgpu_vce_clock_voltage_dependency_table *table =
-                &adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
-
-        if (table->count == 0)
-                return 0;
-
-        switch (msg) {
-        case PPSMC_MSG_SetEclkSoftMin:
-        case PPSMC_MSG_SetEclkHardMin:
-                for (i = 0; i < table->count-1; i++)
-                        if (clock <= table->entries[i].ecclk)
-                                break;
-                break;
-        case PPSMC_MSG_SetEclkSoftMax:
-        case PPSMC_MSG_SetEclkHardMax:
-                for (i = table->count - 1; i > 0; i--)
-                        if (clock >= table->entries[i].ecclk)
-                                break;
-                break;
-        default:
-                break;
-        }
-
-        return i;
-}
-
-static uint32_t cz_get_uvd_level(struct amdgpu_device *adev,
-                                 uint32_t clock, uint16_t msg)
-{
-        int i = 0;
-        struct amdgpu_uvd_clock_voltage_dependency_table *table =
-                &adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table;
-
-        switch (msg) {
-        case PPSMC_MSG_SetUvdSoftMin:
-        case PPSMC_MSG_SetUvdHardMin:
-                for (i = 0; i < table->count; i++)
-                        if (clock <= table->entries[i].vclk)
-                                break;
-                if (i == table->count)
-                        i = table->count - 1;
-                break;
-        case PPSMC_MSG_SetUvdSoftMax:
-        case PPSMC_MSG_SetUvdHardMax:
-                for (i = table->count - 1; i >= 0; i--)
-                        if (clock >= table->entries[i].vclk)
-                                break;
-                if (i < 0)
-                        i = 0;
-                break;
-        default:
-                break;
-        }
-
-        return i;
-}
-
-static int cz_program_bootup_state(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        uint32_t soft_min_clk = 0;
-        uint32_t soft_max_clk = 0;
-        int ret = 0;
-
-        pi->sclk_dpm.soft_min_clk = pi->sys_info.bootup_sclk;
-        pi->sclk_dpm.soft_max_clk = pi->sys_info.bootup_sclk;
-
-        soft_min_clk = cz_get_sclk_level(adev,
-                                pi->sclk_dpm.soft_min_clk,
-                                PPSMC_MSG_SetSclkSoftMin);
-        soft_max_clk = cz_get_sclk_level(adev,
-                                pi->sclk_dpm.soft_max_clk,
-                                PPSMC_MSG_SetSclkSoftMax);
-
-        ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetSclkSoftMin, soft_min_clk);
-        if (ret)
-                return -EINVAL;
-
-        ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetSclkSoftMax, soft_max_clk);
-        if (ret)
-                return -EINVAL;
-
-        return 0;
-}
-
-/* TODO */
-static int cz_disable_cgpg(struct amdgpu_device *adev)
-{
-        return 0;
-}
-
-/* TODO */
-static int cz_enable_cgpg(struct amdgpu_device *adev)
-{
-        return 0;
-}
-
-/* TODO */
-static int cz_program_pt_config_registers(struct amdgpu_device *adev)
-{
-        return 0;
-}
-
-static void cz_do_enable_didt(struct amdgpu_device *adev, bool enable)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        uint32_t reg = 0;
-
-        if (pi->caps_sq_ramping) {
-                reg = RREG32_DIDT(ixDIDT_SQ_CTRL0);
-                if (enable)
-                        reg = REG_SET_FIELD(reg, DIDT_SQ_CTRL0, DIDT_CTRL_EN, 1);
-                else
-                        reg = REG_SET_FIELD(reg, DIDT_SQ_CTRL0, DIDT_CTRL_EN, 0);
-                WREG32_DIDT(ixDIDT_SQ_CTRL0, reg);
-        }
-        if (pi->caps_db_ramping) {
-                reg = RREG32_DIDT(ixDIDT_DB_CTRL0);
-                if (enable)
-                        reg = REG_SET_FIELD(reg, DIDT_DB_CTRL0, DIDT_CTRL_EN, 1);
-                else
-                        reg = REG_SET_FIELD(reg, DIDT_DB_CTRL0, DIDT_CTRL_EN, 0);
-                WREG32_DIDT(ixDIDT_DB_CTRL0, reg);
-        }
-        if (pi->caps_td_ramping) {
-                reg = RREG32_DIDT(ixDIDT_TD_CTRL0);
-                if (enable)
-                        reg = REG_SET_FIELD(reg, DIDT_TD_CTRL0, DIDT_CTRL_EN, 1);
-                else
-                        reg = REG_SET_FIELD(reg, DIDT_TD_CTRL0, DIDT_CTRL_EN, 0);
-                WREG32_DIDT(ixDIDT_TD_CTRL0, reg);
-        }
-        if (pi->caps_tcp_ramping) {
-                reg = RREG32_DIDT(ixDIDT_TCP_CTRL0);
-                if (enable)
-                        reg = REG_SET_FIELD(reg, DIDT_SQ_CTRL0, DIDT_CTRL_EN, 1);
-                else
-                        reg = REG_SET_FIELD(reg, DIDT_SQ_CTRL0, DIDT_CTRL_EN, 0);
-                WREG32_DIDT(ixDIDT_TCP_CTRL0, reg);
-        }
-
-}
-
-static int cz_enable_didt(struct amdgpu_device *adev, bool enable)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        int ret;
-
-        if (pi->caps_sq_ramping || pi->caps_db_ramping ||
-            pi->caps_td_ramping || pi->caps_tcp_ramping) {
-                if (adev->gfx.gfx_current_status != AMDGPU_GFX_SAFE_MODE) {
-                        ret = cz_disable_cgpg(adev);
-                        if (ret) {
-                                DRM_ERROR("Pre Di/Dt disable cg/pg failed\n");
-                                return -EINVAL;
-                        }
-                        adev->gfx.gfx_current_status = AMDGPU_GFX_SAFE_MODE;
-                }
-
-                ret = cz_program_pt_config_registers(adev);
-                if (ret) {
-                        DRM_ERROR("Di/Dt config failed\n");
-                        return -EINVAL;
-                }
-                cz_do_enable_didt(adev, enable);
-
-                if (adev->gfx.gfx_current_status == AMDGPU_GFX_SAFE_MODE) {
-                        ret = cz_enable_cgpg(adev);
-                        if (ret) {
-                                DRM_ERROR("Post Di/Dt enable cg/pg failed\n");
-                                return -EINVAL;
-                        }
-                        adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
-                }
-        }
-
-        return 0;
-}
-
-/* TODO */
-static void cz_reset_acp_boot_level(struct amdgpu_device *adev)
-{
-}
-
-static void cz_update_current_ps(struct amdgpu_device *adev,
-                                        struct amdgpu_ps *rps)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct cz_ps *ps = cz_get_ps(rps);
-
-        pi->current_ps = *ps;
-        pi->current_rps = *rps;
-        pi->current_rps.ps_priv = &pi->current_ps;
-        adev->pm.dpm.current_ps = &pi->current_rps;
-
-}
-
-static void cz_update_requested_ps(struct amdgpu_device *adev,
-                                        struct amdgpu_ps *rps)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct cz_ps *ps = cz_get_ps(rps);
-
-        pi->requested_ps = *ps;
-        pi->requested_rps = *rps;
-        pi->requested_rps.ps_priv = &pi->requested_ps;
-        adev->pm.dpm.requested_ps = &pi->requested_rps;
-
-}
-
-/* PP arbiter support needed TODO */
-static void cz_apply_state_adjust_rules(struct amdgpu_device *adev,
-                                        struct amdgpu_ps *new_rps,
-                                        struct amdgpu_ps *old_rps)
-{
-        struct cz_ps *ps = cz_get_ps(new_rps);
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_clock_and_voltage_limits *limits =
-                &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
-        /* 10kHz memory clock */
-        uint32_t mclk = 0;
-
-        ps->force_high = false;
-        ps->need_dfs_bypass = true;
-        pi->video_start = new_rps->dclk || new_rps->vclk ||
-                          new_rps->evclk || new_rps->ecclk;
-
-        if ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
-                        ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)
-                pi->battery_state = true;
-        else
-                pi->battery_state = false;
-
-        if (pi->caps_stable_power_state)
-                mclk = limits->mclk;
-
-        if (mclk > pi->sys_info.nbp_memory_clock[CZ_NUM_NBPMEMORY_CLOCK - 1])
-                ps->force_high = true;
-
-}
-
-static int cz_dpm_enable(struct amdgpu_device *adev)
-{
-        const char *chip_name;
-        int ret = 0;
-
-        /* renable will hang up SMU, so check first */
-        if (cz_check_for_dpm_enabled(adev))
-                return -EINVAL;
-
-        cz_program_voting_clients(adev);
-
-        switch (adev->asic_type) {
-        case CHIP_CARRIZO:
-                chip_name = "carrizo";
-                break;
-        case CHIP_STONEY:
-                chip_name = "stoney";
-                break;
-        default:
-                BUG();
-        }
-
-
-        ret = cz_start_dpm(adev);
-        if (ret) {
-                DRM_ERROR("%s DPM enable failed\n", chip_name);
-                return -EINVAL;
-        }
-
-        ret = cz_program_bootup_state(adev);
-        if (ret) {
-                DRM_ERROR("%s bootup state program failed\n", chip_name);
-                return -EINVAL;
-        }
-
-        ret = cz_enable_didt(adev, true);
-        if (ret) {
-                DRM_ERROR("%s enable di/dt failed\n", chip_name);
-                return -EINVAL;
-        }
-
-        cz_reset_acp_boot_level(adev);
-        cz_update_current_ps(adev, adev->pm.dpm.boot_ps);
-
-        return 0;
-}
-
-static int cz_dpm_hw_init(void *handle)
-{
-        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-        int ret = 0;
-
-        mutex_lock(&adev->pm.mutex);
-
-        /* smu init only needs to be called at startup, not resume.
-         * It should be in sw_init, but requires the fw info gathered
-         * in sw_init from other IP modules.
-         */
-        ret = cz_smu_init(adev);
-        if (ret) {
-                DRM_ERROR("amdgpu: smc initialization failed\n");
-                mutex_unlock(&adev->pm.mutex);
-                return ret;
-        }
-
-        /* do the actual fw loading */
-        ret = cz_smu_start(adev);
-        if (ret) {
-                DRM_ERROR("amdgpu: smc start failed\n");
-                mutex_unlock(&adev->pm.mutex);
-                return ret;
-        }
-
-        if (!amdgpu_dpm) {
-                adev->pm.dpm_enabled = false;
-                mutex_unlock(&adev->pm.mutex);
-                return ret;
-        }
-
-        /* cz dpm setup asic */
-        cz_dpm_setup_asic(adev);
-
-        /* cz dpm enable */
-        ret = cz_dpm_enable(adev);
-        if (ret)
-                adev->pm.dpm_enabled = false;
-        else
-                adev->pm.dpm_enabled = true;
-
-        mutex_unlock(&adev->pm.mutex);
-
-        return 0;
-}
-
-static int cz_dpm_disable(struct amdgpu_device *adev)
-{
-        int ret = 0;
-
-        if (!cz_check_for_dpm_enabled(adev))
-                return -EINVAL;
-
-        ret = cz_enable_didt(adev, false);
-        if (ret) {
-                DRM_ERROR("disable di/dt failed\n");
-                return -EINVAL;
-        }
-
-        /* powerup blocks */
-        cz_dpm_powergate_uvd(adev, false);
-        cz_dpm_powergate_vce(adev, false);
-
-        cz_clear_voting_clients(adev);
-        cz_stop_dpm(adev);
-        cz_update_current_ps(adev, adev->pm.dpm.boot_ps);
-
-        return 0;
-}
-
-static int cz_dpm_hw_fini(void *handle)
-{
-        int ret = 0;
-        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-        mutex_lock(&adev->pm.mutex);
-
-        /* smu fini only needs to be called at teardown, not suspend.
-         * It should be in sw_fini, but we put it here for symmetry
-         * with smu init.
-         */
-        cz_smu_fini(adev);
-
-        if (adev->pm.dpm_enabled) {
-                ret = cz_dpm_disable(adev);
-
-                adev->pm.dpm.current_ps =
-                        adev->pm.dpm.requested_ps =
-                        adev->pm.dpm.boot_ps;
-        }
-
-        adev->pm.dpm_enabled = false;
-
-        mutex_unlock(&adev->pm.mutex);
-
-        return ret;
-}
-
-static int cz_dpm_suspend(void *handle)
-{
-        int ret = 0;
-        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-        if (adev->pm.dpm_enabled) {
-                mutex_lock(&adev->pm.mutex);
-
-                ret = cz_dpm_disable(adev);
-
-                adev->pm.dpm.current_ps =
-                        adev->pm.dpm.requested_ps =
-                        adev->pm.dpm.boot_ps;
-
-                mutex_unlock(&adev->pm.mutex);
-        }
-
-        return ret;
-}
-
-static int cz_dpm_resume(void *handle)
-{
-        int ret = 0;
-        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-        mutex_lock(&adev->pm.mutex);
-
-        /* do the actual fw loading */
-        ret = cz_smu_start(adev);
-        if (ret) {
-                DRM_ERROR("amdgpu: smc start failed\n");
-                mutex_unlock(&adev->pm.mutex);
-                return ret;
-        }
-
-        if (!amdgpu_dpm) {
-                adev->pm.dpm_enabled = false;
-                mutex_unlock(&adev->pm.mutex);
-                return ret;
-        }
-
-        /* cz dpm setup asic */
-        cz_dpm_setup_asic(adev);
-
-        /* cz dpm enable */
-        ret = cz_dpm_enable(adev);
-        if (ret)
-                adev->pm.dpm_enabled = false;
-        else
-                adev->pm.dpm_enabled = true;
-
-        mutex_unlock(&adev->pm.mutex);
-        /* upon resume, re-compute the clocks */
-        if (adev->pm.dpm_enabled)
-                amdgpu_pm_compute_clocks(adev);
-
-        return 0;
-}
-
-static int cz_dpm_set_clockgating_state(void *handle,
-                                        enum amd_clockgating_state state)
-{
-        return 0;
-}
-
-static int cz_dpm_set_powergating_state(void *handle,
-                                        enum amd_powergating_state state)
-{
-        return 0;
-}
-
-static int cz_dpm_get_temperature(struct amdgpu_device *adev)
-{
-        int actual_temp = 0;
-        uint32_t val = RREG32_SMC(ixTHM_TCON_CUR_TMP);
-        uint32_t temp = REG_GET_FIELD(val, THM_TCON_CUR_TMP, CUR_TEMP);
-
-        if (REG_GET_FIELD(val, THM_TCON_CUR_TMP, CUR_TEMP_RANGE_SEL))
-                actual_temp = 1000 * ((temp / 8) - 49);
-        else
-                actual_temp = 1000 * (temp / 8);
-
-        return actual_temp;
-}
-
-static int cz_dpm_pre_set_power_state(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_ps requested_ps = *adev->pm.dpm.requested_ps;
-        struct amdgpu_ps *new_ps = &requested_ps;
-
-        cz_update_requested_ps(adev, new_ps);
-        cz_apply_state_adjust_rules(adev, &pi->requested_rps,
-                                        &pi->current_rps);
-
-        return 0;
-}
-
-static int cz_dpm_update_sclk_limit(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_clock_and_voltage_limits *limits =
-                &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
-        uint32_t clock, stable_ps_clock = 0;
-
-        clock = pi->sclk_dpm.soft_min_clk;
-
-        if (pi->caps_stable_power_state) {
-                stable_ps_clock = limits->sclk * 75 / 100;
-                if (clock < stable_ps_clock)
-                        clock = stable_ps_clock;
-        }
-
-        if (clock != pi->sclk_dpm.soft_min_clk) {
-                pi->sclk_dpm.soft_min_clk = clock;
-                cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetSclkSoftMin,
-                                cz_get_sclk_level(adev, clock,
-                                        PPSMC_MSG_SetSclkSoftMin));
-        }
-
-        if (pi->caps_stable_power_state &&
-                        pi->sclk_dpm.soft_max_clk != clock) {
-                pi->sclk_dpm.soft_max_clk = clock;
-                cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetSclkSoftMax,
-                                cz_get_sclk_level(adev, clock,
-                                        PPSMC_MSG_SetSclkSoftMax));
-        } else {
-                cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetSclkSoftMax,
-                                cz_get_sclk_level(adev,
-                                        pi->sclk_dpm.soft_max_clk,
-                                        PPSMC_MSG_SetSclkSoftMax));
-        }
-
-        return 0;
-}
-
-static int cz_dpm_set_deep_sleep_sclk_threshold(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        if (pi->caps_sclk_ds) {
-                cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetMinDeepSleepSclk,
-                                CZ_MIN_DEEP_SLEEP_SCLK);
-        }
-
-        return 0;
-}
-
-/* ?? without dal support, is this still needed in setpowerstate list*/
-static int cz_dpm_set_watermark_threshold(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        cz_send_msg_to_smc_with_parameter(adev,
-                        PPSMC_MSG_SetWatermarkFrequency,
-                        pi->sclk_dpm.soft_max_clk);
-
-        return 0;
-}
-
-static int cz_dpm_enable_nbdpm(struct amdgpu_device *adev)
-{
-        int ret = 0;
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        /* also depend on dal NBPStateDisableRequired */
-        if (pi->nb_dpm_enabled_by_driver && !pi->nb_dpm_enabled) {
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_EnableAllSmuFeatures,
-                                NB_DPM_MASK);
-                if (ret) {
-                        DRM_ERROR("amdgpu: nb dpm enable failed\n");
-                        return ret;
-                }
-                pi->nb_dpm_enabled = true;
-        }
-
-        return ret;
-}
-
-static void cz_dpm_nbdpm_lm_pstate_enable(struct amdgpu_device *adev,
-                                                        bool enable)
-{
-        if (enable)
-                cz_send_msg_to_smc(adev, PPSMC_MSG_EnableLowMemoryPstate);
-        else
-                cz_send_msg_to_smc(adev, PPSMC_MSG_DisableLowMemoryPstate);
-
-}
-
-static int cz_dpm_update_low_memory_pstate(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct cz_ps *ps = &pi->requested_ps;
-
-        if (pi->sys_info.nb_dpm_enable) {
-                if (ps->force_high)
-                        cz_dpm_nbdpm_lm_pstate_enable(adev, false);
-                else
-                        cz_dpm_nbdpm_lm_pstate_enable(adev, true);
-        }
-
-        return 0;
-}
-
-/* with dpm enabled */
-static int cz_dpm_set_power_state(struct amdgpu_device *adev)
-{
-        cz_dpm_update_sclk_limit(adev);
-        cz_dpm_set_deep_sleep_sclk_threshold(adev);
-        cz_dpm_set_watermark_threshold(adev);
-        cz_dpm_enable_nbdpm(adev);
-        cz_dpm_update_low_memory_pstate(adev);
-
-        return 0;
-}
-
-static void cz_dpm_post_set_power_state(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_ps *ps = &pi->requested_rps;
-
-        cz_update_current_ps(adev, ps);
-}
-
-static int cz_dpm_force_highest(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        int ret = 0;
-
-        if (pi->sclk_dpm.soft_min_clk != pi->sclk_dpm.soft_max_clk) {
-                pi->sclk_dpm.soft_min_clk =
-                        pi->sclk_dpm.soft_max_clk;
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetSclkSoftMin,
-                                cz_get_sclk_level(adev,
-                                        pi->sclk_dpm.soft_min_clk,
-                                        PPSMC_MSG_SetSclkSoftMin));
-                if (ret)
-                        return ret;
-        }
-
-        return ret;
-}
-
-static int cz_dpm_force_lowest(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        int ret = 0;
-
-        if (pi->sclk_dpm.soft_max_clk != pi->sclk_dpm.soft_min_clk) {
-                pi->sclk_dpm.soft_max_clk = pi->sclk_dpm.soft_min_clk;
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetSclkSoftMax,
-                                cz_get_sclk_level(adev,
-                                        pi->sclk_dpm.soft_max_clk,
-                                        PPSMC_MSG_SetSclkSoftMax));
-                if (ret)
-                        return ret;
-        }
-
-        return ret;
-}
-
-static uint32_t cz_dpm_get_max_sclk_level(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        if (!pi->max_sclk_level) {
-                cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxSclkLevel);
-                pi->max_sclk_level = cz_get_argument(adev) + 1;
-        }
-
-        if (pi->max_sclk_level > CZ_MAX_HARDWARE_POWERLEVELS) {
-                DRM_ERROR("Invalid max sclk level!\n");
-                return -EINVAL;
-        }
-
-        return pi->max_sclk_level;
-}
-
-static int cz_dpm_unforce_dpm_levels(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_clock_voltage_dependency_table *dep_table =
-                &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
-        uint32_t level = 0;
-        int ret = 0;
-
-        pi->sclk_dpm.soft_min_clk = dep_table->entries[0].clk;
-        level = cz_dpm_get_max_sclk_level(adev) - 1;
-        if (level < dep_table->count)
-                pi->sclk_dpm.soft_max_clk = dep_table->entries[level].clk;
-        else
-                pi->sclk_dpm.soft_max_clk =
-                        dep_table->entries[dep_table->count - 1].clk;
-
-        /* get min/max sclk soft value
-         * notify SMU to execute */
-        ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetSclkSoftMin,
-                                cz_get_sclk_level(adev,
-                                        pi->sclk_dpm.soft_min_clk,
-                                        PPSMC_MSG_SetSclkSoftMin));
-        if (ret)
-                return ret;
-
-        ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetSclkSoftMax,
-                                cz_get_sclk_level(adev,
-                                        pi->sclk_dpm.soft_max_clk,
-                                        PPSMC_MSG_SetSclkSoftMax));
-        if (ret)
-                return ret;
-
-        DRM_DEBUG("DPM unforce state min=%d, max=%d.\n",
-                  pi->sclk_dpm.soft_min_clk,
-                  pi->sclk_dpm.soft_max_clk);
-
-        return 0;
-}
-
-static int cz_dpm_uvd_force_highest(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        int ret = 0;
-
-        if (pi->uvd_dpm.soft_min_clk != pi->uvd_dpm.soft_max_clk) {
-                pi->uvd_dpm.soft_min_clk =
-                        pi->uvd_dpm.soft_max_clk;
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetUvdSoftMin,
-                                cz_get_uvd_level(adev,
-                                        pi->uvd_dpm.soft_min_clk,
-                                        PPSMC_MSG_SetUvdSoftMin));
-                if (ret)
-                        return ret;
-        }
-
-        return ret;
-}
-
-static int cz_dpm_uvd_force_lowest(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        int ret = 0;
-
-        if (pi->uvd_dpm.soft_max_clk != pi->uvd_dpm.soft_min_clk) {
-                pi->uvd_dpm.soft_max_clk = pi->uvd_dpm.soft_min_clk;
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetUvdSoftMax,
-                                cz_get_uvd_level(adev,
-                                        pi->uvd_dpm.soft_max_clk,
-                                        PPSMC_MSG_SetUvdSoftMax));
-                if (ret)
-                        return ret;
-        }
-
-        return ret;
-}
-
-static uint32_t cz_dpm_get_max_uvd_level(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        if (!pi->max_uvd_level) {
-                cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxUvdLevel);
-                pi->max_uvd_level = cz_get_argument(adev) + 1;
-        }
-
-        if (pi->max_uvd_level > CZ_MAX_HARDWARE_POWERLEVELS) {
-                DRM_ERROR("Invalid max uvd level!\n");
-                return -EINVAL;
-        }
-
-        return pi->max_uvd_level;
-}
-
-static int cz_dpm_unforce_uvd_dpm_levels(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_uvd_clock_voltage_dependency_table *dep_table =
-                &adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table;
-        uint32_t level = 0;
-        int ret = 0;
-
-        pi->uvd_dpm.soft_min_clk = dep_table->entries[0].vclk;
-        level = cz_dpm_get_max_uvd_level(adev) - 1;
-        if (level < dep_table->count)
-                pi->uvd_dpm.soft_max_clk = dep_table->entries[level].vclk;
-        else
-                pi->uvd_dpm.soft_max_clk =
-                        dep_table->entries[dep_table->count - 1].vclk;
-
-        /* get min/max sclk soft value
-         * notify SMU to execute */
-        ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetUvdSoftMin,
-                                cz_get_uvd_level(adev,
-                                        pi->uvd_dpm.soft_min_clk,
-                                        PPSMC_MSG_SetUvdSoftMin));
-        if (ret)
-                return ret;
-
-        ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetUvdSoftMax,
-                                cz_get_uvd_level(adev,
-                                        pi->uvd_dpm.soft_max_clk,
-                                        PPSMC_MSG_SetUvdSoftMax));
-        if (ret)
-                return ret;
-
-        DRM_DEBUG("DPM uvd unforce state min=%d, max=%d.\n",
-                  pi->uvd_dpm.soft_min_clk,
-                  pi->uvd_dpm.soft_max_clk);
-
-        return 0;
-}
-
-static int cz_dpm_vce_force_highest(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        int ret = 0;
-
-        if (pi->vce_dpm.soft_min_clk != pi->vce_dpm.soft_max_clk) {
-                pi->vce_dpm.soft_min_clk =
-                        pi->vce_dpm.soft_max_clk;
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetEclkSoftMin,
-                                cz_get_eclk_level(adev,
-                                        pi->vce_dpm.soft_min_clk,
-                                        PPSMC_MSG_SetEclkSoftMin));
-                if (ret)
-                        return ret;
-        }
-
-        return ret;
-}
-
-static int cz_dpm_vce_force_lowest(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        int ret = 0;
-
-        if (pi->vce_dpm.soft_max_clk != pi->vce_dpm.soft_min_clk) {
-                pi->vce_dpm.soft_max_clk = pi->vce_dpm.soft_min_clk;
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetEclkSoftMax,
-                                cz_get_uvd_level(adev,
-                                        pi->vce_dpm.soft_max_clk,
-                                        PPSMC_MSG_SetEclkSoftMax));
-                if (ret)
-                        return ret;
-        }
-
-        return ret;
-}
-
-static uint32_t cz_dpm_get_max_vce_level(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        if (!pi->max_vce_level) {
-                cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxEclkLevel);
-                pi->max_vce_level = cz_get_argument(adev) + 1;
-        }
-
-        if (pi->max_vce_level > CZ_MAX_HARDWARE_POWERLEVELS) {
-                DRM_ERROR("Invalid max vce level!\n");
-                return -EINVAL;
-        }
-
-        return pi->max_vce_level;
-}
-
-static int cz_dpm_unforce_vce_dpm_levels(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_vce_clock_voltage_dependency_table *dep_table =
-                &adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
-        uint32_t level = 0;
-        int ret = 0;
-
-        pi->vce_dpm.soft_min_clk = dep_table->entries[0].ecclk;
-        level = cz_dpm_get_max_vce_level(adev) - 1;
-        if (level < dep_table->count)
-                pi->vce_dpm.soft_max_clk = dep_table->entries[level].ecclk;
-        else
-                pi->vce_dpm.soft_max_clk =
-                        dep_table->entries[dep_table->count - 1].ecclk;
-
-        /* get min/max sclk soft value
-         * notify SMU to execute */
-        ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetEclkSoftMin,
-                                cz_get_eclk_level(adev,
-                                        pi->vce_dpm.soft_min_clk,
-                                        PPSMC_MSG_SetEclkSoftMin));
-        if (ret)
-                return ret;
-
-        ret = cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetEclkSoftMax,
-                                cz_get_eclk_level(adev,
-                                        pi->vce_dpm.soft_max_clk,
-                                        PPSMC_MSG_SetEclkSoftMax));
-        if (ret)
-                return ret;
-
-        DRM_DEBUG("DPM vce unforce state min=%d, max=%d.\n",
-                  pi->vce_dpm.soft_min_clk,
-                  pi->vce_dpm.soft_max_clk);
-
-        return 0;
-}
-
-static int cz_dpm_force_dpm_level(struct amdgpu_device *adev,
-                                  enum amdgpu_dpm_forced_level level)
-{
-        int ret = 0;
-
-        switch (level) {
-        case AMDGPU_DPM_FORCED_LEVEL_HIGH:
-                /* sclk */
-                ret = cz_dpm_unforce_dpm_levels(adev);
-                if (ret)
-                        return ret;
-                ret = cz_dpm_force_highest(adev);
-                if (ret)
-                        return ret;
-
-                /* uvd */
-                ret = cz_dpm_unforce_uvd_dpm_levels(adev);
-                if (ret)
-                        return ret;
-                ret = cz_dpm_uvd_force_highest(adev);
-                if (ret)
-                        return ret;
-
-                /* vce */
-                ret = cz_dpm_unforce_vce_dpm_levels(adev);
-                if (ret)
-                        return ret;
-                ret = cz_dpm_vce_force_highest(adev);
-                if (ret)
-                        return ret;
-                break;
-        case AMDGPU_DPM_FORCED_LEVEL_LOW:
-                /* sclk */
-                ret = cz_dpm_unforce_dpm_levels(adev);
-                if (ret)
-                        return ret;
-                ret = cz_dpm_force_lowest(adev);
-                if (ret)
-                        return ret;
-
-                /* uvd */
-                ret = cz_dpm_unforce_uvd_dpm_levels(adev);
-                if (ret)
-                        return ret;
-                ret = cz_dpm_uvd_force_lowest(adev);
-                if (ret)
-                        return ret;
-
-                /* vce */
-                ret = cz_dpm_unforce_vce_dpm_levels(adev);
-                if (ret)
-                        return ret;
-                ret = cz_dpm_vce_force_lowest(adev);
-                if (ret)
-                        return ret;
-                break;
-        case AMDGPU_DPM_FORCED_LEVEL_AUTO:
-                /* sclk */
-                ret = cz_dpm_unforce_dpm_levels(adev);
-                if (ret)
-                        return ret;
-
-                /* uvd */
-                ret = cz_dpm_unforce_uvd_dpm_levels(adev);
-                if (ret)
-                        return ret;
-
-                /* vce */
-                ret = cz_dpm_unforce_vce_dpm_levels(adev);
-                if (ret)
-                        return ret;
-                break;
-        default:
-                break;
-        }
-
-        adev->pm.dpm.forced_level = level;
-
-        return ret;
-}
-
-/* fix me, display configuration change lists here
- * mostly dal related*/
-static void cz_dpm_display_configuration_changed(struct amdgpu_device *adev)
-{
-}
-
-static uint32_t cz_dpm_get_sclk(struct amdgpu_device *adev, bool low)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct cz_ps *requested_state = cz_get_ps(&pi->requested_rps);
-
-        if (low)
-                return requested_state->levels[0].sclk;
-        else
-                return requested_state->levels[requested_state->num_levels - 1].sclk;
-
-}
-
-static uint32_t cz_dpm_get_mclk(struct amdgpu_device *adev, bool low)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        return pi->sys_info.bootup_uma_clk;
-}
-
-static int cz_enable_uvd_dpm(struct amdgpu_device *adev, bool enable)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        int ret = 0;
-
-        if (enable && pi->caps_uvd_dpm ) {
-                pi->dpm_flags |= DPMFlags_UVD_Enabled;
-                DRM_DEBUG("UVD DPM Enabled.\n");
-
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                        PPSMC_MSG_EnableAllSmuFeatures, UVD_DPM_MASK);
-        } else {
-                pi->dpm_flags &= ~DPMFlags_UVD_Enabled;
-                DRM_DEBUG("UVD DPM Stopped\n");
-
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                        PPSMC_MSG_DisableAllSmuFeatures, UVD_DPM_MASK);
-        }
-
-        return ret;
-}
-
-static int cz_update_uvd_dpm(struct amdgpu_device *adev, bool gate)
-{
-        return cz_enable_uvd_dpm(adev, !gate);
-}
-
-
-static void cz_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        int ret;
-
-        if (pi->uvd_power_gated == gate)
-                return;
-
-        pi->uvd_power_gated = gate;
-
-        if (gate) {
-                if (pi->caps_uvd_pg) {
-                        ret = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-                                                            AMD_CG_STATE_GATE);
-                        if (ret) {
-                                DRM_ERROR("UVD DPM Power Gating failed to set clockgating state\n");
-                                return;
-                        }
-
-                        /* shutdown the UVD block */
-                        ret = amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-                                                            AMD_PG_STATE_GATE);
-
-                        if (ret) {
-                                DRM_ERROR("UVD DPM Power Gating failed to set powergating state\n");
-                                return;
-                        }
-                }
-                cz_update_uvd_dpm(adev, gate);
-                if (pi->caps_uvd_pg) {
-                        /* power off the UVD block */
-                        ret = cz_send_msg_to_smc(adev, PPSMC_MSG_UVDPowerOFF);
-                        if (ret) {
-                                DRM_ERROR("UVD DPM Power Gating failed to send SMU PowerOFF message\n");
-                                return;
-                        }
-                }
-        } else {
-                if (pi->caps_uvd_pg) {
-                        /* power on the UVD block */
-                        if (pi->uvd_dynamic_pg)
-                                ret = cz_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_UVDPowerON, 1);
-                        else
-                                ret = cz_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_UVDPowerON, 0);
-
-                        if (ret) {
-                                DRM_ERROR("UVD DPM Power Gating Failed to send SMU PowerON message\n");
-                                return;
-                        }
-
-                        /* re-init the UVD block */
-                        ret = amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-                                                            AMD_PG_STATE_UNGATE);
-
-                        if (ret) {
-                                DRM_ERROR("UVD DPM Power Gating Failed to set powergating state\n");
-                                return;
-                        }
-
-                        ret = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-                                                            AMD_CG_STATE_UNGATE);
-                        if (ret) {
-                                DRM_ERROR("UVD DPM Power Gating Failed to set clockgating state\n");
-                                return;
-                        }
-                }
-                cz_update_uvd_dpm(adev, gate);
-        }
-}
-
-static int cz_enable_vce_dpm(struct amdgpu_device *adev, bool enable)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        int ret = 0;
-
-        if (enable && pi->caps_vce_dpm) {
-                pi->dpm_flags |= DPMFlags_VCE_Enabled;
-                DRM_DEBUG("VCE DPM Enabled.\n");
-
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                        PPSMC_MSG_EnableAllSmuFeatures, VCE_DPM_MASK);
-
-        } else {
-                pi->dpm_flags &= ~DPMFlags_VCE_Enabled;
-                DRM_DEBUG("VCE DPM Stopped\n");
-
-                ret = cz_send_msg_to_smc_with_parameter(adev,
-                        PPSMC_MSG_DisableAllSmuFeatures, VCE_DPM_MASK);
-        }
-
-        return ret;
-}
-
-static int cz_update_vce_dpm(struct amdgpu_device *adev)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-        struct amdgpu_vce_clock_voltage_dependency_table *table =
-                &adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
-
-        /* Stable Pstate is enabled and we need to set the VCE DPM to highest level */
-        if (pi->caps_stable_power_state) {
-                pi->vce_dpm.hard_min_clk = table->entries[table->count-1].ecclk;
-        } else { /* non-stable p-state cases. without vce.Arbiter.EcclkHardMin */
-                /* leave it as set by user */
-                /*pi->vce_dpm.hard_min_clk = table->entries[0].ecclk;*/
-        }
-
-        cz_send_msg_to_smc_with_parameter(adev,
-                PPSMC_MSG_SetEclkHardMin,
-                cz_get_eclk_level(adev,
-                        pi->vce_dpm.hard_min_clk,
-                        PPSMC_MSG_SetEclkHardMin));
-        return 0;
-}
-
-static void cz_dpm_powergate_vce(struct amdgpu_device *adev, bool gate)
-{
-        struct cz_power_info *pi = cz_get_pi(adev);
-
-        if (pi->caps_vce_pg) {
-                if (pi->vce_power_gated != gate) {
-                        if (gate) {
-                                /* disable clockgating so we can properly shut down the block */
-                                amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-                                                            AMD_CG_STATE_UNGATE);
-                                /* shutdown the VCE block */
-                                amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-                                                            AMD_PG_STATE_GATE);
-
-                                cz_enable_vce_dpm(adev, false);
-                                cz_send_msg_to_smc(adev, PPSMC_MSG_VCEPowerOFF);
-                                pi->vce_power_gated = true;
-                        } else {
-                                cz_send_msg_to_smc(adev, PPSMC_MSG_VCEPowerON);
-                                pi->vce_power_gated = false;
-
-                                /* re-init the VCE block */
-                                amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-                                                            AMD_PG_STATE_UNGATE);
-                                /* enable clockgating. hw will dynamically gate/ungate clocks on the fly */
-                                amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-                                                            AMD_CG_STATE_GATE);
-
-                                cz_update_vce_dpm(adev);
-                                cz_enable_vce_dpm(adev, true);
-                        }
-                } else {
-                        if (! pi->vce_power_gated) {
-                                cz_update_vce_dpm(adev);
-                        }
-                }
-        } else { /*pi->caps_vce_pg*/
-                pi->vce_power_gated = gate;
-                cz_update_vce_dpm(adev);
-                cz_enable_vce_dpm(adev, !gate);
-        }
-}
-
-static int cz_check_state_equal(struct amdgpu_device *adev,
-                                struct amdgpu_ps *cps,
-                                struct amdgpu_ps *rps,
-                                bool *equal)
-{
-        if (equal == NULL)
-                return -EINVAL;
-
-        *equal = false;
-        return 0;
-}
-
-const struct amd_ip_funcs cz_dpm_ip_funcs = {
-        .name = "cz_dpm",
-        .early_init = cz_dpm_early_init,
-        .late_init = cz_dpm_late_init,
-        .sw_init = cz_dpm_sw_init,
-        .sw_fini = cz_dpm_sw_fini,
-        .hw_init = cz_dpm_hw_init,
-        .hw_fini = cz_dpm_hw_fini,
-        .suspend = cz_dpm_suspend,
-        .resume = cz_dpm_resume,
-        .is_idle = NULL,
-        .wait_for_idle = NULL,
-        .soft_reset = NULL,
-        .set_clockgating_state = cz_dpm_set_clockgating_state,
-        .set_powergating_state = cz_dpm_set_powergating_state,
-};
-
-static const struct amdgpu_dpm_funcs cz_dpm_funcs = {
-        .get_temperature = cz_dpm_get_temperature,
-        .pre_set_power_state = cz_dpm_pre_set_power_state,
-        .set_power_state = cz_dpm_set_power_state,
-        .post_set_power_state = cz_dpm_post_set_power_state,
-        .display_configuration_changed = cz_dpm_display_configuration_changed,
-        .get_sclk = cz_dpm_get_sclk,
-        .get_mclk = cz_dpm_get_mclk,
-        .print_power_state = cz_dpm_print_power_state,
-        .debugfs_print_current_performance_level =
-                                cz_dpm_debugfs_print_current_performance_level,
-        .force_performance_level = cz_dpm_force_dpm_level,
-        .vblank_too_short = NULL,
-        .powergate_uvd = cz_dpm_powergate_uvd,
-        .powergate_vce = cz_dpm_powergate_vce,
-        .check_state_equal = cz_check_state_equal,
-};
-
-static void cz_dpm_set_funcs(struct amdgpu_device *adev)
-{
-        if (NULL == adev->pm.funcs)
-                adev->pm.funcs = &cz_dpm_funcs;
-}
-
-const struct amdgpu_ip_block_version cz_dpm_ip_block =
-{
-        .type = AMD_IP_BLOCK_TYPE_SMC,
-        .major = 8,
-        .minor = 0,
-        .rev = 0,
-        .funcs = &cz_dpm_ip_funcs,
-};
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_dpm.h b/drivers/gpu/drm/amd/amdgpu/cz_dpm.h
deleted file mode 100644
index 5df8c1faab51..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/cz_dpm.h
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef __CZ_DPM_H__
-#define __CZ_DPM_H__
-
-#include "smu8_fusion.h"
-
-#define CZ_AT_DFLT                                      30
-#define CZ_NUM_NBPSTATES                                4
-#define CZ_NUM_NBPMEMORY_CLOCK                          2
-#define CZ_MAX_HARDWARE_POWERLEVELS                     8
-#define CZ_MAX_DISPLAY_CLOCK_LEVEL                      8
-#define CZ_MAX_DISPLAYPHY_IDS                           10
-
-#define PPCZ_VOTINGRIGHTSCLIENTS_DFLT0                  0x3FFFC102
-
-#define SMC_RAM_END                                     0x40000
-
-#define DPMFlags_SCLK_Enabled                           0x00000001
-#define DPMFlags_UVD_Enabled                            0x00000002
-#define DPMFlags_VCE_Enabled                            0x00000004
-#define DPMFlags_ACP_Enabled                            0x00000008
-#define DPMFlags_ForceHighestValid                      0x40000000
-#define DPMFlags_Debug                                  0x80000000
-
-/* Do not change the following, it is also defined in SMU8.h */
-#define SMU_EnabledFeatureScoreboard_AcpDpmOn           0x00000001
-#define SMU_EnabledFeatureScoreboard_SclkDpmOn          0x00200000
-#define SMU_EnabledFeatureScoreboard_UvdDpmOn           0x00800000
-#define SMU_EnabledFeatureScoreboard_VceDpmOn           0x01000000
-
-/* temporary solution to SetMinDeepSleepSclk
- * should indicate by display adaptor
- * 10k Hz unit*/
-#define CZ_MIN_DEEP_SLEEP_SCLK                          800
-
-enum cz_pt_config_reg_type {
-        CZ_CONFIGREG_MMR = 0,
-        CZ_CONFIGREG_SMC_IND,
-        CZ_CONFIGREG_DIDT_IND,
-        CZ_CONFIGREG_CACHE,
-        CZ_CONFIGREG_MAX
-};
-
-struct cz_pt_config_reg {
-        uint32_t offset;
-        uint32_t mask;
-        uint32_t shift;
-        uint32_t value;
-        enum cz_pt_config_reg_type type;
-};
-
-struct cz_dpm_entry {
-        uint32_t        soft_min_clk;
-        uint32_t        hard_min_clk;
-        uint32_t        soft_max_clk;
-        uint32_t        hard_max_clk;
-};
-
-struct cz_pl {
-        uint32_t sclk;
-        uint8_t vddc_index;
-        uint8_t ds_divider_index;
-        uint8_t ss_divider_index;
-        uint8_t allow_gnb_slow;
-        uint8_t force_nbp_state;
-        uint8_t display_wm;
-        uint8_t vce_wm;
-};
-
-struct cz_ps {
-        struct cz_pl levels[CZ_MAX_HARDWARE_POWERLEVELS];
-        uint32_t num_levels;
-        bool need_dfs_bypass;
-        uint8_t dpm0_pg_nb_ps_lo;
-        uint8_t dpm0_pg_nb_ps_hi;
-        uint8_t dpmx_nb_ps_lo;
-        uint8_t dpmx_nb_ps_hi;
-        bool force_high;
-};
-
-struct cz_displayphy_entry {
-        uint8_t phy_present;
-        uint8_t active_lane_mapping;
-        uint8_t display_conf_type;
-        uint8_t num_active_lanes;
-};
-
-struct cz_displayphy_info {
-        bool phy_access_initialized;
-        struct cz_displayphy_entry entries[CZ_MAX_DISPLAYPHY_IDS];
-};
-
-struct cz_sys_info {
-        uint32_t bootup_uma_clk;
-        uint32_t bootup_sclk;
-        uint32_t dentist_vco_freq;
-        uint32_t nb_dpm_enable;
-        uint32_t nbp_memory_clock[CZ_NUM_NBPMEMORY_CLOCK];
-        uint32_t nbp_n_clock[CZ_NUM_NBPSTATES];
-        uint8_t nbp_voltage_index[CZ_NUM_NBPSTATES];
-        uint32_t display_clock[CZ_MAX_DISPLAY_CLOCK_LEVEL];
-        uint16_t bootup_nb_voltage_index;
-        uint8_t htc_tmp_lmt;
-        uint8_t htc_hyst_lmt;
-        uint32_t uma_channel_number;
-};
-
-struct cz_power_info {
-        uint32_t active_target[CZ_MAX_HARDWARE_POWERLEVELS];
-        struct cz_sys_info sys_info;
-        struct cz_pl boot_pl;
-        bool disable_nb_ps3_in_battery;
-        bool battery_state;
-        uint32_t lowest_valid;
-        uint32_t highest_valid;
-        uint16_t high_voltage_threshold;
-        /* smc offsets */
-        uint32_t sram_end;
-        uint32_t dpm_table_start;
-        uint32_t soft_regs_start;
-        /* dpm SMU tables */
-        uint8_t uvd_level_count;
-        uint8_t vce_level_count;
-        uint8_t acp_level_count;
-        uint32_t fps_high_threshold;
-        uint32_t fps_low_threshold;
-        /* dpm table */
-        uint32_t dpm_flags;
-        struct cz_dpm_entry sclk_dpm;
-        struct cz_dpm_entry uvd_dpm;
-        struct cz_dpm_entry vce_dpm;
-        struct cz_dpm_entry acp_dpm;
-
-        uint8_t uvd_boot_level;
-        uint8_t uvd_interval;
-        uint8_t vce_boot_level;
-        uint8_t vce_interval;
-        uint8_t acp_boot_level;
-        uint8_t acp_interval;
-
-        uint8_t graphics_boot_level;
-        uint8_t graphics_interval;
-        uint8_t graphics_therm_throttle_enable;
-        uint8_t graphics_voltage_change_enable;
-        uint8_t graphics_clk_slow_enable;
-        uint8_t graphics_clk_slow_divider;
-
-        uint32_t low_sclk_interrupt_threshold;
-        bool uvd_power_gated;
-        bool vce_power_gated;
-        bool acp_power_gated;
-
-        uint32_t active_process_mask;
-
-        uint32_t mgcg_cgtt_local0;
-        uint32_t mgcg_cgtt_local1;
-        uint32_t clock_slow_down_step;
-        uint32_t skip_clock_slow_down;
-        bool enable_nb_ps_policy;
-        uint32_t voting_clients;
-        uint32_t voltage_drop_threshold;
-        uint32_t gfx_pg_threshold;
-        uint32_t max_sclk_level;
-        uint32_t max_uvd_level;
-        uint32_t max_vce_level;
-        /* flags */
-        bool didt_enabled;
-        bool video_start;
-        bool cac_enabled;
-        bool bapm_enabled;
-        bool nb_dpm_enabled_by_driver;
-        bool nb_dpm_enabled;
-        bool auto_thermal_throttling_enabled;
-        bool dpm_enabled;
-        bool need_pptable_upload;
-        /* caps */
-        bool caps_cac;
-        bool caps_power_containment;
-        bool caps_sq_ramping;
-        bool caps_db_ramping;
-        bool caps_td_ramping;
-        bool caps_tcp_ramping;
-        bool caps_sclk_throttle_low_notification;
-        bool caps_fps;
-        bool caps_uvd_dpm;
-        bool caps_uvd_pg;
-        bool caps_vce_dpm;
-        bool caps_vce_pg;
-        bool caps_acp_dpm;
-        bool caps_acp_pg;
-        bool caps_stable_power_state;
-        bool caps_enable_dfs_bypass;
-        bool caps_sclk_ds;
-        bool caps_voltage_island;
-        /* power state */
-        struct amdgpu_ps current_rps;
-        struct cz_ps current_ps;
-        struct amdgpu_ps requested_rps;
-        struct cz_ps requested_ps;
-
-        bool uvd_power_down;
-        bool vce_power_down;
-        bool acp_power_down;
-
-        bool uvd_dynamic_pg;
-};
-
-/* cz_smc.c */
-uint32_t cz_get_argument(struct amdgpu_device *adev);
-int cz_send_msg_to_smc(struct amdgpu_device *adev, uint16_t msg);
-int cz_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
-                        uint16_t msg, uint32_t parameter);
-int cz_read_smc_sram_dword(struct amdgpu_device *adev,
-                        uint32_t smc_address, uint32_t *value, uint32_t limit);
-int cz_smu_upload_pptable(struct amdgpu_device *adev);
-int cz_smu_download_pptable(struct amdgpu_device *adev, void **table);
-#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_smc.c b/drivers/gpu/drm/amd/amdgpu/cz_smc.c
deleted file mode 100644
index aed7033c0973..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/cz_smc.c
+++ /dev/null
@@ -1,995 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-#include <linux/firmware.h>
-#include "drmP.h"
-#include "amdgpu.h"
-#include "smu8.h"
-#include "smu8_fusion.h"
-#include "cz_ppsmc.h"
-#include "cz_smumgr.h"
-#include "smu_ucode_xfer_cz.h"
-#include "amdgpu_ucode.h"
-#include "cz_dpm.h"
-#include "vi_dpm.h"
-
-#include "smu/smu_8_0_d.h"
-#include "smu/smu_8_0_sh_mask.h"
-#include "gca/gfx_8_0_d.h"
-#include "gca/gfx_8_0_sh_mask.h"
-
-uint32_t cz_get_argument(struct amdgpu_device *adev)
-{
-        return RREG32(mmSMU_MP1_SRBM2P_ARG_0);
-}
-
-static struct cz_smu_private_data *cz_smu_get_priv(struct amdgpu_device *adev)
-{
-        struct cz_smu_private_data *priv =
-                        (struct cz_smu_private_data *)(adev->smu.priv);
-
-        return priv;
-}
-
-static int cz_send_msg_to_smc_async(struct amdgpu_device *adev, u16 msg)
-{
-        int i;
-        u32 content = 0, tmp;
-
-        for (i = 0; i < adev->usec_timeout; i++) {
-                tmp = REG_GET_FIELD(RREG32(mmSMU_MP1_SRBM2P_RESP_0),
-                                SMU_MP1_SRBM2P_RESP_0, CONTENT);
-                if (content != tmp)
-                        break;
-                udelay(1);
-        }
-
-        /* timeout means wrong logic*/
-        if (i == adev->usec_timeout)
-                return -EINVAL;
-
-        WREG32(mmSMU_MP1_SRBM2P_RESP_0, 0);
-        WREG32(mmSMU_MP1_SRBM2P_MSG_0, msg);
-
-        return 0;
-}
-
-int cz_send_msg_to_smc(struct amdgpu_device *adev, u16 msg)
-{
-        int i;
-        u32 content = 0, tmp = 0;
-
-        if (cz_send_msg_to_smc_async(adev, msg))
-                return -EINVAL;
-
-        for (i = 0; i < adev->usec_timeout; i++) {
-                tmp = REG_GET_FIELD(RREG32(mmSMU_MP1_SRBM2P_RESP_0),
-                                SMU_MP1_SRBM2P_RESP_0, CONTENT);
-                if (content != tmp)
-                        break;
-                udelay(1);
-        }
-
-        /* timeout means wrong logic*/
-        if (i == adev->usec_timeout)
-                return -EINVAL;
-
-        if (PPSMC_Result_OK != tmp) {
-                dev_err(adev->dev, "SMC Failed to send Message.\n");
-                return -EINVAL;
-        }
-
-        return 0;
-}
-
-int cz_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
-                                                u16 msg, u32 parameter)
-{
-        WREG32(mmSMU_MP1_SRBM2P_ARG_0, parameter);
-        return cz_send_msg_to_smc(adev, msg);
-}
-
-static int cz_set_smc_sram_address(struct amdgpu_device *adev,
-                                                u32 smc_address, u32 limit)
-{
-        if (smc_address & 3)
-                return -EINVAL;
-        if ((smc_address + 3) > limit)
-                return -EINVAL;
-
-        WREG32(mmMP0PUB_IND_INDEX_0, SMN_MP1_SRAM_START_ADDR + smc_address);
-
-        return 0;
-}
-
-int cz_read_smc_sram_dword(struct amdgpu_device *adev, u32 smc_address,
-                                                u32 *value, u32 limit)
-{
-        int ret;
-
-        ret = cz_set_smc_sram_address(adev, smc_address, limit);
-        if (ret)
-                return ret;
-
-        *value = RREG32(mmMP0PUB_IND_DATA_0);
-
-        return 0;
-}
-
-static int cz_write_smc_sram_dword(struct amdgpu_device *adev, u32 smc_address,
-                                                u32 value, u32 limit)
-{
-        int ret;
-
-        ret = cz_set_smc_sram_address(adev, smc_address, limit);
-        if (ret)
-                return ret;
-
-        WREG32(mmMP0PUB_IND_DATA_0, value);
-
-        return 0;
-}
-
-static int cz_smu_request_load_fw(struct amdgpu_device *adev)
-{
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-
-        uint32_t smc_addr = SMU8_FIRMWARE_HEADER_LOCATION +
-                        offsetof(struct SMU8_Firmware_Header, UcodeLoadStatus);
-
-        cz_write_smc_sram_dword(adev, smc_addr, 0, smc_addr + 4);
-
-        /*prepare toc buffers*/
-        cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_DriverDramAddrHi,
-                                priv->toc_buffer.mc_addr_high);
-        cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_DriverDramAddrLo,
-                                priv->toc_buffer.mc_addr_low);
-        cz_send_msg_to_smc(adev, PPSMC_MSG_InitJobs);
-
-        /*execute jobs*/
-        cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_ExecuteJob,
-                                priv->toc_entry_aram);
-
-        cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_ExecuteJob,
-                                priv->toc_entry_power_profiling_index);
-
-        cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_ExecuteJob,
-                                priv->toc_entry_initialize_index);
-
-        return 0;
-}
-
-/*
- *Check if the FW has been loaded, SMU will not return if loading
- *has not finished.
- */
-static int cz_smu_check_fw_load_finish(struct amdgpu_device *adev,
-                                                uint32_t fw_mask)
-{
-        int i;
-        uint32_t index = SMN_MP1_SRAM_START_ADDR +
-                        SMU8_FIRMWARE_HEADER_LOCATION +
-                        offsetof(struct SMU8_Firmware_Header, UcodeLoadStatus);
-
-        WREG32(mmMP0PUB_IND_INDEX, index);
-
-        for (i = 0; i < adev->usec_timeout; i++) {
-                if (fw_mask == (RREG32(mmMP0PUB_IND_DATA) & fw_mask))
-                        break;
-                udelay(1);
-        }
-
-        if (i >= adev->usec_timeout) {
-                dev_err(adev->dev,
-                "SMU check loaded firmware failed, expecting 0x%x, getting 0x%x",
-                fw_mask, RREG32(mmMP0PUB_IND_DATA));
-                return -EINVAL;
-        }
-
-        return 0;
-}
-
-/*
- * interfaces for different ip blocks to check firmware loading status
- * 0 for success otherwise failed
- */
-static int cz_smu_check_finished(struct amdgpu_device *adev,
-                                                        enum AMDGPU_UCODE_ID id)
-{
-        switch (id) {
-        case AMDGPU_UCODE_ID_SDMA0:
-                if (adev->smu.fw_flags & AMDGPU_SDMA0_UCODE_LOADED)
-                        return 0;
-                break;
-        case AMDGPU_UCODE_ID_SDMA1:
-                if (adev->smu.fw_flags & AMDGPU_SDMA1_UCODE_LOADED)
-                        return 0;
-                break;
-        case AMDGPU_UCODE_ID_CP_CE:
-                if (adev->smu.fw_flags & AMDGPU_CPCE_UCODE_LOADED)
-                        return 0;
-                break;
-        case AMDGPU_UCODE_ID_CP_PFP:
-                if (adev->smu.fw_flags & AMDGPU_CPPFP_UCODE_LOADED)
-                        return 0;
-        case AMDGPU_UCODE_ID_CP_ME:
-                if (adev->smu.fw_flags & AMDGPU_CPME_UCODE_LOADED)
-                        return 0;
-                break;
-        case AMDGPU_UCODE_ID_CP_MEC1:
-                if (adev->smu.fw_flags & AMDGPU_CPMEC1_UCODE_LOADED)
-                        return 0;
-                break;
-        case AMDGPU_UCODE_ID_CP_MEC2:
-                if (adev->smu.fw_flags & AMDGPU_CPMEC2_UCODE_LOADED)
-                        return 0;
-                break;
-        case AMDGPU_UCODE_ID_RLC_G:
-                if (adev->smu.fw_flags & AMDGPU_CPRLC_UCODE_LOADED)
-                        return 0;
-                break;
-        case AMDGPU_UCODE_ID_MAXIMUM:
-        default:
-                break;
-        }
-
-        return 1;
-}
-
-static int cz_load_mec_firmware(struct amdgpu_device *adev)
-{
-        struct amdgpu_firmware_info *ucode =
-                                &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
-        uint32_t reg_data;
-        uint32_t tmp;
-
-        if (ucode->fw == NULL)
-                return -EINVAL;
-
-        /* Disable MEC parsing/prefetching */
-        tmp = RREG32(mmCP_MEC_CNTL);
-        tmp = REG_SET_FIELD(tmp, CP_MEC_CNTL, MEC_ME1_HALT, 1);
-        tmp = REG_SET_FIELD(tmp, CP_MEC_CNTL, MEC_ME2_HALT, 1);
-        WREG32(mmCP_MEC_CNTL, tmp);
-
-        tmp = RREG32(mmCP_CPC_IC_BASE_CNTL);
-        tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
-        tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ATC, 0);
-        tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
-        tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, MTYPE, 1);
-        WREG32(mmCP_CPC_IC_BASE_CNTL, tmp);
-
-        reg_data = lower_32_bits(ucode->mc_addr) &
-                        REG_FIELD_MASK(CP_CPC_IC_BASE_LO, IC_BASE_LO);
-        WREG32(mmCP_CPC_IC_BASE_LO, reg_data);
-
-        reg_data = upper_32_bits(ucode->mc_addr) &
-                        REG_FIELD_MASK(CP_CPC_IC_BASE_HI, IC_BASE_HI);
-        WREG32(mmCP_CPC_IC_BASE_HI, reg_data);
-
-        return 0;
-}
-
-int cz_smu_start(struct amdgpu_device *adev)
-{
-        int ret = 0;
-
-        uint32_t fw_to_check = UCODE_ID_RLC_G_MASK |
-                                UCODE_ID_SDMA0_MASK |
-                                UCODE_ID_SDMA1_MASK |
-                                UCODE_ID_CP_CE_MASK |
-                                UCODE_ID_CP_ME_MASK |
-                                UCODE_ID_CP_PFP_MASK |
-                                UCODE_ID_CP_MEC_JT1_MASK |
-                                UCODE_ID_CP_MEC_JT2_MASK;
-
-        if (adev->asic_type == CHIP_STONEY)
-                fw_to_check &= ~(UCODE_ID_SDMA1_MASK | UCODE_ID_CP_MEC_JT2_MASK);
-
-        cz_smu_request_load_fw(adev);
-        ret = cz_smu_check_fw_load_finish(adev, fw_to_check);
-        if (ret)
-                return ret;
-
-        /* manually load MEC firmware for CZ */
-        if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY) {
-                ret = cz_load_mec_firmware(adev);
-                if (ret) {
-                        dev_err(adev->dev, "(%d) Mec Firmware load failed\n", ret);
-                        return ret;
-                }
-        }
-
-        /* setup fw load flag */
-        adev->smu.fw_flags = AMDGPU_SDMA0_UCODE_LOADED |
-                                AMDGPU_SDMA1_UCODE_LOADED |
-                                AMDGPU_CPCE_UCODE_LOADED |
-                                AMDGPU_CPPFP_UCODE_LOADED |
-                                AMDGPU_CPME_UCODE_LOADED |
-                                AMDGPU_CPMEC1_UCODE_LOADED |
-                                AMDGPU_CPMEC2_UCODE_LOADED |
-                                AMDGPU_CPRLC_UCODE_LOADED;
-
-        if (adev->asic_type == CHIP_STONEY)
-                adev->smu.fw_flags &= ~(AMDGPU_SDMA1_UCODE_LOADED | AMDGPU_CPMEC2_UCODE_LOADED);
-
-        return ret;
-}
-
-static uint32_t cz_convert_fw_type(uint32_t fw_type)
-{
-        enum AMDGPU_UCODE_ID result = AMDGPU_UCODE_ID_MAXIMUM;
-
-        switch (fw_type) {
-        case UCODE_ID_SDMA0:
-                result = AMDGPU_UCODE_ID_SDMA0;
-                break;
-        case UCODE_ID_SDMA1:
-                result = AMDGPU_UCODE_ID_SDMA1;
-                break;
-        case UCODE_ID_CP_CE:
-                result = AMDGPU_UCODE_ID_CP_CE;
-                break;
-        case UCODE_ID_CP_PFP:
-                result = AMDGPU_UCODE_ID_CP_PFP;
-                break;
-        case UCODE_ID_CP_ME:
-                result = AMDGPU_UCODE_ID_CP_ME;
-                break;
-        case UCODE_ID_CP_MEC_JT1:
-        case UCODE_ID_CP_MEC_JT2:
-                result = AMDGPU_UCODE_ID_CP_MEC1;
-                break;
-        case UCODE_ID_RLC_G:
-                result = AMDGPU_UCODE_ID_RLC_G;
-                break;
-        default:
-                DRM_ERROR("UCode type is out of range!");
-        }
-
-        return result;
-}
-
-static uint8_t cz_smu_translate_firmware_enum_to_arg(
-                        enum cz_scratch_entry firmware_enum)
-{
-        uint8_t ret = 0;
-
-        switch (firmware_enum) {
-        case CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0:
-                ret = UCODE_ID_SDMA0;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1:
-                ret = UCODE_ID_SDMA1;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE:
-                ret = UCODE_ID_CP_CE;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_CP_PFP:
-                ret = UCODE_ID_CP_PFP;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME:
-                ret = UCODE_ID_CP_ME;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1:
-                ret = UCODE_ID_CP_MEC_JT1;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2:
-                ret = UCODE_ID_CP_MEC_JT2;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_GMCON_RENG:
-                ret = UCODE_ID_GMCON_RENG;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G:
-                ret = UCODE_ID_RLC_G;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SCRATCH:
-                ret = UCODE_ID_RLC_SCRATCH;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_ARAM:
-                ret = UCODE_ID_RLC_SRM_ARAM;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_DRAM:
-                ret = UCODE_ID_RLC_SRM_DRAM;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_DMCU_ERAM:
-                ret = UCODE_ID_DMCU_ERAM;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_DMCU_IRAM:
-                ret = UCODE_ID_DMCU_IRAM;
-                break;
-        case CZ_SCRATCH_ENTRY_UCODE_ID_POWER_PROFILING:
-                ret = TASK_ARG_INIT_MM_PWR_LOG;
-                break;
-        case CZ_SCRATCH_ENTRY_DATA_ID_SDMA_HALT:
-        case CZ_SCRATCH_ENTRY_DATA_ID_SYS_CLOCKGATING:
-        case CZ_SCRATCH_ENTRY_DATA_ID_SDMA_RING_REGS:
-        case CZ_SCRATCH_ENTRY_DATA_ID_NONGFX_REINIT:
-        case CZ_SCRATCH_ENTRY_DATA_ID_SDMA_START:
-        case CZ_SCRATCH_ENTRY_DATA_ID_IH_REGISTERS:
-                ret = TASK_ARG_REG_MMIO;
-                break;
-        case CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE:
-                ret = TASK_ARG_INIT_CLK_TABLE;
-                break;
-        }
-
-        return ret;
-}
-
-static int cz_smu_populate_single_firmware_entry(struct amdgpu_device *adev,
-                                        enum cz_scratch_entry firmware_enum,
-                                        struct cz_buffer_entry *entry)
-{
-        uint64_t gpu_addr;
-        uint32_t data_size;
-        uint8_t ucode_id = cz_smu_translate_firmware_enum_to_arg(firmware_enum);
-        enum AMDGPU_UCODE_ID id = cz_convert_fw_type(ucode_id);
-        struct amdgpu_firmware_info *ucode = &adev->firmware.ucode[id];
-        const struct gfx_firmware_header_v1_0 *header;
-
-        if (ucode->fw == NULL)
-                return -EINVAL;
-
-        gpu_addr  = ucode->mc_addr;
-        header = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data;
-        data_size = le32_to_cpu(header->header.ucode_size_bytes);
-
-        if ((firmware_enum == CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1) ||
-            (firmware_enum == CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2)) {
-                gpu_addr += le32_to_cpu(header->jt_offset) << 2;
-                data_size = le32_to_cpu(header->jt_size) << 2;
-        }
-
-        entry->mc_addr_low = lower_32_bits(gpu_addr);
-        entry->mc_addr_high = upper_32_bits(gpu_addr);
-        entry->data_size = data_size;
-        entry->firmware_ID = firmware_enum;
-
-        return 0;
-}
-
-static int cz_smu_populate_single_scratch_entry(struct amdgpu_device *adev,
-                                        enum cz_scratch_entry scratch_type,
-                                        uint32_t size_in_byte,
-                                        struct cz_buffer_entry *entry)
-{
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-        uint64_t mc_addr = (((uint64_t) priv->smu_buffer.mc_addr_high) << 32) |
-                                                priv->smu_buffer.mc_addr_low;
-        mc_addr += size_in_byte;
-
-        priv->smu_buffer_used_bytes += size_in_byte;
-        entry->data_size = size_in_byte;
-        entry->kaddr = priv->smu_buffer.kaddr + priv->smu_buffer_used_bytes;
-        entry->mc_addr_low = lower_32_bits(mc_addr);
-        entry->mc_addr_high = upper_32_bits(mc_addr);
-        entry->firmware_ID = scratch_type;
-
-        return 0;
-}
-
-static int cz_smu_populate_single_ucode_load_task(struct amdgpu_device *adev,
-                                                enum cz_scratch_entry firmware_enum,
-                                                bool is_last)
-{
-        uint8_t i;
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-        struct TOC *toc = (struct TOC *)priv->toc_buffer.kaddr;
-        struct SMU_Task *task = &toc->tasks[priv->toc_entry_used_count++];
-
-        task->type = TASK_TYPE_UCODE_LOAD;
-        task->arg = cz_smu_translate_firmware_enum_to_arg(firmware_enum);
-        task->next = is_last ? END_OF_TASK_LIST : priv->toc_entry_used_count;
-
-        for (i = 0; i < priv->driver_buffer_length; i++)
-                if (priv->driver_buffer[i].firmware_ID == firmware_enum)
-                        break;
-
-        if (i >= priv->driver_buffer_length) {
-                dev_err(adev->dev, "Invalid Firmware Type\n");
-                return -EINVAL;
-        }
-
-        task->addr.low = priv->driver_buffer[i].mc_addr_low;
-        task->addr.high = priv->driver_buffer[i].mc_addr_high;
-        task->size_bytes = priv->driver_buffer[i].data_size;
-
-        return 0;
-}
-
-static int cz_smu_populate_single_scratch_task(struct amdgpu_device *adev,
-                                                enum cz_scratch_entry firmware_enum,
-                                                uint8_t type, bool is_last)
-{
-        uint8_t i;
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-        struct TOC *toc = (struct TOC *)priv->toc_buffer.kaddr;
-        struct SMU_Task *task = &toc->tasks[priv->toc_entry_used_count++];
-
-        task->type = type;
-        task->arg = cz_smu_translate_firmware_enum_to_arg(firmware_enum);
-        task->next = is_last ? END_OF_TASK_LIST : priv->toc_entry_used_count;
-
-        for (i = 0; i < priv->scratch_buffer_length; i++)
-                if (priv->scratch_buffer[i].firmware_ID == firmware_enum)
-                        break;
-
-        if (i >= priv->scratch_buffer_length) {
-                dev_err(adev->dev, "Invalid Firmware Type\n");
-                return -EINVAL;
-        }
-
-        task->addr.low = priv->scratch_buffer[i].mc_addr_low;
-        task->addr.high = priv->scratch_buffer[i].mc_addr_high;
-        task->size_bytes = priv->scratch_buffer[i].data_size;
-
-        if (CZ_SCRATCH_ENTRY_DATA_ID_IH_REGISTERS == firmware_enum) {
-                struct cz_ih_meta_data *pIHReg_restore =
-                        (struct cz_ih_meta_data *)priv->scratch_buffer[i].kaddr;
-                pIHReg_restore->command =
-                        METADATA_CMD_MODE0 | METADATA_PERFORM_ON_LOAD;
-        }
-
-        return 0;
-}
-
-static int cz_smu_construct_toc_for_rlc_aram_save(struct amdgpu_device *adev)
-{
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-        priv->toc_entry_aram = priv->toc_entry_used_count;
-        cz_smu_populate_single_scratch_task(adev,
-                        CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_ARAM,
-                        TASK_TYPE_UCODE_SAVE, true);
-
-        return 0;
-}
-
-static int cz_smu_construct_toc_for_vddgfx_enter(struct amdgpu_device *adev)
-{
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-        struct TOC *toc = (struct TOC *)priv->toc_buffer.kaddr;
-
-        toc->JobList[JOB_GFX_SAVE] = (uint8_t)priv->toc_entry_used_count;
-        cz_smu_populate_single_scratch_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SCRATCH,
-                                TASK_TYPE_UCODE_SAVE, false);
-        cz_smu_populate_single_scratch_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_DRAM,
-                                TASK_TYPE_UCODE_SAVE, true);
-
-        return 0;
-}
-
-static int cz_smu_construct_toc_for_vddgfx_exit(struct amdgpu_device *adev)
-{
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-        struct TOC *toc = (struct TOC *)priv->toc_buffer.kaddr;
-
-        toc->JobList[JOB_GFX_RESTORE] = (uint8_t)priv->toc_entry_used_count;
-
-        /* populate ucode */
-        if (adev->firmware.smu_load) {
-                cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE, false);
-                cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_PFP, false);
-                cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME, false);
-                cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
-                if (adev->asic_type == CHIP_STONEY) {
-                        cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
-                } else {
-                        cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2, false);
-                }
-                cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G, false);
-        }
-
-        /* populate scratch */
-        cz_smu_populate_single_scratch_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SCRATCH,
-                                TASK_TYPE_UCODE_LOAD, false);
-        cz_smu_populate_single_scratch_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_ARAM,
-                                TASK_TYPE_UCODE_LOAD, false);
-        cz_smu_populate_single_scratch_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_DRAM,
-                                TASK_TYPE_UCODE_LOAD, true);
-
-        return 0;
-}
-
-static int cz_smu_construct_toc_for_power_profiling(struct amdgpu_device *adev)
-{
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-
-        priv->toc_entry_power_profiling_index = priv->toc_entry_used_count;
-
-        cz_smu_populate_single_scratch_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_POWER_PROFILING,
-                                TASK_TYPE_INITIALIZE, true);
-        return 0;
-}
-
-static int cz_smu_construct_toc_for_bootup(struct amdgpu_device *adev)
-{
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-
-        priv->toc_entry_initialize_index = priv->toc_entry_used_count;
-
-        if (adev->firmware.smu_load) {
-                cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0, false);
-                if (adev->asic_type == CHIP_STONEY) {
-                        cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0, false);
-                } else {
-                        cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1, false);
-                }
-                cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE, false);
-                cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_PFP, false);
-                cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME, false);
-                cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
-                if (adev->asic_type == CHIP_STONEY) {
-                        cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
-                } else {
-                        cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2, false);
-                }
-                cz_smu_populate_single_ucode_load_task(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G, true);
-        }
-
-        return 0;
-}
-
-static int cz_smu_construct_toc_for_clock_table(struct amdgpu_device *adev)
-{
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-
-        priv->toc_entry_clock_table = priv->toc_entry_used_count;
-
-        cz_smu_populate_single_scratch_task(adev,
-                                CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE,
-                                TASK_TYPE_INITIALIZE, true);
-
-        return 0;
-}
-
-static int cz_smu_initialize_toc_empty_job_list(struct amdgpu_device *adev)
-{
-        int i;
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-        struct TOC *toc = (struct TOC *)priv->toc_buffer.kaddr;
-
-        for (i = 0; i < NUM_JOBLIST_ENTRIES; i++)
-                toc->JobList[i] = (uint8_t)IGNORE_JOB;
-
-        return 0;
-}
-
-/*
- * cz smu uninitialization
- */
-int cz_smu_fini(struct amdgpu_device *adev)
-{
-        amdgpu_bo_unref(&adev->smu.toc_buf);
-        amdgpu_bo_unref(&adev->smu.smu_buf);
-        kfree(adev->smu.priv);
-        adev->smu.priv = NULL;
-        if (adev->firmware.smu_load)
-                amdgpu_ucode_fini_bo(adev);
-
-        return 0;
-}
-
-int cz_smu_download_pptable(struct amdgpu_device *adev, void **table)
-{
-        uint8_t i;
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-
-        for (i = 0; i < priv->scratch_buffer_length; i++)
-                if (priv->scratch_buffer[i].firmware_ID ==
-                                CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE)
-                        break;
-
-        if (i >= priv->scratch_buffer_length) {
-                dev_err(adev->dev, "Invalid Scratch Type\n");
-                return -EINVAL;
-        }
-
-        *table = (struct SMU8_Fusion_ClkTable *)priv->scratch_buffer[i].kaddr;
-
-        /* prepare buffer for pptable */
-        cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetClkTableAddrHi,
-                                priv->scratch_buffer[i].mc_addr_high);
-        cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetClkTableAddrLo,
-                                priv->scratch_buffer[i].mc_addr_low);
-        cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_ExecuteJob,
-                                priv->toc_entry_clock_table);
-
-        /* actual downloading */
-        cz_send_msg_to_smc(adev, PPSMC_MSG_ClkTableXferToDram);
-
-        return 0;
-}
-
-int cz_smu_upload_pptable(struct amdgpu_device *adev)
-{
-        uint8_t i;
-        struct cz_smu_private_data *priv = cz_smu_get_priv(adev);
-
-        for (i = 0; i < priv->scratch_buffer_length; i++)
-                if (priv->scratch_buffer[i].firmware_ID ==
-                                CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE)
-                        break;
-
-        if (i >= priv->scratch_buffer_length) {
-                dev_err(adev->dev, "Invalid Scratch Type\n");
-                return -EINVAL;
-        }
-
-        /* prepare SMU */
-        cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetClkTableAddrHi,
-                                priv->scratch_buffer[i].mc_addr_high);
-        cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_SetClkTableAddrLo,
-                                priv->scratch_buffer[i].mc_addr_low);
-        cz_send_msg_to_smc_with_parameter(adev,
-                                PPSMC_MSG_ExecuteJob,
-                                priv->toc_entry_clock_table);
-
-        /* actual uploading */
-        cz_send_msg_to_smc(adev, PPSMC_MSG_ClkTableXferToSmu);
-
-        return 0;
-}
-
-/*
- * cz smumgr functions initialization
- */
-static const struct amdgpu_smumgr_funcs cz_smumgr_funcs = {
-        .check_fw_load_finish = cz_smu_check_finished,
-        .request_smu_load_fw = NULL,
-        .request_smu_specific_fw = NULL,
-};
-
-/*
- * cz smu initialization
- */
-int cz_smu_init(struct amdgpu_device *adev)
-{
-        int ret = -EINVAL;
-        uint64_t mc_addr = 0;
-        struct amdgpu_bo **toc_buf = &adev->smu.toc_buf;
-        struct amdgpu_bo **smu_buf = &adev->smu.smu_buf;
-        void *toc_buf_ptr = NULL;
-        void *smu_buf_ptr = NULL;
-
-        struct cz_smu_private_data *priv =
-                kzalloc(sizeof(struct cz_smu_private_data), GFP_KERNEL);
-        if (priv == NULL)
-                return -ENOMEM;
-
-        /* allocate firmware buffers */
-        if (adev->firmware.smu_load)
-                amdgpu_ucode_init_bo(adev);
-
-        adev->smu.priv = priv;
-        adev->smu.fw_flags = 0;
-        priv->toc_buffer.data_size = 4096;
-
-        priv->smu_buffer.data_size =
-                                ALIGN(UCODE_ID_RLC_SCRATCH_SIZE_BYTE, 32) +
-                                ALIGN(UCODE_ID_RLC_SRM_ARAM_SIZE_BYTE, 32) +
-                                ALIGN(UCODE_ID_RLC_SRM_DRAM_SIZE_BYTE, 32) +
-                                ALIGN(sizeof(struct SMU8_MultimediaPowerLogData), 32) +
-                                ALIGN(sizeof(struct SMU8_Fusion_ClkTable), 32);
-
-        /* prepare toc buffer and smu buffer:
-        * 1. create amdgpu_bo for toc buffer and smu buffer
-        * 2. pin mc address
-        * 3. map kernel virtual address
-        */
-        ret = amdgpu_bo_create(adev, priv->toc_buffer.data_size, PAGE_SIZE,
-                               true, AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
-                               toc_buf);
-
-        if (ret) {
-                dev_err(adev->dev, "(%d) SMC TOC buffer allocation failed\n", ret);
-                return ret;
-        }
-
-        ret = amdgpu_bo_create(adev, priv->smu_buffer.data_size, PAGE_SIZE,
-                               true, AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
-                               smu_buf);
-
-        if (ret) {
-                dev_err(adev->dev, "(%d) SMC Internal buffer allocation failed\n", ret);
-                return ret;
-        }
-
-        /* toc buffer reserve/pin/map */
-        ret = amdgpu_bo_reserve(adev->smu.toc_buf, false);
-        if (ret) {
-                amdgpu_bo_unref(&adev->smu.toc_buf);
-                dev_err(adev->dev, "(%d) SMC TOC buffer reserve failed\n", ret);
-                return ret;
-        }
-
-        ret = amdgpu_bo_pin(adev->smu.toc_buf, AMDGPU_GEM_DOMAIN_GTT, &mc_addr);
-        if (ret) {
-                amdgpu_bo_unreserve(adev->smu.toc_buf);
-                amdgpu_bo_unref(&adev->smu.toc_buf);
-                dev_err(adev->dev, "(%d) SMC TOC buffer pin failed\n", ret);
-                return ret;
-        }
-
-        ret = amdgpu_bo_kmap(*toc_buf, &toc_buf_ptr);
-        if (ret)
-                goto smu_init_failed;
-
-        amdgpu_bo_unreserve(adev->smu.toc_buf);
-
-        priv->toc_buffer.mc_addr_low = lower_32_bits(mc_addr);
-        priv->toc_buffer.mc_addr_high = upper_32_bits(mc_addr);
-        priv->toc_buffer.kaddr = toc_buf_ptr;
-
-        /* smu buffer reserve/pin/map */
-        ret = amdgpu_bo_reserve(adev->smu.smu_buf, false);
-        if (ret) {
-                amdgpu_bo_unref(&adev->smu.smu_buf);
-                dev_err(adev->dev, "(%d) SMC Internal buffer reserve failed\n", ret);
-                return ret;
-        }
-
-        ret = amdgpu_bo_pin(adev->smu.smu_buf, AMDGPU_GEM_DOMAIN_GTT, &mc_addr);
-        if (ret) {
-                amdgpu_bo_unreserve(adev->smu.smu_buf);
-                amdgpu_bo_unref(&adev->smu.smu_buf);
-                dev_err(adev->dev, "(%d) SMC Internal buffer pin failed\n", ret);
-                return ret;
-        }
-
-        ret = amdgpu_bo_kmap(*smu_buf, &smu_buf_ptr);
-        if (ret)
-                goto smu_init_failed;
-
-        amdgpu_bo_unreserve(adev->smu.smu_buf);
-
-        priv->smu_buffer.mc_addr_low = lower_32_bits(mc_addr);
-        priv->smu_buffer.mc_addr_high = upper_32_bits(mc_addr);
-        priv->smu_buffer.kaddr = smu_buf_ptr;
-
-        if (adev->firmware.smu_load) {
-                if (cz_smu_populate_single_firmware_entry(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0,
-                                &priv->driver_buffer[priv->driver_buffer_length++]))
-                        goto smu_init_failed;
-
-                if (adev->asic_type == CHIP_STONEY) {
-                        if (cz_smu_populate_single_firmware_entry(adev,
-                                        CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0,
-                                        &priv->driver_buffer[priv->driver_buffer_length++]))
-                                goto smu_init_failed;
-                } else {
-                        if (cz_smu_populate_single_firmware_entry(adev,
-                                        CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1,
-                                        &priv->driver_buffer[priv->driver_buffer_length++]))
-                                goto smu_init_failed;
-                }
-                if (cz_smu_populate_single_firmware_entry(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE,
-                                &priv->driver_buffer[priv->driver_buffer_length++]))
-                        goto smu_init_failed;
-                if (cz_smu_populate_single_firmware_entry(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_PFP,
-                                &priv->driver_buffer[priv->driver_buffer_length++]))
-                        goto smu_init_failed;
-                if (cz_smu_populate_single_firmware_entry(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME,
-                                &priv->driver_buffer[priv->driver_buffer_length++]))
-                        goto smu_init_failed;
-                if (cz_smu_populate_single_firmware_entry(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1,
-                                &priv->driver_buffer[priv->driver_buffer_length++]))
-                        goto smu_init_failed;
-                if (adev->asic_type == CHIP_STONEY) {
-                        if (cz_smu_populate_single_firmware_entry(adev,
-                                        CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1,
-                                        &priv->driver_buffer[priv->driver_buffer_length++]))
-                                goto smu_init_failed;
-                } else {
-                        if (cz_smu_populate_single_firmware_entry(adev,
-                                        CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2,
-                                        &priv->driver_buffer[priv->driver_buffer_length++]))
-                                goto smu_init_failed;
-                }
-                if (cz_smu_populate_single_firmware_entry(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G,
-                                &priv->driver_buffer[priv->driver_buffer_length++]))
-                        goto smu_init_failed;
-        }
-
-        if (cz_smu_populate_single_scratch_entry(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SCRATCH,
-                                UCODE_ID_RLC_SCRATCH_SIZE_BYTE,
-                                &priv->scratch_buffer[priv->scratch_buffer_length++]))
-                goto smu_init_failed;
-        if (cz_smu_populate_single_scratch_entry(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_ARAM,
-                                UCODE_ID_RLC_SRM_ARAM_SIZE_BYTE,
-                                &priv->scratch_buffer[priv->scratch_buffer_length++]))
-                goto smu_init_failed;
-        if (cz_smu_populate_single_scratch_entry(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_DRAM,
-                                UCODE_ID_RLC_SRM_DRAM_SIZE_BYTE,
-                                &priv->scratch_buffer[priv->scratch_buffer_length++]))
-                goto smu_init_failed;
-        if (cz_smu_populate_single_scratch_entry(adev,
-                                CZ_SCRATCH_ENTRY_UCODE_ID_POWER_PROFILING,
-                                sizeof(struct SMU8_MultimediaPowerLogData),
-                                &priv->scratch_buffer[priv->scratch_buffer_length++]))
-                goto smu_init_failed;
-        if (cz_smu_populate_single_scratch_entry(adev,
-                                CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE,
-                                sizeof(struct SMU8_Fusion_ClkTable),
-                                &priv->scratch_buffer[priv->scratch_buffer_length++]))
-                goto smu_init_failed;
-
-        cz_smu_initialize_toc_empty_job_list(adev);
-        cz_smu_construct_toc_for_rlc_aram_save(adev);
-        cz_smu_construct_toc_for_vddgfx_enter(adev);
-        cz_smu_construct_toc_for_vddgfx_exit(adev);
-        cz_smu_construct_toc_for_power_profiling(adev);
-        cz_smu_construct_toc_for_bootup(adev);
-        cz_smu_construct_toc_for_clock_table(adev);
-        /* init the smumgr functions */
-        adev->smu.smumgr_funcs = &cz_smumgr_funcs;
-
-        return 0;
-
-smu_init_failed:
-        amdgpu_bo_unref(toc_buf);
-        amdgpu_bo_unref(smu_buf);
-
-        return ret;
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_smumgr.h b/drivers/gpu/drm/amd/amdgpu/cz_smumgr.h
deleted file mode 100644
index 026342fcf0f3..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/cz_smumgr.h
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-#ifndef __CZ_SMC_H__
-#define __CZ_SMC_H__
-
-#define MAX_NUM_FIRMWARE                        8
-#define MAX_NUM_SCRATCH                         11
-#define CZ_SCRATCH_SIZE_NONGFX_CLOCKGATING      1024
-#define CZ_SCRATCH_SIZE_NONGFX_GOLDENSETTING    2048
-#define CZ_SCRATCH_SIZE_SDMA_METADATA           1024
-#define CZ_SCRATCH_SIZE_IH                      ((2*256+1)*4)
-
-enum cz_scratch_entry {
-        CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0 = 0,
-        CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1,
-        CZ_SCRATCH_ENTRY_UCODE_ID_CP_CE,
-        CZ_SCRATCH_ENTRY_UCODE_ID_CP_PFP,
-        CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME,
-        CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1,
-        CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2,
-        CZ_SCRATCH_ENTRY_UCODE_ID_GMCON_RENG,
-        CZ_SCRATCH_ENTRY_UCODE_ID_RLC_G,
-        CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SCRATCH,
-        CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_ARAM,
-        CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_DRAM,
-        CZ_SCRATCH_ENTRY_UCODE_ID_DMCU_ERAM,
-        CZ_SCRATCH_ENTRY_UCODE_ID_DMCU_IRAM,
-        CZ_SCRATCH_ENTRY_UCODE_ID_POWER_PROFILING,
-        CZ_SCRATCH_ENTRY_DATA_ID_SDMA_HALT,
-        CZ_SCRATCH_ENTRY_DATA_ID_SYS_CLOCKGATING,
-        CZ_SCRATCH_ENTRY_DATA_ID_SDMA_RING_REGS,
-        CZ_SCRATCH_ENTRY_DATA_ID_NONGFX_REINIT,
-        CZ_SCRATCH_ENTRY_DATA_ID_SDMA_START,
-        CZ_SCRATCH_ENTRY_DATA_ID_IH_REGISTERS,
-        CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE
-};
-
-struct cz_buffer_entry {
-        uint32_t        data_size;
-        uint32_t        mc_addr_low;
-        uint32_t        mc_addr_high;
-        void            *kaddr;
-        enum cz_scratch_entry firmware_ID;
-};
-
-struct cz_register_index_data_pair {
-        uint32_t        offset;
-        uint32_t        value;
-};
-
-struct cz_ih_meta_data {
-        uint32_t        command;
-        struct cz_register_index_data_pair register_index_value_pair[1];
-};
-
-struct cz_smu_private_data {
-        uint8_t         driver_buffer_length;
-        uint8_t         scratch_buffer_length;
-        uint16_t        toc_entry_used_count;
-        uint16_t        toc_entry_initialize_index;
-        uint16_t        toc_entry_power_profiling_index;
-        uint16_t        toc_entry_aram;
-        uint16_t        toc_entry_ih_register_restore_task_index;
-        uint16_t        toc_entry_clock_table;
-        uint16_t        ih_register_restore_task_size;
-        uint16_t        smu_buffer_used_bytes;
-
-        struct cz_buffer_entry  toc_buffer;
-        struct cz_buffer_entry  smu_buffer;
-        struct cz_buffer_entry  driver_buffer[MAX_NUM_FIRMWARE];
-        struct cz_buffer_entry  scratch_buffer[MAX_NUM_SCRATCH];
-};
-
-#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index b323f5ef64d2..c998f6aaaf36 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -25,7 +25,7 @@
 #include "amdgpu_ih.h"
 #include "amdgpu_gfx.h"
 #include "amdgpu_ucode.h"
-#include "si/clearstate_si.h"
+#include "clearstate_si.h"
 #include "bif/bif_3_0_d.h"
 #include "bif/bif_3_0_sh_mask.h"
 #include "oss/oss_1_0_d.h"
@@ -1794,14 +1794,9 @@ static void gfx_v6_0_gpu_init(struct amdgpu_device *adev)
 
 static void gfx_v6_0_scratch_init(struct amdgpu_device *adev)
 {
-        int i;
-
         adev->gfx.scratch.num_reg = 7;
         adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
-        for (i = 0; i < adev->gfx.scratch.num_reg; i++) {
-                adev->gfx.scratch.free[i] = true;
-                adev->gfx.scratch.reg[i] = adev->gfx.scratch.reg_base + i;
-        }
+        adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
 }
 
 static int gfx_v6_0_ring_test_ring(struct amdgpu_ring *ring)
@@ -1975,7 +1970,7 @@ static int gfx_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
         ib.ptr[2] = 0xDEADBEEF;
         ib.length_dw = 3;
 
-        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, NULL, &f);
+        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
         if (r)
                 goto err2;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index c4e14015ec5b..e3589b55a1e1 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -2003,14 +2003,9 @@ static void gfx_v7_0_gpu_init(struct amdgpu_device *adev)
  */
 static void gfx_v7_0_scratch_init(struct amdgpu_device *adev)
 {
-        int i;
-
         adev->gfx.scratch.num_reg = 7;
         adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
-        for (i = 0; i < adev->gfx.scratch.num_reg; i++) {
-                adev->gfx.scratch.free[i] = true;
-                adev->gfx.scratch.reg[i] = adev->gfx.scratch.reg_base + i;
-        }
+        adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
 }
 
 /**
@@ -2321,7 +2316,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
         ib.ptr[2] = 0xDEADBEEF;
         ib.length_dw = 3;
 
-        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, NULL, &f);
+        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
         if (r)
                 goto err2;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 373374164bd5..35f9cd83b821 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -657,6 +657,8 @@ static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev);
 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev);
 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev);
 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev);
+static void gfx_v8_0_ring_emit_ce_meta_init(struct amdgpu_ring *ring, uint64_t addr);
+static void gfx_v8_0_ring_emit_de_meta_init(struct amdgpu_ring *ring, uint64_t addr);
 
 static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
 {
@@ -749,14 +751,9 @@ static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
 
 static void gfx_v8_0_scratch_init(struct amdgpu_device *adev)
 {
-        int i;
-
         adev->gfx.scratch.num_reg = 7;
         adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
-        for (i = 0; i < adev->gfx.scratch.num_reg; i++) {
-                adev->gfx.scratch.free[i] = true;
-                adev->gfx.scratch.reg[i] = adev->gfx.scratch.reg_base + i;
-        }
+        adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
 }
 
 static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
@@ -829,7 +826,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
         ib.ptr[2] = 0xDEADBEEF;
         ib.length_dw = 3;
 
-        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, NULL, &f);
+        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
         if (r)
                 goto err2;
 
@@ -941,6 +938,13 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
                 goto out;
         cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
         adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
+
+        /* chain ib ucode isn't formal released, just disable it by far
+         * TODO: when ucod ready we should use ucode version to judge if
+         * chain-ib support or not.
+         */
+        adev->virt.chained_ib_support = false;
+
         adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 
         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
@@ -1367,6 +1371,51 @@ static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
         }
 }
 
+static int gfx_v8_0_kiq_init_ring(struct amdgpu_device *adev,
+                                  struct amdgpu_ring *ring,
+                                  struct amdgpu_irq_src *irq)
+{
+        int r = 0;
+
+        if (amdgpu_sriov_vf(adev)) {
+                r = amdgpu_wb_get(adev, &adev->virt.reg_val_offs);
+                if (r)
+                        return r;
+        }
+
+        ring->adev = NULL;
+        ring->ring_obj = NULL;
+        ring->use_doorbell = true;
+        ring->doorbell_index = AMDGPU_DOORBELL_KIQ;
+        if (adev->gfx.mec2_fw) {
+                ring->me = 2;
+                ring->pipe = 0;
+        } else {
+                ring->me = 1;
+                ring->pipe = 1;
+        }
+
+        irq->data = ring;
+        ring->queue = 0;
+        sprintf(ring->name, "kiq %d.%d.%d", ring->me, ring->pipe, ring->queue);
+        r = amdgpu_ring_init(adev, ring, 1024,
+                             irq, AMDGPU_CP_KIQ_IRQ_DRIVER0);
+        if (r)
+                dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r);
+
+        return r;
+}
+
+static void gfx_v8_0_kiq_free_ring(struct amdgpu_ring *ring,
+                                   struct amdgpu_irq_src *irq)
+{
+        if (amdgpu_sriov_vf(ring->adev))
+                amdgpu_wb_free(ring->adev, ring->adev->virt.reg_val_offs);
+
+        amdgpu_ring_fini(ring);
+        irq->data = NULL;
+}
+
 #define MEC_HPD_SIZE 2048
 
 static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
@@ -1421,6 +1470,35 @@ static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
         return 0;
 }
 
+static void gfx_v8_0_kiq_fini(struct amdgpu_device *adev)
+{
+        struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+
+        amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL);
+        kiq->eop_obj = NULL;
+}
+
+static int gfx_v8_0_kiq_init(struct amdgpu_device *adev)
+{
+        int r;
+        u32 *hpd;
+        struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+
+        r = amdgpu_bo_create_kernel(adev, MEC_HPD_SIZE, PAGE_SIZE,
+                                    AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj,
+                                    &kiq->eop_gpu_addr, (void **)&hpd);
+        if (r) {
+                dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r);
+                return r;
+        }
+
+        memset(hpd, 0, MEC_HPD_SIZE);
+
+        amdgpu_bo_kunmap(kiq->eop_obj);
+
+        return 0;
+}
+
 static const u32 vgpr_init_compute_shader[] =
 {
         0x7e000209, 0x7e020208,
@@ -1702,7 +1780,7 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
         ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
 
         /* shedule the ib on the ring */
-        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, NULL, &f);
+        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
         if (r) {
                 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
                 goto fail;
@@ -1997,8 +2075,14 @@ static int gfx_v8_0_sw_init(void *handle)
 {
         int i, r;
         struct amdgpu_ring *ring;
+        struct amdgpu_kiq *kiq;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+        /* KIQ event */
+        r = amdgpu_irq_add_id(adev, 178, &adev->gfx.kiq.irq);
+        if (r)
+                return r;
+
         /* EOP Event */
         r = amdgpu_irq_add_id(adev, 181, &adev->gfx.eop_irq);
         if (r)
@@ -2036,6 +2120,17 @@ static int gfx_v8_0_sw_init(void *handle)
                 return r;
         }
 
+        r = gfx_v8_0_kiq_init(adev);
+        if (r) {
+                DRM_ERROR("Failed to init KIQ BOs!\n");
+                return r;
+        }
+
+        kiq = &adev->gfx.kiq;
+        r = gfx_v8_0_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
+        if (r)
+                return r;
+
         /* set up the gfx ring */
         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
                 ring = &adev->gfx.gfx_ring[i];
@@ -2119,7 +2214,9 @@ static int gfx_v8_0_sw_fini(void *handle)
                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
         for (i = 0; i < adev->gfx.num_compute_rings; i++)
                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
+        gfx_v8_0_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
 
+        gfx_v8_0_kiq_fini(adev);
         gfx_v8_0_mec_fini(adev);
         gfx_v8_0_rlc_fini(adev);
         gfx_v8_0_free_microcode(adev);
@@ -4024,17 +4121,6 @@ static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
                 WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8);
                 gfx_v8_0_init_power_gating(adev);
                 WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
-                if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
-                        cz_enable_sck_slow_down_on_power_up(adev, true);
-                        cz_enable_sck_slow_down_on_power_down(adev, true);
-                } else {
-                        cz_enable_sck_slow_down_on_power_up(adev, false);
-                        cz_enable_sck_slow_down_on_power_down(adev, false);
-                }
-                if (adev->pg_flags & AMD_PG_SUPPORT_CP)
-                        cz_enable_cp_power_gating(adev, true);
-                else
-                        cz_enable_cp_power_gating(adev, false);
         } else if ((adev->asic_type == CHIP_POLARIS11) ||
                    (adev->asic_type == CHIP_POLARIS12)) {
                 gfx_v8_0_init_csb(adev);
@@ -4506,6 +4592,393 @@ static void gfx_v8_0_cp_compute_fini(struct amdgpu_device *adev)
         }
 }
 
+/* KIQ functions */
+static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring)
+{
+        uint32_t tmp;
+        struct amdgpu_device *adev = ring->adev;
+
+        /* tell RLC which is KIQ queue */
+        tmp = RREG32(mmRLC_CP_SCHEDULERS);
+        tmp &= 0xffffff00;
+        tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
+        WREG32(mmRLC_CP_SCHEDULERS, tmp);
+        tmp |= 0x80;
+        WREG32(mmRLC_CP_SCHEDULERS, tmp);
+}
+
+static void gfx_v8_0_kiq_enable(struct amdgpu_ring *ring)
+{
+        amdgpu_ring_alloc(ring, 8);
+        /* set resources */
+        amdgpu_ring_write(ring, PACKET3(PACKET3_SET_RESOURCES, 6));
+        amdgpu_ring_write(ring, 0);     /* vmid_mask:0 queue_type:0 (KIQ) */
+        amdgpu_ring_write(ring, 0x000000FF);    /* queue mask lo */
+        amdgpu_ring_write(ring, 0);     /* queue mask hi */
+        amdgpu_ring_write(ring, 0);     /* gws mask lo */
+        amdgpu_ring_write(ring, 0);     /* gws mask hi */
+        amdgpu_ring_write(ring, 0);     /* oac mask */
+        amdgpu_ring_write(ring, 0);     /* gds heap base:0, gds heap size:0 */
+        amdgpu_ring_commit(ring);
+        udelay(50);
+}
+
+static void gfx_v8_0_map_queue_enable(struct amdgpu_ring *kiq_ring,
+                                   struct amdgpu_ring *ring)
+{
+        struct amdgpu_device *adev = kiq_ring->adev;
+        uint64_t mqd_addr, wptr_addr;
+
+        mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
+        wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+        amdgpu_ring_alloc(kiq_ring, 8);
+
+        amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
+        /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
+        amdgpu_ring_write(kiq_ring, 0x21010000);
+        amdgpu_ring_write(kiq_ring, (ring->doorbell_index << 2) |
+                        (ring->queue << 26) |
+                        (ring->pipe << 29) |
+                        ((ring->me == 1 ? 0 : 1) << 31)); /* doorbell */
+        amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
+        amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
+        amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
+        amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
+        amdgpu_ring_commit(kiq_ring);
+        udelay(50);
+}
+
+static int gfx_v8_0_mqd_init(struct amdgpu_device *adev,
+                             struct vi_mqd *mqd,
+                             uint64_t mqd_gpu_addr,
+                             uint64_t eop_gpu_addr,
+                             struct amdgpu_ring *ring)
+{
+        uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
+        uint32_t tmp;
+
+        mqd->header = 0xC0310800;
+        mqd->compute_pipelinestat_enable = 0x00000001;
+        mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
+        mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
+        mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
+        mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
+        mqd->compute_misc_reserved = 0x00000003;
+
+        eop_base_addr = eop_gpu_addr >> 8;
+        mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
+        mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
+
+        /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
+        tmp = RREG32(mmCP_HQD_EOP_CONTROL);
+        tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
+                        (order_base_2(MEC_HPD_SIZE / 4) - 1));
+
+        mqd->cp_hqd_eop_control = tmp;
+
+        /* enable doorbell? */
+        tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
+
+        if (ring->use_doorbell)
+                tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+                                         DOORBELL_EN, 1);
+        else
+                tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+                                         DOORBELL_EN, 0);
+
+        mqd->cp_hqd_pq_doorbell_control = tmp;
+
+        /* disable the queue if it's active */
+        mqd->cp_hqd_dequeue_request = 0;
+        mqd->cp_hqd_pq_rptr = 0;
+        mqd->cp_hqd_pq_wptr = 0;
+
+        /* set the pointer to the MQD */
+        mqd->cp_mqd_base_addr_lo = mqd_gpu_addr & 0xfffffffc;
+        mqd->cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
+
+        /* set MQD vmid to 0 */
+        tmp = RREG32(mmCP_MQD_CONTROL);
+        tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
+        mqd->cp_mqd_control = tmp;
+
+        /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
+        hqd_gpu_addr = ring->gpu_addr >> 8;
+        mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
+        mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
+
+        /* set up the HQD, this is similar to CP_RB0_CNTL */
+        tmp = RREG32(mmCP_HQD_PQ_CONTROL);
+        tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
+                            (order_base_2(ring->ring_size / 4) - 1));
+        tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
+                        ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
+#ifdef __BIG_ENDIAN
+        tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
+#endif
+        tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
+        tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
+        tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
+        tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
+        mqd->cp_hqd_pq_control = tmp;
+
+        /* set the wb address whether it's enabled or not */
+        wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
+        mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
+        mqd->cp_hqd_pq_rptr_report_addr_hi =
+                upper_32_bits(wb_gpu_addr) & 0xffff;
+
+        /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
+        wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+        mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
+        mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
+
+        tmp = 0;
+        /* enable the doorbell if requested */
+        if (ring->use_doorbell) {
+                tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
+                tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+                                DOORBELL_OFFSET, ring->doorbell_index);
+
+                tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+                                         DOORBELL_EN, 1);
+                tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+                                         DOORBELL_SOURCE, 0);
+                tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+                                         DOORBELL_HIT, 0);
+        }
+
+        mqd->cp_hqd_pq_doorbell_control = tmp;
+
+        /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
+        ring->wptr = 0;
+        mqd->cp_hqd_pq_wptr = ring->wptr;
+        mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
+
+        /* set the vmid for the queue */
+        mqd->cp_hqd_vmid = 0;
+
+        tmp = RREG32(mmCP_HQD_PERSISTENT_STATE);
+        tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
+        mqd->cp_hqd_persistent_state = tmp;
+
+        /* activate the queue */
+        mqd->cp_hqd_active = 1;
+
+        return 0;
+}
+
+static int gfx_v8_0_kiq_init_register(struct amdgpu_device *adev,
+                                      struct vi_mqd *mqd,
+                                      struct amdgpu_ring *ring)
+{
+        uint32_t tmp;
+        int j;
+
+        /* disable wptr polling */
+        tmp = RREG32(mmCP_PQ_WPTR_POLL_CNTL);
+        tmp = REG_SET_FIELD(tmp, CP_PQ_WPTR_POLL_CNTL, EN, 0);
+        WREG32(mmCP_PQ_WPTR_POLL_CNTL, tmp);
+
+        WREG32(mmCP_HQD_EOP_BASE_ADDR, mqd->cp_hqd_eop_base_addr_lo);
+        WREG32(mmCP_HQD_EOP_BASE_ADDR_HI, mqd->cp_hqd_eop_base_addr_hi);
+
+        /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
+        WREG32(mmCP_HQD_EOP_CONTROL, mqd->cp_hqd_eop_control);
+
+        /* enable doorbell? */
+        WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, mqd->cp_hqd_pq_doorbell_control);
+
+        /* disable the queue if it's active */
+        if (RREG32(mmCP_HQD_ACTIVE) & 1) {
+                WREG32(mmCP_HQD_DEQUEUE_REQUEST, 1);
+                for (j = 0; j < adev->usec_timeout; j++) {
+                        if (!(RREG32(mmCP_HQD_ACTIVE) & 1))
+                                break;
+                        udelay(1);
+                }
+                WREG32(mmCP_HQD_DEQUEUE_REQUEST, mqd->cp_hqd_dequeue_request);
+                WREG32(mmCP_HQD_PQ_RPTR, mqd->cp_hqd_pq_rptr);
+                WREG32(mmCP_HQD_PQ_WPTR, mqd->cp_hqd_pq_wptr);
+        }
+
+        /* set the pointer to the MQD */
+        WREG32(mmCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
+        WREG32(mmCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
+
+        /* set MQD vmid to 0 */
+        WREG32(mmCP_MQD_CONTROL, mqd->cp_mqd_control);
+
+        /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
+        WREG32(mmCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
+        WREG32(mmCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
+
+        /* set up the HQD, this is similar to CP_RB0_CNTL */
+        WREG32(mmCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
+
+        /* set the wb address whether it's enabled or not */
+        WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR,
+                                mqd->cp_hqd_pq_rptr_report_addr_lo);
+        WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
+                                mqd->cp_hqd_pq_rptr_report_addr_hi);
+
+        /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
+        WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR, mqd->cp_hqd_pq_wptr_poll_addr_lo);
+        WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, mqd->cp_hqd_pq_wptr_poll_addr_hi);
+
+        /* enable the doorbell if requested */
+        if (ring->use_doorbell) {
+                if ((adev->asic_type == CHIP_CARRIZO) ||
+                                (adev->asic_type == CHIP_FIJI) ||
+                                (adev->asic_type == CHIP_STONEY)) {
+                        WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER,
+                                                AMDGPU_DOORBELL_KIQ << 2);
+                        WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER,
+                                                AMDGPU_DOORBELL_MEC_RING7 << 2);
+                }
+        }
+        WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, mqd->cp_hqd_pq_doorbell_control);
+
+        /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
+        WREG32(mmCP_HQD_PQ_WPTR, mqd->cp_hqd_pq_wptr);
+
+        /* set the vmid for the queue */
+        WREG32(mmCP_HQD_VMID, mqd->cp_hqd_vmid);
+
+        WREG32(mmCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
+
+        /* activate the queue */
+        WREG32(mmCP_HQD_ACTIVE, mqd->cp_hqd_active);
+
+        if (ring->use_doorbell) {
+                tmp = RREG32(mmCP_PQ_STATUS);
+                tmp = REG_SET_FIELD(tmp, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
+                WREG32(mmCP_PQ_STATUS, tmp);
+        }
+
+        return 0;
+}
+
+static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring,
+                                   struct vi_mqd *mqd,
+                                   u64 mqd_gpu_addr)
+{
+        struct amdgpu_device *adev = ring->adev;
+        struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+        uint64_t eop_gpu_addr;
+        bool is_kiq = false;
+
+        if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
+                is_kiq = true;
+
+        if (is_kiq) {
+                eop_gpu_addr = kiq->eop_gpu_addr;
+                gfx_v8_0_kiq_setting(&kiq->ring);
+        } else
+                eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr +
+                                        ring->queue * MEC_HPD_SIZE;
+
+        mutex_lock(&adev->srbm_mutex);
+        vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
+
+        gfx_v8_0_mqd_init(adev, mqd, mqd_gpu_addr, eop_gpu_addr, ring);
+
+        if (is_kiq)
+                gfx_v8_0_kiq_init_register(adev, mqd, ring);
+
+        vi_srbm_select(adev, 0, 0, 0, 0);
+        mutex_unlock(&adev->srbm_mutex);
+
+        if (is_kiq)
+                gfx_v8_0_kiq_enable(ring);
+        else
+                gfx_v8_0_map_queue_enable(&kiq->ring, ring);
+
+        return 0;
+}
+
+static void gfx_v8_0_kiq_free_queue(struct amdgpu_device *adev)
+{
+        struct amdgpu_ring *ring = NULL;
+        int i;
+
+        for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+                ring = &adev->gfx.compute_ring[i];
+                amdgpu_bo_free_kernel(&ring->mqd_obj, NULL, NULL);
+                ring->mqd_obj = NULL;
+        }
+
+        ring = &adev->gfx.kiq.ring;
+        amdgpu_bo_free_kernel(&ring->mqd_obj, NULL, NULL);
+        ring->mqd_obj = NULL;
+}
+
+static int gfx_v8_0_kiq_setup_queue(struct amdgpu_device *adev,
+                                    struct amdgpu_ring *ring)
+{
+        struct vi_mqd *mqd;
+        u64 mqd_gpu_addr;
+        u32 *buf;
+        int r = 0;
+
+        r = amdgpu_bo_create_kernel(adev, sizeof(struct vi_mqd), PAGE_SIZE,
+                                    AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
+                                    &mqd_gpu_addr, (void **)&buf);
+        if (r) {
+                dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
+                return r;
+        }
+
+        /* init the mqd struct */
+        memset(buf, 0, sizeof(struct vi_mqd));
+        mqd = (struct vi_mqd *)buf;
+
+        r = gfx_v8_0_kiq_init_queue(ring, mqd, mqd_gpu_addr);
+        if (r)
+                return r;
+
+        amdgpu_bo_kunmap(ring->mqd_obj);
+
+        return 0;
+}
+
+static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev)
+{
+        struct amdgpu_ring *ring = NULL;
+        int r, i;
+
+        ring = &adev->gfx.kiq.ring;
+        r = gfx_v8_0_kiq_setup_queue(adev, ring);
+        if (r)
+                return r;
+
+        for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+                ring = &adev->gfx.compute_ring[i];
+                r = gfx_v8_0_kiq_setup_queue(adev, ring);
+                if (r)
+                        return r;
+        }
+
+        gfx_v8_0_cp_compute_enable(adev, true);
+
+        for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+                ring = &adev->gfx.compute_ring[i];
+
+                ring->ready = true;
+                r = amdgpu_ring_test_ring(ring);
+                if (r)
+                        ring->ready = false;
+        }
+
+        ring = &adev->gfx.kiq.ring;
+        ring->ready = true;
+        r = amdgpu_ring_test_ring(ring);
+        if (r)
+                ring->ready = false;
+
+        return 0;
+}
+
 static int gfx_v8_0_cp_compute_resume(struct amdgpu_device *adev)
 {
         int r, i, j;
@@ -4806,7 +5279,10 @@ static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
         if (r)
                 return r;
 
-        r = gfx_v8_0_cp_compute_resume(adev);
+        if (amdgpu_sriov_vf(adev))
+                r = gfx_v8_0_kiq_resume(adev);
+        else
+                r = gfx_v8_0_cp_compute_resume(adev);
         if (r)
                 return r;
 
@@ -4845,6 +5321,7 @@ static int gfx_v8_0_hw_fini(void *handle)
         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
         if (amdgpu_sriov_vf(adev)) {
+                gfx_v8_0_kiq_free_queue(adev);
                 pr_debug("For SRIOV client, shouldn't do anything.\n");
                 return 0;
         }
@@ -5360,6 +5837,18 @@ static int gfx_v8_0_set_powergating_state(void *handle,
         case CHIP_CARRIZO:
         case CHIP_STONEY:
 
+                if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
+                        cz_enable_sck_slow_down_on_power_up(adev, true);
+                        cz_enable_sck_slow_down_on_power_down(adev, true);
+                } else {
+                        cz_enable_sck_slow_down_on_power_up(adev, false);
+                        cz_enable_sck_slow_down_on_power_down(adev, false);
+                }
+                if (adev->pg_flags & AMD_PG_SUPPORT_CP)
+                        cz_enable_cp_power_gating(adev, true);
+                else
+                        cz_enable_cp_power_gating(adev, false);
+
                 cz_update_gfx_cg_power_gating(adev, enable);
 
                 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
@@ -5396,6 +5885,45 @@ static int gfx_v8_0_set_powergating_state(void *handle,
         return 0;
 }
 
+static void gfx_v8_0_get_clockgating_state(void *handle, u32 *flags)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int data;
+
+        /* AMD_CG_SUPPORT_GFX_MGCG */
+        data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
+        if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK))
+                *flags |= AMD_CG_SUPPORT_GFX_MGCG;
+
+        /* AMD_CG_SUPPORT_GFX_CGLG */
+        data = RREG32(mmRLC_CGCG_CGLS_CTRL);
+        if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
+                *flags |= AMD_CG_SUPPORT_GFX_CGCG;
+
+        /* AMD_CG_SUPPORT_GFX_CGLS */
+        if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
+                *flags |= AMD_CG_SUPPORT_GFX_CGLS;
+
+        /* AMD_CG_SUPPORT_GFX_CGTS */
+        data = RREG32(mmCGTS_SM_CTRL_REG);
+        if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK))
+                *flags |= AMD_CG_SUPPORT_GFX_CGTS;
+
+        /* AMD_CG_SUPPORT_GFX_CGTS_LS */
+        if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK))
+                *flags |= AMD_CG_SUPPORT_GFX_CGTS_LS;
+
+        /* AMD_CG_SUPPORT_GFX_RLC_LS */
+        data = RREG32(mmRLC_MEM_SLP_CNTL);
+        if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
+                *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
+
+        /* AMD_CG_SUPPORT_GFX_CP_LS */
+        data = RREG32(mmCP_MEM_SLP_CNTL);
+        if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
+                *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
+}
+
 static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
                                      uint32_t reg_addr, uint32_t cmd)
 {
@@ -5444,68 +5972,6 @@ static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
 #define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001
 #define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e
 
-static void cz_enter_rlc_safe_mode(struct amdgpu_device *adev)
-{
-        u32 data = 0;
-        unsigned i;
-
-        data = RREG32(mmRLC_CNTL);
-        if ((data & RLC_CNTL__RLC_ENABLE_F32_MASK) == 0)
-                return;
-
-        if ((adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG)) ||
-            (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | AMD_PG_SUPPORT_GFX_SMG |
-                               AMD_PG_SUPPORT_GFX_DMG))) {
-                data |= RLC_GPR_REG2__REQ_MASK;
-                data &= ~RLC_GPR_REG2__MESSAGE_MASK;
-                data |= (MSG_ENTER_RLC_SAFE_MODE << RLC_GPR_REG2__MESSAGE__SHIFT);
-                WREG32(mmRLC_GPR_REG2, data);
-
-                for (i = 0; i < adev->usec_timeout; i++) {
-                        if ((RREG32(mmRLC_GPM_STAT) &
-                             (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
-                              RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) ==
-                            (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
-                             RLC_GPM_STAT__GFX_POWER_STATUS_MASK))
-                                break;
-                        udelay(1);
-                }
-
-                for (i = 0; i < adev->usec_timeout; i++) {
-                        if (!REG_GET_FIELD(RREG32(mmRLC_GPR_REG2), RLC_GPR_REG2, REQ))
-                                break;
-                        udelay(1);
-                }
-                adev->gfx.rlc.in_safe_mode = true;
-        }
-}
-
-static void cz_exit_rlc_safe_mode(struct amdgpu_device *adev)
-{
-        u32 data;
-        unsigned i;
-
-        data = RREG32(mmRLC_CNTL);
-        if ((data & RLC_CNTL__RLC_ENABLE_F32_MASK) == 0)
-                return;
-
-        if ((adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG)) ||
-            (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | AMD_PG_SUPPORT_GFX_SMG |
-                               AMD_PG_SUPPORT_GFX_DMG))) {
-                data |= RLC_GPR_REG2__REQ_MASK;
-                data &= ~RLC_GPR_REG2__MESSAGE_MASK;
-                data |= (MSG_EXIT_RLC_SAFE_MODE << RLC_GPR_REG2__MESSAGE__SHIFT);
-                WREG32(mmRLC_GPR_REG2, data);
-                adev->gfx.rlc.in_safe_mode = false;
-        }
-
-        for (i = 0; i < adev->usec_timeout; i++) {
-                if (!REG_GET_FIELD(RREG32(mmRLC_GPR_REG2), RLC_GPR_REG2, REQ))
-                        break;
-                udelay(1);
-        }
-}
-
 static void iceland_enter_rlc_safe_mode(struct amdgpu_device *adev)
 {
         u32 data;
@@ -5565,31 +6031,11 @@ static void iceland_exit_rlc_safe_mode(struct amdgpu_device *adev)
         }
 }
 
-static void gfx_v8_0_nop_enter_rlc_safe_mode(struct amdgpu_device *adev)
-{
-        adev->gfx.rlc.in_safe_mode = true;
-}
-
-static void gfx_v8_0_nop_exit_rlc_safe_mode(struct amdgpu_device *adev)
-{
-        adev->gfx.rlc.in_safe_mode = false;
-}
-
-static const struct amdgpu_rlc_funcs cz_rlc_funcs = {
-        .enter_safe_mode = cz_enter_rlc_safe_mode,
-        .exit_safe_mode = cz_exit_rlc_safe_mode
-};
-
 static const struct amdgpu_rlc_funcs iceland_rlc_funcs = {
         .enter_safe_mode = iceland_enter_rlc_safe_mode,
         .exit_safe_mode = iceland_exit_rlc_safe_mode
 };
 
-static const struct amdgpu_rlc_funcs gfx_v8_0_nop_rlc_funcs = {
-        .enter_safe_mode = gfx_v8_0_nop_enter_rlc_safe_mode,
-        .exit_safe_mode = gfx_v8_0_nop_exit_rlc_safe_mode
-};
-
 static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
                                                       bool enable)
 {
@@ -6011,7 +6457,8 @@ static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
         u32 ref_and_mask, reg_mem_engine;
 
-        if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
+        if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) ||
+            (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) {
                 switch (ring->me) {
                 case 1:
                         ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
@@ -6224,6 +6671,31 @@ static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
         amdgpu_ring_write(ring, upper_32_bits(seq));
 }
 
+static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
+                                         u64 seq, unsigned int flags)
+{
+        /* we only allocate 32bit for each seq wb address */
+        BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+
+        /* write fence seq to the "addr" */
+        amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+        amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+                                 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
+        amdgpu_ring_write(ring, lower_32_bits(addr));
+        amdgpu_ring_write(ring, upper_32_bits(addr));
+        amdgpu_ring_write(ring, lower_32_bits(seq));
+
+        if (flags & AMDGPU_FENCE_FLAG_INT) {
+                /* set register to trigger INT */
+                amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+                amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+                                         WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
+                amdgpu_ring_write(ring, mmCPC_INT_STATUS);
+                amdgpu_ring_write(ring, 0);
+                amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
+        }
+}
+
 static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)
 {
         amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
@@ -6234,6 +6706,10 @@ static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
 {
         uint32_t dw2 = 0;
 
+        if (amdgpu_sriov_vf(ring->adev))
+                gfx_v8_0_ring_emit_ce_meta_init(ring,
+                        (flags & AMDGPU_VM_DOMAIN) ? AMDGPU_CSA_VADDR : ring->adev->virt.csa_vmid0_addr);
+
         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
                 gfx_v8_0_ring_emit_vgt_flush(ring);
@@ -6258,6 +6734,36 @@ static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
         amdgpu_ring_write(ring, dw2);
         amdgpu_ring_write(ring, 0);
+
+        if (amdgpu_sriov_vf(ring->adev))
+                gfx_v8_0_ring_emit_de_meta_init(ring,
+                        (flags & AMDGPU_VM_DOMAIN) ? AMDGPU_CSA_VADDR : ring->adev->virt.csa_vmid0_addr);
+}
+
+static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
+{
+        struct amdgpu_device *adev = ring->adev;
+
+        amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
+        amdgpu_ring_write(ring, 0 |     /* src: register*/
+                                (5 << 8) |      /* dst: memory */
+                                (1 << 20));     /* write confirm */
+        amdgpu_ring_write(ring, reg);
+        amdgpu_ring_write(ring, 0);
+        amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
+                                adev->virt.reg_val_offs * 4));
+        amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
+                                adev->virt.reg_val_offs * 4));
+}
+
+static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
+                                  uint32_t val)
+{
+        amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+        amdgpu_ring_write(ring, (1 << 16)); /* no inc addr */
+        amdgpu_ring_write(ring, reg);
+        amdgpu_ring_write(ring, 0);
+        amdgpu_ring_write(ring, val);
 }
 
 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
@@ -6405,6 +6911,72 @@ static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev,
         return 0;
 }
 
+static int gfx_v8_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
+                                            struct amdgpu_irq_src *src,
+                                            unsigned int type,
+                                            enum amdgpu_interrupt_state state)
+{
+        uint32_t tmp, target;
+        struct amdgpu_ring *ring = (struct amdgpu_ring *)src->data;
+
+        BUG_ON(!ring || (ring->funcs->type != AMDGPU_RING_TYPE_KIQ));
+
+        if (ring->me == 1)
+                target = mmCP_ME1_PIPE0_INT_CNTL;
+        else
+                target = mmCP_ME2_PIPE0_INT_CNTL;
+        target += ring->pipe;
+
+        switch (type) {
+        case AMDGPU_CP_KIQ_IRQ_DRIVER0:
+                if (state == AMDGPU_IRQ_STATE_DISABLE) {
+                        tmp = RREG32(mmCPC_INT_CNTL);
+                        tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
+                                                 GENERIC2_INT_ENABLE, 0);
+                        WREG32(mmCPC_INT_CNTL, tmp);
+
+                        tmp = RREG32(target);
+                        tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
+                                                 GENERIC2_INT_ENABLE, 0);
+                        WREG32(target, tmp);
+                } else {
+                        tmp = RREG32(mmCPC_INT_CNTL);
+                        tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
+                                                 GENERIC2_INT_ENABLE, 1);
+                        WREG32(mmCPC_INT_CNTL, tmp);
+
+                        tmp = RREG32(target);
+                        tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
+                                                 GENERIC2_INT_ENABLE, 1);
+                        WREG32(target, tmp);
+                }
+                break;
+        default:
+                BUG(); /* kiq only support GENERIC2_INT now */
+                break;
+        }
+        return 0;
+}
+
+static int gfx_v8_0_kiq_irq(struct amdgpu_device *adev,
+                            struct amdgpu_irq_src *source,
+                            struct amdgpu_iv_entry *entry)
+{
+        u8 me_id, pipe_id, queue_id;
+        struct amdgpu_ring *ring = (struct amdgpu_ring *)source->data;
+
+        BUG_ON(!ring || (ring->funcs->type != AMDGPU_RING_TYPE_KIQ));
+
+        me_id = (entry->ring_id & 0x0c) >> 2;
+        pipe_id = (entry->ring_id & 0x03) >> 0;
+        queue_id = (entry->ring_id & 0x70) >> 4;
+        DRM_DEBUG("IH: CPC GENERIC2_INT, me:%d, pipe:%d, queue:%d\n",
+                   me_id, pipe_id, queue_id);
+
+        amdgpu_fence_process(ring);
+        return 0;
+}
+
 static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
         .name = "gfx_v8_0",
         .early_init = gfx_v8_0_early_init,
@@ -6423,6 +6995,7 @@ static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
         .post_soft_reset = gfx_v8_0_post_soft_reset,
         .set_clockgating_state = gfx_v8_0_set_clockgating_state,
         .set_powergating_state = gfx_v8_0_set_powergating_state,
+        .get_clockgating_state = gfx_v8_0_get_clockgating_state,
 };
 
 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
@@ -6440,7 +7013,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
                 7 + /* gfx_v8_0_ring_emit_pipeline_sync */
                 128 + 19 + /* gfx_v8_0_ring_emit_vm_flush */
                 2 + /* gfx_v8_ring_emit_sb */
-                3 + 4, /* gfx_v8_ring_emit_cntxcntl including vgt flush */
+                3 + 4 + 29, /* gfx_v8_ring_emit_cntxcntl including vgt flush/meta-data */
         .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_gfx */
         .emit_ib = gfx_v8_0_ring_emit_ib_gfx,
         .emit_fence = gfx_v8_0_ring_emit_fence_gfx,
@@ -6485,10 +7058,39 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
         .pad_ib = amdgpu_ring_generic_pad_ib,
 };
 
+static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
+        .type = AMDGPU_RING_TYPE_KIQ,
+        .align_mask = 0xff,
+        .nop = PACKET3(PACKET3_NOP, 0x3FFF),
+        .get_rptr = gfx_v8_0_ring_get_rptr,
+        .get_wptr = gfx_v8_0_ring_get_wptr_compute,
+        .set_wptr = gfx_v8_0_ring_set_wptr_compute,
+        .emit_frame_size =
+                20 + /* gfx_v8_0_ring_emit_gds_switch */
+                7 + /* gfx_v8_0_ring_emit_hdp_flush */
+                5 + /* gfx_v8_0_ring_emit_hdp_invalidate */
+                7 + /* gfx_v8_0_ring_emit_pipeline_sync */
+                17 + /* gfx_v8_0_ring_emit_vm_flush */
+                7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */
+        .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_compute */
+        .emit_ib = gfx_v8_0_ring_emit_ib_compute,
+        .emit_fence = gfx_v8_0_ring_emit_fence_kiq,
+        .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
+        .emit_hdp_invalidate = gfx_v8_0_ring_emit_hdp_invalidate,
+        .test_ring = gfx_v8_0_ring_test_ring,
+        .test_ib = gfx_v8_0_ring_test_ib,
+        .insert_nop = amdgpu_ring_insert_nop,
+        .pad_ib = amdgpu_ring_generic_pad_ib,
+        .emit_rreg = gfx_v8_0_ring_emit_rreg,
+        .emit_wreg = gfx_v8_0_ring_emit_wreg,
+};
+
 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
 {
         int i;
 
+        adev->gfx.kiq.ring.funcs = &gfx_v8_0_ring_funcs_kiq;
+
         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
                 adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx;
 
@@ -6511,6 +7113,11 @@ static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = {
         .process = gfx_v8_0_priv_inst_irq,
 };
 
+static const struct amdgpu_irq_src_funcs gfx_v8_0_kiq_irq_funcs = {
+        .set = gfx_v8_0_kiq_set_interrupt_state,
+        .process = gfx_v8_0_kiq_irq,
+};
+
 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev)
 {
         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
@@ -6521,22 +7128,14 @@ static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev)
 
         adev->gfx.priv_inst_irq.num_types = 1;
         adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs;
+
+        adev->gfx.kiq.irq.num_types = AMDGPU_CP_KIQ_IRQ_LAST;
+        adev->gfx.kiq.irq.funcs = &gfx_v8_0_kiq_irq_funcs;
 }
 
 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev)
 {
-        switch (adev->asic_type) {
-        case CHIP_TOPAZ:
-                adev->gfx.rlc.funcs = &iceland_rlc_funcs;
-                break;
-        case CHIP_STONEY:
-        case CHIP_CARRIZO:
-                adev->gfx.rlc.funcs = &cz_rlc_funcs;
-                break;
-        default:
-                adev->gfx.rlc.funcs = &gfx_v8_0_nop_rlc_funcs;
-                break;
-        }
+        adev->gfx.rlc.funcs = &iceland_rlc_funcs;
 }
 
 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev)
@@ -6653,3 +7252,62 @@ const struct amdgpu_ip_block_version gfx_v8_1_ip_block =
         .rev = 0,
         .funcs = &gfx_v8_0_ip_funcs,
 };
+
+static void gfx_v8_0_ring_emit_ce_meta_init(struct amdgpu_ring *ring, uint64_t csa_addr)
+{
+        uint64_t ce_payload_addr;
+        int cnt_ce;
+        static union {
+                struct amdgpu_ce_ib_state regular;
+                struct amdgpu_ce_ib_state_chained_ib chained;
+        } ce_payload = {0};
+
+        if (ring->adev->virt.chained_ib_support) {
+                ce_payload_addr = csa_addr + offsetof(struct amdgpu_gfx_meta_data_chained_ib, ce_payload);
+                cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
+        } else {
+                ce_payload_addr = csa_addr + offsetof(struct amdgpu_gfx_meta_data, ce_payload);
+                cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;
+        }
+
+        amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce));
+        amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
+                                WRITE_DATA_DST_SEL(8) |
+                                WR_CONFIRM) |
+                                WRITE_DATA_CACHE_POLICY(0));
+        amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr));
+        amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr));
+        amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2);
+}
+
+static void gfx_v8_0_ring_emit_de_meta_init(struct amdgpu_ring *ring, uint64_t csa_addr)
+{
+        uint64_t de_payload_addr, gds_addr;
+        int cnt_de;
+        static union {
+                struct amdgpu_de_ib_state regular;
+                struct amdgpu_de_ib_state_chained_ib chained;
+        } de_payload = {0};
+
+        gds_addr = csa_addr + 4096;
+        if (ring->adev->virt.chained_ib_support) {
+                de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);
+                de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr);
+                de_payload_addr = csa_addr + offsetof(struct amdgpu_gfx_meta_data_chained_ib, de_payload);
+                cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2;
+        } else {
+                de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr);
+                de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr);
+                de_payload_addr = csa_addr + offsetof(struct amdgpu_gfx_meta_data, de_payload);
+                cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2;
+        }
+
+        amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de));
+        amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
+                                WRITE_DATA_DST_SEL(8) |
+                                WR_CONFIRM) |
+                                WRITE_DATA_CACHE_POLICY(0));
+        amdgpu_ring_write(ring, lower_32_bits(de_payload_addr));
+        amdgpu_ring_write(ring, upper_32_bits(de_payload_addr));
+        amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 273b16fb9459..8d05e0c4e3d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -375,9 +375,16 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
         /* size in MB on si */
         adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
         adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
-        adev->mc.visible_vram_size = adev->mc.aper_size;
+
+#ifdef CONFIG_X86_64
+        if (adev->flags & AMD_IS_APU) {
+                adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
+                adev->mc.aper_size = adev->mc.real_vram_size;
+        }
+#endif
 
         /* In case the PCI BAR is larger than the actual amount of vram */
+        adev->mc.visible_vram_size = adev->mc.aper_size;
         if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
                 adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 476bc9f1954b..7669b3259f35 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -467,9 +467,16 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
         /* size in MB on si */
         adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
         adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
-        adev->mc.visible_vram_size = adev->mc.aper_size;
+
+#ifdef CONFIG_X86_64
+        if (adev->flags & AMD_IS_APU) {
+                adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
+                adev->mc.aper_size = adev->mc.real_vram_size;
+        }
+#endif
 
         /* In case the PCI BAR is larger than the actual amount of vram */
+        adev->mc.visible_vram_size = adev->mc.aper_size;
         if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
                 adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
@@ -1439,6 +1446,21 @@ static int gmc_v8_0_set_powergating_state(void *handle,
         return 0;
 }
 
+static void gmc_v8_0_get_clockgating_state(void *handle, u32 *flags)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int data;
+
+        /* AMD_CG_SUPPORT_MC_MGCG */
+        data = RREG32(mmMC_HUB_MISC_HUB_CG);
+        if (data & MC_HUB_MISC_HUB_CG__ENABLE_MASK)
+                *flags |= AMD_CG_SUPPORT_MC_MGCG;
+
+        /* AMD_CG_SUPPORT_MC_LS */
+        if (data & MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK)
+                *flags |= AMD_CG_SUPPORT_MC_LS;
+}
+
 static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
         .name = "gmc_v8_0",
         .early_init = gmc_v8_0_early_init,
@@ -1457,6 +1479,7 @@ static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
         .post_soft_reset = gmc_v8_0_post_soft_reset,
         .set_clockgating_state = gmc_v8_0_set_clockgating_state,
         .set_powergating_state = gmc_v8_0_set_powergating_state,
+        .get_clockgating_state = gmc_v8_0_get_clockgating_state,
 };
 
 static const struct amdgpu_gart_funcs gmc_v8_0_gart_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index 5a1bc358bcb1..8785ca570729 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -1230,6 +1230,7 @@ static void kv_update_current_ps(struct amdgpu_device *adev,
         pi->current_rps = *rps;
         pi->current_ps = *new_ps;
         pi->current_rps.ps_priv = &pi->current_ps;
+        adev->pm.dpm.current_ps = &pi->current_rps;
 }
 
 static void kv_update_requested_ps(struct amdgpu_device *adev,
@@ -1241,6 +1242,7 @@ static void kv_update_requested_ps(struct amdgpu_device *adev,
         pi->requested_rps = *rps;
         pi->requested_ps = *new_ps;
         pi->requested_rps.ps_priv = &pi->requested_ps;
+        adev->pm.dpm.requested_ps = &pi->requested_rps;
 }
 
 static void kv_dpm_enable_bapm(struct amdgpu_device *adev, bool enable)
@@ -1904,19 +1906,19 @@ static int kv_enable_nb_dpm(struct amdgpu_device *adev,
 }
 
 static int kv_dpm_force_performance_level(struct amdgpu_device *adev,
-                                          enum amdgpu_dpm_forced_level level)
+                                          enum amd_dpm_forced_level level)
 {
         int ret;
 
-        if (level == AMDGPU_DPM_FORCED_LEVEL_HIGH) {
+        if (level == AMD_DPM_FORCED_LEVEL_HIGH) {
                 ret = kv_force_dpm_highest(adev);
                 if (ret)
                         return ret;
-        } else if (level == AMDGPU_DPM_FORCED_LEVEL_LOW) {
+        } else if (level == AMD_DPM_FORCED_LEVEL_LOW) {
                 ret = kv_force_dpm_lowest(adev);
                 if (ret)
                         return ret;
-        } else if (level == AMDGPU_DPM_FORCED_LEVEL_AUTO) {
+        } else if (level == AMD_DPM_FORCED_LEVEL_AUTO) {
                 ret = kv_unforce_levels(adev);
                 if (ret)
                         return ret;
@@ -3009,7 +3011,6 @@ static int kv_dpm_late_init(void *handle)
         kv_dpm_powergate_samu(adev, true);
         kv_dpm_powergate_vce(adev, true);
         kv_dpm_powergate_uvd(adev, true);
-
         return 0;
 }
 
@@ -3029,7 +3030,7 @@ static int kv_dpm_sw_init(void *handle)
         /* default to balanced state */
         adev->pm.dpm.state = POWER_STATE_TYPE_BALANCED;
         adev->pm.dpm.user_state = POWER_STATE_TYPE_BALANCED;
-        adev->pm.dpm.forced_level = AMDGPU_DPM_FORCED_LEVEL_AUTO;
+        adev->pm.dpm.forced_level = AMD_DPM_FORCED_LEVEL_AUTO;
         adev->pm.default_sclk = adev->clock.default_sclk;
         adev->pm.default_mclk = adev->clock.default_mclk;
         adev->pm.current_sclk = adev->clock.default_sclk;
@@ -3078,6 +3079,9 @@ static int kv_dpm_hw_init(void *handle)
         int ret;
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+        if (!amdgpu_dpm)
+                return 0;
+
         mutex_lock(&adev->pm.mutex);
         kv_dpm_setup_asic(adev);
         ret = kv_dpm_enable(adev);
@@ -3245,15 +3249,52 @@ static int kv_dpm_set_powergating_state(void *handle,
         return 0;
 }
 
+static inline bool kv_are_power_levels_equal(const struct kv_pl *kv_cpl1,
+                                                const struct kv_pl *kv_cpl2)
+{
+        return ((kv_cpl1->sclk == kv_cpl2->sclk) &&
+                  (kv_cpl1->vddc_index == kv_cpl2->vddc_index) &&
+                  (kv_cpl1->ds_divider_index == kv_cpl2->ds_divider_index) &&
+                  (kv_cpl1->force_nbp_state == kv_cpl2->force_nbp_state));
+}
+
 static int kv_check_state_equal(struct amdgpu_device *adev,
                                 struct amdgpu_ps *cps,
                                 struct amdgpu_ps *rps,
                                 bool *equal)
 {
-        if (equal == NULL)
+        struct kv_ps *kv_cps;
+        struct kv_ps *kv_rps;
+        int i;
+
+        if (adev == NULL || cps == NULL || rps == NULL || equal == NULL)
                 return -EINVAL;
 
-        *equal = false;
+        kv_cps = kv_get_ps(cps);
+        kv_rps = kv_get_ps(rps);
+
+        if (kv_cps == NULL) {
+                *equal = false;
+                return 0;
+        }
+
+        if (kv_cps->num_levels != kv_rps->num_levels) {
+                *equal = false;
+                return 0;
+        }
+
+        for (i = 0; i < kv_cps->num_levels; i++) {
+                if (!kv_are_power_levels_equal(&(kv_cps->levels[i]),
+                                        &(kv_rps->levels[i]))) {
+                        *equal = false;
+                        return 0;
+                }
+        }
+
+        /* If all performance levels are the same try to use the UVD clocks to break the tie.*/
+        *equal = ((cps->vclk == rps->vclk) && (cps->dclk == rps->dclk));
+        *equal &= ((cps->evclk == rps->evclk) && (cps->ecclk == rps->ecclk));
+
         return 0;
 }
 
@@ -3307,12 +3348,3 @@ static void kv_dpm_set_irq_funcs(struct amdgpu_device *adev)
         adev->pm.dpm.thermal.irq.num_types = AMDGPU_THERMAL_IRQ_LAST;
         adev->pm.dpm.thermal.irq.funcs = &kv_dpm_irq_funcs;
 }
-
-const struct amdgpu_ip_block_version kv_dpm_ip_block =
-{
-        .type = AMD_IP_BLOCK_TYPE_SMC,
-        .major = 7,
-        .minor = 0,
-        .rev = 0,
-        .funcs = &kv_dpm_ip_funcs,
-};
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
new file mode 100644
index 000000000000..d2622b6f49fa
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
@@ -0,0 +1,592 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Xiangliang.Yu@amd.com
+ */
+
+#include "amdgpu.h"
+#include "vi.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "vid.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gmc_v8_0.h"
+#include "gfx_v8_0.h"
+#include "sdma_v3_0.h"
+#include "tonga_ih.h"
+#include "gmc/gmc_8_2_d.h"
+#include "gmc/gmc_8_2_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "oss/oss_3_0_sh_mask.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "smu/smu_7_1_3_d.h"
+#include "mxgpu_vi.h"
+
+/* VI golden setting */
+static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
+        mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+        mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
+        mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
+        mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
+        mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+        mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+        mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
+        mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+        mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
+        mmPCIE_INDEX, 0xffffffff, 0x0140001c,
+        mmPCIE_DATA, 0x000f0000, 0x00000000,
+        mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
+        mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
+        mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+        mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+        mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+        mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
+        mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+        mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
+        mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
+        mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_fiji_golden_settings_a10[] = {
+        mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
+        mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+        mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+        mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+        mmFBC_MISC, 0x1f311fff, 0x12300000,
+        mmHDMI_CONTROL, 0x31000111, 0x00000011,
+        mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+        mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+        mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+        mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+        mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+        mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+        mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+        mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
+        mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+        mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+};
+
+static const u32 xgpu_fiji_golden_common_all[] = {
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+        mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
+        mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
+        mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
+        mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+        mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+        mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+        mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+        mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
+};
+
+static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
+        mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
+        mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+        mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
+        mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
+        mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
+        mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
+        mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
+        mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
+        mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
+        mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
+        mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
+        mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
+        mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
+        mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
+        mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
+        mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
+        mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
+        mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
+        mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
+        mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
+        mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
+        mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
+        mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
+        mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+        mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+        mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
+        mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+        mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+        mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+        mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+        mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+        mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+        mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+        mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+        mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
+        mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+        mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+        mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+        mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+        mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
+        mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+        mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+        mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+        mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+        mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
+        mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+        mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+        mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+        mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+        mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+        mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+        mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+        mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+        mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+        mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
+        mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+        mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+        mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+        mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+        mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
+        mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+        mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+        mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+        mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+        mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
+        mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+        mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+        mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
+        mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
+        mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
+        mmPCIE_INDEX,               0xffffffff, 0x0140001c,
+        mmPCIE_DATA,                0x000f0000, 0x00000000,
+        mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
+        mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
+        mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
+        mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
+        mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
+        mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
+        mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
+        mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
+        mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
+        mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_tonga_golden_settings_a11[] = {
+        mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
+        mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
+        mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+        mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+        mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+        mmFBC_MISC, 0x1f311fff, 0x12300000,
+        mmGB_GPU_ID, 0x0000000f, 0x00000000,
+        mmHDMI_CONTROL, 0x31000111, 0x00000011,
+        mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
+        mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
+        mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
+        mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+        mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
+        mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+        mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
+        mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+        mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
+        mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+        mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
+        mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+        mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+        mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+        mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+        mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+        mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
+        mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
+        mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
+        mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+        mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+};
+
+static const u32 xgpu_tonga_golden_common_all[] = {
+        mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
+        mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
+        mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
+        mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
+        mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
+        mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
+        mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+};
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev)
+{
+        switch (adev->asic_type) {
+        case CHIP_FIJI:
+                amdgpu_program_register_sequence(adev,
+                                                 xgpu_fiji_mgcg_cgcg_init,
+                                                 (const u32)ARRAY_SIZE(
+                                                 xgpu_fiji_mgcg_cgcg_init));
+                amdgpu_program_register_sequence(adev,
+                                                 xgpu_fiji_golden_settings_a10,
+                                                 (const u32)ARRAY_SIZE(
+                                                 xgpu_fiji_golden_settings_a10));
+                amdgpu_program_register_sequence(adev,
+                                                 xgpu_fiji_golden_common_all,
+                                                 (const u32)ARRAY_SIZE(
+                                                 xgpu_fiji_golden_common_all));
+                break;
+        case CHIP_TONGA:
+                amdgpu_program_register_sequence(adev,
+                                                 xgpu_tonga_mgcg_cgcg_init,
+                                                 (const u32)ARRAY_SIZE(
+                                                 xgpu_tonga_mgcg_cgcg_init));
+                amdgpu_program_register_sequence(adev,
+                                                 xgpu_tonga_golden_settings_a11,
+                                                 (const u32)ARRAY_SIZE(
+                                                 xgpu_tonga_golden_settings_a11));
+                amdgpu_program_register_sequence(adev,
+                                                 xgpu_tonga_golden_common_all,
+                                                 (const u32)ARRAY_SIZE(
+                                                 xgpu_tonga_golden_common_all));
+                break;
+        default:
+                BUG_ON("Doesn't support chip type.\n");
+                break;
+        }
+}
+
+/*
+ * Mailbox communication between GPU hypervisor and VFs
+ */
+static void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev)
+{
+        u32 reg;
+
+        reg = RREG32(mmMAILBOX_CONTROL);
+        reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
+        WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, bool val)
+{
+        u32 reg;
+
+        reg = RREG32(mmMAILBOX_CONTROL);
+        reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
+                            TRN_MSG_VALID, val ? 1 : 0);
+        WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
+                                      enum idh_event event)
+{
+        u32 reg;
+
+        reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
+        reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
+                            MSGBUF_DATA, event);
+        WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
+
+        xgpu_vi_mailbox_set_valid(adev, true);
+}
+
+static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
+                                   enum idh_event event)
+{
+        u32 reg;
+
+        reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
+        if (reg != event)
+                return -ENOENT;
+
+        /* send ack to PF */
+        xgpu_vi_mailbox_send_ack(adev);
+
+        return 0;
+}
+
+static int xgpu_vi_poll_ack(struct amdgpu_device *adev)
+{
+        int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+        u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
+        u32 reg;
+
+        reg = RREG32(mmMAILBOX_CONTROL);
+        while (!(reg & mask)) {
+                if (timeout <= 0) {
+                        pr_err("Doesn't get ack from pf.\n");
+                        r = -ETIME;
+                        break;
+                }
+                msleep(1);
+                timeout -= 1;
+
+                reg = RREG32(mmMAILBOX_CONTROL);
+        }
+
+        return r;
+}
+
+static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum idh_event event)
+{
+        int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+
+        r = xgpu_vi_mailbox_rcv_msg(adev, event);
+        while (r) {
+                if (timeout <= 0) {
+                        pr_err("Doesn't get ack from pf.\n");
+                        r = -ETIME;
+                        break;
+                }
+                msleep(1);
+                timeout -= 1;
+
+                r = xgpu_vi_mailbox_rcv_msg(adev, event);
+        }
+
+        return r;
+}
+
+static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
+                                        enum idh_request request)
+{
+        int r;
+
+        xgpu_vi_mailbox_trans_msg(adev, request);
+
+        /* start to poll ack */
+        r = xgpu_vi_poll_ack(adev);
+        if (r)
+                return r;
+
+        xgpu_vi_mailbox_set_valid(adev, false);
+
+        /* start to check msg if request is idh_req_gpu_init_access */
+        if (request == IDH_REQ_GPU_INIT_ACCESS) {
+                r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
+                if (r)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int xgpu_vi_request_reset(struct amdgpu_device *adev)
+{
+        return xgpu_vi_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS);
+}
+
+static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
+                                           bool init)
+{
+        enum idh_event event;
+
+        event = init ? IDH_REQ_GPU_INIT_ACCESS : IDH_REQ_GPU_FINI_ACCESS;
+        return xgpu_vi_send_access_requests(adev, event);
+}
+
+static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
+                                           bool init)
+{
+        enum idh_event event;
+        int r = 0;
+
+        event = init ? IDH_REL_GPU_INIT_ACCESS : IDH_REL_GPU_FINI_ACCESS;
+        r = xgpu_vi_send_access_requests(adev, event);
+
+        return r;
+}
+
+/* add support mailbox interrupts */
+static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
+                                   struct amdgpu_irq_src *source,
+                                   struct amdgpu_iv_entry *entry)
+{
+        DRM_DEBUG("get ack intr and do nothing.\n");
+        return 0;
+}
+
+static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
+                                       struct amdgpu_irq_src *src,
+                                       unsigned type,
+                                       enum amdgpu_interrupt_state state)
+{
+        u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+        tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
+                            (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+        WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+        return 0;
+}
+
+static void xgpu_vi_mailbox_flr_work(struct work_struct *work)
+{
+        struct amdgpu_virt *virt = container_of(work,
+                                        struct amdgpu_virt, flr_work.work);
+        struct amdgpu_device *adev = container_of(virt,
+                                        struct amdgpu_device, virt);
+        int r = 0;
+
+        r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
+        if (r)
+                DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
+
+        /* TODO: need to restore gfx states */
+}
+
+static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
+                                       struct amdgpu_irq_src *src,
+                                       unsigned type,
+                                       enum amdgpu_interrupt_state state)
+{
+        u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+        tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
+                            (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+        WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+        return 0;
+}
+
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+                                   struct amdgpu_irq_src *source,
+                                   struct amdgpu_iv_entry *entry)
+{
+        int r;
+
+        adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+        r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+        /* do nothing for other msg */
+        if (r)
+                return 0;
+
+        /* TODO: need to save gfx states */
+        schedule_delayed_work(&adev->virt.flr_work,
+                              msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+        return 0;
+}
+
+static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_ack_irq_funcs = {
+        .set = xgpu_vi_set_mailbox_ack_irq,
+        .process = xgpu_vi_mailbox_ack_irq,
+};
+
+static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_rcv_irq_funcs = {
+        .set = xgpu_vi_set_mailbox_rcv_irq,
+        .process = xgpu_vi_mailbox_rcv_irq,
+};
+
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev)
+{
+        adev->virt.ack_irq.num_types = 1;
+        adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
+        adev->virt.rcv_irq.num_types = 1;
+        adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs;
+}
+
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev)
+{
+        int r;
+
+        r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
+        if (r)
+                return r;
+
+        r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
+        if (r) {
+                amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+                return r;
+        }
+
+        return 0;
+}
+
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev)
+{
+        int r;
+
+        r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
+        if (r)
+                return r;
+        r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
+        if (r) {
+                amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+                return r;
+        }
+
+        INIT_DELAYED_WORK(&adev->virt.flr_work, xgpu_vi_mailbox_flr_work);
+
+        return 0;
+}
+
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev)
+{
+        cancel_delayed_work_sync(&adev->virt.flr_work);
+        amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
+        amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+}
+
+const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
+        .req_full_gpu           = xgpu_vi_request_full_gpu_access,
+        .rel_full_gpu           = xgpu_vi_release_full_gpu_access,
+        .reset_gpu              = xgpu_vi_request_reset,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
new file mode 100644
index 000000000000..fd6216efd2b0
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __MXGPU_VI_H__
+#define __MXGPU_VI_H__
+
+#define VI_MAILBOX_TIMEDOUT     150
+#define VI_MAILBOX_RESET_TIME   12
+
+/* VI mailbox messages request */
+enum idh_request {
+        IDH_REQ_GPU_INIT_ACCESS = 1,
+        IDH_REL_GPU_INIT_ACCESS,
+        IDH_REQ_GPU_FINI_ACCESS,
+        IDH_REL_GPU_FINI_ACCESS,
+        IDH_REQ_GPU_RESET_ACCESS
+};
+
+/* VI mailbox messages data */
+enum idh_event {
+        IDH_CLR_MSG_BUF = 0,
+        IDH_READY_TO_ACCESS_GPU,
+        IDH_FLR_NOTIFICATION,
+        IDH_FLR_NOTIFICATION_CMPL,
+        IDH_EVENT_MAX
+};
+
+extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev);
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev);
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev);
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev);
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index fbe74a33899c..896be64b7013 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -701,7 +701,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
         ib.ptr[7] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
         ib.length_dw = 8;
 
-        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, NULL, &f);
+        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
         if (r)
                 goto err1;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 034ace79ed49..31375bdde6f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -910,7 +910,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
         ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
         ib.length_dw = 8;
 
-        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, NULL, &f);
+        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
         if (r)
                 goto err1;
 
@@ -1533,6 +1533,22 @@ static int sdma_v3_0_set_powergating_state(void *handle,
         return 0;
 }
 
+static void sdma_v3_0_get_clockgating_state(void *handle, u32 *flags)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int data;
+
+        /* AMD_CG_SUPPORT_SDMA_MGCG */
+        data = RREG32(mmSDMA0_CLK_CTRL + sdma_offsets[0]);
+        if (!(data & SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK))
+                *flags |= AMD_CG_SUPPORT_SDMA_MGCG;
+
+        /* AMD_CG_SUPPORT_SDMA_LS */
+        data = RREG32(mmSDMA0_POWER_CNTL + sdma_offsets[0]);
+        if (data & SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
+                *flags |= AMD_CG_SUPPORT_SDMA_LS;
+}
+
 static const struct amd_ip_funcs sdma_v3_0_ip_funcs = {
         .name = "sdma_v3_0",
         .early_init = sdma_v3_0_early_init,
@@ -1551,6 +1567,7 @@ static const struct amd_ip_funcs sdma_v3_0_ip_funcs = {
         .soft_reset = sdma_v3_0_soft_reset,
         .set_clockgating_state = sdma_v3_0_set_clockgating_state,
         .set_powergating_state = sdma_v3_0_set_powergating_state,
+        .get_clockgating_state = sdma_v3_0_get_clockgating_state,
 };
 
 static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index c46b0159007d..da46992f7b18 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -32,7 +32,7 @@
 #include "amdgpu_vce.h"
 #include "atom.h"
 #include "amdgpu_powerplay.h"
-#include "si/sid.h"
+#include "sid.h"
 #include "si_ih.h"
 #include "gfx_v6_0.h"
 #include "gmc_v6_0.h"
@@ -40,337 +40,343 @@
 #include "dce_v6_0.h"
 #include "si.h"
 #include "dce_virtual.h"
+#include "gca/gfx_6_0_d.h"
+#include "oss/oss_1_0_d.h"
+#include "gmc/gmc_6_0_d.h"
+#include "dce/dce_6_0_d.h"
+#include "uvd/uvd_4_0_d.h"
 
 static const u32 tahiti_golden_registers[] =
 {
-        0x17bc, 0x00000030, 0x00000011,
-        0x2684, 0x00010000, 0x00018208,
-        0x260c, 0xffffffff, 0x00000000,
-        0x260d, 0xf00fffff, 0x00000400,
-        0x260e, 0x0002021c, 0x00020200,
-        0x031e, 0x00000080, 0x00000000,
+        mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
+        mmCB_HW_CONTROL, 0x00010000, 0x00018208,
+        mmDB_DEBUG, 0xffffffff, 0x00000000,
+        mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+        mmDB_DEBUG3, 0x0002021c, 0x00020200,
+        mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
         0x340c, 0x000000c0, 0x00800040,
         0x360c, 0x000000c0, 0x00800040,
-        0x16ec, 0x000000f0, 0x00000070,
-        0x16f0, 0x00200000, 0x50100000,
-        0x1c0c, 0x31000311, 0x00000011,
-        0x09df, 0x00000003, 0x000007ff,
-        0x0903, 0x000007ff, 0x00000000,
-        0x2285, 0xf000001f, 0x00000007,
-        0x22c9, 0xffffffff, 0x00ffffff,
-        0x22c4, 0x0000ff0f, 0x00000000,
-        0xa293, 0x07ffffff, 0x4e000000,
-        0xa0d4, 0x3f3f3fff, 0x2a00126a,
+        mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+        mmFBC_MISC, 0x00200000, 0x50100000,
+        mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
+        mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
+        mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
+        mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
+        mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
+        mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+        mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
+        mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
         0x000c, 0xffffffff, 0x0040,
         0x000d, 0x00000040, 0x00004040,
-        0x2440, 0x07ffffff, 0x03000000,
-        0x23a2, 0x01ff1f3f, 0x00000000,
-        0x23a1, 0x01ff1f3f, 0x00000000,
-        0x2418, 0x0000007f, 0x00000020,
-        0x2542, 0x00010000, 0x00010000,
-        0x2b05, 0x00000200, 0x000002fb,
-        0x2b04, 0xffffffff, 0x0000543b,
-        0x2b03, 0xffffffff, 0xa9210876,
-        0x2234, 0xffffffff, 0x000fff40,
-        0x2235, 0x0000001f, 0x00000010,
-        0x0504, 0x20000000, 0x20fffed8,
-        0x0570, 0x000c0fc0, 0x000c0400,
-        0x052c, 0x0fffffff, 0xffffffff,
-        0x052d, 0x0fffffff, 0x0fffffff,
-        0x052e, 0x0fffffff, 0x0fffffff,
-        0x052f, 0x0fffffff, 0x0fffffff
+        mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
+        mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
+        mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
+        mmSX_DEBUG_1, 0x0000007f, 0x00000020,
+        mmTA_CNTL_AUX, 0x00010000, 0x00010000,
+        mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
+        mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
+        mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
+        mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
+        mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
+        mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
+        mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
+        mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
+        mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 };
 
 static const u32 tahiti_golden_registers2[] =
 {
-        0x0319, 0x00000001, 0x00000001
+        mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
 };
 
 static const u32 tahiti_golden_rlc_registers[] =
 {
-        0x263e, 0xffffffff, 0x12011003,
-        0x3109, 0xffffffff, 0x00601005,
+        mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
+        mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
         0x311f, 0xffffffff, 0x10104040,
         0x3122, 0xffffffff, 0x0100000a,
-        0x30c5, 0xffffffff, 0x00000800,
-        0x30c3, 0xffffffff, 0x800000f4,
-        0x3d2a, 0x00000008, 0x00000000
+        mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
+        mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
+        mmUVD_CGC_GATE, 0x00000008, 0x00000000,
 };
 
 static const u32 pitcairn_golden_registers[] =
 {
-        0x17bc, 0x00000030, 0x00000011,
-        0x2684, 0x00010000, 0x00018208,
-        0x260c, 0xffffffff, 0x00000000,
-        0x260d, 0xf00fffff, 0x00000400,
-        0x260e, 0x0002021c, 0x00020200,
-        0x031e, 0x00000080, 0x00000000,
+        mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
+        mmCB_HW_CONTROL, 0x00010000, 0x00018208,
+        mmDB_DEBUG, 0xffffffff, 0x00000000,
+        mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+        mmDB_DEBUG3, 0x0002021c, 0x00020200,
+        mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
         0x340c, 0x000300c0, 0x00800040,
         0x360c, 0x000300c0, 0x00800040,
-        0x16ec, 0x000000f0, 0x00000070,
-        0x16f0, 0x00200000, 0x50100000,
-        0x1c0c, 0x31000311, 0x00000011,
-        0x0ab9, 0x00073ffe, 0x000022a2,
-        0x0903, 0x000007ff, 0x00000000,
-        0x2285, 0xf000001f, 0x00000007,
-        0x22c9, 0xffffffff, 0x00ffffff,
-        0x22c4, 0x0000ff0f, 0x00000000,
-        0xa293, 0x07ffffff, 0x4e000000,
-        0xa0d4, 0x3f3f3fff, 0x2a00126a,
+        mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+        mmFBC_MISC, 0x00200000, 0x50100000,
+        mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
+        mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
+        mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
+        mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
+        mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
+        mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+        mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
+        mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
         0x000c, 0xffffffff, 0x0040,
         0x000d, 0x00000040, 0x00004040,
-        0x2440, 0x07ffffff, 0x03000000,
-        0x2418, 0x0000007f, 0x00000020,
-        0x2542, 0x00010000, 0x00010000,
-        0x2b05, 0x000003ff, 0x000000f7,
-        0x2b04, 0xffffffff, 0x00000000,
-        0x2b03, 0xffffffff, 0x32761054,
-        0x2235, 0x0000001f, 0x00000010,
-        0x0570, 0x000c0fc0, 0x000c0400,
-        0x052c, 0x0fffffff, 0xffffffff,
-        0x052d, 0x0fffffff, 0x0fffffff,
-        0x052e, 0x0fffffff, 0x0fffffff,
-        0x052f, 0x0fffffff, 0x0fffffff
+        mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
+        mmSX_DEBUG_1, 0x0000007f, 0x00000020,
+        mmTA_CNTL_AUX, 0x00010000, 0x00010000,
+        mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
+        mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
+        mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
+        mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
+        mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
+        mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
+        mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 };
 
 static const u32 pitcairn_golden_rlc_registers[] =
 {
-        0x263e, 0xffffffff, 0x12011003,
-        0x3109, 0xffffffff, 0x00601004,
+        mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
+        mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
         0x311f, 0xffffffff, 0x10102020,
         0x3122, 0xffffffff, 0x01000020,
-        0x30c5, 0xffffffff, 0x00000800,
-        0x30c3, 0xffffffff, 0x800000a4
+        mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
+        mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
 };
 
 static const u32 verde_pg_init[] =
 {
-        0x0d4f, 0xffffffff, 0x40000,
-        0x0d4e, 0xffffffff, 0x200010ff,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x7007,
-        0x0d4e, 0xffffffff, 0x300010ff,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x400000,
-        0x0d4e, 0xffffffff, 0x100010ff,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x120200,
-        0x0d4e, 0xffffffff, 0x500010ff,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x1e1e16,
-        0x0d4e, 0xffffffff, 0x600010ff,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x171f1e,
-        0x0d4e, 0xffffffff, 0x700010ff,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4f, 0xffffffff, 0x0,
-        0x0d4e, 0xffffffff, 0x9ff,
-        0x0d40, 0xffffffff, 0x0,
-        0x0d41, 0xffffffff, 0x10000800,
-        0x0d41, 0xffffffff, 0xf,
-        0x0d41, 0xffffffff, 0xf,
-        0x0d40, 0xffffffff, 0x4,
-        0x0d41, 0xffffffff, 0x1000051e,
-        0x0d41, 0xffffffff, 0xffff,
-        0x0d41, 0xffffffff, 0xffff,
-        0x0d40, 0xffffffff, 0x8,
-        0x0d41, 0xffffffff, 0x80500,
-        0x0d40, 0xffffffff, 0x12,
-        0x0d41, 0xffffffff, 0x9050c,
-        0x0d40, 0xffffffff, 0x1d,
-        0x0d41, 0xffffffff, 0xb052c,
-        0x0d40, 0xffffffff, 0x2a,
-        0x0d41, 0xffffffff, 0x1053e,
-        0x0d40, 0xffffffff, 0x2d,
-        0x0d41, 0xffffffff, 0x10546,
-        0x0d40, 0xffffffff, 0x30,
-        0x0d41, 0xffffffff, 0xa054e,
-        0x0d40, 0xffffffff, 0x3c,
-        0x0d41, 0xffffffff, 0x1055f,
-        0x0d40, 0xffffffff, 0x3f,
-        0x0d41, 0xffffffff, 0x10567,
-        0x0d40, 0xffffffff, 0x42,
-        0x0d41, 0xffffffff, 0x1056f,
-        0x0d40, 0xffffffff, 0x45,
-        0x0d41, 0xffffffff, 0x10572,
-        0x0d40, 0xffffffff, 0x48,
-        0x0d41, 0xffffffff, 0x20575,
-        0x0d40, 0xffffffff, 0x4c,
-        0x0d41, 0xffffffff, 0x190801,
-        0x0d40, 0xffffffff, 0x67,
-        0x0d41, 0xffffffff, 0x1082a,
-        0x0d40, 0xffffffff, 0x6a,
-        0x0d41, 0xffffffff, 0x1b082d,
-        0x0d40, 0xffffffff, 0x87,
-        0x0d41, 0xffffffff, 0x310851,
-        0x0d40, 0xffffffff, 0xba,
-        0x0d41, 0xffffffff, 0x891,
-        0x0d40, 0xffffffff, 0xbc,
-        0x0d41, 0xffffffff, 0x893,
-        0x0d40, 0xffffffff, 0xbe,
-        0x0d41, 0xffffffff, 0x20895,
-        0x0d40, 0xffffffff, 0xc2,
-        0x0d41, 0xffffffff, 0x20899,
-        0x0d40, 0xffffffff, 0xc6,
-        0x0d41, 0xffffffff, 0x2089d,
-        0x0d40, 0xffffffff, 0xca,
-        0x0d41, 0xffffffff, 0x8a1,
-        0x0d40, 0xffffffff, 0xcc,
-        0x0d41, 0xffffffff, 0x8a3,
-        0x0d40, 0xffffffff, 0xce,
-        0x0d41, 0xffffffff, 0x308a5,
-        0x0d40, 0xffffffff, 0xd3,
-        0x0d41, 0xffffffff, 0x6d08cd,
-        0x0d40, 0xffffffff, 0x142,
-        0x0d41, 0xffffffff, 0x2000095a,
-        0x0d41, 0xffffffff, 0x1,
-        0x0d40, 0xffffffff, 0x144,
-        0x0d41, 0xffffffff, 0x301f095b,
-        0x0d40, 0xffffffff, 0x165,
-        0x0d41, 0xffffffff, 0xc094d,
-        0x0d40, 0xffffffff, 0x173,
-        0x0d41, 0xffffffff, 0xf096d,
-        0x0d40, 0xffffffff, 0x184,
-        0x0d41, 0xffffffff, 0x15097f,
-        0x0d40, 0xffffffff, 0x19b,
-        0x0d41, 0xffffffff, 0xc0998,
-        0x0d40, 0xffffffff, 0x1a9,
-        0x0d41, 0xffffffff, 0x409a7,
-        0x0d40, 0xffffffff, 0x1af,
-        0x0d41, 0xffffffff, 0xcdc,
-        0x0d40, 0xffffffff, 0x1b1,
-        0x0d41, 0xffffffff, 0x800,
-        0x0d42, 0xffffffff, 0x6c9b2000,
-        0x0d44, 0xfc00, 0x2000,
-        0x0d51, 0xffffffff, 0xfc0,
-        0x0a35, 0x00000100, 0x100
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
+        mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
+        mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
+        mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
+        mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
+        mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
+        mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
+        mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
+        mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
+        mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
+        mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
+        mmGMCON_MISC2, 0xfc00, 0x2000,
+        mmGMCON_MISC3, 0xffffffff, 0xfc0,
+        mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
 };
 
 static const u32 verde_golden_rlc_registers[] =
 {
-        0x263e, 0xffffffff, 0x02010002,
-        0x3109, 0xffffffff, 0x033f1005,
+        mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
+        mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
         0x311f, 0xffffffff, 0x10808020,
         0x3122, 0xffffffff, 0x00800008,
-        0x30c5, 0xffffffff, 0x00001000,
-        0x30c3, 0xffffffff, 0x80010014
+        mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
+        mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
 };
 
 static const u32 verde_golden_registers[] =
 {
-        0x17bc, 0x00000030, 0x00000011,
-        0x2684, 0x00010000, 0x00018208,
-        0x260c, 0xffffffff, 0x00000000,
-        0x260d, 0xf00fffff, 0x00000400,
-        0x260e, 0x0002021c, 0x00020200,
-        0x031e, 0x00000080, 0x00000000,
+        mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
+        mmCB_HW_CONTROL, 0x00010000, 0x00018208,
+        mmDB_DEBUG, 0xffffffff, 0x00000000,
+        mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+        mmDB_DEBUG3, 0x0002021c, 0x00020200,
+        mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
         0x340c, 0x000300c0, 0x00800040,
         0x360c, 0x000300c0, 0x00800040,
-        0x16ec, 0x000000f0, 0x00000070,
-        0x16f0, 0x00200000, 0x50100000,
-        0x1c0c, 0x31000311, 0x00000011,
-        0x0ab9, 0x00073ffe, 0x000022a2,
-        0x0903, 0x000007ff, 0x00000000,
-        0x2285, 0xf000001f, 0x00000007,
-        0x22c9, 0xffffffff, 0x00ffffff,
-        0x22c4, 0x0000ff0f, 0x00000000,
-        0xa293, 0x07ffffff, 0x4e000000,
-        0xa0d4, 0x3f3f3fff, 0x0000124a,
+        mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+        mmFBC_MISC, 0x00200000, 0x50100000,
+        mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
+        mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
+        mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
+        mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
+        mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
+        mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+        mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
+        mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
         0x000c, 0xffffffff, 0x0040,
         0x000d, 0x00000040, 0x00004040,
-        0x2440, 0x07ffffff, 0x03000000,
-        0x23a2, 0x01ff1f3f, 0x00000000,
-        0x23a1, 0x01ff1f3f, 0x00000000,
-        0x2418, 0x0000007f, 0x00000020,
-        0x2542, 0x00010000, 0x00010000,
-        0x2b05, 0x000003ff, 0x00000003,
-        0x2b04, 0xffffffff, 0x00000000,
-        0x2b03, 0xffffffff, 0x00001032,
-        0x2235, 0x0000001f, 0x00000010,
-        0x0570, 0x000c0fc0, 0x000c0400,
-        0x052c, 0x0fffffff, 0xffffffff,
-        0x052d, 0x0fffffff, 0x0fffffff,
-        0x052e, 0x0fffffff, 0x0fffffff,
-        0x052f, 0x0fffffff, 0x0fffffff
+        mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
+        mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
+        mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
+        mmSX_DEBUG_1, 0x0000007f, 0x00000020,
+        mmTA_CNTL_AUX, 0x00010000, 0x00010000,
+        mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
+        mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
+        mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
+        mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
+        mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
+        mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
+        mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 };
 
 static const u32 oland_golden_registers[] =
 {
-        0x17bc, 0x00000030, 0x00000011,
-        0x2684, 0x00010000, 0x00018208,
-        0x260c, 0xffffffff, 0x00000000,
-        0x260d, 0xf00fffff, 0x00000400,
-        0x260e, 0x0002021c, 0x00020200,
-        0x031e, 0x00000080, 0x00000000,
+        mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
+        mmCB_HW_CONTROL, 0x00010000, 0x00018208,
+        mmDB_DEBUG, 0xffffffff, 0x00000000,
+        mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+        mmDB_DEBUG3, 0x0002021c, 0x00020200,
+        mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
         0x340c, 0x000300c0, 0x00800040,
         0x360c, 0x000300c0, 0x00800040,
-        0x16ec, 0x000000f0, 0x00000070,
-        0x16f0, 0x00200000, 0x50100000,
-        0x1c0c, 0x31000311, 0x00000011,
-        0x0ab9, 0x00073ffe, 0x000022a2,
-        0x0903, 0x000007ff, 0x00000000,
-        0x2285, 0xf000001f, 0x00000007,
-        0x22c9, 0xffffffff, 0x00ffffff,
-        0x22c4, 0x0000ff0f, 0x00000000,
-        0xa293, 0x07ffffff, 0x4e000000,
-        0xa0d4, 0x3f3f3fff, 0x00000082,
+        mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+        mmFBC_MISC, 0x00200000, 0x50100000,
+        mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
+        mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
+        mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
+        mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
+        mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
+        mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+        mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
+        mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
         0x000c, 0xffffffff, 0x0040,
         0x000d, 0x00000040, 0x00004040,
-        0x2440, 0x07ffffff, 0x03000000,
-        0x2418, 0x0000007f, 0x00000020,
-        0x2542, 0x00010000, 0x00010000,
-        0x2b05, 0x000003ff, 0x000000f3,
-        0x2b04, 0xffffffff, 0x00000000,
-        0x2b03, 0xffffffff, 0x00003210,
-        0x2235, 0x0000001f, 0x00000010,
-        0x0570, 0x000c0fc0, 0x000c0400,
-        0x052c, 0x0fffffff, 0xffffffff,
-        0x052d, 0x0fffffff, 0x0fffffff,
-        0x052e, 0x0fffffff, 0x0fffffff,
-        0x052f, 0x0fffffff, 0x0fffffff
+        mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
+        mmSX_DEBUG_1, 0x0000007f, 0x00000020,
+        mmTA_CNTL_AUX, 0x00010000, 0x00010000,
+        mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
+        mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
+        mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
+        mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
+        mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
+        mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
+        mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+
 };
 
 static const u32 oland_golden_rlc_registers[] =
 {
-        0x263e, 0xffffffff, 0x02010002,
-        0x3109, 0xffffffff, 0x00601005,
+        mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
+        mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
         0x311f, 0xffffffff, 0x10104040,
         0x3122, 0xffffffff, 0x0100000a,
-        0x30c5, 0xffffffff, 0x00000800,
-        0x30c3, 0xffffffff, 0x800000f4
+        mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
+        mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 };
 
 static const u32 hainan_golden_registers[] =
 {
         0x17bc, 0x00000030, 0x00000011,
-        0x2684, 0x00010000, 0x00018208,
-        0x260c, 0xffffffff, 0x00000000,
-        0x260d, 0xf00fffff, 0x00000400,
-        0x260e, 0x0002021c, 0x00020200,
+        mmCB_HW_CONTROL, 0x00010000, 0x00018208,
+        mmDB_DEBUG, 0xffffffff, 0x00000000,
+        mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+        mmDB_DEBUG3, 0x0002021c, 0x00020200,
         0x031e, 0x00000080, 0x00000000,
         0x3430, 0xff000fff, 0x00000100,
         0x340c, 0x000300c0, 0x00800040,
@@ -379,63 +385,63 @@ static const u32 hainan_golden_registers[] =
         0x16ec, 0x000000f0, 0x00000070,
         0x16f0, 0x00200000, 0x50100000,
         0x1c0c, 0x31000311, 0x00000011,
-        0x0ab9, 0x00073ffe, 0x000022a2,
-        0x0903, 0x000007ff, 0x00000000,
-        0x2285, 0xf000001f, 0x00000007,
-        0x22c9, 0xffffffff, 0x00ffffff,
-        0x22c4, 0x0000ff0f, 0x00000000,
-        0xa293, 0x07ffffff, 0x4e000000,
-        0xa0d4, 0x3f3f3fff, 0x00000000,
+        mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
+        mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
+        mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
+        mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
+        mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+        mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
+        mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
         0x000c, 0xffffffff, 0x0040,
         0x000d, 0x00000040, 0x00004040,
-        0x2440, 0x03e00000, 0x03600000,
-        0x2418, 0x0000007f, 0x00000020,
-        0x2542, 0x00010000, 0x00010000,
-        0x2b05, 0x000003ff, 0x000000f1,
-        0x2b04, 0xffffffff, 0x00000000,
-        0x2b03, 0xffffffff, 0x00003210,
-        0x2235, 0x0000001f, 0x00000010,
-        0x0570, 0x000c0fc0, 0x000c0400,
-        0x052c, 0x0fffffff, 0xffffffff,
-        0x052d, 0x0fffffff, 0x0fffffff,
-        0x052e, 0x0fffffff, 0x0fffffff,
-        0x052f, 0x0fffffff, 0x0fffffff
+        mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
+        mmSX_DEBUG_1, 0x0000007f, 0x00000020,
+        mmTA_CNTL_AUX, 0x00010000, 0x00010000,
+        mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
+        mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
+        mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
+        mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
+        mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
+        mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
+        mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+        mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 };
 
 static const u32 hainan_golden_registers2[] =
 {
-        0x263e, 0xffffffff, 0x2011003
+        mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
 };
 
 static const u32 tahiti_mgcg_cgcg_init[] =
 {
-        0x3100, 0xffffffff, 0xfffffffc,
-        0x200b, 0xffffffff, 0xe0000000,
-        0x2698, 0xffffffff, 0x00000100,
-        0x24a9, 0xffffffff, 0x00000100,
-        0x3059, 0xffffffff, 0x00000100,
-        0x25dd, 0xffffffff, 0x00000100,
-        0x2261, 0xffffffff, 0x06000100,
-        0x2286, 0xffffffff, 0x00000100,
-        0x24a8, 0xffffffff, 0x00000100,
-        0x30e0, 0xffffffff, 0x00000100,
-        0x22ca, 0xffffffff, 0x00000100,
-        0x2451, 0xffffffff, 0x00000100,
-        0x2362, 0xffffffff, 0x00000100,
-        0x2363, 0xffffffff, 0x00000100,
-        0x240c, 0xffffffff, 0x00000100,
-        0x240d, 0xffffffff, 0x00000100,
-        0x240e, 0xffffffff, 0x00000100,
-        0x240f, 0xffffffff, 0x00000100,
-        0x2b60, 0xffffffff, 0x00000100,
-        0x2b15, 0xffffffff, 0x00000100,
-        0x225f, 0xffffffff, 0x06000100,
-        0x261a, 0xffffffff, 0x00000100,
-        0x2544, 0xffffffff, 0x00000100,
-        0x2bc1, 0xffffffff, 0x00000100,
-        0x2b81, 0xffffffff, 0x00000100,
-        0x2527, 0xffffffff, 0x00000100,
-        0x200b, 0xffffffff, 0xe0000000,
+        mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+        mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+        mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+        mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
         0x2458, 0xffffffff, 0x00010000,
         0x2459, 0xffffffff, 0x00030002,
         0x245a, 0xffffffff, 0x00040007,
@@ -516,55 +522,55 @@ static const u32 tahiti_mgcg_cgcg_init[] =
         0x24a5, 0xffffffff, 0x00000015,
         0x24a6, 0xffffffff, 0x00140013,
         0x24a7, 0xffffffff, 0x00170016,
-        0x2454, 0xffffffff, 0x96940200,
-        0x21c2, 0xffffffff, 0x00900100,
-        0x311e, 0xffffffff, 0x00000080,
-        0x3101, 0xffffffff, 0x0020003f,
+        mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
+        mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+        mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
+        mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
         0x000c, 0xffffffff, 0x0000001c,
         0x000d, 0x000f0000, 0x000f0000,
         0x0583, 0xffffffff, 0x00000100,
-        0x0409, 0xffffffff, 0x00000100,
-        0x040b, 0x00000101, 0x00000000,
-        0x082a, 0xffffffff, 0x00000104,
-        0x0993, 0x000c0000, 0x000c0000,
-        0x0992, 0x000c0000, 0x000c0000,
-        0x1579, 0xff000fff, 0x00000100,
+        mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+        mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+        mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+        mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
+        mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
+        mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
         0x157a, 0x00000001, 0x00000001,
-        0x0bd4, 0x00000001, 0x00000001,
-        0x0c33, 0xc0000fff, 0x00000104,
-        0x3079, 0x00000001, 0x00000001,
+        mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
+        mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+        mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
         0x3430, 0xfffffff0, 0x00000100,
-        0x3630, 0xfffffff0, 0x00000100
+        0x3630, 0xfffffff0, 0x00000100,
 };
 static const u32 pitcairn_mgcg_cgcg_init[] =
 {
-        0x3100, 0xffffffff, 0xfffffffc,
-        0x200b, 0xffffffff, 0xe0000000,
-        0x2698, 0xffffffff, 0x00000100,
-        0x24a9, 0xffffffff, 0x00000100,
-        0x3059, 0xffffffff, 0x00000100,
-        0x25dd, 0xffffffff, 0x00000100,
-        0x2261, 0xffffffff, 0x06000100,
-        0x2286, 0xffffffff, 0x00000100,
-        0x24a8, 0xffffffff, 0x00000100,
-        0x30e0, 0xffffffff, 0x00000100,
-        0x22ca, 0xffffffff, 0x00000100,
-        0x2451, 0xffffffff, 0x00000100,
-        0x2362, 0xffffffff, 0x00000100,
-        0x2363, 0xffffffff, 0x00000100,
-        0x240c, 0xffffffff, 0x00000100,
-        0x240d, 0xffffffff, 0x00000100,
-        0x240e, 0xffffffff, 0x00000100,
-        0x240f, 0xffffffff, 0x00000100,
-        0x2b60, 0xffffffff, 0x00000100,
-        0x2b15, 0xffffffff, 0x00000100,
-        0x225f, 0xffffffff, 0x06000100,
-        0x261a, 0xffffffff, 0x00000100,
-        0x2544, 0xffffffff, 0x00000100,
-        0x2bc1, 0xffffffff, 0x00000100,
-        0x2b81, 0xffffffff, 0x00000100,
-        0x2527, 0xffffffff, 0x00000100,
-        0x200b, 0xffffffff, 0xe0000000,
+        mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+        mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+        mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+        mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
         0x2458, 0xffffffff, 0x00010000,
         0x2459, 0xffffffff, 0x00030002,
         0x245a, 0xffffffff, 0x00040007,
@@ -615,53 +621,54 @@ static const u32 pitcairn_mgcg_cgcg_init[] =
         0x2496, 0xffffffff, 0x00100013,
         0x2497, 0xffffffff, 0x00120011,
         0x2498, 0xffffffff, 0x00150014,
-        0x2454, 0xffffffff, 0x96940200,
-        0x21c2, 0xffffffff, 0x00900100,
-        0x311e, 0xffffffff, 0x00000080,
-        0x3101, 0xffffffff, 0x0020003f,
+        mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
+        mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+        mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
+        mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
         0x000c, 0xffffffff, 0x0000001c,
         0x000d, 0x000f0000, 0x000f0000,
         0x0583, 0xffffffff, 0x00000100,
-        0x0409, 0xffffffff, 0x00000100,
-        0x040b, 0x00000101, 0x00000000,
-        0x082a, 0xffffffff, 0x00000104,
-        0x1579, 0xff000fff, 0x00000100,
+        mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+        mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+        mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+        mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
         0x157a, 0x00000001, 0x00000001,
-        0x0bd4, 0x00000001, 0x00000001,
-        0x0c33, 0xc0000fff, 0x00000104,
-        0x3079, 0x00000001, 0x00000001,
+        mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
+        mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+        mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
         0x3430, 0xfffffff0, 0x00000100,
-        0x3630, 0xfffffff0, 0x00000100
+        0x3630, 0xfffffff0, 0x00000100,
 };
+
 static const u32 verde_mgcg_cgcg_init[] =
 {
-        0x3100, 0xffffffff, 0xfffffffc,
-        0x200b, 0xffffffff, 0xe0000000,
-        0x2698, 0xffffffff, 0x00000100,
-        0x24a9, 0xffffffff, 0x00000100,
-        0x3059, 0xffffffff, 0x00000100,
-        0x25dd, 0xffffffff, 0x00000100,
-        0x2261, 0xffffffff, 0x06000100,
-        0x2286, 0xffffffff, 0x00000100,
-        0x24a8, 0xffffffff, 0x00000100,
-        0x30e0, 0xffffffff, 0x00000100,
-        0x22ca, 0xffffffff, 0x00000100,
-        0x2451, 0xffffffff, 0x00000100,
-        0x2362, 0xffffffff, 0x00000100,
-        0x2363, 0xffffffff, 0x00000100,
-        0x240c, 0xffffffff, 0x00000100,
-        0x240d, 0xffffffff, 0x00000100,
-        0x240e, 0xffffffff, 0x00000100,
-        0x240f, 0xffffffff, 0x00000100,
-        0x2b60, 0xffffffff, 0x00000100,
-        0x2b15, 0xffffffff, 0x00000100,
-        0x225f, 0xffffffff, 0x06000100,
-        0x261a, 0xffffffff, 0x00000100,
-        0x2544, 0xffffffff, 0x00000100,
-        0x2bc1, 0xffffffff, 0x00000100,
-        0x2b81, 0xffffffff, 0x00000100,
-        0x2527, 0xffffffff, 0x00000100,
-        0x200b, 0xffffffff, 0xe0000000,
+        mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+        mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+        mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+        mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
         0x2458, 0xffffffff, 0x00010000,
         0x2459, 0xffffffff, 0x00030002,
         0x245a, 0xffffffff, 0x00040007,
@@ -712,55 +719,56 @@ static const u32 verde_mgcg_cgcg_init[] =
         0x2496, 0xffffffff, 0x00100013,
         0x2497, 0xffffffff, 0x00120011,
         0x2498, 0xffffffff, 0x00150014,
-        0x2454, 0xffffffff, 0x96940200,
-        0x21c2, 0xffffffff, 0x00900100,
-        0x311e, 0xffffffff, 0x00000080,
-        0x3101, 0xffffffff, 0x0020003f,
+        mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
+        mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+        mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
+        mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
         0x000c, 0xffffffff, 0x0000001c,
         0x000d, 0x000f0000, 0x000f0000,
         0x0583, 0xffffffff, 0x00000100,
-        0x0409, 0xffffffff, 0x00000100,
-        0x040b, 0x00000101, 0x00000000,
-        0x082a, 0xffffffff, 0x00000104,
-        0x0993, 0x000c0000, 0x000c0000,
-        0x0992, 0x000c0000, 0x000c0000,
-        0x1579, 0xff000fff, 0x00000100,
+        mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+        mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+        mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+        mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
+        mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
+        mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
         0x157a, 0x00000001, 0x00000001,
-        0x0bd4, 0x00000001, 0x00000001,
-        0x0c33, 0xc0000fff, 0x00000104,
-        0x3079, 0x00000001, 0x00000001,
+        mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
+        mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+        mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
         0x3430, 0xfffffff0, 0x00000100,
-        0x3630, 0xfffffff0, 0x00000100
+        0x3630, 0xfffffff0, 0x00000100,
 };
+
 static const u32 oland_mgcg_cgcg_init[] =
 {
-        0x3100, 0xffffffff, 0xfffffffc,
-        0x200b, 0xffffffff, 0xe0000000,
-        0x2698, 0xffffffff, 0x00000100,
-        0x24a9, 0xffffffff, 0x00000100,
-        0x3059, 0xffffffff, 0x00000100,
-        0x25dd, 0xffffffff, 0x00000100,
-        0x2261, 0xffffffff, 0x06000100,
-        0x2286, 0xffffffff, 0x00000100,
-        0x24a8, 0xffffffff, 0x00000100,
-        0x30e0, 0xffffffff, 0x00000100,
-        0x22ca, 0xffffffff, 0x00000100,
-        0x2451, 0xffffffff, 0x00000100,
-        0x2362, 0xffffffff, 0x00000100,
-        0x2363, 0xffffffff, 0x00000100,
-        0x240c, 0xffffffff, 0x00000100,
-        0x240d, 0xffffffff, 0x00000100,
-        0x240e, 0xffffffff, 0x00000100,
-        0x240f, 0xffffffff, 0x00000100,
-        0x2b60, 0xffffffff, 0x00000100,
-        0x2b15, 0xffffffff, 0x00000100,
-        0x225f, 0xffffffff, 0x06000100,
-        0x261a, 0xffffffff, 0x00000100,
-        0x2544, 0xffffffff, 0x00000100,
-        0x2bc1, 0xffffffff, 0x00000100,
-        0x2b81, 0xffffffff, 0x00000100,
-        0x2527, 0xffffffff, 0x00000100,
-        0x200b, 0xffffffff, 0xe0000000,
+        mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+        mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+        mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+        mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
         0x2458, 0xffffffff, 0x00010000,
         0x2459, 0xffffffff, 0x00030002,
         0x245a, 0xffffffff, 0x00040007,
@@ -791,55 +799,56 @@ static const u32 oland_mgcg_cgcg_init[] =
         0x2473, 0xffffffff, 0x0000000b,
         0x2474, 0xffffffff, 0x000a0009,
         0x2475, 0xffffffff, 0x000d000c,
-        0x2454, 0xffffffff, 0x96940200,
-        0x21c2, 0xffffffff, 0x00900100,
-        0x311e, 0xffffffff, 0x00000080,
-        0x3101, 0xffffffff, 0x0020003f,
+        mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
+        mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+        mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
+        mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
         0x000c, 0xffffffff, 0x0000001c,
         0x000d, 0x000f0000, 0x000f0000,
         0x0583, 0xffffffff, 0x00000100,
-        0x0409, 0xffffffff, 0x00000100,
-        0x040b, 0x00000101, 0x00000000,
-        0x082a, 0xffffffff, 0x00000104,
-        0x0993, 0x000c0000, 0x000c0000,
-        0x0992, 0x000c0000, 0x000c0000,
-        0x1579, 0xff000fff, 0x00000100,
+        mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+        mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+        mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+        mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
+        mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
+        mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
         0x157a, 0x00000001, 0x00000001,
-        0x0bd4, 0x00000001, 0x00000001,
-        0x0c33, 0xc0000fff, 0x00000104,
-        0x3079, 0x00000001, 0x00000001,
+        mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
+        mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+        mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
         0x3430, 0xfffffff0, 0x00000100,
-        0x3630, 0xfffffff0, 0x00000100
+        0x3630, 0xfffffff0, 0x00000100,
 };
+
 static const u32 hainan_mgcg_cgcg_init[] =
 {
-        0x3100, 0xffffffff, 0xfffffffc,
-        0x200b, 0xffffffff, 0xe0000000,
-        0x2698, 0xffffffff, 0x00000100,
-        0x24a9, 0xffffffff, 0x00000100,
-        0x3059, 0xffffffff, 0x00000100,
-        0x25dd, 0xffffffff, 0x00000100,
-        0x2261, 0xffffffff, 0x06000100,
-        0x2286, 0xffffffff, 0x00000100,
-        0x24a8, 0xffffffff, 0x00000100,
-        0x30e0, 0xffffffff, 0x00000100,
-        0x22ca, 0xffffffff, 0x00000100,
-        0x2451, 0xffffffff, 0x00000100,
-        0x2362, 0xffffffff, 0x00000100,
-        0x2363, 0xffffffff, 0x00000100,
-        0x240c, 0xffffffff, 0x00000100,
-        0x240d, 0xffffffff, 0x00000100,
-        0x240e, 0xffffffff, 0x00000100,
-        0x240f, 0xffffffff, 0x00000100,
-        0x2b60, 0xffffffff, 0x00000100,
-        0x2b15, 0xffffffff, 0x00000100,
-        0x225f, 0xffffffff, 0x06000100,
-        0x261a, 0xffffffff, 0x00000100,
-        0x2544, 0xffffffff, 0x00000100,
-        0x2bc1, 0xffffffff, 0x00000100,
-        0x2b81, 0xffffffff, 0x00000100,
-        0x2527, 0xffffffff, 0x00000100,
-        0x200b, 0xffffffff, 0xe0000000,
+        mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+        mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+        mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+        mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+        mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+        mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+        mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+        mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+        mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
         0x2458, 0xffffffff, 0x00010000,
         0x2459, 0xffffffff, 0x00030002,
         0x245a, 0xffffffff, 0x00040007,
@@ -870,22 +879,22 @@ static const u32 hainan_mgcg_cgcg_init[] =
         0x2473, 0xffffffff, 0x0000000b,
         0x2474, 0xffffffff, 0x000a0009,
         0x2475, 0xffffffff, 0x000d000c,
-        0x2454, 0xffffffff, 0x96940200,
-        0x21c2, 0xffffffff, 0x00900100,
-        0x311e, 0xffffffff, 0x00000080,
-        0x3101, 0xffffffff, 0x0020003f,
+        mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
+        mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+        mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
+        mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
         0x000c, 0xffffffff, 0x0000001c,
         0x000d, 0x000f0000, 0x000f0000,
         0x0583, 0xffffffff, 0x00000100,
         0x0409, 0xffffffff, 0x00000100,
-        0x082a, 0xffffffff, 0x00000104,
-        0x0993, 0x000c0000, 0x000c0000,
-        0x0992, 0x000c0000, 0x000c0000,
-        0x0bd4, 0x00000001, 0x00000001,
-        0x0c33, 0xc0000fff, 0x00000104,
-        0x3079, 0x00000001, 0x00000001,
+        mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+        mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
+        mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
+        mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
+        mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+        mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
         0x3430, 0xfffffff0, 0x00000100,
-        0x3630, 0xfffffff0, 0x00000100
+        0x3630, 0xfffffff0, 0x00000100,
 };
 
 static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
@@ -1129,13 +1138,12 @@ static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
 static void si_detect_hw_virtualization(struct amdgpu_device *adev)
 {
         if (is_virtual_machine()) /* passthrough mode */
-                adev->virtualization.virtual_caps |= AMDGPU_PASSTHROUGH_MODE;
+                adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
 }
 
 static const struct amdgpu_asic_funcs si_asic_funcs =
 {
         .read_disabled_bios = &si_read_disabled_bios,
-        .detect_hw_virtualization = si_detect_hw_virtualization,
         .read_register = &si_read_register,
         .reset = &si_asic_reset,
         .set_vga_state = &si_vga_set_state,
@@ -1852,6 +1860,8 @@ static const struct amdgpu_ip_block_version si_common_ip_block =
 
 int si_set_ip_blocks(struct amdgpu_device *adev)
 {
+        si_detect_hw_virtualization(adev);
+
         switch (adev->asic_type) {
         case CHIP_VERDE:
         case CHIP_TAHITI:
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 3dd552ae0b59..3372a071bb85 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -24,7 +24,7 @@
 #include <drm/drmP.h>
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
-#include "si/sid.h"
+#include "sid.h"
 
 const u32 sdma_offsets[SDMA_MAX_INSTANCE] =
 {
@@ -301,7 +301,7 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
         ib.ptr[2] = upper_32_bits(gpu_addr) & 0xff;
         ib.ptr[3] = 0xDEADBEEF;
         ib.length_dw = 4;
-        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, NULL, &f);
+        r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
         if (r)
                 goto err1;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index 6e150db8f380..f55e45b52fbc 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -26,7 +26,7 @@
 #include "amdgpu_pm.h"
 #include "amdgpu_dpm.h"
 #include "amdgpu_atombios.h"
-#include "si/sid.h"
+#include "sid.h"
 #include "r600_dpm.h"
 #include "si_dpm.h"
 #include "atom.h"
@@ -3009,29 +3009,6 @@ static int si_init_smc_spll_table(struct amdgpu_device *adev)
         return ret;
 }
 
-struct si_dpm_quirk {
-        u32 chip_vendor;
-        u32 chip_device;
-        u32 subsys_vendor;
-        u32 subsys_device;
-        u32 max_sclk;
-        u32 max_mclk;
-};
-
-/* cards with dpm stability problems */
-static struct si_dpm_quirk si_dpm_quirk_list[] = {
-        /* PITCAIRN - https://bugs.freedesktop.org/show_bug.cgi?id=76490 */
-        { PCI_VENDOR_ID_ATI, 0x6810, 0x1462, 0x3036, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6811, 0x174b, 0xe271, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6811, 0x174b, 0x2015, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6810, 0x174b, 0xe271, 85000, 90000 },
-        { PCI_VENDOR_ID_ATI, 0x6811, 0x1462, 0x2015, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6811, 0x1043, 0x2015, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6811, 0x148c, 0x2015, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6810, 0x1682, 0x9275, 0, 120000 },
-        { 0, 0, 0, 0 },
-};
-
 static u16 si_get_lower_of_leakage_and_vce_voltage(struct amdgpu_device *adev,
                                                    u16 vce_voltage)
 {
@@ -3477,18 +3454,8 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev,
         u32 max_sclk_vddc, max_mclk_vddci, max_mclk_vddc;
         u32 max_sclk = 0, max_mclk = 0;
         int i;
-        struct si_dpm_quirk *p = si_dpm_quirk_list;
 
-        /* limit all SI kickers */
-        if (adev->asic_type == CHIP_PITCAIRN) {
-                if ((adev->pdev->revision == 0x81) ||
-                    (adev->pdev->device == 0x6810) ||
-                    (adev->pdev->device == 0x6811) ||
-                    (adev->pdev->device == 0x6816) ||
-                    (adev->pdev->device == 0x6817) ||
-                    (adev->pdev->device == 0x6806))
-                        max_mclk = 120000;
-        } else if (adev->asic_type == CHIP_HAINAN) {
+        if (adev->asic_type == CHIP_HAINAN) {
                 if ((adev->pdev->revision == 0x81) ||
                     (adev->pdev->revision == 0x83) ||
                     (adev->pdev->revision == 0xC3) ||
@@ -3498,18 +3465,6 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev,
                         max_sclk = 75000;
                 }
         }
-        /* Apply dpm quirks */
-        while (p && p->chip_device != 0) {
-                if (adev->pdev->vendor == p->chip_vendor &&
-                    adev->pdev->device == p->chip_device &&
-                    adev->pdev->subsystem_vendor == p->subsys_vendor &&
-                    adev->pdev->subsystem_device == p->subsys_device) {
-                        max_sclk = p->max_sclk;
-                        max_mclk = p->max_mclk;
-                        break;
-                }
-                ++p;
-        }
 
         if (rps->vce_active) {
                 rps->evclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].evclk;
@@ -3906,25 +3861,25 @@ static int si_restrict_performance_levels_before_switch(struct amdgpu_device *ad
 }
 
 static int si_dpm_force_performance_level(struct amdgpu_device *adev,
-                                   enum amdgpu_dpm_forced_level level)
+                                   enum amd_dpm_forced_level level)
 {
         struct amdgpu_ps *rps = adev->pm.dpm.current_ps;
         struct  si_ps *ps = si_get_ps(rps);
         u32 levels = ps->performance_level_count;
 
-        if (level == AMDGPU_DPM_FORCED_LEVEL_HIGH) {
+        if (level == AMD_DPM_FORCED_LEVEL_HIGH) {
                 if (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetEnabledLevels, levels) != PPSMC_Result_OK)
                         return -EINVAL;
 
                 if (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
                         return -EINVAL;
-        } else if (level == AMDGPU_DPM_FORCED_LEVEL_LOW) {
+        } else if (level == AMD_DPM_FORCED_LEVEL_LOW) {
                 if (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
                         return -EINVAL;
 
                 if (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
                         return -EINVAL;
-        } else if (level == AMDGPU_DPM_FORCED_LEVEL_AUTO) {
+        } else if (level == AMD_DPM_FORCED_LEVEL_AUTO) {
                 if (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
                         return -EINVAL;
 
@@ -7746,7 +7701,7 @@ static int si_dpm_sw_init(void *handle)
         /* default to balanced state */
         adev->pm.dpm.state = POWER_STATE_TYPE_BALANCED;
         adev->pm.dpm.user_state = POWER_STATE_TYPE_BALANCED;
-        adev->pm.dpm.forced_level = AMDGPU_DPM_FORCED_LEVEL_AUTO;
+        adev->pm.dpm.forced_level = AMD_DPM_FORCED_LEVEL_AUTO;
         adev->pm.default_sclk = adev->clock.default_sclk;
         adev->pm.default_mclk = adev->clock.default_mclk;
         adev->pm.current_sclk = adev->clock.default_sclk;
@@ -8072,11 +8027,3 @@ static void si_dpm_set_irq_funcs(struct amdgpu_device *adev)
         adev->pm.dpm.thermal.irq.funcs = &si_dpm_irq_funcs;
 }
 
-const struct amdgpu_ip_block_version si_dpm_ip_block =
-{
-        .type = AMD_IP_BLOCK_TYPE_SMC,
-        .major = 6,
-        .minor = 0,
-        .rev = 0,
-        .funcs = &si_dpm_ip_funcs,
-};
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c
index db0f36846661..81f90800ba73 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c
@@ -23,7 +23,7 @@
 #include "drmP.h"
 #include "amdgpu.h"
 #include "amdgpu_ih.h"
-#include "si/sid.h"
+#include "sid.h"
 #include "si_ih.h"
 
 static void si_ih_set_interrupt_funcs(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/si_smc.c b/drivers/gpu/drm/amd/amdgpu/si_smc.c
index 668ba99d6c05..0726bc3b6f90 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_smc.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_smc.c
@@ -25,7 +25,7 @@
 #include <linux/firmware.h>
 #include "drmP.h"
 #include "amdgpu.h"
-#include "si/sid.h"
+#include "sid.h"
 #include "ppsmc.h"
 #include "amdgpu_ucode.h"
 #include "sislands_smc.h"
diff --git a/drivers/gpu/drm/amd/include/asic_reg/si/sid.h b/drivers/gpu/drm/amd/amdgpu/sid.h
index c57eff159374..c57eff159374 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/si/sid.h
+++ b/drivers/gpu/drm/amd/amdgpu/sid.h
diff --git a/drivers/gpu/drm/amd/amdgpu/smu_ucode_xfer_vi.h b/drivers/gpu/drm/amd/amdgpu/smu_ucode_xfer_vi.h
deleted file mode 100644
index 880152c0f775..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/smu_ucode_xfer_vi.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef SMU_UCODE_XFER_VI_H
-#define SMU_UCODE_XFER_VI_H
-
-#define SMU_DRAMData_TOC_VERSION  1
-#define MAX_IH_REGISTER_COUNT     65535
-#define SMU_DIGEST_SIZE_BYTES     20
-#define SMU_FB_SIZE_BYTES         1048576
-#define SMU_MAX_ENTRIES           12
-
-#define UCODE_ID_SMU              0
-#define UCODE_ID_SDMA0            1
-#define UCODE_ID_SDMA1            2
-#define UCODE_ID_CP_CE            3
-#define UCODE_ID_CP_PFP           4
-#define UCODE_ID_CP_ME            5
-#define UCODE_ID_CP_MEC           6
-#define UCODE_ID_CP_MEC_JT1       7
-#define UCODE_ID_CP_MEC_JT2       8
-#define UCODE_ID_GMCON_RENG       9
-#define UCODE_ID_RLC_G            10
-#define UCODE_ID_IH_REG_RESTORE   11
-#define UCODE_ID_VBIOS            12
-#define UCODE_ID_MISC_METADATA    13
-#define UCODE_ID_SMU_SK               14
-#define UCODE_ID_RLC_SCRATCH      32
-#define UCODE_ID_RLC_SRM_ARAM     33
-#define UCODE_ID_RLC_SRM_DRAM     34
-#define UCODE_ID_MEC_STORAGE      35
-#define UCODE_ID_VBIOS_PARAMETERS 36
-#define UCODE_META_DATA           0xFF
-
-#define UCODE_ID_SMU_MASK             0x00000001
-#define UCODE_ID_SDMA0_MASK           0x00000002
-#define UCODE_ID_SDMA1_MASK           0x00000004
-#define UCODE_ID_CP_CE_MASK           0x00000008
-#define UCODE_ID_CP_PFP_MASK          0x00000010
-#define UCODE_ID_CP_ME_MASK           0x00000020
-#define UCODE_ID_CP_MEC_MASK          0x00000040
-#define UCODE_ID_CP_MEC_JT1_MASK      0x00000080
-#define UCODE_ID_CP_MEC_JT2_MASK      0x00000100
-#define UCODE_ID_GMCON_RENG_MASK      0x00000200
-#define UCODE_ID_RLC_G_MASK           0x00000400
-#define UCODE_ID_IH_REG_RESTORE_MASK  0x00000800
-#define UCODE_ID_VBIOS_MASK           0x00001000
-
-#define UCODE_FLAG_UNHALT_MASK   0x1
-
-struct SMU_Entry {
-#ifndef __BIG_ENDIAN
-        uint16_t id;
-        uint16_t version;
-        uint32_t image_addr_high;
-        uint32_t image_addr_low;
-        uint32_t meta_data_addr_high;
-        uint32_t meta_data_addr_low;
-        uint32_t data_size_byte;
-        uint16_t flags;
-        uint16_t num_register_entries;
-#else
-        uint16_t version;
-        uint16_t id;
-        uint32_t image_addr_high;
-        uint32_t image_addr_low;
-        uint32_t meta_data_addr_high;
-        uint32_t meta_data_addr_low;
-        uint32_t data_size_byte;
-        uint16_t num_register_entries;
-        uint16_t flags;
-#endif
-};
-
-struct SMU_DRAMData_TOC {
-        uint32_t structure_version;
-        uint32_t num_entries;
-        struct SMU_Entry entry[SMU_MAX_ENTRIES];
-};
-
-#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index 6de6becce745..9b49824233ae 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -822,16 +822,44 @@ static int uvd_v5_0_set_powergating_state(void *handle,
          * the smc and the hw blocks
          */
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int ret = 0;
 
         if (!(adev->pg_flags & AMD_PG_SUPPORT_UVD))
                 return 0;
 
         if (state == AMD_PG_STATE_GATE) {
                 uvd_v5_0_stop(adev);
-                return 0;
+                adev->uvd.is_powergated = true;
         } else {
-                return uvd_v5_0_start(adev);
+                ret = uvd_v5_0_start(adev);
+                if (ret)
+                        goto out;
+                adev->uvd.is_powergated = false;
+        }
+
+out:
+        return ret;
+}
+
+static void uvd_v5_0_get_clockgating_state(void *handle, u32 *flags)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int data;
+
+        mutex_lock(&adev->pm.mutex);
+
+        if (adev->uvd.is_powergated) {
+                DRM_INFO("Cannot get clockgating state when UVD is powergated.\n");
+                goto out;
         }
+
+        /* AMD_CG_SUPPORT_UVD_MGCG */
+        data = RREG32(mmUVD_CGC_CTRL);
+        if (data & UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK)
+                *flags |= AMD_CG_SUPPORT_UVD_MGCG;
+
+out:
+        mutex_unlock(&adev->pm.mutex);
 }
 
 static const struct amd_ip_funcs uvd_v5_0_ip_funcs = {
@@ -849,6 +877,7 @@ static const struct amd_ip_funcs uvd_v5_0_ip_funcs = {
         .soft_reset = uvd_v5_0_soft_reset,
         .set_clockgating_state = uvd_v5_0_set_clockgating_state,
         .set_powergating_state = uvd_v5_0_set_powergating_state,
+        .get_clockgating_state = uvd_v5_0_get_clockgating_state,
 };
 
 static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index ba0bbf7138dc..de7e03544d00 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -1047,6 +1047,7 @@ static int uvd_v6_0_set_powergating_state(void *handle,
          * the smc and the hw blocks
          */
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int ret = 0;
 
         if (!(adev->pg_flags & AMD_PG_SUPPORT_UVD))
                 return 0;
@@ -1055,10 +1056,37 @@ static int uvd_v6_0_set_powergating_state(void *handle,
 
         if (state == AMD_PG_STATE_GATE) {
                 uvd_v6_0_stop(adev);
-                return 0;
+                adev->uvd.is_powergated = true;
         } else {
-                return uvd_v6_0_start(adev);
+                ret = uvd_v6_0_start(adev);
+                if (ret)
+                        goto out;
+                adev->uvd.is_powergated = false;
+        }
+
+out:
+        return ret;
+}
+
+static void uvd_v6_0_get_clockgating_state(void *handle, u32 *flags)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int data;
+
+        mutex_lock(&adev->pm.mutex);
+
+        if (adev->uvd.is_powergated) {
+                DRM_INFO("Cannot get clockgating state when UVD is powergated.\n");
+                goto out;
         }
+
+        /* AMD_CG_SUPPORT_UVD_MGCG */
+        data = RREG32(mmUVD_CGC_CTRL);
+        if (data & UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK)
+                *flags |= AMD_CG_SUPPORT_UVD_MGCG;
+
+out:
+        mutex_unlock(&adev->pm.mutex);
 }
 
 static const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
@@ -1079,6 +1107,7 @@ static const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
         .post_soft_reset = uvd_v6_0_post_soft_reset,
         .set_clockgating_state = uvd_v6_0_set_clockgating_state,
         .set_powergating_state = uvd_v6_0_set_powergating_state,
+        .get_clockgating_state = uvd_v6_0_get_clockgating_state,
 };
 
 static const struct amdgpu_ring_funcs uvd_v6_0_ring_phys_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index 37ca685e5a9a..8db26559fd1b 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -230,6 +230,10 @@ static int vce_v3_0_start(struct amdgpu_device *adev)
         struct amdgpu_ring *ring;
         int idx, r;
 
+        vce_v3_0_override_vce_clock_gating(adev, true);
+        if (!(adev->flags & AMD_IS_APU))
+                amdgpu_asic_set_vce_clocks(adev, 10000, 10000);
+
         ring = &adev->vce.ring[0];
         WREG32(mmVCE_RB_RPTR, ring->wptr);
         WREG32(mmVCE_RB_WPTR, ring->wptr);
@@ -708,18 +712,6 @@ static int vce_v3_0_process_interrupt(struct amdgpu_device *adev,
         return 0;
 }
 
-static void vce_v3_0_set_bypass_mode(struct amdgpu_device *adev, bool enable)
-{
-        u32 tmp = RREG32_SMC(ixGCK_DFS_BYPASS_CNTL);
-
-        if (enable)
-                tmp |= GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;
-        else
-                tmp &= ~GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;
-
-        WREG32_SMC(ixGCK_DFS_BYPASS_CNTL, tmp);
-}
-
 static int vce_v3_0_set_clockgating_state(void *handle,
                                           enum amd_clockgating_state state)
 {
@@ -727,11 +719,6 @@ static int vce_v3_0_set_clockgating_state(void *handle,
         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
         int i;
 
-        if ((adev->asic_type == CHIP_POLARIS10) ||
-                (adev->asic_type == CHIP_TONGA) ||
-                (adev->asic_type == CHIP_FIJI))
-                vce_v3_0_set_bypass_mode(adev, enable);
-
         if (!(adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG))
                 return 0;
 
@@ -777,15 +764,46 @@ static int vce_v3_0_set_powergating_state(void *handle,
          * the smc and the hw blocks
          */
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int ret = 0;
 
         if (!(adev->pg_flags & AMD_PG_SUPPORT_VCE))
                 return 0;
 
-        if (state == AMD_PG_STATE_GATE)
+        if (state == AMD_PG_STATE_GATE) {
+                adev->vce.is_powergated = true;
                 /* XXX do we need a vce_v3_0_stop()? */
-                return 0;
-        else
-                return vce_v3_0_start(adev);
+        } else {
+                ret = vce_v3_0_start(adev);
+                if (ret)
+                        goto out;
+                adev->vce.is_powergated = false;
+        }
+
+out:
+        return ret;
+}
+
+static void vce_v3_0_get_clockgating_state(void *handle, u32 *flags)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int data;
+
+        mutex_lock(&adev->pm.mutex);
+
+        if (adev->vce.is_powergated) {
+                DRM_INFO("Cannot get clockgating state when VCE is powergated.\n");
+                goto out;
+        }
+
+        WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, 0);
+
+        /* AMD_CG_SUPPORT_VCE_MGCG */
+        data = RREG32(mmVCE_CLOCK_GATING_A);
+        if (data & (0x04 << 4))
+                *flags |= AMD_CG_SUPPORT_VCE_MGCG;
+
+out:
+        mutex_unlock(&adev->pm.mutex);
 }
 
 static void vce_v3_0_ring_emit_ib(struct amdgpu_ring *ring,
@@ -839,6 +857,7 @@ static const struct amd_ip_funcs vce_v3_0_ip_funcs = {
         .post_soft_reset = vce_v3_0_post_soft_reset,
         .set_clockgating_state = vce_v3_0_set_clockgating_state,
         .set_powergating_state = vce_v3_0_set_powergating_state,
+        .get_clockgating_state = vce_v3_0_get_clockgating_state,
 };
 
 static const struct amdgpu_ring_funcs vce_v3_0_ring_phys_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index c2ac54f11341..4922fff08c3c 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -20,9 +20,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
-#include <linux/firmware.h>
 #include <linux/slab.h>
-#include <linux/module.h>
 #include "drmP.h"
 #include "amdgpu.h"
 #include "amdgpu_atombios.h"
@@ -78,17 +76,7 @@
 #include "amdgpu_acp.h"
 #endif
 #include "dce_virtual.h"
-
-MODULE_FIRMWARE("amdgpu/topaz_smc.bin");
-MODULE_FIRMWARE("amdgpu/topaz_k_smc.bin");
-MODULE_FIRMWARE("amdgpu/tonga_smc.bin");
-MODULE_FIRMWARE("amdgpu/tonga_k_smc.bin");
-MODULE_FIRMWARE("amdgpu/fiji_smc.bin");
-MODULE_FIRMWARE("amdgpu/polaris10_smc.bin");
-MODULE_FIRMWARE("amdgpu/polaris10_smc_sk.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_smc.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_smc_sk.bin");
-MODULE_FIRMWARE("amdgpu/polaris12_smc.bin");
+#include "mxgpu_vi.h"
 
 /*
  * Indirect registers accessor
@@ -285,6 +273,12 @@ static void vi_init_golden_registers(struct amdgpu_device *adev)
         /* Some of the registers might be dependent on GRBM_GFX_INDEX */
         mutex_lock(&adev->grbm_idx_mutex);
 
+        if (amdgpu_sriov_vf(adev)) {
+                xgpu_vi_init_golden_registers(adev);
+                mutex_unlock(&adev->grbm_idx_mutex);
+                return;
+        }
+
         switch (adev->asic_type) {
         case CHIP_TOPAZ:
                 amdgpu_program_register_sequence(adev,
@@ -458,14 +452,14 @@ static void vi_detect_hw_virtualization(struct amdgpu_device *adev)
         /* bit0: 0 means pf and 1 means vf */
         /* bit31: 0 means disable IOV and 1 means enable */
         if (reg & 1)
-                adev->virtualization.virtual_caps |= AMDGPU_SRIOV_CAPS_IS_VF;
+                adev->virt.caps |= AMDGPU_SRIOV_CAPS_IS_VF;
 
         if (reg & 0x80000000)
-                adev->virtualization.virtual_caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV;
+                adev->virt.caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV;
 
         if (reg == 0) {
                 if (is_virtual_machine()) /* passthrough mode exclus sr-iov mode */
-                        adev->virtualization.virtual_caps |= AMDGPU_PASSTHROUGH_MODE;
+                        adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
         }
 }
 
@@ -801,7 +795,37 @@ static int vi_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
 
 static int vi_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
 {
-        /* todo */
+        int r, i;
+        struct atom_clock_dividers dividers;
+        u32 tmp;
+
+        r = amdgpu_atombios_get_clock_dividers(adev,
+                                               COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
+                                               ecclk, false, &dividers);
+        if (r)
+                return r;
+
+        for (i = 0; i < 100; i++) {
+                if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
+                        break;
+                mdelay(10);
+        }
+        if (i == 100)
+                return -ETIMEDOUT;
+
+        tmp = RREG32_SMC(ixCG_ECLK_CNTL);
+        tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
+                CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
+        tmp |= dividers.post_divider;
+        WREG32_SMC(ixCG_ECLK_CNTL, tmp);
+
+        for (i = 0; i < 100; i++) {
+                if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
+                        break;
+                mdelay(10);
+        }
+        if (i == 100)
+                return -ETIMEDOUT;
 
         return 0;
 }
@@ -869,7 +893,6 @@ static const struct amdgpu_asic_funcs vi_asic_funcs =
 {
         .read_disabled_bios = &vi_read_disabled_bios,
         .read_bios_from_rom = &vi_read_bios_from_rom,
-        .detect_hw_virtualization = vi_detect_hw_virtualization,
         .read_register = &vi_read_register,
         .reset = &vi_asic_reset,
         .set_vga_state = &vi_vga_set_state,
@@ -905,6 +928,11 @@ static int vi_common_early_init(void *handle)
                 (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
                 smc_enabled = true;
 
+        if (amdgpu_sriov_vf(adev)) {
+                amdgpu_virt_init_setting(adev);
+                xgpu_vi_mailbox_set_irq_funcs(adev);
+        }
+
         adev->rev_id = vi_get_rev_id(adev);
         adev->external_rev_id = 0xFF;
         switch (adev->asic_type) {
@@ -1061,10 +1089,6 @@ static int vi_common_early_init(void *handle)
                 return -EINVAL;
         }
 
-        /* in early init stage, vbios code won't work */
-        if (adev->asic_funcs->detect_hw_virtualization)
-                amdgpu_asic_detect_hw_virtualization(adev);
-
         if (amdgpu_smc_load_fw && smc_enabled)
                 adev->firmware.smu_load = true;
 
@@ -1073,8 +1097,23 @@ static int vi_common_early_init(void *handle)
         return 0;
 }
 
+static int vi_common_late_init(void *handle)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+        if (amdgpu_sriov_vf(adev))
+                xgpu_vi_mailbox_get_irq(adev);
+
+        return 0;
+}
+
 static int vi_common_sw_init(void *handle)
 {
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+        if (amdgpu_sriov_vf(adev))
+                xgpu_vi_mailbox_add_irq_id(adev);
+
         return 0;
 }
 
@@ -1106,6 +1145,9 @@ static int vi_common_hw_fini(void *handle)
         /* enable the doorbell aperture */
         vi_enable_doorbell_aperture(adev, false);
 
+        if (amdgpu_sriov_vf(adev))
+                xgpu_vi_mailbox_put_irq(adev);
+
         return 0;
 }
 
@@ -1190,6 +1232,23 @@ static void vi_update_hdp_light_sleep(struct amdgpu_device *adev,
                 WREG32(mmHDP_MEM_POWER_LS, data);
 }
 
+static void vi_update_drm_light_sleep(struct amdgpu_device *adev,
+                                      bool enable)
+{
+        uint32_t temp, data;
+
+        temp = data = RREG32(0x157a);
+
+        if (enable && (adev->cg_flags & AMD_CG_SUPPORT_DRM_LS))
+                data |= 1;
+        else
+                data &= ~1;
+
+        if (temp != data)
+                WREG32(0x157a, data);
+}
+
+
 static void vi_update_rom_medium_grain_clock_gating(struct amdgpu_device *adev,
                                                     bool enable)
 {
@@ -1350,6 +1409,8 @@ static int vi_common_set_clockgating_state(void *handle,
                                 state == AMD_CG_STATE_GATE ? true : false);
                 vi_update_hdp_light_sleep(adev,
                                 state == AMD_CG_STATE_GATE ? true : false);
+                vi_update_drm_light_sleep(adev,
+                                state == AMD_CG_STATE_GATE ? true : false);
                 break;
         case CHIP_TONGA:
         case CHIP_POLARIS10:
@@ -1368,10 +1429,36 @@ static int vi_common_set_powergating_state(void *handle,
         return 0;
 }
 
+static void vi_common_get_clockgating_state(void *handle, u32 *flags)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        int data;
+
+        /* AMD_CG_SUPPORT_BIF_LS */
+        data = RREG32_PCIE(ixPCIE_CNTL2);
+        if (data & PCIE_CNTL2__SLV_MEM_LS_EN_MASK)
+                *flags |= AMD_CG_SUPPORT_BIF_LS;
+
+        /* AMD_CG_SUPPORT_HDP_LS */
+        data = RREG32(mmHDP_MEM_POWER_LS);
+        if (data & HDP_MEM_POWER_LS__LS_ENABLE_MASK)
+                *flags |= AMD_CG_SUPPORT_HDP_LS;
+
+        /* AMD_CG_SUPPORT_HDP_MGCG */
+        data = RREG32(mmHDP_HOST_PATH_CNTL);
+        if (!(data & HDP_HOST_PATH_CNTL__CLOCK_GATING_DIS_MASK))
+                *flags |= AMD_CG_SUPPORT_HDP_MGCG;
+
+        /* AMD_CG_SUPPORT_ROM_MGCG */
+        data = RREG32_SMC(ixCGTT_ROM_CLK_CTRL0);
+        if (!(data & CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK))
+                *flags |= AMD_CG_SUPPORT_ROM_MGCG;
+}
+
 static const struct amd_ip_funcs vi_common_ip_funcs = {
         .name = "vi_common",
         .early_init = vi_common_early_init,
-        .late_init = NULL,
+        .late_init = vi_common_late_init,
         .sw_init = vi_common_sw_init,
         .sw_fini = vi_common_sw_fini,
         .hw_init = vi_common_hw_init,
@@ -1383,6 +1470,7 @@ static const struct amd_ip_funcs vi_common_ip_funcs = {
         .soft_reset = vi_common_soft_reset,
         .set_clockgating_state = vi_common_set_clockgating_state,
         .set_powergating_state = vi_common_set_powergating_state,
+        .get_clockgating_state = vi_common_get_clockgating_state,
 };
 
 static const struct amdgpu_ip_block_version vi_common_ip_block =
@@ -1396,6 +1484,12 @@ static const struct amdgpu_ip_block_version vi_common_ip_block =
 
 int vi_set_ip_blocks(struct amdgpu_device *adev)
 {
+        /* in early init stage, vbios code won't work */
+        vi_detect_hw_virtualization(adev);
+
+        if (amdgpu_sriov_vf(adev))
+                adev->virt.ops = &xgpu_vi_virt_ops;
+
         switch (adev->asic_type) {
         case CHIP_TOPAZ:
                 /* topaz has no DCE, UVD, VCE */
@@ -1413,28 +1507,32 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
                 amdgpu_ip_block_add(adev, &gmc_v8_5_ip_block);
                 amdgpu_ip_block_add(adev, &tonga_ih_ip_block);
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
-                if (adev->enable_virtual_display)
+                if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
                 else
                         amdgpu_ip_block_add(adev, &dce_v10_1_ip_block);
                 amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
                 amdgpu_ip_block_add(adev, &sdma_v3_0_ip_block);
-                amdgpu_ip_block_add(adev, &uvd_v6_0_ip_block);
-                amdgpu_ip_block_add(adev, &vce_v3_0_ip_block);
+                if (!amdgpu_sriov_vf(adev)) {
+                        amdgpu_ip_block_add(adev, &uvd_v6_0_ip_block);
+                        amdgpu_ip_block_add(adev, &vce_v3_0_ip_block);
+                }
                 break;
         case CHIP_TONGA:
                 amdgpu_ip_block_add(adev, &vi_common_ip_block);
                 amdgpu_ip_block_add(adev, &gmc_v8_0_ip_block);
                 amdgpu_ip_block_add(adev, &tonga_ih_ip_block);
                 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
-                if (adev->enable_virtual_display)
+                if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
                         amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
                 else
                         amdgpu_ip_block_add(adev, &dce_v10_0_ip_block);
                 amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
                 amdgpu_ip_block_add(adev, &sdma_v3_0_ip_block);
-                amdgpu_ip_block_add(adev, &uvd_v5_0_ip_block);
-                amdgpu_ip_block_add(adev, &vce_v3_0_ip_block);
+                if (!amdgpu_sriov_vf(adev)) {
+                        amdgpu_ip_block_add(adev, &uvd_v5_0_ip_block);
+                        amdgpu_ip_block_add(adev, &vce_v3_0_ip_block);
+                }
                 break;
         case CHIP_POLARIS11:
         case CHIP_POLARIS10:
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.h b/drivers/gpu/drm/amd/amdgpu/vi.h
index 575d7aed5d32..719587b8b0cb 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.h
+++ b/drivers/gpu/drm/amd/amdgpu/vi.h
@@ -28,4 +28,116 @@ void vi_srbm_select(struct amdgpu_device *adev,
                     u32 me, u32 pipe, u32 queue, u32 vmid);
 int vi_set_ip_blocks(struct amdgpu_device *adev);
 
+struct amdgpu_ce_ib_state
+{
+        uint32_t    ce_ib_completion_status;
+        uint32_t    ce_constegnine_count;
+        uint32_t    ce_ibOffset_ib1;
+        uint32_t    ce_ibOffset_ib2;
+}; /* Total of 4 DWORD */
+
+struct amdgpu_de_ib_state
+{
+        uint32_t    ib_completion_status;
+        uint32_t    de_constEngine_count;
+        uint32_t    ib_offset_ib1;
+        uint32_t    ib_offset_ib2;
+        uint32_t    preamble_begin_ib1;
+        uint32_t    preamble_begin_ib2;
+        uint32_t    preamble_end_ib1;
+        uint32_t    preamble_end_ib2;
+        uint32_t    draw_indirect_baseLo;
+        uint32_t    draw_indirect_baseHi;
+        uint32_t    disp_indirect_baseLo;
+        uint32_t    disp_indirect_baseHi;
+        uint32_t    gds_backup_addrlo;
+        uint32_t    gds_backup_addrhi;
+        uint32_t    index_base_addrlo;
+        uint32_t    index_base_addrhi;
+        uint32_t    sample_cntl;
+}; /* Total of 17 DWORD */
+
+struct amdgpu_ce_ib_state_chained_ib
+{
+        /* section of non chained ib part */
+        uint32_t    ce_ib_completion_status;
+        uint32_t    ce_constegnine_count;
+        uint32_t    ce_ibOffset_ib1;
+        uint32_t    ce_ibOffset_ib2;
+
+        /* section of chained ib */
+        uint32_t    ce_chainib_addrlo_ib1;
+        uint32_t    ce_chainib_addrlo_ib2;
+        uint32_t    ce_chainib_addrhi_ib1;
+        uint32_t    ce_chainib_addrhi_ib2;
+        uint32_t    ce_chainib_size_ib1;
+        uint32_t    ce_chainib_size_ib2;
+}; /* total 10 DWORD */
+
+struct amdgpu_de_ib_state_chained_ib
+{
+        /* section of non chained ib part */
+        uint32_t    ib_completion_status;
+        uint32_t    de_constEngine_count;
+        uint32_t    ib_offset_ib1;
+        uint32_t    ib_offset_ib2;
+
+        /* section of chained ib */
+        uint32_t    chain_ib_addrlo_ib1;
+        uint32_t    chain_ib_addrlo_ib2;
+        uint32_t    chain_ib_addrhi_ib1;
+        uint32_t    chain_ib_addrhi_ib2;
+        uint32_t    chain_ib_size_ib1;
+        uint32_t    chain_ib_size_ib2;
+
+        /* section of non chained ib part */
+        uint32_t    preamble_begin_ib1;
+        uint32_t    preamble_begin_ib2;
+        uint32_t    preamble_end_ib1;
+        uint32_t    preamble_end_ib2;
+
+        /* section of chained ib */
+        uint32_t    chain_ib_pream_addrlo_ib1;
+        uint32_t    chain_ib_pream_addrlo_ib2;
+        uint32_t    chain_ib_pream_addrhi_ib1;
+        uint32_t    chain_ib_pream_addrhi_ib2;
+
+        /* section of non chained ib part */
+        uint32_t    draw_indirect_baseLo;
+        uint32_t    draw_indirect_baseHi;
+        uint32_t    disp_indirect_baseLo;
+        uint32_t    disp_indirect_baseHi;
+        uint32_t    gds_backup_addrlo;
+        uint32_t    gds_backup_addrhi;
+        uint32_t    index_base_addrlo;
+        uint32_t    index_base_addrhi;
+        uint32_t    sample_cntl;
+}; /* Total of 27 DWORD */
+
+struct amdgpu_gfx_meta_data
+{
+        /* 4 DWORD, address must be 4KB aligned */
+        struct amdgpu_ce_ib_state    ce_payload;
+        uint32_t                     reserved1[60];
+        /* 17 DWORD, address must be 64B aligned */
+        struct amdgpu_de_ib_state    de_payload;
+        /* PFP IB base address which get pre-empted */
+        uint32_t                     DeIbBaseAddrLo;
+        uint32_t                     DeIbBaseAddrHi;
+        uint32_t                     reserved2[941];
+}; /* Total of 4K Bytes */
+
+struct amdgpu_gfx_meta_data_chained_ib
+{
+        /* 10 DWORD, address must be 4KB aligned */
+        struct amdgpu_ce_ib_state_chained_ib   ce_payload;
+        uint32_t                               reserved1[54];
+        /* 27 DWORD, address must be 64B aligned */
+        struct amdgpu_de_ib_state_chained_ib   de_payload;
+        /* PFP IB base address which get pre-empted */
+        uint32_t                               DeIbBaseAddrLo;
+        uint32_t                               DeIbBaseAddrHi;
+        uint32_t                               reserved2[931];
+}; /* Total of 4K Bytes */
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/vi_dpm.h b/drivers/gpu/drm/amd/amdgpu/vi_dpm.h
index fc120ba18aad..c43e03fddfba 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/vi_dpm.h
@@ -29,8 +29,4 @@ int cz_smu_init(struct amdgpu_device *adev);
 int cz_smu_start(struct amdgpu_device *adev);
 int cz_smu_fini(struct amdgpu_device *adev);
 
-extern const struct amd_ip_funcs tonga_dpm_ip_funcs;
-extern const struct amd_ip_funcs fiji_dpm_ip_funcs;
-extern const struct amd_ip_funcs iceland_dpm_ip_funcs;
-
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/vid.h b/drivers/gpu/drm/amd/amdgpu/vid.h
index 11746f22d0c5..7a3863a45f0a 100644
--- a/drivers/gpu/drm/amd/amdgpu/vid.h
+++ b/drivers/gpu/drm/amd/amdgpu/vid.h
@@ -360,6 +360,8 @@
 #define PACKET3_WAIT_ON_CE_COUNTER                      0x86
 #define PACKET3_WAIT_ON_DE_COUNTER_DIFF                 0x88
 #define PACKET3_SWITCH_BUFFER                           0x8B
+#define PACKET3_SET_RESOURCES                           0xA0
+#define PACKET3_MAP_QUEUES                              0xA2
 
 #define VCE_CMD_NO_OP           0x00000000
 #define VCE_CMD_END             0x00000001
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index 85f358764bbc..43f45adeccd1 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -80,6 +80,18 @@ enum amd_clockgating_state {
         AMD_CG_STATE_UNGATE,
 };
 
+enum amd_dpm_forced_level {
+        AMD_DPM_FORCED_LEVEL_AUTO = 0x1,
+        AMD_DPM_FORCED_LEVEL_MANUAL = 0x2,
+        AMD_DPM_FORCED_LEVEL_LOW = 0x4,
+        AMD_DPM_FORCED_LEVEL_HIGH = 0x8,
+        AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD = 0x10,
+        AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK = 0x20,
+        AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK = 0x40,
+        AMD_DPM_FORCED_LEVEL_PROFILE_PEAK = 0x80,
+        AMD_DPM_FORCED_LEVEL_PROFILE_EXIT = 0x100,
+};
+
 enum amd_powergating_state {
         AMD_PG_STATE_GATE = 0,
         AMD_PG_STATE_UNGATE,
@@ -206,6 +218,8 @@ struct amd_ip_funcs {
         /* enable/disable pg for the IP block */
         int (*set_powergating_state)(void *handle,
                                      enum amd_powergating_state state);
+        /* get current clockgating status */
+        void (*get_clockgating_state)(void *handle, u32 *flags);
 };
 
 #endif /* __AMD_SHARED_H__ */
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_10_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_10_0_d.h
index 95570dbd18bb..813957a17a2d 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_10_0_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_10_0_d.h
@@ -4552,6 +4552,14 @@
 #define mmDP4_DP_DPHY_PRBS_CNTL                                                 0x4eb5
 #define mmDP5_DP_DPHY_PRBS_CNTL                                                 0x4fb5
 #define mmDP6_DP_DPHY_PRBS_CNTL                                                 0x54b5
+#define mmDP_DPHY_SCRAM_CNTL                                                    0x4ab6
+#define mmDP0_DP_DPHY_SCRAM_CNTL                                                0x4ab6
+#define mmDP1_DP_DPHY_SCRAM_CNTL                                                0x4bb6
+#define mmDP2_DP_DPHY_SCRAM_CNTL                                                0x4cb6
+#define mmDP3_DP_DPHY_SCRAM_CNTL                                                0x4db6
+#define mmDP4_DP_DPHY_SCRAM_CNTL                                                0x4eb6
+#define mmDP5_DP_DPHY_SCRAM_CNTL                                                0x4fb6
+#define mmDP6_DP_DPHY_SCRAM_CNTL                                                0x54b6
 #define mmDP_DPHY_CRC_EN                                                        0x4ab7
 #define mmDP0_DP_DPHY_CRC_EN                                                    0x4ab7
 #define mmDP1_DP_DPHY_CRC_EN                                                    0x4bb7
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_10_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_10_0_sh_mask.h
index 8a75eb9d732b..c755f43aaaf8 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_10_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_10_0_sh_mask.h
@@ -8690,6 +8690,10 @@
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL__SHIFT 0x4
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED_MASK 0x7fffff00
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED__SHIFT 0x8
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE_MASK 0x10
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE__SHIFT 0x4
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT_MASK 0x3ff00
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT__SHIFT 0x8
 #define DP_DPHY_CRC_EN__DPHY_CRC_EN_MASK 0x1
 #define DP_DPHY_CRC_EN__DPHY_CRC_EN__SHIFT 0x0
 #define DP_DPHY_CRC_EN__DPHY_CRC_CONT_EN_MASK 0x10
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_0_d.h
index c39234ecedd0..6df651a94b0a 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_0_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_0_d.h
@@ -4544,6 +4544,15 @@
 #define mmDP6_DP_DPHY_PRBS_CNTL                                                 0x54b5
 #define mmDP7_DP_DPHY_PRBS_CNTL                                                 0x56b5
 #define mmDP8_DP_DPHY_PRBS_CNTL                                                 0x57b5
+#define mmDP_DPHY_SCRAM_CNTL                                                    0x4ab6
+#define mmDP0_DP_DPHY_SCRAM_CNTL                                                0x4ab6
+#define mmDP1_DP_DPHY_SCRAM_CNTL                                                0x4bb6
+#define mmDP2_DP_DPHY_SCRAM_CNTL                                                0x4cb6
+#define mmDP3_DP_DPHY_SCRAM_CNTL                                                0x4db6
+#define mmDP4_DP_DPHY_SCRAM_CNTL                                                0x4eb6
+#define mmDP5_DP_DPHY_SCRAM_CNTL                                                0x4fb6
+#define mmDP6_DP_DPHY_SCRAM_CNTL                                                0x54b6
+#define mmDP8_DP_DPHY_SCRAM_CNTL                                                0x56b6
 #define mmDP_DPHY_BS_SR_SWAP_CNTL                                               0x4adc
 #define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL                                           0x4adc
 #define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL                                           0x4bdc
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_0_sh_mask.h
index a438c2b6e280..14a3bacfcfd1 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_0_sh_mask.h
@@ -6004,6 +6004,8 @@
 #define DIG_DISPCLK_SWITCH_STATUS__DIG_DISPCLK_SWITCH_ALLOWED_INT_MASK__SHIFT 0xc
 #define HDMI_CONTROL__HDMI_KEEPOUT_MODE_MASK 0x1
 #define HDMI_CONTROL__HDMI_KEEPOUT_MODE__SHIFT 0x0
+#define HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN_MASK 0x2
+#define HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN__SHIFT 0x1
 #define HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE_MASK 0x4
 #define HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE__SHIFT 0x2
 #define HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED_MASK 0x8
@@ -8364,6 +8366,10 @@
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL__SHIFT 0x4
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED_MASK 0x7fffff00
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED__SHIFT 0x8
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE_MASK 0x10
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE__SHIFT 0x4
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT_MASK 0x3ff00
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT__SHIFT 0x8
 #define DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_MASK 0x3ff
 #define DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT__SHIFT 0x0
 #define DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE_MASK 0x8000
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_2_d.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_2_d.h
index 09a7df17570d..367b191d49fb 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_2_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_2_d.h
@@ -5776,6 +5776,15 @@
 #define mmDP6_DP_DPHY_PRBS_CNTL                                                 0x54b5
 #define mmDP7_DP_DPHY_PRBS_CNTL                                                 0x56b5
 #define mmDP8_DP_DPHY_PRBS_CNTL                                                 0x57b5
+#define mmDP_DPHY_SCRAM_CNTL                                                    0x4ab6
+#define mmDP0_DP_DPHY_SCRAM_CNTL                                                0x4ab6
+#define mmDP1_DP_DPHY_SCRAM_CNTL                                                0x4bb6
+#define mmDP2_DP_DPHY_SCRAM_CNTL                                                0x4cb6
+#define mmDP3_DP_DPHY_SCRAM_CNTL                                                0x4db6
+#define mmDP4_DP_DPHY_SCRAM_CNTL                                                0x4eb6
+#define mmDP5_DP_DPHY_SCRAM_CNTL                                                0x4fb6
+#define mmDP6_DP_DPHY_SCRAM_CNTL                                                0x54b6
+#define mmDP8_DP_DPHY_SCRAM_CNTL                                                0x56b6
 #define mmDP_DPHY_BS_SR_SWAP_CNTL                                               0x4adc
 #define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL                                           0x4adc
 #define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL                                           0x4bdc
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_2_sh_mask.h
index 1ddc4183a1c9..106094ed0661 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_2_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_11_2_sh_mask.h
@@ -7088,6 +7088,8 @@
 #define DIG_DISPCLK_SWITCH_STATUS__DIG_DISPCLK_SWITCH_ALLOWED_INT_MASK__SHIFT 0xc
 #define HDMI_CONTROL__HDMI_KEEPOUT_MODE_MASK 0x1
 #define HDMI_CONTROL__HDMI_KEEPOUT_MODE__SHIFT 0x0
+#define HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN_MASK 0x2
+#define HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN__SHIFT 0x1
 #define HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE_MASK 0x4
 #define HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE__SHIFT 0x2
 #define HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED_MASK 0x8
@@ -9626,6 +9628,10 @@
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL__SHIFT 0x4
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED_MASK 0x7fffff00
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED__SHIFT 0x8
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE_MASK 0x10
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE__SHIFT 0x4
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT_MASK 0x3ff00
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT__SHIFT 0x8
 #define DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_MASK 0x3ff
 #define DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT__SHIFT 0x0
 #define DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE_MASK 0x8000
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_8_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_8_0_d.h
index d3ccf5a86de0..93d84a475134 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_8_0_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_8_0_d.h
@@ -3920,6 +3920,14 @@
 #define mmDP4_DP_DPHY_PRBS_CNTL                                                 0x48d4
 #define mmDP5_DP_DPHY_PRBS_CNTL                                                 0x4bd4
 #define mmDP6_DP_DPHY_PRBS_CNTL                                                 0x4ed4
+#define mmDP_DPHY_SCRAM_CNTL                                                    0x1cd5
+#define mmDP0_DP_DPHY_SCRAM_CNTL                                                0x1cd5
+#define mmDP1_DP_DPHY_SCRAM_CNTL                                                0x1fd5
+#define mmDP2_DP_DPHY_SCRAM_CNTL                                                0x42d5
+#define mmDP3_DP_DPHY_SCRAM_CNTL                                                0x45d5
+#define mmDP4_DP_DPHY_SCRAM_CNTL                                                0x48d5
+#define mmDP5_DP_DPHY_SCRAM_CNTL                                                0x4bd5
+#define mmDP6_DP_DPHY_SCRAM_CNTL                                                0x4ed5
 #define mmDP_DPHY_CRC_EN                                                        0x1cd6
 #define mmDP0_DP_DPHY_CRC_EN                                                    0x1cd6
 #define mmDP1_DP_DPHY_CRC_EN                                                    0x1fd6
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_8_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_8_0_sh_mask.h
index c331c9fe7b81..9b6825b74cc1 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_8_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_8_0_sh_mask.h
@@ -9214,6 +9214,10 @@
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL__SHIFT 0x4
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED_MASK 0x7fffff00
 #define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED__SHIFT 0x8
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE_MASK 0x10
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE__SHIFT 0x4
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT_MASK 0x3ff00
+#define DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT__SHIFT 0x8
 #define DP_DPHY_CRC_EN__DPHY_CRC_EN_MASK 0x1
 #define DP_DPHY_CRC_EN__DPHY_CRC_EN__SHIFT 0x0
 #define DP_DPHY_CRC_EN__DPHY_CRC_CONT_EN_MASK 0x10
diff --git a/drivers/gpu/drm/amd/include/asic_reg/si/si_reg.h b/drivers/gpu/drm/amd/include/asic_reg/si/si_reg.h
deleted file mode 100644
index 895c8e2353e3..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/si/si_reg.h
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2010 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Alex Deucher
- */
-#ifndef __SI_REG_H__
-#define __SI_REG_H__
-
-/* SI */
-#define SI_DC_GPIO_HPD_MASK                      0x196c
-#define SI_DC_GPIO_HPD_A                         0x196d
-#define SI_DC_GPIO_HPD_EN                        0x196e
-#define SI_DC_GPIO_HPD_Y                         0x196f
-
-#define SI_GRPH_CONTROL                          0x1a01
-#       define SI_GRPH_DEPTH(x)                  (((x) & 0x3) << 0)
-#       define SI_GRPH_DEPTH_8BPP                0
-#       define SI_GRPH_DEPTH_16BPP               1
-#       define SI_GRPH_DEPTH_32BPP               2
-#       define SI_GRPH_NUM_BANKS(x)              (((x) & 0x3) << 2)
-#       define SI_ADDR_SURF_2_BANK               0
-#       define SI_ADDR_SURF_4_BANK               1
-#       define SI_ADDR_SURF_8_BANK               2
-#       define SI_ADDR_SURF_16_BANK              3
-#       define SI_GRPH_Z(x)                      (((x) & 0x3) << 4)
-#       define SI_GRPH_BANK_WIDTH(x)             (((x) & 0x3) << 6)
-#       define SI_ADDR_SURF_BANK_WIDTH_1         0
-#       define SI_ADDR_SURF_BANK_WIDTH_2         1
-#       define SI_ADDR_SURF_BANK_WIDTH_4         2
-#       define SI_ADDR_SURF_BANK_WIDTH_8         3
-#       define SI_GRPH_FORMAT(x)                 (((x) & 0x7) << 8)
-/* 8 BPP */
-#       define SI_GRPH_FORMAT_INDEXED            0
-/* 16 BPP */
-#       define SI_GRPH_FORMAT_ARGB1555           0
-#       define SI_GRPH_FORMAT_ARGB565            1
-#       define SI_GRPH_FORMAT_ARGB4444           2
-#       define SI_GRPH_FORMAT_AI88               3
-#       define SI_GRPH_FORMAT_MONO16             4
-#       define SI_GRPH_FORMAT_BGRA5551           5
-/* 32 BPP */
-#       define SI_GRPH_FORMAT_ARGB8888           0
-#       define SI_GRPH_FORMAT_ARGB2101010        1
-#       define SI_GRPH_FORMAT_32BPP_DIG          2
-#       define SI_GRPH_FORMAT_8B_ARGB2101010     3
-#       define SI_GRPH_FORMAT_BGRA1010102        4
-#       define SI_GRPH_FORMAT_8B_BGRA1010102     5
-#       define SI_GRPH_FORMAT_RGB111110          6
-#       define SI_GRPH_FORMAT_BGR101111          7
-#       define SI_GRPH_BANK_HEIGHT(x)            (((x) & 0x3) << 11)
-#       define SI_ADDR_SURF_BANK_HEIGHT_1        0
-#       define SI_ADDR_SURF_BANK_HEIGHT_2        1
-#       define SI_ADDR_SURF_BANK_HEIGHT_4        2
-#       define SI_ADDR_SURF_BANK_HEIGHT_8        3
-#       define SI_GRPH_TILE_SPLIT(x)             (((x) & 0x7) << 13)
-#       define SI_ADDR_SURF_TILE_SPLIT_64B       0
-#       define SI_ADDR_SURF_TILE_SPLIT_128B      1
-#       define SI_ADDR_SURF_TILE_SPLIT_256B      2
-#       define SI_ADDR_SURF_TILE_SPLIT_512B      3
-#       define SI_ADDR_SURF_TILE_SPLIT_1KB       4
-#       define SI_ADDR_SURF_TILE_SPLIT_2KB       5
-#       define SI_ADDR_SURF_TILE_SPLIT_4KB       6
-#       define SI_GRPH_MACRO_TILE_ASPECT(x)      (((x) & 0x3) << 18)
-#       define SI_ADDR_SURF_MACRO_TILE_ASPECT_1  0
-#       define SI_ADDR_SURF_MACRO_TILE_ASPECT_2  1
-#       define SI_ADDR_SURF_MACRO_TILE_ASPECT_4  2
-#       define SI_ADDR_SURF_MACRO_TILE_ASPECT_8  3
-#       define SI_GRPH_ARRAY_MODE(x)             (((x) & 0x7) << 20)
-#       define SI_GRPH_ARRAY_LINEAR_GENERAL      0
-#       define SI_GRPH_ARRAY_LINEAR_ALIGNED      1
-#       define SI_GRPH_ARRAY_1D_TILED_THIN1      2
-#       define SI_GRPH_ARRAY_2D_TILED_THIN1      4
-#       define SI_GRPH_PIPE_CONFIG(x)            (((x) & 0x1f) << 24)
-#       define SI_ADDR_SURF_P2                   0
-#       define SI_ADDR_SURF_P4_8x16              4
-#       define SI_ADDR_SURF_P4_16x16             5
-#       define SI_ADDR_SURF_P4_16x32             6
-#       define SI_ADDR_SURF_P4_32x32             7
-#       define SI_ADDR_SURF_P8_16x16_8x16        8
-#       define SI_ADDR_SURF_P8_16x32_8x16        9
-#       define SI_ADDR_SURF_P8_32x32_8x16        10
-#       define SI_ADDR_SURF_P8_16x32_16x16       11
-#       define SI_ADDR_SURF_P8_32x32_16x16       12
-#       define SI_ADDR_SURF_P8_32x32_16x32       13
-#       define SI_ADDR_SURF_P8_32x64_32x32       14
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/cgs_common.h b/drivers/gpu/drm/amd/include/cgs_common.h
index e4a1697ec1d3..1d26ae768147 100644
--- a/drivers/gpu/drm/amd/include/cgs_common.h
+++ b/drivers/gpu/drm/amd/include/cgs_common.h
@@ -622,6 +622,8 @@ typedef int (*cgs_query_system_info)(struct cgs_device *cgs_device,
 
 typedef int (*cgs_is_virtualization_enabled_t)(void *cgs_device);
 
+typedef int (*cgs_enter_safe_mode)(struct cgs_device *cgs_device, bool en);
+
 struct cgs_ops {
         /* memory management calls (similar to KFD interface) */
         cgs_gpu_mem_info_t gpu_mem_info;
@@ -674,6 +676,7 @@ struct cgs_ops {
         /* get system info */
         cgs_query_system_info query_system_info;
         cgs_is_virtualization_enabled_t is_virtualization_enabled;
+        cgs_enter_safe_mode enter_safe_mode;
 };
 
 struct cgs_os_ops; /* To be define in OS-specific CGS header */
@@ -779,4 +782,8 @@ struct cgs_device
 
 #define cgs_is_virtualization_enabled(cgs_device) \
                 CGS_CALL(is_virtualization_enabled, cgs_device)
+
+#define cgs_enter_safe_mode(cgs_device, en) \
+                CGS_CALL(enter_safe_mode, cgs_device, en)
+
 #endif /* _CGS_COMMON_H */
diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
index c81cf1412728..429f18b99323 100644
--- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
@@ -20,6 +20,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include "pp_debug.h"
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/gfp.h>
@@ -29,153 +30,154 @@
 #include "pp_instance.h"
 #include "power_state.h"
 #include "eventmanager.h"
-#include "pp_debug.h"
 
 
-#define PP_CHECK(handle)                                                \
-        do {                                                            \
-                if ((handle) == NULL || (handle)->pp_valid != PP_VALID) \
-                        return -EINVAL;                                 \
-        } while (0)
+static inline int pp_check(struct pp_instance *handle)
+{
+        if (handle == NULL || handle->pp_valid != PP_VALID)
+                return -EINVAL;
 
-#define PP_CHECK_HW(hwmgr)                                              \
-        do {                                                            \
-                if ((hwmgr) == NULL || (hwmgr)->hwmgr_func == NULL)     \
-                        return 0;                                       \
-        } while (0)
+        if (handle->smu_mgr == NULL || handle->smu_mgr->smumgr_funcs == NULL)
+                return -EINVAL;
+
+        if (handle->pm_en == 0)
+                return PP_DPM_DISABLED;
+
+        if (handle->hwmgr == NULL || handle->hwmgr->hwmgr_func == NULL
+                || handle->eventmgr == NULL)
+                return PP_DPM_DISABLED;
+
+        return 0;
+}
 
 static int pp_early_init(void *handle)
 {
+        int ret;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+
+        ret = smum_early_init(pp_handle);
+        if (ret)
+                return ret;
+
+        if ((pp_handle->pm_en == 0)
+                || cgs_is_virtualization_enabled(pp_handle->device))
+                return PP_DPM_DISABLED;
+
+        ret = hwmgr_early_init(pp_handle);
+        if (ret) {
+                pp_handle->pm_en = 0;
+                return PP_DPM_DISABLED;
+        }
+
+        ret = eventmgr_early_init(pp_handle);
+        if (ret) {
+                kfree(pp_handle->hwmgr);
+                pp_handle->hwmgr = NULL;
+                pp_handle->pm_en = 0;
+                return PP_DPM_DISABLED;
+        }
+
         return 0;
 }
 
 static int pp_sw_init(void *handle)
 {
-        struct pp_instance *pp_handle;
-        struct pp_hwmgr  *hwmgr;
+        struct pp_smumgr *smumgr;
         int ret = 0;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
 
-        if (handle == NULL)
-                return -EINVAL;
-
-        pp_handle = (struct pp_instance *)handle;
-        hwmgr = pp_handle->hwmgr;
-
-        PP_CHECK_HW(hwmgr);
-
-        if (hwmgr->pptable_func == NULL ||
-            hwmgr->pptable_func->pptable_init == NULL ||
-            hwmgr->hwmgr_func->backend_init == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        ret = hwmgr->pptable_func->pptable_init(hwmgr);
-        if (ret)
-                goto err;
+        if (ret == 0 || ret == PP_DPM_DISABLED) {
+                smumgr = pp_handle->smu_mgr;
 
-        ret = hwmgr->hwmgr_func->backend_init(hwmgr);
-        if (ret)
-                goto err1;
+                if (smumgr->smumgr_funcs->smu_init == NULL)
+                        return -EINVAL;
 
-        pr_info("amdgpu: powerplay initialized\n");
+                ret = smumgr->smumgr_funcs->smu_init(smumgr);
 
-        return 0;
-err1:
-        if (hwmgr->pptable_func->pptable_fini)
-                hwmgr->pptable_func->pptable_fini(hwmgr);
-err:
-        pr_err("amdgpu: powerplay initialization failed\n");
+                pr_info("amdgpu: powerplay sw initialized\n");
+        }
         return ret;
 }
 
 static int pp_sw_fini(void *handle)
 {
-        struct pp_instance *pp_handle;
-        struct pp_hwmgr  *hwmgr;
+        struct pp_smumgr *smumgr;
         int ret = 0;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
+        if (ret == 0 || ret == PP_DPM_DISABLED) {
+                smumgr = pp_handle->smu_mgr;
 
-        pp_handle = (struct pp_instance *)handle;
-        hwmgr = pp_handle->hwmgr;
-
-        PP_CHECK_HW(hwmgr);
-
-        if (hwmgr->hwmgr_func->backend_fini != NULL)
-                ret = hwmgr->hwmgr_func->backend_fini(hwmgr);
-
-        if (hwmgr->pptable_func->pptable_fini)
-                hwmgr->pptable_func->pptable_fini(hwmgr);
+                if (smumgr->smumgr_funcs->smu_fini == NULL)
+                        return -EINVAL;
 
+                ret = smumgr->smumgr_funcs->smu_fini(smumgr);
+        }
         return ret;
 }
 
 static int pp_hw_init(void *handle)
 {
-        struct pp_instance *pp_handle;
         struct pp_smumgr *smumgr;
         struct pp_eventmgr *eventmgr;
-        struct pp_hwmgr  *hwmgr;
         int ret = 0;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        pp_handle = (struct pp_instance *)handle;
-        smumgr = pp_handle->smu_mgr;
-        hwmgr = pp_handle->hwmgr;
+        if (ret == 0 || ret == PP_DPM_DISABLED) {
+                smumgr = pp_handle->smu_mgr;
 
-        if (smumgr == NULL || smumgr->smumgr_funcs == NULL ||
-                smumgr->smumgr_funcs->smu_init == NULL ||
-                smumgr->smumgr_funcs->start_smu == NULL)
-                return -EINVAL;
-
-        ret = smumgr->smumgr_funcs->smu_init(smumgr);
-        if (ret) {
-                printk(KERN_ERR "[ powerplay ] smc initialization failed\n");
-                return ret;
-        }
+                if (smumgr->smumgr_funcs->start_smu == NULL)
+                        return -EINVAL;
 
-        ret = smumgr->smumgr_funcs->start_smu(smumgr);
-        if (ret) {
-                printk(KERN_ERR "[ powerplay ] smc start failed\n");
-                smumgr->smumgr_funcs->smu_fini(smumgr);
-                return ret;
+                if(smumgr->smumgr_funcs->start_smu(smumgr)) {
+                        pr_err("smc start failed\n");
+                        smumgr->smumgr_funcs->smu_fini(smumgr);
+                        return -EINVAL;;
+                }
+                if (ret == PP_DPM_DISABLED)
+                        return PP_DPM_DISABLED;
         }
 
-        PP_CHECK_HW(hwmgr);
-
-        hw_init_power_state_table(hwmgr);
+        ret = hwmgr_hw_init(pp_handle);
+        if (ret)
+                goto err;
 
         eventmgr = pp_handle->eventmgr;
-        if (eventmgr == NULL || eventmgr->pp_eventmgr_init == NULL)
-                return -EINVAL;
+        if (eventmgr->pp_eventmgr_init == NULL ||
+                eventmgr->pp_eventmgr_init(eventmgr))
+                goto err;
 
-        ret = eventmgr->pp_eventmgr_init(eventmgr);
         return 0;
+err:
+        pp_handle->pm_en = 0;
+        kfree(pp_handle->eventmgr);
+        kfree(pp_handle->hwmgr);
+        pp_handle->hwmgr = NULL;
+        pp_handle->eventmgr = NULL;
+        return PP_DPM_DISABLED;
 }
 
 static int pp_hw_fini(void *handle)
 {
-        struct pp_instance *pp_handle;
-        struct pp_smumgr *smumgr;
         struct pp_eventmgr *eventmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
-
-        pp_handle = (struct pp_instance *)handle;
-        eventmgr = pp_handle->eventmgr;
+        ret = pp_check(pp_handle);
 
-        if (eventmgr != NULL && eventmgr->pp_eventmgr_fini != NULL)
-                eventmgr->pp_eventmgr_fini(eventmgr);
+        if (ret == 0) {
+                eventmgr = pp_handle->eventmgr;
 
-        smumgr = pp_handle->smu_mgr;
-
-        if (smumgr != NULL && smumgr->smumgr_funcs != NULL &&
-                smumgr->smumgr_funcs->smu_fini != NULL)
-                smumgr->smumgr_funcs->smu_fini(smumgr);
+                if (eventmgr->pp_eventmgr_fini != NULL)
+                        eventmgr->pp_eventmgr_fini(eventmgr);
 
+                hwmgr_hw_fini(pp_handle);
+        }
         return 0;
 }
 
@@ -198,16 +200,18 @@ static int pp_sw_reset(void *handle)
 int amd_set_clockgating_by_smu(void *handle, uint32_t msg_id)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->update_clock_gatings == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -218,16 +222,18 @@ static int pp_set_powergating_state(void *handle,
                                     enum amd_powergating_state state)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->enable_per_cu_power_gating == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -238,49 +244,53 @@ static int pp_set_powergating_state(void *handle,
 
 static int pp_suspend(void *handle)
 {
-        struct pp_instance *pp_handle;
         struct pp_eventmgr *eventmgr;
         struct pem_event_data event_data = { {0} };
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
+
+        if (ret != 0)
+                return ret;
 
-        pp_handle = (struct pp_instance *)handle;
         eventmgr = pp_handle->eventmgr;
+        pem_handle_event(eventmgr, AMD_PP_EVENT_SUSPEND, &event_data);
 
-        if (eventmgr != NULL)
-                pem_handle_event(eventmgr, AMD_PP_EVENT_SUSPEND, &event_data);
         return 0;
 }
 
 static int pp_resume(void *handle)
 {
-        struct pp_instance *pp_handle;
         struct pp_eventmgr *eventmgr;
         struct pem_event_data event_data = { {0} };
         struct pp_smumgr *smumgr;
-        int ret;
+        int ret, ret1;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret1 = pp_check(pp_handle);
+
+        if (ret1 != 0 && ret1 != PP_DPM_DISABLED)
+                return ret1;
 
-        pp_handle = (struct pp_instance *)handle;
         smumgr = pp_handle->smu_mgr;
 
-        if (smumgr == NULL || smumgr->smumgr_funcs == NULL ||
-                smumgr->smumgr_funcs->start_smu == NULL)
+        if (smumgr->smumgr_funcs->start_smu == NULL)
                 return -EINVAL;
 
         ret = smumgr->smumgr_funcs->start_smu(smumgr);
         if (ret) {
-                printk(KERN_ERR "[ powerplay ] smc start failed\n");
+                pr_err("smc start failed\n");
                 smumgr->smumgr_funcs->smu_fini(smumgr);
                 return ret;
         }
 
+        if (ret1 == PP_DPM_DISABLED)
+                return ret1;
+
         eventmgr = pp_handle->eventmgr;
-        if (eventmgr != NULL)
-                pem_handle_event(eventmgr, AMD_PP_EVENT_RESUME, &event_data);
+
+        pem_handle_event(eventmgr, AMD_PP_EVENT_RESUME, &event_data);
 
         return 0;
 }
@@ -315,20 +325,19 @@ static int pp_dpm_fw_loading_complete(void *handle)
 static int pp_dpm_force_performance_level(void *handle,
                                         enum amd_dpm_forced_level level)
 {
-        struct pp_instance *pp_handle;
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        pp_handle = (struct pp_instance *)handle;
+        if (ret != 0)
+                return ret;
 
         hwmgr = pp_handle->hwmgr;
 
-        PP_CHECK_HW(hwmgr);
-
         if (hwmgr->hwmgr_func->force_dpm_level == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -341,30 +350,34 @@ static enum amd_dpm_forced_level pp_dpm_get_performance_level(
                                                                 void *handle)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
-        return (((struct pp_instance *)handle)->hwmgr->dpm_level);
+        return hwmgr->dpm_level;
 }
 
 static int pp_dpm_get_sclk(void *handle, bool low)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->get_sclk == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -374,16 +387,18 @@ static int pp_dpm_get_sclk(void *handle, bool low)
 static int pp_dpm_get_mclk(void *handle, bool low)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->get_mclk == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -393,16 +408,18 @@ static int pp_dpm_get_mclk(void *handle, bool low)
 static int pp_dpm_powergate_vce(void *handle, bool gate)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->powergate_vce == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -412,16 +429,18 @@ static int pp_dpm_powergate_vce(void *handle, bool gate)
 static int pp_dpm_powergate_uvd(void *handle, bool gate)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->powergate_uvd == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -446,16 +465,13 @@ static int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_event event_id,
                 void *input, void *output)
 {
         int ret = 0;
-        struct pp_instance *pp_handle;
         struct pem_event_data data = { {0} };
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
 
-        pp_handle = (struct pp_instance *)handle;
+        ret = pp_check(pp_handle);
 
-        if (pp_handle == NULL)
-                return -EINVAL;
-
-        if (pp_handle->eventmgr == NULL)
-                return 0;
+        if (ret != 0)
+                return ret;
 
         switch (event_id) {
         case AMD_PP_EVENT_DISPLAY_CONFIG_CHANGE:
@@ -489,13 +505,17 @@ static enum amd_pm_state_type pp_dpm_get_current_power_state(void *handle)
 {
         struct pp_hwmgr *hwmgr;
         struct pp_power_state *state;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        if (hwmgr == NULL || hwmgr->current_ps == NULL)
+        hwmgr = pp_handle->hwmgr;
+
+        if (hwmgr->current_ps == NULL)
                 return -EINVAL;
 
         state = hwmgr->current_ps;
@@ -518,16 +538,18 @@ static enum amd_pm_state_type pp_dpm_get_current_power_state(void *handle)
 static int pp_dpm_set_fan_control_mode(void *handle, uint32_t mode)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->set_fan_control_mode == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -537,16 +559,18 @@ static int pp_dpm_set_fan_control_mode(void *handle, uint32_t mode)
 static int pp_dpm_get_fan_control_mode(void *handle)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->get_fan_control_mode == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -556,16 +580,18 @@ static int pp_dpm_get_fan_control_mode(void *handle)
 static int pp_dpm_set_fan_speed_percent(void *handle, uint32_t percent)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->set_fan_speed_percent == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -575,16 +601,18 @@ static int pp_dpm_set_fan_speed_percent(void *handle, uint32_t percent)
 static int pp_dpm_get_fan_speed_percent(void *handle, uint32_t *speed)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->get_fan_speed_percent == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -594,13 +622,15 @@ static int pp_dpm_get_fan_speed_percent(void *handle, uint32_t *speed)
 static int pp_dpm_get_fan_speed_rpm(void *handle, uint32_t *rpm)
 {
         struct pp_hwmgr *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->get_fan_speed_rpm == NULL)
                 return -EINVAL;
@@ -611,16 +641,18 @@ static int pp_dpm_get_fan_speed_rpm(void *handle, uint32_t *rpm)
 static int pp_dpm_get_temperature(void *handle)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->get_temperature == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -632,13 +664,17 @@ static int pp_dpm_get_pp_num_states(void *handle,
 {
         struct pp_hwmgr *hwmgr;
         int i;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (!handle)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
+
+        hwmgr = pp_handle->hwmgr;
 
-        if (hwmgr == NULL || hwmgr->ps == NULL)
+        if (hwmgr->ps == NULL)
                 return -EINVAL;
 
         data->nums = hwmgr->num_ps;
@@ -670,13 +706,15 @@ static int pp_dpm_get_pp_num_states(void *handle,
 static int pp_dpm_get_pp_table(void *handle, char **table)
 {
         struct pp_hwmgr *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (!handle)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (!hwmgr->soft_pp_table)
                 return -EINVAL;
@@ -689,13 +727,15 @@ static int pp_dpm_get_pp_table(void *handle, char **table)
 static int pp_dpm_set_pp_table(void *handle, const char *buf, size_t size)
 {
         struct pp_hwmgr *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (!handle)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (!hwmgr->hardcode_pp_table) {
                 hwmgr->hardcode_pp_table = kmemdup(hwmgr->soft_pp_table,
@@ -717,16 +757,18 @@ static int pp_dpm_force_clock_level(void *handle,
                 enum pp_clock_type type, uint32_t mask)
 {
         struct pp_hwmgr *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (!handle)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->force_clock_level == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -737,16 +779,18 @@ static int pp_dpm_print_clock_levels(void *handle,
                 enum pp_clock_type type, char *buf)
 {
         struct pp_hwmgr *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (!handle)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->print_clock_levels == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
         return hwmgr->hwmgr_func->print_clock_levels(hwmgr, type, buf);
@@ -755,16 +799,18 @@ static int pp_dpm_print_clock_levels(void *handle,
 static int pp_dpm_get_sclk_od(void *handle)
 {
         struct pp_hwmgr *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (!handle)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->get_sclk_od == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -774,16 +820,18 @@ static int pp_dpm_get_sclk_od(void *handle)
 static int pp_dpm_set_sclk_od(void *handle, uint32_t value)
 {
         struct pp_hwmgr *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (!handle)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->set_sclk_od == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -793,16 +841,18 @@ static int pp_dpm_set_sclk_od(void *handle, uint32_t value)
 static int pp_dpm_get_mclk_od(void *handle)
 {
         struct pp_hwmgr *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (!handle)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->get_mclk_od == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -812,16 +862,18 @@ static int pp_dpm_get_mclk_od(void *handle)
 static int pp_dpm_set_mclk_od(void *handle, uint32_t value)
 {
         struct pp_hwmgr *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (!handle)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->set_mclk_od == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -831,16 +883,18 @@ static int pp_dpm_set_mclk_od(void *handle, uint32_t value)
 static int pp_dpm_read_sensor(void *handle, int idx, int32_t *value)
 {
         struct pp_hwmgr *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (!handle)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         if (hwmgr->hwmgr_func->read_sensor == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                pr_info("%s was not implemented.\n", __func__);
                 return 0;
         }
 
@@ -851,13 +905,18 @@ static struct amd_vce_state*
 pp_dpm_get_vce_clock_state(void *handle, unsigned idx)
 {
         struct pp_hwmgr *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        if (handle) {
-                hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        ret = pp_check(pp_handle);
 
-                if (hwmgr && idx < hwmgr->num_vce_state_tables)
-                        return &hwmgr->vce_states[idx];
-        }
+        if (ret != 0)
+                return NULL;
+
+        hwmgr = pp_handle->hwmgr;
+
+        if (hwmgr && idx < hwmgr->num_vce_state_tables)
+                return &hwmgr->vce_states[idx];
 
         return NULL;
 }
@@ -892,89 +951,44 @@ const struct amd_powerplay_funcs pp_dpm_funcs = {
         .get_vce_clock_state = pp_dpm_get_vce_clock_state,
 };
 
-static int amd_pp_instance_init(struct amd_pp_init *pp_init,
-                                struct amd_powerplay *amd_pp)
+int amd_powerplay_create(struct amd_pp_init *pp_init,
+                                void **handle)
 {
-        int ret;
-        struct pp_instance *handle;
-
-        handle = kzalloc(sizeof(struct pp_instance), GFP_KERNEL);
-        if (handle == NULL)
-                return -ENOMEM;
-
-        handle->pp_valid = PP_VALID;
-
-        ret = smum_init(pp_init, handle);
-        if (ret)
-                goto fail_smum;
-
-
-        amd_pp->pp_handle = handle;
+        struct pp_instance *instance;
 
-        if ((amdgpu_dpm == 0)
-                || cgs_is_virtualization_enabled(pp_init->device))
-                return 0;
+        if (pp_init == NULL || handle == NULL)
+                return -EINVAL;
 
-        ret = hwmgr_init(pp_init, handle);
-        if (ret)
-                goto fail_hwmgr;
+        instance = kzalloc(sizeof(struct pp_instance), GFP_KERNEL);
+        if (instance == NULL)
+                return -ENOMEM;
 
-        ret = eventmgr_init(handle);
-        if (ret)
-                goto fail_eventmgr;
+        instance->pp_valid = PP_VALID;
+        instance->chip_family = pp_init->chip_family;
+        instance->chip_id = pp_init->chip_id;
+        instance->pm_en = pp_init->pm_en;
+        instance->feature_mask = pp_init->feature_mask;
+        instance->device = pp_init->device;
+        *handle = instance;
 
         return 0;
-
-fail_eventmgr:
-        hwmgr_fini(handle->hwmgr);
-fail_hwmgr:
-        smum_fini(handle->smu_mgr);
-fail_smum:
-        kfree(handle);
-        return ret;
 }
 
-static int amd_pp_instance_fini(void *handle)
+int amd_powerplay_destroy(void *handle)
 {
         struct pp_instance *instance = (struct pp_instance *)handle;
 
-        if (instance == NULL)
-                return -EINVAL;
-
-        if ((amdgpu_dpm != 0)
-                && !cgs_is_virtualization_enabled(instance->smu_mgr->device)) {
-                eventmgr_fini(instance->eventmgr);
-                hwmgr_fini(instance->hwmgr);
+        if (instance->pm_en) {
+                kfree(instance->eventmgr);
+                kfree(instance->hwmgr);
+                instance->hwmgr = NULL;
+                instance->eventmgr = NULL;
         }
 
-        smum_fini(instance->smu_mgr);
-        kfree(handle);
-        return 0;
-}
-
-int amd_powerplay_init(struct amd_pp_init *pp_init,
-                       struct amd_powerplay *amd_pp)
-{
-        int ret;
-
-        if (pp_init == NULL || amd_pp == NULL)
-                return -EINVAL;
-
-        ret = amd_pp_instance_init(pp_init, amd_pp);
-
-        if (ret)
-                return ret;
-
-        amd_pp->ip_funcs = &pp_ip_funcs;
-        amd_pp->pp_funcs = &pp_dpm_funcs;
-
-        return 0;
-}
-
-int amd_powerplay_fini(void *handle)
-{
-        amd_pp_instance_fini(handle);
-
+        kfree(instance->smu_mgr);
+        instance->smu_mgr = NULL;
+        kfree(instance);
+        instance = NULL;
         return 0;
 }
 
@@ -985,33 +999,25 @@ int amd_powerplay_reset(void *handle)
         struct pem_event_data event_data = { {0} };
         int ret;
 
-        if (instance == NULL)
-                return -EINVAL;
-
-        eventmgr = instance->eventmgr;
-        if (!eventmgr || !eventmgr->pp_eventmgr_fini)
-                return -EINVAL;
-
-        eventmgr->pp_eventmgr_fini(eventmgr);
+        if (cgs_is_virtualization_enabled(instance->smu_mgr->device))
+                return PP_DPM_DISABLED;
 
-        ret = pp_sw_fini(handle);
-        if (ret)
+        ret = pp_check(instance);
+        if (ret != 0)
                 return ret;
 
-        kfree(instance->hwmgr->ps);
-
-        ret = pp_sw_init(handle);
+        ret = pp_hw_fini(handle);
         if (ret)
                 return ret;
 
-        if ((amdgpu_dpm == 0)
-                || cgs_is_virtualization_enabled(instance->smu_mgr->device))
-                return 0;
+        ret = hwmgr_hw_init(instance);
+        if (ret)
+                return PP_DPM_DISABLED;
 
-        hw_init_power_state_table(instance->hwmgr);
+        eventmgr = instance->eventmgr;
 
-        if (eventmgr == NULL || eventmgr->pp_eventmgr_init == NULL)
-                return -EINVAL;
+        if (eventmgr->pp_eventmgr_init == NULL)
+                return PP_DPM_DISABLED;
 
         ret = eventmgr->pp_eventmgr_init(eventmgr);
         if (ret)
@@ -1026,12 +1032,15 @@ int amd_powerplay_display_configuration_change(void *handle,
         const struct amd_pp_display_configuration *display_config)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        PP_CHECK((struct pp_instance *)handle);
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         phm_store_dal_configuration_data(hwmgr, display_config);
 
@@ -1042,15 +1051,18 @@ int amd_powerplay_get_display_power_level(void *handle,
                 struct amd_pp_simple_clock_info *output)
 {
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        PP_CHECK((struct pp_instance *)handle);
+        ret = pp_check(pp_handle);
 
-        if (output == NULL)
-                return -EINVAL;
+        if (ret != 0)
+                return ret;
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        hwmgr = pp_handle->hwmgr;
 
-        PP_CHECK_HW(hwmgr);
+        if (output == NULL)
+                return -EINVAL;
 
         return phm_get_dal_power_level(hwmgr, output);
 }
@@ -1058,18 +1070,18 @@ int amd_powerplay_get_display_power_level(void *handle,
 int amd_powerplay_get_current_clocks(void *handle,
                 struct amd_pp_clock_info *clocks)
 {
-        struct pp_hwmgr  *hwmgr;
         struct amd_pp_simple_clock_info simple_clocks;
         struct pp_clock_info hw_clocks;
+        struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        PP_CHECK((struct pp_instance *)handle);
-
-        if (clocks == NULL)
-                return -EINVAL;
+        ret = pp_check(pp_handle);
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+        if (ret != 0)
+                return ret;
 
-        PP_CHECK_HW(hwmgr);
+        hwmgr = pp_handle->hwmgr;
 
         phm_get_dal_power_level(hwmgr, &simple_clocks);
 
@@ -1105,18 +1117,20 @@ int amd_powerplay_get_current_clocks(void *handle,
 int amd_powerplay_get_clock_by_type(void *handle, enum amd_pp_clock_type type, struct amd_pp_clocks *clocks)
 {
         int result = -1;
+        struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        struct pp_hwmgr *hwmgr;
+        ret = pp_check(pp_handle);
 
-        PP_CHECK((struct pp_instance *)handle);
+        if (ret != 0)
+                return ret;
+
+        hwmgr = pp_handle->hwmgr;
 
         if (clocks == NULL)
                 return -EINVAL;
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
-
-        PP_CHECK_HW(hwmgr);
-
         result = phm_get_clock_by_type(hwmgr, type, clocks);
 
         return result;
@@ -1125,21 +1139,24 @@ int amd_powerplay_get_clock_by_type(void *handle, enum amd_pp_clock_type type, s
 int amd_powerplay_get_display_mode_validation_clocks(void *handle,
                 struct amd_pp_simple_clock_info *clocks)
 {
-        int result = -1;
         struct pp_hwmgr  *hwmgr;
+        struct pp_instance *pp_handle = (struct pp_instance *)handle;
+        int ret = 0;
 
-        PP_CHECK((struct pp_instance *)handle);
+        ret = pp_check(pp_handle);
 
-        if (clocks == NULL)
-                return -EINVAL;
+        if (ret != 0)
+                return ret;
+
+        hwmgr = pp_handle->hwmgr;
 
-        hwmgr = ((struct pp_instance *)handle)->hwmgr;
 
-        PP_CHECK_HW(hwmgr);
+        if (clocks == NULL)
+                return -EINVAL;
 
         if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DynamicPatchPowerState))
-                result = phm_get_max_high_clocks(hwmgr, clocks);
+                ret = phm_get_max_high_clocks(hwmgr, clocks);
 
-        return result;
+        return ret;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventinit.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventinit.c
index d5ec8ccbe97d..a3cd230d636d 100644
--- a/drivers/gpu/drm/amd/powerplay/eventmgr/eventinit.c
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventinit.c
@@ -151,7 +151,7 @@ static int thermal_interrupt_callback(void *private_data,
                                       unsigned src_id, const uint32_t *iv_entry)
 {
         /* TO DO hanle PEM_Event_ThermalNotification (struct pp_eventmgr *)private_data*/
-        printk("current thermal is out of range \n");
+        pr_info("current thermal is out of range \n");
         return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventmgr.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventmgr.c
index fb88e4e5d625..781e53dcf128 100644
--- a/drivers/gpu/drm/amd/powerplay/eventmgr/eventmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventmgr.c
@@ -60,9 +60,8 @@ static void pem_fini(struct pp_eventmgr *eventmgr)
         pem_handle_event(eventmgr, AMD_PP_EVENT_UNINITIALIZE, &event_data);
 }
 
-int eventmgr_init(struct pp_instance *handle)
+int eventmgr_early_init(struct pp_instance *handle)
 {
-        int result = 0;
         struct pp_eventmgr *eventmgr;
 
         if (handle == NULL)
@@ -79,12 +78,6 @@ int eventmgr_init(struct pp_instance *handle)
         eventmgr->pp_eventmgr_init = pem_init;
         eventmgr->pp_eventmgr_fini = pem_fini;
 
-        return result;
-}
-
-int eventmgr_fini(struct pp_eventmgr *eventmgr)
-{
-        kfree(eventmgr);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c
index ec36c0e28388..e04216ec7ee1 100644
--- a/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c
@@ -38,10 +38,13 @@
 int pem_task_update_allowed_performance_levels(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
 {
 
+        if (eventmgr == NULL || eventmgr->hwmgr == NULL)
+                return -EINVAL;
+
         if (pem_is_hw_access_blocked(eventmgr))
                 return 0;
 
-        phm_force_dpm_levels(eventmgr->hwmgr, AMD_DPM_FORCED_LEVEL_AUTO);
+        phm_force_dpm_levels(eventmgr->hwmgr, eventmgr->hwmgr->dpm_level);
 
         return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c
index 6bb79c94cb9f..3eccac735db3 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c
@@ -240,10 +240,16 @@ int cz_dpm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
 
 static const struct phm_master_table_item cz_enable_clock_power_gatings_list[] = {
         /*we don't need an exit table here, because there is only D3 cold on Kv*/
-        { phm_cf_want_uvd_power_gating, cz_tf_uvd_power_gating_initialize },
-        { phm_cf_want_vce_power_gating, cz_tf_vce_power_gating_initialize },
+        {
+          .isFunctionNeededInRuntimeTable = phm_cf_want_uvd_power_gating,
+          .tableFunction = cz_tf_uvd_power_gating_initialize
+        },
+        {
+          .isFunctionNeededInRuntimeTable = phm_cf_want_vce_power_gating,
+          .tableFunction = cz_tf_vce_power_gating_initialize
+        },
         /* to do { NULL, cz_tf_xdma_power_gating_enable }, */
-        { NULL, NULL }
+        { }
 };
 
 const struct phm_master_table_header cz_phm_enable_clock_power_gatings_master = {
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
index 0fb4e8c8f5e1..a4cde3d778b8 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
@@ -20,13 +20,13 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include "pp_debug.h"
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include "atom-types.h"
 #include "atombios.h"
 #include "processpptables.h"
-#include "pp_debug.h"
 #include "cgs_common.h"
 #include "smu/smu_8_0_d.h"
 #include "smu8_fusion.h"
@@ -38,7 +38,6 @@
 #include "cz_hwmgr.h"
 #include "power_state.h"
 #include "cz_clockpowergating.h"
-#include "pp_debug.h"
 
 #define ixSMUSVI_NB_CURRENTVID 0xD8230044
 #define CURRENT_NB_VID_MASK 0xff000000
@@ -288,7 +287,7 @@ static int cz_init_dynamic_state_adjustment_rule_settings(
                                         kzalloc(table_size, GFP_KERNEL);
 
         if (NULL == table_clk_vlt) {
-                printk(KERN_ERR "[ powerplay ] Can not allocate memory!\n");
+                pr_err("Can not allocate memory!\n");
                 return -ENOMEM;
         }
 
@@ -329,12 +328,12 @@ static int cz_get_system_info_data(struct pp_hwmgr *hwmgr)
                         &size, &frev, &crev);
 
         if (crev != 9) {
-                printk(KERN_ERR "[ powerplay ] Unsupported IGP table: %d %d\n", frev, crev);
+                pr_err("Unsupported IGP table: %d %d\n", frev, crev);
                 return -EINVAL;
         }
 
         if (info == NULL) {
-                printk(KERN_ERR "[ powerplay ] Could not retrieve the Integrated System Info Table!\n");
+                pr_err("Could not retrieve the Integrated System Info Table!\n");
                 return -EINVAL;
         }
 
@@ -361,7 +360,7 @@ static int cz_get_system_info_data(struct pp_hwmgr *hwmgr)
 
         if (cz_hwmgr->sys_info.htc_tmp_lmt <=
                         cz_hwmgr->sys_info.htc_hyst_lmt) {
-                printk(KERN_ERR "[ powerplay ] The htcTmpLmt should be larger than htcHystLmt.\n");
+                pr_err("The htcTmpLmt should be larger than htcHystLmt.\n");
                 return -EINVAL;
         }
 
@@ -723,7 +722,7 @@ static int cz_tf_update_sclk_limit(struct pp_hwmgr *hwmgr,
 
         clock = hwmgr->display_config.min_core_set_clock;
         if (clock == 0)
-                printk(KERN_INFO "[ powerplay ] min_core_set_clock not set\n");
+                pr_info("min_core_set_clock not set\n");
 
         if (cz_hwmgr->sclk_dpm.hard_min_clk != clock) {
                 cz_hwmgr->sclk_dpm.hard_min_clk = clock;
@@ -888,13 +887,13 @@ static int cz_tf_update_low_mem_pstate(struct pp_hwmgr *hwmgr,
 }
 
 static const struct phm_master_table_item cz_set_power_state_list[] = {
-        {NULL, cz_tf_update_sclk_limit},
-        {NULL, cz_tf_set_deep_sleep_sclk_threshold},
-        {NULL, cz_tf_set_watermark_threshold},
-        {NULL, cz_tf_set_enabled_levels},
-        {NULL, cz_tf_enable_nb_dpm},
-        {NULL, cz_tf_update_low_mem_pstate},
-        {NULL, NULL}
+        { .tableFunction = cz_tf_update_sclk_limit },
+        { .tableFunction = cz_tf_set_deep_sleep_sclk_threshold },
+        { .tableFunction = cz_tf_set_watermark_threshold },
+        { .tableFunction = cz_tf_set_enabled_levels },
+        { .tableFunction = cz_tf_enable_nb_dpm },
+        { .tableFunction = cz_tf_update_low_mem_pstate },
+        { }
 };
 
 static const struct phm_master_table_header cz_set_power_state_master = {
@@ -904,15 +903,15 @@ static const struct phm_master_table_header cz_set_power_state_master = {
 };
 
 static const struct phm_master_table_item cz_setup_asic_list[] = {
-        {NULL, cz_tf_reset_active_process_mask},
-        {NULL, cz_tf_upload_pptable_to_smu},
-        {NULL, cz_tf_init_sclk_limit},
-        {NULL, cz_tf_init_uvd_limit},
-        {NULL, cz_tf_init_vce_limit},
-        {NULL, cz_tf_init_acp_limit},
-        {NULL, cz_tf_init_power_gate_state},
-        {NULL, cz_tf_init_sclk_threshold},
-        {NULL, NULL}
+        { .tableFunction = cz_tf_reset_active_process_mask },
+        { .tableFunction = cz_tf_upload_pptable_to_smu },
+        { .tableFunction = cz_tf_init_sclk_limit },
+        { .tableFunction = cz_tf_init_uvd_limit },
+        { .tableFunction = cz_tf_init_vce_limit },
+        { .tableFunction = cz_tf_init_acp_limit },
+        { .tableFunction = cz_tf_init_power_gate_state },
+        { .tableFunction = cz_tf_init_sclk_threshold },
+        { }
 };
 
 static const struct phm_master_table_header cz_setup_asic_master = {
@@ -957,10 +956,10 @@ static int cz_tf_reset_cc6_data(struct pp_hwmgr *hwmgr,
 }
 
 static const struct phm_master_table_item cz_power_down_asic_list[] = {
-        {NULL, cz_tf_power_up_display_clock_sys_pll},
-        {NULL, cz_tf_clear_nb_dpm_flag},
-        {NULL, cz_tf_reset_cc6_data},
-        {NULL, NULL}
+        { .tableFunction = cz_tf_power_up_display_clock_sys_pll },
+        { .tableFunction = cz_tf_clear_nb_dpm_flag },
+        { .tableFunction = cz_tf_reset_cc6_data },
+        { }
 };
 
 static const struct phm_master_table_header cz_power_down_asic_master = {
@@ -1068,8 +1067,8 @@ static int cz_tf_check_for_dpm_enabled(struct pp_hwmgr *hwmgr,
 }
 
 static const struct phm_master_table_item cz_disable_dpm_list[] = {
-        { NULL, cz_tf_check_for_dpm_enabled},
-        {NULL, NULL},
+        { .tableFunction = cz_tf_check_for_dpm_enabled },
+        { },
 };
 
 
@@ -1080,13 +1079,13 @@ static const struct phm_master_table_header cz_disable_dpm_master = {
 };
 
 static const struct phm_master_table_item cz_enable_dpm_list[] = {
-        { NULL, cz_tf_check_for_dpm_disabled },
-        { NULL, cz_tf_program_voting_clients },
-        { NULL, cz_tf_start_dpm},
-        { NULL, cz_tf_program_bootup_state},
-        { NULL, cz_tf_enable_didt },
-        { NULL, cz_tf_reset_acp_boot_level },
-        {NULL, NULL},
+        { .tableFunction = cz_tf_check_for_dpm_disabled },
+        { .tableFunction = cz_tf_program_voting_clients },
+        { .tableFunction = cz_tf_start_dpm },
+        { .tableFunction = cz_tf_program_bootup_state },
+        { .tableFunction = cz_tf_enable_didt },
+        { .tableFunction = cz_tf_reset_acp_boot_level },
+        { },
 };
 
 static const struct phm_master_table_header cz_enable_dpm_master = {
@@ -1162,13 +1161,13 @@ static int cz_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 
         result = cz_initialize_dpm_defaults(hwmgr);
         if (result != 0) {
-                printk(KERN_ERR "[ powerplay ] cz_initialize_dpm_defaults failed\n");
+                pr_err("cz_initialize_dpm_defaults failed\n");
                 return result;
         }
 
         result = cz_get_system_info_data(hwmgr);
         if (result != 0) {
-                printk(KERN_ERR "[ powerplay ] cz_get_system_info_data failed\n");
+                pr_err("cz_get_system_info_data failed\n");
                 return result;
         }
 
@@ -1177,40 +1176,40 @@ static int cz_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
         result = phm_construct_table(hwmgr, &cz_setup_asic_master,
                                 &(hwmgr->setup_asic));
         if (result != 0) {
-                printk(KERN_ERR "[ powerplay ] Fail to construct setup ASIC\n");
+                pr_err("Fail to construct setup ASIC\n");
                 return result;
         }
 
         result = phm_construct_table(hwmgr, &cz_power_down_asic_master,
                                 &(hwmgr->power_down_asic));
         if (result != 0) {
-                printk(KERN_ERR "[ powerplay ] Fail to construct power down ASIC\n");
+                pr_err("Fail to construct power down ASIC\n");
                 return result;
         }
 
         result = phm_construct_table(hwmgr, &cz_disable_dpm_master,
                                 &(hwmgr->disable_dynamic_state_management));
         if (result != 0) {
-                printk(KERN_ERR "[ powerplay ] Fail to disable_dynamic_state\n");
+                pr_err("Fail to disable_dynamic_state\n");
                 return result;
         }
         result = phm_construct_table(hwmgr, &cz_enable_dpm_master,
                                 &(hwmgr->enable_dynamic_state_management));
         if (result != 0) {
-                printk(KERN_ERR "[ powerplay ] Fail to enable_dynamic_state\n");
+                pr_err("Fail to enable_dynamic_state\n");
                 return result;
         }
         result = phm_construct_table(hwmgr, &cz_set_power_state_master,
                                 &(hwmgr->set_power_state));
         if (result != 0) {
-                printk(KERN_ERR "[ powerplay ] Fail to construct set_power_state\n");
+                pr_err("Fail to construct set_power_state\n");
                 return result;
         }
         hwmgr->platform_descriptor.hardwareActivityPerformanceLevels =  CZ_MAX_HARDWARE_POWERLEVELS;
 
         result = phm_construct_table(hwmgr, &cz_phm_enable_clock_power_gatings_master, &(hwmgr->enable_clock_power_gatings));
         if (result != 0) {
-                printk(KERN_ERR "[ powerplay ] Fail to construct enable_clock_power_gatings\n");
+                pr_err("Fail to construct enable_clock_power_gatings\n");
                 return result;
         }
         return result;
@@ -1218,9 +1217,15 @@ static int cz_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 
 static int cz_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
 {
-        if (hwmgr != NULL && hwmgr->backend != NULL) {
+        if (hwmgr != NULL) {
+                phm_destroy_table(hwmgr, &(hwmgr->enable_clock_power_gatings));
+                phm_destroy_table(hwmgr, &(hwmgr->set_power_state));
+                phm_destroy_table(hwmgr, &(hwmgr->enable_dynamic_state_management));
+                phm_destroy_table(hwmgr, &(hwmgr->disable_dynamic_state_management));
+                phm_destroy_table(hwmgr, &(hwmgr->power_down_asic));
+                phm_destroy_table(hwmgr, &(hwmgr->setup_asic));
                 kfree(hwmgr->backend);
-                kfree(hwmgr);
+                hwmgr->backend = NULL;
         }
         return 0;
 }
@@ -1939,7 +1944,7 @@ static const struct pp_hwmgr_func cz_hwmgr_funcs = {
         .read_sensor = cz_read_sensor,
 };
 
-int cz_hwmgr_init(struct pp_hwmgr *hwmgr)
+int cz_init_function_pointers(struct pp_hwmgr *hwmgr)
 {
         hwmgr->hwmgr_func = &cz_hwmgr_funcs;
         hwmgr->pptable_func = &pptable_funcs;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.h
index c477f1cf3f23..508b422d6159 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.h
@@ -316,7 +316,6 @@ struct cz_hwmgr {
 
 struct pp_hwmgr;
 
-int cz_hwmgr_init(struct pp_hwmgr *hwmgr);
 int cz_dpm_powerdown_uvd(struct pp_hwmgr *hwmgr);
 int cz_dpm_powerup_uvd(struct pp_hwmgr *hwmgr);
 int cz_dpm_powerdown_vce(struct pp_hwmgr *hwmgr);
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
index 71822ae73a12..bc7d8bd7e7cb 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
@@ -35,7 +35,7 @@ static int phm_run_table(struct pp_hwmgr *hwmgr,
         phm_table_function *function;
 
         if (rt_table->function_list == NULL) {
-                pr_debug("[ powerplay ] this function not implement!\n");
+                pr_debug("this function not implement!\n");
                 return 0;
         }
 
@@ -63,14 +63,14 @@ int phm_dispatch_table(struct pp_hwmgr *hwmgr,
         void *temp_storage;
 
         if (hwmgr == NULL || rt_table == NULL) {
-                printk(KERN_ERR "[ powerplay ] Invalid Parameter!\n");
+                pr_err("Invalid Parameter!\n");
                 return -EINVAL;
         }
 
         if (0 != rt_table->storage_size) {
                 temp_storage = kzalloc(rt_table->storage_size, GFP_KERNEL);
                 if (temp_storage == NULL) {
-                        printk(KERN_ERR "[ powerplay ] Could not allocate table temporary storage\n");
+                        pr_err("Could not allocate table temporary storage\n");
                         return -ENOMEM;
                 }
         } else {
@@ -95,7 +95,7 @@ int phm_construct_table(struct pp_hwmgr *hwmgr,
         phm_table_function *rtf;
 
         if (hwmgr == NULL || master_table == NULL || rt_table == NULL) {
-                printk(KERN_ERR "[ powerplay ] Invalid Parameter!\n");
+                pr_err("Invalid Parameter!\n");
                 return -EINVAL;
         }
 
@@ -116,7 +116,7 @@ int phm_construct_table(struct pp_hwmgr *hwmgr,
         for (table_item = master_table->master_list;
                 NULL != table_item->tableFunction; table_item++) {
                 if ((rtf - run_time_list) > function_count) {
-                        printk(KERN_ERR "[ powerplay ] Check function results have changed\n");
+                        pr_err("Check function results have changed\n");
                         kfree(run_time_list);
                         return -EINVAL;
                 }
@@ -128,7 +128,7 @@ int phm_construct_table(struct pp_hwmgr *hwmgr,
         }
 
         if ((rtf - run_time_list) > function_count) {
-                printk(KERN_ERR "[ powerplay ] Check function results have changed\n");
+                pr_err("Check function results have changed\n");
                 kfree(run_time_list);
                 return -EINVAL;
         }
@@ -144,7 +144,7 @@ int phm_destroy_table(struct pp_hwmgr *hwmgr,
                       struct phm_runtime_table_header *rt_table)
 {
         if (hwmgr == NULL || rt_table == NULL) {
-                printk(KERN_ERR "[ powerplay ] Invalid Parameter\n");
+                pr_err("Invalid Parameter\n");
                 return -EINVAL;
         }
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
index c355a0f51663..0eb8e886bf35 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
@@ -20,11 +20,11 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include "pp_debug.h"
 #include <linux/errno.h>
 #include "hwmgr.h"
 #include "hardwaremanager.h"
 #include "power_state.h"
-#include "pp_debug.h"
 
 #define PHM_FUNC_CHECK(hw) \
         do {                                                    \
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
index b03606405a53..2ea9c0e78689 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
@@ -20,6 +20,8 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+
+#include "pp_debug.h"
 #include "linux/delay.h"
 #include <linux/types.h>
 #include <linux/kernel.h>
@@ -29,13 +31,12 @@
 #include "power_state.h"
 #include "hwmgr.h"
 #include "pppcielanes.h"
-#include "pp_debug.h"
 #include "ppatomctrl.h"
 #include "ppsmc.h"
 #include "pp_acpi.h"
 #include "amd_acpi.h"
 
-extern int cz_hwmgr_init(struct pp_hwmgr *hwmgr);
+extern int cz_init_function_pointers(struct pp_hwmgr *hwmgr);
 
 static int polaris_set_asic_special_caps(struct pp_hwmgr *hwmgr);
 static void hwmgr_init_default_caps(struct pp_hwmgr *hwmgr);
@@ -49,11 +50,11 @@ uint8_t convert_to_vid(uint16_t vddc)
         return (uint8_t) ((6200 - (vddc * VOLTAGE_SCALE)) / 25);
 }
 
-int hwmgr_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
+int hwmgr_early_init(struct pp_instance *handle)
 {
         struct pp_hwmgr *hwmgr;
 
-        if ((handle == NULL) || (pp_init == NULL))
+        if (handle == NULL)
                 return -EINVAL;
 
         hwmgr = kzalloc(sizeof(struct pp_hwmgr), GFP_KERNEL);
@@ -62,19 +63,20 @@ int hwmgr_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
 
         handle->hwmgr = hwmgr;
         hwmgr->smumgr = handle->smu_mgr;
-        hwmgr->device = pp_init->device;
-        hwmgr->chip_family = pp_init->chip_family;
-        hwmgr->chip_id = pp_init->chip_id;
+        hwmgr->device = handle->device;
+        hwmgr->chip_family = handle->chip_family;
+        hwmgr->chip_id = handle->chip_id;
+        hwmgr->feature_mask = handle->feature_mask;
         hwmgr->usec_timeout = AMD_MAX_USEC_TIMEOUT;
         hwmgr->power_source = PP_PowerSource_AC;
         hwmgr->pp_table_version = PP_TABLE_V1;
-
+        hwmgr->dpm_level = AMD_DPM_FORCED_LEVEL_AUTO;
         hwmgr_init_default_caps(hwmgr);
         hwmgr_set_user_specify_caps(hwmgr);
 
         switch (hwmgr->chip_family) {
         case AMDGPU_FAMILY_CZ:
-                cz_hwmgr_init(hwmgr);
+                cz_init_function_pointers(hwmgr);
                 break;
         case AMDGPU_FAMILY_VI:
                 switch (hwmgr->chip_id) {
@@ -102,7 +104,7 @@ int hwmgr_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
                 default:
                         return -EINVAL;
                 }
-                smu7_hwmgr_init(hwmgr);
+                smu7_init_function_pointers(hwmgr);
                 break;
         default:
                 return -EINVAL;
@@ -111,28 +113,7 @@ int hwmgr_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
         return 0;
 }
 
-int hwmgr_fini(struct pp_hwmgr *hwmgr)
-{
-        if (hwmgr == NULL || hwmgr->ps == NULL)
-                return -EINVAL;
-
-        /* do hwmgr finish*/
-        kfree(hwmgr->hardcode_pp_table);
-
-        kfree(hwmgr->backend);
-
-        kfree(hwmgr->start_thermal_controller.function_list);
-
-        kfree(hwmgr->set_temperature_range.function_list);
-
-        kfree(hwmgr->ps);
-        kfree(hwmgr->current_ps);
-        kfree(hwmgr->request_ps);
-        kfree(hwmgr);
-        return 0;
-}
-
-int hw_init_power_state_table(struct pp_hwmgr *hwmgr)
+static int hw_init_power_state_table(struct pp_hwmgr *hwmgr)
 {
         int result;
         unsigned int i;
@@ -156,12 +137,20 @@ int hw_init_power_state_table(struct pp_hwmgr *hwmgr)
                 return -ENOMEM;
 
         hwmgr->request_ps = kzalloc(size, GFP_KERNEL);
-        if (hwmgr->request_ps == NULL)
+        if (hwmgr->request_ps == NULL) {
+                kfree(hwmgr->ps);
+                hwmgr->ps = NULL;
                 return -ENOMEM;
+        }
 
         hwmgr->current_ps = kzalloc(size, GFP_KERNEL);
-        if (hwmgr->current_ps == NULL)
+        if (hwmgr->current_ps == NULL) {
+                kfree(hwmgr->request_ps);
+                kfree(hwmgr->ps);
+                hwmgr->request_ps = NULL;
+                hwmgr->ps = NULL;
                 return -ENOMEM;
+        }
 
         state = hwmgr->ps;
 
@@ -181,10 +170,77 @@ int hw_init_power_state_table(struct pp_hwmgr *hwmgr)
                 state = (struct pp_power_state *)((unsigned long)state + size);
         }
 
+        return 0;
+}
 
+static int hw_fini_power_state_table(struct pp_hwmgr *hwmgr)
+{
+        if (hwmgr == NULL)
+                return -EINVAL;
+
+        kfree(hwmgr->current_ps);
+        kfree(hwmgr->request_ps);
+        kfree(hwmgr->ps);
+        hwmgr->request_ps = NULL;
+        hwmgr->ps = NULL;
+        hwmgr->current_ps = NULL;
         return 0;
 }
 
+int hwmgr_hw_init(struct pp_instance *handle)
+{
+        struct pp_hwmgr *hwmgr;
+        int ret = 0;
+
+        if (handle == NULL)
+                return -EINVAL;
+
+        hwmgr = handle->hwmgr;
+
+        if (hwmgr->pptable_func == NULL ||
+            hwmgr->pptable_func->pptable_init == NULL ||
+            hwmgr->hwmgr_func->backend_init == NULL)
+                return -EINVAL;
+
+        ret = hwmgr->pptable_func->pptable_init(hwmgr);
+        if (ret)
+                goto err;
+
+        ret = hwmgr->hwmgr_func->backend_init(hwmgr);
+        if (ret)
+                goto err1;
+
+        ret = hw_init_power_state_table(hwmgr);
+        if (ret)
+                goto err2;
+        return 0;
+err2:
+        if (hwmgr->hwmgr_func->backend_fini)
+                hwmgr->hwmgr_func->backend_fini(hwmgr);
+err1:
+        if (hwmgr->pptable_func->pptable_fini)
+                hwmgr->pptable_func->pptable_fini(hwmgr);
+err:
+        pr_err("amdgpu: powerplay initialization failed\n");
+        return ret;
+}
+
+int hwmgr_hw_fini(struct pp_instance *handle)
+{
+        struct pp_hwmgr *hwmgr;
+
+        if (handle == NULL)
+                return -EINVAL;
+
+        hwmgr = handle->hwmgr;
+
+        if (hwmgr->hwmgr_func->backend_fini)
+                hwmgr->hwmgr_func->backend_fini(hwmgr);
+        if (hwmgr->pptable_func->pptable_fini)
+                hwmgr->pptable_func->pptable_fini(hwmgr);
+        return hw_fini_power_state_table(hwmgr);
+}
+
 
 /**
  * Returns once the part of the register indicated by the mask has
@@ -197,7 +253,7 @@ int phm_wait_on_register(struct pp_hwmgr *hwmgr, uint32_t index,
         uint32_t cur_value;
 
         if (hwmgr == NULL || hwmgr->device == NULL) {
-                printk(KERN_ERR "[ powerplay ] Invalid Hardware Manager!");
+                pr_err("Invalid Hardware Manager!");
                 return -EINVAL;
         }
 
@@ -227,7 +283,7 @@ void phm_wait_on_indirect_register(struct pp_hwmgr *hwmgr,
                                 uint32_t mask)
 {
         if (hwmgr == NULL || hwmgr->device == NULL) {
-                printk(KERN_ERR "[ powerplay ] Invalid Hardware Manager!");
+                pr_err("Invalid Hardware Manager!");
                 return;
         }
 
@@ -288,7 +344,7 @@ int phm_trim_voltage_table(struct pp_atomctrl_voltage_table *vol_table)
 
         memcpy(vol_table, table, sizeof(struct pp_atomctrl_voltage_table));
         kfree(table);
-
+        table = NULL;
         return 0;
 }
 
@@ -549,7 +605,7 @@ int phm_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr
         table_clk_vlt = kzalloc(table_size, GFP_KERNEL);
 
         if (NULL == table_clk_vlt) {
-                printk(KERN_ERR "[ powerplay ] Can not allocate space for vddc_dep_on_dal_pwrl! \n");
+                pr_err("Can not allocate space for vddc_dep_on_dal_pwrl! \n");
                 return -ENOMEM;
         } else {
                 table_clk_vlt->count = 4;
@@ -569,21 +625,6 @@ int phm_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr
         return 0;
 }
 
-int phm_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
-{
-        if (NULL != hwmgr->dyn_state.vddc_dep_on_dal_pwrl) {
-                kfree(hwmgr->dyn_state.vddc_dep_on_dal_pwrl);
-                hwmgr->dyn_state.vddc_dep_on_dal_pwrl = NULL;
-        }
-
-        if (NULL != hwmgr->backend) {
-                kfree(hwmgr->backend);
-                hwmgr->backend = NULL;
-        }
-
-        return 0;
-}
-
 uint32_t phm_get_lowest_enabled_level(struct pp_hwmgr *hwmgr, uint32_t mask)
 {
         uint32_t level = 0;
@@ -625,7 +666,7 @@ void phm_apply_dal_min_voltage_request(struct pp_hwmgr *hwmgr)
                         return;
                 }
         }
-        printk(KERN_ERR "DAL requested level can not"
+        pr_err("DAL requested level can not"
                         " found a available voltage in VDDC DPM Table \n");
 }
 
@@ -683,14 +724,14 @@ void hwmgr_init_default_caps(struct pp_hwmgr *hwmgr)
 
 int hwmgr_set_user_specify_caps(struct pp_hwmgr *hwmgr)
 {
-        if (amdgpu_pp_feature_mask & PP_SCLK_DEEP_SLEEP_MASK)
+        if (hwmgr->feature_mask & PP_SCLK_DEEP_SLEEP_MASK)
                 phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                         PHM_PlatformCaps_SclkDeepSleep);
         else
                 phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
                         PHM_PlatformCaps_SclkDeepSleep);
 
-        if (amdgpu_pp_feature_mask & PP_POWER_CONTAINMENT_MASK) {
+        if (hwmgr->feature_mask & PP_POWER_CONTAINMENT_MASK) {
                 phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                             PHM_PlatformCaps_PowerContainment);
                 phm_cap_set(hwmgr->platform_descriptor.platformCaps,
@@ -701,7 +742,6 @@ int hwmgr_set_user_specify_caps(struct pp_hwmgr *hwmgr)
                 phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
                         PHM_PlatformCaps_CAC);
         }
-        hwmgr->feature_mask = amdgpu_pp_feature_mask;
 
         return 0;
 }
@@ -727,17 +767,10 @@ int phm_get_voltage_evv_on_sclk(struct pp_hwmgr *hwmgr, uint8_t voltage_type,
 
 int polaris_set_asic_special_caps(struct pp_hwmgr *hwmgr)
 {
-        /* power tune caps Assume disabled */
+
         phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                                                 PHM_PlatformCaps_SQRamping);
         phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-                                                PHM_PlatformCaps_DBRamping);
-        phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-                                                PHM_PlatformCaps_TDRamping);
-        phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-                                                PHM_PlatformCaps_TCPRamping);
-
-        phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                                                 PHM_PlatformCaps_RegulatorHot);
 
         phm_cap_set(hwmgr->platform_descriptor.platformCaps,
@@ -746,9 +779,19 @@ int polaris_set_asic_special_caps(struct pp_hwmgr *hwmgr)
         phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                                 PHM_PlatformCaps_TablelessHardwareInterface);
 
-        if ((hwmgr->chip_id == CHIP_POLARIS11) || (hwmgr->chip_id == CHIP_POLARIS12))
+
+        if (hwmgr->chip_id != CHIP_POLARIS10)
                 phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                                         PHM_PlatformCaps_SPLLShutdownSupport);
+
+        if (hwmgr->chip_id != CHIP_POLARIS11) {
+                phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+                                                        PHM_PlatformCaps_DBRamping);
+                phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+                                                        PHM_PlatformCaps_TDRamping);
+                phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+                                                        PHM_PlatformCaps_TCPRamping);
+        }
         return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
index 0894527d932f..4b0a94cc995e 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
@@ -20,13 +20,13 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include "pp_debug.h"
 #include <linux/module.h>
 #include <linux/slab.h>
 
 #include "ppatomctrl.h"
 #include "atombios.h"
 #include "cgs_common.h"
-#include "pp_debug.h"
 #include "ppevvmath.h"
 
 #define MEM_ID_MASK           0xff000000
@@ -145,10 +145,10 @@ int atomctrl_initialize_mc_reg_table(
                                 GetIndexIntoMasterTable(DATA, VRAM_Info), &size, &frev, &crev);
 
         if (module_index >= vram_info->ucNumOfVRAMModule) {
-                printk(KERN_ERR "[ powerplay ] Invalid VramInfo table.");
+                pr_err("Invalid VramInfo table.");
                 result = -1;
         } else if (vram_info->sHeader.ucTableFormatRevision < 2) {
-                printk(KERN_ERR "[ powerplay ] Invalid VramInfo table.");
+                pr_err("Invalid VramInfo table.");
                 result = -1;
         }
 
@@ -688,7 +688,7 @@ int atomctrl_calculate_voltage_evv_on_sclk(
                 fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM7), 1000);
                 break;
         default:
-                printk(KERN_ERR "DPM Level not supported\n");
+                pr_err("DPM Level not supported\n");
                 fPowerDPMx = Convert_ULONG_ToFraction(1);
                 fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM0), 1000);
         }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
index c45bd2560468..84f01fd33aff 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
@@ -20,13 +20,13 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include "pp_debug.h"
 #include <linux/module.h>
 #include <linux/slab.h>
 
 #include "process_pptables_v1_0.h"
 #include "ppatomctrl.h"
 #include "atombios.h"
-#include "pp_debug.h"
 #include "hwmgr.h"
 #include "cgs_common.h"
 #include "pptable_v1_0.h"
@@ -535,7 +535,7 @@ static int get_pcie_table(
                 if ((uint32_t)atom_pcie_table->ucNumEntries <= pcie_count)
                         pcie_count = (uint32_t)atom_pcie_table->ucNumEntries;
                 else
-                        printk(KERN_ERR "[ powerplay ] Number of Pcie Entries exceed the number of SCLK Dpm Levels! \
+                        pr_err("Number of Pcie Entries exceed the number of SCLK Dpm Levels! \
                         Disregarding the excess entries... \n");
 
                 pcie_table->count = pcie_count;
@@ -577,7 +577,7 @@ static int get_pcie_table(
                 if ((uint32_t)atom_pcie_table->ucNumEntries <= pcie_count)
                         pcie_count = (uint32_t)atom_pcie_table->ucNumEntries;
                 else
-                        printk(KERN_ERR "[ powerplay ] Number of Pcie Entries exceed the number of SCLK Dpm Levels! \
+                        pr_err("Number of Pcie Entries exceed the number of SCLK Dpm Levels! \
                         Disregarding the excess entries... \n");
 
                 pcie_table->count = pcie_count;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
index a4e9cf429e62..ed6c934927fb 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
@@ -20,6 +20,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include "pp_debug.h"
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
@@ -27,7 +28,6 @@
 #include "processpptables.h"
 #include <atom-types.h>
 #include <atombios.h>
-#include "pp_debug.h"
 #include "pptable.h"
 #include "power_state.h"
 #include "hwmgr.h"
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h
index d52a28c343e3..c96ed9ed7eaf 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h
@@ -22,7 +22,7 @@
  */
 
 #ifndef _SMU7_CLOCK_POWER_GATING_H_
-#define _SMU7_CLOCK__POWER_GATING_H_
+#define _SMU7_CLOCK_POWER_GATING_H_
 
 #include "smu7_hwmgr.h"
 #include "pp_asicblocks.h"
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
index a74f60a575ae..0a6c833720df 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
@@ -20,13 +20,13 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include "pp_debug.h"
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/fb.h>
 #include <asm/div64.h>
 #include "linux/delay.h"
 #include "pp_acpi.h"
-#include "pp_debug.h"
 #include "ppatomctrl.h"
 #include "atombios.h"
 #include "pptable_v1_0.h"
@@ -40,6 +40,8 @@
 
 #include "hwmgr.h"
 #include "smu7_hwmgr.h"
+#include "smu7_smumgr.h"
+#include "smu_ucode_xfer_vi.h"
 #include "smu7_powertune.h"
 #include "smu7_dyn_defaults.h"
 #include "smu7_thermal.h"
@@ -88,6 +90,8 @@ enum DPM_EVENT_SRC {
 };
 
 static const unsigned long PhwVIslands_Magic = (unsigned long)(PHM_VIslands_Magic);
+static int smu7_force_clock_level(struct pp_hwmgr *hwmgr,
+                enum pp_clock_type type, uint32_t mask);
 
 static struct smu7_power_state *cast_phw_smu7_power_state(
                                   struct pp_hw_power_state *hw_ps)
@@ -994,7 +998,7 @@ static int smu7_start_dpm(struct pp_hwmgr *hwmgr)
                         SWRST_COMMAND_1, RESETLC, 0x0);
 
         if (smu7_enable_sclk_mclk_dpm(hwmgr)) {
-                printk(KERN_ERR "Failed to enable Sclk DPM and Mclk DPM!");
+                pr_err("Failed to enable Sclk DPM and Mclk DPM!");
                 return -EINVAL;
         }
 
@@ -1079,7 +1083,7 @@ static void smu7_set_dpm_event_sources(struct pp_hwmgr *hwmgr, uint32_t sources)
 
         switch (sources) {
         default:
-                printk(KERN_ERR "Unknown throttling event sources.");
+                pr_err("Unknown throttling event sources.");
                 /* fall through */
         case 0:
                 protection = false;
@@ -1292,6 +1296,10 @@ int smu7_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
         PP_ASSERT_WITH_CODE((tmp_result == 0),
                         "Failed to disable SMC CAC!", result = tmp_result);
 
+        tmp_result = smu7_disable_didt_config(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to disable DIDT!", result = tmp_result);
+
         PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
                         CG_SPLL_SPREAD_SPECTRUM, SSEN, 0);
         PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
@@ -1499,7 +1507,7 @@ static int smu7_get_evv_voltages(struct pp_hwmgr *hwmgr)
                                                 data->vddcgfx_leakage.count++;
                                         }
                                 } else {
-                                        printk("Error retrieving EVV voltage value!\n");
+                                        pr_info("Error retrieving EVV voltage value!\n");
                                 }
                         }
                 } else {
@@ -1527,7 +1535,7 @@ static int smu7_get_evv_voltages(struct pp_hwmgr *hwmgr)
                                         if (vddc >= 2000 || vddc == 0)
                                                 return -EINVAL;
                                 } else {
-                                        printk(KERN_WARNING "failed to retrieving EVV voltage!\n");
+                                        pr_warning("failed to retrieving EVV voltage!\n");
                                         continue;
                                 }
 
@@ -1567,7 +1575,7 @@ static void smu7_patch_ppt_v1_with_vdd_leakage(struct pp_hwmgr *hwmgr,
         }
 
         if (*voltage > ATOM_VIRTUAL_VOLTAGE_ID0)
-                printk(KERN_ERR "Voltage value looks like a Leakage ID but it's not patched \n");
+                pr_err("Voltage value looks like a Leakage ID but it's not patched \n");
 }
 
 /**
@@ -2032,7 +2040,7 @@ static void smu7_patch_ppt_v0_with_vdd_leakage(struct pp_hwmgr *hwmgr,
         }
 
         if (*voltage > ATOM_VIRTUAL_VOLTAGE_ID0)
-                printk(KERN_ERR "Voltage value looks like a Leakage ID but it's not patched \n");
+                pr_err("Voltage value looks like a Leakage ID but it's not patched \n");
 }
 
 
@@ -2267,6 +2275,21 @@ static int smu7_set_private_data_based_on_pptable_v0(struct pp_hwmgr *hwmgr)
         return 0;
 }
 
+static int smu7_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
+{
+        if (NULL != hwmgr->dyn_state.vddc_dep_on_dal_pwrl) {
+                kfree(hwmgr->dyn_state.vddc_dep_on_dal_pwrl);
+                hwmgr->dyn_state.vddc_dep_on_dal_pwrl = NULL;
+        }
+        pp_smu7_thermal_fini(hwmgr);
+        if (NULL != hwmgr->backend) {
+                kfree(hwmgr->backend);
+                hwmgr->backend = NULL;
+        }
+
+        return 0;
+}
+
 static int smu7_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 {
         struct smu7_hwmgr *data;
@@ -2277,6 +2300,7 @@ static int smu7_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
                 return -ENOMEM;
 
         hwmgr->backend = data;
+        pp_smu7_thermal_initialize(hwmgr);
 
         smu7_patch_voltage_workaround(hwmgr);
         smu7_init_dpm_defaults(hwmgr);
@@ -2285,7 +2309,7 @@ static int smu7_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
         result = smu7_get_evv_voltages(hwmgr);
 
         if (result) {
-                printk("Get EVV Voltage Failed.  Abort Driver loading!\n");
+                pr_info("Get EVV Voltage Failed.  Abort Driver loading!\n");
                 return -EINVAL;
         }
 
@@ -2334,7 +2358,7 @@ static int smu7_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
                 smu7_thermal_parameter_init(hwmgr);
         } else {
                 /* Ignore return value in here, we are cleaning up a mess. */
-                phm_hwmgr_backend_fini(hwmgr);
+                smu7_hwmgr_backend_fini(hwmgr);
         }
 
         return 0;
@@ -2466,36 +2490,155 @@ static int smu7_force_dpm_lowest(struct pp_hwmgr *hwmgr)
         }
 
         return 0;
+}
 
+static int smu7_get_profiling_clk(struct pp_hwmgr *hwmgr, enum amd_dpm_forced_level level,
+                                uint32_t *sclk_mask, uint32_t *mclk_mask, uint32_t *pcie_mask)
+{
+        uint32_t percentage;
+        struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+        struct smu7_dpm_table *golden_dpm_table = &data->golden_dpm_table;
+        int32_t tmp_mclk;
+        int32_t tmp_sclk;
+        int32_t count;
+
+        if (golden_dpm_table->mclk_table.count < 1)
+                return -EINVAL;
+
+        percentage = 100 * golden_dpm_table->sclk_table.dpm_levels[golden_dpm_table->sclk_table.count - 1].value /
+                        golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count - 1].value;
+
+        if (golden_dpm_table->mclk_table.count == 1) {
+                percentage = 70;
+                tmp_mclk = golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count - 1].value;
+                *mclk_mask = golden_dpm_table->mclk_table.count - 1;
+        } else {
+                tmp_mclk = golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count - 2].value;
+                *mclk_mask = golden_dpm_table->mclk_table.count - 2;
+        }
+
+        tmp_sclk = tmp_mclk * percentage / 100;
+
+        if (hwmgr->pp_table_version == PP_TABLE_V0) {
+                for (count = hwmgr->dyn_state.vddc_dependency_on_sclk->count-1;
+                        count >= 0; count--) {
+                        if (tmp_sclk >= hwmgr->dyn_state.vddc_dependency_on_sclk->entries[count].clk) {
+                                tmp_sclk = hwmgr->dyn_state.vddc_dependency_on_sclk->entries[count].clk;
+                                *sclk_mask = count;
+                                break;
+                        }
+                }
+                if (count < 0 || level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK)
+                        *sclk_mask = 0;
+
+                if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK)
+                        *sclk_mask = hwmgr->dyn_state.vddc_dependency_on_sclk->count-1;
+        } else if (hwmgr->pp_table_version == PP_TABLE_V1) {
+                struct phm_ppt_v1_information *table_info =
+                                (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+                for (count = table_info->vdd_dep_on_sclk->count-1; count >= 0; count--) {
+                        if (tmp_sclk >= table_info->vdd_dep_on_sclk->entries[count].clk) {
+                                tmp_sclk = table_info->vdd_dep_on_sclk->entries[count].clk;
+                                *sclk_mask = count;
+                                break;
+                        }
+                }
+                if (count < 0 || level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK)
+                        *sclk_mask = 0;
+
+                if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK)
+                        *sclk_mask = table_info->vdd_dep_on_sclk->count - 1;
+        }
+
+        if (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK)
+                *mclk_mask = 0;
+        else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK)
+                *mclk_mask = golden_dpm_table->mclk_table.count - 1;
+
+        *pcie_mask = data->dpm_table.pcie_speed_table.count - 1;
+        return 0;
 }
+
 static int smu7_force_dpm_level(struct pp_hwmgr *hwmgr,
                                 enum amd_dpm_forced_level level)
 {
         int ret = 0;
+        uint32_t sclk_mask = 0;
+        uint32_t mclk_mask = 0;
+        uint32_t pcie_mask = 0;
+        uint32_t profile_mode_mask = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD |
+                                        AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK |
+                                        AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK |
+                                        AMD_DPM_FORCED_LEVEL_PROFILE_PEAK;
+
+        if (level == hwmgr->dpm_level)
+                return ret;
+
+        if (!(hwmgr->dpm_level & profile_mode_mask)) {
+                /* enter profile mode, save current level, disable gfx cg*/
+                if (level & profile_mode_mask) {
+                        hwmgr->saved_dpm_level = hwmgr->dpm_level;
+                        cgs_set_clockgating_state(hwmgr->device,
+                                                AMD_IP_BLOCK_TYPE_GFX,
+                                                AMD_CG_STATE_UNGATE);
+                }
+        } else {
+                /* exit profile mode, restore level, enable gfx cg*/
+                if (!(level & profile_mode_mask)) {
+                        if (level == AMD_DPM_FORCED_LEVEL_PROFILE_EXIT)
+                                level = hwmgr->saved_dpm_level;
+                        cgs_set_clockgating_state(hwmgr->device,
+                                        AMD_IP_BLOCK_TYPE_GFX,
+                                        AMD_CG_STATE_GATE);
+                }
+        }
 
         switch (level) {
         case AMD_DPM_FORCED_LEVEL_HIGH:
                 ret = smu7_force_dpm_highest(hwmgr);
                 if (ret)
                         return ret;
+                hwmgr->dpm_level = level;
                 break;
         case AMD_DPM_FORCED_LEVEL_LOW:
                 ret = smu7_force_dpm_lowest(hwmgr);
                 if (ret)
                         return ret;
+                hwmgr->dpm_level = level;
                 break;
         case AMD_DPM_FORCED_LEVEL_AUTO:
                 ret = smu7_unforce_dpm_levels(hwmgr);
                 if (ret)
                         return ret;
+                hwmgr->dpm_level = level;
+                break;
+        case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
+        case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
+        case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
+        case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
+                ret = smu7_get_profiling_clk(hwmgr, level, &sclk_mask, &mclk_mask, &pcie_mask);
+                if (ret)
+                        return ret;
+                hwmgr->dpm_level = level;
+                smu7_force_clock_level(hwmgr, PP_SCLK, 1<<sclk_mask);
+                smu7_force_clock_level(hwmgr, PP_MCLK, 1<<mclk_mask);
+                smu7_force_clock_level(hwmgr, PP_PCIE, 1<<pcie_mask);
+                break;
+        case AMD_DPM_FORCED_LEVEL_MANUAL:
+                hwmgr->dpm_level = level;
                 break;
+        case AMD_DPM_FORCED_LEVEL_PROFILE_EXIT:
         default:
                 break;
         }
 
-        hwmgr->dpm_level = level;
+        if (level & (AMD_DPM_FORCED_LEVEL_PROFILE_PEAK | AMD_DPM_FORCED_LEVEL_HIGH))
+                smu7_fan_ctrl_set_fan_speed_percent(hwmgr, 100);
+        else
+                smu7_fan_ctrl_reset_fan_speed_to_default(hwmgr);
 
-        return ret;
+        return 0;
 }
 
 static int smu7_get_power_state_size(struct pp_hwmgr *hwmgr)
@@ -2898,11 +3041,11 @@ static int smu7_get_pp_table_entry_v1(struct pp_hwmgr *hwmgr,
         if (dep_mclk_table != NULL && dep_mclk_table->count == 1) {
                 if (dep_mclk_table->entries[0].clk !=
                                 data->vbios_boot_state.mclk_bootup_value)
-                        printk(KERN_ERR "Single MCLK entry VDDCI/MCLK dependency table "
+                        pr_err("Single MCLK entry VDDCI/MCLK dependency table "
                                         "does not match VBIOS boot MCLK level");
                 if (dep_mclk_table->entries[0].vddci !=
                                 data->vbios_boot_state.vddci_bootup_value)
-                        printk(KERN_ERR "Single VDDCI entry VDDCI/MCLK dependency table "
+                        pr_err("Single VDDCI entry VDDCI/MCLK dependency table "
                                         "does not match VBIOS boot VDDCI level");
         }
 
@@ -3046,11 +3189,11 @@ static int smu7_get_pp_table_entry_v0(struct pp_hwmgr *hwmgr,
         if (dep_mclk_table != NULL && dep_mclk_table->count == 1) {
                 if (dep_mclk_table->entries[0].clk !=
                                 data->vbios_boot_state.mclk_bootup_value)
-                        printk(KERN_ERR "Single MCLK entry VDDCI/MCLK dependency table "
+                        pr_err("Single MCLK entry VDDCI/MCLK dependency table "
                                         "does not match VBIOS boot MCLK level");
                 if (dep_mclk_table->entries[0].v !=
                                 data->vbios_boot_state.vddci_bootup_value)
-                        printk(KERN_ERR "Single VDDCI entry VDDCI/MCLK dependency table "
+                        pr_err("Single VDDCI entry VDDCI/MCLK dependency table "
                                         "does not match VBIOS boot VDDCI level");
         }
 
@@ -3590,9 +3733,9 @@ static int smu7_notify_link_speed_change_after_state_change(
 
                 if (acpi_pcie_perf_request(hwmgr->device, request, false)) {
                         if (PP_PCIEGen2 == target_link_speed)
-                                printk("PSPP request to switch to Gen2 from Gen3 Failed!");
+                                pr_info("PSPP request to switch to Gen2 from Gen3 Failed!");
                         else
-                                printk("PSPP request to switch to Gen1 from Gen2 Failed!");
+                                pr_info("PSPP request to switch to Gen1 from Gen2 Failed!");
                 }
         }
 
@@ -4029,7 +4172,9 @@ static int smu7_force_clock_level(struct pp_hwmgr *hwmgr,
 {
         struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
-        if (hwmgr->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL)
+        if (hwmgr->dpm_level & (AMD_DPM_FORCED_LEVEL_AUTO |
+                                AMD_DPM_FORCED_LEVEL_LOW |
+                                AMD_DPM_FORCED_LEVEL_HIGH))
                 return -EINVAL;
 
         switch (type) {
@@ -4324,9 +4469,35 @@ static int smu7_get_clock_by_type(struct pp_hwmgr *hwmgr, enum amd_pp_clock_type
         return 0;
 }
 
+static int smu7_request_firmware(struct pp_hwmgr *hwmgr)
+{
+        int ret;
+        struct cgs_firmware_info info = {0};
+
+        ret = cgs_get_firmware_info(hwmgr->device,
+                                    smu7_convert_fw_type_to_cgs(UCODE_ID_SMU),
+                                    &info);
+        if (ret || !info.kptr)
+                return -EINVAL;
+
+        return 0;
+}
+
+static int smu7_release_firmware(struct pp_hwmgr *hwmgr)
+{
+        int ret;
+
+        ret = cgs_rel_firmware(hwmgr->device,
+                               smu7_convert_fw_type_to_cgs(UCODE_ID_SMU));
+        if (ret)
+                return -EINVAL;
+
+        return 0;
+}
+
 static const struct pp_hwmgr_func smu7_hwmgr_funcs = {
         .backend_init = &smu7_hwmgr_backend_init,
-        .backend_fini = &phm_hwmgr_backend_fini,
+        .backend_fini = &smu7_hwmgr_backend_fini,
         .asic_setup = &smu7_setup_asic_task,
         .dynamic_state_management_enable = &smu7_enable_dpm_tasks,
         .apply_state_adjust_rules = smu7_apply_state_adjust_rules,
@@ -4371,6 +4542,8 @@ static const struct pp_hwmgr_func smu7_hwmgr_funcs = {
         .get_clock_by_type = smu7_get_clock_by_type,
         .read_sensor = smu7_read_sensor,
         .dynamic_state_management_disable = smu7_disable_dpm_tasks,
+        .request_firmware = smu7_request_firmware,
+        .release_firmware = smu7_release_firmware,
 };
 
 uint8_t smu7_get_sleep_divider_id_from_clock(uint32_t clock,
@@ -4390,7 +4563,7 @@ uint8_t smu7_get_sleep_divider_id_from_clock(uint32_t clock,
         return i;
 }
 
-int smu7_hwmgr_init(struct pp_hwmgr *hwmgr)
+int smu7_init_function_pointers(struct pp_hwmgr *hwmgr)
 {
         int ret = 0;
 
@@ -4400,7 +4573,6 @@ int smu7_hwmgr_init(struct pp_hwmgr *hwmgr)
         else if (hwmgr->pp_table_version == PP_TABLE_V1)
                 hwmgr->pptable_func = &pptable_v1_0_funcs;
 
-        pp_smu7_thermal_initialize(hwmgr);
         return ret;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
index 6cd1287a7a8f..3341c0fbd069 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
@@ -20,17 +20,19 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include "pp_debug.h"
 #include "hwmgr.h"
 #include "smumgr.h"
 #include "smu7_hwmgr.h"
 #include "smu7_powertune.h"
-#include "pp_debug.h"
 #include "smu7_common.h"
 
 #define VOLTAGE_SCALE  4
 
 static uint32_t DIDTBlock_Info = SQ_IR_MASK | TCP_IR_MASK | TD_PCC_MASK;
 
+static uint32_t Polaris11_DIDTBlock_Info = SQ_PCC_MASK | TCP_IR_MASK | TD_PCC_MASK;
+
 static const struct gpu_pt_config_reg GCCACConfig_Polaris10[] = {
 /* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  *      Offset                             Mask                                                Shift                                               Value       Type
@@ -261,9 +263,9 @@ static const struct gpu_pt_config_reg DIDTConfig_Polaris11[] = {
         {   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__UNUSED_0_MASK,                    DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0xffff,     GPU_CONFIGREG_DIDT_IND },
 
-        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3853,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3fff,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_0_MASK,                       DIDT_SQ_CTRL2__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
-        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x005a,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x000f,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_1_MASK,                       DIDT_SQ_CTRL2__UNUSED_1__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_2_MASK,                       DIDT_SQ_CTRL2__UNUSED_2__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
@@ -271,12 +273,12 @@ static const struct gpu_pt_config_reg DIDTConfig_Polaris11[] = {
         {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
-        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x0ebb,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__UNUSED_0_MASK,                  DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
 
-        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
-        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x3853,     GPU_CONFIGREG_DIDT_IND },
-        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x3153,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x0dde,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x0dde,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
 
         {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
@@ -284,8 +286,8 @@ static const struct gpu_pt_config_reg DIDTConfig_Polaris11[] = {
         {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__PHASE_OFFSET_MASK,                   DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
-        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
-        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0008,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0008,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__UNUSED_0_MASK,                       DIDT_SQ_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
 
         {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT,                  0x000a,     GPU_CONFIGREG_DIDT_IND },
@@ -373,55 +375,160 @@ static const struct gpu_pt_config_reg DIDTConfig_Polaris11[] = {
         {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
         {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__UNUSED_0_MASK,                       DIDT_TCP_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+        {   0xFFFFFFFF  }
+};
+
+static const struct gpu_pt_config_reg DIDTConfig_Polaris12[] = {
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                Shift                                               Value       Type
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT,                  0x0073,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT,                  0x00ab,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0084,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT,                  0x005a,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0067,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT,                  0x0084,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0027,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0046,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT,                 0x00aa,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MIN_POWER_MASK,                      DIDT_SQ_CTRL1__MIN_POWER__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MAX_POWER_MASK,                      DIDT_SQ_CTRL1__MAX_POWER__SHIFT,                    0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__UNUSED_0_MASK,                    DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0x00ff,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3fff,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_0_MASK,                       DIDT_TD_CTRL2__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x000f,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_1_MASK,                       DIDT_TD_CTRL2__UNUSED_1__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_2_MASK,                       DIDT_TD_CTRL2__UNUSED_2__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__UNUSED_0_MASK,                  DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x0dde,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x0dde,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0008,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0008,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__UNUSED_0_MASK,                       DIDT_TD_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT,                 0x0004,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT,                 0x0037,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT,                 0x00ff,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT,                 0x0054,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MIN_POWER_MASK,                     DIDT_TCP_CTRL1__MIN_POWER__SHIFT,                   0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MAX_POWER_MASK,                     DIDT_TCP_CTRL1__MAX_POWER__SHIFT,                   0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__UNUSED_0_MASK,                   DIDT_TCP_CTRL_OCP__UNUSED_0__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__OCP_MAX_POWER_MASK,              DIDT_TCP_CTRL_OCP__OCP_MAX_POWER__SHIFT,            0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK,               DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT,             0x3dde,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_0_MASK,                      DIDT_TCP_CTRL2__UNUSED_0__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,      DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,    0x0032,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_1_MASK,                      DIDT_TCP_CTRL2__UNUSED_1__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,      DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,    0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_2_MASK,                      DIDT_TCP_CTRL2__UNUSED_2__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,   DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,  DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__UNUSED_0_MASK,                 DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,      DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,    0x3dde,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,    0x3dde,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK,                DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT,              0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__UNUSED_0_MASK,                       DIDT_TCP_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
         {   0xFFFFFFFF  }
 };
 
 
 static int smu7_enable_didt(struct pp_hwmgr *hwmgr, const bool enable)
 {
-
         uint32_t en = enable ? 1 : 0;
+        uint32_t block_en = 0;
         int32_t result = 0;
+        uint32_t didt_block;
         uint32_t data;
 
-        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping)) {
-                data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_CTRL0);
-                data &= ~DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK;
-                data |= ((en << DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK);
-                cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_CTRL0, data);
-                DIDTBlock_Info &= ~SQ_Enable_MASK;
-                DIDTBlock_Info |= en << SQ_Enable_SHIFT;
-        }
+        if (hwmgr->chip_id == CHIP_POLARIS11)
+                didt_block = Polaris11_DIDTBlock_Info;
+        else
+                didt_block = DIDTBlock_Info;
 
-        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping)) {
-                data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_CTRL0);
-                data &= ~DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK;
-                data |= ((en << DIDT_DB_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK);
-                cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_CTRL0, data);
-                DIDTBlock_Info &= ~DB_Enable_MASK;
-                DIDTBlock_Info |= en << DB_Enable_SHIFT;
-        }
+        block_en = phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping) ? en : 0;
 
-        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping)) {
-                data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_CTRL0);
-                data &= ~DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK;
-                data |= ((en << DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK);
-                cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_CTRL0, data);
-                DIDTBlock_Info &= ~TD_Enable_MASK;
-                DIDTBlock_Info |= en << TD_Enable_SHIFT;
-        }
+        data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_CTRL0);
+        data &= ~DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK;
+        data |= ((block_en << DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_CTRL0, data);
+        didt_block &= ~SQ_Enable_MASK;
+        didt_block |= block_en << SQ_Enable_SHIFT;
+
+        block_en = phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping) ? en : 0;
+
+        data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_CTRL0);
+        data &= ~DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK;
+        data |= ((block_en << DIDT_DB_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_CTRL0, data);
+        didt_block &= ~DB_Enable_MASK;
+        didt_block |= block_en << DB_Enable_SHIFT;
+
+        block_en = phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping) ? en : 0;
+        data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_CTRL0);
+        data &= ~DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK;
+        data |= ((block_en << DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_CTRL0, data);
+        didt_block &= ~TD_Enable_MASK;
+        didt_block |= block_en << TD_Enable_SHIFT;
+
+        block_en = phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping) ? en : 0;
+
+        data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_CTRL0);
+        data &= ~DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK;
+        data |= ((block_en << DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_CTRL0, data);
+        didt_block &= ~TCP_Enable_MASK;
+        didt_block |= block_en << TCP_Enable_SHIFT;
 
-        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) {
-                data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_CTRL0);
-                data &= ~DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK;
-                data |= ((en << DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK);
-                cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_CTRL0, data);
-                DIDTBlock_Info &= ~TCP_Enable_MASK;
-                DIDTBlock_Info |= en << TCP_Enable_SHIFT;
-        }
 
         if (enable)
-                result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_Didt_Block_Function, DIDTBlock_Info);
+                result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_Didt_Block_Function, didt_block);
 
         return result;
 }
@@ -498,7 +605,6 @@ int smu7_enable_didt_config(struct pp_hwmgr *hwmgr)
         sys_info.info_id = CGS_SYSTEM_INFO_GFX_SE_INFO;
         result = cgs_query_system_info(hwmgr->device, &sys_info);
 
-
         if (result == 0)
                 num_se = sys_info.value;
 
@@ -507,7 +613,7 @@ int smu7_enable_didt_config(struct pp_hwmgr *hwmgr)
                 phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping) ||
                 phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) {
 
-                /* TO DO Pre DIDT disable clock gating */
+                cgs_enter_safe_mode(hwmgr->device, true);
                 value = 0;
                 value2 = cgs_read_register(hwmgr->device, mmGRBM_GFX_INDEX);
                 for (count = 0; count < num_se; count++) {
@@ -521,11 +627,16 @@ int smu7_enable_didt_config(struct pp_hwmgr *hwmgr)
                                 PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
                                 result = smu7_program_pt_config_registers(hwmgr, DIDTConfig_Polaris10);
                                 PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
-                        } else if ((hwmgr->chip_id == CHIP_POLARIS11) || (hwmgr->chip_id == CHIP_POLARIS12)) {
+                        } else if (hwmgr->chip_id == CHIP_POLARIS11) {
                                 result = smu7_program_pt_config_registers(hwmgr, GCCACConfig_Polaris11);
                                 PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
                                 result = smu7_program_pt_config_registers(hwmgr, DIDTConfig_Polaris11);
                                 PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+                        } else if (hwmgr->chip_id == CHIP_POLARIS12) {
+                                result = smu7_program_pt_config_registers(hwmgr, GCCACConfig_Polaris11);
+                                PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+                                result = smu7_program_pt_config_registers(hwmgr, DIDTConfig_Polaris12);
+                                PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
                         }
                 }
                 cgs_write_register(hwmgr->device, mmGRBM_GFX_INDEX, value2);
@@ -533,7 +644,13 @@ int smu7_enable_didt_config(struct pp_hwmgr *hwmgr)
                 result = smu7_enable_didt(hwmgr, true);
                 PP_ASSERT_WITH_CODE((result == 0), "EnableDiDt failed.", return result);
 
-                /* TO DO Post DIDT enable clock gating */
+                if (hwmgr->chip_id == CHIP_POLARIS11) {
+                        result = smum_send_msg_to_smc(hwmgr->smumgr,
+                                                (uint16_t)(PPSMC_MSG_EnableDpmDidt));
+                        PP_ASSERT_WITH_CODE((0 == result),
+                                        "Failed to enable DPM DIDT.", return result);
+                }
+                cgs_enter_safe_mode(hwmgr->device, false);
         }
 
         return 0;
@@ -547,11 +664,20 @@ int smu7_disable_didt_config(struct pp_hwmgr *hwmgr)
                 phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping) ||
                 phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping) ||
                 phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) {
-                /* TO DO Pre DIDT disable clock gating */
+
+                cgs_enter_safe_mode(hwmgr->device, true);
 
                 result = smu7_enable_didt(hwmgr, false);
-                PP_ASSERT_WITH_CODE((result == 0), "Post DIDT enable clock gating failed.", return result);
-                /* TO DO Post DIDT enable clock gating */
+                PP_ASSERT_WITH_CODE((result == 0),
+                                "Post DIDT enable clock gating failed.",
+                                return result);
+                if (hwmgr->chip_id == CHIP_POLARIS11) {
+                        result = smum_send_msg_to_smc(hwmgr->smumgr,
+                                                (uint16_t)(PPSMC_MSG_DisableDpmDidt));
+                        PP_ASSERT_WITH_CODE((0 == result),
+                                        "Failed to disable DPM DIDT.", return result);
+                }
+                cgs_enter_safe_mode(hwmgr->device, false);
         }
 
         return 0;
@@ -651,7 +777,7 @@ int smu7_enable_power_containment(struct pp_hwmgr *hwmgr)
                                                 POWERCONTAINMENT_FEATURE_PkgPwrLimit;
 
                                 if (smu7_set_power_limit(hwmgr, default_limit))
-                                        printk(KERN_ERR "Failed to set Default Power Limit in SMC!");
+                                        pr_err("Failed to set Default Power Limit in SMC!");
                         }
                 }
         }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.c
index 29d0319b22e6..436ca5ce8248 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.c
@@ -506,18 +506,18 @@ static int tf_smu7_thermal_disable_alert(struct pp_hwmgr *hwmgr,
 
 static const struct phm_master_table_item
 phm_thermal_start_thermal_controller_master_list[] = {
-        {NULL, tf_smu7_thermal_initialize},
-        {NULL, tf_smu7_thermal_set_temperature_range},
-        {NULL, tf_smu7_thermal_enable_alert},
-        {NULL, smum_thermal_avfs_enable},
+        { .tableFunction = tf_smu7_thermal_initialize },
+        { .tableFunction = tf_smu7_thermal_set_temperature_range },
+        { .tableFunction = tf_smu7_thermal_enable_alert },
+        { .tableFunction = smum_thermal_avfs_enable },
 /* We should restrict performance levels to low before we halt the SMC.
  * On the other hand we are still in boot state when we do this
  * so it would be pointless.
  * If this assumption changes we have to revisit this table.
  */
-        {NULL, smum_thermal_setup_fan_table},
-        {NULL, tf_smu7_thermal_start_smc_fan_control},
-        {NULL, NULL}
+        { .tableFunction = smum_thermal_setup_fan_table },
+        { .tableFunction = tf_smu7_thermal_start_smc_fan_control },
+        { }
 };
 
 static const struct phm_master_table_header
@@ -529,10 +529,10 @@ phm_thermal_start_thermal_controller_master = {
 
 static const struct phm_master_table_item
 phm_thermal_set_temperature_range_master_list[] = {
-        {NULL, tf_smu7_thermal_disable_alert},
-        {NULL, tf_smu7_thermal_set_temperature_range},
-        {NULL, tf_smu7_thermal_enable_alert},
-        {NULL, NULL}
+        { .tableFunction = tf_smu7_thermal_disable_alert },
+        { .tableFunction = tf_smu7_thermal_set_temperature_range },
+        { .tableFunction = tf_smu7_thermal_enable_alert },
+        { }
 };
 
 static const struct phm_master_table_header
@@ -575,3 +575,9 @@ int pp_smu7_thermal_initialize(struct pp_hwmgr *hwmgr)
         return result;
 }
 
+void pp_smu7_thermal_fini(struct pp_hwmgr *hwmgr)
+{
+        phm_destroy_table(hwmgr, &(hwmgr->set_temperature_range));
+        phm_destroy_table(hwmgr, &(hwmgr->start_thermal_controller));
+        return;
+}
\ No newline at end of file
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.h
index 6face973be43..2ed774db42c7 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.h
@@ -47,6 +47,7 @@ extern int smu7_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode);
 extern int smu7_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t speed);
 extern int smu7_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr);
 extern int pp_smu7_thermal_initialize(struct pp_hwmgr *hwmgr);
+extern void pp_smu7_thermal_fini(struct pp_hwmgr *hwmgr);
 extern int smu7_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr);
 extern int smu7_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed);
 extern int smu7_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed);
diff --git a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
index 3a883e6c601a..6dd5f0e9ef87 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
@@ -29,7 +29,10 @@
 #include "amd_shared.h"
 #include "cgs_common.h"
 
-extern int amdgpu_dpm;
+extern const struct amd_ip_funcs pp_ip_funcs;
+extern const struct amd_powerplay_funcs pp_dpm_funcs;
+
+#define PP_DPM_DISABLED 0xCCCC
 
 enum amd_pp_sensors {
         AMDGPU_PP_SENSOR_GFX_SCLK = 0,
@@ -135,17 +138,12 @@ enum amd_pp_event {
         AMD_PP_EVENT_MAX
 };
 
-enum amd_dpm_forced_level {
-        AMD_DPM_FORCED_LEVEL_AUTO = 0,
-        AMD_DPM_FORCED_LEVEL_LOW = 1,
-        AMD_DPM_FORCED_LEVEL_HIGH = 2,
-        AMD_DPM_FORCED_LEVEL_MANUAL = 3,
-};
-
 struct amd_pp_init {
         struct cgs_device *device;
         uint32_t chip_family;
         uint32_t chip_id;
+        bool pm_en;
+        uint32_t feature_mask;
 };
 
 enum amd_pp_display_config_type{
@@ -371,10 +369,10 @@ struct amd_powerplay {
         const struct amd_powerplay_funcs *pp_funcs;
 };
 
-int amd_powerplay_init(struct amd_pp_init *pp_init,
-                       struct amd_powerplay *amd_pp);
+int amd_powerplay_create(struct amd_pp_init *pp_init,
+                                void **handle);
 
-int amd_powerplay_fini(void *handle);
+int amd_powerplay_destroy(void *handle);
 
 int amd_powerplay_reset(void *handle);
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/eventmgr.h b/drivers/gpu/drm/amd/powerplay/inc/eventmgr.h
index d63ef83b2628..7bd8a7e57080 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/eventmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/eventmgr.h
@@ -119,7 +119,6 @@ struct pp_eventmgr {
         void (*pp_eventmgr_fini)(struct pp_eventmgr *eventmgr);
 };
 
-int eventmgr_init(struct pp_instance *handle);
-int eventmgr_fini(struct pp_eventmgr *eventmgr);
+int eventmgr_early_init(struct pp_instance *handle);
 
 #endif /* _EVENTMGR_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
index 6cdb7cbf515e..7275a29293eb 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
@@ -38,8 +38,6 @@ struct pp_hwmgr;
 struct phm_fan_speed_info;
 struct pp_atomctrl_voltage_table;
 
-extern unsigned amdgpu_pp_feature_mask;
-
 #define VOLTAGE_SCALE 4
 
 uint8_t convert_to_vid(uint16_t vddc);
@@ -358,6 +356,8 @@ struct pp_hwmgr_func {
         int (*get_mclk_od)(struct pp_hwmgr *hwmgr);
         int (*set_mclk_od)(struct pp_hwmgr *hwmgr, uint32_t value);
         int (*read_sensor)(struct pp_hwmgr *hwmgr, int idx, int32_t *value);
+        int (*request_firmware)(struct pp_hwmgr *hwmgr);
+        int (*release_firmware)(struct pp_hwmgr *hwmgr);
 };
 
 struct pp_table_func {
@@ -612,6 +612,7 @@ struct pp_hwmgr {
         uint32_t num_vce_state_tables;
 
         enum amd_dpm_forced_level dpm_level;
+        enum amd_dpm_forced_level saved_dpm_level;
         bool block_hw_access;
         struct phm_gfx_arbiter gfx_arbiter;
         struct phm_acp_arbiter acp_arbiter;
@@ -651,19 +652,12 @@ struct pp_hwmgr {
         uint32_t feature_mask;
 };
 
-
-extern int hwmgr_init(struct amd_pp_init *pp_init,
-                      struct pp_instance *handle);
-
-extern int hwmgr_fini(struct pp_hwmgr *hwmgr);
-
-extern int hw_init_power_state_table(struct pp_hwmgr *hwmgr);
-
+extern int hwmgr_early_init(struct pp_instance *handle);
+extern int hwmgr_hw_init(struct pp_instance *handle);
+extern int hwmgr_hw_fini(struct pp_instance *handle);
 extern int phm_wait_on_register(struct pp_hwmgr *hwmgr, uint32_t index,
                                 uint32_t value, uint32_t mask);
 
-
-
 extern void phm_wait_on_indirect_register(struct pp_hwmgr *hwmgr,
                                 uint32_t indirect_port,
                                 uint32_t index,
@@ -692,11 +686,10 @@ extern int phm_find_boot_level(void *table, uint32_t value, uint32_t *boot_level
 extern int phm_get_sclk_for_voltage_evv(struct pp_hwmgr *hwmgr, phm_ppt_v1_voltage_lookup_table *lookup_table,
                                                                 uint16_t virtual_voltage_id, int32_t *sclk);
 extern int phm_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr);
-extern int phm_hwmgr_backend_fini(struct pp_hwmgr *hwmgr);
 extern uint32_t phm_get_lowest_enabled_level(struct pp_hwmgr *hwmgr, uint32_t mask);
 extern void phm_apply_dal_min_voltage_request(struct pp_hwmgr *hwmgr);
 
-extern int smu7_hwmgr_init(struct pp_hwmgr *hwmgr);
+extern int smu7_init_function_pointers(struct pp_hwmgr *hwmgr);
 extern int phm_get_voltage_evv_on_sclk(struct pp_hwmgr *hwmgr, uint8_t voltage_type,
                                 uint32_t sclk, uint16_t id, uint16_t *voltage);
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h b/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h
index bfdbec10cdd5..072880130cfb 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h
@@ -24,6 +24,12 @@
 #ifndef PP_DEBUG_H
 #define PP_DEBUG_H
 
+#ifdef pr_fmt
+#undef pr_fmt
+#endif
+
+#define pr_fmt(fmt) "amdgpu: [powerplay] " fmt
+
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
@@ -31,7 +37,7 @@
 #define PP_ASSERT_WITH_CODE(cond, msg, code)    \
         do {                                    \
                 if (!(cond)) {                  \
-                        printk("%s\n", msg);    \
+                        pr_warning("%s\n", msg);        \
                         code;                   \
                 }                               \
         } while (0)
@@ -39,7 +45,7 @@
 
 #define PP_DBG_LOG(fmt, ...) \
         do { \
-                if(0)printk(KERN_INFO "[ pp_dbg ] " fmt, ##__VA_ARGS__); \
+                pr_debug(fmt, ##__VA_ARGS__); \
         } while (0)
 
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_instance.h b/drivers/gpu/drm/amd/powerplay/inc/pp_instance.h
index 4d8ed1f33de4..ab8494fb5c6b 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/pp_instance.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_instance.h
@@ -31,6 +31,11 @@
 
 struct pp_instance {
         uint32_t pp_valid;
+        uint32_t chip_family;
+        uint32_t chip_id;
+        bool pm_en;
+        uint32_t feature_mask;
+        void *device;
         struct pp_smumgr *smu_mgr;
         struct pp_hwmgr *hwmgr;
         struct pp_eventmgr *eventmgr;
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu7_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/smu7_ppsmc.h
index bce00096d80d..fbc504c70b8b 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/smu7_ppsmc.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu7_ppsmc.h
@@ -394,6 +394,9 @@ typedef uint16_t PPSMC_Result;
 
 #define PPSMC_MSG_SetVBITimeout               ((uint16_t) 0x306)
 
+#define PPSMC_MSG_EnableDpmDidt               ((uint16_t) 0x309)
+#define PPSMC_MSG_DisableDpmDidt              ((uint16_t) 0x30A)
+
 #define PPSMC_MSG_SecureSRBMWrite             ((uint16_t) 0x600)
 #define PPSMC_MSG_SecureSRBMRead              ((uint16_t) 0x601)
 #define PPSMC_MSG_SetAddress                  ((uint16_t) 0x800)
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
index 2139072065cc..9b6531bd6350 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
@@ -33,6 +33,12 @@ struct pp_hwmgr;
 #define smu_lower_32_bits(n) ((uint32_t)(n))
 #define smu_upper_32_bits(n) ((uint32_t)(((n)>>16)>>16))
 
+extern const struct pp_smumgr_func cz_smu_funcs;
+extern const struct pp_smumgr_func iceland_smu_funcs;
+extern const struct pp_smumgr_func tonga_smu_funcs;
+extern const struct pp_smumgr_func fiji_smu_funcs;
+extern const struct pp_smumgr_func polaris10_smu_funcs;
+
 enum AVFS_BTC_STATUS {
         AVFS_BTC_BOOT = 0,
         AVFS_BTC_BOOT_STARTEDSMU,
@@ -133,11 +139,7 @@ struct pp_smumgr {
         const struct pp_smumgr_func *smumgr_funcs;
 };
 
-
-extern int smum_init(struct amd_pp_init *pp_init,
-                     struct pp_instance *handle);
-
-extern int smum_fini(struct pp_smumgr *smumgr);
+extern int smum_early_init(struct pp_instance *handle);
 
 extern int smum_get_argument(struct pp_smumgr *smumgr);
 
@@ -172,13 +174,6 @@ extern int smu_allocate_memory(void *device, uint32_t size,
                          void **kptr, void *handle);
 
 extern int smu_free_memory(void *device, void *handle);
-
-extern int cz_smum_init(struct pp_smumgr *smumgr);
-extern int iceland_smum_init(struct pp_smumgr *smumgr);
-extern int tonga_smum_init(struct pp_smumgr *smumgr);
-extern int fiji_smum_init(struct pp_smumgr *smumgr);
-extern int polaris10_smum_init(struct pp_smumgr *smumgr);
-
 extern int smum_update_sclk_threshold(struct pp_hwmgr *hwmgr);
 
 extern int smum_update_smc_table(struct pp_hwmgr *hwmgr, uint32_t type);
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c
index 5a44485526d2..1f6744a443d4 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c
@@ -70,7 +70,7 @@ static int cz_send_msg_to_smc_async(struct pp_smumgr *smumgr,
         result = SMUM_WAIT_FIELD_UNEQUAL(smumgr,
                                         SMU_MP1_SRBM2P_RESP_0, CONTENT, 0);
         if (result != 0) {
-                printk(KERN_ERR "[ powerplay ] cz_send_msg_to_smc_async failed\n");
+                pr_err("cz_send_msg_to_smc_async failed\n");
                 return result;
         }
 
@@ -100,12 +100,12 @@ static int cz_set_smc_sram_address(struct pp_smumgr *smumgr,
                 return -EINVAL;
 
         if (0 != (3 & smc_address)) {
-                printk(KERN_ERR "[ powerplay ] SMC address must be 4 byte aligned\n");
+                pr_err("SMC address must be 4 byte aligned\n");
                 return -EINVAL;
         }
 
         if (limit <= (smc_address + 3)) {
-                printk(KERN_ERR "[ powerplay ] SMC address beyond the SMC RAM area\n");
+                pr_err("SMC address beyond the SMC RAM area\n");
                 return -EINVAL;
         }
 
@@ -141,42 +141,6 @@ static int cz_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr,
         return cz_send_msg_to_smc(smumgr, msg);
 }
 
-static int cz_request_smu_load_fw(struct pp_smumgr *smumgr)
-{
-        struct cz_smumgr *cz_smu = (struct cz_smumgr *)(smumgr->backend);
-        uint32_t smc_address;
-
-        if (!smumgr->reload_fw) {
-                printk(KERN_INFO "[ powerplay ] skip reloading...\n");
-                return 0;
-        }
-
-        smc_address = SMU8_FIRMWARE_HEADER_LOCATION +
-                offsetof(struct SMU8_Firmware_Header, UcodeLoadStatus);
-
-        cz_write_smc_sram_dword(smumgr, smc_address, 0, smc_address+4);
-
-        cz_send_msg_to_smc_with_parameter(smumgr,
-                                        PPSMC_MSG_DriverDramAddrHi,
-                                        cz_smu->toc_buffer.mc_addr_high);
-
-        cz_send_msg_to_smc_with_parameter(smumgr,
-                                        PPSMC_MSG_DriverDramAddrLo,
-                                        cz_smu->toc_buffer.mc_addr_low);
-
-        cz_send_msg_to_smc(smumgr, PPSMC_MSG_InitJobs);
-
-        cz_send_msg_to_smc_with_parameter(smumgr,
-                                        PPSMC_MSG_ExecuteJob,
-                                        cz_smu->toc_entry_aram);
-        cz_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_ExecuteJob,
-                                cz_smu->toc_entry_power_profiling_index);
-
-        return cz_send_msg_to_smc_with_parameter(smumgr,
-                                        PPSMC_MSG_ExecuteJob,
-                                        cz_smu->toc_entry_initialize_index);
-}
-
 static int cz_check_fw_load_finish(struct pp_smumgr *smumgr,
                                    uint32_t firmware)
 {
@@ -198,7 +162,7 @@ static int cz_check_fw_load_finish(struct pp_smumgr *smumgr,
         }
 
         if (i >= smumgr->usec_timeout) {
-                printk(KERN_ERR "[ powerplay ] SMU check loaded firmware failed.\n");
+                pr_err("SMU check loaded firmware failed.\n");
                 return -EINVAL;
         }
 
@@ -250,34 +214,6 @@ static int cz_load_mec_firmware(struct pp_smumgr *smumgr)
         return 0;
 }
 
-static int cz_start_smu(struct pp_smumgr *smumgr)
-{
-        int ret = 0;
-        uint32_t fw_to_check = UCODE_ID_RLC_G_MASK |
-                                UCODE_ID_SDMA0_MASK |
-                                UCODE_ID_SDMA1_MASK |
-                                UCODE_ID_CP_CE_MASK |
-                                UCODE_ID_CP_ME_MASK |
-                                UCODE_ID_CP_PFP_MASK |
-                                UCODE_ID_CP_MEC_JT1_MASK |
-                                UCODE_ID_CP_MEC_JT2_MASK;
-
-        if (smumgr->chip_id == CHIP_STONEY)
-                fw_to_check &= ~(UCODE_ID_SDMA1_MASK | UCODE_ID_CP_MEC_JT2_MASK);
-
-        ret = cz_request_smu_load_fw(smumgr);
-        if (ret)
-                printk(KERN_ERR "[ powerplay] SMU firmware load failed\n");
-
-        cz_check_fw_load_finish(smumgr, fw_to_check);
-
-        ret = cz_load_mec_firmware(smumgr);
-        if (ret)
-                printk(KERN_ERR "[ powerplay ] Mec Firmware load failed\n");
-
-        return ret;
-}
-
 static uint8_t cz_translate_firmware_enum_to_arg(struct pp_smumgr *smumgr,
                         enum cz_scratch_entry firmware_enum)
 {
@@ -406,7 +342,7 @@ static int cz_smu_populate_single_scratch_task(
                         break;
 
         if (i >= cz_smu->scratch_buffer_length) {
-                printk(KERN_ERR "[ powerplay ] Invalid Firmware Type\n");
+                pr_err("Invalid Firmware Type\n");
                 return -EINVAL;
         }
 
@@ -443,7 +379,7 @@ static int cz_smu_populate_single_ucode_load_task(
                         break;
 
         if (i >= cz_smu->driver_buffer_length) {
-                printk(KERN_ERR "[ powerplay ] Invalid Firmware Type\n");
+                pr_err("Invalid Firmware Type\n");
                 return -EINVAL;
         }
 
@@ -729,11 +665,87 @@ static int cz_upload_pptable_settings(struct pp_smumgr *smumgr)
         return 0;
 }
 
+static int cz_request_smu_load_fw(struct pp_smumgr *smumgr)
+{
+        struct cz_smumgr *cz_smu = (struct cz_smumgr *)(smumgr->backend);
+        uint32_t smc_address;
+
+        if (!smumgr->reload_fw) {
+                pr_info("skip reloading...\n");
+                return 0;
+        }
+
+        cz_smu_populate_firmware_entries(smumgr);
+
+        cz_smu_construct_toc(smumgr);
+
+        smc_address = SMU8_FIRMWARE_HEADER_LOCATION +
+                offsetof(struct SMU8_Firmware_Header, UcodeLoadStatus);
+
+        cz_write_smc_sram_dword(smumgr, smc_address, 0, smc_address+4);
+
+        cz_send_msg_to_smc_with_parameter(smumgr,
+                                        PPSMC_MSG_DriverDramAddrHi,
+                                        cz_smu->toc_buffer.mc_addr_high);
+
+        cz_send_msg_to_smc_with_parameter(smumgr,
+                                        PPSMC_MSG_DriverDramAddrLo,
+                                        cz_smu->toc_buffer.mc_addr_low);
+
+        cz_send_msg_to_smc(smumgr, PPSMC_MSG_InitJobs);
+
+        cz_send_msg_to_smc_with_parameter(smumgr,
+                                        PPSMC_MSG_ExecuteJob,
+                                        cz_smu->toc_entry_aram);
+        cz_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_ExecuteJob,
+                                cz_smu->toc_entry_power_profiling_index);
+
+        return cz_send_msg_to_smc_with_parameter(smumgr,
+                                        PPSMC_MSG_ExecuteJob,
+                                        cz_smu->toc_entry_initialize_index);
+}
+
+static int cz_start_smu(struct pp_smumgr *smumgr)
+{
+        int ret = 0;
+        uint32_t fw_to_check = 0;
+
+        fw_to_check = UCODE_ID_RLC_G_MASK |
+                        UCODE_ID_SDMA0_MASK |
+                        UCODE_ID_SDMA1_MASK |
+                        UCODE_ID_CP_CE_MASK |
+                        UCODE_ID_CP_ME_MASK |
+                        UCODE_ID_CP_PFP_MASK |
+                        UCODE_ID_CP_MEC_JT1_MASK |
+                        UCODE_ID_CP_MEC_JT2_MASK;
+
+        if (smumgr->chip_id == CHIP_STONEY)
+                fw_to_check &= ~(UCODE_ID_SDMA1_MASK | UCODE_ID_CP_MEC_JT2_MASK);
+
+        ret = cz_request_smu_load_fw(smumgr);
+        if (ret)
+                pr_err("SMU firmware load failed\n");
+
+        cz_check_fw_load_finish(smumgr, fw_to_check);
+
+        ret = cz_load_mec_firmware(smumgr);
+        if (ret)
+                pr_err("Mec Firmware load failed\n");
+
+        return ret;
+}
+
 static int cz_smu_init(struct pp_smumgr *smumgr)
 {
-        struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
         uint64_t mc_addr = 0;
         int ret = 0;
+        struct cz_smumgr *cz_smu;
+
+        cz_smu = kzalloc(sizeof(struct cz_smumgr), GFP_KERNEL);
+        if (cz_smu == NULL)
+                return -ENOMEM;
+
+        smumgr->backend = cz_smu;
 
         cz_smu->toc_buffer.data_size = 4096;
         cz_smu->smu_buffer.data_size =
@@ -769,12 +781,11 @@ static int cz_smu_init(struct pp_smumgr *smumgr)
         cz_smu->smu_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
         cz_smu->smu_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
 
-        cz_smu_populate_firmware_entries(smumgr);
         if (0 != cz_smu_populate_single_scratch_entry(smumgr,
                 CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SCRATCH,
                 UCODE_ID_RLC_SCRATCH_SIZE_BYTE,
                 &cz_smu->scratch_buffer[cz_smu->scratch_buffer_length++])) {
-                printk(KERN_ERR "[ powerplay ] Error when Populate Firmware Entry.\n");
+                pr_err("Error when Populate Firmware Entry.\n");
                 return -1;
         }
 
@@ -782,14 +793,14 @@ static int cz_smu_init(struct pp_smumgr *smumgr)
                 CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_ARAM,
                 UCODE_ID_RLC_SRM_ARAM_SIZE_BYTE,
                 &cz_smu->scratch_buffer[cz_smu->scratch_buffer_length++])) {
-                printk(KERN_ERR "[ powerplay ] Error when Populate Firmware Entry.\n");
+                pr_err("Error when Populate Firmware Entry.\n");
                 return -1;
         }
         if (0 != cz_smu_populate_single_scratch_entry(smumgr,
                 CZ_SCRATCH_ENTRY_UCODE_ID_RLC_SRM_DRAM,
                 UCODE_ID_RLC_SRM_DRAM_SIZE_BYTE,
                 &cz_smu->scratch_buffer[cz_smu->scratch_buffer_length++])) {
-                printk(KERN_ERR "[ powerplay ] Error when Populate Firmware Entry.\n");
+                pr_err("Error when Populate Firmware Entry.\n");
                 return -1;
         }
 
@@ -797,7 +808,7 @@ static int cz_smu_init(struct pp_smumgr *smumgr)
                 CZ_SCRATCH_ENTRY_UCODE_ID_POWER_PROFILING,
                 sizeof(struct SMU8_MultimediaPowerLogData),
                 &cz_smu->scratch_buffer[cz_smu->scratch_buffer_length++])) {
-                printk(KERN_ERR "[ powerplay ] Error when Populate Firmware Entry.\n");
+                pr_err("Error when Populate Firmware Entry.\n");
                 return -1;
         }
 
@@ -805,10 +816,9 @@ static int cz_smu_init(struct pp_smumgr *smumgr)
                 CZ_SCRATCH_ENTRY_SMU8_FUSION_CLKTABLE,
                 sizeof(struct SMU8_Fusion_ClkTable),
                 &cz_smu->scratch_buffer[cz_smu->scratch_buffer_length++])) {
-                printk(KERN_ERR "[ powerplay ] Error when Populate Firmware Entry.\n");
+                pr_err("Error when Populate Firmware Entry.\n");
                 return -1;
         }
-        cz_smu_construct_toc(smumgr);
 
         return 0;
 }
@@ -827,13 +837,12 @@ static int cz_smu_fini(struct pp_smumgr *smumgr)
                 cgs_free_gpu_mem(smumgr->device,
                                 cz_smu->smu_buffer.handle);
                 kfree(cz_smu);
-                kfree(smumgr);
         }
 
         return 0;
 }
 
-static const struct pp_smumgr_func cz_smu_funcs = {
+const struct pp_smumgr_func cz_smu_funcs = {
         .smu_init = cz_smu_init,
         .smu_fini = cz_smu_fini,
         .start_smu = cz_start_smu,
@@ -847,15 +856,3 @@ static const struct pp_smumgr_func cz_smu_funcs = {
         .upload_pptable_settings = cz_upload_pptable_settings,
 };
 
-int cz_smum_init(struct pp_smumgr *smumgr)
-{
-        struct cz_smumgr *cz_smu;
-
-        cz_smu = kzalloc(sizeof(struct cz_smumgr), GFP_KERNEL);
-        if (cz_smu == NULL)
-                return -ENOMEM;
-
-        smumgr->backend = cz_smu;
-        smumgr->smumgr_funcs = &cz_smu_funcs;
-        return 0;
-}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.h
index 883818039248..7c3a290c8957 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.h
@@ -95,8 +95,4 @@ struct cz_smumgr {
         struct cz_buffer_entry scratch_buffer[MAX_NUM_SCRATCH];
 };
 
-struct pp_smumgr;
-
-extern int cz_smum_init(struct pp_smumgr *smumgr);
-
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c
index 6aeb1d20cc3b..0f7a77b7312e 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c
@@ -21,13 +21,13 @@
  *
  */
 
+#include "pp_debug.h"
 #include "fiji_smc.h"
 #include "smu7_dyn_defaults.h"
 
 #include "smu7_hwmgr.h"
 #include "hardwaremanager.h"
 #include "ppatomctrl.h"
-#include "pp_debug.h"
 #include "cgs_common.h"
 #include "atombios.h"
 #include "fiji_smumgr.h"
@@ -2131,7 +2131,7 @@ uint32_t fiji_get_offsetof(uint32_t type, uint32_t member)
                         return offsetof(SMU73_Discrete_DpmTable, LowSclkInterruptThreshold);
                 }
         }
-        printk(KERN_WARNING "can't get the offset of type %x member %x\n", type, member);
+        pr_warning("can't get the offset of type %x member %x\n", type, member);
         return 0;
 }
 
@@ -2156,7 +2156,7 @@ uint32_t fiji_get_mac_definition(uint32_t value)
                 return SMU73_MAX_LEVELS_MVDD;
         }
 
-        printk(KERN_WARNING "can't get the mac of %x\n", value);
+        pr_warning("can't get the mac of %x\n", value);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
index 26eff56b4a99..54b347366b5d 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
@@ -21,6 +21,7 @@
  *
  */
 
+#include "pp_debug.h"
 #include "smumgr.h"
 #include "smu73.h"
 #include "smu_ucode_xfer_vi.h"
@@ -36,7 +37,6 @@
 #include "gca/gfx_8_0_d.h"
 #include "bif/bif_5_0_d.h"
 #include "bif/bif_5_0_sh_mask.h"
-#include "pp_debug.h"
 #include "fiji_pwrvirus.h"
 #include "fiji_smc.h"
 
@@ -179,7 +179,7 @@ static int fiji_setup_pwr_virus(struct pp_smumgr *smumgr)
                         result = 0;
                         break;
                 default:
-                        printk(KERN_ERR "Table Exit with Invalid Command!");
+                        pr_err("Table Exit with Invalid Command!");
                         priv->avfs.AvfsBtcStatus = AVFS_BTC_VIRUS_FAIL;
                         result = -1;
                         break;
@@ -202,13 +202,13 @@ static int fiji_start_avfs_btc(struct pp_smumgr *smumgr)
                                 priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_UNSAVED;
                                 result = 0;
                         } else {
-                                printk(KERN_ERR "[AVFS][fiji_start_avfs_btc] Attempt"
+                                pr_err("[AVFS][fiji_start_avfs_btc] Attempt"
                                                 " to Enable AVFS Failed!");
                                 smum_send_msg_to_smc(smumgr, PPSMC_MSG_DisableAvfs);
                                 result = -1;
                         }
                 } else {
-                        printk(KERN_ERR "[AVFS][fiji_start_avfs_btc] "
+                        pr_err("[AVFS][fiji_start_avfs_btc] "
                                         "PerformBTC SMU msg failed");
                         result = -1;
                 }
@@ -384,7 +384,7 @@ static int fiji_avfs_event_mgr(struct pp_smumgr *smumgr, bool smu_started)
         case AVFS_BTC_NOTSUPPORTED: /* Do nothing */
                 break;
         default:
-                printk(KERN_ERR "[AVFS] Something is broken. See log!");
+                pr_err("[AVFS] Something is broken. See log!");
                 break;
         }
         return 0;
@@ -464,13 +464,20 @@ static bool fiji_is_hw_avfs_present(struct pp_smumgr *smumgr)
 */
 static int fiji_smu_init(struct pp_smumgr *smumgr)
 {
-        struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
         int i;
+        struct fiji_smumgr *fiji_priv = NULL;
+
+        fiji_priv = kzalloc(sizeof(struct fiji_smumgr), GFP_KERNEL);
+
+        if (fiji_priv == NULL)
+                return -ENOMEM;
+
+        smumgr->backend = fiji_priv;
 
         if (smu7_init(smumgr))
                 return -EINVAL;
 
-        priv->avfs.AvfsBtcStatus = AVFS_BTC_BOOT;
+        fiji_priv->avfs.AvfsBtcStatus = AVFS_BTC_BOOT;
         if (fiji_is_hw_avfs_present(smumgr))
                 /* AVFS Parameter
                  * 0 - BTC DC disabled, BTC AC disabled
@@ -479,18 +486,18 @@ static int fiji_smu_init(struct pp_smumgr *smumgr)
                  * 3 - BTC DC enabled,  BTC AC enabled
                  * Default is 0 - BTC DC disabled, BTC AC disabled
                  */
-                priv->avfs.AvfsBtcParam = 0;
+                fiji_priv->avfs.AvfsBtcParam = 0;
         else
-                priv->avfs.AvfsBtcStatus = AVFS_BTC_NOTSUPPORTED;
+                fiji_priv->avfs.AvfsBtcStatus = AVFS_BTC_NOTSUPPORTED;
 
         for (i = 0; i < SMU73_MAX_LEVELS_GRAPHICS; i++)
-                priv->activity_target[i] = 30;
+                fiji_priv->activity_target[i] = 30;
 
         return 0;
 }
 
 
-static const struct pp_smumgr_func fiji_smu_funcs = {
+const struct pp_smumgr_func fiji_smu_funcs = {
         .smu_init = &fiji_smu_init,
         .smu_fini = &smu7_smu_fini,
         .start_smu = &fiji_start_smu,
@@ -513,18 +520,3 @@ static const struct pp_smumgr_func fiji_smu_funcs = {
         .initialize_mc_reg_table = fiji_initialize_mc_reg_table,
         .is_dpm_running = fiji_is_dpm_running,
 };
-
-int fiji_smum_init(struct pp_smumgr *smumgr)
-{
-        struct fiji_smumgr *fiji_smu = NULL;
-
-        fiji_smu = kzalloc(sizeof(struct fiji_smumgr), GFP_KERNEL);
-
-        if (fiji_smu == NULL)
-                return -ENOMEM;
-
-        smumgr->backend = fiji_smu;
-        smumgr->smumgr_funcs = &fiji_smu_funcs;
-
-        return 0;
-}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c
index a24971a33bfd..ad82161df831 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c
@@ -21,13 +21,13 @@
  *
  */
 
+#include "pp_debug.h"
 #include "iceland_smc.h"
 #include "smu7_dyn_defaults.h"
 
 #include "smu7_hwmgr.h"
 #include "hardwaremanager.h"
 #include "ppatomctrl.h"
-#include "pp_debug.h"
 #include "cgs_common.h"
 #include "atombios.h"
 #include "pppcielanes.h"
@@ -1545,7 +1545,7 @@ static int iceland_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
 
         if (0 != result) {
                 smu_data->smc_state_table.GraphicsBootLevel = 0;
-                printk(KERN_ERR "[ powerplay ] VBIOS did not find boot engine clock value \
+                pr_err("VBIOS did not find boot engine clock value \
                         in dependency table. Using Graphics DPM level 0!");
                 result = 0;
         }
@@ -1556,7 +1556,7 @@ static int iceland_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
 
         if (0 != result) {
                 smu_data->smc_state_table.MemoryBootLevel = 0;
-                printk(KERN_ERR "[ powerplay ] VBIOS did not find boot engine clock value \
+                pr_err("VBIOS did not find boot engine clock value \
                         in dependency table. Using Memory DPM level 0!");
                 result = 0;
         }
@@ -2146,7 +2146,7 @@ uint32_t iceland_get_offsetof(uint32_t type, uint32_t member)
                         return offsetof(SMU71_Discrete_DpmTable, LowSclkInterruptThreshold);
                 }
         }
-        printk(KERN_WARNING "can't get the offset of type %x member %x\n", type, member);
+        pr_warning("can't get the offset of type %x member %x\n", type, member);
         return 0;
 }
 
@@ -2169,7 +2169,7 @@ uint32_t iceland_get_mac_definition(uint32_t value)
                 return SMU71_MAX_LEVELS_MVDD;
         }
 
-        printk(KERN_WARNING "can't get the mac of %x\n", value);
+        pr_warning("can't get the mac of %x\n", value);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c
index eeafefc4acba..0bf2def3b659 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c
@@ -22,6 +22,7 @@
  * Author: Huang Rui <ray.huang@amd.com>
  *
  */
+#include "pp_debug.h"
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
@@ -29,7 +30,6 @@
 
 #include "smumgr.h"
 #include "iceland_smumgr.h"
-#include "pp_debug.h"
 #include "smu_ucode_xfer_vi.h"
 #include "ppsmc.h"
 #include "smu/smu_7_1_1_d.h"
@@ -176,7 +176,7 @@ static int iceland_start_smu(struct pp_smumgr *smumgr)
                 return result;
 
         if (!smu7_is_smc_ram_running(smumgr)) {
-                printk("smu not running, upload firmware again \n");
+                pr_info("smu not running, upload firmware again \n");
                 result = iceland_smu_upload_firmware_image(smumgr);
                 if (result)
                         return result;
@@ -201,17 +201,25 @@ static int iceland_start_smu(struct pp_smumgr *smumgr)
 static int iceland_smu_init(struct pp_smumgr *smumgr)
 {
         int i;
-        struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(smumgr->backend);
+        struct iceland_smumgr *iceland_priv = NULL;
+
+        iceland_priv = kzalloc(sizeof(struct iceland_smumgr), GFP_KERNEL);
+
+        if (iceland_priv == NULL)
+                return -ENOMEM;
+
+        smumgr->backend = iceland_priv;
+
         if (smu7_init(smumgr))
                 return -EINVAL;
 
         for (i = 0; i < SMU71_MAX_LEVELS_GRAPHICS; i++)
-                smu_data->activity_target[i] = 30;
+                iceland_priv->activity_target[i] = 30;
 
         return 0;
 }
 
-static const struct pp_smumgr_func iceland_smu_funcs = {
+const struct pp_smumgr_func iceland_smu_funcs = {
         .smu_init = &iceland_smu_init,
         .smu_fini = &smu7_smu_fini,
         .start_smu = &iceland_start_smu,
@@ -234,17 +242,3 @@ static const struct pp_smumgr_func iceland_smu_funcs = {
         .is_dpm_running = iceland_is_dpm_running,
 };
 
-int iceland_smum_init(struct pp_smumgr *smumgr)
-{
-        struct iceland_smumgr *iceland_smu = NULL;
-
-        iceland_smu = kzalloc(sizeof(struct iceland_smumgr), GFP_KERNEL);
-
-        if (iceland_smu == NULL)
-                return -ENOMEM;
-
-        smumgr->backend = iceland_smu;
-        smumgr->smumgr_funcs = &iceland_smu_funcs;
-
-        return 0;
-}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c
index 5190e821200c..0e26900e459e 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c
@@ -21,13 +21,13 @@
  *
  */
 
+#include "pp_debug.h"
 #include "polaris10_smc.h"
 #include "smu7_dyn_defaults.h"
 
 #include "smu7_hwmgr.h"
 #include "hardwaremanager.h"
 #include "ppatomctrl.h"
-#include "pp_debug.h"
 #include "cgs_common.h"
 #include "atombios.h"
 #include "polaris10_smumgr.h"
@@ -2180,7 +2180,7 @@ uint32_t polaris10_get_offsetof(uint32_t type, uint32_t member)
                         return offsetof(SMU74_Discrete_DpmTable, LowSclkInterruptThreshold);
                 }
         }
-        printk(KERN_WARNING "can't get the offset of type %x member %x\n", type, member);
+        pr_warning("can't get the offset of type %x member %x\n", type, member);
         return 0;
 }
 
@@ -2207,7 +2207,7 @@ uint32_t polaris10_get_mac_definition(uint32_t value)
                 return SMU7_UVD_MCLK_HANDSHAKE_DISABLE;
         }
 
-        printk(KERN_WARNING "can't get the mac of %x\n", value);
+        pr_warning("can't get the mac of %x\n", value);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
index f38a68747df0..ce20ae2e520e 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
@@ -21,6 +21,7 @@
  *
  */
 
+#include "pp_debug.h"
 #include "smumgr.h"
 #include "smu74.h"
 #include "smu_ucode_xfer_vi.h"
@@ -36,7 +37,6 @@
 #include "bif/bif_5_0_sh_mask.h"
 #include "polaris10_pwrvirus.h"
 #include "ppatomctrl.h"
-#include "pp_debug.h"
 #include "cgs_common.h"
 #include "polaris10_smc.h"
 #include "smu7_ppsmc.h"
@@ -84,7 +84,7 @@ static int polaris10_setup_pwr_virus(struct pp_smumgr *smumgr)
                         break;
 
                 default:
-                        printk("Table Exit with Invalid Command!");
+                        pr_info("Table Exit with Invalid Command!");
                         smu_data->avfs.avfs_btc_status = AVFS_BTC_VIRUS_FAIL;
                         result = -1;
                         break;
@@ -102,7 +102,7 @@ static int polaris10_perform_btc(struct pp_smumgr *smumgr)
 
         if (0 != smu_data->avfs.avfs_btc_param) {
                 if (0 != smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_PerformBtc, smu_data->avfs.avfs_btc_param)) {
-                        printk("[AVFS][SmuPolaris10_PerformBtc] PerformBTC SMU msg failed");
+                        pr_info("[AVFS][SmuPolaris10_PerformBtc] PerformBTC SMU msg failed");
                         result = -1;
                 }
         }
@@ -189,7 +189,7 @@ polaris10_avfs_event_mgr(struct pp_smumgr *smumgr, bool SMU_VFT_INTACT)
                 return -1);
 
                 if (smu_data->avfs.avfs_btc_param > 1) {
-                        printk("[AVFS][Polaris10_AVFSEventMgr] AC BTC has not been successfully verified on Fiji. There may be in this setting.");
+                        pr_info("[AVFS][Polaris10_AVFSEventMgr] AC BTC has not been successfully verified on Fiji. There may be in this setting.");
                         smu_data->avfs.avfs_btc_status = AVFS_BTC_VIRUS_FAIL;
                         PP_ASSERT_WITH_CODE(-1 == polaris10_setup_pwr_virus(smumgr),
                         "[AVFS][Polaris10_AVFSEventMgr] Could not setup Pwr Virus for AVFS ",
@@ -208,7 +208,7 @@ polaris10_avfs_event_mgr(struct pp_smumgr *smumgr, bool SMU_VFT_INTACT)
                 break;
 
         default:
-                printk("[AVFS] Something is broken. See log!");
+                pr_info("[AVFS] Something is broken. See log!");
                 break;
         }
 
@@ -328,6 +328,7 @@ static int polaris10_start_smu(struct pp_smumgr *smumgr)
                         /* If failed, try with different security Key. */
                         if (result != 0) {
                                 smu_data->smu7_data.security_hard_key ^= 1;
+                                cgs_rel_firmware(smumgr->device, CGS_UCODE_ID_SMU);
                                 result = polaris10_start_smu_in_protection_mode(smumgr);
                         }
                 }
@@ -363,9 +364,15 @@ static bool polaris10_is_hw_avfs_present(struct pp_smumgr *smumgr)
 
 static int polaris10_smu_init(struct pp_smumgr *smumgr)
 {
-        struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+        struct polaris10_smumgr *smu_data;
         int i;
 
+        smu_data = kzalloc(sizeof(struct polaris10_smumgr), GFP_KERNEL);
+        if (smu_data == NULL)
+                return -ENOMEM;
+
+        smumgr->backend = smu_data;
+
         if (smu7_init(smumgr))
                 return -EINVAL;
 
@@ -380,7 +387,7 @@ static int polaris10_smu_init(struct pp_smumgr *smumgr)
         return 0;
 }
 
-static const struct pp_smumgr_func polaris10_smu_funcs = {
+const struct pp_smumgr_func polaris10_smu_funcs = {
         .smu_init = polaris10_smu_init,
         .smu_fini = smu7_smu_fini,
         .start_smu = polaris10_start_smu,
@@ -403,18 +410,3 @@ static const struct pp_smumgr_func polaris10_smu_funcs = {
         .get_mac_definition = polaris10_get_mac_definition,
         .is_dpm_running = polaris10_is_dpm_running,
 };
-
-int polaris10_smum_init(struct pp_smumgr *smumgr)
-{
-        struct polaris10_smumgr *polaris10_smu = NULL;
-
-        polaris10_smu = kzalloc(sizeof(struct polaris10_smumgr), GFP_KERNEL);
-
-        if (polaris10_smu == NULL)
-                return -EINVAL;
-
-        smumgr->backend = polaris10_smu;
-        smumgr->smumgr_funcs = &polaris10_smu_funcs;
-
-        return 0;
-}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
index f49b5487b951..6749fbe26c74 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
@@ -22,12 +22,12 @@
  */
 
 
+#include "pp_debug.h"
 #include "smumgr.h"
 #include "smu_ucode_xfer_vi.h"
 #include "smu/smu_7_1_3_d.h"
 #include "smu/smu_7_1_3_sh_mask.h"
 #include "ppatomctrl.h"
-#include "pp_debug.h"
 #include "cgs_common.h"
 #include "smu7_ppsmc.h"
 #include "smu7_smumgr.h"
@@ -175,7 +175,7 @@ int smu7_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg)
         ret = SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP);
 
         if (ret != 1)
-                printk("\n failed to send pre message %x ret is %d \n",  msg, ret);
+                pr_info("\n failed to send pre message %x ret is %d \n",  msg, ret);
 
         cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
 
@@ -184,7 +184,7 @@ int smu7_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg)
         ret = SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP);
 
         if (ret != 1)
-                printk("\n failed to send message %x ret is %d \n",  msg, ret);
+                pr_info("\n failed to send message %x ret is %d \n",  msg, ret);
 
         return 0;
 }
@@ -225,7 +225,7 @@ int smu7_send_msg_to_smc_offset(struct pp_smumgr *smumgr)
         SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
 
         if (1 != SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP))
-                printk("Failed to send Message.\n");
+                pr_info("Failed to send Message.\n");
 
         return 0;
 }
@@ -347,7 +347,7 @@ static uint32_t smu7_get_mask_for_firmware_type(uint32_t fw_type)
                 result = UCODE_ID_RLC_G_MASK;
                 break;
         default:
-                printk("UCode type is out of range! \n");
+                pr_info("UCode type is out of range! \n");
                 result = 0;
         }
 
@@ -396,7 +396,7 @@ int smu7_request_smu_load_fw(struct pp_smumgr *smumgr)
         struct SMU_DRAMData_TOC *toc;
 
         if (!smumgr->reload_fw) {
-                printk(KERN_INFO "[ powerplay ] skip reloading...\n");
+                pr_info("skip reloading...\n");
                 return 0;
         }
 
@@ -474,7 +474,7 @@ int smu7_request_smu_load_fw(struct pp_smumgr *smumgr)
         smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_DRV_DRAM_ADDR_LO, smu_data->header_buffer.mc_addr_low);
 
         if (smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_LoadUcodes, fw_to_load))
-                printk(KERN_ERR "Fail to Request SMU Load uCode");
+                pr_err("Fail to Request SMU Load uCode");
 
         return result;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
index 6e618aa20719..60c36928284c 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
@@ -22,6 +22,7 @@
  */
 #include <linux/types.h>
 #include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/slab.h>
 #include <drm/amdgpu_drm.h>
 #include "pp_instance.h"
@@ -29,44 +30,55 @@
 #include "cgs_common.h"
 #include "linux/delay.h"
 
+MODULE_FIRMWARE("amdgpu/topaz_smc.bin");
+MODULE_FIRMWARE("amdgpu/topaz_k_smc.bin");
+MODULE_FIRMWARE("amdgpu/tonga_smc.bin");
+MODULE_FIRMWARE("amdgpu/tonga_k_smc.bin");
+MODULE_FIRMWARE("amdgpu/fiji_smc.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_smc.bin");
+MODULE_FIRMWARE("amdgpu/polaris10_smc_sk.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_smc.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_smc_sk.bin");
+MODULE_FIRMWARE("amdgpu/polaris12_smc.bin");
 
-int smum_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
+
+int smum_early_init(struct pp_instance *handle)
 {
         struct pp_smumgr *smumgr;
 
-        if ((handle == NULL) || (pp_init == NULL))
+        if (handle == NULL)
                 return -EINVAL;
 
         smumgr = kzalloc(sizeof(struct pp_smumgr), GFP_KERNEL);
         if (smumgr == NULL)
                 return -ENOMEM;
 
-        smumgr->device = pp_init->device;
-        smumgr->chip_family = pp_init->chip_family;
-        smumgr->chip_id = pp_init->chip_id;
+        smumgr->device = handle->device;
+        smumgr->chip_family = handle->chip_family;
+        smumgr->chip_id = handle->chip_id;
         smumgr->usec_timeout = AMD_MAX_USEC_TIMEOUT;
         smumgr->reload_fw = 1;
         handle->smu_mgr = smumgr;
 
         switch (smumgr->chip_family) {
         case AMDGPU_FAMILY_CZ:
-                cz_smum_init(smumgr);
+                smumgr->smumgr_funcs = &cz_smu_funcs;
                 break;
         case AMDGPU_FAMILY_VI:
                 switch (smumgr->chip_id) {
                 case CHIP_TOPAZ:
-                        iceland_smum_init(smumgr);
+                        smumgr->smumgr_funcs = &iceland_smu_funcs;
                         break;
                 case CHIP_TONGA:
-                        tonga_smum_init(smumgr);
+                        smumgr->smumgr_funcs = &tonga_smu_funcs;
                         break;
                 case CHIP_FIJI:
-                        fiji_smum_init(smumgr);
+                        smumgr->smumgr_funcs = &fiji_smu_funcs;
                         break;
                 case CHIP_POLARIS11:
                 case CHIP_POLARIS10:
                 case CHIP_POLARIS12:
-                        polaris10_smum_init(smumgr);
+                        smumgr->smumgr_funcs = &polaris10_smu_funcs;
                         break;
                 default:
                         return -EINVAL;
@@ -80,13 +92,6 @@ int smum_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
         return 0;
 }
 
-int smum_fini(struct pp_smumgr *smumgr)
-{
-        kfree(smumgr->device);
-        kfree(smumgr);
-        return 0;
-}
-
 int smum_thermal_avfs_enable(struct pp_hwmgr *hwmgr,
                 void *input, void *output, void *storage, int result)
 {
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c
index 2e1493ce1bb5..331b0aba4a13 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c
@@ -21,13 +21,13 @@
  *
  */
 
+#include "pp_debug.h"
 #include "tonga_smc.h"
 #include "smu7_dyn_defaults.h"
 
 #include "smu7_hwmgr.h"
 #include "hardwaremanager.h"
 #include "ppatomctrl.h"
-#include "pp_debug.h"
 #include "cgs_common.h"
 #include "atombios.h"
 #include "tonga_smumgr.h"
@@ -656,7 +656,7 @@ int tonga_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
                 }
         } else {
                 if (0 == data->dpm_level_enable_mask.pcie_dpm_enable_mask)
-                        printk(KERN_ERR "[ powerplay ] Pcie Dpm Enablemask is 0 !");
+                        pr_err("Pcie Dpm Enablemask is 0 !");
 
                 while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
                                 ((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
@@ -1503,7 +1503,7 @@ static int tonga_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
 
         if (result != 0) {
                 smu_data->smc_state_table.GraphicsBootLevel = 0;
-                printk(KERN_ERR "[powerplay] VBIOS did not find boot engine "
+                pr_err("[powerplay] VBIOS did not find boot engine "
                                 "clock value in dependency table. "
                                 "Using Graphics DPM level 0 !");
                 result = 0;
@@ -1515,7 +1515,7 @@ static int tonga_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
 
         if (result != 0) {
                 smu_data->smc_state_table.MemoryBootLevel = 0;
-                printk(KERN_ERR "[powerplay] VBIOS did not find boot "
+                pr_err("[powerplay] VBIOS did not find boot "
                                 "engine clock value in dependency table."
                                 "Using Memory DPM level 0 !");
                 result = 0;
@@ -1739,7 +1739,7 @@ static int tonga_populate_vr_config(struct pp_hwmgr *hwmgr,
                         config = VR_SVI2_PLANE_2;
                         table->VRConfig |= config;
                 } else {
-                        printk(KERN_ERR "[ powerplay ] VDDC and VDDGFX should "
+                        pr_err("VDDC and VDDGFX should "
                                 "be both on SVI2 control in splitted mode !\n");
                 }
         } else {
@@ -1752,7 +1752,7 @@ static int tonga_populate_vr_config(struct pp_hwmgr *hwmgr,
                         config = VR_SVI2_PLANE_1;
                         table->VRConfig |= config;
                 } else {
-                        printk(KERN_ERR "[ powerplay ] VDDC should be on "
+                        pr_err("VDDC should be on "
                                         "SVI2 control in merged mode !\n");
                 }
         }
@@ -2657,7 +2657,7 @@ uint32_t tonga_get_offsetof(uint32_t type, uint32_t member)
                         return offsetof(SMU72_Discrete_DpmTable, LowSclkInterruptThreshold);
                 }
         }
-        printk(KERN_WARNING "can't get the offset of type %x member %x\n", type, member);
+        pr_warning("can't get the offset of type %x member %x\n", type, member);
         return 0;
 }
 
@@ -2681,7 +2681,7 @@ uint32_t tonga_get_mac_definition(uint32_t value)
         case SMU_MAX_LEVELS_MVDD:
                 return SMU72_MAX_LEVELS_MVDD;
         }
-        printk(KERN_WARNING "can't get the mac value %x\n", value);
+        pr_warning("can't get the mac value %x\n", value);
 
         return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
index eff9a232e72e..a7d55366f2d2 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
@@ -20,6 +20,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#include "pp_debug.h"
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
@@ -27,7 +28,6 @@
 
 #include "smumgr.h"
 #include "tonga_smumgr.h"
-#include "pp_debug.h"
 #include "smu_ucode_xfer_vi.h"
 #include "tonga_ppsmc.h"
 #include "smu/smu_7_1_2_d.h"
@@ -84,7 +84,7 @@ static int tonga_start_in_protection_mode(struct pp_smumgr *smumgr)
         /* Check pass/failed indicator */
         if (1 != SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device,
                                 CGS_IND_REG__SMC, SMU_STATUS, SMU_PASS)) {
-                printk(KERN_ERR "[ powerplay ] SMU Firmware start failed\n");
+                pr_err("SMU Firmware start failed\n");
                 return -EINVAL;
         }
 
@@ -169,20 +169,25 @@ static int tonga_start_smu(struct pp_smumgr *smumgr)
  */
 static int tonga_smu_init(struct pp_smumgr *smumgr)
 {
-        struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(smumgr->backend);
+        struct tonga_smumgr *tonga_priv = NULL;
+        int  i;
+
+        tonga_priv = kzalloc(sizeof(struct tonga_smumgr), GFP_KERNEL);
+        if (tonga_priv == NULL)
+                return -ENOMEM;
 
-        int i;
+        smumgr->backend = tonga_priv;
 
         if (smu7_init(smumgr))
                 return -EINVAL;
 
         for (i = 0; i < SMU72_MAX_LEVELS_GRAPHICS; i++)
-                smu_data->activity_target[i] = 30;
+                tonga_priv->activity_target[i] = 30;
 
         return 0;
 }
 
-static const struct pp_smumgr_func tonga_smu_funcs = {
+const struct pp_smumgr_func tonga_smu_funcs = {
         .smu_init = &tonga_smu_init,
         .smu_fini = &smu7_smu_fini,
         .start_smu = &tonga_start_smu,
@@ -205,18 +210,3 @@ static const struct pp_smumgr_func tonga_smu_funcs = {
         .initialize_mc_reg_table = tonga_initialize_mc_reg_table,
         .is_dpm_running = tonga_is_dpm_running,
 };
-
-int tonga_smum_init(struct pp_smumgr *smumgr)
-{
-        struct tonga_smumgr *tonga_smu = NULL;
-
-        tonga_smu = kzalloc(sizeof(struct tonga_smumgr), GFP_KERNEL);
-
-        if (tonga_smu == NULL)
-                return -ENOMEM;
-
-        smumgr->backend = tonga_smu;
-        smumgr->smumgr_funcs = &tonga_smu_funcs;
-
-        return 0;
-}
diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index 2a1368fac1d1..50c910efa13d 100644
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -236,8 +236,6 @@ struct ttm_bo_driver ast_bo_driver = {
         .verify_access = ast_bo_verify_access,
         .io_mem_reserve = &ast_ttm_io_mem_reserve,
         .io_mem_free = &ast_ttm_io_mem_free,
-        .lru_tail = &ttm_bo_default_lru_tail,
-        .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int ast_mm_init(struct ast_private *ast)
diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c
index ceb1fecf02dd..857755ac2d70 100644
--- a/drivers/gpu/drm/bochs/bochs_mm.c
+++ b/drivers/gpu/drm/bochs/bochs_mm.c
@@ -205,8 +205,6 @@ struct ttm_bo_driver bochs_bo_driver = {
         .verify_access = bochs_bo_verify_access,
         .io_mem_reserve = &bochs_ttm_io_mem_reserve,
         .io_mem_free = &bochs_ttm_io_mem_free,
-        .lru_tail = &ttm_bo_default_lru_tail,
-        .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int bochs_mm_init(struct bochs_device *bochs)
diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c
index d6da848f7c6f..f53aa8f4a143 100644
--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c
+++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c
@@ -236,8 +236,6 @@ struct ttm_bo_driver cirrus_bo_driver = {
         .verify_access = cirrus_bo_verify_access,
         .io_mem_reserve = &cirrus_ttm_io_mem_reserve,
         .io_mem_free = &cirrus_ttm_io_mem_free,
-        .lru_tail = &ttm_bo_default_lru_tail,
-        .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int cirrus_mm_init(struct cirrus_device *cirrus)
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
index 3c6f750389fb..20732b62d4c9 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
@@ -243,8 +243,6 @@ struct ttm_bo_driver hibmc_bo_driver = {
         .verify_access          = hibmc_bo_verify_access,
         .io_mem_reserve         = &hibmc_ttm_io_mem_reserve,
         .io_mem_free            = NULL,
-        .lru_tail               = &ttm_bo_default_lru_tail,
-        .swap_lru_tail          = &ttm_bo_default_swap_lru_tail,
 };
 
 int hibmc_mm_init(struct hibmc_drm_private *hibmc)
diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c
index 5e20220ef4c6..657598bb1e6b 100644
--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c
+++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c
@@ -236,8 +236,6 @@ struct ttm_bo_driver mgag200_bo_driver = {
         .verify_access = mgag200_bo_verify_access,
         .io_mem_reserve = &mgag200_ttm_io_mem_reserve,
         .io_mem_free = &mgag200_ttm_io_mem_free,
-        .lru_tail = &ttm_bo_default_lru_tail,
-        .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int mgag200_mm_init(struct mga_device *mdev)
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index dd07ca140d12..8a528ebe30f3 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -1194,7 +1194,8 @@ out:
 }
 
 static void
-nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
+nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict,
+                     struct ttm_mem_reg *new_mem)
 {
         struct nouveau_bo *nvbo = nouveau_bo(bo);
         struct nvkm_vma *vma;
@@ -1570,8 +1571,6 @@ struct ttm_bo_driver nouveau_bo_driver = {
         .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
         .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
         .io_mem_free = &nouveau_ttm_io_mem_free,
-        .lru_tail = &ttm_bo_default_lru_tail,
-        .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 struct nvkm_vma *
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index bc1c896bc5e1..3dcc48431015 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -366,6 +366,7 @@ static int qxl_bo_move(struct ttm_buffer_object *bo,
 }
 
 static void qxl_bo_move_notify(struct ttm_buffer_object *bo,
+                               bool evict,
                                struct ttm_mem_reg *new_mem)
 {
         struct qxl_bo *qbo;
@@ -393,8 +394,6 @@ static struct ttm_bo_driver qxl_bo_driver = {
         .io_mem_reserve = &qxl_ttm_io_mem_reserve,
         .io_mem_free = &qxl_ttm_io_mem_free,
         .move_notify = &qxl_bo_move_notify,
-        .lru_tail = &ttm_bo_default_lru_tail,
-        .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int qxl_ttm_init(struct qxl_device *qdev)
diff --git a/drivers/gpu/drm/radeon/radeon_bios.c b/drivers/gpu/drm/radeon/radeon_bios.c
index c829cfb02fc4..00cfb5d2875f 100644
--- a/drivers/gpu/drm/radeon/radeon_bios.c
+++ b/drivers/gpu/drm/radeon/radeon_bios.c
@@ -596,52 +596,58 @@ static bool radeon_read_disabled_bios(struct radeon_device *rdev)
 #ifdef CONFIG_ACPI
 static bool radeon_acpi_vfct_bios(struct radeon_device *rdev)
 {
-        bool ret = false;
         struct acpi_table_header *hdr;
         acpi_size tbl_size;
         UEFI_ACPI_VFCT *vfct;
-        GOP_VBIOS_CONTENT *vbios;
-        VFCT_IMAGE_HEADER *vhdr;
+        unsigned offset;
 
         if (!ACPI_SUCCESS(acpi_get_table("VFCT", 1, &hdr)))
                 return false;
         tbl_size = hdr->length;
         if (tbl_size < sizeof(UEFI_ACPI_VFCT)) {
                 DRM_ERROR("ACPI VFCT table present but broken (too short #1)\n");
-                goto out_unmap;
+                return false;
         }
 
         vfct = (UEFI_ACPI_VFCT *)hdr;
-        if (vfct->VBIOSImageOffset + sizeof(VFCT_IMAGE_HEADER) > tbl_size) {
-                DRM_ERROR("ACPI VFCT table present but broken (too short #2)\n");
-                goto out_unmap;
-        }
+        offset = vfct->VBIOSImageOffset;
 
-        vbios = (GOP_VBIOS_CONTENT *)((char *)hdr + vfct->VBIOSImageOffset);
-        vhdr = &vbios->VbiosHeader;
-        DRM_INFO("ACPI VFCT contains a BIOS for %02x:%02x.%d %04x:%04x, size %d\n",
-                        vhdr->PCIBus, vhdr->PCIDevice, vhdr->PCIFunction,
-                        vhdr->VendorID, vhdr->DeviceID, vhdr->ImageLength);
-
-        if (vhdr->PCIBus != rdev->pdev->bus->number ||
-            vhdr->PCIDevice != PCI_SLOT(rdev->pdev->devfn) ||
-            vhdr->PCIFunction != PCI_FUNC(rdev->pdev->devfn) ||
-            vhdr->VendorID != rdev->pdev->vendor ||
-            vhdr->DeviceID != rdev->pdev->device) {
-                DRM_INFO("ACPI VFCT table is not for this card\n");
-                goto out_unmap;
-        }
+        while (offset < tbl_size) {
+                GOP_VBIOS_CONTENT *vbios = (GOP_VBIOS_CONTENT *)((char *)hdr + offset);
+                VFCT_IMAGE_HEADER *vhdr = &vbios->VbiosHeader;
 
-        if (vfct->VBIOSImageOffset + sizeof(VFCT_IMAGE_HEADER) + vhdr->ImageLength > tbl_size) {
-                DRM_ERROR("ACPI VFCT image truncated\n");
-                goto out_unmap;
-        }
+                offset += sizeof(VFCT_IMAGE_HEADER);
+                if (offset > tbl_size) {
+                        DRM_ERROR("ACPI VFCT image header truncated\n");
+                        return false;
+                }
 
-        rdev->bios = kmemdup(&vbios->VbiosContent, vhdr->ImageLength, GFP_KERNEL);
-        ret = !!rdev->bios;
+                offset += vhdr->ImageLength;
+                if (offset > tbl_size) {
+                        DRM_ERROR("ACPI VFCT image truncated\n");
+                        return false;
+                }
+
+                if (vhdr->ImageLength &&
+                    vhdr->PCIBus == rdev->pdev->bus->number &&
+                    vhdr->PCIDevice == PCI_SLOT(rdev->pdev->devfn) &&
+                    vhdr->PCIFunction == PCI_FUNC(rdev->pdev->devfn) &&
+                    vhdr->VendorID == rdev->pdev->vendor &&
+                    vhdr->DeviceID == rdev->pdev->device) {
+                        rdev->bios = kmemdup(&vbios->VbiosContent,
+                                             vhdr->ImageLength,
+                                             GFP_KERNEL);
+
+                        if (!rdev->bios) {
+                                kfree(rdev->bios);
+                                return false;
+                        }
+                        return true;
+                }
+        }
 
-out_unmap:
-        return ret;
+        DRM_ERROR("ACPI VFCT table present but broken (too short #2)\n");
+        return false;
 }
 #else
 static inline bool radeon_acpi_vfct_bios(struct radeon_device *rdev)
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index 41b72ce6613f..74b276060c20 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -765,6 +765,7 @@ int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved,
 }
 
 void radeon_bo_move_notify(struct ttm_buffer_object *bo,
+                           bool evict,
                            struct ttm_mem_reg *new_mem)
 {
         struct radeon_bo *rbo;
diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h
index a10bb3deee54..9ffd8215d38a 100644
--- a/drivers/gpu/drm/radeon/radeon_object.h
+++ b/drivers/gpu/drm/radeon/radeon_object.h
@@ -150,6 +150,7 @@ extern void radeon_bo_get_tiling_flags(struct radeon_bo *bo,
 extern int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved,
                                 bool force_drop);
 extern void radeon_bo_move_notify(struct ttm_buffer_object *bo,
+                                  bool evict,
                                   struct ttm_mem_reg *new_mem);
 extern int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo);
 extern int radeon_bo_get_surface_reg(struct radeon_bo *bo);
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 1888144d0fed..7a10b3852970 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -502,6 +502,8 @@ static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_
                         mem->bus.addr =
                                 ioremap_nocache(mem->bus.base + mem->bus.offset,
                                                 mem->bus.size);
+                if (!mem->bus.addr)
+                        return -ENOMEM;
 
                 /*
                  * Alpha: Use just the bus offset plus
@@ -871,8 +873,6 @@ static struct ttm_bo_driver radeon_bo_driver = {
         .fault_reserve_notify = &radeon_bo_fault_reserve_notify,
         .io_mem_reserve = &radeon_ttm_io_mem_reserve,
         .io_mem_free = &radeon_ttm_io_mem_free,
-        .lru_tail = &ttm_bo_default_lru_tail,
-        .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int radeon_ttm_init(struct radeon_device *rdev)
diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c
index 2944916f7102..d12b8978142f 100644
--- a/drivers/gpu/drm/radeon/si_dpm.c
+++ b/drivers/gpu/drm/radeon/si_dpm.c
@@ -2912,29 +2912,6 @@ static int si_init_smc_spll_table(struct radeon_device *rdev)
         return ret;
 }
 
-struct si_dpm_quirk {
-        u32 chip_vendor;
-        u32 chip_device;
-        u32 subsys_vendor;
-        u32 subsys_device;
-        u32 max_sclk;
-        u32 max_mclk;
-};
-
-/* cards with dpm stability problems */
-static struct si_dpm_quirk si_dpm_quirk_list[] = {
-        /* PITCAIRN - https://bugs.freedesktop.org/show_bug.cgi?id=76490 */
-        { PCI_VENDOR_ID_ATI, 0x6810, 0x1462, 0x3036, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6811, 0x174b, 0xe271, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6811, 0x174b, 0x2015, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6810, 0x174b, 0xe271, 85000, 90000 },
-        { PCI_VENDOR_ID_ATI, 0x6811, 0x1462, 0x2015, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6811, 0x1043, 0x2015, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6811, 0x148c, 0x2015, 0, 120000 },
-        { PCI_VENDOR_ID_ATI, 0x6810, 0x1682, 0x9275, 0, 120000 },
-        { 0, 0, 0, 0 },
-};
-
 static u16 si_get_lower_of_leakage_and_vce_voltage(struct radeon_device *rdev,
                                                    u16 vce_voltage)
 {
@@ -2997,18 +2974,8 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev,
         u32 max_sclk_vddc, max_mclk_vddci, max_mclk_vddc;
         u32 max_sclk = 0, max_mclk = 0;
         int i;
-        struct si_dpm_quirk *p = si_dpm_quirk_list;
 
-        /* limit all SI kickers */
-        if (rdev->family == CHIP_PITCAIRN) {
-                if ((rdev->pdev->revision == 0x81) ||
-                    (rdev->pdev->device == 0x6810) ||
-                    (rdev->pdev->device == 0x6811) ||
-                    (rdev->pdev->device == 0x6816) ||
-                    (rdev->pdev->device == 0x6817) ||
-                    (rdev->pdev->device == 0x6806))
-                        max_mclk = 120000;
-        } else if (rdev->family == CHIP_HAINAN) {
+        if (rdev->family == CHIP_HAINAN) {
                 if ((rdev->pdev->revision == 0x81) ||
                     (rdev->pdev->revision == 0x83) ||
                     (rdev->pdev->revision == 0xC3) ||
@@ -3018,18 +2985,6 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev,
                         max_sclk = 75000;
                 }
         }
-        /* Apply dpm quirks */
-        while (p && p->chip_device != 0) {
-                if (rdev->pdev->vendor == p->chip_vendor &&
-                    rdev->pdev->device == p->chip_device &&
-                    rdev->pdev->subsystem_vendor == p->subsys_vendor &&
-                    rdev->pdev->subsystem_device == p->subsys_device) {
-                        max_sclk = p->max_sclk;
-                        max_mclk = p->max_mclk;
-                        break;
-                }
-                ++p;
-        }
 
         if (rps->vce_active) {
                 rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 86e3b233b722..4562e53c8244 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -163,6 +163,7 @@ static void ttm_bo_release_list(struct kref *list_kref)
 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 {
         struct ttm_bo_device *bdev = bo->bdev;
+        struct ttm_mem_type_manager *man;
 
         lockdep_assert_held(&bo->resv->lock.base);
 
@@ -170,11 +171,13 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 
                 BUG_ON(!list_empty(&bo->lru));
 
-                list_add(&bo->lru, bdev->driver->lru_tail(bo));
+                man = &bdev->man[bo->mem.mem_type];
+                list_add_tail(&bo->lru, &man->lru[bo->priority]);
                 kref_get(&bo->list_kref);
 
                 if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) {
-                        list_add(&bo->swap, bdev->driver->swap_lru_tail(bo));
+                        list_add_tail(&bo->swap,
+                                      &bo->glob->swap_lru[bo->priority]);
                         kref_get(&bo->list_kref);
                 }
         }
@@ -183,12 +186,8 @@ EXPORT_SYMBOL(ttm_bo_add_to_lru);
 
 int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 {
-        struct ttm_bo_device *bdev = bo->bdev;
         int put_count = 0;
 
-        if (bdev->driver->lru_removal)
-                bdev->driver->lru_removal(bo);
-
         if (!list_empty(&bo->swap)) {
                 list_del_init(&bo->swap);
                 ++put_count;
@@ -198,6 +197,11 @@ int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
                 ++put_count;
         }
 
+        /*
+         * TODO: Add a driver hook to delete from
+         * driver-specific LRU's here.
+         */
+
         return put_count;
 }
 
@@ -226,32 +230,16 @@ EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
 
 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
 {
-        struct ttm_bo_device *bdev = bo->bdev;
         int put_count = 0;
 
         lockdep_assert_held(&bo->resv->lock.base);
 
-        if (bdev->driver->lru_removal)
-                bdev->driver->lru_removal(bo);
-
         put_count = ttm_bo_del_from_lru(bo);
         ttm_bo_list_ref_sub(bo, put_count, true);
         ttm_bo_add_to_lru(bo);
 }
 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
 
-struct list_head *ttm_bo_default_lru_tail(struct ttm_buffer_object *bo)
-{
-        return bo->bdev->man[bo->mem.mem_type].lru.prev;
-}
-EXPORT_SYMBOL(ttm_bo_default_lru_tail);
-
-struct list_head *ttm_bo_default_swap_lru_tail(struct ttm_buffer_object *bo)
-{
-        return bo->glob->swap_lru.prev;
-}
-EXPORT_SYMBOL(ttm_bo_default_swap_lru_tail);
-
 /*
  * Call bo->mutex locked.
  */
@@ -342,7 +330,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 
                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
                         if (bdev->driver->move_notify)
-                                bdev->driver->move_notify(bo, mem);
+                                bdev->driver->move_notify(bo, evict, mem);
                         bo->mem = *mem;
                         mem->mm_node = NULL;
                         goto moved;
@@ -350,7 +338,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
         }
 
         if (bdev->driver->move_notify)
-                bdev->driver->move_notify(bo, mem);
+                bdev->driver->move_notify(bo, evict, mem);
 
         if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
             !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
@@ -366,7 +354,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
                         struct ttm_mem_reg tmp_mem = *mem;
                         *mem = bo->mem;
                         bo->mem = tmp_mem;
-                        bdev->driver->move_notify(bo, mem);
+                        bdev->driver->move_notify(bo, false, mem);
                         bo->mem = *mem;
                         *mem = tmp_mem;
                 }
@@ -414,7 +402,7 @@ out_err:
 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 {
         if (bo->bdev->driver->move_notify)
-                bo->bdev->driver->move_notify(bo, NULL);
+                bo->bdev->driver->move_notify(bo, false, NULL);
 
         ttm_tt_destroy(bo->ttm);
         bo->ttm = NULL;
@@ -741,20 +729,27 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
         struct ttm_buffer_object *bo;
         int ret = -EBUSY, put_count;
+        unsigned i;
 
         spin_lock(&glob->lru_lock);
-        list_for_each_entry(bo, &man->lru, lru) {
-                ret = __ttm_bo_reserve(bo, false, true, NULL);
-                if (ret)
-                        continue;
+        for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
+                list_for_each_entry(bo, &man->lru[i], lru) {
+                        ret = __ttm_bo_reserve(bo, false, true, NULL);
+                        if (ret)
+                                continue;
 
-                if (place && !bdev->driver->eviction_valuable(bo, place)) {
-                        __ttm_bo_unreserve(bo);
-                        ret = -EBUSY;
-                        continue;
+                        if (place && !bdev->driver->eviction_valuable(bo,
+                                                                      place)) {
+                                __ttm_bo_unreserve(bo);
+                                ret = -EBUSY;
+                                continue;
+                        }
+
+                        break;
                 }
 
-                break;
+                if (!ret)
+                        break;
         }
 
         if (ret) {
@@ -1197,6 +1192,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
         }
         atomic_inc(&bo->glob->bo_count);
         drm_vma_node_reset(&bo->vma_node);
+        bo->priority = 0;
 
         /*
          * For ttm_bo_type_device buffers, allocate
@@ -1291,29 +1287,27 @@ int ttm_bo_create(struct ttm_bo_device *bdev,
 EXPORT_SYMBOL(ttm_bo_create);
 
 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
-                                        unsigned mem_type, bool allow_errors)
+                                   unsigned mem_type)
 {
         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
         struct ttm_bo_global *glob = bdev->glob;
         struct dma_fence *fence;
         int ret;
+        unsigned i;
 
         /*
          * Can't use standard list traversal since we're unlocking.
          */
 
         spin_lock(&glob->lru_lock);
-        while (!list_empty(&man->lru)) {
-                spin_unlock(&glob->lru_lock);
-                ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false);
-                if (ret) {
-                        if (allow_errors) {
+        for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
+                while (!list_empty(&man->lru[i])) {
+                        spin_unlock(&glob->lru_lock);
+                        ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false);
+                        if (ret)
                                 return ret;
-                        } else {
-                                pr_err("Cleanup eviction failed\n");
-                        }
+                        spin_lock(&glob->lru_lock);
                 }
-                spin_lock(&glob->lru_lock);
         }
         spin_unlock(&glob->lru_lock);
 
@@ -1324,13 +1318,8 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
         if (fence) {
                 ret = dma_fence_wait(fence, false);
                 dma_fence_put(fence);
-                if (ret) {
-                        if (allow_errors) {
-                                return ret;
-                        } else {
-                                pr_err("Cleanup eviction failed\n");
-                        }
-                }
+                if (ret)
+                        return ret;
         }
 
         return 0;
@@ -1359,7 +1348,11 @@ int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
 
         ret = 0;
         if (mem_type > 0) {
-                ttm_bo_force_list_clean(bdev, mem_type, false);
+                ret = ttm_bo_force_list_clean(bdev, mem_type);
+                if (ret) {
+                        pr_err("Cleanup eviction failed\n");
+                        return ret;
+                }
 
                 ret = (*man->func->takedown)(man);
         }
@@ -1382,7 +1375,7 @@ int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
                 return 0;
         }
 
-        return ttm_bo_force_list_clean(bdev, mem_type, true);
+        return ttm_bo_force_list_clean(bdev, mem_type);
 }
 EXPORT_SYMBOL(ttm_bo_evict_mm);
 
@@ -1391,6 +1384,7 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
 {
         int ret = -EINVAL;
         struct ttm_mem_type_manager *man;
+        unsigned i;
 
         BUG_ON(type >= TTM_NUM_MEM_TYPES);
         man = &bdev->man[type];
@@ -1416,7 +1410,8 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
         man->use_type = true;
         man->size = p_size;
 
-        INIT_LIST_HEAD(&man->lru);
+        for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
+                INIT_LIST_HEAD(&man->lru[i]);
         man->move = NULL;
 
         return 0;
@@ -1448,6 +1443,7 @@ int ttm_bo_global_init(struct drm_global_reference *ref)
                 container_of(ref, struct ttm_bo_global_ref, ref);
         struct ttm_bo_global *glob = ref->object;
         int ret;
+        unsigned i;
 
         mutex_init(&glob->device_list_mutex);
         spin_lock_init(&glob->lru_lock);
@@ -1459,7 +1455,8 @@ int ttm_bo_global_init(struct drm_global_reference *ref)
                 goto out_no_drp;
         }
 
-        INIT_LIST_HEAD(&glob->swap_lru);
+        for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
+                INIT_LIST_HEAD(&glob->swap_lru[i]);
         INIT_LIST_HEAD(&glob->device_list);
 
         ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout);
@@ -1518,8 +1515,9 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev)
         if (list_empty(&bdev->ddestroy))
                 TTM_DEBUG("Delayed destroy list was clean\n");
 
-        if (list_empty(&bdev->man[0].lru))
-                TTM_DEBUG("Swap list was clean\n");
+        for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
+                if (list_empty(&bdev->man[0].lru[0]))
+                        TTM_DEBUG("Swap list %d was clean\n", i);
         spin_unlock(&glob->lru_lock);
 
         drm_vma_offset_manager_destroy(&bdev->vma_manager);
@@ -1670,10 +1668,15 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
         struct ttm_buffer_object *bo;
         int ret = -EBUSY;
         int put_count;
+        unsigned i;
 
         spin_lock(&glob->lru_lock);
-        list_for_each_entry(bo, &glob->swap_lru, swap) {
-                ret = __ttm_bo_reserve(bo, false, true, NULL);
+        for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
+                list_for_each_entry(bo, &glob->swap_lru[i], swap) {
+                        ret = __ttm_bo_reserve(bo, false, true, NULL);
+                        if (!ret)
+                                break;
+                }
                 if (!ret)
                         break;
         }
diff --git a/drivers/gpu/drm/virtio/virtgpu_ttm.c b/drivers/gpu/drm/virtio/virtgpu_ttm.c
index 63b3d5d35cf6..9cc7079f7aca 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ttm.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ttm.c
@@ -386,6 +386,7 @@ static int virtio_gpu_bo_move(struct ttm_buffer_object *bo,
 }
 
 static void virtio_gpu_bo_move_notify(struct ttm_buffer_object *tbo,
+                                      bool evict,
                                       struct ttm_mem_reg *new_mem)
 {
         struct virtio_gpu_object *bo;
@@ -433,8 +434,6 @@ static struct ttm_bo_driver virtio_gpu_bo_driver = {
         .io_mem_free = &virtio_gpu_ttm_io_mem_free,
         .move_notify = &virtio_gpu_bo_move_notify,
         .swap_notify = &virtio_gpu_bo_swap_notify,
-        .lru_tail = &ttm_bo_default_lru_tail,
-        .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int virtio_gpu_ttm_init(struct virtio_gpu_device *vgdev)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
index c894a48a74a6..1de96693f2a7 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
@@ -825,6 +825,7 @@ static int vmw_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
  * (currently only resources).
  */
 static void vmw_move_notify(struct ttm_buffer_object *bo,
+                            bool evict,
                             struct ttm_mem_reg *mem)
 {
         vmw_resource_move_notify(bo, mem);
@@ -858,6 +859,4 @@ struct ttm_bo_driver vmw_bo_driver = {
         .fault_reserve_notify = &vmw_ttm_fault_reserve_notify,
         .io_mem_reserve = &vmw_ttm_io_mem_reserve,
         .io_mem_free = &vmw_ttm_io_mem_free,
-        .lru_tail = &ttm_bo_default_lru_tail,
-        .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h
index 652e45be97c8..c356df40ac49 100644
--- a/include/drm/ttm/ttm_bo_api.h
+++ b/include/drm/ttm/ttm_bo_api.h
@@ -215,6 +215,8 @@ struct ttm_buffer_object {
 
         struct drm_vma_offset_node vma_node;
 
+        unsigned priority;
+
         /**
          * Special members that are protected by the reserve lock
          * and the bo::lock when written to. Can be read with
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index cdbdb40eb5bd..7e75fa053473 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -42,6 +42,8 @@
 #include <linux/spinlock.h>
 #include <linux/reservation.h>
 
+#define TTM_MAX_BO_PRIORITY     16
+
 struct ttm_backend_func {
         /**
          * struct ttm_backend_func member bind
@@ -298,7 +300,7 @@ struct ttm_mem_type_manager {
          * Protected by the global->lru_lock.
          */
 
-        struct list_head lru;
+        struct list_head lru[TTM_MAX_BO_PRIORITY];
 
         /*
          * Protected by @move_lock.
@@ -431,9 +433,15 @@ struct ttm_bo_driver {
         int (*verify_access)(struct ttm_buffer_object *bo,
                              struct file *filp);
 
-        /* hook to notify driver about a driver move so it
-         * can do tiling things */
+        /**
+         * Hook to notify driver about a driver move so it
+         * can do tiling things and book-keeping.
+         *
+         * @evict: whether this move is evicting the buffer from the graphics
+         * address space
+         */
         void (*move_notify)(struct ttm_buffer_object *bo,
+                            bool evict,
                             struct ttm_mem_reg *new_mem);
         /* notify the driver we are taking a fault on this BO
          * and have reserved it */
@@ -454,18 +462,6 @@ struct ttm_bo_driver {
                               struct ttm_mem_reg *mem);
         void (*io_mem_free)(struct ttm_bo_device *bdev,
                             struct ttm_mem_reg *mem);
-
-        /**
-         * Optional driver callback for when BO is removed from the LRU.
-         * Called with LRU lock held immediately before the removal.
-         */
-        void (*lru_removal)(struct ttm_buffer_object *bo);
-
-        /**
-         * Return the list_head after which a BO should be inserted in the LRU.
-         */
-        struct list_head *(*lru_tail)(struct ttm_buffer_object *bo);
-        struct list_head *(*swap_lru_tail)(struct ttm_buffer_object *bo);
 };
 
 /**
@@ -512,7 +508,7 @@ struct ttm_bo_global {
         /**
          * Protected by the lru_lock.
          */
-        struct list_head swap_lru;
+        struct list_head swap_lru[TTM_MAX_BO_PRIORITY];
 
         /**
          * Internal protection.
@@ -780,9 +776,6 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man);
 extern void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo);
 extern void ttm_bo_add_to_lru(struct ttm_buffer_object *bo);
 
-struct list_head *ttm_bo_default_lru_tail(struct ttm_buffer_object *bo);
-struct list_head *ttm_bo_default_swap_lru_tail(struct ttm_buffer_object *bo);
-
 /**
  * __ttm_bo_reserve:
  *
diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index 396183628f3c..5797283c2d79 100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -528,6 +528,8 @@ struct drm_amdgpu_cs_chunk_data {
         #define AMDGPU_INFO_VBIOS_SIZE          0x1
         /* Subquery id: Query vbios image */
         #define AMDGPU_INFO_VBIOS_IMAGE         0x2
+/* Query UVD handles */
+#define AMDGPU_INFO_NUM_HANDLES                 0x1C
 
 #define AMDGPU_INFO_MMR_SE_INDEX_SHIFT  0
 #define AMDGPU_INFO_MMR_SE_INDEX_MASK   0xff
@@ -719,6 +721,13 @@ struct drm_amdgpu_info_hw_ip {
         __u32  _pad;
 };
 
+struct drm_amdgpu_info_num_handles {
+        /** Max handles as supported by firmware for UVD */
+        __u32  uvd_max_handles;
+        /** Handles currently in use for UVD */
+        __u32  uvd_used_handles;
+};
+
 #define AMDGPU_VCE_CLOCK_TABLE_ENTRIES          6
 
 struct drm_amdgpu_info_vce_clock_table_entry {