aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/amd/amdgpu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu')
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c23
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c61
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_device.c37
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c20
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c83
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c60
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c25
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c54
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h13
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c37
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/ci_dpm.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cik_ih.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cz_ih.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c87
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/iceland_ih.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/kv_dpm.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c44
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c44
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c20
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c21
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/si_dpm.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/soc15.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/tonga_ih.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c498
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vega10_ih.c11
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vi.c22
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vid.h10
42 files changed, 997 insertions, 368 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index ebfc267467ee..a23b8af95319 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -121,6 +121,7 @@ extern int amdgpu_cntl_sb_buf_per_se;
121extern int amdgpu_param_buf_per_se; 121extern int amdgpu_param_buf_per_se;
122extern int amdgpu_job_hang_limit; 122extern int amdgpu_job_hang_limit;
123extern int amdgpu_lbpw; 123extern int amdgpu_lbpw;
124extern int amdgpu_compute_multipipe;
124 125
125#ifdef CONFIG_DRM_AMDGPU_SI 126#ifdef CONFIG_DRM_AMDGPU_SI
126extern int amdgpu_si_support; 127extern int amdgpu_si_support;
@@ -1310,6 +1311,8 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
1310int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, 1311int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
1311 struct drm_file *filp); 1312 struct drm_file *filp);
1312int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); 1313int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
1314int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
1315 struct drm_file *filp);
1313int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); 1316int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
1314int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data, 1317int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
1315 struct drm_file *filp); 1318 struct drm_file *filp);
@@ -1524,7 +1527,6 @@ struct amdgpu_device {
1524 1527
1525 /* powerplay */ 1528 /* powerplay */
1526 struct amd_powerplay powerplay; 1529 struct amd_powerplay powerplay;
1527 bool pp_enabled;
1528 bool pp_force_state_enabled; 1530 bool pp_force_state_enabled;
1529 1531
1530 /* dpm */ 1532 /* dpm */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index dc7e25cce741..47d1c132ac40 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -338,6 +338,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
338 struct cik_mqd *m; 338 struct cik_mqd *m;
339 uint32_t *mqd_hqd; 339 uint32_t *mqd_hqd;
340 uint32_t reg, wptr_val, data; 340 uint32_t reg, wptr_val, data;
341 bool valid_wptr = false;
341 342
342 m = get_mqd(mqd); 343 m = get_mqd(mqd);
343 344
@@ -356,7 +357,14 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
356 CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); 357 CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
357 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data); 358 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
358 359
359 if (read_user_wptr(mm, wptr, wptr_val)) 360 /* read_user_ptr may take the mm->mmap_sem.
361 * release srbm_mutex to avoid circular dependency between
362 * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
363 */
364 release_queue(kgd);
365 valid_wptr = read_user_wptr(mm, wptr, wptr_val);
366 acquire_queue(kgd, pipe_id, queue_id);
367 if (valid_wptr)
360 WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask); 368 WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
361 369
362 data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); 370 data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index c678c69936a0..056929b8ccd0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -292,6 +292,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
292 struct vi_mqd *m; 292 struct vi_mqd *m;
293 uint32_t *mqd_hqd; 293 uint32_t *mqd_hqd;
294 uint32_t reg, wptr_val, data; 294 uint32_t reg, wptr_val, data;
295 bool valid_wptr = false;
295 296
296 m = get_mqd(mqd); 297 m = get_mqd(mqd);
297 298
@@ -339,7 +340,14 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
339 CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); 340 CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
340 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data); 341 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
341 342
342 if (read_user_wptr(mm, wptr, wptr_val)) 343 /* read_user_ptr may take the mm->mmap_sem.
344 * release srbm_mutex to avoid circular dependency between
345 * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
346 */
347 release_queue(kgd);
348 valid_wptr = read_user_wptr(mm, wptr, wptr_val);
349 acquire_queue(kgd, pipe_id, queue_id);
350 if (valid_wptr)
343 WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask); 351 WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
344 352
345 data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); 353 data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index 383204e911a4..a7afe553e0a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -42,6 +42,28 @@ struct amdgpu_cgs_device {
42 struct amdgpu_device *adev = \ 42 struct amdgpu_device *adev = \
43 ((struct amdgpu_cgs_device *)cgs_device)->adev 43 ((struct amdgpu_cgs_device *)cgs_device)->adev
44 44
45static void *amdgpu_cgs_register_pp_handle(struct cgs_device *cgs_device,
46 int (*call_back_func)(struct amd_pp_init *, void **))
47{
48 CGS_FUNC_ADEV;
49 struct amd_pp_init pp_init;
50 struct amd_powerplay *amd_pp;
51
52 if (call_back_func == NULL)
53 return NULL;
54
55 amd_pp = &(adev->powerplay);
56 pp_init.chip_family = adev->family;
57 pp_init.chip_id = adev->asic_type;
58 pp_init.pm_en = (amdgpu_dpm != 0 && !amdgpu_sriov_vf(adev)) ? true : false;
59 pp_init.feature_mask = amdgpu_pp_feature_mask;
60 pp_init.device = cgs_device;
61 if (call_back_func(&pp_init, &(amd_pp->pp_handle)))
62 return NULL;
63
64 return adev->powerplay.pp_handle;
65}
66
45static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device, 67static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
46 enum cgs_gpu_mem_type type, 68 enum cgs_gpu_mem_type type,
47 uint64_t size, uint64_t align, 69 uint64_t size, uint64_t align,
@@ -1179,6 +1201,7 @@ static const struct cgs_ops amdgpu_cgs_ops = {
1179 .is_virtualization_enabled = amdgpu_cgs_is_virtualization_enabled, 1201 .is_virtualization_enabled = amdgpu_cgs_is_virtualization_enabled,
1180 .enter_safe_mode = amdgpu_cgs_enter_safe_mode, 1202 .enter_safe_mode = amdgpu_cgs_enter_safe_mode,
1181 .lock_grbm_idx = amdgpu_cgs_lock_grbm_idx, 1203 .lock_grbm_idx = amdgpu_cgs_lock_grbm_idx,
1204 .register_pp_handle = amdgpu_cgs_register_pp_handle,
1182}; 1205};
1183 1206
1184static const struct cgs_os_ops amdgpu_cgs_os_ops = { 1207static const struct cgs_os_ops amdgpu_cgs_os_ops = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index c6a214f1e991..ab83dfcabb41 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -25,6 +25,7 @@
25 * Jerome Glisse <glisse@freedesktop.org> 25 * Jerome Glisse <glisse@freedesktop.org>
26 */ 26 */
27#include <linux/pagemap.h> 27#include <linux/pagemap.h>
28#include <linux/sync_file.h>
28#include <drm/drmP.h> 29#include <drm/drmP.h>
29#include <drm/amdgpu_drm.h> 30#include <drm/amdgpu_drm.h>
30#include <drm/drm_syncobj.h> 31#include <drm/drm_syncobj.h>
@@ -1330,6 +1331,66 @@ static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,
1330 return fence; 1331 return fence;
1331} 1332}
1332 1333
1334int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
1335 struct drm_file *filp)
1336{
1337 struct amdgpu_device *adev = dev->dev_private;
1338 struct amdgpu_fpriv *fpriv = filp->driver_priv;
1339 union drm_amdgpu_fence_to_handle *info = data;
1340 struct dma_fence *fence;
1341 struct drm_syncobj *syncobj;
1342 struct sync_file *sync_file;
1343 int fd, r;
1344
1345 if (amdgpu_kms_vram_lost(adev, fpriv))
1346 return -ENODEV;
1347
1348 fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence);
1349 if (IS_ERR(fence))
1350 return PTR_ERR(fence);
1351
1352 switch (info->in.what) {
1353 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ:
1354 r = drm_syncobj_create(&syncobj, 0, fence);
1355 dma_fence_put(fence);
1356 if (r)
1357 return r;
1358 r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle);
1359 drm_syncobj_put(syncobj);
1360 return r;
1361
1362 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD:
1363 r = drm_syncobj_create(&syncobj, 0, fence);
1364 dma_fence_put(fence);
1365 if (r)
1366 return r;
1367 r = drm_syncobj_get_fd(syncobj, (int*)&info->out.handle);
1368 drm_syncobj_put(syncobj);
1369 return r;
1370
1371 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD:
1372 fd = get_unused_fd_flags(O_CLOEXEC);
1373 if (fd < 0) {
1374 dma_fence_put(fence);
1375 return fd;
1376 }
1377
1378 sync_file = sync_file_create(fence);
1379 dma_fence_put(fence);
1380 if (!sync_file) {
1381 put_unused_fd(fd);
1382 return -ENOMEM;
1383 }
1384
1385 fd_install(fd, sync_file->file);
1386 info->out.handle = fd;
1387 return 0;
1388
1389 default:
1390 return -EINVAL;
1391 }
1392}
1393
1333/** 1394/**
1334 * amdgpu_cs_wait_all_fence - wait on all fences to signal 1395 * amdgpu_cs_wait_all_fence - wait on all fences to signal
1335 * 1396 *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 3e84ddf9e3b5..1949d8aedf49 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -56,6 +56,7 @@
56#include "amdgpu_vf_error.h" 56#include "amdgpu_vf_error.h"
57 57
58#include "amdgpu_amdkfd.h" 58#include "amdgpu_amdkfd.h"
59#include "amdgpu_pm.h"
59 60
60MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); 61MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
61MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); 62MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
@@ -1603,6 +1604,7 @@ static int amdgpu_init(struct amdgpu_device *adev)
1603 return r; 1604 return r;
1604 } 1605 }
1605 adev->ip_blocks[i].status.sw = true; 1606 adev->ip_blocks[i].status.sw = true;
1607
1606 /* need to do gmc hw init early so we can allocate gpu mem */ 1608 /* need to do gmc hw init early so we can allocate gpu mem */
1607 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 1609 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1608 r = amdgpu_vram_scratch_init(adev); 1610 r = amdgpu_vram_scratch_init(adev);
@@ -1633,6 +1635,11 @@ static int amdgpu_init(struct amdgpu_device *adev)
1633 } 1635 }
1634 } 1636 }
1635 1637
1638 mutex_lock(&adev->firmware.mutex);
1639 if (amdgpu_ucode_init_bo(adev))
1640 adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
1641 mutex_unlock(&adev->firmware.mutex);
1642
1636 for (i = 0; i < adev->num_ip_blocks; i++) { 1643 for (i = 0; i < adev->num_ip_blocks; i++) {
1637 if (!adev->ip_blocks[i].status.sw) 1644 if (!adev->ip_blocks[i].status.sw)
1638 continue; 1645 continue;
@@ -1768,6 +1775,8 @@ static int amdgpu_fini(struct amdgpu_device *adev)
1768 1775
1769 adev->ip_blocks[i].status.hw = false; 1776 adev->ip_blocks[i].status.hw = false;
1770 } 1777 }
1778 if (adev->firmware.load_type != AMDGPU_FW_LOAD_DIRECT)
1779 amdgpu_ucode_fini_bo(adev);
1771 1780
1772 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 1781 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1773 if (!adev->ip_blocks[i].status.sw) 1782 if (!adev->ip_blocks[i].status.sw)
@@ -2040,6 +2049,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2040 mutex_init(&adev->srbm_mutex); 2049 mutex_init(&adev->srbm_mutex);
2041 mutex_init(&adev->grbm_idx_mutex); 2050 mutex_init(&adev->grbm_idx_mutex);
2042 mutex_init(&adev->mn_lock); 2051 mutex_init(&adev->mn_lock);
2052 mutex_init(&adev->virt.vf_errors.lock);
2043 hash_init(adev->mn_hash); 2053 hash_init(adev->mn_hash);
2044 2054
2045 amdgpu_check_arguments(adev); 2055 amdgpu_check_arguments(adev);
@@ -2125,7 +2135,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2125 r = amdgpu_atombios_init(adev); 2135 r = amdgpu_atombios_init(adev);
2126 if (r) { 2136 if (r) {
2127 dev_err(adev->dev, "amdgpu_atombios_init failed\n"); 2137 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2128 amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0); 2138 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2129 goto failed; 2139 goto failed;
2130 } 2140 }
2131 2141
@@ -2136,7 +2146,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2136 if (amdgpu_vpost_needed(adev)) { 2146 if (amdgpu_vpost_needed(adev)) {
2137 if (!adev->bios) { 2147 if (!adev->bios) {
2138 dev_err(adev->dev, "no vBIOS found\n"); 2148 dev_err(adev->dev, "no vBIOS found\n");
2139 amdgpu_vf_error_put(AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); 2149 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2140 r = -EINVAL; 2150 r = -EINVAL;
2141 goto failed; 2151 goto failed;
2142 } 2152 }
@@ -2144,7 +2154,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2144 r = amdgpu_atom_asic_init(adev->mode_info.atom_context); 2154 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2145 if (r) { 2155 if (r) {
2146 dev_err(adev->dev, "gpu post error!\n"); 2156 dev_err(adev->dev, "gpu post error!\n");
2147 amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_POST_ERROR, 0, 0); 2157 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_POST_ERROR, 0, 0);
2148 goto failed; 2158 goto failed;
2149 } 2159 }
2150 } else { 2160 } else {
@@ -2156,7 +2166,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2156 r = amdgpu_atomfirmware_get_clock_info(adev); 2166 r = amdgpu_atomfirmware_get_clock_info(adev);
2157 if (r) { 2167 if (r) {
2158 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); 2168 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
2159 amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 2169 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2160 goto failed; 2170 goto failed;
2161 } 2171 }
2162 } else { 2172 } else {
@@ -2164,7 +2174,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2164 r = amdgpu_atombios_get_clock_info(adev); 2174 r = amdgpu_atombios_get_clock_info(adev);
2165 if (r) { 2175 if (r) {
2166 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); 2176 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2167 amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 2177 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2168 goto failed; 2178 goto failed;
2169 } 2179 }
2170 /* init i2c buses */ 2180 /* init i2c buses */
@@ -2175,7 +2185,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2175 r = amdgpu_fence_driver_init(adev); 2185 r = amdgpu_fence_driver_init(adev);
2176 if (r) { 2186 if (r) {
2177 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n"); 2187 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
2178 amdgpu_vf_error_put(AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); 2188 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2179 goto failed; 2189 goto failed;
2180 } 2190 }
2181 2191
@@ -2185,7 +2195,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2185 r = amdgpu_init(adev); 2195 r = amdgpu_init(adev);
2186 if (r) { 2196 if (r) {
2187 dev_err(adev->dev, "amdgpu_init failed\n"); 2197 dev_err(adev->dev, "amdgpu_init failed\n");
2188 amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); 2198 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2189 amdgpu_fini(adev); 2199 amdgpu_fini(adev);
2190 goto failed; 2200 goto failed;
2191 } 2201 }
@@ -2205,7 +2215,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2205 r = amdgpu_ib_pool_init(adev); 2215 r = amdgpu_ib_pool_init(adev);
2206 if (r) { 2216 if (r) {
2207 dev_err(adev->dev, "IB initialization failed (%d).\n", r); 2217 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2208 amdgpu_vf_error_put(AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); 2218 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2209 goto failed; 2219 goto failed;
2210 } 2220 }
2211 2221
@@ -2215,6 +2225,10 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2215 2225
2216 amdgpu_fbdev_init(adev); 2226 amdgpu_fbdev_init(adev);
2217 2227
2228 r = amdgpu_pm_sysfs_init(adev);
2229 if (r)
2230 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
2231
2218 r = amdgpu_gem_debugfs_init(adev); 2232 r = amdgpu_gem_debugfs_init(adev);
2219 if (r) 2233 if (r)
2220 DRM_ERROR("registering gem debugfs failed (%d).\n", r); 2234 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
@@ -2254,7 +2268,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2254 r = amdgpu_late_init(adev); 2268 r = amdgpu_late_init(adev);
2255 if (r) { 2269 if (r) {
2256 dev_err(adev->dev, "amdgpu_late_init failed\n"); 2270 dev_err(adev->dev, "amdgpu_late_init failed\n");
2257 amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); 2271 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2258 goto failed; 2272 goto failed;
2259 } 2273 }
2260 2274
@@ -2311,6 +2325,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
2311 iounmap(adev->rmmio); 2325 iounmap(adev->rmmio);
2312 adev->rmmio = NULL; 2326 adev->rmmio = NULL;
2313 amdgpu_doorbell_fini(adev); 2327 amdgpu_doorbell_fini(adev);
2328 amdgpu_pm_sysfs_fini(adev);
2314 amdgpu_debugfs_regs_cleanup(adev); 2329 amdgpu_debugfs_regs_cleanup(adev);
2315} 2330}
2316 2331
@@ -2936,7 +2951,7 @@ out:
2936 } 2951 }
2937 } else { 2952 } else {
2938 dev_err(adev->dev, "asic resume failed (%d).\n", r); 2953 dev_err(adev->dev, "asic resume failed (%d).\n", r);
2939 amdgpu_vf_error_put(AMDGIM_ERROR_VF_ASIC_RESUME_FAIL, 0, r); 2954 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ASIC_RESUME_FAIL, 0, r);
2940 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 2955 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2941 if (adev->rings[i] && adev->rings[i]->sched.thread) { 2956 if (adev->rings[i] && adev->rings[i]->sched.thread) {
2942 kthread_unpark(adev->rings[i]->sched.thread); 2957 kthread_unpark(adev->rings[i]->sched.thread);
@@ -2950,7 +2965,7 @@ out:
2950 if (r) { 2965 if (r) {
2951 /* bad news, how to tell it to userspace ? */ 2966 /* bad news, how to tell it to userspace ? */
2952 dev_info(adev->dev, "GPU reset failed\n"); 2967 dev_info(adev->dev, "GPU reset failed\n");
2953 amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r); 2968 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
2954 } 2969 }
2955 else { 2970 else {
2956 dev_info(adev->dev, "GPU reset successed!\n"); 2971 dev_info(adev->dev, "GPU reset successed!\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index f79f9ea58b17..7279fb5c3abc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -356,6 +356,10 @@ enum amdgpu_pcie_gen {
356 ((adev)->powerplay.pp_funcs->switch_power_profile(\ 356 ((adev)->powerplay.pp_funcs->switch_power_profile(\
357 (adev)->powerplay.pp_handle, type)) 357 (adev)->powerplay.pp_handle, type))
358 358
359#define amdgpu_dpm_set_clockgating_by_smu(adev, msg_id) \
360 ((adev)->powerplay.pp_funcs->set_clockgating_by_smu(\
361 (adev)->powerplay.pp_handle, msg_id))
362
359struct amdgpu_dpm { 363struct amdgpu_dpm {
360 struct amdgpu_ps *ps; 364 struct amdgpu_ps *ps;
361 /* number of valid power states */ 365 /* number of valid power states */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 4f98960e47f9..ad02d3fbb44c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -70,9 +70,10 @@
70 * - 3.18.0 - Export gpu always on cu bitmap 70 * - 3.18.0 - Export gpu always on cu bitmap
71 * - 3.19.0 - Add support for UVD MJPEG decode 71 * - 3.19.0 - Add support for UVD MJPEG decode
72 * - 3.20.0 - Add support for local BOs 72 * - 3.20.0 - Add support for local BOs
73 * - 3.21.0 - Add DRM_AMDGPU_FENCE_TO_HANDLE ioctl
73 */ 74 */
74#define KMS_DRIVER_MAJOR 3 75#define KMS_DRIVER_MAJOR 3
75#define KMS_DRIVER_MINOR 20 76#define KMS_DRIVER_MINOR 21
76#define KMS_DRIVER_PATCHLEVEL 0 77#define KMS_DRIVER_PATCHLEVEL 0
77 78
78int amdgpu_vram_limit = 0; 79int amdgpu_vram_limit = 0;
@@ -122,6 +123,7 @@ int amdgpu_cntl_sb_buf_per_se = 0;
122int amdgpu_param_buf_per_se = 0; 123int amdgpu_param_buf_per_se = 0;
123int amdgpu_job_hang_limit = 0; 124int amdgpu_job_hang_limit = 0;
124int amdgpu_lbpw = -1; 125int amdgpu_lbpw = -1;
126int amdgpu_compute_multipipe = -1;
125 127
126MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes"); 128MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
127module_param_named(vramlimit, amdgpu_vram_limit, int, 0600); 129module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -265,6 +267,9 @@ module_param_named(job_hang_limit, amdgpu_job_hang_limit, int ,0444);
265MODULE_PARM_DESC(lbpw, "Load Balancing Per Watt (LBPW) support (1 = enable, 0 = disable, -1 = auto)"); 267MODULE_PARM_DESC(lbpw, "Load Balancing Per Watt (LBPW) support (1 = enable, 0 = disable, -1 = auto)");
266module_param_named(lbpw, amdgpu_lbpw, int, 0444); 268module_param_named(lbpw, amdgpu_lbpw, int, 0444);
267 269
270MODULE_PARM_DESC(compute_multipipe, "Force compute queues to be spread across pipes (1 = enable, 0 = disable, -1 = auto)");
271module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444);
272
268#ifdef CONFIG_DRM_AMDGPU_SI 273#ifdef CONFIG_DRM_AMDGPU_SI
269 274
270#if defined(CONFIG_DRM_RADEON) || defined(CONFIG_DRM_RADEON_MODULE) 275#if defined(CONFIG_DRM_RADEON) || defined(CONFIG_DRM_RADEON_MODULE)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 4fcd98e65998..83435ccbad44 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -109,9 +109,26 @@ void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_s
109 } 109 }
110} 110}
111 111
112static bool amdgpu_gfx_is_multipipe_capable(struct amdgpu_device *adev)
113{
114 if (amdgpu_compute_multipipe != -1) {
115 DRM_INFO("amdgpu: forcing compute pipe policy %d\n",
116 amdgpu_compute_multipipe);
117 return amdgpu_compute_multipipe == 1;
118 }
119
120 /* FIXME: spreading the queues across pipes causes perf regressions
121 * on POLARIS11 compute workloads */
122 if (adev->asic_type == CHIP_POLARIS11)
123 return false;
124
125 return adev->gfx.mec.num_mec > 1;
126}
127
112void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) 128void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
113{ 129{
114 int i, queue, pipe, mec; 130 int i, queue, pipe, mec;
131 bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev);
115 132
116 /* policy for amdgpu compute queue ownership */ 133 /* policy for amdgpu compute queue ownership */
117 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { 134 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
@@ -125,8 +142,7 @@ void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
125 if (mec >= adev->gfx.mec.num_mec) 142 if (mec >= adev->gfx.mec.num_mec)
126 break; 143 break;
127 144
128 /* FIXME: spreading the queues across pipes causes perf regressions */ 145 if (multipipe_policy) {
129 if (0) {
130 /* policy: amdgpu owns the first two queues of the first MEC */ 146 /* policy: amdgpu owns the first two queues of the first MEC */
131 if (mec == 0 && queue < 2) 147 if (mec == 0 && queue < 2)
132 set_bit(i, adev->gfx.mec.queue_bitmap); 148 set_bit(i, adev->gfx.mec.queue_bitmap);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 4fd06f8d9768..51841259e23f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -1024,6 +1024,7 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
1024 DRM_IOCTL_DEF_DRV(AMDGPU_CTX, amdgpu_ctx_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 1024 DRM_IOCTL_DEF_DRV(AMDGPU_CTX, amdgpu_ctx_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
1025 DRM_IOCTL_DEF_DRV(AMDGPU_VM, amdgpu_vm_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 1025 DRM_IOCTL_DEF_DRV(AMDGPU_VM, amdgpu_vm_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
1026 DRM_IOCTL_DEF_DRV(AMDGPU_BO_LIST, amdgpu_bo_list_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 1026 DRM_IOCTL_DEF_DRV(AMDGPU_BO_LIST, amdgpu_bo_list_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
1027 DRM_IOCTL_DEF_DRV(AMDGPU_FENCE_TO_HANDLE, amdgpu_cs_fence_to_handle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
1027 /* KMS */ 1028 /* KMS */
1028 DRM_IOCTL_DEF_DRV(AMDGPU_GEM_MMAP, amdgpu_gem_mmap_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 1029 DRM_IOCTL_DEF_DRV(AMDGPU_GEM_MMAP, amdgpu_gem_mmap_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
1029 DRM_IOCTL_DEF_DRV(AMDGPU_GEM_WAIT_IDLE, amdgpu_gem_wait_idle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 1030 DRM_IOCTL_DEF_DRV(AMDGPU_GEM_WAIT_IDLE, amdgpu_gem_wait_idle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index f6ce52956e6d..a59e04f3eeba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -64,10 +64,6 @@ static const struct cg_flag_name clocks[] = {
64 64
65void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev) 65void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev)
66{ 66{
67 if (adev->pp_enabled)
68 /* TODO */
69 return;
70
71 if (adev->pm.dpm_enabled) { 67 if (adev->pm.dpm_enabled) {
72 mutex_lock(&adev->pm.mutex); 68 mutex_lock(&adev->pm.mutex);
73 if (power_supply_is_system_supplied() > 0) 69 if (power_supply_is_system_supplied() > 0)
@@ -118,7 +114,7 @@ static ssize_t amdgpu_set_dpm_state(struct device *dev,
118 goto fail; 114 goto fail;
119 } 115 }
120 116
121 if (adev->pp_enabled) { 117 if (adev->powerplay.pp_funcs->dispatch_tasks) {
122 amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_ENABLE_USER_STATE, &state, NULL); 118 amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_ENABLE_USER_STATE, &state, NULL);
123 } else { 119 } else {
124 mutex_lock(&adev->pm.mutex); 120 mutex_lock(&adev->pm.mutex);
@@ -303,7 +299,8 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev,
303 299
304 if (strlen(buf) == 1) 300 if (strlen(buf) == 1)
305 adev->pp_force_state_enabled = false; 301 adev->pp_force_state_enabled = false;
306 else if (adev->pp_enabled) { 302 else if (adev->powerplay.pp_funcs->dispatch_tasks &&
303 adev->powerplay.pp_funcs->get_pp_num_states) {
307 struct pp_states_info data; 304 struct pp_states_info data;
308 305
309 ret = kstrtoul(buf, 0, &idx); 306 ret = kstrtoul(buf, 0, &idx);
@@ -531,7 +528,7 @@ static ssize_t amdgpu_set_pp_sclk_od(struct device *dev,
531 if (adev->powerplay.pp_funcs->set_sclk_od) 528 if (adev->powerplay.pp_funcs->set_sclk_od)
532 amdgpu_dpm_set_sclk_od(adev, (uint32_t)value); 529 amdgpu_dpm_set_sclk_od(adev, (uint32_t)value);
533 530
534 if (adev->pp_enabled) { 531 if (adev->powerplay.pp_funcs->dispatch_tasks) {
535 amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL, NULL); 532 amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL, NULL);
536 } else { 533 } else {
537 adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps; 534 adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
@@ -575,7 +572,7 @@ static ssize_t amdgpu_set_pp_mclk_od(struct device *dev,
575 if (adev->powerplay.pp_funcs->set_mclk_od) 572 if (adev->powerplay.pp_funcs->set_mclk_od)
576 amdgpu_dpm_set_mclk_od(adev, (uint32_t)value); 573 amdgpu_dpm_set_mclk_od(adev, (uint32_t)value);
577 574
578 if (adev->pp_enabled) { 575 if (adev->powerplay.pp_funcs->dispatch_tasks) {
579 amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL, NULL); 576 amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL, NULL);
580 } else { 577 } else {
581 adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps; 578 adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
@@ -959,9 +956,6 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
959 attr == &sensor_dev_attr_pwm1_min.dev_attr.attr)) 956 attr == &sensor_dev_attr_pwm1_min.dev_attr.attr))
960 return 0; 957 return 0;
961 958
962 if (adev->pp_enabled)
963 return effective_mode;
964
965 /* Skip fan attributes if fan is not present */ 959 /* Skip fan attributes if fan is not present */
966 if (adev->pm.no_fan && 960 if (adev->pm.no_fan &&
967 (attr == &sensor_dev_attr_pwm1.dev_attr.attr || 961 (attr == &sensor_dev_attr_pwm1.dev_attr.attr ||
@@ -1317,6 +1311,9 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
1317 if (adev->pm.sysfs_initialized) 1311 if (adev->pm.sysfs_initialized)
1318 return 0; 1312 return 0;
1319 1313
1314 if (adev->pm.dpm_enabled == 0)
1315 return 0;
1316
1320 if (adev->powerplay.pp_funcs->get_temperature == NULL) 1317 if (adev->powerplay.pp_funcs->get_temperature == NULL)
1321 return 0; 1318 return 0;
1322 1319
@@ -1341,27 +1338,26 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
1341 return ret; 1338 return ret;
1342 } 1339 }
1343 1340
1344 if (adev->pp_enabled) { 1341
1345 ret = device_create_file(adev->dev, &dev_attr_pp_num_states); 1342 ret = device_create_file(adev->dev, &dev_attr_pp_num_states);
1346 if (ret) { 1343 if (ret) {
1347 DRM_ERROR("failed to create device file pp_num_states\n"); 1344 DRM_ERROR("failed to create device file pp_num_states\n");
1348 return ret; 1345 return ret;
1349 } 1346 }
1350 ret = device_create_file(adev->dev, &dev_attr_pp_cur_state); 1347 ret = device_create_file(adev->dev, &dev_attr_pp_cur_state);
1351 if (ret) { 1348 if (ret) {
1352 DRM_ERROR("failed to create device file pp_cur_state\n"); 1349 DRM_ERROR("failed to create device file pp_cur_state\n");
1353 return ret; 1350 return ret;
1354 } 1351 }
1355 ret = device_create_file(adev->dev, &dev_attr_pp_force_state); 1352 ret = device_create_file(adev->dev, &dev_attr_pp_force_state);
1356 if (ret) { 1353 if (ret) {
1357 DRM_ERROR("failed to create device file pp_force_state\n"); 1354 DRM_ERROR("failed to create device file pp_force_state\n");
1358 return ret; 1355 return ret;
1359 } 1356 }
1360 ret = device_create_file(adev->dev, &dev_attr_pp_table); 1357 ret = device_create_file(adev->dev, &dev_attr_pp_table);
1361 if (ret) { 1358 if (ret) {
1362 DRM_ERROR("failed to create device file pp_table\n"); 1359 DRM_ERROR("failed to create device file pp_table\n");
1363 return ret; 1360 return ret;
1364 }
1365 } 1361 }
1366 1362
1367 ret = device_create_file(adev->dev, &dev_attr_pp_dpm_sclk); 1363 ret = device_create_file(adev->dev, &dev_attr_pp_dpm_sclk);
@@ -1417,16 +1413,19 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
1417 1413
1418void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev) 1414void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev)
1419{ 1415{
1416 if (adev->pm.dpm_enabled == 0)
1417 return;
1418
1420 if (adev->pm.int_hwmon_dev) 1419 if (adev->pm.int_hwmon_dev)
1421 hwmon_device_unregister(adev->pm.int_hwmon_dev); 1420 hwmon_device_unregister(adev->pm.int_hwmon_dev);
1422 device_remove_file(adev->dev, &dev_attr_power_dpm_state); 1421 device_remove_file(adev->dev, &dev_attr_power_dpm_state);
1423 device_remove_file(adev->dev, &dev_attr_power_dpm_force_performance_level); 1422 device_remove_file(adev->dev, &dev_attr_power_dpm_force_performance_level);
1424 if (adev->pp_enabled) { 1423
1425 device_remove_file(adev->dev, &dev_attr_pp_num_states); 1424 device_remove_file(adev->dev, &dev_attr_pp_num_states);
1426 device_remove_file(adev->dev, &dev_attr_pp_cur_state); 1425 device_remove_file(adev->dev, &dev_attr_pp_cur_state);
1427 device_remove_file(adev->dev, &dev_attr_pp_force_state); 1426 device_remove_file(adev->dev, &dev_attr_pp_force_state);
1428 device_remove_file(adev->dev, &dev_attr_pp_table); 1427 device_remove_file(adev->dev, &dev_attr_pp_table);
1429 } 1428
1430 device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk); 1429 device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk);
1431 device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk); 1430 device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk);
1432 device_remove_file(adev->dev, &dev_attr_pp_dpm_pcie); 1431 device_remove_file(adev->dev, &dev_attr_pp_dpm_pcie);
@@ -1457,7 +1456,7 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
1457 amdgpu_fence_wait_empty(ring); 1456 amdgpu_fence_wait_empty(ring);
1458 } 1457 }
1459 1458
1460 if (adev->pp_enabled) { 1459 if (adev->powerplay.pp_funcs->dispatch_tasks) {
1461 amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, NULL, NULL); 1460 amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, NULL, NULL);
1462 } else { 1461 } else {
1463 mutex_lock(&adev->pm.mutex); 1462 mutex_lock(&adev->pm.mutex);
@@ -1592,15 +1591,15 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data)
1592 if ((adev->flags & AMD_IS_PX) && 1591 if ((adev->flags & AMD_IS_PX) &&
1593 (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) { 1592 (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) {
1594 seq_printf(m, "PX asic powered off\n"); 1593 seq_printf(m, "PX asic powered off\n");
1595 } else if (adev->pp_enabled) { 1594 } else if (adev->powerplay.pp_funcs->debugfs_print_current_performance_level) {
1596 return amdgpu_debugfs_pm_info_pp(m, adev);
1597 } else {
1598 mutex_lock(&adev->pm.mutex); 1595 mutex_lock(&adev->pm.mutex);
1599 if (adev->powerplay.pp_funcs->debugfs_print_current_performance_level) 1596 if (adev->powerplay.pp_funcs->debugfs_print_current_performance_level)
1600 adev->powerplay.pp_funcs->debugfs_print_current_performance_level(adev, m); 1597 adev->powerplay.pp_funcs->debugfs_print_current_performance_level(adev, m);
1601 else 1598 else
1602 seq_printf(m, "Debugfs support not implemented for this asic\n"); 1599 seq_printf(m, "Debugfs support not implemented for this asic\n");
1603 mutex_unlock(&adev->pm.mutex); 1600 mutex_unlock(&adev->pm.mutex);
1601 } else {
1602 return amdgpu_debugfs_pm_info_pp(m, adev);
1604 } 1603 }
1605 1604
1606 return 0; 1605 return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
index 2d2f0960b025..3b42f407971d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
@@ -34,24 +34,6 @@
34#include "cik_dpm.h" 34#include "cik_dpm.h"
35#include "vi_dpm.h" 35#include "vi_dpm.h"
36 36
37static int amdgpu_create_pp_handle(struct amdgpu_device *adev)
38{
39 struct amd_pp_init pp_init;
40 struct amd_powerplay *amd_pp;
41 int ret;
42
43 amd_pp = &(adev->powerplay);
44 pp_init.chip_family = adev->family;
45 pp_init.chip_id = adev->asic_type;
46 pp_init.pm_en = (amdgpu_dpm != 0 && !amdgpu_sriov_vf(adev)) ? true : false;
47 pp_init.feature_mask = amdgpu_pp_feature_mask;
48 pp_init.device = amdgpu_cgs_create_device(adev);
49 ret = amd_powerplay_create(&pp_init, &(amd_pp->pp_handle));
50 if (ret)
51 return -EINVAL;
52 return 0;
53}
54
55static int amdgpu_pp_early_init(void *handle) 37static int amdgpu_pp_early_init(void *handle)
56{ 38{
57 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 39 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -59,7 +41,6 @@ static int amdgpu_pp_early_init(void *handle)
59 int ret = 0; 41 int ret = 0;
60 42
61 amd_pp = &(adev->powerplay); 43 amd_pp = &(adev->powerplay);
62 adev->pp_enabled = false;
63 amd_pp->pp_handle = (void *)adev; 44 amd_pp->pp_handle = (void *)adev;
64 45
65 switch (adev->asic_type) { 46 switch (adev->asic_type) {
@@ -73,9 +54,7 @@ static int amdgpu_pp_early_init(void *handle)
73 case CHIP_STONEY: 54 case CHIP_STONEY:
74 case CHIP_VEGA10: 55 case CHIP_VEGA10:
75 case CHIP_RAVEN: 56 case CHIP_RAVEN:
76 adev->pp_enabled = true; 57 amd_pp->cgs_device = amdgpu_cgs_create_device(adev);
77 if (amdgpu_create_pp_handle(adev))
78 return -EINVAL;
79 amd_pp->ip_funcs = &pp_ip_funcs; 58 amd_pp->ip_funcs = &pp_ip_funcs;
80 amd_pp->pp_funcs = &pp_dpm_funcs; 59 amd_pp->pp_funcs = &pp_dpm_funcs;
81 break; 60 break;
@@ -97,9 +76,7 @@ static int amdgpu_pp_early_init(void *handle)
97 amd_pp->ip_funcs = &ci_dpm_ip_funcs; 76 amd_pp->ip_funcs = &ci_dpm_ip_funcs;
98 amd_pp->pp_funcs = &ci_dpm_funcs; 77 amd_pp->pp_funcs = &ci_dpm_funcs;
99 } else { 78 } else {
100 adev->pp_enabled = true; 79 amd_pp->cgs_device = amdgpu_cgs_create_device(adev);
101 if (amdgpu_create_pp_handle(adev))
102 return -EINVAL;
103 amd_pp->ip_funcs = &pp_ip_funcs; 80 amd_pp->ip_funcs = &pp_ip_funcs;
104 amd_pp->pp_funcs = &pp_dpm_funcs; 81 amd_pp->pp_funcs = &pp_dpm_funcs;
105 } 82 }
@@ -118,12 +95,9 @@ static int amdgpu_pp_early_init(void *handle)
118 95
119 if (adev->powerplay.ip_funcs->early_init) 96 if (adev->powerplay.ip_funcs->early_init)
120 ret = adev->powerplay.ip_funcs->early_init( 97 ret = adev->powerplay.ip_funcs->early_init(
121 adev->powerplay.pp_handle); 98 amd_pp->cgs_device ? amd_pp->cgs_device :
99 amd_pp->pp_handle);
122 100
123 if (ret == PP_DPM_DISABLED) {
124 adev->pm.dpm_enabled = false;
125 return 0;
126 }
127 return ret; 101 return ret;
128} 102}
129 103
@@ -137,11 +111,6 @@ static int amdgpu_pp_late_init(void *handle)
137 ret = adev->powerplay.ip_funcs->late_init( 111 ret = adev->powerplay.ip_funcs->late_init(
138 adev->powerplay.pp_handle); 112 adev->powerplay.pp_handle);
139 113
140 if (adev->pp_enabled && adev->pm.dpm_enabled) {
141 amdgpu_pm_sysfs_init(adev);
142 amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_COMPLETE_INIT, NULL, NULL);
143 }
144
145 return ret; 114 return ret;
146} 115}
147 116
@@ -176,21 +145,11 @@ static int amdgpu_pp_hw_init(void *handle)
176 int ret = 0; 145 int ret = 0;
177 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 146 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
178 147
179 if (adev->pp_enabled && adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
180 amdgpu_ucode_init_bo(adev);
181 148
182 if (adev->powerplay.ip_funcs->hw_init) 149 if (adev->powerplay.ip_funcs->hw_init)
183 ret = adev->powerplay.ip_funcs->hw_init( 150 ret = adev->powerplay.ip_funcs->hw_init(
184 adev->powerplay.pp_handle); 151 adev->powerplay.pp_handle);
185 152
186 if (ret == PP_DPM_DISABLED) {
187 adev->pm.dpm_enabled = false;
188 return 0;
189 }
190
191 if ((amdgpu_dpm != 0) && !amdgpu_sriov_vf(adev))
192 adev->pm.dpm_enabled = true;
193
194 return ret; 153 return ret;
195} 154}
196 155
@@ -199,16 +158,10 @@ static int amdgpu_pp_hw_fini(void *handle)
199 int ret = 0; 158 int ret = 0;
200 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 159 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
201 160
202 if (adev->pp_enabled && adev->pm.dpm_enabled)
203 amdgpu_pm_sysfs_fini(adev);
204
205 if (adev->powerplay.ip_funcs->hw_fini) 161 if (adev->powerplay.ip_funcs->hw_fini)
206 ret = adev->powerplay.ip_funcs->hw_fini( 162 ret = adev->powerplay.ip_funcs->hw_fini(
207 adev->powerplay.pp_handle); 163 adev->powerplay.pp_handle);
208 164
209 if (adev->pp_enabled && adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
210 amdgpu_ucode_fini_bo(adev);
211
212 return ret; 165 return ret;
213} 166}
214 167
@@ -220,9 +173,8 @@ static void amdgpu_pp_late_fini(void *handle)
220 adev->powerplay.ip_funcs->late_fini( 173 adev->powerplay.ip_funcs->late_fini(
221 adev->powerplay.pp_handle); 174 adev->powerplay.pp_handle);
222 175
223 176 if (adev->powerplay.cgs_device)
224 if (adev->pp_enabled) 177 amdgpu_cgs_destroy_device(adev->powerplay.cgs_device);
225 amd_powerplay_destroy(adev->powerplay.pp_handle);
226} 178}
227 179
228static int amdgpu_pp_suspend(void *handle) 180static int amdgpu_pp_suspend(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 447d446b5015..f1035a689d35 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -411,13 +411,6 @@ static int psp_hw_init(void *handle)
411 return 0; 411 return 0;
412 412
413 mutex_lock(&adev->firmware.mutex); 413 mutex_lock(&adev->firmware.mutex);
414 /*
415 * This sequence is just used on hw_init only once, no need on
416 * resume.
417 */
418 ret = amdgpu_ucode_init_bo(adev);
419 if (ret)
420 goto failed;
421 414
422 ret = psp_load_fw(adev); 415 ret = psp_load_fw(adev);
423 if (ret) { 416 if (ret) {
@@ -442,8 +435,6 @@ static int psp_hw_fini(void *handle)
442 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) 435 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
443 return 0; 436 return 0;
444 437
445 amdgpu_ucode_fini_bo(adev);
446
447 psp_ring_destroy(psp, PSP_RING_TYPE__KM); 438 psp_ring_destroy(psp, PSP_RING_TYPE__KM);
448 439
449 amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf); 440 amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
index befc09b68543..190e28cb827e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
@@ -121,7 +121,7 @@ static enum amdgpu_ring_type amdgpu_hw_ip_to_ring_type(int hw_ip)
121 121
122static int amdgpu_lru_map(struct amdgpu_device *adev, 122static int amdgpu_lru_map(struct amdgpu_device *adev,
123 struct amdgpu_queue_mapper *mapper, 123 struct amdgpu_queue_mapper *mapper,
124 int user_ring, 124 int user_ring, bool lru_pipe_order,
125 struct amdgpu_ring **out_ring) 125 struct amdgpu_ring **out_ring)
126{ 126{
127 int r, i, j; 127 int r, i, j;
@@ -139,7 +139,7 @@ static int amdgpu_lru_map(struct amdgpu_device *adev,
139 } 139 }
140 140
141 r = amdgpu_ring_lru_get(adev, ring_type, ring_blacklist, 141 r = amdgpu_ring_lru_get(adev, ring_type, ring_blacklist,
142 j, out_ring); 142 j, lru_pipe_order, out_ring);
143 if (r) 143 if (r)
144 return r; 144 return r;
145 145
@@ -284,8 +284,10 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
284 r = amdgpu_identity_map(adev, mapper, ring, out_ring); 284 r = amdgpu_identity_map(adev, mapper, ring, out_ring);
285 break; 285 break;
286 case AMDGPU_HW_IP_DMA: 286 case AMDGPU_HW_IP_DMA:
287 r = amdgpu_lru_map(adev, mapper, ring, false, out_ring);
288 break;
287 case AMDGPU_HW_IP_COMPUTE: 289 case AMDGPU_HW_IP_COMPUTE:
288 r = amdgpu_lru_map(adev, mapper, ring, out_ring); 290 r = amdgpu_lru_map(adev, mapper, ring, true, out_ring);
289 break; 291 break;
290 default: 292 default:
291 *out_ring = NULL; 293 *out_ring = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 5ce65280b396..019932a7ea3a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -315,14 +315,16 @@ static bool amdgpu_ring_is_blacklisted(struct amdgpu_ring *ring,
315 * @type: amdgpu_ring_type enum 315 * @type: amdgpu_ring_type enum
316 * @blacklist: blacklisted ring ids array 316 * @blacklist: blacklisted ring ids array
317 * @num_blacklist: number of entries in @blacklist 317 * @num_blacklist: number of entries in @blacklist
318 * @lru_pipe_order: find a ring from the least recently used pipe
318 * @ring: output ring 319 * @ring: output ring
319 * 320 *
320 * Retrieve the amdgpu_ring structure for the least recently used ring of 321 * Retrieve the amdgpu_ring structure for the least recently used ring of
321 * a specific IP block (all asics). 322 * a specific IP block (all asics).
322 * Returns 0 on success, error on failure. 323 * Returns 0 on success, error on failure.
323 */ 324 */
324int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type, int *blacklist, 325int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type,
325 int num_blacklist, struct amdgpu_ring **ring) 326 int *blacklist, int num_blacklist,
327 bool lru_pipe_order, struct amdgpu_ring **ring)
326{ 328{
327 struct amdgpu_ring *entry; 329 struct amdgpu_ring *entry;
328 330
@@ -337,10 +339,23 @@ int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type, int *blacklist,
337 if (amdgpu_ring_is_blacklisted(entry, blacklist, num_blacklist)) 339 if (amdgpu_ring_is_blacklisted(entry, blacklist, num_blacklist))
338 continue; 340 continue;
339 341
340 *ring = entry; 342 if (!*ring) {
341 amdgpu_ring_lru_touch_locked(adev, *ring); 343 *ring = entry;
342 break; 344
345 /* We are done for ring LRU */
346 if (!lru_pipe_order)
347 break;
348 }
349
350 /* Move all rings on the same pipe to the end of the list */
351 if (entry->pipe == (*ring)->pipe)
352 amdgpu_ring_lru_touch_locked(adev, entry);
343 } 353 }
354
355 /* Move the ring we found to the end of the list */
356 if (*ring)
357 amdgpu_ring_lru_touch_locked(adev, *ring);
358
344 spin_unlock(&adev->ring_lru_list_lock); 359 spin_unlock(&adev->ring_lru_list_lock);
345 360
346 if (!*ring) { 361 if (!*ring) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 322d25299a00..491bd5512dcc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -201,8 +201,9 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
201 unsigned ring_size, struct amdgpu_irq_src *irq_src, 201 unsigned ring_size, struct amdgpu_irq_src *irq_src,
202 unsigned irq_type); 202 unsigned irq_type);
203void amdgpu_ring_fini(struct amdgpu_ring *ring); 203void amdgpu_ring_fini(struct amdgpu_ring *ring);
204int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type, int *blacklist, 204int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type,
205 int num_blacklist, struct amdgpu_ring **ring); 205 int *blacklist, int num_blacklist,
206 bool lru_pipe_order, struct amdgpu_ring **ring);
206void amdgpu_ring_lru_touch(struct amdgpu_device *adev, struct amdgpu_ring *ring); 207void amdgpu_ring_lru_touch(struct amdgpu_device *adev, struct amdgpu_ring *ring);
207static inline void amdgpu_ring_clear_ring(struct amdgpu_ring *ring) 208static inline void amdgpu_ring_clear_ring(struct amdgpu_ring *ring)
208{ 209{
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c
index 45ac91861965..746b81339835 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c
@@ -25,30 +25,21 @@
25#include "amdgpu_vf_error.h" 25#include "amdgpu_vf_error.h"
26#include "mxgpu_ai.h" 26#include "mxgpu_ai.h"
27 27
28#define AMDGPU_VF_ERROR_ENTRY_SIZE 16 28void amdgpu_vf_error_put(struct amdgpu_device *adev,
29 29 uint16_t sub_error_code,
30/* struct error_entry - amdgpu VF error information. */ 30 uint16_t error_flags,
31struct amdgpu_vf_error_buffer { 31 uint64_t error_data)
32 int read_count;
33 int write_count;
34 uint16_t code[AMDGPU_VF_ERROR_ENTRY_SIZE];
35 uint16_t flags[AMDGPU_VF_ERROR_ENTRY_SIZE];
36 uint64_t data[AMDGPU_VF_ERROR_ENTRY_SIZE];
37};
38
39struct amdgpu_vf_error_buffer admgpu_vf_errors;
40
41
42void amdgpu_vf_error_put(uint16_t sub_error_code, uint16_t error_flags, uint64_t error_data)
43{ 32{
44 int index; 33 int index;
45 uint16_t error_code = AMDGIM_ERROR_CODE(AMDGIM_ERROR_CATEGORY_VF, sub_error_code); 34 uint16_t error_code = AMDGIM_ERROR_CODE(AMDGIM_ERROR_CATEGORY_VF, sub_error_code);
46 35
47 index = admgpu_vf_errors.write_count % AMDGPU_VF_ERROR_ENTRY_SIZE; 36 mutex_lock(&adev->virt.vf_errors.lock);
48 admgpu_vf_errors.code [index] = error_code; 37 index = adev->virt.vf_errors.write_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
49 admgpu_vf_errors.flags [index] = error_flags; 38 adev->virt.vf_errors.code [index] = error_code;
50 admgpu_vf_errors.data [index] = error_data; 39 adev->virt.vf_errors.flags [index] = error_flags;
51 admgpu_vf_errors.write_count ++; 40 adev->virt.vf_errors.data [index] = error_data;
41 adev->virt.vf_errors.write_count ++;
42 mutex_unlock(&adev->virt.vf_errors.lock);
52} 43}
53 44
54 45
@@ -58,7 +49,8 @@ void amdgpu_vf_error_trans_all(struct amdgpu_device *adev)
58 u32 data1, data2, data3; 49 u32 data1, data2, data3;
59 int index; 50 int index;
60 51
61 if ((NULL == adev) || (!amdgpu_sriov_vf(adev)) || (!adev->virt.ops) || (!adev->virt.ops->trans_msg)) { 52 if ((NULL == adev) || (!amdgpu_sriov_vf(adev)) ||
53 (!adev->virt.ops) || (!adev->virt.ops->trans_msg)) {
62 return; 54 return;
63 } 55 }
64/* 56/*
@@ -68,18 +60,22 @@ void amdgpu_vf_error_trans_all(struct amdgpu_device *adev)
68 return; 60 return;
69 } 61 }
70*/ 62*/
63
64 mutex_lock(&adev->virt.vf_errors.lock);
71 /* The errors are overlay of array, correct read_count as full. */ 65 /* The errors are overlay of array, correct read_count as full. */
72 if (admgpu_vf_errors.write_count - admgpu_vf_errors.read_count > AMDGPU_VF_ERROR_ENTRY_SIZE) { 66 if (adev->virt.vf_errors.write_count - adev->virt.vf_errors.read_count > AMDGPU_VF_ERROR_ENTRY_SIZE) {
73 admgpu_vf_errors.read_count = admgpu_vf_errors.write_count - AMDGPU_VF_ERROR_ENTRY_SIZE; 67 adev->virt.vf_errors.read_count = adev->virt.vf_errors.write_count - AMDGPU_VF_ERROR_ENTRY_SIZE;
74 } 68 }
75 69
76 while (admgpu_vf_errors.read_count < admgpu_vf_errors.write_count) { 70 while (adev->virt.vf_errors.read_count < adev->virt.vf_errors.write_count) {
77 index =admgpu_vf_errors.read_count % AMDGPU_VF_ERROR_ENTRY_SIZE; 71 index =adev->virt.vf_errors.read_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
78 data1 = AMDGIM_ERROR_CODE_FLAGS_TO_MAILBOX (admgpu_vf_errors.code[index], admgpu_vf_errors.flags[index]); 72 data1 = AMDGIM_ERROR_CODE_FLAGS_TO_MAILBOX(adev->virt.vf_errors.code[index],
79 data2 = admgpu_vf_errors.data[index] & 0xFFFFFFFF; 73 adev->virt.vf_errors.flags[index]);
80 data3 = (admgpu_vf_errors.data[index] >> 32) & 0xFFFFFFFF; 74 data2 = adev->virt.vf_errors.data[index] & 0xFFFFFFFF;
75 data3 = (adev->virt.vf_errors.data[index] >> 32) & 0xFFFFFFFF;
81 76
82 adev->virt.ops->trans_msg(adev, IDH_LOG_VF_ERROR, data1, data2, data3); 77 adev->virt.ops->trans_msg(adev, IDH_LOG_VF_ERROR, data1, data2, data3);
83 admgpu_vf_errors.read_count ++; 78 adev->virt.vf_errors.read_count ++;
84 } 79 }
80 mutex_unlock(&adev->virt.vf_errors.lock);
85} 81}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h
index 2a3278ec76ba..6436bd053325 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h
@@ -56,7 +56,10 @@ enum AMDGIM_ERROR_CATEGORY {
56 AMDGIM_ERROR_CATEGORY_MAX 56 AMDGIM_ERROR_CATEGORY_MAX
57}; 57};
58 58
59void amdgpu_vf_error_put(uint16_t sub_error_code, uint16_t error_flags, uint64_t error_data); 59void amdgpu_vf_error_put(struct amdgpu_device *adev,
60 uint16_t sub_error_code,
61 uint16_t error_flags,
62 uint64_t error_data);
60void amdgpu_vf_error_trans_all (struct amdgpu_device *adev); 63void amdgpu_vf_error_trans_all (struct amdgpu_device *adev);
61 64
62#endif /* __VF_ERROR_H__ */ 65#endif /* __VF_ERROR_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index afcfb8bcfb65..e5fd0ff6b29d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -36,6 +36,18 @@ struct amdgpu_mm_table {
36 uint64_t gpu_addr; 36 uint64_t gpu_addr;
37}; 37};
38 38
39#define AMDGPU_VF_ERROR_ENTRY_SIZE 16
40
41/* struct error_entry - amdgpu VF error information. */
42struct amdgpu_vf_error_buffer {
43 struct mutex lock;
44 int read_count;
45 int write_count;
46 uint16_t code[AMDGPU_VF_ERROR_ENTRY_SIZE];
47 uint16_t flags[AMDGPU_VF_ERROR_ENTRY_SIZE];
48 uint64_t data[AMDGPU_VF_ERROR_ENTRY_SIZE];
49};
50
39/** 51/**
40 * struct amdgpu_virt_ops - amdgpu device virt operations 52 * struct amdgpu_virt_ops - amdgpu device virt operations
41 */ 53 */
@@ -59,6 +71,7 @@ struct amdgpu_virt {
59 struct work_struct flr_work; 71 struct work_struct flr_work;
60 struct amdgpu_mm_table mm_table; 72 struct amdgpu_mm_table mm_table;
61 const struct amdgpu_virt_ops *ops; 73 const struct amdgpu_virt_ops *ops;
74 struct amdgpu_vf_error_buffer vf_errors;
62}; 75};
63 76
64#define AMDGPU_CSA_SIZE (8 * 1024) 77#define AMDGPU_CSA_SIZE (8 * 1024)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index bbcc67038203..fee0a32ac56f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -2541,7 +2541,8 @@ static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2541 * @adev: amdgpu_device pointer 2541 * @adev: amdgpu_device pointer
2542 * @fragment_size_default: the default fragment size if it's set auto 2542 * @fragment_size_default: the default fragment size if it's set auto
2543 */ 2543 */
2544void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev, uint32_t fragment_size_default) 2544void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev,
2545 uint32_t fragment_size_default)
2545{ 2546{
2546 if (amdgpu_vm_fragment_size == -1) 2547 if (amdgpu_vm_fragment_size == -1)
2547 adev->vm_manager.fragment_size = fragment_size_default; 2548 adev->vm_manager.fragment_size = fragment_size_default;
@@ -2555,7 +2556,8 @@ void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev, uint32_t fragment_s
2555 * @adev: amdgpu_device pointer 2556 * @adev: amdgpu_device pointer
2556 * @vm_size: the default vm size if it's set auto 2557 * @vm_size: the default vm size if it's set auto
2557 */ 2558 */
2558void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_t fragment_size_default) 2559void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size,
2560 uint32_t fragment_size_default)
2559{ 2561{
2560 /* adjust vm size firstly */ 2562 /* adjust vm size firstly */
2561 if (amdgpu_vm_size == -1) 2563 if (amdgpu_vm_size == -1)
@@ -2682,6 +2684,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2682 } 2684 }
2683 2685
2684 INIT_KFIFO(vm->faults); 2686 INIT_KFIFO(vm->faults);
2687 vm->fault_credit = 16;
2685 2688
2686 return 0; 2689 return 0;
2687 2690
@@ -2777,6 +2780,36 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2777} 2780}
2778 2781
2779/** 2782/**
2783 * amdgpu_vm_pasid_fault_credit - Check fault credit for given PASID
2784 *
2785 * @adev: amdgpu_device pointer
2786 * @pasid: PASID do identify the VM
2787 *
2788 * This function is expected to be called in interrupt context. Returns
2789 * true if there was fault credit, false otherwise
2790 */
2791bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
2792 unsigned int pasid)
2793{
2794 struct amdgpu_vm *vm;
2795
2796 spin_lock(&adev->vm_manager.pasid_lock);
2797 vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
2798 spin_unlock(&adev->vm_manager.pasid_lock);
2799 if (!vm)
2800 /* VM not found, can't track fault credit */
2801 return true;
2802
2803 /* No lock needed. only accessed by IRQ handler */
2804 if (!vm->fault_credit)
2805 /* Too many faults in this VM */
2806 return false;
2807
2808 vm->fault_credit--;
2809 return true;
2810}
2811
2812/**
2780 * amdgpu_vm_manager_init - init the VM manager 2813 * amdgpu_vm_manager_init - init the VM manager
2781 * 2814 *
2782 * @adev: amdgpu_device pointer 2815 * @adev: amdgpu_device pointer
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 0af090667dfc..d68f39b4e5e7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -165,8 +165,11 @@ struct amdgpu_vm {
165 /* Flag to indicate ATS support from PTE for GFX9 */ 165 /* Flag to indicate ATS support from PTE for GFX9 */
166 bool pte_support_ats; 166 bool pte_support_ats;
167 167
168 /* Up to 128 pending page faults */ 168 /* Up to 128 pending retry page faults */
169 DECLARE_KFIFO(faults, u64, 128); 169 DECLARE_KFIFO(faults, u64, 128);
170
171 /* Limit non-retry fault storms */
172 unsigned int fault_credit;
170}; 173};
171 174
172struct amdgpu_vm_id { 175struct amdgpu_vm_id {
@@ -244,6 +247,8 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
244int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, 247int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
245 int vm_context, unsigned int pasid); 248 int vm_context, unsigned int pasid);
246void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm); 249void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
250bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
251 unsigned int pasid);
247void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, 252void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
248 struct list_head *validated, 253 struct list_head *validated,
249 struct amdgpu_bo_list_entry *entry); 254 struct amdgpu_bo_list_entry *entry);
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index 68ce1bdaf2fc..68b505c768ad 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -6365,7 +6365,6 @@ static int ci_dpm_sw_fini(void *handle)
6365 flush_work(&adev->pm.dpm.thermal.work); 6365 flush_work(&adev->pm.dpm.thermal.work);
6366 6366
6367 mutex_lock(&adev->pm.mutex); 6367 mutex_lock(&adev->pm.mutex);
6368 amdgpu_pm_sysfs_fini(adev);
6369 ci_dpm_fini(adev); 6368 ci_dpm_fini(adev);
6370 mutex_unlock(&adev->pm.mutex); 6369 mutex_unlock(&adev->pm.mutex);
6371 6370
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
index 07d3d895da10..a870b354e3f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
@@ -237,8 +237,23 @@ static u32 cik_ih_get_wptr(struct amdgpu_device *adev)
237 */ 237 */
238static bool cik_ih_prescreen_iv(struct amdgpu_device *adev) 238static bool cik_ih_prescreen_iv(struct amdgpu_device *adev)
239{ 239{
240 /* Process all interrupts */ 240 u32 ring_index = adev->irq.ih.rptr >> 2;
241 return true; 241 u16 pasid;
242
243 switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
244 case 146:
245 case 147:
246 pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
247 if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
248 return true;
249 break;
250 default:
251 /* Not a VM fault */
252 return true;
253 }
254
255 adev->irq.ih.rptr += 16;
256 return false;
242} 257}
243 258
244 /** 259 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
index b6cdf4afaf46..fa61d649bb44 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
@@ -216,8 +216,23 @@ static u32 cz_ih_get_wptr(struct amdgpu_device *adev)
216 */ 216 */
217static bool cz_ih_prescreen_iv(struct amdgpu_device *adev) 217static bool cz_ih_prescreen_iv(struct amdgpu_device *adev)
218{ 218{
219 /* Process all interrupts */ 219 u32 ring_index = adev->irq.ih.rptr >> 2;
220 return true; 220 u16 pasid;
221
222 switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
223 case 146:
224 case 147:
225 pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
226 if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
227 return true;
228 break;
229 default:
230 /* Not a VM fault */
231 return true;
232 }
233
234 adev->irq.ih.rptr += 16;
235 return false;
221} 236}
222 237
223/** 238/**
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index dfc10b1baea0..147e92b3a959 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -4132,18 +4132,12 @@ static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
4132 gfx_v8_0_rlc_reset(adev); 4132 gfx_v8_0_rlc_reset(adev);
4133 gfx_v8_0_init_pg(adev); 4133 gfx_v8_0_init_pg(adev);
4134 4134
4135 if (!adev->pp_enabled) { 4135
4136 if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) { 4136 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4137 /* legacy rlc firmware loading */ 4137 /* legacy rlc firmware loading */
4138 r = gfx_v8_0_rlc_load_microcode(adev); 4138 r = gfx_v8_0_rlc_load_microcode(adev);
4139 if (r) 4139 if (r)
4140 return r; 4140 return r;
4141 } else {
4142 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
4143 AMDGPU_UCODE_ID_RLC_G);
4144 if (r)
4145 return -EINVAL;
4146 }
4147 } 4141 }
4148 4142
4149 gfx_v8_0_rlc_start(adev); 4143 gfx_v8_0_rlc_start(adev);
@@ -4959,43 +4953,15 @@ static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
4959 if (!(adev->flags & AMD_IS_APU)) 4953 if (!(adev->flags & AMD_IS_APU))
4960 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 4954 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
4961 4955
4962 if (!adev->pp_enabled) { 4956 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4963 if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) {
4964 /* legacy firmware loading */ 4957 /* legacy firmware loading */
4965 r = gfx_v8_0_cp_gfx_load_microcode(adev); 4958 r = gfx_v8_0_cp_gfx_load_microcode(adev);
4966 if (r) 4959 if (r)
4967 return r; 4960 return r;
4968 4961
4969 r = gfx_v8_0_cp_compute_load_microcode(adev); 4962 r = gfx_v8_0_cp_compute_load_microcode(adev);
4970 if (r) 4963 if (r)
4971 return r; 4964 return r;
4972 } else {
4973 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
4974 AMDGPU_UCODE_ID_CP_CE);
4975 if (r)
4976 return -EINVAL;
4977
4978 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
4979 AMDGPU_UCODE_ID_CP_PFP);
4980 if (r)
4981 return -EINVAL;
4982
4983 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
4984 AMDGPU_UCODE_ID_CP_ME);
4985 if (r)
4986 return -EINVAL;
4987
4988 if (adev->asic_type == CHIP_TOPAZ) {
4989 r = gfx_v8_0_cp_compute_load_microcode(adev);
4990 if (r)
4991 return r;
4992 } else {
4993 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
4994 AMDGPU_UCODE_ID_CP_MEC1);
4995 if (r)
4996 return -EINVAL;
4997 }
4998 }
4999 } 4965 }
5000 4966
5001 r = gfx_v8_0_cp_gfx_resume(adev); 4967 r = gfx_v8_0_cp_gfx_resume(adev);
@@ -6018,7 +5984,6 @@ static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
6018{ 5984{
6019 uint32_t msg_id, pp_state = 0; 5985 uint32_t msg_id, pp_state = 0;
6020 uint32_t pp_support_state = 0; 5986 uint32_t pp_support_state = 0;
6021 void *pp_handle = adev->powerplay.pp_handle;
6022 5987
6023 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { 5988 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
6024 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5989 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
@@ -6036,7 +6001,8 @@ static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
6036 PP_BLOCK_GFX_CG, 6001 PP_BLOCK_GFX_CG,
6037 pp_support_state, 6002 pp_support_state,
6038 pp_state); 6003 pp_state);
6039 amd_set_clockgating_by_smu(pp_handle, msg_id); 6004 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
6005 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
6040 } 6006 }
6041 6007
6042 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { 6008 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
@@ -6057,7 +6023,8 @@ static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
6057 PP_BLOCK_GFX_MG, 6023 PP_BLOCK_GFX_MG,
6058 pp_support_state, 6024 pp_support_state,
6059 pp_state); 6025 pp_state);
6060 amd_set_clockgating_by_smu(pp_handle, msg_id); 6026 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
6027 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
6061 } 6028 }
6062 6029
6063 return 0; 6030 return 0;
@@ -6069,7 +6036,6 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
6069 6036
6070 uint32_t msg_id, pp_state = 0; 6037 uint32_t msg_id, pp_state = 0;
6071 uint32_t pp_support_state = 0; 6038 uint32_t pp_support_state = 0;
6072 void *pp_handle = adev->powerplay.pp_handle;
6073 6039
6074 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { 6040 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
6075 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 6041 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
@@ -6087,7 +6053,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
6087 PP_BLOCK_GFX_CG, 6053 PP_BLOCK_GFX_CG,
6088 pp_support_state, 6054 pp_support_state,
6089 pp_state); 6055 pp_state);
6090 amd_set_clockgating_by_smu(pp_handle, msg_id); 6056 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
6057 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
6091 } 6058 }
6092 6059
6093 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) { 6060 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) {
@@ -6106,7 +6073,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
6106 PP_BLOCK_GFX_3D, 6073 PP_BLOCK_GFX_3D,
6107 pp_support_state, 6074 pp_support_state,
6108 pp_state); 6075 pp_state);
6109 amd_set_clockgating_by_smu(pp_handle, msg_id); 6076 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
6077 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
6110 } 6078 }
6111 6079
6112 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { 6080 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
@@ -6127,7 +6095,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
6127 PP_BLOCK_GFX_MG, 6095 PP_BLOCK_GFX_MG,
6128 pp_support_state, 6096 pp_support_state,
6129 pp_state); 6097 pp_state);
6130 amd_set_clockgating_by_smu(pp_handle, msg_id); 6098 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
6099 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
6131 } 6100 }
6132 6101
6133 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 6102 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
@@ -6142,7 +6111,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
6142 PP_BLOCK_GFX_RLC, 6111 PP_BLOCK_GFX_RLC,
6143 pp_support_state, 6112 pp_support_state,
6144 pp_state); 6113 pp_state);
6145 amd_set_clockgating_by_smu(pp_handle, msg_id); 6114 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
6115 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
6146 } 6116 }
6147 6117
6148 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 6118 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
@@ -6156,7 +6126,8 @@ static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
6156 PP_BLOCK_GFX_CP, 6126 PP_BLOCK_GFX_CP,
6157 pp_support_state, 6127 pp_support_state,
6158 pp_state); 6128 pp_state);
6159 amd_set_clockgating_by_smu(pp_handle, msg_id); 6129 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
6130 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
6160 } 6131 }
6161 6132
6162 return 0; 6133 return 0;
@@ -7076,7 +7047,7 @@ static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
7076{ 7047{
7077 uint64_t ce_payload_addr; 7048 uint64_t ce_payload_addr;
7078 int cnt_ce; 7049 int cnt_ce;
7079 static union { 7050 union {
7080 struct vi_ce_ib_state regular; 7051 struct vi_ce_ib_state regular;
7081 struct vi_ce_ib_state_chained_ib chained; 7052 struct vi_ce_ib_state_chained_ib chained;
7082 } ce_payload = {}; 7053 } ce_payload = {};
@@ -7105,7 +7076,7 @@ static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
7105{ 7076{
7106 uint64_t de_payload_addr, gds_addr, csa_addr; 7077 uint64_t de_payload_addr, gds_addr, csa_addr;
7107 int cnt_de; 7078 int cnt_de;
7108 static union { 7079 union {
7109 struct vi_de_ib_state regular; 7080 struct vi_de_ib_state regular;
7110 struct vi_de_ib_state_chained_ib chained; 7081 struct vi_de_ib_state_chained_ib chained;
7111 } de_payload = {}; 7082 } de_payload = {};
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index deeaee1457ef..99a5b3b92e8e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -3583,7 +3583,7 @@ static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
3583static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 3583static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
3584{ 3584{
3585 u32 ref_and_mask, reg_mem_engine; 3585 u32 ref_and_mask, reg_mem_engine;
3586 struct nbio_hdp_flush_reg *nbio_hf_reg; 3586 const struct nbio_hdp_flush_reg *nbio_hf_reg;
3587 3587
3588 if (ring->adev->flags & AMD_IS_APU) 3588 if (ring->adev->flags & AMD_IS_APU)
3589 nbio_hf_reg = &nbio_v7_0_hdp_flush_reg; 3589 nbio_hf_reg = &nbio_v7_0_hdp_flush_reg;
@@ -3806,7 +3806,7 @@ static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
3806 3806
3807static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 3807static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
3808{ 3808{
3809 static struct v9_ce_ib_state ce_payload = {0}; 3809 struct v9_ce_ib_state ce_payload = {0};
3810 uint64_t csa_addr; 3810 uint64_t csa_addr;
3811 int cnt; 3811 int cnt;
3812 3812
@@ -3825,7 +3825,7 @@ static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
3825 3825
3826static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 3826static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
3827{ 3827{
3828 static struct v9_de_ib_state de_payload = {0}; 3828 struct v9_de_ib_state de_payload = {0};
3829 uint64_t csa_addr, gds_addr; 3829 uint64_t csa_addr, gds_addr;
3830 int cnt; 3830 int cnt;
3831 3831
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
index 65ed6d3a8f05..bd592cb39f37 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
@@ -216,8 +216,23 @@ static u32 iceland_ih_get_wptr(struct amdgpu_device *adev)
216 */ 216 */
217static bool iceland_ih_prescreen_iv(struct amdgpu_device *adev) 217static bool iceland_ih_prescreen_iv(struct amdgpu_device *adev)
218{ 218{
219 /* Process all interrupts */ 219 u32 ring_index = adev->irq.ih.rptr >> 2;
220 return true; 220 u16 pasid;
221
222 switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
223 case 146:
224 case 147:
225 pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
226 if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
227 return true;
228 break;
229 default:
230 /* Not a VM fault */
231 return true;
232 }
233
234 adev->irq.ih.rptr += 16;
235 return false;
221} 236}
222 237
223/** 238/**
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index b57399a462c2..f33d1ffdb20b 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -2969,16 +2969,10 @@ static int kv_dpm_late_init(void *handle)
2969{ 2969{
2970 /* powerdown unused blocks for now */ 2970 /* powerdown unused blocks for now */
2971 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2971 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2972 int ret;
2973 2972
2974 if (!amdgpu_dpm) 2973 if (!amdgpu_dpm)
2975 return 0; 2974 return 0;
2976 2975
2977 /* init the sysfs and debugfs files late */
2978 ret = amdgpu_pm_sysfs_init(adev);
2979 if (ret)
2980 return ret;
2981
2982 kv_dpm_powergate_acp(adev, true); 2976 kv_dpm_powergate_acp(adev, true);
2983 kv_dpm_powergate_samu(adev, true); 2977 kv_dpm_powergate_samu(adev, true);
2984 2978
@@ -3040,7 +3034,6 @@ static int kv_dpm_sw_fini(void *handle)
3040 flush_work(&adev->pm.dpm.thermal.work); 3034 flush_work(&adev->pm.dpm.thermal.work);
3041 3035
3042 mutex_lock(&adev->pm.mutex); 3036 mutex_lock(&adev->pm.mutex);
3043 amdgpu_pm_sysfs_fini(adev);
3044 kv_dpm_fini(adev); 3037 kv_dpm_fini(adev);
3045 mutex_unlock(&adev->pm.mutex); 3038 mutex_unlock(&adev->pm.mutex);
3046 3039
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
index 045988b18bc3..904a1bab9b9f 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
@@ -215,31 +215,27 @@ void nbio_v6_1_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
215 *flags |= AMD_CG_SUPPORT_BIF_LS; 215 *flags |= AMD_CG_SUPPORT_BIF_LS;
216} 216}
217 217
218struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg; 218const struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg = {
219struct nbio_pcie_index_data nbio_v6_1_pcie_index_data; 219 .hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_REQ),
220 .hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_DONE),
221 .ref_and_mask_cp0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK,
222 .ref_and_mask_cp1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK,
223 .ref_and_mask_cp2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK,
224 .ref_and_mask_cp3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK,
225 .ref_and_mask_cp4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK,
226 .ref_and_mask_cp5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK,
227 .ref_and_mask_cp6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK,
228 .ref_and_mask_cp7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK,
229 .ref_and_mask_cp8 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK,
230 .ref_and_mask_cp9 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK,
231 .ref_and_mask_sdma0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK,
232 .ref_and_mask_sdma1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK
233};
220 234
221int nbio_v6_1_init(struct amdgpu_device *adev) 235const struct nbio_pcie_index_data nbio_v6_1_pcie_index_data = {
222{ 236 .index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX),
223 nbio_v6_1_hdp_flush_reg.hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_REQ); 237 .data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA),
224 nbio_v6_1_hdp_flush_reg.hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_DONE); 238};
225 nbio_v6_1_hdp_flush_reg.ref_and_mask_cp0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK;
226 nbio_v6_1_hdp_flush_reg.ref_and_mask_cp1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK;
227 nbio_v6_1_hdp_flush_reg.ref_and_mask_cp2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK;
228 nbio_v6_1_hdp_flush_reg.ref_and_mask_cp3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK;
229 nbio_v6_1_hdp_flush_reg.ref_and_mask_cp4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK;
230 nbio_v6_1_hdp_flush_reg.ref_and_mask_cp5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK;
231 nbio_v6_1_hdp_flush_reg.ref_and_mask_cp6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK;
232 nbio_v6_1_hdp_flush_reg.ref_and_mask_cp7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK;
233 nbio_v6_1_hdp_flush_reg.ref_and_mask_cp8 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK;
234 nbio_v6_1_hdp_flush_reg.ref_and_mask_cp9 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK;
235 nbio_v6_1_hdp_flush_reg.ref_and_mask_sdma0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK;
236 nbio_v6_1_hdp_flush_reg.ref_and_mask_sdma1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK;
237
238 nbio_v6_1_pcie_index_data.index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX);
239 nbio_v6_1_pcie_index_data.data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA);
240
241 return 0;
242}
243 239
244void nbio_v6_1_detect_hw_virt(struct amdgpu_device *adev) 240void nbio_v6_1_detect_hw_virt(struct amdgpu_device *adev)
245{ 241{
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
index 686e4b4d296a..14ca8d45a46c 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
@@ -26,8 +26,8 @@
26 26
27#include "soc15_common.h" 27#include "soc15_common.h"
28 28
29extern struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg; 29extern const struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg;
30extern struct nbio_pcie_index_data nbio_v6_1_pcie_index_data; 30extern const struct nbio_pcie_index_data nbio_v6_1_pcie_index_data;
31int nbio_v6_1_init(struct amdgpu_device *adev); 31int nbio_v6_1_init(struct amdgpu_device *adev);
32u32 nbio_v6_1_get_atombios_scratch_regs(struct amdgpu_device *adev, 32u32 nbio_v6_1_get_atombios_scratch_regs(struct amdgpu_device *adev,
33 uint32_t idx); 33 uint32_t idx);
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
index 11b70d601922..f802b973410a 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
@@ -185,28 +185,24 @@ void nbio_v7_0_ih_control(struct amdgpu_device *adev)
185 WREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL, interrupt_cntl); 185 WREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL, interrupt_cntl);
186} 186}
187 187
188struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg; 188const struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg = {
189struct nbio_pcie_index_data nbio_v7_0_pcie_index_data; 189 .hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_REQ),
190 .hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_DONE),
191 .ref_and_mask_cp0 = GPU_HDP_FLUSH_DONE__CP0_MASK,
192 .ref_and_mask_cp1 = GPU_HDP_FLUSH_DONE__CP1_MASK,
193 .ref_and_mask_cp2 = GPU_HDP_FLUSH_DONE__CP2_MASK,
194 .ref_and_mask_cp3 = GPU_HDP_FLUSH_DONE__CP3_MASK,
195 .ref_and_mask_cp4 = GPU_HDP_FLUSH_DONE__CP4_MASK,
196 .ref_and_mask_cp5 = GPU_HDP_FLUSH_DONE__CP5_MASK,
197 .ref_and_mask_cp6 = GPU_HDP_FLUSH_DONE__CP6_MASK,
198 .ref_and_mask_cp7 = GPU_HDP_FLUSH_DONE__CP7_MASK,
199 .ref_and_mask_cp8 = GPU_HDP_FLUSH_DONE__CP8_MASK,
200 .ref_and_mask_cp9 = GPU_HDP_FLUSH_DONE__CP9_MASK,
201 .ref_and_mask_sdma0 = GPU_HDP_FLUSH_DONE__SDMA0_MASK,
202 .ref_and_mask_sdma1 = GPU_HDP_FLUSH_DONE__SDMA1_MASK,
203};
190 204
191int nbio_v7_0_init(struct amdgpu_device *adev) 205const struct nbio_pcie_index_data nbio_v7_0_pcie_index_data = {
192{ 206 .index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2),
193 nbio_v7_0_hdp_flush_reg.hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_REQ); 207 .data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2)
194 nbio_v7_0_hdp_flush_reg.hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_DONE); 208};
195 nbio_v7_0_hdp_flush_reg.ref_and_mask_cp0 = GPU_HDP_FLUSH_DONE__CP0_MASK;
196 nbio_v7_0_hdp_flush_reg.ref_and_mask_cp1 = GPU_HDP_FLUSH_DONE__CP1_MASK;
197 nbio_v7_0_hdp_flush_reg.ref_and_mask_cp2 = GPU_HDP_FLUSH_DONE__CP2_MASK;
198 nbio_v7_0_hdp_flush_reg.ref_and_mask_cp3 = GPU_HDP_FLUSH_DONE__CP3_MASK;
199 nbio_v7_0_hdp_flush_reg.ref_and_mask_cp4 = GPU_HDP_FLUSH_DONE__CP4_MASK;
200 nbio_v7_0_hdp_flush_reg.ref_and_mask_cp5 = GPU_HDP_FLUSH_DONE__CP5_MASK;
201 nbio_v7_0_hdp_flush_reg.ref_and_mask_cp6 = GPU_HDP_FLUSH_DONE__CP6_MASK;
202 nbio_v7_0_hdp_flush_reg.ref_and_mask_cp7 = GPU_HDP_FLUSH_DONE__CP7_MASK;
203 nbio_v7_0_hdp_flush_reg.ref_and_mask_cp8 = GPU_HDP_FLUSH_DONE__CP8_MASK;
204 nbio_v7_0_hdp_flush_reg.ref_and_mask_cp9 = GPU_HDP_FLUSH_DONE__CP9_MASK;
205 nbio_v7_0_hdp_flush_reg.ref_and_mask_sdma0 = GPU_HDP_FLUSH_DONE__SDMA0_MASK;
206 nbio_v7_0_hdp_flush_reg.ref_and_mask_sdma1 = GPU_HDP_FLUSH_DONE__SDMA1_MASK;
207
208 nbio_v7_0_pcie_index_data.index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2);
209 nbio_v7_0_pcie_index_data.data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2);
210
211 return 0;
212}
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
index 054ff49427e6..df8fa90f40d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
@@ -26,8 +26,8 @@
26 26
27#include "soc15_common.h" 27#include "soc15_common.h"
28 28
29extern struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg; 29extern const struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg;
30extern struct nbio_pcie_index_data nbio_v7_0_pcie_index_data; 30extern const struct nbio_pcie_index_data nbio_v7_0_pcie_index_data;
31int nbio_v7_0_init(struct amdgpu_device *adev); 31int nbio_v7_0_init(struct amdgpu_device *adev);
32u32 nbio_v7_0_get_atombios_scratch_regs(struct amdgpu_device *adev, 32u32 nbio_v7_0_get_atombios_scratch_regs(struct amdgpu_device *adev,
33 uint32_t idx); 33 uint32_t idx);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index acdee3a4602c..67f375bfe452 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -561,21 +561,11 @@ static int sdma_v2_4_start(struct amdgpu_device *adev)
561{ 561{
562 int r; 562 int r;
563 563
564 if (!adev->pp_enabled) { 564
565 if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) { 565 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
566 r = sdma_v2_4_load_microcode(adev); 566 r = sdma_v2_4_load_microcode(adev);
567 if (r) 567 if (r)
568 return r; 568 return r;
569 } else {
570 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
571 AMDGPU_UCODE_ID_SDMA0);
572 if (r)
573 return -EINVAL;
574 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
575 AMDGPU_UCODE_ID_SDMA1);
576 if (r)
577 return -EINVAL;
578 }
579 } 569 }
580 570
581 /* halt the engine before programing */ 571 /* halt the engine before programing */
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 72f31cc7df00..6d06f8eb659f 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -819,23 +819,12 @@ static int sdma_v3_0_load_microcode(struct amdgpu_device *adev)
819 */ 819 */
820static int sdma_v3_0_start(struct amdgpu_device *adev) 820static int sdma_v3_0_start(struct amdgpu_device *adev)
821{ 821{
822 int r, i; 822 int r;
823 823
824 if (!adev->pp_enabled) { 824 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
825 if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) { 825 r = sdma_v3_0_load_microcode(adev);
826 r = sdma_v3_0_load_microcode(adev); 826 if (r)
827 if (r) 827 return r;
828 return r;
829 } else {
830 for (i = 0; i < adev->sdma.num_instances; i++) {
831 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
832 (i == 0) ?
833 AMDGPU_UCODE_ID_SDMA0 :
834 AMDGPU_UCODE_ID_SDMA1);
835 if (r)
836 return -EINVAL;
837 }
838 }
839 } 828 }
840 829
841 /* disable sdma engine before programing it */ 830 /* disable sdma engine before programing it */
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index c26d205ff3bf..46009db3d195 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -371,7 +371,7 @@ static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring,
371static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 371static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
372{ 372{
373 u32 ref_and_mask = 0; 373 u32 ref_and_mask = 0;
374 struct nbio_hdp_flush_reg *nbio_hf_reg; 374 const struct nbio_hdp_flush_reg *nbio_hf_reg;
375 375
376 if (ring->adev->flags & AMD_IS_APU) 376 if (ring->adev->flags & AMD_IS_APU)
377 nbio_hf_reg = &nbio_v7_0_hdp_flush_reg; 377 nbio_hf_reg = &nbio_v7_0_hdp_flush_reg;
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index 9b8db6046271..51fd0c9a20a5 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -7604,11 +7604,6 @@ static int si_dpm_late_init(void *handle)
7604 if (!amdgpu_dpm) 7604 if (!amdgpu_dpm)
7605 return 0; 7605 return 0;
7606 7606
7607 /* init the sysfs and debugfs files late */
7608 ret = amdgpu_pm_sysfs_init(adev);
7609 if (ret)
7610 return ret;
7611
7612 ret = si_set_temperature_range(adev); 7607 ret = si_set_temperature_range(adev);
7613 if (ret) 7608 if (ret)
7614 return ret; 7609 return ret;
@@ -7764,7 +7759,6 @@ static int si_dpm_sw_fini(void *handle)
7764 flush_work(&adev->pm.dpm.thermal.work); 7759 flush_work(&adev->pm.dpm.thermal.work);
7765 7760
7766 mutex_lock(&adev->pm.mutex); 7761 mutex_lock(&adev->pm.mutex);
7767 amdgpu_pm_sysfs_fini(adev);
7768 si_dpm_fini(adev); 7762 si_dpm_fini(adev);
7769 mutex_unlock(&adev->pm.mutex); 7763 mutex_unlock(&adev->pm.mutex);
7770 7764
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 245a18aeb389..1c006ba9d826 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -101,7 +101,7 @@ static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg)
101{ 101{
102 unsigned long flags, address, data; 102 unsigned long flags, address, data;
103 u32 r; 103 u32 r;
104 struct nbio_pcie_index_data *nbio_pcie_id; 104 const struct nbio_pcie_index_data *nbio_pcie_id;
105 105
106 if (adev->flags & AMD_IS_APU) 106 if (adev->flags & AMD_IS_APU)
107 nbio_pcie_id = &nbio_v7_0_pcie_index_data; 107 nbio_pcie_id = &nbio_v7_0_pcie_index_data;
@@ -122,7 +122,7 @@ static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg)
122static void soc15_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v) 122static void soc15_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
123{ 123{
124 unsigned long flags, address, data; 124 unsigned long flags, address, data;
125 struct nbio_pcie_index_data *nbio_pcie_id; 125 const struct nbio_pcie_index_data *nbio_pcie_id;
126 126
127 if (adev->flags & AMD_IS_APU) 127 if (adev->flags & AMD_IS_APU)
128 nbio_pcie_id = &nbio_v7_0_pcie_index_data; 128 nbio_pcie_id = &nbio_v7_0_pcie_index_data;
@@ -604,21 +604,6 @@ static int soc15_common_early_init(void *handle)
604 (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_PSP))) 604 (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_PSP)))
605 psp_enabled = true; 605 psp_enabled = true;
606 606
607 /*
608 * nbio need be used for both sdma and gfx9, but only
609 * initializes once
610 */
611 switch(adev->asic_type) {
612 case CHIP_VEGA10:
613 nbio_v6_1_init(adev);
614 break;
615 case CHIP_RAVEN:
616 nbio_v7_0_init(adev);
617 break;
618 default:
619 return -EINVAL;
620 }
621
622 adev->rev_id = soc15_get_rev_id(adev); 607 adev->rev_id = soc15_get_rev_id(adev);
623 adev->external_rev_id = 0xFF; 608 adev->external_rev_id = 0xFF;
624 switch (adev->asic_type) { 609 switch (adev->asic_type) {
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index 5ed00692618e..aa4e320e31f8 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -227,8 +227,23 @@ static u32 tonga_ih_get_wptr(struct amdgpu_device *adev)
227 */ 227 */
228static bool tonga_ih_prescreen_iv(struct amdgpu_device *adev) 228static bool tonga_ih_prescreen_iv(struct amdgpu_device *adev)
229{ 229{
230 /* Process all interrupts */ 230 u32 ring_index = adev->irq.ih.rptr >> 2;
231 return true; 231 u16 pasid;
232
233 switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) {
234 case 146:
235 case 147:
236 pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16;
237 if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid))
238 return true;
239 break;
240 default:
241 /* Not a VM fault */
242 return true;
243 }
244
245 adev->irq.ih.rptr += 16;
246 return false;
232} 247}
233 248
234/** 249/**
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index 31db356476f8..60af7310a234 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -38,6 +38,8 @@
38#include "vi.h" 38#include "vi.h"
39 39
40static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev); 40static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev);
41static void uvd_v6_0_set_enc_ring_funcs(struct amdgpu_device *adev);
42
41static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev); 43static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev);
42static int uvd_v6_0_start(struct amdgpu_device *adev); 44static int uvd_v6_0_start(struct amdgpu_device *adev);
43static void uvd_v6_0_stop(struct amdgpu_device *adev); 45static void uvd_v6_0_stop(struct amdgpu_device *adev);
@@ -48,6 +50,18 @@ static void uvd_v6_0_enable_mgcg(struct amdgpu_device *adev,
48 bool enable); 50 bool enable);
49 51
50/** 52/**
53* uvd_v6_0_enc_support - get encode support status
54*
55* @adev: amdgpu_device pointer
56*
57* Returns the current hardware encode support status
58*/
59static inline bool uvd_v6_0_enc_support(struct amdgpu_device *adev)
60{
61 return ((adev->asic_type >= CHIP_POLARIS10) && (adev->asic_type <= CHIP_POLARIS12));
62}
63
64/**
51 * uvd_v6_0_ring_get_rptr - get read pointer 65 * uvd_v6_0_ring_get_rptr - get read pointer
52 * 66 *
53 * @ring: amdgpu_ring pointer 67 * @ring: amdgpu_ring pointer
@@ -62,6 +76,22 @@ static uint64_t uvd_v6_0_ring_get_rptr(struct amdgpu_ring *ring)
62} 76}
63 77
64/** 78/**
79 * uvd_v6_0_enc_ring_get_rptr - get enc read pointer
80 *
81 * @ring: amdgpu_ring pointer
82 *
83 * Returns the current hardware enc read pointer
84 */
85static uint64_t uvd_v6_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
86{
87 struct amdgpu_device *adev = ring->adev;
88
89 if (ring == &adev->uvd.ring_enc[0])
90 return RREG32(mmUVD_RB_RPTR);
91 else
92 return RREG32(mmUVD_RB_RPTR2);
93}
94/**
65 * uvd_v6_0_ring_get_wptr - get write pointer 95 * uvd_v6_0_ring_get_wptr - get write pointer
66 * 96 *
67 * @ring: amdgpu_ring pointer 97 * @ring: amdgpu_ring pointer
@@ -76,6 +106,23 @@ static uint64_t uvd_v6_0_ring_get_wptr(struct amdgpu_ring *ring)
76} 106}
77 107
78/** 108/**
109 * uvd_v6_0_enc_ring_get_wptr - get enc write pointer
110 *
111 * @ring: amdgpu_ring pointer
112 *
113 * Returns the current hardware enc write pointer
114 */
115static uint64_t uvd_v6_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
116{
117 struct amdgpu_device *adev = ring->adev;
118
119 if (ring == &adev->uvd.ring_enc[0])
120 return RREG32(mmUVD_RB_WPTR);
121 else
122 return RREG32(mmUVD_RB_WPTR2);
123}
124
125/**
79 * uvd_v6_0_ring_set_wptr - set write pointer 126 * uvd_v6_0_ring_set_wptr - set write pointer
80 * 127 *
81 * @ring: amdgpu_ring pointer 128 * @ring: amdgpu_ring pointer
@@ -89,11 +136,247 @@ static void uvd_v6_0_ring_set_wptr(struct amdgpu_ring *ring)
89 WREG32(mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr)); 136 WREG32(mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
90} 137}
91 138
139/**
140 * uvd_v6_0_enc_ring_set_wptr - set enc write pointer
141 *
142 * @ring: amdgpu_ring pointer
143 *
144 * Commits the enc write pointer to the hardware
145 */
146static void uvd_v6_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
147{
148 struct amdgpu_device *adev = ring->adev;
149
150 if (ring == &adev->uvd.ring_enc[0])
151 WREG32(mmUVD_RB_WPTR,
152 lower_32_bits(ring->wptr));
153 else
154 WREG32(mmUVD_RB_WPTR2,
155 lower_32_bits(ring->wptr));
156}
157
158/**
159 * uvd_v6_0_enc_ring_test_ring - test if UVD ENC ring is working
160 *
161 * @ring: the engine to test on
162 *
163 */
164static int uvd_v6_0_enc_ring_test_ring(struct amdgpu_ring *ring)
165{
166 struct amdgpu_device *adev = ring->adev;
167 uint32_t rptr = amdgpu_ring_get_rptr(ring);
168 unsigned i;
169 int r;
170
171 r = amdgpu_ring_alloc(ring, 16);
172 if (r) {
173 DRM_ERROR("amdgpu: uvd enc failed to lock ring %d (%d).\n",
174 ring->idx, r);
175 return r;
176 }
177 amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
178 amdgpu_ring_commit(ring);
179
180 for (i = 0; i < adev->usec_timeout; i++) {
181 if (amdgpu_ring_get_rptr(ring) != rptr)
182 break;
183 DRM_UDELAY(1);
184 }
185
186 if (i < adev->usec_timeout) {
187 DRM_INFO("ring test on %d succeeded in %d usecs\n",
188 ring->idx, i);
189 } else {
190 DRM_ERROR("amdgpu: ring %d test failed\n",
191 ring->idx);
192 r = -ETIMEDOUT;
193 }
194
195 return r;
196}
197
198/**
199 * uvd_v6_0_enc_get_create_msg - generate a UVD ENC create msg
200 *
201 * @adev: amdgpu_device pointer
202 * @ring: ring we should submit the msg to
203 * @handle: session handle to use
204 * @fence: optional fence to return
205 *
206 * Open up a stream for HW test
207 */
208static int uvd_v6_0_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
209 struct dma_fence **fence)
210{
211 const unsigned ib_size_dw = 16;
212 struct amdgpu_job *job;
213 struct amdgpu_ib *ib;
214 struct dma_fence *f = NULL;
215 uint64_t dummy;
216 int i, r;
217
218 r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
219 if (r)
220 return r;
221
222 ib = &job->ibs[0];
223 dummy = ib->gpu_addr + 1024;
224
225 ib->length_dw = 0;
226 ib->ptr[ib->length_dw++] = 0x00000018;
227 ib->ptr[ib->length_dw++] = 0x00000001; /* session info */
228 ib->ptr[ib->length_dw++] = handle;
229 ib->ptr[ib->length_dw++] = 0x00010000;
230 ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
231 ib->ptr[ib->length_dw++] = dummy;
232
233 ib->ptr[ib->length_dw++] = 0x00000014;
234 ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
235 ib->ptr[ib->length_dw++] = 0x0000001c;
236 ib->ptr[ib->length_dw++] = 0x00000001;
237 ib->ptr[ib->length_dw++] = 0x00000000;
238
239 ib->ptr[ib->length_dw++] = 0x00000008;
240 ib->ptr[ib->length_dw++] = 0x08000001; /* op initialize */
241
242 for (i = ib->length_dw; i < ib_size_dw; ++i)
243 ib->ptr[i] = 0x0;
244
245 r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
246 job->fence = dma_fence_get(f);
247 if (r)
248 goto err;
249
250 amdgpu_job_free(job);
251 if (fence)
252 *fence = dma_fence_get(f);
253 dma_fence_put(f);
254 return 0;
255
256err:
257 amdgpu_job_free(job);
258 return r;
259}
260
261/**
262 * uvd_v6_0_enc_get_destroy_msg - generate a UVD ENC destroy msg
263 *
264 * @adev: amdgpu_device pointer
265 * @ring: ring we should submit the msg to
266 * @handle: session handle to use
267 * @fence: optional fence to return
268 *
269 * Close up a stream for HW test or if userspace failed to do so
270 */
271int uvd_v6_0_enc_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
272 bool direct, struct dma_fence **fence)
273{
274 const unsigned ib_size_dw = 16;
275 struct amdgpu_job *job;
276 struct amdgpu_ib *ib;
277 struct dma_fence *f = NULL;
278 uint64_t dummy;
279 int i, r;
280
281 r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
282 if (r)
283 return r;
284
285 ib = &job->ibs[0];
286 dummy = ib->gpu_addr + 1024;
287
288 ib->length_dw = 0;
289 ib->ptr[ib->length_dw++] = 0x00000018;
290 ib->ptr[ib->length_dw++] = 0x00000001; /* session info */
291 ib->ptr[ib->length_dw++] = handle;
292 ib->ptr[ib->length_dw++] = 0x00010000;
293 ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
294 ib->ptr[ib->length_dw++] = dummy;
295
296 ib->ptr[ib->length_dw++] = 0x00000014;
297 ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
298 ib->ptr[ib->length_dw++] = 0x0000001c;
299 ib->ptr[ib->length_dw++] = 0x00000001;
300 ib->ptr[ib->length_dw++] = 0x00000000;
301
302 ib->ptr[ib->length_dw++] = 0x00000008;
303 ib->ptr[ib->length_dw++] = 0x08000002; /* op close session */
304
305 for (i = ib->length_dw; i < ib_size_dw; ++i)
306 ib->ptr[i] = 0x0;
307
308 if (direct) {
309 r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
310 job->fence = dma_fence_get(f);
311 if (r)
312 goto err;
313
314 amdgpu_job_free(job);
315 } else {
316 r = amdgpu_job_submit(job, ring, &ring->adev->vce.entity,
317 AMDGPU_FENCE_OWNER_UNDEFINED, &f);
318 if (r)
319 goto err;
320 }
321
322 if (fence)
323 *fence = dma_fence_get(f);
324 dma_fence_put(f);
325 return 0;
326
327err:
328 amdgpu_job_free(job);
329 return r;
330}
331
332/**
333 * uvd_v6_0_enc_ring_test_ib - test if UVD ENC IBs are working
334 *
335 * @ring: the engine to test on
336 *
337 */
338static int uvd_v6_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
339{
340 struct dma_fence *fence = NULL;
341 long r;
342
343 r = uvd_v6_0_enc_get_create_msg(ring, 1, NULL);
344 if (r) {
345 DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r);
346 goto error;
347 }
348
349 r = uvd_v6_0_enc_get_destroy_msg(ring, 1, true, &fence);
350 if (r) {
351 DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r);
352 goto error;
353 }
354
355 r = dma_fence_wait_timeout(fence, false, timeout);
356 if (r == 0) {
357 DRM_ERROR("amdgpu: IB test timed out.\n");
358 r = -ETIMEDOUT;
359 } else if (r < 0) {
360 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
361 } else {
362 DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
363 r = 0;
364 }
365error:
366 dma_fence_put(fence);
367 return r;
368}
92static int uvd_v6_0_early_init(void *handle) 369static int uvd_v6_0_early_init(void *handle)
93{ 370{
94 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 371 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
95 372
96 uvd_v6_0_set_ring_funcs(adev); 373 uvd_v6_0_set_ring_funcs(adev);
374
375 if (uvd_v6_0_enc_support(adev)) {
376 adev->uvd.num_enc_rings = 2;
377 uvd_v6_0_set_enc_ring_funcs(adev);
378 }
379
97 uvd_v6_0_set_irq_funcs(adev); 380 uvd_v6_0_set_irq_funcs(adev);
98 381
99 return 0; 382 return 0;
@@ -102,7 +385,7 @@ static int uvd_v6_0_early_init(void *handle)
102static int uvd_v6_0_sw_init(void *handle) 385static int uvd_v6_0_sw_init(void *handle)
103{ 386{
104 struct amdgpu_ring *ring; 387 struct amdgpu_ring *ring;
105 int r; 388 int i, r;
106 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 389 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
107 390
108 /* UVD TRAP */ 391 /* UVD TRAP */
@@ -110,10 +393,31 @@ static int uvd_v6_0_sw_init(void *handle)
110 if (r) 393 if (r)
111 return r; 394 return r;
112 395
396 /* UVD ENC TRAP */
397 if (uvd_v6_0_enc_support(adev)) {
398 for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
399 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, i + 119, &adev->uvd.irq);
400 if (r)
401 return r;
402 }
403 }
404
113 r = amdgpu_uvd_sw_init(adev); 405 r = amdgpu_uvd_sw_init(adev);
114 if (r) 406 if (r)
115 return r; 407 return r;
116 408
409 if (uvd_v6_0_enc_support(adev)) {
410 struct amd_sched_rq *rq;
411 ring = &adev->uvd.ring_enc[0];
412 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
413 r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
414 rq, amdgpu_sched_jobs);
415 if (r) {
416 DRM_ERROR("Failed setting up UVD ENC run queue.\n");
417 return r;
418 }
419 }
420
117 r = amdgpu_uvd_resume(adev); 421 r = amdgpu_uvd_resume(adev);
118 if (r) 422 if (r)
119 return r; 423 return r;
@@ -121,19 +425,38 @@ static int uvd_v6_0_sw_init(void *handle)
121 ring = &adev->uvd.ring; 425 ring = &adev->uvd.ring;
122 sprintf(ring->name, "uvd"); 426 sprintf(ring->name, "uvd");
123 r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0); 427 r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
428 if (r)
429 return r;
430
431 if (uvd_v6_0_enc_support(adev)) {
432 for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
433 ring = &adev->uvd.ring_enc[i];
434 sprintf(ring->name, "uvd_enc%d", i);
435 r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
436 if (r)
437 return r;
438 }
439 }
124 440
125 return r; 441 return r;
126} 442}
127 443
128static int uvd_v6_0_sw_fini(void *handle) 444static int uvd_v6_0_sw_fini(void *handle)
129{ 445{
130 int r; 446 int i, r;
131 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 447 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
132 448
133 r = amdgpu_uvd_suspend(adev); 449 r = amdgpu_uvd_suspend(adev);
134 if (r) 450 if (r)
135 return r; 451 return r;
136 452
453 if (uvd_v6_0_enc_support(adev)) {
454 amd_sched_entity_fini(&adev->uvd.ring_enc[0].sched, &adev->uvd.entity_enc);
455
456 for (i = 0; i < adev->uvd.num_enc_rings; ++i)
457 amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
458 }
459
137 return amdgpu_uvd_sw_fini(adev); 460 return amdgpu_uvd_sw_fini(adev);
138} 461}
139 462
@@ -149,7 +472,7 @@ static int uvd_v6_0_hw_init(void *handle)
149 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 472 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
150 struct amdgpu_ring *ring = &adev->uvd.ring; 473 struct amdgpu_ring *ring = &adev->uvd.ring;
151 uint32_t tmp; 474 uint32_t tmp;
152 int r; 475 int i, r;
153 476
154 amdgpu_asic_set_uvd_clocks(adev, 10000, 10000); 477 amdgpu_asic_set_uvd_clocks(adev, 10000, 10000);
155 uvd_v6_0_set_clockgating_state(adev, AMD_CG_STATE_UNGATE); 478 uvd_v6_0_set_clockgating_state(adev, AMD_CG_STATE_UNGATE);
@@ -189,9 +512,25 @@ static int uvd_v6_0_hw_init(void *handle)
189 512
190 amdgpu_ring_commit(ring); 513 amdgpu_ring_commit(ring);
191 514
515 if (uvd_v6_0_enc_support(adev)) {
516 for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
517 ring = &adev->uvd.ring_enc[i];
518 ring->ready = true;
519 r = amdgpu_ring_test_ring(ring);
520 if (r) {
521 ring->ready = false;
522 goto done;
523 }
524 }
525 }
526
192done: 527done:
193 if (!r) 528 if (!r) {
194 DRM_INFO("UVD initialized successfully.\n"); 529 if (uvd_v6_0_enc_support(adev))
530 DRM_INFO("UVD and UVD ENC initialized successfully.\n");
531 else
532 DRM_INFO("UVD initialized successfully.\n");
533 }
195 534
196 return r; 535 return r;
197} 536}
@@ -514,6 +853,22 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
514 853
515 WREG32_FIELD(UVD_RBC_RB_CNTL, RB_NO_FETCH, 0); 854 WREG32_FIELD(UVD_RBC_RB_CNTL, RB_NO_FETCH, 0);
516 855
856 if (uvd_v6_0_enc_support(adev)) {
857 ring = &adev->uvd.ring_enc[0];
858 WREG32(mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
859 WREG32(mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
860 WREG32(mmUVD_RB_BASE_LO, ring->gpu_addr);
861 WREG32(mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
862 WREG32(mmUVD_RB_SIZE, ring->ring_size / 4);
863
864 ring = &adev->uvd.ring_enc[1];
865 WREG32(mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
866 WREG32(mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
867 WREG32(mmUVD_RB_BASE_LO2, ring->gpu_addr);
868 WREG32(mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
869 WREG32(mmUVD_RB_SIZE2, ring->ring_size / 4);
870 }
871
517 return 0; 872 return 0;
518} 873}
519 874
@@ -577,6 +932,26 @@ static void uvd_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq
577} 932}
578 933
579/** 934/**
935 * uvd_v6_0_enc_ring_emit_fence - emit an enc fence & trap command
936 *
937 * @ring: amdgpu_ring pointer
938 * @fence: fence to emit
939 *
940 * Write enc a fence and a trap command to the ring.
941 */
942static void uvd_v6_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
943 u64 seq, unsigned flags)
944{
945 WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
946
947 amdgpu_ring_write(ring, HEVC_ENC_CMD_FENCE);
948 amdgpu_ring_write(ring, addr);
949 amdgpu_ring_write(ring, upper_32_bits(addr));
950 amdgpu_ring_write(ring, seq);
951 amdgpu_ring_write(ring, HEVC_ENC_CMD_TRAP);
952}
953
954/**
580 * uvd_v6_0_ring_emit_hdp_flush - emit an hdp flush 955 * uvd_v6_0_ring_emit_hdp_flush - emit an hdp flush
581 * 956 *
582 * @ring: amdgpu_ring pointer 957 * @ring: amdgpu_ring pointer
@@ -667,6 +1042,24 @@ static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
667 amdgpu_ring_write(ring, ib->length_dw); 1042 amdgpu_ring_write(ring, ib->length_dw);
668} 1043}
669 1044
1045/**
1046 * uvd_v6_0_enc_ring_emit_ib - enc execute indirect buffer
1047 *
1048 * @ring: amdgpu_ring pointer
1049 * @ib: indirect buffer to execute
1050 *
1051 * Write enc ring commands to execute the indirect buffer
1052 */
1053static void uvd_v6_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
1054 struct amdgpu_ib *ib, unsigned int vm_id, bool ctx_switch)
1055{
1056 amdgpu_ring_write(ring, HEVC_ENC_CMD_IB_VM);
1057 amdgpu_ring_write(ring, vm_id);
1058 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1059 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1060 amdgpu_ring_write(ring, ib->length_dw);
1061}
1062
670static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 1063static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
671 unsigned vm_id, uint64_t pd_addr) 1064 unsigned vm_id, uint64_t pd_addr)
672{ 1065{
@@ -718,6 +1111,33 @@ static void uvd_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
718 amdgpu_ring_write(ring, 0xE); 1111 amdgpu_ring_write(ring, 0xE);
719} 1112}
720 1113
1114static void uvd_v6_0_enc_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1115{
1116 uint32_t seq = ring->fence_drv.sync_seq;
1117 uint64_t addr = ring->fence_drv.gpu_addr;
1118
1119 amdgpu_ring_write(ring, HEVC_ENC_CMD_WAIT_GE);
1120 amdgpu_ring_write(ring, lower_32_bits(addr));
1121 amdgpu_ring_write(ring, upper_32_bits(addr));
1122 amdgpu_ring_write(ring, seq);
1123}
1124
1125static void uvd_v6_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1126{
1127 amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
1128}
1129
1130static void uvd_v6_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1131 unsigned int vm_id, uint64_t pd_addr)
1132{
1133 amdgpu_ring_write(ring, HEVC_ENC_CMD_UPDATE_PTB);
1134 amdgpu_ring_write(ring, vm_id);
1135 amdgpu_ring_write(ring, pd_addr >> 12);
1136
1137 amdgpu_ring_write(ring, HEVC_ENC_CMD_FLUSH_TLB);
1138 amdgpu_ring_write(ring, vm_id);
1139}
1140
721static bool uvd_v6_0_is_idle(void *handle) 1141static bool uvd_v6_0_is_idle(void *handle)
722{ 1142{
723 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1143 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -825,8 +1245,31 @@ static int uvd_v6_0_process_interrupt(struct amdgpu_device *adev,
825 struct amdgpu_irq_src *source, 1245 struct amdgpu_irq_src *source,
826 struct amdgpu_iv_entry *entry) 1246 struct amdgpu_iv_entry *entry)
827{ 1247{
1248 bool int_handled = true;
828 DRM_DEBUG("IH: UVD TRAP\n"); 1249 DRM_DEBUG("IH: UVD TRAP\n");
829 amdgpu_fence_process(&adev->uvd.ring); 1250
1251 switch (entry->src_id) {
1252 case 124:
1253 amdgpu_fence_process(&adev->uvd.ring);
1254 break;
1255 case 119:
1256 if (likely(uvd_v6_0_enc_support(adev)))
1257 amdgpu_fence_process(&adev->uvd.ring_enc[0]);
1258 else
1259 int_handled = false;
1260 break;
1261 case 120:
1262 if (likely(uvd_v6_0_enc_support(adev)))
1263 amdgpu_fence_process(&adev->uvd.ring_enc[1]);
1264 else
1265 int_handled = false;
1266 break;
1267 }
1268
1269 if (false == int_handled)
1270 DRM_ERROR("Unhandled interrupt: %d %d\n",
1271 entry->src_id, entry->src_data[0]);
1272
830 return 0; 1273 return 0;
831} 1274}
832 1275
@@ -1153,6 +1596,33 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
1153 .end_use = amdgpu_uvd_ring_end_use, 1596 .end_use = amdgpu_uvd_ring_end_use,
1154}; 1597};
1155 1598
1599static const struct amdgpu_ring_funcs uvd_v6_0_enc_ring_vm_funcs = {
1600 .type = AMDGPU_RING_TYPE_UVD_ENC,
1601 .align_mask = 0x3f,
1602 .nop = HEVC_ENC_CMD_NO_OP,
1603 .support_64bit_ptrs = false,
1604 .get_rptr = uvd_v6_0_enc_ring_get_rptr,
1605 .get_wptr = uvd_v6_0_enc_ring_get_wptr,
1606 .set_wptr = uvd_v6_0_enc_ring_set_wptr,
1607 .emit_frame_size =
1608 4 + /* uvd_v6_0_enc_ring_emit_pipeline_sync */
1609 6 + /* uvd_v6_0_enc_ring_emit_vm_flush */
1610 5 + 5 + /* uvd_v6_0_enc_ring_emit_fence x2 vm fence */
1611 1, /* uvd_v6_0_enc_ring_insert_end */
1612 .emit_ib_size = 5, /* uvd_v6_0_enc_ring_emit_ib */
1613 .emit_ib = uvd_v6_0_enc_ring_emit_ib,
1614 .emit_fence = uvd_v6_0_enc_ring_emit_fence,
1615 .emit_vm_flush = uvd_v6_0_enc_ring_emit_vm_flush,
1616 .emit_pipeline_sync = uvd_v6_0_enc_ring_emit_pipeline_sync,
1617 .test_ring = uvd_v6_0_enc_ring_test_ring,
1618 .test_ib = uvd_v6_0_enc_ring_test_ib,
1619 .insert_nop = amdgpu_ring_insert_nop,
1620 .insert_end = uvd_v6_0_enc_ring_insert_end,
1621 .pad_ib = amdgpu_ring_generic_pad_ib,
1622 .begin_use = amdgpu_uvd_ring_begin_use,
1623 .end_use = amdgpu_uvd_ring_end_use,
1624};
1625
1156static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev) 1626static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
1157{ 1627{
1158 if (adev->asic_type >= CHIP_POLARIS10) { 1628 if (adev->asic_type >= CHIP_POLARIS10) {
@@ -1164,6 +1634,16 @@ static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
1164 } 1634 }
1165} 1635}
1166 1636
1637static void uvd_v6_0_set_enc_ring_funcs(struct amdgpu_device *adev)
1638{
1639 int i;
1640
1641 for (i = 0; i < adev->uvd.num_enc_rings; ++i)
1642 adev->uvd.ring_enc[i].funcs = &uvd_v6_0_enc_ring_vm_funcs;
1643
1644 DRM_INFO("UVD ENC is enabled in VM mode\n");
1645}
1646
1167static const struct amdgpu_irq_src_funcs uvd_v6_0_irq_funcs = { 1647static const struct amdgpu_irq_src_funcs uvd_v6_0_irq_funcs = {
1168 .set = uvd_v6_0_set_interrupt_state, 1648 .set = uvd_v6_0_set_interrupt_state,
1169 .process = uvd_v6_0_process_interrupt, 1649 .process = uvd_v6_0_process_interrupt,
@@ -1171,7 +1651,11 @@ static const struct amdgpu_irq_src_funcs uvd_v6_0_irq_funcs = {
1171 1651
1172static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev) 1652static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev)
1173{ 1653{
1174 adev->uvd.irq.num_types = 1; 1654 if (uvd_v6_0_enc_support(adev))
1655 adev->uvd.irq.num_types = adev->uvd.num_enc_rings + 1;
1656 else
1657 adev->uvd.irq.num_types = 1;
1658
1175 adev->uvd.irq.funcs = &uvd_v6_0_irq_funcs; 1659 adev->uvd.irq.funcs = &uvd_v6_0_irq_funcs;
1176} 1660}
1177 1661
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
index a3b30d84dbb3..697325737ba8 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
@@ -260,15 +260,18 @@ static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
260 return true; 260 return true;
261 } 261 }
262 262
263 /* Not a retry fault */
264 if (!(dw5 & 0x80))
265 return true;
266
267 pasid = dw3 & 0xffff; 263 pasid = dw3 & 0xffff;
268 /* No PASID, can't identify faulting process */ 264 /* No PASID, can't identify faulting process */
269 if (!pasid) 265 if (!pasid)
270 return true; 266 return true;
271 267
268 /* Not a retry fault, check fault credit */
269 if (!(dw5 & 0x80)) {
270 if (!amdgpu_vm_pasid_fault_credit(adev, pasid))
271 goto ignore_iv;
272 return true;
273 }
274
272 addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12); 275 addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12);
273 key = AMDGPU_VM_FAULT(pasid, addr); 276 key = AMDGPU_VM_FAULT(pasid, addr);
274 r = amdgpu_ih_add_fault(adev, key); 277 r = amdgpu_ih_add_fault(adev, key);
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 9ff69b90df36..f3cfef48aa99 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -1254,7 +1254,6 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
1254 uint32_t msg_id, pp_state = 0; 1254 uint32_t msg_id, pp_state = 0;
1255 uint32_t pp_support_state = 0; 1255 uint32_t pp_support_state = 0;
1256 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1256 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1257 void *pp_handle = adev->powerplay.pp_handle;
1258 1257
1259 if (adev->cg_flags & (AMD_CG_SUPPORT_MC_LS | AMD_CG_SUPPORT_MC_MGCG)) { 1258 if (adev->cg_flags & (AMD_CG_SUPPORT_MC_LS | AMD_CG_SUPPORT_MC_MGCG)) {
1260 if (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) { 1259 if (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) {
@@ -1271,7 +1270,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
1271 PP_BLOCK_SYS_MC, 1270 PP_BLOCK_SYS_MC,
1272 pp_support_state, 1271 pp_support_state,
1273 pp_state); 1272 pp_state);
1274 amd_set_clockgating_by_smu(pp_handle, msg_id); 1273 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
1274 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1275 } 1275 }
1276 1276
1277 if (adev->cg_flags & (AMD_CG_SUPPORT_SDMA_LS | AMD_CG_SUPPORT_SDMA_MGCG)) { 1277 if (adev->cg_flags & (AMD_CG_SUPPORT_SDMA_LS | AMD_CG_SUPPORT_SDMA_MGCG)) {
@@ -1289,7 +1289,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
1289 PP_BLOCK_SYS_SDMA, 1289 PP_BLOCK_SYS_SDMA,
1290 pp_support_state, 1290 pp_support_state,
1291 pp_state); 1291 pp_state);
1292 amd_set_clockgating_by_smu(pp_handle, msg_id); 1292 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
1293 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1293 } 1294 }
1294 1295
1295 if (adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_MGCG)) { 1296 if (adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_MGCG)) {
@@ -1307,7 +1308,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
1307 PP_BLOCK_SYS_HDP, 1308 PP_BLOCK_SYS_HDP,
1308 pp_support_state, 1309 pp_support_state,
1309 pp_state); 1310 pp_state);
1310 amd_set_clockgating_by_smu(pp_handle, msg_id); 1311 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
1312 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1311 } 1313 }
1312 1314
1313 1315
@@ -1321,7 +1323,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
1321 PP_BLOCK_SYS_BIF, 1323 PP_BLOCK_SYS_BIF,
1322 PP_STATE_SUPPORT_LS, 1324 PP_STATE_SUPPORT_LS,
1323 pp_state); 1325 pp_state);
1324 amd_set_clockgating_by_smu(pp_handle, msg_id); 1326 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
1327 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1325 } 1328 }
1326 if (adev->cg_flags & AMD_CG_SUPPORT_BIF_MGCG) { 1329 if (adev->cg_flags & AMD_CG_SUPPORT_BIF_MGCG) {
1327 if (state == AMD_CG_STATE_UNGATE) 1330 if (state == AMD_CG_STATE_UNGATE)
@@ -1333,7 +1336,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
1333 PP_BLOCK_SYS_BIF, 1336 PP_BLOCK_SYS_BIF,
1334 PP_STATE_SUPPORT_CG, 1337 PP_STATE_SUPPORT_CG,
1335 pp_state); 1338 pp_state);
1336 amd_set_clockgating_by_smu(pp_handle, msg_id); 1339 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
1340 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1337 } 1341 }
1338 1342
1339 if (adev->cg_flags & AMD_CG_SUPPORT_DRM_LS) { 1343 if (adev->cg_flags & AMD_CG_SUPPORT_DRM_LS) {
@@ -1347,7 +1351,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
1347 PP_BLOCK_SYS_DRM, 1351 PP_BLOCK_SYS_DRM,
1348 PP_STATE_SUPPORT_LS, 1352 PP_STATE_SUPPORT_LS,
1349 pp_state); 1353 pp_state);
1350 amd_set_clockgating_by_smu(pp_handle, msg_id); 1354 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
1355 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1351 } 1356 }
1352 1357
1353 if (adev->cg_flags & AMD_CG_SUPPORT_ROM_MGCG) { 1358 if (adev->cg_flags & AMD_CG_SUPPORT_ROM_MGCG) {
@@ -1361,7 +1366,8 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
1361 PP_BLOCK_SYS_ROM, 1366 PP_BLOCK_SYS_ROM,
1362 PP_STATE_SUPPORT_CG, 1367 PP_STATE_SUPPORT_CG,
1363 pp_state); 1368 pp_state);
1364 amd_set_clockgating_by_smu(pp_handle, msg_id); 1369 if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
1370 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1365 } 1371 }
1366 return 0; 1372 return 0;
1367} 1373}
diff --git a/drivers/gpu/drm/amd/amdgpu/vid.h b/drivers/gpu/drm/amd/amdgpu/vid.h
index a6485254a169..dbf3703cbd1b 100644
--- a/drivers/gpu/drm/amd/amdgpu/vid.h
+++ b/drivers/gpu/drm/amd/amdgpu/vid.h
@@ -465,6 +465,16 @@
465#define VCE_CMD_UPDATE_PTB 0x00000107 465#define VCE_CMD_UPDATE_PTB 0x00000107
466#define VCE_CMD_FLUSH_TLB 0x00000108 466#define VCE_CMD_FLUSH_TLB 0x00000108
467 467
468/* HEVC ENC */
469#define HEVC_ENC_CMD_NO_OP 0x00000000
470#define HEVC_ENC_CMD_END 0x00000001
471#define HEVC_ENC_CMD_FENCE 0x00000003
472#define HEVC_ENC_CMD_TRAP 0x00000004
473#define HEVC_ENC_CMD_IB_VM 0x00000102
474#define HEVC_ENC_CMD_WAIT_GE 0x00000106
475#define HEVC_ENC_CMD_UPDATE_PTB 0x00000107
476#define HEVC_ENC_CMD_FLUSH_TLB 0x00000108
477
468/* mmPA_SC_RASTER_CONFIG mask */ 478/* mmPA_SC_RASTER_CONFIG mask */
469#define RB_MAP_PKR0(x) ((x) << 0) 479#define RB_MAP_PKR0(x) ((x) << 0)
470#define RB_MAP_PKR0_MASK (0x3 << 0) 480#define RB_MAP_PKR0_MASK (0x3 << 0)
generated by cgit v1.2.2 (git 2.25.0) at 2025-05-31 23:04:48 -0400