aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/amd/amdgpu
diff options
context:
space:
mode:
authorDave Airlie <airlied@redhat.com>2017-12-06 15:28:22 -0500
committerDave Airlie <airlied@redhat.com>2017-12-06 15:28:22 -0500
commit9c606cd4117a3c45e04a6616b1a0dbeb18eeee62 (patch)
treeaa6c1db29e1a3f687c81fa03aecd24992a76e993 /drivers/gpu/drm/amd/amdgpu
parentc5dd52f653fa74f8f4771425c6db33609ad21258 (diff)
parent3997eea57caf542e9327df9b6bb2882a57c4c421 (diff)
Merge branch 'drm-next-4.16' of git://people.freedesktop.org/~agd5f/linux into drm-next
First feature request for 4.16. Highlights: - RV and Vega header cleanups - TTM operation context support - 48 bit GPUVM fixes for Vega/RV - More smatch fixes - ECC support for vega10 - Resizeable BAR support - Multi-display sync support in DC - SR-IOV fixes - Various scheduler improvements - GPU reset fixes and vram lost tracking - Clean up DC/powerplay interfaces - DCN display fixes - Various DC fixes * 'drm-next-4.16' of git://people.freedesktop.org/~agd5f/linux: (291 commits) drm/radeon: Use drm_fb_helper_lastclose() and _poll_changed() drm/amdgpu: Use drm_fb_helper_lastclose() and _poll_changed() drm/amd/display: Use drm_fb_helper_poll_changed() drm/ttm: swap consecutive allocated pooled pages v4 drm/amdgpu: fix amdgpu_sync_resv v2 drm/ttm: swap consecutive allocated cached pages v3 drm/amd/amdgpu: set gtt size according to system memory size only drm/amdgpu: Get rid of dep_sync as a seperate object. drm/amdgpu: allow specifying vm_block_size for multi level PDs v2 drm/amdgpu: move validation of the VM size into the VM code drm/amdgpu: allow non pot VM size values drm/amdgpu: choose number of VM levels based on VM size drm/amdgpu: unify VM size handling of Vega10 with older generation drm/amdgpu: fix amdgpu_vm_num_entries drm/amdgpu: fix VM PD addr shift drm/amdgpu: correct vce4.0 fw config for SRIOV (V2) drm/amd/display: Don't call dm_log_to_buffer directly in dc_conn_log drm/amd/display: Add dm_logger_append_va API drm/ttm: Use a static string instead of an array of char * drm/amd/display: remove usage of legacy_cursor_update ...
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu')
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu.h29
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c62
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c44
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_device.c530
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_display.c9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_display.h6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c14
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c27
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c47
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c66
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c22
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c59
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_job.c52
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c24
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.c90
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c22
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c22
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c174
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c45
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c108
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c35
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c126
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h21
-rw-r--r--drivers/gpu/drm/amd/amdgpu/atombios_i2c.c11
-rw-r--r--drivers/gpu/drm/amd/amdgpu/ci_dpm.c16
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cik.c40
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cik_sdma.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v10_0.c20
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v11_0.c22
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v6_0.c12
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v8_0.c12
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_virtual.c14
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c24
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c29
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c100
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c37
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c23
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c29
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c43
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c275
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c15
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c27
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c26
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/psp_v10_0.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/psp_v3_1.c12
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c24
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c30
-rw-r--r--drivers/gpu/drm/amd/amdgpu/si.c34
-rw-r--r--drivers/gpu/drm/amd/amdgpu/si_dma.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/si_dpm.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/soc15.c28
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c30
-rwxr-xr-x[-rw-r--r--]drivers/gpu/drm/amd/amdgpu/vce_v4_0.c50
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c12
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vega10_ih.c52
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vi.c10
81 files changed, 1633 insertions, 1247 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 0b14b5373783..5e2958a79928 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -47,6 +47,8 @@
47#include <drm/amdgpu_drm.h> 47#include <drm/amdgpu_drm.h>
48 48
49#include <kgd_kfd_interface.h> 49#include <kgd_kfd_interface.h>
50#include "dm_pp_interface.h"
51#include "kgd_pp_interface.h"
50 52
51#include "amd_shared.h" 53#include "amd_shared.h"
52#include "amdgpu_mode.h" 54#include "amdgpu_mode.h"
@@ -59,7 +61,6 @@
59#include "amdgpu_sync.h" 61#include "amdgpu_sync.h"
60#include "amdgpu_ring.h" 62#include "amdgpu_ring.h"
61#include "amdgpu_vm.h" 63#include "amdgpu_vm.h"
62#include "amd_powerplay.h"
63#include "amdgpu_dpm.h" 64#include "amdgpu_dpm.h"
64#include "amdgpu_acp.h" 65#include "amdgpu_acp.h"
65#include "amdgpu_uvd.h" 66#include "amdgpu_uvd.h"
@@ -67,11 +68,11 @@
67#include "amdgpu_vcn.h" 68#include "amdgpu_vcn.h"
68#include "amdgpu_mn.h" 69#include "amdgpu_mn.h"
69#include "amdgpu_dm.h" 70#include "amdgpu_dm.h"
70
71#include "gpu_scheduler.h" 71#include "gpu_scheduler.h"
72#include "amdgpu_virt.h" 72#include "amdgpu_virt.h"
73#include "amdgpu_gart.h" 73#include "amdgpu_gart.h"
74 74
75
75/* 76/*
76 * Modules parameters. 77 * Modules parameters.
77 */ 78 */
@@ -177,6 +178,10 @@ extern int amdgpu_cik_support;
177#define CIK_CURSOR_WIDTH 128 178#define CIK_CURSOR_WIDTH 128
178#define CIK_CURSOR_HEIGHT 128 179#define CIK_CURSOR_HEIGHT 128
179 180
181/* GPU RESET flags */
182#define AMDGPU_RESET_INFO_VRAM_LOST (1 << 0)
183#define AMDGPU_RESET_INFO_FULLRESET (1 << 1)
184
180struct amdgpu_device; 185struct amdgpu_device;
181struct amdgpu_ib; 186struct amdgpu_ib;
182struct amdgpu_cs_parser; 187struct amdgpu_cs_parser;
@@ -735,6 +740,7 @@ struct amdgpu_ctx {
735 struct amdgpu_device *adev; 740 struct amdgpu_device *adev;
736 struct amdgpu_queue_mgr queue_mgr; 741 struct amdgpu_queue_mgr queue_mgr;
737 unsigned reset_counter; 742 unsigned reset_counter;
743 unsigned reset_counter_query;
738 uint32_t vram_lost_counter; 744 uint32_t vram_lost_counter;
739 spinlock_t ring_lock; 745 spinlock_t ring_lock;
740 struct dma_fence **fences; 746 struct dma_fence **fences;
@@ -743,6 +749,7 @@ struct amdgpu_ctx {
743 enum amd_sched_priority init_priority; 749 enum amd_sched_priority init_priority;
744 enum amd_sched_priority override_priority; 750 enum amd_sched_priority override_priority;
745 struct mutex lock; 751 struct mutex lock;
752 atomic_t guilty;
746}; 753};
747 754
748struct amdgpu_ctx_mgr { 755struct amdgpu_ctx_mgr {
@@ -1114,7 +1121,6 @@ struct amdgpu_job {
1114 struct amdgpu_vm *vm; 1121 struct amdgpu_vm *vm;
1115 struct amdgpu_ring *ring; 1122 struct amdgpu_ring *ring;
1116 struct amdgpu_sync sync; 1123 struct amdgpu_sync sync;
1117 struct amdgpu_sync dep_sync;
1118 struct amdgpu_sync sched_sync; 1124 struct amdgpu_sync sched_sync;
1119 struct amdgpu_ib *ibs; 1125 struct amdgpu_ib *ibs;
1120 struct dma_fence *fence; /* the hw fence */ 1126 struct dma_fence *fence; /* the hw fence */
@@ -1405,6 +1411,7 @@ struct amdgpu_fw_vram_usage {
1405}; 1411};
1406 1412
1407int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev); 1413int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev);
1414void amdgpu_fw_reserve_vram_fini(struct amdgpu_device *adev);
1408 1415
1409/* 1416/*
1410 * CGS 1417 * CGS
@@ -1421,6 +1428,13 @@ typedef void (*amdgpu_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
1421typedef uint32_t (*amdgpu_block_rreg_t)(struct amdgpu_device*, uint32_t, uint32_t); 1428typedef uint32_t (*amdgpu_block_rreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
1422typedef void (*amdgpu_block_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t, uint32_t); 1429typedef void (*amdgpu_block_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t, uint32_t);
1423 1430
1431struct amd_powerplay {
1432 struct cgs_device *cgs_device;
1433 void *pp_handle;
1434 const struct amd_ip_funcs *ip_funcs;
1435 const struct amd_pm_funcs *pp_funcs;
1436};
1437
1424#define AMDGPU_RESET_MAGIC_NUM 64 1438#define AMDGPU_RESET_MAGIC_NUM 64
1425struct amdgpu_device { 1439struct amdgpu_device {
1426 struct device *dev; 1440 struct device *dev;
@@ -1616,9 +1630,6 @@ struct amdgpu_device {
1616 /* link all shadow bo */ 1630 /* link all shadow bo */
1617 struct list_head shadow_list; 1631 struct list_head shadow_list;
1618 struct mutex shadow_list_lock; 1632 struct mutex shadow_list_lock;
1619 /* link all gtt */
1620 spinlock_t gtt_list_lock;
1621 struct list_head gtt_list;
1622 /* keep an lru list of rings by HW IP */ 1633 /* keep an lru list of rings by HW IP */
1623 struct list_head ring_lru_list; 1634 struct list_head ring_lru_list;
1624 spinlock_t ring_lru_list_lock; 1635 spinlock_t ring_lru_list_lock;
@@ -1629,7 +1640,8 @@ struct amdgpu_device {
1629 1640
1630 /* record last mm index being written through WREG32*/ 1641 /* record last mm index being written through WREG32*/
1631 unsigned long last_mm_index; 1642 unsigned long last_mm_index;
1632 bool in_sriov_reset; 1643 bool in_gpu_reset;
1644 struct mutex lock_reset;
1633}; 1645};
1634 1646
1635static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev) 1647static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
@@ -1823,7 +1835,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
1823#define amdgpu_psp_check_fw_loading_status(adev, i) (adev)->firmware.funcs->check_fw_loading_status((adev), (i)) 1835#define amdgpu_psp_check_fw_loading_status(adev, i) (adev)->firmware.funcs->check_fw_loading_status((adev), (i))
1824 1836
1825/* Common functions */ 1837/* Common functions */
1826int amdgpu_gpu_reset(struct amdgpu_device *adev); 1838int amdgpu_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job* job);
1827bool amdgpu_need_backup(struct amdgpu_device *adev); 1839bool amdgpu_need_backup(struct amdgpu_device *adev);
1828void amdgpu_pci_config_reset(struct amdgpu_device *adev); 1840void amdgpu_pci_config_reset(struct amdgpu_device *adev);
1829bool amdgpu_need_post(struct amdgpu_device *adev); 1841bool amdgpu_need_post(struct amdgpu_device *adev);
@@ -1835,6 +1847,7 @@ void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
1835bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo); 1847bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
1836void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base); 1848void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base);
1837void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc); 1849void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);
1850int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev);
1838void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size); 1851void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size);
1839int amdgpu_ttm_init(struct amdgpu_device *adev); 1852int amdgpu_ttm_init(struct amdgpu_device *adev);
1840void amdgpu_ttm_fini(struct amdgpu_device *adev); 1853void amdgpu_ttm_fini(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 5432af39a674..c70cda04dbfb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -85,7 +85,7 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
85 kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions(); 85 kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions();
86 break; 86 break;
87 default: 87 default:
88 dev_info(adev->dev, "kfd not supported on this ASIC\n"); 88 dev_dbg(adev->dev, "kfd not supported on this ASIC\n");
89 return; 89 return;
90 } 90 }
91 91
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index f450b69323fa..39f4d0df1ada 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -690,12 +690,12 @@ int amdgpu_atombios_get_clock_info(struct amdgpu_device *adev)
690 le32_to_cpu(firmware_info->info_21.ulDefaultDispEngineClkFreq); 690 le32_to_cpu(firmware_info->info_21.ulDefaultDispEngineClkFreq);
691 /* set a reasonable default for DP */ 691 /* set a reasonable default for DP */
692 if (adev->clock.default_dispclk < 53900) { 692 if (adev->clock.default_dispclk < 53900) {
693 DRM_INFO("Changing default dispclk from %dMhz to 600Mhz\n", 693 DRM_DEBUG("Changing default dispclk from %dMhz to 600Mhz\n",
694 adev->clock.default_dispclk / 100); 694 adev->clock.default_dispclk / 100);
695 adev->clock.default_dispclk = 60000; 695 adev->clock.default_dispclk = 60000;
696 } else if (adev->clock.default_dispclk <= 60000) { 696 } else if (adev->clock.default_dispclk <= 60000) {
697 DRM_INFO("Changing default dispclk from %dMhz to 625Mhz\n", 697 DRM_DEBUG("Changing default dispclk from %dMhz to 625Mhz\n",
698 adev->clock.default_dispclk / 100); 698 adev->clock.default_dispclk / 100);
699 adev->clock.default_dispclk = 62500; 699 adev->clock.default_dispclk = 62500;
700 } 700 }
701 adev->clock.dp_extclk = 701 adev->clock.dp_extclk =
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index f2b72c7c6857..85d2149b9dbe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -948,7 +948,6 @@ static int amdgpu_cgs_get_active_displays_info(struct cgs_device *cgs_device,
948 (amdgpu_crtc->v_border * 2); 948 (amdgpu_crtc->v_border * 2);
949 mode_info->vblank_time_us = vblank_lines * line_time_us; 949 mode_info->vblank_time_us = vblank_lines * line_time_us;
950 mode_info->refresh_rate = drm_mode_vrefresh(&amdgpu_crtc->hw_mode); 950 mode_info->refresh_rate = drm_mode_vrefresh(&amdgpu_crtc->hw_mode);
951 mode_info->ref_clock = adev->clock.spll.reference_freq;
952 mode_info = NULL; 951 mode_info = NULL;
953 } 952 }
954 } 953 }
@@ -958,7 +957,6 @@ static int amdgpu_cgs_get_active_displays_info(struct cgs_device *cgs_device,
958 if (mode_info != NULL) { 957 if (mode_info != NULL) {
959 mode_info->vblank_time_us = adev->pm.pm_display_cfg.min_vblank_time; 958 mode_info->vblank_time_us = adev->pm.pm_display_cfg.min_vblank_time;
960 mode_info->refresh_rate = adev->pm.pm_display_cfg.vrefresh; 959 mode_info->refresh_rate = adev->pm.pm_display_cfg.vrefresh;
961 mode_info->ref_clock = adev->clock.spll.reference_freq;
962 } 960 }
963 } 961 }
964 return 0; 962 return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 57abf7abd7a9..4cea9ab237ac 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -90,6 +90,12 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
90 goto free_chunk; 90 goto free_chunk;
91 } 91 }
92 92
93 /* skip guilty context job */
94 if (atomic_read(&p->ctx->guilty) == 1) {
95 ret = -ECANCELED;
96 goto free_chunk;
97 }
98
93 mutex_lock(&p->ctx->lock); 99 mutex_lock(&p->ctx->lock);
94 100
95 /* get chunks */ 101 /* get chunks */
@@ -337,7 +343,7 @@ static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
337 struct amdgpu_bo *bo) 343 struct amdgpu_bo *bo)
338{ 344{
339 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 345 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
340 u64 initial_bytes_moved, bytes_moved; 346 struct ttm_operation_ctx ctx = { true, false };
341 uint32_t domain; 347 uint32_t domain;
342 int r; 348 int r;
343 349
@@ -367,15 +373,13 @@ static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
367 373
368retry: 374retry:
369 amdgpu_ttm_placement_from_domain(bo, domain); 375 amdgpu_ttm_placement_from_domain(bo, domain);
370 initial_bytes_moved = atomic64_read(&adev->num_bytes_moved); 376 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
371 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 377
372 bytes_moved = atomic64_read(&adev->num_bytes_moved) - 378 p->bytes_moved += ctx.bytes_moved;
373 initial_bytes_moved;
374 p->bytes_moved += bytes_moved;
375 if (adev->mc.visible_vram_size < adev->mc.real_vram_size && 379 if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
376 bo->tbo.mem.mem_type == TTM_PL_VRAM && 380 bo->tbo.mem.mem_type == TTM_PL_VRAM &&
377 bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT) 381 bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)
378 p->bytes_moved_vis += bytes_moved; 382 p->bytes_moved_vis += ctx.bytes_moved;
379 383
380 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { 384 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
381 domain = bo->allowed_domains; 385 domain = bo->allowed_domains;
@@ -390,6 +394,7 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
390 struct amdgpu_bo *validated) 394 struct amdgpu_bo *validated)
391{ 395{
392 uint32_t domain = validated->allowed_domains; 396 uint32_t domain = validated->allowed_domains;
397 struct ttm_operation_ctx ctx = { true, false };
393 int r; 398 int r;
394 399
395 if (!p->evictable) 400 if (!p->evictable)
@@ -431,7 +436,7 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
431 bo->tbo.mem.mem_type == TTM_PL_VRAM && 436 bo->tbo.mem.mem_type == TTM_PL_VRAM &&
432 bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT; 437 bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT;
433 initial_bytes_moved = atomic64_read(&adev->num_bytes_moved); 438 initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
434 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 439 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
435 bytes_moved = atomic64_read(&adev->num_bytes_moved) - 440 bytes_moved = atomic64_read(&adev->num_bytes_moved) -
436 initial_bytes_moved; 441 initial_bytes_moved;
437 p->bytes_moved += bytes_moved; 442 p->bytes_moved += bytes_moved;
@@ -470,6 +475,7 @@ static int amdgpu_cs_validate(void *param, struct amdgpu_bo *bo)
470static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, 475static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
471 struct list_head *validated) 476 struct list_head *validated)
472{ 477{
478 struct ttm_operation_ctx ctx = { true, false };
473 struct amdgpu_bo_list_entry *lobj; 479 struct amdgpu_bo_list_entry *lobj;
474 int r; 480 int r;
475 481
@@ -487,8 +493,7 @@ static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
487 lobj->user_pages) { 493 lobj->user_pages) {
488 amdgpu_ttm_placement_from_domain(bo, 494 amdgpu_ttm_placement_from_domain(bo,
489 AMDGPU_GEM_DOMAIN_CPU); 495 AMDGPU_GEM_DOMAIN_CPU);
490 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, 496 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
491 false);
492 if (r) 497 if (r)
493 return r; 498 return r;
494 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, 499 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
@@ -678,7 +683,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
678 if (!r && p->uf_entry.robj) { 683 if (!r && p->uf_entry.robj) {
679 struct amdgpu_bo *uf = p->uf_entry.robj; 684 struct amdgpu_bo *uf = p->uf_entry.robj;
680 685
681 r = amdgpu_ttm_bind(&uf->tbo, &uf->tbo.mem); 686 r = amdgpu_ttm_alloc_gart(&uf->tbo);
682 p->job->uf_addr += amdgpu_bo_gpu_offset(uf); 687 p->job->uf_addr += amdgpu_bo_gpu_offset(uf);
683 } 688 }
684 689
@@ -781,7 +786,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
781 return r; 786 return r;
782 787
783 r = amdgpu_sync_fence(adev, &p->job->sync, 788 r = amdgpu_sync_fence(adev, &p->job->sync,
784 fpriv->prt_va->last_pt_update); 789 fpriv->prt_va->last_pt_update, false);
785 if (r) 790 if (r)
786 return r; 791 return r;
787 792
@@ -795,7 +800,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
795 return r; 800 return r;
796 801
797 f = bo_va->last_pt_update; 802 f = bo_va->last_pt_update;
798 r = amdgpu_sync_fence(adev, &p->job->sync, f); 803 r = amdgpu_sync_fence(adev, &p->job->sync, f, false);
799 if (r) 804 if (r)
800 return r; 805 return r;
801 } 806 }
@@ -818,7 +823,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
818 return r; 823 return r;
819 824
820 f = bo_va->last_pt_update; 825 f = bo_va->last_pt_update;
821 r = amdgpu_sync_fence(adev, &p->job->sync, f); 826 r = amdgpu_sync_fence(adev, &p->job->sync, f, false);
822 if (r) 827 if (r)
823 return r; 828 return r;
824 } 829 }
@@ -829,7 +834,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
829 if (r) 834 if (r)
830 return r; 835 return r;
831 836
832 r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update); 837 r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update, false);
833 if (r) 838 if (r)
834 return r; 839 return r;
835 840
@@ -865,8 +870,8 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
865 struct amdgpu_bo_va_mapping *m; 870 struct amdgpu_bo_va_mapping *m;
866 struct amdgpu_bo *aobj = NULL; 871 struct amdgpu_bo *aobj = NULL;
867 struct amdgpu_cs_chunk *chunk; 872 struct amdgpu_cs_chunk *chunk;
873 uint64_t offset, va_start;
868 struct amdgpu_ib *ib; 874 struct amdgpu_ib *ib;
869 uint64_t offset;
870 uint8_t *kptr; 875 uint8_t *kptr;
871 876
872 chunk = &p->chunks[i]; 877 chunk = &p->chunks[i];
@@ -876,14 +881,14 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
876 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) 881 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
877 continue; 882 continue;
878 883
879 r = amdgpu_cs_find_mapping(p, chunk_ib->va_start, 884 va_start = chunk_ib->va_start & AMDGPU_VA_HOLE_MASK;
880 &aobj, &m); 885 r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);
881 if (r) { 886 if (r) {
882 DRM_ERROR("IB va_start is invalid\n"); 887 DRM_ERROR("IB va_start is invalid\n");
883 return r; 888 return r;
884 } 889 }
885 890
886 if ((chunk_ib->va_start + chunk_ib->ib_bytes) > 891 if ((va_start + chunk_ib->ib_bytes) >
887 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) { 892 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
888 DRM_ERROR("IB va_start+ib_bytes is invalid\n"); 893 DRM_ERROR("IB va_start+ib_bytes is invalid\n");
889 return -EINVAL; 894 return -EINVAL;
@@ -896,7 +901,7 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
896 } 901 }
897 902
898 offset = m->start * AMDGPU_GPU_PAGE_SIZE; 903 offset = m->start * AMDGPU_GPU_PAGE_SIZE;
899 kptr += chunk_ib->va_start - offset; 904 kptr += va_start - offset;
900 905
901 memcpy(ib->ptr, kptr, chunk_ib->ib_bytes); 906 memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
902 amdgpu_bo_kunmap(aobj); 907 amdgpu_bo_kunmap(aobj);
@@ -1033,8 +1038,8 @@ static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
1033 amdgpu_ctx_put(ctx); 1038 amdgpu_ctx_put(ctx);
1034 return r; 1039 return r;
1035 } else if (fence) { 1040 } else if (fence) {
1036 r = amdgpu_sync_fence(p->adev, &p->job->sync, 1041 r = amdgpu_sync_fence(p->adev, &p->job->sync, fence,
1037 fence); 1042 true);
1038 dma_fence_put(fence); 1043 dma_fence_put(fence);
1039 amdgpu_ctx_put(ctx); 1044 amdgpu_ctx_put(ctx);
1040 if (r) 1045 if (r)
@@ -1053,7 +1058,7 @@ static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p,
1053 if (r) 1058 if (r)
1054 return r; 1059 return r;
1055 1060
1056 r = amdgpu_sync_fence(p->adev, &p->job->sync, fence); 1061 r = amdgpu_sync_fence(p->adev, &p->job->sync, fence, true);
1057 dma_fence_put(fence); 1062 dma_fence_put(fence);
1058 1063
1059 return r; 1064 return r;
@@ -1194,11 +1199,10 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
1194 job->uf_sequence = seq; 1199 job->uf_sequence = seq;
1195 1200
1196 amdgpu_job_free_resources(job); 1201 amdgpu_job_free_resources(job);
1197 amdgpu_ring_priority_get(job->ring, 1202 amdgpu_ring_priority_get(job->ring, job->base.s_priority);
1198 amd_sched_get_job_priority(&job->base));
1199 1203
1200 trace_amdgpu_cs_ioctl(job); 1204 trace_amdgpu_cs_ioctl(job);
1201 amd_sched_entity_push_job(&job->base); 1205 amd_sched_entity_push_job(&job->base, entity);
1202 1206
1203 ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence); 1207 ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
1204 amdgpu_mn_unlock(p->mn); 1208 amdgpu_mn_unlock(p->mn);
@@ -1570,6 +1574,7 @@ int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1570 struct amdgpu_bo_va_mapping **map) 1574 struct amdgpu_bo_va_mapping **map)
1571{ 1575{
1572 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 1576 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
1577 struct ttm_operation_ctx ctx = { false, false };
1573 struct amdgpu_vm *vm = &fpriv->vm; 1578 struct amdgpu_vm *vm = &fpriv->vm;
1574 struct amdgpu_bo_va_mapping *mapping; 1579 struct amdgpu_bo_va_mapping *mapping;
1575 int r; 1580 int r;
@@ -1590,11 +1595,10 @@ int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1590 if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) { 1595 if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
1591 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 1596 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1592 amdgpu_ttm_placement_from_domain(*bo, (*bo)->allowed_domains); 1597 amdgpu_ttm_placement_from_domain(*bo, (*bo)->allowed_domains);
1593 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, false, 1598 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx);
1594 false);
1595 if (r) 1599 if (r)
1596 return r; 1600 return r;
1597 } 1601 }
1598 1602
1599 return amdgpu_ttm_bind(&(*bo)->tbo, &(*bo)->tbo.mem); 1603 return amdgpu_ttm_alloc_gart(&(*bo)->tbo);
1600} 1604}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index c184468e2b2b..d71dc164b469 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -75,6 +75,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
75 } 75 }
76 76
77 ctx->reset_counter = atomic_read(&adev->gpu_reset_counter); 77 ctx->reset_counter = atomic_read(&adev->gpu_reset_counter);
78 ctx->reset_counter_query = ctx->reset_counter;
78 ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter); 79 ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
79 ctx->init_priority = priority; 80 ctx->init_priority = priority;
80 ctx->override_priority = AMD_SCHED_PRIORITY_UNSET; 81 ctx->override_priority = AMD_SCHED_PRIORITY_UNSET;
@@ -90,7 +91,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
90 continue; 91 continue;
91 92
92 r = amd_sched_entity_init(&ring->sched, &ctx->rings[i].entity, 93 r = amd_sched_entity_init(&ring->sched, &ctx->rings[i].entity,
93 rq, amdgpu_sched_jobs); 94 rq, amdgpu_sched_jobs, &ctx->guilty);
94 if (r) 95 if (r)
95 goto failed; 96 goto failed;
96 } 97 }
@@ -216,11 +217,45 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
216 /* determine if a GPU reset has occured since the last call */ 217 /* determine if a GPU reset has occured since the last call */
217 reset_counter = atomic_read(&adev->gpu_reset_counter); 218 reset_counter = atomic_read(&adev->gpu_reset_counter);
218 /* TODO: this should ideally return NO, GUILTY, or INNOCENT. */ 219 /* TODO: this should ideally return NO, GUILTY, or INNOCENT. */
219 if (ctx->reset_counter == reset_counter) 220 if (ctx->reset_counter_query == reset_counter)
220 out->state.reset_status = AMDGPU_CTX_NO_RESET; 221 out->state.reset_status = AMDGPU_CTX_NO_RESET;
221 else 222 else
222 out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET; 223 out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
223 ctx->reset_counter = reset_counter; 224 ctx->reset_counter_query = reset_counter;
225
226 mutex_unlock(&mgr->lock);
227 return 0;
228}
229
230static int amdgpu_ctx_query2(struct amdgpu_device *adev,
231 struct amdgpu_fpriv *fpriv, uint32_t id,
232 union drm_amdgpu_ctx_out *out)
233{
234 struct amdgpu_ctx *ctx;
235 struct amdgpu_ctx_mgr *mgr;
236
237 if (!fpriv)
238 return -EINVAL;
239
240 mgr = &fpriv->ctx_mgr;
241 mutex_lock(&mgr->lock);
242 ctx = idr_find(&mgr->ctx_handles, id);
243 if (!ctx) {
244 mutex_unlock(&mgr->lock);
245 return -EINVAL;
246 }
247
248 out->state.flags = 0x0;
249 out->state.hangs = 0x0;
250
251 if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter))
252 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET;
253
254 if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter))
255 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST;
256
257 if (atomic_read(&ctx->guilty))
258 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY;
224 259
225 mutex_unlock(&mgr->lock); 260 mutex_unlock(&mgr->lock);
226 return 0; 261 return 0;
@@ -257,6 +292,9 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
257 case AMDGPU_CTX_OP_QUERY_STATE: 292 case AMDGPU_CTX_OP_QUERY_STATE:
258 r = amdgpu_ctx_query(adev, fpriv, id, &args->out); 293 r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
259 break; 294 break;
295 case AMDGPU_CTX_OP_QUERY_STATE2:
296 r = amdgpu_ctx_query2(adev, fpriv, id, &args->out);
297 break;
260 default: 298 default:
261 return -EINVAL; 299 return -EINVAL;
262 } 300 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 3573ecdb06ee..70c9e5756b02 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -410,6 +410,9 @@ static int amdgpu_doorbell_init(struct amdgpu_device *adev)
410 return 0; 410 return 0;
411 } 411 }
412 412
413 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
414 return -EINVAL;
415
413 /* doorbell bar mapping */ 416 /* doorbell bar mapping */
414 adev->doorbell.base = pci_resource_start(adev->pdev, 2); 417 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
415 adev->doorbell.size = pci_resource_len(adev->pdev, 2); 418 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
@@ -575,41 +578,13 @@ void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
575 * @base: base address at which to put VRAM 578 * @base: base address at which to put VRAM
576 * 579 *
577 * Function will try to place VRAM at base address provided 580 * Function will try to place VRAM at base address provided
578 * as parameter (which is so far either PCI aperture address or 581 * as parameter.
579 * for IGP TOM base address).
580 *
581 * If there is not enough space to fit the unvisible VRAM in the 32bits
582 * address space then we limit the VRAM size to the aperture.
583 *
584 * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
585 * this shouldn't be a problem as we are using the PCI aperture as a reference.
586 * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
587 * not IGP.
588 *
589 * Note: we use mc_vram_size as on some board we need to program the mc to
590 * cover the whole aperture even if VRAM size is inferior to aperture size
591 * Novell bug 204882 + along with lots of ubuntu ones
592 *
593 * Note: when limiting vram it's safe to overwritte real_vram_size because
594 * we are not in case where real_vram_size is inferior to mc_vram_size (ie
595 * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
596 * ones)
597 *
598 * Note: IGP TOM addr should be the same as the aperture addr, we don't
599 * explicitly check for that though.
600 *
601 * FIXME: when reducing VRAM size align new size on power of 2.
602 */ 582 */
603void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base) 583void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
604{ 584{
605 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20; 585 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
606 586
607 mc->vram_start = base; 587 mc->vram_start = base;
608 if (mc->mc_vram_size > (adev->mc.mc_mask - base + 1)) {
609 dev_warn(adev->dev, "limiting VRAM to PCI aperture size\n");
610 mc->real_vram_size = mc->aper_size;
611 mc->mc_vram_size = mc->aper_size;
612 }
613 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; 588 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
614 if (limit && limit < mc->real_vram_size) 589 if (limit && limit < mc->real_vram_size)
615 mc->real_vram_size = limit; 590 mc->real_vram_size = limit;
@@ -647,7 +622,10 @@ void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
647 dev_warn(adev->dev, "limiting GTT\n"); 622 dev_warn(adev->dev, "limiting GTT\n");
648 mc->gart_size = size_af; 623 mc->gart_size = size_af;
649 } 624 }
650 mc->gart_start = mc->vram_end + 1; 625 /* VCE doesn't like it when BOs cross a 4GB segment, so align
626 * the GART base on a 4GB boundary as well.
627 */
628 mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL);
651 } 629 }
652 mc->gart_end = mc->gart_start + mc->gart_size - 1; 630 mc->gart_end = mc->gart_start + mc->gart_size - 1;
653 dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n", 631 dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
@@ -679,9 +657,13 @@ void amdgpu_fw_reserve_vram_fini(struct amdgpu_device *adev)
679 */ 657 */
680int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev) 658int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev)
681{ 659{
660 struct ttm_operation_ctx ctx = { false, false };
682 int r = 0; 661 int r = 0;
683 u64 gpu_addr; 662 int i;
684 u64 vram_size = adev->mc.visible_vram_size; 663 u64 vram_size = adev->mc.visible_vram_size;
664 u64 offset = adev->fw_vram_usage.start_offset;
665 u64 size = adev->fw_vram_usage.size;
666 struct amdgpu_bo *bo;
685 667
686 adev->fw_vram_usage.va = NULL; 668 adev->fw_vram_usage.va = NULL;
687 adev->fw_vram_usage.reserved_bo = NULL; 669 adev->fw_vram_usage.reserved_bo = NULL;
@@ -690,7 +672,7 @@ int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev)
690 adev->fw_vram_usage.size <= vram_size) { 672 adev->fw_vram_usage.size <= vram_size) {
691 673
692 r = amdgpu_bo_create(adev, adev->fw_vram_usage.size, 674 r = amdgpu_bo_create(adev, adev->fw_vram_usage.size,
693 PAGE_SIZE, true, 0, 675 PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
694 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | 676 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
695 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, NULL, NULL, 0, 677 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, NULL, NULL, 0,
696 &adev->fw_vram_usage.reserved_bo); 678 &adev->fw_vram_usage.reserved_bo);
@@ -700,11 +682,28 @@ int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev)
700 r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false); 682 r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
701 if (r) 683 if (r)
702 goto error_reserve; 684 goto error_reserve;
685
686 /* remove the original mem node and create a new one at the
687 * request position
688 */
689 bo = adev->fw_vram_usage.reserved_bo;
690 offset = ALIGN(offset, PAGE_SIZE);
691 for (i = 0; i < bo->placement.num_placement; ++i) {
692 bo->placements[i].fpfn = offset >> PAGE_SHIFT;
693 bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
694 }
695
696 ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem);
697 r = ttm_bo_mem_space(&bo->tbo, &bo->placement,
698 &bo->tbo.mem, &ctx);
699 if (r)
700 goto error_pin;
701
703 r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo, 702 r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
704 AMDGPU_GEM_DOMAIN_VRAM, 703 AMDGPU_GEM_DOMAIN_VRAM,
705 adev->fw_vram_usage.start_offset, 704 adev->fw_vram_usage.start_offset,
706 (adev->fw_vram_usage.start_offset + 705 (adev->fw_vram_usage.start_offset +
707 adev->fw_vram_usage.size), &gpu_addr); 706 adev->fw_vram_usage.size), NULL);
708 if (r) 707 if (r)
709 goto error_pin; 708 goto error_pin;
710 r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo, 709 r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
@@ -728,6 +727,75 @@ error_create:
728 return r; 727 return r;
729} 728}
730 729
730/**
731 * amdgpu_device_resize_fb_bar - try to resize FB BAR
732 *
733 * @adev: amdgpu_device pointer
734 *
735 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
736 * to fail, but if any of the BARs is not accessible after the size we abort
737 * driver loading by returning -ENODEV.
738 */
739int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
740{
741 u64 space_needed = roundup_pow_of_two(adev->mc.real_vram_size);
742 u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
743 struct pci_bus *root;
744 struct resource *res;
745 unsigned i;
746 u16 cmd;
747 int r;
748
749 /* Bypass for VF */
750 if (amdgpu_sriov_vf(adev))
751 return 0;
752
753 /* Check if the root BUS has 64bit memory resources */
754 root = adev->pdev->bus;
755 while (root->parent)
756 root = root->parent;
757
758 pci_bus_for_each_resource(root, res, i) {
759 if (res && res->flags & IORESOURCE_MEM_64 &&
760 res->start > 0x100000000ull)
761 break;
762 }
763
764 /* Trying to resize is pointless without a root hub window above 4GB */
765 if (!res)
766 return 0;
767
768 /* Disable memory decoding while we change the BAR addresses and size */
769 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
770 pci_write_config_word(adev->pdev, PCI_COMMAND,
771 cmd & ~PCI_COMMAND_MEMORY);
772
773 /* Free the VRAM and doorbell BAR, we most likely need to move both. */
774 amdgpu_doorbell_fini(adev);
775 if (adev->asic_type >= CHIP_BONAIRE)
776 pci_release_resource(adev->pdev, 2);
777
778 pci_release_resource(adev->pdev, 0);
779
780 r = pci_resize_resource(adev->pdev, 0, rbar_size);
781 if (r == -ENOSPC)
782 DRM_INFO("Not enough PCI address space for a large BAR.");
783 else if (r && r != -ENOTSUPP)
784 DRM_ERROR("Problem resizing BAR0 (%d).", r);
785
786 pci_assign_unassigned_bus_resources(adev->pdev->bus);
787
788 /* When the doorbell or fb BAR isn't available we have no chance of
789 * using the device.
790 */
791 r = amdgpu_doorbell_init(adev);
792 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
793 return -ENODEV;
794
795 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
796
797 return 0;
798}
731 799
732/* 800/*
733 * GPU helpers function. 801 * GPU helpers function.
@@ -1029,7 +1097,7 @@ static int amdgpu_atombios_init(struct amdgpu_device *adev)
1029 atom_card_info->ioreg_read = cail_ioreg_read; 1097 atom_card_info->ioreg_read = cail_ioreg_read;
1030 atom_card_info->ioreg_write = cail_ioreg_write; 1098 atom_card_info->ioreg_write = cail_ioreg_write;
1031 } else { 1099 } else {
1032 DRM_INFO("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n"); 1100 DRM_DEBUG("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
1033 atom_card_info->ioreg_read = cail_reg_read; 1101 atom_card_info->ioreg_read = cail_reg_read;
1034 atom_card_info->ioreg_write = cail_reg_write; 1102 atom_card_info->ioreg_write = cail_reg_write;
1035 } 1103 }
@@ -1094,20 +1162,8 @@ static void amdgpu_check_block_size(struct amdgpu_device *adev)
1094 if (amdgpu_vm_block_size < 9) { 1162 if (amdgpu_vm_block_size < 9) {
1095 dev_warn(adev->dev, "VM page table size (%d) too small\n", 1163 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1096 amdgpu_vm_block_size); 1164 amdgpu_vm_block_size);
1097 goto def_value; 1165 amdgpu_vm_block_size = -1;
1098 } 1166 }
1099
1100 if (amdgpu_vm_block_size > 24 ||
1101 (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
1102 dev_warn(adev->dev, "VM page table size (%d) too large\n",
1103 amdgpu_vm_block_size);
1104 goto def_value;
1105 }
1106
1107 return;
1108
1109def_value:
1110 amdgpu_vm_block_size = -1;
1111} 1167}
1112 1168
1113static void amdgpu_check_vm_size(struct amdgpu_device *adev) 1169static void amdgpu_check_vm_size(struct amdgpu_device *adev)
@@ -1116,31 +1172,11 @@ static void amdgpu_check_vm_size(struct amdgpu_device *adev)
1116 if (amdgpu_vm_size == -1) 1172 if (amdgpu_vm_size == -1)
1117 return; 1173 return;
1118 1174
1119 if (!is_power_of_2(amdgpu_vm_size)) {
1120 dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
1121 amdgpu_vm_size);
1122 goto def_value;
1123 }
1124
1125 if (amdgpu_vm_size < 1) { 1175 if (amdgpu_vm_size < 1) {
1126 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", 1176 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1127 amdgpu_vm_size); 1177 amdgpu_vm_size);
1128 goto def_value; 1178 amdgpu_vm_size = -1;
1129 } 1179 }
1130
1131 /*
1132 * Max GPUVM size for Cayman, SI, CI VI are 40 bits.
1133 */
1134 if (amdgpu_vm_size > 1024) {
1135 dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
1136 amdgpu_vm_size);
1137 goto def_value;
1138 }
1139
1140 return;
1141
1142def_value:
1143 amdgpu_vm_size = -1;
1144} 1180}
1145 1181
1146/** 1182/**
@@ -1622,10 +1658,12 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
1622 if (r) 1658 if (r)
1623 return r; 1659 return r;
1624 1660
1661 amdgpu_amdkfd_device_probe(adev);
1662
1625 if (amdgpu_sriov_vf(adev)) { 1663 if (amdgpu_sriov_vf(adev)) {
1626 r = amdgpu_virt_request_full_gpu(adev, true); 1664 r = amdgpu_virt_request_full_gpu(adev, true);
1627 if (r) 1665 if (r)
1628 return r; 1666 return -EAGAIN;
1629 } 1667 }
1630 1668
1631 for (i = 0; i < adev->num_ip_blocks; i++) { 1669 for (i = 0; i < adev->num_ip_blocks; i++) {
@@ -1716,6 +1754,11 @@ static int amdgpu_init(struct amdgpu_device *adev)
1716 adev->ip_blocks[i].status.hw = true; 1754 adev->ip_blocks[i].status.hw = true;
1717 } 1755 }
1718 1756
1757 amdgpu_amdkfd_device_init(adev);
1758
1759 if (amdgpu_sriov_vf(adev))
1760 amdgpu_virt_release_full_gpu(adev, true);
1761
1719 return 0; 1762 return 0;
1720} 1763}
1721 1764
@@ -1783,6 +1826,7 @@ static int amdgpu_fini(struct amdgpu_device *adev)
1783{ 1826{
1784 int i, r; 1827 int i, r;
1785 1828
1829 amdgpu_amdkfd_device_fini(adev);
1786 /* need to disable SMC first */ 1830 /* need to disable SMC first */
1787 for (i = 0; i < adev->num_ip_blocks; i++) { 1831 for (i = 0; i < adev->num_ip_blocks; i++) {
1788 if (!adev->ip_blocks[i].status.hw) 1832 if (!adev->ip_blocks[i].status.hw)
@@ -1811,6 +1855,7 @@ static int amdgpu_fini(struct amdgpu_device *adev)
1811 if (!adev->ip_blocks[i].status.hw) 1855 if (!adev->ip_blocks[i].status.hw)
1812 continue; 1856 continue;
1813 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 1857 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1858 amdgpu_free_static_csa(adev);
1814 amdgpu_wb_fini(adev); 1859 amdgpu_wb_fini(adev);
1815 amdgpu_vram_scratch_fini(adev); 1860 amdgpu_vram_scratch_fini(adev);
1816 } 1861 }
@@ -1859,7 +1904,8 @@ static int amdgpu_fini(struct amdgpu_device *adev)
1859 } 1904 }
1860 1905
1861 if (amdgpu_sriov_vf(adev)) 1906 if (amdgpu_sriov_vf(adev))
1862 amdgpu_virt_release_full_gpu(adev, false); 1907 if (amdgpu_virt_release_full_gpu(adev, false))
1908 DRM_ERROR("failed to release exclusive mode on fini\n");
1863 1909
1864 return 0; 1910 return 0;
1865} 1911}
@@ -2163,6 +2209,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2163 mutex_init(&adev->mn_lock); 2209 mutex_init(&adev->mn_lock);
2164 mutex_init(&adev->virt.vf_errors.lock); 2210 mutex_init(&adev->virt.vf_errors.lock);
2165 hash_init(adev->mn_hash); 2211 hash_init(adev->mn_hash);
2212 mutex_init(&adev->lock_reset);
2166 2213
2167 amdgpu_check_arguments(adev); 2214 amdgpu_check_arguments(adev);
2168 2215
@@ -2179,9 +2226,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2179 INIT_LIST_HEAD(&adev->shadow_list); 2226 INIT_LIST_HEAD(&adev->shadow_list);
2180 mutex_init(&adev->shadow_list_lock); 2227 mutex_init(&adev->shadow_list_lock);
2181 2228
2182 INIT_LIST_HEAD(&adev->gtt_list);
2183 spin_lock_init(&adev->gtt_list_lock);
2184
2185 INIT_LIST_HEAD(&adev->ring_lru_list); 2229 INIT_LIST_HEAD(&adev->ring_lru_list);
2186 spin_lock_init(&adev->ring_lru_list_lock); 2230 spin_lock_init(&adev->ring_lru_list_lock);
2187 2231
@@ -2267,8 +2311,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2267 dev_err(adev->dev, "gpu post error!\n"); 2311 dev_err(adev->dev, "gpu post error!\n");
2268 goto failed; 2312 goto failed;
2269 } 2313 }
2270 } else {
2271 DRM_INFO("GPU post is not needed\n");
2272 } 2314 }
2273 2315
2274 if (adev->is_atom_fw) { 2316 if (adev->is_atom_fw) {
@@ -2305,6 +2347,18 @@ int amdgpu_device_init(struct amdgpu_device *adev,
2305 2347
2306 r = amdgpu_init(adev); 2348 r = amdgpu_init(adev);
2307 if (r) { 2349 if (r) {
2350 /* failed in exclusive mode due to timeout */
2351 if (amdgpu_sriov_vf(adev) &&
2352 !amdgpu_sriov_runtime(adev) &&
2353 amdgpu_virt_mmio_blocked(adev) &&
2354 !amdgpu_virt_wait_reset(adev)) {
2355 dev_err(adev->dev, "VF exclusive mode timeout\n");
2356 /* Don't send request since VF is inactive. */
2357 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
2358 adev->virt.ops = NULL;
2359 r = -EAGAIN;
2360 goto failed;
2361 }
2308 dev_err(adev->dev, "amdgpu_init failed\n"); 2362 dev_err(adev->dev, "amdgpu_init failed\n");
2309 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); 2363 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2310 amdgpu_fini(adev); 2364 amdgpu_fini(adev);
@@ -2392,6 +2446,7 @@ failed:
2392 amdgpu_vf_error_trans_all(adev); 2446 amdgpu_vf_error_trans_all(adev);
2393 if (runtime) 2447 if (runtime)
2394 vga_switcheroo_fini_domain_pm_ops(adev->dev); 2448 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2449
2395 return r; 2450 return r;
2396} 2451}
2397 2452
@@ -2414,7 +2469,6 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
2414 /* evict vram memory */ 2469 /* evict vram memory */
2415 amdgpu_bo_evict_vram(adev); 2470 amdgpu_bo_evict_vram(adev);
2416 amdgpu_ib_pool_fini(adev); 2471 amdgpu_ib_pool_fini(adev);
2417 amdgpu_fw_reserve_vram_fini(adev);
2418 amdgpu_fence_driver_fini(adev); 2472 amdgpu_fence_driver_fini(adev);
2419 amdgpu_fbdev_fini(adev); 2473 amdgpu_fbdev_fini(adev);
2420 r = amdgpu_fini(adev); 2474 r = amdgpu_fini(adev);
@@ -2819,163 +2873,172 @@ err:
2819 return r; 2873 return r;
2820} 2874}
2821 2875
2822/** 2876/*
2823 * amdgpu_sriov_gpu_reset - reset the asic 2877 * amdgpu_reset - reset ASIC/GPU for bare-metal or passthrough
2824 * 2878 *
2825 * @adev: amdgpu device pointer 2879 * @adev: amdgpu device pointer
2826 * @job: which job trigger hang 2880 * @reset_flags: output param tells caller the reset result
2827 * 2881 *
2828 * Attempt the reset the GPU if it has hung (all asics). 2882 * attempt to do soft-reset or full-reset and reinitialize Asic
2829 * for SRIOV case. 2883 * return 0 means successed otherwise failed
2830 * Returns 0 for success or an error on failure. 2884*/
2831 */ 2885static int amdgpu_reset(struct amdgpu_device *adev, uint64_t* reset_flags)
2832int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
2833{ 2886{
2834 int i, j, r = 0; 2887 bool need_full_reset, vram_lost = 0;
2835 int resched; 2888 int r;
2836 struct amdgpu_bo *bo, *tmp;
2837 struct amdgpu_ring *ring;
2838 struct dma_fence *fence = NULL, *next = NULL;
2839 2889
2840 mutex_lock(&adev->virt.lock_reset); 2890 need_full_reset = amdgpu_need_full_reset(adev);
2841 atomic_inc(&adev->gpu_reset_counter);
2842 adev->in_sriov_reset = true;
2843 2891
2844 /* block TTM */ 2892 if (!need_full_reset) {
2845 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); 2893 amdgpu_pre_soft_reset(adev);
2894 r = amdgpu_soft_reset(adev);
2895 amdgpu_post_soft_reset(adev);
2896 if (r || amdgpu_check_soft_reset(adev)) {
2897 DRM_INFO("soft reset failed, will fallback to full reset!\n");
2898 need_full_reset = true;
2899 }
2846 2900
2847 /* we start from the ring trigger GPU hang */ 2901 }
2848 j = job ? job->ring->idx : 0;
2849 2902
2850 /* block scheduler */ 2903 if (need_full_reset) {
2851 for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) { 2904 r = amdgpu_suspend(adev);
2852 ring = adev->rings[i % AMDGPU_MAX_RINGS];
2853 if (!ring || !ring->sched.thread)
2854 continue;
2855 2905
2856 kthread_park(ring->sched.thread); 2906retry:
2907 amdgpu_atombios_scratch_regs_save(adev);
2908 r = amdgpu_asic_reset(adev);
2909 amdgpu_atombios_scratch_regs_restore(adev);
2910 /* post card */
2911 amdgpu_atom_asic_init(adev->mode_info.atom_context);
2857 2912
2858 if (job && j != i) 2913 if (!r) {
2859 continue; 2914 dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
2915 r = amdgpu_resume_phase1(adev);
2916 if (r)
2917 goto out;
2860 2918
2861 /* here give the last chance to check if job removed from mirror-list 2919 vram_lost = amdgpu_check_vram_lost(adev);
2862 * since we already pay some time on kthread_park */ 2920 if (vram_lost) {
2863 if (job && list_empty(&job->base.node)) { 2921 DRM_ERROR("VRAM is lost!\n");
2864 kthread_unpark(ring->sched.thread); 2922 atomic_inc(&adev->vram_lost_counter);
2865 goto give_up_reset; 2923 }
2924
2925 r = amdgpu_gtt_mgr_recover(
2926 &adev->mman.bdev.man[TTM_PL_TT]);
2927 if (r)
2928 goto out;
2929
2930 r = amdgpu_resume_phase2(adev);
2931 if (r)
2932 goto out;
2933
2934 if (vram_lost)
2935 amdgpu_fill_reset_magic(adev);
2866 } 2936 }
2937 }
2867 2938
2868 if (amd_sched_invalidate_job(&job->base, amdgpu_job_hang_limit)) 2939out:
2869 amd_sched_job_kickout(&job->base); 2940 if (!r) {
2941 amdgpu_irq_gpu_reset_resume_helper(adev);
2942 r = amdgpu_ib_ring_tests(adev);
2943 if (r) {
2944 dev_err(adev->dev, "ib ring test failed (%d).\n", r);
2945 r = amdgpu_suspend(adev);
2946 need_full_reset = true;
2947 goto retry;
2948 }
2949 }
2870 2950
2871 /* only do job_reset on the hang ring if @job not NULL */ 2951 if (reset_flags) {
2872 amd_sched_hw_job_reset(&ring->sched); 2952 if (vram_lost)
2953 (*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST;
2873 2954
2874 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ 2955 if (need_full_reset)
2875 amdgpu_fence_driver_force_completion_ring(ring); 2956 (*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
2876 } 2957 }
2877 2958
2878 /* request to take full control of GPU before re-initialization */ 2959 return r;
2879 if (job) 2960}
2880 amdgpu_virt_reset_gpu(adev);
2881 else
2882 amdgpu_virt_request_full_gpu(adev, true);
2883 2961
2962/*
2963 * amdgpu_reset_sriov - reset ASIC for SR-IOV vf
2964 *
2965 * @adev: amdgpu device pointer
2966 * @reset_flags: output param tells caller the reset result
2967 *
2968 * do VF FLR and reinitialize Asic
2969 * return 0 means successed otherwise failed
2970*/
2971static int amdgpu_reset_sriov(struct amdgpu_device *adev, uint64_t *reset_flags, bool from_hypervisor)
2972{
2973 int r;
2974
2975 if (from_hypervisor)
2976 r = amdgpu_virt_request_full_gpu(adev, true);
2977 else
2978 r = amdgpu_virt_reset_gpu(adev);
2979 if (r)
2980 return r;
2884 2981
2885 /* Resume IP prior to SMC */ 2982 /* Resume IP prior to SMC */
2886 amdgpu_sriov_reinit_early(adev); 2983 r = amdgpu_sriov_reinit_early(adev);
2984 if (r)
2985 goto error;
2887 2986
2888 /* we need recover gart prior to run SMC/CP/SDMA resume */ 2987 /* we need recover gart prior to run SMC/CP/SDMA resume */
2889 amdgpu_ttm_recover_gart(adev); 2988 amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
2890 2989
2891 /* now we are okay to resume SMC/CP/SDMA */ 2990 /* now we are okay to resume SMC/CP/SDMA */
2892 amdgpu_sriov_reinit_late(adev); 2991 r = amdgpu_sriov_reinit_late(adev);
2992 if (r)
2993 goto error;
2893 2994
2894 amdgpu_irq_gpu_reset_resume_helper(adev); 2995 amdgpu_irq_gpu_reset_resume_helper(adev);
2895 2996 r = amdgpu_ib_ring_tests(adev);
2896 if (amdgpu_ib_ring_tests(adev)) 2997 if (r)
2897 dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r); 2998 dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
2898 2999
3000error:
2899 /* release full control of GPU after ib test */ 3001 /* release full control of GPU after ib test */
2900 amdgpu_virt_release_full_gpu(adev, true); 3002 amdgpu_virt_release_full_gpu(adev, true);
2901 3003
2902 DRM_INFO("recover vram bo from shadow\n"); 3004 if (reset_flags) {
2903 3005 if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
2904 ring = adev->mman.buffer_funcs_ring; 3006 (*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST;
2905 mutex_lock(&adev->shadow_list_lock); 3007 atomic_inc(&adev->vram_lost_counter);
2906 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2907 next = NULL;
2908 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2909 if (fence) {
2910 r = dma_fence_wait(fence, false);
2911 if (r) {
2912 WARN(r, "recovery from shadow isn't completed\n");
2913 break;
2914 }
2915 }
2916
2917 dma_fence_put(fence);
2918 fence = next;
2919 }
2920 mutex_unlock(&adev->shadow_list_lock);
2921
2922 if (fence) {
2923 r = dma_fence_wait(fence, false);
2924 if (r)
2925 WARN(r, "recovery from shadow isn't completed\n");
2926 }
2927 dma_fence_put(fence);
2928
2929 for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
2930 ring = adev->rings[i % AMDGPU_MAX_RINGS];
2931 if (!ring || !ring->sched.thread)
2932 continue;
2933
2934 if (job && j != i) {
2935 kthread_unpark(ring->sched.thread);
2936 continue;
2937 } 3008 }
2938 3009
2939 amd_sched_job_recovery(&ring->sched); 3010 /* VF FLR or hotlink reset is always full-reset */
2940 kthread_unpark(ring->sched.thread); 3011 (*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
2941 }
2942
2943 drm_helper_resume_force_mode(adev->ddev);
2944give_up_reset:
2945 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2946 if (r) {
2947 /* bad news, how to tell it to userspace ? */
2948 dev_info(adev->dev, "GPU reset failed\n");
2949 } else {
2950 dev_info(adev->dev, "GPU reset successed!\n");
2951 } 3012 }
2952 3013
2953 adev->in_sriov_reset = false;
2954 mutex_unlock(&adev->virt.lock_reset);
2955 return r; 3014 return r;
2956} 3015}
2957 3016
2958/** 3017/**
2959 * amdgpu_gpu_reset - reset the asic 3018 * amdgpu_gpu_recover - reset the asic and recover scheduler
2960 * 3019 *
2961 * @adev: amdgpu device pointer 3020 * @adev: amdgpu device pointer
3021 * @job: which job trigger hang
2962 * 3022 *
2963 * Attempt the reset the GPU if it has hung (all asics). 3023 * Attempt to reset the GPU if it has hung (all asics).
2964 * Returns 0 for success or an error on failure. 3024 * Returns 0 for success or an error on failure.
2965 */ 3025 */
2966int amdgpu_gpu_reset(struct amdgpu_device *adev) 3026int amdgpu_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job *job)
2967{ 3027{
2968 struct drm_atomic_state *state = NULL; 3028 struct drm_atomic_state *state = NULL;
2969 int i, r; 3029 uint64_t reset_flags = 0;
2970 int resched; 3030 int i, r, resched;
2971 bool need_full_reset, vram_lost = false;
2972 3031
2973 if (!amdgpu_check_soft_reset(adev)) { 3032 if (!amdgpu_check_soft_reset(adev)) {
2974 DRM_INFO("No hardware hang detected. Did some blocks stall?\n"); 3033 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
2975 return 0; 3034 return 0;
2976 } 3035 }
2977 3036
3037 dev_info(adev->dev, "GPU reset begin!\n");
3038
3039 mutex_lock(&adev->lock_reset);
2978 atomic_inc(&adev->gpu_reset_counter); 3040 atomic_inc(&adev->gpu_reset_counter);
3041 adev->in_gpu_reset = 1;
2979 3042
2980 /* block TTM */ 3043 /* block TTM */
2981 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); 3044 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
@@ -2989,69 +3052,26 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
2989 3052
2990 if (!ring || !ring->sched.thread) 3053 if (!ring || !ring->sched.thread)
2991 continue; 3054 continue;
2992 kthread_park(ring->sched.thread);
2993 amd_sched_hw_job_reset(&ring->sched);
2994 }
2995 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2996 amdgpu_fence_driver_force_completion(adev);
2997 3055
2998 need_full_reset = amdgpu_need_full_reset(adev); 3056 /* only focus on the ring hit timeout if &job not NULL */
3057 if (job && job->ring->idx != i)
3058 continue;
2999 3059
3000 if (!need_full_reset) { 3060 kthread_park(ring->sched.thread);
3001 amdgpu_pre_soft_reset(adev); 3061 amd_sched_hw_job_reset(&ring->sched, &job->base);
3002 r = amdgpu_soft_reset(adev);
3003 amdgpu_post_soft_reset(adev);
3004 if (r || amdgpu_check_soft_reset(adev)) {
3005 DRM_INFO("soft reset failed, will fallback to full reset!\n");
3006 need_full_reset = true;
3007 }
3008 }
3009 3062
3010 if (need_full_reset) { 3063 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
3011 r = amdgpu_suspend(adev); 3064 amdgpu_fence_driver_force_completion(ring);
3065 }
3012 3066
3013retry: 3067 if (amdgpu_sriov_vf(adev))
3014 amdgpu_atombios_scratch_regs_save(adev); 3068 r = amdgpu_reset_sriov(adev, &reset_flags, job ? false : true);
3015 r = amdgpu_asic_reset(adev); 3069 else
3016 amdgpu_atombios_scratch_regs_restore(adev); 3070 r = amdgpu_reset(adev, &reset_flags);
3017 /* post card */
3018 amdgpu_atom_asic_init(adev->mode_info.atom_context);
3019 3071
3020 if (!r) {
3021 dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
3022 r = amdgpu_resume_phase1(adev);
3023 if (r)
3024 goto out;
3025 vram_lost = amdgpu_check_vram_lost(adev);
3026 if (vram_lost) {
3027 DRM_ERROR("VRAM is lost!\n");
3028 atomic_inc(&adev->vram_lost_counter);
3029 }
3030 r = amdgpu_ttm_recover_gart(adev);
3031 if (r)
3032 goto out;
3033 r = amdgpu_resume_phase2(adev);
3034 if (r)
3035 goto out;
3036 if (vram_lost)
3037 amdgpu_fill_reset_magic(adev);
3038 }
3039 }
3040out:
3041 if (!r) { 3072 if (!r) {
3042 amdgpu_irq_gpu_reset_resume_helper(adev); 3073 if (((reset_flags & AMDGPU_RESET_INFO_FULLRESET) && !(adev->flags & AMD_IS_APU)) ||
3043 r = amdgpu_ib_ring_tests(adev); 3074 (reset_flags & AMDGPU_RESET_INFO_VRAM_LOST)) {
3044 if (r) {
3045 dev_err(adev->dev, "ib ring test failed (%d).\n", r);
3046 r = amdgpu_suspend(adev);
3047 need_full_reset = true;
3048 goto retry;
3049 }
3050 /**
3051 * recovery vm page tables, since we cannot depend on VRAM is
3052 * consistent after gpu full reset.
3053 */
3054 if (need_full_reset && amdgpu_need_backup(adev)) {
3055 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; 3075 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
3056 struct amdgpu_bo *bo, *tmp; 3076 struct amdgpu_bo *bo, *tmp;
3057 struct dma_fence *fence = NULL, *next = NULL; 3077 struct dma_fence *fence = NULL, *next = NULL;
@@ -3080,40 +3100,56 @@ out:
3080 } 3100 }
3081 dma_fence_put(fence); 3101 dma_fence_put(fence);
3082 } 3102 }
3103
3083 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 3104 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3084 struct amdgpu_ring *ring = adev->rings[i]; 3105 struct amdgpu_ring *ring = adev->rings[i];
3085 3106
3086 if (!ring || !ring->sched.thread) 3107 if (!ring || !ring->sched.thread)
3087 continue; 3108 continue;
3088 3109
3110 /* only focus on the ring hit timeout if &job not NULL */
3111 if (job && job->ring->idx != i)
3112 continue;
3113
3089 amd_sched_job_recovery(&ring->sched); 3114 amd_sched_job_recovery(&ring->sched);
3090 kthread_unpark(ring->sched.thread); 3115 kthread_unpark(ring->sched.thread);
3091 } 3116 }
3092 } else { 3117 } else {
3093 dev_err(adev->dev, "asic resume failed (%d).\n", r);
3094 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 3118 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3095 if (adev->rings[i] && adev->rings[i]->sched.thread) { 3119 struct amdgpu_ring *ring = adev->rings[i];
3096 kthread_unpark(adev->rings[i]->sched.thread); 3120
3097 } 3121 if (!ring || !ring->sched.thread)
3122 continue;
3123
3124 /* only focus on the ring hit timeout if &job not NULL */
3125 if (job && job->ring->idx != i)
3126 continue;
3127
3128 kthread_unpark(adev->rings[i]->sched.thread);
3098 } 3129 }
3099 } 3130 }
3100 3131
3101 if (amdgpu_device_has_dc_support(adev)) { 3132 if (amdgpu_device_has_dc_support(adev)) {
3102 r = drm_atomic_helper_resume(adev->ddev, state); 3133 if (drm_atomic_helper_resume(adev->ddev, state))
3134 dev_info(adev->dev, "drm resume failed:%d\n", r);
3103 amdgpu_dm_display_resume(adev); 3135 amdgpu_dm_display_resume(adev);
3104 } else 3136 } else {
3105 drm_helper_resume_force_mode(adev->ddev); 3137 drm_helper_resume_force_mode(adev->ddev);
3138 }
3106 3139
3107 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched); 3140 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
3141
3108 if (r) { 3142 if (r) {
3109 /* bad news, how to tell it to userspace ? */ 3143 /* bad news, how to tell it to userspace ? */
3110 dev_info(adev->dev, "GPU reset failed\n"); 3144 dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
3111 } 3145 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
3112 else { 3146 } else {
3113 dev_info(adev->dev, "GPU reset successed!\n"); 3147 dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
3114 } 3148 }
3115 3149
3116 amdgpu_vf_error_trans_all(adev); 3150 amdgpu_vf_error_trans_all(adev);
3151 adev->in_gpu_reset = 0;
3152 mutex_unlock(&adev->lock_reset);
3117 return r; 3153 return r;
3118} 3154}
3119 3155
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index 138beb550a58..38d47559f098 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -34,6 +34,7 @@
34#include <linux/pm_runtime.h> 34#include <linux/pm_runtime.h>
35#include <drm/drm_crtc_helper.h> 35#include <drm/drm_crtc_helper.h>
36#include <drm/drm_edid.h> 36#include <drm/drm_edid.h>
37#include <drm/drm_fb_helper.h>
37 38
38static void amdgpu_flip_callback(struct dma_fence *f, struct dma_fence_cb *cb) 39static void amdgpu_flip_callback(struct dma_fence *f, struct dma_fence_cb *cb)
39{ 40{
@@ -556,15 +557,9 @@ amdgpu_user_framebuffer_create(struct drm_device *dev,
556 return &amdgpu_fb->base; 557 return &amdgpu_fb->base;
557} 558}
558 559
559void amdgpu_output_poll_changed(struct drm_device *dev)
560{
561 struct amdgpu_device *adev = dev->dev_private;
562 amdgpu_fb_output_poll_changed(adev);
563}
564
565const struct drm_mode_config_funcs amdgpu_mode_funcs = { 560const struct drm_mode_config_funcs amdgpu_mode_funcs = {
566 .fb_create = amdgpu_user_framebuffer_create, 561 .fb_create = amdgpu_user_framebuffer_create,
567 .output_poll_changed = amdgpu_output_poll_changed 562 .output_poll_changed = drm_fb_helper_output_poll_changed,
568}; 563};
569 564
570static const struct drm_prop_enum_list amdgpu_underscan_enum_list[] = 565static const struct drm_prop_enum_list amdgpu_underscan_enum_list[] =
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h
index 3cc0ef0c055e..0bcb6c6e0ca9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h
@@ -25,9 +25,7 @@
25 25
26struct drm_framebuffer * 26struct drm_framebuffer *
27amdgpu_user_framebuffer_create(struct drm_device *dev, 27amdgpu_user_framebuffer_create(struct drm_device *dev,
28 struct drm_file *file_priv, 28 struct drm_file *file_priv,
29 const struct drm_mode_fb_cmd2 *mode_cmd); 29 const struct drm_mode_fb_cmd2 *mode_cmd);
30
31void amdgpu_output_poll_changed(struct drm_device *dev);
32 30
33#endif 31#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index 56caaeee6fea..a8437a3296a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -360,6 +360,12 @@ enum amdgpu_pcie_gen {
360 ((adev)->powerplay.pp_funcs->set_clockgating_by_smu(\ 360 ((adev)->powerplay.pp_funcs->set_clockgating_by_smu(\
361 (adev)->powerplay.pp_handle, msg_id)) 361 (adev)->powerplay.pp_handle, msg_id))
362 362
363#define amdgpu_dpm_notify_smu_memory_info(adev, virtual_addr_low, \
364 virtual_addr_hi, mc_addr_low, mc_addr_hi, size) \
365 ((adev)->powerplay.pp_funcs->notify_smu_memory_info)( \
366 (adev)->powerplay.pp_handle, virtual_addr_low, \
367 virtual_addr_hi, mc_addr_low, mc_addr_hi, size)
368
363struct amdgpu_dpm { 369struct amdgpu_dpm {
364 struct amdgpu_ps *ps; 370 struct amdgpu_ps *ps;
365 /* number of valid power states */ 371 /* number of valid power states */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index c2f414ffb2cc..31383e004947 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -216,7 +216,7 @@ module_param_named(exp_hw_support, amdgpu_exp_hw_support, int, 0444);
216MODULE_PARM_DESC(dc, "Display Core driver (1 = enable, 0 = disable, -1 = auto (default))"); 216MODULE_PARM_DESC(dc, "Display Core driver (1 = enable, 0 = disable, -1 = auto (default))");
217module_param_named(dc, amdgpu_dc, int, 0444); 217module_param_named(dc, amdgpu_dc, int, 0444);
218 218
219MODULE_PARM_DESC(dc, "Display Core Log Level (0 = minimal (default), 1 = chatty"); 219MODULE_PARM_DESC(dc_log, "Display Core Log Level (0 = minimal (default), 1 = chatty");
220module_param_named(dc_log, amdgpu_dc_log, int, 0444); 220module_param_named(dc_log, amdgpu_dc_log, int, 0444);
221 221
222MODULE_PARM_DESC(sched_jobs, "the max number of jobs supported in the sw queue (default 32)"); 222MODULE_PARM_DESC(sched_jobs, "the max number of jobs supported in the sw queue (default 32)");
@@ -306,7 +306,6 @@ MODULE_PARM_DESC(cik_support, "CIK support (1 = enabled (default), 0 = disabled)
306module_param_named(cik_support, amdgpu_cik_support, int, 0444); 306module_param_named(cik_support, amdgpu_cik_support, int, 0444);
307#endif 307#endif
308 308
309
310static const struct pci_device_id pciidlist[] = { 309static const struct pci_device_id pciidlist[] = {
311#ifdef CONFIG_DRM_AMDGPU_SI 310#ifdef CONFIG_DRM_AMDGPU_SI
312 {0x1002, 0x6780, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI}, 311 {0x1002, 0x6780, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
@@ -566,12 +565,13 @@ static int amdgpu_kick_out_firmware_fb(struct pci_dev *pdev)
566 return 0; 565 return 0;
567} 566}
568 567
568
569static int amdgpu_pci_probe(struct pci_dev *pdev, 569static int amdgpu_pci_probe(struct pci_dev *pdev,
570 const struct pci_device_id *ent) 570 const struct pci_device_id *ent)
571{ 571{
572 struct drm_device *dev; 572 struct drm_device *dev;
573 unsigned long flags = ent->driver_data; 573 unsigned long flags = ent->driver_data;
574 int ret; 574 int ret, retry = 0;
575 575
576 if ((flags & AMD_EXP_HW_SUPPORT) && !amdgpu_exp_hw_support) { 576 if ((flags & AMD_EXP_HW_SUPPORT) && !amdgpu_exp_hw_support) {
577 DRM_INFO("This hardware requires experimental hardware support.\n" 577 DRM_INFO("This hardware requires experimental hardware support.\n"
@@ -604,8 +604,14 @@ static int amdgpu_pci_probe(struct pci_dev *pdev,
604 604
605 pci_set_drvdata(pdev, dev); 605 pci_set_drvdata(pdev, dev);
606 606
607retry_init:
607 ret = drm_dev_register(dev, ent->driver_data); 608 ret = drm_dev_register(dev, ent->driver_data);
608 if (ret) 609 if (ret == -EAGAIN && ++retry <= 3) {
610 DRM_INFO("retry init %d\n", retry);
611 /* Don't request EX mode too frequently which is attacking */
612 msleep(5000);
613 goto retry_init;
614 } else if (ret)
609 goto err_pci; 615 goto err_pci;
610 616
611 return 0; 617 return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index 90fa8e8bc6fb..ff3e9beb7d19 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -283,12 +283,6 @@ out:
283 return ret; 283 return ret;
284} 284}
285 285
286void amdgpu_fb_output_poll_changed(struct amdgpu_device *adev)
287{
288 if (adev->mode_info.rfbdev)
289 drm_fb_helper_hotplug_event(&adev->mode_info.rfbdev->helper);
290}
291
292static int amdgpu_fbdev_destroy(struct drm_device *dev, struct amdgpu_fbdev *rfbdev) 286static int amdgpu_fbdev_destroy(struct drm_device *dev, struct amdgpu_fbdev *rfbdev)
293{ 287{
294 struct amdgpu_framebuffer *rfb = &rfbdev->rfb; 288 struct amdgpu_framebuffer *rfb = &rfbdev->rfb;
@@ -393,24 +387,3 @@ bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj)
393 return true; 387 return true;
394 return false; 388 return false;
395} 389}
396
397void amdgpu_fbdev_restore_mode(struct amdgpu_device *adev)
398{
399 struct amdgpu_fbdev *afbdev;
400 struct drm_fb_helper *fb_helper;
401 int ret;
402
403 if (!adev)
404 return;
405
406 afbdev = adev->mode_info.rfbdev;
407
408 if (!afbdev)
409 return;
410
411 fb_helper = &afbdev->helper;
412
413 ret = drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper);
414 if (ret)
415 DRM_DEBUG("failed to restore crtc mode\n");
416}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 2fa95aef74d5..604ac03a42e4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -391,9 +391,9 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
391 ring->fence_drv.irq_type = irq_type; 391 ring->fence_drv.irq_type = irq_type;
392 ring->fence_drv.initialized = true; 392 ring->fence_drv.initialized = true;
393 393
394 dev_info(adev->dev, "fence driver on ring %d use gpu addr 0x%016llx, " 394 dev_dbg(adev->dev, "fence driver on ring %d use gpu addr 0x%016llx, "
395 "cpu addr 0x%p\n", ring->idx, 395 "cpu addr 0x%p\n", ring->idx,
396 ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr); 396 ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr);
397 return 0; 397 return 0;
398} 398}
399 399
@@ -446,7 +446,7 @@ int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
446 timeout = MAX_SCHEDULE_TIMEOUT; 446 timeout = MAX_SCHEDULE_TIMEOUT;
447 } 447 }
448 r = amd_sched_init(&ring->sched, &amdgpu_sched_ops, 448 r = amd_sched_init(&ring->sched, &amdgpu_sched_ops,
449 num_hw_submission, 449 num_hw_submission, amdgpu_job_hang_limit,
450 timeout, ring->name); 450 timeout, ring->name);
451 if (r) { 451 if (r) {
452 DRM_ERROR("Failed to create scheduler on ring %s.\n", 452 DRM_ERROR("Failed to create scheduler on ring %s.\n",
@@ -499,7 +499,7 @@ void amdgpu_fence_driver_fini(struct amdgpu_device *adev)
499 r = amdgpu_fence_wait_empty(ring); 499 r = amdgpu_fence_wait_empty(ring);
500 if (r) { 500 if (r) {
501 /* no need to trigger GPU reset as we are unloading */ 501 /* no need to trigger GPU reset as we are unloading */
502 amdgpu_fence_driver_force_completion(adev); 502 amdgpu_fence_driver_force_completion(ring);
503 } 503 }
504 amdgpu_irq_put(adev, ring->fence_drv.irq_src, 504 amdgpu_irq_put(adev, ring->fence_drv.irq_src,
505 ring->fence_drv.irq_type); 505 ring->fence_drv.irq_type);
@@ -534,7 +534,7 @@ void amdgpu_fence_driver_suspend(struct amdgpu_device *adev)
534 r = amdgpu_fence_wait_empty(ring); 534 r = amdgpu_fence_wait_empty(ring);
535 if (r) { 535 if (r) {
536 /* delay GPU reset to resume */ 536 /* delay GPU reset to resume */
537 amdgpu_fence_driver_force_completion(adev); 537 amdgpu_fence_driver_force_completion(ring);
538 } 538 }
539 539
540 /* disable the interrupt */ 540 /* disable the interrupt */
@@ -571,30 +571,15 @@ void amdgpu_fence_driver_resume(struct amdgpu_device *adev)
571} 571}
572 572
573/** 573/**
574 * amdgpu_fence_driver_force_completion - force all fence waiter to complete 574 * amdgpu_fence_driver_force_completion - force signal latest fence of ring
575 * 575 *
576 * @adev: amdgpu device pointer 576 * @ring: fence of the ring to signal
577 * 577 *
578 * In case of GPU reset failure make sure no process keep waiting on fence
579 * that will never complete.
580 */ 578 */
581void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev) 579void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)
582{ 580{
583 int i; 581 amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
584 582 amdgpu_fence_process(ring);
585 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
586 struct amdgpu_ring *ring = adev->rings[i];
587 if (!ring || !ring->fence_drv.initialized)
588 continue;
589
590 amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
591 }
592}
593
594void amdgpu_fence_driver_force_completion_ring(struct amdgpu_ring *ring)
595{
596 if (ring)
597 amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
598} 583}
599 584
600/* 585/*
@@ -709,25 +694,25 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data)
709} 694}
710 695
711/** 696/**
712 * amdgpu_debugfs_gpu_reset - manually trigger a gpu reset 697 * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
713 * 698 *
714 * Manually trigger a gpu reset at the next fence wait. 699 * Manually trigger a gpu reset at the next fence wait.
715 */ 700 */
716static int amdgpu_debugfs_gpu_reset(struct seq_file *m, void *data) 701static int amdgpu_debugfs_gpu_recover(struct seq_file *m, void *data)
717{ 702{
718 struct drm_info_node *node = (struct drm_info_node *) m->private; 703 struct drm_info_node *node = (struct drm_info_node *) m->private;
719 struct drm_device *dev = node->minor->dev; 704 struct drm_device *dev = node->minor->dev;
720 struct amdgpu_device *adev = dev->dev_private; 705 struct amdgpu_device *adev = dev->dev_private;
721 706
722 seq_printf(m, "gpu reset\n"); 707 seq_printf(m, "gpu recover\n");
723 amdgpu_gpu_reset(adev); 708 amdgpu_gpu_recover(adev, NULL);
724 709
725 return 0; 710 return 0;
726} 711}
727 712
728static const struct drm_info_list amdgpu_debugfs_fence_list[] = { 713static const struct drm_info_list amdgpu_debugfs_fence_list[] = {
729 {"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL}, 714 {"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL},
730 {"amdgpu_gpu_reset", &amdgpu_debugfs_gpu_reset, 0, NULL} 715 {"amdgpu_gpu_recover", &amdgpu_debugfs_gpu_recover, 0, NULL}
731}; 716};
732 717
733static const struct drm_info_list amdgpu_debugfs_fence_list_sriov[] = { 718static const struct drm_info_list amdgpu_debugfs_fence_list_sriov[] = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index fe818501c520..1f51897acc5b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -57,63 +57,6 @@
57 */ 57 */
58 58
59/** 59/**
60 * amdgpu_gart_table_ram_alloc - allocate system ram for gart page table
61 *
62 * @adev: amdgpu_device pointer
63 *
64 * Allocate system memory for GART page table
65 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the
66 * gart table to be in system memory.
67 * Returns 0 for success, -ENOMEM for failure.
68 */
69int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev)
70{
71 void *ptr;
72
73 ptr = pci_alloc_consistent(adev->pdev, adev->gart.table_size,
74 &adev->gart.table_addr);
75 if (ptr == NULL) {
76 return -ENOMEM;
77 }
78#ifdef CONFIG_X86
79 if (0) {
80 set_memory_uc((unsigned long)ptr,
81 adev->gart.table_size >> PAGE_SHIFT);
82 }
83#endif
84 adev->gart.ptr = ptr;
85 memset((void *)adev->gart.ptr, 0, adev->gart.table_size);
86 return 0;
87}
88
89/**
90 * amdgpu_gart_table_ram_free - free system ram for gart page table
91 *
92 * @adev: amdgpu_device pointer
93 *
94 * Free system memory for GART page table
95 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the
96 * gart table to be in system memory.
97 */
98void amdgpu_gart_table_ram_free(struct amdgpu_device *adev)
99{
100 if (adev->gart.ptr == NULL) {
101 return;
102 }
103#ifdef CONFIG_X86
104 if (0) {
105 set_memory_wb((unsigned long)adev->gart.ptr,
106 adev->gart.table_size >> PAGE_SHIFT);
107 }
108#endif
109 pci_free_consistent(adev->pdev, adev->gart.table_size,
110 (void *)adev->gart.ptr,
111 adev->gart.table_addr);
112 adev->gart.ptr = NULL;
113 adev->gart.table_addr = 0;
114}
115
116/**
117 * amdgpu_gart_table_vram_alloc - allocate vram for gart page table 60 * amdgpu_gart_table_vram_alloc - allocate vram for gart page table
118 * 61 *
119 * @adev: amdgpu_device pointer 62 * @adev: amdgpu_device pointer
@@ -377,10 +320,8 @@ int amdgpu_gart_init(struct amdgpu_device *adev)
377#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS 320#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
378 /* Allocate pages table */ 321 /* Allocate pages table */
379 adev->gart.pages = vzalloc(sizeof(void *) * adev->gart.num_cpu_pages); 322 adev->gart.pages = vzalloc(sizeof(void *) * adev->gart.num_cpu_pages);
380 if (adev->gart.pages == NULL) { 323 if (adev->gart.pages == NULL)
381 amdgpu_gart_fini(adev);
382 return -ENOMEM; 324 return -ENOMEM;
383 }
384#endif 325#endif
385 326
386 return 0; 327 return 0;
@@ -395,11 +336,6 @@ int amdgpu_gart_init(struct amdgpu_device *adev)
395 */ 336 */
396void amdgpu_gart_fini(struct amdgpu_device *adev) 337void amdgpu_gart_fini(struct amdgpu_device *adev)
397{ 338{
398 if (adev->gart.ready) {
399 /* unbind pages */
400 amdgpu_gart_unbind(adev, 0, adev->gart.num_cpu_pages);
401 }
402 adev->gart.ready = false;
403#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS 339#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
404 vfree(adev->gart.pages); 340 vfree(adev->gart.pages);
405 adev->gart.pages = NULL; 341 adev->gart.pages = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
index afbe803b1a13..d4a43302c2be 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
@@ -39,7 +39,7 @@ struct amdgpu_gart_funcs;
39#define AMDGPU_GPU_PAGE_ALIGN(a) (((a) + AMDGPU_GPU_PAGE_MASK) & ~AMDGPU_GPU_PAGE_MASK) 39#define AMDGPU_GPU_PAGE_ALIGN(a) (((a) + AMDGPU_GPU_PAGE_MASK) & ~AMDGPU_GPU_PAGE_MASK)
40 40
41struct amdgpu_gart { 41struct amdgpu_gart {
42 dma_addr_t table_addr; 42 u64 table_addr;
43 struct amdgpu_bo *robj; 43 struct amdgpu_bo *robj;
44 void *ptr; 44 void *ptr;
45 unsigned num_gpu_pages; 45 unsigned num_gpu_pages;
@@ -56,8 +56,6 @@ struct amdgpu_gart {
56 const struct amdgpu_gart_funcs *gart_funcs; 56 const struct amdgpu_gart_funcs *gart_funcs;
57}; 57};
58 58
59int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev);
60void amdgpu_gart_table_ram_free(struct amdgpu_device *adev);
61int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev); 59int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev);
62void amdgpu_gart_table_vram_free(struct amdgpu_device *adev); 60void amdgpu_gart_table_vram_free(struct amdgpu_device *adev);
63int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev); 61int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index e87eedcc0da9..eb75eb44efc6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -72,7 +72,7 @@ retry:
72 initial_domain |= AMDGPU_GEM_DOMAIN_GTT; 72 initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
73 goto retry; 73 goto retry;
74 } 74 }
75 DRM_ERROR("Failed to allocate GEM object (%ld, %d, %u, %d)\n", 75 DRM_DEBUG("Failed to allocate GEM object (%ld, %d, %u, %d)\n",
76 size, initial_domain, alignment, r); 76 size, initial_domain, alignment, r);
77 } 77 }
78 return r; 78 return r;
@@ -282,6 +282,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
282int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, 282int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
283 struct drm_file *filp) 283 struct drm_file *filp)
284{ 284{
285 struct ttm_operation_ctx ctx = { true, false };
285 struct amdgpu_device *adev = dev->dev_private; 286 struct amdgpu_device *adev = dev->dev_private;
286 struct drm_amdgpu_gem_userptr *args = data; 287 struct drm_amdgpu_gem_userptr *args = data;
287 struct drm_gem_object *gobj; 288 struct drm_gem_object *gobj;
@@ -335,7 +336,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
335 goto free_pages; 336 goto free_pages;
336 337
337 amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 338 amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
338 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 339 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
339 amdgpu_bo_unreserve(bo); 340 amdgpu_bo_unreserve(bo);
340 if (r) 341 if (r)
341 goto free_pages; 342 goto free_pages;
@@ -557,14 +558,25 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
557 int r = 0; 558 int r = 0;
558 559
559 if (args->va_address < AMDGPU_VA_RESERVED_SIZE) { 560 if (args->va_address < AMDGPU_VA_RESERVED_SIZE) {
560 dev_err(&dev->pdev->dev, 561 dev_dbg(&dev->pdev->dev,
561 "va_address 0x%LX is in reserved area 0x%LX\n", 562 "va_address 0x%LX is in reserved area 0x%LX\n",
562 args->va_address, AMDGPU_VA_RESERVED_SIZE); 563 args->va_address, AMDGPU_VA_RESERVED_SIZE);
563 return -EINVAL; 564 return -EINVAL;
564 } 565 }
565 566
567 if (args->va_address >= AMDGPU_VA_HOLE_START &&
568 args->va_address < AMDGPU_VA_HOLE_END) {
569 dev_dbg(&dev->pdev->dev,
570 "va_address 0x%LX is in VA hole 0x%LX-0x%LX\n",
571 args->va_address, AMDGPU_VA_HOLE_START,
572 AMDGPU_VA_HOLE_END);
573 return -EINVAL;
574 }
575
576 args->va_address &= AMDGPU_VA_HOLE_MASK;
577
566 if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) { 578 if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
567 dev_err(&dev->pdev->dev, "invalid flags combination 0x%08X\n", 579 dev_dbg(&dev->pdev->dev, "invalid flags combination 0x%08X\n",
568 args->flags); 580 args->flags);
569 return -EINVAL; 581 return -EINVAL;
570 } 582 }
@@ -576,7 +588,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
576 case AMDGPU_VA_OP_REPLACE: 588 case AMDGPU_VA_OP_REPLACE:
577 break; 589 break;
578 default: 590 default:
579 dev_err(&dev->pdev->dev, "unsupported operation %d\n", 591 dev_dbg(&dev->pdev->dev, "unsupported operation %d\n",
580 args->operation); 592 args->operation);
581 return -EINVAL; 593 return -EINVAL;
582 } 594 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index 00e0ce10862f..e14ab34d8262 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -31,6 +31,11 @@ struct amdgpu_gtt_mgr {
31 atomic64_t available; 31 atomic64_t available;
32}; 32};
33 33
34struct amdgpu_gtt_node {
35 struct drm_mm_node node;
36 struct ttm_buffer_object *tbo;
37};
38
34/** 39/**
35 * amdgpu_gtt_mgr_init - init GTT manager and DRM MM 40 * amdgpu_gtt_mgr_init - init GTT manager and DRM MM
36 * 41 *
@@ -79,17 +84,17 @@ static int amdgpu_gtt_mgr_fini(struct ttm_mem_type_manager *man)
79} 84}
80 85
81/** 86/**
82 * amdgpu_gtt_mgr_is_allocated - Check if mem has address space 87 * amdgpu_gtt_mgr_has_gart_addr - Check if mem has address space
83 * 88 *
84 * @mem: the mem object to check 89 * @mem: the mem object to check
85 * 90 *
86 * Check if a mem object has already address space allocated. 91 * Check if a mem object has already address space allocated.
87 */ 92 */
88bool amdgpu_gtt_mgr_is_allocated(struct ttm_mem_reg *mem) 93bool amdgpu_gtt_mgr_has_gart_addr(struct ttm_mem_reg *mem)
89{ 94{
90 struct drm_mm_node *node = mem->mm_node; 95 struct amdgpu_gtt_node *node = mem->mm_node;
91 96
92 return (node->start != AMDGPU_BO_INVALID_OFFSET); 97 return (node->node.start != AMDGPU_BO_INVALID_OFFSET);
93} 98}
94 99
95/** 100/**
@@ -109,12 +114,12 @@ static int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
109{ 114{
110 struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); 115 struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
111 struct amdgpu_gtt_mgr *mgr = man->priv; 116 struct amdgpu_gtt_mgr *mgr = man->priv;
112 struct drm_mm_node *node = mem->mm_node; 117 struct amdgpu_gtt_node *node = mem->mm_node;
113 enum drm_mm_insert_mode mode; 118 enum drm_mm_insert_mode mode;
114 unsigned long fpfn, lpfn; 119 unsigned long fpfn, lpfn;
115 int r; 120 int r;
116 121
117 if (amdgpu_gtt_mgr_is_allocated(mem)) 122 if (amdgpu_gtt_mgr_has_gart_addr(mem))
118 return 0; 123 return 0;
119 124
120 if (place) 125 if (place)
@@ -132,13 +137,13 @@ static int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
132 mode = DRM_MM_INSERT_HIGH; 137 mode = DRM_MM_INSERT_HIGH;
133 138
134 spin_lock(&mgr->lock); 139 spin_lock(&mgr->lock);
135 r = drm_mm_insert_node_in_range(&mgr->mm, node, 140 r = drm_mm_insert_node_in_range(&mgr->mm, &node->node, mem->num_pages,
136 mem->num_pages, mem->page_alignment, 0, 141 mem->page_alignment, 0, fpfn, lpfn,
137 fpfn, lpfn, mode); 142 mode);
138 spin_unlock(&mgr->lock); 143 spin_unlock(&mgr->lock);
139 144
140 if (!r) 145 if (!r)
141 mem->start = node->start; 146 mem->start = node->node.start;
142 147
143 return r; 148 return r;
144} 149}
@@ -159,7 +164,7 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
159 struct ttm_mem_reg *mem) 164 struct ttm_mem_reg *mem)
160{ 165{
161 struct amdgpu_gtt_mgr *mgr = man->priv; 166 struct amdgpu_gtt_mgr *mgr = man->priv;
162 struct drm_mm_node *node; 167 struct amdgpu_gtt_node *node;
163 int r; 168 int r;
164 169
165 spin_lock(&mgr->lock); 170 spin_lock(&mgr->lock);
@@ -177,8 +182,9 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
177 goto err_out; 182 goto err_out;
178 } 183 }
179 184
180 node->start = AMDGPU_BO_INVALID_OFFSET; 185 node->node.start = AMDGPU_BO_INVALID_OFFSET;
181 node->size = mem->num_pages; 186 node->node.size = mem->num_pages;
187 node->tbo = tbo;
182 mem->mm_node = node; 188 mem->mm_node = node;
183 189
184 if (place->fpfn || place->lpfn || place->flags & TTM_PL_FLAG_TOPDOWN) { 190 if (place->fpfn || place->lpfn || place->flags & TTM_PL_FLAG_TOPDOWN) {
@@ -190,7 +196,7 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
190 goto err_out; 196 goto err_out;
191 } 197 }
192 } else { 198 } else {
193 mem->start = node->start; 199 mem->start = node->node.start;
194 } 200 }
195 201
196 return 0; 202 return 0;
@@ -214,14 +220,14 @@ static void amdgpu_gtt_mgr_del(struct ttm_mem_type_manager *man,
214 struct ttm_mem_reg *mem) 220 struct ttm_mem_reg *mem)
215{ 221{
216 struct amdgpu_gtt_mgr *mgr = man->priv; 222 struct amdgpu_gtt_mgr *mgr = man->priv;
217 struct drm_mm_node *node = mem->mm_node; 223 struct amdgpu_gtt_node *node = mem->mm_node;
218 224
219 if (!node) 225 if (!node)
220 return; 226 return;
221 227
222 spin_lock(&mgr->lock); 228 spin_lock(&mgr->lock);
223 if (node->start != AMDGPU_BO_INVALID_OFFSET) 229 if (node->node.start != AMDGPU_BO_INVALID_OFFSET)
224 drm_mm_remove_node(node); 230 drm_mm_remove_node(&node->node);
225 spin_unlock(&mgr->lock); 231 spin_unlock(&mgr->lock);
226 atomic64_add(mem->num_pages, &mgr->available); 232 atomic64_add(mem->num_pages, &mgr->available);
227 233
@@ -244,6 +250,25 @@ uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man)
244 return (result > 0 ? result : 0) * PAGE_SIZE; 250 return (result > 0 ? result : 0) * PAGE_SIZE;
245} 251}
246 252
253int amdgpu_gtt_mgr_recover(struct ttm_mem_type_manager *man)
254{
255 struct amdgpu_gtt_mgr *mgr = man->priv;
256 struct amdgpu_gtt_node *node;
257 struct drm_mm_node *mm_node;
258 int r = 0;
259
260 spin_lock(&mgr->lock);
261 drm_mm_for_each_node(mm_node, &mgr->mm) {
262 node = container_of(mm_node, struct amdgpu_gtt_node, node);
263 r = amdgpu_ttm_recover_gart(node->tbo);
264 if (r)
265 break;
266 }
267 spin_unlock(&mgr->lock);
268
269 return r;
270}
271
247/** 272/**
248 * amdgpu_gtt_mgr_debug - dump VRAM table 273 * amdgpu_gtt_mgr_debug - dump VRAM table
249 * 274 *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 659997bfff30..0cf86eb357d6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -164,7 +164,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
164 } 164 }
165 165
166 if (ring->funcs->emit_pipeline_sync && job && 166 if (ring->funcs->emit_pipeline_sync && job &&
167 ((tmp = amdgpu_sync_get_fence(&job->sched_sync)) || 167 ((tmp = amdgpu_sync_get_fence(&job->sched_sync, NULL)) ||
168 amdgpu_vm_need_pipeline_sync(ring, job))) { 168 amdgpu_vm_need_pipeline_sync(ring, job))) {
169 need_pipe_sync = true; 169 need_pipe_sync = true;
170 dma_fence_put(tmp); 170 dma_fence_put(tmp);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 47c5ce9807db..c340774082ea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -88,7 +88,7 @@ static void amdgpu_irq_reset_work_func(struct work_struct *work)
88 reset_work); 88 reset_work);
89 89
90 if (!amdgpu_sriov_vf(adev)) 90 if (!amdgpu_sriov_vf(adev))
91 amdgpu_gpu_reset(adev); 91 amdgpu_gpu_recover(adev, NULL);
92} 92}
93 93
94/* Disable *all* interrupts */ 94/* Disable *all* interrupts */
@@ -232,7 +232,7 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
232 int ret = pci_enable_msi(adev->pdev); 232 int ret = pci_enable_msi(adev->pdev);
233 if (!ret) { 233 if (!ret) {
234 adev->irq.msi_enabled = true; 234 adev->irq.msi_enabled = true;
235 dev_info(adev->dev, "amdgpu: using MSI.\n"); 235 dev_dbg(adev->dev, "amdgpu: using MSI.\n");
236 } 236 }
237 } 237 }
238 238
@@ -262,7 +262,7 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
262 return r; 262 return r;
263 } 263 }
264 264
265 DRM_INFO("amdgpu: irq initialized.\n"); 265 DRM_DEBUG("amdgpu: irq initialized.\n");
266 return 0; 266 return 0;
267} 267}
268 268
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 0cfc68db575b..bdc210ac74f8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -37,10 +37,7 @@ static void amdgpu_job_timedout(struct amd_sched_job *s_job)
37 atomic_read(&job->ring->fence_drv.last_seq), 37 atomic_read(&job->ring->fence_drv.last_seq),
38 job->ring->fence_drv.sync_seq); 38 job->ring->fence_drv.sync_seq);
39 39
40 if (amdgpu_sriov_vf(job->adev)) 40 amdgpu_gpu_recover(job->adev, job);
41 amdgpu_sriov_gpu_reset(job->adev, job);
42 else
43 amdgpu_gpu_reset(job->adev);
44} 41}
45 42
46int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, 43int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
@@ -63,7 +60,6 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
63 (*job)->num_ibs = num_ibs; 60 (*job)->num_ibs = num_ibs;
64 61
65 amdgpu_sync_create(&(*job)->sync); 62 amdgpu_sync_create(&(*job)->sync);
66 amdgpu_sync_create(&(*job)->dep_sync);
67 amdgpu_sync_create(&(*job)->sched_sync); 63 amdgpu_sync_create(&(*job)->sched_sync);
68 (*job)->vram_lost_counter = atomic_read(&adev->vram_lost_counter); 64 (*job)->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
69 65
@@ -104,10 +100,9 @@ static void amdgpu_job_free_cb(struct amd_sched_job *s_job)
104{ 100{
105 struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base); 101 struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base);
106 102
107 amdgpu_ring_priority_put(job->ring, amd_sched_get_job_priority(s_job)); 103 amdgpu_ring_priority_put(job->ring, s_job->s_priority);
108 dma_fence_put(job->fence); 104 dma_fence_put(job->fence);
109 amdgpu_sync_free(&job->sync); 105 amdgpu_sync_free(&job->sync);
110 amdgpu_sync_free(&job->dep_sync);
111 amdgpu_sync_free(&job->sched_sync); 106 amdgpu_sync_free(&job->sched_sync);
112 kfree(job); 107 kfree(job);
113} 108}
@@ -118,7 +113,6 @@ void amdgpu_job_free(struct amdgpu_job *job)
118 113
119 dma_fence_put(job->fence); 114 dma_fence_put(job->fence);
120 amdgpu_sync_free(&job->sync); 115 amdgpu_sync_free(&job->sync);
121 amdgpu_sync_free(&job->dep_sync);
122 amdgpu_sync_free(&job->sched_sync); 116 amdgpu_sync_free(&job->sched_sync);
123 kfree(job); 117 kfree(job);
124} 118}
@@ -141,28 +135,29 @@ int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
141 job->fence_ctx = entity->fence_context; 135 job->fence_ctx = entity->fence_context;
142 *f = dma_fence_get(&job->base.s_fence->finished); 136 *f = dma_fence_get(&job->base.s_fence->finished);
143 amdgpu_job_free_resources(job); 137 amdgpu_job_free_resources(job);
144 amdgpu_ring_priority_get(job->ring, 138 amdgpu_ring_priority_get(job->ring, job->base.s_priority);
145 amd_sched_get_job_priority(&job->base)); 139 amd_sched_entity_push_job(&job->base, entity);
146 amd_sched_entity_push_job(&job->base);
147 140
148 return 0; 141 return 0;
149} 142}
150 143
151static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job) 144static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job,
145 struct amd_sched_entity *s_entity)
152{ 146{
153 struct amdgpu_job *job = to_amdgpu_job(sched_job); 147 struct amdgpu_job *job = to_amdgpu_job(sched_job);
154 struct amdgpu_vm *vm = job->vm; 148 struct amdgpu_vm *vm = job->vm;
155 149 bool explicit = false;
156 struct dma_fence *fence = amdgpu_sync_get_fence(&job->dep_sync);
157 int r; 150 int r;
158 151 struct dma_fence *fence = amdgpu_sync_get_fence(&job->sync, &explicit);
159 if (amd_sched_dependency_optimized(fence, sched_job->s_entity)) { 152
160 r = amdgpu_sync_fence(job->adev, &job->sched_sync, fence); 153 if (fence && explicit) {
161 if (r) 154 if (amd_sched_dependency_optimized(fence, s_entity)) {
162 DRM_ERROR("Error adding fence to sync (%d)\n", r); 155 r = amdgpu_sync_fence(job->adev, &job->sched_sync, fence, false);
156 if (r)
157 DRM_ERROR("Error adding fence to sync (%d)\n", r);
158 }
163 } 159 }
164 if (!fence) 160
165 fence = amdgpu_sync_get_fence(&job->sync);
166 while (fence == NULL && vm && !job->vm_id) { 161 while (fence == NULL && vm && !job->vm_id) {
167 struct amdgpu_ring *ring = job->ring; 162 struct amdgpu_ring *ring = job->ring;
168 163
@@ -172,7 +167,7 @@ static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
172 if (r) 167 if (r)
173 DRM_ERROR("Error getting VM ID (%d)\n", r); 168 DRM_ERROR("Error getting VM ID (%d)\n", r);
174 169
175 fence = amdgpu_sync_get_fence(&job->sync); 170 fence = amdgpu_sync_get_fence(&job->sync, NULL);
176 } 171 }
177 172
178 return fence; 173 return fence;
@@ -180,7 +175,7 @@ static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
180 175
181static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job) 176static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
182{ 177{
183 struct dma_fence *fence = NULL; 178 struct dma_fence *fence = NULL, *finished;
184 struct amdgpu_device *adev; 179 struct amdgpu_device *adev;
185 struct amdgpu_job *job; 180 struct amdgpu_job *job;
186 int r; 181 int r;
@@ -190,15 +185,18 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
190 return NULL; 185 return NULL;
191 } 186 }
192 job = to_amdgpu_job(sched_job); 187 job = to_amdgpu_job(sched_job);
188 finished = &job->base.s_fence->finished;
193 adev = job->adev; 189 adev = job->adev;
194 190
195 BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL)); 191 BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
196 192
197 trace_amdgpu_sched_run_job(job); 193 trace_amdgpu_sched_run_job(job);
198 /* skip ib schedule when vram is lost */ 194
199 if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) { 195 if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter))
200 dma_fence_set_error(&job->base.s_fence->finished, -ECANCELED); 196 dma_fence_set_error(finished, -ECANCELED);/* skip IB as well if VRAM lost */
201 DRM_ERROR("Skip scheduling IBs!\n"); 197
198 if (finished->error < 0) {
199 DRM_INFO("Skip scheduling IBs!\n");
202 } else { 200 } else {
203 r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job, 201 r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job,
204 &fence); 202 &fence);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 720139e182a3..bd6e9a40f421 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -63,8 +63,6 @@ void amdgpu_driver_unload_kms(struct drm_device *dev)
63 pm_runtime_forbid(dev->dev); 63 pm_runtime_forbid(dev->dev);
64 } 64 }
65 65
66 amdgpu_amdkfd_device_fini(adev);
67
68 amdgpu_acpi_fini(adev); 66 amdgpu_acpi_fini(adev);
69 67
70 amdgpu_device_fini(adev); 68 amdgpu_device_fini(adev);
@@ -159,9 +157,6 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
159 "Error during ACPI methods call\n"); 157 "Error during ACPI methods call\n");
160 } 158 }
161 159
162 amdgpu_amdkfd_device_probe(adev);
163 amdgpu_amdkfd_device_init(adev);
164
165 if (amdgpu_device_is_px(dev)) { 160 if (amdgpu_device_is_px(dev)) {
166 pm_runtime_use_autosuspend(dev->dev); 161 pm_runtime_use_autosuspend(dev->dev);
167 pm_runtime_set_autosuspend_delay(dev->dev, 5000); 162 pm_runtime_set_autosuspend_delay(dev->dev, 5000);
@@ -171,9 +166,6 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
171 pm_runtime_put_autosuspend(dev->dev); 166 pm_runtime_put_autosuspend(dev->dev);
172 } 167 }
173 168
174 if (amdgpu_sriov_vf(adev))
175 amdgpu_virt_release_full_gpu(adev, true);
176
177out: 169out:
178 if (r) { 170 if (r) {
179 /* balance pm_runtime_get_sync in amdgpu_driver_unload_kms */ 171 /* balance pm_runtime_get_sync in amdgpu_driver_unload_kms */
@@ -558,6 +550,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
558 } 550 }
559 case AMDGPU_INFO_DEV_INFO: { 551 case AMDGPU_INFO_DEV_INFO: {
560 struct drm_amdgpu_info_device dev_info = {}; 552 struct drm_amdgpu_info_device dev_info = {};
553 uint64_t vm_size;
561 554
562 dev_info.device_id = dev->pdev->device; 555 dev_info.device_id = dev->pdev->device;
563 dev_info.chip_rev = adev->rev_id; 556 dev_info.chip_rev = adev->rev_id;
@@ -585,8 +578,17 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
585 dev_info.ids_flags |= AMDGPU_IDS_FLAGS_FUSION; 578 dev_info.ids_flags |= AMDGPU_IDS_FLAGS_FUSION;
586 if (amdgpu_sriov_vf(adev)) 579 if (amdgpu_sriov_vf(adev))
587 dev_info.ids_flags |= AMDGPU_IDS_FLAGS_PREEMPTION; 580 dev_info.ids_flags |= AMDGPU_IDS_FLAGS_PREEMPTION;
581
582 vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
588 dev_info.virtual_address_offset = AMDGPU_VA_RESERVED_SIZE; 583 dev_info.virtual_address_offset = AMDGPU_VA_RESERVED_SIZE;
589 dev_info.virtual_address_max = (uint64_t)adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE; 584 dev_info.virtual_address_max =
585 min(vm_size, AMDGPU_VA_HOLE_START);
586
587 vm_size -= AMDGPU_VA_RESERVED_SIZE;
588 if (vm_size > AMDGPU_VA_HOLE_START) {
589 dev_info.high_va_offset = AMDGPU_VA_HOLE_END;
590 dev_info.high_va_max = AMDGPU_VA_HOLE_END | vm_size;
591 }
590 dev_info.virtual_address_alignment = max((int)PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE); 592 dev_info.virtual_address_alignment = max((int)PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE);
591 dev_info.pte_fragment_size = (1 << adev->vm_manager.fragment_size) * AMDGPU_GPU_PAGE_SIZE; 593 dev_info.pte_fragment_size = (1 << adev->vm_manager.fragment_size) * AMDGPU_GPU_PAGE_SIZE;
592 dev_info.gart_page_size = AMDGPU_GPU_PAGE_SIZE; 594 dev_info.gart_page_size = AMDGPU_GPU_PAGE_SIZE;
@@ -786,9 +788,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
786 */ 788 */
787void amdgpu_driver_lastclose_kms(struct drm_device *dev) 789void amdgpu_driver_lastclose_kms(struct drm_device *dev)
788{ 790{
789 struct amdgpu_device *adev = dev->dev_private; 791 drm_fb_helper_lastclose(dev);
790
791 amdgpu_fbdev_restore_mode(adev);
792 vga_switcheroo_process_delayed_switch(); 792 vga_switcheroo_process_delayed_switch();
793} 793}
794 794
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index ffde1e9666e8..54f06c959340 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -89,7 +89,6 @@ enum amdgpu_hpd_id {
89 AMDGPU_HPD_4, 89 AMDGPU_HPD_4,
90 AMDGPU_HPD_5, 90 AMDGPU_HPD_5,
91 AMDGPU_HPD_6, 91 AMDGPU_HPD_6,
92 AMDGPU_HPD_LAST,
93 AMDGPU_HPD_NONE = 0xff, 92 AMDGPU_HPD_NONE = 0xff,
94}; 93};
95 94
@@ -106,7 +105,6 @@ enum amdgpu_crtc_irq {
106 AMDGPU_CRTC_IRQ_VLINE4, 105 AMDGPU_CRTC_IRQ_VLINE4,
107 AMDGPU_CRTC_IRQ_VLINE5, 106 AMDGPU_CRTC_IRQ_VLINE5,
108 AMDGPU_CRTC_IRQ_VLINE6, 107 AMDGPU_CRTC_IRQ_VLINE6,
109 AMDGPU_CRTC_IRQ_LAST,
110 AMDGPU_CRTC_IRQ_NONE = 0xff 108 AMDGPU_CRTC_IRQ_NONE = 0xff
111}; 109};
112 110
@@ -117,7 +115,6 @@ enum amdgpu_pageflip_irq {
117 AMDGPU_PAGEFLIP_IRQ_D4, 115 AMDGPU_PAGEFLIP_IRQ_D4,
118 AMDGPU_PAGEFLIP_IRQ_D5, 116 AMDGPU_PAGEFLIP_IRQ_D5,
119 AMDGPU_PAGEFLIP_IRQ_D6, 117 AMDGPU_PAGEFLIP_IRQ_D6,
120 AMDGPU_PAGEFLIP_IRQ_LAST,
121 AMDGPU_PAGEFLIP_IRQ_NONE = 0xff 118 AMDGPU_PAGEFLIP_IRQ_NONE = 0xff
122}; 119};
123 120
@@ -661,10 +658,6 @@ void amdgpu_fbdev_fini(struct amdgpu_device *adev);
661void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state); 658void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state);
662int amdgpu_fbdev_total_size(struct amdgpu_device *adev); 659int amdgpu_fbdev_total_size(struct amdgpu_device *adev);
663bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj); 660bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj);
664void amdgpu_fbdev_restore_mode(struct amdgpu_device *adev);
665
666void amdgpu_fb_output_poll_changed(struct amdgpu_device *adev);
667
668 661
669int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int bpp, bool tiled); 662int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int bpp, bool tiled);
670 663
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index ea25164e7f4b..dc0a8be98043 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -281,6 +281,44 @@ void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
281 *cpu_addr = NULL; 281 *cpu_addr = NULL;
282} 282}
283 283
284/* Validate bo size is bit bigger then the request domain */
285static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
286 unsigned long size, u32 domain)
287{
288 struct ttm_mem_type_manager *man = NULL;
289
290 /*
291 * If GTT is part of requested domains the check must succeed to
292 * allow fall back to GTT
293 */
294 if (domain & AMDGPU_GEM_DOMAIN_GTT) {
295 man = &adev->mman.bdev.man[TTM_PL_TT];
296
297 if (size < (man->size << PAGE_SHIFT))
298 return true;
299 else
300 goto fail;
301 }
302
303 if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
304 man = &adev->mman.bdev.man[TTM_PL_VRAM];
305
306 if (size < (man->size << PAGE_SHIFT))
307 return true;
308 else
309 goto fail;
310 }
311
312
313 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */
314 return true;
315
316fail:
317 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size,
318 man->size << PAGE_SHIFT);
319 return false;
320}
321
284static int amdgpu_bo_do_create(struct amdgpu_device *adev, 322static int amdgpu_bo_do_create(struct amdgpu_device *adev,
285 unsigned long size, int byte_align, 323 unsigned long size, int byte_align,
286 bool kernel, u32 domain, u64 flags, 324 bool kernel, u32 domain, u64 flags,
@@ -289,16 +327,19 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
289 uint64_t init_value, 327 uint64_t init_value,
290 struct amdgpu_bo **bo_ptr) 328 struct amdgpu_bo **bo_ptr)
291{ 329{
330 struct ttm_operation_ctx ctx = { !kernel, false };
292 struct amdgpu_bo *bo; 331 struct amdgpu_bo *bo;
293 enum ttm_bo_type type; 332 enum ttm_bo_type type;
294 unsigned long page_align; 333 unsigned long page_align;
295 u64 initial_bytes_moved, bytes_moved;
296 size_t acc_size; 334 size_t acc_size;
297 int r; 335 int r;
298 336
299 page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT; 337 page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
300 size = ALIGN(size, PAGE_SIZE); 338 size = ALIGN(size, PAGE_SIZE);
301 339
340 if (!amdgpu_bo_validate_size(adev, size, domain))
341 return -ENOMEM;
342
302 if (kernel) { 343 if (kernel) {
303 type = ttm_bo_type_kernel; 344 type = ttm_bo_type_kernel;
304 } else if (sg) { 345 } else if (sg) {
@@ -364,22 +405,19 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
364 bo->tbo.bdev = &adev->mman.bdev; 405 bo->tbo.bdev = &adev->mman.bdev;
365 amdgpu_ttm_placement_from_domain(bo, domain); 406 amdgpu_ttm_placement_from_domain(bo, domain);
366 407
367 initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
368 /* Kernel allocation are uninterruptible */
369 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type, 408 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
370 &bo->placement, page_align, !kernel, NULL, 409 &bo->placement, page_align, &ctx, NULL,
371 acc_size, sg, resv, &amdgpu_ttm_bo_destroy); 410 acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
372 if (unlikely(r != 0)) 411 if (unlikely(r != 0))
373 return r; 412 return r;
374 413
375 bytes_moved = atomic64_read(&adev->num_bytes_moved) -
376 initial_bytes_moved;
377 if (adev->mc.visible_vram_size < adev->mc.real_vram_size && 414 if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
378 bo->tbo.mem.mem_type == TTM_PL_VRAM && 415 bo->tbo.mem.mem_type == TTM_PL_VRAM &&
379 bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT) 416 bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)
380 amdgpu_cs_report_moved_bytes(adev, bytes_moved, bytes_moved); 417 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
418 ctx.bytes_moved);
381 else 419 else
382 amdgpu_cs_report_moved_bytes(adev, bytes_moved, 0); 420 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
383 421
384 if (kernel) 422 if (kernel)
385 bo->tbo.priority = 1; 423 bo->tbo.priority = 1;
@@ -511,6 +549,7 @@ err:
511 549
512int amdgpu_bo_validate(struct amdgpu_bo *bo) 550int amdgpu_bo_validate(struct amdgpu_bo *bo)
513{ 551{
552 struct ttm_operation_ctx ctx = { false, false };
514 uint32_t domain; 553 uint32_t domain;
515 int r; 554 int r;
516 555
@@ -521,7 +560,7 @@ int amdgpu_bo_validate(struct amdgpu_bo *bo)
521 560
522retry: 561retry:
523 amdgpu_ttm_placement_from_domain(bo, domain); 562 amdgpu_ttm_placement_from_domain(bo, domain);
524 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); 563 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
525 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { 564 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
526 domain = bo->allowed_domains; 565 domain = bo->allowed_domains;
527 goto retry; 566 goto retry;
@@ -632,6 +671,7 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
632 u64 *gpu_addr) 671 u64 *gpu_addr)
633{ 672{
634 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 673 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
674 struct ttm_operation_ctx ctx = { false, false };
635 int r, i; 675 int r, i;
636 676
637 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) 677 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
@@ -647,7 +687,7 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
647 if (bo->pin_count) { 687 if (bo->pin_count) {
648 uint32_t mem_type = bo->tbo.mem.mem_type; 688 uint32_t mem_type = bo->tbo.mem.mem_type;
649 689
650 if (domain != amdgpu_mem_type_to_domain(mem_type)) 690 if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
651 return -EINVAL; 691 return -EINVAL;
652 692
653 bo->pin_count++; 693 bo->pin_count++;
@@ -682,21 +722,23 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
682 bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT; 722 bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
683 } 723 }
684 724
685 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); 725 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
686 if (unlikely(r)) { 726 if (unlikely(r)) {
687 dev_err(adev->dev, "%p pin failed\n", bo); 727 dev_err(adev->dev, "%p pin failed\n", bo);
688 goto error; 728 goto error;
689 } 729 }
690 730
731 r = amdgpu_ttm_alloc_gart(&bo->tbo);
732 if (unlikely(r)) {
733 dev_err(adev->dev, "%p bind failed\n", bo);
734 goto error;
735 }
736
691 bo->pin_count = 1; 737 bo->pin_count = 1;
692 if (gpu_addr != NULL) { 738 if (gpu_addr != NULL)
693 r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
694 if (unlikely(r)) {
695 dev_err(adev->dev, "%p bind failed\n", bo);
696 goto error;
697 }
698 *gpu_addr = amdgpu_bo_gpu_offset(bo); 739 *gpu_addr = amdgpu_bo_gpu_offset(bo);
699 } 740
741 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
700 if (domain == AMDGPU_GEM_DOMAIN_VRAM) { 742 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
701 adev->vram_pin_size += amdgpu_bo_size(bo); 743 adev->vram_pin_size += amdgpu_bo_size(bo);
702 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 744 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
@@ -717,6 +759,7 @@ int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
717int amdgpu_bo_unpin(struct amdgpu_bo *bo) 759int amdgpu_bo_unpin(struct amdgpu_bo *bo)
718{ 760{
719 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 761 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
762 struct ttm_operation_ctx ctx = { false, false };
720 int r, i; 763 int r, i;
721 764
722 if (!bo->pin_count) { 765 if (!bo->pin_count) {
@@ -730,7 +773,7 @@ int amdgpu_bo_unpin(struct amdgpu_bo *bo)
730 bo->placements[i].lpfn = 0; 773 bo->placements[i].lpfn = 0;
731 bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT; 774 bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
732 } 775 }
733 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); 776 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
734 if (unlikely(r)) { 777 if (unlikely(r)) {
735 dev_err(adev->dev, "%p validate failed for unpin\n", bo); 778 dev_err(adev->dev, "%p validate failed for unpin\n", bo);
736 goto error; 779 goto error;
@@ -779,8 +822,8 @@ int amdgpu_bo_init(struct amdgpu_device *adev)
779 adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base, 822 adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base,
780 adev->mc.aper_size); 823 adev->mc.aper_size);
781 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", 824 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
782 adev->mc.mc_vram_size >> 20, 825 adev->mc.mc_vram_size >> 20,
783 (unsigned long long)adev->mc.aper_size >> 20); 826 (unsigned long long)adev->mc.aper_size >> 20);
784 DRM_INFO("RAM width %dbits %s\n", 827 DRM_INFO("RAM width %dbits %s\n",
785 adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]); 828 adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]);
786 return amdgpu_ttm_init(adev); 829 return amdgpu_ttm_init(adev);
@@ -902,6 +945,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
902int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 945int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
903{ 946{
904 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 947 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
948 struct ttm_operation_ctx ctx = { false, false };
905 struct amdgpu_bo *abo; 949 struct amdgpu_bo *abo;
906 unsigned long offset, size; 950 unsigned long offset, size;
907 int r; 951 int r;
@@ -935,7 +979,7 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
935 abo->placement.num_busy_placement = 1; 979 abo->placement.num_busy_placement = 1;
936 abo->placement.busy_placement = &abo->placements[1]; 980 abo->placement.busy_placement = &abo->placements[1];
937 981
938 r = ttm_bo_validate(bo, &abo->placement, false, false); 982 r = ttm_bo_validate(bo, &abo->placement, &ctx);
939 if (unlikely(r != 0)) 983 if (unlikely(r != 0))
940 return r; 984 return r;
941 985
@@ -980,7 +1024,7 @@ u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
980{ 1024{
981 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM); 1025 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
982 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT && 1026 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT &&
983 !amdgpu_ttm_is_bound(bo->tbo.ttm)); 1027 !amdgpu_gtt_mgr_has_gart_addr(&bo->tbo.mem));
984 WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) && 1028 WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
985 !bo->pin_count); 1029 !bo->pin_count);
986 WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET); 1030 WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 428aae048f4b..33615e2ea2e6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -187,7 +187,7 @@ static inline u64 amdgpu_bo_mmap_offset(struct amdgpu_bo *bo)
187static inline bool amdgpu_bo_gpu_accessible(struct amdgpu_bo *bo) 187static inline bool amdgpu_bo_gpu_accessible(struct amdgpu_bo *bo)
188{ 188{
189 switch (bo->tbo.mem.mem_type) { 189 switch (bo->tbo.mem.mem_type) {
190 case TTM_PL_TT: return amdgpu_ttm_is_bound(bo->tbo.ttm); 190 case TTM_PL_TT: return amdgpu_gtt_mgr_has_gart_addr(&bo->tbo.mem);
191 case TTM_PL_VRAM: return true; 191 case TTM_PL_VRAM: return true;
192 default: return false; 192 default: return false;
193 } 193 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index 6c570d4e4516..6f56ff606e43 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -30,7 +30,6 @@
30#include <linux/hwmon.h> 30#include <linux/hwmon.h>
31#include <linux/hwmon-sysfs.h> 31#include <linux/hwmon-sysfs.h>
32 32
33#include "amd_powerplay.h"
34 33
35static int amdgpu_debugfs_pm_init(struct amdgpu_device *adev); 34static int amdgpu_debugfs_pm_init(struct amdgpu_device *adev);
36 35
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 447d446b5015..2157d4509e84 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -264,7 +264,7 @@ static int psp_hw_start(struct psp_context *psp)
264 struct amdgpu_device *adev = psp->adev; 264 struct amdgpu_device *adev = psp->adev;
265 int ret; 265 int ret;
266 266
267 if (!amdgpu_sriov_vf(adev) || !adev->in_sriov_reset) { 267 if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
268 ret = psp_bootloader_load_sysdrv(psp); 268 ret = psp_bootloader_load_sysdrv(psp);
269 if (ret) 269 if (ret)
270 return ret; 270 return ret;
@@ -334,23 +334,26 @@ static int psp_load_fw(struct amdgpu_device *adev)
334 int ret; 334 int ret;
335 struct psp_context *psp = &adev->psp; 335 struct psp_context *psp = &adev->psp;
336 336
337 if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset != 0)
338 goto skip_memalloc;
339
337 psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 340 psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
338 if (!psp->cmd) 341 if (!psp->cmd)
339 return -ENOMEM; 342 return -ENOMEM;
340 343
341 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG, 344 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
342 AMDGPU_GEM_DOMAIN_GTT, 345 AMDGPU_GEM_DOMAIN_GTT,
343 &psp->fw_pri_bo, 346 &psp->fw_pri_bo,
344 &psp->fw_pri_mc_addr, 347 &psp->fw_pri_mc_addr,
345 &psp->fw_pri_buf); 348 &psp->fw_pri_buf);
346 if (ret) 349 if (ret)
347 goto failed; 350 goto failed;
348 351
349 ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE, 352 ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
350 AMDGPU_GEM_DOMAIN_VRAM, 353 AMDGPU_GEM_DOMAIN_VRAM,
351 &psp->fence_buf_bo, 354 &psp->fence_buf_bo,
352 &psp->fence_buf_mc_addr, 355 &psp->fence_buf_mc_addr,
353 &psp->fence_buf); 356 &psp->fence_buf);
354 if (ret) 357 if (ret)
355 goto failed_mem2; 358 goto failed_mem2;
356 359
@@ -375,6 +378,7 @@ static int psp_load_fw(struct amdgpu_device *adev)
375 if (ret) 378 if (ret)
376 goto failed_mem; 379 goto failed_mem;
377 380
381skip_memalloc:
378 ret = psp_hw_start(psp); 382 ret = psp_hw_start(psp);
379 if (ret) 383 if (ret)
380 goto failed_mem; 384 goto failed_mem;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
index 93d86619e802..262c1267249e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
@@ -225,7 +225,7 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
225 225
226 /* Right now all IPs have only one instance - multiple rings. */ 226 /* Right now all IPs have only one instance - multiple rings. */
227 if (instance != 0) { 227 if (instance != 0) {
228 DRM_ERROR("invalid ip instance: %d\n", instance); 228 DRM_DEBUG("invalid ip instance: %d\n", instance);
229 return -EINVAL; 229 return -EINVAL;
230 } 230 }
231 231
@@ -255,13 +255,13 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
255 ip_num_rings = adev->vcn.num_enc_rings; 255 ip_num_rings = adev->vcn.num_enc_rings;
256 break; 256 break;
257 default: 257 default:
258 DRM_ERROR("unknown ip type: %d\n", hw_ip); 258 DRM_DEBUG("unknown ip type: %d\n", hw_ip);
259 return -EINVAL; 259 return -EINVAL;
260 } 260 }
261 261
262 if (ring >= ip_num_rings) { 262 if (ring >= ip_num_rings) {
263 DRM_ERROR("Ring index:%d exceeds maximum:%d for ip:%d\n", 263 DRM_DEBUG("Ring index:%d exceeds maximum:%d for ip:%d\n",
264 ring, ip_num_rings, hw_ip); 264 ring, ip_num_rings, hw_ip);
265 return -EINVAL; 265 return -EINVAL;
266 } 266 }
267 267
@@ -292,7 +292,7 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
292 default: 292 default:
293 *out_ring = NULL; 293 *out_ring = NULL;
294 r = -EINVAL; 294 r = -EINVAL;
295 DRM_ERROR("unknown HW IP type: %d\n", mapper->hw_ip); 295 DRM_DEBUG("unknown HW IP type: %d\n", mapper->hw_ip);
296 } 296 }
297 297
298out_unlock: 298out_unlock:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index b18c2b96691f..a6b89e3932a5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -79,8 +79,7 @@ struct amdgpu_fence_driver {
79 79
80int amdgpu_fence_driver_init(struct amdgpu_device *adev); 80int amdgpu_fence_driver_init(struct amdgpu_device *adev);
81void amdgpu_fence_driver_fini(struct amdgpu_device *adev); 81void amdgpu_fence_driver_fini(struct amdgpu_device *adev);
82void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev); 82void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring);
83void amdgpu_fence_driver_force_completion_ring(struct amdgpu_ring *ring);
84 83
85int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring, 84int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
86 unsigned num_hw_submission); 85 unsigned num_hw_submission);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index a4bf21f8f1c1..ebe1ffbab0c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -35,6 +35,7 @@
35struct amdgpu_sync_entry { 35struct amdgpu_sync_entry {
36 struct hlist_node node; 36 struct hlist_node node;
37 struct dma_fence *fence; 37 struct dma_fence *fence;
38 bool explicit;
38}; 39};
39 40
40static struct kmem_cache *amdgpu_sync_slab; 41static struct kmem_cache *amdgpu_sync_slab;
@@ -141,7 +142,7 @@ static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f)
141 * 142 *
142 */ 143 */
143int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, 144int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
144 struct dma_fence *f) 145 struct dma_fence *f, bool explicit)
145{ 146{
146 struct amdgpu_sync_entry *e; 147 struct amdgpu_sync_entry *e;
147 148
@@ -159,6 +160,8 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
159 if (!e) 160 if (!e)
160 return -ENOMEM; 161 return -ENOMEM;
161 162
163 e->explicit = explicit;
164
162 hash_add(sync->fences, &e->node, f->context); 165 hash_add(sync->fences, &e->node, f->context);
163 e->fence = dma_fence_get(f); 166 e->fence = dma_fence_get(f);
164 return 0; 167 return 0;
@@ -189,10 +192,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
189 192
190 /* always sync to the exclusive fence */ 193 /* always sync to the exclusive fence */
191 f = reservation_object_get_excl(resv); 194 f = reservation_object_get_excl(resv);
192 r = amdgpu_sync_fence(adev, sync, f); 195 r = amdgpu_sync_fence(adev, sync, f, false);
193
194 if (explicit_sync)
195 return r;
196 196
197 flist = reservation_object_get_list(resv); 197 flist = reservation_object_get_list(resv);
198 if (!flist || r) 198 if (!flist || r)
@@ -212,15 +212,15 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
212 (fence_owner == AMDGPU_FENCE_OWNER_VM))) 212 (fence_owner == AMDGPU_FENCE_OWNER_VM)))
213 continue; 213 continue;
214 214
215 /* Ignore fence from the same owner as 215 /* Ignore fence from the same owner and explicit one as
216 * long as it isn't undefined. 216 * long as it isn't undefined.
217 */ 217 */
218 if (owner != AMDGPU_FENCE_OWNER_UNDEFINED && 218 if (owner != AMDGPU_FENCE_OWNER_UNDEFINED &&
219 fence_owner == owner) 219 (fence_owner == owner || explicit_sync))
220 continue; 220 continue;
221 } 221 }
222 222
223 r = amdgpu_sync_fence(adev, sync, f); 223 r = amdgpu_sync_fence(adev, sync, f, false);
224 if (r) 224 if (r)
225 break; 225 break;
226 } 226 }
@@ -275,19 +275,21 @@ struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
275 * amdgpu_sync_get_fence - get the next fence from the sync object 275 * amdgpu_sync_get_fence - get the next fence from the sync object
276 * 276 *
277 * @sync: sync object to use 277 * @sync: sync object to use
278 * @explicit: true if the next fence is explicit
278 * 279 *
279 * Get and removes the next fence from the sync object not signaled yet. 280 * Get and removes the next fence from the sync object not signaled yet.
280 */ 281 */
281struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync) 282struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync, bool *explicit)
282{ 283{
283 struct amdgpu_sync_entry *e; 284 struct amdgpu_sync_entry *e;
284 struct hlist_node *tmp; 285 struct hlist_node *tmp;
285 struct dma_fence *f; 286 struct dma_fence *f;
286 int i; 287 int i;
287
288 hash_for_each_safe(sync->fences, i, tmp, e, node) { 288 hash_for_each_safe(sync->fences, i, tmp, e, node) {
289 289
290 f = e->fence; 290 f = e->fence;
291 if (explicit)
292 *explicit = e->explicit;
291 293
292 hash_del(&e->node); 294 hash_del(&e->node);
293 kmem_cache_free(amdgpu_sync_slab, e); 295 kmem_cache_free(amdgpu_sync_slab, e);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
index 70d7e3a279a0..7aba38d5c9df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
@@ -41,7 +41,7 @@ struct amdgpu_sync {
41 41
42void amdgpu_sync_create(struct amdgpu_sync *sync); 42void amdgpu_sync_create(struct amdgpu_sync *sync);
43int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, 43int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
44 struct dma_fence *f); 44 struct dma_fence *f, bool explicit);
45int amdgpu_sync_resv(struct amdgpu_device *adev, 45int amdgpu_sync_resv(struct amdgpu_device *adev,
46 struct amdgpu_sync *sync, 46 struct amdgpu_sync *sync,
47 struct reservation_object *resv, 47 struct reservation_object *resv,
@@ -49,7 +49,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
49 bool explicit_sync); 49 bool explicit_sync);
50struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync, 50struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
51 struct amdgpu_ring *ring); 51 struct amdgpu_ring *ring);
52struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync); 52struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync, bool *explicit);
53int amdgpu_sync_wait(struct amdgpu_sync *sync, bool intr); 53int amdgpu_sync_wait(struct amdgpu_sync *sync, bool intr);
54void amdgpu_sync_free(struct amdgpu_sync *sync); 54void amdgpu_sync_free(struct amdgpu_sync *sync);
55int amdgpu_sync_init(void); 55int amdgpu_sync_init(void);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index ad5bf86ee8a3..952e0bf3bc84 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -110,7 +110,7 @@ static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
110 ring = adev->mman.buffer_funcs_ring; 110 ring = adev->mman.buffer_funcs_ring;
111 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL]; 111 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
112 r = amd_sched_entity_init(&ring->sched, &adev->mman.entity, 112 r = amd_sched_entity_init(&ring->sched, &adev->mman.entity,
113 rq, amdgpu_sched_jobs); 113 rq, amdgpu_sched_jobs, NULL);
114 if (r) { 114 if (r) {
115 DRM_ERROR("Failed setting up TTM BO move run queue.\n"); 115 DRM_ERROR("Failed setting up TTM BO move run queue.\n");
116 goto error_entity; 116 goto error_entity;
@@ -282,8 +282,7 @@ static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
282{ 282{
283 uint64_t addr = 0; 283 uint64_t addr = 0;
284 284
285 if (mem->mem_type != TTM_PL_TT || 285 if (mem->mem_type != TTM_PL_TT || amdgpu_gtt_mgr_has_gart_addr(mem)) {
286 amdgpu_gtt_mgr_is_allocated(mem)) {
287 addr = mm_node->start << PAGE_SHIFT; 286 addr = mm_node->start << PAGE_SHIFT;
288 addr += bo->bdev->man[mem->mem_type].gpu_offset; 287 addr += bo->bdev->man[mem->mem_type].gpu_offset;
289 } 288 }
@@ -369,7 +368,7 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
369 * dst to window 1 368 * dst to window 1
370 */ 369 */
371 if (src->mem->mem_type == TTM_PL_TT && 370 if (src->mem->mem_type == TTM_PL_TT &&
372 !amdgpu_gtt_mgr_is_allocated(src->mem)) { 371 !amdgpu_gtt_mgr_has_gart_addr(src->mem)) {
373 r = amdgpu_map_buffer(src->bo, src->mem, 372 r = amdgpu_map_buffer(src->bo, src->mem,
374 PFN_UP(cur_size + src_page_offset), 373 PFN_UP(cur_size + src_page_offset),
375 src_node_start, 0, ring, 374 src_node_start, 0, ring,
@@ -383,7 +382,7 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
383 } 382 }
384 383
385 if (dst->mem->mem_type == TTM_PL_TT && 384 if (dst->mem->mem_type == TTM_PL_TT &&
386 !amdgpu_gtt_mgr_is_allocated(dst->mem)) { 385 !amdgpu_gtt_mgr_has_gart_addr(dst->mem)) {
387 r = amdgpu_map_buffer(dst->bo, dst->mem, 386 r = amdgpu_map_buffer(dst->bo, dst->mem,
388 PFN_UP(cur_size + dst_page_offset), 387 PFN_UP(cur_size + dst_page_offset),
389 dst_node_start, 1, ring, 388 dst_node_start, 1, ring,
@@ -467,9 +466,8 @@ error:
467 return r; 466 return r;
468} 467}
469 468
470static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, 469static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
471 bool evict, bool interruptible, 470 struct ttm_operation_ctx *ctx,
472 bool no_wait_gpu,
473 struct ttm_mem_reg *new_mem) 471 struct ttm_mem_reg *new_mem)
474{ 472{
475 struct amdgpu_device *adev; 473 struct amdgpu_device *adev;
@@ -489,8 +487,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
489 placements.fpfn = 0; 487 placements.fpfn = 0;
490 placements.lpfn = 0; 488 placements.lpfn = 0;
491 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 489 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
492 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, 490 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
493 interruptible, no_wait_gpu);
494 if (unlikely(r)) { 491 if (unlikely(r)) {
495 return r; 492 return r;
496 } 493 }
@@ -504,19 +501,18 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
504 if (unlikely(r)) { 501 if (unlikely(r)) {
505 goto out_cleanup; 502 goto out_cleanup;
506 } 503 }
507 r = amdgpu_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem); 504 r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, &tmp_mem, old_mem);
508 if (unlikely(r)) { 505 if (unlikely(r)) {
509 goto out_cleanup; 506 goto out_cleanup;
510 } 507 }
511 r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, new_mem); 508 r = ttm_bo_move_ttm(bo, ctx->interruptible, ctx->no_wait_gpu, new_mem);
512out_cleanup: 509out_cleanup:
513 ttm_bo_mem_put(bo, &tmp_mem); 510 ttm_bo_mem_put(bo, &tmp_mem);
514 return r; 511 return r;
515} 512}
516 513
517static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, 514static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
518 bool evict, bool interruptible, 515 struct ttm_operation_ctx *ctx,
519 bool no_wait_gpu,
520 struct ttm_mem_reg *new_mem) 516 struct ttm_mem_reg *new_mem)
521{ 517{
522 struct amdgpu_device *adev; 518 struct amdgpu_device *adev;
@@ -536,16 +532,15 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
536 placements.fpfn = 0; 532 placements.fpfn = 0;
537 placements.lpfn = 0; 533 placements.lpfn = 0;
538 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 534 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
539 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, 535 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
540 interruptible, no_wait_gpu);
541 if (unlikely(r)) { 536 if (unlikely(r)) {
542 return r; 537 return r;
543 } 538 }
544 r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, &tmp_mem); 539 r = ttm_bo_move_ttm(bo, ctx->interruptible, ctx->no_wait_gpu, &tmp_mem);
545 if (unlikely(r)) { 540 if (unlikely(r)) {
546 goto out_cleanup; 541 goto out_cleanup;
547 } 542 }
548 r = amdgpu_move_blit(bo, true, no_wait_gpu, new_mem, old_mem); 543 r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, new_mem, old_mem);
549 if (unlikely(r)) { 544 if (unlikely(r)) {
550 goto out_cleanup; 545 goto out_cleanup;
551 } 546 }
@@ -554,10 +549,9 @@ out_cleanup:
554 return r; 549 return r;
555} 550}
556 551
557static int amdgpu_bo_move(struct ttm_buffer_object *bo, 552static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
558 bool evict, bool interruptible, 553 struct ttm_operation_ctx *ctx,
559 bool no_wait_gpu, 554 struct ttm_mem_reg *new_mem)
560 struct ttm_mem_reg *new_mem)
561{ 555{
562 struct amdgpu_device *adev; 556 struct amdgpu_device *adev;
563 struct amdgpu_bo *abo; 557 struct amdgpu_bo *abo;
@@ -592,19 +586,19 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo,
592 586
593 if (old_mem->mem_type == TTM_PL_VRAM && 587 if (old_mem->mem_type == TTM_PL_VRAM &&
594 new_mem->mem_type == TTM_PL_SYSTEM) { 588 new_mem->mem_type == TTM_PL_SYSTEM) {
595 r = amdgpu_move_vram_ram(bo, evict, interruptible, 589 r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem);
596 no_wait_gpu, new_mem);
597 } else if (old_mem->mem_type == TTM_PL_SYSTEM && 590 } else if (old_mem->mem_type == TTM_PL_SYSTEM &&
598 new_mem->mem_type == TTM_PL_VRAM) { 591 new_mem->mem_type == TTM_PL_VRAM) {
599 r = amdgpu_move_ram_vram(bo, evict, interruptible, 592 r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem);
600 no_wait_gpu, new_mem);
601 } else { 593 } else {
602 r = amdgpu_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem); 594 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu,
595 new_mem, old_mem);
603 } 596 }
604 597
605 if (r) { 598 if (r) {
606memcpy: 599memcpy:
607 r = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, new_mem); 600 r = ttm_bo_move_memcpy(bo, ctx->interruptible,
601 ctx->no_wait_gpu, new_mem);
608 if (r) { 602 if (r) {
609 return r; 603 return r;
610 } 604 }
@@ -690,7 +684,6 @@ struct amdgpu_ttm_tt {
690 struct list_head guptasks; 684 struct list_head guptasks;
691 atomic_t mmu_invalidations; 685 atomic_t mmu_invalidations;
692 uint32_t last_set_pages; 686 uint32_t last_set_pages;
693 struct list_head list;
694}; 687};
695 688
696int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages) 689int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
@@ -861,44 +854,35 @@ static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
861 bo_mem->mem_type == AMDGPU_PL_OA) 854 bo_mem->mem_type == AMDGPU_PL_OA)
862 return -EINVAL; 855 return -EINVAL;
863 856
864 if (!amdgpu_gtt_mgr_is_allocated(bo_mem)) 857 if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
858 gtt->offset = AMDGPU_BO_INVALID_OFFSET;
865 return 0; 859 return 0;
860 }
866 861
867 spin_lock(&gtt->adev->gtt_list_lock);
868 flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, bo_mem); 862 flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, bo_mem);
869 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT; 863 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
870 r = amdgpu_gart_bind(gtt->adev, gtt->offset, ttm->num_pages, 864 r = amdgpu_gart_bind(gtt->adev, gtt->offset, ttm->num_pages,
871 ttm->pages, gtt->ttm.dma_address, flags); 865 ttm->pages, gtt->ttm.dma_address, flags);
872 866
873 if (r) { 867 if (r)
874 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n", 868 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
875 ttm->num_pages, gtt->offset); 869 ttm->num_pages, gtt->offset);
876 goto error_gart_bind;
877 }
878
879 list_add_tail(&gtt->list, &gtt->adev->gtt_list);
880error_gart_bind:
881 spin_unlock(&gtt->adev->gtt_list_lock);
882 return r; 870 return r;
883} 871}
884 872
885bool amdgpu_ttm_is_bound(struct ttm_tt *ttm) 873int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
886{
887 struct amdgpu_ttm_tt *gtt = (void *)ttm;
888
889 return gtt && !list_empty(&gtt->list);
890}
891
892int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
893{ 874{
894 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 875 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
895 struct ttm_tt *ttm = bo->ttm; 876 struct ttm_operation_ctx ctx = { false, false };
877 struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;
896 struct ttm_mem_reg tmp; 878 struct ttm_mem_reg tmp;
897 struct ttm_placement placement; 879 struct ttm_placement placement;
898 struct ttm_place placements; 880 struct ttm_place placements;
881 uint64_t flags;
899 int r; 882 int r;
900 883
901 if (!ttm || amdgpu_ttm_is_bound(ttm)) 884 if (bo->mem.mem_type != TTM_PL_TT ||
885 amdgpu_gtt_mgr_has_gart_addr(&bo->mem))
902 return 0; 886 return 0;
903 887
904 tmp = bo->mem; 888 tmp = bo->mem;
@@ -912,43 +896,44 @@ int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
912 placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) | 896 placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
913 TTM_PL_FLAG_TT; 897 TTM_PL_FLAG_TT;
914 898
915 r = ttm_bo_mem_space(bo, &placement, &tmp, true, false); 899 r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
916 if (unlikely(r)) 900 if (unlikely(r))
917 return r; 901 return r;
918 902
919 r = ttm_bo_move_ttm(bo, true, false, &tmp); 903 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
920 if (unlikely(r)) 904 gtt->offset = (u64)tmp.start << PAGE_SHIFT;
905 r = amdgpu_gart_bind(adev, gtt->offset, bo->ttm->num_pages,
906 bo->ttm->pages, gtt->ttm.dma_address, flags);
907 if (unlikely(r)) {
921 ttm_bo_mem_put(bo, &tmp); 908 ttm_bo_mem_put(bo, &tmp);
922 else 909 return r;
923 bo->offset = (bo->mem.start << PAGE_SHIFT) + 910 }
924 bo->bdev->man[bo->mem.mem_type].gpu_offset;
925 911
926 return r; 912 ttm_bo_mem_put(bo, &bo->mem);
913 bo->mem = tmp;
914 bo->offset = (bo->mem.start << PAGE_SHIFT) +
915 bo->bdev->man[bo->mem.mem_type].gpu_offset;
916
917 return 0;
927} 918}
928 919
929int amdgpu_ttm_recover_gart(struct amdgpu_device *adev) 920int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
930{ 921{
931 struct amdgpu_ttm_tt *gtt, *tmp; 922 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
932 struct ttm_mem_reg bo_mem; 923 struct amdgpu_ttm_tt *gtt = (void *)tbo->ttm;
933 uint64_t flags; 924 uint64_t flags;
934 int r; 925 int r;
935 926
936 bo_mem.mem_type = TTM_PL_TT; 927 if (!gtt)
937 spin_lock(&adev->gtt_list_lock); 928 return 0;
938 list_for_each_entry_safe(gtt, tmp, &adev->gtt_list, list) { 929
939 flags = amdgpu_ttm_tt_pte_flags(gtt->adev, &gtt->ttm.ttm, &bo_mem); 930 flags = amdgpu_ttm_tt_pte_flags(adev, &gtt->ttm.ttm, &tbo->mem);
940 r = amdgpu_gart_bind(adev, gtt->offset, gtt->ttm.ttm.num_pages, 931 r = amdgpu_gart_bind(adev, gtt->offset, gtt->ttm.ttm.num_pages,
941 gtt->ttm.ttm.pages, gtt->ttm.dma_address, 932 gtt->ttm.ttm.pages, gtt->ttm.dma_address, flags);
942 flags); 933 if (r)
943 if (r) { 934 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
944 spin_unlock(&adev->gtt_list_lock); 935 gtt->ttm.ttm.num_pages, gtt->offset);
945 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n", 936 return r;
946 gtt->ttm.ttm.num_pages, gtt->offset);
947 return r;
948 }
949 }
950 spin_unlock(&adev->gtt_list_lock);
951 return 0;
952} 937}
953 938
954static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm) 939static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
@@ -959,20 +944,14 @@ static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
959 if (gtt->userptr) 944 if (gtt->userptr)
960 amdgpu_ttm_tt_unpin_userptr(ttm); 945 amdgpu_ttm_tt_unpin_userptr(ttm);
961 946
962 if (!amdgpu_ttm_is_bound(ttm)) 947 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
963 return 0; 948 return 0;
964 949
965 /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */ 950 /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
966 spin_lock(&gtt->adev->gtt_list_lock);
967 r = amdgpu_gart_unbind(gtt->adev, gtt->offset, ttm->num_pages); 951 r = amdgpu_gart_unbind(gtt->adev, gtt->offset, ttm->num_pages);
968 if (r) { 952 if (r)
969 DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n", 953 DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n",
970 gtt->ttm.ttm.num_pages, gtt->offset); 954 gtt->ttm.ttm.num_pages, gtt->offset);
971 goto error_unbind;
972 }
973 list_del_init(&gtt->list);
974error_unbind:
975 spin_unlock(&gtt->adev->gtt_list_lock);
976 return r; 955 return r;
977} 956}
978 957
@@ -1009,7 +988,6 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_bo_device *bdev,
1009 kfree(gtt); 988 kfree(gtt);
1010 return NULL; 989 return NULL;
1011 } 990 }
1012 INIT_LIST_HEAD(&gtt->list);
1013 return &gtt->ttm.ttm; 991 return &gtt->ttm.ttm;
1014} 992}
1015 993
@@ -1348,10 +1326,13 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
1348 DRM_INFO("amdgpu: %uM of VRAM memory ready\n", 1326 DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
1349 (unsigned) (adev->mc.real_vram_size / (1024 * 1024))); 1327 (unsigned) (adev->mc.real_vram_size / (1024 * 1024)));
1350 1328
1351 if (amdgpu_gtt_size == -1) 1329 if (amdgpu_gtt_size == -1) {
1352 gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20), 1330 struct sysinfo si;
1353 adev->mc.mc_vram_size); 1331
1354 else 1332 si_meminfo(&si);
1333 gtt_size = max(AMDGPU_DEFAULT_GTT_SIZE_MB << 20,
1334 (uint64_t)si.totalram * si.mem_unit * 3/4);
1335 } else
1355 gtt_size = (uint64_t)amdgpu_gtt_size << 20; 1336 gtt_size = (uint64_t)amdgpu_gtt_size << 20;
1356 r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT); 1337 r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
1357 if (r) { 1338 if (r) {
@@ -1410,19 +1391,13 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
1410 1391
1411void amdgpu_ttm_fini(struct amdgpu_device *adev) 1392void amdgpu_ttm_fini(struct amdgpu_device *adev)
1412{ 1393{
1413 int r;
1414
1415 if (!adev->mman.initialized) 1394 if (!adev->mman.initialized)
1416 return; 1395 return;
1396
1417 amdgpu_ttm_debugfs_fini(adev); 1397 amdgpu_ttm_debugfs_fini(adev);
1418 if (adev->stolen_vga_memory) { 1398 amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
1419 r = amdgpu_bo_reserve(adev->stolen_vga_memory, true); 1399 amdgpu_fw_reserve_vram_fini(adev);
1420 if (r == 0) { 1400
1421 amdgpu_bo_unpin(adev->stolen_vga_memory);
1422 amdgpu_bo_unreserve(adev->stolen_vga_memory);
1423 }
1424 amdgpu_bo_unref(&adev->stolen_vga_memory);
1425 }
1426 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM); 1401 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
1427 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT); 1402 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
1428 if (adev->gds.mem.total_size) 1403 if (adev->gds.mem.total_size)
@@ -1432,7 +1407,6 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
1432 if (adev->gds.oa.total_size) 1407 if (adev->gds.oa.total_size)
1433 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA); 1408 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
1434 ttm_bo_device_release(&adev->mman.bdev); 1409 ttm_bo_device_release(&adev->mman.bdev);
1435 amdgpu_gart_fini(adev);
1436 amdgpu_ttm_global_fini(adev); 1410 amdgpu_ttm_global_fini(adev);
1437 adev->mman.initialized = false; 1411 adev->mman.initialized = false;
1438 DRM_INFO("amdgpu: ttm finalized\n"); 1412 DRM_INFO("amdgpu: ttm finalized\n");
@@ -1628,7 +1602,7 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
1628 } 1602 }
1629 1603
1630 if (bo->tbo.mem.mem_type == TTM_PL_TT) { 1604 if (bo->tbo.mem.mem_type == TTM_PL_TT) {
1631 r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem); 1605 r = amdgpu_ttm_alloc_gart(&bo->tbo);
1632 if (r) 1606 if (r)
1633 return r; 1607 return r;
1634 } 1608 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index abd4084982a3..4f9433e61406 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -67,8 +67,9 @@ struct amdgpu_copy_mem {
67extern const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func; 67extern const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func;
68extern const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func; 68extern const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func;
69 69
70bool amdgpu_gtt_mgr_is_allocated(struct ttm_mem_reg *mem); 70bool amdgpu_gtt_mgr_has_gart_addr(struct ttm_mem_reg *mem);
71uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man); 71uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man);
72int amdgpu_gtt_mgr_recover(struct ttm_mem_type_manager *man);
72 73
73uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man); 74uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man);
74uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man); 75uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man);
@@ -90,9 +91,8 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
90 struct dma_fence **fence); 91 struct dma_fence **fence);
91 92
92int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma); 93int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
93bool amdgpu_ttm_is_bound(struct ttm_tt *ttm); 94int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);
94int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem); 95int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
95int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);
96 96
97int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages); 97int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
98void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages); 98void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 65649026b836..474f88fbafce 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -359,7 +359,6 @@ static int amdgpu_ucode_patch_jt(struct amdgpu_firmware_info *ucode,
359 359
360int amdgpu_ucode_init_bo(struct amdgpu_device *adev) 360int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
361{ 361{
362 struct amdgpu_bo **bo = &adev->firmware.fw_buf;
363 uint64_t fw_offset = 0; 362 uint64_t fw_offset = 0;
364 int i, err; 363 int i, err;
365 struct amdgpu_firmware_info *ucode = NULL; 364 struct amdgpu_firmware_info *ucode = NULL;
@@ -370,36 +369,16 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
370 return 0; 369 return 0;
371 } 370 }
372 371
373 if (!amdgpu_sriov_vf(adev) || !adev->in_sriov_reset) { 372 if (!adev->in_gpu_reset) {
374 err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true, 373 err = amdgpu_bo_create_kernel(adev, adev->firmware.fw_size, PAGE_SIZE,
375 amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT, 374 amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
376 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, 375 &adev->firmware.fw_buf,
377 NULL, NULL, 0, bo); 376 &adev->firmware.fw_buf_mc,
377 &adev->firmware.fw_buf_ptr);
378 if (err) { 378 if (err) {
379 dev_err(adev->dev, "(%d) Firmware buffer allocate failed\n", err); 379 dev_err(adev->dev, "failed to create kernel buffer for firmware.fw_buf\n");
380 goto failed; 380 goto failed;
381 } 381 }
382
383 err = amdgpu_bo_reserve(*bo, false);
384 if (err) {
385 dev_err(adev->dev, "(%d) Firmware buffer reserve failed\n", err);
386 goto failed_reserve;
387 }
388
389 err = amdgpu_bo_pin(*bo, amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
390 &adev->firmware.fw_buf_mc);
391 if (err) {
392 dev_err(adev->dev, "(%d) Firmware buffer pin failed\n", err);
393 goto failed_pin;
394 }
395
396 err = amdgpu_bo_kmap(*bo, &adev->firmware.fw_buf_ptr);
397 if (err) {
398 dev_err(adev->dev, "(%d) Firmware buffer kmap failed\n", err);
399 goto failed_kmap;
400 }
401
402 amdgpu_bo_unreserve(*bo);
403 } 382 }
404 383
405 memset(adev->firmware.fw_buf_ptr, 0, adev->firmware.fw_size); 384 memset(adev->firmware.fw_buf_ptr, 0, adev->firmware.fw_size);
@@ -436,12 +415,6 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
436 } 415 }
437 return 0; 416 return 0;
438 417
439failed_kmap:
440 amdgpu_bo_unpin(*bo);
441failed_pin:
442 amdgpu_bo_unreserve(*bo);
443failed_reserve:
444 amdgpu_bo_unref(bo);
445failed: 418failed:
446 if (err) 419 if (err)
447 adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT; 420 adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
@@ -464,8 +437,10 @@ int amdgpu_ucode_fini_bo(struct amdgpu_device *adev)
464 ucode->kaddr = NULL; 437 ucode->kaddr = NULL;
465 } 438 }
466 } 439 }
467 amdgpu_bo_unref(&adev->firmware.fw_buf); 440
468 adev->firmware.fw_buf = NULL; 441 amdgpu_bo_free_kernel(&adev->firmware.fw_buf,
442 &adev->firmware.fw_buf_mc,
443 &adev->firmware.fw_buf_ptr);
469 444
470 return 0; 445 return 0;
471} 446}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index e8bd50cf9785..2f2a9e17fdb4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -232,7 +232,7 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
232 ring = &adev->uvd.ring; 232 ring = &adev->uvd.ring;
233 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL]; 233 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
234 r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity, 234 r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity,
235 rq, amdgpu_sched_jobs); 235 rq, amdgpu_sched_jobs, NULL);
236 if (r != 0) { 236 if (r != 0) {
237 DRM_ERROR("Failed setting up UVD run queue.\n"); 237 DRM_ERROR("Failed setting up UVD run queue.\n");
238 return r; 238 return r;
@@ -408,6 +408,7 @@ static u64 amdgpu_uvd_get_addr_from_ctx(struct amdgpu_uvd_cs_ctx *ctx)
408 */ 408 */
409static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx) 409static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx)
410{ 410{
411 struct ttm_operation_ctx tctx = { false, false };
411 struct amdgpu_bo_va_mapping *mapping; 412 struct amdgpu_bo_va_mapping *mapping;
412 struct amdgpu_bo *bo; 413 struct amdgpu_bo *bo;
413 uint32_t cmd; 414 uint32_t cmd;
@@ -430,7 +431,7 @@ static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx)
430 } 431 }
431 amdgpu_uvd_force_into_uvd_segment(bo); 432 amdgpu_uvd_force_into_uvd_segment(bo);
432 433
433 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); 434 r = ttm_bo_validate(&bo->tbo, &bo->placement, &tctx);
434 } 435 }
435 436
436 return r; 437 return r;
@@ -949,6 +950,7 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx)
949static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo, 950static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
950 bool direct, struct dma_fence **fence) 951 bool direct, struct dma_fence **fence)
951{ 952{
953 struct ttm_operation_ctx ctx = { true, false };
952 struct ttm_validate_buffer tv; 954 struct ttm_validate_buffer tv;
953 struct ww_acquire_ctx ticket; 955 struct ww_acquire_ctx ticket;
954 struct list_head head; 956 struct list_head head;
@@ -975,7 +977,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
975 amdgpu_uvd_force_into_uvd_segment(bo); 977 amdgpu_uvd_force_into_uvd_segment(bo);
976 } 978 }
977 979
978 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 980 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
979 if (r) 981 if (r)
980 goto err; 982 goto err;
981 983
@@ -1218,7 +1220,7 @@ int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1218 } else if (r < 0) { 1220 } else if (r < 0) {
1219 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); 1221 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
1220 } else { 1222 } else {
1221 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 1223 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
1222 r = 0; 1224 r = 0;
1223 } 1225 }
1224 1226
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
index 3553b92bf69a..845eea993f75 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
@@ -31,6 +31,10 @@
31#define AMDGPU_UVD_SESSION_SIZE (50*1024) 31#define AMDGPU_UVD_SESSION_SIZE (50*1024)
32#define AMDGPU_UVD_FIRMWARE_OFFSET 256 32#define AMDGPU_UVD_FIRMWARE_OFFSET 256
33 33
34#define AMDGPU_UVD_FIRMWARE_SIZE(adev) \
35 (AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(((const struct common_firmware_header *)(adev)->uvd.fw->data)->ucode_size_bytes) + \
36 8) - AMDGPU_UVD_FIRMWARE_OFFSET)
37
34struct amdgpu_uvd { 38struct amdgpu_uvd {
35 struct amdgpu_bo *vcpu_bo; 39 struct amdgpu_bo *vcpu_bo;
36 void *cpu_addr; 40 void *cpu_addr;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 2918de2f39ec..ba6d846b08ff 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -176,7 +176,7 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
176 ring = &adev->vce.ring[0]; 176 ring = &adev->vce.ring[0];
177 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL]; 177 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
178 r = amd_sched_entity_init(&ring->sched, &adev->vce.entity, 178 r = amd_sched_entity_init(&ring->sched, &adev->vce.entity,
179 rq, amdgpu_sched_jobs); 179 rq, amdgpu_sched_jobs, NULL);
180 if (r != 0) { 180 if (r != 0) {
181 DRM_ERROR("Failed setting up VCE run queue.\n"); 181 DRM_ERROR("Failed setting up VCE run queue.\n");
182 return r; 182 return r;
@@ -544,6 +544,55 @@ err:
544} 544}
545 545
546/** 546/**
547 * amdgpu_vce_cs_validate_bo - make sure not to cross 4GB boundary
548 *
549 * @p: parser context
550 * @lo: address of lower dword
551 * @hi: address of higher dword
552 * @size: minimum size
553 * @index: bs/fb index
554 *
555 * Make sure that no BO cross a 4GB boundary.
556 */
557static int amdgpu_vce_validate_bo(struct amdgpu_cs_parser *p, uint32_t ib_idx,
558 int lo, int hi, unsigned size, int32_t index)
559{
560 int64_t offset = ((uint64_t)size) * ((int64_t)index);
561 struct ttm_operation_ctx ctx = { false, false };
562 struct amdgpu_bo_va_mapping *mapping;
563 unsigned i, fpfn, lpfn;
564 struct amdgpu_bo *bo;
565 uint64_t addr;
566 int r;
567
568 addr = ((uint64_t)amdgpu_get_ib_value(p, ib_idx, lo)) |
569 ((uint64_t)amdgpu_get_ib_value(p, ib_idx, hi)) << 32;
570 if (index >= 0) {
571 addr += offset;
572 fpfn = PAGE_ALIGN(offset) >> PAGE_SHIFT;
573 lpfn = 0x100000000ULL >> PAGE_SHIFT;
574 } else {
575 fpfn = 0;
576 lpfn = (0x100000000ULL - PAGE_ALIGN(offset)) >> PAGE_SHIFT;
577 }
578
579 r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping);
580 if (r) {
581 DRM_ERROR("Can't find BO for addr 0x%010Lx %d %d %d %d\n",
582 addr, lo, hi, size, index);
583 return r;
584 }
585
586 for (i = 0; i < bo->placement.num_placement; ++i) {
587 bo->placements[i].fpfn = max(bo->placements[i].fpfn, fpfn);
588 bo->placements[i].lpfn = bo->placements[i].fpfn ?
589 min(bo->placements[i].fpfn, lpfn) : lpfn;
590 }
591 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
592}
593
594
595/**
547 * amdgpu_vce_cs_reloc - command submission relocation 596 * amdgpu_vce_cs_reloc - command submission relocation
548 * 597 *
549 * @p: parser context 598 * @p: parser context
@@ -648,12 +697,13 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
648 uint32_t allocated = 0; 697 uint32_t allocated = 0;
649 uint32_t tmp, handle = 0; 698 uint32_t tmp, handle = 0;
650 uint32_t *size = &tmp; 699 uint32_t *size = &tmp;
651 int i, r = 0, idx = 0; 700 unsigned idx;
701 int i, r = 0;
652 702
653 p->job->vm = NULL; 703 p->job->vm = NULL;
654 ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo); 704 ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
655 705
656 while (idx < ib->length_dw) { 706 for (idx = 0; idx < ib->length_dw;) {
657 uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx); 707 uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
658 uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1); 708 uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
659 709
@@ -664,6 +714,54 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
664 } 714 }
665 715
666 switch (cmd) { 716 switch (cmd) {
717 case 0x00000002: /* task info */
718 fb_idx = amdgpu_get_ib_value(p, ib_idx, idx + 6);
719 bs_idx = amdgpu_get_ib_value(p, ib_idx, idx + 7);
720 break;
721
722 case 0x03000001: /* encode */
723 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 10,
724 idx + 9, 0, 0);
725 if (r)
726 goto out;
727
728 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 12,
729 idx + 11, 0, 0);
730 if (r)
731 goto out;
732 break;
733
734 case 0x05000001: /* context buffer */
735 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3,
736 idx + 2, 0, 0);
737 if (r)
738 goto out;
739 break;
740
741 case 0x05000004: /* video bitstream buffer */
742 tmp = amdgpu_get_ib_value(p, ib_idx, idx + 4);
743 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, idx + 2,
744 tmp, bs_idx);
745 if (r)
746 goto out;
747 break;
748
749 case 0x05000005: /* feedback buffer */
750 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, idx + 2,
751 4096, fb_idx);
752 if (r)
753 goto out;
754 break;
755 }
756
757 idx += len / 4;
758 }
759
760 for (idx = 0; idx < ib->length_dw;) {
761 uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
762 uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
763
764 switch (cmd) {
667 case 0x00000001: /* session */ 765 case 0x00000001: /* session */
668 handle = amdgpu_get_ib_value(p, ib_idx, idx + 2); 766 handle = amdgpu_get_ib_value(p, ib_idx, idx + 2);
669 session_idx = amdgpu_vce_validate_handle(p, handle, 767 session_idx = amdgpu_vce_validate_handle(p, handle,
@@ -954,7 +1052,7 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
954 } 1052 }
955 1053
956 if (i < timeout) { 1054 if (i < timeout) {
957 DRM_INFO("ring test on %d succeeded in %d usecs\n", 1055 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
958 ring->idx, i); 1056 ring->idx, i);
959 } else { 1057 } else {
960 DRM_ERROR("amdgpu: ring %d test failed\n", 1058 DRM_ERROR("amdgpu: ring %d test failed\n",
@@ -999,7 +1097,7 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
999 } else if (r < 0) { 1097 } else if (r < 0) {
1000 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); 1098 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
1001 } else { 1099 } else {
1002 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 1100 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
1003 r = 0; 1101 r = 0;
1004 } 1102 }
1005error: 1103error:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index 041e0121590c..d7ba048c2f80 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -35,8 +35,8 @@
35#include "soc15d.h" 35#include "soc15d.h"
36#include "soc15_common.h" 36#include "soc15_common.h"
37 37
38#include "vega10/soc15ip.h" 38#include "soc15ip.h"
39#include "raven1/VCN/vcn_1_0_offset.h" 39#include "vcn/vcn_1_0_offset.h"
40 40
41/* 1 second timeout */ 41/* 1 second timeout */
42#define VCN_IDLE_TIMEOUT msecs_to_jiffies(1000) 42#define VCN_IDLE_TIMEOUT msecs_to_jiffies(1000)
@@ -106,7 +106,7 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
106 ring = &adev->vcn.ring_dec; 106 ring = &adev->vcn.ring_dec;
107 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL]; 107 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
108 r = amd_sched_entity_init(&ring->sched, &adev->vcn.entity_dec, 108 r = amd_sched_entity_init(&ring->sched, &adev->vcn.entity_dec,
109 rq, amdgpu_sched_jobs); 109 rq, amdgpu_sched_jobs, NULL);
110 if (r != 0) { 110 if (r != 0) {
111 DRM_ERROR("Failed setting up VCN dec run queue.\n"); 111 DRM_ERROR("Failed setting up VCN dec run queue.\n");
112 return r; 112 return r;
@@ -115,7 +115,7 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
115 ring = &adev->vcn.ring_enc[0]; 115 ring = &adev->vcn.ring_enc[0];
116 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL]; 116 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
117 r = amd_sched_entity_init(&ring->sched, &adev->vcn.entity_enc, 117 r = amd_sched_entity_init(&ring->sched, &adev->vcn.entity_enc,
118 rq, amdgpu_sched_jobs); 118 rq, amdgpu_sched_jobs, NULL);
119 if (r != 0) { 119 if (r != 0) {
120 DRM_ERROR("Failed setting up VCN enc run queue.\n"); 120 DRM_ERROR("Failed setting up VCN enc run queue.\n");
121 return r; 121 return r;
@@ -261,7 +261,7 @@ int amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring *ring)
261 } 261 }
262 262
263 if (i < adev->usec_timeout) { 263 if (i < adev->usec_timeout) {
264 DRM_INFO("ring test on %d succeeded in %d usecs\n", 264 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
265 ring->idx, i); 265 ring->idx, i);
266 } else { 266 } else {
267 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 267 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
@@ -274,6 +274,7 @@ int amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring *ring)
274static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo, 274static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
275 bool direct, struct dma_fence **fence) 275 bool direct, struct dma_fence **fence)
276{ 276{
277 struct ttm_operation_ctx ctx = { true, false };
277 struct ttm_validate_buffer tv; 278 struct ttm_validate_buffer tv;
278 struct ww_acquire_ctx ticket; 279 struct ww_acquire_ctx ticket;
279 struct list_head head; 280 struct list_head head;
@@ -294,7 +295,7 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *b
294 if (r) 295 if (r)
295 return r; 296 return r;
296 297
297 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 298 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
298 if (r) 299 if (r)
299 goto err; 300 goto err;
300 301
@@ -467,7 +468,7 @@ int amdgpu_vcn_dec_ring_test_ib(struct amdgpu_ring *ring, long timeout)
467 } else if (r < 0) { 468 } else if (r < 0) {
468 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); 469 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
469 } else { 470 } else {
470 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 471 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
471 r = 0; 472 r = 0;
472 } 473 }
473 474
@@ -500,7 +501,7 @@ int amdgpu_vcn_enc_ring_test_ring(struct amdgpu_ring *ring)
500 } 501 }
501 502
502 if (i < adev->usec_timeout) { 503 if (i < adev->usec_timeout) {
503 DRM_INFO("ring test on %d succeeded in %d usecs\n", 504 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
504 ring->idx, i); 505 ring->idx, i);
505 } else { 506 } else {
506 DRM_ERROR("amdgpu: ring %d test failed\n", 507 DRM_ERROR("amdgpu: ring %d test failed\n",
@@ -643,7 +644,7 @@ int amdgpu_vcn_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
643 } else if (r < 0) { 644 } else if (r < 0) {
644 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); 645 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
645 } else { 646 } else {
646 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 647 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
647 r = 0; 648 r = 0;
648 } 649 }
649error: 650error:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 6738df836a70..e7dfb7b44b4b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -24,6 +24,14 @@
24#include "amdgpu.h" 24#include "amdgpu.h"
25#define MAX_KIQ_REG_WAIT 100000000 /* in usecs */ 25#define MAX_KIQ_REG_WAIT 100000000 /* in usecs */
26 26
27bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev)
28{
29 /* By now all MMIO pages except mailbox are blocked */
30 /* if blocking is enabled in hypervisor. Choose the */
31 /* SCRATCH_REG0 to test. */
32 return RREG32_NO_KIQ(0xc040) == 0xffffffff;
33}
34
27int amdgpu_allocate_static_csa(struct amdgpu_device *adev) 35int amdgpu_allocate_static_csa(struct amdgpu_device *adev)
28{ 36{
29 int r; 37 int r;
@@ -39,6 +47,12 @@ int amdgpu_allocate_static_csa(struct amdgpu_device *adev)
39 return 0; 47 return 0;
40} 48}
41 49
50void amdgpu_free_static_csa(struct amdgpu_device *adev) {
51 amdgpu_bo_free_kernel(&adev->virt.csa_obj,
52 &adev->virt.csa_vmid0_addr,
53 NULL);
54}
55
42/* 56/*
43 * amdgpu_map_static_csa should be called during amdgpu_vm_init 57 * amdgpu_map_static_csa should be called during amdgpu_vm_init
44 * it maps virtual address "AMDGPU_VA_RESERVED_SIZE - AMDGPU_CSA_SIZE" 58 * it maps virtual address "AMDGPU_VA_RESERVED_SIZE - AMDGPU_CSA_SIZE"
@@ -107,8 +121,6 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
107 adev->enable_virtual_display = true; 121 adev->enable_virtual_display = true;
108 adev->cg_flags = 0; 122 adev->cg_flags = 0;
109 adev->pg_flags = 0; 123 adev->pg_flags = 0;
110
111 mutex_init(&adev->virt.lock_reset);
112} 124}
113 125
114uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) 126uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
@@ -228,6 +240,22 @@ int amdgpu_virt_reset_gpu(struct amdgpu_device *adev)
228} 240}
229 241
230/** 242/**
243 * amdgpu_virt_wait_reset() - wait for reset gpu completed
244 * @amdgpu: amdgpu device.
245 * Wait for GPU reset completed.
246 * Return: Zero if reset success, otherwise will return error.
247 */
248int amdgpu_virt_wait_reset(struct amdgpu_device *adev)
249{
250 struct amdgpu_virt *virt = &adev->virt;
251
252 if (!virt->ops || !virt->ops->wait_reset)
253 return -EINVAL;
254
255 return virt->ops->wait_reset(adev);
256}
257
258/**
231 * amdgpu_virt_alloc_mm_table() - alloc memory for mm table 259 * amdgpu_virt_alloc_mm_table() - alloc memory for mm table
232 * @amdgpu: amdgpu device. 260 * @amdgpu: amdgpu device.
233 * MM table is used by UVD and VCE for its initialization 261 * MM table is used by UVD and VCE for its initialization
@@ -296,7 +324,6 @@ int amdgpu_virt_fw_reserve_get_checksum(void *obj,
296 324
297void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev) 325void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
298{ 326{
299 uint32_t pf2vf_ver = 0;
300 uint32_t pf2vf_size = 0; 327 uint32_t pf2vf_size = 0;
301 uint32_t checksum = 0; 328 uint32_t checksum = 0;
302 uint32_t checkval; 329 uint32_t checkval;
@@ -309,9 +336,9 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
309 adev->virt.fw_reserve.p_pf2vf = 336 adev->virt.fw_reserve.p_pf2vf =
310 (struct amdgim_pf2vf_info_header *)( 337 (struct amdgim_pf2vf_info_header *)(
311 adev->fw_vram_usage.va + AMDGIM_DATAEXCHANGE_OFFSET); 338 adev->fw_vram_usage.va + AMDGIM_DATAEXCHANGE_OFFSET);
312 pf2vf_ver = adev->virt.fw_reserve.p_pf2vf->version;
313 AMDGPU_FW_VRAM_PF2VF_READ(adev, header.size, &pf2vf_size); 339 AMDGPU_FW_VRAM_PF2VF_READ(adev, header.size, &pf2vf_size);
314 AMDGPU_FW_VRAM_PF2VF_READ(adev, checksum, &checksum); 340 AMDGPU_FW_VRAM_PF2VF_READ(adev, checksum, &checksum);
341 AMDGPU_FW_VRAM_PF2VF_READ(adev, feature_flags, &adev->virt.gim_feature);
315 342
316 /* pf2vf message must be in 4K */ 343 /* pf2vf message must be in 4K */
317 if (pf2vf_size > 0 && pf2vf_size < 4096) { 344 if (pf2vf_size > 0 && pf2vf_size < 4096) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index b89d37fc406f..6a83425aa9ed 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -55,6 +55,7 @@ struct amdgpu_virt_ops {
55 int (*req_full_gpu)(struct amdgpu_device *adev, bool init); 55 int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
56 int (*rel_full_gpu)(struct amdgpu_device *adev, bool init); 56 int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
57 int (*reset_gpu)(struct amdgpu_device *adev); 57 int (*reset_gpu)(struct amdgpu_device *adev);
58 int (*wait_reset)(struct amdgpu_device *adev);
58 void (*trans_msg)(struct amdgpu_device *adev, u32 req, u32 data1, u32 data2, u32 data3); 59 void (*trans_msg)(struct amdgpu_device *adev, u32 req, u32 data1, u32 data2, u32 data3);
59}; 60};
60 61
@@ -80,6 +81,8 @@ enum AMDGIM_FEATURE_FLAG {
80 AMDGIM_FEATURE_ERROR_LOG_COLLECT = 0x1, 81 AMDGIM_FEATURE_ERROR_LOG_COLLECT = 0x1,
81 /* GIM supports feature of loading uCodes */ 82 /* GIM supports feature of loading uCodes */
82 AMDGIM_FEATURE_GIM_LOAD_UCODES = 0x2, 83 AMDGIM_FEATURE_GIM_LOAD_UCODES = 0x2,
84 /* VRAM LOST by GIM */
85 AMDGIM_FEATURE_GIM_FLR_VRAMLOST = 0x4,
83}; 86};
84 87
85struct amdgim_pf2vf_info_header { 88struct amdgim_pf2vf_info_header {
@@ -238,7 +241,6 @@ struct amdgpu_virt {
238 uint64_t csa_vmid0_addr; 241 uint64_t csa_vmid0_addr;
239 bool chained_ib_support; 242 bool chained_ib_support;
240 uint32_t reg_val_offs; 243 uint32_t reg_val_offs;
241 struct mutex lock_reset;
242 struct amdgpu_irq_src ack_irq; 244 struct amdgpu_irq_src ack_irq;
243 struct amdgpu_irq_src rcv_irq; 245 struct amdgpu_irq_src rcv_irq;
244 struct work_struct flr_work; 246 struct work_struct flr_work;
@@ -246,6 +248,7 @@ struct amdgpu_virt {
246 const struct amdgpu_virt_ops *ops; 248 const struct amdgpu_virt_ops *ops;
247 struct amdgpu_vf_error_buffer vf_errors; 249 struct amdgpu_vf_error_buffer vf_errors;
248 struct amdgpu_virt_fw_reserve fw_reserve; 250 struct amdgpu_virt_fw_reserve fw_reserve;
251 uint32_t gim_feature;
249}; 252};
250 253
251#define AMDGPU_CSA_SIZE (8 * 1024) 254#define AMDGPU_CSA_SIZE (8 * 1024)
@@ -276,16 +279,18 @@ static inline bool is_virtual_machine(void)
276} 279}
277 280
278struct amdgpu_vm; 281struct amdgpu_vm;
282bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev);
279int amdgpu_allocate_static_csa(struct amdgpu_device *adev); 283int amdgpu_allocate_static_csa(struct amdgpu_device *adev);
280int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm, 284int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm,
281 struct amdgpu_bo_va **bo_va); 285 struct amdgpu_bo_va **bo_va);
286void amdgpu_free_static_csa(struct amdgpu_device *adev);
282void amdgpu_virt_init_setting(struct amdgpu_device *adev); 287void amdgpu_virt_init_setting(struct amdgpu_device *adev);
283uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); 288uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
284void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v); 289void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
285int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init); 290int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
286int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init); 291int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
287int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); 292int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
288int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job); 293int amdgpu_virt_wait_reset(struct amdgpu_device *adev);
289int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev); 294int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev);
290void amdgpu_virt_free_mm_table(struct amdgpu_device *adev); 295void amdgpu_virt_free_mm_table(struct amdgpu_device *adev);
291int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned long obj_size, 296int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned long obj_size,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index c8c26f21993c..3ecdbdfb04dd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -139,6 +139,24 @@ struct amdgpu_prt_cb {
139}; 139};
140 140
141/** 141/**
142 * amdgpu_vm_level_shift - return the addr shift for each level
143 *
144 * @adev: amdgpu_device pointer
145 *
146 * Returns the number of bits the pfn needs to be right shifted for a level.
147 */
148static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
149 unsigned level)
150{
151 if (level != adev->vm_manager.num_level)
152 return 9 * (adev->vm_manager.num_level - level - 1) +
153 adev->vm_manager.block_size;
154 else
155 /* For the page tables on the leaves */
156 return 0;
157}
158
159/**
142 * amdgpu_vm_num_entries - return the number of entries in a PD/PT 160 * amdgpu_vm_num_entries - return the number of entries in a PD/PT
143 * 161 *
144 * @adev: amdgpu_device pointer 162 * @adev: amdgpu_device pointer
@@ -148,17 +166,17 @@ struct amdgpu_prt_cb {
148static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev, 166static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
149 unsigned level) 167 unsigned level)
150{ 168{
169 unsigned shift = amdgpu_vm_level_shift(adev, 0);
170
151 if (level == 0) 171 if (level == 0)
152 /* For the root directory */ 172 /* For the root directory */
153 return adev->vm_manager.max_pfn >> 173 return round_up(adev->vm_manager.max_pfn, 1 << shift) >> shift;
154 (adev->vm_manager.block_size * 174 else if (level != adev->vm_manager.num_level)
155 adev->vm_manager.num_level); 175 /* Everything in between */
156 else if (level == adev->vm_manager.num_level) 176 return 512;
177 else
157 /* For the page tables on the leaves */ 178 /* For the page tables on the leaves */
158 return AMDGPU_VM_PTE_COUNT(adev); 179 return AMDGPU_VM_PTE_COUNT(adev);
159 else
160 /* Everything in between */
161 return 1 << adev->vm_manager.block_size;
162} 180}
163 181
164/** 182/**
@@ -288,8 +306,7 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
288 uint64_t saddr, uint64_t eaddr, 306 uint64_t saddr, uint64_t eaddr,
289 unsigned level) 307 unsigned level)
290{ 308{
291 unsigned shift = (adev->vm_manager.num_level - level) * 309 unsigned shift = amdgpu_vm_level_shift(adev, level);
292 adev->vm_manager.block_size;
293 unsigned pt_idx, from, to; 310 unsigned pt_idx, from, to;
294 int r; 311 int r;
295 u64 flags; 312 u64 flags;
@@ -471,7 +488,7 @@ static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm,
471 id->pd_gpu_addr = 0; 488 id->pd_gpu_addr = 0;
472 tmp = amdgpu_sync_peek_fence(&id->active, ring); 489 tmp = amdgpu_sync_peek_fence(&id->active, ring);
473 if (tmp) { 490 if (tmp) {
474 r = amdgpu_sync_fence(adev, sync, tmp); 491 r = amdgpu_sync_fence(adev, sync, tmp, false);
475 return r; 492 return r;
476 } 493 }
477 } 494 }
@@ -479,7 +496,7 @@ static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm,
479 /* Good we can use this VMID. Remember this submission as 496 /* Good we can use this VMID. Remember this submission as
480 * user of the VMID. 497 * user of the VMID.
481 */ 498 */
482 r = amdgpu_sync_fence(ring->adev, &id->active, fence); 499 r = amdgpu_sync_fence(ring->adev, &id->active, fence, false);
483 if (r) 500 if (r)
484 goto out; 501 goto out;
485 502
@@ -566,7 +583,7 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
566 } 583 }
567 584
568 585
569 r = amdgpu_sync_fence(ring->adev, sync, &array->base); 586 r = amdgpu_sync_fence(ring->adev, sync, &array->base, false);
570 dma_fence_put(&array->base); 587 dma_fence_put(&array->base);
571 if (r) 588 if (r)
572 goto error; 589 goto error;
@@ -609,7 +626,7 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
609 /* Good we can use this VMID. Remember this submission as 626 /* Good we can use this VMID. Remember this submission as
610 * user of the VMID. 627 * user of the VMID.
611 */ 628 */
612 r = amdgpu_sync_fence(ring->adev, &id->active, fence); 629 r = amdgpu_sync_fence(ring->adev, &id->active, fence, false);
613 if (r) 630 if (r)
614 goto error; 631 goto error;
615 632
@@ -629,7 +646,7 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
629 id = idle; 646 id = idle;
630 647
631 /* Remember this submission as user of the VMID */ 648 /* Remember this submission as user of the VMID */
632 r = amdgpu_sync_fence(ring->adev, &id->active, fence); 649 r = amdgpu_sync_fence(ring->adev, &id->active, fence, false);
633 if (r) 650 if (r)
634 goto error; 651 goto error;
635 652
@@ -1302,18 +1319,19 @@ void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
1302 struct amdgpu_vm_pt **entry, 1319 struct amdgpu_vm_pt **entry,
1303 struct amdgpu_vm_pt **parent) 1320 struct amdgpu_vm_pt **parent)
1304{ 1321{
1305 unsigned idx, level = p->adev->vm_manager.num_level; 1322 unsigned level = 0;
1306 1323
1307 *parent = NULL; 1324 *parent = NULL;
1308 *entry = &p->vm->root; 1325 *entry = &p->vm->root;
1309 while ((*entry)->entries) { 1326 while ((*entry)->entries) {
1310 idx = addr >> (p->adev->vm_manager.block_size * level--); 1327 unsigned idx = addr >> amdgpu_vm_level_shift(p->adev, level++);
1328
1311 idx %= amdgpu_bo_size((*entry)->base.bo) / 8; 1329 idx %= amdgpu_bo_size((*entry)->base.bo) / 8;
1312 *parent = *entry; 1330 *parent = *entry;
1313 *entry = &(*entry)->entries[idx]; 1331 *entry = &(*entry)->entries[idx];
1314 } 1332 }
1315 1333
1316 if (level) 1334 if (level != p->adev->vm_manager.num_level)
1317 *entry = NULL; 1335 *entry = NULL;
1318} 1336}
1319 1337
@@ -1639,7 +1657,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1639 addr = 0; 1657 addr = 0;
1640 } 1658 }
1641 1659
1642 r = amdgpu_sync_fence(adev, &job->sync, exclusive); 1660 r = amdgpu_sync_fence(adev, &job->sync, exclusive, false);
1643 if (r) 1661 if (r)
1644 goto error_free; 1662 goto error_free;
1645 1663
@@ -2556,47 +2574,57 @@ static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2556} 2574}
2557 2575
2558/** 2576/**
2559 * amdgpu_vm_set_fragment_size - adjust fragment size in PTE
2560 *
2561 * @adev: amdgpu_device pointer
2562 * @fragment_size_default: the default fragment size if it's set auto
2563 */
2564void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev,
2565 uint32_t fragment_size_default)
2566{
2567 if (amdgpu_vm_fragment_size == -1)
2568 adev->vm_manager.fragment_size = fragment_size_default;
2569 else
2570 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2571}
2572
2573/**
2574 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size 2577 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2575 * 2578 *
2576 * @adev: amdgpu_device pointer 2579 * @adev: amdgpu_device pointer
2577 * @vm_size: the default vm size if it's set auto 2580 * @vm_size: the default vm size if it's set auto
2578 */ 2581 */
2579void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, 2582void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t vm_size,
2580 uint32_t fragment_size_default) 2583 uint32_t fragment_size_default, unsigned max_level,
2584 unsigned max_bits)
2581{ 2585{
2582 /* adjust vm size firstly */ 2586 uint64_t tmp;
2583 if (amdgpu_vm_size == -1)
2584 adev->vm_manager.vm_size = vm_size;
2585 else
2586 adev->vm_manager.vm_size = amdgpu_vm_size;
2587 2587
2588 /* block size depends on vm size */ 2588 /* adjust vm size first */
2589 if (amdgpu_vm_block_size == -1) 2589 if (amdgpu_vm_size != -1) {
2590 unsigned max_size = 1 << (max_bits - 30);
2591
2592 vm_size = amdgpu_vm_size;
2593 if (vm_size > max_size) {
2594 dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2595 amdgpu_vm_size, max_size);
2596 vm_size = max_size;
2597 }
2598 }
2599
2600 adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2601
2602 tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2603 if (amdgpu_vm_block_size != -1)
2604 tmp >>= amdgpu_vm_block_size - 9;
2605 tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2606 adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
2607
2608 /* block size depends on vm size and hw setup*/
2609 if (amdgpu_vm_block_size != -1)
2590 adev->vm_manager.block_size = 2610 adev->vm_manager.block_size =
2591 amdgpu_vm_get_block_size(adev->vm_manager.vm_size); 2611 min((unsigned)amdgpu_vm_block_size, max_bits
2612 - AMDGPU_GPU_PAGE_SHIFT
2613 - 9 * adev->vm_manager.num_level);
2614 else if (adev->vm_manager.num_level > 1)
2615 adev->vm_manager.block_size = 9;
2592 else 2616 else
2593 adev->vm_manager.block_size = amdgpu_vm_block_size; 2617 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2594 2618
2595 amdgpu_vm_set_fragment_size(adev, fragment_size_default); 2619 if (amdgpu_vm_fragment_size == -1)
2620 adev->vm_manager.fragment_size = fragment_size_default;
2621 else
2622 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2596 2623
2597 DRM_INFO("vm size is %llu GB, block size is %u-bit, fragment size is %u-bit\n", 2624 DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2598 adev->vm_manager.vm_size, adev->vm_manager.block_size, 2625 vm_size, adev->vm_manager.num_level + 1,
2599 adev->vm_manager.fragment_size); 2626 adev->vm_manager.block_size,
2627 adev->vm_manager.fragment_size);
2600} 2628}
2601 2629
2602/** 2630/**
@@ -2637,7 +2665,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2637 ring = adev->vm_manager.vm_pte_rings[ring_instance]; 2665 ring = adev->vm_manager.vm_pte_rings[ring_instance];
2638 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL]; 2666 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
2639 r = amd_sched_entity_init(&ring->sched, &vm->entity, 2667 r = amd_sched_entity_init(&ring->sched, &vm->entity,
2640 rq, amdgpu_sched_jobs); 2668 rq, amdgpu_sched_jobs, NULL);
2641 if (r) 2669 if (r)
2642 return r; 2670 return r;
2643 2671
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index bae77353447b..43ea131dd411 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -96,6 +96,19 @@ struct amdgpu_bo_list_entry;
96/* hardcode that limit for now */ 96/* hardcode that limit for now */
97#define AMDGPU_VA_RESERVED_SIZE (8ULL << 20) 97#define AMDGPU_VA_RESERVED_SIZE (8ULL << 20)
98 98
99/* VA hole for 48bit addresses on Vega10 */
100#define AMDGPU_VA_HOLE_START 0x0000800000000000ULL
101#define AMDGPU_VA_HOLE_END 0xffff800000000000ULL
102
103/*
104 * Hardware is programmed as if the hole doesn't exists with start and end
105 * address values.
106 *
107 * This mask is used to remove the upper 16bits of the VA and so come up with
108 * the linear addr value.
109 */
110#define AMDGPU_VA_HOLE_MASK 0x0000ffffffffffffULL
111
99/* max vmids dedicated for process */ 112/* max vmids dedicated for process */
100#define AMDGPU_VM_MAX_RESERVED_VMID 1 113#define AMDGPU_VM_MAX_RESERVED_VMID 1
101 114
@@ -221,7 +234,6 @@ struct amdgpu_vm_manager {
221 234
222 uint64_t max_pfn; 235 uint64_t max_pfn;
223 uint32_t num_level; 236 uint32_t num_level;
224 uint64_t vm_size;
225 uint32_t block_size; 237 uint32_t block_size;
226 uint32_t fragment_size; 238 uint32_t fragment_size;
227 /* vram base address for page table entry */ 239 /* vram base address for page table entry */
@@ -312,10 +324,9 @@ struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
312 uint64_t addr); 324 uint64_t addr);
313void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, 325void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
314 struct amdgpu_bo_va *bo_va); 326 struct amdgpu_bo_va *bo_va);
315void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev, 327void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t vm_size,
316 uint32_t fragment_size_default); 328 uint32_t fragment_size_default, unsigned max_level,
317void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, 329 unsigned max_bits);
318 uint32_t fragment_size_default);
319int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); 330int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
320bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, 331bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
321 struct amdgpu_job *job); 332 struct amdgpu_job *job);
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
index b374653bd6cf..f9b2ce9a98f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
@@ -65,8 +65,15 @@ static int amdgpu_atombios_i2c_process_i2c_ch(struct amdgpu_i2c_chan *chan,
65 args.ucRegIndex = buf[0]; 65 args.ucRegIndex = buf[0];
66 if (num) 66 if (num)
67 num--; 67 num--;
68 if (num) 68 if (num) {
69 memcpy(&out, &buf[1], num); 69 if (buf) {
70 memcpy(&out, &buf[1], num);
71 } else {
72 DRM_ERROR("hw i2c: missing buf with num > 1\n");
73 r = -EINVAL;
74 goto done;
75 }
76 }
70 args.lpI2CDataOut = cpu_to_le16(out); 77 args.lpI2CDataOut = cpu_to_le16(out);
71 } else { 78 } else {
72 if (num > ATOM_MAX_HW_I2C_READ) { 79 if (num > ATOM_MAX_HW_I2C_READ) {
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index 68b505c768ad..f11c0aacf19f 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -4540,9 +4540,9 @@ static int ci_set_mc_special_registers(struct amdgpu_device *adev,
4540 ((temp_reg & 0xffff0000)) | ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16); 4540 ((temp_reg & 0xffff0000)) | ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
4541 } 4541 }
4542 j++; 4542 j++;
4543
4543 if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) 4544 if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4544 return -EINVAL; 4545 return -EINVAL;
4545
4546 temp_reg = RREG32(mmMC_PMG_CMD_MRS); 4546 temp_reg = RREG32(mmMC_PMG_CMD_MRS);
4547 table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS; 4547 table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS;
4548 table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP; 4548 table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP;
@@ -4553,10 +4553,10 @@ static int ci_set_mc_special_registers(struct amdgpu_device *adev,
4553 table->mc_reg_table_entry[k].mc_data[j] |= 0x100; 4553 table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
4554 } 4554 }
4555 j++; 4555 j++;
4556 if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4557 return -EINVAL;
4558 4556
4559 if (adev->mc.vram_type != AMDGPU_VRAM_TYPE_GDDR5) { 4557 if (adev->mc.vram_type != AMDGPU_VRAM_TYPE_GDDR5) {
4558 if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4559 return -EINVAL;
4560 table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD; 4560 table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD;
4561 table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD; 4561 table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD;
4562 for (k = 0; k < table->num_entries; k++) { 4562 for (k = 0; k < table->num_entries; k++) {
@@ -4564,8 +4564,6 @@ static int ci_set_mc_special_registers(struct amdgpu_device *adev,
4564 (table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16; 4564 (table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
4565 } 4565 }
4566 j++; 4566 j++;
4567 if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4568 return -EINVAL;
4569 } 4567 }
4570 break; 4568 break;
4571 case mmMC_SEQ_RESERVE_M: 4569 case mmMC_SEQ_RESERVE_M:
@@ -4577,8 +4575,6 @@ static int ci_set_mc_special_registers(struct amdgpu_device *adev,
4577 (temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); 4575 (temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
4578 } 4576 }
4579 j++; 4577 j++;
4580 if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4581 return -EINVAL;
4582 break; 4578 break;
4583 default: 4579 default:
4584 break; 4580 break;
@@ -6625,9 +6621,9 @@ static int ci_dpm_print_clock_levels(void *handle,
6625 6621
6626 for (i = 0; i < pcie_table->count; i++) 6622 for (i = 0; i < pcie_table->count; i++)
6627 size += sprintf(buf + size, "%d: %s %s\n", i, 6623 size += sprintf(buf + size, "%d: %s %s\n", i,
6628 (pcie_table->dpm_levels[i].value == 0) ? "2.5GB, x1" : 6624 (pcie_table->dpm_levels[i].value == 0) ? "2.5GT/s, x1" :
6629 (pcie_table->dpm_levels[i].value == 1) ? "5.0GB, x16" : 6625 (pcie_table->dpm_levels[i].value == 1) ? "5.0GT/s, x16" :
6630 (pcie_table->dpm_levels[i].value == 2) ? "8.0GB, x16" : "", 6626 (pcie_table->dpm_levels[i].value == 2) ? "8.0GT/s, x16" : "",
6631 (i == now) ? "*" : ""); 6627 (i == now) ? "*" : "");
6632 break; 6628 break;
6633 default: 6629 default:
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index a296f7bbe57c..8ba056a2a5da 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -757,72 +757,72 @@ static void cik_init_golden_registers(struct amdgpu_device *adev)
757 case CHIP_BONAIRE: 757 case CHIP_BONAIRE:
758 amdgpu_program_register_sequence(adev, 758 amdgpu_program_register_sequence(adev,
759 bonaire_mgcg_cgcg_init, 759 bonaire_mgcg_cgcg_init,
760 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init)); 760 ARRAY_SIZE(bonaire_mgcg_cgcg_init));
761 amdgpu_program_register_sequence(adev, 761 amdgpu_program_register_sequence(adev,
762 bonaire_golden_registers, 762 bonaire_golden_registers,
763 (const u32)ARRAY_SIZE(bonaire_golden_registers)); 763 ARRAY_SIZE(bonaire_golden_registers));
764 amdgpu_program_register_sequence(adev, 764 amdgpu_program_register_sequence(adev,
765 bonaire_golden_common_registers, 765 bonaire_golden_common_registers,
766 (const u32)ARRAY_SIZE(bonaire_golden_common_registers)); 766 ARRAY_SIZE(bonaire_golden_common_registers));
767 amdgpu_program_register_sequence(adev, 767 amdgpu_program_register_sequence(adev,
768 bonaire_golden_spm_registers, 768 bonaire_golden_spm_registers,
769 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers)); 769 ARRAY_SIZE(bonaire_golden_spm_registers));
770 break; 770 break;
771 case CHIP_KABINI: 771 case CHIP_KABINI:
772 amdgpu_program_register_sequence(adev, 772 amdgpu_program_register_sequence(adev,
773 kalindi_mgcg_cgcg_init, 773 kalindi_mgcg_cgcg_init,
774 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 774 ARRAY_SIZE(kalindi_mgcg_cgcg_init));
775 amdgpu_program_register_sequence(adev, 775 amdgpu_program_register_sequence(adev,
776 kalindi_golden_registers, 776 kalindi_golden_registers,
777 (const u32)ARRAY_SIZE(kalindi_golden_registers)); 777 ARRAY_SIZE(kalindi_golden_registers));
778 amdgpu_program_register_sequence(adev, 778 amdgpu_program_register_sequence(adev,
779 kalindi_golden_common_registers, 779 kalindi_golden_common_registers,
780 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 780 ARRAY_SIZE(kalindi_golden_common_registers));
781 amdgpu_program_register_sequence(adev, 781 amdgpu_program_register_sequence(adev,
782 kalindi_golden_spm_registers, 782 kalindi_golden_spm_registers,
783 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 783 ARRAY_SIZE(kalindi_golden_spm_registers));
784 break; 784 break;
785 case CHIP_MULLINS: 785 case CHIP_MULLINS:
786 amdgpu_program_register_sequence(adev, 786 amdgpu_program_register_sequence(adev,
787 kalindi_mgcg_cgcg_init, 787 kalindi_mgcg_cgcg_init,
788 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 788 ARRAY_SIZE(kalindi_mgcg_cgcg_init));
789 amdgpu_program_register_sequence(adev, 789 amdgpu_program_register_sequence(adev,
790 godavari_golden_registers, 790 godavari_golden_registers,
791 (const u32)ARRAY_SIZE(godavari_golden_registers)); 791 ARRAY_SIZE(godavari_golden_registers));
792 amdgpu_program_register_sequence(adev, 792 amdgpu_program_register_sequence(adev,
793 kalindi_golden_common_registers, 793 kalindi_golden_common_registers,
794 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 794 ARRAY_SIZE(kalindi_golden_common_registers));
795 amdgpu_program_register_sequence(adev, 795 amdgpu_program_register_sequence(adev,
796 kalindi_golden_spm_registers, 796 kalindi_golden_spm_registers,
797 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 797 ARRAY_SIZE(kalindi_golden_spm_registers));
798 break; 798 break;
799 case CHIP_KAVERI: 799 case CHIP_KAVERI:
800 amdgpu_program_register_sequence(adev, 800 amdgpu_program_register_sequence(adev,
801 spectre_mgcg_cgcg_init, 801 spectre_mgcg_cgcg_init,
802 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init)); 802 ARRAY_SIZE(spectre_mgcg_cgcg_init));
803 amdgpu_program_register_sequence(adev, 803 amdgpu_program_register_sequence(adev,
804 spectre_golden_registers, 804 spectre_golden_registers,
805 (const u32)ARRAY_SIZE(spectre_golden_registers)); 805 ARRAY_SIZE(spectre_golden_registers));
806 amdgpu_program_register_sequence(adev, 806 amdgpu_program_register_sequence(adev,
807 spectre_golden_common_registers, 807 spectre_golden_common_registers,
808 (const u32)ARRAY_SIZE(spectre_golden_common_registers)); 808 ARRAY_SIZE(spectre_golden_common_registers));
809 amdgpu_program_register_sequence(adev, 809 amdgpu_program_register_sequence(adev,
810 spectre_golden_spm_registers, 810 spectre_golden_spm_registers,
811 (const u32)ARRAY_SIZE(spectre_golden_spm_registers)); 811 ARRAY_SIZE(spectre_golden_spm_registers));
812 break; 812 break;
813 case CHIP_HAWAII: 813 case CHIP_HAWAII:
814 amdgpu_program_register_sequence(adev, 814 amdgpu_program_register_sequence(adev,
815 hawaii_mgcg_cgcg_init, 815 hawaii_mgcg_cgcg_init,
816 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init)); 816 ARRAY_SIZE(hawaii_mgcg_cgcg_init));
817 amdgpu_program_register_sequence(adev, 817 amdgpu_program_register_sequence(adev,
818 hawaii_golden_registers, 818 hawaii_golden_registers,
819 (const u32)ARRAY_SIZE(hawaii_golden_registers)); 819 ARRAY_SIZE(hawaii_golden_registers));
820 amdgpu_program_register_sequence(adev, 820 amdgpu_program_register_sequence(adev,
821 hawaii_golden_common_registers, 821 hawaii_golden_common_registers,
822 (const u32)ARRAY_SIZE(hawaii_golden_common_registers)); 822 ARRAY_SIZE(hawaii_golden_common_registers));
823 amdgpu_program_register_sequence(adev, 823 amdgpu_program_register_sequence(adev,
824 hawaii_golden_spm_registers, 824 hawaii_golden_spm_registers,
825 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers)); 825 ARRAY_SIZE(hawaii_golden_spm_registers));
826 break; 826 break;
827 default: 827 default:
828 break; 828 break;
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index 60cecd117705..ed26dcbc4f79 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -657,7 +657,7 @@ static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)
657 } 657 }
658 658
659 if (i < adev->usec_timeout) { 659 if (i < adev->usec_timeout) {
660 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 660 DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
661 } else { 661 } else {
662 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 662 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
663 ring->idx, tmp); 663 ring->idx, tmp);
@@ -724,7 +724,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
724 } 724 }
725 tmp = le32_to_cpu(adev->wb.wb[index]); 725 tmp = le32_to_cpu(adev->wb.wb[index]);
726 if (tmp == 0xDEADBEEF) { 726 if (tmp == 0xDEADBEEF) {
727 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 727 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
728 r = 0; 728 r = 0;
729 } else { 729 } else {
730 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); 730 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index 4e519dc42916..a397111c2ced 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -147,18 +147,18 @@ static void dce_v10_0_init_golden_registers(struct amdgpu_device *adev)
147 case CHIP_FIJI: 147 case CHIP_FIJI:
148 amdgpu_program_register_sequence(adev, 148 amdgpu_program_register_sequence(adev,
149 fiji_mgcg_cgcg_init, 149 fiji_mgcg_cgcg_init,
150 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init)); 150 ARRAY_SIZE(fiji_mgcg_cgcg_init));
151 amdgpu_program_register_sequence(adev, 151 amdgpu_program_register_sequence(adev,
152 golden_settings_fiji_a10, 152 golden_settings_fiji_a10,
153 (const u32)ARRAY_SIZE(golden_settings_fiji_a10)); 153 ARRAY_SIZE(golden_settings_fiji_a10));
154 break; 154 break;
155 case CHIP_TONGA: 155 case CHIP_TONGA:
156 amdgpu_program_register_sequence(adev, 156 amdgpu_program_register_sequence(adev,
157 tonga_mgcg_cgcg_init, 157 tonga_mgcg_cgcg_init,
158 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init)); 158 ARRAY_SIZE(tonga_mgcg_cgcg_init));
159 amdgpu_program_register_sequence(adev, 159 amdgpu_program_register_sequence(adev,
160 golden_settings_tonga_a11, 160 golden_settings_tonga_a11,
161 (const u32)ARRAY_SIZE(golden_settings_tonga_a11)); 161 ARRAY_SIZE(golden_settings_tonga_a11));
162 break; 162 break;
163 default: 163 default:
164 break; 164 break;
@@ -2773,7 +2773,6 @@ static int dce_v10_0_early_init(void *handle)
2773 adev->audio_endpt_wreg = &dce_v10_0_audio_endpt_wreg; 2773 adev->audio_endpt_wreg = &dce_v10_0_audio_endpt_wreg;
2774 2774
2775 dce_v10_0_set_display_funcs(adev); 2775 dce_v10_0_set_display_funcs(adev);
2776 dce_v10_0_set_irq_funcs(adev);
2777 2776
2778 adev->mode_info.num_crtc = dce_v10_0_get_num_crtc(adev); 2777 adev->mode_info.num_crtc = dce_v10_0_get_num_crtc(adev);
2779 2778
@@ -2788,6 +2787,8 @@ static int dce_v10_0_early_init(void *handle)
2788 return -EINVAL; 2787 return -EINVAL;
2789 } 2788 }
2790 2789
2790 dce_v10_0_set_irq_funcs(adev);
2791
2791 return 0; 2792 return 0;
2792} 2793}
2793 2794
@@ -3635,13 +3636,16 @@ static const struct amdgpu_irq_src_funcs dce_v10_0_hpd_irq_funcs = {
3635 3636
3636static void dce_v10_0_set_irq_funcs(struct amdgpu_device *adev) 3637static void dce_v10_0_set_irq_funcs(struct amdgpu_device *adev)
3637{ 3638{
3638 adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST; 3639 if (adev->mode_info.num_crtc > 0)
3640 adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_VLINE1 + adev->mode_info.num_crtc;
3641 else
3642 adev->crtc_irq.num_types = 0;
3639 adev->crtc_irq.funcs = &dce_v10_0_crtc_irq_funcs; 3643 adev->crtc_irq.funcs = &dce_v10_0_crtc_irq_funcs;
3640 3644
3641 adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST; 3645 adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
3642 adev->pageflip_irq.funcs = &dce_v10_0_pageflip_irq_funcs; 3646 adev->pageflip_irq.funcs = &dce_v10_0_pageflip_irq_funcs;
3643 3647
3644 adev->hpd_irq.num_types = AMDGPU_HPD_LAST; 3648 adev->hpd_irq.num_types = adev->mode_info.num_hpd;
3645 adev->hpd_irq.funcs = &dce_v10_0_hpd_irq_funcs; 3649 adev->hpd_irq.funcs = &dce_v10_0_hpd_irq_funcs;
3646} 3650}
3647 3651
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index 11edc75edaa9..67e670989e81 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -156,26 +156,26 @@ static void dce_v11_0_init_golden_registers(struct amdgpu_device *adev)
156 case CHIP_CARRIZO: 156 case CHIP_CARRIZO:
157 amdgpu_program_register_sequence(adev, 157 amdgpu_program_register_sequence(adev,
158 cz_mgcg_cgcg_init, 158 cz_mgcg_cgcg_init,
159 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init)); 159 ARRAY_SIZE(cz_mgcg_cgcg_init));
160 amdgpu_program_register_sequence(adev, 160 amdgpu_program_register_sequence(adev,
161 cz_golden_settings_a11, 161 cz_golden_settings_a11,
162 (const u32)ARRAY_SIZE(cz_golden_settings_a11)); 162 ARRAY_SIZE(cz_golden_settings_a11));
163 break; 163 break;
164 case CHIP_STONEY: 164 case CHIP_STONEY:
165 amdgpu_program_register_sequence(adev, 165 amdgpu_program_register_sequence(adev,
166 stoney_golden_settings_a11, 166 stoney_golden_settings_a11,
167 (const u32)ARRAY_SIZE(stoney_golden_settings_a11)); 167 ARRAY_SIZE(stoney_golden_settings_a11));
168 break; 168 break;
169 case CHIP_POLARIS11: 169 case CHIP_POLARIS11:
170 case CHIP_POLARIS12: 170 case CHIP_POLARIS12:
171 amdgpu_program_register_sequence(adev, 171 amdgpu_program_register_sequence(adev,
172 polaris11_golden_settings_a11, 172 polaris11_golden_settings_a11,
173 (const u32)ARRAY_SIZE(polaris11_golden_settings_a11)); 173 ARRAY_SIZE(polaris11_golden_settings_a11));
174 break; 174 break;
175 case CHIP_POLARIS10: 175 case CHIP_POLARIS10:
176 amdgpu_program_register_sequence(adev, 176 amdgpu_program_register_sequence(adev,
177 polaris10_golden_settings_a11, 177 polaris10_golden_settings_a11,
178 (const u32)ARRAY_SIZE(polaris10_golden_settings_a11)); 178 ARRAY_SIZE(polaris10_golden_settings_a11));
179 break; 179 break;
180 default: 180 default:
181 break; 181 break;
@@ -2876,7 +2876,6 @@ static int dce_v11_0_early_init(void *handle)
2876 adev->audio_endpt_wreg = &dce_v11_0_audio_endpt_wreg; 2876 adev->audio_endpt_wreg = &dce_v11_0_audio_endpt_wreg;
2877 2877
2878 dce_v11_0_set_display_funcs(adev); 2878 dce_v11_0_set_display_funcs(adev);
2879 dce_v11_0_set_irq_funcs(adev);
2880 2879
2881 adev->mode_info.num_crtc = dce_v11_0_get_num_crtc(adev); 2880 adev->mode_info.num_crtc = dce_v11_0_get_num_crtc(adev);
2882 2881
@@ -2903,6 +2902,8 @@ static int dce_v11_0_early_init(void *handle)
2903 return -EINVAL; 2902 return -EINVAL;
2904 } 2903 }
2905 2904
2905 dce_v11_0_set_irq_funcs(adev);
2906
2906 return 0; 2907 return 0;
2907} 2908}
2908 2909
@@ -3759,13 +3760,16 @@ static const struct amdgpu_irq_src_funcs dce_v11_0_hpd_irq_funcs = {
3759 3760
3760static void dce_v11_0_set_irq_funcs(struct amdgpu_device *adev) 3761static void dce_v11_0_set_irq_funcs(struct amdgpu_device *adev)
3761{ 3762{
3762 adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST; 3763 if (adev->mode_info.num_crtc > 0)
3764 adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_VLINE1 + adev->mode_info.num_crtc;
3765 else
3766 adev->crtc_irq.num_types = 0;
3763 adev->crtc_irq.funcs = &dce_v11_0_crtc_irq_funcs; 3767 adev->crtc_irq.funcs = &dce_v11_0_crtc_irq_funcs;
3764 3768
3765 adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST; 3769 adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
3766 adev->pageflip_irq.funcs = &dce_v11_0_pageflip_irq_funcs; 3770 adev->pageflip_irq.funcs = &dce_v11_0_pageflip_irq_funcs;
3767 3771
3768 adev->hpd_irq.num_types = AMDGPU_HPD_LAST; 3772 adev->hpd_irq.num_types = adev->mode_info.num_hpd;
3769 adev->hpd_irq.funcs = &dce_v11_0_hpd_irq_funcs; 3773 adev->hpd_irq.funcs = &dce_v11_0_hpd_irq_funcs;
3770} 3774}
3771 3775
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index a51e35f824a1..bd2c4f727df6 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -2639,7 +2639,6 @@ static int dce_v6_0_early_init(void *handle)
2639 adev->audio_endpt_wreg = &dce_v6_0_audio_endpt_wreg; 2639 adev->audio_endpt_wreg = &dce_v6_0_audio_endpt_wreg;
2640 2640
2641 dce_v6_0_set_display_funcs(adev); 2641 dce_v6_0_set_display_funcs(adev);
2642 dce_v6_0_set_irq_funcs(adev);
2643 2642
2644 adev->mode_info.num_crtc = dce_v6_0_get_num_crtc(adev); 2643 adev->mode_info.num_crtc = dce_v6_0_get_num_crtc(adev);
2645 2644
@@ -2658,6 +2657,8 @@ static int dce_v6_0_early_init(void *handle)
2658 return -EINVAL; 2657 return -EINVAL;
2659 } 2658 }
2660 2659
2660 dce_v6_0_set_irq_funcs(adev);
2661
2661 return 0; 2662 return 0;
2662} 2663}
2663 2664
@@ -3441,13 +3442,16 @@ static const struct amdgpu_irq_src_funcs dce_v6_0_hpd_irq_funcs = {
3441 3442
3442static void dce_v6_0_set_irq_funcs(struct amdgpu_device *adev) 3443static void dce_v6_0_set_irq_funcs(struct amdgpu_device *adev)
3443{ 3444{
3444 adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST; 3445 if (adev->mode_info.num_crtc > 0)
3446 adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_VLINE1 + adev->mode_info.num_crtc;
3447 else
3448 adev->crtc_irq.num_types = 0;
3445 adev->crtc_irq.funcs = &dce_v6_0_crtc_irq_funcs; 3449 adev->crtc_irq.funcs = &dce_v6_0_crtc_irq_funcs;
3446 3450
3447 adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST; 3451 adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
3448 adev->pageflip_irq.funcs = &dce_v6_0_pageflip_irq_funcs; 3452 adev->pageflip_irq.funcs = &dce_v6_0_pageflip_irq_funcs;
3449 3453
3450 adev->hpd_irq.num_types = AMDGPU_HPD_LAST; 3454 adev->hpd_irq.num_types = adev->mode_info.num_hpd;
3451 adev->hpd_irq.funcs = &dce_v6_0_hpd_irq_funcs; 3455 adev->hpd_irq.funcs = &dce_v6_0_hpd_irq_funcs;
3452} 3456}
3453 3457
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 9cf14b8b2db9..c008dc030687 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -2664,7 +2664,6 @@ static int dce_v8_0_early_init(void *handle)
2664 adev->audio_endpt_wreg = &dce_v8_0_audio_endpt_wreg; 2664 adev->audio_endpt_wreg = &dce_v8_0_audio_endpt_wreg;
2665 2665
2666 dce_v8_0_set_display_funcs(adev); 2666 dce_v8_0_set_display_funcs(adev);
2667 dce_v8_0_set_irq_funcs(adev);
2668 2667
2669 adev->mode_info.num_crtc = dce_v8_0_get_num_crtc(adev); 2668 adev->mode_info.num_crtc = dce_v8_0_get_num_crtc(adev);
2670 2669
@@ -2688,6 +2687,8 @@ static int dce_v8_0_early_init(void *handle)
2688 return -EINVAL; 2687 return -EINVAL;
2689 } 2688 }
2690 2689
2690 dce_v8_0_set_irq_funcs(adev);
2691
2691 return 0; 2692 return 0;
2692} 2693}
2693 2694
@@ -3525,13 +3526,16 @@ static const struct amdgpu_irq_src_funcs dce_v8_0_hpd_irq_funcs = {
3525 3526
3526static void dce_v8_0_set_irq_funcs(struct amdgpu_device *adev) 3527static void dce_v8_0_set_irq_funcs(struct amdgpu_device *adev)
3527{ 3528{
3528 adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST; 3529 if (adev->mode_info.num_crtc > 0)
3530 adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_VLINE1 + adev->mode_info.num_crtc;
3531 else
3532 adev->crtc_irq.num_types = 0;
3529 adev->crtc_irq.funcs = &dce_v8_0_crtc_irq_funcs; 3533 adev->crtc_irq.funcs = &dce_v8_0_crtc_irq_funcs;
3530 3534
3531 adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST; 3535 adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
3532 adev->pageflip_irq.funcs = &dce_v8_0_pageflip_irq_funcs; 3536 adev->pageflip_irq.funcs = &dce_v8_0_pageflip_irq_funcs;
3533 3537
3534 adev->hpd_irq.num_types = AMDGPU_HPD_LAST; 3538 adev->hpd_irq.num_types = adev->mode_info.num_hpd;
3535 adev->hpd_irq.funcs = &dce_v8_0_hpd_irq_funcs; 3539 adev->hpd_irq.funcs = &dce_v8_0_hpd_irq_funcs;
3536} 3540}
3537 3541
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
index a8829af120c1..120dd3b26fc2 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -44,6 +44,9 @@ static void dce_virtual_set_display_funcs(struct amdgpu_device *adev);
44static void dce_virtual_set_irq_funcs(struct amdgpu_device *adev); 44static void dce_virtual_set_irq_funcs(struct amdgpu_device *adev);
45static int dce_virtual_connector_encoder_init(struct amdgpu_device *adev, 45static int dce_virtual_connector_encoder_init(struct amdgpu_device *adev,
46 int index); 46 int index);
47static void dce_virtual_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
48 int crtc,
49 enum amdgpu_interrupt_state state);
47 50
48/** 51/**
49 * dce_virtual_vblank_wait - vblank wait asic callback. 52 * dce_virtual_vblank_wait - vblank wait asic callback.
@@ -437,6 +440,8 @@ static int dce_virtual_sw_fini(void *handle)
437 drm_kms_helper_poll_fini(adev->ddev); 440 drm_kms_helper_poll_fini(adev->ddev);
438 441
439 drm_mode_config_cleanup(adev->ddev); 442 drm_mode_config_cleanup(adev->ddev);
443 /* clear crtcs pointer to avoid dce irq finish routine access freed data */
444 memset(adev->mode_info.crtcs, 0, sizeof(adev->mode_info.crtcs[0]) * AMDGPU_MAX_CRTCS);
440 adev->mode_info.mode_config_initialized = false; 445 adev->mode_info.mode_config_initialized = false;
441 return 0; 446 return 0;
442} 447}
@@ -489,6 +494,13 @@ static int dce_virtual_hw_init(void *handle)
489 494
490static int dce_virtual_hw_fini(void *handle) 495static int dce_virtual_hw_fini(void *handle)
491{ 496{
497 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
498 int i = 0;
499
500 for (i = 0; i<adev->mode_info.num_crtc; i++)
501 if (adev->mode_info.crtcs[i])
502 dce_virtual_set_crtc_vblank_interrupt_state(adev, i, AMDGPU_IRQ_STATE_DISABLE);
503
492 return 0; 504 return 0;
493} 505}
494 506
@@ -723,7 +735,7 @@ static void dce_virtual_set_crtc_vblank_interrupt_state(struct amdgpu_device *ad
723 int crtc, 735 int crtc,
724 enum amdgpu_interrupt_state state) 736 enum amdgpu_interrupt_state state)
725{ 737{
726 if (crtc >= adev->mode_info.num_crtc) { 738 if (crtc >= adev->mode_info.num_crtc || !adev->mode_info.crtcs[crtc]) {
727 DRM_DEBUG("invalid crtc %d\n", crtc); 739 DRM_DEBUG("invalid crtc %d\n", crtc);
728 return; 740 return;
729 } 741 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index dbbe986f90f2..edef17d93527 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -1798,7 +1798,7 @@ static int gfx_v6_0_ring_test_ring(struct amdgpu_ring *ring)
1798 DRM_UDELAY(1); 1798 DRM_UDELAY(1);
1799 } 1799 }
1800 if (i < adev->usec_timeout) { 1800 if (i < adev->usec_timeout) {
1801 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 1801 DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
1802 } else { 1802 } else {
1803 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n", 1803 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
1804 ring->idx, scratch, tmp); 1804 ring->idx, scratch, tmp);
@@ -1951,7 +1951,7 @@ static int gfx_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1951 } 1951 }
1952 tmp = RREG32(scratch); 1952 tmp = RREG32(scratch);
1953 if (tmp == 0xDEADBEEF) { 1953 if (tmp == 0xDEADBEEF) {
1954 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 1954 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
1955 r = 0; 1955 r = 0;
1956 } else { 1956 } else {
1957 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n", 1957 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
@@ -2962,25 +2962,7 @@ static void gfx_v6_0_get_csb_buffer(struct amdgpu_device *adev,
2962 2962
2963 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 2963 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
2964 buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 2964 buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
2965 2965 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config);
2966 switch (adev->asic_type) {
2967 case CHIP_TAHITI:
2968 case CHIP_PITCAIRN:
2969 buffer[count++] = cpu_to_le32(0x2a00126a);
2970 break;
2971 case CHIP_VERDE:
2972 buffer[count++] = cpu_to_le32(0x0000124a);
2973 break;
2974 case CHIP_OLAND:
2975 buffer[count++] = cpu_to_le32(0x00000082);
2976 break;
2977 case CHIP_HAINAN:
2978 buffer[count++] = cpu_to_le32(0x00000000);
2979 break;
2980 default:
2981 buffer[count++] = cpu_to_le32(0x00000000);
2982 break;
2983 }
2984 2966
2985 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 2967 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2986 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 2968 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 419ba0ce7ee5..83d94c23aa78 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -2085,7 +2085,7 @@ static int gfx_v7_0_ring_test_ring(struct amdgpu_ring *ring)
2085 DRM_UDELAY(1); 2085 DRM_UDELAY(1);
2086 } 2086 }
2087 if (i < adev->usec_timeout) { 2087 if (i < adev->usec_timeout) {
2088 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 2088 DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2089 } else { 2089 } else {
2090 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n", 2090 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2091 ring->idx, scratch, tmp); 2091 ring->idx, scratch, tmp);
@@ -2365,7 +2365,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
2365 } 2365 }
2366 tmp = RREG32(scratch); 2366 tmp = RREG32(scratch);
2367 if (tmp == 0xDEADBEEF) { 2367 if (tmp == 0xDEADBEEF) {
2368 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 2368 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
2369 r = 0; 2369 r = 0;
2370 } else { 2370 } else {
2371 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n", 2371 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
@@ -2551,29 +2551,8 @@ static int gfx_v7_0_cp_gfx_start(struct amdgpu_device *adev)
2551 2551
2552 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 2552 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2553 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 2553 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
2554 switch (adev->asic_type) { 2554 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);
2555 case CHIP_BONAIRE: 2555 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);
2556 amdgpu_ring_write(ring, 0x16000012);
2557 amdgpu_ring_write(ring, 0x00000000);
2558 break;
2559 case CHIP_KAVERI:
2560 amdgpu_ring_write(ring, 0x00000000); /* XXX */
2561 amdgpu_ring_write(ring, 0x00000000);
2562 break;
2563 case CHIP_KABINI:
2564 case CHIP_MULLINS:
2565 amdgpu_ring_write(ring, 0x00000000); /* XXX */
2566 amdgpu_ring_write(ring, 0x00000000);
2567 break;
2568 case CHIP_HAWAII:
2569 amdgpu_ring_write(ring, 0x3a00161a);
2570 amdgpu_ring_write(ring, 0x0000002e);
2571 break;
2572 default:
2573 amdgpu_ring_write(ring, 0x00000000);
2574 amdgpu_ring_write(ring, 0x00000000);
2575 break;
2576 }
2577 2556
2578 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 2557 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2579 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 2558 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 9ecdf621a74a..d02493cf9175 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -681,53 +681,53 @@ static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
681 case CHIP_TOPAZ: 681 case CHIP_TOPAZ:
682 amdgpu_program_register_sequence(adev, 682 amdgpu_program_register_sequence(adev,
683 iceland_mgcg_cgcg_init, 683 iceland_mgcg_cgcg_init,
684 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init)); 684 ARRAY_SIZE(iceland_mgcg_cgcg_init));
685 amdgpu_program_register_sequence(adev, 685 amdgpu_program_register_sequence(adev,
686 golden_settings_iceland_a11, 686 golden_settings_iceland_a11,
687 (const u32)ARRAY_SIZE(golden_settings_iceland_a11)); 687 ARRAY_SIZE(golden_settings_iceland_a11));
688 amdgpu_program_register_sequence(adev, 688 amdgpu_program_register_sequence(adev,
689 iceland_golden_common_all, 689 iceland_golden_common_all,
690 (const u32)ARRAY_SIZE(iceland_golden_common_all)); 690 ARRAY_SIZE(iceland_golden_common_all));
691 break; 691 break;
692 case CHIP_FIJI: 692 case CHIP_FIJI:
693 amdgpu_program_register_sequence(adev, 693 amdgpu_program_register_sequence(adev,
694 fiji_mgcg_cgcg_init, 694 fiji_mgcg_cgcg_init,
695 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init)); 695 ARRAY_SIZE(fiji_mgcg_cgcg_init));
696 amdgpu_program_register_sequence(adev, 696 amdgpu_program_register_sequence(adev,
697 golden_settings_fiji_a10, 697 golden_settings_fiji_a10,
698 (const u32)ARRAY_SIZE(golden_settings_fiji_a10)); 698 ARRAY_SIZE(golden_settings_fiji_a10));
699 amdgpu_program_register_sequence(adev, 699 amdgpu_program_register_sequence(adev,
700 fiji_golden_common_all, 700 fiji_golden_common_all,
701 (const u32)ARRAY_SIZE(fiji_golden_common_all)); 701 ARRAY_SIZE(fiji_golden_common_all));
702 break; 702 break;
703 703
704 case CHIP_TONGA: 704 case CHIP_TONGA:
705 amdgpu_program_register_sequence(adev, 705 amdgpu_program_register_sequence(adev,
706 tonga_mgcg_cgcg_init, 706 tonga_mgcg_cgcg_init,
707 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init)); 707 ARRAY_SIZE(tonga_mgcg_cgcg_init));
708 amdgpu_program_register_sequence(adev, 708 amdgpu_program_register_sequence(adev,
709 golden_settings_tonga_a11, 709 golden_settings_tonga_a11,
710 (const u32)ARRAY_SIZE(golden_settings_tonga_a11)); 710 ARRAY_SIZE(golden_settings_tonga_a11));
711 amdgpu_program_register_sequence(adev, 711 amdgpu_program_register_sequence(adev,
712 tonga_golden_common_all, 712 tonga_golden_common_all,
713 (const u32)ARRAY_SIZE(tonga_golden_common_all)); 713 ARRAY_SIZE(tonga_golden_common_all));
714 break; 714 break;
715 case CHIP_POLARIS11: 715 case CHIP_POLARIS11:
716 case CHIP_POLARIS12: 716 case CHIP_POLARIS12:
717 amdgpu_program_register_sequence(adev, 717 amdgpu_program_register_sequence(adev,
718 golden_settings_polaris11_a11, 718 golden_settings_polaris11_a11,
719 (const u32)ARRAY_SIZE(golden_settings_polaris11_a11)); 719 ARRAY_SIZE(golden_settings_polaris11_a11));
720 amdgpu_program_register_sequence(adev, 720 amdgpu_program_register_sequence(adev,
721 polaris11_golden_common_all, 721 polaris11_golden_common_all,
722 (const u32)ARRAY_SIZE(polaris11_golden_common_all)); 722 ARRAY_SIZE(polaris11_golden_common_all));
723 break; 723 break;
724 case CHIP_POLARIS10: 724 case CHIP_POLARIS10:
725 amdgpu_program_register_sequence(adev, 725 amdgpu_program_register_sequence(adev,
726 golden_settings_polaris10_a11, 726 golden_settings_polaris10_a11,
727 (const u32)ARRAY_SIZE(golden_settings_polaris10_a11)); 727 ARRAY_SIZE(golden_settings_polaris10_a11));
728 amdgpu_program_register_sequence(adev, 728 amdgpu_program_register_sequence(adev,
729 polaris10_golden_common_all, 729 polaris10_golden_common_all,
730 (const u32)ARRAY_SIZE(polaris10_golden_common_all)); 730 ARRAY_SIZE(polaris10_golden_common_all));
731 WREG32_SMC(ixCG_ACLK_CNTL, 0x0000001C); 731 WREG32_SMC(ixCG_ACLK_CNTL, 0x0000001C);
732 if (adev->pdev->revision == 0xc7 && 732 if (adev->pdev->revision == 0xc7 &&
733 ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) || 733 ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) ||
@@ -740,24 +740,24 @@ static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
740 case CHIP_CARRIZO: 740 case CHIP_CARRIZO:
741 amdgpu_program_register_sequence(adev, 741 amdgpu_program_register_sequence(adev,
742 cz_mgcg_cgcg_init, 742 cz_mgcg_cgcg_init,
743 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init)); 743 ARRAY_SIZE(cz_mgcg_cgcg_init));
744 amdgpu_program_register_sequence(adev, 744 amdgpu_program_register_sequence(adev,
745 cz_golden_settings_a11, 745 cz_golden_settings_a11,
746 (const u32)ARRAY_SIZE(cz_golden_settings_a11)); 746 ARRAY_SIZE(cz_golden_settings_a11));
747 amdgpu_program_register_sequence(adev, 747 amdgpu_program_register_sequence(adev,
748 cz_golden_common_all, 748 cz_golden_common_all,
749 (const u32)ARRAY_SIZE(cz_golden_common_all)); 749 ARRAY_SIZE(cz_golden_common_all));
750 break; 750 break;
751 case CHIP_STONEY: 751 case CHIP_STONEY:
752 amdgpu_program_register_sequence(adev, 752 amdgpu_program_register_sequence(adev,
753 stoney_mgcg_cgcg_init, 753 stoney_mgcg_cgcg_init,
754 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init)); 754 ARRAY_SIZE(stoney_mgcg_cgcg_init));
755 amdgpu_program_register_sequence(adev, 755 amdgpu_program_register_sequence(adev,
756 stoney_golden_settings_a11, 756 stoney_golden_settings_a11,
757 (const u32)ARRAY_SIZE(stoney_golden_settings_a11)); 757 ARRAY_SIZE(stoney_golden_settings_a11));
758 amdgpu_program_register_sequence(adev, 758 amdgpu_program_register_sequence(adev,
759 stoney_golden_common_all, 759 stoney_golden_common_all,
760 (const u32)ARRAY_SIZE(stoney_golden_common_all)); 760 ARRAY_SIZE(stoney_golden_common_all));
761 break; 761 break;
762 default: 762 default:
763 break; 763 break;
@@ -804,7 +804,7 @@ static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
804 DRM_UDELAY(1); 804 DRM_UDELAY(1);
805 } 805 }
806 if (i < adev->usec_timeout) { 806 if (i < adev->usec_timeout) {
807 DRM_INFO("ring test on %d succeeded in %d usecs\n", 807 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
808 ring->idx, i); 808 ring->idx, i);
809 } else { 809 } else {
810 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n", 810 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
@@ -856,7 +856,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
856 } 856 }
857 tmp = RREG32(scratch); 857 tmp = RREG32(scratch);
858 if (tmp == 0xDEADBEEF) { 858 if (tmp == 0xDEADBEEF) {
859 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 859 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
860 r = 0; 860 r = 0;
861 } else { 861 } else {
862 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n", 862 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
@@ -2114,7 +2114,6 @@ static int gfx_v8_0_sw_fini(void *handle)
2114 amdgpu_gfx_compute_mqd_sw_fini(adev); 2114 amdgpu_gfx_compute_mqd_sw_fini(adev);
2115 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq); 2115 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
2116 amdgpu_gfx_kiq_fini(adev); 2116 amdgpu_gfx_kiq_fini(adev);
2117 amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
2118 2117
2119 gfx_v8_0_mec_fini(adev); 2118 gfx_v8_0_mec_fini(adev);
2120 gfx_v8_0_rlc_fini(adev); 2119 gfx_v8_0_rlc_fini(adev);
@@ -3851,6 +3850,14 @@ static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
3851 break; 3850 break;
3852 udelay(1); 3851 udelay(1);
3853 } 3852 }
3853 if (k == adev->usec_timeout) {
3854 gfx_v8_0_select_se_sh(adev, 0xffffffff,
3855 0xffffffff, 0xffffffff);
3856 mutex_unlock(&adev->grbm_idx_mutex);
3857 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
3858 i, j);
3859 return;
3860 }
3854 } 3861 }
3855 } 3862 }
3856 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 3863 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
@@ -4305,37 +4312,8 @@ static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev)
4305 4312
4306 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 4313 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4307 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 4314 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
4308 switch (adev->asic_type) { 4315 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);
4309 case CHIP_TONGA: 4316 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);
4310 case CHIP_POLARIS10:
4311 amdgpu_ring_write(ring, 0x16000012);
4312 amdgpu_ring_write(ring, 0x0000002A);
4313 break;
4314 case CHIP_POLARIS11:
4315 case CHIP_POLARIS12:
4316 amdgpu_ring_write(ring, 0x16000012);
4317 amdgpu_ring_write(ring, 0x00000000);
4318 break;
4319 case CHIP_FIJI:
4320 amdgpu_ring_write(ring, 0x3a00161a);
4321 amdgpu_ring_write(ring, 0x0000002e);
4322 break;
4323 case CHIP_CARRIZO:
4324 amdgpu_ring_write(ring, 0x00000002);
4325 amdgpu_ring_write(ring, 0x00000000);
4326 break;
4327 case CHIP_TOPAZ:
4328 amdgpu_ring_write(ring, adev->gfx.config.num_rbs == 1 ?
4329 0x00000000 : 0x00000002);
4330 amdgpu_ring_write(ring, 0x00000000);
4331 break;
4332 case CHIP_STONEY:
4333 amdgpu_ring_write(ring, 0x00000000);
4334 amdgpu_ring_write(ring, 0x00000000);
4335 break;
4336 default:
4337 BUG();
4338 }
4339 4317
4340 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4318 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4341 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 4319 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
@@ -4816,7 +4794,7 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
4816 4794
4817 gfx_v8_0_kiq_setting(ring); 4795 gfx_v8_0_kiq_setting(ring);
4818 4796
4819 if (adev->in_sriov_reset) { /* for GPU_RESET case */ 4797 if (adev->in_gpu_reset) { /* for GPU_RESET case */
4820 /* reset MQD to a clean status */ 4798 /* reset MQD to a clean status */
4821 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4799 if (adev->gfx.mec.mqd_backup[mqd_idx])
4822 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); 4800 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
@@ -4853,7 +4831,7 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
4853 struct vi_mqd *mqd = ring->mqd_ptr; 4831 struct vi_mqd *mqd = ring->mqd_ptr;
4854 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 4832 int mqd_idx = ring - &adev->gfx.compute_ring[0];
4855 4833
4856 if (!adev->in_sriov_reset && !adev->gfx.in_suspend) { 4834 if (!adev->in_gpu_reset && !adev->gfx.in_suspend) {
4857 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); 4835 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
4858 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 4836 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
4859 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 4837 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
@@ -4865,13 +4843,10 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
4865 4843
4866 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4844 if (adev->gfx.mec.mqd_backup[mqd_idx])
4867 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation)); 4845 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
4868 } else if (adev->in_sriov_reset) { /* for GPU_RESET case */ 4846 } else if (adev->in_gpu_reset) { /* for GPU_RESET case */
4869 /* reset MQD to a clean status */ 4847 /* reset MQD to a clean status */
4870 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4848 if (adev->gfx.mec.mqd_backup[mqd_idx])
4871 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); 4849 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
4872 /* reset ring buffer */
4873 ring->wptr = 0;
4874 amdgpu_ring_clear_ring(ring);
4875 } else { 4850 } else {
4876 amdgpu_ring_clear_ring(ring); 4851 amdgpu_ring_clear_ring(ring);
4877 } 4852 }
@@ -4946,6 +4921,13 @@ static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev)
4946 /* Test KCQs */ 4921 /* Test KCQs */
4947 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4922 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4948 ring = &adev->gfx.compute_ring[i]; 4923 ring = &adev->gfx.compute_ring[i];
4924 if (adev->in_gpu_reset) {
4925 /* move reset ring buffer to here to workaround
4926 * compute ring test failed
4927 */
4928 ring->wptr = 0;
4929 amdgpu_ring_clear_ring(ring);
4930 }
4949 ring->ready = true; 4931 ring->ready = true;
4950 r = amdgpu_ring_test_ring(ring); 4932 r = amdgpu_ring_test_ring(ring);
4951 if (r) 4933 if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index da43813d67a4..6c5289ae67be 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -28,11 +28,11 @@
28#include "soc15.h" 28#include "soc15.h"
29#include "soc15d.h" 29#include "soc15d.h"
30 30
31#include "vega10/soc15ip.h" 31#include "soc15ip.h"
32#include "vega10/GC/gc_9_0_offset.h" 32#include "gc/gc_9_0_offset.h"
33#include "vega10/GC/gc_9_0_sh_mask.h" 33#include "gc/gc_9_0_sh_mask.h"
34#include "vega10/vega10_enum.h" 34#include "vega10_enum.h"
35#include "vega10/HDP/hdp_4_0_offset.h" 35#include "hdp/hdp_4_0_offset.h"
36 36
37#include "soc15_common.h" 37#include "soc15_common.h"
38#include "clearstate_gfx9.h" 38#include "clearstate_gfx9.h"
@@ -232,18 +232,18 @@ static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
232 case CHIP_VEGA10: 232 case CHIP_VEGA10:
233 amdgpu_program_register_sequence(adev, 233 amdgpu_program_register_sequence(adev,
234 golden_settings_gc_9_0, 234 golden_settings_gc_9_0,
235 (const u32)ARRAY_SIZE(golden_settings_gc_9_0)); 235 ARRAY_SIZE(golden_settings_gc_9_0));
236 amdgpu_program_register_sequence(adev, 236 amdgpu_program_register_sequence(adev,
237 golden_settings_gc_9_0_vg10, 237 golden_settings_gc_9_0_vg10,
238 (const u32)ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 238 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
239 break; 239 break;
240 case CHIP_RAVEN: 240 case CHIP_RAVEN:
241 amdgpu_program_register_sequence(adev, 241 amdgpu_program_register_sequence(adev,
242 golden_settings_gc_9_1, 242 golden_settings_gc_9_1,
243 (const u32)ARRAY_SIZE(golden_settings_gc_9_1)); 243 ARRAY_SIZE(golden_settings_gc_9_1));
244 amdgpu_program_register_sequence(adev, 244 amdgpu_program_register_sequence(adev,
245 golden_settings_gc_9_1_rv1, 245 golden_settings_gc_9_1_rv1,
246 (const u32)ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 246 ARRAY_SIZE(golden_settings_gc_9_1_rv1));
247 break; 247 break;
248 default: 248 default:
249 break; 249 break;
@@ -327,7 +327,7 @@ static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
327 DRM_UDELAY(1); 327 DRM_UDELAY(1);
328 } 328 }
329 if (i < adev->usec_timeout) { 329 if (i < adev->usec_timeout) {
330 DRM_INFO("ring test on %d succeeded in %d usecs\n", 330 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
331 ring->idx, i); 331 ring->idx, i);
332 } else { 332 } else {
333 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n", 333 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
@@ -379,7 +379,7 @@ static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
379 } 379 }
380 tmp = RREG32(scratch); 380 tmp = RREG32(scratch);
381 if (tmp == 0xDEADBEEF) { 381 if (tmp == 0xDEADBEEF) {
382 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 382 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
383 r = 0; 383 r = 0;
384 } else { 384 } else {
385 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n", 385 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
@@ -1464,7 +1464,6 @@ static int gfx_v9_0_sw_fini(void *handle)
1464 amdgpu_gfx_compute_mqd_sw_fini(adev); 1464 amdgpu_gfx_compute_mqd_sw_fini(adev);
1465 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq); 1465 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
1466 amdgpu_gfx_kiq_fini(adev); 1466 amdgpu_gfx_kiq_fini(adev);
1467 amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
1468 1467
1469 gfx_v9_0_mec_fini(adev); 1468 gfx_v9_0_mec_fini(adev);
1470 gfx_v9_0_ngg_fini(adev); 1469 gfx_v9_0_ngg_fini(adev);
@@ -1645,6 +1644,14 @@ static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
1645 break; 1644 break;
1646 udelay(1); 1645 udelay(1);
1647 } 1646 }
1647 if (k == adev->usec_timeout) {
1648 gfx_v9_0_select_se_sh(adev, 0xffffffff,
1649 0xffffffff, 0xffffffff);
1650 mutex_unlock(&adev->grbm_idx_mutex);
1651 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
1652 i, j);
1653 return;
1654 }
1648 } 1655 }
1649 } 1656 }
1650 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1657 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
@@ -2749,7 +2756,7 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
2749 2756
2750 gfx_v9_0_kiq_setting(ring); 2757 gfx_v9_0_kiq_setting(ring);
2751 2758
2752 if (adev->in_sriov_reset) { /* for GPU_RESET case */ 2759 if (adev->in_gpu_reset) { /* for GPU_RESET case */
2753 /* reset MQD to a clean status */ 2760 /* reset MQD to a clean status */
2754 if (adev->gfx.mec.mqd_backup[mqd_idx]) 2761 if (adev->gfx.mec.mqd_backup[mqd_idx])
2755 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 2762 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
@@ -2787,7 +2794,7 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
2787 struct v9_mqd *mqd = ring->mqd_ptr; 2794 struct v9_mqd *mqd = ring->mqd_ptr;
2788 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 2795 int mqd_idx = ring - &adev->gfx.compute_ring[0];
2789 2796
2790 if (!adev->in_sriov_reset && !adev->gfx.in_suspend) { 2797 if (!adev->in_gpu_reset && !adev->gfx.in_suspend) {
2791 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 2798 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
2792 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 2799 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
2793 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 2800 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
@@ -2799,7 +2806,7 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
2799 2806
2800 if (adev->gfx.mec.mqd_backup[mqd_idx]) 2807 if (adev->gfx.mec.mqd_backup[mqd_idx])
2801 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 2808 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
2802 } else if (adev->in_sriov_reset) { /* for GPU_RESET case */ 2809 } else if (adev->in_gpu_reset) { /* for GPU_RESET case */
2803 /* reset MQD to a clean status */ 2810 /* reset MQD to a clean status */
2804 if (adev->gfx.mec.mqd_backup[mqd_idx]) 2811 if (adev->gfx.mec.mqd_backup[mqd_idx])
2805 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 2812 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
index c17996e18086..f1effadfbaa6 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
@@ -23,11 +23,11 @@
23#include "amdgpu.h" 23#include "amdgpu.h"
24#include "gfxhub_v1_0.h" 24#include "gfxhub_v1_0.h"
25 25
26#include "vega10/soc15ip.h" 26#include "soc15ip.h"
27#include "vega10/GC/gc_9_0_offset.h" 27#include "gc/gc_9_0_offset.h"
28#include "vega10/GC/gc_9_0_sh_mask.h" 28#include "gc/gc_9_0_sh_mask.h"
29#include "vega10/GC/gc_9_0_default.h" 29#include "gc/gc_9_0_default.h"
30#include "vega10/vega10_enum.h" 30#include "vega10_enum.h"
31 31
32#include "soc15_common.h" 32#include "soc15_common.h"
33 33
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index f4603a7c8ef3..468281f10e8d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -222,11 +222,6 @@ static void gmc_v6_0_vram_gtt_location(struct amdgpu_device *adev,
222 u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF; 222 u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
223 base <<= 24; 223 base <<= 24;
224 224
225 if (mc->mc_vram_size > 0xFFC0000000ULL) {
226 dev_warn(adev->dev, "limiting VRAM\n");
227 mc->real_vram_size = 0xFFC0000000ULL;
228 mc->mc_vram_size = 0xFFC0000000ULL;
229 }
230 amdgpu_vram_location(adev, &adev->mc, base); 225 amdgpu_vram_location(adev, &adev->mc, base);
231 amdgpu_gart_location(adev, mc); 226 amdgpu_gart_location(adev, mc);
232} 227}
@@ -283,6 +278,7 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
283 278
284 u32 tmp; 279 u32 tmp;
285 int chansize, numchan; 280 int chansize, numchan;
281 int r;
286 282
287 tmp = RREG32(mmMC_ARB_RAMCFG); 283 tmp = RREG32(mmMC_ARB_RAMCFG);
288 if (tmp & (1 << 11)) { 284 if (tmp & (1 << 11)) {
@@ -324,12 +320,17 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
324 break; 320 break;
325 } 321 }
326 adev->mc.vram_width = numchan * chansize; 322 adev->mc.vram_width = numchan * chansize;
327 /* Could aper size report 0 ? */
328 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
329 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
330 /* size in MB on si */ 323 /* size in MB on si */
331 adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; 324 adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
332 adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; 325 adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
326
327 if (!(adev->flags & AMD_IS_APU)) {
328 r = amdgpu_device_resize_fb_bar(adev);
329 if (r)
330 return r;
331 }
332 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
333 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
333 adev->mc.visible_vram_size = adev->mc.aper_size; 334 adev->mc.visible_vram_size = adev->mc.aper_size;
334 335
335 /* set the gart size */ 336 /* set the gart size */
@@ -831,8 +832,7 @@ static int gmc_v6_0_sw_init(void *handle)
831 if (r) 832 if (r)
832 return r; 833 return r;
833 834
834 amdgpu_vm_adjust_size(adev, 64, 9); 835 amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);
835 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
836 836
837 adev->mc.mc_mask = 0xffffffffffULL; 837 adev->mc.mc_mask = 0xffffffffffULL;
838 838
@@ -877,7 +877,6 @@ static int gmc_v6_0_sw_init(void *handle)
877 * amdkfd will use VMIDs 8-15 877 * amdkfd will use VMIDs 8-15
878 */ 878 */
879 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS; 879 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
880 adev->vm_manager.num_level = 1;
881 amdgpu_vm_manager_init(adev); 880 amdgpu_vm_manager_init(adev);
882 881
883 /* base offset of vram pages */ 882 /* base offset of vram pages */
@@ -897,9 +896,9 @@ static int gmc_v6_0_sw_fini(void *handle)
897{ 896{
898 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 897 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
899 898
899 amdgpu_gem_force_release(adev);
900 amdgpu_vm_manager_fini(adev); 900 amdgpu_vm_manager_fini(adev);
901 gmc_v6_0_gart_fini(adev); 901 gmc_v6_0_gart_fini(adev);
902 amdgpu_gem_force_release(adev);
903 amdgpu_bo_fini(adev); 902 amdgpu_bo_fini(adev);
904 release_firmware(adev->mc.fw); 903 release_firmware(adev->mc.fw);
905 adev->mc.fw = NULL; 904 adev->mc.fw = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index b0528ca9207b..68a85051f4b7 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -69,10 +69,10 @@ static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev)
69 case CHIP_TOPAZ: 69 case CHIP_TOPAZ:
70 amdgpu_program_register_sequence(adev, 70 amdgpu_program_register_sequence(adev,
71 iceland_mgcg_cgcg_init, 71 iceland_mgcg_cgcg_init,
72 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init)); 72 ARRAY_SIZE(iceland_mgcg_cgcg_init));
73 amdgpu_program_register_sequence(adev, 73 amdgpu_program_register_sequence(adev,
74 golden_settings_iceland_a11, 74 golden_settings_iceland_a11,
75 (const u32)ARRAY_SIZE(golden_settings_iceland_a11)); 75 ARRAY_SIZE(golden_settings_iceland_a11));
76 break; 76 break;
77 default: 77 default:
78 break; 78 break;
@@ -240,12 +240,6 @@ static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,
240 u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF; 240 u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
241 base <<= 24; 241 base <<= 24;
242 242
243 if (mc->mc_vram_size > 0xFFC0000000ULL) {
244 /* leave room for at least 1024M GTT */
245 dev_warn(adev->dev, "limiting VRAM\n");
246 mc->real_vram_size = 0xFFC0000000ULL;
247 mc->mc_vram_size = 0xFFC0000000ULL;
248 }
249 amdgpu_vram_location(adev, &adev->mc, base); 243 amdgpu_vram_location(adev, &adev->mc, base);
250 amdgpu_gart_location(adev, mc); 244 amdgpu_gart_location(adev, mc);
251} 245}
@@ -322,6 +316,8 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
322 */ 316 */
323static int gmc_v7_0_mc_init(struct amdgpu_device *adev) 317static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
324{ 318{
319 int r;
320
325 adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev); 321 adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev);
326 if (!adev->mc.vram_width) { 322 if (!adev->mc.vram_width) {
327 u32 tmp; 323 u32 tmp;
@@ -367,13 +363,18 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
367 } 363 }
368 adev->mc.vram_width = numchan * chansize; 364 adev->mc.vram_width = numchan * chansize;
369 } 365 }
370 /* Could aper size report 0 ? */
371 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
372 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
373 /* size in MB on si */ 366 /* size in MB on si */
374 adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; 367 adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
375 adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; 368 adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
376 369
370 if (!(adev->flags & AMD_IS_APU)) {
371 r = amdgpu_device_resize_fb_bar(adev);
372 if (r)
373 return r;
374 }
375 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
376 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
377
377#ifdef CONFIG_X86_64 378#ifdef CONFIG_X86_64
378 if (adev->flags & AMD_IS_APU) { 379 if (adev->flags & AMD_IS_APU) {
379 adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22; 380 adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
@@ -970,8 +971,7 @@ static int gmc_v7_0_sw_init(void *handle)
970 * Currently set to 4GB ((1 << 20) 4k pages). 971 * Currently set to 4GB ((1 << 20) 4k pages).
971 * Max GPUVM size for cayman and SI is 40 bits. 972 * Max GPUVM size for cayman and SI is 40 bits.
972 */ 973 */
973 amdgpu_vm_adjust_size(adev, 64, 9); 974 amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);
974 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
975 975
976 /* Set the internal MC address mask 976 /* Set the internal MC address mask
977 * This is the max address of the GPU's 977 * This is the max address of the GPU's
@@ -1026,7 +1026,6 @@ static int gmc_v7_0_sw_init(void *handle)
1026 * amdkfd will use VMIDs 8-15 1026 * amdkfd will use VMIDs 8-15
1027 */ 1027 */
1028 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS; 1028 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
1029 adev->vm_manager.num_level = 1;
1030 amdgpu_vm_manager_init(adev); 1029 amdgpu_vm_manager_init(adev);
1031 1030
1032 /* base offset of vram pages */ 1031 /* base offset of vram pages */
@@ -1046,9 +1045,9 @@ static int gmc_v7_0_sw_fini(void *handle)
1046{ 1045{
1047 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1046 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1048 1047
1048 amdgpu_gem_force_release(adev);
1049 amdgpu_vm_manager_fini(adev); 1049 amdgpu_vm_manager_fini(adev);
1050 gmc_v7_0_gart_fini(adev); 1050 gmc_v7_0_gart_fini(adev);
1051 amdgpu_gem_force_release(adev);
1052 amdgpu_bo_fini(adev); 1051 amdgpu_bo_fini(adev);
1053 release_firmware(adev->mc.fw); 1052 release_firmware(adev->mc.fw);
1054 adev->mc.fw = NULL; 1053 adev->mc.fw = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index f368cfe2f585..46ec97e70e5c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -122,42 +122,42 @@ static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
122 case CHIP_FIJI: 122 case CHIP_FIJI:
123 amdgpu_program_register_sequence(adev, 123 amdgpu_program_register_sequence(adev,
124 fiji_mgcg_cgcg_init, 124 fiji_mgcg_cgcg_init,
125 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init)); 125 ARRAY_SIZE(fiji_mgcg_cgcg_init));
126 amdgpu_program_register_sequence(adev, 126 amdgpu_program_register_sequence(adev,
127 golden_settings_fiji_a10, 127 golden_settings_fiji_a10,
128 (const u32)ARRAY_SIZE(golden_settings_fiji_a10)); 128 ARRAY_SIZE(golden_settings_fiji_a10));
129 break; 129 break;
130 case CHIP_TONGA: 130 case CHIP_TONGA:
131 amdgpu_program_register_sequence(adev, 131 amdgpu_program_register_sequence(adev,
132 tonga_mgcg_cgcg_init, 132 tonga_mgcg_cgcg_init,
133 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init)); 133 ARRAY_SIZE(tonga_mgcg_cgcg_init));
134 amdgpu_program_register_sequence(adev, 134 amdgpu_program_register_sequence(adev,
135 golden_settings_tonga_a11, 135 golden_settings_tonga_a11,
136 (const u32)ARRAY_SIZE(golden_settings_tonga_a11)); 136 ARRAY_SIZE(golden_settings_tonga_a11));
137 break; 137 break;
138 case CHIP_POLARIS11: 138 case CHIP_POLARIS11:
139 case CHIP_POLARIS12: 139 case CHIP_POLARIS12:
140 amdgpu_program_register_sequence(adev, 140 amdgpu_program_register_sequence(adev,
141 golden_settings_polaris11_a11, 141 golden_settings_polaris11_a11,
142 (const u32)ARRAY_SIZE(golden_settings_polaris11_a11)); 142 ARRAY_SIZE(golden_settings_polaris11_a11));
143 break; 143 break;
144 case CHIP_POLARIS10: 144 case CHIP_POLARIS10:
145 amdgpu_program_register_sequence(adev, 145 amdgpu_program_register_sequence(adev,
146 golden_settings_polaris10_a11, 146 golden_settings_polaris10_a11,
147 (const u32)ARRAY_SIZE(golden_settings_polaris10_a11)); 147 ARRAY_SIZE(golden_settings_polaris10_a11));
148 break; 148 break;
149 case CHIP_CARRIZO: 149 case CHIP_CARRIZO:
150 amdgpu_program_register_sequence(adev, 150 amdgpu_program_register_sequence(adev,
151 cz_mgcg_cgcg_init, 151 cz_mgcg_cgcg_init,
152 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init)); 152 ARRAY_SIZE(cz_mgcg_cgcg_init));
153 break; 153 break;
154 case CHIP_STONEY: 154 case CHIP_STONEY:
155 amdgpu_program_register_sequence(adev, 155 amdgpu_program_register_sequence(adev,
156 stoney_mgcg_cgcg_init, 156 stoney_mgcg_cgcg_init,
157 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init)); 157 ARRAY_SIZE(stoney_mgcg_cgcg_init));
158 amdgpu_program_register_sequence(adev, 158 amdgpu_program_register_sequence(adev,
159 golden_settings_stoney_common, 159 golden_settings_stoney_common,
160 (const u32)ARRAY_SIZE(golden_settings_stoney_common)); 160 ARRAY_SIZE(golden_settings_stoney_common));
161 break; 161 break;
162 default: 162 default:
163 break; 163 break;
@@ -405,12 +405,6 @@ static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev,
405 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF; 405 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
406 base <<= 24; 406 base <<= 24;
407 407
408 if (mc->mc_vram_size > 0xFFC0000000ULL) {
409 /* leave room for at least 1024M GTT */
410 dev_warn(adev->dev, "limiting VRAM\n");
411 mc->real_vram_size = 0xFFC0000000ULL;
412 mc->mc_vram_size = 0xFFC0000000ULL;
413 }
414 amdgpu_vram_location(adev, &adev->mc, base); 408 amdgpu_vram_location(adev, &adev->mc, base);
415 amdgpu_gart_location(adev, mc); 409 amdgpu_gart_location(adev, mc);
416} 410}
@@ -498,6 +492,8 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
498 */ 492 */
499static int gmc_v8_0_mc_init(struct amdgpu_device *adev) 493static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
500{ 494{
495 int r;
496
501 adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev); 497 adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev);
502 if (!adev->mc.vram_width) { 498 if (!adev->mc.vram_width) {
503 u32 tmp; 499 u32 tmp;
@@ -543,13 +539,18 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
543 } 539 }
544 adev->mc.vram_width = numchan * chansize; 540 adev->mc.vram_width = numchan * chansize;
545 } 541 }
546 /* Could aper size report 0 ? */
547 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
548 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
549 /* size in MB on si */ 542 /* size in MB on si */
550 adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; 543 adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
551 adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; 544 adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
552 545
546 if (!(adev->flags & AMD_IS_APU)) {
547 r = amdgpu_device_resize_fb_bar(adev);
548 if (r)
549 return r;
550 }
551 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
552 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
553
553#ifdef CONFIG_X86_64 554#ifdef CONFIG_X86_64
554 if (adev->flags & AMD_IS_APU) { 555 if (adev->flags & AMD_IS_APU) {
555 adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22; 556 adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
@@ -1067,8 +1068,7 @@ static int gmc_v8_0_sw_init(void *handle)
1067 * Currently set to 4GB ((1 << 20) 4k pages). 1068 * Currently set to 4GB ((1 << 20) 4k pages).
1068 * Max GPUVM size for cayman and SI is 40 bits. 1069 * Max GPUVM size for cayman and SI is 40 bits.
1069 */ 1070 */
1070 amdgpu_vm_adjust_size(adev, 64, 9); 1071 amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);
1071 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
1072 1072
1073 /* Set the internal MC address mask 1073 /* Set the internal MC address mask
1074 * This is the max address of the GPU's 1074 * This is the max address of the GPU's
@@ -1123,7 +1123,6 @@ static int gmc_v8_0_sw_init(void *handle)
1123 * amdkfd will use VMIDs 8-15 1123 * amdkfd will use VMIDs 8-15
1124 */ 1124 */
1125 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS; 1125 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
1126 adev->vm_manager.num_level = 1;
1127 amdgpu_vm_manager_init(adev); 1126 amdgpu_vm_manager_init(adev);
1128 1127
1129 /* base offset of vram pages */ 1128 /* base offset of vram pages */
@@ -1143,9 +1142,9 @@ static int gmc_v8_0_sw_fini(void *handle)
1143{ 1142{
1144 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1143 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1145 1144
1145 amdgpu_gem_force_release(adev);
1146 amdgpu_vm_manager_fini(adev); 1146 amdgpu_vm_manager_fini(adev);
1147 gmc_v8_0_gart_fini(adev); 1147 gmc_v8_0_gart_fini(adev);
1148 amdgpu_gem_force_release(adev);
1149 amdgpu_bo_fini(adev); 1148 amdgpu_bo_fini(adev);
1150 release_firmware(adev->mc.fw); 1149 release_firmware(adev->mc.fw);
1151 adev->mc.fw = NULL; 1150 adev->mc.fw = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index c8f1aebeac7a..cc972153d401 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -25,17 +25,18 @@
25#include "gmc_v9_0.h" 25#include "gmc_v9_0.h"
26#include "amdgpu_atomfirmware.h" 26#include "amdgpu_atomfirmware.h"
27 27
28#include "vega10/soc15ip.h" 28#include "soc15ip.h"
29#include "vega10/HDP/hdp_4_0_offset.h" 29#include "hdp/hdp_4_0_offset.h"
30#include "vega10/HDP/hdp_4_0_sh_mask.h" 30#include "hdp/hdp_4_0_sh_mask.h"
31#include "vega10/GC/gc_9_0_sh_mask.h" 31#include "gc/gc_9_0_sh_mask.h"
32#include "vega10/DC/dce_12_0_offset.h" 32#include "dce/dce_12_0_offset.h"
33#include "vega10/DC/dce_12_0_sh_mask.h" 33#include "dce/dce_12_0_sh_mask.h"
34#include "vega10/vega10_enum.h" 34#include "vega10_enum.h"
35#include "vega10/MMHUB/mmhub_1_0_offset.h" 35#include "mmhub/mmhub_1_0_offset.h"
36#include "vega10/ATHUB/athub_1_0_offset.h" 36#include "athub/athub_1_0_offset.h"
37 37
38#include "soc15_common.h" 38#include "soc15_common.h"
39#include "umc/umc_6_0_sh_mask.h"
39 40
40#include "nbio_v6_1.h" 41#include "nbio_v6_1.h"
41#include "nbio_v7_0.h" 42#include "nbio_v7_0.h"
@@ -85,6 +86,121 @@ static const u32 golden_settings_athub_1_0_0[] =
85 SOC15_REG_OFFSET(ATHUB, 0, mmRPB_ARB_CNTL2), 0x00ff00ff, 0x00080008 86 SOC15_REG_OFFSET(ATHUB, 0, mmRPB_ARB_CNTL2), 0x00ff00ff, 0x00080008
86}; 87};
87 88
89/* Ecc related register addresses, (BASE + reg offset) */
90/* Universal Memory Controller caps (may be fused). */
91/* UMCCH:UmcLocalCap */
92#define UMCLOCALCAPS_ADDR0 (0x00014306 + 0x00000000)
93#define UMCLOCALCAPS_ADDR1 (0x00014306 + 0x00000800)
94#define UMCLOCALCAPS_ADDR2 (0x00014306 + 0x00001000)
95#define UMCLOCALCAPS_ADDR3 (0x00014306 + 0x00001800)
96#define UMCLOCALCAPS_ADDR4 (0x00054306 + 0x00000000)
97#define UMCLOCALCAPS_ADDR5 (0x00054306 + 0x00000800)
98#define UMCLOCALCAPS_ADDR6 (0x00054306 + 0x00001000)
99#define UMCLOCALCAPS_ADDR7 (0x00054306 + 0x00001800)
100#define UMCLOCALCAPS_ADDR8 (0x00094306 + 0x00000000)
101#define UMCLOCALCAPS_ADDR9 (0x00094306 + 0x00000800)
102#define UMCLOCALCAPS_ADDR10 (0x00094306 + 0x00001000)
103#define UMCLOCALCAPS_ADDR11 (0x00094306 + 0x00001800)
104#define UMCLOCALCAPS_ADDR12 (0x000d4306 + 0x00000000)
105#define UMCLOCALCAPS_ADDR13 (0x000d4306 + 0x00000800)
106#define UMCLOCALCAPS_ADDR14 (0x000d4306 + 0x00001000)
107#define UMCLOCALCAPS_ADDR15 (0x000d4306 + 0x00001800)
108
109/* Universal Memory Controller Channel config. */
110/* UMCCH:UMC_CONFIG */
111#define UMCCH_UMC_CONFIG_ADDR0 (0x00014040 + 0x00000000)
112#define UMCCH_UMC_CONFIG_ADDR1 (0x00014040 + 0x00000800)
113#define UMCCH_UMC_CONFIG_ADDR2 (0x00014040 + 0x00001000)
114#define UMCCH_UMC_CONFIG_ADDR3 (0x00014040 + 0x00001800)
115#define UMCCH_UMC_CONFIG_ADDR4 (0x00054040 + 0x00000000)
116#define UMCCH_UMC_CONFIG_ADDR5 (0x00054040 + 0x00000800)
117#define UMCCH_UMC_CONFIG_ADDR6 (0x00054040 + 0x00001000)
118#define UMCCH_UMC_CONFIG_ADDR7 (0x00054040 + 0x00001800)
119#define UMCCH_UMC_CONFIG_ADDR8 (0x00094040 + 0x00000000)
120#define UMCCH_UMC_CONFIG_ADDR9 (0x00094040 + 0x00000800)
121#define UMCCH_UMC_CONFIG_ADDR10 (0x00094040 + 0x00001000)
122#define UMCCH_UMC_CONFIG_ADDR11 (0x00094040 + 0x00001800)
123#define UMCCH_UMC_CONFIG_ADDR12 (0x000d4040 + 0x00000000)
124#define UMCCH_UMC_CONFIG_ADDR13 (0x000d4040 + 0x00000800)
125#define UMCCH_UMC_CONFIG_ADDR14 (0x000d4040 + 0x00001000)
126#define UMCCH_UMC_CONFIG_ADDR15 (0x000d4040 + 0x00001800)
127
128/* Universal Memory Controller Channel Ecc config. */
129/* UMCCH:EccCtrl */
130#define UMCCH_ECCCTRL_ADDR0 (0x00014053 + 0x00000000)
131#define UMCCH_ECCCTRL_ADDR1 (0x00014053 + 0x00000800)
132#define UMCCH_ECCCTRL_ADDR2 (0x00014053 + 0x00001000)
133#define UMCCH_ECCCTRL_ADDR3 (0x00014053 + 0x00001800)
134#define UMCCH_ECCCTRL_ADDR4 (0x00054053 + 0x00000000)
135#define UMCCH_ECCCTRL_ADDR5 (0x00054053 + 0x00000800)
136#define UMCCH_ECCCTRL_ADDR6 (0x00054053 + 0x00001000)
137#define UMCCH_ECCCTRL_ADDR7 (0x00054053 + 0x00001800)
138#define UMCCH_ECCCTRL_ADDR8 (0x00094053 + 0x00000000)
139#define UMCCH_ECCCTRL_ADDR9 (0x00094053 + 0x00000800)
140#define UMCCH_ECCCTRL_ADDR10 (0x00094053 + 0x00001000)
141#define UMCCH_ECCCTRL_ADDR11 (0x00094053 + 0x00001800)
142#define UMCCH_ECCCTRL_ADDR12 (0x000d4053 + 0x00000000)
143#define UMCCH_ECCCTRL_ADDR13 (0x000d4053 + 0x00000800)
144#define UMCCH_ECCCTRL_ADDR14 (0x000d4053 + 0x00001000)
145#define UMCCH_ECCCTRL_ADDR15 (0x000d4053 + 0x00001800)
146
147static const uint32_t ecc_umclocalcap_addrs[] = {
148 UMCLOCALCAPS_ADDR0,
149 UMCLOCALCAPS_ADDR1,
150 UMCLOCALCAPS_ADDR2,
151 UMCLOCALCAPS_ADDR3,
152 UMCLOCALCAPS_ADDR4,
153 UMCLOCALCAPS_ADDR5,
154 UMCLOCALCAPS_ADDR6,
155 UMCLOCALCAPS_ADDR7,
156 UMCLOCALCAPS_ADDR8,
157 UMCLOCALCAPS_ADDR9,
158 UMCLOCALCAPS_ADDR10,
159 UMCLOCALCAPS_ADDR11,
160 UMCLOCALCAPS_ADDR12,
161 UMCLOCALCAPS_ADDR13,
162 UMCLOCALCAPS_ADDR14,
163 UMCLOCALCAPS_ADDR15,
164};
165
166static const uint32_t ecc_umcch_umc_config_addrs[] = {
167 UMCCH_UMC_CONFIG_ADDR0,
168 UMCCH_UMC_CONFIG_ADDR1,
169 UMCCH_UMC_CONFIG_ADDR2,
170 UMCCH_UMC_CONFIG_ADDR3,
171 UMCCH_UMC_CONFIG_ADDR4,
172 UMCCH_UMC_CONFIG_ADDR5,
173 UMCCH_UMC_CONFIG_ADDR6,
174 UMCCH_UMC_CONFIG_ADDR7,
175 UMCCH_UMC_CONFIG_ADDR8,
176 UMCCH_UMC_CONFIG_ADDR9,
177 UMCCH_UMC_CONFIG_ADDR10,
178 UMCCH_UMC_CONFIG_ADDR11,
179 UMCCH_UMC_CONFIG_ADDR12,
180 UMCCH_UMC_CONFIG_ADDR13,
181 UMCCH_UMC_CONFIG_ADDR14,
182 UMCCH_UMC_CONFIG_ADDR15,
183};
184
185static const uint32_t ecc_umcch_eccctrl_addrs[] = {
186 UMCCH_ECCCTRL_ADDR0,
187 UMCCH_ECCCTRL_ADDR1,
188 UMCCH_ECCCTRL_ADDR2,
189 UMCCH_ECCCTRL_ADDR3,
190 UMCCH_ECCCTRL_ADDR4,
191 UMCCH_ECCCTRL_ADDR5,
192 UMCCH_ECCCTRL_ADDR6,
193 UMCCH_ECCCTRL_ADDR7,
194 UMCCH_ECCCTRL_ADDR8,
195 UMCCH_ECCCTRL_ADDR9,
196 UMCCH_ECCCTRL_ADDR10,
197 UMCCH_ECCCTRL_ADDR11,
198 UMCCH_ECCCTRL_ADDR12,
199 UMCCH_ECCCTRL_ADDR13,
200 UMCCH_ECCCTRL_ADDR14,
201 UMCCH_ECCCTRL_ADDR15,
202};
203
88static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 204static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
89 struct amdgpu_irq_src *src, 205 struct amdgpu_irq_src *src,
90 unsigned type, 206 unsigned type,
@@ -389,6 +505,85 @@ static int gmc_v9_0_early_init(void *handle)
389 return 0; 505 return 0;
390} 506}
391 507
508static int gmc_v9_0_ecc_available(struct amdgpu_device *adev)
509{
510 uint32_t reg_val;
511 uint32_t reg_addr;
512 uint32_t field_val;
513 size_t i;
514 uint32_t fv2;
515 size_t lost_sheep;
516
517 DRM_DEBUG("ecc: gmc_v9_0_ecc_available()\n");
518
519 lost_sheep = 0;
520 for (i = 0; i < ARRAY_SIZE(ecc_umclocalcap_addrs); ++i) {
521 reg_addr = ecc_umclocalcap_addrs[i];
522 DRM_DEBUG("ecc: "
523 "UMCCH_UmcLocalCap[%zu]: reg_addr: 0x%08x\n",
524 i, reg_addr);
525 reg_val = RREG32(reg_addr);
526 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UmcLocalCap,
527 EccDis);
528 DRM_DEBUG("ecc: "
529 "reg_val: 0x%08x, "
530 "EccDis: 0x%08x, ",
531 reg_val, field_val);
532 if (field_val) {
533 DRM_ERROR("ecc: UmcLocalCap:EccDis is set.\n");
534 ++lost_sheep;
535 }
536 }
537
538 for (i = 0; i < ARRAY_SIZE(ecc_umcch_umc_config_addrs); ++i) {
539 reg_addr = ecc_umcch_umc_config_addrs[i];
540 DRM_DEBUG("ecc: "
541 "UMCCH0_0_UMC_CONFIG[%zu]: reg_addr: 0x%08x",
542 i, reg_addr);
543 reg_val = RREG32(reg_addr);
544 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UMC_CONFIG,
545 DramReady);
546 DRM_DEBUG("ecc: "
547 "reg_val: 0x%08x, "
548 "DramReady: 0x%08x\n",
549 reg_val, field_val);
550
551 if (!field_val) {
552 DRM_ERROR("ecc: UMC_CONFIG:DramReady is not set.\n");
553 ++lost_sheep;
554 }
555 }
556
557 for (i = 0; i < ARRAY_SIZE(ecc_umcch_eccctrl_addrs); ++i) {
558 reg_addr = ecc_umcch_eccctrl_addrs[i];
559 DRM_DEBUG("ecc: "
560 "UMCCH_EccCtrl[%zu]: reg_addr: 0x%08x, ",
561 i, reg_addr);
562 reg_val = RREG32(reg_addr);
563 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
564 WrEccEn);
565 fv2 = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
566 RdEccEn);
567 DRM_DEBUG("ecc: "
568 "reg_val: 0x%08x, "
569 "WrEccEn: 0x%08x, "
570 "RdEccEn: 0x%08x\n",
571 reg_val, field_val, fv2);
572
573 if (!field_val) {
574 DRM_DEBUG("ecc: WrEccEn is not set\n");
575 ++lost_sheep;
576 }
577 if (!fv2) {
578 DRM_DEBUG("ecc: RdEccEn is not set\n");
579 ++lost_sheep;
580 }
581 }
582
583 DRM_DEBUG("ecc: lost_sheep: %zu\n", lost_sheep);
584 return lost_sheep == 0;
585}
586
392static int gmc_v9_0_late_init(void *handle) 587static int gmc_v9_0_late_init(void *handle)
393{ 588{
394 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 589 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -403,6 +598,7 @@ static int gmc_v9_0_late_init(void *handle)
403 */ 598 */
404 unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 }; 599 unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 };
405 unsigned i; 600 unsigned i;
601 int r;
406 602
407 for(i = 0; i < adev->num_rings; ++i) { 603 for(i = 0; i < adev->num_rings; ++i) {
408 struct amdgpu_ring *ring = adev->rings[i]; 604 struct amdgpu_ring *ring = adev->rings[i];
@@ -418,6 +614,16 @@ static int gmc_v9_0_late_init(void *handle)
418 for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i) 614 for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i)
419 BUG_ON(vm_inv_eng[i] > 16); 615 BUG_ON(vm_inv_eng[i] > 16);
420 616
617 r = gmc_v9_0_ecc_available(adev);
618 if (r == 1) {
619 DRM_INFO("ECC is active.\n");
620 } else if (r == 0) {
621 DRM_INFO("ECC is not present.\n");
622 } else {
623 DRM_ERROR("gmc_v9_0_ecc_available() failed. r: %d\n", r);
624 return r;
625 }
626
421 return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0); 627 return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
422} 628}
423 629
@@ -449,6 +655,7 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
449{ 655{
450 u32 tmp; 656 u32 tmp;
451 int chansize, numchan; 657 int chansize, numchan;
658 int r;
452 659
453 adev->mc.vram_width = amdgpu_atomfirmware_get_vram_width(adev); 660 adev->mc.vram_width = amdgpu_atomfirmware_get_vram_width(adev);
454 if (!adev->mc.vram_width) { 661 if (!adev->mc.vram_width) {
@@ -491,17 +698,22 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
491 adev->mc.vram_width = numchan * chansize; 698 adev->mc.vram_width = numchan * chansize;
492 } 699 }
493 700
494 /* Could aper size report 0 ? */
495 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
496 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
497 /* size in MB on si */ 701 /* size in MB on si */
498 adev->mc.mc_vram_size = 702 adev->mc.mc_vram_size =
499 ((adev->flags & AMD_IS_APU) ? nbio_v7_0_get_memsize(adev) : 703 ((adev->flags & AMD_IS_APU) ? nbio_v7_0_get_memsize(adev) :
500 nbio_v6_1_get_memsize(adev)) * 1024ULL * 1024ULL; 704 nbio_v6_1_get_memsize(adev)) * 1024ULL * 1024ULL;
501 adev->mc.real_vram_size = adev->mc.mc_vram_size; 705 adev->mc.real_vram_size = adev->mc.mc_vram_size;
502 adev->mc.visible_vram_size = adev->mc.aper_size; 706
707 if (!(adev->flags & AMD_IS_APU)) {
708 r = amdgpu_device_resize_fb_bar(adev);
709 if (r)
710 return r;
711 }
712 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
713 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
503 714
504 /* In case the PCI BAR is larger than the actual amount of vram */ 715 /* In case the PCI BAR is larger than the actual amount of vram */
716 adev->mc.visible_vram_size = adev->mc.aper_size;
505 if (adev->mc.visible_vram_size > adev->mc.real_vram_size) 717 if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
506 adev->mc.visible_vram_size = adev->mc.real_vram_size; 718 adev->mc.visible_vram_size = adev->mc.real_vram_size;
507 719
@@ -557,16 +769,11 @@ static int gmc_v9_0_sw_init(void *handle)
557 switch (adev->asic_type) { 769 switch (adev->asic_type) {
558 case CHIP_RAVEN: 770 case CHIP_RAVEN:
559 adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; 771 adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
560 if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 772 if (adev->rev_id == 0x0 || adev->rev_id == 0x1)
561 adev->vm_manager.vm_size = 1U << 18; 773 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
562 adev->vm_manager.block_size = 9; 774 else
563 adev->vm_manager.num_level = 3;
564 amdgpu_vm_set_fragment_size(adev, 9);
565 } else {
566 /* vm_size is 64GB for legacy 2-level page support */ 775 /* vm_size is 64GB for legacy 2-level page support */
567 amdgpu_vm_adjust_size(adev, 64, 9); 776 amdgpu_vm_adjust_size(adev, 64, 9, 1, 48);
568 adev->vm_manager.num_level = 1;
569 }
570 break; 777 break;
571 case CHIP_VEGA10: 778 case CHIP_VEGA10:
572 /* XXX Don't know how to get VRAM type yet. */ 779 /* XXX Don't know how to get VRAM type yet. */
@@ -576,20 +783,12 @@ static int gmc_v9_0_sw_init(void *handle)
576 * vm size is 256TB (48bit), maximum size of Vega10, 783 * vm size is 256TB (48bit), maximum size of Vega10,
577 * block size 512 (9bit) 784 * block size 512 (9bit)
578 */ 785 */
579 adev->vm_manager.vm_size = 1U << 18; 786 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
580 adev->vm_manager.block_size = 9;
581 adev->vm_manager.num_level = 3;
582 amdgpu_vm_set_fragment_size(adev, 9);
583 break; 787 break;
584 default: 788 default:
585 break; 789 break;
586 } 790 }
587 791
588 DRM_INFO("vm size is %llu GB, block size is %u-bit,fragment size is %u-bit\n",
589 adev->vm_manager.vm_size,
590 adev->vm_manager.block_size,
591 adev->vm_manager.fragment_size);
592
593 /* This interrupt is VMC page fault.*/ 792 /* This interrupt is VMC page fault.*/
594 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_VMC, 0, 793 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_VMC, 0,
595 &adev->mc.vm_fault); 794 &adev->mc.vm_fault);
@@ -599,8 +798,6 @@ static int gmc_v9_0_sw_init(void *handle)
599 if (r) 798 if (r)
600 return r; 799 return r;
601 800
602 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
603
604 /* Set the internal MC address mask 801 /* Set the internal MC address mask
605 * This is the max address of the GPU's 802 * This is the max address of the GPU's
606 * internal address space. 803 * internal address space.
@@ -660,7 +857,7 @@ static int gmc_v9_0_sw_init(void *handle)
660} 857}
661 858
662/** 859/**
663 * gmc_v8_0_gart_fini - vm fini callback 860 * gmc_v9_0_gart_fini - vm fini callback
664 * 861 *
665 * @adev: amdgpu_device pointer 862 * @adev: amdgpu_device pointer
666 * 863 *
@@ -676,9 +873,9 @@ static int gmc_v9_0_sw_fini(void *handle)
676{ 873{
677 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 874 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
678 875
876 amdgpu_gem_force_release(adev);
679 amdgpu_vm_manager_fini(adev); 877 amdgpu_vm_manager_fini(adev);
680 gmc_v9_0_gart_fini(adev); 878 gmc_v9_0_gart_fini(adev);
681 amdgpu_gem_force_release(adev);
682 amdgpu_bo_fini(adev); 879 amdgpu_bo_fini(adev);
683 880
684 return 0; 881 return 0;
@@ -690,15 +887,15 @@ static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
690 case CHIP_VEGA10: 887 case CHIP_VEGA10:
691 amdgpu_program_register_sequence(adev, 888 amdgpu_program_register_sequence(adev,
692 golden_settings_mmhub_1_0_0, 889 golden_settings_mmhub_1_0_0,
693 (const u32)ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 890 ARRAY_SIZE(golden_settings_mmhub_1_0_0));
694 amdgpu_program_register_sequence(adev, 891 amdgpu_program_register_sequence(adev,
695 golden_settings_athub_1_0_0, 892 golden_settings_athub_1_0_0,
696 (const u32)ARRAY_SIZE(golden_settings_athub_1_0_0)); 893 ARRAY_SIZE(golden_settings_athub_1_0_0));
697 break; 894 break;
698 case CHIP_RAVEN: 895 case CHIP_RAVEN:
699 amdgpu_program_register_sequence(adev, 896 amdgpu_program_register_sequence(adev,
700 golden_settings_athub_1_0_0, 897 golden_settings_athub_1_0_0,
701 (const u32)ARRAY_SIZE(golden_settings_athub_1_0_0)); 898 ARRAY_SIZE(golden_settings_athub_1_0_0));
702 break; 899 break;
703 default: 900 default:
704 break; 901 break;
@@ -718,7 +915,7 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
718 915
719 amdgpu_program_register_sequence(adev, 916 amdgpu_program_register_sequence(adev,
720 golden_settings_vega10_hdp, 917 golden_settings_vega10_hdp,
721 (const u32)ARRAY_SIZE(golden_settings_vega10_hdp)); 918 ARRAY_SIZE(golden_settings_vega10_hdp));
722 919
723 if (adev->gart.robj == NULL) { 920 if (adev->gart.robj == NULL) {
724 dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 921 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
index cc21c4bdec27..bd160d8700e0 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
@@ -23,14 +23,13 @@
23#include "amdgpu.h" 23#include "amdgpu.h"
24#include "mmhub_v1_0.h" 24#include "mmhub_v1_0.h"
25 25
26#include "vega10/soc15ip.h" 26#include "soc15ip.h"
27#include "vega10/MMHUB/mmhub_1_0_offset.h" 27#include "mmhub/mmhub_1_0_offset.h"
28#include "vega10/MMHUB/mmhub_1_0_sh_mask.h" 28#include "mmhub/mmhub_1_0_sh_mask.h"
29#include "vega10/MMHUB/mmhub_1_0_default.h" 29#include "mmhub/mmhub_1_0_default.h"
30#include "vega10/ATHUB/athub_1_0_offset.h" 30#include "athub/athub_1_0_offset.h"
31#include "vega10/ATHUB/athub_1_0_sh_mask.h" 31#include "athub/athub_1_0_sh_mask.h"
32#include "vega10/ATHUB/athub_1_0_default.h" 32#include "vega10_enum.h"
33#include "vega10/vega10_enum.h"
34 33
35#include "soc15_common.h" 34#include "soc15_common.h"
36 35
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index b4906d2f30d3..ad9054e3903c 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -22,11 +22,11 @@
22 */ 22 */
23 23
24#include "amdgpu.h" 24#include "amdgpu.h"
25#include "vega10/soc15ip.h" 25#include "soc15ip.h"
26#include "vega10/NBIO/nbio_6_1_offset.h" 26#include "nbio/nbio_6_1_offset.h"
27#include "vega10/NBIO/nbio_6_1_sh_mask.h" 27#include "nbio/nbio_6_1_sh_mask.h"
28#include "vega10/GC/gc_9_0_offset.h" 28#include "gc/gc_9_0_offset.h"
29#include "vega10/GC/gc_9_0_sh_mask.h" 29#include "gc/gc_9_0_sh_mask.h"
30#include "soc15.h" 30#include "soc15.h"
31#include "vega10_ih.h" 31#include "vega10_ih.h"
32#include "soc15_common.h" 32#include "soc15_common.h"
@@ -254,7 +254,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
254 } 254 }
255 255
256 /* Trigger recovery due to world switch failure */ 256 /* Trigger recovery due to world switch failure */
257 amdgpu_sriov_gpu_reset(adev, NULL); 257 amdgpu_gpu_recover(adev, NULL);
258} 258}
259 259
260static int xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device *adev, 260static int xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device *adev,
@@ -282,9 +282,17 @@ static int xgpu_ai_mailbox_rcv_irq(struct amdgpu_device *adev,
282 /* see what event we get */ 282 /* see what event we get */
283 r = xgpu_ai_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION); 283 r = xgpu_ai_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
284 284
285 /* only handle FLR_NOTIFY now */ 285 /* sometimes the interrupt is delayed to inject to VM, so under such case
286 if (!r) 286 * the IDH_FLR_NOTIFICATION is overwritten by VF FLR from GIM side, thus
287 schedule_work(&adev->virt.flr_work); 287 * above recieve message could be failed, we should schedule the flr_work
288 * anyway
289 */
290 if (r) {
291 DRM_ERROR("FLR_NOTIFICATION is missed\n");
292 xgpu_ai_mailbox_send_ack(adev);
293 }
294
295 schedule_work(&adev->virt.flr_work);
288 } 296 }
289 297
290 return 0; 298 return 0;
@@ -353,5 +361,6 @@ const struct amdgpu_virt_ops xgpu_ai_virt_ops = {
353 .req_full_gpu = xgpu_ai_request_full_gpu_access, 361 .req_full_gpu = xgpu_ai_request_full_gpu_access,
354 .rel_full_gpu = xgpu_ai_release_full_gpu_access, 362 .rel_full_gpu = xgpu_ai_release_full_gpu_access,
355 .reset_gpu = xgpu_ai_request_reset, 363 .reset_gpu = xgpu_ai_request_reset,
364 .wait_reset = NULL,
356 .trans_msg = xgpu_ai_mailbox_trans_msg, 365 .trans_msg = xgpu_ai_mailbox_trans_msg,
357}; 366};
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
index c25a831f94ec..df52824c0cd4 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
@@ -281,29 +281,29 @@ void xgpu_vi_init_golden_registers(struct amdgpu_device *adev)
281 case CHIP_FIJI: 281 case CHIP_FIJI:
282 amdgpu_program_register_sequence(adev, 282 amdgpu_program_register_sequence(adev,
283 xgpu_fiji_mgcg_cgcg_init, 283 xgpu_fiji_mgcg_cgcg_init,
284 (const u32)ARRAY_SIZE( 284 ARRAY_SIZE(
285 xgpu_fiji_mgcg_cgcg_init)); 285 xgpu_fiji_mgcg_cgcg_init));
286 amdgpu_program_register_sequence(adev, 286 amdgpu_program_register_sequence(adev,
287 xgpu_fiji_golden_settings_a10, 287 xgpu_fiji_golden_settings_a10,
288 (const u32)ARRAY_SIZE( 288 ARRAY_SIZE(
289 xgpu_fiji_golden_settings_a10)); 289 xgpu_fiji_golden_settings_a10));
290 amdgpu_program_register_sequence(adev, 290 amdgpu_program_register_sequence(adev,
291 xgpu_fiji_golden_common_all, 291 xgpu_fiji_golden_common_all,
292 (const u32)ARRAY_SIZE( 292 ARRAY_SIZE(
293 xgpu_fiji_golden_common_all)); 293 xgpu_fiji_golden_common_all));
294 break; 294 break;
295 case CHIP_TONGA: 295 case CHIP_TONGA:
296 amdgpu_program_register_sequence(adev, 296 amdgpu_program_register_sequence(adev,
297 xgpu_tonga_mgcg_cgcg_init, 297 xgpu_tonga_mgcg_cgcg_init,
298 (const u32)ARRAY_SIZE( 298 ARRAY_SIZE(
299 xgpu_tonga_mgcg_cgcg_init)); 299 xgpu_tonga_mgcg_cgcg_init));
300 amdgpu_program_register_sequence(adev, 300 amdgpu_program_register_sequence(adev,
301 xgpu_tonga_golden_settings_a11, 301 xgpu_tonga_golden_settings_a11,
302 (const u32)ARRAY_SIZE( 302 ARRAY_SIZE(
303 xgpu_tonga_golden_settings_a11)); 303 xgpu_tonga_golden_settings_a11));
304 amdgpu_program_register_sequence(adev, 304 amdgpu_program_register_sequence(adev,
305 xgpu_tonga_golden_common_all, 305 xgpu_tonga_golden_common_all,
306 (const u32)ARRAY_SIZE( 306 ARRAY_SIZE(
307 xgpu_tonga_golden_common_all)); 307 xgpu_tonga_golden_common_all));
308 break; 308 break;
309 default: 309 default:
@@ -446,8 +446,10 @@ static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
446 request == IDH_REQ_GPU_FINI_ACCESS || 446 request == IDH_REQ_GPU_FINI_ACCESS ||
447 request == IDH_REQ_GPU_RESET_ACCESS) { 447 request == IDH_REQ_GPU_RESET_ACCESS) {
448 r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU); 448 r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
449 if (r) 449 if (r) {
450 pr_err("Doesn't get ack from pf, continue\n"); 450 pr_err("Doesn't get ack from pf, give up\n");
451 return r;
452 }
451 } 453 }
452 454
453 return 0; 455 return 0;
@@ -458,6 +460,11 @@ static int xgpu_vi_request_reset(struct amdgpu_device *adev)
458 return xgpu_vi_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS); 460 return xgpu_vi_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS);
459} 461}
460 462
463static int xgpu_vi_wait_reset_cmpl(struct amdgpu_device *adev)
464{
465 return xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
466}
467
461static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev, 468static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
462 bool init) 469 bool init)
463{ 470{
@@ -514,7 +521,7 @@ static void xgpu_vi_mailbox_flr_work(struct work_struct *work)
514 } 521 }
515 522
516 /* Trigger recovery due to world switch failure */ 523 /* Trigger recovery due to world switch failure */
517 amdgpu_sriov_gpu_reset(adev, NULL); 524 amdgpu_gpu_recover(adev, NULL);
518} 525}
519 526
520static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev, 527static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
@@ -613,5 +620,6 @@ const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
613 .req_full_gpu = xgpu_vi_request_full_gpu_access, 620 .req_full_gpu = xgpu_vi_request_full_gpu_access,
614 .rel_full_gpu = xgpu_vi_release_full_gpu_access, 621 .rel_full_gpu = xgpu_vi_release_full_gpu_access,
615 .reset_gpu = xgpu_vi_request_reset, 622 .reset_gpu = xgpu_vi_request_reset,
623 .wait_reset = xgpu_vi_wait_reset_cmpl,
616 .trans_msg = NULL, /* Does not need to trans VF errors to host. */ 624 .trans_msg = NULL, /* Does not need to trans VF errors to host. */
617}; 625};
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
index 904a1bab9b9f..76db711097c7 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
@@ -24,11 +24,11 @@
24#include "amdgpu_atombios.h" 24#include "amdgpu_atombios.h"
25#include "nbio_v6_1.h" 25#include "nbio_v6_1.h"
26 26
27#include "vega10/soc15ip.h" 27#include "soc15ip.h"
28#include "vega10/NBIO/nbio_6_1_default.h" 28#include "nbio/nbio_6_1_default.h"
29#include "vega10/NBIO/nbio_6_1_offset.h" 29#include "nbio/nbio_6_1_offset.h"
30#include "vega10/NBIO/nbio_6_1_sh_mask.h" 30#include "nbio/nbio_6_1_sh_mask.h"
31#include "vega10/vega10_enum.h" 31#include "vega10_enum.h"
32 32
33#define smnCPM_CONTROL 0x11180460 33#define smnCPM_CONTROL 0x11180460
34#define smnPCIE_CNTL2 0x11180070 34#define smnPCIE_CNTL2 0x11180070
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
index f802b973410a..1fb77174e02c 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
@@ -24,11 +24,11 @@
24#include "amdgpu_atombios.h" 24#include "amdgpu_atombios.h"
25#include "nbio_v7_0.h" 25#include "nbio_v7_0.h"
26 26
27#include "vega10/soc15ip.h" 27#include "soc15ip.h"
28#include "raven1/NBIO/nbio_7_0_default.h" 28#include "nbio/nbio_7_0_default.h"
29#include "raven1/NBIO/nbio_7_0_offset.h" 29#include "nbio/nbio_7_0_offset.h"
30#include "raven1/NBIO/nbio_7_0_sh_mask.h" 30#include "nbio/nbio_7_0_sh_mask.h"
31#include "vega10/vega10_enum.h" 31#include "vega10_enum.h"
32 32
33#define smnNBIF_MGCG_CTRL_LCLK 0x1013a05c 33#define smnNBIF_MGCG_CTRL_LCLK 0x1013a05c
34 34
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
index 4e20d91d5d50..78fe3f2917a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
@@ -30,10 +30,10 @@
30#include "soc15_common.h" 30#include "soc15_common.h"
31#include "psp_v10_0.h" 31#include "psp_v10_0.h"
32 32
33#include "vega10/soc15ip.h" 33#include "soc15ip.h"
34#include "raven1/MP/mp_10_0_offset.h" 34#include "mp/mp_10_0_offset.h"
35#include "raven1/GC/gc_9_1_offset.h" 35#include "gc/gc_9_1_offset.h"
36#include "raven1/SDMA0/sdma0_4_1_offset.h" 36#include "sdma0/sdma0_4_1_offset.h"
37 37
38MODULE_FIRMWARE("amdgpu/raven_asd.bin"); 38MODULE_FIRMWARE("amdgpu/raven_asd.bin");
39 39
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index c7bcfe8e286c..e75a23d858ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -31,12 +31,12 @@
31#include "soc15_common.h" 31#include "soc15_common.h"
32#include "psp_v3_1.h" 32#include "psp_v3_1.h"
33 33
34#include "vega10/soc15ip.h" 34#include "soc15ip.h"
35#include "vega10/MP/mp_9_0_offset.h" 35#include "mp/mp_9_0_offset.h"
36#include "vega10/MP/mp_9_0_sh_mask.h" 36#include "mp/mp_9_0_sh_mask.h"
37#include "vega10/GC/gc_9_0_offset.h" 37#include "gc/gc_9_0_offset.h"
38#include "vega10/SDMA0/sdma0_4_0_offset.h" 38#include "sdma0/sdma0_4_0_offset.h"
39#include "vega10/NBIO/nbio_6_1_offset.h" 39#include "nbio/nbio_6_1_offset.h"
40 40
41MODULE_FIRMWARE("amdgpu/vega10_sos.bin"); 41MODULE_FIRMWARE("amdgpu/vega10_sos.bin");
42MODULE_FIRMWARE("amdgpu/vega10_asd.bin"); 42MODULE_FIRMWARE("amdgpu/vega10_asd.bin");
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index 67f375bfe452..121e628e7cdb 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -95,10 +95,10 @@ static void sdma_v2_4_init_golden_registers(struct amdgpu_device *adev)
95 case CHIP_TOPAZ: 95 case CHIP_TOPAZ:
96 amdgpu_program_register_sequence(adev, 96 amdgpu_program_register_sequence(adev,
97 iceland_mgcg_cgcg_init, 97 iceland_mgcg_cgcg_init,
98 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init)); 98 ARRAY_SIZE(iceland_mgcg_cgcg_init));
99 amdgpu_program_register_sequence(adev, 99 amdgpu_program_register_sequence(adev,
100 golden_settings_iceland_a11, 100 golden_settings_iceland_a11,
101 (const u32)ARRAY_SIZE(golden_settings_iceland_a11)); 101 ARRAY_SIZE(golden_settings_iceland_a11));
102 break; 102 break;
103 default: 103 default:
104 break; 104 break;
@@ -633,7 +633,7 @@ static int sdma_v2_4_ring_test_ring(struct amdgpu_ring *ring)
633 } 633 }
634 634
635 if (i < adev->usec_timeout) { 635 if (i < adev->usec_timeout) {
636 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 636 DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
637 } else { 637 } else {
638 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 638 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
639 ring->idx, tmp); 639 ring->idx, tmp);
@@ -704,7 +704,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
704 } 704 }
705 tmp = le32_to_cpu(adev->wb.wb[index]); 705 tmp = le32_to_cpu(adev->wb.wb[index]);
706 if (tmp == 0xDEADBEEF) { 706 if (tmp == 0xDEADBEEF) {
707 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 707 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
708 r = 0; 708 r = 0;
709 } else { 709 } else {
710 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); 710 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 6d06f8eb659f..c8c93f9dac21 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -194,45 +194,45 @@ static void sdma_v3_0_init_golden_registers(struct amdgpu_device *adev)
194 case CHIP_FIJI: 194 case CHIP_FIJI:
195 amdgpu_program_register_sequence(adev, 195 amdgpu_program_register_sequence(adev,
196 fiji_mgcg_cgcg_init, 196 fiji_mgcg_cgcg_init,
197 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init)); 197 ARRAY_SIZE(fiji_mgcg_cgcg_init));
198 amdgpu_program_register_sequence(adev, 198 amdgpu_program_register_sequence(adev,
199 golden_settings_fiji_a10, 199 golden_settings_fiji_a10,
200 (const u32)ARRAY_SIZE(golden_settings_fiji_a10)); 200 ARRAY_SIZE(golden_settings_fiji_a10));
201 break; 201 break;
202 case CHIP_TONGA: 202 case CHIP_TONGA:
203 amdgpu_program_register_sequence(adev, 203 amdgpu_program_register_sequence(adev,
204 tonga_mgcg_cgcg_init, 204 tonga_mgcg_cgcg_init,
205 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init)); 205 ARRAY_SIZE(tonga_mgcg_cgcg_init));
206 amdgpu_program_register_sequence(adev, 206 amdgpu_program_register_sequence(adev,
207 golden_settings_tonga_a11, 207 golden_settings_tonga_a11,
208 (const u32)ARRAY_SIZE(golden_settings_tonga_a11)); 208 ARRAY_SIZE(golden_settings_tonga_a11));
209 break; 209 break;
210 case CHIP_POLARIS11: 210 case CHIP_POLARIS11:
211 case CHIP_POLARIS12: 211 case CHIP_POLARIS12:
212 amdgpu_program_register_sequence(adev, 212 amdgpu_program_register_sequence(adev,
213 golden_settings_polaris11_a11, 213 golden_settings_polaris11_a11,
214 (const u32)ARRAY_SIZE(golden_settings_polaris11_a11)); 214 ARRAY_SIZE(golden_settings_polaris11_a11));
215 break; 215 break;
216 case CHIP_POLARIS10: 216 case CHIP_POLARIS10:
217 amdgpu_program_register_sequence(adev, 217 amdgpu_program_register_sequence(adev,
218 golden_settings_polaris10_a11, 218 golden_settings_polaris10_a11,
219 (const u32)ARRAY_SIZE(golden_settings_polaris10_a11)); 219 ARRAY_SIZE(golden_settings_polaris10_a11));
220 break; 220 break;
221 case CHIP_CARRIZO: 221 case CHIP_CARRIZO:
222 amdgpu_program_register_sequence(adev, 222 amdgpu_program_register_sequence(adev,
223 cz_mgcg_cgcg_init, 223 cz_mgcg_cgcg_init,
224 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init)); 224 ARRAY_SIZE(cz_mgcg_cgcg_init));
225 amdgpu_program_register_sequence(adev, 225 amdgpu_program_register_sequence(adev,
226 cz_golden_settings_a11, 226 cz_golden_settings_a11,
227 (const u32)ARRAY_SIZE(cz_golden_settings_a11)); 227 ARRAY_SIZE(cz_golden_settings_a11));
228 break; 228 break;
229 case CHIP_STONEY: 229 case CHIP_STONEY:
230 amdgpu_program_register_sequence(adev, 230 amdgpu_program_register_sequence(adev,
231 stoney_mgcg_cgcg_init, 231 stoney_mgcg_cgcg_init,
232 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init)); 232 ARRAY_SIZE(stoney_mgcg_cgcg_init));
233 amdgpu_program_register_sequence(adev, 233 amdgpu_program_register_sequence(adev,
234 stoney_golden_settings_a11, 234 stoney_golden_settings_a11,
235 (const u32)ARRAY_SIZE(stoney_golden_settings_a11)); 235 ARRAY_SIZE(stoney_golden_settings_a11));
236 break; 236 break;
237 default: 237 default:
238 break; 238 break;
@@ -893,7 +893,7 @@ static int sdma_v3_0_ring_test_ring(struct amdgpu_ring *ring)
893 } 893 }
894 894
895 if (i < adev->usec_timeout) { 895 if (i < adev->usec_timeout) {
896 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 896 DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
897 } else { 897 } else {
898 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 898 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
899 ring->idx, tmp); 899 ring->idx, tmp);
@@ -964,7 +964,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
964 } 964 }
965 tmp = le32_to_cpu(adev->wb.wb[index]); 965 tmp = le32_to_cpu(adev->wb.wb[index]);
966 if (tmp == 0xDEADBEEF) { 966 if (tmp == 0xDEADBEEF) {
967 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 967 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
968 r = 0; 968 r = 0;
969 } else { 969 } else {
970 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); 970 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 46009db3d195..4c55f21e37a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -27,15 +27,15 @@
27#include "amdgpu_ucode.h" 27#include "amdgpu_ucode.h"
28#include "amdgpu_trace.h" 28#include "amdgpu_trace.h"
29 29
30#include "vega10/soc15ip.h" 30#include "soc15ip.h"
31#include "vega10/SDMA0/sdma0_4_0_offset.h" 31#include "sdma0/sdma0_4_0_offset.h"
32#include "vega10/SDMA0/sdma0_4_0_sh_mask.h" 32#include "sdma0/sdma0_4_0_sh_mask.h"
33#include "vega10/SDMA1/sdma1_4_0_offset.h" 33#include "sdma1/sdma1_4_0_offset.h"
34#include "vega10/SDMA1/sdma1_4_0_sh_mask.h" 34#include "sdma1/sdma1_4_0_sh_mask.h"
35#include "vega10/MMHUB/mmhub_1_0_offset.h" 35#include "mmhub/mmhub_1_0_offset.h"
36#include "vega10/MMHUB/mmhub_1_0_sh_mask.h" 36#include "mmhub/mmhub_1_0_sh_mask.h"
37#include "vega10/HDP/hdp_4_0_offset.h" 37#include "hdp/hdp_4_0_offset.h"
38#include "raven1/SDMA0/sdma0_4_1_default.h" 38#include "sdma0/sdma0_4_1_default.h"
39 39
40#include "soc15_common.h" 40#include "soc15_common.h"
41#include "soc15.h" 41#include "soc15.h"
@@ -132,18 +132,18 @@ static void sdma_v4_0_init_golden_registers(struct amdgpu_device *adev)
132 case CHIP_VEGA10: 132 case CHIP_VEGA10:
133 amdgpu_program_register_sequence(adev, 133 amdgpu_program_register_sequence(adev,
134 golden_settings_sdma_4, 134 golden_settings_sdma_4,
135 (const u32)ARRAY_SIZE(golden_settings_sdma_4)); 135 ARRAY_SIZE(golden_settings_sdma_4));
136 amdgpu_program_register_sequence(adev, 136 amdgpu_program_register_sequence(adev,
137 golden_settings_sdma_vg10, 137 golden_settings_sdma_vg10,
138 (const u32)ARRAY_SIZE(golden_settings_sdma_vg10)); 138 ARRAY_SIZE(golden_settings_sdma_vg10));
139 break; 139 break;
140 case CHIP_RAVEN: 140 case CHIP_RAVEN:
141 amdgpu_program_register_sequence(adev, 141 amdgpu_program_register_sequence(adev,
142 golden_settings_sdma_4_1, 142 golden_settings_sdma_4_1,
143 (const u32)ARRAY_SIZE(golden_settings_sdma_4_1)); 143 ARRAY_SIZE(golden_settings_sdma_4_1));
144 amdgpu_program_register_sequence(adev, 144 amdgpu_program_register_sequence(adev,
145 golden_settings_sdma_rv1, 145 golden_settings_sdma_rv1,
146 (const u32)ARRAY_SIZE(golden_settings_sdma_rv1)); 146 ARRAY_SIZE(golden_settings_sdma_rv1));
147 break; 147 break;
148 default: 148 default:
149 break; 149 break;
@@ -919,7 +919,7 @@ static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring)
919 } 919 }
920 920
921 if (i < adev->usec_timeout) { 921 if (i < adev->usec_timeout) {
922 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 922 DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
923 } else { 923 } else {
924 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 924 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
925 ring->idx, tmp); 925 ring->idx, tmp);
@@ -990,7 +990,7 @@ static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
990 } 990 }
991 tmp = le32_to_cpu(adev->wb.wb[index]); 991 tmp = le32_to_cpu(adev->wb.wb[index]);
992 if (tmp == 0xDEADBEEF) { 992 if (tmp == 0xDEADBEEF) {
993 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 993 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
994 r = 0; 994 r = 0;
995 } else { 995 } else {
996 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); 996 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index 8284d5dbfc30..49eef3090f08 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -1392,63 +1392,63 @@ static void si_init_golden_registers(struct amdgpu_device *adev)
1392 case CHIP_TAHITI: 1392 case CHIP_TAHITI:
1393 amdgpu_program_register_sequence(adev, 1393 amdgpu_program_register_sequence(adev,
1394 tahiti_golden_registers, 1394 tahiti_golden_registers,
1395 (const u32)ARRAY_SIZE(tahiti_golden_registers)); 1395 ARRAY_SIZE(tahiti_golden_registers));
1396 amdgpu_program_register_sequence(adev, 1396 amdgpu_program_register_sequence(adev,
1397 tahiti_golden_rlc_registers, 1397 tahiti_golden_rlc_registers,
1398 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers)); 1398 ARRAY_SIZE(tahiti_golden_rlc_registers));
1399 amdgpu_program_register_sequence(adev, 1399 amdgpu_program_register_sequence(adev,
1400 tahiti_mgcg_cgcg_init, 1400 tahiti_mgcg_cgcg_init,
1401 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init)); 1401 ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1402 amdgpu_program_register_sequence(adev, 1402 amdgpu_program_register_sequence(adev,
1403 tahiti_golden_registers2, 1403 tahiti_golden_registers2,
1404 (const u32)ARRAY_SIZE(tahiti_golden_registers2)); 1404 ARRAY_SIZE(tahiti_golden_registers2));
1405 break; 1405 break;
1406 case CHIP_PITCAIRN: 1406 case CHIP_PITCAIRN:
1407 amdgpu_program_register_sequence(adev, 1407 amdgpu_program_register_sequence(adev,
1408 pitcairn_golden_registers, 1408 pitcairn_golden_registers,
1409 (const u32)ARRAY_SIZE(pitcairn_golden_registers)); 1409 ARRAY_SIZE(pitcairn_golden_registers));
1410 amdgpu_program_register_sequence(adev, 1410 amdgpu_program_register_sequence(adev,
1411 pitcairn_golden_rlc_registers, 1411 pitcairn_golden_rlc_registers,
1412 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers)); 1412 ARRAY_SIZE(pitcairn_golden_rlc_registers));
1413 amdgpu_program_register_sequence(adev, 1413 amdgpu_program_register_sequence(adev,
1414 pitcairn_mgcg_cgcg_init, 1414 pitcairn_mgcg_cgcg_init,
1415 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init)); 1415 ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1416 break; 1416 break;
1417 case CHIP_VERDE: 1417 case CHIP_VERDE:
1418 amdgpu_program_register_sequence(adev, 1418 amdgpu_program_register_sequence(adev,
1419 verde_golden_registers, 1419 verde_golden_registers,
1420 (const u32)ARRAY_SIZE(verde_golden_registers)); 1420 ARRAY_SIZE(verde_golden_registers));
1421 amdgpu_program_register_sequence(adev, 1421 amdgpu_program_register_sequence(adev,
1422 verde_golden_rlc_registers, 1422 verde_golden_rlc_registers,
1423 (const u32)ARRAY_SIZE(verde_golden_rlc_registers)); 1423 ARRAY_SIZE(verde_golden_rlc_registers));
1424 amdgpu_program_register_sequence(adev, 1424 amdgpu_program_register_sequence(adev,
1425 verde_mgcg_cgcg_init, 1425 verde_mgcg_cgcg_init,
1426 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init)); 1426 ARRAY_SIZE(verde_mgcg_cgcg_init));
1427 amdgpu_program_register_sequence(adev, 1427 amdgpu_program_register_sequence(adev,
1428 verde_pg_init, 1428 verde_pg_init,
1429 (const u32)ARRAY_SIZE(verde_pg_init)); 1429 ARRAY_SIZE(verde_pg_init));
1430 break; 1430 break;
1431 case CHIP_OLAND: 1431 case CHIP_OLAND:
1432 amdgpu_program_register_sequence(adev, 1432 amdgpu_program_register_sequence(adev,
1433 oland_golden_registers, 1433 oland_golden_registers,
1434 (const u32)ARRAY_SIZE(oland_golden_registers)); 1434 ARRAY_SIZE(oland_golden_registers));
1435 amdgpu_program_register_sequence(adev, 1435 amdgpu_program_register_sequence(adev,
1436 oland_golden_rlc_registers, 1436 oland_golden_rlc_registers,
1437 (const u32)ARRAY_SIZE(oland_golden_rlc_registers)); 1437 ARRAY_SIZE(oland_golden_rlc_registers));
1438 amdgpu_program_register_sequence(adev, 1438 amdgpu_program_register_sequence(adev,
1439 oland_mgcg_cgcg_init, 1439 oland_mgcg_cgcg_init,
1440 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init)); 1440 ARRAY_SIZE(oland_mgcg_cgcg_init));
1441 break; 1441 break;
1442 case CHIP_HAINAN: 1442 case CHIP_HAINAN:
1443 amdgpu_program_register_sequence(adev, 1443 amdgpu_program_register_sequence(adev,
1444 hainan_golden_registers, 1444 hainan_golden_registers,
1445 (const u32)ARRAY_SIZE(hainan_golden_registers)); 1445 ARRAY_SIZE(hainan_golden_registers));
1446 amdgpu_program_register_sequence(adev, 1446 amdgpu_program_register_sequence(adev,
1447 hainan_golden_registers2, 1447 hainan_golden_registers2,
1448 (const u32)ARRAY_SIZE(hainan_golden_registers2)); 1448 ARRAY_SIZE(hainan_golden_registers2));
1449 amdgpu_program_register_sequence(adev, 1449 amdgpu_program_register_sequence(adev,
1450 hainan_mgcg_cgcg_init, 1450 hainan_mgcg_cgcg_init,
1451 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init)); 1451 ARRAY_SIZE(hainan_mgcg_cgcg_init));
1452 break; 1452 break;
1453 1453
1454 1454
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 3fa2fbf8c9a1..ee469a906cd3 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -252,7 +252,7 @@ static int si_dma_ring_test_ring(struct amdgpu_ring *ring)
252 } 252 }
253 253
254 if (i < adev->usec_timeout) { 254 if (i < adev->usec_timeout) {
255 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 255 DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
256 } else { 256 } else {
257 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 257 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
258 ring->idx, tmp); 258 ring->idx, tmp);
@@ -317,7 +317,7 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
317 } 317 }
318 tmp = le32_to_cpu(adev->wb.wb[index]); 318 tmp = le32_to_cpu(adev->wb.wb[index]);
319 if (tmp == 0xDEADBEEF) { 319 if (tmp == 0xDEADBEEF) {
320 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 320 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
321 r = 0; 321 r = 0;
322 } else { 322 } else {
323 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); 323 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index 51fd0c9a20a5..299cb3161b2c 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -5845,9 +5845,9 @@ static int si_set_mc_special_registers(struct amdgpu_device *adev,
5845 ((temp_reg & 0xffff0000)) | 5845 ((temp_reg & 0xffff0000)) |
5846 ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16); 5846 ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
5847 j++; 5847 j++;
5848
5848 if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE) 5849 if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
5849 return -EINVAL; 5850 return -EINVAL;
5850
5851 temp_reg = RREG32(MC_PMG_CMD_MRS); 5851 temp_reg = RREG32(MC_PMG_CMD_MRS);
5852 table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS; 5852 table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS;
5853 table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP; 5853 table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP;
@@ -5859,18 +5859,16 @@ static int si_set_mc_special_registers(struct amdgpu_device *adev,
5859 table->mc_reg_table_entry[k].mc_data[j] |= 0x100; 5859 table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
5860 } 5860 }
5861 j++; 5861 j++;
5862 if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
5863 return -EINVAL;
5864 5862
5865 if (adev->mc.vram_type != AMDGPU_VRAM_TYPE_GDDR5) { 5863 if (adev->mc.vram_type != AMDGPU_VRAM_TYPE_GDDR5) {
5864 if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
5865 return -EINVAL;
5866 table->mc_reg_address[j].s1 = MC_PMG_AUTO_CMD; 5866 table->mc_reg_address[j].s1 = MC_PMG_AUTO_CMD;
5867 table->mc_reg_address[j].s0 = MC_PMG_AUTO_CMD; 5867 table->mc_reg_address[j].s0 = MC_PMG_AUTO_CMD;
5868 for (k = 0; k < table->num_entries; k++) 5868 for (k = 0; k < table->num_entries; k++)
5869 table->mc_reg_table_entry[k].mc_data[j] = 5869 table->mc_reg_table_entry[k].mc_data[j] =
5870 (table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16; 5870 (table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
5871 j++; 5871 j++;
5872 if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
5873 return -EINVAL;
5874 } 5872 }
5875 break; 5873 break;
5876 case MC_SEQ_RESERVE_M: 5874 case MC_SEQ_RESERVE_M:
@@ -5882,8 +5880,6 @@ static int si_set_mc_special_registers(struct amdgpu_device *adev,
5882 (temp_reg & 0xffff0000) | 5880 (temp_reg & 0xffff0000) |
5883 (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); 5881 (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
5884 j++; 5882 j++;
5885 if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
5886 return -EINVAL;
5887 break; 5883 break;
5888 default: 5884 default:
5889 break; 5885 break;
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 4e67fe1e7955..f134ca0c093c 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -34,18 +34,18 @@
34#include "atom.h" 34#include "atom.h"
35#include "amd_pcie.h" 35#include "amd_pcie.h"
36 36
37#include "vega10/soc15ip.h" 37#include "soc15ip.h"
38#include "vega10/UVD/uvd_7_0_offset.h" 38#include "uvd/uvd_7_0_offset.h"
39#include "vega10/GC/gc_9_0_offset.h" 39#include "gc/gc_9_0_offset.h"
40#include "vega10/GC/gc_9_0_sh_mask.h" 40#include "gc/gc_9_0_sh_mask.h"
41#include "vega10/SDMA0/sdma0_4_0_offset.h" 41#include "sdma0/sdma0_4_0_offset.h"
42#include "vega10/SDMA1/sdma1_4_0_offset.h" 42#include "sdma1/sdma1_4_0_offset.h"
43#include "vega10/HDP/hdp_4_0_offset.h" 43#include "hdp/hdp_4_0_offset.h"
44#include "vega10/HDP/hdp_4_0_sh_mask.h" 44#include "hdp/hdp_4_0_sh_mask.h"
45#include "vega10/MP/mp_9_0_offset.h" 45#include "mp/mp_9_0_offset.h"
46#include "vega10/MP/mp_9_0_sh_mask.h" 46#include "mp/mp_9_0_sh_mask.h"
47#include "vega10/SMUIO/smuio_9_0_offset.h" 47#include "smuio/smuio_9_0_offset.h"
48#include "vega10/SMUIO/smuio_9_0_sh_mask.h" 48#include "smuio/smuio_9_0_sh_mask.h"
49 49
50#include "soc15.h" 50#include "soc15.h"
51#include "soc15_common.h" 51#include "soc15_common.h"
@@ -265,12 +265,12 @@ static void soc15_init_golden_registers(struct amdgpu_device *adev)
265 case CHIP_VEGA10: 265 case CHIP_VEGA10:
266 amdgpu_program_register_sequence(adev, 266 amdgpu_program_register_sequence(adev,
267 vega10_golden_init, 267 vega10_golden_init,
268 (const u32)ARRAY_SIZE(vega10_golden_init)); 268 ARRAY_SIZE(vega10_golden_init));
269 break; 269 break;
270 case CHIP_RAVEN: 270 case CHIP_RAVEN:
271 amdgpu_program_register_sequence(adev, 271 amdgpu_program_register_sequence(adev,
272 raven_golden_init, 272 raven_golden_init,
273 (const u32)ARRAY_SIZE(raven_golden_init)); 273 ARRAY_SIZE(raven_golden_init));
274 break; 274 break;
275 default: 275 default:
276 break; 276 break;
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index 8ab0f78794a5..b13ae34be1c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -521,7 +521,7 @@ static int uvd_v4_2_ring_test_ring(struct amdgpu_ring *ring)
521 } 521 }
522 522
523 if (i < adev->usec_timeout) { 523 if (i < adev->usec_timeout) {
524 DRM_INFO("ring test on %d succeeded in %d usecs\n", 524 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
525 ring->idx, i); 525 ring->idx, i);
526 } else { 526 } else {
527 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 527 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
@@ -563,7 +563,7 @@ static void uvd_v4_2_mc_resume(struct amdgpu_device *adev)
563 563
564 /* programm the VCPU memory controller bits 0-27 */ 564 /* programm the VCPU memory controller bits 0-27 */
565 addr = (adev->uvd.gpu_addr + AMDGPU_UVD_FIRMWARE_OFFSET) >> 3; 565 addr = (adev->uvd.gpu_addr + AMDGPU_UVD_FIRMWARE_OFFSET) >> 3;
566 size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4) >> 3; 566 size = AMDGPU_UVD_FIRMWARE_SIZE(adev) >> 3;
567 WREG32(mmUVD_VCPU_CACHE_OFFSET0, addr); 567 WREG32(mmUVD_VCPU_CACHE_OFFSET0, addr);
568 WREG32(mmUVD_VCPU_CACHE_SIZE0, size); 568 WREG32(mmUVD_VCPU_CACHE_SIZE0, size);
569 569
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index bb6d46e168a3..a4b0f1d842b7 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -258,7 +258,7 @@ static void uvd_v5_0_mc_resume(struct amdgpu_device *adev)
258 upper_32_bits(adev->uvd.gpu_addr)); 258 upper_32_bits(adev->uvd.gpu_addr));
259 259
260 offset = AMDGPU_UVD_FIRMWARE_OFFSET; 260 offset = AMDGPU_UVD_FIRMWARE_OFFSET;
261 size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4); 261 size = AMDGPU_UVD_FIRMWARE_SIZE(adev);
262 WREG32(mmUVD_VCPU_CACHE_OFFSET0, offset >> 3); 262 WREG32(mmUVD_VCPU_CACHE_OFFSET0, offset >> 3);
263 WREG32(mmUVD_VCPU_CACHE_SIZE0, size); 263 WREG32(mmUVD_VCPU_CACHE_SIZE0, size);
264 264
@@ -536,7 +536,7 @@ static int uvd_v5_0_ring_test_ring(struct amdgpu_ring *ring)
536 } 536 }
537 537
538 if (i < adev->usec_timeout) { 538 if (i < adev->usec_timeout) {
539 DRM_INFO("ring test on %d succeeded in %d usecs\n", 539 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
540 ring->idx, i); 540 ring->idx, i);
541 } else { 541 } else {
542 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 542 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index 920910ac8663..0e8b887cf03e 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -184,7 +184,7 @@ static int uvd_v6_0_enc_ring_test_ring(struct amdgpu_ring *ring)
184 } 184 }
185 185
186 if (i < adev->usec_timeout) { 186 if (i < adev->usec_timeout) {
187 DRM_INFO("ring test on %d succeeded in %d usecs\n", 187 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
188 ring->idx, i); 188 ring->idx, i);
189 } else { 189 } else {
190 DRM_ERROR("amdgpu: ring %d test failed\n", 190 DRM_ERROR("amdgpu: ring %d test failed\n",
@@ -360,7 +360,7 @@ static int uvd_v6_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
360 } else if (r < 0) { 360 } else if (r < 0) {
361 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); 361 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
362 } else { 362 } else {
363 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 363 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
364 r = 0; 364 r = 0;
365 } 365 }
366error: 366error:
@@ -416,7 +416,7 @@ static int uvd_v6_0_sw_init(void *handle)
416 ring = &adev->uvd.ring_enc[0]; 416 ring = &adev->uvd.ring_enc[0];
417 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL]; 417 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
418 r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity_enc, 418 r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
419 rq, amdgpu_sched_jobs); 419 rq, amdgpu_sched_jobs, NULL);
420 if (r) { 420 if (r) {
421 DRM_ERROR("Failed setting up UVD ENC run queue.\n"); 421 DRM_ERROR("Failed setting up UVD ENC run queue.\n");
422 return r; 422 return r;
@@ -603,7 +603,7 @@ static void uvd_v6_0_mc_resume(struct amdgpu_device *adev)
603 upper_32_bits(adev->uvd.gpu_addr)); 603 upper_32_bits(adev->uvd.gpu_addr));
604 604
605 offset = AMDGPU_UVD_FIRMWARE_OFFSET; 605 offset = AMDGPU_UVD_FIRMWARE_OFFSET;
606 size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4); 606 size = AMDGPU_UVD_FIRMWARE_SIZE(adev);
607 WREG32(mmUVD_VCPU_CACHE_OFFSET0, offset >> 3); 607 WREG32(mmUVD_VCPU_CACHE_OFFSET0, offset >> 3);
608 WREG32(mmUVD_VCPU_CACHE_SIZE0, size); 608 WREG32(mmUVD_VCPU_CACHE_SIZE0, size);
609 609
@@ -1008,7 +1008,7 @@ static int uvd_v6_0_ring_test_ring(struct amdgpu_ring *ring)
1008 } 1008 }
1009 1009
1010 if (i < adev->usec_timeout) { 1010 if (i < adev->usec_timeout) {
1011 DRM_INFO("ring test on %d succeeded in %d usecs\n", 1011 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
1012 ring->idx, i); 1012 ring->idx, i);
1013 } else { 1013 } else {
1014 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 1014 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index 6634545060fd..660fa41dc877 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -29,16 +29,16 @@
29#include "soc15_common.h" 29#include "soc15_common.h"
30#include "mmsch_v1_0.h" 30#include "mmsch_v1_0.h"
31 31
32#include "vega10/soc15ip.h" 32#include "soc15ip.h"
33#include "vega10/UVD/uvd_7_0_offset.h" 33#include "uvd/uvd_7_0_offset.h"
34#include "vega10/UVD/uvd_7_0_sh_mask.h" 34#include "uvd/uvd_7_0_sh_mask.h"
35#include "vega10/VCE/vce_4_0_offset.h" 35#include "vce/vce_4_0_offset.h"
36#include "vega10/VCE/vce_4_0_default.h" 36#include "vce/vce_4_0_default.h"
37#include "vega10/VCE/vce_4_0_sh_mask.h" 37#include "vce/vce_4_0_sh_mask.h"
38#include "vega10/NBIF/nbif_6_1_offset.h" 38#include "nbif/nbif_6_1_offset.h"
39#include "vega10/HDP/hdp_4_0_offset.h" 39#include "hdp/hdp_4_0_offset.h"
40#include "vega10/MMHUB/mmhub_1_0_offset.h" 40#include "mmhub/mmhub_1_0_offset.h"
41#include "vega10/MMHUB/mmhub_1_0_sh_mask.h" 41#include "mmhub/mmhub_1_0_sh_mask.h"
42 42
43static void uvd_v7_0_set_ring_funcs(struct amdgpu_device *adev); 43static void uvd_v7_0_set_ring_funcs(struct amdgpu_device *adev);
44static void uvd_v7_0_set_enc_ring_funcs(struct amdgpu_device *adev); 44static void uvd_v7_0_set_enc_ring_funcs(struct amdgpu_device *adev);
@@ -184,7 +184,7 @@ static int uvd_v7_0_enc_ring_test_ring(struct amdgpu_ring *ring)
184 } 184 }
185 185
186 if (i < adev->usec_timeout) { 186 if (i < adev->usec_timeout) {
187 DRM_INFO("ring test on %d succeeded in %d usecs\n", 187 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
188 ring->idx, i); 188 ring->idx, i);
189 } else { 189 } else {
190 DRM_ERROR("amdgpu: ring %d test failed\n", 190 DRM_ERROR("amdgpu: ring %d test failed\n",
@@ -359,7 +359,7 @@ static int uvd_v7_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
359 } else if (r < 0) { 359 } else if (r < 0) {
360 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); 360 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
361 } else { 361 } else {
362 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 362 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
363 r = 0; 363 r = 0;
364 } 364 }
365error: 365error:
@@ -418,7 +418,7 @@ static int uvd_v7_0_sw_init(void *handle)
418 ring = &adev->uvd.ring_enc[0]; 418 ring = &adev->uvd.ring_enc[0];
419 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL]; 419 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
420 r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity_enc, 420 r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
421 rq, amdgpu_sched_jobs); 421 rq, amdgpu_sched_jobs, NULL);
422 if (r) { 422 if (r) {
423 DRM_ERROR("Failed setting up UVD ENC run queue.\n"); 423 DRM_ERROR("Failed setting up UVD ENC run queue.\n");
424 return r; 424 return r;
@@ -616,7 +616,7 @@ static int uvd_v7_0_resume(void *handle)
616 */ 616 */
617static void uvd_v7_0_mc_resume(struct amdgpu_device *adev) 617static void uvd_v7_0_mc_resume(struct amdgpu_device *adev)
618{ 618{
619 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4); 619 uint32_t size = AMDGPU_UVD_FIRMWARE_SIZE(adev);
620 uint32_t offset; 620 uint32_t offset;
621 621
622 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 622 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
@@ -1192,7 +1192,7 @@ static int uvd_v7_0_ring_test_ring(struct amdgpu_ring *ring)
1192 } 1192 }
1193 1193
1194 if (i < adev->usec_timeout) { 1194 if (i < adev->usec_timeout) {
1195 DRM_INFO("ring test on %d succeeded in %d usecs\n", 1195 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
1196 ring->idx, i); 1196 ring->idx, i);
1197 } else { 1197 } else {
1198 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 1198 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index 75745544600a..f2f713650074 100644..100755
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -32,12 +32,12 @@
32#include "soc15_common.h" 32#include "soc15_common.h"
33#include "mmsch_v1_0.h" 33#include "mmsch_v1_0.h"
34 34
35#include "vega10/soc15ip.h" 35#include "soc15ip.h"
36#include "vega10/VCE/vce_4_0_offset.h" 36#include "vce/vce_4_0_offset.h"
37#include "vega10/VCE/vce_4_0_default.h" 37#include "vce/vce_4_0_default.h"
38#include "vega10/VCE/vce_4_0_sh_mask.h" 38#include "vce/vce_4_0_sh_mask.h"
39#include "vega10/MMHUB/mmhub_1_0_offset.h" 39#include "mmhub/mmhub_1_0_offset.h"
40#include "vega10/MMHUB/mmhub_1_0_sh_mask.h" 40#include "mmhub/mmhub_1_0_sh_mask.h"
41 41
42#define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK 0x02 42#define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK 0x02
43 43
@@ -243,37 +243,49 @@ static int vce_v4_0_sriov_start(struct amdgpu_device *adev)
243 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VM_CTRL), 0); 243 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VM_CTRL), 0);
244 244
245 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 245 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
246 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0), 246 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
247 adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8); 247 mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
248 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR1),
249 adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8);
250 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR2),
251 adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8); 248 adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8);
249 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
250 mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
251 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 40) & 0xff);
252 } else { 252 } else {
253 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0), 253 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
254 mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
254 adev->vce.gpu_addr >> 8); 255 adev->vce.gpu_addr >> 8);
255 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR1), 256 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
257 mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
258 (adev->vce.gpu_addr >> 40) & 0xff);
259 }
260 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
261 mmVCE_LMI_VCPU_CACHE_40BIT_BAR1),
256 adev->vce.gpu_addr >> 8); 262 adev->vce.gpu_addr >> 8);
257 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR2), 263 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
264 mmVCE_LMI_VCPU_CACHE_64BIT_BAR1),
265 (adev->vce.gpu_addr >> 40) & 0xff);
266 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
267 mmVCE_LMI_VCPU_CACHE_40BIT_BAR2),
258 adev->vce.gpu_addr >> 8); 268 adev->vce.gpu_addr >> 8);
259 } 269 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
270 mmVCE_LMI_VCPU_CACHE_64BIT_BAR2),
271 (adev->vce.gpu_addr >> 40) & 0xff);
260 272
261 offset = AMDGPU_VCE_FIRMWARE_OFFSET; 273 offset = AMDGPU_VCE_FIRMWARE_OFFSET;
262 size = VCE_V4_0_FW_SIZE; 274 size = VCE_V4_0_FW_SIZE;
263 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0), 275 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0),
264 offset & 0x7FFFFFFF); 276 offset & ~0x0f000000);
265 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE0), size); 277 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE0), size);
266 278
267 offset += size; 279 offset = (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) ? offset + size : 0;
268 size = VCE_V4_0_STACK_SIZE; 280 size = VCE_V4_0_STACK_SIZE;
269 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET1), 281 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET1),
270 offset & 0x7FFFFFFF); 282 (offset & ~0x0f000000) | (1 << 24));
271 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE1), size); 283 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE1), size);
272 284
273 offset += size; 285 offset += size;
274 size = VCE_V4_0_DATA_SIZE; 286 size = VCE_V4_0_DATA_SIZE;
275 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET2), 287 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET2),
276 offset & 0x7FFFFFFF); 288 (offset & ~0x0f000000) | (2 << 24));
277 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE2), size); 289 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE2), size);
278 290
279 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), ~0x100, 0); 291 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), ~0x100, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
index 0450ac5ba6b6..e4673f792545 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
@@ -28,12 +28,12 @@
28#include "soc15d.h" 28#include "soc15d.h"
29#include "soc15_common.h" 29#include "soc15_common.h"
30 30
31#include "vega10/soc15ip.h" 31#include "soc15ip.h"
32#include "raven1/VCN/vcn_1_0_offset.h" 32#include "vcn/vcn_1_0_offset.h"
33#include "raven1/VCN/vcn_1_0_sh_mask.h" 33#include "vcn/vcn_1_0_sh_mask.h"
34#include "vega10/HDP/hdp_4_0_offset.h" 34#include "hdp/hdp_4_0_offset.h"
35#include "raven1/MMHUB/mmhub_9_1_offset.h" 35#include "mmhub/mmhub_9_1_offset.h"
36#include "raven1/MMHUB/mmhub_9_1_sh_mask.h" 36#include "mmhub/mmhub_9_1_sh_mask.h"
37 37
38static int vcn_v1_0_start(struct amdgpu_device *adev); 38static int vcn_v1_0_start(struct amdgpu_device *adev);
39static int vcn_v1_0_stop(struct amdgpu_device *adev); 39static int vcn_v1_0_stop(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
index 697325737ba8..ca778cd4e6e8 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
@@ -26,9 +26,9 @@
26#include "soc15.h" 26#include "soc15.h"
27 27
28 28
29#include "vega10/soc15ip.h" 29#include "soc15ip.h"
30#include "vega10/OSSSYS/osssys_4_0_offset.h" 30#include "oss/osssys_4_0_offset.h"
31#include "vega10/OSSSYS/osssys_4_0_sh_mask.h" 31#include "oss/osssys_4_0_sh_mask.h"
32 32
33#include "soc15_common.h" 33#include "soc15_common.h"
34#include "vega10_ih.h" 34#include "vega10_ih.h"
@@ -46,11 +46,11 @@ static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
46 */ 46 */
47static void vega10_ih_enable_interrupts(struct amdgpu_device *adev) 47static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
48{ 48{
49 u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL)); 49 u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
50 50
51 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1); 51 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
52 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1); 52 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
53 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl); 53 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
54 adev->irq.ih.enabled = true; 54 adev->irq.ih.enabled = true;
55} 55}
56 56
@@ -63,14 +63,14 @@ static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
63 */ 63 */
64static void vega10_ih_disable_interrupts(struct amdgpu_device *adev) 64static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
65{ 65{
66 u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL)); 66 u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
67 67
68 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0); 68 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
69 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0); 69 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
70 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl); 70 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
71 /* set rptr, wptr to 0 */ 71 /* set rptr, wptr to 0 */
72 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0); 72 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
73 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0); 73 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
74 adev->irq.ih.enabled = false; 74 adev->irq.ih.enabled = false;
75 adev->irq.ih.rptr = 0; 75 adev->irq.ih.rptr = 0;
76} 76}
@@ -102,15 +102,15 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
102 else 102 else
103 nbio_v6_1_ih_control(adev); 103 nbio_v6_1_ih_control(adev);
104 104
105 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL)); 105 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
106 /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ 106 /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
107 if (adev->irq.ih.use_bus_addr) { 107 if (adev->irq.ih.use_bus_addr) {
108 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.rb_dma_addr >> 8); 108 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8);
109 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff); 109 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
110 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1); 110 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
111 } else { 111 } else {
112 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.gpu_addr >> 8); 112 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
113 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), (adev->irq.ih.gpu_addr >> 40) & 0xff); 113 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (adev->irq.ih.gpu_addr >> 40) & 0xff);
114 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4); 114 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
115 } 115 }
116 rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4); 116 rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
@@ -126,21 +126,21 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
126 if (adev->irq.msi_enabled) 126 if (adev->irq.msi_enabled)
127 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1); 127 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);
128 128
129 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl); 129 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
130 130
131 /* set the writeback address whether it's enabled or not */ 131 /* set the writeback address whether it's enabled or not */
132 if (adev->irq.ih.use_bus_addr) 132 if (adev->irq.ih.use_bus_addr)
133 wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4); 133 wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
134 else 134 else
135 wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4); 135 wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
136 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO), lower_32_bits(wptr_off)); 136 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off));
137 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI), upper_32_bits(wptr_off) & 0xFF); 137 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF);
138 138
139 /* set rptr, wptr to 0 */ 139 /* set rptr, wptr to 0 */
140 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0); 140 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
141 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0); 141 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
142 142
143 ih_doorbell_rtpr = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR)); 143 ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
144 if (adev->irq.ih.use_doorbell) { 144 if (adev->irq.ih.use_doorbell) {
145 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR, 145 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
146 OFFSET, adev->irq.ih.doorbell_index); 146 OFFSET, adev->irq.ih.doorbell_index);
@@ -150,20 +150,20 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
150 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR, 150 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
151 ENABLE, 0); 151 ENABLE, 0);
152 } 152 }
153 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR), ih_doorbell_rtpr); 153 WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
154 if (adev->flags & AMD_IS_APU) 154 if (adev->flags & AMD_IS_APU)
155 nbio_v7_0_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index); 155 nbio_v7_0_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
156 else 156 else
157 nbio_v6_1_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index); 157 nbio_v6_1_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
158 158
159 tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL)); 159 tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
160 tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL, 160 tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
161 CLIENT18_IS_STORM_CLIENT, 1); 161 CLIENT18_IS_STORM_CLIENT, 1);
162 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL), tmp); 162 WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
163 163
164 tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL)); 164 tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
165 tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1); 165 tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
166 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL), tmp); 166 WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
167 167
168 pci_set_master(adev->pdev); 168 pci_set_master(adev->pdev);
169 169
@@ -367,7 +367,7 @@ static void vega10_ih_set_rptr(struct amdgpu_device *adev)
367 adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr; 367 adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
368 WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr); 368 WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
369 } else { 369 } else {
370 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), adev->irq.ih.rptr); 370 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, adev->irq.ih.rptr);
371 } 371 }
372} 372}
373 373
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 3a4c2fa7e36d..bb8ca9489546 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -284,27 +284,27 @@ static void vi_init_golden_registers(struct amdgpu_device *adev)
284 case CHIP_TOPAZ: 284 case CHIP_TOPAZ:
285 amdgpu_program_register_sequence(adev, 285 amdgpu_program_register_sequence(adev,
286 iceland_mgcg_cgcg_init, 286 iceland_mgcg_cgcg_init,
287 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init)); 287 ARRAY_SIZE(iceland_mgcg_cgcg_init));
288 break; 288 break;
289 case CHIP_FIJI: 289 case CHIP_FIJI:
290 amdgpu_program_register_sequence(adev, 290 amdgpu_program_register_sequence(adev,
291 fiji_mgcg_cgcg_init, 291 fiji_mgcg_cgcg_init,
292 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init)); 292 ARRAY_SIZE(fiji_mgcg_cgcg_init));
293 break; 293 break;
294 case CHIP_TONGA: 294 case CHIP_TONGA:
295 amdgpu_program_register_sequence(adev, 295 amdgpu_program_register_sequence(adev,
296 tonga_mgcg_cgcg_init, 296 tonga_mgcg_cgcg_init,
297 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init)); 297 ARRAY_SIZE(tonga_mgcg_cgcg_init));
298 break; 298 break;
299 case CHIP_CARRIZO: 299 case CHIP_CARRIZO:
300 amdgpu_program_register_sequence(adev, 300 amdgpu_program_register_sequence(adev,
301 cz_mgcg_cgcg_init, 301 cz_mgcg_cgcg_init,
302 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init)); 302 ARRAY_SIZE(cz_mgcg_cgcg_init));
303 break; 303 break;
304 case CHIP_STONEY: 304 case CHIP_STONEY:
305 amdgpu_program_register_sequence(adev, 305 amdgpu_program_register_sequence(adev,
306 stoney_mgcg_cgcg_init, 306 stoney_mgcg_cgcg_init,
307 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init)); 307 ARRAY_SIZE(stoney_mgcg_cgcg_init));
308 break; 308 break;
309 case CHIP_POLARIS11: 309 case CHIP_POLARIS11:
310 case CHIP_POLARIS10: 310 case CHIP_POLARIS10:
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-27 12:14:43 -0400