aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/amd/amdgpu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu')
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu.h16
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c20
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c13
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c208
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_device.c104
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_display.c140
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c68
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_job.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c21
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h15
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.c23
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.h1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c12
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c229
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h18
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h37
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c36
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c14
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c46
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c155
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h21
-rw-r--r--drivers/gpu/drm/amd/amdgpu/ci_dpm.c30
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v10_0.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v11_0.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v6_0.c13
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v8_0.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_virtual.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c12
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c22
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c517
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c23
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mmsch_v1_0.h57
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/psp_v3_1.c86
-rw-r--r--drivers/gpu/drm/amd/amdgpu/psp_v3_1.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c222
-rw-r--r--drivers/gpu/drm/amd/amdgpu/soc15.c9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/soc15_common.h20
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c466
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vce_v4_0.c224
59 files changed, 1660 insertions, 1362 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 6a8129949333..833c3c16501a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -110,6 +110,7 @@ extern int amdgpu_pos_buf_per_se;
110extern int amdgpu_cntl_sb_buf_per_se; 110extern int amdgpu_cntl_sb_buf_per_se;
111extern int amdgpu_param_buf_per_se; 111extern int amdgpu_param_buf_per_se;
112 112
113#define AMDGPU_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */
113#define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000 114#define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000
114#define AMDGPU_MAX_USEC_TIMEOUT 100000 /* 100 ms */ 115#define AMDGPU_MAX_USEC_TIMEOUT 100000 /* 100 ms */
115#define AMDGPU_FENCE_JIFFIES_TIMEOUT (HZ / 2) 116#define AMDGPU_FENCE_JIFFIES_TIMEOUT (HZ / 2)
@@ -966,6 +967,8 @@ struct amdgpu_gfx_config {
966 unsigned mc_arb_ramcfg; 967 unsigned mc_arb_ramcfg;
967 unsigned gb_addr_config; 968 unsigned gb_addr_config;
968 unsigned num_rbs; 969 unsigned num_rbs;
970 unsigned gs_vgt_table_depth;
971 unsigned gs_prim_buffer_depth;
969 972
970 uint32_t tile_mode_array[32]; 973 uint32_t tile_mode_array[32];
971 uint32_t macrotile_mode_array[16]; 974 uint32_t macrotile_mode_array[16];
@@ -980,6 +983,7 @@ struct amdgpu_gfx_config {
980struct amdgpu_cu_info { 983struct amdgpu_cu_info {
981 uint32_t number; /* total active CU number */ 984 uint32_t number; /* total active CU number */
982 uint32_t ao_cu_mask; 985 uint32_t ao_cu_mask;
986 uint32_t wave_front_size;
983 uint32_t bitmap[4][4]; 987 uint32_t bitmap[4][4];
984}; 988};
985 989
@@ -1000,10 +1004,10 @@ struct amdgpu_ngg_buf {
1000}; 1004};
1001 1005
1002enum { 1006enum {
1003 PRIM = 0, 1007 NGG_PRIM = 0,
1004 POS, 1008 NGG_POS,
1005 CNTL, 1009 NGG_CNTL,
1006 PARAM, 1010 NGG_PARAM,
1007 NGG_BUF_MAX 1011 NGG_BUF_MAX
1008}; 1012};
1009 1013
@@ -1125,6 +1129,7 @@ struct amdgpu_job {
1125 void *owner; 1129 void *owner;
1126 uint64_t fence_ctx; /* the fence_context this job uses */ 1130 uint64_t fence_ctx; /* the fence_context this job uses */
1127 bool vm_needs_flush; 1131 bool vm_needs_flush;
1132 bool need_pipeline_sync;
1128 unsigned vm_id; 1133 unsigned vm_id;
1129 uint64_t vm_pd_addr; 1134 uint64_t vm_pd_addr;
1130 uint32_t gds_base, gds_size; 1135 uint32_t gds_base, gds_size;
@@ -1704,9 +1709,6 @@ void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);
1704#define WREG32_FIELD_OFFSET(reg, offset, field, val) \ 1709#define WREG32_FIELD_OFFSET(reg, offset, field, val) \
1705 WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field)) 1710 WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
1706 1711
1707#define WREG32_FIELD15(ip, idx, reg, field, val) \
1708 WREG32(SOC15_REG_OFFSET(ip, idx, mm##reg), (RREG32(SOC15_REG_OFFSET(ip, idx, mm##reg)) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
1709
1710/* 1712/*
1711 * BIOS helpers. 1713 * BIOS helpers.
1712 */ 1714 */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index ad4329922f79..1cf78f4dd339 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -1727,6 +1727,12 @@ void amdgpu_atombios_scratch_regs_restore(struct amdgpu_device *adev)
1727{ 1727{
1728 int i; 1728 int i;
1729 1729
1730 /*
1731 * VBIOS will check ASIC_INIT_COMPLETE bit to decide if
1732 * execute ASIC_Init posting via driver
1733 */
1734 adev->bios_scratch[7] &= ~ATOM_S7_ASIC_INIT_COMPLETE_MASK;
1735
1730 for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++) 1736 for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)
1731 WREG32(mmBIOS_SCRATCH_0 + i, adev->bios_scratch[i]); 1737 WREG32(mmBIOS_SCRATCH_0 + i, adev->bios_scratch[i]);
1732} 1738}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index 4b9abd68e04f..4bdda56fccee 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -26,6 +26,7 @@
26#include "atomfirmware.h" 26#include "atomfirmware.h"
27#include "amdgpu_atomfirmware.h" 27#include "amdgpu_atomfirmware.h"
28#include "atom.h" 28#include "atom.h"
29#include "atombios.h"
29 30
30#define get_index_into_master_table(master_table, table_name) (offsetof(struct master_table, table_name) / sizeof(uint16_t)) 31#define get_index_into_master_table(master_table, table_name) (offsetof(struct master_table, table_name) / sizeof(uint16_t))
31 32
@@ -77,10 +78,29 @@ void amdgpu_atomfirmware_scratch_regs_restore(struct amdgpu_device *adev)
77{ 78{
78 int i; 79 int i;
79 80
81 /*
82 * VBIOS will check ASIC_INIT_COMPLETE bit to decide if
83 * execute ASIC_Init posting via driver
84 */
85 adev->bios_scratch[7] &= ~ATOM_S7_ASIC_INIT_COMPLETE_MASK;
86
80 for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++) 87 for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)
81 WREG32(adev->bios_scratch_reg_offset + i, adev->bios_scratch[i]); 88 WREG32(adev->bios_scratch_reg_offset + i, adev->bios_scratch[i]);
82} 89}
83 90
91void amdgpu_atomfirmware_scratch_regs_engine_hung(struct amdgpu_device *adev,
92 bool hung)
93{
94 u32 tmp = RREG32(adev->bios_scratch_reg_offset + 3);
95
96 if (hung)
97 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
98 else
99 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
100
101 WREG32(adev->bios_scratch_reg_offset + 3, tmp);
102}
103
84int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) 104int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
85{ 105{
86 struct atom_context *ctx = adev->mode_info.atom_context; 106 struct atom_context *ctx = adev->mode_info.atom_context;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
index d0c4dcd7fa96..a2c3ebe22c71 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
@@ -28,6 +28,8 @@ bool amdgpu_atomfirmware_gpu_supports_virtualization(struct amdgpu_device *adev)
28void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev); 28void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev);
29void amdgpu_atomfirmware_scratch_regs_save(struct amdgpu_device *adev); 29void amdgpu_atomfirmware_scratch_regs_save(struct amdgpu_device *adev);
30void amdgpu_atomfirmware_scratch_regs_restore(struct amdgpu_device *adev); 30void amdgpu_atomfirmware_scratch_regs_restore(struct amdgpu_device *adev);
31void amdgpu_atomfirmware_scratch_regs_engine_hung(struct amdgpu_device *adev,
32 bool hung);
31int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev); 33int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev);
32 34
33#endif 35#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
index cc97eee93226..1beae5b930d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
@@ -117,8 +117,13 @@ static void amdgpu_benchmark_move(struct amdgpu_device *adev, unsigned size,
117 } 117 }
118 118
119out_cleanup: 119out_cleanup:
120 /* Check error value now. The value can be overwritten when clean up.*/
121 if (r) {
122 DRM_ERROR("Error while benchmarking BO move.\n");
123 }
124
120 if (sobj) { 125 if (sobj) {
121 r = amdgpu_bo_reserve(sobj, false); 126 r = amdgpu_bo_reserve(sobj, true);
122 if (likely(r == 0)) { 127 if (likely(r == 0)) {
123 amdgpu_bo_unpin(sobj); 128 amdgpu_bo_unpin(sobj);
124 amdgpu_bo_unreserve(sobj); 129 amdgpu_bo_unreserve(sobj);
@@ -126,17 +131,13 @@ out_cleanup:
126 amdgpu_bo_unref(&sobj); 131 amdgpu_bo_unref(&sobj);
127 } 132 }
128 if (dobj) { 133 if (dobj) {
129 r = amdgpu_bo_reserve(dobj, false); 134 r = amdgpu_bo_reserve(dobj, true);
130 if (likely(r == 0)) { 135 if (likely(r == 0)) {
131 amdgpu_bo_unpin(dobj); 136 amdgpu_bo_unpin(dobj);
132 amdgpu_bo_unreserve(dobj); 137 amdgpu_bo_unreserve(dobj);
133 } 138 }
134 amdgpu_bo_unref(&dobj); 139 amdgpu_bo_unref(&dobj);
135 } 140 }
136
137 if (r) {
138 DRM_ERROR("Error while benchmarking BO move.\n");
139 }
140} 141}
141 142
142void amdgpu_benchmark(struct amdgpu_device *adev, int test_number) 143void amdgpu_benchmark(struct amdgpu_device *adev, int test_number)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index 1c7e6c28f93a..c6dba1eaefbd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -42,82 +42,6 @@ struct amdgpu_cgs_device {
42 struct amdgpu_device *adev = \ 42 struct amdgpu_device *adev = \
43 ((struct amdgpu_cgs_device *)cgs_device)->adev 43 ((struct amdgpu_cgs_device *)cgs_device)->adev
44 44
45static int amdgpu_cgs_gpu_mem_info(struct cgs_device *cgs_device, enum cgs_gpu_mem_type type,
46 uint64_t *mc_start, uint64_t *mc_size,
47 uint64_t *mem_size)
48{
49 CGS_FUNC_ADEV;
50 switch(type) {
51 case CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB:
52 case CGS_GPU_MEM_TYPE__VISIBLE_FB:
53 *mc_start = 0;
54 *mc_size = adev->mc.visible_vram_size;
55 *mem_size = adev->mc.visible_vram_size - adev->vram_pin_size;
56 break;
57 case CGS_GPU_MEM_TYPE__INVISIBLE_CONTIG_FB:
58 case CGS_GPU_MEM_TYPE__INVISIBLE_FB:
59 *mc_start = adev->mc.visible_vram_size;
60 *mc_size = adev->mc.real_vram_size - adev->mc.visible_vram_size;
61 *mem_size = *mc_size;
62 break;
63 case CGS_GPU_MEM_TYPE__GART_CACHEABLE:
64 case CGS_GPU_MEM_TYPE__GART_WRITECOMBINE:
65 *mc_start = adev->mc.gtt_start;
66 *mc_size = adev->mc.gtt_size;
67 *mem_size = adev->mc.gtt_size - adev->gart_pin_size;
68 break;
69 default:
70 return -EINVAL;
71 }
72
73 return 0;
74}
75
76static int amdgpu_cgs_gmap_kmem(struct cgs_device *cgs_device, void *kmem,
77 uint64_t size,
78 uint64_t min_offset, uint64_t max_offset,
79 cgs_handle_t *kmem_handle, uint64_t *mcaddr)
80{
81 CGS_FUNC_ADEV;
82 int ret;
83 struct amdgpu_bo *bo;
84 struct page *kmem_page = vmalloc_to_page(kmem);
85 int npages = ALIGN(size, PAGE_SIZE) >> PAGE_SHIFT;
86
87 struct sg_table *sg = drm_prime_pages_to_sg(&kmem_page, npages);
88 ret = amdgpu_bo_create(adev, size, PAGE_SIZE, false,
89 AMDGPU_GEM_DOMAIN_GTT, 0, sg, NULL, &bo);
90 if (ret)
91 return ret;
92 ret = amdgpu_bo_reserve(bo, false);
93 if (unlikely(ret != 0))
94 return ret;
95
96 /* pin buffer into GTT */
97 ret = amdgpu_bo_pin_restricted(bo, AMDGPU_GEM_DOMAIN_GTT,
98 min_offset, max_offset, mcaddr);
99 amdgpu_bo_unreserve(bo);
100
101 *kmem_handle = (cgs_handle_t)bo;
102 return ret;
103}
104
105static int amdgpu_cgs_gunmap_kmem(struct cgs_device *cgs_device, cgs_handle_t kmem_handle)
106{
107 struct amdgpu_bo *obj = (struct amdgpu_bo *)kmem_handle;
108
109 if (obj) {
110 int r = amdgpu_bo_reserve(obj, false);
111 if (likely(r == 0)) {
112 amdgpu_bo_unpin(obj);
113 amdgpu_bo_unreserve(obj);
114 }
115 amdgpu_bo_unref(&obj);
116
117 }
118 return 0;
119}
120
121static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device, 45static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
122 enum cgs_gpu_mem_type type, 46 enum cgs_gpu_mem_type type,
123 uint64_t size, uint64_t align, 47 uint64_t size, uint64_t align,
@@ -215,7 +139,7 @@ static int amdgpu_cgs_free_gpu_mem(struct cgs_device *cgs_device, cgs_handle_t h
215 struct amdgpu_bo *obj = (struct amdgpu_bo *)handle; 139 struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
216 140
217 if (obj) { 141 if (obj) {
218 int r = amdgpu_bo_reserve(obj, false); 142 int r = amdgpu_bo_reserve(obj, true);
219 if (likely(r == 0)) { 143 if (likely(r == 0)) {
220 amdgpu_bo_kunmap(obj); 144 amdgpu_bo_kunmap(obj);
221 amdgpu_bo_unpin(obj); 145 amdgpu_bo_unpin(obj);
@@ -239,7 +163,7 @@ static int amdgpu_cgs_gmap_gpu_mem(struct cgs_device *cgs_device, cgs_handle_t h
239 min_offset = obj->placements[0].fpfn << PAGE_SHIFT; 163 min_offset = obj->placements[0].fpfn << PAGE_SHIFT;
240 max_offset = obj->placements[0].lpfn << PAGE_SHIFT; 164 max_offset = obj->placements[0].lpfn << PAGE_SHIFT;
241 165
242 r = amdgpu_bo_reserve(obj, false); 166 r = amdgpu_bo_reserve(obj, true);
243 if (unlikely(r != 0)) 167 if (unlikely(r != 0))
244 return r; 168 return r;
245 r = amdgpu_bo_pin_restricted(obj, obj->prefered_domains, 169 r = amdgpu_bo_pin_restricted(obj, obj->prefered_domains,
@@ -252,7 +176,7 @@ static int amdgpu_cgs_gunmap_gpu_mem(struct cgs_device *cgs_device, cgs_handle_t
252{ 176{
253 int r; 177 int r;
254 struct amdgpu_bo *obj = (struct amdgpu_bo *)handle; 178 struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
255 r = amdgpu_bo_reserve(obj, false); 179 r = amdgpu_bo_reserve(obj, true);
256 if (unlikely(r != 0)) 180 if (unlikely(r != 0))
257 return r; 181 return r;
258 r = amdgpu_bo_unpin(obj); 182 r = amdgpu_bo_unpin(obj);
@@ -265,7 +189,7 @@ static int amdgpu_cgs_kmap_gpu_mem(struct cgs_device *cgs_device, cgs_handle_t h
265{ 189{
266 int r; 190 int r;
267 struct amdgpu_bo *obj = (struct amdgpu_bo *)handle; 191 struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
268 r = amdgpu_bo_reserve(obj, false); 192 r = amdgpu_bo_reserve(obj, true);
269 if (unlikely(r != 0)) 193 if (unlikely(r != 0))
270 return r; 194 return r;
271 r = amdgpu_bo_kmap(obj, map); 195 r = amdgpu_bo_kmap(obj, map);
@@ -277,7 +201,7 @@ static int amdgpu_cgs_kunmap_gpu_mem(struct cgs_device *cgs_device, cgs_handle_t
277{ 201{
278 int r; 202 int r;
279 struct amdgpu_bo *obj = (struct amdgpu_bo *)handle; 203 struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
280 r = amdgpu_bo_reserve(obj, false); 204 r = amdgpu_bo_reserve(obj, true);
281 if (unlikely(r != 0)) 205 if (unlikely(r != 0))
282 return r; 206 return r;
283 amdgpu_bo_kunmap(obj); 207 amdgpu_bo_kunmap(obj);
@@ -349,62 +273,6 @@ static void amdgpu_cgs_write_ind_register(struct cgs_device *cgs_device,
349 WARN(1, "Invalid indirect register space"); 273 WARN(1, "Invalid indirect register space");
350} 274}
351 275
352static uint8_t amdgpu_cgs_read_pci_config_byte(struct cgs_device *cgs_device, unsigned addr)
353{
354 CGS_FUNC_ADEV;
355 uint8_t val;
356 int ret = pci_read_config_byte(adev->pdev, addr, &val);
357 if (WARN(ret, "pci_read_config_byte error"))
358 return 0;
359 return val;
360}
361
362static uint16_t amdgpu_cgs_read_pci_config_word(struct cgs_device *cgs_device, unsigned addr)
363{
364 CGS_FUNC_ADEV;
365 uint16_t val;
366 int ret = pci_read_config_word(adev->pdev, addr, &val);
367 if (WARN(ret, "pci_read_config_word error"))
368 return 0;
369 return val;
370}
371
372static uint32_t amdgpu_cgs_read_pci_config_dword(struct cgs_device *cgs_device,
373 unsigned addr)
374{
375 CGS_FUNC_ADEV;
376 uint32_t val;
377 int ret = pci_read_config_dword(adev->pdev, addr, &val);
378 if (WARN(ret, "pci_read_config_dword error"))
379 return 0;
380 return val;
381}
382
383static void amdgpu_cgs_write_pci_config_byte(struct cgs_device *cgs_device, unsigned addr,
384 uint8_t value)
385{
386 CGS_FUNC_ADEV;
387 int ret = pci_write_config_byte(adev->pdev, addr, value);
388 WARN(ret, "pci_write_config_byte error");
389}
390
391static void amdgpu_cgs_write_pci_config_word(struct cgs_device *cgs_device, unsigned addr,
392 uint16_t value)
393{
394 CGS_FUNC_ADEV;
395 int ret = pci_write_config_word(adev->pdev, addr, value);
396 WARN(ret, "pci_write_config_word error");
397}
398
399static void amdgpu_cgs_write_pci_config_dword(struct cgs_device *cgs_device, unsigned addr,
400 uint32_t value)
401{
402 CGS_FUNC_ADEV;
403 int ret = pci_write_config_dword(adev->pdev, addr, value);
404 WARN(ret, "pci_write_config_dword error");
405}
406
407
408static int amdgpu_cgs_get_pci_resource(struct cgs_device *cgs_device, 276static int amdgpu_cgs_get_pci_resource(struct cgs_device *cgs_device,
409 enum cgs_resource_type resource_type, 277 enum cgs_resource_type resource_type,
410 uint64_t size, 278 uint64_t size,
@@ -477,56 +345,6 @@ static int amdgpu_cgs_atom_exec_cmd_table(struct cgs_device *cgs_device, unsigne
477 adev->mode_info.atom_context, table, args); 345 adev->mode_info.atom_context, table, args);
478} 346}
479 347
480static int amdgpu_cgs_create_pm_request(struct cgs_device *cgs_device, cgs_handle_t *request)
481{
482 /* TODO */
483 return 0;
484}
485
486static int amdgpu_cgs_destroy_pm_request(struct cgs_device *cgs_device, cgs_handle_t request)
487{
488 /* TODO */
489 return 0;
490}
491
492static int amdgpu_cgs_set_pm_request(struct cgs_device *cgs_device, cgs_handle_t request,
493 int active)
494{
495 /* TODO */
496 return 0;
497}
498
499static int amdgpu_cgs_pm_request_clock(struct cgs_device *cgs_device, cgs_handle_t request,
500 enum cgs_clock clock, unsigned freq)
501{
502 /* TODO */
503 return 0;
504}
505
506static int amdgpu_cgs_pm_request_engine(struct cgs_device *cgs_device, cgs_handle_t request,
507 enum cgs_engine engine, int powered)
508{
509 /* TODO */
510 return 0;
511}
512
513
514
515static int amdgpu_cgs_pm_query_clock_limits(struct cgs_device *cgs_device,
516 enum cgs_clock clock,
517 struct cgs_clock_limits *limits)
518{
519 /* TODO */
520 return 0;
521}
522
523static int amdgpu_cgs_set_camera_voltages(struct cgs_device *cgs_device, uint32_t mask,
524 const uint32_t *voltages)
525{
526 DRM_ERROR("not implemented");
527 return -EPERM;
528}
529
530struct cgs_irq_params { 348struct cgs_irq_params {
531 unsigned src_id; 349 unsigned src_id;
532 cgs_irq_source_set_func_t set; 350 cgs_irq_source_set_func_t set;
@@ -1269,9 +1087,6 @@ static int amdgpu_cgs_call_acpi_method(struct cgs_device *cgs_device,
1269} 1087}
1270 1088
1271static const struct cgs_ops amdgpu_cgs_ops = { 1089static const struct cgs_ops amdgpu_cgs_ops = {
1272 .gpu_mem_info = amdgpu_cgs_gpu_mem_info,
1273 .gmap_kmem = amdgpu_cgs_gmap_kmem,
1274 .gunmap_kmem = amdgpu_cgs_gunmap_kmem,
1275 .alloc_gpu_mem = amdgpu_cgs_alloc_gpu_mem, 1090 .alloc_gpu_mem = amdgpu_cgs_alloc_gpu_mem,
1276 .free_gpu_mem = amdgpu_cgs_free_gpu_mem, 1091 .free_gpu_mem = amdgpu_cgs_free_gpu_mem,
1277 .gmap_gpu_mem = amdgpu_cgs_gmap_gpu_mem, 1092 .gmap_gpu_mem = amdgpu_cgs_gmap_gpu_mem,
@@ -1282,23 +1097,10 @@ static const struct cgs_ops amdgpu_cgs_ops = {
1282 .write_register = amdgpu_cgs_write_register, 1097 .write_register = amdgpu_cgs_write_register,
1283 .read_ind_register = amdgpu_cgs_read_ind_register, 1098 .read_ind_register = amdgpu_cgs_read_ind_register,
1284 .write_ind_register = amdgpu_cgs_write_ind_register, 1099 .write_ind_register = amdgpu_cgs_write_ind_register,
1285 .read_pci_config_byte = amdgpu_cgs_read_pci_config_byte,
1286 .read_pci_config_word = amdgpu_cgs_read_pci_config_word,
1287 .read_pci_config_dword = amdgpu_cgs_read_pci_config_dword,
1288 .write_pci_config_byte = amdgpu_cgs_write_pci_config_byte,
1289 .write_pci_config_word = amdgpu_cgs_write_pci_config_word,
1290 .write_pci_config_dword = amdgpu_cgs_write_pci_config_dword,
1291 .get_pci_resource = amdgpu_cgs_get_pci_resource, 1100 .get_pci_resource = amdgpu_cgs_get_pci_resource,
1292 .atom_get_data_table = amdgpu_cgs_atom_get_data_table, 1101 .atom_get_data_table = amdgpu_cgs_atom_get_data_table,
1293 .atom_get_cmd_table_revs = amdgpu_cgs_atom_get_cmd_table_revs, 1102 .atom_get_cmd_table_revs = amdgpu_cgs_atom_get_cmd_table_revs,
1294 .atom_exec_cmd_table = amdgpu_cgs_atom_exec_cmd_table, 1103 .atom_exec_cmd_table = amdgpu_cgs_atom_exec_cmd_table,
1295 .create_pm_request = amdgpu_cgs_create_pm_request,
1296 .destroy_pm_request = amdgpu_cgs_destroy_pm_request,
1297 .set_pm_request = amdgpu_cgs_set_pm_request,
1298 .pm_request_clock = amdgpu_cgs_pm_request_clock,
1299 .pm_request_engine = amdgpu_cgs_pm_request_engine,
1300 .pm_query_clock_limits = amdgpu_cgs_pm_query_clock_limits,
1301 .set_camera_voltages = amdgpu_cgs_set_camera_voltages,
1302 .get_firmware_info = amdgpu_cgs_get_firmware_info, 1104 .get_firmware_info = amdgpu_cgs_get_firmware_info,
1303 .rel_firmware = amdgpu_cgs_rel_firmware, 1105 .rel_firmware = amdgpu_cgs_rel_firmware,
1304 .set_powergating_state = amdgpu_cgs_set_powergating_state, 1106 .set_powergating_state = amdgpu_cgs_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index ec71b9320561..4e6b9501ab0a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -1074,6 +1074,7 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
1074 cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence); 1074 cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence);
1075 job->uf_sequence = cs->out.handle; 1075 job->uf_sequence = cs->out.handle;
1076 amdgpu_job_free_resources(job); 1076 amdgpu_job_free_resources(job);
1077 amdgpu_cs_parser_fini(p, 0, true);
1077 1078
1078 trace_amdgpu_cs_ioctl(job); 1079 trace_amdgpu_cs_ioctl(job);
1079 amd_sched_entity_push_job(&job->base); 1080 amd_sched_entity_push_job(&job->base);
@@ -1129,7 +1130,10 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
1129 goto out; 1130 goto out;
1130 1131
1131 r = amdgpu_cs_submit(&parser, cs); 1132 r = amdgpu_cs_submit(&parser, cs);
1133 if (r)
1134 goto out;
1132 1135
1136 return 0;
1133out: 1137out:
1134 amdgpu_cs_parser_fini(&parser, r, reserved_buffers); 1138 amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
1135 return r; 1139 return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index cf0500671353..90d1ac8a80f8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -273,6 +273,9 @@ struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
273 273
274 spin_lock(&ctx->ring_lock); 274 spin_lock(&ctx->ring_lock);
275 275
276 if (seq == ~0ull)
277 seq = ctx->rings[ring->idx].sequence - 1;
278
276 if (seq >= cring->sequence) { 279 if (seq >= cring->sequence) {
277 spin_unlock(&ctx->ring_lock); 280 spin_unlock(&ctx->ring_lock);
278 return ERR_PTR(-EINVAL); 281 return ERR_PTR(-EINVAL);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 483660742f75..43ca16b6eee2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -53,7 +53,6 @@
53#include "bif/bif_4_1_d.h" 53#include "bif/bif_4_1_d.h"
54#include <linux/pci.h> 54#include <linux/pci.h>
55#include <linux/firmware.h> 55#include <linux/firmware.h>
56#include "amdgpu_pm.h"
57 56
58static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev); 57static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
59static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev); 58static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
@@ -350,7 +349,7 @@ static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
350 if (adev->vram_scratch.robj == NULL) { 349 if (adev->vram_scratch.robj == NULL) {
351 return; 350 return;
352 } 351 }
353 r = amdgpu_bo_reserve(adev->vram_scratch.robj, false); 352 r = amdgpu_bo_reserve(adev->vram_scratch.robj, true);
354 if (likely(r == 0)) { 353 if (likely(r == 0)) {
355 amdgpu_bo_kunmap(adev->vram_scratch.robj); 354 amdgpu_bo_kunmap(adev->vram_scratch.robj);
356 amdgpu_bo_unpin(adev->vram_scratch.robj); 355 amdgpu_bo_unpin(adev->vram_scratch.robj);
@@ -422,12 +421,11 @@ static int amdgpu_doorbell_init(struct amdgpu_device *adev)
422 if (adev->doorbell.num_doorbells == 0) 421 if (adev->doorbell.num_doorbells == 0)
423 return -EINVAL; 422 return -EINVAL;
424 423
425 adev->doorbell.ptr = ioremap(adev->doorbell.base, adev->doorbell.num_doorbells * sizeof(u32)); 424 adev->doorbell.ptr = ioremap(adev->doorbell.base,
426 if (adev->doorbell.ptr == NULL) { 425 adev->doorbell.num_doorbells *
426 sizeof(u32));
427 if (adev->doorbell.ptr == NULL)
427 return -ENOMEM; 428 return -ENOMEM;
428 }
429 DRM_INFO("doorbell mmio base: 0x%08X\n", (uint32_t)adev->doorbell.base);
430 DRM_INFO("doorbell mmio size: %u\n", (unsigned)adev->doorbell.size);
431 429
432 return 0; 430 return 0;
433} 431}
@@ -1584,9 +1582,6 @@ static int amdgpu_late_init(struct amdgpu_device *adev)
1584 } 1582 }
1585 } 1583 }
1586 1584
1587 amdgpu_dpm_enable_uvd(adev, false);
1588 amdgpu_dpm_enable_vce(adev, false);
1589
1590 return 0; 1585 return 0;
1591} 1586}
1592 1587
@@ -1854,7 +1849,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
1854 1849
1855 /* mutex initialization are all done here so we 1850 /* mutex initialization are all done here so we
1856 * can recall function without having locking issues */ 1851 * can recall function without having locking issues */
1857 mutex_init(&adev->vm_manager.lock);
1858 atomic_set(&adev->irq.ih.lock, 0); 1852 atomic_set(&adev->irq.ih.lock, 0);
1859 mutex_init(&adev->firmware.mutex); 1853 mutex_init(&adev->firmware.mutex);
1860 mutex_init(&adev->pm.mutex); 1854 mutex_init(&adev->pm.mutex);
@@ -2071,7 +2065,8 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
2071 2065
2072 DRM_INFO("amdgpu: finishing device.\n"); 2066 DRM_INFO("amdgpu: finishing device.\n");
2073 adev->shutdown = true; 2067 adev->shutdown = true;
2074 drm_crtc_force_disable_all(adev->ddev); 2068 if (adev->mode_info.mode_config_initialized)
2069 drm_crtc_force_disable_all(adev->ddev);
2075 /* evict vram memory */ 2070 /* evict vram memory */
2076 amdgpu_bo_evict_vram(adev); 2071 amdgpu_bo_evict_vram(adev);
2077 amdgpu_ib_pool_fini(adev); 2072 amdgpu_ib_pool_fini(adev);
@@ -2146,7 +2141,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2146 2141
2147 if (amdgpu_crtc->cursor_bo) { 2142 if (amdgpu_crtc->cursor_bo) {
2148 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); 2143 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2149 r = amdgpu_bo_reserve(aobj, false); 2144 r = amdgpu_bo_reserve(aobj, true);
2150 if (r == 0) { 2145 if (r == 0) {
2151 amdgpu_bo_unpin(aobj); 2146 amdgpu_bo_unpin(aobj);
2152 amdgpu_bo_unreserve(aobj); 2147 amdgpu_bo_unreserve(aobj);
@@ -2159,7 +2154,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2159 robj = gem_to_amdgpu_bo(rfb->obj); 2154 robj = gem_to_amdgpu_bo(rfb->obj);
2160 /* don't unpin kernel fb objects */ 2155 /* don't unpin kernel fb objects */
2161 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) { 2156 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2162 r = amdgpu_bo_reserve(robj, false); 2157 r = amdgpu_bo_reserve(robj, true);
2163 if (r == 0) { 2158 if (r == 0) {
2164 amdgpu_bo_unpin(robj); 2159 amdgpu_bo_unpin(robj);
2165 amdgpu_bo_unreserve(robj); 2160 amdgpu_bo_unreserve(robj);
@@ -2216,7 +2211,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2216 struct drm_connector *connector; 2211 struct drm_connector *connector;
2217 struct amdgpu_device *adev = dev->dev_private; 2212 struct amdgpu_device *adev = dev->dev_private;
2218 struct drm_crtc *crtc; 2213 struct drm_crtc *crtc;
2219 int r; 2214 int r = 0;
2220 2215
2221 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 2216 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2222 return 0; 2217 return 0;
@@ -2228,11 +2223,8 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2228 pci_set_power_state(dev->pdev, PCI_D0); 2223 pci_set_power_state(dev->pdev, PCI_D0);
2229 pci_restore_state(dev->pdev); 2224 pci_restore_state(dev->pdev);
2230 r = pci_enable_device(dev->pdev); 2225 r = pci_enable_device(dev->pdev);
2231 if (r) { 2226 if (r)
2232 if (fbcon) 2227 goto unlock;
2233 console_unlock();
2234 return r;
2235 }
2236 } 2228 }
2237 if (adev->is_atom_fw) 2229 if (adev->is_atom_fw)
2238 amdgpu_atomfirmware_scratch_regs_restore(adev); 2230 amdgpu_atomfirmware_scratch_regs_restore(adev);
@@ -2249,7 +2241,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2249 r = amdgpu_resume(adev); 2241 r = amdgpu_resume(adev);
2250 if (r) { 2242 if (r) {
2251 DRM_ERROR("amdgpu_resume failed (%d).\n", r); 2243 DRM_ERROR("amdgpu_resume failed (%d).\n", r);
2252 return r; 2244 goto unlock;
2253 } 2245 }
2254 amdgpu_fence_driver_resume(adev); 2246 amdgpu_fence_driver_resume(adev);
2255 2247
@@ -2260,11 +2252,8 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2260 } 2252 }
2261 2253
2262 r = amdgpu_late_init(adev); 2254 r = amdgpu_late_init(adev);
2263 if (r) { 2255 if (r)
2264 if (fbcon) 2256 goto unlock;
2265 console_unlock();
2266 return r;
2267 }
2268 2257
2269 /* pin cursors */ 2258 /* pin cursors */
2270 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { 2259 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
@@ -2272,7 +2261,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2272 2261
2273 if (amdgpu_crtc->cursor_bo) { 2262 if (amdgpu_crtc->cursor_bo) {
2274 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); 2263 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2275 r = amdgpu_bo_reserve(aobj, false); 2264 r = amdgpu_bo_reserve(aobj, true);
2276 if (r == 0) { 2265 if (r == 0) {
2277 r = amdgpu_bo_pin(aobj, 2266 r = amdgpu_bo_pin(aobj,
2278 AMDGPU_GEM_DOMAIN_VRAM, 2267 AMDGPU_GEM_DOMAIN_VRAM,
@@ -2314,12 +2303,14 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2314 dev->dev->power.disable_depth--; 2303 dev->dev->power.disable_depth--;
2315#endif 2304#endif
2316 2305
2317 if (fbcon) { 2306 if (fbcon)
2318 amdgpu_fbdev_set_suspend(adev, 0); 2307 amdgpu_fbdev_set_suspend(adev, 0);
2308
2309unlock:
2310 if (fbcon)
2319 console_unlock(); 2311 console_unlock();
2320 }
2321 2312
2322 return 0; 2313 return r;
2323} 2314}
2324 2315
2325static bool amdgpu_check_soft_reset(struct amdgpu_device *adev) 2316static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
@@ -2430,25 +2421,37 @@ static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
2430 uint32_t domain; 2421 uint32_t domain;
2431 int r; 2422 int r;
2432 2423
2433 if (!bo->shadow) 2424 if (!bo->shadow)
2434 return 0; 2425 return 0;
2426
2427 r = amdgpu_bo_reserve(bo, true);
2428 if (r)
2429 return r;
2430 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2431 /* if bo has been evicted, then no need to recover */
2432 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2433 r = amdgpu_bo_validate(bo->shadow);
2434 if (r) {
2435 DRM_ERROR("bo validate failed!\n");
2436 goto err;
2437 }
2435 2438
2436 r = amdgpu_bo_reserve(bo, false); 2439 r = amdgpu_ttm_bind(&bo->shadow->tbo, &bo->shadow->tbo.mem);
2437 if (r) 2440 if (r) {
2438 return r; 2441 DRM_ERROR("%p bind failed\n", bo->shadow);
2439 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); 2442 goto err;
2440 /* if bo has been evicted, then no need to recover */ 2443 }
2441 if (domain == AMDGPU_GEM_DOMAIN_VRAM) { 2444
2442 r = amdgpu_bo_restore_from_shadow(adev, ring, bo, 2445 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2443 NULL, fence, true); 2446 NULL, fence, true);
2444 if (r) { 2447 if (r) {
2445 DRM_ERROR("recover page table failed!\n"); 2448 DRM_ERROR("recover page table failed!\n");
2446 goto err; 2449 goto err;
2447 } 2450 }
2448 } 2451 }
2449err: 2452err:
2450 amdgpu_bo_unreserve(bo); 2453 amdgpu_bo_unreserve(bo);
2451 return r; 2454 return r;
2452} 2455}
2453 2456
2454/** 2457/**
@@ -2520,6 +2523,7 @@ int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, bool voluntary)
2520 ring = adev->mman.buffer_funcs_ring; 2523 ring = adev->mman.buffer_funcs_ring;
2521 mutex_lock(&adev->shadow_list_lock); 2524 mutex_lock(&adev->shadow_list_lock);
2522 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) { 2525 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2526 next = NULL;
2523 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next); 2527 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2524 if (fence) { 2528 if (fence) {
2525 r = dma_fence_wait(fence, false); 2529 r = dma_fence_wait(fence, false);
@@ -2593,7 +2597,7 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
2593 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 2597 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2594 struct amdgpu_ring *ring = adev->rings[i]; 2598 struct amdgpu_ring *ring = adev->rings[i];
2595 2599
2596 if (!ring) 2600 if (!ring || !ring->sched.thread)
2597 continue; 2601 continue;
2598 kthread_park(ring->sched.thread); 2602 kthread_park(ring->sched.thread);
2599 amd_sched_hw_job_reset(&ring->sched); 2603 amd_sched_hw_job_reset(&ring->sched);
@@ -2666,6 +2670,7 @@ retry:
2666 DRM_INFO("recover vram bo from shadow\n"); 2670 DRM_INFO("recover vram bo from shadow\n");
2667 mutex_lock(&adev->shadow_list_lock); 2671 mutex_lock(&adev->shadow_list_lock);
2668 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) { 2672 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2673 next = NULL;
2669 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next); 2674 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2670 if (fence) { 2675 if (fence) {
2671 r = dma_fence_wait(fence, false); 2676 r = dma_fence_wait(fence, false);
@@ -2688,7 +2693,8 @@ retry:
2688 } 2693 }
2689 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 2694 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2690 struct amdgpu_ring *ring = adev->rings[i]; 2695 struct amdgpu_ring *ring = adev->rings[i];
2691 if (!ring) 2696
2697 if (!ring || !ring->sched.thread)
2692 continue; 2698 continue;
2693 2699
2694 amd_sched_job_recovery(&ring->sched); 2700 amd_sched_job_recovery(&ring->sched);
@@ -2697,7 +2703,7 @@ retry:
2697 } else { 2703 } else {
2698 dev_err(adev->dev, "asic resume failed (%d).\n", r); 2704 dev_err(adev->dev, "asic resume failed (%d).\n", r);
2699 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 2705 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2700 if (adev->rings[i]) { 2706 if (adev->rings[i] && adev->rings[i]->sched.thread) {
2701 kthread_unpark(adev->rings[i]->sched.thread); 2707 kthread_unpark(adev->rings[i]->sched.thread);
2702 } 2708 }
2703 } 2709 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index 96926a221bd5..cdf2ab20166a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -123,7 +123,7 @@ static void amdgpu_unpin_work_func(struct work_struct *__work)
123 int r; 123 int r;
124 124
125 /* unpin of the old buffer */ 125 /* unpin of the old buffer */
126 r = amdgpu_bo_reserve(work->old_abo, false); 126 r = amdgpu_bo_reserve(work->old_abo, true);
127 if (likely(r == 0)) { 127 if (likely(r == 0)) {
128 r = amdgpu_bo_unpin(work->old_abo); 128 r = amdgpu_bo_unpin(work->old_abo);
129 if (unlikely(r != 0)) { 129 if (unlikely(r != 0)) {
@@ -138,52 +138,11 @@ static void amdgpu_unpin_work_func(struct work_struct *__work)
138 kfree(work); 138 kfree(work);
139} 139}
140 140
141 141int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
142static void amdgpu_flip_work_cleanup(struct amdgpu_flip_work *work) 142 struct drm_framebuffer *fb,
143{ 143 struct drm_pending_vblank_event *event,
144 int i; 144 uint32_t page_flip_flags, uint32_t target,
145 145 struct drm_modeset_acquire_ctx *ctx)
146 amdgpu_bo_unref(&work->old_abo);
147 dma_fence_put(work->excl);
148 for (i = 0; i < work->shared_count; ++i)
149 dma_fence_put(work->shared[i]);
150 kfree(work->shared);
151 kfree(work);
152}
153
154static void amdgpu_flip_cleanup_unreserve(struct amdgpu_flip_work *work,
155 struct amdgpu_bo *new_abo)
156{
157 amdgpu_bo_unreserve(new_abo);
158 amdgpu_flip_work_cleanup(work);
159}
160
161static void amdgpu_flip_cleanup_unpin(struct amdgpu_flip_work *work,
162 struct amdgpu_bo *new_abo)
163{
164 if (unlikely(amdgpu_bo_unpin(new_abo) != 0))
165 DRM_ERROR("failed to unpin new abo in error path\n");
166 amdgpu_flip_cleanup_unreserve(work, new_abo);
167}
168
169void amdgpu_crtc_cleanup_flip_ctx(struct amdgpu_flip_work *work,
170 struct amdgpu_bo *new_abo)
171{
172 if (unlikely(amdgpu_bo_reserve(new_abo, false) != 0)) {
173 DRM_ERROR("failed to reserve new abo in error path\n");
174 amdgpu_flip_work_cleanup(work);
175 return;
176 }
177 amdgpu_flip_cleanup_unpin(work, new_abo);
178}
179
180int amdgpu_crtc_prepare_flip(struct drm_crtc *crtc,
181 struct drm_framebuffer *fb,
182 struct drm_pending_vblank_event *event,
183 uint32_t page_flip_flags,
184 uint32_t target,
185 struct amdgpu_flip_work **work_p,
186 struct amdgpu_bo **new_abo_p)
187{ 146{
188 struct drm_device *dev = crtc->dev; 147 struct drm_device *dev = crtc->dev;
189 struct amdgpu_device *adev = dev->dev_private; 148 struct amdgpu_device *adev = dev->dev_private;
@@ -196,7 +155,7 @@ int amdgpu_crtc_prepare_flip(struct drm_crtc *crtc,
196 unsigned long flags; 155 unsigned long flags;
197 u64 tiling_flags; 156 u64 tiling_flags;
198 u64 base; 157 u64 base;
199 int r; 158 int i, r;
200 159
201 work = kzalloc(sizeof *work, GFP_KERNEL); 160 work = kzalloc(sizeof *work, GFP_KERNEL);
202 if (work == NULL) 161 if (work == NULL)
@@ -257,80 +216,41 @@ int amdgpu_crtc_prepare_flip(struct drm_crtc *crtc,
257 spin_unlock_irqrestore(&crtc->dev->event_lock, flags); 216 spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
258 r = -EBUSY; 217 r = -EBUSY;
259 goto pflip_cleanup; 218 goto pflip_cleanup;
260
261 } 219 }
262 spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
263
264 *work_p = work;
265 *new_abo_p = new_abo;
266
267 return 0;
268
269pflip_cleanup:
270 amdgpu_crtc_cleanup_flip_ctx(work, new_abo);
271 return r;
272
273unpin:
274 amdgpu_flip_cleanup_unpin(work, new_abo);
275 return r;
276
277unreserve:
278 amdgpu_flip_cleanup_unreserve(work, new_abo);
279 return r;
280 220
281cleanup:
282 amdgpu_flip_work_cleanup(work);
283 return r;
284
285}
286
287void amdgpu_crtc_submit_flip(struct drm_crtc *crtc,
288 struct drm_framebuffer *fb,
289 struct amdgpu_flip_work *work,
290 struct amdgpu_bo *new_abo)
291{
292 unsigned long flags;
293 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
294
295 spin_lock_irqsave(&crtc->dev->event_lock, flags);
296 amdgpu_crtc->pflip_status = AMDGPU_FLIP_PENDING; 221 amdgpu_crtc->pflip_status = AMDGPU_FLIP_PENDING;
297 amdgpu_crtc->pflip_works = work; 222 amdgpu_crtc->pflip_works = work;
298 223
224
225 DRM_DEBUG_DRIVER("crtc:%d[%p], pflip_stat:AMDGPU_FLIP_PENDING, work: %p,\n",
226 amdgpu_crtc->crtc_id, amdgpu_crtc, work);
299 /* update crtc fb */ 227 /* update crtc fb */
300 crtc->primary->fb = fb; 228 crtc->primary->fb = fb;
301 spin_unlock_irqrestore(&crtc->dev->event_lock, flags); 229 spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
302
303 DRM_DEBUG_DRIVER(
304 "crtc:%d[%p], pflip_stat:AMDGPU_FLIP_PENDING, work: %p,\n",
305 amdgpu_crtc->crtc_id, amdgpu_crtc, work);
306
307 amdgpu_flip_work_func(&work->flip_work.work); 230 amdgpu_flip_work_func(&work->flip_work.work);
308} 231 return 0;
309
310int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
311 struct drm_framebuffer *fb,
312 struct drm_pending_vblank_event *event,
313 uint32_t page_flip_flags,
314 uint32_t target,
315 struct drm_modeset_acquire_ctx *ctx)
316{
317 struct amdgpu_bo *new_abo;
318 struct amdgpu_flip_work *work;
319 int r;
320 232
321 r = amdgpu_crtc_prepare_flip(crtc, 233pflip_cleanup:
322 fb, 234 if (unlikely(amdgpu_bo_reserve(new_abo, false) != 0)) {
323 event, 235 DRM_ERROR("failed to reserve new abo in error path\n");
324 page_flip_flags, 236 goto cleanup;
325 target, 237 }
326 &work, 238unpin:
327 &new_abo); 239 if (unlikely(amdgpu_bo_unpin(new_abo) != 0)) {
328 if (r) 240 DRM_ERROR("failed to unpin new abo in error path\n");
329 return r; 241 }
242unreserve:
243 amdgpu_bo_unreserve(new_abo);
330 244
331 amdgpu_crtc_submit_flip(crtc, fb, work, new_abo); 245cleanup:
246 amdgpu_bo_unref(&work->old_abo);
247 dma_fence_put(work->excl);
248 for (i = 0; i < work->shared_count; ++i)
249 dma_fence_put(work->shared[i]);
250 kfree(work->shared);
251 kfree(work);
332 252
333 return 0; 253 return r;
334} 254}
335 255
336int amdgpu_crtc_set_config(struct drm_mode_set *set, 256int amdgpu_crtc_set_config(struct drm_mode_set *set,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 4e0f7d2d87f1..f2d705e6a75a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -63,9 +63,11 @@
63 * - 3.11.0 - Add support for sensor query info (clocks, temp, etc). 63 * - 3.11.0 - Add support for sensor query info (clocks, temp, etc).
64 * - 3.12.0 - Add query for double offchip LDS buffers 64 * - 3.12.0 - Add query for double offchip LDS buffers
65 * - 3.13.0 - Add PRT support 65 * - 3.13.0 - Add PRT support
66 * - 3.14.0 - Fix race in amdgpu_ctx_get_fence() and note new functionality
67 * - 3.15.0 - Export more gpu info for gfx9
66 */ 68 */
67#define KMS_DRIVER_MAJOR 3 69#define KMS_DRIVER_MAJOR 3
68#define KMS_DRIVER_MINOR 13 70#define KMS_DRIVER_MINOR 15
69#define KMS_DRIVER_PATCHLEVEL 0 71#define KMS_DRIVER_PATCHLEVEL 0
70 72
71int amdgpu_vram_limit = 0; 73int amdgpu_vram_limit = 0;
@@ -453,7 +455,9 @@ static const struct pci_device_id pciidlist[] = {
453 {0x1002, 0x6861, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT}, 455 {0x1002, 0x6861, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
454 {0x1002, 0x6862, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT}, 456 {0x1002, 0x6862, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
455 {0x1002, 0x6863, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT}, 457 {0x1002, 0x6863, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
458 {0x1002, 0x6864, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
456 {0x1002, 0x6867, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT}, 459 {0x1002, 0x6867, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
460 {0x1002, 0x6868, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
457 {0x1002, 0x686c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT}, 461 {0x1002, 0x686c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
458 {0x1002, 0x687f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT}, 462 {0x1002, 0x687f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10|AMD_EXP_HW_SUPPORT},
459 {0, 0, 0} 463 {0, 0, 0}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index a48142d930c6..236d9950221b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -112,7 +112,7 @@ static void amdgpufb_destroy_pinned_object(struct drm_gem_object *gobj)
112 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj); 112 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
113 int ret; 113 int ret;
114 114
115 ret = amdgpu_bo_reserve(abo, false); 115 ret = amdgpu_bo_reserve(abo, true);
116 if (likely(ret == 0)) { 116 if (likely(ret == 0)) {
117 amdgpu_bo_kunmap(abo); 117 amdgpu_bo_kunmap(abo);
118 amdgpu_bo_unpin(abo); 118 amdgpu_bo_unpin(abo);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index 2ee327d69775..902e6015abca 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -186,7 +186,7 @@ void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev)
186 if (adev->gart.robj == NULL) { 186 if (adev->gart.robj == NULL) {
187 return; 187 return;
188 } 188 }
189 r = amdgpu_bo_reserve(adev->gart.robj, false); 189 r = amdgpu_bo_reserve(adev->gart.robj, true);
190 if (likely(r == 0)) { 190 if (likely(r == 0)) {
191 amdgpu_bo_kunmap(adev->gart.robj); 191 amdgpu_bo_kunmap(adev->gart.robj);
192 amdgpu_bo_unpin(adev->gart.robj); 192 amdgpu_bo_unpin(adev->gart.robj);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 03a9c5cad222..94cb91cf93eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -139,6 +139,35 @@ int amdgpu_gem_object_open(struct drm_gem_object *obj,
139 return 0; 139 return 0;
140} 140}
141 141
142static int amdgpu_gem_vm_check(void *param, struct amdgpu_bo *bo)
143{
144 /* if anything is swapped out don't swap it in here,
145 just abort and wait for the next CS */
146 if (!amdgpu_bo_gpu_accessible(bo))
147 return -ERESTARTSYS;
148
149 if (bo->shadow && !amdgpu_bo_gpu_accessible(bo->shadow))
150 return -ERESTARTSYS;
151
152 return 0;
153}
154
155static bool amdgpu_gem_vm_ready(struct amdgpu_device *adev,
156 struct amdgpu_vm *vm,
157 struct list_head *list)
158{
159 struct ttm_validate_buffer *entry;
160
161 list_for_each_entry(entry, list, head) {
162 struct amdgpu_bo *bo =
163 container_of(entry->bo, struct amdgpu_bo, tbo);
164 if (amdgpu_gem_vm_check(NULL, bo))
165 return false;
166 }
167
168 return !amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_gem_vm_check, NULL);
169}
170
142void amdgpu_gem_object_close(struct drm_gem_object *obj, 171void amdgpu_gem_object_close(struct drm_gem_object *obj,
143 struct drm_file *file_priv) 172 struct drm_file *file_priv)
144{ 173{
@@ -148,15 +177,13 @@ void amdgpu_gem_object_close(struct drm_gem_object *obj,
148 struct amdgpu_vm *vm = &fpriv->vm; 177 struct amdgpu_vm *vm = &fpriv->vm;
149 178
150 struct amdgpu_bo_list_entry vm_pd; 179 struct amdgpu_bo_list_entry vm_pd;
151 struct list_head list, duplicates; 180 struct list_head list;
152 struct ttm_validate_buffer tv; 181 struct ttm_validate_buffer tv;
153 struct ww_acquire_ctx ticket; 182 struct ww_acquire_ctx ticket;
154 struct amdgpu_bo_va *bo_va; 183 struct amdgpu_bo_va *bo_va;
155 struct dma_fence *fence = NULL;
156 int r; 184 int r;
157 185
158 INIT_LIST_HEAD(&list); 186 INIT_LIST_HEAD(&list);
159 INIT_LIST_HEAD(&duplicates);
160 187
161 tv.bo = &bo->tbo; 188 tv.bo = &bo->tbo;
162 tv.shared = true; 189 tv.shared = true;
@@ -164,16 +191,18 @@ void amdgpu_gem_object_close(struct drm_gem_object *obj,
164 191
165 amdgpu_vm_get_pd_bo(vm, &list, &vm_pd); 192 amdgpu_vm_get_pd_bo(vm, &list, &vm_pd);
166 193
167 r = ttm_eu_reserve_buffers(&ticket, &list, false, &duplicates); 194 r = ttm_eu_reserve_buffers(&ticket, &list, false, NULL);
168 if (r) { 195 if (r) {
169 dev_err(adev->dev, "leaking bo va because " 196 dev_err(adev->dev, "leaking bo va because "
170 "we fail to reserve bo (%d)\n", r); 197 "we fail to reserve bo (%d)\n", r);
171 return; 198 return;
172 } 199 }
173 bo_va = amdgpu_vm_bo_find(vm, bo); 200 bo_va = amdgpu_vm_bo_find(vm, bo);
174 if (bo_va) { 201 if (bo_va && --bo_va->ref_count == 0) {
175 if (--bo_va->ref_count == 0) { 202 amdgpu_vm_bo_rmv(adev, bo_va);
176 amdgpu_vm_bo_rmv(adev, bo_va); 203
204 if (amdgpu_gem_vm_ready(adev, vm, &list)) {
205 struct dma_fence *fence = NULL;
177 206
178 r = amdgpu_vm_clear_freed(adev, vm, &fence); 207 r = amdgpu_vm_clear_freed(adev, vm, &fence);
179 if (unlikely(r)) { 208 if (unlikely(r)) {
@@ -502,19 +531,6 @@ out:
502 return r; 531 return r;
503} 532}
504 533
505static int amdgpu_gem_va_check(void *param, struct amdgpu_bo *bo)
506{
507 /* if anything is swapped out don't swap it in here,
508 just abort and wait for the next CS */
509 if (!amdgpu_bo_gpu_accessible(bo))
510 return -ERESTARTSYS;
511
512 if (bo->shadow && !amdgpu_bo_gpu_accessible(bo->shadow))
513 return -ERESTARTSYS;
514
515 return 0;
516}
517
518/** 534/**
519 * amdgpu_gem_va_update_vm -update the bo_va in its VM 535 * amdgpu_gem_va_update_vm -update the bo_va in its VM
520 * 536 *
@@ -533,19 +549,9 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
533 struct list_head *list, 549 struct list_head *list,
534 uint32_t operation) 550 uint32_t operation)
535{ 551{
536 struct ttm_validate_buffer *entry;
537 int r = -ERESTARTSYS; 552 int r = -ERESTARTSYS;
538 553
539 list_for_each_entry(entry, list, head) { 554 if (!amdgpu_gem_vm_ready(adev, vm, list))
540 struct amdgpu_bo *bo =
541 container_of(entry->bo, struct amdgpu_bo, tbo);
542 if (amdgpu_gem_va_check(NULL, bo))
543 goto error;
544 }
545
546 r = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_gem_va_check,
547 NULL);
548 if (r)
549 goto error; 555 goto error;
550 556
551 r = amdgpu_vm_update_directories(adev, vm); 557 r = amdgpu_vm_update_directories(adev, vm);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index 0335c2f331e9..f7d22c44034d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -134,6 +134,15 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
134 return r; 134 return r;
135} 135}
136 136
137void amdgpu_gtt_mgr_print(struct seq_file *m, struct ttm_mem_type_manager *man)
138{
139 struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
140 struct amdgpu_gtt_mgr *mgr = man->priv;
141
142 seq_printf(m, "man size:%llu pages, gtt available:%llu pages, usage:%lluMB\n",
143 man->size, mgr->available, (u64)atomic64_read(&adev->gtt_usage) >> 20);
144
145}
137/** 146/**
138 * amdgpu_gtt_mgr_new - allocate a new node 147 * amdgpu_gtt_mgr_new - allocate a new node
139 * 148 *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index aab857d89d03..6e4ae0d983c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -160,6 +160,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
160 dev_err(adev->dev, "scheduling IB failed (%d).\n", r); 160 dev_err(adev->dev, "scheduling IB failed (%d).\n", r);
161 return r; 161 return r;
162 } 162 }
163 if (ring->funcs->emit_pipeline_sync && job && job->need_pipeline_sync)
164 amdgpu_ring_emit_pipeline_sync(ring);
163 165
164 if (vm) { 166 if (vm) {
165 r = amdgpu_vm_flush(ring, job); 167 r = amdgpu_vm_flush(ring, job);
@@ -217,7 +219,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
217 if (r) { 219 if (r) {
218 dev_err(adev->dev, "failed to emit fence (%d)\n", r); 220 dev_err(adev->dev, "failed to emit fence (%d)\n", r);
219 if (job && job->vm_id) 221 if (job && job->vm_id)
220 amdgpu_vm_reset_id(adev, job->vm_id); 222 amdgpu_vm_reset_id(adev, ring->funcs->vmhub,
223 job->vm_id);
221 amdgpu_ring_undo(ring); 224 amdgpu_ring_undo(ring);
222 return r; 225 return r;
223 } 226 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 86a12424c162..7570f2439a11 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -57,6 +57,7 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
57 (*job)->vm = vm; 57 (*job)->vm = vm;
58 (*job)->ibs = (void *)&(*job)[1]; 58 (*job)->ibs = (void *)&(*job)[1];
59 (*job)->num_ibs = num_ibs; 59 (*job)->num_ibs = num_ibs;
60 (*job)->need_pipeline_sync = false;
60 61
61 amdgpu_sync_create(&(*job)->sync); 62 amdgpu_sync_create(&(*job)->sync);
62 63
@@ -139,7 +140,7 @@ static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
139 140
140 struct dma_fence *fence = amdgpu_sync_get_fence(&job->sync); 141 struct dma_fence *fence = amdgpu_sync_get_fence(&job->sync);
141 142
142 if (fence == NULL && vm && !job->vm_id) { 143 while (fence == NULL && vm && !job->vm_id) {
143 struct amdgpu_ring *ring = job->ring; 144 struct amdgpu_ring *ring = job->ring;
144 int r; 145 int r;
145 146
@@ -152,6 +153,9 @@ static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
152 fence = amdgpu_sync_get_fence(&job->sync); 153 fence = amdgpu_sync_get_fence(&job->sync);
153 } 154 }
154 155
156 if (amd_sched_dependency_optimized(fence, sched_job->s_entity))
157 job->need_pipeline_sync = true;
158
155 return fence; 159 return fence;
156} 160}
157 161
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 832be632478f..96c341670782 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -545,11 +545,22 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
545 adev->gfx.config.double_offchip_lds_buf; 545 adev->gfx.config.double_offchip_lds_buf;
546 546
547 if (amdgpu_ngg) { 547 if (amdgpu_ngg) {
548 dev_info.prim_buf_gpu_addr = adev->gfx.ngg.buf[PRIM].gpu_addr; 548 dev_info.prim_buf_gpu_addr = adev->gfx.ngg.buf[NGG_PRIM].gpu_addr;
549 dev_info.pos_buf_gpu_addr = adev->gfx.ngg.buf[POS].gpu_addr; 549 dev_info.prim_buf_size = adev->gfx.ngg.buf[NGG_PRIM].size;
550 dev_info.cntl_sb_buf_gpu_addr = adev->gfx.ngg.buf[CNTL].gpu_addr; 550 dev_info.pos_buf_gpu_addr = adev->gfx.ngg.buf[NGG_POS].gpu_addr;
551 dev_info.param_buf_gpu_addr = adev->gfx.ngg.buf[PARAM].gpu_addr; 551 dev_info.pos_buf_size = adev->gfx.ngg.buf[NGG_POS].size;
552 dev_info.cntl_sb_buf_gpu_addr = adev->gfx.ngg.buf[NGG_CNTL].gpu_addr;
553 dev_info.cntl_sb_buf_size = adev->gfx.ngg.buf[NGG_CNTL].size;
554 dev_info.param_buf_gpu_addr = adev->gfx.ngg.buf[NGG_PARAM].gpu_addr;
555 dev_info.param_buf_size = adev->gfx.ngg.buf[NGG_PARAM].size;
552 } 556 }
557 dev_info.wave_front_size = adev->gfx.cu_info.wave_front_size;
558 dev_info.num_shader_visible_vgprs = adev->gfx.config.max_gprs;
559 dev_info.num_cu_per_sh = adev->gfx.config.max_cu_per_sh;
560 dev_info.num_tcc_blocks = adev->gfx.config.max_texture_channel_caches;
561 dev_info.gs_vgt_table_depth = adev->gfx.config.gs_vgt_table_depth;
562 dev_info.gs_prim_buffer_depth = adev->gfx.config.gs_prim_buffer_depth;
563 dev_info.max_gs_waves_per_vgt = adev->gfx.config.max_gs_threads;
553 564
554 return copy_to_user(out, &dev_info, 565 return copy_to_user(out, &dev_info,
555 min((size_t)size, sizeof(dev_info))) ? -EFAULT : 0; 566 min((size_t)size, sizeof(dev_info))) ? -EFAULT : 0;
@@ -810,7 +821,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
810 821
811 if (amdgpu_sriov_vf(adev)) { 822 if (amdgpu_sriov_vf(adev)) {
812 /* TODO: how to handle reserve failure */ 823 /* TODO: how to handle reserve failure */
813 BUG_ON(amdgpu_bo_reserve(adev->virt.csa_obj, false)); 824 BUG_ON(amdgpu_bo_reserve(adev->virt.csa_obj, true));
814 amdgpu_vm_bo_rmv(adev, fpriv->vm.csa_bo_va); 825 amdgpu_vm_bo_rmv(adev, fpriv->vm.csa_bo_va);
815 fpriv->vm.csa_bo_va = NULL; 826 fpriv->vm.csa_bo_va = NULL;
816 amdgpu_bo_unreserve(adev->virt.csa_obj); 827 amdgpu_bo_unreserve(adev->virt.csa_obj);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index db8f8dda209c..dbd10618ec20 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -597,21 +597,6 @@ int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
597 struct drm_pending_vblank_event *event, 597 struct drm_pending_vblank_event *event,
598 uint32_t page_flip_flags, uint32_t target, 598 uint32_t page_flip_flags, uint32_t target,
599 struct drm_modeset_acquire_ctx *ctx); 599 struct drm_modeset_acquire_ctx *ctx);
600void amdgpu_crtc_cleanup_flip_ctx(struct amdgpu_flip_work *work,
601 struct amdgpu_bo *new_abo);
602int amdgpu_crtc_prepare_flip(struct drm_crtc *crtc,
603 struct drm_framebuffer *fb,
604 struct drm_pending_vblank_event *event,
605 uint32_t page_flip_flags,
606 uint32_t target,
607 struct amdgpu_flip_work **work,
608 struct amdgpu_bo **new_abo);
609
610void amdgpu_crtc_submit_flip(struct drm_crtc *crtc,
611 struct drm_framebuffer *fb,
612 struct amdgpu_flip_work *work,
613 struct amdgpu_bo *new_abo);
614
615extern const struct drm_mode_config_funcs amdgpu_mode_funcs; 600extern const struct drm_mode_config_funcs amdgpu_mode_funcs;
616 601
617#endif 602#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index cb89fff863c0..365883d7948d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -295,7 +295,7 @@ void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
295 if (*bo == NULL) 295 if (*bo == NULL)
296 return; 296 return;
297 297
298 if (likely(amdgpu_bo_reserve(*bo, false) == 0)) { 298 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
299 if (cpu_addr) 299 if (cpu_addr)
300 amdgpu_bo_kunmap(*bo); 300 amdgpu_bo_kunmap(*bo);
301 301
@@ -543,6 +543,27 @@ err:
543 return r; 543 return r;
544} 544}
545 545
546int amdgpu_bo_validate(struct amdgpu_bo *bo)
547{
548 uint32_t domain;
549 int r;
550
551 if (bo->pin_count)
552 return 0;
553
554 domain = bo->prefered_domains;
555
556retry:
557 amdgpu_ttm_placement_from_domain(bo, domain);
558 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
559 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
560 domain = bo->allowed_domains;
561 goto retry;
562 }
563
564 return r;
565}
566
546int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev, 567int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
547 struct amdgpu_ring *ring, 568 struct amdgpu_ring *ring,
548 struct amdgpu_bo *bo, 569 struct amdgpu_bo *bo,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 15a723adca76..382485115b06 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -175,6 +175,7 @@ int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
175 struct amdgpu_bo *bo, 175 struct amdgpu_bo *bo,
176 struct reservation_object *resv, 176 struct reservation_object *resv,
177 struct dma_fence **fence, bool direct); 177 struct dma_fence **fence, bool direct);
178int amdgpu_bo_validate(struct amdgpu_bo *bo);
178int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev, 179int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
179 struct amdgpu_ring *ring, 180 struct amdgpu_ring *ring,
180 struct amdgpu_bo *bo, 181 struct amdgpu_bo *bo,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index 990fde2cf4fd..7df503aedb69 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -867,8 +867,7 @@ static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev,
867 867
868 pwm_mode = amdgpu_dpm_get_fan_control_mode(adev); 868 pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
869 869
870 /* never 0 (full-speed), fuse or smc-controlled always */ 870 return sprintf(buf, "%i\n", pwm_mode);
871 return sprintf(buf, "%i\n", pwm_mode == FDO_PWM_MODE_STATIC ? 1 : 2);
872} 871}
873 872
874static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev, 873static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev,
@@ -887,14 +886,7 @@ static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev,
887 if (err) 886 if (err)
888 return err; 887 return err;
889 888
890 switch (value) { 889 amdgpu_dpm_set_fan_control_mode(adev, value);
891 case 1: /* manual, percent-based */
892 amdgpu_dpm_set_fan_control_mode(adev, FDO_PWM_MODE_STATIC);
893 break;
894 default: /* disable */
895 amdgpu_dpm_set_fan_control_mode(adev, 0);
896 break;
897 }
898 890
899 return count; 891 return count;
900} 892}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
index 3826d5aea0a6..6bdc866570ab 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
@@ -113,7 +113,7 @@ void amdgpu_gem_prime_unpin(struct drm_gem_object *obj)
113 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 113 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
114 int ret = 0; 114 int ret = 0;
115 115
116 ret = amdgpu_bo_reserve(bo, false); 116 ret = amdgpu_bo_reserve(bo, true);
117 if (unlikely(ret != 0)) 117 if (unlikely(ret != 0))
118 return; 118 return;
119 119
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index ed6e5799016e..ac5e92e5d59d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -55,6 +55,8 @@ static int psp_sw_init(void *handle)
55 psp->bootloader_load_sos = psp_v3_1_bootloader_load_sos; 55 psp->bootloader_load_sos = psp_v3_1_bootloader_load_sos;
56 psp->prep_cmd_buf = psp_v3_1_prep_cmd_buf; 56 psp->prep_cmd_buf = psp_v3_1_prep_cmd_buf;
57 psp->ring_init = psp_v3_1_ring_init; 57 psp->ring_init = psp_v3_1_ring_init;
58 psp->ring_create = psp_v3_1_ring_create;
59 psp->ring_destroy = psp_v3_1_ring_destroy;
58 psp->cmd_submit = psp_v3_1_cmd_submit; 60 psp->cmd_submit = psp_v3_1_cmd_submit;
59 psp->compare_sram_data = psp_v3_1_compare_sram_data; 61 psp->compare_sram_data = psp_v3_1_compare_sram_data;
60 psp->smu_reload_quirk = psp_v3_1_smu_reload_quirk; 62 psp->smu_reload_quirk = psp_v3_1_smu_reload_quirk;
@@ -152,11 +154,6 @@ static void psp_prep_tmr_cmd_buf(struct psp_gfx_cmd_resp *cmd,
152static int psp_tmr_init(struct psp_context *psp) 154static int psp_tmr_init(struct psp_context *psp)
153{ 155{
154 int ret; 156 int ret;
155 struct psp_gfx_cmd_resp *cmd;
156
157 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
158 if (!cmd)
159 return -ENOMEM;
160 157
161 /* 158 /*
162 * Allocate 3M memory aligned to 1M from Frame Buffer (local 159 * Allocate 3M memory aligned to 1M from Frame Buffer (local
@@ -168,22 +165,30 @@ static int psp_tmr_init(struct psp_context *psp)
168 ret = amdgpu_bo_create_kernel(psp->adev, 0x300000, 0x100000, 165 ret = amdgpu_bo_create_kernel(psp->adev, 0x300000, 0x100000,
169 AMDGPU_GEM_DOMAIN_VRAM, 166 AMDGPU_GEM_DOMAIN_VRAM,
170 &psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf); 167 &psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
171 if (ret) 168
172 goto failed; 169 return ret;
170}
171
172static int psp_tmr_load(struct psp_context *psp)
173{
174 int ret;
175 struct psp_gfx_cmd_resp *cmd;
176
177 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
178 if (!cmd)
179 return -ENOMEM;
173 180
174 psp_prep_tmr_cmd_buf(cmd, psp->tmr_mc_addr, 0x300000); 181 psp_prep_tmr_cmd_buf(cmd, psp->tmr_mc_addr, 0x300000);
175 182
176 ret = psp_cmd_submit_buf(psp, NULL, cmd, 183 ret = psp_cmd_submit_buf(psp, NULL, cmd,
177 psp->fence_buf_mc_addr, 1); 184 psp->fence_buf_mc_addr, 1);
178 if (ret) 185 if (ret)
179 goto failed_mem; 186 goto failed;
180 187
181 kfree(cmd); 188 kfree(cmd);
182 189
183 return 0; 190 return 0;
184 191
185failed_mem:
186 amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
187failed: 192failed:
188 kfree(cmd); 193 kfree(cmd);
189 return ret; 194 return ret;
@@ -203,104 +208,78 @@ static void psp_prep_asd_cmd_buf(struct psp_gfx_cmd_resp *cmd,
203 cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size; 208 cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
204} 209}
205 210
206static int psp_asd_load(struct psp_context *psp) 211static int psp_asd_init(struct psp_context *psp)
207{ 212{
208 int ret; 213 int ret;
209 struct amdgpu_bo *asd_bo, *asd_shared_bo;
210 uint64_t asd_mc_addr, asd_shared_mc_addr;
211 void *asd_buf, *asd_shared_buf;
212 struct psp_gfx_cmd_resp *cmd;
213
214 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
215 if (!cmd)
216 return -ENOMEM;
217 214
218 /* 215 /*
219 * Allocate 16k memory aligned to 4k from Frame Buffer (local 216 * Allocate 16k memory aligned to 4k from Frame Buffer (local
220 * physical) for shared ASD <-> Driver 217 * physical) for shared ASD <-> Driver
221 */ 218 */
222 ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE, PAGE_SIZE, 219 ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE,
223 AMDGPU_GEM_DOMAIN_VRAM, 220 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
224 &asd_shared_bo, &asd_shared_mc_addr, &asd_buf); 221 &psp->asd_shared_bo,
225 if (ret) 222 &psp->asd_shared_mc_addr,
226 goto failed; 223 &psp->asd_shared_buf);
227 224
228 /* 225 return ret;
229 * Allocate 256k memory aligned to 4k from Frame Buffer (local 226}
230 * physical) for ASD firmware 227
231 */ 228static int psp_asd_load(struct psp_context *psp)
232 ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_BIN_SIZE, PAGE_SIZE, 229{
233 AMDGPU_GEM_DOMAIN_VRAM, 230 int ret;
234 &asd_bo, &asd_mc_addr, &asd_buf); 231 struct psp_gfx_cmd_resp *cmd;
235 if (ret) 232
236 goto failed_mem; 233 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
234 if (!cmd)
235 return -ENOMEM;
237 236
238 memcpy(asd_buf, psp->asd_start_addr, psp->asd_ucode_size); 237 memset(psp->fw_pri_buf, 0, PSP_1_MEG);
238 memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
239 239
240 psp_prep_asd_cmd_buf(cmd, asd_mc_addr, asd_shared_mc_addr, 240 psp_prep_asd_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_shared_mc_addr,
241 psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE); 241 psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE);
242 242
243 ret = psp_cmd_submit_buf(psp, NULL, cmd, 243 ret = psp_cmd_submit_buf(psp, NULL, cmd,
244 psp->fence_buf_mc_addr, 2); 244 psp->fence_buf_mc_addr, 2);
245 if (ret)
246 goto failed_mem1;
247 245
248 amdgpu_bo_free_kernel(&asd_bo, &asd_mc_addr, &asd_buf);
249 amdgpu_bo_free_kernel(&asd_shared_bo, &asd_shared_mc_addr, &asd_shared_buf);
250 kfree(cmd); 246 kfree(cmd);
251 247
252 return 0;
253
254failed_mem1:
255 amdgpu_bo_free_kernel(&asd_bo, &asd_mc_addr, &asd_buf);
256failed_mem:
257 amdgpu_bo_free_kernel(&asd_shared_bo, &asd_shared_mc_addr, &asd_shared_buf);
258failed:
259 kfree(cmd);
260 return ret; 248 return ret;
261} 249}
262 250
263static int psp_load_fw(struct amdgpu_device *adev) 251static int psp_hw_start(struct psp_context *psp)
264{ 252{
265 int ret; 253 int ret;
266 struct psp_gfx_cmd_resp *cmd;
267 int i;
268 struct amdgpu_firmware_info *ucode;
269 struct psp_context *psp = &adev->psp;
270
271 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
272 if (!cmd)
273 return -ENOMEM;
274 254
275 ret = psp_bootloader_load_sysdrv(psp); 255 ret = psp_bootloader_load_sysdrv(psp);
276 if (ret) 256 if (ret)
277 goto failed; 257 return ret;
278 258
279 ret = psp_bootloader_load_sos(psp); 259 ret = psp_bootloader_load_sos(psp);
280 if (ret) 260 if (ret)
281 goto failed; 261 return ret;
282
283 ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
284 if (ret)
285 goto failed;
286 262
287 ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE, 263 ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
288 AMDGPU_GEM_DOMAIN_VRAM,
289 &psp->fence_buf_bo,
290 &psp->fence_buf_mc_addr,
291 &psp->fence_buf);
292 if (ret) 264 if (ret)
293 goto failed; 265 return ret;
294
295 memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
296 266
297 ret = psp_tmr_init(psp); 267 ret = psp_tmr_load(psp);
298 if (ret) 268 if (ret)
299 goto failed_mem; 269 return ret;
300 270
301 ret = psp_asd_load(psp); 271 ret = psp_asd_load(psp);
302 if (ret) 272 if (ret)
303 goto failed_mem; 273 return ret;
274
275 return 0;
276}
277
278static int psp_np_fw_load(struct psp_context *psp)
279{
280 int i, ret;
281 struct amdgpu_firmware_info *ucode;
282 struct amdgpu_device* adev = psp->adev;
304 283
305 for (i = 0; i < adev->firmware.max_ucodes; i++) { 284 for (i = 0; i < adev->firmware.max_ucodes; i++) {
306 ucode = &adev->firmware.ucode[i]; 285 ucode = &adev->firmware.ucode[i];
@@ -310,15 +289,21 @@ static int psp_load_fw(struct amdgpu_device *adev)
310 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC && 289 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
311 psp_smu_reload_quirk(psp)) 290 psp_smu_reload_quirk(psp))
312 continue; 291 continue;
292 if (amdgpu_sriov_vf(adev) &&
293 (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
294 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
295 || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G))
296 /*skip ucode loading in SRIOV VF */
297 continue;
313 298
314 ret = psp_prep_cmd_buf(ucode, cmd); 299 ret = psp_prep_cmd_buf(ucode, psp->cmd);
315 if (ret) 300 if (ret)
316 goto failed_mem; 301 return ret;
317 302
318 ret = psp_cmd_submit_buf(psp, ucode, cmd, 303 ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
319 psp->fence_buf_mc_addr, i + 3); 304 psp->fence_buf_mc_addr, i + 3);
320 if (ret) 305 if (ret)
321 goto failed_mem; 306 return ret;
322 307
323#if 0 308#if 0
324 /* check if firmware loaded sucessfully */ 309 /* check if firmware loaded sucessfully */
@@ -327,8 +312,59 @@ static int psp_load_fw(struct amdgpu_device *adev)
327#endif 312#endif
328 } 313 }
329 314
330 amdgpu_bo_free_kernel(&psp->fence_buf_bo, 315 return 0;
331 &psp->fence_buf_mc_addr, &psp->fence_buf); 316}
317
318static int psp_load_fw(struct amdgpu_device *adev)
319{
320 int ret;
321 struct psp_context *psp = &adev->psp;
322 struct psp_gfx_cmd_resp *cmd;
323
324 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
325 if (!cmd)
326 return -ENOMEM;
327
328 psp->cmd = cmd;
329
330 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
331 AMDGPU_GEM_DOMAIN_GTT,
332 &psp->fw_pri_bo,
333 &psp->fw_pri_mc_addr,
334 &psp->fw_pri_buf);
335 if (ret)
336 goto failed;
337
338 ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
339 AMDGPU_GEM_DOMAIN_VRAM,
340 &psp->fence_buf_bo,
341 &psp->fence_buf_mc_addr,
342 &psp->fence_buf);
343 if (ret)
344 goto failed_mem1;
345
346 memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
347
348 ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
349 if (ret)
350 goto failed_mem1;
351
352 ret = psp_tmr_init(psp);
353 if (ret)
354 goto failed_mem;
355
356 ret = psp_asd_init(psp);
357 if (ret)
358 goto failed_mem;
359
360 ret = psp_hw_start(psp);
361 if (ret)
362 goto failed_mem;
363
364 ret = psp_np_fw_load(psp);
365 if (ret)
366 goto failed_mem;
367
332 kfree(cmd); 368 kfree(cmd);
333 369
334 return 0; 370 return 0;
@@ -336,6 +372,9 @@ static int psp_load_fw(struct amdgpu_device *adev)
336failed_mem: 372failed_mem:
337 amdgpu_bo_free_kernel(&psp->fence_buf_bo, 373 amdgpu_bo_free_kernel(&psp->fence_buf_bo,
338 &psp->fence_buf_mc_addr, &psp->fence_buf); 374 &psp->fence_buf_mc_addr, &psp->fence_buf);
375failed_mem1:
376 amdgpu_bo_free_kernel(&psp->fw_pri_bo,
377 &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
339failed: 378failed:
340 kfree(cmd); 379 kfree(cmd);
341 return ret; 380 return ret;
@@ -379,12 +418,24 @@ static int psp_hw_fini(void *handle)
379 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 418 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
380 struct psp_context *psp = &adev->psp; 419 struct psp_context *psp = &adev->psp;
381 420
382 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) 421 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
383 amdgpu_ucode_fini_bo(adev); 422 return 0;
423
424 amdgpu_ucode_fini_bo(adev);
425
426 psp_ring_destroy(psp, PSP_RING_TYPE__KM);
384 427
385 if (psp->tmr_buf) 428 if (psp->tmr_buf)
386 amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf); 429 amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
387 430
431 if (psp->fw_pri_buf)
432 amdgpu_bo_free_kernel(&psp->fw_pri_bo,
433 &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
434
435 if (psp->fence_buf_bo)
436 amdgpu_bo_free_kernel(&psp->fence_buf_bo,
437 &psp->fence_buf_mc_addr, &psp->fence_buf);
438
388 return 0; 439 return 0;
389} 440}
390 441
@@ -397,18 +448,30 @@ static int psp_resume(void *handle)
397{ 448{
398 int ret; 449 int ret;
399 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 450 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
451 struct psp_context *psp = &adev->psp;
400 452
401 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) 453 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
402 return 0; 454 return 0;
403 455
456 DRM_INFO("PSP is resuming...\n");
457
404 mutex_lock(&adev->firmware.mutex); 458 mutex_lock(&adev->firmware.mutex);
405 459
406 ret = psp_load_fw(adev); 460 ret = psp_hw_start(psp);
407 if (ret) 461 if (ret)
408 DRM_ERROR("PSP resume failed\n"); 462 goto failed;
463
464 ret = psp_np_fw_load(psp);
465 if (ret)
466 goto failed;
409 467
410 mutex_unlock(&adev->firmware.mutex); 468 mutex_unlock(&adev->firmware.mutex);
411 469
470 return 0;
471
472failed:
473 DRM_ERROR("PSP resume failed\n");
474 mutex_unlock(&adev->firmware.mutex);
412 return ret; 475 return ret;
413} 476}
414 477
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index e9f35e025b59..0301e4e0b297 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -30,8 +30,8 @@
30 30
31#define PSP_FENCE_BUFFER_SIZE 0x1000 31#define PSP_FENCE_BUFFER_SIZE 0x1000
32#define PSP_CMD_BUFFER_SIZE 0x1000 32#define PSP_CMD_BUFFER_SIZE 0x1000
33#define PSP_ASD_BIN_SIZE 0x40000
34#define PSP_ASD_SHARED_MEM_SIZE 0x4000 33#define PSP_ASD_SHARED_MEM_SIZE 0x4000
34#define PSP_1_MEG 0x100000
35 35
36enum psp_ring_type 36enum psp_ring_type
37{ 37{
@@ -57,6 +57,7 @@ struct psp_context
57{ 57{
58 struct amdgpu_device *adev; 58 struct amdgpu_device *adev;
59 struct psp_ring km_ring; 59 struct psp_ring km_ring;
60 struct psp_gfx_cmd_resp *cmd;
60 61
61 int (*init_microcode)(struct psp_context *psp); 62 int (*init_microcode)(struct psp_context *psp);
62 int (*bootloader_load_sysdrv)(struct psp_context *psp); 63 int (*bootloader_load_sysdrv)(struct psp_context *psp);
@@ -64,6 +65,9 @@ struct psp_context
64 int (*prep_cmd_buf)(struct amdgpu_firmware_info *ucode, 65 int (*prep_cmd_buf)(struct amdgpu_firmware_info *ucode,
65 struct psp_gfx_cmd_resp *cmd); 66 struct psp_gfx_cmd_resp *cmd);
66 int (*ring_init)(struct psp_context *psp, enum psp_ring_type ring_type); 67 int (*ring_init)(struct psp_context *psp, enum psp_ring_type ring_type);
68 int (*ring_create)(struct psp_context *psp, enum psp_ring_type ring_type);
69 int (*ring_destroy)(struct psp_context *psp,
70 enum psp_ring_type ring_type);
67 int (*cmd_submit)(struct psp_context *psp, struct amdgpu_firmware_info *ucode, 71 int (*cmd_submit)(struct psp_context *psp, struct amdgpu_firmware_info *ucode,
68 uint64_t cmd_buf_mc_addr, uint64_t fence_mc_addr, int index); 72 uint64_t cmd_buf_mc_addr, uint64_t fence_mc_addr, int index);
69 bool (*compare_sram_data)(struct psp_context *psp, 73 bool (*compare_sram_data)(struct psp_context *psp,
@@ -71,6 +75,11 @@ struct psp_context
71 enum AMDGPU_UCODE_ID ucode_type); 75 enum AMDGPU_UCODE_ID ucode_type);
72 bool (*smu_reload_quirk)(struct psp_context *psp); 76 bool (*smu_reload_quirk)(struct psp_context *psp);
73 77
78 /* fence buffer */
79 struct amdgpu_bo *fw_pri_bo;
80 uint64_t fw_pri_mc_addr;
81 void *fw_pri_buf;
82
74 /* sos firmware */ 83 /* sos firmware */
75 const struct firmware *sos_fw; 84 const struct firmware *sos_fw;
76 uint32_t sos_fw_version; 85 uint32_t sos_fw_version;
@@ -85,12 +94,15 @@ struct psp_context
85 uint64_t tmr_mc_addr; 94 uint64_t tmr_mc_addr;
86 void *tmr_buf; 95 void *tmr_buf;
87 96
88 /* asd firmware */ 97 /* asd firmware and buffer */
89 const struct firmware *asd_fw; 98 const struct firmware *asd_fw;
90 uint32_t asd_fw_version; 99 uint32_t asd_fw_version;
91 uint32_t asd_feature_version; 100 uint32_t asd_feature_version;
92 uint32_t asd_ucode_size; 101 uint32_t asd_ucode_size;
93 uint8_t *asd_start_addr; 102 uint8_t *asd_start_addr;
103 struct amdgpu_bo *asd_shared_bo;
104 uint64_t asd_shared_mc_addr;
105 void *asd_shared_buf;
94 106
95 /* fence buffer */ 107 /* fence buffer */
96 struct amdgpu_bo *fence_buf_bo; 108 struct amdgpu_bo *fence_buf_bo;
@@ -105,6 +117,8 @@ struct amdgpu_psp_funcs {
105 117
106#define psp_prep_cmd_buf(ucode, type) (psp)->prep_cmd_buf((ucode), (type)) 118#define psp_prep_cmd_buf(ucode, type) (psp)->prep_cmd_buf((ucode), (type))
107#define psp_ring_init(psp, type) (psp)->ring_init((psp), (type)) 119#define psp_ring_init(psp, type) (psp)->ring_init((psp), (type))
120#define psp_ring_create(psp, type) (psp)->ring_create((psp), (type))
121#define psp_ring_destroy(psp, type) ((psp)->ring_destroy((psp), (type)))
108#define psp_cmd_submit(psp, ucode, cmd_mc, fence_mc, index) \ 122#define psp_cmd_submit(psp, ucode, cmd_mc, fence_mc, index) \
109 (psp)->cmd_submit((psp), (ucode), (cmd_mc), (fence_mc), (index)) 123 (psp)->cmd_submit((psp), (ucode), (cmd_mc), (fence_mc), (index))
110#define psp_compare_sram_data(psp, ucode, type) \ 124#define psp_compare_sram_data(psp, ucode, type) \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 63e56398ca9a..944443c5b90a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -99,6 +99,7 @@ struct amdgpu_ring_funcs {
99 uint32_t align_mask; 99 uint32_t align_mask;
100 u32 nop; 100 u32 nop;
101 bool support_64bit_ptrs; 101 bool support_64bit_ptrs;
102 unsigned vmhub;
102 103
103 /* ring read/write ptr handling */ 104 /* ring read/write ptr handling */
104 u64 (*get_rptr)(struct amdgpu_ring *ring); 105 u64 (*get_rptr)(struct amdgpu_ring *ring);
@@ -178,6 +179,7 @@ struct amdgpu_ring {
178 unsigned cond_exe_offs; 179 unsigned cond_exe_offs;
179 u64 cond_exe_gpu_addr; 180 u64 cond_exe_gpu_addr;
180 volatile u32 *cond_exe_cpu_addr; 181 volatile u32 *cond_exe_cpu_addr;
182 unsigned vm_inv_eng;
181#if defined(CONFIG_DEBUG_FS) 183#if defined(CONFIG_DEBUG_FS)
182 struct dentry *ent; 184 struct dentry *ent;
183#endif 185#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
index de9f919ae336..5ca75a456ad2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
@@ -130,7 +130,7 @@ int amdgpu_sa_bo_manager_suspend(struct amdgpu_device *adev,
130 return -EINVAL; 130 return -EINVAL;
131 } 131 }
132 132
133 r = amdgpu_bo_reserve(sa_manager->bo, false); 133 r = amdgpu_bo_reserve(sa_manager->bo, true);
134 if (!r) { 134 if (!r) {
135 amdgpu_bo_kunmap(sa_manager->bo); 135 amdgpu_bo_kunmap(sa_manager->bo);
136 amdgpu_bo_unpin(sa_manager->bo); 136 amdgpu_bo_unpin(sa_manager->bo);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index ee9d0f346d75..8601904e670a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -190,26 +190,29 @@ TRACE_EVENT(amdgpu_sched_run_job,
190 190
191 191
192TRACE_EVENT(amdgpu_vm_grab_id, 192TRACE_EVENT(amdgpu_vm_grab_id,
193 TP_PROTO(struct amdgpu_vm *vm, int ring, struct amdgpu_job *job), 193 TP_PROTO(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
194 struct amdgpu_job *job),
194 TP_ARGS(vm, ring, job), 195 TP_ARGS(vm, ring, job),
195 TP_STRUCT__entry( 196 TP_STRUCT__entry(
196 __field(struct amdgpu_vm *, vm) 197 __field(struct amdgpu_vm *, vm)
197 __field(u32, ring) 198 __field(u32, ring)
198 __field(u32, vmid) 199 __field(u32, vm_id)
200 __field(u32, vm_hub)
199 __field(u64, pd_addr) 201 __field(u64, pd_addr)
200 __field(u32, needs_flush) 202 __field(u32, needs_flush)
201 ), 203 ),
202 204
203 TP_fast_assign( 205 TP_fast_assign(
204 __entry->vm = vm; 206 __entry->vm = vm;
205 __entry->ring = ring; 207 __entry->ring = ring->idx;
206 __entry->vmid = job->vm_id; 208 __entry->vm_id = job->vm_id;
209 __entry->vm_hub = ring->funcs->vmhub,
207 __entry->pd_addr = job->vm_pd_addr; 210 __entry->pd_addr = job->vm_pd_addr;
208 __entry->needs_flush = job->vm_needs_flush; 211 __entry->needs_flush = job->vm_needs_flush;
209 ), 212 ),
210 TP_printk("vm=%p, ring=%u, id=%u, pd_addr=%010Lx needs_flush=%u", 213 TP_printk("vm=%p, ring=%u, id=%u, hub=%u, pd_addr=%010Lx needs_flush=%u",
211 __entry->vm, __entry->ring, __entry->vmid, 214 __entry->vm, __entry->ring, __entry->vm_id,
212 __entry->pd_addr, __entry->needs_flush) 215 __entry->vm_hub, __entry->pd_addr, __entry->needs_flush)
213); 216);
214 217
215TRACE_EVENT(amdgpu_vm_bo_map, 218TRACE_EVENT(amdgpu_vm_bo_map,
@@ -331,21 +334,25 @@ TRACE_EVENT(amdgpu_vm_copy_ptes,
331); 334);
332 335
333TRACE_EVENT(amdgpu_vm_flush, 336TRACE_EVENT(amdgpu_vm_flush,
334 TP_PROTO(uint64_t pd_addr, unsigned ring, unsigned id), 337 TP_PROTO(struct amdgpu_ring *ring, unsigned vm_id,
335 TP_ARGS(pd_addr, ring, id), 338 uint64_t pd_addr),
339 TP_ARGS(ring, vm_id, pd_addr),
336 TP_STRUCT__entry( 340 TP_STRUCT__entry(
337 __field(u64, pd_addr)
338 __field(u32, ring) 341 __field(u32, ring)
339 __field(u32, id) 342 __field(u32, vm_id)
343 __field(u32, vm_hub)
344 __field(u64, pd_addr)
340 ), 345 ),
341 346
342 TP_fast_assign( 347 TP_fast_assign(
348 __entry->ring = ring->idx;
349 __entry->vm_id = vm_id;
350 __entry->vm_hub = ring->funcs->vmhub;
343 __entry->pd_addr = pd_addr; 351 __entry->pd_addr = pd_addr;
344 __entry->ring = ring;
345 __entry->id = id;
346 ), 352 ),
347 TP_printk("ring=%u, id=%u, pd_addr=%010Lx", 353 TP_printk("ring=%u, id=%u, hub=%u, pd_addr=%010Lx",
348 __entry->ring, __entry->id, __entry->pd_addr) 354 __entry->ring, __entry->vm_id,
355 __entry->vm_hub,__entry->pd_addr)
349); 356);
350 357
351TRACE_EVENT(amdgpu_bo_list_set, 358TRACE_EVENT(amdgpu_bo_list_set,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 35d53a0d9ba6..5db0230e45c6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -203,7 +203,9 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
203 abo = container_of(bo, struct amdgpu_bo, tbo); 203 abo = container_of(bo, struct amdgpu_bo, tbo);
204 switch (bo->mem.mem_type) { 204 switch (bo->mem.mem_type) {
205 case TTM_PL_VRAM: 205 case TTM_PL_VRAM:
206 if (adev->mman.buffer_funcs_ring->ready == false) { 206 if (adev->mman.buffer_funcs &&
207 adev->mman.buffer_funcs_ring &&
208 adev->mman.buffer_funcs_ring->ready == false) {
207 amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); 209 amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
208 } else { 210 } else {
209 amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT); 211 amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
@@ -763,7 +765,7 @@ int amdgpu_ttm_recover_gart(struct amdgpu_device *adev)
763{ 765{
764 struct amdgpu_ttm_tt *gtt, *tmp; 766 struct amdgpu_ttm_tt *gtt, *tmp;
765 struct ttm_mem_reg bo_mem; 767 struct ttm_mem_reg bo_mem;
766 uint32_t flags; 768 uint64_t flags;
767 int r; 769 int r;
768 770
769 bo_mem.mem_type = TTM_PL_TT; 771 bo_mem.mem_type = TTM_PL_TT;
@@ -1038,11 +1040,17 @@ uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
1038static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 1040static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
1039 const struct ttm_place *place) 1041 const struct ttm_place *place)
1040{ 1042{
1041 if (bo->mem.mem_type == TTM_PL_VRAM && 1043 unsigned long num_pages = bo->mem.num_pages;
1042 bo->mem.start == AMDGPU_BO_INVALID_OFFSET) { 1044 struct drm_mm_node *node = bo->mem.mm_node;
1043 unsigned long num_pages = bo->mem.num_pages; 1045
1044 struct drm_mm_node *node = bo->mem.mm_node; 1046 if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET)
1047 return ttm_bo_eviction_valuable(bo, place);
1048
1049 switch (bo->mem.mem_type) {
1050 case TTM_PL_TT:
1051 return true;
1045 1052
1053 case TTM_PL_VRAM:
1046 /* Check each drm MM node individually */ 1054 /* Check each drm MM node individually */
1047 while (num_pages) { 1055 while (num_pages) {
1048 if (place->fpfn < (node->start + node->size) && 1056 if (place->fpfn < (node->start + node->size) &&
@@ -1052,8 +1060,10 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
1052 num_pages -= node->size; 1060 num_pages -= node->size;
1053 ++node; 1061 ++node;
1054 } 1062 }
1063 break;
1055 1064
1056 return false; 1065 default:
1066 break;
1057 } 1067 }
1058 1068
1059 return ttm_bo_eviction_valuable(bo, place); 1069 return ttm_bo_eviction_valuable(bo, place);
@@ -1188,7 +1198,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
1188 return; 1198 return;
1189 amdgpu_ttm_debugfs_fini(adev); 1199 amdgpu_ttm_debugfs_fini(adev);
1190 if (adev->stollen_vga_memory) { 1200 if (adev->stollen_vga_memory) {
1191 r = amdgpu_bo_reserve(adev->stollen_vga_memory, false); 1201 r = amdgpu_bo_reserve(adev->stollen_vga_memory, true);
1192 if (r == 0) { 1202 if (r == 0) {
1193 amdgpu_bo_unpin(adev->stollen_vga_memory); 1203 amdgpu_bo_unpin(adev->stollen_vga_memory);
1194 amdgpu_bo_unreserve(adev->stollen_vga_memory); 1204 amdgpu_bo_unreserve(adev->stollen_vga_memory);
@@ -1401,6 +1411,8 @@ error_free:
1401 1411
1402#if defined(CONFIG_DEBUG_FS) 1412#if defined(CONFIG_DEBUG_FS)
1403 1413
1414extern void amdgpu_gtt_mgr_print(struct seq_file *m, struct ttm_mem_type_manager
1415 *man);
1404static int amdgpu_mm_dump_table(struct seq_file *m, void *data) 1416static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
1405{ 1417{
1406 struct drm_info_node *node = (struct drm_info_node *)m->private; 1418 struct drm_info_node *node = (struct drm_info_node *)m->private;
@@ -1414,11 +1426,17 @@ static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
1414 spin_lock(&glob->lru_lock); 1426 spin_lock(&glob->lru_lock);
1415 drm_mm_print(mm, &p); 1427 drm_mm_print(mm, &p);
1416 spin_unlock(&glob->lru_lock); 1428 spin_unlock(&glob->lru_lock);
1417 if (ttm_pl == TTM_PL_VRAM) 1429 switch (ttm_pl) {
1430 case TTM_PL_VRAM:
1418 seq_printf(m, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n", 1431 seq_printf(m, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n",
1419 adev->mman.bdev.man[ttm_pl].size, 1432 adev->mman.bdev.man[ttm_pl].size,
1420 (u64)atomic64_read(&adev->vram_usage) >> 20, 1433 (u64)atomic64_read(&adev->vram_usage) >> 20,
1421 (u64)atomic64_read(&adev->vram_vis_usage) >> 20); 1434 (u64)atomic64_read(&adev->vram_vis_usage) >> 20);
1435 break;
1436 case TTM_PL_TT:
1437 amdgpu_gtt_mgr_print(m, &adev->mman.bdev.man[TTM_PL_TT]);
1438 break;
1439 }
1422 return 0; 1440 return 0;
1423} 1441}
1424 1442
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index a1891c93cdbf..dfd1c98efa7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -382,10 +382,14 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
382 * if SMU loaded firmware, it needn't add SMC, UVD, and VCE 382 * if SMU loaded firmware, it needn't add SMC, UVD, and VCE
383 * ucode info here 383 * ucode info here
384 */ 384 */
385 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) 385 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
386 adev->firmware.max_ucodes = AMDGPU_UCODE_ID_MAXIMUM - 4; 386 if (amdgpu_sriov_vf(adev))
387 else 387 adev->firmware.max_ucodes = AMDGPU_UCODE_ID_MAXIMUM - 3;
388 else
389 adev->firmware.max_ucodes = AMDGPU_UCODE_ID_MAXIMUM - 4;
390 } else {
388 adev->firmware.max_ucodes = AMDGPU_UCODE_ID_MAXIMUM; 391 adev->firmware.max_ucodes = AMDGPU_UCODE_ID_MAXIMUM;
392 }
389 393
390 for (i = 0; i < adev->firmware.max_ucodes; i++) { 394 for (i = 0; i < adev->firmware.max_ucodes; i++) {
391 ucode = &adev->firmware.ucode[i]; 395 ucode = &adev->firmware.ucode[i];
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index c853400805d1..735c38d7db0d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -955,11 +955,11 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
955 struct amdgpu_device *adev = ring->adev; 955 struct amdgpu_device *adev = ring->adev;
956 uint32_t rptr = amdgpu_ring_get_rptr(ring); 956 uint32_t rptr = amdgpu_ring_get_rptr(ring);
957 unsigned i; 957 unsigned i;
958 int r; 958 int r, timeout = adev->usec_timeout;
959 959
960 /* TODO: remove it if VCE can work for sriov */ 960 /* workaround VCE ring test slow issue for sriov*/
961 if (amdgpu_sriov_vf(adev)) 961 if (amdgpu_sriov_vf(adev))
962 return 0; 962 timeout *= 10;
963 963
964 r = amdgpu_ring_alloc(ring, 16); 964 r = amdgpu_ring_alloc(ring, 16);
965 if (r) { 965 if (r) {
@@ -970,13 +970,13 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
970 amdgpu_ring_write(ring, VCE_CMD_END); 970 amdgpu_ring_write(ring, VCE_CMD_END);
971 amdgpu_ring_commit(ring); 971 amdgpu_ring_commit(ring);
972 972
973 for (i = 0; i < adev->usec_timeout; i++) { 973 for (i = 0; i < timeout; i++) {
974 if (amdgpu_ring_get_rptr(ring) != rptr) 974 if (amdgpu_ring_get_rptr(ring) != rptr)
975 break; 975 break;
976 DRM_UDELAY(1); 976 DRM_UDELAY(1);
977 } 977 }
978 978
979 if (i < adev->usec_timeout) { 979 if (i < timeout) {
980 DRM_INFO("ring test on %d succeeded in %d usecs\n", 980 DRM_INFO("ring test on %d succeeded in %d usecs\n",
981 ring->idx, i); 981 ring->idx, i);
982 } else { 982 } else {
@@ -999,10 +999,6 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
999 struct dma_fence *fence = NULL; 999 struct dma_fence *fence = NULL;
1000 long r; 1000 long r;
1001 1001
1002 /* TODO: remove it if VCE can work for sriov */
1003 if (amdgpu_sriov_vf(ring->adev))
1004 return 0;
1005
1006 /* skip vce ring1/2 ib test for now, since it's not reliable */ 1002 /* skip vce ring1/2 ib test for now, since it's not reliable */
1007 if (ring != &ring->adev->vce.ring[0]) 1003 if (ring != &ring->adev->vce.ring[0])
1008 return 0; 1004 return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index ba8b8ae6234f..6bf5cea294f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -225,3 +225,49 @@ int amdgpu_virt_reset_gpu(struct amdgpu_device *adev)
225 225
226 return 0; 226 return 0;
227} 227}
228
229/**
230 * amdgpu_virt_alloc_mm_table() - alloc memory for mm table
231 * @amdgpu: amdgpu device.
232 * MM table is used by UVD and VCE for its initialization
233 * Return: Zero if allocate success.
234 */
235int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev)
236{
237 int r;
238
239 if (!amdgpu_sriov_vf(adev) || adev->virt.mm_table.gpu_addr)
240 return 0;
241
242 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
243 AMDGPU_GEM_DOMAIN_VRAM,
244 &adev->virt.mm_table.bo,
245 &adev->virt.mm_table.gpu_addr,
246 (void *)&adev->virt.mm_table.cpu_addr);
247 if (r) {
248 DRM_ERROR("failed to alloc mm table and error = %d.\n", r);
249 return r;
250 }
251
252 memset((void *)adev->virt.mm_table.cpu_addr, 0, PAGE_SIZE);
253 DRM_INFO("MM table gpu addr = 0x%llx, cpu addr = %p.\n",
254 adev->virt.mm_table.gpu_addr,
255 adev->virt.mm_table.cpu_addr);
256 return 0;
257}
258
259/**
260 * amdgpu_virt_free_mm_table() - free mm table memory
261 * @amdgpu: amdgpu device.
262 * Free MM table memory
263 */
264void amdgpu_virt_free_mm_table(struct amdgpu_device *adev)
265{
266 if (!amdgpu_sriov_vf(adev) || !adev->virt.mm_table.gpu_addr)
267 return;
268
269 amdgpu_bo_free_kernel(&adev->virt.mm_table.bo,
270 &adev->virt.mm_table.gpu_addr,
271 (void *)&adev->virt.mm_table.cpu_addr);
272 adev->virt.mm_table.gpu_addr = 0;
273}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 1ee0a190b33b..a8ed162cc0bc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -98,5 +98,7 @@ int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
98int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init); 98int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
99int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); 99int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
100int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, bool voluntary); 100int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, bool voluntary);
101int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev);
102void amdgpu_virt_free_mm_table(struct amdgpu_device *adev);
101 103
102#endif 104#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 7ed5302b511a..07ff3b1514f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -406,6 +406,8 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
406 struct amdgpu_job *job) 406 struct amdgpu_job *job)
407{ 407{
408 struct amdgpu_device *adev = ring->adev; 408 struct amdgpu_device *adev = ring->adev;
409 unsigned vmhub = ring->funcs->vmhub;
410 struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
409 uint64_t fence_context = adev->fence_context + ring->idx; 411 uint64_t fence_context = adev->fence_context + ring->idx;
410 struct dma_fence *updates = sync->last_vm_update; 412 struct dma_fence *updates = sync->last_vm_update;
411 struct amdgpu_vm_id *id, *idle; 413 struct amdgpu_vm_id *id, *idle;
@@ -413,16 +415,15 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
413 unsigned i; 415 unsigned i;
414 int r = 0; 416 int r = 0;
415 417
416 fences = kmalloc_array(sizeof(void *), adev->vm_manager.num_ids, 418 fences = kmalloc_array(sizeof(void *), id_mgr->num_ids, GFP_KERNEL);
417 GFP_KERNEL);
418 if (!fences) 419 if (!fences)
419 return -ENOMEM; 420 return -ENOMEM;
420 421
421 mutex_lock(&adev->vm_manager.lock); 422 mutex_lock(&id_mgr->lock);
422 423
423 /* Check if we have an idle VMID */ 424 /* Check if we have an idle VMID */
424 i = 0; 425 i = 0;
425 list_for_each_entry(idle, &adev->vm_manager.ids_lru, list) { 426 list_for_each_entry(idle, &id_mgr->ids_lru, list) {
426 fences[i] = amdgpu_sync_peek_fence(&idle->active, ring); 427 fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
427 if (!fences[i]) 428 if (!fences[i])
428 break; 429 break;
@@ -430,7 +431,7 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
430 } 431 }
431 432
432 /* If we can't find a idle VMID to use, wait till one becomes available */ 433 /* If we can't find a idle VMID to use, wait till one becomes available */
433 if (&idle->list == &adev->vm_manager.ids_lru) { 434 if (&idle->list == &id_mgr->ids_lru) {
434 u64 fence_context = adev->vm_manager.fence_context + ring->idx; 435 u64 fence_context = adev->vm_manager.fence_context + ring->idx;
435 unsigned seqno = ++adev->vm_manager.seqno[ring->idx]; 436 unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
436 struct dma_fence_array *array; 437 struct dma_fence_array *array;
@@ -455,25 +456,19 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
455 if (r) 456 if (r)
456 goto error; 457 goto error;
457 458
458 mutex_unlock(&adev->vm_manager.lock); 459 mutex_unlock(&id_mgr->lock);
459 return 0; 460 return 0;
460 461
461 } 462 }
462 kfree(fences); 463 kfree(fences);
463 464
464 job->vm_needs_flush = true; 465 job->vm_needs_flush = false;
465 /* Check if we can use a VMID already assigned to this VM */ 466 /* Check if we can use a VMID already assigned to this VM */
466 i = ring->idx; 467 list_for_each_entry_reverse(id, &id_mgr->ids_lru, list) {
467 do {
468 struct dma_fence *flushed; 468 struct dma_fence *flushed;
469 469 bool needs_flush = false;
470 id = vm->ids[i++];
471 if (i == AMDGPU_MAX_RINGS)
472 i = 0;
473 470
474 /* Check all the prerequisites to using this VMID */ 471 /* Check all the prerequisites to using this VMID */
475 if (!id)
476 continue;
477 if (amdgpu_vm_had_gpu_reset(adev, id)) 472 if (amdgpu_vm_had_gpu_reset(adev, id))
478 continue; 473 continue;
479 474
@@ -483,16 +478,17 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
483 if (job->vm_pd_addr != id->pd_gpu_addr) 478 if (job->vm_pd_addr != id->pd_gpu_addr)
484 continue; 479 continue;
485 480
486 if (!id->last_flush) 481 if (!id->last_flush ||
487 continue; 482 (id->last_flush->context != fence_context &&
488 483 !dma_fence_is_signaled(id->last_flush)))
489 if (id->last_flush->context != fence_context && 484 needs_flush = true;
490 !dma_fence_is_signaled(id->last_flush))
491 continue;
492 485
493 flushed = id->flushed_updates; 486 flushed = id->flushed_updates;
494 if (updates && 487 if (updates && (!flushed || dma_fence_is_later(updates, flushed)))
495 (!flushed || dma_fence_is_later(updates, flushed))) 488 needs_flush = true;
489
490 /* Concurrent flushes are only possible starting with Vega10 */
491 if (adev->asic_type < CHIP_VEGA10 && needs_flush)
496 continue; 492 continue;
497 493
498 /* Good we can use this VMID. Remember this submission as 494 /* Good we can use this VMID. Remember this submission as
@@ -502,17 +498,17 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
502 if (r) 498 if (r)
503 goto error; 499 goto error;
504 500
505 list_move_tail(&id->list, &adev->vm_manager.ids_lru); 501 if (updates && (!flushed || dma_fence_is_later(updates, flushed))) {
506 vm->ids[ring->idx] = id; 502 dma_fence_put(id->flushed_updates);
507 503 id->flushed_updates = dma_fence_get(updates);
508 job->vm_id = id - adev->vm_manager.ids; 504 }
509 job->vm_needs_flush = false;
510 trace_amdgpu_vm_grab_id(vm, ring->idx, job);
511 505
512 mutex_unlock(&adev->vm_manager.lock); 506 if (needs_flush)
513 return 0; 507 goto needs_flush;
508 else
509 goto no_flush_needed;
514 510
515 } while (i != ring->idx); 511 };
516 512
517 /* Still no ID to use? Then use the idle one found earlier */ 513 /* Still no ID to use? Then use the idle one found earlier */
518 id = idle; 514 id = idle;
@@ -522,23 +518,25 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
522 if (r) 518 if (r)
523 goto error; 519 goto error;
524 520
525 dma_fence_put(id->last_flush); 521 id->pd_gpu_addr = job->vm_pd_addr;
526 id->last_flush = NULL;
527
528 dma_fence_put(id->flushed_updates); 522 dma_fence_put(id->flushed_updates);
529 id->flushed_updates = dma_fence_get(updates); 523 id->flushed_updates = dma_fence_get(updates);
530
531 id->pd_gpu_addr = job->vm_pd_addr;
532 id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter); 524 id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
533 list_move_tail(&id->list, &adev->vm_manager.ids_lru);
534 atomic64_set(&id->owner, vm->client_id); 525 atomic64_set(&id->owner, vm->client_id);
535 vm->ids[ring->idx] = id;
536 526
537 job->vm_id = id - adev->vm_manager.ids; 527needs_flush:
538 trace_amdgpu_vm_grab_id(vm, ring->idx, job); 528 job->vm_needs_flush = true;
529 dma_fence_put(id->last_flush);
530 id->last_flush = NULL;
531
532no_flush_needed:
533 list_move_tail(&id->list, &id_mgr->ids_lru);
534
535 job->vm_id = id - id_mgr->ids;
536 trace_amdgpu_vm_grab_id(vm, ring, job);
539 537
540error: 538error:
541 mutex_unlock(&adev->vm_manager.lock); 539 mutex_unlock(&id_mgr->lock);
542 return r; 540 return r;
543} 541}
544 542
@@ -590,7 +588,9 @@ static u64 amdgpu_vm_adjust_mc_addr(struct amdgpu_device *adev, u64 mc_addr)
590int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job) 588int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
591{ 589{
592 struct amdgpu_device *adev = ring->adev; 590 struct amdgpu_device *adev = ring->adev;
593 struct amdgpu_vm_id *id = &adev->vm_manager.ids[job->vm_id]; 591 unsigned vmhub = ring->funcs->vmhub;
592 struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
593 struct amdgpu_vm_id *id = &id_mgr->ids[job->vm_id];
594 bool gds_switch_needed = ring->funcs->emit_gds_switch && ( 594 bool gds_switch_needed = ring->funcs->emit_gds_switch && (
595 id->gds_base != job->gds_base || 595 id->gds_base != job->gds_base ||
596 id->gds_size != job->gds_size || 596 id->gds_size != job->gds_size ||
@@ -614,24 +614,24 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
614 if (ring->funcs->init_cond_exec) 614 if (ring->funcs->init_cond_exec)
615 patch_offset = amdgpu_ring_init_cond_exec(ring); 615 patch_offset = amdgpu_ring_init_cond_exec(ring);
616 616
617 if (ring->funcs->emit_pipeline_sync) 617 if (ring->funcs->emit_pipeline_sync && !job->need_pipeline_sync)
618 amdgpu_ring_emit_pipeline_sync(ring); 618 amdgpu_ring_emit_pipeline_sync(ring);
619 619
620 if (ring->funcs->emit_vm_flush && vm_flush_needed) { 620 if (ring->funcs->emit_vm_flush && vm_flush_needed) {
621 u64 pd_addr = amdgpu_vm_adjust_mc_addr(adev, job->vm_pd_addr); 621 u64 pd_addr = amdgpu_vm_adjust_mc_addr(adev, job->vm_pd_addr);
622 struct dma_fence *fence; 622 struct dma_fence *fence;
623 623
624 trace_amdgpu_vm_flush(pd_addr, ring->idx, job->vm_id); 624 trace_amdgpu_vm_flush(ring, job->vm_id, pd_addr);
625 amdgpu_ring_emit_vm_flush(ring, job->vm_id, pd_addr); 625 amdgpu_ring_emit_vm_flush(ring, job->vm_id, pd_addr);
626 626
627 r = amdgpu_fence_emit(ring, &fence); 627 r = amdgpu_fence_emit(ring, &fence);
628 if (r) 628 if (r)
629 return r; 629 return r;
630 630
631 mutex_lock(&adev->vm_manager.lock); 631 mutex_lock(&id_mgr->lock);
632 dma_fence_put(id->last_flush); 632 dma_fence_put(id->last_flush);
633 id->last_flush = fence; 633 id->last_flush = fence;
634 mutex_unlock(&adev->vm_manager.lock); 634 mutex_unlock(&id_mgr->lock);
635 } 635 }
636 636
637 if (gds_switch_needed) { 637 if (gds_switch_needed) {
@@ -666,9 +666,11 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
666 * 666 *
667 * Reset saved GDW, GWS and OA to force switch on next flush. 667 * Reset saved GDW, GWS and OA to force switch on next flush.
668 */ 668 */
669void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id) 669void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
670 unsigned vmid)
670{ 671{
671 struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id]; 672 struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
673 struct amdgpu_vm_id *id = &id_mgr->ids[vmid];
672 674
673 id->gds_base = 0; 675 id->gds_base = 0;
674 id->gds_size = 0; 676 id->gds_size = 0;
@@ -1336,6 +1338,12 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1336 flags &= ~AMDGPU_PTE_MTYPE_MASK; 1338 flags &= ~AMDGPU_PTE_MTYPE_MASK;
1337 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK); 1339 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);
1338 1340
1341 if ((mapping->flags & AMDGPU_PTE_PRT) &&
1342 (adev->asic_type >= CHIP_VEGA10)) {
1343 flags |= AMDGPU_PTE_PRT;
1344 flags &= ~AMDGPU_PTE_VALID;
1345 }
1346
1339 trace_amdgpu_vm_bo_update(mapping); 1347 trace_amdgpu_vm_bo_update(mapping);
1340 1348
1341 pfn = mapping->offset >> PAGE_SHIFT; 1349 pfn = mapping->offset >> PAGE_SHIFT;
@@ -1629,8 +1637,9 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1629 struct amdgpu_bo_va_mapping, list); 1637 struct amdgpu_bo_va_mapping, list);
1630 list_del(&mapping->list); 1638 list_del(&mapping->list);
1631 1639
1632 r = amdgpu_vm_bo_split_mapping(adev, NULL, 0, NULL, vm, mapping, 1640 r = amdgpu_vm_bo_update_mapping(adev, NULL, 0, NULL, vm,
1633 0, 0, &f); 1641 mapping->start, mapping->last,
1642 0, 0, &f);
1634 amdgpu_vm_free_mapping(adev, vm, mapping, f); 1643 amdgpu_vm_free_mapping(adev, vm, mapping, f);
1635 if (r) { 1644 if (r) {
1636 dma_fence_put(f); 1645 dma_fence_put(f);
@@ -2117,10 +2126,8 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2117 unsigned ring_instance; 2126 unsigned ring_instance;
2118 struct amdgpu_ring *ring; 2127 struct amdgpu_ring *ring;
2119 struct amd_sched_rq *rq; 2128 struct amd_sched_rq *rq;
2120 int i, r; 2129 int r;
2121 2130
2122 for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
2123 vm->ids[i] = NULL;
2124 vm->va = RB_ROOT; 2131 vm->va = RB_ROOT;
2125 vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter); 2132 vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
2126 spin_lock_init(&vm->status_lock); 2133 spin_lock_init(&vm->status_lock);
@@ -2241,16 +2248,21 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2241 */ 2248 */
2242void amdgpu_vm_manager_init(struct amdgpu_device *adev) 2249void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2243{ 2250{
2244 unsigned i; 2251 unsigned i, j;
2252
2253 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
2254 struct amdgpu_vm_id_manager *id_mgr =
2255 &adev->vm_manager.id_mgr[i];
2245 2256
2246 INIT_LIST_HEAD(&adev->vm_manager.ids_lru); 2257 mutex_init(&id_mgr->lock);
2258 INIT_LIST_HEAD(&id_mgr->ids_lru);
2247 2259
2248 /* skip over VMID 0, since it is the system VM */ 2260 /* skip over VMID 0, since it is the system VM */
2249 for (i = 1; i < adev->vm_manager.num_ids; ++i) { 2261 for (j = 1; j < id_mgr->num_ids; ++j) {
2250 amdgpu_vm_reset_id(adev, i); 2262 amdgpu_vm_reset_id(adev, i, j);
2251 amdgpu_sync_create(&adev->vm_manager.ids[i].active); 2263 amdgpu_sync_create(&id_mgr->ids[i].active);
2252 list_add_tail(&adev->vm_manager.ids[i].list, 2264 list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru);
2253 &adev->vm_manager.ids_lru); 2265 }
2254 } 2266 }
2255 2267
2256 adev->vm_manager.fence_context = 2268 adev->vm_manager.fence_context =
@@ -2258,6 +2270,7 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2258 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) 2270 for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
2259 adev->vm_manager.seqno[i] = 0; 2271 adev->vm_manager.seqno[i] = 0;
2260 2272
2273
2261 atomic_set(&adev->vm_manager.vm_pte_next_ring, 0); 2274 atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
2262 atomic64_set(&adev->vm_manager.client_counter, 0); 2275 atomic64_set(&adev->vm_manager.client_counter, 0);
2263 spin_lock_init(&adev->vm_manager.prt_lock); 2276 spin_lock_init(&adev->vm_manager.prt_lock);
@@ -2273,13 +2286,19 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2273 */ 2286 */
2274void amdgpu_vm_manager_fini(struct amdgpu_device *adev) 2287void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2275{ 2288{
2276 unsigned i; 2289 unsigned i, j;
2277 2290
2278 for (i = 0; i < AMDGPU_NUM_VM; ++i) { 2291 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
2279 struct amdgpu_vm_id *id = &adev->vm_manager.ids[i]; 2292 struct amdgpu_vm_id_manager *id_mgr =
2293 &adev->vm_manager.id_mgr[i];
2280 2294
2281 amdgpu_sync_free(&adev->vm_manager.ids[i].active); 2295 mutex_destroy(&id_mgr->lock);
2282 dma_fence_put(id->flushed_updates); 2296 for (j = 0; j < AMDGPU_NUM_VM; ++j) {
2283 dma_fence_put(id->last_flush); 2297 struct amdgpu_vm_id *id = &id_mgr->ids[j];
2298
2299 amdgpu_sync_free(&id->active);
2300 dma_fence_put(id->flushed_updates);
2301 dma_fence_put(id->last_flush);
2302 }
2284 } 2303 }
2285} 2304}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index d9e57290dc71..d97e28b4bdc4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -65,7 +65,8 @@ struct amdgpu_bo_list_entry;
65 65
66#define AMDGPU_PTE_FRAG(x) ((x & 0x1fULL) << 7) 66#define AMDGPU_PTE_FRAG(x) ((x & 0x1fULL) << 7)
67 67
68#define AMDGPU_PTE_PRT (1ULL << 63) 68/* TILED for VEGA10, reserved for older ASICs */
69#define AMDGPU_PTE_PRT (1ULL << 51)
69 70
70/* VEGA10 only */ 71/* VEGA10 only */
71#define AMDGPU_PTE_MTYPE(a) ((uint64_t)a << 57) 72#define AMDGPU_PTE_MTYPE(a) ((uint64_t)a << 57)
@@ -114,9 +115,6 @@ struct amdgpu_vm {
114 struct dma_fence *last_dir_update; 115 struct dma_fence *last_dir_update;
115 uint64_t last_eviction_counter; 116 uint64_t last_eviction_counter;
116 117
117 /* for id and flush management per ring */
118 struct amdgpu_vm_id *ids[AMDGPU_MAX_RINGS];
119
120 /* protecting freed */ 118 /* protecting freed */
121 spinlock_t freed_lock; 119 spinlock_t freed_lock;
122 120
@@ -149,12 +147,16 @@ struct amdgpu_vm_id {
149 uint32_t oa_size; 147 uint32_t oa_size;
150}; 148};
151 149
150struct amdgpu_vm_id_manager {
151 struct mutex lock;
152 unsigned num_ids;
153 struct list_head ids_lru;
154 struct amdgpu_vm_id ids[AMDGPU_NUM_VM];
155};
156
152struct amdgpu_vm_manager { 157struct amdgpu_vm_manager {
153 /* Handling of VMIDs */ 158 /* Handling of VMIDs */
154 struct mutex lock; 159 struct amdgpu_vm_id_manager id_mgr[AMDGPU_MAX_VMHUBS];
155 unsigned num_ids;
156 struct list_head ids_lru;
157 struct amdgpu_vm_id ids[AMDGPU_NUM_VM];
158 160
159 /* Handling of VM fences */ 161 /* Handling of VM fences */
160 u64 fence_context; 162 u64 fence_context;
@@ -200,7 +202,8 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
200 struct amdgpu_sync *sync, struct dma_fence *fence, 202 struct amdgpu_sync *sync, struct dma_fence *fence,
201 struct amdgpu_job *job); 203 struct amdgpu_job *job);
202int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job); 204int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job);
203void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id); 205void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
206 unsigned vmid);
204int amdgpu_vm_update_directories(struct amdgpu_device *adev, 207int amdgpu_vm_update_directories(struct amdgpu_device *adev,
205 struct amdgpu_vm *vm); 208 struct amdgpu_vm *vm);
206int amdgpu_vm_clear_freed(struct amdgpu_device *adev, 209int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index 11ccda83d767..6dc1410b380f 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -1267,30 +1267,33 @@ static int ci_dpm_set_fan_speed_percent(struct amdgpu_device *adev,
1267 1267
1268static void ci_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode) 1268static void ci_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode)
1269{ 1269{
1270 if (mode) { 1270 switch (mode) {
1271 /* stop auto-manage */ 1271 case AMD_FAN_CTRL_NONE:
1272 if (adev->pm.dpm.fan.ucode_fan_control) 1272 if (adev->pm.dpm.fan.ucode_fan_control)
1273 ci_fan_ctrl_stop_smc_fan_control(adev); 1273 ci_fan_ctrl_stop_smc_fan_control(adev);
1274 ci_fan_ctrl_set_static_mode(adev, mode); 1274 ci_dpm_set_fan_speed_percent(adev, 100);
1275 } else { 1275 break;
1276 /* restart auto-manage */ 1276 case AMD_FAN_CTRL_MANUAL:
1277 if (adev->pm.dpm.fan.ucode_fan_control)
1278 ci_fan_ctrl_stop_smc_fan_control(adev);
1279 break;
1280 case AMD_FAN_CTRL_AUTO:
1277 if (adev->pm.dpm.fan.ucode_fan_control) 1281 if (adev->pm.dpm.fan.ucode_fan_control)
1278 ci_thermal_start_smc_fan_control(adev); 1282 ci_thermal_start_smc_fan_control(adev);
1279 else 1283 break;
1280 ci_fan_ctrl_set_default_mode(adev); 1284 default:
1285 break;
1281 } 1286 }
1282} 1287}
1283 1288
1284static u32 ci_dpm_get_fan_control_mode(struct amdgpu_device *adev) 1289static u32 ci_dpm_get_fan_control_mode(struct amdgpu_device *adev)
1285{ 1290{
1286 struct ci_power_info *pi = ci_get_pi(adev); 1291 struct ci_power_info *pi = ci_get_pi(adev);
1287 u32 tmp;
1288 1292
1289 if (pi->fan_is_controlled_by_smc) 1293 if (pi->fan_is_controlled_by_smc)
1290 return 0; 1294 return AMD_FAN_CTRL_AUTO;
1291 1295 else
1292 tmp = RREG32_SMC(ixCG_FDO_CTRL2) & CG_FDO_CTRL2__FDO_PWM_MODE_MASK; 1296 return AMD_FAN_CTRL_MANUAL;
1293 return (tmp >> CG_FDO_CTRL2__FDO_PWM_MODE__SHIFT);
1294} 1297}
1295 1298
1296#if 0 1299#if 0
@@ -3036,6 +3039,7 @@ static int ci_populate_single_memory_level(struct amdgpu_device *adev,
3036 memory_clock, 3039 memory_clock,
3037 &memory_level->MinVddcPhases); 3040 &memory_level->MinVddcPhases);
3038 3041
3042 memory_level->EnabledForActivity = 1;
3039 memory_level->EnabledForThrottle = 1; 3043 memory_level->EnabledForThrottle = 1;
3040 memory_level->UpH = 0; 3044 memory_level->UpH = 0;
3041 memory_level->DownH = 100; 3045 memory_level->DownH = 100;
@@ -3468,8 +3472,6 @@ static int ci_populate_all_memory_levels(struct amdgpu_device *adev)
3468 return ret; 3472 return ret;
3469 } 3473 }
3470 3474
3471 pi->smc_state_table.MemoryLevel[0].EnabledForActivity = 1;
3472
3473 if ((dpm_table->mclk_table.count >= 2) && 3475 if ((dpm_table->mclk_table.count >= 2) &&
3474 ((adev->pdev->device == 0x67B0) || (adev->pdev->device == 0x67B1))) { 3476 ((adev->pdev->device == 0x67B0) || (adev->pdev->device == 0x67B1))) {
3475 pi->smc_state_table.MemoryLevel[1].MinVddc = 3477 pi->smc_state_table.MemoryLevel[1].MinVddc =
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index ba98d35340a3..0cdeb6a2e4a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -2230,7 +2230,7 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
2230 if (!atomic && fb && fb != crtc->primary->fb) { 2230 if (!atomic && fb && fb != crtc->primary->fb) {
2231 amdgpu_fb = to_amdgpu_framebuffer(fb); 2231 amdgpu_fb = to_amdgpu_framebuffer(fb);
2232 abo = gem_to_amdgpu_bo(amdgpu_fb->obj); 2232 abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
2233 r = amdgpu_bo_reserve(abo, false); 2233 r = amdgpu_bo_reserve(abo, true);
2234 if (unlikely(r != 0)) 2234 if (unlikely(r != 0))
2235 return r; 2235 return r;
2236 amdgpu_bo_unpin(abo); 2236 amdgpu_bo_unpin(abo);
@@ -2589,7 +2589,7 @@ static int dce_v10_0_crtc_cursor_set2(struct drm_crtc *crtc,
2589unpin: 2589unpin:
2590 if (amdgpu_crtc->cursor_bo) { 2590 if (amdgpu_crtc->cursor_bo) {
2591 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); 2591 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2592 ret = amdgpu_bo_reserve(aobj, false); 2592 ret = amdgpu_bo_reserve(aobj, true);
2593 if (likely(ret == 0)) { 2593 if (likely(ret == 0)) {
2594 amdgpu_bo_unpin(aobj); 2594 amdgpu_bo_unpin(aobj);
2595 amdgpu_bo_unreserve(aobj); 2595 amdgpu_bo_unreserve(aobj);
@@ -2720,7 +2720,7 @@ static void dce_v10_0_crtc_disable(struct drm_crtc *crtc)
2720 2720
2721 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); 2721 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
2722 abo = gem_to_amdgpu_bo(amdgpu_fb->obj); 2722 abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
2723 r = amdgpu_bo_reserve(abo, false); 2723 r = amdgpu_bo_reserve(abo, true);
2724 if (unlikely(r)) 2724 if (unlikely(r))
2725 DRM_ERROR("failed to reserve abo before unpin\n"); 2725 DRM_ERROR("failed to reserve abo before unpin\n");
2726 else { 2726 else {
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index e59bc42df18c..773654a19749 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -2214,7 +2214,7 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
2214 if (!atomic && fb && fb != crtc->primary->fb) { 2214 if (!atomic && fb && fb != crtc->primary->fb) {
2215 amdgpu_fb = to_amdgpu_framebuffer(fb); 2215 amdgpu_fb = to_amdgpu_framebuffer(fb);
2216 abo = gem_to_amdgpu_bo(amdgpu_fb->obj); 2216 abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
2217 r = amdgpu_bo_reserve(abo, false); 2217 r = amdgpu_bo_reserve(abo, true);
2218 if (unlikely(r != 0)) 2218 if (unlikely(r != 0))
2219 return r; 2219 return r;
2220 amdgpu_bo_unpin(abo); 2220 amdgpu_bo_unpin(abo);
@@ -2609,7 +2609,7 @@ static int dce_v11_0_crtc_cursor_set2(struct drm_crtc *crtc,
2609unpin: 2609unpin:
2610 if (amdgpu_crtc->cursor_bo) { 2610 if (amdgpu_crtc->cursor_bo) {
2611 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); 2611 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2612 ret = amdgpu_bo_reserve(aobj, false); 2612 ret = amdgpu_bo_reserve(aobj, true);
2613 if (likely(ret == 0)) { 2613 if (likely(ret == 0)) {
2614 amdgpu_bo_unpin(aobj); 2614 amdgpu_bo_unpin(aobj);
2615 amdgpu_bo_unreserve(aobj); 2615 amdgpu_bo_unreserve(aobj);
@@ -2740,7 +2740,7 @@ static void dce_v11_0_crtc_disable(struct drm_crtc *crtc)
2740 2740
2741 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); 2741 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
2742 abo = gem_to_amdgpu_bo(amdgpu_fb->obj); 2742 abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
2743 r = amdgpu_bo_reserve(abo, false); 2743 r = amdgpu_bo_reserve(abo, true);
2744 if (unlikely(r)) 2744 if (unlikely(r))
2745 DRM_ERROR("failed to reserve abo before unpin\n"); 2745 DRM_ERROR("failed to reserve abo before unpin\n");
2746 else { 2746 else {
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index 307269bda4fa..1f3552967ba3 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -979,7 +979,7 @@ static void dce_v6_0_program_watermarks(struct amdgpu_device *adev,
979 u32 priority_a_mark = 0, priority_b_mark = 0; 979 u32 priority_a_mark = 0, priority_b_mark = 0;
980 u32 priority_a_cnt = PRIORITY_OFF; 980 u32 priority_a_cnt = PRIORITY_OFF;
981 u32 priority_b_cnt = PRIORITY_OFF; 981 u32 priority_b_cnt = PRIORITY_OFF;
982 u32 tmp, arb_control3; 982 u32 tmp, arb_control3, lb_vblank_lead_lines = 0;
983 fixed20_12 a, b, c; 983 fixed20_12 a, b, c;
984 984
985 if (amdgpu_crtc->base.enabled && num_heads && mode) { 985 if (amdgpu_crtc->base.enabled && num_heads && mode) {
@@ -1091,6 +1091,8 @@ static void dce_v6_0_program_watermarks(struct amdgpu_device *adev,
1091 c.full = dfixed_div(c, a); 1091 c.full = dfixed_div(c, a);
1092 priority_b_mark = dfixed_trunc(c); 1092 priority_b_mark = dfixed_trunc(c);
1093 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 1093 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
1094
1095 lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
1094 } 1096 }
1095 1097
1096 /* select wm A */ 1098 /* select wm A */
@@ -1120,6 +1122,9 @@ static void dce_v6_0_program_watermarks(struct amdgpu_device *adev,
1120 /* save values for DPM */ 1122 /* save values for DPM */
1121 amdgpu_crtc->line_time = line_time; 1123 amdgpu_crtc->line_time = line_time;
1122 amdgpu_crtc->wm_high = latency_watermark_a; 1124 amdgpu_crtc->wm_high = latency_watermark_a;
1125
1126 /* Save number of lines the linebuffer leads before the scanout */
1127 amdgpu_crtc->lb_vblank_lead_lines = lb_vblank_lead_lines;
1123} 1128}
1124 1129
1125/* watermark setup */ 1130/* watermark setup */
@@ -1640,7 +1645,7 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
1640 if (!atomic && fb && fb != crtc->primary->fb) { 1645 if (!atomic && fb && fb != crtc->primary->fb) {
1641 amdgpu_fb = to_amdgpu_framebuffer(fb); 1646 amdgpu_fb = to_amdgpu_framebuffer(fb);
1642 abo = gem_to_amdgpu_bo(amdgpu_fb->obj); 1647 abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
1643 r = amdgpu_bo_reserve(abo, false); 1648 r = amdgpu_bo_reserve(abo, true);
1644 if (unlikely(r != 0)) 1649 if (unlikely(r != 0))
1645 return r; 1650 return r;
1646 amdgpu_bo_unpin(abo); 1651 amdgpu_bo_unpin(abo);
@@ -1957,7 +1962,7 @@ static int dce_v6_0_crtc_cursor_set2(struct drm_crtc *crtc,
1957unpin: 1962unpin:
1958 if (amdgpu_crtc->cursor_bo) { 1963 if (amdgpu_crtc->cursor_bo) {
1959 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); 1964 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
1960 ret = amdgpu_bo_reserve(aobj, false); 1965 ret = amdgpu_bo_reserve(aobj, true);
1961 if (likely(ret == 0)) { 1966 if (likely(ret == 0)) {
1962 amdgpu_bo_unpin(aobj); 1967 amdgpu_bo_unpin(aobj);
1963 amdgpu_bo_unreserve(aobj); 1968 amdgpu_bo_unreserve(aobj);
@@ -2083,7 +2088,7 @@ static void dce_v6_0_crtc_disable(struct drm_crtc *crtc)
2083 2088
2084 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); 2089 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
2085 abo = gem_to_amdgpu_bo(amdgpu_fb->obj); 2090 abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
2086 r = amdgpu_bo_reserve(abo, false); 2091 r = amdgpu_bo_reserve(abo, true);
2087 if (unlikely(r)) 2092 if (unlikely(r))
2088 DRM_ERROR("failed to reserve abo before unpin\n"); 2093 DRM_ERROR("failed to reserve abo before unpin\n");
2089 else { 2094 else {
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 6df7a28e8aac..3c558c170e5e 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -2089,7 +2089,7 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
2089 if (!atomic && fb && fb != crtc->primary->fb) { 2089 if (!atomic && fb && fb != crtc->primary->fb) {
2090 amdgpu_fb = to_amdgpu_framebuffer(fb); 2090 amdgpu_fb = to_amdgpu_framebuffer(fb);
2091 abo = gem_to_amdgpu_bo(amdgpu_fb->obj); 2091 abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
2092 r = amdgpu_bo_reserve(abo, false); 2092 r = amdgpu_bo_reserve(abo, true);
2093 if (unlikely(r != 0)) 2093 if (unlikely(r != 0))
2094 return r; 2094 return r;
2095 amdgpu_bo_unpin(abo); 2095 amdgpu_bo_unpin(abo);
@@ -2440,7 +2440,7 @@ static int dce_v8_0_crtc_cursor_set2(struct drm_crtc *crtc,
2440unpin: 2440unpin:
2441 if (amdgpu_crtc->cursor_bo) { 2441 if (amdgpu_crtc->cursor_bo) {
2442 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); 2442 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2443 ret = amdgpu_bo_reserve(aobj, false); 2443 ret = amdgpu_bo_reserve(aobj, true);
2444 if (likely(ret == 0)) { 2444 if (likely(ret == 0)) {
2445 amdgpu_bo_unpin(aobj); 2445 amdgpu_bo_unpin(aobj);
2446 amdgpu_bo_unreserve(aobj); 2446 amdgpu_bo_unreserve(aobj);
@@ -2571,7 +2571,7 @@ static void dce_v8_0_crtc_disable(struct drm_crtc *crtc)
2571 2571
2572 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); 2572 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
2573 abo = gem_to_amdgpu_bo(amdgpu_fb->obj); 2573 abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
2574 r = amdgpu_bo_reserve(abo, false); 2574 r = amdgpu_bo_reserve(abo, true);
2575 if (unlikely(r)) 2575 if (unlikely(r))
2576 DRM_ERROR("failed to reserve abo before unpin\n"); 2576 DRM_ERROR("failed to reserve abo before unpin\n");
2577 else { 2577 else {
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
index 81a24b6b4846..f1b479b6ac98 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -248,7 +248,7 @@ static void dce_virtual_crtc_disable(struct drm_crtc *crtc)
248 248
249 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); 249 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
250 abo = gem_to_amdgpu_bo(amdgpu_fb->obj); 250 abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
251 r = amdgpu_bo_reserve(abo, false); 251 r = amdgpu_bo_reserve(abo, true);
252 if (unlikely(r)) 252 if (unlikely(r))
253 DRM_ERROR("failed to reserve abo before unpin\n"); 253 DRM_ERROR("failed to reserve abo before unpin\n");
254 else { 254 else {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index 4c4874fdf59f..a125f9d44577 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -1579,7 +1579,7 @@ static void gfx_v6_0_setup_spi(struct amdgpu_device *adev)
1579 1579
1580static void gfx_v6_0_config_init(struct amdgpu_device *adev) 1580static void gfx_v6_0_config_init(struct amdgpu_device *adev)
1581{ 1581{
1582 adev->gfx.config.double_offchip_lds_buf = 1; 1582 adev->gfx.config.double_offchip_lds_buf = 0;
1583} 1583}
1584 1584
1585static void gfx_v6_0_gpu_init(struct amdgpu_device *adev) 1585static void gfx_v6_0_gpu_init(struct amdgpu_device *adev)
@@ -2437,7 +2437,7 @@ static void gfx_v6_0_rlc_fini(struct amdgpu_device *adev)
2437 int r; 2437 int r;
2438 2438
2439 if (adev->gfx.rlc.save_restore_obj) { 2439 if (adev->gfx.rlc.save_restore_obj) {
2440 r = amdgpu_bo_reserve(adev->gfx.rlc.save_restore_obj, false); 2440 r = amdgpu_bo_reserve(adev->gfx.rlc.save_restore_obj, true);
2441 if (unlikely(r != 0)) 2441 if (unlikely(r != 0))
2442 dev_warn(adev->dev, "(%d) reserve RLC sr bo failed\n", r); 2442 dev_warn(adev->dev, "(%d) reserve RLC sr bo failed\n", r);
2443 amdgpu_bo_unpin(adev->gfx.rlc.save_restore_obj); 2443 amdgpu_bo_unpin(adev->gfx.rlc.save_restore_obj);
@@ -2448,7 +2448,7 @@ static void gfx_v6_0_rlc_fini(struct amdgpu_device *adev)
2448 } 2448 }
2449 2449
2450 if (adev->gfx.rlc.clear_state_obj) { 2450 if (adev->gfx.rlc.clear_state_obj) {
2451 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false); 2451 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
2452 if (unlikely(r != 0)) 2452 if (unlikely(r != 0))
2453 dev_warn(adev->dev, "(%d) reserve RLC c bo failed\n", r); 2453 dev_warn(adev->dev, "(%d) reserve RLC c bo failed\n", r);
2454 amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj); 2454 amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
@@ -2459,7 +2459,7 @@ static void gfx_v6_0_rlc_fini(struct amdgpu_device *adev)
2459 } 2459 }
2460 2460
2461 if (adev->gfx.rlc.cp_table_obj) { 2461 if (adev->gfx.rlc.cp_table_obj) {
2462 r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false); 2462 r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, true);
2463 if (unlikely(r != 0)) 2463 if (unlikely(r != 0))
2464 dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r); 2464 dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
2465 amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj); 2465 amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj);
@@ -3292,7 +3292,7 @@ static int gfx_v6_0_sw_init(void *handle)
3292 ring->me = 1; 3292 ring->me = 1;
3293 ring->pipe = i; 3293 ring->pipe = i;
3294 ring->queue = i; 3294 ring->queue = i;
3295 sprintf(ring->name, "comp %d.%d.%d", ring->me, ring->pipe, ring->queue); 3295 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
3296 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe; 3296 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe;
3297 r = amdgpu_ring_init(adev, ring, 1024, 3297 r = amdgpu_ring_init(adev, ring, 1024,
3298 &adev->gfx.eop_irq, irq_type); 3298 &adev->gfx.eop_irq, irq_type);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 8a8bc2fe6f2e..ee2f2139e2eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -1935,7 +1935,7 @@ static void gfx_v7_0_gpu_init(struct amdgpu_device *adev)
1935 INDEX_STRIDE, 3); 1935 INDEX_STRIDE, 3);
1936 1936
1937 mutex_lock(&adev->srbm_mutex); 1937 mutex_lock(&adev->srbm_mutex);
1938 for (i = 0; i < adev->vm_manager.num_ids; i++) { 1938 for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
1939 if (i == 0) 1939 if (i == 0)
1940 sh_mem_base = 0; 1940 sh_mem_base = 0;
1941 else 1941 else
@@ -2792,7 +2792,7 @@ static void gfx_v7_0_cp_compute_fini(struct amdgpu_device *adev)
2792 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 2792 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
2793 2793
2794 if (ring->mqd_obj) { 2794 if (ring->mqd_obj) {
2795 r = amdgpu_bo_reserve(ring->mqd_obj, false); 2795 r = amdgpu_bo_reserve(ring->mqd_obj, true);
2796 if (unlikely(r != 0)) 2796 if (unlikely(r != 0))
2797 dev_warn(adev->dev, "(%d) reserve MQD bo failed\n", r); 2797 dev_warn(adev->dev, "(%d) reserve MQD bo failed\n", r);
2798 2798
@@ -2810,7 +2810,7 @@ static void gfx_v7_0_mec_fini(struct amdgpu_device *adev)
2810 int r; 2810 int r;
2811 2811
2812 if (adev->gfx.mec.hpd_eop_obj) { 2812 if (adev->gfx.mec.hpd_eop_obj) {
2813 r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, false); 2813 r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, true);
2814 if (unlikely(r != 0)) 2814 if (unlikely(r != 0))
2815 dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r); 2815 dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r);
2816 amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj); 2816 amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj);
@@ -3359,7 +3359,7 @@ static void gfx_v7_0_rlc_fini(struct amdgpu_device *adev)
3359 3359
3360 /* save restore block */ 3360 /* save restore block */
3361 if (adev->gfx.rlc.save_restore_obj) { 3361 if (adev->gfx.rlc.save_restore_obj) {
3362 r = amdgpu_bo_reserve(adev->gfx.rlc.save_restore_obj, false); 3362 r = amdgpu_bo_reserve(adev->gfx.rlc.save_restore_obj, true);
3363 if (unlikely(r != 0)) 3363 if (unlikely(r != 0))
3364 dev_warn(adev->dev, "(%d) reserve RLC sr bo failed\n", r); 3364 dev_warn(adev->dev, "(%d) reserve RLC sr bo failed\n", r);
3365 amdgpu_bo_unpin(adev->gfx.rlc.save_restore_obj); 3365 amdgpu_bo_unpin(adev->gfx.rlc.save_restore_obj);
@@ -3371,7 +3371,7 @@ static void gfx_v7_0_rlc_fini(struct amdgpu_device *adev)
3371 3371
3372 /* clear state block */ 3372 /* clear state block */
3373 if (adev->gfx.rlc.clear_state_obj) { 3373 if (adev->gfx.rlc.clear_state_obj) {
3374 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false); 3374 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
3375 if (unlikely(r != 0)) 3375 if (unlikely(r != 0))
3376 dev_warn(adev->dev, "(%d) reserve RLC c bo failed\n", r); 3376 dev_warn(adev->dev, "(%d) reserve RLC c bo failed\n", r);
3377 amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj); 3377 amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
@@ -3383,7 +3383,7 @@ static void gfx_v7_0_rlc_fini(struct amdgpu_device *adev)
3383 3383
3384 /* clear state block */ 3384 /* clear state block */
3385 if (adev->gfx.rlc.cp_table_obj) { 3385 if (adev->gfx.rlc.cp_table_obj) {
3386 r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false); 3386 r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, true);
3387 if (unlikely(r != 0)) 3387 if (unlikely(r != 0))
3388 dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r); 3388 dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3389 amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj); 3389 amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index dad8a4cd1b37..758d636a6f52 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -1239,7 +1239,7 @@ static void gfx_v8_0_rlc_fini(struct amdgpu_device *adev)
1239 1239
1240 /* clear state block */ 1240 /* clear state block */
1241 if (adev->gfx.rlc.clear_state_obj) { 1241 if (adev->gfx.rlc.clear_state_obj) {
1242 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false); 1242 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
1243 if (unlikely(r != 0)) 1243 if (unlikely(r != 0))
1244 dev_warn(adev->dev, "(%d) reserve RLC cbs bo failed\n", r); 1244 dev_warn(adev->dev, "(%d) reserve RLC cbs bo failed\n", r);
1245 amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj); 1245 amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
@@ -1250,7 +1250,7 @@ static void gfx_v8_0_rlc_fini(struct amdgpu_device *adev)
1250 1250
1251 /* jump table block */ 1251 /* jump table block */
1252 if (adev->gfx.rlc.cp_table_obj) { 1252 if (adev->gfx.rlc.cp_table_obj) {
1253 r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false); 1253 r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, true);
1254 if (unlikely(r != 0)) 1254 if (unlikely(r != 0))
1255 dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r); 1255 dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
1256 amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj); 1256 amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj);
@@ -1363,7 +1363,7 @@ static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
1363 int r; 1363 int r;
1364 1364
1365 if (adev->gfx.mec.hpd_eop_obj) { 1365 if (adev->gfx.mec.hpd_eop_obj) {
1366 r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, false); 1366 r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, true);
1367 if (unlikely(r != 0)) 1367 if (unlikely(r != 0))
1368 dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r); 1368 dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r);
1369 amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj); 1369 amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj);
@@ -1490,7 +1490,7 @@ static int gfx_v8_0_kiq_init(struct amdgpu_device *adev)
1490 1490
1491 memset(hpd, 0, MEC_HPD_SIZE); 1491 memset(hpd, 0, MEC_HPD_SIZE);
1492 1492
1493 r = amdgpu_bo_reserve(kiq->eop_obj, false); 1493 r = amdgpu_bo_reserve(kiq->eop_obj, true);
1494 if (unlikely(r != 0)) 1494 if (unlikely(r != 0))
1495 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r); 1495 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r);
1496 amdgpu_bo_kunmap(kiq->eop_obj); 1496 amdgpu_bo_kunmap(kiq->eop_obj);
@@ -1932,6 +1932,7 @@ static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
1932 case 0xca: 1932 case 0xca:
1933 case 0xce: 1933 case 0xce:
1934 case 0x88: 1934 case 0x88:
1935 case 0xe6:
1935 /* B6 */ 1936 /* B6 */
1936 adev->gfx.config.max_cu_per_sh = 6; 1937 adev->gfx.config.max_cu_per_sh = 6;
1937 break; 1938 break;
@@ -1964,17 +1965,28 @@ static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
1964 adev->gfx.config.max_backends_per_se = 1; 1965 adev->gfx.config.max_backends_per_se = 1;
1965 1966
1966 switch (adev->pdev->revision) { 1967 switch (adev->pdev->revision) {
1968 case 0x80:
1969 case 0x81:
1967 case 0xc0: 1970 case 0xc0:
1968 case 0xc1: 1971 case 0xc1:
1969 case 0xc2: 1972 case 0xc2:
1970 case 0xc4: 1973 case 0xc4:
1971 case 0xc8: 1974 case 0xc8:
1972 case 0xc9: 1975 case 0xc9:
1976 case 0xd6:
1977 case 0xda:
1978 case 0xe9:
1979 case 0xea:
1973 adev->gfx.config.max_cu_per_sh = 3; 1980 adev->gfx.config.max_cu_per_sh = 3;
1974 break; 1981 break;
1982 case 0x83:
1975 case 0xd0: 1983 case 0xd0:
1976 case 0xd1: 1984 case 0xd1:
1977 case 0xd2: 1985 case 0xd2:
1986 case 0xd4:
1987 case 0xdb:
1988 case 0xe1:
1989 case 0xe2:
1978 default: 1990 default:
1979 adev->gfx.config.max_cu_per_sh = 2; 1991 adev->gfx.config.max_cu_per_sh = 2;
1980 break; 1992 break;
@@ -3890,7 +3902,7 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
3890 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG, 3902 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
3891 INDEX_STRIDE, 3); 3903 INDEX_STRIDE, 3);
3892 mutex_lock(&adev->srbm_mutex); 3904 mutex_lock(&adev->srbm_mutex);
3893 for (i = 0; i < adev->vm_manager.num_ids; i++) { 3905 for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
3894 vi_srbm_select(adev, 0, 0, 0, i); 3906 vi_srbm_select(adev, 0, 0, 0, i);
3895 /* CP and shaders */ 3907 /* CP and shaders */
3896 if (i == 0) { 3908 if (i == 0) {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index a447b70841c9..0c16b7563b73 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -39,7 +39,6 @@
39 39
40#define GFX9_NUM_GFX_RINGS 1 40#define GFX9_NUM_GFX_RINGS 1
41#define GFX9_NUM_COMPUTE_RINGS 8 41#define GFX9_NUM_COMPUTE_RINGS 8
42#define GFX9_NUM_SE 4
43#define RLCG_UCODE_LOADING_START_ADDRESS 0x2000 42#define RLCG_UCODE_LOADING_START_ADDRESS 0x2000
44 43
45MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 44MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
@@ -453,7 +452,7 @@ static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
453 int r; 452 int r;
454 453
455 if (adev->gfx.mec.hpd_eop_obj) { 454 if (adev->gfx.mec.hpd_eop_obj) {
456 r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, false); 455 r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, true);
457 if (unlikely(r != 0)) 456 if (unlikely(r != 0))
458 dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r); 457 dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r);
459 amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj); 458 amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj);
@@ -463,7 +462,7 @@ static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
463 adev->gfx.mec.hpd_eop_obj = NULL; 462 adev->gfx.mec.hpd_eop_obj = NULL;
464 } 463 }
465 if (adev->gfx.mec.mec_fw_obj) { 464 if (adev->gfx.mec.mec_fw_obj) {
466 r = amdgpu_bo_reserve(adev->gfx.mec.mec_fw_obj, false); 465 r = amdgpu_bo_reserve(adev->gfx.mec.mec_fw_obj, true);
467 if (unlikely(r != 0)) 466 if (unlikely(r != 0))
468 dev_warn(adev->dev, "(%d) reserve mec firmware bo failed\n", r); 467 dev_warn(adev->dev, "(%d) reserve mec firmware bo failed\n", r);
469 amdgpu_bo_unpin(adev->gfx.mec.mec_fw_obj); 468 amdgpu_bo_unpin(adev->gfx.mec.mec_fw_obj);
@@ -599,7 +598,7 @@ static int gfx_v9_0_kiq_init(struct amdgpu_device *adev)
599 598
600 memset(hpd, 0, MEC_HPD_SIZE); 599 memset(hpd, 0, MEC_HPD_SIZE);
601 600
602 r = amdgpu_bo_reserve(kiq->eop_obj, false); 601 r = amdgpu_bo_reserve(kiq->eop_obj, true);
603 if (unlikely(r != 0)) 602 if (unlikely(r != 0))
604 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r); 603 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r);
605 amdgpu_bo_kunmap(kiq->eop_obj); 604 amdgpu_bo_kunmap(kiq->eop_obj);
@@ -631,7 +630,6 @@ static int gfx_v9_0_kiq_init_ring(struct amdgpu_device *adev,
631 ring->pipe = 1; 630 ring->pipe = 1;
632 } 631 }
633 632
634 irq->data = ring;
635 ring->queue = 0; 633 ring->queue = 0;
636 ring->eop_gpu_addr = kiq->eop_gpu_addr; 634 ring->eop_gpu_addr = kiq->eop_gpu_addr;
637 sprintf(ring->name, "kiq %d.%d.%d", ring->me, ring->pipe, ring->queue); 635 sprintf(ring->name, "kiq %d.%d.%d", ring->me, ring->pipe, ring->queue);
@@ -647,7 +645,6 @@ static void gfx_v9_0_kiq_free_ring(struct amdgpu_ring *ring,
647{ 645{
648 amdgpu_wb_free(ring->adev, ring->adev->virt.reg_val_offs); 646 amdgpu_wb_free(ring->adev, ring->adev->virt.reg_val_offs);
649 amdgpu_ring_fini(ring); 647 amdgpu_ring_fini(ring);
650 irq->data = NULL;
651} 648}
652 649
653/* create MQD for each compute queue */ 650/* create MQD for each compute queue */
@@ -705,19 +702,19 @@ static void gfx_v9_0_compute_mqd_sw_fini(struct amdgpu_device *adev)
705 702
706static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 703static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
707{ 704{
708 WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_IND_INDEX), 705 WREG32_SOC15(GC, 0, mmSQ_IND_INDEX,
709 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 706 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
710 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 707 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
711 (address << SQ_IND_INDEX__INDEX__SHIFT) | 708 (address << SQ_IND_INDEX__INDEX__SHIFT) |
712 (SQ_IND_INDEX__FORCE_READ_MASK)); 709 (SQ_IND_INDEX__FORCE_READ_MASK));
713 return RREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_IND_DATA)); 710 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
714} 711}
715 712
716static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 713static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
717 uint32_t wave, uint32_t thread, 714 uint32_t wave, uint32_t thread,
718 uint32_t regno, uint32_t num, uint32_t *out) 715 uint32_t regno, uint32_t num, uint32_t *out)
719{ 716{
720 WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_IND_INDEX), 717 WREG32_SOC15(GC, 0, mmSQ_IND_INDEX,
721 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 718 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
722 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 719 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
723 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 720 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
@@ -725,7 +722,7 @@ static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
725 (SQ_IND_INDEX__FORCE_READ_MASK) | 722 (SQ_IND_INDEX__FORCE_READ_MASK) |
726 (SQ_IND_INDEX__AUTO_INCR_MASK)); 723 (SQ_IND_INDEX__AUTO_INCR_MASK));
727 while (num--) 724 while (num--)
728 *(out++) = RREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_IND_DATA)); 725 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
729} 726}
730 727
731static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 728static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
@@ -774,7 +771,6 @@ static void gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
774 switch (adev->asic_type) { 771 switch (adev->asic_type) {
775 case CHIP_VEGA10: 772 case CHIP_VEGA10:
776 adev->gfx.config.max_shader_engines = 4; 773 adev->gfx.config.max_shader_engines = 4;
777 adev->gfx.config.max_tile_pipes = 8; //??
778 adev->gfx.config.max_cu_per_sh = 16; 774 adev->gfx.config.max_cu_per_sh = 16;
779 adev->gfx.config.max_sh_per_se = 1; 775 adev->gfx.config.max_sh_per_se = 1;
780 adev->gfx.config.max_backends_per_se = 4; 776 adev->gfx.config.max_backends_per_se = 4;
@@ -787,6 +783,8 @@ static void gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
787 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 783 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
788 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 784 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
789 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 785 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
786 adev->gfx.config.gs_vgt_table_depth = 32;
787 adev->gfx.config.gs_prim_buffer_depth = 1792;
790 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 788 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
791 break; 789 break;
792 default: 790 default:
@@ -801,6 +799,10 @@ static void gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
801 adev->gfx.config.gb_addr_config, 799 adev->gfx.config.gb_addr_config,
802 GB_ADDR_CONFIG, 800 GB_ADDR_CONFIG,
803 NUM_PIPES); 801 NUM_PIPES);
802
803 adev->gfx.config.max_tile_pipes =
804 adev->gfx.config.gb_addr_config_fields.num_pipes;
805
804 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 806 adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
805 REG_GET_FIELD( 807 REG_GET_FIELD(
806 adev->gfx.config.gb_addr_config, 808 adev->gfx.config.gb_addr_config,
@@ -841,7 +843,7 @@ static int gfx_v9_0_ngg_create_buf(struct amdgpu_device *adev,
841 } 843 }
842 size_se = size_se ? size_se : default_size_se; 844 size_se = size_se ? size_se : default_size_se;
843 845
844 ngg_buf->size = size_se * GFX9_NUM_SE; 846 ngg_buf->size = size_se * adev->gfx.config.max_shader_engines;
845 r = amdgpu_bo_create_kernel(adev, ngg_buf->size, 847 r = amdgpu_bo_create_kernel(adev, ngg_buf->size,
846 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, 848 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
847 &ngg_buf->bo, 849 &ngg_buf->bo,
@@ -888,7 +890,7 @@ static int gfx_v9_0_ngg_init(struct amdgpu_device *adev)
888 adev->gfx.ngg.gds_reserve_addr += adev->gds.mem.gfx_partition_size; 890 adev->gfx.ngg.gds_reserve_addr += adev->gds.mem.gfx_partition_size;
889 891
890 /* Primitive Buffer */ 892 /* Primitive Buffer */
891 r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[PRIM], 893 r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_PRIM],
892 amdgpu_prim_buf_per_se, 894 amdgpu_prim_buf_per_se,
893 64 * 1024); 895 64 * 1024);
894 if (r) { 896 if (r) {
@@ -897,7 +899,7 @@ static int gfx_v9_0_ngg_init(struct amdgpu_device *adev)
897 } 899 }
898 900
899 /* Position Buffer */ 901 /* Position Buffer */
900 r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[POS], 902 r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_POS],
901 amdgpu_pos_buf_per_se, 903 amdgpu_pos_buf_per_se,
902 256 * 1024); 904 256 * 1024);
903 if (r) { 905 if (r) {
@@ -906,7 +908,7 @@ static int gfx_v9_0_ngg_init(struct amdgpu_device *adev)
906 } 908 }
907 909
908 /* Control Sideband */ 910 /* Control Sideband */
909 r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[CNTL], 911 r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_CNTL],
910 amdgpu_cntl_sb_buf_per_se, 912 amdgpu_cntl_sb_buf_per_se,
911 256); 913 256);
912 if (r) { 914 if (r) {
@@ -918,7 +920,7 @@ static int gfx_v9_0_ngg_init(struct amdgpu_device *adev)
918 if (amdgpu_param_buf_per_se <= 0) 920 if (amdgpu_param_buf_per_se <= 0)
919 goto out; 921 goto out;
920 922
921 r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[PARAM], 923 r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_PARAM],
922 amdgpu_param_buf_per_se, 924 amdgpu_param_buf_per_se,
923 512 * 1024); 925 512 * 1024);
924 if (r) { 926 if (r) {
@@ -947,47 +949,47 @@ static int gfx_v9_0_ngg_en(struct amdgpu_device *adev)
947 949
948 /* Program buffer size */ 950 /* Program buffer size */
949 data = 0; 951 data = 0;
950 size = adev->gfx.ngg.buf[PRIM].size / 256; 952 size = adev->gfx.ngg.buf[NGG_PRIM].size / 256;
951 data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, INDEX_BUF_SIZE, size); 953 data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, INDEX_BUF_SIZE, size);
952 954
953 size = adev->gfx.ngg.buf[POS].size / 256; 955 size = adev->gfx.ngg.buf[NGG_POS].size / 256;
954 data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, POS_BUF_SIZE, size); 956 data = REG_SET_FIELD(data, WD_BUF_RESOURCE_1, POS_BUF_SIZE, size);
955 957
956 WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_BUF_RESOURCE_1), data); 958 WREG32_SOC15(GC, 0, mmWD_BUF_RESOURCE_1, data);
957 959
958 data = 0; 960 data = 0;
959 size = adev->gfx.ngg.buf[CNTL].size / 256; 961 size = adev->gfx.ngg.buf[NGG_CNTL].size / 256;
960 data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, CNTL_SB_BUF_SIZE, size); 962 data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, CNTL_SB_BUF_SIZE, size);
961 963
962 size = adev->gfx.ngg.buf[PARAM].size / 1024; 964 size = adev->gfx.ngg.buf[NGG_PARAM].size / 1024;
963 data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, PARAM_BUF_SIZE, size); 965 data = REG_SET_FIELD(data, WD_BUF_RESOURCE_2, PARAM_BUF_SIZE, size);
964 966
965 WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_BUF_RESOURCE_2), data); 967 WREG32_SOC15(GC, 0, mmWD_BUF_RESOURCE_2, data);
966 968
967 /* Program buffer base address */ 969 /* Program buffer base address */
968 base = lower_32_bits(adev->gfx.ngg.buf[PRIM].gpu_addr); 970 base = lower_32_bits(adev->gfx.ngg.buf[NGG_PRIM].gpu_addr);
969 data = REG_SET_FIELD(0, WD_INDEX_BUF_BASE, BASE, base); 971 data = REG_SET_FIELD(0, WD_INDEX_BUF_BASE, BASE, base);
970 WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_INDEX_BUF_BASE), data); 972 WREG32_SOC15(GC, 0, mmWD_INDEX_BUF_BASE, data);
971 973
972 base = upper_32_bits(adev->gfx.ngg.buf[PRIM].gpu_addr); 974 base = upper_32_bits(adev->gfx.ngg.buf[NGG_PRIM].gpu_addr);
973 data = REG_SET_FIELD(0, WD_INDEX_BUF_BASE_HI, BASE_HI, base); 975 data = REG_SET_FIELD(0, WD_INDEX_BUF_BASE_HI, BASE_HI, base);
974 WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_INDEX_BUF_BASE_HI), data); 976 WREG32_SOC15(GC, 0, mmWD_INDEX_BUF_BASE_HI, data);
975 977
976 base = lower_32_bits(adev->gfx.ngg.buf[POS].gpu_addr); 978 base = lower_32_bits(adev->gfx.ngg.buf[NGG_POS].gpu_addr);
977 data = REG_SET_FIELD(0, WD_POS_BUF_BASE, BASE, base); 979 data = REG_SET_FIELD(0, WD_POS_BUF_BASE, BASE, base);
978 WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_POS_BUF_BASE), data); 980 WREG32_SOC15(GC, 0, mmWD_POS_BUF_BASE, data);
979 981
980 base = upper_32_bits(adev->gfx.ngg.buf[POS].gpu_addr); 982 base = upper_32_bits(adev->gfx.ngg.buf[NGG_POS].gpu_addr);
981 data = REG_SET_FIELD(0, WD_POS_BUF_BASE_HI, BASE_HI, base); 983 data = REG_SET_FIELD(0, WD_POS_BUF_BASE_HI, BASE_HI, base);
982 WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_POS_BUF_BASE_HI), data); 984 WREG32_SOC15(GC, 0, mmWD_POS_BUF_BASE_HI, data);
983 985
984 base = lower_32_bits(adev->gfx.ngg.buf[CNTL].gpu_addr); 986 base = lower_32_bits(adev->gfx.ngg.buf[NGG_CNTL].gpu_addr);
985 data = REG_SET_FIELD(0, WD_CNTL_SB_BUF_BASE, BASE, base); 987 data = REG_SET_FIELD(0, WD_CNTL_SB_BUF_BASE, BASE, base);
986 WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_CNTL_SB_BUF_BASE), data); 988 WREG32_SOC15(GC, 0, mmWD_CNTL_SB_BUF_BASE, data);
987 989
988 base = upper_32_bits(adev->gfx.ngg.buf[CNTL].gpu_addr); 990 base = upper_32_bits(adev->gfx.ngg.buf[NGG_CNTL].gpu_addr);
989 data = REG_SET_FIELD(0, WD_CNTL_SB_BUF_BASE_HI, BASE_HI, base); 991 data = REG_SET_FIELD(0, WD_CNTL_SB_BUF_BASE_HI, BASE_HI, base);
990 WREG32(SOC15_REG_OFFSET(GC, 0, mmWD_CNTL_SB_BUF_BASE_HI), data); 992 WREG32_SOC15(GC, 0, mmWD_CNTL_SB_BUF_BASE_HI, data);
991 993
992 /* Clear GDS reserved memory */ 994 /* Clear GDS reserved memory */
993 r = amdgpu_ring_alloc(ring, 17); 995 r = amdgpu_ring_alloc(ring, 17);
@@ -1096,7 +1098,7 @@ static int gfx_v9_0_sw_init(void *handle)
1096 ring->pipe = i / 8; 1098 ring->pipe = i / 8;
1097 ring->queue = i % 8; 1099 ring->queue = i % 8;
1098 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE); 1100 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE);
1099 sprintf(ring->name, "comp %d.%d.%d", ring->me, ring->pipe, ring->queue); 1101 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1100 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe; 1102 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe;
1101 /* type-2 packets are deprecated on MEC, use type-3 instead */ 1103 /* type-2 packets are deprecated on MEC, use type-3 instead */
1102 r = amdgpu_ring_init(adev, ring, 1024, 1104 r = amdgpu_ring_init(adev, ring, 1024,
@@ -1203,7 +1205,7 @@ static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh
1203 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 1205 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
1204 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 1206 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1205 } 1207 }
1206 WREG32( SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data); 1208 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, data);
1207} 1209}
1208 1210
1209static u32 gfx_v9_0_create_bitmask(u32 bit_width) 1211static u32 gfx_v9_0_create_bitmask(u32 bit_width)
@@ -1215,8 +1217,8 @@ static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1215{ 1217{
1216 u32 data, mask; 1218 u32 data, mask;
1217 1219
1218 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCC_RB_BACKEND_DISABLE)); 1220 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
1219 data |= RREG32(SOC15_REG_OFFSET(GC, 0, mmGC_USER_RB_BACKEND_DISABLE)); 1221 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
1220 1222
1221 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 1223 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
1222 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 1224 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
@@ -1276,8 +1278,8 @@ static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
1276 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) { 1278 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
1277 soc15_grbm_select(adev, 0, 0, 0, i); 1279 soc15_grbm_select(adev, 0, 0, 0, i);
1278 /* CP and shaders */ 1280 /* CP and shaders */
1279 WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config); 1281 WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
1280 WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases); 1282 WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
1281 } 1283 }
1282 soc15_grbm_select(adev, 0, 0, 0, 0); 1284 soc15_grbm_select(adev, 0, 0, 0, 0);
1283 mutex_unlock(&adev->srbm_mutex); 1285 mutex_unlock(&adev->srbm_mutex);
@@ -1304,8 +1306,8 @@ static void gfx_v9_0_gpu_init(struct amdgpu_device *adev)
1304 tmp = 0; 1306 tmp = 0;
1305 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE, 1307 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
1306 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 1308 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
1307 WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), tmp); 1309 WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, tmp);
1308 WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), 0); 1310 WREG32_SOC15(GC, 0, mmSH_MEM_BASES, 0);
1309 } 1311 }
1310 soc15_grbm_select(adev, 0, 0, 0, 0); 1312 soc15_grbm_select(adev, 0, 0, 0, 0);
1311 1313
@@ -1320,7 +1322,7 @@ static void gfx_v9_0_gpu_init(struct amdgpu_device *adev)
1320 */ 1322 */
1321 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1323 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1322 1324
1323 WREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_FIFO_SIZE), 1325 WREG32_SOC15(GC, 0, mmPA_SC_FIFO_SIZE,
1324 (adev->gfx.config.sc_prim_fifo_size_frontend << 1326 (adev->gfx.config.sc_prim_fifo_size_frontend <<
1325 PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) | 1327 PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
1326 (adev->gfx.config.sc_prim_fifo_size_backend << 1328 (adev->gfx.config.sc_prim_fifo_size_backend <<
@@ -1343,7 +1345,7 @@ static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
1343 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1345 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1344 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1346 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1345 for (k = 0; k < adev->usec_timeout; k++) { 1347 for (k = 0; k < adev->usec_timeout; k++) {
1346 if (RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY)) == 0) 1348 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
1347 break; 1349 break;
1348 udelay(1); 1350 udelay(1);
1349 } 1351 }
@@ -1357,7 +1359,7 @@ static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
1357 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 1359 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
1358 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 1360 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
1359 for (k = 0; k < adev->usec_timeout; k++) { 1361 for (k = 0; k < adev->usec_timeout; k++) {
1360 if ((RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY)) & mask) == 0) 1362 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
1361 break; 1363 break;
1362 udelay(1); 1364 udelay(1);
1363 } 1365 }
@@ -1366,7 +1368,7 @@ static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
1366static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 1368static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1367 bool enable) 1369 bool enable)
1368{ 1370{
1369 u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0)); 1371 u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
1370 1372
1371 if (enable) 1373 if (enable)
1372 return; 1374 return;
@@ -1376,15 +1378,15 @@ static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1376 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 1378 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
1377 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 1379 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
1378 1380
1379 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), tmp); 1381 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
1380} 1382}
1381 1383
1382void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 1384void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
1383{ 1385{
1384 u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL)); 1386 u32 tmp = RREG32_SOC15(GC, 0, mmRLC_CNTL);
1385 1387
1386 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0); 1388 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
1387 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL), tmp); 1389 WREG32_SOC15(GC, 0, mmRLC_CNTL, tmp);
1388 1390
1389 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 1391 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
1390 1392
@@ -1415,17 +1417,17 @@ static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
1415 1417
1416#ifdef AMDGPU_RLC_DEBUG_RETRY 1418#ifdef AMDGPU_RLC_DEBUG_RETRY
1417 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 1419 /* RLC_GPM_GENERAL_6 : RLC Ucode version */
1418 rlc_ucode_ver = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_GENERAL_6)); 1420 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
1419 if(rlc_ucode_ver == 0x108) { 1421 if(rlc_ucode_ver == 0x108) {
1420 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 1422 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
1421 rlc_ucode_ver, adev->gfx.rlc_fw_version); 1423 rlc_ucode_ver, adev->gfx.rlc_fw_version);
1422 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 1424 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
1423 * default is 0x9C4 to create a 100us interval */ 1425 * default is 0x9C4 to create a 100us interval */
1424 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_TIMER_INT_3), 0x9C4); 1426 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
1425 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 1427 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
1426 * to disable the page fault retry interrupts, default is 1428 * to disable the page fault retry interrupts, default is
1427 * 0x100 (256) */ 1429 * 0x100 (256) */
1428 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_GENERAL_12), 0x100); 1430 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
1429 } 1431 }
1430#endif 1432#endif
1431} 1433}
@@ -1446,11 +1448,11 @@ static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
1446 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1448 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1447 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 1449 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1448 1450
1449 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UCODE_ADDR), 1451 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
1450 RLCG_UCODE_LOADING_START_ADDRESS); 1452 RLCG_UCODE_LOADING_START_ADDRESS);
1451 for (i = 0; i < fw_size; i++) 1453 for (i = 0; i < fw_size; i++)
1452 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UCODE_DATA), le32_to_cpup(fw_data++)); 1454 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
1453 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UCODE_ADDR), adev->gfx.rlc_fw_version); 1455 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
1454 1456
1455 return 0; 1457 return 0;
1456} 1458}
@@ -1465,10 +1467,10 @@ static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
1465 gfx_v9_0_rlc_stop(adev); 1467 gfx_v9_0_rlc_stop(adev);
1466 1468
1467 /* disable CG */ 1469 /* disable CG */
1468 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL), 0); 1470 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
1469 1471
1470 /* disable PG */ 1472 /* disable PG */
1471 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), 0); 1473 WREG32_SOC15(GC, 0, mmRLC_PG_CNTL, 0);
1472 1474
1473 gfx_v9_0_rlc_reset(adev); 1475 gfx_v9_0_rlc_reset(adev);
1474 1476
@@ -1487,7 +1489,7 @@ static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
1487static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 1489static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
1488{ 1490{
1489 int i; 1491 int i;
1490 u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL)); 1492 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
1491 1493
1492 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 1494 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
1493 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 1495 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
@@ -1496,7 +1498,7 @@ static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
1496 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 1498 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1497 adev->gfx.gfx_ring[i].ready = false; 1499 adev->gfx.gfx_ring[i].ready = false;
1498 } 1500 }
1499 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL), tmp); 1501 WREG32_SOC15(GC, 0, mmCP_ME_CNTL, tmp);
1500 udelay(50); 1502 udelay(50);
1501} 1503}
1502 1504
@@ -1529,30 +1531,30 @@ static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
1529 (adev->gfx.pfp_fw->data + 1531 (adev->gfx.pfp_fw->data +
1530 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 1532 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
1531 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 1533 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
1532 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PFP_UCODE_ADDR), 0); 1534 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
1533 for (i = 0; i < fw_size; i++) 1535 for (i = 0; i < fw_size; i++)
1534 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PFP_UCODE_DATA), le32_to_cpup(fw_data++)); 1536 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
1535 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PFP_UCODE_ADDR), adev->gfx.pfp_fw_version); 1537 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
1536 1538
1537 /* CE */ 1539 /* CE */
1538 fw_data = (const __le32 *) 1540 fw_data = (const __le32 *)
1539 (adev->gfx.ce_fw->data + 1541 (adev->gfx.ce_fw->data +
1540 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 1542 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
1541 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 1543 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
1542 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CE_UCODE_ADDR), 0); 1544 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
1543 for (i = 0; i < fw_size; i++) 1545 for (i = 0; i < fw_size; i++)
1544 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CE_UCODE_DATA), le32_to_cpup(fw_data++)); 1546 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
1545 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CE_UCODE_ADDR), adev->gfx.ce_fw_version); 1547 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
1546 1548
1547 /* ME */ 1549 /* ME */
1548 fw_data = (const __le32 *) 1550 fw_data = (const __le32 *)
1549 (adev->gfx.me_fw->data + 1551 (adev->gfx.me_fw->data +
1550 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 1552 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
1551 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 1553 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
1552 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_RAM_WADDR), 0); 1554 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
1553 for (i = 0; i < fw_size; i++) 1555 for (i = 0; i < fw_size; i++)
1554 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_RAM_DATA), le32_to_cpup(fw_data++)); 1556 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
1555 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_RAM_WADDR), adev->gfx.me_fw_version); 1557 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
1556 1558
1557 return 0; 1559 return 0;
1558} 1560}
@@ -1594,8 +1596,8 @@ static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
1594 int r, i; 1596 int r, i;
1595 1597
1596 /* init the CP */ 1598 /* init the CP */
1597 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MAX_CONTEXT), adev->gfx.config.max_hw_contexts - 1); 1599 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
1598 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_DEVICE_ID), 1); 1600 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
1599 1601
1600 gfx_v9_0_cp_gfx_enable(adev, true); 1602 gfx_v9_0_cp_gfx_enable(adev, true);
1601 1603
@@ -1650,10 +1652,10 @@ static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
1650 u64 rb_addr, rptr_addr, wptr_gpu_addr; 1652 u64 rb_addr, rptr_addr, wptr_gpu_addr;
1651 1653
1652 /* Set the write pointer delay */ 1654 /* Set the write pointer delay */
1653 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_DELAY), 0); 1655 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
1654 1656
1655 /* set the RB to use vmid 0 */ 1657 /* set the RB to use vmid 0 */
1656 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_VMID), 0); 1658 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
1657 1659
1658 /* Set ring buffer size */ 1660 /* Set ring buffer size */
1659 ring = &adev->gfx.gfx_ring[0]; 1661 ring = &adev->gfx.gfx_ring[0];
@@ -1663,30 +1665,30 @@ static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
1663#ifdef __BIG_ENDIAN 1665#ifdef __BIG_ENDIAN
1664 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 1666 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
1665#endif 1667#endif
1666 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_CNTL), tmp); 1668 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
1667 1669
1668 /* Initialize the ring buffer's write pointers */ 1670 /* Initialize the ring buffer's write pointers */
1669 ring->wptr = 0; 1671 ring->wptr = 0;
1670 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR), lower_32_bits(ring->wptr)); 1672 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
1671 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR_HI), upper_32_bits(ring->wptr)); 1673 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
1672 1674
1673 /* set the wb address wether it's enabled or not */ 1675 /* set the wb address wether it's enabled or not */
1674 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 1676 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
1675 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_RPTR_ADDR), lower_32_bits(rptr_addr)); 1677 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
1676 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_RPTR_ADDR_HI), upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 1678 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
1677 1679
1678 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 1680 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
1679 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO), lower_32_bits(wptr_gpu_addr)); 1681 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
1680 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI), upper_32_bits(wptr_gpu_addr)); 1682 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
1681 1683
1682 mdelay(1); 1684 mdelay(1);
1683 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_CNTL), tmp); 1685 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
1684 1686
1685 rb_addr = ring->gpu_addr >> 8; 1687 rb_addr = ring->gpu_addr >> 8;
1686 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_BASE), rb_addr); 1688 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
1687 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_BASE_HI), upper_32_bits(rb_addr)); 1689 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
1688 1690
1689 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_DOORBELL_CONTROL)); 1691 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
1690 if (ring->use_doorbell) { 1692 if (ring->use_doorbell) {
1691 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 1693 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
1692 DOORBELL_OFFSET, ring->doorbell_index); 1694 DOORBELL_OFFSET, ring->doorbell_index);
@@ -1695,13 +1697,13 @@ static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
1695 } else { 1697 } else {
1696 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 1698 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
1697 } 1699 }
1698 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_DOORBELL_CONTROL), tmp); 1700 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
1699 1701
1700 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 1702 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
1701 DOORBELL_RANGE_LOWER, ring->doorbell_index); 1703 DOORBELL_RANGE_LOWER, ring->doorbell_index);
1702 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER), tmp); 1704 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
1703 1705
1704 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER), 1706 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
1705 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 1707 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
1706 1708
1707 1709
@@ -1717,9 +1719,9 @@ static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
1717 int i; 1719 int i;
1718 1720
1719 if (enable) { 1721 if (enable) {
1720 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_CNTL), 0); 1722 WREG32_SOC15(GC, 0, mmCP_MEC_CNTL, 0);
1721 } else { 1723 } else {
1722 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_CNTL), 1724 WREG32_SOC15(GC, 0, mmCP_MEC_CNTL,
1723 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 1725 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
1724 for (i = 0; i < adev->gfx.num_compute_rings; i++) 1726 for (i = 0; i < adev->gfx.num_compute_rings; i++)
1725 adev->gfx.compute_ring[i].ready = false; 1727 adev->gfx.compute_ring[i].ready = false;
@@ -1756,21 +1758,21 @@ static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
1756 tmp = 0; 1758 tmp = 0;
1757 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 1759 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
1758 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 1760 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
1759 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_IC_BASE_CNTL), tmp); 1761 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
1760 1762
1761 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_IC_BASE_LO), 1763 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
1762 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 1764 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
1763 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_IC_BASE_HI), 1765 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
1764 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 1766 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
1765 1767
1766 /* MEC1 */ 1768 /* MEC1 */
1767 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_ME1_UCODE_ADDR), 1769 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
1768 mec_hdr->jt_offset); 1770 mec_hdr->jt_offset);
1769 for (i = 0; i < mec_hdr->jt_size; i++) 1771 for (i = 0; i < mec_hdr->jt_size; i++)
1770 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_ME1_UCODE_DATA), 1772 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
1771 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 1773 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
1772 1774
1773 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_ME1_UCODE_ADDR), 1775 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
1774 adev->gfx.mec_fw_version); 1776 adev->gfx.mec_fw_version);
1775 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 1777 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
1776 1778
@@ -1785,7 +1787,7 @@ static void gfx_v9_0_cp_compute_fini(struct amdgpu_device *adev)
1785 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 1787 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
1786 1788
1787 if (ring->mqd_obj) { 1789 if (ring->mqd_obj) {
1788 r = amdgpu_bo_reserve(ring->mqd_obj, false); 1790 r = amdgpu_bo_reserve(ring->mqd_obj, true);
1789 if (unlikely(r != 0)) 1791 if (unlikely(r != 0))
1790 dev_warn(adev->dev, "(%d) reserve MQD bo failed\n", r); 1792 dev_warn(adev->dev, "(%d) reserve MQD bo failed\n", r);
1791 1793
@@ -1823,12 +1825,12 @@ static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
1823 struct amdgpu_device *adev = ring->adev; 1825 struct amdgpu_device *adev = ring->adev;
1824 1826
1825 /* tell RLC which is KIQ queue */ 1827 /* tell RLC which is KIQ queue */
1826 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS)); 1828 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
1827 tmp &= 0xffffff00; 1829 tmp &= 0xffffff00;
1828 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 1830 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1829 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), tmp); 1831 WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
1830 tmp |= 0x80; 1832 tmp |= 0x80;
1831 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), tmp); 1833 WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
1832} 1834}
1833 1835
1834static void gfx_v9_0_kiq_enable(struct amdgpu_ring *ring) 1836static void gfx_v9_0_kiq_enable(struct amdgpu_ring *ring)
@@ -1898,14 +1900,14 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
1898 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 1900 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
1899 1901
1900 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 1902 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1901 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_CONTROL)); 1903 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
1902 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 1904 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
1903 (order_base_2(MEC_HPD_SIZE / 4) - 1)); 1905 (order_base_2(MEC_HPD_SIZE / 4) - 1));
1904 1906
1905 mqd->cp_hqd_eop_control = tmp; 1907 mqd->cp_hqd_eop_control = tmp;
1906 1908
1907 /* enable doorbell? */ 1909 /* enable doorbell? */
1908 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL)); 1910 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
1909 1911
1910 if (ring->use_doorbell) { 1912 if (ring->use_doorbell) {
1911 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 1913 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
@@ -1935,7 +1937,7 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
1935 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 1937 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
1936 1938
1937 /* set MQD vmid to 0 */ 1939 /* set MQD vmid to 0 */
1938 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_CONTROL)); 1940 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
1939 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 1941 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
1940 mqd->cp_mqd_control = tmp; 1942 mqd->cp_mqd_control = tmp;
1941 1943
@@ -1945,7 +1947,7 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
1945 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 1947 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
1946 1948
1947 /* set up the HQD, this is similar to CP_RB0_CNTL */ 1949 /* set up the HQD, this is similar to CP_RB0_CNTL */
1948 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_CONTROL)); 1950 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
1949 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 1951 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
1950 (order_base_2(ring->ring_size / 4) - 1)); 1952 (order_base_2(ring->ring_size / 4) - 1));
1951 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 1953 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
@@ -1973,7 +1975,7 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
1973 tmp = 0; 1975 tmp = 0;
1974 /* enable the doorbell if requested */ 1976 /* enable the doorbell if requested */
1975 if (ring->use_doorbell) { 1977 if (ring->use_doorbell) {
1976 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL)); 1978 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
1977 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 1979 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1978 DOORBELL_OFFSET, ring->doorbell_index); 1980 DOORBELL_OFFSET, ring->doorbell_index);
1979 1981
@@ -1989,15 +1991,20 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
1989 1991
1990 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 1992 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
1991 ring->wptr = 0; 1993 ring->wptr = 0;
1992 mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR); 1994 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
1993 1995
1994 /* set the vmid for the queue */ 1996 /* set the vmid for the queue */
1995 mqd->cp_hqd_vmid = 0; 1997 mqd->cp_hqd_vmid = 0;
1996 1998
1997 tmp = RREG32(mmCP_HQD_PERSISTENT_STATE); 1999 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
1998 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 2000 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
1999 mqd->cp_hqd_persistent_state = tmp; 2001 mqd->cp_hqd_persistent_state = tmp;
2000 2002
2003 /* set MIN_IB_AVAIL_SIZE */
2004 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
2005 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
2006 mqd->cp_hqd_ib_control = tmp;
2007
2001 /* activate the queue */ 2008 /* activate the queue */
2002 mqd->cp_hqd_active = 1; 2009 mqd->cp_hqd_active = 1;
2003 2010
@@ -2013,94 +2020,94 @@ static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
2013 /* disable wptr polling */ 2020 /* disable wptr polling */
2014 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 2021 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
2015 2022
2016 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_BASE_ADDR), 2023 WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
2017 mqd->cp_hqd_eop_base_addr_lo); 2024 mqd->cp_hqd_eop_base_addr_lo);
2018 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI), 2025 WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
2019 mqd->cp_hqd_eop_base_addr_hi); 2026 mqd->cp_hqd_eop_base_addr_hi);
2020 2027
2021 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 2028 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2022 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_CONTROL), 2029 WREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL,
2023 mqd->cp_hqd_eop_control); 2030 mqd->cp_hqd_eop_control);
2024 2031
2025 /* enable doorbell? */ 2032 /* enable doorbell? */
2026 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), 2033 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
2027 mqd->cp_hqd_pq_doorbell_control); 2034 mqd->cp_hqd_pq_doorbell_control);
2028 2035
2029 /* disable the queue if it's active */ 2036 /* disable the queue if it's active */
2030 if (RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)) & 1) { 2037 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
2031 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), 1); 2038 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
2032 for (j = 0; j < adev->usec_timeout; j++) { 2039 for (j = 0; j < adev->usec_timeout; j++) {
2033 if (!(RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)) & 1)) 2040 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
2034 break; 2041 break;
2035 udelay(1); 2042 udelay(1);
2036 } 2043 }
2037 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), 2044 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
2038 mqd->cp_hqd_dequeue_request); 2045 mqd->cp_hqd_dequeue_request);
2039 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR), 2046 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR,
2040 mqd->cp_hqd_pq_rptr); 2047 mqd->cp_hqd_pq_rptr);
2041 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO), 2048 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO,
2042 mqd->cp_hqd_pq_wptr_lo); 2049 mqd->cp_hqd_pq_wptr_lo);
2043 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI), 2050 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI,
2044 mqd->cp_hqd_pq_wptr_hi); 2051 mqd->cp_hqd_pq_wptr_hi);
2045 } 2052 }
2046 2053
2047 /* set the pointer to the MQD */ 2054 /* set the pointer to the MQD */
2048 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR), 2055 WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR,
2049 mqd->cp_mqd_base_addr_lo); 2056 mqd->cp_mqd_base_addr_lo);
2050 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR_HI), 2057 WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR_HI,
2051 mqd->cp_mqd_base_addr_hi); 2058 mqd->cp_mqd_base_addr_hi);
2052 2059
2053 /* set MQD vmid to 0 */ 2060 /* set MQD vmid to 0 */
2054 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_CONTROL), 2061 WREG32_SOC15(GC, 0, mmCP_MQD_CONTROL,
2055 mqd->cp_mqd_control); 2062 mqd->cp_mqd_control);
2056 2063
2057 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 2064 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2058 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE), 2065 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE,
2059 mqd->cp_hqd_pq_base_lo); 2066 mqd->cp_hqd_pq_base_lo);
2060 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI), 2067 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE_HI,
2061 mqd->cp_hqd_pq_base_hi); 2068 mqd->cp_hqd_pq_base_hi);
2062 2069
2063 /* set up the HQD, this is similar to CP_RB0_CNTL */ 2070 /* set up the HQD, this is similar to CP_RB0_CNTL */
2064 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_CONTROL), 2071 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL,
2065 mqd->cp_hqd_pq_control); 2072 mqd->cp_hqd_pq_control);
2066 2073
2067 /* set the wb address whether it's enabled or not */ 2074 /* set the wb address whether it's enabled or not */
2068 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR), 2075 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
2069 mqd->cp_hqd_pq_rptr_report_addr_lo); 2076 mqd->cp_hqd_pq_rptr_report_addr_lo);
2070 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI), 2077 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
2071 mqd->cp_hqd_pq_rptr_report_addr_hi); 2078 mqd->cp_hqd_pq_rptr_report_addr_hi);
2072 2079
2073 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 2080 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2074 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR), 2081 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
2075 mqd->cp_hqd_pq_wptr_poll_addr_lo); 2082 mqd->cp_hqd_pq_wptr_poll_addr_lo);
2076 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI), 2083 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
2077 mqd->cp_hqd_pq_wptr_poll_addr_hi); 2084 mqd->cp_hqd_pq_wptr_poll_addr_hi);
2078 2085
2079 /* enable the doorbell if requested */ 2086 /* enable the doorbell if requested */
2080 if (ring->use_doorbell) { 2087 if (ring->use_doorbell) {
2081 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER), 2088 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
2082 (AMDGPU_DOORBELL64_KIQ *2) << 2); 2089 (AMDGPU_DOORBELL64_KIQ *2) << 2);
2083 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER), 2090 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
2084 (AMDGPU_DOORBELL64_USERQUEUE_END * 2) << 2); 2091 (AMDGPU_DOORBELL64_USERQUEUE_END * 2) << 2);
2085 } 2092 }
2086 2093
2087 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), 2094 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
2088 mqd->cp_hqd_pq_doorbell_control); 2095 mqd->cp_hqd_pq_doorbell_control);
2089 2096
2090 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 2097 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2091 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO), 2098 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO,
2092 mqd->cp_hqd_pq_wptr_lo); 2099 mqd->cp_hqd_pq_wptr_lo);
2093 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI), 2100 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI,
2094 mqd->cp_hqd_pq_wptr_hi); 2101 mqd->cp_hqd_pq_wptr_hi);
2095 2102
2096 /* set the vmid for the queue */ 2103 /* set the vmid for the queue */
2097 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_VMID), mqd->cp_hqd_vmid); 2104 WREG32_SOC15(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
2098 2105
2099 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PERSISTENT_STATE), 2106 WREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE,
2100 mqd->cp_hqd_persistent_state); 2107 mqd->cp_hqd_persistent_state);
2101 2108
2102 /* activate the queue */ 2109 /* activate the queue */
2103 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), 2110 WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE,
2104 mqd->cp_hqd_active); 2111 mqd->cp_hqd_active);
2105 2112
2106 if (ring->use_doorbell) 2113 if (ring->use_doorbell)
@@ -2323,7 +2330,7 @@ static bool gfx_v9_0_is_idle(void *handle)
2323{ 2330{
2324 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2331 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2325 2332
2326 if (REG_GET_FIELD(RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS)), 2333 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
2327 GRBM_STATUS, GUI_ACTIVE)) 2334 GRBM_STATUS, GUI_ACTIVE))
2328 return false; 2335 return false;
2329 else 2336 else
@@ -2338,7 +2345,7 @@ static int gfx_v9_0_wait_for_idle(void *handle)
2338 2345
2339 for (i = 0; i < adev->usec_timeout; i++) { 2346 for (i = 0; i < adev->usec_timeout; i++) {
2340 /* read MC_STATUS */ 2347 /* read MC_STATUS */
2341 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS)) & 2348 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS) &
2342 GRBM_STATUS__GUI_ACTIVE_MASK; 2349 GRBM_STATUS__GUI_ACTIVE_MASK;
2343 2350
2344 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE)) 2351 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
@@ -2355,7 +2362,7 @@ static int gfx_v9_0_soft_reset(void *handle)
2355 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2362 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2356 2363
2357 /* GRBM_STATUS */ 2364 /* GRBM_STATUS */
2358 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS)); 2365 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
2359 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 2366 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
2360 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 2367 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
2361 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 2368 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
@@ -2374,7 +2381,7 @@ static int gfx_v9_0_soft_reset(void *handle)
2374 } 2381 }
2375 2382
2376 /* GRBM_STATUS2 */ 2383 /* GRBM_STATUS2 */
2377 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS2)); 2384 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
2378 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 2385 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
2379 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 2386 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2380 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 2387 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
@@ -2391,17 +2398,17 @@ static int gfx_v9_0_soft_reset(void *handle)
2391 gfx_v9_0_cp_compute_enable(adev, false); 2398 gfx_v9_0_cp_compute_enable(adev, false);
2392 2399
2393 if (grbm_soft_reset) { 2400 if (grbm_soft_reset) {
2394 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET)); 2401 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
2395 tmp |= grbm_soft_reset; 2402 tmp |= grbm_soft_reset;
2396 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 2403 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
2397 WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET), tmp); 2404 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
2398 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET)); 2405 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
2399 2406
2400 udelay(50); 2407 udelay(50);
2401 2408
2402 tmp &= ~grbm_soft_reset; 2409 tmp &= ~grbm_soft_reset;
2403 WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET), tmp); 2410 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
2404 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET)); 2411 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
2405 } 2412 }
2406 2413
2407 /* Wait a little for things to settle down */ 2414 /* Wait a little for things to settle down */
@@ -2415,9 +2422,9 @@ static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
2415 uint64_t clock; 2422 uint64_t clock;
2416 2423
2417 mutex_lock(&adev->gfx.gpu_clock_mutex); 2424 mutex_lock(&adev->gfx.gpu_clock_mutex);
2418 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT), 1); 2425 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
2419 clock = (uint64_t)RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB)) | 2426 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
2420 ((uint64_t)RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB)) << 32ULL); 2427 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
2421 mutex_unlock(&adev->gfx.gpu_clock_mutex); 2428 mutex_unlock(&adev->gfx.gpu_clock_mutex);
2422 return clock; 2429 return clock;
2423} 2430}
@@ -2497,7 +2504,7 @@ static void gfx_v9_0_enter_rlc_safe_mode(struct amdgpu_device *adev)
2497 return; 2504 return;
2498 2505
2499 /* if RLC is not enabled, do nothing */ 2506 /* if RLC is not enabled, do nothing */
2500 rlc_setting = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL)); 2507 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
2501 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 2508 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
2502 return; 2509 return;
2503 2510
@@ -2506,7 +2513,7 @@ static void gfx_v9_0_enter_rlc_safe_mode(struct amdgpu_device *adev)
2506 AMD_CG_SUPPORT_GFX_3D_CGCG)) { 2513 AMD_CG_SUPPORT_GFX_3D_CGCG)) {
2507 data = RLC_SAFE_MODE__CMD_MASK; 2514 data = RLC_SAFE_MODE__CMD_MASK;
2508 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 2515 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
2509 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SAFE_MODE), data); 2516 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
2510 2517
2511 /* wait for RLC_SAFE_MODE */ 2518 /* wait for RLC_SAFE_MODE */
2512 for (i = 0; i < adev->usec_timeout; i++) { 2519 for (i = 0; i < adev->usec_timeout; i++) {
@@ -2526,7 +2533,7 @@ static void gfx_v9_0_exit_rlc_safe_mode(struct amdgpu_device *adev)
2526 return; 2533 return;
2527 2534
2528 /* if RLC is not enabled, do nothing */ 2535 /* if RLC is not enabled, do nothing */
2529 rlc_setting = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL)); 2536 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
2530 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 2537 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
2531 return; 2538 return;
2532 2539
@@ -2537,7 +2544,7 @@ static void gfx_v9_0_exit_rlc_safe_mode(struct amdgpu_device *adev)
2537 * mode. 2544 * mode.
2538 */ 2545 */
2539 data = RLC_SAFE_MODE__CMD_MASK; 2546 data = RLC_SAFE_MODE__CMD_MASK;
2540 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SAFE_MODE), data); 2547 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
2541 adev->gfx.rlc.in_safe_mode = false; 2548 adev->gfx.rlc.in_safe_mode = false;
2542 } 2549 }
2543} 2550}
@@ -2550,7 +2557,7 @@ static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev
2550 /* It is disabled by HW by default */ 2557 /* It is disabled by HW by default */
2551 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 2558 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
2552 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 2559 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
2553 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 2560 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
2554 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK | 2561 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK |
2555 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 2562 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
2556 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 2563 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
@@ -2560,48 +2567,48 @@ static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev
2560 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 2567 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
2561 2568
2562 if (def != data) 2569 if (def != data)
2563 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE), data); 2570 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
2564 2571
2565 /* MGLS is a global flag to control all MGLS in GFX */ 2572 /* MGLS is a global flag to control all MGLS in GFX */
2566 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 2573 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
2567 /* 2 - RLC memory Light sleep */ 2574 /* 2 - RLC memory Light sleep */
2568 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 2575 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
2569 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 2576 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
2570 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 2577 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
2571 if (def != data) 2578 if (def != data)
2572 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL), data); 2579 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
2573 } 2580 }
2574 /* 3 - CP memory Light sleep */ 2581 /* 3 - CP memory Light sleep */
2575 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 2582 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
2576 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 2583 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
2577 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 2584 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
2578 if (def != data) 2585 if (def != data)
2579 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL), data); 2586 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
2580 } 2587 }
2581 } 2588 }
2582 } else { 2589 } else {
2583 /* 1 - MGCG_OVERRIDE */ 2590 /* 1 - MGCG_OVERRIDE */
2584 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 2591 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
2585 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK | 2592 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK |
2586 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 2593 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
2587 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 2594 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
2588 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 2595 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
2589 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 2596 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
2590 if (def != data) 2597 if (def != data)
2591 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE), data); 2598 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
2592 2599
2593 /* 2 - disable MGLS in RLC */ 2600 /* 2 - disable MGLS in RLC */
2594 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 2601 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
2595 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 2602 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
2596 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 2603 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
2597 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL), data); 2604 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
2598 } 2605 }
2599 2606
2600 /* 3 - disable MGLS in CP */ 2607 /* 3 - disable MGLS in CP */
2601 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 2608 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
2602 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 2609 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
2603 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 2610 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
2604 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL), data); 2611 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
2605 } 2612 }
2606 } 2613 }
2607} 2614}
@@ -2616,37 +2623,37 @@ static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
2616 /* Enable 3D CGCG/CGLS */ 2623 /* Enable 3D CGCG/CGLS */
2617 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) { 2624 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
2618 /* write cmd to clear cgcg/cgls ov */ 2625 /* write cmd to clear cgcg/cgls ov */
2619 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 2626 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
2620 /* unset CGCG override */ 2627 /* unset CGCG override */
2621 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 2628 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
2622 /* update CGCG and CGLS override bits */ 2629 /* update CGCG and CGLS override bits */
2623 if (def != data) 2630 if (def != data)
2624 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE), data); 2631 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
2625 /* enable 3Dcgcg FSM(0x0020003f) */ 2632 /* enable 3Dcgcg FSM(0x0020003f) */
2626 def = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 2633 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
2627 data = (0x2000 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 2634 data = (0x2000 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
2628 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 2635 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
2629 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 2636 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
2630 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 2637 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
2631 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 2638 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
2632 if (def != data) 2639 if (def != data)
2633 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D), data); 2640 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
2634 2641
2635 /* set IDLE_POLL_COUNT(0x00900100) */ 2642 /* set IDLE_POLL_COUNT(0x00900100) */
2636 def = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2643 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
2637 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 2644 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
2638 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2645 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2639 if (def != data) 2646 if (def != data)
2640 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2647 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
2641 } else { 2648 } else {
2642 /* Disable CGCG/CGLS */ 2649 /* Disable CGCG/CGLS */
2643 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 2650 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
2644 /* disable cgcg, cgls should be disabled */ 2651 /* disable cgcg, cgls should be disabled */
2645 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 2652 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
2646 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 2653 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
2647 /* disable cgcg and cgls in FSM */ 2654 /* disable cgcg and cgls in FSM */
2648 if (def != data) 2655 if (def != data)
2649 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D), data); 2656 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
2650 } 2657 }
2651 2658
2652 adev->gfx.rlc.funcs->exit_safe_mode(adev); 2659 adev->gfx.rlc.funcs->exit_safe_mode(adev);
@@ -2660,7 +2667,7 @@ static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev
2660 adev->gfx.rlc.funcs->enter_safe_mode(adev); 2667 adev->gfx.rlc.funcs->enter_safe_mode(adev);
2661 2668
2662 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 2669 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
2663 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 2670 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
2664 /* unset CGCG override */ 2671 /* unset CGCG override */
2665 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 2672 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
2666 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 2673 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
@@ -2669,31 +2676,31 @@ static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev
2669 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 2676 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
2670 /* update CGCG and CGLS override bits */ 2677 /* update CGCG and CGLS override bits */
2671 if (def != data) 2678 if (def != data)
2672 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE), data); 2679 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
2673 2680
2674 /* enable cgcg FSM(0x0020003F) */ 2681 /* enable cgcg FSM(0x0020003F) */
2675 def = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 2682 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
2676 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 2683 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
2677 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 2684 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
2678 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 2685 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
2679 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 2686 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
2680 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 2687 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
2681 if (def != data) 2688 if (def != data)
2682 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL), data); 2689 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
2683 2690
2684 /* set IDLE_POLL_COUNT(0x00900100) */ 2691 /* set IDLE_POLL_COUNT(0x00900100) */
2685 def = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2692 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
2686 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 2693 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
2687 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2694 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2688 if (def != data) 2695 if (def != data)
2689 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2696 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
2690 } else { 2697 } else {
2691 def = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 2698 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
2692 /* reset CGCG/CGLS bits */ 2699 /* reset CGCG/CGLS bits */
2693 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 2700 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
2694 /* disable cgcg and cgls in FSM */ 2701 /* disable cgcg and cgls in FSM */
2695 if (def != data) 2702 if (def != data)
2696 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL), data); 2703 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
2697 } 2704 }
2698 2705
2699 adev->gfx.rlc.funcs->exit_safe_mode(adev); 2706 adev->gfx.rlc.funcs->exit_safe_mode(adev);
@@ -2740,6 +2747,9 @@ static int gfx_v9_0_set_clockgating_state(void *handle,
2740{ 2747{
2741 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2748 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2742 2749
2750 if (amdgpu_sriov_vf(adev))
2751 return 0;
2752
2743 switch (adev->asic_type) { 2753 switch (adev->asic_type) {
2744 case CHIP_VEGA10: 2754 case CHIP_VEGA10:
2745 gfx_v9_0_update_gfx_clock_gating(adev, 2755 gfx_v9_0_update_gfx_clock_gating(adev,
@@ -2760,12 +2770,12 @@ static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
2760 *flags = 0; 2770 *flags = 0;
2761 2771
2762 /* AMD_CG_SUPPORT_GFX_MGCG */ 2772 /* AMD_CG_SUPPORT_GFX_MGCG */
2763 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 2773 data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
2764 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 2774 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
2765 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 2775 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
2766 2776
2767 /* AMD_CG_SUPPORT_GFX_CGCG */ 2777 /* AMD_CG_SUPPORT_GFX_CGCG */
2768 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 2778 data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
2769 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 2779 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
2770 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 2780 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
2771 2781
@@ -2774,17 +2784,17 @@ static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
2774 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 2784 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
2775 2785
2776 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 2786 /* AMD_CG_SUPPORT_GFX_RLC_LS */
2777 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 2787 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
2778 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 2788 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
2779 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 2789 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
2780 2790
2781 /* AMD_CG_SUPPORT_GFX_CP_LS */ 2791 /* AMD_CG_SUPPORT_GFX_CP_LS */
2782 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 2792 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
2783 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 2793 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
2784 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 2794 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
2785 2795
2786 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 2796 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
2787 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 2797 data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
2788 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 2798 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
2789 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 2799 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
2790 2800
@@ -2807,8 +2817,8 @@ static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
2807 if (ring->use_doorbell) { 2817 if (ring->use_doorbell) {
2808 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 2818 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
2809 } else { 2819 } else {
2810 wptr = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR)); 2820 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
2811 wptr += (u64)RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR_HI)) << 32; 2821 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
2812 } 2822 }
2813 2823
2814 return wptr; 2824 return wptr;
@@ -2823,8 +2833,8 @@ static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
2823 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr); 2833 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr);
2824 WDOORBELL64(ring->doorbell_index, ring->wptr); 2834 WDOORBELL64(ring->doorbell_index, ring->wptr);
2825 } else { 2835 } else {
2826 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR), lower_32_bits(ring->wptr)); 2836 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
2827 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR_HI), upper_32_bits(ring->wptr)); 2837 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
2828 } 2838 }
2829} 2839}
2830 2840
@@ -2956,35 +2966,29 @@ static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
2956static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 2966static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
2957 unsigned vm_id, uint64_t pd_addr) 2967 unsigned vm_id, uint64_t pd_addr)
2958{ 2968{
2969 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
2959 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 2970 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
2960 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id); 2971 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
2961 unsigned eng = ring->idx; 2972 unsigned eng = ring->vm_inv_eng;
2962 unsigned i;
2963 2973
2964 pd_addr = pd_addr | 0x1; /* valid bit */ 2974 pd_addr = pd_addr | 0x1; /* valid bit */
2965 /* now only use physical base address of PDE and valid */ 2975 /* now only use physical base address of PDE and valid */
2966 BUG_ON(pd_addr & 0xFFFF00000000003EULL); 2976 BUG_ON(pd_addr & 0xFFFF00000000003EULL);
2967 2977
2968 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 2978 gfx_v9_0_write_data_to_reg(ring, usepfp, true,
2969 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i]; 2979 hub->ctx0_ptb_addr_lo32 + (2 * vm_id),
2970 2980 lower_32_bits(pd_addr));
2971 gfx_v9_0_write_data_to_reg(ring, usepfp, true,
2972 hub->ctx0_ptb_addr_lo32
2973 + (2 * vm_id),
2974 lower_32_bits(pd_addr));
2975 2981
2976 gfx_v9_0_write_data_to_reg(ring, usepfp, true, 2982 gfx_v9_0_write_data_to_reg(ring, usepfp, true,
2977 hub->ctx0_ptb_addr_hi32 2983 hub->ctx0_ptb_addr_hi32 + (2 * vm_id),
2978 + (2 * vm_id), 2984 upper_32_bits(pd_addr));
2979 upper_32_bits(pd_addr));
2980 2985
2981 gfx_v9_0_write_data_to_reg(ring, usepfp, true, 2986 gfx_v9_0_write_data_to_reg(ring, usepfp, true,
2982 hub->vm_inv_eng0_req + eng, req); 2987 hub->vm_inv_eng0_req + eng, req);
2983 2988
2984 /* wait for the invalidate to complete */ 2989 /* wait for the invalidate to complete */
2985 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, hub->vm_inv_eng0_ack + 2990 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, hub->vm_inv_eng0_ack +
2986 eng, 0, 1 << vm_id, 1 << vm_id, 0x20); 2991 eng, 0, 1 << vm_id, 1 << vm_id, 0x20);
2987 }
2988 2992
2989 /* compute doesn't have PFP */ 2993 /* compute doesn't have PFP */
2990 if (usepfp) { 2994 if (usepfp) {
@@ -3373,9 +3377,7 @@ static int gfx_v9_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
3373 enum amdgpu_interrupt_state state) 3377 enum amdgpu_interrupt_state state)
3374{ 3378{
3375 uint32_t tmp, target; 3379 uint32_t tmp, target;
3376 struct amdgpu_ring *ring = (struct amdgpu_ring *)src->data; 3380 struct amdgpu_ring *ring = &(adev->gfx.kiq.ring);
3377
3378 BUG_ON(!ring || (ring->funcs->type != AMDGPU_RING_TYPE_KIQ));
3379 3381
3380 if (ring->me == 1) 3382 if (ring->me == 1)
3381 target = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 3383 target = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
@@ -3386,20 +3388,20 @@ static int gfx_v9_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
3386 switch (type) { 3388 switch (type) {
3387 case AMDGPU_CP_KIQ_IRQ_DRIVER0: 3389 case AMDGPU_CP_KIQ_IRQ_DRIVER0:
3388 if (state == AMDGPU_IRQ_STATE_DISABLE) { 3390 if (state == AMDGPU_IRQ_STATE_DISABLE) {
3389 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL)); 3391 tmp = RREG32_SOC15(GC, 0, mmCPC_INT_CNTL);
3390 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 3392 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
3391 GENERIC2_INT_ENABLE, 0); 3393 GENERIC2_INT_ENABLE, 0);
3392 WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL), tmp); 3394 WREG32_SOC15(GC, 0, mmCPC_INT_CNTL, tmp);
3393 3395
3394 tmp = RREG32(target); 3396 tmp = RREG32(target);
3395 tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL, 3397 tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
3396 GENERIC2_INT_ENABLE, 0); 3398 GENERIC2_INT_ENABLE, 0);
3397 WREG32(target, tmp); 3399 WREG32(target, tmp);
3398 } else { 3400 } else {
3399 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL)); 3401 tmp = RREG32_SOC15(GC, 0, mmCPC_INT_CNTL);
3400 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 3402 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
3401 GENERIC2_INT_ENABLE, 1); 3403 GENERIC2_INT_ENABLE, 1);
3402 WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL), tmp); 3404 WREG32_SOC15(GC, 0, mmCPC_INT_CNTL, tmp);
3403 3405
3404 tmp = RREG32(target); 3406 tmp = RREG32(target);
3405 tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL, 3407 tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
@@ -3419,9 +3421,7 @@ static int gfx_v9_0_kiq_irq(struct amdgpu_device *adev,
3419 struct amdgpu_iv_entry *entry) 3421 struct amdgpu_iv_entry *entry)
3420{ 3422{
3421 u8 me_id, pipe_id, queue_id; 3423 u8 me_id, pipe_id, queue_id;
3422 struct amdgpu_ring *ring = (struct amdgpu_ring *)source->data; 3424 struct amdgpu_ring *ring = &(adev->gfx.kiq.ring);
3423
3424 BUG_ON(!ring || (ring->funcs->type != AMDGPU_RING_TYPE_KIQ));
3425 3425
3426 me_id = (entry->ring_id & 0x0c) >> 2; 3426 me_id = (entry->ring_id & 0x0c) >> 2;
3427 pipe_id = (entry->ring_id & 0x03) >> 0; 3427 pipe_id = (entry->ring_id & 0x03) >> 0;
@@ -3456,13 +3456,14 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
3456 .align_mask = 0xff, 3456 .align_mask = 0xff,
3457 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 3457 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
3458 .support_64bit_ptrs = true, 3458 .support_64bit_ptrs = true,
3459 .vmhub = AMDGPU_GFXHUB,
3459 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 3460 .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
3460 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 3461 .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
3461 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 3462 .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
3462 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 3463 .emit_frame_size = /* totally 242 maximum if 16 IBs */
3463 5 + /* COND_EXEC */ 3464 5 + /* COND_EXEC */
3464 7 + /* PIPELINE_SYNC */ 3465 7 + /* PIPELINE_SYNC */
3465 46 + /* VM_FLUSH */ 3466 24 + /* VM_FLUSH */
3466 8 + /* FENCE for VM_FLUSH */ 3467 8 + /* FENCE for VM_FLUSH */
3467 20 + /* GDS switch */ 3468 20 + /* GDS switch */
3468 4 + /* double SWITCH_BUFFER, 3469 4 + /* double SWITCH_BUFFER,
@@ -3500,6 +3501,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
3500 .align_mask = 0xff, 3501 .align_mask = 0xff,
3501 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 3502 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
3502 .support_64bit_ptrs = true, 3503 .support_64bit_ptrs = true,
3504 .vmhub = AMDGPU_GFXHUB,
3503 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 3505 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
3504 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 3506 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
3505 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 3507 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
@@ -3508,7 +3510,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
3508 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 3510 7 + /* gfx_v9_0_ring_emit_hdp_flush */
3509 5 + /* gfx_v9_0_ring_emit_hdp_invalidate */ 3511 5 + /* gfx_v9_0_ring_emit_hdp_invalidate */
3510 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 3512 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
3511 64 + /* gfx_v9_0_ring_emit_vm_flush */ 3513 24 + /* gfx_v9_0_ring_emit_vm_flush */
3512 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 3514 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
3513 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */ 3515 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */
3514 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 3516 .emit_ib = gfx_v9_0_ring_emit_ib_compute,
@@ -3529,6 +3531,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
3529 .align_mask = 0xff, 3531 .align_mask = 0xff,
3530 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 3532 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
3531 .support_64bit_ptrs = true, 3533 .support_64bit_ptrs = true,
3534 .vmhub = AMDGPU_GFXHUB,
3532 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 3535 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
3533 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 3536 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
3534 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 3537 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
@@ -3537,7 +3540,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
3537 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 3540 7 + /* gfx_v9_0_ring_emit_hdp_flush */
3538 5 + /* gfx_v9_0_ring_emit_hdp_invalidate */ 3541 5 + /* gfx_v9_0_ring_emit_hdp_invalidate */
3539 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 3542 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
3540 64 + /* gfx_v9_0_ring_emit_vm_flush */ 3543 24 + /* gfx_v9_0_ring_emit_vm_flush */
3541 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 3544 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
3542 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */ 3545 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */
3543 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 3546 .emit_ib = gfx_v9_0_ring_emit_ib_compute,
@@ -3612,7 +3615,7 @@ static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
3612static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 3615static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
3613{ 3616{
3614 /* init asci gds info */ 3617 /* init asci gds info */
3615 adev->gds.mem.total_size = RREG32(SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE)); 3618 adev->gds.mem.total_size = RREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE);
3616 adev->gds.gws.total_size = 64; 3619 adev->gds.gws.total_size = 64;
3617 adev->gds.oa.total_size = 16; 3620 adev->gds.oa.total_size = 16;
3618 3621
@@ -3641,8 +3644,8 @@ static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
3641{ 3644{
3642 u32 data, mask; 3645 u32 data, mask;
3643 3646
3644 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG)); 3647 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
3645 data |= RREG32(SOC15_REG_OFFSET(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG)); 3648 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
3646 3649
3647 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 3650 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
3648 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 3651 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
@@ -3763,25 +3766,25 @@ static int gfx_v9_0_init_queue(struct amdgpu_ring *ring)
3763 eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + (ring->queue * MEC_HPD_SIZE); 3766 eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + (ring->queue * MEC_HPD_SIZE);
3764 eop_gpu_addr >>= 8; 3767 eop_gpu_addr >>= 8;
3765 3768
3766 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_BASE_ADDR), lower_32_bits(eop_gpu_addr)); 3769 WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR, lower_32_bits(eop_gpu_addr));
3767 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI), upper_32_bits(eop_gpu_addr)); 3770 WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr));
3768 mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_gpu_addr); 3771 mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_gpu_addr);
3769 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_gpu_addr); 3772 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_gpu_addr);
3770 3773
3771 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3774 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3772 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_CONTROL)); 3775 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3773 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3776 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3774 (order_base_2(MEC_HPD_SIZE / 4) - 1)); 3777 (order_base_2(MEC_HPD_SIZE / 4) - 1));
3775 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_CONTROL), tmp); 3778 WREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL, tmp);
3776 3779
3777 /* enable doorbell? */ 3780 /* enable doorbell? */
3778 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL)); 3781 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3779 if (use_doorbell) 3782 if (use_doorbell)
3780 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); 3783 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
3781 else 3784 else
3782 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 0); 3785 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 0);
3783 3786
3784 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), tmp); 3787 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, tmp);
3785 mqd->cp_hqd_pq_doorbell_control = tmp; 3788 mqd->cp_hqd_pq_doorbell_control = tmp;
3786 3789
3787 /* disable the queue if it's active */ 3790 /* disable the queue if it's active */
@@ -3790,40 +3793,40 @@ static int gfx_v9_0_init_queue(struct amdgpu_ring *ring)
3790 mqd->cp_hqd_pq_rptr = 0; 3793 mqd->cp_hqd_pq_rptr = 0;
3791 mqd->cp_hqd_pq_wptr_lo = 0; 3794 mqd->cp_hqd_pq_wptr_lo = 0;
3792 mqd->cp_hqd_pq_wptr_hi = 0; 3795 mqd->cp_hqd_pq_wptr_hi = 0;
3793 if (RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)) & 1) { 3796 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3794 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), 1); 3797 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3795 for (j = 0; j < adev->usec_timeout; j++) { 3798 for (j = 0; j < adev->usec_timeout; j++) {
3796 if (!(RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)) & 1)) 3799 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3797 break; 3800 break;
3798 udelay(1); 3801 udelay(1);
3799 } 3802 }
3800 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), mqd->cp_hqd_dequeue_request); 3803 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, mqd->cp_hqd_dequeue_request);
3801 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR), mqd->cp_hqd_pq_rptr); 3804 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR, mqd->cp_hqd_pq_rptr);
3802 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO), mqd->cp_hqd_pq_wptr_lo); 3805 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO, mqd->cp_hqd_pq_wptr_lo);
3803 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI), mqd->cp_hqd_pq_wptr_hi); 3806 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI, mqd->cp_hqd_pq_wptr_hi);
3804 } 3807 }
3805 3808
3806 /* set the pointer to the MQD */ 3809 /* set the pointer to the MQD */
3807 mqd->cp_mqd_base_addr_lo = mqd_gpu_addr & 0xfffffffc; 3810 mqd->cp_mqd_base_addr_lo = mqd_gpu_addr & 0xfffffffc;
3808 mqd->cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr); 3811 mqd->cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
3809 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR), mqd->cp_mqd_base_addr_lo); 3812 WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
3810 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR_HI), mqd->cp_mqd_base_addr_hi); 3813 WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
3811 3814
3812 /* set MQD vmid to 0 */ 3815 /* set MQD vmid to 0 */
3813 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_CONTROL)); 3816 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3814 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3817 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3815 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MQD_CONTROL), tmp); 3818 WREG32_SOC15(GC, 0, mmCP_MQD_CONTROL, tmp);
3816 mqd->cp_mqd_control = tmp; 3819 mqd->cp_mqd_control = tmp;
3817 3820
3818 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3821 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3819 hqd_gpu_addr = ring->gpu_addr >> 8; 3822 hqd_gpu_addr = ring->gpu_addr >> 8;
3820 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3823 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3821 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3824 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3822 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE), mqd->cp_hqd_pq_base_lo); 3825 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
3823 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI), mqd->cp_hqd_pq_base_hi); 3826 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
3824 3827
3825 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3828 /* set up the HQD, this is similar to CP_RB0_CNTL */
3826 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_CONTROL)); 3829 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3827 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3830 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3828 (order_base_2(ring->ring_size / 4) - 1)); 3831 (order_base_2(ring->ring_size / 4) - 1));
3829 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3832 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
@@ -3835,7 +3838,7 @@ static int gfx_v9_0_init_queue(struct amdgpu_ring *ring)
3835 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3838 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3836 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3839 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3837 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3840 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3838 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_CONTROL), tmp); 3841 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL, tmp);
3839 mqd->cp_hqd_pq_control = tmp; 3842 mqd->cp_hqd_pq_control = tmp;
3840 3843
3841 /* set the wb address wether it's enabled or not */ 3844 /* set the wb address wether it's enabled or not */
@@ -3843,27 +3846,27 @@ static int gfx_v9_0_init_queue(struct amdgpu_ring *ring)
3843 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3846 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3844 mqd->cp_hqd_pq_rptr_report_addr_hi = 3847 mqd->cp_hqd_pq_rptr_report_addr_hi =
3845 upper_32_bits(wb_gpu_addr) & 0xffff; 3848 upper_32_bits(wb_gpu_addr) & 0xffff;
3846 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR), 3849 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3847 mqd->cp_hqd_pq_rptr_report_addr_lo); 3850 mqd->cp_hqd_pq_rptr_report_addr_lo);
3848 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI), 3851 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3849 mqd->cp_hqd_pq_rptr_report_addr_hi); 3852 mqd->cp_hqd_pq_rptr_report_addr_hi);
3850 3853
3851 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3854 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3852 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3855 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3853 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3856 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3854 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3857 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3855 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR), 3858 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3856 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3859 mqd->cp_hqd_pq_wptr_poll_addr_lo);
3857 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI), 3860 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3858 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3861 mqd->cp_hqd_pq_wptr_poll_addr_hi);
3859 3862
3860 /* enable the doorbell if requested */ 3863 /* enable the doorbell if requested */
3861 if (use_doorbell) { 3864 if (use_doorbell) {
3862 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER), 3865 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3863 (AMDGPU_DOORBELL64_KIQ * 2) << 2); 3866 (AMDGPU_DOORBELL64_KIQ * 2) << 2);
3864 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER), 3867 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3865 (AMDGPU_DOORBELL64_MEC_RING7 * 2) << 2); 3868 (AMDGPU_DOORBELL64_MEC_RING7 * 2) << 2);
3866 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL)); 3869 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3867 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3870 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3868 DOORBELL_OFFSET, ring->doorbell_index); 3871 DOORBELL_OFFSET, ring->doorbell_index);
3869 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); 3872 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
@@ -3874,25 +3877,25 @@ static int gfx_v9_0_init_queue(struct amdgpu_ring *ring)
3874 } else { 3877 } else {
3875 mqd->cp_hqd_pq_doorbell_control = 0; 3878 mqd->cp_hqd_pq_doorbell_control = 0;
3876 } 3879 }
3877 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), 3880 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3878 mqd->cp_hqd_pq_doorbell_control); 3881 mqd->cp_hqd_pq_doorbell_control);
3879 3882
3880 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3883 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3881 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO), mqd->cp_hqd_pq_wptr_lo); 3884 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO, mqd->cp_hqd_pq_wptr_lo);
3882 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI), mqd->cp_hqd_pq_wptr_hi); 3885 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI, mqd->cp_hqd_pq_wptr_hi);
3883 3886
3884 /* set the vmid for the queue */ 3887 /* set the vmid for the queue */
3885 mqd->cp_hqd_vmid = 0; 3888 mqd->cp_hqd_vmid = 0;
3886 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_VMID), mqd->cp_hqd_vmid); 3889 WREG32_SOC15(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3887 3890
3888 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PERSISTENT_STATE)); 3891 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3889 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3892 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3890 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PERSISTENT_STATE), tmp); 3893 WREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE, tmp);
3891 mqd->cp_hqd_persistent_state = tmp; 3894 mqd->cp_hqd_persistent_state = tmp;
3892 3895
3893 /* activate the queue */ 3896 /* activate the queue */
3894 mqd->cp_hqd_active = 1; 3897 mqd->cp_hqd_active = 1;
3895 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), mqd->cp_hqd_active); 3898 WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, mqd->cp_hqd_active);
3896 3899
3897 soc15_grbm_select(adev, 0, 0, 0, 0); 3900 soc15_grbm_select(adev, 0, 0, 0, 0);
3898 mutex_unlock(&adev->srbm_mutex); 3901 mutex_unlock(&adev->srbm_mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index 631aef38126d..a572979f186c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -346,7 +346,8 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
346 * size equal to the 1024 or vram, whichever is larger. 346 * size equal to the 1024 or vram, whichever is larger.
347 */ 347 */
348 if (amdgpu_gart_size == -1) 348 if (amdgpu_gart_size == -1)
349 adev->mc.gtt_size = max((1024ULL << 20), adev->mc.mc_vram_size); 349 adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
350 adev->mc.mc_vram_size);
350 else 351 else
351 adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20; 352 adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
352 353
@@ -621,7 +622,7 @@ static int gmc_v6_0_vm_init(struct amdgpu_device *adev)
621 * amdgpu graphics/compute will use VMIDs 1-7 622 * amdgpu graphics/compute will use VMIDs 1-7
622 * amdkfd will use VMIDs 8-15 623 * amdkfd will use VMIDs 8-15
623 */ 624 */
624 adev->vm_manager.num_ids = AMDGPU_NUM_OF_VMIDS; 625 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
625 adev->vm_manager.num_level = 1; 626 adev->vm_manager.num_level = 1;
626 amdgpu_vm_manager_init(adev); 627 amdgpu_vm_manager_init(adev);
627 628
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 92abe12d92bb..a9083a16a250 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -395,7 +395,8 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
395 * size equal to the 1024 or vram, whichever is larger. 395 * size equal to the 1024 or vram, whichever is larger.
396 */ 396 */
397 if (amdgpu_gart_size == -1) 397 if (amdgpu_gart_size == -1)
398 adev->mc.gtt_size = max((1024ULL << 20), adev->mc.mc_vram_size); 398 adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
399 adev->mc.mc_vram_size);
399 else 400 else
400 adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20; 401 adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
401 402
@@ -746,7 +747,7 @@ static int gmc_v7_0_vm_init(struct amdgpu_device *adev)
746 * amdgpu graphics/compute will use VMIDs 1-7 747 * amdgpu graphics/compute will use VMIDs 1-7
747 * amdkfd will use VMIDs 8-15 748 * amdkfd will use VMIDs 8-15
748 */ 749 */
749 adev->vm_manager.num_ids = AMDGPU_NUM_OF_VMIDS; 750 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
750 adev->vm_manager.num_level = 1; 751 adev->vm_manager.num_level = 1;
751 amdgpu_vm_manager_init(adev); 752 amdgpu_vm_manager_init(adev);
752 753
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index f2ccefc66fd4..4ac99784160a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -557,7 +557,8 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
557 * size equal to the 1024 or vram, whichever is larger. 557 * size equal to the 1024 or vram, whichever is larger.
558 */ 558 */
559 if (amdgpu_gart_size == -1) 559 if (amdgpu_gart_size == -1)
560 adev->mc.gtt_size = max((1024ULL << 20), adev->mc.mc_vram_size); 560 adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
561 adev->mc.mc_vram_size);
561 else 562 else
562 adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20; 563 adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
563 564
@@ -949,7 +950,7 @@ static int gmc_v8_0_vm_init(struct amdgpu_device *adev)
949 * amdgpu graphics/compute will use VMIDs 1-7 950 * amdgpu graphics/compute will use VMIDs 1-7
950 * amdkfd will use VMIDs 8-15 951 * amdkfd will use VMIDs 8-15
951 */ 952 */
952 adev->vm_manager.num_ids = AMDGPU_NUM_OF_VMIDS; 953 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
953 adev->vm_manager.num_level = 1; 954 adev->vm_manager.num_level = 1;
954 amdgpu_vm_manager_init(adev); 955 amdgpu_vm_manager_init(adev);
955 956
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 3b045e0b114e..dc1e1c1d6b24 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -386,6 +386,23 @@ static int gmc_v9_0_early_init(void *handle)
386static int gmc_v9_0_late_init(void *handle) 386static int gmc_v9_0_late_init(void *handle)
387{ 387{
388 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 388 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
389 unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 3, 3 };
390 unsigned i;
391
392 for(i = 0; i < adev->num_rings; ++i) {
393 struct amdgpu_ring *ring = adev->rings[i];
394 unsigned vmhub = ring->funcs->vmhub;
395
396 ring->vm_inv_eng = vm_inv_eng[vmhub]++;
397 dev_info(adev->dev, "ring %u(%s) uses VM inv eng %u on hub %u\n",
398 ring->idx, ring->name, ring->vm_inv_eng,
399 ring->funcs->vmhub);
400 }
401
402 /* Engine 17 is used for GART flushes */
403 for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i)
404 BUG_ON(vm_inv_eng[i] > 17);
405
389 return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0); 406 return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
390} 407}
391 408
@@ -469,7 +486,8 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
469 * size equal to the 1024 or vram, whichever is larger. 486 * size equal to the 1024 or vram, whichever is larger.
470 */ 487 */
471 if (amdgpu_gart_size == -1) 488 if (amdgpu_gart_size == -1)
472 adev->mc.gtt_size = max((1024ULL << 20), adev->mc.mc_vram_size); 489 adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
490 adev->mc.mc_vram_size);
473 else 491 else
474 adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20; 492 adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
475 493
@@ -519,7 +537,8 @@ static int gmc_v9_0_vm_init(struct amdgpu_device *adev)
519 * amdgpu graphics/compute will use VMIDs 1-7 537 * amdgpu graphics/compute will use VMIDs 1-7
520 * amdkfd will use VMIDs 8-15 538 * amdkfd will use VMIDs 8-15
521 */ 539 */
522 adev->vm_manager.num_ids = AMDGPU_NUM_OF_VMIDS; 540 adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
541 adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
523 542
524 /* TODO: fix num_level for APU when updating vm size and block size */ 543 /* TODO: fix num_level for APU when updating vm size and block size */
525 if (adev->flags & AMD_IS_APU) 544 if (adev->flags & AMD_IS_APU)
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
index 62684510ddcd..dbfe48d1207a 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
@@ -511,6 +511,9 @@ static int mmhub_v1_0_set_clockgating_state(void *handle,
511{ 511{
512 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 512 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
513 513
514 if (amdgpu_sriov_vf(adev))
515 return 0;
516
514 switch (adev->asic_type) { 517 switch (adev->asic_type) {
515 case CHIP_VEGA10: 518 case CHIP_VEGA10:
516 mmhub_v1_0_update_medium_grain_clock_gating(adev, 519 mmhub_v1_0_update_medium_grain_clock_gating(adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/mmsch_v1_0.h b/drivers/gpu/drm/amd/amdgpu/mmsch_v1_0.h
index 5f0fc8bf16a9..8af0bddf85e4 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmsch_v1_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/mmsch_v1_0.h
@@ -84,4 +84,61 @@ struct mmsch_v1_0_cmd_indirect_write {
84 uint32_t reg_value; 84 uint32_t reg_value;
85}; 85};
86 86
87static inline void mmsch_v1_0_insert_direct_wt(struct mmsch_v1_0_cmd_direct_write *direct_wt,
88 uint32_t *init_table,
89 uint32_t reg_offset,
90 uint32_t value)
91{
92 direct_wt->cmd_header.reg_offset = reg_offset;
93 direct_wt->reg_value = value;
94 memcpy((void *)init_table, direct_wt, sizeof(struct mmsch_v1_0_cmd_direct_write));
95}
96
97static inline void mmsch_v1_0_insert_direct_rd_mod_wt(struct mmsch_v1_0_cmd_direct_read_modify_write *direct_rd_mod_wt,
98 uint32_t *init_table,
99 uint32_t reg_offset,
100 uint32_t mask, uint32_t data)
101{
102 direct_rd_mod_wt->cmd_header.reg_offset = reg_offset;
103 direct_rd_mod_wt->mask_value = mask;
104 direct_rd_mod_wt->write_data = data;
105 memcpy((void *)init_table, direct_rd_mod_wt,
106 sizeof(struct mmsch_v1_0_cmd_direct_read_modify_write));
107}
108
109static inline void mmsch_v1_0_insert_direct_poll(struct mmsch_v1_0_cmd_direct_polling *direct_poll,
110 uint32_t *init_table,
111 uint32_t reg_offset,
112 uint32_t mask, uint32_t wait)
113{
114 direct_poll->cmd_header.reg_offset = reg_offset;
115 direct_poll->mask_value = mask;
116 direct_poll->wait_value = wait;
117 memcpy((void *)init_table, direct_poll, sizeof(struct mmsch_v1_0_cmd_direct_polling));
118}
119
120#define MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(reg, mask, data) { \
121 mmsch_v1_0_insert_direct_rd_mod_wt(&direct_rd_mod_wt, \
122 init_table, (reg), \
123 (mask), (data)); \
124 init_table += sizeof(struct mmsch_v1_0_cmd_direct_read_modify_write)/4; \
125 table_size += sizeof(struct mmsch_v1_0_cmd_direct_read_modify_write)/4; \
126}
127
128#define MMSCH_V1_0_INSERT_DIRECT_WT(reg, value) { \
129 mmsch_v1_0_insert_direct_wt(&direct_wt, \
130 init_table, (reg), \
131 (value)); \
132 init_table += sizeof(struct mmsch_v1_0_cmd_direct_write)/4; \
133 table_size += sizeof(struct mmsch_v1_0_cmd_direct_write)/4; \
134}
135
136#define MMSCH_V1_0_INSERT_DIRECT_POLL(reg, mask, wait) { \
137 mmsch_v1_0_insert_direct_poll(&direct_poll, \
138 init_table, (reg), \
139 (mask), (wait)); \
140 init_table += sizeof(struct mmsch_v1_0_cmd_direct_polling)/4; \
141 table_size += sizeof(struct mmsch_v1_0_cmd_direct_polling)/4; \
142}
143
87#endif 144#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
index 70a3dd13cb02..7bdc51b02326 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
@@ -368,9 +368,12 @@ static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
368 u32 reg; 368 u32 reg;
369 u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, RCV_MSG_VALID); 369 u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, RCV_MSG_VALID);
370 370
371 reg = RREG32_NO_KIQ(mmMAILBOX_CONTROL); 371 /* workaround: host driver doesn't set VALID for CMPL now */
372 if (!(reg & mask)) 372 if (event != IDH_FLR_NOTIFICATION_CMPL) {
373 return -ENOENT; 373 reg = RREG32_NO_KIQ(mmMAILBOX_CONTROL);
374 if (!(reg & mask))
375 return -ENOENT;
376 }
374 377
375 reg = RREG32_NO_KIQ(mmMAILBOX_MSGBUF_RCV_DW0); 378 reg = RREG32_NO_KIQ(mmMAILBOX_MSGBUF_RCV_DW0);
376 if (reg != event) 379 if (reg != event)
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index c3588d1c7cb0..60a6407ba267 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -166,11 +166,8 @@ int psp_v3_1_bootloader_load_sysdrv(struct psp_context *psp)
166{ 166{
167 int ret; 167 int ret;
168 uint32_t psp_gfxdrv_command_reg = 0; 168 uint32_t psp_gfxdrv_command_reg = 0;
169 struct amdgpu_bo *psp_sysdrv;
170 void *psp_sysdrv_virt = NULL;
171 uint64_t psp_sysdrv_mem;
172 struct amdgpu_device *adev = psp->adev; 169 struct amdgpu_device *adev = psp->adev;
173 uint32_t size, sol_reg; 170 uint32_t sol_reg;
174 171
175 /* Check sOS sign of life register to confirm sys driver and sOS 172 /* Check sOS sign of life register to confirm sys driver and sOS
176 * are already been loaded. 173 * are already been loaded.
@@ -185,27 +182,14 @@ int psp_v3_1_bootloader_load_sysdrv(struct psp_context *psp)
185 if (ret) 182 if (ret)
186 return ret; 183 return ret;
187 184
188 /* 185 memset(psp->fw_pri_buf, 0, PSP_1_MEG);
189 * Create a 1 meg GART memory to store the psp sys driver
190 * binary with a 1 meg aligned address
191 */
192 size = (psp->sys_bin_size + (PSP_BOOTLOADER_1_MEG_ALIGNMENT - 1)) &
193 (~(PSP_BOOTLOADER_1_MEG_ALIGNMENT - 1));
194
195 ret = amdgpu_bo_create_kernel(adev, size, PSP_BOOTLOADER_1_MEG_ALIGNMENT,
196 AMDGPU_GEM_DOMAIN_GTT,
197 &psp_sysdrv,
198 &psp_sysdrv_mem,
199 &psp_sysdrv_virt);
200 if (ret)
201 return ret;
202 186
203 /* Copy PSP System Driver binary to memory */ 187 /* Copy PSP System Driver binary to memory */
204 memcpy(psp_sysdrv_virt, psp->sys_start_addr, psp->sys_bin_size); 188 memcpy(psp->fw_pri_buf, psp->sys_start_addr, psp->sys_bin_size);
205 189
206 /* Provide the sys driver to bootrom */ 190 /* Provide the sys driver to bootrom */
207 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_36), 191 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_36),
208 (uint32_t)(psp_sysdrv_mem >> 20)); 192 (uint32_t)(psp->fw_pri_mc_addr >> 20));
209 psp_gfxdrv_command_reg = 1 << 16; 193 psp_gfxdrv_command_reg = 1 << 16;
210 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_35), 194 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_35),
211 psp_gfxdrv_command_reg); 195 psp_gfxdrv_command_reg);
@@ -216,8 +200,6 @@ int psp_v3_1_bootloader_load_sysdrv(struct psp_context *psp)
216 ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_35), 200 ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_35),
217 0x80000000, 0x80000000, false); 201 0x80000000, 0x80000000, false);
218 202
219 amdgpu_bo_free_kernel(&psp_sysdrv, &psp_sysdrv_mem, &psp_sysdrv_virt);
220
221 return ret; 203 return ret;
222} 204}
223 205
@@ -225,11 +207,8 @@ int psp_v3_1_bootloader_load_sos(struct psp_context *psp)
225{ 207{
226 int ret; 208 int ret;
227 unsigned int psp_gfxdrv_command_reg = 0; 209 unsigned int psp_gfxdrv_command_reg = 0;
228 struct amdgpu_bo *psp_sos;
229 void *psp_sos_virt = NULL;
230 uint64_t psp_sos_mem;
231 struct amdgpu_device *adev = psp->adev; 210 struct amdgpu_device *adev = psp->adev;
232 uint32_t size, sol_reg; 211 uint32_t sol_reg;
233 212
234 /* Check sOS sign of life register to confirm sys driver and sOS 213 /* Check sOS sign of life register to confirm sys driver and sOS
235 * are already been loaded. 214 * are already been loaded.
@@ -244,23 +223,14 @@ int psp_v3_1_bootloader_load_sos(struct psp_context *psp)
244 if (ret) 223 if (ret)
245 return ret; 224 return ret;
246 225
247 size = (psp->sos_bin_size + (PSP_BOOTLOADER_1_MEG_ALIGNMENT - 1)) & 226 memset(psp->fw_pri_buf, 0, PSP_1_MEG);
248 (~((uint64_t)PSP_BOOTLOADER_1_MEG_ALIGNMENT - 1));
249
250 ret = amdgpu_bo_create_kernel(adev, size, PSP_BOOTLOADER_1_MEG_ALIGNMENT,
251 AMDGPU_GEM_DOMAIN_GTT,
252 &psp_sos,
253 &psp_sos_mem,
254 &psp_sos_virt);
255 if (ret)
256 return ret;
257 227
258 /* Copy Secure OS binary to PSP memory */ 228 /* Copy Secure OS binary to PSP memory */
259 memcpy(psp_sos_virt, psp->sos_start_addr, psp->sos_bin_size); 229 memcpy(psp->fw_pri_buf, psp->sos_start_addr, psp->sos_bin_size);
260 230
261 /* Provide the PSP secure OS to bootrom */ 231 /* Provide the PSP secure OS to bootrom */
262 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_36), 232 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_36),
263 (uint32_t)(psp_sos_mem >> 20)); 233 (uint32_t)(psp->fw_pri_mc_addr >> 20));
264 psp_gfxdrv_command_reg = 2 << 16; 234 psp_gfxdrv_command_reg = 2 << 16;
265 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_35), 235 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_35),
266 psp_gfxdrv_command_reg); 236 psp_gfxdrv_command_reg);
@@ -273,8 +243,6 @@ int psp_v3_1_bootloader_load_sos(struct psp_context *psp)
273 0, true); 243 0, true);
274#endif 244#endif
275 245
276 amdgpu_bo_free_kernel(&psp_sos, &psp_sos_mem, &psp_sos_virt);
277
278 return ret; 246 return ret;
279} 247}
280 248
@@ -300,7 +268,6 @@ int psp_v3_1_prep_cmd_buf(struct amdgpu_firmware_info *ucode, struct psp_gfx_cmd
300int psp_v3_1_ring_init(struct psp_context *psp, enum psp_ring_type ring_type) 268int psp_v3_1_ring_init(struct psp_context *psp, enum psp_ring_type ring_type)
301{ 269{
302 int ret = 0; 270 int ret = 0;
303 unsigned int psp_ring_reg = 0;
304 struct psp_ring *ring; 271 struct psp_ring *ring;
305 struct amdgpu_device *adev = psp->adev; 272 struct amdgpu_device *adev = psp->adev;
306 273
@@ -320,6 +287,16 @@ int psp_v3_1_ring_init(struct psp_context *psp, enum psp_ring_type ring_type)
320 return ret; 287 return ret;
321 } 288 }
322 289
290 return 0;
291}
292
293int psp_v3_1_ring_create(struct psp_context *psp, enum psp_ring_type ring_type)
294{
295 int ret = 0;
296 unsigned int psp_ring_reg = 0;
297 struct psp_ring *ring = &psp->km_ring;
298 struct amdgpu_device *adev = psp->adev;
299
323 /* Write low address of the ring to C2PMSG_69 */ 300 /* Write low address of the ring to C2PMSG_69 */
324 psp_ring_reg = lower_32_bits(ring->ring_mem_mc_addr); 301 psp_ring_reg = lower_32_bits(ring->ring_mem_mc_addr);
325 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_69), psp_ring_reg); 302 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_69), psp_ring_reg);
@@ -344,6 +321,33 @@ int psp_v3_1_ring_init(struct psp_context *psp, enum psp_ring_type ring_type)
344 return ret; 321 return ret;
345} 322}
346 323
324int psp_v3_1_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
325{
326 int ret = 0;
327 struct psp_ring *ring;
328 unsigned int psp_ring_reg = 0;
329 struct amdgpu_device *adev = psp->adev;
330
331 ring = &psp->km_ring;
332
333 /* Write the ring destroy command to C2PMSG_64 */
334 psp_ring_reg = 3 << 16;
335 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64), psp_ring_reg);
336
337 /* there might be handshake issue with hardware which needs delay */
338 mdelay(20);
339
340 /* Wait for response flag (bit 31) in C2PMSG_64 */
341 ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
342 0x80000000, 0x80000000, false);
343
344 if (ring->ring_mem)
345 amdgpu_bo_free_kernel(&adev->firmware.rbuf,
346 &ring->ring_mem_mc_addr,
347 (void **)&ring->ring_mem);
348 return ret;
349}
350
347int psp_v3_1_cmd_submit(struct psp_context *psp, 351int psp_v3_1_cmd_submit(struct psp_context *psp,
348 struct amdgpu_firmware_info *ucode, 352 struct amdgpu_firmware_info *ucode,
349 uint64_t cmd_buf_mc_addr, uint64_t fence_mc_addr, 353 uint64_t cmd_buf_mc_addr, uint64_t fence_mc_addr,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h
index e82eff741a08..9dcd0b25c4c6 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.h
@@ -39,6 +39,10 @@ extern int psp_v3_1_prep_cmd_buf(struct amdgpu_firmware_info *ucode,
39 struct psp_gfx_cmd_resp *cmd); 39 struct psp_gfx_cmd_resp *cmd);
40extern int psp_v3_1_ring_init(struct psp_context *psp, 40extern int psp_v3_1_ring_init(struct psp_context *psp,
41 enum psp_ring_type ring_type); 41 enum psp_ring_type ring_type);
42extern int psp_v3_1_ring_create(struct psp_context *psp,
43 enum psp_ring_type ring_type);
44extern int psp_v3_1_ring_destroy(struct psp_context *psp,
45 enum psp_ring_type ring_type);
42extern int psp_v3_1_cmd_submit(struct psp_context *psp, 46extern int psp_v3_1_cmd_submit(struct psp_context *psp,
43 struct amdgpu_firmware_info *ucode, 47 struct amdgpu_firmware_info *ucode,
44 uint64_t cmd_buf_mc_addr, uint64_t fence_mc_addr, 48 uint64_t cmd_buf_mc_addr, uint64_t fence_mc_addr,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 21f38d882335..ecc70a730a54 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -48,8 +48,7 @@ static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev);
48static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev); 48static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev);
49static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev); 49static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev);
50 50
51static const u32 golden_settings_sdma_4[] = 51static const u32 golden_settings_sdma_4[] = {
52{
53 SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831f07, 52 SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831f07,
54 SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CLK_CTRL), 0xff000ff0, 0x3f000100, 53 SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CLK_CTRL), 0xff000ff0, 0x3f000100,
55 SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_IB_CNTL), 0x800f0100, 0x00000100, 54 SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_IB_CNTL), 0x800f0100, 0x00000100,
@@ -76,8 +75,7 @@ static const u32 golden_settings_sdma_4[] =
76 SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_UTCL1_PAGE), 0x000003ff, 0x000003c0 75 SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_UTCL1_PAGE), 0x000003ff, 0x000003c0
77}; 76};
78 77
79static const u32 golden_settings_sdma_vg10[] = 78static const u32 golden_settings_sdma_vg10[] = {
80{
81 SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG), 0x0018773f, 0x00104002, 79 SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG), 0x0018773f, 0x00104002,
82 SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ), 0x0018773f, 0x00104002, 80 SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ), 0x0018773f, 0x00104002,
83 SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG), 0x0018773f, 0x00104002, 81 SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG), 0x0018773f, 0x00104002,
@@ -87,16 +85,17 @@ static const u32 golden_settings_sdma_vg10[] =
87static u32 sdma_v4_0_get_reg_offset(u32 instance, u32 internal_offset) 85static u32 sdma_v4_0_get_reg_offset(u32 instance, u32 internal_offset)
88{ 86{
89 u32 base = 0; 87 u32 base = 0;
88
90 switch (instance) { 89 switch (instance) {
91 case 0: 90 case 0:
92 base = SDMA0_BASE.instance[0].segment[0]; 91 base = SDMA0_BASE.instance[0].segment[0];
93 break; 92 break;
94 case 1: 93 case 1:
95 base = SDMA1_BASE.instance[0].segment[0]; 94 base = SDMA1_BASE.instance[0].segment[0];
96 break; 95 break;
97 default: 96 default:
98 BUG(); 97 BUG();
99 break; 98 break;
100 } 99 }
101 100
102 return base + internal_offset; 101 return base + internal_offset;
@@ -159,7 +158,8 @@ static int sdma_v4_0_init_microcode(struct amdgpu_device *adev)
159 case CHIP_VEGA10: 158 case CHIP_VEGA10:
160 chip_name = "vega10"; 159 chip_name = "vega10";
161 break; 160 break;
162 default: BUG(); 161 default:
162 BUG();
163 } 163 }
164 164
165 for (i = 0; i < adev->sdma.num_instances; i++) { 165 for (i = 0; i < adev->sdma.num_instances; i++) {
@@ -179,7 +179,7 @@ static int sdma_v4_0_init_microcode(struct amdgpu_device *adev)
179 if (adev->sdma.instance[i].feature_version >= 20) 179 if (adev->sdma.instance[i].feature_version >= 20)
180 adev->sdma.instance[i].burst_nop = true; 180 adev->sdma.instance[i].burst_nop = true;
181 DRM_DEBUG("psp_load == '%s'\n", 181 DRM_DEBUG("psp_load == '%s'\n",
182 adev->firmware.load_type == AMDGPU_FW_LOAD_PSP? "true": "false"); 182 adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
183 183
184 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 184 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
185 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i]; 185 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
@@ -192,9 +192,7 @@ static int sdma_v4_0_init_microcode(struct amdgpu_device *adev)
192 } 192 }
193out: 193out:
194 if (err) { 194 if (err) {
195 printk(KERN_ERR 195 DRM_ERROR("sdma_v4_0: Failed to load firmware \"%s\"\n", fw_name);
196 "sdma_v4_0: Failed to load firmware \"%s\"\n",
197 fw_name);
198 for (i = 0; i < adev->sdma.num_instances; i++) { 196 for (i = 0; i < adev->sdma.num_instances; i++) {
199 release_firmware(adev->sdma.instance[i].fw); 197 release_firmware(adev->sdma.instance[i].fw);
200 adev->sdma.instance[i].fw = NULL; 198 adev->sdma.instance[i].fw = NULL;
@@ -212,10 +210,10 @@ out:
212 */ 210 */
213static uint64_t sdma_v4_0_ring_get_rptr(struct amdgpu_ring *ring) 211static uint64_t sdma_v4_0_ring_get_rptr(struct amdgpu_ring *ring)
214{ 212{
215 u64* rptr; 213 u64 *rptr;
216 214
217 /* XXX check if swapping is necessary on BE */ 215 /* XXX check if swapping is necessary on BE */
218 rptr =((u64*)&ring->adev->wb.wb[ring->rptr_offs]); 216 rptr = ((u64 *)&ring->adev->wb.wb[ring->rptr_offs]);
219 217
220 DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr); 218 DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
221 return ((*rptr) >> 2); 219 return ((*rptr) >> 2);
@@ -231,19 +229,20 @@ static uint64_t sdma_v4_0_ring_get_rptr(struct amdgpu_ring *ring)
231static uint64_t sdma_v4_0_ring_get_wptr(struct amdgpu_ring *ring) 229static uint64_t sdma_v4_0_ring_get_wptr(struct amdgpu_ring *ring)
232{ 230{
233 struct amdgpu_device *adev = ring->adev; 231 struct amdgpu_device *adev = ring->adev;
234 u64* wptr = NULL; 232 u64 *wptr = NULL;
235 uint64_t local_wptr=0; 233 uint64_t local_wptr = 0;
236 234
237 if (ring->use_doorbell) { 235 if (ring->use_doorbell) {
238 /* XXX check if swapping is necessary on BE */ 236 /* XXX check if swapping is necessary on BE */
239 wptr = ((u64*)&adev->wb.wb[ring->wptr_offs]); 237 wptr = ((u64 *)&adev->wb.wb[ring->wptr_offs]);
240 DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", *wptr); 238 DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", *wptr);
241 *wptr = (*wptr) >> 2; 239 *wptr = (*wptr) >> 2;
242 DRM_DEBUG("wptr/doorbell after shift == 0x%016llx\n", *wptr); 240 DRM_DEBUG("wptr/doorbell after shift == 0x%016llx\n", *wptr);
243 } else { 241 } else {
244 u32 lowbit, highbit; 242 u32 lowbit, highbit;
245 int me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1; 243 int me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;
246 wptr=&local_wptr; 244
245 wptr = &local_wptr;
247 lowbit = RREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR)) >> 2; 246 lowbit = RREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR)) >> 2;
248 highbit = RREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR_HI)) >> 2; 247 highbit = RREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR_HI)) >> 2;
249 248
@@ -285,12 +284,13 @@ static void sdma_v4_0_ring_set_wptr(struct amdgpu_ring *ring)
285 WDOORBELL64(ring->doorbell_index, ring->wptr << 2); 284 WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
286 } else { 285 } else {
287 int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1; 286 int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
287
288 DRM_DEBUG("Not using doorbell -- " 288 DRM_DEBUG("Not using doorbell -- "
289 "mmSDMA%i_GFX_RB_WPTR == 0x%08x " 289 "mmSDMA%i_GFX_RB_WPTR == 0x%08x "
290 "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x \n", 290 "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
291 me,
292 me, 291 me,
293 lower_32_bits(ring->wptr << 2), 292 lower_32_bits(ring->wptr << 2),
293 me,
294 upper_32_bits(ring->wptr << 2)); 294 upper_32_bits(ring->wptr << 2));
295 WREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2)); 295 WREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2));
296 WREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2)); 296 WREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
@@ -319,22 +319,22 @@ static void sdma_v4_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
319 * Schedule an IB in the DMA ring (VEGA10). 319 * Schedule an IB in the DMA ring (VEGA10).
320 */ 320 */
321static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring, 321static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring,
322 struct amdgpu_ib *ib, 322 struct amdgpu_ib *ib,
323 unsigned vm_id, bool ctx_switch) 323 unsigned vm_id, bool ctx_switch)
324{ 324{
325 u32 vmid = vm_id & 0xf; 325 u32 vmid = vm_id & 0xf;
326 326
327 /* IB packet must end on a 8 DW boundary */ 327 /* IB packet must end on a 8 DW boundary */
328 sdma_v4_0_ring_insert_nop(ring, (10 - (lower_32_bits(ring->wptr) & 7)) % 8); 328 sdma_v4_0_ring_insert_nop(ring, (10 - (lower_32_bits(ring->wptr) & 7)) % 8);
329 329
330 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) | 330 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
331 SDMA_PKT_INDIRECT_HEADER_VMID(vmid)); 331 SDMA_PKT_INDIRECT_HEADER_VMID(vmid));
332 /* base must be 32 byte aligned */ 332 /* base must be 32 byte aligned */
333 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0); 333 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
334 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 334 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
335 amdgpu_ring_write(ring, ib->length_dw); 335 amdgpu_ring_write(ring, ib->length_dw);
336 amdgpu_ring_write(ring, 0); 336 amdgpu_ring_write(ring, 0);
337 amdgpu_ring_write(ring, 0); 337 amdgpu_ring_write(ring, 0);
338 338
339} 339}
340 340
@@ -523,7 +523,7 @@ static int sdma_v4_0_gfx_resume(struct amdgpu_device *adev)
523 u32 doorbell; 523 u32 doorbell;
524 u32 doorbell_offset; 524 u32 doorbell_offset;
525 u32 temp; 525 u32 temp;
526 int i,r; 526 int i, r;
527 527
528 for (i = 0; i < adev->sdma.num_instances; i++) { 528 for (i = 0; i < adev->sdma.num_instances; i++) {
529 ring = &adev->sdma.instance[i].ring; 529 ring = &adev->sdma.instance[i].ring;
@@ -572,7 +572,7 @@ static int sdma_v4_0_gfx_resume(struct amdgpu_device *adev)
572 doorbell = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_DOORBELL)); 572 doorbell = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_DOORBELL));
573 doorbell_offset = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_DOORBELL_OFFSET)); 573 doorbell_offset = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_DOORBELL_OFFSET));
574 574
575 if (ring->use_doorbell){ 575 if (ring->use_doorbell) {
576 doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 1); 576 doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 1);
577 doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_GFX_DOORBELL_OFFSET, 577 doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_GFX_DOORBELL_OFFSET,
578 OFFSET, ring->doorbell_index); 578 OFFSET, ring->doorbell_index);
@@ -694,9 +694,7 @@ static int sdma_v4_0_load_microcode(struct amdgpu_device *adev)
694 694
695 695
696 for (j = 0; j < fw_size; j++) 696 for (j = 0; j < fw_size; j++)
697 {
698 WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_UCODE_DATA), le32_to_cpup(fw_data++)); 697 WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
699 }
700 698
701 WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_UCODE_ADDR), adev->sdma.instance[i].fw_version); 699 WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_UCODE_ADDR), adev->sdma.instance[i].fw_version);
702 } 700 }
@@ -744,10 +742,8 @@ static int sdma_v4_0_start(struct amdgpu_device *adev)
744 if (r) 742 if (r)
745 return r; 743 return r;
746 r = sdma_v4_0_rlc_resume(adev); 744 r = sdma_v4_0_rlc_resume(adev);
747 if (r)
748 return r;
749 745
750 return 0; 746 return r;
751} 747}
752 748
753/** 749/**
@@ -797,9 +793,8 @@ static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring)
797 793
798 for (i = 0; i < adev->usec_timeout; i++) { 794 for (i = 0; i < adev->usec_timeout; i++) {
799 tmp = le32_to_cpu(adev->wb.wb[index]); 795 tmp = le32_to_cpu(adev->wb.wb[index]);
800 if (tmp == 0xDEADBEEF) { 796 if (tmp == 0xDEADBEEF)
801 break; 797 break;
802 }
803 DRM_UDELAY(1); 798 DRM_UDELAY(1);
804 } 799 }
805 800
@@ -864,29 +859,29 @@ static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
864 if (r) 859 if (r)
865 goto err1; 860 goto err1;
866 861
867 r = dma_fence_wait_timeout(f, false, timeout); 862 r = dma_fence_wait_timeout(f, false, timeout);
868 if (r == 0) { 863 if (r == 0) {
869 DRM_ERROR("amdgpu: IB test timed out\n"); 864 DRM_ERROR("amdgpu: IB test timed out\n");
870 r = -ETIMEDOUT; 865 r = -ETIMEDOUT;
871 goto err1; 866 goto err1;
872 } else if (r < 0) { 867 } else if (r < 0) {
873 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); 868 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
874 goto err1; 869 goto err1;
875 } 870 }
876 tmp = le32_to_cpu(adev->wb.wb[index]); 871 tmp = le32_to_cpu(adev->wb.wb[index]);
877 if (tmp == 0xDEADBEEF) { 872 if (tmp == 0xDEADBEEF) {
878 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 873 DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
879 r = 0; 874 r = 0;
880 } else { 875 } else {
881 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); 876 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
882 r = -EINVAL; 877 r = -EINVAL;
883 } 878 }
884err1: 879err1:
885 amdgpu_ib_free(adev, &ib, NULL); 880 amdgpu_ib_free(adev, &ib, NULL);
886 dma_fence_put(f); 881 dma_fence_put(f);
887err0: 882err0:
888 amdgpu_wb_free(adev, index); 883 amdgpu_wb_free(adev, index);
889 return r; 884 return r;
890} 885}
891 886
892 887
@@ -1039,44 +1034,40 @@ static void sdma_v4_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1039static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 1034static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1040 unsigned vm_id, uint64_t pd_addr) 1035 unsigned vm_id, uint64_t pd_addr)
1041{ 1036{
1037 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1042 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id); 1038 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
1043 unsigned eng = ring->idx; 1039 unsigned eng = ring->vm_inv_eng;
1044 unsigned i;
1045 1040
1046 pd_addr = pd_addr | 0x1; /* valid bit */ 1041 pd_addr = pd_addr | 0x1; /* valid bit */
1047 /* now only use physical base address of PDE and valid */ 1042 /* now only use physical base address of PDE and valid */
1048 BUG_ON(pd_addr & 0xFFFF00000000003EULL); 1043 BUG_ON(pd_addr & 0xFFFF00000000003EULL);
1049 1044
1050 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 1045 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1051 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i]; 1046 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1052 1047 amdgpu_ring_write(ring, hub->ctx0_ptb_addr_lo32 + vm_id * 2);
1053 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | 1048 amdgpu_ring_write(ring, lower_32_bits(pd_addr));
1054 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); 1049
1055 amdgpu_ring_write(ring, hub->ctx0_ptb_addr_lo32 + vm_id * 2); 1050 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1056 amdgpu_ring_write(ring, lower_32_bits(pd_addr)); 1051 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1057 1052 amdgpu_ring_write(ring, hub->ctx0_ptb_addr_hi32 + vm_id * 2);
1058 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | 1053 amdgpu_ring_write(ring, upper_32_bits(pd_addr));
1059 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); 1054
1060 amdgpu_ring_write(ring, hub->ctx0_ptb_addr_hi32 + vm_id * 2); 1055 /* flush TLB */
1061 amdgpu_ring_write(ring, upper_32_bits(pd_addr)); 1056 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1062 1057 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1063 /* flush TLB */ 1058 amdgpu_ring_write(ring, hub->vm_inv_eng0_req + eng);
1064 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | 1059 amdgpu_ring_write(ring, req);
1065 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); 1060
1066 amdgpu_ring_write(ring, hub->vm_inv_eng0_req + eng); 1061 /* wait for flush */
1067 amdgpu_ring_write(ring, req); 1062 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1068 1063 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1069 /* wait for flush */ 1064 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
1070 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | 1065 amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
1071 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) | 1066 amdgpu_ring_write(ring, 0);
1072 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */ 1067 amdgpu_ring_write(ring, 1 << vm_id); /* reference */
1073 amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2); 1068 amdgpu_ring_write(ring, 1 << vm_id); /* mask */
1074 amdgpu_ring_write(ring, 0); 1069 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1075 amdgpu_ring_write(ring, 1 << vm_id); /* reference */ 1070 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
1076 amdgpu_ring_write(ring, 1 << vm_id); /* mask */
1077 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1078 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
1079 }
1080} 1071}
1081 1072
1082static int sdma_v4_0_early_init(void *handle) 1073static int sdma_v4_0_early_init(void *handle)
@@ -1162,8 +1153,6 @@ static int sdma_v4_0_hw_init(void *handle)
1162 sdma_v4_0_init_golden_registers(adev); 1153 sdma_v4_0_init_golden_registers(adev);
1163 1154
1164 r = sdma_v4_0_start(adev); 1155 r = sdma_v4_0_start(adev);
1165 if (r)
1166 return r;
1167 1156
1168 return r; 1157 return r;
1169} 1158}
@@ -1199,10 +1188,12 @@ static bool sdma_v4_0_is_idle(void *handle)
1199{ 1188{
1200 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1189 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1201 u32 i; 1190 u32 i;
1191
1202 for (i = 0; i < adev->sdma.num_instances; i++) { 1192 for (i = 0; i < adev->sdma.num_instances; i++) {
1203 u32 tmp = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_STATUS_REG)); 1193 u32 tmp = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_STATUS_REG));
1194
1204 if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK)) 1195 if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1205 return false; 1196 return false;
1206 } 1197 }
1207 1198
1208 return true; 1199 return true;
@@ -1211,8 +1202,9 @@ static bool sdma_v4_0_is_idle(void *handle)
1211static int sdma_v4_0_wait_for_idle(void *handle) 1202static int sdma_v4_0_wait_for_idle(void *handle)
1212{ 1203{
1213 unsigned i; 1204 unsigned i;
1214 u32 sdma0,sdma1; 1205 u32 sdma0, sdma1;
1215 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1206 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1207
1216 for (i = 0; i < adev->usec_timeout; i++) { 1208 for (i = 0; i < adev->usec_timeout; i++) {
1217 sdma0 = RREG32(sdma_v4_0_get_reg_offset(0, mmSDMA0_STATUS_REG)); 1209 sdma0 = RREG32(sdma_v4_0_get_reg_offset(0, mmSDMA0_STATUS_REG));
1218 sdma1 = RREG32(sdma_v4_0_get_reg_offset(1, mmSDMA0_STATUS_REG)); 1210 sdma1 = RREG32(sdma_v4_0_get_reg_offset(1, mmSDMA0_STATUS_REG));
@@ -1240,7 +1232,7 @@ static int sdma_v4_0_set_trap_irq_state(struct amdgpu_device *adev,
1240 1232
1241 u32 reg_offset = (type == AMDGPU_SDMA_IRQ_TRAP0) ? 1233 u32 reg_offset = (type == AMDGPU_SDMA_IRQ_TRAP0) ?
1242 sdma_v4_0_get_reg_offset(0, mmSDMA0_CNTL) : 1234 sdma_v4_0_get_reg_offset(0, mmSDMA0_CNTL) :
1243 sdma_v4_0_get_reg_offset(1, mmSDMA0_CNTL); 1235 sdma_v4_0_get_reg_offset(1, mmSDMA0_CNTL);
1244 1236
1245 sdma_cntl = RREG32(reg_offset); 1237 sdma_cntl = RREG32(reg_offset);
1246 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE, 1238 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
@@ -1332,7 +1324,7 @@ static void sdma_v4_0_update_medium_grain_clock_gating(
1332 SDMA1_CLK_CTRL__SOFT_OVERRIDE2_MASK | 1324 SDMA1_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1333 SDMA1_CLK_CTRL__SOFT_OVERRIDE1_MASK | 1325 SDMA1_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1334 SDMA1_CLK_CTRL__SOFT_OVERRIDE0_MASK); 1326 SDMA1_CLK_CTRL__SOFT_OVERRIDE0_MASK);
1335 if(def != data) 1327 if (def != data)
1336 WREG32(SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_CLK_CTRL), data); 1328 WREG32(SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_CLK_CTRL), data);
1337 } 1329 }
1338 } else { 1330 } else {
@@ -1382,17 +1374,17 @@ static void sdma_v4_0_update_medium_grain_light_sleep(
1382 1374
1383 /* 1-not override: enable sdma1 mem light sleep */ 1375 /* 1-not override: enable sdma1 mem light sleep */
1384 if (adev->asic_type == CHIP_VEGA10) { 1376 if (adev->asic_type == CHIP_VEGA10) {
1385 def = data = RREG32(SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_POWER_CNTL)); 1377 def = data = RREG32(SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_POWER_CNTL));
1386 data |= SDMA1_POWER_CNTL__MEM_POWER_OVERRIDE_MASK; 1378 data |= SDMA1_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1387 if (def != data) 1379 if (def != data)
1388 WREG32(SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_POWER_CNTL), data); 1380 WREG32(SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_POWER_CNTL), data);
1389 } 1381 }
1390 } else { 1382 } else {
1391 /* 0-override:disable sdma0 mem light sleep */ 1383 /* 0-override:disable sdma0 mem light sleep */
1392 def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL)); 1384 def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL));
1393 data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK; 1385 data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1394 if (def != data) 1386 if (def != data)
1395 WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL), data); 1387 WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL), data);
1396 1388
1397 /* 0-override:disable sdma1 mem light sleep */ 1389 /* 0-override:disable sdma1 mem light sleep */
1398 if (adev->asic_type == CHIP_VEGA10) { 1390 if (adev->asic_type == CHIP_VEGA10) {
@@ -1473,6 +1465,7 @@ static const struct amdgpu_ring_funcs sdma_v4_0_ring_funcs = {
1473 .align_mask = 0xf, 1465 .align_mask = 0xf,
1474 .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), 1466 .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1475 .support_64bit_ptrs = true, 1467 .support_64bit_ptrs = true,
1468 .vmhub = AMDGPU_MMHUB,
1476 .get_rptr = sdma_v4_0_ring_get_rptr, 1469 .get_rptr = sdma_v4_0_ring_get_rptr,
1477 .get_wptr = sdma_v4_0_ring_get_wptr, 1470 .get_wptr = sdma_v4_0_ring_get_wptr,
1478 .set_wptr = sdma_v4_0_ring_set_wptr, 1471 .set_wptr = sdma_v4_0_ring_set_wptr,
@@ -1480,7 +1473,7 @@ static const struct amdgpu_ring_funcs sdma_v4_0_ring_funcs = {
1480 6 + /* sdma_v4_0_ring_emit_hdp_flush */ 1473 6 + /* sdma_v4_0_ring_emit_hdp_flush */
1481 3 + /* sdma_v4_0_ring_emit_hdp_invalidate */ 1474 3 + /* sdma_v4_0_ring_emit_hdp_invalidate */
1482 6 + /* sdma_v4_0_ring_emit_pipeline_sync */ 1475 6 + /* sdma_v4_0_ring_emit_pipeline_sync */
1483 36 + /* sdma_v4_0_ring_emit_vm_flush */ 1476 18 + /* sdma_v4_0_ring_emit_vm_flush */
1484 10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */ 1477 10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */
1485 .emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */ 1478 .emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */
1486 .emit_ib = sdma_v4_0_ring_emit_ib, 1479 .emit_ib = sdma_v4_0_ring_emit_ib,
@@ -1606,8 +1599,7 @@ static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev)
1606 } 1599 }
1607} 1600}
1608 1601
1609const struct amdgpu_ip_block_version sdma_v4_0_ip_block = 1602const struct amdgpu_ip_block_version sdma_v4_0_ip_block = {
1610{
1611 .type = AMD_IP_BLOCK_TYPE_SDMA, 1603 .type = AMD_IP_BLOCK_TYPE_SDMA,
1612 .major = 4, 1604 .major = 4,
1613 .minor = 0, 1605 .minor = 0,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 385de8617075..6b55d451ae7f 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -25,7 +25,7 @@
25#include <linux/module.h> 25#include <linux/module.h>
26#include "drmP.h" 26#include "drmP.h"
27#include "amdgpu.h" 27#include "amdgpu.h"
28#include "amdgpu_atombios.h" 28#include "amdgpu_atomfirmware.h"
29#include "amdgpu_ih.h" 29#include "amdgpu_ih.h"
30#include "amdgpu_uvd.h" 30#include "amdgpu_uvd.h"
31#include "amdgpu_vce.h" 31#include "amdgpu_vce.h"
@@ -405,11 +405,11 @@ static void soc15_gpu_pci_config_reset(struct amdgpu_device *adev)
405 405
406static int soc15_asic_reset(struct amdgpu_device *adev) 406static int soc15_asic_reset(struct amdgpu_device *adev)
407{ 407{
408 amdgpu_atombios_scratch_regs_engine_hung(adev, true); 408 amdgpu_atomfirmware_scratch_regs_engine_hung(adev, true);
409 409
410 soc15_gpu_pci_config_reset(adev); 410 soc15_gpu_pci_config_reset(adev);
411 411
412 amdgpu_atombios_scratch_regs_engine_hung(adev, false); 412 amdgpu_atomfirmware_scratch_regs_engine_hung(adev, false);
413 413
414 return 0; 414 return 0;
415} 415}
@@ -505,8 +505,7 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev)
505 amdgpu_ip_block_add(adev, &dce_virtual_ip_block); 505 amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
506 amdgpu_ip_block_add(adev, &gfx_v9_0_ip_block); 506 amdgpu_ip_block_add(adev, &gfx_v9_0_ip_block);
507 amdgpu_ip_block_add(adev, &sdma_v4_0_ip_block); 507 amdgpu_ip_block_add(adev, &sdma_v4_0_ip_block);
508 if (!amdgpu_sriov_vf(adev)) 508 amdgpu_ip_block_add(adev, &uvd_v7_0_ip_block);
509 amdgpu_ip_block_add(adev, &uvd_v7_0_ip_block);
510 amdgpu_ip_block_add(adev, &vce_v4_0_ip_block); 509 amdgpu_ip_block_add(adev, &vce_v4_0_ip_block);
511 break; 510 break;
512 default: 511 default:
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
index 2b96c806baa1..e8df6d820dbe 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
@@ -45,13 +45,31 @@ struct nbio_pcie_index_data {
45 u32 index_offset; 45 u32 index_offset;
46 u32 data_offset; 46 u32 data_offset;
47}; 47};
48// Register Access Macro 48
49/* Register Access Macros */
49#define SOC15_REG_OFFSET(ip, inst, reg) (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \ 50#define SOC15_REG_OFFSET(ip, inst, reg) (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \
50 (1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \ 51 (1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \
51 (2 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG2 + reg : \ 52 (2 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG2 + reg : \
52 (3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \ 53 (3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \
53 (ip##_BASE__INST##inst##_SEG4 + reg))))) 54 (ip##_BASE__INST##inst##_SEG4 + reg)))))
54 55
56#define WREG32_FIELD15(ip, idx, reg, field, val) \
57 WREG32(SOC15_REG_OFFSET(ip, idx, mm##reg), (RREG32(SOC15_REG_OFFSET(ip, idx, mm##reg)) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
58
59#define RREG32_SOC15(ip, inst, reg) \
60 RREG32( (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \
61 (1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \
62 (2 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG2 + reg : \
63 (3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \
64 (ip##_BASE__INST##inst##_SEG4 + reg))))))
65
66#define WREG32_SOC15(ip, inst, reg, value) \
67 WREG32( (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \
68 (1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \
69 (2 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG2 + reg : \
70 (3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \
71 (ip##_BASE__INST##inst##_SEG4 + reg))))), value)
72
55#endif 73#endif
56 74
57 75
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index 9bcf01469282..eca8f6e01e97 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -27,10 +27,14 @@
27#include "amdgpu_uvd.h" 27#include "amdgpu_uvd.h"
28#include "soc15d.h" 28#include "soc15d.h"
29#include "soc15_common.h" 29#include "soc15_common.h"
30#include "mmsch_v1_0.h"
30 31
31#include "vega10/soc15ip.h" 32#include "vega10/soc15ip.h"
32#include "vega10/UVD/uvd_7_0_offset.h" 33#include "vega10/UVD/uvd_7_0_offset.h"
33#include "vega10/UVD/uvd_7_0_sh_mask.h" 34#include "vega10/UVD/uvd_7_0_sh_mask.h"
35#include "vega10/VCE/vce_4_0_offset.h"
36#include "vega10/VCE/vce_4_0_default.h"
37#include "vega10/VCE/vce_4_0_sh_mask.h"
34#include "vega10/NBIF/nbif_6_1_offset.h" 38#include "vega10/NBIF/nbif_6_1_offset.h"
35#include "vega10/HDP/hdp_4_0_offset.h" 39#include "vega10/HDP/hdp_4_0_offset.h"
36#include "vega10/MMHUB/mmhub_1_0_offset.h" 40#include "vega10/MMHUB/mmhub_1_0_offset.h"
@@ -41,6 +45,7 @@ static void uvd_v7_0_set_enc_ring_funcs(struct amdgpu_device *adev);
41static void uvd_v7_0_set_irq_funcs(struct amdgpu_device *adev); 45static void uvd_v7_0_set_irq_funcs(struct amdgpu_device *adev);
42static int uvd_v7_0_start(struct amdgpu_device *adev); 46static int uvd_v7_0_start(struct amdgpu_device *adev);
43static void uvd_v7_0_stop(struct amdgpu_device *adev); 47static void uvd_v7_0_stop(struct amdgpu_device *adev);
48static int uvd_v7_0_sriov_start(struct amdgpu_device *adev);
44 49
45/** 50/**
46 * uvd_v7_0_ring_get_rptr - get read pointer 51 * uvd_v7_0_ring_get_rptr - get read pointer
@@ -98,6 +103,9 @@ static uint64_t uvd_v7_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
98{ 103{
99 struct amdgpu_device *adev = ring->adev; 104 struct amdgpu_device *adev = ring->adev;
100 105
106 if (ring->use_doorbell)
107 return adev->wb.wb[ring->wptr_offs];
108
101 if (ring == &adev->uvd.ring_enc[0]) 109 if (ring == &adev->uvd.ring_enc[0])
102 return RREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_WPTR)); 110 return RREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_WPTR));
103 else 111 else
@@ -129,6 +137,13 @@ static void uvd_v7_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
129{ 137{
130 struct amdgpu_device *adev = ring->adev; 138 struct amdgpu_device *adev = ring->adev;
131 139
140 if (ring->use_doorbell) {
141 /* XXX check if swapping is necessary on BE */
142 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
143 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
144 return;
145 }
146
132 if (ring == &adev->uvd.ring_enc[0]) 147 if (ring == &adev->uvd.ring_enc[0])
133 WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_WPTR), 148 WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_WPTR),
134 lower_32_bits(ring->wptr)); 149 lower_32_bits(ring->wptr));
@@ -353,7 +368,10 @@ static int uvd_v7_0_early_init(void *handle)
353{ 368{
354 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 369 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
355 370
356 adev->uvd.num_enc_rings = 2; 371 if (amdgpu_sriov_vf(adev))
372 adev->uvd.num_enc_rings = 1;
373 else
374 adev->uvd.num_enc_rings = 2;
357 uvd_v7_0_set_ring_funcs(adev); 375 uvd_v7_0_set_ring_funcs(adev);
358 uvd_v7_0_set_enc_ring_funcs(adev); 376 uvd_v7_0_set_enc_ring_funcs(adev);
359 uvd_v7_0_set_irq_funcs(adev); 377 uvd_v7_0_set_irq_funcs(adev);
@@ -406,21 +424,31 @@ static int uvd_v7_0_sw_init(void *handle)
406 r = amdgpu_uvd_resume(adev); 424 r = amdgpu_uvd_resume(adev);
407 if (r) 425 if (r)
408 return r; 426 return r;
427 if (!amdgpu_sriov_vf(adev)) {
428 ring = &adev->uvd.ring;
429 sprintf(ring->name, "uvd");
430 r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
431 if (r)
432 return r;
433 }
409 434
410 ring = &adev->uvd.ring;
411 sprintf(ring->name, "uvd");
412 r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
413 if (r)
414 return r;
415 435
416 for (i = 0; i < adev->uvd.num_enc_rings; ++i) { 436 for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
417 ring = &adev->uvd.ring_enc[i]; 437 ring = &adev->uvd.ring_enc[i];
418 sprintf(ring->name, "uvd_enc%d", i); 438 sprintf(ring->name, "uvd_enc%d", i);
439 if (amdgpu_sriov_vf(adev)) {
440 ring->use_doorbell = true;
441 ring->doorbell_index = AMDGPU_DOORBELL64_UVD_RING0_1 * 2;
442 }
419 r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0); 443 r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
420 if (r) 444 if (r)
421 return r; 445 return r;
422 } 446 }
423 447
448 r = amdgpu_virt_alloc_mm_table(adev);
449 if (r)
450 return r;
451
424 return r; 452 return r;
425} 453}
426 454
@@ -429,6 +457,8 @@ static int uvd_v7_0_sw_fini(void *handle)
429 int i, r; 457 int i, r;
430 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 458 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
431 459
460 amdgpu_virt_free_mm_table(adev);
461
432 r = amdgpu_uvd_suspend(adev); 462 r = amdgpu_uvd_suspend(adev);
433 if (r) 463 if (r)
434 return r; 464 return r;
@@ -455,48 +485,53 @@ static int uvd_v7_0_hw_init(void *handle)
455 uint32_t tmp; 485 uint32_t tmp;
456 int i, r; 486 int i, r;
457 487
458 r = uvd_v7_0_start(adev); 488 if (amdgpu_sriov_vf(adev))
489 r = uvd_v7_0_sriov_start(adev);
490 else
491 r = uvd_v7_0_start(adev);
459 if (r) 492 if (r)
460 goto done; 493 goto done;
461 494
462 ring->ready = true; 495 if (!amdgpu_sriov_vf(adev)) {
463 r = amdgpu_ring_test_ring(ring); 496 ring->ready = true;
464 if (r) { 497 r = amdgpu_ring_test_ring(ring);
465 ring->ready = false; 498 if (r) {
466 goto done; 499 ring->ready = false;
467 } 500 goto done;
501 }
468 502
469 r = amdgpu_ring_alloc(ring, 10); 503 r = amdgpu_ring_alloc(ring, 10);
470 if (r) { 504 if (r) {
471 DRM_ERROR("amdgpu: ring failed to lock UVD ring (%d).\n", r); 505 DRM_ERROR("amdgpu: ring failed to lock UVD ring (%d).\n", r);
472 goto done; 506 goto done;
473 } 507 }
474 508
475 tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0, 509 tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0,
476 mmUVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL), 0); 510 mmUVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL), 0);
477 amdgpu_ring_write(ring, tmp); 511 amdgpu_ring_write(ring, tmp);
478 amdgpu_ring_write(ring, 0xFFFFF); 512 amdgpu_ring_write(ring, 0xFFFFF);
479 513
480 tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0, 514 tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0,
481 mmUVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL), 0); 515 mmUVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL), 0);
482 amdgpu_ring_write(ring, tmp); 516 amdgpu_ring_write(ring, tmp);
483 amdgpu_ring_write(ring, 0xFFFFF); 517 amdgpu_ring_write(ring, 0xFFFFF);
484 518
485 tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0, 519 tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0,
486 mmUVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL), 0); 520 mmUVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL), 0);
487 amdgpu_ring_write(ring, tmp); 521 amdgpu_ring_write(ring, tmp);
488 amdgpu_ring_write(ring, 0xFFFFF); 522 amdgpu_ring_write(ring, 0xFFFFF);
489 523
490 /* Clear timeout status bits */ 524 /* Clear timeout status bits */
491 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, 525 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0,
492 mmUVD_SEMA_TIMEOUT_STATUS), 0)); 526 mmUVD_SEMA_TIMEOUT_STATUS), 0));
493 amdgpu_ring_write(ring, 0x8); 527 amdgpu_ring_write(ring, 0x8);
494 528
495 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, 529 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0,
496 mmUVD_SEMA_CNTL), 0)); 530 mmUVD_SEMA_CNTL), 0));
497 amdgpu_ring_write(ring, 3); 531 amdgpu_ring_write(ring, 3);
498 532
499 amdgpu_ring_commit(ring); 533 amdgpu_ring_commit(ring);
534 }
500 535
501 for (i = 0; i < adev->uvd.num_enc_rings; ++i) { 536 for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
502 ring = &adev->uvd.ring_enc[i]; 537 ring = &adev->uvd.ring_enc[i];
@@ -618,6 +653,241 @@ static void uvd_v7_0_mc_resume(struct amdgpu_device *adev)
618 WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH4), adev->uvd.max_handles); 653 WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH4), adev->uvd.max_handles);
619} 654}
620 655
656static int uvd_v7_0_mmsch_start(struct amdgpu_device *adev,
657 struct amdgpu_mm_table *table)
658{
659 uint32_t data = 0, loop;
660 uint64_t addr = table->gpu_addr;
661 struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)table->cpu_addr;
662 uint32_t size;
663
664 size = header->header_size + header->vce_table_size + header->uvd_table_size;
665
666 /* 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of memory descriptor location */
667 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_LO), lower_32_bits(addr));
668 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_HI), upper_32_bits(addr));
669
670 /* 2, update vmid of descriptor */
671 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_VMID));
672 data &= ~VCE_MMSCH_VF_VMID__VF_CTX_VMID_MASK;
673 data |= (0 << VCE_MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); /* use domain0 for MM scheduler */
674 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_VMID), data);
675
676 /* 3, notify mmsch about the size of this descriptor */
677 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_SIZE), size);
678
679 /* 4, set resp to zero */
680 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP), 0);
681
682 /* 5, kick off the initialization and wait until VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero */
683 WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_HOST), 0x10000001);
684
685 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP));
686 loop = 1000;
687 while ((data & 0x10000002) != 0x10000002) {
688 udelay(10);
689 data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP));
690 loop--;
691 if (!loop)
692 break;
693 }
694
695 if (!loop) {
696 dev_err(adev->dev, "failed to init MMSCH, mmVCE_MMSCH_VF_MAILBOX_RESP = %x\n", data);
697 return -EBUSY;
698 }
699
700 return 0;
701}
702
703static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)
704{
705 struct amdgpu_ring *ring;
706 uint32_t offset, size, tmp;
707 uint32_t table_size = 0;
708 struct mmsch_v1_0_cmd_direct_write direct_wt = { {0} };
709 struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { {0} };
710 struct mmsch_v1_0_cmd_direct_polling direct_poll = { {0} };
711 struct mmsch_v1_0_cmd_end end = { {0} };
712 uint32_t *init_table = adev->virt.mm_table.cpu_addr;
713 struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)init_table;
714
715 direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;
716 direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
717 direct_poll.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_POLLING;
718 end.cmd_header.command_type = MMSCH_COMMAND__END;
719
720 if (header->uvd_table_offset == 0 && header->uvd_table_size == 0) {
721 header->version = MMSCH_VERSION;
722 header->header_size = sizeof(struct mmsch_v1_0_init_header) >> 2;
723
724 if (header->vce_table_offset == 0 && header->vce_table_size == 0)
725 header->uvd_table_offset = header->header_size;
726 else
727 header->uvd_table_offset = header->vce_table_size + header->vce_table_offset;
728
729 init_table += header->uvd_table_offset;
730
731 ring = &adev->uvd.ring;
732 size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4);
733
734 /* disable clock gating */
735 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS),
736 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK, 0);
737 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS),
738 0xFFFFFFFF, 0x00000004);
739 /* mc resume*/
740 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
741 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
742 lower_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
743 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
744 upper_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
745 offset = 0;
746 } else {
747 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
748 lower_32_bits(adev->uvd.gpu_addr));
749 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
750 upper_32_bits(adev->uvd.gpu_addr));
751 offset = size;
752 }
753
754 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0),
755 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
756 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size);
757
758 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
759 lower_32_bits(adev->uvd.gpu_addr + offset));
760 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
761 upper_32_bits(adev->uvd.gpu_addr + offset));
762 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), (1 << 21));
763 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_UVD_HEAP_SIZE);
764
765 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
766 lower_32_bits(adev->uvd.gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
767 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
768 upper_32_bits(adev->uvd.gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
769 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), (2 << 21));
770 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE2),
771 AMDGPU_UVD_STACK_SIZE + (AMDGPU_UVD_SESSION_SIZE * 40));
772
773 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_UDEC_ADDR_CONFIG),
774 adev->gfx.config.gb_addr_config);
775 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG),
776 adev->gfx.config.gb_addr_config);
777 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG),
778 adev->gfx.config.gb_addr_config);
779 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH4), adev->uvd.max_handles);
780 /* mc resume end*/
781
782 /* disable clock gating */
783 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_CGC_CTRL),
784 ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK, 0);
785
786 /* disable interupt */
787 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
788 ~UVD_MASTINT_EN__VCPU_EN_MASK, 0);
789
790 /* stall UMC and register bus before resetting VCPU */
791 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
792 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
793 UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
794
795 /* put LMI, VCPU, RBC etc... into reset */
796 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
797 (uint32_t)(UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
798 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
799 UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
800 UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
801 UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
802 UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
803 UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
804 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK));
805
806 /* initialize UVD memory controller */
807 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL),
808 (uint32_t)((0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
809 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
810 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
811 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
812 UVD_LMI_CTRL__REQ_MODE_MASK |
813 0x00100000L));
814
815 /* disable byte swapping */
816 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_SWAP_CNTL), 0);
817 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MP_SWAP_CNTL), 0);
818
819 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_MUXA0), 0x40c2040);
820 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_MUXA1), 0x0);
821 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_MUXB0), 0x40c2040);
822 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_MUXB1), 0x0);
823 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_ALU), 0);
824 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_MUX), 0x88);
825
826 /* take all subblocks out of reset, except VCPU */
827 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
828 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
829
830 /* enable VCPU clock */
831 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL),
832 UVD_VCPU_CNTL__CLK_EN_MASK);
833
834 /* enable UMC */
835 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
836 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, 0);
837
838 /* boot up the VCPU */
839 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0);
840
841 MMSCH_V1_0_INSERT_DIRECT_POLL(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0x02, 0x02);
842
843 /* enable master interrupt */
844 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
845 ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
846 (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
847
848 /* clear the bit 4 of UVD_STATUS */
849 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS),
850 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT), 0);
851
852 /* force RBC into idle state */
853 size = order_base_2(ring->ring_size);
854 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, size);
855 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
856 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
857 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
858 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
859 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
860 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), tmp);
861
862 /* set the write pointer delay */
863 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL), 0);
864
865 /* set the wb address */
866 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR),
867 (upper_32_bits(ring->gpu_addr) >> 2));
868
869 /* programm the RB_BASE for ring buffer */
870 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
871 lower_32_bits(ring->gpu_addr));
872 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
873 upper_32_bits(ring->gpu_addr));
874
875 ring->wptr = 0;
876 ring = &adev->uvd.ring_enc[0];
877 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_LO), ring->gpu_addr);
878 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_HI), upper_32_bits(ring->gpu_addr));
879 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_SIZE), ring->ring_size / 4);
880
881 /* add end packet */
882 memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
883 table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;
884 header->uvd_table_size = table_size;
885
886 return uvd_v7_0_mmsch_start(adev, &adev->virt.mm_table);
887 }
888 return -EINVAL; /* already initializaed ? */
889}
890
621/** 891/**
622 * uvd_v7_0_start - start UVD block 892 * uvd_v7_0_start - start UVD block
623 * 893 *
@@ -1034,42 +1304,38 @@ static void uvd_v7_0_vm_reg_wait(struct amdgpu_ring *ring,
1034static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 1304static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1035 unsigned vm_id, uint64_t pd_addr) 1305 unsigned vm_id, uint64_t pd_addr)
1036{ 1306{
1307 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1037 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id); 1308 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
1038 uint32_t data0, data1, mask; 1309 uint32_t data0, data1, mask;
1039 unsigned eng = ring->idx; 1310 unsigned eng = ring->vm_inv_eng;
1040 unsigned i;
1041 1311
1042 pd_addr = pd_addr | 0x1; /* valid bit */ 1312 pd_addr = pd_addr | 0x1; /* valid bit */
1043 /* now only use physical base address of PDE and valid */ 1313 /* now only use physical base address of PDE and valid */
1044 BUG_ON(pd_addr & 0xFFFF00000000003EULL); 1314 BUG_ON(pd_addr & 0xFFFF00000000003EULL);
1045 1315
1046 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 1316 data0 = (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2;
1047 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i]; 1317 data1 = upper_32_bits(pd_addr);
1048 1318 uvd_v7_0_vm_reg_write(ring, data0, data1);
1049 data0 = (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2; 1319
1050 data1 = upper_32_bits(pd_addr); 1320 data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2;
1051 uvd_v7_0_vm_reg_write(ring, data0, data1); 1321 data1 = lower_32_bits(pd_addr);
1052 1322 uvd_v7_0_vm_reg_write(ring, data0, data1);
1053 data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2; 1323
1054 data1 = lower_32_bits(pd_addr); 1324 data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2;
1055 uvd_v7_0_vm_reg_write(ring, data0, data1); 1325 data1 = lower_32_bits(pd_addr);
1056 1326 mask = 0xffffffff;
1057 data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2; 1327 uvd_v7_0_vm_reg_wait(ring, data0, data1, mask);
1058 data1 = lower_32_bits(pd_addr); 1328
1059 mask = 0xffffffff; 1329 /* flush TLB */
1060 uvd_v7_0_vm_reg_wait(ring, data0, data1, mask); 1330 data0 = (hub->vm_inv_eng0_req + eng) << 2;
1061 1331 data1 = req;
1062 /* flush TLB */ 1332 uvd_v7_0_vm_reg_write(ring, data0, data1);
1063 data0 = (hub->vm_inv_eng0_req + eng) << 2; 1333
1064 data1 = req; 1334 /* wait for flush */
1065 uvd_v7_0_vm_reg_write(ring, data0, data1); 1335 data0 = (hub->vm_inv_eng0_ack + eng) << 2;
1066 1336 data1 = 1 << vm_id;
1067 /* wait for flush */ 1337 mask = 1 << vm_id;
1068 data0 = (hub->vm_inv_eng0_ack + eng) << 2; 1338 uvd_v7_0_vm_reg_wait(ring, data0, data1, mask);
1069 data1 = 1 << vm_id;
1070 mask = 1 << vm_id;
1071 uvd_v7_0_vm_reg_wait(ring, data0, data1, mask);
1072 }
1073} 1339}
1074 1340
1075static void uvd_v7_0_enc_ring_insert_end(struct amdgpu_ring *ring) 1341static void uvd_v7_0_enc_ring_insert_end(struct amdgpu_ring *ring)
@@ -1080,44 +1346,37 @@ static void uvd_v7_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1080static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring, 1346static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1081 unsigned int vm_id, uint64_t pd_addr) 1347 unsigned int vm_id, uint64_t pd_addr)
1082{ 1348{
1349 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1083 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id); 1350 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
1084 unsigned eng = ring->idx; 1351 unsigned eng = ring->vm_inv_eng;
1085 unsigned i;
1086 1352
1087 pd_addr = pd_addr | 0x1; /* valid bit */ 1353 pd_addr = pd_addr | 0x1; /* valid bit */
1088 /* now only use physical base address of PDE and valid */ 1354 /* now only use physical base address of PDE and valid */
1089 BUG_ON(pd_addr & 0xFFFF00000000003EULL); 1355 BUG_ON(pd_addr & 0xFFFF00000000003EULL);
1090 1356
1091 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 1357 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE);
1092 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i]; 1358 amdgpu_ring_write(ring, (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2);
1093 1359 amdgpu_ring_write(ring, upper_32_bits(pd_addr));
1094 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE); 1360
1095 amdgpu_ring_write(ring, 1361 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE);
1096 (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2); 1362 amdgpu_ring_write(ring, (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
1097 amdgpu_ring_write(ring, upper_32_bits(pd_addr)); 1363 amdgpu_ring_write(ring, lower_32_bits(pd_addr));
1098 1364
1099 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE); 1365 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WAIT);
1100 amdgpu_ring_write(ring, 1366 amdgpu_ring_write(ring, (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
1101 (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2); 1367 amdgpu_ring_write(ring, 0xffffffff);
1102 amdgpu_ring_write(ring, lower_32_bits(pd_addr)); 1368 amdgpu_ring_write(ring, lower_32_bits(pd_addr));
1103 1369
1104 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WAIT); 1370 /* flush TLB */
1105 amdgpu_ring_write(ring, 1371 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE);
1106 (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2); 1372 amdgpu_ring_write(ring, (hub->vm_inv_eng0_req + eng) << 2);
1107 amdgpu_ring_write(ring, 0xffffffff); 1373 amdgpu_ring_write(ring, req);
1108 amdgpu_ring_write(ring, lower_32_bits(pd_addr)); 1374
1109 1375 /* wait for flush */
1110 /* flush TLB */ 1376 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WAIT);
1111 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE); 1377 amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
1112 amdgpu_ring_write(ring, (hub->vm_inv_eng0_req + eng) << 2); 1378 amdgpu_ring_write(ring, 1 << vm_id);
1113 amdgpu_ring_write(ring, req); 1379 amdgpu_ring_write(ring, 1 << vm_id);
1114
1115 /* wait for flush */
1116 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WAIT);
1117 amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
1118 amdgpu_ring_write(ring, 1 << vm_id);
1119 amdgpu_ring_write(ring, 1 << vm_id);
1120 }
1121} 1380}
1122 1381
1123#if 0 1382#if 0
@@ -1240,7 +1499,8 @@ static int uvd_v7_0_process_interrupt(struct amdgpu_device *adev,
1240 amdgpu_fence_process(&adev->uvd.ring_enc[0]); 1499 amdgpu_fence_process(&adev->uvd.ring_enc[0]);
1241 break; 1500 break;
1242 case 120: 1501 case 120:
1243 amdgpu_fence_process(&adev->uvd.ring_enc[1]); 1502 if (!amdgpu_sriov_vf(adev))
1503 amdgpu_fence_process(&adev->uvd.ring_enc[1]);
1244 break; 1504 break;
1245 default: 1505 default:
1246 DRM_ERROR("Unhandled interrupt: %d %d\n", 1506 DRM_ERROR("Unhandled interrupt: %d %d\n",
@@ -1448,13 +1708,14 @@ static const struct amdgpu_ring_funcs uvd_v7_0_ring_vm_funcs = {
1448 .align_mask = 0xf, 1708 .align_mask = 0xf,
1449 .nop = PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0), 1709 .nop = PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0),
1450 .support_64bit_ptrs = false, 1710 .support_64bit_ptrs = false,
1711 .vmhub = AMDGPU_MMHUB,
1451 .get_rptr = uvd_v7_0_ring_get_rptr, 1712 .get_rptr = uvd_v7_0_ring_get_rptr,
1452 .get_wptr = uvd_v7_0_ring_get_wptr, 1713 .get_wptr = uvd_v7_0_ring_get_wptr,
1453 .set_wptr = uvd_v7_0_ring_set_wptr, 1714 .set_wptr = uvd_v7_0_ring_set_wptr,
1454 .emit_frame_size = 1715 .emit_frame_size =
1455 2 + /* uvd_v7_0_ring_emit_hdp_flush */ 1716 2 + /* uvd_v7_0_ring_emit_hdp_flush */
1456 2 + /* uvd_v7_0_ring_emit_hdp_invalidate */ 1717 2 + /* uvd_v7_0_ring_emit_hdp_invalidate */
1457 34 * AMDGPU_MAX_VMHUBS + /* uvd_v7_0_ring_emit_vm_flush */ 1718 34 + /* uvd_v7_0_ring_emit_vm_flush */
1458 14 + 14, /* uvd_v7_0_ring_emit_fence x2 vm fence */ 1719 14 + 14, /* uvd_v7_0_ring_emit_fence x2 vm fence */
1459 .emit_ib_size = 8, /* uvd_v7_0_ring_emit_ib */ 1720 .emit_ib_size = 8, /* uvd_v7_0_ring_emit_ib */
1460 .emit_ib = uvd_v7_0_ring_emit_ib, 1721 .emit_ib = uvd_v7_0_ring_emit_ib,
@@ -1475,11 +1736,12 @@ static const struct amdgpu_ring_funcs uvd_v7_0_enc_ring_vm_funcs = {
1475 .align_mask = 0x3f, 1736 .align_mask = 0x3f,
1476 .nop = HEVC_ENC_CMD_NO_OP, 1737 .nop = HEVC_ENC_CMD_NO_OP,
1477 .support_64bit_ptrs = false, 1738 .support_64bit_ptrs = false,
1739 .vmhub = AMDGPU_MMHUB,
1478 .get_rptr = uvd_v7_0_enc_ring_get_rptr, 1740 .get_rptr = uvd_v7_0_enc_ring_get_rptr,
1479 .get_wptr = uvd_v7_0_enc_ring_get_wptr, 1741 .get_wptr = uvd_v7_0_enc_ring_get_wptr,
1480 .set_wptr = uvd_v7_0_enc_ring_set_wptr, 1742 .set_wptr = uvd_v7_0_enc_ring_set_wptr,
1481 .emit_frame_size = 1743 .emit_frame_size =
1482 17 * AMDGPU_MAX_VMHUBS + /* uvd_v7_0_enc_ring_emit_vm_flush */ 1744 17 + /* uvd_v7_0_enc_ring_emit_vm_flush */
1483 5 + 5 + /* uvd_v7_0_enc_ring_emit_fence x2 vm fence */ 1745 5 + 5 + /* uvd_v7_0_enc_ring_emit_fence x2 vm fence */
1484 1, /* uvd_v7_0_enc_ring_insert_end */ 1746 1, /* uvd_v7_0_enc_ring_insert_end */
1485 .emit_ib_size = 5, /* uvd_v7_0_enc_ring_emit_ib */ 1747 .emit_ib_size = 5, /* uvd_v7_0_enc_ring_emit_ib */
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index edde5fe938d6..139f964196b4 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -49,63 +49,6 @@ static void vce_v4_0_mc_resume(struct amdgpu_device *adev);
49static void vce_v4_0_set_ring_funcs(struct amdgpu_device *adev); 49static void vce_v4_0_set_ring_funcs(struct amdgpu_device *adev);
50static void vce_v4_0_set_irq_funcs(struct amdgpu_device *adev); 50static void vce_v4_0_set_irq_funcs(struct amdgpu_device *adev);
51 51
52static inline void mmsch_insert_direct_wt(struct mmsch_v1_0_cmd_direct_write *direct_wt,
53 uint32_t *init_table,
54 uint32_t reg_offset,
55 uint32_t value)
56{
57 direct_wt->cmd_header.reg_offset = reg_offset;
58 direct_wt->reg_value = value;
59 memcpy((void *)init_table, direct_wt, sizeof(struct mmsch_v1_0_cmd_direct_write));
60}
61
62static inline void mmsch_insert_direct_rd_mod_wt(struct mmsch_v1_0_cmd_direct_read_modify_write *direct_rd_mod_wt,
63 uint32_t *init_table,
64 uint32_t reg_offset,
65 uint32_t mask, uint32_t data)
66{
67 direct_rd_mod_wt->cmd_header.reg_offset = reg_offset;
68 direct_rd_mod_wt->mask_value = mask;
69 direct_rd_mod_wt->write_data = data;
70 memcpy((void *)init_table, direct_rd_mod_wt,
71 sizeof(struct mmsch_v1_0_cmd_direct_read_modify_write));
72}
73
74static inline void mmsch_insert_direct_poll(struct mmsch_v1_0_cmd_direct_polling *direct_poll,
75 uint32_t *init_table,
76 uint32_t reg_offset,
77 uint32_t mask, uint32_t wait)
78{
79 direct_poll->cmd_header.reg_offset = reg_offset;
80 direct_poll->mask_value = mask;
81 direct_poll->wait_value = wait;
82 memcpy((void *)init_table, direct_poll, sizeof(struct mmsch_v1_0_cmd_direct_polling));
83}
84
85#define INSERT_DIRECT_RD_MOD_WT(reg, mask, data) { \
86 mmsch_insert_direct_rd_mod_wt(&direct_rd_mod_wt, \
87 init_table, (reg), \
88 (mask), (data)); \
89 init_table += sizeof(struct mmsch_v1_0_cmd_direct_read_modify_write)/4; \
90 table_size += sizeof(struct mmsch_v1_0_cmd_direct_read_modify_write)/4; \
91}
92
93#define INSERT_DIRECT_WT(reg, value) { \
94 mmsch_insert_direct_wt(&direct_wt, \
95 init_table, (reg), \
96 (value)); \
97 init_table += sizeof(struct mmsch_v1_0_cmd_direct_write)/4; \
98 table_size += sizeof(struct mmsch_v1_0_cmd_direct_write)/4; \
99}
100
101#define INSERT_DIRECT_POLL(reg, mask, wait) { \
102 mmsch_insert_direct_poll(&direct_poll, \
103 init_table, (reg), \
104 (mask), (wait)); \
105 init_table += sizeof(struct mmsch_v1_0_cmd_direct_polling)/4; \
106 table_size += sizeof(struct mmsch_v1_0_cmd_direct_polling)/4; \
107}
108
109/** 52/**
110 * vce_v4_0_ring_get_rptr - get read pointer 53 * vce_v4_0_ring_get_rptr - get read pointer
111 * 54 *
@@ -280,60 +223,73 @@ static int vce_v4_0_sriov_start(struct amdgpu_device *adev)
280 init_table += header->vce_table_offset; 223 init_table += header->vce_table_offset;
281 224
282 ring = &adev->vce.ring[0]; 225 ring = &adev->vce.ring[0];
283 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR), ring->wptr); 226 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO),
284 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR), ring->wptr); 227 lower_32_bits(ring->gpu_addr));
285 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO), lower_32_bits(ring->gpu_addr)); 228 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI),
286 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI), upper_32_bits(ring->gpu_addr)); 229 upper_32_bits(ring->gpu_addr));
287 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE), ring->ring_size / 4); 230 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE),
231 ring->ring_size / 4);
288 232
289 /* BEGING OF MC_RESUME */ 233 /* BEGING OF MC_RESUME */
290 INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_A), ~(1 << 16), 0); 234 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL), 0x398000);
291 INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING), ~0xFF9FF000, 0x1FF000); 235 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CACHE_CTRL), ~0x1, 0);
292 INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING), ~0x3F, 0x3F); 236 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL), 0);
293 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B), 0x1FF); 237 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL1), 0);
294 238 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VM_CTRL), 0);
295 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL), 0x398000); 239
296 INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CACHE_CTRL), ~0x1, 0); 240 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
297 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL), 0); 241 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
298 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL1), 0); 242 adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8);
299 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VM_CTRL), 0); 243 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR1),
300 244 adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8);
301 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0), adev->vce.gpu_addr >> 8); 245 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR2),
302 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR1), adev->vce.gpu_addr >> 8); 246 adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8);
303 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR2), adev->vce.gpu_addr >> 8); 247 } else {
248 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
249 adev->vce.gpu_addr >> 8);
250 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR1),
251 adev->vce.gpu_addr >> 8);
252 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR2),
253 adev->vce.gpu_addr >> 8);
254 }
304 255
305 offset = AMDGPU_VCE_FIRMWARE_OFFSET; 256 offset = AMDGPU_VCE_FIRMWARE_OFFSET;
306 size = VCE_V4_0_FW_SIZE; 257 size = VCE_V4_0_FW_SIZE;
307 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0), offset & 0x7FFFFFFF); 258 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0),
308 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE0), size); 259 offset & 0x7FFFFFFF);
260 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE0), size);
309 261
310 offset += size; 262 offset += size;
311 size = VCE_V4_0_STACK_SIZE; 263 size = VCE_V4_0_STACK_SIZE;
312 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET1), offset & 0x7FFFFFFF); 264 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET1),
313 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE1), size); 265 offset & 0x7FFFFFFF);
266 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE1), size);
314 267
315 offset += size; 268 offset += size;
316 size = VCE_V4_0_DATA_SIZE; 269 size = VCE_V4_0_DATA_SIZE;
317 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET2), offset & 0x7FFFFFFF); 270 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET2),
318 INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE2), size); 271 offset & 0x7FFFFFFF);
272 MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE2), size);
319 273
320 INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), ~0x100, 0); 274 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), ~0x100, 0);
321 INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN), 275 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN),
322 0xffffffff, VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK); 276 0xffffffff, VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
323 277
324 /* end of MC_RESUME */ 278 /* end of MC_RESUME */
325 INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL), 279 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),
326 ~0x200001, VCE_VCPU_CNTL__CLK_EN_MASK); 280 VCE_STATUS__JOB_BUSY_MASK, ~VCE_STATUS__JOB_BUSY_MASK);
327 INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET), 281 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL),
328 ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, 0); 282 ~0x200001, VCE_VCPU_CNTL__CLK_EN_MASK);
283 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET),
284 ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, 0);
329 285
330 INSERT_DIRECT_POLL(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), 286 MMSCH_V1_0_INSERT_DIRECT_POLL(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),
331 VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK, 287 VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK,
332 VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK); 288 VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK);
333 289
334 /* clear BUSY flag */ 290 /* clear BUSY flag */
335 INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), 291 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),
336 ~VCE_STATUS__JOB_BUSY_MASK, 0); 292 ~VCE_STATUS__JOB_BUSY_MASK, 0);
337 293
338 /* add end packet */ 294 /* add end packet */
339 memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end)); 295 memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
@@ -494,20 +450,9 @@ static int vce_v4_0_sw_init(void *handle)
494 return r; 450 return r;
495 } 451 }
496 452
497 if (amdgpu_sriov_vf(adev)) { 453 r = amdgpu_virt_alloc_mm_table(adev);
498 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE, 454 if (r)
499 AMDGPU_GEM_DOMAIN_VRAM,
500 &adev->virt.mm_table.bo,
501 &adev->virt.mm_table.gpu_addr,
502 (void *)&adev->virt.mm_table.cpu_addr);
503 if (!r) {
504 memset((void *)adev->virt.mm_table.cpu_addr, 0, PAGE_SIZE);
505 printk("mm table gpu addr = 0x%llx, cpu addr = %p. \n",
506 adev->virt.mm_table.gpu_addr,
507 adev->virt.mm_table.cpu_addr);
508 }
509 return r; 455 return r;
510 }
511 456
512 return r; 457 return r;
513} 458}
@@ -518,10 +463,7 @@ static int vce_v4_0_sw_fini(void *handle)
518 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 463 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
519 464
520 /* free MM table */ 465 /* free MM table */
521 if (amdgpu_sriov_vf(adev)) 466 amdgpu_virt_free_mm_table(adev);
522 amdgpu_bo_free_kernel(&adev->virt.mm_table.bo,
523 &adev->virt.mm_table.gpu_addr,
524 (void *)&adev->virt.mm_table.cpu_addr);
525 467
526 r = amdgpu_vce_suspend(adev); 468 r = amdgpu_vce_suspend(adev);
527 if (r) 469 if (r)
@@ -973,44 +915,37 @@ static void vce_v4_0_ring_insert_end(struct amdgpu_ring *ring)
973static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring, 915static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring,
974 unsigned int vm_id, uint64_t pd_addr) 916 unsigned int vm_id, uint64_t pd_addr)
975{ 917{
918 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
976 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id); 919 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
977 unsigned eng = ring->idx; 920 unsigned eng = ring->vm_inv_eng;
978 unsigned i;
979 921
980 pd_addr = pd_addr | 0x1; /* valid bit */ 922 pd_addr = pd_addr | 0x1; /* valid bit */
981 /* now only use physical base address of PDE and valid */ 923 /* now only use physical base address of PDE and valid */
982 BUG_ON(pd_addr & 0xFFFF00000000003EULL); 924 BUG_ON(pd_addr & 0xFFFF00000000003EULL);
983 925
984 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 926 amdgpu_ring_write(ring, VCE_CMD_REG_WRITE);
985 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i]; 927 amdgpu_ring_write(ring, (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2);
986 928 amdgpu_ring_write(ring, upper_32_bits(pd_addr));
987 amdgpu_ring_write(ring, VCE_CMD_REG_WRITE); 929
988 amdgpu_ring_write(ring, 930 amdgpu_ring_write(ring, VCE_CMD_REG_WRITE);
989 (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2); 931 amdgpu_ring_write(ring, (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
990 amdgpu_ring_write(ring, upper_32_bits(pd_addr)); 932 amdgpu_ring_write(ring, lower_32_bits(pd_addr));
991 933
992 amdgpu_ring_write(ring, VCE_CMD_REG_WRITE); 934 amdgpu_ring_write(ring, VCE_CMD_REG_WAIT);
993 amdgpu_ring_write(ring, 935 amdgpu_ring_write(ring, (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
994 (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2); 936 amdgpu_ring_write(ring, 0xffffffff);
995 amdgpu_ring_write(ring, lower_32_bits(pd_addr)); 937 amdgpu_ring_write(ring, lower_32_bits(pd_addr));
996 938
997 amdgpu_ring_write(ring, VCE_CMD_REG_WAIT); 939 /* flush TLB */
998 amdgpu_ring_write(ring, 940 amdgpu_ring_write(ring, VCE_CMD_REG_WRITE);
999 (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2); 941 amdgpu_ring_write(ring, (hub->vm_inv_eng0_req + eng) << 2);
1000 amdgpu_ring_write(ring, 0xffffffff); 942 amdgpu_ring_write(ring, req);
1001 amdgpu_ring_write(ring, lower_32_bits(pd_addr)); 943
1002 944 /* wait for flush */
1003 /* flush TLB */ 945 amdgpu_ring_write(ring, VCE_CMD_REG_WAIT);
1004 amdgpu_ring_write(ring, VCE_CMD_REG_WRITE); 946 amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
1005 amdgpu_ring_write(ring, (hub->vm_inv_eng0_req + eng) << 2); 947 amdgpu_ring_write(ring, 1 << vm_id);
1006 amdgpu_ring_write(ring, req); 948 amdgpu_ring_write(ring, 1 << vm_id);
1007
1008 /* wait for flush */
1009 amdgpu_ring_write(ring, VCE_CMD_REG_WAIT);
1010 amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
1011 amdgpu_ring_write(ring, 1 << vm_id);
1012 amdgpu_ring_write(ring, 1 << vm_id);
1013 }
1014} 949}
1015 950
1016static int vce_v4_0_set_interrupt_state(struct amdgpu_device *adev, 951static int vce_v4_0_set_interrupt_state(struct amdgpu_device *adev,
@@ -1078,12 +1013,13 @@ static const struct amdgpu_ring_funcs vce_v4_0_ring_vm_funcs = {
1078 .align_mask = 0x3f, 1013 .align_mask = 0x3f,
1079 .nop = VCE_CMD_NO_OP, 1014 .nop = VCE_CMD_NO_OP,
1080 .support_64bit_ptrs = false, 1015 .support_64bit_ptrs = false,
1016 .vmhub = AMDGPU_MMHUB,
1081 .get_rptr = vce_v4_0_ring_get_rptr, 1017 .get_rptr = vce_v4_0_ring_get_rptr,
1082 .get_wptr = vce_v4_0_ring_get_wptr, 1018 .get_wptr = vce_v4_0_ring_get_wptr,
1083 .set_wptr = vce_v4_0_ring_set_wptr, 1019 .set_wptr = vce_v4_0_ring_set_wptr,
1084 .parse_cs = amdgpu_vce_ring_parse_cs_vm, 1020 .parse_cs = amdgpu_vce_ring_parse_cs_vm,
1085 .emit_frame_size = 1021 .emit_frame_size =
1086 17 * AMDGPU_MAX_VMHUBS + /* vce_v4_0_emit_vm_flush */ 1022 17 + /* vce_v4_0_emit_vm_flush */
1087 5 + 5 + /* amdgpu_vce_ring_emit_fence x2 vm fence */ 1023 5 + 5 + /* amdgpu_vce_ring_emit_fence x2 vm fence */
1088 1, /* vce_v4_0_ring_insert_end */ 1024 1, /* vce_v4_0_ring_insert_end */
1089 .emit_ib_size = 5, /* vce_v4_0_ring_emit_ib */ 1025 .emit_ib_size = 5, /* vce_v4_0_ring_emit_ib */
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-06 13:28:47 -0500