aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDave Airlie <airlied@redhat.com>2017-04-06 15:41:42 -0400
committerDave Airlie <airlied@redhat.com>2017-04-06 15:49:12 -0400
commit0168778115687486575a6831df865dbc4f5369fe (patch)
tree29a3f1e3348f1ddbc9924611fa0cad738507ba64
parentaed93ee7d03eac9b7d21f08aebe8a7d9ea069e20 (diff)
parentf4e7c7c1b4ed4c28caf679bc94ca5aa096310c10 (diff)
Merge branch 'drm-next-4.12' of git://people.freedesktop.org/~agd5f/linux into drm-next
A few more things for 4.12: - ttm and amdgpu support for non-contiguous vram CPU mappings - lots of bug fixes and cleanups for vega10 - misc bug fixes and code cleanups [airlied: fix do_div error on 32-bit arm, not sure it's 100% correct] * 'drm-next-4.12' of git://people.freedesktop.org/~agd5f/linux: (58 commits) drm/amdgpu: use uintptr_t instead of unsigned long to store pointer drm/amdgpu: Avoid using signed integer to store pointer value drm/amdgpu:invoke new implemented AI MB func drm/amdgpu/vega10:timeout set to equal with VI drm/amdgpu:implement the reset MB func for vega10 drm/amdgpu:fix typo for mxgpu_ai drm/amdgpu:no need to involv HDP in KIQ drm/amdgpu:add PSP block only load_type=PSP (v2) drm/amdgpu/smu9: update to latest driver interface drm/amd/amdgpu: cleanup gfx_v9_0_gpu_init() drm/amd/amdgpu: cleanup gfx_v9_0_rlc_reset() drm/amd/amdgpu: cleanup gfx_v9_0_rlc_start() drm/amd/amdgpu: simplify gfx_v9_0_cp_gfx_enable() drm/amd/amdgpu: cleanup gfx_v9_0_kiq_init_register() drm/amd/amdgpu: Drop gfx_v9_0_print_status() drm/amd/amdgpu: cleanup gfx_v9_0_set_gfx_eop_interrupt_state() drm/amd/amdgpu: cleanup gfx_v9_0_set_priv_reg_fault_state() drm/amd/amdgpu: cleanup gfx_v9_0_set_priv_inst_fault_state() drm/amd/amdgpu: cleanup gfx_v9_0_init_queue() drm/amdgpu: Move function amdgpu_has_atpx near other similar functions ...
Diffstat
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu.h31
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c29
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c20
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_device.c64
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.c15
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h12
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c40
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c230
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h13
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c17
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c82
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c274
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c35
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c96
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c96
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c90
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c35
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c137
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/psp_v3_1.c14
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/soc15.c27
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c18
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c17
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c21
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vce_v2_0.c18
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vce_v3_0.c18
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vce_v4_0.c20
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vid.h82
-rw-r--r--drivers/gpu/drm/amd/powerplay/amd_powerplay.c12
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/smu9_driver_if.h57
-rw-r--r--drivers/gpu/drm/ast/ast_ttm.c1
-rw-r--r--drivers/gpu/drm/bochs/bochs_mm.c1
-rw-r--r--drivers/gpu/drm/cirrus/cirrus_ttm.c1
-rw-r--r--drivers/gpu/drm/mgag200/mgag200_ttm.c1
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_bo.c1
-rw-r--r--drivers/gpu/drm/qxl/qxl_ttm.c1
-rw-r--r--drivers/gpu/drm/radeon/radeon_ttm.c1
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo.c49
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo_vm.c10
-rw-r--r--drivers/gpu/drm/virtio/virtgpu_ttm.c1
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c1
-rw-r--r--include/drm/ttm/ttm_bo_api.h11
-rw-r--r--include/drm/ttm/ttm_bo_driver.h9
-rw-r--r--include/drm/ttm/ttm_placement.h1
57 files changed, 900 insertions, 883 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 262056778f52..6a8129949333 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -32,7 +32,7 @@
32#include <linux/wait.h> 32#include <linux/wait.h>
33#include <linux/list.h> 33#include <linux/list.h>
34#include <linux/kref.h> 34#include <linux/kref.h>
35#include <linux/interval_tree.h> 35#include <linux/rbtree.h>
36#include <linux/hashtable.h> 36#include <linux/hashtable.h>
37#include <linux/dma-fence.h> 37#include <linux/dma-fence.h>
38 38
@@ -122,14 +122,6 @@ extern int amdgpu_param_buf_per_se;
122/* max number of IP instances */ 122/* max number of IP instances */
123#define AMDGPU_MAX_SDMA_INSTANCES 2 123#define AMDGPU_MAX_SDMA_INSTANCES 2
124 124
125/* max number of VMHUB */
126#define AMDGPU_MAX_VMHUBS 2
127#define AMDGPU_MMHUB 0
128#define AMDGPU_GFXHUB 1
129
130/* hardcode that limit for now */
131#define AMDGPU_VA_RESERVED_SIZE (8 << 20)
132
133/* hard reset data */ 125/* hard reset data */
134#define AMDGPU_ASIC_RESET_DATA 0x39d5e86b 126#define AMDGPU_ASIC_RESET_DATA 0x39d5e86b
135 127
@@ -312,12 +304,9 @@ struct amdgpu_gart_funcs {
312 /* set pte flags based per asic */ 304 /* set pte flags based per asic */
313 uint64_t (*get_vm_pte_flags)(struct amdgpu_device *adev, 305 uint64_t (*get_vm_pte_flags)(struct amdgpu_device *adev,
314 uint32_t flags); 306 uint32_t flags);
315};
316
317/* provided by the mc block */
318struct amdgpu_mc_funcs {
319 /* adjust mc addr in fb for APU case */ 307 /* adjust mc addr in fb for APU case */
320 u64 (*adjust_mc_addr)(struct amdgpu_device *adev, u64 addr); 308 u64 (*adjust_mc_addr)(struct amdgpu_device *adev, u64 addr);
309 uint32_t (*get_invalidate_req)(unsigned int vm_id);
321}; 310};
322 311
323/* provided by the ih block */ 312/* provided by the ih block */
@@ -379,7 +368,10 @@ struct amdgpu_bo_list_entry {
379 368
380struct amdgpu_bo_va_mapping { 369struct amdgpu_bo_va_mapping {
381 struct list_head list; 370 struct list_head list;
382 struct interval_tree_node it; 371 struct rb_node rb;
372 uint64_t start;
373 uint64_t last;
374 uint64_t __subtree_last;
383 uint64_t offset; 375 uint64_t offset;
384 uint64_t flags; 376 uint64_t flags;
385}; 377};
@@ -579,8 +571,6 @@ struct amdgpu_vmhub {
579 uint32_t vm_context0_cntl; 571 uint32_t vm_context0_cntl;
580 uint32_t vm_l2_pro_fault_status; 572 uint32_t vm_l2_pro_fault_status;
581 uint32_t vm_l2_pro_fault_cntl; 573 uint32_t vm_l2_pro_fault_cntl;
582 uint32_t (*get_invalidate_req)(unsigned int vm_id);
583 uint32_t (*get_vm_protection_bits)(void);
584}; 574};
585 575
586/* 576/*
@@ -618,7 +608,6 @@ struct amdgpu_mc {
618 u64 private_aperture_end; 608 u64 private_aperture_end;
619 /* protects concurrent invalidation */ 609 /* protects concurrent invalidation */
620 spinlock_t invalidate_lock; 610 spinlock_t invalidate_lock;
621 const struct amdgpu_mc_funcs *mc_funcs;
622}; 611};
623 612
624/* 613/*
@@ -1712,6 +1701,12 @@ void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);
1712#define WREG32_FIELD(reg, field, val) \ 1701#define WREG32_FIELD(reg, field, val) \
1713 WREG32(mm##reg, (RREG32(mm##reg) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field)) 1702 WREG32(mm##reg, (RREG32(mm##reg) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
1714 1703
1704#define WREG32_FIELD_OFFSET(reg, offset, field, val) \
1705 WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
1706
1707#define WREG32_FIELD15(ip, idx, reg, field, val) \
1708 WREG32(SOC15_REG_OFFSET(ip, idx, mm##reg), (RREG32(SOC15_REG_OFFSET(ip, idx, mm##reg)) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
1709
1715/* 1710/*
1716 * BIOS helpers. 1711 * BIOS helpers.
1717 */ 1712 */
@@ -1887,12 +1882,14 @@ void amdgpu_unregister_atpx_handler(void);
1887bool amdgpu_has_atpx_dgpu_power_cntl(void); 1882bool amdgpu_has_atpx_dgpu_power_cntl(void);
1888bool amdgpu_is_atpx_hybrid(void); 1883bool amdgpu_is_atpx_hybrid(void);
1889bool amdgpu_atpx_dgpu_req_power_for_displays(void); 1884bool amdgpu_atpx_dgpu_req_power_for_displays(void);
1885bool amdgpu_has_atpx(void);
1890#else 1886#else
1891static inline void amdgpu_register_atpx_handler(void) {} 1887static inline void amdgpu_register_atpx_handler(void) {}
1892static inline void amdgpu_unregister_atpx_handler(void) {} 1888static inline void amdgpu_unregister_atpx_handler(void) {}
1893static inline bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false; } 1889static inline bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false; }
1894static inline bool amdgpu_is_atpx_hybrid(void) { return false; } 1890static inline bool amdgpu_is_atpx_hybrid(void) { return false; }
1895static inline bool amdgpu_atpx_dgpu_req_power_for_displays(void) { return false; } 1891static inline bool amdgpu_atpx_dgpu_req_power_for_displays(void) { return false; }
1892static inline bool amdgpu_has_atpx(void) { return false; }
1896#endif 1893#endif
1897 1894
1898/* 1895/*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index f52b1bf3d3d9..ad4329922f79 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -754,6 +754,35 @@ union igp_info {
754 struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_9 info_9; 754 struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_9 info_9;
755}; 755};
756 756
757/*
758 * Return vram width from integrated system info table, if available,
759 * or 0 if not.
760 */
761int amdgpu_atombios_get_vram_width(struct amdgpu_device *adev)
762{
763 struct amdgpu_mode_info *mode_info = &adev->mode_info;
764 int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
765 u16 data_offset, size;
766 union igp_info *igp_info;
767 u8 frev, crev;
768
769 /* get any igp specific overrides */
770 if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, &size,
771 &frev, &crev, &data_offset)) {
772 igp_info = (union igp_info *)
773 (mode_info->atom_context->bios + data_offset);
774 switch (crev) {
775 case 8:
776 case 9:
777 return igp_info->info_8.ucUMAChannelNumber * 64;
778 default:
779 return 0;
780 }
781 }
782
783 return 0;
784}
785
757static void amdgpu_atombios_get_igp_ss_overrides(struct amdgpu_device *adev, 786static void amdgpu_atombios_get_igp_ss_overrides(struct amdgpu_device *adev,
758 struct amdgpu_atom_ss *ss, 787 struct amdgpu_atom_ss *ss,
759 int id) 788 int id)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
index 4e0f488487f3..38d0fe32e5cd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
@@ -148,6 +148,8 @@ int amdgpu_atombios_get_clock_info(struct amdgpu_device *adev);
148 148
149int amdgpu_atombios_get_gfx_info(struct amdgpu_device *adev); 149int amdgpu_atombios_get_gfx_info(struct amdgpu_device *adev);
150 150
151int amdgpu_atombios_get_vram_width(struct amdgpu_device *adev);
152
151bool amdgpu_atombios_get_asic_ss_info(struct amdgpu_device *adev, 153bool amdgpu_atombios_get_asic_ss_info(struct amdgpu_device *adev,
152 struct amdgpu_atom_ss *ss, 154 struct amdgpu_atom_ss *ss,
153 int id, u32 clock); 155 int id, u32 clock);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
index 0218cea6be4d..a6649874e6ce 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
@@ -237,7 +237,7 @@ int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
237 struct amdgpu_fpriv *fpriv = filp->driver_priv; 237 struct amdgpu_fpriv *fpriv = filp->driver_priv;
238 union drm_amdgpu_bo_list *args = data; 238 union drm_amdgpu_bo_list *args = data;
239 uint32_t handle = args->in.list_handle; 239 uint32_t handle = args->in.list_handle;
240 const void __user *uptr = (const void*)(long)args->in.bo_info_ptr; 240 const void __user *uptr = (const void*)(uintptr_t)args->in.bo_info_ptr;
241 241
242 struct drm_amdgpu_bo_list_entry *info; 242 struct drm_amdgpu_bo_list_entry *info;
243 struct amdgpu_bo_list *list; 243 struct amdgpu_bo_list *list;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 97f661372a1c..9cd4e1e05b97 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -161,7 +161,7 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
161 } 161 }
162 162
163 /* get chunks */ 163 /* get chunks */
164 chunk_array_user = (uint64_t __user *)(unsigned long)(cs->in.chunks); 164 chunk_array_user = (uint64_t __user *)(uintptr_t)(cs->in.chunks);
165 if (copy_from_user(chunk_array, chunk_array_user, 165 if (copy_from_user(chunk_array, chunk_array_user,
166 sizeof(uint64_t)*cs->in.num_chunks)) { 166 sizeof(uint64_t)*cs->in.num_chunks)) {
167 ret = -EFAULT; 167 ret = -EFAULT;
@@ -181,7 +181,7 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
181 struct drm_amdgpu_cs_chunk user_chunk; 181 struct drm_amdgpu_cs_chunk user_chunk;
182 uint32_t __user *cdata; 182 uint32_t __user *cdata;
183 183
184 chunk_ptr = (void __user *)(unsigned long)chunk_array[i]; 184 chunk_ptr = (void __user *)(uintptr_t)chunk_array[i];
185 if (copy_from_user(&user_chunk, chunk_ptr, 185 if (copy_from_user(&user_chunk, chunk_ptr,
186 sizeof(struct drm_amdgpu_cs_chunk))) { 186 sizeof(struct drm_amdgpu_cs_chunk))) {
187 ret = -EFAULT; 187 ret = -EFAULT;
@@ -192,7 +192,7 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
192 p->chunks[i].length_dw = user_chunk.length_dw; 192 p->chunks[i].length_dw = user_chunk.length_dw;
193 193
194 size = p->chunks[i].length_dw; 194 size = p->chunks[i].length_dw;
195 cdata = (void __user *)(unsigned long)user_chunk.chunk_data; 195 cdata = (void __user *)(uintptr_t)user_chunk.chunk_data;
196 196
197 p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t)); 197 p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
198 if (p->chunks[i].kdata == NULL) { 198 if (p->chunks[i].kdata == NULL) {
@@ -949,7 +949,7 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
949 } 949 }
950 950
951 if ((chunk_ib->va_start + chunk_ib->ib_bytes) > 951 if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
952 (m->it.last + 1) * AMDGPU_GPU_PAGE_SIZE) { 952 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
953 DRM_ERROR("IB va_start+ib_bytes is invalid\n"); 953 DRM_ERROR("IB va_start+ib_bytes is invalid\n");
954 return -EINVAL; 954 return -EINVAL;
955 } 955 }
@@ -960,7 +960,7 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
960 return r; 960 return r;
961 } 961 }
962 962
963 offset = ((uint64_t)m->it.start) * AMDGPU_GPU_PAGE_SIZE; 963 offset = m->start * AMDGPU_GPU_PAGE_SIZE;
964 kptr += chunk_ib->va_start - offset; 964 kptr += chunk_ib->va_start - offset;
965 965
966 r = amdgpu_ib_get(adev, vm, chunk_ib->ib_bytes, ib); 966 r = amdgpu_ib_get(adev, vm, chunk_ib->ib_bytes, ib);
@@ -1339,7 +1339,7 @@ int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
1339 if (fences == NULL) 1339 if (fences == NULL)
1340 return -ENOMEM; 1340 return -ENOMEM;
1341 1341
1342 fences_user = (void __user *)(unsigned long)(wait->in.fences); 1342 fences_user = (void __user *)(uintptr_t)(wait->in.fences);
1343 if (copy_from_user(fences, fences_user, 1343 if (copy_from_user(fences, fences_user,
1344 sizeof(struct drm_amdgpu_fence) * fence_count)) { 1344 sizeof(struct drm_amdgpu_fence) * fence_count)) {
1345 r = -EFAULT; 1345 r = -EFAULT;
@@ -1388,8 +1388,8 @@ amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1388 continue; 1388 continue;
1389 1389
1390 list_for_each_entry(mapping, &lobj->bo_va->valids, list) { 1390 list_for_each_entry(mapping, &lobj->bo_va->valids, list) {
1391 if (mapping->it.start > addr || 1391 if (mapping->start > addr ||
1392 addr > mapping->it.last) 1392 addr > mapping->last)
1393 continue; 1393 continue;
1394 1394
1395 *bo = lobj->bo_va->bo; 1395 *bo = lobj->bo_va->bo;
@@ -1397,8 +1397,8 @@ amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1397 } 1397 }
1398 1398
1399 list_for_each_entry(mapping, &lobj->bo_va->invalids, list) { 1399 list_for_each_entry(mapping, &lobj->bo_va->invalids, list) {
1400 if (mapping->it.start > addr || 1400 if (mapping->start > addr ||
1401 addr > mapping->it.last) 1401 addr > mapping->last)
1402 continue; 1402 continue;
1403 1403
1404 *bo = lobj->bo_va->bo; 1404 *bo = lobj->bo_va->bo;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 83dda05325b8..831c2bfd2072 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -1042,14 +1042,6 @@ static bool amdgpu_check_pot_argument(int arg)
1042 1042
1043static void amdgpu_get_block_size(struct amdgpu_device *adev) 1043static void amdgpu_get_block_size(struct amdgpu_device *adev)
1044{ 1044{
1045 /* from AI, asic starts to support multiple level VMPT */
1046 if (adev->asic_type >= CHIP_VEGA10) {
1047 if (amdgpu_vm_block_size != 9)
1048 dev_warn(adev->dev,
1049 "Multi-VMPT limits block size to one page!\n");
1050 amdgpu_vm_block_size = 9;
1051 return;
1052 }
1053 /* defines number of bits in page table versus page directory, 1045 /* defines number of bits in page table versus page directory,
1054 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1046 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1055 * page table and the remaining bits are in the page directory */ 1047 * page table and the remaining bits are in the page directory */
@@ -1079,6 +1071,36 @@ static void amdgpu_get_block_size(struct amdgpu_device *adev)
1079 } 1071 }
1080} 1072}
1081 1073
1074static void amdgpu_check_vm_size(struct amdgpu_device *adev)
1075{
1076 if (!amdgpu_check_pot_argument(amdgpu_vm_size)) {
1077 dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
1078 amdgpu_vm_size);
1079 goto def_value;
1080 }
1081
1082 if (amdgpu_vm_size < 1) {
1083 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1084 amdgpu_vm_size);
1085 goto def_value;
1086 }
1087
1088 /*
1089 * Max GPUVM size for Cayman, SI, CI VI are 40 bits.
1090 */
1091 if (amdgpu_vm_size > 1024) {
1092 dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
1093 amdgpu_vm_size);
1094 goto def_value;
1095 }
1096
1097 return;
1098
1099def_value:
1100 amdgpu_vm_size = 8;
1101 dev_info(adev->dev, "set default VM size %dGB\n", amdgpu_vm_size);
1102}
1103
1082/** 1104/**
1083 * amdgpu_check_arguments - validate module params 1105 * amdgpu_check_arguments - validate module params
1084 * 1106 *
@@ -1108,26 +1130,7 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
1108 } 1130 }
1109 } 1131 }
1110 1132
1111 if (!amdgpu_check_pot_argument(amdgpu_vm_size)) { 1133 amdgpu_check_vm_size(adev);
1112 dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
1113 amdgpu_vm_size);
1114 amdgpu_vm_size = 8;
1115 }
1116
1117 if (amdgpu_vm_size < 1) {
1118 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1119 amdgpu_vm_size);
1120 amdgpu_vm_size = 8;
1121 }
1122
1123 /*
1124 * Max GPUVM size for Cayman, SI and CI are 40 bits.
1125 */
1126 if (amdgpu_vm_size > 1024) {
1127 dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
1128 amdgpu_vm_size);
1129 amdgpu_vm_size = 8;
1130 }
1131 1134
1132 amdgpu_get_block_size(adev); 1135 amdgpu_get_block_size(adev);
1133 1136
@@ -2249,9 +2252,10 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2249 } 2252 }
2250 2253
2251 r = amdgpu_resume(adev); 2254 r = amdgpu_resume(adev);
2252 if (r) 2255 if (r) {
2253 DRM_ERROR("amdgpu_resume failed (%d).\n", r); 2256 DRM_ERROR("amdgpu_resume failed (%d).\n", r);
2254 2257 return r;
2258 }
2255 amdgpu_fence_driver_resume(adev); 2259 amdgpu_fence_driver_resume(adev);
2256 2260
2257 if (resume) { 2261 if (resume) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index f85520d4e711..bb710e0ace01 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -717,7 +717,7 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
717 switch (args->op) { 717 switch (args->op) {
718 case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: { 718 case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
719 struct drm_amdgpu_gem_create_in info; 719 struct drm_amdgpu_gem_create_in info;
720 void __user *out = (void __user *)(long)args->value; 720 void __user *out = (void __user *)(uintptr_t)args->value;
721 721
722 info.bo_size = robj->gem_base.size; 722 info.bo_size = robj->gem_base.size;
723 info.alignment = robj->tbo.mem.page_alignment << PAGE_SHIFT; 723 info.alignment = robj->tbo.mem.page_alignment << PAGE_SHIFT;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 13b487235a8b..a6b7e367a860 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -316,9 +316,10 @@ int amdgpu_irq_add_id(struct amdgpu_device *adev,
316 return -EINVAL; 316 return -EINVAL;
317 317
318 if (!adev->irq.client[client_id].sources) { 318 if (!adev->irq.client[client_id].sources) {
319 adev->irq.client[client_id].sources = kcalloc(AMDGPU_MAX_IRQ_SRC_ID, 319 adev->irq.client[client_id].sources =
320 sizeof(struct amdgpu_irq_src), 320 kcalloc(AMDGPU_MAX_IRQ_SRC_ID,
321 GFP_KERNEL); 321 sizeof(struct amdgpu_irq_src *),
322 GFP_KERNEL);
322 if (!adev->irq.client[client_id].sources) 323 if (!adev->irq.client[client_id].sources)
323 return -ENOMEM; 324 return -ENOMEM;
324 } 325 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index dfb029ab3448..832be632478f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -36,12 +36,6 @@
36#include <linux/pm_runtime.h> 36#include <linux/pm_runtime.h>
37#include "amdgpu_amdkfd.h" 37#include "amdgpu_amdkfd.h"
38 38
39#if defined(CONFIG_VGA_SWITCHEROO)
40bool amdgpu_has_atpx(void);
41#else
42static inline bool amdgpu_has_atpx(void) { return false; }
43#endif
44
45/** 39/**
46 * amdgpu_driver_unload_kms - Main unload function for KMS. 40 * amdgpu_driver_unload_kms - Main unload function for KMS.
47 * 41 *
@@ -243,7 +237,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
243 struct amdgpu_device *adev = dev->dev_private; 237 struct amdgpu_device *adev = dev->dev_private;
244 struct drm_amdgpu_info *info = data; 238 struct drm_amdgpu_info *info = data;
245 struct amdgpu_mode_info *minfo = &adev->mode_info; 239 struct amdgpu_mode_info *minfo = &adev->mode_info;
246 void __user *out = (void __user *)(long)info->return_pointer; 240 void __user *out = (void __user *)(uintptr_t)info->return_pointer;
247 uint32_t size = info->return_size; 241 uint32_t size = info->return_size;
248 struct drm_crtc *crtc; 242 struct drm_crtc *crtc;
249 uint32_t ui32 = 0; 243 uint32_t ui32 = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
index 7ea3cacf9f9f..38f739fb727b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
@@ -31,6 +31,7 @@
31#include <linux/firmware.h> 31#include <linux/firmware.h>
32#include <linux/module.h> 32#include <linux/module.h>
33#include <linux/mmu_notifier.h> 33#include <linux/mmu_notifier.h>
34#include <linux/interval_tree.h>
34#include <drm/drmP.h> 35#include <drm/drmP.h>
35#include <drm/drm.h> 36#include <drm/drm.h>
36 37
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 5aac350b007f..758d8cf8fd17 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -122,20 +122,19 @@ static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
122 122
123 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 123 if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
124 unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT; 124 unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
125 unsigned lpfn = 0;
126
127 /* This forces a reallocation if the flag wasn't set before */
128 if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
129 lpfn = adev->mc.real_vram_size >> PAGE_SHIFT;
130 125
131 places[c].fpfn = 0; 126 places[c].fpfn = 0;
132 places[c].lpfn = lpfn; 127 places[c].lpfn = 0;
133 places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | 128 places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
134 TTM_PL_FLAG_VRAM; 129 TTM_PL_FLAG_VRAM;
130
135 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) 131 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
136 places[c].lpfn = visible_pfn; 132 places[c].lpfn = visible_pfn;
137 else 133 else
138 places[c].flags |= TTM_PL_FLAG_TOPDOWN; 134 places[c].flags |= TTM_PL_FLAG_TOPDOWN;
135
136 if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
137 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
139 c++; 138 c++;
140 } 139 }
141 140
@@ -928,8 +927,7 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
928 size = bo->mem.num_pages << PAGE_SHIFT; 927 size = bo->mem.num_pages << PAGE_SHIFT;
929 offset = bo->mem.start << PAGE_SHIFT; 928 offset = bo->mem.start << PAGE_SHIFT;
930 /* TODO: figure out how to map scattered VRAM to the CPU */ 929 /* TODO: figure out how to map scattered VRAM to the CPU */
931 if ((offset + size) <= adev->mc.visible_vram_size && 930 if ((offset + size) <= adev->mc.visible_vram_size)
932 (abo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS))
933 return 0; 931 return 0;
934 932
935 /* Can't move a pinned BO to visible VRAM */ 933 /* Can't move a pinned BO to visible VRAM */
@@ -937,7 +935,6 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
937 return -EINVAL; 935 return -EINVAL;
938 936
939 /* hurrah the memory is not visible ! */ 937 /* hurrah the memory is not visible ! */
940 abo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
941 amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM); 938 amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM);
942 lpfn = adev->mc.visible_vram_size >> PAGE_SHIFT; 939 lpfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
943 for (i = 0; i < abo->placement.num_placement; i++) { 940 for (i = 0; i < abo->placement.num_placement; i++) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 4731015f6101..ed6e5799016e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -130,7 +130,7 @@ psp_cmd_submit_buf(struct psp_context *psp,
130 130
131 while (*((unsigned int *)psp->fence_buf) != index) { 131 while (*((unsigned int *)psp->fence_buf) != index) {
132 msleep(1); 132 msleep(1);
133 }; 133 }
134 134
135 amdgpu_bo_free_kernel(&cmd_buf_bo, 135 amdgpu_bo_free_kernel(&cmd_buf_bo,
136 &cmd_buf_mc_addr, 136 &cmd_buf_mc_addr,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index a87de18160a8..ee9d0f346d75 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -226,8 +226,8 @@ TRACE_EVENT(amdgpu_vm_bo_map,
226 226
227 TP_fast_assign( 227 TP_fast_assign(
228 __entry->bo = bo_va ? bo_va->bo : NULL; 228 __entry->bo = bo_va ? bo_va->bo : NULL;
229 __entry->start = mapping->it.start; 229 __entry->start = mapping->start;
230 __entry->last = mapping->it.last; 230 __entry->last = mapping->last;
231 __entry->offset = mapping->offset; 231 __entry->offset = mapping->offset;
232 __entry->flags = mapping->flags; 232 __entry->flags = mapping->flags;
233 ), 233 ),
@@ -250,8 +250,8 @@ TRACE_EVENT(amdgpu_vm_bo_unmap,
250 250
251 TP_fast_assign( 251 TP_fast_assign(
252 __entry->bo = bo_va->bo; 252 __entry->bo = bo_va->bo;
253 __entry->start = mapping->it.start; 253 __entry->start = mapping->start;
254 __entry->last = mapping->it.last; 254 __entry->last = mapping->last;
255 __entry->offset = mapping->offset; 255 __entry->offset = mapping->offset;
256 __entry->flags = mapping->flags; 256 __entry->flags = mapping->flags;
257 ), 257 ),
@@ -270,8 +270,8 @@ DECLARE_EVENT_CLASS(amdgpu_vm_mapping,
270 ), 270 ),
271 271
272 TP_fast_assign( 272 TP_fast_assign(
273 __entry->soffset = mapping->it.start; 273 __entry->soffset = mapping->start;
274 __entry->eoffset = mapping->it.last + 1; 274 __entry->eoffset = mapping->last + 1;
275 __entry->flags = mapping->flags; 275 __entry->flags = mapping->flags;
276 ), 276 ),
277 TP_printk("soffs=%010llx, eoffs=%010llx, flags=%08x", 277 TP_printk("soffs=%010llx, eoffs=%010llx, flags=%08x",
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 244bb9aacf86..7e993259c2e9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -529,40 +529,12 @@ static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_
529 case TTM_PL_TT: 529 case TTM_PL_TT:
530 break; 530 break;
531 case TTM_PL_VRAM: 531 case TTM_PL_VRAM:
532 if (mem->start == AMDGPU_BO_INVALID_OFFSET)
533 return -EINVAL;
534
535 mem->bus.offset = mem->start << PAGE_SHIFT; 532 mem->bus.offset = mem->start << PAGE_SHIFT;
536 /* check if it's visible */ 533 /* check if it's visible */
537 if ((mem->bus.offset + mem->bus.size) > adev->mc.visible_vram_size) 534 if ((mem->bus.offset + mem->bus.size) > adev->mc.visible_vram_size)
538 return -EINVAL; 535 return -EINVAL;
539 mem->bus.base = adev->mc.aper_base; 536 mem->bus.base = adev->mc.aper_base;
540 mem->bus.is_iomem = true; 537 mem->bus.is_iomem = true;
541#ifdef __alpha__
542 /*
543 * Alpha: use bus.addr to hold the ioremap() return,
544 * so we can modify bus.base below.
545 */
546 if (mem->placement & TTM_PL_FLAG_WC)
547 mem->bus.addr =
548 ioremap_wc(mem->bus.base + mem->bus.offset,
549 mem->bus.size);
550 else
551 mem->bus.addr =
552 ioremap_nocache(mem->bus.base + mem->bus.offset,
553 mem->bus.size);
554 if (!mem->bus.addr)
555 return -ENOMEM;
556
557 /*
558 * Alpha: Use just the bus offset plus
559 * the hose/domain memory base for bus.base.
560 * It then can be used to build PTEs for VRAM
561 * access, as done in ttm_bo_vm_fault().
562 */
563 mem->bus.base = (mem->bus.base & 0x0ffffffffUL) +
564 adev->ddev->hose->dense_mem_base;
565#endif
566 break; 538 break;
567 default: 539 default:
568 return -EINVAL; 540 return -EINVAL;
@@ -574,6 +546,17 @@ static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_re
574{ 546{
575} 547}
576 548
549static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
550 unsigned long page_offset)
551{
552 struct drm_mm_node *mm = bo->mem.mm_node;
553 uint64_t size = mm->size;
554 uint64_t offset = page_offset;
555
556 page_offset = do_div(offset, size);
557 return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start + page_offset;
558}
559
577/* 560/*
578 * TTM backend functions. 561 * TTM backend functions.
579 */ 562 */
@@ -1089,6 +1072,7 @@ static struct ttm_bo_driver amdgpu_bo_driver = {
1089 .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify, 1072 .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
1090 .io_mem_reserve = &amdgpu_ttm_io_mem_reserve, 1073 .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
1091 .io_mem_free = &amdgpu_ttm_io_mem_free, 1074 .io_mem_free = &amdgpu_ttm_io_mem_free,
1075 .io_mem_pfn = amdgpu_ttm_io_mem_pfn,
1092}; 1076};
1093 1077
1094int amdgpu_ttm_init(struct amdgpu_device *adev) 1078int amdgpu_ttm_init(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index 0b92dd0c1d70..2ca09f111f08 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -741,10 +741,10 @@ static int amdgpu_uvd_cs_pass2(struct amdgpu_uvd_cs_ctx *ctx)
741 741
742 start = amdgpu_bo_gpu_offset(bo); 742 start = amdgpu_bo_gpu_offset(bo);
743 743
744 end = (mapping->it.last + 1 - mapping->it.start); 744 end = (mapping->last + 1 - mapping->start);
745 end = end * AMDGPU_GPU_PAGE_SIZE + start; 745 end = end * AMDGPU_GPU_PAGE_SIZE + start;
746 746
747 addr -= ((uint64_t)mapping->it.start) * AMDGPU_GPU_PAGE_SIZE; 747 addr -= mapping->start * AMDGPU_GPU_PAGE_SIZE;
748 start += addr; 748 start += addr;
749 749
750 amdgpu_set_ib_value(ctx->parser, ctx->ib_idx, ctx->data0, 750 amdgpu_set_ib_value(ctx->parser, ctx->ib_idx, ctx->data0,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 0184197eb000..c853400805d1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -595,13 +595,13 @@ static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx,
595 } 595 }
596 596
597 if ((addr + (uint64_t)size) > 597 if ((addr + (uint64_t)size) >
598 ((uint64_t)mapping->it.last + 1) * AMDGPU_GPU_PAGE_SIZE) { 598 (mapping->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
599 DRM_ERROR("BO to small for addr 0x%010Lx %d %d\n", 599 DRM_ERROR("BO to small for addr 0x%010Lx %d %d\n",
600 addr, lo, hi); 600 addr, lo, hi);
601 return -EINVAL; 601 return -EINVAL;
602 } 602 }
603 603
604 addr -= ((uint64_t)mapping->it.start) * AMDGPU_GPU_PAGE_SIZE; 604 addr -= mapping->start * AMDGPU_GPU_PAGE_SIZE;
605 addr += amdgpu_bo_gpu_offset(bo); 605 addr += amdgpu_bo_gpu_offset(bo);
606 addr -= ((uint64_t)size) * ((uint64_t)index); 606 addr -= ((uint64_t)size) * ((uint64_t)index);
607 607
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index ecef35a1fe33..ba8b8ae6234f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -122,9 +122,7 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
122 122
123 mutex_lock(&adev->virt.lock_kiq); 123 mutex_lock(&adev->virt.lock_kiq);
124 amdgpu_ring_alloc(ring, 32); 124 amdgpu_ring_alloc(ring, 32);
125 amdgpu_ring_emit_hdp_flush(ring);
126 amdgpu_ring_emit_rreg(ring, reg); 125 amdgpu_ring_emit_rreg(ring, reg);
127 amdgpu_ring_emit_hdp_invalidate(ring);
128 amdgpu_fence_emit(ring, &f); 126 amdgpu_fence_emit(ring, &f);
129 amdgpu_ring_commit(ring); 127 amdgpu_ring_commit(ring);
130 mutex_unlock(&adev->virt.lock_kiq); 128 mutex_unlock(&adev->virt.lock_kiq);
@@ -150,9 +148,7 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
150 148
151 mutex_lock(&adev->virt.lock_kiq); 149 mutex_lock(&adev->virt.lock_kiq);
152 amdgpu_ring_alloc(ring, 32); 150 amdgpu_ring_alloc(ring, 32);
153 amdgpu_ring_emit_hdp_flush(ring);
154 amdgpu_ring_emit_wreg(ring, reg, v); 151 amdgpu_ring_emit_wreg(ring, reg, v);
155 amdgpu_ring_emit_hdp_invalidate(ring);
156 amdgpu_fence_emit(ring, &f); 152 amdgpu_fence_emit(ring, &f);
157 amdgpu_ring_commit(ring); 153 amdgpu_ring_commit(ring);
158 mutex_unlock(&adev->virt.lock_kiq); 154 mutex_unlock(&adev->virt.lock_kiq);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 0235d7933efd..2895d9d86f29 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -26,6 +26,7 @@
26 * Jerome Glisse 26 * Jerome Glisse
27 */ 27 */
28#include <linux/dma-fence-array.h> 28#include <linux/dma-fence-array.h>
29#include <linux/interval_tree_generic.h>
29#include <drm/drmP.h> 30#include <drm/drmP.h>
30#include <drm/amdgpu_drm.h> 31#include <drm/amdgpu_drm.h>
31#include "amdgpu.h" 32#include "amdgpu.h"
@@ -51,6 +52,15 @@
51 * SI supports 16. 52 * SI supports 16.
52 */ 53 */
53 54
55#define START(node) ((node)->start)
56#define LAST(node) ((node)->last)
57
58INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
59 START, LAST, static, amdgpu_vm_it)
60
61#undef START
62#undef LAST
63
54/* Local structure. Encapsulate some VM table update parameters to reduce 64/* Local structure. Encapsulate some VM table update parameters to reduce
55 * the number of function parameters 65 * the number of function parameters
56 */ 66 */
@@ -90,13 +100,14 @@ static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
90 if (level == 0) 100 if (level == 0)
91 /* For the root directory */ 101 /* For the root directory */
92 return adev->vm_manager.max_pfn >> 102 return adev->vm_manager.max_pfn >>
93 (amdgpu_vm_block_size * adev->vm_manager.num_level); 103 (adev->vm_manager.block_size *
104 adev->vm_manager.num_level);
94 else if (level == adev->vm_manager.num_level) 105 else if (level == adev->vm_manager.num_level)
95 /* For the page tables on the leaves */ 106 /* For the page tables on the leaves */
96 return AMDGPU_VM_PTE_COUNT; 107 return AMDGPU_VM_PTE_COUNT(adev);
97 else 108 else
98 /* Everything in between */ 109 /* Everything in between */
99 return 1 << amdgpu_vm_block_size; 110 return 1 << adev->vm_manager.block_size;
100} 111}
101 112
102/** 113/**
@@ -261,7 +272,7 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
261 unsigned level) 272 unsigned level)
262{ 273{
263 unsigned shift = (adev->vm_manager.num_level - level) * 274 unsigned shift = (adev->vm_manager.num_level - level) *
264 amdgpu_vm_block_size; 275 adev->vm_manager.block_size;
265 unsigned pt_idx, from, to; 276 unsigned pt_idx, from, to;
266 int r; 277 int r;
267 278
@@ -365,11 +376,19 @@ int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
365 return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0); 376 return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0);
366} 377}
367 378
368static bool amdgpu_vm_is_gpu_reset(struct amdgpu_device *adev, 379/**
369 struct amdgpu_vm_id *id) 380 * amdgpu_vm_had_gpu_reset - check if reset occured since last use
381 *
382 * @adev: amdgpu_device pointer
383 * @id: VMID structure
384 *
385 * Check if GPU reset occured since last use of the VMID.
386 */
387static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device *adev,
388 struct amdgpu_vm_id *id)
370{ 389{
371 return id->current_gpu_reset_count != 390 return id->current_gpu_reset_count !=
372 atomic_read(&adev->gpu_reset_counter) ? true : false; 391 atomic_read(&adev->gpu_reset_counter);
373} 392}
374 393
375/** 394/**
@@ -455,7 +474,7 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
455 /* Check all the prerequisites to using this VMID */ 474 /* Check all the prerequisites to using this VMID */
456 if (!id) 475 if (!id)
457 continue; 476 continue;
458 if (amdgpu_vm_is_gpu_reset(adev, id)) 477 if (amdgpu_vm_had_gpu_reset(adev, id))
459 continue; 478 continue;
460 479
461 if (atomic64_read(&id->owner) != vm->client_id) 480 if (atomic64_read(&id->owner) != vm->client_id)
@@ -483,7 +502,6 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
483 if (r) 502 if (r)
484 goto error; 503 goto error;
485 504
486 id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
487 list_move_tail(&id->list, &adev->vm_manager.ids_lru); 505 list_move_tail(&id->list, &adev->vm_manager.ids_lru);
488 vm->ids[ring->idx] = id; 506 vm->ids[ring->idx] = id;
489 507
@@ -504,9 +522,6 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
504 if (r) 522 if (r)
505 goto error; 523 goto error;
506 524
507 dma_fence_put(id->first);
508 id->first = dma_fence_get(fence);
509
510 dma_fence_put(id->last_flush); 525 dma_fence_put(id->last_flush);
511 id->last_flush = NULL; 526 id->last_flush = NULL;
512 527
@@ -557,8 +572,8 @@ static u64 amdgpu_vm_adjust_mc_addr(struct amdgpu_device *adev, u64 mc_addr)
557{ 572{
558 u64 addr = mc_addr; 573 u64 addr = mc_addr;
559 574
560 if (adev->mc.mc_funcs && adev->mc.mc_funcs->adjust_mc_addr) 575 if (adev->gart.gart_funcs->adjust_mc_addr)
561 addr = adev->mc.mc_funcs->adjust_mc_addr(adev, addr); 576 addr = adev->gart.gart_funcs->adjust_mc_addr(adev, addr);
562 577
563 return addr; 578 return addr;
564} 579}
@@ -583,60 +598,62 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
583 id->gws_size != job->gws_size || 598 id->gws_size != job->gws_size ||
584 id->oa_base != job->oa_base || 599 id->oa_base != job->oa_base ||
585 id->oa_size != job->oa_size); 600 id->oa_size != job->oa_size);
601 bool vm_flush_needed = job->vm_needs_flush ||
602 amdgpu_vm_ring_has_compute_vm_bug(ring);
603 unsigned patch_offset = 0;
586 int r; 604 int r;
587 605
588 if (job->vm_needs_flush || gds_switch_needed || 606 if (amdgpu_vm_had_gpu_reset(adev, id)) {
589 amdgpu_vm_is_gpu_reset(adev, id) || 607 gds_switch_needed = true;
590 amdgpu_vm_ring_has_compute_vm_bug(ring)) { 608 vm_flush_needed = true;
591 unsigned patch_offset = 0; 609 }
592 610
593 if (ring->funcs->init_cond_exec) 611 if (!vm_flush_needed && !gds_switch_needed)
594 patch_offset = amdgpu_ring_init_cond_exec(ring); 612 return 0;
595 613
596 if (ring->funcs->emit_pipeline_sync && 614 if (ring->funcs->init_cond_exec)
597 (job->vm_needs_flush || gds_switch_needed || 615 patch_offset = amdgpu_ring_init_cond_exec(ring);
598 amdgpu_vm_ring_has_compute_vm_bug(ring)))
599 amdgpu_ring_emit_pipeline_sync(ring);
600 616
601 if (ring->funcs->emit_vm_flush && (job->vm_needs_flush || 617 if (ring->funcs->emit_pipeline_sync)
602 amdgpu_vm_is_gpu_reset(adev, id))) { 618 amdgpu_ring_emit_pipeline_sync(ring);
603 struct dma_fence *fence;
604 u64 pd_addr = amdgpu_vm_adjust_mc_addr(adev, job->vm_pd_addr);
605 619
606 trace_amdgpu_vm_flush(pd_addr, ring->idx, job->vm_id); 620 if (ring->funcs->emit_vm_flush && vm_flush_needed) {
607 amdgpu_ring_emit_vm_flush(ring, job->vm_id, pd_addr); 621 u64 pd_addr = amdgpu_vm_adjust_mc_addr(adev, job->vm_pd_addr);
622 struct dma_fence *fence;
608 623
609 r = amdgpu_fence_emit(ring, &fence); 624 trace_amdgpu_vm_flush(pd_addr, ring->idx, job->vm_id);
610 if (r) 625 amdgpu_ring_emit_vm_flush(ring, job->vm_id, pd_addr);
611 return r;
612 626
613 mutex_lock(&adev->vm_manager.lock); 627 r = amdgpu_fence_emit(ring, &fence);
614 dma_fence_put(id->last_flush); 628 if (r)
615 id->last_flush = fence; 629 return r;
616 mutex_unlock(&adev->vm_manager.lock);
617 }
618 630
619 if (gds_switch_needed) { 631 mutex_lock(&adev->vm_manager.lock);
620 id->gds_base = job->gds_base; 632 dma_fence_put(id->last_flush);
621 id->gds_size = job->gds_size; 633 id->last_flush = fence;
622 id->gws_base = job->gws_base; 634 mutex_unlock(&adev->vm_manager.lock);
623 id->gws_size = job->gws_size; 635 }
624 id->oa_base = job->oa_base;
625 id->oa_size = job->oa_size;
626 amdgpu_ring_emit_gds_switch(ring, job->vm_id,
627 job->gds_base, job->gds_size,
628 job->gws_base, job->gws_size,
629 job->oa_base, job->oa_size);
630 }
631 636
632 if (ring->funcs->patch_cond_exec) 637 if (gds_switch_needed) {
633 amdgpu_ring_patch_cond_exec(ring, patch_offset); 638 id->gds_base = job->gds_base;
639 id->gds_size = job->gds_size;
640 id->gws_base = job->gws_base;
641 id->gws_size = job->gws_size;
642 id->oa_base = job->oa_base;
643 id->oa_size = job->oa_size;
644 amdgpu_ring_emit_gds_switch(ring, job->vm_id, job->gds_base,
645 job->gds_size, job->gws_base,
646 job->gws_size, job->oa_base,
647 job->oa_size);
648 }
634 649
635 /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */ 650 if (ring->funcs->patch_cond_exec)
636 if (ring->funcs->emit_switch_buffer) { 651 amdgpu_ring_patch_cond_exec(ring, patch_offset);
637 amdgpu_ring_emit_switch_buffer(ring); 652
638 amdgpu_ring_emit_switch_buffer(ring); 653 /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
639 } 654 if (ring->funcs->emit_switch_buffer) {
655 amdgpu_ring_emit_switch_buffer(ring);
656 amdgpu_ring_emit_switch_buffer(ring);
640 } 657 }
641 return 0; 658 return 0;
642} 659}
@@ -960,7 +977,7 @@ static struct amdgpu_bo *amdgpu_vm_get_pt(struct amdgpu_pte_update_params *p,
960 unsigned idx, level = p->adev->vm_manager.num_level; 977 unsigned idx, level = p->adev->vm_manager.num_level;
961 978
962 while (entry->entries) { 979 while (entry->entries) {
963 idx = addr >> (amdgpu_vm_block_size * level--); 980 idx = addr >> (p->adev->vm_manager.block_size * level--);
964 idx %= amdgpu_bo_size(entry->bo) / 8; 981 idx %= amdgpu_bo_size(entry->bo) / 8;
965 entry = &entry->entries[idx]; 982 entry = &entry->entries[idx];
966 } 983 }
@@ -987,7 +1004,8 @@ static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
987 uint64_t start, uint64_t end, 1004 uint64_t start, uint64_t end,
988 uint64_t dst, uint64_t flags) 1005 uint64_t dst, uint64_t flags)
989{ 1006{
990 const uint64_t mask = AMDGPU_VM_PTE_COUNT - 1; 1007 struct amdgpu_device *adev = params->adev;
1008 const uint64_t mask = AMDGPU_VM_PTE_COUNT(adev) - 1;
991 1009
992 uint64_t cur_pe_start, cur_nptes, cur_dst; 1010 uint64_t cur_pe_start, cur_nptes, cur_dst;
993 uint64_t addr; /* next GPU address to be updated */ 1011 uint64_t addr; /* next GPU address to be updated */
@@ -1011,7 +1029,7 @@ static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
1011 if ((addr & ~mask) == (end & ~mask)) 1029 if ((addr & ~mask) == (end & ~mask))
1012 nptes = end - addr; 1030 nptes = end - addr;
1013 else 1031 else
1014 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask); 1032 nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
1015 1033
1016 cur_pe_start = amdgpu_bo_gpu_offset(pt); 1034 cur_pe_start = amdgpu_bo_gpu_offset(pt);
1017 cur_pe_start += (addr & mask) * 8; 1035 cur_pe_start += (addr & mask) * 8;
@@ -1039,7 +1057,7 @@ static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
1039 if ((addr & ~mask) == (end & ~mask)) 1057 if ((addr & ~mask) == (end & ~mask))
1040 nptes = end - addr; 1058 nptes = end - addr;
1041 else 1059 else
1042 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask); 1060 nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
1043 1061
1044 next_pe_start = amdgpu_bo_gpu_offset(pt); 1062 next_pe_start = amdgpu_bo_gpu_offset(pt);
1045 next_pe_start += (addr & mask) * 8; 1063 next_pe_start += (addr & mask) * 8;
@@ -1186,7 +1204,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1186 * reserve space for one command every (1 << BLOCK_SIZE) 1204 * reserve space for one command every (1 << BLOCK_SIZE)
1187 * entries or 2k dwords (whatever is smaller) 1205 * entries or 2k dwords (whatever is smaller)
1188 */ 1206 */
1189 ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1; 1207 ncmds = (nptes >> min(adev->vm_manager.block_size, 11u)) + 1;
1190 1208
1191 /* padding, etc. */ 1209 /* padding, etc. */
1192 ndw = 64; 1210 ndw = 64;
@@ -1301,7 +1319,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1301 struct drm_mm_node *nodes, 1319 struct drm_mm_node *nodes,
1302 struct dma_fence **fence) 1320 struct dma_fence **fence)
1303{ 1321{
1304 uint64_t pfn, src = 0, start = mapping->it.start; 1322 uint64_t pfn, src = 0, start = mapping->start;
1305 int r; 1323 int r;
1306 1324
1307 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here 1325 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
@@ -1353,7 +1371,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1353 } 1371 }
1354 addr += pfn << PAGE_SHIFT; 1372 addr += pfn << PAGE_SHIFT;
1355 1373
1356 last = min((uint64_t)mapping->it.last, start + max_entries - 1); 1374 last = min((uint64_t)mapping->last, start + max_entries - 1);
1357 r = amdgpu_vm_bo_update_mapping(adev, exclusive, 1375 r = amdgpu_vm_bo_update_mapping(adev, exclusive,
1358 src, pages_addr, vm, 1376 src, pages_addr, vm,
1359 start, last, flags, addr, 1377 start, last, flags, addr,
@@ -1368,7 +1386,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1368 } 1386 }
1369 start = last + 1; 1387 start = last + 1;
1370 1388
1371 } while (unlikely(start != mapping->it.last + 1)); 1389 } while (unlikely(start != mapping->last + 1));
1372 1390
1373 return 0; 1391 return 0;
1374} 1392}
@@ -1518,7 +1536,7 @@ static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1518 if (fence) 1536 if (fence)
1519 dma_fence_wait(fence, false); 1537 dma_fence_wait(fence, false);
1520 1538
1521 amdgpu_vm_prt_put(cb->adev); 1539 amdgpu_vm_prt_put(adev);
1522 } else { 1540 } else {
1523 cb->adev = adev; 1541 cb->adev = adev;
1524 if (!fence || dma_fence_add_callback(fence, &cb->cb, 1542 if (!fence || dma_fence_add_callback(fence, &cb->cb,
@@ -1724,9 +1742,8 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1724 uint64_t saddr, uint64_t offset, 1742 uint64_t saddr, uint64_t offset,
1725 uint64_t size, uint64_t flags) 1743 uint64_t size, uint64_t flags)
1726{ 1744{
1727 struct amdgpu_bo_va_mapping *mapping; 1745 struct amdgpu_bo_va_mapping *mapping, *tmp;
1728 struct amdgpu_vm *vm = bo_va->vm; 1746 struct amdgpu_vm *vm = bo_va->vm;
1729 struct interval_tree_node *it;
1730 uint64_t eaddr; 1747 uint64_t eaddr;
1731 1748
1732 /* validate the parameters */ 1749 /* validate the parameters */
@@ -1743,14 +1760,12 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1743 saddr /= AMDGPU_GPU_PAGE_SIZE; 1760 saddr /= AMDGPU_GPU_PAGE_SIZE;
1744 eaddr /= AMDGPU_GPU_PAGE_SIZE; 1761 eaddr /= AMDGPU_GPU_PAGE_SIZE;
1745 1762
1746 it = interval_tree_iter_first(&vm->va, saddr, eaddr); 1763 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1747 if (it) { 1764 if (tmp) {
1748 struct amdgpu_bo_va_mapping *tmp;
1749 tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
1750 /* bo and tmp overlap, invalid addr */ 1765 /* bo and tmp overlap, invalid addr */
1751 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " 1766 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1752 "0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr, 1767 "0x%010Lx-0x%010Lx\n", bo_va->bo, saddr, eaddr,
1753 tmp->it.start, tmp->it.last + 1); 1768 tmp->start, tmp->last + 1);
1754 return -EINVAL; 1769 return -EINVAL;
1755 } 1770 }
1756 1771
@@ -1759,13 +1774,13 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1759 return -ENOMEM; 1774 return -ENOMEM;
1760 1775
1761 INIT_LIST_HEAD(&mapping->list); 1776 INIT_LIST_HEAD(&mapping->list);
1762 mapping->it.start = saddr; 1777 mapping->start = saddr;
1763 mapping->it.last = eaddr; 1778 mapping->last = eaddr;
1764 mapping->offset = offset; 1779 mapping->offset = offset;
1765 mapping->flags = flags; 1780 mapping->flags = flags;
1766 1781
1767 list_add(&mapping->list, &bo_va->invalids); 1782 list_add(&mapping->list, &bo_va->invalids);
1768 interval_tree_insert(&mapping->it, &vm->va); 1783 amdgpu_vm_it_insert(mapping, &vm->va);
1769 1784
1770 if (flags & AMDGPU_PTE_PRT) 1785 if (flags & AMDGPU_PTE_PRT)
1771 amdgpu_vm_prt_get(adev); 1786 amdgpu_vm_prt_get(adev);
@@ -1823,13 +1838,13 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
1823 saddr /= AMDGPU_GPU_PAGE_SIZE; 1838 saddr /= AMDGPU_GPU_PAGE_SIZE;
1824 eaddr /= AMDGPU_GPU_PAGE_SIZE; 1839 eaddr /= AMDGPU_GPU_PAGE_SIZE;
1825 1840
1826 mapping->it.start = saddr; 1841 mapping->start = saddr;
1827 mapping->it.last = eaddr; 1842 mapping->last = eaddr;
1828 mapping->offset = offset; 1843 mapping->offset = offset;
1829 mapping->flags = flags; 1844 mapping->flags = flags;
1830 1845
1831 list_add(&mapping->list, &bo_va->invalids); 1846 list_add(&mapping->list, &bo_va->invalids);
1832 interval_tree_insert(&mapping->it, &vm->va); 1847 amdgpu_vm_it_insert(mapping, &vm->va);
1833 1848
1834 if (flags & AMDGPU_PTE_PRT) 1849 if (flags & AMDGPU_PTE_PRT)
1835 amdgpu_vm_prt_get(adev); 1850 amdgpu_vm_prt_get(adev);
@@ -1860,7 +1875,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1860 saddr /= AMDGPU_GPU_PAGE_SIZE; 1875 saddr /= AMDGPU_GPU_PAGE_SIZE;
1861 1876
1862 list_for_each_entry(mapping, &bo_va->valids, list) { 1877 list_for_each_entry(mapping, &bo_va->valids, list) {
1863 if (mapping->it.start == saddr) 1878 if (mapping->start == saddr)
1864 break; 1879 break;
1865 } 1880 }
1866 1881
@@ -1868,7 +1883,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1868 valid = false; 1883 valid = false;
1869 1884
1870 list_for_each_entry(mapping, &bo_va->invalids, list) { 1885 list_for_each_entry(mapping, &bo_va->invalids, list) {
1871 if (mapping->it.start == saddr) 1886 if (mapping->start == saddr)
1872 break; 1887 break;
1873 } 1888 }
1874 1889
@@ -1877,7 +1892,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1877 } 1892 }
1878 1893
1879 list_del(&mapping->list); 1894 list_del(&mapping->list);
1880 interval_tree_remove(&mapping->it, &vm->va); 1895 amdgpu_vm_it_remove(mapping, &vm->va);
1881 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1896 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1882 1897
1883 if (valid) 1898 if (valid)
@@ -1905,7 +1920,6 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
1905 uint64_t saddr, uint64_t size) 1920 uint64_t saddr, uint64_t size)
1906{ 1921{
1907 struct amdgpu_bo_va_mapping *before, *after, *tmp, *next; 1922 struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
1908 struct interval_tree_node *it;
1909 LIST_HEAD(removed); 1923 LIST_HEAD(removed);
1910 uint64_t eaddr; 1924 uint64_t eaddr;
1911 1925
@@ -1927,43 +1941,42 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
1927 INIT_LIST_HEAD(&after->list); 1941 INIT_LIST_HEAD(&after->list);
1928 1942
1929 /* Now gather all removed mappings */ 1943 /* Now gather all removed mappings */
1930 it = interval_tree_iter_first(&vm->va, saddr, eaddr); 1944 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1931 while (it) { 1945 while (tmp) {
1932 tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
1933 it = interval_tree_iter_next(it, saddr, eaddr);
1934
1935 /* Remember mapping split at the start */ 1946 /* Remember mapping split at the start */
1936 if (tmp->it.start < saddr) { 1947 if (tmp->start < saddr) {
1937 before->it.start = tmp->it.start; 1948 before->start = tmp->start;
1938 before->it.last = saddr - 1; 1949 before->last = saddr - 1;
1939 before->offset = tmp->offset; 1950 before->offset = tmp->offset;
1940 before->flags = tmp->flags; 1951 before->flags = tmp->flags;
1941 list_add(&before->list, &tmp->list); 1952 list_add(&before->list, &tmp->list);
1942 } 1953 }
1943 1954
1944 /* Remember mapping split at the end */ 1955 /* Remember mapping split at the end */
1945 if (tmp->it.last > eaddr) { 1956 if (tmp->last > eaddr) {
1946 after->it.start = eaddr + 1; 1957 after->start = eaddr + 1;
1947 after->it.last = tmp->it.last; 1958 after->last = tmp->last;
1948 after->offset = tmp->offset; 1959 after->offset = tmp->offset;
1949 after->offset += after->it.start - tmp->it.start; 1960 after->offset += after->start - tmp->start;
1950 after->flags = tmp->flags; 1961 after->flags = tmp->flags;
1951 list_add(&after->list, &tmp->list); 1962 list_add(&after->list, &tmp->list);
1952 } 1963 }
1953 1964
1954 list_del(&tmp->list); 1965 list_del(&tmp->list);
1955 list_add(&tmp->list, &removed); 1966 list_add(&tmp->list, &removed);
1967
1968 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
1956 } 1969 }
1957 1970
1958 /* And free them up */ 1971 /* And free them up */
1959 list_for_each_entry_safe(tmp, next, &removed, list) { 1972 list_for_each_entry_safe(tmp, next, &removed, list) {
1960 interval_tree_remove(&tmp->it, &vm->va); 1973 amdgpu_vm_it_remove(tmp, &vm->va);
1961 list_del(&tmp->list); 1974 list_del(&tmp->list);
1962 1975
1963 if (tmp->it.start < saddr) 1976 if (tmp->start < saddr)
1964 tmp->it.start = saddr; 1977 tmp->start = saddr;
1965 if (tmp->it.last > eaddr) 1978 if (tmp->last > eaddr)
1966 tmp->it.last = eaddr; 1979 tmp->last = eaddr;
1967 1980
1968 list_add(&tmp->list, &vm->freed); 1981 list_add(&tmp->list, &vm->freed);
1969 trace_amdgpu_vm_bo_unmap(NULL, tmp); 1982 trace_amdgpu_vm_bo_unmap(NULL, tmp);
@@ -1971,7 +1984,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
1971 1984
1972 /* Insert partial mapping before the range */ 1985 /* Insert partial mapping before the range */
1973 if (!list_empty(&before->list)) { 1986 if (!list_empty(&before->list)) {
1974 interval_tree_insert(&before->it, &vm->va); 1987 amdgpu_vm_it_insert(before, &vm->va);
1975 if (before->flags & AMDGPU_PTE_PRT) 1988 if (before->flags & AMDGPU_PTE_PRT)
1976 amdgpu_vm_prt_get(adev); 1989 amdgpu_vm_prt_get(adev);
1977 } else { 1990 } else {
@@ -1980,7 +1993,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
1980 1993
1981 /* Insert partial mapping after the range */ 1994 /* Insert partial mapping after the range */
1982 if (!list_empty(&after->list)) { 1995 if (!list_empty(&after->list)) {
1983 interval_tree_insert(&after->it, &vm->va); 1996 amdgpu_vm_it_insert(after, &vm->va);
1984 if (after->flags & AMDGPU_PTE_PRT) 1997 if (after->flags & AMDGPU_PTE_PRT)
1985 amdgpu_vm_prt_get(adev); 1998 amdgpu_vm_prt_get(adev);
1986 } else { 1999 } else {
@@ -2014,13 +2027,13 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
2014 2027
2015 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 2028 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2016 list_del(&mapping->list); 2029 list_del(&mapping->list);
2017 interval_tree_remove(&mapping->it, &vm->va); 2030 amdgpu_vm_it_remove(mapping, &vm->va);
2018 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 2031 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2019 list_add(&mapping->list, &vm->freed); 2032 list_add(&mapping->list, &vm->freed);
2020 } 2033 }
2021 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 2034 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2022 list_del(&mapping->list); 2035 list_del(&mapping->list);
2023 interval_tree_remove(&mapping->it, &vm->va); 2036 amdgpu_vm_it_remove(mapping, &vm->va);
2024 amdgpu_vm_free_mapping(adev, vm, mapping, 2037 amdgpu_vm_free_mapping(adev, vm, mapping,
2025 bo_va->last_pt_update); 2038 bo_va->last_pt_update);
2026 } 2039 }
@@ -2062,7 +2075,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
2062int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2075int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2063{ 2076{
2064 const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE, 2077 const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
2065 AMDGPU_VM_PTE_COUNT * 8); 2078 AMDGPU_VM_PTE_COUNT(adev) * 8);
2066 unsigned ring_instance; 2079 unsigned ring_instance;
2067 struct amdgpu_ring *ring; 2080 struct amdgpu_ring *ring;
2068 struct amd_sched_rq *rq; 2081 struct amd_sched_rq *rq;
@@ -2162,9 +2175,9 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2162 if (!RB_EMPTY_ROOT(&vm->va)) { 2175 if (!RB_EMPTY_ROOT(&vm->va)) {
2163 dev_err(adev->dev, "still active bo inside vm\n"); 2176 dev_err(adev->dev, "still active bo inside vm\n");
2164 } 2177 }
2165 rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) { 2178 rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, rb) {
2166 list_del(&mapping->list); 2179 list_del(&mapping->list);
2167 interval_tree_remove(&mapping->it, &vm->va); 2180 amdgpu_vm_it_remove(mapping, &vm->va);
2168 kfree(mapping); 2181 kfree(mapping);
2169 } 2182 }
2170 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { 2183 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
@@ -2227,7 +2240,6 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2227 for (i = 0; i < AMDGPU_NUM_VM; ++i) { 2240 for (i = 0; i < AMDGPU_NUM_VM; ++i) {
2228 struct amdgpu_vm_id *id = &adev->vm_manager.ids[i]; 2241 struct amdgpu_vm_id *id = &adev->vm_manager.ids[i];
2229 2242
2230 dma_fence_put(adev->vm_manager.ids[i].first);
2231 amdgpu_sync_free(&adev->vm_manager.ids[i].active); 2243 amdgpu_sync_free(&adev->vm_manager.ids[i].active);
2232 dma_fence_put(id->flushed_updates); 2244 dma_fence_put(id->flushed_updates);
2233 dma_fence_put(id->last_flush); 2245 dma_fence_put(id->last_flush);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index fbe17bf73a00..02b0dd3b135f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -45,7 +45,7 @@ struct amdgpu_bo_list_entry;
45#define AMDGPU_VM_MAX_UPDATE_SIZE 0x3FFFF 45#define AMDGPU_VM_MAX_UPDATE_SIZE 0x3FFFF
46 46
47/* number of entries in page table */ 47/* number of entries in page table */
48#define AMDGPU_VM_PTE_COUNT (1 << amdgpu_vm_block_size) 48#define AMDGPU_VM_PTE_COUNT(adev) (1 << (adev)->vm_manager.block_size)
49 49
50/* PTBs (Page Table Blocks) need to be aligned to 32K */ 50/* PTBs (Page Table Blocks) need to be aligned to 32K */
51#define AMDGPU_VM_PTB_ALIGN_SIZE 32768 51#define AMDGPU_VM_PTB_ALIGN_SIZE 32768
@@ -76,6 +76,14 @@ struct amdgpu_bo_list_entry;
76#define AMDGPU_VM_FAULT_STOP_FIRST 1 76#define AMDGPU_VM_FAULT_STOP_FIRST 1
77#define AMDGPU_VM_FAULT_STOP_ALWAYS 2 77#define AMDGPU_VM_FAULT_STOP_ALWAYS 2
78 78
79/* max number of VMHUB */
80#define AMDGPU_MAX_VMHUBS 2
81#define AMDGPU_GFXHUB 0
82#define AMDGPU_MMHUB 1
83
84/* hardcode that limit for now */
85#define AMDGPU_VA_RESERVED_SIZE (8 << 20)
86
79struct amdgpu_vm_pt { 87struct amdgpu_vm_pt {
80 struct amdgpu_bo *bo; 88 struct amdgpu_bo *bo;
81 uint64_t addr; 89 uint64_t addr;
@@ -123,7 +131,6 @@ struct amdgpu_vm {
123 131
124struct amdgpu_vm_id { 132struct amdgpu_vm_id {
125 struct list_head list; 133 struct list_head list;
126 struct dma_fence *first;
127 struct amdgpu_sync active; 134 struct amdgpu_sync active;
128 struct dma_fence *last_flush; 135 struct dma_fence *last_flush;
129 atomic64_t owner; 136 atomic64_t owner;
@@ -155,6 +162,8 @@ struct amdgpu_vm_manager {
155 162
156 uint64_t max_pfn; 163 uint64_t max_pfn;
157 uint32_t num_level; 164 uint32_t num_level;
165 uint64_t vm_size;
166 uint32_t block_size;
158 /* vram base address for page table entry */ 167 /* vram base address for page table entry */
159 u64 vram_base_offset; 168 u64 vram_base_offset;
160 /* is vm enabled? */ 169 /* is vm enabled? */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 9e577e3d3147..a4831fe0223b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -93,7 +93,6 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
93 const struct ttm_place *place, 93 const struct ttm_place *place,
94 struct ttm_mem_reg *mem) 94 struct ttm_mem_reg *mem)
95{ 95{
96 struct amdgpu_bo *bo = container_of(tbo, struct amdgpu_bo, tbo);
97 struct amdgpu_vram_mgr *mgr = man->priv; 96 struct amdgpu_vram_mgr *mgr = man->priv;
98 struct drm_mm *mm = &mgr->mm; 97 struct drm_mm *mm = &mgr->mm;
99 struct drm_mm_node *nodes; 98 struct drm_mm_node *nodes;
@@ -106,8 +105,8 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
106 if (!lpfn) 105 if (!lpfn)
107 lpfn = man->size; 106 lpfn = man->size;
108 107
109 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS || 108 if (place->flags & TTM_PL_FLAG_CONTIGUOUS ||
110 place->lpfn || amdgpu_vram_page_split == -1) { 109 amdgpu_vram_page_split == -1) {
111 pages_per_node = ~0ul; 110 pages_per_node = ~0ul;
112 num_nodes = 1; 111 num_nodes = 1;
113 } else { 112 } else {
@@ -124,12 +123,14 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
124 if (place->flags & TTM_PL_FLAG_TOPDOWN) 123 if (place->flags & TTM_PL_FLAG_TOPDOWN)
125 mode = DRM_MM_INSERT_HIGH; 124 mode = DRM_MM_INSERT_HIGH;
126 125
126 mem->start = 0;
127 pages_left = mem->num_pages; 127 pages_left = mem->num_pages;
128 128
129 spin_lock(&mgr->lock); 129 spin_lock(&mgr->lock);
130 for (i = 0; i < num_nodes; ++i) { 130 for (i = 0; i < num_nodes; ++i) {
131 unsigned long pages = min(pages_left, pages_per_node); 131 unsigned long pages = min(pages_left, pages_per_node);
132 uint32_t alignment = mem->page_alignment; 132 uint32_t alignment = mem->page_alignment;
133 unsigned long start;
133 134
134 if (pages == pages_per_node) 135 if (pages == pages_per_node)
135 alignment = pages_per_node; 136 alignment = pages_per_node;
@@ -141,11 +142,19 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
141 if (unlikely(r)) 142 if (unlikely(r))
142 goto error; 143 goto error;
143 144
145 /* Calculate a virtual BO start address to easily check if
146 * everything is CPU accessible.
147 */
148 start = nodes[i].start + nodes[i].size;
149 if (start > mem->num_pages)
150 start -= mem->num_pages;
151 else
152 start = 0;
153 mem->start = max(mem->start, start);
144 pages_left -= pages; 154 pages_left -= pages;
145 } 155 }
146 spin_unlock(&mgr->lock); 156 spin_unlock(&mgr->lock);
147 157
148 mem->start = num_nodes == 1 ? nodes[0].start : AMDGPU_BO_INVALID_OFFSET;
149 mem->mm_node = nodes; 158 mem->mm_node = nodes;
150 159
151 return 0; 160 return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index e0fa0d30e162..dad8a4cd1b37 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -4565,6 +4565,7 @@ static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
4565 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 4565 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
4566 for (i = 0; i < adev->gfx.num_compute_rings; i++) 4566 for (i = 0; i < adev->gfx.num_compute_rings; i++)
4567 adev->gfx.compute_ring[i].ready = false; 4567 adev->gfx.compute_ring[i].ready = false;
4568 adev->gfx.kiq.ring.ready = false;
4568 } 4569 }
4569 udelay(50); 4570 udelay(50);
4570} 4571}
@@ -4721,14 +4722,10 @@ static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
4721 mqd->cp_hqd_eop_control = tmp; 4722 mqd->cp_hqd_eop_control = tmp;
4722 4723
4723 /* enable doorbell? */ 4724 /* enable doorbell? */
4724 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL); 4725 tmp = REG_SET_FIELD(RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL),
4725 4726 CP_HQD_PQ_DOORBELL_CONTROL,
4726 if (ring->use_doorbell) 4727 DOORBELL_EN,
4727 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4728 ring->use_doorbell ? 1 : 0);
4728 DOORBELL_EN, 1);
4729 else
4730 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4731 DOORBELL_EN, 0);
4732 4729
4733 mqd->cp_hqd_pq_doorbell_control = tmp; 4730 mqd->cp_hqd_pq_doorbell_control = tmp;
4734 4731
@@ -4816,13 +4813,10 @@ static int gfx_v8_0_kiq_init_register(struct amdgpu_ring *ring)
4816{ 4813{
4817 struct amdgpu_device *adev = ring->adev; 4814 struct amdgpu_device *adev = ring->adev;
4818 struct vi_mqd *mqd = ring->mqd_ptr; 4815 struct vi_mqd *mqd = ring->mqd_ptr;
4819 uint32_t tmp;
4820 int j; 4816 int j;
4821 4817
4822 /* disable wptr polling */ 4818 /* disable wptr polling */
4823 tmp = RREG32(mmCP_PQ_WPTR_POLL_CNTL); 4819 WREG32_FIELD(CP_PQ_WPTR_POLL_CNTL, EN, 0);
4824 tmp = REG_SET_FIELD(tmp, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4825 WREG32(mmCP_PQ_WPTR_POLL_CNTL, tmp);
4826 4820
4827 WREG32(mmCP_HQD_EOP_BASE_ADDR, mqd->cp_hqd_eop_base_addr_lo); 4821 WREG32(mmCP_HQD_EOP_BASE_ADDR, mqd->cp_hqd_eop_base_addr_lo);
4828 WREG32(mmCP_HQD_EOP_BASE_ADDR_HI, mqd->cp_hqd_eop_base_addr_hi); 4822 WREG32(mmCP_HQD_EOP_BASE_ADDR_HI, mqd->cp_hqd_eop_base_addr_hi);
@@ -4834,10 +4828,10 @@ static int gfx_v8_0_kiq_init_register(struct amdgpu_ring *ring)
4834 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, mqd->cp_hqd_pq_doorbell_control); 4828 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, mqd->cp_hqd_pq_doorbell_control);
4835 4829
4836 /* disable the queue if it's active */ 4830 /* disable the queue if it's active */
4837 if (RREG32(mmCP_HQD_ACTIVE) & 1) { 4831 if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) {
4838 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 1); 4832 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 1);
4839 for (j = 0; j < adev->usec_timeout; j++) { 4833 for (j = 0; j < adev->usec_timeout; j++) {
4840 if (!(RREG32(mmCP_HQD_ACTIVE) & 1)) 4834 if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK))
4841 break; 4835 break;
4842 udelay(1); 4836 udelay(1);
4843 } 4837 }
@@ -4894,11 +4888,8 @@ static int gfx_v8_0_kiq_init_register(struct amdgpu_ring *ring)
4894 /* activate the queue */ 4888 /* activate the queue */
4895 WREG32(mmCP_HQD_ACTIVE, mqd->cp_hqd_active); 4889 WREG32(mmCP_HQD_ACTIVE, mqd->cp_hqd_active);
4896 4890
4897 if (ring->use_doorbell) { 4891 if (ring->use_doorbell)
4898 tmp = RREG32(mmCP_PQ_STATUS); 4892 WREG32_FIELD(CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4899 tmp = REG_SET_FIELD(tmp, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4900 WREG32(mmCP_PQ_STATUS, tmp);
4901 }
4902 4893
4903 return 0; 4894 return 0;
4904} 4895}
@@ -5471,19 +5462,18 @@ static void gfx_v8_0_inactive_hqd(struct amdgpu_device *adev,
5471{ 5462{
5472 int i; 5463 int i;
5473 5464
5465 mutex_lock(&adev->srbm_mutex);
5474 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 5466 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5475 if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) { 5467 if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) {
5476 u32 tmp; 5468 WREG32_FIELD(CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQ, 2);
5477 tmp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
5478 tmp = REG_SET_FIELD(tmp, CP_HQD_DEQUEUE_REQUEST,
5479 DEQUEUE_REQ, 2);
5480 WREG32(mmCP_HQD_DEQUEUE_REQUEST, tmp);
5481 for (i = 0; i < adev->usec_timeout; i++) { 5469 for (i = 0; i < adev->usec_timeout; i++) {
5482 if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK)) 5470 if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK))
5483 break; 5471 break;
5484 udelay(1); 5472 udelay(1);
5485 } 5473 }
5486 } 5474 }
5475 vi_srbm_select(adev, 0, 0, 0, 0);
5476 mutex_unlock(&adev->srbm_mutex);
5487} 5477}
5488 5478
5489static int gfx_v8_0_pre_soft_reset(void *handle) 5479static int gfx_v8_0_pre_soft_reset(void *handle)
@@ -5589,11 +5579,13 @@ static int gfx_v8_0_soft_reset(void *handle)
5589static void gfx_v8_0_init_hqd(struct amdgpu_device *adev, 5579static void gfx_v8_0_init_hqd(struct amdgpu_device *adev,
5590 struct amdgpu_ring *ring) 5580 struct amdgpu_ring *ring)
5591{ 5581{
5582 mutex_lock(&adev->srbm_mutex);
5592 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 5583 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5593 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0); 5584 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0);
5594 WREG32(mmCP_HQD_PQ_RPTR, 0); 5585 WREG32(mmCP_HQD_PQ_RPTR, 0);
5595 WREG32(mmCP_HQD_PQ_WPTR, 0); 5586 WREG32(mmCP_HQD_PQ_WPTR, 0);
5596 vi_srbm_select(adev, 0, 0, 0, 0); 5587 vi_srbm_select(adev, 0, 0, 0, 0);
5588 mutex_unlock(&adev->srbm_mutex);
5597} 5589}
5598 5590
5599static int gfx_v8_0_post_soft_reset(void *handle) 5591static int gfx_v8_0_post_soft_reset(void *handle)
@@ -6986,40 +6978,24 @@ static int gfx_v8_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
6986 unsigned int type, 6978 unsigned int type,
6987 enum amdgpu_interrupt_state state) 6979 enum amdgpu_interrupt_state state)
6988{ 6980{
6989 uint32_t tmp, target;
6990 struct amdgpu_ring *ring = &(adev->gfx.kiq.ring); 6981 struct amdgpu_ring *ring = &(adev->gfx.kiq.ring);
6991 6982
6992 BUG_ON(ring->funcs->type != AMDGPU_RING_TYPE_KIQ); 6983 BUG_ON(ring->funcs->type != AMDGPU_RING_TYPE_KIQ);
6993 6984
6994 if (ring->me == 1)
6995 target = mmCP_ME1_PIPE0_INT_CNTL;
6996 else
6997 target = mmCP_ME2_PIPE0_INT_CNTL;
6998 target += ring->pipe;
6999
7000 switch (type) { 6985 switch (type) {
7001 case AMDGPU_CP_KIQ_IRQ_DRIVER0: 6986 case AMDGPU_CP_KIQ_IRQ_DRIVER0:
7002 if (state == AMDGPU_IRQ_STATE_DISABLE) { 6987 WREG32_FIELD(CPC_INT_CNTL, GENERIC2_INT_ENABLE,
7003 tmp = RREG32(mmCPC_INT_CNTL); 6988 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
7004 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 6989 if (ring->me == 1)
7005 GENERIC2_INT_ENABLE, 0); 6990 WREG32_FIELD_OFFSET(CP_ME1_PIPE0_INT_CNTL,
7006 WREG32(mmCPC_INT_CNTL, tmp); 6991 ring->pipe,
7007 6992 GENERIC2_INT_ENABLE,
7008 tmp = RREG32(target); 6993 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
7009 tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL, 6994 else
7010 GENERIC2_INT_ENABLE, 0); 6995 WREG32_FIELD_OFFSET(CP_ME2_PIPE0_INT_CNTL,
7011 WREG32(target, tmp); 6996 ring->pipe,
7012 } else { 6997 GENERIC2_INT_ENABLE,
7013 tmp = RREG32(mmCPC_INT_CNTL); 6998 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
7014 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
7015 GENERIC2_INT_ENABLE, 1);
7016 WREG32(mmCPC_INT_CNTL, tmp);
7017
7018 tmp = RREG32(target);
7019 tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
7020 GENERIC2_INT_ENABLE, 1);
7021 WREG32(target, tmp);
7022 }
7023 break; 6999 break;
7024 default: 7000 default:
7025 BUG(); /* kiq only support GENERIC2_INT now */ 7001 BUG(); /* kiq only support GENERIC2_INT now */
@@ -7159,8 +7135,6 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
7159 .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_compute */ 7135 .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_compute */
7160 .emit_ib = gfx_v8_0_ring_emit_ib_compute, 7136 .emit_ib = gfx_v8_0_ring_emit_ib_compute,
7161 .emit_fence = gfx_v8_0_ring_emit_fence_kiq, 7137 .emit_fence = gfx_v8_0_ring_emit_fence_kiq,
7162 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
7163 .emit_hdp_invalidate = gfx_v8_0_ring_emit_hdp_invalidate,
7164 .test_ring = gfx_v8_0_ring_test_ring, 7138 .test_ring = gfx_v8_0_ring_test_ring,
7165 .test_ib = gfx_v8_0_ring_test_ib, 7139 .test_ib = gfx_v8_0_ring_test_ib,
7166 .insert_nop = amdgpu_ring_insert_nop, 7140 .insert_nop = amdgpu_ring_insert_nop,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 669bb98fc45d..a447b70841c9 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -1288,9 +1288,7 @@ static void gfx_v9_0_gpu_init(struct amdgpu_device *adev)
1288 u32 tmp; 1288 u32 tmp;
1289 int i; 1289 int i;
1290 1290
1291 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_CNTL)); 1291 WREG32_FIELD15(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1292 tmp = REG_SET_FIELD(tmp, GRBM_CNTL, READ_TIMEOUT, 0xff);
1293 WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_CNTL), tmp);
1294 1292
1295 gfx_v9_0_tiling_mode_table_init(adev); 1293 gfx_v9_0_tiling_mode_table_init(adev);
1296 1294
@@ -1395,13 +1393,9 @@ void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
1395 1393
1396static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 1394static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
1397{ 1395{
1398 u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET)); 1396 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
1399
1400 tmp = REG_SET_FIELD(tmp, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
1401 WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET), tmp);
1402 udelay(50); 1397 udelay(50);
1403 tmp = REG_SET_FIELD(tmp, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 1398 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
1404 WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_SOFT_RESET), tmp);
1405 udelay(50); 1399 udelay(50);
1406} 1400}
1407 1401
@@ -1410,10 +1404,8 @@ static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
1410#ifdef AMDGPU_RLC_DEBUG_RETRY 1404#ifdef AMDGPU_RLC_DEBUG_RETRY
1411 u32 rlc_ucode_ver; 1405 u32 rlc_ucode_ver;
1412#endif 1406#endif
1413 u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL));
1414 1407
1415 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 1); 1408 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
1416 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL), tmp);
1417 1409
1418 /* carrizo do enable cp interrupt after cp inited */ 1410 /* carrizo do enable cp interrupt after cp inited */
1419 if (!(adev->flags & AMD_IS_APU)) 1411 if (!(adev->flags & AMD_IS_APU))
@@ -1497,14 +1489,10 @@ static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
1497 int i; 1489 int i;
1498 u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL)); 1490 u32 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL));
1499 1491
1500 if (enable) { 1492 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
1501 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0); 1493 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
1502 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0); 1494 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
1503 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0); 1495 if (!enable) {
1504 } else {
1505 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1);
1506 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1);
1507 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1);
1508 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 1496 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1509 adev->gfx.gfx_ring[i].ready = false; 1497 adev->gfx.gfx_ring[i].ready = false;
1510 } 1498 }
@@ -2020,13 +2008,10 @@ static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
2020{ 2008{
2021 struct amdgpu_device *adev = ring->adev; 2009 struct amdgpu_device *adev = ring->adev;
2022 struct v9_mqd *mqd = ring->mqd_ptr; 2010 struct v9_mqd *mqd = ring->mqd_ptr;
2023 uint32_t tmp;
2024 int j; 2011 int j;
2025 2012
2026 /* disable wptr polling */ 2013 /* disable wptr polling */
2027 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL)); 2014 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
2028 tmp = REG_SET_FIELD(tmp, CP_PQ_WPTR_POLL_CNTL, EN, 0);
2029 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL), tmp);
2030 2015
2031 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_BASE_ADDR), 2016 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_BASE_ADDR),
2032 mqd->cp_hqd_eop_base_addr_lo); 2017 mqd->cp_hqd_eop_base_addr_lo);
@@ -2118,11 +2103,8 @@ static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
2118 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), 2103 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE),
2119 mqd->cp_hqd_active); 2104 mqd->cp_hqd_active);
2120 2105
2121 if (ring->use_doorbell) { 2106 if (ring->use_doorbell)
2122 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_STATUS)); 2107 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
2123 tmp = REG_SET_FIELD(tmp, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
2124 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_STATUS), tmp);
2125 }
2126 2108
2127 return 0; 2109 return 0;
2128} 2110}
@@ -2366,177 +2348,6 @@ static int gfx_v9_0_wait_for_idle(void *handle)
2366 return -ETIMEDOUT; 2348 return -ETIMEDOUT;
2367} 2349}
2368 2350
2369static void gfx_v9_0_print_status(void *handle)
2370{
2371 int i;
2372 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2373
2374 dev_info(adev->dev, "GFX 9.x registers\n");
2375 dev_info(adev->dev, " GRBM_STATUS=0x%08X\n",
2376 RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS)));
2377 dev_info(adev->dev, " GRBM_STATUS2=0x%08X\n",
2378 RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS2)));
2379 dev_info(adev->dev, " GRBM_STATUS_SE0=0x%08X\n",
2380 RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE0)));
2381 dev_info(adev->dev, " GRBM_STATUS_SE1=0x%08X\n",
2382 RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE1)));
2383 dev_info(adev->dev, " GRBM_STATUS_SE2=0x%08X\n",
2384 RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE2)));
2385 dev_info(adev->dev, " GRBM_STATUS_SE3=0x%08X\n",
2386 RREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE3)));
2387 dev_info(adev->dev, " CP_STAT = 0x%08x\n", RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_STAT)));
2388 dev_info(adev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
2389 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_STALLED_STAT1)));
2390 dev_info(adev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
2391 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_STALLED_STAT2)));
2392 dev_info(adev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
2393 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_STALLED_STAT3)));
2394 dev_info(adev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
2395 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPF_BUSY_STAT)));
2396 dev_info(adev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
2397 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPF_STALLED_STAT1)));
2398 dev_info(adev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPF_STATUS)));
2399 dev_info(adev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_BUSY_STAT)));
2400 dev_info(adev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
2401 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_STALLED_STAT1)));
2402 dev_info(adev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_CPC_STATUS)));
2403
2404 for (i = 0; i < 32; i++) {
2405 dev_info(adev->dev, " GB_TILE_MODE%d=0x%08X\n",
2406 i, RREG32(SOC15_REG_OFFSET(GC, 0, mmGB_TILE_MODE0 ) + i*4));
2407 }
2408 for (i = 0; i < 16; i++) {
2409 dev_info(adev->dev, " GB_MACROTILE_MODE%d=0x%08X\n",
2410 i, RREG32(SOC15_REG_OFFSET(GC, 0, mmGB_MACROTILE_MODE0) + i*4));
2411 }
2412 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2413 dev_info(adev->dev, " se: %d\n", i);
2414 gfx_v9_0_select_se_sh(adev, i, 0xffffffff, 0xffffffff);
2415 dev_info(adev->dev, " PA_SC_RASTER_CONFIG=0x%08X\n",
2416 RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_RASTER_CONFIG)));
2417 dev_info(adev->dev, " PA_SC_RASTER_CONFIG_1=0x%08X\n",
2418 RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_RASTER_CONFIG_1)));
2419 }
2420 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2421
2422 dev_info(adev->dev, " GB_ADDR_CONFIG=0x%08X\n",
2423 RREG32(SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG)));
2424
2425 dev_info(adev->dev, " CP_MEQ_THRESHOLDS=0x%08X\n",
2426 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEQ_THRESHOLDS)));
2427 dev_info(adev->dev, " SX_DEBUG_1=0x%08X\n",
2428 RREG32(SOC15_REG_OFFSET(GC, 0, mmSX_DEBUG_1)));
2429 dev_info(adev->dev, " TA_CNTL_AUX=0x%08X\n",
2430 RREG32(SOC15_REG_OFFSET(GC, 0, mmTA_CNTL_AUX)));
2431 dev_info(adev->dev, " SPI_CONFIG_CNTL=0x%08X\n",
2432 RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_CONFIG_CNTL)));
2433 dev_info(adev->dev, " SQ_CONFIG=0x%08X\n",
2434 RREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CONFIG)));
2435 dev_info(adev->dev, " DB_DEBUG=0x%08X\n",
2436 RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG)));
2437 dev_info(adev->dev, " DB_DEBUG2=0x%08X\n",
2438 RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG2)));
2439 dev_info(adev->dev, " DB_DEBUG3=0x%08X\n",
2440 RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG3)));
2441 dev_info(adev->dev, " CB_HW_CONTROL=0x%08X\n",
2442 RREG32(SOC15_REG_OFFSET(GC, 0, mmCB_HW_CONTROL)));
2443 dev_info(adev->dev, " SPI_CONFIG_CNTL_1=0x%08X\n",
2444 RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_CONFIG_CNTL_1)));
2445 dev_info(adev->dev, " PA_SC_FIFO_SIZE=0x%08X\n",
2446 RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_FIFO_SIZE)));
2447 dev_info(adev->dev, " VGT_NUM_INSTANCES=0x%08X\n",
2448 RREG32(SOC15_REG_OFFSET(GC, 0, mmVGT_NUM_INSTANCES)));
2449 dev_info(adev->dev, " CP_PERFMON_CNTL=0x%08X\n",
2450 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PERFMON_CNTL)));
2451 dev_info(adev->dev, " PA_SC_FORCE_EOV_MAX_CNTS=0x%08X\n",
2452 RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_FORCE_EOV_MAX_CNTS)));
2453 dev_info(adev->dev, " VGT_CACHE_INVALIDATION=0x%08X\n",
2454 RREG32(SOC15_REG_OFFSET(GC, 0, mmVGT_CACHE_INVALIDATION)));
2455 dev_info(adev->dev, " VGT_GS_VERTEX_REUSE=0x%08X\n",
2456 RREG32(SOC15_REG_OFFSET(GC, 0, mmVGT_GS_VERTEX_REUSE)));
2457 dev_info(adev->dev, " PA_SC_LINE_STIPPLE_STATE=0x%08X\n",
2458 RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_LINE_STIPPLE_STATE)));
2459 dev_info(adev->dev, " PA_CL_ENHANCE=0x%08X\n",
2460 RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_CL_ENHANCE)));
2461 dev_info(adev->dev, " PA_SC_ENHANCE=0x%08X\n",
2462 RREG32(SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE)));
2463
2464 dev_info(adev->dev, " CP_ME_CNTL=0x%08X\n",
2465 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL)));
2466 dev_info(adev->dev, " CP_MAX_CONTEXT=0x%08X\n",
2467 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MAX_CONTEXT)));
2468 dev_info(adev->dev, " CP_DEVICE_ID=0x%08X\n",
2469 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_DEVICE_ID)));
2470
2471 dev_info(adev->dev, " CP_SEM_WAIT_TIMER=0x%08X\n",
2472 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_SEM_WAIT_TIMER)));
2473
2474 dev_info(adev->dev, " CP_RB_WPTR_DELAY=0x%08X\n",
2475 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_DELAY)));
2476 dev_info(adev->dev, " CP_RB_VMID=0x%08X\n",
2477 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_VMID)));
2478 dev_info(adev->dev, " CP_RB0_CNTL=0x%08X\n",
2479 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_CNTL)));
2480 dev_info(adev->dev, " CP_RB0_WPTR=0x%08X\n",
2481 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_WPTR)));
2482 dev_info(adev->dev, " CP_RB0_RPTR_ADDR=0x%08X\n",
2483 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_RPTR_ADDR)));
2484 dev_info(adev->dev, " CP_RB0_RPTR_ADDR_HI=0x%08X\n",
2485 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_RPTR_ADDR_HI)));
2486 dev_info(adev->dev, " CP_RB0_CNTL=0x%08X\n",
2487 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_CNTL)));
2488 dev_info(adev->dev, " CP_RB0_BASE=0x%08X\n",
2489 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_BASE)));
2490 dev_info(adev->dev, " CP_RB0_BASE_HI=0x%08X\n",
2491 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB0_BASE_HI)));
2492 dev_info(adev->dev, " CP_MEC_CNTL=0x%08X\n",
2493 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_CNTL)));
2494
2495 dev_info(adev->dev, " SCRATCH_ADDR=0x%08X\n",
2496 RREG32(SOC15_REG_OFFSET(GC, 0, mmSCRATCH_ADDR)));
2497 dev_info(adev->dev, " SCRATCH_UMSK=0x%08X\n",
2498 RREG32(SOC15_REG_OFFSET(GC, 0, mmSCRATCH_UMSK)));
2499
2500 dev_info(adev->dev, " CP_INT_CNTL_RING0=0x%08X\n",
2501 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0)));
2502 dev_info(adev->dev, " RLC_LB_CNTL=0x%08X\n",
2503 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_CNTL)));
2504 dev_info(adev->dev, " RLC_CNTL=0x%08X\n",
2505 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CNTL)));
2506 dev_info(adev->dev, " RLC_CGCG_CGLS_CTRL=0x%08X\n",
2507 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)));
2508 dev_info(adev->dev, " RLC_LB_CNTR_INIT=0x%08X\n",
2509 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_CNTR_INIT)));
2510 dev_info(adev->dev, " RLC_LB_CNTR_MAX=0x%08X\n",
2511 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_CNTR_MAX)));
2512 dev_info(adev->dev, " RLC_LB_INIT_CU_MASK=0x%08X\n",
2513 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_INIT_CU_MASK)));
2514 dev_info(adev->dev, " RLC_LB_PARAMS=0x%08X\n",
2515 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_PARAMS)));
2516 dev_info(adev->dev, " RLC_LB_CNTL=0x%08X\n",
2517 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_LB_CNTL)));
2518 dev_info(adev->dev, " RLC_UCODE_CNTL=0x%08X\n",
2519 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_UCODE_CNTL)));
2520
2521 dev_info(adev->dev, " RLC_GPM_GENERAL_6=0x%08X\n",
2522 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_GENERAL_6)));
2523 dev_info(adev->dev, " RLC_GPM_GENERAL_12=0x%08X\n",
2524 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_GENERAL_12)));
2525 dev_info(adev->dev, " RLC_GPM_TIMER_INT_3=0x%08X\n",
2526 RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_TIMER_INT_3)));
2527 mutex_lock(&adev->srbm_mutex);
2528 for (i = 0; i < 16; i++) {
2529 soc15_grbm_select(adev, 0, 0, 0, i);
2530 dev_info(adev->dev, " VM %d:\n", i);
2531 dev_info(adev->dev, " SH_MEM_CONFIG=0x%08X\n",
2532 RREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG)));
2533 dev_info(adev->dev, " SH_MEM_BASES=0x%08X\n",
2534 RREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES)));
2535 }
2536 soc15_grbm_select(adev, 0, 0, 0, 0);
2537 mutex_unlock(&adev->srbm_mutex);
2538}
2539
2540static int gfx_v9_0_soft_reset(void *handle) 2351static int gfx_v9_0_soft_reset(void *handle)
2541{ 2352{
2542 u32 grbm_soft_reset = 0; 2353 u32 grbm_soft_reset = 0;
@@ -2569,8 +2380,7 @@ static int gfx_v9_0_soft_reset(void *handle)
2569 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 2380 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2570 2381
2571 2382
2572 if (grbm_soft_reset ) { 2383 if (grbm_soft_reset) {
2573 gfx_v9_0_print_status((void *)adev);
2574 /* stop the rlc */ 2384 /* stop the rlc */
2575 gfx_v9_0_rlc_stop(adev); 2385 gfx_v9_0_rlc_stop(adev);
2576 2386
@@ -2596,7 +2406,6 @@ static int gfx_v9_0_soft_reset(void *handle)
2596 2406
2597 /* Wait a little for things to settle down */ 2407 /* Wait a little for things to settle down */
2598 udelay(50); 2408 udelay(50);
2599 gfx_v9_0_print_status((void *)adev);
2600 } 2409 }
2601 return 0; 2410 return 0;
2602} 2411}
@@ -3148,6 +2957,7 @@ static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
3148 unsigned vm_id, uint64_t pd_addr) 2957 unsigned vm_id, uint64_t pd_addr)
3149{ 2958{
3150 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 2959 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
2960 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
3151 unsigned eng = ring->idx; 2961 unsigned eng = ring->idx;
3152 unsigned i; 2962 unsigned i;
3153 2963
@@ -3157,7 +2967,6 @@ static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
3157 2967
3158 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 2968 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
3159 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i]; 2969 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i];
3160 uint32_t req = hub->get_invalidate_req(vm_id);
3161 2970
3162 gfx_v9_0_write_data_to_reg(ring, usepfp, true, 2971 gfx_v9_0_write_data_to_reg(ring, usepfp, true,
3163 hub->ctx0_ptb_addr_lo32 2972 hub->ctx0_ptb_addr_lo32
@@ -3376,21 +3185,12 @@ static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
3376static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 3185static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
3377 enum amdgpu_interrupt_state state) 3186 enum amdgpu_interrupt_state state)
3378{ 3187{
3379 u32 cp_int_cntl;
3380
3381 switch (state) { 3188 switch (state) {
3382 case AMDGPU_IRQ_STATE_DISABLE: 3189 case AMDGPU_IRQ_STATE_DISABLE:
3383 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0));
3384 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
3385 TIME_STAMP_INT_ENABLE, 0);
3386 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3387 break;
3388 case AMDGPU_IRQ_STATE_ENABLE: 3190 case AMDGPU_IRQ_STATE_ENABLE:
3389 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0)); 3191 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
3390 cp_int_cntl = 3192 TIME_STAMP_INT_ENABLE,
3391 REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 3193 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3392 TIME_STAMP_INT_ENABLE, 1);
3393 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3394 break; 3194 break;
3395 default: 3195 default:
3396 break; 3196 break;
@@ -3446,20 +3246,12 @@ static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
3446 unsigned type, 3246 unsigned type,
3447 enum amdgpu_interrupt_state state) 3247 enum amdgpu_interrupt_state state)
3448{ 3248{
3449 u32 cp_int_cntl;
3450
3451 switch (state) { 3249 switch (state) {
3452 case AMDGPU_IRQ_STATE_DISABLE: 3250 case AMDGPU_IRQ_STATE_DISABLE:
3453 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0));
3454 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
3455 PRIV_REG_INT_ENABLE, 0);
3456 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3457 break;
3458 case AMDGPU_IRQ_STATE_ENABLE: 3251 case AMDGPU_IRQ_STATE_ENABLE:
3459 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0)); 3252 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
3460 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 3253 PRIV_REG_INT_ENABLE,
3461 PRIV_REG_INT_ENABLE, 1); 3254 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3462 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3463 break; 3255 break;
3464 default: 3256 default:
3465 break; 3257 break;
@@ -3473,21 +3265,12 @@ static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
3473 unsigned type, 3265 unsigned type,
3474 enum amdgpu_interrupt_state state) 3266 enum amdgpu_interrupt_state state)
3475{ 3267{
3476 u32 cp_int_cntl;
3477
3478 switch (state) { 3268 switch (state) {
3479 case AMDGPU_IRQ_STATE_DISABLE: 3269 case AMDGPU_IRQ_STATE_DISABLE:
3480 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0));
3481 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
3482 PRIV_INSTR_INT_ENABLE, 0);
3483 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3484 break;
3485 case AMDGPU_IRQ_STATE_ENABLE: 3270 case AMDGPU_IRQ_STATE_ENABLE:
3486 cp_int_cntl = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0)); 3271 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
3487 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 3272 PRIV_INSTR_INT_ENABLE,
3488 PRIV_INSTR_INT_ENABLE, 1); 3273 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3489 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0), cp_int_cntl);
3490 break;
3491 default: 3274 default:
3492 break; 3275 break;
3493 } 3276 }
@@ -3759,8 +3542,6 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
3759 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */ 3542 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_compute */
3760 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 3543 .emit_ib = gfx_v9_0_ring_emit_ib_compute,
3761 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 3544 .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
3762 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
3763 .emit_hdp_invalidate = gfx_v9_0_ring_emit_hdp_invalidate,
3764 .test_ring = gfx_v9_0_ring_test_ring, 3545 .test_ring = gfx_v9_0_ring_test_ring,
3765 .test_ib = gfx_v9_0_ring_test_ib, 3546 .test_ib = gfx_v9_0_ring_test_ib,
3766 .insert_nop = amdgpu_ring_insert_nop, 3547 .insert_nop = amdgpu_ring_insert_nop,
@@ -3975,9 +3756,7 @@ static int gfx_v9_0_init_queue(struct amdgpu_ring *ring)
3975 ring->pipe, 3756 ring->pipe,
3976 ring->queue, 0); 3757 ring->queue, 0);
3977 /* disable wptr polling */ 3758 /* disable wptr polling */
3978 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL)); 3759 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3979 tmp = REG_SET_FIELD(tmp, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3980 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL), tmp);
3981 3760
3982 /* write the EOP addr */ 3761 /* write the EOP addr */
3983 BUG_ON(ring->me != 1 || ring->pipe != 0); /* can't handle other cases eop address */ 3762 BUG_ON(ring->me != 1 || ring->pipe != 0); /* can't handle other cases eop address */
@@ -4121,11 +3900,8 @@ static int gfx_v9_0_init_queue(struct amdgpu_ring *ring)
4121 amdgpu_bo_kunmap(ring->mqd_obj); 3900 amdgpu_bo_kunmap(ring->mqd_obj);
4122 amdgpu_bo_unreserve(ring->mqd_obj); 3901 amdgpu_bo_unreserve(ring->mqd_obj);
4123 3902
4124 if (use_doorbell) { 3903 if (use_doorbell)
4125 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_STATUS)); 3904 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4126 tmp = REG_SET_FIELD(tmp, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4127 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_STATUS), tmp);
4128 }
4129 3905
4130 return 0; 3906 return 0;
4131} 3907}
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
index 30ef3126c8a9..005075ff00f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
@@ -222,7 +222,7 @@ int gfxhub_v1_0_gart_enable(struct amdgpu_device *adev)
222 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 222 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
223 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 223 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
224 PAGE_TABLE_BLOCK_SIZE, 224 PAGE_TABLE_BLOCK_SIZE,
225 amdgpu_vm_block_size - 9); 225 adev->vm_manager.block_size - 9);
226 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL) + i, tmp); 226 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL) + i, tmp);
227 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32) + i*2, 0); 227 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32) + i*2, 0);
228 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32) + i*2, 0); 228 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32) + i*2, 0);
@@ -299,36 +299,6 @@ void gfxhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev,
299 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL), tmp); 299 WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL), tmp);
300} 300}
301 301
302static uint32_t gfxhub_v1_0_get_invalidate_req(unsigned int vm_id)
303{
304 u32 req = 0;
305
306 /* invalidate using legacy mode on vm_id*/
307 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
308 PER_VMID_INVALIDATE_REQ, 1 << vm_id);
309 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0);
310 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
311 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
312 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
313 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
314 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
315 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
316 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
317
318 return req;
319}
320
321static uint32_t gfxhub_v1_0_get_vm_protection_bits(void)
322{
323 return (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
324 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
325 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
326 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
327 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
328 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
329 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
330}
331
332static int gfxhub_v1_0_early_init(void *handle) 302static int gfxhub_v1_0_early_init(void *handle)
333{ 303{
334 return 0; 304 return 0;
@@ -361,9 +331,6 @@ static int gfxhub_v1_0_sw_init(void *handle)
361 hub->vm_l2_pro_fault_cntl = 331 hub->vm_l2_pro_fault_cntl =
362 SOC15_REG_OFFSET(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL); 332 SOC15_REG_OFFSET(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
363 333
364 hub->get_invalidate_req = gfxhub_v1_0_get_invalidate_req;
365 hub->get_vm_protection_bits = gfxhub_v1_0_get_vm_protection_bits;
366
367 return 0; 334 return 0;
368} 335}
369 336
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index d9586601a437..8f18d14f8eda 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -543,7 +543,8 @@ static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
543 WREG32(mmVM_CONTEXT1_CNTL, 543 WREG32(mmVM_CONTEXT1_CNTL,
544 VM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK | 544 VM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK |
545 (1UL << VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT) | 545 (1UL << VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT) |
546 ((amdgpu_vm_block_size - 9) << VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT)); 546 ((adev->vm_manager.block_size - 9)
547 << VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT));
547 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 548 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
548 gmc_v6_0_set_fault_enable_default(adev, false); 549 gmc_v6_0_set_fault_enable_default(adev, false);
549 else 550 else
@@ -848,7 +849,12 @@ static int gmc_v6_0_sw_init(void *handle)
848 if (r) 849 if (r)
849 return r; 850 return r;
850 851
851 adev->vm_manager.max_pfn = amdgpu_vm_size << 18; 852 adev->vm_manager.vm_size = amdgpu_vm_size;
853 adev->vm_manager.block_size = amdgpu_vm_block_size;
854 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
855
856 DRM_INFO("vm size is %llu GB, block size is %d-bit\n",
857 adev->vm_manager.vm_size, adev->vm_manager.block_size);
852 858
853 adev->mc.mc_mask = 0xffffffffffULL; 859 adev->mc.mc_mask = 0xffffffffffULL;
854 860
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 0c0a6015cca5..b86b454197f8 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -37,6 +37,8 @@
37#include "oss/oss_2_0_d.h" 37#include "oss/oss_2_0_d.h"
38#include "oss/oss_2_0_sh_mask.h" 38#include "oss/oss_2_0_sh_mask.h"
39 39
40#include "amdgpu_atombios.h"
41
40static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev); 42static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev);
41static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev); 43static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev);
42static int gmc_v7_0_wait_for_idle(void *handle); 44static int gmc_v7_0_wait_for_idle(void *handle);
@@ -325,48 +327,51 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
325 */ 327 */
326static int gmc_v7_0_mc_init(struct amdgpu_device *adev) 328static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
327{ 329{
328 u32 tmp; 330 adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev);
329 int chansize, numchan; 331 if (!adev->mc.vram_width) {
330 332 u32 tmp;
331 /* Get VRAM informations */ 333 int chansize, numchan;
332 tmp = RREG32(mmMC_ARB_RAMCFG); 334
333 if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) { 335 /* Get VRAM informations */
334 chansize = 64; 336 tmp = RREG32(mmMC_ARB_RAMCFG);
335 } else { 337 if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) {
336 chansize = 32; 338 chansize = 64;
337 } 339 } else {
338 tmp = RREG32(mmMC_SHARED_CHMAP); 340 chansize = 32;
339 switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) { 341 }
340 case 0: 342 tmp = RREG32(mmMC_SHARED_CHMAP);
341 default: 343 switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {
342 numchan = 1; 344 case 0:
343 break; 345 default:
344 case 1: 346 numchan = 1;
345 numchan = 2; 347 break;
346 break; 348 case 1:
347 case 2: 349 numchan = 2;
348 numchan = 4; 350 break;
349 break; 351 case 2:
350 case 3: 352 numchan = 4;
351 numchan = 8; 353 break;
352 break; 354 case 3:
353 case 4: 355 numchan = 8;
354 numchan = 3; 356 break;
355 break; 357 case 4:
356 case 5: 358 numchan = 3;
357 numchan = 6; 359 break;
358 break; 360 case 5:
359 case 6: 361 numchan = 6;
360 numchan = 10; 362 break;
361 break; 363 case 6:
362 case 7: 364 numchan = 10;
363 numchan = 12; 365 break;
364 break; 366 case 7:
365 case 8: 367 numchan = 12;
366 numchan = 16; 368 break;
367 break; 369 case 8:
370 numchan = 16;
371 break;
372 }
373 adev->mc.vram_width = numchan * chansize;
368 } 374 }
369 adev->mc.vram_width = numchan * chansize;
370 /* Could aper size report 0 ? */ 375 /* Could aper size report 0 ? */
371 adev->mc.aper_base = pci_resource_start(adev->pdev, 0); 376 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
372 adev->mc.aper_size = pci_resource_len(adev->pdev, 0); 377 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
@@ -639,7 +644,7 @@ static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
639 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); 644 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
640 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1); 645 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
641 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE, 646 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
642 amdgpu_vm_block_size - 9); 647 adev->vm_manager.block_size - 9);
643 WREG32(mmVM_CONTEXT1_CNTL, tmp); 648 WREG32(mmVM_CONTEXT1_CNTL, tmp);
644 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 649 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
645 gmc_v7_0_set_fault_enable_default(adev, false); 650 gmc_v7_0_set_fault_enable_default(adev, false);
@@ -998,7 +1003,12 @@ static int gmc_v7_0_sw_init(void *handle)
998 * Currently set to 4GB ((1 << 20) 4k pages). 1003 * Currently set to 4GB ((1 << 20) 4k pages).
999 * Max GPUVM size for cayman and SI is 40 bits. 1004 * Max GPUVM size for cayman and SI is 40 bits.
1000 */ 1005 */
1001 adev->vm_manager.max_pfn = amdgpu_vm_size << 18; 1006 adev->vm_manager.vm_size = amdgpu_vm_size;
1007 adev->vm_manager.block_size = amdgpu_vm_block_size;
1008 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
1009
1010 DRM_INFO("vm size is %llu GB, block size is %d-bit\n",
1011 adev->vm_manager.vm_size, adev->vm_manager.block_size);
1002 1012
1003 /* Set the internal MC address mask 1013 /* Set the internal MC address mask
1004 * This is the max address of the GPU's 1014 * This is the max address of the GPU's
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index d19d1c5e2847..108a20e832cf 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -38,6 +38,8 @@
38#include "vid.h" 38#include "vid.h"
39#include "vi.h" 39#include "vi.h"
40 40
41#include "amdgpu_atombios.h"
42
41 43
42static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev); 44static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev);
43static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev); 45static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev);
@@ -487,48 +489,51 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
487 */ 489 */
488static int gmc_v8_0_mc_init(struct amdgpu_device *adev) 490static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
489{ 491{
490 u32 tmp; 492 adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev);
491 int chansize, numchan; 493 if (!adev->mc.vram_width) {
492 494 u32 tmp;
493 /* Get VRAM informations */ 495 int chansize, numchan;
494 tmp = RREG32(mmMC_ARB_RAMCFG); 496
495 if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) { 497 /* Get VRAM informations */
496 chansize = 64; 498 tmp = RREG32(mmMC_ARB_RAMCFG);
497 } else { 499 if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) {
498 chansize = 32; 500 chansize = 64;
499 } 501 } else {
500 tmp = RREG32(mmMC_SHARED_CHMAP); 502 chansize = 32;
501 switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) { 503 }
502 case 0: 504 tmp = RREG32(mmMC_SHARED_CHMAP);
503 default: 505 switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {
504 numchan = 1; 506 case 0:
505 break; 507 default:
506 case 1: 508 numchan = 1;
507 numchan = 2; 509 break;
508 break; 510 case 1:
509 case 2: 511 numchan = 2;
510 numchan = 4; 512 break;
511 break; 513 case 2:
512 case 3: 514 numchan = 4;
513 numchan = 8; 515 break;
514 break; 516 case 3:
515 case 4: 517 numchan = 8;
516 numchan = 3; 518 break;
517 break; 519 case 4:
518 case 5: 520 numchan = 3;
519 numchan = 6; 521 break;
520 break; 522 case 5:
521 case 6: 523 numchan = 6;
522 numchan = 10; 524 break;
523 break; 525 case 6:
524 case 7: 526 numchan = 10;
525 numchan = 12; 527 break;
526 break; 528 case 7:
527 case 8: 529 numchan = 12;
528 numchan = 16; 530 break;
529 break; 531 case 8:
532 numchan = 16;
533 break;
534 }
535 adev->mc.vram_width = numchan * chansize;
530 } 536 }
531 adev->mc.vram_width = numchan * chansize;
532 /* Could aper size report 0 ? */ 537 /* Could aper size report 0 ? */
533 adev->mc.aper_base = pci_resource_start(adev->pdev, 0); 538 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
534 adev->mc.aper_size = pci_resource_len(adev->pdev, 0); 539 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
@@ -848,7 +853,7 @@ static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
848 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 853 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
849 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 854 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
850 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE, 855 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
851 amdgpu_vm_block_size - 9); 856 adev->vm_manager.block_size - 9);
852 WREG32(mmVM_CONTEXT1_CNTL, tmp); 857 WREG32(mmVM_CONTEXT1_CNTL, tmp);
853 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 858 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
854 gmc_v8_0_set_fault_enable_default(adev, false); 859 gmc_v8_0_set_fault_enable_default(adev, false);
@@ -1082,7 +1087,12 @@ static int gmc_v8_0_sw_init(void *handle)
1082 * Currently set to 4GB ((1 << 20) 4k pages). 1087 * Currently set to 4GB ((1 << 20) 4k pages).
1083 * Max GPUVM size for cayman and SI is 40 bits. 1088 * Max GPUVM size for cayman and SI is 40 bits.
1084 */ 1089 */
1085 adev->vm_manager.max_pfn = amdgpu_vm_size << 18; 1090 adev->vm_manager.vm_size = amdgpu_vm_size;
1091 adev->vm_manager.block_size = amdgpu_vm_block_size;
1092 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
1093
1094 DRM_INFO("vm size is %llu GB, block size is %d-bit\n",
1095 adev->vm_manager.vm_size, adev->vm_manager.block_size);
1086 1096
1087 /* Set the internal MC address mask 1097 /* Set the internal MC address mask
1088 * This is the max address of the GPU's 1098 * This is the max address of the GPU's
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index df69aae99df4..6329be81f260 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -75,11 +75,18 @@ static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
75 struct amdgpu_vmhub *hub; 75 struct amdgpu_vmhub *hub;
76 u32 tmp, reg, bits, i; 76 u32 tmp, reg, bits, i;
77 77
78 bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
79 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
80 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
81 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
82 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
83 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
84 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
85
78 switch (state) { 86 switch (state) {
79 case AMDGPU_IRQ_STATE_DISABLE: 87 case AMDGPU_IRQ_STATE_DISABLE:
80 /* MM HUB */ 88 /* MM HUB */
81 hub = &adev->vmhub[AMDGPU_MMHUB]; 89 hub = &adev->vmhub[AMDGPU_MMHUB];
82 bits = hub->get_vm_protection_bits();
83 for (i = 0; i< 16; i++) { 90 for (i = 0; i< 16; i++) {
84 reg = hub->vm_context0_cntl + i; 91 reg = hub->vm_context0_cntl + i;
85 tmp = RREG32(reg); 92 tmp = RREG32(reg);
@@ -89,7 +96,6 @@ static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
89 96
90 /* GFX HUB */ 97 /* GFX HUB */
91 hub = &adev->vmhub[AMDGPU_GFXHUB]; 98 hub = &adev->vmhub[AMDGPU_GFXHUB];
92 bits = hub->get_vm_protection_bits();
93 for (i = 0; i < 16; i++) { 99 for (i = 0; i < 16; i++) {
94 reg = hub->vm_context0_cntl + i; 100 reg = hub->vm_context0_cntl + i;
95 tmp = RREG32(reg); 101 tmp = RREG32(reg);
@@ -100,7 +106,6 @@ static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
100 case AMDGPU_IRQ_STATE_ENABLE: 106 case AMDGPU_IRQ_STATE_ENABLE:
101 /* MM HUB */ 107 /* MM HUB */
102 hub = &adev->vmhub[AMDGPU_MMHUB]; 108 hub = &adev->vmhub[AMDGPU_MMHUB];
103 bits = hub->get_vm_protection_bits();
104 for (i = 0; i< 16; i++) { 109 for (i = 0; i< 16; i++) {
105 reg = hub->vm_context0_cntl + i; 110 reg = hub->vm_context0_cntl + i;
106 tmp = RREG32(reg); 111 tmp = RREG32(reg);
@@ -110,7 +115,6 @@ static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
110 115
111 /* GFX HUB */ 116 /* GFX HUB */
112 hub = &adev->vmhub[AMDGPU_GFXHUB]; 117 hub = &adev->vmhub[AMDGPU_GFXHUB];
113 bits = hub->get_vm_protection_bits();
114 for (i = 0; i < 16; i++) { 118 for (i = 0; i < 16; i++) {
115 reg = hub->vm_context0_cntl + i; 119 reg = hub->vm_context0_cntl + i;
116 tmp = RREG32(reg); 120 tmp = RREG32(reg);
@@ -129,8 +133,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
129 struct amdgpu_irq_src *source, 133 struct amdgpu_irq_src *source,
130 struct amdgpu_iv_entry *entry) 134 struct amdgpu_iv_entry *entry)
131{ 135{
132 struct amdgpu_vmhub *gfxhub = &adev->vmhub[AMDGPU_GFXHUB]; 136 struct amdgpu_vmhub *hub = &adev->vmhub[entry->vm_id_src];
133 struct amdgpu_vmhub *mmhub = &adev->vmhub[AMDGPU_MMHUB];
134 uint32_t status = 0; 137 uint32_t status = 0;
135 u64 addr; 138 u64 addr;
136 139
@@ -138,13 +141,8 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
138 addr |= ((u64)entry->src_data[1] & 0xf) << 44; 141 addr |= ((u64)entry->src_data[1] & 0xf) << 44;
139 142
140 if (!amdgpu_sriov_vf(adev)) { 143 if (!amdgpu_sriov_vf(adev)) {
141 if (entry->vm_id_src) { 144 status = RREG32(hub->vm_l2_pro_fault_status);
142 status = RREG32(mmhub->vm_l2_pro_fault_status); 145 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
143 WREG32_P(mmhub->vm_l2_pro_fault_cntl, 1, ~1);
144 } else {
145 status = RREG32(gfxhub->vm_l2_pro_fault_status);
146 WREG32_P(gfxhub->vm_l2_pro_fault_cntl, 1, ~1);
147 }
148 } 146 }
149 147
150 if (printk_ratelimit()) { 148 if (printk_ratelimit()) {
@@ -175,6 +173,25 @@ static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
175 adev->mc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 173 adev->mc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
176} 174}
177 175
176static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vm_id)
177{
178 u32 req = 0;
179
180 /* invalidate using legacy mode on vm_id*/
181 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
182 PER_VMID_INVALIDATE_REQ, 1 << vm_id);
183 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0);
184 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
185 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
186 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
187 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
188 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
189 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
190 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
191
192 return req;
193}
194
178/* 195/*
179 * GART 196 * GART
180 * VMID 0 is the physical GPU addresses as used by the kernel. 197 * VMID 0 is the physical GPU addresses as used by the kernel.
@@ -204,7 +221,7 @@ static void gmc_v9_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
204 221
205 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 222 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
206 struct amdgpu_vmhub *hub = &adev->vmhub[i]; 223 struct amdgpu_vmhub *hub = &adev->vmhub[i];
207 u32 tmp = hub->get_invalidate_req(vmid); 224 u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
208 225
209 WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp); 226 WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
210 227
@@ -337,30 +354,23 @@ static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev,
337 return pte_flag; 354 return pte_flag;
338} 355}
339 356
340static const struct amdgpu_gart_funcs gmc_v9_0_gart_funcs = {
341 .flush_gpu_tlb = gmc_v9_0_gart_flush_gpu_tlb,
342 .set_pte_pde = gmc_v9_0_gart_set_pte_pde,
343 .get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags
344};
345
346static void gmc_v9_0_set_gart_funcs(struct amdgpu_device *adev)
347{
348 if (adev->gart.gart_funcs == NULL)
349 adev->gart.gart_funcs = &gmc_v9_0_gart_funcs;
350}
351
352static u64 gmc_v9_0_adjust_mc_addr(struct amdgpu_device *adev, u64 mc_addr) 357static u64 gmc_v9_0_adjust_mc_addr(struct amdgpu_device *adev, u64 mc_addr)
353{ 358{
354 return adev->vm_manager.vram_base_offset + mc_addr - adev->mc.vram_start; 359 return adev->vm_manager.vram_base_offset + mc_addr - adev->mc.vram_start;
355} 360}
356 361
357static const struct amdgpu_mc_funcs gmc_v9_0_mc_funcs = { 362static const struct amdgpu_gart_funcs gmc_v9_0_gart_funcs = {
363 .flush_gpu_tlb = gmc_v9_0_gart_flush_gpu_tlb,
364 .set_pte_pde = gmc_v9_0_gart_set_pte_pde,
365 .get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags,
358 .adjust_mc_addr = gmc_v9_0_adjust_mc_addr, 366 .adjust_mc_addr = gmc_v9_0_adjust_mc_addr,
367 .get_invalidate_req = gmc_v9_0_get_invalidate_req,
359}; 368};
360 369
361static void gmc_v9_0_set_mc_funcs(struct amdgpu_device *adev) 370static void gmc_v9_0_set_gart_funcs(struct amdgpu_device *adev)
362{ 371{
363 adev->mc.mc_funcs = &gmc_v9_0_mc_funcs; 372 if (adev->gart.gart_funcs == NULL)
373 adev->gart.gart_funcs = &gmc_v9_0_gart_funcs;
364} 374}
365 375
366static int gmc_v9_0_early_init(void *handle) 376static int gmc_v9_0_early_init(void *handle)
@@ -368,7 +378,6 @@ static int gmc_v9_0_early_init(void *handle)
368 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 378 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
369 379
370 gmc_v9_0_set_gart_funcs(adev); 380 gmc_v9_0_set_gart_funcs(adev);
371 gmc_v9_0_set_mc_funcs(adev);
372 gmc_v9_0_set_irq_funcs(adev); 381 gmc_v9_0_set_irq_funcs(adev);
373 382
374 return 0; 383 return 0;
@@ -543,11 +552,23 @@ static int gmc_v9_0_sw_init(void *handle)
543 552
544 if (adev->flags & AMD_IS_APU) { 553 if (adev->flags & AMD_IS_APU) {
545 adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; 554 adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
555 adev->vm_manager.vm_size = amdgpu_vm_size;
556 adev->vm_manager.block_size = amdgpu_vm_block_size;
546 } else { 557 } else {
547 /* XXX Don't know how to get VRAM type yet. */ 558 /* XXX Don't know how to get VRAM type yet. */
548 adev->mc.vram_type = AMDGPU_VRAM_TYPE_HBM; 559 adev->mc.vram_type = AMDGPU_VRAM_TYPE_HBM;
560 /*
561 * To fulfill 4-level page support,
562 * vm size is 256TB (48bit), maximum size of Vega10,
563 * block size 512 (9bit)
564 */
565 adev->vm_manager.vm_size = 1U << 18;
566 adev->vm_manager.block_size = 9;
549 } 567 }
550 568
569 DRM_INFO("vm size is %llu GB, block size is %d-bit\n",
570 adev->vm_manager.vm_size, adev->vm_manager.block_size);
571
551 /* This interrupt is VMC page fault.*/ 572 /* This interrupt is VMC page fault.*/
552 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_VMC, 0, 573 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_VMC, 0,
553 &adev->mc.vm_fault); 574 &adev->mc.vm_fault);
@@ -557,14 +578,7 @@ static int gmc_v9_0_sw_init(void *handle)
557 if (r) 578 if (r)
558 return r; 579 return r;
559 580
560 /* Because of four level VMPTs, vm size is at least 512GB. 581 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
561 * The maximum size is 256TB (48bit).
562 */
563 if (amdgpu_vm_size < 512) {
564 DRM_WARN("VM size is at least 512GB!\n");
565 amdgpu_vm_size = 512;
566 }
567 adev->vm_manager.max_pfn = (uint64_t)amdgpu_vm_size << 18;
568 582
569 /* Set the internal MC address mask 583 /* Set the internal MC address mask
570 * This is the max address of the GPU's 584 * This is the max address of the GPU's
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
index 266a0f47a908..62684510ddcd 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
@@ -242,7 +242,7 @@ int mmhub_v1_0_gart_enable(struct amdgpu_device *adev)
242 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 242 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
243 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 243 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
244 PAGE_TABLE_BLOCK_SIZE, 244 PAGE_TABLE_BLOCK_SIZE,
245 amdgpu_vm_block_size - 9); 245 adev->vm_manager.block_size - 9);
246 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL) + i, tmp); 246 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL) + i, tmp);
247 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32) + i*2, 0); 247 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32) + i*2, 0);
248 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32) + i*2, 0); 248 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32) + i*2, 0);
@@ -317,36 +317,6 @@ void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, bool value)
317 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL), tmp); 317 WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL), tmp);
318} 318}
319 319
320static uint32_t mmhub_v1_0_get_invalidate_req(unsigned int vm_id)
321{
322 u32 req = 0;
323
324 /* invalidate using legacy mode on vm_id*/
325 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
326 PER_VMID_INVALIDATE_REQ, 1 << vm_id);
327 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0);
328 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
329 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
330 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
331 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
332 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
333 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
334 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
335
336 return req;
337}
338
339static uint32_t mmhub_v1_0_get_vm_protection_bits(void)
340{
341 return (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
342 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
343 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
344 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
345 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
346 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
347 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
348}
349
350static int mmhub_v1_0_early_init(void *handle) 320static int mmhub_v1_0_early_init(void *handle)
351{ 321{
352 return 0; 322 return 0;
@@ -379,9 +349,6 @@ static int mmhub_v1_0_sw_init(void *handle)
379 hub->vm_l2_pro_fault_cntl = 349 hub->vm_l2_pro_fault_cntl =
380 SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL); 350 SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
381 351
382 hub->get_invalidate_req = mmhub_v1_0_get_invalidate_req;
383 hub->get_vm_protection_bits = mmhub_v1_0_get_vm_protection_bits;
384
385 return 0; 352 return 0;
386} 353}
387 354
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index cfd5e54777bb..1493301b6a94 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -28,6 +28,7 @@
28#include "vega10/GC/gc_9_0_offset.h" 28#include "vega10/GC/gc_9_0_offset.h"
29#include "vega10/GC/gc_9_0_sh_mask.h" 29#include "vega10/GC/gc_9_0_sh_mask.h"
30#include "soc15.h" 30#include "soc15.h"
31#include "vega10_ih.h"
31#include "soc15_common.h" 32#include "soc15_common.h"
32#include "mxgpu_ai.h" 33#include "mxgpu_ai.h"
33 34
@@ -133,7 +134,7 @@ static int xgpu_ai_poll_ack(struct amdgpu_device *adev)
133 return r; 134 return r;
134} 135}
135 136
136static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum idh_event event) 137static int xgpu_ai_poll_msg(struct amdgpu_device *adev, enum idh_event event)
137{ 138{
138 int r = 0, timeout = AI_MAILBOX_TIMEDOUT; 139 int r = 0, timeout = AI_MAILBOX_TIMEDOUT;
139 140
@@ -172,7 +173,7 @@ static int xgpu_ai_send_access_requests(struct amdgpu_device *adev,
172 if (req == IDH_REQ_GPU_INIT_ACCESS || 173 if (req == IDH_REQ_GPU_INIT_ACCESS ||
173 req == IDH_REQ_GPU_FINI_ACCESS || 174 req == IDH_REQ_GPU_FINI_ACCESS ||
174 req == IDH_REQ_GPU_RESET_ACCESS) { 175 req == IDH_REQ_GPU_RESET_ACCESS) {
175 r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU); 176 r = xgpu_ai_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
176 if (r) 177 if (r)
177 return r; 178 return r;
178 } 179 }
@@ -180,6 +181,11 @@ static int xgpu_ai_send_access_requests(struct amdgpu_device *adev,
180 return 0; 181 return 0;
181} 182}
182 183
184static int xgpu_ai_request_reset(struct amdgpu_device *adev)
185{
186 return xgpu_ai_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS);
187}
188
183static int xgpu_ai_request_full_gpu_access(struct amdgpu_device *adev, 189static int xgpu_ai_request_full_gpu_access(struct amdgpu_device *adev,
184 bool init) 190 bool init)
185{ 191{
@@ -201,7 +207,134 @@ static int xgpu_ai_release_full_gpu_access(struct amdgpu_device *adev,
201 return r; 207 return r;
202} 208}
203 209
210static int xgpu_ai_mailbox_ack_irq(struct amdgpu_device *adev,
211 struct amdgpu_irq_src *source,
212 struct amdgpu_iv_entry *entry)
213{
214 DRM_DEBUG("get ack intr and do nothing.\n");
215 return 0;
216}
217
218static int xgpu_ai_set_mailbox_ack_irq(struct amdgpu_device *adev,
219 struct amdgpu_irq_src *source,
220 unsigned type,
221 enum amdgpu_interrupt_state state)
222{
223 u32 tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL));
224
225 tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_MAILBOX_INT_CNTL, ACK_INT_EN,
226 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
227 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL), tmp);
228
229 return 0;
230}
231
232static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
233{
234 struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work);
235 struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);
236
237 /* wait until RCV_MSG become 3 */
238 if (xgpu_ai_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL)) {
239 pr_err("failed to recieve FLR_CMPL\n");
240 return;
241 }
242
243 /* Trigger recovery due to world switch failure */
244 amdgpu_sriov_gpu_reset(adev, false);
245}
246
247static int xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device *adev,
248 struct amdgpu_irq_src *src,
249 unsigned type,
250 enum amdgpu_interrupt_state state)
251{
252 u32 tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL));
253
254 tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_MAILBOX_INT_CNTL, VALID_INT_EN,
255 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
256 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL), tmp);
257
258 return 0;
259}
260
261static int xgpu_ai_mailbox_rcv_irq(struct amdgpu_device *adev,
262 struct amdgpu_irq_src *source,
263 struct amdgpu_iv_entry *entry)
264{
265 int r;
266
267 /* see what event we get */
268 r = xgpu_ai_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
269
270 /* only handle FLR_NOTIFY now */
271 if (!r)
272 schedule_work(&adev->virt.flr_work);
273
274 return 0;
275}
276
277static const struct amdgpu_irq_src_funcs xgpu_ai_mailbox_ack_irq_funcs = {
278 .set = xgpu_ai_set_mailbox_ack_irq,
279 .process = xgpu_ai_mailbox_ack_irq,
280};
281
282static const struct amdgpu_irq_src_funcs xgpu_ai_mailbox_rcv_irq_funcs = {
283 .set = xgpu_ai_set_mailbox_rcv_irq,
284 .process = xgpu_ai_mailbox_rcv_irq,
285};
286
287void xgpu_ai_mailbox_set_irq_funcs(struct amdgpu_device *adev)
288{
289 adev->virt.ack_irq.num_types = 1;
290 adev->virt.ack_irq.funcs = &xgpu_ai_mailbox_ack_irq_funcs;
291 adev->virt.rcv_irq.num_types = 1;
292 adev->virt.rcv_irq.funcs = &xgpu_ai_mailbox_rcv_irq_funcs;
293}
294
295int xgpu_ai_mailbox_add_irq_id(struct amdgpu_device *adev)
296{
297 int r;
298
299 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 135, &adev->virt.rcv_irq);
300 if (r)
301 return r;
302
303 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 138, &adev->virt.ack_irq);
304 if (r) {
305 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
306 return r;
307 }
308
309 return 0;
310}
311
312int xgpu_ai_mailbox_get_irq(struct amdgpu_device *adev)
313{
314 int r;
315
316 r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
317 if (r)
318 return r;
319 r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
320 if (r) {
321 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
322 return r;
323 }
324
325 INIT_WORK(&adev->virt.flr_work, xgpu_ai_mailbox_flr_work);
326
327 return 0;
328}
329
330void xgpu_ai_mailbox_put_irq(struct amdgpu_device *adev)
331{
332 amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
333 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
334}
335
204const struct amdgpu_virt_ops xgpu_ai_virt_ops = { 336const struct amdgpu_virt_ops xgpu_ai_virt_ops = {
205 .req_full_gpu = xgpu_ai_request_full_gpu_access, 337 .req_full_gpu = xgpu_ai_request_full_gpu_access,
206 .rel_full_gpu = xgpu_ai_release_full_gpu_access, 338 .rel_full_gpu = xgpu_ai_release_full_gpu_access,
339 .reset_gpu = xgpu_ai_request_reset,
207}; 340};
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
index bf8ab8fd4367..9aefc44d2c34 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
@@ -24,7 +24,7 @@
24#ifndef __MXGPU_AI_H__ 24#ifndef __MXGPU_AI_H__
25#define __MXGPU_AI_H__ 25#define __MXGPU_AI_H__
26 26
27#define AI_MAILBOX_TIMEDOUT 150000 27#define AI_MAILBOX_TIMEDOUT 5000
28 28
29enum idh_request { 29enum idh_request {
30 IDH_REQ_GPU_INIT_ACCESS = 1, 30 IDH_REQ_GPU_INIT_ACCESS = 1,
@@ -44,4 +44,9 @@ enum idh_event {
44 44
45extern const struct amdgpu_virt_ops xgpu_ai_virt_ops; 45extern const struct amdgpu_virt_ops xgpu_ai_virt_ops;
46 46
47void xgpu_ai_mailbox_set_irq_funcs(struct amdgpu_device *adev);
48int xgpu_ai_mailbox_add_irq_id(struct amdgpu_device *adev);
49int xgpu_ai_mailbox_get_irq(struct amdgpu_device *adev);
50void xgpu_ai_mailbox_put_irq(struct amdgpu_device *adev);
51
47#endif 52#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index 5191c45ffdf3..c3588d1c7cb0 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -491,7 +491,7 @@ bool psp_v3_1_compare_sram_data(struct psp_context *psp,
491 491
492 ucode_size = ucode->ucode_size; 492 ucode_size = ucode->ucode_size;
493 ucode_mem = (uint32_t *)ucode->kaddr; 493 ucode_mem = (uint32_t *)ucode->kaddr;
494 while (!ucode_size) { 494 while (ucode_size) {
495 fw_sram_reg_val = RREG32(fw_sram_data_reg_offset); 495 fw_sram_reg_val = RREG32(fw_sram_data_reg_offset);
496 496
497 if (*ucode_mem != fw_sram_reg_val) 497 if (*ucode_mem != fw_sram_reg_val)
@@ -508,14 +508,10 @@ bool psp_v3_1_compare_sram_data(struct psp_context *psp,
508bool psp_v3_1_smu_reload_quirk(struct psp_context *psp) 508bool psp_v3_1_smu_reload_quirk(struct psp_context *psp)
509{ 509{
510 struct amdgpu_device *adev = psp->adev; 510 struct amdgpu_device *adev = psp->adev;
511 uint32_t reg, reg_val; 511 uint32_t reg;
512 512
513 reg_val = (smnMP1_FIRMWARE_FLAGS & 0xffffffff) | 0x03b00000; 513 reg = smnMP1_FIRMWARE_FLAGS | 0x03b00000;
514 WREG32(SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2), reg_val); 514 WREG32(SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2), reg);
515 reg = RREG32(SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2)); 515 reg = RREG32(SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2));
516 if ((reg & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) >> 516 return (reg & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) ? true : false;
517 MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT)
518 return true;
519
520 return false;
521} 517}
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 2dd2b20d727e..21f38d882335 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -1039,6 +1039,7 @@ static void sdma_v4_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1039static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 1039static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1040 unsigned vm_id, uint64_t pd_addr) 1040 unsigned vm_id, uint64_t pd_addr)
1041{ 1041{
1042 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
1042 unsigned eng = ring->idx; 1043 unsigned eng = ring->idx;
1043 unsigned i; 1044 unsigned i;
1044 1045
@@ -1048,7 +1049,6 @@ static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1048 1049
1049 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 1050 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
1050 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i]; 1051 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i];
1051 uint32_t req = hub->get_invalidate_req(vm_id);
1052 1052
1053 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | 1053 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1054 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); 1054 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index bb14a45997b5..385de8617075 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -106,6 +106,8 @@ static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg)
106 106
107 if (adev->asic_type == CHIP_VEGA10) 107 if (adev->asic_type == CHIP_VEGA10)
108 nbio_pcie_id = &nbio_v6_1_pcie_index_data; 108 nbio_pcie_id = &nbio_v6_1_pcie_index_data;
109 else
110 BUG();
109 111
110 address = nbio_pcie_id->index_offset; 112 address = nbio_pcie_id->index_offset;
111 data = nbio_pcie_id->data_offset; 113 data = nbio_pcie_id->data_offset;
@@ -125,6 +127,8 @@ static void soc15_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
125 127
126 if (adev->asic_type == CHIP_VEGA10) 128 if (adev->asic_type == CHIP_VEGA10)
127 nbio_pcie_id = &nbio_v6_1_pcie_index_data; 129 nbio_pcie_id = &nbio_v6_1_pcie_index_data;
130 else
131 BUG();
128 132
129 address = nbio_pcie_id->index_offset; 133 address = nbio_pcie_id->index_offset;
130 data = nbio_pcie_id->data_offset; 134 data = nbio_pcie_id->data_offset;
@@ -493,7 +497,8 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev)
493 amdgpu_ip_block_add(adev, &mmhub_v1_0_ip_block); 497 amdgpu_ip_block_add(adev, &mmhub_v1_0_ip_block);
494 amdgpu_ip_block_add(adev, &gmc_v9_0_ip_block); 498 amdgpu_ip_block_add(adev, &gmc_v9_0_ip_block);
495 amdgpu_ip_block_add(adev, &vega10_ih_ip_block); 499 amdgpu_ip_block_add(adev, &vega10_ih_ip_block);
496 amdgpu_ip_block_add(adev, &psp_v3_1_ip_block); 500 if (amdgpu_fw_load_type == 2 || amdgpu_fw_load_type == -1)
501 amdgpu_ip_block_add(adev, &psp_v3_1_ip_block);
497 if (!amdgpu_sriov_vf(adev)) 502 if (!amdgpu_sriov_vf(adev))
498 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block); 503 amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
499 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) 504 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
@@ -558,6 +563,7 @@ static int soc15_common_early_init(void *handle)
558 563
559 if (amdgpu_sriov_vf(adev)) { 564 if (amdgpu_sriov_vf(adev)) {
560 amdgpu_virt_init_setting(adev); 565 amdgpu_virt_init_setting(adev);
566 xgpu_ai_mailbox_set_irq_funcs(adev);
561 } 567 }
562 568
563 /* 569 /*
@@ -610,8 +616,23 @@ static int soc15_common_early_init(void *handle)
610 return 0; 616 return 0;
611} 617}
612 618
619static int soc15_common_late_init(void *handle)
620{
621 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
622
623 if (amdgpu_sriov_vf(adev))
624 xgpu_ai_mailbox_get_irq(adev);
625
626 return 0;
627}
628
613static int soc15_common_sw_init(void *handle) 629static int soc15_common_sw_init(void *handle)
614{ 630{
631 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
632
633 if (amdgpu_sriov_vf(adev))
634 xgpu_ai_mailbox_add_irq_id(adev);
635
615 return 0; 636 return 0;
616} 637}
617 638
@@ -642,6 +663,8 @@ static int soc15_common_hw_fini(void *handle)
642 663
643 /* disable the doorbell aperture */ 664 /* disable the doorbell aperture */
644 soc15_enable_doorbell_aperture(adev, false); 665 soc15_enable_doorbell_aperture(adev, false);
666 if (amdgpu_sriov_vf(adev))
667 xgpu_ai_mailbox_put_irq(adev);
645 668
646 return 0; 669 return 0;
647} 670}
@@ -855,7 +878,7 @@ static int soc15_common_set_powergating_state(void *handle,
855const struct amd_ip_funcs soc15_common_ip_funcs = { 878const struct amd_ip_funcs soc15_common_ip_funcs = {
856 .name = "soc15_common", 879 .name = "soc15_common",
857 .early_init = soc15_common_early_init, 880 .early_init = soc15_common_early_init,
858 .late_init = NULL, 881 .late_init = soc15_common_late_init,
859 .sw_init = soc15_common_sw_init, 882 .sw_init = soc15_common_sw_init,
860 .sw_fini = soc15_common_sw_fini, 883 .sw_fini = soc15_common_sw_fini,
861 .hw_init = soc15_common_hw_init, 884 .hw_init = soc15_common_hw_init,
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index 9a4129d881aa..8ab0f78794a5 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -135,12 +135,9 @@ static int uvd_v4_2_sw_fini(void *handle)
135 if (r) 135 if (r)
136 return r; 136 return r;
137 137
138 r = amdgpu_uvd_sw_fini(adev); 138 return amdgpu_uvd_sw_fini(adev);
139 if (r)
140 return r;
141
142 return r;
143} 139}
140
144static void uvd_v4_2_enable_mgcg(struct amdgpu_device *adev, 141static void uvd_v4_2_enable_mgcg(struct amdgpu_device *adev,
145 bool enable); 142 bool enable);
146/** 143/**
@@ -230,11 +227,7 @@ static int uvd_v4_2_suspend(void *handle)
230 if (r) 227 if (r)
231 return r; 228 return r;
232 229
233 r = amdgpu_uvd_suspend(adev); 230 return amdgpu_uvd_suspend(adev);
234 if (r)
235 return r;
236
237 return r;
238} 231}
239 232
240static int uvd_v4_2_resume(void *handle) 233static int uvd_v4_2_resume(void *handle)
@@ -246,11 +239,7 @@ static int uvd_v4_2_resume(void *handle)
246 if (r) 239 if (r)
247 return r; 240 return r;
248 241
249 r = uvd_v4_2_hw_init(adev); 242 return uvd_v4_2_hw_init(adev);
250 if (r)
251 return r;
252
253 return r;
254} 243}
255 244
256/** 245/**
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index e448f7d86bc0..bb6d46e168a3 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -131,11 +131,7 @@ static int uvd_v5_0_sw_fini(void *handle)
131 if (r) 131 if (r)
132 return r; 132 return r;
133 133
134 r = amdgpu_uvd_sw_fini(adev); 134 return amdgpu_uvd_sw_fini(adev);
135 if (r)
136 return r;
137
138 return r;
139} 135}
140 136
141/** 137/**
@@ -228,11 +224,7 @@ static int uvd_v5_0_suspend(void *handle)
228 return r; 224 return r;
229 uvd_v5_0_set_clockgating_state(adev, AMD_CG_STATE_GATE); 225 uvd_v5_0_set_clockgating_state(adev, AMD_CG_STATE_GATE);
230 226
231 r = amdgpu_uvd_suspend(adev); 227 return amdgpu_uvd_suspend(adev);
232 if (r)
233 return r;
234
235 return r;
236} 228}
237 229
238static int uvd_v5_0_resume(void *handle) 230static int uvd_v5_0_resume(void *handle)
@@ -244,11 +236,7 @@ static int uvd_v5_0_resume(void *handle)
244 if (r) 236 if (r)
245 return r; 237 return r;
246 238
247 r = uvd_v5_0_hw_init(adev); 239 return uvd_v5_0_hw_init(adev);
248 if (r)
249 return r;
250
251 return r;
252} 240}
253 241
254/** 242/**
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index 5679a4249bd9..31db356476f8 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -134,11 +134,7 @@ static int uvd_v6_0_sw_fini(void *handle)
134 if (r) 134 if (r)
135 return r; 135 return r;
136 136
137 r = amdgpu_uvd_sw_fini(adev); 137 return amdgpu_uvd_sw_fini(adev);
138 if (r)
139 return r;
140
141 return r;
142} 138}
143 139
144/** 140/**
@@ -230,11 +226,8 @@ static int uvd_v6_0_suspend(void *handle)
230 return r; 226 return r;
231 227
232 /* Skip this for APU for now */ 228 /* Skip this for APU for now */
233 if (!(adev->flags & AMD_IS_APU)) { 229 if (!(adev->flags & AMD_IS_APU))
234 r = amdgpu_uvd_suspend(adev); 230 r = amdgpu_uvd_suspend(adev);
235 if (r)
236 return r;
237 }
238 231
239 return r; 232 return r;
240} 233}
@@ -250,11 +243,7 @@ static int uvd_v6_0_resume(void *handle)
250 if (r) 243 if (r)
251 return r; 244 return r;
252 } 245 }
253 r = uvd_v6_0_hw_init(adev); 246 return uvd_v6_0_hw_init(adev);
254 if (r)
255 return r;
256
257 return r;
258} 247}
259 248
260/** 249/**
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index 13f52e0af9b8..9bcf01469282 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -438,11 +438,7 @@ static int uvd_v7_0_sw_fini(void *handle)
438 for (i = 0; i < adev->uvd.num_enc_rings; ++i) 438 for (i = 0; i < adev->uvd.num_enc_rings; ++i)
439 amdgpu_ring_fini(&adev->uvd.ring_enc[i]); 439 amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
440 440
441 r = amdgpu_uvd_sw_fini(adev); 441 return amdgpu_uvd_sw_fini(adev);
442 if (r)
443 return r;
444
445 return r;
446} 442}
447 443
448/** 444/**
@@ -547,11 +543,8 @@ static int uvd_v7_0_suspend(void *handle)
547 return r; 543 return r;
548 544
549 /* Skip this for APU for now */ 545 /* Skip this for APU for now */
550 if (!(adev->flags & AMD_IS_APU)) { 546 if (!(adev->flags & AMD_IS_APU))
551 r = amdgpu_uvd_suspend(adev); 547 r = amdgpu_uvd_suspend(adev);
552 if (r)
553 return r;
554 }
555 548
556 return r; 549 return r;
557} 550}
@@ -567,11 +560,7 @@ static int uvd_v7_0_resume(void *handle)
567 if (r) 560 if (r)
568 return r; 561 return r;
569 } 562 }
570 r = uvd_v7_0_hw_init(adev); 563 return uvd_v7_0_hw_init(adev);
571 if (r)
572 return r;
573
574 return r;
575} 564}
576 565
577/** 566/**
@@ -1045,6 +1034,7 @@ static void uvd_v7_0_vm_reg_wait(struct amdgpu_ring *ring,
1045static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 1034static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1046 unsigned vm_id, uint64_t pd_addr) 1035 unsigned vm_id, uint64_t pd_addr)
1047{ 1036{
1037 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
1048 uint32_t data0, data1, mask; 1038 uint32_t data0, data1, mask;
1049 unsigned eng = ring->idx; 1039 unsigned eng = ring->idx;
1050 unsigned i; 1040 unsigned i;
@@ -1055,7 +1045,6 @@ static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1055 1045
1056 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 1046 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
1057 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i]; 1047 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i];
1058 uint32_t req = hub->get_invalidate_req(vm_id);
1059 1048
1060 data0 = (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2; 1049 data0 = (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2;
1061 data1 = upper_32_bits(pd_addr); 1050 data1 = upper_32_bits(pd_addr);
@@ -1091,6 +1080,7 @@ static void uvd_v7_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1091static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring, 1080static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1092 unsigned int vm_id, uint64_t pd_addr) 1081 unsigned int vm_id, uint64_t pd_addr)
1093{ 1082{
1083 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
1094 unsigned eng = ring->idx; 1084 unsigned eng = ring->idx;
1095 unsigned i; 1085 unsigned i;
1096 1086
@@ -1100,7 +1090,6 @@ static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1100 1090
1101 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 1091 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
1102 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i]; 1092 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i];
1103 uint32_t req = hub->get_invalidate_req(vm_id);
1104 1093
1105 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE); 1094 amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE);
1106 amdgpu_ring_write(ring, 1095 amdgpu_ring_write(ring,
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index 49a6c45e65be..47f70827195b 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -451,11 +451,7 @@ static int vce_v2_0_sw_fini(void *handle)
451 if (r) 451 if (r)
452 return r; 452 return r;
453 453
454 r = amdgpu_vce_sw_fini(adev); 454 return amdgpu_vce_sw_fini(adev);
455 if (r)
456 return r;
457
458 return r;
459} 455}
460 456
461static int vce_v2_0_hw_init(void *handle) 457static int vce_v2_0_hw_init(void *handle)
@@ -495,11 +491,7 @@ static int vce_v2_0_suspend(void *handle)
495 if (r) 491 if (r)
496 return r; 492 return r;
497 493
498 r = amdgpu_vce_suspend(adev); 494 return amdgpu_vce_suspend(adev);
499 if (r)
500 return r;
501
502 return r;
503} 495}
504 496
505static int vce_v2_0_resume(void *handle) 497static int vce_v2_0_resume(void *handle)
@@ -511,11 +503,7 @@ static int vce_v2_0_resume(void *handle)
511 if (r) 503 if (r)
512 return r; 504 return r;
513 505
514 r = vce_v2_0_hw_init(adev); 506 return vce_v2_0_hw_init(adev);
515 if (r)
516 return r;
517
518 return r;
519} 507}
520 508
521static int vce_v2_0_soft_reset(void *handle) 509static int vce_v2_0_soft_reset(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index db0adac073c6..fb0819359909 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -417,11 +417,7 @@ static int vce_v3_0_sw_fini(void *handle)
417 if (r) 417 if (r)
418 return r; 418 return r;
419 419
420 r = amdgpu_vce_sw_fini(adev); 420 return amdgpu_vce_sw_fini(adev);
421 if (r)
422 return r;
423
424 return r;
425} 421}
426 422
427static int vce_v3_0_hw_init(void *handle) 423static int vce_v3_0_hw_init(void *handle)
@@ -471,11 +467,7 @@ static int vce_v3_0_suspend(void *handle)
471 if (r) 467 if (r)
472 return r; 468 return r;
473 469
474 r = amdgpu_vce_suspend(adev); 470 return amdgpu_vce_suspend(adev);
475 if (r)
476 return r;
477
478 return r;
479} 471}
480 472
481static int vce_v3_0_resume(void *handle) 473static int vce_v3_0_resume(void *handle)
@@ -487,11 +479,7 @@ static int vce_v3_0_resume(void *handle)
487 if (r) 479 if (r)
488 return r; 480 return r;
489 481
490 r = vce_v3_0_hw_init(adev); 482 return vce_v3_0_hw_init(adev);
491 if (r)
492 return r;
493
494 return r;
495} 483}
496 484
497static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx) 485static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx)
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index becc5f744a98..edde5fe938d6 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -527,11 +527,7 @@ static int vce_v4_0_sw_fini(void *handle)
527 if (r) 527 if (r)
528 return r; 528 return r;
529 529
530 r = amdgpu_vce_sw_fini(adev); 530 return amdgpu_vce_sw_fini(adev);
531 if (r)
532 return r;
533
534 return r;
535} 531}
536 532
537static int vce_v4_0_hw_init(void *handle) 533static int vce_v4_0_hw_init(void *handle)
@@ -584,11 +580,7 @@ static int vce_v4_0_suspend(void *handle)
584 if (r) 580 if (r)
585 return r; 581 return r;
586 582
587 r = amdgpu_vce_suspend(adev); 583 return amdgpu_vce_suspend(adev);
588 if (r)
589 return r;
590
591 return r;
592} 584}
593 585
594static int vce_v4_0_resume(void *handle) 586static int vce_v4_0_resume(void *handle)
@@ -600,11 +592,7 @@ static int vce_v4_0_resume(void *handle)
600 if (r) 592 if (r)
601 return r; 593 return r;
602 594
603 r = vce_v4_0_hw_init(adev); 595 return vce_v4_0_hw_init(adev);
604 if (r)
605 return r;
606
607 return r;
608} 596}
609 597
610static void vce_v4_0_mc_resume(struct amdgpu_device *adev) 598static void vce_v4_0_mc_resume(struct amdgpu_device *adev)
@@ -985,6 +973,7 @@ static void vce_v4_0_ring_insert_end(struct amdgpu_ring *ring)
985static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring, 973static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring,
986 unsigned int vm_id, uint64_t pd_addr) 974 unsigned int vm_id, uint64_t pd_addr)
987{ 975{
976 uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
988 unsigned eng = ring->idx; 977 unsigned eng = ring->idx;
989 unsigned i; 978 unsigned i;
990 979
@@ -994,7 +983,6 @@ static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring,
994 983
995 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 984 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
996 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i]; 985 struct amdgpu_vmhub *hub = &ring->adev->vmhub[i];
997 uint32_t req = hub->get_invalidate_req(vm_id);
998 986
999 amdgpu_ring_write(ring, VCE_CMD_REG_WRITE); 987 amdgpu_ring_write(ring, VCE_CMD_REG_WRITE);
1000 amdgpu_ring_write(ring, 988 amdgpu_ring_write(ring,
diff --git a/drivers/gpu/drm/amd/amdgpu/vid.h b/drivers/gpu/drm/amd/amdgpu/vid.h
index b3a86e0e96e6..5f2ab9c1609a 100644
--- a/drivers/gpu/drm/amd/amdgpu/vid.h
+++ b/drivers/gpu/drm/amd/amdgpu/vid.h
@@ -362,7 +362,89 @@
362#define PACKET3_WAIT_ON_DE_COUNTER_DIFF 0x88 362#define PACKET3_WAIT_ON_DE_COUNTER_DIFF 0x88
363#define PACKET3_SWITCH_BUFFER 0x8B 363#define PACKET3_SWITCH_BUFFER 0x8B
364#define PACKET3_SET_RESOURCES 0xA0 364#define PACKET3_SET_RESOURCES 0xA0
365/* 1. header
366 * 2. CONTROL
367 * 3. QUEUE_MASK_LO [31:0]
368 * 4. QUEUE_MASK_HI [31:0]
369 * 5. GWS_MASK_LO [31:0]
370 * 6. GWS_MASK_HI [31:0]
371 * 7. OAC_MASK [15:0]
372 * 8. GDS_HEAP_SIZE [16:11] | GDS_HEAP_BASE [5:0]
373 */
374# define PACKET3_SET_RESOURCES_VMID_MASK(x) ((x) << 0)
375# define PACKET3_SET_RESOURCES_UNMAP_LATENTY(x) ((x) << 16)
376# define PACKET3_SET_RESOURCES_QUEUE_TYPE(x) ((x) << 29)
365#define PACKET3_MAP_QUEUES 0xA2 377#define PACKET3_MAP_QUEUES 0xA2
378/* 1. header
379 * 2. CONTROL
380 * 3. CONTROL2
381 * 4. MQD_ADDR_LO [31:0]
382 * 5. MQD_ADDR_HI [31:0]
383 * 6. WPTR_ADDR_LO [31:0]
384 * 7. WPTR_ADDR_HI [31:0]
385 */
386/* CONTROL */
387# define PACKET3_MAP_QUEUES_QUEUE_SEL(x) ((x) << 4)
388# define PACKET3_MAP_QUEUES_VMID(x) ((x) << 8)
389# define PACKET3_MAP_QUEUES_QUEUE_TYPE(x) ((x) << 21)
390# define PACKET3_MAP_QUEUES_ALLOC_FORMAT(x) ((x) << 24)
391# define PACKET3_MAP_QUEUES_ENGINE_SEL(x) ((x) << 26)
392# define PACKET3_MAP_QUEUES_NUM_QUEUES(x) ((x) << 29)
393/* CONTROL2 */
394# define PACKET3_MAP_QUEUES_CHECK_DISABLE(x) ((x) << 1)
395# define PACKET3_MAP_QUEUES_DOORBELL_OFFSET(x) ((x) << 2)
396# define PACKET3_MAP_QUEUES_QUEUE(x) ((x) << 26)
397# define PACKET3_MAP_QUEUES_PIPE(x) ((x) << 29)
398# define PACKET3_MAP_QUEUES_ME(x) ((x) << 31)
399#define PACKET3_UNMAP_QUEUES 0xA3
400/* 1. header
401 * 2. CONTROL
402 * 3. CONTROL2
403 * 4. CONTROL3
404 * 5. CONTROL4
405 * 6. CONTROL5
406 */
407/* CONTROL */
408# define PACKET3_UNMAP_QUEUES_ACTION(x) ((x) << 0)
409 /* 0 - PREEMPT_QUEUES
410 * 1 - RESET_QUEUES
411 * 2 - DISABLE_PROCESS_QUEUES
412 * 3 - PREEMPT_QUEUES_NO_UNMAP
413 */
414# define PACKET3_UNMAP_QUEUES_QUEUE_SEL(x) ((x) << 4)
415# define PACKET3_UNMAP_QUEUES_ENGINE_SEL(x) ((x) << 26)
416# define PACKET3_UNMAP_QUEUES_NUM_QUEUES(x) ((x) << 29)
417/* CONTROL2a */
418# define PACKET3_UNMAP_QUEUES_PASID(x) ((x) << 0)
419/* CONTROL2b */
420# define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(x) ((x) << 2)
421/* CONTROL3a */
422# define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET1(x) ((x) << 2)
423/* CONTROL3b */
424# define PACKET3_UNMAP_QUEUES_RB_WPTR(x) ((x) << 0)
425/* CONTROL4 */
426# define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET2(x) ((x) << 2)
427/* CONTROL5 */
428# define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET3(x) ((x) << 2)
429#define PACKET3_QUERY_STATUS 0xA4
430/* 1. header
431 * 2. CONTROL
432 * 3. CONTROL2
433 * 4. ADDR_LO [31:0]
434 * 5. ADDR_HI [31:0]
435 * 6. DATA_LO [31:0]
436 * 7. DATA_HI [31:0]
437 */
438/* CONTROL */
439# define PACKET3_QUERY_STATUS_CONTEXT_ID(x) ((x) << 0)
440# define PACKET3_QUERY_STATUS_INTERRUPT_SEL(x) ((x) << 28)
441# define PACKET3_QUERY_STATUS_COMMAND(x) ((x) << 30)
442/* CONTROL2a */
443# define PACKET3_QUERY_STATUS_PASID(x) ((x) << 0)
444/* CONTROL2b */
445# define PACKET3_QUERY_STATUS_DOORBELL_OFFSET(x) ((x) << 2)
446# define PACKET3_QUERY_STATUS_ENG_SEL(x) ((x) << 25)
447
366 448
367#define VCE_CMD_NO_OP 0x00000000 449#define VCE_CMD_NO_OP 0x00000000
368#define VCE_CMD_END 0x00000001 450#define VCE_CMD_END 0x00000001
diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
index dfd4fe6f0578..9da5b0bb66d8 100644
--- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
@@ -493,8 +493,10 @@ static int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_event event_id,
493 { 493 {
494 enum amd_pm_state_type ps; 494 enum amd_pm_state_type ps;
495 495
496 if (input == NULL) 496 if (input == NULL) {
497 return -EINVAL; 497 ret = -EINVAL;
498 break;
499 }
498 ps = *(unsigned long *)input; 500 ps = *(unsigned long *)input;
499 501
500 data.requested_ui_label = power_state_convert(ps); 502 data.requested_ui_label = power_state_convert(ps);
@@ -539,15 +541,19 @@ static enum amd_pm_state_type pp_dpm_get_current_power_state(void *handle)
539 switch (state->classification.ui_label) { 541 switch (state->classification.ui_label) {
540 case PP_StateUILabel_Battery: 542 case PP_StateUILabel_Battery:
541 pm_type = POWER_STATE_TYPE_BATTERY; 543 pm_type = POWER_STATE_TYPE_BATTERY;
544 break;
542 case PP_StateUILabel_Balanced: 545 case PP_StateUILabel_Balanced:
543 pm_type = POWER_STATE_TYPE_BALANCED; 546 pm_type = POWER_STATE_TYPE_BALANCED;
547 break;
544 case PP_StateUILabel_Performance: 548 case PP_StateUILabel_Performance:
545 pm_type = POWER_STATE_TYPE_PERFORMANCE; 549 pm_type = POWER_STATE_TYPE_PERFORMANCE;
550 break;
546 default: 551 default:
547 if (state->classification.flags & PP_StateClassificationFlag_Boot) 552 if (state->classification.flags & PP_StateClassificationFlag_Boot)
548 pm_type = POWER_STATE_TYPE_INTERNAL_BOOT; 553 pm_type = POWER_STATE_TYPE_INTERNAL_BOOT;
549 else 554 else
550 pm_type = POWER_STATE_TYPE_DEFAULT; 555 pm_type = POWER_STATE_TYPE_DEFAULT;
556 break;
551 } 557 }
552 mutex_unlock(&pp_handle->pp_lock); 558 mutex_unlock(&pp_handle->pp_lock);
553 559
@@ -894,7 +900,7 @@ static int pp_dpm_set_sclk_od(void *handle, uint32_t value)
894 900
895 mutex_lock(&pp_handle->pp_lock); 901 mutex_lock(&pp_handle->pp_lock);
896 ret = hwmgr->hwmgr_func->set_sclk_od(hwmgr, value); 902 ret = hwmgr->hwmgr_func->set_sclk_od(hwmgr, value);
897 mutex_lock(&pp_handle->pp_lock); 903 mutex_unlock(&pp_handle->pp_lock);
898 return ret; 904 return ret;
899} 905}
900 906
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu9_driver_if.h b/drivers/gpu/drm/amd/powerplay/inc/smu9_driver_if.h
index aee021451d35..2037910adcb1 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/smu9_driver_if.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu9_driver_if.h
@@ -30,7 +30,9 @@
30 * SMU TEAM: Always increment the interface version if 30 * SMU TEAM: Always increment the interface version if
31 * any structure is changed in this file 31 * any structure is changed in this file
32 */ 32 */
33#define SMU9_DRIVER_IF_VERSION 0xa 33#define SMU9_DRIVER_IF_VERSION 0xB
34
35#define PPTABLE_V10_SMU_VERSION 1
34 36
35#define NUM_GFXCLK_DPM_LEVELS 8 37#define NUM_GFXCLK_DPM_LEVELS 8
36#define NUM_UVD_DPM_LEVELS 8 38#define NUM_UVD_DPM_LEVELS 8
@@ -87,6 +89,11 @@ typedef struct {
87 int32_t a0; 89 int32_t a0;
88 int32_t a1; 90 int32_t a1;
89 int32_t a2; 91 int32_t a2;
92
93 uint8_t a0_shift;
94 uint8_t a1_shift;
95 uint8_t a2_shift;
96 uint8_t padding;
90} GbVdroopTable_t; 97} GbVdroopTable_t;
91 98
92typedef struct { 99typedef struct {
@@ -293,7 +300,9 @@ typedef struct {
293 uint16_t Platform_sigma; 300 uint16_t Platform_sigma;
294 uint16_t PSM_Age_CompFactor; 301 uint16_t PSM_Age_CompFactor;
295 302
296 uint32_t Reserved[20]; 303 uint32_t DpmLevelPowerDelta;
304
305 uint32_t Reserved[19];
297 306
298 /* Padding - ignore */ 307 /* Padding - ignore */
299 uint32_t MmHubPadding[7]; /* SMU internal use */ 308 uint32_t MmHubPadding[7]; /* SMU internal use */
@@ -350,8 +359,8 @@ typedef struct {
350typedef struct { 359typedef struct {
351 uint16_t avgPsmCount[30]; 360 uint16_t avgPsmCount[30];
352 uint16_t minPsmCount[30]; 361 uint16_t minPsmCount[30];
353 uint16_t avgPsmVoltage[30]; /* in mV with 2 fractional bits */ 362 float avgPsmVoltage[30];
354 uint16_t minPsmVoltage[30]; /* in mV with 2 fractional bits */ 363 float minPsmVoltage[30];
355 364
356 uint32_t MmHubPadding[7]; /* SMU internal use */ 365 uint32_t MmHubPadding[7]; /* SMU internal use */
357} AvfsDebugTable_t; 366} AvfsDebugTable_t;
@@ -414,5 +423,45 @@ typedef struct {
414#define UCLK_SWITCH_SLOW 0 423#define UCLK_SWITCH_SLOW 0
415#define UCLK_SWITCH_FAST 1 424#define UCLK_SWITCH_FAST 1
416 425
426/* GFX DIDT Configuration */
427#define SQ_Enable_MASK 0x1
428#define SQ_IR_MASK 0x2
429#define SQ_PCC_MASK 0x4
430#define SQ_EDC_MASK 0x8
431
432#define TCP_Enable_MASK 0x100
433#define TCP_IR_MASK 0x200
434#define TCP_PCC_MASK 0x400
435#define TCP_EDC_MASK 0x800
436
437#define TD_Enable_MASK 0x10000
438#define TD_IR_MASK 0x20000
439#define TD_PCC_MASK 0x40000
440#define TD_EDC_MASK 0x80000
441
442#define DB_Enable_MASK 0x1000000
443#define DB_IR_MASK 0x2000000
444#define DB_PCC_MASK 0x4000000
445#define DB_EDC_MASK 0x8000000
446
447#define SQ_Enable_SHIFT 0
448#define SQ_IR_SHIFT 1
449#define SQ_PCC_SHIFT 2
450#define SQ_EDC_SHIFT 3
451
452#define TCP_Enable_SHIFT 8
453#define TCP_IR_SHIFT 9
454#define TCP_PCC_SHIFT 10
455#define TCP_EDC_SHIFT 11
456
457#define TD_Enable_SHIFT 16
458#define TD_IR_SHIFT 17
459#define TD_PCC_SHIFT 18
460#define TD_EDC_SHIFT 19
461
462#define DB_Enable_SHIFT 24
463#define DB_IR_SHIFT 25
464#define DB_PCC_SHIFT 26
465#define DB_EDC_SHIFT 27
417 466
418#endif 467#endif
diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index 50c910efa13d..e879496b8a42 100644
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -236,6 +236,7 @@ struct ttm_bo_driver ast_bo_driver = {
236 .verify_access = ast_bo_verify_access, 236 .verify_access = ast_bo_verify_access,
237 .io_mem_reserve = &ast_ttm_io_mem_reserve, 237 .io_mem_reserve = &ast_ttm_io_mem_reserve,
238 .io_mem_free = &ast_ttm_io_mem_free, 238 .io_mem_free = &ast_ttm_io_mem_free,
239 .io_mem_pfn = ttm_bo_default_io_mem_pfn,
239}; 240};
240 241
241int ast_mm_init(struct ast_private *ast) 242int ast_mm_init(struct ast_private *ast)
diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c
index 857755ac2d70..c4cadb638460 100644
--- a/drivers/gpu/drm/bochs/bochs_mm.c
+++ b/drivers/gpu/drm/bochs/bochs_mm.c
@@ -205,6 +205,7 @@ struct ttm_bo_driver bochs_bo_driver = {
205 .verify_access = bochs_bo_verify_access, 205 .verify_access = bochs_bo_verify_access,
206 .io_mem_reserve = &bochs_ttm_io_mem_reserve, 206 .io_mem_reserve = &bochs_ttm_io_mem_reserve,
207 .io_mem_free = &bochs_ttm_io_mem_free, 207 .io_mem_free = &bochs_ttm_io_mem_free,
208 .io_mem_pfn = ttm_bo_default_io_mem_pfn,
208}; 209};
209 210
210int bochs_mm_init(struct bochs_device *bochs) 211int bochs_mm_init(struct bochs_device *bochs)
diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c
index f53aa8f4a143..93dbcd38355d 100644
--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c
+++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c
@@ -236,6 +236,7 @@ struct ttm_bo_driver cirrus_bo_driver = {
236 .verify_access = cirrus_bo_verify_access, 236 .verify_access = cirrus_bo_verify_access,
237 .io_mem_reserve = &cirrus_ttm_io_mem_reserve, 237 .io_mem_reserve = &cirrus_ttm_io_mem_reserve,
238 .io_mem_free = &cirrus_ttm_io_mem_free, 238 .io_mem_free = &cirrus_ttm_io_mem_free,
239 .io_mem_pfn = ttm_bo_default_io_mem_pfn,
239}; 240};
240 241
241int cirrus_mm_init(struct cirrus_device *cirrus) 242int cirrus_mm_init(struct cirrus_device *cirrus)
diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c
index 657598bb1e6b..565a217b46f2 100644
--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c
+++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c
@@ -236,6 +236,7 @@ struct ttm_bo_driver mgag200_bo_driver = {
236 .verify_access = mgag200_bo_verify_access, 236 .verify_access = mgag200_bo_verify_access,
237 .io_mem_reserve = &mgag200_ttm_io_mem_reserve, 237 .io_mem_reserve = &mgag200_ttm_io_mem_reserve,
238 .io_mem_free = &mgag200_ttm_io_mem_free, 238 .io_mem_free = &mgag200_ttm_io_mem_free,
239 .io_mem_pfn = ttm_bo_default_io_mem_pfn,
239}; 240};
240 241
241int mgag200_mm_init(struct mga_device *mdev) 242int mgag200_mm_init(struct mga_device *mdev)
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 548f36d33924..e427f80344c4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -1574,6 +1574,7 @@ struct ttm_bo_driver nouveau_bo_driver = {
1574 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify, 1574 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
1575 .io_mem_reserve = &nouveau_ttm_io_mem_reserve, 1575 .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
1576 .io_mem_free = &nouveau_ttm_io_mem_free, 1576 .io_mem_free = &nouveau_ttm_io_mem_free,
1577 .io_mem_pfn = ttm_bo_default_io_mem_pfn,
1577}; 1578};
1578 1579
1579struct nvkm_vma * 1580struct nvkm_vma *
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index 7d1cab57c89e..0fdedee4509d 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -393,6 +393,7 @@ static struct ttm_bo_driver qxl_bo_driver = {
393 .verify_access = &qxl_verify_access, 393 .verify_access = &qxl_verify_access,
394 .io_mem_reserve = &qxl_ttm_io_mem_reserve, 394 .io_mem_reserve = &qxl_ttm_io_mem_reserve,
395 .io_mem_free = &qxl_ttm_io_mem_free, 395 .io_mem_free = &qxl_ttm_io_mem_free,
396 .io_mem_pfn = ttm_bo_default_io_mem_pfn,
396 .move_notify = &qxl_bo_move_notify, 397 .move_notify = &qxl_bo_move_notify,
397}; 398};
398 399
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 684f1703aa5c..bfb06469db46 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -873,6 +873,7 @@ static struct ttm_bo_driver radeon_bo_driver = {
873 .fault_reserve_notify = &radeon_bo_fault_reserve_notify, 873 .fault_reserve_notify = &radeon_bo_fault_reserve_notify,
874 .io_mem_reserve = &radeon_ttm_io_mem_reserve, 874 .io_mem_reserve = &radeon_ttm_io_mem_reserve,
875 .io_mem_free = &radeon_ttm_io_mem_free, 875 .io_mem_free = &radeon_ttm_io_mem_free,
876 .io_mem_pfn = ttm_bo_default_io_mem_pfn,
876}; 877};
877 878
878int radeon_ttm_init(struct radeon_device *rdev) 879int radeon_ttm_init(struct radeon_device *rdev)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 412240a3ba90..e44626a2e698 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -1020,37 +1020,44 @@ out_unlock:
1020 return ret; 1020 return ret;
1021} 1021}
1022 1022
1023bool ttm_bo_mem_compat(struct ttm_placement *placement, 1023static bool ttm_bo_places_compat(const struct ttm_place *places,
1024 struct ttm_mem_reg *mem, 1024 unsigned num_placement,
1025 uint32_t *new_flags) 1025 struct ttm_mem_reg *mem,
1026 uint32_t *new_flags)
1026{ 1027{
1027 int i; 1028 unsigned i;
1028
1029 for (i = 0; i < placement->num_placement; i++) {
1030 const struct ttm_place *heap = &placement->placement[i];
1031 if (mem->mm_node &&
1032 (mem->start < heap->fpfn ||
1033 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1034 continue;
1035 1029
1036 *new_flags = heap->flags; 1030 for (i = 0; i < num_placement; i++) {
1037 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1031 const struct ttm_place *heap = &places[i];
1038 (*new_flags & mem->placement & TTM_PL_MASK_MEM))
1039 return true;
1040 }
1041 1032
1042 for (i = 0; i < placement->num_busy_placement; i++) { 1033 if (mem->mm_node && (mem->start < heap->fpfn ||
1043 const struct ttm_place *heap = &placement->busy_placement[i];
1044 if (mem->mm_node &&
1045 (mem->start < heap->fpfn ||
1046 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) 1034 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1047 continue; 1035 continue;
1048 1036
1049 *new_flags = heap->flags; 1037 *new_flags = heap->flags;
1050 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1038 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1051 (*new_flags & mem->placement & TTM_PL_MASK_MEM)) 1039 (*new_flags & mem->placement & TTM_PL_MASK_MEM) &&
1040 (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
1041 (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1052 return true; 1042 return true;
1053 } 1043 }
1044 return false;
1045}
1046
1047bool ttm_bo_mem_compat(struct ttm_placement *placement,
1048 struct ttm_mem_reg *mem,
1049 uint32_t *new_flags)
1050{
1051 if (ttm_bo_places_compat(placement->placement, placement->num_placement,
1052 mem, new_flags))
1053 return true;
1054
1055 if ((placement->busy_placement != placement->placement ||
1056 placement->num_busy_placement > placement->num_placement) &&
1057 ttm_bo_places_compat(placement->busy_placement,
1058 placement->num_busy_placement,
1059 mem, new_flags))
1060 return true;
1054 1061
1055 return false; 1062 return false;
1056} 1063}
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index 35ffb3754feb..9f53df95f35c 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -231,7 +231,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf)
231 */ 231 */
232 for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { 232 for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) {
233 if (bo->mem.bus.is_iomem) 233 if (bo->mem.bus.is_iomem)
234 pfn = ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + page_offset; 234 pfn = bdev->driver->io_mem_pfn(bo, page_offset);
235 else { 235 else {
236 page = ttm->pages[page_offset]; 236 page = ttm->pages[page_offset];
237 if (unlikely(!page && i == 0)) { 237 if (unlikely(!page && i == 0)) {
@@ -324,6 +324,14 @@ static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
324 return bo; 324 return bo;
325} 325}
326 326
327unsigned long ttm_bo_default_io_mem_pfn(struct ttm_buffer_object *bo,
328 unsigned long page_offset)
329{
330 return ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT)
331 + page_offset;
332}
333EXPORT_SYMBOL(ttm_bo_default_io_mem_pfn);
334
327int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, 335int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
328 struct ttm_bo_device *bdev) 336 struct ttm_bo_device *bdev)
329{ 337{
diff --git a/drivers/gpu/drm/virtio/virtgpu_ttm.c b/drivers/gpu/drm/virtio/virtgpu_ttm.c
index 70ec8ca8d9b1..4e8e27d50922 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ttm.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ttm.c
@@ -431,6 +431,7 @@ static struct ttm_bo_driver virtio_gpu_bo_driver = {
431 .verify_access = &virtio_gpu_verify_access, 431 .verify_access = &virtio_gpu_verify_access,
432 .io_mem_reserve = &virtio_gpu_ttm_io_mem_reserve, 432 .io_mem_reserve = &virtio_gpu_ttm_io_mem_reserve,
433 .io_mem_free = &virtio_gpu_ttm_io_mem_free, 433 .io_mem_free = &virtio_gpu_ttm_io_mem_free,
434 .io_mem_pfn = ttm_bo_default_io_mem_pfn,
434 .move_notify = &virtio_gpu_bo_move_notify, 435 .move_notify = &virtio_gpu_bo_move_notify,
435 .swap_notify = &virtio_gpu_bo_swap_notify, 436 .swap_notify = &virtio_gpu_bo_swap_notify,
436}; 437};
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
index 4c7f24a67a2e..35bf781e418e 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
@@ -859,4 +859,5 @@ struct ttm_bo_driver vmw_bo_driver = {
859 .fault_reserve_notify = &vmw_ttm_fault_reserve_notify, 859 .fault_reserve_notify = &vmw_ttm_fault_reserve_notify,
860 .io_mem_reserve = &vmw_ttm_io_mem_reserve, 860 .io_mem_reserve = &vmw_ttm_io_mem_reserve,
861 .io_mem_free = &vmw_ttm_io_mem_free, 861 .io_mem_free = &vmw_ttm_io_mem_free,
862 .io_mem_pfn = ttm_bo_default_io_mem_pfn,
862}; 863};
diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h
index 0b1ce05e2c2e..fa07be197945 100644
--- a/include/drm/ttm/ttm_bo_api.h
+++ b/include/drm/ttm/ttm_bo_api.h
@@ -711,6 +711,17 @@ extern int ttm_fbdev_mmap(struct vm_area_struct *vma,
711 struct ttm_buffer_object *bo); 711 struct ttm_buffer_object *bo);
712 712
713/** 713/**
714 * ttm_bo_default_iomem_pfn - get a pfn for a page offset
715 *
716 * @bo: the BO we need to look up the pfn for
717 * @page_offset: offset inside the BO to look up.
718 *
719 * Calculate the PFN for iomem based mappings during page fault
720 */
721unsigned long ttm_bo_default_io_mem_pfn(struct ttm_buffer_object *bo,
722 unsigned long page_offset);
723
724/**
714 * ttm_bo_mmap - mmap out of the ttm device address space. 725 * ttm_bo_mmap - mmap out of the ttm device address space.
715 * 726 *
716 * @filp: filp as input from the mmap method. 727 * @filp: filp as input from the mmap method.
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 3641c6128ac2..6bbd34d25a8d 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -462,6 +462,15 @@ struct ttm_bo_driver {
462 struct ttm_mem_reg *mem); 462 struct ttm_mem_reg *mem);
463 void (*io_mem_free)(struct ttm_bo_device *bdev, 463 void (*io_mem_free)(struct ttm_bo_device *bdev,
464 struct ttm_mem_reg *mem); 464 struct ttm_mem_reg *mem);
465
466 /**
467 * Return the pfn for a given page_offset inside the BO.
468 *
469 * @bo: the BO to look up the pfn for
470 * @page_offset: the offset to look up
471 */
472 unsigned long (*io_mem_pfn)(struct ttm_buffer_object *bo,
473 unsigned long page_offset);
465}; 474};
466 475
467/** 476/**
diff --git a/include/drm/ttm/ttm_placement.h b/include/drm/ttm/ttm_placement.h
index 932be0c8086e..e88a8e39767b 100644
--- a/include/drm/ttm/ttm_placement.h
+++ b/include/drm/ttm/ttm_placement.h
@@ -63,6 +63,7 @@
63#define TTM_PL_FLAG_CACHED (1 << 16) 63#define TTM_PL_FLAG_CACHED (1 << 16)
64#define TTM_PL_FLAG_UNCACHED (1 << 17) 64#define TTM_PL_FLAG_UNCACHED (1 << 17)
65#define TTM_PL_FLAG_WC (1 << 18) 65#define TTM_PL_FLAG_WC (1 << 18)
66#define TTM_PL_FLAG_CONTIGUOUS (1 << 19)
66#define TTM_PL_FLAG_NO_EVICT (1 << 21) 67#define TTM_PL_FLAG_NO_EVICT (1 << 21)
67#define TTM_PL_FLAG_TOPDOWN (1 << 22) 68#define TTM_PL_FLAG_TOPDOWN (1 << 22)
68 69
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-12 21:31:09 -0400