aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--MAINTAINERS4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu.h41
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c25
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c117
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_device.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c31
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c57
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c33
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_job.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c88
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c101
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c125
-rw-r--r--drivers/gpu/drm/amd/amdgpu/ci_dpm.c49
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cik.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cik_sdma.c36
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v10_0.c33
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v11_0.c38
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v8_0.c33
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c54
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c31
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c37
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c38
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vi.c4
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c3
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c9
-rw-r--r--drivers/gpu/drm/radeon/atombios_crtc.c4
-rw-r--r--drivers/gpu/drm/radeon/atombios_encoders.c6
-rw-r--r--drivers/gpu/drm/radeon/radeon_atpx_handler.c8
-rw-r--r--drivers/gpu/drm/radeon/radeon_device.c8
-rw-r--r--drivers/gpu/drm/radeon/radeon_legacy_crtc.c4
36 files changed, 741 insertions, 312 deletions
diff --git a/MAINTAINERS b/MAINTAINERS
index 10562167b85e..a32d2ffd81a8 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -3710,7 +3710,7 @@ F: drivers/gpu/vga/
3710F: include/drm/ 3710F: include/drm/
3711F: include/uapi/drm/ 3711F: include/uapi/drm/
3712 3712
3713RADEON DRM DRIVERS 3713RADEON and AMDGPU DRM DRIVERS
3714M: Alex Deucher <alexander.deucher@amd.com> 3714M: Alex Deucher <alexander.deucher@amd.com>
3715M: Christian König <christian.koenig@amd.com> 3715M: Christian König <christian.koenig@amd.com>
3716L: dri-devel@lists.freedesktop.org 3716L: dri-devel@lists.freedesktop.org
@@ -3718,6 +3718,8 @@ T: git git://people.freedesktop.org/~agd5f/linux
3718S: Supported 3718S: Supported
3719F: drivers/gpu/drm/radeon/ 3719F: drivers/gpu/drm/radeon/
3720F: include/uapi/drm/radeon* 3720F: include/uapi/drm/radeon*
3721F: drivers/gpu/drm/amd/
3722F: include/uapi/drm/amdgpu*
3721 3723
3722DRM PANEL DRIVERS 3724DRM PANEL DRIVERS
3723M: Thierry Reding <thierry.reding@gmail.com> 3725M: Thierry Reding <thierry.reding@gmail.com>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index d0489722fc7e..a80c8cea7609 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -287,9 +287,11 @@ struct amdgpu_ring_funcs {
287 struct amdgpu_ib *ib); 287 struct amdgpu_ib *ib);
288 void (*emit_fence)(struct amdgpu_ring *ring, uint64_t addr, 288 void (*emit_fence)(struct amdgpu_ring *ring, uint64_t addr,
289 uint64_t seq, unsigned flags); 289 uint64_t seq, unsigned flags);
290 void (*emit_pipeline_sync)(struct amdgpu_ring *ring);
290 void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vm_id, 291 void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vm_id,
291 uint64_t pd_addr); 292 uint64_t pd_addr);
292 void (*emit_hdp_flush)(struct amdgpu_ring *ring); 293 void (*emit_hdp_flush)(struct amdgpu_ring *ring);
294 void (*emit_hdp_invalidate)(struct amdgpu_ring *ring);
293 void (*emit_gds_switch)(struct amdgpu_ring *ring, uint32_t vmid, 295 void (*emit_gds_switch)(struct amdgpu_ring *ring, uint32_t vmid,
294 uint32_t gds_base, uint32_t gds_size, 296 uint32_t gds_base, uint32_t gds_size,
295 uint32_t gws_base, uint32_t gws_size, 297 uint32_t gws_base, uint32_t gws_size,
@@ -369,9 +371,6 @@ struct amdgpu_fence {
369 struct amdgpu_ring *ring; 371 struct amdgpu_ring *ring;
370 uint64_t seq; 372 uint64_t seq;
371 373
372 /* filp or special value for fence creator */
373 void *owner;
374
375 wait_queue_t fence_wake; 374 wait_queue_t fence_wake;
376}; 375};
377 376
@@ -392,8 +391,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
392 unsigned irq_type); 391 unsigned irq_type);
393void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); 392void amdgpu_fence_driver_suspend(struct amdgpu_device *adev);
394void amdgpu_fence_driver_resume(struct amdgpu_device *adev); 393void amdgpu_fence_driver_resume(struct amdgpu_device *adev);
395int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner, 394int amdgpu_fence_emit(struct amdgpu_ring *ring, struct fence **fence);
396 struct amdgpu_fence **fence);
397void amdgpu_fence_process(struct amdgpu_ring *ring); 395void amdgpu_fence_process(struct amdgpu_ring *ring);
398int amdgpu_fence_wait_next(struct amdgpu_ring *ring); 396int amdgpu_fence_wait_next(struct amdgpu_ring *ring);
399int amdgpu_fence_wait_empty(struct amdgpu_ring *ring); 397int amdgpu_fence_wait_empty(struct amdgpu_ring *ring);
@@ -434,6 +432,8 @@ struct amdgpu_bo_list_entry {
434 struct ttm_validate_buffer tv; 432 struct ttm_validate_buffer tv;
435 struct amdgpu_bo_va *bo_va; 433 struct amdgpu_bo_va *bo_va;
436 uint32_t priority; 434 uint32_t priority;
435 struct page **user_pages;
436 int user_invalidated;
437}; 437};
438 438
439struct amdgpu_bo_va_mapping { 439struct amdgpu_bo_va_mapping {
@@ -445,7 +445,6 @@ struct amdgpu_bo_va_mapping {
445 445
446/* bo virtual addresses in a specific vm */ 446/* bo virtual addresses in a specific vm */
447struct amdgpu_bo_va { 447struct amdgpu_bo_va {
448 struct mutex mutex;
449 /* protected by bo being reserved */ 448 /* protected by bo being reserved */
450 struct list_head bo_list; 449 struct list_head bo_list;
451 struct fence *last_pt_update; 450 struct fence *last_pt_update;
@@ -596,6 +595,8 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
596struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync); 595struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync);
597int amdgpu_sync_wait(struct amdgpu_sync *sync); 596int amdgpu_sync_wait(struct amdgpu_sync *sync);
598void amdgpu_sync_free(struct amdgpu_sync *sync); 597void amdgpu_sync_free(struct amdgpu_sync *sync);
598int amdgpu_sync_init(void);
599void amdgpu_sync_fini(void);
599 600
600/* 601/*
601 * GART structures, functions & helpers 602 * GART structures, functions & helpers
@@ -726,7 +727,7 @@ struct amdgpu_ib {
726 uint32_t length_dw; 727 uint32_t length_dw;
727 uint64_t gpu_addr; 728 uint64_t gpu_addr;
728 uint32_t *ptr; 729 uint32_t *ptr;
729 struct amdgpu_fence *fence; 730 struct fence *fence;
730 struct amdgpu_user_fence *user; 731 struct amdgpu_user_fence *user;
731 struct amdgpu_vm *vm; 732 struct amdgpu_vm *vm;
732 unsigned vm_id; 733 unsigned vm_id;
@@ -845,7 +846,6 @@ struct amdgpu_vm_id {
845 846
846struct amdgpu_vm { 847struct amdgpu_vm {
847 /* tree of virtual addresses mapped */ 848 /* tree of virtual addresses mapped */
848 spinlock_t it_lock;
849 struct rb_root va; 849 struct rb_root va;
850 850
851 /* protecting invalidated */ 851 /* protecting invalidated */
@@ -882,6 +882,13 @@ struct amdgpu_vm_manager_id {
882 struct list_head list; 882 struct list_head list;
883 struct fence *active; 883 struct fence *active;
884 atomic_long_t owner; 884 atomic_long_t owner;
885
886 uint32_t gds_base;
887 uint32_t gds_size;
888 uint32_t gws_base;
889 uint32_t gws_size;
890 uint32_t oa_base;
891 uint32_t oa_size;
885}; 892};
886 893
887struct amdgpu_vm_manager { 894struct amdgpu_vm_manager {
@@ -917,8 +924,11 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
917 struct amdgpu_sync *sync, struct fence *fence, 924 struct amdgpu_sync *sync, struct fence *fence,
918 unsigned *vm_id, uint64_t *vm_pd_addr); 925 unsigned *vm_id, uint64_t *vm_pd_addr);
919void amdgpu_vm_flush(struct amdgpu_ring *ring, 926void amdgpu_vm_flush(struct amdgpu_ring *ring,
920 unsigned vmid, 927 unsigned vm_id, uint64_t pd_addr,
921 uint64_t pd_addr); 928 uint32_t gds_base, uint32_t gds_size,
929 uint32_t gws_base, uint32_t gws_size,
930 uint32_t oa_base, uint32_t oa_size);
931void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id);
922uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr); 932uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr);
923int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, 933int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
924 struct amdgpu_vm *vm); 934 struct amdgpu_vm *vm);
@@ -1006,7 +1016,7 @@ struct amdgpu_bo_list {
1006 struct amdgpu_bo *gds_obj; 1016 struct amdgpu_bo *gds_obj;
1007 struct amdgpu_bo *gws_obj; 1017 struct amdgpu_bo *gws_obj;
1008 struct amdgpu_bo *oa_obj; 1018 struct amdgpu_bo *oa_obj;
1009 bool has_userptr; 1019 unsigned first_userptr;
1010 unsigned num_entries; 1020 unsigned num_entries;
1011 struct amdgpu_bo_list_entry *array; 1021 struct amdgpu_bo_list_entry *array;
1012}; 1022};
@@ -1135,8 +1145,7 @@ int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
1135 unsigned size, struct amdgpu_ib *ib); 1145 unsigned size, struct amdgpu_ib *ib);
1136void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib); 1146void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib);
1137int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, 1147int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
1138 struct amdgpu_ib *ib, void *owner, 1148 struct amdgpu_ib *ib, struct fence *last_vm_update,
1139 struct fence *last_vm_update,
1140 struct fence **f); 1149 struct fence **f);
1141int amdgpu_ib_pool_init(struct amdgpu_device *adev); 1150int amdgpu_ib_pool_init(struct amdgpu_device *adev);
1142void amdgpu_ib_pool_fini(struct amdgpu_device *adev); 1151void amdgpu_ib_pool_fini(struct amdgpu_device *adev);
@@ -2012,7 +2021,6 @@ struct amdgpu_device {
2012 struct amdgpu_sdma sdma; 2021 struct amdgpu_sdma sdma;
2013 2022
2014 /* uvd */ 2023 /* uvd */
2015 bool has_uvd;
2016 struct amdgpu_uvd uvd; 2024 struct amdgpu_uvd uvd;
2017 2025
2018 /* vce */ 2026 /* vce */
@@ -2186,10 +2194,12 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
2186#define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r)) 2194#define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r))
2187#define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r)) 2195#define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r))
2188#define amdgpu_ring_emit_ib(r, ib) (r)->funcs->emit_ib((r), (ib)) 2196#define amdgpu_ring_emit_ib(r, ib) (r)->funcs->emit_ib((r), (ib))
2197#define amdgpu_ring_emit_pipeline_sync(r) (r)->funcs->emit_pipeline_sync((r))
2189#define amdgpu_ring_emit_vm_flush(r, vmid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (addr)) 2198#define amdgpu_ring_emit_vm_flush(r, vmid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (addr))
2190#define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags)) 2199#define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags))
2191#define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as)) 2200#define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as))
2192#define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r)) 2201#define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r))
2202#define amdgpu_ring_emit_hdp_invalidate(r) (r)->funcs->emit_hdp_invalidate((r))
2193#define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib))) 2203#define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
2194#define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev)) 2204#define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev))
2195#define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv)) 2205#define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv))
@@ -2314,12 +2324,15 @@ int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
2314 struct amdgpu_ring **out_ring); 2324 struct amdgpu_ring **out_ring);
2315void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *rbo, u32 domain); 2325void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *rbo, u32 domain);
2316bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo); 2326bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
2327int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
2317int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, 2328int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
2318 uint32_t flags); 2329 uint32_t flags);
2319bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm); 2330bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm);
2320struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm); 2331struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm);
2321bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, 2332bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
2322 unsigned long end); 2333 unsigned long end);
2334bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
2335 int *last_invalidated);
2323bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm); 2336bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
2324uint32_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm, 2337uint32_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
2325 struct ttm_mem_reg *mem); 2338 struct ttm_mem_reg *mem);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
index fa948dcbdd5d..0020a0ea43ff 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
@@ -63,6 +63,10 @@ bool amdgpu_has_atpx(void) {
63 return amdgpu_atpx_priv.atpx_detected; 63 return amdgpu_atpx_priv.atpx_detected;
64} 64}
65 65
66bool amdgpu_has_atpx_dgpu_power_cntl(void) {
67 return amdgpu_atpx_priv.atpx.functions.power_cntl;
68}
69
66/** 70/**
67 * amdgpu_atpx_call - call an ATPX method 71 * amdgpu_atpx_call - call an ATPX method
68 * 72 *
@@ -142,10 +146,6 @@ static void amdgpu_atpx_parse_functions(struct amdgpu_atpx_functions *f, u32 mas
142 */ 146 */
143static int amdgpu_atpx_validate(struct amdgpu_atpx *atpx) 147static int amdgpu_atpx_validate(struct amdgpu_atpx *atpx)
144{ 148{
145 /* make sure required functions are enabled */
146 /* dGPU power control is required */
147 atpx->functions.power_cntl = true;
148
149 if (atpx->functions.px_params) { 149 if (atpx->functions.px_params) {
150 union acpi_object *info; 150 union acpi_object *info;
151 struct atpx_px_params output; 151 struct atpx_px_params output;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
index 4792f9d0b7d4..eacd810fc09b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
@@ -91,7 +91,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev,
91 struct amdgpu_bo *gws_obj = adev->gds.gws_gfx_bo; 91 struct amdgpu_bo *gws_obj = adev->gds.gws_gfx_bo;
92 struct amdgpu_bo *oa_obj = adev->gds.oa_gfx_bo; 92 struct amdgpu_bo *oa_obj = adev->gds.oa_gfx_bo;
93 93
94 bool has_userptr = false; 94 unsigned last_entry = 0, first_userptr = num_entries;
95 unsigned i; 95 unsigned i;
96 int r; 96 int r;
97 97
@@ -101,8 +101,9 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev,
101 memset(array, 0, num_entries * sizeof(struct amdgpu_bo_list_entry)); 101 memset(array, 0, num_entries * sizeof(struct amdgpu_bo_list_entry));
102 102
103 for (i = 0; i < num_entries; ++i) { 103 for (i = 0; i < num_entries; ++i) {
104 struct amdgpu_bo_list_entry *entry = &array[i]; 104 struct amdgpu_bo_list_entry *entry;
105 struct drm_gem_object *gobj; 105 struct drm_gem_object *gobj;
106 struct amdgpu_bo *bo;
106 struct mm_struct *usermm; 107 struct mm_struct *usermm;
107 108
108 gobj = drm_gem_object_lookup(adev->ddev, filp, info[i].bo_handle); 109 gobj = drm_gem_object_lookup(adev->ddev, filp, info[i].bo_handle);
@@ -111,19 +112,24 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev,
111 goto error_free; 112 goto error_free;
112 } 113 }
113 114
114 entry->robj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); 115 bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
115 drm_gem_object_unreference_unlocked(gobj); 116 drm_gem_object_unreference_unlocked(gobj);
116 entry->priority = min(info[i].bo_priority, 117
117 AMDGPU_BO_LIST_MAX_PRIORITY); 118 usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
118 usermm = amdgpu_ttm_tt_get_usermm(entry->robj->tbo.ttm);
119 if (usermm) { 119 if (usermm) {
120 if (usermm != current->mm) { 120 if (usermm != current->mm) {
121 amdgpu_bo_unref(&entry->robj); 121 amdgpu_bo_unref(&bo);
122 r = -EPERM; 122 r = -EPERM;
123 goto error_free; 123 goto error_free;
124 } 124 }
125 has_userptr = true; 125 entry = &array[--first_userptr];
126 } else {
127 entry = &array[last_entry++];
126 } 128 }
129
130 entry->robj = bo;
131 entry->priority = min(info[i].bo_priority,
132 AMDGPU_BO_LIST_MAX_PRIORITY);
127 entry->tv.bo = &entry->robj->tbo; 133 entry->tv.bo = &entry->robj->tbo;
128 entry->tv.shared = true; 134 entry->tv.shared = true;
129 135
@@ -145,7 +151,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev,
145 list->gds_obj = gds_obj; 151 list->gds_obj = gds_obj;
146 list->gws_obj = gws_obj; 152 list->gws_obj = gws_obj;
147 list->oa_obj = oa_obj; 153 list->oa_obj = oa_obj;
148 list->has_userptr = has_userptr; 154 list->first_userptr = first_userptr;
149 list->array = array; 155 list->array = array;
150 list->num_entries = num_entries; 156 list->num_entries = num_entries;
151 157
@@ -194,6 +200,7 @@ void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
194 200
195 list_add_tail(&list->array[i].tv.head, 201 list_add_tail(&list->array[i].tv.head,
196 &bucket[priority]); 202 &bucket[priority]);
203 list->array[i].user_pages = NULL;
197 } 204 }
198 205
199 /* Connect the sorted buckets in the output list. */ 206 /* Connect the sorted buckets in the output list. */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 52c3eb96b199..4f5ef4149e87 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -111,6 +111,7 @@ static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
111 p->uf_entry.priority = 0; 111 p->uf_entry.priority = 0;
112 p->uf_entry.tv.bo = &p->uf_entry.robj->tbo; 112 p->uf_entry.tv.bo = &p->uf_entry.robj->tbo;
113 p->uf_entry.tv.shared = true; 113 p->uf_entry.tv.shared = true;
114 p->uf_entry.user_pages = NULL;
114 115
115 drm_gem_object_unreference_unlocked(gobj); 116 drm_gem_object_unreference_unlocked(gobj);
116 return 0; 117 return 0;
@@ -297,6 +298,7 @@ int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
297 298
298 list_for_each_entry(lobj, validated, tv.head) { 299 list_for_each_entry(lobj, validated, tv.head) {
299 struct amdgpu_bo *bo = lobj->robj; 300 struct amdgpu_bo *bo = lobj->robj;
301 bool binding_userptr = false;
300 struct mm_struct *usermm; 302 struct mm_struct *usermm;
301 uint32_t domain; 303 uint32_t domain;
302 304
@@ -304,6 +306,15 @@ int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
304 if (usermm && usermm != current->mm) 306 if (usermm && usermm != current->mm)
305 return -EPERM; 307 return -EPERM;
306 308
309 /* Check if we have user pages and nobody bound the BO already */
310 if (lobj->user_pages && bo->tbo.ttm->state != tt_bound) {
311 size_t size = sizeof(struct page *);
312
313 size *= bo->tbo.ttm->num_pages;
314 memcpy(bo->tbo.ttm->pages, lobj->user_pages, size);
315 binding_userptr = true;
316 }
317
307 if (bo->pin_count) 318 if (bo->pin_count)
308 continue; 319 continue;
309 320
@@ -334,6 +345,11 @@ int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
334 } 345 }
335 return r; 346 return r;
336 } 347 }
348
349 if (binding_userptr) {
350 drm_free_large(lobj->user_pages);
351 lobj->user_pages = NULL;
352 }
337 } 353 }
338 return 0; 354 return 0;
339} 355}
@@ -342,15 +358,18 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
342 union drm_amdgpu_cs *cs) 358 union drm_amdgpu_cs *cs)
343{ 359{
344 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 360 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
361 struct amdgpu_bo_list_entry *e;
345 struct list_head duplicates; 362 struct list_head duplicates;
346 bool need_mmap_lock = false; 363 bool need_mmap_lock = false;
364 unsigned i, tries = 10;
347 int r; 365 int r;
348 366
349 INIT_LIST_HEAD(&p->validated); 367 INIT_LIST_HEAD(&p->validated);
350 368
351 p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle); 369 p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
352 if (p->bo_list) { 370 if (p->bo_list) {
353 need_mmap_lock = p->bo_list->has_userptr; 371 need_mmap_lock = p->bo_list->first_userptr !=
372 p->bo_list->num_entries;
354 amdgpu_bo_list_get_list(p->bo_list, &p->validated); 373 amdgpu_bo_list_get_list(p->bo_list, &p->validated);
355 } 374 }
356 375
@@ -363,9 +382,81 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
363 if (need_mmap_lock) 382 if (need_mmap_lock)
364 down_read(&current->mm->mmap_sem); 383 down_read(&current->mm->mmap_sem);
365 384
366 r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, &duplicates); 385 while (1) {
367 if (unlikely(r != 0)) 386 struct list_head need_pages;
368 goto error_reserve; 387 unsigned i;
388
389 r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
390 &duplicates);
391 if (unlikely(r != 0))
392 goto error_free_pages;
393
394 /* Without a BO list we don't have userptr BOs */
395 if (!p->bo_list)
396 break;
397
398 INIT_LIST_HEAD(&need_pages);
399 for (i = p->bo_list->first_userptr;
400 i < p->bo_list->num_entries; ++i) {
401
402 e = &p->bo_list->array[i];
403
404 if (amdgpu_ttm_tt_userptr_invalidated(e->robj->tbo.ttm,
405 &e->user_invalidated) && e->user_pages) {
406
407 /* We acquired a page array, but somebody
408 * invalidated it. Free it an try again
409 */
410 release_pages(e->user_pages,
411 e->robj->tbo.ttm->num_pages,
412 false);
413 drm_free_large(e->user_pages);
414 e->user_pages = NULL;
415 }
416
417 if (e->robj->tbo.ttm->state != tt_bound &&
418 !e->user_pages) {
419 list_del(&e->tv.head);
420 list_add(&e->tv.head, &need_pages);
421
422 amdgpu_bo_unreserve(e->robj);
423 }
424 }
425
426 if (list_empty(&need_pages))
427 break;
428
429 /* Unreserve everything again. */
430 ttm_eu_backoff_reservation(&p->ticket, &p->validated);
431
432 /* We tried to often, just abort */
433 if (!--tries) {
434 r = -EDEADLK;
435 goto error_free_pages;
436 }
437
438 /* Fill the page arrays for all useptrs. */
439 list_for_each_entry(e, &need_pages, tv.head) {
440 struct ttm_tt *ttm = e->robj->tbo.ttm;
441
442 e->user_pages = drm_calloc_large(ttm->num_pages,
443 sizeof(struct page*));
444 if (!e->user_pages) {
445 r = -ENOMEM;
446 goto error_free_pages;
447 }
448
449 r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages);
450 if (r) {
451 drm_free_large(e->user_pages);
452 e->user_pages = NULL;
453 goto error_free_pages;
454 }
455 }
456
457 /* And try again. */
458 list_splice(&need_pages, &p->validated);
459 }
369 460
370 amdgpu_vm_get_pt_bos(&fpriv->vm, &duplicates); 461 amdgpu_vm_get_pt_bos(&fpriv->vm, &duplicates);
371 462
@@ -397,10 +488,26 @@ error_validate:
397 ttm_eu_backoff_reservation(&p->ticket, &p->validated); 488 ttm_eu_backoff_reservation(&p->ticket, &p->validated);
398 } 489 }
399 490
400error_reserve: 491error_free_pages:
492
401 if (need_mmap_lock) 493 if (need_mmap_lock)
402 up_read(&current->mm->mmap_sem); 494 up_read(&current->mm->mmap_sem);
403 495
496 if (p->bo_list) {
497 for (i = p->bo_list->first_userptr;
498 i < p->bo_list->num_entries; ++i) {
499 e = &p->bo_list->array[i];
500
501 if (!e->user_pages)
502 continue;
503
504 release_pages(e->user_pages,
505 e->robj->tbo.ttm->num_pages,
506 false);
507 drm_free_large(e->user_pages);
508 }
509 }
510
404 return r; 511 return r;
405} 512}
406 513
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 2139da773da6..612117478b57 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -62,6 +62,12 @@ static const char *amdgpu_asic_name[] = {
62 "LAST", 62 "LAST",
63}; 63};
64 64
65#if defined(CONFIG_VGA_SWITCHEROO)
66bool amdgpu_has_atpx_dgpu_power_cntl(void);
67#else
68static inline bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false; }
69#endif
70
65bool amdgpu_device_is_px(struct drm_device *dev) 71bool amdgpu_device_is_px(struct drm_device *dev)
66{ 72{
67 struct amdgpu_device *adev = dev->dev_private; 73 struct amdgpu_device *adev = dev->dev_private;
@@ -1479,7 +1485,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
1479 1485
1480 if (amdgpu_runtime_pm == 1) 1486 if (amdgpu_runtime_pm == 1)
1481 runtime = true; 1487 runtime = true;
1482 if (amdgpu_device_is_px(ddev)) 1488 if (amdgpu_device_is_px(ddev) && amdgpu_has_atpx_dgpu_power_cntl())
1483 runtime = true; 1489 runtime = true;
1484 vga_switcheroo_register_client(adev->pdev, &amdgpu_switcheroo_ops, runtime); 1490 vga_switcheroo_register_client(adev->pdev, &amdgpu_switcheroo_ops, runtime);
1485 if (runtime) 1491 if (runtime)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 74a2f8a6be1f..f1e17d60055a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -555,6 +555,7 @@ static struct pci_driver amdgpu_kms_pci_driver = {
555 555
556static int __init amdgpu_init(void) 556static int __init amdgpu_init(void)
557{ 557{
558 amdgpu_sync_init();
558#ifdef CONFIG_VGA_CONSOLE 559#ifdef CONFIG_VGA_CONSOLE
559 if (vgacon_text_force()) { 560 if (vgacon_text_force()) {
560 DRM_ERROR("VGACON disables amdgpu kernel modesetting.\n"); 561 DRM_ERROR("VGACON disables amdgpu kernel modesetting.\n");
@@ -577,6 +578,7 @@ static void __exit amdgpu_exit(void)
577 amdgpu_amdkfd_fini(); 578 amdgpu_amdkfd_fini();
578 drm_pci_exit(driver, pdriver); 579 drm_pci_exit(driver, pdriver);
579 amdgpu_unregister_atpx_handler(); 580 amdgpu_unregister_atpx_handler();
581 amdgpu_sync_fini();
580} 582}
581 583
582module_init(amdgpu_init); 584module_init(amdgpu_init);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 97db196dc6f8..83599f2a0387 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -91,32 +91,29 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
91 * amdgpu_fence_emit - emit a fence on the requested ring 91 * amdgpu_fence_emit - emit a fence on the requested ring
92 * 92 *
93 * @ring: ring the fence is associated with 93 * @ring: ring the fence is associated with
94 * @owner: creator of the fence 94 * @f: resulting fence object
95 * @fence: amdgpu fence object
96 * 95 *
97 * Emits a fence command on the requested ring (all asics). 96 * Emits a fence command on the requested ring (all asics).
98 * Returns 0 on success, -ENOMEM on failure. 97 * Returns 0 on success, -ENOMEM on failure.
99 */ 98 */
100int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner, 99int amdgpu_fence_emit(struct amdgpu_ring *ring, struct fence **f)
101 struct amdgpu_fence **fence)
102{ 100{
103 struct amdgpu_device *adev = ring->adev; 101 struct amdgpu_device *adev = ring->adev;
102 struct amdgpu_fence *fence;
104 103
105 /* we are protected by the ring emission mutex */ 104 fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
106 *fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL); 105 if (fence == NULL)
107 if ((*fence) == NULL) {
108 return -ENOMEM; 106 return -ENOMEM;
109 } 107
110 (*fence)->seq = ++ring->fence_drv.sync_seq; 108 fence->seq = ++ring->fence_drv.sync_seq;
111 (*fence)->ring = ring; 109 fence->ring = ring;
112 (*fence)->owner = owner; 110 fence_init(&fence->base, &amdgpu_fence_ops,
113 fence_init(&(*fence)->base, &amdgpu_fence_ops, 111 &ring->fence_drv.fence_queue.lock,
114 &ring->fence_drv.fence_queue.lock, 112 adev->fence_context + ring->idx,
115 adev->fence_context + ring->idx, 113 fence->seq);
116 (*fence)->seq);
117 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr, 114 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
118 (*fence)->seq, 115 fence->seq, AMDGPU_FENCE_FLAG_INT);
119 AMDGPU_FENCE_FLAG_INT); 116 *f = &fence->base;
120 return 0; 117 return 0;
121} 118}
122 119
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 7a47c45b2131..1ecdf6c01368 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -140,25 +140,40 @@ int amdgpu_gem_object_open(struct drm_gem_object *obj, struct drm_file *file_pri
140void amdgpu_gem_object_close(struct drm_gem_object *obj, 140void amdgpu_gem_object_close(struct drm_gem_object *obj,
141 struct drm_file *file_priv) 141 struct drm_file *file_priv)
142{ 142{
143 struct amdgpu_bo *rbo = gem_to_amdgpu_bo(obj); 143 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
144 struct amdgpu_device *adev = rbo->adev; 144 struct amdgpu_device *adev = bo->adev;
145 struct amdgpu_fpriv *fpriv = file_priv->driver_priv; 145 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
146 struct amdgpu_vm *vm = &fpriv->vm; 146 struct amdgpu_vm *vm = &fpriv->vm;
147
148 struct amdgpu_bo_list_entry vm_pd;
149 struct list_head list, duplicates;
150 struct ttm_validate_buffer tv;
151 struct ww_acquire_ctx ticket;
147 struct amdgpu_bo_va *bo_va; 152 struct amdgpu_bo_va *bo_va;
148 int r; 153 int r;
149 r = amdgpu_bo_reserve(rbo, true); 154
155 INIT_LIST_HEAD(&list);
156 INIT_LIST_HEAD(&duplicates);
157
158 tv.bo = &bo->tbo;
159 tv.shared = true;
160 list_add(&tv.head, &list);
161
162 amdgpu_vm_get_pd_bo(vm, &list, &vm_pd);
163
164 r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates);
150 if (r) { 165 if (r) {
151 dev_err(adev->dev, "leaking bo va because " 166 dev_err(adev->dev, "leaking bo va because "
152 "we fail to reserve bo (%d)\n", r); 167 "we fail to reserve bo (%d)\n", r);
153 return; 168 return;
154 } 169 }
155 bo_va = amdgpu_vm_bo_find(vm, rbo); 170 bo_va = amdgpu_vm_bo_find(vm, bo);
156 if (bo_va) { 171 if (bo_va) {
157 if (--bo_va->ref_count == 0) { 172 if (--bo_va->ref_count == 0) {
158 amdgpu_vm_bo_rmv(adev, bo_va); 173 amdgpu_vm_bo_rmv(adev, bo_va);
159 } 174 }
160 } 175 }
161 amdgpu_bo_unreserve(rbo); 176 ttm_eu_backoff_reservation(&ticket, &list);
162} 177}
163 178
164static int amdgpu_gem_handle_lockup(struct amdgpu_device *adev, int r) 179static int amdgpu_gem_handle_lockup(struct amdgpu_device *adev, int r)
@@ -274,18 +289,23 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
274 289
275 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) { 290 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
276 down_read(&current->mm->mmap_sem); 291 down_read(&current->mm->mmap_sem);
292
293 r = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
294 bo->tbo.ttm->pages);
295 if (r)
296 goto unlock_mmap_sem;
297
277 r = amdgpu_bo_reserve(bo, true); 298 r = amdgpu_bo_reserve(bo, true);
278 if (r) { 299 if (r)
279 up_read(&current->mm->mmap_sem); 300 goto free_pages;
280 goto release_object;
281 }
282 301
283 amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 302 amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
284 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 303 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
285 amdgpu_bo_unreserve(bo); 304 amdgpu_bo_unreserve(bo);
286 up_read(&current->mm->mmap_sem);
287 if (r) 305 if (r)
288 goto release_object; 306 goto free_pages;
307
308 up_read(&current->mm->mmap_sem);
289 } 309 }
290 310
291 r = drm_gem_handle_create(filp, gobj, &handle); 311 r = drm_gem_handle_create(filp, gobj, &handle);
@@ -297,6 +317,12 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
297 args->handle = handle; 317 args->handle = handle;
298 return 0; 318 return 0;
299 319
320free_pages:
321 release_pages(bo->tbo.ttm->pages, bo->tbo.ttm->num_pages, false);
322
323unlock_mmap_sem:
324 up_read(&current->mm->mmap_sem);
325
300release_object: 326release_object:
301 drm_gem_object_unreference_unlocked(gobj); 327 drm_gem_object_unreference_unlocked(gobj);
302 328
@@ -569,11 +595,10 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
569 tv.shared = true; 595 tv.shared = true;
570 list_add(&tv.head, &list); 596 list_add(&tv.head, &list);
571 597
572 if (args->operation == AMDGPU_VA_OP_MAP) { 598 tv_pd.bo = &fpriv->vm.page_directory->tbo;
573 tv_pd.bo = &fpriv->vm.page_directory->tbo; 599 tv_pd.shared = true;
574 tv_pd.shared = true; 600 list_add(&tv_pd.head, &list);
575 list_add(&tv_pd.head, &list); 601
576 }
577 r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates); 602 r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates);
578 if (r) { 603 if (r) {
579 drm_gem_object_unreference_unlocked(gobj); 604 drm_gem_object_unreference_unlocked(gobj);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index db14a7bbb8f4..798d46626820 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -90,9 +90,8 @@ int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
90 */ 90 */
91void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib) 91void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib)
92{ 92{
93 amdgpu_sa_bo_free(adev, &ib->sa_bo, &ib->fence->base); 93 amdgpu_sa_bo_free(adev, &ib->sa_bo, ib->fence);
94 if (ib->fence) 94 fence_put(ib->fence);
95 fence_put(&ib->fence->base);
96} 95}
97 96
98/** 97/**
@@ -101,7 +100,6 @@ void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib)
101 * @adev: amdgpu_device pointer 100 * @adev: amdgpu_device pointer
102 * @num_ibs: number of IBs to schedule 101 * @num_ibs: number of IBs to schedule
103 * @ibs: IB objects to schedule 102 * @ibs: IB objects to schedule
104 * @owner: owner for creating the fences
105 * @f: fence created during this submission 103 * @f: fence created during this submission
106 * 104 *
107 * Schedule an IB on the associated ring (all asics). 105 * Schedule an IB on the associated ring (all asics).
@@ -118,8 +116,7 @@ void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib)
118 * to SI there was just a DE IB. 116 * to SI there was just a DE IB.
119 */ 117 */
120int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, 118int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
121 struct amdgpu_ib *ibs, void *owner, 119 struct amdgpu_ib *ibs, struct fence *last_vm_update,
122 struct fence *last_vm_update,
123 struct fence **f) 120 struct fence **f)
124{ 121{
125 struct amdgpu_device *adev = ring->adev; 122 struct amdgpu_device *adev = ring->adev;
@@ -153,13 +150,10 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
153 150
154 if (vm) { 151 if (vm) {
155 /* do context switch */ 152 /* do context switch */
156 amdgpu_vm_flush(ring, ib->vm_id, ib->vm_pd_addr); 153 amdgpu_vm_flush(ring, ib->vm_id, ib->vm_pd_addr,
157 154 ib->gds_base, ib->gds_size,
158 if (ring->funcs->emit_gds_switch) 155 ib->gws_base, ib->gws_size,
159 amdgpu_ring_emit_gds_switch(ring, ib->vm_id, 156 ib->oa_base, ib->oa_size);
160 ib->gds_base, ib->gds_size,
161 ib->gws_base, ib->gws_size,
162 ib->oa_base, ib->oa_size);
163 157
164 if (ring->funcs->emit_hdp_flush) 158 if (ring->funcs->emit_hdp_flush)
165 amdgpu_ring_emit_hdp_flush(ring); 159 amdgpu_ring_emit_hdp_flush(ring);
@@ -171,6 +165,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
171 165
172 if (ib->ctx != ctx || ib->vm != vm) { 166 if (ib->ctx != ctx || ib->vm != vm) {
173 ring->current_ctx = old_ctx; 167 ring->current_ctx = old_ctx;
168 if (ib->vm_id)
169 amdgpu_vm_reset_id(adev, ib->vm_id);
174 amdgpu_ring_undo(ring); 170 amdgpu_ring_undo(ring);
175 return -EINVAL; 171 return -EINVAL;
176 } 172 }
@@ -178,10 +174,17 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
178 ring->current_ctx = ctx; 174 ring->current_ctx = ctx;
179 } 175 }
180 176
181 r = amdgpu_fence_emit(ring, owner, &ib->fence); 177 if (vm) {
178 if (ring->funcs->emit_hdp_invalidate)
179 amdgpu_ring_emit_hdp_invalidate(ring);
180 }
181
182 r = amdgpu_fence_emit(ring, &ib->fence);
182 if (r) { 183 if (r) {
183 dev_err(adev->dev, "failed to emit fence (%d)\n", r); 184 dev_err(adev->dev, "failed to emit fence (%d)\n", r);
184 ring->current_ctx = old_ctx; 185 ring->current_ctx = old_ctx;
186 if (ib->vm_id)
187 amdgpu_vm_reset_id(adev, ib->vm_id);
185 amdgpu_ring_undo(ring); 188 amdgpu_ring_undo(ring);
186 return r; 189 return r;
187 } 190 }
@@ -195,7 +198,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
195 } 198 }
196 199
197 if (f) 200 if (f)
198 *f = fence_get(&ib->fence->base); 201 *f = fence_get(ib->fence);
199 202
200 amdgpu_ring_commit(ring); 203 amdgpu_ring_commit(ring);
201 return 0; 204 return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 90e52f7e17a0..692b45560d0a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -148,7 +148,7 @@ static struct fence *amdgpu_job_run(struct amd_sched_job *sched_job)
148 } 148 }
149 149
150 trace_amdgpu_sched_run_job(job); 150 trace_amdgpu_sched_run_job(job);
151 r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job->owner, 151 r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs,
152 job->sync.last_vm_update, &fence); 152 job->sync.last_vm_update, &fence);
153 if (r) { 153 if (r) {
154 DRM_ERROR("Error scheduling IBs (%d)\n", r); 154 DRM_ERROR("Error scheduling IBs (%d)\n", r);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index c15be00de904..c48b4fce5e57 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -37,6 +37,8 @@ struct amdgpu_sync_entry {
37 struct fence *fence; 37 struct fence *fence;
38}; 38};
39 39
40static struct kmem_cache *amdgpu_sync_slab;
41
40/** 42/**
41 * amdgpu_sync_create - zero init sync object 43 * amdgpu_sync_create - zero init sync object
42 * 44 *
@@ -50,14 +52,18 @@ void amdgpu_sync_create(struct amdgpu_sync *sync)
50 sync->last_vm_update = NULL; 52 sync->last_vm_update = NULL;
51} 53}
52 54
55/**
56 * amdgpu_sync_same_dev - test if fence belong to us
57 *
58 * @adev: amdgpu device to use for the test
59 * @f: fence to test
60 *
61 * Test if the fence was issued by us.
62 */
53static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f) 63static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f)
54{ 64{
55 struct amdgpu_fence *a_fence = to_amdgpu_fence(f);
56 struct amd_sched_fence *s_fence = to_amd_sched_fence(f); 65 struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
57 66
58 if (a_fence)
59 return a_fence->ring->adev == adev;
60
61 if (s_fence) { 67 if (s_fence) {
62 struct amdgpu_ring *ring; 68 struct amdgpu_ring *ring;
63 69
@@ -68,17 +74,31 @@ static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f)
68 return false; 74 return false;
69} 75}
70 76
71static bool amdgpu_sync_test_owner(struct fence *f, void *owner) 77/**
78 * amdgpu_sync_get_owner - extract the owner of a fence
79 *
80 * @fence: fence get the owner from
81 *
82 * Extract who originally created the fence.
83 */
84static void *amdgpu_sync_get_owner(struct fence *f)
72{ 85{
73 struct amdgpu_fence *a_fence = to_amdgpu_fence(f);
74 struct amd_sched_fence *s_fence = to_amd_sched_fence(f); 86 struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
87
75 if (s_fence) 88 if (s_fence)
76 return s_fence->owner == owner; 89 return s_fence->owner;
77 if (a_fence) 90
78 return a_fence->owner == owner; 91 return AMDGPU_FENCE_OWNER_UNDEFINED;
79 return false;
80} 92}
81 93
94/**
95 * amdgpu_sync_keep_later - Keep the later fence
96 *
97 * @keep: existing fence to test
98 * @fence: new fence
99 *
100 * Either keep the existing fence or the new one, depending which one is later.
101 */
82static void amdgpu_sync_keep_later(struct fence **keep, struct fence *fence) 102static void amdgpu_sync_keep_later(struct fence **keep, struct fence *fence)
83{ 103{
84 if (*keep && fence_is_later(*keep, fence)) 104 if (*keep && fence_is_later(*keep, fence))
@@ -104,7 +124,7 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
104 return 0; 124 return 0;
105 125
106 if (amdgpu_sync_same_dev(adev, f) && 126 if (amdgpu_sync_same_dev(adev, f) &&
107 amdgpu_sync_test_owner(f, AMDGPU_FENCE_OWNER_VM)) 127 amdgpu_sync_get_owner(f) == AMDGPU_FENCE_OWNER_VM)
108 amdgpu_sync_keep_later(&sync->last_vm_update, f); 128 amdgpu_sync_keep_later(&sync->last_vm_update, f);
109 129
110 hash_for_each_possible(sync->fences, e, node, f->context) { 130 hash_for_each_possible(sync->fences, e, node, f->context) {
@@ -115,7 +135,7 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
115 return 0; 135 return 0;
116 } 136 }
117 137
118 e = kmalloc(sizeof(struct amdgpu_sync_entry), GFP_KERNEL); 138 e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
119 if (!e) 139 if (!e)
120 return -ENOMEM; 140 return -ENOMEM;
121 141
@@ -124,18 +144,6 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
124 return 0; 144 return 0;
125} 145}
126 146
127static void *amdgpu_sync_get_owner(struct fence *f)
128{
129 struct amdgpu_fence *a_fence = to_amdgpu_fence(f);
130 struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
131
132 if (s_fence)
133 return s_fence->owner;
134 else if (a_fence)
135 return a_fence->owner;
136 return AMDGPU_FENCE_OWNER_UNDEFINED;
137}
138
139/** 147/**
140 * amdgpu_sync_resv - sync to a reservation object 148 * amdgpu_sync_resv - sync to a reservation object
141 * 149 *
@@ -208,7 +216,7 @@ struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
208 f = e->fence; 216 f = e->fence;
209 217
210 hash_del(&e->node); 218 hash_del(&e->node);
211 kfree(e); 219 kmem_cache_free(amdgpu_sync_slab, e);
212 220
213 if (!fence_is_signaled(f)) 221 if (!fence_is_signaled(f))
214 return f; 222 return f;
@@ -231,7 +239,7 @@ int amdgpu_sync_wait(struct amdgpu_sync *sync)
231 239
232 hash_del(&e->node); 240 hash_del(&e->node);
233 fence_put(e->fence); 241 fence_put(e->fence);
234 kfree(e); 242 kmem_cache_free(amdgpu_sync_slab, e);
235 } 243 }
236 244
237 return 0; 245 return 0;
@@ -253,8 +261,34 @@ void amdgpu_sync_free(struct amdgpu_sync *sync)
253 hash_for_each_safe(sync->fences, i, tmp, e, node) { 261 hash_for_each_safe(sync->fences, i, tmp, e, node) {
254 hash_del(&e->node); 262 hash_del(&e->node);
255 fence_put(e->fence); 263 fence_put(e->fence);
256 kfree(e); 264 kmem_cache_free(amdgpu_sync_slab, e);
257 } 265 }
258 266
259 fence_put(sync->last_vm_update); 267 fence_put(sync->last_vm_update);
260} 268}
269
270/**
271 * amdgpu_sync_init - init sync object subsystem
272 *
273 * Allocate the slab allocator.
274 */
275int amdgpu_sync_init(void)
276{
277 amdgpu_sync_slab = kmem_cache_create(
278 "amdgpu_sync", sizeof(struct amdgpu_sync_entry), 0,
279 SLAB_HWCACHE_ALIGN, NULL);
280 if (!amdgpu_sync_slab)
281 return -ENOMEM;
282
283 return 0;
284}
285
286/**
287 * amdgpu_sync_fini - fini sync object subsystem
288 *
289 * Free the slab allocator.
290 */
291void amdgpu_sync_fini(void)
292{
293 kmem_cache_destroy(amdgpu_sync_slab);
294}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 9ccdd189d717..0f42b1a24446 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -494,29 +494,32 @@ static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_re
494/* 494/*
495 * TTM backend functions. 495 * TTM backend functions.
496 */ 496 */
497struct amdgpu_ttm_gup_task_list {
498 struct list_head list;
499 struct task_struct *task;
500};
501
497struct amdgpu_ttm_tt { 502struct amdgpu_ttm_tt {
498 struct ttm_dma_tt ttm; 503 struct ttm_dma_tt ttm;
499 struct amdgpu_device *adev; 504 struct amdgpu_device *adev;
500 u64 offset; 505 u64 offset;
501 uint64_t userptr; 506 uint64_t userptr;
502 struct mm_struct *usermm; 507 struct mm_struct *usermm;
503 uint32_t userflags; 508 uint32_t userflags;
509 spinlock_t guptasklock;
510 struct list_head guptasks;
511 atomic_t mmu_invalidations;
504}; 512};
505 513
506/* prepare the sg table with the user pages */ 514int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
507static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
508{ 515{
509 struct amdgpu_device *adev = amdgpu_get_adev(ttm->bdev);
510 struct amdgpu_ttm_tt *gtt = (void *)ttm; 516 struct amdgpu_ttm_tt *gtt = (void *)ttm;
511 unsigned pinned = 0, nents;
512 int r;
513
514 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); 517 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
515 enum dma_data_direction direction = write ? 518 unsigned pinned = 0;
516 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 519 int r;
517 520
518 if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) { 521 if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
519 /* check that we only pin down anonymous memory 522 /* check that we only use anonymous memory
520 to prevent problems with writeback */ 523 to prevent problems with writeback */
521 unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE; 524 unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
522 struct vm_area_struct *vma; 525 struct vm_area_struct *vma;
@@ -529,10 +532,21 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
529 do { 532 do {
530 unsigned num_pages = ttm->num_pages - pinned; 533 unsigned num_pages = ttm->num_pages - pinned;
531 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; 534 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
532 struct page **pages = ttm->pages + pinned; 535 struct page **p = pages + pinned;
536 struct amdgpu_ttm_gup_task_list guptask;
537
538 guptask.task = current;
539 spin_lock(&gtt->guptasklock);
540 list_add(&guptask.list, &gtt->guptasks);
541 spin_unlock(&gtt->guptasklock);
533 542
534 r = get_user_pages(current, current->mm, userptr, num_pages, 543 r = get_user_pages(current, current->mm, userptr, num_pages,
535 write, 0, pages, NULL); 544 write, 0, p, NULL);
545
546 spin_lock(&gtt->guptasklock);
547 list_del(&guptask.list);
548 spin_unlock(&gtt->guptasklock);
549
536 if (r < 0) 550 if (r < 0)
537 goto release_pages; 551 goto release_pages;
538 552
@@ -540,6 +554,25 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
540 554
541 } while (pinned < ttm->num_pages); 555 } while (pinned < ttm->num_pages);
542 556
557 return 0;
558
559release_pages:
560 release_pages(pages, pinned, 0);
561 return r;
562}
563
564/* prepare the sg table with the user pages */
565static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
566{
567 struct amdgpu_device *adev = amdgpu_get_adev(ttm->bdev);
568 struct amdgpu_ttm_tt *gtt = (void *)ttm;
569 unsigned nents;
570 int r;
571
572 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
573 enum dma_data_direction direction = write ?
574 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
575
543 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, 576 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
544 ttm->num_pages << PAGE_SHIFT, 577 ttm->num_pages << PAGE_SHIFT,
545 GFP_KERNEL); 578 GFP_KERNEL);
@@ -558,9 +591,6 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
558 591
559release_sg: 592release_sg:
560 kfree(ttm->sg); 593 kfree(ttm->sg);
561
562release_pages:
563 release_pages(ttm->pages, pinned, 0);
564 return r; 594 return r;
565} 595}
566 596
@@ -783,6 +813,10 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
783 gtt->userptr = addr; 813 gtt->userptr = addr;
784 gtt->usermm = current->mm; 814 gtt->usermm = current->mm;
785 gtt->userflags = flags; 815 gtt->userflags = flags;
816 spin_lock_init(&gtt->guptasklock);
817 INIT_LIST_HEAD(&gtt->guptasks);
818 atomic_set(&gtt->mmu_invalidations, 0);
819
786 return 0; 820 return 0;
787} 821}
788 822
@@ -800,21 +834,40 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
800 unsigned long end) 834 unsigned long end)
801{ 835{
802 struct amdgpu_ttm_tt *gtt = (void *)ttm; 836 struct amdgpu_ttm_tt *gtt = (void *)ttm;
837 struct amdgpu_ttm_gup_task_list *entry;
803 unsigned long size; 838 unsigned long size;
804 839
805 if (gtt == NULL) 840 if (gtt == NULL || !gtt->userptr)
806 return false;
807
808 if (gtt->ttm.ttm.state != tt_bound || !gtt->userptr)
809 return false; 841 return false;
810 842
811 size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE; 843 size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;
812 if (gtt->userptr > end || gtt->userptr + size <= start) 844 if (gtt->userptr > end || gtt->userptr + size <= start)
813 return false; 845 return false;
814 846
847 spin_lock(&gtt->guptasklock);
848 list_for_each_entry(entry, &gtt->guptasks, list) {
849 if (entry->task == current) {
850 spin_unlock(&gtt->guptasklock);
851 return false;
852 }
853 }
854 spin_unlock(&gtt->guptasklock);
855
856 atomic_inc(&gtt->mmu_invalidations);
857
815 return true; 858 return true;
816} 859}
817 860
861bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
862 int *last_invalidated)
863{
864 struct amdgpu_ttm_tt *gtt = (void *)ttm;
865 int prev_invalidated = *last_invalidated;
866
867 *last_invalidated = atomic_read(&gtt->mmu_invalidations);
868 return prev_invalidated != *last_invalidated;
869}
870
818bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm) 871bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
819{ 872{
820 struct amdgpu_ttm_tt *gtt = (void *)ttm; 873 struct amdgpu_ttm_tt *gtt = (void *)ttm;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index 1de82bf4fc79..e5f0a5e29551 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -886,8 +886,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
886 ib->length_dw = 16; 886 ib->length_dw = 16;
887 887
888 if (direct) { 888 if (direct) {
889 r = amdgpu_ib_schedule(ring, 1, ib, 889 r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
890 AMDGPU_FENCE_OWNER_UNDEFINED, NULL, &f);
891 if (r) 890 if (r)
892 goto err_free; 891 goto err_free;
893 892
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 39c3aa60381a..6d191fb40b38 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -425,8 +425,7 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
425 for (i = ib->length_dw; i < ib_size_dw; ++i) 425 for (i = ib->length_dw; i < ib_size_dw; ++i)
426 ib->ptr[i] = 0x0; 426 ib->ptr[i] = 0x0;
427 427
428 r = amdgpu_ib_schedule(ring, 1, ib, AMDGPU_FENCE_OWNER_UNDEFINED, 428 r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
429 NULL, &f);
430 if (r) 429 if (r)
431 goto err; 430 goto err;
432 431
@@ -487,9 +486,7 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
487 ib->ptr[i] = 0x0; 486 ib->ptr[i] = 0x0;
488 487
489 if (direct) { 488 if (direct) {
490 r = amdgpu_ib_schedule(ring, 1, ib, 489 r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
491 AMDGPU_FENCE_OWNER_UNDEFINED,
492 NULL, &f);
493 if (r) 490 if (r)
494 goto err; 491 goto err;
495 492
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index d9dc8bea5e98..b6c011b83641 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -95,6 +95,7 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
95 entry->priority = 0; 95 entry->priority = 0;
96 entry->tv.bo = &vm->page_directory->tbo; 96 entry->tv.bo = &vm->page_directory->tbo;
97 entry->tv.shared = true; 97 entry->tv.shared = true;
98 entry->user_pages = NULL;
98 list_add(&entry->tv.head, validated); 99 list_add(&entry->tv.head, validated);
99} 100}
100 101
@@ -188,6 +189,13 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
188 if (!is_later && owner == (long)id && 189 if (!is_later && owner == (long)id &&
189 pd_addr == id->pd_gpu_addr) { 190 pd_addr == id->pd_gpu_addr) {
190 191
192 r = amdgpu_sync_fence(ring->adev, sync,
193 id->mgr_id->active);
194 if (r) {
195 mutex_unlock(&adev->vm_manager.lock);
196 return r;
197 }
198
191 fence_put(id->mgr_id->active); 199 fence_put(id->mgr_id->active);
192 id->mgr_id->active = fence_get(fence); 200 id->mgr_id->active = fence_get(fence);
193 201
@@ -234,19 +242,68 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
234 * amdgpu_vm_flush - hardware flush the vm 242 * amdgpu_vm_flush - hardware flush the vm
235 * 243 *
236 * @ring: ring to use for flush 244 * @ring: ring to use for flush
237 * @vmid: vmid number to use 245 * @vm_id: vmid number to use
238 * @pd_addr: address of the page directory 246 * @pd_addr: address of the page directory
239 * 247 *
240 * Emit a VM flush when it is necessary. 248 * Emit a VM flush when it is necessary.
241 */ 249 */
242void amdgpu_vm_flush(struct amdgpu_ring *ring, 250void amdgpu_vm_flush(struct amdgpu_ring *ring,
243 unsigned vmid, 251 unsigned vm_id, uint64_t pd_addr,
244 uint64_t pd_addr) 252 uint32_t gds_base, uint32_t gds_size,
253 uint32_t gws_base, uint32_t gws_size,
254 uint32_t oa_base, uint32_t oa_size)
245{ 255{
256 struct amdgpu_device *adev = ring->adev;
257 struct amdgpu_vm_manager_id *mgr_id = &adev->vm_manager.ids[vm_id];
258 bool gds_switch_needed = ring->funcs->emit_gds_switch && (
259 mgr_id->gds_base != gds_base ||
260 mgr_id->gds_size != gds_size ||
261 mgr_id->gws_base != gws_base ||
262 mgr_id->gws_size != gws_size ||
263 mgr_id->oa_base != oa_base ||
264 mgr_id->oa_size != oa_size);
265
266 if (ring->funcs->emit_pipeline_sync && (
267 pd_addr != AMDGPU_VM_NO_FLUSH || gds_switch_needed))
268 amdgpu_ring_emit_pipeline_sync(ring);
269
246 if (pd_addr != AMDGPU_VM_NO_FLUSH) { 270 if (pd_addr != AMDGPU_VM_NO_FLUSH) {
247 trace_amdgpu_vm_flush(pd_addr, ring->idx, vmid); 271 trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id);
248 amdgpu_ring_emit_vm_flush(ring, vmid, pd_addr); 272 amdgpu_ring_emit_vm_flush(ring, vm_id, pd_addr);
249 } 273 }
274
275 if (gds_switch_needed) {
276 mgr_id->gds_base = gds_base;
277 mgr_id->gds_size = gds_size;
278 mgr_id->gws_base = gws_base;
279 mgr_id->gws_size = gws_size;
280 mgr_id->oa_base = oa_base;
281 mgr_id->oa_size = oa_size;
282 amdgpu_ring_emit_gds_switch(ring, vm_id,
283 gds_base, gds_size,
284 gws_base, gws_size,
285 oa_base, oa_size);
286 }
287}
288
289/**
290 * amdgpu_vm_reset_id - reset VMID to zero
291 *
292 * @adev: amdgpu device structure
293 * @vm_id: vmid number to use
294 *
295 * Reset saved GDW, GWS and OA to force switch on next flush.
296 */
297void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id)
298{
299 struct amdgpu_vm_manager_id *mgr_id = &adev->vm_manager.ids[vm_id];
300
301 mgr_id->gds_base = 0;
302 mgr_id->gds_size = 0;
303 mgr_id->gws_base = 0;
304 mgr_id->gws_size = 0;
305 mgr_id->oa_base = 0;
306 mgr_id->oa_size = 0;
250} 307}
251 308
252/** 309/**
@@ -810,7 +867,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
810 while (start != mapping->it.last + 1) { 867 while (start != mapping->it.last + 1) {
811 uint64_t last; 868 uint64_t last;
812 869
813 last = min((uint64_t)mapping->it.last, start + max_size); 870 last = min((uint64_t)mapping->it.last, start + max_size - 1);
814 r = amdgpu_vm_bo_update_mapping(adev, gtt, gtt_flags, vm, 871 r = amdgpu_vm_bo_update_mapping(adev, gtt, gtt_flags, vm,
815 start, last, flags, addr, 872 start, last, flags, addr,
816 fence); 873 fence);
@@ -818,7 +875,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
818 return r; 875 return r;
819 876
820 start = last + 1; 877 start = last + 1;
821 addr += max_size; 878 addr += max_size * AMDGPU_GPU_PAGE_SIZE;
822 } 879 }
823 880
824 return 0; 881 return 0;
@@ -914,22 +971,18 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
914 struct amdgpu_bo_va_mapping *mapping; 971 struct amdgpu_bo_va_mapping *mapping;
915 int r; 972 int r;
916 973
917 spin_lock(&vm->freed_lock);
918 while (!list_empty(&vm->freed)) { 974 while (!list_empty(&vm->freed)) {
919 mapping = list_first_entry(&vm->freed, 975 mapping = list_first_entry(&vm->freed,
920 struct amdgpu_bo_va_mapping, list); 976 struct amdgpu_bo_va_mapping, list);
921 list_del(&mapping->list); 977 list_del(&mapping->list);
922 spin_unlock(&vm->freed_lock); 978
923 r = amdgpu_vm_bo_split_mapping(adev, NULL, 0, vm, mapping, 979 r = amdgpu_vm_bo_split_mapping(adev, NULL, 0, vm, mapping,
924 0, NULL); 980 0, NULL);
925 kfree(mapping); 981 kfree(mapping);
926 if (r) 982 if (r)
927 return r; 983 return r;
928 984
929 spin_lock(&vm->freed_lock);
930 } 985 }
931 spin_unlock(&vm->freed_lock);
932
933 return 0; 986 return 0;
934 987
935} 988}
@@ -956,9 +1009,8 @@ int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
956 bo_va = list_first_entry(&vm->invalidated, 1009 bo_va = list_first_entry(&vm->invalidated,
957 struct amdgpu_bo_va, vm_status); 1010 struct amdgpu_bo_va, vm_status);
958 spin_unlock(&vm->status_lock); 1011 spin_unlock(&vm->status_lock);
959 mutex_lock(&bo_va->mutex); 1012
960 r = amdgpu_vm_bo_update(adev, bo_va, NULL); 1013 r = amdgpu_vm_bo_update(adev, bo_va, NULL);
961 mutex_unlock(&bo_va->mutex);
962 if (r) 1014 if (r)
963 return r; 1015 return r;
964 1016
@@ -1002,7 +1054,7 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1002 INIT_LIST_HEAD(&bo_va->valids); 1054 INIT_LIST_HEAD(&bo_va->valids);
1003 INIT_LIST_HEAD(&bo_va->invalids); 1055 INIT_LIST_HEAD(&bo_va->invalids);
1004 INIT_LIST_HEAD(&bo_va->vm_status); 1056 INIT_LIST_HEAD(&bo_va->vm_status);
1005 mutex_init(&bo_va->mutex); 1057
1006 list_add_tail(&bo_va->bo_list, &bo->va); 1058 list_add_tail(&bo_va->bo_list, &bo->va);
1007 1059
1008 return bo_va; 1060 return bo_va;
@@ -1054,9 +1106,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1054 saddr /= AMDGPU_GPU_PAGE_SIZE; 1106 saddr /= AMDGPU_GPU_PAGE_SIZE;
1055 eaddr /= AMDGPU_GPU_PAGE_SIZE; 1107 eaddr /= AMDGPU_GPU_PAGE_SIZE;
1056 1108
1057 spin_lock(&vm->it_lock);
1058 it = interval_tree_iter_first(&vm->va, saddr, eaddr); 1109 it = interval_tree_iter_first(&vm->va, saddr, eaddr);
1059 spin_unlock(&vm->it_lock);
1060 if (it) { 1110 if (it) {
1061 struct amdgpu_bo_va_mapping *tmp; 1111 struct amdgpu_bo_va_mapping *tmp;
1062 tmp = container_of(it, struct amdgpu_bo_va_mapping, it); 1112 tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
@@ -1080,13 +1130,8 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1080 mapping->offset = offset; 1130 mapping->offset = offset;
1081 mapping->flags = flags; 1131 mapping->flags = flags;
1082 1132
1083 mutex_lock(&bo_va->mutex);
1084 list_add(&mapping->list, &bo_va->invalids); 1133 list_add(&mapping->list, &bo_va->invalids);
1085 mutex_unlock(&bo_va->mutex);
1086 spin_lock(&vm->it_lock);
1087 interval_tree_insert(&mapping->it, &vm->va); 1134 interval_tree_insert(&mapping->it, &vm->va);
1088 spin_unlock(&vm->it_lock);
1089 trace_amdgpu_vm_bo_map(bo_va, mapping);
1090 1135
1091 /* Make sure the page tables are allocated */ 1136 /* Make sure the page tables are allocated */
1092 saddr >>= amdgpu_vm_block_size; 1137 saddr >>= amdgpu_vm_block_size;
@@ -1130,6 +1175,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1130 entry->priority = 0; 1175 entry->priority = 0;
1131 entry->tv.bo = &entry->robj->tbo; 1176 entry->tv.bo = &entry->robj->tbo;
1132 entry->tv.shared = true; 1177 entry->tv.shared = true;
1178 entry->user_pages = NULL;
1133 vm->page_tables[pt_idx].addr = 0; 1179 vm->page_tables[pt_idx].addr = 0;
1134 } 1180 }
1135 1181
@@ -1137,9 +1183,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1137 1183
1138error_free: 1184error_free:
1139 list_del(&mapping->list); 1185 list_del(&mapping->list);
1140 spin_lock(&vm->it_lock);
1141 interval_tree_remove(&mapping->it, &vm->va); 1186 interval_tree_remove(&mapping->it, &vm->va);
1142 spin_unlock(&vm->it_lock);
1143 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1187 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1144 kfree(mapping); 1188 kfree(mapping);
1145 1189
@@ -1168,7 +1212,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1168 bool valid = true; 1212 bool valid = true;
1169 1213
1170 saddr /= AMDGPU_GPU_PAGE_SIZE; 1214 saddr /= AMDGPU_GPU_PAGE_SIZE;
1171 mutex_lock(&bo_va->mutex); 1215
1172 list_for_each_entry(mapping, &bo_va->valids, list) { 1216 list_for_each_entry(mapping, &bo_va->valids, list) {
1173 if (mapping->it.start == saddr) 1217 if (mapping->it.start == saddr)
1174 break; 1218 break;
@@ -1182,25 +1226,18 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1182 break; 1226 break;
1183 } 1227 }
1184 1228
1185 if (&mapping->list == &bo_va->invalids) { 1229 if (&mapping->list == &bo_va->invalids)
1186 mutex_unlock(&bo_va->mutex);
1187 return -ENOENT; 1230 return -ENOENT;
1188 }
1189 } 1231 }
1190 mutex_unlock(&bo_va->mutex); 1232
1191 list_del(&mapping->list); 1233 list_del(&mapping->list);
1192 spin_lock(&vm->it_lock);
1193 interval_tree_remove(&mapping->it, &vm->va); 1234 interval_tree_remove(&mapping->it, &vm->va);
1194 spin_unlock(&vm->it_lock);
1195 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1235 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1196 1236
1197 if (valid) { 1237 if (valid)
1198 spin_lock(&vm->freed_lock);
1199 list_add(&mapping->list, &vm->freed); 1238 list_add(&mapping->list, &vm->freed);
1200 spin_unlock(&vm->freed_lock); 1239 else
1201 } else {
1202 kfree(mapping); 1240 kfree(mapping);
1203 }
1204 1241
1205 return 0; 1242 return 0;
1206} 1243}
@@ -1229,23 +1266,17 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
1229 1266
1230 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 1267 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
1231 list_del(&mapping->list); 1268 list_del(&mapping->list);
1232 spin_lock(&vm->it_lock);
1233 interval_tree_remove(&mapping->it, &vm->va); 1269 interval_tree_remove(&mapping->it, &vm->va);
1234 spin_unlock(&vm->it_lock);
1235 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1270 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1236 spin_lock(&vm->freed_lock);
1237 list_add(&mapping->list, &vm->freed); 1271 list_add(&mapping->list, &vm->freed);
1238 spin_unlock(&vm->freed_lock);
1239 } 1272 }
1240 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 1273 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
1241 list_del(&mapping->list); 1274 list_del(&mapping->list);
1242 spin_lock(&vm->it_lock);
1243 interval_tree_remove(&mapping->it, &vm->va); 1275 interval_tree_remove(&mapping->it, &vm->va);
1244 spin_unlock(&vm->it_lock);
1245 kfree(mapping); 1276 kfree(mapping);
1246 } 1277 }
1278
1247 fence_put(bo_va->last_pt_update); 1279 fence_put(bo_va->last_pt_update);
1248 mutex_destroy(&bo_va->mutex);
1249 kfree(bo_va); 1280 kfree(bo_va);
1250} 1281}
1251 1282
@@ -1298,8 +1329,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1298 INIT_LIST_HEAD(&vm->invalidated); 1329 INIT_LIST_HEAD(&vm->invalidated);
1299 INIT_LIST_HEAD(&vm->cleared); 1330 INIT_LIST_HEAD(&vm->cleared);
1300 INIT_LIST_HEAD(&vm->freed); 1331 INIT_LIST_HEAD(&vm->freed);
1301 spin_lock_init(&vm->it_lock); 1332
1302 spin_lock_init(&vm->freed_lock);
1303 pd_size = amdgpu_vm_directory_size(adev); 1333 pd_size = amdgpu_vm_directory_size(adev);
1304 pd_entries = amdgpu_vm_num_pdes(adev); 1334 pd_entries = amdgpu_vm_num_pdes(adev);
1305 1335
@@ -1386,6 +1416,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1386 1416
1387 amdgpu_bo_unref(&vm->page_directory); 1417 amdgpu_bo_unref(&vm->page_directory);
1388 fence_put(vm->page_directory_fence); 1418 fence_put(vm->page_directory_fence);
1419
1389 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 1420 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1390 struct amdgpu_vm_id *id = &vm->ids[i]; 1421 struct amdgpu_vm_id *id = &vm->ids[i];
1391 1422
@@ -1410,9 +1441,11 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
1410 INIT_LIST_HEAD(&adev->vm_manager.ids_lru); 1441 INIT_LIST_HEAD(&adev->vm_manager.ids_lru);
1411 1442
1412 /* skip over VMID 0, since it is the system VM */ 1443 /* skip over VMID 0, since it is the system VM */
1413 for (i = 1; i < adev->vm_manager.num_ids; ++i) 1444 for (i = 1; i < adev->vm_manager.num_ids; ++i) {
1445 amdgpu_vm_reset_id(adev, i);
1414 list_add_tail(&adev->vm_manager.ids[i].list, 1446 list_add_tail(&adev->vm_manager.ids[i].list,
1415 &adev->vm_manager.ids_lru); 1447 &adev->vm_manager.ids_lru);
1448 }
1416 1449
1417 atomic_set(&adev->vm_manager.vm_pte_next_ring, 0); 1450 atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
1418} 1451}
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index 474ca02b0949..1f9109d3348b 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -3017,7 +3017,6 @@ static int ci_populate_single_memory_level(struct amdgpu_device *adev,
3017 &memory_level->MinVddcPhases); 3017 &memory_level->MinVddcPhases);
3018 3018
3019 memory_level->EnabledForThrottle = 1; 3019 memory_level->EnabledForThrottle = 1;
3020 memory_level->EnabledForActivity = 1;
3021 memory_level->UpH = 0; 3020 memory_level->UpH = 0;
3022 memory_level->DownH = 100; 3021 memory_level->DownH = 100;
3023 memory_level->VoltageDownH = 0; 3022 memory_level->VoltageDownH = 0;
@@ -3376,7 +3375,6 @@ static int ci_populate_single_graphic_level(struct amdgpu_device *adev,
3376 graphic_level->SpllSpreadSpectrum2 = cpu_to_be32(graphic_level->SpllSpreadSpectrum2); 3375 graphic_level->SpllSpreadSpectrum2 = cpu_to_be32(graphic_level->SpllSpreadSpectrum2);
3377 graphic_level->CcPwrDynRm = cpu_to_be32(graphic_level->CcPwrDynRm); 3376 graphic_level->CcPwrDynRm = cpu_to_be32(graphic_level->CcPwrDynRm);
3378 graphic_level->CcPwrDynRm1 = cpu_to_be32(graphic_level->CcPwrDynRm1); 3377 graphic_level->CcPwrDynRm1 = cpu_to_be32(graphic_level->CcPwrDynRm1);
3379 graphic_level->EnabledForActivity = 1;
3380 3378
3381 return 0; 3379 return 0;
3382} 3380}
@@ -3407,6 +3405,7 @@ static int ci_populate_all_graphic_levels(struct amdgpu_device *adev)
3407 pi->smc_state_table.GraphicsLevel[i].DisplayWatermark = 3405 pi->smc_state_table.GraphicsLevel[i].DisplayWatermark =
3408 PPSMC_DISPLAY_WATERMARK_HIGH; 3406 PPSMC_DISPLAY_WATERMARK_HIGH;
3409 } 3407 }
3408 pi->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1;
3410 3409
3411 pi->smc_state_table.GraphicsDpmLevelCount = (u8)dpm_table->sclk_table.count; 3410 pi->smc_state_table.GraphicsDpmLevelCount = (u8)dpm_table->sclk_table.count;
3412 pi->dpm_level_enable_mask.sclk_dpm_enable_mask = 3411 pi->dpm_level_enable_mask.sclk_dpm_enable_mask =
@@ -3450,6 +3449,8 @@ static int ci_populate_all_memory_levels(struct amdgpu_device *adev)
3450 return ret; 3449 return ret;
3451 } 3450 }
3452 3451
3452 pi->smc_state_table.MemoryLevel[0].EnabledForActivity = 1;
3453
3453 if ((dpm_table->mclk_table.count >= 2) && 3454 if ((dpm_table->mclk_table.count >= 2) &&
3454 ((adev->pdev->device == 0x67B0) || (adev->pdev->device == 0x67B1))) { 3455 ((adev->pdev->device == 0x67B0) || (adev->pdev->device == 0x67B1))) {
3455 pi->smc_state_table.MemoryLevel[1].MinVddc = 3456 pi->smc_state_table.MemoryLevel[1].MinVddc =
@@ -4381,26 +4382,6 @@ static int ci_dpm_force_performance_level(struct amdgpu_device *adev,
4381 } 4382 }
4382 } 4383 }
4383 } 4384 }
4384 if ((!pi->pcie_dpm_key_disabled) &&
4385 pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
4386 levels = 0;
4387 tmp = pi->dpm_level_enable_mask.pcie_dpm_enable_mask;
4388 while (tmp >>= 1)
4389 levels++;
4390 if (levels) {
4391 ret = ci_dpm_force_state_pcie(adev, level);
4392 if (ret)
4393 return ret;
4394 for (i = 0; i < adev->usec_timeout; i++) {
4395 tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX_1) &
4396 TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX_MASK) >>
4397 TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX__SHIFT;
4398 if (tmp == levels)
4399 break;
4400 udelay(1);
4401 }
4402 }
4403 }
4404 } else if (level == AMDGPU_DPM_FORCED_LEVEL_LOW) { 4385 } else if (level == AMDGPU_DPM_FORCED_LEVEL_LOW) {
4405 if ((!pi->sclk_dpm_key_disabled) && 4386 if ((!pi->sclk_dpm_key_disabled) &&
4406 pi->dpm_level_enable_mask.sclk_dpm_enable_mask) { 4387 pi->dpm_level_enable_mask.sclk_dpm_enable_mask) {
@@ -5395,30 +5376,6 @@ static int ci_dpm_enable(struct amdgpu_device *adev)
5395 5376
5396 ci_update_current_ps(adev, boot_ps); 5377 ci_update_current_ps(adev, boot_ps);
5397 5378
5398 if (adev->irq.installed &&
5399 amdgpu_is_internal_thermal_sensor(adev->pm.int_thermal_type)) {
5400#if 0
5401 PPSMC_Result result;
5402#endif
5403 ret = ci_thermal_set_temperature_range(adev, CISLANDS_TEMP_RANGE_MIN,
5404 CISLANDS_TEMP_RANGE_MAX);
5405 if (ret) {
5406 DRM_ERROR("ci_thermal_set_temperature_range failed\n");
5407 return ret;
5408 }
5409 amdgpu_irq_get(adev, &adev->pm.dpm.thermal.irq,
5410 AMDGPU_THERMAL_IRQ_LOW_TO_HIGH);
5411 amdgpu_irq_get(adev, &adev->pm.dpm.thermal.irq,
5412 AMDGPU_THERMAL_IRQ_HIGH_TO_LOW);
5413
5414#if 0
5415 result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_EnableThermalInterrupt);
5416
5417 if (result != PPSMC_Result_OK)
5418 DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
5419#endif
5420 }
5421
5422 return 0; 5379 return 0;
5423} 5380}
5424 5381
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 192ab13e9f05..bddc9ba11495 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -2028,8 +2028,6 @@ static int cik_common_early_init(void *handle)
2028 2028
2029 adev->asic_funcs = &cik_asic_funcs; 2029 adev->asic_funcs = &cik_asic_funcs;
2030 2030
2031 adev->has_uvd = true;
2032
2033 adev->rev_id = cik_get_rev_id(adev); 2031 adev->rev_id = cik_get_rev_id(adev);
2034 adev->external_rev_id = 0xFF; 2032 adev->external_rev_id = 0xFF;
2035 switch (adev->asic_type) { 2033 switch (adev->asic_type) {
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index 266db15daf2c..02122197d2b6 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -261,6 +261,13 @@ static void cik_sdma_ring_emit_hdp_flush(struct amdgpu_ring *ring)
261 amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */ 261 amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */
262} 262}
263 263
264static void cik_sdma_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
265{
266 amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
267 amdgpu_ring_write(ring, mmHDP_DEBUG0);
268 amdgpu_ring_write(ring, 1);
269}
270
264/** 271/**
265 * cik_sdma_ring_emit_fence - emit a fence on the DMA ring 272 * cik_sdma_ring_emit_fence - emit a fence on the DMA ring
266 * 273 *
@@ -636,8 +643,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring)
636 ib.ptr[3] = 1; 643 ib.ptr[3] = 1;
637 ib.ptr[4] = 0xDEADBEEF; 644 ib.ptr[4] = 0xDEADBEEF;
638 ib.length_dw = 5; 645 ib.length_dw = 5;
639 r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, 646 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
640 NULL, &f);
641 if (r) 647 if (r)
642 goto err1; 648 goto err1;
643 649
@@ -816,6 +822,30 @@ static void cik_sdma_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
816} 822}
817 823
818/** 824/**
825 * cik_sdma_ring_emit_pipeline_sync - sync the pipeline
826 *
827 * @ring: amdgpu_ring pointer
828 *
829 * Make sure all previous operations are completed (CIK).
830 */
831static void cik_sdma_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
832{
833 uint32_t seq = ring->fence_drv.sync_seq;
834 uint64_t addr = ring->fence_drv.gpu_addr;
835
836 /* wait for idle */
837 amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0,
838 SDMA_POLL_REG_MEM_EXTRA_OP(0) |
839 SDMA_POLL_REG_MEM_EXTRA_FUNC(3) | /* equal */
840 SDMA_POLL_REG_MEM_EXTRA_M));
841 amdgpu_ring_write(ring, addr & 0xfffffffc);
842 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
843 amdgpu_ring_write(ring, seq); /* reference */
844 amdgpu_ring_write(ring, 0xfffffff); /* mask */
845 amdgpu_ring_write(ring, (0xfff << 16) | 4); /* retry count, poll interval */
846}
847
848/**
819 * cik_sdma_ring_emit_vm_flush - cik vm flush using sDMA 849 * cik_sdma_ring_emit_vm_flush - cik vm flush using sDMA
820 * 850 *
821 * @ring: amdgpu_ring pointer 851 * @ring: amdgpu_ring pointer
@@ -1270,8 +1300,10 @@ static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = {
1270 .parse_cs = NULL, 1300 .parse_cs = NULL,
1271 .emit_ib = cik_sdma_ring_emit_ib, 1301 .emit_ib = cik_sdma_ring_emit_ib,
1272 .emit_fence = cik_sdma_ring_emit_fence, 1302 .emit_fence = cik_sdma_ring_emit_fence,
1303 .emit_pipeline_sync = cik_sdma_ring_emit_pipeline_sync,
1273 .emit_vm_flush = cik_sdma_ring_emit_vm_flush, 1304 .emit_vm_flush = cik_sdma_ring_emit_vm_flush,
1274 .emit_hdp_flush = cik_sdma_ring_emit_hdp_flush, 1305 .emit_hdp_flush = cik_sdma_ring_emit_hdp_flush,
1306 .emit_hdp_invalidate = cik_sdma_ring_emit_hdp_invalidate,
1275 .test_ring = cik_sdma_ring_test_ring, 1307 .test_ring = cik_sdma_ring_test_ring,
1276 .test_ib = cik_sdma_ring_test_ib, 1308 .test_ib = cik_sdma_ring_test_ib,
1277 .insert_nop = cik_sdma_ring_insert_nop, 1309 .insert_nop = cik_sdma_ring_insert_nop,
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index e3ff809a0cae..6de2ce535e37 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -1668,6 +1668,9 @@ static void dce_v10_0_audio_fini(struct amdgpu_device *adev)
1668{ 1668{
1669 int i; 1669 int i;
1670 1670
1671 if (!amdgpu_audio)
1672 return;
1673
1671 if (!adev->mode_info.audio.enabled) 1674 if (!adev->mode_info.audio.enabled)
1672 return; 1675 return;
1673 1676
@@ -1973,7 +1976,7 @@ static void dce_v10_0_afmt_enable(struct drm_encoder *encoder, bool enable)
1973 enable ? "En" : "Dis", dig->afmt->offset, amdgpu_encoder->encoder_id); 1976 enable ? "En" : "Dis", dig->afmt->offset, amdgpu_encoder->encoder_id);
1974} 1977}
1975 1978
1976static void dce_v10_0_afmt_init(struct amdgpu_device *adev) 1979static int dce_v10_0_afmt_init(struct amdgpu_device *adev)
1977{ 1980{
1978 int i; 1981 int i;
1979 1982
@@ -1986,8 +1989,16 @@ static void dce_v10_0_afmt_init(struct amdgpu_device *adev)
1986 if (adev->mode_info.afmt[i]) { 1989 if (adev->mode_info.afmt[i]) {
1987 adev->mode_info.afmt[i]->offset = dig_offsets[i]; 1990 adev->mode_info.afmt[i]->offset = dig_offsets[i];
1988 adev->mode_info.afmt[i]->id = i; 1991 adev->mode_info.afmt[i]->id = i;
1992 } else {
1993 int j;
1994 for (j = 0; j < i; j++) {
1995 kfree(adev->mode_info.afmt[j]);
1996 adev->mode_info.afmt[j] = NULL;
1997 }
1998 return -ENOMEM;
1989 } 1999 }
1990 } 2000 }
2001 return 0;
1991} 2002}
1992 2003
1993static void dce_v10_0_afmt_fini(struct amdgpu_device *adev) 2004static void dce_v10_0_afmt_fini(struct amdgpu_device *adev)
@@ -2064,8 +2075,7 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
2064 if (atomic) { 2075 if (atomic) {
2065 amdgpu_fb = to_amdgpu_framebuffer(fb); 2076 amdgpu_fb = to_amdgpu_framebuffer(fb);
2066 target_fb = fb; 2077 target_fb = fb;
2067 } 2078 } else {
2068 else {
2069 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); 2079 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
2070 target_fb = crtc->primary->fb; 2080 target_fb = crtc->primary->fb;
2071 } 2081 }
@@ -2079,9 +2089,9 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
2079 if (unlikely(r != 0)) 2089 if (unlikely(r != 0))
2080 return r; 2090 return r;
2081 2091
2082 if (atomic) 2092 if (atomic) {
2083 fb_location = amdgpu_bo_gpu_offset(rbo); 2093 fb_location = amdgpu_bo_gpu_offset(rbo);
2084 else { 2094 } else {
2085 r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location); 2095 r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
2086 if (unlikely(r != 0)) { 2096 if (unlikely(r != 0)) {
2087 amdgpu_bo_unreserve(rbo); 2097 amdgpu_bo_unreserve(rbo);
@@ -2700,13 +2710,13 @@ static void dce_v10_0_crtc_dpms(struct drm_crtc *crtc, int mode)
2700 type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id); 2710 type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id);
2701 amdgpu_irq_update(adev, &adev->crtc_irq, type); 2711 amdgpu_irq_update(adev, &adev->crtc_irq, type);
2702 amdgpu_irq_update(adev, &adev->pageflip_irq, type); 2712 amdgpu_irq_update(adev, &adev->pageflip_irq, type);
2703 drm_vblank_post_modeset(dev, amdgpu_crtc->crtc_id); 2713 drm_vblank_on(dev, amdgpu_crtc->crtc_id);
2704 dce_v10_0_crtc_load_lut(crtc); 2714 dce_v10_0_crtc_load_lut(crtc);
2705 break; 2715 break;
2706 case DRM_MODE_DPMS_STANDBY: 2716 case DRM_MODE_DPMS_STANDBY:
2707 case DRM_MODE_DPMS_SUSPEND: 2717 case DRM_MODE_DPMS_SUSPEND:
2708 case DRM_MODE_DPMS_OFF: 2718 case DRM_MODE_DPMS_OFF:
2709 drm_vblank_pre_modeset(dev, amdgpu_crtc->crtc_id); 2719 drm_vblank_off(dev, amdgpu_crtc->crtc_id);
2710 if (amdgpu_crtc->enabled) { 2720 if (amdgpu_crtc->enabled) {
2711 dce_v10_0_vga_enable(crtc, true); 2721 dce_v10_0_vga_enable(crtc, true);
2712 amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE); 2722 amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE);
@@ -2980,8 +2990,6 @@ static int dce_v10_0_sw_init(void *handle)
2980 if (r) 2990 if (r)
2981 return r; 2991 return r;
2982 2992
2983 adev->mode_info.mode_config_initialized = true;
2984
2985 adev->ddev->mode_config.funcs = &amdgpu_mode_funcs; 2993 adev->ddev->mode_config.funcs = &amdgpu_mode_funcs;
2986 2994
2987 adev->ddev->mode_config.max_width = 16384; 2995 adev->ddev->mode_config.max_width = 16384;
@@ -3012,7 +3020,9 @@ static int dce_v10_0_sw_init(void *handle)
3012 return -EINVAL; 3020 return -EINVAL;
3013 3021
3014 /* setup afmt */ 3022 /* setup afmt */
3015 dce_v10_0_afmt_init(adev); 3023 r = dce_v10_0_afmt_init(adev);
3024 if (r)
3025 return r;
3016 3026
3017 r = dce_v10_0_audio_init(adev); 3027 r = dce_v10_0_audio_init(adev);
3018 if (r) 3028 if (r)
@@ -3020,7 +3030,8 @@ static int dce_v10_0_sw_init(void *handle)
3020 3030
3021 drm_kms_helper_poll_init(adev->ddev); 3031 drm_kms_helper_poll_init(adev->ddev);
3022 3032
3023 return r; 3033 adev->mode_info.mode_config_initialized = true;
3034 return 0;
3024} 3035}
3025 3036
3026static int dce_v10_0_sw_fini(void *handle) 3037static int dce_v10_0_sw_fini(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index 6b6c9b6879ae..e9ccc6b787f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -1658,6 +1658,9 @@ static void dce_v11_0_audio_fini(struct amdgpu_device *adev)
1658{ 1658{
1659 int i; 1659 int i;
1660 1660
1661 if (!amdgpu_audio)
1662 return;
1663
1661 if (!adev->mode_info.audio.enabled) 1664 if (!adev->mode_info.audio.enabled)
1662 return; 1665 return;
1663 1666
@@ -1963,7 +1966,7 @@ static void dce_v11_0_afmt_enable(struct drm_encoder *encoder, bool enable)
1963 enable ? "En" : "Dis", dig->afmt->offset, amdgpu_encoder->encoder_id); 1966 enable ? "En" : "Dis", dig->afmt->offset, amdgpu_encoder->encoder_id);
1964} 1967}
1965 1968
1966static void dce_v11_0_afmt_init(struct amdgpu_device *adev) 1969static int dce_v11_0_afmt_init(struct amdgpu_device *adev)
1967{ 1970{
1968 int i; 1971 int i;
1969 1972
@@ -1976,8 +1979,16 @@ static void dce_v11_0_afmt_init(struct amdgpu_device *adev)
1976 if (adev->mode_info.afmt[i]) { 1979 if (adev->mode_info.afmt[i]) {
1977 adev->mode_info.afmt[i]->offset = dig_offsets[i]; 1980 adev->mode_info.afmt[i]->offset = dig_offsets[i];
1978 adev->mode_info.afmt[i]->id = i; 1981 adev->mode_info.afmt[i]->id = i;
1982 } else {
1983 int j;
1984 for (j = 0; j < i; j++) {
1985 kfree(adev->mode_info.afmt[j]);
1986 adev->mode_info.afmt[j] = NULL;
1987 }
1988 return -ENOMEM;
1979 } 1989 }
1980 } 1990 }
1991 return 0;
1981} 1992}
1982 1993
1983static void dce_v11_0_afmt_fini(struct amdgpu_device *adev) 1994static void dce_v11_0_afmt_fini(struct amdgpu_device *adev)
@@ -2054,8 +2065,7 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
2054 if (atomic) { 2065 if (atomic) {
2055 amdgpu_fb = to_amdgpu_framebuffer(fb); 2066 amdgpu_fb = to_amdgpu_framebuffer(fb);
2056 target_fb = fb; 2067 target_fb = fb;
2057 } 2068 } else {
2058 else {
2059 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); 2069 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
2060 target_fb = crtc->primary->fb; 2070 target_fb = crtc->primary->fb;
2061 } 2071 }
@@ -2069,9 +2079,9 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
2069 if (unlikely(r != 0)) 2079 if (unlikely(r != 0))
2070 return r; 2080 return r;
2071 2081
2072 if (atomic) 2082 if (atomic) {
2073 fb_location = amdgpu_bo_gpu_offset(rbo); 2083 fb_location = amdgpu_bo_gpu_offset(rbo);
2074 else { 2084 } else {
2075 r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location); 2085 r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
2076 if (unlikely(r != 0)) { 2086 if (unlikely(r != 0)) {
2077 amdgpu_bo_unreserve(rbo); 2087 amdgpu_bo_unreserve(rbo);
@@ -2691,13 +2701,13 @@ static void dce_v11_0_crtc_dpms(struct drm_crtc *crtc, int mode)
2691 type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id); 2701 type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id);
2692 amdgpu_irq_update(adev, &adev->crtc_irq, type); 2702 amdgpu_irq_update(adev, &adev->crtc_irq, type);
2693 amdgpu_irq_update(adev, &adev->pageflip_irq, type); 2703 amdgpu_irq_update(adev, &adev->pageflip_irq, type);
2694 drm_vblank_post_modeset(dev, amdgpu_crtc->crtc_id); 2704 drm_vblank_on(dev, amdgpu_crtc->crtc_id);
2695 dce_v11_0_crtc_load_lut(crtc); 2705 dce_v11_0_crtc_load_lut(crtc);
2696 break; 2706 break;
2697 case DRM_MODE_DPMS_STANDBY: 2707 case DRM_MODE_DPMS_STANDBY:
2698 case DRM_MODE_DPMS_SUSPEND: 2708 case DRM_MODE_DPMS_SUSPEND:
2699 case DRM_MODE_DPMS_OFF: 2709 case DRM_MODE_DPMS_OFF:
2700 drm_vblank_pre_modeset(dev, amdgpu_crtc->crtc_id); 2710 drm_vblank_off(dev, amdgpu_crtc->crtc_id);
2701 if (amdgpu_crtc->enabled) { 2711 if (amdgpu_crtc->enabled) {
2702 dce_v11_0_vga_enable(crtc, true); 2712 dce_v11_0_vga_enable(crtc, true);
2703 amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE); 2713 amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE);
@@ -2961,7 +2971,7 @@ static int dce_v11_0_sw_init(void *handle)
2961 for (i = 0; i < adev->mode_info.num_crtc; i++) { 2971 for (i = 0; i < adev->mode_info.num_crtc; i++) {
2962 r = amdgpu_irq_add_id(adev, i + 1, &adev->crtc_irq); 2972 r = amdgpu_irq_add_id(adev, i + 1, &adev->crtc_irq);
2963 if (r) 2973 if (r)
2964 return r; 2974 return r;
2965 } 2975 }
2966 2976
2967 for (i = 8; i < 20; i += 2) { 2977 for (i = 8; i < 20; i += 2) {
@@ -2973,9 +2983,7 @@ static int dce_v11_0_sw_init(void *handle)
2973 /* HPD hotplug */ 2983 /* HPD hotplug */
2974 r = amdgpu_irq_add_id(adev, 42, &adev->hpd_irq); 2984 r = amdgpu_irq_add_id(adev, 42, &adev->hpd_irq);
2975 if (r) 2985 if (r)
2976 return r; 2986 return r;
2977
2978 adev->mode_info.mode_config_initialized = true;
2979 2987
2980 adev->ddev->mode_config.funcs = &amdgpu_mode_funcs; 2988 adev->ddev->mode_config.funcs = &amdgpu_mode_funcs;
2981 2989
@@ -2994,6 +3002,7 @@ static int dce_v11_0_sw_init(void *handle)
2994 adev->ddev->mode_config.max_width = 16384; 3002 adev->ddev->mode_config.max_width = 16384;
2995 adev->ddev->mode_config.max_height = 16384; 3003 adev->ddev->mode_config.max_height = 16384;
2996 3004
3005
2997 /* allocate crtcs */ 3006 /* allocate crtcs */
2998 for (i = 0; i < adev->mode_info.num_crtc; i++) { 3007 for (i = 0; i < adev->mode_info.num_crtc; i++) {
2999 r = dce_v11_0_crtc_init(adev, i); 3008 r = dce_v11_0_crtc_init(adev, i);
@@ -3007,7 +3016,9 @@ static int dce_v11_0_sw_init(void *handle)
3007 return -EINVAL; 3016 return -EINVAL;
3008 3017
3009 /* setup afmt */ 3018 /* setup afmt */
3010 dce_v11_0_afmt_init(adev); 3019 r = dce_v11_0_afmt_init(adev);
3020 if (r)
3021 return r;
3011 3022
3012 r = dce_v11_0_audio_init(adev); 3023 r = dce_v11_0_audio_init(adev);
3013 if (r) 3024 if (r)
@@ -3015,7 +3026,8 @@ static int dce_v11_0_sw_init(void *handle)
3015 3026
3016 drm_kms_helper_poll_init(adev->ddev); 3027 drm_kms_helper_poll_init(adev->ddev);
3017 3028
3018 return r; 3029 adev->mode_info.mode_config_initialized = true;
3030 return 0;
3019} 3031}
3020 3032
3021static int dce_v11_0_sw_fini(void *handle) 3033static int dce_v11_0_sw_fini(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 56bea36a6b18..e56b55d8c280 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -1639,6 +1639,9 @@ static void dce_v8_0_audio_fini(struct amdgpu_device *adev)
1639{ 1639{
1640 int i; 1640 int i;
1641 1641
1642 if (!amdgpu_audio)
1643 return;
1644
1642 if (!adev->mode_info.audio.enabled) 1645 if (!adev->mode_info.audio.enabled)
1643 return; 1646 return;
1644 1647
@@ -1910,7 +1913,7 @@ static void dce_v8_0_afmt_enable(struct drm_encoder *encoder, bool enable)
1910 enable ? "En" : "Dis", dig->afmt->offset, amdgpu_encoder->encoder_id); 1913 enable ? "En" : "Dis", dig->afmt->offset, amdgpu_encoder->encoder_id);
1911} 1914}
1912 1915
1913static void dce_v8_0_afmt_init(struct amdgpu_device *adev) 1916static int dce_v8_0_afmt_init(struct amdgpu_device *adev)
1914{ 1917{
1915 int i; 1918 int i;
1916 1919
@@ -1923,8 +1926,16 @@ static void dce_v8_0_afmt_init(struct amdgpu_device *adev)
1923 if (adev->mode_info.afmt[i]) { 1926 if (adev->mode_info.afmt[i]) {
1924 adev->mode_info.afmt[i]->offset = dig_offsets[i]; 1927 adev->mode_info.afmt[i]->offset = dig_offsets[i];
1925 adev->mode_info.afmt[i]->id = i; 1928 adev->mode_info.afmt[i]->id = i;
1929 } else {
1930 int j;
1931 for (j = 0; j < i; j++) {
1932 kfree(adev->mode_info.afmt[j]);
1933 adev->mode_info.afmt[j] = NULL;
1934 }
1935 return -ENOMEM;
1926 } 1936 }
1927 } 1937 }
1938 return 0;
1928} 1939}
1929 1940
1930static void dce_v8_0_afmt_fini(struct amdgpu_device *adev) 1941static void dce_v8_0_afmt_fini(struct amdgpu_device *adev)
@@ -2001,8 +2012,7 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
2001 if (atomic) { 2012 if (atomic) {
2002 amdgpu_fb = to_amdgpu_framebuffer(fb); 2013 amdgpu_fb = to_amdgpu_framebuffer(fb);
2003 target_fb = fb; 2014 target_fb = fb;
2004 } 2015 } else {
2005 else {
2006 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); 2016 amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
2007 target_fb = crtc->primary->fb; 2017 target_fb = crtc->primary->fb;
2008 } 2018 }
@@ -2016,9 +2026,9 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
2016 if (unlikely(r != 0)) 2026 if (unlikely(r != 0))
2017 return r; 2027 return r;
2018 2028
2019 if (atomic) 2029 if (atomic) {
2020 fb_location = amdgpu_bo_gpu_offset(rbo); 2030 fb_location = amdgpu_bo_gpu_offset(rbo);
2021 else { 2031 } else {
2022 r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location); 2032 r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
2023 if (unlikely(r != 0)) { 2033 if (unlikely(r != 0)) {
2024 amdgpu_bo_unreserve(rbo); 2034 amdgpu_bo_unreserve(rbo);
@@ -2612,13 +2622,13 @@ static void dce_v8_0_crtc_dpms(struct drm_crtc *crtc, int mode)
2612 type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id); 2622 type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id);
2613 amdgpu_irq_update(adev, &adev->crtc_irq, type); 2623 amdgpu_irq_update(adev, &adev->crtc_irq, type);
2614 amdgpu_irq_update(adev, &adev->pageflip_irq, type); 2624 amdgpu_irq_update(adev, &adev->pageflip_irq, type);
2615 drm_vblank_post_modeset(dev, amdgpu_crtc->crtc_id); 2625 drm_vblank_on(dev, amdgpu_crtc->crtc_id);
2616 dce_v8_0_crtc_load_lut(crtc); 2626 dce_v8_0_crtc_load_lut(crtc);
2617 break; 2627 break;
2618 case DRM_MODE_DPMS_STANDBY: 2628 case DRM_MODE_DPMS_STANDBY:
2619 case DRM_MODE_DPMS_SUSPEND: 2629 case DRM_MODE_DPMS_SUSPEND:
2620 case DRM_MODE_DPMS_OFF: 2630 case DRM_MODE_DPMS_OFF:
2621 drm_vblank_pre_modeset(dev, amdgpu_crtc->crtc_id); 2631 drm_vblank_off(dev, amdgpu_crtc->crtc_id);
2622 if (amdgpu_crtc->enabled) { 2632 if (amdgpu_crtc->enabled) {
2623 dce_v8_0_vga_enable(crtc, true); 2633 dce_v8_0_vga_enable(crtc, true);
2624 amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE); 2634 amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE);
@@ -2890,8 +2900,6 @@ static int dce_v8_0_sw_init(void *handle)
2890 if (r) 2900 if (r)
2891 return r; 2901 return r;
2892 2902
2893 adev->mode_info.mode_config_initialized = true;
2894
2895 adev->ddev->mode_config.funcs = &amdgpu_mode_funcs; 2903 adev->ddev->mode_config.funcs = &amdgpu_mode_funcs;
2896 2904
2897 adev->ddev->mode_config.max_width = 16384; 2905 adev->ddev->mode_config.max_width = 16384;
@@ -2922,7 +2930,9 @@ static int dce_v8_0_sw_init(void *handle)
2922 return -EINVAL; 2930 return -EINVAL;
2923 2931
2924 /* setup afmt */ 2932 /* setup afmt */
2925 dce_v8_0_afmt_init(adev); 2933 r = dce_v8_0_afmt_init(adev);
2934 if (r)
2935 return r;
2926 2936
2927 r = dce_v8_0_audio_init(adev); 2937 r = dce_v8_0_audio_init(adev);
2928 if (r) 2938 if (r)
@@ -2930,7 +2940,8 @@ static int dce_v8_0_sw_init(void *handle)
2930 2940
2931 drm_kms_helper_poll_init(adev->ddev); 2941 drm_kms_helper_poll_init(adev->ddev);
2932 2942
2933 return r; 2943 adev->mode_info.mode_config_initialized = true;
2944 return 0;
2934} 2945}
2935 2946
2936static int dce_v8_0_sw_fini(void *handle) 2947static int dce_v8_0_sw_fini(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 4411b94775db..a06045f040f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -1925,6 +1925,25 @@ static void gfx_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
1925} 1925}
1926 1926
1927/** 1927/**
1928 * gfx_v7_0_ring_emit_hdp_invalidate - emit an hdp invalidate on the cp
1929 *
1930 * @adev: amdgpu_device pointer
1931 * @ridx: amdgpu ring index
1932 *
1933 * Emits an hdp invalidate on the cp.
1934 */
1935static void gfx_v7_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
1936{
1937 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1938 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
1939 WRITE_DATA_DST_SEL(0) |
1940 WR_CONFIRM));
1941 amdgpu_ring_write(ring, mmHDP_DEBUG0);
1942 amdgpu_ring_write(ring, 0);
1943 amdgpu_ring_write(ring, 1);
1944}
1945
1946/**
1928 * gfx_v7_0_ring_emit_fence_gfx - emit a fence on the gfx ring 1947 * gfx_v7_0_ring_emit_fence_gfx - emit a fence on the gfx ring
1929 * 1948 *
1930 * @adev: amdgpu_device pointer 1949 * @adev: amdgpu_device pointer
@@ -2117,8 +2136,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring)
2117 ib.ptr[2] = 0xDEADBEEF; 2136 ib.ptr[2] = 0xDEADBEEF;
2118 ib.length_dw = 3; 2137 ib.length_dw = 3;
2119 2138
2120 r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, 2139 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
2121 NULL, &f);
2122 if (r) 2140 if (r)
2123 goto err2; 2141 goto err2;
2124 2142
@@ -3023,6 +3041,26 @@ static int gfx_v7_0_cp_resume(struct amdgpu_device *adev)
3023 return 0; 3041 return 0;
3024} 3042}
3025 3043
3044/**
3045 * gfx_v7_0_ring_emit_vm_flush - cik vm flush using the CP
3046 *
3047 * @ring: the ring to emmit the commands to
3048 *
3049 * Sync the command pipeline with the PFP. E.g. wait for everything
3050 * to be completed.
3051 */
3052static void gfx_v7_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
3053{
3054 int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
3055 if (usepfp) {
3056 /* synce CE with ME to prevent CE fetch CEIB before context switch done */
3057 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3058 amdgpu_ring_write(ring, 0);
3059 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3060 amdgpu_ring_write(ring, 0);
3061 }
3062}
3063
3026/* 3064/*
3027 * vm 3065 * vm
3028 * VMID 0 is the physical GPU addresses as used by the kernel. 3066 * VMID 0 is the physical GPU addresses as used by the kernel.
@@ -3054,14 +3092,6 @@ static void gfx_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
3054 amdgpu_ring_write(ring, 0xffffffff); 3092 amdgpu_ring_write(ring, 0xffffffff);
3055 amdgpu_ring_write(ring, 4); /* poll interval */ 3093 amdgpu_ring_write(ring, 4); /* poll interval */
3056 3094
3057 if (usepfp) {
3058 /* synce CE with ME to prevent CE fetch CEIB before context switch done */
3059 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3060 amdgpu_ring_write(ring, 0);
3061 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3062 amdgpu_ring_write(ring, 0);
3063 }
3064
3065 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3095 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3066 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 3096 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
3067 WRITE_DATA_DST_SEL(0))); 3097 WRITE_DATA_DST_SEL(0)));
@@ -5142,9 +5172,11 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = {
5142 .parse_cs = NULL, 5172 .parse_cs = NULL,
5143 .emit_ib = gfx_v7_0_ring_emit_ib_gfx, 5173 .emit_ib = gfx_v7_0_ring_emit_ib_gfx,
5144 .emit_fence = gfx_v7_0_ring_emit_fence_gfx, 5174 .emit_fence = gfx_v7_0_ring_emit_fence_gfx,
5175 .emit_pipeline_sync = gfx_v7_0_ring_emit_pipeline_sync,
5145 .emit_vm_flush = gfx_v7_0_ring_emit_vm_flush, 5176 .emit_vm_flush = gfx_v7_0_ring_emit_vm_flush,
5146 .emit_gds_switch = gfx_v7_0_ring_emit_gds_switch, 5177 .emit_gds_switch = gfx_v7_0_ring_emit_gds_switch,
5147 .emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush, 5178 .emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush,
5179 .emit_hdp_invalidate = gfx_v7_0_ring_emit_hdp_invalidate,
5148 .test_ring = gfx_v7_0_ring_test_ring, 5180 .test_ring = gfx_v7_0_ring_test_ring,
5149 .test_ib = gfx_v7_0_ring_test_ib, 5181 .test_ib = gfx_v7_0_ring_test_ib,
5150 .insert_nop = amdgpu_ring_insert_nop, 5182 .insert_nop = amdgpu_ring_insert_nop,
@@ -5158,9 +5190,11 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = {
5158 .parse_cs = NULL, 5190 .parse_cs = NULL,
5159 .emit_ib = gfx_v7_0_ring_emit_ib_compute, 5191 .emit_ib = gfx_v7_0_ring_emit_ib_compute,
5160 .emit_fence = gfx_v7_0_ring_emit_fence_compute, 5192 .emit_fence = gfx_v7_0_ring_emit_fence_compute,
5193 .emit_pipeline_sync = gfx_v7_0_ring_emit_pipeline_sync,
5161 .emit_vm_flush = gfx_v7_0_ring_emit_vm_flush, 5194 .emit_vm_flush = gfx_v7_0_ring_emit_vm_flush,
5162 .emit_gds_switch = gfx_v7_0_ring_emit_gds_switch, 5195 .emit_gds_switch = gfx_v7_0_ring_emit_gds_switch,
5163 .emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush, 5196 .emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush,
5197 .emit_hdp_invalidate = gfx_v7_0_ring_emit_hdp_invalidate,
5164 .test_ring = gfx_v7_0_ring_test_ring, 5198 .test_ring = gfx_v7_0_ring_test_ring,
5165 .test_ib = gfx_v7_0_ring_test_ib, 5199 .test_ib = gfx_v7_0_ring_test_ib,
5166 .insert_nop = amdgpu_ring_insert_nop, 5200 .insert_nop = amdgpu_ring_insert_nop,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 1b85c001f860..509d0baaeaae 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -706,8 +706,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring)
706 ib.ptr[2] = 0xDEADBEEF; 706 ib.ptr[2] = 0xDEADBEEF;
707 ib.length_dw = 3; 707 ib.length_dw = 3;
708 708
709 r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, 709 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
710 NULL, &f);
711 if (r) 710 if (r)
712 goto err2; 711 goto err2;
713 712
@@ -1262,8 +1261,7 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
1262 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1261 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1263 1262
1264 /* shedule the ib on the ring */ 1263 /* shedule the ib on the ring */
1265 r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, 1264 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1266 NULL, &f);
1267 if (r) { 1265 if (r) {
1268 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 1266 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
1269 goto fail; 1267 goto fail;
@@ -4589,6 +4587,18 @@ static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
4589 amdgpu_ring_write(ring, 0x20); /* poll interval */ 4587 amdgpu_ring_write(ring, 0x20); /* poll interval */
4590} 4588}
4591 4589
4590static void gfx_v8_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
4591{
4592 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4593 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4594 WRITE_DATA_DST_SEL(0) |
4595 WR_CONFIRM));
4596 amdgpu_ring_write(ring, mmHDP_DEBUG0);
4597 amdgpu_ring_write(ring, 0);
4598 amdgpu_ring_write(ring, 1);
4599
4600}
4601
4592static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 4602static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
4593 struct amdgpu_ib *ib) 4603 struct amdgpu_ib *ib)
4594{ 4604{
@@ -4682,8 +4692,7 @@ static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
4682 4692
4683} 4693}
4684 4694
4685static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 4695static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
4686 unsigned vm_id, uint64_t pd_addr)
4687{ 4696{
4688 int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX); 4697 int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
4689 uint32_t seq = ring->fence_drv.sync_seq; 4698 uint32_t seq = ring->fence_drv.sync_seq;
@@ -4706,6 +4715,12 @@ static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
4706 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 4715 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4707 amdgpu_ring_write(ring, 0); 4716 amdgpu_ring_write(ring, 0);
4708 } 4717 }
4718}
4719
4720static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
4721 unsigned vm_id, uint64_t pd_addr)
4722{
4723 int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
4709 4724
4710 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4725 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4711 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 4726 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
@@ -5028,9 +5043,11 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
5028 .parse_cs = NULL, 5043 .parse_cs = NULL,
5029 .emit_ib = gfx_v8_0_ring_emit_ib_gfx, 5044 .emit_ib = gfx_v8_0_ring_emit_ib_gfx,
5030 .emit_fence = gfx_v8_0_ring_emit_fence_gfx, 5045 .emit_fence = gfx_v8_0_ring_emit_fence_gfx,
5046 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
5031 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, 5047 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
5032 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, 5048 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
5033 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 5049 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
5050 .emit_hdp_invalidate = gfx_v8_0_ring_emit_hdp_invalidate,
5034 .test_ring = gfx_v8_0_ring_test_ring, 5051 .test_ring = gfx_v8_0_ring_test_ring,
5035 .test_ib = gfx_v8_0_ring_test_ib, 5052 .test_ib = gfx_v8_0_ring_test_ib,
5036 .insert_nop = amdgpu_ring_insert_nop, 5053 .insert_nop = amdgpu_ring_insert_nop,
@@ -5044,9 +5061,11 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
5044 .parse_cs = NULL, 5061 .parse_cs = NULL,
5045 .emit_ib = gfx_v8_0_ring_emit_ib_compute, 5062 .emit_ib = gfx_v8_0_ring_emit_ib_compute,
5046 .emit_fence = gfx_v8_0_ring_emit_fence_compute, 5063 .emit_fence = gfx_v8_0_ring_emit_fence_compute,
5064 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
5047 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, 5065 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
5048 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, 5066 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
5049 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 5067 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
5068 .emit_hdp_invalidate = gfx_v8_0_ring_emit_hdp_invalidate,
5050 .test_ring = gfx_v8_0_ring_test_ring, 5069 .test_ring = gfx_v8_0_ring_test_ring,
5051 .test_ib = gfx_v8_0_ring_test_ib, 5070 .test_ib = gfx_v8_0_ring_test_ib,
5052 .insert_nop = amdgpu_ring_insert_nop, 5071 .insert_nop = amdgpu_ring_insert_nop,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 711840a23bd3..82ce7d943884 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -339,7 +339,7 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
339 WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK); 339 WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
340 340
341 tmp = RREG32(mmHDP_MISC_CNTL); 341 tmp = RREG32(mmHDP_MISC_CNTL);
342 tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 342 tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0);
343 WREG32(mmHDP_MISC_CNTL, tmp); 343 WREG32(mmHDP_MISC_CNTL, tmp);
344 344
345 tmp = RREG32(mmHDP_HOST_PATH_CNTL); 345 tmp = RREG32(mmHDP_HOST_PATH_CNTL);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 757803ae7c4a..29bd7b57dc91 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -386,7 +386,7 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
386 WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK); 386 WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
387 387
388 tmp = RREG32(mmHDP_MISC_CNTL); 388 tmp = RREG32(mmHDP_MISC_CNTL);
389 tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 389 tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0);
390 WREG32(mmHDP_MISC_CNTL, tmp); 390 WREG32(mmHDP_MISC_CNTL, tmp);
391 391
392 tmp = RREG32(mmHDP_HOST_PATH_CNTL); 392 tmp = RREG32(mmHDP_HOST_PATH_CNTL);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index dddb8d6a81f3..ab9ff89a3096 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -300,6 +300,13 @@ static void sdma_v2_4_ring_emit_hdp_flush(struct amdgpu_ring *ring)
300 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */ 300 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
301} 301}
302 302
303static void sdma_v2_4_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
304{
305 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
306 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
307 amdgpu_ring_write(ring, mmHDP_DEBUG0);
308 amdgpu_ring_write(ring, 1);
309}
303/** 310/**
304 * sdma_v2_4_ring_emit_fence - emit a fence on the DMA ring 311 * sdma_v2_4_ring_emit_fence - emit a fence on the DMA ring
305 * 312 *
@@ -694,8 +701,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring)
694 ib.ptr[7] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP); 701 ib.ptr[7] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
695 ib.length_dw = 8; 702 ib.length_dw = 8;
696 703
697 r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, 704 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
698 NULL, &f);
699 if (r) 705 if (r)
700 goto err1; 706 goto err1;
701 707
@@ -874,6 +880,31 @@ static void sdma_v2_4_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib
874} 880}
875 881
876/** 882/**
883 * sdma_v2_4_ring_emit_pipeline_sync - sync the pipeline
884 *
885 * @ring: amdgpu_ring pointer
886 *
887 * Make sure all previous operations are completed (CIK).
888 */
889static void sdma_v2_4_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
890{
891 uint32_t seq = ring->fence_drv.sync_seq;
892 uint64_t addr = ring->fence_drv.gpu_addr;
893
894 /* wait for idle */
895 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
896 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
897 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
898 SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
899 amdgpu_ring_write(ring, addr & 0xfffffffc);
900 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
901 amdgpu_ring_write(ring, seq); /* reference */
902 amdgpu_ring_write(ring, 0xfffffff); /* mask */
903 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
904 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
905}
906
907/**
877 * sdma_v2_4_ring_emit_vm_flush - cik vm flush using sDMA 908 * sdma_v2_4_ring_emit_vm_flush - cik vm flush using sDMA
878 * 909 *
879 * @ring: amdgpu_ring pointer 910 * @ring: amdgpu_ring pointer
@@ -1274,8 +1305,10 @@ static const struct amdgpu_ring_funcs sdma_v2_4_ring_funcs = {
1274 .parse_cs = NULL, 1305 .parse_cs = NULL,
1275 .emit_ib = sdma_v2_4_ring_emit_ib, 1306 .emit_ib = sdma_v2_4_ring_emit_ib,
1276 .emit_fence = sdma_v2_4_ring_emit_fence, 1307 .emit_fence = sdma_v2_4_ring_emit_fence,
1308 .emit_pipeline_sync = sdma_v2_4_ring_emit_pipeline_sync,
1277 .emit_vm_flush = sdma_v2_4_ring_emit_vm_flush, 1309 .emit_vm_flush = sdma_v2_4_ring_emit_vm_flush,
1278 .emit_hdp_flush = sdma_v2_4_ring_emit_hdp_flush, 1310 .emit_hdp_flush = sdma_v2_4_ring_emit_hdp_flush,
1311 .emit_hdp_invalidate = sdma_v2_4_ring_emit_hdp_invalidate,
1279 .test_ring = sdma_v2_4_ring_test_ring, 1312 .test_ring = sdma_v2_4_ring_test_ring,
1280 .test_ib = sdma_v2_4_ring_test_ib, 1313 .test_ib = sdma_v2_4_ring_test_ib,
1281 .insert_nop = sdma_v2_4_ring_insert_nop, 1314 .insert_nop = sdma_v2_4_ring_insert_nop,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 19e02f7a06f3..4c24c371fec7 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -410,6 +410,14 @@ static void sdma_v3_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
410 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */ 410 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
411} 411}
412 412
413static void sdma_v3_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
414{
415 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
416 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
417 amdgpu_ring_write(ring, mmHDP_DEBUG0);
418 amdgpu_ring_write(ring, 1);
419}
420
413/** 421/**
414 * sdma_v3_0_ring_emit_fence - emit a fence on the DMA ring 422 * sdma_v3_0_ring_emit_fence - emit a fence on the DMA ring
415 * 423 *
@@ -845,8 +853,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring)
845 ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); 853 ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
846 ib.length_dw = 8; 854 ib.length_dw = 8;
847 855
848 r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, 856 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
849 NULL, &f);
850 if (r) 857 if (r)
851 goto err1; 858 goto err1;
852 859
@@ -1024,6 +1031,31 @@ static void sdma_v3_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib
1024} 1031}
1025 1032
1026/** 1033/**
1034 * sdma_v3_0_ring_emit_pipeline_sync - sync the pipeline
1035 *
1036 * @ring: amdgpu_ring pointer
1037 *
1038 * Make sure all previous operations are completed (CIK).
1039 */
1040static void sdma_v3_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1041{
1042 uint32_t seq = ring->fence_drv.sync_seq;
1043 uint64_t addr = ring->fence_drv.gpu_addr;
1044
1045 /* wait for idle */
1046 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1047 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1048 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
1049 SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
1050 amdgpu_ring_write(ring, addr & 0xfffffffc);
1051 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
1052 amdgpu_ring_write(ring, seq); /* reference */
1053 amdgpu_ring_write(ring, 0xfffffff); /* mask */
1054 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1055 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
1056}
1057
1058/**
1027 * sdma_v3_0_ring_emit_vm_flush - cik vm flush using sDMA 1059 * sdma_v3_0_ring_emit_vm_flush - cik vm flush using sDMA
1028 * 1060 *
1029 * @ring: amdgpu_ring pointer 1061 * @ring: amdgpu_ring pointer
@@ -1541,8 +1573,10 @@ static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = {
1541 .parse_cs = NULL, 1573 .parse_cs = NULL,
1542 .emit_ib = sdma_v3_0_ring_emit_ib, 1574 .emit_ib = sdma_v3_0_ring_emit_ib,
1543 .emit_fence = sdma_v3_0_ring_emit_fence, 1575 .emit_fence = sdma_v3_0_ring_emit_fence,
1576 .emit_pipeline_sync = sdma_v3_0_ring_emit_pipeline_sync,
1544 .emit_vm_flush = sdma_v3_0_ring_emit_vm_flush, 1577 .emit_vm_flush = sdma_v3_0_ring_emit_vm_flush,
1545 .emit_hdp_flush = sdma_v3_0_ring_emit_hdp_flush, 1578 .emit_hdp_flush = sdma_v3_0_ring_emit_hdp_flush,
1579 .emit_hdp_invalidate = sdma_v3_0_ring_emit_hdp_invalidate,
1546 .test_ring = sdma_v3_0_ring_test_ring, 1580 .test_ring = sdma_v3_0_ring_test_ring,
1547 .test_ib = sdma_v3_0_ring_test_ib, 1581 .test_ib = sdma_v3_0_ring_test_ib,
1548 .insert_nop = sdma_v3_0_ring_insert_nop, 1582 .insert_nop = sdma_v3_0_ring_insert_nop,
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index b72cf063df1a..1c120efa292c 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -1071,26 +1071,22 @@ static int vi_common_early_init(void *handle)
1071 adev->external_rev_id = 0xFF; 1071 adev->external_rev_id = 0xFF;
1072 switch (adev->asic_type) { 1072 switch (adev->asic_type) {
1073 case CHIP_TOPAZ: 1073 case CHIP_TOPAZ:
1074 adev->has_uvd = false;
1075 adev->cg_flags = 0; 1074 adev->cg_flags = 0;
1076 adev->pg_flags = 0; 1075 adev->pg_flags = 0;
1077 adev->external_rev_id = 0x1; 1076 adev->external_rev_id = 0x1;
1078 break; 1077 break;
1079 case CHIP_FIJI: 1078 case CHIP_FIJI:
1080 adev->has_uvd = true;
1081 adev->cg_flags = 0; 1079 adev->cg_flags = 0;
1082 adev->pg_flags = 0; 1080 adev->pg_flags = 0;
1083 adev->external_rev_id = adev->rev_id + 0x3c; 1081 adev->external_rev_id = adev->rev_id + 0x3c;
1084 break; 1082 break;
1085 case CHIP_TONGA: 1083 case CHIP_TONGA:
1086 adev->has_uvd = true;
1087 adev->cg_flags = 0; 1084 adev->cg_flags = 0;
1088 adev->pg_flags = 0; 1085 adev->pg_flags = 0;
1089 adev->external_rev_id = adev->rev_id + 0x14; 1086 adev->external_rev_id = adev->rev_id + 0x14;
1090 break; 1087 break;
1091 case CHIP_CARRIZO: 1088 case CHIP_CARRIZO:
1092 case CHIP_STONEY: 1089 case CHIP_STONEY:
1093 adev->has_uvd = true;
1094 adev->cg_flags = 0; 1090 adev->cg_flags = 0;
1095 adev->pg_flags = 0; 1091 adev->pg_flags = 0;
1096 adev->external_rev_id = adev->rev_id + 0x1; 1092 adev->external_rev_id = adev->rev_id + 0x1;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
index b8d6a82c1be2..727d5c9ea04c 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
@@ -744,8 +744,9 @@ static int cz_tf_update_sclk_limit(struct pp_hwmgr *hwmgr,
744 cz_hwmgr->sclk_dpm.soft_max_clk = table->entries[table->count - 1].clk; 744 cz_hwmgr->sclk_dpm.soft_max_clk = table->entries[table->count - 1].clk;
745 745
746 clock = hwmgr->display_config.min_core_set_clock; 746 clock = hwmgr->display_config.min_core_set_clock;
747;
747 if (clock == 0) 748 if (clock == 0)
748 printk(KERN_ERR "[ powerplay ] min_core_set_clock not set\n"); 749 printk(KERN_INFO "[ powerplay ] min_core_set_clock not set\n");
749 750
750 if (cz_hwmgr->sclk_dpm.hard_min_clk != clock) { 751 if (cz_hwmgr->sclk_dpm.hard_min_clk != clock) {
751 cz_hwmgr->sclk_dpm.hard_min_clk = clock; 752 cz_hwmgr->sclk_dpm.hard_min_clk = clock;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
index 9deadabbc81c..72cfecc4f9f7 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
@@ -34,6 +34,11 @@ static int phm_run_table(struct pp_hwmgr *hwmgr,
34 int result = 0; 34 int result = 0;
35 phm_table_function *function; 35 phm_table_function *function;
36 36
37 if (rt_table->function_list == NULL) {
38 printk(KERN_INFO "[ powerplay ] this function not implement!\n");
39 return 0;
40 }
41
37 for (function = rt_table->function_list; NULL != *function; function++) { 42 for (function = rt_table->function_list; NULL != *function; function++) {
38 int tmp = (*function)(hwmgr, input, output, temp_storage, result); 43 int tmp = (*function)(hwmgr, input, output, temp_storage, result);
39 44
@@ -57,9 +62,9 @@ int phm_dispatch_table(struct pp_hwmgr *hwmgr,
57 int result = 0; 62 int result = 0;
58 void *temp_storage = NULL; 63 void *temp_storage = NULL;
59 64
60 if (hwmgr == NULL || rt_table == NULL || rt_table->function_list == NULL) { 65 if (hwmgr == NULL || rt_table == NULL) {
61 printk(KERN_ERR "[ powerplay ] Invalid Parameter!\n"); 66 printk(KERN_ERR "[ powerplay ] Invalid Parameter!\n");
62 return 0; /*temp return ture because some function not implement on some asic */ 67 return -EINVAL;
63 } 68 }
64 69
65 if (0 != rt_table->storage_size) { 70 if (0 != rt_table->storage_size) {
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index 801dd60ac192..e187beca38f7 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -275,13 +275,13 @@ void atombios_crtc_dpms(struct drm_crtc *crtc, int mode)
275 if (ASIC_IS_DCE3(rdev) && !ASIC_IS_DCE6(rdev)) 275 if (ASIC_IS_DCE3(rdev) && !ASIC_IS_DCE6(rdev))
276 atombios_enable_crtc_memreq(crtc, ATOM_ENABLE); 276 atombios_enable_crtc_memreq(crtc, ATOM_ENABLE);
277 atombios_blank_crtc(crtc, ATOM_DISABLE); 277 atombios_blank_crtc(crtc, ATOM_DISABLE);
278 drm_vblank_post_modeset(dev, radeon_crtc->crtc_id); 278 drm_vblank_on(dev, radeon_crtc->crtc_id);
279 radeon_crtc_load_lut(crtc); 279 radeon_crtc_load_lut(crtc);
280 break; 280 break;
281 case DRM_MODE_DPMS_STANDBY: 281 case DRM_MODE_DPMS_STANDBY:
282 case DRM_MODE_DPMS_SUSPEND: 282 case DRM_MODE_DPMS_SUSPEND:
283 case DRM_MODE_DPMS_OFF: 283 case DRM_MODE_DPMS_OFF:
284 drm_vblank_pre_modeset(dev, radeon_crtc->crtc_id); 284 drm_vblank_off(dev, radeon_crtc->crtc_id);
285 if (radeon_crtc->enabled) 285 if (radeon_crtc->enabled)
286 atombios_blank_crtc(crtc, ATOM_ENABLE); 286 atombios_blank_crtc(crtc, ATOM_ENABLE);
287 if (ASIC_IS_DCE3(rdev) && !ASIC_IS_DCE6(rdev)) 287 if (ASIC_IS_DCE3(rdev) && !ASIC_IS_DCE6(rdev))
diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c
index 1603751b1164..edd05cdb0cd8 100644
--- a/drivers/gpu/drm/radeon/atombios_encoders.c
+++ b/drivers/gpu/drm/radeon/atombios_encoders.c
@@ -892,8 +892,6 @@ atombios_dig_encoder_setup2(struct drm_encoder *encoder, int action, int panel_m
892 else 892 else
893 args.v1.ucLaneNum = 4; 893 args.v1.ucLaneNum = 4;
894 894
895 if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode) && (dp_clock == 270000))
896 args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ;
897 switch (radeon_encoder->encoder_id) { 895 switch (radeon_encoder->encoder_id) {
898 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: 896 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
899 args.v1.ucConfig = ATOM_ENCODER_CONFIG_V2_TRANSMITTER1; 897 args.v1.ucConfig = ATOM_ENCODER_CONFIG_V2_TRANSMITTER1;
@@ -910,6 +908,10 @@ atombios_dig_encoder_setup2(struct drm_encoder *encoder, int action, int panel_m
910 args.v1.ucConfig |= ATOM_ENCODER_CONFIG_LINKB; 908 args.v1.ucConfig |= ATOM_ENCODER_CONFIG_LINKB;
911 else 909 else
912 args.v1.ucConfig |= ATOM_ENCODER_CONFIG_LINKA; 910 args.v1.ucConfig |= ATOM_ENCODER_CONFIG_LINKA;
911
912 if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode) && (dp_clock == 270000))
913 args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ;
914
913 break; 915 break;
914 case 2: 916 case 2:
915 case 3: 917 case 3:
diff --git a/drivers/gpu/drm/radeon/radeon_atpx_handler.c b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
index 56482e35d43e..fd8c4d317e60 100644
--- a/drivers/gpu/drm/radeon/radeon_atpx_handler.c
+++ b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
@@ -62,6 +62,10 @@ bool radeon_has_atpx(void) {
62 return radeon_atpx_priv.atpx_detected; 62 return radeon_atpx_priv.atpx_detected;
63} 63}
64 64
65bool radeon_has_atpx_dgpu_power_cntl(void) {
66 return radeon_atpx_priv.atpx.functions.power_cntl;
67}
68
65/** 69/**
66 * radeon_atpx_call - call an ATPX method 70 * radeon_atpx_call - call an ATPX method
67 * 71 *
@@ -141,10 +145,6 @@ static void radeon_atpx_parse_functions(struct radeon_atpx_functions *f, u32 mas
141 */ 145 */
142static int radeon_atpx_validate(struct radeon_atpx *atpx) 146static int radeon_atpx_validate(struct radeon_atpx *atpx)
143{ 147{
144 /* make sure required functions are enabled */
145 /* dGPU power control is required */
146 atpx->functions.power_cntl = true;
147
148 if (atpx->functions.px_params) { 148 if (atpx->functions.px_params) {
149 union acpi_object *info; 149 union acpi_object *info;
150 struct atpx_px_params output; 150 struct atpx_px_params output;
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 4197ca1bb1e4..e2396336f9e8 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -103,6 +103,12 @@ static const char radeon_family_name[][16] = {
103 "LAST", 103 "LAST",
104}; 104};
105 105
106#if defined(CONFIG_VGA_SWITCHEROO)
107bool radeon_has_atpx_dgpu_power_cntl(void);
108#else
109static inline bool radeon_has_atpx_dgpu_power_cntl(void) { return false; }
110#endif
111
106#define RADEON_PX_QUIRK_DISABLE_PX (1 << 0) 112#define RADEON_PX_QUIRK_DISABLE_PX (1 << 0)
107#define RADEON_PX_QUIRK_LONG_WAKEUP (1 << 1) 113#define RADEON_PX_QUIRK_LONG_WAKEUP (1 << 1)
108 114
@@ -1433,7 +1439,7 @@ int radeon_device_init(struct radeon_device *rdev,
1433 * ignore it */ 1439 * ignore it */
1434 vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode); 1440 vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode);
1435 1441
1436 if (rdev->flags & RADEON_IS_PX) 1442 if ((rdev->flags & RADEON_IS_PX) && radeon_has_atpx_dgpu_power_cntl())
1437 runtime = true; 1443 runtime = true;
1438 vga_switcheroo_register_client(rdev->pdev, &radeon_switcheroo_ops, runtime); 1444 vga_switcheroo_register_client(rdev->pdev, &radeon_switcheroo_ops, runtime);
1439 if (runtime) 1445 if (runtime)
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index 32b338ff436b..24152dfef199 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -331,13 +331,13 @@ static void radeon_crtc_dpms(struct drm_crtc *crtc, int mode)
331 RADEON_CRTC_DISP_REQ_EN_B)); 331 RADEON_CRTC_DISP_REQ_EN_B));
332 WREG32_P(RADEON_CRTC_EXT_CNTL, crtc_ext_cntl, ~(mask | crtc_ext_cntl)); 332 WREG32_P(RADEON_CRTC_EXT_CNTL, crtc_ext_cntl, ~(mask | crtc_ext_cntl));
333 } 333 }
334 drm_vblank_post_modeset(dev, radeon_crtc->crtc_id); 334 drm_vblank_on(dev, radeon_crtc->crtc_id);
335 radeon_crtc_load_lut(crtc); 335 radeon_crtc_load_lut(crtc);
336 break; 336 break;
337 case DRM_MODE_DPMS_STANDBY: 337 case DRM_MODE_DPMS_STANDBY:
338 case DRM_MODE_DPMS_SUSPEND: 338 case DRM_MODE_DPMS_SUSPEND:
339 case DRM_MODE_DPMS_OFF: 339 case DRM_MODE_DPMS_OFF:
340 drm_vblank_pre_modeset(dev, radeon_crtc->crtc_id); 340 drm_vblank_off(dev, radeon_crtc->crtc_id);
341 if (radeon_crtc->crtc_id) 341 if (radeon_crtc->crtc_id)
342 WREG32_P(RADEON_CRTC2_GEN_CNTL, mask, ~(RADEON_CRTC2_EN | mask)); 342 WREG32_P(RADEON_CRTC2_GEN_CNTL, mask, ~(RADEON_CRTC2_EN | mask));
343 else { 343 else {
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-15 20:45:37 -0400