aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorOded Gabbay <oded.gabbay@amd.com>2014-07-15 06:53:32 -0400
committerOded Gabbay <oded.gabbay@amd.com>2014-07-15 06:53:32 -0400
commite28740ece34d314002b1ddfa14e8fb7c7b909489 (patch)
treeb428227c964e601e090662183c422440e0613090
parent1c0a46255f8d7daf5b601668836e185fd1294e94 (diff)
drm/radeon: Add radeon <--> amdkfd interface
This patch adds the interface between the radeon driver and the amdkfd driver. The interface implementation is contained in radeon_kfd.c and radeon_kfd.h. The interface itself is represented by a pointer to struct kfd_dev. The pointer is located inside radeon_device structure. All the register accesses that amdkfd need are done using this interface. This allows us to avoid direct register accesses in amdkfd proper, while also avoiding locking between amdkfd and radeon. The single exception is the doorbells that are used in both of the drivers. However, because they are located in separate pci bar pages, the danger of sharing registers between the drivers is minimal. Having said that, we are planning to move the doorbells as well to radeon. v3: Add interface for sa manager init and fini. The init function will allocate a buffer on system memory and pin it to the GART address space via the radeon sa manager. All mappings of buffers to GART address space are done via the radeon sa manager. The interface of allocate memory will use the radeon sa manager to sub allocate from the single buffer that was allocated during the init function. Change lower_32/upper_32 calls to use linux macros Add documentation for the interface v4: Change ptr field type in kgd_mem from uint32_t* to void* to match to type that is returned by radeon_sa_bo_cpu_addr v5: Change format of mqd structure to work with latest KV firmware Add support for AQL queues creation to enable working with open-source HSA runtime. Move generic kfd-->kgd interface and other generic kgd definitions to a generic header file that will be used by AMD's radeon and amdgpu drivers Signed-off-by: Oded Gabbay <oded.gabbay@amd.com>
Diffstat
-rw-r--r--drivers/gpu/drm/amd/include/kgd_kfd_interface.h185
-rw-r--r--drivers/gpu/drm/radeon/Makefile1
-rw-r--r--drivers/gpu/drm/radeon/cik.c10
-rw-r--r--drivers/gpu/drm/radeon/cik_reg.h136
-rw-r--r--drivers/gpu/drm/radeon/cikd.h51
-rw-r--r--drivers/gpu/drm/radeon/radeon.h4
-rw-r--r--drivers/gpu/drm/radeon/radeon_drv.c5
-rw-r--r--drivers/gpu/drm/radeon/radeon_kfd.c563
-rw-r--r--drivers/gpu/drm/radeon/radeon_kfd.h47
-rw-r--r--drivers/gpu/drm/radeon/radeon_kms.c7
10 files changed, 1008 insertions, 1 deletions
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
new file mode 100644
index 000000000000..9c729dd8dd50
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -0,0 +1,185 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23/*
24 * This file defines the private interface between the
25 * AMD kernel graphics drivers and the AMD KFD.
26 */
27
28#ifndef KGD_KFD_INTERFACE_H_INCLUDED
29#define KGD_KFD_INTERFACE_H_INCLUDED
30
31#include <linux/types.h>
32
33struct pci_dev;
34
35#define KFD_INTERFACE_VERSION 1
36
37struct kfd_dev;
38struct kgd_dev;
39
40struct kgd_mem;
41
42enum kgd_memory_pool {
43 KGD_POOL_SYSTEM_CACHEABLE = 1,
44 KGD_POOL_SYSTEM_WRITECOMBINE = 2,
45 KGD_POOL_FRAMEBUFFER = 3,
46};
47
48struct kgd2kfd_shared_resources {
49 /* Bit n == 1 means VMID n is available for KFD. */
50 unsigned int compute_vmid_bitmap;
51
52 /* Compute pipes are counted starting from MEC0/pipe0 as 0. */
53 unsigned int first_compute_pipe;
54
55 /* Number of MEC pipes available for KFD. */
56 unsigned int compute_pipe_count;
57
58 /* Base address of doorbell aperture. */
59 phys_addr_t doorbell_physical_address;
60
61 /* Size in bytes of doorbell aperture. */
62 size_t doorbell_aperture_size;
63
64 /* Number of bytes at start of aperture reserved for KGD. */
65 size_t doorbell_start_offset;
66};
67
68/**
69 * struct kgd2kfd_calls
70 *
71 * @exit: Notifies amdkfd that kgd module is unloaded
72 *
73 * @probe: Notifies amdkfd about a probe done on a device in the kgd driver.
74 *
75 * @device_init: Initialize the newly probed device (if it is a device that
76 * amdkfd supports)
77 *
78 * @device_exit: Notifies amdkfd about a removal of a kgd device
79 *
80 * @suspend: Notifies amdkfd about a suspend action done to a kgd device
81 *
82 * @resume: Notifies amdkfd about a resume action done to a kgd device
83 *
84 * This structure contains function callback pointers so the kgd driver
85 * will notify to the amdkfd about certain status changes.
86 *
87 */
88struct kgd2kfd_calls {
89 void (*exit)(void);
90 struct kfd_dev* (*probe)(struct kgd_dev *kgd, struct pci_dev *pdev);
91 bool (*device_init)(struct kfd_dev *kfd,
92 const struct kgd2kfd_shared_resources *gpu_resources);
93 void (*device_exit)(struct kfd_dev *kfd);
94 void (*interrupt)(struct kfd_dev *kfd, const void *ih_ring_entry);
95 void (*suspend)(struct kfd_dev *kfd);
96 int (*resume)(struct kfd_dev *kfd);
97};
98
99/**
100 * struct kfd2kgd_calls
101 *
102 * @init_sa_manager: Initialize an instance of the sa manager, used by
103 * amdkfd for all system memory allocations that are mapped to the GART
104 * address space
105 *
106 * @fini_sa_manager: Releases all memory allocations for amdkfd that are
107 * handled by kgd sa manager
108 *
109 * @allocate_mem: Allocate a buffer from amdkfd's sa manager. The buffer can
110 * be used for mqds, hpds, kernel queue, fence and runlists
111 *
112 * @free_mem: Frees a buffer that was allocated by amdkfd's sa manager
113 *
114 * @get_vmem_size: Retrieves (physical) size of VRAM
115 *
116 * @get_gpu_clock_counter: Retrieves GPU clock counter
117 *
118 * @get_max_engine_clock_in_mhz: Retrieves maximum GPU clock in MHz
119 *
120 * @program_sh_mem_settings: A function that should initiate the memory
121 * properties such as main aperture memory type (cache / non cached) and
122 * secondary aperture base address, size and memory type.
123 * This function is used only for no cp scheduling mode.
124 *
125 * @set_pasid_vmid_mapping: Exposes pasid/vmid pair to the H/W for no cp
126 * scheduling mode. Only used for no cp scheduling mode.
127 *
128 * @init_memory: Initializes memory apertures to fixed base/limit address
129 * and non cached memory types.
130 *
131 * @init_pipeline: Initialized the compute pipelines.
132 *
133 * @hqd_load: Loads the mqd structure to a H/W hqd slot. used only for no cp
134 * sceduling mode.
135 *
136 * @hqd_is_occupies: Checks if a hqd slot is occupied.
137 *
138 * @hqd_destroy: Destructs and preempts the queue assigned to that hqd slot.
139 *
140 * This structure contains function pointers to services that the kgd driver
141 * provides to amdkfd driver.
142 *
143 */
144struct kfd2kgd_calls {
145 /* Memory management. */
146 int (*init_sa_manager)(struct kgd_dev *kgd, unsigned int size);
147 void (*fini_sa_manager)(struct kgd_dev *kgd);
148 int (*allocate_mem)(struct kgd_dev *kgd, size_t size, size_t alignment,
149 enum kgd_memory_pool pool, struct kgd_mem **mem);
150
151 void (*free_mem)(struct kgd_dev *kgd, struct kgd_mem *mem);
152
153 uint64_t (*get_vmem_size)(struct kgd_dev *kgd);
154 uint64_t (*get_gpu_clock_counter)(struct kgd_dev *kgd);
155
156 uint32_t (*get_max_engine_clock_in_mhz)(struct kgd_dev *kgd);
157
158 /* Register access functions */
159 void (*program_sh_mem_settings)(struct kgd_dev *kgd, uint32_t vmid,
160 uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
161 uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
162
163 int (*set_pasid_vmid_mapping)(struct kgd_dev *kgd, unsigned int pasid,
164 unsigned int vmid);
165
166 int (*init_memory)(struct kgd_dev *kgd);
167 int (*init_pipeline)(struct kgd_dev *kgd, uint32_t pipe_id,
168 uint32_t hpd_size, uint64_t hpd_gpu_addr);
169
170 int (*hqd_load)(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
171 uint32_t queue_id, uint32_t __user *wptr);
172
173 bool (*hqd_is_occupies)(struct kgd_dev *kgd, uint64_t queue_address,
174 uint32_t pipe_id, uint32_t queue_id);
175
176 int (*hqd_destroy)(struct kgd_dev *kgd, uint32_t reset_type,
177 unsigned int timeout, uint32_t pipe_id,
178 uint32_t queue_id);
179};
180
181bool kgd2kfd_init(unsigned interface_version,
182 const struct kfd2kgd_calls *f2g,
183 const struct kgd2kfd_calls **g2f);
184
185#endif /* KGD_KFD_INTERFACE_H_INCLUDED */
diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile
index d01b87991422..bad6caa0a727 100644
--- a/drivers/gpu/drm/radeon/Makefile
+++ b/drivers/gpu/drm/radeon/Makefile
@@ -104,6 +104,7 @@ radeon-y += \
104 radeon_vce.o \ 104 radeon_vce.o \
105 vce_v1_0.o \ 105 vce_v1_0.o \
106 vce_v2_0.o \ 106 vce_v2_0.o \
107 radeon_kfd.o
107 108
108radeon-$(CONFIG_COMPAT) += radeon_ioc32.o 109radeon-$(CONFIG_COMPAT) += radeon_ioc32.o
109radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o 110radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o
diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c
index 2e742a591dfc..bce73b6203ac 100644
--- a/drivers/gpu/drm/radeon/cik.c
+++ b/drivers/gpu/drm/radeon/cik.c
@@ -32,6 +32,7 @@
32#include "cik_blit_shaders.h" 32#include "cik_blit_shaders.h"
33#include "radeon_ucode.h" 33#include "radeon_ucode.h"
34#include "clearstate_ci.h" 34#include "clearstate_ci.h"
35#include "radeon_kfd.h"
35 36
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); 37MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); 38MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
@@ -7798,6 +7799,10 @@ restart_ih:
7798 while (rptr != wptr) { 7799 while (rptr != wptr) {
7799 /* wptr/rptr are in bytes! */ 7800 /* wptr/rptr are in bytes! */
7800 ring_index = rptr / 4; 7801 ring_index = rptr / 4;
7802
7803 radeon_kfd_interrupt(rdev,
7804 (const void *) &rdev->ih.ring[ring_index]);
7805
7801 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 7806 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7802 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 7807 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7803 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 7808 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
@@ -8487,6 +8492,10 @@ static int cik_startup(struct radeon_device *rdev)
8487 if (r) 8492 if (r)
8488 return r; 8493 return r;
8489 8494
8495 r = radeon_kfd_resume(rdev);
8496 if (r)
8497 return r;
8498
8490 return 0; 8499 return 0;
8491} 8500}
8492 8501
@@ -8535,6 +8544,7 @@ int cik_resume(struct radeon_device *rdev)
8535 */ 8544 */
8536int cik_suspend(struct radeon_device *rdev) 8545int cik_suspend(struct radeon_device *rdev)
8537{ 8546{
8547 radeon_kfd_suspend(rdev);
8538 radeon_pm_suspend(rdev); 8548 radeon_pm_suspend(rdev);
8539 dce6_audio_fini(rdev); 8549 dce6_audio_fini(rdev);
8540 radeon_vm_manager_fini(rdev); 8550 radeon_vm_manager_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/cik_reg.h b/drivers/gpu/drm/radeon/cik_reg.h
index ca1bb6133580..79c45e8a536b 100644
--- a/drivers/gpu/drm/radeon/cik_reg.h
+++ b/drivers/gpu/drm/radeon/cik_reg.h
@@ -147,4 +147,140 @@
147 147
148#define CIK_LB_DESKTOP_HEIGHT 0x6b0c 148#define CIK_LB_DESKTOP_HEIGHT 0x6b0c
149 149
150#define CP_HQD_IQ_RPTR 0xC970u
151#define AQL_ENABLE (1U << 0)
152
153#define IDLE (1 << 2)
154
155struct cik_mqd {
156 uint32_t header;
157 uint32_t compute_dispatch_initiator;
158 uint32_t compute_dim_x;
159 uint32_t compute_dim_y;
160 uint32_t compute_dim_z;
161 uint32_t compute_start_x;
162 uint32_t compute_start_y;
163 uint32_t compute_start_z;
164 uint32_t compute_num_thread_x;
165 uint32_t compute_num_thread_y;
166 uint32_t compute_num_thread_z;
167 uint32_t compute_pipelinestat_enable;
168 uint32_t compute_perfcount_enable;
169 uint32_t compute_pgm_lo;
170 uint32_t compute_pgm_hi;
171 uint32_t compute_tba_lo;
172 uint32_t compute_tba_hi;
173 uint32_t compute_tma_lo;
174 uint32_t compute_tma_hi;
175 uint32_t compute_pgm_rsrc1;
176 uint32_t compute_pgm_rsrc2;
177 uint32_t compute_vmid;
178 uint32_t compute_resource_limits;
179 uint32_t compute_static_thread_mgmt_se0;
180 uint32_t compute_static_thread_mgmt_se1;
181 uint32_t compute_tmpring_size;
182 uint32_t compute_static_thread_mgmt_se2;
183 uint32_t compute_static_thread_mgmt_se3;
184 uint32_t compute_restart_x;
185 uint32_t compute_restart_y;
186 uint32_t compute_restart_z;
187 uint32_t compute_thread_trace_enable;
188 uint32_t compute_misc_reserved;
189 uint32_t compute_user_data_0;
190 uint32_t compute_user_data_1;
191 uint32_t compute_user_data_2;
192 uint32_t compute_user_data_3;
193 uint32_t compute_user_data_4;
194 uint32_t compute_user_data_5;
195 uint32_t compute_user_data_6;
196 uint32_t compute_user_data_7;
197 uint32_t compute_user_data_8;
198 uint32_t compute_user_data_9;
199 uint32_t compute_user_data_10;
200 uint32_t compute_user_data_11;
201 uint32_t compute_user_data_12;
202 uint32_t compute_user_data_13;
203 uint32_t compute_user_data_14;
204 uint32_t compute_user_data_15;
205 uint32_t cp_compute_csinvoc_count_lo;
206 uint32_t cp_compute_csinvoc_count_hi;
207 uint32_t cp_mqd_base_addr_lo;
208 uint32_t cp_mqd_base_addr_hi;
209 uint32_t cp_hqd_active;
210 uint32_t cp_hqd_vmid;
211 uint32_t cp_hqd_persistent_state;
212 uint32_t cp_hqd_pipe_priority;
213 uint32_t cp_hqd_queue_priority;
214 uint32_t cp_hqd_quantum;
215 uint32_t cp_hqd_pq_base_lo;
216 uint32_t cp_hqd_pq_base_hi;
217 uint32_t cp_hqd_pq_rptr;
218 uint32_t cp_hqd_pq_rptr_report_addr_lo;
219 uint32_t cp_hqd_pq_rptr_report_addr_hi;
220 uint32_t cp_hqd_pq_wptr_poll_addr_lo;
221 uint32_t cp_hqd_pq_wptr_poll_addr_hi;
222 uint32_t cp_hqd_pq_doorbell_control;
223 uint32_t cp_hqd_pq_wptr;
224 uint32_t cp_hqd_pq_control;
225 uint32_t cp_hqd_ib_base_addr_lo;
226 uint32_t cp_hqd_ib_base_addr_hi;
227 uint32_t cp_hqd_ib_rptr;
228 uint32_t cp_hqd_ib_control;
229 uint32_t cp_hqd_iq_timer;
230 uint32_t cp_hqd_iq_rptr;
231 uint32_t cp_hqd_dequeue_request;
232 uint32_t cp_hqd_dma_offload;
233 uint32_t cp_hqd_sema_cmd;
234 uint32_t cp_hqd_msg_type;
235 uint32_t cp_hqd_atomic0_preop_lo;
236 uint32_t cp_hqd_atomic0_preop_hi;
237 uint32_t cp_hqd_atomic1_preop_lo;
238 uint32_t cp_hqd_atomic1_preop_hi;
239 uint32_t cp_hqd_hq_status0;
240 uint32_t cp_hqd_hq_control0;
241 uint32_t cp_mqd_control;
242 uint32_t cp_mqd_query_time_lo;
243 uint32_t cp_mqd_query_time_hi;
244 uint32_t cp_mqd_connect_start_time_lo;
245 uint32_t cp_mqd_connect_start_time_hi;
246 uint32_t cp_mqd_connect_end_time_lo;
247 uint32_t cp_mqd_connect_end_time_hi;
248 uint32_t cp_mqd_connect_end_wf_count;
249 uint32_t cp_mqd_connect_end_pq_rptr;
250 uint32_t cp_mqd_connect_end_pq_wptr;
251 uint32_t cp_mqd_connect_end_ib_rptr;
252 uint32_t reserved_96;
253 uint32_t reserved_97;
254 uint32_t reserved_98;
255 uint32_t reserved_99;
256 uint32_t iqtimer_pkt_header;
257 uint32_t iqtimer_pkt_dw0;
258 uint32_t iqtimer_pkt_dw1;
259 uint32_t iqtimer_pkt_dw2;
260 uint32_t iqtimer_pkt_dw3;
261 uint32_t iqtimer_pkt_dw4;
262 uint32_t iqtimer_pkt_dw5;
263 uint32_t iqtimer_pkt_dw6;
264 uint32_t reserved_108;
265 uint32_t reserved_109;
266 uint32_t reserved_110;
267 uint32_t reserved_111;
268 uint32_t queue_doorbell_id0;
269 uint32_t queue_doorbell_id1;
270 uint32_t queue_doorbell_id2;
271 uint32_t queue_doorbell_id3;
272 uint32_t queue_doorbell_id4;
273 uint32_t queue_doorbell_id5;
274 uint32_t queue_doorbell_id6;
275 uint32_t queue_doorbell_id7;
276 uint32_t queue_doorbell_id8;
277 uint32_t queue_doorbell_id9;
278 uint32_t queue_doorbell_id10;
279 uint32_t queue_doorbell_id11;
280 uint32_t queue_doorbell_id12;
281 uint32_t queue_doorbell_id13;
282 uint32_t queue_doorbell_id14;
283 uint32_t queue_doorbell_id15;
284};
285
150#endif 286#endif
diff --git a/drivers/gpu/drm/radeon/cikd.h b/drivers/gpu/drm/radeon/cikd.h
index fae4d0c68478..068cbb019326 100644
--- a/drivers/gpu/drm/radeon/cikd.h
+++ b/drivers/gpu/drm/radeon/cikd.h
@@ -1139,6 +1139,9 @@
1139#define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3 1139#define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3
1140#define DEFAULT_MTYPE(x) ((x) << 4) 1140#define DEFAULT_MTYPE(x) ((x) << 4)
1141#define APE1_MTYPE(x) ((x) << 7) 1141#define APE1_MTYPE(x) ((x) << 7)
1142/* valid for both DEFAULT_MTYPE and APE1_MTYPE */
1143#define MTYPE_CACHED 0
1144#define MTYPE_NONCACHED 3
1142 1145
1143#define SX_DEBUG_1 0x9060 1146#define SX_DEBUG_1 0x9060
1144 1147
@@ -1449,6 +1452,16 @@
1449#define CP_HQD_ACTIVE 0xC91C 1452#define CP_HQD_ACTIVE 0xC91C
1450#define CP_HQD_VMID 0xC920 1453#define CP_HQD_VMID 0xC920
1451 1454
1455#define CP_HQD_PERSISTENT_STATE 0xC924u
1456#define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8)
1457
1458#define CP_HQD_PIPE_PRIORITY 0xC928u
1459#define CP_HQD_QUEUE_PRIORITY 0xC92Cu
1460#define CP_HQD_QUANTUM 0xC930u
1461#define QUANTUM_EN 1U
1462#define QUANTUM_SCALE_1MS (1U << 4)
1463#define QUANTUM_DURATION(x) ((x) << 8)
1464
1452#define CP_HQD_PQ_BASE 0xC934 1465#define CP_HQD_PQ_BASE 0xC934
1453#define CP_HQD_PQ_BASE_HI 0xC938 1466#define CP_HQD_PQ_BASE_HI 0xC938
1454#define CP_HQD_PQ_RPTR 0xC93C 1467#define CP_HQD_PQ_RPTR 0xC93C
@@ -1476,12 +1489,32 @@
1476#define PRIV_STATE (1 << 30) 1489#define PRIV_STATE (1 << 30)
1477#define KMD_QUEUE (1 << 31) 1490#define KMD_QUEUE (1 << 31)
1478 1491
1479#define CP_HQD_DEQUEUE_REQUEST 0xC974 1492#define CP_HQD_IB_BASE_ADDR 0xC95Cu
1493#define CP_HQD_IB_BASE_ADDR_HI 0xC960u
1494#define CP_HQD_IB_RPTR 0xC964u
1495#define CP_HQD_IB_CONTROL 0xC968u
1496#define IB_ATC_EN (1U << 23)
1497#define DEFAULT_MIN_IB_AVAIL_SIZE (3U << 20)
1498
1499#define CP_HQD_DEQUEUE_REQUEST 0xC974
1500#define DEQUEUE_REQUEST_DRAIN 1
1501#define DEQUEUE_REQUEST_RESET 2
1480 1502
1481#define CP_MQD_CONTROL 0xC99C 1503#define CP_MQD_CONTROL 0xC99C
1482#define MQD_VMID(x) ((x) << 0) 1504#define MQD_VMID(x) ((x) << 0)
1483#define MQD_VMID_MASK (0xf << 0) 1505#define MQD_VMID_MASK (0xf << 0)
1484 1506
1507#define CP_HQD_SEMA_CMD 0xC97Cu
1508#define CP_HQD_MSG_TYPE 0xC980u
1509#define CP_HQD_ATOMIC0_PREOP_LO 0xC984u
1510#define CP_HQD_ATOMIC0_PREOP_HI 0xC988u
1511#define CP_HQD_ATOMIC1_PREOP_LO 0xC98Cu
1512#define CP_HQD_ATOMIC1_PREOP_HI 0xC990u
1513#define CP_HQD_HQ_SCHEDULER0 0xC994u
1514#define CP_HQD_HQ_SCHEDULER1 0xC998u
1515
1516#define SH_STATIC_MEM_CONFIG 0x9604u
1517
1485#define DB_RENDER_CONTROL 0x28000 1518#define DB_RENDER_CONTROL 0x28000
1486 1519
1487#define PA_SC_RASTER_CONFIG 0x28350 1520#define PA_SC_RASTER_CONFIG 0x28350
@@ -2071,4 +2104,20 @@
2071#define VCE_CMD_IB_AUTO 0x00000005 2104#define VCE_CMD_IB_AUTO 0x00000005
2072#define VCE_CMD_SEMAPHORE 0x00000006 2105#define VCE_CMD_SEMAPHORE 0x00000006
2073 2106
2107#define ATC_VMID0_PASID_MAPPING 0x339Cu
2108#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x3398u
2109#define ATC_VMID_PASID_MAPPING_VALID (1U << 31)
2110
2111#define ATC_VM_APERTURE0_CNTL 0x3310u
2112#define ATS_ACCESS_MODE_NEVER 0
2113#define ATS_ACCESS_MODE_ALWAYS 1
2114
2115#define ATC_VM_APERTURE0_CNTL2 0x3318u
2116#define ATC_VM_APERTURE0_HIGH_ADDR 0x3308u
2117#define ATC_VM_APERTURE0_LOW_ADDR 0x3300u
2118#define ATC_VM_APERTURE1_CNTL 0x3314u
2119#define ATC_VM_APERTURE1_CNTL2 0x331Cu
2120#define ATC_VM_APERTURE1_HIGH_ADDR 0x330Cu
2121#define ATC_VM_APERTURE1_LOW_ADDR 0x3304u
2122
2074#endif 2123#endif
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index e95718f02de3..1f61ff089c9e 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -2411,6 +2411,10 @@ struct radeon_device {
2411 u64 vram_pin_size; 2411 u64 vram_pin_size;
2412 u64 gart_pin_size; 2412 u64 gart_pin_size;
2413 2413
2414 /* amdkfd interface */
2415 struct kfd_dev *kfd;
2416 struct radeon_sa_manager kfd_bo;
2417
2414 struct mutex mn_lock; 2418 struct mutex mn_lock;
2415 DECLARE_HASHTABLE(mn_hash, 7); 2419 DECLARE_HASHTABLE(mn_hash, 7);
2416}; 2420};
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index dcffa30ee2db..4f50fb0e3d93 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -41,6 +41,8 @@
41#include <drm/drm_gem.h> 41#include <drm/drm_gem.h>
42 42
43#include "drm_crtc_helper.h" 43#include "drm_crtc_helper.h"
44#include "radeon_kfd.h"
45
44/* 46/*
45 * KMS wrapper. 47 * KMS wrapper.
46 * - 2.0.0 - initial interface 48 * - 2.0.0 - initial interface
@@ -654,12 +656,15 @@ static int __init radeon_init(void)
654#endif 656#endif
655 } 657 }
656 658
659 radeon_kfd_init();
660
657 /* let modprobe override vga console setting */ 661 /* let modprobe override vga console setting */
658 return drm_pci_init(driver, pdriver); 662 return drm_pci_init(driver, pdriver);
659} 663}
660 664
661static void __exit radeon_exit(void) 665static void __exit radeon_exit(void)
662{ 666{
667 radeon_kfd_fini();
663 drm_pci_exit(driver, pdriver); 668 drm_pci_exit(driver, pdriver);
664 radeon_unregister_atpx_handler(); 669 radeon_unregister_atpx_handler();
665} 670}
diff --git a/drivers/gpu/drm/radeon/radeon_kfd.c b/drivers/gpu/drm/radeon/radeon_kfd.c
new file mode 100644
index 000000000000..065d02068ec3
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_kfd.c
@@ -0,0 +1,563 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/module.h>
24#include <linux/fdtable.h>
25#include <linux/uaccess.h>
26#include <drm/drmP.h>
27#include "radeon.h"
28#include "cikd.h"
29#include "cik_reg.h"
30#include "radeon_kfd.h"
31
32#define CIK_PIPE_PER_MEC (4)
33
34struct kgd_mem {
35 struct radeon_sa_bo *sa_bo;
36 uint64_t gpu_addr;
37 void *ptr;
38};
39
40static int init_sa_manager(struct kgd_dev *kgd, unsigned int size);
41static void fini_sa_manager(struct kgd_dev *kgd);
42
43static int allocate_mem(struct kgd_dev *kgd, size_t size, size_t alignment,
44 enum kgd_memory_pool pool, struct kgd_mem **mem);
45
46static void free_mem(struct kgd_dev *kgd, struct kgd_mem *mem);
47
48static uint64_t get_vmem_size(struct kgd_dev *kgd);
49static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
50
51static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
52
53/*
54 * Register access functions
55 */
56
57static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
58 uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
59 uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
60
61static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
62 unsigned int vmid);
63
64static int kgd_init_memory(struct kgd_dev *kgd);
65
66static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
67 uint32_t hpd_size, uint64_t hpd_gpu_addr);
68
69static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
70 uint32_t queue_id, uint32_t __user *wptr);
71
72static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
73 uint32_t pipe_id, uint32_t queue_id);
74
75static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
76 unsigned int timeout, uint32_t pipe_id,
77 uint32_t queue_id);
78
79static const struct kfd2kgd_calls kfd2kgd = {
80 .init_sa_manager = init_sa_manager,
81 .fini_sa_manager = fini_sa_manager,
82 .allocate_mem = allocate_mem,
83 .free_mem = free_mem,
84 .get_vmem_size = get_vmem_size,
85 .get_gpu_clock_counter = get_gpu_clock_counter,
86 .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
87 .program_sh_mem_settings = kgd_program_sh_mem_settings,
88 .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
89 .init_memory = kgd_init_memory,
90 .init_pipeline = kgd_init_pipeline,
91 .hqd_load = kgd_hqd_load,
92 .hqd_is_occupies = kgd_hqd_is_occupies,
93 .hqd_destroy = kgd_hqd_destroy,
94};
95
96static const struct kgd2kfd_calls *kgd2kfd;
97
98bool radeon_kfd_init(void)
99{
100 bool (*kgd2kfd_init_p)(unsigned, const struct kfd2kgd_calls*,
101 const struct kgd2kfd_calls**);
102
103 kgd2kfd_init_p = symbol_request(kgd2kfd_init);
104
105 if (kgd2kfd_init_p == NULL)
106 return false;
107
108 if (!kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kfd2kgd, &kgd2kfd)) {
109 symbol_put(kgd2kfd_init);
110 kgd2kfd = NULL;
111
112 return false;
113 }
114
115 return true;
116}
117
118void radeon_kfd_fini(void)
119{
120 if (kgd2kfd) {
121 kgd2kfd->exit();
122 symbol_put(kgd2kfd_init);
123 }
124}
125
126void radeon_kfd_device_probe(struct radeon_device *rdev)
127{
128 if (kgd2kfd)
129 rdev->kfd = kgd2kfd->probe((struct kgd_dev *)rdev, rdev->pdev);
130}
131
132void radeon_kfd_device_init(struct radeon_device *rdev)
133{
134 if (rdev->kfd) {
135 struct kgd2kfd_shared_resources gpu_resources = {
136 .compute_vmid_bitmap = 0xFF00,
137
138 .first_compute_pipe = 1,
139 .compute_pipe_count = 8 - 1,
140 };
141
142 radeon_doorbell_get_kfd_info(rdev,
143 &gpu_resources.doorbell_physical_address,
144 &gpu_resources.doorbell_aperture_size,
145 &gpu_resources.doorbell_start_offset);
146
147 kgd2kfd->device_init(rdev->kfd, &gpu_resources);
148 }
149}
150
151void radeon_kfd_device_fini(struct radeon_device *rdev)
152{
153 if (rdev->kfd) {
154 kgd2kfd->device_exit(rdev->kfd);
155 rdev->kfd = NULL;
156 }
157}
158
159void radeon_kfd_interrupt(struct radeon_device *rdev, const void *ih_ring_entry)
160{
161 if (rdev->kfd)
162 kgd2kfd->interrupt(rdev->kfd, ih_ring_entry);
163}
164
165void radeon_kfd_suspend(struct radeon_device *rdev)
166{
167 if (rdev->kfd)
168 kgd2kfd->suspend(rdev->kfd);
169}
170
171int radeon_kfd_resume(struct radeon_device *rdev)
172{
173 int r = 0;
174
175 if (rdev->kfd)
176 r = kgd2kfd->resume(rdev->kfd);
177
178 return r;
179}
180
181static u32 pool_to_domain(enum kgd_memory_pool p)
182{
183 switch (p) {
184 case KGD_POOL_FRAMEBUFFER: return RADEON_GEM_DOMAIN_VRAM;
185 default: return RADEON_GEM_DOMAIN_GTT;
186 }
187}
188
189static int init_sa_manager(struct kgd_dev *kgd, unsigned int size)
190{
191 struct radeon_device *rdev = (struct radeon_device *)kgd;
192 int r;
193
194 BUG_ON(kgd == NULL);
195
196 r = radeon_sa_bo_manager_init(rdev, &rdev->kfd_bo,
197 size,
198 RADEON_GPU_PAGE_SIZE,
199 RADEON_GEM_DOMAIN_GTT,
200 RADEON_GEM_GTT_WC);
201
202 if (r)
203 return r;
204
205 r = radeon_sa_bo_manager_start(rdev, &rdev->kfd_bo);
206 if (r)
207 radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo);
208
209 return r;
210}
211
212static void fini_sa_manager(struct kgd_dev *kgd)
213{
214 struct radeon_device *rdev = (struct radeon_device *)kgd;
215
216 BUG_ON(kgd == NULL);
217
218 radeon_sa_bo_manager_suspend(rdev, &rdev->kfd_bo);
219 radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo);
220}
221
222static int allocate_mem(struct kgd_dev *kgd, size_t size, size_t alignment,
223 enum kgd_memory_pool pool, struct kgd_mem **mem)
224{
225 struct radeon_device *rdev = (struct radeon_device *)kgd;
226 u32 domain;
227 int r;
228
229 BUG_ON(kgd == NULL);
230
231 domain = pool_to_domain(pool);
232 if (domain != RADEON_GEM_DOMAIN_GTT) {
233 dev_err(rdev->dev,
234 "Only allowed to allocate gart memory for kfd\n");
235 return -EINVAL;
236 }
237
238 *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL);
239 if ((*mem) == NULL)
240 return -ENOMEM;
241
242 r = radeon_sa_bo_new(rdev, &rdev->kfd_bo, &(*mem)->sa_bo, size,
243 alignment);
244 if (r) {
245 dev_err(rdev->dev, "failed to get memory for kfd (%d)\n", r);
246 return r;
247 }
248
249 (*mem)->ptr = radeon_sa_bo_cpu_addr((*mem)->sa_bo);
250 (*mem)->gpu_addr = radeon_sa_bo_gpu_addr((*mem)->sa_bo);
251
252 return 0;
253}
254
255static void free_mem(struct kgd_dev *kgd, struct kgd_mem *mem)
256{
257 struct radeon_device *rdev = (struct radeon_device *)kgd;
258
259 BUG_ON(kgd == NULL);
260
261 radeon_sa_bo_free(rdev, &mem->sa_bo, NULL);
262 kfree(mem);
263}
264
265static uint64_t get_vmem_size(struct kgd_dev *kgd)
266{
267 struct radeon_device *rdev = (struct radeon_device *)kgd;
268
269 BUG_ON(kgd == NULL);
270
271 return rdev->mc.real_vram_size;
272}
273
274static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
275{
276 struct radeon_device *rdev = (struct radeon_device *)kgd;
277
278 return rdev->asic->get_gpu_clock_counter(rdev);
279}
280
281static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
282{
283 struct radeon_device *rdev = (struct radeon_device *)kgd;
284
285 /* The sclk is in quantas of 10kHz */
286 return rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100;
287}
288
289static inline struct radeon_device *get_radeon_device(struct kgd_dev *kgd)
290{
291 return (struct radeon_device *)kgd;
292}
293
294static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value)
295{
296 struct radeon_device *rdev = get_radeon_device(kgd);
297
298 writel(value, (void __iomem *)(rdev->rmmio + offset));
299}
300
301static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset)
302{
303 struct radeon_device *rdev = get_radeon_device(kgd);
304
305 return readl((void __iomem *)(rdev->rmmio + offset));
306}
307
308static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
309 uint32_t queue, uint32_t vmid)
310{
311 struct radeon_device *rdev = get_radeon_device(kgd);
312 uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);
313
314 mutex_lock(&rdev->srbm_mutex);
315 write_register(kgd, SRBM_GFX_CNTL, value);
316}
317
318static void unlock_srbm(struct kgd_dev *kgd)
319{
320 struct radeon_device *rdev = get_radeon_device(kgd);
321
322 write_register(kgd, SRBM_GFX_CNTL, 0);
323 mutex_unlock(&rdev->srbm_mutex);
324}
325
326static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
327 uint32_t queue_id)
328{
329 uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
330 uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
331
332 lock_srbm(kgd, mec, pipe, queue_id, 0);
333}
334
335static void release_queue(struct kgd_dev *kgd)
336{
337 unlock_srbm(kgd);
338}
339
340static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
341 uint32_t sh_mem_config,
342 uint32_t sh_mem_ape1_base,
343 uint32_t sh_mem_ape1_limit,
344 uint32_t sh_mem_bases)
345{
346 lock_srbm(kgd, 0, 0, 0, vmid);
347
348 write_register(kgd, SH_MEM_CONFIG, sh_mem_config);
349 write_register(kgd, SH_MEM_APE1_BASE, sh_mem_ape1_base);
350 write_register(kgd, SH_MEM_APE1_LIMIT, sh_mem_ape1_limit);
351 write_register(kgd, SH_MEM_BASES, sh_mem_bases);
352
353 unlock_srbm(kgd);
354}
355
356static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
357 unsigned int vmid)
358{
359 /*
360 * We have to assume that there is no outstanding mapping.
361 * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0
362 * because a mapping is in progress or because a mapping finished and
363 * the SW cleared it.
364 * So the protocol is to always wait & clear.
365 */
366 uint32_t pasid_mapping = (pasid == 0) ? 0 :
367 (uint32_t)pasid | ATC_VMID_PASID_MAPPING_VALID;
368
369 write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t),
370 pasid_mapping);
371
372 while (!(read_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) &
373 (1U << vmid)))
374 cpu_relax();
375 write_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);
376
377 return 0;
378}
379
380static int kgd_init_memory(struct kgd_dev *kgd)
381{
382 /*
383 * Configure apertures:
384 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
385 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
386 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
387 */
388 int i;
389 uint32_t sh_mem_bases = PRIVATE_BASE(0x6000) | SHARED_BASE(0x6000);
390
391 for (i = 8; i < 16; i++) {
392 uint32_t sh_mem_config;
393
394 lock_srbm(kgd, 0, 0, 0, i);
395
396 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
397 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
398
399 write_register(kgd, SH_MEM_CONFIG, sh_mem_config);
400
401 write_register(kgd, SH_MEM_BASES, sh_mem_bases);
402
403 /* Scratch aperture is not supported for now. */
404 write_register(kgd, SH_STATIC_MEM_CONFIG, 0);
405
406 /* APE1 disabled for now. */
407 write_register(kgd, SH_MEM_APE1_BASE, 1);
408 write_register(kgd, SH_MEM_APE1_LIMIT, 0);
409
410 unlock_srbm(kgd);
411 }
412
413 return 0;
414}
415
416static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
417 uint32_t hpd_size, uint64_t hpd_gpu_addr)
418{
419 uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
420 uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
421
422 lock_srbm(kgd, mec, pipe, 0, 0);
423 write_register(kgd, CP_HPD_EOP_BASE_ADDR,
424 lower_32_bits(hpd_gpu_addr >> 8));
425 write_register(kgd, CP_HPD_EOP_BASE_ADDR_HI,
426 upper_32_bits(hpd_gpu_addr >> 8));
427 write_register(kgd, CP_HPD_EOP_VMID, 0);
428 write_register(kgd, CP_HPD_EOP_CONTROL, hpd_size);
429 unlock_srbm(kgd);
430
431 return 0;
432}
433
434static inline struct cik_mqd *get_mqd(void *mqd)
435{
436 return (struct cik_mqd *)mqd;
437}
438
439static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
440 uint32_t queue_id, uint32_t __user *wptr)
441{
442 uint32_t wptr_shadow, is_wptr_shadow_valid;
443 struct cik_mqd *m;
444
445 m = get_mqd(mqd);
446
447 is_wptr_shadow_valid = !get_user(wptr_shadow, wptr);
448
449 acquire_queue(kgd, pipe_id, queue_id);
450 write_register(kgd, CP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo);
451 write_register(kgd, CP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi);
452 write_register(kgd, CP_MQD_CONTROL, m->cp_mqd_control);
453
454 write_register(kgd, CP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo);
455 write_register(kgd, CP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi);
456 write_register(kgd, CP_HQD_PQ_CONTROL, m->cp_hqd_pq_control);
457
458 write_register(kgd, CP_HQD_IB_CONTROL, m->cp_hqd_ib_control);
459 write_register(kgd, CP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo);
460 write_register(kgd, CP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi);
461
462 write_register(kgd, CP_HQD_IB_RPTR, m->cp_hqd_ib_rptr);
463
464 write_register(kgd, CP_HQD_PERSISTENT_STATE,
465 m->cp_hqd_persistent_state);
466 write_register(kgd, CP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd);
467 write_register(kgd, CP_HQD_MSG_TYPE, m->cp_hqd_msg_type);
468
469 write_register(kgd, CP_HQD_ATOMIC0_PREOP_LO,
470 m->cp_hqd_atomic0_preop_lo);
471
472 write_register(kgd, CP_HQD_ATOMIC0_PREOP_HI,
473 m->cp_hqd_atomic0_preop_hi);
474
475 write_register(kgd, CP_HQD_ATOMIC1_PREOP_LO,
476 m->cp_hqd_atomic1_preop_lo);
477
478 write_register(kgd, CP_HQD_ATOMIC1_PREOP_HI,
479 m->cp_hqd_atomic1_preop_hi);
480
481 write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR,
482 m->cp_hqd_pq_rptr_report_addr_lo);
483
484 write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
485 m->cp_hqd_pq_rptr_report_addr_hi);
486
487 write_register(kgd, CP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr);
488
489 write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR,
490 m->cp_hqd_pq_wptr_poll_addr_lo);
491
492 write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR_HI,
493 m->cp_hqd_pq_wptr_poll_addr_hi);
494
495 write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL,
496 m->cp_hqd_pq_doorbell_control);
497
498 write_register(kgd, CP_HQD_VMID, m->cp_hqd_vmid);
499
500 write_register(kgd, CP_HQD_QUANTUM, m->cp_hqd_quantum);
501
502 write_register(kgd, CP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority);
503 write_register(kgd, CP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority);
504
505 write_register(kgd, CP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr);
506
507 if (is_wptr_shadow_valid)
508 write_register(kgd, CP_HQD_PQ_WPTR, wptr_shadow);
509
510 write_register(kgd, CP_HQD_ACTIVE, m->cp_hqd_active);
511 release_queue(kgd);
512
513 return 0;
514}
515
516static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
517 uint32_t pipe_id, uint32_t queue_id)
518{
519 uint32_t act;
520 bool retval = false;
521 uint32_t low, high;
522
523 acquire_queue(kgd, pipe_id, queue_id);
524 act = read_register(kgd, CP_HQD_ACTIVE);
525 if (act) {
526 low = lower_32_bits(queue_address >> 8);
527 high = upper_32_bits(queue_address >> 8);
528
529 if (low == read_register(kgd, CP_HQD_PQ_BASE) &&
530 high == read_register(kgd, CP_HQD_PQ_BASE_HI))
531 retval = true;
532 }
533 release_queue(kgd);
534 return retval;
535}
536
537static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
538 unsigned int timeout, uint32_t pipe_id,
539 uint32_t queue_id)
540{
541 uint32_t temp;
542
543 acquire_queue(kgd, pipe_id, queue_id);
544 write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, 0);
545
546 write_register(kgd, CP_HQD_DEQUEUE_REQUEST, reset_type);
547
548 while (true) {
549 temp = read_register(kgd, CP_HQD_ACTIVE);
550 if (temp & 0x1)
551 break;
552 if (timeout == 0) {
553 pr_err("kfd: cp queue preemption time out (%dms)\n",
554 temp);
555 return -ETIME;
556 }
557 msleep(20);
558 timeout -= 20;
559 }
560
561 release_queue(kgd);
562 return 0;
563}
diff --git a/drivers/gpu/drm/radeon/radeon_kfd.h b/drivers/gpu/drm/radeon/radeon_kfd.h
new file mode 100644
index 000000000000..f90e161ca507
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_kfd.h
@@ -0,0 +1,47 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23/*
24 * radeon_kfd.h defines the private interface between the
25 * AMD kernel graphics drivers and the AMD KFD.
26 */
27
28#ifndef RADEON_KFD_H_INCLUDED
29#define RADEON_KFD_H_INCLUDED
30
31#include <linux/types.h>
32#include "../amd/include/kgd_kfd_interface.h"
33
34struct radeon_device;
35
36bool radeon_kfd_init(void);
37void radeon_kfd_fini(void);
38
39void radeon_kfd_suspend(struct radeon_device *rdev);
40int radeon_kfd_resume(struct radeon_device *rdev);
41void radeon_kfd_interrupt(struct radeon_device *rdev,
42 const void *ih_ring_entry);
43void radeon_kfd_device_probe(struct radeon_device *rdev);
44void radeon_kfd_device_init(struct radeon_device *rdev);
45void radeon_kfd_device_fini(struct radeon_device *rdev);
46
47#endif /* RADEON_KFD_H_INCLUDED */
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 8309b11e674d..6eb561d33eba 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -34,6 +34,8 @@
34#include <linux/slab.h> 34#include <linux/slab.h>
35#include <linux/pm_runtime.h> 35#include <linux/pm_runtime.h>
36 36
37#include "radeon_kfd.h"
38
37#if defined(CONFIG_VGA_SWITCHEROO) 39#if defined(CONFIG_VGA_SWITCHEROO)
38bool radeon_has_atpx(void); 40bool radeon_has_atpx(void);
39#else 41#else
@@ -63,6 +65,8 @@ int radeon_driver_unload_kms(struct drm_device *dev)
63 65
64 pm_runtime_get_sync(dev->dev); 66 pm_runtime_get_sync(dev->dev);
65 67
68 radeon_kfd_device_fini(rdev);
69
66 radeon_acpi_fini(rdev); 70 radeon_acpi_fini(rdev);
67 71
68 radeon_modeset_fini(rdev); 72 radeon_modeset_fini(rdev);
@@ -142,6 +146,9 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags)
142 "Error during ACPI methods call\n"); 146 "Error during ACPI methods call\n");
143 } 147 }
144 148
149 radeon_kfd_device_probe(rdev);
150 radeon_kfd_device_init(rdev);
151
145 if (radeon_is_px(dev)) { 152 if (radeon_is_px(dev)) {
146 pm_runtime_use_autosuspend(dev->dev); 153 pm_runtime_use_autosuspend(dev->dev);
147 pm_runtime_set_autosuspend_delay(dev->dev, 5000); 154 pm_runtime_set_autosuspend_delay(dev->dev, 5000);
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-12 08:24:03 -0500