aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBen Goz <ben.goz@amd.com>2015-01-04 04:24:25 -0500
committerOded Gabbay <oded.gabbay@amd.com>2015-01-04 04:24:25 -0500
commit4b8f589b052fe800e36f11eb2d29d4cf364bbed0 (patch)
treeedb12b56e6c40bff6960b9be667d057c6543b96c
parent0da7558c690708259617417d3ddc5b5042f1cca4 (diff)
drm/amdkfd: Change MQD manager to be H/W specific
The MQDs for CI and VI are different. Therefore, the MQD manager module need to be H/W specific. This patch splits the current MQD manager into three files: - kfd_mqd_manager.c, which contains common functions and initializes the specific mqd manager module according to the H/W - kfd_mqd_manager_cik.c, which contains Kaveri specific functions. This is basically the old kfd_mqd_manager.c - kfd_mqd_manager_vi.c, which will contain VI specific functions. Currently it is not implemented except for returning NULL on initialization. Signed-off-by: Ben Goz <ben.goz@amd.com> Signed-off-by: Oded Gabbay <oded.gabbay@amd.com> Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
Diffstat
-rw-r--r--drivers/gpu/drm/amd/amdkfd/Makefile1
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c434
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c454
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c33
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_priv.h4
5 files changed, 498 insertions, 428 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
index be6246de5091..bc6053f8b843 100644
--- a/drivers/gpu/drm/amd/amdkfd/Makefile
+++ b/drivers/gpu/drm/amd/amdkfd/Makefile
@@ -7,6 +7,7 @@ ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/
7amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ 7amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \
8 kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \ 8 kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \
9 kfd_process.o kfd_queue.o kfd_mqd_manager.o \ 9 kfd_process.o kfd_queue.o kfd_mqd_manager.o \
10 kfd_mqd_manager_cik.o kfd_mqd_manager_vi.o \
10 kfd_kernel_queue.o kfd_packet_manager.o \ 11 kfd_kernel_queue.o kfd_packet_manager.o \
11 kfd_process_queue_manager.o kfd_device_queue_manager.o \ 12 kfd_process_queue_manager.o kfd_device_queue_manager.o \
12 kfd_interrupt.o 13 kfd_interrupt.o
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
index be989fb22846..b1ef1368c3bb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
@@ -21,439 +21,17 @@
21 * 21 *
22 */ 22 */
23 23
24#include <linux/printk.h>
25#include <linux/slab.h>
26#include "kfd_priv.h" 24#include "kfd_priv.h"
27#include "kfd_mqd_manager.h"
28#include "cik_regs.h"
29#include "cik_structs.h"
30
31inline void busy_wait(unsigned long ms)
32{
33 while (time_before(jiffies, ms))
34 cpu_relax();
35}
36
37static inline struct cik_mqd *get_mqd(void *mqd)
38{
39 return (struct cik_mqd *)mqd;
40}
41
42static int init_mqd(struct mqd_manager *mm, void **mqd,
43 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
44 struct queue_properties *q)
45{
46 uint64_t addr;
47 struct cik_mqd *m;
48 int retval;
49
50 BUG_ON(!mm || !q || !mqd);
51
52 pr_debug("kfd: In func %s\n", __func__);
53
54 retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd),
55 mqd_mem_obj);
56
57 if (retval != 0)
58 return -ENOMEM;
59
60 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr;
61 addr = (*mqd_mem_obj)->gpu_addr;
62
63 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
64
65 m->header = 0xC0310800;
66 m->compute_pipelinestat_enable = 1;
67 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
68 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
69 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
70 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
71
72 /*
73 * Make sure to use the last queue state saved on mqd when the cp
74 * reassigns the queue, so when queue is switched on/off (e.g over
75 * subscription or quantum timeout) the context will be consistent
76 */
77 m->cp_hqd_persistent_state =
78 DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ;
79
80 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
81 m->cp_mqd_base_addr_lo = lower_32_bits(addr);
82 m->cp_mqd_base_addr_hi = upper_32_bits(addr);
83
84 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN;
85 /* Although WinKFD writes this, I suspect it should not be necessary */
86 m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE;
87
88 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
89 QUANTUM_DURATION(10);
90
91 /*
92 * Pipe Priority
93 * Identifies the pipe relative priority when this queue is connected
94 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
95 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
96 * 0 = CS_LOW (typically below GFX)
97 * 1 = CS_MEDIUM (typically between HP3D and GFX
98 * 2 = CS_HIGH (typically above HP3D)
99 */
100 m->cp_hqd_pipe_priority = 1;
101 m->cp_hqd_queue_priority = 15;
102
103 *mqd = m;
104 if (gart_addr != NULL)
105 *gart_addr = addr;
106 retval = mm->update_mqd(mm, m, q);
107
108 return retval;
109}
110
111static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
112 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
113 struct queue_properties *q)
114{
115 int retval;
116 struct cik_sdma_rlc_registers *m;
117
118 BUG_ON(!mm || !mqd || !mqd_mem_obj);
119
120 retval = kfd_gtt_sa_allocate(mm->dev,
121 sizeof(struct cik_sdma_rlc_registers),
122 mqd_mem_obj);
123
124 if (retval != 0)
125 return -ENOMEM;
126
127 m = (struct cik_sdma_rlc_registers *) (*mqd_mem_obj)->cpu_ptr;
128
129 memset(m, 0, sizeof(struct cik_sdma_rlc_registers));
130
131 *mqd = m;
132 if (gart_addr != NULL)
133 *gart_addr = (*mqd_mem_obj)->gpu_addr;
134
135 retval = mm->update_mqd(mm, m, q);
136
137 return retval;
138}
139
140static void uninit_mqd(struct mqd_manager *mm, void *mqd,
141 struct kfd_mem_obj *mqd_mem_obj)
142{
143 BUG_ON(!mm || !mqd);
144 kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
145}
146
147static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
148 struct kfd_mem_obj *mqd_mem_obj)
149{
150 BUG_ON(!mm || !mqd);
151 kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
152}
153
154static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
155 uint32_t queue_id, uint32_t __user *wptr)
156{
157 return kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
158}
159
160static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
161 uint32_t pipe_id, uint32_t queue_id,
162 uint32_t __user *wptr)
163{
164 return kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd);
165}
166
167static int update_mqd(struct mqd_manager *mm, void *mqd,
168 struct queue_properties *q)
169{
170 struct cik_mqd *m;
171
172 BUG_ON(!mm || !q || !mqd);
173
174 pr_debug("kfd: In func %s\n", __func__);
175
176 m = get_mqd(mqd);
177 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
178 DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN;
179
180 /*
181 * Calculating queue size which is log base 2 of actual queue size -1
182 * dwords and another -1 for ffs
183 */
184 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
185 - 1 - 1;
186 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
187 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
188 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
189 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
190 m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
191 DOORBELL_OFFSET(q->doorbell_off);
192
193 m->cp_hqd_vmid = q->vmid;
194
195 if (q->format == KFD_QUEUE_FORMAT_AQL) {
196 m->cp_hqd_iq_rptr = AQL_ENABLE;
197 m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
198 }
199
200 m->cp_hqd_active = 0;
201 q->is_active = false;
202 if (q->queue_size > 0 &&
203 q->queue_address != 0 &&
204 q->queue_percent > 0) {
205 m->cp_hqd_active = 1;
206 q->is_active = true;
207 }
208
209 return 0;
210}
211
212static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
213 struct queue_properties *q)
214{
215 struct cik_sdma_rlc_registers *m;
216
217 BUG_ON(!mm || !mqd || !q);
218
219 m = get_sdma_mqd(mqd);
220 m->sdma_rlc_rb_cntl =
221 SDMA_RB_SIZE((ffs(q->queue_size / sizeof(unsigned int)))) |
222 SDMA_RB_VMID(q->vmid) |
223 SDMA_RPTR_WRITEBACK_ENABLE |
224 SDMA_RPTR_WRITEBACK_TIMER(6);
225
226 m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8);
227 m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8);
228 m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
229 m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
230 m->sdma_rlc_doorbell = SDMA_OFFSET(q->doorbell_off) | SDMA_DB_ENABLE;
231 m->sdma_rlc_virtual_addr = q->sdma_vm_addr;
232
233 m->sdma_engine_id = q->sdma_engine_id;
234 m->sdma_queue_id = q->sdma_queue_id;
235
236 q->is_active = false;
237 if (q->queue_size > 0 &&
238 q->queue_address != 0 &&
239 q->queue_percent > 0) {
240 m->sdma_rlc_rb_cntl |= SDMA_RB_ENABLE;
241 q->is_active = true;
242 }
243
244 return 0;
245}
246
247static int destroy_mqd(struct mqd_manager *mm, void *mqd,
248 enum kfd_preempt_type type,
249 unsigned int timeout, uint32_t pipe_id,
250 uint32_t queue_id)
251{
252 return kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout,
253 pipe_id, queue_id);
254}
255
256/*
257 * preempt type here is ignored because there is only one way
258 * to preempt sdma queue
259 */
260static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
261 enum kfd_preempt_type type,
262 unsigned int timeout, uint32_t pipe_id,
263 uint32_t queue_id)
264{
265 return kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
266}
267
268static bool is_occupied(struct mqd_manager *mm, void *mqd,
269 uint64_t queue_address, uint32_t pipe_id,
270 uint32_t queue_id)
271{
272
273 return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address,
274 pipe_id, queue_id);
275
276}
277
278static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
279 uint64_t queue_address, uint32_t pipe_id,
280 uint32_t queue_id)
281{
282 return kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
283}
284
285/*
286 * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation.
287 * The HIQ queue in Kaveri is using the same MQD structure as all the user mode
288 * queues but with different initial values.
289 */
290
291static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
292 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
293 struct queue_properties *q)
294{
295 uint64_t addr;
296 struct cik_mqd *m;
297 int retval;
298
299 BUG_ON(!mm || !q || !mqd || !mqd_mem_obj);
300
301 pr_debug("kfd: In func %s\n", __func__);
302
303 retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd),
304 mqd_mem_obj);
305
306 if (retval != 0)
307 return -ENOMEM;
308
309 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr;
310 addr = (*mqd_mem_obj)->gpu_addr;
311
312 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
313
314 m->header = 0xC0310800;
315 m->compute_pipelinestat_enable = 1;
316 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
317 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
318 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
319 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
320
321 m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE |
322 PRELOAD_REQ;
323 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
324 QUANTUM_DURATION(10);
325
326 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
327 m->cp_mqd_base_addr_lo = lower_32_bits(addr);
328 m->cp_mqd_base_addr_hi = upper_32_bits(addr);
329
330 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE;
331
332 /*
333 * Pipe Priority
334 * Identifies the pipe relative priority when this queue is connected
335 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
336 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
337 * 0 = CS_LOW (typically below GFX)
338 * 1 = CS_MEDIUM (typically between HP3D and GFX
339 * 2 = CS_HIGH (typically above HP3D)
340 */
341 m->cp_hqd_pipe_priority = 1;
342 m->cp_hqd_queue_priority = 15;
343
344 *mqd = m;
345 if (gart_addr)
346 *gart_addr = addr;
347 retval = mm->update_mqd(mm, m, q);
348
349 return retval;
350}
351
352static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
353 struct queue_properties *q)
354{
355 struct cik_mqd *m;
356
357 BUG_ON(!mm || !q || !mqd);
358
359 pr_debug("kfd: In func %s\n", __func__);
360
361 m = get_mqd(mqd);
362 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
363 DEFAULT_MIN_AVAIL_SIZE |
364 PRIV_STATE |
365 KMD_QUEUE;
366
367 /*
368 * Calculating queue size which is log base 2 of actual queue
369 * size -1 dwords
370 */
371 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
372 - 1 - 1;
373 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
374 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
375 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
376 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
377 m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
378 DOORBELL_OFFSET(q->doorbell_off);
379
380 m->cp_hqd_vmid = q->vmid;
381
382 m->cp_hqd_active = 0;
383 q->is_active = false;
384 if (q->queue_size > 0 &&
385 q->queue_address != 0 &&
386 q->queue_percent > 0) {
387 m->cp_hqd_active = 1;
388 q->is_active = true;
389 }
390
391 return 0;
392}
393
394/*
395 * SDMA MQD Implementation
396 */
397
398struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
399{
400 struct cik_sdma_rlc_registers *m;
401
402 BUG_ON(!mqd);
403
404 m = (struct cik_sdma_rlc_registers *)mqd;
405
406 return m;
407}
408 25
409struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, 26struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
410 struct kfd_dev *dev) 27 struct kfd_dev *dev)
411{ 28{
412 struct mqd_manager *mqd; 29 switch (dev->device_info->asic_family) {
413 30 case CHIP_KAVERI:
414 BUG_ON(!dev); 31 return mqd_manager_init_cik(type, dev);
415 BUG_ON(type >= KFD_MQD_TYPE_MAX); 32 case CHIP_CARRIZO:
416 33 return mqd_manager_init_vi(type, dev);
417 pr_debug("kfd: In func %s\n", __func__);
418
419 mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL);
420 if (!mqd)
421 return NULL;
422
423 mqd->dev = dev;
424
425 switch (type) {
426 case KFD_MQD_TYPE_CP:
427 case KFD_MQD_TYPE_COMPUTE:
428 mqd->init_mqd = init_mqd;
429 mqd->uninit_mqd = uninit_mqd;
430 mqd->load_mqd = load_mqd;
431 mqd->update_mqd = update_mqd;
432 mqd->destroy_mqd = destroy_mqd;
433 mqd->is_occupied = is_occupied;
434 break;
435 case KFD_MQD_TYPE_HIQ:
436 mqd->init_mqd = init_mqd_hiq;
437 mqd->uninit_mqd = uninit_mqd;
438 mqd->load_mqd = load_mqd;
439 mqd->update_mqd = update_mqd_hiq;
440 mqd->destroy_mqd = destroy_mqd;
441 mqd->is_occupied = is_occupied;
442 break;
443 case KFD_MQD_TYPE_SDMA:
444 mqd->init_mqd = init_mqd_sdma;
445 mqd->uninit_mqd = uninit_mqd_sdma;
446 mqd->load_mqd = load_mqd_sdma;
447 mqd->update_mqd = update_mqd_sdma;
448 mqd->destroy_mqd = destroy_mqd_sdma;
449 mqd->is_occupied = is_occupied_sdma;
450 break;
451 default:
452 kfree(mqd);
453 return NULL;
454 } 34 }
455 35
456 return mqd; 36 return NULL;
457} 37}
458
459/* SDMA queues should be implemented here when the cp will supports them */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
new file mode 100644
index 000000000000..7b28f6e74c32
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -0,0 +1,454 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/printk.h>
25#include <linux/slab.h>
26#include "kfd_priv.h"
27#include "kfd_mqd_manager.h"
28#include "cik_regs.h"
29#include "cik_structs.h"
30
31inline void busy_wait(unsigned long ms)
32{
33 while (time_before(jiffies, ms))
34 cpu_relax();
35}
36
37static inline struct cik_mqd *get_mqd(void *mqd)
38{
39 return (struct cik_mqd *)mqd;
40}
41
42static int init_mqd(struct mqd_manager *mm, void **mqd,
43 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
44 struct queue_properties *q)
45{
46 uint64_t addr;
47 struct cik_mqd *m;
48 int retval;
49
50 BUG_ON(!mm || !q || !mqd);
51
52 pr_debug("kfd: In func %s\n", __func__);
53
54 retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd),
55 mqd_mem_obj);
56
57 if (retval != 0)
58 return -ENOMEM;
59
60 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr;
61 addr = (*mqd_mem_obj)->gpu_addr;
62
63 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
64
65 m->header = 0xC0310800;
66 m->compute_pipelinestat_enable = 1;
67 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
68 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
69 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
70 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
71
72 /*
73 * Make sure to use the last queue state saved on mqd when the cp
74 * reassigns the queue, so when queue is switched on/off (e.g over
75 * subscription or quantum timeout) the context will be consistent
76 */
77 m->cp_hqd_persistent_state =
78 DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ;
79
80 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
81 m->cp_mqd_base_addr_lo = lower_32_bits(addr);
82 m->cp_mqd_base_addr_hi = upper_32_bits(addr);
83
84 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN;
85 /* Although WinKFD writes this, I suspect it should not be necessary */
86 m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE;
87
88 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
89 QUANTUM_DURATION(10);
90
91 /*
92 * Pipe Priority
93 * Identifies the pipe relative priority when this queue is connected
94 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
95 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
96 * 0 = CS_LOW (typically below GFX)
97 * 1 = CS_MEDIUM (typically between HP3D and GFX
98 * 2 = CS_HIGH (typically above HP3D)
99 */
100 m->cp_hqd_pipe_priority = 1;
101 m->cp_hqd_queue_priority = 15;
102
103 *mqd = m;
104 if (gart_addr != NULL)
105 *gart_addr = addr;
106 retval = mm->update_mqd(mm, m, q);
107
108 return retval;
109}
110
111static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
112 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
113 struct queue_properties *q)
114{
115 int retval;
116 struct cik_sdma_rlc_registers *m;
117
118 BUG_ON(!mm || !mqd || !mqd_mem_obj);
119
120 retval = kfd_gtt_sa_allocate(mm->dev,
121 sizeof(struct cik_sdma_rlc_registers),
122 mqd_mem_obj);
123
124 if (retval != 0)
125 return -ENOMEM;
126
127 m = (struct cik_sdma_rlc_registers *) (*mqd_mem_obj)->cpu_ptr;
128
129 memset(m, 0, sizeof(struct cik_sdma_rlc_registers));
130
131 *mqd = m;
132 if (gart_addr != NULL)
133 *gart_addr = (*mqd_mem_obj)->gpu_addr;
134
135 retval = mm->update_mqd(mm, m, q);
136
137 return retval;
138}
139
140static void uninit_mqd(struct mqd_manager *mm, void *mqd,
141 struct kfd_mem_obj *mqd_mem_obj)
142{
143 BUG_ON(!mm || !mqd);
144 kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
145}
146
147static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
148 struct kfd_mem_obj *mqd_mem_obj)
149{
150 BUG_ON(!mm || !mqd);
151 kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
152}
153
154static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
155 uint32_t queue_id, uint32_t __user *wptr)
156{
157 return kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
158}
159
160static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
161 uint32_t pipe_id, uint32_t queue_id,
162 uint32_t __user *wptr)
163{
164 return kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd);
165}
166
167static int update_mqd(struct mqd_manager *mm, void *mqd,
168 struct queue_properties *q)
169{
170 struct cik_mqd *m;
171
172 BUG_ON(!mm || !q || !mqd);
173
174 pr_debug("kfd: In func %s\n", __func__);
175
176 m = get_mqd(mqd);
177 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
178 DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN;
179
180 /*
181 * Calculating queue size which is log base 2 of actual queue size -1
182 * dwords and another -1 for ffs
183 */
184 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
185 - 1 - 1;
186 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
187 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
188 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
189 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
190 m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
191 DOORBELL_OFFSET(q->doorbell_off);
192
193 m->cp_hqd_vmid = q->vmid;
194
195 if (q->format == KFD_QUEUE_FORMAT_AQL) {
196 m->cp_hqd_iq_rptr = AQL_ENABLE;
197 m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
198 }
199
200 m->cp_hqd_active = 0;
201 q->is_active = false;
202 if (q->queue_size > 0 &&
203 q->queue_address != 0 &&
204 q->queue_percent > 0) {
205 m->cp_hqd_active = 1;
206 q->is_active = true;
207 }
208
209 return 0;
210}
211
212static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
213 struct queue_properties *q)
214{
215 struct cik_sdma_rlc_registers *m;
216
217 BUG_ON(!mm || !mqd || !q);
218
219 m = get_sdma_mqd(mqd);
220 m->sdma_rlc_rb_cntl =
221 SDMA_RB_SIZE((ffs(q->queue_size / sizeof(unsigned int)))) |
222 SDMA_RB_VMID(q->vmid) |
223 SDMA_RPTR_WRITEBACK_ENABLE |
224 SDMA_RPTR_WRITEBACK_TIMER(6);
225
226 m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8);
227 m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8);
228 m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
229 m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
230 m->sdma_rlc_doorbell = SDMA_OFFSET(q->doorbell_off) | SDMA_DB_ENABLE;
231 m->sdma_rlc_virtual_addr = q->sdma_vm_addr;
232
233 m->sdma_engine_id = q->sdma_engine_id;
234 m->sdma_queue_id = q->sdma_queue_id;
235
236 q->is_active = false;
237 if (q->queue_size > 0 &&
238 q->queue_address != 0 &&
239 q->queue_percent > 0) {
240 m->sdma_rlc_rb_cntl |= SDMA_RB_ENABLE;
241 q->is_active = true;
242 }
243
244 return 0;
245}
246
247static int destroy_mqd(struct mqd_manager *mm, void *mqd,
248 enum kfd_preempt_type type,
249 unsigned int timeout, uint32_t pipe_id,
250 uint32_t queue_id)
251{
252 return kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout,
253 pipe_id, queue_id);
254}
255
256/*
257 * preempt type here is ignored because there is only one way
258 * to preempt sdma queue
259 */
260static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
261 enum kfd_preempt_type type,
262 unsigned int timeout, uint32_t pipe_id,
263 uint32_t queue_id)
264{
265 return kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
266}
267
268static bool is_occupied(struct mqd_manager *mm, void *mqd,
269 uint64_t queue_address, uint32_t pipe_id,
270 uint32_t queue_id)
271{
272
273 return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address,
274 pipe_id, queue_id);
275
276}
277
278static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
279 uint64_t queue_address, uint32_t pipe_id,
280 uint32_t queue_id)
281{
282 return kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
283}
284
285/*
286 * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation.
287 * The HIQ queue in Kaveri is using the same MQD structure as all the user mode
288 * queues but with different initial values.
289 */
290
291static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
292 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
293 struct queue_properties *q)
294{
295 uint64_t addr;
296 struct cik_mqd *m;
297 int retval;
298
299 BUG_ON(!mm || !q || !mqd || !mqd_mem_obj);
300
301 pr_debug("kfd: In func %s\n", __func__);
302
303 retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd),
304 mqd_mem_obj);
305
306 if (retval != 0)
307 return -ENOMEM;
308
309 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr;
310 addr = (*mqd_mem_obj)->gpu_addr;
311
312 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
313
314 m->header = 0xC0310800;
315 m->compute_pipelinestat_enable = 1;
316 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
317 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
318 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
319 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
320
321 m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE |
322 PRELOAD_REQ;
323 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
324 QUANTUM_DURATION(10);
325
326 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN;
327 m->cp_mqd_base_addr_lo = lower_32_bits(addr);
328 m->cp_mqd_base_addr_hi = upper_32_bits(addr);
329
330 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE;
331
332 /*
333 * Pipe Priority
334 * Identifies the pipe relative priority when this queue is connected
335 * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
336 * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
337 * 0 = CS_LOW (typically below GFX)
338 * 1 = CS_MEDIUM (typically between HP3D and GFX
339 * 2 = CS_HIGH (typically above HP3D)
340 */
341 m->cp_hqd_pipe_priority = 1;
342 m->cp_hqd_queue_priority = 15;
343
344 *mqd = m;
345 if (gart_addr)
346 *gart_addr = addr;
347 retval = mm->update_mqd(mm, m, q);
348
349 return retval;
350}
351
352static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
353 struct queue_properties *q)
354{
355 struct cik_mqd *m;
356
357 BUG_ON(!mm || !q || !mqd);
358
359 pr_debug("kfd: In func %s\n", __func__);
360
361 m = get_mqd(mqd);
362 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
363 DEFAULT_MIN_AVAIL_SIZE |
364 PRIV_STATE |
365 KMD_QUEUE;
366
367 /*
368 * Calculating queue size which is log base 2 of actual queue
369 * size -1 dwords
370 */
371 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
372 - 1 - 1;
373 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
374 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
375 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
376 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
377 m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
378 DOORBELL_OFFSET(q->doorbell_off);
379
380 m->cp_hqd_vmid = q->vmid;
381
382 m->cp_hqd_active = 0;
383 q->is_active = false;
384 if (q->queue_size > 0 &&
385 q->queue_address != 0 &&
386 q->queue_percent > 0) {
387 m->cp_hqd_active = 1;
388 q->is_active = true;
389 }
390
391 return 0;
392}
393
394struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
395{
396 struct cik_sdma_rlc_registers *m;
397
398 BUG_ON(!mqd);
399
400 m = (struct cik_sdma_rlc_registers *)mqd;
401
402 return m;
403}
404
405struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
406 struct kfd_dev *dev)
407{
408 struct mqd_manager *mqd;
409
410 BUG_ON(!dev);
411 BUG_ON(type >= KFD_MQD_TYPE_MAX);
412
413 pr_debug("kfd: In func %s\n", __func__);
414
415 mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL);
416 if (!mqd)
417 return NULL;
418
419 mqd->dev = dev;
420
421 switch (type) {
422 case KFD_MQD_TYPE_CP:
423 case KFD_MQD_TYPE_COMPUTE:
424 mqd->init_mqd = init_mqd;
425 mqd->uninit_mqd = uninit_mqd;
426 mqd->load_mqd = load_mqd;
427 mqd->update_mqd = update_mqd;
428 mqd->destroy_mqd = destroy_mqd;
429 mqd->is_occupied = is_occupied;
430 break;
431 case KFD_MQD_TYPE_HIQ:
432 mqd->init_mqd = init_mqd_hiq;
433 mqd->uninit_mqd = uninit_mqd;
434 mqd->load_mqd = load_mqd;
435 mqd->update_mqd = update_mqd_hiq;
436 mqd->destroy_mqd = destroy_mqd;
437 mqd->is_occupied = is_occupied;
438 break;
439 case KFD_MQD_TYPE_SDMA:
440 mqd->init_mqd = init_mqd_sdma;
441 mqd->uninit_mqd = uninit_mqd_sdma;
442 mqd->load_mqd = load_mqd_sdma;
443 mqd->update_mqd = update_mqd_sdma;
444 mqd->destroy_mqd = destroy_mqd_sdma;
445 mqd->is_occupied = is_occupied_sdma;
446 break;
447 default:
448 kfree(mqd);
449 return NULL;
450 }
451
452 return mqd;
453}
454
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
new file mode 100644
index 000000000000..b3a7e3ba1e38
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -0,0 +1,33 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/printk.h>
25#include "kfd_priv.h"
26#include "kfd_mqd_manager.h"
27
28struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
29 struct kfd_dev *dev)
30{
31 pr_warn("amdkfd: VI MQD is not currently supported\n");
32 return NULL;
33}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 872a1da4c02e..bfcf45f30b76 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -573,6 +573,10 @@ void print_queue(struct queue *q);
573 573
574struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, 574struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
575 struct kfd_dev *dev); 575 struct kfd_dev *dev);
576struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
577 struct kfd_dev *dev);
578struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
579 struct kfd_dev *dev);
576struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev); 580struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
577void device_queue_manager_uninit(struct device_queue_manager *dqm); 581void device_queue_manager_uninit(struct device_queue_manager *dqm);
578struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, 582struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
generated by cgit v1.2.2 (git 2.25.0) at 2026-05-29 17:35:02 -0400