aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/amd/amdkfd
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd')
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_chardev.c316
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c28
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c2
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_pasid.c2
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_priv.h18
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_topology.c2
6 files changed, 215 insertions, 153 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 7d4974b83af7..fcfdf23e1913 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -31,7 +31,6 @@
31#include <uapi/linux/kfd_ioctl.h> 31#include <uapi/linux/kfd_ioctl.h>
32#include <linux/time.h> 32#include <linux/time.h>
33#include <linux/mm.h> 33#include <linux/mm.h>
34#include <linux/uaccess.h>
35#include <uapi/asm-generic/mman-common.h> 34#include <uapi/asm-generic/mman-common.h>
36#include <asm/processor.h> 35#include <asm/processor.h>
37#include "kfd_priv.h" 36#include "kfd_priv.h"
@@ -127,17 +126,14 @@ static int kfd_open(struct inode *inode, struct file *filep)
127 return 0; 126 return 0;
128} 127}
129 128
130static long kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, 129static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
131 void __user *arg) 130 void *data)
132{ 131{
133 struct kfd_ioctl_get_version_args args; 132 struct kfd_ioctl_get_version_args *args = data;
134 int err = 0; 133 int err = 0;
135 134
136 args.major_version = KFD_IOCTL_MAJOR_VERSION; 135 args->major_version = KFD_IOCTL_MAJOR_VERSION;
137 args.minor_version = KFD_IOCTL_MINOR_VERSION; 136 args->minor_version = KFD_IOCTL_MINOR_VERSION;
138
139 if (copy_to_user(arg, &args, sizeof(args)))
140 err = -EFAULT;
141 137
142 return err; 138 return err;
143} 139}
@@ -221,10 +217,10 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
221 return 0; 217 return 0;
222} 218}
223 219
224static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 220static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
225 void __user *arg) 221 void *data)
226{ 222{
227 struct kfd_ioctl_create_queue_args args; 223 struct kfd_ioctl_create_queue_args *args = data;
228 struct kfd_dev *dev; 224 struct kfd_dev *dev;
229 int err = 0; 225 int err = 0;
230 unsigned int queue_id; 226 unsigned int queue_id;
@@ -233,16 +229,13 @@ static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
233 229
234 memset(&q_properties, 0, sizeof(struct queue_properties)); 230 memset(&q_properties, 0, sizeof(struct queue_properties));
235 231
236 if (copy_from_user(&args, arg, sizeof(args)))
237 return -EFAULT;
238
239 pr_debug("kfd: creating queue ioctl\n"); 232 pr_debug("kfd: creating queue ioctl\n");
240 233
241 err = set_queue_properties_from_user(&q_properties, &args); 234 err = set_queue_properties_from_user(&q_properties, args);
242 if (err) 235 if (err)
243 return err; 236 return err;
244 237
245 dev = kfd_device_by_id(args.gpu_id); 238 dev = kfd_device_by_id(args->gpu_id);
246 if (dev == NULL) 239 if (dev == NULL)
247 return -EINVAL; 240 return -EINVAL;
248 241
@@ -250,7 +243,7 @@ static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
250 243
251 pdd = kfd_bind_process_to_device(dev, p); 244 pdd = kfd_bind_process_to_device(dev, p);
252 if (IS_ERR(pdd)) { 245 if (IS_ERR(pdd)) {
253 err = PTR_ERR(pdd); 246 err = -ESRCH;
254 goto err_bind_process; 247 goto err_bind_process;
255 } 248 }
256 249
@@ -263,33 +256,26 @@ static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
263 if (err != 0) 256 if (err != 0)
264 goto err_create_queue; 257 goto err_create_queue;
265 258
266 args.queue_id = queue_id; 259 args->queue_id = queue_id;
267 260
268 /* Return gpu_id as doorbell offset for mmap usage */ 261 /* Return gpu_id as doorbell offset for mmap usage */
269 args.doorbell_offset = args.gpu_id << PAGE_SHIFT; 262 args->doorbell_offset = args->gpu_id << PAGE_SHIFT;
270
271 if (copy_to_user(arg, &args, sizeof(args))) {
272 err = -EFAULT;
273 goto err_copy_args_out;
274 }
275 263
276 mutex_unlock(&p->mutex); 264 mutex_unlock(&p->mutex);
277 265
278 pr_debug("kfd: queue id %d was created successfully\n", args.queue_id); 266 pr_debug("kfd: queue id %d was created successfully\n", args->queue_id);
279 267
280 pr_debug("ring buffer address == 0x%016llX\n", 268 pr_debug("ring buffer address == 0x%016llX\n",
281 args.ring_base_address); 269 args->ring_base_address);
282 270
283 pr_debug("read ptr address == 0x%016llX\n", 271 pr_debug("read ptr address == 0x%016llX\n",
284 args.read_pointer_address); 272 args->read_pointer_address);
285 273
286 pr_debug("write ptr address == 0x%016llX\n", 274 pr_debug("write ptr address == 0x%016llX\n",
287 args.write_pointer_address); 275 args->write_pointer_address);
288 276
289 return 0; 277 return 0;
290 278
291err_copy_args_out:
292 pqm_destroy_queue(&p->pqm, queue_id);
293err_create_queue: 279err_create_queue:
294err_bind_process: 280err_bind_process:
295 mutex_unlock(&p->mutex); 281 mutex_unlock(&p->mutex);
@@ -297,99 +283,90 @@ err_bind_process:
297} 283}
298 284
299static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, 285static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
300 void __user *arg) 286 void *data)
301{ 287{
302 int retval; 288 int retval;
303 struct kfd_ioctl_destroy_queue_args args; 289 struct kfd_ioctl_destroy_queue_args *args = data;
304
305 if (copy_from_user(&args, arg, sizeof(args)))
306 return -EFAULT;
307 290
308 pr_debug("kfd: destroying queue id %d for PASID %d\n", 291 pr_debug("kfd: destroying queue id %d for PASID %d\n",
309 args.queue_id, 292 args->queue_id,
310 p->pasid); 293 p->pasid);
311 294
312 mutex_lock(&p->mutex); 295 mutex_lock(&p->mutex);
313 296
314 retval = pqm_destroy_queue(&p->pqm, args.queue_id); 297 retval = pqm_destroy_queue(&p->pqm, args->queue_id);
315 298
316 mutex_unlock(&p->mutex); 299 mutex_unlock(&p->mutex);
317 return retval; 300 return retval;
318} 301}
319 302
320static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, 303static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
321 void __user *arg) 304 void *data)
322{ 305{
323 int retval; 306 int retval;
324 struct kfd_ioctl_update_queue_args args; 307 struct kfd_ioctl_update_queue_args *args = data;
325 struct queue_properties properties; 308 struct queue_properties properties;
326 309
327 if (copy_from_user(&args, arg, sizeof(args))) 310 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
328 return -EFAULT;
329
330 if (args.queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
331 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 311 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
332 return -EINVAL; 312 return -EINVAL;
333 } 313 }
334 314
335 if (args.queue_priority > KFD_MAX_QUEUE_PRIORITY) { 315 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
336 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 316 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
337 return -EINVAL; 317 return -EINVAL;
338 } 318 }
339 319
340 if ((args.ring_base_address) && 320 if ((args->ring_base_address) &&
341 (!access_ok(VERIFY_WRITE, 321 (!access_ok(VERIFY_WRITE,
342 (const void __user *) args.ring_base_address, 322 (const void __user *) args->ring_base_address,
343 sizeof(uint64_t)))) { 323 sizeof(uint64_t)))) {
344 pr_err("kfd: can't access ring base address\n"); 324 pr_err("kfd: can't access ring base address\n");
345 return -EFAULT; 325 return -EFAULT;
346 } 326 }
347 327
348 if (!is_power_of_2(args.ring_size) && (args.ring_size != 0)) { 328 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
349 pr_err("kfd: ring size must be a power of 2 or 0\n"); 329 pr_err("kfd: ring size must be a power of 2 or 0\n");
350 return -EINVAL; 330 return -EINVAL;
351 } 331 }
352 332
353 properties.queue_address = args.ring_base_address; 333 properties.queue_address = args->ring_base_address;
354 properties.queue_size = args.ring_size; 334 properties.queue_size = args->ring_size;
355 properties.queue_percent = args.queue_percentage; 335 properties.queue_percent = args->queue_percentage;
356 properties.priority = args.queue_priority; 336 properties.priority = args->queue_priority;
357 337
358 pr_debug("kfd: updating queue id %d for PASID %d\n", 338 pr_debug("kfd: updating queue id %d for PASID %d\n",
359 args.queue_id, p->pasid); 339 args->queue_id, p->pasid);
360 340
361 mutex_lock(&p->mutex); 341 mutex_lock(&p->mutex);
362 342
363 retval = pqm_update_queue(&p->pqm, args.queue_id, &properties); 343 retval = pqm_update_queue(&p->pqm, args->queue_id, &properties);
364 344
365 mutex_unlock(&p->mutex); 345 mutex_unlock(&p->mutex);
366 346
367 return retval; 347 return retval;
368} 348}
369 349
370static long kfd_ioctl_set_memory_policy(struct file *filep, 350static int kfd_ioctl_set_memory_policy(struct file *filep,
371 struct kfd_process *p, void __user *arg) 351 struct kfd_process *p, void *data)
372{ 352{
373 struct kfd_ioctl_set_memory_policy_args args; 353 struct kfd_ioctl_set_memory_policy_args *args = data;
374 struct kfd_dev *dev; 354 struct kfd_dev *dev;
375 int err = 0; 355 int err = 0;
376 struct kfd_process_device *pdd; 356 struct kfd_process_device *pdd;
377 enum cache_policy default_policy, alternate_policy; 357 enum cache_policy default_policy, alternate_policy;
378 358
379 if (copy_from_user(&args, arg, sizeof(args))) 359 if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
380 return -EFAULT; 360 && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
381
382 if (args.default_policy != KFD_IOC_CACHE_POLICY_COHERENT
383 && args.default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
384 return -EINVAL; 361 return -EINVAL;
385 } 362 }
386 363
387 if (args.alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT 364 if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
388 && args.alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 365 && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
389 return -EINVAL; 366 return -EINVAL;
390 } 367 }
391 368
392 dev = kfd_device_by_id(args.gpu_id); 369 dev = kfd_device_by_id(args->gpu_id);
393 if (dev == NULL) 370 if (dev == NULL)
394 return -EINVAL; 371 return -EINVAL;
395 372
@@ -397,23 +374,23 @@ static long kfd_ioctl_set_memory_policy(struct file *filep,
397 374
398 pdd = kfd_bind_process_to_device(dev, p); 375 pdd = kfd_bind_process_to_device(dev, p);
399 if (IS_ERR(pdd)) { 376 if (IS_ERR(pdd)) {
400 err = PTR_ERR(pdd); 377 err = -ESRCH;
401 goto out; 378 goto out;
402 } 379 }
403 380
404 default_policy = (args.default_policy == KFD_IOC_CACHE_POLICY_COHERENT) 381 default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
405 ? cache_policy_coherent : cache_policy_noncoherent; 382 ? cache_policy_coherent : cache_policy_noncoherent;
406 383
407 alternate_policy = 384 alternate_policy =
408 (args.alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) 385 (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
409 ? cache_policy_coherent : cache_policy_noncoherent; 386 ? cache_policy_coherent : cache_policy_noncoherent;
410 387
411 if (!dev->dqm->set_cache_memory_policy(dev->dqm, 388 if (!dev->dqm->set_cache_memory_policy(dev->dqm,
412 &pdd->qpd, 389 &pdd->qpd,
413 default_policy, 390 default_policy,
414 alternate_policy, 391 alternate_policy,
415 (void __user *)args.alternate_aperture_base, 392 (void __user *)args->alternate_aperture_base,
416 args.alternate_aperture_size)) 393 args->alternate_aperture_size))
417 err = -EINVAL; 394 err = -EINVAL;
418 395
419out: 396out:
@@ -422,53 +399,44 @@ out:
422 return err; 399 return err;
423} 400}
424 401
425static long kfd_ioctl_get_clock_counters(struct file *filep, 402static int kfd_ioctl_get_clock_counters(struct file *filep,
426 struct kfd_process *p, void __user *arg) 403 struct kfd_process *p, void *data)
427{ 404{
428 struct kfd_ioctl_get_clock_counters_args args; 405 struct kfd_ioctl_get_clock_counters_args *args = data;
429 struct kfd_dev *dev; 406 struct kfd_dev *dev;
430 struct timespec time; 407 struct timespec time;
431 408
432 if (copy_from_user(&args, arg, sizeof(args))) 409 dev = kfd_device_by_id(args->gpu_id);
433 return -EFAULT;
434
435 dev = kfd_device_by_id(args.gpu_id);
436 if (dev == NULL) 410 if (dev == NULL)
437 return -EINVAL; 411 return -EINVAL;
438 412
439 /* Reading GPU clock counter from KGD */ 413 /* Reading GPU clock counter from KGD */
440 args.gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd); 414 args->gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd);
441 415
442 /* No access to rdtsc. Using raw monotonic time */ 416 /* No access to rdtsc. Using raw monotonic time */
443 getrawmonotonic(&time); 417 getrawmonotonic(&time);
444 args.cpu_clock_counter = (uint64_t)timespec_to_ns(&time); 418 args->cpu_clock_counter = (uint64_t)timespec_to_ns(&time);
445 419
446 get_monotonic_boottime(&time); 420 get_monotonic_boottime(&time);
447 args.system_clock_counter = (uint64_t)timespec_to_ns(&time); 421 args->system_clock_counter = (uint64_t)timespec_to_ns(&time);
448 422
449 /* Since the counter is in nano-seconds we use 1GHz frequency */ 423 /* Since the counter is in nano-seconds we use 1GHz frequency */
450 args.system_clock_freq = 1000000000; 424 args->system_clock_freq = 1000000000;
451
452 if (copy_to_user(arg, &args, sizeof(args)))
453 return -EFAULT;
454 425
455 return 0; 426 return 0;
456} 427}
457 428
458 429
459static int kfd_ioctl_get_process_apertures(struct file *filp, 430static int kfd_ioctl_get_process_apertures(struct file *filp,
460 struct kfd_process *p, void __user *arg) 431 struct kfd_process *p, void *data)
461{ 432{
462 struct kfd_ioctl_get_process_apertures_args args; 433 struct kfd_ioctl_get_process_apertures_args *args = data;
463 struct kfd_process_device_apertures *pAperture; 434 struct kfd_process_device_apertures *pAperture;
464 struct kfd_process_device *pdd; 435 struct kfd_process_device *pdd;
465 436
466 dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid); 437 dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid);
467 438
468 if (copy_from_user(&args, arg, sizeof(args))) 439 args->num_of_nodes = 0;
469 return -EFAULT;
470
471 args.num_of_nodes = 0;
472 440
473 mutex_lock(&p->mutex); 441 mutex_lock(&p->mutex);
474 442
@@ -477,7 +445,8 @@ static int kfd_ioctl_get_process_apertures(struct file *filp,
477 /* Run over all pdd of the process */ 445 /* Run over all pdd of the process */
478 pdd = kfd_get_first_process_device_data(p); 446 pdd = kfd_get_first_process_device_data(p);
479 do { 447 do {
480 pAperture = &args.process_apertures[args.num_of_nodes]; 448 pAperture =
449 &args->process_apertures[args->num_of_nodes];
481 pAperture->gpu_id = pdd->dev->id; 450 pAperture->gpu_id = pdd->dev->id;
482 pAperture->lds_base = pdd->lds_base; 451 pAperture->lds_base = pdd->lds_base;
483 pAperture->lds_limit = pdd->lds_limit; 452 pAperture->lds_limit = pdd->lds_limit;
@@ -487,7 +456,7 @@ static int kfd_ioctl_get_process_apertures(struct file *filp,
487 pAperture->scratch_limit = pdd->scratch_limit; 456 pAperture->scratch_limit = pdd->scratch_limit;
488 457
489 dev_dbg(kfd_device, 458 dev_dbg(kfd_device,
490 "node id %u\n", args.num_of_nodes); 459 "node id %u\n", args->num_of_nodes);
491 dev_dbg(kfd_device, 460 dev_dbg(kfd_device,
492 "gpu id %u\n", pdd->dev->id); 461 "gpu id %u\n", pdd->dev->id);
493 dev_dbg(kfd_device, 462 dev_dbg(kfd_device,
@@ -503,80 +472,131 @@ static int kfd_ioctl_get_process_apertures(struct file *filp,
503 dev_dbg(kfd_device, 472 dev_dbg(kfd_device,
504 "scratch_limit %llX\n", pdd->scratch_limit); 473 "scratch_limit %llX\n", pdd->scratch_limit);
505 474
506 args.num_of_nodes++; 475 args->num_of_nodes++;
507 } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL && 476 } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL &&
508 (args.num_of_nodes < NUM_OF_SUPPORTED_GPUS)); 477 (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS));
509 } 478 }
510 479
511 mutex_unlock(&p->mutex); 480 mutex_unlock(&p->mutex);
512 481
513 if (copy_to_user(arg, &args, sizeof(args)))
514 return -EFAULT;
515
516 return 0; 482 return 0;
517} 483}
518 484
485#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
486 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl}
487
488/** Ioctl table */
489static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
490 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
491 kfd_ioctl_get_version, 0),
492
493 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
494 kfd_ioctl_create_queue, 0),
495
496 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
497 kfd_ioctl_destroy_queue, 0),
498
499 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
500 kfd_ioctl_set_memory_policy, 0),
501
502 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
503 kfd_ioctl_get_clock_counters, 0),
504
505 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
506 kfd_ioctl_get_process_apertures, 0),
507
508 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
509 kfd_ioctl_update_queue, 0),
510};
511
512#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
513
519static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) 514static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
520{ 515{
521 struct kfd_process *process; 516 struct kfd_process *process;
522 long err = -EINVAL; 517 amdkfd_ioctl_t *func;
518 const struct amdkfd_ioctl_desc *ioctl = NULL;
519 unsigned int nr = _IOC_NR(cmd);
520 char stack_kdata[128];
521 char *kdata = NULL;
522 unsigned int usize, asize;
523 int retcode = -EINVAL;
523 524
524 dev_dbg(kfd_device, 525 if (nr >= AMDKFD_CORE_IOCTL_COUNT)
525 "ioctl cmd 0x%x (#%d), arg 0x%lx\n", 526 goto err_i1;
526 cmd, _IOC_NR(cmd), arg); 527
528 if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
529 u32 amdkfd_size;
530
531 ioctl = &amdkfd_ioctls[nr];
532
533 amdkfd_size = _IOC_SIZE(ioctl->cmd);
534 usize = asize = _IOC_SIZE(cmd);
535 if (amdkfd_size > asize)
536 asize = amdkfd_size;
537
538 cmd = ioctl->cmd;
539 } else
540 goto err_i1;
541
542 dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg);
527 543
528 process = kfd_get_process(current); 544 process = kfd_get_process(current);
529 if (IS_ERR(process)) 545 if (IS_ERR(process)) {
530 return PTR_ERR(process); 546 dev_dbg(kfd_device, "no process\n");
547 goto err_i1;
548 }
531 549
532 switch (cmd) { 550 /* Do not trust userspace, use our own definition */
533 case KFD_IOC_GET_VERSION: 551 func = ioctl->func;
534 err = kfd_ioctl_get_version(filep, process, (void __user *)arg); 552
535 break; 553 if (unlikely(!func)) {
536 case KFD_IOC_CREATE_QUEUE: 554 dev_dbg(kfd_device, "no function\n");
537 err = kfd_ioctl_create_queue(filep, process, 555 retcode = -EINVAL;
538 (void __user *)arg); 556 goto err_i1;
539 break;
540
541 case KFD_IOC_DESTROY_QUEUE:
542 err = kfd_ioctl_destroy_queue(filep, process,
543 (void __user *)arg);
544 break;
545
546 case KFD_IOC_SET_MEMORY_POLICY:
547 err = kfd_ioctl_set_memory_policy(filep, process,
548 (void __user *)arg);
549 break;
550
551 case KFD_IOC_GET_CLOCK_COUNTERS:
552 err = kfd_ioctl_get_clock_counters(filep, process,
553 (void __user *)arg);
554 break;
555
556 case KFD_IOC_GET_PROCESS_APERTURES:
557 err = kfd_ioctl_get_process_apertures(filep, process,
558 (void __user *)arg);
559 break;
560
561 case KFD_IOC_UPDATE_QUEUE:
562 err = kfd_ioctl_update_queue(filep, process,
563 (void __user *)arg);
564 break;
565
566 default:
567 dev_err(kfd_device,
568 "unknown ioctl cmd 0x%x, arg 0x%lx)\n",
569 cmd, arg);
570 err = -EINVAL;
571 break;
572 } 557 }
573 558
574 if (err < 0) 559 if (cmd & (IOC_IN | IOC_OUT)) {
575 dev_err(kfd_device, 560 if (asize <= sizeof(stack_kdata)) {
576 "ioctl error %ld for ioctl cmd 0x%x (#%d)\n", 561 kdata = stack_kdata;
577 err, cmd, _IOC_NR(cmd)); 562 } else {
563 kdata = kmalloc(asize, GFP_KERNEL);
564 if (!kdata) {
565 retcode = -ENOMEM;
566 goto err_i1;
567 }
568 }
569 if (asize > usize)
570 memset(kdata + usize, 0, asize - usize);
571 }
578 572
579 return err; 573 if (cmd & IOC_IN) {
574 if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {
575 retcode = -EFAULT;
576 goto err_i1;
577 }
578 } else if (cmd & IOC_OUT) {
579 memset(kdata, 0, usize);
580 }
581
582 retcode = func(filep, process, kdata);
583
584 if (cmd & IOC_OUT)
585 if (copy_to_user((void __user *)arg, kdata, usize) != 0)
586 retcode = -EFAULT;
587
588err_i1:
589 if (!ioctl)
590 dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
591 task_pid_nr(current), cmd, nr);
592
593 if (kdata != stack_kdata)
594 kfree(kdata);
595
596 if (retcode)
597 dev_dbg(kfd_device, "ret = %d\n", retcode);
598
599 return retcode;
580} 600}
581 601
582static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) 602static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 924e90c072e5..9c8961d22360 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -161,6 +161,9 @@ static void deallocate_vmid(struct device_queue_manager *dqm,
161{ 161{
162 int bit = qpd->vmid - KFD_VMID_START_OFFSET; 162 int bit = qpd->vmid - KFD_VMID_START_OFFSET;
163 163
164 /* Release the vmid mapping */
165 set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
166
164 set_bit(bit, (unsigned long *)&dqm->vmid_bitmap); 167 set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
165 qpd->vmid = 0; 168 qpd->vmid = 0;
166 q->properties.vmid = 0; 169 q->properties.vmid = 0;
@@ -272,6 +275,18 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
272 return retval; 275 return retval;
273 } 276 }
274 277
278 pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n",
279 q->pipe,
280 q->queue);
281
282 retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
283 q->queue, q->properties.write_ptr);
284 if (retval != 0) {
285 deallocate_hqd(dqm, q);
286 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
287 return retval;
288 }
289
275 return 0; 290 return 0;
276} 291}
277 292
@@ -320,6 +335,7 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
320{ 335{
321 int retval; 336 int retval;
322 struct mqd_manager *mqd; 337 struct mqd_manager *mqd;
338 bool prev_active = false;
323 339
324 BUG_ON(!dqm || !q || !q->mqd); 340 BUG_ON(!dqm || !q || !q->mqd);
325 341
@@ -330,10 +346,18 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
330 return -ENOMEM; 346 return -ENOMEM;
331 } 347 }
332 348
333 retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
334 if (q->properties.is_active == true) 349 if (q->properties.is_active == true)
350 prev_active = true;
351
352 /*
353 *
354 * check active state vs. the previous state
355 * and modify counter accordingly
356 */
357 retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
358 if ((q->properties.is_active == true) && (prev_active == false))
335 dqm->queue_count++; 359 dqm->queue_count++;
336 else 360 else if ((q->properties.is_active == false) && (prev_active == true))
337 dqm->queue_count--; 361 dqm->queue_count--;
338 362
339 if (sched_policy != KFD_SCHED_POLICY_NO_HWS) 363 if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
index adc31474e786..4c3828cf45bf 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
@@ -184,7 +184,7 @@ static bool is_occupied(struct mqd_manager *mm, void *mqd,
184 uint32_t queue_id) 184 uint32_t queue_id)
185{ 185{
186 186
187 return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address, 187 return kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address,
188 pipe_id, queue_id); 188 pipe_id, queue_id);
189 189
190} 190}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c b/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
index 71699ad97d74..4c25ef504f79 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
@@ -32,7 +32,7 @@ int kfd_pasid_init(void)
32{ 32{
33 pasid_limit = max_num_of_processes; 33 pasid_limit = max_num_of_processes;
34 34
35 pasid_bitmap = kzalloc(BITS_TO_LONGS(pasid_limit), GFP_KERNEL); 35 pasid_bitmap = kcalloc(BITS_TO_LONGS(pasid_limit), sizeof(long), GFP_KERNEL);
36 if (!pasid_bitmap) 36 if (!pasid_bitmap)
37 return -ENOMEM; 37 return -ENOMEM;
38 38
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index f9fb81e3bb09..a5edb29507e3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -463,6 +463,24 @@ struct kfd_process {
463 bool is_32bit_user_mode; 463 bool is_32bit_user_mode;
464}; 464};
465 465
466/**
467 * Ioctl function type.
468 *
469 * \param filep pointer to file structure.
470 * \param p amdkfd process pointer.
471 * \param data pointer to arg that was copied from user.
472 */
473typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
474 void *data);
475
476struct amdkfd_ioctl_desc {
477 unsigned int cmd;
478 int flags;
479 amdkfd_ioctl_t *func;
480 unsigned int cmd_drv;
481 const char *name;
482};
483
466void kfd_process_create_wq(void); 484void kfd_process_create_wq(void);
467void kfd_process_destroy_wq(void); 485void kfd_process_destroy_wq(void);
468struct kfd_process *kfd_create_process(const struct task_struct *); 486struct kfd_process *kfd_create_process(const struct task_struct *);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index b11792d7e70e..cca1708fd811 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -921,7 +921,7 @@ static int kfd_build_sysfs_node_tree(void)
921 uint32_t i = 0; 921 uint32_t i = 0;
922 922
923 list_for_each_entry(dev, &topology_device_list, list) { 923 list_for_each_entry(dev, &topology_device_list, list) {
924 ret = kfd_build_sysfs_node_entry(dev, 0); 924 ret = kfd_build_sysfs_node_entry(dev, i);
925 if (ret < 0) 925 if (ret < 0)
926 return ret; 926 return ret;
927 i++; 927 i++;