diff options
Diffstat (limited to 'drivers/gpu/drm/amd')
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 316 | ||||
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 28 | ||||
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c | 2 | ||||
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_pasid.c | 2 | ||||
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 18 | ||||
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 2 | ||||
-rw-r--r-- | drivers/gpu/drm/amd/include/kgd_kfd_interface.h | 2 |
7 files changed, 216 insertions, 154 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 7d4974b83af7..fcfdf23e1913 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | |||
@@ -31,7 +31,6 @@ | |||
31 | #include <uapi/linux/kfd_ioctl.h> | 31 | #include <uapi/linux/kfd_ioctl.h> |
32 | #include <linux/time.h> | 32 | #include <linux/time.h> |
33 | #include <linux/mm.h> | 33 | #include <linux/mm.h> |
34 | #include <linux/uaccess.h> | ||
35 | #include <uapi/asm-generic/mman-common.h> | 34 | #include <uapi/asm-generic/mman-common.h> |
36 | #include <asm/processor.h> | 35 | #include <asm/processor.h> |
37 | #include "kfd_priv.h" | 36 | #include "kfd_priv.h" |
@@ -127,17 +126,14 @@ static int kfd_open(struct inode *inode, struct file *filep) | |||
127 | return 0; | 126 | return 0; |
128 | } | 127 | } |
129 | 128 | ||
130 | static long kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, | 129 | static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, |
131 | void __user *arg) | 130 | void *data) |
132 | { | 131 | { |
133 | struct kfd_ioctl_get_version_args args; | 132 | struct kfd_ioctl_get_version_args *args = data; |
134 | int err = 0; | 133 | int err = 0; |
135 | 134 | ||
136 | args.major_version = KFD_IOCTL_MAJOR_VERSION; | 135 | args->major_version = KFD_IOCTL_MAJOR_VERSION; |
137 | args.minor_version = KFD_IOCTL_MINOR_VERSION; | 136 | args->minor_version = KFD_IOCTL_MINOR_VERSION; |
138 | |||
139 | if (copy_to_user(arg, &args, sizeof(args))) | ||
140 | err = -EFAULT; | ||
141 | 137 | ||
142 | return err; | 138 | return err; |
143 | } | 139 | } |
@@ -221,10 +217,10 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties, | |||
221 | return 0; | 217 | return 0; |
222 | } | 218 | } |
223 | 219 | ||
224 | static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, | 220 | static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, |
225 | void __user *arg) | 221 | void *data) |
226 | { | 222 | { |
227 | struct kfd_ioctl_create_queue_args args; | 223 | struct kfd_ioctl_create_queue_args *args = data; |
228 | struct kfd_dev *dev; | 224 | struct kfd_dev *dev; |
229 | int err = 0; | 225 | int err = 0; |
230 | unsigned int queue_id; | 226 | unsigned int queue_id; |
@@ -233,16 +229,13 @@ static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, | |||
233 | 229 | ||
234 | memset(&q_properties, 0, sizeof(struct queue_properties)); | 230 | memset(&q_properties, 0, sizeof(struct queue_properties)); |
235 | 231 | ||
236 | if (copy_from_user(&args, arg, sizeof(args))) | ||
237 | return -EFAULT; | ||
238 | |||
239 | pr_debug("kfd: creating queue ioctl\n"); | 232 | pr_debug("kfd: creating queue ioctl\n"); |
240 | 233 | ||
241 | err = set_queue_properties_from_user(&q_properties, &args); | 234 | err = set_queue_properties_from_user(&q_properties, args); |
242 | if (err) | 235 | if (err) |
243 | return err; | 236 | return err; |
244 | 237 | ||
245 | dev = kfd_device_by_id(args.gpu_id); | 238 | dev = kfd_device_by_id(args->gpu_id); |
246 | if (dev == NULL) | 239 | if (dev == NULL) |
247 | return -EINVAL; | 240 | return -EINVAL; |
248 | 241 | ||
@@ -250,7 +243,7 @@ static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, | |||
250 | 243 | ||
251 | pdd = kfd_bind_process_to_device(dev, p); | 244 | pdd = kfd_bind_process_to_device(dev, p); |
252 | if (IS_ERR(pdd)) { | 245 | if (IS_ERR(pdd)) { |
253 | err = PTR_ERR(pdd); | 246 | err = -ESRCH; |
254 | goto err_bind_process; | 247 | goto err_bind_process; |
255 | } | 248 | } |
256 | 249 | ||
@@ -263,33 +256,26 @@ static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, | |||
263 | if (err != 0) | 256 | if (err != 0) |
264 | goto err_create_queue; | 257 | goto err_create_queue; |
265 | 258 | ||
266 | args.queue_id = queue_id; | 259 | args->queue_id = queue_id; |
267 | 260 | ||
268 | /* Return gpu_id as doorbell offset for mmap usage */ | 261 | /* Return gpu_id as doorbell offset for mmap usage */ |
269 | args.doorbell_offset = args.gpu_id << PAGE_SHIFT; | 262 | args->doorbell_offset = args->gpu_id << PAGE_SHIFT; |
270 | |||
271 | if (copy_to_user(arg, &args, sizeof(args))) { | ||
272 | err = -EFAULT; | ||
273 | goto err_copy_args_out; | ||
274 | } | ||
275 | 263 | ||
276 | mutex_unlock(&p->mutex); | 264 | mutex_unlock(&p->mutex); |
277 | 265 | ||
278 | pr_debug("kfd: queue id %d was created successfully\n", args.queue_id); | 266 | pr_debug("kfd: queue id %d was created successfully\n", args->queue_id); |
279 | 267 | ||
280 | pr_debug("ring buffer address == 0x%016llX\n", | 268 | pr_debug("ring buffer address == 0x%016llX\n", |
281 | args.ring_base_address); | 269 | args->ring_base_address); |
282 | 270 | ||
283 | pr_debug("read ptr address == 0x%016llX\n", | 271 | pr_debug("read ptr address == 0x%016llX\n", |
284 | args.read_pointer_address); | 272 | args->read_pointer_address); |
285 | 273 | ||
286 | pr_debug("write ptr address == 0x%016llX\n", | 274 | pr_debug("write ptr address == 0x%016llX\n", |
287 | args.write_pointer_address); | 275 | args->write_pointer_address); |
288 | 276 | ||
289 | return 0; | 277 | return 0; |
290 | 278 | ||
291 | err_copy_args_out: | ||
292 | pqm_destroy_queue(&p->pqm, queue_id); | ||
293 | err_create_queue: | 279 | err_create_queue: |
294 | err_bind_process: | 280 | err_bind_process: |
295 | mutex_unlock(&p->mutex); | 281 | mutex_unlock(&p->mutex); |
@@ -297,99 +283,90 @@ err_bind_process: | |||
297 | } | 283 | } |
298 | 284 | ||
299 | static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, | 285 | static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, |
300 | void __user *arg) | 286 | void *data) |
301 | { | 287 | { |
302 | int retval; | 288 | int retval; |
303 | struct kfd_ioctl_destroy_queue_args args; | 289 | struct kfd_ioctl_destroy_queue_args *args = data; |
304 | |||
305 | if (copy_from_user(&args, arg, sizeof(args))) | ||
306 | return -EFAULT; | ||
307 | 290 | ||
308 | pr_debug("kfd: destroying queue id %d for PASID %d\n", | 291 | pr_debug("kfd: destroying queue id %d for PASID %d\n", |
309 | args.queue_id, | 292 | args->queue_id, |
310 | p->pasid); | 293 | p->pasid); |
311 | 294 | ||
312 | mutex_lock(&p->mutex); | 295 | mutex_lock(&p->mutex); |
313 | 296 | ||
314 | retval = pqm_destroy_queue(&p->pqm, args.queue_id); | 297 | retval = pqm_destroy_queue(&p->pqm, args->queue_id); |
315 | 298 | ||
316 | mutex_unlock(&p->mutex); | 299 | mutex_unlock(&p->mutex); |
317 | return retval; | 300 | return retval; |
318 | } | 301 | } |
319 | 302 | ||
320 | static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, | 303 | static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, |
321 | void __user *arg) | 304 | void *data) |
322 | { | 305 | { |
323 | int retval; | 306 | int retval; |
324 | struct kfd_ioctl_update_queue_args args; | 307 | struct kfd_ioctl_update_queue_args *args = data; |
325 | struct queue_properties properties; | 308 | struct queue_properties properties; |
326 | 309 | ||
327 | if (copy_from_user(&args, arg, sizeof(args))) | 310 | if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { |
328 | return -EFAULT; | ||
329 | |||
330 | if (args.queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { | ||
331 | pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); | 311 | pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); |
332 | return -EINVAL; | 312 | return -EINVAL; |
333 | } | 313 | } |
334 | 314 | ||
335 | if (args.queue_priority > KFD_MAX_QUEUE_PRIORITY) { | 315 | if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { |
336 | pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); | 316 | pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); |
337 | return -EINVAL; | 317 | return -EINVAL; |
338 | } | 318 | } |
339 | 319 | ||
340 | if ((args.ring_base_address) && | 320 | if ((args->ring_base_address) && |
341 | (!access_ok(VERIFY_WRITE, | 321 | (!access_ok(VERIFY_WRITE, |
342 | (const void __user *) args.ring_base_address, | 322 | (const void __user *) args->ring_base_address, |
343 | sizeof(uint64_t)))) { | 323 | sizeof(uint64_t)))) { |
344 | pr_err("kfd: can't access ring base address\n"); | 324 | pr_err("kfd: can't access ring base address\n"); |
345 | return -EFAULT; | 325 | return -EFAULT; |
346 | } | 326 | } |
347 | 327 | ||
348 | if (!is_power_of_2(args.ring_size) && (args.ring_size != 0)) { | 328 | if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { |
349 | pr_err("kfd: ring size must be a power of 2 or 0\n"); | 329 | pr_err("kfd: ring size must be a power of 2 or 0\n"); |
350 | return -EINVAL; | 330 | return -EINVAL; |
351 | } | 331 | } |
352 | 332 | ||
353 | properties.queue_address = args.ring_base_address; | 333 | properties.queue_address = args->ring_base_address; |
354 | properties.queue_size = args.ring_size; | 334 | properties.queue_size = args->ring_size; |
355 | properties.queue_percent = args.queue_percentage; | 335 | properties.queue_percent = args->queue_percentage; |
356 | properties.priority = args.queue_priority; | 336 | properties.priority = args->queue_priority; |
357 | 337 | ||
358 | pr_debug("kfd: updating queue id %d for PASID %d\n", | 338 | pr_debug("kfd: updating queue id %d for PASID %d\n", |
359 | args.queue_id, p->pasid); | 339 | args->queue_id, p->pasid); |
360 | 340 | ||
361 | mutex_lock(&p->mutex); | 341 | mutex_lock(&p->mutex); |
362 | 342 | ||
363 | retval = pqm_update_queue(&p->pqm, args.queue_id, &properties); | 343 | retval = pqm_update_queue(&p->pqm, args->queue_id, &properties); |
364 | 344 | ||
365 | mutex_unlock(&p->mutex); | 345 | mutex_unlock(&p->mutex); |
366 | 346 | ||
367 | return retval; | 347 | return retval; |
368 | } | 348 | } |
369 | 349 | ||
370 | static long kfd_ioctl_set_memory_policy(struct file *filep, | 350 | static int kfd_ioctl_set_memory_policy(struct file *filep, |
371 | struct kfd_process *p, void __user *arg) | 351 | struct kfd_process *p, void *data) |
372 | { | 352 | { |
373 | struct kfd_ioctl_set_memory_policy_args args; | 353 | struct kfd_ioctl_set_memory_policy_args *args = data; |
374 | struct kfd_dev *dev; | 354 | struct kfd_dev *dev; |
375 | int err = 0; | 355 | int err = 0; |
376 | struct kfd_process_device *pdd; | 356 | struct kfd_process_device *pdd; |
377 | enum cache_policy default_policy, alternate_policy; | 357 | enum cache_policy default_policy, alternate_policy; |
378 | 358 | ||
379 | if (copy_from_user(&args, arg, sizeof(args))) | 359 | if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT |
380 | return -EFAULT; | 360 | && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { |
381 | |||
382 | if (args.default_policy != KFD_IOC_CACHE_POLICY_COHERENT | ||
383 | && args.default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { | ||
384 | return -EINVAL; | 361 | return -EINVAL; |
385 | } | 362 | } |
386 | 363 | ||
387 | if (args.alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT | 364 | if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT |
388 | && args.alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { | 365 | && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { |
389 | return -EINVAL; | 366 | return -EINVAL; |
390 | } | 367 | } |
391 | 368 | ||
392 | dev = kfd_device_by_id(args.gpu_id); | 369 | dev = kfd_device_by_id(args->gpu_id); |
393 | if (dev == NULL) | 370 | if (dev == NULL) |
394 | return -EINVAL; | 371 | return -EINVAL; |
395 | 372 | ||
@@ -397,23 +374,23 @@ static long kfd_ioctl_set_memory_policy(struct file *filep, | |||
397 | 374 | ||
398 | pdd = kfd_bind_process_to_device(dev, p); | 375 | pdd = kfd_bind_process_to_device(dev, p); |
399 | if (IS_ERR(pdd)) { | 376 | if (IS_ERR(pdd)) { |
400 | err = PTR_ERR(pdd); | 377 | err = -ESRCH; |
401 | goto out; | 378 | goto out; |
402 | } | 379 | } |
403 | 380 | ||
404 | default_policy = (args.default_policy == KFD_IOC_CACHE_POLICY_COHERENT) | 381 | default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT) |
405 | ? cache_policy_coherent : cache_policy_noncoherent; | 382 | ? cache_policy_coherent : cache_policy_noncoherent; |
406 | 383 | ||
407 | alternate_policy = | 384 | alternate_policy = |
408 | (args.alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) | 385 | (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) |
409 | ? cache_policy_coherent : cache_policy_noncoherent; | 386 | ? cache_policy_coherent : cache_policy_noncoherent; |
410 | 387 | ||
411 | if (!dev->dqm->set_cache_memory_policy(dev->dqm, | 388 | if (!dev->dqm->set_cache_memory_policy(dev->dqm, |
412 | &pdd->qpd, | 389 | &pdd->qpd, |
413 | default_policy, | 390 | default_policy, |
414 | alternate_policy, | 391 | alternate_policy, |
415 | (void __user *)args.alternate_aperture_base, | 392 | (void __user *)args->alternate_aperture_base, |
416 | args.alternate_aperture_size)) | 393 | args->alternate_aperture_size)) |
417 | err = -EINVAL; | 394 | err = -EINVAL; |
418 | 395 | ||
419 | out: | 396 | out: |
@@ -422,53 +399,44 @@ out: | |||
422 | return err; | 399 | return err; |
423 | } | 400 | } |
424 | 401 | ||
425 | static long kfd_ioctl_get_clock_counters(struct file *filep, | 402 | static int kfd_ioctl_get_clock_counters(struct file *filep, |
426 | struct kfd_process *p, void __user *arg) | 403 | struct kfd_process *p, void *data) |
427 | { | 404 | { |
428 | struct kfd_ioctl_get_clock_counters_args args; | 405 | struct kfd_ioctl_get_clock_counters_args *args = data; |
429 | struct kfd_dev *dev; | 406 | struct kfd_dev *dev; |
430 | struct timespec time; | 407 | struct timespec time; |
431 | 408 | ||
432 | if (copy_from_user(&args, arg, sizeof(args))) | 409 | dev = kfd_device_by_id(args->gpu_id); |
433 | return -EFAULT; | ||
434 | |||
435 | dev = kfd_device_by_id(args.gpu_id); | ||
436 | if (dev == NULL) | 410 | if (dev == NULL) |
437 | return -EINVAL; | 411 | return -EINVAL; |
438 | 412 | ||
439 | /* Reading GPU clock counter from KGD */ | 413 | /* Reading GPU clock counter from KGD */ |
440 | args.gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd); | 414 | args->gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd); |
441 | 415 | ||
442 | /* No access to rdtsc. Using raw monotonic time */ | 416 | /* No access to rdtsc. Using raw monotonic time */ |
443 | getrawmonotonic(&time); | 417 | getrawmonotonic(&time); |
444 | args.cpu_clock_counter = (uint64_t)timespec_to_ns(&time); | 418 | args->cpu_clock_counter = (uint64_t)timespec_to_ns(&time); |
445 | 419 | ||
446 | get_monotonic_boottime(&time); | 420 | get_monotonic_boottime(&time); |
447 | args.system_clock_counter = (uint64_t)timespec_to_ns(&time); | 421 | args->system_clock_counter = (uint64_t)timespec_to_ns(&time); |
448 | 422 | ||
449 | /* Since the counter is in nano-seconds we use 1GHz frequency */ | 423 | /* Since the counter is in nano-seconds we use 1GHz frequency */ |
450 | args.system_clock_freq = 1000000000; | 424 | args->system_clock_freq = 1000000000; |
451 | |||
452 | if (copy_to_user(arg, &args, sizeof(args))) | ||
453 | return -EFAULT; | ||
454 | 425 | ||
455 | return 0; | 426 | return 0; |
456 | } | 427 | } |
457 | 428 | ||
458 | 429 | ||
459 | static int kfd_ioctl_get_process_apertures(struct file *filp, | 430 | static int kfd_ioctl_get_process_apertures(struct file *filp, |
460 | struct kfd_process *p, void __user *arg) | 431 | struct kfd_process *p, void *data) |
461 | { | 432 | { |
462 | struct kfd_ioctl_get_process_apertures_args args; | 433 | struct kfd_ioctl_get_process_apertures_args *args = data; |
463 | struct kfd_process_device_apertures *pAperture; | 434 | struct kfd_process_device_apertures *pAperture; |
464 | struct kfd_process_device *pdd; | 435 | struct kfd_process_device *pdd; |
465 | 436 | ||
466 | dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid); | 437 | dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid); |
467 | 438 | ||
468 | if (copy_from_user(&args, arg, sizeof(args))) | 439 | args->num_of_nodes = 0; |
469 | return -EFAULT; | ||
470 | |||
471 | args.num_of_nodes = 0; | ||
472 | 440 | ||
473 | mutex_lock(&p->mutex); | 441 | mutex_lock(&p->mutex); |
474 | 442 | ||
@@ -477,7 +445,8 @@ static int kfd_ioctl_get_process_apertures(struct file *filp, | |||
477 | /* Run over all pdd of the process */ | 445 | /* Run over all pdd of the process */ |
478 | pdd = kfd_get_first_process_device_data(p); | 446 | pdd = kfd_get_first_process_device_data(p); |
479 | do { | 447 | do { |
480 | pAperture = &args.process_apertures[args.num_of_nodes]; | 448 | pAperture = |
449 | &args->process_apertures[args->num_of_nodes]; | ||
481 | pAperture->gpu_id = pdd->dev->id; | 450 | pAperture->gpu_id = pdd->dev->id; |
482 | pAperture->lds_base = pdd->lds_base; | 451 | pAperture->lds_base = pdd->lds_base; |
483 | pAperture->lds_limit = pdd->lds_limit; | 452 | pAperture->lds_limit = pdd->lds_limit; |
@@ -487,7 +456,7 @@ static int kfd_ioctl_get_process_apertures(struct file *filp, | |||
487 | pAperture->scratch_limit = pdd->scratch_limit; | 456 | pAperture->scratch_limit = pdd->scratch_limit; |
488 | 457 | ||
489 | dev_dbg(kfd_device, | 458 | dev_dbg(kfd_device, |
490 | "node id %u\n", args.num_of_nodes); | 459 | "node id %u\n", args->num_of_nodes); |
491 | dev_dbg(kfd_device, | 460 | dev_dbg(kfd_device, |
492 | "gpu id %u\n", pdd->dev->id); | 461 | "gpu id %u\n", pdd->dev->id); |
493 | dev_dbg(kfd_device, | 462 | dev_dbg(kfd_device, |
@@ -503,80 +472,131 @@ static int kfd_ioctl_get_process_apertures(struct file *filp, | |||
503 | dev_dbg(kfd_device, | 472 | dev_dbg(kfd_device, |
504 | "scratch_limit %llX\n", pdd->scratch_limit); | 473 | "scratch_limit %llX\n", pdd->scratch_limit); |
505 | 474 | ||
506 | args.num_of_nodes++; | 475 | args->num_of_nodes++; |
507 | } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL && | 476 | } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL && |
508 | (args.num_of_nodes < NUM_OF_SUPPORTED_GPUS)); | 477 | (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS)); |
509 | } | 478 | } |
510 | 479 | ||
511 | mutex_unlock(&p->mutex); | 480 | mutex_unlock(&p->mutex); |
512 | 481 | ||
513 | if (copy_to_user(arg, &args, sizeof(args))) | ||
514 | return -EFAULT; | ||
515 | |||
516 | return 0; | 482 | return 0; |
517 | } | 483 | } |
518 | 484 | ||
485 | #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ | ||
486 | [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl} | ||
487 | |||
488 | /** Ioctl table */ | ||
489 | static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { | ||
490 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION, | ||
491 | kfd_ioctl_get_version, 0), | ||
492 | |||
493 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE, | ||
494 | kfd_ioctl_create_queue, 0), | ||
495 | |||
496 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE, | ||
497 | kfd_ioctl_destroy_queue, 0), | ||
498 | |||
499 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY, | ||
500 | kfd_ioctl_set_memory_policy, 0), | ||
501 | |||
502 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS, | ||
503 | kfd_ioctl_get_clock_counters, 0), | ||
504 | |||
505 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES, | ||
506 | kfd_ioctl_get_process_apertures, 0), | ||
507 | |||
508 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE, | ||
509 | kfd_ioctl_update_queue, 0), | ||
510 | }; | ||
511 | |||
512 | #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) | ||
513 | |||
519 | static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) | 514 | static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) |
520 | { | 515 | { |
521 | struct kfd_process *process; | 516 | struct kfd_process *process; |
522 | long err = -EINVAL; | 517 | amdkfd_ioctl_t *func; |
518 | const struct amdkfd_ioctl_desc *ioctl = NULL; | ||
519 | unsigned int nr = _IOC_NR(cmd); | ||
520 | char stack_kdata[128]; | ||
521 | char *kdata = NULL; | ||
522 | unsigned int usize, asize; | ||
523 | int retcode = -EINVAL; | ||
523 | 524 | ||
524 | dev_dbg(kfd_device, | 525 | if (nr >= AMDKFD_CORE_IOCTL_COUNT) |
525 | "ioctl cmd 0x%x (#%d), arg 0x%lx\n", | 526 | goto err_i1; |
526 | cmd, _IOC_NR(cmd), arg); | 527 | |
528 | if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) { | ||
529 | u32 amdkfd_size; | ||
530 | |||
531 | ioctl = &amdkfd_ioctls[nr]; | ||
532 | |||
533 | amdkfd_size = _IOC_SIZE(ioctl->cmd); | ||
534 | usize = asize = _IOC_SIZE(cmd); | ||
535 | if (amdkfd_size > asize) | ||
536 | asize = amdkfd_size; | ||
537 | |||
538 | cmd = ioctl->cmd; | ||
539 | } else | ||
540 | goto err_i1; | ||
541 | |||
542 | dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg); | ||
527 | 543 | ||
528 | process = kfd_get_process(current); | 544 | process = kfd_get_process(current); |
529 | if (IS_ERR(process)) | 545 | if (IS_ERR(process)) { |
530 | return PTR_ERR(process); | 546 | dev_dbg(kfd_device, "no process\n"); |
547 | goto err_i1; | ||
548 | } | ||
531 | 549 | ||
532 | switch (cmd) { | 550 | /* Do not trust userspace, use our own definition */ |
533 | case KFD_IOC_GET_VERSION: | 551 | func = ioctl->func; |
534 | err = kfd_ioctl_get_version(filep, process, (void __user *)arg); | 552 | |
535 | break; | 553 | if (unlikely(!func)) { |
536 | case KFD_IOC_CREATE_QUEUE: | 554 | dev_dbg(kfd_device, "no function\n"); |
537 | err = kfd_ioctl_create_queue(filep, process, | 555 | retcode = -EINVAL; |
538 | (void __user *)arg); | 556 | goto err_i1; |
539 | break; | ||
540 | |||
541 | case KFD_IOC_DESTROY_QUEUE: | ||
542 | err = kfd_ioctl_destroy_queue(filep, process, | ||
543 | (void __user *)arg); | ||
544 | break; | ||
545 | |||
546 | case KFD_IOC_SET_MEMORY_POLICY: | ||
547 | err = kfd_ioctl_set_memory_policy(filep, process, | ||
548 | (void __user *)arg); | ||
549 | break; | ||
550 | |||
551 | case KFD_IOC_GET_CLOCK_COUNTERS: | ||
552 | err = kfd_ioctl_get_clock_counters(filep, process, | ||
553 | (void __user *)arg); | ||
554 | break; | ||
555 | |||
556 | case KFD_IOC_GET_PROCESS_APERTURES: | ||
557 | err = kfd_ioctl_get_process_apertures(filep, process, | ||
558 | (void __user *)arg); | ||
559 | break; | ||
560 | |||
561 | case KFD_IOC_UPDATE_QUEUE: | ||
562 | err = kfd_ioctl_update_queue(filep, process, | ||
563 | (void __user *)arg); | ||
564 | break; | ||
565 | |||
566 | default: | ||
567 | dev_err(kfd_device, | ||
568 | "unknown ioctl cmd 0x%x, arg 0x%lx)\n", | ||
569 | cmd, arg); | ||
570 | err = -EINVAL; | ||
571 | break; | ||
572 | } | 557 | } |
573 | 558 | ||
574 | if (err < 0) | 559 | if (cmd & (IOC_IN | IOC_OUT)) { |
575 | dev_err(kfd_device, | 560 | if (asize <= sizeof(stack_kdata)) { |
576 | "ioctl error %ld for ioctl cmd 0x%x (#%d)\n", | 561 | kdata = stack_kdata; |
577 | err, cmd, _IOC_NR(cmd)); | 562 | } else { |
563 | kdata = kmalloc(asize, GFP_KERNEL); | ||
564 | if (!kdata) { | ||
565 | retcode = -ENOMEM; | ||
566 | goto err_i1; | ||
567 | } | ||
568 | } | ||
569 | if (asize > usize) | ||
570 | memset(kdata + usize, 0, asize - usize); | ||
571 | } | ||
578 | 572 | ||
579 | return err; | 573 | if (cmd & IOC_IN) { |
574 | if (copy_from_user(kdata, (void __user *)arg, usize) != 0) { | ||
575 | retcode = -EFAULT; | ||
576 | goto err_i1; | ||
577 | } | ||
578 | } else if (cmd & IOC_OUT) { | ||
579 | memset(kdata, 0, usize); | ||
580 | } | ||
581 | |||
582 | retcode = func(filep, process, kdata); | ||
583 | |||
584 | if (cmd & IOC_OUT) | ||
585 | if (copy_to_user((void __user *)arg, kdata, usize) != 0) | ||
586 | retcode = -EFAULT; | ||
587 | |||
588 | err_i1: | ||
589 | if (!ioctl) | ||
590 | dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n", | ||
591 | task_pid_nr(current), cmd, nr); | ||
592 | |||
593 | if (kdata != stack_kdata) | ||
594 | kfree(kdata); | ||
595 | |||
596 | if (retcode) | ||
597 | dev_dbg(kfd_device, "ret = %d\n", retcode); | ||
598 | |||
599 | return retcode; | ||
580 | } | 600 | } |
581 | 601 | ||
582 | static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) | 602 | static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 924e90c072e5..9c8961d22360 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | |||
@@ -161,6 +161,9 @@ static void deallocate_vmid(struct device_queue_manager *dqm, | |||
161 | { | 161 | { |
162 | int bit = qpd->vmid - KFD_VMID_START_OFFSET; | 162 | int bit = qpd->vmid - KFD_VMID_START_OFFSET; |
163 | 163 | ||
164 | /* Release the vmid mapping */ | ||
165 | set_pasid_vmid_mapping(dqm, 0, qpd->vmid); | ||
166 | |||
164 | set_bit(bit, (unsigned long *)&dqm->vmid_bitmap); | 167 | set_bit(bit, (unsigned long *)&dqm->vmid_bitmap); |
165 | qpd->vmid = 0; | 168 | qpd->vmid = 0; |
166 | q->properties.vmid = 0; | 169 | q->properties.vmid = 0; |
@@ -272,6 +275,18 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, | |||
272 | return retval; | 275 | return retval; |
273 | } | 276 | } |
274 | 277 | ||
278 | pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n", | ||
279 | q->pipe, | ||
280 | q->queue); | ||
281 | |||
282 | retval = mqd->load_mqd(mqd, q->mqd, q->pipe, | ||
283 | q->queue, q->properties.write_ptr); | ||
284 | if (retval != 0) { | ||
285 | deallocate_hqd(dqm, q); | ||
286 | mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); | ||
287 | return retval; | ||
288 | } | ||
289 | |||
275 | return 0; | 290 | return 0; |
276 | } | 291 | } |
277 | 292 | ||
@@ -320,6 +335,7 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q) | |||
320 | { | 335 | { |
321 | int retval; | 336 | int retval; |
322 | struct mqd_manager *mqd; | 337 | struct mqd_manager *mqd; |
338 | bool prev_active = false; | ||
323 | 339 | ||
324 | BUG_ON(!dqm || !q || !q->mqd); | 340 | BUG_ON(!dqm || !q || !q->mqd); |
325 | 341 | ||
@@ -330,10 +346,18 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q) | |||
330 | return -ENOMEM; | 346 | return -ENOMEM; |
331 | } | 347 | } |
332 | 348 | ||
333 | retval = mqd->update_mqd(mqd, q->mqd, &q->properties); | ||
334 | if (q->properties.is_active == true) | 349 | if (q->properties.is_active == true) |
350 | prev_active = true; | ||
351 | |||
352 | /* | ||
353 | * | ||
354 | * check active state vs. the previous state | ||
355 | * and modify counter accordingly | ||
356 | */ | ||
357 | retval = mqd->update_mqd(mqd, q->mqd, &q->properties); | ||
358 | if ((q->properties.is_active == true) && (prev_active == false)) | ||
335 | dqm->queue_count++; | 359 | dqm->queue_count++; |
336 | else | 360 | else if ((q->properties.is_active == false) && (prev_active == true)) |
337 | dqm->queue_count--; | 361 | dqm->queue_count--; |
338 | 362 | ||
339 | if (sched_policy != KFD_SCHED_POLICY_NO_HWS) | 363 | if (sched_policy != KFD_SCHED_POLICY_NO_HWS) |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c index adc31474e786..4c3828cf45bf 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c | |||
@@ -184,7 +184,7 @@ static bool is_occupied(struct mqd_manager *mm, void *mqd, | |||
184 | uint32_t queue_id) | 184 | uint32_t queue_id) |
185 | { | 185 | { |
186 | 186 | ||
187 | return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address, | 187 | return kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address, |
188 | pipe_id, queue_id); | 188 | pipe_id, queue_id); |
189 | 189 | ||
190 | } | 190 | } |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c b/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c index 71699ad97d74..4c25ef504f79 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c | |||
@@ -32,7 +32,7 @@ int kfd_pasid_init(void) | |||
32 | { | 32 | { |
33 | pasid_limit = max_num_of_processes; | 33 | pasid_limit = max_num_of_processes; |
34 | 34 | ||
35 | pasid_bitmap = kzalloc(BITS_TO_LONGS(pasid_limit), GFP_KERNEL); | 35 | pasid_bitmap = kcalloc(BITS_TO_LONGS(pasid_limit), sizeof(long), GFP_KERNEL); |
36 | if (!pasid_bitmap) | 36 | if (!pasid_bitmap) |
37 | return -ENOMEM; | 37 | return -ENOMEM; |
38 | 38 | ||
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index f9fb81e3bb09..a5edb29507e3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h | |||
@@ -463,6 +463,24 @@ struct kfd_process { | |||
463 | bool is_32bit_user_mode; | 463 | bool is_32bit_user_mode; |
464 | }; | 464 | }; |
465 | 465 | ||
466 | /** | ||
467 | * Ioctl function type. | ||
468 | * | ||
469 | * \param filep pointer to file structure. | ||
470 | * \param p amdkfd process pointer. | ||
471 | * \param data pointer to arg that was copied from user. | ||
472 | */ | ||
473 | typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p, | ||
474 | void *data); | ||
475 | |||
476 | struct amdkfd_ioctl_desc { | ||
477 | unsigned int cmd; | ||
478 | int flags; | ||
479 | amdkfd_ioctl_t *func; | ||
480 | unsigned int cmd_drv; | ||
481 | const char *name; | ||
482 | }; | ||
483 | |||
466 | void kfd_process_create_wq(void); | 484 | void kfd_process_create_wq(void); |
467 | void kfd_process_destroy_wq(void); | 485 | void kfd_process_destroy_wq(void); |
468 | struct kfd_process *kfd_create_process(const struct task_struct *); | 486 | struct kfd_process *kfd_create_process(const struct task_struct *); |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index b11792d7e70e..cca1708fd811 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c | |||
@@ -921,7 +921,7 @@ static int kfd_build_sysfs_node_tree(void) | |||
921 | uint32_t i = 0; | 921 | uint32_t i = 0; |
922 | 922 | ||
923 | list_for_each_entry(dev, &topology_device_list, list) { | 923 | list_for_each_entry(dev, &topology_device_list, list) { |
924 | ret = kfd_build_sysfs_node_entry(dev, 0); | 924 | ret = kfd_build_sysfs_node_entry(dev, i); |
925 | if (ret < 0) | 925 | if (ret < 0) |
926 | return ret; | 926 | return ret; |
927 | i++; | 927 | i++; |
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h index 47b551970a14..96a512208fad 100644 --- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h | |||
@@ -183,7 +183,7 @@ struct kfd2kgd_calls { | |||
183 | int (*hqd_load)(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, | 183 | int (*hqd_load)(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, |
184 | uint32_t queue_id, uint32_t __user *wptr); | 184 | uint32_t queue_id, uint32_t __user *wptr); |
185 | 185 | ||
186 | bool (*hqd_is_occupies)(struct kgd_dev *kgd, uint64_t queue_address, | 186 | bool (*hqd_is_occupied)(struct kgd_dev *kgd, uint64_t queue_address, |
187 | uint32_t pipe_id, uint32_t queue_id); | 187 | uint32_t pipe_id, uint32_t queue_id); |
188 | 188 | ||
189 | int (*hqd_destroy)(struct kgd_dev *kgd, uint32_t reset_type, | 189 | int (*hqd_destroy)(struct kgd_dev *kgd, uint32_t reset_type, |