aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c')
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c111
1 files changed, 62 insertions, 49 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index ea708cb94862..9f36ed30ba11 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -53,6 +53,18 @@
53/* Special value that no flush is necessary */ 53/* Special value that no flush is necessary */
54#define AMDGPU_VM_NO_FLUSH (~0ll) 54#define AMDGPU_VM_NO_FLUSH (~0ll)
55 55
56/* Local structure. Encapsulate some VM table update parameters to reduce
57 * the number of function parameters
58 */
59struct amdgpu_vm_update_params {
60 /* address where to copy page table entries from */
61 uint64_t src;
62 /* DMA addresses to use for mapping */
63 dma_addr_t *pages_addr;
64 /* indirect buffer to fill with commands */
65 struct amdgpu_ib *ib;
66};
67
56/** 68/**
57 * amdgpu_vm_num_pde - return the number of page directory entries 69 * amdgpu_vm_num_pde - return the number of page directory entries
58 * 70 *
@@ -389,9 +401,7 @@ struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
389 * amdgpu_vm_update_pages - helper to call the right asic function 401 * amdgpu_vm_update_pages - helper to call the right asic function
390 * 402 *
391 * @adev: amdgpu_device pointer 403 * @adev: amdgpu_device pointer
392 * @src: address where to copy page table entries from 404 * @vm_update_params: see amdgpu_vm_update_params definition
393 * @pages_addr: DMA addresses to use for mapping
394 * @ib: indirect buffer to fill with commands
395 * @pe: addr of the page entry 405 * @pe: addr of the page entry
396 * @addr: dst addr to write into pe 406 * @addr: dst addr to write into pe
397 * @count: number of page entries to update 407 * @count: number of page entries to update
@@ -402,29 +412,29 @@ struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
402 * to setup the page table using the DMA. 412 * to setup the page table using the DMA.
403 */ 413 */
404static void amdgpu_vm_update_pages(struct amdgpu_device *adev, 414static void amdgpu_vm_update_pages(struct amdgpu_device *adev,
405 uint64_t src, 415 struct amdgpu_vm_update_params
406 dma_addr_t *pages_addr, 416 *vm_update_params,
407 struct amdgpu_ib *ib,
408 uint64_t pe, uint64_t addr, 417 uint64_t pe, uint64_t addr,
409 unsigned count, uint32_t incr, 418 unsigned count, uint32_t incr,
410 uint32_t flags) 419 uint32_t flags)
411{ 420{
412 trace_amdgpu_vm_set_page(pe, addr, count, incr, flags); 421 trace_amdgpu_vm_set_page(pe, addr, count, incr, flags);
413 422
414 if (src) { 423 if (vm_update_params->src) {
415 src += (addr >> 12) * 8; 424 amdgpu_vm_copy_pte(adev, vm_update_params->ib,
416 amdgpu_vm_copy_pte(adev, ib, pe, src, count); 425 pe, (vm_update_params->src + (addr >> 12) * 8), count);
417 426
418 } else if (pages_addr) { 427 } else if (vm_update_params->pages_addr) {
419 amdgpu_vm_write_pte(adev, ib, pages_addr, pe, addr, 428 amdgpu_vm_write_pte(adev, vm_update_params->ib,
420 count, incr, flags); 429 vm_update_params->pages_addr,
430 pe, addr, count, incr, flags);
421 431
422 } else if (count < 3) { 432 } else if (count < 3) {
423 amdgpu_vm_write_pte(adev, ib, NULL, pe, addr, 433 amdgpu_vm_write_pte(adev, vm_update_params->ib, NULL, pe, addr,
424 count, incr, flags); 434 count, incr, flags);
425 435
426 } else { 436 } else {
427 amdgpu_vm_set_pte_pde(adev, ib, pe, addr, 437 amdgpu_vm_set_pte_pde(adev, vm_update_params->ib, pe, addr,
428 count, incr, flags); 438 count, incr, flags);
429 } 439 }
430} 440}
@@ -444,10 +454,12 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
444 struct amdgpu_ring *ring; 454 struct amdgpu_ring *ring;
445 struct fence *fence = NULL; 455 struct fence *fence = NULL;
446 struct amdgpu_job *job; 456 struct amdgpu_job *job;
457 struct amdgpu_vm_update_params vm_update_params;
447 unsigned entries; 458 unsigned entries;
448 uint64_t addr; 459 uint64_t addr;
449 int r; 460 int r;
450 461
462 memset(&vm_update_params, 0, sizeof(vm_update_params));
451 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched); 463 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
452 464
453 r = reservation_object_reserve_shared(bo->tbo.resv); 465 r = reservation_object_reserve_shared(bo->tbo.resv);
@@ -465,7 +477,8 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
465 if (r) 477 if (r)
466 goto error; 478 goto error;
467 479
468 amdgpu_vm_update_pages(adev, 0, NULL, &job->ibs[0], addr, 0, entries, 480 vm_update_params.ib = &job->ibs[0];
481 amdgpu_vm_update_pages(adev, &vm_update_params, addr, 0, entries,
469 0, 0); 482 0, 0);
470 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 483 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
471 484
@@ -538,11 +551,12 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
538 uint64_t last_pde = ~0, last_pt = ~0; 551 uint64_t last_pde = ~0, last_pt = ~0;
539 unsigned count = 0, pt_idx, ndw; 552 unsigned count = 0, pt_idx, ndw;
540 struct amdgpu_job *job; 553 struct amdgpu_job *job;
541 struct amdgpu_ib *ib; 554 struct amdgpu_vm_update_params vm_update_params;
542 struct fence *fence = NULL; 555 struct fence *fence = NULL;
543 556
544 int r; 557 int r;
545 558
559 memset(&vm_update_params, 0, sizeof(vm_update_params));
546 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched); 560 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
547 561
548 /* padding, etc. */ 562 /* padding, etc. */
@@ -555,7 +569,7 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
555 if (r) 569 if (r)
556 return r; 570 return r;
557 571
558 ib = &job->ibs[0]; 572 vm_update_params.ib = &job->ibs[0];
559 573
560 /* walk over the address space and update the page directory */ 574 /* walk over the address space and update the page directory */
561 for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) { 575 for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
@@ -575,7 +589,7 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
575 ((last_pt + incr * count) != pt)) { 589 ((last_pt + incr * count) != pt)) {
576 590
577 if (count) { 591 if (count) {
578 amdgpu_vm_update_pages(adev, 0, NULL, ib, 592 amdgpu_vm_update_pages(adev, &vm_update_params,
579 last_pde, last_pt, 593 last_pde, last_pt,
580 count, incr, 594 count, incr,
581 AMDGPU_PTE_VALID); 595 AMDGPU_PTE_VALID);
@@ -590,14 +604,15 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
590 } 604 }
591 605
592 if (count) 606 if (count)
593 amdgpu_vm_update_pages(adev, 0, NULL, ib, last_pde, last_pt, 607 amdgpu_vm_update_pages(adev, &vm_update_params,
594 count, incr, AMDGPU_PTE_VALID); 608 last_pde, last_pt,
609 count, incr, AMDGPU_PTE_VALID);
595 610
596 if (ib->length_dw != 0) { 611 if (vm_update_params.ib->length_dw != 0) {
597 amdgpu_ring_pad_ib(ring, ib); 612 amdgpu_ring_pad_ib(ring, vm_update_params.ib);
598 amdgpu_sync_resv(adev, &job->sync, pd->tbo.resv, 613 amdgpu_sync_resv(adev, &job->sync, pd->tbo.resv,
599 AMDGPU_FENCE_OWNER_VM); 614 AMDGPU_FENCE_OWNER_VM);
600 WARN_ON(ib->length_dw > ndw); 615 WARN_ON(vm_update_params.ib->length_dw > ndw);
601 r = amdgpu_job_submit(job, ring, &vm->entity, 616 r = amdgpu_job_submit(job, ring, &vm->entity,
602 AMDGPU_FENCE_OWNER_VM, &fence); 617 AMDGPU_FENCE_OWNER_VM, &fence);
603 if (r) 618 if (r)
@@ -623,18 +638,15 @@ error_free:
623 * amdgpu_vm_frag_ptes - add fragment information to PTEs 638 * amdgpu_vm_frag_ptes - add fragment information to PTEs
624 * 639 *
625 * @adev: amdgpu_device pointer 640 * @adev: amdgpu_device pointer
626 * @src: address where to copy page table entries from 641 * @vm_update_params: see amdgpu_vm_update_params definition
627 * @pages_addr: DMA addresses to use for mapping
628 * @ib: IB for the update
629 * @pe_start: first PTE to handle 642 * @pe_start: first PTE to handle
630 * @pe_end: last PTE to handle 643 * @pe_end: last PTE to handle
631 * @addr: addr those PTEs should point to 644 * @addr: addr those PTEs should point to
632 * @flags: hw mapping flags 645 * @flags: hw mapping flags
633 */ 646 */
634static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev, 647static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
635 uint64_t src, 648 struct amdgpu_vm_update_params
636 dma_addr_t *pages_addr, 649 *vm_update_params,
637 struct amdgpu_ib *ib,
638 uint64_t pe_start, uint64_t pe_end, 650 uint64_t pe_start, uint64_t pe_end,
639 uint64_t addr, uint32_t flags) 651 uint64_t addr, uint32_t flags)
640{ 652{
@@ -671,11 +683,11 @@ static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
671 return; 683 return;
672 684
673 /* system pages are non continuously */ 685 /* system pages are non continuously */
674 if (src || pages_addr || !(flags & AMDGPU_PTE_VALID) || 686 if (vm_update_params->src || vm_update_params->pages_addr ||
675 (frag_start >= frag_end)) { 687 !(flags & AMDGPU_PTE_VALID) || (frag_start >= frag_end)) {
676 688
677 count = (pe_end - pe_start) / 8; 689 count = (pe_end - pe_start) / 8;
678 amdgpu_vm_update_pages(adev, src, pages_addr, ib, pe_start, 690 amdgpu_vm_update_pages(adev, vm_update_params, pe_start,
679 addr, count, AMDGPU_GPU_PAGE_SIZE, 691 addr, count, AMDGPU_GPU_PAGE_SIZE,
680 flags); 692 flags);
681 return; 693 return;
@@ -684,21 +696,21 @@ static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
684 /* handle the 4K area at the beginning */ 696 /* handle the 4K area at the beginning */
685 if (pe_start != frag_start) { 697 if (pe_start != frag_start) {
686 count = (frag_start - pe_start) / 8; 698 count = (frag_start - pe_start) / 8;
687 amdgpu_vm_update_pages(adev, 0, NULL, ib, pe_start, addr, 699 amdgpu_vm_update_pages(adev, vm_update_params, pe_start, addr,
688 count, AMDGPU_GPU_PAGE_SIZE, flags); 700 count, AMDGPU_GPU_PAGE_SIZE, flags);
689 addr += AMDGPU_GPU_PAGE_SIZE * count; 701 addr += AMDGPU_GPU_PAGE_SIZE * count;
690 } 702 }
691 703
692 /* handle the area in the middle */ 704 /* handle the area in the middle */
693 count = (frag_end - frag_start) / 8; 705 count = (frag_end - frag_start) / 8;
694 amdgpu_vm_update_pages(adev, 0, NULL, ib, frag_start, addr, count, 706 amdgpu_vm_update_pages(adev, vm_update_params, frag_start, addr, count,
695 AMDGPU_GPU_PAGE_SIZE, flags | frag_flags); 707 AMDGPU_GPU_PAGE_SIZE, flags | frag_flags);
696 708
697 /* handle the 4K area at the end */ 709 /* handle the 4K area at the end */
698 if (frag_end != pe_end) { 710 if (frag_end != pe_end) {
699 addr += AMDGPU_GPU_PAGE_SIZE * count; 711 addr += AMDGPU_GPU_PAGE_SIZE * count;
700 count = (pe_end - frag_end) / 8; 712 count = (pe_end - frag_end) / 8;
701 amdgpu_vm_update_pages(adev, 0, NULL, ib, frag_end, addr, 713 amdgpu_vm_update_pages(adev, vm_update_params, frag_end, addr,
702 count, AMDGPU_GPU_PAGE_SIZE, flags); 714 count, AMDGPU_GPU_PAGE_SIZE, flags);
703 } 715 }
704} 716}
@@ -707,8 +719,7 @@ static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
707 * amdgpu_vm_update_ptes - make sure that page tables are valid 719 * amdgpu_vm_update_ptes - make sure that page tables are valid
708 * 720 *
709 * @adev: amdgpu_device pointer 721 * @adev: amdgpu_device pointer
710 * @src: address where to copy page table entries from 722 * @vm_update_params: see amdgpu_vm_update_params definition
711 * @pages_addr: DMA addresses to use for mapping
712 * @vm: requested vm 723 * @vm: requested vm
713 * @start: start of GPU address range 724 * @start: start of GPU address range
714 * @end: end of GPU address range 725 * @end: end of GPU address range
@@ -718,10 +729,9 @@ static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
718 * Update the page tables in the range @start - @end. 729 * Update the page tables in the range @start - @end.
719 */ 730 */
720static void amdgpu_vm_update_ptes(struct amdgpu_device *adev, 731static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
721 uint64_t src, 732 struct amdgpu_vm_update_params
722 dma_addr_t *pages_addr, 733 *vm_update_params,
723 struct amdgpu_vm *vm, 734 struct amdgpu_vm *vm,
724 struct amdgpu_ib *ib,
725 uint64_t start, uint64_t end, 735 uint64_t start, uint64_t end,
726 uint64_t dst, uint32_t flags) 736 uint64_t dst, uint32_t flags)
727{ 737{
@@ -747,7 +757,7 @@ static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
747 757
748 if (last_pe_end != pe_start) { 758 if (last_pe_end != pe_start) {
749 759
750 amdgpu_vm_frag_ptes(adev, src, pages_addr, ib, 760 amdgpu_vm_frag_ptes(adev, vm_update_params,
751 last_pe_start, last_pe_end, 761 last_pe_start, last_pe_end,
752 last_dst, flags); 762 last_dst, flags);
753 763
@@ -762,7 +772,7 @@ static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
762 dst += nptes * AMDGPU_GPU_PAGE_SIZE; 772 dst += nptes * AMDGPU_GPU_PAGE_SIZE;
763 } 773 }
764 774
765 amdgpu_vm_frag_ptes(adev, src, pages_addr, ib, last_pe_start, 775 amdgpu_vm_frag_ptes(adev, vm_update_params, last_pe_start,
766 last_pe_end, last_dst, flags); 776 last_pe_end, last_dst, flags);
767} 777}
768 778
@@ -794,11 +804,14 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
794 void *owner = AMDGPU_FENCE_OWNER_VM; 804 void *owner = AMDGPU_FENCE_OWNER_VM;
795 unsigned nptes, ncmds, ndw; 805 unsigned nptes, ncmds, ndw;
796 struct amdgpu_job *job; 806 struct amdgpu_job *job;
797 struct amdgpu_ib *ib; 807 struct amdgpu_vm_update_params vm_update_params;
798 struct fence *f = NULL; 808 struct fence *f = NULL;
799 int r; 809 int r;
800 810
801 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched); 811 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
812 memset(&vm_update_params, 0, sizeof(vm_update_params));
813 vm_update_params.src = src;
814 vm_update_params.pages_addr = pages_addr;
802 815
803 /* sync to everything on unmapping */ 816 /* sync to everything on unmapping */
804 if (!(flags & AMDGPU_PTE_VALID)) 817 if (!(flags & AMDGPU_PTE_VALID))
@@ -815,11 +828,11 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
815 /* padding, etc. */ 828 /* padding, etc. */
816 ndw = 64; 829 ndw = 64;
817 830
818 if (src) { 831 if (vm_update_params.src) {
819 /* only copy commands needed */ 832 /* only copy commands needed */
820 ndw += ncmds * 7; 833 ndw += ncmds * 7;
821 834
822 } else if (pages_addr) { 835 } else if (vm_update_params.pages_addr) {
823 /* header for write data commands */ 836 /* header for write data commands */
824 ndw += ncmds * 4; 837 ndw += ncmds * 4;
825 838
@@ -838,7 +851,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
838 if (r) 851 if (r)
839 return r; 852 return r;
840 853
841 ib = &job->ibs[0]; 854 vm_update_params.ib = &job->ibs[0];
842 855
843 r = amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv, 856 r = amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
844 owner); 857 owner);
@@ -849,11 +862,11 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
849 if (r) 862 if (r)
850 goto error_free; 863 goto error_free;
851 864
852 amdgpu_vm_update_ptes(adev, src, pages_addr, vm, ib, start, 865 amdgpu_vm_update_ptes(adev, &vm_update_params, vm, start,
853 last + 1, addr, flags); 866 last + 1, addr, flags);
854 867
855 amdgpu_ring_pad_ib(ring, ib); 868 amdgpu_ring_pad_ib(ring, vm_update_params.ib);
856 WARN_ON(ib->length_dw > ndw); 869 WARN_ON(vm_update_params.ib->length_dw > ndw);
857 r = amdgpu_job_submit(job, ring, &vm->entity, 870 r = amdgpu_job_submit(job, ring, &vm->entity,
858 AMDGPU_FENCE_OWNER_VM, &f); 871 AMDGPU_FENCE_OWNER_VM, &f);
859 if (r) 872 if (r)