diff options
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/mm_gk20a.c | 122 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/mm_gk20a.h | 5 |
2 files changed, 62 insertions, 65 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c index 579441f0..3bb5ddb4 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c | |||
@@ -568,26 +568,26 @@ static int alloc_gmmu_phys_pages(struct vm_gk20a *vm, u32 order, | |||
568 | gk20a_dbg(gpu_dbg_pte, "alloc_pages failed\n"); | 568 | gk20a_dbg(gpu_dbg_pte, "alloc_pages failed\n"); |
569 | goto err_out; | 569 | goto err_out; |
570 | } | 570 | } |
571 | entry->sgt = kzalloc(sizeof(*entry->sgt), GFP_KERNEL); | 571 | entry->mem.sgt = kzalloc(sizeof(*entry->mem.sgt), GFP_KERNEL); |
572 | if (!entry->sgt) { | 572 | if (!entry->mem.sgt) { |
573 | gk20a_dbg(gpu_dbg_pte, "cannot allocate sg table"); | 573 | gk20a_dbg(gpu_dbg_pte, "cannot allocate sg table"); |
574 | goto err_alloced; | 574 | goto err_alloced; |
575 | } | 575 | } |
576 | err = sg_alloc_table(entry->sgt, 1, GFP_KERNEL); | 576 | err = sg_alloc_table(entry->mem.sgt, 1, GFP_KERNEL); |
577 | if (err) { | 577 | if (err) { |
578 | gk20a_dbg(gpu_dbg_pte, "sg_alloc_table failed\n"); | 578 | gk20a_dbg(gpu_dbg_pte, "sg_alloc_table failed\n"); |
579 | goto err_sg_table; | 579 | goto err_sg_table; |
580 | } | 580 | } |
581 | sg_set_page(entry->sgt->sgl, pages, len, 0); | 581 | sg_set_page(entry->mem.sgt->sgl, pages, len, 0); |
582 | entry->cpu_va = page_address(pages); | 582 | entry->mem.cpu_va = page_address(pages); |
583 | memset(entry->cpu_va, 0, len); | 583 | memset(entry->mem.cpu_va, 0, len); |
584 | entry->size = len; | 584 | entry->mem.size = len; |
585 | FLUSH_CPU_DCACHE(entry->cpu_va, sg_phys(entry->sgt->sgl), len); | 585 | FLUSH_CPU_DCACHE(entry->mem.cpu_va, sg_phys(entry->mem.sgt->sgl), len); |
586 | 586 | ||
587 | return 0; | 587 | return 0; |
588 | 588 | ||
589 | err_sg_table: | 589 | err_sg_table: |
590 | kfree(entry->sgt); | 590 | kfree(entry->mem.sgt); |
591 | err_alloced: | 591 | err_alloced: |
592 | __free_pages(pages, order); | 592 | __free_pages(pages, order); |
593 | err_out: | 593 | err_out: |
@@ -598,27 +598,27 @@ static void free_gmmu_phys_pages(struct vm_gk20a *vm, | |||
598 | struct gk20a_mm_entry *entry) | 598 | struct gk20a_mm_entry *entry) |
599 | { | 599 | { |
600 | gk20a_dbg_fn(""); | 600 | gk20a_dbg_fn(""); |
601 | free_pages((unsigned long)entry->cpu_va, get_order(entry->size)); | 601 | free_pages((unsigned long)entry->mem.cpu_va, get_order(entry->mem.size)); |
602 | entry->cpu_va = NULL; | 602 | entry->mem.cpu_va = NULL; |
603 | 603 | ||
604 | sg_free_table(entry->sgt); | 604 | sg_free_table(entry->mem.sgt); |
605 | kfree(entry->sgt); | 605 | kfree(entry->mem.sgt); |
606 | entry->sgt = NULL; | 606 | entry->mem.sgt = NULL; |
607 | } | 607 | } |
608 | 608 | ||
609 | static int map_gmmu_phys_pages(struct gk20a_mm_entry *entry) | 609 | static int map_gmmu_phys_pages(struct gk20a_mm_entry *entry) |
610 | { | 610 | { |
611 | FLUSH_CPU_DCACHE(entry->cpu_va, | 611 | FLUSH_CPU_DCACHE(entry->mem.cpu_va, |
612 | sg_phys(entry->sgt->sgl), | 612 | sg_phys(entry->mem.sgt->sgl), |
613 | entry->sgt->sgl->length); | 613 | entry->mem.sgt->sgl->length); |
614 | return 0; | 614 | return 0; |
615 | } | 615 | } |
616 | 616 | ||
617 | static void unmap_gmmu_phys_pages(struct gk20a_mm_entry *entry) | 617 | static void unmap_gmmu_phys_pages(struct gk20a_mm_entry *entry) |
618 | { | 618 | { |
619 | FLUSH_CPU_DCACHE(entry->cpu_va, | 619 | FLUSH_CPU_DCACHE(entry->mem.cpu_va, |
620 | sg_phys(entry->sgt->sgl), | 620 | sg_phys(entry->mem.sgt->sgl), |
621 | entry->sgt->sgl->length); | 621 | entry->mem.sgt->sgl->length); |
622 | } | 622 | } |
623 | 623 | ||
624 | static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order, | 624 | static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order, |
@@ -637,7 +637,7 @@ static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order, | |||
637 | if (tegra_platform_is_linsim()) | 637 | if (tegra_platform_is_linsim()) |
638 | return alloc_gmmu_phys_pages(vm, order, entry); | 638 | return alloc_gmmu_phys_pages(vm, order, entry); |
639 | 639 | ||
640 | entry->size = len; | 640 | entry->mem.size = len; |
641 | 641 | ||
642 | /* | 642 | /* |
643 | * On arm32 we're limited by vmalloc space, so we do not map pages by | 643 | * On arm32 we're limited by vmalloc space, so we do not map pages by |
@@ -650,13 +650,13 @@ static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order, | |||
650 | goto err_out; | 650 | goto err_out; |
651 | } | 651 | } |
652 | 652 | ||
653 | err = gk20a_get_sgtable(d, &entry->sgt, cpuva, iova, len); | 653 | err = gk20a_get_sgtable(d, &entry->mem.sgt, cpuva, iova, len); |
654 | if (err) { | 654 | if (err) { |
655 | gk20a_err(d, "sgt allocation failed\n"); | 655 | gk20a_err(d, "sgt allocation failed\n"); |
656 | goto err_free; | 656 | goto err_free; |
657 | } | 657 | } |
658 | 658 | ||
659 | entry->cpu_va = cpuva; | 659 | entry->mem.cpu_va = cpuva; |
660 | } else { | 660 | } else { |
661 | struct page **pages; | 661 | struct page **pages; |
662 | 662 | ||
@@ -667,25 +667,25 @@ static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order, | |||
667 | goto err_out; | 667 | goto err_out; |
668 | } | 668 | } |
669 | 669 | ||
670 | err = gk20a_get_sgtable_from_pages(d, &entry->sgt, pages, | 670 | err = gk20a_get_sgtable_from_pages(d, &entry->mem.sgt, pages, |
671 | iova, len); | 671 | iova, len); |
672 | if (err) { | 672 | if (err) { |
673 | gk20a_err(d, "sgt allocation failed\n"); | 673 | gk20a_err(d, "sgt allocation failed\n"); |
674 | goto err_free; | 674 | goto err_free; |
675 | } | 675 | } |
676 | 676 | ||
677 | entry->pages = pages; | 677 | entry->mem.pages = pages; |
678 | } | 678 | } |
679 | 679 | ||
680 | return 0; | 680 | return 0; |
681 | 681 | ||
682 | err_free: | 682 | err_free: |
683 | if (IS_ENABLED(CONFIG_ARM64)) { | 683 | if (IS_ENABLED(CONFIG_ARM64)) { |
684 | dma_free_coherent(d, len, entry->cpu_va, iova); | 684 | dma_free_coherent(d, len, entry->mem.cpu_va, iova); |
685 | cpuva = NULL; | 685 | cpuva = NULL; |
686 | } else { | 686 | } else { |
687 | dma_free_attrs(d, len, entry->pages, iova, &attrs); | 687 | dma_free_attrs(d, len, entry->mem.pages, iova, &attrs); |
688 | entry->pages = NULL; | 688 | entry->mem.pages = NULL; |
689 | } | 689 | } |
690 | iova = 0; | 690 | iova = 0; |
691 | err_out: | 691 | err_out: |
@@ -701,7 +701,7 @@ void free_gmmu_pages(struct vm_gk20a *vm, | |||
701 | 701 | ||
702 | gk20a_dbg_fn(""); | 702 | gk20a_dbg_fn(""); |
703 | 703 | ||
704 | if (!entry->sgt) | 704 | if (!entry->mem.sgt) |
705 | return; | 705 | return; |
706 | 706 | ||
707 | if (tegra_platform_is_linsim()) { | 707 | if (tegra_platform_is_linsim()) { |
@@ -709,29 +709,29 @@ void free_gmmu_pages(struct vm_gk20a *vm, | |||
709 | return; | 709 | return; |
710 | } | 710 | } |
711 | 711 | ||
712 | iova = sg_dma_address(entry->sgt->sgl); | 712 | iova = sg_dma_address(entry->mem.sgt->sgl); |
713 | 713 | ||
714 | gk20a_free_sgtable(&entry->sgt); | 714 | gk20a_free_sgtable(&entry->mem.sgt); |
715 | 715 | ||
716 | /* | 716 | /* |
717 | * On arm32 we're limited by vmalloc space, so we do not map pages by | 717 | * On arm32 we're limited by vmalloc space, so we do not map pages by |
718 | * default. | 718 | * default. |
719 | */ | 719 | */ |
720 | if (IS_ENABLED(CONFIG_ARM64)) { | 720 | if (IS_ENABLED(CONFIG_ARM64)) { |
721 | dma_free_coherent(d, entry->size, entry->cpu_va, iova); | 721 | dma_free_coherent(d, entry->mem.size, entry->mem.cpu_va, iova); |
722 | entry->cpu_va = NULL; | 722 | entry->mem.cpu_va = NULL; |
723 | } else { | 723 | } else { |
724 | dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs); | 724 | dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs); |
725 | dma_free_attrs(d, entry->size, entry->pages, iova, &attrs); | 725 | dma_free_attrs(d, entry->mem.size, entry->mem.pages, iova, &attrs); |
726 | entry->pages = NULL; | 726 | entry->mem.pages = NULL; |
727 | } | 727 | } |
728 | entry->size = 0; | 728 | entry->mem.size = 0; |
729 | entry->sgt = NULL; | 729 | entry->mem.sgt = NULL; |
730 | } | 730 | } |
731 | 731 | ||
732 | int map_gmmu_pages(struct gk20a_mm_entry *entry) | 732 | int map_gmmu_pages(struct gk20a_mm_entry *entry) |
733 | { | 733 | { |
734 | int count = PAGE_ALIGN(entry->size) >> PAGE_SHIFT; | 734 | int count = PAGE_ALIGN(entry->mem.size) >> PAGE_SHIFT; |
735 | struct page **pages; | 735 | struct page **pages; |
736 | gk20a_dbg_fn(""); | 736 | gk20a_dbg_fn(""); |
737 | 737 | ||
@@ -739,14 +739,14 @@ int map_gmmu_pages(struct gk20a_mm_entry *entry) | |||
739 | return map_gmmu_phys_pages(entry); | 739 | return map_gmmu_phys_pages(entry); |
740 | 740 | ||
741 | if (IS_ENABLED(CONFIG_ARM64)) { | 741 | if (IS_ENABLED(CONFIG_ARM64)) { |
742 | FLUSH_CPU_DCACHE(entry->cpu_va, | 742 | FLUSH_CPU_DCACHE(entry->mem.cpu_va, |
743 | sg_phys(entry->sgt->sgl), | 743 | sg_phys(entry->mem.sgt->sgl), |
744 | entry->size); | 744 | entry->mem.size); |
745 | } else { | 745 | } else { |
746 | pages = entry->pages; | 746 | pages = entry->mem.pages; |
747 | entry->cpu_va = vmap(pages, count, 0, | 747 | entry->mem.cpu_va = vmap(pages, count, 0, |
748 | pgprot_writecombine(PAGE_KERNEL)); | 748 | pgprot_writecombine(PAGE_KERNEL)); |
749 | if (!entry->cpu_va) | 749 | if (!entry->mem.cpu_va) |
750 | return -ENOMEM; | 750 | return -ENOMEM; |
751 | } | 751 | } |
752 | 752 | ||
@@ -763,12 +763,12 @@ void unmap_gmmu_pages(struct gk20a_mm_entry *entry) | |||
763 | } | 763 | } |
764 | 764 | ||
765 | if (IS_ENABLED(CONFIG_ARM64)) { | 765 | if (IS_ENABLED(CONFIG_ARM64)) { |
766 | FLUSH_CPU_DCACHE(entry->cpu_va, | 766 | FLUSH_CPU_DCACHE(entry->mem.cpu_va, |
767 | sg_phys(entry->sgt->sgl), | 767 | sg_phys(entry->mem.sgt->sgl), |
768 | entry->size); | 768 | entry->mem.size); |
769 | } else { | 769 | } else { |
770 | vunmap(entry->cpu_va); | 770 | vunmap(entry->mem.cpu_va); |
771 | entry->cpu_va = NULL; | 771 | entry->mem.cpu_va = NULL; |
772 | } | 772 | } |
773 | } | 773 | } |
774 | 774 | ||
@@ -796,7 +796,7 @@ static int gk20a_zalloc_gmmu_page_table(struct vm_gk20a *vm, | |||
796 | 796 | ||
797 | err = alloc_gmmu_pages(vm, order, entry); | 797 | err = alloc_gmmu_pages(vm, order, entry); |
798 | gk20a_dbg(gpu_dbg_pte, "entry = 0x%p, addr=%08llx, size %d", | 798 | gk20a_dbg(gpu_dbg_pte, "entry = 0x%p, addr=%08llx, size %d", |
799 | entry, g->ops.mm.get_iova_addr(g, entry->sgt->sgl, 0), | 799 | entry, g->ops.mm.get_iova_addr(g, entry->mem.sgt->sgl, 0), |
800 | order); | 800 | order); |
801 | if (err) | 801 | if (err) |
802 | return err; | 802 | return err; |
@@ -827,7 +827,7 @@ void pde_range_from_vaddr_range(struct vm_gk20a *vm, | |||
827 | 827 | ||
828 | u32 *pde_from_index(struct vm_gk20a *vm, u32 i) | 828 | u32 *pde_from_index(struct vm_gk20a *vm, u32 i) |
829 | { | 829 | { |
830 | return (u32 *) (((u8 *)vm->pdb.cpu_va) + i*gmmu_pde__size_v()); | 830 | return (u32 *) (((u8 *)vm->pdb.mem.cpu_va) + i*gmmu_pde__size_v()); |
831 | } | 831 | } |
832 | 832 | ||
833 | u32 pte_index_from_vaddr(struct vm_gk20a *vm, | 833 | u32 pte_index_from_vaddr(struct vm_gk20a *vm, |
@@ -2370,14 +2370,14 @@ static int update_gmmu_pde_locked(struct vm_gk20a *vm, | |||
2370 | 2370 | ||
2371 | gk20a_dbg_fn(""); | 2371 | gk20a_dbg_fn(""); |
2372 | 2372 | ||
2373 | small_valid = entry->size && entry->pgsz == gmmu_page_size_small; | 2373 | small_valid = entry->mem.size && entry->pgsz == gmmu_page_size_small; |
2374 | big_valid = entry->size && entry->pgsz == gmmu_page_size_big; | 2374 | big_valid = entry->mem.size && entry->pgsz == gmmu_page_size_big; |
2375 | 2375 | ||
2376 | if (small_valid) | 2376 | if (small_valid) |
2377 | pte_addr_small = g->ops.mm.get_iova_addr(g, entry->sgt->sgl, 0); | 2377 | pte_addr_small = g->ops.mm.get_iova_addr(g, entry->mem.sgt->sgl, 0); |
2378 | 2378 | ||
2379 | if (big_valid) | 2379 | if (big_valid) |
2380 | pte_addr_big = g->ops.mm.get_iova_addr(g, entry->sgt->sgl, 0); | 2380 | pte_addr_big = g->ops.mm.get_iova_addr(g, entry->mem.sgt->sgl, 0); |
2381 | 2381 | ||
2382 | pde_v[0] = gmmu_pde_size_full_f(); | 2382 | pde_v[0] = gmmu_pde_size_full_f(); |
2383 | pde_v[0] |= big_valid ? big_valid_pde0_bits(g, pte_addr_big) : | 2383 | pde_v[0] |= big_valid ? big_valid_pde0_bits(g, pte_addr_big) : |
@@ -2475,8 +2475,8 @@ static int update_gmmu_pte_locked(struct vm_gk20a *vm, | |||
2475 | gk20a_dbg(gpu_dbg_pte, "pte_cur=%d [0x0,0x0]", i); | 2475 | gk20a_dbg(gpu_dbg_pte, "pte_cur=%d [0x0,0x0]", i); |
2476 | } | 2476 | } |
2477 | 2477 | ||
2478 | gk20a_mem_wr32(pte->cpu_va + i*8, 0, pte_w[0]); | 2478 | gk20a_mem_wr32(pte->mem.cpu_va + i*8, 0, pte_w[0]); |
2479 | gk20a_mem_wr32(pte->cpu_va + i*8, 1, pte_w[1]); | 2479 | gk20a_mem_wr32(pte->mem.cpu_va + i*8, 1, pte_w[1]); |
2480 | 2480 | ||
2481 | if (*iova) { | 2481 | if (*iova) { |
2482 | *iova += page_size; | 2482 | *iova += page_size; |
@@ -2548,7 +2548,7 @@ static int update_gmmu_level_locked(struct vm_gk20a *vm, | |||
2548 | } | 2548 | } |
2549 | next_pte = pte->entries + pde_i; | 2549 | next_pte = pte->entries + pde_i; |
2550 | 2550 | ||
2551 | if (!next_pte->size) { | 2551 | if (!next_pte->mem.size) { |
2552 | err = gk20a_zalloc_gmmu_page_table(vm, | 2552 | err = gk20a_zalloc_gmmu_page_table(vm, |
2553 | pgsz_idx, next_l, next_pte); | 2553 | pgsz_idx, next_l, next_pte); |
2554 | if (err) | 2554 | if (err) |
@@ -2761,7 +2761,7 @@ static void gk20a_vm_free_entries(struct vm_gk20a *vm, | |||
2761 | for (i = 0; i < parent->num_entries; i++) | 2761 | for (i = 0; i < parent->num_entries; i++) |
2762 | gk20a_vm_free_entries(vm, &parent->entries[i], level+1); | 2762 | gk20a_vm_free_entries(vm, &parent->entries[i], level+1); |
2763 | 2763 | ||
2764 | if (parent->size) | 2764 | if (parent->mem.size) |
2765 | free_gmmu_pages(vm, parent); | 2765 | free_gmmu_pages(vm, parent); |
2766 | vfree(parent->entries); | 2766 | vfree(parent->entries); |
2767 | parent->entries = NULL; | 2767 | parent->entries = NULL; |
@@ -3552,7 +3552,7 @@ void gk20a_init_inst_block(struct mem_desc *inst_block, struct vm_gk20a *vm, | |||
3552 | u32 big_page_size) | 3552 | u32 big_page_size) |
3553 | { | 3553 | { |
3554 | struct gk20a *g = gk20a_from_vm(vm); | 3554 | struct gk20a *g = gk20a_from_vm(vm); |
3555 | u64 pde_addr = g->ops.mm.get_iova_addr(g, vm->pdb.sgt->sgl, 0); | 3555 | u64 pde_addr = g->ops.mm.get_iova_addr(g, vm->pdb.mem.sgt->sgl, 0); |
3556 | void *inst_ptr = inst_block->cpu_va; | 3556 | void *inst_ptr = inst_block->cpu_va; |
3557 | 3557 | ||
3558 | gk20a_dbg_info("inst block phys = 0x%llx, kv = 0x%p", | 3558 | gk20a_dbg_info("inst block phys = 0x%llx, kv = 0x%p", |
@@ -3796,7 +3796,7 @@ void gk20a_mm_tlb_invalidate(struct vm_gk20a *vm) | |||
3796 | { | 3796 | { |
3797 | struct gk20a *g = gk20a_from_vm(vm); | 3797 | struct gk20a *g = gk20a_from_vm(vm); |
3798 | u32 addr_lo = u64_lo32(g->ops.mm.get_iova_addr(vm->mm->g, | 3798 | u32 addr_lo = u64_lo32(g->ops.mm.get_iova_addr(vm->mm->g, |
3799 | vm->pdb.sgt->sgl, 0) >> 12); | 3799 | vm->pdb.mem.sgt->sgl, 0) >> 12); |
3800 | u32 data; | 3800 | u32 data; |
3801 | s32 retry = 2000; | 3801 | s32 retry = 2000; |
3802 | static DEFINE_MUTEX(tlb_lock); | 3802 | static DEFINE_MUTEX(tlb_lock); |
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h index e7154c84..3bc568d6 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h | |||
@@ -152,10 +152,7 @@ struct gk20a_comptags { | |||
152 | 152 | ||
153 | struct gk20a_mm_entry { | 153 | struct gk20a_mm_entry { |
154 | /* backing for */ | 154 | /* backing for */ |
155 | void *cpu_va; | 155 | struct mem_desc mem; |
156 | struct sg_table *sgt; | ||
157 | struct page **pages; | ||
158 | size_t size; | ||
159 | int pgsz; | 156 | int pgsz; |
160 | struct gk20a_mm_entry *entries; | 157 | struct gk20a_mm_entry *entries; |
161 | int num_entries; | 158 | int num_entries; |