diff options
Diffstat (limited to 'drivers')
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/vm.c | 202 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/vm_priv.h | 10 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/vm.c | 134 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/vm_area.c | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/gk20a.h | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/mm_gk20a.c | 348 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/mm_gk20a.h | 1 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gm20b/hal_gm20b.c | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gm20b/mm_gm20b.c | 10 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gm20b/mm_gm20b.h | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gp106/hal_gp106.c | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gp10b/hal_gp10b.c | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/vm.h | 7 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/vgpu/gm20b/vgpu_hal_gm20b.c | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c | 2 |
15 files changed, 366 insertions, 362 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/vm.c b/drivers/gpu/nvgpu/common/linux/vm.c index b686d616..638d3e51 100644 --- a/drivers/gpu/nvgpu/common/linux/vm.c +++ b/drivers/gpu/nvgpu/common/linux/vm.c | |||
@@ -115,6 +115,108 @@ static u64 nvgpu_get_buffer_alignment(struct gk20a *g, struct scatterlist *sgl, | |||
115 | return align; | 115 | return align; |
116 | } | 116 | } |
117 | 117 | ||
118 | static int setup_kind_legacy(struct vm_gk20a *vm, struct buffer_attrs *bfr, | ||
119 | bool *pkind_compressible) | ||
120 | { | ||
121 | struct gk20a *g = gk20a_from_vm(vm); | ||
122 | bool kind_compressible; | ||
123 | |||
124 | if (unlikely(bfr->kind_v == g->ops.mm.get_kind_invalid())) | ||
125 | bfr->kind_v = g->ops.mm.get_kind_pitch(); | ||
126 | |||
127 | if (unlikely(!gk20a_kind_is_supported(bfr->kind_v))) { | ||
128 | nvgpu_err(g, "kind 0x%x not supported", bfr->kind_v); | ||
129 | return -EINVAL; | ||
130 | } | ||
131 | |||
132 | bfr->uc_kind_v = g->ops.mm.get_kind_invalid(); | ||
133 | /* find a suitable incompressible kind if it becomes necessary later */ | ||
134 | kind_compressible = gk20a_kind_is_compressible(bfr->kind_v); | ||
135 | if (kind_compressible) { | ||
136 | bfr->uc_kind_v = gk20a_get_uncompressed_kind(bfr->kind_v); | ||
137 | if (unlikely(bfr->uc_kind_v == g->ops.mm.get_kind_invalid())) { | ||
138 | /* shouldn't happen, but it is worth cross-checking */ | ||
139 | nvgpu_err(g, "comptag kind 0x%x can't be" | ||
140 | " downgraded to uncompressed kind", | ||
141 | bfr->kind_v); | ||
142 | return -EINVAL; | ||
143 | } | ||
144 | } | ||
145 | |||
146 | *pkind_compressible = kind_compressible; | ||
147 | return 0; | ||
148 | } | ||
149 | |||
150 | static int setup_buffer_kind_and_compression(struct vm_gk20a *vm, | ||
151 | u32 flags, | ||
152 | struct buffer_attrs *bfr, | ||
153 | enum gmmu_pgsz_gk20a pgsz_idx) | ||
154 | { | ||
155 | bool kind_compressible; | ||
156 | struct gk20a *g = gk20a_from_vm(vm); | ||
157 | int ctag_granularity = g->ops.fb.compression_page_size(g); | ||
158 | |||
159 | if (!bfr->use_kind_v) | ||
160 | bfr->kind_v = g->ops.mm.get_kind_invalid(); | ||
161 | if (!bfr->use_uc_kind_v) | ||
162 | bfr->uc_kind_v = g->ops.mm.get_kind_invalid(); | ||
163 | |||
164 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL) { | ||
165 | kind_compressible = (bfr->kind_v != | ||
166 | g->ops.mm.get_kind_invalid()); | ||
167 | if (!kind_compressible) | ||
168 | bfr->kind_v = bfr->uc_kind_v; | ||
169 | } else { | ||
170 | int err = setup_kind_legacy(vm, bfr, &kind_compressible); | ||
171 | |||
172 | if (err) | ||
173 | return err; | ||
174 | } | ||
175 | |||
176 | /* comptags only supported for suitable kinds, 128KB pagesize */ | ||
177 | if (kind_compressible && | ||
178 | vm->gmmu_page_sizes[pgsz_idx] < | ||
179 | g->ops.fb.compressible_page_size(g)) { | ||
180 | /* it is safe to fall back to uncompressed as | ||
181 | functionality is not harmed */ | ||
182 | bfr->kind_v = bfr->uc_kind_v; | ||
183 | kind_compressible = false; | ||
184 | } | ||
185 | if (kind_compressible) | ||
186 | bfr->ctag_lines = DIV_ROUND_UP_ULL(bfr->size, ctag_granularity); | ||
187 | else | ||
188 | bfr->ctag_lines = 0; | ||
189 | |||
190 | bfr->use_kind_v = (bfr->kind_v != g->ops.mm.get_kind_invalid()); | ||
191 | bfr->use_uc_kind_v = (bfr->uc_kind_v != g->ops.mm.get_kind_invalid()); | ||
192 | |||
193 | return 0; | ||
194 | } | ||
195 | |||
196 | int nvgpu_vm_find_buf(struct vm_gk20a *vm, u64 gpu_va, | ||
197 | struct dma_buf **dmabuf, | ||
198 | u64 *offset) | ||
199 | { | ||
200 | struct nvgpu_mapped_buf *mapped_buffer; | ||
201 | |||
202 | gk20a_dbg_fn("gpu_va=0x%llx", gpu_va); | ||
203 | |||
204 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
205 | |||
206 | mapped_buffer = __nvgpu_vm_find_mapped_buf_range(vm, gpu_va); | ||
207 | if (!mapped_buffer) { | ||
208 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
209 | return -EINVAL; | ||
210 | } | ||
211 | |||
212 | *dmabuf = mapped_buffer->dmabuf; | ||
213 | *offset = gpu_va - mapped_buffer->addr; | ||
214 | |||
215 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
216 | |||
217 | return 0; | ||
218 | } | ||
219 | |||
118 | /* | 220 | /* |
119 | * vm->update_gmmu_lock must be held. This checks to see if we already have | 221 | * vm->update_gmmu_lock must be held. This checks to see if we already have |
120 | * mapped the passed buffer into this VM. If so, just return the existing | 222 | * mapped the passed buffer into this VM. If so, just return the existing |
@@ -478,6 +580,67 @@ clean_up: | |||
478 | return 0; | 580 | return 0; |
479 | } | 581 | } |
480 | 582 | ||
583 | int nvgpu_vm_map_buffer(struct vm_gk20a *vm, | ||
584 | int dmabuf_fd, | ||
585 | u64 *offset_align, | ||
586 | u32 flags, /*NVGPU_AS_MAP_BUFFER_FLAGS_*/ | ||
587 | s16 compr_kind, | ||
588 | s16 incompr_kind, | ||
589 | u64 buffer_offset, | ||
590 | u64 mapping_size, | ||
591 | struct vm_gk20a_mapping_batch *batch) | ||
592 | { | ||
593 | int err = 0; | ||
594 | struct dma_buf *dmabuf; | ||
595 | u64 ret_va; | ||
596 | |||
597 | gk20a_dbg_fn(""); | ||
598 | |||
599 | /* get ref to the mem handle (released on unmap_locked) */ | ||
600 | dmabuf = dma_buf_get(dmabuf_fd); | ||
601 | if (IS_ERR(dmabuf)) { | ||
602 | nvgpu_warn(gk20a_from_vm(vm), "%s: fd %d is not a dmabuf", | ||
603 | __func__, dmabuf_fd); | ||
604 | return PTR_ERR(dmabuf); | ||
605 | } | ||
606 | |||
607 | /* verify that we're not overflowing the buffer, i.e. | ||
608 | * (buffer_offset + mapping_size)> dmabuf->size. | ||
609 | * | ||
610 | * Since buffer_offset + mapping_size could overflow, first check | ||
611 | * that mapping size < dmabuf_size, at which point we can subtract | ||
612 | * mapping_size from both sides for the final comparison. | ||
613 | */ | ||
614 | if ((mapping_size > dmabuf->size) || | ||
615 | (buffer_offset > (dmabuf->size - mapping_size))) { | ||
616 | nvgpu_err(gk20a_from_vm(vm), | ||
617 | "buf size %llx < (offset(%llx) + map_size(%llx))\n", | ||
618 | (u64)dmabuf->size, buffer_offset, mapping_size); | ||
619 | return -EINVAL; | ||
620 | } | ||
621 | |||
622 | err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev_from_vm(vm)); | ||
623 | if (err) { | ||
624 | dma_buf_put(dmabuf); | ||
625 | return err; | ||
626 | } | ||
627 | |||
628 | ret_va = nvgpu_vm_map(vm, dmabuf, *offset_align, | ||
629 | flags, compr_kind, incompr_kind, true, | ||
630 | gk20a_mem_flag_none, | ||
631 | buffer_offset, | ||
632 | mapping_size, | ||
633 | batch); | ||
634 | |||
635 | *offset_align = ret_va; | ||
636 | if (!ret_va) { | ||
637 | dma_buf_put(dmabuf); | ||
638 | err = -EINVAL; | ||
639 | } | ||
640 | |||
641 | return err; | ||
642 | } | ||
643 | |||
481 | void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset) | 644 | void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset) |
482 | { | 645 | { |
483 | struct gk20a *g = vm->mm->g; | 646 | struct gk20a *g = vm->mm->g; |
@@ -491,6 +654,43 @@ void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset) | |||
491 | return; | 654 | return; |
492 | } | 655 | } |
493 | 656 | ||
494 | nvgpu_ref_put(&mapped_buffer->ref, gk20a_vm_unmap_locked_ref); | 657 | nvgpu_ref_put(&mapped_buffer->ref, nvgpu_vm_unmap_locked_ref); |
495 | nvgpu_mutex_release(&vm->update_gmmu_lock); | 658 | nvgpu_mutex_release(&vm->update_gmmu_lock); |
496 | } | 659 | } |
660 | |||
661 | /* NOTE! mapped_buffers lock must be held */ | ||
662 | void nvgpu_vm_unmap_locked(struct nvgpu_mapped_buf *mapped_buffer, | ||
663 | struct vm_gk20a_mapping_batch *batch) | ||
664 | { | ||
665 | struct vm_gk20a *vm = mapped_buffer->vm; | ||
666 | struct gk20a *g = vm->mm->g; | ||
667 | |||
668 | g->ops.mm.gmmu_unmap(vm, | ||
669 | mapped_buffer->addr, | ||
670 | mapped_buffer->size, | ||
671 | mapped_buffer->pgsz_idx, | ||
672 | mapped_buffer->va_allocated, | ||
673 | gk20a_mem_flag_none, | ||
674 | mapped_buffer->vm_area ? | ||
675 | mapped_buffer->vm_area->sparse : false, | ||
676 | batch); | ||
677 | |||
678 | gk20a_mm_unpin(dev_from_vm(vm), mapped_buffer->dmabuf, | ||
679 | mapped_buffer->sgt); | ||
680 | |||
681 | /* remove from mapped buffer tree and remove list, free */ | ||
682 | nvgpu_remove_mapped_buf(vm, mapped_buffer); | ||
683 | if (!nvgpu_list_empty(&mapped_buffer->buffer_list)) | ||
684 | nvgpu_list_del(&mapped_buffer->buffer_list); | ||
685 | |||
686 | /* keep track of mapped buffers */ | ||
687 | if (mapped_buffer->user_mapped) | ||
688 | vm->num_user_mapped_buffers--; | ||
689 | |||
690 | if (mapped_buffer->own_mem_ref) | ||
691 | dma_buf_put(mapped_buffer->dmabuf); | ||
692 | |||
693 | nvgpu_kfree(g, mapped_buffer); | ||
694 | |||
695 | return; | ||
696 | } | ||
diff --git a/drivers/gpu/nvgpu/common/linux/vm_priv.h b/drivers/gpu/nvgpu/common/linux/vm_priv.h index fa173d59..be7efa8b 100644 --- a/drivers/gpu/nvgpu/common/linux/vm_priv.h +++ b/drivers/gpu/nvgpu/common/linux/vm_priv.h | |||
@@ -88,9 +88,9 @@ int nvgpu_vm_map_buffer(struct vm_gk20a *vm, | |||
88 | void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset); | 88 | void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset); |
89 | 89 | ||
90 | /* find buffer corresponding to va */ | 90 | /* find buffer corresponding to va */ |
91 | int nvgpu_vm_find_buffer(struct vm_gk20a *vm, u64 gpu_va, | 91 | int nvgpu_vm_find_buf(struct vm_gk20a *vm, u64 gpu_va, |
92 | struct dma_buf **dmabuf, | 92 | struct dma_buf **dmabuf, |
93 | u64 *offset); | 93 | u64 *offset); |
94 | 94 | ||
95 | enum nvgpu_aperture gk20a_dmabuf_aperture(struct gk20a *g, | 95 | enum nvgpu_aperture gk20a_dmabuf_aperture(struct gk20a *g, |
96 | struct dma_buf *dmabuf); | 96 | struct dma_buf *dmabuf); |
@@ -98,9 +98,5 @@ int validate_fixed_buffer(struct vm_gk20a *vm, | |||
98 | struct buffer_attrs *bfr, | 98 | struct buffer_attrs *bfr, |
99 | u64 map_offset, u64 map_size, | 99 | u64 map_offset, u64 map_size, |
100 | struct nvgpu_vm_area **pva_node); | 100 | struct nvgpu_vm_area **pva_node); |
101 | int setup_buffer_kind_and_compression(struct vm_gk20a *vm, | ||
102 | u32 flags, | ||
103 | struct buffer_attrs *bfr, | ||
104 | enum gmmu_pgsz_gk20a pgsz_idx); | ||
105 | 101 | ||
106 | #endif | 102 | #endif |
diff --git a/drivers/gpu/nvgpu/common/mm/vm.c b/drivers/gpu/nvgpu/common/mm/vm.c index f8d58349..9f04ee01 100644 --- a/drivers/gpu/nvgpu/common/mm/vm.c +++ b/drivers/gpu/nvgpu/common/mm/vm.c | |||
@@ -641,3 +641,137 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_less_than( | |||
641 | 641 | ||
642 | return mapped_buffer_from_rbtree_node(node); | 642 | return mapped_buffer_from_rbtree_node(node); |
643 | } | 643 | } |
644 | |||
645 | int nvgpu_vm_get_buffers(struct vm_gk20a *vm, | ||
646 | struct nvgpu_mapped_buf ***mapped_buffers, | ||
647 | int *num_buffers) | ||
648 | { | ||
649 | struct nvgpu_mapped_buf *mapped_buffer; | ||
650 | struct nvgpu_mapped_buf **buffer_list; | ||
651 | struct nvgpu_rbtree_node *node = NULL; | ||
652 | int i = 0; | ||
653 | |||
654 | if (vm->userspace_managed) { | ||
655 | *mapped_buffers = NULL; | ||
656 | *num_buffers = 0; | ||
657 | return 0; | ||
658 | } | ||
659 | |||
660 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
661 | |||
662 | buffer_list = nvgpu_big_zalloc(vm->mm->g, sizeof(*buffer_list) * | ||
663 | vm->num_user_mapped_buffers); | ||
664 | if (!buffer_list) { | ||
665 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
666 | return -ENOMEM; | ||
667 | } | ||
668 | |||
669 | nvgpu_rbtree_enum_start(0, &node, vm->mapped_buffers); | ||
670 | while (node) { | ||
671 | mapped_buffer = mapped_buffer_from_rbtree_node(node); | ||
672 | if (mapped_buffer->user_mapped) { | ||
673 | buffer_list[i] = mapped_buffer; | ||
674 | nvgpu_ref_get(&mapped_buffer->ref); | ||
675 | i++; | ||
676 | } | ||
677 | nvgpu_rbtree_enum_next(&node, node); | ||
678 | } | ||
679 | |||
680 | BUG_ON(i != vm->num_user_mapped_buffers); | ||
681 | |||
682 | *num_buffers = vm->num_user_mapped_buffers; | ||
683 | *mapped_buffers = buffer_list; | ||
684 | |||
685 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
686 | |||
687 | return 0; | ||
688 | } | ||
689 | |||
690 | void nvgpu_vm_unmap_locked_ref(struct nvgpu_ref *ref) | ||
691 | { | ||
692 | struct nvgpu_mapped_buf *mapped_buffer = | ||
693 | container_of(ref, struct nvgpu_mapped_buf, ref); | ||
694 | nvgpu_vm_unmap_locked(mapped_buffer, mapped_buffer->vm->kref_put_batch); | ||
695 | } | ||
696 | |||
697 | void nvgpu_vm_put_buffers(struct vm_gk20a *vm, | ||
698 | struct nvgpu_mapped_buf **mapped_buffers, | ||
699 | int num_buffers) | ||
700 | { | ||
701 | int i; | ||
702 | struct vm_gk20a_mapping_batch batch; | ||
703 | |||
704 | if (num_buffers == 0) | ||
705 | return; | ||
706 | |||
707 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
708 | nvgpu_vm_mapping_batch_start(&batch); | ||
709 | vm->kref_put_batch = &batch; | ||
710 | |||
711 | for (i = 0; i < num_buffers; ++i) | ||
712 | nvgpu_ref_put(&mapped_buffers[i]->ref, | ||
713 | nvgpu_vm_unmap_locked_ref); | ||
714 | |||
715 | vm->kref_put_batch = NULL; | ||
716 | nvgpu_vm_mapping_batch_finish_locked(vm, &batch); | ||
717 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
718 | |||
719 | nvgpu_big_free(vm->mm->g, mapped_buffers); | ||
720 | } | ||
721 | |||
722 | static void nvgpu_vm_unmap_user(struct vm_gk20a *vm, u64 offset, | ||
723 | struct vm_gk20a_mapping_batch *batch) | ||
724 | { | ||
725 | struct gk20a *g = vm->mm->g; | ||
726 | struct nvgpu_mapped_buf *mapped_buffer; | ||
727 | |||
728 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
729 | |||
730 | mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, offset); | ||
731 | if (!mapped_buffer) { | ||
732 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
733 | nvgpu_err(g, "invalid addr to unmap 0x%llx", offset); | ||
734 | return; | ||
735 | } | ||
736 | |||
737 | if (mapped_buffer->flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) { | ||
738 | struct nvgpu_timeout timeout; | ||
739 | |||
740 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
741 | |||
742 | nvgpu_timeout_init(vm->mm->g, &timeout, 10000, | ||
743 | NVGPU_TIMER_RETRY_TIMER); | ||
744 | do { | ||
745 | if (nvgpu_atomic_read( | ||
746 | &mapped_buffer->ref.refcount) == 1) | ||
747 | break; | ||
748 | nvgpu_udelay(5); | ||
749 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
750 | "sync-unmap failed on 0x%llx")); | ||
751 | |||
752 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
753 | } | ||
754 | |||
755 | if (mapped_buffer->user_mapped == 0) { | ||
756 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
757 | nvgpu_err(g, "addr already unmapped from user 0x%llx", offset); | ||
758 | return; | ||
759 | } | ||
760 | |||
761 | mapped_buffer->user_mapped--; | ||
762 | if (mapped_buffer->user_mapped == 0) | ||
763 | vm->num_user_mapped_buffers--; | ||
764 | |||
765 | vm->kref_put_batch = batch; | ||
766 | nvgpu_ref_put(&mapped_buffer->ref, nvgpu_vm_unmap_locked_ref); | ||
767 | vm->kref_put_batch = NULL; | ||
768 | |||
769 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
770 | } | ||
771 | |||
772 | int nvgpu_vm_unmap_buffer(struct vm_gk20a *vm, u64 offset, | ||
773 | struct vm_gk20a_mapping_batch *batch) | ||
774 | { | ||
775 | nvgpu_vm_unmap_user(vm, offset, batch); | ||
776 | return 0; | ||
777 | } | ||
diff --git a/drivers/gpu/nvgpu/common/mm/vm_area.c b/drivers/gpu/nvgpu/common/mm/vm_area.c index de1623bc..88758b85 100644 --- a/drivers/gpu/nvgpu/common/mm/vm_area.c +++ b/drivers/gpu/nvgpu/common/mm/vm_area.c | |||
@@ -208,7 +208,7 @@ int nvgpu_vm_area_free(struct vm_gk20a *vm, u64 addr) | |||
208 | &vm_area->buffer_list_head, | 208 | &vm_area->buffer_list_head, |
209 | nvgpu_mapped_buf, buffer_list) { | 209 | nvgpu_mapped_buf, buffer_list) { |
210 | nvgpu_list_del(&buffer->buffer_list); | 210 | nvgpu_list_del(&buffer->buffer_list); |
211 | nvgpu_ref_put(&buffer->ref, gk20a_vm_unmap_locked_ref); | 211 | nvgpu_ref_put(&buffer->ref, nvgpu_vm_unmap_locked_ref); |
212 | } | 212 | } |
213 | 213 | ||
214 | /* if this was a sparse mapping, free the va */ | 214 | /* if this was a sparse mapping, free the va */ |
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index 9d272646..b55f4517 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h | |||
@@ -759,6 +759,8 @@ struct gpu_ops { | |||
759 | struct vm_gk20a *vm, u32 big_page_size); | 759 | struct vm_gk20a *vm, u32 big_page_size); |
760 | bool (*mmu_fault_pending)(struct gk20a *g); | 760 | bool (*mmu_fault_pending)(struct gk20a *g); |
761 | void (*fault_info_mem_destroy)(struct gk20a *g); | 761 | void (*fault_info_mem_destroy)(struct gk20a *g); |
762 | u32 (*get_kind_invalid)(void); | ||
763 | u32 (*get_kind_pitch)(void); | ||
762 | } mm; | 764 | } mm; |
763 | /* | 765 | /* |
764 | * This function is called to allocate secure memory (memory | 766 | * This function is called to allocate secure memory (memory |
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c index 69d9e983..687951a9 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c | |||
@@ -22,11 +22,6 @@ | |||
22 | * DEALINGS IN THE SOFTWARE. | 22 | * DEALINGS IN THE SOFTWARE. |
23 | */ | 23 | */ |
24 | 24 | ||
25 | #include <linux/dma-buf.h> | ||
26 | #include <linux/dma-mapping.h> | ||
27 | #include <linux/dma-attrs.h> | ||
28 | #include <linux/lcm.h> | ||
29 | #include <uapi/linux/nvgpu.h> | ||
30 | #include <trace/events/gk20a.h> | 25 | #include <trace/events/gk20a.h> |
31 | 26 | ||
32 | #include <nvgpu/vm.h> | 27 | #include <nvgpu/vm.h> |
@@ -46,8 +41,6 @@ | |||
46 | #include <nvgpu/enabled.h> | 41 | #include <nvgpu/enabled.h> |
47 | #include <nvgpu/vidmem.h> | 42 | #include <nvgpu/vidmem.h> |
48 | 43 | ||
49 | #include <nvgpu/linux/dma.h> | ||
50 | |||
51 | #include "gk20a.h" | 44 | #include "gk20a.h" |
52 | #include "platform_gk20a.h" | 45 | #include "platform_gk20a.h" |
53 | #include "mm_gk20a.h" | 46 | #include "mm_gk20a.h" |
@@ -65,13 +58,6 @@ | |||
65 | #include <nvgpu/hw/gk20a/hw_ltc_gk20a.h> | 58 | #include <nvgpu/hw/gk20a/hw_ltc_gk20a.h> |
66 | 59 | ||
67 | /* | 60 | /* |
68 | * Necessary while transitioning to less coupled code. Will be removed once | ||
69 | * all the common APIs no longers have Linux stuff in them. | ||
70 | */ | ||
71 | #include "common/linux/vm_priv.h" | ||
72 | #include "common/linux/dmabuf.h" | ||
73 | |||
74 | /* | ||
75 | * GPU mapping life cycle | 61 | * GPU mapping life cycle |
76 | * ====================== | 62 | * ====================== |
77 | * | 63 | * |
@@ -330,209 +316,6 @@ int gk20a_mm_pde_coverage_bit_count(struct vm_gk20a *vm) | |||
330 | return vm->mmu_levels[0].lo_bit[0]; | 316 | return vm->mmu_levels[0].lo_bit[0]; |
331 | } | 317 | } |
332 | 318 | ||
333 | int nvgpu_vm_get_buffers(struct vm_gk20a *vm, | ||
334 | struct nvgpu_mapped_buf ***mapped_buffers, | ||
335 | int *num_buffers) | ||
336 | { | ||
337 | struct nvgpu_mapped_buf *mapped_buffer; | ||
338 | struct nvgpu_mapped_buf **buffer_list; | ||
339 | struct nvgpu_rbtree_node *node = NULL; | ||
340 | int i = 0; | ||
341 | |||
342 | if (vm->userspace_managed) { | ||
343 | *mapped_buffers = NULL; | ||
344 | *num_buffers = 0; | ||
345 | return 0; | ||
346 | } | ||
347 | |||
348 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
349 | |||
350 | buffer_list = nvgpu_big_zalloc(vm->mm->g, sizeof(*buffer_list) * | ||
351 | vm->num_user_mapped_buffers); | ||
352 | if (!buffer_list) { | ||
353 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
354 | return -ENOMEM; | ||
355 | } | ||
356 | |||
357 | nvgpu_rbtree_enum_start(0, &node, vm->mapped_buffers); | ||
358 | while (node) { | ||
359 | mapped_buffer = mapped_buffer_from_rbtree_node(node); | ||
360 | if (mapped_buffer->user_mapped) { | ||
361 | buffer_list[i] = mapped_buffer; | ||
362 | nvgpu_ref_get(&mapped_buffer->ref); | ||
363 | i++; | ||
364 | } | ||
365 | nvgpu_rbtree_enum_next(&node, node); | ||
366 | } | ||
367 | |||
368 | BUG_ON(i != vm->num_user_mapped_buffers); | ||
369 | |||
370 | *num_buffers = vm->num_user_mapped_buffers; | ||
371 | *mapped_buffers = buffer_list; | ||
372 | |||
373 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
374 | |||
375 | return 0; | ||
376 | } | ||
377 | |||
378 | void gk20a_vm_unmap_locked_ref(struct nvgpu_ref *ref) | ||
379 | { | ||
380 | struct nvgpu_mapped_buf *mapped_buffer = | ||
381 | container_of(ref, struct nvgpu_mapped_buf, ref); | ||
382 | nvgpu_vm_unmap_locked(mapped_buffer, mapped_buffer->vm->kref_put_batch); | ||
383 | } | ||
384 | |||
385 | void nvgpu_vm_put_buffers(struct vm_gk20a *vm, | ||
386 | struct nvgpu_mapped_buf **mapped_buffers, | ||
387 | int num_buffers) | ||
388 | { | ||
389 | int i; | ||
390 | struct vm_gk20a_mapping_batch batch; | ||
391 | |||
392 | if (num_buffers == 0) | ||
393 | return; | ||
394 | |||
395 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
396 | nvgpu_vm_mapping_batch_start(&batch); | ||
397 | vm->kref_put_batch = &batch; | ||
398 | |||
399 | for (i = 0; i < num_buffers; ++i) | ||
400 | nvgpu_ref_put(&mapped_buffers[i]->ref, | ||
401 | gk20a_vm_unmap_locked_ref); | ||
402 | |||
403 | vm->kref_put_batch = NULL; | ||
404 | nvgpu_vm_mapping_batch_finish_locked(vm, &batch); | ||
405 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
406 | |||
407 | nvgpu_big_free(vm->mm->g, mapped_buffers); | ||
408 | } | ||
409 | |||
410 | static void nvgpu_vm_unmap_user(struct vm_gk20a *vm, u64 offset, | ||
411 | struct vm_gk20a_mapping_batch *batch) | ||
412 | { | ||
413 | struct gk20a *g = vm->mm->g; | ||
414 | struct nvgpu_mapped_buf *mapped_buffer; | ||
415 | |||
416 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
417 | |||
418 | mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, offset); | ||
419 | if (!mapped_buffer) { | ||
420 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
421 | nvgpu_err(g, "invalid addr to unmap 0x%llx", offset); | ||
422 | return; | ||
423 | } | ||
424 | |||
425 | if (mapped_buffer->flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) { | ||
426 | struct nvgpu_timeout timeout; | ||
427 | |||
428 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
429 | |||
430 | nvgpu_timeout_init(vm->mm->g, &timeout, 10000, | ||
431 | NVGPU_TIMER_RETRY_TIMER); | ||
432 | do { | ||
433 | if (nvgpu_atomic_read( | ||
434 | &mapped_buffer->ref.refcount) == 1) | ||
435 | break; | ||
436 | nvgpu_udelay(5); | ||
437 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
438 | "sync-unmap failed on 0x%llx")); | ||
439 | |||
440 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
441 | } | ||
442 | |||
443 | if (mapped_buffer->user_mapped == 0) { | ||
444 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
445 | nvgpu_err(g, "addr already unmapped from user 0x%llx", offset); | ||
446 | return; | ||
447 | } | ||
448 | |||
449 | mapped_buffer->user_mapped--; | ||
450 | if (mapped_buffer->user_mapped == 0) | ||
451 | vm->num_user_mapped_buffers--; | ||
452 | |||
453 | vm->kref_put_batch = batch; | ||
454 | nvgpu_ref_put(&mapped_buffer->ref, gk20a_vm_unmap_locked_ref); | ||
455 | vm->kref_put_batch = NULL; | ||
456 | |||
457 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
458 | } | ||
459 | |||
460 | static int setup_kind_legacy(struct vm_gk20a *vm, struct buffer_attrs *bfr, | ||
461 | bool *pkind_compressible) | ||
462 | { | ||
463 | struct gk20a *g = gk20a_from_vm(vm); | ||
464 | bool kind_compressible; | ||
465 | |||
466 | if (unlikely(bfr->kind_v == gmmu_pte_kind_invalid_v())) | ||
467 | bfr->kind_v = gmmu_pte_kind_pitch_v(); | ||
468 | |||
469 | if (unlikely(!gk20a_kind_is_supported(bfr->kind_v))) { | ||
470 | nvgpu_err(g, "kind 0x%x not supported", bfr->kind_v); | ||
471 | return -EINVAL; | ||
472 | } | ||
473 | |||
474 | bfr->uc_kind_v = gmmu_pte_kind_invalid_v(); | ||
475 | /* find a suitable incompressible kind if it becomes necessary later */ | ||
476 | kind_compressible = gk20a_kind_is_compressible(bfr->kind_v); | ||
477 | if (kind_compressible) { | ||
478 | bfr->uc_kind_v = gk20a_get_uncompressed_kind(bfr->kind_v); | ||
479 | if (unlikely(bfr->uc_kind_v == gmmu_pte_kind_invalid_v())) { | ||
480 | /* shouldn't happen, but it is worth cross-checking */ | ||
481 | nvgpu_err(g, "comptag kind 0x%x can't be" | ||
482 | " downgraded to uncompressed kind", | ||
483 | bfr->kind_v); | ||
484 | return -EINVAL; | ||
485 | } | ||
486 | } | ||
487 | |||
488 | *pkind_compressible = kind_compressible; | ||
489 | return 0; | ||
490 | } | ||
491 | |||
492 | int setup_buffer_kind_and_compression(struct vm_gk20a *vm, | ||
493 | u32 flags, | ||
494 | struct buffer_attrs *bfr, | ||
495 | enum gmmu_pgsz_gk20a pgsz_idx) | ||
496 | { | ||
497 | bool kind_compressible; | ||
498 | struct gk20a *g = gk20a_from_vm(vm); | ||
499 | int ctag_granularity = g->ops.fb.compression_page_size(g); | ||
500 | |||
501 | if (!bfr->use_kind_v) | ||
502 | bfr->kind_v = gmmu_pte_kind_invalid_v(); | ||
503 | if (!bfr->use_uc_kind_v) | ||
504 | bfr->uc_kind_v = gmmu_pte_kind_invalid_v(); | ||
505 | |||
506 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL) { | ||
507 | kind_compressible = (bfr->kind_v != gmmu_pte_kind_invalid_v()); | ||
508 | if (!kind_compressible) | ||
509 | bfr->kind_v = bfr->uc_kind_v; | ||
510 | } else { | ||
511 | int err = setup_kind_legacy(vm, bfr, &kind_compressible); | ||
512 | |||
513 | if (err) | ||
514 | return err; | ||
515 | } | ||
516 | |||
517 | /* comptags only supported for suitable kinds, 128KB pagesize */ | ||
518 | if (kind_compressible && | ||
519 | vm->gmmu_page_sizes[pgsz_idx] < g->ops.fb.compressible_page_size(g)) { | ||
520 | /* it is safe to fall back to uncompressed as | ||
521 | functionality is not harmed */ | ||
522 | bfr->kind_v = bfr->uc_kind_v; | ||
523 | kind_compressible = false; | ||
524 | } | ||
525 | if (kind_compressible) | ||
526 | bfr->ctag_lines = DIV_ROUND_UP_ULL(bfr->size, ctag_granularity); | ||
527 | else | ||
528 | bfr->ctag_lines = 0; | ||
529 | |||
530 | bfr->use_kind_v = (bfr->kind_v != gmmu_pte_kind_invalid_v()); | ||
531 | bfr->use_uc_kind_v = (bfr->uc_kind_v != gmmu_pte_kind_invalid_v()); | ||
532 | |||
533 | return 0; | ||
534 | } | ||
535 | |||
536 | /* for gk20a the "video memory" apertures here are misnomers. */ | 319 | /* for gk20a the "video memory" apertures here are misnomers. */ |
537 | static inline u32 big_valid_pde0_bits(struct gk20a *g, | 320 | static inline u32 big_valid_pde0_bits(struct gk20a *g, |
538 | struct nvgpu_gmmu_pd *pd, u64 addr) | 321 | struct nvgpu_gmmu_pd *pd, u64 addr) |
@@ -698,43 +481,6 @@ static void update_gmmu_pte_locked(struct vm_gk20a *vm, | |||
698 | pd_write(g, pd, pd_offset + 1, pte_w[1]); | 481 | pd_write(g, pd, pd_offset + 1, pte_w[1]); |
699 | } | 482 | } |
700 | 483 | ||
701 | /* NOTE! mapped_buffers lock must be held */ | ||
702 | void nvgpu_vm_unmap_locked(struct nvgpu_mapped_buf *mapped_buffer, | ||
703 | struct vm_gk20a_mapping_batch *batch) | ||
704 | { | ||
705 | struct vm_gk20a *vm = mapped_buffer->vm; | ||
706 | struct gk20a *g = vm->mm->g; | ||
707 | |||
708 | g->ops.mm.gmmu_unmap(vm, | ||
709 | mapped_buffer->addr, | ||
710 | mapped_buffer->size, | ||
711 | mapped_buffer->pgsz_idx, | ||
712 | mapped_buffer->va_allocated, | ||
713 | gk20a_mem_flag_none, | ||
714 | mapped_buffer->vm_area ? | ||
715 | mapped_buffer->vm_area->sparse : false, | ||
716 | batch); | ||
717 | |||
718 | gk20a_mm_unpin(dev_from_vm(vm), mapped_buffer->dmabuf, | ||
719 | mapped_buffer->sgt); | ||
720 | |||
721 | /* remove from mapped buffer tree and remove list, free */ | ||
722 | nvgpu_remove_mapped_buf(vm, mapped_buffer); | ||
723 | if (!nvgpu_list_empty(&mapped_buffer->buffer_list)) | ||
724 | nvgpu_list_del(&mapped_buffer->buffer_list); | ||
725 | |||
726 | /* keep track of mapped buffers */ | ||
727 | if (mapped_buffer->user_mapped) | ||
728 | vm->num_user_mapped_buffers--; | ||
729 | |||
730 | if (mapped_buffer->own_mem_ref) | ||
731 | dma_buf_put(mapped_buffer->dmabuf); | ||
732 | |||
733 | nvgpu_kfree(g, mapped_buffer); | ||
734 | |||
735 | return; | ||
736 | } | ||
737 | |||
738 | const struct gk20a_mmu_level gk20a_mm_levels_64k[] = { | 484 | const struct gk20a_mmu_level gk20a_mm_levels_64k[] = { |
739 | {.hi_bit = {NV_GMMU_VA_RANGE-1, NV_GMMU_VA_RANGE-1}, | 485 | {.hi_bit = {NV_GMMU_VA_RANGE-1, NV_GMMU_VA_RANGE-1}, |
740 | .lo_bit = {26, 26}, | 486 | .lo_bit = {26, 26}, |
@@ -853,76 +599,6 @@ int gk20a_vm_bind_channel(struct gk20a_as_share *as_share, | |||
853 | return __gk20a_vm_bind_channel(as_share->vm, ch); | 599 | return __gk20a_vm_bind_channel(as_share->vm, ch); |
854 | } | 600 | } |
855 | 601 | ||
856 | int nvgpu_vm_map_buffer(struct vm_gk20a *vm, | ||
857 | int dmabuf_fd, | ||
858 | u64 *offset_align, | ||
859 | u32 flags, /*NVGPU_AS_MAP_BUFFER_FLAGS_*/ | ||
860 | s16 compr_kind, | ||
861 | s16 incompr_kind, | ||
862 | u64 buffer_offset, | ||
863 | u64 mapping_size, | ||
864 | struct vm_gk20a_mapping_batch *batch) | ||
865 | { | ||
866 | int err = 0; | ||
867 | struct dma_buf *dmabuf; | ||
868 | u64 ret_va; | ||
869 | |||
870 | gk20a_dbg_fn(""); | ||
871 | |||
872 | /* get ref to the mem handle (released on unmap_locked) */ | ||
873 | dmabuf = dma_buf_get(dmabuf_fd); | ||
874 | if (IS_ERR(dmabuf)) { | ||
875 | nvgpu_warn(gk20a_from_vm(vm), "%s: fd %d is not a dmabuf", | ||
876 | __func__, dmabuf_fd); | ||
877 | return PTR_ERR(dmabuf); | ||
878 | } | ||
879 | |||
880 | /* verify that we're not overflowing the buffer, i.e. | ||
881 | * (buffer_offset + mapping_size)> dmabuf->size. | ||
882 | * | ||
883 | * Since buffer_offset + mapping_size could overflow, first check | ||
884 | * that mapping size < dmabuf_size, at which point we can subtract | ||
885 | * mapping_size from both sides for the final comparison. | ||
886 | */ | ||
887 | if ((mapping_size > dmabuf->size) || | ||
888 | (buffer_offset > (dmabuf->size - mapping_size))) { | ||
889 | nvgpu_err(gk20a_from_vm(vm), | ||
890 | "buf size %llx < (offset(%llx) + map_size(%llx))\n", | ||
891 | (u64)dmabuf->size, buffer_offset, mapping_size); | ||
892 | return -EINVAL; | ||
893 | } | ||
894 | |||
895 | err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev_from_vm(vm)); | ||
896 | if (err) { | ||
897 | dma_buf_put(dmabuf); | ||
898 | return err; | ||
899 | } | ||
900 | |||
901 | ret_va = nvgpu_vm_map(vm, dmabuf, *offset_align, | ||
902 | flags, compr_kind, incompr_kind, true, | ||
903 | gk20a_mem_flag_none, | ||
904 | buffer_offset, | ||
905 | mapping_size, | ||
906 | batch); | ||
907 | |||
908 | *offset_align = ret_va; | ||
909 | if (!ret_va) { | ||
910 | dma_buf_put(dmabuf); | ||
911 | err = -EINVAL; | ||
912 | } | ||
913 | |||
914 | return err; | ||
915 | } | ||
916 | |||
917 | int nvgpu_vm_unmap_buffer(struct vm_gk20a *vm, u64 offset, | ||
918 | struct vm_gk20a_mapping_batch *batch) | ||
919 | { | ||
920 | gk20a_dbg_fn(""); | ||
921 | |||
922 | nvgpu_vm_unmap_user(vm, offset, batch); | ||
923 | return 0; | ||
924 | } | ||
925 | |||
926 | int gk20a_alloc_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block) | 602 | int gk20a_alloc_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block) |
927 | { | 603 | { |
928 | int err; | 604 | int err; |
@@ -1298,30 +974,6 @@ hw_was_off: | |||
1298 | gk20a_idle_nosuspend(g); | 974 | gk20a_idle_nosuspend(g); |
1299 | } | 975 | } |
1300 | 976 | ||
1301 | int nvgpu_vm_find_buf(struct vm_gk20a *vm, u64 gpu_va, | ||
1302 | struct dma_buf **dmabuf, | ||
1303 | u64 *offset) | ||
1304 | { | ||
1305 | struct nvgpu_mapped_buf *mapped_buffer; | ||
1306 | |||
1307 | gk20a_dbg_fn("gpu_va=0x%llx", gpu_va); | ||
1308 | |||
1309 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
1310 | |||
1311 | mapped_buffer = __nvgpu_vm_find_mapped_buf_range(vm, gpu_va); | ||
1312 | if (!mapped_buffer) { | ||
1313 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
1314 | return -EINVAL; | ||
1315 | } | ||
1316 | |||
1317 | *dmabuf = mapped_buffer->dmabuf; | ||
1318 | *offset = gpu_va - mapped_buffer->addr; | ||
1319 | |||
1320 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
1321 | |||
1322 | return 0; | ||
1323 | } | ||
1324 | |||
1325 | int gk20a_mm_suspend(struct gk20a *g) | 977 | int gk20a_mm_suspend(struct gk20a *g) |
1326 | { | 978 | { |
1327 | gk20a_dbg_fn(""); | 979 | gk20a_dbg_fn(""); |
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h index 04034d84..556cb234 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h | |||
@@ -386,6 +386,5 @@ extern const struct gk20a_mmu_level gk20a_mm_levels_128k[]; | |||
386 | 386 | ||
387 | int gk20a_mm_get_buffer_info(struct device *dev, int dmabuf_fd, | 387 | int gk20a_mm_get_buffer_info(struct device *dev, int dmabuf_fd, |
388 | u64 *buffer_id, u64 *buffer_len); | 388 | u64 *buffer_id, u64 *buffer_len); |
389 | void gk20a_vm_unmap_locked_ref(struct nvgpu_ref *ref); | ||
390 | 389 | ||
391 | #endif /* MM_GK20A_H */ | 390 | #endif /* MM_GK20A_H */ |
diff --git a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c index 50cc759a..e146d3ca 100644 --- a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c | |||
@@ -443,6 +443,8 @@ static const struct gpu_ops gm20b_ops = { | |||
443 | .is_bar1_supported = gm20b_mm_is_bar1_supported, | 443 | .is_bar1_supported = gm20b_mm_is_bar1_supported, |
444 | .init_inst_block = gk20a_init_inst_block, | 444 | .init_inst_block = gk20a_init_inst_block, |
445 | .mmu_fault_pending = gk20a_fifo_mmu_fault_pending, | 445 | .mmu_fault_pending = gk20a_fifo_mmu_fault_pending, |
446 | .get_kind_invalid = gm20b_get_kind_invalid, | ||
447 | .get_kind_pitch = gm20b_get_kind_pitch, | ||
446 | }, | 448 | }, |
447 | .therm = { | 449 | .therm = { |
448 | .init_therm_setup_hw = gm20b_init_therm_setup_hw, | 450 | .init_therm_setup_hw = gm20b_init_therm_setup_hw, |
diff --git a/drivers/gpu/nvgpu/gm20b/mm_gm20b.c b/drivers/gpu/nvgpu/gm20b/mm_gm20b.c index dc332075..5cd7706d 100644 --- a/drivers/gpu/nvgpu/gm20b/mm_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/mm_gm20b.c | |||
@@ -74,3 +74,13 @@ u64 gm20b_gpu_phys_addr(struct gk20a *g, | |||
74 | { | 74 | { |
75 | return phys; | 75 | return phys; |
76 | } | 76 | } |
77 | |||
78 | u32 gm20b_get_kind_invalid(void) | ||
79 | { | ||
80 | return gmmu_pte_kind_invalid_v(); | ||
81 | } | ||
82 | |||
83 | u32 gm20b_get_kind_pitch(void) | ||
84 | { | ||
85 | return gmmu_pte_kind_pitch_v(); | ||
86 | } | ||
diff --git a/drivers/gpu/nvgpu/gm20b/mm_gm20b.h b/drivers/gpu/nvgpu/gm20b/mm_gm20b.h index 602e18dc..af67845a 100644 --- a/drivers/gpu/nvgpu/gm20b/mm_gm20b.h +++ b/drivers/gpu/nvgpu/gm20b/mm_gm20b.h | |||
@@ -38,4 +38,6 @@ bool gm20b_mm_is_bar1_supported(struct gk20a *g); | |||
38 | int gm20b_mm_mmu_vpr_info_fetch(struct gk20a *g); | 38 | int gm20b_mm_mmu_vpr_info_fetch(struct gk20a *g); |
39 | u64 gm20b_gpu_phys_addr(struct gk20a *g, | 39 | u64 gm20b_gpu_phys_addr(struct gk20a *g, |
40 | struct nvgpu_gmmu_attrs *attrs, u64 phys); | 40 | struct nvgpu_gmmu_attrs *attrs, u64 phys); |
41 | u32 gm20b_get_kind_invalid(void); | ||
42 | u32 gm20b_get_kind_pitch(void); | ||
41 | #endif | 43 | #endif |
diff --git a/drivers/gpu/nvgpu/gp106/hal_gp106.c b/drivers/gpu/nvgpu/gp106/hal_gp106.c index 204c376b..54960b96 100644 --- a/drivers/gpu/nvgpu/gp106/hal_gp106.c +++ b/drivers/gpu/nvgpu/gp106/hal_gp106.c | |||
@@ -529,6 +529,8 @@ static const struct gpu_ops gp106_ops = { | |||
529 | .init_bar2_mm_hw_setup = gb10b_init_bar2_mm_hw_setup, | 529 | .init_bar2_mm_hw_setup = gb10b_init_bar2_mm_hw_setup, |
530 | .remove_bar2_vm = gp10b_remove_bar2_vm, | 530 | .remove_bar2_vm = gp10b_remove_bar2_vm, |
531 | .get_vidmem_size = gp106_mm_get_vidmem_size, | 531 | .get_vidmem_size = gp106_mm_get_vidmem_size, |
532 | .get_kind_invalid = gm20b_get_kind_invalid, | ||
533 | .get_kind_pitch = gm20b_get_kind_pitch, | ||
532 | }, | 534 | }, |
533 | .pramin = { | 535 | .pramin = { |
534 | .enter = gk20a_pramin_enter, | 536 | .enter = gk20a_pramin_enter, |
diff --git a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c index 90e8137b..b40ce4a1 100644 --- a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c | |||
@@ -488,6 +488,8 @@ static const struct gpu_ops gp10b_ops = { | |||
488 | .init_bar2_vm = gb10b_init_bar2_vm, | 488 | .init_bar2_vm = gb10b_init_bar2_vm, |
489 | .init_bar2_mm_hw_setup = gb10b_init_bar2_mm_hw_setup, | 489 | .init_bar2_mm_hw_setup = gb10b_init_bar2_mm_hw_setup, |
490 | .remove_bar2_vm = gp10b_remove_bar2_vm, | 490 | .remove_bar2_vm = gp10b_remove_bar2_vm, |
491 | .get_kind_invalid = gm20b_get_kind_invalid, | ||
492 | .get_kind_pitch = gm20b_get_kind_pitch, | ||
491 | }, | 493 | }, |
492 | .pramin = { | 494 | .pramin = { |
493 | .enter = gk20a_pramin_enter, | 495 | .enter = gk20a_pramin_enter, |
diff --git a/drivers/gpu/nvgpu/include/nvgpu/vm.h b/drivers/gpu/nvgpu/include/nvgpu/vm.h index f8868065..8c56461c 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/vm.h +++ b/drivers/gpu/nvgpu/include/nvgpu/vm.h | |||
@@ -31,9 +31,10 @@ | |||
31 | #include <nvgpu/nvgpu_mem.h> | 31 | #include <nvgpu/nvgpu_mem.h> |
32 | #include <nvgpu/allocator.h> | 32 | #include <nvgpu/allocator.h> |
33 | 33 | ||
34 | struct dma_buf; | ||
35 | |||
34 | struct vm_gk20a; | 36 | struct vm_gk20a; |
35 | struct nvgpu_vm_area; | 37 | struct nvgpu_vm_area; |
36 | struct buffer_attrs; | ||
37 | struct gk20a_comptag_allocator; | 38 | struct gk20a_comptag_allocator; |
38 | 39 | ||
39 | /** | 40 | /** |
@@ -212,6 +213,7 @@ int nvgpu_vm_unmap_buffer(struct vm_gk20a *vm, u64 offset, | |||
212 | 213 | ||
213 | void nvgpu_vm_unmap_locked(struct nvgpu_mapped_buf *mapped_buffer, | 214 | void nvgpu_vm_unmap_locked(struct nvgpu_mapped_buf *mapped_buffer, |
214 | struct vm_gk20a_mapping_batch *batch); | 215 | struct vm_gk20a_mapping_batch *batch); |
216 | void nvgpu_vm_unmap_locked_ref(struct nvgpu_ref *ref); | ||
215 | 217 | ||
216 | /* | 218 | /* |
217 | * These all require the VM update lock to be held. | 219 | * These all require the VM update lock to be held. |
@@ -223,9 +225,6 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_range( | |||
223 | struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_less_than( | 225 | struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_less_than( |
224 | struct vm_gk20a *vm, u64 addr); | 226 | struct vm_gk20a *vm, u64 addr); |
225 | 227 | ||
226 | int nvgpu_vm_find_buf(struct vm_gk20a *vm, u64 gpu_va, | ||
227 | struct dma_buf **dmabuf, | ||
228 | u64 *offset); | ||
229 | int nvgpu_insert_mapped_buf(struct vm_gk20a *vm, | 228 | int nvgpu_insert_mapped_buf(struct vm_gk20a *vm, |
230 | struct nvgpu_mapped_buf *mapped_buffer); | 229 | struct nvgpu_mapped_buf *mapped_buffer); |
231 | void nvgpu_remove_mapped_buf(struct vm_gk20a *vm, | 230 | void nvgpu_remove_mapped_buf(struct vm_gk20a *vm, |
diff --git a/drivers/gpu/nvgpu/vgpu/gm20b/vgpu_hal_gm20b.c b/drivers/gpu/nvgpu/vgpu/gm20b/vgpu_hal_gm20b.c index fc9b0e83..38d07ee2 100644 --- a/drivers/gpu/nvgpu/vgpu/gm20b/vgpu_hal_gm20b.c +++ b/drivers/gpu/nvgpu/vgpu/gm20b/vgpu_hal_gm20b.c | |||
@@ -342,6 +342,8 @@ static const struct gpu_ops vgpu_gm20b_ops = { | |||
342 | .is_bar1_supported = gm20b_mm_is_bar1_supported, | 342 | .is_bar1_supported = gm20b_mm_is_bar1_supported, |
343 | .init_inst_block = gk20a_init_inst_block, | 343 | .init_inst_block = gk20a_init_inst_block, |
344 | .mmu_fault_pending = gk20a_fifo_mmu_fault_pending, | 344 | .mmu_fault_pending = gk20a_fifo_mmu_fault_pending, |
345 | .get_kind_invalid = gm20b_get_kind_invalid, | ||
346 | .get_kind_pitch = gm20b_get_kind_pitch, | ||
345 | }, | 347 | }, |
346 | .therm = { | 348 | .therm = { |
347 | .init_therm_setup_hw = gm20b_init_therm_setup_hw, | 349 | .init_therm_setup_hw = gm20b_init_therm_setup_hw, |
diff --git a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c index 2c59d847..7f18bfe4 100644 --- a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c +++ b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c | |||
@@ -392,6 +392,8 @@ static const struct gpu_ops vgpu_gp10b_ops = { | |||
392 | .init_bar2_vm = gb10b_init_bar2_vm, | 392 | .init_bar2_vm = gb10b_init_bar2_vm, |
393 | .init_bar2_mm_hw_setup = gb10b_init_bar2_mm_hw_setup, | 393 | .init_bar2_mm_hw_setup = gb10b_init_bar2_mm_hw_setup, |
394 | .remove_bar2_vm = gp10b_remove_bar2_vm, | 394 | .remove_bar2_vm = gp10b_remove_bar2_vm, |
395 | .get_kind_invalid = gm20b_get_kind_invalid, | ||
396 | .get_kind_pitch = gm20b_get_kind_pitch, | ||
395 | }, | 397 | }, |
396 | .pramin = { | 398 | .pramin = { |
397 | .enter = gk20a_pramin_enter, | 399 | .enter = gk20a_pramin_enter, |