summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common')
-rw-r--r--drivers/gpu/nvgpu/common/linux/ioctl_as.c21
-rw-r--r--drivers/gpu/nvgpu/common/linux/vm_priv.h62
-rw-r--r--drivers/gpu/nvgpu/common/mm/vm.c129
-rw-r--r--drivers/gpu/nvgpu/common/mm/vm_area.c223
4 files changed, 425 insertions, 10 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/ioctl_as.c b/drivers/gpu/nvgpu/common/linux/ioctl_as.c
index 4bbcedda..7a24a14f 100644
--- a/drivers/gpu/nvgpu/common/linux/ioctl_as.c
+++ b/drivers/gpu/nvgpu/common/linux/ioctl_as.c
@@ -28,6 +28,7 @@
28#include "gk20a/gk20a.h" 28#include "gk20a/gk20a.h"
29#include "gk20a/platform_gk20a.h" 29#include "gk20a/platform_gk20a.h"
30#include "ioctl_as.h" 30#include "ioctl_as.h"
31#include "vm_priv.h"
31 32
32static int gk20a_as_ioctl_bind_channel( 33static int gk20a_as_ioctl_bind_channel(
33 struct gk20a_as_share *as_share, 34 struct gk20a_as_share *as_share,
@@ -72,7 +73,7 @@ static int gk20a_as_ioctl_map_buffer_ex(
72{ 73{
73 gk20a_dbg_fn(""); 74 gk20a_dbg_fn("");
74 75
75 return gk20a_vm_map_buffer(as_share->vm, args->dmabuf_fd, 76 return nvgpu_vm_map_buffer(as_share->vm, args->dmabuf_fd,
76 &args->offset, args->flags, 77 &args->offset, args->flags,
77 args->kind, 78 args->kind,
78 args->buffer_offset, 79 args->buffer_offset,
@@ -85,7 +86,7 @@ static int gk20a_as_ioctl_map_buffer(
85 struct nvgpu_as_map_buffer_args *args) 86 struct nvgpu_as_map_buffer_args *args)
86{ 87{
87 gk20a_dbg_fn(""); 88 gk20a_dbg_fn("");
88 return gk20a_vm_map_buffer(as_share->vm, args->dmabuf_fd, 89 return nvgpu_vm_map_buffer(as_share->vm, args->dmabuf_fd,
89 &args->o_a.offset, 90 &args->o_a.offset,
90 args->flags, NV_KIND_DEFAULT, 91 args->flags, NV_KIND_DEFAULT,
91 0, 0, NULL); 92 0, 0, NULL);
@@ -97,7 +98,7 @@ static int gk20a_as_ioctl_unmap_buffer(
97 struct nvgpu_as_unmap_buffer_args *args) 98 struct nvgpu_as_unmap_buffer_args *args)
98{ 99{
99 gk20a_dbg_fn(""); 100 gk20a_dbg_fn("");
100 return gk20a_vm_unmap_buffer(as_share->vm, args->offset, NULL); 101 return nvgpu_vm_unmap_buffer(as_share->vm, args->offset, NULL);
101} 102}
102 103
103static int gk20a_as_ioctl_map_buffer_batch( 104static int gk20a_as_ioctl_map_buffer_batch(
@@ -123,7 +124,7 @@ static int gk20a_as_ioctl_map_buffer_batch(
123 args->num_maps > g->gpu_characteristics.map_buffer_batch_limit) 124 args->num_maps > g->gpu_characteristics.map_buffer_batch_limit)
124 return -EINVAL; 125 return -EINVAL;
125 126
126 gk20a_vm_mapping_batch_start(&batch); 127 nvgpu_vm_mapping_batch_start(&batch);
127 128
128 for (i = 0; i < args->num_unmaps; ++i) { 129 for (i = 0; i < args->num_unmaps; ++i) {
129 struct nvgpu_as_unmap_buffer_args unmap_args; 130 struct nvgpu_as_unmap_buffer_args unmap_args;
@@ -134,14 +135,14 @@ static int gk20a_as_ioctl_map_buffer_batch(
134 break; 135 break;
135 } 136 }
136 137
137 err = gk20a_vm_unmap_buffer(as_share->vm, unmap_args.offset, 138 err = nvgpu_vm_unmap_buffer(as_share->vm, unmap_args.offset,
138 &batch); 139 &batch);
139 if (err) 140 if (err)
140 break; 141 break;
141 } 142 }
142 143
143 if (err) { 144 if (err) {
144 gk20a_vm_mapping_batch_finish(as_share->vm, &batch); 145 nvgpu_vm_mapping_batch_finish(as_share->vm, &batch);
145 146
146 args->num_unmaps = i; 147 args->num_unmaps = i;
147 args->num_maps = 0; 148 args->num_maps = 0;
@@ -158,7 +159,7 @@ static int gk20a_as_ioctl_map_buffer_batch(
158 break; 159 break;
159 } 160 }
160 161
161 err = gk20a_vm_map_buffer( 162 err = nvgpu_vm_map_buffer(
162 as_share->vm, map_args.dmabuf_fd, 163 as_share->vm, map_args.dmabuf_fd,
163 &map_args.offset, map_args.flags, 164 &map_args.offset, map_args.flags,
164 map_args.kind, 165 map_args.kind,
@@ -169,7 +170,7 @@ static int gk20a_as_ioctl_map_buffer_batch(
169 break; 170 break;
170 } 171 }
171 172
172 gk20a_vm_mapping_batch_finish(as_share->vm, &batch); 173 nvgpu_vm_mapping_batch_finish(as_share->vm, &batch);
173 174
174 if (err) 175 if (err)
175 args->num_maps = i; 176 args->num_maps = i;
@@ -228,7 +229,7 @@ static int gk20a_as_ioctl_get_buffer_compbits_info(
228 struct nvgpu_as_get_buffer_compbits_info_args *args) 229 struct nvgpu_as_get_buffer_compbits_info_args *args)
229{ 230{
230 gk20a_dbg_fn(""); 231 gk20a_dbg_fn("");
231 return gk20a_vm_get_compbits_info(as_share->vm, 232 return nvgpu_vm_get_compbits_info(as_share->vm,
232 args->mapping_gva, 233 args->mapping_gva,
233 &args->compbits_win_size, 234 &args->compbits_win_size,
234 &args->compbits_win_ctagline, 235 &args->compbits_win_ctagline,
@@ -241,7 +242,7 @@ static int gk20a_as_ioctl_map_buffer_compbits(
241 struct nvgpu_as_map_buffer_compbits_args *args) 242 struct nvgpu_as_map_buffer_compbits_args *args)
242{ 243{
243 gk20a_dbg_fn(""); 244 gk20a_dbg_fn("");
244 return gk20a_vm_map_compbits(as_share->vm, 245 return nvgpu_vm_map_compbits(as_share->vm,
245 args->mapping_gva, 246 args->mapping_gva,
246 &args->compbits_win_gva, 247 &args->compbits_win_gva,
247 &args->mapping_iova, 248 &args->mapping_iova,
diff --git a/drivers/gpu/nvgpu/common/linux/vm_priv.h b/drivers/gpu/nvgpu/common/linux/vm_priv.h
new file mode 100644
index 00000000..c0fb0ffe
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/vm_priv.h
@@ -0,0 +1,62 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#ifndef __COMMON_LINUX_VM_PRIV_H__
18#define __COMMON_LINUX_VM_PRIV_H__
19
20#include <nvgpu/types.h>
21
22struct sg_table;
23struct dma_buf;
24
25struct vm_gk20a;
26struct vm_gk20a_mapping_batch;
27
28u64 nvgpu_vm_map(struct vm_gk20a *vm,
29 struct dma_buf *dmabuf,
30 u64 offset_align,
31 u32 flags /*NVGPU_AS_MAP_BUFFER_FLAGS_*/,
32 int kind,
33 struct sg_table **sgt,
34 bool user_mapped,
35 int rw_flag,
36 u64 buffer_offset,
37 u64 mapping_size,
38 struct vm_gk20a_mapping_batch *mapping_batch);
39
40int nvgpu_vm_map_compbits(struct vm_gk20a *vm,
41 u64 mapping_gva,
42 u64 *compbits_win_gva,
43 u64 *mapping_iova,
44 u32 flags);
45
46/* Note: batch may be NULL if map op is not part of a batch */
47int nvgpu_vm_map_buffer(struct vm_gk20a *vm,
48 int dmabuf_fd,
49 u64 *offset_align,
50 u32 flags, /* NVGPU_AS_MAP_BUFFER_FLAGS_ */
51 int kind,
52 u64 buffer_offset,
53 u64 mapping_size,
54 struct vm_gk20a_mapping_batch *batch);
55
56void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset);
57
58/* find buffer corresponding to va */
59int nvgpu_vm_find_buffer(struct vm_gk20a *vm, u64 gpu_va,
60 struct dma_buf **dmabuf,
61 u64 *offset);
62#endif
diff --git a/drivers/gpu/nvgpu/common/mm/vm.c b/drivers/gpu/nvgpu/common/mm/vm.c
new file mode 100644
index 00000000..eaf30fd0
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/vm.c
@@ -0,0 +1,129 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#include <nvgpu/vm.h>
18#include <nvgpu/lock.h>
19#include <nvgpu/list.h>
20#include <nvgpu/rbtree.h>
21#include <nvgpu/semaphore.h>
22
23#include "gk20a/gk20a.h"
24#include "gk20a/mm_gk20a.h"
25
26void nvgpu_vm_mapping_batch_start(struct vm_gk20a_mapping_batch *mapping_batch)
27{
28 memset(mapping_batch, 0, sizeof(*mapping_batch));
29 mapping_batch->gpu_l2_flushed = false;
30 mapping_batch->need_tlb_invalidate = false;
31}
32
33void nvgpu_vm_mapping_batch_finish_locked(
34 struct vm_gk20a *vm, struct vm_gk20a_mapping_batch *mapping_batch)
35{
36 /* hanging kref_put batch pointer? */
37 WARN_ON(vm->kref_put_batch == mapping_batch);
38
39 if (mapping_batch->need_tlb_invalidate) {
40 struct gk20a *g = gk20a_from_vm(vm);
41 g->ops.fb.tlb_invalidate(g, &vm->pdb.mem);
42 }
43}
44
45void nvgpu_vm_mapping_batch_finish(struct vm_gk20a *vm,
46 struct vm_gk20a_mapping_batch *mapping_batch)
47{
48 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
49 nvgpu_vm_mapping_batch_finish_locked(vm, mapping_batch);
50 nvgpu_mutex_release(&vm->update_gmmu_lock);
51}
52
53void nvgpu_vm_remove_support_nofree(struct vm_gk20a *vm)
54{
55 struct mapped_buffer_node *mapped_buffer;
56 struct vm_reserved_va_node *va_node, *va_node_tmp;
57 struct nvgpu_rbtree_node *node = NULL;
58 struct gk20a *g = vm->mm->g;
59
60 gk20a_dbg_fn("");
61
62 /*
63 * Do this outside of the update_gmmu_lock since unmapping the semaphore
64 * pool involves unmapping a GMMU mapping which means aquiring the
65 * update_gmmu_lock.
66 */
67 if (!(g->gpu_characteristics.flags & NVGPU_GPU_FLAGS_HAS_SYNCPOINTS)) {
68 if (vm->sema_pool) {
69 nvgpu_semaphore_pool_unmap(vm->sema_pool, vm);
70 nvgpu_semaphore_pool_put(vm->sema_pool);
71 }
72 }
73
74 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
75
76 nvgpu_rbtree_enum_start(0, &node, vm->mapped_buffers);
77 while (node) {
78 mapped_buffer = mapped_buffer_from_rbtree_node(node);
79 nvgpu_vm_unmap_locked(mapped_buffer, NULL);
80 nvgpu_rbtree_enum_start(0, &node, vm->mapped_buffers);
81 }
82
83 /* destroy remaining reserved memory areas */
84 nvgpu_list_for_each_entry_safe(va_node, va_node_tmp,
85 &vm->reserved_va_list,
86 vm_reserved_va_node, reserved_va_list) {
87 nvgpu_list_del(&va_node->reserved_va_list);
88 nvgpu_kfree(vm->mm->g, va_node);
89 }
90
91 nvgpu_deinit_vm(vm);
92
93 nvgpu_mutex_release(&vm->update_gmmu_lock);
94}
95
96void nvgpu_vm_remove_support(struct vm_gk20a *vm)
97{
98 nvgpu_vm_remove_support_nofree(vm);
99 /* vm is not used anymore. release it. */
100 nvgpu_kfree(vm->mm->g, vm);
101}
102
103static void nvgpu_vm_remove_support_kref(struct kref *ref)
104{
105 struct vm_gk20a *vm = container_of(ref, struct vm_gk20a, ref);
106 struct gk20a *g = gk20a_from_vm(vm);
107
108 g->ops.mm.vm_remove(vm);
109}
110
111void nvgpu_vm_get(struct vm_gk20a *vm)
112{
113 kref_get(&vm->ref);
114}
115
116void nvgpu_vm_put(struct vm_gk20a *vm)
117{
118 kref_put(&vm->ref, nvgpu_vm_remove_support_kref);
119}
120
121void gk20a_remove_vm(struct vm_gk20a *vm, struct nvgpu_mem *inst_block)
122{
123 struct gk20a *g = vm->mm->g;
124
125 gk20a_dbg_fn("");
126
127 gk20a_free_inst_block(g, inst_block);
128 nvgpu_vm_remove_support_nofree(vm);
129}
diff --git a/drivers/gpu/nvgpu/common/mm/vm_area.c b/drivers/gpu/nvgpu/common/mm/vm_area.c
new file mode 100644
index 00000000..7b831947
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/vm_area.c
@@ -0,0 +1,223 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#include <nvgpu/vm.h>
18#include <nvgpu/vm_area.h>
19
20#include "gk20a/gk20a.h"
21#include "gk20a/mm_gk20a.h"
22
23struct nvgpu_vm_area *nvgpu_vm_area_find(struct vm_gk20a *vm, u64 addr)
24{
25 struct nvgpu_vm_area *vm_area;
26
27 nvgpu_list_for_each_entry(vm_area, &vm->vm_area_list,
28 nvgpu_vm_area, vm_area_list) {
29 if (addr >= vm_area->addr &&
30 addr < (u64)vm_area->addr + (u64)vm_area->size)
31 return vm_area;
32 }
33
34 return NULL;
35}
36
37int nvgpu_vm_area_validate_buffer(struct vm_gk20a *vm,
38 u64 map_addr, u64 map_size, int pgsz_idx,
39 struct nvgpu_vm_area **pvm_area)
40{
41 struct gk20a *g = vm->mm->g;
42 struct nvgpu_vm_area *vm_area;
43 struct nvgpu_mapped_buf *buffer;
44 u64 map_end = map_addr + map_size;
45
46 /* can wrap around with insane map_size; zero is disallowed too */
47 if (map_end <= map_addr) {
48 nvgpu_warn(g, "fixed offset mapping with invalid map_size");
49 return -EINVAL;
50 }
51
52 if (map_addr & (vm->gmmu_page_sizes[pgsz_idx] - 1)) {
53 nvgpu_err(g, "map offset must be buffer page size aligned 0x%llx",
54 map_addr);
55 return -EINVAL;
56 }
57
58 /* Find the space reservation, but it's ok to have none for
59 * userspace-managed address spaces */
60 vm_area = nvgpu_vm_area_find(vm, map_addr);
61 if (!vm_area && !vm->userspace_managed) {
62 nvgpu_warn(g, "fixed offset mapping without space allocation");
63 return -EINVAL;
64 }
65
66 /* Mapped area should fit inside va, if there's one */
67 if (vm_area && map_end > vm_area->addr + vm_area->size) {
68 nvgpu_warn(g, "fixed offset mapping size overflows va node");
69 return -EINVAL;
70 }
71
72 /* check that this mapping does not collide with existing
73 * mappings by checking the buffer with the highest GPU VA
74 * that is less than our buffer end */
75 buffer = __nvgpu_vm_find_mapped_buf_less_than(
76 vm, map_addr + map_size);
77 if (buffer && buffer->addr + buffer->size > map_addr) {
78 nvgpu_warn(g, "overlapping buffer map requested");
79 return -EINVAL;
80 }
81
82 *pvm_area = vm_area;
83
84 return 0;
85}
86
87int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u32 pages, u32 page_size,
88 u64 *addr, u32 flags)
89{
90 struct gk20a *g = vm->mm->g;
91 struct nvgpu_allocator *vma;
92 struct nvgpu_vm_area *vm_area;
93 u64 vaddr_start = 0;
94 int pgsz_idx = gmmu_page_size_small;
95
96 nvgpu_log(g, gpu_dbg_map,
97 "ADD vm_area: pgsz=%#-8x pages=%-9u addr=%#-14llx flags=0x%x",
98 page_size, pages, *addr, flags);
99
100 for (; pgsz_idx < gmmu_nr_page_sizes; pgsz_idx++) {
101 if (vm->gmmu_page_sizes[pgsz_idx] == page_size)
102 break;
103 }
104
105 if (pgsz_idx > gmmu_page_size_big)
106 return -EINVAL;
107
108 if (!vm->big_pages && pgsz_idx == gmmu_page_size_big)
109 return -EINVAL;
110
111 vm_area = nvgpu_kzalloc(g, sizeof(*vm_area));
112 if (!vm_area)
113 goto clean_up_err;
114
115 vma = vm->vma[pgsz_idx];
116 if (flags & NVGPU_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET)
117 vaddr_start = nvgpu_alloc_fixed(vma, *addr,
118 (u64)pages *
119 (u64)page_size,
120 page_size);
121 else
122 vaddr_start = nvgpu_alloc(vma,
123 (u64)pages *
124 (u64)page_size);
125
126 if (!vaddr_start)
127 goto clean_up_err;
128
129 vm_area->flags = flags;
130 vm_area->addr = vaddr_start;
131 vm_area->size = (u64)page_size * (u64)pages;
132 vm_area->pgsz_idx = pgsz_idx;
133 nvgpu_init_list_node(&vm_area->buffer_list_head);
134 nvgpu_init_list_node(&vm_area->vm_area_list);
135
136 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
137
138 if (flags & NVGPU_AS_ALLOC_SPACE_FLAGS_SPARSE) {
139 u64 map_addr = g->ops.mm.gmmu_map(vm, vaddr_start,
140 NULL,
141 0,
142 vm_area->size,
143 pgsz_idx,
144 0,
145 0,
146 flags,
147 gk20a_mem_flag_none,
148 false,
149 true,
150 false,
151 NULL,
152 APERTURE_INVALID);
153 if (!map_addr) {
154 nvgpu_mutex_release(&vm->update_gmmu_lock);
155 goto clean_up_err;
156 }
157
158 vm_area->sparse = true;
159 }
160 nvgpu_list_add_tail(&vm_area->vm_area_list, &vm->vm_area_list);
161
162 nvgpu_mutex_release(&vm->update_gmmu_lock);
163
164 *addr = vaddr_start;
165 return 0;
166
167clean_up_err:
168 if (vaddr_start)
169 nvgpu_free(vma, vaddr_start);
170 if (vm_area)
171 nvgpu_kfree(g, vm_area);
172 return -ENOMEM;
173}
174
175int nvgpu_vm_area_free(struct vm_gk20a *vm, u64 addr)
176{
177 struct gk20a *g = gk20a_from_vm(vm);
178 struct nvgpu_mapped_buf *buffer, *n;
179 struct nvgpu_vm_area *vm_area;
180
181 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
182 vm_area = nvgpu_vm_area_find(vm, addr);
183 if (!vm_area) {
184 nvgpu_mutex_release(&vm->update_gmmu_lock);
185 return 0;
186 }
187 nvgpu_list_del(&vm_area->vm_area_list);
188 nvgpu_mutex_release(&vm->update_gmmu_lock);
189
190 nvgpu_log(g, gpu_dbg_map,
191 "DEL vm_area: pgsz=%#-8x pages=%-9llu "
192 "addr=%#-14llx flags=0x%x",
193 vm->gmmu_page_sizes[vm_area->pgsz_idx],
194 vm_area->size / vm->gmmu_page_sizes[vm_area->pgsz_idx],
195 vm_area->addr,
196 vm_area->flags);
197
198 /* Decrement the ref count on all buffers in this vm_area. This
199 * allows userspace to let the kernel free mappings that are
200 * only used by this vm_area. */
201 nvgpu_list_for_each_entry_safe(buffer, n,
202 &vm_area->buffer_list_head,
203 nvgpu_mapped_buf, buffer_list) {
204 nvgpu_list_del(&buffer->buffer_list);
205 kref_put(&buffer->ref, gk20a_vm_unmap_locked_kref);
206 }
207
208 /* if this was a sparse mapping, free the va */
209 if (vm_area->sparse)
210 g->ops.mm.gmmu_unmap(vm,
211 vm_area->addr,
212 vm_area->size,
213 vm_area->pgsz_idx,
214 true,
215 gk20a_mem_flag_none,
216 true,
217 NULL);
218
219 nvgpu_free(vm->vma[vm_area->pgsz_idx], vm_area->addr);
220 nvgpu_kfree(g, vm_area);
221
222 return 0;
223}
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-17 01:13:11 -0400