diff options
Diffstat (limited to 'drivers/gpu/nvgpu/os/linux/vm.c')
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/vm.c | 332 |
1 files changed, 332 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/os/linux/vm.c b/drivers/gpu/nvgpu/os/linux/vm.c new file mode 100644 index 00000000..baa77515 --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/vm.c | |||
@@ -0,0 +1,332 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify it | ||
5 | * under the terms and conditions of the GNU General Public License, | ||
6 | * version 2, as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | * more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | |||
17 | #include <linux/dma-buf.h> | ||
18 | #include <linux/scatterlist.h> | ||
19 | #include <uapi/linux/nvgpu.h> | ||
20 | |||
21 | #include <nvgpu/log.h> | ||
22 | #include <nvgpu/lock.h> | ||
23 | #include <nvgpu/rbtree.h> | ||
24 | #include <nvgpu/vm_area.h> | ||
25 | #include <nvgpu/nvgpu_mem.h> | ||
26 | #include <nvgpu/page_allocator.h> | ||
27 | #include <nvgpu/vidmem.h> | ||
28 | |||
29 | #include <nvgpu/linux/vm.h> | ||
30 | #include <nvgpu/linux/vidmem.h> | ||
31 | #include <nvgpu/linux/nvgpu_mem.h> | ||
32 | |||
33 | #include "gk20a/gk20a.h" | ||
34 | #include "gk20a/mm_gk20a.h" | ||
35 | |||
36 | #include "platform_gk20a.h" | ||
37 | #include "os_linux.h" | ||
38 | #include "dmabuf.h" | ||
39 | |||
40 | static u32 nvgpu_vm_translate_linux_flags(struct gk20a *g, u32 flags) | ||
41 | { | ||
42 | u32 core_flags = 0; | ||
43 | |||
44 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) | ||
45 | core_flags |= NVGPU_VM_MAP_FIXED_OFFSET; | ||
46 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_CACHEABLE) | ||
47 | core_flags |= NVGPU_VM_MAP_CACHEABLE; | ||
48 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_IO_COHERENT) | ||
49 | core_flags |= NVGPU_VM_MAP_IO_COHERENT; | ||
50 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_UNMAPPED_PTE) | ||
51 | core_flags |= NVGPU_VM_MAP_UNMAPPED_PTE; | ||
52 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_L3_ALLOC) | ||
53 | core_flags |= NVGPU_VM_MAP_L3_ALLOC; | ||
54 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL) | ||
55 | core_flags |= NVGPU_VM_MAP_DIRECT_KIND_CTRL; | ||
56 | |||
57 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_MAPPABLE_COMPBITS) | ||
58 | nvgpu_warn(g, "Ignoring deprecated flag: " | ||
59 | "NVGPU_AS_MAP_BUFFER_FLAGS_MAPPABLE_COMPBITS"); | ||
60 | |||
61 | return core_flags; | ||
62 | } | ||
63 | |||
64 | static struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_reverse( | ||
65 | struct vm_gk20a *vm, struct dma_buf *dmabuf, u32 kind) | ||
66 | { | ||
67 | struct nvgpu_rbtree_node *node = NULL; | ||
68 | struct nvgpu_rbtree_node *root = vm->mapped_buffers; | ||
69 | |||
70 | nvgpu_rbtree_enum_start(0, &node, root); | ||
71 | |||
72 | while (node) { | ||
73 | struct nvgpu_mapped_buf *mapped_buffer = | ||
74 | mapped_buffer_from_rbtree_node(node); | ||
75 | |||
76 | if (mapped_buffer->os_priv.dmabuf == dmabuf && | ||
77 | mapped_buffer->kind == kind) | ||
78 | return mapped_buffer; | ||
79 | |||
80 | nvgpu_rbtree_enum_next(&node, node); | ||
81 | } | ||
82 | |||
83 | return NULL; | ||
84 | } | ||
85 | |||
86 | int nvgpu_vm_find_buf(struct vm_gk20a *vm, u64 gpu_va, | ||
87 | struct dma_buf **dmabuf, | ||
88 | u64 *offset) | ||
89 | { | ||
90 | struct nvgpu_mapped_buf *mapped_buffer; | ||
91 | struct gk20a *g = gk20a_from_vm(vm); | ||
92 | |||
93 | nvgpu_log_fn(g, "gpu_va=0x%llx", gpu_va); | ||
94 | |||
95 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
96 | |||
97 | mapped_buffer = __nvgpu_vm_find_mapped_buf_range(vm, gpu_va); | ||
98 | if (!mapped_buffer) { | ||
99 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
100 | return -EINVAL; | ||
101 | } | ||
102 | |||
103 | *dmabuf = mapped_buffer->os_priv.dmabuf; | ||
104 | *offset = gpu_va - mapped_buffer->addr; | ||
105 | |||
106 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
107 | |||
108 | return 0; | ||
109 | } | ||
110 | |||
111 | u64 nvgpu_os_buf_get_size(struct nvgpu_os_buffer *os_buf) | ||
112 | { | ||
113 | return os_buf->dmabuf->size; | ||
114 | } | ||
115 | |||
116 | /* | ||
117 | * vm->update_gmmu_lock must be held. This checks to see if we already have | ||
118 | * mapped the passed buffer into this VM. If so, just return the existing | ||
119 | * mapping address. | ||
120 | */ | ||
121 | struct nvgpu_mapped_buf *nvgpu_vm_find_mapping(struct vm_gk20a *vm, | ||
122 | struct nvgpu_os_buffer *os_buf, | ||
123 | u64 map_addr, | ||
124 | u32 flags, | ||
125 | int kind) | ||
126 | { | ||
127 | struct gk20a *g = gk20a_from_vm(vm); | ||
128 | struct nvgpu_mapped_buf *mapped_buffer = NULL; | ||
129 | |||
130 | if (flags & NVGPU_VM_MAP_FIXED_OFFSET) { | ||
131 | mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, map_addr); | ||
132 | if (!mapped_buffer) | ||
133 | return NULL; | ||
134 | |||
135 | if (mapped_buffer->os_priv.dmabuf != os_buf->dmabuf || | ||
136 | mapped_buffer->kind != (u32)kind) | ||
137 | return NULL; | ||
138 | } else { | ||
139 | mapped_buffer = | ||
140 | __nvgpu_vm_find_mapped_buf_reverse(vm, | ||
141 | os_buf->dmabuf, | ||
142 | kind); | ||
143 | if (!mapped_buffer) | ||
144 | return NULL; | ||
145 | } | ||
146 | |||
147 | if (mapped_buffer->flags != flags) | ||
148 | return NULL; | ||
149 | |||
150 | /* | ||
151 | * If we find the mapping here then that means we have mapped it already | ||
152 | * and the prior pin and get must be undone. | ||
153 | */ | ||
154 | gk20a_mm_unpin(os_buf->dev, os_buf->dmabuf, os_buf->attachment, | ||
155 | mapped_buffer->os_priv.sgt); | ||
156 | dma_buf_put(os_buf->dmabuf); | ||
157 | |||
158 | nvgpu_log(g, gpu_dbg_map, | ||
159 | "gv: 0x%04x_%08x + 0x%-7zu " | ||
160 | "[dma: 0x%010llx, pa: 0x%010llx] " | ||
161 | "pgsz=%-3dKb as=%-2d " | ||
162 | "flags=0x%x apt=%s (reused)", | ||
163 | u64_hi32(mapped_buffer->addr), u64_lo32(mapped_buffer->addr), | ||
164 | os_buf->dmabuf->size, | ||
165 | (u64)sg_dma_address(mapped_buffer->os_priv.sgt->sgl), | ||
166 | (u64)sg_phys(mapped_buffer->os_priv.sgt->sgl), | ||
167 | vm->gmmu_page_sizes[mapped_buffer->pgsz_idx] >> 10, | ||
168 | vm_aspace_id(vm), | ||
169 | mapped_buffer->flags, | ||
170 | nvgpu_aperture_str(g, | ||
171 | gk20a_dmabuf_aperture(g, os_buf->dmabuf))); | ||
172 | |||
173 | return mapped_buffer; | ||
174 | } | ||
175 | |||
176 | int nvgpu_vm_map_linux(struct vm_gk20a *vm, | ||
177 | struct dma_buf *dmabuf, | ||
178 | u64 offset_align, | ||
179 | u32 flags, | ||
180 | s16 compr_kind, | ||
181 | s16 incompr_kind, | ||
182 | int rw_flag, | ||
183 | u64 buffer_offset, | ||
184 | u64 mapping_size, | ||
185 | struct vm_gk20a_mapping_batch *batch, | ||
186 | u64 *gpu_va) | ||
187 | { | ||
188 | struct gk20a *g = gk20a_from_vm(vm); | ||
189 | struct device *dev = dev_from_gk20a(g); | ||
190 | struct nvgpu_os_buffer os_buf; | ||
191 | struct sg_table *sgt; | ||
192 | struct nvgpu_sgt *nvgpu_sgt = NULL; | ||
193 | struct nvgpu_mapped_buf *mapped_buffer = NULL; | ||
194 | struct dma_buf_attachment *attachment; | ||
195 | u64 map_addr = 0ULL; | ||
196 | int err = 0; | ||
197 | |||
198 | if (flags & NVGPU_VM_MAP_FIXED_OFFSET) | ||
199 | map_addr = offset_align; | ||
200 | |||
201 | sgt = gk20a_mm_pin(dev, dmabuf, &attachment); | ||
202 | if (IS_ERR(sgt)) { | ||
203 | nvgpu_warn(g, "Failed to pin dma_buf!"); | ||
204 | return PTR_ERR(sgt); | ||
205 | } | ||
206 | os_buf.dmabuf = dmabuf; | ||
207 | os_buf.attachment = attachment; | ||
208 | os_buf.dev = dev; | ||
209 | |||
210 | if (gk20a_dmabuf_aperture(g, dmabuf) == APERTURE_INVALID) { | ||
211 | err = -EINVAL; | ||
212 | goto clean_up; | ||
213 | } | ||
214 | |||
215 | nvgpu_sgt = nvgpu_linux_sgt_create(g, sgt); | ||
216 | if (!nvgpu_sgt) { | ||
217 | err = -ENOMEM; | ||
218 | goto clean_up; | ||
219 | } | ||
220 | |||
221 | mapped_buffer = nvgpu_vm_map(vm, | ||
222 | &os_buf, | ||
223 | nvgpu_sgt, | ||
224 | map_addr, | ||
225 | mapping_size, | ||
226 | buffer_offset, | ||
227 | rw_flag, | ||
228 | flags, | ||
229 | compr_kind, | ||
230 | incompr_kind, | ||
231 | batch, | ||
232 | gk20a_dmabuf_aperture(g, dmabuf)); | ||
233 | |||
234 | nvgpu_sgt_free(g, nvgpu_sgt); | ||
235 | |||
236 | if (IS_ERR(mapped_buffer)) { | ||
237 | err = PTR_ERR(mapped_buffer); | ||
238 | goto clean_up; | ||
239 | } | ||
240 | |||
241 | mapped_buffer->os_priv.dmabuf = dmabuf; | ||
242 | mapped_buffer->os_priv.attachment = attachment; | ||
243 | mapped_buffer->os_priv.sgt = sgt; | ||
244 | |||
245 | *gpu_va = mapped_buffer->addr; | ||
246 | return 0; | ||
247 | |||
248 | clean_up: | ||
249 | gk20a_mm_unpin(dev, dmabuf, attachment, sgt); | ||
250 | |||
251 | return err; | ||
252 | } | ||
253 | |||
254 | int nvgpu_vm_map_buffer(struct vm_gk20a *vm, | ||
255 | int dmabuf_fd, | ||
256 | u64 *offset_align, | ||
257 | u32 flags, /*NVGPU_AS_MAP_BUFFER_FLAGS_*/ | ||
258 | s16 compr_kind, | ||
259 | s16 incompr_kind, | ||
260 | u64 buffer_offset, | ||
261 | u64 mapping_size, | ||
262 | struct vm_gk20a_mapping_batch *batch) | ||
263 | { | ||
264 | struct gk20a *g = gk20a_from_vm(vm); | ||
265 | struct dma_buf *dmabuf; | ||
266 | u64 ret_va; | ||
267 | int err = 0; | ||
268 | |||
269 | /* get ref to the mem handle (released on unmap_locked) */ | ||
270 | dmabuf = dma_buf_get(dmabuf_fd); | ||
271 | if (IS_ERR(dmabuf)) { | ||
272 | nvgpu_warn(g, "%s: fd %d is not a dmabuf", | ||
273 | __func__, dmabuf_fd); | ||
274 | return PTR_ERR(dmabuf); | ||
275 | } | ||
276 | |||
277 | /* verify that we're not overflowing the buffer, i.e. | ||
278 | * (buffer_offset + mapping_size)> dmabuf->size. | ||
279 | * | ||
280 | * Since buffer_offset + mapping_size could overflow, first check | ||
281 | * that mapping size < dmabuf_size, at which point we can subtract | ||
282 | * mapping_size from both sides for the final comparison. | ||
283 | */ | ||
284 | if ((mapping_size > dmabuf->size) || | ||
285 | (buffer_offset > (dmabuf->size - mapping_size))) { | ||
286 | nvgpu_err(g, | ||
287 | "buf size %llx < (offset(%llx) + map_size(%llx))\n", | ||
288 | (u64)dmabuf->size, buffer_offset, mapping_size); | ||
289 | dma_buf_put(dmabuf); | ||
290 | return -EINVAL; | ||
291 | } | ||
292 | |||
293 | err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev_from_vm(vm)); | ||
294 | if (err) { | ||
295 | dma_buf_put(dmabuf); | ||
296 | return err; | ||
297 | } | ||
298 | |||
299 | err = nvgpu_vm_map_linux(vm, dmabuf, *offset_align, | ||
300 | nvgpu_vm_translate_linux_flags(g, flags), | ||
301 | compr_kind, incompr_kind, | ||
302 | gk20a_mem_flag_none, | ||
303 | buffer_offset, | ||
304 | mapping_size, | ||
305 | batch, | ||
306 | &ret_va); | ||
307 | |||
308 | if (!err) | ||
309 | *offset_align = ret_va; | ||
310 | else | ||
311 | dma_buf_put(dmabuf); | ||
312 | |||
313 | return err; | ||
314 | } | ||
315 | |||
316 | /* | ||
317 | * This is the function call-back for freeing OS specific components of an | ||
318 | * nvgpu_mapped_buf. This should most likely never be called outside of the | ||
319 | * core MM framework! | ||
320 | * | ||
321 | * Note: the VM lock will be held. | ||
322 | */ | ||
323 | void nvgpu_vm_unmap_system(struct nvgpu_mapped_buf *mapped_buffer) | ||
324 | { | ||
325 | struct vm_gk20a *vm = mapped_buffer->vm; | ||
326 | |||
327 | gk20a_mm_unpin(dev_from_vm(vm), mapped_buffer->os_priv.dmabuf, | ||
328 | mapped_buffer->os_priv.attachment, | ||
329 | mapped_buffer->os_priv.sgt); | ||
330 | |||
331 | dma_buf_put(mapped_buffer->os_priv.dmabuf); | ||
332 | } | ||