diff options
Diffstat (limited to 'drivers/gpu/nvgpu/os/linux/dma.c')
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/dma.c | 694 |
1 files changed, 694 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/os/linux/dma.c b/drivers/gpu/nvgpu/os/linux/dma.c new file mode 100644 index 00000000..f513dcd6 --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/dma.c | |||
@@ -0,0 +1,694 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify it | ||
5 | * under the terms and conditions of the GNU General Public License, | ||
6 | * version 2, as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | * more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | |||
17 | #include <linux/dma-mapping.h> | ||
18 | #include <linux/version.h> | ||
19 | |||
20 | #include <nvgpu/log.h> | ||
21 | #include <nvgpu/dma.h> | ||
22 | #include <nvgpu/lock.h> | ||
23 | #include <nvgpu/bug.h> | ||
24 | #include <nvgpu/gmmu.h> | ||
25 | #include <nvgpu/kmem.h> | ||
26 | #include <nvgpu/enabled.h> | ||
27 | #include <nvgpu/vidmem.h> | ||
28 | |||
29 | #include <nvgpu/linux/dma.h> | ||
30 | #include <nvgpu/linux/vidmem.h> | ||
31 | |||
32 | #include "gk20a/gk20a.h" | ||
33 | |||
34 | #include "platform_gk20a.h" | ||
35 | #include "os_linux.h" | ||
36 | |||
37 | #ifdef __DMA_ATTRS_LONGS | ||
38 | #define NVGPU_DEFINE_DMA_ATTRS(x) \ | ||
39 | struct dma_attrs x = { \ | ||
40 | .flags = { [0 ... __DMA_ATTRS_LONGS-1] = 0 }, \ | ||
41 | } | ||
42 | #define NVGPU_DMA_ATTR(attrs) &attrs | ||
43 | #else | ||
44 | #define NVGPU_DEFINE_DMA_ATTRS(attrs) unsigned long attrs = 0 | ||
45 | #define NVGPU_DMA_ATTR(attrs) attrs | ||
46 | #endif | ||
47 | |||
48 | /* | ||
49 | * Enough to hold all the possible flags in string form. When a new flag is | ||
50 | * added it must be added here as well!! | ||
51 | */ | ||
52 | #define NVGPU_DMA_STR_SIZE \ | ||
53 | sizeof("NO_KERNEL_MAPPING FORCE_CONTIGUOUS") | ||
54 | |||
55 | /* | ||
56 | * The returned string is kmalloc()ed here but must be freed by the caller. | ||
57 | */ | ||
58 | static char *nvgpu_dma_flags_to_str(struct gk20a *g, unsigned long flags) | ||
59 | { | ||
60 | char *buf = nvgpu_kzalloc(g, NVGPU_DMA_STR_SIZE); | ||
61 | int bytes_available = NVGPU_DMA_STR_SIZE; | ||
62 | |||
63 | /* | ||
64 | * Return the empty buffer if there's no flags. Makes it easier on the | ||
65 | * calling code to just print it instead of any if (NULL) type logic. | ||
66 | */ | ||
67 | if (!flags) | ||
68 | return buf; | ||
69 | |||
70 | #define APPEND_FLAG(flag, str_flag) \ | ||
71 | do { \ | ||
72 | if (flags & flag) { \ | ||
73 | strncat(buf, str_flag, bytes_available); \ | ||
74 | bytes_available -= strlen(str_flag); \ | ||
75 | } \ | ||
76 | } while (0) | ||
77 | |||
78 | APPEND_FLAG(NVGPU_DMA_NO_KERNEL_MAPPING, "NO_KERNEL_MAPPING "); | ||
79 | APPEND_FLAG(NVGPU_DMA_FORCE_CONTIGUOUS, "FORCE_CONTIGUOUS "); | ||
80 | #undef APPEND_FLAG | ||
81 | |||
82 | return buf; | ||
83 | } | ||
84 | |||
85 | /** | ||
86 | * __dma_dbg - Debug print for DMA allocs and frees. | ||
87 | * | ||
88 | * @g - The GPU. | ||
89 | * @size - The requested size of the alloc (size_t). | ||
90 | * @flags - The flags (unsigned long). | ||
91 | * @type - A string describing the type (i.e: sysmem or vidmem). | ||
92 | * @what - A string with 'alloc' or 'free'. | ||
93 | * | ||
94 | * @flags is the DMA flags. If there are none or it doesn't make sense to print | ||
95 | * flags just pass 0. | ||
96 | * | ||
97 | * Please use dma_dbg_alloc() and dma_dbg_free() instead of this function. | ||
98 | */ | ||
99 | static void __dma_dbg(struct gk20a *g, size_t size, unsigned long flags, | ||
100 | const char *type, const char *what) | ||
101 | { | ||
102 | char *flags_str = NULL; | ||
103 | |||
104 | /* | ||
105 | * Don't bother making the flags_str if debugging is | ||
106 | * not enabled. This saves a malloc and a free. | ||
107 | */ | ||
108 | if (!nvgpu_log_mask_enabled(g, gpu_dbg_dma)) | ||
109 | return; | ||
110 | |||
111 | flags_str = nvgpu_dma_flags_to_str(g, flags); | ||
112 | |||
113 | __nvgpu_log_dbg(g, gpu_dbg_dma, | ||
114 | __func__, __LINE__, | ||
115 | "DMA %s: [%s] size=%-7zu " | ||
116 | "aligned=%-7zu total=%-10llukB %s", | ||
117 | what, type, | ||
118 | size, PAGE_ALIGN(size), | ||
119 | g->dma_memory_used >> 10, | ||
120 | flags_str); | ||
121 | |||
122 | if (flags_str) | ||
123 | nvgpu_kfree(g, flags_str); | ||
124 | } | ||
125 | |||
126 | #define dma_dbg_alloc(g, size, flags, type) \ | ||
127 | __dma_dbg(g, size, flags, type, "alloc") | ||
128 | #define dma_dbg_free(g, size, flags, type) \ | ||
129 | __dma_dbg(g, size, flags, type, "free") | ||
130 | |||
131 | /* | ||
132 | * For after the DMA alloc is done. | ||
133 | */ | ||
134 | #define __dma_dbg_done(g, size, type, what) \ | ||
135 | nvgpu_log(g, gpu_dbg_dma, \ | ||
136 | "DMA %s: [%s] size=%-7zu Done!", \ | ||
137 | what, type, size); \ | ||
138 | |||
139 | #define dma_dbg_alloc_done(g, size, type) \ | ||
140 | __dma_dbg_done(g, size, type, "alloc") | ||
141 | #define dma_dbg_free_done(g, size, type) \ | ||
142 | __dma_dbg_done(g, size, type, "free") | ||
143 | |||
144 | #if defined(CONFIG_GK20A_VIDMEM) | ||
145 | static u64 __nvgpu_dma_alloc(struct nvgpu_allocator *allocator, u64 at, | ||
146 | size_t size) | ||
147 | { | ||
148 | u64 addr = 0; | ||
149 | |||
150 | if (at) | ||
151 | addr = nvgpu_alloc_fixed(allocator, at, size, 0); | ||
152 | else | ||
153 | addr = nvgpu_alloc(allocator, size); | ||
154 | |||
155 | return addr; | ||
156 | } | ||
157 | #endif | ||
158 | |||
159 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) | ||
160 | static void nvgpu_dma_flags_to_attrs(unsigned long *attrs, | ||
161 | unsigned long flags) | ||
162 | #define ATTR_ARG(x) *x | ||
163 | #else | ||
164 | static void nvgpu_dma_flags_to_attrs(struct dma_attrs *attrs, | ||
165 | unsigned long flags) | ||
166 | #define ATTR_ARG(x) x | ||
167 | #endif | ||
168 | { | ||
169 | if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) | ||
170 | dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, ATTR_ARG(attrs)); | ||
171 | if (flags & NVGPU_DMA_FORCE_CONTIGUOUS) | ||
172 | dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, ATTR_ARG(attrs)); | ||
173 | #undef ATTR_ARG | ||
174 | } | ||
175 | |||
176 | int nvgpu_dma_alloc(struct gk20a *g, size_t size, struct nvgpu_mem *mem) | ||
177 | { | ||
178 | return nvgpu_dma_alloc_flags(g, 0, size, mem); | ||
179 | } | ||
180 | |||
181 | int nvgpu_dma_alloc_flags(struct gk20a *g, unsigned long flags, size_t size, | ||
182 | struct nvgpu_mem *mem) | ||
183 | { | ||
184 | if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) { | ||
185 | /* | ||
186 | * Force the no-kernel-mapping flag on because we don't support | ||
187 | * the lack of it for vidmem - the user should not care when | ||
188 | * using nvgpu_gmmu_alloc_map and it's vidmem, or if there's a | ||
189 | * difference, the user should use the flag explicitly anyway. | ||
190 | * | ||
191 | * Incoming flags are ignored here, since bits other than the | ||
192 | * no-kernel-mapping flag are ignored by the vidmem mapping | ||
193 | * functions anyway. | ||
194 | */ | ||
195 | int err = nvgpu_dma_alloc_flags_vid(g, | ||
196 | NVGPU_DMA_NO_KERNEL_MAPPING, | ||
197 | size, mem); | ||
198 | |||
199 | if (!err) | ||
200 | return 0; | ||
201 | /* | ||
202 | * Fall back to sysmem (which may then also fail) in case | ||
203 | * vidmem is exhausted. | ||
204 | */ | ||
205 | } | ||
206 | |||
207 | return nvgpu_dma_alloc_flags_sys(g, flags, size, mem); | ||
208 | } | ||
209 | |||
210 | int nvgpu_dma_alloc_sys(struct gk20a *g, size_t size, struct nvgpu_mem *mem) | ||
211 | { | ||
212 | return nvgpu_dma_alloc_flags_sys(g, 0, size, mem); | ||
213 | } | ||
214 | |||
215 | int nvgpu_dma_alloc_flags_sys(struct gk20a *g, unsigned long flags, | ||
216 | size_t size, struct nvgpu_mem *mem) | ||
217 | { | ||
218 | struct device *d = dev_from_gk20a(g); | ||
219 | int err; | ||
220 | dma_addr_t iova; | ||
221 | NVGPU_DEFINE_DMA_ATTRS(dma_attrs); | ||
222 | void *alloc_ret; | ||
223 | |||
224 | if (nvgpu_mem_is_valid(mem)) { | ||
225 | nvgpu_warn(g, "memory leak !!"); | ||
226 | WARN_ON(1); | ||
227 | } | ||
228 | |||
229 | /* | ||
230 | * WAR for IO coherent chips: the DMA API does not seem to generate | ||
231 | * mappings that work correctly. Unclear why - Bug ID: 2040115. | ||
232 | * | ||
233 | * Basically we just tell the DMA API not to map with NO_KERNEL_MAPPING | ||
234 | * and then make a vmap() ourselves. | ||
235 | */ | ||
236 | if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) | ||
237 | flags |= NVGPU_DMA_NO_KERNEL_MAPPING; | ||
238 | |||
239 | /* | ||
240 | * Before the debug print so we see this in the total. But during | ||
241 | * cleanup in the fail path this has to be subtracted. | ||
242 | */ | ||
243 | g->dma_memory_used += PAGE_ALIGN(size); | ||
244 | |||
245 | dma_dbg_alloc(g, size, flags, "sysmem"); | ||
246 | |||
247 | /* | ||
248 | * Save the old size but for actual allocation purposes the size is | ||
249 | * going to be page aligned. | ||
250 | */ | ||
251 | mem->size = size; | ||
252 | size = PAGE_ALIGN(size); | ||
253 | |||
254 | nvgpu_dma_flags_to_attrs(&dma_attrs, flags); | ||
255 | |||
256 | alloc_ret = dma_alloc_attrs(d, size, &iova, | ||
257 | GFP_KERNEL|__GFP_ZERO, | ||
258 | NVGPU_DMA_ATTR(dma_attrs)); | ||
259 | if (!alloc_ret) | ||
260 | return -ENOMEM; | ||
261 | |||
262 | if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) { | ||
263 | mem->priv.pages = alloc_ret; | ||
264 | err = nvgpu_get_sgtable_from_pages(g, &mem->priv.sgt, | ||
265 | mem->priv.pages, | ||
266 | iova, size); | ||
267 | } else { | ||
268 | mem->cpu_va = alloc_ret; | ||
269 | err = nvgpu_get_sgtable_attrs(g, &mem->priv.sgt, mem->cpu_va, | ||
270 | iova, size, flags); | ||
271 | } | ||
272 | if (err) | ||
273 | goto fail_free_dma; | ||
274 | |||
275 | if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) { | ||
276 | mem->cpu_va = vmap(mem->priv.pages, | ||
277 | size >> PAGE_SHIFT, | ||
278 | 0, PAGE_KERNEL); | ||
279 | if (!mem->cpu_va) { | ||
280 | err = -ENOMEM; | ||
281 | goto fail_free_sgt; | ||
282 | } | ||
283 | } | ||
284 | |||
285 | mem->aligned_size = size; | ||
286 | mem->aperture = APERTURE_SYSMEM; | ||
287 | mem->priv.flags = flags; | ||
288 | |||
289 | dma_dbg_alloc_done(g, mem->size, "sysmem"); | ||
290 | |||
291 | return 0; | ||
292 | |||
293 | fail_free_sgt: | ||
294 | nvgpu_free_sgtable(g, &mem->priv.sgt); | ||
295 | fail_free_dma: | ||
296 | dma_free_attrs(d, size, alloc_ret, iova, NVGPU_DMA_ATTR(dma_attrs)); | ||
297 | mem->cpu_va = NULL; | ||
298 | mem->priv.sgt = NULL; | ||
299 | mem->size = 0; | ||
300 | g->dma_memory_used -= mem->aligned_size; | ||
301 | return err; | ||
302 | } | ||
303 | |||
304 | int nvgpu_dma_alloc_vid(struct gk20a *g, size_t size, struct nvgpu_mem *mem) | ||
305 | { | ||
306 | return nvgpu_dma_alloc_flags_vid(g, | ||
307 | NVGPU_DMA_NO_KERNEL_MAPPING, size, mem); | ||
308 | } | ||
309 | |||
310 | int nvgpu_dma_alloc_flags_vid(struct gk20a *g, unsigned long flags, | ||
311 | size_t size, struct nvgpu_mem *mem) | ||
312 | { | ||
313 | return nvgpu_dma_alloc_flags_vid_at(g, flags, size, mem, 0); | ||
314 | } | ||
315 | |||
316 | int nvgpu_dma_alloc_flags_vid_at(struct gk20a *g, unsigned long flags, | ||
317 | size_t size, struct nvgpu_mem *mem, u64 at) | ||
318 | { | ||
319 | #if defined(CONFIG_GK20A_VIDMEM) | ||
320 | u64 addr; | ||
321 | int err; | ||
322 | struct nvgpu_allocator *vidmem_alloc = g->mm.vidmem.cleared ? | ||
323 | &g->mm.vidmem.allocator : | ||
324 | &g->mm.vidmem.bootstrap_allocator; | ||
325 | int before_pending; | ||
326 | |||
327 | if (nvgpu_mem_is_valid(mem)) { | ||
328 | nvgpu_warn(g, "memory leak !!"); | ||
329 | WARN_ON(1); | ||
330 | } | ||
331 | |||
332 | dma_dbg_alloc(g, size, flags, "vidmem"); | ||
333 | |||
334 | mem->size = size; | ||
335 | size = PAGE_ALIGN(size); | ||
336 | |||
337 | if (!nvgpu_alloc_initialized(&g->mm.vidmem.allocator)) | ||
338 | return -ENOSYS; | ||
339 | |||
340 | /* | ||
341 | * Our own allocator doesn't have any flags yet, and we can't | ||
342 | * kernel-map these, so require explicit flags. | ||
343 | */ | ||
344 | WARN_ON(flags != NVGPU_DMA_NO_KERNEL_MAPPING); | ||
345 | |||
346 | nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex); | ||
347 | before_pending = atomic64_read(&g->mm.vidmem.bytes_pending.atomic_var); | ||
348 | addr = __nvgpu_dma_alloc(vidmem_alloc, at, size); | ||
349 | nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); | ||
350 | if (!addr) { | ||
351 | /* | ||
352 | * If memory is known to be freed soon, let the user know that | ||
353 | * it may be available after a while. | ||
354 | */ | ||
355 | if (before_pending) | ||
356 | return -EAGAIN; | ||
357 | else | ||
358 | return -ENOMEM; | ||
359 | } | ||
360 | |||
361 | if (at) | ||
362 | mem->mem_flags |= NVGPU_MEM_FLAG_FIXED; | ||
363 | |||
364 | mem->priv.sgt = nvgpu_kzalloc(g, sizeof(struct sg_table)); | ||
365 | if (!mem->priv.sgt) { | ||
366 | err = -ENOMEM; | ||
367 | goto fail_physfree; | ||
368 | } | ||
369 | |||
370 | err = sg_alloc_table(mem->priv.sgt, 1, GFP_KERNEL); | ||
371 | if (err) | ||
372 | goto fail_kfree; | ||
373 | |||
374 | nvgpu_vidmem_set_page_alloc(mem->priv.sgt->sgl, addr); | ||
375 | sg_set_page(mem->priv.sgt->sgl, NULL, size, 0); | ||
376 | |||
377 | mem->aligned_size = size; | ||
378 | mem->aperture = APERTURE_VIDMEM; | ||
379 | mem->vidmem_alloc = (struct nvgpu_page_alloc *)(uintptr_t)addr; | ||
380 | mem->allocator = vidmem_alloc; | ||
381 | mem->priv.flags = flags; | ||
382 | |||
383 | nvgpu_init_list_node(&mem->clear_list_entry); | ||
384 | |||
385 | dma_dbg_alloc_done(g, mem->size, "vidmem"); | ||
386 | |||
387 | return 0; | ||
388 | |||
389 | fail_kfree: | ||
390 | nvgpu_kfree(g, mem->priv.sgt); | ||
391 | fail_physfree: | ||
392 | nvgpu_free(&g->mm.vidmem.allocator, addr); | ||
393 | mem->size = 0; | ||
394 | return err; | ||
395 | #else | ||
396 | return -ENOSYS; | ||
397 | #endif | ||
398 | } | ||
399 | |||
400 | int nvgpu_dma_alloc_map(struct vm_gk20a *vm, size_t size, | ||
401 | struct nvgpu_mem *mem) | ||
402 | { | ||
403 | return nvgpu_dma_alloc_map_flags(vm, 0, size, mem); | ||
404 | } | ||
405 | |||
406 | int nvgpu_dma_alloc_map_flags(struct vm_gk20a *vm, unsigned long flags, | ||
407 | size_t size, struct nvgpu_mem *mem) | ||
408 | { | ||
409 | if (!nvgpu_is_enabled(gk20a_from_vm(vm), NVGPU_MM_UNIFIED_MEMORY)) { | ||
410 | /* | ||
411 | * Force the no-kernel-mapping flag on because we don't support | ||
412 | * the lack of it for vidmem - the user should not care when | ||
413 | * using nvgpu_dma_alloc_map and it's vidmem, or if there's a | ||
414 | * difference, the user should use the flag explicitly anyway. | ||
415 | */ | ||
416 | int err = nvgpu_dma_alloc_map_flags_vid(vm, | ||
417 | flags | NVGPU_DMA_NO_KERNEL_MAPPING, | ||
418 | size, mem); | ||
419 | |||
420 | if (!err) | ||
421 | return 0; | ||
422 | /* | ||
423 | * Fall back to sysmem (which may then also fail) in case | ||
424 | * vidmem is exhausted. | ||
425 | */ | ||
426 | } | ||
427 | |||
428 | return nvgpu_dma_alloc_map_flags_sys(vm, flags, size, mem); | ||
429 | } | ||
430 | |||
431 | int nvgpu_dma_alloc_map_sys(struct vm_gk20a *vm, size_t size, | ||
432 | struct nvgpu_mem *mem) | ||
433 | { | ||
434 | return nvgpu_dma_alloc_map_flags_sys(vm, 0, size, mem); | ||
435 | } | ||
436 | |||
437 | int nvgpu_dma_alloc_map_flags_sys(struct vm_gk20a *vm, unsigned long flags, | ||
438 | size_t size, struct nvgpu_mem *mem) | ||
439 | { | ||
440 | int err = nvgpu_dma_alloc_flags_sys(vm->mm->g, flags, size, mem); | ||
441 | |||
442 | if (err) | ||
443 | return err; | ||
444 | |||
445 | mem->gpu_va = nvgpu_gmmu_map(vm, mem, size, 0, | ||
446 | gk20a_mem_flag_none, false, | ||
447 | mem->aperture); | ||
448 | if (!mem->gpu_va) { | ||
449 | err = -ENOMEM; | ||
450 | goto fail_free; | ||
451 | } | ||
452 | |||
453 | return 0; | ||
454 | |||
455 | fail_free: | ||
456 | nvgpu_dma_free(vm->mm->g, mem); | ||
457 | return err; | ||
458 | } | ||
459 | |||
460 | int nvgpu_dma_alloc_map_vid(struct vm_gk20a *vm, size_t size, | ||
461 | struct nvgpu_mem *mem) | ||
462 | { | ||
463 | return nvgpu_dma_alloc_map_flags_vid(vm, | ||
464 | NVGPU_DMA_NO_KERNEL_MAPPING, size, mem); | ||
465 | } | ||
466 | |||
467 | int nvgpu_dma_alloc_map_flags_vid(struct vm_gk20a *vm, unsigned long flags, | ||
468 | size_t size, struct nvgpu_mem *mem) | ||
469 | { | ||
470 | int err = nvgpu_dma_alloc_flags_vid(vm->mm->g, flags, size, mem); | ||
471 | |||
472 | if (err) | ||
473 | return err; | ||
474 | |||
475 | mem->gpu_va = nvgpu_gmmu_map(vm, mem, size, 0, | ||
476 | gk20a_mem_flag_none, false, | ||
477 | mem->aperture); | ||
478 | if (!mem->gpu_va) { | ||
479 | err = -ENOMEM; | ||
480 | goto fail_free; | ||
481 | } | ||
482 | |||
483 | return 0; | ||
484 | |||
485 | fail_free: | ||
486 | nvgpu_dma_free(vm->mm->g, mem); | ||
487 | return err; | ||
488 | } | ||
489 | |||
490 | static void nvgpu_dma_free_sys(struct gk20a *g, struct nvgpu_mem *mem) | ||
491 | { | ||
492 | struct device *d = dev_from_gk20a(g); | ||
493 | |||
494 | g->dma_memory_used -= mem->aligned_size; | ||
495 | |||
496 | dma_dbg_free(g, mem->size, mem->priv.flags, "sysmem"); | ||
497 | |||
498 | if (!(mem->mem_flags & NVGPU_MEM_FLAG_SHADOW_COPY) && | ||
499 | !(mem->mem_flags & __NVGPU_MEM_FLAG_NO_DMA) && | ||
500 | (mem->cpu_va || mem->priv.pages)) { | ||
501 | /* | ||
502 | * Free side of WAR for bug 2040115. | ||
503 | */ | ||
504 | if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) | ||
505 | vunmap(mem->cpu_va); | ||
506 | |||
507 | if (mem->priv.flags) { | ||
508 | NVGPU_DEFINE_DMA_ATTRS(dma_attrs); | ||
509 | |||
510 | nvgpu_dma_flags_to_attrs(&dma_attrs, mem->priv.flags); | ||
511 | |||
512 | if (mem->priv.flags & NVGPU_DMA_NO_KERNEL_MAPPING) { | ||
513 | dma_free_attrs(d, mem->aligned_size, mem->priv.pages, | ||
514 | sg_dma_address(mem->priv.sgt->sgl), | ||
515 | NVGPU_DMA_ATTR(dma_attrs)); | ||
516 | } else { | ||
517 | dma_free_attrs(d, mem->aligned_size, mem->cpu_va, | ||
518 | sg_dma_address(mem->priv.sgt->sgl), | ||
519 | NVGPU_DMA_ATTR(dma_attrs)); | ||
520 | } | ||
521 | } else { | ||
522 | dma_free_coherent(d, mem->aligned_size, mem->cpu_va, | ||
523 | sg_dma_address(mem->priv.sgt->sgl)); | ||
524 | } | ||
525 | mem->cpu_va = NULL; | ||
526 | mem->priv.pages = NULL; | ||
527 | } | ||
528 | |||
529 | /* | ||
530 | * When this flag is set we expect that pages is still populated but not | ||
531 | * by the DMA API. | ||
532 | */ | ||
533 | if (mem->mem_flags & __NVGPU_MEM_FLAG_NO_DMA) | ||
534 | nvgpu_kfree(g, mem->priv.pages); | ||
535 | |||
536 | if (mem->priv.sgt) | ||
537 | nvgpu_free_sgtable(g, &mem->priv.sgt); | ||
538 | |||
539 | dma_dbg_free_done(g, mem->size, "sysmem"); | ||
540 | |||
541 | mem->size = 0; | ||
542 | mem->aligned_size = 0; | ||
543 | mem->aperture = APERTURE_INVALID; | ||
544 | } | ||
545 | |||
546 | static void nvgpu_dma_free_vid(struct gk20a *g, struct nvgpu_mem *mem) | ||
547 | { | ||
548 | #if defined(CONFIG_GK20A_VIDMEM) | ||
549 | size_t mem_size = mem->size; | ||
550 | |||
551 | dma_dbg_free(g, mem->size, mem->priv.flags, "vidmem"); | ||
552 | |||
553 | /* Sanity check - only this supported when allocating. */ | ||
554 | WARN_ON(mem->priv.flags != NVGPU_DMA_NO_KERNEL_MAPPING); | ||
555 | |||
556 | if (mem->mem_flags & NVGPU_MEM_FLAG_USER_MEM) { | ||
557 | int err = nvgpu_vidmem_clear_list_enqueue(g, mem); | ||
558 | |||
559 | /* | ||
560 | * If there's an error here then that means we can't clear the | ||
561 | * vidmem. That's too bad; however, we still own the nvgpu_mem | ||
562 | * buf so we have to free that. | ||
563 | * | ||
564 | * We don't need to worry about the vidmem allocator itself | ||
565 | * since when that gets cleaned up in the driver shutdown path | ||
566 | * all the outstanding allocs are force freed. | ||
567 | */ | ||
568 | if (err) | ||
569 | nvgpu_kfree(g, mem); | ||
570 | } else { | ||
571 | nvgpu_memset(g, mem, 0, 0, mem->aligned_size); | ||
572 | nvgpu_free(mem->allocator, | ||
573 | (u64)nvgpu_vidmem_get_page_alloc(mem->priv.sgt->sgl)); | ||
574 | nvgpu_free_sgtable(g, &mem->priv.sgt); | ||
575 | |||
576 | mem->size = 0; | ||
577 | mem->aligned_size = 0; | ||
578 | mem->aperture = APERTURE_INVALID; | ||
579 | } | ||
580 | |||
581 | dma_dbg_free_done(g, mem_size, "vidmem"); | ||
582 | #endif | ||
583 | } | ||
584 | |||
585 | void nvgpu_dma_free(struct gk20a *g, struct nvgpu_mem *mem) | ||
586 | { | ||
587 | switch (mem->aperture) { | ||
588 | case APERTURE_SYSMEM: | ||
589 | return nvgpu_dma_free_sys(g, mem); | ||
590 | case APERTURE_VIDMEM: | ||
591 | return nvgpu_dma_free_vid(g, mem); | ||
592 | default: | ||
593 | break; /* like free() on "null" memory */ | ||
594 | } | ||
595 | } | ||
596 | |||
597 | void nvgpu_dma_unmap_free(struct vm_gk20a *vm, struct nvgpu_mem *mem) | ||
598 | { | ||
599 | if (mem->gpu_va) | ||
600 | nvgpu_gmmu_unmap(vm, mem, mem->gpu_va); | ||
601 | mem->gpu_va = 0; | ||
602 | |||
603 | nvgpu_dma_free(vm->mm->g, mem); | ||
604 | } | ||
605 | |||
606 | int nvgpu_get_sgtable_attrs(struct gk20a *g, struct sg_table **sgt, | ||
607 | void *cpuva, u64 iova, size_t size, unsigned long flags) | ||
608 | { | ||
609 | int err = 0; | ||
610 | struct sg_table *tbl; | ||
611 | NVGPU_DEFINE_DMA_ATTRS(dma_attrs); | ||
612 | |||
613 | tbl = nvgpu_kzalloc(g, sizeof(struct sg_table)); | ||
614 | if (!tbl) { | ||
615 | err = -ENOMEM; | ||
616 | goto fail; | ||
617 | } | ||
618 | |||
619 | nvgpu_dma_flags_to_attrs(&dma_attrs, flags); | ||
620 | err = dma_get_sgtable_attrs(dev_from_gk20a(g), tbl, cpuva, iova, | ||
621 | size, NVGPU_DMA_ATTR(dma_attrs)); | ||
622 | if (err) | ||
623 | goto fail; | ||
624 | |||
625 | sg_dma_address(tbl->sgl) = iova; | ||
626 | *sgt = tbl; | ||
627 | |||
628 | return 0; | ||
629 | |||
630 | fail: | ||
631 | if (tbl) | ||
632 | nvgpu_kfree(g, tbl); | ||
633 | |||
634 | return err; | ||
635 | } | ||
636 | |||
637 | int nvgpu_get_sgtable(struct gk20a *g, struct sg_table **sgt, | ||
638 | void *cpuva, u64 iova, size_t size) | ||
639 | { | ||
640 | return nvgpu_get_sgtable_attrs(g, sgt, cpuva, iova, size, 0); | ||
641 | } | ||
642 | |||
643 | int nvgpu_get_sgtable_from_pages(struct gk20a *g, struct sg_table **sgt, | ||
644 | struct page **pages, u64 iova, size_t size) | ||
645 | { | ||
646 | int err = 0; | ||
647 | struct sg_table *tbl; | ||
648 | |||
649 | tbl = nvgpu_kzalloc(g, sizeof(struct sg_table)); | ||
650 | if (!tbl) { | ||
651 | err = -ENOMEM; | ||
652 | goto fail; | ||
653 | } | ||
654 | |||
655 | err = sg_alloc_table_from_pages(tbl, pages, | ||
656 | DIV_ROUND_UP(size, PAGE_SIZE), | ||
657 | 0, size, GFP_KERNEL); | ||
658 | if (err) | ||
659 | goto fail; | ||
660 | |||
661 | sg_dma_address(tbl->sgl) = iova; | ||
662 | *sgt = tbl; | ||
663 | |||
664 | return 0; | ||
665 | |||
666 | fail: | ||
667 | if (tbl) | ||
668 | nvgpu_kfree(g, tbl); | ||
669 | |||
670 | return err; | ||
671 | } | ||
672 | |||
673 | void nvgpu_free_sgtable(struct gk20a *g, struct sg_table **sgt) | ||
674 | { | ||
675 | sg_free_table(*sgt); | ||
676 | nvgpu_kfree(g, *sgt); | ||
677 | *sgt = NULL; | ||
678 | } | ||
679 | |||
680 | bool nvgpu_iommuable(struct gk20a *g) | ||
681 | { | ||
682 | #ifdef CONFIG_TEGRA_GK20A | ||
683 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
684 | |||
685 | /* | ||
686 | * Check against the nvgpu device to see if it's been marked as | ||
687 | * IOMMU'able. | ||
688 | */ | ||
689 | if (!device_is_iommuable(l->dev)) | ||
690 | return false; | ||
691 | #endif | ||
692 | |||
693 | return true; | ||
694 | } | ||