summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/mm/gmmu.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common/mm/gmmu.c')
-rw-r--r--drivers/gpu/nvgpu/common/mm/gmmu.c920
1 files changed, 920 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/gmmu.c b/drivers/gpu/nvgpu/common/mm/gmmu.c
new file mode 100644
index 00000000..568da8c4
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/gmmu.c
@@ -0,0 +1,920 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <uapi/linux/nvgpu.h>
24
25#include <nvgpu/log.h>
26#include <nvgpu/list.h>
27#include <nvgpu/dma.h>
28#include <nvgpu/gmmu.h>
29#include <nvgpu/nvgpu_mem.h>
30#include <nvgpu/enabled.h>
31#include <nvgpu/page_allocator.h>
32#include <nvgpu/barrier.h>
33#include <nvgpu/vidmem.h>
34
35#include "gk20a/gk20a.h"
36#include "gk20a/mm_gk20a.h"
37
38#define __gmmu_dbg(g, attrs, fmt, args...) \
39 do { \
40 if (attrs->debug) \
41 nvgpu_info(g, fmt, ##args); \
42 else \
43 nvgpu_log(g, gpu_dbg_map, fmt, ##args); \
44 } while (0)
45
46#define __gmmu_dbg_v(g, attrs, fmt, args...) \
47 do { \
48 if (attrs->debug) \
49 nvgpu_info(g, fmt, ##args); \
50 else \
51 nvgpu_log(g, gpu_dbg_map_v, fmt, ##args); \
52 } while (0)
53
54static int pd_allocate(struct vm_gk20a *vm,
55 struct nvgpu_gmmu_pd *pd,
56 const struct gk20a_mmu_level *l,
57 struct nvgpu_gmmu_attrs *attrs);
58static u32 pd_size(const struct gk20a_mmu_level *l,
59 struct nvgpu_gmmu_attrs *attrs);
60/*
61 * Core GMMU map function for the kernel to use. If @addr is 0 then the GPU
62 * VA will be allocated for you. If addr is non-zero then the buffer will be
63 * mapped at @addr.
64 */
65static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm,
66 struct nvgpu_mem *mem,
67 u64 addr,
68 u64 size,
69 u32 flags,
70 int rw_flag,
71 bool priv,
72 enum nvgpu_aperture aperture)
73{
74 struct gk20a *g = gk20a_from_vm(vm);
75 u64 vaddr;
76
77 struct nvgpu_sgt *sgt = nvgpu_sgt_create_from_mem(g, mem);
78
79 if (!sgt)
80 return -ENOMEM;
81
82 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
83 vaddr = g->ops.mm.gmmu_map(vm, addr,
84 sgt, /* sg list */
85 0, /* sg offset */
86 size,
87 gmmu_page_size_kernel,
88 0, /* kind */
89 0, /* ctag_offset */
90 flags, rw_flag,
91 false, /* clear_ctags */
92 false, /* sparse */
93 priv, /* priv */
94 NULL, /* mapping_batch handle */
95 aperture);
96 nvgpu_mutex_release(&vm->update_gmmu_lock);
97
98 nvgpu_sgt_free(g, sgt);
99
100 if (!vaddr) {
101 nvgpu_err(g, "failed to map buffer!");
102 return 0;
103 }
104
105 return vaddr;
106}
107
108/*
109 * Map a nvgpu_mem into the GMMU. This is for kernel space to use.
110 */
111u64 nvgpu_gmmu_map(struct vm_gk20a *vm,
112 struct nvgpu_mem *mem,
113 u64 size,
114 u32 flags,
115 int rw_flag,
116 bool priv,
117 enum nvgpu_aperture aperture)
118{
119 return __nvgpu_gmmu_map(vm, mem, 0, size, flags, rw_flag, priv,
120 aperture);
121}
122
123/*
124 * Like nvgpu_gmmu_map() except this can work on a fixed address.
125 */
126u64 nvgpu_gmmu_map_fixed(struct vm_gk20a *vm,
127 struct nvgpu_mem *mem,
128 u64 addr,
129 u64 size,
130 u32 flags,
131 int rw_flag,
132 bool priv,
133 enum nvgpu_aperture aperture)
134{
135 return __nvgpu_gmmu_map(vm, mem, addr, size, flags, rw_flag, priv,
136 aperture);
137}
138
139void nvgpu_gmmu_unmap(struct vm_gk20a *vm, struct nvgpu_mem *mem, u64 gpu_va)
140{
141 struct gk20a *g = gk20a_from_vm(vm);
142
143 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
144 g->ops.mm.gmmu_unmap(vm,
145 gpu_va,
146 mem->size,
147 gmmu_page_size_kernel,
148 true, /*va_allocated */
149 gk20a_mem_flag_none,
150 false,
151 NULL);
152
153 nvgpu_mutex_release(&vm->update_gmmu_lock);
154}
155
156int nvgpu_gmmu_init_page_table(struct vm_gk20a *vm)
157{
158 u32 pdb_size;
159 int err;
160
161 /*
162 * Need this just for page size. Everything else can be ignored. Also
163 * note that we can just use pgsz 0 (i.e small pages) since the number
164 * of bits present in the top level PDE are the same for small/large
165 * page VMs.
166 */
167 struct nvgpu_gmmu_attrs attrs = {
168 .pgsz = 0,
169 };
170
171 /*
172 * PDB size here must be one page so that its address is page size
173 * aligned. Although lower PDE tables can be aligned at 256B boundaries
174 * the main PDB must be page aligned.
175 */
176 pdb_size = ALIGN(pd_size(&vm->mmu_levels[0], &attrs), PAGE_SIZE);
177
178 err = __nvgpu_pd_cache_alloc_direct(vm->mm->g, &vm->pdb, pdb_size);
179 if (WARN_ON(err))
180 return err;
181
182 /*
183 * One nvgpu_smp_mb() is done after all mapping operations. Don't need
184 * individual barriers for each PD write.
185 */
186 vm->pdb.mem->skip_wmb = true;
187
188 return 0;
189}
190
191/*
192 * Ensure that there's a CPU mapping for the page directory memory. This won't
193 * always be the case for 32 bit systems since we may need to save kernel
194 * virtual memory.
195 */
196static int map_gmmu_pages(struct gk20a *g, struct nvgpu_gmmu_pd *pd)
197{
198 return nvgpu_mem_begin(g, pd->mem);
199}
200
201/*
202 * Handle any necessary CPU unmap semantics for a page directories DMA memory.
203 * For 64 bit platforms this is a noop.
204 */
205static void unmap_gmmu_pages(struct gk20a *g, struct nvgpu_gmmu_pd *pd)
206{
207 nvgpu_mem_end(g, pd->mem);
208}
209
210/*
211 * Return the _physical_ address of a page directory.
212 */
213u64 nvgpu_pde_phys_addr(struct gk20a *g, struct nvgpu_gmmu_pd *pd)
214{
215 u64 page_addr;
216
217 if (g->mm.has_physical_mode)
218 page_addr = nvgpu_mem_get_phys_addr(g, pd->mem);
219 else
220 page_addr = nvgpu_mem_get_addr(g, pd->mem);
221
222 return page_addr + pd->mem_offs;
223}
224
225/*
226 * Return the aligned length based on the page size in attrs.
227 */
228static u64 nvgpu_align_map_length(struct vm_gk20a *vm, u64 length,
229 struct nvgpu_gmmu_attrs *attrs)
230{
231 u64 page_size = vm->gmmu_page_sizes[attrs->pgsz];
232
233 return ALIGN(length, page_size);
234}
235
236static u32 pd_entries(const struct gk20a_mmu_level *l,
237 struct nvgpu_gmmu_attrs *attrs)
238{
239 /*
240 * Number of entries in a PD is easy to compute from the number of bits
241 * used to index the page directory. That is simply 2 raised to the
242 * number of bits.
243 */
244 return 1UL << (l->hi_bit[attrs->pgsz] - l->lo_bit[attrs->pgsz] + 1UL);
245}
246
247/*
248 * Computes the size of a PD table.
249 */
250static u32 pd_size(const struct gk20a_mmu_level *l,
251 struct nvgpu_gmmu_attrs *attrs)
252{
253 return pd_entries(l, attrs) * l->entry_size;
254}
255
256/*
257 * Allocate a physically contiguous region big enough for a gmmu page table
258 * of the specified level and page size. The whole range is zeroed so that any
259 * accesses will fault until proper values are programmed.
260 */
261static int pd_allocate(struct vm_gk20a *vm,
262 struct nvgpu_gmmu_pd *pd,
263 const struct gk20a_mmu_level *l,
264 struct nvgpu_gmmu_attrs *attrs)
265{
266 int err;
267
268 if (pd->mem)
269 return 0;
270
271 err = __nvgpu_pd_alloc(vm, pd, pd_size(l, attrs));
272 if (err) {
273 nvgpu_info(vm->mm->g, "error allocating page directory!");
274 return err;
275 }
276
277 /*
278 * One nvgpu_smp_mb() is done after all mapping operations. Don't need
279 * individual barriers for each PD write.
280 */
281 pd->mem->skip_wmb = true;
282
283 return 0;
284}
285
286/*
287 * Compute what page directory index at the passed level the passed virtual
288 * address corresponds to. @attrs is necessary for determining the page size
289 * which is used to pick the right bit offsets for the GMMU level.
290 */
291static u32 pd_index(const struct gk20a_mmu_level *l, u64 virt,
292 struct nvgpu_gmmu_attrs *attrs)
293{
294 u64 pd_mask = (1ULL << ((u64)l->hi_bit[attrs->pgsz] + 1)) - 1ULL;
295 u32 pd_shift = (u64)l->lo_bit[attrs->pgsz];
296
297 /*
298 * For convenience we don't bother computing the lower bound of the
299 * mask; it's easier to just shift it off.
300 */
301 return (virt & pd_mask) >> pd_shift;
302}
303
304static int pd_allocate_children(struct vm_gk20a *vm,
305 const struct gk20a_mmu_level *l,
306 struct nvgpu_gmmu_pd *pd,
307 struct nvgpu_gmmu_attrs *attrs)
308{
309 struct gk20a *g = gk20a_from_vm(vm);
310
311 if (pd->entries)
312 return 0;
313
314 pd->num_entries = pd_entries(l, attrs);
315 pd->entries = nvgpu_vzalloc(g, sizeof(struct nvgpu_gmmu_pd) *
316 pd->num_entries);
317 if (!pd->entries)
318 return -ENOMEM;
319
320 return 0;
321}
322
323/*
324 * This function programs the GMMU based on two ranges: a physical range and a
325 * GPU virtual range. The virtual is mapped to the physical. Physical in this
326 * case can mean either a real physical sysmem address or a IO virtual address
327 * (for instance when a system has an IOMMU running).
328 *
329 * The rest of the parameters are for describing the actual mapping itself.
330 *
331 * This function recursively calls itself for handling PDEs. At the final level
332 * a PTE handler is called. The phys and virt ranges are adjusted for each
333 * recursion so that each invocation of this function need only worry about the
334 * range it is passed.
335 *
336 * phys_addr will always point to a contiguous range - the discontiguous nature
337 * of DMA buffers is taken care of at the layer above this.
338 */
339static int __set_pd_level(struct vm_gk20a *vm,
340 struct nvgpu_gmmu_pd *pd,
341 int lvl,
342 u64 phys_addr,
343 u64 virt_addr, u64 length,
344 struct nvgpu_gmmu_attrs *attrs)
345{
346 int err = 0;
347 u64 pde_range;
348 struct gk20a *g = gk20a_from_vm(vm);
349 struct nvgpu_gmmu_pd *next_pd = NULL;
350 const struct gk20a_mmu_level *l = &vm->mmu_levels[lvl];
351 const struct gk20a_mmu_level *next_l = &vm->mmu_levels[lvl + 1];
352
353 /*
354 * 5 levels for Pascal+. For pre-pascal we only have 2. This puts
355 * offsets into the page table debugging code which makes it easier to
356 * see what level prints are from.
357 */
358 static const char *__lvl_debug[] = {
359 "", /* L=0 */
360 " ", /* L=1 */
361 " ", /* L=2 */
362 " ", /* L=3 */
363 " ", /* L=4 */
364 };
365
366 pde_range = 1ULL << (u64)l->lo_bit[attrs->pgsz];
367
368 __gmmu_dbg_v(g, attrs,
369 "L=%d %sGPU virt %#-12llx +%#-9llx -> phys %#-12llx",
370 lvl,
371 __lvl_debug[lvl],
372 virt_addr,
373 length,
374 phys_addr);
375
376 /*
377 * Iterate across the mapping in chunks the size of this level's PDE.
378 * For each of those chunks program our level's PDE and then, if there's
379 * a next level, program the next level's PDEs/PTEs.
380 */
381 while (length) {
382 u32 pd_idx = pd_index(l, virt_addr, attrs);
383 u64 chunk_size;
384 u64 target_addr;
385
386 /*
387 * Truncate the pde_range when the virtual address does not
388 * start at a PDE boundary.
389 */
390 chunk_size = min(length,
391 pde_range - (virt_addr & (pde_range - 1)));
392
393 /*
394 * If the next level has an update_entry function then we know
395 * that _this_ level points to PDEs (not PTEs). Thus we need to
396 * have a bunch of children PDs.
397 */
398 if (next_l->update_entry) {
399 if (pd_allocate_children(vm, l, pd, attrs))
400 return -ENOMEM;
401
402 /*
403 * Get the next PD so that we know what to put in this
404 * current PD. If the next level is actually PTEs then
405 * we don't need this - we will just use the real
406 * physical target.
407 */
408 next_pd = &pd->entries[pd_idx];
409
410 /*
411 * Allocate the backing memory for next_pd.
412 */
413 if (pd_allocate(vm, next_pd, next_l, attrs))
414 return -ENOMEM;
415 }
416
417 /*
418 * This is the address we want to program into the actual PDE/
419 * PTE. When the next level is PDEs we need the target address
420 * to be the table of PDEs. When the next level is PTEs the
421 * target addr is the real physical address we are aiming for.
422 */
423 target_addr = next_pd ?
424 nvgpu_pde_phys_addr(g, next_pd) :
425 phys_addr;
426
427 l->update_entry(vm, l,
428 pd, pd_idx,
429 virt_addr,
430 target_addr,
431 attrs);
432
433 if (next_l->update_entry) {
434 err = map_gmmu_pages(g, next_pd);
435 if (err) {
436 nvgpu_err(g,
437 "couldn't map ptes for update as=%d",
438 vm_aspace_id(vm));
439 return err;
440 }
441
442 err = __set_pd_level(vm, next_pd,
443 lvl + 1,
444 phys_addr,
445 virt_addr,
446 chunk_size,
447 attrs);
448 unmap_gmmu_pages(g, next_pd);
449
450 if (err)
451 return err;
452 }
453
454 virt_addr += chunk_size;
455
456 /*
457 * Only add to phys_addr if it's non-zero. A zero value implies
458 * we are unmapping as as a result we don't want to place
459 * non-zero phys addresses in the PTEs. A non-zero phys-addr
460 * would also confuse the lower level PTE programming code.
461 */
462 if (phys_addr)
463 phys_addr += chunk_size;
464 length -= chunk_size;
465 }
466
467 __gmmu_dbg_v(g, attrs, "L=%d %s%s", lvl, __lvl_debug[lvl], "ret!");
468
469 return 0;
470}
471
472static int __nvgpu_gmmu_do_update_page_table(struct vm_gk20a *vm,
473 struct nvgpu_sgt *sgt,
474 u64 space_to_skip,
475 u64 virt_addr,
476 u64 length,
477 struct nvgpu_gmmu_attrs *attrs)
478{
479 struct gk20a *g = gk20a_from_vm(vm);
480 void *sgl;
481 int err = 0;
482
483 if (!sgt) {
484 /*
485 * This is considered an unmap. Just pass in 0 as the physical
486 * address for the entire GPU range.
487 */
488 err = __set_pd_level(vm, &vm->pdb,
489 0,
490 0,
491 virt_addr, length,
492 attrs);
493 return err;
494 }
495
496 /*
497 * At this point we have a scatter-gather list pointing to some number
498 * of discontiguous chunks of memory. We must iterate over that list and
499 * generate a GMMU map call for each chunk. There are two possibilities:
500 * either an IOMMU is enabled or not. When an IOMMU is enabled the
501 * mapping is simple since the "physical" address is actually a virtual
502 * IO address and will be contiguous.
503 */
504 if (attrs->aperture == APERTURE_SYSMEM && !g->mm.bypass_smmu) {
505 u64 io_addr = nvgpu_sgt_get_gpu_addr(g, sgt, sgt->sgl, attrs);
506
507 io_addr += space_to_skip;
508
509 err = __set_pd_level(vm, &vm->pdb,
510 0,
511 io_addr,
512 virt_addr,
513 length,
514 attrs);
515
516 return err;
517 }
518
519 /*
520 * Finally: last possible case: do the no-IOMMU mapping. In this case we
521 * really are mapping physical pages directly.
522 */
523 nvgpu_sgt_for_each_sgl(sgl, sgt) {
524 u64 phys_addr;
525 u64 chunk_length;
526
527 /*
528 * Cut out sgl ents for space_to_skip.
529 */
530 if (space_to_skip &&
531 space_to_skip >= nvgpu_sgt_get_length(sgt, sgl)) {
532 space_to_skip -= nvgpu_sgt_get_length(sgt, sgl);
533 continue;
534 }
535
536 phys_addr = nvgpu_sgt_get_phys(sgt, sgl) + space_to_skip;
537 chunk_length = min(length,
538 nvgpu_sgt_get_length(sgt, sgl) - space_to_skip);
539
540 err = __set_pd_level(vm, &vm->pdb,
541 0,
542 phys_addr,
543 virt_addr,
544 chunk_length,
545 attrs);
546 if (err)
547 break;
548
549 /* Space has been skipped so zero this for future chunks. */
550 space_to_skip = 0;
551
552 /*
553 * Update the map pointer and the remaining length.
554 */
555 virt_addr += chunk_length;
556 length -= chunk_length;
557
558 if (length == 0)
559 break;
560 }
561
562 return err;
563}
564
565/*
566 * This is the true top level GMMU mapping logic. This breaks down the incoming
567 * scatter gather table and does actual programming of GPU virtual address to
568 * physical* address.
569 *
570 * The update of each level of the page tables is farmed out to chip specific
571 * implementations. But the logic around that is generic to all chips. Every
572 * chip has some number of PDE levels and then a PTE level.
573 *
574 * Each chunk of the incoming SGL is sent to the chip specific implementation
575 * of page table update.
576 *
577 * [*] Note: the "physical" address may actually be an IO virtual address in the
578 * case of SMMU usage.
579 */
580static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm,
581 struct nvgpu_sgt *sgt,
582 u64 space_to_skip,
583 u64 virt_addr,
584 u64 length,
585 struct nvgpu_gmmu_attrs *attrs)
586{
587 struct gk20a *g = gk20a_from_vm(vm);
588 u32 page_size;
589 int err;
590
591 /* note: here we need to map kernel to small, since the
592 * low-level mmu code assumes 0 is small and 1 is big pages */
593 if (attrs->pgsz == gmmu_page_size_kernel)
594 attrs->pgsz = gmmu_page_size_small;
595
596 page_size = vm->gmmu_page_sizes[attrs->pgsz];
597
598 if (space_to_skip & (page_size - 1))
599 return -EINVAL;
600
601 /*
602 * Update length to be aligned to the passed page size.
603 */
604 length = nvgpu_align_map_length(vm, length, attrs);
605
606 err = map_gmmu_pages(g, &vm->pdb);
607 if (err) {
608 nvgpu_err(g, "couldn't map ptes for update as=%d",
609 vm_aspace_id(vm));
610 return err;
611 }
612
613 __gmmu_dbg(g, attrs,
614 "vm=%s "
615 "%-5s GPU virt %#-12llx +%#-9llx phys %#-12llx "
616 "phys offset: %#-4llx; pgsz: %3dkb perm=%-2s | "
617 "kind=%#02x APT=%-6s %c%c%c%c%c",
618 vm->name,
619 sgt ? "MAP" : "UNMAP",
620 virt_addr,
621 length,
622 sgt ? nvgpu_sgt_get_phys(sgt, sgt->sgl) : 0,
623 space_to_skip,
624 page_size >> 10,
625 nvgpu_gmmu_perm_str(attrs->rw_flag),
626 attrs->kind_v,
627 nvgpu_aperture_str(attrs->aperture),
628 attrs->cacheable ? 'C' : 'c', /* C = cached, V = volatile. */
629 attrs->sparse ? 'S' : '-',
630 attrs->priv ? 'P' : '-',
631 attrs->coherent ? 'c' : '-',
632 attrs->valid ? 'V' : '-');
633
634 err = __nvgpu_gmmu_do_update_page_table(vm,
635 sgt,
636 space_to_skip,
637 virt_addr,
638 length,
639 attrs);
640
641 unmap_gmmu_pages(g, &vm->pdb);
642 nvgpu_smp_mb();
643
644 __gmmu_dbg(g, attrs, "%-5s Done!", sgt ? "MAP" : "UNMAP");
645
646 return err;
647}
648
649/**
650 * gk20a_locked_gmmu_map - Map a buffer into the GMMU
651 *
652 * This is for non-vGPU chips. It's part of the HAL at the moment but really
653 * should not be. Chip specific stuff is handled at the PTE/PDE programming
654 * layer. The rest of the logic is essentially generic for all chips.
655 *
656 * To call this function you must have locked the VM lock: vm->update_gmmu_lock.
657 * However, note: this function is not called directly. It's used through the
658 * mm.gmmu_lock() HAL. So before calling the mm.gmmu_lock() HAL make sure you
659 * have the update_gmmu_lock aquired.
660 */
661u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
662 u64 vaddr,
663 struct nvgpu_sgt *sgt,
664 u64 buffer_offset,
665 u64 size,
666 int pgsz_idx,
667 u8 kind_v,
668 u32 ctag_offset,
669 u32 flags,
670 int rw_flag,
671 bool clear_ctags,
672 bool sparse,
673 bool priv,
674 struct vm_gk20a_mapping_batch *batch,
675 enum nvgpu_aperture aperture)
676{
677 struct gk20a *g = gk20a_from_vm(vm);
678 int err = 0;
679 bool allocated = false;
680 int ctag_granularity = g->ops.fb.compression_page_size(g);
681 struct nvgpu_gmmu_attrs attrs = {
682 .pgsz = pgsz_idx,
683 .kind_v = kind_v,
684 .ctag = (u64)ctag_offset * (u64)ctag_granularity,
685 .cacheable = flags & NVGPU_AS_MAP_BUFFER_FLAGS_CACHEABLE,
686 .rw_flag = rw_flag,
687 .sparse = sparse,
688 .priv = priv,
689 .coherent = flags & NVGPU_AS_MAP_BUFFER_FLAGS_IO_COHERENT,
690 .valid = !(flags & NVGPU_AS_MAP_BUFFER_FLAGS_UNMAPPED_PTE),
691 .aperture = aperture
692 };
693
694#ifdef CONFIG_TEGRA_19x_GPU
695 nvgpu_gmmu_add_t19x_attrs(&attrs, flags);
696#endif
697
698 /*
699 * Only allocate a new GPU VA range if we haven't already been passed a
700 * GPU VA range. This facilitates fixed mappings.
701 */
702 if (!vaddr) {
703 vaddr = __nvgpu_vm_alloc_va(vm, size, pgsz_idx);
704 if (!vaddr) {
705 nvgpu_err(g, "failed to allocate va space");
706 err = -ENOMEM;
707 goto fail_alloc;
708 }
709 allocated = true;
710 }
711
712 err = __nvgpu_gmmu_update_page_table(vm, sgt, buffer_offset,
713 vaddr, size, &attrs);
714 if (err) {
715 nvgpu_err(g, "failed to update ptes on map");
716 goto fail_validate;
717 }
718
719 if (!batch)
720 g->ops.fb.tlb_invalidate(g, vm->pdb.mem);
721 else
722 batch->need_tlb_invalidate = true;
723
724 return vaddr;
725
726fail_validate:
727 if (allocated)
728 __nvgpu_vm_free_va(vm, vaddr, pgsz_idx);
729fail_alloc:
730 nvgpu_err(g, "%s: failed with err=%d", __func__, err);
731 return 0;
732}
733
734void gk20a_locked_gmmu_unmap(struct vm_gk20a *vm,
735 u64 vaddr,
736 u64 size,
737 int pgsz_idx,
738 bool va_allocated,
739 int rw_flag,
740 bool sparse,
741 struct vm_gk20a_mapping_batch *batch)
742{
743 int err = 0;
744 struct gk20a *g = gk20a_from_vm(vm);
745 struct nvgpu_gmmu_attrs attrs = {
746 .pgsz = pgsz_idx,
747 .kind_v = 0,
748 .ctag = 0,
749 .cacheable = 0,
750 .rw_flag = rw_flag,
751 .sparse = sparse,
752 .priv = 0,
753 .coherent = 0,
754 .valid = 0,
755 .aperture = APERTURE_INVALID,
756 };
757
758 if (va_allocated) {
759 err = __nvgpu_vm_free_va(vm, vaddr, pgsz_idx);
760 if (err) {
761 nvgpu_err(g, "failed to free va");
762 return;
763 }
764 }
765
766 /* unmap here needs to know the page size we assigned at mapping */
767 err = __nvgpu_gmmu_update_page_table(vm, NULL, 0,
768 vaddr, size, &attrs);
769 if (err)
770 nvgpu_err(g, "failed to update gmmu ptes on unmap");
771
772 if (!batch) {
773 gk20a_mm_l2_flush(g, true);
774 g->ops.fb.tlb_invalidate(g, vm->pdb.mem);
775 } else {
776 if (!batch->gpu_l2_flushed) {
777 gk20a_mm_l2_flush(g, true);
778 batch->gpu_l2_flushed = true;
779 }
780 batch->need_tlb_invalidate = true;
781 }
782}
783
784u32 __nvgpu_pte_words(struct gk20a *g)
785{
786 const struct gk20a_mmu_level *l = g->ops.mm.get_mmu_levels(g, SZ_64K);
787 const struct gk20a_mmu_level *next_l;
788
789 /*
790 * Iterate to the bottom GMMU level - the PTE level. The levels array
791 * is always NULL terminated (by the update_entry function).
792 */
793 do {
794 next_l = l + 1;
795 if (!next_l->update_entry)
796 break;
797
798 l++;
799 } while (true);
800
801 return (u32)(l->entry_size / sizeof(u32));
802}
803
804/*
805 * Recursively walk the pages tables to find the PTE.
806 */
807static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm,
808 struct nvgpu_gmmu_pd *pd,
809 u64 vaddr, int lvl,
810 struct nvgpu_gmmu_attrs *attrs,
811 u32 *data,
812 struct nvgpu_gmmu_pd **pd_out, u32 *pd_idx_out,
813 u32 *pd_offs_out)
814{
815 const struct gk20a_mmu_level *l = &vm->mmu_levels[lvl];
816 const struct gk20a_mmu_level *next_l = &vm->mmu_levels[lvl + 1];
817 u32 pd_idx = pd_index(l, vaddr, attrs);
818 u32 pte_base;
819 u32 pte_size;
820 u32 i;
821
822 /*
823 * If this isn't the final level (i.e there's a valid next level)
824 * then find the next level PD and recurse.
825 */
826 if (next_l->update_entry) {
827 struct nvgpu_gmmu_pd *pd_next = pd->entries + pd_idx;
828
829 /* Invalid entry! */
830 if (!pd_next->mem)
831 return -EINVAL;
832
833 attrs->pgsz = l->get_pgsz(g, pd, pd_idx);
834
835 if (attrs->pgsz >= gmmu_nr_page_sizes)
836 return -EINVAL;
837
838 return __nvgpu_locate_pte(g, vm, pd_next,
839 vaddr, lvl + 1, attrs,
840 data, pd_out, pd_idx_out,
841 pd_offs_out);
842 }
843
844 if (!pd->mem)
845 return -EINVAL;
846
847 /*
848 * Take into account the real offset into the nvgpu_mem since the PD
849 * may be located at an offset other than 0 (due to PD packing).
850 */
851 pte_base = (pd->mem_offs / sizeof(u32)) +
852 pd_offset_from_index(l, pd_idx);
853 pte_size = (u32)(l->entry_size / sizeof(u32));
854
855 if (data) {
856 map_gmmu_pages(g, pd);
857 for (i = 0; i < pte_size; i++)
858 data[i] = nvgpu_mem_rd32(g, pd->mem, pte_base + i);
859 unmap_gmmu_pages(g, pd);
860 }
861
862 if (pd_out)
863 *pd_out = pd;
864
865 if (pd_idx_out)
866 *pd_idx_out = pd_idx;
867
868 if (pd_offs_out)
869 *pd_offs_out = pd_offset_from_index(l, pd_idx);
870
871 return 0;
872}
873
874int __nvgpu_get_pte(struct gk20a *g, struct vm_gk20a *vm, u64 vaddr, u32 *pte)
875{
876 struct nvgpu_gmmu_attrs attrs = {
877 .pgsz = 0,
878 };
879
880 return __nvgpu_locate_pte(g, vm, &vm->pdb,
881 vaddr, 0, &attrs,
882 pte, NULL, NULL, NULL);
883}
884
885int __nvgpu_set_pte(struct gk20a *g, struct vm_gk20a *vm, u64 vaddr, u32 *pte)
886{
887 struct nvgpu_gmmu_pd *pd;
888 u32 pd_idx, pd_offs, pte_size, i;
889 int err;
890 struct nvgpu_gmmu_attrs attrs = {
891 .pgsz = 0,
892 };
893 struct nvgpu_gmmu_attrs *attrs_ptr = &attrs;
894
895 err = __nvgpu_locate_pte(g, vm, &vm->pdb,
896 vaddr, 0, &attrs,
897 NULL, &pd, &pd_idx, &pd_offs);
898 if (err)
899 return err;
900
901 pte_size = __nvgpu_pte_words(g);
902
903 map_gmmu_pages(g, pd);
904 for (i = 0; i < pte_size; i++) {
905 pd_write(g, pd, pd_offs + i, pte[i]);
906 pte_dbg(g, attrs_ptr,
907 "PTE: idx=%-4u (%d) 0x%08x", pd_idx, i, pte[i]);
908 }
909 unmap_gmmu_pages(g, pd);
910
911 /*
912 * Ensures the pd_write()s are done. The pd_write() does not do this
913 * since generally there's lots of pd_write()s called one after another.
914 * There probably also needs to be a TLB invalidate as well but we leave
915 * that to the caller of this function.
916 */
917 nvgpu_smp_wmb();
918
919 return 0;
920}
generated by cgit v1.2.2 (git 2.25.0) at 2024-08-22 14:47:52 -0400