1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
|
/*
* Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <nvgpu/dma.h>
#include <nvgpu/gmmu.h>
#include <nvgpu/nvgpu_mem.h>
#include <nvgpu/page_allocator.h>
#include <nvgpu/log.h>
#include <nvgpu/bug.h>
#include <nvgpu/enabled.h>
#include <nvgpu/kmem.h>
#include <nvgpu/vidmem.h>
#include <nvgpu/linux/dma.h>
#include <nvgpu/linux/vidmem.h>
#include "os_linux.h"
#include "gk20a/gk20a.h"
#include "gk20a/mm_gk20a.h"
u32 __nvgpu_aperture_mask(struct gk20a *g, enum nvgpu_aperture aperture,
u32 sysmem_mask, u32 vidmem_mask)
{
switch (aperture) {
case APERTURE_SYSMEM:
/* some igpus consider system memory vidmem */
return nvgpu_is_enabled(g, NVGPU_MM_HONORS_APERTURE)
? sysmem_mask : vidmem_mask;
case APERTURE_VIDMEM:
/* for dgpus only */
return vidmem_mask;
case APERTURE_INVALID:
WARN_ON("Bad aperture");
}
return 0;
}
u32 nvgpu_aperture_mask(struct gk20a *g, struct nvgpu_mem *mem,
u32 sysmem_mask, u32 vidmem_mask)
{
return __nvgpu_aperture_mask(g, mem->aperture,
sysmem_mask, vidmem_mask);
}
int nvgpu_mem_begin(struct gk20a *g, struct nvgpu_mem *mem)
{
void *cpu_va;
if (WARN_ON(mem->cpu_accessible)) {
nvgpu_warn(g, "nested");
return -EBUSY;
}
/* flag that the intent is to allow CPU access to the memory. */
mem->cpu_accessible = true;
if (mem->aperture != APERTURE_SYSMEM || g->mm.force_pramin)
return 0;
/*
* A CPU mapping is implicitly made for all SYSMEM DMA allocations that
* don't have NVGPU_DMA_NO_KERNEL_MAPPING. Thus we don't need to make
* another CPU mapping.
*/
if (!(mem->priv.flags & NVGPU_DMA_NO_KERNEL_MAPPING))
return 0;
cpu_va = vmap(mem->priv.pages,
PAGE_ALIGN(mem->size) >> PAGE_SHIFT,
0, pgprot_writecombine(PAGE_KERNEL));
if (WARN_ON(!cpu_va)) {
mem->cpu_accessible = false;
return -ENOMEM;
}
mem->cpu_va = cpu_va;
return 0;
}
void nvgpu_mem_end(struct gk20a *g, struct nvgpu_mem *mem)
{
mem->cpu_accessible = false;
if (mem->aperture != APERTURE_SYSMEM || g->mm.force_pramin)
return;
/*
* Similar to nvgpu_mem_begin() we don't need to unmap the CPU mapping
* already made by the DMA API.
*/
if (!(mem->priv.flags & NVGPU_DMA_NO_KERNEL_MAPPING))
return;
vunmap(mem->cpu_va);
mem->cpu_va = NULL;
}
static void pramin_access_batch_rd_n(struct gk20a *g, u32 start, u32 words, u32 **arg)
{
struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
u32 r = start, *dest_u32 = *arg;
if (!l->regs) {
__gk20a_warn_on_no_regs();
return;
}
while (words--) {
*dest_u32++ = gk20a_readl(g, r);
r += sizeof(u32);
}
*arg = dest_u32;
}
u32 nvgpu_mem_rd32(struct gk20a *g, struct nvgpu_mem *mem, u32 w)
{
u32 data = 0;
if (mem->aperture == APERTURE_SYSMEM && !g->mm.force_pramin) {
u32 *ptr = mem->cpu_va;
WARN_ON(!ptr);
data = ptr[w];
#ifdef CONFIG_TEGRA_SIMULATION_PLATFORM
gk20a_dbg(gpu_dbg_mem, " %p = 0x%x", ptr + w, data);
#endif
} else if (mem->aperture == APERTURE_VIDMEM || g->mm.force_pramin) {
u32 value;
u32 *p = &value;
nvgpu_pramin_access_batched(g, mem, w * sizeof(u32),
sizeof(u32), pramin_access_batch_rd_n, &p);
data = value;
} else {
WARN_ON("Accessing unallocated nvgpu_mem");
}
return data;
}
u32 nvgpu_mem_rd(struct gk20a *g, struct nvgpu_mem *mem, u32 offset)
{
WARN_ON(offset & 3);
return nvgpu_mem_rd32(g, mem, offset / sizeof(u32));
}
void nvgpu_mem_rd_n(struct gk20a *g, struct nvgpu_mem *mem,
u32 offset, void *dest, u32 size)
{
WARN_ON(offset & 3);
WARN_ON(size & 3);
if (mem->aperture == APERTURE_SYSMEM && !g->mm.force_pramin) {
u8 *src = (u8 *)mem->cpu_va + offset;
WARN_ON(!mem->cpu_va);
memcpy(dest, src, size);
#ifdef CONFIG_TEGRA_SIMULATION_PLATFORM
if (size)
gk20a_dbg(gpu_dbg_mem, " %p = 0x%x ... [%d bytes]",
src, *dest, size);
#endif
} else if (mem->aperture == APERTURE_VIDMEM || g->mm.force_pramin) {
u32 *dest_u32 = dest;
nvgpu_pramin_access_batched(g, mem, offset, size,
pramin_access_batch_rd_n, &dest_u32);
} else {
WARN_ON("Accessing unallocated nvgpu_mem");
}
}
static void pramin_access_batch_wr_n(struct gk20a *g, u32 start, u32 words, u32 **arg)
{
struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
u32 r = start, *src_u32 = *arg;
if (!l->regs) {
__gk20a_warn_on_no_regs();
return;
}
while (words--) {
writel_relaxed(*src_u32++, l->regs + r);
r += sizeof(u32);
}
*arg = src_u32;
}
void nvgpu_mem_wr32(struct gk20a *g, struct nvgpu_mem *mem, u32 w, u32 data)
{
if (mem->aperture == APERTURE_SYSMEM && !g->mm.force_pramin) {
u32 *ptr = mem->cpu_va;
WARN_ON(!ptr);
#ifdef CONFIG_TEGRA_SIMULATION_PLATFORM
gk20a_dbg(gpu_dbg_mem, " %p = 0x%x", ptr + w, data);
#endif
ptr[w] = data;
} else if (mem->aperture == APERTURE_VIDMEM || g->mm.force_pramin) {
u32 value = data;
u32 *p = &value;
nvgpu_pramin_access_batched(g, mem, w * sizeof(u32),
sizeof(u32), pramin_access_batch_wr_n, &p);
if (!mem->skip_wmb)
wmb();
} else {
WARN_ON("Accessing unallocated nvgpu_mem");
}
}
void nvgpu_mem_wr(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, u32 data)
{
WARN_ON(offset & 3);
nvgpu_mem_wr32(g, mem, offset / sizeof(u32), data);
}
void nvgpu_mem_wr_n(struct gk20a *g, struct nvgpu_mem *mem, u32 offset,
void *src, u32 size)
{
WARN_ON(offset & 3);
WARN_ON(size & 3);
if (mem->aperture == APERTURE_SYSMEM && !g->mm.force_pramin) {
u8 *dest = (u8 *)mem->cpu_va + offset;
WARN_ON(!mem->cpu_va);
#ifdef CONFIG_TEGRA_SIMULATION_PLATFORM
if (size)
gk20a_dbg(gpu_dbg_mem, " %p = 0x%x ... [%d bytes]",
dest, *src, size);
#endif
memcpy(dest, src, size);
} else if (mem->aperture == APERTURE_VIDMEM || g->mm.force_pramin) {
u32 *src_u32 = src;
nvgpu_pramin_access_batched(g, mem, offset, size,
pramin_access_batch_wr_n, &src_u32);
if (!mem->skip_wmb)
wmb();
} else {
WARN_ON("Accessing unallocated nvgpu_mem");
}
}
static void pramin_access_batch_set(struct gk20a *g, u32 start, u32 words, u32 **arg)
{
struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
u32 r = start, repeat = **arg;
if (!l->regs) {
__gk20a_warn_on_no_regs();
return;
}
while (words--) {
writel_relaxed(repeat, l->regs + r);
r += sizeof(u32);
}
}
void nvgpu_memset(struct gk20a *g, struct nvgpu_mem *mem, u32 offset,
u32 c, u32 size)
{
WARN_ON(offset & 3);
WARN_ON(size & 3);
WARN_ON(c & ~0xff);
c &= 0xff;
if (mem->aperture == APERTURE_SYSMEM && !g->mm.force_pramin) {
u8 *dest = (u8 *)mem->cpu_va + offset;
WARN_ON(!mem->cpu_va);
#ifdef CONFIG_TEGRA_SIMULATION_PLATFORM
if (size)
gk20a_dbg(gpu_dbg_mem, " %p = 0x%x [times %d]",
dest, c, size);
#endif
memset(dest, c, size);
} else if (mem->aperture == APERTURE_VIDMEM || g->mm.force_pramin) {
u32 repeat_value = c | (c << 8) | (c << 16) | (c << 24);
u32 *p = &repeat_value;
nvgpu_pramin_access_batched(g, mem, offset, size,
pramin_access_batch_set, &p);
if (!mem->skip_wmb)
wmb();
} else {
WARN_ON("Accessing unallocated nvgpu_mem");
}
}
/*
* Obtain a SYSMEM address from a Linux SGL. This should eventually go away
* and/or become private to this file once all bad usages of Linux SGLs are
* cleaned up in the driver.
*/
u64 nvgpu_mem_get_addr_sgl(struct gk20a *g, struct scatterlist *sgl)
{
struct nvgpu_os_linux *l = container_of(g, struct nvgpu_os_linux, g);
if (!device_is_iommuable(l->dev))
return g->ops.mm.gpu_phys_addr(g, NULL, sg_phys(sgl));
if (sg_dma_address(sgl) == 0)
return g->ops.mm.gpu_phys_addr(g, NULL, sg_phys(sgl));
if (sg_dma_address(sgl) == DMA_ERROR_CODE)
return 0;
return nvgpu_mem_iommu_translate(g, sg_dma_address(sgl));
}
/*
* Obtain the address the GPU should use from the %mem assuming this is a SYSMEM
* allocation.
*/
static u64 nvgpu_mem_get_addr_sysmem(struct gk20a *g, struct nvgpu_mem *mem)
{
return nvgpu_mem_get_addr_sgl(g, mem->priv.sgt->sgl);
}
/*
* Return the base address of %mem. Handles whether this is a VIDMEM or SYSMEM
* allocation.
*
* Note: this API does not make sense to use for _VIDMEM_ buffers with greater
* than one scatterlist chunk. If there's more than one scatterlist chunk then
* the buffer will not be contiguous. As such the base address probably isn't
* very useful. This is true for SYSMEM as well, if there's no IOMMU.
*
* However! It _is_ OK to use this on discontiguous sysmem buffers _if_ there's
* an IOMMU present and enabled for the GPU.
*
* %attrs can be NULL. If it is not NULL then it may be inspected to determine
* if the address needs to be modified before writing into a PTE.
*/
u64 nvgpu_mem_get_addr(struct gk20a *g, struct nvgpu_mem *mem)
{
struct nvgpu_page_alloc *alloc;
if (mem->aperture == APERTURE_SYSMEM)
return nvgpu_mem_get_addr_sysmem(g, mem);
/*
* Otherwise get the vidmem address.
*/
alloc = mem->vidmem_alloc;
/* This API should not be used with > 1 chunks */
WARN_ON(alloc->nr_chunks != 1);
return alloc->base;
}
/*
* This should only be used on contiguous buffers regardless of whether
* there's an IOMMU present/enabled. This applies to both SYSMEM and
* VIDMEM.
*/
u64 nvgpu_mem_get_phys_addr(struct gk20a *g, struct nvgpu_mem *mem)
{
/*
* For a VIDMEM buf, this is identical to simply get_addr() so just fall
* back to that.
*/
if (mem->aperture == APERTURE_VIDMEM)
return nvgpu_mem_get_addr(g, mem);
return sg_phys(mem->priv.sgt->sgl);
}
/*
* Be careful how you use this! You are responsible for correctly freeing this
* memory.
*/
int nvgpu_mem_create_from_mem(struct gk20a *g,
struct nvgpu_mem *dest, struct nvgpu_mem *src,
int start_page, int nr_pages)
{
int ret;
u64 start = start_page * PAGE_SIZE;
u64 size = nr_pages * PAGE_SIZE;
dma_addr_t new_iova;
if (src->aperture != APERTURE_SYSMEM)
return -EINVAL;
/* Some silly things a caller might do... */
if (size > src->size)
return -EINVAL;
if ((start + size) > src->size)
return -EINVAL;
dest->mem_flags = src->mem_flags | NVGPU_MEM_FLAG_SHADOW_COPY;
dest->aperture = src->aperture;
dest->skip_wmb = src->skip_wmb;
dest->size = size;
/*
* Re-use the CPU mapping only if the mapping was made by the DMA API.
*/
if (!(src->priv.flags & NVGPU_DMA_NO_KERNEL_MAPPING))
dest->cpu_va = src->cpu_va + (PAGE_SIZE * start_page);
dest->priv.pages = src->priv.pages + start_page;
dest->priv.flags = src->priv.flags;
new_iova = sg_dma_address(src->priv.sgt->sgl) ?
sg_dma_address(src->priv.sgt->sgl) + start : 0;
/*
* Make a new SG table that is based only on the subset of pages that
* is passed to us. This table gets freed by the dma free routines.
*/
if (src->priv.flags & NVGPU_DMA_NO_KERNEL_MAPPING)
ret = nvgpu_get_sgtable_from_pages(g, &dest->priv.sgt,
src->priv.pages + start_page,
new_iova, size);
else
ret = nvgpu_get_sgtable(g, &dest->priv.sgt, dest->cpu_va,
new_iova, size);
return ret;
}
int __nvgpu_mem_create_from_pages(struct gk20a *g, struct nvgpu_mem *dest,
struct page **pages, int nr_pages)
{
struct sg_table *sgt;
struct page **our_pages =
nvgpu_kmalloc(g, sizeof(struct page *) * nr_pages);
if (!our_pages)
return -ENOMEM;
memcpy(our_pages, pages, sizeof(struct page *) * nr_pages);
if (nvgpu_get_sgtable_from_pages(g, &sgt, pages, 0,
nr_pages * PAGE_SIZE)) {
nvgpu_kfree(g, our_pages);
return -ENOMEM;
}
/*
* If we are making an SGT from physical pages we can be reasonably
* certain that this should bypass the SMMU - thus we set the DMA (aka
* IOVA) address to 0. This tells the GMMU mapping code to not make a
* mapping directed to the SMMU.
*/
sg_dma_address(sgt->sgl) = 0;
dest->mem_flags = __NVGPU_MEM_FLAG_NO_DMA;
dest->aperture = APERTURE_SYSMEM;
dest->skip_wmb = 0;
dest->size = PAGE_SIZE * nr_pages;
dest->priv.flags = 0;
dest->priv.pages = our_pages;
dest->priv.sgt = sgt;
return 0;
}
int __nvgpu_mem_create_from_phys(struct gk20a *g, struct nvgpu_mem *dest,
u64 src_phys, int nr_pages)
{
struct page **pages =
nvgpu_kmalloc(g, sizeof(struct page *) * nr_pages);
int i, ret = 0;
if (!pages)
return -ENOMEM;
for (i = 0; i < nr_pages; i++)
pages[i] = phys_to_page(src_phys + PAGE_SIZE * i);
ret = __nvgpu_mem_create_from_pages(g, dest, pages, nr_pages);
nvgpu_kfree(g, pages);
return ret;
}
static void *nvgpu_mem_linux_sgl_next(void *sgl)
{
return sg_next((struct scatterlist *)sgl);
}
static u64 nvgpu_mem_linux_sgl_phys(void *sgl)
{
return (u64)sg_phys((struct scatterlist *)sgl);
}
static u64 nvgpu_mem_linux_sgl_dma(void *sgl)
{
return (u64)sg_dma_address((struct scatterlist *)sgl);
}
static u64 nvgpu_mem_linux_sgl_length(void *sgl)
{
return (u64)((struct scatterlist *)sgl)->length;
}
static u64 nvgpu_mem_linux_sgl_gpu_addr(struct gk20a *g, void *sgl,
struct nvgpu_gmmu_attrs *attrs)
{
if (sg_dma_address((struct scatterlist *)sgl) == 0)
return g->ops.mm.gpu_phys_addr(g, attrs,
sg_phys((struct scatterlist *)sgl));
if (sg_dma_address((struct scatterlist *)sgl) == DMA_ERROR_CODE)
return 0;
return nvgpu_mem_iommu_translate(g,
sg_dma_address((struct scatterlist *)sgl));
}
static void nvgpu_mem_linux_sgl_free(struct gk20a *g, struct nvgpu_sgt *sgt)
{
/*
* Free this SGT. All we do is free the passed SGT. The actual Linux
* SGT/SGL needs to be freed separately.
*/
nvgpu_kfree(g, sgt);
}
static const struct nvgpu_sgt_ops nvgpu_linux_sgt_ops = {
.sgl_next = nvgpu_mem_linux_sgl_next,
.sgl_phys = nvgpu_mem_linux_sgl_phys,
.sgl_dma = nvgpu_mem_linux_sgl_dma,
.sgl_length = nvgpu_mem_linux_sgl_length,
.sgl_gpu_addr = nvgpu_mem_linux_sgl_gpu_addr,
.sgt_free = nvgpu_mem_linux_sgl_free,
};
static struct nvgpu_sgt *__nvgpu_mem_get_sgl_from_vidmem(
struct gk20a *g,
struct scatterlist *linux_sgl)
{
struct nvgpu_page_alloc *vidmem_alloc;
vidmem_alloc = nvgpu_vidmem_get_page_alloc(linux_sgl);
if (!vidmem_alloc)
return NULL;
return &vidmem_alloc->sgt;
}
struct nvgpu_sgt *nvgpu_linux_sgt_create(struct gk20a *g, struct sg_table *sgt)
{
struct nvgpu_sgt *nvgpu_sgt;
struct scatterlist *linux_sgl = sgt->sgl;
if (nvgpu_addr_is_vidmem_page_alloc(sg_dma_address(linux_sgl)))
return __nvgpu_mem_get_sgl_from_vidmem(g, linux_sgl);
nvgpu_sgt = nvgpu_kzalloc(g, sizeof(*nvgpu_sgt));
if (!nvgpu_sgt)
return NULL;
nvgpu_log(g, gpu_dbg_sgl, "Making Linux SGL!");
nvgpu_sgt->sgl = sgt->sgl;
nvgpu_sgt->ops = &nvgpu_linux_sgt_ops;
return nvgpu_sgt;
}
struct nvgpu_sgt *nvgpu_sgt_create_from_mem(struct gk20a *g,
struct nvgpu_mem *mem)
{
return nvgpu_linux_sgt_create(g, mem->priv.sgt);
}
|