1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
|
/*
* GK20A memory management
*
* Copyright (c) 2011-2015, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef MM_GK20A_H
#define MM_GK20A_H
#include <linux/scatterlist.h>
#include <linux/dma-attrs.h>
#include <linux/iommu.h>
#include <asm/dma-iommu.h>
#include <asm/cacheflush.h>
#include "gk20a_allocator.h"
/*
* Amount of the GVA space we actually use is smaller than the available space.
* The bottom 16GB of the space are used for small pages, the remaining high
* memory is for large pages.
*/
#define NV_GMMU_VA_RANGE 37ULL
#define NV_GMMU_VA_IS_UPPER(x) ((x) >= ((u64)SZ_1G * 16))
#ifdef CONFIG_ARM64
#define outer_flush_range(a, b)
#define __cpuc_flush_dcache_area __flush_dcache_area
#endif
#define FLUSH_CPU_DCACHE(va, pa, size) \
do { \
__cpuc_flush_dcache_area((void *)(va), (size_t)(size)); \
outer_flush_range(pa, pa + (size_t)(size)); \
} while (0)
struct mem_desc {
void *cpu_va;
struct page **pages;
struct sg_table *sgt;
size_t size;
u64 gpu_va;
};
struct mem_desc_sub {
u32 offset;
u32 size;
};
struct gpfifo_desc {
struct mem_desc mem;
u32 entry_num;
u32 get;
u32 put;
bool wrap;
};
struct patch_desc {
struct mem_desc mem;
u32 data_count;
};
struct zcull_ctx_desc {
u64 gpu_va;
u32 ctx_attr;
u32 ctx_sw_mode;
};
struct gk20a;
struct gr_ctx_buffer_desc {
void (*destroy)(struct gk20a *, struct gr_ctx_buffer_desc *);
struct mem_desc mem;
void *priv;
};
#ifdef CONFIG_ARCH_TEGRA_18x_SOC
#include "gr_t18x.h"
#endif
struct gr_ctx_desc {
struct mem_desc mem;
int preempt_mode;
#ifdef CONFIG_ARCH_TEGRA_18x_SOC
struct gr_ctx_desc_t18x t18x;
#endif
};
#define NVGPU_GR_PREEMPTION_MODE_WFI 0
#define NVGPU_GR_PREEMPTION_MODE_CTA 2
struct compbit_store_desc {
struct mem_desc mem;
/* The value that is written to the hardware. This depends on
* on the number of ltcs and is not an address. */
u64 base_hw;
};
struct gk20a_buffer_state {
struct list_head list;
/* The valid compbits and the fence must be changed atomically. */
struct mutex lock;
/* Offset of the surface within the dma-buf whose state is
* described by this struct (one dma-buf can contain multiple
* surfaces with different states). */
size_t offset;
/* A bitmask of valid sets of compbits (0 = uncompressed). */
u32 valid_compbits;
/* The ZBC color used on this buffer. */
u32 zbc_color;
/* This struct reflects the state of the buffer when this
* fence signals. */
struct gk20a_fence *fence;
};
enum gmmu_pgsz_gk20a {
gmmu_page_size_small = 0,
gmmu_page_size_big = 1,
gmmu_nr_page_sizes = 2
};
struct gk20a_comptags {
u32 offset;
u32 lines;
};
struct gk20a_mm_entry {
/* backing for */
void *cpu_va;
struct sg_table *sgt;
struct page **pages;
size_t size;
int pgsz;
struct gk20a_mm_entry *entries;
};
struct priv_cmd_queue {
struct mem_desc mem;
u32 size; /* num of entries in words */
u32 put; /* put for priv cmd queue */
u32 get; /* get for priv cmd queue */
struct list_head free; /* list of pre-allocated free entries */
struct list_head head; /* list of used entries */
};
struct priv_cmd_entry {
u32 *ptr;
u64 gva;
u32 get; /* start of entry in queue */
u32 size; /* in words */
u32 gp_get; /* gp_get when submitting last priv cmd */
u32 gp_put; /* gp_put when submitting last priv cmd */
u32 gp_wrap; /* wrap when submitting last priv cmd */
bool pre_alloc; /* prealloc entry, free to free list */
struct list_head list; /* node for lists */
};
struct mapped_buffer_node {
struct vm_gk20a *vm;
struct rb_node node;
struct list_head unmap_list;
struct list_head va_buffers_list;
struct vm_reserved_va_node *va_node;
u64 addr;
u64 size;
struct dma_buf *dmabuf;
struct sg_table *sgt;
struct kref ref;
u32 user_mapped;
bool own_mem_ref;
u32 pgsz_idx;
u32 ctag_offset;
u32 ctag_lines;
u32 flags;
u32 kind;
bool va_allocated;
};
struct vm_reserved_va_node {
struct list_head reserved_va_list;
struct list_head va_buffers_list;
u32 pgsz_idx;
u64 vaddr_start;
u64 size;
bool sparse;
};
struct gk20a_mmu_level {
int hi_bit[2];
int lo_bit[2];
int (*update_entry)(struct vm_gk20a *vm,
struct gk20a_mm_entry *pte,
u32 i, u32 gmmu_pgsz_idx,
u64 iova,
u32 kind_v, u32 *ctag,
bool cacheable, bool unmapped_pte,
int rw_flag, bool sparse, u32 flags);
size_t entry_size;
};
struct vm_gk20a {
struct mm_gk20a *mm;
struct gk20a_as_share *as_share; /* as_share this represents */
u64 va_start;
u64 va_limit;
int num_user_mapped_buffers;
bool big_pages; /* enable large page support */
bool enable_ctag;
bool mapped;
u32 big_page_size;
const struct gk20a_mmu_level *mmu_levels;
struct kref ref;
struct mutex update_gmmu_lock;
struct gk20a_mm_entry pdb;
struct gk20a_allocator vma[gmmu_nr_page_sizes];
struct rb_root mapped_buffers;
struct list_head reserved_va_list;
#ifdef CONFIG_TEGRA_GR_VIRTUALIZATION
u64 handle;
#endif
u32 gmmu_page_sizes[gmmu_nr_page_sizes];
};
struct gk20a;
struct channel_gk20a;
int gk20a_init_mm_support(struct gk20a *g);
int gk20a_init_mm_setup_sw(struct gk20a *g);
int gk20a_init_mm_setup_hw(struct gk20a *g);
int gk20a_mm_fb_flush(struct gk20a *g);
void gk20a_mm_l2_flush(struct gk20a *g, bool invalidate);
void gk20a_mm_l2_invalidate(struct gk20a *g);
struct mm_gk20a {
struct gk20a *g;
struct {
u64 size;
} channel;
struct {
u32 aperture_size;
struct vm_gk20a vm;
struct mem_desc inst_block;
} bar1;
struct {
u32 aperture_size;
struct vm_gk20a vm;
struct mem_desc inst_block;
} bar2;
struct {
u32 aperture_size;
struct vm_gk20a vm;
struct mem_desc inst_block;
} pmu;
struct {
/* using pmu vm currently */
struct mem_desc inst_block;
} hwpm;
struct mutex l2_op_lock;
#ifdef CONFIG_ARCH_TEGRA_18x_SOC
struct mem_desc bar2_desc;
#endif
void (*remove_support)(struct mm_gk20a *mm);
bool sw_ready;
int physical_bits;
#ifdef CONFIG_DEBUG_FS
u32 ltc_enabled;
u32 ltc_enabled_debug;
#endif
};
int gk20a_mm_init(struct mm_gk20a *mm);
#define gk20a_from_mm(mm) ((mm)->g)
#define gk20a_from_vm(vm) ((vm)->mm->g)
#define dev_from_vm(vm) dev_from_gk20a(vm->mm->g)
#define DEFAULT_ALLOC_ALIGNMENT (4*1024)
static inline int bar1_aperture_size_mb_gk20a(void)
{
return 128; /*TBD read this from fuses?*/
}
/* max address bits */
static inline int max_physaddr_bits_gk20a(void)
{
return 40;/*"old" sys physaddr, meaningful? */
}
static inline int max_vid_physaddr_bits_gk20a(void)
{
/* "vid phys" is asid/smmu phys?,
* i.e. is this the real sys physaddr? */
return 37;
}
static inline int max_vaddr_bits_gk20a(void)
{
return 40; /* chopped for area? */
}
#if 0 /*related to addr bits above, concern below TBD on which is accurate */
#define bar1_instance_block_shift_gk20a() (max_physaddr_bits_gk20a() -\
bus_bar1_block_ptr_s())
#else
#define bar1_instance_block_shift_gk20a() bus_bar1_block_ptr_shift_v()
#endif
int gk20a_alloc_inst_block(struct gk20a *g, struct mem_desc *inst_block);
void gk20a_free_inst_block(struct gk20a *g, struct mem_desc *inst_block);
void gk20a_init_inst_block(struct mem_desc *inst_block, struct vm_gk20a *vm,
u32 big_page_size);
void gk20a_mm_dump_vm(struct vm_gk20a *vm,
u64 va_begin, u64 va_end, char *label);
int gk20a_mm_suspend(struct gk20a *g);
phys_addr_t gk20a_get_phys_from_iova(struct device *d,
u64 dma_addr);
int gk20a_get_sgtable(struct device *d, struct sg_table **sgt,
void *cpuva, u64 iova,
size_t size);
int gk20a_get_sgtable_from_pages(struct device *d, struct sg_table **sgt,
struct page **pages, u64 iova,
size_t size);
void gk20a_free_sgtable(struct sg_table **sgt);
u64 gk20a_mm_iova_addr(struct gk20a *g, struct scatterlist *sgl,
u32 flags);
u64 gk20a_mm_smmu_vaddr_translate(struct gk20a *g, dma_addr_t iova);
void gk20a_mm_ltc_isr(struct gk20a *g);
bool gk20a_mm_mmu_debug_mode_enabled(struct gk20a *g);
int gk20a_mm_mmu_vpr_info_fetch(struct gk20a *g);
u64 gk20a_gmmu_map(struct vm_gk20a *vm,
struct sg_table **sgt,
u64 size,
u32 flags,
int rw_flag);
int gk20a_gmmu_alloc_map(struct vm_gk20a *vm,
size_t size,
struct mem_desc *mem);
int gk20a_gmmu_alloc_map_attr(struct vm_gk20a *vm,
enum dma_attr attr,
size_t size,
struct mem_desc *mem);
void gk20a_gmmu_unmap_free(struct vm_gk20a *vm,
struct mem_desc *mem);
int gk20a_gmmu_alloc(struct gk20a *g,
size_t size,
struct mem_desc *mem);
int gk20a_gmmu_alloc_attr(struct gk20a *g,
enum dma_attr attr,
size_t size,
struct mem_desc *mem);
void gk20a_gmmu_free(struct gk20a *g,
struct mem_desc *mem);
void gk20a_gmmu_free_attr(struct gk20a *g,
enum dma_attr attr,
struct mem_desc *mem);
static inline phys_addr_t gk20a_mem_phys(struct mem_desc *mem)
{
return sg_phys(mem->sgt->sgl);
}
u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
u64 map_offset,
struct sg_table *sgt,
u64 buffer_offset,
u64 size,
int pgsz_idx,
u8 kind_v,
u32 ctag_offset,
u32 flags,
int rw_flag,
bool clear_ctags,
bool sparse);
void gk20a_gmmu_unmap(struct vm_gk20a *vm,
u64 vaddr,
u64 size,
int rw_flag);
void gk20a_locked_gmmu_unmap(struct vm_gk20a *vm,
u64 vaddr,
u64 size,
int pgsz_idx,
bool va_allocated,
int rw_flag,
bool sparse);
struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf);
void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf,
struct sg_table *sgt);
u64 gk20a_vm_map(struct vm_gk20a *vm,
struct dma_buf *dmabuf,
u64 offset_align,
u32 flags /*NVGPU_AS_MAP_BUFFER_FLAGS_*/,
int kind,
struct sg_table **sgt,
bool user_mapped,
int rw_flag,
u64 buffer_offset,
u64 mapping_size);
/* unmap handle from kernel */
void gk20a_vm_unmap(struct vm_gk20a *vm, u64 offset);
void gk20a_vm_unmap_locked(struct mapped_buffer_node *mapped_buffer);
/* get reference to all currently mapped buffers */
int gk20a_vm_get_buffers(struct vm_gk20a *vm,
struct mapped_buffer_node ***mapped_buffers,
int *num_buffers);
/* put references on the given buffers */
void gk20a_vm_put_buffers(struct vm_gk20a *vm,
struct mapped_buffer_node **mapped_buffers,
int num_buffers);
/* invalidate tlbs for the vm area */
void gk20a_mm_tlb_invalidate(struct vm_gk20a *vm);
/* find buffer corresponding to va */
int gk20a_vm_find_buffer(struct vm_gk20a *vm, u64 gpu_va,
struct dma_buf **dmabuf,
u64 *offset);
void gk20a_vm_get(struct vm_gk20a *vm);
void gk20a_vm_put(struct vm_gk20a *vm);
void gk20a_vm_remove_support(struct vm_gk20a *vm);
u64 gk20a_vm_alloc_va(struct vm_gk20a *vm,
u64 size,
enum gmmu_pgsz_gk20a gmmu_pgsz_idx);
int gk20a_vm_free_va(struct vm_gk20a *vm,
u64 offset, u64 size,
enum gmmu_pgsz_gk20a pgsz_idx);
/* vm-as interface */
struct nvgpu_as_alloc_space_args;
struct nvgpu_as_free_space_args;
int gk20a_vm_alloc_share(struct gk20a_as_share *as_share, u32 flags);
int gk20a_vm_release_share(struct gk20a_as_share *as_share);
int gk20a_vm_alloc_space(struct gk20a_as_share *as_share,
struct nvgpu_as_alloc_space_args *args);
int gk20a_vm_free_space(struct gk20a_as_share *as_share,
struct nvgpu_as_free_space_args *args);
int gk20a_vm_bind_channel(struct gk20a_as_share *as_share,
struct channel_gk20a *ch);
int gk20a_vm_map_buffer(struct vm_gk20a *vm,
int dmabuf_fd,
u64 *offset_align,
u32 flags, /* NVGPU_AS_MAP_BUFFER_FLAGS_ */
int kind,
u64 buffer_offset,
u64 mapping_size);
int gk20a_init_vm(struct mm_gk20a *mm,
struct vm_gk20a *vm,
u32 big_page_size,
u64 low_hole,
u64 aperture_size,
bool big_pages,
char *name);
void gk20a_deinit_vm(struct vm_gk20a *vm);
int gk20a_vm_unmap_buffer(struct vm_gk20a *vm, u64 offset);
void gk20a_get_comptags(struct device *dev, struct dma_buf *dmabuf,
struct gk20a_comptags *comptags);
dma_addr_t gk20a_mm_gpuva_to_iova_base(struct vm_gk20a *vm, u64 gpu_vaddr);
int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev);
int gk20a_dmabuf_get_state(struct dma_buf *dmabuf, struct device *dev,
u64 offset, struct gk20a_buffer_state **state);
int map_gmmu_pages(struct gk20a_mm_entry *entry);
void unmap_gmmu_pages(struct gk20a_mm_entry *entry);
void pde_range_from_vaddr_range(struct vm_gk20a *vm,
u64 addr_lo, u64 addr_hi,
u32 *pde_lo, u32 *pde_hi);
int gk20a_mm_pde_coverage_bit_count(struct vm_gk20a *vm);
u32 *pde_from_index(struct vm_gk20a *vm, u32 i);
u32 pte_index_from_vaddr(struct vm_gk20a *vm,
u64 addr, enum gmmu_pgsz_gk20a pgsz_idx);
void free_gmmu_pages(struct vm_gk20a *vm,
struct gk20a_mm_entry *entry);
u32 gk20a_mm_get_physical_addr_bits(struct gk20a *g);
struct gpu_ops;
void gk20a_init_mm(struct gpu_ops *gops);
const struct gk20a_mmu_level *gk20a_mm_get_mmu_levels(struct gk20a *g,
u32 big_page_size);
void gk20a_mm_init_pdb(struct gk20a *g, void *inst_ptr, u64 pdb_addr);
extern const struct gk20a_mmu_level gk20a_mm_levels_64k[];
extern const struct gk20a_mmu_level gk20a_mm_levels_128k[];
#endif /* MM_GK20A_H */
|