summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/mm
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common/mm')
-rw-r--r--drivers/gpu/nvgpu/common/mm/bitmap_allocator.c438
-rw-r--r--drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h87
-rw-r--r--drivers/gpu/nvgpu/common/mm/buddy_allocator.c1323
-rw-r--r--drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h222
-rw-r--r--drivers/gpu/nvgpu/common/mm/comptags.c95
-rw-r--r--drivers/gpu/nvgpu/common/mm/gmmu.c920
-rw-r--r--drivers/gpu/nvgpu/common/mm/lockless_allocator.c225
-rw-r--r--drivers/gpu/nvgpu/common/mm/lockless_allocator_priv.h127
-rw-r--r--drivers/gpu/nvgpu/common/mm/mm.c450
-rw-r--r--drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c162
-rw-r--r--drivers/gpu/nvgpu/common/mm/nvgpu_mem.c119
-rw-r--r--drivers/gpu/nvgpu/common/mm/page_allocator.c1047
-rw-r--r--drivers/gpu/nvgpu/common/mm/pd_cache.c444
-rw-r--r--drivers/gpu/nvgpu/common/mm/vidmem.c554
-rw-r--r--drivers/gpu/nvgpu/common/mm/vm.c1145
-rw-r--r--drivers/gpu/nvgpu/common/mm/vm_area.c231
16 files changed, 7589 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c b/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c
new file mode 100644
index 00000000..6bd654b8
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c
@@ -0,0 +1,438 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/bitops.h>
24#include <nvgpu/allocator.h>
25#include <nvgpu/kmem.h>
26#include <nvgpu/bug.h>
27#include <nvgpu/barrier.h>
28
29#include "bitmap_allocator_priv.h"
30
31static u64 nvgpu_bitmap_alloc_length(struct nvgpu_allocator *a)
32{
33 struct nvgpu_bitmap_allocator *ba = a->priv;
34
35 return ba->length;
36}
37
38static u64 nvgpu_bitmap_alloc_base(struct nvgpu_allocator *a)
39{
40 struct nvgpu_bitmap_allocator *ba = a->priv;
41
42 return ba->base;
43}
44
45static int nvgpu_bitmap_alloc_inited(struct nvgpu_allocator *a)
46{
47 struct nvgpu_bitmap_allocator *ba = a->priv;
48 int inited = ba->inited;
49
50 nvgpu_smp_rmb();
51 return inited;
52}
53
54static u64 nvgpu_bitmap_alloc_end(struct nvgpu_allocator *a)
55{
56 struct nvgpu_bitmap_allocator *ba = a->priv;
57
58 return ba->base + ba->length;
59}
60
61/*
62 * @page_size is ignored.
63 */
64static u64 nvgpu_bitmap_alloc_fixed(struct nvgpu_allocator *__a,
65 u64 base, u64 len, u32 page_size)
66{
67 struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a);
68 u64 blks, offs, ret;
69
70 /* Compute the bit offset and make sure it's aligned to a block. */
71 offs = base >> a->blk_shift;
72 if (offs * a->blk_size != base)
73 return 0;
74
75 offs -= a->bit_offs;
76
77 blks = len >> a->blk_shift;
78 if (blks * a->blk_size != len)
79 blks++;
80
81 alloc_lock(__a);
82
83 /* Check if the space requested is already occupied. */
84 ret = bitmap_find_next_zero_area(a->bitmap, a->num_bits, offs, blks, 0);
85 if (ret != offs)
86 goto fail;
87
88 bitmap_set(a->bitmap, offs, blks);
89
90 a->bytes_alloced += blks * a->blk_size;
91 a->nr_fixed_allocs++;
92 alloc_unlock(__a);
93
94 alloc_dbg(__a, "Alloc-fixed 0x%-10llx 0x%-5llx [bits=0x%llx (%llu)]\n",
95 base, len, blks, blks);
96 return base;
97
98fail:
99 alloc_unlock(__a);
100 alloc_dbg(__a, "Alloc-fixed failed! (0x%llx)\n", base);
101 return 0;
102}
103
104/*
105 * Two possibilities for this function: either we are freeing a fixed allocation
106 * or we are freeing a regular alloc but with GPU_ALLOC_NO_ALLOC_PAGE defined.
107 *
108 * Note: this function won't do much error checking. Thus you could really
109 * confuse the allocator if you misuse this function.
110 */
111static void nvgpu_bitmap_free_fixed(struct nvgpu_allocator *__a,
112 u64 base, u64 len)
113{
114 struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a);
115 u64 blks, offs;
116
117 offs = base >> a->blk_shift;
118 if (WARN_ON(offs * a->blk_size != base))
119 return;
120
121 offs -= a->bit_offs;
122
123 blks = len >> a->blk_shift;
124 if (blks * a->blk_size != len)
125 blks++;
126
127 alloc_lock(__a);
128 bitmap_clear(a->bitmap, offs, blks);
129 a->bytes_freed += blks * a->blk_size;
130 alloc_unlock(__a);
131
132 alloc_dbg(__a, "Free-fixed 0x%-10llx 0x%-5llx [bits=0x%llx (%llu)]\n",
133 base, len, blks, blks);
134}
135
136/*
137 * Add the passed alloc to the tree of stored allocations.
138 */
139static void insert_alloc_metadata(struct nvgpu_bitmap_allocator *a,
140 struct nvgpu_bitmap_alloc *alloc)
141{
142 alloc->alloc_entry.key_start = alloc->base;
143 alloc->alloc_entry.key_end = alloc->base + alloc->length;
144
145 nvgpu_rbtree_insert(&alloc->alloc_entry, &a->allocs);
146}
147
148/*
149 * Find and remove meta-data from the outstanding allocations.
150 */
151static struct nvgpu_bitmap_alloc *find_alloc_metadata(
152 struct nvgpu_bitmap_allocator *a, u64 addr)
153{
154 struct nvgpu_bitmap_alloc *alloc;
155 struct nvgpu_rbtree_node *node = NULL;
156
157 nvgpu_rbtree_search(addr, &node, a->allocs);
158 if (!node)
159 return NULL;
160
161 alloc = nvgpu_bitmap_alloc_from_rbtree_node(node);
162
163 nvgpu_rbtree_unlink(node, &a->allocs);
164
165 return alloc;
166}
167
168/*
169 * Tree of alloc meta data stores the address of the alloc not the bit offset.
170 */
171static int __nvgpu_bitmap_store_alloc(struct nvgpu_bitmap_allocator *a,
172 u64 addr, u64 len)
173{
174 struct nvgpu_bitmap_alloc *alloc =
175 nvgpu_kmem_cache_alloc(a->meta_data_cache);
176
177 if (!alloc)
178 return -ENOMEM;
179
180 alloc->base = addr;
181 alloc->length = len;
182
183 insert_alloc_metadata(a, alloc);
184
185 return 0;
186}
187
188/*
189 * @len is in bytes. This routine will figure out the right number of bits to
190 * actually allocate. The return is the address in bytes as well.
191 */
192static u64 nvgpu_bitmap_alloc(struct nvgpu_allocator *__a, u64 len)
193{
194 u64 blks, addr;
195 unsigned long offs, adjusted_offs, limit;
196 struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a);
197
198 blks = len >> a->blk_shift;
199
200 if (blks * a->blk_size != len)
201 blks++;
202
203 alloc_lock(__a);
204
205 /*
206 * First look from next_blk and onwards...
207 */
208 offs = bitmap_find_next_zero_area(a->bitmap, a->num_bits,
209 a->next_blk, blks, 0);
210 if (offs >= a->num_bits) {
211 /*
212 * If that didn't work try the remaining area. Since there can
213 * be available space that spans across a->next_blk we need to
214 * search up to the first set bit after that.
215 */
216 limit = find_next_bit(a->bitmap, a->num_bits, a->next_blk);
217 offs = bitmap_find_next_zero_area(a->bitmap, limit,
218 0, blks, 0);
219 if (offs >= a->next_blk)
220 goto fail;
221 }
222
223 bitmap_set(a->bitmap, offs, blks);
224 a->next_blk = offs + blks;
225
226 adjusted_offs = offs + a->bit_offs;
227 addr = ((u64)adjusted_offs) * a->blk_size;
228
229 /*
230 * Only do meta-data storage if we are allowed to allocate storage for
231 * that meta-data. The issue with using malloc and friends is that
232 * in latency and success critical paths an alloc_page() call can either
233 * sleep for potentially a long time or fail. Since we might not want
234 * either of these possibilities assume that the caller will keep what
235 * data it needs around to successfully free this allocation.
236 */
237 if (!(a->flags & GPU_ALLOC_NO_ALLOC_PAGE) &&
238 __nvgpu_bitmap_store_alloc(a, addr, blks * a->blk_size))
239 goto fail_reset_bitmap;
240
241 alloc_dbg(__a, "Alloc 0x%-10llx 0x%-5llx [bits=0x%llx (%llu)]\n",
242 addr, len, blks, blks);
243
244 a->nr_allocs++;
245 a->bytes_alloced += (blks * a->blk_size);
246 alloc_unlock(__a);
247
248 return addr;
249
250fail_reset_bitmap:
251 bitmap_clear(a->bitmap, offs, blks);
252fail:
253 a->next_blk = 0;
254 alloc_unlock(__a);
255 alloc_dbg(__a, "Alloc failed!\n");
256 return 0;
257}
258
259static void nvgpu_bitmap_free(struct nvgpu_allocator *__a, u64 addr)
260{
261 struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a);
262 struct nvgpu_bitmap_alloc *alloc = NULL;
263 u64 offs, adjusted_offs, blks;
264
265 alloc_lock(__a);
266
267 if (a->flags & GPU_ALLOC_NO_ALLOC_PAGE) {
268 WARN(1, "Using wrong free for NO_ALLOC_PAGE bitmap allocator");
269 goto done;
270 }
271
272 alloc = find_alloc_metadata(a, addr);
273 if (!alloc)
274 goto done;
275
276 /*
277 * Address comes from adjusted offset (i.e the bit offset with
278 * a->bit_offs added. So start with that and then work out the real
279 * offs into the bitmap.
280 */
281 adjusted_offs = addr >> a->blk_shift;
282 offs = adjusted_offs - a->bit_offs;
283 blks = alloc->length >> a->blk_shift;
284
285 bitmap_clear(a->bitmap, offs, blks);
286 alloc_dbg(__a, "Free 0x%-10llx\n", addr);
287
288 a->bytes_freed += alloc->length;
289
290done:
291 if (a->meta_data_cache && alloc)
292 nvgpu_kmem_cache_free(a->meta_data_cache, alloc);
293 alloc_unlock(__a);
294}
295
296static void nvgpu_bitmap_alloc_destroy(struct nvgpu_allocator *__a)
297{
298 struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a);
299 struct nvgpu_bitmap_alloc *alloc;
300 struct nvgpu_rbtree_node *node;
301
302 /*
303 * Kill any outstanding allocations.
304 */
305 nvgpu_rbtree_enum_start(0, &node, a->allocs);
306 while (node) {
307 alloc = nvgpu_bitmap_alloc_from_rbtree_node(node);
308
309 nvgpu_rbtree_unlink(node, &a->allocs);
310 nvgpu_kmem_cache_free(a->meta_data_cache, alloc);
311
312 nvgpu_rbtree_enum_start(0, &node, a->allocs);
313 }
314
315 nvgpu_kmem_cache_destroy(a->meta_data_cache);
316 nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a->bitmap);
317 nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a);
318}
319
320#ifdef __KERNEL__
321static void nvgpu_bitmap_print_stats(struct nvgpu_allocator *__a,
322 struct seq_file *s, int lock)
323{
324 struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a);
325
326 __alloc_pstat(s, __a, "Bitmap allocator params:\n");
327 __alloc_pstat(s, __a, " start = 0x%llx\n", a->base);
328 __alloc_pstat(s, __a, " end = 0x%llx\n", a->base + a->length);
329 __alloc_pstat(s, __a, " blks = 0x%llx\n", a->num_bits);
330
331 /* Actual stats. */
332 __alloc_pstat(s, __a, "Stats:\n");
333 __alloc_pstat(s, __a, " Number allocs = 0x%llx\n", a->nr_allocs);
334 __alloc_pstat(s, __a, " Number fixed = 0x%llx\n", a->nr_fixed_allocs);
335 __alloc_pstat(s, __a, " Bytes alloced = 0x%llx\n", a->bytes_alloced);
336 __alloc_pstat(s, __a, " Bytes freed = 0x%llx\n", a->bytes_freed);
337 __alloc_pstat(s, __a, " Outstanding = 0x%llx\n",
338 a->bytes_alloced - a->bytes_freed);
339}
340#endif
341
342static const struct nvgpu_allocator_ops bitmap_ops = {
343 .alloc = nvgpu_bitmap_alloc,
344 .free = nvgpu_bitmap_free,
345
346 .alloc_fixed = nvgpu_bitmap_alloc_fixed,
347 .free_fixed = nvgpu_bitmap_free_fixed,
348
349 .base = nvgpu_bitmap_alloc_base,
350 .length = nvgpu_bitmap_alloc_length,
351 .end = nvgpu_bitmap_alloc_end,
352 .inited = nvgpu_bitmap_alloc_inited,
353
354 .fini = nvgpu_bitmap_alloc_destroy,
355
356#ifdef __KERNEL__
357 .print_stats = nvgpu_bitmap_print_stats,
358#endif
359};
360
361
362int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
363 const char *name, u64 base, u64 length,
364 u64 blk_size, u64 flags)
365{
366 int err;
367 struct nvgpu_bitmap_allocator *a;
368
369 if (WARN_ON(blk_size & (blk_size - 1)))
370 return -EINVAL;
371
372 /*
373 * blk_size must be a power-of-2; base length also need to be aligned
374 * to blk_size.
375 */
376 if (blk_size & (blk_size - 1) ||
377 base & (blk_size - 1) || length & (blk_size - 1))
378 return -EINVAL;
379
380 if (base == 0) {
381 base = blk_size;
382 length -= blk_size;
383 }
384
385 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_bitmap_allocator));
386 if (!a)
387 return -ENOMEM;
388
389 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &bitmap_ops);
390 if (err)
391 goto fail;
392
393 if (!(flags & GPU_ALLOC_NO_ALLOC_PAGE)) {
394 a->meta_data_cache = nvgpu_kmem_cache_create(g,
395 sizeof(struct nvgpu_bitmap_alloc));
396 if (!a->meta_data_cache) {
397 err = -ENOMEM;
398 goto fail;
399 }
400 }
401
402 a->base = base;
403 a->length = length;
404 a->blk_size = blk_size;
405 a->blk_shift = __ffs(a->blk_size);
406 a->num_bits = length >> a->blk_shift;
407 a->bit_offs = a->base >> a->blk_shift;
408 a->flags = flags;
409 a->allocs = NULL;
410
411 a->bitmap = nvgpu_kcalloc(g, BITS_TO_LONGS(a->num_bits),
412 sizeof(*a->bitmap));
413 if (!a->bitmap) {
414 err = -ENOMEM;
415 goto fail;
416 }
417
418 nvgpu_smp_wmb();
419 a->inited = true;
420
421#ifdef CONFIG_DEBUG_FS
422 nvgpu_init_alloc_debug(g, __a);
423#endif
424 alloc_dbg(__a, "New allocator: type bitmap\n");
425 alloc_dbg(__a, " base 0x%llx\n", a->base);
426 alloc_dbg(__a, " bit_offs 0x%llx\n", a->bit_offs);
427 alloc_dbg(__a, " size 0x%llx\n", a->length);
428 alloc_dbg(__a, " blk_size 0x%llx\n", a->blk_size);
429 alloc_dbg(__a, " flags 0x%llx\n", a->flags);
430
431 return 0;
432
433fail:
434 if (a->meta_data_cache)
435 nvgpu_kmem_cache_destroy(a->meta_data_cache);
436 nvgpu_kfree(g, a);
437 return err;
438}
diff --git a/drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h
new file mode 100644
index 00000000..1750447d
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h
@@ -0,0 +1,87 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef BITMAP_ALLOCATOR_PRIV_H
24#define BITMAP_ALLOCATOR_PRIV_H
25
26
27#include <nvgpu/rbtree.h>
28#include <nvgpu/kmem.h>
29
30struct nvgpu_allocator;
31
32struct nvgpu_bitmap_allocator {
33 struct nvgpu_allocator *owner;
34
35 u64 base; /* Base address of the space. */
36 u64 length; /* Length of the space. */
37 u64 blk_size; /* Size that corresponds to 1 bit. */
38 u64 blk_shift; /* Bit shift to divide by blk_size. */
39 u64 num_bits; /* Number of allocatable bits. */
40 u64 bit_offs; /* Offset of bitmap. */
41
42 /*
43 * Optimization for making repeated allocations faster. Keep track of
44 * the next bit after the most recent allocation. This is where the next
45 * search will start from. This should make allocation faster in cases
46 * where lots of allocations get made one after another. It shouldn't
47 * have a negative impact on the case where the allocator is fragmented.
48 */
49 u64 next_blk;
50
51 unsigned long *bitmap; /* The actual bitmap! */
52 struct nvgpu_rbtree_node *allocs; /* Tree of outstanding allocations */
53
54 struct nvgpu_kmem_cache *meta_data_cache;
55
56 u64 flags;
57
58 bool inited;
59
60 /* Statistics */
61 u64 nr_allocs;
62 u64 nr_fixed_allocs;
63 u64 bytes_alloced;
64 u64 bytes_freed;
65};
66
67struct nvgpu_bitmap_alloc {
68 u64 base;
69 u64 length;
70 struct nvgpu_rbtree_node alloc_entry; /* RB tree of allocations. */
71};
72
73static inline struct nvgpu_bitmap_alloc *
74nvgpu_bitmap_alloc_from_rbtree_node(struct nvgpu_rbtree_node *node)
75{
76 return (struct nvgpu_bitmap_alloc *)
77 ((uintptr_t)node - offsetof(struct nvgpu_bitmap_alloc, alloc_entry));
78};
79
80static inline struct nvgpu_bitmap_allocator *bitmap_allocator(
81 struct nvgpu_allocator *a)
82{
83 return (struct nvgpu_bitmap_allocator *)(a)->priv;
84}
85
86
87#endif
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
new file mode 100644
index 00000000..a2546e9d
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
@@ -0,0 +1,1323 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/allocator.h>
24#include <nvgpu/kmem.h>
25#include <nvgpu/bug.h>
26#include <nvgpu/log2.h>
27#include <nvgpu/barrier.h>
28#include <nvgpu/mm.h>
29#include <nvgpu/vm.h>
30
31#include "buddy_allocator_priv.h"
32
33/* Some other buddy allocator functions. */
34static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a,
35 u64 addr);
36static void balloc_coalesce(struct nvgpu_buddy_allocator *a,
37 struct nvgpu_buddy *b);
38static void __balloc_do_free_fixed(struct nvgpu_buddy_allocator *a,
39 struct nvgpu_fixed_alloc *falloc);
40
41/*
42 * This function is not present in older kernel's list.h code.
43 */
44#ifndef list_last_entry
45#define list_last_entry(ptr, type, member) \
46 list_entry((ptr)->prev, type, member)
47#endif
48
49/*
50 * GPU buddy allocator for various address spaces.
51 *
52 * Current limitations:
53 * o A fixed allocation could potentially be made that borders PDEs with
54 * different PTE sizes. This would require that fixed buffer to have
55 * different sized PTEs for different parts of the allocation. Probably
56 * best to just require PDE alignment for fixed address allocs.
57 *
58 * o It is currently possible to make an allocator that has a buddy alignment
59 * out of sync with the PDE block size alignment. A simple example is a
60 * 32GB address space starting at byte 1. Every buddy is shifted off by 1
61 * which means each buddy corresponf to more than one actual GPU page. The
62 * best way to fix this is probably just require PDE blocksize alignment
63 * for the start of the address space. At the moment all allocators are
64 * easily PDE aligned so this hasn't been a problem.
65 */
66
67/*
68 * Pick a suitable maximum order for this allocator.
69 *
70 * Hueristic: Just guessing that the best max order is the largest single
71 * block that will fit in the address space.
72 */
73static void balloc_compute_max_order(struct nvgpu_buddy_allocator *a)
74{
75 u64 true_max_order = ilog2(a->blks);
76
77 if (a->max_order == 0) {
78 a->max_order = true_max_order;
79 return;
80 }
81
82 if (a->max_order > true_max_order)
83 a->max_order = true_max_order;
84 if (a->max_order > GPU_BALLOC_MAX_ORDER)
85 a->max_order = GPU_BALLOC_MAX_ORDER;
86}
87
88/*
89 * Since we can only allocate in chucks of a->blk_size we need to trim off
90 * any excess data that is not aligned to a->blk_size.
91 */
92static void balloc_allocator_align(struct nvgpu_buddy_allocator *a)
93{
94 a->start = ALIGN(a->base, a->blk_size);
95 WARN_ON(a->start != a->base);
96 a->end = (a->base + a->length) & ~(a->blk_size - 1);
97 a->count = a->end - a->start;
98 a->blks = a->count >> a->blk_shift;
99}
100
101/*
102 * Pass NULL for parent if you want a top level buddy.
103 */
104static struct nvgpu_buddy *balloc_new_buddy(struct nvgpu_buddy_allocator *a,
105 struct nvgpu_buddy *parent,
106 u64 start, u64 order)
107{
108 struct nvgpu_buddy *new_buddy;
109
110 new_buddy = nvgpu_kmem_cache_alloc(a->buddy_cache);
111 if (!new_buddy)
112 return NULL;
113
114 memset(new_buddy, 0, sizeof(struct nvgpu_buddy));
115
116 new_buddy->parent = parent;
117 new_buddy->start = start;
118 new_buddy->order = order;
119 new_buddy->end = start + (1 << order) * a->blk_size;
120 new_buddy->pte_size = BALLOC_PTE_SIZE_ANY;
121
122 return new_buddy;
123}
124
125static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a,
126 struct nvgpu_buddy *b,
127 struct nvgpu_list_node *list)
128{
129 if (buddy_is_in_list(b)) {
130 alloc_dbg(balloc_owner(a),
131 "Oops: adding added buddy (%llu:0x%llx)\n",
132 b->order, b->start);
133 BUG();
134 }
135
136 /*
137 * Add big PTE blocks to the tail, small to the head for GVA spaces.
138 * This lets the code that checks if there are available blocks check
139 * without cycling through the entire list.
140 */
141 if (a->flags & GPU_ALLOC_GVA_SPACE &&
142 b->pte_size == gmmu_page_size_big)
143 nvgpu_list_add_tail(&b->buddy_entry, list);
144 else
145 nvgpu_list_add(&b->buddy_entry, list);
146
147 buddy_set_in_list(b);
148}
149
150static void __balloc_buddy_list_rem(struct nvgpu_buddy_allocator *a,
151 struct nvgpu_buddy *b)
152{
153 if (!buddy_is_in_list(b)) {
154 alloc_dbg(balloc_owner(a),
155 "Oops: removing removed buddy (%llu:0x%llx)\n",
156 b->order, b->start);
157 BUG();
158 }
159
160 nvgpu_list_del(&b->buddy_entry);
161 buddy_clr_in_list(b);
162}
163
164/*
165 * Add a buddy to one of the buddy lists and deal with the necessary
166 * book keeping. Adds the buddy to the list specified by the buddy's order.
167 */
168static void balloc_blist_add(struct nvgpu_buddy_allocator *a,
169 struct nvgpu_buddy *b)
170{
171 __balloc_buddy_list_add(a, b, balloc_get_order_list(a, b->order));
172 a->buddy_list_len[b->order]++;
173}
174
175static void balloc_blist_rem(struct nvgpu_buddy_allocator *a,
176 struct nvgpu_buddy *b)
177{
178 __balloc_buddy_list_rem(a, b);
179 a->buddy_list_len[b->order]--;
180}
181
182static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len)
183{
184 if (len == 0)
185 return 0;
186
187 len--;
188 len >>= a->blk_shift;
189
190 return fls(len);
191}
192
193static u64 __balloc_max_order_in(struct nvgpu_buddy_allocator *a,
194 u64 start, u64 end)
195{
196 u64 size = (end - start) >> a->blk_shift;
197
198 if (size > 0)
199 return min_t(u64, ilog2(size), a->max_order);
200 else
201 return GPU_BALLOC_MAX_ORDER;
202}
203
204/*
205 * Initialize the buddy lists.
206 */
207static int balloc_init_lists(struct nvgpu_buddy_allocator *a)
208{
209 int i;
210 u64 bstart, bend, order;
211 struct nvgpu_buddy *buddy;
212
213 bstart = a->start;
214 bend = a->end;
215
216 /* First make sure the LLs are valid. */
217 for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++)
218 nvgpu_init_list_node(balloc_get_order_list(a, i));
219
220 while (bstart < bend) {
221 order = __balloc_max_order_in(a, bstart, bend);
222
223 buddy = balloc_new_buddy(a, NULL, bstart, order);
224 if (!buddy)
225 goto cleanup;
226
227 balloc_blist_add(a, buddy);
228 bstart += balloc_order_to_len(a, order);
229 }
230
231 return 0;
232
233cleanup:
234 for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++) {
235 if (!nvgpu_list_empty(balloc_get_order_list(a, i))) {
236 buddy = nvgpu_list_first_entry(
237 balloc_get_order_list(a, i),
238 nvgpu_buddy, buddy_entry);
239 balloc_blist_rem(a, buddy);
240 nvgpu_kmem_cache_free(a->buddy_cache, buddy);
241 }
242 }
243
244 return -ENOMEM;
245}
246
247/*
248 * Clean up and destroy the passed allocator.
249 */
250static void nvgpu_buddy_allocator_destroy(struct nvgpu_allocator *__a)
251{
252 int i;
253 struct nvgpu_rbtree_node *node = NULL;
254 struct nvgpu_buddy *bud;
255 struct nvgpu_fixed_alloc *falloc;
256 struct nvgpu_buddy_allocator *a = __a->priv;
257
258 alloc_lock(__a);
259
260#ifdef CONFIG_DEBUG_FS
261 nvgpu_fini_alloc_debug(__a);
262#endif
263
264 /*
265 * Free the fixed allocs first.
266 */
267 nvgpu_rbtree_enum_start(0, &node, a->fixed_allocs);
268 while (node) {
269 falloc = nvgpu_fixed_alloc_from_rbtree_node(node);
270
271 nvgpu_rbtree_unlink(node, &a->fixed_allocs);
272 __balloc_do_free_fixed(a, falloc);
273
274 nvgpu_rbtree_enum_start(0, &node, a->fixed_allocs);
275 }
276
277 /*
278 * And now free all outstanding allocations.
279 */
280 nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies);
281 while (node) {
282 bud = nvgpu_buddy_from_rbtree_node(node);
283
284 balloc_free_buddy(a, bud->start);
285 balloc_blist_add(a, bud);
286 balloc_coalesce(a, bud);
287
288 nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies);
289 }
290
291 /*
292 * Now clean up the unallocated buddies.
293 */
294 for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++) {
295 BUG_ON(a->buddy_list_alloced[i] != 0);
296
297 while (!nvgpu_list_empty(balloc_get_order_list(a, i))) {
298 bud = nvgpu_list_first_entry(
299 balloc_get_order_list(a, i),
300 nvgpu_buddy, buddy_entry);
301 balloc_blist_rem(a, bud);
302 nvgpu_kmem_cache_free(a->buddy_cache, bud);
303 }
304
305 if (a->buddy_list_len[i] != 0) {
306 nvgpu_info(__a->g,
307 "Excess buddies!!! (%d: %llu)\n",
308 i, a->buddy_list_len[i]);
309 BUG();
310 }
311 if (a->buddy_list_split[i] != 0) {
312 nvgpu_info(__a->g,
313 "Excess split nodes!!! (%d: %llu)\n",
314 i, a->buddy_list_split[i]);
315 BUG();
316 }
317 if (a->buddy_list_alloced[i] != 0) {
318 nvgpu_info(__a->g,
319 "Excess alloced nodes!!! (%d: %llu)\n",
320 i, a->buddy_list_alloced[i]);
321 BUG();
322 }
323 }
324
325 nvgpu_kmem_cache_destroy(a->buddy_cache);
326 nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a);
327
328 alloc_unlock(__a);
329}
330
331/*
332 * Combine the passed buddy if possible. The pointer in @b may not be valid
333 * after this as the buddy may be freed.
334 *
335 * @a must be locked.
336 */
337static void balloc_coalesce(struct nvgpu_buddy_allocator *a,
338 struct nvgpu_buddy *b)
339{
340 struct nvgpu_buddy *parent;
341
342 if (buddy_is_alloced(b) || buddy_is_split(b))
343 return;
344
345 /*
346 * If both our buddy and I are both not allocated and not split then
347 * we can coalesce ourselves.
348 */
349 if (!b->buddy)
350 return;
351 if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy))
352 return;
353
354 parent = b->parent;
355
356 balloc_blist_rem(a, b);
357 balloc_blist_rem(a, b->buddy);
358
359 buddy_clr_split(parent);
360 a->buddy_list_split[parent->order]--;
361 balloc_blist_add(a, parent);
362
363 /*
364 * Recursively coalesce as far as we can go.
365 */
366 balloc_coalesce(a, parent);
367
368 /* Clean up the remains. */
369 nvgpu_kmem_cache_free(a->buddy_cache, b->buddy);
370 nvgpu_kmem_cache_free(a->buddy_cache, b);
371}
372
373/*
374 * Split a buddy into two new buddies who are 1/2 the size of the parent buddy.
375 *
376 * @a must be locked.
377 */
378static int balloc_split_buddy(struct nvgpu_buddy_allocator *a,
379 struct nvgpu_buddy *b, int pte_size)
380{
381 struct nvgpu_buddy *left, *right;
382 u64 half;
383
384 left = balloc_new_buddy(a, b, b->start, b->order - 1);
385 if (!left)
386 return -ENOMEM;
387
388 half = (b->end - b->start) / 2;
389
390 right = balloc_new_buddy(a, b, b->start + half, b->order - 1);
391 if (!right) {
392 nvgpu_kmem_cache_free(a->buddy_cache, left);
393 return -ENOMEM;
394 }
395
396 buddy_set_split(b);
397 a->buddy_list_split[b->order]++;
398
399 b->left = left;
400 b->right = right;
401 left->buddy = right;
402 right->buddy = left;
403 left->parent = b;
404 right->parent = b;
405
406 /* PTE considerations. */
407 if (a->flags & GPU_ALLOC_GVA_SPACE &&
408 left->order <= a->pte_blk_order) {
409 left->pte_size = pte_size;
410 right->pte_size = pte_size;
411 }
412
413 balloc_blist_rem(a, b);
414 balloc_blist_add(a, left);
415 balloc_blist_add(a, right);
416
417 return 0;
418}
419
420/*
421 * Place the passed buddy into the RB tree for allocated buddies. Never fails
422 * unless the passed entry is a duplicate which is a bug.
423 *
424 * @a must be locked.
425 */
426static void balloc_alloc_buddy(struct nvgpu_buddy_allocator *a,
427 struct nvgpu_buddy *b)
428{
429 b->alloced_entry.key_start = b->start;
430 b->alloced_entry.key_end = b->end;
431
432 nvgpu_rbtree_insert(&b->alloced_entry, &a->alloced_buddies);
433
434 buddy_set_alloced(b);
435 a->buddy_list_alloced[b->order]++;
436}
437
438/*
439 * Remove the passed buddy from the allocated buddy RB tree. Returns the
440 * deallocated buddy for further processing.
441 *
442 * @a must be locked.
443 */
444static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a,
445 u64 addr)
446{
447 struct nvgpu_rbtree_node *node = NULL;
448 struct nvgpu_buddy *bud;
449
450 nvgpu_rbtree_search(addr, &node, a->alloced_buddies);
451 if (!node)
452 return NULL;
453
454 bud = nvgpu_buddy_from_rbtree_node(node);
455
456 nvgpu_rbtree_unlink(node, &a->alloced_buddies);
457 buddy_clr_alloced(bud);
458 a->buddy_list_alloced[bud->order]--;
459
460 return bud;
461}
462
463/*
464 * Find a suitable buddy for the given order and PTE type (big or little).
465 */
466static struct nvgpu_buddy *__balloc_find_buddy(struct nvgpu_buddy_allocator *a,
467 u64 order, int pte_size)
468{
469 struct nvgpu_buddy *bud;
470
471 if (order > a->max_order ||
472 nvgpu_list_empty(balloc_get_order_list(a, order)))
473 return NULL;
474
475 if (a->flags & GPU_ALLOC_GVA_SPACE &&
476 pte_size == gmmu_page_size_big)
477 bud = nvgpu_list_last_entry(balloc_get_order_list(a, order),
478 nvgpu_buddy, buddy_entry);
479 else
480 bud = nvgpu_list_first_entry(balloc_get_order_list(a, order),
481 nvgpu_buddy, buddy_entry);
482
483 if (pte_size != BALLOC_PTE_SIZE_ANY &&
484 pte_size != bud->pte_size &&
485 bud->pte_size != BALLOC_PTE_SIZE_ANY)
486 return NULL;
487
488 return bud;
489}
490
491/*
492 * Allocate a suitably sized buddy. If no suitable buddy exists split higher
493 * order buddies until we have a suitable buddy to allocate.
494 *
495 * For PDE grouping add an extra check to see if a buddy is suitable: that the
496 * buddy exists in a PDE who's PTE size is reasonable
497 *
498 * @a must be locked.
499 */
500static u64 __balloc_do_alloc(struct nvgpu_buddy_allocator *a,
501 u64 order, int pte_size)
502{
503 u64 split_order;
504 struct nvgpu_buddy *bud = NULL;
505
506 split_order = order;
507 while (split_order <= a->max_order &&
508 !(bud = __balloc_find_buddy(a, split_order, pte_size)))
509 split_order++;
510
511 /* Out of memory! */
512 if (!bud)
513 return 0;
514
515 while (bud->order != order) {
516 if (balloc_split_buddy(a, bud, pte_size))
517 return 0; /* No mem... */
518 bud = bud->left;
519 }
520
521 balloc_blist_rem(a, bud);
522 balloc_alloc_buddy(a, bud);
523
524 return bud->start;
525}
526
527/*
528 * See if the passed range is actually available for allocation. If so, then
529 * return 1, otherwise return 0.
530 *
531 * TODO: Right now this uses the unoptimal approach of going through all
532 * outstanding allocations and checking their base/ends. This could be better.
533 */
534static int balloc_is_range_free(struct nvgpu_buddy_allocator *a,
535 u64 base, u64 end)
536{
537 struct nvgpu_rbtree_node *node = NULL;
538 struct nvgpu_buddy *bud;
539
540 nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies);
541 if (!node)
542 return 1; /* No allocs yet. */
543
544 bud = nvgpu_buddy_from_rbtree_node(node);
545
546 while (bud->start < end) {
547 if ((bud->start > base && bud->start < end) ||
548 (bud->end > base && bud->end < end))
549 return 0;
550
551 nvgpu_rbtree_enum_next(&node, node);
552 if (!node)
553 break;
554 bud = nvgpu_buddy_from_rbtree_node(node);
555 }
556
557 return 1;
558}
559
560static void balloc_alloc_fixed(struct nvgpu_buddy_allocator *a,
561 struct nvgpu_fixed_alloc *f)
562{
563 f->alloced_entry.key_start = f->start;
564 f->alloced_entry.key_end = f->end;
565
566 nvgpu_rbtree_insert(&f->alloced_entry, &a->fixed_allocs);
567}
568
569/*
570 * Remove the passed buddy from the allocated buddy RB tree. Returns the
571 * deallocated buddy for further processing.
572 *
573 * @a must be locked.
574 */
575static struct nvgpu_fixed_alloc *balloc_free_fixed(
576 struct nvgpu_buddy_allocator *a, u64 addr)
577{
578 struct nvgpu_fixed_alloc *falloc;
579 struct nvgpu_rbtree_node *node = NULL;
580
581 nvgpu_rbtree_search(addr, &node, a->fixed_allocs);
582 if (!node)
583 return NULL;
584
585 falloc = nvgpu_fixed_alloc_from_rbtree_node(node);
586
587 nvgpu_rbtree_unlink(node, &a->fixed_allocs);
588
589 return falloc;
590}
591
592/*
593 * Find the parent range - doesn't necessarily need the parent to actually exist
594 * as a buddy. Finding an existing parent comes later...
595 */
596static void __balloc_get_parent_range(struct nvgpu_buddy_allocator *a,
597 u64 base, u64 order,
598 u64 *pbase, u64 *porder)
599{
600 u64 base_mask;
601 u64 shifted_base = balloc_base_shift(a, base);
602
603 order++;
604 base_mask = ~((a->blk_size << order) - 1);
605
606 shifted_base &= base_mask;
607
608 *pbase = balloc_base_unshift(a, shifted_base);
609 *porder = order;
610}
611
612/*
613 * Makes a buddy at the passed address. This will make all parent buddies
614 * necessary for this buddy to exist as well.
615 */
616static struct nvgpu_buddy *__balloc_make_fixed_buddy(
617 struct nvgpu_buddy_allocator *a, u64 base, u64 order, int pte_size)
618{
619 struct nvgpu_buddy *bud = NULL;
620 struct nvgpu_list_node *order_list;
621 u64 cur_order = order, cur_base = base;
622
623 /*
624 * Algo:
625 * 1. Keep jumping up a buddy order until we find the real buddy that
626 * this buddy exists in.
627 * 2. Then work our way down through the buddy tree until we hit a dead
628 * end.
629 * 3. Start splitting buddies until we split to the one we need to
630 * make.
631 */
632 while (cur_order <= a->max_order) {
633 int found = 0;
634
635 order_list = balloc_get_order_list(a, cur_order);
636 nvgpu_list_for_each_entry(bud, order_list,
637 nvgpu_buddy, buddy_entry) {
638 if (bud->start == cur_base) {
639 /*
640 * Make sure page size matches if it's smaller
641 * than a PDE sized buddy.
642 */
643 if (bud->order <= a->pte_blk_order &&
644 bud->pte_size != BALLOC_PTE_SIZE_ANY &&
645 bud->pte_size != pte_size) {
646 /* Welp, that's the end of that. */
647 alloc_dbg(balloc_owner(a),
648 "Fixed buddy PTE "
649 "size mismatch!\n");
650 return NULL;
651 }
652
653 found = 1;
654 break;
655 }
656 }
657
658 if (found)
659 break;
660
661 __balloc_get_parent_range(a, cur_base, cur_order,
662 &cur_base, &cur_order);
663 }
664
665 if (cur_order > a->max_order) {
666 alloc_dbg(balloc_owner(a), "No buddy for range ???\n");
667 return NULL;
668 }
669
670 /* Split this buddy as necessary until we get the target buddy. */
671 while (bud->start != base || bud->order != order) {
672 if (balloc_split_buddy(a, bud, pte_size)) {
673 alloc_dbg(balloc_owner(a),
674 "split buddy failed? {0x%llx, %llu}\n",
675 bud->start, bud->order);
676 balloc_coalesce(a, bud);
677 return NULL;
678 }
679
680 if (base < bud->right->start)
681 bud = bud->left;
682 else
683 bud = bud->right;
684
685 }
686
687 return bud;
688}
689
690static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a,
691 struct nvgpu_fixed_alloc *falloc,
692 u64 base, u64 len, int pte_size)
693{
694 u64 shifted_base, inc_base;
695 u64 align_order;
696
697 shifted_base = balloc_base_shift(a, base);
698 if (shifted_base == 0)
699 align_order = __fls(len >> a->blk_shift);
700 else
701 align_order = min_t(u64,
702 __ffs(shifted_base >> a->blk_shift),
703 __fls(len >> a->blk_shift));
704
705 if (align_order > a->max_order) {
706 alloc_dbg(balloc_owner(a),
707 "Align order too big: %llu > %llu\n",
708 align_order, a->max_order);
709 return 0;
710 }
711
712 /*
713 * Generate a list of buddies that satisfy this allocation.
714 */
715 inc_base = shifted_base;
716 while (inc_base < (shifted_base + len)) {
717 u64 order_len = balloc_order_to_len(a, align_order);
718 u64 remaining;
719 struct nvgpu_buddy *bud;
720
721 bud = __balloc_make_fixed_buddy(a,
722 balloc_base_unshift(a, inc_base),
723 align_order, pte_size);
724 if (!bud) {
725 alloc_dbg(balloc_owner(a),
726 "Fixed buddy failed: {0x%llx, %llu}!\n",
727 balloc_base_unshift(a, inc_base),
728 align_order);
729 goto err_and_cleanup;
730 }
731
732 balloc_blist_rem(a, bud);
733 balloc_alloc_buddy(a, bud);
734 __balloc_buddy_list_add(a, bud, &falloc->buddies);
735
736 /* Book keeping. */
737 inc_base += order_len;
738 remaining = (shifted_base + len) - inc_base;
739 align_order = __ffs(inc_base >> a->blk_shift);
740
741 /* If we don't have much left - trim down align_order. */
742 if (balloc_order_to_len(a, align_order) > remaining)
743 align_order = __balloc_max_order_in(a, inc_base,
744 inc_base + remaining);
745 }
746
747 return base;
748
749err_and_cleanup:
750 while (!nvgpu_list_empty(&falloc->buddies)) {
751 struct nvgpu_buddy *bud = nvgpu_list_first_entry(
752 &falloc->buddies,
753 nvgpu_buddy, buddy_entry);
754
755 __balloc_buddy_list_rem(a, bud);
756 balloc_free_buddy(a, bud->start);
757 nvgpu_kmem_cache_free(a->buddy_cache, bud);
758 }
759
760 return 0;
761}
762
763static void __balloc_do_free_fixed(struct nvgpu_buddy_allocator *a,
764 struct nvgpu_fixed_alloc *falloc)
765{
766 struct nvgpu_buddy *bud;
767
768 while (!nvgpu_list_empty(&falloc->buddies)) {
769 bud = nvgpu_list_first_entry(&falloc->buddies,
770 nvgpu_buddy,
771 buddy_entry);
772 __balloc_buddy_list_rem(a, bud);
773
774 balloc_free_buddy(a, bud->start);
775 balloc_blist_add(a, bud);
776 a->bytes_freed += balloc_order_to_len(a, bud->order);
777
778 /*
779 * Attemp to defrag the allocation.
780 */
781 balloc_coalesce(a, bud);
782 }
783
784 nvgpu_kfree(nvgpu_alloc_to_gpu(a->owner), falloc);
785}
786
787/*
788 * Allocate memory from the passed allocator.
789 */
790static u64 nvgpu_buddy_balloc(struct nvgpu_allocator *__a, u64 len)
791{
792 u64 order, addr;
793 int pte_size;
794 struct nvgpu_buddy_allocator *a = __a->priv;
795
796 alloc_lock(__a);
797
798 order = balloc_get_order(a, len);
799
800 if (order > a->max_order) {
801 alloc_unlock(__a);
802 alloc_dbg(balloc_owner(a), "Alloc fail\n");
803 return 0;
804 }
805
806 if (a->flags & GPU_ALLOC_GVA_SPACE)
807 pte_size = __get_pte_size(a->vm, 0, len);
808 else
809 pte_size = BALLOC_PTE_SIZE_ANY;
810
811 addr = __balloc_do_alloc(a, order, pte_size);
812
813 if (addr) {
814 a->bytes_alloced += len;
815 a->bytes_alloced_real += balloc_order_to_len(a, order);
816 alloc_dbg(balloc_owner(a),
817 "Alloc 0x%-10llx %3lld:0x%-10llx pte_size=%s\n",
818 addr, order, len,
819 pte_size == gmmu_page_size_big ? "big" :
820 pte_size == gmmu_page_size_small ? "small" :
821 "NA/any");
822 } else {
823 alloc_dbg(balloc_owner(a), "Alloc failed: no mem!\n");
824 }
825
826 a->alloc_made = 1;
827
828 alloc_unlock(__a);
829
830 return addr;
831}
832
833/*
834 * Requires @__a to be locked.
835 */
836static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a,
837 u64 base, u64 len, u32 page_size)
838{
839 int pte_size = BALLOC_PTE_SIZE_ANY;
840 u64 ret, real_bytes = 0;
841 struct nvgpu_buddy *bud;
842 struct nvgpu_fixed_alloc *falloc = NULL;
843 struct nvgpu_buddy_allocator *a = __a->priv;
844
845 /* If base isn't aligned to an order 0 block, fail. */
846 if (base & (a->blk_size - 1))
847 goto fail;
848
849 if (len == 0)
850 goto fail;
851
852 /* Check that the page size is valid. */
853 if (a->flags & GPU_ALLOC_GVA_SPACE && a->vm->big_pages) {
854 if (page_size == a->vm->big_page_size)
855 pte_size = gmmu_page_size_big;
856 else if (page_size == SZ_4K)
857 pte_size = gmmu_page_size_small;
858 else
859 goto fail;
860 }
861
862 falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc));
863 if (!falloc)
864 goto fail;
865
866 nvgpu_init_list_node(&falloc->buddies);
867 falloc->start = base;
868 falloc->end = base + len;
869
870 if (!balloc_is_range_free(a, base, base + len)) {
871 alloc_dbg(balloc_owner(a),
872 "Range not free: 0x%llx -> 0x%llx\n",
873 base, base + len);
874 goto fail_unlock;
875 }
876
877 ret = __balloc_do_alloc_fixed(a, falloc, base, len, pte_size);
878 if (!ret) {
879 alloc_dbg(balloc_owner(a),
880 "Alloc-fixed failed ?? 0x%llx -> 0x%llx\n",
881 base, base + len);
882 goto fail_unlock;
883 }
884
885 balloc_alloc_fixed(a, falloc);
886
887 nvgpu_list_for_each_entry(bud, &falloc->buddies,
888 nvgpu_buddy, buddy_entry)
889 real_bytes += (bud->end - bud->start);
890
891 a->bytes_alloced += len;
892 a->bytes_alloced_real += real_bytes;
893
894 alloc_dbg(balloc_owner(a), "Alloc (fixed) 0x%llx\n", base);
895
896 return base;
897
898fail_unlock:
899 alloc_unlock(__a);
900fail:
901 nvgpu_kfree(nvgpu_alloc_to_gpu(__a), falloc);
902 return 0;
903}
904
905/*
906 * Allocate a fixed address allocation. The address of the allocation is @base
907 * and the length is @len. This is not a typical buddy allocator operation and
908 * as such has a high posibility of failure if the address space is heavily in
909 * use.
910 *
911 * Please do not use this function unless _absolutely_ necessary.
912 */
913static u64 nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a,
914 u64 base, u64 len, u32 page_size)
915{
916 u64 alloc;
917 struct nvgpu_buddy_allocator *a = __a->priv;
918
919 alloc_lock(__a);
920 alloc = __nvgpu_balloc_fixed_buddy(__a, base, len, page_size);
921 a->alloc_made = 1;
922 alloc_unlock(__a);
923
924 return alloc;
925}
926
927/*
928 * Free the passed allocation.
929 */
930static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr)
931{
932 struct nvgpu_buddy *bud;
933 struct nvgpu_fixed_alloc *falloc;
934 struct nvgpu_buddy_allocator *a = __a->priv;
935
936 if (!addr)
937 return;
938
939 alloc_lock(__a);
940
941 /*
942 * First see if this is a fixed alloc. If not fall back to a regular
943 * buddy.
944 */
945 falloc = balloc_free_fixed(a, addr);
946 if (falloc) {
947 __balloc_do_free_fixed(a, falloc);
948 goto done;
949 }
950
951 bud = balloc_free_buddy(a, addr);
952 if (!bud)
953 goto done;
954
955 balloc_blist_add(a, bud);
956 a->bytes_freed += balloc_order_to_len(a, bud->order);
957
958 /*
959 * Attemp to defrag the allocation.
960 */
961 balloc_coalesce(a, bud);
962
963done:
964 alloc_unlock(__a);
965 alloc_dbg(balloc_owner(a), "Free 0x%llx\n", addr);
966 return;
967}
968
969static bool nvgpu_buddy_reserve_is_possible(struct nvgpu_buddy_allocator *a,
970 struct nvgpu_alloc_carveout *co)
971{
972 struct nvgpu_alloc_carveout *tmp;
973 u64 co_base, co_end;
974
975 co_base = co->base;
976 co_end = co->base + co->length;
977
978 /*
979 * Not the fastest approach but we should not have that many carveouts
980 * for any reasonable allocator.
981 */
982 nvgpu_list_for_each_entry(tmp, &a->co_list,
983 nvgpu_alloc_carveout, co_entry) {
984 if ((co_base >= tmp->base &&
985 co_base < (tmp->base + tmp->length)) ||
986 (co_end >= tmp->base &&
987 co_end < (tmp->base + tmp->length)))
988 return false;
989 }
990
991 return true;
992}
993
994/*
995 * Carveouts can only be reserved before any regular allocations have been
996 * made.
997 */
998static int nvgpu_buddy_reserve_co(struct nvgpu_allocator *__a,
999 struct nvgpu_alloc_carveout *co)
1000{
1001 struct nvgpu_buddy_allocator *a = __a->priv;
1002 u64 addr;
1003 int err = 0;
1004
1005 if (co->base < a->start || (co->base + co->length) > a->end ||
1006 a->alloc_made)
1007 return -EINVAL;
1008
1009 alloc_lock(__a);
1010
1011 if (!nvgpu_buddy_reserve_is_possible(a, co)) {
1012 err = -EBUSY;
1013 goto done;
1014 }
1015
1016 /* Should not be possible to fail... */
1017 addr = __nvgpu_balloc_fixed_buddy(__a, co->base, co->length, 0);
1018 if (!addr) {
1019 err = -ENOMEM;
1020 nvgpu_warn(__a->g,
1021 "%s: Failed to reserve a valid carveout!\n",
1022 __func__);
1023 goto done;
1024 }
1025
1026 nvgpu_list_add(&co->co_entry, &a->co_list);
1027
1028done:
1029 alloc_unlock(__a);
1030 return err;
1031}
1032
1033/*
1034 * Carveouts can be release at any time.
1035 */
1036static void nvgpu_buddy_release_co(struct nvgpu_allocator *__a,
1037 struct nvgpu_alloc_carveout *co)
1038{
1039 alloc_lock(__a);
1040
1041 nvgpu_list_del(&co->co_entry);
1042 nvgpu_free(__a, co->base);
1043
1044 alloc_unlock(__a);
1045}
1046
1047static u64 nvgpu_buddy_alloc_length(struct nvgpu_allocator *a)
1048{
1049 struct nvgpu_buddy_allocator *ba = a->priv;
1050
1051 return ba->length;
1052}
1053
1054static u64 nvgpu_buddy_alloc_base(struct nvgpu_allocator *a)
1055{
1056 struct nvgpu_buddy_allocator *ba = a->priv;
1057
1058 return ba->start;
1059}
1060
1061static int nvgpu_buddy_alloc_inited(struct nvgpu_allocator *a)
1062{
1063 struct nvgpu_buddy_allocator *ba = a->priv;
1064 int inited = ba->initialized;
1065
1066 nvgpu_smp_rmb();
1067 return inited;
1068}
1069
1070static u64 nvgpu_buddy_alloc_end(struct nvgpu_allocator *a)
1071{
1072 struct nvgpu_buddy_allocator *ba = a->priv;
1073
1074 return ba->end;
1075}
1076
1077static u64 nvgpu_buddy_alloc_space(struct nvgpu_allocator *a)
1078{
1079 struct nvgpu_buddy_allocator *ba = a->priv;
1080 u64 space;
1081
1082 alloc_lock(a);
1083 space = ba->end - ba->start -
1084 (ba->bytes_alloced_real - ba->bytes_freed);
1085 alloc_unlock(a);
1086
1087 return space;
1088}
1089
1090#ifdef __KERNEL__
1091/*
1092 * Print the buddy allocator top level stats. If you pass @s as NULL then the
1093 * stats are printed to the kernel log. This lets this code be used for
1094 * debugging purposes internal to the allocator.
1095 */
1096static void nvgpu_buddy_print_stats(struct nvgpu_allocator *__a,
1097 struct seq_file *s, int lock)
1098{
1099 int i = 0;
1100 struct nvgpu_rbtree_node *node = NULL;
1101 struct nvgpu_fixed_alloc *falloc;
1102 struct nvgpu_alloc_carveout *tmp;
1103 struct nvgpu_buddy_allocator *a = __a->priv;
1104
1105 __alloc_pstat(s, __a, "base = %llu, limit = %llu, blk_size = %llu\n",
1106 a->base, a->length, a->blk_size);
1107 __alloc_pstat(s, __a, "Internal params:\n");
1108 __alloc_pstat(s, __a, " start = 0x%llx\n", a->start);
1109 __alloc_pstat(s, __a, " end = 0x%llx\n", a->end);
1110 __alloc_pstat(s, __a, " count = 0x%llx\n", a->count);
1111 __alloc_pstat(s, __a, " blks = 0x%llx\n", a->blks);
1112 __alloc_pstat(s, __a, " max_order = %llu\n", a->max_order);
1113
1114 if (lock)
1115 alloc_lock(__a);
1116
1117 if (!nvgpu_list_empty(&a->co_list)) {
1118 __alloc_pstat(s, __a, "\n");
1119 __alloc_pstat(s, __a, "Carveouts:\n");
1120 nvgpu_list_for_each_entry(tmp, &a->co_list,
1121 nvgpu_alloc_carveout, co_entry)
1122 __alloc_pstat(s, __a,
1123 " CO %2d: %-20s 0x%010llx + 0x%llx\n",
1124 i++, tmp->name, tmp->base, tmp->length);
1125 }
1126
1127 __alloc_pstat(s, __a, "\n");
1128 __alloc_pstat(s, __a, "Buddy blocks:\n");
1129 __alloc_pstat(s, __a, " Order Free Alloced Split\n");
1130 __alloc_pstat(s, __a, " ----- ---- ------- -----\n");
1131
1132 for (i = a->max_order; i >= 0; i--) {
1133 if (a->buddy_list_len[i] == 0 &&
1134 a->buddy_list_alloced[i] == 0 &&
1135 a->buddy_list_split[i] == 0)
1136 continue;
1137
1138 __alloc_pstat(s, __a, " %3d %-7llu %-9llu %llu\n", i,
1139 a->buddy_list_len[i],
1140 a->buddy_list_alloced[i],
1141 a->buddy_list_split[i]);
1142 }
1143
1144 __alloc_pstat(s, __a, "\n");
1145
1146 nvgpu_rbtree_enum_start(0, &node, a->fixed_allocs);
1147 i = 1;
1148 while (node) {
1149 falloc = nvgpu_fixed_alloc_from_rbtree_node(node);
1150
1151 __alloc_pstat(s, __a, "Fixed alloc (%d): [0x%llx -> 0x%llx]\n",
1152 i, falloc->start, falloc->end);
1153
1154 nvgpu_rbtree_enum_next(&node, a->fixed_allocs);
1155 }
1156
1157 __alloc_pstat(s, __a, "\n");
1158 __alloc_pstat(s, __a, "Bytes allocated: %llu\n",
1159 a->bytes_alloced);
1160 __alloc_pstat(s, __a, "Bytes allocated (real): %llu\n",
1161 a->bytes_alloced_real);
1162 __alloc_pstat(s, __a, "Bytes freed: %llu\n",
1163 a->bytes_freed);
1164
1165 if (lock)
1166 alloc_unlock(__a);
1167}
1168#endif
1169
1170static const struct nvgpu_allocator_ops buddy_ops = {
1171 .alloc = nvgpu_buddy_balloc,
1172 .free = nvgpu_buddy_bfree,
1173
1174 .alloc_fixed = nvgpu_balloc_fixed_buddy,
1175 /* .free_fixed not needed. */
1176
1177 .reserve_carveout = nvgpu_buddy_reserve_co,
1178 .release_carveout = nvgpu_buddy_release_co,
1179
1180 .base = nvgpu_buddy_alloc_base,
1181 .length = nvgpu_buddy_alloc_length,
1182 .end = nvgpu_buddy_alloc_end,
1183 .inited = nvgpu_buddy_alloc_inited,
1184 .space = nvgpu_buddy_alloc_space,
1185
1186 .fini = nvgpu_buddy_allocator_destroy,
1187
1188#ifdef __KERNEL__
1189 .print_stats = nvgpu_buddy_print_stats,
1190#endif
1191};
1192
1193/*
1194 * Initialize a buddy allocator. Returns 0 on success. This allocator does
1195 * not necessarily manage bytes. It manages distinct ranges of resources. This
1196 * allows the allocator to work for things like comp_tags, semaphores, etc.
1197 *
1198 * @allocator: Ptr to an allocator struct to init.
1199 * @vm: GPU VM to associate this allocator with. Can be NULL. Will be used to
1200 * get PTE size for GVA spaces.
1201 * @name: Name of the allocator. Doesn't have to be static storage.
1202 * @base: The base address of the resource pool being managed.
1203 * @size: Number of resources in the pool.
1204 * @blk_size: Minimum number of resources to allocate at once. For things like
1205 * semaphores this is 1. For GVA this might be as much as 64k. This
1206 * corresponds to order 0. Must be power of 2.
1207 * @max_order: Pick a maximum order. If you leave this as 0, the buddy allocator
1208 * will try and pick a reasonable max order.
1209 * @flags: Extra flags necessary. See GPU_BALLOC_*.
1210 */
1211int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
1212 struct vm_gk20a *vm, const char *name,
1213 u64 base, u64 size, u64 blk_size,
1214 u64 max_order, u64 flags)
1215{
1216 int err;
1217 u64 pde_size;
1218 struct nvgpu_buddy_allocator *a;
1219
1220 /* blk_size must be greater than 0 and a power of 2. */
1221 if (blk_size == 0)
1222 return -EINVAL;
1223 if (blk_size & (blk_size - 1))
1224 return -EINVAL;
1225
1226 if (max_order > GPU_BALLOC_MAX_ORDER)
1227 return -EINVAL;
1228
1229 /* If this is to manage a GVA space we need a VM. */
1230 if (flags & GPU_ALLOC_GVA_SPACE && !vm)
1231 return -EINVAL;
1232
1233 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator));
1234 if (!a)
1235 return -ENOMEM;
1236
1237 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops);
1238 if (err)
1239 goto fail;
1240
1241 a->base = base;
1242 a->length = size;
1243 a->blk_size = blk_size;
1244 a->blk_shift = __ffs(blk_size);
1245 a->owner = __a;
1246
1247 /*
1248 * If base is 0 then modfy base to be the size of one block so that we
1249 * can return errors by returning addr == 0.
1250 */
1251 if (a->base == 0) {
1252 a->base = a->blk_size;
1253 a->length -= a->blk_size;
1254 }
1255
1256 a->vm = vm;
1257 if (flags & GPU_ALLOC_GVA_SPACE) {
1258 pde_size = ((u64)vm->big_page_size) << 10;
1259 a->pte_blk_order = balloc_get_order(a, pde_size);
1260 }
1261
1262 /*
1263 * When we have a GVA space with big_pages enabled the size and base
1264 * must be PDE aligned. If big_pages are not enabled then this
1265 * requirement is not necessary.
1266 */
1267 if (flags & GPU_ALLOC_GVA_SPACE && vm->big_pages &&
1268 (base & ((vm->big_page_size << 10) - 1) ||
1269 size & ((vm->big_page_size << 10) - 1)))
1270 return -EINVAL;
1271
1272 a->flags = flags;
1273 a->max_order = max_order;
1274
1275 balloc_allocator_align(a);
1276 balloc_compute_max_order(a);
1277
1278 a->buddy_cache = nvgpu_kmem_cache_create(g, sizeof(struct nvgpu_buddy));
1279 if (!a->buddy_cache) {
1280 err = -ENOMEM;
1281 goto fail;
1282 }
1283
1284 a->alloced_buddies = NULL;
1285 a->fixed_allocs = NULL;
1286 nvgpu_init_list_node(&a->co_list);
1287 err = balloc_init_lists(a);
1288 if (err)
1289 goto fail;
1290
1291 nvgpu_smp_wmb();
1292 a->initialized = 1;
1293
1294#ifdef CONFIG_DEBUG_FS
1295 nvgpu_init_alloc_debug(g, __a);
1296#endif
1297 alloc_dbg(__a, "New allocator: type buddy\n");
1298 alloc_dbg(__a, " base 0x%llx\n", a->base);
1299 alloc_dbg(__a, " size 0x%llx\n", a->length);
1300 alloc_dbg(__a, " blk_size 0x%llx\n", a->blk_size);
1301 if (flags & GPU_ALLOC_GVA_SPACE)
1302 alloc_dbg(balloc_owner(a),
1303 " pde_size 0x%llx\n",
1304 balloc_order_to_len(a, a->pte_blk_order));
1305 alloc_dbg(__a, " max_order %llu\n", a->max_order);
1306 alloc_dbg(__a, " flags 0x%llx\n", a->flags);
1307
1308 return 0;
1309
1310fail:
1311 if (a->buddy_cache)
1312 nvgpu_kmem_cache_destroy(a->buddy_cache);
1313 nvgpu_kfree(g, a);
1314 return err;
1315}
1316
1317int nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *a,
1318 const char *name, u64 base, u64 size,
1319 u64 blk_size, u64 flags)
1320{
1321 return __nvgpu_buddy_allocator_init(g, a, NULL, name,
1322 base, size, blk_size, 0, 0);
1323}
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h
new file mode 100644
index 00000000..c9e332a5
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h
@@ -0,0 +1,222 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef BUDDY_ALLOCATOR_PRIV_H
24#define BUDDY_ALLOCATOR_PRIV_H
25
26#include <nvgpu/rbtree.h>
27#include <nvgpu/list.h>
28
29struct nvgpu_kmem_cache;
30struct nvgpu_allocator;
31struct vm_gk20a;
32
33/*
34 * Each buddy is an element in a binary tree.
35 */
36struct nvgpu_buddy {
37 struct nvgpu_buddy *parent; /* Parent node. */
38 struct nvgpu_buddy *buddy; /* This node's buddy. */
39 struct nvgpu_buddy *left; /* Lower address sub-node. */
40 struct nvgpu_buddy *right; /* Higher address sub-node. */
41
42 struct nvgpu_list_node buddy_entry; /* List entry for various lists. */
43 struct nvgpu_rbtree_node alloced_entry; /* RB tree of allocations. */
44
45 u64 start; /* Start address of this buddy. */
46 u64 end; /* End address of this buddy. */
47 u64 order; /* Buddy order. */
48
49#define BALLOC_BUDDY_ALLOCED 0x1
50#define BALLOC_BUDDY_SPLIT 0x2
51#define BALLOC_BUDDY_IN_LIST 0x4
52 int flags; /* List of associated flags. */
53
54 /*
55 * Size of the PDE this buddy is using. This allows for grouping like
56 * sized allocations into the same PDE. This uses the gmmu_pgsz_gk20a
57 * enum except for the BALLOC_PTE_SIZE_ANY specifier.
58 */
59#define BALLOC_PTE_SIZE_ANY -1
60 int pte_size;
61};
62
63static inline struct nvgpu_buddy *
64nvgpu_buddy_from_buddy_entry(struct nvgpu_list_node *node)
65{
66 return (struct nvgpu_buddy *)
67 ((uintptr_t)node - offsetof(struct nvgpu_buddy, buddy_entry));
68};
69
70static inline struct nvgpu_buddy *
71nvgpu_buddy_from_rbtree_node(struct nvgpu_rbtree_node *node)
72{
73 return (struct nvgpu_buddy *)
74 ((uintptr_t)node - offsetof(struct nvgpu_buddy, alloced_entry));
75};
76
77#define __buddy_flag_ops(flag, flag_up) \
78 static inline int buddy_is_ ## flag(struct nvgpu_buddy *b) \
79 { \
80 return b->flags & BALLOC_BUDDY_ ## flag_up; \
81 } \
82 static inline void buddy_set_ ## flag(struct nvgpu_buddy *b) \
83 { \
84 b->flags |= BALLOC_BUDDY_ ## flag_up; \
85 } \
86 static inline void buddy_clr_ ## flag(struct nvgpu_buddy *b) \
87 { \
88 b->flags &= ~BALLOC_BUDDY_ ## flag_up; \
89 }
90
91/*
92 * int buddy_is_alloced(struct nvgpu_buddy *b);
93 * void buddy_set_alloced(struct nvgpu_buddy *b);
94 * void buddy_clr_alloced(struct nvgpu_buddy *b);
95 *
96 * int buddy_is_split(struct nvgpu_buddy *b);
97 * void buddy_set_split(struct nvgpu_buddy *b);
98 * void buddy_clr_split(struct nvgpu_buddy *b);
99 *
100 * int buddy_is_in_list(struct nvgpu_buddy *b);
101 * void buddy_set_in_list(struct nvgpu_buddy *b);
102 * void buddy_clr_in_list(struct nvgpu_buddy *b);
103 */
104__buddy_flag_ops(alloced, ALLOCED);
105__buddy_flag_ops(split, SPLIT);
106__buddy_flag_ops(in_list, IN_LIST);
107
108/*
109 * Keeps info for a fixed allocation.
110 */
111struct nvgpu_fixed_alloc {
112 struct nvgpu_list_node buddies; /* List of buddies. */
113 struct nvgpu_rbtree_node alloced_entry; /* RB tree of fixed allocations. */
114
115 u64 start; /* Start of fixed block. */
116 u64 end; /* End address. */
117};
118
119static inline struct nvgpu_fixed_alloc *
120nvgpu_fixed_alloc_from_rbtree_node(struct nvgpu_rbtree_node *node)
121{
122 return (struct nvgpu_fixed_alloc *)
123 ((uintptr_t)node - offsetof(struct nvgpu_fixed_alloc, alloced_entry));
124};
125
126/*
127 * GPU buddy allocator for the various GPU address spaces. Each addressable unit
128 * doesn't have to correspond to a byte. In some cases each unit is a more
129 * complex object such as a comp_tag line or the like.
130 *
131 * The max order is computed based on the size of the minimum order and the size
132 * of the address space.
133 *
134 * order_size is the size of an order 0 buddy.
135 */
136struct nvgpu_buddy_allocator {
137 struct nvgpu_allocator *owner; /* Owner of this buddy allocator. */
138 struct vm_gk20a *vm; /* Parent VM - can be NULL. */
139
140 u64 base; /* Base address of the space. */
141 u64 length; /* Length of the space. */
142 u64 blk_size; /* Size of order 0 allocation. */
143 u64 blk_shift; /* Shift to divide by blk_size. */
144
145 /* Internal stuff. */
146 u64 start; /* Real start (aligned to blk_size). */
147 u64 end; /* Real end, trimmed if needed. */
148 u64 count; /* Count of objects in space. */
149 u64 blks; /* Count of blks in the space. */
150 u64 max_order; /* Specific maximum order. */
151
152 struct nvgpu_rbtree_node *alloced_buddies; /* Outstanding allocations. */
153 struct nvgpu_rbtree_node *fixed_allocs; /* Outstanding fixed allocations. */
154
155 struct nvgpu_list_node co_list;
156
157 struct nvgpu_kmem_cache *buddy_cache;
158
159 /*
160 * Impose an upper bound on the maximum order.
161 */
162#define GPU_BALLOC_ORDER_LIST_LEN (GPU_BALLOC_MAX_ORDER + 1)
163
164 struct nvgpu_list_node buddy_list[GPU_BALLOC_ORDER_LIST_LEN];
165 u64 buddy_list_len[GPU_BALLOC_ORDER_LIST_LEN];
166 u64 buddy_list_split[GPU_BALLOC_ORDER_LIST_LEN];
167 u64 buddy_list_alloced[GPU_BALLOC_ORDER_LIST_LEN];
168
169 /*
170 * This is for when the allocator is managing a GVA space (the
171 * GPU_ALLOC_GVA_SPACE bit is set in @flags). This requires
172 * that we group like sized allocations into PDE blocks.
173 */
174 u64 pte_blk_order;
175
176 int initialized;
177 int alloc_made; /* True after the first alloc. */
178
179 u64 flags;
180
181 u64 bytes_alloced;
182 u64 bytes_alloced_real;
183 u64 bytes_freed;
184};
185
186static inline struct nvgpu_buddy_allocator *buddy_allocator(
187 struct nvgpu_allocator *a)
188{
189 return (struct nvgpu_buddy_allocator *)(a)->priv;
190}
191
192static inline struct nvgpu_list_node *balloc_get_order_list(
193 struct nvgpu_buddy_allocator *a, int order)
194{
195 return &a->buddy_list[order];
196}
197
198static inline u64 balloc_order_to_len(struct nvgpu_buddy_allocator *a,
199 int order)
200{
201 return (1 << order) * a->blk_size;
202}
203
204static inline u64 balloc_base_shift(struct nvgpu_buddy_allocator *a,
205 u64 base)
206{
207 return base - a->start;
208}
209
210static inline u64 balloc_base_unshift(struct nvgpu_buddy_allocator *a,
211 u64 base)
212{
213 return base + a->start;
214}
215
216static inline struct nvgpu_allocator *balloc_owner(
217 struct nvgpu_buddy_allocator *a)
218{
219 return a->owner;
220}
221
222#endif
diff --git a/drivers/gpu/nvgpu/common/mm/comptags.c b/drivers/gpu/nvgpu/common/mm/comptags.c
new file mode 100644
index 00000000..8f2fe90f
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/comptags.c
@@ -0,0 +1,95 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/bitops.h>
24#include <nvgpu/comptags.h>
25
26#include "gk20a/gk20a.h"
27
28int gk20a_comptaglines_alloc(struct gk20a_comptag_allocator *allocator,
29 u32 *offset, u32 len)
30{
31 unsigned long addr;
32 int err = 0;
33
34 nvgpu_mutex_acquire(&allocator->lock);
35 addr = bitmap_find_next_zero_area(allocator->bitmap, allocator->size,
36 0, len, 0);
37 if (addr < allocator->size) {
38 /* number zero is reserved; bitmap base is 1 */
39 *offset = 1 + addr;
40 bitmap_set(allocator->bitmap, addr, len);
41 } else {
42 err = -ENOMEM;
43 }
44 nvgpu_mutex_release(&allocator->lock);
45
46 return err;
47}
48
49void gk20a_comptaglines_free(struct gk20a_comptag_allocator *allocator,
50 u32 offset, u32 len)
51{
52 /* number zero is reserved; bitmap base is 1 */
53 u32 addr = offset - 1;
54
55 WARN_ON(offset == 0);
56 WARN_ON(addr > allocator->size);
57 WARN_ON(addr + len > allocator->size);
58
59 nvgpu_mutex_acquire(&allocator->lock);
60 bitmap_clear(allocator->bitmap, addr, len);
61 nvgpu_mutex_release(&allocator->lock);
62}
63
64int gk20a_comptag_allocator_init(struct gk20a *g,
65 struct gk20a_comptag_allocator *allocator,
66 unsigned long size)
67{
68 nvgpu_mutex_init(&allocator->lock);
69
70 /*
71 * 0th comptag is special and is never used. The base for this bitmap
72 * is 1, and its size is one less than the size of comptag store.
73 */
74 size--;
75 allocator->bitmap = nvgpu_vzalloc(g,
76 BITS_TO_LONGS(size) * sizeof(long));
77 if (!allocator->bitmap)
78 return -ENOMEM;
79
80 allocator->size = size;
81
82 return 0;
83}
84
85void gk20a_comptag_allocator_destroy(struct gk20a *g,
86 struct gk20a_comptag_allocator *allocator)
87{
88 /*
89 * called only when exiting the driver (gk20a_remove, or unwinding the
90 * init stage); no users should be active, so taking the mutex is
91 * unnecessary here.
92 */
93 allocator->size = 0;
94 nvgpu_vfree(g, allocator->bitmap);
95}
diff --git a/drivers/gpu/nvgpu/common/mm/gmmu.c b/drivers/gpu/nvgpu/common/mm/gmmu.c
new file mode 100644
index 00000000..568da8c4
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/gmmu.c
@@ -0,0 +1,920 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <uapi/linux/nvgpu.h>
24
25#include <nvgpu/log.h>
26#include <nvgpu/list.h>
27#include <nvgpu/dma.h>
28#include <nvgpu/gmmu.h>
29#include <nvgpu/nvgpu_mem.h>
30#include <nvgpu/enabled.h>
31#include <nvgpu/page_allocator.h>
32#include <nvgpu/barrier.h>
33#include <nvgpu/vidmem.h>
34
35#include "gk20a/gk20a.h"
36#include "gk20a/mm_gk20a.h"
37
38#define __gmmu_dbg(g, attrs, fmt, args...) \
39 do { \
40 if (attrs->debug) \
41 nvgpu_info(g, fmt, ##args); \
42 else \
43 nvgpu_log(g, gpu_dbg_map, fmt, ##args); \
44 } while (0)
45
46#define __gmmu_dbg_v(g, attrs, fmt, args...) \
47 do { \
48 if (attrs->debug) \
49 nvgpu_info(g, fmt, ##args); \
50 else \
51 nvgpu_log(g, gpu_dbg_map_v, fmt, ##args); \
52 } while (0)
53
54static int pd_allocate(struct vm_gk20a *vm,
55 struct nvgpu_gmmu_pd *pd,
56 const struct gk20a_mmu_level *l,
57 struct nvgpu_gmmu_attrs *attrs);
58static u32 pd_size(const struct gk20a_mmu_level *l,
59 struct nvgpu_gmmu_attrs *attrs);
60/*
61 * Core GMMU map function for the kernel to use. If @addr is 0 then the GPU
62 * VA will be allocated for you. If addr is non-zero then the buffer will be
63 * mapped at @addr.
64 */
65static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm,
66 struct nvgpu_mem *mem,
67 u64 addr,
68 u64 size,
69 u32 flags,
70 int rw_flag,
71 bool priv,
72 enum nvgpu_aperture aperture)
73{
74 struct gk20a *g = gk20a_from_vm(vm);
75 u64 vaddr;
76
77 struct nvgpu_sgt *sgt = nvgpu_sgt_create_from_mem(g, mem);
78
79 if (!sgt)
80 return -ENOMEM;
81
82 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
83 vaddr = g->ops.mm.gmmu_map(vm, addr,
84 sgt, /* sg list */
85 0, /* sg offset */
86 size,
87 gmmu_page_size_kernel,
88 0, /* kind */
89 0, /* ctag_offset */
90 flags, rw_flag,
91 false, /* clear_ctags */
92 false, /* sparse */
93 priv, /* priv */
94 NULL, /* mapping_batch handle */
95 aperture);
96 nvgpu_mutex_release(&vm->update_gmmu_lock);
97
98 nvgpu_sgt_free(g, sgt);
99
100 if (!vaddr) {
101 nvgpu_err(g, "failed to map buffer!");
102 return 0;
103 }
104
105 return vaddr;
106}
107
108/*
109 * Map a nvgpu_mem into the GMMU. This is for kernel space to use.
110 */
111u64 nvgpu_gmmu_map(struct vm_gk20a *vm,
112 struct nvgpu_mem *mem,
113 u64 size,
114 u32 flags,
115 int rw_flag,
116 bool priv,
117 enum nvgpu_aperture aperture)
118{
119 return __nvgpu_gmmu_map(vm, mem, 0, size, flags, rw_flag, priv,
120 aperture);
121}
122
123/*
124 * Like nvgpu_gmmu_map() except this can work on a fixed address.
125 */
126u64 nvgpu_gmmu_map_fixed(struct vm_gk20a *vm,
127 struct nvgpu_mem *mem,
128 u64 addr,
129 u64 size,
130 u32 flags,
131 int rw_flag,
132 bool priv,
133 enum nvgpu_aperture aperture)
134{
135 return __nvgpu_gmmu_map(vm, mem, addr, size, flags, rw_flag, priv,
136 aperture);
137}
138
139void nvgpu_gmmu_unmap(struct vm_gk20a *vm, struct nvgpu_mem *mem, u64 gpu_va)
140{
141 struct gk20a *g = gk20a_from_vm(vm);
142
143 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
144 g->ops.mm.gmmu_unmap(vm,
145 gpu_va,
146 mem->size,
147 gmmu_page_size_kernel,
148 true, /*va_allocated */
149 gk20a_mem_flag_none,
150 false,
151 NULL);
152
153 nvgpu_mutex_release(&vm->update_gmmu_lock);
154}
155
156int nvgpu_gmmu_init_page_table(struct vm_gk20a *vm)
157{
158 u32 pdb_size;
159 int err;
160
161 /*
162 * Need this just for page size. Everything else can be ignored. Also
163 * note that we can just use pgsz 0 (i.e small pages) since the number
164 * of bits present in the top level PDE are the same for small/large
165 * page VMs.
166 */
167 struct nvgpu_gmmu_attrs attrs = {
168 .pgsz = 0,
169 };
170
171 /*
172 * PDB size here must be one page so that its address is page size
173 * aligned. Although lower PDE tables can be aligned at 256B boundaries
174 * the main PDB must be page aligned.
175 */
176 pdb_size = ALIGN(pd_size(&vm->mmu_levels[0], &attrs), PAGE_SIZE);
177
178 err = __nvgpu_pd_cache_alloc_direct(vm->mm->g, &vm->pdb, pdb_size);
179 if (WARN_ON(err))
180 return err;
181
182 /*
183 * One nvgpu_smp_mb() is done after all mapping operations. Don't need
184 * individual barriers for each PD write.
185 */
186 vm->pdb.mem->skip_wmb = true;
187
188 return 0;
189}
190
191/*
192 * Ensure that there's a CPU mapping for the page directory memory. This won't
193 * always be the case for 32 bit systems since we may need to save kernel
194 * virtual memory.
195 */
196static int map_gmmu_pages(struct gk20a *g, struct nvgpu_gmmu_pd *pd)
197{
198 return nvgpu_mem_begin(g, pd->mem);
199}
200
201/*
202 * Handle any necessary CPU unmap semantics for a page directories DMA memory.
203 * For 64 bit platforms this is a noop.
204 */
205static void unmap_gmmu_pages(struct gk20a *g, struct nvgpu_gmmu_pd *pd)
206{
207 nvgpu_mem_end(g, pd->mem);
208}
209
210/*
211 * Return the _physical_ address of a page directory.
212 */
213u64 nvgpu_pde_phys_addr(struct gk20a *g, struct nvgpu_gmmu_pd *pd)
214{
215 u64 page_addr;
216
217 if (g->mm.has_physical_mode)
218 page_addr = nvgpu_mem_get_phys_addr(g, pd->mem);
219 else
220 page_addr = nvgpu_mem_get_addr(g, pd->mem);
221
222 return page_addr + pd->mem_offs;
223}
224
225/*
226 * Return the aligned length based on the page size in attrs.
227 */
228static u64 nvgpu_align_map_length(struct vm_gk20a *vm, u64 length,
229 struct nvgpu_gmmu_attrs *attrs)
230{
231 u64 page_size = vm->gmmu_page_sizes[attrs->pgsz];
232
233 return ALIGN(length, page_size);
234}
235
236static u32 pd_entries(const struct gk20a_mmu_level *l,
237 struct nvgpu_gmmu_attrs *attrs)
238{
239 /*
240 * Number of entries in a PD is easy to compute from the number of bits
241 * used to index the page directory. That is simply 2 raised to the
242 * number of bits.
243 */
244 return 1UL << (l->hi_bit[attrs->pgsz] - l->lo_bit[attrs->pgsz] + 1UL);
245}
246
247/*
248 * Computes the size of a PD table.
249 */
250static u32 pd_size(const struct gk20a_mmu_level *l,
251 struct nvgpu_gmmu_attrs *attrs)
252{
253 return pd_entries(l, attrs) * l->entry_size;
254}
255
256/*
257 * Allocate a physically contiguous region big enough for a gmmu page table
258 * of the specified level and page size. The whole range is zeroed so that any
259 * accesses will fault until proper values are programmed.
260 */
261static int pd_allocate(struct vm_gk20a *vm,
262 struct nvgpu_gmmu_pd *pd,
263 const struct gk20a_mmu_level *l,
264 struct nvgpu_gmmu_attrs *attrs)
265{
266 int err;
267
268 if (pd->mem)
269 return 0;
270
271 err = __nvgpu_pd_alloc(vm, pd, pd_size(l, attrs));
272 if (err) {
273 nvgpu_info(vm->mm->g, "error allocating page directory!");
274 return err;
275 }
276
277 /*
278 * One nvgpu_smp_mb() is done after all mapping operations. Don't need
279 * individual barriers for each PD write.
280 */
281 pd->mem->skip_wmb = true;
282
283 return 0;
284}
285
286/*
287 * Compute what page directory index at the passed level the passed virtual
288 * address corresponds to. @attrs is necessary for determining the page size
289 * which is used to pick the right bit offsets for the GMMU level.
290 */
291static u32 pd_index(const struct gk20a_mmu_level *l, u64 virt,
292 struct nvgpu_gmmu_attrs *attrs)
293{
294 u64 pd_mask = (1ULL << ((u64)l->hi_bit[attrs->pgsz] + 1)) - 1ULL;
295 u32 pd_shift = (u64)l->lo_bit[attrs->pgsz];
296
297 /*
298 * For convenience we don't bother computing the lower bound of the
299 * mask; it's easier to just shift it off.
300 */
301 return (virt & pd_mask) >> pd_shift;
302}
303
304static int pd_allocate_children(struct vm_gk20a *vm,
305 const struct gk20a_mmu_level *l,
306 struct nvgpu_gmmu_pd *pd,
307 struct nvgpu_gmmu_attrs *attrs)
308{
309 struct gk20a *g = gk20a_from_vm(vm);
310
311 if (pd->entries)
312 return 0;
313
314 pd->num_entries = pd_entries(l, attrs);
315 pd->entries = nvgpu_vzalloc(g, sizeof(struct nvgpu_gmmu_pd) *
316 pd->num_entries);
317 if (!pd->entries)
318 return -ENOMEM;
319
320 return 0;
321}
322
323/*
324 * This function programs the GMMU based on two ranges: a physical range and a
325 * GPU virtual range. The virtual is mapped to the physical. Physical in this
326 * case can mean either a real physical sysmem address or a IO virtual address
327 * (for instance when a system has an IOMMU running).
328 *
329 * The rest of the parameters are for describing the actual mapping itself.
330 *
331 * This function recursively calls itself for handling PDEs. At the final level
332 * a PTE handler is called. The phys and virt ranges are adjusted for each
333 * recursion so that each invocation of this function need only worry about the
334 * range it is passed.
335 *
336 * phys_addr will always point to a contiguous range - the discontiguous nature
337 * of DMA buffers is taken care of at the layer above this.
338 */
339static int __set_pd_level(struct vm_gk20a *vm,
340 struct nvgpu_gmmu_pd *pd,
341 int lvl,
342 u64 phys_addr,
343 u64 virt_addr, u64 length,
344 struct nvgpu_gmmu_attrs *attrs)
345{
346 int err = 0;
347 u64 pde_range;
348 struct gk20a *g = gk20a_from_vm(vm);
349 struct nvgpu_gmmu_pd *next_pd = NULL;
350 const struct gk20a_mmu_level *l = &vm->mmu_levels[lvl];
351 const struct gk20a_mmu_level *next_l = &vm->mmu_levels[lvl + 1];
352
353 /*
354 * 5 levels for Pascal+. For pre-pascal we only have 2. This puts
355 * offsets into the page table debugging code which makes it easier to
356 * see what level prints are from.
357 */
358 static const char *__lvl_debug[] = {
359 "", /* L=0 */
360 " ", /* L=1 */
361 " ", /* L=2 */
362 " ", /* L=3 */
363 " ", /* L=4 */
364 };
365
366 pde_range = 1ULL << (u64)l->lo_bit[attrs->pgsz];
367
368 __gmmu_dbg_v(g, attrs,
369 "L=%d %sGPU virt %#-12llx +%#-9llx -> phys %#-12llx",
370 lvl,
371 __lvl_debug[lvl],
372 virt_addr,
373 length,
374 phys_addr);
375
376 /*
377 * Iterate across the mapping in chunks the size of this level's PDE.
378 * For each of those chunks program our level's PDE and then, if there's
379 * a next level, program the next level's PDEs/PTEs.
380 */
381 while (length) {
382 u32 pd_idx = pd_index(l, virt_addr, attrs);
383 u64 chunk_size;
384 u64 target_addr;
385
386 /*
387 * Truncate the pde_range when the virtual address does not
388 * start at a PDE boundary.
389 */
390 chunk_size = min(length,
391 pde_range - (virt_addr & (pde_range - 1)));
392
393 /*
394 * If the next level has an update_entry function then we know
395 * that _this_ level points to PDEs (not PTEs). Thus we need to
396 * have a bunch of children PDs.
397 */
398 if (next_l->update_entry) {
399 if (pd_allocate_children(vm, l, pd, attrs))
400 return -ENOMEM;
401
402 /*
403 * Get the next PD so that we know what to put in this
404 * current PD. If the next level is actually PTEs then
405 * we don't need this - we will just use the real
406 * physical target.
407 */
408 next_pd = &pd->entries[pd_idx];
409
410 /*
411 * Allocate the backing memory for next_pd.
412 */
413 if (pd_allocate(vm, next_pd, next_l, attrs))
414 return -ENOMEM;
415 }
416
417 /*
418 * This is the address we want to program into the actual PDE/
419 * PTE. When the next level is PDEs we need the target address
420 * to be the table of PDEs. When the next level is PTEs the
421 * target addr is the real physical address we are aiming for.
422 */
423 target_addr = next_pd ?
424 nvgpu_pde_phys_addr(g, next_pd) :
425 phys_addr;
426
427 l->update_entry(vm, l,
428 pd, pd_idx,
429 virt_addr,
430 target_addr,
431 attrs);
432
433 if (next_l->update_entry) {
434 err = map_gmmu_pages(g, next_pd);
435 if (err) {
436 nvgpu_err(g,
437 "couldn't map ptes for update as=%d",
438 vm_aspace_id(vm));
439 return err;
440 }
441
442 err = __set_pd_level(vm, next_pd,
443 lvl + 1,
444 phys_addr,
445 virt_addr,
446 chunk_size,
447 attrs);
448 unmap_gmmu_pages(g, next_pd);
449
450 if (err)
451 return err;
452 }
453
454 virt_addr += chunk_size;
455
456 /*
457 * Only add to phys_addr if it's non-zero. A zero value implies
458 * we are unmapping as as a result we don't want to place
459 * non-zero phys addresses in the PTEs. A non-zero phys-addr
460 * would also confuse the lower level PTE programming code.
461 */
462 if (phys_addr)
463 phys_addr += chunk_size;
464 length -= chunk_size;
465 }
466
467 __gmmu_dbg_v(g, attrs, "L=%d %s%s", lvl, __lvl_debug[lvl], "ret!");
468
469 return 0;
470}
471
472static int __nvgpu_gmmu_do_update_page_table(struct vm_gk20a *vm,
473 struct nvgpu_sgt *sgt,
474 u64 space_to_skip,
475 u64 virt_addr,
476 u64 length,
477 struct nvgpu_gmmu_attrs *attrs)
478{
479 struct gk20a *g = gk20a_from_vm(vm);
480 void *sgl;
481 int err = 0;
482
483 if (!sgt) {
484 /*
485 * This is considered an unmap. Just pass in 0 as the physical
486 * address for the entire GPU range.
487 */
488 err = __set_pd_level(vm, &vm->pdb,
489 0,
490 0,
491 virt_addr, length,
492 attrs);
493 return err;
494 }
495
496 /*
497 * At this point we have a scatter-gather list pointing to some number
498 * of discontiguous chunks of memory. We must iterate over that list and
499 * generate a GMMU map call for each chunk. There are two possibilities:
500 * either an IOMMU is enabled or not. When an IOMMU is enabled the
501 * mapping is simple since the "physical" address is actually a virtual
502 * IO address and will be contiguous.
503 */
504 if (attrs->aperture == APERTURE_SYSMEM && !g->mm.bypass_smmu) {
505 u64 io_addr = nvgpu_sgt_get_gpu_addr(g, sgt, sgt->sgl, attrs);
506
507 io_addr += space_to_skip;
508
509 err = __set_pd_level(vm, &vm->pdb,
510 0,
511 io_addr,
512 virt_addr,
513 length,
514 attrs);
515
516 return err;
517 }
518
519 /*
520 * Finally: last possible case: do the no-IOMMU mapping. In this case we
521 * really are mapping physical pages directly.
522 */
523 nvgpu_sgt_for_each_sgl(sgl, sgt) {
524 u64 phys_addr;
525 u64 chunk_length;
526
527 /*
528 * Cut out sgl ents for space_to_skip.
529 */
530 if (space_to_skip &&
531 space_to_skip >= nvgpu_sgt_get_length(sgt, sgl)) {
532 space_to_skip -= nvgpu_sgt_get_length(sgt, sgl);
533 continue;
534 }
535
536 phys_addr = nvgpu_sgt_get_phys(sgt, sgl) + space_to_skip;
537 chunk_length = min(length,
538 nvgpu_sgt_get_length(sgt, sgl) - space_to_skip);
539
540 err = __set_pd_level(vm, &vm->pdb,
541 0,
542 phys_addr,
543 virt_addr,
544 chunk_length,
545 attrs);
546 if (err)
547 break;
548
549 /* Space has been skipped so zero this for future chunks. */
550 space_to_skip = 0;
551
552 /*
553 * Update the map pointer and the remaining length.
554 */
555 virt_addr += chunk_length;
556 length -= chunk_length;
557
558 if (length == 0)
559 break;
560 }
561
562 return err;
563}
564
565/*
566 * This is the true top level GMMU mapping logic. This breaks down the incoming
567 * scatter gather table and does actual programming of GPU virtual address to
568 * physical* address.
569 *
570 * The update of each level of the page tables is farmed out to chip specific
571 * implementations. But the logic around that is generic to all chips. Every
572 * chip has some number of PDE levels and then a PTE level.
573 *
574 * Each chunk of the incoming SGL is sent to the chip specific implementation
575 * of page table update.
576 *
577 * [*] Note: the "physical" address may actually be an IO virtual address in the
578 * case of SMMU usage.
579 */
580static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm,
581 struct nvgpu_sgt *sgt,
582 u64 space_to_skip,
583 u64 virt_addr,
584 u64 length,
585 struct nvgpu_gmmu_attrs *attrs)
586{
587 struct gk20a *g = gk20a_from_vm(vm);
588 u32 page_size;
589 int err;
590
591 /* note: here we need to map kernel to small, since the
592 * low-level mmu code assumes 0 is small and 1 is big pages */
593 if (attrs->pgsz == gmmu_page_size_kernel)
594 attrs->pgsz = gmmu_page_size_small;
595
596 page_size = vm->gmmu_page_sizes[attrs->pgsz];
597
598 if (space_to_skip & (page_size - 1))
599 return -EINVAL;
600
601 /*
602 * Update length to be aligned to the passed page size.
603 */
604 length = nvgpu_align_map_length(vm, length, attrs);
605
606 err = map_gmmu_pages(g, &vm->pdb);
607 if (err) {
608 nvgpu_err(g, "couldn't map ptes for update as=%d",
609 vm_aspace_id(vm));
610 return err;
611 }
612
613 __gmmu_dbg(g, attrs,
614 "vm=%s "
615 "%-5s GPU virt %#-12llx +%#-9llx phys %#-12llx "
616 "phys offset: %#-4llx; pgsz: %3dkb perm=%-2s | "
617 "kind=%#02x APT=%-6s %c%c%c%c%c",
618 vm->name,
619 sgt ? "MAP" : "UNMAP",
620 virt_addr,
621 length,
622 sgt ? nvgpu_sgt_get_phys(sgt, sgt->sgl) : 0,
623 space_to_skip,
624 page_size >> 10,
625 nvgpu_gmmu_perm_str(attrs->rw_flag),
626 attrs->kind_v,
627 nvgpu_aperture_str(attrs->aperture),
628 attrs->cacheable ? 'C' : 'c', /* C = cached, V = volatile. */
629 attrs->sparse ? 'S' : '-',
630 attrs->priv ? 'P' : '-',
631 attrs->coherent ? 'c' : '-',
632 attrs->valid ? 'V' : '-');
633
634 err = __nvgpu_gmmu_do_update_page_table(vm,
635 sgt,
636 space_to_skip,
637 virt_addr,
638 length,
639 attrs);
640
641 unmap_gmmu_pages(g, &vm->pdb);
642 nvgpu_smp_mb();
643
644 __gmmu_dbg(g, attrs, "%-5s Done!", sgt ? "MAP" : "UNMAP");
645
646 return err;
647}
648
649/**
650 * gk20a_locked_gmmu_map - Map a buffer into the GMMU
651 *
652 * This is for non-vGPU chips. It's part of the HAL at the moment but really
653 * should not be. Chip specific stuff is handled at the PTE/PDE programming
654 * layer. The rest of the logic is essentially generic for all chips.
655 *
656 * To call this function you must have locked the VM lock: vm->update_gmmu_lock.
657 * However, note: this function is not called directly. It's used through the
658 * mm.gmmu_lock() HAL. So before calling the mm.gmmu_lock() HAL make sure you
659 * have the update_gmmu_lock aquired.
660 */
661u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
662 u64 vaddr,
663 struct nvgpu_sgt *sgt,
664 u64 buffer_offset,
665 u64 size,
666 int pgsz_idx,
667 u8 kind_v,
668 u32 ctag_offset,
669 u32 flags,
670 int rw_flag,
671 bool clear_ctags,
672 bool sparse,
673 bool priv,
674 struct vm_gk20a_mapping_batch *batch,
675 enum nvgpu_aperture aperture)
676{
677 struct gk20a *g = gk20a_from_vm(vm);
678 int err = 0;
679 bool allocated = false;
680 int ctag_granularity = g->ops.fb.compression_page_size(g);
681 struct nvgpu_gmmu_attrs attrs = {
682 .pgsz = pgsz_idx,
683 .kind_v = kind_v,
684 .ctag = (u64)ctag_offset * (u64)ctag_granularity,
685 .cacheable = flags & NVGPU_AS_MAP_BUFFER_FLAGS_CACHEABLE,
686 .rw_flag = rw_flag,
687 .sparse = sparse,
688 .priv = priv,
689 .coherent = flags & NVGPU_AS_MAP_BUFFER_FLAGS_IO_COHERENT,
690 .valid = !(flags & NVGPU_AS_MAP_BUFFER_FLAGS_UNMAPPED_PTE),
691 .aperture = aperture
692 };
693
694#ifdef CONFIG_TEGRA_19x_GPU
695 nvgpu_gmmu_add_t19x_attrs(&attrs, flags);
696#endif
697
698 /*
699 * Only allocate a new GPU VA range if we haven't already been passed a
700 * GPU VA range. This facilitates fixed mappings.
701 */
702 if (!vaddr) {
703 vaddr = __nvgpu_vm_alloc_va(vm, size, pgsz_idx);
704 if (!vaddr) {
705 nvgpu_err(g, "failed to allocate va space");
706 err = -ENOMEM;
707 goto fail_alloc;
708 }
709 allocated = true;
710 }
711
712 err = __nvgpu_gmmu_update_page_table(vm, sgt, buffer_offset,
713 vaddr, size, &attrs);
714 if (err) {
715 nvgpu_err(g, "failed to update ptes on map");
716 goto fail_validate;
717 }
718
719 if (!batch)
720 g->ops.fb.tlb_invalidate(g, vm->pdb.mem);
721 else
722 batch->need_tlb_invalidate = true;
723
724 return vaddr;
725
726fail_validate:
727 if (allocated)
728 __nvgpu_vm_free_va(vm, vaddr, pgsz_idx);
729fail_alloc:
730 nvgpu_err(g, "%s: failed with err=%d", __func__, err);
731 return 0;
732}
733
734void gk20a_locked_gmmu_unmap(struct vm_gk20a *vm,
735 u64 vaddr,
736 u64 size,
737 int pgsz_idx,
738 bool va_allocated,
739 int rw_flag,
740 bool sparse,
741 struct vm_gk20a_mapping_batch *batch)
742{
743 int err = 0;
744 struct gk20a *g = gk20a_from_vm(vm);
745 struct nvgpu_gmmu_attrs attrs = {
746 .pgsz = pgsz_idx,
747 .kind_v = 0,
748 .ctag = 0,
749 .cacheable = 0,
750 .rw_flag = rw_flag,
751 .sparse = sparse,
752 .priv = 0,
753 .coherent = 0,
754 .valid = 0,
755 .aperture = APERTURE_INVALID,
756 };
757
758 if (va_allocated) {
759 err = __nvgpu_vm_free_va(vm, vaddr, pgsz_idx);
760 if (err) {
761 nvgpu_err(g, "failed to free va");
762 return;
763 }
764 }
765
766 /* unmap here needs to know the page size we assigned at mapping */
767 err = __nvgpu_gmmu_update_page_table(vm, NULL, 0,
768 vaddr, size, &attrs);
769 if (err)
770 nvgpu_err(g, "failed to update gmmu ptes on unmap");
771
772 if (!batch) {
773 gk20a_mm_l2_flush(g, true);
774 g->ops.fb.tlb_invalidate(g, vm->pdb.mem);
775 } else {
776 if (!batch->gpu_l2_flushed) {
777 gk20a_mm_l2_flush(g, true);
778 batch->gpu_l2_flushed = true;
779 }
780 batch->need_tlb_invalidate = true;
781 }
782}
783
784u32 __nvgpu_pte_words(struct gk20a *g)
785{
786 const struct gk20a_mmu_level *l = g->ops.mm.get_mmu_levels(g, SZ_64K);
787 const struct gk20a_mmu_level *next_l;
788
789 /*
790 * Iterate to the bottom GMMU level - the PTE level. The levels array
791 * is always NULL terminated (by the update_entry function).
792 */
793 do {
794 next_l = l + 1;
795 if (!next_l->update_entry)
796 break;
797
798 l++;
799 } while (true);
800
801 return (u32)(l->entry_size / sizeof(u32));
802}
803
804/*
805 * Recursively walk the pages tables to find the PTE.
806 */
807static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm,
808 struct nvgpu_gmmu_pd *pd,
809 u64 vaddr, int lvl,
810 struct nvgpu_gmmu_attrs *attrs,
811 u32 *data,
812 struct nvgpu_gmmu_pd **pd_out, u32 *pd_idx_out,
813 u32 *pd_offs_out)
814{
815 const struct gk20a_mmu_level *l = &vm->mmu_levels[lvl];
816 const struct gk20a_mmu_level *next_l = &vm->mmu_levels[lvl + 1];
817 u32 pd_idx = pd_index(l, vaddr, attrs);
818 u32 pte_base;
819 u32 pte_size;
820 u32 i;
821
822 /*
823 * If this isn't the final level (i.e there's a valid next level)
824 * then find the next level PD and recurse.
825 */
826 if (next_l->update_entry) {
827 struct nvgpu_gmmu_pd *pd_next = pd->entries + pd_idx;
828
829 /* Invalid entry! */
830 if (!pd_next->mem)
831 return -EINVAL;
832
833 attrs->pgsz = l->get_pgsz(g, pd, pd_idx);
834
835 if (attrs->pgsz >= gmmu_nr_page_sizes)
836 return -EINVAL;
837
838 return __nvgpu_locate_pte(g, vm, pd_next,
839 vaddr, lvl + 1, attrs,
840 data, pd_out, pd_idx_out,
841 pd_offs_out);
842 }
843
844 if (!pd->mem)
845 return -EINVAL;
846
847 /*
848 * Take into account the real offset into the nvgpu_mem since the PD
849 * may be located at an offset other than 0 (due to PD packing).
850 */
851 pte_base = (pd->mem_offs / sizeof(u32)) +
852 pd_offset_from_index(l, pd_idx);
853 pte_size = (u32)(l->entry_size / sizeof(u32));
854
855 if (data) {
856 map_gmmu_pages(g, pd);
857 for (i = 0; i < pte_size; i++)
858 data[i] = nvgpu_mem_rd32(g, pd->mem, pte_base + i);
859 unmap_gmmu_pages(g, pd);
860 }
861
862 if (pd_out)
863 *pd_out = pd;
864
865 if (pd_idx_out)
866 *pd_idx_out = pd_idx;
867
868 if (pd_offs_out)
869 *pd_offs_out = pd_offset_from_index(l, pd_idx);
870
871 return 0;
872}
873
874int __nvgpu_get_pte(struct gk20a *g, struct vm_gk20a *vm, u64 vaddr, u32 *pte)
875{
876 struct nvgpu_gmmu_attrs attrs = {
877 .pgsz = 0,
878 };
879
880 return __nvgpu_locate_pte(g, vm, &vm->pdb,
881 vaddr, 0, &attrs,
882 pte, NULL, NULL, NULL);
883}
884
885int __nvgpu_set_pte(struct gk20a *g, struct vm_gk20a *vm, u64 vaddr, u32 *pte)
886{
887 struct nvgpu_gmmu_pd *pd;
888 u32 pd_idx, pd_offs, pte_size, i;
889 int err;
890 struct nvgpu_gmmu_attrs attrs = {
891 .pgsz = 0,
892 };
893 struct nvgpu_gmmu_attrs *attrs_ptr = &attrs;
894
895 err = __nvgpu_locate_pte(g, vm, &vm->pdb,
896 vaddr, 0, &attrs,
897 NULL, &pd, &pd_idx, &pd_offs);
898 if (err)
899 return err;
900
901 pte_size = __nvgpu_pte_words(g);
902
903 map_gmmu_pages(g, pd);
904 for (i = 0; i < pte_size; i++) {
905 pd_write(g, pd, pd_offs + i, pte[i]);
906 pte_dbg(g, attrs_ptr,
907 "PTE: idx=%-4u (%d) 0x%08x", pd_idx, i, pte[i]);
908 }
909 unmap_gmmu_pages(g, pd);
910
911 /*
912 * Ensures the pd_write()s are done. The pd_write() does not do this
913 * since generally there's lots of pd_write()s called one after another.
914 * There probably also needs to be a TLB invalidate as well but we leave
915 * that to the caller of this function.
916 */
917 nvgpu_smp_wmb();
918
919 return 0;
920}
diff --git a/drivers/gpu/nvgpu/common/mm/lockless_allocator.c b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c
new file mode 100644
index 00000000..3eb10fc4
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c
@@ -0,0 +1,225 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/atomic.h>
24#include <nvgpu/allocator.h>
25#include <nvgpu/kmem.h>
26#include <nvgpu/barrier.h>
27
28#include "lockless_allocator_priv.h"
29
30static u64 nvgpu_lockless_alloc_length(struct nvgpu_allocator *a)
31{
32 struct nvgpu_lockless_allocator *pa = a->priv;
33
34 return pa->length;
35}
36
37static u64 nvgpu_lockless_alloc_base(struct nvgpu_allocator *a)
38{
39 struct nvgpu_lockless_allocator *pa = a->priv;
40
41 return pa->base;
42}
43
44static int nvgpu_lockless_alloc_inited(struct nvgpu_allocator *a)
45{
46 struct nvgpu_lockless_allocator *pa = a->priv;
47 int inited = pa->inited;
48
49 nvgpu_smp_rmb();
50 return inited;
51}
52
53static u64 nvgpu_lockless_alloc_end(struct nvgpu_allocator *a)
54{
55 struct nvgpu_lockless_allocator *pa = a->priv;
56
57 return pa->base + pa->length;
58}
59
60static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len)
61{
62 struct nvgpu_lockless_allocator *pa = a->priv;
63 int head, new_head, ret;
64 u64 addr = 0;
65
66 if (len != pa->blk_size)
67 return 0;
68
69 head = NV_ACCESS_ONCE(pa->head);
70 while (head >= 0) {
71 new_head = NV_ACCESS_ONCE(pa->next[head]);
72 ret = cmpxchg(&pa->head, head, new_head);
73 if (ret == head) {
74 addr = pa->base + head * pa->blk_size;
75 nvgpu_atomic_inc(&pa->nr_allocs);
76 alloc_dbg(a, "Alloc node # %d @ addr 0x%llx\n", head,
77 addr);
78 break;
79 }
80 head = NV_ACCESS_ONCE(pa->head);
81 }
82
83 if (addr)
84 alloc_dbg(a, "Alloc node # %d @ addr 0x%llx\n", head, addr);
85 else
86 alloc_dbg(a, "Alloc failed!\n");
87
88 return addr;
89}
90
91static void nvgpu_lockless_free(struct nvgpu_allocator *a, u64 addr)
92{
93 struct nvgpu_lockless_allocator *pa = a->priv;
94 int head, ret;
95 u64 cur_idx;
96
97 cur_idx = (addr - pa->base) / pa->blk_size;
98
99 alloc_dbg(a, "Free node # %llu @ addr 0x%llx\n", cur_idx, addr);
100
101 while (1) {
102 head = NV_ACCESS_ONCE(pa->head);
103 NV_ACCESS_ONCE(pa->next[cur_idx]) = head;
104 ret = cmpxchg(&pa->head, head, cur_idx);
105 if (ret == head) {
106 nvgpu_atomic_dec(&pa->nr_allocs);
107 alloc_dbg(a, "Free node # %llu\n", cur_idx);
108 break;
109 }
110 }
111}
112
113static void nvgpu_lockless_alloc_destroy(struct nvgpu_allocator *a)
114{
115 struct nvgpu_lockless_allocator *pa = a->priv;
116
117#ifdef CONFIG_DEBUG_FS
118 nvgpu_fini_alloc_debug(a);
119#endif
120
121 nvgpu_vfree(a->g, pa->next);
122 nvgpu_kfree(nvgpu_alloc_to_gpu(a), pa);
123}
124
125#ifdef __KERNEL__
126static void nvgpu_lockless_print_stats(struct nvgpu_allocator *a,
127 struct seq_file *s, int lock)
128{
129 struct nvgpu_lockless_allocator *pa = a->priv;
130
131 __alloc_pstat(s, a, "Lockless allocator params:\n");
132 __alloc_pstat(s, a, " start = 0x%llx\n", pa->base);
133 __alloc_pstat(s, a, " end = 0x%llx\n", pa->base + pa->length);
134
135 /* Actual stats. */
136 __alloc_pstat(s, a, "Stats:\n");
137 __alloc_pstat(s, a, " Number allocs = %d\n",
138 nvgpu_atomic_read(&pa->nr_allocs));
139 __alloc_pstat(s, a, " Number free = %d\n",
140 pa->nr_nodes - nvgpu_atomic_read(&pa->nr_allocs));
141}
142#endif
143
144static const struct nvgpu_allocator_ops pool_ops = {
145 .alloc = nvgpu_lockless_alloc,
146 .free = nvgpu_lockless_free,
147
148 .base = nvgpu_lockless_alloc_base,
149 .length = nvgpu_lockless_alloc_length,
150 .end = nvgpu_lockless_alloc_end,
151 .inited = nvgpu_lockless_alloc_inited,
152
153 .fini = nvgpu_lockless_alloc_destroy,
154
155#ifdef __KERNEL__
156 .print_stats = nvgpu_lockless_print_stats,
157#endif
158};
159
160int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
161 const char *name, u64 base, u64 length,
162 u64 blk_size, u64 flags)
163{
164 int i;
165 int err;
166 int nr_nodes;
167 u64 count;
168 struct nvgpu_lockless_allocator *a;
169
170 if (!blk_size)
171 return -EINVAL;
172
173 /*
174 * Ensure we have space for at least one node & there's no overflow.
175 * In order to control memory footprint, we require count < INT_MAX
176 */
177 count = length / blk_size;
178 if (!base || !count || count > INT_MAX)
179 return -EINVAL;
180
181 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_lockless_allocator));
182 if (!a)
183 return -ENOMEM;
184
185 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &pool_ops);
186 if (err)
187 goto fail;
188
189 a->next = nvgpu_vzalloc(g, sizeof(*a->next) * count);
190 if (!a->next) {
191 err = -ENOMEM;
192 goto fail;
193 }
194
195 /* chain the elements together to form the initial free list */
196 nr_nodes = (int)count;
197 for (i = 0; i < nr_nodes; i++)
198 a->next[i] = i + 1;
199 a->next[nr_nodes - 1] = -1;
200
201 a->base = base;
202 a->length = length;
203 a->blk_size = blk_size;
204 a->nr_nodes = nr_nodes;
205 a->flags = flags;
206 nvgpu_atomic_set(&a->nr_allocs, 0);
207
208 nvgpu_smp_wmb();
209 a->inited = true;
210
211#ifdef CONFIG_DEBUG_FS
212 nvgpu_init_alloc_debug(g, __a);
213#endif
214 alloc_dbg(__a, "New allocator: type lockless\n");
215 alloc_dbg(__a, " base 0x%llx\n", a->base);
216 alloc_dbg(__a, " nodes %d\n", a->nr_nodes);
217 alloc_dbg(__a, " blk_size 0x%llx\n", a->blk_size);
218 alloc_dbg(__a, " flags 0x%llx\n", a->flags);
219
220 return 0;
221
222fail:
223 nvgpu_kfree(g, a);
224 return err;
225}
diff --git a/drivers/gpu/nvgpu/common/mm/lockless_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/lockless_allocator_priv.h
new file mode 100644
index 00000000..c2f6649a
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/lockless_allocator_priv.h
@@ -0,0 +1,127 @@
1/*
2 * Copyright (c) 2016 - 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23/*
24 * Basics:
25 *
26 * - Lockless memory allocator for fixed-size structures, whose
27 * size is defined up front at init time.
28 * - Memory footprint scales linearly w/ the number of structures in
29 * the pool. It is ~= sizeof(int) * N.
30 * - Memory is pre-allocated by the client. The allocator itself
31 * only computes the addresses for allocations.
32 * - Limit of MAX_INT nodes that the allocator can be responsible for.
33 *
34 * Implementation details:
35 *
36 * The allocator maintains a single list of free nodes. We allocate &
37 * free nodes from the head of the list. We rely on the cmpxchg() operator
38 * to maintain atomicity on the head.
39 *
40 * So, both allocs & frees are O(1)!!
41 *
42 * -- Definitions --
43 * Block Size - size of a single structure that this allocator will
44 * allocate.
45 * Node - one of the elements of size blk_size in the
46 * client-allocated buffer.
47 * Node Index - zero-based index of a node in the client-allocated
48 * contiguous buffer.
49 *
50 * -- Initial State --
51 * We maintain the following to track the state of the free list:
52 *
53 * 1) A "head" index to track the index of the first free node in the list
54 * 2) A "next" array to track the index of the next free node in the list
55 * for every node. So next[head], will give the index to the 2nd free
56 * element in the list.
57 *
58 * So, to begin with, the free list consists of all node indices, and each
59 * position in the next array contains index N + 1:
60 *
61 * head = 0
62 * next = [1, 2, 3, 4, -1] : Example for a user-allocated buffer of 5 nodes
63 * free_list = 0->1->2->3->4->-1
64 *
65 * -- Allocations --
66 * 1) Read the current head (aka acq_head)
67 * 2) Read next[acq_head], to get the 2nd free element (aka new_head)
68 * 3) cmp_xchg(&head, acq_head, new_head)
69 * 4) If it succeeds, compute the address of the node, based on
70 * base address, blk_size, & acq_head.
71 *
72 * head = 1;
73 * next = [1, 2, 3, 4, -1] : Example after allocating Node #0
74 * free_list = 1->2->3->4->-1
75 *
76 * head = 2;
77 * next = [1, 2, 3, 4, -1] : Example after allocating Node #1
78 * free_list = 2->3->4->-1
79 *
80 * -- Frees --
81 * 1) Based on the address to be freed, calculate the index of the node
82 * being freed (cur_idx)
83 * 2) Read the current head (old_head)
84 * 3) So the freed node is going to go at the head of the list, and we
85 * want to put the old_head after it. So next[cur_idx] = old_head
86 * 4) cmpxchg(head, old_head, cur_idx)
87 *
88 * head = 0
89 * next = [2, 2, 3, 4, -1]
90 * free_list = 0->2->3->4->-1 : Example after freeing Node #0
91 *
92 * head = 1
93 * next = [2, 0, 3, 4, -1]
94 * free_list = 1->0->2->3->4->-1 : Example after freeing Node #1
95 */
96
97#ifndef LOCKLESS_ALLOCATOR_PRIV_H
98#define LOCKLESS_ALLOCATOR_PRIV_H
99
100struct nvgpu_allocator;
101
102struct nvgpu_lockless_allocator {
103 struct nvgpu_allocator *owner;
104
105 u64 base; /* Base address of the space. */
106 u64 length; /* Length of the space. */
107 u64 blk_size; /* Size of the structure being allocated */
108 int nr_nodes; /* Number of nodes available for allocation */
109
110 int *next; /* An array holding the next indices per node */
111 int head; /* Current node at the top of the stack */
112
113 u64 flags;
114
115 bool inited;
116
117 /* Statistics */
118 nvgpu_atomic_t nr_allocs;
119};
120
121static inline struct nvgpu_lockless_allocator *lockless_allocator(
122 struct nvgpu_allocator *a)
123{
124 return (struct nvgpu_lockless_allocator *)(a)->priv;
125}
126
127#endif
diff --git a/drivers/gpu/nvgpu/common/mm/mm.c b/drivers/gpu/nvgpu/common/mm/mm.c
new file mode 100644
index 00000000..db87c4c4
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/mm.c
@@ -0,0 +1,450 @@
1/*
2 * Permission is hereby granted, free of charge, to any person obtaining a
3 * copy of this software and associated documentation files (the "Software"),
4 * to deal in the Software without restriction, including without limitation
5 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
6 * and/or sell copies of the Software, and to permit persons to whom the
7 * Software is furnished to do so, subject to the following conditions:
8 *
9 * The above copyright notice and this permission notice shall be included in
10 * all copies or substantial portions of the Software.
11 *
12 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
15 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
16 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
17 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
18 * DEALINGS IN THE SOFTWARE.
19 */
20
21#include <nvgpu/mm.h>
22#include <nvgpu/vm.h>
23#include <nvgpu/dma.h>
24#include <nvgpu/vm_area.h>
25#include <nvgpu/gmmu.h>
26#include <nvgpu/vidmem.h>
27#include <nvgpu/semaphore.h>
28#include <nvgpu/pramin.h>
29#include <nvgpu/enabled.h>
30
31#include "gk20a/gk20a.h"
32
33/*
34 * Attempt to find a reserved memory area to determine PTE size for the passed
35 * mapping. If no reserved area can be found use small pages.
36 */
37enum gmmu_pgsz_gk20a __get_pte_size_fixed_map(struct vm_gk20a *vm,
38 u64 base, u64 size)
39{
40 struct nvgpu_vm_area *vm_area;
41
42 vm_area = nvgpu_vm_area_find(vm, base);
43 if (!vm_area)
44 return gmmu_page_size_small;
45
46 return vm_area->pgsz_idx;
47}
48
49/*
50 * This is for when the address space does not support unified address spaces.
51 */
52static enum gmmu_pgsz_gk20a __get_pte_size_split_addr(struct vm_gk20a *vm,
53 u64 base, u64 size)
54{
55 if (!base) {
56 if (size >= vm->gmmu_page_sizes[gmmu_page_size_big])
57 return gmmu_page_size_big;
58 return gmmu_page_size_small;
59 } else {
60 if (base < __nv_gmmu_va_small_page_limit())
61 return gmmu_page_size_small;
62 else
63 return gmmu_page_size_big;
64 }
65}
66
67/*
68 * This determines the PTE size for a given alloc. Used by both the GVA space
69 * allocator and the mm core code so that agreement can be reached on how to
70 * map allocations.
71 *
72 * The page size of a buffer is this:
73 *
74 * o If the VM doesn't support large pages then obviously small pages
75 * must be used.
76 * o If the base address is non-zero (fixed address map):
77 * - Attempt to find a reserved memory area and use the page size
78 * based on that.
79 * - If no reserved page size is available, default to small pages.
80 * o If the base is zero:
81 * - If the size is larger than or equal to the big page size, use big
82 * pages.
83 * - Otherwise use small pages.
84 */
85enum gmmu_pgsz_gk20a __get_pte_size(struct vm_gk20a *vm, u64 base, u64 size)
86{
87 struct gk20a *g = gk20a_from_vm(vm);
88
89 if (!vm->big_pages)
90 return gmmu_page_size_small;
91
92 if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES))
93 return __get_pte_size_split_addr(vm, base, size);
94
95 if (base)
96 return __get_pte_size_fixed_map(vm, base, size);
97
98 if (size >= vm->gmmu_page_sizes[gmmu_page_size_big])
99 return gmmu_page_size_big;
100 return gmmu_page_size_small;
101}
102
103int nvgpu_mm_suspend(struct gk20a *g)
104{
105 nvgpu_log_info(g, "MM suspend running...");
106
107 nvgpu_vidmem_thread_pause_sync(&g->mm);
108
109 g->ops.mm.cbc_clean(g);
110 g->ops.mm.l2_flush(g, false);
111
112 nvgpu_log_info(g, "MM suspend done!");
113
114 return 0;
115}
116
117u64 nvgpu_inst_block_addr(struct gk20a *g, struct nvgpu_mem *inst_block)
118{
119 if (g->mm.has_physical_mode)
120 return nvgpu_mem_get_phys_addr(g, inst_block);
121 else
122 return nvgpu_mem_get_addr(g, inst_block);
123}
124
125void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block)
126{
127 if (nvgpu_mem_is_valid(inst_block))
128 nvgpu_dma_free(g, inst_block);
129}
130
131static int nvgpu_alloc_sysmem_flush(struct gk20a *g)
132{
133 return nvgpu_dma_alloc_sys(g, SZ_4K, &g->mm.sysmem_flush);
134}
135
136static void nvgpu_remove_mm_ce_support(struct mm_gk20a *mm)
137{
138 struct gk20a *g = gk20a_from_mm(mm);
139
140 if (mm->vidmem.ce_ctx_id != (u32)~0)
141 gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id);
142
143 mm->vidmem.ce_ctx_id = (u32)~0;
144
145 nvgpu_vm_put(mm->ce.vm);
146}
147
148static void nvgpu_remove_mm_support(struct mm_gk20a *mm)
149{
150 struct gk20a *g = gk20a_from_mm(mm);
151
152 if (g->ops.mm.fault_info_mem_destroy)
153 g->ops.mm.fault_info_mem_destroy(g);
154
155 if (g->ops.mm.remove_bar2_vm)
156 g->ops.mm.remove_bar2_vm(g);
157
158 if (g->ops.mm.is_bar1_supported(g)) {
159 nvgpu_free_inst_block(g, &mm->bar1.inst_block);
160 nvgpu_vm_put(mm->bar1.vm);
161 }
162
163 nvgpu_free_inst_block(g, &mm->pmu.inst_block);
164 nvgpu_free_inst_block(g, &mm->hwpm.inst_block);
165 nvgpu_vm_put(mm->pmu.vm);
166 nvgpu_vm_put(mm->cde.vm);
167
168 nvgpu_semaphore_sea_destroy(g);
169 nvgpu_vidmem_destroy(g);
170 nvgpu_pd_cache_fini(g);
171}
172
173/* pmu vm, share channel_vm interfaces */
174static int nvgpu_init_system_vm(struct mm_gk20a *mm)
175{
176 int err;
177 struct gk20a *g = gk20a_from_mm(mm);
178 struct nvgpu_mem *inst_block = &mm->pmu.inst_block;
179 u32 big_page_size = g->ops.mm.get_default_big_page_size();
180 u32 low_hole, aperture_size;
181
182 /*
183 * No user region - so we will pass that as zero sized.
184 */
185 low_hole = SZ_4K * 16;
186 aperture_size = GK20A_PMU_VA_SIZE * 2;
187
188 mm->pmu.aperture_size = GK20A_PMU_VA_SIZE;
189 nvgpu_log_info(g, "pmu vm size = 0x%x", mm->pmu.aperture_size);
190
191 mm->pmu.vm = nvgpu_vm_init(g, big_page_size,
192 low_hole,
193 aperture_size - low_hole,
194 aperture_size,
195 true,
196 false,
197 "system");
198 if (!mm->pmu.vm)
199 return -ENOMEM;
200
201 err = g->ops.mm.alloc_inst_block(g, inst_block);
202 if (err)
203 goto clean_up_vm;
204 g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size);
205
206 return 0;
207
208clean_up_vm:
209 nvgpu_vm_put(mm->pmu.vm);
210 return err;
211}
212
213static int nvgpu_init_hwpm(struct mm_gk20a *mm)
214{
215 int err;
216 struct gk20a *g = gk20a_from_mm(mm);
217 struct nvgpu_mem *inst_block = &mm->hwpm.inst_block;
218
219 err = g->ops.mm.alloc_inst_block(g, inst_block);
220 if (err)
221 return err;
222 g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0);
223
224 return 0;
225}
226
227static int nvgpu_init_cde_vm(struct mm_gk20a *mm)
228{
229 struct gk20a *g = gk20a_from_mm(mm);
230 u32 big_page_size = g->ops.mm.get_default_big_page_size();
231
232 mm->cde.vm = nvgpu_vm_init(g, big_page_size,
233 big_page_size << 10,
234 NV_MM_DEFAULT_KERNEL_SIZE,
235 NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE,
236 false, false, "cde");
237 if (!mm->cde.vm)
238 return -ENOMEM;
239 return 0;
240}
241
242static int nvgpu_init_ce_vm(struct mm_gk20a *mm)
243{
244 struct gk20a *g = gk20a_from_mm(mm);
245 u32 big_page_size = g->ops.mm.get_default_big_page_size();
246
247 mm->ce.vm = nvgpu_vm_init(g, big_page_size,
248 big_page_size << 10,
249 NV_MM_DEFAULT_KERNEL_SIZE,
250 NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE,
251 false, false, "ce");
252 if (!mm->ce.vm)
253 return -ENOMEM;
254 return 0;
255}
256
257void nvgpu_init_mm_ce_context(struct gk20a *g)
258{
259#if defined(CONFIG_GK20A_VIDMEM)
260 if (g->mm.vidmem.size && (g->mm.vidmem.ce_ctx_id == (u32)~0)) {
261 g->mm.vidmem.ce_ctx_id =
262 gk20a_ce_create_context(g,
263 gk20a_fifo_get_fast_ce_runlist_id(g),
264 -1,
265 -1);
266
267 if (g->mm.vidmem.ce_ctx_id == (u32)~0)
268 nvgpu_err(g,
269 "Failed to allocate CE context for vidmem page clearing support");
270 }
271#endif
272}
273
274static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g)
275{
276 if (g->ops.fb.reset)
277 g->ops.fb.reset(g);
278
279 if (g->ops.clock_gating.slcg_fb_load_gating_prod)
280 g->ops.clock_gating.slcg_fb_load_gating_prod(g,
281 g->slcg_enabled);
282 if (g->ops.clock_gating.slcg_ltc_load_gating_prod)
283 g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
284 g->slcg_enabled);
285 if (g->ops.clock_gating.blcg_fb_load_gating_prod)
286 g->ops.clock_gating.blcg_fb_load_gating_prod(g,
287 g->blcg_enabled);
288 if (g->ops.clock_gating.blcg_ltc_load_gating_prod)
289 g->ops.clock_gating.blcg_ltc_load_gating_prod(g,
290 g->blcg_enabled);
291
292 if (g->ops.fb.init_fs_state)
293 g->ops.fb.init_fs_state(g);
294
295 return 0;
296}
297
298static int nvgpu_init_bar1_vm(struct mm_gk20a *mm)
299{
300 int err;
301 struct gk20a *g = gk20a_from_mm(mm);
302 struct nvgpu_mem *inst_block = &mm->bar1.inst_block;
303 u32 big_page_size = g->ops.mm.get_default_big_page_size();
304
305 mm->bar1.aperture_size = bar1_aperture_size_mb_gk20a() << 20;
306 nvgpu_log_info(g, "bar1 vm size = 0x%x", mm->bar1.aperture_size);
307 mm->bar1.vm = nvgpu_vm_init(g,
308 big_page_size,
309 SZ_4K,
310 mm->bar1.aperture_size - SZ_4K,
311 mm->bar1.aperture_size,
312 true, false,
313 "bar1");
314 if (!mm->bar1.vm)
315 return -ENOMEM;
316
317 err = g->ops.mm.alloc_inst_block(g, inst_block);
318 if (err)
319 goto clean_up_vm;
320 g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size);
321
322 return 0;
323
324clean_up_vm:
325 nvgpu_vm_put(mm->bar1.vm);
326 return err;
327}
328
329static int nvgpu_init_mm_setup_sw(struct gk20a *g)
330{
331 struct mm_gk20a *mm = &g->mm;
332 int err;
333
334 if (mm->sw_ready) {
335 nvgpu_log_info(g, "skip init");
336 return 0;
337 }
338
339 mm->g = g;
340 nvgpu_mutex_init(&mm->l2_op_lock);
341
342 /*TBD: make channel vm size configurable */
343 mm->channel.user_size = NV_MM_DEFAULT_USER_SIZE -
344 NV_MM_DEFAULT_KERNEL_SIZE;
345 mm->channel.kernel_size = NV_MM_DEFAULT_KERNEL_SIZE;
346
347 nvgpu_log_info(g, "channel vm size: user %dMB kernel %dMB",
348 (int)(mm->channel.user_size >> 20),
349 (int)(mm->channel.kernel_size >> 20));
350
351 nvgpu_init_pramin(mm);
352
353 mm->vidmem.ce_ctx_id = (u32)~0;
354
355 err = nvgpu_vidmem_init(mm);
356 if (err)
357 return err;
358
359 /*
360 * this requires fixed allocations in vidmem which must be
361 * allocated before all other buffers
362 */
363 if (g->ops.pmu.alloc_blob_space
364 && !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) {
365 err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob);
366 if (err)
367 return err;
368 }
369
370 err = nvgpu_alloc_sysmem_flush(g);
371 if (err)
372 return err;
373
374 if (g->ops.mm.is_bar1_supported(g)) {
375 err = nvgpu_init_bar1_vm(mm);
376 if (err)
377 return err;
378 }
379 if (g->ops.mm.init_bar2_vm) {
380 err = g->ops.mm.init_bar2_vm(g);
381 if (err)
382 return err;
383 }
384 err = nvgpu_init_system_vm(mm);
385 if (err)
386 return err;
387
388 err = nvgpu_init_hwpm(mm);
389 if (err)
390 return err;
391
392 err = nvgpu_init_cde_vm(mm);
393 if (err)
394 return err;
395
396 err = nvgpu_init_ce_vm(mm);
397 if (err)
398 return err;
399
400 mm->remove_support = nvgpu_remove_mm_support;
401 mm->remove_ce_support = nvgpu_remove_mm_ce_support;
402
403 mm->sw_ready = true;
404
405 return 0;
406}
407
408int nvgpu_init_mm_support(struct gk20a *g)
409{
410 u32 err;
411
412 err = nvgpu_init_mm_reset_enable_hw(g);
413 if (err)
414 return err;
415
416 err = nvgpu_init_mm_setup_sw(g);
417 if (err)
418 return err;
419
420 if (g->ops.mm.init_mm_setup_hw)
421 err = g->ops.mm.init_mm_setup_hw(g);
422
423 return err;
424}
425
426u32 nvgpu_mm_get_default_big_page_size(struct gk20a *g)
427{
428 u32 big_page_size;
429
430 big_page_size = g->ops.mm.get_default_big_page_size();
431
432 if (g->mm.disable_bigpage)
433 big_page_size = 0;
434
435 return big_page_size;
436}
437
438u32 nvgpu_mm_get_available_big_page_sizes(struct gk20a *g)
439{
440 u32 available_big_page_sizes = 0;
441
442 if (!g->mm.disable_bigpage) {
443 available_big_page_sizes =
444 g->ops.mm.get_default_big_page_size();
445 if (g->ops.mm.get_big_page_sizes)
446 available_big_page_sizes |= g->ops.mm.get_big_page_sizes();
447 }
448
449 return available_big_page_sizes;
450}
diff --git a/drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c b/drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c
new file mode 100644
index 00000000..7a4e7705
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c
@@ -0,0 +1,162 @@
1/*
2 * gk20a allocator
3 *
4 * Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <nvgpu/allocator.h>
26
27#include "gk20a/gk20a.h"
28#include "gk20a/mm_gk20a.h"
29
30u64 nvgpu_alloc_length(struct nvgpu_allocator *a)
31{
32 if (a->ops->length)
33 return a->ops->length(a);
34
35 return 0;
36}
37
38u64 nvgpu_alloc_base(struct nvgpu_allocator *a)
39{
40 if (a->ops->base)
41 return a->ops->base(a);
42
43 return 0;
44}
45
46u64 nvgpu_alloc_initialized(struct nvgpu_allocator *a)
47{
48 if (!a->ops || !a->ops->inited)
49 return 0;
50
51 return a->ops->inited(a);
52}
53
54u64 nvgpu_alloc_end(struct nvgpu_allocator *a)
55{
56 if (a->ops->end)
57 return a->ops->end(a);
58
59 return 0;
60}
61
62u64 nvgpu_alloc_space(struct nvgpu_allocator *a)
63{
64 if (a->ops->space)
65 return a->ops->space(a);
66
67 return 0;
68}
69
70u64 nvgpu_alloc(struct nvgpu_allocator *a, u64 len)
71{
72 return a->ops->alloc(a, len);
73}
74
75void nvgpu_free(struct nvgpu_allocator *a, u64 addr)
76{
77 a->ops->free(a, addr);
78}
79
80u64 nvgpu_alloc_fixed(struct nvgpu_allocator *a, u64 base, u64 len,
81 u32 page_size)
82{
83 if (a->ops->alloc_fixed)
84 return a->ops->alloc_fixed(a, base, len, page_size);
85
86 return 0;
87}
88
89void nvgpu_free_fixed(struct nvgpu_allocator *a, u64 base, u64 len)
90{
91 /*
92 * If this operation is not defined for the allocator then just do
93 * nothing. The alternative would be to fall back on the regular
94 * free but that may be harmful in unexpected ways.
95 */
96 if (a->ops->free_fixed)
97 a->ops->free_fixed(a, base, len);
98}
99
100int nvgpu_alloc_reserve_carveout(struct nvgpu_allocator *a,
101 struct nvgpu_alloc_carveout *co)
102{
103 if (a->ops->reserve_carveout)
104 return a->ops->reserve_carveout(a, co);
105
106 return -ENODEV;
107}
108
109void nvgpu_alloc_release_carveout(struct nvgpu_allocator *a,
110 struct nvgpu_alloc_carveout *co)
111{
112 if (a->ops->release_carveout)
113 a->ops->release_carveout(a, co);
114}
115
116void nvgpu_alloc_destroy(struct nvgpu_allocator *a)
117{
118 a->ops->fini(a);
119 nvgpu_mutex_destroy(&a->lock);
120 memset(a, 0, sizeof(*a));
121}
122
123#ifdef __KERNEL__
124void nvgpu_alloc_print_stats(struct nvgpu_allocator *__a,
125 struct seq_file *s, int lock)
126{
127 __a->ops->print_stats(__a, s, lock);
128}
129#endif
130
131/*
132 * Handle the common init stuff for a nvgpu_allocator.
133 */
134int __nvgpu_alloc_common_init(struct nvgpu_allocator *a, struct gk20a *g,
135 const char *name, void *priv, bool dbg,
136 const struct nvgpu_allocator_ops *ops)
137{
138 int err;
139
140 if (!ops)
141 return -EINVAL;
142
143 /*
144 * This is the bare minimum operations required for a sensible
145 * allocator.
146 */
147 if (!ops->alloc || !ops->free || !ops->fini)
148 return -EINVAL;
149
150 err = nvgpu_mutex_init(&a->lock);
151 if (err)
152 return err;
153
154 a->g = g;
155 a->ops = ops;
156 a->priv = priv;
157 a->debug = dbg;
158
159 strlcpy(a->name, name, sizeof(a->name));
160
161 return 0;
162}
diff --git a/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
new file mode 100644
index 00000000..b4e718b4
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
@@ -0,0 +1,119 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/kmem.h>
24#include <nvgpu/nvgpu_mem.h>
25#include <nvgpu/dma.h>
26#include <nvgpu/vidmem.h>
27
28#include "gk20a/gk20a.h"
29
30void *nvgpu_sgt_get_next(struct nvgpu_sgt *sgt, void *sgl)
31{
32 return sgt->ops->sgl_next(sgl);
33}
34
35u64 nvgpu_sgt_get_phys(struct nvgpu_sgt *sgt, void *sgl)
36{
37 return sgt->ops->sgl_phys(sgl);
38}
39
40u64 nvgpu_sgt_get_dma(struct nvgpu_sgt *sgt, void *sgl)
41{
42 return sgt->ops->sgl_dma(sgl);
43}
44
45u64 nvgpu_sgt_get_length(struct nvgpu_sgt *sgt, void *sgl)
46{
47 return sgt->ops->sgl_length(sgl);
48}
49
50u64 nvgpu_sgt_get_gpu_addr(struct gk20a *g, struct nvgpu_sgt *sgt, void *sgl,
51 struct nvgpu_gmmu_attrs *attrs)
52{
53 return sgt->ops->sgl_gpu_addr(g, sgl, attrs);
54}
55
56bool nvgpu_sgt_iommuable(struct gk20a *g, struct nvgpu_sgt *sgt)
57{
58 if (sgt->ops->sgt_iommuable)
59 return sgt->ops->sgt_iommuable(g, sgt);
60 return false;
61}
62
63void nvgpu_sgt_free(struct gk20a *g, struct nvgpu_sgt *sgt)
64{
65 if (sgt && sgt->ops->sgt_free)
66 sgt->ops->sgt_free(g, sgt);
67}
68
69u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys)
70{
71 /* ensure it is not vidmem allocation */
72 WARN_ON(nvgpu_addr_is_vidmem_page_alloc(phys));
73
74 if (nvgpu_iommuable(g) && g->ops.mm.get_iommu_bit)
75 return phys | 1ULL << g->ops.mm.get_iommu_bit(g);
76
77 return phys;
78}
79
80/*
81 * Determine alignment for a passed buffer. Necessary since the buffer may
82 * appear big enough to map with large pages but the SGL may have chunks that
83 * are not aligned on a 64/128kB large page boundary. There's also the
84 * possibility chunks are odd sizes which will necessitate small page mappings
85 * to correctly glue them together into a contiguous virtual mapping.
86 */
87u64 nvgpu_sgt_alignment(struct gk20a *g, struct nvgpu_sgt *sgt)
88{
89 u64 align = 0, chunk_align = 0;
90 void *sgl;
91
92 /*
93 * If this SGT is iommuable and we want to use the IOMMU address then
94 * the SGT's first entry has the IOMMU address. We will align on this
95 * and double check length of buffer later. Also, since there's an
96 * IOMMU we know that this DMA address is contiguous.
97 */
98 if (!g->mm.bypass_smmu &&
99 nvgpu_sgt_iommuable(g, sgt) &&
100 nvgpu_sgt_get_dma(sgt, sgt->sgl))
101 return 1ULL << __ffs(nvgpu_sgt_get_dma(sgt, sgt->sgl));
102
103 /*
104 * Otherwise the buffer is not iommuable (VIDMEM, for example) or we are
105 * bypassing the IOMMU and need to use the underlying physical entries
106 * of the SGT.
107 */
108 nvgpu_sgt_for_each_sgl(sgl, sgt) {
109 chunk_align = 1ULL << __ffs(nvgpu_sgt_get_phys(sgt, sgl) |
110 nvgpu_sgt_get_length(sgt, sgl));
111
112 if (align)
113 align = min(align, chunk_align);
114 else
115 align = chunk_align;
116 }
117
118 return align;
119}
diff --git a/drivers/gpu/nvgpu/common/mm/page_allocator.c b/drivers/gpu/nvgpu/common/mm/page_allocator.c
new file mode 100644
index 00000000..d5ce5d8e
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/page_allocator.c
@@ -0,0 +1,1047 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/bitops.h>
24#include <nvgpu/allocator.h>
25#include <nvgpu/page_allocator.h>
26#include <nvgpu/kmem.h>
27#include <nvgpu/bug.h>
28#include <nvgpu/log2.h>
29
30#include "buddy_allocator_priv.h"
31
32#define palloc_dbg(a, fmt, arg...) \
33 alloc_dbg(palloc_owner(a), fmt, ##arg)
34
35/*
36 * Since some Linux headers are still leaked into common code this is necessary
37 * for some builds.
38 */
39#ifdef PAGE_SIZE
40#undef PAGE_SIZE
41#endif
42
43#ifdef PAGE_ALIGN
44#undef PAGE_ALIGN
45#endif
46
47/*
48 * VIDMEM page size is 4k.
49 */
50#define PAGE_SIZE 0x1000
51#define PAGE_ALIGN(addr) ((addr + (PAGE_SIZE - 1)) & \
52 ((typeof(addr)) ~(PAGE_SIZE - 1)))
53
54/*
55 * Handle the book-keeping for these operations.
56 */
57static inline void add_slab_page_to_empty(struct page_alloc_slab *slab,
58 struct page_alloc_slab_page *page)
59{
60 BUG_ON(page->state != SP_NONE);
61 nvgpu_list_add(&page->list_entry, &slab->empty);
62 slab->nr_empty++;
63 page->state = SP_EMPTY;
64}
65static inline void add_slab_page_to_partial(struct page_alloc_slab *slab,
66 struct page_alloc_slab_page *page)
67{
68 BUG_ON(page->state != SP_NONE);
69 nvgpu_list_add(&page->list_entry, &slab->partial);
70 slab->nr_partial++;
71 page->state = SP_PARTIAL;
72}
73static inline void add_slab_page_to_full(struct page_alloc_slab *slab,
74 struct page_alloc_slab_page *page)
75{
76 BUG_ON(page->state != SP_NONE);
77 nvgpu_list_add(&page->list_entry, &slab->full);
78 slab->nr_full++;
79 page->state = SP_FULL;
80}
81
82static inline void del_slab_page_from_empty(struct page_alloc_slab *slab,
83 struct page_alloc_slab_page *page)
84{
85 nvgpu_list_del(&page->list_entry);
86 slab->nr_empty--;
87 page->state = SP_NONE;
88}
89static inline void del_slab_page_from_partial(struct page_alloc_slab *slab,
90 struct page_alloc_slab_page *page)
91{
92 nvgpu_list_del(&page->list_entry);
93 slab->nr_partial--;
94 page->state = SP_NONE;
95}
96static inline void del_slab_page_from_full(struct page_alloc_slab *slab,
97 struct page_alloc_slab_page *page)
98{
99 nvgpu_list_del(&page->list_entry);
100 slab->nr_full--;
101 page->state = SP_NONE;
102}
103
104static u64 nvgpu_page_alloc_length(struct nvgpu_allocator *a)
105{
106 struct nvgpu_page_allocator *va = a->priv;
107
108 return nvgpu_alloc_length(&va->source_allocator);
109}
110
111static u64 nvgpu_page_alloc_base(struct nvgpu_allocator *a)
112{
113 struct nvgpu_page_allocator *va = a->priv;
114
115 return nvgpu_alloc_base(&va->source_allocator);
116}
117
118static int nvgpu_page_alloc_inited(struct nvgpu_allocator *a)
119{
120 struct nvgpu_page_allocator *va = a->priv;
121
122 return nvgpu_alloc_initialized(&va->source_allocator);
123}
124
125static u64 nvgpu_page_alloc_end(struct nvgpu_allocator *a)
126{
127 struct nvgpu_page_allocator *va = a->priv;
128
129 return nvgpu_alloc_end(&va->source_allocator);
130}
131
132static u64 nvgpu_page_alloc_space(struct nvgpu_allocator *a)
133{
134 struct nvgpu_page_allocator *va = a->priv;
135
136 return nvgpu_alloc_space(&va->source_allocator);
137}
138
139static int nvgpu_page_reserve_co(struct nvgpu_allocator *a,
140 struct nvgpu_alloc_carveout *co)
141{
142 struct nvgpu_page_allocator *va = a->priv;
143
144 return nvgpu_alloc_reserve_carveout(&va->source_allocator, co);
145}
146
147static void nvgpu_page_release_co(struct nvgpu_allocator *a,
148 struct nvgpu_alloc_carveout *co)
149{
150 struct nvgpu_page_allocator *va = a->priv;
151
152 nvgpu_alloc_release_carveout(&va->source_allocator, co);
153}
154
155static void *nvgpu_page_alloc_sgl_next(void *sgl)
156{
157 struct nvgpu_mem_sgl *nvgpu_sgl = sgl;
158
159 return nvgpu_sgl->next;
160}
161
162static u64 nvgpu_page_alloc_sgl_phys(void *sgl)
163{
164 struct nvgpu_mem_sgl *nvgpu_sgl = sgl;
165
166 return nvgpu_sgl->phys;
167}
168
169static u64 nvgpu_page_alloc_sgl_dma(void *sgl)
170{
171 struct nvgpu_mem_sgl *nvgpu_sgl = sgl;
172
173 return nvgpu_sgl->dma;
174}
175
176static u64 nvgpu_page_alloc_sgl_length(void *sgl)
177{
178 struct nvgpu_mem_sgl *nvgpu_sgl = sgl;
179
180 return nvgpu_sgl->length;
181}
182
183static u64 nvgpu_page_alloc_sgl_gpu_addr(struct gk20a *g, void *sgl,
184 struct nvgpu_gmmu_attrs *attrs)
185{
186 struct nvgpu_mem_sgl *nvgpu_sgl = sgl;
187
188 return nvgpu_sgl->phys;
189}
190
191static void nvgpu_page_alloc_sgt_free(struct gk20a *g, struct nvgpu_sgt *sgt)
192{
193 /*
194 * No-op here. The free is handled by the page_alloc free() functions.
195 */
196}
197
198/*
199 * These implement the generic scatter gather ops for pages allocated
200 * by the page allocator. however, the primary aim for this, is of course,
201 * vidmem.
202 */
203static const struct nvgpu_sgt_ops page_alloc_sgl_ops = {
204 .sgl_next = nvgpu_page_alloc_sgl_next,
205 .sgl_phys = nvgpu_page_alloc_sgl_phys,
206 .sgl_dma = nvgpu_page_alloc_sgl_dma,
207 .sgl_length = nvgpu_page_alloc_sgl_length,
208 .sgl_gpu_addr = nvgpu_page_alloc_sgl_gpu_addr,
209 .sgt_free = nvgpu_page_alloc_sgt_free,
210};
211
212/*
213 * This actually frees the sgl memory. Used by the page_alloc free() functions.
214 */
215static void nvgpu_page_alloc_sgl_proper_free(struct gk20a *g,
216 struct nvgpu_mem_sgl *sgl)
217{
218 struct nvgpu_mem_sgl *next;
219
220 while (sgl) {
221 next = sgl->next;
222 nvgpu_kfree(g, sgl);
223 sgl = next;
224 }
225}
226
227static void __nvgpu_free_pages(struct nvgpu_page_allocator *a,
228 struct nvgpu_page_alloc *alloc,
229 bool free_buddy_alloc)
230{
231 struct nvgpu_mem_sgl *sgl = alloc->sgt.sgl;
232
233 if (free_buddy_alloc) {
234 while (sgl) {
235 nvgpu_free(&a->source_allocator,
236 nvgpu_sgt_get_phys(&alloc->sgt, sgl));
237 sgl = nvgpu_sgt_get_next(&alloc->sgt, sgl);
238 }
239 }
240
241 nvgpu_page_alloc_sgl_proper_free(a->owner->g, sgl);
242 nvgpu_kmem_cache_free(a->alloc_cache, alloc);
243}
244
245static int __insert_page_alloc(struct nvgpu_page_allocator *a,
246 struct nvgpu_page_alloc *alloc)
247{
248 alloc->tree_entry.key_start = alloc->base;
249 alloc->tree_entry.key_end = alloc->base + alloc->length;
250
251 nvgpu_rbtree_insert(&alloc->tree_entry, &a->allocs);
252 return 0;
253}
254
255static struct nvgpu_page_alloc *__find_page_alloc(
256 struct nvgpu_page_allocator *a,
257 u64 addr)
258{
259 struct nvgpu_page_alloc *alloc;
260 struct nvgpu_rbtree_node *node = NULL;
261
262 nvgpu_rbtree_search(addr, &node, a->allocs);
263 if (!node)
264 return NULL;
265
266 alloc = nvgpu_page_alloc_from_rbtree_node(node);
267
268 nvgpu_rbtree_unlink(node, &a->allocs);
269
270 return alloc;
271}
272
273static struct page_alloc_slab_page *alloc_slab_page(
274 struct nvgpu_page_allocator *a,
275 struct page_alloc_slab *slab)
276{
277 struct page_alloc_slab_page *slab_page;
278
279 slab_page = nvgpu_kmem_cache_alloc(a->slab_page_cache);
280 if (!slab_page) {
281 palloc_dbg(a, "OOM: unable to alloc slab_page struct!\n");
282 return NULL;
283 }
284
285 memset(slab_page, 0, sizeof(*slab_page));
286
287 slab_page->page_addr = nvgpu_alloc(&a->source_allocator, a->page_size);
288 if (!slab_page->page_addr) {
289 nvgpu_kmem_cache_free(a->slab_page_cache, slab_page);
290 palloc_dbg(a, "OOM: vidmem is full!\n");
291 return NULL;
292 }
293
294 nvgpu_init_list_node(&slab_page->list_entry);
295 slab_page->slab_size = slab->slab_size;
296 slab_page->nr_objects = (u32)a->page_size / slab->slab_size;
297 slab_page->nr_objects_alloced = 0;
298 slab_page->owner = slab;
299 slab_page->state = SP_NONE;
300
301 a->pages_alloced++;
302
303 palloc_dbg(a, "Allocated new slab page @ 0x%012llx size=%u\n",
304 slab_page->page_addr, slab_page->slab_size);
305
306 return slab_page;
307}
308
309static void free_slab_page(struct nvgpu_page_allocator *a,
310 struct page_alloc_slab_page *slab_page)
311{
312 palloc_dbg(a, "Freeing slab page @ 0x%012llx\n", slab_page->page_addr);
313
314 BUG_ON((slab_page->state != SP_NONE && slab_page->state != SP_EMPTY) ||
315 slab_page->nr_objects_alloced != 0 ||
316 slab_page->bitmap != 0);
317
318 nvgpu_free(&a->source_allocator, slab_page->page_addr);
319 a->pages_freed++;
320
321 nvgpu_kmem_cache_free(a->slab_page_cache, slab_page);
322}
323
324/*
325 * This expects @alloc to have 1 empty sgl_entry ready for usage.
326 */
327static int __do_slab_alloc(struct nvgpu_page_allocator *a,
328 struct page_alloc_slab *slab,
329 struct nvgpu_page_alloc *alloc)
330{
331 struct page_alloc_slab_page *slab_page = NULL;
332 struct nvgpu_mem_sgl *sgl;
333 unsigned long offs;
334
335 /*
336 * Check the partial and empty lists to see if we have some space
337 * readily available. Take the slab_page out of what ever list it
338 * was in since it may be put back into a different list later.
339 */
340 if (!nvgpu_list_empty(&slab->partial)) {
341 slab_page = nvgpu_list_first_entry(&slab->partial,
342 page_alloc_slab_page,
343 list_entry);
344 del_slab_page_from_partial(slab, slab_page);
345 } else if (!nvgpu_list_empty(&slab->empty)) {
346 slab_page = nvgpu_list_first_entry(&slab->empty,
347 page_alloc_slab_page,
348 list_entry);
349 del_slab_page_from_empty(slab, slab_page);
350 }
351
352 if (!slab_page) {
353 slab_page = alloc_slab_page(a, slab);
354 if (!slab_page)
355 return -ENOMEM;
356 }
357
358 /*
359 * We now have a slab_page. Do the alloc.
360 */
361 offs = bitmap_find_next_zero_area(&slab_page->bitmap,
362 slab_page->nr_objects,
363 0, 1, 0);
364 if (offs >= slab_page->nr_objects) {
365 WARN(1, "Empty/partial slab with no free objects?");
366
367 /* Add the buggy page to the full list... This isn't ideal. */
368 add_slab_page_to_full(slab, slab_page);
369 return -ENOMEM;
370 }
371
372 bitmap_set(&slab_page->bitmap, offs, 1);
373 slab_page->nr_objects_alloced++;
374
375 if (slab_page->nr_objects_alloced < slab_page->nr_objects)
376 add_slab_page_to_partial(slab, slab_page);
377 else if (slab_page->nr_objects_alloced == slab_page->nr_objects)
378 add_slab_page_to_full(slab, slab_page);
379 else
380 BUG(); /* Should be impossible to hit this. */
381
382 /*
383 * Handle building the nvgpu_page_alloc struct. We expect one sgl
384 * to be present.
385 */
386 alloc->slab_page = slab_page;
387 alloc->nr_chunks = 1;
388 alloc->length = slab_page->slab_size;
389 alloc->base = slab_page->page_addr + (offs * slab_page->slab_size);
390
391 sgl = alloc->sgt.sgl;
392 sgl->phys = alloc->base;
393 sgl->dma = alloc->base;
394 sgl->length = alloc->length;
395 sgl->next = NULL;
396
397 return 0;
398}
399
400/*
401 * Allocate from a slab instead of directly from the page allocator.
402 */
403static struct nvgpu_page_alloc *__nvgpu_alloc_slab(
404 struct nvgpu_page_allocator *a, u64 len)
405{
406 int err, slab_nr;
407 struct page_alloc_slab *slab;
408 struct nvgpu_page_alloc *alloc = NULL;
409 struct nvgpu_mem_sgl *sgl = NULL;
410
411 /*
412 * Align the length to a page and then divide by the page size (4k for
413 * this code). ilog2() of that then gets us the correct slab to use.
414 */
415 slab_nr = (int)ilog2(PAGE_ALIGN(len) >> 12);
416 slab = &a->slabs[slab_nr];
417
418 alloc = nvgpu_kmem_cache_alloc(a->alloc_cache);
419 if (!alloc) {
420 palloc_dbg(a, "OOM: could not alloc page_alloc struct!\n");
421 goto fail;
422 }
423
424 alloc->sgt.ops = &page_alloc_sgl_ops;
425
426 sgl = nvgpu_kzalloc(a->owner->g, sizeof(*sgl));
427 if (!sgl) {
428 palloc_dbg(a, "OOM: could not alloc sgl struct!\n");
429 goto fail;
430 }
431
432 alloc->sgt.sgl = sgl;
433 err = __do_slab_alloc(a, slab, alloc);
434 if (err)
435 goto fail;
436
437 palloc_dbg(a, "Alloc 0x%04llx sr=%d id=0x%010llx [slab]\n",
438 len, slab_nr, alloc->base);
439 a->nr_slab_allocs++;
440
441 return alloc;
442
443fail:
444 if (alloc)
445 nvgpu_kmem_cache_free(a->alloc_cache, alloc);
446 if (sgl)
447 nvgpu_kfree(a->owner->g, sgl);
448 return NULL;
449}
450
451static void __nvgpu_free_slab(struct nvgpu_page_allocator *a,
452 struct nvgpu_page_alloc *alloc)
453{
454 struct page_alloc_slab_page *slab_page = alloc->slab_page;
455 struct page_alloc_slab *slab = slab_page->owner;
456 enum slab_page_state new_state;
457 int offs;
458
459 offs = (u32)(alloc->base - slab_page->page_addr) / slab_page->slab_size;
460 bitmap_clear(&slab_page->bitmap, offs, 1);
461
462 slab_page->nr_objects_alloced--;
463
464 if (slab_page->nr_objects_alloced == 0)
465 new_state = SP_EMPTY;
466 else
467 new_state = SP_PARTIAL;
468
469 /*
470 * Need to migrate the page to a different list.
471 */
472 if (new_state != slab_page->state) {
473 /* Delete - can't be in empty. */
474 if (slab_page->state == SP_PARTIAL)
475 del_slab_page_from_partial(slab, slab_page);
476 else
477 del_slab_page_from_full(slab, slab_page);
478
479 /* And add. */
480 if (new_state == SP_EMPTY) {
481 if (nvgpu_list_empty(&slab->empty))
482 add_slab_page_to_empty(slab, slab_page);
483 else
484 free_slab_page(a, slab_page);
485 } else {
486 add_slab_page_to_partial(slab, slab_page);
487 }
488 }
489
490 /*
491 * Now handle the page_alloc.
492 */
493 __nvgpu_free_pages(a, alloc, false);
494 a->nr_slab_frees++;
495
496 return;
497}
498
499/*
500 * Allocate physical pages. Since the underlying allocator is a buddy allocator
501 * the returned pages are always contiguous. However, since there could be
502 * fragmentation in the space this allocator will collate smaller non-contiguous
503 * allocations together if necessary.
504 */
505static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages(
506 struct nvgpu_page_allocator *a, u64 pages)
507{
508 struct nvgpu_page_alloc *alloc;
509 struct nvgpu_mem_sgl *sgl, *prev_sgl = NULL;
510 u64 max_chunk_len = pages << a->page_shift;
511 int i = 0;
512
513 alloc = nvgpu_kmem_cache_alloc(a->alloc_cache);
514 if (!alloc)
515 goto fail;
516
517 memset(alloc, 0, sizeof(*alloc));
518
519 alloc->length = pages << a->page_shift;
520 alloc->sgt.ops = &page_alloc_sgl_ops;
521
522 while (pages) {
523 u64 chunk_addr = 0;
524 u64 chunk_pages = (u64)1 << __fls(pages);
525 u64 chunk_len = chunk_pages << a->page_shift;
526
527 /*
528 * Take care of the possibility that the allocation must be
529 * contiguous. If this is not the first iteration then that
530 * means the first iteration failed to alloc the entire
531 * requested size. The buddy allocator guarantees any given
532 * single alloc is contiguous.
533 */
534 if (a->flags & GPU_ALLOC_FORCE_CONTIG && i != 0)
535 goto fail_cleanup;
536
537 if (chunk_len > max_chunk_len)
538 chunk_len = max_chunk_len;
539
540 /*
541 * Keep attempting to allocate in smaller chunks until the alloc
542 * either succeeds or is smaller than the page_size of the
543 * allocator (i.e the allocator is OOM).
544 */
545 do {
546 chunk_addr = nvgpu_alloc(&a->source_allocator,
547 chunk_len);
548
549 /* Divide by 2 and try again */
550 if (!chunk_addr) {
551 palloc_dbg(a, "balloc failed: 0x%llx\n",
552 chunk_len);
553 chunk_len >>= 1;
554 max_chunk_len = chunk_len;
555 }
556 } while (!chunk_addr && chunk_len >= a->page_size);
557
558 chunk_pages = chunk_len >> a->page_shift;
559
560 if (!chunk_addr) {
561 palloc_dbg(a, "bailing @ 0x%llx\n", chunk_len);
562 goto fail_cleanup;
563 }
564
565 sgl = nvgpu_kzalloc(a->owner->g, sizeof(*sgl));
566 if (!sgl) {
567 nvgpu_free(&a->source_allocator, chunk_addr);
568 goto fail_cleanup;
569 }
570
571 pages -= chunk_pages;
572
573 sgl->phys = chunk_addr;
574 sgl->dma = chunk_addr;
575 sgl->length = chunk_len;
576
577 /*
578 * Build the singly linked list with a head node that is part of
579 * the list.
580 */
581 if (prev_sgl)
582 prev_sgl->next = sgl;
583 else
584 alloc->sgt.sgl = sgl;
585
586 prev_sgl = sgl;
587
588 i++;
589 }
590
591 alloc->nr_chunks = i;
592 alloc->base = ((struct nvgpu_mem_sgl *)alloc->sgt.sgl)->phys;
593
594 return alloc;
595
596fail_cleanup:
597 sgl = alloc->sgt.sgl;
598 while (sgl) {
599 struct nvgpu_mem_sgl *next = sgl->next;
600
601 nvgpu_free(&a->source_allocator, sgl->phys);
602 nvgpu_kfree(a->owner->g, sgl);
603
604 sgl = next;
605 }
606
607 nvgpu_kmem_cache_free(a->alloc_cache, alloc);
608fail:
609 return NULL;
610}
611
612static struct nvgpu_page_alloc *__nvgpu_alloc_pages(
613 struct nvgpu_page_allocator *a, u64 len)
614{
615 struct nvgpu_page_alloc *alloc = NULL;
616 struct nvgpu_mem_sgl *sgl;
617 u64 pages;
618 int i = 0;
619
620 pages = ALIGN(len, a->page_size) >> a->page_shift;
621
622 alloc = __do_nvgpu_alloc_pages(a, pages);
623 if (!alloc) {
624 palloc_dbg(a, "Alloc 0x%llx (%llu) (failed)\n",
625 pages << a->page_shift, pages);
626 return NULL;
627 }
628
629 palloc_dbg(a, "Alloc 0x%llx (%llu) id=0x%010llx\n",
630 pages << a->page_shift, pages, alloc->base);
631 sgl = alloc->sgt.sgl;
632 while (sgl) {
633 palloc_dbg(a, " Chunk %2d: 0x%010llx + 0x%llx\n",
634 i++,
635 nvgpu_sgt_get_phys(&alloc->sgt, sgl),
636 nvgpu_sgt_get_length(&alloc->sgt, sgl));
637 sgl = nvgpu_sgt_get_next(&alloc->sgt, sgl);
638 }
639 palloc_dbg(a, "Alloc done\n");
640
641 return alloc;
642}
643
644/*
645 * Allocate enough pages to satisfy @len. Page size is determined at
646 * initialization of the allocator.
647 *
648 * The return is actually a pointer to a struct nvgpu_page_alloc pointer. This
649 * is because it doesn't make a lot of sense to return the address of the first
650 * page in the list of pages (since they could be discontiguous). This has
651 * precedent in the dma_alloc APIs, though, it's really just an annoying
652 * artifact of the fact that the nvgpu_alloc() API requires a u64 return type.
653 */
654static u64 nvgpu_page_alloc(struct nvgpu_allocator *__a, u64 len)
655{
656 struct nvgpu_page_allocator *a = page_allocator(__a);
657 struct nvgpu_page_alloc *alloc = NULL;
658 u64 real_len;
659
660 /*
661 * If we want contig pages we have to round up to a power of two. It's
662 * easier to do that here than in the buddy allocator.
663 */
664 real_len = a->flags & GPU_ALLOC_FORCE_CONTIG ?
665 roundup_pow_of_two(len) : len;
666
667 alloc_lock(__a);
668 if (a->flags & GPU_ALLOC_4K_VIDMEM_PAGES &&
669 real_len <= (a->page_size / 2))
670 alloc = __nvgpu_alloc_slab(a, real_len);
671 else
672 alloc = __nvgpu_alloc_pages(a, real_len);
673
674 if (!alloc) {
675 alloc_unlock(__a);
676 return 0;
677 }
678
679 __insert_page_alloc(a, alloc);
680
681 a->nr_allocs++;
682 if (real_len > a->page_size / 2)
683 a->pages_alloced += alloc->length >> a->page_shift;
684 alloc_unlock(__a);
685
686 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER)
687 return alloc->base;
688 else
689 return (u64) (uintptr_t) alloc;
690}
691
692/*
693 * Note: this will remove the nvgpu_page_alloc struct from the RB tree
694 * if it's found.
695 */
696static void nvgpu_page_free(struct nvgpu_allocator *__a, u64 base)
697{
698 struct nvgpu_page_allocator *a = page_allocator(__a);
699 struct nvgpu_page_alloc *alloc;
700
701 alloc_lock(__a);
702
703 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER)
704 alloc = __find_page_alloc(a, base);
705 else
706 alloc = __find_page_alloc(a,
707 ((struct nvgpu_page_alloc *)(uintptr_t)base)->base);
708
709 if (!alloc) {
710 palloc_dbg(a, "Hrm, found no alloc?\n");
711 goto done;
712 }
713
714 a->nr_frees++;
715
716 palloc_dbg(a, "Free 0x%llx id=0x%010llx\n",
717 alloc->length, alloc->base);
718
719 /*
720 * Frees *alloc.
721 */
722 if (alloc->slab_page) {
723 __nvgpu_free_slab(a, alloc);
724 } else {
725 a->pages_freed += (alloc->length >> a->page_shift);
726 __nvgpu_free_pages(a, alloc, true);
727 }
728
729done:
730 alloc_unlock(__a);
731}
732
733static struct nvgpu_page_alloc *__nvgpu_alloc_pages_fixed(
734 struct nvgpu_page_allocator *a, u64 base, u64 length, u32 unused)
735{
736 struct nvgpu_page_alloc *alloc;
737 struct nvgpu_mem_sgl *sgl;
738
739 alloc = nvgpu_kmem_cache_alloc(a->alloc_cache);
740 sgl = nvgpu_kzalloc(a->owner->g, sizeof(*sgl));
741 if (!alloc || !sgl)
742 goto fail;
743
744 alloc->sgt.ops = &page_alloc_sgl_ops;
745 alloc->base = nvgpu_alloc_fixed(&a->source_allocator, base, length, 0);
746 if (!alloc->base) {
747 WARN(1, "nvgpu: failed to fixed alloc pages @ 0x%010llx", base);
748 goto fail;
749 }
750
751 alloc->nr_chunks = 1;
752 alloc->length = length;
753 alloc->sgt.sgl = sgl;
754
755 sgl->phys = alloc->base;
756 sgl->dma = alloc->base;
757 sgl->length = length;
758 sgl->next = NULL;
759
760 return alloc;
761
762fail:
763 if (sgl)
764 nvgpu_kfree(a->owner->g, sgl);
765 if (alloc)
766 nvgpu_kmem_cache_free(a->alloc_cache, alloc);
767 return NULL;
768}
769
770/*
771 * @page_size is ignored.
772 */
773static u64 nvgpu_page_alloc_fixed(struct nvgpu_allocator *__a,
774 u64 base, u64 len, u32 page_size)
775{
776 struct nvgpu_page_allocator *a = page_allocator(__a);
777 struct nvgpu_page_alloc *alloc = NULL;
778 struct nvgpu_mem_sgl *sgl;
779 u64 aligned_len, pages;
780 int i = 0;
781
782 aligned_len = ALIGN(len, a->page_size);
783 pages = aligned_len >> a->page_shift;
784
785 alloc_lock(__a);
786
787 alloc = __nvgpu_alloc_pages_fixed(a, base, aligned_len, 0);
788 if (!alloc) {
789 alloc_unlock(__a);
790 return 0;
791 }
792
793 __insert_page_alloc(a, alloc);
794 alloc_unlock(__a);
795
796 palloc_dbg(a, "Alloc [fixed] @ 0x%010llx + 0x%llx (%llu)\n",
797 alloc->base, aligned_len, pages);
798 sgl = alloc->sgt.sgl;
799 while (sgl) {
800 palloc_dbg(a, " Chunk %2d: 0x%010llx + 0x%llx\n",
801 i++,
802 nvgpu_sgt_get_phys(&alloc->sgt, sgl),
803 nvgpu_sgt_get_length(&alloc->sgt, sgl));
804 sgl = nvgpu_sgt_get_next(&alloc->sgt, sgl);
805 }
806
807 a->nr_fixed_allocs++;
808 a->pages_alloced += pages;
809
810 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER)
811 return alloc->base;
812 else
813 return (u64) (uintptr_t) alloc;
814}
815
816static void nvgpu_page_free_fixed(struct nvgpu_allocator *__a,
817 u64 base, u64 len)
818{
819 struct nvgpu_page_allocator *a = page_allocator(__a);
820 struct nvgpu_page_alloc *alloc;
821
822 alloc_lock(__a);
823
824 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) {
825 alloc = __find_page_alloc(a, base);
826 if (!alloc)
827 goto done;
828 } else {
829 alloc = (struct nvgpu_page_alloc *) (uintptr_t) base;
830 }
831
832 palloc_dbg(a, "Free [fixed] 0x%010llx + 0x%llx\n",
833 alloc->base, alloc->length);
834
835 a->nr_fixed_frees++;
836 a->pages_freed += (alloc->length >> a->page_shift);
837
838 /*
839 * This works for the time being since the buddy allocator
840 * uses the same free function for both fixed and regular
841 * allocs. This would have to be updated if the underlying
842 * allocator were to change.
843 */
844 __nvgpu_free_pages(a, alloc, true);
845
846done:
847 alloc_unlock(__a);
848}
849
850static void nvgpu_page_allocator_destroy(struct nvgpu_allocator *__a)
851{
852 struct nvgpu_page_allocator *a = page_allocator(__a);
853
854 alloc_lock(__a);
855 nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a);
856 __a->priv = NULL;
857 alloc_unlock(__a);
858}
859
860#ifdef __KERNEL__
861static void nvgpu_page_print_stats(struct nvgpu_allocator *__a,
862 struct seq_file *s, int lock)
863{
864 struct nvgpu_page_allocator *a = page_allocator(__a);
865 int i;
866
867 if (lock)
868 alloc_lock(__a);
869
870 __alloc_pstat(s, __a, "Page allocator:\n");
871 __alloc_pstat(s, __a, " allocs %lld\n", a->nr_allocs);
872 __alloc_pstat(s, __a, " frees %lld\n", a->nr_frees);
873 __alloc_pstat(s, __a, " fixed_allocs %lld\n", a->nr_fixed_allocs);
874 __alloc_pstat(s, __a, " fixed_frees %lld\n", a->nr_fixed_frees);
875 __alloc_pstat(s, __a, " slab_allocs %lld\n", a->nr_slab_allocs);
876 __alloc_pstat(s, __a, " slab_frees %lld\n", a->nr_slab_frees);
877 __alloc_pstat(s, __a, " pages alloced %lld\n", a->pages_alloced);
878 __alloc_pstat(s, __a, " pages freed %lld\n", a->pages_freed);
879 __alloc_pstat(s, __a, "\n");
880
881 __alloc_pstat(s, __a, "Page size: %lld KB\n",
882 a->page_size >> 10);
883 __alloc_pstat(s, __a, "Total pages: %lld (%lld MB)\n",
884 a->length / a->page_size,
885 a->length >> 20);
886 __alloc_pstat(s, __a, "Available pages: %lld (%lld MB)\n",
887 nvgpu_alloc_space(&a->source_allocator) / a->page_size,
888 nvgpu_alloc_space(&a->source_allocator) >> 20);
889 __alloc_pstat(s, __a, "\n");
890
891 /*
892 * Slab info.
893 */
894 if (a->flags & GPU_ALLOC_4K_VIDMEM_PAGES) {
895 __alloc_pstat(s, __a, "Slabs:\n");
896 __alloc_pstat(s, __a, " size empty partial full\n");
897 __alloc_pstat(s, __a, " ---- ----- ------- ----\n");
898
899 for (i = 0; i < a->nr_slabs; i++) {
900 struct page_alloc_slab *slab = &a->slabs[i];
901
902 __alloc_pstat(s, __a, " %-9u %-9d %-9u %u\n",
903 slab->slab_size,
904 slab->nr_empty, slab->nr_partial,
905 slab->nr_full);
906 }
907 __alloc_pstat(s, __a, "\n");
908 }
909
910 __alloc_pstat(s, __a, "Source alloc: %s\n",
911 a->source_allocator.name);
912 nvgpu_alloc_print_stats(&a->source_allocator, s, lock);
913
914 if (lock)
915 alloc_unlock(__a);
916}
917#endif
918
919static const struct nvgpu_allocator_ops page_ops = {
920 .alloc = nvgpu_page_alloc,
921 .free = nvgpu_page_free,
922
923 .alloc_fixed = nvgpu_page_alloc_fixed,
924 .free_fixed = nvgpu_page_free_fixed,
925
926 .reserve_carveout = nvgpu_page_reserve_co,
927 .release_carveout = nvgpu_page_release_co,
928
929 .base = nvgpu_page_alloc_base,
930 .length = nvgpu_page_alloc_length,
931 .end = nvgpu_page_alloc_end,
932 .inited = nvgpu_page_alloc_inited,
933 .space = nvgpu_page_alloc_space,
934
935 .fini = nvgpu_page_allocator_destroy,
936
937#ifdef __KERNEL__
938 .print_stats = nvgpu_page_print_stats,
939#endif
940};
941
942/*
943 * nr_slabs is computed as follows: divide page_size by 4096 to get number of
944 * 4k pages in page_size. Then take the base 2 log of that to get number of
945 * slabs. For 64k page_size that works on like:
946 *
947 * 1024*64 / 1024*4 = 16
948 * ilog2(16) = 4
949 *
950 * That gives buckets of 1, 2, 4, and 8 pages (i.e 4k, 8k, 16k, 32k).
951 */
952static int nvgpu_page_alloc_init_slabs(struct nvgpu_page_allocator *a)
953{
954 size_t nr_slabs = ilog2(a->page_size >> 12);
955 unsigned int i;
956
957 a->slabs = nvgpu_kcalloc(nvgpu_alloc_to_gpu(a->owner),
958 nr_slabs,
959 sizeof(struct page_alloc_slab));
960 if (!a->slabs)
961 return -ENOMEM;
962 a->nr_slabs = nr_slabs;
963
964 for (i = 0; i < nr_slabs; i++) {
965 struct page_alloc_slab *slab = &a->slabs[i];
966
967 slab->slab_size = SZ_4K * (1 << i);
968 nvgpu_init_list_node(&slab->empty);
969 nvgpu_init_list_node(&slab->partial);
970 nvgpu_init_list_node(&slab->full);
971 slab->nr_empty = 0;
972 slab->nr_partial = 0;
973 slab->nr_full = 0;
974 }
975
976 return 0;
977}
978
979int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
980 const char *name, u64 base, u64 length,
981 u64 blk_size, u64 flags)
982{
983 struct nvgpu_page_allocator *a;
984 char buddy_name[sizeof(__a->name)];
985 int err;
986
987 if (blk_size < SZ_4K)
988 return -EINVAL;
989
990 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_page_allocator));
991 if (!a)
992 return -ENOMEM;
993
994 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &page_ops);
995 if (err)
996 goto fail;
997
998 a->alloc_cache = nvgpu_kmem_cache_create(g,
999 sizeof(struct nvgpu_page_alloc));
1000 a->slab_page_cache = nvgpu_kmem_cache_create(g,
1001 sizeof(struct page_alloc_slab_page));
1002 if (!a->alloc_cache || !a->slab_page_cache) {
1003 err = -ENOMEM;
1004 goto fail;
1005 }
1006
1007 a->base = base;
1008 a->length = length;
1009 a->page_size = blk_size;
1010 a->page_shift = __ffs(blk_size);
1011 a->allocs = NULL;
1012 a->owner = __a;
1013 a->flags = flags;
1014
1015 if (flags & GPU_ALLOC_4K_VIDMEM_PAGES && blk_size > SZ_4K) {
1016 err = nvgpu_page_alloc_init_slabs(a);
1017 if (err)
1018 goto fail;
1019 }
1020
1021 snprintf(buddy_name, sizeof(buddy_name), "%s-src", name);
1022
1023 err = nvgpu_buddy_allocator_init(g, &a->source_allocator, buddy_name,
1024 base, length, blk_size, 0);
1025 if (err)
1026 goto fail;
1027
1028#ifdef CONFIG_DEBUG_FS
1029 nvgpu_init_alloc_debug(g, __a);
1030#endif
1031 palloc_dbg(a, "New allocator: type page\n");
1032 palloc_dbg(a, " base 0x%llx\n", a->base);
1033 palloc_dbg(a, " size 0x%llx\n", a->length);
1034 palloc_dbg(a, " page_size 0x%llx\n", a->page_size);
1035 palloc_dbg(a, " flags 0x%llx\n", a->flags);
1036 palloc_dbg(a, " slabs: %d\n", a->nr_slabs);
1037
1038 return 0;
1039
1040fail:
1041 if (a->alloc_cache)
1042 nvgpu_kmem_cache_destroy(a->alloc_cache);
1043 if (a->slab_page_cache)
1044 nvgpu_kmem_cache_destroy(a->slab_page_cache);
1045 nvgpu_kfree(g, a);
1046 return err;
1047}
diff --git a/drivers/gpu/nvgpu/common/mm/pd_cache.c b/drivers/gpu/nvgpu/common/mm/pd_cache.c
new file mode 100644
index 00000000..4c3e06ba
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/pd_cache.c
@@ -0,0 +1,444 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/log.h>
24#include <nvgpu/dma.h>
25#include <nvgpu/gmmu.h>
26#include <nvgpu/nvgpu_mem.h>
27#include <nvgpu/list.h>
28#include <nvgpu/log2.h>
29
30#include "gk20a/gk20a.h"
31#include "gk20a/mm_gk20a.h"
32
33#define pd_dbg(g, fmt, args...) nvgpu_log(g, gpu_dbg_pd_cache, fmt, ##args)
34
35/**
36 * DOC: PD cache
37 *
38 * In the name of saving memory with the many sub-page sized PD levels in Pascal
39 * and beyond a way of packing PD tables together is necessary. This code here
40 * does just that. If a PD table only requires 1024 bytes, then it is possible
41 * to have 4 of these PDs in one page. This is even more pronounced for 256 byte
42 * PD tables.
43 *
44 * The pd cache is basially just a slab allocator. Each instance of the nvgpu
45 * driver makes one of these structs:
46 *
47 * struct nvgpu_pd_cache {
48 * struct nvgpu_list_node full[NVGPU_PD_CACHE_COUNT];
49 * struct nvgpu_list_node partial[NVGPU_PD_CACHE_COUNT];
50 *
51 * struct nvgpu_rbtree_node *mem_tree;
52 * };
53 *
54 * There are two sets of lists, the full and the partial. The full lists contain
55 * pages of memory for which all the memory in that page is in use. The partial
56 * lists contain partially full pages of memory which can be used for more PD
57 * allocations. There a couple of assumptions here:
58 *
59 * 1. PDs greater than or equal to the page size bypass the pd cache.
60 * 2. PDs are always power of 2 and greater than %NVGPU_PD_CACHE_MIN bytes.
61 *
62 * There are NVGPU_PD_CACHE_COUNT full lists and the same number of partial
63 * lists. For a 4Kb page NVGPU_PD_CACHE_COUNT is 4. This is enough space for
64 * 256, 512, 1024, and 2048 byte PDs.
65 *
66 * __nvgpu_pd_alloc() will allocate a PD for the GMMU. It will check if the PD
67 * size is page size or larger and choose the correct allocation scheme - either
68 * from the PD cache or directly. Similarly __nvgpu_pd_free() will free a PD
69 * allocated by __nvgpu_pd_alloc().
70 *
71 * Since the top level PD (the PDB) is a page aligned pointer but less than a
72 * page size the direct functions must be used for allocating PDBs. Otherwise
73 * there would be alignment issues for the PDBs when they get packed.
74 */
75
76static u32 nvgpu_pd_cache_nr(u32 bytes)
77{
78 return ilog2(bytes >> (NVGPU_PD_CACHE_MIN_SHIFT - 1));
79}
80
81static u32 nvgpu_pd_cache_get_mask(struct nvgpu_pd_mem_entry *pentry)
82{
83 u32 mask_offset = 1 << (PAGE_SIZE / pentry->pd_size);
84
85 return mask_offset - 1;
86}
87
88int nvgpu_pd_cache_init(struct gk20a *g)
89{
90 struct nvgpu_pd_cache *cache;
91 int i;
92
93 /*
94 * This gets called from finalize_poweron() so we need to make sure we
95 * don't reinit the pd_cache over and over.
96 */
97 if (g->mm.pd_cache)
98 return 0;
99
100 cache = nvgpu_kzalloc(g, sizeof(*cache));
101 if (!cache) {
102 nvgpu_err(g, "Failed to alloc pd_cache!");
103 return -ENOMEM;
104 }
105
106 for (i = 0; i < NVGPU_PD_CACHE_COUNT; i++) {
107 nvgpu_init_list_node(&cache->full[i]);
108 nvgpu_init_list_node(&cache->partial[i]);
109 }
110
111 cache->mem_tree = NULL;
112 g->mm.pd_cache = cache;
113 nvgpu_mutex_init(&cache->lock);
114
115 pd_dbg(g, "PD cache initialized!");
116
117 return 0;
118}
119
120void nvgpu_pd_cache_fini(struct gk20a *g)
121{
122 int i;
123 struct nvgpu_pd_cache *cache = g->mm.pd_cache;
124
125 if (!cache)
126 return;
127
128 for (i = 0; i < NVGPU_PD_CACHE_COUNT; i++) {
129 WARN_ON(!nvgpu_list_empty(&cache->full[i]));
130 WARN_ON(!nvgpu_list_empty(&cache->partial[i]));
131 }
132
133 nvgpu_kfree(g, g->mm.pd_cache);
134}
135
136/*
137 * This is the simple pass-through for greater than page or page sized PDs.
138 *
139 * Note: this does not need the cache lock since it does not modify any of the
140 * PD cache data structures.
141 */
142int __nvgpu_pd_cache_alloc_direct(struct gk20a *g,
143 struct nvgpu_gmmu_pd *pd, u32 bytes)
144{
145 int err;
146 unsigned long flags = 0;
147
148 pd_dbg(g, "PD-Alloc [D] %u bytes", bytes);
149
150 pd->mem = nvgpu_kzalloc(g, sizeof(*pd->mem));
151 if (!pd->mem) {
152 nvgpu_err(g, "OOM allocating nvgpu_mem struct!");
153 return -ENOMEM;
154 }
155
156 /*
157 * If bytes == PAGE_SIZE then it's impossible to get a discontiguous DMA
158 * allocation. Some DMA implementations may, despite this fact, still
159 * use the contiguous pool for page sized allocations. As such only
160 * request explicitly contiguous allocs if the page directory is larger
161 * than the page size. Also, of course, this is all only revelant for
162 * GPUs not using an IOMMU. If there is an IOMMU DMA allocs are always
163 * going to be virtually contiguous and we don't have to force the
164 * underlying allocations to be physically contiguous as well.
165 */
166 if (!nvgpu_iommuable(g) && bytes > PAGE_SIZE)
167 flags = NVGPU_DMA_FORCE_CONTIGUOUS;
168
169 err = nvgpu_dma_alloc_flags(g, flags, bytes, pd->mem);
170 if (err) {
171 nvgpu_err(g, "OOM allocating page directory!");
172 nvgpu_kfree(g, pd->mem);
173 return -ENOMEM;
174 }
175
176 pd->cached = false;
177 pd->mem_offs = 0;
178
179 return 0;
180}
181
182/*
183 * Make a new nvgpu_pd_cache_entry and allocate a PD from it. Update the passed
184 * pd to reflect this allocation.
185 */
186static int nvgpu_pd_cache_alloc_new(struct gk20a *g,
187 struct nvgpu_pd_cache *cache,
188 struct nvgpu_gmmu_pd *pd,
189 u32 bytes)
190{
191 struct nvgpu_pd_mem_entry *pentry;
192
193 pd_dbg(g, "PD-Alloc [C] New: offs=0");
194
195 pentry = nvgpu_kzalloc(g, sizeof(*pentry));
196 if (!pentry) {
197 nvgpu_err(g, "OOM allocating pentry!");
198 return -ENOMEM;
199 }
200
201 if (nvgpu_dma_alloc(g, PAGE_SIZE, &pentry->mem)) {
202 nvgpu_kfree(g, pentry);
203 nvgpu_err(g, "Unable to DMA alloc!");
204 return -ENOMEM;
205 }
206
207 pentry->pd_size = bytes;
208 nvgpu_list_add(&pentry->list_entry,
209 &cache->partial[nvgpu_pd_cache_nr(bytes)]);
210
211 /*
212 * This allocates the very first PD table in the set of tables in this
213 * nvgpu_pd_mem_entry.
214 */
215 pentry->alloc_map = 1;
216
217 /*
218 * Now update the nvgpu_gmmu_pd to reflect this allocation.
219 */
220 pd->mem = &pentry->mem;
221 pd->mem_offs = 0;
222 pd->cached = true;
223
224 pentry->tree_entry.key_start = (u64)(uintptr_t)&pentry->mem;
225 nvgpu_rbtree_insert(&pentry->tree_entry, &cache->mem_tree);
226
227 return 0;
228}
229
230static int nvgpu_pd_cache_alloc_from_partial(struct gk20a *g,
231 struct nvgpu_pd_cache *cache,
232 struct nvgpu_pd_mem_entry *pentry,
233 struct nvgpu_gmmu_pd *pd)
234{
235 unsigned long bit_offs;
236 u32 mem_offs;
237 u32 pentry_mask = nvgpu_pd_cache_get_mask(pentry);
238
239 /*
240 * Find and allocate an open PD.
241 */
242 bit_offs = ffz(pentry->alloc_map);
243 mem_offs = bit_offs * pentry->pd_size;
244
245 /* Bit map full. Somethings wrong. */
246 if (WARN_ON(bit_offs >= ffz(pentry_mask)))
247 return -ENOMEM;
248
249 pentry->alloc_map |= 1 << bit_offs;
250
251 pd_dbg(g, "PD-Alloc [C] Partial: offs=%lu", bit_offs);
252
253 /*
254 * First update the pd.
255 */
256 pd->mem = &pentry->mem;
257 pd->mem_offs = mem_offs;
258 pd->cached = true;
259
260 /*
261 * Now make sure the pentry is in the correct list (full vs partial).
262 */
263 if ((pentry->alloc_map & pentry_mask) == pentry_mask) {
264 pd_dbg(g, "Adding pentry to full list!");
265 nvgpu_list_del(&pentry->list_entry);
266 nvgpu_list_add(&pentry->list_entry,
267 &cache->full[nvgpu_pd_cache_nr(pentry->pd_size)]);
268 }
269
270 return 0;
271}
272
273/*
274 * Get a partially full nvgpu_pd_mem_entry. Returns NULL if there is no partial
275 * nvgpu_pd_mem_entry's.
276 */
277static struct nvgpu_pd_mem_entry *nvgpu_pd_cache_get_partial(
278 struct nvgpu_pd_cache *cache, u32 bytes)
279{
280 struct nvgpu_list_node *list =
281 &cache->partial[nvgpu_pd_cache_nr(bytes)];
282
283 if (nvgpu_list_empty(list))
284 return NULL;
285
286 return nvgpu_list_first_entry(list,
287 nvgpu_pd_mem_entry,
288 list_entry);
289}
290
291/*
292 * Allocate memory from an nvgpu_mem for the page directory.
293 */
294static int nvgpu_pd_cache_alloc(struct gk20a *g, struct nvgpu_pd_cache *cache,
295 struct nvgpu_gmmu_pd *pd, u32 bytes)
296{
297 struct nvgpu_pd_mem_entry *pentry;
298 int err;
299
300 pd_dbg(g, "PD-Alloc [C] %u bytes", bytes);
301
302 if (bytes & (bytes - 1) ||
303 (bytes >= PAGE_SIZE ||
304 bytes < NVGPU_PD_CACHE_MIN)) {
305 pd_dbg(g, "PD-Alloc [C] Invalid (bytes=%u)!", bytes);
306 return -EINVAL;
307 }
308
309 pentry = nvgpu_pd_cache_get_partial(cache, bytes);
310 if (!pentry)
311 err = nvgpu_pd_cache_alloc_new(g, cache, pd, bytes);
312 else
313 err = nvgpu_pd_cache_alloc_from_partial(g, cache, pentry, pd);
314
315 if (err)
316 nvgpu_err(g, "PD-Alloc [C] Failed!");
317
318 return err;
319}
320
321/*
322 * Allocate the DMA memory for a page directory. This handles the necessary PD
323 * cache logistics. Since on Parker and later GPUs some of the page directories
324 * are smaller than a page packing these PDs together saves a lot of memory.
325 */
326int __nvgpu_pd_alloc(struct vm_gk20a *vm, struct nvgpu_gmmu_pd *pd, u32 bytes)
327{
328 struct gk20a *g = gk20a_from_vm(vm);
329 int err;
330
331 /*
332 * Simple case: PD is bigger than a page so just do a regular DMA
333 * alloc.
334 */
335 if (bytes >= PAGE_SIZE) {
336 err = __nvgpu_pd_cache_alloc_direct(g, pd, bytes);
337 if (err)
338 return err;
339
340 return 0;
341 }
342
343 if (WARN_ON(!g->mm.pd_cache))
344 return -ENOMEM;
345
346 nvgpu_mutex_acquire(&g->mm.pd_cache->lock);
347 err = nvgpu_pd_cache_alloc(g, g->mm.pd_cache, pd, bytes);
348 nvgpu_mutex_release(&g->mm.pd_cache->lock);
349
350 return err;
351}
352
353void __nvgpu_pd_cache_free_direct(struct gk20a *g, struct nvgpu_gmmu_pd *pd)
354{
355 pd_dbg(g, "PD-Free [D] 0x%p", pd->mem);
356
357 if (!pd->mem)
358 return;
359
360 nvgpu_dma_free(g, pd->mem);
361 nvgpu_kfree(g, pd->mem);
362 pd->mem = NULL;
363}
364
365static void nvgpu_pd_cache_free_mem_entry(struct gk20a *g,
366 struct nvgpu_pd_cache *cache,
367 struct nvgpu_pd_mem_entry *pentry)
368{
369 nvgpu_dma_free(g, &pentry->mem);
370 nvgpu_list_del(&pentry->list_entry);
371 nvgpu_rbtree_unlink(&pentry->tree_entry, &cache->mem_tree);
372 nvgpu_kfree(g, pentry);
373}
374
375static void nvgpu_pd_cache_do_free(struct gk20a *g,
376 struct nvgpu_pd_cache *cache,
377 struct nvgpu_pd_mem_entry *pentry,
378 struct nvgpu_gmmu_pd *pd)
379{
380 u32 index = pd->mem_offs / pentry->pd_size;
381 u32 bit = 1 << index;
382
383 /* Mark entry as free. */
384 pentry->alloc_map &= ~bit;
385
386 if (pentry->alloc_map & nvgpu_pd_cache_get_mask(pentry)) {
387 /*
388 * Partially full still. If it was already on the partial list
389 * this just re-adds it.
390 */
391 nvgpu_list_del(&pentry->list_entry);
392 nvgpu_list_add(&pentry->list_entry,
393 &cache->partial[nvgpu_pd_cache_nr(pentry->pd_size)]);
394 } else {
395 /* Empty now so free it. */
396 nvgpu_pd_cache_free_mem_entry(g, cache, pentry);
397 }
398}
399
400static struct nvgpu_pd_mem_entry *nvgpu_pd_cache_look_up(
401 struct gk20a *g,
402 struct nvgpu_pd_cache *cache,
403 struct nvgpu_gmmu_pd *pd)
404{
405 struct nvgpu_rbtree_node *node;
406
407 nvgpu_rbtree_search((u64)(uintptr_t)pd->mem, &node,
408 cache->mem_tree);
409 if (!node)
410 return NULL;
411
412 return nvgpu_pd_mem_entry_from_tree_entry(node);
413}
414
415static void nvgpu_pd_cache_free(struct gk20a *g, struct nvgpu_pd_cache *cache,
416 struct nvgpu_gmmu_pd *pd)
417{
418 struct nvgpu_pd_mem_entry *pentry;
419
420 pd_dbg(g, "PD-Free [C] 0x%p", pd->mem);
421
422 pentry = nvgpu_pd_cache_look_up(g, cache, pd);
423 if (!pentry) {
424 WARN(1, "Attempting to free non-existent pd");
425 return;
426 }
427
428 nvgpu_pd_cache_do_free(g, cache, pentry, pd);
429}
430
431void __nvgpu_pd_free(struct vm_gk20a *vm, struct nvgpu_gmmu_pd *pd)
432{
433 struct gk20a *g = gk20a_from_vm(vm);
434
435 /*
436 * Simple case: just DMA free.
437 */
438 if (!pd->cached)
439 return __nvgpu_pd_cache_free_direct(g, pd);
440
441 nvgpu_mutex_acquire(&g->mm.pd_cache->lock);
442 nvgpu_pd_cache_free(g, g->mm.pd_cache, pd);
443 nvgpu_mutex_release(&g->mm.pd_cache->lock);
444}
diff --git a/drivers/gpu/nvgpu/common/mm/vidmem.c b/drivers/gpu/nvgpu/common/mm/vidmem.c
new file mode 100644
index 00000000..3526fce5
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/vidmem.c
@@ -0,0 +1,554 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/scatterlist.h>
24
25#include <nvgpu/timers.h>
26#include <nvgpu/dma.h>
27#include <nvgpu/vidmem.h>
28#include <nvgpu/page_allocator.h>
29#include <nvgpu/enabled.h>
30
31#include "gk20a/gk20a.h"
32#include "gk20a/mm_gk20a.h"
33
34/*
35 * This is expected to be called from the shutdown path (or the error path in
36 * the vidmem init code). As such we do not expect new vidmem frees to be
37 * enqueued.
38 */
39void nvgpu_vidmem_destroy(struct gk20a *g)
40{
41 struct nvgpu_timeout timeout;
42
43 nvgpu_timeout_init(g, &timeout, 100, NVGPU_TIMER_RETRY_TIMER);
44
45 /*
46 * Ensure that the thread runs one last time to flush anything in the
47 * queue.
48 */
49 nvgpu_cond_signal_interruptible(&g->mm.vidmem.clearing_thread_cond);
50
51 /*
52 * Wait for at most 1 second before just continuing on. It doesn't make
53 * sense to hang the system over some potential memory leaks.
54 */
55 do {
56 bool empty;
57
58 nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex);
59 empty = nvgpu_list_empty(&g->mm.vidmem.clear_list_head);
60 nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex);
61
62 if (empty)
63 break;
64
65 nvgpu_msleep(10);
66 } while (!nvgpu_timeout_expired(&timeout));
67
68 /*
69 * Kill the vidmem clearing thread now. This will wake the thread up
70 * automatically and cause the wait_interruptible condition trigger.
71 */
72 nvgpu_thread_stop(&g->mm.vidmem.clearing_thread);
73
74 if (nvgpu_alloc_initialized(&g->mm.vidmem.allocator))
75 nvgpu_alloc_destroy(&g->mm.vidmem.allocator);
76}
77
78static int __nvgpu_vidmem_do_clear_all(struct gk20a *g)
79{
80 struct mm_gk20a *mm = &g->mm;
81 struct gk20a_fence *gk20a_fence_out = NULL;
82 u64 region2_base = 0;
83 int err = 0;
84
85 if (mm->vidmem.ce_ctx_id == (u32)~0)
86 return -EINVAL;
87
88 vidmem_dbg(g, "Clearing all VIDMEM:");
89
90 err = gk20a_ce_execute_ops(g,
91 mm->vidmem.ce_ctx_id,
92 0,
93 mm->vidmem.base,
94 mm->vidmem.bootstrap_base - mm->vidmem.base,
95 0x00000000,
96 NVGPU_CE_DST_LOCATION_LOCAL_FB,
97 NVGPU_CE_MEMSET,
98 NULL,
99 0,
100 NULL);
101 if (err) {
102 nvgpu_err(g,
103 "Failed to clear vidmem region 1 : %d", err);
104 return err;
105 }
106
107 region2_base = mm->vidmem.bootstrap_base + mm->vidmem.bootstrap_size;
108
109 err = gk20a_ce_execute_ops(g,
110 mm->vidmem.ce_ctx_id,
111 0,
112 region2_base,
113 mm->vidmem.size - region2_base,
114 0x00000000,
115 NVGPU_CE_DST_LOCATION_LOCAL_FB,
116 NVGPU_CE_MEMSET,
117 NULL,
118 0,
119 &gk20a_fence_out);
120 if (err) {
121 nvgpu_err(g,
122 "Failed to clear vidmem region 2 : %d", err);
123 return err;
124 }
125
126 if (gk20a_fence_out) {
127 struct nvgpu_timeout timeout;
128
129 nvgpu_timeout_init(g, &timeout,
130 gk20a_get_gr_idle_timeout(g),
131 NVGPU_TIMER_CPU_TIMER);
132
133 do {
134 err = gk20a_fence_wait(g, gk20a_fence_out,
135 gk20a_get_gr_idle_timeout(g));
136 } while (err == -ERESTARTSYS &&
137 !nvgpu_timeout_expired(&timeout));
138
139 gk20a_fence_put(gk20a_fence_out);
140 if (err) {
141 nvgpu_err(g,
142 "fence wait failed for CE execute ops");
143 return err;
144 }
145 }
146
147 mm->vidmem.cleared = true;
148
149 vidmem_dbg(g, "Done!");
150
151 return 0;
152}
153
154void nvgpu_vidmem_thread_pause_sync(struct mm_gk20a *mm)
155{
156 /*
157 * On the first increment of the pause_count (0 -> 1) take the pause
158 * lock and prevent the vidmem clearing thread from processing work
159 * items.
160 *
161 * Otherwise the increment is all that's needed - it's essentially a
162 * ref-count for the number of pause() calls.
163 *
164 * The sync component is implemented by waiting for the lock to be
165 * released by the clearing thread in case the thread is currently
166 * processing work items.
167 */
168 if (nvgpu_atomic_inc_return(&mm->vidmem.pause_count) == 1)
169 nvgpu_mutex_acquire(&mm->vidmem.clearing_thread_lock);
170
171 vidmem_dbg(mm->g, "Clearing thread paused; new count=%d",
172 nvgpu_atomic_read(&mm->vidmem.pause_count));
173}
174
175void nvgpu_vidmem_thread_unpause(struct mm_gk20a *mm)
176{
177 vidmem_dbg(mm->g, "Unpausing clearing thread; current count=%d",
178 nvgpu_atomic_read(&mm->vidmem.pause_count));
179
180 /*
181 * And on the last decrement (1 -> 0) release the pause lock and let
182 * the vidmem clearing thread continue.
183 */
184 if (nvgpu_atomic_dec_return(&mm->vidmem.pause_count) == 0) {
185 nvgpu_mutex_release(&mm->vidmem.clearing_thread_lock);
186 vidmem_dbg(mm->g, " > Clearing thread really unpaused!");
187 }
188}
189
190int nvgpu_vidmem_clear_list_enqueue(struct gk20a *g, struct nvgpu_mem *mem)
191{
192 struct mm_gk20a *mm = &g->mm;
193
194 /*
195 * Crap. Can't enqueue new vidmem bufs! CE may be gone!
196 *
197 * However, an errant app can hold a vidmem dma_buf FD open past when
198 * the nvgpu driver has exited. Thus when the FD does get closed
199 * eventually the dma_buf release function will try to call the vidmem
200 * free function which will attempt to enqueue the vidmem into the
201 * vidmem clearing thread.
202 */
203 if (nvgpu_is_enabled(g, NVGPU_DRIVER_IS_DYING))
204 return -ENOSYS;
205
206 nvgpu_mutex_acquire(&mm->vidmem.clear_list_mutex);
207 nvgpu_list_add_tail(&mem->clear_list_entry,
208 &mm->vidmem.clear_list_head);
209 nvgpu_atomic64_add(mem->aligned_size, &mm->vidmem.bytes_pending);
210 nvgpu_mutex_release(&mm->vidmem.clear_list_mutex);
211
212 nvgpu_cond_signal_interruptible(&mm->vidmem.clearing_thread_cond);
213
214 return 0;
215}
216
217static struct nvgpu_mem *nvgpu_vidmem_clear_list_dequeue(struct mm_gk20a *mm)
218{
219 struct nvgpu_mem *mem = NULL;
220
221 nvgpu_mutex_acquire(&mm->vidmem.clear_list_mutex);
222 if (!nvgpu_list_empty(&mm->vidmem.clear_list_head)) {
223 mem = nvgpu_list_first_entry(&mm->vidmem.clear_list_head,
224 nvgpu_mem, clear_list_entry);
225 nvgpu_list_del(&mem->clear_list_entry);
226 }
227 nvgpu_mutex_release(&mm->vidmem.clear_list_mutex);
228
229 return mem;
230}
231
232static void nvgpu_vidmem_clear_pending_allocs(struct mm_gk20a *mm)
233{
234 struct gk20a *g = mm->g;
235 struct nvgpu_mem *mem;
236
237 vidmem_dbg(g, "Running VIDMEM clearing thread:");
238
239 while ((mem = nvgpu_vidmem_clear_list_dequeue(mm)) != NULL) {
240 nvgpu_vidmem_clear(g, mem);
241
242 WARN_ON(nvgpu_atomic64_sub_return(mem->aligned_size,
243 &g->mm.vidmem.bytes_pending) < 0);
244 mem->size = 0;
245 mem->aperture = APERTURE_INVALID;
246
247 __nvgpu_mem_free_vidmem_alloc(g, mem);
248 nvgpu_kfree(g, mem);
249 }
250
251 vidmem_dbg(g, "Done!");
252}
253
254static int nvgpu_vidmem_clear_pending_allocs_thr(void *mm_ptr)
255{
256 struct mm_gk20a *mm = mm_ptr;
257
258 /*
259 * Simple thread who's sole job is to periodically clear userspace
260 * vidmem allocations that have been recently freed.
261 *
262 * Since it doesn't make sense to run unless there's pending work a
263 * condition field is used to wait for work. When the DMA API frees a
264 * userspace vidmem buf it enqueues it into the clear list and alerts us
265 * that we have some work to do.
266 */
267
268 while (!nvgpu_thread_should_stop(&mm->vidmem.clearing_thread)) {
269 int ret;
270
271 /*
272 * Wait for work but also make sure we should not be paused.
273 */
274 ret = NVGPU_COND_WAIT_INTERRUPTIBLE(
275 &mm->vidmem.clearing_thread_cond,
276 nvgpu_thread_should_stop(
277 &mm->vidmem.clearing_thread) ||
278 !nvgpu_list_empty(&mm->vidmem.clear_list_head),
279 0);
280 if (ret == -ERESTARTSYS)
281 continue;
282
283 /*
284 * Use this lock to implement a pause mechanism. By taking this
285 * lock some other code can prevent this thread from processing
286 * work items.
287 */
288 if (!nvgpu_mutex_tryacquire(&mm->vidmem.clearing_thread_lock))
289 continue;
290
291 nvgpu_vidmem_clear_pending_allocs(mm);
292
293 nvgpu_mutex_release(&mm->vidmem.clearing_thread_lock);
294 }
295
296 return 0;
297}
298
299int nvgpu_vidmem_init(struct mm_gk20a *mm)
300{
301 struct gk20a *g = mm->g;
302 size_t size = g->ops.mm.get_vidmem_size ?
303 g->ops.mm.get_vidmem_size(g) : 0;
304 u64 bootstrap_base, bootstrap_size, base;
305 u64 default_page_size = SZ_64K;
306 int err;
307
308 static struct nvgpu_alloc_carveout wpr_co =
309 NVGPU_CARVEOUT("wpr-region", 0, SZ_16M);
310
311 if (!size)
312 return 0;
313
314 vidmem_dbg(g, "init begin");
315
316 wpr_co.base = size - SZ_256M;
317 bootstrap_base = wpr_co.base;
318 bootstrap_size = SZ_16M;
319 base = default_page_size;
320
321 /*
322 * Bootstrap allocator for use before the CE is initialized (CE
323 * initialization requires vidmem but we want to use the CE to zero
324 * out vidmem before allocating it...
325 */
326 err = nvgpu_page_allocator_init(g, &g->mm.vidmem.bootstrap_allocator,
327 "vidmem-bootstrap",
328 bootstrap_base, bootstrap_size,
329 SZ_4K, 0);
330
331 err = nvgpu_page_allocator_init(g, &g->mm.vidmem.allocator,
332 "vidmem",
333 base, size - base,
334 default_page_size,
335 GPU_ALLOC_4K_VIDMEM_PAGES);
336 if (err) {
337 nvgpu_err(g, "Failed to register vidmem for size %zu: %d",
338 size, err);
339 return err;
340 }
341
342 /* Reserve bootstrap region in vidmem allocator */
343 nvgpu_alloc_reserve_carveout(&g->mm.vidmem.allocator, &wpr_co);
344
345 mm->vidmem.base = base;
346 mm->vidmem.size = size - base;
347 mm->vidmem.bootstrap_base = bootstrap_base;
348 mm->vidmem.bootstrap_size = bootstrap_size;
349
350 err = nvgpu_cond_init(&mm->vidmem.clearing_thread_cond);
351 if (err)
352 goto fail;
353
354 nvgpu_atomic64_set(&mm->vidmem.bytes_pending, 0);
355 nvgpu_init_list_node(&mm->vidmem.clear_list_head);
356 nvgpu_mutex_init(&mm->vidmem.clear_list_mutex);
357 nvgpu_mutex_init(&mm->vidmem.clearing_thread_lock);
358 nvgpu_mutex_init(&mm->vidmem.first_clear_mutex);
359 nvgpu_atomic_set(&mm->vidmem.pause_count, 0);
360
361 /*
362 * Start the thread off in the paused state. The thread doesn't have to
363 * be running for this to work. It will be woken up later on in
364 * finalize_poweron(). We won't necessarily have a CE context yet
365 * either, so hypothetically one could cause a race where we try to
366 * clear a vidmem struct before we have a CE context to do so.
367 */
368 nvgpu_vidmem_thread_pause_sync(mm);
369
370 err = nvgpu_thread_create(&mm->vidmem.clearing_thread, mm,
371 nvgpu_vidmem_clear_pending_allocs_thr,
372 "vidmem-clear");
373 if (err)
374 goto fail;
375
376 vidmem_dbg(g, "VIDMEM Total: %zu MB", size >> 20);
377 vidmem_dbg(g, "VIDMEM Ranges:");
378 vidmem_dbg(g, " 0x%-10llx -> 0x%-10llx Primary",
379 mm->vidmem.base, mm->vidmem.base + mm->vidmem.size);
380 vidmem_dbg(g, " 0x%-10llx -> 0x%-10llx Bootstrap",
381 mm->vidmem.bootstrap_base,
382 mm->vidmem.bootstrap_base + mm->vidmem.bootstrap_size);
383 vidmem_dbg(g, "VIDMEM carveouts:");
384 vidmem_dbg(g, " 0x%-10llx -> 0x%-10llx %s",
385 wpr_co.base, wpr_co.base + wpr_co.length, wpr_co.name);
386
387 return 0;
388
389fail:
390 nvgpu_cond_destroy(&mm->vidmem.clearing_thread_cond);
391 nvgpu_vidmem_destroy(g);
392 return err;
393}
394
395int nvgpu_vidmem_get_space(struct gk20a *g, u64 *space)
396{
397 struct nvgpu_allocator *allocator = &g->mm.vidmem.allocator;
398
399 gk20a_dbg_fn("");
400
401 if (!nvgpu_alloc_initialized(allocator))
402 return -ENOSYS;
403
404 nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex);
405 *space = nvgpu_alloc_space(allocator) +
406 nvgpu_atomic64_read(&g->mm.vidmem.bytes_pending);
407 nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex);
408 return 0;
409}
410
411int nvgpu_vidmem_clear(struct gk20a *g, struct nvgpu_mem *mem)
412{
413 struct gk20a_fence *gk20a_fence_out = NULL;
414 struct gk20a_fence *gk20a_last_fence = NULL;
415 struct nvgpu_page_alloc *alloc = NULL;
416 void *sgl = NULL;
417 int err = 0;
418
419 if (g->mm.vidmem.ce_ctx_id == (u32)~0)
420 return -EINVAL;
421
422 alloc = mem->vidmem_alloc;
423
424 vidmem_dbg(g, "Clearing VIDMEM buf:");
425
426 nvgpu_sgt_for_each_sgl(sgl, &alloc->sgt) {
427 if (gk20a_last_fence)
428 gk20a_fence_put(gk20a_last_fence);
429
430 err = gk20a_ce_execute_ops(g,
431 g->mm.vidmem.ce_ctx_id,
432 0,
433 nvgpu_sgt_get_phys(&alloc->sgt, sgl),
434 nvgpu_sgt_get_length(&alloc->sgt, sgl),
435 0x00000000,
436 NVGPU_CE_DST_LOCATION_LOCAL_FB,
437 NVGPU_CE_MEMSET,
438 NULL,
439 0,
440 &gk20a_fence_out);
441
442 if (err) {
443 nvgpu_err(g,
444 "Failed gk20a_ce_execute_ops[%d]", err);
445 return err;
446 }
447
448 vidmem_dbg(g, " > [0x%llx +0x%llx]",
449 nvgpu_sgt_get_phys(&alloc->sgt, sgl),
450 nvgpu_sgt_get_length(&alloc->sgt, sgl));
451
452 gk20a_last_fence = gk20a_fence_out;
453 }
454
455 if (gk20a_last_fence) {
456 struct nvgpu_timeout timeout;
457
458 nvgpu_timeout_init(g, &timeout,
459 gk20a_get_gr_idle_timeout(g),
460 NVGPU_TIMER_CPU_TIMER);
461
462 do {
463 err = gk20a_fence_wait(g, gk20a_last_fence,
464 gk20a_get_gr_idle_timeout(g));
465 } while (err == -ERESTARTSYS &&
466 !nvgpu_timeout_expired(&timeout));
467
468 gk20a_fence_put(gk20a_last_fence);
469 if (err)
470 nvgpu_err(g,
471 "fence wait failed for CE execute ops");
472 }
473
474 vidmem_dbg(g, " Done");
475
476 return err;
477}
478
479static int nvgpu_vidmem_clear_all(struct gk20a *g)
480{
481 int err;
482
483 if (g->mm.vidmem.cleared)
484 return 0;
485
486 nvgpu_mutex_acquire(&g->mm.vidmem.first_clear_mutex);
487 if (!g->mm.vidmem.cleared) {
488 err = __nvgpu_vidmem_do_clear_all(g);
489 if (err) {
490 nvgpu_mutex_release(&g->mm.vidmem.first_clear_mutex);
491 nvgpu_err(g, "failed to clear whole vidmem");
492 return err;
493 }
494 }
495 nvgpu_mutex_release(&g->mm.vidmem.first_clear_mutex);
496
497 return 0;
498}
499
500struct nvgpu_vidmem_buf *nvgpu_vidmem_user_alloc(struct gk20a *g, size_t bytes)
501{
502 struct nvgpu_vidmem_buf *buf;
503 int err;
504
505 err = nvgpu_vidmem_clear_all(g);
506 if (err)
507 return NULL;
508
509 buf = nvgpu_kzalloc(g, sizeof(*buf));
510 if (!buf)
511 return NULL;
512
513 buf->g = g;
514 buf->mem = nvgpu_kzalloc(g, sizeof(*buf->mem));
515 if (!buf->mem)
516 goto fail;
517
518 err = nvgpu_dma_alloc_vid(g, bytes, buf->mem);
519 if (err)
520 goto fail;
521
522 /*
523 * Alerts the DMA API that when we free this vidmem buf we have to
524 * clear it to avoid leaking data to userspace.
525 */
526 buf->mem->mem_flags |= NVGPU_MEM_FLAG_USER_MEM;
527
528 return buf;
529
530fail:
531 /* buf will never be NULL here. */
532 nvgpu_kfree(g, buf->mem);
533 nvgpu_kfree(g, buf);
534 return NULL;
535}
536
537void nvgpu_vidmem_buf_free(struct gk20a *g, struct nvgpu_vidmem_buf *buf)
538{
539 /*
540 * In some error paths it's convenient to be able to "free" a NULL buf.
541 */
542 if (!buf)
543 return;
544
545 nvgpu_dma_free(g, buf->mem);
546
547 /*
548 * We don't free buf->mem here. This is handled by nvgpu_dma_free()!
549 * Since these buffers are cleared in the background the nvgpu_mem
550 * struct must live on through that. We transfer ownership here to the
551 * DMA API and let the DMA API free the buffer.
552 */
553 nvgpu_kfree(g, buf);
554}
diff --git a/drivers/gpu/nvgpu/common/mm/vm.c b/drivers/gpu/nvgpu/common/mm/vm.c
new file mode 100644
index 00000000..ebe8e381
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/vm.c
@@ -0,0 +1,1145 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/bug.h>
24#include <uapi/linux/nvgpu.h>
25
26#include <nvgpu/log.h>
27#include <nvgpu/dma.h>
28#include <nvgpu/vm.h>
29#include <nvgpu/vm_area.h>
30#include <nvgpu/gmmu.h>
31#include <nvgpu/lock.h>
32#include <nvgpu/list.h>
33#include <nvgpu/rbtree.h>
34#include <nvgpu/semaphore.h>
35#include <nvgpu/enabled.h>
36
37#include <nvgpu/vgpu/vm.h>
38
39#include "gk20a/gk20a.h"
40#include "gk20a/mm_gk20a.h"
41
42struct nvgpu_ctag_buffer_info {
43 u64 size;
44 enum gmmu_pgsz_gk20a pgsz_idx;
45 u32 flags;
46
47 s16 compr_kind;
48 s16 incompr_kind;
49
50 u32 ctag_lines;
51};
52
53static int nvgpu_vm_compute_compression(struct vm_gk20a *vm,
54 struct nvgpu_ctag_buffer_info *binfo);
55
56static void __nvgpu_vm_unmap(struct nvgpu_mapped_buf *mapped_buffer,
57 struct vm_gk20a_mapping_batch *batch);
58
59int vm_aspace_id(struct vm_gk20a *vm)
60{
61 return vm->as_share ? vm->as_share->id : -1;
62}
63
64static void __nvgpu_vm_free_entries(struct vm_gk20a *vm,
65 struct nvgpu_gmmu_pd *pd,
66 int level)
67{
68 int i;
69
70 if (pd->mem) {
71 __nvgpu_pd_free(vm, pd);
72 pd->mem = NULL;
73 }
74
75 if (pd->entries) {
76 for (i = 0; i < pd->num_entries; i++)
77 __nvgpu_vm_free_entries(vm, &pd->entries[i],
78 level + 1);
79 nvgpu_vfree(vm->mm->g, pd->entries);
80 pd->entries = NULL;
81 }
82}
83
84static void nvgpu_vm_free_entries(struct vm_gk20a *vm,
85 struct nvgpu_gmmu_pd *pdb)
86{
87 struct gk20a *g = vm->mm->g;
88 int i;
89
90 __nvgpu_pd_cache_free_direct(g, pdb);
91
92 if (!pdb->entries)
93 return;
94
95 for (i = 0; i < pdb->num_entries; i++)
96 __nvgpu_vm_free_entries(vm, &pdb->entries[i], 1);
97
98 nvgpu_vfree(g, pdb->entries);
99 pdb->entries = NULL;
100}
101
102u64 __nvgpu_vm_alloc_va(struct vm_gk20a *vm, u64 size,
103 enum gmmu_pgsz_gk20a pgsz_idx)
104
105{
106 struct gk20a *g = vm->mm->g;
107 struct nvgpu_allocator *vma = NULL;
108 u64 addr;
109 u64 page_size = vm->gmmu_page_sizes[pgsz_idx];
110
111 vma = vm->vma[pgsz_idx];
112
113 if (pgsz_idx >= gmmu_nr_page_sizes) {
114 nvgpu_err(g, "(%s) invalid page size requested", vma->name);
115 return 0;
116 }
117
118 if ((pgsz_idx == gmmu_page_size_big) && !vm->big_pages) {
119 nvgpu_err(g, "(%s) unsupportd page size requested", vma->name);
120 return 0;
121 }
122
123 /* Be certain we round up to page_size if needed */
124 size = (size + ((u64)page_size - 1)) & ~((u64)page_size - 1);
125
126 addr = nvgpu_alloc(vma, size);
127 if (!addr) {
128 nvgpu_err(g, "(%s) oom: sz=0x%llx", vma->name, size);
129 return 0;
130 }
131
132 return addr;
133}
134
135int __nvgpu_vm_free_va(struct vm_gk20a *vm, u64 addr,
136 enum gmmu_pgsz_gk20a pgsz_idx)
137{
138 struct nvgpu_allocator *vma = vm->vma[pgsz_idx];
139
140 nvgpu_free(vma, addr);
141
142 return 0;
143}
144
145void nvgpu_vm_mapping_batch_start(struct vm_gk20a_mapping_batch *mapping_batch)
146{
147 memset(mapping_batch, 0, sizeof(*mapping_batch));
148 mapping_batch->gpu_l2_flushed = false;
149 mapping_batch->need_tlb_invalidate = false;
150}
151
152void nvgpu_vm_mapping_batch_finish_locked(
153 struct vm_gk20a *vm, struct vm_gk20a_mapping_batch *mapping_batch)
154{
155 /* hanging kref_put batch pointer? */
156 WARN_ON(vm->kref_put_batch == mapping_batch);
157
158 if (mapping_batch->need_tlb_invalidate) {
159 struct gk20a *g = gk20a_from_vm(vm);
160 g->ops.fb.tlb_invalidate(g, vm->pdb.mem);
161 }
162}
163
164void nvgpu_vm_mapping_batch_finish(struct vm_gk20a *vm,
165 struct vm_gk20a_mapping_batch *mapping_batch)
166{
167 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
168 nvgpu_vm_mapping_batch_finish_locked(vm, mapping_batch);
169 nvgpu_mutex_release(&vm->update_gmmu_lock);
170}
171
172/*
173 * Determine if the passed address space can support big pages or not.
174 */
175int nvgpu_big_pages_possible(struct vm_gk20a *vm, u64 base, u64 size)
176{
177 u64 mask = ((u64)vm->big_page_size << 10) - 1;
178
179 if (base & mask || size & mask)
180 return 0;
181 return 1;
182}
183
184/*
185 * Initialize a semaphore pool. Just return successfully if we do not need
186 * semaphores (i.e when sync-pts are active).
187 */
188static int nvgpu_init_sema_pool(struct vm_gk20a *vm)
189{
190 struct nvgpu_semaphore_sea *sema_sea;
191 struct mm_gk20a *mm = vm->mm;
192 struct gk20a *g = mm->g;
193 int err;
194
195 /*
196 * Don't waste the memory on semaphores if we don't need them.
197 */
198 if (nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS))
199 return 0;
200
201 if (vm->sema_pool)
202 return 0;
203
204 sema_sea = nvgpu_semaphore_sea_create(g);
205 if (!sema_sea)
206 return -ENOMEM;
207
208 vm->sema_pool = nvgpu_semaphore_pool_alloc(sema_sea);
209 if (!vm->sema_pool)
210 return -ENOMEM;
211
212 /*
213 * Allocate a chunk of GPU VA space for mapping the semaphores. We will
214 * do a fixed alloc in the kernel VM so that all channels have the same
215 * RO address range for the semaphores.
216 *
217 * !!! TODO: cleanup.
218 */
219 sema_sea->gpu_va = nvgpu_alloc_fixed(&vm->kernel,
220 vm->va_limit -
221 mm->channel.kernel_size,
222 512 * PAGE_SIZE,
223 SZ_4K);
224 if (!sema_sea->gpu_va) {
225 nvgpu_free(&vm->kernel, sema_sea->gpu_va);
226 nvgpu_vm_put(vm);
227 return -ENOMEM;
228 }
229
230 err = nvgpu_semaphore_pool_map(vm->sema_pool, vm);
231 if (err) {
232 nvgpu_semaphore_pool_unmap(vm->sema_pool, vm);
233 nvgpu_free(vm->vma[gmmu_page_size_small],
234 vm->sema_pool->gpu_va);
235 return err;
236 }
237
238 return 0;
239}
240
241static int __nvgpu_vm_init(struct mm_gk20a *mm,
242 struct vm_gk20a *vm,
243 u32 big_page_size,
244 u64 low_hole,
245 u64 kernel_reserved,
246 u64 aperture_size,
247 bool big_pages,
248 bool userspace_managed,
249 char *name)
250{
251 int err;
252 char alloc_name[32];
253 u64 kernel_vma_flags;
254 u64 user_vma_start, user_vma_limit;
255 u64 user_lp_vma_start, user_lp_vma_limit;
256 u64 kernel_vma_start, kernel_vma_limit;
257 struct gk20a *g = gk20a_from_mm(mm);
258
259 if (WARN_ON(kernel_reserved + low_hole > aperture_size))
260 return -ENOMEM;
261
262 nvgpu_log_info(g, "Init space for %s: valimit=0x%llx, "
263 "LP size=0x%x lowhole=0x%llx",
264 name, aperture_size,
265 (unsigned int)big_page_size, low_hole);
266
267 vm->mm = mm;
268
269 vm->gmmu_page_sizes[gmmu_page_size_small] = SZ_4K;
270 vm->gmmu_page_sizes[gmmu_page_size_big] = big_page_size;
271 vm->gmmu_page_sizes[gmmu_page_size_kernel] = SZ_4K;
272
273 /* Set up vma pointers. */
274 vm->vma[gmmu_page_size_small] = &vm->user;
275 vm->vma[gmmu_page_size_big] = &vm->user;
276 vm->vma[gmmu_page_size_kernel] = &vm->kernel;
277 if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES))
278 vm->vma[gmmu_page_size_big] = &vm->user_lp;
279
280 vm->va_start = low_hole;
281 vm->va_limit = aperture_size;
282
283 vm->big_page_size = vm->gmmu_page_sizes[gmmu_page_size_big];
284 vm->userspace_managed = userspace_managed;
285 vm->mmu_levels = g->ops.mm.get_mmu_levels(g, vm->big_page_size);
286
287#ifdef CONFIG_TEGRA_GR_VIRTUALIZATION
288 if (g->is_virtual && userspace_managed) {
289 nvgpu_err(g, "vGPU: no userspace managed addr space support");
290 return -ENOSYS;
291 }
292 if (g->is_virtual && vgpu_vm_init(g, vm)) {
293 nvgpu_err(g, "Failed to init vGPU VM!");
294 return -ENOMEM;
295 }
296#endif
297
298 /* Initialize the page table data structures. */
299 strncpy(vm->name, name, min(strlen(name), sizeof(vm->name)));
300 err = nvgpu_gmmu_init_page_table(vm);
301 if (err)
302 goto clean_up_vgpu_vm;
303
304 /* Setup vma limits. */
305 if (kernel_reserved + low_hole < aperture_size) {
306 /*
307 * If big_pages are disabled for this VM then it only makes
308 * sense to make one VM, same as if the unified address flag
309 * is set.
310 */
311 if (!big_pages ||
312 nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
313 user_vma_start = low_hole;
314 user_vma_limit = vm->va_limit - kernel_reserved;
315 user_lp_vma_start = user_vma_limit;
316 user_lp_vma_limit = user_vma_limit;
317 } else {
318 user_vma_start = low_hole;
319 user_vma_limit = __nv_gmmu_va_small_page_limit();
320 user_lp_vma_start = __nv_gmmu_va_small_page_limit();
321 user_lp_vma_limit = vm->va_limit - kernel_reserved;
322 }
323 } else {
324 user_vma_start = 0;
325 user_vma_limit = 0;
326 user_lp_vma_start = 0;
327 user_lp_vma_limit = 0;
328 }
329 kernel_vma_start = vm->va_limit - kernel_reserved;
330 kernel_vma_limit = vm->va_limit;
331
332 nvgpu_log_info(g, "user_vma [0x%llx,0x%llx)",
333 user_vma_start, user_vma_limit);
334 nvgpu_log_info(g, "user_lp_vma [0x%llx,0x%llx)",
335 user_lp_vma_start, user_lp_vma_limit);
336 nvgpu_log_info(g, "kernel_vma [0x%llx,0x%llx)",
337 kernel_vma_start, kernel_vma_limit);
338
339 if (WARN_ON(user_vma_start > user_vma_limit) ||
340 WARN_ON(user_lp_vma_start > user_lp_vma_limit) ||
341 WARN_ON(kernel_vma_start >= kernel_vma_limit)) {
342 err = -EINVAL;
343 goto clean_up_page_tables;
344 }
345
346 kernel_vma_flags = (kernel_reserved + low_hole) == aperture_size ?
347 0 : GPU_ALLOC_GVA_SPACE;
348
349 /*
350 * A "user" area only makes sense for the GVA spaces. For VMs where
351 * there is no "user" area user_vma_start will be equal to
352 * user_vma_limit (i.e a 0 sized space). In such a situation the kernel
353 * area must be non-zero in length.
354 */
355 if (user_vma_start >= user_vma_limit &&
356 kernel_vma_start >= kernel_vma_limit) {
357 err = -EINVAL;
358 goto clean_up_page_tables;
359 }
360
361 /*
362 * Determine if big pages are possible in this VM. If a split address
363 * space is used then check the user_lp vma instead of the user vma.
364 */
365 if (nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES))
366 vm->big_pages = big_pages &&
367 nvgpu_big_pages_possible(vm, user_vma_start,
368 user_vma_limit - user_vma_start);
369 else
370 vm->big_pages = big_pages &&
371 nvgpu_big_pages_possible(vm, user_lp_vma_start,
372 user_lp_vma_limit - user_lp_vma_start);
373
374 /*
375 * User VMA.
376 */
377 if (user_vma_start < user_vma_limit) {
378 snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s", name);
379 err = __nvgpu_buddy_allocator_init(g, &vm->user,
380 vm, alloc_name,
381 user_vma_start,
382 user_vma_limit -
383 user_vma_start,
384 SZ_4K,
385 GPU_BALLOC_MAX_ORDER,
386 GPU_ALLOC_GVA_SPACE);
387 if (err)
388 goto clean_up_page_tables;
389 } else {
390 /*
391 * Make these allocator pointers point to the kernel allocator
392 * since we still use the legacy notion of page size to choose
393 * the allocator.
394 */
395 vm->vma[0] = &vm->kernel;
396 vm->vma[1] = &vm->kernel;
397 }
398
399 /*
400 * User VMA for large pages when a split address range is used.
401 */
402 if (user_lp_vma_start < user_lp_vma_limit) {
403 snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s_lp", name);
404 err = __nvgpu_buddy_allocator_init(g, &vm->user_lp,
405 vm, alloc_name,
406 user_lp_vma_start,
407 user_lp_vma_limit -
408 user_lp_vma_start,
409 vm->big_page_size,
410 GPU_BALLOC_MAX_ORDER,
411 GPU_ALLOC_GVA_SPACE);
412 if (err)
413 goto clean_up_allocators;
414 }
415
416 /*
417 * Kernel VMA. Must always exist for an address space.
418 */
419 snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-sys", name);
420 err = __nvgpu_buddy_allocator_init(g, &vm->kernel,
421 vm, alloc_name,
422 kernel_vma_start,
423 kernel_vma_limit - kernel_vma_start,
424 SZ_4K,
425 GPU_BALLOC_MAX_ORDER,
426 kernel_vma_flags);
427 if (err)
428 goto clean_up_allocators;
429
430 vm->mapped_buffers = NULL;
431
432 nvgpu_mutex_init(&vm->update_gmmu_lock);
433 nvgpu_ref_init(&vm->ref);
434 nvgpu_init_list_node(&vm->vm_area_list);
435
436 /*
437 * This is only necessary for channel address spaces. The best way to
438 * distinguish channel address spaces from other address spaces is by
439 * size - if the address space is 4GB or less, it's not a channel.
440 */
441 if (vm->va_limit > SZ_4G) {
442 err = nvgpu_init_sema_pool(vm);
443 if (err)
444 goto clean_up_allocators;
445 }
446
447 return 0;
448
449clean_up_allocators:
450 if (nvgpu_alloc_initialized(&vm->kernel))
451 nvgpu_alloc_destroy(&vm->kernel);
452 if (nvgpu_alloc_initialized(&vm->user))
453 nvgpu_alloc_destroy(&vm->user);
454 if (nvgpu_alloc_initialized(&vm->user_lp))
455 nvgpu_alloc_destroy(&vm->user_lp);
456clean_up_page_tables:
457 /* Cleans up nvgpu_gmmu_init_page_table() */
458 __nvgpu_pd_cache_free_direct(g, &vm->pdb);
459clean_up_vgpu_vm:
460#ifdef CONFIG_TEGRA_GR_VIRTUALIZATION
461 if (g->is_virtual)
462 vgpu_vm_remove(vm);
463#endif
464 return err;
465}
466
467/**
468 * nvgpu_init_vm() - Initialize an address space.
469 *
470 * @mm - Parent MM.
471 * @vm - The VM to init.
472 * @big_page_size - Size of big pages associated with this VM.
473 * @low_hole - The size of the low hole (unaddressable memory at the bottom of
474 * the address space).
475 * @kernel_reserved - Space reserved for kernel only allocations.
476 * @aperture_size - Total size of the aperture.
477 * @big_pages - If true then big pages are possible in the VM. Note this does
478 * not guarantee that big pages will be possible.
479 * @name - Name of the address space.
480 *
481 * This function initializes an address space according to the following map:
482 *
483 * +--+ 0x0
484 * | |
485 * +--+ @low_hole
486 * | |
487 * ~ ~ This is the "user" section.
488 * | |
489 * +--+ @aperture_size - @kernel_reserved
490 * | |
491 * ~ ~ This is the "kernel" section.
492 * | |
493 * +--+ @aperture_size
494 *
495 * The user section is therefor what ever is left over after the @low_hole and
496 * @kernel_reserved memory have been portioned out. The @kernel_reserved is
497 * always persent at the top of the memory space and the @low_hole is always at
498 * the bottom.
499 *
500 * For certain address spaces a "user" section makes no sense (bar1, etc) so in
501 * such cases the @kernel_reserved and @low_hole should sum to exactly
502 * @aperture_size.
503 */
504struct vm_gk20a *nvgpu_vm_init(struct gk20a *g,
505 u32 big_page_size,
506 u64 low_hole,
507 u64 kernel_reserved,
508 u64 aperture_size,
509 bool big_pages,
510 bool userspace_managed,
511 char *name)
512{
513 struct vm_gk20a *vm = nvgpu_kzalloc(g, sizeof(*vm));
514
515 if (!vm)
516 return NULL;
517
518 if (__nvgpu_vm_init(&g->mm, vm, big_page_size, low_hole,
519 kernel_reserved, aperture_size, big_pages,
520 userspace_managed, name)) {
521 nvgpu_kfree(g, vm);
522 return NULL;
523 }
524
525 return vm;
526}
527
528/*
529 * Cleanup the VM!
530 */
531static void __nvgpu_vm_remove(struct vm_gk20a *vm)
532{
533 struct nvgpu_mapped_buf *mapped_buffer;
534 struct nvgpu_vm_area *vm_area, *vm_area_tmp;
535 struct nvgpu_rbtree_node *node = NULL;
536 struct gk20a *g = vm->mm->g;
537
538 /*
539 * Do this outside of the update_gmmu_lock since unmapping the semaphore
540 * pool involves unmapping a GMMU mapping which means aquiring the
541 * update_gmmu_lock.
542 */
543 if (!nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS)) {
544 if (vm->sema_pool) {
545 nvgpu_semaphore_pool_unmap(vm->sema_pool, vm);
546 nvgpu_semaphore_pool_put(vm->sema_pool);
547 }
548 }
549
550#if defined(CONFIG_TEGRA_GK20A_NVHOST) && defined(CONFIG_TEGRA_19x_GPU)
551 if (nvgpu_mem_is_valid(&g->syncpt_mem) && vm->syncpt_ro_map_gpu_va)
552 nvgpu_gmmu_unmap(vm, &g->syncpt_mem,
553 vm->syncpt_ro_map_gpu_va);
554#endif
555
556 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
557
558 nvgpu_rbtree_enum_start(0, &node, vm->mapped_buffers);
559 while (node) {
560 mapped_buffer = mapped_buffer_from_rbtree_node(node);
561 __nvgpu_vm_unmap(mapped_buffer, NULL);
562 nvgpu_rbtree_enum_start(0, &node, vm->mapped_buffers);
563 }
564
565 /* destroy remaining reserved memory areas */
566 nvgpu_list_for_each_entry_safe(vm_area, vm_area_tmp,
567 &vm->vm_area_list,
568 nvgpu_vm_area, vm_area_list) {
569 nvgpu_list_del(&vm_area->vm_area_list);
570 nvgpu_kfree(vm->mm->g, vm_area);
571 }
572
573 if (nvgpu_alloc_initialized(&vm->kernel))
574 nvgpu_alloc_destroy(&vm->kernel);
575 if (nvgpu_alloc_initialized(&vm->user))
576 nvgpu_alloc_destroy(&vm->user);
577 if (nvgpu_alloc_initialized(&vm->user_lp))
578 nvgpu_alloc_destroy(&vm->user_lp);
579
580 nvgpu_vm_free_entries(vm, &vm->pdb);
581
582#ifdef CONFIG_TEGRA_GR_VIRTUALIZATION
583 if (g->is_virtual)
584 vgpu_vm_remove(vm);
585#endif
586
587 nvgpu_mutex_release(&vm->update_gmmu_lock);
588
589 nvgpu_kfree(g, vm);
590}
591
592static void __nvgpu_vm_remove_ref(struct nvgpu_ref *ref)
593{
594 struct vm_gk20a *vm = container_of(ref, struct vm_gk20a, ref);
595
596 __nvgpu_vm_remove(vm);
597}
598
599void nvgpu_vm_get(struct vm_gk20a *vm)
600{
601 nvgpu_ref_get(&vm->ref);
602}
603
604void nvgpu_vm_put(struct vm_gk20a *vm)
605{
606 nvgpu_ref_put(&vm->ref, __nvgpu_vm_remove_ref);
607}
608
609int nvgpu_insert_mapped_buf(struct vm_gk20a *vm,
610 struct nvgpu_mapped_buf *mapped_buffer)
611{
612 mapped_buffer->node.key_start = mapped_buffer->addr;
613 mapped_buffer->node.key_end = mapped_buffer->addr + mapped_buffer->size;
614
615 nvgpu_rbtree_insert(&mapped_buffer->node, &vm->mapped_buffers);
616
617 return 0;
618}
619
620void nvgpu_remove_mapped_buf(struct vm_gk20a *vm,
621 struct nvgpu_mapped_buf *mapped_buffer)
622{
623 nvgpu_rbtree_unlink(&mapped_buffer->node, &vm->mapped_buffers);
624}
625
626struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf(
627 struct vm_gk20a *vm, u64 addr)
628{
629 struct nvgpu_rbtree_node *node = NULL;
630 struct nvgpu_rbtree_node *root = vm->mapped_buffers;
631
632 nvgpu_rbtree_search(addr, &node, root);
633 if (!node)
634 return NULL;
635
636 return mapped_buffer_from_rbtree_node(node);
637}
638
639struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_range(
640 struct vm_gk20a *vm, u64 addr)
641{
642 struct nvgpu_rbtree_node *node = NULL;
643 struct nvgpu_rbtree_node *root = vm->mapped_buffers;
644
645 nvgpu_rbtree_range_search(addr, &node, root);
646 if (!node)
647 return NULL;
648
649 return mapped_buffer_from_rbtree_node(node);
650}
651
652struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_less_than(
653 struct vm_gk20a *vm, u64 addr)
654{
655 struct nvgpu_rbtree_node *node = NULL;
656 struct nvgpu_rbtree_node *root = vm->mapped_buffers;
657
658 nvgpu_rbtree_less_than_search(addr, &node, root);
659 if (!node)
660 return NULL;
661
662 return mapped_buffer_from_rbtree_node(node);
663}
664
665int nvgpu_vm_get_buffers(struct vm_gk20a *vm,
666 struct nvgpu_mapped_buf ***mapped_buffers,
667 int *num_buffers)
668{
669 struct nvgpu_mapped_buf *mapped_buffer;
670 struct nvgpu_mapped_buf **buffer_list;
671 struct nvgpu_rbtree_node *node = NULL;
672 int i = 0;
673
674 if (vm->userspace_managed) {
675 *mapped_buffers = NULL;
676 *num_buffers = 0;
677 return 0;
678 }
679
680 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
681
682 buffer_list = nvgpu_big_zalloc(vm->mm->g, sizeof(*buffer_list) *
683 vm->num_user_mapped_buffers);
684 if (!buffer_list) {
685 nvgpu_mutex_release(&vm->update_gmmu_lock);
686 return -ENOMEM;
687 }
688
689 nvgpu_rbtree_enum_start(0, &node, vm->mapped_buffers);
690 while (node) {
691 mapped_buffer = mapped_buffer_from_rbtree_node(node);
692 buffer_list[i] = mapped_buffer;
693 nvgpu_ref_get(&mapped_buffer->ref);
694 i++;
695 nvgpu_rbtree_enum_next(&node, node);
696 }
697
698 BUG_ON(i != vm->num_user_mapped_buffers);
699
700 *num_buffers = vm->num_user_mapped_buffers;
701 *mapped_buffers = buffer_list;
702
703 nvgpu_mutex_release(&vm->update_gmmu_lock);
704
705 return 0;
706}
707
708void nvgpu_vm_put_buffers(struct vm_gk20a *vm,
709 struct nvgpu_mapped_buf **mapped_buffers,
710 int num_buffers)
711{
712 int i;
713 struct vm_gk20a_mapping_batch batch;
714
715 if (num_buffers == 0)
716 return;
717
718 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
719 nvgpu_vm_mapping_batch_start(&batch);
720 vm->kref_put_batch = &batch;
721
722 for (i = 0; i < num_buffers; ++i)
723 nvgpu_ref_put(&mapped_buffers[i]->ref, __nvgpu_vm_unmap_ref);
724
725 vm->kref_put_batch = NULL;
726 nvgpu_vm_mapping_batch_finish_locked(vm, &batch);
727 nvgpu_mutex_release(&vm->update_gmmu_lock);
728
729 nvgpu_big_free(vm->mm->g, mapped_buffers);
730}
731
732struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
733 struct nvgpu_os_buffer *os_buf,
734 struct nvgpu_sgt *sgt,
735 u64 map_addr,
736 u64 map_size,
737 u64 phys_offset,
738 int rw,
739 u32 flags,
740 s16 compr_kind,
741 s16 incompr_kind,
742 struct vm_gk20a_mapping_batch *batch,
743 enum nvgpu_aperture aperture)
744{
745 struct gk20a *g = gk20a_from_vm(vm);
746 struct nvgpu_mapped_buf *mapped_buffer = NULL;
747 struct nvgpu_ctag_buffer_info binfo = { 0 };
748 struct nvgpu_vm_area *vm_area = NULL;
749 int err = 0;
750 u64 align;
751 u32 ctag_offset = 0;
752 bool clear_ctags = false;
753 bool va_allocated = true;
754
755 /*
756 * The kind used as part of the key for map caching. HW may
757 * actually be programmed with the fallback kind in case the
758 * key kind is compressible but we're out of comptags.
759 */
760 s16 map_key_kind;
761
762 /*
763 * The actual GMMU PTE kind
764 */
765 u8 pte_kind;
766
767 if (vm->userspace_managed &&
768 !(flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET)) {
769 nvgpu_err(g,
770 "non-fixed-offset mapping not available on "
771 "userspace managed address spaces");
772 return ERR_PTR(-EINVAL);
773 }
774
775 binfo.flags = flags;
776 binfo.size = nvgpu_os_buf_get_size(os_buf);
777 binfo.compr_kind = (vm->enable_ctag && compr_kind != NV_KIND_INVALID ?
778 compr_kind : NV_KIND_INVALID);
779 binfo.incompr_kind = incompr_kind;
780
781 if (compr_kind != NV_KIND_INVALID)
782 map_key_kind = compr_kind;
783 else
784 map_key_kind = incompr_kind;
785
786 /*
787 * Check if this buffer is already mapped.
788 */
789 if (!vm->userspace_managed) {
790 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
791 mapped_buffer = nvgpu_vm_find_mapping(vm,
792 os_buf,
793 map_addr,
794 flags,
795 map_key_kind);
796 nvgpu_mutex_release(&vm->update_gmmu_lock);
797
798 if (mapped_buffer) {
799 nvgpu_ref_get(&mapped_buffer->ref);
800 return mapped_buffer;
801 }
802 }
803
804 /*
805 * Generate a new mapping!
806 */
807 mapped_buffer = nvgpu_kzalloc(g, sizeof(*mapped_buffer));
808 if (!mapped_buffer) {
809 nvgpu_warn(g, "oom allocating tracking buffer");
810 return ERR_PTR(-ENOMEM);
811 }
812
813 align = nvgpu_sgt_alignment(g, sgt);
814 if (g->mm.disable_bigpage)
815 binfo.pgsz_idx = gmmu_page_size_small;
816 else
817 binfo.pgsz_idx = __get_pte_size(vm, map_addr,
818 min_t(u64, binfo.size, align));
819 map_size = map_size ? map_size : binfo.size;
820 map_size = ALIGN(map_size, SZ_4K);
821
822 if ((map_size > binfo.size) ||
823 (phys_offset > (binfo.size - map_size))) {
824 err = -EINVAL;
825 goto clean_up_nolock;
826 }
827
828 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
829
830 /*
831 * Check if we should use a fixed offset for mapping this buffer.
832 */
833 if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) {
834 err = nvgpu_vm_area_validate_buffer(vm,
835 map_addr,
836 map_size,
837 binfo.pgsz_idx,
838 &vm_area);
839 if (err)
840 goto clean_up;
841
842 va_allocated = false;
843 }
844
845 err = nvgpu_vm_compute_compression(vm, &binfo);
846 if (err) {
847 nvgpu_err(g, "failure setting up compression");
848 goto clean_up;
849 }
850
851 if (binfo.compr_kind != NV_KIND_INVALID) {
852 struct gk20a_comptags comptags = { 0 };
853
854 /*
855 * Get the comptags state, alloc if necessary
856 */
857 err = gk20a_alloc_or_get_comptags(g, os_buf,
858 &g->gr.comp_tags,
859 &comptags);
860 if (err) {
861 /*
862 * This is an irrecoverable failure and we need to
863 * abort. In particular, it is not safe to proceed with
864 * the incompressible fallback, since we cannot not mark
865 * our alloc failure anywere. Later we would retry
866 * allocation and break compressible map aliasing.
867 */
868 nvgpu_err(g, "Error %d setting up comptags", err);
869 goto clean_up;
870 }
871
872 /*
873 * Newly allocated comptags needs to be cleared
874 */
875 if (comptags.needs_clear) {
876 if (g->ops.ltc.cbc_ctrl) {
877 if (gk20a_comptags_start_clear(os_buf)) {
878 err = g->ops.ltc.cbc_ctrl(
879 g, gk20a_cbc_op_clear,
880 comptags.offset,
881 (comptags.offset +
882 comptags.lines - 1));
883 gk20a_comptags_finish_clear(
884 os_buf, err == 0);
885 if (err)
886 goto clean_up;
887 }
888 } else {
889 /*
890 * Cleared as part of gmmu map
891 */
892 clear_ctags = true;
893 }
894 }
895
896 /*
897 * Store the ctag offset for later use if we got the comptags
898 */
899 if (comptags.lines)
900 ctag_offset = comptags.offset;
901 }
902
903 /*
904 * Figure out the kind and ctag offset for the GMMU page tables
905 */
906 if (binfo.compr_kind != NV_KIND_INVALID && ctag_offset) {
907 /*
908 * Adjust the ctag_offset as per the buffer map offset
909 */
910 ctag_offset += phys_offset >>
911 ilog2(g->ops.fb.compression_page_size(g));
912 pte_kind = binfo.compr_kind;
913 } else if (binfo.incompr_kind != NV_KIND_INVALID) {
914 /*
915 * Incompressible kind, ctag offset will not be programmed
916 */
917 ctag_offset = 0;
918 pte_kind = binfo.incompr_kind;
919 } else {
920 /*
921 * Caller required compression, but we cannot provide it
922 */
923 nvgpu_err(g, "No comptags and no incompressible fallback kind");
924 err = -ENOMEM;
925 goto clean_up;
926 }
927
928 if (clear_ctags)
929 clear_ctags = gk20a_comptags_start_clear(os_buf);
930
931 map_addr = g->ops.mm.gmmu_map(vm,
932 map_addr,
933 sgt,
934 phys_offset,
935 map_size,
936 binfo.pgsz_idx,
937 pte_kind,
938 ctag_offset,
939 flags,
940 rw,
941 clear_ctags,
942 false,
943 false,
944 batch,
945 aperture);
946
947 if (clear_ctags)
948 gk20a_comptags_finish_clear(os_buf, map_addr != 0);
949
950 if (!map_addr) {
951 err = -ENOMEM;
952 goto clean_up;
953 }
954
955 nvgpu_init_list_node(&mapped_buffer->buffer_list);
956 nvgpu_ref_init(&mapped_buffer->ref);
957 mapped_buffer->addr = map_addr;
958 mapped_buffer->size = map_size;
959 mapped_buffer->pgsz_idx = binfo.pgsz_idx;
960 mapped_buffer->vm = vm;
961 mapped_buffer->flags = flags;
962 mapped_buffer->kind = map_key_kind;
963 mapped_buffer->va_allocated = va_allocated;
964 mapped_buffer->vm_area = vm_area;
965
966 err = nvgpu_insert_mapped_buf(vm, mapped_buffer);
967 if (err) {
968 nvgpu_err(g, "failed to insert into mapped buffer tree");
969 goto clean_up;
970 }
971
972 vm->num_user_mapped_buffers++;
973
974 if (vm_area) {
975 nvgpu_list_add_tail(&mapped_buffer->buffer_list,
976 &vm_area->buffer_list_head);
977 mapped_buffer->vm_area = vm_area;
978 }
979
980 nvgpu_mutex_release(&vm->update_gmmu_lock);
981
982 return mapped_buffer;
983
984clean_up:
985 if (mapped_buffer->addr)
986 g->ops.mm.gmmu_unmap(vm,
987 mapped_buffer->addr,
988 mapped_buffer->size,
989 mapped_buffer->pgsz_idx,
990 mapped_buffer->va_allocated,
991 gk20a_mem_flag_none,
992 mapped_buffer->vm_area ?
993 mapped_buffer->vm_area->sparse : false,
994 NULL);
995 nvgpu_mutex_release(&vm->update_gmmu_lock);
996clean_up_nolock:
997 nvgpu_kfree(g, mapped_buffer);
998
999 return ERR_PTR(err);
1000}
1001
1002/*
1003 * Really unmap. This does the real GMMU unmap and removes the mapping from the
1004 * VM map tracking tree (and vm_area list if necessary).
1005 */
1006static void __nvgpu_vm_unmap(struct nvgpu_mapped_buf *mapped_buffer,
1007 struct vm_gk20a_mapping_batch *batch)
1008{
1009 struct vm_gk20a *vm = mapped_buffer->vm;
1010 struct gk20a *g = vm->mm->g;
1011
1012 vm->num_user_mapped_buffers--;
1013
1014 g->ops.mm.gmmu_unmap(vm,
1015 mapped_buffer->addr,
1016 mapped_buffer->size,
1017 mapped_buffer->pgsz_idx,
1018 mapped_buffer->va_allocated,
1019 gk20a_mem_flag_none,
1020 mapped_buffer->vm_area ?
1021 mapped_buffer->vm_area->sparse : false,
1022 batch);
1023
1024 /*
1025 * Remove from mapped buffer tree. Then delete the buffer from the
1026 * linked list of mapped buffers; though note: not all mapped buffers
1027 * are part of a vm_area.
1028 */
1029 nvgpu_remove_mapped_buf(vm, mapped_buffer);
1030 nvgpu_list_del(&mapped_buffer->buffer_list);
1031
1032 /*
1033 * OS specific freeing. This is after the generic freeing incase the
1034 * generic freeing relies on some component of the OS specific
1035 * nvgpu_mapped_buf in some abstraction or the like.
1036 */
1037 nvgpu_vm_unmap_system(mapped_buffer);
1038
1039 nvgpu_kfree(g, mapped_buffer);
1040}
1041
1042void __nvgpu_vm_unmap_ref(struct nvgpu_ref *ref)
1043{
1044 struct nvgpu_mapped_buf *mapped_buffer =
1045 container_of(ref, struct nvgpu_mapped_buf, ref);
1046
1047 __nvgpu_vm_unmap(mapped_buffer, mapped_buffer->vm->kref_put_batch);
1048}
1049
1050/*
1051 * For fixed-offset buffers we must sync the buffer. That means we wait for the
1052 * buffer to hit a ref-count of 1 before proceeding.
1053 *
1054 * Note: this requires the update_gmmu_lock to be held since we release it and
1055 * re-aquire it in this function.
1056 */
1057static int nvgpu_vm_unmap_sync_buffer(struct vm_gk20a *vm,
1058 struct nvgpu_mapped_buf *mapped_buffer)
1059{
1060 struct nvgpu_timeout timeout;
1061 int ret = 0;
1062
1063 nvgpu_mutex_release(&vm->update_gmmu_lock);
1064
1065 /*
1066 * 500ms second timer.
1067 */
1068 nvgpu_timeout_init(vm->mm->g, &timeout, 50, NVGPU_TIMER_CPU_TIMER);
1069
1070 do {
1071 if (nvgpu_atomic_read(&mapped_buffer->ref.refcount) == 1)
1072 break;
1073 nvgpu_msleep(10);
1074 } while (!nvgpu_timeout_expired_msg(&timeout,
1075 "sync-unmap failed on 0x%llx"));
1076
1077 if (nvgpu_timeout_expired(&timeout))
1078 ret = -ETIMEDOUT;
1079
1080 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
1081
1082 return ret;
1083}
1084
1085void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset,
1086 struct vm_gk20a_mapping_batch *batch)
1087{
1088 struct nvgpu_mapped_buf *mapped_buffer;
1089
1090 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
1091
1092 mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, offset);
1093 if (!mapped_buffer)
1094 goto done;
1095
1096 if (mapped_buffer->flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) {
1097 if (nvgpu_vm_unmap_sync_buffer(vm, mapped_buffer))
1098 /*
1099 * Looks like we have failed... Better not continue in
1100 * case the buffer is in use.
1101 */
1102 goto done;
1103 }
1104
1105 /*
1106 * Make sure we have access to the batch if we end up calling through to
1107 * the unmap_ref function.
1108 */
1109 vm->kref_put_batch = batch;
1110 nvgpu_ref_put(&mapped_buffer->ref, __nvgpu_vm_unmap_ref);
1111 vm->kref_put_batch = NULL;
1112
1113done:
1114 nvgpu_mutex_release(&vm->update_gmmu_lock);
1115 return;
1116}
1117
1118static int nvgpu_vm_compute_compression(struct vm_gk20a *vm,
1119 struct nvgpu_ctag_buffer_info *binfo)
1120{
1121 bool kind_compressible = (binfo->compr_kind != NV_KIND_INVALID);
1122 struct gk20a *g = gk20a_from_vm(vm);
1123
1124 if (kind_compressible &&
1125 vm->gmmu_page_sizes[binfo->pgsz_idx] <
1126 g->ops.fb.compressible_page_size(g)) {
1127 /*
1128 * Let's double check that there is a fallback kind
1129 */
1130 if (binfo->incompr_kind == NV_KIND_INVALID) {
1131 nvgpu_err(g,
1132 "Unsupported page size for compressible "
1133 "kind, but no fallback kind");
1134 return -EINVAL;
1135 } else {
1136 nvgpu_log(g, gpu_dbg_map,
1137 "Unsupported page size for compressible "
1138 "kind, demoting to incompressible");
1139 binfo->compr_kind = NV_KIND_INVALID;
1140 kind_compressible = false;
1141 }
1142 }
1143
1144 return 0;
1145}
diff --git a/drivers/gpu/nvgpu/common/mm/vm_area.c b/drivers/gpu/nvgpu/common/mm/vm_area.c
new file mode 100644
index 00000000..b6286c43
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/mm/vm_area.c
@@ -0,0 +1,231 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <uapi/linux/nvgpu.h>
24
25#include <nvgpu/vm.h>
26#include <nvgpu/vm_area.h>
27
28#include "gk20a/gk20a.h"
29#include "gk20a/mm_gk20a.h"
30
31struct nvgpu_vm_area *nvgpu_vm_area_find(struct vm_gk20a *vm, u64 addr)
32{
33 struct nvgpu_vm_area *vm_area;
34
35 nvgpu_list_for_each_entry(vm_area, &vm->vm_area_list,
36 nvgpu_vm_area, vm_area_list) {
37 if (addr >= vm_area->addr &&
38 addr < (u64)vm_area->addr + (u64)vm_area->size)
39 return vm_area;
40 }
41
42 return NULL;
43}
44
45int nvgpu_vm_area_validate_buffer(struct vm_gk20a *vm,
46 u64 map_addr, u64 map_size, int pgsz_idx,
47 struct nvgpu_vm_area **pvm_area)
48{
49 struct gk20a *g = vm->mm->g;
50 struct nvgpu_vm_area *vm_area;
51 struct nvgpu_mapped_buf *buffer;
52 u64 map_end = map_addr + map_size;
53
54 /* can wrap around with insane map_size; zero is disallowed too */
55 if (map_end <= map_addr) {
56 nvgpu_warn(g, "fixed offset mapping with invalid map_size");
57 return -EINVAL;
58 }
59
60 if (map_addr & (vm->gmmu_page_sizes[pgsz_idx] - 1)) {
61 nvgpu_err(g, "map offset must be buffer page size aligned 0x%llx",
62 map_addr);
63 return -EINVAL;
64 }
65
66 /* Find the space reservation, but it's ok to have none for
67 * userspace-managed address spaces */
68 vm_area = nvgpu_vm_area_find(vm, map_addr);
69 if (!vm_area && !vm->userspace_managed) {
70 nvgpu_warn(g, "fixed offset mapping without space allocation");
71 return -EINVAL;
72 }
73
74 /* Mapped area should fit inside va, if there's one */
75 if (vm_area && map_end > vm_area->addr + vm_area->size) {
76 nvgpu_warn(g, "fixed offset mapping size overflows va node");
77 return -EINVAL;
78 }
79
80 /* check that this mapping does not collide with existing
81 * mappings by checking the buffer with the highest GPU VA
82 * that is less than our buffer end */
83 buffer = __nvgpu_vm_find_mapped_buf_less_than(
84 vm, map_addr + map_size);
85 if (buffer && buffer->addr + buffer->size > map_addr) {
86 nvgpu_warn(g, "overlapping buffer map requested");
87 return -EINVAL;
88 }
89
90 *pvm_area = vm_area;
91
92 return 0;
93}
94
95int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u32 pages, u32 page_size,
96 u64 *addr, u32 flags)
97{
98 struct gk20a *g = vm->mm->g;
99 struct nvgpu_allocator *vma;
100 struct nvgpu_vm_area *vm_area;
101 u64 vaddr_start = 0;
102 int pgsz_idx = gmmu_page_size_small;
103
104 nvgpu_log(g, gpu_dbg_map,
105 "ADD vm_area: pgsz=%#-8x pages=%-9u addr=%#-14llx flags=0x%x",
106 page_size, pages, *addr, flags);
107
108 for (; pgsz_idx < gmmu_nr_page_sizes; pgsz_idx++) {
109 if (vm->gmmu_page_sizes[pgsz_idx] == page_size)
110 break;
111 }
112
113 if (pgsz_idx > gmmu_page_size_big)
114 return -EINVAL;
115
116 if (!vm->big_pages && pgsz_idx == gmmu_page_size_big)
117 return -EINVAL;
118
119 vm_area = nvgpu_kzalloc(g, sizeof(*vm_area));
120 if (!vm_area)
121 goto clean_up_err;
122
123 vma = vm->vma[pgsz_idx];
124 if (flags & NVGPU_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET)
125 vaddr_start = nvgpu_alloc_fixed(vma, *addr,
126 (u64)pages *
127 (u64)page_size,
128 page_size);
129 else
130 vaddr_start = nvgpu_alloc(vma,
131 (u64)pages *
132 (u64)page_size);
133
134 if (!vaddr_start)
135 goto clean_up_err;
136
137 vm_area->flags = flags;
138 vm_area->addr = vaddr_start;
139 vm_area->size = (u64)page_size * (u64)pages;
140 vm_area->pgsz_idx = pgsz_idx;
141 nvgpu_init_list_node(&vm_area->buffer_list_head);
142 nvgpu_init_list_node(&vm_area->vm_area_list);
143
144 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
145
146 if (flags & NVGPU_AS_ALLOC_SPACE_FLAGS_SPARSE) {
147 u64 map_addr = g->ops.mm.gmmu_map(vm, vaddr_start,
148 NULL,
149 0,
150 vm_area->size,
151 pgsz_idx,
152 0,
153 0,
154 flags,
155 gk20a_mem_flag_none,
156 false,
157 true,
158 false,
159 NULL,
160 APERTURE_INVALID);
161 if (!map_addr) {
162 nvgpu_mutex_release(&vm->update_gmmu_lock);
163 goto clean_up_err;
164 }
165
166 vm_area->sparse = true;
167 }
168 nvgpu_list_add_tail(&vm_area->vm_area_list, &vm->vm_area_list);
169
170 nvgpu_mutex_release(&vm->update_gmmu_lock);
171
172 *addr = vaddr_start;
173 return 0;
174
175clean_up_err:
176 if (vaddr_start)
177 nvgpu_free(vma, vaddr_start);
178 if (vm_area)
179 nvgpu_kfree(g, vm_area);
180 return -ENOMEM;
181}
182
183int nvgpu_vm_area_free(struct vm_gk20a *vm, u64 addr)
184{
185 struct gk20a *g = gk20a_from_vm(vm);
186 struct nvgpu_mapped_buf *buffer, *n;
187 struct nvgpu_vm_area *vm_area;
188
189 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
190 vm_area = nvgpu_vm_area_find(vm, addr);
191 if (!vm_area) {
192 nvgpu_mutex_release(&vm->update_gmmu_lock);
193 return 0;
194 }
195 nvgpu_list_del(&vm_area->vm_area_list);
196 nvgpu_mutex_release(&vm->update_gmmu_lock);
197
198 nvgpu_log(g, gpu_dbg_map,
199 "DEL vm_area: pgsz=%#-8x pages=%-9llu "
200 "addr=%#-14llx flags=0x%x",
201 vm->gmmu_page_sizes[vm_area->pgsz_idx],
202 vm_area->size / vm->gmmu_page_sizes[vm_area->pgsz_idx],
203 vm_area->addr,
204 vm_area->flags);
205
206 /* Decrement the ref count on all buffers in this vm_area. This
207 * allows userspace to let the kernel free mappings that are
208 * only used by this vm_area. */
209 nvgpu_list_for_each_entry_safe(buffer, n,
210 &vm_area->buffer_list_head,
211 nvgpu_mapped_buf, buffer_list) {
212 nvgpu_list_del(&buffer->buffer_list);
213 nvgpu_ref_put(&buffer->ref, __nvgpu_vm_unmap_ref);
214 }
215
216 /* if this was a sparse mapping, free the va */
217 if (vm_area->sparse)
218 g->ops.mm.gmmu_unmap(vm,
219 vm_area->addr,
220 vm_area->size,
221 vm_area->pgsz_idx,
222 true,
223 gk20a_mem_flag_none,
224 true,
225 NULL);
226
227 nvgpu_free(vm->vma[vm_area->pgsz_idx], vm_area->addr);
228 nvgpu_kfree(g, vm_area);
229
230 return 0;
231}
generated by cgit v1.2.2 (git 2.25.0) at 2024-08-24 12:11:22 -0400