summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/gk20a
diff options
context:
space:
mode:
authorTerje Bergstrom <tbergstrom@nvidia.com>2015-05-11 22:28:11 -0400
committerHiroshi Doyu <hdoyu@nvidia.com>2015-05-12 05:46:39 -0400
commitaa25a952ea2b19a081fa746f043228c270f43f94 (patch)
treee640f945d82c598645b0b2c5cbc6eb194444026d /drivers/gpu/nvgpu/gk20a
parenta2e852364582e9c337f52bc53ccc33877c8f3b47 (diff)
Revert "gpu: nvgpu: New allocator for VA space"
This reverts commit 2e235ac150fa4af8632c9abf0f109a10973a0bf5. Change-Id: I3aa745152124c2bc09c6c6dc5aeb1084ae7e08a4 Signed-off-by: Terje Bergstrom <tbergstrom@nvidia.com> Reviewed-on: http://git-master/r/741469 Reviewed-by: Automatic_Commit_Validation_User Reviewed-by: Hiroshi Doyu <hdoyu@nvidia.com> Tested-by: Hiroshi Doyu <hdoyu@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a')
-rw-r--r--drivers/gpu/nvgpu/gk20a/as_gk20a.c15
-rw-r--r--drivers/gpu/nvgpu/gk20a/gk20a.c2
-rw-r--r--drivers/gpu/nvgpu/gk20a/gk20a_allocator.c1167
-rw-r--r--drivers/gpu/nvgpu/gk20a/gk20a_allocator.h213
-rw-r--r--drivers/gpu/nvgpu/gk20a/ltc_gk20a.c5
-rw-r--r--drivers/gpu/nvgpu/gk20a/mm_gk20a.c194
-rw-r--r--drivers/gpu/nvgpu/gk20a/mm_gk20a.h1
-rw-r--r--drivers/gpu/nvgpu/gk20a/pmu_gk20a.c68
-rw-r--r--drivers/gpu/nvgpu/gk20a/pmu_gk20a.h4
-rw-r--r--drivers/gpu/nvgpu/gk20a/semaphore_gk20a.c15
10 files changed, 316 insertions, 1368 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/as_gk20a.c b/drivers/gpu/nvgpu/gk20a/as_gk20a.c
index d832b792..038fa4c8 100644
--- a/drivers/gpu/nvgpu/gk20a/as_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/as_gk20a.c
@@ -199,14 +199,21 @@ static int gk20a_as_ioctl_get_va_regions(
199 199
200 for (i = 0; i < write_entries; ++i) { 200 for (i = 0; i < write_entries; ++i) {
201 struct nvgpu_as_va_region region; 201 struct nvgpu_as_va_region region;
202 u32 base, limit;
202 203
203 memset(&region, 0, sizeof(struct nvgpu_as_va_region)); 204 memset(&region, 0, sizeof(struct nvgpu_as_va_region));
204 205
206 if (!vm->vma[i].constraint.enable) {
207 base = vm->vma[i].base;
208 limit = vm->vma[i].limit;
209 } else {
210 base = vm->vma[i].constraint.base;
211 limit = vm->vma[i].constraint.limit;
212 }
213
205 region.page_size = vm->gmmu_page_sizes[i]; 214 region.page_size = vm->gmmu_page_sizes[i];
206 region.offset = vm->vma[i].base; 215 region.offset = (u64)base * region.page_size;
207 /* No __aeabi_uldivmod() on some platforms... */ 216 region.pages = limit - base; /* NOTE: limit is exclusive */
208 region.pages = (vm->vma[i].end - vm->vma[i].start) >>
209 ilog2(region.page_size);
210 217
211 if (copy_to_user(user_region_ptr + i, &region, sizeof(region))) 218 if (copy_to_user(user_region_ptr + i, &region, sizeof(region)))
212 return -EFAULT; 219 return -EFAULT;
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c
index eb52f28d..f6c9f901 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.c
@@ -59,7 +59,6 @@
59#include "hw_fb_gk20a.h" 59#include "hw_fb_gk20a.h"
60#include "gk20a_scale.h" 60#include "gk20a_scale.h"
61#include "dbg_gpu_gk20a.h" 61#include "dbg_gpu_gk20a.h"
62#include "gk20a_allocator.h"
63#include "hal.h" 62#include "hal.h"
64#include "vgpu/vgpu.h" 63#include "vgpu/vgpu.h"
65 64
@@ -1511,7 +1510,6 @@ static int gk20a_probe(struct platform_device *dev)
1511 &gk20a->mm.disable_bigpage); 1510 &gk20a->mm.disable_bigpage);
1512 gk20a_pmu_debugfs_init(dev); 1511 gk20a_pmu_debugfs_init(dev);
1513 gk20a_cde_debugfs_init(dev); 1512 gk20a_cde_debugfs_init(dev);
1514 gk20a_alloc_debugfs_init(dev);
1515#endif 1513#endif
1516 1514
1517 gk20a_init_gr(gk20a); 1515 gk20a_init_gr(gk20a);
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_allocator.c b/drivers/gpu/nvgpu/gk20a/gk20a_allocator.c
index 56fb22df..0037257c 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a_allocator.c
+++ b/drivers/gpu/nvgpu/gk20a/gk20a_allocator.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * gk20a allocator 2 * gk20a allocator
3 * 3 *
4 * Copyright (c) 2011-2015, NVIDIA CORPORATION. All rights reserved. 4 * Copyright (c) 2011-2014, NVIDIA CORPORATION. All rights reserved.
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify it 6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License, 7 * under the terms and conditions of the GNU General Public License,
@@ -16,1149 +16,112 @@
16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */ 17 */
18 18
19#include <linux/kernel.h>
20#include <linux/seq_file.h>
21#include <linux/slab.h>
22#include <linux/debugfs.h>
23
24#include "platform_gk20a.h"
25#include "gk20a_allocator.h" 19#include "gk20a_allocator.h"
20#include <linux/vmalloc.h>
26 21
27#include "mm_gk20a.h" 22/* init allocator struct */
28 23int gk20a_allocator_init(struct gk20a_allocator *allocator,
29static struct dentry *balloc_debugfs_root; 24 const char *name, u32 start, u32 len)
30
31static struct kmem_cache *buddy_cache; /* slab cache for meta data. */
32
33static u32 balloc_tracing_on;
34
35#define balloc_trace_func() \
36 do { \
37 if (balloc_tracing_on) \
38 trace_printk("%s\n", __func__); \
39 } while (0)
40
41#define balloc_trace_func_done() \
42 do { \
43 if (balloc_tracing_on) \
44 trace_printk("%s_done\n", __func__); \
45 } while (0)
46
47
48static void balloc_init_alloc_debug(struct gk20a_allocator *a);
49static void balloc_print_stats(struct gk20a_allocator *a, struct seq_file *s,
50 int lock);
51static struct gk20a_buddy *balloc_free_buddy(struct gk20a_allocator *a,
52 u64 addr);
53static void balloc_coalesce(struct gk20a_allocator *a, struct gk20a_buddy *b);
54static void __balloc_do_free_fixed(struct gk20a_allocator *a,
55 struct gk20a_fixed_alloc *falloc);
56
57/*
58 * This function is not present in older kernel's list.h code.
59 */
60#ifndef list_last_entry
61#define list_last_entry(ptr, type, member) \
62 list_entry((ptr)->prev, type, member)
63#endif
64
65/*
66 * GPU buddy allocator for various address spaces.
67 *
68 * Current limitations:
69 * o A fixed allocation could potentially be made that borders PDEs with
70 * different PTE sizes. This would require that fixed buffer to have
71 * different sized PTEs for different parts of the allocation. Probably
72 * best to just require PDE alignment for fixed address allocs.
73 *
74 * o It is currently possible to make an allocator that has a buddy alignment
75 * out of sync with the PDE block size alignment. A simple example is a
76 * 32GB address space starting at byte 1. Every buddy is shifted off by 1
77 * which means each buddy corresponf to more than one actual GPU page. The
78 * best way to fix this is probably just require PDE blocksize alignment
79 * for the start of the address space. At the moment all allocators are
80 * easily PDE aligned so this hasn't been a problem.
81 */
82
83/*
84 * Pick a suitable maximum order for this allocator.
85 *
86 * Hueristic: Just guessing that the best max order is the largest single
87 * block that will fit in the address space.
88 */
89static void balloc_compute_max_order(struct gk20a_allocator *a)
90{
91 u64 true_max_order = ilog2(a->blks);
92
93 if (a->max_order > true_max_order)
94 a->max_order = true_max_order;
95 if (a->max_order > GPU_BALLOC_MAX_ORDER)
96 a->max_order = GPU_BALLOC_MAX_ORDER;
97}
98
99/*
100 * Since we can only allocate in chucks of a->blk_size we need to trim off
101 * any excess data that is not aligned to a->blk_size.
102 */
103static void balloc_allocator_align(struct gk20a_allocator *a)
104{
105 a->start = ALIGN(a->base, a->blk_size);
106 a->end = (a->base + a->length) & ~(a->blk_size - 1);
107 a->count = a->end - a->start;
108 a->blks = a->count >> a->blk_shift;
109}
110
111/*
112 * Pass NULL for parent if you want a top level buddy.
113 */
114static struct gk20a_buddy *balloc_new_buddy(struct gk20a_allocator *a,
115 struct gk20a_buddy *parent,
116 u64 start, u64 order)
117{
118 struct gk20a_buddy *new_buddy;
119
120 new_buddy = kmem_cache_alloc(buddy_cache, GFP_KERNEL);
121 if (!new_buddy)
122 return NULL;
123
124 memset(new_buddy, 0, sizeof(struct gk20a_buddy));
125
126 new_buddy->parent = parent;
127 new_buddy->start = start;
128 new_buddy->order = order;
129 new_buddy->end = start + (1 << order) * a->blk_size;
130
131 return new_buddy;
132}
133
134static void __balloc_buddy_list_add(struct gk20a_allocator *a,
135 struct gk20a_buddy *b,
136 struct list_head *list)
137{
138 if (buddy_is_in_list(b)) {
139 balloc_dbg(a, "Oops: adding added buddy (%llu:0x%llx)\n",
140 b->order, b->start);
141 BUG();
142 }
143
144 /*
145 * Add big PTE blocks to the tail, small to the head for GVA spaces.
146 * This lets the code that checks if there are available blocks check
147 * without cycling through the entire list.
148 */
149 if (a->flags & GPU_BALLOC_GVA_SPACE &&
150 b->pte_size == BALLOC_PTE_SIZE_BIG)
151 list_add_tail(&b->buddy_entry, list);
152 else
153 list_add(&b->buddy_entry, list);
154
155 buddy_set_in_list(b);
156}
157
158static void __balloc_buddy_list_rem(struct gk20a_allocator *a,
159 struct gk20a_buddy *b)
160{
161 if (!buddy_is_in_list(b)) {
162 balloc_dbg(a, "Oops: removing removed buddy (%llu:0x%llx)\n",
163 b->order, b->start);
164 BUG();
165 }
166
167 list_del_init(&b->buddy_entry);
168 buddy_clr_in_list(b);
169}
170
171/*
172 * Add a buddy to one of the buddy lists and deal with the necessary
173 * book keeping. Adds the buddy to the list specified by the buddy's order.
174 */
175static void balloc_blist_add(struct gk20a_allocator *a, struct gk20a_buddy *b)
176{
177 __balloc_buddy_list_add(a, b, balloc_get_order_list(a, b->order));
178 a->buddy_list_len[b->order]++;
179}
180
181static void balloc_blist_rem(struct gk20a_allocator *a, struct gk20a_buddy *b)
182{
183 __balloc_buddy_list_rem(a, b);
184 a->buddy_list_len[b->order]--;
185}
186
187static u64 balloc_get_order(struct gk20a_allocator *a, u64 len)
188{
189 if (len == 0)
190 return 0;
191
192 len--;
193 len >>= a->blk_shift;
194
195 return fls(len);
196}
197
198static u64 __balloc_max_order_in(struct gk20a_allocator *a, u64 start, u64 end)
199{
200 u64 size = (end - start) >> a->blk_shift;
201
202 if (size > 0)
203 return min_t(u64, ilog2(size), a->max_order);
204 else
205 return GPU_BALLOC_MAX_ORDER;
206}
207
208/*
209 * Initialize the buddy lists.
210 */
211static int balloc_init_lists(struct gk20a_allocator *a)
212{
213 int i;
214 u64 bstart, bend, order;
215 struct gk20a_buddy *buddy;
216
217 bstart = a->start;
218 bend = a->end;
219
220 /* First make sure the LLs are valid. */
221 for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++)
222 INIT_LIST_HEAD(balloc_get_order_list(a, i));
223
224 while (bstart < bend) {
225 order = __balloc_max_order_in(a, bstart, bend);
226
227 buddy = balloc_new_buddy(a, NULL, bstart, order);
228 if (!buddy)
229 goto cleanup;
230
231 balloc_blist_add(a, buddy);
232 bstart += balloc_order_to_len(a, order);
233 }
234
235 return 0;
236
237cleanup:
238 for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++) {
239 if (!list_empty(balloc_get_order_list(a, i))) {
240 buddy = list_first_entry(balloc_get_order_list(a, i),
241 struct gk20a_buddy, buddy_entry);
242 balloc_blist_rem(a, buddy);
243 kmem_cache_free(buddy_cache, buddy);
244 }
245 }
246
247 return -ENOMEM;
248}
249
250/*
251 * Initialize a buddy allocator. Returns 0 on success. This allocator does
252 * not necessarily manage bytes. It manages distinct ranges of resources. This
253 * allows the allocator to work for things like comp_tags, semaphores, etc.
254 *
255 * @allocator: Ptr to an allocator struct to init.
256 * @vm: GPU VM to associate this allocator with. Can be NULL. Will be used to
257 * get PTE size for GVA spaces.
258 * @name: Name of the allocator. Doesn't have to be static storage.
259 * @base: The base address of the resource pool being managed.
260 * @size: Number of resources in the pool.
261 * @blk_size: Minimum number of resources to allocate at once. For things like
262 * semaphores this is 1. For GVA this might be as much as 64k. This
263 * corresponds to order 0. Must be power of 2.
264 * @max_order: Pick a maximum order. If you leave this as 0, the buddy allocator
265 * will try and pick a reasonable max order.
266 * @flags: Extra flags necessary. See GPU_BALLOC_*.
267 */
268int __gk20a_allocator_init(struct gk20a_allocator *a,
269 struct vm_gk20a *vm, const char *name,
270 u64 base, u64 size, u64 blk_size, u64 max_order,
271 u64 flags)
272{ 25{
273 int err; 26 memset(allocator, 0, sizeof(struct gk20a_allocator));
274
275 memset(a, 0, sizeof(struct gk20a_allocator));
276 strncpy(a->name, name, 32);
277
278 a->base = base;
279 a->length = size;
280 a->blk_size = blk_size;
281 a->blk_shift = __ffs(blk_size);
282
283 /* blk_size must be greater than 0 and a power of 2. */
284 if (blk_size == 0)
285 return -EINVAL;
286 if (blk_size & (blk_size - 1))
287 return -EINVAL;
288
289 if (max_order > GPU_BALLOC_MAX_ORDER)
290 return -EINVAL;
291
292 /* If this is to manage a GVA space we need a VM. */
293 if (flags & GPU_BALLOC_GVA_SPACE && !vm)
294 return -EINVAL;
295
296 a->vm = vm;
297 if (flags & GPU_BALLOC_GVA_SPACE)
298 a->pte_blk_order = balloc_get_order(a, vm->big_page_size << 10);
299 27
300 a->flags = flags; 28 strncpy(allocator->name, name, 32);
301 a->max_order = max_order;
302 29
303 balloc_allocator_align(a); 30 allocator->base = start;
304 balloc_compute_max_order(a); 31 allocator->limit = start + len - 1;
305 32
306 /* Shared buddy kmem_cache for all allocators. */ 33 allocator->bitmap = vzalloc(BITS_TO_LONGS(len) * sizeof(long));
307 if (!buddy_cache) 34 if (!allocator->bitmap)
308 buddy_cache = KMEM_CACHE(gk20a_buddy, 0);
309 if (!buddy_cache)
310 return -ENOMEM; 35 return -ENOMEM;
311 36
312 a->alloced_buddies = RB_ROOT; 37 allocator_dbg(allocator, "%s : base %d, limit %d",
313 err = balloc_init_lists(a); 38 allocator->name, allocator->base);
314 if (err)
315 return err;
316 39
317 mutex_init(&a->lock); 40 init_rwsem(&allocator->rw_sema);
318 41
319 a->init = 1; 42 allocator->alloc = gk20a_allocator_block_alloc;
320 43 allocator->free = gk20a_allocator_block_free;
321 balloc_init_alloc_debug(a);
322 balloc_dbg(a, "New allocator: base 0x%llx\n", a->base);
323 balloc_dbg(a, " size 0x%llx\n", a->length);
324 balloc_dbg(a, " blk_size 0x%llx\n", a->blk_size);
325 balloc_dbg(a, " max_order %llu\n", a->max_order);
326 balloc_dbg(a, " flags 0x%llx\n", a->flags);
327 44
328 return 0; 45 return 0;
329} 46}
330 47
331int gk20a_allocator_init(struct gk20a_allocator *a, const char *name, 48/* destroy allocator, free all remaining blocks if any */
332 u64 base, u64 size, u64 blk_size) 49void gk20a_allocator_destroy(struct gk20a_allocator *allocator)
333{
334 return __gk20a_allocator_init(a, NULL, name,
335 base, size, blk_size, 0, 0);
336}
337
338/*
339 * Clean up and destroy the passed allocator.
340 */
341void gk20a_allocator_destroy(struct gk20a_allocator *a)
342{ 50{
343 struct rb_node *node; 51 down_write(&allocator->rw_sema);
344 struct gk20a_buddy *bud;
345 struct gk20a_fixed_alloc *falloc;
346 int i;
347
348 balloc_lock(a);
349
350 if (!IS_ERR_OR_NULL(a->debugfs_entry))
351 debugfs_remove(a->debugfs_entry);
352
353 /*
354 * Free the fixed allocs first.
355 */
356 while ((node = rb_first(&a->fixed_allocs)) != NULL) {
357 falloc = container_of(node,
358 struct gk20a_fixed_alloc, alloced_entry);
359
360 __balloc_do_free_fixed(a, falloc);
361 rb_erase(node, &a->fixed_allocs);
362 }
363
364 /*
365 * And now free all outstanding allocations.
366 */
367 while ((node = rb_first(&a->alloced_buddies)) != NULL) {
368 bud = container_of(node, struct gk20a_buddy, alloced_entry);
369 balloc_free_buddy(a, bud->start);
370 balloc_blist_add(a, bud);
371 balloc_coalesce(a, bud);
372 }
373 52
374 /* 53 vfree(allocator->bitmap);
375 * Now clean up the unallocated buddies.
376 */
377 for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++) {
378 BUG_ON(a->buddy_list_alloced[i] != 0);
379
380 while (!list_empty(balloc_get_order_list(a, i))) {
381 bud = list_first_entry(balloc_get_order_list(a, i),
382 struct gk20a_buddy, buddy_entry);
383 balloc_blist_rem(a, bud);
384 kmem_cache_free(buddy_cache, bud);
385 }
386
387 if (a->buddy_list_len[i] != 0) {
388 pr_info("Excess buddies!!! (%d: %llu)\n",
389 i, a->buddy_list_len[i]);
390 BUG();
391 }
392 if (a->buddy_list_split[i] != 0) {
393 pr_info("Excess split nodes!!! (%d: %llu)\n",
394 i, a->buddy_list_split[i]);
395 BUG();
396 }
397 if (a->buddy_list_alloced[i] != 0) {
398 pr_info("Excess alloced nodes!!! (%d: %llu)\n",
399 i, a->buddy_list_alloced[i]);
400 BUG();
401 }
402 }
403 54
404 a->init = 0; 55 memset(allocator, 0, sizeof(struct gk20a_allocator));
405
406 balloc_unlock(a);
407
408 /*
409 * We cant unlock an allocator after memsetting it. That wipes the
410 * state of the mutex. Hopefully no one uses the allocator after
411 * destroying it...
412 */
413 memset(a, 0, sizeof(struct gk20a_allocator));
414} 56}
415 57
416/* 58/*
417 * Combine the passed buddy if possible. The pointer in @b may not be valid 59 * *addr != ~0 for fixed address allocation. if *addr == 0, base addr is
418 * after this as the buddy may be freed. 60 * returned to caller in *addr.
419 * 61 *
420 * @a must be locked. 62 * contiguous allocation, which allocates one block of
421 */ 63 * contiguous address.
422static void balloc_coalesce(struct gk20a_allocator *a, struct gk20a_buddy *b) 64*/
65int gk20a_allocator_block_alloc(struct gk20a_allocator *allocator,
66 u32 *addr, u32 len, u32 align)
423{ 67{
424 struct gk20a_buddy *parent; 68 unsigned long _addr;
425
426 if (buddy_is_alloced(b) || buddy_is_split(b))
427 return;
428
429 /*
430 * If both our buddy and I are both not allocated and not split then
431 * we can coalesce ourselves.
432 */
433 if (!b->buddy)
434 return;
435 if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy))
436 return;
437 69
438 parent = b->parent; 70 allocator_dbg(allocator, "[in] addr %d, len %d", *addr, len);
439 71
440 balloc_blist_rem(a, b); 72 if ((*addr != 0 && *addr < allocator->base) || /* check addr range */
441 balloc_blist_rem(a, b->buddy); 73 *addr + len > allocator->limit || /* check addr range */
442 74 *addr & (align - 1) || /* check addr alignment */
443 buddy_clr_split(parent); 75 len == 0) /* check len */
444 a->buddy_list_split[parent->order]--; 76 return -EINVAL;
445 balloc_blist_add(a, parent);
446
447 /*
448 * Recursively coalesce as far as we can go.
449 */
450 balloc_coalesce(a, parent);
451
452 /* Clean up the remains. */
453 kmem_cache_free(buddy_cache, b->buddy);
454 kmem_cache_free(buddy_cache, b);
455}
456
457/*
458 * Split a buddy into two new buddies who are 1/2 the size of the parent buddy.
459 *
460 * @a must be locked.
461 */
462static int balloc_split_buddy(struct gk20a_allocator *a, struct gk20a_buddy *b,
463 int pte_size)
464{
465 struct gk20a_buddy *left, *right;
466 u64 half;
467 77
468 left = balloc_new_buddy(a, b, b->start, b->order - 1); 78 len = ALIGN(len, align);
469 if (!left) 79 if (!len)
470 return -ENOMEM; 80 return -ENOMEM;
471 81
472 half = (b->end - b->start) / 2; 82 down_write(&allocator->rw_sema);
473 83
474 right = balloc_new_buddy(a, b, b->start + half, b->order - 1); 84 _addr = bitmap_find_next_zero_area(allocator->bitmap,
475 if (!right) { 85 allocator->limit - allocator->base + 1,
476 kmem_cache_free(buddy_cache, left); 86 *addr ? (*addr - allocator->base) : 0,
87 len,
88 align - 1);
89 if ((_addr > allocator->limit - allocator->base + 1) ||
90 (*addr && *addr != (_addr + allocator->base))) {
91 up_write(&allocator->rw_sema);
477 return -ENOMEM; 92 return -ENOMEM;
478 } 93 }
479 94
480 buddy_set_split(b); 95 bitmap_set(allocator->bitmap, _addr, len);
481 a->buddy_list_split[b->order]++; 96 *addr = allocator->base + _addr;
482
483 b->left = left;
484 b->right = right;
485 left->buddy = right;
486 right->buddy = left;
487 left->parent = b;
488 right->parent = b;
489
490 /* PTE considerations. */
491 if (a->flags & GPU_BALLOC_GVA_SPACE &&
492 left->order <= a->pte_blk_order) {
493 left->pte_size = pte_size;
494 right->pte_size = pte_size;
495 }
496
497 balloc_blist_rem(a, b);
498 balloc_blist_add(a, left);
499 balloc_blist_add(a, right);
500
501 return 0;
502}
503
504/*
505 * Place the passed buddy into the RB tree for allocated buddies. Never fails
506 * unless the passed entry is a duplicate which is a bug.
507 *
508 * @a must be locked.
509 */
510void balloc_alloc_buddy(struct gk20a_allocator *a, struct gk20a_buddy *b)
511{
512 struct rb_node **new = &(a->alloced_buddies.rb_node);
513 struct rb_node *parent = NULL;
514
515 while (*new) {
516 struct gk20a_buddy *bud = container_of(*new, struct gk20a_buddy,
517 alloced_entry);
518
519 parent = *new;
520 if (b->start < bud->start)
521 new = &((*new)->rb_left);
522 else if (b->start > bud->start)
523 new = &((*new)->rb_right);
524 else
525 BUG_ON("Duplicate entries in allocated list!\n");
526 }
527
528 rb_link_node(&b->alloced_entry, parent, new);
529 rb_insert_color(&b->alloced_entry, &a->alloced_buddies);
530
531 buddy_set_alloced(b);
532 a->buddy_list_alloced[b->order]++;
533}
534
535/*
536 * Remove the passed buddy from the allocated buddy RB tree. Returns the
537 * deallocated buddy for further processing.
538 *
539 * @a must be locked.
540 */
541static struct gk20a_buddy *balloc_free_buddy(struct gk20a_allocator *a,
542 u64 addr)
543{
544 struct rb_node *node = a->alloced_buddies.rb_node;
545 struct gk20a_buddy *bud;
546
547 while (node) {
548 bud = container_of(node, struct gk20a_buddy, alloced_entry);
549
550 if (addr < bud->start)
551 node = node->rb_left;
552 else if (addr > bud->start)
553 node = node->rb_right;
554 else
555 break;
556 }
557
558 if (!node)
559 return NULL;
560
561 rb_erase(node, &a->alloced_buddies);
562 buddy_clr_alloced(bud);
563 a->buddy_list_alloced[bud->order]--;
564
565 return bud;
566}
567
568/*
569 * Find a suitable buddy for the given order and PTE type (big or little).
570 */
571static struct gk20a_buddy *__balloc_find_buddy(struct gk20a_allocator *a,
572 u64 order, int pte_size)
573{
574 struct gk20a_buddy *bud;
575
576 if (list_empty(balloc_get_order_list(a, order)))
577 return NULL;
578
579 if (a->flags & GPU_BALLOC_GVA_SPACE &&
580 pte_size == BALLOC_PTE_SIZE_BIG)
581 bud = list_last_entry(balloc_get_order_list(a, order),
582 struct gk20a_buddy, buddy_entry);
583 else
584 bud = list_first_entry(balloc_get_order_list(a, order),
585 struct gk20a_buddy, buddy_entry);
586
587 if (bud->pte_size != BALLOC_PTE_SIZE_ANY &&
588 bud->pte_size != pte_size)
589 return NULL;
590
591 return bud;
592}
593
594/*
595 * Allocate a suitably sized buddy. If no suitable buddy exists split higher
596 * order buddies until we have a suitable buddy to allocate.
597 *
598 * For PDE grouping add an extra check to see if a buddy is suitable: that the
599 * buddy exists in a PDE who's PTE size is reasonable
600 *
601 * @a must be locked.
602 */
603static u64 __balloc_do_alloc(struct gk20a_allocator *a, u64 order, int pte_size)
604{
605 u64 split_order;
606 struct gk20a_buddy *bud;
607
608 split_order = order;
609 while (!(bud = __balloc_find_buddy(a, split_order, pte_size)))
610 split_order++;
611
612 while (bud->order != order) {
613 if (balloc_split_buddy(a, bud, pte_size))
614 return 0; /* No mem... */
615 bud = bud->left;
616 }
617
618 balloc_blist_rem(a, bud);
619 balloc_alloc_buddy(a, bud);
620
621 return bud->start;
622}
623
624/*
625 * Allocate memory from the passed allocator.
626 */
627u64 gk20a_balloc(struct gk20a_allocator *a, u64 len)
628{
629 u64 order, addr;
630 int pte_size;
631
632 balloc_trace_func();
633
634 balloc_lock(a);
635
636 order = balloc_get_order(a, len);
637
638 if (order > a->max_order) {
639 balloc_unlock(a);
640 balloc_dbg(a, "Alloc fail\n");
641 balloc_trace_func_done();
642 return 0;
643 }
644
645 /*
646 * For now pass the base address of the allocator's region to
647 * __get_pte_size(). This ensures we get the right page size for
648 * the alloc but we don't have to know what the real address is
649 * going to be quite yet.
650 *
651 * TODO: once userspace supports a unified address space pass 0 for
652 * the base. This will make only 'len' affect the PTE size.
653 */
654 if (a->flags & GPU_BALLOC_GVA_SPACE)
655 pte_size = __get_pte_size(a->vm, a->base, len);
656 else
657 pte_size = BALLOC_PTE_SIZE_ANY;
658
659 addr = __balloc_do_alloc(a, order, pte_size);
660
661 a->bytes_alloced += len;
662 a->bytes_alloced_real += balloc_order_to_len(a, order);
663
664 balloc_unlock(a);
665 balloc_dbg(a, "Alloc 0x%-10llx %3lld:0x%-10llx pte_size=%s\n",
666 addr, order, len,
667 pte_size == gmmu_page_size_big ? "big" :
668 pte_size == gmmu_page_size_small ? "small" :
669 "NA/any");
670
671 balloc_trace_func_done();
672 return addr;
673}
674
675/*
676 * See if the passed range is actually available for allocation. If so, then
677 * return 1, otherwise return 0.
678 *
679 * TODO: Right now this uses the unoptimal approach of going through all
680 * outstanding allocations and checking their base/ends. This could be better.
681 */
682static int balloc_is_range_free(struct gk20a_allocator *a, u64 base, u64 end)
683{
684 struct rb_node *node;
685 struct gk20a_buddy *bud;
686
687 node = rb_first(&a->alloced_buddies);
688 if (!node)
689 return 1; /* No allocs yet. */
690
691 bud = container_of(node, struct gk20a_buddy, alloced_entry);
692
693 while (bud->start < end) {
694 if ((bud->start > base && bud->start < end) ||
695 (bud->end > base && bud->end < end))
696 return 0;
697
698 node = rb_next(node);
699 if (!node)
700 break;
701 bud = container_of(node, struct gk20a_buddy, alloced_entry);
702 }
703
704 return 1;
705}
706
707static void balloc_alloc_fixed(struct gk20a_allocator *a,
708 struct gk20a_fixed_alloc *f)
709{
710 struct rb_node **new = &(a->fixed_allocs.rb_node);
711 struct rb_node *parent = NULL;
712
713 while (*new) {
714 struct gk20a_fixed_alloc *falloc =
715 container_of(*new, struct gk20a_fixed_alloc,
716 alloced_entry);
717
718 parent = *new;
719 if (f->start < falloc->start)
720 new = &((*new)->rb_left);
721 else if (f->start > falloc->start)
722 new = &((*new)->rb_right);
723 else
724 BUG_ON("Duplicate entries in allocated list!\n");
725 }
726
727 rb_link_node(&f->alloced_entry, parent, new);
728 rb_insert_color(&f->alloced_entry, &a->fixed_allocs);
729}
730
731/*
732 * Remove the passed buddy from the allocated buddy RB tree. Returns the
733 * deallocated buddy for further processing.
734 *
735 * @a must be locked.
736 */
737static struct gk20a_fixed_alloc *balloc_free_fixed(struct gk20a_allocator *a,
738 u64 addr)
739{
740 struct rb_node *node = a->fixed_allocs.rb_node;
741 struct gk20a_fixed_alloc *falloc;
742
743 while (node) {
744 falloc = container_of(node,
745 struct gk20a_fixed_alloc, alloced_entry);
746
747 if (addr < falloc->start)
748 node = node->rb_left;
749 else if (addr > falloc->start)
750 node = node->rb_right;
751 else
752 break;
753 }
754
755 if (!node)
756 return NULL;
757
758 rb_erase(node, &a->fixed_allocs);
759
760 return falloc;
761}
762
763/*
764 * Find the parent range - doesn't necessarily need the parent to actually exist
765 * as a buddy. Finding an existing parent comes later...
766 */
767static void __balloc_get_parent_range(struct gk20a_allocator *a,
768 u64 base, u64 order,
769 u64 *pbase, u64 *porder)
770{
771 u64 base_mask;
772 u64 shifted_base = balloc_base_shift(a, base);
773
774 order++;
775 base_mask = ~((a->blk_size << order) - 1);
776
777 shifted_base &= base_mask;
778
779 *pbase = balloc_base_unshift(a, shifted_base);
780 *porder = order;
781}
782
783/*
784 * Makes a buddy at the passed address. This will make all parent buddies
785 * necessary for this buddy to exist as well.
786 */
787static struct gk20a_buddy *__balloc_make_fixed_buddy(struct gk20a_allocator *a,
788 u64 base, u64 order)
789{
790 struct gk20a_buddy *bud = NULL;
791 struct list_head *order_list;
792 u64 cur_order = order, cur_base = base;
793
794 /*
795 * Algo:
796 * 1. Keep jumping up a buddy order until we find the real buddy that
797 * this buddy exists in.
798 * 2. Then work our way down through the buddy tree until we hit a dead
799 * end.
800 * 3. Start splitting buddies until we split to the one we need to
801 * make.
802 */
803 while (cur_order <= a->max_order) {
804 int found = 0;
805
806 order_list = balloc_get_order_list(a, cur_order);
807 list_for_each_entry(bud, order_list, buddy_entry) {
808 if (bud->start == cur_base) {
809 found = 1;
810 break;
811 }
812 }
813
814 if (found)
815 break;
816
817 __balloc_get_parent_range(a, cur_base, cur_order,
818 &cur_base, &cur_order);
819 }
820
821 if (cur_order > a->max_order) {
822 balloc_dbg(a, "No buddy for range ???\n");
823 return NULL;
824 }
825
826 /* Split this buddy as necessary until we get the target buddy. */
827 while (bud->start != base || bud->order != order) {
828 if (balloc_split_buddy(a, bud, BALLOC_PTE_SIZE_ANY)) {
829 balloc_coalesce(a, bud);
830 return NULL;
831 }
832
833 if (base < bud->right->start)
834 bud = bud->left;
835 else
836 bud = bud->right;
837
838 }
839
840 return bud;
841}
842
843static u64 __balloc_do_alloc_fixed(struct gk20a_allocator *a,
844 struct gk20a_fixed_alloc *falloc,
845 u64 base, u64 len)
846{
847 u64 shifted_base, inc_base;
848 u64 align_order;
849
850 shifted_base = balloc_base_shift(a, base);
851 if (shifted_base == 0)
852 align_order = __fls(len >> a->blk_shift);
853 else
854 align_order = min_t(u64,
855 __ffs(shifted_base >> a->blk_shift),
856 __fls(len >> a->blk_shift));
857
858 if (align_order > a->max_order) {
859 balloc_dbg(a, "Align order too big: %llu > %llu\n",
860 align_order, a->max_order);
861 return 0;
862 }
863
864 /*
865 * Generate a list of buddies that satisfy this allocation.
866 */
867 inc_base = shifted_base;
868 while (inc_base < (shifted_base + len)) {
869 u64 order_len = balloc_order_to_len(a, align_order);
870 u64 remaining;
871 struct gk20a_buddy *bud;
872
873 bud = __balloc_make_fixed_buddy(a,
874 balloc_base_unshift(a, inc_base),
875 align_order);
876 if (!bud) {
877 balloc_dbg(a, "Fixed buddy failed: {0x%llx, %llu}!\n",
878 balloc_base_unshift(a, inc_base),
879 align_order);
880 goto err_and_cleanup;
881 }
882
883 balloc_blist_rem(a, bud);
884 balloc_alloc_buddy(a, bud);
885 __balloc_buddy_list_add(a, bud, &falloc->buddies);
886
887 /* Book keeping. */
888 inc_base += order_len;
889 remaining = (shifted_base + len) - inc_base;
890 align_order = __ffs(inc_base >> a->blk_shift);
891
892 /* If we don't have much left - trim down align_order. */
893 if (balloc_order_to_len(a, align_order) > remaining)
894 align_order = __balloc_max_order_in(a, inc_base,
895 inc_base + remaining);
896 }
897
898 return base;
899
900err_and_cleanup:
901 while (!list_empty(&falloc->buddies)) {
902 struct gk20a_buddy *bud = list_first_entry(&falloc->buddies,
903 struct gk20a_buddy,
904 buddy_entry);
905
906 __balloc_buddy_list_rem(a, bud);
907 balloc_free_buddy(a, bud->start);
908 kmem_cache_free(buddy_cache, bud);
909 }
910
911 return 0;
912}
913
914/*
915 * Allocate a fixed address allocation. The address of the allocation is @base
916 * and the length is @len. This is not a typical buddy allocator operation and
917 * as such has a high posibility of failure if the address space is heavily in
918 * use.
919 *
920 * Please do not use this function unless _absolutely_ necessary.
921 */
922u64 gk20a_balloc_fixed(struct gk20a_allocator *a, u64 base, u64 len)
923{
924 struct gk20a_fixed_alloc *falloc = NULL;
925 struct gk20a_buddy *bud;
926 u64 ret, real_bytes = 0;
927
928 balloc_trace_func();
929
930 /* If base isn't aligned to an order 0 block, fail. */
931 if (base & (a->blk_size - 1))
932 goto fail;
933
934 if (len == 0)
935 goto fail;
936
937 falloc = kmalloc(sizeof(*falloc), GFP_KERNEL);
938 if (!falloc)
939 goto fail;
940
941 INIT_LIST_HEAD(&falloc->buddies);
942 falloc->start = base;
943 falloc->end = base + len;
944
945 balloc_lock(a);
946 if (!balloc_is_range_free(a, base, base + len)) {
947 balloc_dbg(a, "Range not free: 0x%llx -> 0x%llx\n",
948 base, base + len);
949 goto fail_unlock;
950 }
951
952 ret = __balloc_do_alloc_fixed(a, falloc, base, len);
953 if (!ret) {
954 balloc_dbg(a, "Alloc-fixed failed ?? 0x%llx -> 0x%llx\n",
955 base, base + len);
956 goto fail_unlock;
957 }
958
959 balloc_alloc_fixed(a, falloc);
960
961 list_for_each_entry(bud, &falloc->buddies, buddy_entry)
962 real_bytes += (bud->end - bud->start);
963 97
964 a->bytes_alloced += len; 98 up_write(&allocator->rw_sema);
965 a->bytes_alloced_real += real_bytes;
966 99
967 balloc_unlock(a); 100 allocator_dbg(allocator, "[out] addr %d, len %d", *addr, len);
968 balloc_dbg(a, "Alloc (fixed) 0x%llx\n", base);
969 101
970 balloc_trace_func_done();
971 return base;
972
973fail_unlock:
974 balloc_unlock(a);
975fail:
976 kfree(falloc);
977 balloc_trace_func_done();
978 return 0; 102 return 0;
979} 103}
980 104
981static void __balloc_do_free_fixed(struct gk20a_allocator *a, 105/* free all blocks between start and end */
982 struct gk20a_fixed_alloc *falloc) 106int gk20a_allocator_block_free(struct gk20a_allocator *allocator,
983{ 107 u32 addr, u32 len, u32 align)
984 struct gk20a_buddy *bud;
985
986 while (!list_empty(&falloc->buddies)) {
987 bud = list_first_entry(&falloc->buddies,
988 struct gk20a_buddy,
989 buddy_entry);
990 __balloc_buddy_list_rem(a, bud);
991
992 balloc_free_buddy(a, bud->start);
993 balloc_blist_add(a, bud);
994 a->bytes_freed += balloc_order_to_len(a, bud->order);
995
996 /*
997 * Attemp to defrag the allocation.
998 */
999 balloc_coalesce(a, bud);
1000 }
1001
1002 kfree(falloc);
1003}
1004
1005/*
1006 * Free the passed allocation.
1007 */
1008void gk20a_bfree(struct gk20a_allocator *a, u64 addr)
1009{
1010 struct gk20a_buddy *bud;
1011 struct gk20a_fixed_alloc *falloc;
1012
1013 balloc_trace_func();
1014
1015 if (!addr) {
1016 balloc_trace_func_done();
1017 return;
1018 }
1019
1020 balloc_lock(a);
1021
1022 /*
1023 * First see if this is a fixed alloc. If not fall back to a regular
1024 * buddy.
1025 */
1026 falloc = balloc_free_fixed(a, addr);
1027 if (falloc) {
1028 __balloc_do_free_fixed(a, falloc);
1029 goto done;
1030 }
1031
1032 bud = balloc_free_buddy(a, addr);
1033 if (!bud)
1034 goto done;
1035
1036 balloc_blist_add(a, bud);
1037 a->bytes_freed += balloc_order_to_len(a, bud->order);
1038
1039 /*
1040 * Attemp to defrag the allocation.
1041 */
1042 balloc_coalesce(a, bud);
1043
1044done:
1045 balloc_unlock(a);
1046 balloc_dbg(a, "Free 0x%llx\n", addr);
1047 balloc_trace_func_done();
1048 return;
1049}
1050
1051/*
1052 * Print the buddy allocator top level stats. If you pass @s as NULL then the
1053 * stats are printed to the kernel log. This lets this code be used for
1054 * debugging purposes internal to the allocator.
1055 */
1056static void balloc_print_stats(struct gk20a_allocator *a, struct seq_file *s,
1057 int lock)
1058{ 108{
1059#define __balloc_pstat(s, fmt, arg...) \ 109 allocator_dbg(allocator, "[in] addr %d, len %d", addr, len);
1060 do { \
1061 if (s) \
1062 seq_printf(s, fmt, ##arg); \
1063 else \
1064 balloc_dbg(a, fmt, ##arg); \
1065 } while (0)
1066
1067 int i;
1068 struct rb_node *node;
1069 struct gk20a_fixed_alloc *falloc;
1070
1071 __balloc_pstat(s, "base = %llu, limit = %llu, blk_size = %llu\n",
1072 a->base, a->length, a->blk_size);
1073 __balloc_pstat(s, "Internal params:\n");
1074 __balloc_pstat(s, " start = %llu\n", a->start);
1075 __balloc_pstat(s, " end = %llu\n", a->end);
1076 __balloc_pstat(s, " count = %llu\n", a->count);
1077 __balloc_pstat(s, " blks = %llu\n", a->blks);
1078 __balloc_pstat(s, " max_order = %llu\n", a->max_order);
1079
1080 __balloc_pstat(s, "Buddy blocks:\n");
1081 __balloc_pstat(s, " Order Free Alloced Split\n");
1082 __balloc_pstat(s, " ----- ---- ------- -----\n");
1083
1084 if (lock)
1085 balloc_lock(a);
1086 for (i = a->max_order; i >= 0; i--) {
1087 if (a->buddy_list_len[i] == 0 &&
1088 a->buddy_list_alloced[i] == 0 &&
1089 a->buddy_list_split[i] == 0)
1090 continue;
1091
1092 __balloc_pstat(s, " %3d %-7llu %-9llu %llu\n", i,
1093 a->buddy_list_len[i],
1094 a->buddy_list_alloced[i],
1095 a->buddy_list_split[i]);
1096 }
1097
1098 __balloc_pstat(s, "\n");
1099 110
1100 for (node = rb_first(&a->fixed_allocs), i = 1; 111 if (addr + len > allocator->limit || /* check addr range */
1101 node != NULL; 112 addr < allocator->base ||
1102 node = rb_next(node)) { 113 addr & (align - 1)) /* check addr alignment */
1103 falloc = container_of(node, 114 return -EINVAL;
1104 struct gk20a_fixed_alloc, alloced_entry);
1105
1106 __balloc_pstat(s, "Fixed alloc (%d): [0x%llx -> 0x%llx]\n",
1107 i, falloc->start, falloc->end);
1108 }
1109
1110 __balloc_pstat(s, "\n");
1111 __balloc_pstat(s, "Bytes allocated: %llu\n", a->bytes_alloced);
1112 __balloc_pstat(s, "Bytes allocated (real): %llu\n",
1113 a->bytes_alloced_real);
1114 __balloc_pstat(s, "Bytes freed: %llu\n", a->bytes_freed);
1115
1116 if (lock)
1117 balloc_unlock(a);
1118 115
1119#undef __balloc_pstats 116 len = ALIGN(len, align);
1120} 117 if (!len)
118 return -EINVAL;
1121 119
1122static int __alloc_show(struct seq_file *s, void *unused) 120 down_write(&allocator->rw_sema);
1123{ 121 bitmap_clear(allocator->bitmap, addr - allocator->base, len);
1124 struct gk20a_allocator *a = s->private; 122 up_write(&allocator->rw_sema);
1125 123
1126 balloc_print_stats(a, s, 1); 124 allocator_dbg(allocator, "[out] addr %d, len %d", addr, len);
1127 125
1128 return 0; 126 return 0;
1129} 127}
1130
1131static int __alloc_open(struct inode *inode, struct file *file)
1132{
1133 return single_open(file, __alloc_show, inode->i_private);
1134}
1135
1136static const struct file_operations __alloc_fops = {
1137 .open = __alloc_open,
1138 .read = seq_read,
1139 .llseek = seq_lseek,
1140 .release = single_release,
1141};
1142
1143static void balloc_init_alloc_debug(struct gk20a_allocator *a)
1144{
1145 if (!balloc_debugfs_root)
1146 return;
1147
1148 a->debugfs_entry = debugfs_create_file(a->name, S_IRUGO,
1149 balloc_debugfs_root,
1150 a, &__alloc_fops);
1151}
1152
1153void gk20a_alloc_debugfs_init(struct platform_device *pdev)
1154{
1155 struct gk20a_platform *platform = platform_get_drvdata(pdev);
1156 struct dentry *gpu_root = platform->debugfs;
1157
1158 balloc_debugfs_root = debugfs_create_dir("allocators", gpu_root);
1159 if (IS_ERR_OR_NULL(balloc_debugfs_root))
1160 return;
1161
1162 debugfs_create_u32("tracing", 0664, balloc_debugfs_root,
1163 &balloc_tracing_on);
1164}
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_allocator.h b/drivers/gpu/nvgpu/gk20a/gk20a_allocator.h
index e86e053b..69a227bd 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a_allocator.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a_allocator.h
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (c) 2011-2015, NVIDIA CORPORATION. All rights reserved. 2 * Copyright (c) 2011-2014, NVIDIA CORPORATION. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify it 4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License, 5 * under the terms and conditions of the GNU General Public License,
@@ -17,190 +17,75 @@
17#ifndef GK20A_ALLOCATOR_H 17#ifndef GK20A_ALLOCATOR_H
18#define GK20A_ALLOCATOR_H 18#define GK20A_ALLOCATOR_H
19 19
20#include <linux/list.h>
21#include <linux/rbtree.h> 20#include <linux/rbtree.h>
22#include <linux/debugfs.h> 21#include <linux/rwsem.h>
23#include <linux/platform_device.h> 22#include <linux/slab.h>
24 23
25/* #define ALLOCATOR_DEBUG */ 24/* #define ALLOCATOR_DEBUG */
26 25
27/* 26/* main struct */
28 * Each buddy is an element in a binary tree.
29 */
30struct gk20a_buddy {
31 struct gk20a_buddy *parent; /* Parent node. */
32 struct gk20a_buddy *buddy; /* This node's buddy. */
33 struct gk20a_buddy *left; /* Lower address sub-node. */
34 struct gk20a_buddy *right; /* Higher address sub-node. */
35
36 struct list_head buddy_entry; /* List entry for various lists. */
37 struct rb_node alloced_entry; /* RB tree of allocations. */
38
39 u64 start; /* Start address of this buddy. */
40 u64 end; /* End address of this buddy. */
41 u64 order; /* Buddy order. */
42
43#define BALLOC_BUDDY_ALLOCED 0x1
44#define BALLOC_BUDDY_SPLIT 0x2
45#define BALLOC_BUDDY_IN_LIST 0x4
46 int flags; /* List of associated flags. */
47
48 /*
49 * Size of the PDE this buddy is using. This allows for grouping like
50 * sized allocations into the same PDE.
51 */
52#define BALLOC_PTE_SIZE_ANY 0x0
53#define BALLOC_PTE_SIZE_SMALL 0x1
54#define BALLOC_PTE_SIZE_BIG 0x2
55 int pte_size;
56};
57
58#define __buddy_flag_ops(flag, flag_up) \
59 static inline int buddy_is_ ## flag(struct gk20a_buddy *b) \
60 { \
61 return b->flags & BALLOC_BUDDY_ ## flag_up; \
62 } \
63 static inline void buddy_set_ ## flag(struct gk20a_buddy *b) \
64 { \
65 b->flags |= BALLOC_BUDDY_ ## flag_up; \
66 } \
67 static inline void buddy_clr_ ## flag(struct gk20a_buddy *b) \
68 { \
69 b->flags &= ~BALLOC_BUDDY_ ## flag_up; \
70 }
71
72/*
73 * int buddy_is_alloced(struct gk20a_buddy *b);
74 * void buddy_set_alloced(struct gk20a_buddy *b);
75 * void buddy_clr_alloced(struct gk20a_buddy *b);
76 *
77 * int buddy_is_split(struct gk20a_buddy *b);
78 * void buddy_set_split(struct gk20a_buddy *b);
79 * void buddy_clr_split(struct gk20a_buddy *b);
80 *
81 * int buddy_is_in_list(struct gk20a_buddy *b);
82 * void buddy_set_in_list(struct gk20a_buddy *b);
83 * void buddy_clr_in_list(struct gk20a_buddy *b);
84 */
85__buddy_flag_ops(alloced, ALLOCED);
86__buddy_flag_ops(split, SPLIT);
87__buddy_flag_ops(in_list, IN_LIST);
88
89/*
90 * Keeps info for a fixed allocation.
91 */
92struct gk20a_fixed_alloc {
93 struct list_head buddies; /* List of buddies. */
94 struct rb_node alloced_entry; /* RB tree of fixed allocations. */
95
96 u64 start; /* Start of fixed block. */
97 u64 end; /* End address. */
98};
99
100struct vm_gk20a;
101
102/*
103 * GPU buddy allocator for the various GPU address spaces. Each addressable unit
104 * doesn't have to correspond to a byte. In some cases each unit is a more
105 * complex object such as a comp_tag line or the like.
106 *
107 * The max order is computed based on the size of the minimum order and the size
108 * of the address space.
109 *
110 * order_size is the size of an order 0 buddy.
111 */
112struct gk20a_allocator { 27struct gk20a_allocator {
113 28
114 struct vm_gk20a *vm; /* Parent VM - can be NULL. */ 29 char name[32]; /* name for allocator */
115 30 struct rb_root rb_root; /* rb tree root for blocks */
116 char name[32]; /* Name of allocator. */
117
118 u64 base; /* Base address of the space. */
119 u64 length; /* Length of the space. */
120 u64 blk_size; /* Size of order 0 allocation. */
121 u64 blk_shift; /* Shift to divide by blk_size. */
122
123 int init; /* Non-zero if initialized. */
124 31
125 /* Internal stuff. */ 32 u32 base; /* min value of this linear space */
126 u64 start; /* Real start (aligned to blk_size). */ 33 u32 limit; /* max value = limit - 1 */
127 u64 end; /* Real end, trimmed if needed. */
128 u64 count; /* Count of objects in space. */
129 u64 blks; /* Count of blks in the space. */
130 u64 max_order; /* Specific maximum order. */
131 34
132 struct rb_root alloced_buddies; /* Outstanding allocations. */ 35 unsigned long *bitmap; /* bitmap */
133 struct rb_root fixed_allocs; /* Outstanding fixed allocations. */
134 36
135 struct mutex lock; /* Protects buddy access. */ 37 struct gk20a_alloc_block *block_first; /* first block in list */
38 struct gk20a_alloc_block *block_recent; /* last visited block */
136 39
137#define GPU_BALLOC_GVA_SPACE 0x1 40 u32 first_free_addr; /* first free addr, non-contigous
138 u64 flags; 41 allocation preferred start,
42 in order to pick up small holes */
43 u32 last_free_addr; /* last free addr, contiguous
44 allocation preferred start */
45 u32 cached_hole_size; /* max free hole size up to
46 last_free_addr */
47 u32 block_count; /* number of blocks */
139 48
140 /* 49 struct rw_semaphore rw_sema; /* lock */
141 * Impose an upper bound on the maximum order. 50 struct kmem_cache *block_cache; /* slab cache */
142 */
143#define GPU_BALLOC_MAX_ORDER 31
144#define GPU_BALLOC_ORDER_LIST_LEN (GPU_BALLOC_MAX_ORDER + 1)
145 51
146 struct list_head buddy_list[GPU_BALLOC_ORDER_LIST_LEN]; 52 /* if enabled, constrain to [base, limit) */
147 u64 buddy_list_len[GPU_BALLOC_ORDER_LIST_LEN]; 53 struct {
148 u64 buddy_list_split[GPU_BALLOC_ORDER_LIST_LEN]; 54 bool enable;
149 u64 buddy_list_alloced[GPU_BALLOC_ORDER_LIST_LEN]; 55 u32 base;
56 u32 limit;
57 } constraint;
150 58
151 /* 59 int (*alloc)(struct gk20a_allocator *allocator,
152 * This is for when the allocator is managing a GVA space (the 60 u32 *addr, u32 len, u32 align);
153 * GPU_BALLOC_GVA_SPACE bit is set in @flags). This requires 61 int (*free)(struct gk20a_allocator *allocator,
154 * that we group like sized allocations into PDE blocks. 62 u32 addr, u32 len, u32 align);
155 */
156 u64 pte_blk_order;
157 63
158 struct dentry *debugfs_entry;
159
160 u64 bytes_alloced;
161 u64 bytes_alloced_real;
162 u64 bytes_freed;
163}; 64};
164 65
165#define balloc_lock(a) mutex_lock(&(a)->lock) 66int gk20a_allocator_init(struct gk20a_allocator *allocator,
166#define balloc_unlock(a) mutex_unlock(&(a)->lock) 67 const char *name, u32 base, u32 size);
68void gk20a_allocator_destroy(struct gk20a_allocator *allocator);
167 69
168#define balloc_get_order_list(a, order) (&(a)->buddy_list[(order)]) 70int gk20a_allocator_block_alloc(struct gk20a_allocator *allocator,
169#define balloc_order_to_len(a, order) ((1 << order) * (a)->blk_size) 71 u32 *addr, u32 len, u32 align);
170#define balloc_base_shift(a, base) ((base) - (a)->start)
171#define balloc_base_unshift(a, base) ((base) + (a)->start)
172 72
173int gk20a_allocator_init(struct gk20a_allocator *allocator, 73int gk20a_allocator_block_free(struct gk20a_allocator *allocator,
174 const char *name, u64 base, u64 size, u64 order0); 74 u32 addr, u32 len, u32 align);
175int __gk20a_allocator_init(struct gk20a_allocator *allocator,
176 struct vm_gk20a *vm, const char *name,
177 u64 base, u64 size, u64 order0,
178 u64 max_order, u64 flags);
179void gk20a_allocator_destroy(struct gk20a_allocator *allocator);
180 75
181/* 76#if defined(ALLOCATOR_DEBUG)
182 * Normal alloc/free operations for the buddy allocator.
183 */
184u64 gk20a_balloc(struct gk20a_allocator *allocator, u64 len);
185void gk20a_bfree(struct gk20a_allocator *allocator, u64 addr);
186 77
187/* 78#define allocator_dbg(alloctor, format, arg...) \
188 * Special interface to allocate a memory regions with a specific starting 79do { \
189 * address. Yikes. 80 if (1) \
190 */ 81 pr_debug("gk20a_allocator (%s) %s: " format "\n",\
191u64 gk20a_balloc_fixed(struct gk20a_allocator *allocator, u64 base, u64 len); 82 alloctor->name, __func__, ##arg);\
83} while (0)
192 84
193/* 85#else /* ALLOCATOR_DEBUG */
194 * Debugfs init.
195 */
196void gk20a_alloc_debugfs_init(struct platform_device *pdev);
197 86
198#if defined(ALLOCATOR_DEBUG) 87#define allocator_dbg(format, arg...)
199#define balloc_dbg(alloctor, format, arg...) \ 88
200 pr_info("%-25s %25s() " format, \ 89#endif /* ALLOCATOR_DEBUG */
201 alloctor->name, __func__, ##arg)
202#else
203#define balloc_dbg(allocator, format, arg...)
204#endif
205 90
206#endif /* GK20A_ALLOCATOR_H */ 91#endif /* GK20A_ALLOCATOR_H */
diff --git a/drivers/gpu/nvgpu/gk20a/ltc_gk20a.c b/drivers/gpu/nvgpu/gk20a/ltc_gk20a.c
index bcadde93..c5d0f0c4 100644
--- a/drivers/gpu/nvgpu/gk20a/ltc_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/ltc_gk20a.c
@@ -89,8 +89,9 @@ static int gk20a_ltc_init_comptags(struct gk20a *g, struct gr_gk20a *gr)
89 if (err) 89 if (err)
90 return err; 90 return err;
91 91
92 __gk20a_allocator_init(&gr->comp_tags, NULL, "comptag", 92 gk20a_allocator_init(&gr->comp_tags, "comptag",
93 1, max_comptag_lines - 1, 1, 10, 0); 93 1, /* start */
94 max_comptag_lines - 1); /* length*/
94 95
95 gr->comptags_per_cacheline = comptags_per_cacheline; 96 gr->comptags_per_cacheline = comptags_per_cacheline;
96 gr->slices_per_ltc = slices_per_fbp / g->ltc_count; 97 gr->slices_per_ltc = slices_per_fbp / g->ltc_count;
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
index c11414b5..5d1ff563 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
@@ -132,8 +132,10 @@ static void gk20a_mm_delete_priv(void *_priv)
132 132
133 if (priv->comptags.lines) { 133 if (priv->comptags.lines) {
134 BUG_ON(!priv->comptag_allocator); 134 BUG_ON(!priv->comptag_allocator);
135 gk20a_bfree(priv->comptag_allocator, 135 priv->comptag_allocator->free(priv->comptag_allocator,
136 priv->comptags.real_offset); 136 priv->comptags.offset,
137 priv->comptags.allocated_lines,
138 1);
137 } 139 }
138 140
139 /* Free buffer states */ 141 /* Free buffer states */
@@ -222,9 +224,10 @@ static int gk20a_alloc_comptags(struct gk20a *g,
222 u32 lines, bool user_mappable) 224 u32 lines, bool user_mappable)
223{ 225{
224 struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev); 226 struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
227 u32 offset = 0;
228 int err;
225 u32 ctaglines_to_allocate; 229 u32 ctaglines_to_allocate;
226 u32 ctagline_align = 1; 230 u32 ctagline_align;
227 u32 offset;
228 const u32 aggregate_cacheline_sz = 231 const u32 aggregate_cacheline_sz =
229 g->gr.cacheline_size * g->gr.slices_per_ltc * 232 g->gr.cacheline_size * g->gr.slices_per_ltc *
230 g->ltc_count; 233 g->ltc_count;
@@ -238,6 +241,7 @@ static int gk20a_alloc_comptags(struct gk20a *g,
238 241
239 if (!user_mappable) { 242 if (!user_mappable) {
240 ctaglines_to_allocate = lines; 243 ctaglines_to_allocate = lines;
244 ctagline_align = 1;
241 } else { 245 } else {
242 /* Unfortunately, we cannot use allocation alignment 246 /* Unfortunately, we cannot use allocation alignment
243 * here, since compbits per cacheline is not always a 247 * here, since compbits per cacheline is not always a
@@ -269,25 +273,71 @@ static int gk20a_alloc_comptags(struct gk20a *g,
269 273
270 if (ctaglines_to_allocate < lines) 274 if (ctaglines_to_allocate < lines)
271 return -EINVAL; /* integer overflow */ 275 return -EINVAL; /* integer overflow */
272 pr_info("user-mapped CTAGS: %u\n", ctaglines_to_allocate);
273 } 276 }
274 277
275 /* store the allocator so we can use it when we free the ctags */ 278 /* store the allocator so we can use it when we free the ctags */
276 priv->comptag_allocator = allocator; 279 priv->comptag_allocator = allocator;
277 offset = gk20a_balloc(allocator, ctaglines_to_allocate); 280 err = allocator->alloc(allocator, &offset,
278 if (!offset) 281 ctaglines_to_allocate, 1);
279 return -ENOMEM; 282 if (!err) {
283 const u32 alignment_lines =
284 DIV_ROUND_UP(offset, ctagline_align) * ctagline_align -
285 offset;
286
287 /* prune the preceding ctaglines that were allocated
288 for alignment */
289 if (alignment_lines) {
290 /* free alignment lines */
291 int tmp=
292 allocator->free(allocator, offset,
293 alignment_lines,
294 1);
295 WARN_ON(tmp);
296
297 offset += alignment_lines;
298 ctaglines_to_allocate -= alignment_lines;
299 }
280 300
281 priv->comptags.lines = lines; 301 /* check if we can prune the trailing, too */
282 priv->comptags.real_offset = offset; 302 if (user_mappable)
303 {
304 u32 needed_cachelines =
305 DIV_ROUND_UP(lines, g->gr.comptags_per_cacheline);
306
307 u32 first_unneeded_cacheline =
308 DIV_ROUND_UP(round_up(needed_cachelines *
309 aggregate_cacheline_sz,
310 small_pgsz),
311 aggregate_cacheline_sz);
312 u32 needed_ctaglines =
313 first_unneeded_cacheline *
314 g->gr.comptags_per_cacheline;
315
316 if (needed_ctaglines < ctaglines_to_allocate) {
317 /* free alignment lines */
318 int tmp=
319 allocator->free(
320 allocator,
321 offset + needed_ctaglines,
322 (ctaglines_to_allocate -
323 needed_ctaglines),
324 1);
325 WARN_ON(tmp);
326
327 ctaglines_to_allocate = needed_ctaglines;
328 }
329 }
330
331 priv->comptags.offset = offset;
332 priv->comptags.lines = lines;
333 priv->comptags.allocated_lines = ctaglines_to_allocate;
334 priv->comptags.user_mappable = user_mappable;
335 }
336 return err;
337}
283 338
284 if (user_mappable)
285 offset = DIV_ROUND_UP(offset, ctagline_align) * ctagline_align;
286 339
287 priv->comptags.offset = offset;
288 340
289 return 0;
290}
291 341
292static int gk20a_init_mm_reset_enable_hw(struct gk20a *g) 342static int gk20a_init_mm_reset_enable_hw(struct gk20a *g)
293{ 343{
@@ -839,12 +889,14 @@ static void gk20a_vm_unmap_user(struct vm_gk20a *vm, u64 offset)
839} 889}
840 890
841u64 gk20a_vm_alloc_va(struct vm_gk20a *vm, 891u64 gk20a_vm_alloc_va(struct vm_gk20a *vm,
842 u64 size, 892 u64 size,
843 enum gmmu_pgsz_gk20a gmmu_pgsz_idx) 893 enum gmmu_pgsz_gk20a gmmu_pgsz_idx)
844 894
845{ 895{
846 struct gk20a_allocator *vma = &vm->vma[gmmu_pgsz_idx]; 896 struct gk20a_allocator *vma = &vm->vma[gmmu_pgsz_idx];
897 int err;
847 u64 offset; 898 u64 offset;
899 u32 start_page_nr = 0, num_pages;
848 u64 gmmu_page_size = vm->gmmu_page_sizes[gmmu_pgsz_idx]; 900 u64 gmmu_page_size = vm->gmmu_page_sizes[gmmu_pgsz_idx];
849 901
850 if (gmmu_pgsz_idx >= gmmu_nr_page_sizes) { 902 if (gmmu_pgsz_idx >= gmmu_nr_page_sizes) {
@@ -860,19 +912,28 @@ u64 gk20a_vm_alloc_va(struct vm_gk20a *vm,
860 912
861 } 913 }
862 914
863 /* Be certain we round up to gmmu_page_size if needed */ 915 /* be certain we round up to gmmu_page_size if needed */
916 /* TBD: DIV_ROUND_UP -> undefined reference to __aeabi_uldivmod */
864 size = (size + ((u64)gmmu_page_size - 1)) & ~((u64)gmmu_page_size - 1); 917 size = (size + ((u64)gmmu_page_size - 1)) & ~((u64)gmmu_page_size - 1);
918
865 gk20a_dbg_info("size=0x%llx @ pgsz=%dKB", size, 919 gk20a_dbg_info("size=0x%llx @ pgsz=%dKB", size,
866 vm->gmmu_page_sizes[gmmu_pgsz_idx]>>10); 920 vm->gmmu_page_sizes[gmmu_pgsz_idx]>>10);
867 921
868 offset = gk20a_balloc(vma, size); 922 /* The vma allocator represents page accounting. */
869 if (!offset) { 923 num_pages = size >> ilog2(vm->gmmu_page_sizes[gmmu_pgsz_idx]);
924
925 err = vma->alloc(vma, &start_page_nr, num_pages, 1);
926
927 if (err) {
870 gk20a_err(dev_from_vm(vm), 928 gk20a_err(dev_from_vm(vm),
871 "%s oom: sz=0x%llx", vma->name, size); 929 "%s oom: sz=0x%llx", vma->name, size);
872 return 0; 930 return 0;
873 } 931 }
874 932
933 offset = (u64)start_page_nr <<
934 ilog2(vm->gmmu_page_sizes[gmmu_pgsz_idx]);
875 gk20a_dbg_fn("%s found addr: 0x%llx", vma->name, offset); 935 gk20a_dbg_fn("%s found addr: 0x%llx", vma->name, offset);
936
876 return offset; 937 return offset;
877} 938}
878 939
@@ -881,12 +942,25 @@ int gk20a_vm_free_va(struct vm_gk20a *vm,
881 enum gmmu_pgsz_gk20a pgsz_idx) 942 enum gmmu_pgsz_gk20a pgsz_idx)
882{ 943{
883 struct gk20a_allocator *vma = &vm->vma[pgsz_idx]; 944 struct gk20a_allocator *vma = &vm->vma[pgsz_idx];
945 u32 page_size = vm->gmmu_page_sizes[pgsz_idx];
946 u32 page_shift = ilog2(page_size);
947 u32 start_page_nr, num_pages;
948 int err;
884 949
885 gk20a_dbg_info("%s free addr=0x%llx, size=0x%llx", 950 gk20a_dbg_info("%s free addr=0x%llx, size=0x%llx",
886 vma->name, offset, size); 951 vma->name, offset, size);
887 gk20a_bfree(vma, offset);
888 952
889 return 0; 953 start_page_nr = (u32)(offset >> page_shift);
954 num_pages = (u32)((size + page_size - 1) >> page_shift);
955
956 err = vma->free(vma, start_page_nr, num_pages, 1);
957 if (err) {
958 gk20a_err(dev_from_vm(vm),
959 "not found: offset=0x%llx, sz=0x%llx",
960 offset, size);
961 }
962
963 return err;
890} 964}
891 965
892static int insert_mapped_buffer(struct rb_root *root, 966static int insert_mapped_buffer(struct rb_root *root,
@@ -1062,7 +1136,7 @@ static int validate_fixed_buffer(struct vm_gk20a *vm,
1062 1136
1063 if (map_offset & (vm->gmmu_page_sizes[bfr->pgsz_idx] - 1)) { 1137 if (map_offset & (vm->gmmu_page_sizes[bfr->pgsz_idx] - 1)) {
1064 gk20a_err(dev, "map offset must be buffer page size aligned 0x%llx", 1138 gk20a_err(dev, "map offset must be buffer page size aligned 0x%llx",
1065 map_offset); 1139 map_offset);
1066 return -EINVAL; 1140 return -EINVAL;
1067 } 1141 }
1068 1142
@@ -2359,6 +2433,7 @@ int gk20a_init_vm(struct mm_gk20a *mm,
2359 char *name) 2433 char *name)
2360{ 2434{
2361 int err, i; 2435 int err, i;
2436 u32 num_small_pages, num_large_pages, low_hole_pages;
2362 char alloc_name[32]; 2437 char alloc_name[32];
2363 u64 small_vma_size, large_vma_size; 2438 u64 small_vma_size, large_vma_size;
2364 u32 pde_lo, pde_hi; 2439 u32 pde_lo, pde_hi;
@@ -2419,31 +2494,34 @@ int gk20a_init_vm(struct mm_gk20a *mm,
2419 large_vma_size = vm->va_limit - small_vma_size; 2494 large_vma_size = vm->va_limit - small_vma_size;
2420 } 2495 }
2421 2496
2497 num_small_pages = (u32)(small_vma_size >>
2498 ilog2(vm->gmmu_page_sizes[gmmu_page_size_small]));
2499
2500 /* num_pages above is without regard to the low-side hole. */
2501 low_hole_pages = (vm->va_start >>
2502 ilog2(vm->gmmu_page_sizes[gmmu_page_size_small]));
2503
2422 snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-%dKB", name, 2504 snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-%dKB", name,
2423 vm->gmmu_page_sizes[gmmu_page_size_small]>>10); 2505 vm->gmmu_page_sizes[gmmu_page_size_small]>>10);
2424 err = __gk20a_allocator_init(&vm->vma[gmmu_page_size_small], 2506 err = gk20a_allocator_init(&vm->vma[gmmu_page_size_small],
2425 vm, alloc_name, 2507 alloc_name,
2426 vm->va_start, 2508 low_hole_pages, /*start*/
2427 small_vma_size - vm->va_start, 2509 num_small_pages - low_hole_pages);/* length*/
2428 SZ_4K,
2429 GPU_BALLOC_MAX_ORDER,
2430 GPU_BALLOC_GVA_SPACE);
2431 if (err) 2510 if (err)
2432 goto clean_up_ptes; 2511 goto clean_up_ptes;
2433 2512
2434 if (big_pages) { 2513 if (big_pages) {
2514 u32 start = (u32)(small_vma_size >>
2515 ilog2(vm->gmmu_page_sizes[gmmu_page_size_big]));
2516 num_large_pages = (u32)(large_vma_size >>
2517 ilog2(vm->gmmu_page_sizes[gmmu_page_size_big]));
2518
2435 snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-%dKB", 2519 snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-%dKB",
2436 name, vm->gmmu_page_sizes[gmmu_page_size_big]>>10); 2520 name, vm->gmmu_page_sizes[gmmu_page_size_big]>>10);
2437 /* 2521 err = gk20a_allocator_init(&vm->vma[gmmu_page_size_big],
2438 * Big page VMA starts at the end of the small page VMA. 2522 alloc_name,
2439 */ 2523 start, /* start */
2440 err = __gk20a_allocator_init(&vm->vma[gmmu_page_size_big], 2524 num_large_pages); /* length */
2441 vm, alloc_name,
2442 small_vma_size,
2443 large_vma_size,
2444 big_page_size,
2445 GPU_BALLOC_MAX_ORDER,
2446 GPU_BALLOC_GVA_SPACE);
2447 if (err) 2525 if (err)
2448 goto clean_up_small_allocator; 2526 goto clean_up_small_allocator;
2449 } 2527 }
@@ -2524,9 +2602,9 @@ int gk20a_vm_release_share(struct gk20a_as_share *as_share)
2524int gk20a_vm_alloc_space(struct gk20a_as_share *as_share, 2602int gk20a_vm_alloc_space(struct gk20a_as_share *as_share,
2525 struct nvgpu_as_alloc_space_args *args) 2603 struct nvgpu_as_alloc_space_args *args)
2526 2604
2527{ 2605{ int err = -ENOMEM;
2528 int err = -ENOMEM;
2529 int pgsz_idx = gmmu_page_size_small; 2606 int pgsz_idx = gmmu_page_size_small;
2607 u32 start_page_nr;
2530 struct gk20a_allocator *vma; 2608 struct gk20a_allocator *vma;
2531 struct vm_gk20a *vm = as_share->vm; 2609 struct vm_gk20a *vm = as_share->vm;
2532 struct gk20a *g = vm->mm->g; 2610 struct gk20a *g = vm->mm->g;
@@ -2557,19 +2635,21 @@ int gk20a_vm_alloc_space(struct gk20a_as_share *as_share,
2557 goto clean_up; 2635 goto clean_up;
2558 } 2636 }
2559 2637
2560 vma = &vm->vma[pgsz_idx]; 2638 start_page_nr = 0;
2561 if (args->flags & NVGPU_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET) 2639 if (args->flags & NVGPU_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET)
2562 vaddr_start = gk20a_balloc_fixed(vma, args->o_a.offset, 2640 start_page_nr = (u32)(args->o_a.offset >>
2563 (u64)args->pages * 2641 ilog2(vm->gmmu_page_sizes[pgsz_idx]));
2564 (u64)args->page_size);
2565 else
2566 vaddr_start = gk20a_balloc(vma, args->pages * args->page_size);
2567 2642
2568 if (!vaddr_start) { 2643 vma = &vm->vma[pgsz_idx];
2644 err = vma->alloc(vma, &start_page_nr, args->pages, 1);
2645 if (err) {
2569 kfree(va_node); 2646 kfree(va_node);
2570 goto clean_up; 2647 goto clean_up;
2571 } 2648 }
2572 2649
2650 vaddr_start = (u64)start_page_nr <<
2651 ilog2(vm->gmmu_page_sizes[pgsz_idx]);
2652
2573 va_node->vaddr_start = vaddr_start; 2653 va_node->vaddr_start = vaddr_start;
2574 va_node->size = (u64)args->page_size * (u64)args->pages; 2654 va_node->size = (u64)args->page_size * (u64)args->pages;
2575 va_node->pgsz_idx = pgsz_idx; 2655 va_node->pgsz_idx = pgsz_idx;
@@ -2593,7 +2673,7 @@ int gk20a_vm_alloc_space(struct gk20a_as_share *as_share,
2593 true); 2673 true);
2594 if (!map_offset) { 2674 if (!map_offset) {
2595 mutex_unlock(&vm->update_gmmu_lock); 2675 mutex_unlock(&vm->update_gmmu_lock);
2596 gk20a_bfree(vma, vaddr_start); 2676 vma->free(vma, start_page_nr, args->pages, 1);
2597 kfree(va_node); 2677 kfree(va_node);
2598 goto clean_up; 2678 goto clean_up;
2599 } 2679 }
@@ -2605,7 +2685,6 @@ int gk20a_vm_alloc_space(struct gk20a_as_share *as_share,
2605 mutex_unlock(&vm->update_gmmu_lock); 2685 mutex_unlock(&vm->update_gmmu_lock);
2606 2686
2607 args->o_a.offset = vaddr_start; 2687 args->o_a.offset = vaddr_start;
2608 err = 0;
2609 2688
2610clean_up: 2689clean_up:
2611 return err; 2690 return err;
@@ -2616,6 +2695,7 @@ int gk20a_vm_free_space(struct gk20a_as_share *as_share,
2616{ 2695{
2617 int err = -ENOMEM; 2696 int err = -ENOMEM;
2618 int pgsz_idx; 2697 int pgsz_idx;
2698 u32 start_page_nr;
2619 struct gk20a_allocator *vma; 2699 struct gk20a_allocator *vma;
2620 struct vm_gk20a *vm = as_share->vm; 2700 struct vm_gk20a *vm = as_share->vm;
2621 struct vm_reserved_va_node *va_node; 2701 struct vm_reserved_va_node *va_node;
@@ -2628,8 +2708,14 @@ int gk20a_vm_free_space(struct gk20a_as_share *as_share,
2628 pgsz_idx = __nv_gmmu_va_is_upper(vm, args->offset) ? 2708 pgsz_idx = __nv_gmmu_va_is_upper(vm, args->offset) ?
2629 gmmu_page_size_big : gmmu_page_size_small; 2709 gmmu_page_size_big : gmmu_page_size_small;
2630 2710
2711 start_page_nr = (u32)(args->offset >>
2712 ilog2(vm->gmmu_page_sizes[pgsz_idx]));
2713
2631 vma = &vm->vma[pgsz_idx]; 2714 vma = &vm->vma[pgsz_idx];
2632 gk20a_bfree(vma, args->offset); 2715 err = vma->free(vma, start_page_nr, args->pages, 1);
2716
2717 if (err)
2718 goto clean_up;
2633 2719
2634 mutex_lock(&vm->update_gmmu_lock); 2720 mutex_lock(&vm->update_gmmu_lock);
2635 va_node = addr_to_reservation(vm, args->offset); 2721 va_node = addr_to_reservation(vm, args->offset);
@@ -2659,8 +2745,8 @@ int gk20a_vm_free_space(struct gk20a_as_share *as_share,
2659 kfree(va_node); 2745 kfree(va_node);
2660 } 2746 }
2661 mutex_unlock(&vm->update_gmmu_lock); 2747 mutex_unlock(&vm->update_gmmu_lock);
2662 err = 0;
2663 2748
2749clean_up:
2664 return err; 2750 return err;
2665} 2751}
2666 2752
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
index cf246744..57f7a373 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
@@ -131,7 +131,6 @@ enum gmmu_pgsz_gk20a {
131}; 131};
132 132
133struct gk20a_comptags { 133struct gk20a_comptags {
134 u32 real_offset;
135 u32 offset; 134 u32 offset;
136 u32 lines; 135 u32 lines;
137 u32 allocated_lines; 136 u32 allocated_lines;
diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
index fc8d130c..275fbd4e 100644
--- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
@@ -2438,6 +2438,7 @@ static int pmu_init_perfmon(struct pmu_gk20a *pmu)
2438 struct pmu_payload payload; 2438 struct pmu_payload payload;
2439 u32 seq; 2439 u32 seq;
2440 u32 data; 2440 u32 data;
2441 int err = 0;
2441 2442
2442 gk20a_dbg_fn(""); 2443 gk20a_dbg_fn("");
2443 2444
@@ -2488,11 +2489,12 @@ static int pmu_init_perfmon(struct pmu_gk20a *pmu)
2488 gk20a_writel(g, pwr_pmu_idle_ctrl_r(2), data); 2489 gk20a_writel(g, pwr_pmu_idle_ctrl_r(2), data);
2489 2490
2490 if (!pmu->sample_buffer) 2491 if (!pmu->sample_buffer)
2491 pmu->sample_buffer = gk20a_balloc(&pmu->dmem, 2492 err = pmu->dmem.alloc(&pmu->dmem,
2492 2 * sizeof(u16)); 2493 &pmu->sample_buffer, 2 * sizeof(u16),
2493 if (!pmu->sample_buffer) { 2494 PMU_DMEM_ALLOC_ALIGNMENT);
2495 if (err) {
2494 gk20a_err(dev_from_gk20a(g), 2496 gk20a_err(dev_from_gk20a(g),
2495 "failed to allocate perfmon sample buffer"); 2497 "failed to allocate perfmon sample buffer");
2496 return -ENOMEM; 2498 return -ENOMEM;
2497 } 2499 }
2498 2500
@@ -2590,17 +2592,15 @@ static int pmu_process_init_msg(struct pmu_gk20a *pmu,
2590 for (i = 0; i < PMU_QUEUE_COUNT; i++) 2592 for (i = 0; i < PMU_QUEUE_COUNT; i++)
2591 pmu_queue_init(pmu, i, init); 2593 pmu_queue_init(pmu, i, init);
2592 2594
2593 if (!pmu->dmem.init) { 2595 if (!pmu->dmem.alloc) {
2594 /* Align start and end addresses */ 2596 /*Align start and end addresses*/
2595 u32 start = ALIGN(pv->get_pmu_init_msg_pmu_sw_mg_off(init), 2597 u32 start = ALIGN(pv->get_pmu_init_msg_pmu_sw_mg_off(init),
2596 PMU_DMEM_ALLOC_ALIGNMENT); 2598 PMU_DMEM_ALLOC_ALIGNMENT);
2597 u32 end = (pv->get_pmu_init_msg_pmu_sw_mg_off(init) + 2599 u32 end = (pv->get_pmu_init_msg_pmu_sw_mg_off(init) +
2598 pv->get_pmu_init_msg_pmu_sw_mg_size(init)) & 2600 pv->get_pmu_init_msg_pmu_sw_mg_size(init)) &
2599 ~(PMU_DMEM_ALLOC_ALIGNMENT - 1); 2601 ~(PMU_DMEM_ALLOC_ALIGNMENT - 1);
2600 u32 size = end - start; 2602 u32 size = end - start;
2601 __gk20a_allocator_init(&pmu->dmem, NULL, "gk20a_pmu_dmem", 2603 gk20a_allocator_init(&pmu->dmem, "gk20a_pmu_dmem", start, size);
2602 start, size,
2603 PMU_DMEM_ALLOC_ALIGNMENT, 4, 0);
2604 } 2604 }
2605 2605
2606 pmu->pmu_ready = true; 2606 pmu->pmu_ready = true;
@@ -2737,14 +2737,20 @@ static int pmu_response_handle(struct pmu_gk20a *pmu,
2737 seq->callback = NULL; 2737 seq->callback = NULL;
2738 if (pv->pmu_allocation_get_dmem_size(pmu, 2738 if (pv->pmu_allocation_get_dmem_size(pmu,
2739 pv->get_pmu_seq_in_a_ptr(seq)) != 0) 2739 pv->get_pmu_seq_in_a_ptr(seq)) != 0)
2740 gk20a_bfree(&pmu->dmem, 2740 pmu->dmem.free(&pmu->dmem,
2741 pv->pmu_allocation_get_dmem_offset(pmu, 2741 pv->pmu_allocation_get_dmem_offset(pmu,
2742 pv->get_pmu_seq_in_a_ptr(seq))); 2742 pv->get_pmu_seq_in_a_ptr(seq)),
2743 pv->pmu_allocation_get_dmem_size(pmu,
2744 pv->get_pmu_seq_in_a_ptr(seq)),
2745 PMU_DMEM_ALLOC_ALIGNMENT);
2743 if (pv->pmu_allocation_get_dmem_size(pmu, 2746 if (pv->pmu_allocation_get_dmem_size(pmu,
2744 pv->get_pmu_seq_out_a_ptr(seq)) != 0) 2747 pv->get_pmu_seq_out_a_ptr(seq)) != 0)
2745 gk20a_bfree(&pmu->dmem, 2748 pmu->dmem.free(&pmu->dmem,
2746 pv->pmu_allocation_get_dmem_offset(pmu, 2749 pv->pmu_allocation_get_dmem_offset(pmu,
2747 pv->get_pmu_seq_out_a_ptr(seq))); 2750 pv->get_pmu_seq_out_a_ptr(seq)),
2751 pv->pmu_allocation_get_dmem_size(pmu,
2752 pv->get_pmu_seq_out_a_ptr(seq)),
2753 PMU_DMEM_ALLOC_ALIGNMENT);
2748 2754
2749 if (seq->callback) 2755 if (seq->callback)
2750 seq->callback(g, msg, seq->cb_params, seq->desc, ret); 2756 seq->callback(g, msg, seq->cb_params, seq->desc, ret);
@@ -3381,10 +3387,11 @@ int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd,
3381 pv->pmu_allocation_set_dmem_size(pmu, in, 3387 pv->pmu_allocation_set_dmem_size(pmu, in,
3382 (u16)max(payload->in.size, payload->out.size)); 3388 (u16)max(payload->in.size, payload->out.size));
3383 3389
3384 *(pv->pmu_allocation_get_dmem_offset_addr(pmu, in)) = 3390 err = pmu->dmem.alloc(&pmu->dmem,
3385 gk20a_balloc(&pmu->dmem, 3391 pv->pmu_allocation_get_dmem_offset_addr(pmu, in),
3386 pv->pmu_allocation_get_dmem_size(pmu, in)); 3392 pv->pmu_allocation_get_dmem_size(pmu, in),
3387 if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu, in))) 3393 PMU_DMEM_ALLOC_ALIGNMENT);
3394 if (err)
3388 goto clean_up; 3395 goto clean_up;
3389 3396
3390 pmu_copy_to_dmem(pmu, (pv->pmu_allocation_get_dmem_offset(pmu, 3397 pmu_copy_to_dmem(pmu, (pv->pmu_allocation_get_dmem_offset(pmu,
@@ -3405,12 +3412,11 @@ int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd,
3405 (u16)payload->out.size); 3412 (u16)payload->out.size);
3406 3413
3407 if (payload->out.buf != payload->in.buf) { 3414 if (payload->out.buf != payload->in.buf) {
3408 3415 err = pmu->dmem.alloc(&pmu->dmem,
3409 *(pv->pmu_allocation_get_dmem_offset_addr(pmu, out)) = 3416 pv->pmu_allocation_get_dmem_offset_addr(pmu, out),
3410 gk20a_balloc(&pmu->dmem, 3417 pv->pmu_allocation_get_dmem_size(pmu, out),
3411 pv->pmu_allocation_get_dmem_size(pmu, out)); 3418 PMU_DMEM_ALLOC_ALIGNMENT);
3412 if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu, 3419 if (err)
3413 out)))
3414 goto clean_up; 3420 goto clean_up;
3415 } else { 3421 } else {
3416 BUG_ON(in == NULL); 3422 BUG_ON(in == NULL);
@@ -3438,11 +3444,15 @@ int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd,
3438clean_up: 3444clean_up:
3439 gk20a_dbg_fn("fail"); 3445 gk20a_dbg_fn("fail");
3440 if (in) 3446 if (in)
3441 gk20a_bfree(&pmu->dmem, 3447 pmu->dmem.free(&pmu->dmem,
3442 pv->pmu_allocation_get_dmem_offset(pmu, in)); 3448 pv->pmu_allocation_get_dmem_offset(pmu, in),
3449 pv->pmu_allocation_get_dmem_size(pmu, in),
3450 PMU_DMEM_ALLOC_ALIGNMENT);
3443 if (out) 3451 if (out)
3444 gk20a_bfree(&pmu->dmem, 3452 pmu->dmem.free(&pmu->dmem,
3445 pv->pmu_allocation_get_dmem_offset(pmu, out)); 3453 pv->pmu_allocation_get_dmem_offset(pmu, out),
3454 pv->pmu_allocation_get_dmem_size(pmu, out),
3455 PMU_DMEM_ALLOC_ALIGNMENT);
3446 3456
3447 pmu_seq_release(pmu, seq); 3457 pmu_seq_release(pmu, seq);
3448 return err; 3458 return err;
diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h
index e54805a6..6cd173e8 100644
--- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h
@@ -3,7 +3,7 @@
3 * 3 *
4 * GK20A PMU (aka. gPMU outside gk20a context) 4 * GK20A PMU (aka. gPMU outside gk20a context)
5 * 5 *
6 * Copyright (c) 2011-2015, NVIDIA CORPORATION. All rights reserved. 6 * Copyright (c) 2011-2014, NVIDIA CORPORATION. All rights reserved.
7 * 7 *
8 * This program is free software; you can redistribute it and/or modify it 8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License, 9 * under the terms and conditions of the GNU General Public License,
@@ -437,7 +437,7 @@ struct pmu_ucode_desc {
437#define PMU_UNIT_ID_IS_VALID(id) \ 437#define PMU_UNIT_ID_IS_VALID(id) \
438 (((id) < PMU_UNIT_END) || ((id) >= PMU_UNIT_TEST_START)) 438 (((id) < PMU_UNIT_END) || ((id) >= PMU_UNIT_TEST_START))
439 439
440#define PMU_DMEM_ALLOC_ALIGNMENT (4) 440#define PMU_DMEM_ALLOC_ALIGNMENT (32)
441#define PMU_DMEM_ALIGNMENT (4) 441#define PMU_DMEM_ALIGNMENT (4)
442 442
443#define PMU_CMD_FLAGS_PMU_MASK (0xF0) 443#define PMU_CMD_FLAGS_PMU_MASK (0xF0)
diff --git a/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.c b/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.c
index 053550f6..04f61c58 100644
--- a/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.c
@@ -3,7 +3,7 @@
3 * 3 *
4 * GK20A Semaphores 4 * GK20A Semaphores
5 * 5 *
6 * Copyright (c) 2014-2015, NVIDIA CORPORATION. All rights reserved. 6 * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.
7 * 7 *
8 * This program is free software; you can redistribute it and/or modify it 8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License, 9 * under the terms and conditions of the GNU General Public License,
@@ -44,10 +44,8 @@ struct gk20a_semaphore_pool *gk20a_semaphore_pool_alloc(struct device *d,
44 if (gk20a_get_sgtable(d, &p->sgt, p->cpu_va, p->iova, p->size)) 44 if (gk20a_get_sgtable(d, &p->sgt, p->cpu_va, p->iova, p->size))
45 goto clean_up; 45 goto clean_up;
46 46
47 /* Sacrifice one semaphore in the name of returning error codes. */ 47 if (gk20a_allocator_init(&p->alloc, unique_name, 0,
48 if (gk20a_allocator_init(&p->alloc, unique_name, 48 p->size))
49 SEMAPHORE_SIZE, p->size - SEMAPHORE_SIZE,
50 SEMAPHORE_SIZE))
51 goto clean_up; 49 goto clean_up;
52 50
53 gk20a_dbg_info("cpuva=%p iova=%llx phys=%llx", p->cpu_va, 51 gk20a_dbg_info("cpuva=%p iova=%llx phys=%llx", p->cpu_va,
@@ -165,8 +163,8 @@ struct gk20a_semaphore *gk20a_semaphore_alloc(struct gk20a_semaphore_pool *pool)
165 if (!s) 163 if (!s)
166 return NULL; 164 return NULL;
167 165
168 s->offset = gk20a_balloc(&pool->alloc, SEMAPHORE_SIZE); 166 if (pool->alloc.alloc(&pool->alloc, &s->offset, SEMAPHORE_SIZE,
169 if (!s->offset) { 167 SEMAPHORE_SIZE)) {
170 gk20a_err(pool->dev, "failed to allocate semaphore"); 168 gk20a_err(pool->dev, "failed to allocate semaphore");
171 kfree(s); 169 kfree(s);
172 return NULL; 170 return NULL;
@@ -188,7 +186,8 @@ static void gk20a_semaphore_free(struct kref *ref)
188 struct gk20a_semaphore *s = 186 struct gk20a_semaphore *s =
189 container_of(ref, struct gk20a_semaphore, ref); 187 container_of(ref, struct gk20a_semaphore, ref);
190 188
191 gk20a_bfree(&s->pool->alloc, s->offset); 189 s->pool->alloc.free(&s->pool->alloc, s->offset, SEMAPHORE_SIZE,
190 SEMAPHORE_SIZE);
192 gk20a_semaphore_pool_put(s->pool); 191 gk20a_semaphore_pool_put(s->pool);
193 kfree(s); 192 kfree(s);
194} 193}
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-07 15:30:59 -0400