summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common/mm/buddy_allocator.c')
-rw-r--r--drivers/gpu/nvgpu/common/mm/buddy_allocator.c148
1 files changed, 96 insertions, 52 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
index 5a0a28cb..365f3b7b 100644
--- a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
+++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
@@ -79,10 +79,12 @@ static void balloc_compute_max_order(struct nvgpu_buddy_allocator *a)
79 return; 79 return;
80 } 80 }
81 81
82 if (a->max_order > true_max_order) 82 if (a->max_order > true_max_order) {
83 a->max_order = true_max_order; 83 a->max_order = true_max_order;
84 if (a->max_order > GPU_BALLOC_MAX_ORDER) 84 }
85 if (a->max_order > GPU_BALLOC_MAX_ORDER) {
85 a->max_order = GPU_BALLOC_MAX_ORDER; 86 a->max_order = GPU_BALLOC_MAX_ORDER;
87 }
86} 88}
87 89
88/* 90/*
@@ -108,8 +110,9 @@ static struct nvgpu_buddy *balloc_new_buddy(struct nvgpu_buddy_allocator *a,
108 struct nvgpu_buddy *new_buddy; 110 struct nvgpu_buddy *new_buddy;
109 111
110 new_buddy = nvgpu_kmem_cache_alloc(a->buddy_cache); 112 new_buddy = nvgpu_kmem_cache_alloc(a->buddy_cache);
111 if (!new_buddy) 113 if (!new_buddy) {
112 return NULL; 114 return NULL;
115 }
113 116
114 memset(new_buddy, 0, sizeof(struct nvgpu_buddy)); 117 memset(new_buddy, 0, sizeof(struct nvgpu_buddy));
115 118
@@ -139,10 +142,11 @@ static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a,
139 * without cycling through the entire list. 142 * without cycling through the entire list.
140 */ 143 */
141 if (a->flags & GPU_ALLOC_GVA_SPACE && 144 if (a->flags & GPU_ALLOC_GVA_SPACE &&
142 b->pte_size == gmmu_page_size_big) 145 b->pte_size == gmmu_page_size_big) {
143 nvgpu_list_add_tail(&b->buddy_entry, list); 146 nvgpu_list_add_tail(&b->buddy_entry, list);
144 else 147 } else {
145 nvgpu_list_add(&b->buddy_entry, list); 148 nvgpu_list_add(&b->buddy_entry, list);
149 }
146 150
147 buddy_set_in_list(b); 151 buddy_set_in_list(b);
148} 152}
@@ -181,8 +185,9 @@ static void balloc_blist_rem(struct nvgpu_buddy_allocator *a,
181 185
182static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len) 186static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len)
183{ 187{
184 if (len == 0) 188 if (len == 0) {
185 return 0; 189 return 0;
190 }
186 191
187 len--; 192 len--;
188 len >>= a->blk_shift; 193 len >>= a->blk_shift;
@@ -195,10 +200,11 @@ static u64 __balloc_max_order_in(struct nvgpu_buddy_allocator *a,
195{ 200{
196 u64 size = (end - start) >> a->blk_shift; 201 u64 size = (end - start) >> a->blk_shift;
197 202
198 if (size > 0) 203 if (size > 0) {
199 return min_t(u64, ilog2(size), a->max_order); 204 return min_t(u64, ilog2(size), a->max_order);
200 else 205 } else {
201 return GPU_BALLOC_MAX_ORDER; 206 return GPU_BALLOC_MAX_ORDER;
207 }
202} 208}
203 209
204/* 210/*
@@ -222,8 +228,9 @@ static int balloc_init_lists(struct nvgpu_buddy_allocator *a)
222 order = __balloc_max_order_in(a, bstart, bend); 228 order = __balloc_max_order_in(a, bstart, bend);
223 229
224 buddy = balloc_new_buddy(a, NULL, bstart, order); 230 buddy = balloc_new_buddy(a, NULL, bstart, order);
225 if (!buddy) 231 if (!buddy) {
226 goto cleanup; 232 goto cleanup;
233 }
227 234
228 balloc_blist_add(a, buddy); 235 balloc_blist_add(a, buddy);
229 bstart += balloc_order_to_len(a, order); 236 bstart += balloc_order_to_len(a, order);
@@ -340,17 +347,20 @@ static void balloc_coalesce(struct nvgpu_buddy_allocator *a,
340{ 347{
341 struct nvgpu_buddy *parent; 348 struct nvgpu_buddy *parent;
342 349
343 if (buddy_is_alloced(b) || buddy_is_split(b)) 350 if (buddy_is_alloced(b) || buddy_is_split(b)) {
344 return; 351 return;
352 }
345 353
346 /* 354 /*
347 * If both our buddy and I are both not allocated and not split then 355 * If both our buddy and I are both not allocated and not split then
348 * we can coalesce ourselves. 356 * we can coalesce ourselves.
349 */ 357 */
350 if (!b->buddy) 358 if (!b->buddy) {
351 return; 359 return;
352 if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy)) 360 }
361 if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy)) {
353 return; 362 return;
363 }
354 364
355 parent = b->parent; 365 parent = b->parent;
356 366
@@ -383,8 +393,9 @@ static int balloc_split_buddy(struct nvgpu_buddy_allocator *a,
383 u64 half; 393 u64 half;
384 394
385 left = balloc_new_buddy(a, b, b->start, b->order - 1); 395 left = balloc_new_buddy(a, b, b->start, b->order - 1);
386 if (!left) 396 if (!left) {
387 return -ENOMEM; 397 return -ENOMEM;
398 }
388 399
389 half = (b->end - b->start) / 2; 400 half = (b->end - b->start) / 2;
390 401
@@ -449,8 +460,9 @@ static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a,
449 struct nvgpu_buddy *bud; 460 struct nvgpu_buddy *bud;
450 461
451 nvgpu_rbtree_search(addr, &node, a->alloced_buddies); 462 nvgpu_rbtree_search(addr, &node, a->alloced_buddies);
452 if (!node) 463 if (!node) {
453 return NULL; 464 return NULL;
465 }
454 466
455 bud = nvgpu_buddy_from_rbtree_node(node); 467 bud = nvgpu_buddy_from_rbtree_node(node);
456 468
@@ -470,21 +482,24 @@ static struct nvgpu_buddy *__balloc_find_buddy(struct nvgpu_buddy_allocator *a,
470 struct nvgpu_buddy *bud; 482 struct nvgpu_buddy *bud;
471 483
472 if (order > a->max_order || 484 if (order > a->max_order ||
473 nvgpu_list_empty(balloc_get_order_list(a, order))) 485 nvgpu_list_empty(balloc_get_order_list(a, order))) {
474 return NULL; 486 return NULL;
487 }
475 488
476 if (a->flags & GPU_ALLOC_GVA_SPACE && 489 if (a->flags & GPU_ALLOC_GVA_SPACE &&
477 pte_size == gmmu_page_size_big) 490 pte_size == gmmu_page_size_big) {
478 bud = nvgpu_list_last_entry(balloc_get_order_list(a, order), 491 bud = nvgpu_list_last_entry(balloc_get_order_list(a, order),
479 nvgpu_buddy, buddy_entry); 492 nvgpu_buddy, buddy_entry);
480 else 493 } else {
481 bud = nvgpu_list_first_entry(balloc_get_order_list(a, order), 494 bud = nvgpu_list_first_entry(balloc_get_order_list(a, order),
482 nvgpu_buddy, buddy_entry); 495 nvgpu_buddy, buddy_entry);
496 }
483 497
484 if (pte_size != BALLOC_PTE_SIZE_ANY && 498 if (pte_size != BALLOC_PTE_SIZE_ANY &&
485 pte_size != bud->pte_size && 499 pte_size != bud->pte_size &&
486 bud->pte_size != BALLOC_PTE_SIZE_ANY) 500 bud->pte_size != BALLOC_PTE_SIZE_ANY) {
487 return NULL; 501 return NULL;
502 }
488 503
489 return bud; 504 return bud;
490} 505}
@@ -511,12 +526,14 @@ static u64 __balloc_do_alloc(struct nvgpu_buddy_allocator *a,
511 } 526 }
512 527
513 /* Out of memory! */ 528 /* Out of memory! */
514 if (!bud) 529 if (!bud) {
515 return 0; 530 return 0;
531 }
516 532
517 while (bud->order != order) { 533 while (bud->order != order) {
518 if (balloc_split_buddy(a, bud, pte_size)) 534 if (balloc_split_buddy(a, bud, pte_size)) {
519 return 0; /* No mem... */ 535 return 0; /* No mem... */
536 }
520 bud = bud->left; 537 bud = bud->left;
521 } 538 }
522 539
@@ -540,19 +557,22 @@ static int balloc_is_range_free(struct nvgpu_buddy_allocator *a,
540 struct nvgpu_buddy *bud; 557 struct nvgpu_buddy *bud;
541 558
542 nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies); 559 nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies);
543 if (!node) 560 if (!node) {
544 return 1; /* No allocs yet. */ 561 return 1; /* No allocs yet. */
562 }
545 563
546 bud = nvgpu_buddy_from_rbtree_node(node); 564 bud = nvgpu_buddy_from_rbtree_node(node);
547 565
548 while (bud->start < end) { 566 while (bud->start < end) {
549 if ((bud->start > base && bud->start < end) || 567 if ((bud->start > base && bud->start < end) ||
550 (bud->end > base && bud->end < end)) 568 (bud->end > base && bud->end < end)) {
551 return 0; 569 return 0;
570 }
552 571
553 nvgpu_rbtree_enum_next(&node, node); 572 nvgpu_rbtree_enum_next(&node, node);
554 if (!node) 573 if (!node) {
555 break; 574 break;
575 }
556 bud = nvgpu_buddy_from_rbtree_node(node); 576 bud = nvgpu_buddy_from_rbtree_node(node);
557 } 577 }
558 578
@@ -581,8 +601,9 @@ static struct nvgpu_fixed_alloc *balloc_free_fixed(
581 struct nvgpu_rbtree_node *node = NULL; 601 struct nvgpu_rbtree_node *node = NULL;
582 602
583 nvgpu_rbtree_search(addr, &node, a->fixed_allocs); 603 nvgpu_rbtree_search(addr, &node, a->fixed_allocs);
584 if (!node) 604 if (!node) {
585 return NULL; 605 return NULL;
606 }
586 607
587 falloc = nvgpu_fixed_alloc_from_rbtree_node(node); 608 falloc = nvgpu_fixed_alloc_from_rbtree_node(node);
588 609
@@ -657,8 +678,9 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy(
657 } 678 }
658 } 679 }
659 680
660 if (found) 681 if (found) {
661 break; 682 break;
683 }
662 684
663 __balloc_get_parent_range(a, cur_base, cur_order, 685 __balloc_get_parent_range(a, cur_base, cur_order,
664 &cur_base, &cur_order); 686 &cur_base, &cur_order);
@@ -679,10 +701,11 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy(
679 return NULL; 701 return NULL;
680 } 702 }
681 703
682 if (base < bud->right->start) 704 if (base < bud->right->start) {
683 bud = bud->left; 705 bud = bud->left;
684 else 706 } else {
685 bud = bud->right; 707 bud = bud->right;
708 }
686 709
687 } 710 }
688 711
@@ -697,12 +720,13 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a,
697 u64 align_order; 720 u64 align_order;
698 721
699 shifted_base = balloc_base_shift(a, base); 722 shifted_base = balloc_base_shift(a, base);
700 if (shifted_base == 0) 723 if (shifted_base == 0) {
701 align_order = __fls(len >> a->blk_shift); 724 align_order = __fls(len >> a->blk_shift);
702 else 725 } else {
703 align_order = min_t(u64, 726 align_order = min_t(u64,
704 __ffs(shifted_base >> a->blk_shift), 727 __ffs(shifted_base >> a->blk_shift),
705 __fls(len >> a->blk_shift)); 728 __fls(len >> a->blk_shift));
729 }
706 730
707 if (align_order > a->max_order) { 731 if (align_order > a->max_order) {
708 alloc_dbg(balloc_owner(a), 732 alloc_dbg(balloc_owner(a),
@@ -741,9 +765,10 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a,
741 align_order = __ffs(inc_base >> a->blk_shift); 765 align_order = __ffs(inc_base >> a->blk_shift);
742 766
743 /* If we don't have much left - trim down align_order. */ 767 /* If we don't have much left - trim down align_order. */
744 if (balloc_order_to_len(a, align_order) > remaining) 768 if (balloc_order_to_len(a, align_order) > remaining) {
745 align_order = __balloc_max_order_in(a, inc_base, 769 align_order = __balloc_max_order_in(a, inc_base,
746 inc_base + remaining); 770 inc_base + remaining);
771 }
747 } 772 }
748 773
749 return base; 774 return base;
@@ -805,10 +830,11 @@ static u64 nvgpu_buddy_balloc(struct nvgpu_allocator *__a, u64 len)
805 return 0; 830 return 0;
806 } 831 }
807 832
808 if (a->flags & GPU_ALLOC_GVA_SPACE) 833 if (a->flags & GPU_ALLOC_GVA_SPACE) {
809 pte_size = __get_pte_size(a->vm, 0, len); 834 pte_size = __get_pte_size(a->vm, 0, len);
810 else 835 } else {
811 pte_size = BALLOC_PTE_SIZE_ANY; 836 pte_size = BALLOC_PTE_SIZE_ANY;
837 }
812 838
813 addr = __balloc_do_alloc(a, order, pte_size); 839 addr = __balloc_do_alloc(a, order, pte_size);
814 840
@@ -845,25 +871,29 @@ static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a,
845 struct nvgpu_buddy_allocator *a = __a->priv; 871 struct nvgpu_buddy_allocator *a = __a->priv;
846 872
847 /* If base isn't aligned to an order 0 block, fail. */ 873 /* If base isn't aligned to an order 0 block, fail. */
848 if (base & (a->blk_size - 1)) 874 if (base & (a->blk_size - 1)) {
849 goto fail; 875 goto fail;
876 }
850 877
851 if (len == 0) 878 if (len == 0) {
852 goto fail; 879 goto fail;
880 }
853 881
854 /* Check that the page size is valid. */ 882 /* Check that the page size is valid. */
855 if (a->flags & GPU_ALLOC_GVA_SPACE && a->vm->big_pages) { 883 if (a->flags & GPU_ALLOC_GVA_SPACE && a->vm->big_pages) {
856 if (page_size == a->vm->big_page_size) 884 if (page_size == a->vm->big_page_size) {
857 pte_size = gmmu_page_size_big; 885 pte_size = gmmu_page_size_big;
858 else if (page_size == SZ_4K) 886 } else if (page_size == SZ_4K) {
859 pte_size = gmmu_page_size_small; 887 pte_size = gmmu_page_size_small;
860 else 888 } else {
861 goto fail; 889 goto fail;
890 }
862 } 891 }
863 892
864 falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc)); 893 falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc));
865 if (!falloc) 894 if (!falloc) {
866 goto fail; 895 goto fail;
896 }
867 897
868 nvgpu_init_list_node(&falloc->buddies); 898 nvgpu_init_list_node(&falloc->buddies);
869 falloc->start = base; 899 falloc->start = base;
@@ -936,8 +966,9 @@ static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr)
936 struct nvgpu_fixed_alloc *falloc; 966 struct nvgpu_fixed_alloc *falloc;
937 struct nvgpu_buddy_allocator *a = __a->priv; 967 struct nvgpu_buddy_allocator *a = __a->priv;
938 968
939 if (!addr) 969 if (!addr) {
940 return; 970 return;
971 }
941 972
942 alloc_lock(__a); 973 alloc_lock(__a);
943 974
@@ -952,8 +983,9 @@ static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr)
952 } 983 }
953 984
954 bud = balloc_free_buddy(a, addr); 985 bud = balloc_free_buddy(a, addr);
955 if (!bud) 986 if (!bud) {
956 goto done; 987 goto done;
988 }
957 989
958 balloc_blist_add(a, bud); 990 balloc_blist_add(a, bud);
959 a->bytes_freed += balloc_order_to_len(a, bud->order); 991 a->bytes_freed += balloc_order_to_len(a, bud->order);
@@ -987,8 +1019,9 @@ static bool nvgpu_buddy_reserve_is_possible(struct nvgpu_buddy_allocator *a,
987 if ((co_base >= tmp->base && 1019 if ((co_base >= tmp->base &&
988 co_base < (tmp->base + tmp->length)) || 1020 co_base < (tmp->base + tmp->length)) ||
989 (co_end >= tmp->base && 1021 (co_end >= tmp->base &&
990 co_end < (tmp->base + tmp->length))) 1022 co_end < (tmp->base + tmp->length))) {
991 return false; 1023 return false;
1024 }
992 } 1025 }
993 1026
994 return true; 1027 return true;
@@ -1006,8 +1039,9 @@ static int nvgpu_buddy_reserve_co(struct nvgpu_allocator *__a,
1006 int err = 0; 1039 int err = 0;
1007 1040
1008 if (co->base < a->start || (co->base + co->length) > a->end || 1041 if (co->base < a->start || (co->base + co->length) > a->end ||
1009 a->alloc_made) 1042 a->alloc_made) {
1010 return -EINVAL; 1043 return -EINVAL;
1044 }
1011 1045
1012 alloc_lock(__a); 1046 alloc_lock(__a);
1013 1047
@@ -1221,25 +1255,31 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
1221 struct nvgpu_buddy_allocator *a; 1255 struct nvgpu_buddy_allocator *a;
1222 1256
1223 /* blk_size must be greater than 0 and a power of 2. */ 1257 /* blk_size must be greater than 0 and a power of 2. */
1224 if (blk_size == 0) 1258 if (blk_size == 0) {
1225 return -EINVAL; 1259 return -EINVAL;
1226 if (blk_size & (blk_size - 1)) 1260 }
1261 if (blk_size & (blk_size - 1)) {
1227 return -EINVAL; 1262 return -EINVAL;
1263 }
1228 1264
1229 if (max_order > GPU_BALLOC_MAX_ORDER) 1265 if (max_order > GPU_BALLOC_MAX_ORDER) {
1230 return -EINVAL; 1266 return -EINVAL;
1267 }
1231 1268
1232 /* If this is to manage a GVA space we need a VM. */ 1269 /* If this is to manage a GVA space we need a VM. */
1233 if (flags & GPU_ALLOC_GVA_SPACE && !vm) 1270 if (flags & GPU_ALLOC_GVA_SPACE && !vm) {
1234 return -EINVAL; 1271 return -EINVAL;
1272 }
1235 1273
1236 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator)); 1274 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator));
1237 if (!a) 1275 if (!a) {
1238 return -ENOMEM; 1276 return -ENOMEM;
1277 }
1239 1278
1240 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops); 1279 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops);
1241 if (err) 1280 if (err) {
1242 goto fail; 1281 goto fail;
1282 }
1243 1283
1244 a->base = base; 1284 a->base = base;
1245 a->length = size; 1285 a->length = size;
@@ -1269,8 +1309,9 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
1269 */ 1309 */
1270 if (flags & GPU_ALLOC_GVA_SPACE && vm->big_pages && 1310 if (flags & GPU_ALLOC_GVA_SPACE && vm->big_pages &&
1271 (base & ((vm->big_page_size << 10) - 1) || 1311 (base & ((vm->big_page_size << 10) - 1) ||
1272 size & ((vm->big_page_size << 10) - 1))) 1312 size & ((vm->big_page_size << 10) - 1))) {
1273 return -EINVAL; 1313 return -EINVAL;
1314 }
1274 1315
1275 a->flags = flags; 1316 a->flags = flags;
1276 a->max_order = max_order; 1317 a->max_order = max_order;
@@ -1288,8 +1329,9 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
1288 a->fixed_allocs = NULL; 1329 a->fixed_allocs = NULL;
1289 nvgpu_init_list_node(&a->co_list); 1330 nvgpu_init_list_node(&a->co_list);
1290 err = balloc_init_lists(a); 1331 err = balloc_init_lists(a);
1291 if (err) 1332 if (err) {
1292 goto fail; 1333 goto fail;
1334 }
1293 1335
1294 nvgpu_smp_wmb(); 1336 nvgpu_smp_wmb();
1295 a->initialized = 1; 1337 a->initialized = 1;
@@ -1301,18 +1343,20 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
1301 alloc_dbg(__a, " base 0x%llx", a->base); 1343 alloc_dbg(__a, " base 0x%llx", a->base);
1302 alloc_dbg(__a, " size 0x%llx", a->length); 1344 alloc_dbg(__a, " size 0x%llx", a->length);
1303 alloc_dbg(__a, " blk_size 0x%llx", a->blk_size); 1345 alloc_dbg(__a, " blk_size 0x%llx", a->blk_size);
1304 if (flags & GPU_ALLOC_GVA_SPACE) 1346 if (flags & GPU_ALLOC_GVA_SPACE) {
1305 alloc_dbg(balloc_owner(a), 1347 alloc_dbg(balloc_owner(a),
1306 " pde_size 0x%llx", 1348 " pde_size 0x%llx",
1307 balloc_order_to_len(a, a->pte_blk_order)); 1349 balloc_order_to_len(a, a->pte_blk_order));
1350 }
1308 alloc_dbg(__a, " max_order %llu", a->max_order); 1351 alloc_dbg(__a, " max_order %llu", a->max_order);
1309 alloc_dbg(__a, " flags 0x%llx", a->flags); 1352 alloc_dbg(__a, " flags 0x%llx", a->flags);
1310 1353
1311 return 0; 1354 return 0;
1312 1355
1313fail: 1356fail:
1314 if (a->buddy_cache) 1357 if (a->buddy_cache) {
1315 nvgpu_kmem_cache_destroy(a->buddy_cache); 1358 nvgpu_kmem_cache_destroy(a->buddy_cache);
1359 }
1316 nvgpu_kfree(g, a); 1360 nvgpu_kfree(g, a);
1317 return err; 1361 return err;
1318} 1362}