summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSrirangan <smadhavan@nvidia.com>2018-08-14 01:27:15 -0400
committermobile promotions <svcmobile_promotions@nvidia.com>2018-08-17 16:54:08 -0400
commit70c20bb75be7815ebc67ac82d6999f46bc25ed6d (patch)
tree19d6b5299e09b71e9afe2967a758f036bb9b79bc
parent553fdf3534f856edce73744fd54914b9b7a829cc (diff)
gpu: nvgpu: common: mm: Fix MISRA 15.6 violations
MISRA Rule-15.6 requires that all if-else blocks be enclosed in braces, including single statement blocks. Fix errors due to single statement if blocks without braces, introducing the braces. JIRA NVGPU-671 Change-Id: Ieeecf719dca9acc1a116d2893637bf770caf4f5b Signed-off-by: Srirangan <smadhavan@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/1794241 GVS: Gerrit_Virtual_Submit Reviewed-by: Adeel Raza <araza@nvidia.com> Reviewed-by: Alex Waterman <alexw@nvidia.com> Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com> Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
-rw-r--r--drivers/gpu/nvgpu/common/mm/buddy_allocator.c148
-rw-r--r--drivers/gpu/nvgpu/common/mm/gmmu.c98
-rw-r--r--drivers/gpu/nvgpu/common/mm/lockless_allocator.c20
-rw-r--r--drivers/gpu/nvgpu/common/mm/mm.c119
-rw-r--r--drivers/gpu/nvgpu/common/mm/nvgpu_mem.c23
-rw-r--r--drivers/gpu/nvgpu/common/mm/vm.c123
6 files changed, 348 insertions, 183 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
index 5a0a28cb..365f3b7b 100644
--- a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
+++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c
@@ -79,10 +79,12 @@ static void balloc_compute_max_order(struct nvgpu_buddy_allocator *a)
79 return; 79 return;
80 } 80 }
81 81
82 if (a->max_order > true_max_order) 82 if (a->max_order > true_max_order) {
83 a->max_order = true_max_order; 83 a->max_order = true_max_order;
84 if (a->max_order > GPU_BALLOC_MAX_ORDER) 84 }
85 if (a->max_order > GPU_BALLOC_MAX_ORDER) {
85 a->max_order = GPU_BALLOC_MAX_ORDER; 86 a->max_order = GPU_BALLOC_MAX_ORDER;
87 }
86} 88}
87 89
88/* 90/*
@@ -108,8 +110,9 @@ static struct nvgpu_buddy *balloc_new_buddy(struct nvgpu_buddy_allocator *a,
108 struct nvgpu_buddy *new_buddy; 110 struct nvgpu_buddy *new_buddy;
109 111
110 new_buddy = nvgpu_kmem_cache_alloc(a->buddy_cache); 112 new_buddy = nvgpu_kmem_cache_alloc(a->buddy_cache);
111 if (!new_buddy) 113 if (!new_buddy) {
112 return NULL; 114 return NULL;
115 }
113 116
114 memset(new_buddy, 0, sizeof(struct nvgpu_buddy)); 117 memset(new_buddy, 0, sizeof(struct nvgpu_buddy));
115 118
@@ -139,10 +142,11 @@ static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a,
139 * without cycling through the entire list. 142 * without cycling through the entire list.
140 */ 143 */
141 if (a->flags & GPU_ALLOC_GVA_SPACE && 144 if (a->flags & GPU_ALLOC_GVA_SPACE &&
142 b->pte_size == gmmu_page_size_big) 145 b->pte_size == gmmu_page_size_big) {
143 nvgpu_list_add_tail(&b->buddy_entry, list); 146 nvgpu_list_add_tail(&b->buddy_entry, list);
144 else 147 } else {
145 nvgpu_list_add(&b->buddy_entry, list); 148 nvgpu_list_add(&b->buddy_entry, list);
149 }
146 150
147 buddy_set_in_list(b); 151 buddy_set_in_list(b);
148} 152}
@@ -181,8 +185,9 @@ static void balloc_blist_rem(struct nvgpu_buddy_allocator *a,
181 185
182static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len) 186static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len)
183{ 187{
184 if (len == 0) 188 if (len == 0) {
185 return 0; 189 return 0;
190 }
186 191
187 len--; 192 len--;
188 len >>= a->blk_shift; 193 len >>= a->blk_shift;
@@ -195,10 +200,11 @@ static u64 __balloc_max_order_in(struct nvgpu_buddy_allocator *a,
195{ 200{
196 u64 size = (end - start) >> a->blk_shift; 201 u64 size = (end - start) >> a->blk_shift;
197 202
198 if (size > 0) 203 if (size > 0) {
199 return min_t(u64, ilog2(size), a->max_order); 204 return min_t(u64, ilog2(size), a->max_order);
200 else 205 } else {
201 return GPU_BALLOC_MAX_ORDER; 206 return GPU_BALLOC_MAX_ORDER;
207 }
202} 208}
203 209
204/* 210/*
@@ -222,8 +228,9 @@ static int balloc_init_lists(struct nvgpu_buddy_allocator *a)
222 order = __balloc_max_order_in(a, bstart, bend); 228 order = __balloc_max_order_in(a, bstart, bend);
223 229
224 buddy = balloc_new_buddy(a, NULL, bstart, order); 230 buddy = balloc_new_buddy(a, NULL, bstart, order);
225 if (!buddy) 231 if (!buddy) {
226 goto cleanup; 232 goto cleanup;
233 }
227 234
228 balloc_blist_add(a, buddy); 235 balloc_blist_add(a, buddy);
229 bstart += balloc_order_to_len(a, order); 236 bstart += balloc_order_to_len(a, order);
@@ -340,17 +347,20 @@ static void balloc_coalesce(struct nvgpu_buddy_allocator *a,
340{ 347{
341 struct nvgpu_buddy *parent; 348 struct nvgpu_buddy *parent;
342 349
343 if (buddy_is_alloced(b) || buddy_is_split(b)) 350 if (buddy_is_alloced(b) || buddy_is_split(b)) {
344 return; 351 return;
352 }
345 353
346 /* 354 /*
347 * If both our buddy and I are both not allocated and not split then 355 * If both our buddy and I are both not allocated and not split then
348 * we can coalesce ourselves. 356 * we can coalesce ourselves.
349 */ 357 */
350 if (!b->buddy) 358 if (!b->buddy) {
351 return; 359 return;
352 if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy)) 360 }
361 if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy)) {
353 return; 362 return;
363 }
354 364
355 parent = b->parent; 365 parent = b->parent;
356 366
@@ -383,8 +393,9 @@ static int balloc_split_buddy(struct nvgpu_buddy_allocator *a,
383 u64 half; 393 u64 half;
384 394
385 left = balloc_new_buddy(a, b, b->start, b->order - 1); 395 left = balloc_new_buddy(a, b, b->start, b->order - 1);
386 if (!left) 396 if (!left) {
387 return -ENOMEM; 397 return -ENOMEM;
398 }
388 399
389 half = (b->end - b->start) / 2; 400 half = (b->end - b->start) / 2;
390 401
@@ -449,8 +460,9 @@ static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a,
449 struct nvgpu_buddy *bud; 460 struct nvgpu_buddy *bud;
450 461
451 nvgpu_rbtree_search(addr, &node, a->alloced_buddies); 462 nvgpu_rbtree_search(addr, &node, a->alloced_buddies);
452 if (!node) 463 if (!node) {
453 return NULL; 464 return NULL;
465 }
454 466
455 bud = nvgpu_buddy_from_rbtree_node(node); 467 bud = nvgpu_buddy_from_rbtree_node(node);
456 468
@@ -470,21 +482,24 @@ static struct nvgpu_buddy *__balloc_find_buddy(struct nvgpu_buddy_allocator *a,
470 struct nvgpu_buddy *bud; 482 struct nvgpu_buddy *bud;
471 483
472 if (order > a->max_order || 484 if (order > a->max_order ||
473 nvgpu_list_empty(balloc_get_order_list(a, order))) 485 nvgpu_list_empty(balloc_get_order_list(a, order))) {
474 return NULL; 486 return NULL;
487 }
475 488
476 if (a->flags & GPU_ALLOC_GVA_SPACE && 489 if (a->flags & GPU_ALLOC_GVA_SPACE &&
477 pte_size == gmmu_page_size_big) 490 pte_size == gmmu_page_size_big) {
478 bud = nvgpu_list_last_entry(balloc_get_order_list(a, order), 491 bud = nvgpu_list_last_entry(balloc_get_order_list(a, order),
479 nvgpu_buddy, buddy_entry); 492 nvgpu_buddy, buddy_entry);
480 else 493 } else {
481 bud = nvgpu_list_first_entry(balloc_get_order_list(a, order), 494 bud = nvgpu_list_first_entry(balloc_get_order_list(a, order),
482 nvgpu_buddy, buddy_entry); 495 nvgpu_buddy, buddy_entry);
496 }
483 497
484 if (pte_size != BALLOC_PTE_SIZE_ANY && 498 if (pte_size != BALLOC_PTE_SIZE_ANY &&
485 pte_size != bud->pte_size && 499 pte_size != bud->pte_size &&
486 bud->pte_size != BALLOC_PTE_SIZE_ANY) 500 bud->pte_size != BALLOC_PTE_SIZE_ANY) {
487 return NULL; 501 return NULL;
502 }
488 503
489 return bud; 504 return bud;
490} 505}
@@ -511,12 +526,14 @@ static u64 __balloc_do_alloc(struct nvgpu_buddy_allocator *a,
511 } 526 }
512 527
513 /* Out of memory! */ 528 /* Out of memory! */
514 if (!bud) 529 if (!bud) {
515 return 0; 530 return 0;
531 }
516 532
517 while (bud->order != order) { 533 while (bud->order != order) {
518 if (balloc_split_buddy(a, bud, pte_size)) 534 if (balloc_split_buddy(a, bud, pte_size)) {
519 return 0; /* No mem... */ 535 return 0; /* No mem... */
536 }
520 bud = bud->left; 537 bud = bud->left;
521 } 538 }
522 539
@@ -540,19 +557,22 @@ static int balloc_is_range_free(struct nvgpu_buddy_allocator *a,
540 struct nvgpu_buddy *bud; 557 struct nvgpu_buddy *bud;
541 558
542 nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies); 559 nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies);
543 if (!node) 560 if (!node) {
544 return 1; /* No allocs yet. */ 561 return 1; /* No allocs yet. */
562 }
545 563
546 bud = nvgpu_buddy_from_rbtree_node(node); 564 bud = nvgpu_buddy_from_rbtree_node(node);
547 565
548 while (bud->start < end) { 566 while (bud->start < end) {
549 if ((bud->start > base && bud->start < end) || 567 if ((bud->start > base && bud->start < end) ||
550 (bud->end > base && bud->end < end)) 568 (bud->end > base && bud->end < end)) {
551 return 0; 569 return 0;
570 }
552 571
553 nvgpu_rbtree_enum_next(&node, node); 572 nvgpu_rbtree_enum_next(&node, node);
554 if (!node) 573 if (!node) {
555 break; 574 break;
575 }
556 bud = nvgpu_buddy_from_rbtree_node(node); 576 bud = nvgpu_buddy_from_rbtree_node(node);
557 } 577 }
558 578
@@ -581,8 +601,9 @@ static struct nvgpu_fixed_alloc *balloc_free_fixed(
581 struct nvgpu_rbtree_node *node = NULL; 601 struct nvgpu_rbtree_node *node = NULL;
582 602
583 nvgpu_rbtree_search(addr, &node, a->fixed_allocs); 603 nvgpu_rbtree_search(addr, &node, a->fixed_allocs);
584 if (!node) 604 if (!node) {
585 return NULL; 605 return NULL;
606 }
586 607
587 falloc = nvgpu_fixed_alloc_from_rbtree_node(node); 608 falloc = nvgpu_fixed_alloc_from_rbtree_node(node);
588 609
@@ -657,8 +678,9 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy(
657 } 678 }
658 } 679 }
659 680
660 if (found) 681 if (found) {
661 break; 682 break;
683 }
662 684
663 __balloc_get_parent_range(a, cur_base, cur_order, 685 __balloc_get_parent_range(a, cur_base, cur_order,
664 &cur_base, &cur_order); 686 &cur_base, &cur_order);
@@ -679,10 +701,11 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy(
679 return NULL; 701 return NULL;
680 } 702 }
681 703
682 if (base < bud->right->start) 704 if (base < bud->right->start) {
683 bud = bud->left; 705 bud = bud->left;
684 else 706 } else {
685 bud = bud->right; 707 bud = bud->right;
708 }
686 709
687 } 710 }
688 711
@@ -697,12 +720,13 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a,
697 u64 align_order; 720 u64 align_order;
698 721
699 shifted_base = balloc_base_shift(a, base); 722 shifted_base = balloc_base_shift(a, base);
700 if (shifted_base == 0) 723 if (shifted_base == 0) {
701 align_order = __fls(len >> a->blk_shift); 724 align_order = __fls(len >> a->blk_shift);
702 else 725 } else {
703 align_order = min_t(u64, 726 align_order = min_t(u64,
704 __ffs(shifted_base >> a->blk_shift), 727 __ffs(shifted_base >> a->blk_shift),
705 __fls(len >> a->blk_shift)); 728 __fls(len >> a->blk_shift));
729 }
706 730
707 if (align_order > a->max_order) { 731 if (align_order > a->max_order) {
708 alloc_dbg(balloc_owner(a), 732 alloc_dbg(balloc_owner(a),
@@ -741,9 +765,10 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a,
741 align_order = __ffs(inc_base >> a->blk_shift); 765 align_order = __ffs(inc_base >> a->blk_shift);
742 766
743 /* If we don't have much left - trim down align_order. */ 767 /* If we don't have much left - trim down align_order. */
744 if (balloc_order_to_len(a, align_order) > remaining) 768 if (balloc_order_to_len(a, align_order) > remaining) {
745 align_order = __balloc_max_order_in(a, inc_base, 769 align_order = __balloc_max_order_in(a, inc_base,
746 inc_base + remaining); 770 inc_base + remaining);
771 }
747 } 772 }
748 773
749 return base; 774 return base;
@@ -805,10 +830,11 @@ static u64 nvgpu_buddy_balloc(struct nvgpu_allocator *__a, u64 len)
805 return 0; 830 return 0;
806 } 831 }
807 832
808 if (a->flags & GPU_ALLOC_GVA_SPACE) 833 if (a->flags & GPU_ALLOC_GVA_SPACE) {
809 pte_size = __get_pte_size(a->vm, 0, len); 834 pte_size = __get_pte_size(a->vm, 0, len);
810 else 835 } else {
811 pte_size = BALLOC_PTE_SIZE_ANY; 836 pte_size = BALLOC_PTE_SIZE_ANY;
837 }
812 838
813 addr = __balloc_do_alloc(a, order, pte_size); 839 addr = __balloc_do_alloc(a, order, pte_size);
814 840
@@ -845,25 +871,29 @@ static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a,
845 struct nvgpu_buddy_allocator *a = __a->priv; 871 struct nvgpu_buddy_allocator *a = __a->priv;
846 872
847 /* If base isn't aligned to an order 0 block, fail. */ 873 /* If base isn't aligned to an order 0 block, fail. */
848 if (base & (a->blk_size - 1)) 874 if (base & (a->blk_size - 1)) {
849 goto fail; 875 goto fail;
876 }
850 877
851 if (len == 0) 878 if (len == 0) {
852 goto fail; 879 goto fail;
880 }
853 881
854 /* Check that the page size is valid. */ 882 /* Check that the page size is valid. */
855 if (a->flags & GPU_ALLOC_GVA_SPACE && a->vm->big_pages) { 883 if (a->flags & GPU_ALLOC_GVA_SPACE && a->vm->big_pages) {
856 if (page_size == a->vm->big_page_size) 884 if (page_size == a->vm->big_page_size) {
857 pte_size = gmmu_page_size_big; 885 pte_size = gmmu_page_size_big;
858 else if (page_size == SZ_4K) 886 } else if (page_size == SZ_4K) {
859 pte_size = gmmu_page_size_small; 887 pte_size = gmmu_page_size_small;
860 else 888 } else {
861 goto fail; 889 goto fail;
890 }
862 } 891 }
863 892
864 falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc)); 893 falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc));
865 if (!falloc) 894 if (!falloc) {
866 goto fail; 895 goto fail;
896 }
867 897
868 nvgpu_init_list_node(&falloc->buddies); 898 nvgpu_init_list_node(&falloc->buddies);
869 falloc->start = base; 899 falloc->start = base;
@@ -936,8 +966,9 @@ static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr)
936 struct nvgpu_fixed_alloc *falloc; 966 struct nvgpu_fixed_alloc *falloc;
937 struct nvgpu_buddy_allocator *a = __a->priv; 967 struct nvgpu_buddy_allocator *a = __a->priv;
938 968
939 if (!addr) 969 if (!addr) {
940 return; 970 return;
971 }
941 972
942 alloc_lock(__a); 973 alloc_lock(__a);
943 974
@@ -952,8 +983,9 @@ static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr)
952 } 983 }
953 984
954 bud = balloc_free_buddy(a, addr); 985 bud = balloc_free_buddy(a, addr);
955 if (!bud) 986 if (!bud) {
956 goto done; 987 goto done;
988 }
957 989
958 balloc_blist_add(a, bud); 990 balloc_blist_add(a, bud);
959 a->bytes_freed += balloc_order_to_len(a, bud->order); 991 a->bytes_freed += balloc_order_to_len(a, bud->order);
@@ -987,8 +1019,9 @@ static bool nvgpu_buddy_reserve_is_possible(struct nvgpu_buddy_allocator *a,
987 if ((co_base >= tmp->base && 1019 if ((co_base >= tmp->base &&
988 co_base < (tmp->base + tmp->length)) || 1020 co_base < (tmp->base + tmp->length)) ||
989 (co_end >= tmp->base && 1021 (co_end >= tmp->base &&
990 co_end < (tmp->base + tmp->length))) 1022 co_end < (tmp->base + tmp->length))) {
991 return false; 1023 return false;
1024 }
992 } 1025 }
993 1026
994 return true; 1027 return true;
@@ -1006,8 +1039,9 @@ static int nvgpu_buddy_reserve_co(struct nvgpu_allocator *__a,
1006 int err = 0; 1039 int err = 0;
1007 1040
1008 if (co->base < a->start || (co->base + co->length) > a->end || 1041 if (co->base < a->start || (co->base + co->length) > a->end ||
1009 a->alloc_made) 1042 a->alloc_made) {
1010 return -EINVAL; 1043 return -EINVAL;
1044 }
1011 1045
1012 alloc_lock(__a); 1046 alloc_lock(__a);
1013 1047
@@ -1221,25 +1255,31 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
1221 struct nvgpu_buddy_allocator *a; 1255 struct nvgpu_buddy_allocator *a;
1222 1256
1223 /* blk_size must be greater than 0 and a power of 2. */ 1257 /* blk_size must be greater than 0 and a power of 2. */
1224 if (blk_size == 0) 1258 if (blk_size == 0) {
1225 return -EINVAL; 1259 return -EINVAL;
1226 if (blk_size & (blk_size - 1)) 1260 }
1261 if (blk_size & (blk_size - 1)) {
1227 return -EINVAL; 1262 return -EINVAL;
1263 }
1228 1264
1229 if (max_order > GPU_BALLOC_MAX_ORDER) 1265 if (max_order > GPU_BALLOC_MAX_ORDER) {
1230 return -EINVAL; 1266 return -EINVAL;
1267 }
1231 1268
1232 /* If this is to manage a GVA space we need a VM. */ 1269 /* If this is to manage a GVA space we need a VM. */
1233 if (flags & GPU_ALLOC_GVA_SPACE && !vm) 1270 if (flags & GPU_ALLOC_GVA_SPACE && !vm) {
1234 return -EINVAL; 1271 return -EINVAL;
1272 }
1235 1273
1236 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator)); 1274 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator));
1237 if (!a) 1275 if (!a) {
1238 return -ENOMEM; 1276 return -ENOMEM;
1277 }
1239 1278
1240 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops); 1279 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops);
1241 if (err) 1280 if (err) {
1242 goto fail; 1281 goto fail;
1282 }
1243 1283
1244 a->base = base; 1284 a->base = base;
1245 a->length = size; 1285 a->length = size;
@@ -1269,8 +1309,9 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
1269 */ 1309 */
1270 if (flags & GPU_ALLOC_GVA_SPACE && vm->big_pages && 1310 if (flags & GPU_ALLOC_GVA_SPACE && vm->big_pages &&
1271 (base & ((vm->big_page_size << 10) - 1) || 1311 (base & ((vm->big_page_size << 10) - 1) ||
1272 size & ((vm->big_page_size << 10) - 1))) 1312 size & ((vm->big_page_size << 10) - 1))) {
1273 return -EINVAL; 1313 return -EINVAL;
1314 }
1274 1315
1275 a->flags = flags; 1316 a->flags = flags;
1276 a->max_order = max_order; 1317 a->max_order = max_order;
@@ -1288,8 +1329,9 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
1288 a->fixed_allocs = NULL; 1329 a->fixed_allocs = NULL;
1289 nvgpu_init_list_node(&a->co_list); 1330 nvgpu_init_list_node(&a->co_list);
1290 err = balloc_init_lists(a); 1331 err = balloc_init_lists(a);
1291 if (err) 1332 if (err) {
1292 goto fail; 1333 goto fail;
1334 }
1293 1335
1294 nvgpu_smp_wmb(); 1336 nvgpu_smp_wmb();
1295 a->initialized = 1; 1337 a->initialized = 1;
@@ -1301,18 +1343,20 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
1301 alloc_dbg(__a, " base 0x%llx", a->base); 1343 alloc_dbg(__a, " base 0x%llx", a->base);
1302 alloc_dbg(__a, " size 0x%llx", a->length); 1344 alloc_dbg(__a, " size 0x%llx", a->length);
1303 alloc_dbg(__a, " blk_size 0x%llx", a->blk_size); 1345 alloc_dbg(__a, " blk_size 0x%llx", a->blk_size);
1304 if (flags & GPU_ALLOC_GVA_SPACE) 1346 if (flags & GPU_ALLOC_GVA_SPACE) {
1305 alloc_dbg(balloc_owner(a), 1347 alloc_dbg(balloc_owner(a),
1306 " pde_size 0x%llx", 1348 " pde_size 0x%llx",
1307 balloc_order_to_len(a, a->pte_blk_order)); 1349 balloc_order_to_len(a, a->pte_blk_order));
1350 }
1308 alloc_dbg(__a, " max_order %llu", a->max_order); 1351 alloc_dbg(__a, " max_order %llu", a->max_order);
1309 alloc_dbg(__a, " flags 0x%llx", a->flags); 1352 alloc_dbg(__a, " flags 0x%llx", a->flags);
1310 1353
1311 return 0; 1354 return 0;
1312 1355
1313fail: 1356fail:
1314 if (a->buddy_cache) 1357 if (a->buddy_cache) {
1315 nvgpu_kmem_cache_destroy(a->buddy_cache); 1358 nvgpu_kmem_cache_destroy(a->buddy_cache);
1359 }
1316 nvgpu_kfree(g, a); 1360 nvgpu_kfree(g, a);
1317 return err; 1361 return err;
1318} 1362}
diff --git a/drivers/gpu/nvgpu/common/mm/gmmu.c b/drivers/gpu/nvgpu/common/mm/gmmu.c
index f6f2b9ad..498cdf06 100644
--- a/drivers/gpu/nvgpu/common/mm/gmmu.c
+++ b/drivers/gpu/nvgpu/common/mm/gmmu.c
@@ -38,18 +38,20 @@
38 38
39#define __gmmu_dbg(g, attrs, fmt, args...) \ 39#define __gmmu_dbg(g, attrs, fmt, args...) \
40 do { \ 40 do { \
41 if (attrs->debug) \ 41 if (attrs->debug) { \
42 nvgpu_info(g, fmt, ##args); \ 42 nvgpu_info(g, fmt, ##args); \
43 else \ 43 } else { \
44 nvgpu_log(g, gpu_dbg_map, fmt, ##args); \ 44 nvgpu_log(g, gpu_dbg_map, fmt, ##args); \
45 } \
45 } while (0) 46 } while (0)
46 47
47#define __gmmu_dbg_v(g, attrs, fmt, args...) \ 48#define __gmmu_dbg_v(g, attrs, fmt, args...) \
48 do { \ 49 do { \
49 if (attrs->debug) \ 50 if (attrs->debug) { \
50 nvgpu_info(g, fmt, ##args); \ 51 nvgpu_info(g, fmt, ##args); \
51 else \ 52 } else { \
52 nvgpu_log(g, gpu_dbg_map_v, fmt, ##args); \ 53 nvgpu_log(g, gpu_dbg_map_v, fmt, ##args); \
54 } \
53 } while (0) 55 } while (0)
54 56
55static int pd_allocate(struct vm_gk20a *vm, 57static int pd_allocate(struct vm_gk20a *vm,
@@ -77,15 +79,17 @@ static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm,
77 79
78 struct nvgpu_sgt *sgt = nvgpu_sgt_create_from_mem(g, mem); 80 struct nvgpu_sgt *sgt = nvgpu_sgt_create_from_mem(g, mem);
79 81
80 if (!sgt) 82 if (!sgt) {
81 return -ENOMEM; 83 return -ENOMEM;
84 }
82 85
83 /* 86 /*
84 * If the GPU is IO coherent and the DMA API is giving us IO coherent 87 * If the GPU is IO coherent and the DMA API is giving us IO coherent
85 * CPU mappings then we gotta make sure we use the IO coherent aperture. 88 * CPU mappings then we gotta make sure we use the IO coherent aperture.
86 */ 89 */
87 if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) 90 if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) {
88 flags |= NVGPU_VM_MAP_IO_COHERENT; 91 flags |= NVGPU_VM_MAP_IO_COHERENT;
92 }
89 93
90 /* 94 /*
91 * Later on, when we free this nvgpu_mem's GPU mapping, we are going to 95 * Later on, when we free this nvgpu_mem's GPU mapping, we are going to
@@ -94,10 +98,11 @@ static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm,
94 * therefor we should not try and free it. But otherwise, if we do 98 * therefor we should not try and free it. But otherwise, if we do
95 * manage the VA alloc, we obviously must free it. 99 * manage the VA alloc, we obviously must free it.
96 */ 100 */
97 if (addr != 0) 101 if (addr != 0) {
98 mem->free_gpu_va = false; 102 mem->free_gpu_va = false;
99 else 103 } else {
100 mem->free_gpu_va = true; 104 mem->free_gpu_va = true;
105 }
101 106
102 nvgpu_mutex_acquire(&vm->update_gmmu_lock); 107 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
103 vaddr = g->ops.mm.gmmu_map(vm, addr, 108 vaddr = g->ops.mm.gmmu_map(vm, addr,
@@ -196,8 +201,9 @@ int nvgpu_gmmu_init_page_table(struct vm_gk20a *vm)
196 pdb_size = ALIGN(pd_size(&vm->mmu_levels[0], &attrs), PAGE_SIZE); 201 pdb_size = ALIGN(pd_size(&vm->mmu_levels[0], &attrs), PAGE_SIZE);
197 202
198 err = __nvgpu_pd_cache_alloc_direct(vm->mm->g, &vm->pdb, pdb_size); 203 err = __nvgpu_pd_cache_alloc_direct(vm->mm->g, &vm->pdb, pdb_size);
199 if (WARN_ON(err)) 204 if (WARN_ON(err)) {
200 return err; 205 return err;
206 }
201 207
202 /* 208 /*
203 * One nvgpu_mb() is done after all mapping operations. Don't need 209 * One nvgpu_mb() is done after all mapping operations. Don't need
@@ -267,8 +273,9 @@ static int pd_allocate(struct vm_gk20a *vm,
267{ 273{
268 int err; 274 int err;
269 275
270 if (pd->mem) 276 if (pd->mem) {
271 return 0; 277 return 0;
278 }
272 279
273 err = __nvgpu_pd_alloc(vm, pd, pd_size(l, attrs)); 280 err = __nvgpu_pd_alloc(vm, pd, pd_size(l, attrs));
274 if (err) { 281 if (err) {
@@ -310,14 +317,16 @@ static int pd_allocate_children(struct vm_gk20a *vm,
310{ 317{
311 struct gk20a *g = gk20a_from_vm(vm); 318 struct gk20a *g = gk20a_from_vm(vm);
312 319
313 if (pd->entries) 320 if (pd->entries) {
314 return 0; 321 return 0;
322 }
315 323
316 pd->num_entries = pd_entries(l, attrs); 324 pd->num_entries = pd_entries(l, attrs);
317 pd->entries = nvgpu_vzalloc(g, sizeof(struct nvgpu_gmmu_pd) * 325 pd->entries = nvgpu_vzalloc(g, sizeof(struct nvgpu_gmmu_pd) *
318 pd->num_entries); 326 pd->num_entries);
319 if (!pd->entries) 327 if (!pd->entries) {
320 return -ENOMEM; 328 return -ENOMEM;
329 }
321 330
322 return 0; 331 return 0;
323} 332}
@@ -398,8 +407,9 @@ static int __set_pd_level(struct vm_gk20a *vm,
398 * have a bunch of children PDs. 407 * have a bunch of children PDs.
399 */ 408 */
400 if (next_l->update_entry) { 409 if (next_l->update_entry) {
401 if (pd_allocate_children(vm, l, pd, attrs)) 410 if (pd_allocate_children(vm, l, pd, attrs)) {
402 return -ENOMEM; 411 return -ENOMEM;
412 }
403 413
404 /* 414 /*
405 * Get the next PD so that we know what to put in this 415 * Get the next PD so that we know what to put in this
@@ -412,8 +422,9 @@ static int __set_pd_level(struct vm_gk20a *vm,
412 /* 422 /*
413 * Allocate the backing memory for next_pd. 423 * Allocate the backing memory for next_pd.
414 */ 424 */
415 if (pd_allocate(vm, next_pd, next_l, attrs)) 425 if (pd_allocate(vm, next_pd, next_l, attrs)) {
416 return -ENOMEM; 426 return -ENOMEM;
427 }
417 } 428 }
418 429
419 /* 430 /*
@@ -440,8 +451,9 @@ static int __set_pd_level(struct vm_gk20a *vm,
440 chunk_size, 451 chunk_size,
441 attrs); 452 attrs);
442 453
443 if (err) 454 if (err) {
444 return err; 455 return err;
456 }
445 } 457 }
446 458
447 virt_addr += chunk_size; 459 virt_addr += chunk_size;
@@ -452,8 +464,9 @@ static int __set_pd_level(struct vm_gk20a *vm,
452 * non-zero phys addresses in the PTEs. A non-zero phys-addr 464 * non-zero phys addresses in the PTEs. A non-zero phys-addr
453 * would also confuse the lower level PTE programming code. 465 * would also confuse the lower level PTE programming code.
454 */ 466 */
455 if (phys_addr) 467 if (phys_addr) {
456 phys_addr += chunk_size; 468 phys_addr += chunk_size;
469 }
457 length -= chunk_size; 470 length -= chunk_size;
458 } 471 }
459 472
@@ -547,8 +560,9 @@ static int __nvgpu_gmmu_do_update_page_table(struct vm_gk20a *vm,
547 virt_addr, 560 virt_addr,
548 chunk_length, 561 chunk_length,
549 attrs); 562 attrs);
550 if (err) 563 if (err) {
551 break; 564 break;
565 }
552 566
553 /* Space has been skipped so zero this for future chunks. */ 567 /* Space has been skipped so zero this for future chunks. */
554 space_to_skip = 0; 568 space_to_skip = 0;
@@ -559,8 +573,9 @@ static int __nvgpu_gmmu_do_update_page_table(struct vm_gk20a *vm,
559 virt_addr += chunk_length; 573 virt_addr += chunk_length;
560 length -= chunk_length; 574 length -= chunk_length;
561 575
562 if (length == 0) 576 if (length == 0) {
563 break; 577 break;
578 }
564 } 579 }
565 580
566 return err; 581 return err;
@@ -594,13 +609,15 @@ static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm,
594 609
595 /* note: here we need to map kernel to small, since the 610 /* note: here we need to map kernel to small, since the
596 * low-level mmu code assumes 0 is small and 1 is big pages */ 611 * low-level mmu code assumes 0 is small and 1 is big pages */
597 if (attrs->pgsz == gmmu_page_size_kernel) 612 if (attrs->pgsz == gmmu_page_size_kernel) {
598 attrs->pgsz = gmmu_page_size_small; 613 attrs->pgsz = gmmu_page_size_small;
614 }
599 615
600 page_size = vm->gmmu_page_sizes[attrs->pgsz]; 616 page_size = vm->gmmu_page_sizes[attrs->pgsz];
601 617
602 if (space_to_skip & (page_size - 1)) 618 if (space_to_skip & (page_size - 1)) {
603 return -EINVAL; 619 return -EINVAL;
620 }
604 621
605 /* 622 /*
606 * Update length to be aligned to the passed page size. 623 * Update length to be aligned to the passed page size.
@@ -692,8 +709,9 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
692 * the programmed ctagline gets increased at compression_page_size 709 * the programmed ctagline gets increased at compression_page_size
693 * boundaries. 710 * boundaries.
694 */ 711 */
695 if (attrs.ctag) 712 if (attrs.ctag) {
696 attrs.ctag += buffer_offset & (ctag_granularity - 1U); 713 attrs.ctag += buffer_offset & (ctag_granularity - 1U);
714 }
697 715
698 attrs.l3_alloc = (bool)(flags & NVGPU_VM_MAP_L3_ALLOC); 716 attrs.l3_alloc = (bool)(flags & NVGPU_VM_MAP_L3_ALLOC);
699 717
@@ -701,8 +719,9 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
701 * Handle the IO coherency aperture: make sure the .aperture field is 719 * Handle the IO coherency aperture: make sure the .aperture field is
702 * correct based on the IO coherency flag. 720 * correct based on the IO coherency flag.
703 */ 721 */
704 if (attrs.coherent && attrs.aperture == APERTURE_SYSMEM) 722 if (attrs.coherent && attrs.aperture == APERTURE_SYSMEM) {
705 attrs.aperture = __APERTURE_SYSMEM_COH; 723 attrs.aperture = __APERTURE_SYSMEM_COH;
724 }
706 725
707 /* 726 /*
708 * Only allocate a new GPU VA range if we haven't already been passed a 727 * Only allocate a new GPU VA range if we haven't already been passed a
@@ -725,16 +744,18 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
725 goto fail_validate; 744 goto fail_validate;
726 } 745 }
727 746
728 if (!batch) 747 if (!batch) {
729 g->ops.fb.tlb_invalidate(g, vm->pdb.mem); 748 g->ops.fb.tlb_invalidate(g, vm->pdb.mem);
730 else 749 } else {
731 batch->need_tlb_invalidate = true; 750 batch->need_tlb_invalidate = true;
751 }
732 752
733 return vaddr; 753 return vaddr;
734 754
735fail_validate: 755fail_validate:
736 if (allocated) 756 if (allocated) {
737 __nvgpu_vm_free_va(vm, vaddr, pgsz_idx); 757 __nvgpu_vm_free_va(vm, vaddr, pgsz_idx);
758 }
738fail_alloc: 759fail_alloc:
739 nvgpu_err(g, "%s: failed with err=%d", __func__, err); 760 nvgpu_err(g, "%s: failed with err=%d", __func__, err);
740 return 0; 761 return 0;
@@ -775,8 +796,9 @@ void gk20a_locked_gmmu_unmap(struct vm_gk20a *vm,
775 /* unmap here needs to know the page size we assigned at mapping */ 796 /* unmap here needs to know the page size we assigned at mapping */
776 err = __nvgpu_gmmu_update_page_table(vm, NULL, 0, 797 err = __nvgpu_gmmu_update_page_table(vm, NULL, 0,
777 vaddr, size, &attrs); 798 vaddr, size, &attrs);
778 if (err) 799 if (err) {
779 nvgpu_err(g, "failed to update gmmu ptes on unmap"); 800 nvgpu_err(g, "failed to update gmmu ptes on unmap");
801 }
780 802
781 if (!batch) { 803 if (!batch) {
782 gk20a_mm_l2_flush(g, true); 804 gk20a_mm_l2_flush(g, true);
@@ -801,8 +823,9 @@ u32 __nvgpu_pte_words(struct gk20a *g)
801 */ 823 */
802 do { 824 do {
803 next_l = l + 1; 825 next_l = l + 1;
804 if (!next_l->update_entry) 826 if (!next_l->update_entry) {
805 break; 827 break;
828 }
806 829
807 l++; 830 l++;
808 } while (true); 831 } while (true);
@@ -836,13 +859,15 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm,
836 struct nvgpu_gmmu_pd *pd_next = pd->entries + pd_idx; 859 struct nvgpu_gmmu_pd *pd_next = pd->entries + pd_idx;
837 860
838 /* Invalid entry! */ 861 /* Invalid entry! */
839 if (!pd_next->mem) 862 if (!pd_next->mem) {
840 return -EINVAL; 863 return -EINVAL;
864 }
841 865
842 attrs->pgsz = l->get_pgsz(g, l, pd, pd_idx); 866 attrs->pgsz = l->get_pgsz(g, l, pd, pd_idx);
843 867
844 if (attrs->pgsz >= gmmu_nr_page_sizes) 868 if (attrs->pgsz >= gmmu_nr_page_sizes) {
845 return -EINVAL; 869 return -EINVAL;
870 }
846 871
847 return __nvgpu_locate_pte(g, vm, pd_next, 872 return __nvgpu_locate_pte(g, vm, pd_next,
848 vaddr, lvl + 1, attrs, 873 vaddr, lvl + 1, attrs,
@@ -850,8 +875,9 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm,
850 pd_offs_out); 875 pd_offs_out);
851 } 876 }
852 877
853 if (!pd->mem) 878 if (!pd->mem) {
854 return -EINVAL; 879 return -EINVAL;
880 }
855 881
856 /* 882 /*
857 * Take into account the real offset into the nvgpu_mem since the PD 883 * Take into account the real offset into the nvgpu_mem since the PD
@@ -867,14 +893,17 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm,
867 } 893 }
868 } 894 }
869 895
870 if (pd_out) 896 if (pd_out) {
871 *pd_out = pd; 897 *pd_out = pd;
898 }
872 899
873 if (pd_idx_out) 900 if (pd_idx_out) {
874 *pd_idx_out = pd_idx; 901 *pd_idx_out = pd_idx;
902 }
875 903
876 if (pd_offs_out) 904 if (pd_offs_out) {
877 *pd_offs_out = pd_offset_from_index(l, pd_idx); 905 *pd_offs_out = pd_offset_from_index(l, pd_idx);
906 }
878 907
879 return 0; 908 return 0;
880} 909}
@@ -903,8 +932,9 @@ int __nvgpu_set_pte(struct gk20a *g, struct vm_gk20a *vm, u64 vaddr, u32 *pte)
903 err = __nvgpu_locate_pte(g, vm, &vm->pdb, 932 err = __nvgpu_locate_pte(g, vm, &vm->pdb,
904 vaddr, 0, &attrs, 933 vaddr, 0, &attrs,
905 NULL, &pd, &pd_idx, &pd_offs); 934 NULL, &pd, &pd_idx, &pd_offs);
906 if (err) 935 if (err) {
907 return err; 936 return err;
937 }
908 938
909 pte_size = __nvgpu_pte_words(g); 939 pte_size = __nvgpu_pte_words(g);
910 940
diff --git a/drivers/gpu/nvgpu/common/mm/lockless_allocator.c b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c
index 4a3d6c9d..7a5f67e9 100644
--- a/drivers/gpu/nvgpu/common/mm/lockless_allocator.c
+++ b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c
@@ -63,8 +63,9 @@ static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len)
63 int head, new_head, ret; 63 int head, new_head, ret;
64 u64 addr = 0; 64 u64 addr = 0;
65 65
66 if (len != pa->blk_size) 66 if (len != pa->blk_size) {
67 return 0; 67 return 0;
68 }
68 69
69 head = NV_ACCESS_ONCE(pa->head); 70 head = NV_ACCESS_ONCE(pa->head);
70 while (head >= 0) { 71 while (head >= 0) {
@@ -80,10 +81,11 @@ static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len)
80 head = NV_ACCESS_ONCE(pa->head); 81 head = NV_ACCESS_ONCE(pa->head);
81 } 82 }
82 83
83 if (addr) 84 if (addr) {
84 alloc_dbg(a, "Alloc node # %d @ addr 0x%llx", head, addr); 85 alloc_dbg(a, "Alloc node # %d @ addr 0x%llx", head, addr);
85 else 86 } else {
86 alloc_dbg(a, "Alloc failed!"); 87 alloc_dbg(a, "Alloc failed!");
88 }
87 89
88 return addr; 90 return addr;
89} 91}
@@ -167,24 +169,28 @@ int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
167 u64 count; 169 u64 count;
168 struct nvgpu_lockless_allocator *a; 170 struct nvgpu_lockless_allocator *a;
169 171
170 if (!blk_size) 172 if (!blk_size) {
171 return -EINVAL; 173 return -EINVAL;
174 }
172 175
173 /* 176 /*
174 * Ensure we have space for at least one node & there's no overflow. 177 * Ensure we have space for at least one node & there's no overflow.
175 * In order to control memory footprint, we require count < INT_MAX 178 * In order to control memory footprint, we require count < INT_MAX
176 */ 179 */
177 count = length / blk_size; 180 count = length / blk_size;
178 if (!base || !count || count > INT_MAX) 181 if (!base || !count || count > INT_MAX) {
179 return -EINVAL; 182 return -EINVAL;
183 }
180 184
181 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_lockless_allocator)); 185 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_lockless_allocator));
182 if (!a) 186 if (!a) {
183 return -ENOMEM; 187 return -ENOMEM;
188 }
184 189
185 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &pool_ops); 190 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &pool_ops);
186 if (err) 191 if (err) {
187 goto fail; 192 goto fail;
193 }
188 194
189 a->next = nvgpu_vzalloc(g, sizeof(*a->next) * count); 195 a->next = nvgpu_vzalloc(g, sizeof(*a->next) * count);
190 if (!a->next) { 196 if (!a->next) {
diff --git a/drivers/gpu/nvgpu/common/mm/mm.c b/drivers/gpu/nvgpu/common/mm/mm.c
index 2c3a1cd6..0608d66a 100644
--- a/drivers/gpu/nvgpu/common/mm/mm.c
+++ b/drivers/gpu/nvgpu/common/mm/mm.c
@@ -40,8 +40,9 @@ enum gmmu_pgsz_gk20a __get_pte_size_fixed_map(struct vm_gk20a *vm,
40 struct nvgpu_vm_area *vm_area; 40 struct nvgpu_vm_area *vm_area;
41 41
42 vm_area = nvgpu_vm_area_find(vm, base); 42 vm_area = nvgpu_vm_area_find(vm, base);
43 if (!vm_area) 43 if (!vm_area) {
44 return gmmu_page_size_small; 44 return gmmu_page_size_small;
45 }
45 46
46 return vm_area->pgsz_idx; 47 return vm_area->pgsz_idx;
47} 48}
@@ -53,14 +54,16 @@ static enum gmmu_pgsz_gk20a __get_pte_size_split_addr(struct vm_gk20a *vm,
53 u64 base, u64 size) 54 u64 base, u64 size)
54{ 55{
55 if (!base) { 56 if (!base) {
56 if (size >= vm->gmmu_page_sizes[gmmu_page_size_big]) 57 if (size >= vm->gmmu_page_sizes[gmmu_page_size_big]) {
57 return gmmu_page_size_big; 58 return gmmu_page_size_big;
59 }
58 return gmmu_page_size_small; 60 return gmmu_page_size_small;
59 } else { 61 } else {
60 if (base < __nv_gmmu_va_small_page_limit()) 62 if (base < __nv_gmmu_va_small_page_limit()) {
61 return gmmu_page_size_small; 63 return gmmu_page_size_small;
62 else 64 } else {
63 return gmmu_page_size_big; 65 return gmmu_page_size_big;
66 }
64 } 67 }
65} 68}
66 69
@@ -89,18 +92,22 @@ enum gmmu_pgsz_gk20a __get_pte_size(struct vm_gk20a *vm, u64 base, u64 size)
89{ 92{
90 struct gk20a *g = gk20a_from_vm(vm); 93 struct gk20a *g = gk20a_from_vm(vm);
91 94
92 if (!vm->big_pages) 95 if (!vm->big_pages) {
93 return gmmu_page_size_small; 96 return gmmu_page_size_small;
97 }
94 98
95 if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) 99 if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
96 return __get_pte_size_split_addr(vm, base, size); 100 return __get_pte_size_split_addr(vm, base, size);
101 }
97 102
98 if (base) 103 if (base) {
99 return __get_pte_size_fixed_map(vm, base, size); 104 return __get_pte_size_fixed_map(vm, base, size);
105 }
100 106
101 if (size >= vm->gmmu_page_sizes[gmmu_page_size_big] && 107 if (size >= vm->gmmu_page_sizes[gmmu_page_size_big] &&
102 nvgpu_iommuable(g)) 108 nvgpu_iommuable(g)) {
103 return gmmu_page_size_big; 109 return gmmu_page_size_big;
110 }
104 return gmmu_page_size_small; 111 return gmmu_page_size_small;
105} 112}
106 113
@@ -137,8 +144,9 @@ u64 nvgpu_inst_block_addr(struct gk20a *g, struct nvgpu_mem *inst_block)
137 144
138void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block) 145void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block)
139{ 146{
140 if (nvgpu_mem_is_valid(inst_block)) 147 if (nvgpu_mem_is_valid(inst_block)) {
141 nvgpu_dma_free(g, inst_block); 148 nvgpu_dma_free(g, inst_block);
149 }
142} 150}
143 151
144static int nvgpu_alloc_sysmem_flush(struct gk20a *g) 152static int nvgpu_alloc_sysmem_flush(struct gk20a *g)
@@ -150,8 +158,9 @@ static void nvgpu_remove_mm_ce_support(struct mm_gk20a *mm)
150{ 158{
151 struct gk20a *g = gk20a_from_mm(mm); 159 struct gk20a *g = gk20a_from_mm(mm);
152 160
153 if (mm->vidmem.ce_ctx_id != (u32)~0) 161 if (mm->vidmem.ce_ctx_id != (u32)~0) {
154 gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id); 162 gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id);
163 }
155 164
156 mm->vidmem.ce_ctx_id = (u32)~0; 165 mm->vidmem.ce_ctx_id = (u32)~0;
157 166
@@ -162,11 +171,13 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm)
162{ 171{
163 struct gk20a *g = gk20a_from_mm(mm); 172 struct gk20a *g = gk20a_from_mm(mm);
164 173
165 if (g->ops.mm.fault_info_mem_destroy) 174 if (g->ops.mm.fault_info_mem_destroy) {
166 g->ops.mm.fault_info_mem_destroy(g); 175 g->ops.mm.fault_info_mem_destroy(g);
176 }
167 177
168 if (g->ops.mm.remove_bar2_vm) 178 if (g->ops.mm.remove_bar2_vm) {
169 g->ops.mm.remove_bar2_vm(g); 179 g->ops.mm.remove_bar2_vm(g);
180 }
170 181
171 nvgpu_free_inst_block(g, &mm->bar1.inst_block); 182 nvgpu_free_inst_block(g, &mm->bar1.inst_block);
172 nvgpu_vm_put(mm->bar1.vm); 183 nvgpu_vm_put(mm->bar1.vm);
@@ -175,8 +186,9 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm)
175 nvgpu_free_inst_block(g, &mm->hwpm.inst_block); 186 nvgpu_free_inst_block(g, &mm->hwpm.inst_block);
176 nvgpu_vm_put(mm->pmu.vm); 187 nvgpu_vm_put(mm->pmu.vm);
177 188
178 if (g->has_cde) 189 if (g->has_cde) {
179 nvgpu_vm_put(mm->cde.vm); 190 nvgpu_vm_put(mm->cde.vm);
191 }
180 192
181 nvgpu_semaphore_sea_destroy(g); 193 nvgpu_semaphore_sea_destroy(g);
182 nvgpu_vidmem_destroy(g); 194 nvgpu_vidmem_destroy(g);
@@ -208,12 +220,14 @@ static int nvgpu_init_system_vm(struct mm_gk20a *mm)
208 true, 220 true,
209 false, 221 false,
210 "system"); 222 "system");
211 if (!mm->pmu.vm) 223 if (!mm->pmu.vm) {
212 return -ENOMEM; 224 return -ENOMEM;
225 }
213 226
214 err = g->ops.mm.alloc_inst_block(g, inst_block); 227 err = g->ops.mm.alloc_inst_block(g, inst_block);
215 if (err) 228 if (err) {
216 goto clean_up_vm; 229 goto clean_up_vm;
230 }
217 g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size); 231 g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size);
218 232
219 return 0; 233 return 0;
@@ -230,8 +244,9 @@ static int nvgpu_init_hwpm(struct mm_gk20a *mm)
230 struct nvgpu_mem *inst_block = &mm->hwpm.inst_block; 244 struct nvgpu_mem *inst_block = &mm->hwpm.inst_block;
231 245
232 err = g->ops.mm.alloc_inst_block(g, inst_block); 246 err = g->ops.mm.alloc_inst_block(g, inst_block);
233 if (err) 247 if (err) {
234 return err; 248 return err;
249 }
235 g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0); 250 g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0);
236 251
237 return 0; 252 return 0;
@@ -247,8 +262,9 @@ static int nvgpu_init_cde_vm(struct mm_gk20a *mm)
247 NV_MM_DEFAULT_KERNEL_SIZE, 262 NV_MM_DEFAULT_KERNEL_SIZE,
248 NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, 263 NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE,
249 false, false, "cde"); 264 false, false, "cde");
250 if (!mm->cde.vm) 265 if (!mm->cde.vm) {
251 return -ENOMEM; 266 return -ENOMEM;
267 }
252 return 0; 268 return 0;
253} 269}
254 270
@@ -262,8 +278,9 @@ static int nvgpu_init_ce_vm(struct mm_gk20a *mm)
262 NV_MM_DEFAULT_KERNEL_SIZE, 278 NV_MM_DEFAULT_KERNEL_SIZE,
263 NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, 279 NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE,
264 false, false, "ce"); 280 false, false, "ce");
265 if (!mm->ce.vm) 281 if (!mm->ce.vm) {
266 return -ENOMEM; 282 return -ENOMEM;
283 }
267 return 0; 284 return 0;
268} 285}
269 286
@@ -286,24 +303,30 @@ void nvgpu_init_mm_ce_context(struct gk20a *g)
286 303
287static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g) 304static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g)
288{ 305{
289 if (g->ops.fb.reset) 306 if (g->ops.fb.reset) {
290 g->ops.fb.reset(g); 307 g->ops.fb.reset(g);
308 }
291 309
292 if (g->ops.clock_gating.slcg_fb_load_gating_prod) 310 if (g->ops.clock_gating.slcg_fb_load_gating_prod) {
293 g->ops.clock_gating.slcg_fb_load_gating_prod(g, 311 g->ops.clock_gating.slcg_fb_load_gating_prod(g,
294 g->slcg_enabled); 312 g->slcg_enabled);
295 if (g->ops.clock_gating.slcg_ltc_load_gating_prod) 313 }
314 if (g->ops.clock_gating.slcg_ltc_load_gating_prod) {
296 g->ops.clock_gating.slcg_ltc_load_gating_prod(g, 315 g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
297 g->slcg_enabled); 316 g->slcg_enabled);
298 if (g->ops.clock_gating.blcg_fb_load_gating_prod) 317 }
318 if (g->ops.clock_gating.blcg_fb_load_gating_prod) {
299 g->ops.clock_gating.blcg_fb_load_gating_prod(g, 319 g->ops.clock_gating.blcg_fb_load_gating_prod(g,
300 g->blcg_enabled); 320 g->blcg_enabled);
301 if (g->ops.clock_gating.blcg_ltc_load_gating_prod) 321 }
322 if (g->ops.clock_gating.blcg_ltc_load_gating_prod) {
302 g->ops.clock_gating.blcg_ltc_load_gating_prod(g, 323 g->ops.clock_gating.blcg_ltc_load_gating_prod(g,
303 g->blcg_enabled); 324 g->blcg_enabled);
325 }
304 326
305 if (g->ops.fb.init_fs_state) 327 if (g->ops.fb.init_fs_state) {
306 g->ops.fb.init_fs_state(g); 328 g->ops.fb.init_fs_state(g);
329 }
307 330
308 return 0; 331 return 0;
309} 332}
@@ -324,12 +347,14 @@ static int nvgpu_init_bar1_vm(struct mm_gk20a *mm)
324 mm->bar1.aperture_size, 347 mm->bar1.aperture_size,
325 true, false, 348 true, false,
326 "bar1"); 349 "bar1");
327 if (!mm->bar1.vm) 350 if (!mm->bar1.vm) {
328 return -ENOMEM; 351 return -ENOMEM;
352 }
329 353
330 err = g->ops.mm.alloc_inst_block(g, inst_block); 354 err = g->ops.mm.alloc_inst_block(g, inst_block);
331 if (err) 355 if (err) {
332 goto clean_up_vm; 356 goto clean_up_vm;
357 }
333 g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size); 358 g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size);
334 359
335 return 0; 360 return 0;
@@ -366,8 +391,9 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g)
366 mm->vidmem.ce_ctx_id = (u32)~0; 391 mm->vidmem.ce_ctx_id = (u32)~0;
367 392
368 err = nvgpu_vidmem_init(mm); 393 err = nvgpu_vidmem_init(mm);
369 if (err) 394 if (err) {
370 return err; 395 return err;
396 }
371 397
372 /* 398 /*
373 * this requires fixed allocations in vidmem which must be 399 * this requires fixed allocations in vidmem which must be
@@ -376,40 +402,48 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g)
376 if (g->ops.pmu.alloc_blob_space 402 if (g->ops.pmu.alloc_blob_space
377 && !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) { 403 && !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) {
378 err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob); 404 err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob);
379 if (err) 405 if (err) {
380 return err; 406 return err;
407 }
381 } 408 }
382 409
383 err = nvgpu_alloc_sysmem_flush(g); 410 err = nvgpu_alloc_sysmem_flush(g);
384 if (err) 411 if (err) {
385 return err; 412 return err;
413 }
386 414
387 err = nvgpu_init_bar1_vm(mm); 415 err = nvgpu_init_bar1_vm(mm);
388 if (err) 416 if (err) {
389 return err; 417 return err;
418 }
390 419
391 if (g->ops.mm.init_bar2_vm) { 420 if (g->ops.mm.init_bar2_vm) {
392 err = g->ops.mm.init_bar2_vm(g); 421 err = g->ops.mm.init_bar2_vm(g);
393 if (err) 422 if (err) {
394 return err; 423 return err;
424 }
395 } 425 }
396 err = nvgpu_init_system_vm(mm); 426 err = nvgpu_init_system_vm(mm);
397 if (err) 427 if (err) {
398 return err; 428 return err;
429 }
399 430
400 err = nvgpu_init_hwpm(mm); 431 err = nvgpu_init_hwpm(mm);
401 if (err) 432 if (err) {
402 return err; 433 return err;
434 }
403 435
404 if (g->has_cde) { 436 if (g->has_cde) {
405 err = nvgpu_init_cde_vm(mm); 437 err = nvgpu_init_cde_vm(mm);
406 if (err) 438 if (err) {
407 return err; 439 return err;
440 }
408 } 441 }
409 442
410 err = nvgpu_init_ce_vm(mm); 443 err = nvgpu_init_ce_vm(mm);
411 if (err) 444 if (err) {
412 return err; 445 return err;
446 }
413 447
414 mm->remove_support = nvgpu_remove_mm_support; 448 mm->remove_support = nvgpu_remove_mm_support;
415 mm->remove_ce_support = nvgpu_remove_mm_ce_support; 449 mm->remove_ce_support = nvgpu_remove_mm_ce_support;
@@ -424,15 +458,18 @@ int nvgpu_init_mm_support(struct gk20a *g)
424 u32 err; 458 u32 err;
425 459
426 err = nvgpu_init_mm_reset_enable_hw(g); 460 err = nvgpu_init_mm_reset_enable_hw(g);
427 if (err) 461 if (err) {
428 return err; 462 return err;
463 }
429 464
430 err = nvgpu_init_mm_setup_sw(g); 465 err = nvgpu_init_mm_setup_sw(g);
431 if (err) 466 if (err) {
432 return err; 467 return err;
468 }
433 469
434 if (g->ops.mm.init_mm_setup_hw) 470 if (g->ops.mm.init_mm_setup_hw) {
435 err = g->ops.mm.init_mm_setup_hw(g); 471 err = g->ops.mm.init_mm_setup_hw(g);
472 }
436 473
437 return err; 474 return err;
438} 475}
@@ -443,8 +480,9 @@ u32 nvgpu_mm_get_default_big_page_size(struct gk20a *g)
443 480
444 big_page_size = g->ops.mm.get_default_big_page_size(); 481 big_page_size = g->ops.mm.get_default_big_page_size();
445 482
446 if (g->mm.disable_bigpage) 483 if (g->mm.disable_bigpage) {
447 big_page_size = 0; 484 big_page_size = 0;
485 }
448 486
449 return big_page_size; 487 return big_page_size;
450} 488}
@@ -456,8 +494,9 @@ u32 nvgpu_mm_get_available_big_page_sizes(struct gk20a *g)
456 if (!g->mm.disable_bigpage) { 494 if (!g->mm.disable_bigpage) {
457 available_big_page_sizes = 495 available_big_page_sizes =
458 g->ops.mm.get_default_big_page_size(); 496 g->ops.mm.get_default_big_page_size();
459 if (g->ops.mm.get_big_page_sizes) 497 if (g->ops.mm.get_big_page_sizes) {
460 available_big_page_sizes |= g->ops.mm.get_big_page_sizes(); 498 available_big_page_sizes |= g->ops.mm.get_big_page_sizes();
499 }
461 } 500 }
462 501
463 return available_big_page_sizes; 502 return available_big_page_sizes;
diff --git a/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
index 8ba119a6..345b947d 100644
--- a/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
+++ b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
@@ -40,8 +40,9 @@ u32 __nvgpu_aperture_mask(struct gk20a *g, enum nvgpu_aperture aperture,
40 * Some iGPUs treat sysmem (i.e SoC DRAM) as vidmem. In these cases the 40 * Some iGPUs treat sysmem (i.e SoC DRAM) as vidmem. In these cases the
41 * "sysmem" aperture should really be translated to VIDMEM. 41 * "sysmem" aperture should really be translated to VIDMEM.
42 */ 42 */
43 if (!nvgpu_is_enabled(g, NVGPU_MM_HONORS_APERTURE)) 43 if (!nvgpu_is_enabled(g, NVGPU_MM_HONORS_APERTURE)) {
44 aperture = APERTURE_VIDMEM; 44 aperture = APERTURE_VIDMEM;
45 }
45 46
46 switch (aperture) { 47 switch (aperture) {
47 case __APERTURE_SYSMEM_COH: 48 case __APERTURE_SYSMEM_COH:
@@ -67,8 +68,9 @@ u32 nvgpu_aperture_mask(struct gk20a *g, struct nvgpu_mem *mem,
67 * we add this translation step here. 68 * we add this translation step here.
68 */ 69 */
69 if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) && 70 if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) &&
70 ap == APERTURE_SYSMEM) 71 ap == APERTURE_SYSMEM) {
71 ap = __APERTURE_SYSMEM_COH; 72 ap = __APERTURE_SYSMEM_COH;
73 }
72 74
73 return __nvgpu_aperture_mask(g, ap, 75 return __nvgpu_aperture_mask(g, ap,
74 sysmem_mask, sysmem_coh_mask, vidmem_mask); 76 sysmem_mask, sysmem_coh_mask, vidmem_mask);
@@ -115,15 +117,17 @@ u64 nvgpu_sgt_get_gpu_addr(struct gk20a *g, struct nvgpu_sgt *sgt,
115 117
116bool nvgpu_sgt_iommuable(struct gk20a *g, struct nvgpu_sgt *sgt) 118bool nvgpu_sgt_iommuable(struct gk20a *g, struct nvgpu_sgt *sgt)
117{ 119{
118 if (sgt->ops->sgt_iommuable) 120 if (sgt->ops->sgt_iommuable) {
119 return sgt->ops->sgt_iommuable(g, sgt); 121 return sgt->ops->sgt_iommuable(g, sgt);
122 }
120 return false; 123 return false;
121} 124}
122 125
123void nvgpu_sgt_free(struct gk20a *g, struct nvgpu_sgt *sgt) 126void nvgpu_sgt_free(struct gk20a *g, struct nvgpu_sgt *sgt)
124{ 127{
125 if (sgt && sgt->ops->sgt_free) 128 if (sgt && sgt->ops->sgt_free) {
126 sgt->ops->sgt_free(g, sgt); 129 sgt->ops->sgt_free(g, sgt);
130 }
127} 131}
128 132
129u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys) 133u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys)
@@ -131,8 +135,9 @@ u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys)
131 /* ensure it is not vidmem allocation */ 135 /* ensure it is not vidmem allocation */
132 WARN_ON(nvgpu_addr_is_vidmem_page_alloc(phys)); 136 WARN_ON(nvgpu_addr_is_vidmem_page_alloc(phys));
133 137
134 if (nvgpu_iommuable(g) && g->ops.mm.get_iommu_bit) 138 if (nvgpu_iommuable(g) && g->ops.mm.get_iommu_bit) {
135 return phys | 1ULL << g->ops.mm.get_iommu_bit(g); 139 return phys | 1ULL << g->ops.mm.get_iommu_bit(g);
140 }
136 141
137 return phys; 142 return phys;
138} 143}
@@ -157,8 +162,9 @@ u64 nvgpu_sgt_alignment(struct gk20a *g, struct nvgpu_sgt *sgt)
157 */ 162 */
158 if (nvgpu_iommuable(g) && 163 if (nvgpu_iommuable(g) &&
159 nvgpu_sgt_iommuable(g, sgt) && 164 nvgpu_sgt_iommuable(g, sgt) &&
160 nvgpu_sgt_get_dma(sgt, sgt->sgl)) 165 nvgpu_sgt_get_dma(sgt, sgt->sgl)) {
161 return 1ULL << __ffs(nvgpu_sgt_get_dma(sgt, sgt->sgl)); 166 return 1ULL << __ffs(nvgpu_sgt_get_dma(sgt, sgt->sgl));
167 }
162 168
163 /* 169 /*
164 * Otherwise the buffer is not iommuable (VIDMEM, for example) or we are 170 * Otherwise the buffer is not iommuable (VIDMEM, for example) or we are
@@ -169,10 +175,11 @@ u64 nvgpu_sgt_alignment(struct gk20a *g, struct nvgpu_sgt *sgt)
169 chunk_align = 1ULL << __ffs(nvgpu_sgt_get_phys(g, sgt, sgl) | 175 chunk_align = 1ULL << __ffs(nvgpu_sgt_get_phys(g, sgt, sgl) |
170 nvgpu_sgt_get_length(sgt, sgl)); 176 nvgpu_sgt_get_length(sgt, sgl));
171 177
172 if (align) 178 if (align) {
173 align = min(align, chunk_align); 179 align = min(align, chunk_align);
174 else 180 } else {
175 align = chunk_align; 181 align = chunk_align;
182 }
176 } 183 }
177 184
178 return align; 185 return align;
diff --git a/drivers/gpu/nvgpu/common/mm/vm.c b/drivers/gpu/nvgpu/common/mm/vm.c
index 3b682e28..7d97b7b7 100644
--- a/drivers/gpu/nvgpu/common/mm/vm.c
+++ b/drivers/gpu/nvgpu/common/mm/vm.c
@@ -111,8 +111,9 @@ static void nvgpu_vm_free_entries(struct vm_gk20a *vm,
111 111
112 __nvgpu_pd_cache_free_direct(g, pdb); 112 __nvgpu_pd_cache_free_direct(g, pdb);
113 113
114 if (!pdb->entries) 114 if (!pdb->entries) {
115 return; 115 return;
116 }
116 117
117 for (i = 0; i < pdb->num_entries; i++) { 118 for (i = 0; i < pdb->num_entries; i++) {
118 __nvgpu_vm_free_entries(vm, &pdb->entries[i], 1); 119 __nvgpu_vm_free_entries(vm, &pdb->entries[i], 1);
@@ -204,8 +205,9 @@ int nvgpu_big_pages_possible(struct vm_gk20a *vm, u64 base, u64 size)
204{ 205{
205 u64 mask = ((u64)vm->big_page_size << 10) - 1; 206 u64 mask = ((u64)vm->big_page_size << 10) - 1;
206 207
207 if (base & mask || size & mask) 208 if (base & mask || size & mask) {
208 return 0; 209 return 0;
210 }
209 return 1; 211 return 1;
210} 212}
211 213
@@ -223,19 +225,23 @@ static int nvgpu_init_sema_pool(struct vm_gk20a *vm)
223 /* 225 /*
224 * Don't waste the memory on semaphores if we don't need them. 226 * Don't waste the memory on semaphores if we don't need them.
225 */ 227 */
226 if (nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS)) 228 if (nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS)) {
227 return 0; 229 return 0;
230 }
228 231
229 if (vm->sema_pool) 232 if (vm->sema_pool) {
230 return 0; 233 return 0;
234 }
231 235
232 sema_sea = nvgpu_semaphore_sea_create(g); 236 sema_sea = nvgpu_semaphore_sea_create(g);
233 if (!sema_sea) 237 if (!sema_sea) {
234 return -ENOMEM; 238 return -ENOMEM;
239 }
235 240
236 err = nvgpu_semaphore_pool_alloc(sema_sea, &vm->sema_pool); 241 err = nvgpu_semaphore_pool_alloc(sema_sea, &vm->sema_pool);
237 if (err) 242 if (err) {
238 return err; 243 return err;
244 }
239 245
240 /* 246 /*
241 * Allocate a chunk of GPU VA space for mapping the semaphores. We will 247 * Allocate a chunk of GPU VA space for mapping the semaphores. We will
@@ -287,11 +293,13 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
287 u64 kernel_vma_start, kernel_vma_limit; 293 u64 kernel_vma_start, kernel_vma_limit;
288 struct gk20a *g = gk20a_from_mm(mm); 294 struct gk20a *g = gk20a_from_mm(mm);
289 295
290 if (WARN_ON(kernel_reserved + low_hole > aperture_size)) 296 if (WARN_ON(kernel_reserved + low_hole > aperture_size)) {
291 return -ENOMEM; 297 return -ENOMEM;
298 }
292 299
293 if (WARN_ON(vm->guest_managed && kernel_reserved != 0)) 300 if (WARN_ON(vm->guest_managed && kernel_reserved != 0)) {
294 return -EINVAL; 301 return -EINVAL;
302 }
295 303
296 nvgpu_log_info(g, "Init space for %s: valimit=0x%llx, " 304 nvgpu_log_info(g, "Init space for %s: valimit=0x%llx, "
297 "LP size=0x%x lowhole=0x%llx", 305 "LP size=0x%x lowhole=0x%llx",
@@ -308,8 +316,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
308 vm->vma[gmmu_page_size_small] = &vm->user; 316 vm->vma[gmmu_page_size_small] = &vm->user;
309 vm->vma[gmmu_page_size_big] = &vm->user; 317 vm->vma[gmmu_page_size_big] = &vm->user;
310 vm->vma[gmmu_page_size_kernel] = &vm->kernel; 318 vm->vma[gmmu_page_size_kernel] = &vm->kernel;
311 if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) 319 if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
312 vm->vma[gmmu_page_size_big] = &vm->user_lp; 320 vm->vma[gmmu_page_size_big] = &vm->user_lp;
321 }
313 322
314 vm->va_start = low_hole; 323 vm->va_start = low_hole;
315 vm->va_limit = aperture_size; 324 vm->va_limit = aperture_size;
@@ -332,8 +341,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
332 /* Initialize the page table data structures. */ 341 /* Initialize the page table data structures. */
333 strncpy(vm->name, name, min(strlen(name), sizeof(vm->name))); 342 strncpy(vm->name, name, min(strlen(name), sizeof(vm->name)));
334 err = nvgpu_gmmu_init_page_table(vm); 343 err = nvgpu_gmmu_init_page_table(vm);
335 if (err) 344 if (err) {
336 goto clean_up_vgpu_vm; 345 goto clean_up_vgpu_vm;
346 }
337 347
338 /* Setup vma limits. */ 348 /* Setup vma limits. */
339 if (kernel_reserved + low_hole < aperture_size) { 349 if (kernel_reserved + low_hole < aperture_size) {
@@ -396,14 +406,15 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
396 * Determine if big pages are possible in this VM. If a split address 406 * Determine if big pages are possible in this VM. If a split address
397 * space is used then check the user_lp vma instead of the user vma. 407 * space is used then check the user_lp vma instead of the user vma.
398 */ 408 */
399 if (nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) 409 if (nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
400 vm->big_pages = big_pages && 410 vm->big_pages = big_pages &&
401 nvgpu_big_pages_possible(vm, user_vma_start, 411 nvgpu_big_pages_possible(vm, user_vma_start,
402 user_vma_limit - user_vma_start); 412 user_vma_limit - user_vma_start);
403 else 413 } else {
404 vm->big_pages = big_pages && 414 vm->big_pages = big_pages &&
405 nvgpu_big_pages_possible(vm, user_lp_vma_start, 415 nvgpu_big_pages_possible(vm, user_lp_vma_start,
406 user_lp_vma_limit - user_lp_vma_start); 416 user_lp_vma_limit - user_lp_vma_start);
417 }
407 418
408 /* 419 /*
409 * User VMA. 420 * User VMA.
@@ -418,8 +429,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
418 SZ_4K, 429 SZ_4K,
419 GPU_BALLOC_MAX_ORDER, 430 GPU_BALLOC_MAX_ORDER,
420 GPU_ALLOC_GVA_SPACE); 431 GPU_ALLOC_GVA_SPACE);
421 if (err) 432 if (err) {
422 goto clean_up_page_tables; 433 goto clean_up_page_tables;
434 }
423 } else { 435 } else {
424 /* 436 /*
425 * Make these allocator pointers point to the kernel allocator 437 * Make these allocator pointers point to the kernel allocator
@@ -443,8 +455,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
443 vm->big_page_size, 455 vm->big_page_size,
444 GPU_BALLOC_MAX_ORDER, 456 GPU_BALLOC_MAX_ORDER,
445 GPU_ALLOC_GVA_SPACE); 457 GPU_ALLOC_GVA_SPACE);
446 if (err) 458 if (err) {
447 goto clean_up_allocators; 459 goto clean_up_allocators;
460 }
448 } 461 }
449 462
450 /* 463 /*
@@ -458,8 +471,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
458 SZ_4K, 471 SZ_4K,
459 GPU_BALLOC_MAX_ORDER, 472 GPU_BALLOC_MAX_ORDER,
460 kernel_vma_flags); 473 kernel_vma_flags);
461 if (err) 474 if (err) {
462 goto clean_up_allocators; 475 goto clean_up_allocators;
476 }
463 477
464 vm->mapped_buffers = NULL; 478 vm->mapped_buffers = NULL;
465 479
@@ -475,19 +489,23 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
475 */ 489 */
476 if (vm->va_limit > 4ULL * SZ_1G) { 490 if (vm->va_limit > 4ULL * SZ_1G) {
477 err = nvgpu_init_sema_pool(vm); 491 err = nvgpu_init_sema_pool(vm);
478 if (err) 492 if (err) {
479 goto clean_up_allocators; 493 goto clean_up_allocators;
494 }
480 } 495 }
481 496
482 return 0; 497 return 0;
483 498
484clean_up_allocators: 499clean_up_allocators:
485 if (nvgpu_alloc_initialized(&vm->kernel)) 500 if (nvgpu_alloc_initialized(&vm->kernel)) {
486 nvgpu_alloc_destroy(&vm->kernel); 501 nvgpu_alloc_destroy(&vm->kernel);
487 if (nvgpu_alloc_initialized(&vm->user)) 502 }
503 if (nvgpu_alloc_initialized(&vm->user)) {
488 nvgpu_alloc_destroy(&vm->user); 504 nvgpu_alloc_destroy(&vm->user);
489 if (nvgpu_alloc_initialized(&vm->user_lp)) 505 }
506 if (nvgpu_alloc_initialized(&vm->user_lp)) {
490 nvgpu_alloc_destroy(&vm->user_lp); 507 nvgpu_alloc_destroy(&vm->user_lp);
508 }
491clean_up_page_tables: 509clean_up_page_tables:
492 /* Cleans up nvgpu_gmmu_init_page_table() */ 510 /* Cleans up nvgpu_gmmu_init_page_table() */
493 __nvgpu_pd_cache_free_direct(g, &vm->pdb); 511 __nvgpu_pd_cache_free_direct(g, &vm->pdb);
@@ -547,8 +565,9 @@ struct vm_gk20a *nvgpu_vm_init(struct gk20a *g,
547{ 565{
548 struct vm_gk20a *vm = nvgpu_kzalloc(g, sizeof(*vm)); 566 struct vm_gk20a *vm = nvgpu_kzalloc(g, sizeof(*vm));
549 567
550 if (!vm) 568 if (!vm) {
551 return NULL; 569 return NULL;
570 }
552 571
553 if (__nvgpu_vm_init(&g->mm, vm, big_page_size, low_hole, 572 if (__nvgpu_vm_init(&g->mm, vm, big_page_size, low_hole,
554 kernel_reserved, aperture_size, big_pages, 573 kernel_reserved, aperture_size, big_pages,
@@ -582,9 +601,10 @@ static void __nvgpu_vm_remove(struct vm_gk20a *vm)
582 } 601 }
583 } 602 }
584 603
585 if (nvgpu_mem_is_valid(&g->syncpt_mem) && vm->syncpt_ro_map_gpu_va) 604 if (nvgpu_mem_is_valid(&g->syncpt_mem) && vm->syncpt_ro_map_gpu_va) {
586 nvgpu_gmmu_unmap(vm, &g->syncpt_mem, 605 nvgpu_gmmu_unmap(vm, &g->syncpt_mem,
587 vm->syncpt_ro_map_gpu_va); 606 vm->syncpt_ro_map_gpu_va);
607 }
588 608
589 nvgpu_mutex_acquire(&vm->update_gmmu_lock); 609 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
590 610
@@ -603,12 +623,15 @@ static void __nvgpu_vm_remove(struct vm_gk20a *vm)
603 nvgpu_kfree(vm->mm->g, vm_area); 623 nvgpu_kfree(vm->mm->g, vm_area);
604 } 624 }
605 625
606 if (nvgpu_alloc_initialized(&vm->kernel)) 626 if (nvgpu_alloc_initialized(&vm->kernel)) {
607 nvgpu_alloc_destroy(&vm->kernel); 627 nvgpu_alloc_destroy(&vm->kernel);
608 if (nvgpu_alloc_initialized(&vm->user)) 628 }
629 if (nvgpu_alloc_initialized(&vm->user)) {
609 nvgpu_alloc_destroy(&vm->user); 630 nvgpu_alloc_destroy(&vm->user);
610 if (nvgpu_alloc_initialized(&vm->user_lp)) 631 }
632 if (nvgpu_alloc_initialized(&vm->user_lp)) {
611 nvgpu_alloc_destroy(&vm->user_lp); 633 nvgpu_alloc_destroy(&vm->user_lp);
634 }
612 635
613 nvgpu_vm_free_entries(vm, &vm->pdb); 636 nvgpu_vm_free_entries(vm, &vm->pdb);
614 637
@@ -664,8 +687,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf(
664 struct nvgpu_rbtree_node *root = vm->mapped_buffers; 687 struct nvgpu_rbtree_node *root = vm->mapped_buffers;
665 688
666 nvgpu_rbtree_search(addr, &node, root); 689 nvgpu_rbtree_search(addr, &node, root);
667 if (!node) 690 if (!node) {
668 return NULL; 691 return NULL;
692 }
669 693
670 return mapped_buffer_from_rbtree_node(node); 694 return mapped_buffer_from_rbtree_node(node);
671} 695}
@@ -677,8 +701,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_range(
677 struct nvgpu_rbtree_node *root = vm->mapped_buffers; 701 struct nvgpu_rbtree_node *root = vm->mapped_buffers;
678 702
679 nvgpu_rbtree_range_search(addr, &node, root); 703 nvgpu_rbtree_range_search(addr, &node, root);
680 if (!node) 704 if (!node) {
681 return NULL; 705 return NULL;
706 }
682 707
683 return mapped_buffer_from_rbtree_node(node); 708 return mapped_buffer_from_rbtree_node(node);
684} 709}
@@ -690,8 +715,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_less_than(
690 struct nvgpu_rbtree_node *root = vm->mapped_buffers; 715 struct nvgpu_rbtree_node *root = vm->mapped_buffers;
691 716
692 nvgpu_rbtree_less_than_search(addr, &node, root); 717 nvgpu_rbtree_less_than_search(addr, &node, root);
693 if (!node) 718 if (!node) {
694 return NULL; 719 return NULL;
720 }
695 721
696 return mapped_buffer_from_rbtree_node(node); 722 return mapped_buffer_from_rbtree_node(node);
697} 723}
@@ -746,8 +772,9 @@ void nvgpu_vm_put_buffers(struct vm_gk20a *vm,
746 int i; 772 int i;
747 struct vm_gk20a_mapping_batch batch; 773 struct vm_gk20a_mapping_batch batch;
748 774
749 if (num_buffers == 0) 775 if (num_buffers == 0) {
750 return; 776 return;
777 }
751 778
752 nvgpu_mutex_acquire(&vm->update_gmmu_lock); 779 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
753 nvgpu_vm_mapping_batch_start(&batch); 780 nvgpu_vm_mapping_batch_start(&batch);
@@ -814,10 +841,11 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
814 compr_kind : NVGPU_KIND_INVALID); 841 compr_kind : NVGPU_KIND_INVALID);
815 binfo.incompr_kind = incompr_kind; 842 binfo.incompr_kind = incompr_kind;
816 843
817 if (compr_kind != NVGPU_KIND_INVALID) 844 if (compr_kind != NVGPU_KIND_INVALID) {
818 map_key_kind = compr_kind; 845 map_key_kind = compr_kind;
819 else 846 } else {
820 map_key_kind = incompr_kind; 847 map_key_kind = incompr_kind;
848 }
821 849
822 /* 850 /*
823 * Check if this buffer is already mapped. 851 * Check if this buffer is already mapped.
@@ -847,11 +875,12 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
847 } 875 }
848 876
849 align = nvgpu_sgt_alignment(g, sgt); 877 align = nvgpu_sgt_alignment(g, sgt);
850 if (g->mm.disable_bigpage) 878 if (g->mm.disable_bigpage) {
851 binfo.pgsz_idx = gmmu_page_size_small; 879 binfo.pgsz_idx = gmmu_page_size_small;
852 else 880 } else {
853 binfo.pgsz_idx = __get_pte_size(vm, map_addr, 881 binfo.pgsz_idx = __get_pte_size(vm, map_addr,
854 min_t(u64, binfo.size, align)); 882 min_t(u64, binfo.size, align));
883 }
855 map_size = map_size ? map_size : binfo.size; 884 map_size = map_size ? map_size : binfo.size;
856 map_size = ALIGN(map_size, SZ_4K); 885 map_size = ALIGN(map_size, SZ_4K);
857 886
@@ -872,8 +901,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
872 map_size, 901 map_size,
873 binfo.pgsz_idx, 902 binfo.pgsz_idx,
874 &vm_area); 903 &vm_area);
875 if (err) 904 if (err) {
876 goto clean_up; 905 goto clean_up;
906 }
877 907
878 va_allocated = false; 908 va_allocated = false;
879 } 909 }
@@ -941,8 +971,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
941 comptags.lines - 1)); 971 comptags.lines - 1));
942 gk20a_comptags_finish_clear( 972 gk20a_comptags_finish_clear(
943 os_buf, err == 0); 973 os_buf, err == 0);
944 if (err) 974 if (err) {
945 goto clean_up; 975 goto clean_up;
976 }
946 } 977 }
947 } else { 978 } else {
948 /* 979 /*
@@ -955,8 +986,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
955 /* 986 /*
956 * Store the ctag offset for later use if we got the comptags 987 * Store the ctag offset for later use if we got the comptags
957 */ 988 */
958 if (comptags.lines) 989 if (comptags.lines) {
959 ctag_offset = comptags.offset; 990 ctag_offset = comptags.offset;
991 }
960 } 992 }
961 993
962 /* 994 /*
@@ -984,8 +1016,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
984 goto clean_up; 1016 goto clean_up;
985 } 1017 }
986 1018
987 if (clear_ctags) 1019 if (clear_ctags) {
988 clear_ctags = gk20a_comptags_start_clear(os_buf); 1020 clear_ctags = gk20a_comptags_start_clear(os_buf);
1021 }
989 1022
990 map_addr = g->ops.mm.gmmu_map(vm, 1023 map_addr = g->ops.mm.gmmu_map(vm,
991 map_addr, 1024 map_addr,
@@ -1003,8 +1036,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
1003 batch, 1036 batch,
1004 aperture); 1037 aperture);
1005 1038
1006 if (clear_ctags) 1039 if (clear_ctags) {
1007 gk20a_comptags_finish_clear(os_buf, map_addr != 0); 1040 gk20a_comptags_finish_clear(os_buf, map_addr != 0);
1041 }
1008 1042
1009 if (!map_addr) { 1043 if (!map_addr) {
1010 err = -ENOMEM; 1044 err = -ENOMEM;
@@ -1041,7 +1075,7 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
1041 return mapped_buffer; 1075 return mapped_buffer;
1042 1076
1043clean_up: 1077clean_up:
1044 if (mapped_buffer->addr) 1078 if (mapped_buffer->addr) {
1045 g->ops.mm.gmmu_unmap(vm, 1079 g->ops.mm.gmmu_unmap(vm,
1046 mapped_buffer->addr, 1080 mapped_buffer->addr,
1047 mapped_buffer->size, 1081 mapped_buffer->size,
@@ -1051,6 +1085,7 @@ clean_up:
1051 mapped_buffer->vm_area ? 1085 mapped_buffer->vm_area ?
1052 mapped_buffer->vm_area->sparse : false, 1086 mapped_buffer->vm_area->sparse : false,
1053 NULL); 1087 NULL);
1088 }
1054 nvgpu_mutex_release(&vm->update_gmmu_lock); 1089 nvgpu_mutex_release(&vm->update_gmmu_lock);
1055clean_up_nolock: 1090clean_up_nolock:
1056 nvgpu_kfree(g, mapped_buffer); 1091 nvgpu_kfree(g, mapped_buffer);
@@ -1132,14 +1167,16 @@ static int nvgpu_vm_unmap_sync_buffer(struct vm_gk20a *vm,
1132 nvgpu_timeout_init(vm->mm->g, &timeout, 50, NVGPU_TIMER_CPU_TIMER); 1167 nvgpu_timeout_init(vm->mm->g, &timeout, 50, NVGPU_TIMER_CPU_TIMER);
1133 1168
1134 do { 1169 do {
1135 if (nvgpu_atomic_read(&mapped_buffer->ref.refcount) == 1) 1170 if (nvgpu_atomic_read(&mapped_buffer->ref.refcount) == 1) {
1136 break; 1171 break;
1172 }
1137 nvgpu_msleep(10); 1173 nvgpu_msleep(10);
1138 } while (!nvgpu_timeout_expired_msg(&timeout, 1174 } while (!nvgpu_timeout_expired_msg(&timeout,
1139 "sync-unmap failed on 0x%llx")); 1175 "sync-unmap failed on 0x%llx"));
1140 1176
1141 if (nvgpu_timeout_expired(&timeout)) 1177 if (nvgpu_timeout_expired(&timeout)) {
1142 ret = -ETIMEDOUT; 1178 ret = -ETIMEDOUT;
1179 }
1143 1180
1144 nvgpu_mutex_acquire(&vm->update_gmmu_lock); 1181 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
1145 1182
@@ -1154,16 +1191,18 @@ void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset,
1154 nvgpu_mutex_acquire(&vm->update_gmmu_lock); 1191 nvgpu_mutex_acquire(&vm->update_gmmu_lock);
1155 1192
1156 mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, offset); 1193 mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, offset);
1157 if (!mapped_buffer) 1194 if (!mapped_buffer) {
1158 goto done; 1195 goto done;
1196 }
1159 1197
1160 if (mapped_buffer->flags & NVGPU_VM_MAP_FIXED_OFFSET) { 1198 if (mapped_buffer->flags & NVGPU_VM_MAP_FIXED_OFFSET) {
1161 if (nvgpu_vm_unmap_sync_buffer(vm, mapped_buffer)) 1199 if (nvgpu_vm_unmap_sync_buffer(vm, mapped_buffer)) {
1162 /* 1200 /*
1163 * Looks like we have failed... Better not continue in 1201 * Looks like we have failed... Better not continue in
1164 * case the buffer is in use. 1202 * case the buffer is in use.
1165 */ 1203 */
1166 goto done; 1204 goto done;
1205 }
1167 } 1206 }
1168 1207
1169 /* 1208 /*