diff options
author | Srirangan <smadhavan@nvidia.com> | 2018-08-14 01:27:15 -0400 |
---|---|---|
committer | mobile promotions <svcmobile_promotions@nvidia.com> | 2018-08-17 16:54:08 -0400 |
commit | 70c20bb75be7815ebc67ac82d6999f46bc25ed6d (patch) | |
tree | 19d6b5299e09b71e9afe2967a758f036bb9b79bc | |
parent | 553fdf3534f856edce73744fd54914b9b7a829cc (diff) |
gpu: nvgpu: common: mm: Fix MISRA 15.6 violations
MISRA Rule-15.6 requires that all if-else blocks be enclosed in braces,
including single statement blocks. Fix errors due to single statement
if blocks without braces, introducing the braces.
JIRA NVGPU-671
Change-Id: Ieeecf719dca9acc1a116d2893637bf770caf4f5b
Signed-off-by: Srirangan <smadhavan@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1794241
GVS: Gerrit_Virtual_Submit
Reviewed-by: Adeel Raza <araza@nvidia.com>
Reviewed-by: Alex Waterman <alexw@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/buddy_allocator.c | 148 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/gmmu.c | 98 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/lockless_allocator.c | 20 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/mm.c | 119 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/nvgpu_mem.c | 23 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/vm.c | 123 |
6 files changed, 348 insertions, 183 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c index 5a0a28cb..365f3b7b 100644 --- a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c +++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c | |||
@@ -79,10 +79,12 @@ static void balloc_compute_max_order(struct nvgpu_buddy_allocator *a) | |||
79 | return; | 79 | return; |
80 | } | 80 | } |
81 | 81 | ||
82 | if (a->max_order > true_max_order) | 82 | if (a->max_order > true_max_order) { |
83 | a->max_order = true_max_order; | 83 | a->max_order = true_max_order; |
84 | if (a->max_order > GPU_BALLOC_MAX_ORDER) | 84 | } |
85 | if (a->max_order > GPU_BALLOC_MAX_ORDER) { | ||
85 | a->max_order = GPU_BALLOC_MAX_ORDER; | 86 | a->max_order = GPU_BALLOC_MAX_ORDER; |
87 | } | ||
86 | } | 88 | } |
87 | 89 | ||
88 | /* | 90 | /* |
@@ -108,8 +110,9 @@ static struct nvgpu_buddy *balloc_new_buddy(struct nvgpu_buddy_allocator *a, | |||
108 | struct nvgpu_buddy *new_buddy; | 110 | struct nvgpu_buddy *new_buddy; |
109 | 111 | ||
110 | new_buddy = nvgpu_kmem_cache_alloc(a->buddy_cache); | 112 | new_buddy = nvgpu_kmem_cache_alloc(a->buddy_cache); |
111 | if (!new_buddy) | 113 | if (!new_buddy) { |
112 | return NULL; | 114 | return NULL; |
115 | } | ||
113 | 116 | ||
114 | memset(new_buddy, 0, sizeof(struct nvgpu_buddy)); | 117 | memset(new_buddy, 0, sizeof(struct nvgpu_buddy)); |
115 | 118 | ||
@@ -139,10 +142,11 @@ static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a, | |||
139 | * without cycling through the entire list. | 142 | * without cycling through the entire list. |
140 | */ | 143 | */ |
141 | if (a->flags & GPU_ALLOC_GVA_SPACE && | 144 | if (a->flags & GPU_ALLOC_GVA_SPACE && |
142 | b->pte_size == gmmu_page_size_big) | 145 | b->pte_size == gmmu_page_size_big) { |
143 | nvgpu_list_add_tail(&b->buddy_entry, list); | 146 | nvgpu_list_add_tail(&b->buddy_entry, list); |
144 | else | 147 | } else { |
145 | nvgpu_list_add(&b->buddy_entry, list); | 148 | nvgpu_list_add(&b->buddy_entry, list); |
149 | } | ||
146 | 150 | ||
147 | buddy_set_in_list(b); | 151 | buddy_set_in_list(b); |
148 | } | 152 | } |
@@ -181,8 +185,9 @@ static void balloc_blist_rem(struct nvgpu_buddy_allocator *a, | |||
181 | 185 | ||
182 | static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len) | 186 | static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len) |
183 | { | 187 | { |
184 | if (len == 0) | 188 | if (len == 0) { |
185 | return 0; | 189 | return 0; |
190 | } | ||
186 | 191 | ||
187 | len--; | 192 | len--; |
188 | len >>= a->blk_shift; | 193 | len >>= a->blk_shift; |
@@ -195,10 +200,11 @@ static u64 __balloc_max_order_in(struct nvgpu_buddy_allocator *a, | |||
195 | { | 200 | { |
196 | u64 size = (end - start) >> a->blk_shift; | 201 | u64 size = (end - start) >> a->blk_shift; |
197 | 202 | ||
198 | if (size > 0) | 203 | if (size > 0) { |
199 | return min_t(u64, ilog2(size), a->max_order); | 204 | return min_t(u64, ilog2(size), a->max_order); |
200 | else | 205 | } else { |
201 | return GPU_BALLOC_MAX_ORDER; | 206 | return GPU_BALLOC_MAX_ORDER; |
207 | } | ||
202 | } | 208 | } |
203 | 209 | ||
204 | /* | 210 | /* |
@@ -222,8 +228,9 @@ static int balloc_init_lists(struct nvgpu_buddy_allocator *a) | |||
222 | order = __balloc_max_order_in(a, bstart, bend); | 228 | order = __balloc_max_order_in(a, bstart, bend); |
223 | 229 | ||
224 | buddy = balloc_new_buddy(a, NULL, bstart, order); | 230 | buddy = balloc_new_buddy(a, NULL, bstart, order); |
225 | if (!buddy) | 231 | if (!buddy) { |
226 | goto cleanup; | 232 | goto cleanup; |
233 | } | ||
227 | 234 | ||
228 | balloc_blist_add(a, buddy); | 235 | balloc_blist_add(a, buddy); |
229 | bstart += balloc_order_to_len(a, order); | 236 | bstart += balloc_order_to_len(a, order); |
@@ -340,17 +347,20 @@ static void balloc_coalesce(struct nvgpu_buddy_allocator *a, | |||
340 | { | 347 | { |
341 | struct nvgpu_buddy *parent; | 348 | struct nvgpu_buddy *parent; |
342 | 349 | ||
343 | if (buddy_is_alloced(b) || buddy_is_split(b)) | 350 | if (buddy_is_alloced(b) || buddy_is_split(b)) { |
344 | return; | 351 | return; |
352 | } | ||
345 | 353 | ||
346 | /* | 354 | /* |
347 | * If both our buddy and I are both not allocated and not split then | 355 | * If both our buddy and I are both not allocated and not split then |
348 | * we can coalesce ourselves. | 356 | * we can coalesce ourselves. |
349 | */ | 357 | */ |
350 | if (!b->buddy) | 358 | if (!b->buddy) { |
351 | return; | 359 | return; |
352 | if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy)) | 360 | } |
361 | if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy)) { | ||
353 | return; | 362 | return; |
363 | } | ||
354 | 364 | ||
355 | parent = b->parent; | 365 | parent = b->parent; |
356 | 366 | ||
@@ -383,8 +393,9 @@ static int balloc_split_buddy(struct nvgpu_buddy_allocator *a, | |||
383 | u64 half; | 393 | u64 half; |
384 | 394 | ||
385 | left = balloc_new_buddy(a, b, b->start, b->order - 1); | 395 | left = balloc_new_buddy(a, b, b->start, b->order - 1); |
386 | if (!left) | 396 | if (!left) { |
387 | return -ENOMEM; | 397 | return -ENOMEM; |
398 | } | ||
388 | 399 | ||
389 | half = (b->end - b->start) / 2; | 400 | half = (b->end - b->start) / 2; |
390 | 401 | ||
@@ -449,8 +460,9 @@ static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a, | |||
449 | struct nvgpu_buddy *bud; | 460 | struct nvgpu_buddy *bud; |
450 | 461 | ||
451 | nvgpu_rbtree_search(addr, &node, a->alloced_buddies); | 462 | nvgpu_rbtree_search(addr, &node, a->alloced_buddies); |
452 | if (!node) | 463 | if (!node) { |
453 | return NULL; | 464 | return NULL; |
465 | } | ||
454 | 466 | ||
455 | bud = nvgpu_buddy_from_rbtree_node(node); | 467 | bud = nvgpu_buddy_from_rbtree_node(node); |
456 | 468 | ||
@@ -470,21 +482,24 @@ static struct nvgpu_buddy *__balloc_find_buddy(struct nvgpu_buddy_allocator *a, | |||
470 | struct nvgpu_buddy *bud; | 482 | struct nvgpu_buddy *bud; |
471 | 483 | ||
472 | if (order > a->max_order || | 484 | if (order > a->max_order || |
473 | nvgpu_list_empty(balloc_get_order_list(a, order))) | 485 | nvgpu_list_empty(balloc_get_order_list(a, order))) { |
474 | return NULL; | 486 | return NULL; |
487 | } | ||
475 | 488 | ||
476 | if (a->flags & GPU_ALLOC_GVA_SPACE && | 489 | if (a->flags & GPU_ALLOC_GVA_SPACE && |
477 | pte_size == gmmu_page_size_big) | 490 | pte_size == gmmu_page_size_big) { |
478 | bud = nvgpu_list_last_entry(balloc_get_order_list(a, order), | 491 | bud = nvgpu_list_last_entry(balloc_get_order_list(a, order), |
479 | nvgpu_buddy, buddy_entry); | 492 | nvgpu_buddy, buddy_entry); |
480 | else | 493 | } else { |
481 | bud = nvgpu_list_first_entry(balloc_get_order_list(a, order), | 494 | bud = nvgpu_list_first_entry(balloc_get_order_list(a, order), |
482 | nvgpu_buddy, buddy_entry); | 495 | nvgpu_buddy, buddy_entry); |
496 | } | ||
483 | 497 | ||
484 | if (pte_size != BALLOC_PTE_SIZE_ANY && | 498 | if (pte_size != BALLOC_PTE_SIZE_ANY && |
485 | pte_size != bud->pte_size && | 499 | pte_size != bud->pte_size && |
486 | bud->pte_size != BALLOC_PTE_SIZE_ANY) | 500 | bud->pte_size != BALLOC_PTE_SIZE_ANY) { |
487 | return NULL; | 501 | return NULL; |
502 | } | ||
488 | 503 | ||
489 | return bud; | 504 | return bud; |
490 | } | 505 | } |
@@ -511,12 +526,14 @@ static u64 __balloc_do_alloc(struct nvgpu_buddy_allocator *a, | |||
511 | } | 526 | } |
512 | 527 | ||
513 | /* Out of memory! */ | 528 | /* Out of memory! */ |
514 | if (!bud) | 529 | if (!bud) { |
515 | return 0; | 530 | return 0; |
531 | } | ||
516 | 532 | ||
517 | while (bud->order != order) { | 533 | while (bud->order != order) { |
518 | if (balloc_split_buddy(a, bud, pte_size)) | 534 | if (balloc_split_buddy(a, bud, pte_size)) { |
519 | return 0; /* No mem... */ | 535 | return 0; /* No mem... */ |
536 | } | ||
520 | bud = bud->left; | 537 | bud = bud->left; |
521 | } | 538 | } |
522 | 539 | ||
@@ -540,19 +557,22 @@ static int balloc_is_range_free(struct nvgpu_buddy_allocator *a, | |||
540 | struct nvgpu_buddy *bud; | 557 | struct nvgpu_buddy *bud; |
541 | 558 | ||
542 | nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies); | 559 | nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies); |
543 | if (!node) | 560 | if (!node) { |
544 | return 1; /* No allocs yet. */ | 561 | return 1; /* No allocs yet. */ |
562 | } | ||
545 | 563 | ||
546 | bud = nvgpu_buddy_from_rbtree_node(node); | 564 | bud = nvgpu_buddy_from_rbtree_node(node); |
547 | 565 | ||
548 | while (bud->start < end) { | 566 | while (bud->start < end) { |
549 | if ((bud->start > base && bud->start < end) || | 567 | if ((bud->start > base && bud->start < end) || |
550 | (bud->end > base && bud->end < end)) | 568 | (bud->end > base && bud->end < end)) { |
551 | return 0; | 569 | return 0; |
570 | } | ||
552 | 571 | ||
553 | nvgpu_rbtree_enum_next(&node, node); | 572 | nvgpu_rbtree_enum_next(&node, node); |
554 | if (!node) | 573 | if (!node) { |
555 | break; | 574 | break; |
575 | } | ||
556 | bud = nvgpu_buddy_from_rbtree_node(node); | 576 | bud = nvgpu_buddy_from_rbtree_node(node); |
557 | } | 577 | } |
558 | 578 | ||
@@ -581,8 +601,9 @@ static struct nvgpu_fixed_alloc *balloc_free_fixed( | |||
581 | struct nvgpu_rbtree_node *node = NULL; | 601 | struct nvgpu_rbtree_node *node = NULL; |
582 | 602 | ||
583 | nvgpu_rbtree_search(addr, &node, a->fixed_allocs); | 603 | nvgpu_rbtree_search(addr, &node, a->fixed_allocs); |
584 | if (!node) | 604 | if (!node) { |
585 | return NULL; | 605 | return NULL; |
606 | } | ||
586 | 607 | ||
587 | falloc = nvgpu_fixed_alloc_from_rbtree_node(node); | 608 | falloc = nvgpu_fixed_alloc_from_rbtree_node(node); |
588 | 609 | ||
@@ -657,8 +678,9 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy( | |||
657 | } | 678 | } |
658 | } | 679 | } |
659 | 680 | ||
660 | if (found) | 681 | if (found) { |
661 | break; | 682 | break; |
683 | } | ||
662 | 684 | ||
663 | __balloc_get_parent_range(a, cur_base, cur_order, | 685 | __balloc_get_parent_range(a, cur_base, cur_order, |
664 | &cur_base, &cur_order); | 686 | &cur_base, &cur_order); |
@@ -679,10 +701,11 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy( | |||
679 | return NULL; | 701 | return NULL; |
680 | } | 702 | } |
681 | 703 | ||
682 | if (base < bud->right->start) | 704 | if (base < bud->right->start) { |
683 | bud = bud->left; | 705 | bud = bud->left; |
684 | else | 706 | } else { |
685 | bud = bud->right; | 707 | bud = bud->right; |
708 | } | ||
686 | 709 | ||
687 | } | 710 | } |
688 | 711 | ||
@@ -697,12 +720,13 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a, | |||
697 | u64 align_order; | 720 | u64 align_order; |
698 | 721 | ||
699 | shifted_base = balloc_base_shift(a, base); | 722 | shifted_base = balloc_base_shift(a, base); |
700 | if (shifted_base == 0) | 723 | if (shifted_base == 0) { |
701 | align_order = __fls(len >> a->blk_shift); | 724 | align_order = __fls(len >> a->blk_shift); |
702 | else | 725 | } else { |
703 | align_order = min_t(u64, | 726 | align_order = min_t(u64, |
704 | __ffs(shifted_base >> a->blk_shift), | 727 | __ffs(shifted_base >> a->blk_shift), |
705 | __fls(len >> a->blk_shift)); | 728 | __fls(len >> a->blk_shift)); |
729 | } | ||
706 | 730 | ||
707 | if (align_order > a->max_order) { | 731 | if (align_order > a->max_order) { |
708 | alloc_dbg(balloc_owner(a), | 732 | alloc_dbg(balloc_owner(a), |
@@ -741,9 +765,10 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a, | |||
741 | align_order = __ffs(inc_base >> a->blk_shift); | 765 | align_order = __ffs(inc_base >> a->blk_shift); |
742 | 766 | ||
743 | /* If we don't have much left - trim down align_order. */ | 767 | /* If we don't have much left - trim down align_order. */ |
744 | if (balloc_order_to_len(a, align_order) > remaining) | 768 | if (balloc_order_to_len(a, align_order) > remaining) { |
745 | align_order = __balloc_max_order_in(a, inc_base, | 769 | align_order = __balloc_max_order_in(a, inc_base, |
746 | inc_base + remaining); | 770 | inc_base + remaining); |
771 | } | ||
747 | } | 772 | } |
748 | 773 | ||
749 | return base; | 774 | return base; |
@@ -805,10 +830,11 @@ static u64 nvgpu_buddy_balloc(struct nvgpu_allocator *__a, u64 len) | |||
805 | return 0; | 830 | return 0; |
806 | } | 831 | } |
807 | 832 | ||
808 | if (a->flags & GPU_ALLOC_GVA_SPACE) | 833 | if (a->flags & GPU_ALLOC_GVA_SPACE) { |
809 | pte_size = __get_pte_size(a->vm, 0, len); | 834 | pte_size = __get_pte_size(a->vm, 0, len); |
810 | else | 835 | } else { |
811 | pte_size = BALLOC_PTE_SIZE_ANY; | 836 | pte_size = BALLOC_PTE_SIZE_ANY; |
837 | } | ||
812 | 838 | ||
813 | addr = __balloc_do_alloc(a, order, pte_size); | 839 | addr = __balloc_do_alloc(a, order, pte_size); |
814 | 840 | ||
@@ -845,25 +871,29 @@ static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a, | |||
845 | struct nvgpu_buddy_allocator *a = __a->priv; | 871 | struct nvgpu_buddy_allocator *a = __a->priv; |
846 | 872 | ||
847 | /* If base isn't aligned to an order 0 block, fail. */ | 873 | /* If base isn't aligned to an order 0 block, fail. */ |
848 | if (base & (a->blk_size - 1)) | 874 | if (base & (a->blk_size - 1)) { |
849 | goto fail; | 875 | goto fail; |
876 | } | ||
850 | 877 | ||
851 | if (len == 0) | 878 | if (len == 0) { |
852 | goto fail; | 879 | goto fail; |
880 | } | ||
853 | 881 | ||
854 | /* Check that the page size is valid. */ | 882 | /* Check that the page size is valid. */ |
855 | if (a->flags & GPU_ALLOC_GVA_SPACE && a->vm->big_pages) { | 883 | if (a->flags & GPU_ALLOC_GVA_SPACE && a->vm->big_pages) { |
856 | if (page_size == a->vm->big_page_size) | 884 | if (page_size == a->vm->big_page_size) { |
857 | pte_size = gmmu_page_size_big; | 885 | pte_size = gmmu_page_size_big; |
858 | else if (page_size == SZ_4K) | 886 | } else if (page_size == SZ_4K) { |
859 | pte_size = gmmu_page_size_small; | 887 | pte_size = gmmu_page_size_small; |
860 | else | 888 | } else { |
861 | goto fail; | 889 | goto fail; |
890 | } | ||
862 | } | 891 | } |
863 | 892 | ||
864 | falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc)); | 893 | falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc)); |
865 | if (!falloc) | 894 | if (!falloc) { |
866 | goto fail; | 895 | goto fail; |
896 | } | ||
867 | 897 | ||
868 | nvgpu_init_list_node(&falloc->buddies); | 898 | nvgpu_init_list_node(&falloc->buddies); |
869 | falloc->start = base; | 899 | falloc->start = base; |
@@ -936,8 +966,9 @@ static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr) | |||
936 | struct nvgpu_fixed_alloc *falloc; | 966 | struct nvgpu_fixed_alloc *falloc; |
937 | struct nvgpu_buddy_allocator *a = __a->priv; | 967 | struct nvgpu_buddy_allocator *a = __a->priv; |
938 | 968 | ||
939 | if (!addr) | 969 | if (!addr) { |
940 | return; | 970 | return; |
971 | } | ||
941 | 972 | ||
942 | alloc_lock(__a); | 973 | alloc_lock(__a); |
943 | 974 | ||
@@ -952,8 +983,9 @@ static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr) | |||
952 | } | 983 | } |
953 | 984 | ||
954 | bud = balloc_free_buddy(a, addr); | 985 | bud = balloc_free_buddy(a, addr); |
955 | if (!bud) | 986 | if (!bud) { |
956 | goto done; | 987 | goto done; |
988 | } | ||
957 | 989 | ||
958 | balloc_blist_add(a, bud); | 990 | balloc_blist_add(a, bud); |
959 | a->bytes_freed += balloc_order_to_len(a, bud->order); | 991 | a->bytes_freed += balloc_order_to_len(a, bud->order); |
@@ -987,8 +1019,9 @@ static bool nvgpu_buddy_reserve_is_possible(struct nvgpu_buddy_allocator *a, | |||
987 | if ((co_base >= tmp->base && | 1019 | if ((co_base >= tmp->base && |
988 | co_base < (tmp->base + tmp->length)) || | 1020 | co_base < (tmp->base + tmp->length)) || |
989 | (co_end >= tmp->base && | 1021 | (co_end >= tmp->base && |
990 | co_end < (tmp->base + tmp->length))) | 1022 | co_end < (tmp->base + tmp->length))) { |
991 | return false; | 1023 | return false; |
1024 | } | ||
992 | } | 1025 | } |
993 | 1026 | ||
994 | return true; | 1027 | return true; |
@@ -1006,8 +1039,9 @@ static int nvgpu_buddy_reserve_co(struct nvgpu_allocator *__a, | |||
1006 | int err = 0; | 1039 | int err = 0; |
1007 | 1040 | ||
1008 | if (co->base < a->start || (co->base + co->length) > a->end || | 1041 | if (co->base < a->start || (co->base + co->length) > a->end || |
1009 | a->alloc_made) | 1042 | a->alloc_made) { |
1010 | return -EINVAL; | 1043 | return -EINVAL; |
1044 | } | ||
1011 | 1045 | ||
1012 | alloc_lock(__a); | 1046 | alloc_lock(__a); |
1013 | 1047 | ||
@@ -1221,25 +1255,31 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
1221 | struct nvgpu_buddy_allocator *a; | 1255 | struct nvgpu_buddy_allocator *a; |
1222 | 1256 | ||
1223 | /* blk_size must be greater than 0 and a power of 2. */ | 1257 | /* blk_size must be greater than 0 and a power of 2. */ |
1224 | if (blk_size == 0) | 1258 | if (blk_size == 0) { |
1225 | return -EINVAL; | 1259 | return -EINVAL; |
1226 | if (blk_size & (blk_size - 1)) | 1260 | } |
1261 | if (blk_size & (blk_size - 1)) { | ||
1227 | return -EINVAL; | 1262 | return -EINVAL; |
1263 | } | ||
1228 | 1264 | ||
1229 | if (max_order > GPU_BALLOC_MAX_ORDER) | 1265 | if (max_order > GPU_BALLOC_MAX_ORDER) { |
1230 | return -EINVAL; | 1266 | return -EINVAL; |
1267 | } | ||
1231 | 1268 | ||
1232 | /* If this is to manage a GVA space we need a VM. */ | 1269 | /* If this is to manage a GVA space we need a VM. */ |
1233 | if (flags & GPU_ALLOC_GVA_SPACE && !vm) | 1270 | if (flags & GPU_ALLOC_GVA_SPACE && !vm) { |
1234 | return -EINVAL; | 1271 | return -EINVAL; |
1272 | } | ||
1235 | 1273 | ||
1236 | a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator)); | 1274 | a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator)); |
1237 | if (!a) | 1275 | if (!a) { |
1238 | return -ENOMEM; | 1276 | return -ENOMEM; |
1277 | } | ||
1239 | 1278 | ||
1240 | err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops); | 1279 | err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops); |
1241 | if (err) | 1280 | if (err) { |
1242 | goto fail; | 1281 | goto fail; |
1282 | } | ||
1243 | 1283 | ||
1244 | a->base = base; | 1284 | a->base = base; |
1245 | a->length = size; | 1285 | a->length = size; |
@@ -1269,8 +1309,9 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
1269 | */ | 1309 | */ |
1270 | if (flags & GPU_ALLOC_GVA_SPACE && vm->big_pages && | 1310 | if (flags & GPU_ALLOC_GVA_SPACE && vm->big_pages && |
1271 | (base & ((vm->big_page_size << 10) - 1) || | 1311 | (base & ((vm->big_page_size << 10) - 1) || |
1272 | size & ((vm->big_page_size << 10) - 1))) | 1312 | size & ((vm->big_page_size << 10) - 1))) { |
1273 | return -EINVAL; | 1313 | return -EINVAL; |
1314 | } | ||
1274 | 1315 | ||
1275 | a->flags = flags; | 1316 | a->flags = flags; |
1276 | a->max_order = max_order; | 1317 | a->max_order = max_order; |
@@ -1288,8 +1329,9 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
1288 | a->fixed_allocs = NULL; | 1329 | a->fixed_allocs = NULL; |
1289 | nvgpu_init_list_node(&a->co_list); | 1330 | nvgpu_init_list_node(&a->co_list); |
1290 | err = balloc_init_lists(a); | 1331 | err = balloc_init_lists(a); |
1291 | if (err) | 1332 | if (err) { |
1292 | goto fail; | 1333 | goto fail; |
1334 | } | ||
1293 | 1335 | ||
1294 | nvgpu_smp_wmb(); | 1336 | nvgpu_smp_wmb(); |
1295 | a->initialized = 1; | 1337 | a->initialized = 1; |
@@ -1301,18 +1343,20 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
1301 | alloc_dbg(__a, " base 0x%llx", a->base); | 1343 | alloc_dbg(__a, " base 0x%llx", a->base); |
1302 | alloc_dbg(__a, " size 0x%llx", a->length); | 1344 | alloc_dbg(__a, " size 0x%llx", a->length); |
1303 | alloc_dbg(__a, " blk_size 0x%llx", a->blk_size); | 1345 | alloc_dbg(__a, " blk_size 0x%llx", a->blk_size); |
1304 | if (flags & GPU_ALLOC_GVA_SPACE) | 1346 | if (flags & GPU_ALLOC_GVA_SPACE) { |
1305 | alloc_dbg(balloc_owner(a), | 1347 | alloc_dbg(balloc_owner(a), |
1306 | " pde_size 0x%llx", | 1348 | " pde_size 0x%llx", |
1307 | balloc_order_to_len(a, a->pte_blk_order)); | 1349 | balloc_order_to_len(a, a->pte_blk_order)); |
1350 | } | ||
1308 | alloc_dbg(__a, " max_order %llu", a->max_order); | 1351 | alloc_dbg(__a, " max_order %llu", a->max_order); |
1309 | alloc_dbg(__a, " flags 0x%llx", a->flags); | 1352 | alloc_dbg(__a, " flags 0x%llx", a->flags); |
1310 | 1353 | ||
1311 | return 0; | 1354 | return 0; |
1312 | 1355 | ||
1313 | fail: | 1356 | fail: |
1314 | if (a->buddy_cache) | 1357 | if (a->buddy_cache) { |
1315 | nvgpu_kmem_cache_destroy(a->buddy_cache); | 1358 | nvgpu_kmem_cache_destroy(a->buddy_cache); |
1359 | } | ||
1316 | nvgpu_kfree(g, a); | 1360 | nvgpu_kfree(g, a); |
1317 | return err; | 1361 | return err; |
1318 | } | 1362 | } |
diff --git a/drivers/gpu/nvgpu/common/mm/gmmu.c b/drivers/gpu/nvgpu/common/mm/gmmu.c index f6f2b9ad..498cdf06 100644 --- a/drivers/gpu/nvgpu/common/mm/gmmu.c +++ b/drivers/gpu/nvgpu/common/mm/gmmu.c | |||
@@ -38,18 +38,20 @@ | |||
38 | 38 | ||
39 | #define __gmmu_dbg(g, attrs, fmt, args...) \ | 39 | #define __gmmu_dbg(g, attrs, fmt, args...) \ |
40 | do { \ | 40 | do { \ |
41 | if (attrs->debug) \ | 41 | if (attrs->debug) { \ |
42 | nvgpu_info(g, fmt, ##args); \ | 42 | nvgpu_info(g, fmt, ##args); \ |
43 | else \ | 43 | } else { \ |
44 | nvgpu_log(g, gpu_dbg_map, fmt, ##args); \ | 44 | nvgpu_log(g, gpu_dbg_map, fmt, ##args); \ |
45 | } \ | ||
45 | } while (0) | 46 | } while (0) |
46 | 47 | ||
47 | #define __gmmu_dbg_v(g, attrs, fmt, args...) \ | 48 | #define __gmmu_dbg_v(g, attrs, fmt, args...) \ |
48 | do { \ | 49 | do { \ |
49 | if (attrs->debug) \ | 50 | if (attrs->debug) { \ |
50 | nvgpu_info(g, fmt, ##args); \ | 51 | nvgpu_info(g, fmt, ##args); \ |
51 | else \ | 52 | } else { \ |
52 | nvgpu_log(g, gpu_dbg_map_v, fmt, ##args); \ | 53 | nvgpu_log(g, gpu_dbg_map_v, fmt, ##args); \ |
54 | } \ | ||
53 | } while (0) | 55 | } while (0) |
54 | 56 | ||
55 | static int pd_allocate(struct vm_gk20a *vm, | 57 | static int pd_allocate(struct vm_gk20a *vm, |
@@ -77,15 +79,17 @@ static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm, | |||
77 | 79 | ||
78 | struct nvgpu_sgt *sgt = nvgpu_sgt_create_from_mem(g, mem); | 80 | struct nvgpu_sgt *sgt = nvgpu_sgt_create_from_mem(g, mem); |
79 | 81 | ||
80 | if (!sgt) | 82 | if (!sgt) { |
81 | return -ENOMEM; | 83 | return -ENOMEM; |
84 | } | ||
82 | 85 | ||
83 | /* | 86 | /* |
84 | * If the GPU is IO coherent and the DMA API is giving us IO coherent | 87 | * If the GPU is IO coherent and the DMA API is giving us IO coherent |
85 | * CPU mappings then we gotta make sure we use the IO coherent aperture. | 88 | * CPU mappings then we gotta make sure we use the IO coherent aperture. |
86 | */ | 89 | */ |
87 | if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) | 90 | if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) { |
88 | flags |= NVGPU_VM_MAP_IO_COHERENT; | 91 | flags |= NVGPU_VM_MAP_IO_COHERENT; |
92 | } | ||
89 | 93 | ||
90 | /* | 94 | /* |
91 | * Later on, when we free this nvgpu_mem's GPU mapping, we are going to | 95 | * Later on, when we free this nvgpu_mem's GPU mapping, we are going to |
@@ -94,10 +98,11 @@ static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm, | |||
94 | * therefor we should not try and free it. But otherwise, if we do | 98 | * therefor we should not try and free it. But otherwise, if we do |
95 | * manage the VA alloc, we obviously must free it. | 99 | * manage the VA alloc, we obviously must free it. |
96 | */ | 100 | */ |
97 | if (addr != 0) | 101 | if (addr != 0) { |
98 | mem->free_gpu_va = false; | 102 | mem->free_gpu_va = false; |
99 | else | 103 | } else { |
100 | mem->free_gpu_va = true; | 104 | mem->free_gpu_va = true; |
105 | } | ||
101 | 106 | ||
102 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | 107 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); |
103 | vaddr = g->ops.mm.gmmu_map(vm, addr, | 108 | vaddr = g->ops.mm.gmmu_map(vm, addr, |
@@ -196,8 +201,9 @@ int nvgpu_gmmu_init_page_table(struct vm_gk20a *vm) | |||
196 | pdb_size = ALIGN(pd_size(&vm->mmu_levels[0], &attrs), PAGE_SIZE); | 201 | pdb_size = ALIGN(pd_size(&vm->mmu_levels[0], &attrs), PAGE_SIZE); |
197 | 202 | ||
198 | err = __nvgpu_pd_cache_alloc_direct(vm->mm->g, &vm->pdb, pdb_size); | 203 | err = __nvgpu_pd_cache_alloc_direct(vm->mm->g, &vm->pdb, pdb_size); |
199 | if (WARN_ON(err)) | 204 | if (WARN_ON(err)) { |
200 | return err; | 205 | return err; |
206 | } | ||
201 | 207 | ||
202 | /* | 208 | /* |
203 | * One nvgpu_mb() is done after all mapping operations. Don't need | 209 | * One nvgpu_mb() is done after all mapping operations. Don't need |
@@ -267,8 +273,9 @@ static int pd_allocate(struct vm_gk20a *vm, | |||
267 | { | 273 | { |
268 | int err; | 274 | int err; |
269 | 275 | ||
270 | if (pd->mem) | 276 | if (pd->mem) { |
271 | return 0; | 277 | return 0; |
278 | } | ||
272 | 279 | ||
273 | err = __nvgpu_pd_alloc(vm, pd, pd_size(l, attrs)); | 280 | err = __nvgpu_pd_alloc(vm, pd, pd_size(l, attrs)); |
274 | if (err) { | 281 | if (err) { |
@@ -310,14 +317,16 @@ static int pd_allocate_children(struct vm_gk20a *vm, | |||
310 | { | 317 | { |
311 | struct gk20a *g = gk20a_from_vm(vm); | 318 | struct gk20a *g = gk20a_from_vm(vm); |
312 | 319 | ||
313 | if (pd->entries) | 320 | if (pd->entries) { |
314 | return 0; | 321 | return 0; |
322 | } | ||
315 | 323 | ||
316 | pd->num_entries = pd_entries(l, attrs); | 324 | pd->num_entries = pd_entries(l, attrs); |
317 | pd->entries = nvgpu_vzalloc(g, sizeof(struct nvgpu_gmmu_pd) * | 325 | pd->entries = nvgpu_vzalloc(g, sizeof(struct nvgpu_gmmu_pd) * |
318 | pd->num_entries); | 326 | pd->num_entries); |
319 | if (!pd->entries) | 327 | if (!pd->entries) { |
320 | return -ENOMEM; | 328 | return -ENOMEM; |
329 | } | ||
321 | 330 | ||
322 | return 0; | 331 | return 0; |
323 | } | 332 | } |
@@ -398,8 +407,9 @@ static int __set_pd_level(struct vm_gk20a *vm, | |||
398 | * have a bunch of children PDs. | 407 | * have a bunch of children PDs. |
399 | */ | 408 | */ |
400 | if (next_l->update_entry) { | 409 | if (next_l->update_entry) { |
401 | if (pd_allocate_children(vm, l, pd, attrs)) | 410 | if (pd_allocate_children(vm, l, pd, attrs)) { |
402 | return -ENOMEM; | 411 | return -ENOMEM; |
412 | } | ||
403 | 413 | ||
404 | /* | 414 | /* |
405 | * Get the next PD so that we know what to put in this | 415 | * Get the next PD so that we know what to put in this |
@@ -412,8 +422,9 @@ static int __set_pd_level(struct vm_gk20a *vm, | |||
412 | /* | 422 | /* |
413 | * Allocate the backing memory for next_pd. | 423 | * Allocate the backing memory for next_pd. |
414 | */ | 424 | */ |
415 | if (pd_allocate(vm, next_pd, next_l, attrs)) | 425 | if (pd_allocate(vm, next_pd, next_l, attrs)) { |
416 | return -ENOMEM; | 426 | return -ENOMEM; |
427 | } | ||
417 | } | 428 | } |
418 | 429 | ||
419 | /* | 430 | /* |
@@ -440,8 +451,9 @@ static int __set_pd_level(struct vm_gk20a *vm, | |||
440 | chunk_size, | 451 | chunk_size, |
441 | attrs); | 452 | attrs); |
442 | 453 | ||
443 | if (err) | 454 | if (err) { |
444 | return err; | 455 | return err; |
456 | } | ||
445 | } | 457 | } |
446 | 458 | ||
447 | virt_addr += chunk_size; | 459 | virt_addr += chunk_size; |
@@ -452,8 +464,9 @@ static int __set_pd_level(struct vm_gk20a *vm, | |||
452 | * non-zero phys addresses in the PTEs. A non-zero phys-addr | 464 | * non-zero phys addresses in the PTEs. A non-zero phys-addr |
453 | * would also confuse the lower level PTE programming code. | 465 | * would also confuse the lower level PTE programming code. |
454 | */ | 466 | */ |
455 | if (phys_addr) | 467 | if (phys_addr) { |
456 | phys_addr += chunk_size; | 468 | phys_addr += chunk_size; |
469 | } | ||
457 | length -= chunk_size; | 470 | length -= chunk_size; |
458 | } | 471 | } |
459 | 472 | ||
@@ -547,8 +560,9 @@ static int __nvgpu_gmmu_do_update_page_table(struct vm_gk20a *vm, | |||
547 | virt_addr, | 560 | virt_addr, |
548 | chunk_length, | 561 | chunk_length, |
549 | attrs); | 562 | attrs); |
550 | if (err) | 563 | if (err) { |
551 | break; | 564 | break; |
565 | } | ||
552 | 566 | ||
553 | /* Space has been skipped so zero this for future chunks. */ | 567 | /* Space has been skipped so zero this for future chunks. */ |
554 | space_to_skip = 0; | 568 | space_to_skip = 0; |
@@ -559,8 +573,9 @@ static int __nvgpu_gmmu_do_update_page_table(struct vm_gk20a *vm, | |||
559 | virt_addr += chunk_length; | 573 | virt_addr += chunk_length; |
560 | length -= chunk_length; | 574 | length -= chunk_length; |
561 | 575 | ||
562 | if (length == 0) | 576 | if (length == 0) { |
563 | break; | 577 | break; |
578 | } | ||
564 | } | 579 | } |
565 | 580 | ||
566 | return err; | 581 | return err; |
@@ -594,13 +609,15 @@ static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm, | |||
594 | 609 | ||
595 | /* note: here we need to map kernel to small, since the | 610 | /* note: here we need to map kernel to small, since the |
596 | * low-level mmu code assumes 0 is small and 1 is big pages */ | 611 | * low-level mmu code assumes 0 is small and 1 is big pages */ |
597 | if (attrs->pgsz == gmmu_page_size_kernel) | 612 | if (attrs->pgsz == gmmu_page_size_kernel) { |
598 | attrs->pgsz = gmmu_page_size_small; | 613 | attrs->pgsz = gmmu_page_size_small; |
614 | } | ||
599 | 615 | ||
600 | page_size = vm->gmmu_page_sizes[attrs->pgsz]; | 616 | page_size = vm->gmmu_page_sizes[attrs->pgsz]; |
601 | 617 | ||
602 | if (space_to_skip & (page_size - 1)) | 618 | if (space_to_skip & (page_size - 1)) { |
603 | return -EINVAL; | 619 | return -EINVAL; |
620 | } | ||
604 | 621 | ||
605 | /* | 622 | /* |
606 | * Update length to be aligned to the passed page size. | 623 | * Update length to be aligned to the passed page size. |
@@ -692,8 +709,9 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm, | |||
692 | * the programmed ctagline gets increased at compression_page_size | 709 | * the programmed ctagline gets increased at compression_page_size |
693 | * boundaries. | 710 | * boundaries. |
694 | */ | 711 | */ |
695 | if (attrs.ctag) | 712 | if (attrs.ctag) { |
696 | attrs.ctag += buffer_offset & (ctag_granularity - 1U); | 713 | attrs.ctag += buffer_offset & (ctag_granularity - 1U); |
714 | } | ||
697 | 715 | ||
698 | attrs.l3_alloc = (bool)(flags & NVGPU_VM_MAP_L3_ALLOC); | 716 | attrs.l3_alloc = (bool)(flags & NVGPU_VM_MAP_L3_ALLOC); |
699 | 717 | ||
@@ -701,8 +719,9 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm, | |||
701 | * Handle the IO coherency aperture: make sure the .aperture field is | 719 | * Handle the IO coherency aperture: make sure the .aperture field is |
702 | * correct based on the IO coherency flag. | 720 | * correct based on the IO coherency flag. |
703 | */ | 721 | */ |
704 | if (attrs.coherent && attrs.aperture == APERTURE_SYSMEM) | 722 | if (attrs.coherent && attrs.aperture == APERTURE_SYSMEM) { |
705 | attrs.aperture = __APERTURE_SYSMEM_COH; | 723 | attrs.aperture = __APERTURE_SYSMEM_COH; |
724 | } | ||
706 | 725 | ||
707 | /* | 726 | /* |
708 | * Only allocate a new GPU VA range if we haven't already been passed a | 727 | * Only allocate a new GPU VA range if we haven't already been passed a |
@@ -725,16 +744,18 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm, | |||
725 | goto fail_validate; | 744 | goto fail_validate; |
726 | } | 745 | } |
727 | 746 | ||
728 | if (!batch) | 747 | if (!batch) { |
729 | g->ops.fb.tlb_invalidate(g, vm->pdb.mem); | 748 | g->ops.fb.tlb_invalidate(g, vm->pdb.mem); |
730 | else | 749 | } else { |
731 | batch->need_tlb_invalidate = true; | 750 | batch->need_tlb_invalidate = true; |
751 | } | ||
732 | 752 | ||
733 | return vaddr; | 753 | return vaddr; |
734 | 754 | ||
735 | fail_validate: | 755 | fail_validate: |
736 | if (allocated) | 756 | if (allocated) { |
737 | __nvgpu_vm_free_va(vm, vaddr, pgsz_idx); | 757 | __nvgpu_vm_free_va(vm, vaddr, pgsz_idx); |
758 | } | ||
738 | fail_alloc: | 759 | fail_alloc: |
739 | nvgpu_err(g, "%s: failed with err=%d", __func__, err); | 760 | nvgpu_err(g, "%s: failed with err=%d", __func__, err); |
740 | return 0; | 761 | return 0; |
@@ -775,8 +796,9 @@ void gk20a_locked_gmmu_unmap(struct vm_gk20a *vm, | |||
775 | /* unmap here needs to know the page size we assigned at mapping */ | 796 | /* unmap here needs to know the page size we assigned at mapping */ |
776 | err = __nvgpu_gmmu_update_page_table(vm, NULL, 0, | 797 | err = __nvgpu_gmmu_update_page_table(vm, NULL, 0, |
777 | vaddr, size, &attrs); | 798 | vaddr, size, &attrs); |
778 | if (err) | 799 | if (err) { |
779 | nvgpu_err(g, "failed to update gmmu ptes on unmap"); | 800 | nvgpu_err(g, "failed to update gmmu ptes on unmap"); |
801 | } | ||
780 | 802 | ||
781 | if (!batch) { | 803 | if (!batch) { |
782 | gk20a_mm_l2_flush(g, true); | 804 | gk20a_mm_l2_flush(g, true); |
@@ -801,8 +823,9 @@ u32 __nvgpu_pte_words(struct gk20a *g) | |||
801 | */ | 823 | */ |
802 | do { | 824 | do { |
803 | next_l = l + 1; | 825 | next_l = l + 1; |
804 | if (!next_l->update_entry) | 826 | if (!next_l->update_entry) { |
805 | break; | 827 | break; |
828 | } | ||
806 | 829 | ||
807 | l++; | 830 | l++; |
808 | } while (true); | 831 | } while (true); |
@@ -836,13 +859,15 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm, | |||
836 | struct nvgpu_gmmu_pd *pd_next = pd->entries + pd_idx; | 859 | struct nvgpu_gmmu_pd *pd_next = pd->entries + pd_idx; |
837 | 860 | ||
838 | /* Invalid entry! */ | 861 | /* Invalid entry! */ |
839 | if (!pd_next->mem) | 862 | if (!pd_next->mem) { |
840 | return -EINVAL; | 863 | return -EINVAL; |
864 | } | ||
841 | 865 | ||
842 | attrs->pgsz = l->get_pgsz(g, l, pd, pd_idx); | 866 | attrs->pgsz = l->get_pgsz(g, l, pd, pd_idx); |
843 | 867 | ||
844 | if (attrs->pgsz >= gmmu_nr_page_sizes) | 868 | if (attrs->pgsz >= gmmu_nr_page_sizes) { |
845 | return -EINVAL; | 869 | return -EINVAL; |
870 | } | ||
846 | 871 | ||
847 | return __nvgpu_locate_pte(g, vm, pd_next, | 872 | return __nvgpu_locate_pte(g, vm, pd_next, |
848 | vaddr, lvl + 1, attrs, | 873 | vaddr, lvl + 1, attrs, |
@@ -850,8 +875,9 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm, | |||
850 | pd_offs_out); | 875 | pd_offs_out); |
851 | } | 876 | } |
852 | 877 | ||
853 | if (!pd->mem) | 878 | if (!pd->mem) { |
854 | return -EINVAL; | 879 | return -EINVAL; |
880 | } | ||
855 | 881 | ||
856 | /* | 882 | /* |
857 | * Take into account the real offset into the nvgpu_mem since the PD | 883 | * Take into account the real offset into the nvgpu_mem since the PD |
@@ -867,14 +893,17 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm, | |||
867 | } | 893 | } |
868 | } | 894 | } |
869 | 895 | ||
870 | if (pd_out) | 896 | if (pd_out) { |
871 | *pd_out = pd; | 897 | *pd_out = pd; |
898 | } | ||
872 | 899 | ||
873 | if (pd_idx_out) | 900 | if (pd_idx_out) { |
874 | *pd_idx_out = pd_idx; | 901 | *pd_idx_out = pd_idx; |
902 | } | ||
875 | 903 | ||
876 | if (pd_offs_out) | 904 | if (pd_offs_out) { |
877 | *pd_offs_out = pd_offset_from_index(l, pd_idx); | 905 | *pd_offs_out = pd_offset_from_index(l, pd_idx); |
906 | } | ||
878 | 907 | ||
879 | return 0; | 908 | return 0; |
880 | } | 909 | } |
@@ -903,8 +932,9 @@ int __nvgpu_set_pte(struct gk20a *g, struct vm_gk20a *vm, u64 vaddr, u32 *pte) | |||
903 | err = __nvgpu_locate_pte(g, vm, &vm->pdb, | 932 | err = __nvgpu_locate_pte(g, vm, &vm->pdb, |
904 | vaddr, 0, &attrs, | 933 | vaddr, 0, &attrs, |
905 | NULL, &pd, &pd_idx, &pd_offs); | 934 | NULL, &pd, &pd_idx, &pd_offs); |
906 | if (err) | 935 | if (err) { |
907 | return err; | 936 | return err; |
937 | } | ||
908 | 938 | ||
909 | pte_size = __nvgpu_pte_words(g); | 939 | pte_size = __nvgpu_pte_words(g); |
910 | 940 | ||
diff --git a/drivers/gpu/nvgpu/common/mm/lockless_allocator.c b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c index 4a3d6c9d..7a5f67e9 100644 --- a/drivers/gpu/nvgpu/common/mm/lockless_allocator.c +++ b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c | |||
@@ -63,8 +63,9 @@ static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len) | |||
63 | int head, new_head, ret; | 63 | int head, new_head, ret; |
64 | u64 addr = 0; | 64 | u64 addr = 0; |
65 | 65 | ||
66 | if (len != pa->blk_size) | 66 | if (len != pa->blk_size) { |
67 | return 0; | 67 | return 0; |
68 | } | ||
68 | 69 | ||
69 | head = NV_ACCESS_ONCE(pa->head); | 70 | head = NV_ACCESS_ONCE(pa->head); |
70 | while (head >= 0) { | 71 | while (head >= 0) { |
@@ -80,10 +81,11 @@ static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len) | |||
80 | head = NV_ACCESS_ONCE(pa->head); | 81 | head = NV_ACCESS_ONCE(pa->head); |
81 | } | 82 | } |
82 | 83 | ||
83 | if (addr) | 84 | if (addr) { |
84 | alloc_dbg(a, "Alloc node # %d @ addr 0x%llx", head, addr); | 85 | alloc_dbg(a, "Alloc node # %d @ addr 0x%llx", head, addr); |
85 | else | 86 | } else { |
86 | alloc_dbg(a, "Alloc failed!"); | 87 | alloc_dbg(a, "Alloc failed!"); |
88 | } | ||
87 | 89 | ||
88 | return addr; | 90 | return addr; |
89 | } | 91 | } |
@@ -167,24 +169,28 @@ int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
167 | u64 count; | 169 | u64 count; |
168 | struct nvgpu_lockless_allocator *a; | 170 | struct nvgpu_lockless_allocator *a; |
169 | 171 | ||
170 | if (!blk_size) | 172 | if (!blk_size) { |
171 | return -EINVAL; | 173 | return -EINVAL; |
174 | } | ||
172 | 175 | ||
173 | /* | 176 | /* |
174 | * Ensure we have space for at least one node & there's no overflow. | 177 | * Ensure we have space for at least one node & there's no overflow. |
175 | * In order to control memory footprint, we require count < INT_MAX | 178 | * In order to control memory footprint, we require count < INT_MAX |
176 | */ | 179 | */ |
177 | count = length / blk_size; | 180 | count = length / blk_size; |
178 | if (!base || !count || count > INT_MAX) | 181 | if (!base || !count || count > INT_MAX) { |
179 | return -EINVAL; | 182 | return -EINVAL; |
183 | } | ||
180 | 184 | ||
181 | a = nvgpu_kzalloc(g, sizeof(struct nvgpu_lockless_allocator)); | 185 | a = nvgpu_kzalloc(g, sizeof(struct nvgpu_lockless_allocator)); |
182 | if (!a) | 186 | if (!a) { |
183 | return -ENOMEM; | 187 | return -ENOMEM; |
188 | } | ||
184 | 189 | ||
185 | err = __nvgpu_alloc_common_init(__a, g, name, a, false, &pool_ops); | 190 | err = __nvgpu_alloc_common_init(__a, g, name, a, false, &pool_ops); |
186 | if (err) | 191 | if (err) { |
187 | goto fail; | 192 | goto fail; |
193 | } | ||
188 | 194 | ||
189 | a->next = nvgpu_vzalloc(g, sizeof(*a->next) * count); | 195 | a->next = nvgpu_vzalloc(g, sizeof(*a->next) * count); |
190 | if (!a->next) { | 196 | if (!a->next) { |
diff --git a/drivers/gpu/nvgpu/common/mm/mm.c b/drivers/gpu/nvgpu/common/mm/mm.c index 2c3a1cd6..0608d66a 100644 --- a/drivers/gpu/nvgpu/common/mm/mm.c +++ b/drivers/gpu/nvgpu/common/mm/mm.c | |||
@@ -40,8 +40,9 @@ enum gmmu_pgsz_gk20a __get_pte_size_fixed_map(struct vm_gk20a *vm, | |||
40 | struct nvgpu_vm_area *vm_area; | 40 | struct nvgpu_vm_area *vm_area; |
41 | 41 | ||
42 | vm_area = nvgpu_vm_area_find(vm, base); | 42 | vm_area = nvgpu_vm_area_find(vm, base); |
43 | if (!vm_area) | 43 | if (!vm_area) { |
44 | return gmmu_page_size_small; | 44 | return gmmu_page_size_small; |
45 | } | ||
45 | 46 | ||
46 | return vm_area->pgsz_idx; | 47 | return vm_area->pgsz_idx; |
47 | } | 48 | } |
@@ -53,14 +54,16 @@ static enum gmmu_pgsz_gk20a __get_pte_size_split_addr(struct vm_gk20a *vm, | |||
53 | u64 base, u64 size) | 54 | u64 base, u64 size) |
54 | { | 55 | { |
55 | if (!base) { | 56 | if (!base) { |
56 | if (size >= vm->gmmu_page_sizes[gmmu_page_size_big]) | 57 | if (size >= vm->gmmu_page_sizes[gmmu_page_size_big]) { |
57 | return gmmu_page_size_big; | 58 | return gmmu_page_size_big; |
59 | } | ||
58 | return gmmu_page_size_small; | 60 | return gmmu_page_size_small; |
59 | } else { | 61 | } else { |
60 | if (base < __nv_gmmu_va_small_page_limit()) | 62 | if (base < __nv_gmmu_va_small_page_limit()) { |
61 | return gmmu_page_size_small; | 63 | return gmmu_page_size_small; |
62 | else | 64 | } else { |
63 | return gmmu_page_size_big; | 65 | return gmmu_page_size_big; |
66 | } | ||
64 | } | 67 | } |
65 | } | 68 | } |
66 | 69 | ||
@@ -89,18 +92,22 @@ enum gmmu_pgsz_gk20a __get_pte_size(struct vm_gk20a *vm, u64 base, u64 size) | |||
89 | { | 92 | { |
90 | struct gk20a *g = gk20a_from_vm(vm); | 93 | struct gk20a *g = gk20a_from_vm(vm); |
91 | 94 | ||
92 | if (!vm->big_pages) | 95 | if (!vm->big_pages) { |
93 | return gmmu_page_size_small; | 96 | return gmmu_page_size_small; |
97 | } | ||
94 | 98 | ||
95 | if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) | 99 | if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) { |
96 | return __get_pte_size_split_addr(vm, base, size); | 100 | return __get_pte_size_split_addr(vm, base, size); |
101 | } | ||
97 | 102 | ||
98 | if (base) | 103 | if (base) { |
99 | return __get_pte_size_fixed_map(vm, base, size); | 104 | return __get_pte_size_fixed_map(vm, base, size); |
105 | } | ||
100 | 106 | ||
101 | if (size >= vm->gmmu_page_sizes[gmmu_page_size_big] && | 107 | if (size >= vm->gmmu_page_sizes[gmmu_page_size_big] && |
102 | nvgpu_iommuable(g)) | 108 | nvgpu_iommuable(g)) { |
103 | return gmmu_page_size_big; | 109 | return gmmu_page_size_big; |
110 | } | ||
104 | return gmmu_page_size_small; | 111 | return gmmu_page_size_small; |
105 | } | 112 | } |
106 | 113 | ||
@@ -137,8 +144,9 @@ u64 nvgpu_inst_block_addr(struct gk20a *g, struct nvgpu_mem *inst_block) | |||
137 | 144 | ||
138 | void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block) | 145 | void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block) |
139 | { | 146 | { |
140 | if (nvgpu_mem_is_valid(inst_block)) | 147 | if (nvgpu_mem_is_valid(inst_block)) { |
141 | nvgpu_dma_free(g, inst_block); | 148 | nvgpu_dma_free(g, inst_block); |
149 | } | ||
142 | } | 150 | } |
143 | 151 | ||
144 | static int nvgpu_alloc_sysmem_flush(struct gk20a *g) | 152 | static int nvgpu_alloc_sysmem_flush(struct gk20a *g) |
@@ -150,8 +158,9 @@ static void nvgpu_remove_mm_ce_support(struct mm_gk20a *mm) | |||
150 | { | 158 | { |
151 | struct gk20a *g = gk20a_from_mm(mm); | 159 | struct gk20a *g = gk20a_from_mm(mm); |
152 | 160 | ||
153 | if (mm->vidmem.ce_ctx_id != (u32)~0) | 161 | if (mm->vidmem.ce_ctx_id != (u32)~0) { |
154 | gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id); | 162 | gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id); |
163 | } | ||
155 | 164 | ||
156 | mm->vidmem.ce_ctx_id = (u32)~0; | 165 | mm->vidmem.ce_ctx_id = (u32)~0; |
157 | 166 | ||
@@ -162,11 +171,13 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm) | |||
162 | { | 171 | { |
163 | struct gk20a *g = gk20a_from_mm(mm); | 172 | struct gk20a *g = gk20a_from_mm(mm); |
164 | 173 | ||
165 | if (g->ops.mm.fault_info_mem_destroy) | 174 | if (g->ops.mm.fault_info_mem_destroy) { |
166 | g->ops.mm.fault_info_mem_destroy(g); | 175 | g->ops.mm.fault_info_mem_destroy(g); |
176 | } | ||
167 | 177 | ||
168 | if (g->ops.mm.remove_bar2_vm) | 178 | if (g->ops.mm.remove_bar2_vm) { |
169 | g->ops.mm.remove_bar2_vm(g); | 179 | g->ops.mm.remove_bar2_vm(g); |
180 | } | ||
170 | 181 | ||
171 | nvgpu_free_inst_block(g, &mm->bar1.inst_block); | 182 | nvgpu_free_inst_block(g, &mm->bar1.inst_block); |
172 | nvgpu_vm_put(mm->bar1.vm); | 183 | nvgpu_vm_put(mm->bar1.vm); |
@@ -175,8 +186,9 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm) | |||
175 | nvgpu_free_inst_block(g, &mm->hwpm.inst_block); | 186 | nvgpu_free_inst_block(g, &mm->hwpm.inst_block); |
176 | nvgpu_vm_put(mm->pmu.vm); | 187 | nvgpu_vm_put(mm->pmu.vm); |
177 | 188 | ||
178 | if (g->has_cde) | 189 | if (g->has_cde) { |
179 | nvgpu_vm_put(mm->cde.vm); | 190 | nvgpu_vm_put(mm->cde.vm); |
191 | } | ||
180 | 192 | ||
181 | nvgpu_semaphore_sea_destroy(g); | 193 | nvgpu_semaphore_sea_destroy(g); |
182 | nvgpu_vidmem_destroy(g); | 194 | nvgpu_vidmem_destroy(g); |
@@ -208,12 +220,14 @@ static int nvgpu_init_system_vm(struct mm_gk20a *mm) | |||
208 | true, | 220 | true, |
209 | false, | 221 | false, |
210 | "system"); | 222 | "system"); |
211 | if (!mm->pmu.vm) | 223 | if (!mm->pmu.vm) { |
212 | return -ENOMEM; | 224 | return -ENOMEM; |
225 | } | ||
213 | 226 | ||
214 | err = g->ops.mm.alloc_inst_block(g, inst_block); | 227 | err = g->ops.mm.alloc_inst_block(g, inst_block); |
215 | if (err) | 228 | if (err) { |
216 | goto clean_up_vm; | 229 | goto clean_up_vm; |
230 | } | ||
217 | g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size); | 231 | g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size); |
218 | 232 | ||
219 | return 0; | 233 | return 0; |
@@ -230,8 +244,9 @@ static int nvgpu_init_hwpm(struct mm_gk20a *mm) | |||
230 | struct nvgpu_mem *inst_block = &mm->hwpm.inst_block; | 244 | struct nvgpu_mem *inst_block = &mm->hwpm.inst_block; |
231 | 245 | ||
232 | err = g->ops.mm.alloc_inst_block(g, inst_block); | 246 | err = g->ops.mm.alloc_inst_block(g, inst_block); |
233 | if (err) | 247 | if (err) { |
234 | return err; | 248 | return err; |
249 | } | ||
235 | g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0); | 250 | g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0); |
236 | 251 | ||
237 | return 0; | 252 | return 0; |
@@ -247,8 +262,9 @@ static int nvgpu_init_cde_vm(struct mm_gk20a *mm) | |||
247 | NV_MM_DEFAULT_KERNEL_SIZE, | 262 | NV_MM_DEFAULT_KERNEL_SIZE, |
248 | NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, | 263 | NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, |
249 | false, false, "cde"); | 264 | false, false, "cde"); |
250 | if (!mm->cde.vm) | 265 | if (!mm->cde.vm) { |
251 | return -ENOMEM; | 266 | return -ENOMEM; |
267 | } | ||
252 | return 0; | 268 | return 0; |
253 | } | 269 | } |
254 | 270 | ||
@@ -262,8 +278,9 @@ static int nvgpu_init_ce_vm(struct mm_gk20a *mm) | |||
262 | NV_MM_DEFAULT_KERNEL_SIZE, | 278 | NV_MM_DEFAULT_KERNEL_SIZE, |
263 | NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, | 279 | NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, |
264 | false, false, "ce"); | 280 | false, false, "ce"); |
265 | if (!mm->ce.vm) | 281 | if (!mm->ce.vm) { |
266 | return -ENOMEM; | 282 | return -ENOMEM; |
283 | } | ||
267 | return 0; | 284 | return 0; |
268 | } | 285 | } |
269 | 286 | ||
@@ -286,24 +303,30 @@ void nvgpu_init_mm_ce_context(struct gk20a *g) | |||
286 | 303 | ||
287 | static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g) | 304 | static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g) |
288 | { | 305 | { |
289 | if (g->ops.fb.reset) | 306 | if (g->ops.fb.reset) { |
290 | g->ops.fb.reset(g); | 307 | g->ops.fb.reset(g); |
308 | } | ||
291 | 309 | ||
292 | if (g->ops.clock_gating.slcg_fb_load_gating_prod) | 310 | if (g->ops.clock_gating.slcg_fb_load_gating_prod) { |
293 | g->ops.clock_gating.slcg_fb_load_gating_prod(g, | 311 | g->ops.clock_gating.slcg_fb_load_gating_prod(g, |
294 | g->slcg_enabled); | 312 | g->slcg_enabled); |
295 | if (g->ops.clock_gating.slcg_ltc_load_gating_prod) | 313 | } |
314 | if (g->ops.clock_gating.slcg_ltc_load_gating_prod) { | ||
296 | g->ops.clock_gating.slcg_ltc_load_gating_prod(g, | 315 | g->ops.clock_gating.slcg_ltc_load_gating_prod(g, |
297 | g->slcg_enabled); | 316 | g->slcg_enabled); |
298 | if (g->ops.clock_gating.blcg_fb_load_gating_prod) | 317 | } |
318 | if (g->ops.clock_gating.blcg_fb_load_gating_prod) { | ||
299 | g->ops.clock_gating.blcg_fb_load_gating_prod(g, | 319 | g->ops.clock_gating.blcg_fb_load_gating_prod(g, |
300 | g->blcg_enabled); | 320 | g->blcg_enabled); |
301 | if (g->ops.clock_gating.blcg_ltc_load_gating_prod) | 321 | } |
322 | if (g->ops.clock_gating.blcg_ltc_load_gating_prod) { | ||
302 | g->ops.clock_gating.blcg_ltc_load_gating_prod(g, | 323 | g->ops.clock_gating.blcg_ltc_load_gating_prod(g, |
303 | g->blcg_enabled); | 324 | g->blcg_enabled); |
325 | } | ||
304 | 326 | ||
305 | if (g->ops.fb.init_fs_state) | 327 | if (g->ops.fb.init_fs_state) { |
306 | g->ops.fb.init_fs_state(g); | 328 | g->ops.fb.init_fs_state(g); |
329 | } | ||
307 | 330 | ||
308 | return 0; | 331 | return 0; |
309 | } | 332 | } |
@@ -324,12 +347,14 @@ static int nvgpu_init_bar1_vm(struct mm_gk20a *mm) | |||
324 | mm->bar1.aperture_size, | 347 | mm->bar1.aperture_size, |
325 | true, false, | 348 | true, false, |
326 | "bar1"); | 349 | "bar1"); |
327 | if (!mm->bar1.vm) | 350 | if (!mm->bar1.vm) { |
328 | return -ENOMEM; | 351 | return -ENOMEM; |
352 | } | ||
329 | 353 | ||
330 | err = g->ops.mm.alloc_inst_block(g, inst_block); | 354 | err = g->ops.mm.alloc_inst_block(g, inst_block); |
331 | if (err) | 355 | if (err) { |
332 | goto clean_up_vm; | 356 | goto clean_up_vm; |
357 | } | ||
333 | g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size); | 358 | g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size); |
334 | 359 | ||
335 | return 0; | 360 | return 0; |
@@ -366,8 +391,9 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g) | |||
366 | mm->vidmem.ce_ctx_id = (u32)~0; | 391 | mm->vidmem.ce_ctx_id = (u32)~0; |
367 | 392 | ||
368 | err = nvgpu_vidmem_init(mm); | 393 | err = nvgpu_vidmem_init(mm); |
369 | if (err) | 394 | if (err) { |
370 | return err; | 395 | return err; |
396 | } | ||
371 | 397 | ||
372 | /* | 398 | /* |
373 | * this requires fixed allocations in vidmem which must be | 399 | * this requires fixed allocations in vidmem which must be |
@@ -376,40 +402,48 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g) | |||
376 | if (g->ops.pmu.alloc_blob_space | 402 | if (g->ops.pmu.alloc_blob_space |
377 | && !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) { | 403 | && !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) { |
378 | err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob); | 404 | err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob); |
379 | if (err) | 405 | if (err) { |
380 | return err; | 406 | return err; |
407 | } | ||
381 | } | 408 | } |
382 | 409 | ||
383 | err = nvgpu_alloc_sysmem_flush(g); | 410 | err = nvgpu_alloc_sysmem_flush(g); |
384 | if (err) | 411 | if (err) { |
385 | return err; | 412 | return err; |
413 | } | ||
386 | 414 | ||
387 | err = nvgpu_init_bar1_vm(mm); | 415 | err = nvgpu_init_bar1_vm(mm); |
388 | if (err) | 416 | if (err) { |
389 | return err; | 417 | return err; |
418 | } | ||
390 | 419 | ||
391 | if (g->ops.mm.init_bar2_vm) { | 420 | if (g->ops.mm.init_bar2_vm) { |
392 | err = g->ops.mm.init_bar2_vm(g); | 421 | err = g->ops.mm.init_bar2_vm(g); |
393 | if (err) | 422 | if (err) { |
394 | return err; | 423 | return err; |
424 | } | ||
395 | } | 425 | } |
396 | err = nvgpu_init_system_vm(mm); | 426 | err = nvgpu_init_system_vm(mm); |
397 | if (err) | 427 | if (err) { |
398 | return err; | 428 | return err; |
429 | } | ||
399 | 430 | ||
400 | err = nvgpu_init_hwpm(mm); | 431 | err = nvgpu_init_hwpm(mm); |
401 | if (err) | 432 | if (err) { |
402 | return err; | 433 | return err; |
434 | } | ||
403 | 435 | ||
404 | if (g->has_cde) { | 436 | if (g->has_cde) { |
405 | err = nvgpu_init_cde_vm(mm); | 437 | err = nvgpu_init_cde_vm(mm); |
406 | if (err) | 438 | if (err) { |
407 | return err; | 439 | return err; |
440 | } | ||
408 | } | 441 | } |
409 | 442 | ||
410 | err = nvgpu_init_ce_vm(mm); | 443 | err = nvgpu_init_ce_vm(mm); |
411 | if (err) | 444 | if (err) { |
412 | return err; | 445 | return err; |
446 | } | ||
413 | 447 | ||
414 | mm->remove_support = nvgpu_remove_mm_support; | 448 | mm->remove_support = nvgpu_remove_mm_support; |
415 | mm->remove_ce_support = nvgpu_remove_mm_ce_support; | 449 | mm->remove_ce_support = nvgpu_remove_mm_ce_support; |
@@ -424,15 +458,18 @@ int nvgpu_init_mm_support(struct gk20a *g) | |||
424 | u32 err; | 458 | u32 err; |
425 | 459 | ||
426 | err = nvgpu_init_mm_reset_enable_hw(g); | 460 | err = nvgpu_init_mm_reset_enable_hw(g); |
427 | if (err) | 461 | if (err) { |
428 | return err; | 462 | return err; |
463 | } | ||
429 | 464 | ||
430 | err = nvgpu_init_mm_setup_sw(g); | 465 | err = nvgpu_init_mm_setup_sw(g); |
431 | if (err) | 466 | if (err) { |
432 | return err; | 467 | return err; |
468 | } | ||
433 | 469 | ||
434 | if (g->ops.mm.init_mm_setup_hw) | 470 | if (g->ops.mm.init_mm_setup_hw) { |
435 | err = g->ops.mm.init_mm_setup_hw(g); | 471 | err = g->ops.mm.init_mm_setup_hw(g); |
472 | } | ||
436 | 473 | ||
437 | return err; | 474 | return err; |
438 | } | 475 | } |
@@ -443,8 +480,9 @@ u32 nvgpu_mm_get_default_big_page_size(struct gk20a *g) | |||
443 | 480 | ||
444 | big_page_size = g->ops.mm.get_default_big_page_size(); | 481 | big_page_size = g->ops.mm.get_default_big_page_size(); |
445 | 482 | ||
446 | if (g->mm.disable_bigpage) | 483 | if (g->mm.disable_bigpage) { |
447 | big_page_size = 0; | 484 | big_page_size = 0; |
485 | } | ||
448 | 486 | ||
449 | return big_page_size; | 487 | return big_page_size; |
450 | } | 488 | } |
@@ -456,8 +494,9 @@ u32 nvgpu_mm_get_available_big_page_sizes(struct gk20a *g) | |||
456 | if (!g->mm.disable_bigpage) { | 494 | if (!g->mm.disable_bigpage) { |
457 | available_big_page_sizes = | 495 | available_big_page_sizes = |
458 | g->ops.mm.get_default_big_page_size(); | 496 | g->ops.mm.get_default_big_page_size(); |
459 | if (g->ops.mm.get_big_page_sizes) | 497 | if (g->ops.mm.get_big_page_sizes) { |
460 | available_big_page_sizes |= g->ops.mm.get_big_page_sizes(); | 498 | available_big_page_sizes |= g->ops.mm.get_big_page_sizes(); |
499 | } | ||
461 | } | 500 | } |
462 | 501 | ||
463 | return available_big_page_sizes; | 502 | return available_big_page_sizes; |
diff --git a/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c index 8ba119a6..345b947d 100644 --- a/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c +++ b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c | |||
@@ -40,8 +40,9 @@ u32 __nvgpu_aperture_mask(struct gk20a *g, enum nvgpu_aperture aperture, | |||
40 | * Some iGPUs treat sysmem (i.e SoC DRAM) as vidmem. In these cases the | 40 | * Some iGPUs treat sysmem (i.e SoC DRAM) as vidmem. In these cases the |
41 | * "sysmem" aperture should really be translated to VIDMEM. | 41 | * "sysmem" aperture should really be translated to VIDMEM. |
42 | */ | 42 | */ |
43 | if (!nvgpu_is_enabled(g, NVGPU_MM_HONORS_APERTURE)) | 43 | if (!nvgpu_is_enabled(g, NVGPU_MM_HONORS_APERTURE)) { |
44 | aperture = APERTURE_VIDMEM; | 44 | aperture = APERTURE_VIDMEM; |
45 | } | ||
45 | 46 | ||
46 | switch (aperture) { | 47 | switch (aperture) { |
47 | case __APERTURE_SYSMEM_COH: | 48 | case __APERTURE_SYSMEM_COH: |
@@ -67,8 +68,9 @@ u32 nvgpu_aperture_mask(struct gk20a *g, struct nvgpu_mem *mem, | |||
67 | * we add this translation step here. | 68 | * we add this translation step here. |
68 | */ | 69 | */ |
69 | if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) && | 70 | if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) && |
70 | ap == APERTURE_SYSMEM) | 71 | ap == APERTURE_SYSMEM) { |
71 | ap = __APERTURE_SYSMEM_COH; | 72 | ap = __APERTURE_SYSMEM_COH; |
73 | } | ||
72 | 74 | ||
73 | return __nvgpu_aperture_mask(g, ap, | 75 | return __nvgpu_aperture_mask(g, ap, |
74 | sysmem_mask, sysmem_coh_mask, vidmem_mask); | 76 | sysmem_mask, sysmem_coh_mask, vidmem_mask); |
@@ -115,15 +117,17 @@ u64 nvgpu_sgt_get_gpu_addr(struct gk20a *g, struct nvgpu_sgt *sgt, | |||
115 | 117 | ||
116 | bool nvgpu_sgt_iommuable(struct gk20a *g, struct nvgpu_sgt *sgt) | 118 | bool nvgpu_sgt_iommuable(struct gk20a *g, struct nvgpu_sgt *sgt) |
117 | { | 119 | { |
118 | if (sgt->ops->sgt_iommuable) | 120 | if (sgt->ops->sgt_iommuable) { |
119 | return sgt->ops->sgt_iommuable(g, sgt); | 121 | return sgt->ops->sgt_iommuable(g, sgt); |
122 | } | ||
120 | return false; | 123 | return false; |
121 | } | 124 | } |
122 | 125 | ||
123 | void nvgpu_sgt_free(struct gk20a *g, struct nvgpu_sgt *sgt) | 126 | void nvgpu_sgt_free(struct gk20a *g, struct nvgpu_sgt *sgt) |
124 | { | 127 | { |
125 | if (sgt && sgt->ops->sgt_free) | 128 | if (sgt && sgt->ops->sgt_free) { |
126 | sgt->ops->sgt_free(g, sgt); | 129 | sgt->ops->sgt_free(g, sgt); |
130 | } | ||
127 | } | 131 | } |
128 | 132 | ||
129 | u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys) | 133 | u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys) |
@@ -131,8 +135,9 @@ u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys) | |||
131 | /* ensure it is not vidmem allocation */ | 135 | /* ensure it is not vidmem allocation */ |
132 | WARN_ON(nvgpu_addr_is_vidmem_page_alloc(phys)); | 136 | WARN_ON(nvgpu_addr_is_vidmem_page_alloc(phys)); |
133 | 137 | ||
134 | if (nvgpu_iommuable(g) && g->ops.mm.get_iommu_bit) | 138 | if (nvgpu_iommuable(g) && g->ops.mm.get_iommu_bit) { |
135 | return phys | 1ULL << g->ops.mm.get_iommu_bit(g); | 139 | return phys | 1ULL << g->ops.mm.get_iommu_bit(g); |
140 | } | ||
136 | 141 | ||
137 | return phys; | 142 | return phys; |
138 | } | 143 | } |
@@ -157,8 +162,9 @@ u64 nvgpu_sgt_alignment(struct gk20a *g, struct nvgpu_sgt *sgt) | |||
157 | */ | 162 | */ |
158 | if (nvgpu_iommuable(g) && | 163 | if (nvgpu_iommuable(g) && |
159 | nvgpu_sgt_iommuable(g, sgt) && | 164 | nvgpu_sgt_iommuable(g, sgt) && |
160 | nvgpu_sgt_get_dma(sgt, sgt->sgl)) | 165 | nvgpu_sgt_get_dma(sgt, sgt->sgl)) { |
161 | return 1ULL << __ffs(nvgpu_sgt_get_dma(sgt, sgt->sgl)); | 166 | return 1ULL << __ffs(nvgpu_sgt_get_dma(sgt, sgt->sgl)); |
167 | } | ||
162 | 168 | ||
163 | /* | 169 | /* |
164 | * Otherwise the buffer is not iommuable (VIDMEM, for example) or we are | 170 | * Otherwise the buffer is not iommuable (VIDMEM, for example) or we are |
@@ -169,10 +175,11 @@ u64 nvgpu_sgt_alignment(struct gk20a *g, struct nvgpu_sgt *sgt) | |||
169 | chunk_align = 1ULL << __ffs(nvgpu_sgt_get_phys(g, sgt, sgl) | | 175 | chunk_align = 1ULL << __ffs(nvgpu_sgt_get_phys(g, sgt, sgl) | |
170 | nvgpu_sgt_get_length(sgt, sgl)); | 176 | nvgpu_sgt_get_length(sgt, sgl)); |
171 | 177 | ||
172 | if (align) | 178 | if (align) { |
173 | align = min(align, chunk_align); | 179 | align = min(align, chunk_align); |
174 | else | 180 | } else { |
175 | align = chunk_align; | 181 | align = chunk_align; |
182 | } | ||
176 | } | 183 | } |
177 | 184 | ||
178 | return align; | 185 | return align; |
diff --git a/drivers/gpu/nvgpu/common/mm/vm.c b/drivers/gpu/nvgpu/common/mm/vm.c index 3b682e28..7d97b7b7 100644 --- a/drivers/gpu/nvgpu/common/mm/vm.c +++ b/drivers/gpu/nvgpu/common/mm/vm.c | |||
@@ -111,8 +111,9 @@ static void nvgpu_vm_free_entries(struct vm_gk20a *vm, | |||
111 | 111 | ||
112 | __nvgpu_pd_cache_free_direct(g, pdb); | 112 | __nvgpu_pd_cache_free_direct(g, pdb); |
113 | 113 | ||
114 | if (!pdb->entries) | 114 | if (!pdb->entries) { |
115 | return; | 115 | return; |
116 | } | ||
116 | 117 | ||
117 | for (i = 0; i < pdb->num_entries; i++) { | 118 | for (i = 0; i < pdb->num_entries; i++) { |
118 | __nvgpu_vm_free_entries(vm, &pdb->entries[i], 1); | 119 | __nvgpu_vm_free_entries(vm, &pdb->entries[i], 1); |
@@ -204,8 +205,9 @@ int nvgpu_big_pages_possible(struct vm_gk20a *vm, u64 base, u64 size) | |||
204 | { | 205 | { |
205 | u64 mask = ((u64)vm->big_page_size << 10) - 1; | 206 | u64 mask = ((u64)vm->big_page_size << 10) - 1; |
206 | 207 | ||
207 | if (base & mask || size & mask) | 208 | if (base & mask || size & mask) { |
208 | return 0; | 209 | return 0; |
210 | } | ||
209 | return 1; | 211 | return 1; |
210 | } | 212 | } |
211 | 213 | ||
@@ -223,19 +225,23 @@ static int nvgpu_init_sema_pool(struct vm_gk20a *vm) | |||
223 | /* | 225 | /* |
224 | * Don't waste the memory on semaphores if we don't need them. | 226 | * Don't waste the memory on semaphores if we don't need them. |
225 | */ | 227 | */ |
226 | if (nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS)) | 228 | if (nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS)) { |
227 | return 0; | 229 | return 0; |
230 | } | ||
228 | 231 | ||
229 | if (vm->sema_pool) | 232 | if (vm->sema_pool) { |
230 | return 0; | 233 | return 0; |
234 | } | ||
231 | 235 | ||
232 | sema_sea = nvgpu_semaphore_sea_create(g); | 236 | sema_sea = nvgpu_semaphore_sea_create(g); |
233 | if (!sema_sea) | 237 | if (!sema_sea) { |
234 | return -ENOMEM; | 238 | return -ENOMEM; |
239 | } | ||
235 | 240 | ||
236 | err = nvgpu_semaphore_pool_alloc(sema_sea, &vm->sema_pool); | 241 | err = nvgpu_semaphore_pool_alloc(sema_sea, &vm->sema_pool); |
237 | if (err) | 242 | if (err) { |
238 | return err; | 243 | return err; |
244 | } | ||
239 | 245 | ||
240 | /* | 246 | /* |
241 | * Allocate a chunk of GPU VA space for mapping the semaphores. We will | 247 | * Allocate a chunk of GPU VA space for mapping the semaphores. We will |
@@ -287,11 +293,13 @@ int __nvgpu_vm_init(struct mm_gk20a *mm, | |||
287 | u64 kernel_vma_start, kernel_vma_limit; | 293 | u64 kernel_vma_start, kernel_vma_limit; |
288 | struct gk20a *g = gk20a_from_mm(mm); | 294 | struct gk20a *g = gk20a_from_mm(mm); |
289 | 295 | ||
290 | if (WARN_ON(kernel_reserved + low_hole > aperture_size)) | 296 | if (WARN_ON(kernel_reserved + low_hole > aperture_size)) { |
291 | return -ENOMEM; | 297 | return -ENOMEM; |
298 | } | ||
292 | 299 | ||
293 | if (WARN_ON(vm->guest_managed && kernel_reserved != 0)) | 300 | if (WARN_ON(vm->guest_managed && kernel_reserved != 0)) { |
294 | return -EINVAL; | 301 | return -EINVAL; |
302 | } | ||
295 | 303 | ||
296 | nvgpu_log_info(g, "Init space for %s: valimit=0x%llx, " | 304 | nvgpu_log_info(g, "Init space for %s: valimit=0x%llx, " |
297 | "LP size=0x%x lowhole=0x%llx", | 305 | "LP size=0x%x lowhole=0x%llx", |
@@ -308,8 +316,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm, | |||
308 | vm->vma[gmmu_page_size_small] = &vm->user; | 316 | vm->vma[gmmu_page_size_small] = &vm->user; |
309 | vm->vma[gmmu_page_size_big] = &vm->user; | 317 | vm->vma[gmmu_page_size_big] = &vm->user; |
310 | vm->vma[gmmu_page_size_kernel] = &vm->kernel; | 318 | vm->vma[gmmu_page_size_kernel] = &vm->kernel; |
311 | if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) | 319 | if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) { |
312 | vm->vma[gmmu_page_size_big] = &vm->user_lp; | 320 | vm->vma[gmmu_page_size_big] = &vm->user_lp; |
321 | } | ||
313 | 322 | ||
314 | vm->va_start = low_hole; | 323 | vm->va_start = low_hole; |
315 | vm->va_limit = aperture_size; | 324 | vm->va_limit = aperture_size; |
@@ -332,8 +341,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm, | |||
332 | /* Initialize the page table data structures. */ | 341 | /* Initialize the page table data structures. */ |
333 | strncpy(vm->name, name, min(strlen(name), sizeof(vm->name))); | 342 | strncpy(vm->name, name, min(strlen(name), sizeof(vm->name))); |
334 | err = nvgpu_gmmu_init_page_table(vm); | 343 | err = nvgpu_gmmu_init_page_table(vm); |
335 | if (err) | 344 | if (err) { |
336 | goto clean_up_vgpu_vm; | 345 | goto clean_up_vgpu_vm; |
346 | } | ||
337 | 347 | ||
338 | /* Setup vma limits. */ | 348 | /* Setup vma limits. */ |
339 | if (kernel_reserved + low_hole < aperture_size) { | 349 | if (kernel_reserved + low_hole < aperture_size) { |
@@ -396,14 +406,15 @@ int __nvgpu_vm_init(struct mm_gk20a *mm, | |||
396 | * Determine if big pages are possible in this VM. If a split address | 406 | * Determine if big pages are possible in this VM. If a split address |
397 | * space is used then check the user_lp vma instead of the user vma. | 407 | * space is used then check the user_lp vma instead of the user vma. |
398 | */ | 408 | */ |
399 | if (nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) | 409 | if (nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) { |
400 | vm->big_pages = big_pages && | 410 | vm->big_pages = big_pages && |
401 | nvgpu_big_pages_possible(vm, user_vma_start, | 411 | nvgpu_big_pages_possible(vm, user_vma_start, |
402 | user_vma_limit - user_vma_start); | 412 | user_vma_limit - user_vma_start); |
403 | else | 413 | } else { |
404 | vm->big_pages = big_pages && | 414 | vm->big_pages = big_pages && |
405 | nvgpu_big_pages_possible(vm, user_lp_vma_start, | 415 | nvgpu_big_pages_possible(vm, user_lp_vma_start, |
406 | user_lp_vma_limit - user_lp_vma_start); | 416 | user_lp_vma_limit - user_lp_vma_start); |
417 | } | ||
407 | 418 | ||
408 | /* | 419 | /* |
409 | * User VMA. | 420 | * User VMA. |
@@ -418,8 +429,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm, | |||
418 | SZ_4K, | 429 | SZ_4K, |
419 | GPU_BALLOC_MAX_ORDER, | 430 | GPU_BALLOC_MAX_ORDER, |
420 | GPU_ALLOC_GVA_SPACE); | 431 | GPU_ALLOC_GVA_SPACE); |
421 | if (err) | 432 | if (err) { |
422 | goto clean_up_page_tables; | 433 | goto clean_up_page_tables; |
434 | } | ||
423 | } else { | 435 | } else { |
424 | /* | 436 | /* |
425 | * Make these allocator pointers point to the kernel allocator | 437 | * Make these allocator pointers point to the kernel allocator |
@@ -443,8 +455,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm, | |||
443 | vm->big_page_size, | 455 | vm->big_page_size, |
444 | GPU_BALLOC_MAX_ORDER, | 456 | GPU_BALLOC_MAX_ORDER, |
445 | GPU_ALLOC_GVA_SPACE); | 457 | GPU_ALLOC_GVA_SPACE); |
446 | if (err) | 458 | if (err) { |
447 | goto clean_up_allocators; | 459 | goto clean_up_allocators; |
460 | } | ||
448 | } | 461 | } |
449 | 462 | ||
450 | /* | 463 | /* |
@@ -458,8 +471,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm, | |||
458 | SZ_4K, | 471 | SZ_4K, |
459 | GPU_BALLOC_MAX_ORDER, | 472 | GPU_BALLOC_MAX_ORDER, |
460 | kernel_vma_flags); | 473 | kernel_vma_flags); |
461 | if (err) | 474 | if (err) { |
462 | goto clean_up_allocators; | 475 | goto clean_up_allocators; |
476 | } | ||
463 | 477 | ||
464 | vm->mapped_buffers = NULL; | 478 | vm->mapped_buffers = NULL; |
465 | 479 | ||
@@ -475,19 +489,23 @@ int __nvgpu_vm_init(struct mm_gk20a *mm, | |||
475 | */ | 489 | */ |
476 | if (vm->va_limit > 4ULL * SZ_1G) { | 490 | if (vm->va_limit > 4ULL * SZ_1G) { |
477 | err = nvgpu_init_sema_pool(vm); | 491 | err = nvgpu_init_sema_pool(vm); |
478 | if (err) | 492 | if (err) { |
479 | goto clean_up_allocators; | 493 | goto clean_up_allocators; |
494 | } | ||
480 | } | 495 | } |
481 | 496 | ||
482 | return 0; | 497 | return 0; |
483 | 498 | ||
484 | clean_up_allocators: | 499 | clean_up_allocators: |
485 | if (nvgpu_alloc_initialized(&vm->kernel)) | 500 | if (nvgpu_alloc_initialized(&vm->kernel)) { |
486 | nvgpu_alloc_destroy(&vm->kernel); | 501 | nvgpu_alloc_destroy(&vm->kernel); |
487 | if (nvgpu_alloc_initialized(&vm->user)) | 502 | } |
503 | if (nvgpu_alloc_initialized(&vm->user)) { | ||
488 | nvgpu_alloc_destroy(&vm->user); | 504 | nvgpu_alloc_destroy(&vm->user); |
489 | if (nvgpu_alloc_initialized(&vm->user_lp)) | 505 | } |
506 | if (nvgpu_alloc_initialized(&vm->user_lp)) { | ||
490 | nvgpu_alloc_destroy(&vm->user_lp); | 507 | nvgpu_alloc_destroy(&vm->user_lp); |
508 | } | ||
491 | clean_up_page_tables: | 509 | clean_up_page_tables: |
492 | /* Cleans up nvgpu_gmmu_init_page_table() */ | 510 | /* Cleans up nvgpu_gmmu_init_page_table() */ |
493 | __nvgpu_pd_cache_free_direct(g, &vm->pdb); | 511 | __nvgpu_pd_cache_free_direct(g, &vm->pdb); |
@@ -547,8 +565,9 @@ struct vm_gk20a *nvgpu_vm_init(struct gk20a *g, | |||
547 | { | 565 | { |
548 | struct vm_gk20a *vm = nvgpu_kzalloc(g, sizeof(*vm)); | 566 | struct vm_gk20a *vm = nvgpu_kzalloc(g, sizeof(*vm)); |
549 | 567 | ||
550 | if (!vm) | 568 | if (!vm) { |
551 | return NULL; | 569 | return NULL; |
570 | } | ||
552 | 571 | ||
553 | if (__nvgpu_vm_init(&g->mm, vm, big_page_size, low_hole, | 572 | if (__nvgpu_vm_init(&g->mm, vm, big_page_size, low_hole, |
554 | kernel_reserved, aperture_size, big_pages, | 573 | kernel_reserved, aperture_size, big_pages, |
@@ -582,9 +601,10 @@ static void __nvgpu_vm_remove(struct vm_gk20a *vm) | |||
582 | } | 601 | } |
583 | } | 602 | } |
584 | 603 | ||
585 | if (nvgpu_mem_is_valid(&g->syncpt_mem) && vm->syncpt_ro_map_gpu_va) | 604 | if (nvgpu_mem_is_valid(&g->syncpt_mem) && vm->syncpt_ro_map_gpu_va) { |
586 | nvgpu_gmmu_unmap(vm, &g->syncpt_mem, | 605 | nvgpu_gmmu_unmap(vm, &g->syncpt_mem, |
587 | vm->syncpt_ro_map_gpu_va); | 606 | vm->syncpt_ro_map_gpu_va); |
607 | } | ||
588 | 608 | ||
589 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | 609 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); |
590 | 610 | ||
@@ -603,12 +623,15 @@ static void __nvgpu_vm_remove(struct vm_gk20a *vm) | |||
603 | nvgpu_kfree(vm->mm->g, vm_area); | 623 | nvgpu_kfree(vm->mm->g, vm_area); |
604 | } | 624 | } |
605 | 625 | ||
606 | if (nvgpu_alloc_initialized(&vm->kernel)) | 626 | if (nvgpu_alloc_initialized(&vm->kernel)) { |
607 | nvgpu_alloc_destroy(&vm->kernel); | 627 | nvgpu_alloc_destroy(&vm->kernel); |
608 | if (nvgpu_alloc_initialized(&vm->user)) | 628 | } |
629 | if (nvgpu_alloc_initialized(&vm->user)) { | ||
609 | nvgpu_alloc_destroy(&vm->user); | 630 | nvgpu_alloc_destroy(&vm->user); |
610 | if (nvgpu_alloc_initialized(&vm->user_lp)) | 631 | } |
632 | if (nvgpu_alloc_initialized(&vm->user_lp)) { | ||
611 | nvgpu_alloc_destroy(&vm->user_lp); | 633 | nvgpu_alloc_destroy(&vm->user_lp); |
634 | } | ||
612 | 635 | ||
613 | nvgpu_vm_free_entries(vm, &vm->pdb); | 636 | nvgpu_vm_free_entries(vm, &vm->pdb); |
614 | 637 | ||
@@ -664,8 +687,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf( | |||
664 | struct nvgpu_rbtree_node *root = vm->mapped_buffers; | 687 | struct nvgpu_rbtree_node *root = vm->mapped_buffers; |
665 | 688 | ||
666 | nvgpu_rbtree_search(addr, &node, root); | 689 | nvgpu_rbtree_search(addr, &node, root); |
667 | if (!node) | 690 | if (!node) { |
668 | return NULL; | 691 | return NULL; |
692 | } | ||
669 | 693 | ||
670 | return mapped_buffer_from_rbtree_node(node); | 694 | return mapped_buffer_from_rbtree_node(node); |
671 | } | 695 | } |
@@ -677,8 +701,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_range( | |||
677 | struct nvgpu_rbtree_node *root = vm->mapped_buffers; | 701 | struct nvgpu_rbtree_node *root = vm->mapped_buffers; |
678 | 702 | ||
679 | nvgpu_rbtree_range_search(addr, &node, root); | 703 | nvgpu_rbtree_range_search(addr, &node, root); |
680 | if (!node) | 704 | if (!node) { |
681 | return NULL; | 705 | return NULL; |
706 | } | ||
682 | 707 | ||
683 | return mapped_buffer_from_rbtree_node(node); | 708 | return mapped_buffer_from_rbtree_node(node); |
684 | } | 709 | } |
@@ -690,8 +715,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_less_than( | |||
690 | struct nvgpu_rbtree_node *root = vm->mapped_buffers; | 715 | struct nvgpu_rbtree_node *root = vm->mapped_buffers; |
691 | 716 | ||
692 | nvgpu_rbtree_less_than_search(addr, &node, root); | 717 | nvgpu_rbtree_less_than_search(addr, &node, root); |
693 | if (!node) | 718 | if (!node) { |
694 | return NULL; | 719 | return NULL; |
720 | } | ||
695 | 721 | ||
696 | return mapped_buffer_from_rbtree_node(node); | 722 | return mapped_buffer_from_rbtree_node(node); |
697 | } | 723 | } |
@@ -746,8 +772,9 @@ void nvgpu_vm_put_buffers(struct vm_gk20a *vm, | |||
746 | int i; | 772 | int i; |
747 | struct vm_gk20a_mapping_batch batch; | 773 | struct vm_gk20a_mapping_batch batch; |
748 | 774 | ||
749 | if (num_buffers == 0) | 775 | if (num_buffers == 0) { |
750 | return; | 776 | return; |
777 | } | ||
751 | 778 | ||
752 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | 779 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); |
753 | nvgpu_vm_mapping_batch_start(&batch); | 780 | nvgpu_vm_mapping_batch_start(&batch); |
@@ -814,10 +841,11 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm, | |||
814 | compr_kind : NVGPU_KIND_INVALID); | 841 | compr_kind : NVGPU_KIND_INVALID); |
815 | binfo.incompr_kind = incompr_kind; | 842 | binfo.incompr_kind = incompr_kind; |
816 | 843 | ||
817 | if (compr_kind != NVGPU_KIND_INVALID) | 844 | if (compr_kind != NVGPU_KIND_INVALID) { |
818 | map_key_kind = compr_kind; | 845 | map_key_kind = compr_kind; |
819 | else | 846 | } else { |
820 | map_key_kind = incompr_kind; | 847 | map_key_kind = incompr_kind; |
848 | } | ||
821 | 849 | ||
822 | /* | 850 | /* |
823 | * Check if this buffer is already mapped. | 851 | * Check if this buffer is already mapped. |
@@ -847,11 +875,12 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm, | |||
847 | } | 875 | } |
848 | 876 | ||
849 | align = nvgpu_sgt_alignment(g, sgt); | 877 | align = nvgpu_sgt_alignment(g, sgt); |
850 | if (g->mm.disable_bigpage) | 878 | if (g->mm.disable_bigpage) { |
851 | binfo.pgsz_idx = gmmu_page_size_small; | 879 | binfo.pgsz_idx = gmmu_page_size_small; |
852 | else | 880 | } else { |
853 | binfo.pgsz_idx = __get_pte_size(vm, map_addr, | 881 | binfo.pgsz_idx = __get_pte_size(vm, map_addr, |
854 | min_t(u64, binfo.size, align)); | 882 | min_t(u64, binfo.size, align)); |
883 | } | ||
855 | map_size = map_size ? map_size : binfo.size; | 884 | map_size = map_size ? map_size : binfo.size; |
856 | map_size = ALIGN(map_size, SZ_4K); | 885 | map_size = ALIGN(map_size, SZ_4K); |
857 | 886 | ||
@@ -872,8 +901,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm, | |||
872 | map_size, | 901 | map_size, |
873 | binfo.pgsz_idx, | 902 | binfo.pgsz_idx, |
874 | &vm_area); | 903 | &vm_area); |
875 | if (err) | 904 | if (err) { |
876 | goto clean_up; | 905 | goto clean_up; |
906 | } | ||
877 | 907 | ||
878 | va_allocated = false; | 908 | va_allocated = false; |
879 | } | 909 | } |
@@ -941,8 +971,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm, | |||
941 | comptags.lines - 1)); | 971 | comptags.lines - 1)); |
942 | gk20a_comptags_finish_clear( | 972 | gk20a_comptags_finish_clear( |
943 | os_buf, err == 0); | 973 | os_buf, err == 0); |
944 | if (err) | 974 | if (err) { |
945 | goto clean_up; | 975 | goto clean_up; |
976 | } | ||
946 | } | 977 | } |
947 | } else { | 978 | } else { |
948 | /* | 979 | /* |
@@ -955,8 +986,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm, | |||
955 | /* | 986 | /* |
956 | * Store the ctag offset for later use if we got the comptags | 987 | * Store the ctag offset for later use if we got the comptags |
957 | */ | 988 | */ |
958 | if (comptags.lines) | 989 | if (comptags.lines) { |
959 | ctag_offset = comptags.offset; | 990 | ctag_offset = comptags.offset; |
991 | } | ||
960 | } | 992 | } |
961 | 993 | ||
962 | /* | 994 | /* |
@@ -984,8 +1016,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm, | |||
984 | goto clean_up; | 1016 | goto clean_up; |
985 | } | 1017 | } |
986 | 1018 | ||
987 | if (clear_ctags) | 1019 | if (clear_ctags) { |
988 | clear_ctags = gk20a_comptags_start_clear(os_buf); | 1020 | clear_ctags = gk20a_comptags_start_clear(os_buf); |
1021 | } | ||
989 | 1022 | ||
990 | map_addr = g->ops.mm.gmmu_map(vm, | 1023 | map_addr = g->ops.mm.gmmu_map(vm, |
991 | map_addr, | 1024 | map_addr, |
@@ -1003,8 +1036,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm, | |||
1003 | batch, | 1036 | batch, |
1004 | aperture); | 1037 | aperture); |
1005 | 1038 | ||
1006 | if (clear_ctags) | 1039 | if (clear_ctags) { |
1007 | gk20a_comptags_finish_clear(os_buf, map_addr != 0); | 1040 | gk20a_comptags_finish_clear(os_buf, map_addr != 0); |
1041 | } | ||
1008 | 1042 | ||
1009 | if (!map_addr) { | 1043 | if (!map_addr) { |
1010 | err = -ENOMEM; | 1044 | err = -ENOMEM; |
@@ -1041,7 +1075,7 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm, | |||
1041 | return mapped_buffer; | 1075 | return mapped_buffer; |
1042 | 1076 | ||
1043 | clean_up: | 1077 | clean_up: |
1044 | if (mapped_buffer->addr) | 1078 | if (mapped_buffer->addr) { |
1045 | g->ops.mm.gmmu_unmap(vm, | 1079 | g->ops.mm.gmmu_unmap(vm, |
1046 | mapped_buffer->addr, | 1080 | mapped_buffer->addr, |
1047 | mapped_buffer->size, | 1081 | mapped_buffer->size, |
@@ -1051,6 +1085,7 @@ clean_up: | |||
1051 | mapped_buffer->vm_area ? | 1085 | mapped_buffer->vm_area ? |
1052 | mapped_buffer->vm_area->sparse : false, | 1086 | mapped_buffer->vm_area->sparse : false, |
1053 | NULL); | 1087 | NULL); |
1088 | } | ||
1054 | nvgpu_mutex_release(&vm->update_gmmu_lock); | 1089 | nvgpu_mutex_release(&vm->update_gmmu_lock); |
1055 | clean_up_nolock: | 1090 | clean_up_nolock: |
1056 | nvgpu_kfree(g, mapped_buffer); | 1091 | nvgpu_kfree(g, mapped_buffer); |
@@ -1132,14 +1167,16 @@ static int nvgpu_vm_unmap_sync_buffer(struct vm_gk20a *vm, | |||
1132 | nvgpu_timeout_init(vm->mm->g, &timeout, 50, NVGPU_TIMER_CPU_TIMER); | 1167 | nvgpu_timeout_init(vm->mm->g, &timeout, 50, NVGPU_TIMER_CPU_TIMER); |
1133 | 1168 | ||
1134 | do { | 1169 | do { |
1135 | if (nvgpu_atomic_read(&mapped_buffer->ref.refcount) == 1) | 1170 | if (nvgpu_atomic_read(&mapped_buffer->ref.refcount) == 1) { |
1136 | break; | 1171 | break; |
1172 | } | ||
1137 | nvgpu_msleep(10); | 1173 | nvgpu_msleep(10); |
1138 | } while (!nvgpu_timeout_expired_msg(&timeout, | 1174 | } while (!nvgpu_timeout_expired_msg(&timeout, |
1139 | "sync-unmap failed on 0x%llx")); | 1175 | "sync-unmap failed on 0x%llx")); |
1140 | 1176 | ||
1141 | if (nvgpu_timeout_expired(&timeout)) | 1177 | if (nvgpu_timeout_expired(&timeout)) { |
1142 | ret = -ETIMEDOUT; | 1178 | ret = -ETIMEDOUT; |
1179 | } | ||
1143 | 1180 | ||
1144 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | 1181 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); |
1145 | 1182 | ||
@@ -1154,16 +1191,18 @@ void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset, | |||
1154 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | 1191 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); |
1155 | 1192 | ||
1156 | mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, offset); | 1193 | mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, offset); |
1157 | if (!mapped_buffer) | 1194 | if (!mapped_buffer) { |
1158 | goto done; | 1195 | goto done; |
1196 | } | ||
1159 | 1197 | ||
1160 | if (mapped_buffer->flags & NVGPU_VM_MAP_FIXED_OFFSET) { | 1198 | if (mapped_buffer->flags & NVGPU_VM_MAP_FIXED_OFFSET) { |
1161 | if (nvgpu_vm_unmap_sync_buffer(vm, mapped_buffer)) | 1199 | if (nvgpu_vm_unmap_sync_buffer(vm, mapped_buffer)) { |
1162 | /* | 1200 | /* |
1163 | * Looks like we have failed... Better not continue in | 1201 | * Looks like we have failed... Better not continue in |
1164 | * case the buffer is in use. | 1202 | * case the buffer is in use. |
1165 | */ | 1203 | */ |
1166 | goto done; | 1204 | goto done; |
1205 | } | ||
1167 | } | 1206 | } |
1168 | 1207 | ||
1169 | /* | 1208 | /* |