diff options
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/buddy_allocator.c | 87 |
1 files changed, 45 insertions, 42 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c index 3b8d9939..516e5035 100644 --- a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c +++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c | |||
@@ -35,8 +35,8 @@ static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a, | |||
35 | u64 addr); | 35 | u64 addr); |
36 | static void balloc_coalesce(struct nvgpu_buddy_allocator *a, | 36 | static void balloc_coalesce(struct nvgpu_buddy_allocator *a, |
37 | struct nvgpu_buddy *b); | 37 | struct nvgpu_buddy *b); |
38 | static void __balloc_do_free_fixed(struct nvgpu_buddy_allocator *a, | 38 | static void balloc_do_free_fixed(struct nvgpu_buddy_allocator *a, |
39 | struct nvgpu_fixed_alloc *falloc); | 39 | struct nvgpu_fixed_alloc *falloc); |
40 | 40 | ||
41 | /* | 41 | /* |
42 | * This function is not present in older kernel's list.h code. | 42 | * This function is not present in older kernel's list.h code. |
@@ -144,9 +144,9 @@ static struct nvgpu_buddy *balloc_new_buddy(struct nvgpu_buddy_allocator *a, | |||
144 | return new_buddy; | 144 | return new_buddy; |
145 | } | 145 | } |
146 | 146 | ||
147 | static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a, | 147 | static void balloc_buddy_list_do_add(struct nvgpu_buddy_allocator *a, |
148 | struct nvgpu_buddy *b, | 148 | struct nvgpu_buddy *b, |
149 | struct nvgpu_list_node *list) | 149 | struct nvgpu_list_node *list) |
150 | { | 150 | { |
151 | if (buddy_is_in_list(b)) { | 151 | if (buddy_is_in_list(b)) { |
152 | alloc_dbg(balloc_owner(a), | 152 | alloc_dbg(balloc_owner(a), |
@@ -170,8 +170,8 @@ static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a, | |||
170 | buddy_set_in_list(b); | 170 | buddy_set_in_list(b); |
171 | } | 171 | } |
172 | 172 | ||
173 | static void __balloc_buddy_list_rem(struct nvgpu_buddy_allocator *a, | 173 | static void balloc_buddy_list_do_rem(struct nvgpu_buddy_allocator *a, |
174 | struct nvgpu_buddy *b) | 174 | struct nvgpu_buddy *b) |
175 | { | 175 | { |
176 | if (!buddy_is_in_list(b)) { | 176 | if (!buddy_is_in_list(b)) { |
177 | alloc_dbg(balloc_owner(a), | 177 | alloc_dbg(balloc_owner(a), |
@@ -191,14 +191,14 @@ static void __balloc_buddy_list_rem(struct nvgpu_buddy_allocator *a, | |||
191 | static void balloc_blist_add(struct nvgpu_buddy_allocator *a, | 191 | static void balloc_blist_add(struct nvgpu_buddy_allocator *a, |
192 | struct nvgpu_buddy *b) | 192 | struct nvgpu_buddy *b) |
193 | { | 193 | { |
194 | __balloc_buddy_list_add(a, b, balloc_get_order_list(a, b->order)); | 194 | balloc_buddy_list_do_add(a, b, balloc_get_order_list(a, b->order)); |
195 | a->buddy_list_len[b->order]++; | 195 | a->buddy_list_len[b->order]++; |
196 | } | 196 | } |
197 | 197 | ||
198 | static void balloc_blist_rem(struct nvgpu_buddy_allocator *a, | 198 | static void balloc_blist_rem(struct nvgpu_buddy_allocator *a, |
199 | struct nvgpu_buddy *b) | 199 | struct nvgpu_buddy *b) |
200 | { | 200 | { |
201 | __balloc_buddy_list_rem(a, b); | 201 | balloc_buddy_list_do_rem(a, b); |
202 | a->buddy_list_len[b->order]--; | 202 | a->buddy_list_len[b->order]--; |
203 | } | 203 | } |
204 | 204 | ||
@@ -214,7 +214,7 @@ static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len) | |||
214 | return fls(len); | 214 | return fls(len); |
215 | } | 215 | } |
216 | 216 | ||
217 | static u64 __balloc_max_order_in(struct nvgpu_buddy_allocator *a, | 217 | static u64 balloc_max_order_in(struct nvgpu_buddy_allocator *a, |
218 | u64 start, u64 end) | 218 | u64 start, u64 end) |
219 | { | 219 | { |
220 | u64 size = (end - start) >> a->blk_shift; | 220 | u64 size = (end - start) >> a->blk_shift; |
@@ -244,7 +244,7 @@ static int balloc_init_lists(struct nvgpu_buddy_allocator *a) | |||
244 | } | 244 | } |
245 | 245 | ||
246 | while (bstart < bend) { | 246 | while (bstart < bend) { |
247 | order = __balloc_max_order_in(a, bstart, bend); | 247 | order = balloc_max_order_in(a, bstart, bend); |
248 | 248 | ||
249 | buddy = balloc_new_buddy(a, NULL, bstart, order); | 249 | buddy = balloc_new_buddy(a, NULL, bstart, order); |
250 | if (!buddy) { | 250 | if (!buddy) { |
@@ -296,7 +296,7 @@ static void nvgpu_buddy_allocator_destroy(struct nvgpu_allocator *na) | |||
296 | falloc = nvgpu_fixed_alloc_from_rbtree_node(node); | 296 | falloc = nvgpu_fixed_alloc_from_rbtree_node(node); |
297 | 297 | ||
298 | nvgpu_rbtree_unlink(node, &a->fixed_allocs); | 298 | nvgpu_rbtree_unlink(node, &a->fixed_allocs); |
299 | __balloc_do_free_fixed(a, falloc); | 299 | balloc_do_free_fixed(a, falloc); |
300 | 300 | ||
301 | nvgpu_rbtree_enum_start(0, &node, a->fixed_allocs); | 301 | nvgpu_rbtree_enum_start(0, &node, a->fixed_allocs); |
302 | } | 302 | } |
@@ -508,8 +508,8 @@ static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a, | |||
508 | /* | 508 | /* |
509 | * Find a suitable buddy for the given order and PTE type (big or little). | 509 | * Find a suitable buddy for the given order and PTE type (big or little). |
510 | */ | 510 | */ |
511 | static struct nvgpu_buddy *__balloc_find_buddy(struct nvgpu_buddy_allocator *a, | 511 | static struct nvgpu_buddy *balloc_find_buddy(struct nvgpu_buddy_allocator *a, |
512 | u64 order, u32 pte_size) | 512 | u64 order, u32 pte_size) |
513 | { | 513 | { |
514 | struct nvgpu_buddy *bud; | 514 | struct nvgpu_buddy *bud; |
515 | 515 | ||
@@ -545,15 +545,15 @@ static struct nvgpu_buddy *__balloc_find_buddy(struct nvgpu_buddy_allocator *a, | |||
545 | * | 545 | * |
546 | * @a must be locked. | 546 | * @a must be locked. |
547 | */ | 547 | */ |
548 | static u64 __balloc_do_alloc(struct nvgpu_buddy_allocator *a, | 548 | static u64 balloc_do_alloc(struct nvgpu_buddy_allocator *a, |
549 | u64 order, u32 pte_size) | 549 | u64 order, u32 pte_size) |
550 | { | 550 | { |
551 | u64 split_order; | 551 | u64 split_order; |
552 | struct nvgpu_buddy *bud = NULL; | 552 | struct nvgpu_buddy *bud = NULL; |
553 | 553 | ||
554 | split_order = order; | 554 | split_order = order; |
555 | while (split_order <= a->max_order && | 555 | while (split_order <= a->max_order && |
556 | !(bud = __balloc_find_buddy(a, split_order, pte_size))) { | 556 | !(bud = balloc_find_buddy(a, split_order, pte_size))) { |
557 | split_order++; | 557 | split_order++; |
558 | } | 558 | } |
559 | 559 | ||
@@ -648,9 +648,9 @@ static struct nvgpu_fixed_alloc *balloc_free_fixed( | |||
648 | * Find the parent range - doesn't necessarily need the parent to actually exist | 648 | * Find the parent range - doesn't necessarily need the parent to actually exist |
649 | * as a buddy. Finding an existing parent comes later... | 649 | * as a buddy. Finding an existing parent comes later... |
650 | */ | 650 | */ |
651 | static void __balloc_get_parent_range(struct nvgpu_buddy_allocator *a, | 651 | static void balloc_get_parent_range(struct nvgpu_buddy_allocator *a, |
652 | u64 base, u64 order, | 652 | u64 base, u64 order, |
653 | u64 *pbase, u64 *porder) | 653 | u64 *pbase, u64 *porder) |
654 | { | 654 | { |
655 | u64 base_mask; | 655 | u64 base_mask; |
656 | u64 shifted_base = balloc_base_shift(a, base); | 656 | u64 shifted_base = balloc_base_shift(a, base); |
@@ -668,7 +668,7 @@ static void __balloc_get_parent_range(struct nvgpu_buddy_allocator *a, | |||
668 | * Makes a buddy at the passed address. This will make all parent buddies | 668 | * Makes a buddy at the passed address. This will make all parent buddies |
669 | * necessary for this buddy to exist as well. | 669 | * necessary for this buddy to exist as well. |
670 | */ | 670 | */ |
671 | static struct nvgpu_buddy *__balloc_make_fixed_buddy( | 671 | static struct nvgpu_buddy *balloc_make_fixed_buddy( |
672 | struct nvgpu_buddy_allocator *a, u64 base, u64 order, u32 pte_size) | 672 | struct nvgpu_buddy_allocator *a, u64 base, u64 order, u32 pte_size) |
673 | { | 673 | { |
674 | struct nvgpu_buddy *bud = NULL; | 674 | struct nvgpu_buddy *bud = NULL; |
@@ -714,8 +714,8 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy( | |||
714 | break; | 714 | break; |
715 | } | 715 | } |
716 | 716 | ||
717 | __balloc_get_parent_range(a, cur_base, cur_order, | 717 | balloc_get_parent_range(a, cur_base, cur_order, |
718 | &cur_base, &cur_order); | 718 | &cur_base, &cur_order); |
719 | } | 719 | } |
720 | 720 | ||
721 | if (cur_order > a->max_order) { | 721 | if (cur_order > a->max_order) { |
@@ -744,9 +744,9 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy( | |||
744 | return bud; | 744 | return bud; |
745 | } | 745 | } |
746 | 746 | ||
747 | static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a, | 747 | static u64 balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a, |
748 | struct nvgpu_fixed_alloc *falloc, | 748 | struct nvgpu_fixed_alloc *falloc, |
749 | u64 base, u64 len, u32 pte_size) | 749 | u64 base, u64 len, u32 pte_size) |
750 | { | 750 | { |
751 | u64 shifted_base, inc_base; | 751 | u64 shifted_base, inc_base; |
752 | u64 align_order; | 752 | u64 align_order; |
@@ -784,7 +784,7 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a, | |||
784 | u64 remaining; | 784 | u64 remaining; |
785 | struct nvgpu_buddy *bud; | 785 | struct nvgpu_buddy *bud; |
786 | 786 | ||
787 | bud = __balloc_make_fixed_buddy(a, | 787 | bud = balloc_make_fixed_buddy(a, |
788 | balloc_base_unshift(a, inc_base), | 788 | balloc_base_unshift(a, inc_base), |
789 | align_order, pte_size); | 789 | align_order, pte_size); |
790 | if (!bud) { | 790 | if (!bud) { |
@@ -797,7 +797,7 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a, | |||
797 | 797 | ||
798 | balloc_blist_rem(a, bud); | 798 | balloc_blist_rem(a, bud); |
799 | balloc_alloc_buddy(a, bud); | 799 | balloc_alloc_buddy(a, bud); |
800 | __balloc_buddy_list_add(a, bud, &falloc->buddies); | 800 | balloc_buddy_list_do_add(a, bud, &falloc->buddies); |
801 | 801 | ||
802 | /* Book keeping. */ | 802 | /* Book keeping. */ |
803 | inc_base += order_len; | 803 | inc_base += order_len; |
@@ -806,8 +806,8 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a, | |||
806 | 806 | ||
807 | /* If we don't have much left - trim down align_order. */ | 807 | /* If we don't have much left - trim down align_order. */ |
808 | if (balloc_order_to_len(a, align_order) > remaining) { | 808 | if (balloc_order_to_len(a, align_order) > remaining) { |
809 | align_order = __balloc_max_order_in(a, inc_base, | 809 | align_order = balloc_max_order_in(a, inc_base, |
810 | inc_base + remaining); | 810 | inc_base + remaining); |
811 | } | 811 | } |
812 | } | 812 | } |
813 | 813 | ||
@@ -819,7 +819,7 @@ err_and_cleanup: | |||
819 | &falloc->buddies, | 819 | &falloc->buddies, |
820 | nvgpu_buddy, buddy_entry); | 820 | nvgpu_buddy, buddy_entry); |
821 | 821 | ||
822 | __balloc_buddy_list_rem(a, bud); | 822 | balloc_buddy_list_do_rem(a, bud); |
823 | balloc_free_buddy(a, bud->start); | 823 | balloc_free_buddy(a, bud->start); |
824 | nvgpu_kmem_cache_free(a->buddy_cache, bud); | 824 | nvgpu_kmem_cache_free(a->buddy_cache, bud); |
825 | } | 825 | } |
@@ -827,8 +827,8 @@ err_and_cleanup: | |||
827 | return 0; | 827 | return 0; |
828 | } | 828 | } |
829 | 829 | ||
830 | static void __balloc_do_free_fixed(struct nvgpu_buddy_allocator *a, | 830 | static void balloc_do_free_fixed(struct nvgpu_buddy_allocator *a, |
831 | struct nvgpu_fixed_alloc *falloc) | 831 | struct nvgpu_fixed_alloc *falloc) |
832 | { | 832 | { |
833 | struct nvgpu_buddy *bud; | 833 | struct nvgpu_buddy *bud; |
834 | 834 | ||
@@ -836,7 +836,7 @@ static void __balloc_do_free_fixed(struct nvgpu_buddy_allocator *a, | |||
836 | bud = nvgpu_list_first_entry(&falloc->buddies, | 836 | bud = nvgpu_list_first_entry(&falloc->buddies, |
837 | nvgpu_buddy, | 837 | nvgpu_buddy, |
838 | buddy_entry); | 838 | buddy_entry); |
839 | __balloc_buddy_list_rem(a, bud); | 839 | balloc_buddy_list_do_rem(a, bud); |
840 | 840 | ||
841 | balloc_free_buddy(a, bud->start); | 841 | balloc_free_buddy(a, bud->start); |
842 | balloc_blist_add(a, bud); | 842 | balloc_blist_add(a, bud); |
@@ -876,7 +876,7 @@ static u64 nvgpu_buddy_balloc_pte(struct nvgpu_allocator *na, u64 len, | |||
876 | return 0ULL; | 876 | return 0ULL; |
877 | } | 877 | } |
878 | 878 | ||
879 | addr = __balloc_do_alloc(a, order, pte_size); | 879 | addr = balloc_do_alloc(a, order, pte_size); |
880 | 880 | ||
881 | if (addr != 0ULL) { | 881 | if (addr != 0ULL) { |
882 | a->bytes_alloced += len; | 882 | a->bytes_alloced += len; |
@@ -903,8 +903,11 @@ static u64 nvgpu_buddy_balloc(struct nvgpu_allocator *na, u64 len) | |||
903 | return nvgpu_buddy_balloc_pte(na, len, BALLOC_PTE_SIZE_ANY); | 903 | return nvgpu_buddy_balloc_pte(na, len, BALLOC_PTE_SIZE_ANY); |
904 | } | 904 | } |
905 | 905 | ||
906 | static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *na, | 906 | /* |
907 | u64 base, u64 len, u32 page_size) | 907 | * Requires @na to be locked. |
908 | */ | ||
909 | static u64 nvgpu_balloc_fixed_buddy_locked(struct nvgpu_allocator *na, | ||
910 | u64 base, u64 len, u32 page_size) | ||
908 | { | 911 | { |
909 | u32 pte_size; | 912 | u32 pte_size; |
910 | u64 ret, real_bytes = 0; | 913 | u64 ret, real_bytes = 0; |
@@ -942,7 +945,7 @@ static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *na, | |||
942 | goto fail; | 945 | goto fail; |
943 | } | 946 | } |
944 | 947 | ||
945 | ret = __balloc_do_alloc_fixed(a, falloc, base, len, pte_size); | 948 | ret = balloc_do_alloc_fixed(a, falloc, base, len, pte_size); |
946 | if (!ret) { | 949 | if (!ret) { |
947 | alloc_dbg(balloc_owner(a), | 950 | alloc_dbg(balloc_owner(a), |
948 | "Alloc-fixed failed ?? 0x%llx -> 0x%llx", | 951 | "Alloc-fixed failed ?? 0x%llx -> 0x%llx", |
@@ -984,7 +987,7 @@ static u64 nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *na, | |||
984 | struct nvgpu_buddy_allocator *a = na->priv; | 987 | struct nvgpu_buddy_allocator *a = na->priv; |
985 | 988 | ||
986 | alloc_lock(na); | 989 | alloc_lock(na); |
987 | alloc = __nvgpu_balloc_fixed_buddy(na, base, len, page_size); | 990 | alloc = nvgpu_balloc_fixed_buddy_locked(na, base, len, page_size); |
988 | a->alloc_made = 1; | 991 | a->alloc_made = 1; |
989 | alloc_unlock(na); | 992 | alloc_unlock(na); |
990 | 993 | ||
@@ -1012,7 +1015,7 @@ static void nvgpu_buddy_bfree(struct nvgpu_allocator *na, u64 addr) | |||
1012 | */ | 1015 | */ |
1013 | falloc = balloc_free_fixed(a, addr); | 1016 | falloc = balloc_free_fixed(a, addr); |
1014 | if (falloc) { | 1017 | if (falloc) { |
1015 | __balloc_do_free_fixed(a, falloc); | 1018 | balloc_do_free_fixed(a, falloc); |
1016 | goto done; | 1019 | goto done; |
1017 | } | 1020 | } |
1018 | 1021 | ||
@@ -1085,8 +1088,8 @@ static int nvgpu_buddy_reserve_co(struct nvgpu_allocator *na, | |||
1085 | } | 1088 | } |
1086 | 1089 | ||
1087 | /* Should not be possible to fail... */ | 1090 | /* Should not be possible to fail... */ |
1088 | addr = __nvgpu_balloc_fixed_buddy(na, co->base, co->length, | 1091 | addr = nvgpu_balloc_fixed_buddy_locked(na, co->base, co->length, |
1089 | BALLOC_PTE_SIZE_ANY); | 1092 | BALLOC_PTE_SIZE_ANY); |
1090 | if (!addr) { | 1093 | if (!addr) { |
1091 | err = -ENOMEM; | 1094 | err = -ENOMEM; |
1092 | nvgpu_warn(na->g, | 1095 | nvgpu_warn(na->g, |