diff options
author | Deepak Nibade <dnibade@nvidia.com> | 2017-04-11 09:59:10 -0400 |
---|---|---|
committer | mobile promotions <svcmobile_promotions@nvidia.com> | 2017-04-19 15:15:57 -0400 |
commit | a54fee533a2e3244343642b1ba1b211e2b53961c (patch) | |
tree | e17d4a8ef4a6559c8a1aa0a5bf17228f705914f6 /drivers/gpu/nvgpu | |
parent | 71c85c225ec205388fbaf5b482bfc3e54e7aa12e (diff) |
gpu: nvgpu: use nvgpu list for buddy allocator
Use nvgpu list APIs instead of linux list APIs
for buddy allocator lists
Jira NVGPU-13
Change-Id: I69a506a9aef77eaa9da0f89609627f4c2f5a7b28
Signed-off-by: Deepak Nibade <dnibade@nvidia.com>
Reviewed-on: http://git-master/r/1462079
Reviewed-by: svccoveritychecker <svccoveritychecker@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu')
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/buddy_allocator.c | 69 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h | 20 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/allocator.h | 10 |
3 files changed, 60 insertions, 39 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c index c015d64f..59227e4e 100644 --- a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c +++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c | |||
@@ -117,7 +117,7 @@ static struct nvgpu_buddy *balloc_new_buddy(struct nvgpu_buddy_allocator *a, | |||
117 | 117 | ||
118 | static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a, | 118 | static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a, |
119 | struct nvgpu_buddy *b, | 119 | struct nvgpu_buddy *b, |
120 | struct list_head *list) | 120 | struct nvgpu_list_node *list) |
121 | { | 121 | { |
122 | if (buddy_is_in_list(b)) { | 122 | if (buddy_is_in_list(b)) { |
123 | alloc_dbg(balloc_owner(a), | 123 | alloc_dbg(balloc_owner(a), |
@@ -133,9 +133,9 @@ static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a, | |||
133 | */ | 133 | */ |
134 | if (a->flags & GPU_ALLOC_GVA_SPACE && | 134 | if (a->flags & GPU_ALLOC_GVA_SPACE && |
135 | b->pte_size == gmmu_page_size_big) | 135 | b->pte_size == gmmu_page_size_big) |
136 | list_add_tail(&b->buddy_entry, list); | 136 | nvgpu_list_add_tail(&b->buddy_entry, list); |
137 | else | 137 | else |
138 | list_add(&b->buddy_entry, list); | 138 | nvgpu_list_add(&b->buddy_entry, list); |
139 | 139 | ||
140 | buddy_set_in_list(b); | 140 | buddy_set_in_list(b); |
141 | } | 141 | } |
@@ -150,7 +150,7 @@ static void __balloc_buddy_list_rem(struct nvgpu_buddy_allocator *a, | |||
150 | BUG(); | 150 | BUG(); |
151 | } | 151 | } |
152 | 152 | ||
153 | list_del_init(&b->buddy_entry); | 153 | nvgpu_list_del(&b->buddy_entry); |
154 | buddy_clr_in_list(b); | 154 | buddy_clr_in_list(b); |
155 | } | 155 | } |
156 | 156 | ||
@@ -208,7 +208,7 @@ static int balloc_init_lists(struct nvgpu_buddy_allocator *a) | |||
208 | 208 | ||
209 | /* First make sure the LLs are valid. */ | 209 | /* First make sure the LLs are valid. */ |
210 | for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++) | 210 | for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++) |
211 | INIT_LIST_HEAD(balloc_get_order_list(a, i)); | 211 | nvgpu_init_list_node(balloc_get_order_list(a, i)); |
212 | 212 | ||
213 | while (bstart < bend) { | 213 | while (bstart < bend) { |
214 | order = __balloc_max_order_in(a, bstart, bend); | 214 | order = __balloc_max_order_in(a, bstart, bend); |
@@ -225,9 +225,10 @@ static int balloc_init_lists(struct nvgpu_buddy_allocator *a) | |||
225 | 225 | ||
226 | cleanup: | 226 | cleanup: |
227 | for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++) { | 227 | for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++) { |
228 | if (!list_empty(balloc_get_order_list(a, i))) { | 228 | if (!nvgpu_list_empty(balloc_get_order_list(a, i))) { |
229 | buddy = list_first_entry(balloc_get_order_list(a, i), | 229 | buddy = nvgpu_list_first_entry( |
230 | struct nvgpu_buddy, buddy_entry); | 230 | balloc_get_order_list(a, i), |
231 | nvgpu_buddy, buddy_entry); | ||
231 | balloc_blist_rem(a, buddy); | 232 | balloc_blist_rem(a, buddy); |
232 | nvgpu_kmem_cache_free(a->buddy_cache, buddy); | 233 | nvgpu_kmem_cache_free(a->buddy_cache, buddy); |
233 | } | 234 | } |
@@ -278,9 +279,10 @@ static void nvgpu_buddy_allocator_destroy(struct nvgpu_allocator *__a) | |||
278 | for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++) { | 279 | for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++) { |
279 | BUG_ON(a->buddy_list_alloced[i] != 0); | 280 | BUG_ON(a->buddy_list_alloced[i] != 0); |
280 | 281 | ||
281 | while (!list_empty(balloc_get_order_list(a, i))) { | 282 | while (!nvgpu_list_empty(balloc_get_order_list(a, i))) { |
282 | bud = list_first_entry(balloc_get_order_list(a, i), | 283 | bud = nvgpu_list_first_entry( |
283 | struct nvgpu_buddy, buddy_entry); | 284 | balloc_get_order_list(a, i), |
285 | nvgpu_buddy, buddy_entry); | ||
284 | balloc_blist_rem(a, bud); | 286 | balloc_blist_rem(a, bud); |
285 | nvgpu_kmem_cache_free(a->buddy_cache, bud); | 287 | nvgpu_kmem_cache_free(a->buddy_cache, bud); |
286 | } | 288 | } |
@@ -471,16 +473,16 @@ static struct nvgpu_buddy *__balloc_find_buddy(struct nvgpu_buddy_allocator *a, | |||
471 | struct nvgpu_buddy *bud; | 473 | struct nvgpu_buddy *bud; |
472 | 474 | ||
473 | if (order > a->max_order || | 475 | if (order > a->max_order || |
474 | list_empty(balloc_get_order_list(a, order))) | 476 | nvgpu_list_empty(balloc_get_order_list(a, order))) |
475 | return NULL; | 477 | return NULL; |
476 | 478 | ||
477 | if (a->flags & GPU_ALLOC_GVA_SPACE && | 479 | if (a->flags & GPU_ALLOC_GVA_SPACE && |
478 | pte_size == gmmu_page_size_big) | 480 | pte_size == gmmu_page_size_big) |
479 | bud = list_last_entry(balloc_get_order_list(a, order), | 481 | bud = nvgpu_list_last_entry(balloc_get_order_list(a, order), |
480 | struct nvgpu_buddy, buddy_entry); | 482 | nvgpu_buddy, buddy_entry); |
481 | else | 483 | else |
482 | bud = list_first_entry(balloc_get_order_list(a, order), | 484 | bud = nvgpu_list_first_entry(balloc_get_order_list(a, order), |
483 | struct nvgpu_buddy, buddy_entry); | 485 | nvgpu_buddy, buddy_entry); |
484 | 486 | ||
485 | if (pte_size != BALLOC_PTE_SIZE_ANY && | 487 | if (pte_size != BALLOC_PTE_SIZE_ANY && |
486 | pte_size != bud->pte_size && | 488 | pte_size != bud->pte_size && |
@@ -645,7 +647,7 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy( | |||
645 | struct nvgpu_buddy_allocator *a, u64 base, u64 order, int pte_size) | 647 | struct nvgpu_buddy_allocator *a, u64 base, u64 order, int pte_size) |
646 | { | 648 | { |
647 | struct nvgpu_buddy *bud = NULL; | 649 | struct nvgpu_buddy *bud = NULL; |
648 | struct list_head *order_list; | 650 | struct nvgpu_list_node *order_list; |
649 | u64 cur_order = order, cur_base = base; | 651 | u64 cur_order = order, cur_base = base; |
650 | 652 | ||
651 | /* | 653 | /* |
@@ -661,7 +663,8 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy( | |||
661 | int found = 0; | 663 | int found = 0; |
662 | 664 | ||
663 | order_list = balloc_get_order_list(a, cur_order); | 665 | order_list = balloc_get_order_list(a, cur_order); |
664 | list_for_each_entry(bud, order_list, buddy_entry) { | 666 | nvgpu_list_for_each_entry(bud, order_list, |
667 | nvgpu_buddy, buddy_entry) { | ||
665 | if (bud->start == cur_base) { | 668 | if (bud->start == cur_base) { |
666 | /* | 669 | /* |
667 | * Make sure page size matches if it's smaller | 670 | * Make sure page size matches if it's smaller |
@@ -774,10 +777,10 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a, | |||
774 | return base; | 777 | return base; |
775 | 778 | ||
776 | err_and_cleanup: | 779 | err_and_cleanup: |
777 | while (!list_empty(&falloc->buddies)) { | 780 | while (!nvgpu_list_empty(&falloc->buddies)) { |
778 | struct nvgpu_buddy *bud = list_first_entry(&falloc->buddies, | 781 | struct nvgpu_buddy *bud = nvgpu_list_first_entry( |
779 | struct nvgpu_buddy, | 782 | &falloc->buddies, |
780 | buddy_entry); | 783 | nvgpu_buddy, buddy_entry); |
781 | 784 | ||
782 | __balloc_buddy_list_rem(a, bud); | 785 | __balloc_buddy_list_rem(a, bud); |
783 | balloc_free_buddy(a, bud->start); | 786 | balloc_free_buddy(a, bud->start); |
@@ -792,9 +795,9 @@ static void __balloc_do_free_fixed(struct nvgpu_buddy_allocator *a, | |||
792 | { | 795 | { |
793 | struct nvgpu_buddy *bud; | 796 | struct nvgpu_buddy *bud; |
794 | 797 | ||
795 | while (!list_empty(&falloc->buddies)) { | 798 | while (!nvgpu_list_empty(&falloc->buddies)) { |
796 | bud = list_first_entry(&falloc->buddies, | 799 | bud = nvgpu_list_first_entry(&falloc->buddies, |
797 | struct nvgpu_buddy, | 800 | nvgpu_buddy, |
798 | buddy_entry); | 801 | buddy_entry); |
799 | __balloc_buddy_list_rem(a, bud); | 802 | __balloc_buddy_list_rem(a, bud); |
800 | 803 | ||
@@ -896,7 +899,7 @@ static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a, | |||
896 | if (!falloc) | 899 | if (!falloc) |
897 | goto fail; | 900 | goto fail; |
898 | 901 | ||
899 | INIT_LIST_HEAD(&falloc->buddies); | 902 | nvgpu_init_list_node(&falloc->buddies); |
900 | falloc->start = base; | 903 | falloc->start = base; |
901 | falloc->end = base + len; | 904 | falloc->end = base + len; |
902 | 905 | ||
@@ -1018,7 +1021,8 @@ static bool nvgpu_buddy_reserve_is_possible(struct nvgpu_buddy_allocator *a, | |||
1018 | * Not the fastest approach but we should not have that many carveouts | 1021 | * Not the fastest approach but we should not have that many carveouts |
1019 | * for any reasonable allocator. | 1022 | * for any reasonable allocator. |
1020 | */ | 1023 | */ |
1021 | list_for_each_entry(tmp, &a->co_list, co_entry) { | 1024 | nvgpu_list_for_each_entry(tmp, &a->co_list, |
1025 | nvgpu_alloc_carveout, co_entry) { | ||
1022 | if ((co_base >= tmp->base && | 1026 | if ((co_base >= tmp->base && |
1023 | co_base < (tmp->base + tmp->length)) || | 1027 | co_base < (tmp->base + tmp->length)) || |
1024 | (co_end >= tmp->base && | 1028 | (co_end >= tmp->base && |
@@ -1059,7 +1063,7 @@ static int nvgpu_buddy_reserve_co(struct nvgpu_allocator *__a, | |||
1059 | goto done; | 1063 | goto done; |
1060 | } | 1064 | } |
1061 | 1065 | ||
1062 | list_add(&co->co_entry, &a->co_list); | 1066 | nvgpu_list_add(&co->co_entry, &a->co_list); |
1063 | 1067 | ||
1064 | done: | 1068 | done: |
1065 | alloc_unlock(__a); | 1069 | alloc_unlock(__a); |
@@ -1074,7 +1078,7 @@ static void nvgpu_buddy_release_co(struct nvgpu_allocator *__a, | |||
1074 | { | 1078 | { |
1075 | alloc_lock(__a); | 1079 | alloc_lock(__a); |
1076 | 1080 | ||
1077 | list_del_init(&co->co_entry); | 1081 | nvgpu_list_del(&co->co_entry); |
1078 | nvgpu_free(__a, co->base); | 1082 | nvgpu_free(__a, co->base); |
1079 | 1083 | ||
1080 | alloc_unlock(__a); | 1084 | alloc_unlock(__a); |
@@ -1149,10 +1153,11 @@ static void nvgpu_buddy_print_stats(struct nvgpu_allocator *__a, | |||
1149 | if (lock) | 1153 | if (lock) |
1150 | alloc_lock(__a); | 1154 | alloc_lock(__a); |
1151 | 1155 | ||
1152 | if (!list_empty(&a->co_list)) { | 1156 | if (!nvgpu_list_empty(&a->co_list)) { |
1153 | __alloc_pstat(s, __a, "\n"); | 1157 | __alloc_pstat(s, __a, "\n"); |
1154 | __alloc_pstat(s, __a, "Carveouts:\n"); | 1158 | __alloc_pstat(s, __a, "Carveouts:\n"); |
1155 | list_for_each_entry(tmp, &a->co_list, co_entry) | 1159 | nvgpu_list_for_each_entry(tmp, &a->co_list, |
1160 | nvgpu_alloc_carveout, co_entry) | ||
1156 | __alloc_pstat(s, __a, | 1161 | __alloc_pstat(s, __a, |
1157 | " CO %2d: %-20s 0x%010llx + 0x%llx\n", | 1162 | " CO %2d: %-20s 0x%010llx + 0x%llx\n", |
1158 | i++, tmp->name, tmp->base, tmp->length); | 1163 | i++, tmp->name, tmp->base, tmp->length); |
@@ -1313,7 +1318,7 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
1313 | 1318 | ||
1314 | a->alloced_buddies = RB_ROOT; | 1319 | a->alloced_buddies = RB_ROOT; |
1315 | a->fixed_allocs = RB_ROOT; | 1320 | a->fixed_allocs = RB_ROOT; |
1316 | INIT_LIST_HEAD(&a->co_list); | 1321 | nvgpu_init_list_node(&a->co_list); |
1317 | err = balloc_init_lists(a); | 1322 | err = balloc_init_lists(a); |
1318 | if (err) | 1323 | if (err) |
1319 | goto fail; | 1324 | goto fail; |
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h index 935b3f1c..549ab3d1 100644 --- a/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h +++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h | |||
@@ -17,9 +17,10 @@ | |||
17 | #ifndef BUDDY_ALLOCATOR_PRIV_H | 17 | #ifndef BUDDY_ALLOCATOR_PRIV_H |
18 | #define BUDDY_ALLOCATOR_PRIV_H | 18 | #define BUDDY_ALLOCATOR_PRIV_H |
19 | 19 | ||
20 | #include <linux/list.h> | ||
21 | #include <linux/rbtree.h> | 20 | #include <linux/rbtree.h> |
22 | 21 | ||
22 | #include <nvgpu/list.h> | ||
23 | |||
23 | struct nvgpu_kmem_cache; | 24 | struct nvgpu_kmem_cache; |
24 | struct nvgpu_allocator; | 25 | struct nvgpu_allocator; |
25 | struct vm_gk20a; | 26 | struct vm_gk20a; |
@@ -33,7 +34,7 @@ struct nvgpu_buddy { | |||
33 | struct nvgpu_buddy *left; /* Lower address sub-node. */ | 34 | struct nvgpu_buddy *left; /* Lower address sub-node. */ |
34 | struct nvgpu_buddy *right; /* Higher address sub-node. */ | 35 | struct nvgpu_buddy *right; /* Higher address sub-node. */ |
35 | 36 | ||
36 | struct list_head buddy_entry; /* List entry for various lists. */ | 37 | struct nvgpu_list_node buddy_entry; /* List entry for various lists. */ |
37 | struct rb_node alloced_entry; /* RB tree of allocations. */ | 38 | struct rb_node alloced_entry; /* RB tree of allocations. */ |
38 | 39 | ||
39 | u64 start; /* Start address of this buddy. */ | 40 | u64 start; /* Start address of this buddy. */ |
@@ -54,6 +55,13 @@ struct nvgpu_buddy { | |||
54 | int pte_size; | 55 | int pte_size; |
55 | }; | 56 | }; |
56 | 57 | ||
58 | static inline struct nvgpu_buddy * | ||
59 | nvgpu_buddy_from_buddy_entry(struct nvgpu_list_node *node) | ||
60 | { | ||
61 | return (struct nvgpu_buddy *) | ||
62 | ((uintptr_t)node - offsetof(struct nvgpu_buddy, buddy_entry)); | ||
63 | }; | ||
64 | |||
57 | #define __buddy_flag_ops(flag, flag_up) \ | 65 | #define __buddy_flag_ops(flag, flag_up) \ |
58 | static inline int buddy_is_ ## flag(struct nvgpu_buddy *b) \ | 66 | static inline int buddy_is_ ## flag(struct nvgpu_buddy *b) \ |
59 | { \ | 67 | { \ |
@@ -89,7 +97,7 @@ __buddy_flag_ops(in_list, IN_LIST); | |||
89 | * Keeps info for a fixed allocation. | 97 | * Keeps info for a fixed allocation. |
90 | */ | 98 | */ |
91 | struct nvgpu_fixed_alloc { | 99 | struct nvgpu_fixed_alloc { |
92 | struct list_head buddies; /* List of buddies. */ | 100 | struct nvgpu_list_node buddies; /* List of buddies. */ |
93 | struct rb_node alloced_entry; /* RB tree of fixed allocations. */ | 101 | struct rb_node alloced_entry; /* RB tree of fixed allocations. */ |
94 | 102 | ||
95 | u64 start; /* Start of fixed block. */ | 103 | u64 start; /* Start of fixed block. */ |
@@ -125,7 +133,7 @@ struct nvgpu_buddy_allocator { | |||
125 | struct rb_root alloced_buddies; /* Outstanding allocations. */ | 133 | struct rb_root alloced_buddies; /* Outstanding allocations. */ |
126 | struct rb_root fixed_allocs; /* Outstanding fixed allocations. */ | 134 | struct rb_root fixed_allocs; /* Outstanding fixed allocations. */ |
127 | 135 | ||
128 | struct list_head co_list; | 136 | struct nvgpu_list_node co_list; |
129 | 137 | ||
130 | struct nvgpu_kmem_cache *buddy_cache; | 138 | struct nvgpu_kmem_cache *buddy_cache; |
131 | 139 | ||
@@ -134,7 +142,7 @@ struct nvgpu_buddy_allocator { | |||
134 | */ | 142 | */ |
135 | #define GPU_BALLOC_ORDER_LIST_LEN (GPU_BALLOC_MAX_ORDER + 1) | 143 | #define GPU_BALLOC_ORDER_LIST_LEN (GPU_BALLOC_MAX_ORDER + 1) |
136 | 144 | ||
137 | struct list_head buddy_list[GPU_BALLOC_ORDER_LIST_LEN]; | 145 | struct nvgpu_list_node buddy_list[GPU_BALLOC_ORDER_LIST_LEN]; |
138 | u64 buddy_list_len[GPU_BALLOC_ORDER_LIST_LEN]; | 146 | u64 buddy_list_len[GPU_BALLOC_ORDER_LIST_LEN]; |
139 | u64 buddy_list_split[GPU_BALLOC_ORDER_LIST_LEN]; | 147 | u64 buddy_list_split[GPU_BALLOC_ORDER_LIST_LEN]; |
140 | u64 buddy_list_alloced[GPU_BALLOC_ORDER_LIST_LEN]; | 148 | u64 buddy_list_alloced[GPU_BALLOC_ORDER_LIST_LEN]; |
@@ -162,7 +170,7 @@ static inline struct nvgpu_buddy_allocator *buddy_allocator( | |||
162 | return (struct nvgpu_buddy_allocator *)(a)->priv; | 170 | return (struct nvgpu_buddy_allocator *)(a)->priv; |
163 | } | 171 | } |
164 | 172 | ||
165 | static inline struct list_head *balloc_get_order_list( | 173 | static inline struct nvgpu_list_node *balloc_get_order_list( |
166 | struct nvgpu_buddy_allocator *a, int order) | 174 | struct nvgpu_buddy_allocator *a, int order) |
167 | { | 175 | { |
168 | return &a->buddy_list[order]; | 176 | return &a->buddy_list[order]; |
diff --git a/drivers/gpu/nvgpu/include/nvgpu/allocator.h b/drivers/gpu/nvgpu/include/nvgpu/allocator.h index 1bde290f..61914d8c 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/allocator.h +++ b/drivers/gpu/nvgpu/include/nvgpu/allocator.h | |||
@@ -22,6 +22,7 @@ | |||
22 | #include <linux/platform_device.h> | 22 | #include <linux/platform_device.h> |
23 | 23 | ||
24 | #include <nvgpu/lock.h> | 24 | #include <nvgpu/lock.h> |
25 | #include <nvgpu/list.h> | ||
25 | 26 | ||
26 | /* #define ALLOCATOR_DEBUG */ | 27 | /* #define ALLOCATOR_DEBUG */ |
27 | 28 | ||
@@ -101,7 +102,14 @@ struct nvgpu_alloc_carveout { | |||
101 | /* | 102 | /* |
102 | * For usage by the allocator implementation. | 103 | * For usage by the allocator implementation. |
103 | */ | 104 | */ |
104 | struct list_head co_entry; | 105 | struct nvgpu_list_node co_entry; |
106 | }; | ||
107 | |||
108 | static inline struct nvgpu_alloc_carveout * | ||
109 | nvgpu_alloc_carveout_from_co_entry(struct nvgpu_list_node *node) | ||
110 | { | ||
111 | return (struct nvgpu_alloc_carveout *) | ||
112 | ((uintptr_t)node - offsetof(struct nvgpu_alloc_carveout, co_entry)); | ||
105 | }; | 113 | }; |
106 | 114 | ||
107 | #define NVGPU_CARVEOUT(__name, __base, __length) \ | 115 | #define NVGPU_CARVEOUT(__name, __base, __length) \ |