diff options
Diffstat (limited to 'drivers')
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/buddy_allocator_priv.h | 3 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/gk20a_allocator_buddy.c | 134 |
2 files changed, 124 insertions, 13 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/buddy_allocator_priv.h b/drivers/gpu/nvgpu/gk20a/buddy_allocator_priv.h index 1a6aed48..bb8b307b 100644 --- a/drivers/gpu/nvgpu/gk20a/buddy_allocator_priv.h +++ b/drivers/gpu/nvgpu/gk20a/buddy_allocator_priv.h | |||
@@ -124,6 +124,8 @@ struct gk20a_buddy_allocator { | |||
124 | struct rb_root alloced_buddies; /* Outstanding allocations. */ | 124 | struct rb_root alloced_buddies; /* Outstanding allocations. */ |
125 | struct rb_root fixed_allocs; /* Outstanding fixed allocations. */ | 125 | struct rb_root fixed_allocs; /* Outstanding fixed allocations. */ |
126 | 126 | ||
127 | struct list_head co_list; | ||
128 | |||
127 | /* | 129 | /* |
128 | * Impose an upper bound on the maximum order. | 130 | * Impose an upper bound on the maximum order. |
129 | */ | 131 | */ |
@@ -142,6 +144,7 @@ struct gk20a_buddy_allocator { | |||
142 | u64 pte_blk_order; | 144 | u64 pte_blk_order; |
143 | 145 | ||
144 | int initialized; | 146 | int initialized; |
147 | int alloc_made; /* True after the first alloc. */ | ||
145 | 148 | ||
146 | u64 flags; | 149 | u64 flags; |
147 | 150 | ||
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_allocator_buddy.c b/drivers/gpu/nvgpu/gk20a/gk20a_allocator_buddy.c index dd7f4d27..70a10c24 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a_allocator_buddy.c +++ b/drivers/gpu/nvgpu/gk20a/gk20a_allocator_buddy.c | |||
@@ -844,6 +844,8 @@ static u64 gk20a_buddy_balloc(struct gk20a_allocator *__a, u64 len) | |||
844 | alloc_dbg(balloc_owner(a), "Alloc failed: no mem!\n"); | 844 | alloc_dbg(balloc_owner(a), "Alloc failed: no mem!\n"); |
845 | } | 845 | } |
846 | 846 | ||
847 | a->alloc_made = 1; | ||
848 | |||
847 | alloc_unlock(__a); | 849 | alloc_unlock(__a); |
848 | 850 | ||
849 | gk20a_alloc_trace_func_done(); | 851 | gk20a_alloc_trace_func_done(); |
@@ -851,15 +853,10 @@ static u64 gk20a_buddy_balloc(struct gk20a_allocator *__a, u64 len) | |||
851 | } | 853 | } |
852 | 854 | ||
853 | /* | 855 | /* |
854 | * Allocate a fixed address allocation. The address of the allocation is @base | 856 | * Requires @__a to be locked. |
855 | * and the length is @len. This is not a typical buddy allocator operation and | ||
856 | * as such has a high posibility of failure if the address space is heavily in | ||
857 | * use. | ||
858 | * | ||
859 | * Please do not use this function unless _absolutely_ necessary. | ||
860 | */ | 857 | */ |
861 | static u64 gk20a_balloc_fixed_buddy(struct gk20a_allocator *__a, | 858 | static u64 __gk20a_balloc_fixed_buddy(struct gk20a_allocator *__a, |
862 | u64 base, u64 len) | 859 | u64 base, u64 len) |
863 | { | 860 | { |
864 | u64 ret, real_bytes = 0; | 861 | u64 ret, real_bytes = 0; |
865 | struct gk20a_buddy *bud; | 862 | struct gk20a_buddy *bud; |
@@ -883,7 +880,6 @@ static u64 gk20a_balloc_fixed_buddy(struct gk20a_allocator *__a, | |||
883 | falloc->start = base; | 880 | falloc->start = base; |
884 | falloc->end = base + len; | 881 | falloc->end = base + len; |
885 | 882 | ||
886 | alloc_lock(__a); | ||
887 | if (!balloc_is_range_free(a, base, base + len)) { | 883 | if (!balloc_is_range_free(a, base, base + len)) { |
888 | alloc_dbg(balloc_owner(a), | 884 | alloc_dbg(balloc_owner(a), |
889 | "Range not free: 0x%llx -> 0x%llx\n", | 885 | "Range not free: 0x%llx -> 0x%llx\n", |
@@ -907,7 +903,6 @@ static u64 gk20a_balloc_fixed_buddy(struct gk20a_allocator *__a, | |||
907 | a->bytes_alloced += len; | 903 | a->bytes_alloced += len; |
908 | a->bytes_alloced_real += real_bytes; | 904 | a->bytes_alloced_real += real_bytes; |
909 | 905 | ||
910 | alloc_unlock(__a); | ||
911 | alloc_dbg(balloc_owner(a), "Alloc (fixed) 0x%llx\n", base); | 906 | alloc_dbg(balloc_owner(a), "Alloc (fixed) 0x%llx\n", base); |
912 | 907 | ||
913 | gk20a_alloc_trace_func_done(); | 908 | gk20a_alloc_trace_func_done(); |
@@ -922,6 +917,28 @@ fail: | |||
922 | } | 917 | } |
923 | 918 | ||
924 | /* | 919 | /* |
920 | * Allocate a fixed address allocation. The address of the allocation is @base | ||
921 | * and the length is @len. This is not a typical buddy allocator operation and | ||
922 | * as such has a high posibility of failure if the address space is heavily in | ||
923 | * use. | ||
924 | * | ||
925 | * Please do not use this function unless _absolutely_ necessary. | ||
926 | */ | ||
927 | static u64 gk20a_balloc_fixed_buddy(struct gk20a_allocator *__a, | ||
928 | u64 base, u64 len) | ||
929 | { | ||
930 | u64 alloc; | ||
931 | struct gk20a_buddy_allocator *a = __a->priv; | ||
932 | |||
933 | alloc_lock(__a); | ||
934 | alloc = __gk20a_balloc_fixed_buddy(__a, base, len); | ||
935 | a->alloc_made = 1; | ||
936 | alloc_unlock(__a); | ||
937 | |||
938 | return alloc; | ||
939 | } | ||
940 | |||
941 | /* | ||
925 | * Free the passed allocation. | 942 | * Free the passed allocation. |
926 | */ | 943 | */ |
927 | static void gk20a_buddy_bfree(struct gk20a_allocator *__a, u64 addr) | 944 | static void gk20a_buddy_bfree(struct gk20a_allocator *__a, u64 addr) |
@@ -968,6 +985,81 @@ done: | |||
968 | return; | 985 | return; |
969 | } | 986 | } |
970 | 987 | ||
988 | static bool gk20a_buddy_reserve_is_possible(struct gk20a_buddy_allocator *a, | ||
989 | struct gk20a_alloc_carveout *co) | ||
990 | { | ||
991 | struct gk20a_alloc_carveout *tmp; | ||
992 | u64 co_base, co_end; | ||
993 | |||
994 | co_base = co->base; | ||
995 | co_end = co->base + co->length; | ||
996 | |||
997 | /* | ||
998 | * Not the fastest approach but we should not have that many carveouts | ||
999 | * for any reasonable allocator. | ||
1000 | */ | ||
1001 | list_for_each_entry(tmp, &a->co_list, co_entry) { | ||
1002 | if ((co_base >= tmp->base && | ||
1003 | co_base < (tmp->base + tmp->length)) || | ||
1004 | (co_end >= tmp->base && | ||
1005 | co_end < (tmp->base + tmp->length))) | ||
1006 | return false; | ||
1007 | } | ||
1008 | |||
1009 | return true; | ||
1010 | } | ||
1011 | |||
1012 | /* | ||
1013 | * Carveouts can only be reserved before any regular allocations have been | ||
1014 | * made. | ||
1015 | */ | ||
1016 | static int gk20a_buddy_reserve_co(struct gk20a_allocator *__a, | ||
1017 | struct gk20a_alloc_carveout *co) | ||
1018 | { | ||
1019 | struct gk20a_buddy_allocator *a = __a->priv; | ||
1020 | u64 addr; | ||
1021 | int err = 0; | ||
1022 | |||
1023 | if (co->base < a->start || (co->base + co->length) > a->end || | ||
1024 | a->alloc_made) | ||
1025 | return -EINVAL; | ||
1026 | |||
1027 | alloc_lock(__a); | ||
1028 | |||
1029 | if (!gk20a_buddy_reserve_is_possible(a, co)) { | ||
1030 | err = -EBUSY; | ||
1031 | goto done; | ||
1032 | } | ||
1033 | |||
1034 | /* Should not be possible to fail... */ | ||
1035 | addr = __gk20a_balloc_fixed_buddy(__a, co->base, co->length); | ||
1036 | if (!addr) { | ||
1037 | err = -ENOMEM; | ||
1038 | pr_warn("%s: Failed to reserve a valid carveout!\n", __func__); | ||
1039 | goto done; | ||
1040 | } | ||
1041 | |||
1042 | list_add(&co->co_entry, &a->co_list); | ||
1043 | |||
1044 | done: | ||
1045 | alloc_unlock(__a); | ||
1046 | return err; | ||
1047 | } | ||
1048 | |||
1049 | /* | ||
1050 | * Carveouts can be release at any time. | ||
1051 | */ | ||
1052 | static void gk20a_buddy_release_co(struct gk20a_allocator *__a, | ||
1053 | struct gk20a_alloc_carveout *co) | ||
1054 | { | ||
1055 | alloc_lock(__a); | ||
1056 | |||
1057 | list_del_init(&co->co_entry); | ||
1058 | gk20a_free(__a, co->base); | ||
1059 | |||
1060 | alloc_unlock(__a); | ||
1061 | } | ||
1062 | |||
971 | static u64 gk20a_buddy_alloc_length(struct gk20a_allocator *a) | 1063 | static u64 gk20a_buddy_alloc_length(struct gk20a_allocator *a) |
972 | { | 1064 | { |
973 | struct gk20a_buddy_allocator *ba = a->priv; | 1065 | struct gk20a_buddy_allocator *ba = a->priv; |
@@ -1004,9 +1096,10 @@ static u64 gk20a_buddy_alloc_end(struct gk20a_allocator *a) | |||
1004 | static void gk20a_buddy_print_stats(struct gk20a_allocator *__a, | 1096 | static void gk20a_buddy_print_stats(struct gk20a_allocator *__a, |
1005 | struct seq_file *s, int lock) | 1097 | struct seq_file *s, int lock) |
1006 | { | 1098 | { |
1007 | int i; | 1099 | int i = 0; |
1008 | struct rb_node *node; | 1100 | struct rb_node *node; |
1009 | struct gk20a_fixed_alloc *falloc; | 1101 | struct gk20a_fixed_alloc *falloc; |
1102 | struct gk20a_alloc_carveout *tmp; | ||
1010 | struct gk20a_buddy_allocator *a = __a->priv; | 1103 | struct gk20a_buddy_allocator *a = __a->priv; |
1011 | 1104 | ||
1012 | __alloc_pstat(s, __a, "base = %llu, limit = %llu, blk_size = %llu\n", | 1105 | __alloc_pstat(s, __a, "base = %llu, limit = %llu, blk_size = %llu\n", |
@@ -1018,12 +1111,23 @@ static void gk20a_buddy_print_stats(struct gk20a_allocator *__a, | |||
1018 | __alloc_pstat(s, __a, " blks = 0x%llx\n", a->blks); | 1111 | __alloc_pstat(s, __a, " blks = 0x%llx\n", a->blks); |
1019 | __alloc_pstat(s, __a, " max_order = %llu\n", a->max_order); | 1112 | __alloc_pstat(s, __a, " max_order = %llu\n", a->max_order); |
1020 | 1113 | ||
1114 | if (lock) | ||
1115 | alloc_lock(__a); | ||
1116 | |||
1117 | if (!list_empty(&a->co_list)) { | ||
1118 | __alloc_pstat(s, __a, "\n"); | ||
1119 | __alloc_pstat(s, __a, "Carveouts:\n"); | ||
1120 | list_for_each_entry(tmp, &a->co_list, co_entry) | ||
1121 | __alloc_pstat(s, __a, | ||
1122 | " CO %2d: %-20s 0x%010llx + 0x%llx\n", | ||
1123 | i++, tmp->name, tmp->base, tmp->length); | ||
1124 | } | ||
1125 | |||
1126 | __alloc_pstat(s, __a, "\n"); | ||
1021 | __alloc_pstat(s, __a, "Buddy blocks:\n"); | 1127 | __alloc_pstat(s, __a, "Buddy blocks:\n"); |
1022 | __alloc_pstat(s, __a, " Order Free Alloced Split\n"); | 1128 | __alloc_pstat(s, __a, " Order Free Alloced Split\n"); |
1023 | __alloc_pstat(s, __a, " ----- ---- ------- -----\n"); | 1129 | __alloc_pstat(s, __a, " ----- ---- ------- -----\n"); |
1024 | 1130 | ||
1025 | if (lock) | ||
1026 | alloc_lock(__a); | ||
1027 | for (i = a->max_order; i >= 0; i--) { | 1131 | for (i = a->max_order; i >= 0; i--) { |
1028 | if (a->buddy_list_len[i] == 0 && | 1132 | if (a->buddy_list_len[i] == 0 && |
1029 | a->buddy_list_alloced[i] == 0 && | 1133 | a->buddy_list_alloced[i] == 0 && |
@@ -1067,6 +1171,9 @@ static const struct gk20a_allocator_ops buddy_ops = { | |||
1067 | .alloc_fixed = gk20a_balloc_fixed_buddy, | 1171 | .alloc_fixed = gk20a_balloc_fixed_buddy, |
1068 | /* .free_fixed not needed. */ | 1172 | /* .free_fixed not needed. */ |
1069 | 1173 | ||
1174 | .reserve_carveout = gk20a_buddy_reserve_co, | ||
1175 | .release_carveout = gk20a_buddy_release_co, | ||
1176 | |||
1070 | .base = gk20a_buddy_alloc_base, | 1177 | .base = gk20a_buddy_alloc_base, |
1071 | .length = gk20a_buddy_alloc_length, | 1178 | .length = gk20a_buddy_alloc_length, |
1072 | .end = gk20a_buddy_alloc_end, | 1179 | .end = gk20a_buddy_alloc_end, |
@@ -1172,6 +1279,7 @@ int __gk20a_buddy_allocator_init(struct gk20a_allocator *__a, | |||
1172 | 1279 | ||
1173 | a->alloced_buddies = RB_ROOT; | 1280 | a->alloced_buddies = RB_ROOT; |
1174 | a->fixed_allocs = RB_ROOT; | 1281 | a->fixed_allocs = RB_ROOT; |
1282 | INIT_LIST_HEAD(&a->co_list); | ||
1175 | err = balloc_init_lists(a); | 1283 | err = balloc_init_lists(a); |
1176 | if (err) | 1284 | if (err) |
1177 | goto fail; | 1285 | goto fail; |