diff options
Diffstat (limited to 'drivers/gpu/nvgpu/include')
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/allocator.h | 302 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/page_allocator.h | 164 |
2 files changed, 466 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/include/nvgpu/allocator.h b/drivers/gpu/nvgpu/include/nvgpu/allocator.h new file mode 100644 index 00000000..dee9b562 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/allocator.h | |||
@@ -0,0 +1,302 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2011-2016, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify it | ||
5 | * under the terms and conditions of the GNU General Public License, | ||
6 | * version 2, as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | * more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | |||
17 | #ifndef NVGPU_ALLOCATOR_H | ||
18 | #define NVGPU_ALLOCATOR_H | ||
19 | |||
20 | #include <linux/debugfs.h> | ||
21 | #include <linux/seq_file.h> | ||
22 | #include <linux/platform_device.h> | ||
23 | |||
24 | /* #define ALLOCATOR_DEBUG */ | ||
25 | |||
26 | struct nvgpu_allocator; | ||
27 | struct nvgpu_alloc_carveout; | ||
28 | struct vm_gk20a; | ||
29 | struct gk20a; | ||
30 | |||
31 | /* | ||
32 | * Operations for an allocator to implement. | ||
33 | */ | ||
34 | struct nvgpu_allocator_ops { | ||
35 | u64 (*alloc)(struct nvgpu_allocator *allocator, u64 len); | ||
36 | void (*free)(struct nvgpu_allocator *allocator, u64 addr); | ||
37 | |||
38 | /* | ||
39 | * Special interface to allocate a memory region with a specific | ||
40 | * starting address. Yikes. Note: if free() works for freeing both | ||
41 | * regular and fixed allocations then free_fixed() does not need to | ||
42 | * be implemented. This behavior exists for legacy reasons and should | ||
43 | * not be propagated to new allocators. | ||
44 | */ | ||
45 | u64 (*alloc_fixed)(struct nvgpu_allocator *allocator, | ||
46 | u64 base, u64 len); | ||
47 | void (*free_fixed)(struct nvgpu_allocator *allocator, | ||
48 | u64 base, u64 len); | ||
49 | |||
50 | /* | ||
51 | * Allow allocators to reserve space for carveouts. | ||
52 | */ | ||
53 | int (*reserve_carveout)(struct nvgpu_allocator *allocator, | ||
54 | struct nvgpu_alloc_carveout *co); | ||
55 | void (*release_carveout)(struct nvgpu_allocator *allocator, | ||
56 | struct nvgpu_alloc_carveout *co); | ||
57 | |||
58 | /* | ||
59 | * Returns info about the allocator. | ||
60 | */ | ||
61 | u64 (*base)(struct nvgpu_allocator *allocator); | ||
62 | u64 (*length)(struct nvgpu_allocator *allocator); | ||
63 | u64 (*end)(struct nvgpu_allocator *allocator); | ||
64 | int (*inited)(struct nvgpu_allocator *allocator); | ||
65 | u64 (*space)(struct nvgpu_allocator *allocator); | ||
66 | |||
67 | /* Destructor. */ | ||
68 | void (*fini)(struct nvgpu_allocator *allocator); | ||
69 | |||
70 | /* Debugging. */ | ||
71 | void (*print_stats)(struct nvgpu_allocator *allocator, | ||
72 | struct seq_file *s, int lock); | ||
73 | }; | ||
74 | |||
75 | struct nvgpu_allocator { | ||
76 | char name[32]; | ||
77 | struct mutex lock; | ||
78 | |||
79 | void *priv; | ||
80 | const struct nvgpu_allocator_ops *ops; | ||
81 | |||
82 | struct dentry *debugfs_entry; | ||
83 | bool debug; /* Control for debug msgs. */ | ||
84 | }; | ||
85 | |||
86 | struct nvgpu_alloc_carveout { | ||
87 | const char *name; | ||
88 | u64 base; | ||
89 | u64 length; | ||
90 | |||
91 | struct nvgpu_allocator *allocator; | ||
92 | |||
93 | /* | ||
94 | * For usage by the allocator implementation. | ||
95 | */ | ||
96 | struct list_head co_entry; | ||
97 | }; | ||
98 | |||
99 | #define NVGPU_CARVEOUT(__name, __base, __length) \ | ||
100 | { \ | ||
101 | .name = (__name), \ | ||
102 | .base = (__base), \ | ||
103 | .length = (__length) \ | ||
104 | } | ||
105 | |||
106 | /* | ||
107 | * These are the available allocator flags. | ||
108 | * | ||
109 | * GPU_ALLOC_GVA_SPACE | ||
110 | * | ||
111 | * This flag makes sense for the buddy allocator only. It specifies that the | ||
112 | * allocator will be used for managing a GVA space. When managing GVA spaces | ||
113 | * special care has to be taken to ensure that allocations of similar PTE | ||
114 | * sizes are placed in the same PDE block. This allows the higher level | ||
115 | * code to skip defining both small and large PTE tables for every PDE. That | ||
116 | * can save considerable memory for address spaces that have a lot of | ||
117 | * allocations. | ||
118 | * | ||
119 | * GPU_ALLOC_NO_ALLOC_PAGE | ||
120 | * | ||
121 | * For any allocator that needs to manage a resource in a latency critical | ||
122 | * path this flag specifies that the allocator should not use any kmalloc() | ||
123 | * or similar functions during normal operation. Initialization routines | ||
124 | * may still use kmalloc(). This prevents the possibility of long waits for | ||
125 | * pages when using alloc_page(). Currently only the bitmap allocator | ||
126 | * implements this functionality. | ||
127 | * | ||
128 | * Also note that if you accept this flag then you must also define the | ||
129 | * free_fixed() function. Since no meta-data is allocated to help free | ||
130 | * allocations you need to keep track of the meta-data yourself (in this | ||
131 | * case the base and length of the allocation as opposed to just the base | ||
132 | * of the allocation). | ||
133 | * | ||
134 | * GPU_ALLOC_4K_VIDMEM_PAGES | ||
135 | * | ||
136 | * We manage vidmem pages at a large page granularity for performance | ||
137 | * reasons; however, this can lead to wasting memory. For page allocators | ||
138 | * setting this flag will tell the allocator to manage pools of 4K pages | ||
139 | * inside internally allocated large pages. | ||
140 | * | ||
141 | * Currently this flag is ignored since the only usage of the page allocator | ||
142 | * uses a 4K block size already. However, this flag has been reserved since | ||
143 | * it will be necessary in the future. | ||
144 | * | ||
145 | * GPU_ALLOC_FORCE_CONTIG | ||
146 | * | ||
147 | * Force allocations to be contiguous. Currently only relevant for page | ||
148 | * allocators since all other allocators are naturally contiguous. | ||
149 | * | ||
150 | * GPU_ALLOC_NO_SCATTER_GATHER | ||
151 | * | ||
152 | * The page allocator normally returns a scatter gather data structure for | ||
153 | * allocations (to handle discontiguous pages). However, at times that can | ||
154 | * be annoying so this flag forces the page allocator to return a u64 | ||
155 | * pointing to the allocation base (requires GPU_ALLOC_FORCE_CONTIG to be | ||
156 | * set as well). | ||
157 | */ | ||
158 | #define GPU_ALLOC_GVA_SPACE 0x1 | ||
159 | #define GPU_ALLOC_NO_ALLOC_PAGE 0x2 | ||
160 | #define GPU_ALLOC_4K_VIDMEM_PAGES 0x4 | ||
161 | #define GPU_ALLOC_FORCE_CONTIG 0x8 | ||
162 | #define GPU_ALLOC_NO_SCATTER_GATHER 0x10 | ||
163 | |||
164 | static inline void alloc_lock(struct nvgpu_allocator *a) | ||
165 | { | ||
166 | mutex_lock(&a->lock); | ||
167 | } | ||
168 | |||
169 | static inline void alloc_unlock(struct nvgpu_allocator *a) | ||
170 | { | ||
171 | mutex_unlock(&a->lock); | ||
172 | } | ||
173 | |||
174 | /* | ||
175 | * Buddy allocator specific initializers. | ||
176 | */ | ||
177 | int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, | ||
178 | struct vm_gk20a *vm, const char *name, | ||
179 | u64 base, u64 size, u64 blk_size, | ||
180 | u64 max_order, u64 flags); | ||
181 | int nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, | ||
182 | const char *name, u64 base, u64 size, | ||
183 | u64 blk_size, u64 flags); | ||
184 | |||
185 | /* | ||
186 | * Bitmap initializers. | ||
187 | */ | ||
188 | int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, | ||
189 | const char *name, u64 base, u64 length, | ||
190 | u64 blk_size, u64 flags); | ||
191 | |||
192 | /* | ||
193 | * Page allocator initializers. | ||
194 | */ | ||
195 | int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, | ||
196 | const char *name, u64 base, u64 length, | ||
197 | u64 blk_size, u64 flags); | ||
198 | |||
199 | /* | ||
200 | * Lockless allocatior initializers. | ||
201 | * Note: This allocator can only allocate fixed-size structures of a | ||
202 | * pre-defined size. | ||
203 | */ | ||
204 | int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, | ||
205 | const char *name, u64 base, u64 length, | ||
206 | u64 struct_size, u64 flags); | ||
207 | |||
208 | #define GPU_BALLOC_MAX_ORDER 31 | ||
209 | |||
210 | /* | ||
211 | * Allocator APIs. | ||
212 | */ | ||
213 | u64 nvgpu_alloc(struct nvgpu_allocator *allocator, u64 len); | ||
214 | void nvgpu_free(struct nvgpu_allocator *allocator, u64 addr); | ||
215 | |||
216 | u64 nvgpu_alloc_fixed(struct nvgpu_allocator *allocator, u64 base, u64 len); | ||
217 | void nvgpu_free_fixed(struct nvgpu_allocator *allocator, u64 base, u64 len); | ||
218 | |||
219 | int nvgpu_alloc_reserve_carveout(struct nvgpu_allocator *a, | ||
220 | struct nvgpu_alloc_carveout *co); | ||
221 | void nvgpu_alloc_release_carveout(struct nvgpu_allocator *a, | ||
222 | struct nvgpu_alloc_carveout *co); | ||
223 | |||
224 | u64 nvgpu_alloc_base(struct nvgpu_allocator *a); | ||
225 | u64 nvgpu_alloc_length(struct nvgpu_allocator *a); | ||
226 | u64 nvgpu_alloc_end(struct nvgpu_allocator *a); | ||
227 | u64 nvgpu_alloc_initialized(struct nvgpu_allocator *a); | ||
228 | u64 nvgpu_alloc_space(struct nvgpu_allocator *a); | ||
229 | |||
230 | void nvgpu_alloc_destroy(struct nvgpu_allocator *allocator); | ||
231 | |||
232 | void nvgpu_alloc_print_stats(struct nvgpu_allocator *a, | ||
233 | struct seq_file *s, int lock); | ||
234 | |||
235 | /* | ||
236 | * Common functionality for the internals of the allocators. | ||
237 | */ | ||
238 | void nvgpu_init_alloc_debug(struct gk20a *g, struct nvgpu_allocator *a); | ||
239 | void nvgpu_fini_alloc_debug(struct nvgpu_allocator *a); | ||
240 | |||
241 | int __nvgpu_alloc_common_init(struct nvgpu_allocator *a, | ||
242 | const char *name, void *priv, bool dbg, | ||
243 | const struct nvgpu_allocator_ops *ops); | ||
244 | |||
245 | static inline void nvgpu_alloc_enable_dbg(struct nvgpu_allocator *a) | ||
246 | { | ||
247 | a->debug = true; | ||
248 | } | ||
249 | |||
250 | static inline void nvgpu_alloc_disable_dbg(struct nvgpu_allocator *a) | ||
251 | { | ||
252 | a->debug = false; | ||
253 | } | ||
254 | |||
255 | /* | ||
256 | * Debug stuff. | ||
257 | */ | ||
258 | extern u32 nvgpu_alloc_tracing_on; | ||
259 | |||
260 | void nvgpu_alloc_debugfs_init(struct device *dev); | ||
261 | |||
262 | #define nvgpu_alloc_trace_func() \ | ||
263 | do { \ | ||
264 | if (nvgpu_alloc_tracing_on) \ | ||
265 | trace_printk("%s\n", __func__); \ | ||
266 | } while (0) | ||
267 | |||
268 | #define nvgpu_alloc_trace_func_done() \ | ||
269 | do { \ | ||
270 | if (nvgpu_alloc_tracing_on) \ | ||
271 | trace_printk("%s_done\n", __func__); \ | ||
272 | } while (0) | ||
273 | |||
274 | #define __alloc_pstat(seq, allocator, fmt, arg...) \ | ||
275 | do { \ | ||
276 | if (s) \ | ||
277 | seq_printf(seq, fmt, ##arg); \ | ||
278 | else \ | ||
279 | alloc_dbg(allocator, fmt, ##arg); \ | ||
280 | } while (0) | ||
281 | |||
282 | #define __alloc_dbg(a, fmt, arg...) \ | ||
283 | pr_info("%-25s %25s() " fmt, (a)->name, __func__, ##arg) | ||
284 | |||
285 | #if defined(ALLOCATOR_DEBUG) | ||
286 | /* | ||
287 | * Always print the debug messages... | ||
288 | */ | ||
289 | #define alloc_dbg(a, fmt, arg...) __alloc_dbg(a, fmt, ##arg) | ||
290 | #else | ||
291 | /* | ||
292 | * Only print debug messages if debug is enabled for a given allocator. | ||
293 | */ | ||
294 | #define alloc_dbg(a, fmt, arg...) \ | ||
295 | do { \ | ||
296 | if ((a)->debug) \ | ||
297 | __alloc_dbg((a), fmt, ##arg); \ | ||
298 | } while (0) | ||
299 | |||
300 | #endif | ||
301 | |||
302 | #endif /* NVGPU_ALLOCATOR_H */ | ||
diff --git a/drivers/gpu/nvgpu/include/nvgpu/page_allocator.h b/drivers/gpu/nvgpu/include/nvgpu/page_allocator.h new file mode 100644 index 00000000..7c21c117 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/page_allocator.h | |||
@@ -0,0 +1,164 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify it | ||
5 | * under the terms and conditions of the GNU General Public License, | ||
6 | * version 2, as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | * more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | |||
17 | #ifndef PAGE_ALLOCATOR_PRIV_H | ||
18 | #define PAGE_ALLOCATOR_PRIV_H | ||
19 | |||
20 | #include <linux/list.h> | ||
21 | #include <linux/rbtree.h> | ||
22 | |||
23 | #include <nvgpu/allocator.h> | ||
24 | |||
25 | struct nvgpu_allocator; | ||
26 | |||
27 | /* | ||
28 | * This allocator implements the ability to do SLAB style allocation since the | ||
29 | * GPU has two page sizes available - 4k and 64k/128k. When the default | ||
30 | * granularity is the large page size (64k/128k) small allocations become very | ||
31 | * space inefficient. This is most notable in PDE and PTE blocks which are 4k | ||
32 | * in size. | ||
33 | * | ||
34 | * Thus we need the ability to suballocate in 64k pages. The way we do this for | ||
35 | * the GPU is as follows. We have several buckets for sub-64K allocations: | ||
36 | * | ||
37 | * B0 - 4k | ||
38 | * B1 - 8k | ||
39 | * B3 - 16k | ||
40 | * B4 - 32k | ||
41 | * B5 - 64k (for when large pages are 128k) | ||
42 | * | ||
43 | * When an allocation comes in for less than the large page size (from now on | ||
44 | * assumed to be 64k) the allocation is satisfied by one of the buckets. | ||
45 | */ | ||
46 | struct page_alloc_slab { | ||
47 | struct list_head empty; | ||
48 | struct list_head partial; | ||
49 | struct list_head full; | ||
50 | |||
51 | int nr_empty; | ||
52 | int nr_partial; | ||
53 | int nr_full; | ||
54 | |||
55 | u32 slab_size; | ||
56 | }; | ||
57 | |||
58 | enum slab_page_state { | ||
59 | SP_EMPTY, | ||
60 | SP_PARTIAL, | ||
61 | SP_FULL, | ||
62 | SP_NONE | ||
63 | }; | ||
64 | |||
65 | struct page_alloc_slab_page { | ||
66 | unsigned long bitmap; | ||
67 | u64 page_addr; | ||
68 | u32 slab_size; | ||
69 | |||
70 | u32 nr_objects; | ||
71 | u32 nr_objects_alloced; | ||
72 | |||
73 | enum slab_page_state state; | ||
74 | |||
75 | struct page_alloc_slab *owner; | ||
76 | struct list_head list_entry; | ||
77 | }; | ||
78 | |||
79 | struct page_alloc_chunk { | ||
80 | struct list_head list_entry; | ||
81 | |||
82 | u64 base; | ||
83 | u64 length; | ||
84 | }; | ||
85 | |||
86 | /* | ||
87 | * Struct to handle internal management of page allocation. It holds a list | ||
88 | * of the chunks of pages that make up the overall allocation - much like a | ||
89 | * scatter gather table. | ||
90 | */ | ||
91 | struct nvgpu_page_alloc { | ||
92 | struct list_head alloc_chunks; | ||
93 | |||
94 | int nr_chunks; | ||
95 | u64 length; | ||
96 | |||
97 | /* | ||
98 | * Only useful for the RB tree - since the alloc may have discontiguous | ||
99 | * pages the base is essentially irrelevant except for the fact that it | ||
100 | * is guarenteed to be unique. | ||
101 | */ | ||
102 | u64 base; | ||
103 | |||
104 | struct rb_node tree_entry; | ||
105 | |||
106 | /* | ||
107 | * Set if this is a slab alloc. Points back to the slab page that owns | ||
108 | * this particular allocation. nr_chunks will always be 1 if this is | ||
109 | * set. | ||
110 | */ | ||
111 | struct page_alloc_slab_page *slab_page; | ||
112 | }; | ||
113 | |||
114 | struct nvgpu_page_allocator { | ||
115 | struct nvgpu_allocator *owner; /* Owner of this allocator. */ | ||
116 | |||
117 | /* | ||
118 | * Use a buddy allocator to manage the allocation of the underlying | ||
119 | * pages. This lets us abstract the discontiguous allocation handling | ||
120 | * out of the annoyingly complicated buddy allocator. | ||
121 | */ | ||
122 | struct nvgpu_allocator source_allocator; | ||
123 | |||
124 | /* | ||
125 | * Page params. | ||
126 | */ | ||
127 | u64 base; | ||
128 | u64 length; | ||
129 | u64 page_size; | ||
130 | u32 page_shift; | ||
131 | |||
132 | struct rb_root allocs; /* Outstanding allocations. */ | ||
133 | |||
134 | struct page_alloc_slab *slabs; | ||
135 | int nr_slabs; | ||
136 | |||
137 | u64 flags; | ||
138 | |||
139 | /* | ||
140 | * Stat tracking. | ||
141 | */ | ||
142 | u64 nr_allocs; | ||
143 | u64 nr_frees; | ||
144 | u64 nr_fixed_allocs; | ||
145 | u64 nr_fixed_frees; | ||
146 | u64 nr_slab_allocs; | ||
147 | u64 nr_slab_frees; | ||
148 | u64 pages_alloced; | ||
149 | u64 pages_freed; | ||
150 | }; | ||
151 | |||
152 | static inline struct nvgpu_page_allocator *page_allocator( | ||
153 | struct nvgpu_allocator *a) | ||
154 | { | ||
155 | return (struct nvgpu_page_allocator *)(a)->priv; | ||
156 | } | ||
157 | |||
158 | static inline struct nvgpu_allocator *palloc_owner( | ||
159 | struct nvgpu_page_allocator *a) | ||
160 | { | ||
161 | return a->owner; | ||
162 | } | ||
163 | |||
164 | #endif | ||