diff options
author | Alex Waterman <alexw@nvidia.com> | 2016-10-04 17:04:01 -0400 |
---|---|---|
committer | mobile promotions <svcmobile_promotions@nvidia.com> | 2017-02-10 14:57:31 -0500 |
commit | cf0ef133e6e3075c808bb3f553b35e94fc000afd (patch) | |
tree | 13cace143fe43034791570514a05bf06cfcc47cb /drivers/gpu/nvgpu/common/mm | |
parent | 24e8ee192a02ff75a569f8c734efa9b4b5267fc8 (diff) |
gpu: nvgpu: Move kmem_caches to allocator
Instead of using a single static kmem_cache for each type of
data structure the allocators may want to allocate each
allocator now has its own instance of the kmem_cache. This is
done so that each GPU driver instance can accurately track how
much memory it is using.
In order to support this on older kernels a new NVGPU API has
been made,
nvgpu_kmem_cache_create(struct gk20a *g, size_t size)
To handle the possibility that caches cannot be created with
the same name.
This patch also fixes numerous places where kfree() was wrongly
used to free kmem_cache allocs.
Bug 1799159
Bug 1823380
Change-Id: Id674f9a5445fde3f95db65ad6bf3ea990444603d
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: http://git-master/r/1283826
Reviewed-by: svccoveritychecker <svccoveritychecker@nvidia.com>
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
GVS: Gerrit_Virtual_Submit
Diffstat (limited to 'drivers/gpu/nvgpu/common/mm')
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/bitmap_allocator.c | 36 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h | 6 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/buddy_allocator.c | 27 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h | 6 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/page_allocator.c | 73 |
5 files changed, 81 insertions, 67 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c b/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c index 5042980f..6fc508d6 100644 --- a/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c +++ b/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c | |||
@@ -1,5 +1,5 @@ | |||
1 | /* | 1 | /* |
2 | * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. | 2 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. |
3 | * | 3 | * |
4 | * This program is free software; you can redistribute it and/or modify it | 4 | * This program is free software; you can redistribute it and/or modify it |
5 | * under the terms and conditions of the GNU General Public License, | 5 | * under the terms and conditions of the GNU General Public License, |
@@ -22,9 +22,6 @@ | |||
22 | 22 | ||
23 | #include "bitmap_allocator_priv.h" | 23 | #include "bitmap_allocator_priv.h" |
24 | 24 | ||
25 | static struct kmem_cache *meta_data_cache; /* slab cache for meta data. */ | ||
26 | static DEFINE_MUTEX(meta_data_cache_lock); | ||
27 | |||
28 | static u64 nvgpu_bitmap_alloc_length(struct nvgpu_allocator *a) | 25 | static u64 nvgpu_bitmap_alloc_length(struct nvgpu_allocator *a) |
29 | { | 26 | { |
30 | struct nvgpu_bitmap_allocator *ba = a->priv; | 27 | struct nvgpu_bitmap_allocator *ba = a->priv; |
@@ -195,7 +192,7 @@ static int __nvgpu_bitmap_store_alloc(struct nvgpu_bitmap_allocator *a, | |||
195 | u64 addr, u64 len) | 192 | u64 addr, u64 len) |
196 | { | 193 | { |
197 | struct nvgpu_bitmap_alloc *alloc = | 194 | struct nvgpu_bitmap_alloc *alloc = |
198 | kmem_cache_alloc(meta_data_cache, GFP_KERNEL); | 195 | nvgpu_kmem_cache_alloc(a->meta_data_cache); |
199 | 196 | ||
200 | if (!alloc) | 197 | if (!alloc) |
201 | return -ENOMEM; | 198 | return -ENOMEM; |
@@ -312,7 +309,8 @@ static void nvgpu_bitmap_free(struct nvgpu_allocator *__a, u64 addr) | |||
312 | a->bytes_freed += alloc->length; | 309 | a->bytes_freed += alloc->length; |
313 | 310 | ||
314 | done: | 311 | done: |
315 | kfree(alloc); | 312 | if (a->meta_data_cache && alloc) |
313 | nvgpu_kmem_cache_free(a->meta_data_cache, alloc); | ||
316 | alloc_unlock(__a); | 314 | alloc_unlock(__a); |
317 | } | 315 | } |
318 | 316 | ||
@@ -330,9 +328,10 @@ static void nvgpu_bitmap_alloc_destroy(struct nvgpu_allocator *__a) | |||
330 | alloc_entry); | 328 | alloc_entry); |
331 | 329 | ||
332 | rb_erase(node, &a->allocs); | 330 | rb_erase(node, &a->allocs); |
333 | kfree(alloc); | 331 | nvgpu_kmem_cache_free(a->meta_data_cache, alloc); |
334 | } | 332 | } |
335 | 333 | ||
334 | nvgpu_kmem_cache_destroy(a->meta_data_cache); | ||
336 | kfree(a->bitmap); | 335 | kfree(a->bitmap); |
337 | kfree(a); | 336 | kfree(a); |
338 | } | 337 | } |
@@ -382,14 +381,6 @@ int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
382 | int err; | 381 | int err; |
383 | struct nvgpu_bitmap_allocator *a; | 382 | struct nvgpu_bitmap_allocator *a; |
384 | 383 | ||
385 | mutex_lock(&meta_data_cache_lock); | ||
386 | if (!meta_data_cache) | ||
387 | meta_data_cache = KMEM_CACHE(nvgpu_bitmap_alloc, 0); | ||
388 | mutex_unlock(&meta_data_cache_lock); | ||
389 | |||
390 | if (!meta_data_cache) | ||
391 | return -ENOMEM; | ||
392 | |||
393 | if (WARN_ON(blk_size & (blk_size - 1))) | 384 | if (WARN_ON(blk_size & (blk_size - 1))) |
394 | return -EINVAL; | 385 | return -EINVAL; |
395 | 386 | ||
@@ -414,6 +405,15 @@ int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
414 | if (err) | 405 | if (err) |
415 | goto fail; | 406 | goto fail; |
416 | 407 | ||
408 | if (!(flags & GPU_ALLOC_NO_ALLOC_PAGE)) { | ||
409 | a->meta_data_cache = nvgpu_kmem_cache_create(g, | ||
410 | sizeof(struct nvgpu_bitmap_alloc)); | ||
411 | if (!a->meta_data_cache) { | ||
412 | err = -ENOMEM; | ||
413 | goto fail; | ||
414 | } | ||
415 | } | ||
416 | |||
417 | a->base = base; | 417 | a->base = base; |
418 | a->length = length; | 418 | a->length = length; |
419 | a->blk_size = blk_size; | 419 | a->blk_size = blk_size; |
@@ -424,8 +424,10 @@ int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
424 | 424 | ||
425 | a->bitmap = kcalloc(BITS_TO_LONGS(a->num_bits), sizeof(*a->bitmap), | 425 | a->bitmap = kcalloc(BITS_TO_LONGS(a->num_bits), sizeof(*a->bitmap), |
426 | GFP_KERNEL); | 426 | GFP_KERNEL); |
427 | if (!a->bitmap) | 427 | if (!a->bitmap) { |
428 | err = -ENOMEM; | ||
428 | goto fail; | 429 | goto fail; |
430 | } | ||
429 | 431 | ||
430 | wmb(); | 432 | wmb(); |
431 | a->inited = true; | 433 | a->inited = true; |
@@ -441,6 +443,8 @@ int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
441 | return 0; | 443 | return 0; |
442 | 444 | ||
443 | fail: | 445 | fail: |
446 | if (a->meta_data_cache) | ||
447 | nvgpu_kmem_cache_destroy(a->meta_data_cache); | ||
444 | kfree(a); | 448 | kfree(a); |
445 | return err; | 449 | return err; |
446 | } | 450 | } |
diff --git a/drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h index 9802b9db..95780202 100644 --- a/drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h +++ b/drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h | |||
@@ -1,5 +1,5 @@ | |||
1 | /* | 1 | /* |
2 | * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. | 2 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. |
3 | * | 3 | * |
4 | * This program is free software; you can redistribute it and/or modify it | 4 | * This program is free software; you can redistribute it and/or modify it |
5 | * under the terms and conditions of the GNU General Public License, | 5 | * under the terms and conditions of the GNU General Public License, |
@@ -19,6 +19,8 @@ | |||
19 | 19 | ||
20 | #include <linux/rbtree.h> | 20 | #include <linux/rbtree.h> |
21 | 21 | ||
22 | #include <nvgpu/kmem.h> | ||
23 | |||
22 | struct nvgpu_allocator; | 24 | struct nvgpu_allocator; |
23 | 25 | ||
24 | struct nvgpu_bitmap_allocator { | 26 | struct nvgpu_bitmap_allocator { |
@@ -43,6 +45,8 @@ struct nvgpu_bitmap_allocator { | |||
43 | unsigned long *bitmap; /* The actual bitmap! */ | 45 | unsigned long *bitmap; /* The actual bitmap! */ |
44 | struct rb_root allocs; /* Tree of outstanding allocations. */ | 46 | struct rb_root allocs; /* Tree of outstanding allocations. */ |
45 | 47 | ||
48 | struct nvgpu_kmem_cache *meta_data_cache; | ||
49 | |||
46 | u64 flags; | 50 | u64 flags; |
47 | 51 | ||
48 | bool inited; | 52 | bool inited; |
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c index eee0b634..6f4c670a 100644 --- a/drivers/gpu/nvgpu/common/mm/buddy_allocator.c +++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c | |||
@@ -1,5 +1,5 @@ | |||
1 | /* | 1 | /* |
2 | * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. | 2 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. |
3 | * | 3 | * |
4 | * This program is free software; you can redistribute it and/or modify it | 4 | * This program is free software; you can redistribute it and/or modify it |
5 | * under the terms and conditions of the GNU General Public License, | 5 | * under the terms and conditions of the GNU General Public License, |
@@ -24,8 +24,6 @@ | |||
24 | 24 | ||
25 | #include "buddy_allocator_priv.h" | 25 | #include "buddy_allocator_priv.h" |
26 | 26 | ||
27 | static struct kmem_cache *buddy_cache; /* slab cache for meta data. */ | ||
28 | |||
29 | /* Some other buddy allocator functions. */ | 27 | /* Some other buddy allocator functions. */ |
30 | static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a, | 28 | static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a, |
31 | u64 addr); | 29 | u64 addr); |
@@ -103,7 +101,7 @@ static struct nvgpu_buddy *balloc_new_buddy(struct nvgpu_buddy_allocator *a, | |||
103 | { | 101 | { |
104 | struct nvgpu_buddy *new_buddy; | 102 | struct nvgpu_buddy *new_buddy; |
105 | 103 | ||
106 | new_buddy = kmem_cache_alloc(buddy_cache, GFP_KERNEL); | 104 | new_buddy = nvgpu_kmem_cache_alloc(a->buddy_cache); |
107 | if (!new_buddy) | 105 | if (!new_buddy) |
108 | return NULL; | 106 | return NULL; |
109 | 107 | ||
@@ -232,7 +230,7 @@ cleanup: | |||
232 | buddy = list_first_entry(balloc_get_order_list(a, i), | 230 | buddy = list_first_entry(balloc_get_order_list(a, i), |
233 | struct nvgpu_buddy, buddy_entry); | 231 | struct nvgpu_buddy, buddy_entry); |
234 | balloc_blist_rem(a, buddy); | 232 | balloc_blist_rem(a, buddy); |
235 | kmem_cache_free(buddy_cache, buddy); | 233 | nvgpu_kmem_cache_free(a->buddy_cache, buddy); |
236 | } | 234 | } |
237 | } | 235 | } |
238 | 236 | ||
@@ -285,7 +283,7 @@ static void nvgpu_buddy_allocator_destroy(struct nvgpu_allocator *__a) | |||
285 | bud = list_first_entry(balloc_get_order_list(a, i), | 283 | bud = list_first_entry(balloc_get_order_list(a, i), |
286 | struct nvgpu_buddy, buddy_entry); | 284 | struct nvgpu_buddy, buddy_entry); |
287 | balloc_blist_rem(a, bud); | 285 | balloc_blist_rem(a, bud); |
288 | kmem_cache_free(buddy_cache, bud); | 286 | nvgpu_kmem_cache_free(a->buddy_cache, bud); |
289 | } | 287 | } |
290 | 288 | ||
291 | if (a->buddy_list_len[i] != 0) { | 289 | if (a->buddy_list_len[i] != 0) { |
@@ -305,6 +303,7 @@ static void nvgpu_buddy_allocator_destroy(struct nvgpu_allocator *__a) | |||
305 | } | 303 | } |
306 | } | 304 | } |
307 | 305 | ||
306 | nvgpu_kmem_cache_destroy(a->buddy_cache); | ||
308 | kfree(a); | 307 | kfree(a); |
309 | 308 | ||
310 | alloc_unlock(__a); | 309 | alloc_unlock(__a); |
@@ -348,8 +347,8 @@ static void balloc_coalesce(struct nvgpu_buddy_allocator *a, | |||
348 | balloc_coalesce(a, parent); | 347 | balloc_coalesce(a, parent); |
349 | 348 | ||
350 | /* Clean up the remains. */ | 349 | /* Clean up the remains. */ |
351 | kmem_cache_free(buddy_cache, b->buddy); | 350 | nvgpu_kmem_cache_free(a->buddy_cache, b->buddy); |
352 | kmem_cache_free(buddy_cache, b); | 351 | nvgpu_kmem_cache_free(a->buddy_cache, b); |
353 | } | 352 | } |
354 | 353 | ||
355 | /* | 354 | /* |
@@ -371,7 +370,7 @@ static int balloc_split_buddy(struct nvgpu_buddy_allocator *a, | |||
371 | 370 | ||
372 | right = balloc_new_buddy(a, b, b->start + half, b->order - 1); | 371 | right = balloc_new_buddy(a, b, b->start + half, b->order - 1); |
373 | if (!right) { | 372 | if (!right) { |
374 | kmem_cache_free(buddy_cache, left); | 373 | nvgpu_kmem_cache_free(a->buddy_cache, left); |
375 | return -ENOMEM; | 374 | return -ENOMEM; |
376 | } | 375 | } |
377 | 376 | ||
@@ -783,7 +782,7 @@ err_and_cleanup: | |||
783 | 782 | ||
784 | __balloc_buddy_list_rem(a, bud); | 783 | __balloc_buddy_list_rem(a, bud); |
785 | balloc_free_buddy(a, bud->start); | 784 | balloc_free_buddy(a, bud->start); |
786 | kmem_cache_free(buddy_cache, bud); | 785 | nvgpu_kmem_cache_free(a->buddy_cache, bud); |
787 | } | 786 | } |
788 | 787 | ||
789 | return 0; | 788 | return 0; |
@@ -1307,10 +1306,8 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
1307 | balloc_allocator_align(a); | 1306 | balloc_allocator_align(a); |
1308 | balloc_compute_max_order(a); | 1307 | balloc_compute_max_order(a); |
1309 | 1308 | ||
1310 | /* Shared buddy kmem_cache for all allocators. */ | 1309 | a->buddy_cache = nvgpu_kmem_cache_create(g, sizeof(struct nvgpu_buddy)); |
1311 | if (!buddy_cache) | 1310 | if (!a->buddy_cache) { |
1312 | buddy_cache = KMEM_CACHE(nvgpu_buddy, 0); | ||
1313 | if (!buddy_cache) { | ||
1314 | err = -ENOMEM; | 1311 | err = -ENOMEM; |
1315 | goto fail; | 1312 | goto fail; |
1316 | } | 1313 | } |
@@ -1340,6 +1337,8 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
1340 | return 0; | 1337 | return 0; |
1341 | 1338 | ||
1342 | fail: | 1339 | fail: |
1340 | if (a->buddy_cache) | ||
1341 | nvgpu_kmem_cache_destroy(a->buddy_cache); | ||
1343 | kfree(a); | 1342 | kfree(a); |
1344 | return err; | 1343 | return err; |
1345 | } | 1344 | } |
diff --git a/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h index 50a11f14..56aaea62 100644 --- a/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h +++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h | |||
@@ -1,5 +1,5 @@ | |||
1 | /* | 1 | /* |
2 | * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. | 2 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. |
3 | * | 3 | * |
4 | * This program is free software; you can redistribute it and/or modify it | 4 | * This program is free software; you can redistribute it and/or modify it |
5 | * under the terms and conditions of the GNU General Public License, | 5 | * under the terms and conditions of the GNU General Public License, |
@@ -20,6 +20,8 @@ | |||
20 | #include <linux/list.h> | 20 | #include <linux/list.h> |
21 | #include <linux/rbtree.h> | 21 | #include <linux/rbtree.h> |
22 | 22 | ||
23 | #include <nvgpu/kmem.h> | ||
24 | |||
23 | struct nvgpu_allocator; | 25 | struct nvgpu_allocator; |
24 | struct vm_gk20a; | 26 | struct vm_gk20a; |
25 | 27 | ||
@@ -126,6 +128,8 @@ struct nvgpu_buddy_allocator { | |||
126 | 128 | ||
127 | struct list_head co_list; | 129 | struct list_head co_list; |
128 | 130 | ||
131 | struct nvgpu_kmem_cache *buddy_cache; | ||
132 | |||
129 | /* | 133 | /* |
130 | * Impose an upper bound on the maximum order. | 134 | * Impose an upper bound on the maximum order. |
131 | */ | 135 | */ |
diff --git a/drivers/gpu/nvgpu/common/mm/page_allocator.c b/drivers/gpu/nvgpu/common/mm/page_allocator.c index 056c0c8f..193decc9 100644 --- a/drivers/gpu/nvgpu/common/mm/page_allocator.c +++ b/drivers/gpu/nvgpu/common/mm/page_allocator.c | |||
@@ -27,11 +27,6 @@ | |||
27 | #define palloc_dbg(a, fmt, arg...) \ | 27 | #define palloc_dbg(a, fmt, arg...) \ |
28 | alloc_dbg(palloc_owner(a), fmt, ##arg) | 28 | alloc_dbg(palloc_owner(a), fmt, ##arg) |
29 | 29 | ||
30 | static struct kmem_cache *page_alloc_cache; | ||
31 | static struct kmem_cache *page_alloc_chunk_cache; | ||
32 | static struct kmem_cache *page_alloc_slab_page_cache; | ||
33 | static DEFINE_MUTEX(meta_data_cache_lock); | ||
34 | |||
35 | /* | 30 | /* |
36 | * Handle the book-keeping for these operations. | 31 | * Handle the book-keeping for these operations. |
37 | */ | 32 | */ |
@@ -147,10 +142,10 @@ static void __nvgpu_free_pages(struct nvgpu_page_allocator *a, | |||
147 | 142 | ||
148 | if (free_buddy_alloc) | 143 | if (free_buddy_alloc) |
149 | nvgpu_free(&a->source_allocator, chunk->base); | 144 | nvgpu_free(&a->source_allocator, chunk->base); |
150 | kfree(chunk); | 145 | nvgpu_kmem_cache_free(a->chunk_cache, chunk); |
151 | } | 146 | } |
152 | 147 | ||
153 | kmem_cache_free(page_alloc_cache, alloc); | 148 | nvgpu_kmem_cache_free(a->alloc_cache, alloc); |
154 | } | 149 | } |
155 | 150 | ||
156 | static int __insert_page_alloc(struct nvgpu_page_allocator *a, | 151 | static int __insert_page_alloc(struct nvgpu_page_allocator *a, |
@@ -213,7 +208,7 @@ static struct page_alloc_slab_page *alloc_slab_page( | |||
213 | { | 208 | { |
214 | struct page_alloc_slab_page *slab_page; | 209 | struct page_alloc_slab_page *slab_page; |
215 | 210 | ||
216 | slab_page = kmem_cache_alloc(page_alloc_slab_page_cache, GFP_KERNEL); | 211 | slab_page = nvgpu_kmem_cache_alloc(a->slab_page_cache); |
217 | if (!slab_page) { | 212 | if (!slab_page) { |
218 | palloc_dbg(a, "OOM: unable to alloc slab_page struct!\n"); | 213 | palloc_dbg(a, "OOM: unable to alloc slab_page struct!\n"); |
219 | return ERR_PTR(-ENOMEM); | 214 | return ERR_PTR(-ENOMEM); |
@@ -223,7 +218,7 @@ static struct page_alloc_slab_page *alloc_slab_page( | |||
223 | 218 | ||
224 | slab_page->page_addr = nvgpu_alloc(&a->source_allocator, a->page_size); | 219 | slab_page->page_addr = nvgpu_alloc(&a->source_allocator, a->page_size); |
225 | if (!slab_page->page_addr) { | 220 | if (!slab_page->page_addr) { |
226 | kfree(slab_page); | 221 | nvgpu_kmem_cache_free(a->slab_page_cache, slab_page); |
227 | palloc_dbg(a, "OOM: vidmem is full!\n"); | 222 | palloc_dbg(a, "OOM: vidmem is full!\n"); |
228 | return ERR_PTR(-ENOMEM); | 223 | return ERR_PTR(-ENOMEM); |
229 | } | 224 | } |
@@ -255,7 +250,7 @@ static void free_slab_page(struct nvgpu_page_allocator *a, | |||
255 | nvgpu_free(&a->source_allocator, slab_page->page_addr); | 250 | nvgpu_free(&a->source_allocator, slab_page->page_addr); |
256 | a->pages_freed++; | 251 | a->pages_freed++; |
257 | 252 | ||
258 | kmem_cache_free(page_alloc_slab_page_cache, slab_page); | 253 | nvgpu_kmem_cache_free(a->slab_page_cache, slab_page); |
259 | } | 254 | } |
260 | 255 | ||
261 | /* | 256 | /* |
@@ -352,12 +347,12 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_slab( | |||
352 | slab_nr = (int)ilog2(PAGE_ALIGN(len) >> 12); | 347 | slab_nr = (int)ilog2(PAGE_ALIGN(len) >> 12); |
353 | slab = &a->slabs[slab_nr]; | 348 | slab = &a->slabs[slab_nr]; |
354 | 349 | ||
355 | alloc = kmem_cache_alloc(page_alloc_cache, GFP_KERNEL); | 350 | alloc = nvgpu_kmem_cache_alloc(a->alloc_cache); |
356 | if (!alloc) { | 351 | if (!alloc) { |
357 | palloc_dbg(a, "OOM: could not alloc page_alloc struct!\n"); | 352 | palloc_dbg(a, "OOM: could not alloc page_alloc struct!\n"); |
358 | goto fail; | 353 | goto fail; |
359 | } | 354 | } |
360 | chunk = kmem_cache_alloc(page_alloc_chunk_cache, GFP_KERNEL); | 355 | chunk = nvgpu_kmem_cache_alloc(a->chunk_cache); |
361 | if (!chunk) { | 356 | if (!chunk) { |
362 | palloc_dbg(a, "OOM: could not alloc alloc_chunk struct!\n"); | 357 | palloc_dbg(a, "OOM: could not alloc alloc_chunk struct!\n"); |
363 | goto fail; | 358 | goto fail; |
@@ -377,8 +372,10 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_slab( | |||
377 | return alloc; | 372 | return alloc; |
378 | 373 | ||
379 | fail: | 374 | fail: |
380 | kfree(alloc); | 375 | if (alloc) |
381 | kfree(chunk); | 376 | nvgpu_kmem_cache_free(a->alloc_cache, alloc); |
377 | if (chunk) | ||
378 | nvgpu_kmem_cache_free(a->chunk_cache, chunk); | ||
382 | return NULL; | 379 | return NULL; |
383 | } | 380 | } |
384 | 381 | ||
@@ -444,7 +441,7 @@ static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages( | |||
444 | u64 max_chunk_len = pages << a->page_shift; | 441 | u64 max_chunk_len = pages << a->page_shift; |
445 | int i = 0; | 442 | int i = 0; |
446 | 443 | ||
447 | alloc = kmem_cache_alloc(page_alloc_cache, GFP_KERNEL); | 444 | alloc = nvgpu_kmem_cache_alloc(a->alloc_cache); |
448 | if (!alloc) | 445 | if (!alloc) |
449 | goto fail; | 446 | goto fail; |
450 | 447 | ||
@@ -496,7 +493,7 @@ static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages( | |||
496 | goto fail_cleanup; | 493 | goto fail_cleanup; |
497 | } | 494 | } |
498 | 495 | ||
499 | c = kmem_cache_alloc(page_alloc_chunk_cache, GFP_KERNEL); | 496 | c = nvgpu_kmem_cache_alloc(a->chunk_cache); |
500 | if (!c) { | 497 | if (!c) { |
501 | nvgpu_free(&a->source_allocator, chunk_addr); | 498 | nvgpu_free(&a->source_allocator, chunk_addr); |
502 | goto fail_cleanup; | 499 | goto fail_cleanup; |
@@ -524,9 +521,9 @@ fail_cleanup: | |||
524 | struct page_alloc_chunk, list_entry); | 521 | struct page_alloc_chunk, list_entry); |
525 | list_del(&c->list_entry); | 522 | list_del(&c->list_entry); |
526 | nvgpu_free(&a->source_allocator, c->base); | 523 | nvgpu_free(&a->source_allocator, c->base); |
527 | kfree(c); | 524 | nvgpu_kmem_cache_free(a->chunk_cache, c); |
528 | } | 525 | } |
529 | kfree(alloc); | 526 | nvgpu_kmem_cache_free(a->alloc_cache, alloc); |
530 | fail: | 527 | fail: |
531 | return ERR_PTR(-ENOMEM); | 528 | return ERR_PTR(-ENOMEM); |
532 | } | 529 | } |
@@ -653,8 +650,8 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_pages_fixed( | |||
653 | struct nvgpu_page_alloc *alloc; | 650 | struct nvgpu_page_alloc *alloc; |
654 | struct page_alloc_chunk *c; | 651 | struct page_alloc_chunk *c; |
655 | 652 | ||
656 | alloc = kmem_cache_alloc(page_alloc_cache, GFP_KERNEL); | 653 | alloc = nvgpu_kmem_cache_alloc(a->alloc_cache); |
657 | c = kmem_cache_alloc(page_alloc_chunk_cache, GFP_KERNEL); | 654 | c = nvgpu_kmem_cache_alloc(a->chunk_cache); |
658 | if (!alloc || !c) | 655 | if (!alloc || !c) |
659 | goto fail; | 656 | goto fail; |
660 | 657 | ||
@@ -675,8 +672,10 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_pages_fixed( | |||
675 | return alloc; | 672 | return alloc; |
676 | 673 | ||
677 | fail: | 674 | fail: |
678 | kfree(c); | 675 | if (c) |
679 | kfree(alloc); | 676 | nvgpu_kmem_cache_free(a->chunk_cache, c); |
677 | if (alloc) | ||
678 | nvgpu_kmem_cache_free(a->alloc_cache, alloc); | ||
680 | return ERR_PTR(-ENOMEM); | 679 | return ERR_PTR(-ENOMEM); |
681 | } | 680 | } |
682 | 681 | ||
@@ -879,19 +878,6 @@ int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
879 | char buddy_name[sizeof(__a->name)]; | 878 | char buddy_name[sizeof(__a->name)]; |
880 | int err; | 879 | int err; |
881 | 880 | ||
882 | mutex_lock(&meta_data_cache_lock); | ||
883 | if (!page_alloc_cache) | ||
884 | page_alloc_cache = KMEM_CACHE(nvgpu_page_alloc, 0); | ||
885 | if (!page_alloc_chunk_cache) | ||
886 | page_alloc_chunk_cache = KMEM_CACHE(page_alloc_chunk, 0); | ||
887 | if (!page_alloc_slab_page_cache) | ||
888 | page_alloc_slab_page_cache = | ||
889 | KMEM_CACHE(page_alloc_slab_page, 0); | ||
890 | mutex_unlock(&meta_data_cache_lock); | ||
891 | |||
892 | if (!page_alloc_cache || !page_alloc_chunk_cache) | ||
893 | return -ENOMEM; | ||
894 | |||
895 | if (blk_size < SZ_4K) | 881 | if (blk_size < SZ_4K) |
896 | return -EINVAL; | 882 | return -EINVAL; |
897 | 883 | ||
@@ -903,6 +889,17 @@ int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
903 | if (err) | 889 | if (err) |
904 | goto fail; | 890 | goto fail; |
905 | 891 | ||
892 | a->alloc_cache = nvgpu_kmem_cache_create(g, | ||
893 | sizeof(struct nvgpu_page_alloc)); | ||
894 | a->chunk_cache = nvgpu_kmem_cache_create(g, | ||
895 | sizeof(struct page_alloc_chunk)); | ||
896 | a->slab_page_cache = nvgpu_kmem_cache_create(g, | ||
897 | sizeof(struct page_alloc_slab_page)); | ||
898 | if (!a->alloc_cache || !a->chunk_cache || !a->slab_page_cache) { | ||
899 | err = -ENOMEM; | ||
900 | goto fail; | ||
901 | } | ||
902 | |||
906 | a->base = base; | 903 | a->base = base; |
907 | a->length = length; | 904 | a->length = length; |
908 | a->page_size = blk_size; | 905 | a->page_size = blk_size; |
@@ -935,6 +932,12 @@ int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
935 | return 0; | 932 | return 0; |
936 | 933 | ||
937 | fail: | 934 | fail: |
935 | if (a->alloc_cache) | ||
936 | nvgpu_kmem_cache_destroy(a->alloc_cache); | ||
937 | if (a->chunk_cache) | ||
938 | nvgpu_kmem_cache_destroy(a->chunk_cache); | ||
939 | if (a->slab_page_cache) | ||
940 | nvgpu_kmem_cache_destroy(a->slab_page_cache); | ||
938 | kfree(a); | 941 | kfree(a); |
939 | return err; | 942 | return err; |
940 | } | 943 | } |