summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/mm/page_allocator.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common/mm/page_allocator.c')
-rw-r--r--drivers/gpu/nvgpu/common/mm/page_allocator.c110
1 files changed, 70 insertions, 40 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/page_allocator.c b/drivers/gpu/nvgpu/common/mm/page_allocator.c
index 773d33ef..d001a2aa 100644
--- a/drivers/gpu/nvgpu/common/mm/page_allocator.c
+++ b/drivers/gpu/nvgpu/common/mm/page_allocator.c
@@ -264,8 +264,9 @@ static struct nvgpu_page_alloc *__find_page_alloc(
264 struct nvgpu_rbtree_node *node = NULL; 264 struct nvgpu_rbtree_node *node = NULL;
265 265
266 nvgpu_rbtree_search(addr, &node, a->allocs); 266 nvgpu_rbtree_search(addr, &node, a->allocs);
267 if (!node) 267 if (!node) {
268 return NULL; 268 return NULL;
269 }
269 270
270 alloc = nvgpu_page_alloc_from_rbtree_node(node); 271 alloc = nvgpu_page_alloc_from_rbtree_node(node);
271 272
@@ -355,8 +356,9 @@ static int __do_slab_alloc(struct nvgpu_page_allocator *a,
355 356
356 if (!slab_page) { 357 if (!slab_page) {
357 slab_page = alloc_slab_page(a, slab); 358 slab_page = alloc_slab_page(a, slab);
358 if (!slab_page) 359 if (!slab_page) {
359 return -ENOMEM; 360 return -ENOMEM;
361 }
360 } 362 }
361 363
362 /* 364 /*
@@ -376,12 +378,13 @@ static int __do_slab_alloc(struct nvgpu_page_allocator *a,
376 bitmap_set(&slab_page->bitmap, offs, 1); 378 bitmap_set(&slab_page->bitmap, offs, 1);
377 slab_page->nr_objects_alloced++; 379 slab_page->nr_objects_alloced++;
378 380
379 if (slab_page->nr_objects_alloced < slab_page->nr_objects) 381 if (slab_page->nr_objects_alloced < slab_page->nr_objects) {
380 add_slab_page_to_partial(slab, slab_page); 382 add_slab_page_to_partial(slab, slab_page);
381 else if (slab_page->nr_objects_alloced == slab_page->nr_objects) 383 } else if (slab_page->nr_objects_alloced == slab_page->nr_objects) {
382 add_slab_page_to_full(slab, slab_page); 384 add_slab_page_to_full(slab, slab_page);
383 else 385 } else {
384 BUG(); /* Should be impossible to hit this. */ 386 BUG(); /* Should be impossible to hit this. */
387 }
385 388
386 /* 389 /*
387 * Handle building the nvgpu_page_alloc struct. We expect one sgl 390 * Handle building the nvgpu_page_alloc struct. We expect one sgl
@@ -435,8 +438,9 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_slab(
435 438
436 alloc->sgt.sgl = (struct nvgpu_sgl *)sgl; 439 alloc->sgt.sgl = (struct nvgpu_sgl *)sgl;
437 err = __do_slab_alloc(a, slab, alloc); 440 err = __do_slab_alloc(a, slab, alloc);
438 if (err) 441 if (err) {
439 goto fail; 442 goto fail;
443 }
440 444
441 palloc_dbg(a, "Alloc 0x%04llx sr=%d id=0x%010llx [slab]", 445 palloc_dbg(a, "Alloc 0x%04llx sr=%d id=0x%010llx [slab]",
442 len, slab_nr, alloc->base); 446 len, slab_nr, alloc->base);
@@ -445,10 +449,12 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_slab(
445 return alloc; 449 return alloc;
446 450
447fail: 451fail:
448 if (alloc) 452 if (alloc) {
449 nvgpu_kmem_cache_free(a->alloc_cache, alloc); 453 nvgpu_kmem_cache_free(a->alloc_cache, alloc);
450 if (sgl) 454 }
455 if (sgl) {
451 nvgpu_kfree(a->owner->g, sgl); 456 nvgpu_kfree(a->owner->g, sgl);
457 }
452 return NULL; 458 return NULL;
453} 459}
454 460
@@ -465,27 +471,30 @@ static void __nvgpu_free_slab(struct nvgpu_page_allocator *a,
465 471
466 slab_page->nr_objects_alloced--; 472 slab_page->nr_objects_alloced--;
467 473
468 if (slab_page->nr_objects_alloced == 0) 474 if (slab_page->nr_objects_alloced == 0) {
469 new_state = SP_EMPTY; 475 new_state = SP_EMPTY;
470 else 476 } else {
471 new_state = SP_PARTIAL; 477 new_state = SP_PARTIAL;
478 }
472 479
473 /* 480 /*
474 * Need to migrate the page to a different list. 481 * Need to migrate the page to a different list.
475 */ 482 */
476 if (new_state != slab_page->state) { 483 if (new_state != slab_page->state) {
477 /* Delete - can't be in empty. */ 484 /* Delete - can't be in empty. */
478 if (slab_page->state == SP_PARTIAL) 485 if (slab_page->state == SP_PARTIAL) {
479 del_slab_page_from_partial(slab, slab_page); 486 del_slab_page_from_partial(slab, slab_page);
480 else 487 } else {
481 del_slab_page_from_full(slab, slab_page); 488 del_slab_page_from_full(slab, slab_page);
489 }
482 490
483 /* And add. */ 491 /* And add. */
484 if (new_state == SP_EMPTY) { 492 if (new_state == SP_EMPTY) {
485 if (nvgpu_list_empty(&slab->empty)) 493 if (nvgpu_list_empty(&slab->empty)) {
486 add_slab_page_to_empty(slab, slab_page); 494 add_slab_page_to_empty(slab, slab_page);
487 else 495 } else {
488 free_slab_page(a, slab_page); 496 free_slab_page(a, slab_page);
497 }
489 } else { 498 } else {
490 add_slab_page_to_partial(slab, slab_page); 499 add_slab_page_to_partial(slab, slab_page);
491 } 500 }
@@ -515,8 +524,9 @@ static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages(
515 int i = 0; 524 int i = 0;
516 525
517 alloc = nvgpu_kmem_cache_alloc(a->alloc_cache); 526 alloc = nvgpu_kmem_cache_alloc(a->alloc_cache);
518 if (!alloc) 527 if (!alloc) {
519 goto fail; 528 goto fail;
529 }
520 530
521 memset(alloc, 0, sizeof(*alloc)); 531 memset(alloc, 0, sizeof(*alloc));
522 532
@@ -535,11 +545,13 @@ static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages(
535 * requested size. The buddy allocator guarantees any given 545 * requested size. The buddy allocator guarantees any given
536 * single alloc is contiguous. 546 * single alloc is contiguous.
537 */ 547 */
538 if (a->flags & GPU_ALLOC_FORCE_CONTIG && i != 0) 548 if (a->flags & GPU_ALLOC_FORCE_CONTIG && i != 0) {
539 goto fail_cleanup; 549 goto fail_cleanup;
550 }
540 551
541 if (chunk_len > max_chunk_len) 552 if (chunk_len > max_chunk_len) {
542 chunk_len = max_chunk_len; 553 chunk_len = max_chunk_len;
554 }
543 555
544 /* 556 /*
545 * Keep attempting to allocate in smaller chunks until the alloc 557 * Keep attempting to allocate in smaller chunks until the alloc
@@ -582,10 +594,11 @@ static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages(
582 * Build the singly linked list with a head node that is part of 594 * Build the singly linked list with a head node that is part of
583 * the list. 595 * the list.
584 */ 596 */
585 if (prev_sgl) 597 if (prev_sgl) {
586 prev_sgl->next = sgl; 598 prev_sgl->next = sgl;
587 else 599 } else {
588 alloc->sgt.sgl = (struct nvgpu_sgl *)sgl; 600 alloc->sgt.sgl = (struct nvgpu_sgl *)sgl;
601 }
589 602
590 prev_sgl = sgl; 603 prev_sgl = sgl;
591 604
@@ -671,10 +684,11 @@ static u64 nvgpu_page_alloc(struct nvgpu_allocator *__a, u64 len)
671 684
672 alloc_lock(__a); 685 alloc_lock(__a);
673 if (a->flags & GPU_ALLOC_4K_VIDMEM_PAGES && 686 if (a->flags & GPU_ALLOC_4K_VIDMEM_PAGES &&
674 real_len <= (a->page_size / 2)) 687 real_len <= (a->page_size / 2)) {
675 alloc = __nvgpu_alloc_slab(a, real_len); 688 alloc = __nvgpu_alloc_slab(a, real_len);
676 else 689 } else {
677 alloc = __nvgpu_alloc_pages(a, real_len); 690 alloc = __nvgpu_alloc_pages(a, real_len);
691 }
678 692
679 if (!alloc) { 693 if (!alloc) {
680 alloc_unlock(__a); 694 alloc_unlock(__a);
@@ -684,14 +698,16 @@ static u64 nvgpu_page_alloc(struct nvgpu_allocator *__a, u64 len)
684 __insert_page_alloc(a, alloc); 698 __insert_page_alloc(a, alloc);
685 699
686 a->nr_allocs++; 700 a->nr_allocs++;
687 if (real_len > a->page_size / 2) 701 if (real_len > a->page_size / 2) {
688 a->pages_alloced += alloc->length >> a->page_shift; 702 a->pages_alloced += alloc->length >> a->page_shift;
703 }
689 alloc_unlock(__a); 704 alloc_unlock(__a);
690 705
691 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) 706 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) {
692 return alloc->base; 707 return alloc->base;
693 else 708 } else {
694 return (u64) (uintptr_t) alloc; 709 return (u64) (uintptr_t) alloc;
710 }
695} 711}
696 712
697/* 713/*
@@ -705,11 +721,12 @@ static void nvgpu_page_free(struct nvgpu_allocator *__a, u64 base)
705 721
706 alloc_lock(__a); 722 alloc_lock(__a);
707 723
708 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) 724 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) {
709 alloc = __find_page_alloc(a, base); 725 alloc = __find_page_alloc(a, base);
710 else 726 } else {
711 alloc = __find_page_alloc(a, 727 alloc = __find_page_alloc(a,
712 ((struct nvgpu_page_alloc *)(uintptr_t)base)->base); 728 ((struct nvgpu_page_alloc *)(uintptr_t)base)->base);
729 }
713 730
714 if (!alloc) { 731 if (!alloc) {
715 palloc_dbg(a, "Hrm, found no alloc?"); 732 palloc_dbg(a, "Hrm, found no alloc?");
@@ -743,8 +760,9 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_pages_fixed(
743 760
744 alloc = nvgpu_kmem_cache_alloc(a->alloc_cache); 761 alloc = nvgpu_kmem_cache_alloc(a->alloc_cache);
745 sgl = nvgpu_kzalloc(a->owner->g, sizeof(*sgl)); 762 sgl = nvgpu_kzalloc(a->owner->g, sizeof(*sgl));
746 if (!alloc || !sgl) 763 if (!alloc || !sgl) {
747 goto fail; 764 goto fail;
765 }
748 766
749 alloc->sgt.ops = &page_alloc_sgl_ops; 767 alloc->sgt.ops = &page_alloc_sgl_ops;
750 alloc->base = nvgpu_alloc_fixed(&a->source_allocator, base, length, 0); 768 alloc->base = nvgpu_alloc_fixed(&a->source_allocator, base, length, 0);
@@ -765,10 +783,12 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_pages_fixed(
765 return alloc; 783 return alloc;
766 784
767fail: 785fail:
768 if (sgl) 786 if (sgl) {
769 nvgpu_kfree(a->owner->g, sgl); 787 nvgpu_kfree(a->owner->g, sgl);
770 if (alloc) 788 }
789 if (alloc) {
771 nvgpu_kmem_cache_free(a->alloc_cache, alloc); 790 nvgpu_kmem_cache_free(a->alloc_cache, alloc);
791 }
772 return NULL; 792 return NULL;
773} 793}
774 794
@@ -813,10 +833,11 @@ static u64 nvgpu_page_alloc_fixed(struct nvgpu_allocator *__a,
813 a->nr_fixed_allocs++; 833 a->nr_fixed_allocs++;
814 a->pages_alloced += pages; 834 a->pages_alloced += pages;
815 835
816 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) 836 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) {
817 return alloc->base; 837 return alloc->base;
818 else 838 } else {
819 return (u64) (uintptr_t) alloc; 839 return (u64) (uintptr_t) alloc;
840 }
820} 841}
821 842
822static void nvgpu_page_free_fixed(struct nvgpu_allocator *__a, 843static void nvgpu_page_free_fixed(struct nvgpu_allocator *__a,
@@ -829,8 +850,9 @@ static void nvgpu_page_free_fixed(struct nvgpu_allocator *__a,
829 850
830 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) { 851 if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) {
831 alloc = __find_page_alloc(a, base); 852 alloc = __find_page_alloc(a, base);
832 if (!alloc) 853 if (!alloc) {
833 goto done; 854 goto done;
855 }
834 } else { 856 } else {
835 alloc = (struct nvgpu_page_alloc *) (uintptr_t) base; 857 alloc = (struct nvgpu_page_alloc *) (uintptr_t) base;
836 } 858 }
@@ -963,8 +985,9 @@ static int nvgpu_page_alloc_init_slabs(struct nvgpu_page_allocator *a)
963 a->slabs = nvgpu_kcalloc(nvgpu_alloc_to_gpu(a->owner), 985 a->slabs = nvgpu_kcalloc(nvgpu_alloc_to_gpu(a->owner),
964 nr_slabs, 986 nr_slabs,
965 sizeof(struct page_alloc_slab)); 987 sizeof(struct page_alloc_slab));
966 if (!a->slabs) 988 if (!a->slabs) {
967 return -ENOMEM; 989 return -ENOMEM;
990 }
968 a->nr_slabs = nr_slabs; 991 a->nr_slabs = nr_slabs;
969 992
970 for (i = 0; i < nr_slabs; i++) { 993 for (i = 0; i < nr_slabs; i++) {
@@ -990,16 +1013,19 @@ int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
990 char buddy_name[sizeof(__a->name)]; 1013 char buddy_name[sizeof(__a->name)];
991 int err; 1014 int err;
992 1015
993 if (blk_size < SZ_4K) 1016 if (blk_size < SZ_4K) {
994 return -EINVAL; 1017 return -EINVAL;
1018 }
995 1019
996 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_page_allocator)); 1020 a = nvgpu_kzalloc(g, sizeof(struct nvgpu_page_allocator));
997 if (!a) 1021 if (!a) {
998 return -ENOMEM; 1022 return -ENOMEM;
1023 }
999 1024
1000 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &page_ops); 1025 err = __nvgpu_alloc_common_init(__a, g, name, a, false, &page_ops);
1001 if (err) 1026 if (err) {
1002 goto fail; 1027 goto fail;
1028 }
1003 1029
1004 a->alloc_cache = nvgpu_kmem_cache_create(g, 1030 a->alloc_cache = nvgpu_kmem_cache_create(g,
1005 sizeof(struct nvgpu_page_alloc)); 1031 sizeof(struct nvgpu_page_alloc));
@@ -1020,16 +1046,18 @@ int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
1020 1046
1021 if (flags & GPU_ALLOC_4K_VIDMEM_PAGES && blk_size > SZ_4K) { 1047 if (flags & GPU_ALLOC_4K_VIDMEM_PAGES && blk_size > SZ_4K) {
1022 err = nvgpu_page_alloc_init_slabs(a); 1048 err = nvgpu_page_alloc_init_slabs(a);
1023 if (err) 1049 if (err) {
1024 goto fail; 1050 goto fail;
1051 }
1025 } 1052 }
1026 1053
1027 snprintf(buddy_name, sizeof(buddy_name), "%s-src", name); 1054 snprintf(buddy_name, sizeof(buddy_name), "%s-src", name);
1028 1055
1029 err = nvgpu_buddy_allocator_init(g, &a->source_allocator, buddy_name, 1056 err = nvgpu_buddy_allocator_init(g, &a->source_allocator, buddy_name,
1030 base, length, blk_size, 0); 1057 base, length, blk_size, 0);
1031 if (err) 1058 if (err) {
1032 goto fail; 1059 goto fail;
1060 }
1033 1061
1034#ifdef CONFIG_DEBUG_FS 1062#ifdef CONFIG_DEBUG_FS
1035 nvgpu_init_alloc_debug(g, __a); 1063 nvgpu_init_alloc_debug(g, __a);
@@ -1044,10 +1072,12 @@ int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
1044 return 0; 1072 return 0;
1045 1073
1046fail: 1074fail:
1047 if (a->alloc_cache) 1075 if (a->alloc_cache) {
1048 nvgpu_kmem_cache_destroy(a->alloc_cache); 1076 nvgpu_kmem_cache_destroy(a->alloc_cache);
1049 if (a->slab_page_cache) 1077 }
1078 if (a->slab_page_cache) {
1050 nvgpu_kmem_cache_destroy(a->slab_page_cache); 1079 nvgpu_kmem_cache_destroy(a->slab_page_cache);
1080 }
1051 nvgpu_kfree(g, a); 1081 nvgpu_kfree(g, a);
1052 return err; 1082 return err;
1053} 1083}