diff options
Diffstat (limited to 'drivers/gpu/nvgpu/common/mm/page_allocator.c')
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/page_allocator.c | 110 |
1 files changed, 70 insertions, 40 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/page_allocator.c b/drivers/gpu/nvgpu/common/mm/page_allocator.c index 773d33ef..d001a2aa 100644 --- a/drivers/gpu/nvgpu/common/mm/page_allocator.c +++ b/drivers/gpu/nvgpu/common/mm/page_allocator.c | |||
@@ -264,8 +264,9 @@ static struct nvgpu_page_alloc *__find_page_alloc( | |||
264 | struct nvgpu_rbtree_node *node = NULL; | 264 | struct nvgpu_rbtree_node *node = NULL; |
265 | 265 | ||
266 | nvgpu_rbtree_search(addr, &node, a->allocs); | 266 | nvgpu_rbtree_search(addr, &node, a->allocs); |
267 | if (!node) | 267 | if (!node) { |
268 | return NULL; | 268 | return NULL; |
269 | } | ||
269 | 270 | ||
270 | alloc = nvgpu_page_alloc_from_rbtree_node(node); | 271 | alloc = nvgpu_page_alloc_from_rbtree_node(node); |
271 | 272 | ||
@@ -355,8 +356,9 @@ static int __do_slab_alloc(struct nvgpu_page_allocator *a, | |||
355 | 356 | ||
356 | if (!slab_page) { | 357 | if (!slab_page) { |
357 | slab_page = alloc_slab_page(a, slab); | 358 | slab_page = alloc_slab_page(a, slab); |
358 | if (!slab_page) | 359 | if (!slab_page) { |
359 | return -ENOMEM; | 360 | return -ENOMEM; |
361 | } | ||
360 | } | 362 | } |
361 | 363 | ||
362 | /* | 364 | /* |
@@ -376,12 +378,13 @@ static int __do_slab_alloc(struct nvgpu_page_allocator *a, | |||
376 | bitmap_set(&slab_page->bitmap, offs, 1); | 378 | bitmap_set(&slab_page->bitmap, offs, 1); |
377 | slab_page->nr_objects_alloced++; | 379 | slab_page->nr_objects_alloced++; |
378 | 380 | ||
379 | if (slab_page->nr_objects_alloced < slab_page->nr_objects) | 381 | if (slab_page->nr_objects_alloced < slab_page->nr_objects) { |
380 | add_slab_page_to_partial(slab, slab_page); | 382 | add_slab_page_to_partial(slab, slab_page); |
381 | else if (slab_page->nr_objects_alloced == slab_page->nr_objects) | 383 | } else if (slab_page->nr_objects_alloced == slab_page->nr_objects) { |
382 | add_slab_page_to_full(slab, slab_page); | 384 | add_slab_page_to_full(slab, slab_page); |
383 | else | 385 | } else { |
384 | BUG(); /* Should be impossible to hit this. */ | 386 | BUG(); /* Should be impossible to hit this. */ |
387 | } | ||
385 | 388 | ||
386 | /* | 389 | /* |
387 | * Handle building the nvgpu_page_alloc struct. We expect one sgl | 390 | * Handle building the nvgpu_page_alloc struct. We expect one sgl |
@@ -435,8 +438,9 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_slab( | |||
435 | 438 | ||
436 | alloc->sgt.sgl = (struct nvgpu_sgl *)sgl; | 439 | alloc->sgt.sgl = (struct nvgpu_sgl *)sgl; |
437 | err = __do_slab_alloc(a, slab, alloc); | 440 | err = __do_slab_alloc(a, slab, alloc); |
438 | if (err) | 441 | if (err) { |
439 | goto fail; | 442 | goto fail; |
443 | } | ||
440 | 444 | ||
441 | palloc_dbg(a, "Alloc 0x%04llx sr=%d id=0x%010llx [slab]", | 445 | palloc_dbg(a, "Alloc 0x%04llx sr=%d id=0x%010llx [slab]", |
442 | len, slab_nr, alloc->base); | 446 | len, slab_nr, alloc->base); |
@@ -445,10 +449,12 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_slab( | |||
445 | return alloc; | 449 | return alloc; |
446 | 450 | ||
447 | fail: | 451 | fail: |
448 | if (alloc) | 452 | if (alloc) { |
449 | nvgpu_kmem_cache_free(a->alloc_cache, alloc); | 453 | nvgpu_kmem_cache_free(a->alloc_cache, alloc); |
450 | if (sgl) | 454 | } |
455 | if (sgl) { | ||
451 | nvgpu_kfree(a->owner->g, sgl); | 456 | nvgpu_kfree(a->owner->g, sgl); |
457 | } | ||
452 | return NULL; | 458 | return NULL; |
453 | } | 459 | } |
454 | 460 | ||
@@ -465,27 +471,30 @@ static void __nvgpu_free_slab(struct nvgpu_page_allocator *a, | |||
465 | 471 | ||
466 | slab_page->nr_objects_alloced--; | 472 | slab_page->nr_objects_alloced--; |
467 | 473 | ||
468 | if (slab_page->nr_objects_alloced == 0) | 474 | if (slab_page->nr_objects_alloced == 0) { |
469 | new_state = SP_EMPTY; | 475 | new_state = SP_EMPTY; |
470 | else | 476 | } else { |
471 | new_state = SP_PARTIAL; | 477 | new_state = SP_PARTIAL; |
478 | } | ||
472 | 479 | ||
473 | /* | 480 | /* |
474 | * Need to migrate the page to a different list. | 481 | * Need to migrate the page to a different list. |
475 | */ | 482 | */ |
476 | if (new_state != slab_page->state) { | 483 | if (new_state != slab_page->state) { |
477 | /* Delete - can't be in empty. */ | 484 | /* Delete - can't be in empty. */ |
478 | if (slab_page->state == SP_PARTIAL) | 485 | if (slab_page->state == SP_PARTIAL) { |
479 | del_slab_page_from_partial(slab, slab_page); | 486 | del_slab_page_from_partial(slab, slab_page); |
480 | else | 487 | } else { |
481 | del_slab_page_from_full(slab, slab_page); | 488 | del_slab_page_from_full(slab, slab_page); |
489 | } | ||
482 | 490 | ||
483 | /* And add. */ | 491 | /* And add. */ |
484 | if (new_state == SP_EMPTY) { | 492 | if (new_state == SP_EMPTY) { |
485 | if (nvgpu_list_empty(&slab->empty)) | 493 | if (nvgpu_list_empty(&slab->empty)) { |
486 | add_slab_page_to_empty(slab, slab_page); | 494 | add_slab_page_to_empty(slab, slab_page); |
487 | else | 495 | } else { |
488 | free_slab_page(a, slab_page); | 496 | free_slab_page(a, slab_page); |
497 | } | ||
489 | } else { | 498 | } else { |
490 | add_slab_page_to_partial(slab, slab_page); | 499 | add_slab_page_to_partial(slab, slab_page); |
491 | } | 500 | } |
@@ -515,8 +524,9 @@ static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages( | |||
515 | int i = 0; | 524 | int i = 0; |
516 | 525 | ||
517 | alloc = nvgpu_kmem_cache_alloc(a->alloc_cache); | 526 | alloc = nvgpu_kmem_cache_alloc(a->alloc_cache); |
518 | if (!alloc) | 527 | if (!alloc) { |
519 | goto fail; | 528 | goto fail; |
529 | } | ||
520 | 530 | ||
521 | memset(alloc, 0, sizeof(*alloc)); | 531 | memset(alloc, 0, sizeof(*alloc)); |
522 | 532 | ||
@@ -535,11 +545,13 @@ static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages( | |||
535 | * requested size. The buddy allocator guarantees any given | 545 | * requested size. The buddy allocator guarantees any given |
536 | * single alloc is contiguous. | 546 | * single alloc is contiguous. |
537 | */ | 547 | */ |
538 | if (a->flags & GPU_ALLOC_FORCE_CONTIG && i != 0) | 548 | if (a->flags & GPU_ALLOC_FORCE_CONTIG && i != 0) { |
539 | goto fail_cleanup; | 549 | goto fail_cleanup; |
550 | } | ||
540 | 551 | ||
541 | if (chunk_len > max_chunk_len) | 552 | if (chunk_len > max_chunk_len) { |
542 | chunk_len = max_chunk_len; | 553 | chunk_len = max_chunk_len; |
554 | } | ||
543 | 555 | ||
544 | /* | 556 | /* |
545 | * Keep attempting to allocate in smaller chunks until the alloc | 557 | * Keep attempting to allocate in smaller chunks until the alloc |
@@ -582,10 +594,11 @@ static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages( | |||
582 | * Build the singly linked list with a head node that is part of | 594 | * Build the singly linked list with a head node that is part of |
583 | * the list. | 595 | * the list. |
584 | */ | 596 | */ |
585 | if (prev_sgl) | 597 | if (prev_sgl) { |
586 | prev_sgl->next = sgl; | 598 | prev_sgl->next = sgl; |
587 | else | 599 | } else { |
588 | alloc->sgt.sgl = (struct nvgpu_sgl *)sgl; | 600 | alloc->sgt.sgl = (struct nvgpu_sgl *)sgl; |
601 | } | ||
589 | 602 | ||
590 | prev_sgl = sgl; | 603 | prev_sgl = sgl; |
591 | 604 | ||
@@ -671,10 +684,11 @@ static u64 nvgpu_page_alloc(struct nvgpu_allocator *__a, u64 len) | |||
671 | 684 | ||
672 | alloc_lock(__a); | 685 | alloc_lock(__a); |
673 | if (a->flags & GPU_ALLOC_4K_VIDMEM_PAGES && | 686 | if (a->flags & GPU_ALLOC_4K_VIDMEM_PAGES && |
674 | real_len <= (a->page_size / 2)) | 687 | real_len <= (a->page_size / 2)) { |
675 | alloc = __nvgpu_alloc_slab(a, real_len); | 688 | alloc = __nvgpu_alloc_slab(a, real_len); |
676 | else | 689 | } else { |
677 | alloc = __nvgpu_alloc_pages(a, real_len); | 690 | alloc = __nvgpu_alloc_pages(a, real_len); |
691 | } | ||
678 | 692 | ||
679 | if (!alloc) { | 693 | if (!alloc) { |
680 | alloc_unlock(__a); | 694 | alloc_unlock(__a); |
@@ -684,14 +698,16 @@ static u64 nvgpu_page_alloc(struct nvgpu_allocator *__a, u64 len) | |||
684 | __insert_page_alloc(a, alloc); | 698 | __insert_page_alloc(a, alloc); |
685 | 699 | ||
686 | a->nr_allocs++; | 700 | a->nr_allocs++; |
687 | if (real_len > a->page_size / 2) | 701 | if (real_len > a->page_size / 2) { |
688 | a->pages_alloced += alloc->length >> a->page_shift; | 702 | a->pages_alloced += alloc->length >> a->page_shift; |
703 | } | ||
689 | alloc_unlock(__a); | 704 | alloc_unlock(__a); |
690 | 705 | ||
691 | if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) | 706 | if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) { |
692 | return alloc->base; | 707 | return alloc->base; |
693 | else | 708 | } else { |
694 | return (u64) (uintptr_t) alloc; | 709 | return (u64) (uintptr_t) alloc; |
710 | } | ||
695 | } | 711 | } |
696 | 712 | ||
697 | /* | 713 | /* |
@@ -705,11 +721,12 @@ static void nvgpu_page_free(struct nvgpu_allocator *__a, u64 base) | |||
705 | 721 | ||
706 | alloc_lock(__a); | 722 | alloc_lock(__a); |
707 | 723 | ||
708 | if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) | 724 | if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) { |
709 | alloc = __find_page_alloc(a, base); | 725 | alloc = __find_page_alloc(a, base); |
710 | else | 726 | } else { |
711 | alloc = __find_page_alloc(a, | 727 | alloc = __find_page_alloc(a, |
712 | ((struct nvgpu_page_alloc *)(uintptr_t)base)->base); | 728 | ((struct nvgpu_page_alloc *)(uintptr_t)base)->base); |
729 | } | ||
713 | 730 | ||
714 | if (!alloc) { | 731 | if (!alloc) { |
715 | palloc_dbg(a, "Hrm, found no alloc?"); | 732 | palloc_dbg(a, "Hrm, found no alloc?"); |
@@ -743,8 +760,9 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_pages_fixed( | |||
743 | 760 | ||
744 | alloc = nvgpu_kmem_cache_alloc(a->alloc_cache); | 761 | alloc = nvgpu_kmem_cache_alloc(a->alloc_cache); |
745 | sgl = nvgpu_kzalloc(a->owner->g, sizeof(*sgl)); | 762 | sgl = nvgpu_kzalloc(a->owner->g, sizeof(*sgl)); |
746 | if (!alloc || !sgl) | 763 | if (!alloc || !sgl) { |
747 | goto fail; | 764 | goto fail; |
765 | } | ||
748 | 766 | ||
749 | alloc->sgt.ops = &page_alloc_sgl_ops; | 767 | alloc->sgt.ops = &page_alloc_sgl_ops; |
750 | alloc->base = nvgpu_alloc_fixed(&a->source_allocator, base, length, 0); | 768 | alloc->base = nvgpu_alloc_fixed(&a->source_allocator, base, length, 0); |
@@ -765,10 +783,12 @@ static struct nvgpu_page_alloc *__nvgpu_alloc_pages_fixed( | |||
765 | return alloc; | 783 | return alloc; |
766 | 784 | ||
767 | fail: | 785 | fail: |
768 | if (sgl) | 786 | if (sgl) { |
769 | nvgpu_kfree(a->owner->g, sgl); | 787 | nvgpu_kfree(a->owner->g, sgl); |
770 | if (alloc) | 788 | } |
789 | if (alloc) { | ||
771 | nvgpu_kmem_cache_free(a->alloc_cache, alloc); | 790 | nvgpu_kmem_cache_free(a->alloc_cache, alloc); |
791 | } | ||
772 | return NULL; | 792 | return NULL; |
773 | } | 793 | } |
774 | 794 | ||
@@ -813,10 +833,11 @@ static u64 nvgpu_page_alloc_fixed(struct nvgpu_allocator *__a, | |||
813 | a->nr_fixed_allocs++; | 833 | a->nr_fixed_allocs++; |
814 | a->pages_alloced += pages; | 834 | a->pages_alloced += pages; |
815 | 835 | ||
816 | if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) | 836 | if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) { |
817 | return alloc->base; | 837 | return alloc->base; |
818 | else | 838 | } else { |
819 | return (u64) (uintptr_t) alloc; | 839 | return (u64) (uintptr_t) alloc; |
840 | } | ||
820 | } | 841 | } |
821 | 842 | ||
822 | static void nvgpu_page_free_fixed(struct nvgpu_allocator *__a, | 843 | static void nvgpu_page_free_fixed(struct nvgpu_allocator *__a, |
@@ -829,8 +850,9 @@ static void nvgpu_page_free_fixed(struct nvgpu_allocator *__a, | |||
829 | 850 | ||
830 | if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) { | 851 | if (a->flags & GPU_ALLOC_NO_SCATTER_GATHER) { |
831 | alloc = __find_page_alloc(a, base); | 852 | alloc = __find_page_alloc(a, base); |
832 | if (!alloc) | 853 | if (!alloc) { |
833 | goto done; | 854 | goto done; |
855 | } | ||
834 | } else { | 856 | } else { |
835 | alloc = (struct nvgpu_page_alloc *) (uintptr_t) base; | 857 | alloc = (struct nvgpu_page_alloc *) (uintptr_t) base; |
836 | } | 858 | } |
@@ -963,8 +985,9 @@ static int nvgpu_page_alloc_init_slabs(struct nvgpu_page_allocator *a) | |||
963 | a->slabs = nvgpu_kcalloc(nvgpu_alloc_to_gpu(a->owner), | 985 | a->slabs = nvgpu_kcalloc(nvgpu_alloc_to_gpu(a->owner), |
964 | nr_slabs, | 986 | nr_slabs, |
965 | sizeof(struct page_alloc_slab)); | 987 | sizeof(struct page_alloc_slab)); |
966 | if (!a->slabs) | 988 | if (!a->slabs) { |
967 | return -ENOMEM; | 989 | return -ENOMEM; |
990 | } | ||
968 | a->nr_slabs = nr_slabs; | 991 | a->nr_slabs = nr_slabs; |
969 | 992 | ||
970 | for (i = 0; i < nr_slabs; i++) { | 993 | for (i = 0; i < nr_slabs; i++) { |
@@ -990,16 +1013,19 @@ int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
990 | char buddy_name[sizeof(__a->name)]; | 1013 | char buddy_name[sizeof(__a->name)]; |
991 | int err; | 1014 | int err; |
992 | 1015 | ||
993 | if (blk_size < SZ_4K) | 1016 | if (blk_size < SZ_4K) { |
994 | return -EINVAL; | 1017 | return -EINVAL; |
1018 | } | ||
995 | 1019 | ||
996 | a = nvgpu_kzalloc(g, sizeof(struct nvgpu_page_allocator)); | 1020 | a = nvgpu_kzalloc(g, sizeof(struct nvgpu_page_allocator)); |
997 | if (!a) | 1021 | if (!a) { |
998 | return -ENOMEM; | 1022 | return -ENOMEM; |
1023 | } | ||
999 | 1024 | ||
1000 | err = __nvgpu_alloc_common_init(__a, g, name, a, false, &page_ops); | 1025 | err = __nvgpu_alloc_common_init(__a, g, name, a, false, &page_ops); |
1001 | if (err) | 1026 | if (err) { |
1002 | goto fail; | 1027 | goto fail; |
1028 | } | ||
1003 | 1029 | ||
1004 | a->alloc_cache = nvgpu_kmem_cache_create(g, | 1030 | a->alloc_cache = nvgpu_kmem_cache_create(g, |
1005 | sizeof(struct nvgpu_page_alloc)); | 1031 | sizeof(struct nvgpu_page_alloc)); |
@@ -1020,16 +1046,18 @@ int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
1020 | 1046 | ||
1021 | if (flags & GPU_ALLOC_4K_VIDMEM_PAGES && blk_size > SZ_4K) { | 1047 | if (flags & GPU_ALLOC_4K_VIDMEM_PAGES && blk_size > SZ_4K) { |
1022 | err = nvgpu_page_alloc_init_slabs(a); | 1048 | err = nvgpu_page_alloc_init_slabs(a); |
1023 | if (err) | 1049 | if (err) { |
1024 | goto fail; | 1050 | goto fail; |
1051 | } | ||
1025 | } | 1052 | } |
1026 | 1053 | ||
1027 | snprintf(buddy_name, sizeof(buddy_name), "%s-src", name); | 1054 | snprintf(buddy_name, sizeof(buddy_name), "%s-src", name); |
1028 | 1055 | ||
1029 | err = nvgpu_buddy_allocator_init(g, &a->source_allocator, buddy_name, | 1056 | err = nvgpu_buddy_allocator_init(g, &a->source_allocator, buddy_name, |
1030 | base, length, blk_size, 0); | 1057 | base, length, blk_size, 0); |
1031 | if (err) | 1058 | if (err) { |
1032 | goto fail; | 1059 | goto fail; |
1060 | } | ||
1033 | 1061 | ||
1034 | #ifdef CONFIG_DEBUG_FS | 1062 | #ifdef CONFIG_DEBUG_FS |
1035 | nvgpu_init_alloc_debug(g, __a); | 1063 | nvgpu_init_alloc_debug(g, __a); |
@@ -1044,10 +1072,12 @@ int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
1044 | return 0; | 1072 | return 0; |
1045 | 1073 | ||
1046 | fail: | 1074 | fail: |
1047 | if (a->alloc_cache) | 1075 | if (a->alloc_cache) { |
1048 | nvgpu_kmem_cache_destroy(a->alloc_cache); | 1076 | nvgpu_kmem_cache_destroy(a->alloc_cache); |
1049 | if (a->slab_page_cache) | 1077 | } |
1078 | if (a->slab_page_cache) { | ||
1050 | nvgpu_kmem_cache_destroy(a->slab_page_cache); | 1079 | nvgpu_kmem_cache_destroy(a->slab_page_cache); |
1080 | } | ||
1051 | nvgpu_kfree(g, a); | 1081 | nvgpu_kfree(g, a); |
1052 | return err; | 1082 | return err; |
1053 | } | 1083 | } |