diff options
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a/mm_gk20a.c')
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/mm_gk20a.c | 393 |
1 files changed, 1 insertions, 392 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c index cdd0e541..79654af3 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c | |||
@@ -30,6 +30,7 @@ | |||
30 | #include <uapi/linux/nvgpu.h> | 30 | #include <uapi/linux/nvgpu.h> |
31 | #include <trace/events/gk20a.h> | 31 | #include <trace/events/gk20a.h> |
32 | 32 | ||
33 | #include <nvgpu/dma.h> | ||
33 | #include <nvgpu/kmem.h> | 34 | #include <nvgpu/kmem.h> |
34 | #include <nvgpu/timers.h> | 35 | #include <nvgpu/timers.h> |
35 | #include <nvgpu/pramin.h> | 36 | #include <nvgpu/pramin.h> |
@@ -2519,152 +2520,6 @@ u64 gk20a_gmmu_fixed_map(struct vm_gk20a *vm, | |||
2519 | aperture); | 2520 | aperture); |
2520 | } | 2521 | } |
2521 | 2522 | ||
2522 | int gk20a_gmmu_alloc(struct gk20a *g, size_t size, struct nvgpu_mem *mem) | ||
2523 | { | ||
2524 | return gk20a_gmmu_alloc_flags(g, 0, size, mem); | ||
2525 | } | ||
2526 | |||
2527 | int gk20a_gmmu_alloc_flags(struct gk20a *g, unsigned long flags, size_t size, | ||
2528 | struct nvgpu_mem *mem) | ||
2529 | { | ||
2530 | if (g->mm.vidmem_is_vidmem) { | ||
2531 | /* | ||
2532 | * Force the no-kernel-mapping flag on because we don't support | ||
2533 | * the lack of it for vidmem - the user should not care when | ||
2534 | * using gk20a_gmmu_alloc_map and it's vidmem, or if there's a | ||
2535 | * difference, the user should use the flag explicitly anyway. | ||
2536 | */ | ||
2537 | int err = gk20a_gmmu_alloc_flags_vid(g, | ||
2538 | flags | NVGPU_DMA_NO_KERNEL_MAPPING, | ||
2539 | size, mem); | ||
2540 | |||
2541 | if (!err) | ||
2542 | return 0; | ||
2543 | /* | ||
2544 | * Fall back to sysmem (which may then also fail) in case | ||
2545 | * vidmem is exhausted. | ||
2546 | */ | ||
2547 | } | ||
2548 | |||
2549 | return gk20a_gmmu_alloc_flags_sys(g, flags, size, mem); | ||
2550 | } | ||
2551 | |||
2552 | int gk20a_gmmu_alloc_sys(struct gk20a *g, size_t size, struct nvgpu_mem *mem) | ||
2553 | { | ||
2554 | return gk20a_gmmu_alloc_flags_sys(g, 0, size, mem); | ||
2555 | } | ||
2556 | |||
2557 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) | ||
2558 | static void gk20a_dma_flags_to_attrs(unsigned long *attrs, | ||
2559 | unsigned long flags) | ||
2560 | #define ATTR_ARG(x) *x | ||
2561 | #else | ||
2562 | static void gk20a_dma_flags_to_attrs(struct dma_attrs *attrs, | ||
2563 | unsigned long flags) | ||
2564 | #define ATTR_ARG(x) x | ||
2565 | #endif | ||
2566 | { | ||
2567 | if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) | ||
2568 | dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, ATTR_ARG(attrs)); | ||
2569 | if (flags & NVGPU_DMA_FORCE_CONTIGUOUS) | ||
2570 | dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, ATTR_ARG(attrs)); | ||
2571 | if (flags & NVGPU_DMA_READ_ONLY) | ||
2572 | dma_set_attr(DMA_ATTR_READ_ONLY, ATTR_ARG(attrs)); | ||
2573 | #undef ATTR_ARG | ||
2574 | } | ||
2575 | |||
2576 | int gk20a_gmmu_alloc_flags_sys(struct gk20a *g, unsigned long flags, | ||
2577 | size_t size, struct nvgpu_mem *mem) | ||
2578 | { | ||
2579 | struct device *d = dev_from_gk20a(g); | ||
2580 | int err; | ||
2581 | dma_addr_t iova; | ||
2582 | |||
2583 | gk20a_dbg_fn(""); | ||
2584 | |||
2585 | if (flags) { | ||
2586 | DEFINE_DMA_ATTRS(dma_attrs); | ||
2587 | |||
2588 | gk20a_dma_flags_to_attrs(&dma_attrs, flags); | ||
2589 | |||
2590 | if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) { | ||
2591 | mem->pages = dma_alloc_attrs(d, | ||
2592 | size, &iova, GFP_KERNEL, | ||
2593 | __DMA_ATTR(dma_attrs)); | ||
2594 | if (!mem->pages) | ||
2595 | return -ENOMEM; | ||
2596 | } else { | ||
2597 | mem->cpu_va = dma_alloc_attrs(d, | ||
2598 | size, &iova, GFP_KERNEL, | ||
2599 | __DMA_ATTR(dma_attrs)); | ||
2600 | if (!mem->cpu_va) | ||
2601 | return -ENOMEM; | ||
2602 | } | ||
2603 | } else { | ||
2604 | mem->cpu_va = dma_alloc_coherent(d, size, &iova, GFP_KERNEL); | ||
2605 | if (!mem->cpu_va) | ||
2606 | return -ENOMEM; | ||
2607 | } | ||
2608 | |||
2609 | if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) | ||
2610 | err = gk20a_get_sgtable_from_pages(d, &mem->sgt, mem->pages, | ||
2611 | iova, size); | ||
2612 | else { | ||
2613 | err = gk20a_get_sgtable(d, &mem->sgt, mem->cpu_va, iova, size); | ||
2614 | memset(mem->cpu_va, 0, size); | ||
2615 | } | ||
2616 | if (err) | ||
2617 | goto fail_free; | ||
2618 | |||
2619 | mem->size = size; | ||
2620 | mem->aperture = APERTURE_SYSMEM; | ||
2621 | mem->flags = flags; | ||
2622 | |||
2623 | gk20a_dbg_fn("done"); | ||
2624 | |||
2625 | return 0; | ||
2626 | |||
2627 | fail_free: | ||
2628 | dma_free_coherent(d, size, mem->cpu_va, iova); | ||
2629 | mem->cpu_va = NULL; | ||
2630 | mem->sgt = NULL; | ||
2631 | return err; | ||
2632 | } | ||
2633 | |||
2634 | static void gk20a_gmmu_free_sys(struct gk20a *g, struct nvgpu_mem *mem) | ||
2635 | { | ||
2636 | struct device *d = dev_from_gk20a(g); | ||
2637 | |||
2638 | if (mem->cpu_va || mem->pages) { | ||
2639 | if (mem->flags) { | ||
2640 | DEFINE_DMA_ATTRS(dma_attrs); | ||
2641 | |||
2642 | gk20a_dma_flags_to_attrs(&dma_attrs, mem->flags); | ||
2643 | |||
2644 | if (mem->flags & NVGPU_DMA_NO_KERNEL_MAPPING) { | ||
2645 | dma_free_attrs(d, mem->size, mem->pages, | ||
2646 | sg_dma_address(mem->sgt->sgl), | ||
2647 | __DMA_ATTR(dma_attrs)); | ||
2648 | } else { | ||
2649 | dma_free_attrs(d, mem->size, mem->cpu_va, | ||
2650 | sg_dma_address(mem->sgt->sgl), | ||
2651 | __DMA_ATTR(dma_attrs)); | ||
2652 | } | ||
2653 | } else { | ||
2654 | dma_free_coherent(d, mem->size, mem->cpu_va, | ||
2655 | sg_dma_address(mem->sgt->sgl)); | ||
2656 | } | ||
2657 | mem->cpu_va = NULL; | ||
2658 | mem->pages = NULL; | ||
2659 | } | ||
2660 | |||
2661 | if (mem->sgt) | ||
2662 | gk20a_free_sgtable(g, &mem->sgt); | ||
2663 | |||
2664 | mem->size = 0; | ||
2665 | mem->aperture = APERTURE_INVALID; | ||
2666 | } | ||
2667 | |||
2668 | #if defined(CONFIG_GK20A_VIDMEM) | 2523 | #if defined(CONFIG_GK20A_VIDMEM) |
2669 | static int gk20a_gmmu_clear_vidmem_mem(struct gk20a *g, struct nvgpu_mem *mem) | 2524 | static int gk20a_gmmu_clear_vidmem_mem(struct gk20a *g, struct nvgpu_mem *mem) |
2670 | { | 2525 | { |
@@ -2728,153 +2583,6 @@ static int gk20a_gmmu_clear_vidmem_mem(struct gk20a *g, struct nvgpu_mem *mem) | |||
2728 | } | 2583 | } |
2729 | #endif | 2584 | #endif |
2730 | 2585 | ||
2731 | int gk20a_gmmu_alloc_vid(struct gk20a *g, size_t size, struct nvgpu_mem *mem) | ||
2732 | { | ||
2733 | return gk20a_gmmu_alloc_flags_vid(g, | ||
2734 | NVGPU_DMA_NO_KERNEL_MAPPING, size, mem); | ||
2735 | } | ||
2736 | |||
2737 | int gk20a_gmmu_alloc_flags_vid(struct gk20a *g, unsigned long flags, | ||
2738 | size_t size, struct nvgpu_mem *mem) | ||
2739 | { | ||
2740 | return gk20a_gmmu_alloc_flags_vid_at(g, flags, size, mem, 0); | ||
2741 | } | ||
2742 | |||
2743 | #if defined(CONFIG_GK20A_VIDMEM) | ||
2744 | static u64 __gk20a_gmmu_alloc(struct nvgpu_allocator *allocator, dma_addr_t at, | ||
2745 | size_t size) | ||
2746 | { | ||
2747 | u64 addr = 0; | ||
2748 | |||
2749 | if (at) | ||
2750 | addr = nvgpu_alloc_fixed(allocator, at, size, 0); | ||
2751 | else | ||
2752 | addr = nvgpu_alloc(allocator, size); | ||
2753 | |||
2754 | return addr; | ||
2755 | } | ||
2756 | #endif | ||
2757 | |||
2758 | int gk20a_gmmu_alloc_flags_vid_at(struct gk20a *g, unsigned long flags, | ||
2759 | size_t size, struct nvgpu_mem *mem, dma_addr_t at) | ||
2760 | { | ||
2761 | #if defined(CONFIG_GK20A_VIDMEM) | ||
2762 | u64 addr; | ||
2763 | int err; | ||
2764 | struct nvgpu_allocator *vidmem_alloc = g->mm.vidmem.cleared ? | ||
2765 | &g->mm.vidmem.allocator : | ||
2766 | &g->mm.vidmem.bootstrap_allocator; | ||
2767 | int before_pending; | ||
2768 | |||
2769 | gk20a_dbg_fn(""); | ||
2770 | |||
2771 | if (!nvgpu_alloc_initialized(&g->mm.vidmem.allocator)) | ||
2772 | return -ENOSYS; | ||
2773 | |||
2774 | /* | ||
2775 | * Our own allocator doesn't have any flags yet, and we can't | ||
2776 | * kernel-map these, so require explicit flags. | ||
2777 | */ | ||
2778 | WARN_ON(flags != NVGPU_DMA_NO_KERNEL_MAPPING); | ||
2779 | |||
2780 | nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex); | ||
2781 | before_pending = atomic64_read(&g->mm.vidmem.bytes_pending); | ||
2782 | addr = __gk20a_gmmu_alloc(vidmem_alloc, at, size); | ||
2783 | nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); | ||
2784 | if (!addr) { | ||
2785 | /* | ||
2786 | * If memory is known to be freed soon, let the user know that | ||
2787 | * it may be available after a while. | ||
2788 | */ | ||
2789 | if (before_pending) | ||
2790 | return -EAGAIN; | ||
2791 | else | ||
2792 | return -ENOMEM; | ||
2793 | } | ||
2794 | |||
2795 | if (at) | ||
2796 | mem->fixed = true; | ||
2797 | else | ||
2798 | mem->fixed = false; | ||
2799 | |||
2800 | mem->sgt = nvgpu_kzalloc(g, sizeof(struct sg_table)); | ||
2801 | if (!mem->sgt) { | ||
2802 | err = -ENOMEM; | ||
2803 | goto fail_physfree; | ||
2804 | } | ||
2805 | |||
2806 | err = sg_alloc_table(mem->sgt, 1, GFP_KERNEL); | ||
2807 | if (err) | ||
2808 | goto fail_kfree; | ||
2809 | |||
2810 | set_vidmem_page_alloc(mem->sgt->sgl, addr); | ||
2811 | sg_set_page(mem->sgt->sgl, NULL, size, 0); | ||
2812 | |||
2813 | mem->size = size; | ||
2814 | mem->aperture = APERTURE_VIDMEM; | ||
2815 | mem->allocator = vidmem_alloc; | ||
2816 | mem->flags = flags; | ||
2817 | |||
2818 | nvgpu_init_list_node(&mem->clear_list_entry); | ||
2819 | |||
2820 | gk20a_dbg_fn("done at 0x%llx size %zu", addr, size); | ||
2821 | |||
2822 | return 0; | ||
2823 | |||
2824 | fail_kfree: | ||
2825 | nvgpu_kfree(g, mem->sgt); | ||
2826 | fail_physfree: | ||
2827 | nvgpu_free(&g->mm.vidmem.allocator, addr); | ||
2828 | return err; | ||
2829 | #else | ||
2830 | return -ENOSYS; | ||
2831 | #endif | ||
2832 | } | ||
2833 | |||
2834 | static void gk20a_gmmu_free_vid(struct gk20a *g, struct nvgpu_mem *mem) | ||
2835 | { | ||
2836 | #if defined(CONFIG_GK20A_VIDMEM) | ||
2837 | bool was_empty; | ||
2838 | |||
2839 | /* Sanity check - only this supported when allocating. */ | ||
2840 | WARN_ON(mem->flags != NVGPU_DMA_NO_KERNEL_MAPPING); | ||
2841 | |||
2842 | if (mem->user_mem) { | ||
2843 | nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex); | ||
2844 | was_empty = nvgpu_list_empty(&g->mm.vidmem.clear_list_head); | ||
2845 | nvgpu_list_add_tail(&mem->clear_list_entry, | ||
2846 | &g->mm.vidmem.clear_list_head); | ||
2847 | atomic64_add(mem->size, &g->mm.vidmem.bytes_pending); | ||
2848 | nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); | ||
2849 | |||
2850 | if (was_empty) { | ||
2851 | cancel_work_sync(&g->mm.vidmem.clear_mem_worker); | ||
2852 | schedule_work(&g->mm.vidmem.clear_mem_worker); | ||
2853 | } | ||
2854 | } else { | ||
2855 | nvgpu_memset(g, mem, 0, 0, mem->size); | ||
2856 | nvgpu_free(mem->allocator, | ||
2857 | (u64)get_vidmem_page_alloc(mem->sgt->sgl)); | ||
2858 | gk20a_free_sgtable(g, &mem->sgt); | ||
2859 | |||
2860 | mem->size = 0; | ||
2861 | mem->aperture = APERTURE_INVALID; | ||
2862 | } | ||
2863 | #endif | ||
2864 | } | ||
2865 | |||
2866 | void gk20a_gmmu_free(struct gk20a *g, struct nvgpu_mem *mem) | ||
2867 | { | ||
2868 | switch (mem->aperture) { | ||
2869 | case APERTURE_SYSMEM: | ||
2870 | return gk20a_gmmu_free_sys(g, mem); | ||
2871 | case APERTURE_VIDMEM: | ||
2872 | return gk20a_gmmu_free_vid(g, mem); | ||
2873 | default: | ||
2874 | break; /* like free() on "null" memory */ | ||
2875 | } | ||
2876 | } | ||
2877 | |||
2878 | /* | 2586 | /* |
2879 | * If mem is in VIDMEM, return base address in vidmem | 2587 | * If mem is in VIDMEM, return base address in vidmem |
2880 | * else return IOVA address for SYSMEM | 2588 | * else return IOVA address for SYSMEM |
@@ -2938,105 +2646,6 @@ static void gk20a_vidmem_clear_mem_worker(struct work_struct *work) | |||
2938 | } | 2646 | } |
2939 | #endif | 2647 | #endif |
2940 | 2648 | ||
2941 | int gk20a_gmmu_alloc_map(struct vm_gk20a *vm, size_t size, | ||
2942 | struct nvgpu_mem *mem) | ||
2943 | { | ||
2944 | return gk20a_gmmu_alloc_map_flags(vm, 0, size, mem); | ||
2945 | } | ||
2946 | |||
2947 | int gk20a_gmmu_alloc_map_flags(struct vm_gk20a *vm, unsigned long flags, | ||
2948 | size_t size, struct nvgpu_mem *mem) | ||
2949 | { | ||
2950 | if (vm->mm->vidmem_is_vidmem) { | ||
2951 | /* | ||
2952 | * Force the no-kernel-mapping flag on because we don't support | ||
2953 | * the lack of it for vidmem - the user should not care when | ||
2954 | * using gk20a_gmmu_alloc_map and it's vidmem, or if there's a | ||
2955 | * difference, the user should use the flag explicitly anyway. | ||
2956 | */ | ||
2957 | int err = gk20a_gmmu_alloc_map_flags_vid(vm, | ||
2958 | flags | NVGPU_DMA_NO_KERNEL_MAPPING, | ||
2959 | size, mem); | ||
2960 | |||
2961 | if (!err) | ||
2962 | return 0; | ||
2963 | /* | ||
2964 | * Fall back to sysmem (which may then also fail) in case | ||
2965 | * vidmem is exhausted. | ||
2966 | */ | ||
2967 | } | ||
2968 | |||
2969 | return gk20a_gmmu_alloc_map_flags_sys(vm, flags, size, mem); | ||
2970 | } | ||
2971 | |||
2972 | int gk20a_gmmu_alloc_map_sys(struct vm_gk20a *vm, size_t size, | ||
2973 | struct nvgpu_mem *mem) | ||
2974 | { | ||
2975 | return gk20a_gmmu_alloc_map_flags_sys(vm, 0, size, mem); | ||
2976 | } | ||
2977 | |||
2978 | int gk20a_gmmu_alloc_map_flags_sys(struct vm_gk20a *vm, unsigned long flags, | ||
2979 | size_t size, struct nvgpu_mem *mem) | ||
2980 | { | ||
2981 | int err = gk20a_gmmu_alloc_flags_sys(vm->mm->g, flags, size, mem); | ||
2982 | |||
2983 | if (err) | ||
2984 | return err; | ||
2985 | |||
2986 | mem->gpu_va = gk20a_gmmu_map(vm, &mem->sgt, size, 0, | ||
2987 | gk20a_mem_flag_none, false, | ||
2988 | mem->aperture); | ||
2989 | if (!mem->gpu_va) { | ||
2990 | err = -ENOMEM; | ||
2991 | goto fail_free; | ||
2992 | } | ||
2993 | |||
2994 | return 0; | ||
2995 | |||
2996 | fail_free: | ||
2997 | gk20a_gmmu_free(vm->mm->g, mem); | ||
2998 | return err; | ||
2999 | } | ||
3000 | |||
3001 | int gk20a_gmmu_alloc_map_vid(struct vm_gk20a *vm, size_t size, | ||
3002 | struct nvgpu_mem *mem) | ||
3003 | { | ||
3004 | return gk20a_gmmu_alloc_map_flags_vid(vm, | ||
3005 | NVGPU_DMA_NO_KERNEL_MAPPING, size, mem); | ||
3006 | } | ||
3007 | |||
3008 | int gk20a_gmmu_alloc_map_flags_vid(struct vm_gk20a *vm, unsigned long flags, | ||
3009 | size_t size, struct nvgpu_mem *mem) | ||
3010 | { | ||
3011 | int err = gk20a_gmmu_alloc_flags_vid(vm->mm->g, flags, size, mem); | ||
3012 | |||
3013 | if (err) | ||
3014 | return err; | ||
3015 | |||
3016 | mem->gpu_va = gk20a_gmmu_map(vm, &mem->sgt, size, 0, | ||
3017 | gk20a_mem_flag_none, false, | ||
3018 | mem->aperture); | ||
3019 | if (!mem->gpu_va) { | ||
3020 | err = -ENOMEM; | ||
3021 | goto fail_free; | ||
3022 | } | ||
3023 | |||
3024 | return 0; | ||
3025 | |||
3026 | fail_free: | ||
3027 | gk20a_gmmu_free(vm->mm->g, mem); | ||
3028 | return err; | ||
3029 | } | ||
3030 | |||
3031 | void gk20a_gmmu_unmap_free(struct vm_gk20a *vm, struct nvgpu_mem *mem) | ||
3032 | { | ||
3033 | if (mem->gpu_va) | ||
3034 | gk20a_gmmu_unmap(vm, mem->gpu_va, mem->size, gk20a_mem_flag_none); | ||
3035 | mem->gpu_va = 0; | ||
3036 | |||
3037 | gk20a_gmmu_free(vm->mm->g, mem); | ||
3038 | } | ||
3039 | |||
3040 | dma_addr_t gk20a_mm_gpuva_to_iova_base(struct vm_gk20a *vm, u64 gpu_vaddr) | 2649 | dma_addr_t gk20a_mm_gpuva_to_iova_base(struct vm_gk20a *vm, u64 gpu_vaddr) |
3041 | { | 2650 | { |
3042 | struct mapped_buffer_node *buffer; | 2651 | struct mapped_buffer_node *buffer; |