diff options
Diffstat (limited to 'drivers/gpu/nvgpu')
-rw-r--r-- | drivers/gpu/nvgpu/Makefile.nvgpu | 1 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/ioctl_as.c | 6 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/vm.c | 14 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/vm_priv.h | 3 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/vm.c | 13 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/mm_gk20a.c | 241 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/mm_gk20a.h | 21 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/as.h | 10 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/vm.h | 41 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/vm_area.h | 63 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/vgpu/mm_vgpu.c | 15 |
11 files changed, 146 insertions, 282 deletions
diff --git a/drivers/gpu/nvgpu/Makefile.nvgpu b/drivers/gpu/nvgpu/Makefile.nvgpu index 2f9d1b36..6bbf49a5 100644 --- a/drivers/gpu/nvgpu/Makefile.nvgpu +++ b/drivers/gpu/nvgpu/Makefile.nvgpu | |||
@@ -47,6 +47,7 @@ nvgpu-y := \ | |||
47 | common/mm/lockless_allocator.o \ | 47 | common/mm/lockless_allocator.o \ |
48 | common/mm/gmmu.o \ | 48 | common/mm/gmmu.o \ |
49 | common/mm/vm.o \ | 49 | common/mm/vm.o \ |
50 | common/mm/vm_area.o \ | ||
50 | common/pramin.o \ | 51 | common/pramin.o \ |
51 | common/semaphore.o \ | 52 | common/semaphore.o \ |
52 | common/as.o \ | 53 | common/as.o \ |
diff --git a/drivers/gpu/nvgpu/common/linux/ioctl_as.c b/drivers/gpu/nvgpu/common/linux/ioctl_as.c index 7a24a14f..023f8236 100644 --- a/drivers/gpu/nvgpu/common/linux/ioctl_as.c +++ b/drivers/gpu/nvgpu/common/linux/ioctl_as.c | |||
@@ -24,6 +24,7 @@ | |||
24 | #include <uapi/linux/nvgpu.h> | 24 | #include <uapi/linux/nvgpu.h> |
25 | 25 | ||
26 | #include <nvgpu/gmmu.h> | 26 | #include <nvgpu/gmmu.h> |
27 | #include <nvgpu/vm_area.h> | ||
27 | 28 | ||
28 | #include "gk20a/gk20a.h" | 29 | #include "gk20a/gk20a.h" |
29 | #include "gk20a/platform_gk20a.h" | 30 | #include "gk20a/platform_gk20a.h" |
@@ -56,7 +57,8 @@ static int gk20a_as_ioctl_alloc_space( | |||
56 | struct nvgpu_as_alloc_space_args *args) | 57 | struct nvgpu_as_alloc_space_args *args) |
57 | { | 58 | { |
58 | gk20a_dbg_fn(""); | 59 | gk20a_dbg_fn(""); |
59 | return gk20a_vm_alloc_space(as_share, args); | 60 | return nvgpu_vm_area_alloc(as_share->vm, args->pages, args->page_size, |
61 | &args->o_a.offset, args->flags); | ||
60 | } | 62 | } |
61 | 63 | ||
62 | static int gk20a_as_ioctl_free_space( | 64 | static int gk20a_as_ioctl_free_space( |
@@ -64,7 +66,7 @@ static int gk20a_as_ioctl_free_space( | |||
64 | struct nvgpu_as_free_space_args *args) | 66 | struct nvgpu_as_free_space_args *args) |
65 | { | 67 | { |
66 | gk20a_dbg_fn(""); | 68 | gk20a_dbg_fn(""); |
67 | return gk20a_vm_free_space(as_share, args); | 69 | return nvgpu_vm_area_free(as_share->vm, args->offset); |
68 | } | 70 | } |
69 | 71 | ||
70 | static int gk20a_as_ioctl_map_buffer_ex( | 72 | static int gk20a_as_ioctl_map_buffer_ex( |
diff --git a/drivers/gpu/nvgpu/common/linux/vm.c b/drivers/gpu/nvgpu/common/linux/vm.c index 8b9d6f96..5470d9ee 100644 --- a/drivers/gpu/nvgpu/common/linux/vm.c +++ b/drivers/gpu/nvgpu/common/linux/vm.c | |||
@@ -20,6 +20,7 @@ | |||
20 | #include <nvgpu/log.h> | 20 | #include <nvgpu/log.h> |
21 | #include <nvgpu/lock.h> | 21 | #include <nvgpu/lock.h> |
22 | #include <nvgpu/rbtree.h> | 22 | #include <nvgpu/rbtree.h> |
23 | #include <nvgpu/vm_area.h> | ||
23 | #include <nvgpu/page_allocator.h> | 24 | #include <nvgpu/page_allocator.h> |
24 | 25 | ||
25 | #include "gk20a/gk20a.h" | 26 | #include "gk20a/gk20a.h" |
@@ -196,7 +197,7 @@ u64 nvgpu_vm_map(struct vm_gk20a *vm, | |||
196 | struct scatterlist *sgl; | 197 | struct scatterlist *sgl; |
197 | u64 ctag_map_win_size = 0; | 198 | u64 ctag_map_win_size = 0; |
198 | u32 ctag_map_win_ctagline = 0; | 199 | u32 ctag_map_win_ctagline = 0; |
199 | struct vm_reserved_va_node *va_node = NULL; | 200 | struct nvgpu_vm_area *vm_area = NULL; |
200 | u32 ctag_offset; | 201 | u32 ctag_offset; |
201 | enum nvgpu_aperture aperture; | 202 | enum nvgpu_aperture aperture; |
202 | 203 | ||
@@ -256,9 +257,8 @@ u64 nvgpu_vm_map(struct vm_gk20a *vm, | |||
256 | 257 | ||
257 | /* Check if we should use a fixed offset for mapping this buffer */ | 258 | /* Check if we should use a fixed offset for mapping this buffer */ |
258 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) { | 259 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) { |
259 | err = validate_fixed_buffer(vm, &bfr, | 260 | err = nvgpu_vm_area_validate_buffer(vm, offset_align, mapping_size, |
260 | offset_align, mapping_size, | 261 | bfr.pgsz_idx, &vm_area); |
261 | &va_node); | ||
262 | if (err) | 262 | if (err) |
263 | goto clean_up; | 263 | goto clean_up; |
264 | 264 | ||
@@ -376,10 +376,10 @@ u64 nvgpu_vm_map(struct vm_gk20a *vm, | |||
376 | if (user_mapped) | 376 | if (user_mapped) |
377 | vm->num_user_mapped_buffers++; | 377 | vm->num_user_mapped_buffers++; |
378 | 378 | ||
379 | if (va_node) { | 379 | if (vm_area) { |
380 | nvgpu_list_add_tail(&mapped_buffer->buffer_list, | 380 | nvgpu_list_add_tail(&mapped_buffer->buffer_list, |
381 | &va_node->buffer_list_head); | 381 | &vm_area->buffer_list_head); |
382 | mapped_buffer->va_node = va_node; | 382 | mapped_buffer->vm_area = vm_area; |
383 | } | 383 | } |
384 | 384 | ||
385 | nvgpu_mutex_release(&vm->update_gmmu_lock); | 385 | nvgpu_mutex_release(&vm->update_gmmu_lock); |
diff --git a/drivers/gpu/nvgpu/common/linux/vm_priv.h b/drivers/gpu/nvgpu/common/linux/vm_priv.h index 9e064d76..14852264 100644 --- a/drivers/gpu/nvgpu/common/linux/vm_priv.h +++ b/drivers/gpu/nvgpu/common/linux/vm_priv.h | |||
@@ -77,7 +77,7 @@ enum nvgpu_aperture gk20a_dmabuf_aperture(struct gk20a *g, | |||
77 | int validate_fixed_buffer(struct vm_gk20a *vm, | 77 | int validate_fixed_buffer(struct vm_gk20a *vm, |
78 | struct buffer_attrs *bfr, | 78 | struct buffer_attrs *bfr, |
79 | u64 map_offset, u64 map_size, | 79 | u64 map_offset, u64 map_size, |
80 | struct vm_reserved_va_node **pva_node); | 80 | struct nvgpu_vm_area **pva_node); |
81 | int setup_buffer_kind_and_compression(struct vm_gk20a *vm, | 81 | int setup_buffer_kind_and_compression(struct vm_gk20a *vm, |
82 | u32 flags, | 82 | u32 flags, |
83 | struct buffer_attrs *bfr, | 83 | struct buffer_attrs *bfr, |
@@ -89,6 +89,5 @@ int gk20a_alloc_comptags(struct gk20a *g, | |||
89 | u32 lines, bool user_mappable, | 89 | u32 lines, bool user_mappable, |
90 | u64 *ctag_map_win_size, | 90 | u64 *ctag_map_win_size, |
91 | u32 *ctag_map_win_ctagline); | 91 | u32 *ctag_map_win_ctagline); |
92 | void gk20a_vm_unmap_locked_kref(struct kref *ref); | ||
93 | 92 | ||
94 | #endif | 93 | #endif |
diff --git a/drivers/gpu/nvgpu/common/mm/vm.c b/drivers/gpu/nvgpu/common/mm/vm.c index 635ac0fb..3bdc905e 100644 --- a/drivers/gpu/nvgpu/common/mm/vm.c +++ b/drivers/gpu/nvgpu/common/mm/vm.c | |||
@@ -15,6 +15,7 @@ | |||
15 | */ | 15 | */ |
16 | 16 | ||
17 | #include <nvgpu/vm.h> | 17 | #include <nvgpu/vm.h> |
18 | #include <nvgpu/vm_area.h> | ||
18 | #include <nvgpu/lock.h> | 19 | #include <nvgpu/lock.h> |
19 | #include <nvgpu/list.h> | 20 | #include <nvgpu/list.h> |
20 | #include <nvgpu/rbtree.h> | 21 | #include <nvgpu/rbtree.h> |
@@ -58,7 +59,7 @@ void nvgpu_vm_mapping_batch_finish(struct vm_gk20a *vm, | |||
58 | void nvgpu_vm_remove_support_nofree(struct vm_gk20a *vm) | 59 | void nvgpu_vm_remove_support_nofree(struct vm_gk20a *vm) |
59 | { | 60 | { |
60 | struct nvgpu_mapped_buf *mapped_buffer; | 61 | struct nvgpu_mapped_buf *mapped_buffer; |
61 | struct vm_reserved_va_node *va_node, *va_node_tmp; | 62 | struct nvgpu_vm_area *vm_area, *vm_area_tmp; |
62 | struct nvgpu_rbtree_node *node = NULL; | 63 | struct nvgpu_rbtree_node *node = NULL; |
63 | struct gk20a *g = vm->mm->g; | 64 | struct gk20a *g = vm->mm->g; |
64 | 65 | ||
@@ -86,11 +87,11 @@ void nvgpu_vm_remove_support_nofree(struct vm_gk20a *vm) | |||
86 | } | 87 | } |
87 | 88 | ||
88 | /* destroy remaining reserved memory areas */ | 89 | /* destroy remaining reserved memory areas */ |
89 | nvgpu_list_for_each_entry_safe(va_node, va_node_tmp, | 90 | nvgpu_list_for_each_entry_safe(vm_area, vm_area_tmp, |
90 | &vm->reserved_va_list, | 91 | &vm->vm_area_list, |
91 | vm_reserved_va_node, reserved_va_list) { | 92 | nvgpu_vm_area, vm_area_list) { |
92 | nvgpu_list_del(&va_node->reserved_va_list); | 93 | nvgpu_list_del(&vm_area->vm_area_list); |
93 | nvgpu_kfree(vm->mm->g, va_node); | 94 | nvgpu_kfree(vm->mm->g, vm_area); |
94 | } | 95 | } |
95 | 96 | ||
96 | nvgpu_deinit_vm(vm); | 97 | nvgpu_deinit_vm(vm); |
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c index 84919d50..5051f028 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c | |||
@@ -26,6 +26,7 @@ | |||
26 | #include <trace/events/gk20a.h> | 26 | #include <trace/events/gk20a.h> |
27 | 27 | ||
28 | #include <nvgpu/vm.h> | 28 | #include <nvgpu/vm.h> |
29 | #include <nvgpu/vm_area.h> | ||
29 | #include <nvgpu/dma.h> | 30 | #include <nvgpu/dma.h> |
30 | #include <nvgpu/kmem.h> | 31 | #include <nvgpu/kmem.h> |
31 | #include <nvgpu/timers.h> | 32 | #include <nvgpu/timers.h> |
@@ -1065,19 +1066,6 @@ u32 pte_index_from_vaddr(struct vm_gk20a *vm, | |||
1065 | return ret; | 1066 | return ret; |
1066 | } | 1067 | } |
1067 | 1068 | ||
1068 | static struct vm_reserved_va_node *addr_to_reservation(struct vm_gk20a *vm, | ||
1069 | u64 addr) | ||
1070 | { | ||
1071 | struct vm_reserved_va_node *va_node; | ||
1072 | nvgpu_list_for_each_entry(va_node, &vm->reserved_va_list, | ||
1073 | vm_reserved_va_node, reserved_va_list) | ||
1074 | if (addr >= va_node->vaddr_start && | ||
1075 | addr < (u64)va_node->vaddr_start + (u64)va_node->size) | ||
1076 | return va_node; | ||
1077 | |||
1078 | return NULL; | ||
1079 | } | ||
1080 | |||
1081 | int nvgpu_vm_get_buffers(struct vm_gk20a *vm, | 1069 | int nvgpu_vm_get_buffers(struct vm_gk20a *vm, |
1082 | struct nvgpu_mapped_buf ***mapped_buffers, | 1070 | struct nvgpu_mapped_buf ***mapped_buffers, |
1083 | int *num_buffers) | 1071 | int *num_buffers) |
@@ -1301,57 +1289,6 @@ int setup_buffer_kind_and_compression(struct vm_gk20a *vm, | |||
1301 | return 0; | 1289 | return 0; |
1302 | } | 1290 | } |
1303 | 1291 | ||
1304 | int validate_fixed_buffer(struct vm_gk20a *vm, | ||
1305 | struct buffer_attrs *bfr, | ||
1306 | u64 map_offset, u64 map_size, | ||
1307 | struct vm_reserved_va_node **pva_node) | ||
1308 | { | ||
1309 | struct gk20a *g = vm->mm->g; | ||
1310 | struct vm_reserved_va_node *va_node; | ||
1311 | struct nvgpu_mapped_buf *buffer; | ||
1312 | u64 map_end = map_offset + map_size; | ||
1313 | |||
1314 | /* can wrap around with insane map_size; zero is disallowed too */ | ||
1315 | if (map_end <= map_offset) { | ||
1316 | nvgpu_warn(g, "fixed offset mapping with invalid map_size"); | ||
1317 | return -EINVAL; | ||
1318 | } | ||
1319 | |||
1320 | if (map_offset & (vm->gmmu_page_sizes[bfr->pgsz_idx] - 1)) { | ||
1321 | nvgpu_err(g, "map offset must be buffer page size aligned 0x%llx", | ||
1322 | map_offset); | ||
1323 | return -EINVAL; | ||
1324 | } | ||
1325 | |||
1326 | /* Find the space reservation, but it's ok to have none for | ||
1327 | * userspace-managed address spaces */ | ||
1328 | va_node = addr_to_reservation(vm, map_offset); | ||
1329 | if (!va_node && !vm->userspace_managed) { | ||
1330 | nvgpu_warn(g, "fixed offset mapping without space allocation"); | ||
1331 | return -EINVAL; | ||
1332 | } | ||
1333 | |||
1334 | /* Mapped area should fit inside va, if there's one */ | ||
1335 | if (va_node && map_end > va_node->vaddr_start + va_node->size) { | ||
1336 | nvgpu_warn(g, "fixed offset mapping size overflows va node"); | ||
1337 | return -EINVAL; | ||
1338 | } | ||
1339 | |||
1340 | /* check that this mapping does not collide with existing | ||
1341 | * mappings by checking the buffer with the highest GPU VA | ||
1342 | * that is less than our buffer end */ | ||
1343 | buffer = __nvgpu_vm_find_mapped_buf_less_than( | ||
1344 | vm, map_offset + map_size); | ||
1345 | if (buffer && buffer->addr + buffer->size > map_offset) { | ||
1346 | nvgpu_warn(g, "overlapping buffer map requested"); | ||
1347 | return -EINVAL; | ||
1348 | } | ||
1349 | |||
1350 | *pva_node = va_node; | ||
1351 | |||
1352 | return 0; | ||
1353 | } | ||
1354 | |||
1355 | u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm, | 1292 | u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm, |
1356 | u64 map_offset, | 1293 | u64 map_offset, |
1357 | struct sg_table *sgt, | 1294 | struct sg_table *sgt, |
@@ -1850,22 +1787,22 @@ int nvgpu_vm_map_compbits(struct vm_gk20a *vm, | |||
1850 | if (fixed_mapping) { | 1787 | if (fixed_mapping) { |
1851 | struct buffer_attrs bfr; | 1788 | struct buffer_attrs bfr; |
1852 | int err; | 1789 | int err; |
1853 | struct vm_reserved_va_node *va_node = NULL; | 1790 | struct nvgpu_vm_area *vm_area = NULL; |
1854 | 1791 | ||
1855 | memset(&bfr, 0, sizeof(bfr)); | 1792 | memset(&bfr, 0, sizeof(bfr)); |
1856 | 1793 | ||
1857 | bfr.pgsz_idx = small_pgsz_index; | 1794 | bfr.pgsz_idx = small_pgsz_index; |
1858 | 1795 | ||
1859 | err = validate_fixed_buffer( | 1796 | err = nvgpu_vm_area_validate_buffer( |
1860 | vm, &bfr, *compbits_win_gva, | 1797 | vm, *compbits_win_gva, mapped_buffer->ctag_map_win_size, |
1861 | mapped_buffer->ctag_map_win_size, &va_node); | 1798 | bfr.pgsz_idx, &vm_area); |
1862 | 1799 | ||
1863 | if (err) { | 1800 | if (err) { |
1864 | nvgpu_mutex_release(&vm->update_gmmu_lock); | 1801 | nvgpu_mutex_release(&vm->update_gmmu_lock); |
1865 | return err; | 1802 | return err; |
1866 | } | 1803 | } |
1867 | 1804 | ||
1868 | if (va_node) { | 1805 | if (vm_area) { |
1869 | /* this would create a dangling GPU VA | 1806 | /* this would create a dangling GPU VA |
1870 | * pointer if the space is freed | 1807 | * pointer if the space is freed |
1871 | * before before the buffer is | 1808 | * before before the buffer is |
@@ -2564,8 +2501,8 @@ void nvgpu_vm_unmap_locked(struct nvgpu_mapped_buf *mapped_buffer, | |||
2564 | mapped_buffer->pgsz_idx, | 2501 | mapped_buffer->pgsz_idx, |
2565 | mapped_buffer->va_allocated, | 2502 | mapped_buffer->va_allocated, |
2566 | gk20a_mem_flag_none, | 2503 | gk20a_mem_flag_none, |
2567 | mapped_buffer->va_node ? | 2504 | mapped_buffer->vm_area ? |
2568 | mapped_buffer->va_node->sparse : false, | 2505 | mapped_buffer->vm_area->sparse : false, |
2569 | batch); | 2506 | batch); |
2570 | 2507 | ||
2571 | gk20a_dbg(gpu_dbg_map, | 2508 | gk20a_dbg(gpu_dbg_map, |
@@ -2712,13 +2649,13 @@ int gk20a_big_pages_possible(struct vm_gk20a *vm, u64 base, u64 size) | |||
2712 | enum gmmu_pgsz_gk20a __get_pte_size_fixed_map(struct vm_gk20a *vm, | 2649 | enum gmmu_pgsz_gk20a __get_pte_size_fixed_map(struct vm_gk20a *vm, |
2713 | u64 base, u64 size) | 2650 | u64 base, u64 size) |
2714 | { | 2651 | { |
2715 | struct vm_reserved_va_node *node; | 2652 | struct nvgpu_vm_area *vm_area; |
2716 | 2653 | ||
2717 | node = addr_to_reservation(vm, base); | 2654 | vm_area = nvgpu_vm_area_find(vm, base); |
2718 | if (!node) | 2655 | if (!vm_area) |
2719 | return gmmu_page_size_small; | 2656 | return gmmu_page_size_small; |
2720 | 2657 | ||
2721 | return node->pgsz_idx; | 2658 | return vm_area->pgsz_idx; |
2722 | } | 2659 | } |
2723 | 2660 | ||
2724 | /* | 2661 | /* |
@@ -3012,7 +2949,7 @@ int nvgpu_init_vm(struct mm_gk20a *mm, | |||
3012 | 2949 | ||
3013 | nvgpu_mutex_init(&vm->update_gmmu_lock); | 2950 | nvgpu_mutex_init(&vm->update_gmmu_lock); |
3014 | kref_init(&vm->ref); | 2951 | kref_init(&vm->ref); |
3015 | nvgpu_init_list_node(&vm->reserved_va_list); | 2952 | nvgpu_init_list_node(&vm->vm_area_list); |
3016 | 2953 | ||
3017 | /* | 2954 | /* |
3018 | * This is only necessary for channel address spaces. The best way to | 2955 | * This is only necessary for channel address spaces. The best way to |
@@ -3100,158 +3037,6 @@ int gk20a_vm_release_share(struct gk20a_as_share *as_share) | |||
3100 | return 0; | 3037 | return 0; |
3101 | } | 3038 | } |
3102 | 3039 | ||
3103 | |||
3104 | int gk20a_vm_alloc_space(struct gk20a_as_share *as_share, | ||
3105 | struct nvgpu_as_alloc_space_args *args) | ||
3106 | |||
3107 | { | ||
3108 | int err = -ENOMEM; | ||
3109 | int pgsz_idx = gmmu_page_size_small; | ||
3110 | struct nvgpu_allocator *vma; | ||
3111 | struct vm_gk20a *vm = as_share->vm; | ||
3112 | struct gk20a *g = vm->mm->g; | ||
3113 | struct vm_reserved_va_node *va_node; | ||
3114 | u64 vaddr_start = 0; | ||
3115 | int page_sizes = gmmu_nr_page_sizes; | ||
3116 | |||
3117 | gk20a_dbg_fn("flags=0x%x pgsz=0x%x nr_pages=0x%x o/a=0x%llx", | ||
3118 | args->flags, args->page_size, args->pages, | ||
3119 | args->o_a.offset); | ||
3120 | |||
3121 | if (!vm->big_pages) | ||
3122 | page_sizes--; | ||
3123 | |||
3124 | for (; pgsz_idx < page_sizes; pgsz_idx++) { | ||
3125 | if (vm->gmmu_page_sizes[pgsz_idx] == args->page_size) | ||
3126 | break; | ||
3127 | } | ||
3128 | |||
3129 | if (pgsz_idx >= page_sizes) { | ||
3130 | err = -EINVAL; | ||
3131 | goto clean_up; | ||
3132 | } | ||
3133 | |||
3134 | va_node = nvgpu_kzalloc(g, sizeof(*va_node)); | ||
3135 | if (!va_node) { | ||
3136 | err = -ENOMEM; | ||
3137 | goto clean_up; | ||
3138 | } | ||
3139 | |||
3140 | vma = vm->vma[pgsz_idx]; | ||
3141 | if (args->flags & NVGPU_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET) | ||
3142 | vaddr_start = nvgpu_alloc_fixed(vma, args->o_a.offset, | ||
3143 | (u64)args->pages * | ||
3144 | (u64)args->page_size, | ||
3145 | args->page_size); | ||
3146 | else | ||
3147 | vaddr_start = nvgpu_alloc(vma, | ||
3148 | (u64)args->pages * | ||
3149 | (u64)args->page_size); | ||
3150 | |||
3151 | if (!vaddr_start) { | ||
3152 | nvgpu_kfree(g, va_node); | ||
3153 | goto clean_up; | ||
3154 | } | ||
3155 | |||
3156 | va_node->vaddr_start = vaddr_start; | ||
3157 | va_node->size = (u64)args->page_size * (u64)args->pages; | ||
3158 | va_node->pgsz_idx = pgsz_idx; | ||
3159 | nvgpu_init_list_node(&va_node->buffer_list_head); | ||
3160 | nvgpu_init_list_node(&va_node->reserved_va_list); | ||
3161 | |||
3162 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
3163 | |||
3164 | /* mark that we need to use sparse mappings here */ | ||
3165 | if (args->flags & NVGPU_AS_ALLOC_SPACE_FLAGS_SPARSE) { | ||
3166 | u64 map_offset = g->ops.mm.gmmu_map(vm, vaddr_start, | ||
3167 | NULL, | ||
3168 | 0, | ||
3169 | va_node->size, | ||
3170 | pgsz_idx, | ||
3171 | 0, | ||
3172 | 0, | ||
3173 | args->flags, | ||
3174 | gk20a_mem_flag_none, | ||
3175 | false, | ||
3176 | true, | ||
3177 | false, | ||
3178 | NULL, | ||
3179 | APERTURE_INVALID); | ||
3180 | if (!map_offset) { | ||
3181 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
3182 | nvgpu_free(vma, vaddr_start); | ||
3183 | nvgpu_kfree(g, va_node); | ||
3184 | goto clean_up; | ||
3185 | } | ||
3186 | |||
3187 | va_node->sparse = true; | ||
3188 | } | ||
3189 | nvgpu_list_add_tail(&va_node->reserved_va_list, &vm->reserved_va_list); | ||
3190 | |||
3191 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
3192 | |||
3193 | args->o_a.offset = vaddr_start; | ||
3194 | err = 0; | ||
3195 | |||
3196 | clean_up: | ||
3197 | return err; | ||
3198 | } | ||
3199 | |||
3200 | int gk20a_vm_free_space(struct gk20a_as_share *as_share, | ||
3201 | struct nvgpu_as_free_space_args *args) | ||
3202 | { | ||
3203 | int err = -ENOMEM; | ||
3204 | int pgsz_idx; | ||
3205 | struct nvgpu_allocator *vma; | ||
3206 | struct vm_gk20a *vm = as_share->vm; | ||
3207 | struct vm_reserved_va_node *va_node; | ||
3208 | struct gk20a *g = gk20a_from_vm(vm); | ||
3209 | |||
3210 | gk20a_dbg_fn("pgsz=0x%x nr_pages=0x%x o/a=0x%llx", args->page_size, | ||
3211 | args->pages, args->offset); | ||
3212 | |||
3213 | /* determine pagesz idx */ | ||
3214 | pgsz_idx = __get_pte_size(vm, args->offset, | ||
3215 | args->page_size * args->pages); | ||
3216 | |||
3217 | vma = vm->vma[pgsz_idx]; | ||
3218 | nvgpu_free(vma, args->offset); | ||
3219 | |||
3220 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
3221 | va_node = addr_to_reservation(vm, args->offset); | ||
3222 | if (va_node) { | ||
3223 | struct nvgpu_mapped_buf *buffer, *n; | ||
3224 | |||
3225 | /* Decrement the ref count on all buffers in this va_node. This | ||
3226 | * allows userspace to let the kernel free mappings that are | ||
3227 | * only used by this va_node. */ | ||
3228 | nvgpu_list_for_each_entry_safe(buffer, n, | ||
3229 | &va_node->buffer_list_head, | ||
3230 | nvgpu_mapped_buf, buffer_list) { | ||
3231 | nvgpu_list_del(&buffer->buffer_list); | ||
3232 | kref_put(&buffer->ref, gk20a_vm_unmap_locked_kref); | ||
3233 | } | ||
3234 | |||
3235 | nvgpu_list_del(&va_node->reserved_va_list); | ||
3236 | |||
3237 | /* if this was a sparse mapping, free the va */ | ||
3238 | if (va_node->sparse) | ||
3239 | g->ops.mm.gmmu_unmap(vm, | ||
3240 | va_node->vaddr_start, | ||
3241 | va_node->size, | ||
3242 | va_node->pgsz_idx, | ||
3243 | true, | ||
3244 | gk20a_mem_flag_none, | ||
3245 | true, | ||
3246 | NULL); | ||
3247 | nvgpu_kfree(g, va_node); | ||
3248 | } | ||
3249 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
3250 | err = 0; | ||
3251 | |||
3252 | return err; | ||
3253 | } | ||
3254 | |||
3255 | int __gk20a_vm_bind_channel(struct vm_gk20a *vm, struct channel_gk20a *ch) | 3040 | int __gk20a_vm_bind_channel(struct vm_gk20a *vm, struct channel_gk20a *ch) |
3256 | { | 3041 | { |
3257 | int err = 0; | 3042 | int err = 0; |
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h index 357962c7..6ddf842a 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h | |||
@@ -140,22 +140,6 @@ struct priv_cmd_entry { | |||
140 | u32 size; /* in words */ | 140 | u32 size; /* in words */ |
141 | }; | 141 | }; |
142 | 142 | ||
143 | struct vm_reserved_va_node { | ||
144 | struct nvgpu_list_node reserved_va_list; | ||
145 | struct nvgpu_list_node buffer_list_head; | ||
146 | u32 pgsz_idx; | ||
147 | u64 vaddr_start; | ||
148 | u64 size; | ||
149 | bool sparse; | ||
150 | }; | ||
151 | |||
152 | static inline struct vm_reserved_va_node * | ||
153 | vm_reserved_va_node_from_reserved_va_list(struct nvgpu_list_node *node) | ||
154 | { | ||
155 | return (struct vm_reserved_va_node *) | ||
156 | ((uintptr_t)node - offsetof(struct vm_reserved_va_node, reserved_va_list)); | ||
157 | }; | ||
158 | |||
159 | struct gk20a; | 143 | struct gk20a; |
160 | struct channel_gk20a; | 144 | struct channel_gk20a; |
161 | 145 | ||
@@ -442,10 +426,6 @@ struct nvgpu_as_free_space_args; | |||
442 | int gk20a_vm_alloc_share(struct gk20a_as_share *as_share, u32 big_page_size, | 426 | int gk20a_vm_alloc_share(struct gk20a_as_share *as_share, u32 big_page_size, |
443 | u32 flags); | 427 | u32 flags); |
444 | int gk20a_vm_release_share(struct gk20a_as_share *as_share); | 428 | int gk20a_vm_release_share(struct gk20a_as_share *as_share); |
445 | int gk20a_vm_alloc_space(struct gk20a_as_share *as_share, | ||
446 | struct nvgpu_as_alloc_space_args *args); | ||
447 | int gk20a_vm_free_space(struct gk20a_as_share *as_share, | ||
448 | struct nvgpu_as_free_space_args *args); | ||
449 | int gk20a_vm_bind_channel(struct gk20a_as_share *as_share, | 429 | int gk20a_vm_bind_channel(struct gk20a_as_share *as_share, |
450 | struct channel_gk20a *ch); | 430 | struct channel_gk20a *ch); |
451 | int __gk20a_vm_bind_channel(struct vm_gk20a *vm, struct channel_gk20a *ch); | 431 | int __gk20a_vm_bind_channel(struct vm_gk20a *vm, struct channel_gk20a *ch); |
@@ -491,5 +471,6 @@ extern const struct gk20a_mmu_level gk20a_mm_levels_128k[]; | |||
491 | 471 | ||
492 | int gk20a_mm_get_buffer_info(struct device *dev, int dmabuf_fd, | 472 | int gk20a_mm_get_buffer_info(struct device *dev, int dmabuf_fd, |
493 | u64 *buffer_id, u64 *buffer_len); | 473 | u64 *buffer_id, u64 *buffer_len); |
474 | void gk20a_vm_unmap_locked_kref(struct kref *ref); | ||
494 | 475 | ||
495 | #endif /* MM_GK20A_H */ | 476 | #endif /* MM_GK20A_H */ |
diff --git a/drivers/gpu/nvgpu/include/nvgpu/as.h b/drivers/gpu/nvgpu/include/nvgpu/as.h index 0e784396..e3233f87 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/as.h +++ b/drivers/gpu/nvgpu/include/nvgpu/as.h | |||
@@ -17,14 +17,14 @@ | |||
17 | 17 | ||
18 | struct vm_gk20a; | 18 | struct vm_gk20a; |
19 | 19 | ||
20 | struct gk20a_as { | ||
21 | int last_share_id; /* dummy allocator for now */ | ||
22 | }; | ||
23 | |||
20 | struct gk20a_as_share { | 24 | struct gk20a_as_share { |
21 | struct gk20a_as *as; | 25 | struct gk20a_as *as; |
22 | int id; | ||
23 | struct vm_gk20a *vm; | 26 | struct vm_gk20a *vm; |
24 | }; | 27 | int id; |
25 | |||
26 | struct gk20a_as { | ||
27 | int last_share_id; /* dummy allocator for now */ | ||
28 | }; | 28 | }; |
29 | 29 | ||
30 | int gk20a_as_release_share(struct gk20a_as_share *as_share); | 30 | int gk20a_as_release_share(struct gk20a_as_share *as_share); |
diff --git a/drivers/gpu/nvgpu/include/nvgpu/vm.h b/drivers/gpu/nvgpu/include/nvgpu/vm.h index e1ceffd4..69c08c77 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/vm.h +++ b/drivers/gpu/nvgpu/include/nvgpu/vm.h | |||
@@ -26,11 +26,10 @@ | |||
26 | #include <nvgpu/allocator.h> | 26 | #include <nvgpu/allocator.h> |
27 | 27 | ||
28 | struct vm_gk20a; | 28 | struct vm_gk20a; |
29 | struct vm_reserved_va_node; | 29 | struct nvgpu_vm_area; |
30 | struct buffer_attrs; | 30 | struct buffer_attrs; |
31 | struct gk20a_comptag_allocator; | 31 | struct gk20a_comptag_allocator; |
32 | 32 | ||
33 | |||
34 | /** | 33 | /** |
35 | * This header contains the OS agnostic APIs for dealing with VMs. Most of the | 34 | * This header contains the OS agnostic APIs for dealing with VMs. Most of the |
36 | * VM implementation is system specific - it must translate from a platform's | 35 | * VM implementation is system specific - it must translate from a platform's |
@@ -39,6 +38,38 @@ struct gk20a_comptag_allocator; | |||
39 | * However, some stuff is platform agnostic. VM ref-counting and the VM struct | 38 | * However, some stuff is platform agnostic. VM ref-counting and the VM struct |
40 | * itself are platform agnostic. Also, the initialization and destruction of | 39 | * itself are platform agnostic. Also, the initialization and destruction of |
41 | * VMs is the same across all platforms (for now). | 40 | * VMs is the same across all platforms (for now). |
41 | * | ||
42 | * VM Architecture: | ||
43 | * ---------------- | ||
44 | * | ||
45 | * The VM managment in nvgpu is split up as follows: a vm_gk20a struct which | ||
46 | * defines an address space. Each address space is a set of page tables and a | ||
47 | * GPU Virtual Address (GVA) allocator. Any number of channels may bind to a VM. | ||
48 | * | ||
49 | * +----+ +----+ +----+ +-----+ +-----+ | ||
50 | * | C1 | | C2 | ... | Cn | | VM1 | ... | VMn | | ||
51 | * +-+--+ +-+--+ +-+--+ +--+--+ +--+--+ | ||
52 | * | | | | | | ||
53 | * | | +----->-----+ | | ||
54 | * | +---------------->-----+ | | ||
55 | * +------------------------>-----------------+ | ||
56 | * | ||
57 | * Each VM also manages a set of mapped buffers (struct nvgpu_mapped_buf) | ||
58 | * which corresponds to _user space_ buffers which have been mapped into this VM. | ||
59 | * Kernel space mappings (created by nvgpu_gmmu_map()) are not tracked by VMs. | ||
60 | * This may be an architectural bug, but for now it seems to be OK. VMs can be | ||
61 | * closed in various ways - refs counts hitting zero, direct calls to the remove | ||
62 | * routine, etc. Note: this is going to change. VM cleanup is going to be | ||
63 | * homogonized around ref-counts. When a VM is closed all mapped buffers in the | ||
64 | * VM are unmapped from the GMMU. This means that those mappings will no longer | ||
65 | * be valid and any subsequent access by the GPU will fault. That means one must | ||
66 | * ensure the VM is not in use before closing it. | ||
67 | * | ||
68 | * VMs may also contain VM areas (struct nvgpu_vm_area) which are created for | ||
69 | * the purpose of sparse and/or fixed mappings. If userspace wishes to create a | ||
70 | * fixed mapping it must first create a VM area - either with a fixed address or | ||
71 | * not. VM areas are reserved - other mapping operations will not use the space. | ||
72 | * Userspace may then create fixed mappings within that VM area. | ||
42 | */ | 73 | */ |
43 | 74 | ||
44 | /* map/unmap batch state */ | 75 | /* map/unmap batch state */ |
@@ -49,9 +80,10 @@ struct vm_gk20a_mapping_batch { | |||
49 | 80 | ||
50 | struct nvgpu_mapped_buf { | 81 | struct nvgpu_mapped_buf { |
51 | struct vm_gk20a *vm; | 82 | struct vm_gk20a *vm; |
83 | struct nvgpu_vm_area *vm_area; | ||
84 | |||
52 | struct nvgpu_rbtree_node node; | 85 | struct nvgpu_rbtree_node node; |
53 | struct nvgpu_list_node buffer_list; | 86 | struct nvgpu_list_node buffer_list; |
54 | struct vm_reserved_va_node *va_node; | ||
55 | u64 addr; | 87 | u64 addr; |
56 | u64 size; | 88 | u64 size; |
57 | struct dma_buf *dmabuf; | 89 | struct dma_buf *dmabuf; |
@@ -102,7 +134,6 @@ struct vm_gk20a { | |||
102 | 134 | ||
103 | bool big_pages; /* enable large page support */ | 135 | bool big_pages; /* enable large page support */ |
104 | bool enable_ctag; | 136 | bool enable_ctag; |
105 | bool mapped; | ||
106 | 137 | ||
107 | u32 big_page_size; | 138 | u32 big_page_size; |
108 | 139 | ||
@@ -129,7 +160,7 @@ struct vm_gk20a { | |||
129 | 160 | ||
130 | struct nvgpu_rbtree_node *mapped_buffers; | 161 | struct nvgpu_rbtree_node *mapped_buffers; |
131 | 162 | ||
132 | struct nvgpu_list_node reserved_va_list; | 163 | struct nvgpu_list_node vm_area_list; |
133 | 164 | ||
134 | #ifdef CONFIG_TEGRA_GR_VIRTUALIZATION | 165 | #ifdef CONFIG_TEGRA_GR_VIRTUALIZATION |
135 | u64 handle; | 166 | u64 handle; |
diff --git a/drivers/gpu/nvgpu/include/nvgpu/vm_area.h b/drivers/gpu/nvgpu/include/nvgpu/vm_area.h new file mode 100644 index 00000000..ffe4b99b --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/vm_area.h | |||
@@ -0,0 +1,63 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify it | ||
5 | * under the terms and conditions of the GNU General Public License, | ||
6 | * version 2, as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | * more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | |||
17 | #ifndef __NVGPU_VM_AREA_H__ | ||
18 | #define __NVGPU_VM_AREA_H__ | ||
19 | |||
20 | #include <nvgpu/list.h> | ||
21 | #include <nvgpu/types.h> | ||
22 | |||
23 | struct vm_gk20a; | ||
24 | struct gk20a_as_share; | ||
25 | struct nvgpu_as_alloc_space_args; | ||
26 | struct nvgpu_as_free_space_args; | ||
27 | |||
28 | struct nvgpu_vm_area { | ||
29 | /* | ||
30 | * Entry into the list of VM areas owned by a VM. | ||
31 | */ | ||
32 | struct nvgpu_list_node vm_area_list; | ||
33 | |||
34 | /* | ||
35 | * List of buffers mapped into this vm_area. | ||
36 | */ | ||
37 | struct nvgpu_list_node buffer_list_head; | ||
38 | |||
39 | u32 flags; | ||
40 | u32 pgsz_idx; | ||
41 | u64 addr; | ||
42 | u64 size; | ||
43 | bool sparse; | ||
44 | }; | ||
45 | |||
46 | static inline struct nvgpu_vm_area * | ||
47 | nvgpu_vm_area_from_vm_area_list(struct nvgpu_list_node *node) | ||
48 | { | ||
49 | return (struct nvgpu_vm_area *) | ||
50 | ((uintptr_t)node - offsetof(struct nvgpu_vm_area, | ||
51 | vm_area_list)); | ||
52 | }; | ||
53 | |||
54 | int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u32 pages, u32 page_size, | ||
55 | u64 *addr, u32 flags); | ||
56 | int nvgpu_vm_area_free(struct vm_gk20a *vm, u64 addr); | ||
57 | |||
58 | struct nvgpu_vm_area *nvgpu_vm_area_find(struct vm_gk20a *vm, u64 addr); | ||
59 | int nvgpu_vm_area_validate_buffer(struct vm_gk20a *vm, | ||
60 | u64 map_offset, u64 map_size, int pgsz_idx, | ||
61 | struct nvgpu_vm_area **pvm_area); | ||
62 | |||
63 | #endif | ||
diff --git a/drivers/gpu/nvgpu/vgpu/mm_vgpu.c b/drivers/gpu/nvgpu/vgpu/mm_vgpu.c index f4004f42..b42fbcb3 100644 --- a/drivers/gpu/nvgpu/vgpu/mm_vgpu.c +++ b/drivers/gpu/nvgpu/vgpu/mm_vgpu.c | |||
@@ -18,6 +18,7 @@ | |||
18 | #include <nvgpu/kmem.h> | 18 | #include <nvgpu/kmem.h> |
19 | #include <nvgpu/dma.h> | 19 | #include <nvgpu/dma.h> |
20 | #include <nvgpu/bug.h> | 20 | #include <nvgpu/bug.h> |
21 | #include <nvgpu/vm_area.h> | ||
21 | 22 | ||
22 | #include "vgpu/vgpu.h" | 23 | #include "vgpu/vgpu.h" |
23 | #include "gk20a/mm_gk20a.h" | 24 | #include "gk20a/mm_gk20a.h" |
@@ -203,7 +204,7 @@ static void vgpu_vm_remove_support(struct vm_gk20a *vm) | |||
203 | { | 204 | { |
204 | struct gk20a *g = vm->mm->g; | 205 | struct gk20a *g = vm->mm->g; |
205 | struct nvgpu_mapped_buf *mapped_buffer; | 206 | struct nvgpu_mapped_buf *mapped_buffer; |
206 | struct vm_reserved_va_node *va_node, *va_node_tmp; | 207 | struct nvgpu_vm_area *vm_area, *vm_area_tmp; |
207 | struct tegra_vgpu_cmd_msg msg; | 208 | struct tegra_vgpu_cmd_msg msg; |
208 | struct tegra_vgpu_as_share_params *p = &msg.params.as_share; | 209 | struct tegra_vgpu_as_share_params *p = &msg.params.as_share; |
209 | struct nvgpu_rbtree_node *node = NULL; | 210 | struct nvgpu_rbtree_node *node = NULL; |
@@ -223,11 +224,11 @@ static void vgpu_vm_remove_support(struct vm_gk20a *vm) | |||
223 | } | 224 | } |
224 | 225 | ||
225 | /* destroy remaining reserved memory areas */ | 226 | /* destroy remaining reserved memory areas */ |
226 | nvgpu_list_for_each_entry_safe(va_node, va_node_tmp, | 227 | nvgpu_list_for_each_entry_safe(vm_area, vm_area_tmp, |
227 | &vm->reserved_va_list, | 228 | &vm->vm_area_list, |
228 | vm_reserved_va_node, reserved_va_list) { | 229 | nvgpu_vm_area, vm_area_list) { |
229 | nvgpu_list_del(&va_node->reserved_va_list); | 230 | nvgpu_list_del(&vm_area->vm_area_list); |
230 | nvgpu_kfree(g, va_node); | 231 | nvgpu_kfree(g, vm_area); |
231 | } | 232 | } |
232 | 233 | ||
233 | msg.cmd = TEGRA_VGPU_CMD_AS_FREE_SHARE; | 234 | msg.cmd = TEGRA_VGPU_CMD_AS_FREE_SHARE; |
@@ -413,7 +414,7 @@ static int vgpu_vm_alloc_share(struct gk20a_as_share *as_share, | |||
413 | 414 | ||
414 | nvgpu_mutex_init(&vm->update_gmmu_lock); | 415 | nvgpu_mutex_init(&vm->update_gmmu_lock); |
415 | kref_init(&vm->ref); | 416 | kref_init(&vm->ref); |
416 | nvgpu_init_list_node(&vm->reserved_va_list); | 417 | nvgpu_init_list_node(&vm->vm_area_list); |
417 | 418 | ||
418 | vm->enable_ctag = true; | 419 | vm->enable_ctag = true; |
419 | 420 | ||