diff options
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a/mm_gk20a.c')
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/mm_gk20a.c | 115 |
1 files changed, 69 insertions, 46 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c index 654938b2..3feb675b 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c | |||
@@ -88,7 +88,6 @@ static inline u32 lo32(u64 f) | |||
88 | return (u32)(f & 0xffffffff); | 88 | return (u32)(f & 0xffffffff); |
89 | } | 89 | } |
90 | 90 | ||
91 | static void gk20a_vm_unmap_locked(struct mapped_buffer_node *mapped_buffer); | ||
92 | static struct mapped_buffer_node *find_mapped_buffer_locked( | 91 | static struct mapped_buffer_node *find_mapped_buffer_locked( |
93 | struct rb_root *root, u64 addr); | 92 | struct rb_root *root, u64 addr); |
94 | static struct mapped_buffer_node *find_mapped_buffer_reverse_locked( | 93 | static struct mapped_buffer_node *find_mapped_buffer_reverse_locked( |
@@ -100,7 +99,6 @@ static int update_gmmu_ptes_locked(struct vm_gk20a *vm, | |||
100 | u64 first_vaddr, u64 last_vaddr, | 99 | u64 first_vaddr, u64 last_vaddr, |
101 | u8 kind_v, u32 ctag_offset, bool cacheable, | 100 | u8 kind_v, u32 ctag_offset, bool cacheable, |
102 | int rw_flag); | 101 | int rw_flag); |
103 | static void gk20a_vm_remove_support(struct vm_gk20a *vm); | ||
104 | static int gk20a_init_system_vm(struct mm_gk20a *mm); | 102 | static int gk20a_init_system_vm(struct mm_gk20a *mm); |
105 | static int gk20a_init_bar1_vm(struct mm_gk20a *mm); | 103 | static int gk20a_init_bar1_vm(struct mm_gk20a *mm); |
106 | 104 | ||
@@ -335,6 +333,8 @@ int gk20a_init_mm_setup_sw(struct gk20a *g) | |||
335 | gk20a_init_bar1_vm(mm); | 333 | gk20a_init_bar1_vm(mm); |
336 | gk20a_init_system_vm(mm); | 334 | gk20a_init_system_vm(mm); |
337 | 335 | ||
336 | /* set vm_alloc_share op here as gk20a_as_alloc_share needs it */ | ||
337 | g->ops.mm.vm_alloc_share = gk20a_vm_alloc_share; | ||
338 | mm->remove_support = gk20a_remove_mm_support; | 338 | mm->remove_support = gk20a_remove_mm_support; |
339 | mm->sw_ready = true; | 339 | mm->sw_ready = true; |
340 | 340 | ||
@@ -833,9 +833,9 @@ static void gk20a_vm_unmap_user(struct vm_gk20a *vm, u64 offset) | |||
833 | mutex_unlock(&vm->update_gmmu_lock); | 833 | mutex_unlock(&vm->update_gmmu_lock); |
834 | } | 834 | } |
835 | 835 | ||
836 | static u64 gk20a_vm_alloc_va(struct vm_gk20a *vm, | 836 | u64 gk20a_vm_alloc_va(struct vm_gk20a *vm, |
837 | u64 size, | 837 | u64 size, |
838 | enum gmmu_pgsz_gk20a gmmu_pgsz_idx) | 838 | enum gmmu_pgsz_gk20a gmmu_pgsz_idx) |
839 | 839 | ||
840 | { | 840 | { |
841 | struct gk20a_allocator *vma = &vm->vma[gmmu_pgsz_idx]; | 841 | struct gk20a_allocator *vma = &vm->vma[gmmu_pgsz_idx]; |
@@ -881,9 +881,9 @@ static u64 gk20a_vm_alloc_va(struct vm_gk20a *vm, | |||
881 | return offset; | 881 | return offset; |
882 | } | 882 | } |
883 | 883 | ||
884 | static int gk20a_vm_free_va(struct vm_gk20a *vm, | 884 | int gk20a_vm_free_va(struct vm_gk20a *vm, |
885 | u64 offset, u64 size, | 885 | u64 offset, u64 size, |
886 | enum gmmu_pgsz_gk20a pgsz_idx) | 886 | enum gmmu_pgsz_gk20a pgsz_idx) |
887 | { | 887 | { |
888 | struct gk20a_allocator *vma = &vm->vma[pgsz_idx]; | 888 | struct gk20a_allocator *vma = &vm->vma[pgsz_idx]; |
889 | u32 page_size = gmmu_page_sizes[pgsz_idx]; | 889 | u32 page_size = gmmu_page_sizes[pgsz_idx]; |
@@ -1100,21 +1100,32 @@ static int validate_fixed_buffer(struct vm_gk20a *vm, | |||
1100 | return 0; | 1100 | return 0; |
1101 | } | 1101 | } |
1102 | 1102 | ||
1103 | static u64 __locked_gmmu_map(struct vm_gk20a *vm, | 1103 | u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm, |
1104 | u64 map_offset, | 1104 | u64 map_offset, |
1105 | struct sg_table *sgt, | 1105 | struct sg_table *sgt, |
1106 | u64 buffer_offset, | 1106 | u64 buffer_offset, |
1107 | u64 size, | 1107 | u64 size, |
1108 | int pgsz_idx, | 1108 | int pgsz_idx, |
1109 | u8 kind_v, | 1109 | u8 kind_v, |
1110 | u32 ctag_offset, | 1110 | u32 ctag_offset, |
1111 | u32 flags, | 1111 | u32 flags, |
1112 | int rw_flag) | 1112 | int rw_flag, |
1113 | bool clear_ctags) | ||
1113 | { | 1114 | { |
1114 | int err = 0, i = 0; | 1115 | int err = 0, i = 0; |
1115 | bool allocated = false; | 1116 | bool allocated = false; |
1116 | u32 pde_lo, pde_hi; | 1117 | u32 pde_lo, pde_hi; |
1117 | struct device *d = dev_from_vm(vm); | 1118 | struct device *d = dev_from_vm(vm); |
1119 | struct gk20a *g = gk20a_from_vm(vm); | ||
1120 | |||
1121 | if (clear_ctags && ctag_offset) { | ||
1122 | u32 ctag_lines = ALIGN(size, COMP_TAG_LINE_SIZE) >> | ||
1123 | COMP_TAG_LINE_SIZE_SHIFT; | ||
1124 | |||
1125 | /* init/clear the ctag buffer */ | ||
1126 | g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_clear, | ||
1127 | ctag_offset, ctag_offset + ctag_lines - 1); | ||
1128 | } | ||
1118 | 1129 | ||
1119 | /* Allocate (or validate when map_offset != 0) the virtual address. */ | 1130 | /* Allocate (or validate when map_offset != 0) the virtual address. */ |
1120 | if (!map_offset) { | 1131 | if (!map_offset) { |
@@ -1167,12 +1178,12 @@ fail_alloc: | |||
1167 | return 0; | 1178 | return 0; |
1168 | } | 1179 | } |
1169 | 1180 | ||
1170 | static void __locked_gmmu_unmap(struct vm_gk20a *vm, | 1181 | void gk20a_locked_gmmu_unmap(struct vm_gk20a *vm, |
1171 | u64 vaddr, | 1182 | u64 vaddr, |
1172 | u64 size, | 1183 | u64 size, |
1173 | int pgsz_idx, | 1184 | int pgsz_idx, |
1174 | bool va_allocated, | 1185 | bool va_allocated, |
1175 | int rw_flag) | 1186 | int rw_flag) |
1176 | { | 1187 | { |
1177 | int err = 0; | 1188 | int err = 0; |
1178 | struct gk20a *g = gk20a_from_vm(vm); | 1189 | struct gk20a *g = gk20a_from_vm(vm); |
@@ -1298,6 +1309,7 @@ u64 gk20a_vm_map(struct vm_gk20a *vm, | |||
1298 | struct buffer_attrs bfr = {0}; | 1309 | struct buffer_attrs bfr = {0}; |
1299 | struct gk20a_comptags comptags; | 1310 | struct gk20a_comptags comptags; |
1300 | u64 buf_addr; | 1311 | u64 buf_addr; |
1312 | bool clear_ctags = false; | ||
1301 | 1313 | ||
1302 | mutex_lock(&vm->update_gmmu_lock); | 1314 | mutex_lock(&vm->update_gmmu_lock); |
1303 | 1315 | ||
@@ -1402,11 +1414,7 @@ u64 gk20a_vm_map(struct vm_gk20a *vm, | |||
1402 | bfr.kind_v = bfr.uc_kind_v; | 1414 | bfr.kind_v = bfr.uc_kind_v; |
1403 | } else { | 1415 | } else { |
1404 | gk20a_get_comptags(d, dmabuf, &comptags); | 1416 | gk20a_get_comptags(d, dmabuf, &comptags); |
1405 | 1417 | clear_ctags = true; | |
1406 | /* init/clear the ctag buffer */ | ||
1407 | g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_clear, | ||
1408 | comptags.offset, | ||
1409 | comptags.offset + comptags.lines - 1); | ||
1410 | } | 1418 | } |
1411 | } | 1419 | } |
1412 | 1420 | ||
@@ -1414,15 +1422,15 @@ u64 gk20a_vm_map(struct vm_gk20a *vm, | |||
1414 | bfr.ctag_offset = comptags.offset; | 1422 | bfr.ctag_offset = comptags.offset; |
1415 | 1423 | ||
1416 | /* update gmmu ptes */ | 1424 | /* update gmmu ptes */ |
1417 | map_offset = __locked_gmmu_map(vm, map_offset, | 1425 | map_offset = g->ops.mm.gmmu_map(vm, map_offset, |
1418 | bfr.sgt, | 1426 | bfr.sgt, |
1419 | buffer_offset, /* sg offset */ | 1427 | buffer_offset, /* sg offset */ |
1420 | mapping_size, | 1428 | mapping_size, |
1421 | bfr.pgsz_idx, | 1429 | bfr.pgsz_idx, |
1422 | bfr.kind_v, | 1430 | bfr.kind_v, |
1423 | bfr.ctag_offset, | 1431 | bfr.ctag_offset, |
1424 | flags, rw_flag); | 1432 | flags, rw_flag, |
1425 | 1433 | clear_ctags); | |
1426 | if (!map_offset) | 1434 | if (!map_offset) |
1427 | goto clean_up; | 1435 | goto clean_up; |
1428 | 1436 | ||
@@ -1531,17 +1539,18 @@ u64 gk20a_gmmu_map(struct vm_gk20a *vm, | |||
1531 | u32 flags, | 1539 | u32 flags, |
1532 | int rw_flag) | 1540 | int rw_flag) |
1533 | { | 1541 | { |
1542 | struct gk20a *g = gk20a_from_vm(vm); | ||
1534 | u64 vaddr; | 1543 | u64 vaddr; |
1535 | 1544 | ||
1536 | mutex_lock(&vm->update_gmmu_lock); | 1545 | mutex_lock(&vm->update_gmmu_lock); |
1537 | vaddr = __locked_gmmu_map(vm, 0, /* already mapped? - No */ | 1546 | vaddr = g->ops.mm.gmmu_map(vm, 0, /* already mapped? - No */ |
1538 | *sgt, /* sg table */ | 1547 | *sgt, /* sg table */ |
1539 | 0, /* sg offset */ | 1548 | 0, /* sg offset */ |
1540 | size, | 1549 | size, |
1541 | 0, /* page size index = 0 i.e. SZ_4K */ | 1550 | 0, /* page size index = 0 i.e. SZ_4K */ |
1542 | 0, /* kind */ | 1551 | 0, /* kind */ |
1543 | 0, /* ctag_offset */ | 1552 | 0, /* ctag_offset */ |
1544 | flags, rw_flag); | 1553 | flags, rw_flag, false); |
1545 | mutex_unlock(&vm->update_gmmu_lock); | 1554 | mutex_unlock(&vm->update_gmmu_lock); |
1546 | if (!vaddr) { | 1555 | if (!vaddr) { |
1547 | gk20a_err(dev_from_vm(vm), "failed to allocate va space"); | 1556 | gk20a_err(dev_from_vm(vm), "failed to allocate va space"); |
@@ -1549,7 +1558,7 @@ u64 gk20a_gmmu_map(struct vm_gk20a *vm, | |||
1549 | } | 1558 | } |
1550 | 1559 | ||
1551 | /* Invalidate kernel mappings immediately */ | 1560 | /* Invalidate kernel mappings immediately */ |
1552 | gk20a_mm_tlb_invalidate(vm); | 1561 | g->ops.mm.tlb_invalidate(vm); |
1553 | 1562 | ||
1554 | return vaddr; | 1563 | return vaddr; |
1555 | } | 1564 | } |
@@ -1573,8 +1582,10 @@ void gk20a_gmmu_unmap(struct vm_gk20a *vm, | |||
1573 | u64 size, | 1582 | u64 size, |
1574 | int rw_flag) | 1583 | int rw_flag) |
1575 | { | 1584 | { |
1585 | struct gk20a *g = gk20a_from_vm(vm); | ||
1586 | |||
1576 | mutex_lock(&vm->update_gmmu_lock); | 1587 | mutex_lock(&vm->update_gmmu_lock); |
1577 | __locked_gmmu_unmap(vm, | 1588 | g->ops.mm.gmmu_unmap(vm, |
1578 | vaddr, | 1589 | vaddr, |
1579 | size, | 1590 | size, |
1580 | 0, /* page size 4K */ | 1591 | 0, /* page size 4K */ |
@@ -1970,10 +1981,10 @@ static int gk20a_vm_put_empty(struct vm_gk20a *vm, u64 vaddr, | |||
1970 | } | 1981 | } |
1971 | 1982 | ||
1972 | for (i = 0; i < num_pages; i++) { | 1983 | for (i = 0; i < num_pages; i++) { |
1973 | u64 page_vaddr = __locked_gmmu_map(vm, vaddr, | 1984 | u64 page_vaddr = g->ops.mm.gmmu_map(vm, vaddr, |
1974 | vm->zero_page_sgt, 0, pgsz, pgsz_idx, 0, 0, | 1985 | vm->zero_page_sgt, 0, pgsz, pgsz_idx, 0, 0, |
1975 | NVHOST_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET, | 1986 | NVHOST_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET, |
1976 | gk20a_mem_flag_none); | 1987 | gk20a_mem_flag_none, false); |
1977 | 1988 | ||
1978 | if (!page_vaddr) { | 1989 | if (!page_vaddr) { |
1979 | gk20a_err(dev_from_vm(vm), "failed to remap clean buffers!"); | 1990 | gk20a_err(dev_from_vm(vm), "failed to remap clean buffers!"); |
@@ -1990,7 +2001,7 @@ err_unmap: | |||
1990 | /* something went wrong. unmap pages */ | 2001 | /* something went wrong. unmap pages */ |
1991 | while (i--) { | 2002 | while (i--) { |
1992 | vaddr -= pgsz; | 2003 | vaddr -= pgsz; |
1993 | __locked_gmmu_unmap(vm, vaddr, pgsz, pgsz_idx, 0, | 2004 | g->ops.mm.gmmu_unmap(vm, vaddr, pgsz, pgsz_idx, 0, |
1994 | gk20a_mem_flag_none); | 2005 | gk20a_mem_flag_none); |
1995 | } | 2006 | } |
1996 | 2007 | ||
@@ -2005,12 +2016,14 @@ static int gk20a_vm_put_sparse(struct vm_gk20a *vm, u64 vaddr, | |||
2005 | 2016 | ||
2006 | void gk20a_vm_clear_sparse(struct vm_gk20a *vm, u64 vaddr, | 2017 | void gk20a_vm_clear_sparse(struct vm_gk20a *vm, u64 vaddr, |
2007 | u64 size, u32 pgsz_idx) { | 2018 | u64 size, u32 pgsz_idx) { |
2008 | __locked_gmmu_unmap(vm, vaddr, size, pgsz_idx, | 2019 | struct gk20a *g = vm->mm->g; |
2009 | false, gk20a_mem_flag_none); | 2020 | |
2021 | g->ops.mm.gmmu_unmap(vm, vaddr, size, pgsz_idx, | ||
2022 | false, gk20a_mem_flag_none); | ||
2010 | } | 2023 | } |
2011 | 2024 | ||
2012 | /* NOTE! mapped_buffers lock must be held */ | 2025 | /* NOTE! mapped_buffers lock must be held */ |
2013 | static void gk20a_vm_unmap_locked(struct mapped_buffer_node *mapped_buffer) | 2026 | void gk20a_vm_unmap_locked(struct mapped_buffer_node *mapped_buffer) |
2014 | { | 2027 | { |
2015 | struct vm_gk20a *vm = mapped_buffer->vm; | 2028 | struct vm_gk20a *vm = mapped_buffer->vm; |
2016 | struct gk20a *g = vm->mm->g; | 2029 | struct gk20a *g = vm->mm->g; |
@@ -2026,7 +2039,7 @@ static void gk20a_vm_unmap_locked(struct mapped_buffer_node *mapped_buffer) | |||
2026 | if (g->ops.mm.put_empty) { | 2039 | if (g->ops.mm.put_empty) { |
2027 | g->ops.mm.put_empty(vm, vaddr, num_pages, pgsz_idx); | 2040 | g->ops.mm.put_empty(vm, vaddr, num_pages, pgsz_idx); |
2028 | } else { | 2041 | } else { |
2029 | __locked_gmmu_unmap(vm, | 2042 | g->ops.mm.gmmu_unmap(vm, |
2030 | mapped_buffer->addr, | 2043 | mapped_buffer->addr, |
2031 | mapped_buffer->size, | 2044 | mapped_buffer->size, |
2032 | mapped_buffer->pgsz_idx, | 2045 | mapped_buffer->pgsz_idx, |
@@ -2036,7 +2049,7 @@ static void gk20a_vm_unmap_locked(struct mapped_buffer_node *mapped_buffer) | |||
2036 | num_pages, pgsz_idx, false); | 2049 | num_pages, pgsz_idx, false); |
2037 | } | 2050 | } |
2038 | } else | 2051 | } else |
2039 | __locked_gmmu_unmap(vm, | 2052 | g->ops.mm.gmmu_unmap(vm, |
2040 | mapped_buffer->addr, | 2053 | mapped_buffer->addr, |
2041 | mapped_buffer->size, | 2054 | mapped_buffer->size, |
2042 | mapped_buffer->pgsz_idx, | 2055 | mapped_buffer->pgsz_idx, |
@@ -2085,7 +2098,7 @@ void gk20a_vm_unmap(struct vm_gk20a *vm, u64 offset) | |||
2085 | mutex_unlock(&vm->update_gmmu_lock); | 2098 | mutex_unlock(&vm->update_gmmu_lock); |
2086 | } | 2099 | } |
2087 | 2100 | ||
2088 | static void gk20a_vm_remove_support(struct vm_gk20a *vm) | 2101 | void gk20a_vm_remove_support(struct vm_gk20a *vm) |
2089 | { | 2102 | { |
2090 | struct gk20a *g = vm->mm->g; | 2103 | struct gk20a *g = vm->mm->g; |
2091 | struct mapped_buffer_node *mapped_buffer; | 2104 | struct mapped_buffer_node *mapped_buffer; |
@@ -2156,7 +2169,8 @@ static void gk20a_vm_remove_support(struct vm_gk20a *vm) | |||
2156 | static void gk20a_vm_remove_support_kref(struct kref *ref) | 2169 | static void gk20a_vm_remove_support_kref(struct kref *ref) |
2157 | { | 2170 | { |
2158 | struct vm_gk20a *vm = container_of(ref, struct vm_gk20a, ref); | 2171 | struct vm_gk20a *vm = container_of(ref, struct vm_gk20a, ref); |
2159 | gk20a_vm_remove_support(vm); | 2172 | struct gk20a *g = gk20a_from_vm(vm); |
2173 | g->ops.mm.vm_remove(vm); | ||
2160 | } | 2174 | } |
2161 | 2175 | ||
2162 | void gk20a_vm_get(struct vm_gk20a *vm) | 2176 | void gk20a_vm_get(struct vm_gk20a *vm) |
@@ -3124,5 +3138,14 @@ void gk20a_init_mm(struct gpu_ops *gops) | |||
3124 | gops->mm.put_empty = gk20a_vm_put_empty; | 3138 | gops->mm.put_empty = gk20a_vm_put_empty; |
3125 | gops->mm.clear_sparse = gk20a_vm_clear_sparse; | 3139 | gops->mm.clear_sparse = gk20a_vm_clear_sparse; |
3126 | gops->mm.is_debug_mode_enabled = gk20a_mm_mmu_debug_mode_enabled; | 3140 | gops->mm.is_debug_mode_enabled = gk20a_mm_mmu_debug_mode_enabled; |
3141 | gops->mm.gmmu_map = gk20a_locked_gmmu_map; | ||
3142 | gops->mm.gmmu_unmap = gk20a_locked_gmmu_unmap; | ||
3143 | gops->mm.vm_remove = gk20a_vm_remove_support; | ||
3144 | gops->mm.vm_alloc_share = gk20a_vm_alloc_share; | ||
3145 | gops->mm.vm_bind_channel = gk20a_vm_bind_channel; | ||
3146 | gops->mm.fb_flush = gk20a_mm_fb_flush; | ||
3147 | gops->mm.l2_invalidate = gk20a_mm_l2_invalidate; | ||
3148 | gops->mm.l2_flush = gk20a_mm_l2_flush; | ||
3149 | gops->mm.tlb_invalidate = gk20a_mm_tlb_invalidate; | ||
3127 | } | 3150 | } |
3128 | 3151 | ||