From 01e6fac4d61fdd7fff5433942ec93fc2ea1e4df1 Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Wed, 28 Jun 2023 18:24:25 -0400 Subject: Include nvgpu headers These are needed to build on NVIDIA's Jetson boards for the time being. Only a couple structs are required, so it should be fairly easy to remove this dependency at some point in the future. --- include/gk20a/mm_gk20a.c | 654 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 654 insertions(+) create mode 100644 include/gk20a/mm_gk20a.c (limited to 'include/gk20a/mm_gk20a.c') diff --git a/include/gk20a/mm_gk20a.c b/include/gk20a/mm_gk20a.c new file mode 100644 index 0000000..10ca84d --- /dev/null +++ b/include/gk20a/mm_gk20a.c @@ -0,0 +1,654 @@ +/* + * Copyright (c) 2011-2020, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gk20a.h" +#include "mm_gk20a.h" +#include "fence_gk20a.h" + +#include +#include +#include +#include + +/* + * GPU mapping life cycle + * ====================== + * + * Kernel mappings + * --------------- + * + * Kernel mappings are created through vm.map(..., false): + * + * - Mappings to the same allocations are reused and refcounted. + * - This path does not support deferred unmapping (i.e. kernel must wait for + * all hw operations on the buffer to complete before unmapping). + * - References to dmabuf are owned and managed by the (kernel) clients of + * the gk20a_vm layer. + * + * + * User space mappings + * ------------------- + * + * User space mappings are created through as.map_buffer -> vm.map(..., true): + * + * - Mappings to the same allocations are reused and refcounted. + * - This path supports deferred unmapping (i.e. we delay the actual unmapping + * until all hw operations have completed). + * - References to dmabuf are owned and managed by the vm_gk20a + * layer itself. vm.map acquires these refs, and sets + * mapped_buffer->own_mem_ref to record that we must release the refs when we + * actually unmap. + * + */ + +/* make sure gk20a_init_mm_support is called before */ +int gk20a_init_mm_setup_hw(struct gk20a *g) +{ + struct mm_gk20a *mm = &g->mm; + int err; + + nvgpu_log_fn(g, " "); + + if (g->ops.fb.set_mmu_page_size) { + g->ops.fb.set_mmu_page_size(g); + } + + if (g->ops.fb.set_use_full_comp_tag_line) { + mm->use_full_comp_tag_line = + g->ops.fb.set_use_full_comp_tag_line(g); + } + + g->ops.fb.init_hw(g); + + if (g->ops.bus.bar1_bind) { + g->ops.bus.bar1_bind(g, &mm->bar1.inst_block); + } + + if (g->ops.bus.bar2_bind) { + err = g->ops.bus.bar2_bind(g, &mm->bar2.inst_block); + if (err) { + return err; + } + } + + if (gk20a_mm_fb_flush(g) || gk20a_mm_fb_flush(g)) { + return -EBUSY; + } + + nvgpu_log_fn(g, "done"); + return 0; +} + +/* for gk20a the "video memory" apertures here are misnomers. */ +static inline u32 big_valid_pde0_bits(struct gk20a *g, + struct nvgpu_gmmu_pd *pd, u64 addr) +{ + u32 pde0_bits = + nvgpu_aperture_mask(g, pd->mem, + gmmu_pde_aperture_big_sys_mem_ncoh_f(), + gmmu_pde_aperture_big_sys_mem_coh_f(), + gmmu_pde_aperture_big_video_memory_f()) | + gmmu_pde_address_big_sys_f( + (u32)(addr >> gmmu_pde_address_shift_v())); + + return pde0_bits; +} + +static inline u32 small_valid_pde1_bits(struct gk20a *g, + struct nvgpu_gmmu_pd *pd, u64 addr) +{ + u32 pde1_bits = + nvgpu_aperture_mask(g, pd->mem, + gmmu_pde_aperture_small_sys_mem_ncoh_f(), + gmmu_pde_aperture_small_sys_mem_coh_f(), + gmmu_pde_aperture_small_video_memory_f()) | + gmmu_pde_vol_small_true_f() | /* tbd: why? */ + gmmu_pde_address_small_sys_f( + (u32)(addr >> gmmu_pde_address_shift_v())); + + return pde1_bits; +} + +static void update_gmmu_pde_locked(struct vm_gk20a *vm, + const struct gk20a_mmu_level *l, + struct nvgpu_gmmu_pd *pd, + u32 pd_idx, + u64 virt_addr, + u64 phys_addr, + struct nvgpu_gmmu_attrs *attrs) +{ + struct gk20a *g = gk20a_from_vm(vm); + bool small_valid, big_valid; + u32 pd_offset = pd_offset_from_index(l, pd_idx); + u32 pde_v[2] = {0, 0}; + + small_valid = attrs->pgsz == GMMU_PAGE_SIZE_SMALL; + big_valid = attrs->pgsz == GMMU_PAGE_SIZE_BIG; + + pde_v[0] = gmmu_pde_size_full_f(); + pde_v[0] |= big_valid ? + big_valid_pde0_bits(g, pd, phys_addr) : + gmmu_pde_aperture_big_invalid_f(); + + pde_v[1] |= (small_valid ? small_valid_pde1_bits(g, pd, phys_addr) : + (gmmu_pde_aperture_small_invalid_f() | + gmmu_pde_vol_small_false_f())) + | + (big_valid ? (gmmu_pde_vol_big_true_f()) : + gmmu_pde_vol_big_false_f()); + + pte_dbg(g, attrs, + "PDE: i=%-4u size=%-2u offs=%-4u pgsz: %c%c | " + "GPU %#-12llx phys %#-12llx " + "[0x%08x, 0x%08x]", + pd_idx, l->entry_size, pd_offset, + small_valid ? 'S' : '-', + big_valid ? 'B' : '-', + virt_addr, phys_addr, + pde_v[1], pde_v[0]); + + pd_write(g, &vm->pdb, pd_offset + 0, pde_v[0]); + pd_write(g, &vm->pdb, pd_offset + 1, pde_v[1]); +} + +static void __update_pte_sparse(u32 *pte_w) +{ + pte_w[0] = gmmu_pte_valid_false_f(); + pte_w[1] |= gmmu_pte_vol_true_f(); +} + +static void __update_pte(struct vm_gk20a *vm, + u32 *pte_w, + u64 phys_addr, + struct nvgpu_gmmu_attrs *attrs) +{ + struct gk20a *g = gk20a_from_vm(vm); + u32 page_size = vm->gmmu_page_sizes[attrs->pgsz]; + u32 pte_valid = attrs->valid ? + gmmu_pte_valid_true_f() : + gmmu_pte_valid_false_f(); + u32 phys_shifted = phys_addr >> gmmu_pte_address_shift_v(); + u32 addr = attrs->aperture == APERTURE_SYSMEM ? + gmmu_pte_address_sys_f(phys_shifted) : + gmmu_pte_address_vid_f(phys_shifted); + int ctag_shift = ilog2(g->ops.fb.compression_page_size(g)); + + pte_w[0] = pte_valid | addr; + + if (attrs->priv) { + pte_w[0] |= gmmu_pte_privilege_true_f(); + } + + pte_w[1] = nvgpu_aperture_mask_raw(g, attrs->aperture, + gmmu_pte_aperture_sys_mem_ncoh_f(), + gmmu_pte_aperture_sys_mem_coh_f(), + gmmu_pte_aperture_video_memory_f()) | + gmmu_pte_kind_f(attrs->kind_v) | + gmmu_pte_comptagline_f((u32)(attrs->ctag >> ctag_shift)); + + if (attrs->ctag && vm->mm->use_full_comp_tag_line && + phys_addr & 0x10000) { + pte_w[1] |= gmmu_pte_comptagline_f( + 1 << (gmmu_pte_comptagline_s() - 1)); + } + + if (attrs->rw_flag == gk20a_mem_flag_read_only) { + pte_w[0] |= gmmu_pte_read_only_true_f(); + pte_w[1] |= gmmu_pte_write_disable_true_f(); + } else if (attrs->rw_flag == gk20a_mem_flag_write_only) { + pte_w[1] |= gmmu_pte_read_disable_true_f(); + } + + if (!attrs->cacheable) { + pte_w[1] |= gmmu_pte_vol_true_f(); + } + + if (attrs->ctag) { + attrs->ctag += page_size; + } +} + +static void update_gmmu_pte_locked(struct vm_gk20a *vm, + const struct gk20a_mmu_level *l, + struct nvgpu_gmmu_pd *pd, + u32 pd_idx, + u64 virt_addr, + u64 phys_addr, + struct nvgpu_gmmu_attrs *attrs) +{ + struct gk20a *g = gk20a_from_vm(vm); + u32 page_size = vm->gmmu_page_sizes[attrs->pgsz]; + u32 pd_offset = pd_offset_from_index(l, pd_idx); + u32 pte_w[2] = {0, 0}; + int ctag_shift = ilog2(g->ops.fb.compression_page_size(g)); + + if (phys_addr) { + __update_pte(vm, pte_w, phys_addr, attrs); + } else if (attrs->sparse) { + __update_pte_sparse(pte_w); + } + + pte_dbg(g, attrs, + "PTE: i=%-4u size=%-2u offs=%-4u | " + "GPU %#-12llx phys %#-12llx " + "pgsz: %3dkb perm=%-2s kind=%#02x APT=%-6s %c%c%c%c " + "ctag=0x%08x " + "[0x%08x, 0x%08x]", + pd_idx, l->entry_size, pd_offset, + virt_addr, phys_addr, + page_size >> 10, + nvgpu_gmmu_perm_str(attrs->rw_flag), + attrs->kind_v, + nvgpu_aperture_str(g, attrs->aperture), + attrs->cacheable ? 'C' : '-', + attrs->sparse ? 'S' : '-', + attrs->priv ? 'P' : '-', + attrs->valid ? 'V' : '-', + (u32)attrs->ctag >> ctag_shift, + pte_w[1], pte_w[0]); + + pd_write(g, pd, pd_offset + 0, pte_w[0]); + pd_write(g, pd, pd_offset + 1, pte_w[1]); +} + +u32 gk20a_get_pde_pgsz(struct gk20a *g, const struct gk20a_mmu_level *l, + struct nvgpu_gmmu_pd *pd, u32 pd_idx) +{ + /* + * big and small page sizes are the same + */ + return GMMU_PAGE_SIZE_SMALL; +} + +u32 gk20a_get_pte_pgsz(struct gk20a *g, const struct gk20a_mmu_level *l, + struct nvgpu_gmmu_pd *pd, u32 pd_idx) +{ + /* + * return invalid + */ + return GMMU_NR_PAGE_SIZES; +} + +const struct gk20a_mmu_level gk20a_mm_levels_64k[] = { + {.hi_bit = {NV_GMMU_VA_RANGE-1, NV_GMMU_VA_RANGE-1}, + .lo_bit = {26, 26}, + .update_entry = update_gmmu_pde_locked, + .entry_size = 8, + .get_pgsz = gk20a_get_pde_pgsz}, + {.hi_bit = {25, 25}, + .lo_bit = {12, 16}, + .update_entry = update_gmmu_pte_locked, + .entry_size = 8, + .get_pgsz = gk20a_get_pte_pgsz}, + {.update_entry = NULL} +}; + +const struct gk20a_mmu_level gk20a_mm_levels_128k[] = { + {.hi_bit = {NV_GMMU_VA_RANGE-1, NV_GMMU_VA_RANGE-1}, + .lo_bit = {27, 27}, + .update_entry = update_gmmu_pde_locked, + .entry_size = 8, + .get_pgsz = gk20a_get_pde_pgsz}, + {.hi_bit = {26, 26}, + .lo_bit = {12, 17}, + .update_entry = update_gmmu_pte_locked, + .entry_size = 8, + .get_pgsz = gk20a_get_pte_pgsz}, + {.update_entry = NULL} +}; + +int gk20a_vm_bind_channel(struct vm_gk20a *vm, struct channel_gk20a *ch) +{ + int err = 0; + + nvgpu_log_fn(ch->g, " "); + + nvgpu_vm_get(vm); + ch->vm = vm; + err = channel_gk20a_commit_va(ch); + if (err) { + ch->vm = NULL; + } + + nvgpu_log(gk20a_from_vm(vm), gpu_dbg_map, "Binding ch=%d -> VM:%s", + ch->chid, vm->name); + + return err; +} + +void gk20a_mm_init_pdb(struct gk20a *g, struct nvgpu_mem *inst_block, + struct vm_gk20a *vm) +{ + u64 pdb_addr = nvgpu_mem_get_addr(g, vm->pdb.mem); + u32 pdb_addr_lo = u64_lo32(pdb_addr >> ram_in_base_shift_v()); + u32 pdb_addr_hi = u64_hi32(pdb_addr); + + nvgpu_log_info(g, "pde pa=0x%llx", pdb_addr); + + nvgpu_mem_wr32(g, inst_block, ram_in_page_dir_base_lo_w(), + nvgpu_aperture_mask(g, vm->pdb.mem, + ram_in_page_dir_base_target_sys_mem_ncoh_f(), + ram_in_page_dir_base_target_sys_mem_coh_f(), + ram_in_page_dir_base_target_vid_mem_f()) | + ram_in_page_dir_base_vol_true_f() | + ram_in_page_dir_base_lo_f(pdb_addr_lo)); + + nvgpu_mem_wr32(g, inst_block, ram_in_page_dir_base_hi_w(), + ram_in_page_dir_base_hi_f(pdb_addr_hi)); +} + +void gk20a_init_inst_block(struct nvgpu_mem *inst_block, struct vm_gk20a *vm, + u32 big_page_size) +{ + struct gk20a *g = gk20a_from_vm(vm); + + nvgpu_log_info(g, "inst block phys = 0x%llx, kv = 0x%p", + nvgpu_inst_block_addr(g, inst_block), inst_block->cpu_va); + + g->ops.mm.init_pdb(g, inst_block, vm); + + nvgpu_mem_wr32(g, inst_block, ram_in_adr_limit_lo_w(), + u64_lo32(vm->va_limit - 1) & ~0xfff); + + nvgpu_mem_wr32(g, inst_block, ram_in_adr_limit_hi_w(), + ram_in_adr_limit_hi_f(u64_hi32(vm->va_limit - 1))); + + if (big_page_size && g->ops.mm.set_big_page_size) { + g->ops.mm.set_big_page_size(g, inst_block, big_page_size); + } +} + +int gk20a_alloc_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block) +{ + int err; + + nvgpu_log_fn(g, " "); + + err = nvgpu_dma_alloc(g, ram_in_alloc_size_v(), inst_block); + if (err) { + nvgpu_err(g, "%s: memory allocation failed", __func__); + return err; + } + + nvgpu_log_fn(g, "done"); + return 0; +} + +int gk20a_mm_fb_flush(struct gk20a *g) +{ + struct mm_gk20a *mm = &g->mm; + u32 data; + int ret = 0; + struct nvgpu_timeout timeout; + u32 retries; + + nvgpu_log_fn(g, " "); + + gk20a_busy_noresume(g); + if (!g->power_on) { + gk20a_idle_nosuspend(g); + return 0; + } + + retries = 100; + + if (g->ops.mm.get_flush_retries) { + retries = g->ops.mm.get_flush_retries(g, NVGPU_FLUSH_FB); + } + + nvgpu_timeout_init(g, &timeout, retries, NVGPU_TIMER_RETRY_TIMER); + + nvgpu_mutex_acquire(&mm->l2_op_lock); + + /* Make sure all previous writes are committed to the L2. There's no + guarantee that writes are to DRAM. This will be a sysmembar internal + to the L2. */ + + trace_gk20a_mm_fb_flush(g->name); + + gk20a_writel(g, flush_fb_flush_r(), + flush_fb_flush_pending_busy_f()); + + do { + data = gk20a_readl(g, flush_fb_flush_r()); + + if (flush_fb_flush_outstanding_v(data) == + flush_fb_flush_outstanding_true_v() || + flush_fb_flush_pending_v(data) == + flush_fb_flush_pending_busy_v()) { + nvgpu_log_info(g, "fb_flush 0x%x", data); + nvgpu_udelay(5); + } else { + break; + } + } while (!nvgpu_timeout_expired(&timeout)); + + if (nvgpu_timeout_peek_expired(&timeout)) { + if (g->ops.fb.dump_vpr_info) { + g->ops.fb.dump_vpr_info(g); + } + if (g->ops.fb.dump_wpr_info) { + g->ops.fb.dump_wpr_info(g); + } + ret = -EBUSY; + } + + trace_gk20a_mm_fb_flush_done(g->name); + + nvgpu_mutex_release(&mm->l2_op_lock); + + gk20a_idle_nosuspend(g); + + return ret; +} + +static void gk20a_mm_l2_invalidate_locked(struct gk20a *g) +{ + u32 data; + struct nvgpu_timeout timeout; + u32 retries = 200; + + trace_gk20a_mm_l2_invalidate(g->name); + + if (g->ops.mm.get_flush_retries) { + retries = g->ops.mm.get_flush_retries(g, NVGPU_FLUSH_L2_INV); + } + + nvgpu_timeout_init(g, &timeout, retries, NVGPU_TIMER_RETRY_TIMER); + + /* Invalidate any clean lines from the L2 so subsequent reads go to + DRAM. Dirty lines are not affected by this operation. */ + gk20a_writel(g, flush_l2_system_invalidate_r(), + flush_l2_system_invalidate_pending_busy_f()); + + do { + data = gk20a_readl(g, flush_l2_system_invalidate_r()); + + if (flush_l2_system_invalidate_outstanding_v(data) == + flush_l2_system_invalidate_outstanding_true_v() || + flush_l2_system_invalidate_pending_v(data) == + flush_l2_system_invalidate_pending_busy_v()) { + nvgpu_log_info(g, "l2_system_invalidate 0x%x", + data); + nvgpu_udelay(5); + } else { + break; + } + } while (!nvgpu_timeout_expired(&timeout)); + + if (nvgpu_timeout_peek_expired(&timeout)) { + nvgpu_warn(g, "l2_system_invalidate too many retries"); + } + + trace_gk20a_mm_l2_invalidate_done(g->name); +} + +void gk20a_mm_l2_invalidate(struct gk20a *g) +{ + struct mm_gk20a *mm = &g->mm; + gk20a_busy_noresume(g); + if (g->power_on) { + nvgpu_mutex_acquire(&mm->l2_op_lock); + gk20a_mm_l2_invalidate_locked(g); + nvgpu_mutex_release(&mm->l2_op_lock); + } + gk20a_idle_nosuspend(g); +} + +void gk20a_mm_l2_flush(struct gk20a *g, bool invalidate) +{ + struct mm_gk20a *mm = &g->mm; + u32 data; + struct nvgpu_timeout timeout; + u32 retries = 2000; + + nvgpu_log_fn(g, " "); + + gk20a_busy_noresume(g); + if (!g->power_on) { + goto hw_was_off; + } + + if (g->ops.mm.get_flush_retries) { + retries = g->ops.mm.get_flush_retries(g, NVGPU_FLUSH_L2_FLUSH); + } + + nvgpu_timeout_init(g, &timeout, retries, NVGPU_TIMER_RETRY_TIMER); + + nvgpu_mutex_acquire(&mm->l2_op_lock); + + trace_gk20a_mm_l2_flush(g->name); + + /* Flush all dirty lines from the L2 to DRAM. Lines are left in the L2 + as clean, so subsequent reads might hit in the L2. */ + gk20a_writel(g, flush_l2_flush_dirty_r(), + flush_l2_flush_dirty_pending_busy_f()); + + do { + data = gk20a_readl(g, flush_l2_flush_dirty_r()); + + if (flush_l2_flush_dirty_outstanding_v(data) == + flush_l2_flush_dirty_outstanding_true_v() || + flush_l2_flush_dirty_pending_v(data) == + flush_l2_flush_dirty_pending_busy_v()) { + nvgpu_log_info(g, "l2_flush_dirty 0x%x", data); + nvgpu_udelay(5); + } else { + break; + } + } while (!nvgpu_timeout_expired_msg(&timeout, + "l2_flush_dirty too many retries")); + + trace_gk20a_mm_l2_flush_done(g->name); + + if (invalidate) { + gk20a_mm_l2_invalidate_locked(g); + } + + nvgpu_mutex_release(&mm->l2_op_lock); + +hw_was_off: + gk20a_idle_nosuspend(g); +} + +void gk20a_mm_cbc_clean(struct gk20a *g) +{ + struct mm_gk20a *mm = &g->mm; + u32 data; + struct nvgpu_timeout timeout; + u32 retries = 200; + + nvgpu_log_fn(g, " "); + + gk20a_busy_noresume(g); + if (!g->power_on) { + goto hw_was_off; + } + + if (g->ops.mm.get_flush_retries) { + retries = g->ops.mm.get_flush_retries(g, NVGPU_FLUSH_CBC_CLEAN); + } + + nvgpu_timeout_init(g, &timeout, retries, NVGPU_TIMER_RETRY_TIMER); + + nvgpu_mutex_acquire(&mm->l2_op_lock); + + /* Flush all dirty lines from the CBC to L2 */ + gk20a_writel(g, flush_l2_clean_comptags_r(), + flush_l2_clean_comptags_pending_busy_f()); + + do { + data = gk20a_readl(g, flush_l2_clean_comptags_r()); + + if (flush_l2_clean_comptags_outstanding_v(data) == + flush_l2_clean_comptags_outstanding_true_v() || + flush_l2_clean_comptags_pending_v(data) == + flush_l2_clean_comptags_pending_busy_v()) { + nvgpu_log_info(g, "l2_clean_comptags 0x%x", data); + nvgpu_udelay(5); + } else { + break; + } + } while (!nvgpu_timeout_expired_msg(&timeout, + "l2_clean_comptags too many retries")); + + nvgpu_mutex_release(&mm->l2_op_lock); + +hw_was_off: + gk20a_idle_nosuspend(g); +} + +u32 gk20a_mm_get_iommu_bit(struct gk20a *g) +{ + return 34; +} + +const struct gk20a_mmu_level *gk20a_mm_get_mmu_levels(struct gk20a *g, + u32 big_page_size) +{ + return (big_page_size == SZ_64K) ? + gk20a_mm_levels_64k : gk20a_mm_levels_128k; +} -- cgit v1.2.2