From b97bcb3c689426a1b099e88ceef4d55584e2362b Mon Sep 17 00:00:00 2001 From: Terje Bergstrom Date: Mon, 2 Jul 2018 14:30:26 -0700 Subject: gpu: nvgpu: Move FB to common Move all FB HAL implementations to common/fb. JIRA NVGPU-596 Change-Id: Id4ea09d608f5d6d1b245bddac09ecf1444b8ab30 Signed-off-by: Terje Bergstrom Reviewed-on: https://git-master.nvidia.com/r/1769724 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/Makefile | 14 +- drivers/gpu/nvgpu/Makefile.sources | 12 +- drivers/gpu/nvgpu/common/fb/fb_gk20a.c | 123 ++ drivers/gpu/nvgpu/common/fb/fb_gk20a.h | 33 + drivers/gpu/nvgpu/common/fb/fb_gm20b.c | 229 ++++ drivers/gpu/nvgpu/common/fb/fb_gm20b.h | 47 + drivers/gpu/nvgpu/common/fb/fb_gp106.c | 51 + drivers/gpu/nvgpu/common/fb/fb_gp106.h | 28 + drivers/gpu/nvgpu/common/fb/fb_gp10b.c | 38 + drivers/gpu/nvgpu/common/fb/fb_gp10b.h | 32 + drivers/gpu/nvgpu/common/fb/fb_gv100.c | 257 +++++ drivers/gpu/nvgpu/common/fb/fb_gv100.h | 36 + drivers/gpu/nvgpu/common/fb/fb_gv11b.c | 1516 +++++++++++++++++++++++++ drivers/gpu/nvgpu/common/fb/fb_gv11b.h | 82 ++ drivers/gpu/nvgpu/gk20a/fb_gk20a.c | 122 -- drivers/gpu/nvgpu/gk20a/fb_gk20a.h | 33 - drivers/gpu/nvgpu/gm20b/fb_gm20b.c | 228 ---- drivers/gpu/nvgpu/gm20b/fb_gm20b.h | 45 - drivers/gpu/nvgpu/gm20b/hal_gm20b.c | 4 +- drivers/gpu/nvgpu/gp106/fb_gp106.c | 51 - drivers/gpu/nvgpu/gp106/fb_gp106.h | 28 - drivers/gpu/nvgpu/gp106/hal_gp106.c | 8 +- drivers/gpu/nvgpu/gp10b/fb_gp10b.c | 38 - drivers/gpu/nvgpu/gp10b/fb_gp10b.h | 32 - drivers/gpu/nvgpu/gp10b/hal_gp10b.c | 6 +- drivers/gpu/nvgpu/gv100/fb_gv100.c | 256 ----- drivers/gpu/nvgpu/gv100/fb_gv100.h | 36 - drivers/gpu/nvgpu/gv100/hal_gv100.c | 10 +- drivers/gpu/nvgpu/gv100/mc_gv100.c | 2 - drivers/gpu/nvgpu/gv11b/fb_gv11b.c | 1516 ------------------------- drivers/gpu/nvgpu/gv11b/fb_gv11b.h | 81 -- drivers/gpu/nvgpu/gv11b/hal_gv11b.c | 8 +- drivers/gpu/nvgpu/gv11b/mc_gv11b.c | 1 - drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c | 6 +- drivers/gpu/nvgpu/vgpu/gv11b/vgpu_hal_gv11b.c | 8 +- 35 files changed, 2510 insertions(+), 2507 deletions(-) create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gk20a.c create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gk20a.h create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gm20b.c create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gm20b.h create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gp106.c create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gp106.h create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gp10b.c create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gp10b.h create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gv100.c create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gv100.h create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gv11b.c create mode 100644 drivers/gpu/nvgpu/common/fb/fb_gv11b.h delete mode 100644 drivers/gpu/nvgpu/gk20a/fb_gk20a.c delete mode 100644 drivers/gpu/nvgpu/gk20a/fb_gk20a.h delete mode 100644 drivers/gpu/nvgpu/gm20b/fb_gm20b.c delete mode 100644 drivers/gpu/nvgpu/gm20b/fb_gm20b.h delete mode 100644 drivers/gpu/nvgpu/gp106/fb_gp106.c delete mode 100644 drivers/gpu/nvgpu/gp106/fb_gp106.h delete mode 100644 drivers/gpu/nvgpu/gp10b/fb_gp10b.c delete mode 100644 drivers/gpu/nvgpu/gp10b/fb_gp10b.h delete mode 100644 drivers/gpu/nvgpu/gv100/fb_gv100.c delete mode 100644 drivers/gpu/nvgpu/gv100/fb_gv100.h delete mode 100644 drivers/gpu/nvgpu/gv11b/fb_gv11b.c delete mode 100644 drivers/gpu/nvgpu/gv11b/fb_gv11b.h diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 9ca762a3..e715e385 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -28,7 +28,13 @@ nvgpu-y += common/bus/bus_gk20a.o \ common/bus/bus_gp10b.o \ common/bus/bus_gv100.o \ common/ptimer/ptimer.o \ - common/ptimer/ptimer_gk20a.o + common/ptimer/ptimer_gk20a.o \ + common/fb/fb_gk20a.o \ + common/fb/fb_gm20b.o \ + common/fb/fb_gp10b.o \ + common/fb/fb_gp106.o \ + common/fb/fb_gv11b.o \ + common/fb/fb_gv100.o # Linux specific parts of nvgpu. nvgpu-y += \ @@ -194,7 +200,6 @@ nvgpu-y += \ gk20a/therm_gk20a.o \ gk20a/gr_ctx_gk20a_sim.o \ gk20a/gr_ctx_gk20a.o \ - gk20a/fb_gk20a.o \ gk20a/hal.o \ gk20a/tsg_gk20a.o \ gk20a/fecs_trace_gk20a.o \ @@ -203,7 +208,6 @@ nvgpu-y += \ gm20b/ltc_gm20b.o \ gm20b/gr_gm20b.o \ gm20b/clk_gm20b.o \ - gm20b/fb_gm20b.o \ gm20b/fifo_gm20b.o \ gm20b/gr_ctx_gm20b.o \ gm20b/acr_gm20b.o \ @@ -254,7 +258,6 @@ nvgpu-y += \ gp10b/fifo_gp10b.o \ gp10b/ltc_gp10b.o \ gp10b/mm_gp10b.o \ - gp10b/fb_gp10b.o \ gp10b/pmu_gp10b.o \ gp10b/hal_gp10b.o \ gp10b/regops_gp10b.o \ @@ -273,7 +276,6 @@ nvgpu-y += \ gp106/acr_gp106.o \ gp106/sec2_gp106.o \ gp106/fifo_gp106.o \ - gp106/fb_gp106.o \ gp106/regops_gp106.o \ gp106/bios_gp106.o \ gp106/fuse_gp106.o \ @@ -284,7 +286,6 @@ nvgpu-y += \ gv11b/ltc_gv11b.o \ gv11b/hal_gv11b.o \ gv11b/gr_gv11b.o \ - gv11b/fb_gv11b.o \ gv11b/fifo_gv11b.o \ gv11b/mm_gv11b.o \ gv11b/ce_gv11b.o \ @@ -296,7 +297,6 @@ nvgpu-y += \ gv11b/therm_gv11b.o \ gv100/mm_gv100.o \ gv100/gr_ctx_gv100.o \ - gv100/fb_gv100.o \ gv100/bios_gv100.o \ gv100/fifo_gv100.o \ gv100/gr_gv100.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index 6fa815ab..c0340545 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -36,6 +36,12 @@ srcs := common/mm/nvgpu_allocator.c \ common/bus/bus_gm20b.c \ common/bus/bus_gp10b.c \ common/bus/bus_gv100.c \ + common/fb/fb_gk20a.c \ + common/fb/fb_gm20b.c \ + common/fb/fb_gp10b.c \ + common/fb/fb_gp106.c \ + common/fb/fb_gv100.c \ + common/fb/fb_gv11b.c \ common/enabled.c \ common/pramin.c \ common/semaphore.c \ @@ -129,7 +135,6 @@ srcs := common/mm/nvgpu_allocator.c \ gk20a/therm_gk20a.c \ gk20a/gr_ctx_gk20a_sim.c \ gk20a/gr_ctx_gk20a.c \ - gk20a/fb_gk20a.c \ gk20a/hal.c \ gk20a/tsg_gk20a.c \ gk20a/mc_gk20a.c \ @@ -137,7 +142,6 @@ srcs := common/mm/nvgpu_allocator.c \ gm20b/ltc_gm20b.c \ gm20b/gr_gm20b.c \ gm20b/clk_gm20b.c \ - gm20b/fb_gm20b.c \ gm20b/fifo_gm20b.c \ gm20b/gr_ctx_gm20b.c \ gm20b/acr_gm20b.c \ @@ -153,7 +157,6 @@ srcs := common/mm/nvgpu_allocator.c \ gp10b/fifo_gp10b.c \ gp10b/ltc_gp10b.c \ gp10b/mm_gp10b.c \ - gp10b/fb_gp10b.c \ gp10b/pmu_gp10b.c \ gp10b/hal_gp10b.c \ gp10b/regops_gp10b.c \ @@ -168,7 +171,6 @@ srcs := common/mm/nvgpu_allocator.c \ gv11b/ltc_gv11b.c \ gv11b/hal_gv11b.c \ gv11b/gr_gv11b.c \ - gv11b/fb_gv11b.c \ gv11b/fifo_gv11b.c \ gv11b/mm_gv11b.c \ gv11b/ce_gv11b.c \ @@ -188,7 +190,6 @@ srcs := common/mm/nvgpu_allocator.c \ gp106/acr_gp106.c \ gp106/sec2_gp106.c \ gp106/fifo_gp106.c \ - gp106/fb_gp106.c \ gp106/regops_gp106.c \ gp106/bios_gp106.c \ gp106/fuse_gp106.c \ @@ -198,7 +199,6 @@ srcs := common/mm/nvgpu_allocator.c \ gp106/therm_gp106.c \ gv100/mm_gv100.c \ gv100/gr_ctx_gv100.c \ - gv100/fb_gv100.c \ gv100/bios_gv100.c \ gv100/fifo_gv100.c \ gv100/gr_gv100.c \ diff --git a/drivers/gpu/nvgpu/common/fb/fb_gk20a.c b/drivers/gpu/nvgpu/common/fb/fb_gk20a.c new file mode 100644 index 00000000..d27ac9d0 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gk20a.c @@ -0,0 +1,123 @@ +/* + * GK20A memory interface + * + * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include + +#include "gk20a/gk20a.h" + +#include "fb_gk20a.h" + +#include + +#include +#include + +void fb_gk20a_reset(struct gk20a *g) +{ + u32 val; + + nvgpu_log_info(g, "reset gk20a fb"); + + g->ops.mc.reset(g, mc_enable_pfb_enabled_f() | + mc_enable_l2_enabled_f() | + mc_enable_xbar_enabled_f() | + mc_enable_hub_enabled_f()); + + val = gk20a_readl(g, mc_elpg_enable_r()); + val |= mc_elpg_enable_xbar_enabled_f() + | mc_elpg_enable_pfb_enabled_f() + | mc_elpg_enable_hub_enabled_f(); + gk20a_writel(g, mc_elpg_enable_r(), val); +} + +void gk20a_fb_init_hw(struct gk20a *g) +{ + u32 addr = nvgpu_mem_get_addr(g, &g->mm.sysmem_flush) >> 8; + + gk20a_writel(g, fb_niso_flush_sysmem_addr_r(), addr); +} + +void gk20a_fb_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb) +{ + struct nvgpu_timeout timeout; + u32 addr_lo; + u32 data; + + nvgpu_log_fn(g, " "); + + /* pagetables are considered sw states which are preserved after + prepare_poweroff. When gk20a deinit releases those pagetables, + common code in vm unmap path calls tlb invalidate that touches + hw. Use the power_on flag to skip tlb invalidation when gpu + power is turned off */ + + if (!g->power_on) + return; + + addr_lo = u64_lo32(nvgpu_mem_get_addr(g, pdb) >> 12); + + nvgpu_mutex_acquire(&g->mm.tlb_lock); + + trace_gk20a_mm_tlb_invalidate(g->name); + + nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER); + + do { + data = gk20a_readl(g, fb_mmu_ctrl_r()); + if (fb_mmu_ctrl_pri_fifo_space_v(data) != 0) + break; + nvgpu_udelay(2); + } while (!nvgpu_timeout_expired_msg(&timeout, + "wait mmu fifo space")); + + if (nvgpu_timeout_peek_expired(&timeout)) + goto out; + + nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER); + + gk20a_writel(g, fb_mmu_invalidate_pdb_r(), + fb_mmu_invalidate_pdb_addr_f(addr_lo) | + nvgpu_aperture_mask(g, pdb, + fb_mmu_invalidate_pdb_aperture_sys_mem_f(), + fb_mmu_invalidate_pdb_aperture_sys_mem_f(), + fb_mmu_invalidate_pdb_aperture_vid_mem_f())); + + gk20a_writel(g, fb_mmu_invalidate_r(), + fb_mmu_invalidate_all_va_true_f() | + fb_mmu_invalidate_trigger_true_f()); + + do { + data = gk20a_readl(g, fb_mmu_ctrl_r()); + if (fb_mmu_ctrl_pri_fifo_empty_v(data) != + fb_mmu_ctrl_pri_fifo_empty_false_f()) + break; + nvgpu_udelay(2); + } while (!nvgpu_timeout_expired_msg(&timeout, + "wait mmu invalidate")); + + trace_gk20a_mm_tlb_invalidate_done(g->name); + +out: + nvgpu_mutex_release(&g->mm.tlb_lock); +} diff --git a/drivers/gpu/nvgpu/common/fb/fb_gk20a.h b/drivers/gpu/nvgpu/common/fb/fb_gk20a.h new file mode 100644 index 00000000..072c9027 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gk20a.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef FB_GK20A_H +#define FB_GK20A_H + +struct gk20a; +struct nvgpu_mem; + +void fb_gk20a_reset(struct gk20a *g); +void gk20a_fb_init_hw(struct gk20a *g); +void gk20a_fb_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb); + +#endif diff --git a/drivers/gpu/nvgpu/common/fb/fb_gm20b.c b/drivers/gpu/nvgpu/common/fb/fb_gm20b.c new file mode 100644 index 00000000..bd093b31 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gm20b.c @@ -0,0 +1,229 @@ +/* + * GM20B GPC MMU + * + * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. +* + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include + +#include "gk20a/gk20a.h" + +#include "fb_gk20a.h" +#include "fb_gm20b.h" + +#include +#include +#include +#include + +#define VPR_INFO_FETCH_WAIT (5) +#define WPR_INFO_ADDR_ALIGNMENT 0x0000000c + +void fb_gm20b_init_fs_state(struct gk20a *g) +{ + nvgpu_log_info(g, "initialize gm20b fb"); + + gk20a_writel(g, fb_fbhub_num_active_ltcs_r(), + g->ltc_count); + + if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) { + /* Bypass MMU check for non-secure boot. For + * secure-boot,this register write has no-effect */ + gk20a_writel(g, fb_priv_mmu_phy_secure_r(), 0xffffffffU); + } +} + +void gm20b_fb_set_mmu_page_size(struct gk20a *g) +{ + /* set large page size in fb */ + u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r()); + fb_mmu_ctrl |= fb_mmu_ctrl_use_pdb_big_page_size_true_f(); + gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl); +} + +bool gm20b_fb_set_use_full_comp_tag_line(struct gk20a *g) +{ + /* set large page size in fb */ + u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r()); + fb_mmu_ctrl |= fb_mmu_ctrl_use_full_comp_tag_line_true_f(); + gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl); + + return true; +} + +u32 gm20b_fb_mmu_ctrl(struct gk20a *g) +{ + return gk20a_readl(g, fb_mmu_ctrl_r()); +} + +u32 gm20b_fb_mmu_debug_ctrl(struct gk20a *g) +{ + return gk20a_readl(g, fb_mmu_debug_ctrl_r()); +} + +u32 gm20b_fb_mmu_debug_wr(struct gk20a *g) +{ + return gk20a_readl(g, fb_mmu_debug_wr_r()); +} + +u32 gm20b_fb_mmu_debug_rd(struct gk20a *g) +{ + return gk20a_readl(g, fb_mmu_debug_rd_r()); +} + +unsigned int gm20b_fb_compression_page_size(struct gk20a *g) +{ + return SZ_128K; +} + +unsigned int gm20b_fb_compressible_page_size(struct gk20a *g) +{ + return SZ_64K; +} + +u32 gm20b_fb_compression_align_mask(struct gk20a *g) +{ + return SZ_64K - 1; +} + +void gm20b_fb_dump_vpr_wpr_info(struct gk20a *g) +{ + u32 val; + + /* print vpr and wpr info */ + val = gk20a_readl(g, fb_mmu_vpr_info_r()); + val &= ~0x3; + val |= fb_mmu_vpr_info_index_addr_lo_v(); + gk20a_writel(g, fb_mmu_vpr_info_r(), val); + nvgpu_err(g, "VPR: %08x %08x %08x %08x", + gk20a_readl(g, fb_mmu_vpr_info_r()), + gk20a_readl(g, fb_mmu_vpr_info_r()), + gk20a_readl(g, fb_mmu_vpr_info_r()), + gk20a_readl(g, fb_mmu_vpr_info_r())); + + val = gk20a_readl(g, fb_mmu_wpr_info_r()); + val &= ~0xf; + val |= (fb_mmu_wpr_info_index_allow_read_v()); + gk20a_writel(g, fb_mmu_wpr_info_r(), val); + nvgpu_err(g, "WPR: %08x %08x %08x %08x %08x %08x", + gk20a_readl(g, fb_mmu_wpr_info_r()), + gk20a_readl(g, fb_mmu_wpr_info_r()), + gk20a_readl(g, fb_mmu_wpr_info_r()), + gk20a_readl(g, fb_mmu_wpr_info_r()), + gk20a_readl(g, fb_mmu_wpr_info_r()), + gk20a_readl(g, fb_mmu_wpr_info_r())); + +} + +static int gm20b_fb_vpr_info_fetch_wait(struct gk20a *g, + unsigned int msec) +{ + struct nvgpu_timeout timeout; + + nvgpu_timeout_init(g, &timeout, msec, NVGPU_TIMER_CPU_TIMER); + + do { + u32 val; + + val = gk20a_readl(g, fb_mmu_vpr_info_r()); + if (fb_mmu_vpr_info_fetch_v(val) == + fb_mmu_vpr_info_fetch_false_v()) + return 0; + + } while (!nvgpu_timeout_expired(&timeout)); + + return -ETIMEDOUT; +} + +int gm20b_fb_vpr_info_fetch(struct gk20a *g) +{ + if (gm20b_fb_vpr_info_fetch_wait(g, VPR_INFO_FETCH_WAIT)) { + return -ETIMEDOUT; + } + + gk20a_writel(g, fb_mmu_vpr_info_r(), + fb_mmu_vpr_info_fetch_true_v()); + + return gm20b_fb_vpr_info_fetch_wait(g, VPR_INFO_FETCH_WAIT); +} + +void gm20b_fb_read_wpr_info(struct gk20a *g, struct wpr_carveout_info *inf) +{ + u32 val = 0; + u64 wpr_start = 0; + u64 wpr_end = 0; + + val = gk20a_readl(g, fb_mmu_wpr_info_r()); + val &= ~0xF; + val |= fb_mmu_wpr_info_index_wpr1_addr_lo_v(); + gk20a_writel(g, fb_mmu_wpr_info_r(), val); + + val = gk20a_readl(g, fb_mmu_wpr_info_r()) >> 0x4; + wpr_start = hi32_lo32_to_u64( + (val >> (32 - WPR_INFO_ADDR_ALIGNMENT)), + (val << WPR_INFO_ADDR_ALIGNMENT)); + + val = gk20a_readl(g, fb_mmu_wpr_info_r()); + val &= ~0xF; + val |= fb_mmu_wpr_info_index_wpr1_addr_hi_v(); + gk20a_writel(g, fb_mmu_wpr_info_r(), val); + + val = gk20a_readl(g, fb_mmu_wpr_info_r()) >> 0x4; + wpr_end = hi32_lo32_to_u64( + (val >> (32 - WPR_INFO_ADDR_ALIGNMENT)), + (val << WPR_INFO_ADDR_ALIGNMENT)); + + inf->wpr_base = wpr_start; + inf->nonwpr_base = 0; + inf->size = (wpr_end - wpr_start); +} + +bool gm20b_fb_debug_mode_enabled(struct gk20a *g) +{ + u32 debug_ctrl = gk20a_readl(g, gr_gpcs_pri_mmu_debug_ctrl_r()); + return gr_gpcs_pri_mmu_debug_ctrl_debug_v(debug_ctrl) == + gr_gpcs_pri_mmu_debug_ctrl_debug_enabled_v(); +} + +void gm20b_fb_set_debug_mode(struct gk20a *g, bool enable) +{ + u32 reg_val, fb_debug_ctrl, gpc_debug_ctrl; + + if (enable) { + fb_debug_ctrl = fb_mmu_debug_ctrl_debug_enabled_f(); + gpc_debug_ctrl = gr_gpcs_pri_mmu_debug_ctrl_debug_enabled_f(); + g->mmu_debug_ctrl = true; + } else { + fb_debug_ctrl = fb_mmu_debug_ctrl_debug_disabled_f(); + gpc_debug_ctrl = gr_gpcs_pri_mmu_debug_ctrl_debug_disabled_f(); + g->mmu_debug_ctrl = false; + } + + reg_val = gk20a_readl(g, fb_mmu_debug_ctrl_r()); + reg_val = set_field(reg_val, + fb_mmu_debug_ctrl_debug_m(), fb_debug_ctrl); + gk20a_writel(g, fb_mmu_debug_ctrl_r(), reg_val); + + reg_val = gk20a_readl(g, gr_gpcs_pri_mmu_debug_ctrl_r()); + reg_val = set_field(reg_val, + gr_gpcs_pri_mmu_debug_ctrl_debug_m(), gpc_debug_ctrl); + gk20a_writel(g, gr_gpcs_pri_mmu_debug_ctrl_r(), reg_val); +} diff --git a/drivers/gpu/nvgpu/common/fb/fb_gm20b.h b/drivers/gpu/nvgpu/common/fb/fb_gm20b.h new file mode 100644 index 00000000..eb868b01 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gm20b.h @@ -0,0 +1,47 @@ +/* + * GM20B FB + * + * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NVHOST_GM20B_FB +#define _NVHOST_GM20B_FB + +struct gk20a; +struct wpr_carveout_info; + +void fb_gm20b_init_fs_state(struct gk20a *g); +void gm20b_fb_set_mmu_page_size(struct gk20a *g); +bool gm20b_fb_set_use_full_comp_tag_line(struct gk20a *g); +u32 gm20b_fb_mmu_ctrl(struct gk20a *g); +u32 gm20b_fb_mmu_debug_ctrl(struct gk20a *g); +u32 gm20b_fb_mmu_debug_wr(struct gk20a *g); +u32 gm20b_fb_mmu_debug_rd(struct gk20a *g); +unsigned int gm20b_fb_compression_page_size(struct gk20a *g); +unsigned int gm20b_fb_compressible_page_size(struct gk20a *g); +u32 gm20b_fb_compression_align_mask(struct gk20a *g); +void gm20b_fb_dump_vpr_wpr_info(struct gk20a *g); +void gm20b_fb_read_wpr_info(struct gk20a *g, struct wpr_carveout_info *inf); +int gm20b_fb_vpr_info_fetch(struct gk20a *g); +bool gm20b_fb_debug_mode_enabled(struct gk20a *g); +void gm20b_fb_set_debug_mode(struct gk20a *g, bool enable); + +#endif diff --git a/drivers/gpu/nvgpu/common/fb/fb_gp106.c b/drivers/gpu/nvgpu/common/fb/fb_gp106.c new file mode 100644 index 00000000..6f257db4 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gp106.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "gk20a/gk20a.h" + +#include "fb_gp10b.h" +#include "fb_gp106.h" + +#include + +#define HW_SCRUB_TIMEOUT_DEFAULT 100 /* usec */ +#define HW_SCRUB_TIMEOUT_MAX 2000000 /* usec */ + +void gp106_fb_reset(struct gk20a *g) +{ + u32 val; + + int retries = HW_SCRUB_TIMEOUT_MAX / HW_SCRUB_TIMEOUT_DEFAULT; + /* wait for memory to be accessible */ + do { + u32 w = gk20a_readl(g, fb_niso_scrub_status_r()); + if (fb_niso_scrub_status_flag_v(w)) { + nvgpu_log_fn(g, "done"); + break; + } + nvgpu_udelay(HW_SCRUB_TIMEOUT_DEFAULT); + } while (--retries); + + val = gk20a_readl(g, fb_mmu_priv_level_mask_r()); + val &= ~fb_mmu_priv_level_mask_write_violation_m(); + gk20a_writel(g, fb_mmu_priv_level_mask_r(), val); +} diff --git a/drivers/gpu/nvgpu/common/fb/fb_gp106.h b/drivers/gpu/nvgpu/common/fb/fb_gp106.h new file mode 100644 index 00000000..d5ee87f4 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gp106.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef FB_GP106_H +#define FB_GP106_H +struct gpu_ops; + +void gp106_fb_reset(struct gk20a *g); +#endif diff --git a/drivers/gpu/nvgpu/common/fb/fb_gp10b.c b/drivers/gpu/nvgpu/common/fb/fb_gp10b.c new file mode 100644 index 00000000..45fc8373 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gp10b.c @@ -0,0 +1,38 @@ +/* + * GP10B FB + * + * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. +* + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include + +#include "gk20a/gk20a.h" +#include "fb_gp10b.h" + +unsigned int gp10b_fb_compression_page_size(struct gk20a *g) +{ + return SZ_64K; +} + +unsigned int gp10b_fb_compressible_page_size(struct gk20a *g) +{ + return SZ_4K; +} diff --git a/drivers/gpu/nvgpu/common/fb/fb_gp10b.h b/drivers/gpu/nvgpu/common/fb/fb_gp10b.h new file mode 100644 index 00000000..52aa2a75 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gp10b.h @@ -0,0 +1,32 @@ +/* + * GP10B FB + * + * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NVGPU_GP10B_FB +#define _NVGPU_GP10B_FB +struct gk20a; + +unsigned int gp10b_fb_compression_page_size(struct gk20a *g); +unsigned int gp10b_fb_compressible_page_size(struct gk20a *g); + +#endif diff --git a/drivers/gpu/nvgpu/common/fb/fb_gv100.c b/drivers/gpu/nvgpu/common/fb/fb_gv100.c new file mode 100644 index 00000000..155c1e8b --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gv100.c @@ -0,0 +1,257 @@ +/* + * GV100 FB + * + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gk20a/gk20a.h" +#include "gm20b/acr_gm20b.h" + +#include "fb_gv100.h" + +#include +#include +#include + +#define HW_SCRUB_TIMEOUT_DEFAULT 100 /* usec */ +#define HW_SCRUB_TIMEOUT_MAX 2000000 /* usec */ +#define MEM_UNLOCK_TIMEOUT 3500 /* msec */ + +void gv100_fb_reset(struct gk20a *g) +{ + u32 val; + int retries = HW_SCRUB_TIMEOUT_MAX / HW_SCRUB_TIMEOUT_DEFAULT; + + nvgpu_info(g, "reset gv100 fb"); + + /* wait for memory to be accessible */ + do { + u32 w = gk20a_readl(g, fb_niso_scrub_status_r()); + if (fb_niso_scrub_status_flag_v(w)) { + nvgpu_info(g, "done"); + break; + } + nvgpu_udelay(HW_SCRUB_TIMEOUT_DEFAULT); + } while (--retries); + + val = gk20a_readl(g, fb_mmu_priv_level_mask_r()); + val &= ~fb_mmu_priv_level_mask_write_violation_m(); + gk20a_writel(g, fb_mmu_priv_level_mask_r(), val); +} + +void gv100_fb_enable_hub_intr(struct gk20a *g) +{ + u32 mask = 0; + + mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | + fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | + fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | + fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | + fb_niso_intr_en_set_mmu_replayable_fault_overflow_m(); + + gk20a_writel(g, fb_niso_intr_en_set_r(0), + mask); +} + +void gv100_fb_disable_hub_intr(struct gk20a *g) +{ + u32 mask = 0; + + mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | + fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | + fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | + fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | + fb_niso_intr_en_set_mmu_replayable_fault_overflow_m(); + + gk20a_writel(g, fb_niso_intr_en_clr_r(0), + mask); +} + +int gv100_fb_memory_unlock(struct gk20a *g) +{ + struct nvgpu_firmware *mem_unlock_fw = NULL; + struct bin_hdr *hsbin_hdr = NULL; + struct acr_fw_header *fw_hdr = NULL; + u32 *mem_unlock_ucode = NULL; + u32 *mem_unlock_ucode_header = NULL; + u32 sec_imem_dest = 0; + u32 val = 0; + int err = 0; + + nvgpu_log_fn(g, " "); + + nvgpu_log_info(g, "fb_mmu_vpr_info = 0x%08x", + gk20a_readl(g, fb_mmu_vpr_info_r())); + /* + * mem_unlock.bin should be written to install + * traps even if VPR isn’t actually supported + */ + mem_unlock_fw = nvgpu_request_firmware(g, "mem_unlock.bin", 0); + if (!mem_unlock_fw) { + nvgpu_err(g, "mem unlock ucode get fail"); + err = -ENOENT; + goto exit; + } + + /* Enable nvdec */ + g->ops.mc.enable(g, mc_enable_nvdec_enabled_f()); + + /* nvdec falcon reset */ + nvgpu_flcn_reset(&g->nvdec_flcn); + + hsbin_hdr = (struct bin_hdr *)mem_unlock_fw->data; + fw_hdr = (struct acr_fw_header *)(mem_unlock_fw->data + + hsbin_hdr->header_offset); + + mem_unlock_ucode_header = (u32 *)(mem_unlock_fw->data + + fw_hdr->hdr_offset); + mem_unlock_ucode = (u32 *)(mem_unlock_fw->data + + hsbin_hdr->data_offset); + + /* Patch Ucode singnatures */ + if (acr_ucode_patch_sig(g, mem_unlock_ucode, + (u32 *)(mem_unlock_fw->data + fw_hdr->sig_prod_offset), + (u32 *)(mem_unlock_fw->data + fw_hdr->sig_dbg_offset), + (u32 *)(mem_unlock_fw->data + fw_hdr->patch_loc), + (u32 *)(mem_unlock_fw->data + fw_hdr->patch_sig)) < 0) { + nvgpu_err(g, "mem unlock patch signatures fail"); + err = -EPERM; + goto exit; + } + + /* Clear interrupts */ + nvgpu_flcn_set_irq(&g->nvdec_flcn, false, 0x0, 0x0); + + /* Copy Non Secure IMEM code */ + nvgpu_flcn_copy_to_imem(&g->nvdec_flcn, 0, + (u8 *)&mem_unlock_ucode[ + mem_unlock_ucode_header[OS_CODE_OFFSET] >> 2], + mem_unlock_ucode_header[OS_CODE_SIZE], 0, false, + GET_IMEM_TAG(mem_unlock_ucode_header[OS_CODE_OFFSET])); + + /* Put secure code after non-secure block */ + sec_imem_dest = GET_NEXT_BLOCK(mem_unlock_ucode_header[OS_CODE_SIZE]); + + nvgpu_flcn_copy_to_imem(&g->nvdec_flcn, sec_imem_dest, + (u8 *)&mem_unlock_ucode[ + mem_unlock_ucode_header[APP_0_CODE_OFFSET] >> 2], + mem_unlock_ucode_header[APP_0_CODE_SIZE], 0, true, + GET_IMEM_TAG(mem_unlock_ucode_header[APP_0_CODE_OFFSET])); + + /* load DMEM: ensure that signatures are patched */ + nvgpu_flcn_copy_to_dmem(&g->nvdec_flcn, 0, (u8 *)&mem_unlock_ucode[ + mem_unlock_ucode_header[OS_DATA_OFFSET] >> 2], + mem_unlock_ucode_header[OS_DATA_SIZE], 0); + + nvgpu_log_info(g, "nvdec sctl reg %x\n", + gk20a_readl(g, g->nvdec_flcn.flcn_base + + falcon_falcon_sctl_r())); + + /* set BOOTVEC to start of non-secure code */ + nvgpu_flcn_bootstrap(&g->nvdec_flcn, 0); + + /* wait for complete & halt */ + nvgpu_flcn_wait_for_halt(&g->nvdec_flcn, MEM_UNLOCK_TIMEOUT); + + /* check mem unlock status */ + val = nvgpu_flcn_mailbox_read(&g->nvdec_flcn, 0); + if (val) { + nvgpu_err(g, "memory unlock failed, err %x", val); + err = -1; + goto exit; + } + + nvgpu_log_info(g, "nvdec sctl reg %x\n", + gk20a_readl(g, g->nvdec_flcn.flcn_base + + falcon_falcon_sctl_r())); + +exit: + if (mem_unlock_fw) + nvgpu_release_firmware(g, mem_unlock_fw); + + nvgpu_log_fn(g, "done, status - %d", err); + + return err; +} + +int gv100_fb_init_nvlink(struct gk20a *g) +{ + u32 data; + u32 mask = g->nvlink.enabled_links; + + /* Map enabled link to SYSMEM */ + data = nvgpu_readl(g, fb_hshub_config0_r()); + data = set_field(data, fb_hshub_config0_sysmem_nvlink_mask_m(), + fb_hshub_config0_sysmem_nvlink_mask_f(mask)); + nvgpu_writel(g, fb_hshub_config0_r(), data); + + return 0; +} + +int gv100_fb_enable_nvlink(struct gk20a *g) +{ + u32 data; + + nvgpu_log(g, gpu_dbg_nvlink|gpu_dbg_info, "enabling nvlink"); + + /* Enable nvlink for NISO FBHUB */ + data = nvgpu_readl(g, fb_niso_cfg1_r()); + data = set_field(data, fb_niso_cfg1_sysmem_nvlink_m(), + fb_niso_cfg1_sysmem_nvlink_enabled_f()); + nvgpu_writel(g, fb_niso_cfg1_r(), data); + + /* Setup atomics */ + data = nvgpu_readl(g, fb_mmu_ctrl_r()); + data = set_field(data, fb_mmu_ctrl_atomic_capability_mode_m(), + fb_mmu_ctrl_atomic_capability_mode_rmw_f()); + nvgpu_writel(g, fb_mmu_ctrl_r(), data); + + data = nvgpu_readl(g, fb_hsmmu_pri_mmu_ctrl_r()); + data = set_field(data, fb_hsmmu_pri_mmu_ctrl_atomic_capability_mode_m(), + fb_hsmmu_pri_mmu_ctrl_atomic_capability_mode_rmw_f()); + nvgpu_writel(g, fb_hsmmu_pri_mmu_ctrl_r(), data); + + data = nvgpu_readl(g, fb_fbhub_num_active_ltcs_r()); + data = set_field(data, fb_fbhub_num_active_ltcs_hub_sys_atomic_mode_m(), + fb_fbhub_num_active_ltcs_hub_sys_atomic_mode_use_rmw_f()); + nvgpu_writel(g, fb_fbhub_num_active_ltcs_r(), data); + + data = nvgpu_readl(g, fb_hshub_num_active_ltcs_r()); + data = set_field(data, fb_hshub_num_active_ltcs_hub_sys_atomic_mode_m(), + fb_hshub_num_active_ltcs_hub_sys_atomic_mode_use_rmw_f()); + nvgpu_writel(g, fb_hshub_num_active_ltcs_r(), data); + + return 0; +} diff --git a/drivers/gpu/nvgpu/common/fb/fb_gv100.h b/drivers/gpu/nvgpu/common/fb/fb_gv100.h new file mode 100644 index 00000000..195baccf --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gv100.h @@ -0,0 +1,36 @@ +/* + * GV100 FB + * + * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NVGPU_GV100_FB +#define _NVGPU_GV100_FB + +struct gk20a; + +void gv100_fb_reset(struct gk20a *g); +void gv100_fb_enable_hub_intr(struct gk20a *g); +void gv100_fb_disable_hub_intr(struct gk20a *g); +int gv100_fb_memory_unlock(struct gk20a *g); +int gv100_fb_init_nvlink(struct gk20a *g); +int gv100_fb_enable_nvlink(struct gk20a *g); +#endif diff --git a/drivers/gpu/nvgpu/common/fb/fb_gv11b.c b/drivers/gpu/nvgpu/common/fb/fb_gv11b.c new file mode 100644 index 00000000..69a71575 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gv11b.c @@ -0,0 +1,1516 @@ +/* + * GV11B FB + * + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "gk20a/gk20a.h" +#include "gk20a/mm_gk20a.h" + +#include "gv11b/fifo_gv11b.h" +#include "gv11b/ce_gv11b.h" + +#include "fb_gk20a.h" +#include "fb_gp10b.h" +#include "fb_gv11b.h" + +#include +#include +#include +#include + +static int gv11b_fb_fix_page_fault(struct gk20a *g, + struct mmu_fault_info *mmfault); + +static void gv11b_init_nvlink_soc_credits(struct gk20a *g) +{ + if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) { + nvgpu_log(g, gpu_dbg_info, "nvlink soc credits init done by bpmp"); + } else { +#ifndef __NVGPU_POSIX__ + nvgpu_mss_nvlink_init_credits(g); +#endif + } +} + +void gv11b_fb_init_hw(struct gk20a *g) +{ + gk20a_fb_init_hw(g); + + g->ops.fb.enable_hub_intr(g); +} + +void gv11b_fb_init_fs_state(struct gk20a *g) +{ + nvgpu_log(g, gpu_dbg_fn, "initialize gv11b fb"); + + nvgpu_log(g, gpu_dbg_info, "fbhub active ltcs %x", + gk20a_readl(g, fb_fbhub_num_active_ltcs_r())); + + nvgpu_log(g, gpu_dbg_info, "mmu active ltcs %u", + fb_mmu_num_active_ltcs_count_v( + gk20a_readl(g, fb_mmu_num_active_ltcs_r()))); + + if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) { + /* Bypass MMU check for non-secure boot. For + * secure-boot,this register write has no-effect */ + gk20a_writel(g, fb_priv_mmu_phy_secure_r(), 0xffffffffU); + } +} + +void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr) +{ + u32 max_size = gr->max_comptag_mem; + /* one tag line covers 64KB */ + u32 max_comptag_lines = max_size << 4; + u32 compbit_base_post_divide; + u64 compbit_base_post_multiply64; + u64 compbit_store_iova; + u64 compbit_base_post_divide64; + + if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) + compbit_store_iova = nvgpu_mem_get_phys_addr(g, + &gr->compbit_store.mem); + else + compbit_store_iova = nvgpu_mem_get_addr(g, + &gr->compbit_store.mem); + + compbit_base_post_divide64 = compbit_store_iova >> + fb_mmu_cbc_base_address_alignment_shift_v(); + + do_div(compbit_base_post_divide64, g->ltc_count); + compbit_base_post_divide = u64_lo32(compbit_base_post_divide64); + + compbit_base_post_multiply64 = ((u64)compbit_base_post_divide * + g->ltc_count) << fb_mmu_cbc_base_address_alignment_shift_v(); + + if (compbit_base_post_multiply64 < compbit_store_iova) + compbit_base_post_divide++; + + if (g->ops.ltc.cbc_fix_config) + compbit_base_post_divide = + g->ops.ltc.cbc_fix_config(g, compbit_base_post_divide); + + gk20a_writel(g, fb_mmu_cbc_base_r(), + fb_mmu_cbc_base_address_f(compbit_base_post_divide)); + + nvgpu_log(g, gpu_dbg_info | gpu_dbg_map_v | gpu_dbg_pte, + "compbit base.pa: 0x%x,%08x cbc_base:0x%08x\n", + (u32)(compbit_store_iova >> 32), + (u32)(compbit_store_iova & 0xffffffff), + compbit_base_post_divide); + nvgpu_log(g, gpu_dbg_fn, "cbc base %x", + gk20a_readl(g, fb_mmu_cbc_base_r())); + + gr->compbit_store.base_hw = compbit_base_post_divide; + + g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_invalidate, + 0, max_comptag_lines - 1); + +} + +void gv11b_fb_reset(struct gk20a *g) +{ + gv11b_init_nvlink_soc_credits(g); +} + +static const char * const invalid_str = "invalid"; + +static const char *const fault_type_descs_gv11b[] = { + "invalid pde", + "invalid pde size", + "invalid pte", + "limit violation", + "unbound inst block", + "priv violation", + "write", + "read", + "pitch mask violation", + "work creation", + "unsupported aperture", + "compression failure", + "unsupported kind", + "region violation", + "poison", + "atomic" +}; + +static const char *const fault_client_type_descs_gv11b[] = { + "gpc", + "hub", +}; + +static const char *const fault_access_type_descs_gv11b[] = { + "virt read", + "virt write", + "virt atomic strong", + "virt prefetch", + "virt atomic weak", + "xxx", + "xxx", + "xxx", + "phys read", + "phys write", + "phys atomic", + "phys prefetch", +}; + +static const char *const hub_client_descs_gv11b[] = { + "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu", + "host cpu nb", "iso", "mmu", "nvdec", "nvenc1", "nvenc2", + "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc", + "scc nb", "sec", "ssync", "gr copy", "xv", "mmu nb", + "nvenc", "d falcon", "sked", "a falcon", "hsce0", "hsce1", + "hsce2", "hsce3", "hsce4", "hsce5", "hsce6", "hsce7", "hsce8", + "hsce9", "hshub", "ptp x0", "ptp x1", "ptp x2", "ptp x3", + "ptp x4", "ptp x5", "ptp x6", "ptp x7", "vpr scrubber0", + "vpr scrubber1", "dwbif", "fbfalcon", "ce shim", "gsp", + "dont care" +}; + +static const char *const gpc_client_descs_gv11b[] = { + "t1 0", "t1 1", "t1 2", "t1 3", + "t1 4", "t1 5", "t1 6", "t1 7", + "pe 0", "pe 1", "pe 2", "pe 3", + "pe 4", "pe 5", "pe 6", "pe 7", + "rast", "gcc", "gpccs", + "prop 0", "prop 1", "prop 2", "prop 3", + "gpm", + "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3", + "ltp utlb 4", "ltp utlb 5", "ltp utlb 6", "ltp utlb 7", + "utlb", + "t1 8", "t1 9", "t1 10", "t1 11", + "t1 12", "t1 13", "t1 14", "t1 15", + "tpccs 0", "tpccs 1", "tpccs 2", "tpccs 3", + "tpccs 4", "tpccs 5", "tpccs 6", "tpccs 7", + "pe 8", "pe 9", "tpccs 8", "tpccs 9", + "t1 16", "t1 17", "t1 18", "t1 19", + "pe 10", "pe 11", "tpccs 10", "tpccs 11", + "t1 20", "t1 21", "t1 22", "t1 23", + "pe 12", "pe 13", "tpccs 12", "tpccs 13", + "t1 24", "t1 25", "t1 26", "t1 27", + "pe 14", "pe 15", "tpccs 14", "tpccs 15", + "t1 28", "t1 29", "t1 30", "t1 31", + "pe 16", "pe 17", "tpccs 16", "tpccs 17", + "t1 32", "t1 33", "t1 34", "t1 35", + "pe 18", "pe 19", "tpccs 18", "tpccs 19", + "t1 36", "t1 37", "t1 38", "t1 39", +}; + +bool gv11b_fb_is_fault_buf_enabled(struct gk20a *g, u32 index) +{ + u32 reg_val; + + reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); + return fb_mmu_fault_buffer_size_enable_v(reg_val) != 0U; +} + +static void gv11b_fb_fault_buffer_get_ptr_update(struct gk20a *g, + u32 index, u32 next) +{ + u32 reg_val; + + nvgpu_log(g, gpu_dbg_intr, "updating get index with = %d", next); + + reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); + reg_val = set_field(reg_val, fb_mmu_fault_buffer_get_ptr_m(), + fb_mmu_fault_buffer_get_ptr_f(next)); + + /* while the fault is being handled it is possible for overflow + * to happen, + */ + if (reg_val & fb_mmu_fault_buffer_get_overflow_m()) + reg_val |= fb_mmu_fault_buffer_get_overflow_clear_f(); + + g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); + + /* make sure get ptr update is visible to everyone to avoid + * reading already read entry + */ + nvgpu_mb(); +} + +static u32 gv11b_fb_fault_buffer_get_index(struct gk20a *g, u32 index) +{ + u32 reg_val; + + reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); + return fb_mmu_fault_buffer_get_ptr_v(reg_val); +} + +static u32 gv11b_fb_fault_buffer_put_index(struct gk20a *g, u32 index) +{ + u32 reg_val; + + reg_val = g->ops.fb.read_mmu_fault_buffer_put(g, index); + return fb_mmu_fault_buffer_put_ptr_v(reg_val); +} + +static u32 gv11b_fb_fault_buffer_size_val(struct gk20a *g, u32 index) +{ + u32 reg_val; + + reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); + return fb_mmu_fault_buffer_size_val_v(reg_val); +} + +static bool gv11b_fb_is_fault_buffer_empty(struct gk20a *g, + u32 index, u32 *get_idx) +{ + u32 put_idx; + + *get_idx = gv11b_fb_fault_buffer_get_index(g, index); + put_idx = gv11b_fb_fault_buffer_put_index(g, index); + + return *get_idx == put_idx; +} + +static bool gv11b_fb_is_fault_buffer_full(struct gk20a *g, u32 index) +{ + u32 get_idx, put_idx, entries; + + + get_idx = gv11b_fb_fault_buffer_get_index(g, index); + + put_idx = gv11b_fb_fault_buffer_put_index(g, index); + + entries = gv11b_fb_fault_buffer_size_val(g, index); + + return get_idx == ((put_idx + 1) % entries); +} + +void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g, + u32 index, u32 state) +{ + u32 fault_status; + u32 reg_val; + + nvgpu_log_fn(g, " "); + + reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); + if (state == NVGPU_FB_MMU_FAULT_BUF_ENABLED) { + if (gv11b_fb_is_fault_buf_enabled(g, index)) { + nvgpu_log_info(g, "fault buffer is already enabled"); + } else { + reg_val |= fb_mmu_fault_buffer_size_enable_true_f(); + g->ops.fb.write_mmu_fault_buffer_size(g, index, + reg_val); + } + + } else { + struct nvgpu_timeout timeout; + u32 delay = GR_IDLE_CHECK_DEFAULT; + + nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), + NVGPU_TIMER_CPU_TIMER); + + reg_val &= (~(fb_mmu_fault_buffer_size_enable_m())); + g->ops.fb.write_mmu_fault_buffer_size(g, index, reg_val); + + fault_status = g->ops.fb.read_mmu_fault_status(g); + + do { + if (!(fault_status & fb_mmu_fault_status_busy_true_f())) + break; + /* + * Make sure fault buffer is disabled. + * This is to avoid accessing fault buffer by hw + * during the window BAR2 is being unmapped by s/w + */ + nvgpu_log_info(g, "fault status busy set, check again"); + fault_status = g->ops.fb.read_mmu_fault_status(g); + + nvgpu_usleep_range(delay, delay * 2); + delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); + } while (!nvgpu_timeout_expired_msg(&timeout, + "fault status busy set")); + } +} + +void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, u32 index) +{ + u32 addr_lo; + u32 addr_hi; + + nvgpu_log_fn(g, " "); + + gv11b_fb_fault_buf_set_state_hw(g, index, + NVGPU_FB_MMU_FAULT_BUF_DISABLED); + addr_lo = u64_lo32(g->mm.hw_fault_buf[index].gpu_va >> + ram_in_base_shift_v()); + addr_hi = u64_hi32(g->mm.hw_fault_buf[index].gpu_va); + + g->ops.fb.write_mmu_fault_buffer_lo_hi(g, index, + fb_mmu_fault_buffer_lo_addr_f(addr_lo), + fb_mmu_fault_buffer_hi_addr_f(addr_hi)); + + g->ops.fb.write_mmu_fault_buffer_size(g, index, + fb_mmu_fault_buffer_size_val_f(g->ops.fifo.get_num_fifos(g)) | + fb_mmu_fault_buffer_size_overflow_intr_enable_f()); + + gv11b_fb_fault_buf_set_state_hw(g, index, NVGPU_FB_MMU_FAULT_BUF_ENABLED); +} + +void gv11b_fb_enable_hub_intr(struct gk20a *g) +{ + u32 mask = 0; + + mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | + fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | + fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | + fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | + fb_niso_intr_en_set_mmu_replayable_fault_overflow_m() | + fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); + + gk20a_writel(g, fb_niso_intr_en_set_r(0), + mask); +} + +void gv11b_fb_disable_hub_intr(struct gk20a *g) +{ + u32 mask = 0; + + mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | + fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | + fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | + fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | + fb_niso_intr_en_set_mmu_replayable_fault_overflow_m() | + fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); + + gk20a_writel(g, fb_niso_intr_en_clr_r(0), + mask); +} + +void gv11b_handle_l2tlb_ecc_isr(struct gk20a *g, u32 ecc_status) +{ + u32 ecc_addr, corrected_cnt, uncorrected_cnt; + u32 corrected_delta, uncorrected_delta; + u32 corrected_overflow, uncorrected_overflow; + + ecc_addr = gk20a_readl(g, fb_mmu_l2tlb_ecc_address_r()); + corrected_cnt = gk20a_readl(g, + fb_mmu_l2tlb_ecc_corrected_err_count_r()); + uncorrected_cnt = gk20a_readl(g, + fb_mmu_l2tlb_ecc_uncorrected_err_count_r()); + + corrected_delta = fb_mmu_l2tlb_ecc_corrected_err_count_total_v( + corrected_cnt); + uncorrected_delta = fb_mmu_l2tlb_ecc_uncorrected_err_count_total_v( + uncorrected_cnt); + corrected_overflow = ecc_status & + fb_mmu_l2tlb_ecc_status_corrected_err_total_counter_overflow_m(); + + uncorrected_overflow = ecc_status & + fb_mmu_l2tlb_ecc_status_uncorrected_err_total_counter_overflow_m(); + + /* clear the interrupt */ + if ((corrected_delta > 0) || corrected_overflow) + gk20a_writel(g, fb_mmu_l2tlb_ecc_corrected_err_count_r(), 0); + if ((uncorrected_delta > 0) || uncorrected_overflow) + gk20a_writel(g, fb_mmu_l2tlb_ecc_uncorrected_err_count_r(), 0); + + gk20a_writel(g, fb_mmu_l2tlb_ecc_status_r(), + fb_mmu_l2tlb_ecc_status_reset_clear_f()); + + /* Handle overflow */ + if (corrected_overflow) + corrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_corrected_err_count_total_s()); + if (uncorrected_overflow) + uncorrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_uncorrected_err_count_total_s()); + + + g->ecc.fb.mmu_l2tlb_corrected_err_count.counters[0] += + corrected_delta; + g->ecc.fb.mmu_l2tlb_uncorrected_err_count.counters[0] += + uncorrected_delta; + + if (ecc_status & fb_mmu_l2tlb_ecc_status_corrected_err_l2tlb_sa_data_m()) + nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); + if (ecc_status & fb_mmu_l2tlb_ecc_status_uncorrected_err_l2tlb_sa_data_m()) + nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); + if (corrected_overflow || uncorrected_overflow) + nvgpu_info(g, "mmu l2tlb ecc counter overflow!"); + + nvgpu_log(g, gpu_dbg_intr, + "ecc error address: 0x%x", ecc_addr); + nvgpu_log(g, gpu_dbg_intr, + "ecc error count corrected: %d, uncorrected %d", + g->ecc.fb.mmu_l2tlb_corrected_err_count.counters[0], + g->ecc.fb.mmu_l2tlb_uncorrected_err_count.counters[0]); +} + +void gv11b_handle_hubtlb_ecc_isr(struct gk20a *g, u32 ecc_status) +{ + u32 ecc_addr, corrected_cnt, uncorrected_cnt; + u32 corrected_delta, uncorrected_delta; + u32 corrected_overflow, uncorrected_overflow; + + ecc_addr = gk20a_readl(g, fb_mmu_hubtlb_ecc_address_r()); + corrected_cnt = gk20a_readl(g, + fb_mmu_hubtlb_ecc_corrected_err_count_r()); + uncorrected_cnt = gk20a_readl(g, + fb_mmu_hubtlb_ecc_uncorrected_err_count_r()); + + corrected_delta = fb_mmu_hubtlb_ecc_corrected_err_count_total_v( + corrected_cnt); + uncorrected_delta = fb_mmu_hubtlb_ecc_uncorrected_err_count_total_v( + uncorrected_cnt); + corrected_overflow = ecc_status & + fb_mmu_hubtlb_ecc_status_corrected_err_total_counter_overflow_m(); + + uncorrected_overflow = ecc_status & + fb_mmu_hubtlb_ecc_status_uncorrected_err_total_counter_overflow_m(); + + /* clear the interrupt */ + if ((corrected_delta > 0) || corrected_overflow) + gk20a_writel(g, fb_mmu_hubtlb_ecc_corrected_err_count_r(), 0); + if ((uncorrected_delta > 0) || uncorrected_overflow) + gk20a_writel(g, fb_mmu_hubtlb_ecc_uncorrected_err_count_r(), 0); + + gk20a_writel(g, fb_mmu_hubtlb_ecc_status_r(), + fb_mmu_hubtlb_ecc_status_reset_clear_f()); + + /* Handle overflow */ + if (corrected_overflow) + corrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_corrected_err_count_total_s()); + if (uncorrected_overflow) + uncorrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_uncorrected_err_count_total_s()); + + + g->ecc.fb.mmu_hubtlb_corrected_err_count.counters[0] += + corrected_delta; + g->ecc.fb.mmu_hubtlb_uncorrected_err_count.counters[0] += + uncorrected_delta; + + if (ecc_status & fb_mmu_hubtlb_ecc_status_corrected_err_sa_data_m()) + nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); + if (ecc_status & fb_mmu_hubtlb_ecc_status_uncorrected_err_sa_data_m()) + nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); + if (corrected_overflow || uncorrected_overflow) + nvgpu_info(g, "mmu hubtlb ecc counter overflow!"); + + nvgpu_log(g, gpu_dbg_intr, + "ecc error address: 0x%x", ecc_addr); + nvgpu_log(g, gpu_dbg_intr, + "ecc error count corrected: %d, uncorrected %d", + g->ecc.fb.mmu_hubtlb_corrected_err_count.counters[0], + g->ecc.fb.mmu_hubtlb_uncorrected_err_count.counters[0]); +} + +void gv11b_handle_fillunit_ecc_isr(struct gk20a *g, u32 ecc_status) +{ + u32 ecc_addr, corrected_cnt, uncorrected_cnt; + u32 corrected_delta, uncorrected_delta; + u32 corrected_overflow, uncorrected_overflow; + + ecc_addr = gk20a_readl(g, fb_mmu_fillunit_ecc_address_r()); + corrected_cnt = gk20a_readl(g, + fb_mmu_fillunit_ecc_corrected_err_count_r()); + uncorrected_cnt = gk20a_readl(g, + fb_mmu_fillunit_ecc_uncorrected_err_count_r()); + + corrected_delta = fb_mmu_fillunit_ecc_corrected_err_count_total_v( + corrected_cnt); + uncorrected_delta = fb_mmu_fillunit_ecc_uncorrected_err_count_total_v( + uncorrected_cnt); + corrected_overflow = ecc_status & + fb_mmu_fillunit_ecc_status_corrected_err_total_counter_overflow_m(); + + uncorrected_overflow = ecc_status & + fb_mmu_fillunit_ecc_status_uncorrected_err_total_counter_overflow_m(); + + /* clear the interrupt */ + if ((corrected_delta > 0) || corrected_overflow) + gk20a_writel(g, fb_mmu_fillunit_ecc_corrected_err_count_r(), 0); + if ((uncorrected_delta > 0) || uncorrected_overflow) + gk20a_writel(g, fb_mmu_fillunit_ecc_uncorrected_err_count_r(), 0); + + gk20a_writel(g, fb_mmu_fillunit_ecc_status_r(), + fb_mmu_fillunit_ecc_status_reset_clear_f()); + + /* Handle overflow */ + if (corrected_overflow) + corrected_delta += (0x1UL << fb_mmu_fillunit_ecc_corrected_err_count_total_s()); + if (uncorrected_overflow) + uncorrected_delta += (0x1UL << fb_mmu_fillunit_ecc_uncorrected_err_count_total_s()); + + + g->ecc.fb.mmu_fillunit_corrected_err_count.counters[0] += + corrected_delta; + g->ecc.fb.mmu_fillunit_uncorrected_err_count.counters[0] += + uncorrected_delta; + + if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pte_data_m()) + nvgpu_log(g, gpu_dbg_intr, "corrected ecc pte data error"); + if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pte_data_m()) + nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pte data error"); + if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pde0_data_m()) + nvgpu_log(g, gpu_dbg_intr, "corrected ecc pde0 data error"); + if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pde0_data_m()) + nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pde0 data error"); + + if (corrected_overflow || uncorrected_overflow) + nvgpu_info(g, "mmu fillunit ecc counter overflow!"); + + nvgpu_log(g, gpu_dbg_intr, + "ecc error address: 0x%x", ecc_addr); + nvgpu_log(g, gpu_dbg_intr, + "ecc error count corrected: %d, uncorrected %d", + g->ecc.fb.mmu_fillunit_corrected_err_count.counters[0], + g->ecc.fb.mmu_fillunit_uncorrected_err_count.counters[0]); +} + +static void gv11b_fb_parse_mmfault(struct mmu_fault_info *mmfault) +{ + if (WARN_ON(mmfault->fault_type >= + ARRAY_SIZE(fault_type_descs_gv11b))) + mmfault->fault_type_desc = invalid_str; + else + mmfault->fault_type_desc = + fault_type_descs_gv11b[mmfault->fault_type]; + + if (WARN_ON(mmfault->client_type >= + ARRAY_SIZE(fault_client_type_descs_gv11b))) + mmfault->client_type_desc = invalid_str; + else + mmfault->client_type_desc = + fault_client_type_descs_gv11b[mmfault->client_type]; + + mmfault->client_id_desc = invalid_str; + if (mmfault->client_type == + gmmu_fault_client_type_hub_v()) { + + if (!(WARN_ON(mmfault->client_id >= + ARRAY_SIZE(hub_client_descs_gv11b)))) + mmfault->client_id_desc = + hub_client_descs_gv11b[mmfault->client_id]; + } else if (mmfault->client_type == + gmmu_fault_client_type_gpc_v()) { + if (!(WARN_ON(mmfault->client_id >= + ARRAY_SIZE(gpc_client_descs_gv11b)))) + mmfault->client_id_desc = + gpc_client_descs_gv11b[mmfault->client_id]; + } + +} + +static void gv11b_fb_print_fault_info(struct gk20a *g, + struct mmu_fault_info *mmfault) +{ + if (mmfault && mmfault->valid) { + nvgpu_err(g, "[MMU FAULT] " + "mmu engine id: %d, " + "ch id: %d, " + "fault addr: 0x%llx, " + "fault addr aperture: %d, " + "fault type: %s, " + "access type: %s, ", + mmfault->mmu_engine_id, + mmfault->chid, + mmfault->fault_addr, + mmfault->fault_addr_aperture, + mmfault->fault_type_desc, + fault_access_type_descs_gv11b[mmfault->access_type]); + nvgpu_err(g, "[MMU FAULT] " + "protected mode: %d, " + "client type: %s, " + "client id: %s, " + "gpc id if client type is gpc: %d, ", + mmfault->protected_mode, + mmfault->client_type_desc, + mmfault->client_id_desc, + mmfault->gpc_id); + + nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " + "faulted act eng id if any: 0x%x, " + "faulted veid if any: 0x%x, " + "faulted pbdma id if any: 0x%x, ", + mmfault->faulted_engine, + mmfault->faulted_subid, + mmfault->faulted_pbdma); + nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " + "inst ptr: 0x%llx, " + "inst ptr aperture: %d, " + "replayable fault: %d, " + "replayable fault en: %d " + "timestamp hi:lo 0x%08x:0x%08x, ", + mmfault->inst_ptr, + mmfault->inst_aperture, + mmfault->replayable_fault, + mmfault->replay_fault_en, + mmfault->timestamp_hi, mmfault->timestamp_lo); + } +} + +/* + *Fault buffer format + * + * 31 28 24 23 16 15 8 7 4 0 + *.-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. + *| inst_lo |0 0|apr|0 0 0 0 0 0 0 0| + *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' + *| inst_hi | + *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' + *| addr_31_12 | |AP | + *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' + *| addr_63_32 | + *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' + *| timestamp_lo | + *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' + *| timestamp_hi | + *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' + *| (reserved) | engine_id | + *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' + *|V|R|P| gpc_id |0 0 0|t|0|acctp|0| client |RF0 0|faulttype| + */ + +static void gv11b_fb_copy_from_hw_fault_buf(struct gk20a *g, + struct nvgpu_mem *mem, u32 offset, struct mmu_fault_info *mmfault) +{ + u32 rd32_val; + u32 addr_lo, addr_hi; + u64 inst_ptr; + u32 chid = FIFO_INVAL_CHANNEL_ID; + struct channel_gk20a *refch; + + memset(mmfault, 0, sizeof(*mmfault)); + + rd32_val = nvgpu_mem_rd32(g, mem, offset + + gmmu_fault_buf_entry_inst_lo_w()); + addr_lo = gmmu_fault_buf_entry_inst_lo_v(rd32_val); + addr_lo = addr_lo << ram_in_base_shift_v(); + + addr_hi = nvgpu_mem_rd32(g, mem, offset + + gmmu_fault_buf_entry_inst_hi_w()); + addr_hi = gmmu_fault_buf_entry_inst_hi_v(addr_hi); + + inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); + + /* refch will be put back after fault is handled */ + refch = gk20a_refch_from_inst_ptr(g, inst_ptr); + if (refch) + chid = refch->chid; + + /* it is ok to continue even if refch is NULL */ + mmfault->refch = refch; + mmfault->chid = chid; + mmfault->inst_ptr = inst_ptr; + mmfault->inst_aperture = gmmu_fault_buf_entry_inst_aperture_v(rd32_val); + + rd32_val = nvgpu_mem_rd32(g, mem, offset + + gmmu_fault_buf_entry_addr_lo_w()); + + mmfault->fault_addr_aperture = + gmmu_fault_buf_entry_addr_phys_aperture_v(rd32_val); + addr_lo = gmmu_fault_buf_entry_addr_lo_v(rd32_val); + addr_lo = addr_lo << ram_in_base_shift_v(); + + rd32_val = nvgpu_mem_rd32(g, mem, offset + + gmmu_fault_buf_entry_addr_hi_w()); + addr_hi = gmmu_fault_buf_entry_addr_hi_v(rd32_val); + mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); + + rd32_val = nvgpu_mem_rd32(g, mem, offset + + gmmu_fault_buf_entry_timestamp_lo_w()); + mmfault->timestamp_lo = + gmmu_fault_buf_entry_timestamp_lo_v(rd32_val); + + rd32_val = nvgpu_mem_rd32(g, mem, offset + + gmmu_fault_buf_entry_timestamp_hi_w()); + mmfault->timestamp_hi = + gmmu_fault_buf_entry_timestamp_hi_v(rd32_val); + + rd32_val = nvgpu_mem_rd32(g, mem, offset + + gmmu_fault_buf_entry_engine_id_w()); + + mmfault->mmu_engine_id = + gmmu_fault_buf_entry_engine_id_v(rd32_val); + gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, + &mmfault->faulted_engine, &mmfault->faulted_subid, + &mmfault->faulted_pbdma); + + rd32_val = nvgpu_mem_rd32(g, mem, offset + + gmmu_fault_buf_entry_fault_type_w()); + mmfault->client_id = + gmmu_fault_buf_entry_client_v(rd32_val); + mmfault->replayable_fault = + gmmu_fault_buf_entry_replayable_fault_v(rd32_val); + + mmfault->fault_type = + gmmu_fault_buf_entry_fault_type_v(rd32_val); + mmfault->access_type = + gmmu_fault_buf_entry_access_type_v(rd32_val); + + mmfault->client_type = + gmmu_fault_buf_entry_mmu_client_type_v(rd32_val); + + mmfault->gpc_id = + gmmu_fault_buf_entry_gpc_id_v(rd32_val); + mmfault->protected_mode = + gmmu_fault_buf_entry_protected_mode_v(rd32_val); + + mmfault->replay_fault_en = + gmmu_fault_buf_entry_replayable_fault_en_v(rd32_val); + + mmfault->valid = gmmu_fault_buf_entry_valid_v(rd32_val); + + rd32_val = nvgpu_mem_rd32(g, mem, offset + + gmmu_fault_buf_entry_fault_type_w()); + rd32_val &= ~(gmmu_fault_buf_entry_valid_m()); + nvgpu_mem_wr32(g, mem, offset + gmmu_fault_buf_entry_valid_w(), + rd32_val); + + gv11b_fb_parse_mmfault(mmfault); +} + +static void gv11b_fb_handle_mmu_fault_common(struct gk20a *g, + struct mmu_fault_info *mmfault, u32 *invalidate_replay_val) +{ + unsigned int id_type; + u32 num_lce, act_eng_bitmask = 0; + int err = 0; + u32 id = ((u32)~0); + + if (!mmfault->valid) + return; + + gv11b_fb_print_fault_info(g, mmfault); + + num_lce = gv11b_ce_get_num_lce(g); + if ((mmfault->mmu_engine_id >= + gmmu_fault_mmu_eng_id_ce0_v()) && + (mmfault->mmu_engine_id < + gmmu_fault_mmu_eng_id_ce0_v() + num_lce)) { + /* CE page faults are not reported as replayable */ + nvgpu_log(g, gpu_dbg_intr, "CE Faulted"); + err = gv11b_fb_fix_page_fault(g, mmfault); + gv11b_fifo_reset_pbdma_and_eng_faulted(g, mmfault->refch, + mmfault->faulted_pbdma, mmfault->faulted_engine); + if (!err) { + nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Fixed"); + *invalidate_replay_val = 0; + /* refch in mmfault is assigned at the time of copying + * fault info from snap reg or bar2 fault buf + */ + gk20a_channel_put(mmfault->refch); + return; + } + /* Do recovery. Channel recovery needs refch */ + nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Not Fixed"); + } + + if (!mmfault->replayable_fault) { + if (mmfault->fault_type == + gmmu_fault_type_unbound_inst_block_v()) { + /* + * Bug 1847172: When an engine faults due to an unbound + * instance block, the fault cannot be isolated to a + * single context so we need to reset the entire runlist + */ + id_type = ID_TYPE_UNKNOWN; + + } else if (mmfault->refch) { + if (gk20a_is_channel_marked_as_tsg(mmfault->refch)) { + id = mmfault->refch->tsgid; + id_type = ID_TYPE_TSG; + } else { + id = mmfault->chid; + id_type = ID_TYPE_CHANNEL; + } + if (mmfault->refch->mmu_nack_handled) { + /* We have already recovered for the same + * context, skip doing another recovery. + */ + mmfault->refch->mmu_nack_handled = false; + /* + * Recovery path can be entered twice for the + * same error in case of mmu nack. If mmu + * nack interrupt is handled before mmu fault + * then channel reference is increased to avoid + * closing the channel by userspace. Decrement + * channel reference. + */ + gk20a_channel_put(mmfault->refch); + /* refch in mmfault is assigned at the time + * of copying fault info from snap reg or bar2 + * fault buf. + */ + gk20a_channel_put(mmfault->refch); + return; + } + } else { + id_type = ID_TYPE_UNKNOWN; + } + if (mmfault->faulted_engine != FIFO_INVAL_ENGINE_ID) + act_eng_bitmask = BIT(mmfault->faulted_engine); + + /* Indicate recovery is handled if mmu fault is a result of + * mmu nack. + */ + mmfault->refch->mmu_nack_handled = true; + g->ops.fifo.teardown_ch_tsg(g, act_eng_bitmask, + id, id_type, RC_TYPE_MMU_FAULT, mmfault); + } else { + if (mmfault->fault_type == gmmu_fault_type_pte_v()) { + nvgpu_log(g, gpu_dbg_intr, "invalid pte! try to fix"); + err = gv11b_fb_fix_page_fault(g, mmfault); + if (err) + *invalidate_replay_val |= + fb_mmu_invalidate_replay_cancel_global_f(); + else + *invalidate_replay_val |= + fb_mmu_invalidate_replay_start_ack_all_f(); + } else { + /* cancel faults other than invalid pte */ + *invalidate_replay_val |= + fb_mmu_invalidate_replay_cancel_global_f(); + } + /* refch in mmfault is assigned at the time of copying + * fault info from snap reg or bar2 fault buf + */ + gk20a_channel_put(mmfault->refch); + } +} + +static int gv11b_fb_replay_or_cancel_faults(struct gk20a *g, + u32 invalidate_replay_val) +{ + int err = 0; + + nvgpu_log_fn(g, " "); + + if (invalidate_replay_val & + fb_mmu_invalidate_replay_cancel_global_f()) { + /* + * cancel faults so that next time it faults as + * replayable faults and channel recovery can be done + */ + err = g->ops.fb.mmu_invalidate_replay(g, + fb_mmu_invalidate_replay_cancel_global_f()); + } else if (invalidate_replay_val & + fb_mmu_invalidate_replay_start_ack_all_f()) { + /* pte valid is fixed. replay faulting request */ + err = g->ops.fb.mmu_invalidate_replay(g, + fb_mmu_invalidate_replay_start_ack_all_f()); + } + + return err; +} + +void gv11b_fb_handle_mmu_nonreplay_replay_fault(struct gk20a *g, + u32 fault_status, u32 index) +{ + u32 get_indx, offset, rd32_val, entries; + struct nvgpu_mem *mem; + struct mmu_fault_info *mmfault; + u32 invalidate_replay_val = 0; + u64 prev_fault_addr = 0ULL; + u64 next_fault_addr = 0ULL; + + if (gv11b_fb_is_fault_buffer_empty(g, index, &get_indx)) { + nvgpu_log(g, gpu_dbg_intr, + "SPURIOUS mmu fault: reg index:%d", index); + return; + } + nvgpu_log(g, gpu_dbg_intr, "%s MMU FAULT" , + index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX ? + "REPLAY" : "NON-REPLAY"); + + nvgpu_log(g, gpu_dbg_intr, "get ptr = %d", get_indx); + + mem = &g->mm.hw_fault_buf[index]; + mmfault = &g->mm.fault_info[index]; + + entries = gv11b_fb_fault_buffer_size_val(g, index); + nvgpu_log(g, gpu_dbg_intr, "buffer num entries = %d", entries); + + offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); + nvgpu_log(g, gpu_dbg_intr, "starting word offset = 0x%x", offset); + + rd32_val = nvgpu_mem_rd32(g, mem, + offset + gmmu_fault_buf_entry_valid_w()); + nvgpu_log(g, gpu_dbg_intr, "entry valid offset val = 0x%x", rd32_val); + + while ((rd32_val & gmmu_fault_buf_entry_valid_m())) { + + nvgpu_log(g, gpu_dbg_intr, "entry valid = 0x%x", rd32_val); + + gv11b_fb_copy_from_hw_fault_buf(g, mem, offset, mmfault); + + get_indx = (get_indx + 1) % entries; + nvgpu_log(g, gpu_dbg_intr, "new get index = %d", get_indx); + + gv11b_fb_fault_buffer_get_ptr_update(g, index, get_indx); + + offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); + nvgpu_log(g, gpu_dbg_intr, "next word offset = 0x%x", offset); + + rd32_val = nvgpu_mem_rd32(g, mem, + offset + gmmu_fault_buf_entry_valid_w()); + + if (index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX && + mmfault->fault_addr != 0ULL) { + /* fault_addr "0" is not supposed to be fixed ever. + * For the first time when prev = 0, next = 0 and + * fault addr is also 0 then handle_mmu_fault_common will + * not be called. Fix by checking fault_addr not equal to 0 + */ + prev_fault_addr = next_fault_addr; + next_fault_addr = mmfault->fault_addr; + if (prev_fault_addr == next_fault_addr) { + nvgpu_log(g, gpu_dbg_intr, "pte already scanned"); + if (mmfault->refch) + gk20a_channel_put(mmfault->refch); + continue; + } + } + + gv11b_fb_handle_mmu_fault_common(g, mmfault, + &invalidate_replay_val); + + } + if (index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX && + invalidate_replay_val != 0U) + gv11b_fb_replay_or_cancel_faults(g, invalidate_replay_val); +} + +static void gv11b_mm_copy_from_fault_snap_reg(struct gk20a *g, + u32 fault_status, struct mmu_fault_info *mmfault) +{ + u32 reg_val; + u32 addr_lo, addr_hi; + u64 inst_ptr; + int chid = FIFO_INVAL_CHANNEL_ID; + struct channel_gk20a *refch; + + memset(mmfault, 0, sizeof(*mmfault)); + + if (!(fault_status & fb_mmu_fault_status_valid_set_f())) { + + nvgpu_log(g, gpu_dbg_intr, "mmu fault status valid not set"); + return; + } + + g->ops.fb.read_mmu_fault_inst_lo_hi(g, ®_val, &addr_hi); + + addr_lo = fb_mmu_fault_inst_lo_addr_v(reg_val); + addr_lo = addr_lo << ram_in_base_shift_v(); + + addr_hi = fb_mmu_fault_inst_hi_addr_v(addr_hi); + inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); + + /* refch will be put back after fault is handled */ + refch = gk20a_refch_from_inst_ptr(g, inst_ptr); + if (refch) + chid = refch->chid; + + /* It is still ok to continue if refch is NULL */ + mmfault->refch = refch; + mmfault->chid = chid; + mmfault->inst_ptr = inst_ptr; + mmfault->inst_aperture = fb_mmu_fault_inst_lo_aperture_v(reg_val); + mmfault->mmu_engine_id = fb_mmu_fault_inst_lo_engine_id_v(reg_val); + + gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, + &mmfault->faulted_engine, &mmfault->faulted_subid, + &mmfault->faulted_pbdma); + + g->ops.fb.read_mmu_fault_addr_lo_hi(g, ®_val, &addr_hi); + + addr_lo = fb_mmu_fault_addr_lo_addr_v(reg_val); + addr_lo = addr_lo << ram_in_base_shift_v(); + + mmfault->fault_addr_aperture = + fb_mmu_fault_addr_lo_phys_aperture_v(reg_val); + + addr_hi = fb_mmu_fault_addr_hi_addr_v(addr_hi); + mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); + + reg_val = g->ops.fb.read_mmu_fault_info(g); + mmfault->fault_type = fb_mmu_fault_info_fault_type_v(reg_val); + mmfault->replayable_fault = + fb_mmu_fault_info_replayable_fault_v(reg_val); + mmfault->client_id = fb_mmu_fault_info_client_v(reg_val); + mmfault->access_type = fb_mmu_fault_info_access_type_v(reg_val); + mmfault->client_type = fb_mmu_fault_info_client_type_v(reg_val); + mmfault->gpc_id = fb_mmu_fault_info_gpc_id_v(reg_val); + mmfault->protected_mode = + fb_mmu_fault_info_protected_mode_v(reg_val); + mmfault->replay_fault_en = + fb_mmu_fault_info_replayable_fault_en_v(reg_val); + + mmfault->valid = fb_mmu_fault_info_valid_v(reg_val); + + fault_status &= ~(fb_mmu_fault_status_valid_m()); + g->ops.fb.write_mmu_fault_status(g, fault_status); + + gv11b_fb_parse_mmfault(mmfault); + +} + +void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g, + u32 fault_status) +{ + u32 reg_val; + u32 index = NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX; + + reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); + + if (fault_status & + fb_mmu_fault_status_replayable_getptr_corrupted_m()) { + + nvgpu_err(g, "replayable getptr corrupted set"); + + gv11b_fb_fault_buf_configure_hw(g, index); + + reg_val = set_field(reg_val, + fb_mmu_fault_buffer_get_getptr_corrupted_m(), + fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); + } + + if (fault_status & + fb_mmu_fault_status_replayable_overflow_m()) { + bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); + + nvgpu_err(g, "replayable overflow: buffer full:%s", + buffer_full?"true":"false"); + + reg_val = set_field(reg_val, + fb_mmu_fault_buffer_get_overflow_m(), + fb_mmu_fault_buffer_get_overflow_clear_f()); + } + + g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); +} + +void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g, + u32 fault_status) +{ + u32 reg_val; + u32 index = NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX; + + reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); + + if (fault_status & + fb_mmu_fault_status_non_replayable_getptr_corrupted_m()) { + + nvgpu_err(g, "non replayable getptr corrupted set"); + + gv11b_fb_fault_buf_configure_hw(g, index); + + reg_val = set_field(reg_val, + fb_mmu_fault_buffer_get_getptr_corrupted_m(), + fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); + } + + if (fault_status & + fb_mmu_fault_status_non_replayable_overflow_m()) { + + bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); + + nvgpu_err(g, "non replayable overflow: buffer full:%s", + buffer_full?"true":"false"); + + reg_val = set_field(reg_val, + fb_mmu_fault_buffer_get_overflow_m(), + fb_mmu_fault_buffer_get_overflow_clear_f()); + } + + g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); +} + +static void gv11b_fb_handle_bar2_fault(struct gk20a *g, + struct mmu_fault_info *mmfault, u32 fault_status) +{ + if (fault_status & fb_mmu_fault_status_non_replayable_error_m()) { + if (gv11b_fb_is_fault_buf_enabled(g, + NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) + gv11b_fb_fault_buf_configure_hw(g, NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX); + } + + if (fault_status & fb_mmu_fault_status_replayable_error_m()) { + if (gv11b_fb_is_fault_buf_enabled(g, + NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX)) + gv11b_fb_fault_buf_configure_hw(g, + NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); + } + gv11b_ce_mthd_buffer_fault_in_bar2_fault(g); + + g->ops.bus.bar2_bind(g, &g->mm.bar2.inst_block); + + if (mmfault->refch) { + gk20a_channel_put(mmfault->refch); + mmfault->refch = NULL; + } +} + +void gv11b_fb_handle_other_fault_notify(struct gk20a *g, + u32 fault_status) +{ + struct mmu_fault_info *mmfault; + u32 invalidate_replay_val = 0; + + mmfault = &g->mm.fault_info[NVGPU_MM_MMU_FAULT_TYPE_OTHER_AND_NONREPLAY]; + + gv11b_mm_copy_from_fault_snap_reg(g, fault_status, mmfault); + + /* BAR2/Physical faults will not be snapped in hw fault buf */ + if (mmfault->mmu_engine_id == gmmu_fault_mmu_eng_id_bar2_v()) { + nvgpu_err(g, "BAR2 MMU FAULT"); + gv11b_fb_handle_bar2_fault(g, mmfault, fault_status); + + } else if (mmfault->mmu_engine_id == + gmmu_fault_mmu_eng_id_physical_v()) { + /* usually means VPR or out of bounds physical accesses */ + nvgpu_err(g, "PHYSICAL MMU FAULT"); + + } else { + gv11b_fb_handle_mmu_fault_common(g, mmfault, + &invalidate_replay_val); + + if (invalidate_replay_val) + gv11b_fb_replay_or_cancel_faults(g, + invalidate_replay_val); + } +} + +void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status) +{ + u32 dropped_faults = 0; + + dropped_faults = fb_mmu_fault_status_dropped_bar1_phys_set_f() | + fb_mmu_fault_status_dropped_bar1_virt_set_f() | + fb_mmu_fault_status_dropped_bar2_phys_set_f() | + fb_mmu_fault_status_dropped_bar2_virt_set_f() | + fb_mmu_fault_status_dropped_ifb_phys_set_f() | + fb_mmu_fault_status_dropped_ifb_virt_set_f() | + fb_mmu_fault_status_dropped_other_phys_set_f()| + fb_mmu_fault_status_dropped_other_virt_set_f(); + + if (fault_status & dropped_faults) { + nvgpu_err(g, "dropped mmu fault (0x%08x)", + fault_status & dropped_faults); + g->ops.fb.write_mmu_fault_status(g, dropped_faults); + } +} + +void gv11b_fb_handle_replayable_mmu_fault(struct gk20a *g) +{ + u32 fault_status = gk20a_readl(g, fb_mmu_fault_status_r()); + + if (!(fault_status & fb_mmu_fault_status_replayable_m())) + return; + + if (gv11b_fb_is_fault_buf_enabled(g, + NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) { + gv11b_fb_handle_mmu_nonreplay_replay_fault(g, + fault_status, + NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); + } +} + +static void gv11b_fb_handle_mmu_fault(struct gk20a *g, u32 niso_intr) +{ + u32 fault_status = g->ops.fb.read_mmu_fault_status(g); + + nvgpu_log(g, gpu_dbg_intr, "mmu_fault_status = 0x%08x", fault_status); + + if (niso_intr & + fb_niso_intr_mmu_other_fault_notify_m()) { + + gv11b_fb_handle_dropped_mmu_fault(g, fault_status); + + gv11b_fb_handle_other_fault_notify(g, fault_status); + } + + if (gv11b_fb_is_fault_buf_enabled(g, NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) { + + if (niso_intr & + fb_niso_intr_mmu_nonreplayable_fault_notify_m()) { + + gv11b_fb_handle_mmu_nonreplay_replay_fault(g, + fault_status, + NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX); + + /* + * When all the faults are processed, + * GET and PUT will have same value and mmu fault status + * bit will be reset by HW + */ + } + if (niso_intr & + fb_niso_intr_mmu_nonreplayable_fault_overflow_m()) { + + gv11b_fb_handle_nonreplay_fault_overflow(g, + fault_status); + } + + } + + if (gv11b_fb_is_fault_buf_enabled(g, NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX)) { + + if (niso_intr & + fb_niso_intr_mmu_replayable_fault_notify_m()) { + + gv11b_fb_handle_mmu_nonreplay_replay_fault(g, + fault_status, + NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); + } + if (niso_intr & + fb_niso_intr_mmu_replayable_fault_overflow_m()) { + + gv11b_fb_handle_replay_fault_overflow(g, + fault_status); + } + + } + + nvgpu_log(g, gpu_dbg_intr, "clear mmu fault status"); + g->ops.fb.write_mmu_fault_status(g, + fb_mmu_fault_status_valid_clear_f()); +} + +void gv11b_fb_hub_isr(struct gk20a *g) +{ + u32 status, niso_intr; + + nvgpu_mutex_acquire(&g->mm.hub_isr_mutex); + + niso_intr = gk20a_readl(g, fb_niso_intr_r()); + + nvgpu_log(g, gpu_dbg_intr, "enter hub isr, niso_intr = 0x%08x", + niso_intr); + + if (niso_intr & + (fb_niso_intr_hub_access_counter_notify_m() | + fb_niso_intr_hub_access_counter_error_m())) { + + nvgpu_info(g, "hub access counter notify/error"); + } + if (niso_intr & + fb_niso_intr_mmu_ecc_uncorrected_error_notify_pending_f()) { + + nvgpu_info(g, "ecc uncorrected error notify"); + + status = gk20a_readl(g, fb_mmu_l2tlb_ecc_status_r()); + if (status) + gv11b_handle_l2tlb_ecc_isr(g, status); + + status = gk20a_readl(g, fb_mmu_hubtlb_ecc_status_r()); + if (status) + gv11b_handle_hubtlb_ecc_isr(g, status); + + status = gk20a_readl(g, fb_mmu_fillunit_ecc_status_r()); + if (status) + gv11b_handle_fillunit_ecc_isr(g, status); + } + if (niso_intr & + (fb_niso_intr_mmu_other_fault_notify_m() | + fb_niso_intr_mmu_replayable_fault_notify_m() | + fb_niso_intr_mmu_replayable_fault_overflow_m() | + fb_niso_intr_mmu_nonreplayable_fault_notify_m() | + fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) { + + nvgpu_log(g, gpu_dbg_intr, "MMU Fault"); + gv11b_fb_handle_mmu_fault(g, niso_intr); + } + + nvgpu_mutex_release(&g->mm.hub_isr_mutex); +} + +bool gv11b_fb_mmu_fault_pending(struct gk20a *g) +{ + if (gk20a_readl(g, fb_niso_intr_r()) & + (fb_niso_intr_mmu_other_fault_notify_m() | + fb_niso_intr_mmu_ecc_uncorrected_error_notify_m() | + fb_niso_intr_mmu_replayable_fault_notify_m() | + fb_niso_intr_mmu_replayable_fault_overflow_m() | + fb_niso_intr_mmu_nonreplayable_fault_notify_m() | + fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) + return true; + + return false; +} + +int gv11b_fb_mmu_invalidate_replay(struct gk20a *g, + u32 invalidate_replay_val) +{ + int err = -ETIMEDOUT; + u32 reg_val; + struct nvgpu_timeout timeout; + + nvgpu_log_fn(g, " "); + + nvgpu_mutex_acquire(&g->mm.tlb_lock); + + reg_val = gk20a_readl(g, fb_mmu_invalidate_r()); + + reg_val |= fb_mmu_invalidate_all_va_true_f() | + fb_mmu_invalidate_all_pdb_true_f() | + invalidate_replay_val | + fb_mmu_invalidate_trigger_true_f(); + + gk20a_writel(g, fb_mmu_invalidate_r(), reg_val); + + /* retry 200 times */ + nvgpu_timeout_init(g, &timeout, 200, NVGPU_TIMER_RETRY_TIMER); + do { + reg_val = gk20a_readl(g, fb_mmu_ctrl_r()); + if (fb_mmu_ctrl_pri_fifo_empty_v(reg_val) != + fb_mmu_ctrl_pri_fifo_empty_false_f()) { + err = 0; + break; + } + nvgpu_udelay(5); + } while (!nvgpu_timeout_expired_msg(&timeout, + "invalidate replay failed on 0x%llx")); + if (err) + nvgpu_err(g, "invalidate replay timedout"); + + nvgpu_mutex_release(&g->mm.tlb_lock); + + return err; +} + +static int gv11b_fb_fix_page_fault(struct gk20a *g, + struct mmu_fault_info *mmfault) +{ + int err = 0; + u32 pte[2]; + + if (mmfault->refch == NULL) { + nvgpu_log(g, gpu_dbg_intr, "refch from mmu_fault_info is NULL"); + return -EINVAL; + } + + err = __nvgpu_get_pte(g, + mmfault->refch->vm, mmfault->fault_addr, &pte[0]); + if (err) { + nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not found"); + return err; + } + nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, + "pte: %#08x %#08x", pte[1], pte[0]); + + if (pte[0] == 0x0 && pte[1] == 0x0) { + nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, + "pte all zeros, do not set valid"); + return -1; + } + if (pte[0] & gmmu_new_pte_valid_true_f()) { + nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, + "pte valid already set"); + return -1; + } + + pte[0] |= gmmu_new_pte_valid_true_f(); + if (pte[0] & gmmu_new_pte_read_only_true_f()) + pte[0] &= ~(gmmu_new_pte_read_only_true_f()); + nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, + "new pte: %#08x %#08x", pte[1], pte[0]); + + err = __nvgpu_set_pte(g, + mmfault->refch->vm, mmfault->fault_addr, &pte[0]); + if (err) { + nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not fixed"); + return err; + } + /* invalidate tlb so that GMMU does not use old cached translation */ + g->ops.fb.tlb_invalidate(g, mmfault->refch->vm->pdb.mem); + + err = __nvgpu_get_pte(g, + mmfault->refch->vm, mmfault->fault_addr, &pte[0]); + nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, + "pte after tlb invalidate: %#08x %#08x", + pte[1], pte[0]); + return err; +} + +void fb_gv11b_write_mmu_fault_buffer_lo_hi(struct gk20a *g, u32 index, + u32 addr_lo, u32 addr_hi) +{ + nvgpu_writel(g, fb_mmu_fault_buffer_lo_r(index), addr_lo); + nvgpu_writel(g, fb_mmu_fault_buffer_hi_r(index), addr_hi); +} + +u32 fb_gv11b_read_mmu_fault_buffer_get(struct gk20a *g, u32 index) +{ + return nvgpu_readl(g, fb_mmu_fault_buffer_get_r(index)); +} + +void fb_gv11b_write_mmu_fault_buffer_get(struct gk20a *g, u32 index, + u32 reg_val) +{ + nvgpu_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val); +} + +u32 fb_gv11b_read_mmu_fault_buffer_put(struct gk20a *g, u32 index) +{ + return nvgpu_readl(g, fb_mmu_fault_buffer_put_r(index)); +} + +u32 fb_gv11b_read_mmu_fault_buffer_size(struct gk20a *g, u32 index) +{ + return nvgpu_readl(g, fb_mmu_fault_buffer_size_r(index)); +} + +void fb_gv11b_write_mmu_fault_buffer_size(struct gk20a *g, u32 index, + u32 reg_val) +{ + nvgpu_writel(g, fb_mmu_fault_buffer_size_r(index), reg_val); +} + +void fb_gv11b_read_mmu_fault_addr_lo_hi(struct gk20a *g, + u32 *addr_lo, u32 *addr_hi) +{ + *addr_lo = nvgpu_readl(g, fb_mmu_fault_addr_lo_r()); + *addr_hi = nvgpu_readl(g, fb_mmu_fault_addr_hi_r()); +} + +void fb_gv11b_read_mmu_fault_inst_lo_hi(struct gk20a *g, + u32 *inst_lo, u32 *inst_hi) +{ + *inst_lo = nvgpu_readl(g, fb_mmu_fault_inst_lo_r()); + *inst_hi = nvgpu_readl(g, fb_mmu_fault_inst_hi_r()); +} + +u32 fb_gv11b_read_mmu_fault_info(struct gk20a *g) +{ + return nvgpu_readl(g, fb_mmu_fault_info_r()); +} + +u32 fb_gv11b_read_mmu_fault_status(struct gk20a *g) +{ + return nvgpu_readl(g, fb_mmu_fault_status_r()); +} + +void fb_gv11b_write_mmu_fault_status(struct gk20a *g, u32 reg_val) +{ + nvgpu_writel(g, fb_mmu_fault_status_r(), reg_val); +} diff --git a/drivers/gpu/nvgpu/common/fb/fb_gv11b.h b/drivers/gpu/nvgpu/common/fb/fb_gv11b.h new file mode 100644 index 00000000..71fb3a41 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gv11b.h @@ -0,0 +1,82 @@ +/* + * GV11B FB + * + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NVGPU_GV11B_FB +#define _NVGPU_GV11B_FB + +#define NONREPLAY_REG_INDEX 0 +#define REPLAY_REG_INDEX 1 + +struct gk20a; +struct gr_gk20a; + +void gv11b_fb_init_hw(struct gk20a *g); + +void gv11b_fb_init_fs_state(struct gk20a *g); +void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr); +void gv11b_fb_reset(struct gk20a *g); +void gv11b_fb_hub_isr(struct gk20a *g); + +bool gv11b_fb_is_fault_buf_enabled(struct gk20a *g, u32 index ); +void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g, + u32 index, u32 state); +void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, u32 index); +void gv11b_fb_enable_hub_intr(struct gk20a *g); +void gv11b_fb_disable_hub_intr(struct gk20a *g); +bool gv11b_fb_mmu_fault_pending(struct gk20a *g); +void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status); +void gv11b_fb_handle_other_fault_notify(struct gk20a *g, + u32 fault_status); +void gv11b_fb_handle_mmu_nonreplay_replay_fault(struct gk20a *g, + u32 fault_status, u32 index); +void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g, + u32 fault_status); +void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g, + u32 fault_status); +void gv11b_fb_handle_replayable_mmu_fault(struct gk20a *g); +void gv11b_handle_l2tlb_ecc_isr(struct gk20a *g, u32 ecc_status); +void gv11b_handle_hubtlb_ecc_isr(struct gk20a *g, u32 ecc_status); +void gv11b_handle_fillunit_ecc_isr(struct gk20a *g, u32 ecc_status); + +void fb_gv11b_write_mmu_fault_buffer_lo_hi(struct gk20a *g, u32 index, + u32 addr_lo, u32 addr_hi); +u32 fb_gv11b_read_mmu_fault_buffer_get(struct gk20a *g, u32 index); +void fb_gv11b_write_mmu_fault_buffer_get(struct gk20a *g, u32 index, + u32 reg_val); +u32 fb_gv11b_read_mmu_fault_buffer_put(struct gk20a *g, u32 index); +u32 fb_gv11b_read_mmu_fault_buffer_size(struct gk20a *g, u32 index); +void fb_gv11b_write_mmu_fault_buffer_size(struct gk20a *g, u32 index, + u32 reg_val); +void fb_gv11b_read_mmu_fault_addr_lo_hi(struct gk20a *g, + u32 *addr_lo, u32 *addr_hi); +void fb_gv11b_read_mmu_fault_inst_lo_hi(struct gk20a *g, + u32 *inst_lo, u32 *inst_hi); +u32 fb_gv11b_read_mmu_fault_info(struct gk20a *g); +u32 fb_gv11b_read_mmu_fault_status(struct gk20a *g); +void fb_gv11b_write_mmu_fault_status(struct gk20a *g, u32 reg_val); + +int gv11b_fb_mmu_invalidate_replay(struct gk20a *g, + u32 invalidate_replay_val); + +#endif diff --git a/drivers/gpu/nvgpu/gk20a/fb_gk20a.c b/drivers/gpu/nvgpu/gk20a/fb_gk20a.c deleted file mode 100644 index c4be3313..00000000 --- a/drivers/gpu/nvgpu/gk20a/fb_gk20a.c +++ /dev/null @@ -1,122 +0,0 @@ -/* - * GK20A memory interface - * - * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include - -#include "gk20a.h" -#include "fb_gk20a.h" - -#include - -#include -#include - -void fb_gk20a_reset(struct gk20a *g) -{ - u32 val; - - nvgpu_log_info(g, "reset gk20a fb"); - - g->ops.mc.reset(g, mc_enable_pfb_enabled_f() | - mc_enable_l2_enabled_f() | - mc_enable_xbar_enabled_f() | - mc_enable_hub_enabled_f()); - - val = gk20a_readl(g, mc_elpg_enable_r()); - val |= mc_elpg_enable_xbar_enabled_f() - | mc_elpg_enable_pfb_enabled_f() - | mc_elpg_enable_hub_enabled_f(); - gk20a_writel(g, mc_elpg_enable_r(), val); -} - -void gk20a_fb_init_hw(struct gk20a *g) -{ - u32 addr = nvgpu_mem_get_addr(g, &g->mm.sysmem_flush) >> 8; - - gk20a_writel(g, fb_niso_flush_sysmem_addr_r(), addr); -} - -void gk20a_fb_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb) -{ - struct nvgpu_timeout timeout; - u32 addr_lo; - u32 data; - - nvgpu_log_fn(g, " "); - - /* pagetables are considered sw states which are preserved after - prepare_poweroff. When gk20a deinit releases those pagetables, - common code in vm unmap path calls tlb invalidate that touches - hw. Use the power_on flag to skip tlb invalidation when gpu - power is turned off */ - - if (!g->power_on) - return; - - addr_lo = u64_lo32(nvgpu_mem_get_addr(g, pdb) >> 12); - - nvgpu_mutex_acquire(&g->mm.tlb_lock); - - trace_gk20a_mm_tlb_invalidate(g->name); - - nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER); - - do { - data = gk20a_readl(g, fb_mmu_ctrl_r()); - if (fb_mmu_ctrl_pri_fifo_space_v(data) != 0) - break; - nvgpu_udelay(2); - } while (!nvgpu_timeout_expired_msg(&timeout, - "wait mmu fifo space")); - - if (nvgpu_timeout_peek_expired(&timeout)) - goto out; - - nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER); - - gk20a_writel(g, fb_mmu_invalidate_pdb_r(), - fb_mmu_invalidate_pdb_addr_f(addr_lo) | - nvgpu_aperture_mask(g, pdb, - fb_mmu_invalidate_pdb_aperture_sys_mem_f(), - fb_mmu_invalidate_pdb_aperture_sys_mem_f(), - fb_mmu_invalidate_pdb_aperture_vid_mem_f())); - - gk20a_writel(g, fb_mmu_invalidate_r(), - fb_mmu_invalidate_all_va_true_f() | - fb_mmu_invalidate_trigger_true_f()); - - do { - data = gk20a_readl(g, fb_mmu_ctrl_r()); - if (fb_mmu_ctrl_pri_fifo_empty_v(data) != - fb_mmu_ctrl_pri_fifo_empty_false_f()) - break; - nvgpu_udelay(2); - } while (!nvgpu_timeout_expired_msg(&timeout, - "wait mmu invalidate")); - - trace_gk20a_mm_tlb_invalidate_done(g->name); - -out: - nvgpu_mutex_release(&g->mm.tlb_lock); -} diff --git a/drivers/gpu/nvgpu/gk20a/fb_gk20a.h b/drivers/gpu/nvgpu/gk20a/fb_gk20a.h deleted file mode 100644 index 072c9027..00000000 --- a/drivers/gpu/nvgpu/gk20a/fb_gk20a.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#ifndef FB_GK20A_H -#define FB_GK20A_H - -struct gk20a; -struct nvgpu_mem; - -void fb_gk20a_reset(struct gk20a *g); -void gk20a_fb_init_hw(struct gk20a *g); -void gk20a_fb_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb); - -#endif diff --git a/drivers/gpu/nvgpu/gm20b/fb_gm20b.c b/drivers/gpu/nvgpu/gm20b/fb_gm20b.c deleted file mode 100644 index 37a7944b..00000000 --- a/drivers/gpu/nvgpu/gm20b/fb_gm20b.c +++ /dev/null @@ -1,228 +0,0 @@ -/* - * GM20B GPC MMU - * - * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. -* - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include - -#include "gk20a/gk20a.h" -#include "gk20a/fb_gk20a.h" -#include "gm20b/fb_gm20b.h" - -#include -#include -#include -#include - -#define VPR_INFO_FETCH_WAIT (5) -#define WPR_INFO_ADDR_ALIGNMENT 0x0000000c - -void fb_gm20b_init_fs_state(struct gk20a *g) -{ - nvgpu_log_info(g, "initialize gm20b fb"); - - gk20a_writel(g, fb_fbhub_num_active_ltcs_r(), - g->ltc_count); - - if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) { - /* Bypass MMU check for non-secure boot. For - * secure-boot,this register write has no-effect */ - gk20a_writel(g, fb_priv_mmu_phy_secure_r(), 0xffffffffU); - } -} - -void gm20b_fb_set_mmu_page_size(struct gk20a *g) -{ - /* set large page size in fb */ - u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r()); - fb_mmu_ctrl |= fb_mmu_ctrl_use_pdb_big_page_size_true_f(); - gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl); -} - -bool gm20b_fb_set_use_full_comp_tag_line(struct gk20a *g) -{ - /* set large page size in fb */ - u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r()); - fb_mmu_ctrl |= fb_mmu_ctrl_use_full_comp_tag_line_true_f(); - gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl); - - return true; -} - -u32 gm20b_fb_mmu_ctrl(struct gk20a *g) -{ - return gk20a_readl(g, fb_mmu_ctrl_r()); -} - -u32 gm20b_fb_mmu_debug_ctrl(struct gk20a *g) -{ - return gk20a_readl(g, fb_mmu_debug_ctrl_r()); -} - -u32 gm20b_fb_mmu_debug_wr(struct gk20a *g) -{ - return gk20a_readl(g, fb_mmu_debug_wr_r()); -} - -u32 gm20b_fb_mmu_debug_rd(struct gk20a *g) -{ - return gk20a_readl(g, fb_mmu_debug_rd_r()); -} - -unsigned int gm20b_fb_compression_page_size(struct gk20a *g) -{ - return SZ_128K; -} - -unsigned int gm20b_fb_compressible_page_size(struct gk20a *g) -{ - return SZ_64K; -} - -u32 gm20b_fb_compression_align_mask(struct gk20a *g) -{ - return SZ_64K - 1; -} - -void gm20b_fb_dump_vpr_wpr_info(struct gk20a *g) -{ - u32 val; - - /* print vpr and wpr info */ - val = gk20a_readl(g, fb_mmu_vpr_info_r()); - val &= ~0x3; - val |= fb_mmu_vpr_info_index_addr_lo_v(); - gk20a_writel(g, fb_mmu_vpr_info_r(), val); - nvgpu_err(g, "VPR: %08x %08x %08x %08x", - gk20a_readl(g, fb_mmu_vpr_info_r()), - gk20a_readl(g, fb_mmu_vpr_info_r()), - gk20a_readl(g, fb_mmu_vpr_info_r()), - gk20a_readl(g, fb_mmu_vpr_info_r())); - - val = gk20a_readl(g, fb_mmu_wpr_info_r()); - val &= ~0xf; - val |= (fb_mmu_wpr_info_index_allow_read_v()); - gk20a_writel(g, fb_mmu_wpr_info_r(), val); - nvgpu_err(g, "WPR: %08x %08x %08x %08x %08x %08x", - gk20a_readl(g, fb_mmu_wpr_info_r()), - gk20a_readl(g, fb_mmu_wpr_info_r()), - gk20a_readl(g, fb_mmu_wpr_info_r()), - gk20a_readl(g, fb_mmu_wpr_info_r()), - gk20a_readl(g, fb_mmu_wpr_info_r()), - gk20a_readl(g, fb_mmu_wpr_info_r())); - -} - -static int gm20b_fb_vpr_info_fetch_wait(struct gk20a *g, - unsigned int msec) -{ - struct nvgpu_timeout timeout; - - nvgpu_timeout_init(g, &timeout, msec, NVGPU_TIMER_CPU_TIMER); - - do { - u32 val; - - val = gk20a_readl(g, fb_mmu_vpr_info_r()); - if (fb_mmu_vpr_info_fetch_v(val) == - fb_mmu_vpr_info_fetch_false_v()) - return 0; - - } while (!nvgpu_timeout_expired(&timeout)); - - return -ETIMEDOUT; -} - -int gm20b_fb_vpr_info_fetch(struct gk20a *g) -{ - if (gm20b_fb_vpr_info_fetch_wait(g, VPR_INFO_FETCH_WAIT)) { - return -ETIMEDOUT; - } - - gk20a_writel(g, fb_mmu_vpr_info_r(), - fb_mmu_vpr_info_fetch_true_v()); - - return gm20b_fb_vpr_info_fetch_wait(g, VPR_INFO_FETCH_WAIT); -} - -void gm20b_fb_read_wpr_info(struct gk20a *g, struct wpr_carveout_info *inf) -{ - u32 val = 0; - u64 wpr_start = 0; - u64 wpr_end = 0; - - val = gk20a_readl(g, fb_mmu_wpr_info_r()); - val &= ~0xF; - val |= fb_mmu_wpr_info_index_wpr1_addr_lo_v(); - gk20a_writel(g, fb_mmu_wpr_info_r(), val); - - val = gk20a_readl(g, fb_mmu_wpr_info_r()) >> 0x4; - wpr_start = hi32_lo32_to_u64( - (val >> (32 - WPR_INFO_ADDR_ALIGNMENT)), - (val << WPR_INFO_ADDR_ALIGNMENT)); - - val = gk20a_readl(g, fb_mmu_wpr_info_r()); - val &= ~0xF; - val |= fb_mmu_wpr_info_index_wpr1_addr_hi_v(); - gk20a_writel(g, fb_mmu_wpr_info_r(), val); - - val = gk20a_readl(g, fb_mmu_wpr_info_r()) >> 0x4; - wpr_end = hi32_lo32_to_u64( - (val >> (32 - WPR_INFO_ADDR_ALIGNMENT)), - (val << WPR_INFO_ADDR_ALIGNMENT)); - - inf->wpr_base = wpr_start; - inf->nonwpr_base = 0; - inf->size = (wpr_end - wpr_start); -} - -bool gm20b_fb_debug_mode_enabled(struct gk20a *g) -{ - u32 debug_ctrl = gk20a_readl(g, gr_gpcs_pri_mmu_debug_ctrl_r()); - return gr_gpcs_pri_mmu_debug_ctrl_debug_v(debug_ctrl) == - gr_gpcs_pri_mmu_debug_ctrl_debug_enabled_v(); -} - -void gm20b_fb_set_debug_mode(struct gk20a *g, bool enable) -{ - u32 reg_val, fb_debug_ctrl, gpc_debug_ctrl; - - if (enable) { - fb_debug_ctrl = fb_mmu_debug_ctrl_debug_enabled_f(); - gpc_debug_ctrl = gr_gpcs_pri_mmu_debug_ctrl_debug_enabled_f(); - g->mmu_debug_ctrl = true; - } else { - fb_debug_ctrl = fb_mmu_debug_ctrl_debug_disabled_f(); - gpc_debug_ctrl = gr_gpcs_pri_mmu_debug_ctrl_debug_disabled_f(); - g->mmu_debug_ctrl = false; - } - - reg_val = gk20a_readl(g, fb_mmu_debug_ctrl_r()); - reg_val = set_field(reg_val, - fb_mmu_debug_ctrl_debug_m(), fb_debug_ctrl); - gk20a_writel(g, fb_mmu_debug_ctrl_r(), reg_val); - - reg_val = gk20a_readl(g, gr_gpcs_pri_mmu_debug_ctrl_r()); - reg_val = set_field(reg_val, - gr_gpcs_pri_mmu_debug_ctrl_debug_m(), gpc_debug_ctrl); - gk20a_writel(g, gr_gpcs_pri_mmu_debug_ctrl_r(), reg_val); -} diff --git a/drivers/gpu/nvgpu/gm20b/fb_gm20b.h b/drivers/gpu/nvgpu/gm20b/fb_gm20b.h deleted file mode 100644 index 1a5e9187..00000000 --- a/drivers/gpu/nvgpu/gm20b/fb_gm20b.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * GM20B FB - * - * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#ifndef _NVHOST_GM20B_FB -#define _NVHOST_GM20B_FB -struct gk20a; - -void fb_gm20b_init_fs_state(struct gk20a *g); -void gm20b_fb_set_mmu_page_size(struct gk20a *g); -bool gm20b_fb_set_use_full_comp_tag_line(struct gk20a *g); -u32 gm20b_fb_mmu_ctrl(struct gk20a *g); -u32 gm20b_fb_mmu_debug_ctrl(struct gk20a *g); -u32 gm20b_fb_mmu_debug_wr(struct gk20a *g); -u32 gm20b_fb_mmu_debug_rd(struct gk20a *g); -unsigned int gm20b_fb_compression_page_size(struct gk20a *g); -unsigned int gm20b_fb_compressible_page_size(struct gk20a *g); -u32 gm20b_fb_compression_align_mask(struct gk20a *g); -void gm20b_fb_dump_vpr_wpr_info(struct gk20a *g); -void gm20b_fb_read_wpr_info(struct gk20a *g, struct wpr_carveout_info *inf); -int gm20b_fb_vpr_info_fetch(struct gk20a *g); -bool gm20b_fb_debug_mode_enabled(struct gk20a *g); -void gm20b_fb_set_debug_mode(struct gk20a *g, bool enable); - -#endif diff --git a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c index 02355450..30349304 100644 --- a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c @@ -26,11 +26,12 @@ #include "common/bus/bus_gm20b.h" #include "common/bus/bus_gk20a.h" #include "common/ptimer/ptimer_gk20a.h" +#include "common/fb/fb_gk20a.h" +#include "common/fb/fb_gm20b.h" #include "gk20a/gk20a.h" #include "gk20a/ce2_gk20a.h" #include "gk20a/dbg_gpu_gk20a.h" -#include "gk20a/fb_gk20a.h" #include "gk20a/fifo_gk20a.h" #include "gk20a/therm_gk20a.h" #include "gk20a/mm_gk20a.h" @@ -46,7 +47,6 @@ #include "ltc_gm20b.h" #include "gr_gm20b.h" #include "ltc_gm20b.h" -#include "fb_gm20b.h" #include "fifo_gm20b.h" #include "gr_ctx_gm20b.h" #include "mm_gm20b.h" diff --git a/drivers/gpu/nvgpu/gp106/fb_gp106.c b/drivers/gpu/nvgpu/gp106/fb_gp106.c deleted file mode 100644 index 2bf97f61..00000000 --- a/drivers/gpu/nvgpu/gp106/fb_gp106.c +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include "gk20a/gk20a.h" -#include "gp10b/fb_gp10b.h" - -#include "fb_gp106.h" - -#include - -#define HW_SCRUB_TIMEOUT_DEFAULT 100 /* usec */ -#define HW_SCRUB_TIMEOUT_MAX 2000000 /* usec */ - -void gp106_fb_reset(struct gk20a *g) -{ - u32 val; - - int retries = HW_SCRUB_TIMEOUT_MAX / HW_SCRUB_TIMEOUT_DEFAULT; - /* wait for memory to be accessible */ - do { - u32 w = gk20a_readl(g, fb_niso_scrub_status_r()); - if (fb_niso_scrub_status_flag_v(w)) { - nvgpu_log_fn(g, "done"); - break; - } - nvgpu_udelay(HW_SCRUB_TIMEOUT_DEFAULT); - } while (--retries); - - val = gk20a_readl(g, fb_mmu_priv_level_mask_r()); - val &= ~fb_mmu_priv_level_mask_write_violation_m(); - gk20a_writel(g, fb_mmu_priv_level_mask_r(), val); -} diff --git a/drivers/gpu/nvgpu/gp106/fb_gp106.h b/drivers/gpu/nvgpu/gp106/fb_gp106.h deleted file mode 100644 index d5ee87f4..00000000 --- a/drivers/gpu/nvgpu/gp106/fb_gp106.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#ifndef FB_GP106_H -#define FB_GP106_H -struct gpu_ops; - -void gp106_fb_reset(struct gk20a *g); -#endif diff --git a/drivers/gpu/nvgpu/gp106/hal_gp106.c b/drivers/gpu/nvgpu/gp106/hal_gp106.c index 75e2f5a1..63e6206a 100644 --- a/drivers/gpu/nvgpu/gp106/hal_gp106.c +++ b/drivers/gpu/nvgpu/gp106/hal_gp106.c @@ -27,6 +27,10 @@ #include "common/ptimer/ptimer_gk20a.h" #include "common/bus/bus_gm20b.h" #include "common/bus/bus_gp10b.h" +#include "common/fb/fb_gk20a.h" +#include "common/fb/fb_gp10b.h" +#include "common/fb/fb_gm20b.h" +#include "common/fb/fb_gp106.h" #include "gk20a/gk20a.h" #include "gk20a/fifo_gk20a.h" @@ -37,7 +41,6 @@ #include "gk20a/flcn_gk20a.h" #include "gk20a/regops_gk20a.h" #include "gk20a/mc_gk20a.h" -#include "gk20a/fb_gk20a.h" #include "gk20a/pmu_gk20a.h" #include "gk20a/gr_gk20a.h" @@ -50,7 +53,6 @@ #include "gp10b/regops_gp10b.h" #include "gp10b/priv_ring_gp10b.h" #include "gp10b/fifo_gp10b.h" -#include "gp10b/fb_gp10b.h" #include "gp10b/pmu_gp10b.h" #include "gp10b/gr_gp10b.h" #include "gp10b/priv_ring_gp10b.h" @@ -63,7 +65,6 @@ #include "gm20b/fifo_gm20b.h" #include "gm20b/mm_gm20b.h" #include "gm20b/pmu_gm20b.h" -#include "gm20b/fb_gm20b.h" #include "gm20b/acr_gm20b.h" #include "gm20b/gr_gm20b.h" @@ -81,7 +82,6 @@ #include "gp106/pmu_gp106.h" #include "gp106/gr_ctx_gp106.h" #include "gp106/gr_gp106.h" -#include "gp106/fb_gp106.h" #include "gp106/flcn_gp106.h" #include "gp106/fuse_gp106.h" diff --git a/drivers/gpu/nvgpu/gp10b/fb_gp10b.c b/drivers/gpu/nvgpu/gp10b/fb_gp10b.c deleted file mode 100644 index 45fc8373..00000000 --- a/drivers/gpu/nvgpu/gp10b/fb_gp10b.c +++ /dev/null @@ -1,38 +0,0 @@ -/* - * GP10B FB - * - * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. -* - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include - -#include "gk20a/gk20a.h" -#include "fb_gp10b.h" - -unsigned int gp10b_fb_compression_page_size(struct gk20a *g) -{ - return SZ_64K; -} - -unsigned int gp10b_fb_compressible_page_size(struct gk20a *g) -{ - return SZ_4K; -} diff --git a/drivers/gpu/nvgpu/gp10b/fb_gp10b.h b/drivers/gpu/nvgpu/gp10b/fb_gp10b.h deleted file mode 100644 index 52aa2a75..00000000 --- a/drivers/gpu/nvgpu/gp10b/fb_gp10b.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * GP10B FB - * - * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#ifndef _NVGPU_GP10B_FB -#define _NVGPU_GP10B_FB -struct gk20a; - -unsigned int gp10b_fb_compression_page_size(struct gk20a *g); -unsigned int gp10b_fb_compressible_page_size(struct gk20a *g); - -#endif diff --git a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c index a40608df..bbfce287 100644 --- a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c @@ -27,6 +27,9 @@ #include "common/ptimer/ptimer_gk20a.h" #include "common/bus/bus_gm20b.h" #include "common/bus/bus_gp10b.h" +#include "common/fb/fb_gk20a.h" +#include "common/fb/fb_gm20b.h" +#include "common/fb/fb_gp10b.h" #include "gk20a/gk20a.h" #include "gk20a/fifo_gk20a.h" @@ -37,7 +40,6 @@ #include "gk20a/flcn_gk20a.h" #include "gk20a/regops_gk20a.h" #include "gk20a/mc_gk20a.h" -#include "gk20a/fb_gk20a.h" #include "gk20a/pmu_gk20a.h" #include "gk20a/gr_gk20a.h" #include "gk20a/tsg_gk20a.h" @@ -48,7 +50,6 @@ #include "gp10b/ltc_gp10b.h" #include "gp10b/mm_gp10b.h" #include "gp10b/ce_gp10b.h" -#include "gp10b/fb_gp10b.h" #include "gp10b/pmu_gp10b.h" #include "gp10b/gr_ctx_gp10b.h" #include "gp10b/fifo_gp10b.h" @@ -62,7 +63,6 @@ #include "gm20b/acr_gm20b.h" #include "gm20b/pmu_gm20b.h" #include "gm20b/clk_gm20b.h" -#include "gm20b/fb_gm20b.h" #include "gm20b/mm_gm20b.h" #include "gp10b.h" diff --git a/drivers/gpu/nvgpu/gv100/fb_gv100.c b/drivers/gpu/nvgpu/gv100/fb_gv100.c deleted file mode 100644 index 1889268f..00000000 --- a/drivers/gpu/nvgpu/gv100/fb_gv100.c +++ /dev/null @@ -1,256 +0,0 @@ -/* - * GV100 FB - * - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "gk20a/gk20a.h" -#include "gv100/fb_gv100.h" -#include "gm20b/acr_gm20b.h" - -#include -#include -#include - -#define HW_SCRUB_TIMEOUT_DEFAULT 100 /* usec */ -#define HW_SCRUB_TIMEOUT_MAX 2000000 /* usec */ -#define MEM_UNLOCK_TIMEOUT 3500 /* msec */ - -void gv100_fb_reset(struct gk20a *g) -{ - u32 val; - int retries = HW_SCRUB_TIMEOUT_MAX / HW_SCRUB_TIMEOUT_DEFAULT; - - nvgpu_info(g, "reset gv100 fb"); - - /* wait for memory to be accessible */ - do { - u32 w = gk20a_readl(g, fb_niso_scrub_status_r()); - if (fb_niso_scrub_status_flag_v(w)) { - nvgpu_info(g, "done"); - break; - } - nvgpu_udelay(HW_SCRUB_TIMEOUT_DEFAULT); - } while (--retries); - - val = gk20a_readl(g, fb_mmu_priv_level_mask_r()); - val &= ~fb_mmu_priv_level_mask_write_violation_m(); - gk20a_writel(g, fb_mmu_priv_level_mask_r(), val); -} - -void gv100_fb_enable_hub_intr(struct gk20a *g) -{ - u32 mask = 0; - - mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | - fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | - fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | - fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | - fb_niso_intr_en_set_mmu_replayable_fault_overflow_m(); - - gk20a_writel(g, fb_niso_intr_en_set_r(0), - mask); -} - -void gv100_fb_disable_hub_intr(struct gk20a *g) -{ - u32 mask = 0; - - mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | - fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | - fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | - fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | - fb_niso_intr_en_set_mmu_replayable_fault_overflow_m(); - - gk20a_writel(g, fb_niso_intr_en_clr_r(0), - mask); -} - -int gv100_fb_memory_unlock(struct gk20a *g) -{ - struct nvgpu_firmware *mem_unlock_fw = NULL; - struct bin_hdr *hsbin_hdr = NULL; - struct acr_fw_header *fw_hdr = NULL; - u32 *mem_unlock_ucode = NULL; - u32 *mem_unlock_ucode_header = NULL; - u32 sec_imem_dest = 0; - u32 val = 0; - int err = 0; - - nvgpu_log_fn(g, " "); - - nvgpu_log_info(g, "fb_mmu_vpr_info = 0x%08x", - gk20a_readl(g, fb_mmu_vpr_info_r())); - /* - * mem_unlock.bin should be written to install - * traps even if VPR isn’t actually supported - */ - mem_unlock_fw = nvgpu_request_firmware(g, "mem_unlock.bin", 0); - if (!mem_unlock_fw) { - nvgpu_err(g, "mem unlock ucode get fail"); - err = -ENOENT; - goto exit; - } - - /* Enable nvdec */ - g->ops.mc.enable(g, mc_enable_nvdec_enabled_f()); - - /* nvdec falcon reset */ - nvgpu_flcn_reset(&g->nvdec_flcn); - - hsbin_hdr = (struct bin_hdr *)mem_unlock_fw->data; - fw_hdr = (struct acr_fw_header *)(mem_unlock_fw->data + - hsbin_hdr->header_offset); - - mem_unlock_ucode_header = (u32 *)(mem_unlock_fw->data + - fw_hdr->hdr_offset); - mem_unlock_ucode = (u32 *)(mem_unlock_fw->data + - hsbin_hdr->data_offset); - - /* Patch Ucode singnatures */ - if (acr_ucode_patch_sig(g, mem_unlock_ucode, - (u32 *)(mem_unlock_fw->data + fw_hdr->sig_prod_offset), - (u32 *)(mem_unlock_fw->data + fw_hdr->sig_dbg_offset), - (u32 *)(mem_unlock_fw->data + fw_hdr->patch_loc), - (u32 *)(mem_unlock_fw->data + fw_hdr->patch_sig)) < 0) { - nvgpu_err(g, "mem unlock patch signatures fail"); - err = -EPERM; - goto exit; - } - - /* Clear interrupts */ - nvgpu_flcn_set_irq(&g->nvdec_flcn, false, 0x0, 0x0); - - /* Copy Non Secure IMEM code */ - nvgpu_flcn_copy_to_imem(&g->nvdec_flcn, 0, - (u8 *)&mem_unlock_ucode[ - mem_unlock_ucode_header[OS_CODE_OFFSET] >> 2], - mem_unlock_ucode_header[OS_CODE_SIZE], 0, false, - GET_IMEM_TAG(mem_unlock_ucode_header[OS_CODE_OFFSET])); - - /* Put secure code after non-secure block */ - sec_imem_dest = GET_NEXT_BLOCK(mem_unlock_ucode_header[OS_CODE_SIZE]); - - nvgpu_flcn_copy_to_imem(&g->nvdec_flcn, sec_imem_dest, - (u8 *)&mem_unlock_ucode[ - mem_unlock_ucode_header[APP_0_CODE_OFFSET] >> 2], - mem_unlock_ucode_header[APP_0_CODE_SIZE], 0, true, - GET_IMEM_TAG(mem_unlock_ucode_header[APP_0_CODE_OFFSET])); - - /* load DMEM: ensure that signatures are patched */ - nvgpu_flcn_copy_to_dmem(&g->nvdec_flcn, 0, (u8 *)&mem_unlock_ucode[ - mem_unlock_ucode_header[OS_DATA_OFFSET] >> 2], - mem_unlock_ucode_header[OS_DATA_SIZE], 0); - - nvgpu_log_info(g, "nvdec sctl reg %x\n", - gk20a_readl(g, g->nvdec_flcn.flcn_base + - falcon_falcon_sctl_r())); - - /* set BOOTVEC to start of non-secure code */ - nvgpu_flcn_bootstrap(&g->nvdec_flcn, 0); - - /* wait for complete & halt */ - nvgpu_flcn_wait_for_halt(&g->nvdec_flcn, MEM_UNLOCK_TIMEOUT); - - /* check mem unlock status */ - val = nvgpu_flcn_mailbox_read(&g->nvdec_flcn, 0); - if (val) { - nvgpu_err(g, "memory unlock failed, err %x", val); - err = -1; - goto exit; - } - - nvgpu_log_info(g, "nvdec sctl reg %x\n", - gk20a_readl(g, g->nvdec_flcn.flcn_base + - falcon_falcon_sctl_r())); - -exit: - if (mem_unlock_fw) - nvgpu_release_firmware(g, mem_unlock_fw); - - nvgpu_log_fn(g, "done, status - %d", err); - - return err; -} - -int gv100_fb_init_nvlink(struct gk20a *g) -{ - u32 data; - u32 mask = g->nvlink.enabled_links; - - /* Map enabled link to SYSMEM */ - data = nvgpu_readl(g, fb_hshub_config0_r()); - data = set_field(data, fb_hshub_config0_sysmem_nvlink_mask_m(), - fb_hshub_config0_sysmem_nvlink_mask_f(mask)); - nvgpu_writel(g, fb_hshub_config0_r(), data); - - return 0; -} - -int gv100_fb_enable_nvlink(struct gk20a *g) -{ - u32 data; - - nvgpu_log(g, gpu_dbg_nvlink|gpu_dbg_info, "enabling nvlink"); - - /* Enable nvlink for NISO FBHUB */ - data = nvgpu_readl(g, fb_niso_cfg1_r()); - data = set_field(data, fb_niso_cfg1_sysmem_nvlink_m(), - fb_niso_cfg1_sysmem_nvlink_enabled_f()); - nvgpu_writel(g, fb_niso_cfg1_r(), data); - - /* Setup atomics */ - data = nvgpu_readl(g, fb_mmu_ctrl_r()); - data = set_field(data, fb_mmu_ctrl_atomic_capability_mode_m(), - fb_mmu_ctrl_atomic_capability_mode_rmw_f()); - nvgpu_writel(g, fb_mmu_ctrl_r(), data); - - data = nvgpu_readl(g, fb_hsmmu_pri_mmu_ctrl_r()); - data = set_field(data, fb_hsmmu_pri_mmu_ctrl_atomic_capability_mode_m(), - fb_hsmmu_pri_mmu_ctrl_atomic_capability_mode_rmw_f()); - nvgpu_writel(g, fb_hsmmu_pri_mmu_ctrl_r(), data); - - data = nvgpu_readl(g, fb_fbhub_num_active_ltcs_r()); - data = set_field(data, fb_fbhub_num_active_ltcs_hub_sys_atomic_mode_m(), - fb_fbhub_num_active_ltcs_hub_sys_atomic_mode_use_rmw_f()); - nvgpu_writel(g, fb_fbhub_num_active_ltcs_r(), data); - - data = nvgpu_readl(g, fb_hshub_num_active_ltcs_r()); - data = set_field(data, fb_hshub_num_active_ltcs_hub_sys_atomic_mode_m(), - fb_hshub_num_active_ltcs_hub_sys_atomic_mode_use_rmw_f()); - nvgpu_writel(g, fb_hshub_num_active_ltcs_r(), data); - - return 0; -} diff --git a/drivers/gpu/nvgpu/gv100/fb_gv100.h b/drivers/gpu/nvgpu/gv100/fb_gv100.h deleted file mode 100644 index 195baccf..00000000 --- a/drivers/gpu/nvgpu/gv100/fb_gv100.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * GV100 FB - * - * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#ifndef _NVGPU_GV100_FB -#define _NVGPU_GV100_FB - -struct gk20a; - -void gv100_fb_reset(struct gk20a *g); -void gv100_fb_enable_hub_intr(struct gk20a *g); -void gv100_fb_disable_hub_intr(struct gk20a *g); -int gv100_fb_memory_unlock(struct gk20a *g); -int gv100_fb_init_nvlink(struct gk20a *g); -int gv100_fb_enable_nvlink(struct gk20a *g); -#endif diff --git a/drivers/gpu/nvgpu/gv100/hal_gv100.c b/drivers/gpu/nvgpu/gv100/hal_gv100.c index 9c3eb91a..6f340e6a 100644 --- a/drivers/gpu/nvgpu/gv100/hal_gv100.c +++ b/drivers/gpu/nvgpu/gv100/hal_gv100.c @@ -27,6 +27,11 @@ #include "common/bus/bus_gv100.h" #include "common/clock_gating/gv100_gating_reglist.h" #include "common/ptimer/ptimer_gk20a.h" +#include "common/fb/fb_gk20a.h" +#include "common/fb/fb_gm20b.h" +#include "common/fb/fb_gp10b.h" +#include "common/fb/fb_gv11b.h" +#include "common/fb/fb_gv100.h" #include "gk20a/gk20a.h" #include "gk20a/fifo_gk20a.h" @@ -36,7 +41,6 @@ #include "gk20a/dbg_gpu_gk20a.h" #include "gk20a/flcn_gk20a.h" #include "gk20a/regops_gk20a.h" -#include "gk20a/fb_gk20a.h" #include "gk20a/mm_gk20a.h" #include "gk20a/pmu_gk20a.h" #include "gk20a/gr_gk20a.h" @@ -44,7 +48,6 @@ #include "gm20b/ltc_gm20b.h" #include "gm20b/gr_gm20b.h" #include "gm20b/fifo_gm20b.h" -#include "gm20b/fb_gm20b.h" #include "gm20b/mm_gm20b.h" #include "gm20b/pmu_gm20b.h" #include "gm20b/acr_gm20b.h" @@ -60,7 +63,6 @@ #include "gp106/clk_gp106.h" #include "gp106/flcn_gp106.h" -#include "gp10b/fb_gp10b.h" #include "gp10b/gr_gp10b.h" #include "gp10b/ltc_gp10b.h" #include "gp10b/therm_gp10b.h" @@ -82,7 +84,6 @@ #include "gv11b/ce_gv11b.h" #include "gv11b/mm_gv11b.h" #include "gv11b/pmu_gv11b.h" -#include "gv11b/fb_gv11b.h" #include "gv11b/pmu_gv11b.h" #include "gv11b/fifo_gv11b.h" #include "gv11b/regops_gv11b.h" @@ -91,7 +92,6 @@ #include "gv100.h" #include "hal_gv100.h" #include "gv100/bios_gv100.h" -#include "gv100/fb_gv100.h" #include "gv100/fifo_gv100.h" #include "gv100/flcn_gv100.h" #include "gv100/gr_ctx_gv100.h" diff --git a/drivers/gpu/nvgpu/gv100/mc_gv100.c b/drivers/gpu/nvgpu/gv100/mc_gv100.c index 3d6f0bd3..46af100a 100644 --- a/drivers/gpu/nvgpu/gv100/mc_gv100.c +++ b/drivers/gpu/nvgpu/gv100/mc_gv100.c @@ -29,7 +29,6 @@ #include "gp10b/mc_gp10b.h" #include "mc_gv100.h" -#include "gv11b/fb_gv11b.h" #include @@ -41,7 +40,6 @@ void mc_gv100_intr_enable(struct gk20a *g) 0xffffffffU); gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), 0xffffffffU); - g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING] = mc_intr_pfifo_pending_f() | mc_intr_hub_pending_f() | diff --git a/drivers/gpu/nvgpu/gv11b/fb_gv11b.c b/drivers/gpu/nvgpu/gv11b/fb_gv11b.c deleted file mode 100644 index c0fcf051..00000000 --- a/drivers/gpu/nvgpu/gv11b/fb_gv11b.c +++ /dev/null @@ -1,1516 +0,0 @@ -/* - * GV11B FB - * - * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "gk20a/gk20a.h" -#include "gk20a/mm_gk20a.h" -#include "gk20a/fb_gk20a.h" - -#include "gp10b/fb_gp10b.h" - -#include "gv11b/fifo_gv11b.h" -#include "gv11b/fb_gv11b.h" -#include "gv11b/ce_gv11b.h" - -#include -#include -#include -#include - -static int gv11b_fb_fix_page_fault(struct gk20a *g, - struct mmu_fault_info *mmfault); - -static void gv11b_init_nvlink_soc_credits(struct gk20a *g) -{ - if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) { - nvgpu_log(g, gpu_dbg_info, "nvlink soc credits init done by bpmp"); - } else { -#ifndef __NVGPU_POSIX__ - nvgpu_mss_nvlink_init_credits(g); -#endif - } -} - -void gv11b_fb_init_hw(struct gk20a *g) -{ - gk20a_fb_init_hw(g); - - g->ops.fb.enable_hub_intr(g); -} - -void gv11b_fb_init_fs_state(struct gk20a *g) -{ - nvgpu_log(g, gpu_dbg_fn, "initialize gv11b fb"); - - nvgpu_log(g, gpu_dbg_info, "fbhub active ltcs %x", - gk20a_readl(g, fb_fbhub_num_active_ltcs_r())); - - nvgpu_log(g, gpu_dbg_info, "mmu active ltcs %u", - fb_mmu_num_active_ltcs_count_v( - gk20a_readl(g, fb_mmu_num_active_ltcs_r()))); - - if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) { - /* Bypass MMU check for non-secure boot. For - * secure-boot,this register write has no-effect */ - gk20a_writel(g, fb_priv_mmu_phy_secure_r(), 0xffffffffU); - } -} - -void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr) -{ - u32 max_size = gr->max_comptag_mem; - /* one tag line covers 64KB */ - u32 max_comptag_lines = max_size << 4; - u32 compbit_base_post_divide; - u64 compbit_base_post_multiply64; - u64 compbit_store_iova; - u64 compbit_base_post_divide64; - - if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) - compbit_store_iova = nvgpu_mem_get_phys_addr(g, - &gr->compbit_store.mem); - else - compbit_store_iova = nvgpu_mem_get_addr(g, - &gr->compbit_store.mem); - - compbit_base_post_divide64 = compbit_store_iova >> - fb_mmu_cbc_base_address_alignment_shift_v(); - - do_div(compbit_base_post_divide64, g->ltc_count); - compbit_base_post_divide = u64_lo32(compbit_base_post_divide64); - - compbit_base_post_multiply64 = ((u64)compbit_base_post_divide * - g->ltc_count) << fb_mmu_cbc_base_address_alignment_shift_v(); - - if (compbit_base_post_multiply64 < compbit_store_iova) - compbit_base_post_divide++; - - if (g->ops.ltc.cbc_fix_config) - compbit_base_post_divide = - g->ops.ltc.cbc_fix_config(g, compbit_base_post_divide); - - gk20a_writel(g, fb_mmu_cbc_base_r(), - fb_mmu_cbc_base_address_f(compbit_base_post_divide)); - - nvgpu_log(g, gpu_dbg_info | gpu_dbg_map_v | gpu_dbg_pte, - "compbit base.pa: 0x%x,%08x cbc_base:0x%08x\n", - (u32)(compbit_store_iova >> 32), - (u32)(compbit_store_iova & 0xffffffff), - compbit_base_post_divide); - nvgpu_log(g, gpu_dbg_fn, "cbc base %x", - gk20a_readl(g, fb_mmu_cbc_base_r())); - - gr->compbit_store.base_hw = compbit_base_post_divide; - - g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_invalidate, - 0, max_comptag_lines - 1); - -} - -void gv11b_fb_reset(struct gk20a *g) -{ - gv11b_init_nvlink_soc_credits(g); -} - -static const char * const invalid_str = "invalid"; - -static const char *const fault_type_descs_gv11b[] = { - "invalid pde", - "invalid pde size", - "invalid pte", - "limit violation", - "unbound inst block", - "priv violation", - "write", - "read", - "pitch mask violation", - "work creation", - "unsupported aperture", - "compression failure", - "unsupported kind", - "region violation", - "poison", - "atomic" -}; - -static const char *const fault_client_type_descs_gv11b[] = { - "gpc", - "hub", -}; - -static const char *const fault_access_type_descs_gv11b[] = { - "virt read", - "virt write", - "virt atomic strong", - "virt prefetch", - "virt atomic weak", - "xxx", - "xxx", - "xxx", - "phys read", - "phys write", - "phys atomic", - "phys prefetch", -}; - -static const char *const hub_client_descs_gv11b[] = { - "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu", - "host cpu nb", "iso", "mmu", "nvdec", "nvenc1", "nvenc2", - "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc", - "scc nb", "sec", "ssync", "gr copy", "xv", "mmu nb", - "nvenc", "d falcon", "sked", "a falcon", "hsce0", "hsce1", - "hsce2", "hsce3", "hsce4", "hsce5", "hsce6", "hsce7", "hsce8", - "hsce9", "hshub", "ptp x0", "ptp x1", "ptp x2", "ptp x3", - "ptp x4", "ptp x5", "ptp x6", "ptp x7", "vpr scrubber0", - "vpr scrubber1", "dwbif", "fbfalcon", "ce shim", "gsp", - "dont care" -}; - -static const char *const gpc_client_descs_gv11b[] = { - "t1 0", "t1 1", "t1 2", "t1 3", - "t1 4", "t1 5", "t1 6", "t1 7", - "pe 0", "pe 1", "pe 2", "pe 3", - "pe 4", "pe 5", "pe 6", "pe 7", - "rast", "gcc", "gpccs", - "prop 0", "prop 1", "prop 2", "prop 3", - "gpm", - "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3", - "ltp utlb 4", "ltp utlb 5", "ltp utlb 6", "ltp utlb 7", - "utlb", - "t1 8", "t1 9", "t1 10", "t1 11", - "t1 12", "t1 13", "t1 14", "t1 15", - "tpccs 0", "tpccs 1", "tpccs 2", "tpccs 3", - "tpccs 4", "tpccs 5", "tpccs 6", "tpccs 7", - "pe 8", "pe 9", "tpccs 8", "tpccs 9", - "t1 16", "t1 17", "t1 18", "t1 19", - "pe 10", "pe 11", "tpccs 10", "tpccs 11", - "t1 20", "t1 21", "t1 22", "t1 23", - "pe 12", "pe 13", "tpccs 12", "tpccs 13", - "t1 24", "t1 25", "t1 26", "t1 27", - "pe 14", "pe 15", "tpccs 14", "tpccs 15", - "t1 28", "t1 29", "t1 30", "t1 31", - "pe 16", "pe 17", "tpccs 16", "tpccs 17", - "t1 32", "t1 33", "t1 34", "t1 35", - "pe 18", "pe 19", "tpccs 18", "tpccs 19", - "t1 36", "t1 37", "t1 38", "t1 39", -}; - -bool gv11b_fb_is_fault_buf_enabled(struct gk20a *g, u32 index) -{ - u32 reg_val; - - reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); - return fb_mmu_fault_buffer_size_enable_v(reg_val) != 0U; -} - -static void gv11b_fb_fault_buffer_get_ptr_update(struct gk20a *g, - u32 index, u32 next) -{ - u32 reg_val; - - nvgpu_log(g, gpu_dbg_intr, "updating get index with = %d", next); - - reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); - reg_val = set_field(reg_val, fb_mmu_fault_buffer_get_ptr_m(), - fb_mmu_fault_buffer_get_ptr_f(next)); - - /* while the fault is being handled it is possible for overflow - * to happen, - */ - if (reg_val & fb_mmu_fault_buffer_get_overflow_m()) - reg_val |= fb_mmu_fault_buffer_get_overflow_clear_f(); - - g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); - - /* make sure get ptr update is visible to everyone to avoid - * reading already read entry - */ - nvgpu_mb(); -} - -static u32 gv11b_fb_fault_buffer_get_index(struct gk20a *g, u32 index) -{ - u32 reg_val; - - reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); - return fb_mmu_fault_buffer_get_ptr_v(reg_val); -} - -static u32 gv11b_fb_fault_buffer_put_index(struct gk20a *g, u32 index) -{ - u32 reg_val; - - reg_val = g->ops.fb.read_mmu_fault_buffer_put(g, index); - return fb_mmu_fault_buffer_put_ptr_v(reg_val); -} - -static u32 gv11b_fb_fault_buffer_size_val(struct gk20a *g, u32 index) -{ - u32 reg_val; - - reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); - return fb_mmu_fault_buffer_size_val_v(reg_val); -} - -static bool gv11b_fb_is_fault_buffer_empty(struct gk20a *g, - u32 index, u32 *get_idx) -{ - u32 put_idx; - - *get_idx = gv11b_fb_fault_buffer_get_index(g, index); - put_idx = gv11b_fb_fault_buffer_put_index(g, index); - - return *get_idx == put_idx; -} - -static bool gv11b_fb_is_fault_buffer_full(struct gk20a *g, u32 index) -{ - u32 get_idx, put_idx, entries; - - - get_idx = gv11b_fb_fault_buffer_get_index(g, index); - - put_idx = gv11b_fb_fault_buffer_put_index(g, index); - - entries = gv11b_fb_fault_buffer_size_val(g, index); - - return get_idx == ((put_idx + 1) % entries); -} - -void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g, - u32 index, u32 state) -{ - u32 fault_status; - u32 reg_val; - - nvgpu_log_fn(g, " "); - - reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); - if (state == NVGPU_FB_MMU_FAULT_BUF_ENABLED) { - if (gv11b_fb_is_fault_buf_enabled(g, index)) { - nvgpu_log_info(g, "fault buffer is already enabled"); - } else { - reg_val |= fb_mmu_fault_buffer_size_enable_true_f(); - g->ops.fb.write_mmu_fault_buffer_size(g, index, - reg_val); - } - - } else { - struct nvgpu_timeout timeout; - u32 delay = GR_IDLE_CHECK_DEFAULT; - - nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), - NVGPU_TIMER_CPU_TIMER); - - reg_val &= (~(fb_mmu_fault_buffer_size_enable_m())); - g->ops.fb.write_mmu_fault_buffer_size(g, index, reg_val); - - fault_status = g->ops.fb.read_mmu_fault_status(g); - - do { - if (!(fault_status & fb_mmu_fault_status_busy_true_f())) - break; - /* - * Make sure fault buffer is disabled. - * This is to avoid accessing fault buffer by hw - * during the window BAR2 is being unmapped by s/w - */ - nvgpu_log_info(g, "fault status busy set, check again"); - fault_status = g->ops.fb.read_mmu_fault_status(g); - - nvgpu_usleep_range(delay, delay * 2); - delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); - } while (!nvgpu_timeout_expired_msg(&timeout, - "fault status busy set")); - } -} - -void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, u32 index) -{ - u32 addr_lo; - u32 addr_hi; - - nvgpu_log_fn(g, " "); - - gv11b_fb_fault_buf_set_state_hw(g, index, - NVGPU_FB_MMU_FAULT_BUF_DISABLED); - addr_lo = u64_lo32(g->mm.hw_fault_buf[index].gpu_va >> - ram_in_base_shift_v()); - addr_hi = u64_hi32(g->mm.hw_fault_buf[index].gpu_va); - - g->ops.fb.write_mmu_fault_buffer_lo_hi(g, index, - fb_mmu_fault_buffer_lo_addr_f(addr_lo), - fb_mmu_fault_buffer_hi_addr_f(addr_hi)); - - g->ops.fb.write_mmu_fault_buffer_size(g, index, - fb_mmu_fault_buffer_size_val_f(g->ops.fifo.get_num_fifos(g)) | - fb_mmu_fault_buffer_size_overflow_intr_enable_f()); - - gv11b_fb_fault_buf_set_state_hw(g, index, NVGPU_FB_MMU_FAULT_BUF_ENABLED); -} - -void gv11b_fb_enable_hub_intr(struct gk20a *g) -{ - u32 mask = 0; - - mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | - fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | - fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | - fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | - fb_niso_intr_en_set_mmu_replayable_fault_overflow_m() | - fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); - - gk20a_writel(g, fb_niso_intr_en_set_r(0), - mask); -} - -void gv11b_fb_disable_hub_intr(struct gk20a *g) -{ - u32 mask = 0; - - mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | - fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | - fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | - fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | - fb_niso_intr_en_set_mmu_replayable_fault_overflow_m() | - fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); - - gk20a_writel(g, fb_niso_intr_en_clr_r(0), - mask); -} - -void gv11b_handle_l2tlb_ecc_isr(struct gk20a *g, u32 ecc_status) -{ - u32 ecc_addr, corrected_cnt, uncorrected_cnt; - u32 corrected_delta, uncorrected_delta; - u32 corrected_overflow, uncorrected_overflow; - - ecc_addr = gk20a_readl(g, fb_mmu_l2tlb_ecc_address_r()); - corrected_cnt = gk20a_readl(g, - fb_mmu_l2tlb_ecc_corrected_err_count_r()); - uncorrected_cnt = gk20a_readl(g, - fb_mmu_l2tlb_ecc_uncorrected_err_count_r()); - - corrected_delta = fb_mmu_l2tlb_ecc_corrected_err_count_total_v( - corrected_cnt); - uncorrected_delta = fb_mmu_l2tlb_ecc_uncorrected_err_count_total_v( - uncorrected_cnt); - corrected_overflow = ecc_status & - fb_mmu_l2tlb_ecc_status_corrected_err_total_counter_overflow_m(); - - uncorrected_overflow = ecc_status & - fb_mmu_l2tlb_ecc_status_uncorrected_err_total_counter_overflow_m(); - - /* clear the interrupt */ - if ((corrected_delta > 0) || corrected_overflow) - gk20a_writel(g, fb_mmu_l2tlb_ecc_corrected_err_count_r(), 0); - if ((uncorrected_delta > 0) || uncorrected_overflow) - gk20a_writel(g, fb_mmu_l2tlb_ecc_uncorrected_err_count_r(), 0); - - gk20a_writel(g, fb_mmu_l2tlb_ecc_status_r(), - fb_mmu_l2tlb_ecc_status_reset_clear_f()); - - /* Handle overflow */ - if (corrected_overflow) - corrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_corrected_err_count_total_s()); - if (uncorrected_overflow) - uncorrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_uncorrected_err_count_total_s()); - - - g->ecc.fb.mmu_l2tlb_corrected_err_count.counters[0] += - corrected_delta; - g->ecc.fb.mmu_l2tlb_uncorrected_err_count.counters[0] += - uncorrected_delta; - - if (ecc_status & fb_mmu_l2tlb_ecc_status_corrected_err_l2tlb_sa_data_m()) - nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); - if (ecc_status & fb_mmu_l2tlb_ecc_status_uncorrected_err_l2tlb_sa_data_m()) - nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); - if (corrected_overflow || uncorrected_overflow) - nvgpu_info(g, "mmu l2tlb ecc counter overflow!"); - - nvgpu_log(g, gpu_dbg_intr, - "ecc error address: 0x%x", ecc_addr); - nvgpu_log(g, gpu_dbg_intr, - "ecc error count corrected: %d, uncorrected %d", - g->ecc.fb.mmu_l2tlb_corrected_err_count.counters[0], - g->ecc.fb.mmu_l2tlb_uncorrected_err_count.counters[0]); -} - -void gv11b_handle_hubtlb_ecc_isr(struct gk20a *g, u32 ecc_status) -{ - u32 ecc_addr, corrected_cnt, uncorrected_cnt; - u32 corrected_delta, uncorrected_delta; - u32 corrected_overflow, uncorrected_overflow; - - ecc_addr = gk20a_readl(g, fb_mmu_hubtlb_ecc_address_r()); - corrected_cnt = gk20a_readl(g, - fb_mmu_hubtlb_ecc_corrected_err_count_r()); - uncorrected_cnt = gk20a_readl(g, - fb_mmu_hubtlb_ecc_uncorrected_err_count_r()); - - corrected_delta = fb_mmu_hubtlb_ecc_corrected_err_count_total_v( - corrected_cnt); - uncorrected_delta = fb_mmu_hubtlb_ecc_uncorrected_err_count_total_v( - uncorrected_cnt); - corrected_overflow = ecc_status & - fb_mmu_hubtlb_ecc_status_corrected_err_total_counter_overflow_m(); - - uncorrected_overflow = ecc_status & - fb_mmu_hubtlb_ecc_status_uncorrected_err_total_counter_overflow_m(); - - /* clear the interrupt */ - if ((corrected_delta > 0) || corrected_overflow) - gk20a_writel(g, fb_mmu_hubtlb_ecc_corrected_err_count_r(), 0); - if ((uncorrected_delta > 0) || uncorrected_overflow) - gk20a_writel(g, fb_mmu_hubtlb_ecc_uncorrected_err_count_r(), 0); - - gk20a_writel(g, fb_mmu_hubtlb_ecc_status_r(), - fb_mmu_hubtlb_ecc_status_reset_clear_f()); - - /* Handle overflow */ - if (corrected_overflow) - corrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_corrected_err_count_total_s()); - if (uncorrected_overflow) - uncorrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_uncorrected_err_count_total_s()); - - - g->ecc.fb.mmu_hubtlb_corrected_err_count.counters[0] += - corrected_delta; - g->ecc.fb.mmu_hubtlb_uncorrected_err_count.counters[0] += - uncorrected_delta; - - if (ecc_status & fb_mmu_hubtlb_ecc_status_corrected_err_sa_data_m()) - nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); - if (ecc_status & fb_mmu_hubtlb_ecc_status_uncorrected_err_sa_data_m()) - nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); - if (corrected_overflow || uncorrected_overflow) - nvgpu_info(g, "mmu hubtlb ecc counter overflow!"); - - nvgpu_log(g, gpu_dbg_intr, - "ecc error address: 0x%x", ecc_addr); - nvgpu_log(g, gpu_dbg_intr, - "ecc error count corrected: %d, uncorrected %d", - g->ecc.fb.mmu_hubtlb_corrected_err_count.counters[0], - g->ecc.fb.mmu_hubtlb_uncorrected_err_count.counters[0]); -} - -void gv11b_handle_fillunit_ecc_isr(struct gk20a *g, u32 ecc_status) -{ - u32 ecc_addr, corrected_cnt, uncorrected_cnt; - u32 corrected_delta, uncorrected_delta; - u32 corrected_overflow, uncorrected_overflow; - - ecc_addr = gk20a_readl(g, fb_mmu_fillunit_ecc_address_r()); - corrected_cnt = gk20a_readl(g, - fb_mmu_fillunit_ecc_corrected_err_count_r()); - uncorrected_cnt = gk20a_readl(g, - fb_mmu_fillunit_ecc_uncorrected_err_count_r()); - - corrected_delta = fb_mmu_fillunit_ecc_corrected_err_count_total_v( - corrected_cnt); - uncorrected_delta = fb_mmu_fillunit_ecc_uncorrected_err_count_total_v( - uncorrected_cnt); - corrected_overflow = ecc_status & - fb_mmu_fillunit_ecc_status_corrected_err_total_counter_overflow_m(); - - uncorrected_overflow = ecc_status & - fb_mmu_fillunit_ecc_status_uncorrected_err_total_counter_overflow_m(); - - /* clear the interrupt */ - if ((corrected_delta > 0) || corrected_overflow) - gk20a_writel(g, fb_mmu_fillunit_ecc_corrected_err_count_r(), 0); - if ((uncorrected_delta > 0) || uncorrected_overflow) - gk20a_writel(g, fb_mmu_fillunit_ecc_uncorrected_err_count_r(), 0); - - gk20a_writel(g, fb_mmu_fillunit_ecc_status_r(), - fb_mmu_fillunit_ecc_status_reset_clear_f()); - - /* Handle overflow */ - if (corrected_overflow) - corrected_delta += (0x1UL << fb_mmu_fillunit_ecc_corrected_err_count_total_s()); - if (uncorrected_overflow) - uncorrected_delta += (0x1UL << fb_mmu_fillunit_ecc_uncorrected_err_count_total_s()); - - - g->ecc.fb.mmu_fillunit_corrected_err_count.counters[0] += - corrected_delta; - g->ecc.fb.mmu_fillunit_uncorrected_err_count.counters[0] += - uncorrected_delta; - - if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pte_data_m()) - nvgpu_log(g, gpu_dbg_intr, "corrected ecc pte data error"); - if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pte_data_m()) - nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pte data error"); - if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pde0_data_m()) - nvgpu_log(g, gpu_dbg_intr, "corrected ecc pde0 data error"); - if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pde0_data_m()) - nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pde0 data error"); - - if (corrected_overflow || uncorrected_overflow) - nvgpu_info(g, "mmu fillunit ecc counter overflow!"); - - nvgpu_log(g, gpu_dbg_intr, - "ecc error address: 0x%x", ecc_addr); - nvgpu_log(g, gpu_dbg_intr, - "ecc error count corrected: %d, uncorrected %d", - g->ecc.fb.mmu_fillunit_corrected_err_count.counters[0], - g->ecc.fb.mmu_fillunit_uncorrected_err_count.counters[0]); -} - -static void gv11b_fb_parse_mmfault(struct mmu_fault_info *mmfault) -{ - if (WARN_ON(mmfault->fault_type >= - ARRAY_SIZE(fault_type_descs_gv11b))) - mmfault->fault_type_desc = invalid_str; - else - mmfault->fault_type_desc = - fault_type_descs_gv11b[mmfault->fault_type]; - - if (WARN_ON(mmfault->client_type >= - ARRAY_SIZE(fault_client_type_descs_gv11b))) - mmfault->client_type_desc = invalid_str; - else - mmfault->client_type_desc = - fault_client_type_descs_gv11b[mmfault->client_type]; - - mmfault->client_id_desc = invalid_str; - if (mmfault->client_type == - gmmu_fault_client_type_hub_v()) { - - if (!(WARN_ON(mmfault->client_id >= - ARRAY_SIZE(hub_client_descs_gv11b)))) - mmfault->client_id_desc = - hub_client_descs_gv11b[mmfault->client_id]; - } else if (mmfault->client_type == - gmmu_fault_client_type_gpc_v()) { - if (!(WARN_ON(mmfault->client_id >= - ARRAY_SIZE(gpc_client_descs_gv11b)))) - mmfault->client_id_desc = - gpc_client_descs_gv11b[mmfault->client_id]; - } - -} - -static void gv11b_fb_print_fault_info(struct gk20a *g, - struct mmu_fault_info *mmfault) -{ - if (mmfault && mmfault->valid) { - nvgpu_err(g, "[MMU FAULT] " - "mmu engine id: %d, " - "ch id: %d, " - "fault addr: 0x%llx, " - "fault addr aperture: %d, " - "fault type: %s, " - "access type: %s, ", - mmfault->mmu_engine_id, - mmfault->chid, - mmfault->fault_addr, - mmfault->fault_addr_aperture, - mmfault->fault_type_desc, - fault_access_type_descs_gv11b[mmfault->access_type]); - nvgpu_err(g, "[MMU FAULT] " - "protected mode: %d, " - "client type: %s, " - "client id: %s, " - "gpc id if client type is gpc: %d, ", - mmfault->protected_mode, - mmfault->client_type_desc, - mmfault->client_id_desc, - mmfault->gpc_id); - - nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " - "faulted act eng id if any: 0x%x, " - "faulted veid if any: 0x%x, " - "faulted pbdma id if any: 0x%x, ", - mmfault->faulted_engine, - mmfault->faulted_subid, - mmfault->faulted_pbdma); - nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " - "inst ptr: 0x%llx, " - "inst ptr aperture: %d, " - "replayable fault: %d, " - "replayable fault en: %d " - "timestamp hi:lo 0x%08x:0x%08x, ", - mmfault->inst_ptr, - mmfault->inst_aperture, - mmfault->replayable_fault, - mmfault->replay_fault_en, - mmfault->timestamp_hi, mmfault->timestamp_lo); - } -} - -/* - *Fault buffer format - * - * 31 28 24 23 16 15 8 7 4 0 - *.-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. - *| inst_lo |0 0|apr|0 0 0 0 0 0 0 0| - *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' - *| inst_hi | - *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' - *| addr_31_12 | |AP | - *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' - *| addr_63_32 | - *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' - *| timestamp_lo | - *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' - *| timestamp_hi | - *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' - *| (reserved) | engine_id | - *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' - *|V|R|P| gpc_id |0 0 0|t|0|acctp|0| client |RF0 0|faulttype| - */ - -static void gv11b_fb_copy_from_hw_fault_buf(struct gk20a *g, - struct nvgpu_mem *mem, u32 offset, struct mmu_fault_info *mmfault) -{ - u32 rd32_val; - u32 addr_lo, addr_hi; - u64 inst_ptr; - u32 chid = FIFO_INVAL_CHANNEL_ID; - struct channel_gk20a *refch; - - memset(mmfault, 0, sizeof(*mmfault)); - - rd32_val = nvgpu_mem_rd32(g, mem, offset + - gmmu_fault_buf_entry_inst_lo_w()); - addr_lo = gmmu_fault_buf_entry_inst_lo_v(rd32_val); - addr_lo = addr_lo << ram_in_base_shift_v(); - - addr_hi = nvgpu_mem_rd32(g, mem, offset + - gmmu_fault_buf_entry_inst_hi_w()); - addr_hi = gmmu_fault_buf_entry_inst_hi_v(addr_hi); - - inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); - - /* refch will be put back after fault is handled */ - refch = gk20a_refch_from_inst_ptr(g, inst_ptr); - if (refch) - chid = refch->chid; - - /* it is ok to continue even if refch is NULL */ - mmfault->refch = refch; - mmfault->chid = chid; - mmfault->inst_ptr = inst_ptr; - mmfault->inst_aperture = gmmu_fault_buf_entry_inst_aperture_v(rd32_val); - - rd32_val = nvgpu_mem_rd32(g, mem, offset + - gmmu_fault_buf_entry_addr_lo_w()); - - mmfault->fault_addr_aperture = - gmmu_fault_buf_entry_addr_phys_aperture_v(rd32_val); - addr_lo = gmmu_fault_buf_entry_addr_lo_v(rd32_val); - addr_lo = addr_lo << ram_in_base_shift_v(); - - rd32_val = nvgpu_mem_rd32(g, mem, offset + - gmmu_fault_buf_entry_addr_hi_w()); - addr_hi = gmmu_fault_buf_entry_addr_hi_v(rd32_val); - mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); - - rd32_val = nvgpu_mem_rd32(g, mem, offset + - gmmu_fault_buf_entry_timestamp_lo_w()); - mmfault->timestamp_lo = - gmmu_fault_buf_entry_timestamp_lo_v(rd32_val); - - rd32_val = nvgpu_mem_rd32(g, mem, offset + - gmmu_fault_buf_entry_timestamp_hi_w()); - mmfault->timestamp_hi = - gmmu_fault_buf_entry_timestamp_hi_v(rd32_val); - - rd32_val = nvgpu_mem_rd32(g, mem, offset + - gmmu_fault_buf_entry_engine_id_w()); - - mmfault->mmu_engine_id = - gmmu_fault_buf_entry_engine_id_v(rd32_val); - gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, - &mmfault->faulted_engine, &mmfault->faulted_subid, - &mmfault->faulted_pbdma); - - rd32_val = nvgpu_mem_rd32(g, mem, offset + - gmmu_fault_buf_entry_fault_type_w()); - mmfault->client_id = - gmmu_fault_buf_entry_client_v(rd32_val); - mmfault->replayable_fault = - gmmu_fault_buf_entry_replayable_fault_v(rd32_val); - - mmfault->fault_type = - gmmu_fault_buf_entry_fault_type_v(rd32_val); - mmfault->access_type = - gmmu_fault_buf_entry_access_type_v(rd32_val); - - mmfault->client_type = - gmmu_fault_buf_entry_mmu_client_type_v(rd32_val); - - mmfault->gpc_id = - gmmu_fault_buf_entry_gpc_id_v(rd32_val); - mmfault->protected_mode = - gmmu_fault_buf_entry_protected_mode_v(rd32_val); - - mmfault->replay_fault_en = - gmmu_fault_buf_entry_replayable_fault_en_v(rd32_val); - - mmfault->valid = gmmu_fault_buf_entry_valid_v(rd32_val); - - rd32_val = nvgpu_mem_rd32(g, mem, offset + - gmmu_fault_buf_entry_fault_type_w()); - rd32_val &= ~(gmmu_fault_buf_entry_valid_m()); - nvgpu_mem_wr32(g, mem, offset + gmmu_fault_buf_entry_valid_w(), - rd32_val); - - gv11b_fb_parse_mmfault(mmfault); -} - -static void gv11b_fb_handle_mmu_fault_common(struct gk20a *g, - struct mmu_fault_info *mmfault, u32 *invalidate_replay_val) -{ - unsigned int id_type; - u32 num_lce, act_eng_bitmask = 0; - int err = 0; - u32 id = ((u32)~0); - - if (!mmfault->valid) - return; - - gv11b_fb_print_fault_info(g, mmfault); - - num_lce = gv11b_ce_get_num_lce(g); - if ((mmfault->mmu_engine_id >= - gmmu_fault_mmu_eng_id_ce0_v()) && - (mmfault->mmu_engine_id < - gmmu_fault_mmu_eng_id_ce0_v() + num_lce)) { - /* CE page faults are not reported as replayable */ - nvgpu_log(g, gpu_dbg_intr, "CE Faulted"); - err = gv11b_fb_fix_page_fault(g, mmfault); - gv11b_fifo_reset_pbdma_and_eng_faulted(g, mmfault->refch, - mmfault->faulted_pbdma, mmfault->faulted_engine); - if (!err) { - nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Fixed"); - *invalidate_replay_val = 0; - /* refch in mmfault is assigned at the time of copying - * fault info from snap reg or bar2 fault buf - */ - gk20a_channel_put(mmfault->refch); - return; - } - /* Do recovery. Channel recovery needs refch */ - nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Not Fixed"); - } - - if (!mmfault->replayable_fault) { - if (mmfault->fault_type == - gmmu_fault_type_unbound_inst_block_v()) { - /* - * Bug 1847172: When an engine faults due to an unbound - * instance block, the fault cannot be isolated to a - * single context so we need to reset the entire runlist - */ - id_type = ID_TYPE_UNKNOWN; - - } else if (mmfault->refch) { - if (gk20a_is_channel_marked_as_tsg(mmfault->refch)) { - id = mmfault->refch->tsgid; - id_type = ID_TYPE_TSG; - } else { - id = mmfault->chid; - id_type = ID_TYPE_CHANNEL; - } - if (mmfault->refch->mmu_nack_handled) { - /* We have already recovered for the same - * context, skip doing another recovery. - */ - mmfault->refch->mmu_nack_handled = false; - /* - * Recovery path can be entered twice for the - * same error in case of mmu nack. If mmu - * nack interrupt is handled before mmu fault - * then channel reference is increased to avoid - * closing the channel by userspace. Decrement - * channel reference. - */ - gk20a_channel_put(mmfault->refch); - /* refch in mmfault is assigned at the time - * of copying fault info from snap reg or bar2 - * fault buf. - */ - gk20a_channel_put(mmfault->refch); - return; - } - } else { - id_type = ID_TYPE_UNKNOWN; - } - if (mmfault->faulted_engine != FIFO_INVAL_ENGINE_ID) - act_eng_bitmask = BIT(mmfault->faulted_engine); - - /* Indicate recovery is handled if mmu fault is a result of - * mmu nack. - */ - mmfault->refch->mmu_nack_handled = true; - g->ops.fifo.teardown_ch_tsg(g, act_eng_bitmask, - id, id_type, RC_TYPE_MMU_FAULT, mmfault); - } else { - if (mmfault->fault_type == gmmu_fault_type_pte_v()) { - nvgpu_log(g, gpu_dbg_intr, "invalid pte! try to fix"); - err = gv11b_fb_fix_page_fault(g, mmfault); - if (err) - *invalidate_replay_val |= - fb_mmu_invalidate_replay_cancel_global_f(); - else - *invalidate_replay_val |= - fb_mmu_invalidate_replay_start_ack_all_f(); - } else { - /* cancel faults other than invalid pte */ - *invalidate_replay_val |= - fb_mmu_invalidate_replay_cancel_global_f(); - } - /* refch in mmfault is assigned at the time of copying - * fault info from snap reg or bar2 fault buf - */ - gk20a_channel_put(mmfault->refch); - } -} - -static int gv11b_fb_replay_or_cancel_faults(struct gk20a *g, - u32 invalidate_replay_val) -{ - int err = 0; - - nvgpu_log_fn(g, " "); - - if (invalidate_replay_val & - fb_mmu_invalidate_replay_cancel_global_f()) { - /* - * cancel faults so that next time it faults as - * replayable faults and channel recovery can be done - */ - err = g->ops.fb.mmu_invalidate_replay(g, - fb_mmu_invalidate_replay_cancel_global_f()); - } else if (invalidate_replay_val & - fb_mmu_invalidate_replay_start_ack_all_f()) { - /* pte valid is fixed. replay faulting request */ - err = g->ops.fb.mmu_invalidate_replay(g, - fb_mmu_invalidate_replay_start_ack_all_f()); - } - - return err; -} - -void gv11b_fb_handle_mmu_nonreplay_replay_fault(struct gk20a *g, - u32 fault_status, u32 index) -{ - u32 get_indx, offset, rd32_val, entries; - struct nvgpu_mem *mem; - struct mmu_fault_info *mmfault; - u32 invalidate_replay_val = 0; - u64 prev_fault_addr = 0ULL; - u64 next_fault_addr = 0ULL; - - if (gv11b_fb_is_fault_buffer_empty(g, index, &get_indx)) { - nvgpu_log(g, gpu_dbg_intr, - "SPURIOUS mmu fault: reg index:%d", index); - return; - } - nvgpu_log(g, gpu_dbg_intr, "%s MMU FAULT" , - index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX ? - "REPLAY" : "NON-REPLAY"); - - nvgpu_log(g, gpu_dbg_intr, "get ptr = %d", get_indx); - - mem = &g->mm.hw_fault_buf[index]; - mmfault = &g->mm.fault_info[index]; - - entries = gv11b_fb_fault_buffer_size_val(g, index); - nvgpu_log(g, gpu_dbg_intr, "buffer num entries = %d", entries); - - offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); - nvgpu_log(g, gpu_dbg_intr, "starting word offset = 0x%x", offset); - - rd32_val = nvgpu_mem_rd32(g, mem, - offset + gmmu_fault_buf_entry_valid_w()); - nvgpu_log(g, gpu_dbg_intr, "entry valid offset val = 0x%x", rd32_val); - - while ((rd32_val & gmmu_fault_buf_entry_valid_m())) { - - nvgpu_log(g, gpu_dbg_intr, "entry valid = 0x%x", rd32_val); - - gv11b_fb_copy_from_hw_fault_buf(g, mem, offset, mmfault); - - get_indx = (get_indx + 1) % entries; - nvgpu_log(g, gpu_dbg_intr, "new get index = %d", get_indx); - - gv11b_fb_fault_buffer_get_ptr_update(g, index, get_indx); - - offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); - nvgpu_log(g, gpu_dbg_intr, "next word offset = 0x%x", offset); - - rd32_val = nvgpu_mem_rd32(g, mem, - offset + gmmu_fault_buf_entry_valid_w()); - - if (index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX && - mmfault->fault_addr != 0ULL) { - /* fault_addr "0" is not supposed to be fixed ever. - * For the first time when prev = 0, next = 0 and - * fault addr is also 0 then handle_mmu_fault_common will - * not be called. Fix by checking fault_addr not equal to 0 - */ - prev_fault_addr = next_fault_addr; - next_fault_addr = mmfault->fault_addr; - if (prev_fault_addr == next_fault_addr) { - nvgpu_log(g, gpu_dbg_intr, "pte already scanned"); - if (mmfault->refch) - gk20a_channel_put(mmfault->refch); - continue; - } - } - - gv11b_fb_handle_mmu_fault_common(g, mmfault, - &invalidate_replay_val); - - } - if (index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX && - invalidate_replay_val != 0U) - gv11b_fb_replay_or_cancel_faults(g, invalidate_replay_val); -} - -static void gv11b_mm_copy_from_fault_snap_reg(struct gk20a *g, - u32 fault_status, struct mmu_fault_info *mmfault) -{ - u32 reg_val; - u32 addr_lo, addr_hi; - u64 inst_ptr; - int chid = FIFO_INVAL_CHANNEL_ID; - struct channel_gk20a *refch; - - memset(mmfault, 0, sizeof(*mmfault)); - - if (!(fault_status & fb_mmu_fault_status_valid_set_f())) { - - nvgpu_log(g, gpu_dbg_intr, "mmu fault status valid not set"); - return; - } - - g->ops.fb.read_mmu_fault_inst_lo_hi(g, ®_val, &addr_hi); - - addr_lo = fb_mmu_fault_inst_lo_addr_v(reg_val); - addr_lo = addr_lo << ram_in_base_shift_v(); - - addr_hi = fb_mmu_fault_inst_hi_addr_v(addr_hi); - inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); - - /* refch will be put back after fault is handled */ - refch = gk20a_refch_from_inst_ptr(g, inst_ptr); - if (refch) - chid = refch->chid; - - /* It is still ok to continue if refch is NULL */ - mmfault->refch = refch; - mmfault->chid = chid; - mmfault->inst_ptr = inst_ptr; - mmfault->inst_aperture = fb_mmu_fault_inst_lo_aperture_v(reg_val); - mmfault->mmu_engine_id = fb_mmu_fault_inst_lo_engine_id_v(reg_val); - - gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, - &mmfault->faulted_engine, &mmfault->faulted_subid, - &mmfault->faulted_pbdma); - - g->ops.fb.read_mmu_fault_addr_lo_hi(g, ®_val, &addr_hi); - - addr_lo = fb_mmu_fault_addr_lo_addr_v(reg_val); - addr_lo = addr_lo << ram_in_base_shift_v(); - - mmfault->fault_addr_aperture = - fb_mmu_fault_addr_lo_phys_aperture_v(reg_val); - - addr_hi = fb_mmu_fault_addr_hi_addr_v(addr_hi); - mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); - - reg_val = g->ops.fb.read_mmu_fault_info(g); - mmfault->fault_type = fb_mmu_fault_info_fault_type_v(reg_val); - mmfault->replayable_fault = - fb_mmu_fault_info_replayable_fault_v(reg_val); - mmfault->client_id = fb_mmu_fault_info_client_v(reg_val); - mmfault->access_type = fb_mmu_fault_info_access_type_v(reg_val); - mmfault->client_type = fb_mmu_fault_info_client_type_v(reg_val); - mmfault->gpc_id = fb_mmu_fault_info_gpc_id_v(reg_val); - mmfault->protected_mode = - fb_mmu_fault_info_protected_mode_v(reg_val); - mmfault->replay_fault_en = - fb_mmu_fault_info_replayable_fault_en_v(reg_val); - - mmfault->valid = fb_mmu_fault_info_valid_v(reg_val); - - fault_status &= ~(fb_mmu_fault_status_valid_m()); - g->ops.fb.write_mmu_fault_status(g, fault_status); - - gv11b_fb_parse_mmfault(mmfault); - -} - -void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g, - u32 fault_status) -{ - u32 reg_val; - u32 index = NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX; - - reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); - - if (fault_status & - fb_mmu_fault_status_replayable_getptr_corrupted_m()) { - - nvgpu_err(g, "replayable getptr corrupted set"); - - gv11b_fb_fault_buf_configure_hw(g, index); - - reg_val = set_field(reg_val, - fb_mmu_fault_buffer_get_getptr_corrupted_m(), - fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); - } - - if (fault_status & - fb_mmu_fault_status_replayable_overflow_m()) { - bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); - - nvgpu_err(g, "replayable overflow: buffer full:%s", - buffer_full?"true":"false"); - - reg_val = set_field(reg_val, - fb_mmu_fault_buffer_get_overflow_m(), - fb_mmu_fault_buffer_get_overflow_clear_f()); - } - - g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); -} - -void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g, - u32 fault_status) -{ - u32 reg_val; - u32 index = NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX; - - reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); - - if (fault_status & - fb_mmu_fault_status_non_replayable_getptr_corrupted_m()) { - - nvgpu_err(g, "non replayable getptr corrupted set"); - - gv11b_fb_fault_buf_configure_hw(g, index); - - reg_val = set_field(reg_val, - fb_mmu_fault_buffer_get_getptr_corrupted_m(), - fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); - } - - if (fault_status & - fb_mmu_fault_status_non_replayable_overflow_m()) { - - bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); - - nvgpu_err(g, "non replayable overflow: buffer full:%s", - buffer_full?"true":"false"); - - reg_val = set_field(reg_val, - fb_mmu_fault_buffer_get_overflow_m(), - fb_mmu_fault_buffer_get_overflow_clear_f()); - } - - g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); -} - -static void gv11b_fb_handle_bar2_fault(struct gk20a *g, - struct mmu_fault_info *mmfault, u32 fault_status) -{ - if (fault_status & fb_mmu_fault_status_non_replayable_error_m()) { - if (gv11b_fb_is_fault_buf_enabled(g, - NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) - gv11b_fb_fault_buf_configure_hw(g, NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX); - } - - if (fault_status & fb_mmu_fault_status_replayable_error_m()) { - if (gv11b_fb_is_fault_buf_enabled(g, - NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX)) - gv11b_fb_fault_buf_configure_hw(g, - NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); - } - gv11b_ce_mthd_buffer_fault_in_bar2_fault(g); - - g->ops.bus.bar2_bind(g, &g->mm.bar2.inst_block); - - if (mmfault->refch) { - gk20a_channel_put(mmfault->refch); - mmfault->refch = NULL; - } -} - -void gv11b_fb_handle_other_fault_notify(struct gk20a *g, - u32 fault_status) -{ - struct mmu_fault_info *mmfault; - u32 invalidate_replay_val = 0; - - mmfault = &g->mm.fault_info[NVGPU_MM_MMU_FAULT_TYPE_OTHER_AND_NONREPLAY]; - - gv11b_mm_copy_from_fault_snap_reg(g, fault_status, mmfault); - - /* BAR2/Physical faults will not be snapped in hw fault buf */ - if (mmfault->mmu_engine_id == gmmu_fault_mmu_eng_id_bar2_v()) { - nvgpu_err(g, "BAR2 MMU FAULT"); - gv11b_fb_handle_bar2_fault(g, mmfault, fault_status); - - } else if (mmfault->mmu_engine_id == - gmmu_fault_mmu_eng_id_physical_v()) { - /* usually means VPR or out of bounds physical accesses */ - nvgpu_err(g, "PHYSICAL MMU FAULT"); - - } else { - gv11b_fb_handle_mmu_fault_common(g, mmfault, - &invalidate_replay_val); - - if (invalidate_replay_val) - gv11b_fb_replay_or_cancel_faults(g, - invalidate_replay_val); - } -} - -void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status) -{ - u32 dropped_faults = 0; - - dropped_faults = fb_mmu_fault_status_dropped_bar1_phys_set_f() | - fb_mmu_fault_status_dropped_bar1_virt_set_f() | - fb_mmu_fault_status_dropped_bar2_phys_set_f() | - fb_mmu_fault_status_dropped_bar2_virt_set_f() | - fb_mmu_fault_status_dropped_ifb_phys_set_f() | - fb_mmu_fault_status_dropped_ifb_virt_set_f() | - fb_mmu_fault_status_dropped_other_phys_set_f()| - fb_mmu_fault_status_dropped_other_virt_set_f(); - - if (fault_status & dropped_faults) { - nvgpu_err(g, "dropped mmu fault (0x%08x)", - fault_status & dropped_faults); - g->ops.fb.write_mmu_fault_status(g, dropped_faults); - } -} - -void gv11b_fb_handle_replayable_mmu_fault(struct gk20a *g) -{ - u32 fault_status = gk20a_readl(g, fb_mmu_fault_status_r()); - - if (!(fault_status & fb_mmu_fault_status_replayable_m())) - return; - - if (gv11b_fb_is_fault_buf_enabled(g, - NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) { - gv11b_fb_handle_mmu_nonreplay_replay_fault(g, - fault_status, - NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); - } -} - -static void gv11b_fb_handle_mmu_fault(struct gk20a *g, u32 niso_intr) -{ - u32 fault_status = g->ops.fb.read_mmu_fault_status(g); - - nvgpu_log(g, gpu_dbg_intr, "mmu_fault_status = 0x%08x", fault_status); - - if (niso_intr & - fb_niso_intr_mmu_other_fault_notify_m()) { - - gv11b_fb_handle_dropped_mmu_fault(g, fault_status); - - gv11b_fb_handle_other_fault_notify(g, fault_status); - } - - if (gv11b_fb_is_fault_buf_enabled(g, NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) { - - if (niso_intr & - fb_niso_intr_mmu_nonreplayable_fault_notify_m()) { - - gv11b_fb_handle_mmu_nonreplay_replay_fault(g, - fault_status, - NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX); - - /* - * When all the faults are processed, - * GET and PUT will have same value and mmu fault status - * bit will be reset by HW - */ - } - if (niso_intr & - fb_niso_intr_mmu_nonreplayable_fault_overflow_m()) { - - gv11b_fb_handle_nonreplay_fault_overflow(g, - fault_status); - } - - } - - if (gv11b_fb_is_fault_buf_enabled(g, NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX)) { - - if (niso_intr & - fb_niso_intr_mmu_replayable_fault_notify_m()) { - - gv11b_fb_handle_mmu_nonreplay_replay_fault(g, - fault_status, - NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); - } - if (niso_intr & - fb_niso_intr_mmu_replayable_fault_overflow_m()) { - - gv11b_fb_handle_replay_fault_overflow(g, - fault_status); - } - - } - - nvgpu_log(g, gpu_dbg_intr, "clear mmu fault status"); - g->ops.fb.write_mmu_fault_status(g, - fb_mmu_fault_status_valid_clear_f()); -} - -void gv11b_fb_hub_isr(struct gk20a *g) -{ - u32 status, niso_intr; - - nvgpu_mutex_acquire(&g->mm.hub_isr_mutex); - - niso_intr = gk20a_readl(g, fb_niso_intr_r()); - - nvgpu_log(g, gpu_dbg_intr, "enter hub isr, niso_intr = 0x%08x", - niso_intr); - - if (niso_intr & - (fb_niso_intr_hub_access_counter_notify_m() | - fb_niso_intr_hub_access_counter_error_m())) { - - nvgpu_info(g, "hub access counter notify/error"); - } - if (niso_intr & - fb_niso_intr_mmu_ecc_uncorrected_error_notify_pending_f()) { - - nvgpu_info(g, "ecc uncorrected error notify"); - - status = gk20a_readl(g, fb_mmu_l2tlb_ecc_status_r()); - if (status) - gv11b_handle_l2tlb_ecc_isr(g, status); - - status = gk20a_readl(g, fb_mmu_hubtlb_ecc_status_r()); - if (status) - gv11b_handle_hubtlb_ecc_isr(g, status); - - status = gk20a_readl(g, fb_mmu_fillunit_ecc_status_r()); - if (status) - gv11b_handle_fillunit_ecc_isr(g, status); - } - if (niso_intr & - (fb_niso_intr_mmu_other_fault_notify_m() | - fb_niso_intr_mmu_replayable_fault_notify_m() | - fb_niso_intr_mmu_replayable_fault_overflow_m() | - fb_niso_intr_mmu_nonreplayable_fault_notify_m() | - fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) { - - nvgpu_log(g, gpu_dbg_intr, "MMU Fault"); - gv11b_fb_handle_mmu_fault(g, niso_intr); - } - - nvgpu_mutex_release(&g->mm.hub_isr_mutex); -} - -bool gv11b_fb_mmu_fault_pending(struct gk20a *g) -{ - if (gk20a_readl(g, fb_niso_intr_r()) & - (fb_niso_intr_mmu_other_fault_notify_m() | - fb_niso_intr_mmu_ecc_uncorrected_error_notify_m() | - fb_niso_intr_mmu_replayable_fault_notify_m() | - fb_niso_intr_mmu_replayable_fault_overflow_m() | - fb_niso_intr_mmu_nonreplayable_fault_notify_m() | - fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) - return true; - - return false; -} - -int gv11b_fb_mmu_invalidate_replay(struct gk20a *g, - u32 invalidate_replay_val) -{ - int err = -ETIMEDOUT; - u32 reg_val; - struct nvgpu_timeout timeout; - - nvgpu_log_fn(g, " "); - - nvgpu_mutex_acquire(&g->mm.tlb_lock); - - reg_val = gk20a_readl(g, fb_mmu_invalidate_r()); - - reg_val |= fb_mmu_invalidate_all_va_true_f() | - fb_mmu_invalidate_all_pdb_true_f() | - invalidate_replay_val | - fb_mmu_invalidate_trigger_true_f(); - - gk20a_writel(g, fb_mmu_invalidate_r(), reg_val); - - /* retry 200 times */ - nvgpu_timeout_init(g, &timeout, 200, NVGPU_TIMER_RETRY_TIMER); - do { - reg_val = gk20a_readl(g, fb_mmu_ctrl_r()); - if (fb_mmu_ctrl_pri_fifo_empty_v(reg_val) != - fb_mmu_ctrl_pri_fifo_empty_false_f()) { - err = 0; - break; - } - nvgpu_udelay(5); - } while (!nvgpu_timeout_expired_msg(&timeout, - "invalidate replay failed on 0x%llx")); - if (err) - nvgpu_err(g, "invalidate replay timedout"); - - nvgpu_mutex_release(&g->mm.tlb_lock); - - return err; -} - -static int gv11b_fb_fix_page_fault(struct gk20a *g, - struct mmu_fault_info *mmfault) -{ - int err = 0; - u32 pte[2]; - - if (mmfault->refch == NULL) { - nvgpu_log(g, gpu_dbg_intr, "refch from mmu_fault_info is NULL"); - return -EINVAL; - } - - err = __nvgpu_get_pte(g, - mmfault->refch->vm, mmfault->fault_addr, &pte[0]); - if (err) { - nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not found"); - return err; - } - nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, - "pte: %#08x %#08x", pte[1], pte[0]); - - if (pte[0] == 0x0 && pte[1] == 0x0) { - nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, - "pte all zeros, do not set valid"); - return -1; - } - if (pte[0] & gmmu_new_pte_valid_true_f()) { - nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, - "pte valid already set"); - return -1; - } - - pte[0] |= gmmu_new_pte_valid_true_f(); - if (pte[0] & gmmu_new_pte_read_only_true_f()) - pte[0] &= ~(gmmu_new_pte_read_only_true_f()); - nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, - "new pte: %#08x %#08x", pte[1], pte[0]); - - err = __nvgpu_set_pte(g, - mmfault->refch->vm, mmfault->fault_addr, &pte[0]); - if (err) { - nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not fixed"); - return err; - } - /* invalidate tlb so that GMMU does not use old cached translation */ - g->ops.fb.tlb_invalidate(g, mmfault->refch->vm->pdb.mem); - - err = __nvgpu_get_pte(g, - mmfault->refch->vm, mmfault->fault_addr, &pte[0]); - nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, - "pte after tlb invalidate: %#08x %#08x", - pte[1], pte[0]); - return err; -} - -void fb_gv11b_write_mmu_fault_buffer_lo_hi(struct gk20a *g, u32 index, - u32 addr_lo, u32 addr_hi) -{ - nvgpu_writel(g, fb_mmu_fault_buffer_lo_r(index), addr_lo); - nvgpu_writel(g, fb_mmu_fault_buffer_hi_r(index), addr_hi); -} - -u32 fb_gv11b_read_mmu_fault_buffer_get(struct gk20a *g, u32 index) -{ - return nvgpu_readl(g, fb_mmu_fault_buffer_get_r(index)); -} - -void fb_gv11b_write_mmu_fault_buffer_get(struct gk20a *g, u32 index, - u32 reg_val) -{ - nvgpu_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val); -} - -u32 fb_gv11b_read_mmu_fault_buffer_put(struct gk20a *g, u32 index) -{ - return nvgpu_readl(g, fb_mmu_fault_buffer_put_r(index)); -} - -u32 fb_gv11b_read_mmu_fault_buffer_size(struct gk20a *g, u32 index) -{ - return nvgpu_readl(g, fb_mmu_fault_buffer_size_r(index)); -} - -void fb_gv11b_write_mmu_fault_buffer_size(struct gk20a *g, u32 index, - u32 reg_val) -{ - nvgpu_writel(g, fb_mmu_fault_buffer_size_r(index), reg_val); -} - -void fb_gv11b_read_mmu_fault_addr_lo_hi(struct gk20a *g, - u32 *addr_lo, u32 *addr_hi) -{ - *addr_lo = nvgpu_readl(g, fb_mmu_fault_addr_lo_r()); - *addr_hi = nvgpu_readl(g, fb_mmu_fault_addr_hi_r()); -} - -void fb_gv11b_read_mmu_fault_inst_lo_hi(struct gk20a *g, - u32 *inst_lo, u32 *inst_hi) -{ - *inst_lo = nvgpu_readl(g, fb_mmu_fault_inst_lo_r()); - *inst_hi = nvgpu_readl(g, fb_mmu_fault_inst_hi_r()); -} - -u32 fb_gv11b_read_mmu_fault_info(struct gk20a *g) -{ - return nvgpu_readl(g, fb_mmu_fault_info_r()); -} - -u32 fb_gv11b_read_mmu_fault_status(struct gk20a *g) -{ - return nvgpu_readl(g, fb_mmu_fault_status_r()); -} - -void fb_gv11b_write_mmu_fault_status(struct gk20a *g, u32 reg_val) -{ - nvgpu_writel(g, fb_mmu_fault_status_r(), reg_val); -} diff --git a/drivers/gpu/nvgpu/gv11b/fb_gv11b.h b/drivers/gpu/nvgpu/gv11b/fb_gv11b.h deleted file mode 100644 index fe2c733d..00000000 --- a/drivers/gpu/nvgpu/gv11b/fb_gv11b.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * GV11B FB - * - * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#ifndef _NVGPU_GV11B_FB -#define _NVGPU_GV11B_FB - -#define NONREPLAY_REG_INDEX 0 -#define REPLAY_REG_INDEX 1 - -struct gk20a; - -void gv11b_fb_init_hw(struct gk20a *g); - -void gv11b_fb_init_fs_state(struct gk20a *g); -void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr); -void gv11b_fb_reset(struct gk20a *g); -void gv11b_fb_hub_isr(struct gk20a *g); - -bool gv11b_fb_is_fault_buf_enabled(struct gk20a *g, u32 index ); -void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g, - u32 index, u32 state); -void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, u32 index); -void gv11b_fb_enable_hub_intr(struct gk20a *g); -void gv11b_fb_disable_hub_intr(struct gk20a *g); -bool gv11b_fb_mmu_fault_pending(struct gk20a *g); -void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status); -void gv11b_fb_handle_other_fault_notify(struct gk20a *g, - u32 fault_status); -void gv11b_fb_handle_mmu_nonreplay_replay_fault(struct gk20a *g, - u32 fault_status, u32 index); -void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g, - u32 fault_status); -void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g, - u32 fault_status); -void gv11b_fb_handle_replayable_mmu_fault(struct gk20a *g); -void gv11b_handle_l2tlb_ecc_isr(struct gk20a *g, u32 ecc_status); -void gv11b_handle_hubtlb_ecc_isr(struct gk20a *g, u32 ecc_status); -void gv11b_handle_fillunit_ecc_isr(struct gk20a *g, u32 ecc_status); - -void fb_gv11b_write_mmu_fault_buffer_lo_hi(struct gk20a *g, u32 index, - u32 addr_lo, u32 addr_hi); -u32 fb_gv11b_read_mmu_fault_buffer_get(struct gk20a *g, u32 index); -void fb_gv11b_write_mmu_fault_buffer_get(struct gk20a *g, u32 index, - u32 reg_val); -u32 fb_gv11b_read_mmu_fault_buffer_put(struct gk20a *g, u32 index); -u32 fb_gv11b_read_mmu_fault_buffer_size(struct gk20a *g, u32 index); -void fb_gv11b_write_mmu_fault_buffer_size(struct gk20a *g, u32 index, - u32 reg_val); -void fb_gv11b_read_mmu_fault_addr_lo_hi(struct gk20a *g, - u32 *addr_lo, u32 *addr_hi); -void fb_gv11b_read_mmu_fault_inst_lo_hi(struct gk20a *g, - u32 *inst_lo, u32 *inst_hi); -u32 fb_gv11b_read_mmu_fault_info(struct gk20a *g); -u32 fb_gv11b_read_mmu_fault_status(struct gk20a *g); -void fb_gv11b_write_mmu_fault_status(struct gk20a *g, u32 reg_val); - -int gv11b_fb_mmu_invalidate_replay(struct gk20a *g, - u32 invalidate_replay_val); - -#endif diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c index 09f74042..325285a6 100644 --- a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c @@ -27,6 +27,10 @@ #include "common/bus/bus_gm20b.h" #include "common/clock_gating/gv11b_gating_reglist.h" #include "common/ptimer/ptimer_gk20a.h" +#include "common/fb/fb_gk20a.h" +#include "common/fb/fb_gm20b.h" +#include "common/fb/fb_gp10b.h" +#include "common/fb/fb_gv11b.h" #include "gk20a/gk20a.h" #include "gk20a/fifo_gk20a.h" @@ -37,13 +41,11 @@ #include "gk20a/dbg_gpu_gk20a.h" #include "gk20a/flcn_gk20a.h" #include "gk20a/regops_gk20a.h" -#include "gk20a/fb_gk20a.h" #include "gk20a/pmu_gk20a.h" #include "gk20a/gr_gk20a.h" #include "gm20b/ltc_gm20b.h" #include "gm20b/gr_gm20b.h" -#include "gm20b/fb_gm20b.h" #include "gm20b/fifo_gm20b.h" #include "gm20b/mm_gm20b.h" #include "gm20b/acr_gm20b.h" @@ -56,7 +58,6 @@ #include "gp10b/priv_ring_gp10b.h" #include "gp10b/fifo_gp10b.h" #include "gp10b/fecs_trace_gp10b.h" -#include "gp10b/fb_gp10b.h" #include "gp10b/mm_gp10b.h" #include "gp10b/pmu_gp10b.h" #include "gp10b/gr_gp10b.h" @@ -79,7 +80,6 @@ #include "mm_gv11b.h" #include "pmu_gv11b.h" #include "acr_gv11b.h" -#include "fb_gv11b.h" #include "fifo_gv11b.h" #include "regops_gv11b.h" #include "subctx_gv11b.h" diff --git a/drivers/gpu/nvgpu/gv11b/mc_gv11b.c b/drivers/gpu/nvgpu/gv11b/mc_gv11b.c index b7a3ce62..64680fc6 100644 --- a/drivers/gpu/nvgpu/gv11b/mc_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/mc_gv11b.c @@ -29,7 +29,6 @@ #include "gp10b/mc_gp10b.h" #include "mc_gv11b.h" -#include "fb_gv11b.h" #include diff --git a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c index 349548cd..421f3692 100644 --- a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c +++ b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c @@ -23,6 +23,9 @@ #include "common/bus/bus_gk20a.h" #include "common/bus/bus_gm20b.h" #include "common/clock_gating/gp10b_gating_reglist.h" +#include "common/fb/fb_gk20a.h" +#include "common/fb/fb_gm20b.h" +#include "common/fb/fb_gp10b.h" #include "vgpu/fifo_vgpu.h" #include "vgpu/gr_vgpu.h" @@ -40,14 +43,12 @@ #include "gk20a/flcn_gk20a.h" #include "gk20a/mc_gk20a.h" -#include "gk20a/fb_gk20a.h" #include "gk20a/dbg_gpu_gk20a.h" #include "gp10b/mc_gp10b.h" #include "gp10b/ltc_gp10b.h" #include "gp10b/mm_gp10b.h" #include "gp10b/ce_gp10b.h" -#include "gp10b/fb_gp10b.h" #include "gp10b/pmu_gp10b.h" #include "gp10b/gr_gp10b.h" #include "gp10b/gr_ctx_gp10b.h" @@ -61,7 +62,6 @@ #include "gm20b/fifo_gm20b.h" #include "gm20b/acr_gm20b.h" #include "gm20b/pmu_gm20b.h" -#include "gm20b/fb_gm20b.h" #include "gm20b/mm_gm20b.h" #include diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_hal_gv11b.c b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_hal_gv11b.c index 577c8f6e..38f9a184 100644 --- a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_hal_gv11b.c +++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_hal_gv11b.c @@ -23,6 +23,10 @@ #include "common/bus/bus_gk20a.h" #include "common/bus/bus_gm20b.h" #include "common/clock_gating/gv11b_gating_reglist.h" +#include "common/fb/fb_gk20a.h" +#include "common/fb/fb_gm20b.h" +#include "common/fb/fb_gp10b.h" +#include "common/fb/fb_gv11b.h" #include #include @@ -40,20 +44,17 @@ #include "vgpu/gp10b/vgpu_mm_gp10b.h" #include "vgpu/gp10b/vgpu_gr_gp10b.h" -#include #include #include #include "gk20a/dbg_gpu_gk20a.h" #include -#include #include #include #include #include #include -#include #include #include #include @@ -67,7 +68,6 @@ #include #include -#include #include #include #include -- cgit v1.2.2