From f56874aec2ec61f2c341b813cc76de5acc51ea12 Mon Sep 17 00:00:00 2001 From: Ranjanikar Nikhil Prabhakarrao Date: Thu, 13 Dec 2018 17:29:20 +0530 Subject: gpu: nvgpu: add speculative barrier Data can be speculativerly stored and code flow can be hijacked. To mitigate this problem insert a speculation barrier. Bug 200447167 Change-Id: Ia865ff2add8b30de49aa970715625b13e8f71c08 Signed-off-by: Ranjanikar Nikhil Prabhakarrao Reviewed-on: https://git-master.nvidia.com/r/1972221 (cherry picked from commit f0762ed4831b3fe6cc953a4a4ec26c2537dcb69f) Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/1996052 Reviewed-by: automaticguardword Reviewed-by: Deepak Nibade Reviewed-by: mobile promotions GVS: Gerrit_Virtual_Submit Tested-by: Deepak Nibade Tested-by: mobile promotions --- drivers/gpu/nvgpu/tu104/gr_tu104.c | 549 +++++++++++++++++++++++++++++++++++++ 1 file changed, 549 insertions(+) create mode 100644 drivers/gpu/nvgpu/tu104/gr_tu104.c (limited to 'drivers/gpu/nvgpu/tu104/gr_tu104.c') diff --git a/drivers/gpu/nvgpu/tu104/gr_tu104.c b/drivers/gpu/nvgpu/tu104/gr_tu104.c new file mode 100644 index 00000000..fa6995ac --- /dev/null +++ b/drivers/gpu/nvgpu/tu104/gr_tu104.c @@ -0,0 +1,549 @@ +<<<<<<< HEAD (bbef4c gpu: nvgpu: initialize masks for the perfmon counters 3) +======= +/* + * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "gk20a/gr_gk20a.h" +#include "gk20a/gr_pri_gk20a.h" + +#include "gp10b/gr_gp10b.h" + +#include "gv11b/gr_gv11b.h" + +#include "tu104/gr_tu104.h" + +#include + +bool gr_tu104_is_valid_class(struct gk20a *g, u32 class_num) +{ + nvgpu_speculation_barrier(); + switch (class_num) { + case TURING_CHANNEL_GPFIFO_A: + case TURING_A: + case TURING_COMPUTE_A: + case TURING_DMA_COPY_A: + return true; + default: + break; + } + + return gr_gv11b_is_valid_class(g, class_num); +}; + +bool gr_tu104_is_valid_gfx_class(struct gk20a *g, u32 class_num) +{ + nvgpu_speculation_barrier(); + switch (class_num) { + case TURING_A: + return true; + default: + break; + } + + return gr_gv11b_is_valid_gfx_class(g, class_num); +} + +bool gr_tu104_is_valid_compute_class(struct gk20a *g, u32 class_num) +{ + nvgpu_speculation_barrier(); + switch (class_num) { + case TURING_COMPUTE_A: + return true; + default: + break; + } + + return gr_gv11b_is_valid_compute_class(g, class_num); +} + +int gr_tu104_init_sw_bundle64(struct gk20a *g) +{ + u32 i; + u32 last_bundle_data_lo = 0; + u32 last_bundle_data_hi = 0; + int err = 0; + struct netlist_av64_list *sw_bundle64_init = + &g->netlist_vars->sw_bundle64_init; + + for (i = 0U; i < sw_bundle64_init->count; i++) { + if (i == 0U || + (last_bundle_data_lo != sw_bundle64_init->l[i].value_lo) || + (last_bundle_data_hi != sw_bundle64_init->l[i].value_hi)) { + nvgpu_writel(g, gr_pipe_bundle_data_r(), + sw_bundle64_init->l[i].value_lo); + nvgpu_writel(g, gr_pipe_bundle_data_hi_r(), + sw_bundle64_init->l[i].value_hi); + + last_bundle_data_lo = sw_bundle64_init->l[i].value_lo; + last_bundle_data_hi = sw_bundle64_init->l[i].value_hi; + } + + nvgpu_writel(g, gr_pipe_bundle_address_r(), + sw_bundle64_init->l[i].addr); + + if (gr_pipe_bundle_address_value_v(sw_bundle64_init->l[i].addr) + == GR_GO_IDLE_BUNDLE) { + err = gr_gk20a_wait_idle(g, + gk20a_get_gr_idle_timeout(g), + GR_IDLE_CHECK_DEFAULT); + } else if (nvgpu_platform_is_silicon(g)) { + err = gr_gk20a_wait_fe_idle(g, + gk20a_get_gr_idle_timeout(g), + GR_IDLE_CHECK_DEFAULT); + } + if (err != 0) { + break; + } + } + + return err; +} + +int gr_tu104_alloc_global_ctx_buffers(struct gk20a *g) +{ + int err; + struct gr_gk20a *gr = &g->gr; + u32 rtv_circular_buffer_size; + + nvgpu_log_fn(g, " "); + + rtv_circular_buffer_size = + (gr_scc_rm_rtv_cb_size_div_256b_default_f() + + gr_scc_rm_rtv_cb_size_div_256b_db_adder_f()) * + gr_scc_bundle_cb_size_div_256b_byte_granularity_v(); + nvgpu_log_info(g, "rtv_circular_buffer_size : %u", + rtv_circular_buffer_size); + + err = gk20a_gr_alloc_ctx_buffer(g, + &gr->global_ctx_buffer[RTV_CIRCULAR_BUFFER], + rtv_circular_buffer_size); + if (err != 0) { + return err; + } + + err = gr_gk20a_alloc_global_ctx_buffers(g); + if (err != 0) { + goto clean_up; + } + + return 0; + +clean_up: + nvgpu_err(g, "fail"); + gk20a_gr_destroy_ctx_buffer(g, + &gr->global_ctx_buffer[RTV_CIRCULAR_BUFFER]); + + return err; +} + +int gr_tu104_map_global_ctx_buffers(struct gk20a *g, struct vm_gk20a *vm, + struct nvgpu_gr_ctx *gr_ctx, bool vpr) +{ + int err; + u64 *g_bfr_va; + u64 *g_bfr_size; + int *g_bfr_index; + struct gr_gk20a *gr = &g->gr; + struct nvgpu_mem *mem; + u64 gpu_va; + + nvgpu_log_fn(g, " "); + + g_bfr_va = gr_ctx->global_ctx_buffer_va; + g_bfr_size = gr_ctx->global_ctx_buffer_size; + g_bfr_index = gr_ctx->global_ctx_buffer_index; + + /* RTV circular buffer */ + mem = &gr->global_ctx_buffer[RTV_CIRCULAR_BUFFER].mem; + gpu_va = nvgpu_gmmu_map(vm, mem, mem->size, 0, + gk20a_mem_flag_none, true, mem->aperture); + if (gpu_va == 0ULL) { + return -ENOMEM; + } + + g_bfr_va[RTV_CIRCULAR_BUFFER_VA] = gpu_va; + g_bfr_size[RTV_CIRCULAR_BUFFER_VA] = mem->size; + g_bfr_index[RTV_CIRCULAR_BUFFER_VA] = RTV_CIRCULAR_BUFFER; + + err = gr_gk20a_map_global_ctx_buffers(g, vm, gr_ctx, vpr); + if (err != 0) { + goto clean_up; + } + + return 0; + +clean_up: + nvgpu_err(g, "fail"); + nvgpu_gmmu_unmap(vm, mem, gpu_va); + + return err; +} + +static void gr_tu104_commit_rtv_circular_buffer(struct gk20a *g, + struct nvgpu_gr_ctx *gr_ctx, + u64 addr, u32 size, u32 gfxpAddSize, bool patch) +{ + gr_gk20a_ctx_patch_write(g, gr_ctx, gr_scc_rm_rtv_cb_base_r(), + gr_scc_rm_rtv_cb_base_addr_39_8_f(addr), patch); + gr_gk20a_ctx_patch_write(g, gr_ctx, gr_scc_rm_rtv_cb_size_r(), + gr_scc_rm_rtv_cb_size_div_256b_f(size), patch); + gr_gk20a_ctx_patch_write(g, gr_ctx, gr_gpcs_gcc_rm_rtv_cb_base_r(), + gr_gpcs_gcc_rm_rtv_cb_base_addr_39_8_f(addr), patch); + gr_gk20a_ctx_patch_write(g, gr_ctx, gr_scc_rm_gfxp_reserve_r(), + gr_scc_rm_gfxp_reserve_rtv_cb_size_div_256b_f(gfxpAddSize), + patch); +} + +int gr_tu104_commit_global_ctx_buffers(struct gk20a *g, + struct nvgpu_gr_ctx *gr_ctx, bool patch) +{ + int err; + u64 addr; + u32 size; + u32 gfxpaddsize = 0; + + nvgpu_log_fn(g, " "); + + err = gr_gk20a_commit_global_ctx_buffers(g, gr_ctx, patch); + if (err != 0) { + return err; + } + + if (patch) { + int err; + err = gr_gk20a_ctx_patch_write_begin(g, gr_ctx, false); + if (err != 0) { + return err; + } + } + + /* RTV circular buffer */ + addr = gr_ctx->global_ctx_buffer_va[RTV_CIRCULAR_BUFFER_VA] >> + U64(gr_scc_rm_rtv_cb_base_addr_39_8_align_bits_f()); + + size = (gr_scc_rm_rtv_cb_size_div_256b_default_f() + + gr_scc_rm_rtv_cb_size_div_256b_db_adder_f()); + + gr_tu104_commit_rtv_circular_buffer(g, gr_ctx, addr, size, + gfxpaddsize, patch); + + if (patch) { + gr_gk20a_ctx_patch_write_end(g, gr_ctx, false); + } + + return 0; +} + +int gr_tu104_alloc_gfxp_rtv_cb(struct gk20a *g, + struct nvgpu_gr_ctx *gr_ctx, struct vm_gk20a *vm) +{ + int err; + u32 rtv_cb_size; + + nvgpu_log_fn(g, " "); + + rtv_cb_size = + (gr_scc_rm_rtv_cb_size_div_256b_default_f() + + gr_scc_rm_rtv_cb_size_div_256b_db_adder_f() + + gr_scc_rm_rtv_cb_size_div_256b_gfxp_adder_f()) * + gr_scc_rm_rtv_cb_size_div_256b_byte_granularity_v(); + + err = gr_gp10b_alloc_buffer(vm, + rtv_cb_size, + &gr_ctx->gfxp_rtvcb_ctxsw_buffer); + + return err; +} + +void gr_tu104_commit_gfxp_rtv_cb(struct gk20a *g, + struct nvgpu_gr_ctx *gr_ctx, bool patch) +{ + u64 addr; + u32 rtv_cb_size; + u32 gfxp_addr_size; + + nvgpu_log_fn(g, " "); + + rtv_cb_size = + (gr_scc_rm_rtv_cb_size_div_256b_default_f() + + gr_scc_rm_rtv_cb_size_div_256b_db_adder_f() + + gr_scc_rm_rtv_cb_size_div_256b_gfxp_adder_f()); + gfxp_addr_size = gr_scc_rm_rtv_cb_size_div_256b_gfxp_adder_f(); + + /* GFXP RTV circular buffer */ + addr = (u64)(u64_lo32(gr_ctx->gfxp_rtvcb_ctxsw_buffer.gpu_va) >> + gr_scc_rm_rtv_cb_base_addr_39_8_align_bits_f()) | + (u64)(u64_hi32(gr_ctx->gfxp_rtvcb_ctxsw_buffer.gpu_va) << + (32U - gr_scc_rm_rtv_cb_base_addr_39_8_align_bits_f())); + + + gr_tu104_commit_rtv_circular_buffer(g, gr_ctx, addr, + rtv_cb_size, + gfxp_addr_size, + patch); +} + +void gr_tu104_bundle_cb_defaults(struct gk20a *g) +{ + struct gr_gk20a *gr = &g->gr; + + gr->bundle_cb_default_size = + gr_scc_bundle_cb_size_div_256b__prod_v(); + gr->min_gpm_fifo_depth = + gr_pd_ab_dist_cfg2_state_limit_min_gpm_fifo_depths_v(); + gr->bundle_cb_token_limit = + gr_pd_ab_dist_cfg2_token_limit_init_v(); +} + +void gr_tu104_cb_size_default(struct gk20a *g) +{ + struct gr_gk20a *gr = &g->gr; + + if (gr->attrib_cb_default_size == 0U) { + gr->attrib_cb_default_size = + gr_gpc0_ppc0_cbm_beta_cb_size_v_default_v(); + } + gr->alpha_cb_default_size = + gr_gpc0_ppc0_cbm_alpha_cb_size_v_default_v(); + gr->attrib_cb_gfxp_default_size = + gr_gpc0_ppc0_cbm_beta_cb_size_v_gfxp_v(); + gr->attrib_cb_gfxp_size = + gr_gpc0_ppc0_cbm_beta_cb_size_v_gfxp_v(); +} + +void gr_tu104_free_gr_ctx(struct gk20a *g, + struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx) +{ + nvgpu_log_fn(g, " "); + + if (gr_ctx != NULL) { + nvgpu_dma_unmap_free(vm, &gr_ctx->gfxp_rtvcb_ctxsw_buffer); + } + + gr_gk20a_free_gr_ctx(g, vm, gr_ctx); +} + +void gr_tu104_enable_gpc_exceptions(struct gk20a *g) +{ + struct gr_gk20a *gr = &g->gr; + u32 tpc_mask; + + gk20a_writel(g, gr_gpcs_tpcs_tpccs_tpc_exception_en_r(), + gr_gpcs_tpcs_tpccs_tpc_exception_en_sm_enabled_f()); + + tpc_mask = + gr_gpcs_gpccs_gpc_exception_en_tpc_f((1 << gr->max_tpc_per_gpc_count) - 1); + + gk20a_writel(g, gr_gpcs_gpccs_gpc_exception_en_r(), + (tpc_mask | gr_gpcs_gpccs_gpc_exception_en_gcc_f(1) | + gr_gpcs_gpccs_gpc_exception_en_gpccs_f(1) | + gr_gpcs_gpccs_gpc_exception_en_gpcmmu_f(1))); +} + +int gr_tu104_get_offset_in_gpccs_segment(struct gk20a *g, + enum ctxsw_addr_type addr_type, + u32 num_tpcs, + u32 num_ppcs, + u32 reg_list_ppc_count, + u32 *__offset_in_segment) +{ + u32 offset_in_segment = 0; + u32 num_pes_per_gpc = nvgpu_get_litter_value(g, + GPU_LIT_NUM_PES_PER_GPC); + + if (addr_type == CTXSW_ADDR_TYPE_TPC) { + /* + * reg = g->netlist_vars->ctxsw_regs.tpc.l; + * offset_in_segment = 0; + */ + } else if (addr_type == CTXSW_ADDR_TYPE_PPC) { + /* + * The ucode stores TPC data before PPC data. + * Advance offset past TPC data to PPC data. + */ + offset_in_segment = + ((g->netlist_vars->ctxsw_regs.tpc.count * + num_tpcs) << 2); + } else if (addr_type == CTXSW_ADDR_TYPE_GPC) { + /* + * The ucode stores TPC/PPC data before GPC data. + * Advance offset past TPC/PPC data to GPC data. + * + * Note 1 PES_PER_GPC case + */ + if (num_pes_per_gpc > 1U) { + offset_in_segment = + (((g->netlist_vars->ctxsw_regs.tpc.count * + num_tpcs) << 2) + + ((reg_list_ppc_count * num_ppcs) << 2)); + } else { + offset_in_segment = + ((g->netlist_vars->ctxsw_regs.tpc.count * + num_tpcs) << 2); + } + } else if ((addr_type == CTXSW_ADDR_TYPE_EGPC) || + (addr_type == CTXSW_ADDR_TYPE_ETPC)) { + if (num_pes_per_gpc > 1U) { + offset_in_segment = + ((g->netlist_vars->ctxsw_regs.tpc.count * + num_tpcs) << 2) + + ((reg_list_ppc_count * num_ppcs) << 2) + + (g->netlist_vars->ctxsw_regs.gpc.count << 2); + } else { + offset_in_segment = + ((g->netlist_vars->ctxsw_regs.tpc.count * + num_tpcs) << 2) + + (g->netlist_vars->ctxsw_regs.gpc.count << 2); + } + + /* aligned to next 256 byte */ + offset_in_segment = ALIGN(offset_in_segment, 256); + + nvgpu_log(g, gpu_dbg_info | gpu_dbg_gpu_dbg, + "egpc etpc offset_in_segment 0x%#08x", + offset_in_segment); + } else { + nvgpu_log_fn(g, "Unknown address type."); + return -EINVAL; + } + + *__offset_in_segment = offset_in_segment; + return 0; +} + +static void gr_tu104_set_sm_disp_ctrl(struct gk20a *g, u32 data) +{ + u32 reg_val; + + nvgpu_log_fn(g, " "); + + reg_val = nvgpu_readl(g, gr_gpcs_tpcs_sm_disp_ctrl_r()); + + if ((data & NVC5C0_SET_SM_DISP_CTRL_COMPUTE_SHADER_QUAD_MASK) + == NVC5C0_SET_SM_DISP_CTRL_COMPUTE_SHADER_QUAD_DISABLE) { + reg_val = set_field(reg_val, + gr_gpcs_tpcs_sm_disp_ctrl_compute_shader_quad_m(), + gr_gpcs_tpcs_sm_disp_ctrl_compute_shader_quad_disable_f() + ); + } else if ((data & NVC5C0_SET_SM_DISP_CTRL_COMPUTE_SHADER_QUAD_MASK) + == NVC5C0_SET_SM_DISP_CTRL_COMPUTE_SHADER_QUAD_ENABLE) { + reg_val = set_field(reg_val, + gr_gpcs_tpcs_sm_disp_ctrl_compute_shader_quad_m(), + gr_gpcs_tpcs_sm_disp_ctrl_compute_shader_quad_enable_f() + ); + } + + nvgpu_writel(g, gr_gpcs_tpcs_sm_disp_ctrl_r(), reg_val); +} + +int gr_tu104_handle_sw_method(struct gk20a *g, u32 addr, + u32 class_num, u32 offset, u32 data) +{ + nvgpu_log_fn(g, " "); + + if (class_num == TURING_COMPUTE_A) { + switch (offset << 2) { + case NVC5C0_SET_SHADER_EXCEPTIONS: + gv11b_gr_set_shader_exceptions(g, data); + break; + case NVC5C0_SET_SKEDCHECK: + gr_gv11b_set_skedcheck(g, data); + break; + case NVC5C0_SET_SM_DISP_CTRL: + gr_tu104_set_sm_disp_ctrl(g, data); + break; + case NVC5C0_SET_SHADER_CUT_COLLECTOR: + gr_gv11b_set_shader_cut_collector(g, data); + break; + default: + goto fail; + } + } + + if (class_num == TURING_A) { + switch (offset << 2) { + case NVC597_SET_SHADER_EXCEPTIONS: + gv11b_gr_set_shader_exceptions(g, data); + break; + case NVC597_SET_CIRCULAR_BUFFER_SIZE: + g->ops.gr.set_circular_buffer_size(g, data); + break; + case NVC597_SET_ALPHA_CIRCULAR_BUFFER_SIZE: + g->ops.gr.set_alpha_circular_buffer_size(g, data); + break; + case NVC597_SET_GO_IDLE_TIMEOUT: + gr_gv11b_set_go_idle_timeout(g, data); + break; + case NVC097_SET_COALESCE_BUFFER_SIZE: + gr_gv11b_set_coalesce_buffer_size(g, data); + break; + case NVC597_SET_TEX_IN_DBG: + gr_gv11b_set_tex_in_dbg(g, data); + break; + case NVC597_SET_SKEDCHECK: + gr_gv11b_set_skedcheck(g, data); + break; + case NVC597_SET_BES_CROP_DEBUG3: + g->ops.gr.set_bes_crop_debug3(g, data); + break; + case NVC597_SET_BES_CROP_DEBUG4: + g->ops.gr.set_bes_crop_debug4(g, data); + break; + case NVC597_SET_SM_DISP_CTRL: + gr_tu104_set_sm_disp_ctrl(g, data); + break; + case NVC597_SET_SHADER_CUT_COLLECTOR: + gr_gv11b_set_shader_cut_collector(g, data); + break; + default: + goto fail; + } + } + return 0; + +fail: + return -EINVAL; +} + +void gr_tu104_init_sm_dsm_reg_info(void) +{ + return; +} + +void gr_tu104_get_sm_dsm_perf_ctrl_regs(struct gk20a *g, + u32 *num_sm_dsm_perf_ctrl_regs, + u32 **sm_dsm_perf_ctrl_regs, + u32 *ctrl_register_stride) +{ + *num_sm_dsm_perf_ctrl_regs = 0; + *sm_dsm_perf_ctrl_regs = NULL; + *ctrl_register_stride = 0; +} +>>>>>>> CHANGE (f0762e gpu: nvgpu: add speculative barrier) -- cgit v1.2.2