/* * GM20B GPC MMU * * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #include "gk20a/gk20a.h" #include "gk20a/kind_gk20a.h" #include "gk20a/fb_gk20a.h" #include "gm20b/fb_gm20b.h" #include #include #include #include #define VPR_INFO_FETCH_WAIT (5) #define WPR_INFO_ADDR_ALIGNMENT 0x0000000c void fb_gm20b_init_fs_state(struct gk20a *g) { gk20a_dbg_info("initialize gm20b fb"); gk20a_writel(g, fb_fbhub_num_active_ltcs_r(), g->ltc_count); } noinline_for_stack void gm20b_init_uncompressed_kind_map(void) { int i; for (i = 0; i < 256; i++) gk20a_uc_kind_map[i] = gmmu_pte_kind_invalid_v(); /* From gm20b */ gk20a_uc_kind_map[gmmu_pte_kind_s8_v()] = gk20a_uc_kind_map[gmmu_pte_kind_s8_2s_v()] = gmmu_pte_kind_s8_v(); /* From gk20a */ gk20a_uc_kind_map[gmmu_pte_kind_z16_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z16_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z16_ms2_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z16_ms4_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z16_ms8_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z16_2z_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z16_ms2_2z_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z16_ms4_2z_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z16_ms8_2z_v()] = gmmu_pte_kind_z16_v(); gk20a_uc_kind_map[gmmu_pte_kind_s8z24_v()] = gk20a_uc_kind_map[gmmu_pte_kind_s8z24_2cz_v()] = gk20a_uc_kind_map[gmmu_pte_kind_s8z24_ms2_2cz_v()] = gk20a_uc_kind_map[gmmu_pte_kind_s8z24_ms4_2cz_v()] = gk20a_uc_kind_map[gmmu_pte_kind_s8z24_ms8_2cz_v()] = gk20a_uc_kind_map[gmmu_pte_kind_s8z24_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_s8z24_ms2_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_s8z24_ms4_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_s8z24_ms8_2cs_v()] = gmmu_pte_kind_s8z24_v(); gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms4_vc4_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms4_vc4_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms4_vc4_2czv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms4_vc4_2zv_v()] = gmmu_pte_kind_v8z24_ms4_vc4_v(); gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms8_vc8_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms8_vc8_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms8_vc8_2czv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms8_vc8_2zv_v()] = gmmu_pte_kind_v8z24_ms8_vc8_v(); gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms4_vc12_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms4_vc12_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms4_vc12_2czv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms4_vc12_2zv_v()] = gmmu_pte_kind_v8z24_ms4_vc12_v(); gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms8_vc24_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms8_vc24_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms8_vc24_2czv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_v8z24_ms8_vc24_2zv_v()] = gmmu_pte_kind_v8z24_ms8_vc24_v(); gk20a_uc_kind_map[gmmu_pte_kind_z24s8_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24s8_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24s8_ms2_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24s8_ms4_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24s8_ms8_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24s8_2cz_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24s8_ms2_2cz_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24s8_ms4_2cz_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24s8_ms8_2cz_v()] = gmmu_pte_kind_z24s8_v(); gk20a_uc_kind_map[gmmu_pte_kind_zf32_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_ms2_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_ms4_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_ms8_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_2cz_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_ms2_2cz_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_ms4_2cz_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_ms8_2cz_v()] = gmmu_pte_kind_zf32_v(); gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms4_vc12_v()] = gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms4_vc12_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms4_vc12_2cszv_v()] = gmmu_pte_kind_x8z24_x16v8s8_ms4_vc12_v(); gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms4_vc4_v()] = gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms4_vc4_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms4_vc4_2cszv_v()] = gmmu_pte_kind_x8z24_x16v8s8_ms4_vc4_v(); gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms8_vc8_v()] = gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms8_vc8_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms8_vc8_2cszv_v()] = gmmu_pte_kind_x8z24_x16v8s8_ms8_vc8_v(); gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms8_vc24_v()] = gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms8_vc24_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_x8z24_x16v8s8_ms8_vc24_2cszv_v()] = gmmu_pte_kind_x8z24_x16v8s8_ms8_vc24_v(); gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms4_vc12_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms4_vc12_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms4_vc12_2cszv_v()] = gmmu_pte_kind_zf32_x16v8s8_ms4_vc12_v(); gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms4_vc4_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms4_vc4_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms4_vc4_2cszv_v()] = gmmu_pte_kind_zf32_x16v8s8_ms4_vc4_v(); gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms8_vc8_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms8_vc8_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms8_vc8_2cszv_v()] = gmmu_pte_kind_zf32_x16v8s8_ms8_vc8_v(); gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms8_vc24_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms8_vc24_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x16v8s8_ms8_vc24_2cszv_v()] = gmmu_pte_kind_zf32_x16v8s8_ms8_vc24_v(); gk20a_uc_kind_map[gmmu_pte_kind_zf32_x24s8_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x24s8_2cszv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x24s8_ms2_2cszv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x24s8_ms4_2cszv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x24s8_ms8_2cszv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x24s8_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x24s8_ms2_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x24s8_ms4_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_zf32_x24s8_ms8_2cs_v()] = gmmu_pte_kind_zf32_x24s8_v(); gk20a_uc_kind_map[gmmu_pte_kind_c32_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_2cba_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_2cra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_2bra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_ms2_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_ms2_2cra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_ms4_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_ms4_2cbr_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_ms4_2cba_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_ms4_2cra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_ms4_2bra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_ms8_ms16_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c32_ms8_ms16_2cra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_2cbr_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_2cba_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_2cra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_2bra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_ms2_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_ms2_2cra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_ms4_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_ms4_2cbr_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_ms4_2cba_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_ms4_2cra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_ms4_2bra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_ms8_ms16_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c64_ms8_ms16_2cra_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c128_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c128_2cr_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c128_ms2_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c128_ms2_2cr_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c128_ms4_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c128_ms4_2cr_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c128_ms8_ms16_2c_v()] = gk20a_uc_kind_map[gmmu_pte_kind_c128_ms8_ms16_2cr_v()] = gmmu_pte_kind_generic_16bx2_v(); gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms4_vc4_2czv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms4_vc4_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms4_vc4_2zv_v()] = gmmu_pte_kind_z24v8_ms4_vc4_v(); gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms4_vc12_2czv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms4_vc12_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms4_vc12_2zv_v()] = gmmu_pte_kind_z24v8_ms4_vc12_v(); gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms8_vc8_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms8_vc8_2czv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms8_vc8_2zv_v()] = gmmu_pte_kind_z24v8_ms8_vc8_v(); gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms8_vc24_2cs_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms8_vc24_2czv_v()] = gk20a_uc_kind_map[gmmu_pte_kind_z24v8_ms8_vc24_2zv_v()] = gmmu_pte_kind_z24v8_ms8_vc24_v(); gk20a_uc_kind_map[gmmu_pte_kind_x8c24_v()] = gmmu_pte_kind_x8c24_v(); } static noinline_for_stack bool gm20b_kind_supported(u8 k) { return /* From gm20b */ (k == gmmu_pte_kind_smsked_message_v()) || (k >= gmmu_pte_kind_s8_v() && k <= gmmu_pte_kind_s8_2s_v()) /* From gk20a */ || gk20a_kind_work_creation(k) || (k == gmmu_pte_kind_invalid_v()) || (k == gmmu_pte_kind_pitch_v()) || (k >= gmmu_pte_kind_z16_v() && k <= gmmu_pte_kind_z16_ms8_2c_v()) || (k >= gmmu_pte_kind_z16_2z_v() && k <= gmmu_pte_kind_z16_ms8_2z_v()) || (k == gmmu_pte_kind_s8z24_v()) || (k >= gmmu_pte_kind_s8z24_2cz_v() && k <= gmmu_pte_kind_s8z24_ms8_2cz_v()) || (k >= gmmu_pte_kind_v8z24_ms4_vc12_v() && k <= gmmu_pte_kind_v8z24_ms8_vc24_v()) || (k >= gmmu_pte_kind_v8z24_ms4_vc12_2czv_v() && k <= gmmu_pte_kind_v8z24_ms8_vc24_2zv_v()) || (k == gmmu_pte_kind_z24s8_v()) || (k >= gmmu_pte_kind_z24s8_2cz_v() && k <= gmmu_pte_kind_z24s8_ms8_2cz_v()) || (k == gmmu_pte_kind_zf32_v()) || (k >= gmmu_pte_kind_zf32_2cz_v() && k <= gmmu_pte_kind_zf32_ms8_2cz_v()) || (k >= gmmu_pte_kind_x8z24_x16v8s8_ms4_vc12_v() && k <= gmmu_pte_kind_x8z24_x16v8s8_ms8_vc24_v()) || (k >= gmmu_pte_kind_x8z24_x16v8s8_ms4_vc12_2cszv_v() && k <= gmmu_pte_kind_zf32_x16v8s8_ms8_vc24_v()) || (k >= gmmu_pte_kind_zf32_x16v8s8_ms4_vc12_2cszv_v() && k <= gmmu_pte_kind_zf32_x24s8_v()) || (k >= gmmu_pte_kind_zf32_x24s8_2cszv_v() && k <= gmmu_pte_kind_zf32_x24s8_ms8_2cszv_v()) || (k == gmmu_pte_kind_generic_16bx2_v()) || (k == gmmu_pte_kind_c32_2c_v()) || (k == gmmu_pte_kind_c32_2cra_v()) || (k == gmmu_pte_kind_c32_ms2_2c_v()) || (k == gmmu_pte_kind_c32_ms2_2cra_v()) || (k >= gmmu_pte_kind_c32_ms4_2c_v() && k <= gmmu_pte_kind_c32_ms4_2cbr_v()) || (k >= gmmu_pte_kind_c32_ms4_2cra_v() && k <= gmmu_pte_kind_c64_2c_v()) || (k == gmmu_pte_kind_c64_2cra_v()) || (k == gmmu_pte_kind_c64_ms2_2c_v()) || (k == gmmu_pte_kind_c64_ms2_2cra_v()) || (k >= gmmu_pte_kind_c64_ms4_2c_v() && k <= gmmu_pte_kind_c64_ms4_2cbr_v()) || (k >= gmmu_pte_kind_c64_ms4_2cra_v() && k <= gmmu_pte_kind_c128_ms8_ms16_2cr_v()) || (k == gmmu_pte_kind_pitch_no_swizzle_v()); } static noinline_for_stack bool gm20b_kind_z(u8 k) { return /* From gm20b */ (k >= gmmu_pte_kind_s8_v() && k <= gmmu_pte_kind_s8_2s_v()) /* From gk20a */ || (k >= gmmu_pte_kind_z16_v() && k <= gmmu_pte_kind_v8z24_ms8_vc24_v()) || (k >= gmmu_pte_kind_v8z24_ms4_vc12_1zv_v() && k <= gmmu_pte_kind_v8z24_ms8_vc24_2cs_v()) || (k >= gmmu_pte_kind_v8z24_ms4_vc12_2czv_v() && k <= gmmu_pte_kind_z24v8_ms8_vc24_v()) || (k >= gmmu_pte_kind_z24v8_ms4_vc12_1zv_v() && k <= gmmu_pte_kind_z24v8_ms8_vc24_2cs_v()) || (k >= gmmu_pte_kind_z24v8_ms4_vc12_2czv_v() && k <= gmmu_pte_kind_x8z24_x16v8s8_ms8_vc24_1cs_v()) || (k >= gmmu_pte_kind_x8z24_x16v8s8_ms4_vc12_1zv_v() && k <= gmmu_pte_kind_zf32_x16v8s8_ms8_vc24_1cs_v()) || (k >= gmmu_pte_kind_zf32_x16v8s8_ms4_vc12_1zv_v() && k <= gmmu_pte_kind_zf32_x24s8_ms16_1cs_v()); } static noinline_for_stack bool gm20b_kind_c(u8 k) { return gk20a_kind_work_creation(k) || (k == gmmu_pte_kind_pitch_v()) || (k == gmmu_pte_kind_generic_16bx2_v()) || (k >= gmmu_pte_kind_c32_2c_v() && k <= gmmu_pte_kind_c32_ms2_2cbr_v()) || (k == gmmu_pte_kind_c32_ms2_2cra_v()) || (k >= gmmu_pte_kind_c32_ms4_2c_v() && k <= gmmu_pte_kind_c64_ms2_2cbr_v()) || (k == gmmu_pte_kind_c64_ms2_2cra_v()) || (k >= gmmu_pte_kind_c64_ms4_2c_v() && k <= gmmu_pte_kind_pitch_no_swizzle_v()); } static noinline_for_stack bool gm20b_kind_compressible(u8 k) { return /* From gm20b */ (k >= gmmu_pte_kind_s8_v() && k <= gmmu_pte_kind_s8_2s_v()) /* From gk20a */ || (k >= gmmu_pte_kind_z16_2c_v() && k <= gmmu_pte_kind_z16_ms16_4cz_v()) || (k >= gmmu_pte_kind_s8z24_1z_v() && k <= gmmu_pte_kind_s8z24_ms16_4cszv_v()) || (k >= gmmu_pte_kind_v8z24_ms4_vc12_1zv_v() && k <= gmmu_pte_kind_v8z24_ms8_vc24_2cs_v()) || (k >= gmmu_pte_kind_v8z24_ms4_vc12_2czv_v() && k <= gmmu_pte_kind_v8z24_ms8_vc24_4cszv_v()) || (k >= gmmu_pte_kind_z24s8_1z_v() && k <= gmmu_pte_kind_z24s8_ms16_4cszv_v()) || (k >= gmmu_pte_kind_z24v8_ms4_vc12_1zv_v() && k <= gmmu_pte_kind_z24v8_ms8_vc24_2cs_v()) || (k >= gmmu_pte_kind_z24v8_ms4_vc12_2czv_v() && k <= gmmu_pte_kind_z24v8_ms8_vc24_4cszv_v()) || (k >= gmmu_pte_kind_zf32_1z_v() && k <= gmmu_pte_kind_zf32_ms16_2cz_v()) || (k >= gmmu_pte_kind_x8z24_x16v8s8_ms4_vc12_1cs_v() && k <= gmmu_pte_kind_x8z24_x16v8s8_ms8_vc24_1cs_v()) || (k >= gmmu_pte_kind_x8z24_x16v8s8_ms4_vc12_1zv_v() && k <= gmmu_pte_kind_x8z24_x16v8s8_ms8_vc24_2cszv_v()) || (k >= gmmu_pte_kind_zf32_x16v8s8_ms4_vc12_1cs_v() && k <= gmmu_pte_kind_zf32_x16v8s8_ms8_vc24_1cs_v()) || (k >= gmmu_pte_kind_zf32_x16v8s8_ms4_vc12_1zv_v() && k <= gmmu_pte_kind_zf32_x16v8s8_ms8_vc24_2cszv_v()) || (k >= gmmu_pte_kind_zf32_x24s8_1cs_v() && k <= gmmu_pte_kind_zf32_x24s8_ms16_1cs_v()) || (k >= gmmu_pte_kind_zf32_x24s8_2cszv_v() && k <= gmmu_pte_kind_c32_ms2_2cbr_v()) || (k == gmmu_pte_kind_c32_ms2_2cra_v()) || (k >= gmmu_pte_kind_c32_ms4_2c_v() && k <= gmmu_pte_kind_c64_ms2_2cbr_v()) || (k == gmmu_pte_kind_c64_ms2_2cra_v()) || (k >= gmmu_pte_kind_c64_ms4_2c_v() && k <= gmmu_pte_kind_c128_ms8_ms16_2cr_v()); } static noinline_for_stack bool gm20b_kind_zbc(u8 k) { return /* From gm20b */ (k >= gmmu_pte_kind_s8_v() && k <= gmmu_pte_kind_s8_2s_v()) /* From gk20a */ || (k >= gmmu_pte_kind_z16_2c_v() && k <= gmmu_pte_kind_z16_ms16_2c_v()) || (k >= gmmu_pte_kind_z16_4cz_v() && k <= gmmu_pte_kind_z16_ms16_4cz_v()) || (k >= gmmu_pte_kind_s8z24_2cz_v() && k <= gmmu_pte_kind_s8z24_ms16_4cszv_v()) || (k >= gmmu_pte_kind_v8z24_ms4_vc12_2cs_v() && k <= gmmu_pte_kind_v8z24_ms8_vc24_2cs_v()) || (k >= gmmu_pte_kind_v8z24_ms4_vc12_2czv_v() && k <= gmmu_pte_kind_v8z24_ms8_vc24_2czv_v()) || (k >= gmmu_pte_kind_v8z24_ms4_vc12_4cszv_v() && k <= gmmu_pte_kind_v8z24_ms8_vc24_4cszv_v()) || (k >= gmmu_pte_kind_z24s8_2cs_v() && k <= gmmu_pte_kind_z24s8_ms16_4cszv_v()) || (k >= gmmu_pte_kind_z24v8_ms4_vc12_2cs_v() && k <= gmmu_pte_kind_z24v8_ms8_vc24_2cs_v()) || (k >= gmmu_pte_kind_z24v8_ms4_vc12_2czv_v() && k <= gmmu_pte_kind_z24v8_ms8_vc24_2czv_v()) || (k >= gmmu_pte_kind_z24v8_ms4_vc12_4cszv_v() && k <= gmmu_pte_kind_z24v8_ms8_vc24_4cszv_v()) || (k >= gmmu_pte_kind_zf32_2cs_v() && k <= gmmu_pte_kind_zf32_ms16_2cz_v()) || (k >= gmmu_pte_kind_x8z24_x16v8s8_ms4_vc12_1cs_v() && k <= gmmu_pte_kind_x8z24_x16v8s8_ms8_vc24_1cs_v()) || (k >= gmmu_pte_kind_x8z24_x16v8s8_ms4_vc12_1czv_v() && k <= gmmu_pte_kind_x8z24_x16v8s8_ms8_vc24_2cszv_v()) || (k >= gmmu_pte_kind_zf32_x16v8s8_ms4_vc12_1cs_v() && k <= gmmu_pte_kind_zf32_x16v8s8_ms8_vc24_1cs_v()) || (k >= gmmu_pte_kind_zf32_x16v8s8_ms4_vc12_1czv_v() && k <= gmmu_pte_kind_zf32_x16v8s8_ms8_vc24_2cszv_v()) || (k >= gmmu_pte_kind_zf32_x24s8_1cs_v() && k <= gmmu_pte_kind_zf32_x24s8_ms16_1cs_v()) || (k >= gmmu_pte_kind_zf32_x24s8_2cszv_v() && k <= gmmu_pte_kind_c32_2cra_v()) || (k >= gmmu_pte_kind_c32_ms2_2c_v() && k <= gmmu_pte_kind_c32_ms2_2cbr_v()) || (k == gmmu_pte_kind_c32_ms2_2cra_v()) || (k >= gmmu_pte_kind_c32_ms4_2c_v() && k <= gmmu_pte_kind_c32_ms4_2cra_v()) || (k >= gmmu_pte_kind_c32_ms8_ms16_2c_v() && k <= gmmu_pte_kind_c64_2cra_v()) || (k >= gmmu_pte_kind_c64_ms2_2c_v() && k <= gmmu_pte_kind_c64_ms2_2cbr_v()) || (k == gmmu_pte_kind_c64_ms2_2cra_v()) || (k >= gmmu_pte_kind_c64_ms4_2c_v() && k <= gmmu_pte_kind_c64_ms4_2cra_v()) || (k >= gmmu_pte_kind_c64_ms8_ms16_2c_v() && k <= gmmu_pte_kind_c128_ms8_ms16_2cr_v()); } void gm20b_init_kind_attr(void) { u16 k; for (k = 0; k < NV_KIND_ATTR_SIZE; k++) { gk20a_kind_attr[k] = 0; if (gm20b_kind_supported((u8)k)) gk20a_kind_attr[k] |= GK20A_KIND_ATTR_SUPPORTED; if (gm20b_kind_compressible((u8)k)) gk20a_kind_attr[k] |= GK20A_KIND_ATTR_COMPRESSIBLE; if (gm20b_kind_z((u8)k)) gk20a_kind_attr[k] |= GK20A_KIND_ATTR_Z; if (gm20b_kind_c((u8)k)) gk20a_kind_attr[k] |= GK20A_KIND_ATTR_C; if (gm20b_kind_zbc((u8)k)) gk20a_kind_attr[k] |= GK20A_KIND_ATTR_ZBC; } } void gm20b_fb_set_mmu_page_size(struct gk20a *g) { /* set large page size in fb */ u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r()); fb_mmu_ctrl |= fb_mmu_ctrl_use_pdb_big_page_size_true_f(); gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl); } bool gm20b_fb_set_use_full_comp_tag_line(struct gk20a *g) { /* set large page size in fb */ u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r()); fb_mmu_ctrl |= fb_mmu_ctrl_use_full_comp_tag_line_true_f(); gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl); return true; } unsigned int gm20b_fb_compression_page_size(struct gk20a *g) { return SZ_128K; } unsigned int gm20b_fb_compressible_page_size(struct gk20a *g) { return SZ_64K; } void gm20b_fb_dump_vpr_wpr_info(struct gk20a *g) { u32 val; /* print vpr and wpr info */ val = gk20a_readl(g, fb_mmu_vpr_info_r()); val &= ~0x3; val |= fb_mmu_vpr_info_index_addr_lo_v(); gk20a_writel(g, fb_mmu_vpr_info_r(), val); nvgpu_err(g, "VPR: %08x %08x %08x %08x", gk20a_readl(g, fb_mmu_vpr_info_r()), gk20a_readl(g, fb_mmu_vpr_info_r()), gk20a_readl(g, fb_mmu_vpr_info_r()), gk20a_readl(g, fb_mmu_vpr_info_r())); val = gk20a_readl(g, fb_mmu_wpr_info_r()); val &= ~0xf; val |= (fb_mmu_wpr_info_index_allow_read_v()); gk20a_writel(g, fb_mmu_wpr_info_r(), val); nvgpu_err(g, "WPR: %08x %08x %08x %08x %08x %08x", gk20a_readl(g, fb_mmu_wpr_info_r()), gk20a_readl(g, fb_mmu_wpr_info_r()), gk20a_readl(g, fb_mmu_wpr_info_r()), gk20a_readl(g, fb_mmu_wpr_info_r()), gk20a_readl(g, fb_mmu_wpr_info_r()), gk20a_readl(g, fb_mmu_wpr_info_r())); } static int gm20b_fb_vpr_info_fetch_wait(struct gk20a *g, unsigned int msec) { struct nvgpu_timeout timeout; nvgpu_timeout_init(g, &timeout, msec, NVGPU_TIMER_CPU_TIMER); do { u32 val; val = gk20a_readl(g, fb_mmu_vpr_info_r()); if (fb_mmu_vpr_info_fetch_v(val) == fb_mmu_vpr_info_fetch_false_v()) return 0; } while (!nvgpu_timeout_expired(&timeout)); return -ETIMEDOUT; } int gm20b_fb_vpr_info_fetch(struct gk20a *g) { if (gm20b_fb_vpr_info_fetch_wait(g, VPR_INFO_FETCH_WAIT)) { return -ETIME; } gk20a_writel(g, fb_mmu_vpr_info_r(), fb_mmu_vpr_info_fetch_true_v()); return gm20b_fb_vpr_info_fetch_wait(g, VPR_INFO_FETCH_WAIT); } void gm20b_fb_read_wpr_info(struct gk20a *g, struct wpr_carveout_info *inf) { u32 val = 0; u64 wpr_start = 0; u64 wpr_end = 0; val = gk20a_readl(g, fb_mmu_wpr_info_r()); val &= ~0xF; val |= fb_mmu_wpr_info_index_wpr1_addr_lo_v(); gk20a_writel(g, fb_mmu_wpr_info_r(), val); val = gk20a_readl(g, fb_mmu_wpr_info_r()) >> 0x4; wpr_start = hi32_lo32_to_u64( (val >> (32 - WPR_INFO_ADDR_ALIGNMENT)), (val << WPR_INFO_ADDR_ALIGNMENT)); val = gk20a_readl(g, fb_mmu_wpr_info_r()); val &= ~0xF; val |= fb_mmu_wpr_info_index_wpr1_addr_hi_v(); gk20a_writel(g, fb_mmu_wpr_info_r(), val); val = gk20a_readl(g, fb_mmu_wpr_info_r()) >> 0x4; wpr_end = hi32_lo32_to_u64( (val >> (32 - WPR_INFO_ADDR_ALIGNMENT)), (val << WPR_INFO_ADDR_ALIGNMENT)); inf->wpr_base = wpr_start; inf->nonwpr_base = 0; inf->size = (wpr_end - wpr_start); } bool gm20b_fb_debug_mode_enabled(struct gk20a *g) { u32 debug_ctrl = gk20a_readl(g, gr_gpcs_pri_mmu_debug_ctrl_r()); return gr_gpcs_pri_mmu_debug_ctrl_debug_v(debug_ctrl) == gr_gpcs_pri_mmu_debug_ctrl_debug_enabled_v(); } void gm20b_fb_set_debug_mode(struct gk20a *g, bool enable) { u32 reg_val, fb_debug_ctrl, gpc_debug_ctrl; if (enable) { fb_debug_ctrl = fb_mmu_debug_ctrl_debug_enabled_f(); gpc_debug_ctrl = gr_gpcs_pri_mmu_debug_ctrl_debug_enabled_f(); g->mmu_debug_ctrl = true; } else { fb_debug_ctrl = fb_mmu_debug_ctrl_debug_disabled_f(); gpc_debug_ctrl = gr_gpcs_pri_mmu_debug_ctrl_debug_disabled_f(); g->mmu_debug_ctrl = false; } reg_val = gk20a_readl(g, fb_mmu_debug_ctrl_r()); reg_val = set_field(reg_val, fb_mmu_debug_ctrl_debug_m(), fb_debug_ctrl); gk20a_writel(g, fb_mmu_debug_ctrl_r(), reg_val); reg_val = gk20a_readl(g, gr_gpcs_pri_mmu_debug_ctrl_r()); reg_val = set_field(reg_val, gr_gpcs_pri_mmu_debug_ctrl_debug_m(), gpc_debug_ctrl); gk20a_writel(g, gr_gpcs_pri_mmu_debug_ctrl_r(), reg_val); }