/* * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #ifdef CONFIG_TEGRA_NVLINK #include #include #include #include #include #include #include #include #include #include #include "nvlink_gv100.h" #include #include #include #include #include #include #include #include #include #define NVLINK_PLL_ON_TIMEOUT_MS 30 #define NVLINK_SUBLINK_TIMEOUT_MS 200 /* * The manuals are missing some useful defines * we add them for now */ #define IPT_INTR_CONTROL_LINK(i) (nvlipt_intr_control_link0_r() + (i)*4) #define IPT_ERR_UC_STATUS_LINK(i) (nvlipt_err_uc_status_link0_r() + (i)*36) #define IPT_ERR_UC_MASK_LINK(i) (nvlipt_err_uc_mask_link0_r() + (i)*36) #define IPT_ERR_UC_SEVERITY_LINK(i) (nvlipt_err_uc_severity_link0_r() + (i)*36) #define IPT_ERR_UC_FIRST_LINK(i) (nvlipt_err_uc_first_link0_r() + (i)*36) #define IPT_ERR_UC_ADVISORY_LINK(i) (nvlipt_err_uc_advisory_link0_r() + (i)*36) #define IPT_ERR_C_STATUS_LINK(i) (nvlipt_err_c_status_link0_r() + (i)*36) #define IPT_ERR_C_MASK_LINK(i) (nvlipt_err_c_mask_link0_r() + (i)*36) #define IPT_ERR_C_FIRST_LINK(i) (nvlipt_err_c_first_link0_r() + (i)*36) #define IPT_ERR_CONTROL_LINK(i) (nvlipt_err_control_link0_r() + (i)*4) #define IPT_ERR_UC_ACTIVE_BITS (nvlipt_err_uc_status_link0_dlprotocol_f(1) | \ nvlipt_err_uc_status_link0_datapoisoned_f(1) | \ nvlipt_err_uc_status_link0_flowcontrol_f(1) | \ nvlipt_err_uc_status_link0_responsetimeout_f(1) | \ nvlipt_err_uc_status_link0_targeterror_f(1) | \ nvlipt_err_uc_status_link0_unexpectedresponse_f(1) | \ nvlipt_err_uc_status_link0_receiveroverflow_f(1) | \ nvlipt_err_uc_status_link0_malformedpacket_f(1) | \ nvlipt_err_uc_status_link0_stompedpacketreceived_f(1) | \ nvlipt_err_uc_status_link0_unsupportedrequest_f(1) | \ nvlipt_err_uc_status_link0_ucinternal_f(1)) #define MINION_FALCON_INTR_MASK (minion_falcon_irqmset_wdtmr_set_f() | \ minion_falcon_irqmset_halt_set_f() | \ minion_falcon_irqmset_exterr_set_f()| \ minion_falcon_irqmset_swgen0_set_f()| \ minion_falcon_irqmset_swgen1_set_f()) #define MINION_FALCON_INTR_DEST ( \ minion_falcon_irqdest_host_wdtmr_host_f() | \ minion_falcon_irqdest_host_halt_host_f() | \ minion_falcon_irqdest_host_exterr_host_f() | \ minion_falcon_irqdest_host_swgen0_host_f() | \ minion_falcon_irqdest_host_swgen1_host_f() | \ minion_falcon_irqdest_target_wdtmr_host_normal_f() | \ minion_falcon_irqdest_target_halt_host_normal_f() | \ minion_falcon_irqdest_target_exterr_host_normal_f() | \ minion_falcon_irqdest_target_swgen0_host_normal_f() | \ minion_falcon_irqdest_target_swgen1_host_normal_f()) struct __nvlink_reginit { u32 addr; u32 value; }; static const struct __nvlink_reginit __nvlink_reginit_per_link_tegra[] = { /* NVTLC when connected to Tegra */ { 0x300U, 0x00800040U }, { 0x304U, 0x00000000U }, { 0x308U, 0x00000000U }, { 0x30CU, 0x00000000U }, { 0x310U, 0x00000000U }, { 0x314U, 0x00800040U }, { 0x318U, 0x00000000U }, { 0x31CU, 0x00000000U }, { 0x200U, 0x007F003FU }, { 0x204U, 0x007F003FU }, { 0x208U, 0x007F003FU }, { 0x20CU, 0x007F003FU }, { 0x210U, 0x007F003FU }, { 0x214U, 0x00FF007FU }, { 0x218U, 0x00FF007FU }, { 0x21CU, 0x00FF007FU }, { 0xB00U, 0x010000C0U }, { 0xB04U, 0x00000000U }, { 0xB08U, 0x00000000U }, { 0xB0CU, 0x00000000U }, { 0xB10U, 0x00000000U }, { 0xB14U, 0x010000C0U }, { 0xB18U, 0x00000000U }, { 0xB1CU, 0x00000000U }, { 0xA00U, 0x00FF00BFU }, { 0xA04U, 0x00FF00BFU }, { 0xA08U, 0x00FF00BFU }, { 0xA0CU, 0x00FF00BFU }, { 0xA10U, 0x00FF00BFU }, { 0xA14U, 0x01FF017FU }, { 0xA18U, 0x01FF017FU }, { 0xA1CU, 0x01FF017FU }, { 0xF04U, 0x00FFFFFFU }, { 0xF0CU, 0x00FFFFFFU }, { 0xF1CU, 0x003FFFFFU }, { 0xF24U, 0x003FFFFFU }, { 0x704U, 0x003FFFFFU }, { 0x70CU, 0x003FFFFFU }, { 0x400U, 0x00000001U }, { 0xC00U, 0x00000001U }, }; static const struct __nvlink_reginit __nvlink_reginit_per_link_gpu[] = { /* NVTLC for PEER GPU */ { 0x300U, 0x00800040U }, { 0x304U, 0x00000000U }, { 0x308U, 0x00000000U }, { 0x30CU, 0x00000000U }, { 0x310U, 0x00000000U }, { 0x314U, 0x00800040U }, { 0x318U, 0x00000000U }, { 0x31CU, 0x00000000U }, { 0x200U, 0x007F003FU }, { 0x204U, 0x007F003FU }, { 0x208U, 0x007F003FU }, { 0x20CU, 0x007F003FU }, { 0x210U, 0x007F003FU }, { 0x214U, 0x00FF007FU }, { 0x218U, 0x00FF007FU }, { 0x21CU, 0x00FF007FU }, { 0xB00U, 0x010000C0U }, { 0xB04U, 0x00000000U }, { 0xB08U, 0x00000000U }, { 0xB0CU, 0x00000000U }, { 0xB10U, 0x00000000U }, { 0xB14U, 0x010000C0U }, { 0xB18U, 0x00000000U }, { 0xB1CU, 0x00000000U }, { 0xA00U, 0x00FF00BFU }, { 0xA04U, 0x00FF00BFU }, { 0xA08U, 0x00FF00BFU }, { 0xA0CU, 0x00FF00BFU }, { 0xA10U, 0x00FF00BFU }, { 0xA14U, 0x01FF017FU }, { 0xA18U, 0x01FF017FU }, { 0xA1CU, 0x01FF017FU }, { 0xF04U, 0x00FFFFFFU }, { 0xF0CU, 0x00FFFFFFU }, { 0xF1CU, 0x003FFFFFU }, { 0xF24U, 0x003FFFFFU }, { 0x704U, 0x003FFFFFU }, { 0x70CU, 0x003FFFFFU }, { 0x400U, 0x00000001U }, { 0xC00U, 0x00000001U }, }; #define NVL_DEVICE(str) nvlinkip_discovery_common_device_##str##_v() static const char *__gv100_device_type_to_str(u32 type) { if (type == NVL_DEVICE(ioctrl)) return "IOCTRL"; if (type == NVL_DEVICE(dlpl)) return "DL/PL"; if (type == NVL_DEVICE(nvltlc)) return "NVLTLC"; if (type == NVL_DEVICE(ioctrlmif)) return "IOCTRLMIF"; if (type == NVL_DEVICE(nvlipt)) return "NVLIPT"; if (type == NVL_DEVICE(minion)) return "MINION"; if (type == NVL_DEVICE(dlpl_multicast)) return "DL/PL MULTICAST"; if (type == NVL_DEVICE(nvltlc_multicast)) return "NVLTLC MULTICAST"; if (type == NVL_DEVICE(ioctrlmif_multicast)) return "IOCTRLMIF MULTICAST"; if (type == NVL_DEVICE(nvltlc_multicast)) return "NVLTLC MULTICAST"; return "UNKNOWN"; } /* * Function prototypes */ static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g); static u32 gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask); /* ******************************************************************************* * IP specific functions * ******************************************************************************* */ /* *-----------------------------------------------------------------------------* * MINION API *-----------------------------------------------------------------------------* */ /* * Initialization of link specific interrupts */ static void gv100_nvlink_minion_link_intr_enable(struct gk20a *g, u32 link_id, bool enable) { u32 intr, links; /* Only stall interrupts for now */ intr = MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); links = minion_minion_intr_stall_en_link_v(intr); if (enable) links |= BIT(link_id); else links &= ~BIT(link_id); intr = set_field(intr, minion_minion_intr_stall_en_link_m(), minion_minion_intr_stall_en_link_f(links)); MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), intr); } /* * Initialization of falcon interrupts */ static void gv100_nvlink_minion_falcon_intr_enable(struct gk20a *g, bool enable) { u32 reg; reg = MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); if (enable) { reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(), minion_minion_intr_stall_en_fatal_enable_f()); reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(), minion_minion_intr_stall_en_nonfatal_enable_f()); reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(), minion_minion_intr_stall_en_falcon_stall_enable_f()); reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(), minion_minion_intr_stall_en_falcon_nostall_enable_f()); } else { reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(), minion_minion_intr_stall_en_fatal_disable_f()); reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(), minion_minion_intr_stall_en_nonfatal_disable_f()); reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(), minion_minion_intr_stall_en_falcon_stall_disable_f()); reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(), minion_minion_intr_stall_en_falcon_nostall_disable_f()); } MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), reg); } /* * Initialize minion IP interrupts */ static void gv100_nvlink_initialize_minion(struct gk20a *g) { /* Disable non-stall tree */ MINION_REG_WR32(g, minion_minion_intr_nonstall_en_r(), 0x0); gv100_nvlink_minion_falcon_intr_enable(g, true); } /* * Check if minion is up */ static bool __gv100_nvlink_minion_is_running(struct gk20a *g) { /* if minion is booted and not halted, it is running */ if ((MINION_REG_RD32(g, minion_minion_status_r()) & minion_minion_status_status_f(1)) && (!minion_falcon_irqstat_halt_v( MINION_REG_RD32(g, minion_falcon_irqstat_r())))) return true; return false; } /* * Falcon specific ISR handling */ static bool gv100_nvlink_minion_falcon_isr(struct gk20a *g) { u32 intr; intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) & MINION_REG_RD32(g, minion_falcon_irqmask_r()); if (!intr) return true; if (intr & minion_falcon_irqstat_exterr_true_f()) { nvgpu_err(g, "FALCON EXT ADDR: 0x%x 0x%x 0x%x", MINION_REG_RD32(g, 0x244), MINION_REG_RD32(g, 0x248), MINION_REG_RD32(g, 0x24c)); } MINION_REG_WR32(g, minion_falcon_irqsclr_r(), intr); nvgpu_err(g, "FATAL minion IRQ: 0x%08x", intr); intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) & MINION_REG_RD32(g, minion_falcon_irqmask_r()); return (intr == 0); } /* * Link Specific ISR */ static bool gv100_nvlink_minion_link_isr(struct gk20a *g, u32 link_id) { u32 intr, code; bool fatal = false; intr = MINION_REG_RD32(g, minion_nvlink_link_intr_r(link_id)); code = minion_nvlink_link_intr_code_v(intr); if (code == minion_nvlink_link_intr_code_swreq_v()) { nvgpu_err(g, " Intr SWREQ, link: %d subcode: %x", link_id, minion_nvlink_link_intr_subcode_v(intr)); } else if (code == minion_nvlink_link_intr_code_pmdisabled_v()) { nvgpu_err(g, " Fatal Intr PMDISABLED, link: %d subcode: %x", link_id, minion_nvlink_link_intr_subcode_v(intr)); fatal = true; } else if (code == minion_nvlink_link_intr_code_na_v()) { nvgpu_err(g, " Fatal Intr NA, link: %d subcode: %x", link_id, minion_nvlink_link_intr_subcode_v(intr)); fatal = true; } else if (code == minion_nvlink_link_intr_code_dlreq_v()) { nvgpu_err(g, " Fatal Intr DLREQ, link: %d subcode: %x", link_id, minion_nvlink_link_intr_subcode_v(intr)); fatal = true; } else { nvgpu_err(g, " Fatal Intr UNKN:%x, link: %d subcode: %x", code, link_id, minion_nvlink_link_intr_subcode_v(intr)); fatal = true; } if (fatal) gv100_nvlink_minion_link_intr_enable(g, link_id, false); intr = set_field(intr, minion_nvlink_link_intr_state_m(), minion_nvlink_link_intr_state_f(1)); MINION_REG_WR32(g, minion_nvlink_link_intr_r(link_id), intr); return true; } /* * Global minion routine to service interrupts */ static bool gv100_nvlink_minion_isr(struct gk20a *g) { u32 intr, i; unsigned long links; intr = MINION_REG_RD32(g, minion_minion_intr_r()) & MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); if (minion_minion_intr_falcon_stall_v(intr) || minion_minion_intr_falcon_nostall_v(intr)) gv100_nvlink_minion_falcon_isr(g); if (minion_minion_intr_fatal_v(intr)) { gv100_nvlink_minion_falcon_intr_enable(g, false); MINION_REG_WR32(g, minion_minion_intr_r(), minion_minion_intr_fatal_f(1)); } if (minion_minion_intr_nonfatal_v(intr)) MINION_REG_WR32(g, minion_minion_intr_r(), minion_minion_intr_nonfatal_f(1)); links = minion_minion_intr_link_v(intr) & g->nvlink.enabled_links; if (links) for_each_set_bit(i, &links, 32) gv100_nvlink_minion_link_isr(g, i); /* Re-test interrupt status */ intr = MINION_REG_RD32(g, minion_minion_intr_r()) & MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); return (intr == 0); } /* Extract a WORD from the MINION ucode */ static inline u32 minion_extract_word(struct nvgpu_firmware *fw, u32 idx) { u32 out_data = 0; u8 byte = 0; u32 i = 0; for (i = 0; i < 4; i++) { byte = fw->data[idx + i]; out_data |= ((u32)byte) << (8 * i); } return out_data; } /* * Load minion FW and set up bootstrap */ static u32 gv100_nvlink_minion_load(struct gk20a *g) { u32 err = 0; struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; struct nvgpu_firmware *nvgpu_minion_fw = NULL; struct minion_hdr *minion_hdr = &ndev->minion_hdr; u32 data_idx = 0; u32 app = 0; struct nvgpu_timeout timeout; u32 delay = GR_IDLE_CHECK_DEFAULT; u32 reg; nvgpu_log_fn(g, " "); if (__gv100_nvlink_minion_is_running(g)) return 0; /* get mem unlock ucode binary */ nvgpu_minion_fw = nvgpu_request_firmware(g, "minion.bin", 0); if (!nvgpu_minion_fw) { nvgpu_err(g, "minion ucode get fail"); err = -ENOENT; goto exit; } /* nvdec falcon reset */ nvgpu_flcn_reset(&g->minion_flcn); /* Read ucode header */ minion_hdr->os_code_offset = minion_extract_word(nvgpu_minion_fw, data_idx); data_idx += 4; minion_hdr->os_code_size = minion_extract_word(nvgpu_minion_fw, data_idx); data_idx += 4; minion_hdr->os_data_offset = minion_extract_word(nvgpu_minion_fw, data_idx); data_idx += 4; minion_hdr->os_data_size = minion_extract_word(nvgpu_minion_fw, data_idx); data_idx += 4; minion_hdr->num_apps = minion_extract_word(nvgpu_minion_fw, data_idx); data_idx += 4; nvgpu_log(g, gpu_dbg_nvlink, "MINION Ucode Header Info:"); nvgpu_log(g, gpu_dbg_nvlink, "-------------------------"); nvgpu_log(g, gpu_dbg_nvlink, " - OS Code Offset = %u", minion_hdr->os_code_offset); nvgpu_log(g, gpu_dbg_nvlink, " - OS Code Size = %u", minion_hdr->os_code_size); nvgpu_log(g, gpu_dbg_nvlink, " - OS Data Offset = %u", minion_hdr->os_data_offset); nvgpu_log(g, gpu_dbg_nvlink, " - OS Data Size = %u", minion_hdr->os_data_size); nvgpu_log(g, gpu_dbg_nvlink, " - Num Apps = %u", minion_hdr->num_apps); /* Allocate offset/size arrays for all the ucode apps */ minion_hdr->app_code_offsets = nvgpu_kcalloc(g, minion_hdr->num_apps, sizeof(u32)); if (!minion_hdr->app_code_offsets) { nvgpu_err(g, "Couldn't allocate MINION app_code_offsets array"); err = -ENOMEM; goto exit; } minion_hdr->app_code_sizes = nvgpu_kcalloc(g, minion_hdr->num_apps, sizeof(u32)); if (!minion_hdr->app_code_sizes) { nvgpu_err(g, "Couldn't allocate MINION app_code_sizes array"); err = -ENOMEM; goto exit; } minion_hdr->app_data_offsets = nvgpu_kcalloc(g, minion_hdr->num_apps, sizeof(u32)); if (!minion_hdr->app_data_offsets) { nvgpu_err(g, "Couldn't allocate MINION app_data_offsets array"); err = -ENOMEM; goto exit; } minion_hdr->app_data_sizes = nvgpu_kcalloc(g, minion_hdr->num_apps, sizeof(u32)); if (!minion_hdr->app_data_sizes) { nvgpu_err(g, "Couldn't allocate MINION app_data_sizes array"); err = -ENOMEM; goto exit; } /* Get app code offsets and sizes */ for (app = 0; app < minion_hdr->num_apps; app++) { minion_hdr->app_code_offsets[app] = minion_extract_word(nvgpu_minion_fw, data_idx); data_idx += 4; minion_hdr->app_code_sizes[app] = minion_extract_word(nvgpu_minion_fw, data_idx); data_idx += 4; nvgpu_log(g, gpu_dbg_nvlink, " - App Code:"); nvgpu_log(g, gpu_dbg_nvlink, " - App #%d: Code Offset = %u, Code Size = %u", app, minion_hdr->app_code_offsets[app], minion_hdr->app_code_sizes[app]); } /* Get app data offsets and sizes */ for (app = 0; app < minion_hdr->num_apps; app++) { minion_hdr->app_data_offsets[app] = minion_extract_word(nvgpu_minion_fw, data_idx); data_idx += 4; minion_hdr->app_data_sizes[app] = minion_extract_word(nvgpu_minion_fw, data_idx); data_idx += 4; nvgpu_log(g, gpu_dbg_nvlink, " - App Data:"); nvgpu_log(g, gpu_dbg_nvlink, " - App #%d: Data Offset = %u, Data Size = %u", app, minion_hdr->app_data_offsets[app], minion_hdr->app_data_sizes[app]); } minion_hdr->ovl_offset = minion_extract_word(nvgpu_minion_fw, data_idx); data_idx += 4; minion_hdr->ovl_size = minion_extract_word(nvgpu_minion_fw, data_idx); data_idx += 4; ndev->minion_img = &(nvgpu_minion_fw->data[data_idx]); minion_hdr->ucode_data_size = nvgpu_minion_fw->size - data_idx; nvgpu_log(g, gpu_dbg_nvlink, " - Overlay Offset = %u", minion_hdr->ovl_offset); nvgpu_log(g, gpu_dbg_nvlink, " - Overlay Size = %u", minion_hdr->ovl_size); nvgpu_log(g, gpu_dbg_nvlink, " - Ucode Data Size = %u", minion_hdr->ucode_data_size); /* Clear interrupts */ nvgpu_flcn_set_irq(&g->minion_flcn, true, MINION_FALCON_INTR_MASK, MINION_FALCON_INTR_DEST); /* Copy Non Secure IMEM code */ nvgpu_flcn_copy_to_imem(&g->minion_flcn, 0, (u8 *)&ndev->minion_img[minion_hdr->os_code_offset], minion_hdr->os_code_size, 0, false, GET_IMEM_TAG(minion_hdr->os_code_offset)); /* Copy Non Secure DMEM code */ nvgpu_flcn_copy_to_dmem(&g->minion_flcn, 0, (u8 *)&ndev->minion_img[minion_hdr->os_data_offset], minion_hdr->os_data_size, 0); /* Load the apps securely */ for (app = 0; app < minion_hdr->num_apps; app++) { u32 app_code_start = minion_hdr->app_code_offsets[app]; u32 app_code_size = minion_hdr->app_code_sizes[app]; u32 app_data_start = minion_hdr->app_data_offsets[app]; u32 app_data_size = minion_hdr->app_data_sizes[app]; if (app_code_size) nvgpu_flcn_copy_to_imem(&g->minion_flcn, app_code_start, (u8 *)&ndev->minion_img[app_code_start], app_code_size, 0, true, GET_IMEM_TAG(app_code_start)); if (app_data_size) nvgpu_flcn_copy_to_dmem(&g->minion_flcn, app_data_start, (u8 *)&ndev->minion_img[app_data_start], app_data_size, 0); } /* set BOOTVEC to start of non-secure code */ nvgpu_flcn_bootstrap(&g->minion_flcn, 0x0); nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), NVGPU_TIMER_CPU_TIMER); do { reg = MINION_REG_RD32(g, minion_minion_status_r()); if (minion_minion_status_status_v(reg)) { /* Minion sequence completed, check status */ if (minion_minion_status_status_v(reg) != minion_minion_status_status_boot_v()) { nvgpu_err(g, "MINION init sequence failed: 0x%x", minion_minion_status_status_v(reg)); err = -EINVAL; goto exit; } nvgpu_log(g, gpu_dbg_nvlink, "MINION boot successful: 0x%x", reg); err = 0; break; } nvgpu_usleep_range(delay, delay * 2); delay = min_t(unsigned long, delay << 1, GR_IDLE_CHECK_MAX); } while (!nvgpu_timeout_expired_msg(&timeout, " minion boot timeout")); /* Service interrupts */ gv100_nvlink_minion_falcon_isr(g); if (nvgpu_timeout_peek_expired(&timeout)) { err = -ETIMEDOUT; goto exit; } gv100_nvlink_initialize_minion(g); exit: nvgpu_kfree(g, minion_hdr->app_code_offsets); nvgpu_kfree(g, minion_hdr->app_code_sizes); nvgpu_kfree(g, minion_hdr->app_data_offsets); nvgpu_kfree(g, minion_hdr->app_data_sizes); if (nvgpu_minion_fw) { nvgpu_release_firmware(g, nvgpu_minion_fw); ndev->minion_img = NULL; } return err; } /* * Check if MINION command is complete */ static u32 gv100_nvlink_minion_command_complete(struct gk20a *g, u32 link_id) { u32 reg; struct nvgpu_timeout timeout; u32 delay = GR_IDLE_CHECK_DEFAULT; nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), NVGPU_TIMER_CPU_TIMER); do { reg = MINION_REG_RD32(g, minion_nvlink_dl_cmd_r(link_id)); if (minion_nvlink_dl_cmd_ready_v(reg) == 1) { /* Command completed, check sucess */ if (minion_nvlink_dl_cmd_fault_v(reg) == 1) { nvgpu_err(g, "minion cmd(%d) error: 0x%x", link_id, reg); reg = minion_nvlink_dl_cmd_fault_f(1); MINION_REG_WR32(g, minion_nvlink_dl_cmd_r(link_id), reg); return -EINVAL; } /* Commnand success */ break; } nvgpu_usleep_range(delay, delay * 2); delay = min_t(unsigned long, delay << 1, GR_IDLE_CHECK_MAX); } while (!nvgpu_timeout_expired_msg(&timeout, " minion cmd timeout")); if (nvgpu_timeout_peek_expired(&timeout)) return -ETIMEDOUT; nvgpu_log(g, gpu_dbg_nvlink, "minion cmd Complete"); return 0; } /* * Send Minion command (can be async) */ int gv100_nvlink_minion_send_command(struct gk20a *g, u32 link_id, u32 command, u32 scratch_0, bool sync) { int err = 0; /* Check last command succeded */ err = gv100_nvlink_minion_command_complete(g, link_id); if (err) return -EINVAL; nvgpu_log(g, gpu_dbg_nvlink, "sending MINION command 0x%x to link %d", command, link_id); if (command == minion_nvlink_dl_cmd_command_configeom_v()) MINION_REG_WR32(g, minion_misc_0_r(), minion_misc_0_scratch_swrw_0_f(scratch_0)); MINION_REG_WR32(g, minion_nvlink_dl_cmd_r(link_id), minion_nvlink_dl_cmd_command_f(command) | minion_nvlink_dl_cmd_fault_f(1)); if (sync) err = gv100_nvlink_minion_command_complete(g, link_id); return err; } /* MINION API COMMANDS */ /* * Init UPHY */ static u32 gv100_nvlink_minion_init_uphy(struct gk20a *g, unsigned long mask, bool sync) { u32 err = 0; u32 init_pll_cmd; u32 link_id, master_pll, slave_pll; u32 master_state, slave_state; unsigned long link_enable; switch(g->nvlink.speed) { case nvgpu_nvlink_speed_25G: init_pll_cmd = minion_nvlink_dl_cmd_command_initpll_0_v(); break; case nvgpu_nvlink_speed_20G: init_pll_cmd = minion_nvlink_dl_cmd_command_initpll_1_v(); break; default: nvgpu_err(g, "Unsupported UPHY speed"); return -EINVAL; } link_enable = __gv100_nvlink_get_link_reset_mask(g); for_each_set_bit(link_id, &mask, 32) { master_pll = g->nvlink.links[link_id].pll_master_link_id; slave_pll = g->nvlink.links[link_id].pll_slave_link_id; master_state = nvl_link_state_state_init_v(); slave_state = nvl_link_state_state_init_v(); if (BIT(master_pll) & link_enable) master_state = nvl_link_state_state_v( g->ops.nvlink.link_get_state(g, master_pll)); if (BIT(slave_pll) & link_enable) slave_state = nvl_link_state_state_v( g->ops.nvlink.link_get_state(g, slave_pll)); if ((slave_state != nvl_link_state_state_init_v()) || (master_state != nvl_link_state_state_init_v())) { nvgpu_err(g, "INIT PLL can only be executed when both " "master and slave links are in init state"); return -EINVAL; } /* Check if INIT PLL is done on link */ if (!(BIT(master_pll) & g->nvlink.init_pll_done)) { err = gv100_nvlink_minion_send_command(g, master_pll, init_pll_cmd, 0, sync); if (err) { nvgpu_err(g, " Error sending INITPLL to minion"); return err; } g->nvlink.init_pll_done |= BIT(master_pll); } } err = g->ops.nvlink.setup_pll(g, mask); if (err) { nvgpu_err(g, "Error setting up PLL"); return err; } /* INITPHY commands */ for_each_set_bit(link_id, &mask, 32) { err = gv100_nvlink_minion_send_command(g, link_id, minion_nvlink_dl_cmd_command_initphy_v(), 0, sync); if (err) { nvgpu_err(g, "Error on INITPHY minion DL command %u", link_id); return err; } } return 0; } /* * Configure AC coupling */ static u32 gv100_nvlink_minion_configure_ac_coupling(struct gk20a *g, unsigned long mask, bool sync) { u32 err = 0; u32 i; u32 temp; for_each_set_bit(i, &mask, 32) { temp = DLPL_REG_RD32(g, i, nvl_link_config_r()); temp &= ~nvl_link_config_ac_safe_en_m(); temp |= nvl_link_config_ac_safe_en_on_f(); DLPL_REG_WR32(g, i, nvl_link_config_r(), temp); err = gv100_nvlink_minion_send_command(g, i, minion_nvlink_dl_cmd_command_setacmode_v(), 0, sync); if (err) return err; } return err; } /* * Set Data ready */ int gv100_nvlink_minion_data_ready_en(struct gk20a *g, unsigned long link_mask, bool sync) { int ret = 0; u32 link_id; for_each_set_bit(link_id, &link_mask, 32) { ret = gv100_nvlink_minion_send_command(g, link_id, minion_nvlink_dl_cmd_command_initlaneenable_v(), 0, sync); if (ret) { nvgpu_err(g, "Failed initlaneenable on link %u", link_id); return ret; } } for_each_set_bit(link_id, &link_mask, 32) { ret = gv100_nvlink_minion_send_command(g, link_id, minion_nvlink_dl_cmd_command_initdlpl_v(), 0, sync); if (ret) { nvgpu_err(g, "Failed initdlpl on link %u", link_id); return ret; } } return ret; } /* * Request that minion disable the lane */ static u32 gv100_nvlink_minion_lane_disable(struct gk20a *g, u32 link_id, bool sync) { u32 err = 0; err = gv100_nvlink_minion_send_command(g, link_id, minion_nvlink_dl_cmd_command_lanedisable_v(), 0, sync); if (err) nvgpu_err(g, " failed to disable lane on %d", link_id); return err; } /* * Request that minion shutdown the lane */ static u32 gv100_nvlink_minion_lane_shutdown(struct gk20a *g, u32 link_id, bool sync) { u32 err = 0; err = gv100_nvlink_minion_send_command(g, link_id, minion_nvlink_dl_cmd_command_laneshutdown_v(), 0, sync); if (err) nvgpu_err(g, " failed to shutdown lane on %d", link_id); return err; } /* *-----------------------------------------------------------------------------* * TLC API *-----------------------------------------------------------------------------* */ static int gv100_nvlink_get_tlc_reginit(enum nvgpu_nvlink_endp endp, struct __nvlink_reginit **reg, u32 *count) { switch(endp) { case nvgpu_nvlink_endp_tegra: *reg = (struct __nvlink_reginit *) __nvlink_reginit_per_link_tegra; *count = ARRAY_SIZE(__nvlink_reginit_per_link_tegra); break; case nvgpu_nvlink_endp_gpu: *reg = (struct __nvlink_reginit *) __nvlink_reginit_per_link_gpu; *count = ARRAY_SIZE(__nvlink_reginit_per_link_gpu); break; default: return -EINVAL; } return 0; } /* * Init TLC IP and prod regs */ static void gv100_nvlink_initialize_tlc(struct gk20a *g, u32 link_id) { } /* * Init TLC per link interrupts */ static void gv100_nvlink_tlc_intr_enable(struct gk20a *g, u32 link_id, bool enable) { u32 reg_rx0 = 0, reg_rx1 = 0, reg_tx = 0; if (enable) { /* Set PROD values */ reg_rx0 = 0x0FFFFFF; reg_rx1 = 0x03FFFFF; reg_tx = 0x1FFFFFF; } TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_0_r(), reg_rx0); TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_1_r(), reg_rx1); TLC_REG_WR32(g, link_id, nvtlc_tx_err_report_en_0_r(), reg_tx); } /* * helper function to get TLC intr status in common structure */ static void gv100_nvlink_tlc_get_intr(struct gk20a *g, u32 link_id) { g->nvlink.tlc_rx_err_status_0[link_id] = TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_0_r()); g->nvlink.tlc_rx_err_status_1[link_id] = TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_1_r()); g->nvlink.tlc_tx_err_status_0[link_id] = TLC_REG_RD32(g, link_id, nvtlc_tx_err_status_0_r()); } /* * Interrupt routine handler for TLC */ static void gv100_nvlink_tlc_isr(struct gk20a *g, u32 link_id) { if (g->nvlink.tlc_rx_err_status_0[link_id]) { /* All TLC RX 0 errors are fatal. Notify and disable */ nvgpu_err(g, "Fatal TLC RX 0 interrupt on link %d mask: %x", link_id, g->nvlink.tlc_rx_err_status_0[link_id]); TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_0_r(), g->nvlink.tlc_rx_err_status_0[link_id]); TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_0_r(), g->nvlink.tlc_rx_err_status_0[link_id]); } if (g->nvlink.tlc_rx_err_status_1[link_id]) { /* All TLC RX 1 errors are fatal. Notify and disable */ nvgpu_err(g, "Fatal TLC RX 1 interrupt on link %d mask: %x", link_id, g->nvlink.tlc_rx_err_status_1[link_id]); TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_1_r(), g->nvlink.tlc_rx_err_status_1[link_id]); TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_1_r(), g->nvlink.tlc_rx_err_status_1[link_id]); } if (g->nvlink.tlc_tx_err_status_0[link_id]) { /* All TLC TX 0 errors are fatal. Notify and disable */ nvgpu_err(g, "Fatal TLC TX 0 interrupt on link %d mask: %x", link_id, g->nvlink.tlc_tx_err_status_0[link_id]); TLC_REG_WR32(g, link_id, nvtlc_tx_err_first_0_r(), g->nvlink.tlc_tx_err_status_0[link_id]); TLC_REG_WR32(g, link_id, nvtlc_tx_err_status_0_r(), g->nvlink.tlc_tx_err_status_0[link_id]); } } /* *-----------------------------------------------------------------------------* * DLPL API *-----------------------------------------------------------------------------* */ /* * DLPL interrupt enable helper */ static void gv100_nvlink_dlpl_intr_enable(struct gk20a *g, u32 link_id, bool enable) { u32 reg = 0; /* Always disable nonstall tree */ DLPL_REG_WR32(g, link_id, nvl_intr_nonstall_en_r(), 0); if (!enable) { DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), 0); return; } /* Clear interrupt register to get rid of stale state (W1C) */ DLPL_REG_WR32(g, link_id, nvl_intr_r(), 0xffffffff); DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff); reg = nvl_intr_stall_en_tx_recovery_long_enable_f() | nvl_intr_stall_en_tx_fault_ram_enable_f() | nvl_intr_stall_en_tx_fault_interface_enable_f() | nvl_intr_stall_en_tx_fault_sublink_change_enable_f() | nvl_intr_stall_en_rx_fault_sublink_change_enable_f() | nvl_intr_stall_en_rx_fault_dl_protocol_enable_f() | nvl_intr_stall_en_ltssm_fault_enable_f(); DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), reg); /* Configure error threshold */ reg = DLPL_REG_RD32(g, link_id, nvl_sl1_error_rate_ctrl_r()); reg = set_field(reg, nvl_sl1_error_rate_ctrl_short_threshold_man_m(), nvl_sl1_error_rate_ctrl_short_threshold_man_f(0x2)); reg = set_field(reg, nvl_sl1_error_rate_ctrl_long_threshold_man_m(), nvl_sl1_error_rate_ctrl_long_threshold_man_f(0x2)); DLPL_REG_WR32(g, link_id, nvl_sl1_error_rate_ctrl_r(), reg); } /* * DLPL per-link isr */ #define DLPL_NON_FATAL_INTR_MASK (nvl_intr_tx_replay_f(1) | \ nvl_intr_tx_recovery_short_f(1) | \ nvl_intr_tx_recovery_long_f(1) | \ nvl_intr_rx_short_error_rate_f(1) | \ nvl_intr_rx_long_error_rate_f(1) | \ nvl_intr_rx_ila_trigger_f(1) | \ nvl_intr_ltssm_protocol_f(1)) #define DLPL_FATAL_INTR_MASK ( nvl_intr_ltssm_fault_f(1) | \ nvl_intr_rx_fault_dl_protocol_f(1) | \ nvl_intr_rx_fault_sublink_change_f(1) | \ nvl_intr_tx_fault_sublink_change_f(1) | \ nvl_intr_tx_fault_interface_f(1) | \ nvl_intr_tx_fault_ram_f(1)) static void gv100_nvlink_dlpl_isr(struct gk20a *g, u32 link_id) { u32 non_fatal_mask = 0; u32 fatal_mask = 0; u32 intr = 0; bool retrain = false; u32 err; intr = DLPL_REG_RD32(g, link_id, nvl_intr_r()) & DLPL_REG_RD32(g, link_id, nvl_intr_stall_en_r()); if (!intr) return; fatal_mask = intr & DLPL_FATAL_INTR_MASK; non_fatal_mask = intr & DLPL_NON_FATAL_INTR_MASK; nvgpu_err(g, " handling DLPL %d isr. Fatal: %x non-Fatal: %x", link_id, fatal_mask, non_fatal_mask); /* Check if we are not handling an interupt */ if ((fatal_mask | non_fatal_mask) & ~intr) nvgpu_err(g, "Unable to service DLPL intr on link %d", link_id); if (non_fatal_mask & nvl_intr_tx_recovery_long_f(1)) retrain = true; if (fatal_mask) retrain = false; if (retrain) { err = nvgpu_nvlink_train(g, link_id, false); if (err) nvgpu_err(g, "failed to retrain link %d", link_id); } /* Clear interrupts */ DLPL_REG_WR32(g, link_id, nvl_intr_r(), (non_fatal_mask | fatal_mask)); DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff); } /* *-----------------------------------------------------------------------------* * MIF API *-----------------------------------------------------------------------------* */ /* * Initialize MIF API with PROD settings */ static void gv100_nvlink_initialize_mif(struct gk20a *g, u32 link_id) { u32 tmp; /* Enable MIF RX error */ /* Containment (make fatal) */ tmp = 0; tmp = set_field(tmp, ioctrlmif_rx_err_contain_en_0_rxramdataparityerr_m(), ioctrlmif_rx_err_contain_en_0_rxramdataparityerr__prod_f()); tmp = set_field(tmp, ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr_m(), ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr__prod_f()); MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_contain_en_0_r(), tmp); /* Logging (do not ignore) */ tmp = 0; tmp = set_field(tmp, ioctrlmif_rx_err_log_en_0_rxramdataparityerr_m(), ioctrlmif_rx_err_log_en_0_rxramdataparityerr_f(1)); tmp = set_field(tmp, ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_m(), ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_f(1)); MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_log_en_0_r(), tmp); /* Tx Error */ /* Containment (make fatal) */ tmp = 0; tmp = set_field(tmp, ioctrlmif_tx_err_contain_en_0_txramdataparityerr_m(), ioctrlmif_tx_err_contain_en_0_txramdataparityerr__prod_f()); tmp = set_field(tmp, ioctrlmif_tx_err_contain_en_0_txramhdrparityerr_m(), ioctrlmif_tx_err_contain_en_0_txramhdrparityerr__prod_f()); MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_contain_en_0_r(), tmp); /* Logging (do not ignore) */ tmp = 0; tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramdataparityerr_m(), ioctrlmif_tx_err_log_en_0_txramdataparityerr_f(1)); tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramhdrparityerr_m(), ioctrlmif_tx_err_log_en_0_txramhdrparityerr_f(1)); MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_log_en_0_r(), tmp); /* Credit release */ MIF_REG_WR32(g, link_id, ioctrlmif_rx_ctrl_buffer_ready_r(), 0x1); MIF_REG_WR32(g, link_id, ioctrlmif_tx_ctrl_buffer_ready_r(), 0x1); } /* * Enable per-link MIF interrupts */ static void gv100_nvlink_mif_intr_enable(struct gk20a *g, u32 link_id, bool enable) { u32 reg0 = 0, reg1 = 0; if (enable) { reg0 = set_field(reg0, ioctrlmif_rx_err_report_en_0_rxramdataparityerr_m(), ioctrlmif_rx_err_report_en_0_rxramdataparityerr_f(1)); reg0 = set_field(reg0, ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_m(), ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_f(1)); reg1 = set_field(reg1, ioctrlmif_tx_err_report_en_0_txramdataparityerr_m(), ioctrlmif_tx_err_report_en_0_txramdataparityerr_f(1)); reg1 = set_field(reg1, ioctrlmif_tx_err_report_en_0_txramhdrparityerr_m(), ioctrlmif_tx_err_report_en_0_txramhdrparityerr_f(1)); } MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_report_en_0_r(), reg0); MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_report_en_0_r(), reg1); } /* * Handle per-link MIF interrupts */ static void gv100_nvlink_mif_isr(struct gk20a *g, u32 link_id) { u32 intr, fatal_mask = 0; /* RX Errors */ intr = MIF_REG_RD32(g, link_id, ioctrlmif_rx_err_status_0_r()); if (intr) { if (intr & ioctrlmif_rx_err_status_0_rxramdataparityerr_m()) { nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_DATA_PARITY", link_id); fatal_mask |= ioctrlmif_rx_err_status_0_rxramdataparityerr_f(1); } if (intr & ioctrlmif_rx_err_status_0_rxramhdrparityerr_m()) { nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_HDR_PARITY", link_id); fatal_mask |= ioctrlmif_rx_err_status_0_rxramhdrparityerr_f(1); } if (fatal_mask) { MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_first_0_r(), fatal_mask); MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_status_0_r(), fatal_mask); } } /* TX Errors */ fatal_mask = 0; intr = MIF_REG_RD32(g, link_id, ioctrlmif_tx_err_status_0_r()); if (intr) { if (intr & ioctrlmif_tx_err_status_0_txramdataparityerr_m()) { nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_DATA_PARITY", link_id); fatal_mask |= ioctrlmif_tx_err_status_0_txramdataparityerr_f(1); } if (intr & ioctrlmif_tx_err_status_0_txramhdrparityerr_m()) { nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_HDR_PARITY", link_id); fatal_mask |= ioctrlmif_tx_err_status_0_txramhdrparityerr_f(1); } if (fatal_mask) { MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_first_0_r(), fatal_mask); MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_status_0_r(), fatal_mask); } } } /* *-----------------------------------------------------------------------------* * NVLIPT API *-----------------------------------------------------------------------------* */ /* * NVLIPT IP initialization (per-link) */ static void gv100_nvlink_initialize_nvlipt(struct gk20a *g, u32 link_id) { /* init persistent scratch registers */ IPT_REG_WR32(g, nvlipt_scratch_cold_r(), nvlipt_scratch_cold_data_init_v()); /* * AErr settings (top level) */ /* UC first and status reg (W1C) need to be cleared byt arch */ IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); /* AErr Severity */ IPT_REG_WR32(g, IPT_ERR_UC_SEVERITY_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); /* AErr Control settings */ IPT_REG_WR32(g, IPT_ERR_CONTROL_LINK(link_id), nvlipt_err_control_link0_fatalenable_f(1) | nvlipt_err_control_link0_nonfatalenable_f(1)); } /* * Enable NVLIPT interrupts */ static void gv100_nvlink_nvlipt_intr_enable(struct gk20a *g, u32 link_id, bool enable) { u32 val = 0; u32 reg; if (enable) val = 1; reg = IPT_REG_RD32(g, IPT_INTR_CONTROL_LINK(link_id)); reg = set_field(reg, nvlipt_intr_control_link0_stallenable_m(), nvlipt_intr_control_link0_stallenable_f(val)); reg = set_field(reg, nvlipt_intr_control_link0_nostallenable_m(), nvlipt_intr_control_link0_nostallenable_f(val)); IPT_REG_WR32(g, IPT_INTR_CONTROL_LINK(link_id), reg); } /* * Per-link NVLIPT ISR handler */ static bool gv100_nvlink_nvlipt_isr(struct gk20a *g, u32 link_id) { /* * Interrupt handling happens in leaf handlers. Assume all interrupts * were handled and clear roll ups/ */ IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); return true; } /* ******************************************************************************* * Interrupt handling functions * ******************************************************************************* */ /* * Enable common interrupts */ static void gv100_nvlink_common_intr_enable(struct gk20a *g, unsigned long mask) { u32 reg, i; /* Init IOCTRL */ for_each_set_bit(i, &mask, 32) { reg = IOCTRL_REG_RD32(g, ioctrl_link_intr_0_mask_r(i)); reg |= (ioctrl_link_intr_0_mask_fatal_f(1) | ioctrl_link_intr_0_mask_nonfatal_f(1) | ioctrl_link_intr_0_mask_correctable_f(1) | ioctrl_link_intr_0_mask_intra_f(1)); IOCTRL_REG_WR32(g, ioctrl_link_intr_0_mask_r(i), reg); } reg = IOCTRL_REG_RD32(g, ioctrl_common_intr_0_mask_r()); reg |= (ioctrl_common_intr_0_mask_fatal_f(1) | ioctrl_common_intr_0_mask_nonfatal_f(1) | ioctrl_common_intr_0_mask_correctable_f(1) | ioctrl_common_intr_0_mask_intra_f(1)); IOCTRL_REG_WR32(g, ioctrl_common_intr_0_mask_r(), reg); /* Init NVLIPT */ IPT_REG_WR32(g, nvlipt_intr_control_common_r(), nvlipt_intr_control_common_stallenable_f(1) | nvlipt_intr_control_common_nonstallenable_f(1)); } /* * Enable link specific interrupts (top-level) */ static void gv100_nvlink_enable_link_intr(struct gk20a *g, u32 link_id, bool enable) { gv100_nvlink_minion_link_intr_enable(g, link_id, enable); gv100_nvlink_dlpl_intr_enable(g, link_id, enable); gv100_nvlink_tlc_intr_enable(g, link_id, enable); gv100_nvlink_mif_intr_enable(g, link_id, enable); gv100_nvlink_nvlipt_intr_enable(g, link_id, enable); } /* * Top level interrupt handler */ int gv100_nvlink_isr(struct gk20a *g) { unsigned long links; u32 link_id; links = ioctrl_top_intr_0_status_link_v( IOCTRL_REG_RD32(g, ioctrl_top_intr_0_status_r())); links &= g->nvlink.enabled_links; /* As per ARCH minion must be serviced first */ gv100_nvlink_minion_isr(g); for_each_set_bit(link_id, &links, 32) { /* Cache error logs from TLC, DL handler may clear them */ gv100_nvlink_tlc_get_intr(g, link_id); gv100_nvlink_dlpl_isr(g, link_id); gv100_nvlink_tlc_isr(g, link_id); gv100_nvlink_mif_isr(g, link_id); /* NVLIPT is top-level. Do it last */ gv100_nvlink_nvlipt_isr(g, link_id); } return 0; } /******************************************************************************* * Helper functions * ******************************************************************************* */ static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g) { u32 reg_data; reg_data = IOCTRL_REG_RD32(g, ioctrl_reset_r()); return ioctrl_reset_linkreset_v(reg_data); } static u32 __gv100_nvlink_state_load_hal(struct gk20a *g) { unsigned long discovered = g->nvlink.discovered_links; gv100_nvlink_common_intr_enable(g, discovered); return gv100_nvlink_minion_load(g); } #define TRIM_SYS_NVLINK_CTRL(i) (trim_sys_nvlink0_ctrl_r() + 16*i) #define TRIM_SYS_NVLINK_STATUS(i) (trim_sys_nvlink0_status_r() + 16*i) int gv100_nvlink_setup_pll(struct gk20a *g, unsigned long link_mask) { u32 reg; u32 i; u32 links_off; struct nvgpu_timeout timeout; u32 pad_ctrl = 0; u32 swap_ctrl = 0; u32 pll_id; reg = gk20a_readl(g, trim_sys_nvlink_uphy_cfg_r()); reg = set_field(reg, trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_m(), trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_f(1)); gk20a_writel(g, trim_sys_nvlink_uphy_cfg_r(), reg); if (g->ops.top.get_nvhsclk_ctrl_e_clk_nvl) { pad_ctrl = g->ops.top.get_nvhsclk_ctrl_e_clk_nvl(g); } if (g->ops.top.get_nvhsclk_ctrl_swap_clk_nvl) { swap_ctrl = g->ops.top.get_nvhsclk_ctrl_swap_clk_nvl(g); } for_each_set_bit(i, &link_mask, 32) { /* There are 3 PLLs for 6 links. We have 3 bits for each PLL. * The PLL bit corresponding to a link is /2 of its master link. */ pll_id = g->nvlink.links[i].pll_master_link_id >> 1; pad_ctrl |= BIT(pll_id); swap_ctrl |= BIT(pll_id); } if (g->ops.top.set_nvhsclk_ctrl_e_clk_nvl) { g->ops.top.set_nvhsclk_ctrl_e_clk_nvl(g, pad_ctrl); } if (g->ops.top.set_nvhsclk_ctrl_swap_clk_nvl) { g->ops.top.set_nvhsclk_ctrl_swap_clk_nvl(g, swap_ctrl); } for_each_set_bit(i, &link_mask, 32) { reg = gk20a_readl(g, TRIM_SYS_NVLINK_CTRL(i)); reg = set_field(reg, trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_m(), trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_f(0)); gk20a_writel(g, TRIM_SYS_NVLINK_CTRL(i), reg); } /* Poll for links to go up */ links_off = link_mask; nvgpu_timeout_init(g, &timeout, NVLINK_PLL_ON_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER); do { for_each_set_bit(i, &link_mask, 32) { reg = gk20a_readl(g, TRIM_SYS_NVLINK_STATUS(i)); if (trim_sys_nvlink0_status_pll_off_v(reg) == 0) links_off &= ~BIT(i); } nvgpu_udelay(5); } while((!nvgpu_timeout_expired_msg(&timeout, "timeout on pll on")) && links_off); if (nvgpu_timeout_peek_expired(&timeout)) return -ETIMEDOUT; return 0; } static void gv100_nvlink_prog_alt_clk(struct gk20a *g) { u32 tmp; /* RMW registers need to be separate */ tmp = gk20a_readl(g, trim_sys_nvl_common_clk_alt_switch_r()); tmp &= ~trim_sys_nvl_common_clk_alt_switch_slowclk_m(); tmp |= trim_sys_nvl_common_clk_alt_switch_slowclk_xtal4x_f(); gk20a_writel(g, trim_sys_nvl_common_clk_alt_switch_r(), tmp); } static int gv100_nvlink_enable_links_pre_top(struct gk20a *g, u32 links) { u32 link_id; unsigned long enabled_links = links; u32 tmp; u32 reg; u32 delay = ioctrl_reset_sw_post_reset_delay_microseconds_v(); u32 err; nvgpu_log(g, gpu_dbg_nvlink, " enabling 0x%lx links", enabled_links); /* Take links out of reset */ for_each_set_bit(link_id, &enabled_links, 32) { reg = IOCTRL_REG_RD32(g, ioctrl_reset_r()); tmp = (BIT(link_id) | BIT(g->nvlink.links[link_id].pll_master_link_id)); reg = set_field(reg, ioctrl_reset_linkreset_m(), ioctrl_reset_linkreset_f( ioctrl_reset_linkreset_v(reg) | tmp)); IOCTRL_REG_WR32(g, ioctrl_reset_r(), reg); nvgpu_udelay(delay); reg = IOCTRL_REG_RD32(g, ioctrl_debug_reset_r()); reg &= ~ioctrl_debug_reset_link_f(BIT(link_id)); IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg); nvgpu_udelay(delay); reg |= ioctrl_debug_reset_link_f(BIT(link_id)); IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg); nvgpu_udelay(delay); /* Before doing any link initialization, run RXDET to check * if link is connected on other end. */ if (g->ops.nvlink.rxdet) { err = g->ops.nvlink.rxdet(g, link_id); if (err) return err; } /* Enable Link DLPL for AN0 */ reg = DLPL_REG_RD32(g, link_id, nvl_link_config_r()); reg = set_field(reg, nvl_link_config_link_en_m(), nvl_link_config_link_en_f(1)); DLPL_REG_WR32(g, link_id, nvl_link_config_r(), reg); /* This should be done by the NVLINK API */ err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true); if (err) { nvgpu_err(g, "Failed to init phy of link: %u", link_id); return err; } err = gv100_nvlink_rxcal_en(g, BIT(link_id)); if (err) { nvgpu_err(g, "Failed to RXcal on link: %u", link_id); return err; } err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true); if (err) { nvgpu_err(g, "Failed to set data ready link:%u", link_id); return err; } g->nvlink.enabled_links |= BIT(link_id); } nvgpu_log(g, gpu_dbg_nvlink, "enabled_links=0x%08x", g->nvlink.enabled_links); if (g->nvlink.enabled_links) return 0; nvgpu_err(g, " No links were enabled"); return -EINVAL; } void gv100_nvlink_set_sw_war(struct gk20a *g, u32 link_id) { u32 reg; /* WAR for HW bug 1888034 */ reg = DLPL_REG_RD32(g, link_id, nvl_sl0_safe_ctrl2_tx_r()); reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_init_m(), nvl_sl0_safe_ctrl2_tx_ctr_init_init_f()); reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_initscl_m(), nvl_sl0_safe_ctrl2_tx_ctr_initscl_init_f()); DLPL_REG_WR32(g, link_id, nvl_sl0_safe_ctrl2_tx_r(), reg); } static int gv100_nvlink_enable_links_post_top(struct gk20a *g, u32 links) { u32 link_id; unsigned long enabled_links = (links & g->nvlink.enabled_links) & ~g->nvlink.initialized_links; for_each_set_bit(link_id, &enabled_links, 32) { if (g->ops.nvlink.set_sw_war) g->ops.nvlink.set_sw_war(g, link_id); gv100_nvlink_initialize_tlc(g, link_id); gv100_nvlink_initialize_nvlipt(g, link_id); gv100_nvlink_enable_link_intr(g, link_id, true); g->nvlink.initialized_links |= BIT(link_id); }; return 0; } static u32 gv100_nvlink_prbs_gen_en(struct gk20a *g, unsigned long mask) { u32 reg; u32 link_id; for_each_set_bit(link_id, &mask, 32) { /* Write is required as part of HW sequence */ DLPL_REG_WR32(g, link_id, nvl_sl1_rxslsm_timeout_2_r(), 0); reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r()); reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(), nvl_txiobist_config_dpg_prbsseedld_f(0x1)); DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg); reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r()); reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(), nvl_txiobist_config_dpg_prbsseedld_f(0x0)); DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg); } return 0; } static u32 gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask) { u32 link_id; struct nvgpu_timeout timeout; u32 reg; for_each_set_bit(link_id, &mask, 32) { /* Timeout from HW specs */ nvgpu_timeout_init(g, &timeout, 8*NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER); reg = DLPL_REG_RD32(g, link_id, nvl_br0_cfg_cal_r()); reg = set_field(reg, nvl_br0_cfg_cal_rxcal_m(), nvl_br0_cfg_cal_rxcal_on_f()); DLPL_REG_WR32(g, link_id, nvl_br0_cfg_cal_r(), reg); do { reg = DLPL_REG_RD32(g, link_id, nvl_br0_cfg_status_cal_r()); if (nvl_br0_cfg_status_cal_rxcal_done_v(reg) == 1) break; nvgpu_udelay(5); } while(!nvgpu_timeout_expired_msg(&timeout, "timeout on rxcal")); if (nvgpu_timeout_peek_expired(&timeout)) return -ETIMEDOUT; } return 0; } /* ******************************************************************************* * Internal "ops" functions * ******************************************************************************* */ /* * Main Nvlink init function. Calls into the Nvlink core API */ int gv100_nvlink_init(struct gk20a *g) { int err = 0; if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) return -ENODEV; err = nvgpu_nvlink_enumerate(g); if (err) { nvgpu_err(g, "failed to enumerate nvlink"); goto fail; } /* Set HSHUB and SG_PHY */ __nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, true); err = g->ops.fb.enable_nvlink(g); if (err) { nvgpu_err(g, "failed switch to nvlink sysmem"); goto fail; } return err; fail: __nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, false); __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, false); return err; } /* * Query internal device topology and discover devices in nvlink local * infrastructure. Initialize register base and offsets */ int gv100_nvlink_discover_link(struct gk20a *g) { u32 i; u32 ioctrl_entry_addr; u8 ioctrl_device_type; u32 table_entry; u32 ioctrl_info_entry_type; u8 ioctrl_discovery_size; bool is_chain = false; u8 nvlink_num_devices = 0; unsigned long available_links = 0; struct nvgpu_nvlink_device_list *device_table; u32 err = 0; /* * Process Entry 0 & 1 of IOCTRL table to find table size */ if (g->nvlink.ioctrl_table && g->nvlink.ioctrl_table[0].pri_base_addr) { ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr; table_entry = gk20a_readl(g, ioctrl_entry_addr); ioctrl_info_entry_type = nvlinkip_discovery_common_device_v(table_entry); } else { nvgpu_err(g, " Bad IOCTRL PRI Base addr"); return -EINVAL; } if (ioctrl_info_entry_type == NVL_DEVICE(ioctrl)) { ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr + 4; table_entry = gk20a_readl(g, ioctrl_entry_addr); ioctrl_discovery_size = nvlinkip_discovery_common_ioctrl_length_v(table_entry); nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL size: %d", ioctrl_discovery_size); } else { nvgpu_err(g, " First entry of IOCTRL_DISCOVERY invalid"); return -EINVAL; } device_table = nvgpu_kzalloc(g, ioctrl_discovery_size * sizeof(struct nvgpu_nvlink_device_list)); if (!device_table) { nvgpu_err(g, " Unable to allocate nvlink device table"); return -ENOMEM; } for (i = 0; i < ioctrl_discovery_size; i++) { ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr + 4*i; table_entry = gk20a_readl(g, ioctrl_entry_addr); nvgpu_log(g, gpu_dbg_nvlink, "parsing ioctrl %d: 0x%08x", i, table_entry); ioctrl_info_entry_type = nvlinkip_discovery_common_entry_v(table_entry); if (ioctrl_info_entry_type == nvlinkip_discovery_common_entry_invalid_v()) continue; if (ioctrl_info_entry_type == nvlinkip_discovery_common_entry_enum_v()) { nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is ENUM", i); ioctrl_device_type = nvlinkip_discovery_common_device_v(table_entry); if (nvlinkip_discovery_common_chain_v(table_entry) != nvlinkip_discovery_common_chain_enable_v()) { nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is ENUM but no chain", i); err = -EINVAL; break; } is_chain = true; device_table[nvlink_num_devices].valid = true; device_table[nvlink_num_devices].device_type = ioctrl_device_type; device_table[nvlink_num_devices].device_id = nvlinkip_discovery_common_id_v(table_entry); device_table[nvlink_num_devices].device_version = nvlinkip_discovery_common_version_v( table_entry); continue; } if (ioctrl_info_entry_type == nvlinkip_discovery_common_entry_data1_v()) { nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA1", i); if (is_chain) { device_table[nvlink_num_devices].pri_base_addr = nvlinkip_discovery_common_pri_base_v( table_entry) << 12; device_table[nvlink_num_devices].intr_enum = nvlinkip_discovery_common_intr_v( table_entry); device_table[nvlink_num_devices].reset_enum = nvlinkip_discovery_common_reset_v( table_entry); nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d type = %d base: 0x%08x intr: %d reset: %d", i, device_table[nvlink_num_devices].device_type, device_table[nvlink_num_devices].pri_base_addr, device_table[nvlink_num_devices].intr_enum, device_table[nvlink_num_devices].reset_enum); if (device_table[nvlink_num_devices].device_type == NVL_DEVICE(dlpl)) { device_table[nvlink_num_devices].num_tx = nvlinkip_discovery_common_dlpl_num_tx_v(table_entry); device_table[nvlink_num_devices].num_rx = nvlinkip_discovery_common_dlpl_num_rx_v(table_entry); nvgpu_log(g, gpu_dbg_nvlink, "DLPL tx: %d rx: %d", device_table[nvlink_num_devices].num_tx, device_table[nvlink_num_devices].num_rx); } if (nvlinkip_discovery_common_chain_v(table_entry) != nvlinkip_discovery_common_chain_enable_v()) { is_chain = false; nvlink_num_devices++; } } continue; } if (ioctrl_info_entry_type == nvlinkip_discovery_common_entry_data2_v()) { nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA2", i); if (is_chain) { if (nvlinkip_discovery_common_dlpl_data2_type_v(table_entry)) { device_table[nvlink_num_devices].pll_master = nvlinkip_discovery_common_dlpl_data2_master_v(table_entry); device_table[nvlink_num_devices].pll_master_id = nvlinkip_discovery_common_dlpl_data2_masterid_v(table_entry); nvgpu_log(g, gpu_dbg_nvlink, "PLL info: Master: %d, Master ID: %d", device_table[nvlink_num_devices].pll_master, device_table[nvlink_num_devices].pll_master_id); } if (nvlinkip_discovery_common_chain_v(table_entry) != nvlinkip_discovery_common_chain_enable_v()) { is_chain = false; nvlink_num_devices++; } } continue; } } g->nvlink.device_table = device_table; g->nvlink.num_devices = nvlink_num_devices; /* * Print table */ for (i = 0; i < nvlink_num_devices; i++) { if (device_table[i].valid) { nvgpu_log(g, gpu_dbg_nvlink, "Device %d - %s", i, __gv100_device_type_to_str( device_table[i].device_type)); nvgpu_log(g, gpu_dbg_nvlink, "+Link/Device Id: %d", device_table[i].device_id); nvgpu_log(g, gpu_dbg_nvlink, "+Version: %d", device_table[i].device_version); nvgpu_log(g, gpu_dbg_nvlink, "+Base Addr: 0x%08x", device_table[i].pri_base_addr); nvgpu_log(g, gpu_dbg_nvlink, "+Intr Enum: %d", device_table[i].intr_enum); nvgpu_log(g, gpu_dbg_nvlink, "+Reset Enum: %d", device_table[i].reset_enum); if ((device_table[i].device_type == NVL_DEVICE(dlpl)) || (device_table[i].device_type == NVL_DEVICE(nvlink))) { nvgpu_log(g, gpu_dbg_nvlink, "+TX: %d", device_table[i].num_tx); nvgpu_log(g, gpu_dbg_nvlink, "+RX: %d", device_table[i].num_rx); nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master: %d", device_table[i].pll_master); nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master ID: %d", device_table[i].pll_master_id); } } } for (i = 0; i < nvlink_num_devices; i++) { if (device_table[i].valid) { if (device_table[i].device_type == NVL_DEVICE(ioctrl)) { g->nvlink.ioctrl_type = device_table[i].device_type; g->nvlink.ioctrl_base = device_table[i].pri_base_addr; continue; } if (device_table[i].device_type == NVL_DEVICE(dlpl)) { g->nvlink.dlpl_type = device_table[i].device_type; g->nvlink.dlpl_base[device_table[i].device_id] = device_table[i].pri_base_addr; g->nvlink.links[device_table[i].device_id].valid = true; g->nvlink.links[device_table[i].device_id].g = g; g->nvlink.links[device_table[i].device_id].dlpl_version = device_table[i].device_version; g->nvlink.links[device_table[i].device_id].dlpl_base = device_table[i].pri_base_addr; g->nvlink.links[device_table[i].device_id].intr_enum = device_table[i].intr_enum; g->nvlink.links[device_table[i].device_id].reset_enum = device_table[i].reset_enum; g->nvlink.links[device_table[i].device_id].link_id = device_table[i].device_id; /* initiate the PLL master and slave link id to max */ g->nvlink.links[device_table[i].device_id].pll_master_link_id = NVLINK_MAX_LINKS_SW; g->nvlink.links[device_table[i].device_id].pll_slave_link_id = NVLINK_MAX_LINKS_SW; /* Update Pll master */ if (device_table[i].pll_master) g->nvlink.links[device_table[i].device_id].pll_master_link_id = g->nvlink.links[device_table[i].device_id].link_id; else { g->nvlink.links[device_table[i].device_id].pll_master_link_id = device_table[i].pll_master_id; g->nvlink.links[device_table[i].device_id].pll_slave_link_id = g->nvlink.links[device_table[i].device_id].link_id; g->nvlink.links[device_table[i].pll_master_id].pll_slave_link_id = g->nvlink.links[device_table[i].device_id].link_id; } available_links |= BIT(device_table[i].device_id); continue; } if (device_table[i].device_type == NVL_DEVICE(nvltlc)) { g->nvlink.tl_type = device_table[i].device_type; g->nvlink.tl_base[device_table[i].device_id] = device_table[i].pri_base_addr; g->nvlink.links[device_table[i].device_id].tl_base = device_table[i].pri_base_addr; g->nvlink.links[device_table[i].device_id].tl_version = device_table[i].device_version; continue; } if (device_table[i].device_type == NVL_DEVICE(nvltlc)) { g->nvlink.tl_type = device_table[i].device_type; g->nvlink.tl_base[device_table[i].device_id] = device_table[i].pri_base_addr; g->nvlink.links[device_table[i].device_id].tl_base = device_table[i].pri_base_addr; g->nvlink.links[device_table[i].device_id].tl_version = device_table[i].device_version; continue; } if (device_table[i].device_type == NVL_DEVICE(ioctrlmif)) { g->nvlink.mif_type = device_table[i].device_type; g->nvlink.mif_base[device_table[i].device_id] = device_table[i].pri_base_addr; g->nvlink.links[device_table[i].device_id].mif_base = device_table[i].pri_base_addr; g->nvlink.links[device_table[i].device_id].mif_version = device_table[i].device_version; continue; } if (device_table[i].device_type == NVL_DEVICE(nvlipt)) { g->nvlink.ipt_type = device_table[i].device_type; g->nvlink.ipt_base = device_table[i].pri_base_addr; g->nvlink.ipt_version = device_table[i].device_version; continue; } if (device_table[i].device_type == NVL_DEVICE(minion)) { g->nvlink.minion_type = device_table[i].device_type; g->nvlink.minion_base = device_table[i].pri_base_addr; g->nvlink.minion_version = device_table[i].device_version; continue; } if (device_table[i].device_type == NVL_DEVICE(dlpl_multicast)) { g->nvlink.dlpl_multicast_type = device_table[i].device_type; g->nvlink.dlpl_multicast_base = device_table[i].pri_base_addr; g->nvlink.dlpl_multicast_version = device_table[i].device_version; continue; } if (device_table[i].device_type == NVL_DEVICE(nvltlc_multicast)) { g->nvlink.tl_multicast_type = device_table[i].device_type; g->nvlink.tl_multicast_base = device_table[i].pri_base_addr; g->nvlink.tl_multicast_version = device_table[i].device_version; continue; } if (device_table[i].device_type == NVL_DEVICE(ioctrlmif_multicast)) { g->nvlink.mif_multicast_type = device_table[i].device_type; g->nvlink.mif_multicast_base = device_table[i].pri_base_addr; g->nvlink.mif_multicast_version = device_table[i].device_version; continue; } } } g->nvlink.discovered_links = (u32) available_links; nvgpu_log(g, gpu_dbg_nvlink, "Nvlink Tree:"); nvgpu_log(g, gpu_dbg_nvlink, "+ Available Links: 0x%08lx", available_links); nvgpu_log(g, gpu_dbg_nvlink, "+ Per-Link Devices:"); for_each_set_bit(i, &available_links, 32) { nvgpu_log(g, gpu_dbg_nvlink, "-- Link %d Dl/Pl Base: 0x%08x TLC Base: 0x%08x MIF Base: 0x%08x", i, g->nvlink.dlpl_base[i], g->nvlink.tl_base[i], g->nvlink.mif_base[i]); } nvgpu_log(g, gpu_dbg_nvlink, "+ IOCTRL Base: 0x%08x", g->nvlink.ioctrl_base); nvgpu_log(g, gpu_dbg_nvlink, "+ NVLIPT Base: 0x%08x", g->nvlink.ipt_base); nvgpu_log(g, gpu_dbg_nvlink, "+ MINION Base: 0x%08x", g->nvlink.minion_base); nvgpu_log(g, gpu_dbg_nvlink, "+ DLPL MCAST Base: 0x%08x", g->nvlink.dlpl_multicast_base); nvgpu_log(g, gpu_dbg_nvlink, "+ TLC MCAST Base: 0x%08x", g->nvlink.tl_multicast_base); nvgpu_log(g, gpu_dbg_nvlink, "+ MIF MCAST Base: 0x%08x", g->nvlink.mif_multicast_base); if (!g->nvlink.minion_version) { nvgpu_err(g, "Unsupported MINION version"); nvgpu_kfree(g, device_table); g->nvlink.device_table = NULL; g->nvlink.num_devices = 0; return -EINVAL; } return err; } /* * Query IOCTRL for device discovery */ int gv100_nvlink_discover_ioctrl(struct gk20a *g) { u32 i; struct nvgpu_nvlink_ioctrl_list *ioctrl_table; u32 table_entry; u32 devinfo_type; u32 io_num_entries = 0; u32 entry_engine = 0; u32 entry_enum = 0; u32 entry_data = 0; ioctrl_table = nvgpu_kzalloc(g, top_device_info__size_1_v() * sizeof(struct nvgpu_nvlink_ioctrl_list)); if (!ioctrl_table) { nvgpu_err(g, "failed to allocate memory for nvlink io table"); return -ENOMEM; } for (i = 0; i < top_device_info__size_1_v(); i++) { table_entry = gk20a_readl(g, top_device_info_r(i)); nvgpu_log(g, gpu_dbg_nvlink, "Table entry: 0x%x", table_entry); devinfo_type = top_device_info_entry_v(table_entry); if (devinfo_type == top_device_info_entry_not_valid_v()) { nvgpu_log(g, gpu_dbg_nvlink, "Invalid entry"); continue; } if (devinfo_type == top_device_info_entry_engine_type_v()) { entry_engine = table_entry; } else if (devinfo_type == top_device_info_entry_data_v()) { entry_data = table_entry; } else if (devinfo_type == top_device_info_entry_enum_v()) { entry_enum = table_entry; } if (top_device_info_chain_v(table_entry) == top_device_info_chain_enable_v()) { continue; } if (top_device_info_type_enum_v(entry_engine) == top_device_info_type_enum_ioctrl_v()) { nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entries"); nvgpu_log(g, gpu_dbg_nvlink, " enum: 0x%x, engine = 0x%x, data = 0x%x", entry_enum, entry_engine, entry_data); ioctrl_table[io_num_entries].valid = true; ioctrl_table[io_num_entries].intr_enum = top_device_info_intr_enum_v(entry_enum); ioctrl_table[io_num_entries].reset_enum = top_device_info_reset_enum_v(entry_enum); ioctrl_table[io_num_entries].pri_base_addr = top_device_info_data_pri_base_v(entry_data) << top_device_info_data_pri_base_align_v(); io_num_entries++; } } if (io_num_entries == 0 || !ioctrl_table[0].pri_base_addr) { nvgpu_err(g, "No NVLINK io found"); nvgpu_kfree(g, ioctrl_table); return -EINVAL; } g->nvlink.ioctrl_table = ioctrl_table; g->nvlink.io_num_entries = io_num_entries; for (i =0; i < io_num_entries; i++) nvgpu_log(g, gpu_dbg_nvlink, "Device %d : Pri Base Addr = 0x%0x Intr = %d Reset = %d", i, ioctrl_table[i].pri_base_addr, ioctrl_table[i].intr_enum, ioctrl_table[i].reset_enum); return 0; } /* ******************************************************************************* * NVLINK API FUNCTIONS * ******************************************************************************* */ /* * Performs link level initialization like phy inits, AN0 and interrupts */ int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask) { int err; err = gv100_nvlink_enable_links_pre_top(g, mask); if (err) { nvgpu_err(g, "Pre topology failed for links %lx", mask); return err; } nvgpu_log(g, gpu_dbg_nvlink, "pretopology enabled: 0x%lx", mask & g->nvlink.enabled_links); err = gv100_nvlink_enable_links_post_top(g, mask); return err; } /* * Performs memory interface initialization */ int gv100_nvlink_interface_init(struct gk20a *g) { unsigned long mask = g->nvlink.enabled_links; u32 link_id; int err; for_each_set_bit(link_id, &mask, 32) { gv100_nvlink_initialize_mif(g, link_id); gv100_nvlink_mif_intr_enable(g, link_id, true); } err = g->ops.fb.init_nvlink(g); if (err) { nvgpu_err(g, "failed to setup nvlinks for sysmem"); return err; } return 0; } int gv100_nvlink_interface_disable(struct gk20a *g) { return 0; } int gv100_nvlink_reg_init(struct gk20a *g) { u32 i = 0; u32 count = 0; struct __nvlink_reginit *reg; enum nvgpu_nvlink_endp endp; int err; u32 link_id; unsigned long mask = g->nvlink.enabled_links; struct nvgpu_nvlink_link *link; /* Apply automated reg init flow for PROD settings */ for_each_set_bit(link_id, &mask, 32) { link = &g->nvlink.links[link_id]; if (!link->remote_info.is_connected) continue; endp = link->remote_info.device_type; err = gv100_nvlink_get_tlc_reginit(endp, ®, &count); if (err) { nvgpu_err(g, "no reginit for endp=%u", endp); continue; } for (i = 0; i < count; i++) { TLC_REG_WR32(g, link_id, reg->addr, reg->value); reg++; } } return 0; } /* * Shutdown device. This should tear down Nvlink connection. * For now return. */ int gv100_nvlink_shutdown(struct gk20a *g) { return 0; } /* * Get link state */ u32 gv100_nvlink_link_get_state(struct gk20a *g, u32 link_id) { return DLPL_REG_RD32(g, link_id, nvl_link_state_r()) & nvl_link_state_state_m(); } /* Get link mode */ u32 gv100_nvlink_link_get_mode(struct gk20a *g, u32 link_id) { u32 state; if (!(BIT(link_id) & g->nvlink.discovered_links)) return nvgpu_nvlink_link__last; state = nvl_link_state_state_v( g->ops.nvlink.link_get_state(g, link_id)); if (state == nvl_link_state_state_init_v()) return nvgpu_nvlink_link_off; if (state == nvl_link_state_state_hwcfg_v()) return nvgpu_nvlink_link_detect; if (state == nvl_link_state_state_swcfg_v()) return nvgpu_nvlink_link_safe; if (state == nvl_link_state_state_active_v()) return nvgpu_nvlink_link_hs; if (state == nvl_link_state_state_fault_v()) return nvgpu_nvlink_link_fault; if (state == nvl_link_state_state_rcvy_ac_v()) return nvgpu_nvlink_link_rcvy_ac; if (state == nvl_link_state_state_rcvy_sw_v()) return nvgpu_nvlink_link_rcvy_sw; if (state == nvl_link_state_state_rcvy_rx_v()) return nvgpu_nvlink_link_rcvy_rx; return nvgpu_nvlink_link_off; } /* Set Link mode */ int gv100_nvlink_link_set_mode(struct gk20a *g, u32 link_id, u32 mode) { u32 state; u32 reg; u32 err = 0; nvgpu_log(g, gpu_dbg_nvlink, "link :%d, mode:%u", link_id, mode); if (!(BIT(link_id) & g->nvlink.enabled_links)) return -EINVAL; state = nvl_link_state_state_v( g->ops.nvlink.link_get_state(g, link_id)); switch (mode) { case nvgpu_nvlink_link_safe: if (state == nvl_link_state_state_swcfg_v()) { nvgpu_warn(g, "link is already in safe mode"); break; } if (state == nvl_link_state_state_hwcfg_v()) { nvgpu_warn(g, "link is transitioning to safe mode"); break; } if (state == nvl_link_state_state_init_v()) { /* Off to Safe transition */ reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r()); reg = set_field(reg, nvl_link_change_newstate_m(), nvl_link_change_newstate_hwcfg_f()); reg = set_field(reg, nvl_link_change_oldstate_mask_m(), nvl_link_change_oldstate_mask_dontcare_f()); reg = set_field(reg, nvl_link_change_action_m(), nvl_link_change_action_ltssm_change_f()); DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg); } else if (state == nvl_link_state_state_active_v()) { /* TODO: * Disable PM first since we are moving out active * state */ reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r()); reg = set_field(reg, nvl_link_change_newstate_m(), nvl_link_change_newstate_swcfg_f()); reg = set_field(reg, nvl_link_change_oldstate_mask_m(), nvl_link_change_oldstate_mask_dontcare_f()); reg = set_field(reg, nvl_link_change_action_m(), nvl_link_change_action_ltssm_change_f()); DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg); } break; case nvgpu_nvlink_link_hs: if (state == nvl_link_state_state_active_v()) { nvgpu_err(g, "link is already in active mode"); break; } if (state == nvl_link_state_state_init_v()) { nvgpu_err(g, "link cannot be taken from init state"); return -EPERM; } reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r()); reg = set_field(reg, nvl_link_change_newstate_m(), nvl_link_change_newstate_active_f()); reg = set_field(reg, nvl_link_change_oldstate_mask_m(), nvl_link_change_oldstate_mask_dontcare_f()); reg = set_field(reg, nvl_link_change_action_m(), nvl_link_change_action_ltssm_change_f()); DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg); break; case nvgpu_nvlink_link_off: if (state == nvl_link_state_state_active_v()) { nvgpu_err(g, "link cannot be taken from active to init"); return -EPERM; } if (state == nvl_link_state_state_init_v()) { nvgpu_err(g, "link already in init state"); } /* GV100 UPHY is handled by MINION */ break; /* 1/8 th mode not supported */ case nvgpu_nvlink_link_enable_pm: case nvgpu_nvlink_link_disable_pm: return -EPERM; case nvgpu_nvlink_link_disable_err_detect: /* Disable Link interrupts */ gv100_nvlink_dlpl_intr_enable(g, link_id, false); break; case nvgpu_nvlink_link_lane_disable: err = gv100_nvlink_minion_lane_disable(g, link_id, true); break; case nvgpu_nvlink_link_lane_shutdown: err = gv100_nvlink_minion_lane_shutdown(g, link_id, true); break; default: nvgpu_err(g, "Unhandled mode %x", mode); break; } return err; } static u32 gv100_nvlink_link_sublink_check_change(struct gk20a *g, u32 link_id) { struct nvgpu_timeout timeout; u32 reg; nvgpu_timeout_init(g, &timeout, NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER); /* Poll for sublink status */ do { reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); if (nvl_sublink_change_status_v(reg) == nvl_sublink_change_status_done_v()) break; if (nvl_sublink_change_status_v(reg) == nvl_sublink_change_status_fault_v()) { nvgpu_err(g, "Fault detected in sublink change"); return -EFAULT; } nvgpu_udelay(5); } while(!nvgpu_timeout_expired_msg(&timeout, "timeout on sublink rdy")); if (nvgpu_timeout_peek_expired(&timeout)) return -ETIMEDOUT; return-0; } int gv100_nvlink_link_set_sublink_mode(struct gk20a *g, u32 link_id, bool is_rx_sublink, u32 mode) { int err = 0; u32 rx_sublink_state = nvgpu_nvlink_sublink_rx__last; u32 tx_sublink_state = nvgpu_nvlink_sublink_tx__last; u32 reg; if (!(BIT(link_id) & g->nvlink.enabled_links)) return -EINVAL; err = gv100_nvlink_link_sublink_check_change(g, link_id); if (err) return err; if (is_rx_sublink) rx_sublink_state = g->ops.nvlink.get_rx_sublink_state(g, link_id); else tx_sublink_state = g->ops.nvlink.get_tx_sublink_state(g, link_id); switch (mode) { case nvgpu_nvlink_sublink_tx_hs: if (tx_sublink_state == nvl_sl0_slsm_status_tx_primary_state_hs_v()) { nvgpu_err(g, " TX already in HS"); break; } else if (tx_sublink_state == nvl_sl0_slsm_status_tx_primary_state_off_v()) { nvgpu_err(g, "TX cannot be do from OFF to HS"); return -EPERM; } reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); reg = set_field(reg, nvl_sublink_change_newstate_m(), nvl_sublink_change_newstate_hs_f()); reg = set_field(reg, nvl_sublink_change_sublink_m(), nvl_sublink_change_sublink_tx_f()); reg = set_field(reg, nvl_sublink_change_action_m(), nvl_sublink_change_action_slsm_change_f()); DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); err = gv100_nvlink_link_sublink_check_change(g, link_id); if (err) { nvgpu_err(g, "Error in TX to HS"); return err; } break; case nvgpu_nvlink_sublink_tx_common: err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true); break; case nvgpu_nvlink_sublink_tx_common_disable: /* NOP */ break; case nvgpu_nvlink_sublink_tx_data_ready: err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true); break; case nvgpu_nvlink_sublink_tx_prbs_en: err = gv100_nvlink_prbs_gen_en(g, BIT(link_id)); break; case nvgpu_nvlink_sublink_tx_safe: if (tx_sublink_state == nvl_sl0_slsm_status_tx_primary_state_safe_v()) { nvgpu_err(g, "TX already SAFE: %d", link_id); break; } reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); reg = set_field(reg, nvl_sublink_change_newstate_m(), nvl_sublink_change_newstate_safe_f()); reg = set_field(reg, nvl_sublink_change_sublink_m(), nvl_sublink_change_sublink_tx_f()); reg = set_field(reg, nvl_sublink_change_action_m(), nvl_sublink_change_action_slsm_change_f()); DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); err = gv100_nvlink_link_sublink_check_change(g, link_id); if (err) { nvgpu_err(g, "Error in TX to SAFE"); return err; } break; case nvgpu_nvlink_sublink_tx_off: if (tx_sublink_state == nvl_sl0_slsm_status_tx_primary_state_off_v()) { nvgpu_err(g, "TX already OFF: %d", link_id); break; } else if (tx_sublink_state == nvl_sl0_slsm_status_tx_primary_state_hs_v()) { nvgpu_err(g, " TX cannot go off from HS %d", link_id); return -EPERM; } reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); reg = set_field(reg, nvl_sublink_change_newstate_m(), nvl_sublink_change_newstate_off_f()); reg = set_field(reg, nvl_sublink_change_sublink_m(), nvl_sublink_change_sublink_tx_f()); reg = set_field(reg, nvl_sublink_change_action_m(), nvl_sublink_change_action_slsm_change_f()); DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); err = gv100_nvlink_link_sublink_check_change(g, link_id); if (err) { nvgpu_err(g, "Error in TX to OFF"); return err; } break; /* RX modes */ case nvgpu_nvlink_sublink_rx_hs: case nvgpu_nvlink_sublink_rx_safe: break; case nvgpu_nvlink_sublink_rx_off: if (rx_sublink_state == nvl_sl1_slsm_status_rx_primary_state_off_v()) { nvgpu_err(g, "RX already OFF: %d", link_id); break; } else if (rx_sublink_state == nvl_sl1_slsm_status_rx_primary_state_hs_v()) { nvgpu_err(g, " RX cannot go off from HS %d", link_id); return -EPERM; } reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); reg = set_field(reg, nvl_sublink_change_newstate_m(), nvl_sublink_change_newstate_off_f()); reg = set_field(reg, nvl_sublink_change_sublink_m(), nvl_sublink_change_sublink_rx_f()); reg = set_field(reg, nvl_sublink_change_action_m(), nvl_sublink_change_action_slsm_change_f()); DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); err = gv100_nvlink_link_sublink_check_change(g, link_id); if (err) { nvgpu_err(g, "Error in RX to OFF"); return err; } break; case nvgpu_nvlink_sublink_rx_rxcal: err = gv100_nvlink_rxcal_en(g, BIT(link_id)); break; default: if ((is_rx_sublink) && ((mode < nvgpu_nvlink_sublink_rx_hs) || (mode >= nvgpu_nvlink_sublink_rx__last))) { nvgpu_err(g, "Unsupported RX mode %u", mode); return -EINVAL; } if (mode >= nvgpu_nvlink_sublink_tx__last) { nvgpu_err(g, "Unsupported TX mode %u", mode); return -EINVAL; } nvgpu_err(g, "MODE %u", mode); } if (err) nvgpu_err(g, " failed on set_sublink_mode"); return err; } u32 gv100_nvlink_link_get_sublink_mode(struct gk20a *g, u32 link_id, bool is_rx_sublink) { u32 state; if (!(BIT(link_id) & g->nvlink.discovered_links)) { if (!is_rx_sublink) return nvgpu_nvlink_sublink_tx__last; return nvgpu_nvlink_sublink_rx__last; } if (!is_rx_sublink) { state = g->ops.nvlink.get_tx_sublink_state(g, link_id); if (state == nvl_sl0_slsm_status_tx_primary_state_hs_v()) return nvgpu_nvlink_sublink_tx_hs; if (state == nvl_sl0_slsm_status_tx_primary_state_eighth_v()) return nvgpu_nvlink_sublink_tx_single_lane; if (state == nvl_sl0_slsm_status_tx_primary_state_safe_v()) return nvgpu_nvlink_sublink_tx_safe; if (state == nvl_sl0_slsm_status_tx_primary_state_off_v()) return nvgpu_nvlink_sublink_tx_off; return nvgpu_nvlink_sublink_tx__last; } else { state = g->ops.nvlink.get_rx_sublink_state(g, link_id); if (state == nvl_sl1_slsm_status_rx_primary_state_hs_v()) return nvgpu_nvlink_sublink_rx_hs; if (state == nvl_sl1_slsm_status_rx_primary_state_eighth_v()) return nvgpu_nvlink_sublink_rx_single_lane; if (state == nvl_sl1_slsm_status_rx_primary_state_safe_v()) return nvgpu_nvlink_sublink_rx_safe; if (state == nvl_sl1_slsm_status_rx_primary_state_off_v()) return nvgpu_nvlink_sublink_rx_off; return nvgpu_nvlink_sublink_rx__last; } return nvgpu_nvlink_sublink_tx__last; } /* * Get TX sublink state */ u32 gv100_nvlink_link_get_tx_sublink_state(struct gk20a *g, u32 link_id) { u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl0_slsm_status_tx_r()); return nvl_sl0_slsm_status_tx_primary_state_v(reg); } /* * Get RX sublink state */ u32 gv100_nvlink_link_get_rx_sublink_state(struct gk20a *g, u32 link_id) { u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl1_slsm_status_rx_r()); return nvl_sl1_slsm_status_rx_primary_state_v(reg); } /* Hardcode the link_mask while we wait for VBIOS link_disable_mask field * to be updated. */ void gv100_nvlink_get_connected_link_mask(u32 *link_mask) { *link_mask = GV100_CONNECTED_LINK_MASK; } /* * Performs nvlink device level initialization by discovering the topology * taking device out of reset, boot minion, set clocks up and common interrupts */ int gv100_nvlink_early_init(struct gk20a *g) { int err = 0; u32 mc_reset_nvlink_mask; if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) return -EINVAL; err = nvgpu_bios_get_nvlink_config_data(g); if (err) { nvgpu_err(g, "failed to read nvlink vbios data"); goto nvlink_init_exit; } err = g->ops.nvlink.discover_ioctrl(g); if (err) goto nvlink_init_exit; /* Enable NVLINK in MC */ mc_reset_nvlink_mask = BIT32(g->nvlink.ioctrl_table[0].reset_enum); nvgpu_log(g, gpu_dbg_nvlink, "mc_reset_nvlink_mask: 0x%x", mc_reset_nvlink_mask); g->ops.mc.reset(g, mc_reset_nvlink_mask); err = g->ops.nvlink.discover_link(g); if (err || g->nvlink.discovered_links == 0) { nvgpu_err(g, "No links available"); goto nvlink_init_exit; } err = nvgpu_flcn_sw_init(g, FALCON_ID_MINION); if (err != 0) { nvgpu_err(g, "failed to sw init FALCON_ID_MINION"); goto nvlink_init_exit; } g->nvlink.discovered_links &= ~g->nvlink.link_disable_mask; nvgpu_log(g, gpu_dbg_nvlink, "link_disable_mask = 0x%08x (from VBIOS)", g->nvlink.link_disable_mask); /* Links in reset should be removed from initialized link sw state */ g->nvlink.initialized_links &= __gv100_nvlink_get_link_reset_mask(g); /* VBIOS link_disable_mask should be sufficient to find the connected * links. As VBIOS is not updated with correct mask, we parse the DT * node where we hardcode the link_id. DT method is not scalable as same * DT node is used for different dGPUs connected over PCIE. * Remove the DT parsing of link id and use HAL to get link_mask based * on the GPU. This is temporary WAR while we get the VBIOS updated with * correct mask. */ g->ops.nvlink.get_connected_link_mask(&(g->nvlink.connected_links)); nvgpu_log(g, gpu_dbg_nvlink, "connected_links = 0x%08x", g->nvlink.connected_links); /* Track only connected links */ g->nvlink.discovered_links &= g->nvlink.connected_links; nvgpu_log(g, gpu_dbg_nvlink, "discovered_links = 0x%08x (combination)", g->nvlink.discovered_links); if (hweight32(g->nvlink.discovered_links) > 1) { nvgpu_err(g, "more than one link enabled"); err = -EINVAL; goto nvlink_init_exit; } g->nvlink.speed = nvgpu_nvlink_speed_20G; err = __gv100_nvlink_state_load_hal(g); if (err) { nvgpu_err(g, " failed Nvlink state load"); goto nvlink_init_exit; } err = gv100_nvlink_minion_configure_ac_coupling(g, g->nvlink.ac_coupling_mask, true); if (err) { nvgpu_err(g, " failed Nvlink state load"); goto nvlink_init_exit; } /* Program clocks */ gv100_nvlink_prog_alt_clk(g); nvlink_init_exit: return err; } #endif /* CONFIG_TEGRA_NVLINK */