/* Copyright 2024 Joshua Bakita * Helpers for dealing with the PCIe and platform bus * * = Design Notes = * We have to use PRAMIN to access the BAR2 page table. While it's typically also * mapped into BAR2, we have no way to know where without the table. If the table * changes, the new sections will have new mappings in BAR2, repeating the * problem, and making caching insufficient. * * = Terms = * VRAM/VID_MEM: Video RAM; Addresses to physical frames in the on-GPU RAM. * SYS_MEM: System Memory; "Bus addresses"; Addresses which can be presented to * the PCIe Host for resolution. On x86_64 without an IOMMU, these are * just physical addresses, but may be I/O Virtual Addresses (IOVAs) * or translated via an I/O MMU on other platforms; DMA Addresses. * PEER: Addresses to RAM on another GPU. */ #include // For printk() #include // For error defines #include // For readl() #include "nvdebug.h" /* Obtain the PRAMIN offset at which `addr` can be accessed @param addr Address to find @param target Which address space to use (VRAM, SYS_MEM, PEER(?)) @return positive offset, -EINVAL on invalid arguments, or -EOPNOTSUPP on an unsupported platform. Note: Will move the PRAMIN window to accomodate the request. Only guarantees that the surrounding 64-KiB-aligned window will be accessible. Note: Moving the PRAMIN window will cause problems if it races with driver code that tries to do the same, or expects the window not to move. Bugs: Untested on PEER. */ int addr_to_pramin_mut(struct nvdebug_state *g, uint64_t addr, enum INST_TARGET target) { bar0_window_t window; uint64_t pramin_base; uint32_t window_reg; // For us, accuracy and robustness is more important than speed // Check that the address is valid (49 bits are addressable on-GPU, but // PRAMIN only supports up to 40 bits). if (addr & ~0x000000ffffffffff) { printk(KERN_ERR "[nvdebug] Invalid address %llx passed to %s!\n", addr, __func__); return -EINVAL; } // Register relocated on Hopper and Blackwell+ if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL) window_reg = NV_XAL_EP_BAR0_WINDOW_BASE; else window_reg = NV_PBUS_BAR0_WINDOW; if ((window.raw = nvdebug_readl(g, window_reg)) == -1) { printk(KERN_ERR "[nvdebug] PRAMIN window configuration inaccessible; " "failing %s\n", __func__); return -EOPNOTSUPP; } if (window.target != target) { // On Hopper and Blackwell+, the window always points at VID_MEM if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL) return -EOPNOTSUPP; else goto relocate; } pramin_base = ((uint64_t)window.base) << 16; if (addr < pramin_base || addr >= pramin_base + NV_PRAMIN_LEN) goto relocate; return addr - pramin_base; // Guaranteed to be < 1MiB, so safe for int relocate: printk(KERN_INFO "[nvdebug] [SIDE EFFECT] Moving PRAMIN window from base " "%llx (%s) to %llx (%s) to accomodate %#018llx\n", ((uint64_t)window.base) << 16, target_to_text(window.target), (addr >> 16) << 16, target_to_text(target), addr); // Move PRAMIN window to a 64KiB-aligned address window.base = (u32)(addr >> 16); // Safe, due to above range check window.target = target; nvdebug_writel(g, window_reg, window.raw); // Wait for the window to move by re-reading (as done in nvgpu driver) (void) nvdebug_readl(g, window_reg); return (int)(addr & 0xffffull); } /* Get a copy of the BAR2 page directory configuration (base and aperture) @param pd Pointer at which to store the configuration, including a pointer and aperture for the zeroth entry of the top-level page directory (PD3 for V2 page tables). This pointer **may not** be directly dereferencable, and the caller may need to shift the BAR2 window. @return 0 on success, -errno on error. Note: This may move the PRAMIN window. */ int get_bar2_pdb(struct nvdebug_state *g, page_dir_config_t* pd) { int ret; uint64_t bar2_ptr; enum INST_TARGET bar2_target; bool bar2_is_virtual; if (!pd) return -EINVAL; if (!g->bar2) return -ENXIO; // BAR2 has its own instance block (typically in VRAM) which contains the // Page Directory Base (PDB), a pointer to a page directory/table // hierarchy used to translate BAR2 offsets to VRAM or SYS_MEM addresses. // Determine location of BAR2 instance block if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL) { // Register layout updated on Hopper and Blackwell+ to support 52-bit // instance block pointers (vs. 40 bits before) bar_config_block_gh100_t bar2_block; if ((bar2_block.raw = nvdebug_readq(g, NV_VIRTUAL_FUNCTION_PRIV_FUNC_BAR2_BLOCK)) == -1) { printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n"); return -EOPNOTSUPP; } bar2_ptr = (uint64_t)bar2_block.ptr << 12; bar2_target = bar2_block.target; bar2_is_virtual = bar2_block.is_virtual; } else { bar_config_block_t bar2_block; if ((bar2_block.raw = nvdebug_readl(g, NV_PBUS_BAR2_BLOCK)) == -1) { printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n"); return -EOPNOTSUPP; } bar2_ptr = (uint64_t)bar2_block.ptr << 12; bar2_target = bar2_block.target; bar2_is_virtual = bar2_block.is_virtual; } printk(KERN_INFO "[nvdebug] BAR2 inst block @ %llx in %s's %s address space.\n", bar2_ptr, target_to_text(bar2_target), bar2_is_virtual ? "virtual" : "physical"); // Setup PRAMIN to point at the BAR2 instance block // TODO: This won't work if the instance block is in SYS_MEM on Hopper or // Blackwell+. Going through the I/O MMU appears to be fairly // reliable, so I need to switch to using that logic whenever // SYS_MEM may be accessed. if ((ret = addr_to_pramin_mut(g, bar2_ptr, bar2_target)) < 0) { printk(KERN_ERR "[nvdebug] Unable to access BAR2/3 Instance Block configuration via PRAMIN! BAR2/3 inaccessible.\n"); return ret; } // Pull the page directory base configuration from the instance block if ((pd->raw = nvdebug_readq(g, NV_PRAMIN + ret + NV_PRAMIN_PDB_CONFIG_OFF)) == -1) { printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 PDB configuration! BAR2/3 inaccessible.\n"); return -EOPNOTSUPP; } return 0; }