From 16163d89b564029cabf4842815590b0a484cc172 Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Tue, 10 Sep 2024 13:39:59 -0400 Subject: Add Hopper and Blackwell support to bus.c - Handle relocation of PRAMIN window configuration register - Handle new format for BAR2 configuration - Catch unreadable PRAMIN configuration Tested on A100, H100, and AD102 (RTX 6000 Ada). --- bus.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++++-------------- nvdebug.h | 67 +++++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 111 insertions(+), 29 deletions(-) diff --git a/bus.c b/bus.c index c4d991e..f2369eb 100644 --- a/bus.c +++ b/bus.c @@ -24,10 +24,11 @@ /* Obtain the PRAMIN offset at which `addr` can be accessed @param addr Address to find @param target Which address space to use (VRAM, SYS_MEM, PEER(?)) - @return positive offset or -EINVAL on invalid arguments + @return positive offset, -EINVAL on invalid arguments, or -EOPNOTSUPP on + an unsupported platform. Note: Will move the PRAMIN window to accomodate the request. Only guarantees - that the surrounding 64KiB window will be accessible. + that the surrounding 64-KiB-aligned window will be accessible. Note: Moving the PRAMIN window will cause problems if it races with driver code that tries to do the same, or expects the window not to move. Bugs: Untested on PEER. @@ -36,6 +37,7 @@ int addr_to_pramin_mut(struct nvdebug_state *g, uint64_t addr, enum INST_TARGET target) { bar0_window_t window; uint64_t pramin_base; + uint32_t window_reg; // For us, accuracy and robustness is more important than speed // Check that the address is valid (49 bits are addressable on-GPU, but // PRAMIN only supports up to 40 bits). @@ -44,21 +46,38 @@ int addr_to_pramin_mut(struct nvdebug_state *g, addr, __func__); return -EINVAL; } - window.raw = nvdebug_readl(g, NV_PBUS_BAR0_WINDOW); - if (window.target != target) - goto relocate; + // Register relocated on Hopper and Blackwell+ + if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL) + window_reg = NV_XAL_EP_BAR0_WINDOW_BASE; + else + window_reg = NV_PBUS_BAR0_WINDOW; + if ((window.raw = nvdebug_readl(g, window_reg)) == -1) { + printk(KERN_ERR "[nvdebug] PRAMIN window configuration inaccessible; " + "failing %s\n", __func__); + return -EOPNOTSUPP; + } + if (window.target != target) { + // On Hopper and Blackwell+, the window always points at VID_MEM + if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL) + return -EOPNOTSUPP; + else + goto relocate; + } pramin_base = ((uint64_t)window.base) << 16; if (addr < pramin_base || addr > pramin_base + NV_PRAMIN_LEN) goto relocate; return addr - pramin_base; // Guaranteed to be < 1MiB, so safe for int relocate: - printk(KERN_INFO "[nvdebug] Moving PRAMIN win from base %llx (%s) to %llx (%s) to accomodate %#018llx\n", pramin_base, target_to_text(window.target), (addr >> 16) << 16, target_to_text(target), addr); + printk(KERN_INFO "[nvdebug] [SIDE EFFECT] Moving PRAMIN window from base " + "%llx (%s) to %llx (%s) to accomodate %#018llx\n", + ((uint64_t)window.base) << 16, target_to_text(window.target), + (addr >> 16) << 16, target_to_text(target), addr); // Move PRAMIN window to a 64KiB-aligned address window.base = (u32)(addr >> 16); // Safe, due to above range check window.target = target; - nvdebug_writel(g, NV_PBUS_BAR0_WINDOW, window.raw); + nvdebug_writel(g, window_reg, window.raw); // Wait for the window to move by re-reading (as done in nvgpu driver) - (void) nvdebug_readl(g, NV_PBUS_BAR0_WINDOW); + (void) nvdebug_readl(g, window_reg); return (int)(addr & 0xffffull); } @@ -72,7 +91,9 @@ relocate: */ int get_bar2_pdb(struct nvdebug_state *g, page_dir_config_t* pd) { int ret; - bar_config_block_t bar2_block; + uint64_t bar2_ptr; + enum INST_TARGET bar2_target; + bool bar2_is_virtual; if (!pd) return -EINVAL; @@ -85,17 +106,37 @@ int get_bar2_pdb(struct nvdebug_state *g, page_dir_config_t* pd) { // hierarchy used to translate BAR2 offsets to VRAM or SYS_MEM addresses. // Determine location of BAR2 instance block - if ((bar2_block.raw = nvdebug_readl(g, NV_PBUS_BAR2_BLOCK)) == -1) { - printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n"); - return -EOPNOTSUPP; + if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL) { + // Register layout updated on Hopper and Blackwell+ to support 52-bit + // instance block pointers (vs. 40 bits before) + bar_config_block_gh100_t bar2_block; + if ((bar2_block.raw = nvdebug_readq(g, NV_VIRTUAL_FUNCTION_PRIV_FUNC_BAR2_BLOCK)) == -1) { + printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n"); + return -EOPNOTSUPP; + } + bar2_ptr = (uint64_t)bar2_block.ptr << 12; + bar2_target = bar2_block.target; + bar2_is_virtual = bar2_block.is_virtual; + } else { + bar_config_block_t bar2_block; + if ((bar2_block.raw = nvdebug_readl(g, NV_PBUS_BAR2_BLOCK)) == -1) { + printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n"); + return -EOPNOTSUPP; + } + bar2_ptr = (uint64_t)bar2_block.ptr << 12; + bar2_target = bar2_block.target; + bar2_is_virtual = bar2_block.is_virtual; } - printk(KERN_INFO "[nvdebug] BAR2 inst block @ %llx in %s's %s address space.\n", ((u64)bar2_block.ptr) << 12, target_to_text(bar2_block.target), bar2_block.is_virtual ? "virtual" : "physical"); + printk(KERN_INFO "[nvdebug] BAR2 inst block @ %llx in %s's %s address space.\n", bar2_ptr, target_to_text(bar2_target), bar2_is_virtual ? "virtual" : "physical"); // Setup PRAMIN to point at the BAR2 instance block - if ((ret = addr_to_pramin_mut(g, (uint64_t)bar2_block.ptr << 12, bar2_block.target)) < 0) { - printk(KERN_ERR "[nvdebug] Invalid BAR2/3 Instance Block configuration! BAR2/3 inaccessible.\n"); + // TODO: This won't work if the instance block is in SYS_MEM on Hopper or + // Blackwell+. Going through the I/O MMU appears to be fairly + // reliable, so I need to switch to using that logic whenever + // SYS_MEM may be accessed. + if ((ret = addr_to_pramin_mut(g, bar2_ptr, bar2_target)) < 0) { + printk(KERN_ERR "[nvdebug] Unable to access BAR2/3 Instance Block configuration via PRAMIN! BAR2/3 inaccessible.\n"); return ret; } - printk(KERN_INFO "[nvdebug] BAR2 inst block at off %x in PRAMIN\n", ret); // Pull the page directory base configuration from the instance block if ((pd->raw = nvdebug_readq(g, NV_PRAMIN + ret + NV_PRAMIN_PDB_CONFIG_OFF)) == -1) { printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 PDB configuration! BAR2/3 inaccessible.\n"); diff --git a/nvdebug.h b/nvdebug.h index f644500..409b013 100644 --- a/nvdebug.h +++ b/nvdebug.h @@ -484,6 +484,7 @@ typedef union { #define NV_CHIP_ID_AMPERE 0x170 #define NV_CHIP_ID_HOPPER 0x180 #define NV_CHIP_ID_ADA 0x190 +#define NV_CHIP_ID_BLACKWELL 0x1A0 inline static const char* ARCH2NAME(uint32_t arch) { switch (arch) { @@ -521,14 +522,18 @@ inline static const char* ARCH2NAME(uint32_t arch) { return "Turing"; case 0x17: return "Ampere"; - case 0x18: + case 0x18: // Despite the Chip ID, Hopper functionally proceeds Ada return "Hopper"; case 0x19: return "Ada Lovelace"; - case 0x20: - return "Blackwell (?)"; + case 0x1A: + return "Blackwell"; + case 0x1B: + return "Rubin (?)"; + case 0x1F: // NVIDIA-internal simulator + return "AMODEL"; default: - if (arch < 0x19) + if (arch < 0x1A) return "[unknown historical architecture]"; else return "[future]"; @@ -881,25 +886,57 @@ union reg_range { #define NV_CE_PCE_MAP_SIZE 32 -/* Location of the 1Kb instance block with page tables for BAR1 and BAR2. - Support: Fermi+ (?), Pascal +/* Location of the 1Kb instance block with page tables for the BAR1/2 regions. + + On the H100, the "BAR1 block" describes what is actually BAR2, and the + "BAR2 block" describes BAR4. + + PTR : Upper 28 bits of the 40-bit, (4k-aligned) address where the instance + block configuration is for the listed BAR region. + + "Hopper+ uses 64-bit BARs, so GPU BAR2 should be at BAR4/5 and GPU BAR1 is at + BAR2/3" (open-gpu-kernel-modules) */ +// Support: Fermi through Ampere, Ada #define NV_PBUS_BAR1_BLOCK 0x00001704 #define NV_PBUS_BAR2_BLOCK 0x00001714 typedef union { struct { uint32_t ptr:28; enum INST_TARGET target:2; - uint32_t padding0:1; + uint32_t :1; bool is_virtual:1; } __attribute__((packed)); uint32_t raw; struct { uint32_t map:30; - uint32_t padding1:2; + uint32_t :2; } __attribute__((packed)); } bar_config_block_t; +// Support: Hopper, Blackwell+ +// This is a "VREG" (virtual register?) in the documentation, meaning that it +// needs the VREG base added first. +#define NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET 0x00B80000 +#define NV_VIRTUAL_FUNCTION_PRIV_FUNC_BAR2_BLOCK (NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET+0x00000F70) +typedef union { + struct { + bool is_pending:1; + bool is_outstanding:1; + uint32_t :7; + bool is_virtual:1; + enum INST_TARGET target:2; + uint64_t ptr:40; + uint32_t :12; + } __attribute__((packed)); + uint64_t raw; + struct { + uint32_t :10; + uint32_t map:22; + uint32_t :32; + } __attribute__((packed)); +} bar_config_block_gh100_t; + /* BAR0 PRAMIN (Private RAM Instance) window configuration One of the oldest ways to access video memory on NVIDIA GPUs is by using a configurable 1MB window into VRAM which is mapped into BAR0 (register) @@ -914,21 +951,25 @@ typedef union { TARGET : Which address space BASE points into Note: This seems to be set to 0x0bff00000 - 0x0c0000000 at least sometimes - - Support: Tesla 2.0, Fermi, Kepler, Maxwell, Pascal, Turing, Ampere */ +// Support: Tesla 2.0, Fermi, Kepler, Maxwell, Pascal, Turing, Ampere, Ada #define NV_PBUS_BAR0_WINDOW 0x00001700 -#define NV_PRAMIN 0x00700000 // Goes until 0x00800000 (1MB window) -#define NV_PRAMIN_LEN 0x00100000 +// On Hopper, and Blackwell+, TARGET must always be 0 (VIDMEM) +// Support: Hopper, Blackwell+ +#define NV_XAL_EP_BAR0_WINDOW_BASE 0x0010fd40 typedef union { struct { uint32_t base:24; enum INST_TARGET target:2; - uint32_t padding0:6; + uint32_t :6; } __attribute__((packed)); uint32_t raw; } bar0_window_t; +// Support: Tesla 2.0 through (at least) Blackwell +#define NV_PRAMIN 0x00700000 // Goes until 0x00800000 (1MB window) +#define NV_PRAMIN_LEN 0x00100000 + // Support: Tesla 2.0, Fermi, Kepler, Maxwell, Pascal, Turing, Ampere #define NV_PRAMIN_PDB_CONFIG_OFF 0x200 typedef union { -- cgit v1.2.2