1 files changed, 57 insertions, 16 deletions

diff --git a/bus.c b/bus.c
index c4d991e..f2369eb 100644
--- a/bus.c
+++ b/bus.c
@@ -24,10 +24,11 @@
 /* Obtain the PRAMIN offset at which `addr` can be accessed
   @param addr   Address to find
   @param target Which address space to use (VRAM, SYS_MEM, PEER(?))
-  @return positive offset or -EINVAL on invalid arguments
+  @return positive offset, -EINVAL on invalid arguments, or -EOPNOTSUPP on
+          an unsupported platform.
 
   Note: Will move the PRAMIN window to accomodate the request. Only guarantees
-        that the surrounding 64KiB window will be accessible.
+        that the surrounding 64-KiB-aligned window will be accessible.
   Note: Moving the PRAMIN window will cause problems if it races with driver
         code that tries to do the same, or expects the window not to move.
   Bugs: Untested on PEER.
@@ -36,6 +37,7 @@ int addr_to_pramin_mut(struct nvdebug_state *g,
                        uint64_t addr, enum INST_TARGET target) {
         bar0_window_t window;
         uint64_t pramin_base;
+        uint32_t window_reg;
         // For us, accuracy and robustness is more important than speed
         // Check that the address is valid (49 bits are addressable on-GPU, but
         // PRAMIN only supports up to 40 bits).
@@ -44,21 +46,38 @@ int addr_to_pramin_mut(struct nvdebug_state *g,
                        addr, __func__);
                 return -EINVAL;
         }
-        window.raw = nvdebug_readl(g, NV_PBUS_BAR0_WINDOW);
-        if (window.target != target)
-                goto relocate;
+        // Register relocated on Hopper and Blackwell+
+        if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL)
+                window_reg = NV_XAL_EP_BAR0_WINDOW_BASE;
+        else
+                window_reg = NV_PBUS_BAR0_WINDOW;
+        if ((window.raw = nvdebug_readl(g, window_reg)) == -1) {
+                printk(KERN_ERR "[nvdebug] PRAMIN window configuration inaccessible; "
+                       "failing %s\n", __func__);
+                return -EOPNOTSUPP;
+        }
+        if (window.target != target) {
+                // On Hopper and Blackwell+, the window always points at VID_MEM
+                if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL)
+                        return -EOPNOTSUPP;
+                else
+                        goto relocate;
+        }
         pramin_base = ((uint64_t)window.base) << 16;
         if (addr < pramin_base || addr > pramin_base + NV_PRAMIN_LEN)
                 goto relocate;
         return addr - pramin_base; // Guaranteed to be < 1MiB, so safe for int
 relocate:
-        printk(KERN_INFO "[nvdebug] Moving PRAMIN win from base %llx (%s) to %llx (%s) to accomodate %#018llx\n", pramin_base, target_to_text(window.target), (addr >> 16) << 16, target_to_text(target), addr);
+        printk(KERN_INFO "[nvdebug] [SIDE EFFECT] Moving PRAMIN window from base "
+               "%llx (%s) to %llx (%s) to accomodate %#018llx\n",
+               ((uint64_t)window.base) << 16, target_to_text(window.target),
+               (addr >> 16) << 16, target_to_text(target), addr);
         // Move PRAMIN window to a 64KiB-aligned address
         window.base = (u32)(addr >> 16); // Safe, due to above range check
         window.target = target;
-        nvdebug_writel(g, NV_PBUS_BAR0_WINDOW, window.raw);
+        nvdebug_writel(g, window_reg, window.raw);
         // Wait for the window to move by re-reading (as done in nvgpu driver)
-        (void) nvdebug_readl(g, NV_PBUS_BAR0_WINDOW);
+        (void) nvdebug_readl(g, window_reg);
         return (int)(addr & 0xffffull);
 }
 
@@ -72,7 +91,9 @@ relocate:
 */
 int get_bar2_pdb(struct nvdebug_state *g, page_dir_config_t* pd) {
         int ret;
-        bar_config_block_t bar2_block;
+        uint64_t bar2_ptr;
+        enum INST_TARGET bar2_target;
+        bool bar2_is_virtual;
 
         if (!pd)
                 return -EINVAL;
@@ -85,17 +106,37 @@ int get_bar2_pdb(struct nvdebug_state *g, page_dir_config_t* pd) {
         // hierarchy used to translate BAR2 offsets to VRAM or SYS_MEM addresses.
 
         // Determine location of BAR2 instance block
-        if ((bar2_block.raw = nvdebug_readl(g, NV_PBUS_BAR2_BLOCK)) == -1) {
-                printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n");
-                return -EOPNOTSUPP;
+        if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL) {
+                // Register layout updated on Hopper and Blackwell+ to support 52-bit
+                // instance block pointers (vs. 40 bits before)
+                bar_config_block_gh100_t bar2_block;
+                if ((bar2_block.raw = nvdebug_readq(g, NV_VIRTUAL_FUNCTION_PRIV_FUNC_BAR2_BLOCK)) == -1) {
+                        printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n");
+                        return -EOPNOTSUPP;
+                }
+                bar2_ptr = (uint64_t)bar2_block.ptr << 12;
+                bar2_target = bar2_block.target;
+                bar2_is_virtual = bar2_block.is_virtual;
+        } else {
+                bar_config_block_t bar2_block;
+                if ((bar2_block.raw = nvdebug_readl(g, NV_PBUS_BAR2_BLOCK)) == -1) {
+                        printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n");
+                        return -EOPNOTSUPP;
+                }
+                bar2_ptr = (uint64_t)bar2_block.ptr << 12;
+                bar2_target = bar2_block.target;
+                bar2_is_virtual = bar2_block.is_virtual;
         }
-        printk(KERN_INFO "[nvdebug] BAR2 inst block @ %llx in %s's %s address space.\n", ((u64)bar2_block.ptr) << 12, target_to_text(bar2_block.target), bar2_block.is_virtual ? "virtual" : "physical");
+        printk(KERN_INFO "[nvdebug] BAR2 inst block @ %llx in %s's %s address space.\n", bar2_ptr, target_to_text(bar2_target), bar2_is_virtual ? "virtual" : "physical");
         // Setup PRAMIN to point at the BAR2 instance block
-        if ((ret = addr_to_pramin_mut(g, (uint64_t)bar2_block.ptr << 12, bar2_block.target)) < 0) {
-                printk(KERN_ERR "[nvdebug] Invalid BAR2/3 Instance Block configuration! BAR2/3 inaccessible.\n");
+        // TODO: This won't work if the instance block is in SYS_MEM on Hopper or
+        //       Blackwell+. Going through the I/O MMU appears to be fairly
+        //       reliable, so I need to switch to using that logic whenever
+        //       SYS_MEM may be accessed.
+        if ((ret = addr_to_pramin_mut(g, bar2_ptr, bar2_target)) < 0) {
+                printk(KERN_ERR "[nvdebug] Unable to access BAR2/3 Instance Block configuration via PRAMIN! BAR2/3 inaccessible.\n");
                 return ret;
         }
-        printk(KERN_INFO "[nvdebug] BAR2 inst block at off %x in PRAMIN\n", ret);
         // Pull the page directory base configuration from the instance block
         if ((pd->raw = nvdebug_readq(g, NV_PRAMIN + ret + NV_PRAMIN_PDB_CONFIG_OFF)) == -1) {
                 printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 PDB configuration! BAR2/3 inaccessible.\n");