2 files changed, 111 insertions, 29 deletions

diff --git a/bus.c b/bus.c
index c4d991e..f2369eb 100644
--- a/bus.c
+++ b/bus.c
@@ -24,10 +24,11 @@
 /* Obtain the PRAMIN offset at which `addr` can be accessed
   @param addr   Address to find
   @param target Which address space to use (VRAM, SYS_MEM, PEER(?))
-  @return positive offset or -EINVAL on invalid arguments
+  @return positive offset, -EINVAL on invalid arguments, or -EOPNOTSUPP on
+          an unsupported platform.
 
   Note: Will move the PRAMIN window to accomodate the request. Only guarantees
-        that the surrounding 64KiB window will be accessible.
+        that the surrounding 64-KiB-aligned window will be accessible.
   Note: Moving the PRAMIN window will cause problems if it races with driver
         code that tries to do the same, or expects the window not to move.
   Bugs: Untested on PEER.
@@ -36,6 +37,7 @@ int addr_to_pramin_mut(struct nvdebug_state *g,
                        uint64_t addr, enum INST_TARGET target) {
         bar0_window_t window;
         uint64_t pramin_base;
+        uint32_t window_reg;
         // For us, accuracy and robustness is more important than speed
         // Check that the address is valid (49 bits are addressable on-GPU, but
         // PRAMIN only supports up to 40 bits).
@@ -44,21 +46,38 @@ int addr_to_pramin_mut(struct nvdebug_state *g,
                        addr, __func__);
                 return -EINVAL;
         }
-        window.raw = nvdebug_readl(g, NV_PBUS_BAR0_WINDOW);
-        if (window.target != target)
-                goto relocate;
+        // Register relocated on Hopper and Blackwell+
+        if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL)
+                window_reg = NV_XAL_EP_BAR0_WINDOW_BASE;
+        else
+                window_reg = NV_PBUS_BAR0_WINDOW;
+        if ((window.raw = nvdebug_readl(g, window_reg)) == -1) {
+                printk(KERN_ERR "[nvdebug] PRAMIN window configuration inaccessible; "
+                       "failing %s\n", __func__);
+                return -EOPNOTSUPP;
+        }
+        if (window.target != target) {
+                // On Hopper and Blackwell+, the window always points at VID_MEM
+                if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL)
+                        return -EOPNOTSUPP;
+                else
+                        goto relocate;
+        }
         pramin_base = ((uint64_t)window.base) << 16;
         if (addr < pramin_base || addr > pramin_base + NV_PRAMIN_LEN)
                 goto relocate;
         return addr - pramin_base; // Guaranteed to be < 1MiB, so safe for int
 relocate:
-        printk(KERN_INFO "[nvdebug] Moving PRAMIN win from base %llx (%s) to %llx (%s) to accomodate %#018llx\n", pramin_base, target_to_text(window.target), (addr >> 16) << 16, target_to_text(target), addr);
+        printk(KERN_INFO "[nvdebug] [SIDE EFFECT] Moving PRAMIN window from base "
+               "%llx (%s) to %llx (%s) to accomodate %#018llx\n",
+               ((uint64_t)window.base) << 16, target_to_text(window.target),
+               (addr >> 16) << 16, target_to_text(target), addr);
         // Move PRAMIN window to a 64KiB-aligned address
         window.base = (u32)(addr >> 16); // Safe, due to above range check
         window.target = target;
-        nvdebug_writel(g, NV_PBUS_BAR0_WINDOW, window.raw);
+        nvdebug_writel(g, window_reg, window.raw);
         // Wait for the window to move by re-reading (as done in nvgpu driver)
-        (void) nvdebug_readl(g, NV_PBUS_BAR0_WINDOW);
+        (void) nvdebug_readl(g, window_reg);
         return (int)(addr & 0xffffull);
 }
 
@@ -72,7 +91,9 @@ relocate:
 */
 int get_bar2_pdb(struct nvdebug_state *g, page_dir_config_t* pd) {
         int ret;
-        bar_config_block_t bar2_block;
+        uint64_t bar2_ptr;
+        enum INST_TARGET bar2_target;
+        bool bar2_is_virtual;
 
         if (!pd)
                 return -EINVAL;
@@ -85,17 +106,37 @@ int get_bar2_pdb(struct nvdebug_state *g, page_dir_config_t* pd) {
         // hierarchy used to translate BAR2 offsets to VRAM or SYS_MEM addresses.
 
         // Determine location of BAR2 instance block
-        if ((bar2_block.raw = nvdebug_readl(g, NV_PBUS_BAR2_BLOCK)) == -1) {
-                printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n");
-                return -EOPNOTSUPP;
+        if ((g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) || g->chip_id >= NV_CHIP_ID_BLACKWELL) {
+                // Register layout updated on Hopper and Blackwell+ to support 52-bit
+                // instance block pointers (vs. 40 bits before)
+                bar_config_block_gh100_t bar2_block;
+                if ((bar2_block.raw = nvdebug_readq(g, NV_VIRTUAL_FUNCTION_PRIV_FUNC_BAR2_BLOCK)) == -1) {
+                        printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n");
+                        return -EOPNOTSUPP;
+                }
+                bar2_ptr = (uint64_t)bar2_block.ptr << 12;
+                bar2_target = bar2_block.target;
+                bar2_is_virtual = bar2_block.is_virtual;
+        } else {
+                bar_config_block_t bar2_block;
+                if ((bar2_block.raw = nvdebug_readl(g, NV_PBUS_BAR2_BLOCK)) == -1) {
+                        printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n");
+                        return -EOPNOTSUPP;
+                }
+                bar2_ptr = (uint64_t)bar2_block.ptr << 12;
+                bar2_target = bar2_block.target;
+                bar2_is_virtual = bar2_block.is_virtual;
         }
-        printk(KERN_INFO "[nvdebug] BAR2 inst block @ %llx in %s's %s address space.\n", ((u64)bar2_block.ptr) << 12, target_to_text(bar2_block.target), bar2_block.is_virtual ? "virtual" : "physical");
+        printk(KERN_INFO "[nvdebug] BAR2 inst block @ %llx in %s's %s address space.\n", bar2_ptr, target_to_text(bar2_target), bar2_is_virtual ? "virtual" : "physical");
         // Setup PRAMIN to point at the BAR2 instance block
-        if ((ret = addr_to_pramin_mut(g, (uint64_t)bar2_block.ptr << 12, bar2_block.target)) < 0) {
-                printk(KERN_ERR "[nvdebug] Invalid BAR2/3 Instance Block configuration! BAR2/3 inaccessible.\n");
+        // TODO: This won't work if the instance block is in SYS_MEM on Hopper or
+        //       Blackwell+. Going through the I/O MMU appears to be fairly
+        //       reliable, so I need to switch to using that logic whenever
+        //       SYS_MEM may be accessed.
+        if ((ret = addr_to_pramin_mut(g, bar2_ptr, bar2_target)) < 0) {
+                printk(KERN_ERR "[nvdebug] Unable to access BAR2/3 Instance Block configuration via PRAMIN! BAR2/3 inaccessible.\n");
                 return ret;
         }
-        printk(KERN_INFO "[nvdebug] BAR2 inst block at off %x in PRAMIN\n", ret);
         // Pull the page directory base configuration from the instance block
         if ((pd->raw = nvdebug_readq(g, NV_PRAMIN + ret + NV_PRAMIN_PDB_CONFIG_OFF)) == -1) {
                 printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 PDB configuration! BAR2/3 inaccessible.\n");
diff --git a/nvdebug.h b/nvdebug.h
index f644500..409b013 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -484,6 +484,7 @@ typedef union {
 #define NV_CHIP_ID_AMPERE 0x170
 #define NV_CHIP_ID_HOPPER 0x180
 #define NV_CHIP_ID_ADA 0x190
+#define NV_CHIP_ID_BLACKWELL 0x1A0
 
 inline static const char* ARCH2NAME(uint32_t arch) {
         switch (arch) {
@@ -521,14 +522,18 @@ inline static const char* ARCH2NAME(uint32_t arch) {
                 return "Turing";
         case 0x17:
                 return "Ampere";
-        case 0x18:
+        case 0x18: // Despite the Chip ID, Hopper functionally proceeds Ada
                 return "Hopper";
         case 0x19:
                 return "Ada Lovelace";
-        case 0x20:
-                return "Blackwell (?)";
+        case 0x1A:
+                return "Blackwell";
+        case 0x1B:
+                return "Rubin (?)";
+        case 0x1F: // NVIDIA-internal simulator
+                return "AMODEL";
         default:
-                if (arch < 0x19)
+                if (arch < 0x1A)
                         return "[unknown historical architecture]";
                 else
                         return "[future]";
@@ -881,25 +886,57 @@ union reg_range {
 #define NV_CE_PCE_MAP_SIZE 32
 
 
-/* Location of the 1Kb instance block with page tables for BAR1 and BAR2.
-  Support: Fermi+ (?), Pascal
+/* Location of the 1Kb instance block with page tables for the BAR1/2 regions.
+
+  On the H100, the "BAR1 block" describes what is actually BAR2, and the
+  "BAR2 block" describes BAR4.
+
+  PTR : Upper 28 bits of the 40-bit, (4k-aligned) address where the instance
+        block configuration is for the listed BAR region.
+
+  "Hopper+ uses 64-bit BARs, so GPU BAR2 should be at BAR4/5 and GPU BAR1 is at
+  BAR2/3" (open-gpu-kernel-modules)
 */
+// Support: Fermi through Ampere, Ada
 #define NV_PBUS_BAR1_BLOCK 0x00001704
 #define NV_PBUS_BAR2_BLOCK 0x00001714
 typedef union {
         struct {
                 uint32_t ptr:28;
                 enum INST_TARGET target:2;
-                 uint32_t padding0:1;
+                 uint32_t :1;
                 bool is_virtual:1;
         } __attribute__((packed));
         uint32_t raw;
         struct {
                 uint32_t map:30;
-                 uint32_t padding1:2;
+                 uint32_t :2;
         } __attribute__((packed));
 } bar_config_block_t;
 
+// Support: Hopper, Blackwell+
+// This is a "VREG" (virtual register?) in the documentation, meaning that it
+// needs the VREG base added first.
+#define NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET 0x00B80000
+#define NV_VIRTUAL_FUNCTION_PRIV_FUNC_BAR2_BLOCK (NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET+0x00000F70)
+typedef union {
+        struct {
+                bool is_pending:1;
+                bool is_outstanding:1;
+                 uint32_t :7;
+                bool is_virtual:1;
+                enum INST_TARGET target:2;
+                uint64_t ptr:40;
+                 uint32_t :12;
+        } __attribute__((packed));
+        uint64_t raw;
+        struct {
+                 uint32_t :10;
+                uint32_t map:22;
+                 uint32_t :32;
+        } __attribute__((packed));
+} bar_config_block_gh100_t;
+
 /* BAR0 PRAMIN (Private RAM Instance) window configuration
   One of the oldest ways to access video memory on NVIDIA GPUs is by using
   a configurable 1MB window into VRAM which is mapped into BAR0 (register)
@@ -914,21 +951,25 @@ typedef union {
   TARGET  : Which address space BASE points into
 
   Note: This seems to be set to 0x0bff00000 - 0x0c0000000 at least sometimes
-
-  Support: Tesla 2.0, Fermi, Kepler, Maxwell, Pascal, Turing, Ampere
 */
+// Support: Tesla 2.0, Fermi, Kepler, Maxwell, Pascal, Turing, Ampere, Ada
 #define NV_PBUS_BAR0_WINDOW 0x00001700
-#define NV_PRAMIN 0x00700000  // Goes until 0x00800000 (1MB window)
-#define NV_PRAMIN_LEN 0x00100000
+// On Hopper, and Blackwell+, TARGET must always be 0 (VIDMEM)
+// Support: Hopper, Blackwell+
+#define NV_XAL_EP_BAR0_WINDOW_BASE 0x0010fd40
 typedef union {
         struct {
                 uint32_t base:24;
                 enum INST_TARGET target:2;
-                 uint32_t padding0:6;
+                 uint32_t :6;
         } __attribute__((packed));
         uint32_t raw;
 } bar0_window_t;
 
+// Support: Tesla 2.0 through (at least) Blackwell
+#define NV_PRAMIN 0x00700000  // Goes until 0x00800000 (1MB window)
+#define NV_PRAMIN_LEN 0x00100000
+
 // Support: Tesla 2.0, Fermi, Kepler, Maxwell, Pascal, Turing, Ampere
 #define NV_PRAMIN_PDB_CONFIG_OFF 0x200
 typedef union {