5 files changed, 268 insertions, 181 deletions

diff --git a/bus.c b/bus.c
index 802b6df..951ac77 100644
--- a/bus.c
+++ b/bus.c
@@ -57,35 +57,25 @@ relocate:
         window.base = (u32)(addr >> 16); // Safe, due to above range check
         window.target = target;
         nvdebug_writel(g, NV_PBUS_BAR0_WINDOW, window.raw);
+        // Wait for the window to move by re-reading (as done in nvgpu driver)
+        (void) nvdebug_readl(g, NV_PBUS_BAR0_WINDOW);
         return (int)(addr & 0xffffull);
 }
 
-
-/* Get a persistent pointer to the page directory base
-  @param pdb Dereferencable pointer to the zeroeth entry of top-level page
-             directory (PD3) for the BAR2 register region.
-  Note: The returned pointer will be into the PRAMIN space. If the PRAMIN
-        window is moved to a region that does not cover the BAR2 page table,
-        this ***will move the window***.
-  Note: Even if the page table is located in SYS_MEM, we route reads/writes via
-        PRAMIN. This ensures that we always see what the GPU sees, and that
-        includes any passes through I/O MMUs or IOVA spaces.
+/* Get a copy of the BAR2 page directory configuration (base and aperture)
+  @param pd Pointer at which to store the configuration, including a pointer
+            and aperture for the zeroth entry of the top-level page directory
+            (PD3 for V2 page tables). This pointer **may not** be directly
+            dereferencable, and the caller may need to shift the BAR2 window.
+  @return 0 on success, -errno on error.
+  Note: This may move the PRAMIN window.
 */
-int get_bar2_pdb(struct nvdebug_state *g, void **pdb, bool *is_v2_pdb) {
-        static void* cached_pdb = NULL;
-        static bool cached_is_v2_pdb = false;
-        static long pd_hash = 0;
+int get_bar2_pdb(struct nvdebug_state *g, page_dir_config_t* pd) {
         int ret;
         bar_config_block_t bar2_block;
-        page_dir_config_t pd_config;
-        uint64_t pdb_vram;
 
-        // Use cached base as long as it's still pointing to the same thing
-        if (cached_pdb && readl(cached_pdb) == pd_hash) {
-                *pdb = cached_pdb;
-                *is_v2_pdb = cached_is_v2_pdb;
-                return 0;
-        }
+        if (!pd)
+                return -EINVAL;
 
         if (!g->bar2)
                 return -ENXIO;
@@ -107,24 +97,10 @@ int get_bar2_pdb(struct nvdebug_state *g, void **pdb, bool *is_v2_pdb) {
         }
         printk(KERN_INFO "[nvdebug] BAR2 inst block at off %x in PRAMIN\n", ret);
         // Pull the page directory base configuration from the instance block
-        if ((pd_config.raw = nvdebug_readq(g, NV_PRAMIN + ret + NV_PRAMIN_PDB_CONFIG_OFF)) == -1) {
+        if ((pd->raw = nvdebug_readq(g, NV_PRAMIN + ret + NV_PRAMIN_PDB_CONFIG_OFF)) == -1) {
                 printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 PDB configuration! BAR2/3 inaccessible.\n");
                 return -ENOTSUPP;
         }
-        pdb_vram = pd_config.page_dir_hi;
-        pdb_vram <<= 20;
-        pdb_vram |= pd_config.page_dir_lo;
-        pdb_vram <<= 12;
-        printk(KERN_INFO "[nvdebug] BAR2 PDB @ %llx (config raw: %llx)\n", pdb_vram, pd_config.raw);
-        // Setup PRAMIN to point at the page directory
-        if ((ret = addr_to_pramin_mut(g, pdb_vram, pd_config.target)) < 0) {
-                printk(KERN_ERR "[nvdebug] Invalid BAR2/3 PDB configuration! BAR2/3 inaccessible.\n");
-                return ret;
-        }
-
-        *pdb = cached_pdb = g->regs + NV_PRAMIN + ret;
-        pd_hash = readl(cached_pdb);
-        *is_v2_pdb = cached_is_v2_pdb = pd_config.is_ver2;
 
         return 0;
 }
diff --git a/mmu.c b/mmu.c
index e420864..70c00f9 100644
--- a/mmu.c
+++ b/mmu.c
@@ -1,117 +1,129 @@
-// Helpers to deal with NVIDIA's MMU and associated page tables
+/* Copyright 2024 Joshua Bakita
+ * Helpers to deal with NVIDIA's MMU and associated page tables
+ */
+#include <linux/err.h>  // ERR_PTR() etc.
+#include <linux/iommu.h>  // iommu_get_domain_for_dev() and iommu_iova_to_phys()
 #include <linux/kernel.h>  // Kernel types
 
 #include "nvdebug.h"
 
-/* One of the oldest ways to access video memory on NVIDIA GPUs is by using
-  a configurable 1MB window into VRAM which is mapped into BAR0 (register)
-  space starting at offset NV_PRAMIN. This is still supported on NVIDIA GPUs
-  and appear to be used today to bootstrap page table configuration.
+// Uncomment to print every PDE and PTE walked for debugging
+//#define DEBUG
+#ifdef DEBUG
+#define printk_debug printk
+#else
+#define printk_debug(...)
+#endif
 
-  Why is it mapped at a location called NVIDIA Private RAM Instance? Because
-  this used to point to the entirety of intance RAM, which was seperate from
-  VRAM on older NVIDIA GPUs.
-*/
+/* Convert a page directory (PD) pointer and aperture to be kernel-accessible
 
-/* Convert a physical VRAM address to an offset in the PRAMIN window
-  @param addr VRAM address to convert
-  @return -errno on error, PRAMIN offset on success
+  I/O MMU handling inspired by amdgpu_iomem_read() in amdgpu_ttm.c of the
+  AMDGPU driver.
 
-  Note: Use off2PRAMIN() instead if you want a dereferenceable address
-  Note: PRAMIN window is only 1MB, so returning an int is safe
-*/
-static int vram2PRAMIN(struct nvdebug_state *g, uint64_t addr) {
-        uint64_t pramin_base_va;
-        bar0_window_t window;
-        window.raw = nvdebug_readl(g, NV_PBUS_BAR0_WINDOW);
-        // Check if the address is valid (49 bits are addressable on-GPU)
-        if (addr & ~0x0001ffffffffffff) {
-                printk(KERN_ERR "[nvdebug] Invalid address %llx passed to %s!\n",
-                       addr, __func__);
-                return -EINVAL;
+  @param addr  Pointer from page directory entry (PDE)
+  @param pd_ap PD-type aperture (target address space) for `addr`
+  @return A dereferencable kernel address, or an ERR_PTR-wrapped error
+ */
+void __iomem *pd_deref(struct nvdebug_state *g, uintptr_t addr, enum PD_TARGET pd_ap) {
+        struct iommu_domain *dom;
+        phys_addr_t phys;
+
+        // Validate arguments
+        if (unlikely(!IS_PD_TARGET(pd_ap) || pd_ap == PD_AND_TARGET_INVALID || !addr))
+                return ERR_PTR(-EINVAL);
+
+        // VID_MEM accesses are the simple common-case
+        if (pd_ap == PD_AND_TARGET_VID_MEM) {
+                // Using BAR2 requires a page-table traversal. As this function is part
+                // of the page-table traversal process, it must instead use PRAMIN.
+                int off = addr_to_pramin_mut(g, addr, TARGET_VID_MEM);
+                if (off < 0)
+                        return ERR_PTR(off);
+                return g->regs + NV_PRAMIN + off;
         }
-        // For unclear (debugging?) reasons, PRAMIN can point to SYSMEM
-        if (window.target != TARGET_VID_MEM)
-                return -EFAULT;
-        pramin_base_va = ((uint64_t)window.base) << 16;
-        // Protect against out-of-bounds accesses
-        if (addr < pramin_base_va || addr > pramin_base_va + NV_PRAMIN_LEN)
-                return -ERANGE;
-        return addr - pramin_base_va;
-}
+        /* SYS_MEM accesses are rare. Only nvgpu (Jetson driver), nouveau, and this
+         * driver are known to create page directory entries in SYS_MEM.
+         *
+         * On systems using an I/O MMU, or some other I/O virtual address space,
+         * these are **not** physical addresses, and must first be translated
+         * through the I/O MMU before use.
+         * Example default meaning of a SYS_MEM address for a few CPUs:
+         * - Jetson Xavier : physical address
+         * - AMD 3950X     : I/O MMU address
+         * - Phenom II x4  : physical address
+         */
+        // Check for, and translate through, the I/O MMU (if any)
+        if ((dom = iommu_get_domain_for_dev(g->dev))) {
+                phys = iommu_iova_to_phys(dom, addr);
+                printk(KERN_ERR "[nvdebug] I/O MMU translated SYS_MEM I/O VA %#lx to physical address %llx.\n", addr, phys);
+        } else
+                phys = addr;
 
-// Convert a GPU physical address to CPU virtual address via the PRAMIN window
-// @return A dereferencable address, or 0 (an invalid physical address) on err
-void __iomem *phy2PRAMIN(struct nvdebug_state* g, uint64_t phy) {
-        int off = vram2PRAMIN(g, phy);
-        if (off == -ERANGE)
-                printk(KERN_ERR "[nvdebug] Page table walk off end of PRAMIN!\n");
-        if (off < 0)
+        if (!phys)
                 return 0;
-        return g->regs + NV_PRAMIN + vram2PRAMIN(g, phy);
-}
 
-/* FIXME
-void __iomem *off2BAR2(struct nvdebug_state* g, uint32_t off) {
-        return g->bar2 + off;
+        return phys_to_virt(addr);
 }
-*/
 
 // Internal helper for search_page_directory().
 uint64_t search_page_directory_subtree(struct nvdebug_state *g,
-                                       void __iomem *pde_offset,
-                                       void __iomem *(*off2addr)(struct nvdebug_state*, uint64_t),
+                                       uintptr_t pde_addr,
+                                       enum PD_TARGET pde_target,
                                        uint64_t addr_to_find,
                                        uint32_t level) {
         uint64_t res, i;
-        void __iomem *next;
+        void __iomem *pde_kern;
         page_dir_entry_t entry;
         if (level > sizeof(NV_MMU_PT_V2_SZ))
                 return 0;
         // Hack to workaround PDE0 being double-size and strangely formatted
         if (NV_MMU_PT_V2_ENTRY_SZ[level] == 16)
-                pde_offset += 8;
-        entry.raw_w = readq(pde_offset);
+                pde_addr += 8;
+        // Translate a VID_MEM/SYS_MEM-space address to something kernel-accessible
+        pde_kern = pd_deref(g, pde_addr, pde_target);
+        if (IS_ERR_OR_NULL(pde_kern)) {
+                printk(KERN_ERR "[nvdebug] %s: Unable to resolve %#lx in GPU %s to a kernel-accessible address. Error %ld.\n", __func__, pde_addr, pd_target_to_text(pde_target), PTR_ERR(pde_kern));
+                return 0;
+        }
+        // Read the page directory entry (a pointer to another directory, or a PTE)
+        entry.raw_w = readq(pde_kern);
         // If we reached an invalid (unpopulated) PDE, walk back up the tree
         if (entry.target == PD_AND_TARGET_INVALID)
                 return 0;
         // Succeed when we reach a PTE with the address we want