Support page directories outside PRAMIN or in SYS_MEM

- Re-read PRAMIN configuration after update to verify change applies
- Return a page_dir_config_t rather than just an address and page.
  table version from `get_bar2_pdb()`.
- Less verbose logging for MMU-related functions by default.
- Perform all conversion from SYS_MEM/VID_MEM addresses to kernel
  addresses inside the translation functions, via the new function
  'pd_deref()`.
- Support use of an I/O MMU, page tables/directories outside the
  current PRAMIN window, and page tables/directories arbitrarially
  located in SYS_MEM or VID_MEM on different levels of the same tree.
- Heavily improve documentation and add references for Version 1 and
  Version 0 page tables.
- Improve logging in `runlist.c` to include runlist and chip IDs.
- Update all users of search_page_directory* to use the new API.
- Remove now-unused supporting functions from `mmu.c`.

Tested on GTX 970, GTX 1060 3GB, Jetson TX2, Titan V, Jetson Xavier,
and RTX 2080 Ti.

1 files changed, 110 insertions, 96 deletions

diff --git a/mmu.c b/mmu.c
index e420864..70c00f9 100644
--- a/mmu.c
+++ b/mmu.c
@@ -1,117 +1,129 @@
-// Helpers to deal with NVIDIA's MMU and associated page tables
+/* Copyright 2024 Joshua Bakita
+ * Helpers to deal with NVIDIA's MMU and associated page tables
+ */
+#include <linux/err.h>  // ERR_PTR() etc.
+#include <linux/iommu.h>  // iommu_get_domain_for_dev() and iommu_iova_to_phys()
 #include <linux/kernel.h>  // Kernel types
 
 #include "nvdebug.h"
 
-/* One of the oldest ways to access video memory on NVIDIA GPUs is by using
-  a configurable 1MB window into VRAM which is mapped into BAR0 (register)
-  space starting at offset NV_PRAMIN. This is still supported on NVIDIA GPUs
-  and appear to be used today to bootstrap page table configuration.
+// Uncomment to print every PDE and PTE walked for debugging
+//#define DEBUG
+#ifdef DEBUG
+#define printk_debug printk
+#else
+#define printk_debug(...)
+#endif
 
-  Why is it mapped at a location called NVIDIA Private RAM Instance? Because
-  this used to point to the entirety of intance RAM, which was seperate from
-  VRAM on older NVIDIA GPUs.
-*/
+/* Convert a page directory (PD) pointer and aperture to be kernel-accessible
 
-/* Convert a physical VRAM address to an offset in the PRAMIN window
-  @param addr VRAM address to convert
-  @return -errno on error, PRAMIN offset on success
+  I/O MMU handling inspired by amdgpu_iomem_read() in amdgpu_ttm.c of the
+  AMDGPU driver.
 
-  Note: Use off2PRAMIN() instead if you want a dereferenceable address
-  Note: PRAMIN window is only 1MB, so returning an int is safe
-*/
-static int vram2PRAMIN(struct nvdebug_state *g, uint64_t addr) {
-        uint64_t pramin_base_va;
-        bar0_window_t window;
-        window.raw = nvdebug_readl(g, NV_PBUS_BAR0_WINDOW);
-        // Check if the address is valid (49 bits are addressable on-GPU)
-        if (addr & ~0x0001ffffffffffff) {
-                printk(KERN_ERR "[nvdebug] Invalid address %llx passed to %s!\n",
-                       addr, __func__);
-                return -EINVAL;
+  @param addr  Pointer from page directory entry (PDE)
+  @param pd_ap PD-type aperture (target address space) for `addr`
+  @return A dereferencable kernel address, or an ERR_PTR-wrapped error
+ */
+void __iomem *pd_deref(struct nvdebug_state *g, uintptr_t addr, enum PD_TARGET pd_ap) {
+        struct iommu_domain *dom;
+        phys_addr_t phys;
+
+        // Validate arguments
+        if (unlikely(!IS_PD_TARGET(pd_ap) || pd_ap == PD_AND_TARGET_INVALID || !addr))
+                return ERR_PTR(-EINVAL);
+
+        // VID_MEM accesses are the simple common-case
+        if (pd_ap == PD_AND_TARGET_VID_MEM) {
+                // Using BAR2 requires a page-table traversal. As this function is part
+                // of the page-table traversal process, it must instead use PRAMIN.
+                int off = addr_to_pramin_mut(g, addr, TARGET_VID_MEM);
+                if (off < 0)
+                        return ERR_PTR(off);
+                return g->regs + NV_PRAMIN + off;
         }
-        // For unclear (debugging?) reasons, PRAMIN can point to SYSMEM
-        if (window.target != TARGET_VID_MEM)
-                return -EFAULT;
-        pramin_base_va = ((uint64_t)window.base) << 16;
-        // Protect against out-of-bounds accesses
-        if (addr < pramin_base_va || addr > pramin_base_va + NV_PRAMIN_LEN)
-                return -ERANGE;
-        return addr - pramin_base_va;
-}
+        /* SYS_MEM accesses are rare. Only nvgpu (Jetson driver), nouveau, and this
+         * driver are known to create page directory entries in SYS_MEM.
+         *
+         * On systems using an I/O MMU, or some other I/O virtual address space,
+         * these are **not** physical addresses, and must first be translated
+         * through the I/O MMU before use.
+         * Example default meaning of a SYS_MEM address for a few CPUs:
+         * - Jetson Xavier : physical address
+         * - AMD 3950X     : I/O MMU address
+         * - Phenom II x4  : physical address
+         */
+        // Check for, and translate through, the I/O MMU (if any)
+        if ((dom = iommu_get_domain_for_dev(g->dev))) {
+                phys = iommu_iova_to_phys(dom, addr);
+                printk(KERN_ERR "[nvdebug] I/O MMU translated SYS_MEM I/O VA %#lx to physical address %llx.\n", addr, phys);
+        } else
+                phys = addr;
 
-// Convert a GPU physical address to CPU virtual address via the PRAMIN window
-// @return A dereferencable address, or 0 (an invalid physical address) on err
-void __iomem *phy2PRAMIN(struct nvdebug_state* g, uint64_t phy) {
-        int off = vram2PRAMIN(g, phy);
-        if (off == -ERANGE)
-                printk(KERN_ERR "[nvdebug] Page table walk off end of PRAMIN!\n");
-        if (off < 0)
+        if (!phys)
                 return 0;
-        return g->regs + NV_PRAMIN + vram2PRAMIN(g, phy);
-}
 
-/* FIXME
-void __iomem *off2BAR2(struct nvdebug_state* g, uint32_t off) {
-        return g->bar2 + off;
+        return phys_to_virt(addr);
 }
-*/
 
 // Internal helper for search_page_directory().
 uint64_t search_page_directory_subtree(struct nvdebug_state *g,
-                                       void __iomem *pde_offset,
-                                       void __iomem *(*off2addr)(struct nvdebug_state*, uint64_t),
+                                       uintptr_t pde_addr,
+                                       enum PD_TARGET pde_target,
                                        uint64_t addr_to_find,
                                        uint32_t level) {
         uint64_t res, i;
-        void __iomem *next;
+        void __iomem *pde_kern;
         page_dir_entry_t entry;
         if (level > sizeof(NV_MMU_PT_V2_SZ))
                 return 0;
         // Hack to workaround PDE0 being double-size and strangely formatted
         if (NV_MMU_PT_V2_ENTRY_SZ[level] == 16)
-                pde_offset += 8;
-        entry.raw_w = readq(pde_offset);
+                pde_addr += 8;
+        // Translate a VID_MEM/SYS_MEM-space address to something kernel-accessible
+        pde_kern = pd_deref(g, pde_addr, pde_target);
+        if (IS_ERR_OR_NULL(pde_kern)) {
+                printk(KERN_ERR "[nvdebug] %s: Unable to resolve %#lx in GPU %s to a kernel-accessible address. Error %ld.\n", __func__, pde_addr, pd_target_to_text(pde_target), PTR_ERR(pde_kern));
+                return 0;
+        }
+        // Read the page directory entry (a pointer to another directory, or a PTE)
+        entry.raw_w = readq(pde_kern);
         // If we reached an invalid (unpopulated) PDE, walk back up the tree
         if (entry.target == PD_AND_TARGET_INVALID)
                 return 0;
         // Succeed when we reach a PTE with the address we want
         if (entry.is_pte) {
                 // TODO: Handle huge pages here
-                printk(KERN_INFO "[nvdebug] PTE for phy addr %#018llx, ap '%s', vol '%d', priv '%d', ro '%d', no_atomics '%d' (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, entry.is_privileged, entry.is_readonly, entry.atomics_disabled, entry.raw_w);
+                printk_debug(KERN_INFO "[nvdebug] PTE for phy addr %#018llx, ap '%s', vol '%d', priv '%d', ro '%d', no_atomics '%d' (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, entry.is_privileged, entry.is_readonly, entry.atomics_disabled, entry.raw_w);
                 return (uint64_t)entry.addr << 12 == addr_to_find;
         }
-        printk(KERN_INFO "[nvdebug] Found PDE pointing to %#018llx in ap '%s' vol '%d' at lvl %d (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, level, entry.raw_w);
+        printk_debug(KERN_INFO "[nvdebug] Found PDE pointing to %#018llx in ap '%s' vol '%d' at lvl %d (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, level, entry.raw_w);
         // Depth-first search of the page table
         for (i = 0; i < NV_MMU_PT_V2_SZ[level + 1]; i++) {
-                next = off2addr(g, ((uint64_t)entry.addr << 12) + NV_MMU_PT_V2_ENTRY_SZ[level + 1] * i);
-                // off2addr can fail
-                if (!next || !entry.addr_w) {
-                        printk(KERN_ERR "[nvdebug] %s: Unable to resolve GPU PA to CPU PA\n", __func__);
-                        return 0;
-                }
-                res = search_page_directory_subtree(g, next, off2addr, addr_to_find, level + 1);
+                uint64_t next = ((uint64_t)entry.addr << 12) + NV_MMU_PT_V2_ENTRY_SZ[level + 1] * i;
+                res = search_page_directory_subtree(g, next, entry.target, addr_to_find, level + 1);
                 if (res)
                         return res | (i << NV_MMU_PT_V2_LSB[level + 1]);
         }
         return 0;
 }
 
-/* GPU Physical address -> Virtual address ("reverse" translation)
+/* GPU Physical address -> Virtual address ("reverse" translation) for V2 tables
+
+  Depth-first search a page directory of the GPU MMU for where a particular
+  physical address is mapped. Upon finding a mapping, the virtual address is
+  returned.
 
-   Depth-first search a page directory of the GPU MMU for where a particular
-   physical address is mapped. Upon finding a mapping, the virtual address is
-   returned.
+  The page directory may be located in VID_MEM, SYS_MEM, or some combination of
+  the two.
 
-  @param pde_offset   Dereferenceable pointer to the start of the PDE3 entries
-  @param off2addr     Func to convert VRAM phys addresses to valid CPU VAs
+  @param pd_config    Page Directory configuration, containing pointer and
+                      aperture for the start of the PDE3 entries
   @param addr_to_find Physical address to reconstruct the virtual address of
   @return 0 on error, otherwise the virtual address at which addr_to_find is
           mapped into by this page table. (Zero is not a valid virtual address)
 */
 uint64_t search_page_directory(struct nvdebug_state *g,
-                               void __iomem *pde_offset,
-                               void __iomem *(*off2addr)(struct nvdebug_state*, uint64_t),
+                               page_dir_config_t pd_config,
                                uint64_t addr_to_find) {
         uint64_t res, i;
         // Make sure that the query is page-aligned
@@ -119,57 +131,62 @@ uint64_t search_page_directory(struct nvdebug_state *g,
                 printk(KERN_WARNING "[nvdebug] Attempting to search for unaligned address %llx in search_page_directory()!\n", addr_to_find);
                 return 0;
         }