1 files changed, 267 insertions, 8 deletions

diff --git a/runlist.c b/runlist.c
index 7bb2ee4..3076d27 100644
--- a/runlist.c
+++ b/runlist.c
@@ -1,19 +1,13 @@
 /* Copyright 2024 Joshua Bakita
  * Helpers for dealing with the runlist and other Host (PFIFO) registers
  */
+#include <linux/iommu.h>  // iommu_get_domain_for_dev() and iommu_iova_to_phys()
 #include <linux/printk.h> // For printk()
 #include <asm/errno.h> // For error defines
 #include <asm/io.h> // For phys_to_virt()
 
 #include "nvdebug.h"
 
-// Uncomment to, upon BAR2 access failure, return a PRAMIN-based runlist pointer
-// in get_runlist_iter(). In order for this pointer to remain valid, PRAMIN
-// **must** not be moved during runlist traversal.
-// The Jetson TX2 has no BAR2, and stores the runlist in VID_MEM, so this must
-// be enabled to print the runlist on the TX2.
-//#define FALLBACK_TO_PRAMIN
-
 /* Get RunList RAM (RLRAM) offset for a runlist from the device topology
   @param rl_id      Which runlist to obtain [numbered in order of appearance in
                     the device topology (PTOP) registers]
@@ -116,6 +110,7 @@ int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl
                 runlist_len = submit.len;
                 printk(KERN_INFO "[nvdebug] Runlist %d for %x: %d entries @ %llx in %s (config raw: %#018llx %#018llx)\n",
                        rl_id, g->chip_id, submit.len, runlist_iova, target_to_text(runlist_target), base.raw, submit.raw);
+                printk(KERN_INFO "[nvdebug] Runlist offset is %d\n", submit.offset);
                 rl_iter->runlist_pri_base = runlist_pri_base;
         }
         // Return early on an empty runlist
@@ -130,6 +125,12 @@ int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl
                 if ((err = get_bar2_pdb(g, &pd_config)) < 0)
                         goto attempt_pramin_access;
 
+                // XXX: PD version detection not working on Hopper [is_ver2 errantly (?) unset]
+                if (g->chip_id >= NV_CHIP_ID_HOPPER && g->chip_id < NV_CHIP_ID_ADA) {
+                        printk(KERN_WARNING "[nvdebug] V3 page tables do not currently work on Hopper! Mystery config: %llx\n", pd_config.raw);
+                        err = -EOPNOTSUPP;
+                        goto attempt_pramin_access;
+                }
                 if (pd_config.is_ver2)
                         runlist_bar_vaddr = search_page_directory(g, pd_config, runlist_iova, TARGET_VID_MEM);
                 else
@@ -233,7 +234,7 @@ int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id) {
 }
 
 // Read and write runlist configuration, triggering a resubmit
-int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
+int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id, uint32_t off) {
         // Necessary registers do not exist pre-Fermi
         if (g->chip_id < NV_CHIP_ID_FERMI)
                 return -EOPNOTSUPP;
@@ -252,6 +253,9 @@ int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
                         return -EINVAL;
                 if ((submit.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id))) == -1)
                         return -EIO;
+                preempt_runlist(g, rl_id);
+                if (off != -1)
+                        submit.offset = off;
                 nvdebug_writeq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id), submit.raw);
         } else {
                 int err;
@@ -261,6 +265,9 @@ int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
                         return err;
                 if ((submit.raw = nvdebug_readq(g, runlist_pri_base + NV_RUNLIST_SUBMIT_GA100)) == -1)
                         return -EIO;
+                preempt_runlist(g, rl_id);
+                if (off != -1)
+                        submit.offset = off;
                 // On Ampere, this does not appear to trigger a preempt of the
                 // currently-running channel (even if the currently running channel
                 // becomes disabled), but will cause newly re-enabled channels
@@ -270,3 +277,255 @@ int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
         }
         return 0;
 }
+
+/* Get a CPU-accessible pointer to an arbitrary-address-space instance block
+  @param instance_addr  Address of instance block
+  @param intasce_target Aperture/taget of instance block address
+  @return A dereferencable KVA, NULL if not found, or an ERR_PTR-wrapped error
+
+  Note: The returned address will be a BAR2 or physical address, mapped into
+        kernel space, /not/ a PRAMIN-derived address. Thus, the returned
+        address will have an indefinite lifetime, and will be uneffected by use
+        of PRAMIN elsewhere (such as to read the CTXSW block).
+*/
+instance_ctrl_t *instance_deref(struct nvdebug_state *g, uint64_t instance_addr,
+                                enum INST_TARGET instance_target) {
+        if (!instance_addr || instance_target == TARGET_INVALID)
+                return ERR_PTR(-EINVAL);
+        if (instance_target == TARGET_VID_MEM) {
+                int err;
+                uint64_t inst_bar_vaddr;
+                page_dir_config_t pd_config;
+                // Only access VID_MEM via BAR2; do not fall back to PRAMIN
+                if (!g->bar2)
+                        return NULL;
+                // Find page tables which define how BAR2/3 offsets are translated to
+                // physical VID/SYS_MEM addresses.
+                if ((err = get_bar2_pdb(g, &pd_config)) < 0) {
+                        printk(KERN_ERR "[nvdebug] Error: Unable to access page directory "
+                               "configuration for BAR2/3. Error %d.\n", err);
+                        return ERR_PTR(err);
+                }
+                // Search the BAR2/3 page tables for the offset at which the instance
+                // block is mapped (reverse translation).
+                if (pd_config.is_ver2)
+                        inst_bar_vaddr = search_page_directory(g, pd_config, instance_addr, instance_target);
+                else
+                        inst_bar_vaddr = search_v1_page_directory(g, pd_config, instance_addr, instance_target);
+                if (!inst_bar_vaddr) {
+                        printk(KERN_WARNING "[nvdebug] Warning: Instance block %#018llx "
+                               "(%s) appears unmapped in BAR2/3.\n", instance_addr,
+                               target_to_text(instance_target));
+                        return NULL;
+                }
+                return g->bar2 + inst_bar_vaddr;
+        } else {
+                struct iommu_domain *dom;
+                // SYS_MEM addresses are physical addresses *from the perspective of
+                // the device* ("bus addresses"), and may not necessarially correspond
+                // to physical addresses from the perspective of the CPU. The I/O MMU
+                // is responsible for mapping bus addresses to CPU-relative physical
+                // addresses when there is no direct correspondence. If an I/O MMU is
+                // enabled on this GPU, ask it to translate the bus address to a
+                // CPU-relative physical address.
+                if ((dom = iommu_get_domain_for_dev(g->dev))) {
+                        // XXX: As of Aug 2024, this is not tested, so include extra logging
+                        printk(KERN_DEBUG "[nvdebug] I/O MMU translated SYS_MEM I/O VA %#llx for instance block", instance_addr);
+                        if (!(instance_addr = iommu_iova_to_phys(dom, instance_addr))) {
+                                printk(KERN_ERR "[nvdebug] Error: I/O MMU failed to translate "
+                                       "%#018llx (%s) to a CPU-relative physical address.\n",
+                                       instance_addr, target_to_text(instance_target));
+                                return ERR_PTR(-EADDRNOTAVAIL);
+                        }
+                        printk(KERN_DEBUG " to physical address %#llx.\n", instance_addr);
+                }
+                // Convert from a physical address to a kernel virtual address (KVA)
+                return phys_to_virt(instance_addr);
+        }
+}
+
+/* Get a CPU-accessible pointer to the CTXSW block for a channel intance block
+  @param inst Dereferencable pointer to the start of a complete instance block
+  @return A dereferencable KVA, NULL if not found, or an ERR_PTR-wrapped error
+
+  Note: The returned address **will** be a PRAMIN-based address. Any changes to
+        PRAMIN **will** invalidate the returned pointer. `inst` **cannot** be a
+        pointer into the PRAMIN space.
+*/
+context_switch_ctrl_t *get_ctxsw(struct nvdebug_state *g,
+                                 instance_ctrl_t *inst) {
+        int err;
+        context_switch_ctrl_t *wfi = NULL;
+        uint64_t wfi_virt, wfi_phys, ctxsw_virt, ctxsw_phys;
+        enum INST_TARGET wfi_phys_aperture, ctxsw_phys_aperture;
+
+        // The WFI block contains a pointer to the CTXSW block, which contains the
+        // preemption mode configuration for the context. (As best I can tell, the WFI
+        // block is subcontext-specific, whereas the CTXSW block is context-wide.
+        wfi_virt = (uint64_t)inst->engine_wfi_ptr << 12;
+
+        // WFI may not be configured
+        if (!wfi_virt)
+                goto out;
+
+        // Determine the physical location of the WFI block
+        if (inst->engine_wfi_is_virtual) {
+                if (inst->pdb.is_ver2)
+                        err = translate_page_directory(g, inst->pdb, wfi_virt, &wfi_phys, &wfi_phys_aperture);
+                else
+                        err = translate_v1_page_directory(g, inst->pdb, wfi_virt, &wfi_phys, &wfi_phys_aperture);
+                if (err) {
+                        printk(KERN_ERR "[nvdebug] Critical: Inconsistent GPU state; WFI block "
+                               "pointer %#018llx (virt) cannot be found in process page tables! "
+                               "Translation error %d.\n", wfi_virt, -err);
+                        return ERR_PTR(-ENOTRECOVERABLE);
+                }
+        } else {
+                wfi_phys = (uint64_t)inst->engine_wfi_ptr << 12;
+                wfi_phys_aperture = inst->engine_wfi_target;
+        }
+
+        // Get a dereferencible pointer to the WFI block (the WFI and CTXSW blocks
+        // have not been observed as mapped in BAR2/3, so we use the PRAMIN window).
+        // Note: On Jetson boards, we could attempt to avoid PRAMIN since CTXSW is in
+        //       SYS_MEM, but this function will always need to use PRAMIN to work
+        //       around the WFI and CTXSW blocks not being accessible via BAR2/3 on
+        //       PCIe GPU, so always use PRAMIN for simplicity.
+        if ((wfi_phys = addr_to_pramin_mut(g, wfi_phys, wfi_phys_aperture)) == -1)
+                goto out;
+        wfi = g->regs + wfi_phys + NV_PRAMIN;
+
+// XXX
+//      return wfi;
+// End XXX
+
+        // While the WFI block uses the same layout as the context switch (CTXSW)
+        // control block, it is mostly unpopulated except for a few pointers on GPUs
+        // after Volta. This appears to be related to subcontexts, where each
+        // subcontext has its own WFI block containing a pointer to the overarching
+        // CTXSW block. Only attempt to find the overarching CTXSW block if at least