Heavily refactor runlist code for correctness and Turing support

- Support differently-formatted runlist registers on Turing
- Support different runlist register offsets on Turing
- Fix incorrect indenting when printing the runlist
- Fix `preempt_tsg` and `switch_to_tsg` API implementations to
  correctly interface with the hardware (previously, they would try
  to disable scheduling for the last-updated runlist pointer, which
  was nonsense, and just an artifact of my early misunderstandings
  of how the NV_PFIFO_RUNLIST* registers worked).
- Remove misused NV_PFIFO_RUNLIST and NV_PFIFO_RUNLIST_BASE registers
- Refactor `runlist.c` to use the APIs from `bus.c`

1 files changed, 87 insertions, 125 deletions

diff --git a/runlist.c b/runlist.c
index ed35c7e..c725e77 100644
--- a/runlist.c
+++ b/runlist.c
@@ -1,172 +1,134 @@
-#include <linux/kernel.h>  // Kernel types
+/* Copyright 2024 Joshua Bakita
+ * Helpers for dealing with the runlist and other Host (PFIFO) registers
+ */
+#include <linux/printk.h> // For printk()
+#include <asm/errno.h> // For error defines
+#include <asm/io.h> // For phys_to_virt()
 
 #include "nvdebug.h"
 
+// Uncomment to, upon BAR2 access failure, return a PRAMIN-based runlist pointer
+// **If enabled, PRAMIN may not be otherwise used while walking the runlist!**
 #define FALLBACK_TO_PRAMIN
 
 /* Get runlist head and info (incl. length)
-   @param rl_iter Location at which to store output
-   @param rl_id   Which runlist to obtain?
+  @param rl_id   Which runlist to obtain?
+  @param rl_iter Location at which to store output
+  @return 0 or -errno on error
 */
 int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl_iter) {
-        runlist_base_t rl_base;
-        runlist_info_t rl_info;
-        u64 runlist_iova;
+        uint64_t runlist_iova;
+        enum INST_TARGET runlist_target;
+        uint16_t runlist_len;
+#ifdef FALLBACK_TO_PRAMIN
+        int off;
+#endif // FALLBACK_TO_PRAMIN
+        // Zero-initialize the runlist iterator
         *rl_iter = (struct runlist_iter){0};
-        rl_base.raw = nvdebug_readl(g, NV_PFIFO_ENG_RUNLIST_BASE(rl_id));
-        // Check that reads are working
-        if (rl_base.raw == -1)
-                return -EIO;
-        // The address pointed to `regs` + NV_PFIFO_RUNLIST_BASE seems to not be:
-        // - A GPU address (type is sysmem_coherent)
-        // - A physical address (dereferencing after ioremap crashes)
-        // - A kernel virtual address (dereferencing segfaults)
-        // So maybe it's some sort of custom thing? This is an address that the GPU
-        // can use, so it would make most sense for it to be a physical address.
-        //
-        // BUT, it can't possibly be a physical address, as it would refer to an
-        // address greater than the maximum one on our system (by a lot!).
-        // Maybe I'm reading the runlist base wrong?
-        // Aha, the driver calls it runlist_iova. Sounds like runlist I/O virtual
-        // address! So, what's this I/O address space? All I know is that it's what
-        // nvgpu_mem_get_addr() returns. That function returns the result of either:
-        // - gpu_phys_addr which is  __nvgpu_sgl_phys on our platform which (?)
-        //   converts an IPA to a PA?
-        // - nvgpu_mem_iommu_translate
-        //
-        // The original memory is allocated with nvgpu_dma_alloc_flags_sys(), which
-        // returns SYSMEM.
-        //
-        // To convert a physical address to a IOMMU address, we add a bit
-        //
-        // BUT, it turns out that it IS JUST A PHYSICAL ADDRESS! It wasn't working
-        // before because the GPU had simply gone to sleep and invalidated its
-        // register state, so nvgpu_readl() was simply returning garbage.
-        rl_info.raw = nvdebug_readl(g, NV_PFIFO_ENG_RUNLIST(rl_id));
-        if (rl_info.raw == -1)
-                return -EIO;
-        runlist_iova = ((u64)rl_base.ptr) << 12;
-        printk(KERN_INFO "[nvdebug] Runlist %d @ %llx in %s (config raw: %x)\n",
-               rl_id, runlist_iova, target_to_text(rl_base.target), rl_base.raw);
-        printk(KERN_INFO "[nvdebug] Runlist length %d, ID %d\n", rl_info.len, rl_info.id);
+
+        // Get runlist location and length using architecture-dependent logic
+        if (g->chip_id < NV_CHIP_ID_TURING) {
+                eng_runlist_gf100_t rl;
+                if ((rl.raw = nvdebug_readq(g, NV_PFIFO_ENG_RUNLIST_BASE_GF100(rl_id))) == -1)
+                        return -EIO;
+                runlist_iova = ((uint64_t)rl.ptr) << 12;
+                runlist_target = rl.target;
+                printk(KERN_INFO "[nvdebug] Runlist %d: %d entries @ %llx in %s (config raw: %#018llx)\n",
+                           rl_id, rl.len, runlist_iova, target_to_text(rl.target), rl.raw);
+                runlist_len = rl.len;
+        } else if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                runlist_base_tu102_t base;
+                runlist_submit_tu102_t submit;
+                if ((base.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_BASE_TU102(rl_id))) == -1)
+                        return -EIO;
+                if ((submit.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id))) == -1)
+                        return -EIO;
+                runlist_iova = ((uint64_t)base.ptr) << 12;
+                runlist_target = base.target;
+                runlist_len = submit.len;
+        }
         // Return early on an empty runlist
-        if (!rl_info.len)
+        if (!runlist_len)
                 return 0;
+
         // If the runlist is in VID_MEM, search the BAR2/3 page tables for a mapping
-        if (rl_base.target == TARGET_VID_MEM) {
-                printk(KERN_WARNING "[nvdebug] Runlist is located in video memory. Access to video memory is experimental.");
-                bar_config_block_t bar1_block, bar2_block;
-                bar1_block.raw = nvdebug_readl(g, NV_PBUS_BAR1_BLOCK);
-                printk(KERN_INFO "[nvdebug] BAR1 inst block @ %llx in %s's %s address space.\n", ((u64)bar1_block.ptr) << 12, target_to_text(bar1_block.target), bar1_block.is_virtual ? "virtual" : "physical");
-                bar2_block.raw = nvdebug_readl(g, NV_PBUS_BAR2_BLOCK);
-                printk(KERN_INFO "[nvdebug] BAR2 inst block @ %llx in %s's %s address space.\n", ((u64)bar2_block.ptr) << 12, target_to_text(bar2_block.target), bar1_block.is_virtual ? "virtual" : "physical");
-                uint32_t bar_inst_pramin_offset = vram2PRAMIN(g, (uint64_t)bar2_block.ptr << 12);
-                if (!bar_inst_pramin_offset) {
-                        printk(KERN_WARNING "[nvdebug] Unable to find instance block for BAR2/3 in the current NV_PRAMIN window. VRAM inaccessible.\n");
-                        goto attempt_pramin_access;
-                }
-                /* TODO: Support BAR1?
-                bar_inst_pramin_offset = vram2PRAMIN(g, bar1_block.ptr << 12);
-                if (!bar_inst_pramin_offset) {
-                        printk(KERN_WARNING "[nvdebug] Unable to find instance block for BAR1 in the current NV_PRAMIN window. VRAM inaccessible.\n");
-                        return -EOPNOTSUPP;
-                }*/
-                // Instance blocks (size == 1kb) contain many things, but we only care about
-                // the section which describes the location of the page directory (page table)
-                uint32_t bar_pdb_config_pramin_offset = bar_inst_pramin_offset + NV_PRAMIN_PDB_CONFIG_OFF;
-                page_dir_config_t pd_config;
-                pd_config.raw = nvdebug_readq(g, bar_pdb_config_pramin_offset + NV_PRAMIN);
-                uint64_t bar_pdb_vram_addr = pd_config.page_dir_hi;
-                bar_pdb_vram_addr <<= 20;
-                bar_pdb_vram_addr |= pd_config.page_dir_lo;
-                bar_pdb_vram_addr <<= 12;
-                printk(KERN_INFO "[nvdebug] BAR2 PDB @ %llx in %s of version %s (config raw: %llx)\n", bar_pdb_vram_addr, target_to_text(pd_config.target), pd_config.is_ver2 ? "2" : "1", pd_config.raw);
-                // TODO: SYSMEM support for page table location
-                if (pd_config.target != TARGET_VID_MEM) {
-                        printk(KERN_WARNING "[nvdebug] BAR2 PDB is in an unsupported location.\n");
-                        goto attempt_pramin_access;
-                }
-                uint32_t bar_pdb_pramin_offset = vram2PRAMIN(g, bar_pdb_vram_addr);
-                if (!bar_pdb_pramin_offset) {
-                        printk(KERN_WARNING "[nvdebug] Unable to find page directory BAR2/3 in the current NV_PRAMIN window. VRAM inaccessible.\n");
-                        goto attempt_pramin_access;
-                }
+        if (runlist_target == TARGET_VID_MEM) {
+                void __iomem *bar2_page_dir;
+                bool pdb_is_ver2;
                 uint64_t runlist_bar_vaddr;
-                if (pd_config.is_ver2)
-                        runlist_bar_vaddr = search_page_directory(g, g->regs + NV_PRAMIN + bar_pdb_pramin_offset, phy2PRAMIN, runlist_iova);
+
+                if (get_bar2_pdb(g, &bar2_page_dir, &pdb_is_ver2) < 0)
+                        return -EIO;
+
+                if (pdb_is_ver2)
+                        runlist_bar_vaddr = search_page_directory(g, bar2_page_dir, phy2PRAMIN, runlist_iova);
                 else
-                        runlist_bar_vaddr = search_v1_page_directory(g, g->regs + NV_PRAMIN + bar_pdb_pramin_offset, phy2PRAMIN, runlist_iova);
+                        runlist_bar_vaddr = search_v1_page_directory(g, bar2_page_dir, phy2PRAMIN, runlist_iova);
                 if (!runlist_bar_vaddr) {
                         printk(KERN_WARNING "[nvdebug] Unable to find runlist mapping in BAR2/3 page tables.\n");
                         goto attempt_pramin_access;
                 }
                 printk(KERN_INFO "[nvdebug] Runlist @ %llx in BAR2 virtual address space.\n", runlist_bar_vaddr);
-                /* XXX: Old test code
-                uint32_t bar2_pd_pramin_offset = vram_to_pramin_off(bar2_pd);
-                //walk_pd_subtree(bar2_pd_pramin_offset);
-                uint64_t runlist_bar2_vaddr = search_pd_subtree(bar2_pd_pramin_offset, runlist_iova);
-                page_dir_entry_t pde_0;
-                pde_0.raw = nvdebug_readl(g, NV_PRAMIN + bar2_pd_pramin_offset);
-                uint32_t pde_1 = nvdebug_readl(g, NV_PRAMIN + vram_to_pramin_off(((u64)pde_0.addr) << 12));
-                uint64_t pde_bar2_vaddr = search_pd_subtree(bar2_pd_pramin_offset, ((u64)pde_0.addr) << 12);
-                uint32_t pde_2 = readl(g->bar3 + pde_bar2_vaddr);
-                printk(KERN_INFO "[nvdebug] PDE0 via PRAMIN: %x, via BAR3: %x\n", pde_1, pde_2);
-                */
-                if (!g->bar3) {
+                if (!g->bar2) {
                         printk(KERN_WARNING "[nvdebug] BAR2/3 not mapped.\n");
                         return -ENODEV;
                 }
                 rl_iter->curr_entry = g->bar2 + runlist_bar_vaddr;
         } else {
                 // Directly access the runlist if stored in SYS_MEM (physically addressed)
-                rl_iter->curr_entry = phys_to_virt(runlist_iova);
+                // XXX: SYS_MEM is an IOMMU address on some platforms, causing this to crash
+                rl_iter->curr_entry = (void*)phys_to_virt(runlist_iova);
         }
-        rl_iter->rl_info = rl_info;
+        rl_iter->len = runlist_len;
         return 0;
+
 attempt_pramin_access:
 #ifdef FALLBACK_TO_PRAMIN
         printk(KERN_INFO "[nvdebug] Attempting to move PRAMIN window to runlist as BAR2/3-based access failed [DANGEROUS SIDE EFFECTS]!\n");
-        bar0_window_t win;
-        win.base = (runlist_iova >> 16);
-        win.target = TARGET_VID_MEM;
-        // Shift PRAMIN window. This will cause problems if it races with driver code
-        // that tries to do the same, or expects the window not to move.
-        nvdebug_writel(g, NV_PBUS_BAR0_WINDOW, win.raw);
-        uint32_t off = vram2PRAMIN(g, runlist_iova);
-        // Workaround bug for if `off` should be zero (vram2PRAMIN normally returns
-        // this on error)
-        if (!off && (runlist_iova & 0xffffull != runlist_iova)) {
-                printk(KERN_INFO "[nvdebug] Unable to shift PRAMIN to runlist. Aborting...\n");
-                return -EOPNOTSUPP;
-        }
+        if ((off = addr_to_pramin_mut(g, runlist_iova, runlist_target)) == -1)
+                return off;
         rl_iter->curr_entry = g->regs + NV_PRAMIN + off;
-        rl_iter->rl_info = rl_info;
+        rl_iter->len = runlist_len;
         return 0;
 #else
         return -EOPNOTSUPP;
 #endif // FALLBACK_TO_PRAMIN
 }
 
+/* Trigger a preempt of the specified TSG
+  @param tsg_id ID of TSG to preempt.
+  @return 0 or -errno on error
+
+  Note: If no other TSGs exist in the associated runlist, this TSG may
+        continue executing, unless NV_PFIFO_SCHED_DISABLE is set, or all the
+        channels of the TSG to be preempted are disabled.
+*/
 int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) {
-        runlist_info_t rl_info;
         pfifo_preempt_t pfifo_preempt;
-        runlist_disable_t rl_disable;
-        if (!g)
-                return -EIO;
-        rl_info.raw = nvdebug_readl(g, NV_PFIFO_RUNLIST);
+        if (g->chip_id < NV_CHIP_ID_KEPLER)
+                return -EOPNOTSUPP;
+
+        pfifo_preempt.raw = 0;
         pfifo_preempt.id = tsg_id;
         pfifo_preempt.is_pending = 0;
         pfifo_preempt.type = PREEMPT_TYPE_TSG;
-        // There may be a bug (?) that requires us to disable scheduling before preempting
-        rl_disable.raw = nvdebug_readl(g, NV_PFIFO_SCHED_DISABLE);
-        rl_disable.raw |= BIT(rl_info.id);  // Disable runlist rl_info.id
-        nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw);
+
         // Actually trigger the preemption
         nvdebug_writel(g, NV_PFIFO_PREEMPT, pfifo_preempt.raw);
-        // Renable scheduling
-        rl_disable.raw &= ~BIT(rl_info.id);  // Enable runlist rl_info.id
-        nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw);
+        return 0;
+}
+
+/* Trigger a preempt of the specified runlist
+  @param rl_id ID of runlist to preempt.
+  @return 0 or -errno on error
+*/
+int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id) {
+        runlist_preempt_t rl_preempt;
+        if (g->chip_id < NV_CHIP_ID_VOLTA)
+                return -EOPNOTSUPP;
 
-        printk(KERN_INFO "[nvdebug] TSG %d preempted (runlist %d)\n", tsg_id, rl_info.id);
+        // Overwrite, as the register contains nothing to preserve
+        rl_preempt.raw = BIT(rl_id);
+        nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
         return 0;
 }