Snapshot for ECRTS'25 artifact evaluation

9 files changed, 2154 insertions, 46 deletions

diff --git a/Makefile b/Makefile
index fea3819..9d6d374 100644
--- a/Makefile
+++ b/Makefile
@@ -8,3 +8,6 @@ all:
         make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules
 clean:
         make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean
+
+nvdebug_user.so: runlist.c mmu.c bus.c nvdebug_user.c
+        gcc $< -shared -o $@ $(KBULID_CFLAGS)
diff --git a/README.md b/README.md
index da3e5d7..2889b29 100644
--- a/README.md
+++ b/README.md
@@ -59,6 +59,7 @@ Not all these TPCs will necessarially be enabled in every GPC.
 Use `cat gpcX_tpc_mask` to get a bit mask of which TPCs are disabled for GPC X.
 A set bit indicates a disabled TPC.
 This API is only available on enabled GPCs.
+Bits greater than the number of on-chip TPCs per GPC should be ignored (it may appear than non-existent TPCs are "disabled").
 
 Example usage: To get the number of on-chip SMs on Volta+ GPUs, multiply the return of `cat num_gpcs` with `cat num_tpc_per_gpc` and multiply by 2 (SMs per TPC).
 
@@ -83,6 +84,13 @@ Use `echo Z > runlistY/switch_to_tsg` to switch the GPU to run only the specifie
 
 Use `echo Y > resubmit_runlist` to resubmit runlist Y (useful to prompt newer GPUs to pick up on re-enabled channels).
 
+## Error Interpretation
+First check the kernel log to see if in includes more information about the error.
+The following conventions are used for certain error codes:
+
+- EIO, "Input/Output Error," is returned when an operation fails due to a bad register read.
+- (Other errors may not have a consistent conventional meaning; see the implementation.)
+
 ## General Codebase Structure
 - `nvdebug.h` defines and describes all GPU data structures. This does not depend on any kernel-internal headers.
 - `nvdebug_entry.h` contains module startup, device detection, initialization, and module teardown logic.
@@ -94,4 +102,4 @@ Use `echo Y > resubmit_runlist` to resubmit runlist Y (useful to prompt newer GP
 
 - The runlist-printing API does not work when runlist management is delegated to the GPU System Processor (GSP) (most Turing+ datacenter GPUs).
   To workaround, enable the `FALLBACK_TO_PRAMIN` define in `runlist.c`, or reload the `nvidia` kernel module with the `NVreg_EnableGpuFirmware=0` parameter setting.
-  (Eg. on A100: end all GPU-using processes, then `sudo rmmod nvidia_uvm nvidia; sudo modprobe nvidia NVreg_EnableGpuFirmware=0`.)
+  (Eg. on A100: end all GPU-using processes, then `sudo rmmod nvidia_drm nvidia_modeset nvidia_uvm nvidia; sudo modprobe nvidia NVreg_EnableGpuFirmware=0`.)
diff --git a/device_info_procfs.c b/device_info_procfs.c
index 4e4ab03..105e731 100644
--- a/device_info_procfs.c
+++ b/device_info_procfs.c
@@ -18,7 +18,7 @@ static ssize_t nvdebug_reg32_read(struct file *f, char __user *buf, size_t size,
                 return 0;
 
         if ((read = nvdebug_readl(g, (uintptr_t)pde_data(file_inode(f)))) == -1)
-                return -EOPNOTSUPP;
+                return -EIO;
         // 32 bit register will always take less than 16 characters to print
         chars_written = scnprintf(out, 16, "%#0x\n", read);
         if (copy_to_user(buf, out, chars_written))
@@ -32,12 +32,85 @@ struct file_operations nvdebug_read_reg32_file_ops = {
         .llseek = default_llseek,
 };
 
+typedef union {
+        struct {
+                uint8_t partitioning_select:2;
+                uint8_t table_select:2;
+                 uint32_t pad_1:12;
+                uint8_t veid_offset:6;
+                 uint32_t pad_2:2;
+                uint8_t table_offset:6;
+                 uint32_t pad_3:2;
+        };
+        uint32_t raw;
+} partition_ctl_t;
+
+static ssize_t nvdebug_read_part(struct file *f, char __user *buf, size_t size, loff_t *off) {
+        char out[12*64+2];
+        int i, chars_written = 0;
+        partition_ctl_t part_ctl;
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
+        if (size < 16 || *off != 0)
+                return 0;
+        // 32 bit register will always take less than 16 characters to print
+        part_ctl.raw = nvdebug_readl(g, 0x00405b2c);
+        //part_ctl.partitioning_select = 0; // XXX XXX XXX Temp; 06/18/2024
+        //part_ctl.table_select = 3; // 3 == ???
+        //part_ctl.table_select = 2; // 2 == TBL_SEL_PARTITIONING_LMEM_BLK
+        part_ctl.table_select = 1; // 1 == TBL_SEL_PARTITIONING_ENABLE
+        //part_ctl.table_select = 0; // 0 == TBL_SEL_NONE
+        part_ctl.veid_offset = (uintptr_t)pde_data(file_inode(f)); // Range of [0, 0x3f], aka [0, 63]
+        for (i = 0; i < 64; i++) {
+                // Increment to next table offset in PARTITION_CTL
+                part_ctl.table_offset = i;
+                nvdebug_writel(g, 0x00405b2c, part_ctl.raw);
+                // Verify write applied to PARTITION_CTL
+                part_ctl.raw = nvdebug_readl(g, 0x00405b2c);
+                if (part_ctl.table_offset != i)
+                        return -ENOTRECOVERABLE;
+                // Read PARTITION_DATA and print
+                // ---
+                // I get back 0x000000ff on Volta and 0x00000003 on Turing from
+                // PARTITION_DATA for all possible VEID_OFFSET, TBL_OFFSET, and TBL_SEL
+                // combinations.
+                // ---
+                // There's a 48-byte (12-word) gap after the address for PARTITION_DATA.
+                // Exploring this on Turing for TBL_SEL_PARTITIONING_ENABLE, VEID 1, 62, and
+                // 63, with CUDA_MPS_ACTIVE_THREAD_PERCENTAGE=5 for constant_cycles_kernel
+                // running under MPS:
+                // +0x0: 0x3
+                // +0x4: 0
+                // +0x8: 0x100
+                // +0xC: 0
+                // +0x10: 0xffffffff
+                // +0x14: 0
+                // +0x18: 0
+                // +0x1C: 0xffffffff
+                // +0x20: 0
+                // +0x24: 0xffffffff
+                // +0x28: 0xffffffff
+                // +0x2C: 0xffffffff
+                chars_written += scnprintf(out + chars_written, 12, "%#010x ", nvdebug_readl(g, 0x00405b30));
+        }
+        chars_written += scnprintf(out + chars_written, 2, "\n");
+        if (copy_to_user(buf, out, chars_written))
+                printk(KERN_WARNING "Unable to copy all data for %s\n", file_dentry(f)->d_name.name);
+        *off += chars_written;
+        return chars_written;
+}
+
+struct file_operations nvdebug_read_part_file_ops = {
+        .read = nvdebug_read_part,
+        .llseek = default_llseek,
+};
+
 static ssize_t nvdebug_reg_range_read(struct file *f, char __user *buf, size_t size, loff_t *off) {
         char out[12];
         int chars_written;
         uint32_t read, mask;
         struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
-        // See comment in nvdebug_entry.c to understand `union reg_range`
+        // `start_bit` is included, `stop_bit` is not, so to print lower eight bits
+        // from a register, use `start_bit = 0` and `stop_bit = 8`.
         union reg_range range;
         range.raw = (uintptr_t)pde_data(file_inode(f));
 
@@ -47,7 +120,7 @@ static ssize_t nvdebug_reg_range_read(struct file *f, char __user *buf, size_t s
 
         // Print bits `start_bit` to `stop_bit` from 32 bits at address `offset`
         if ((read = nvdebug_readl(g, range.offset)) == -1)
-                return -EOPNOTSUPP;
+                return -EIO;
         // Setup `mask` used to throw out unused upper bits
         mask = -1u >> (32 - range.stop_bit + range.start_bit);
         // Throw out unused lower bits via a shift, apply the mask, and print
diff --git a/mmu.c b/mmu.c
index ababef5..e2b9a91 100644
--- a/mmu.c
+++ b/mmu.c
@@ -1,9 +1,13 @@
 /* Copyright 2024 Joshua Bakita
  * Helpers to deal with NVIDIA's MMU and associated page tables
  */
+#include <linux/dma-mapping.h>  // dma_map_page() and dma_unmap_page()
 #include <linux/err.h>  // ERR_PTR() etc.
+#include <linux/gfp.h>  // alloc_pages()
 #include <linux/iommu.h>  // iommu_get_domain_for_dev() and iommu_iova_to_phys()
 #include <linux/kernel.h>  // Kernel types
+#include <linux/list.h>  // struct list_head and associated functions
+#include <linux/mm.h>  // put_page()
 
 #include "nvdebug.h"
 
@@ -15,6 +19,11 @@ int g_verbose = 0;
 #define printk_debug if (g_verbose >= 2) printk
 #define printk_info  if (g_verbose >= 1) printk
 
+// At least map_page_directory() assumes that pages are 4 KiB
+#if PAGE_SIZE != 4096
+#error nvdebug assumes and requires a 4 KiB page size.
+#endif
+
 /* Convert a page directory (PD) pointer and aperture to be kernel-accessible
 
   I/O MMU handling inspired by amdgpu_iomem_read() in amdgpu_ttm.c of the
@@ -22,7 +31,8 @@ int g_verbose = 0;
 
   @param addr  Pointer from page directory entry (PDE)
   @param pd_ap PD-type aperture (target address space) for `addr`
-  @return A dereferencable kernel address, or an ERR_PTR-wrapped error
+  @return A dereferencable kernel address, 0 if an I/O MMU is in use and has
+          no available mapping for the bus address, or an ERR_PTR-wrapped error
  */