From 293430fcb5d4013b573556c58457ee706e482b7f Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Mon, 5 May 2025 03:53:01 -0400 Subject: Snapshot for ECRTS'25 artifact evaluation --- device_info_procfs.c | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 3 deletions(-) (limited to 'device_info_procfs.c') diff --git a/device_info_procfs.c b/device_info_procfs.c index 4e4ab03..105e731 100644 --- a/device_info_procfs.c +++ b/device_info_procfs.c @@ -18,7 +18,7 @@ static ssize_t nvdebug_reg32_read(struct file *f, char __user *buf, size_t size, return 0; if ((read = nvdebug_readl(g, (uintptr_t)pde_data(file_inode(f)))) == -1) - return -EOPNOTSUPP; + return -EIO; // 32 bit register will always take less than 16 characters to print chars_written = scnprintf(out, 16, "%#0x\n", read); if (copy_to_user(buf, out, chars_written)) @@ -32,12 +32,85 @@ struct file_operations nvdebug_read_reg32_file_ops = { .llseek = default_llseek, }; +typedef union { + struct { + uint8_t partitioning_select:2; + uint8_t table_select:2; + uint32_t pad_1:12; + uint8_t veid_offset:6; + uint32_t pad_2:2; + uint8_t table_offset:6; + uint32_t pad_3:2; + }; + uint32_t raw; +} partition_ctl_t; + +static ssize_t nvdebug_read_part(struct file *f, char __user *buf, size_t size, loff_t *off) { + char out[12*64+2]; + int i, chars_written = 0; + partition_ctl_t part_ctl; + struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)]; + if (size < 16 || *off != 0) + return 0; + // 32 bit register will always take less than 16 characters to print + part_ctl.raw = nvdebug_readl(g, 0x00405b2c); + //part_ctl.partitioning_select = 0; // XXX XXX XXX Temp; 06/18/2024 + //part_ctl.table_select = 3; // 3 == ??? + //part_ctl.table_select = 2; // 2 == TBL_SEL_PARTITIONING_LMEM_BLK + part_ctl.table_select = 1; // 1 == TBL_SEL_PARTITIONING_ENABLE + //part_ctl.table_select = 0; // 0 == TBL_SEL_NONE + part_ctl.veid_offset = (uintptr_t)pde_data(file_inode(f)); // Range of [0, 0x3f], aka [0, 63] + for (i = 0; i < 64; i++) { + // Increment to next table offset in PARTITION_CTL + part_ctl.table_offset = i; + nvdebug_writel(g, 0x00405b2c, part_ctl.raw); + // Verify write applied to PARTITION_CTL + part_ctl.raw = nvdebug_readl(g, 0x00405b2c); + if (part_ctl.table_offset != i) + return -ENOTRECOVERABLE; + // Read PARTITION_DATA and print + // --- + // I get back 0x000000ff on Volta and 0x00000003 on Turing from + // PARTITION_DATA for all possible VEID_OFFSET, TBL_OFFSET, and TBL_SEL + // combinations. + // --- + // There's a 48-byte (12-word) gap after the address for PARTITION_DATA. + // Exploring this on Turing for TBL_SEL_PARTITIONING_ENABLE, VEID 1, 62, and + // 63, with CUDA_MPS_ACTIVE_THREAD_PERCENTAGE=5 for constant_cycles_kernel + // running under MPS: + // +0x0: 0x3 + // +0x4: 0 + // +0x8: 0x100 + // +0xC: 0 + // +0x10: 0xffffffff + // +0x14: 0 + // +0x18: 0 + // +0x1C: 0xffffffff + // +0x20: 0 + // +0x24: 0xffffffff + // +0x28: 0xffffffff + // +0x2C: 0xffffffff + chars_written += scnprintf(out + chars_written, 12, "%#010x ", nvdebug_readl(g, 0x00405b30)); + } + chars_written += scnprintf(out + chars_written, 2, "\n"); + if (copy_to_user(buf, out, chars_written)) + printk(KERN_WARNING "Unable to copy all data for %s\n", file_dentry(f)->d_name.name); + *off += chars_written; + return chars_written; +} + +struct file_operations nvdebug_read_part_file_ops = { + .read = nvdebug_read_part, + .llseek = default_llseek, +}; + static ssize_t nvdebug_reg_range_read(struct file *f, char __user *buf, size_t size, loff_t *off) { char out[12]; int chars_written; uint32_t read, mask; struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)]; - // See comment in nvdebug_entry.c to understand `union reg_range` + // `start_bit` is included, `stop_bit` is not, so to print lower eight bits + // from a register, use `start_bit = 0` and `stop_bit = 8`. union reg_range range; range.raw = (uintptr_t)pde_data(file_inode(f)); @@ -47,7 +120,7 @@ static ssize_t nvdebug_reg_range_read(struct file *f, char __user *buf, size_t s // Print bits `start_bit` to `stop_bit` from 32 bits at address `offset` if ((read = nvdebug_readl(g, range.offset)) == -1) - return -EOPNOTSUPP; + return -EIO; // Setup `mask` used to throw out unused upper bits mask = -1u >> (32 - range.stop_bit + range.start_bit); // Throw out unused lower bits via a shift, apply the mask, and print -- cgit v1.2.2