Quick dump of current state for Ben to review.

1 files changed, 253 insertions, 35 deletions

diff --git a/nvdebug_entry.c b/nvdebug_entry.c
index 0854b8b..695b5fd 100644
--- a/nvdebug_entry.c
+++ b/nvdebug_entry.c
@@ -2,64 +2,282 @@
  * SPDX-License-Identifier: MIT
  */
 
-/* TODO
- * - Add sysfs trigger for a preemption
- */
-
 #include <linux/device.h>  // For struct device, bus_find_device*(), struct bus_type
+#include <linux/interrupt.h> // For hooking the nvidia driver interrupts
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/proc_fs.h> // So we can set up entries in /proc
+#include <linux/pci.h>  // For PCI device scanning
+#include <linux/proc_fs.h>  // So we can set up entries in /proc
 
 #include "nvdebug.h"
+#include "stubs.h"
 
-// LIAR. But without this we can't use GPL-only exported symbols like
+// MIT is GPL-compatible. We need to be GPL-compatible for symbols like
 // platform_bus_type or bus_find_device_by_name...
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("Dual MIT/GPL");
 MODULE_AUTHOR("Joshua Bakita");
 MODULE_DESCRIPTION("A scheduling debugging module for NVIDIA GPUs");
-MODULE_SOFTDEP("pre: nvgpu"); // We only support the Jetson boards for now
 
 extern const struct file_operations runlist_file_ops;
 extern const struct file_operations preempt_tsg_file_ops;
 extern const struct file_operations disable_channel_file_ops;
 extern const struct file_operations enable_channel_file_ops;
 extern const struct file_operations switch_to_tsg_file_ops;
+extern const struct file_operations device_info_file_ops;
+extern const struct file_operations nvdebug_read_reg32_file_ops;
+
+// Bus types are global symbols in the kernel
+extern struct bus_type platform_bus_type;
+struct nvdebug_state g_nvdebug_state[NVDEBUG_MAX_DEVICES];
+unsigned int g_nvdebug_devices = 0;
+
+// TEMP
+irqreturn_t nvdebug_irq_tap(int irq_num, void * dev) {
+        printk(KERN_INFO "[nvdebug] Interrupt tap triggered on IRQ %d.\n", irq_num);
+        return IRQ_NONE; // We don't actually handle any interrupts. Pass them on.
+}
+
+// Find any and all NVIDIA GPUs in the system
+// Note: This function fails if any of them are in a bad state
+int probe_and_cache_device(void) {
+        // platform bus (SoC) iterators
+        struct device *dev = NULL;
+        struct device *temp_dev;
+        // PCI search iterator and search query
+        struct pci_dev *pcid = NULL;
+        // This query pattern is mirrored off nouveau
+        struct pci_device_id query = {
+                .vendor = NV_PCI_VENDOR,  // Match NVIDIA devices
+                .device = PCI_ANY_ID,
+                .subvendor = PCI_ANY_ID,
+                .subdevice = PCI_ANY_ID,
+                .class_mask = 0xff << 16,
+                .class = PCI_BASE_CLASS_DISPLAY << 16,  // Match display devs
+        };
+        int i = 0;
+        // Search the platform bus for the first device that matches our name
+        // Search for GV10B (Jetson Xavier)
+        while (!dev && (temp_dev = bus_find_device_by_name(&platform_bus_type, dev, "17000000.gv11b")))
+                dev = temp_dev;
+        // Search for GP10B (Jetson TX2)
+        while (!dev && (temp_dev = bus_find_device_by_name(&platform_bus_type, dev, "17000000.gp10b")))
+                dev = temp_dev;
+        // TODO: Support other platform bus devices (gk20a, gm20b)
+        if (dev) {
+                struct nvgpu_os_linux *l;
+                mc_boot_0_t ids;
+                g_nvdebug_state[i].g = get_gk20a(dev);
+                l = container_of(g_nvdebug_state[i].g, struct nvgpu_os_linux, g);
+                g_nvdebug_state[i].regs = l->regs;
+                if (!g_nvdebug_state[i].regs)
+                        return -EADDRNOTAVAIL;
+                ids.raw = nvdebug_readl(&g_nvdebug_state[i], NV_MC_BOOT_0);
+                if (ids.raw == -1)
+                        return -EADDRNOTAVAIL;
+                g_nvdebug_state[i].chip_id = ids.chip_id;
+                printk(KERN_INFO "[nvdebug] Chip ID %x (architecture %s) detected on platform bus and initialized.",
+                       ids.chip_id, ARCH2NAME(ids.architecture));
+                i++;
+        }
+        // Search the PCI bus and iterate through all matches
+        // FIXME: State rollback
+        while ((pcid = pci_get_dev_by_id(&query, pcid)) && i < NVDEBUG_MAX_DEVICES) {
+                mc_boot_0_t ids;
+                g_nvdebug_state[i].g = NULL;
+                // Map BAR0 (GPU control registers)
+                g_nvdebug_state[i].regs = pci_iomap(pcid, 0, 0);
+                if (!g_nvdebug_state[i].regs) {
+                        pci_err(pcid, "[nvdebug] Unable to map BAR0 on this GPU\n");
+                        return -EADDRNOTAVAIL;
+                }
+                // Map BAR3 (CPU-accessible mappings of GPU DRAM)
+                g_nvdebug_state[i].bar3 = pci_iomap(pcid, 3, 0);
+                // Try mapping only the lower half of BAR3 on fail
+                // (vesafb may map the top half for display)
+                if (!g_nvdebug_state[i].bar3)
+                        g_nvdebug_state[i].bar3 = pci_iomap(pcid, 3, pci_resource_len(pcid, 3)/2);
+                g_nvdebug_state[i].pcid = pcid;
+                ids.raw = nvdebug_readl(&g_nvdebug_state[i], NV_MC_BOOT_0);
+                if (ids.raw == -1) {
+                        pci_err(pcid, "[nvdebug] Unable to read config from Master Controller on this GPU\n");
+                        return -EADDRNOTAVAIL;
+                }
+                g_nvdebug_state[i].chip_id = ids.chip_id;
+                printk(KERN_INFO "[nvdebug] Chip ID %x (architecture %s) detected on PCI bus and initialized.",
+                       ids.chip_id, ARCH2NAME(ids.architecture));
+                // TEMP
+                if (request_irq(pcid->irq, nvdebug_irq_tap, IRQF_SHARED, "nvdebug tap", pcid)) {
+                        printk(KERN_WARNING "[nvdebug] Unable to initialize IRQ tap\n");
+                }
+                i++;
+        }
+        // Return the number of devices we found
+        if (i > 0)
+                return i;
+        return -ENODEV;
+}
+
+// Create files `/proc/gpu#/runlist#`, world readable
+int create_runlist_files(int device_id, struct proc_dir_entry *dir) {
+        ptop_device_info_t info;
+        struct proc_dir_entry *rl_entry;
+        int i, rl_id;
+        char runlist_name[12];
+        int max_rl_id = 0; // Always at least one runlist
+        // Figure out how many runlists there are by checking the device info
+        // registers. Runlists are always numbered sequentially, so we just have
+        // to find the highest-valued one and add 1 to get the number of runlists.
+        for (i = 0; i < NV_PTOP_DEVICE_INFO__SIZE_1; i++) {
+                info.raw = nvdebug_readl(&g_nvdebug_state[device_id], NV_PTOP_DEVICE_INFO(i));
+                if (info.info_type != INFO_TYPE_ENUM || !info.runlist_is_valid)
+                        continue;
+                if (info.runlist_enum > max_rl_id)
+                        max_rl_id = info.runlist_enum;
+        }
+        // Create files to read each runlist. The read handling code looks at the
+        // PDE_DATA associated with the file to determine what the runlist ID is.
+        for (rl_id = 0; rl_id <= max_rl_id; rl_id++) {
+                snprintf(runlist_name, 12, "runlist%d", rl_id);
+                rl_entry = proc_create_data(
+                        runlist_name, 0444, dir, &runlist_file_ops,
+                        (void*)(uintptr_t)rl_id);
+                if (!rl_entry)
+                        return -ENOMEM;
+        }
+        return 0;
+}
+
+// Create files /proc/gpu#
+// TODO: Don't run this on unsupported GPUs
+int create_tpc_mask_files(int device_id, struct proc_dir_entry *dir) {
+        char file_name[20];
+        int i;
+        struct proc_dir_entry *gpc_tpc_mask_entry;
+        // Get a bitmask of which GPCs are disabled
+        uint32_t gpcs_mask = nvdebug_readl(&g_nvdebug_state[device_id], NV_FUSE_GPC);
+        // Get maximum number of enabled GPCs for this chip
+        uint32_t max_gpcs = nvdebug_readl(&g_nvdebug_state[device_id], NV_PTOP_SCAL_NUM_GPCS);
+        // For each enabled GPC, expose a mask of disabled TPCs
+        for (i = 0; i < max_gpcs; i++) {
+                // Do nothing if GPC is disabled
+                if ((1 << i) & gpcs_mask)
+                        continue;
+                // If GPC is enabled, create an entry to read disabled TPCs mask
+                snprintf(file_name, 20, "gpc%d_tpc_mask", i);
+                gpc_tpc_mask_entry = proc_create_data(
+                        file_name, 0444, dir, &nvdebug_read_reg32_file_ops,
+                        (void*)(uintptr_t)NV_FUSE_TPC_FOR_GPC(i));
+                if (!gpc_tpc_mask_entry)
+                        return -ENOMEM;
+        }
+        return 0;
+}
 
 int __init nvdebug_init(void) {
-        struct proc_dir_entry *rl_entry, *preempt_entry, *disable_channel_entry,
-                              *enable_channel_entry, *switch_to_tsg_entry;
-        // Create file `/proc/preempt_tsg`, world readable
-        rl_entry = proc_create("runlist", 0444, NULL, &runlist_file_ops);
-        // Create file `/proc/preempt_tsg`, world writable
-        preempt_entry = proc_create("preempt_tsg", 0222, NULL, &preempt_tsg_file_ops);
-        // Create file `/proc/disable_channel`, world writable
-        disable_channel_entry = proc_create("disable_channel", 0222, NULL, &disable_channel_file_ops);
-        // Create file `/proc/enable_channel`, world writable
-        enable_channel_entry = proc_create("enable_channel", 0222, NULL, &enable_channel_file_ops);
-        // Create file `/proc/switch_to_tsg`, world writable
-        switch_to_tsg_entry = proc_create("switch_to_tsg", 0222, NULL, &switch_to_tsg_file_ops);
-        // ProcFS entry creation only fails if out of memory
-        if (!rl_entry || !preempt_entry || !disable_channel_entry || !enable_channel_entry || !switch_to_tsg_entry) {
-                remove_proc_entry("runlist", NULL);
-                remove_proc_entry("preempt_tsg", NULL);
-                remove_proc_entry("disable_channel", NULL);
-                remove_proc_entry("enable_channel", NULL);
-                remove_proc_entry("switch_to_tsg", NULL);
-                printk(KERN_ERR "[nvdebug] Unable to initialize procfs entries!\n");
-                return -ENOMEM;
+        struct proc_dir_entry *dir, *preempt_entry, *disable_channel_entry,
+                              *enable_channel_entry, *switch_to_tsg_entry, *device_info_entry,
+                              *num_gpcs_entry;
+        int rl_create_err, tpc_masks_create_err;
+        // Check that an NVIDIA GPU is present and initialize g_nvdebug_state
+        int res = probe_and_cache_device();
+        if (res < 0)
+                return res;
+        g_nvdebug_devices = res;
+        // Create seperate ProcFS directories for each gpu
+        while (res--) {
+                char device_id_str[7];
+                uintptr_t device_id = res;  // This is uintptr as we abuse the *data field on proc_dir_entry to store the GPU id
+                // Create directory /proc/gpu# where # is the GPU number
+                snprintf(device_id_str, 7, "gpu%ld", device_id);
+                if (!(dir = proc_mkdir_data(device_id_str, 0555, NULL, (void*)device_id)))
+                        goto out_nomem;
+                // Create files `/proc/gpu#/runlist#`, world readable
+                rl_create_err = create_runlist_files(device_id, dir);
+                // Create files `/proc/gpu#/gpc#_tpc_mask`, world readable
+                tpc_masks_create_err = create_tpc_mask_files(device_id, dir);
+                // Create file `/proc/gpu#/preempt_tsg`, world writable
+                preempt_entry = proc_create_data(
+                        "preempt_tsg", 0222, dir, &preempt_tsg_file_ops,
+                        (void*)device_id);
+                // Create file `/proc/gpu#/disable_channel`, world writable
+                disable_channel_entry = proc_create_data(
+                        "disable_channel", 0222, dir, &disable_channel_file_ops,
+                        (void*)device_id);
+                // Create file `/proc/gpu#/enable_channel`, world writable
+                enable_channel_entry = proc_create_data(
+                        "enable_channel", 0222, dir, &enable_channel_file_ops,
+                        (void*)device_id);
+                // Create file `/proc/gpu#/switch_to_tsg`, world writable
+                switch_to_tsg_entry = proc_create_data(
+                        "switch_to_tsg", 0222, dir, &switch_to_tsg_file_ops,
+                        (void*)device_id);
+                // Create file `/proc/gpu#/device_info`, world readable
+                device_info_entry = proc_create_data(
+                        "device_info", 0444, dir, &device_info_file_ops,
+                        (void*)device_id);
+                // Create file `/proc/gpu#/num_gpcs`, world readable
+                num_gpcs_entry = proc_create_data(
+                        "num_gpcs", 0444, dir, &nvdebug_read_reg32_file_ops,
+                        (void*)NV_PTOP_SCAL_NUM_GPCS);
+                // Create file `/proc/gpu#/num_tpc_per_gpc`, world readable
+                num_gpcs_entry = proc_create_data(
+                        "num_tpc_per_gpc", 0444, dir, &nvdebug_read_reg32_file_ops,
+                        (void*)NV_PTOP_SCAL_NUM_TPC_PER_GPC);
+                // Create file `/proc/gpu#/num_ces`, world readable
+                num_gpcs_entry = proc_create_data(
+                        "num_ces", 0444, dir, &nvdebug_read_reg32_file_ops,
+                        (void*)NV_PTOP_SCAL_NUM_CES);
+                // Create file `/proc/gpu#/num_ces`, world readable
+                num_gpcs_entry = proc_create_data(
+                        "gpc_mask", 0444, dir, &nvdebug_read_reg32_file_ops,
+                        (void*)NV_FUSE_GPC);
+                // In both nouveau and nvgpu, the PCE_MAP register is only available on Volta+
+                if (g_nvdebug_state[res].chip_id >= NV_CHIP_ID_VOLTA) {
+                        // TODO: Redo to num_pces
+                        // Create file `/proc/gpu#/pce_map`, world readable
+                        num_gpcs_entry = proc_create_data(
+                                "pce_map", 0444, dir, &nvdebug_read_reg32_file_ops,
+                                (void*)NV_CE_PCE_MAP);
+                }
+                // ProcFS entry creation only fails if out of memory
+                if (rl_create_err || tpc_masks_create_err || !preempt_entry ||
+                    !disable_channel_entry || !enable_channel_entry ||
+                    !switch_to_tsg_entry || !device_info_entry || !num_gpcs_entry)
+                        goto out_nomem;
         }
+        // (See Makefile if you want to know the origin of GIT_HASH.)
         printk(KERN_INFO "[nvdebug] Module version "GIT_HASH" initialized\n");
         return 0;
+out_nomem:
+        // Make sure to clear all ProcFS directories on error
+        while (res < g_nvdebug_devices) {
+                char device_id_str[7];
+                snprintf(device_id_str, 7, "gpu%d", res);
+                remove_proc_subtree(device_id_str, NULL);
+                res++;
+        }
+        return -ENOMEM;
 }
 
 static void __exit nvdebug_exit(void) {
-        remove_proc_entry("runlist", NULL);
-        remove_proc_entry("preempt_tsg", NULL);
-        remove_proc_entry("disable_channel", NULL);
-        remove_proc_entry("enable_channel", NULL);
-        remove_proc_entry("switch_to_tsg", NULL);
-        printk(KERN_INFO "[nvdebug] Exiting...\n");
+        struct nvdebug_state *g;
+        // Deinitialize each device
+        while (g_nvdebug_devices--) {
+                // Remove procfs directory
+                char device_id[7];
+                snprintf(device_id, 7, "gpu%d", g_nvdebug_devices);
+                remove_proc_subtree(device_id, NULL);
+                // Free BAR mappings
+                g = &g_nvdebug_state[g_nvdebug_devices];
+                if (g && g->regs)
+                        pci_iounmap(g->pcid, g->regs);
+                if (g && g->bar2)
+                        pci_iounmap(g->pcid, g->bar2);
+                // TEMP
+                free_irq(g->pcid->irq, g->pcid);
+                printk(KERN_INFO "[nvdebug] Chip ID %x deinitialized.", g->chip_id);
+        }
+        printk(KERN_INFO "[nvdebug] Module exit complete.\n");
 }
 
 module_init(nvdebug_init);