path: root/nvdebug_entry.c

/* Copyright 2024 Joshua Bakita
 * SPDX-License-Identifier: MIT
 */

#include <linux/device.h>  // For struct device, bus_find_device*(), struct bus_type
#include <linux/interrupt.h> // For hooking the nvidia driver interrupts
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>  // For PCI device scanning
#include <linux/platform_device.h>  // For platform_device struct
#include <linux/proc_fs.h>  // So we can set up entries in /proc

#include "nvdebug_linux.h"
#include "stubs.h"

// Enable to intercept and log GPU interrupts. Historically used to benchmark
// interrupt latency.
#define INTERRUPT_DEBUG

// MIT is GPL-compatible. We need to be GPL-compatible for symbols like
// platform_bus_type or bus_find_device_by_name...
MODULE_LICENSE("Dual MIT/GPL");
MODULE_AUTHOR("Joshua Bakita");
MODULE_DESCRIPTION("A scheduling debugging module for NVIDIA GPUs");

// runlist_procfs.c
extern struct file_operations runlist_file_ops;
extern struct file_operations preempt_tsg_file_ops;
extern struct file_operations disable_channel_file_ops;
extern struct file_operations enable_channel_file_ops;
extern struct file_operations wfi_preempt_channel_file_ops;
extern struct file_operations cta_preempt_channel_file_ops;
extern struct file_operations cil_preempt_channel_file_ops;
extern struct file_operations resubmit_runlist_file_ops;
extern struct file_operations preempt_runlist_file_ops;
extern struct file_operations ack_bad_tsg_file_ops;
extern struct file_operations map_mem_chid_file_ops;
extern struct file_operations map_mem_ctxid_file_ops;
extern struct file_operations switch_to_tsg_file_ops;
// device_info_procfs.c
extern struct file_operations device_info_file_ops;
extern struct file_operations nvdebug_read_reg32_file_ops;
extern struct file_operations nvdebug_read_reg_range_file_ops;
extern struct file_operations nvdebug_read_part_file_ops;
extern struct file_operations local_memory_file_ops;
// copy_topology_procfs.c
extern struct file_operations copy_topology_file_ops;

struct nvdebug_state g_nvdebug_state[NVDEBUG_MAX_DEVICES];
unsigned int g_nvdebug_devices = 0;
// Bus types are global symbols in the kernel
extern struct bus_type platform_bus_type;

// Starting in Kernel 5.6, proc_ops is required instead of file_operations.
// As file_operations is larger than proc_ops, we can overwrite the memory
// backing the file_operations struct to follow the proc_ops layout, and then
// cast on newer kernels.
// We use the last byte of the file_operations struct to flag that the memory
// layout has been rearranged.
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)
const struct proc_ops* compat_ops(const struct file_operations* ops) {
	struct proc_ops new_ops = {};
	// Don't re-layout if it's already been done
	if (*((uint8_t*)(ops + 1) - 1))
		return (struct proc_ops*)ops;
	new_ops.proc_open = ops->open;
	new_ops.proc_read = ops->read;
	new_ops.proc_write = ops->write;
	new_ops.proc_lseek = ops->llseek;
	new_ops.proc_release = ops->release;
	memcpy((void*)ops, &new_ops, sizeof(new_ops));
	// Flag re-layout as complete in last byte of structure
	*((uint8_t*)(ops + 1) - 1) = 1;
	return (struct proc_ops*)ops;
}
#else
const struct file_operations* compat_ops(const struct file_operations* ops) {
	return ops;
}
#endif

#ifdef INTERRUPT_DEBUG

void nvdebug_fifo_intr(struct nvdebug_state *g) {
	uint32_t fifo_intr_mask;// = nvdebug_readl(g, 0x02100); // PFIFO_INTR_0
	fifo_intr_mask = nvdebug_readl(g, 0x02100); // PFIFO_INTR_0
	if (fifo_intr_mask & 1 << 0)
		printk(KERN_INFO "[nvdebug]   - Interrupt BIND_ERROR.\n");
	if (fifo_intr_mask & 1 << 1)
		printk(KERN_INFO "[nvdebug]   - Interrupt CTXSW_TIMEOUT.\n");
	if (fifo_intr_mask & 1 << 4)
		printk(KERN_INFO "[nvdebug]   - Interrupt RUNLIST_IDLE.\n");
	if (fifo_intr_mask & 1 << 5)
		printk(KERN_INFO "[nvdebug]   - Interrupt RUNLIST_AND_ENG_IDLE.\n");
	if (fifo_intr_mask & 1 << 6)
		printk(KERN_INFO "[nvdebug]   - Interrupt RUNLIST_ACQUIRE.\n");
	if (fifo_intr_mask & 1 << 7)
		printk(KERN_INFO "[nvdebug]   - Interrupt RUNLIST_ACQUIRE_AND_ENG_IDLE.\n");
	if (fifo_intr_mask & 1 << 8)
		printk(KERN_INFO "[nvdebug]   - Interrupt SCHED_ERROR.\n");
	if (fifo_intr_mask & 1 << 16)
		printk(KERN_INFO "[nvdebug]   - Interrupt CHSW_ERROR.\n");
	if (fifo_intr_mask & 1 << 23)
		printk(KERN_INFO "[nvdebug]   - Interrupt MEMOP_TIMEOUT.\n");
	if (fifo_intr_mask & 1 << 24)
		printk(KERN_INFO "[nvdebug]   - Interrupt LB_ERROR.\n");
	if (fifo_intr_mask & 1 << 25) // OLD; Pascal
		printk(KERN_INFO "[nvdebug]   - Interrupt REPLAYABLE_FAULT_ERROR.\n");
	if (fifo_intr_mask & 1 << 27) // OLD; Pascal
		printk(KERN_INFO "[nvdebug]   - Interrupt DROPPED_MMU_FAULT.\n");
	if (fifo_intr_mask & 1 << 28) { // On Pascal, this is MMU_FAULT
		if (g->chip_id <= NV_CHIP_ID_VOLTA) // MMU_FAULT on Pascal (nvgpu, l4t/l4t-r28.1:drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_fifo_gp10b.h)
			printk(KERN_INFO "[nvdebug]   - Interrupt MMU_FAULT.\n");
		else // Repurposed starting with Turing: open-gpu-doc/manuals/turing/tu104/dev_fifo.ref.txt
			printk(KERN_INFO "[nvdebug]   - Interrupt TSG_PREEMPT_COMPLETE.\n");
	}
	if (fifo_intr_mask & 1 << 29)
		printk(KERN_INFO "[nvdebug]   - Interrupt PBDMA_INTR.\n");
	if (fifo_intr_mask & 1 << 30) {
		printk(KERN_INFO "[nvdebug]   - Interrupt RUNLIST_EVENT.\n");
		uint32_t fifo_runlist_intr_mask = nvdebug_readl(g, 0x02A00); // PFIFO_INTR_RUNLIST
		printk(KERN_INFO "[nvdebug]     - Event %#x.\n", fifo_runlist_intr_mask);
	}
	if (fifo_intr_mask & 1 << 31)
		printk(KERN_INFO "[nvdebug]   - Interrupt CHANNEL_INTR.\n");
}

irqreturn_t nvdebug_irq_tap(int irq_num, void * dev) {
	struct nvdebug_state *g = dev;
	u64 time = ktime_get_raw_ns(); // CLOCK_MONOTONTIC_RAW
	// NV_PMC_INTR does not exist on Ada, so use NV_FUNC_PRIV_CPU_INTR_TOP
	// Note that this also appears to exist on Turing
	if (g->chip_id >= NV_CHIP_ID_TURING) {//AMPERE) {
		int i;
		// Despite being an indexed register, it is only documented to have on, and could only support two
		uint32_t intr_mask0 = nvdebug_readl(g, NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET + 0x1600); // NV_FUNC_PRIV_CPU_INTR_TOP(0)
		uint32_t intr_mask1 = nvdebug_readl(g, NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET + 0x1604); // NV_FUNC_PRIV_CPU_INTR_TOP(1)
		printk(KERN_INFO "[nvdebug] Interrupt on IRQ %d with CPU_INTR_TOP(0) %#010x, ...(1) %#010x @ %llu.\n", irq_num, intr_mask0, intr_mask1, time);
		for (i = 0; i < 8; i++) {
			uint32_t leaf = nvdebug_readl(g, NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET + 0x1000 + i*4); // NV_FUNC_PRIV_CPU_INTR_LEAF(0) to ...(7)
			if (leaf)
				printk(KERN_INFO "[nvdebug] - Interrupt leaf %d: %#010x\n", i, leaf);
			// 131-133 & 64 are faults on tu104??? (open-gpu-doc/manuals/turing/tu104/pri_mmu_hub.ref.txt)
			if (136 / 32 == i && 1 << (136 % 32) & leaf) // PFIFO0 ga100
				printk(KERN_INFO "[nvdebug] - Interrupt on PFIFO0.\n");
			if (137 / 32 == i && 1 << (137 % 32) & leaf) // PFIFO1 ga100
				printk(KERN_INFO "[nvdebug] - Interrupt on PFIFO1.\n");
			if (148 / 32 == i && 1 << (148 % 32) & leaf) // TIMER ga100
				printk(KERN_INFO "[nvdebug] - Interrupt on PTIMER.\n");
			if (152 / 32 == i && 1 << (152 % 32) & leaf) // PMU ga100
				printk(KERN_INFO "[nvdebug] - Interrupt on PMU.\n");
			if (156 / 32 == i && 1 << (156 % 32) & leaf) { // PBUS ga100
				printk(KERN_INFO "[nvdebug] - Interrupt on PBUS.\n");
				uint32_t bus_intr = nvdebug_readl(g, 0x1100); // BUS_INTR_0
				if (bus_intr & 1 << 2) {
					// use timer_pri_timeout_save_0_r
					uint32_t SAVE_0 = nvdebug_readl(g, 0x00009084); // NV_PTIMER_PRI_TIMEOUT_SAVE_0
					printk(KERN_INFO "[nvdebug]   - Interrupt PRI_FECSERR on %s to address %#010x %stargeting FECS.\n", SAVE_0 & 0x2 ? "write" : "read", SAVE_0 & 0x00fffffc, SAVE_0 & 0x80000000 ? "" : "not ");
					uint32_t SAVE_1 = nvdebug_readl(g, 0x00009088); // NV_PTIMER_PRI_TIMEOUT_SAVE_1
					if (SAVE_1)
						printk(KERN_INFO "[nvdebug]     Data written: %#010x\n", SAVE_1);
					uint32_t errcode = readl(g->regs + 0x0000908C); // NV_PTIMER_PRI_TIMEOUT_FECS_ERRCODE
					if (errcode)
						printk(KERN_INFO "[nvdebug]     FECS Error Code: %#010x\n", errcode);
					// badf5040 is a "client error" (0) of "no such address" (40)
					// See linux-nvgpu/drivers/gpu/nvgpu/hal/priv_ring/priv_ring_ga10b_fusa.c
					// for how to decode.
				}
				if (bus_intr & 1 << 3)
					printk(KERN_INFO "[nvdebug]   - Interrupt PRI_TIMEOUT.\n");
				if (bus_intr & 1 << 4)
					printk(KERN_INFO "[nvdebug]   - Interrupt FB_REQ_TIMEOUT.\n");
				if (bus_intr <