path: root/mmu.c

/* Copyright 2024 Joshua Bakita
 * Helpers to deal with NVIDIA's MMU and associated page tables
 */
#include <linux/err.h>  // ERR_PTR() etc.
#include <linux/iommu.h>  // iommu_get_domain_for_dev() and iommu_iova_to_phys()
#include <linux/kernel.h>  // Kernel types

#include "nvdebug.h"

// Uncomment to print every PDE and PTE walked for debugging
//#define DEBUG
#ifdef DEBUG
#define printk_debug printk
#else
#define printk_debug(...)
#endif

/* Convert a page directory (PD) pointer and aperture to be kernel-accessible

  I/O MMU handling inspired by amdgpu_iomem_read() in amdgpu_ttm.c of the
  AMDGPU driver.

  @param addr  Pointer from page directory entry (PDE)
  @param pd_ap PD-type aperture (target address space) for `addr`
  @return A dereferencable kernel address, or an ERR_PTR-wrapped error
 */
void __iomem *pd_deref(struct nvdebug_state *g, uintptr_t addr, enum PD_TARGET pd_ap) {
	struct iommu_domain *dom;
	phys_addr_t phys;

	// Validate arguments
	if (unlikely(!IS_PD_TARGET(pd_ap) || pd_ap == PD_AND_TARGET_INVALID || !addr))
		return ERR_PTR(-EINVAL);

	// VID_MEM accesses are the simple common-case
	if (pd_ap == PD_AND_TARGET_VID_MEM) {
		// Using BAR2 requires a page-table traversal. As this function is part
		// of the page-table traversal process, it must instead use PRAMIN.
		int off = addr_to_pramin_mut(g, addr, TARGET_VID_MEM);
		if (off < 0)
			return ERR_PTR(off);
		return g->regs + NV_PRAMIN + off;
	}
	/* SYS_MEM accesses are rare. Only nvgpu (Jetson driver), nouveau, and this
	 * driver are known to create page directory entries in SYS_MEM.
	 *
	 * On systems using an I/O MMU, or some other I/O virtual address space,
	 * these are **not** physical addresses, and must first be translated
	 * through the I/O MMU before use.
	 * Example default meaning of a SYS_MEM address for a few CPUs:
	 * - Jetson Xavier : physical address
	 * - AMD 3950X     : I/O MMU address
	 * - Phenom II x4  : physical address
	 */
	// Check for, and translate through, the I/O MMU (if any)
	if ((dom = iommu_get_domain_for_dev(g->dev))) {
		phys = iommu_iova_to_phys(dom, addr);
		printk(KERN_ERR "[nvdebug] I/O MMU translated SYS_MEM I/O VA %#lx to physical address %llx.\n", addr, phys);
	} else
		phys = addr;

	if (!phys)
		return 0;

	return phys_to_virt(addr);
}

// Internal helper for search_page_directory().
uint64_t search_page_directory_subtree(struct nvdebug_state *g,
				       uintptr_t pde_addr,
				       enum PD_TARGET pde_target,
				       uint64_t addr_to_find,
				       uint32_t level) {
	uint64_t res, i;
	void __iomem *pde_kern;
	page_dir_entry_t entry;
	if (level > sizeof(NV_MMU_PT_V2_SZ))
		return 0;
	// Hack to workaround PDE0 being double-size and strangely formatted
	if (NV_MMU_PT_V2_ENTRY_SZ[level] == 16)
		pde_addr += 8;
	// Translate a VID_MEM/SYS_MEM-space address to something kernel-accessible
	pde_kern = pd_deref(g, pde_addr, pde_target);
	if (IS_ERR_OR_NULL(pde_kern)) {
		printk(KERN_ERR "[nvdebug] %s: Unable to resolve %#lx in GPU %s to a kernel-accessible address. Error %ld.\n", __func__, pde_addr, pd_target_to_text(pde_target), PTR_ERR(pde_kern));
		return 0;
	}
	// Read the page directory entry (a pointer to another directory, or a PTE)
	entry.raw_w = readq(pde_kern);
	// If we reached an invalid (unpopulated) PDE, walk back up the tree
	if (entry.target == PD_AND_TARGET_INVALID)
		return 0;
	// Succeed when we reach a PTE with the address we want
	if (entry.is_pte) {
		// TODO: Handle huge pages here
		printk_debug(KERN_INFO "[nvdebug] PTE for phy addr %#018llx, ap '%s', vol '%d', priv '%d', ro '%d', no_atomics '%d' (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, entry.is_privileged, entry.is_readonly, entry.atomics_disabled, entry.raw_w);
		return (uint64_t)entry.addr << 12 == addr_to_find;
	}
	printk_debug(KERN_INFO "[nvdebug] Found PDE pointing to %#018llx in ap '%s' vol '%d' at lvl %d (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, level, entry.raw_w);
	// Depth-first search of the page table
	for (i = 0; i < NV_MMU_PT_V2_SZ[level + 1]; i++) {
		uint64_t next = ((uint64_t)entry.addr << 12) + NV_MMU_PT_V2_ENTRY_SZ[level + 1] * i;
		res = search_page_directory_subtree(g, next, entry.target, addr_to_find, level + 1);
		if (res)
			return res | (i << NV_MMU_PT_V2_LSB[level + 1]);
	}
	return 0;
}

/* GPU Physical address -> Virtual address ("reverse" translation) for V2 tables

  Depth-first search a page directory of the GPU MMU for where a particular
  physical address is mapped. Upon finding a mapping, the virtual address is
  returned.

  The page directory may be located in VID_MEM, SYS_MEM, or some combination of
  the two.

  @param pd_config    Page Directory configuration, containing pointer and
                      aperture for the start of the PDE3 entries
  @param addr_to_find Physical address to reconstruct the virtual address of
  @return 0 on error, otherwise the virtual address at which addr_to_find is
          mapped into by this page table. (Zero is not a valid virtual address)
*/
uint64_t search_page_directory(struct nvdebug_state *g,
			       page_dir_config_t pd_config,
			       uint64_t addr_to_find) {
	uint64_t res, i;
	// Make sure that the query is page-aligned
	if (addr_to_find & 0xfff) {
		printk(KERN_WARNING "[nvdebug] Attempting to search for unaligned address %llx in search_page_directory()!\n", addr_to_find);
		return 0;
	}
	printk(KERN_INFO "[nvdebug] Searching for addr %#018llx in page table with base %#018lx\n", addr_to_find, (uintptr_t)pd_config.page_dir << 12);
	// Search the top-level page directory (PDE3)
	for (i = 0; i < NV_MMU_PT_V2_SZ[0]; i++)
		if ((res = search_page_directory_subtree(g, ((uintptr_t)pd_config.page_dir << 12) + NV_MMU_PT_V2_ENTRY_SZ[0] * i, INST2PD_TARGET(pd_config.target), addr_to_find, 0)))
			return (res & ~0xfff) | (i << NV_MMU_PT_V2_LSB[0]);
	return 0;
}

/* GPU Physical address -> Virtual address ("reverse" translation) for V1 tables
  (See `search_page_directory()` for documentation.)
 */
uint64_t search_v1_page_directory(struct nvdebug_state *g,
				  page_dir_config_t pd_config,
				  uint64_t addr_to_find) {
	uint64_t j, i = 0;
	page_dir_entry_v1_t pde;
	page_tbl_entry_v1_t pte;
	uintptr_t pte_offset, pde_offset;
	void __iomem *pte_addr, *pde_addr;
	// For each PDE
	do {
		// Index the list of page directory entries
		pde_offset = ((uint64_t)pd_config.page_dir << 12) + i * sizeof(page_dir_entry_v1_t);
		// Convert the VID_MEM/SYS_MEM address to a kernel-accessible addr
		pde_addr = pd_deref(g, pde_offset, INST2PD_TARGET(pd_config.target));
		if (IS_ERR_OR_NULL(pde_addr)) {
			printk(KERN_ERR "[nvdebug] %s: Unable to resolve %#lx in GPU %s to a kernel-accessible address. Error %ld.\n", __func__, pde_offset, pd_target_to_text(INST2PD_TARGET(pd_config.target)), -PTR_ERR(pde_addr));
			return 0;
		}
		// readq doesn't seem to work on BAR0
		pde.raw = readl(pde_addr + 4);
		pde.raw <<= 32;
		pde.raw |= readl(pde_addr);
		// Verify PDE is present
		if (pde.target == PD_TARGET_INVALID && pde.alt_target == PD_TARGET_INVALID)
			continue;
//		printk(KERN_INFO "[nvdebug] Found %s PDE pointing to PTEs @ %llx in ap '%d' (raw: %llx)\n", pde.is_volatile ? "volatile" : "non-volatile", ((u64)pde.addr) << 12, pde.target, pde.raw);
		printk_debug(KERN_INFO "[nvdebug] Found %s PDE pointing to PTEs @ %llx in ap '%d' (raw: %llx)\n", pde.alt_is_volatile ? "volatile" : "non-volatile", ((u64)pde.alt_addr) << 12, pde.alt_target, pde.raw);
		// For each PTE
		for (j = 0; j < NV_MMU_PT_V1_SZ[1]; j++) {
			// Index the list of page table entries starting at pde.alt_addr
			pte_offset = ((uint64_t)pde.alt_addr << 12) + j * sizeof(page_tbl_entry_v1_t);
			// Convert the VID_MEM/SYS_MEM address to a kernel-accessible addr
			pte_addr = pd_deref(g, pte_offset, V12PD_TARGET(pde.alt_target));
			if (IS_ERR_OR_NULL(pte_addr)) {
				printk(KERN_ERR "[nvdebug] %s: Unable to resolve %#lx in GPU %s to a kernel-accessible address. Error %ld.\n", __func__, pte_offset, pd_target_to_text(V12PD_TARGET(pde.alt_target)), -PTR_ERR(pte_addr));
				return 0;
			}
			// Read page table entry, avoiding readq
			pte.raw = readl(pte_addr + 4);
			pte.raw <<= 32;
			pte.raw |= readl(pte_addr);
			// Skip non-present PTEs
			if (!pte.is_present)
				continue;
			printk_debug(KERN_INFO "[nvdebug] PTE for phy addr %llx %s (raw: %llx)\n", ((u64)pte.addr) << 12, pte.is_present ? "present" : "non-present", pte.raw);
			// If we find a matching PTE, return its virtual address
			if ((uint64_t)pte.addr << 12 == addr_to_find)
				return i << NV_MMU_PT_V1_LSB[0] | j << NV_MMU_PT_V1_LSB[1];
		}
	} while (++i < NV_MMU_PT_V1_SZ[0]);
	return 0;
}

/* *** UNTESTED ***
#define NV_MMU_PT_V0_SZ 2048
#define NV_MMU_PT_V0_LSB 29
uint64_t search_v0_page_directory(struct nvdebug_state *g,
				  void __iomem *pde_offset,
				  void __iomem *(*off2addr)(struct nvdebug_state*, uint32_t),
				  uint32_t addr_to_find) {
	int j, i = 0;
	page_dir_entry_v0_t pde;
	page_tbl_entry_v0_t pte;
	void __iomem *pte_offset;
	// For each PDE
	do {
		// readq doesn't seem to work on BAR0
		pde.raw = readl(pde_offset + i * sizeof(page_dir_entry_v0_t) + 4);
		pde.raw <<= 32;
		pde.raw |= readl(pde_offset + i * sizeof(page_dir_entry_v0_t));
		//if (pde.raw)
		//printk(KERN_INFO "[nvdebug] Read raw PDE @ %x: %llx\n", pde_offset + i * sizeof(page_dir_entry_v1_t), pde.raw);
		// Skip unpopulated PDEs
		if (pde.type == NOT_PRESENT)
			continue;
		//printk(KERN_INFO "[nvdebug] PDE to %llx present\n", ((uint64_t)pde.addr) << 12);
		pte_offset = off2addr(g, ((uint64_t)pde.addr) << 12);
		// For each PTE
		for (j = 0; j < V0_PDE_SIZE2NUM[pde.sublevel_size]; j++) {
			pte.raw = readl(pte_offset + j * sizeof(page_tbl_entry_v0_t) + 4);
			pte.raw <<= 32;
			pte.raw |= readl(pte_offset + j * sizeof(page_tbl_entry_v0_t));
			// Skip non-present PTEs
			if (!pte.is_present)
				continue;
			// If we find a matching PTE, return its virtual address
			//if (pte.addr != 0x5555555)
			//	printk(KERN_INFO "[nvdebug] PTE for phy addr %llx %s\n", ((uint64_t)pte.addr) << 12, pte.is_present ? "present" : "non-present");
			if (pte.addr << 12 == addr_to_find)
				return i << NV_MMU_PT_V0_LSB | j << 12;
		}
	} while (++i < NV_MMU_PT_V0_SZ);
	return 0;  // No match
}
*/