path: root/mmu.c

/* Copyright 2024 Joshua Bakita
 * Helpers to deal with NVIDIA's MMU and associated page tables
 */
#include <linux/dma-mapping.h>  // dma_map_page() and dma_unmap_page()
#include <linux/err.h>  // ERR_PTR() etc.
#include <linux/gfp.h>  // alloc_pages()
#include <linux/iommu.h>  // iommu_get_domain_for_dev() and iommu_iova_to_phys()
#include <linux/kernel.h>  // Kernel types
#include <linux/list.h>  // struct list_head and associated functions
#include <linux/mm.h>  // put_page()

#include "nvdebug.h"

/* Set logging level for MMU operations
  g_verbose >= 1: Log a single message describing the MMU operation
  g_verbose >= 2: Log every PDE and PTE traversed
*/
int g_verbose = 0;
#define printk_debug if (g_verbose >= 2) printk
#define printk_info  if (g_verbose >= 1) printk

// At least map_page_directory() assumes that pages are 4 KiB
#if PAGE_SIZE != 4096
#error nvdebug assumes and requires a 4 KiB page size.
#endif

/* Convert a page directory (PD) pointer and aperture to be kernel-accessible

  I/O MMU handling inspired by amdgpu_iomem_read() in amdgpu_ttm.c of the
  AMDGPU driver.

  @param addr  Pointer from page directory entry (PDE)
  @param pd_ap PD-type aperture (target address space) for `addr`
  @return A dereferencable kernel address, 0 if an I/O MMU is in use and has
          no available mapping for the bus address, or an ERR_PTR-wrapped error
 */
static void __iomem *pd_deref(struct nvdebug_state *g, uintptr_t addr,
                              enum PD_TARGET pd_ap) {
	struct iommu_domain *dom;
	phys_addr_t phys;

	// Validate arguments
	if (unlikely(!IS_PD_TARGET(pd_ap) || pd_ap == PD_AND_TARGET_INVALID || !addr))
		return ERR_PTR(-EINVAL);

	// VID_MEM accesses are the simple common-case
	if (pd_ap == PD_AND_TARGET_VID_MEM) {
		// Using BAR2 requires a page-table traversal. As this function is part
		// of the page-table traversal process, it must instead use PRAMIN.
		int off = addr_to_pramin_mut(g, addr, TARGET_VID_MEM);
		if (off < 0)
			return ERR_PTR(off);
		return g->regs + NV_PRAMIN + off;
	}
	/* SYS_MEM accesses are rare. Only nvgpu (Jetson driver), nouveau, and this
	 * driver are known to create page directory entries in SYS_MEM.
	 *
	 * On systems using an I/O MMU, or some other I/O virtual address space,
	 * these are **not** physical addresses, and must first be translated
	 * through the I/O MMU before use.
	 * Example default meaning of a SYS_MEM address for a few CPUs:
	 * - Jetson Xavier : physical address
	 * - AMD 3950X     : I/O MMU address
	 * - Phenom II x4  : physical address
	 */
	// Check for, and translate through, the I/O MMU (if any)
	if ((dom = iommu_get_domain_for_dev(g->dev))) {
		phys = iommu_iova_to_phys(dom, addr);
		printk_debug(KERN_DEBUG "[nvdebug] %s: I/O MMU translated SYS_MEM I/O VA %#lx to physical address %#llx.\n", __func__, addr, phys);
	} else
		phys = addr;

	if (!phys)
		return 0;

	return phys_to_virt(phys);
}

// Internal helper for search_page_directory().
uint64_t search_page_directory_subtree(struct nvdebug_state *g,
                                       uintptr_t pde_addr,
                                       enum PD_TARGET pde_target,
                                       uint64_t addr_to_find,
                                       enum INST_TARGET addr_to_find_aperture,
                                       uint32_t level) {
	uint64_t res, i;
	void __iomem *pde_kern;
	page_dir_entry_t entry;
	if (level > sizeof(NV_MMU_PT_V2_SZ))
		return 0;
	// Hack to workaround PDE0 being double-size and strangely formatted
	if (NV_MMU_PT_V2_ENTRY_SZ[level] == 16)
		pde_addr += 8;
	// Translate a VID_MEM/SYS_MEM-space address to something kernel-accessible
	pde_kern = pd_deref(g, pde_addr, pde_target);
	if (IS_ERR_OR_NULL(pde_kern)) {
		printk(KERN_ERR "[nvdebug] %s: Unable to resolve %#lx in GPU %s to a kernel-accessible address. Error %ld.\n", __func__, pde_addr, pd_target_to_text(pde_target), PTR_ERR(pde_kern));
		return 0;
	}
	// Read the page directory entry (a pointer to another directory, or a PTE)
	entry.raw_w = readq(pde_kern);
	// If we reached an invalid (unpopulated) PDE, walk back up the tree
	if (entry.target == PD_AND_TARGET_INVALID)
		return 0;
	// Succeed when we reach a PTE with the address we want
	if (entry.is_pte) {
		// TODO: Handle huge pages here
		printk_debug(KERN_DEBUG "[nvdebug] PTE for phy addr %#018llx, ap '%s', vol '%d', priv '%d', ro '%d', no_atomics '%d' (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, entry.is_privileged, entry.is_readonly, entry.atomics_disabled, entry.raw_w);
		return (uint64_t)entry.addr << 12 == addr_to_find && entry.aperture == addr_to_find_aperture;
	}
	printk_debug(KERN_DEBUG "[nvdebug] Found PDE pointing to %#018llx in ap '%s' vol '%d' at lvl %d (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, level, entry.raw_w);
	// Depth-first search of the page table
	for (i = 0; i < NV_MMU_PT_V2_SZ[level + 1]; i++) {
		uint64_t next = ((uint64_t)entry.addr << 12) + NV_MMU_PT_V2_ENTRY_SZ[level + 1] * i;
		printk_debug(KERN_DEBUG "[nvdebug] Searching index %llu in lvl %d\n", i, level + 1);
		res = search_page_directory_subtree(g, next, entry.target, addr_to_find, addr_to_find_aperture, level + 1);
		if (res)
			return res | (i << NV_MMU_PT_V2_LSB[level + 1]);
	}
	return 0;
}

/* GPU Physical address -> Virtual address ("reverse" translation) for V2 tables
  Depth-first search a page directory of the GPU MMU for where a particular
  physical address is mapped. Upon finding a mapping, the virtual address is
  returned.

  The page directory and tables may be located in VID_MEM, SYS_MEM, or spread
  across multiple apertures.

  @param pd_config    Page Directory configuration, containing pointer and
                      aperture for the start of the PDE3 entries
  @param addr_to_find Physical address to reconstruct the virtual address of
  @param addr_to_find_aperture Aperture (SYS_MEM or VID_MEM) of addr_to_find
  @return 0 on error, otherwise the virtual address at which addr_to_find is
          mapped into by this page table. (Zero is not a valid virtual address)
*/
uint64_t search_page_directory(struct nvdebug_state *g,
                               page_dir_config_t pd_config,
                               uint64_t addr_to_find,
                               enum INST_TARGET addr_to_find_aperture) {
	uint64_t res, i;
	// Make sure that the query is page-aligned
	if (addr_to_find & 0xfff) {
		printk(KERN_WARNING "[nvdebug] Attempting to search for unaligned address %llx in search_page_directory()!\n", addr_to_find);
		return 0;
	}
	printk_info(KERN_INFO "[nvdebug] Searching for addr %#018llx in page table with base %#018lx\n", addr_to_find, (uintptr_t)pd_config.page_dir << 12);
	// Search the top-level page directory (PDE3)
	for (i = 0; i < NV_MMU_PT_V2_SZ[0]; i++)
		if ((res = search_page_directory_subtree(g, ((uintptr_t)pd_config.page_dir << 12) + NV_MMU_PT_V2_ENTRY_SZ[0] * i, INST2PD_TARGET(pd_config.target), addr_to_find, addr_to_find_aperture, 0)))
			return (res & ~0xfff) | (i << NV_MMU_PT_V2_LSB[0]);
	return 0;
}

/* GPU Virtual address -> Physical address ("forward" translation) for V2 tables
  Index the page directories and tables used by the GPU MMU to determine which
  physical address a given GPU virtual address has been mapped to.

  The page directory and tables may be located in VID_MEM, SYS_MEM, or spread
  across multiple apertures.

  @param pd_config      Page Directory configuration, containing pointer and
                        aperture for the start of the PDE3 entries
  @param addr_to_find   Virtual address to translate to a physical address
  @param found_addr     Where to store found physical address (0 if unfound)
  @param found_aperture Where to store aperture of found physical address
  @return 0 on success, -ENXIO if not found, and -errno on error.
*/
int translate_page_directory(struct nvdebug_state *g,
                             page_dir_config_t pd_config,
                             uint64_t addr_to_find,
                             uint64_t *found_addr /* out */,
                             enum INST_TARGET *found_aperture /* out */) {
	page_dir_entry_t entry;
	void __iomem *next_kva;
	unsigned int level, pde_idx;
	uintptr_t next = (uintptr_t)pd_config.page_dir << 12;
	enum PD_TARGET next_target = INST2PD_TARGET(pd_config.target);

	*found_addr = 0;
	*found_aperture = TARGET_INVALID;

	// Make sure that the query is page-aligned (likely mistake otherwise)
	if (addr_to_find & 0xfff) {
		printk(KERN_WARNING "[nvdebug] Attempting to translate unaligned address %#llx in translate_page_directory()!\n", addr_to_find);
		return -EINVAL;
	}

	printk_info(KERN_INFO "[nvdebug] Translating addr %#018llx in V2 page table with base %#018llx\n", (u64)addr_to_find, (u64)next);

	// Step through each PDE level and the PTE level
	for (level = 0; level < 5; level++) {
		// Index into this level
		pde_idx = (addr_to_find >> NV_MMU_PT_V2_LSB[level]) & (NV_MMU_PT_V2_SZ[level] - 1);
		printk_debug(KERN_DEBUG "[nvdebug] Using index %u in lvl %d\n", pde_idx, level);
		// Hack to workaround PDE0 being double-size and strangely formatted
		if (NV_MMU_PT_V2_ENTRY_SZ[level] == 16)
			next += 8;
		// Obtain a kernel-dereferencable address
		next_kva = pd_deref(g, next, next_target);
		if (IS_ERR_OR_NULL(next_kva)) {
			printk(KERN_ERR "[nvdebug] %s: Unable to resolve %#lx in GPU %s to a kernel-accessible address. Error %ld.\n", __func__, next, pd_target_to_text(next_target), PTR_ERR(next_kva));
			return PTR_ERR(next_kva);
		}
		// Obtain entry at this level
		entry.raw_w = readq(next_kva + NV_MMU_PT_V2_ENTRY_SZ[level] * pde_idx);
		if (entry.target == PD_AND_TARGET_INVALID)
			return -ENXIO;
		printk_debug(KERN_DEBUG "[nvdebug] Found %s pointing to %#018llx in ap '%s' at lvl %d (raw: %#018llx)\n", entry.is_pte ? "PTE" : "PDE", ((u64)entry.addr) << 12, pd_target_to_text(entry.target), level, entry.raw_w);
		// Just return the physical address if this is the PTE level
		if (entry.is_pte) { // level == 4 for 4 KiB pages, == 3 for 2 MiB
			*found_addr = ((uint64_t)entry.addr) << 12;
			*found_aperture = entry.aperture;
			return 0;
		}
		// Otherwise step to the next table level
		// TODO: Use addr_w as appropriate
		next = (uint64_t)entry.addr << 12;
		next_target = entry.target;
	}

	return 0;
}

// This struct is very special. We will never directly allocate this struct;
// its sole purpose is to provide more intuitive names to the offsets at which
// we store data in Linux's struct page. Such (ab)use of struct page is
// explictly permitted (see linux/mm_types.h). This struct is thus used by
// casting a pointer of struct page to a pointer of struct nvdebug_pd_page,
// then accessing the associated fields. This pointer may also be freely cast
// back to a sturct page pointer.
// We have 24 (32-bit) or 44 (64-bit) bytes available in the page struct
// (according to the documentation on struct page). Our comments indicate what
// available parts of struct page we repurpose for our own needs.
struct nvdebug_pd_page {
	unsigned long __flags; // From struct page; do not touch!
	// Overlaps struct page.lru
	struct list_head list; // 4/8 bytes
	// Overlaps struct page.mapping (and page.share on 32-bit)
	uintptr_t parent_addr; // 8 bytes
	// Overlaps struct page.share (page.private on 32-bit)
	enum PD_TARGET parent_aperture; // 4 bytes
	// Overlaps page.private (page.page_type on 32-bit)
	dma_addr_t dma_addr; // 4/8 bytes
};

/* Collect and free any now-unused page directory/table allocations

  @param force Deallocate all page directories/tables created by this module,
               no matter if they appear to be in-use or not.
  @returns Number of freed pages on success, -errno on error.
*/
int gc_page_directory(struct nvdebug_state *g, bool force) {
	struct nvdebug_pd_page  *page, *_page;
	void __iomem *parent_kva;
	page_dir_entry_t parent_entry;
	int freed_pages = 0;

	// Depth-first traversal (from perspective of each page table) of page
	// allocations.
	// (This is depth-first because map_page_directory() always allocates and
	// pushes page directory allocations before page table allocations.)
	list_for_each_entry_safe_reverse(page, _page, &g->pd_allocs, list) {
		printk_debug(KERN_DEBUG "[nvdebug] %s: Checking if page directory/table at %llx (SYS_MEM_?) with parent at %lx (%s) is unused...\n", __func__, page->dma_addr, page->parent_addr, pd_target_to_text(page->parent_aperture));
		// Try to determine if we're still in-use. We consider ourselves
		// potentially in-use if our parent still points to us.
		parent_kva = pd_deref(g, page->parent_addr, page->parent_aperture);
		if (IS_ERR(parent_kva)) {
			printk(KERN_ERR "[nvdebug] %s: Error resolving %#lx in GPU %s to a kernel-accessible address. Error %ld.\n", __func__, page->parent_addr, pd_target_to_text(page->parent_aperture), PTR_ERR(parent_kva));
			return -ENOTRECOVERABLE;
		}
		// A NULL kva indicates parent no longer exists
		parent_entry.raw_w = parent_kva ? readq(parent_kva) : 0;
		// Page directory/table still in-use; do not free unless forced
		if (parent_entry.addr_w == (page->dma_addr >> 12) && !force)
			continue;
		// Free this page table/directory and delete our parent's pointer to us
		if (parent_entry.addr_w == (