/* Copyright 2024 Joshua Bakita
* Helpers for dealing with the PCIe and platform bus
*
* = Design Notes =
* We have to use PRAMIN to access the BAR2 page table. While it's typically also
* mapped into BAR2, we have no way to know where without the table. If the table
* changes, the new sections will have new mappings in BAR2, repeating the
* problem, and making caching insufficient.
*
* = Terms =
* VRAM/VID_MEM: Video RAM; Addresses to physical frames in the on-GPU RAM.
* SYS_MEM: System Memory; "Bus addresses"; Addresses which can be presented to
* the PCIe Host for resolution. On x86_64 without an IOMMU, these are
* just physical addresses, but may be I/O Virtual Addresses (IOVAs)
* or translated via an I/O MMU on other platforms; DMA Addresses.
* PEER: Addresses to RAM on another GPU.
*/
#include <linux/printk.h> // For printk()
#include <asm/errno.h> // For error defines
#include <asm/io.h> // For readl()
#include "nvdebug.h"
/* Obtain the PRAMIN offset at which `addr` can be accessed
@param addr Address to find
@param target Which address space to use (VRAM, SYS_MEM, PEER(?))
@return positive offset or -EINVAL on invalid arguments
Note: Will move the PRAMIN window to accomodate the request. Only guarantees
that the surrounding 64KiB window will be accessible.
Note: Moving the PRAMIN window will cause problems if it races with driver
code that tries to do the same, or expects the window not to move.
Bugs: Untested on PEER.
*/
int addr_to_pramin_mut(struct nvdebug_state *g,
uint64_t addr, enum INST_TARGET target) {
bar0_window_t window;
uint64_t pramin_base;
// For us, accuracy and robustness is more important than speed
// Check that the address is valid (49 bits are addressable on-GPU, but
// PRAMIN only supports up to 40 bits).
if (addr & ~0x000000ffffffffff) {
printk(KERN_ERR "[nvdebug] Invalid address %llx passed to %s!\n",
addr, __func__);
return -EINVAL;
}
window.raw = nvdebug_readl(g, NV_PBUS_BAR0_WINDOW);
if (window.target != target)
goto relocate;
pramin_base = ((uint64_t)window.base) << 16;
if (addr < pramin_base || addr > pramin_base + NV_PRAMIN_LEN)
goto relocate;
return addr - pramin_base; // Guaranteed to be < 1MiB, so safe for int
relocate:
printk(KERN_INFO "[nvdebug] Moving PRAMIN win from base %llx to %llx to accomodate %#018llx\n", pramin_base, (addr >> 16) << 16, addr);
// Move PRAMIN window to a 64KiB-aligned address
window.base = (u32)(addr >> 16); // Safe, due to above range check
window.target = target;
nvdebug_writel(g, NV_PBUS_BAR0_WINDOW, window.raw);
return (int)(addr & 0xffffull);
}
/* Get a persistent pointer to the page directory base
@param pdb Dereferencable pointer to the zeroeth entry of top-level page
directory (PD3) for the BAR2 register region.
Note: The returned pointer will be into the PRAMIN space. If the PRAMIN
window is moved to a region that does not cover the BAR2 page table,
this ***will move the window***.
Note: Even if the page table is located in SYS_MEM, we route reads/writes via
PRAMIN. This ensures that we always see what the GPU sees, and that
includes any passes through I/O MMUs or IOVA spaces.
*/
int get_bar2_pdb(struct nvdebug_state *g, void **pdb, bool *is_v2_pdb) {
static void* cached_pdb = NULL;
static bool cached_is_v2_pdb = false;
static long pd_hash = 0;
int ret;
bar_config_block_t bar2_block;
page_dir_config_t pd_config;
uint64_t pdb_vram;
// Use cached base as long as it's still pointing to the same thing
if (cached_pdb && readl(cached_pdb) == pd_hash) {
*pdb = cached_pdb;
*is_v2_pdb = cached_is_v2_pdb;
return 0;
}
if (!g->bar2)
return -ENXIO;
// BAR2 has its own instance block (typically in VRAM) which contains the
// Page Directory Base (PDB), a pointer to a page directory/table
// hierarchy used to translate BAR2 offsets to VRAM or SYS_MEM addresses.
// Determine location of BAR2 instance block
if ((bar2_block.raw = nvdebug_readl(g, NV_PBUS_BAR2_BLOCK)) == -1) {
printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 configuration! BAR2/3 inaccessible.\n");
return -ENOTSUPP;
}
printk(KERN_INFO "[nvdebug] BAR2 inst block @ %llx in %s's %s address space.\n", ((u64)bar2_block.ptr) << 12, target_to_text(bar2_block.target), bar2_block.is_virtual ? "virtual" : "physical");
// Setup PRAMIN to point at the BAR2 instance block
if ((ret = addr_to_pramin_mut(g, (uint64_t)bar2_block.ptr << 12, bar2_block.target)) < 0) {
printk(KERN_ERR "[nvdebug] Invalid BAR2/3 Instance Block configuration! BAR2/3 inaccessible.\n");
return ret;
}
printk(KERN_INFO "[nvdebug] BAR2 inst block at off %x in PRAMIN\n", ret);
// Pull the page directory base configuration from the instance block
if ((pd_config.raw = nvdebug_readq(g, NV_PRAMIN + ret + NV_PRAMIN_PDB_CONFIG_OFF)) == -1) {
printk(KERN_ERR "[nvdebug] Unable to read BAR2/3 PDB configuration! BAR2/3 inaccessible.\n");
return -ENOTSUPP;
}
pdb_vram = pd_config.page_dir_hi;
pdb_vram <<= 20;
pdb_vram |= pd_config.page_dir_lo;
pdb_vram <<= 12;
printk(KERN_INFO "[nvdebug] BAR2 PDB @ %llx (config raw: %llx)\n", pdb_vram, pd_config.raw);
// Setup PRAMIN to point at the page directory
if ((ret = addr_to_pramin_mut(g, pdb_vram, pd_config.target)) < 0) {
printk(KERN_ERR "[nvdebug] Invalid BAR2/3 PDB configuration! BAR2/3 inaccessible.\n");
return ret;
}
*pdb = cached_pdb = g->regs + NV_PRAMIN + ret;
pd_hash = readl(cached_pdb);
*is_v2_pdb = cached_is_v2_pdb = pd_config.is_ver2;
return 0;
}
