/* Copyright 2024 Joshua Bakita
* Helpers for dealing with the runlist and other Host (PFIFO) registers
*/
#include <linux/printk.h> // For printk()
#include <asm/errno.h> // For error defines
#include <asm/io.h> // For phys_to_virt()
#include "nvdebug.h"
// Uncomment to, upon BAR2 access failure, return a PRAMIN-based runlist pointer
// in get_runlist_iter(). In order for this pointer to remain valid, PRAMIN
// **must** not be moved during runlist traversal.
// The Jetson TX2 has no BAR2, and stores the runlist in VID_MEM, so this must
// be enabled to print the runlist on the TX2.
//#define FALLBACK_TO_PRAMIN
/* Get RunList RAM (RLRAM) offset for a runlist from the device topology
@param rl_id Which runlist to obtain [numbered in order of appearance in
the device topology (PTOP) registers]
@param rl_ram_off Location at which to store runlist private register
interface base address (PRI base); an offset into the BAR0
register range.
@return 0 or -errno on error
*/
int get_runlist_ram(struct nvdebug_state *g, int rl_id, uint32_t *rl_ram_off) {
int i;
int curr_rl_id = 0;
int ptop_size = NV_PTOP_DEVICE_INFO__SIZE_1_GA100(g);
// Each PTOP entry is composed of 1--3 subrows, and the fields available
// on each row vary. The runlist RAM location is only available on row 3
int ptop_entry_subrow = 0;
ptop_device_info_ga100_t ptop_entry;
// Iterate through all PTOP entries
for (i = 0; i < ptop_size; i++) {
if ((ptop_entry.raw = nvdebug_readl(g, NV_PTOP_DEVICE_INFO_GA100(i))) == -1)
return -EIO;
// Skip empty entries
if (!ptop_entry.raw)
continue;
// If on subrow 3 (zero-base-index 2), runlist info is available
// Multiple engines may be associated with a single runlist, so
// multiple PTOP entries may refer to the same runlist. Only match when
// on the 0th-associated entry.
if (ptop_entry_subrow == 2 && ptop_entry.rleng_id == 0) {
// If this is the requested runlist, return it
if (curr_rl_id == rl_id) {
*rl_ram_off = (uint32_t)ptop_entry.runlist_pri_base << 10;
return 0;
}
// Otherwise, update our accounting of what the next runlist ID is
curr_rl_id++;
}
// Track if the next row is a subrow of the current entry
if (ptop_entry.has_next_entry)
ptop_entry_subrow += 1;
else
ptop_entry_subrow = 0;
}
// Search failed; requested index does not exist
return -EINVAL;
}
/* Get runlist head and info (incl. length)
@param rl_id Which runlist to obtain?
@param rl_iter Location at which to store output
@return 0 or -errno on error
*/
int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl_iter) {
uint64_t runlist_iova;
enum INST_TARGET runlist_target;
uint16_t runlist_len;
int err;
#ifdef FALLBACK_TO_PRAMIN
int off;
#endif // FALLBACK_TO_PRAMIN
// Zero-initialize the runlist iterator
*rl_iter = (struct runlist_iter){0};
// Get runlist location and length using architecture-dependent logic
if (g->chip_id < NV_CHIP_ID_TURING) {
eng_runlist_gf100_t rl;
if ((rl.raw = nvdebug_readq(g, NV_PFIFO_ENG_RUNLIST_BASE_GF100(rl_id))) == -1)
return -EIO;
runlist_iova = ((uint64_t)rl.ptr) << 12;
runlist_target = rl.target;
runlist_len = rl.len;
printk(KERN_INFO "[nvdebug] Runlist %d for %x: %d entries @ %llx in %s (config raw: %#018llx)\n",
rl_id, g->chip_id, rl.len, runlist_iova, target_to_text(rl.target), rl.raw);
} else if (g->chip_id < NV_CHIP_ID_AMPERE) {
runlist_base_tu102_t base;
runlist_submit_tu102_t submit;
if ((base.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_BASE_TU102(rl_id))) == -1)
return -EIO;
if ((submit.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id))) == -1)
return -EIO;
runlist_iova = ((uint64_t)base.ptr) << 12;
runlist_target = base.target;
runlist_len = submit.len;
printk(KERN_INFO "[nvdebug] Runlist %d for %x: %d entries @ %llx in %s (config raw: %#018llx %#018llx)\n",
rl_id, g->chip_id, submit.len, runlist_iova, target_to_text(runlist_target), base.raw, submit.raw);
} else {
runlist_base_tu102_t base;
runlist_submit_tu102_t submit;
uint32_t runlist_pri_base;
// Runlist configurations are stored in per-runlist regions on Ampere+
if ((err = get_runlist_ram(g, rl_id, &runlist_pri_base)) < 0)
return err;
// The runlist configuration region (RLRAM) contains Turing-like BASE
// and SUBMIT registers at static offsets
if ((base.raw = nvdebug_readq(g, runlist_pri_base + NV_RUNLIST_BASE_GA100)) == -1)
return -EIO;
if ((submit.raw = nvdebug_readq(g, runlist_pri_base + NV_RUNLIST_SUBMIT_GA100)) == -1)
return -EIO;
runlist_iova = ((uint64_t)base.ptr) << 12;
runlist_target = base.target;
runlist_len = submit.len;
printk(KERN_INFO "[nvdebug] Runlist %d for %x: %d entries @ %llx in %s (config raw: %#018llx %#018llx)\n",
rl_id, g->chip_id, submit.len, runlist_iova, target_to_text(runlist_target), base.raw, submit.raw);
rl_iter->runlist_pri_base = runlist_pri_base;
}
// Return early on an empty runlist
if (!runlist_len)
return 0;
// If the runlist is in VID_MEM, search the BAR2/3 page tables for a mapping
if (runlist_target == TARGET_VID_MEM) {
uint64_t runlist_bar_vaddr;
page_dir_config_t pd_config;
if ((err = get_bar2_pdb(g, &pd_config)) < 0)
goto attempt_pramin_access;
if (pd_config.is_ver2)
runlist_bar_vaddr = search_page_directory(g, pd_config, runlist_iova, TARGET_VID_MEM);
else
runlist_bar_vaddr = search_v1_page_directory(g, pd_config, runlist_iova, TARGET_VID_MEM);
if (!runlist_bar_vaddr) {
printk(KERN_WARNING "[nvdebug] Unable to find runlist %d mapping in BAR2/3 page tables for %x.\n", rl_id, g->chip_id);
err = -EOPNOTSUPP;
goto attempt_pramin_access;
}
printk(KERN_INFO "[nvdebug] Runlist %d for %x @ %llx in BAR2 virtual address space.\n", rl_id, g->chip_id, runlist_bar_vaddr);
if (!g->bar2) {
printk(KERN_WARNING "[nvdebug] BAR2/3 not mapped for %x.\n", g->chip_id);
return -ENODEV;
}
rl_iter->curr_entry = g->bar2 + runlist_bar_vaddr;
} else {
// Directly access the runlist if stored in SYS_MEM (physically addressed)
// XXX: SYS_MEM is an IOMMU address on some platforms, causing this to crash
rl_iter->curr_entry = (void*)phys_to_virt(runlist_iova);
}
rl_iter->len = runlist_len;
return 0;
attempt_pramin_access:
#ifdef FALLBACK_TO_PRAMIN
printk(KERN_INFO "[nvdebug] Attempting to move PRAMIN window to runlist as BAR2/3-based access failed [DANGEROUS SIDE EFFECTS]!\n");
if ((off = addr_to_pramin_mut(g, runlist_iova, runlist_target)) == -1)
return off;
rl_iter->curr_entry = g->regs + NV_PRAMIN + off;
rl_iter->len = runlist_len;
return 0;
#else
return err;
#endif // FALLBACK_TO_PRAMIN
}
/* Trigger a preempt of the specified TSG
@param tsg_id ID of TSG to preempt.
@param rl_id Which channel RAM address space to search?
@return 0 or -errno on error
Note: If no other TSGs exist in the associated runlist, this TSG may
continue executing, unless NV_PFIFO_SCHED_DISABLE is set, or all the
channels of the TSG to be preempted are disabled.
*/
int preempt_tsg(struct nvdebug_state *g, uint32_t rl_id, uint32_t tsg_id) {
pfifo_preempt_t preempt;
// Fermi does not support time-slice groups
if (g->chip_id < NV_CHIP_ID_KEPLER)
return -EOPNOTSUPP;
preempt.raw = 0;
preempt.id = tsg_id;
preempt.type = PREEMPT_TYPE_TSG;
// Actually trigger the preemption
if (g->chip_id < NV_CHIP_ID_AMPERE) {
nvdebug_writel(g, NV_PFIFO_PREEMPT, preempt.raw);
} else {
uint32_t runlist_reg_base;
int err;
// As TSG and channel IDs are namespaced per-runlist starting with
// Ampere, the PREEMPT register is also per-runlist.
if ((err = get_runlist_ram(g, rl_id, &runlist_reg_base)))
return err;
nvdebug_writel(g, runlist_reg_base + NV_RUNLIST_PREEMPT_GA100, preempt.raw);
}
return 0;
}
/* Trigger a preempt of the specified runlist
@param rl_id ID of runlist to preempt.
@return 0 or -errno on error
*/
int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id) {
// The runlist preempt register does not exist on Kepler (tested gk104)
if (g->chip_id < NV_CHIP_ID_MAXWELL)
return -EOPNOTSUPP;
// Write to trigger the preemption (the register contains nothing to
// preserve, and can thus just be overwritten)
if (g->chip_id < NV_CHIP_ID_AMPERE) {
runlist_preempt_t rl_preempt;
rl_preempt.raw = BIT(rl_id);
nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
} else {
int err;
uint32_t runlist_regs_base;
pfifo_preempt_t preempt;
// The RUNLIST_PREEMPT register was deleted, and the _PREEMPT register
// was extended to support runlist-level preemptions starting on Ampere
preempt.id = rl_id;
preempt.type = PREEMPT_TYPE_RUNLIST;
// The preempt register is scoped per-runlist on Ampere+
if ((err = get_runlist_ram(g, rl_id, &runlist_regs_base)))
return err;
nvdebug_writel(g, runlist_regs_base + NV_RUNLIST_PREEMPT_GA100, preempt.raw);
}
return 0;
}
// Read and write runlist configuration, triggering a resubmit
int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
// Necessary registers do not exist pre-Fermi
if (g->chip_id < NV_CHIP_ID_FERMI)
return -EOPNOTSUPP;
if (g->chip_id < NV_CHIP_ID_TURING) {
eng_runlist_gf100_t rl;
if (rl_id > MAX_RUNLISTS_GF100)
return -EINVAL;
if ((rl.raw = nvdebug_readq(g, NV_PFIFO_ENG_RUNLIST_BASE_GF100(rl_id))) == -1)
return -EIO;
rl.id = rl_id;
nvdebug_writeq(g, NV_PFIFO_RUNLIST_BASE_GF100, rl.raw);
} else if (g->chip_id < NV_CHIP_ID_AMPERE) {
runlist_submit_tu102_t submit;
if (rl_id > MAX_RUNLISTS_TU102)
return -EINVAL;
if ((submit.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id))) == -1)
return -EIO;
nvdebug_writeq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id), submit.raw);
} else {
int err;
uint32_t runlist_pri_base;
runlist_submit_tu102_t submit;
if ((err = get_runlist_ram(g, rl_id, &runlist_pri_base)) < 0)
return err;
if ((submit.raw = nvdebug_readq(g, runlist_pri_base + NV_RUNLIST_SUBMIT_GA100)) == -1)
return -EIO;
// On Ampere, this does not appear to trigger a preempt of the
// currently-running channel (even if the currently running channel
// becomes disabled), but will cause newly re-enabled channels
// (at least if nothing else is pending) to become ready (tested on
// Jetson Orin).
nvdebug_writeq(g, runlist_pri_base + NV_RUNLIST_SUBMIT_GA100, submit.raw);
}
return 0;
}
