path: root/runlist.c

/* Copyright 2024 Joshua Bakita
 * Helpers for dealing with the runlist and other Host (PFIFO) registers
 */
#include <linux/printk.h> // For printk()
#include <asm/errno.h> // For error defines
#include <asm/io.h> // For phys_to_virt()

#include "nvdebug.h"

// Uncomment to, upon BAR2 access failure, return a PRAMIN-based runlist pointer
// **If enabled, PRAMIN may not be otherwise used while walking the runlist!**
#define FALLBACK_TO_PRAMIN

/* Get runlist head and info (incl. length)
  @param rl_id   Which runlist to obtain?
  @param rl_iter Location at which to store output
  @return 0 or -errno on error
*/
int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl_iter) {
	uint64_t runlist_iova;
	enum INST_TARGET runlist_target;
	uint16_t runlist_len;
#ifdef FALLBACK_TO_PRAMIN
	int off;
#endif // FALLBACK_TO_PRAMIN
	// Zero-initialize the runlist iterator
	*rl_iter = (struct runlist_iter){0};

	// Get runlist location and length using architecture-dependent logic
	if (g->chip_id < NV_CHIP_ID_TURING) {
		eng_runlist_gf100_t rl;
		if ((rl.raw = nvdebug_readq(g, NV_PFIFO_ENG_RUNLIST_BASE_GF100(rl_id))) == -1)
			return -EIO;
		runlist_iova = ((uint64_t)rl.ptr) << 12;
		runlist_target = rl.target;
		printk(KERN_INFO "[nvdebug] Runlist %d: %d entries @ %llx in %s (config raw: %#018llx)\n",
			   rl_id, rl.len, runlist_iova, target_to_text(rl.target), rl.raw);
		runlist_len = rl.len;
	} else if (g->chip_id < NV_CHIP_ID_AMPERE) {
		runlist_base_tu102_t base;
		runlist_submit_tu102_t submit;
		if ((base.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_BASE_TU102(rl_id))) == -1)
			return -EIO;
		if ((submit.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id))) == -1)
			return -EIO;
		runlist_iova = ((uint64_t)base.ptr) << 12;
		runlist_target = base.target;
		runlist_len = submit.len;
	}
	// Return early on an empty runlist
	if (!runlist_len)
		return 0;

	// If the runlist is in VID_MEM, search the BAR2/3 page tables for a mapping
	if (runlist_target == TARGET_VID_MEM) {
		void __iomem *bar2_page_dir;
		bool pdb_is_ver2;
		uint64_t runlist_bar_vaddr;

		if (get_bar2_pdb(g, &bar2_page_dir, &pdb_is_ver2) < 0)
			return -EIO;

		if (pdb_is_ver2)
			runlist_bar_vaddr = search_page_directory(g, bar2_page_dir, phy2PRAMIN, runlist_iova);
		else
			runlist_bar_vaddr = search_v1_page_directory(g, bar2_page_dir, phy2PRAMIN, runlist_iova);
		if (!runlist_bar_vaddr) {
			printk(KERN_WARNING "[nvdebug] Unable to find runlist mapping in BAR2/3 page tables.\n");
			goto attempt_pramin_access;
		}
		printk(KERN_INFO "[nvdebug] Runlist @ %llx in BAR2 virtual address space.\n", runlist_bar_vaddr);
		if (!g->bar2) {
			printk(KERN_WARNING "[nvdebug] BAR2/3 not mapped.\n");
			return -ENODEV;
		}
		rl_iter->curr_entry = g->bar2 + runlist_bar_vaddr;
	} else {
		// Directly access the runlist if stored in SYS_MEM (physically addressed)
		// XXX: SYS_MEM is an IOMMU address on some platforms, causing this to crash
		rl_iter->curr_entry = (void*)phys_to_virt(runlist_iova);
	}
	rl_iter->len = runlist_len;
	return 0;

attempt_pramin_access:
#ifdef FALLBACK_TO_PRAMIN
	printk(KERN_INFO "[nvdebug] Attempting to move PRAMIN window to runlist as BAR2/3-based access failed [DANGEROUS SIDE EFFECTS]!\n");
	if ((off = addr_to_pramin_mut(g, runlist_iova, runlist_target)) == -1)
		return off;
	rl_iter->curr_entry = g->regs + NV_PRAMIN + off;
	rl_iter->len = runlist_len;
	return 0;
#else
	return -EOPNOTSUPP;
#endif // FALLBACK_TO_PRAMIN
}

/* Trigger a preempt of the specified TSG
  @param tsg_id ID of TSG to preempt.
  @return 0 or -errno on error

  Note: If no other TSGs exist in the associated runlist, this TSG may
        continue executing, unless NV_PFIFO_SCHED_DISABLE is set, or all the
        channels of the TSG to be preempted are disabled.
*/
int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) {
	pfifo_preempt_t pfifo_preempt;
	if (g->chip_id < NV_CHIP_ID_KEPLER)
		return -EOPNOTSUPP;

	pfifo_preempt.raw = 0;
	pfifo_preempt.id = tsg_id;
	pfifo_preempt.is_pending = 0;
	pfifo_preempt.type = PREEMPT_TYPE_TSG;

	// Actually trigger the preemption
	nvdebug_writel(g, NV_PFIFO_PREEMPT, pfifo_preempt.raw);
	return 0;
}

/* Trigger a preempt of the specified runlist
  @param rl_id ID of runlist to preempt.
  @return 0 or -errno on error
*/
int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id) {
	runlist_preempt_t rl_preempt;
	if (g->chip_id < NV_CHIP_ID_VOLTA)
		return -EOPNOTSUPP;

	// Overwrite, as the register contains nothing to preserve
	rl_preempt.raw = BIT(rl_id);
	nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
	return 0;
}