1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
/* Copyright 2024 Joshua Bakita
* Helpers for dealing with the runlist and other Host (PFIFO) registers
*/
#include <linux/printk.h> // For printk()
#include <asm/errno.h> // For error defines
#include <asm/io.h> // For phys_to_virt()
#include "nvdebug.h"
// Uncomment to, upon BAR2 access failure, return a PRAMIN-based runlist pointer
// **If enabled, PRAMIN may not be otherwise used while walking the runlist!**
#define FALLBACK_TO_PRAMIN
/* Get runlist head and info (incl. length)
@param rl_id Which runlist to obtain?
@param rl_iter Location at which to store output
@return 0 or -errno on error
*/
int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl_iter) {
uint64_t runlist_iova;
enum INST_TARGET runlist_target;
uint16_t runlist_len;
#ifdef FALLBACK_TO_PRAMIN
int off;
#endif // FALLBACK_TO_PRAMIN
// Zero-initialize the runlist iterator
*rl_iter = (struct runlist_iter){0};
// Get runlist location and length using architecture-dependent logic
if (g->chip_id < NV_CHIP_ID_TURING) {
eng_runlist_gf100_t rl;
if ((rl.raw = nvdebug_readq(g, NV_PFIFO_ENG_RUNLIST_BASE_GF100(rl_id))) == -1)
return -EIO;
runlist_iova = ((uint64_t)rl.ptr) << 12;
runlist_target = rl.target;
printk(KERN_INFO "[nvdebug] Runlist %d: %d entries @ %llx in %s (config raw: %#018llx)\n",
rl_id, rl.len, runlist_iova, target_to_text(rl.target), rl.raw);
runlist_len = rl.len;
} else if (g->chip_id < NV_CHIP_ID_AMPERE) {
runlist_base_tu102_t base;
runlist_submit_tu102_t submit;
if ((base.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_BASE_TU102(rl_id))) == -1)
return -EIO;
if ((submit.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id))) == -1)
return -EIO;
runlist_iova = ((uint64_t)base.ptr) << 12;
runlist_target = base.target;
runlist_len = submit.len;
}
// Return early on an empty runlist
if (!runlist_len)
return 0;
// If the runlist is in VID_MEM, search the BAR2/3 page tables for a mapping
if (runlist_target == TARGET_VID_MEM) {
void __iomem *bar2_page_dir;
bool pdb_is_ver2;
uint64_t runlist_bar_vaddr;
if (get_bar2_pdb(g, &bar2_page_dir, &pdb_is_ver2) < 0)
return -EIO;
if (pdb_is_ver2)
runlist_bar_vaddr = search_page_directory(g, bar2_page_dir, phy2PRAMIN, runlist_iova);
else
runlist_bar_vaddr = search_v1_page_directory(g, bar2_page_dir, phy2PRAMIN, runlist_iova);
if (!runlist_bar_vaddr) {
printk(KERN_WARNING "[nvdebug] Unable to find runlist mapping in BAR2/3 page tables.\n");
goto attempt_pramin_access;
}
printk(KERN_INFO "[nvdebug] Runlist @ %llx in BAR2 virtual address space.\n", runlist_bar_vaddr);
if (!g->bar2) {
printk(KERN_WARNING "[nvdebug] BAR2/3 not mapped.\n");
return -ENODEV;
}
rl_iter->curr_entry = g->bar2 + runlist_bar_vaddr;
} else {
// Directly access the runlist if stored in SYS_MEM (physically addressed)
// XXX: SYS_MEM is an IOMMU address on some platforms, causing this to crash
rl_iter->curr_entry = (void*)phys_to_virt(runlist_iova);
}
rl_iter->len = runlist_len;
return 0;
attempt_pramin_access:
#ifdef FALLBACK_TO_PRAMIN
printk(KERN_INFO "[nvdebug] Attempting to move PRAMIN window to runlist as BAR2/3-based access failed [DANGEROUS SIDE EFFECTS]!\n");
if ((off = addr_to_pramin_mut(g, runlist_iova, runlist_target)) == -1)
return off;
rl_iter->curr_entry = g->regs + NV_PRAMIN + off;
rl_iter->len = runlist_len;
return 0;
#else
return -EOPNOTSUPP;
#endif // FALLBACK_TO_PRAMIN
}
/* Trigger a preempt of the specified TSG
@param tsg_id ID of TSG to preempt.
@return 0 or -errno on error
Note: If no other TSGs exist in the associated runlist, this TSG may
continue executing, unless NV_PFIFO_SCHED_DISABLE is set, or all the
channels of the TSG to be preempted are disabled.
*/
int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) {
pfifo_preempt_t pfifo_preempt;
if (g->chip_id < NV_CHIP_ID_KEPLER)
return -EOPNOTSUPP;
pfifo_preempt.raw = 0;
pfifo_preempt.id = tsg_id;
pfifo_preempt.is_pending = 0;
pfifo_preempt.type = PREEMPT_TYPE_TSG;
// Actually trigger the preemption
nvdebug_writel(g, NV_PFIFO_PREEMPT, pfifo_preempt.raw);
return 0;
}
/* Trigger a preempt of the specified runlist
@param rl_id ID of runlist to preempt.
@return 0 or -errno on error
*/
int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id) {
runlist_preempt_t rl_preempt;
if (g->chip_id < NV_CHIP_ID_VOLTA)
return -EOPNOTSUPP;
// Overwrite, as the register contains nothing to preserve
rl_preempt.raw = BIT(rl_id);
nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
return 0;
}
|
