1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
|
/* Copyright 2024 Joshua Bakita
* Helpers for dealing with the runlist and other Host (PFIFO) registers
*/
#include <linux/printk.h> // For printk()
#include <asm/errno.h> // For error defines
#include <asm/io.h> // For phys_to_virt()
#include "nvdebug.h"
// Uncomment to, upon BAR2 access failure, return a PRAMIN-based runlist pointer
// in get_runlist_iter(). In order for this pointer to remain valid, PRAMIN
// **must** not be moved during runlist traversal.
// The Jetson TX2 has no BAR2, and stores the runlist in VID_MEM, so this must
// be enabled to print the runlist on the TX2.
//#define FALLBACK_TO_PRAMIN
/* Get RunList RAM (RLRAM) offset for a runlist from the device topology
@param rl_id Which runlist to obtain [numbered in order of appearance in
the device topology (PTOP) registers]
@param rl_ram_off Location at which to store runlist private register
interface base address (PRI base); an offset into the BAR0
register range.
@return 0 or -errno on error
*/
int get_runlist_ram(struct nvdebug_state *g, int rl_id, uint32_t *rl_ram_off) {
int i;
int curr_rl_id = 0;
int ptop_size = NV_PTOP_DEVICE_INFO__SIZE_1_GA100(g);
// Each PTOP entry is composed of 1--3 subrows, and the fields available
// on each row vary. The runlist RAM location is only available on row 3
int ptop_entry_subrow = 0;
ptop_device_info_ga100_t ptop_entry;
// Iterate through all PTOP entries
for (i = 0; i < ptop_size; i++) {
if ((ptop_entry.raw = nvdebug_readl(g, NV_PTOP_DEVICE_INFO_GA100(i))) == -1)
return -EIO;
// Skip empty entries
if (!ptop_entry.raw)
continue;
// If on subrow 3 (zero-base-index 2), runlist info is available
// Multiple engines may be associated with a single runlist, so
// multiple PTOP entries may refer to the same runlist. Only match when
// on the 0th-associated entry.
if (ptop_entry_subrow == 2 && ptop_entry.rleng_id == 0) {
// If this is the requested runlist, return it
if (curr_rl_id == rl_id) {
*rl_ram_off = (uint32_t)ptop_entry.runlist_pri_base << 10;
return 0;
}
// Otherwise, update our accounting of what the next runlist ID is
curr_rl_id++;
}
// Track if the next row is a subrow of the current entry
if (ptop_entry.has_next_entry)
ptop_entry_subrow += 1;
else
ptop_entry_subrow = 0;
}
// Search failed; requested index does not exist
return -EINVAL;
}
/* Get runlist head and info (incl. length)
@param rl_id Which runlist to obtain?
@param rl_iter Location at which to store output
@return 0 or -errno on error
*/
int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl_iter) {
uint64_t runlist_iova;
enum INST_TARGET runlist_target;
uint16_t runlist_len;
int err;
#ifdef FALLBACK_TO_PRAMIN
int off;
#endif // FALLBACK_TO_PRAMIN
// Zero-initialize the runlist iterator
*rl_iter = (struct runlist_iter){0};
// Get runlist location and length using architecture-dependent logic
if (g->chip_id < NV_CHIP_ID_TURING) {
eng_runlist_gf100_t rl;
if ((rl.raw = nvdebug_readq(g, NV_PFIFO_ENG_RUNLIST_BASE_GF100(rl_id))) == -1)
return -EIO;
runlist_iova = ((uint64_t)rl.ptr) << 12;
runlist_target = rl.target;
runlist_len = rl.len;
printk(KERN_INFO "[nvdebug] Runlist %d for %x: %d entries @ %llx in %s (config raw: %#018llx)\n",
rl_id, g->chip_id, rl.len, runlist_iova, target_to_text(rl.target), rl.raw);
} else if (g->chip_id < NV_CHIP_ID_AMPERE) {
runlist_base_tu102_t base;
runlist_submit_tu102_t submit;
if ((base.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_BASE_TU102(rl_id))) == -1)
return -EIO;
if ((submit.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id))) == -1)
return -EIO;
runlist_iova = ((uint64_t)base.ptr) << 12;
runlist_target = base.target;
runlist_len = submit.len;
printk(KERN_INFO "[nvdebug] Runlist %d for %x: %d entries @ %llx in %s (config raw: %#018llx %#018llx)\n",
rl_id, g->chip_id, submit.len, runlist_iova, target_to_text(runlist_target), base.raw, submit.raw);
} else {
runlist_base_tu102_t base;
runlist_submit_tu102_t submit;
uint32_t runlist_pri_base;
// Runlist configurations are stored in per-runlist regions on Ampere+
if ((err = get_runlist_ram(g, rl_id, &runlist_pri_base)) < 0)
return err;
// The runlist configuration region (RLRAM) contains Turing-like BASE
// and SUBMIT registers at static offsets
if ((base.raw = nvdebug_readq(g, runlist_pri_base + NV_RUNLIST_BASE_GA100)) == -1)
return -EIO;
if ((submit.raw = nvdebug_readq(g, runlist_pri_base + NV_RUNLIST_SUBMIT_GA100)) == -1)
return -EIO;
runlist_iova = ((uint64_t)base.ptr) << 12;
runlist_target = base.target;
runlist_len = submit.len;
printk(KERN_INFO "[nvdebug] Runlist %d for %x: %d entries @ %llx in %s (config raw: %#018llx %#018llx)\n",
rl_id, g->chip_id, submit.len, runlist_iova, target_to_text(runlist_target), base.raw, submit.raw);
rl_iter->runlist_pri_base = runlist_pri_base;
}
// Return early on an empty runlist
if (!runlist_len)
return 0;
// If the runlist is in VID_MEM, search the BAR2/3 page tables for a mapping
if (runlist_target == TARGET_VID_MEM) {
uint64_t runlist_bar_vaddr;
page_dir_config_t pd_config;
if ((err = get_bar2_pdb(g, &pd_config)) < 0)
goto attempt_pramin_access;
if (pd_config.is_ver2)
runlist_bar_vaddr = search_page_directory(g, pd_config, runlist_iova, TARGET_VID_MEM);
else
runlist_bar_vaddr = search_v1_page_directory(g, pd_config, runlist_iova, TARGET_VID_MEM);
if (!runlist_bar_vaddr) {
printk(KERN_WARNING "[nvdebug] Unable to find runlist %d mapping in BAR2/3 page tables for %x.\n", rl_id, g->chip_id);
err = -EOPNOTSUPP;
goto attempt_pramin_access;
}
printk(KERN_INFO "[nvdebug] Runlist %d for %x @ %llx in BAR2 virtual address space.\n", rl_id, g->chip_id, runlist_bar_vaddr);
if (!g->bar2) {
printk(KERN_WARNING "[nvdebug] BAR2/3 not mapped for %x.\n", g->chip_id);
return -ENODEV;
}
rl_iter->curr_entry = g->bar2 + runlist_bar_vaddr;
} else {
// Directly access the runlist if stored in SYS_MEM (physically addressed)
// XXX: SYS_MEM is an IOMMU address on some platforms, causing this to crash
rl_iter->curr_entry = (void*)phys_to_virt(runlist_iova);
}
rl_iter->len = runlist_len;
return 0;
attempt_pramin_access:
#ifdef FALLBACK_TO_PRAMIN
printk(KERN_INFO "[nvdebug] Attempting to move PRAMIN window to runlist as BAR2/3-based access failed [DANGEROUS SIDE EFFECTS]!\n");
if ((off = addr_to_pramin_mut(g, runlist_iova, runlist_target)) == -1)
return off;
rl_iter->curr_entry = g->regs + NV_PRAMIN + off;
rl_iter->len = runlist_len;
return 0;
#else
return err;
#endif // FALLBACK_TO_PRAMIN
}
/* Trigger a preempt of the specified TSG
@param tsg_id ID of TSG to preempt.
@return 0 or -errno on error
Note: If no other TSGs exist in the associated runlist, this TSG may
continue executing, unless NV_PFIFO_SCHED_DISABLE is set, or all the
channels of the TSG to be preempted are disabled.
*/
int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) {
pfifo_preempt_t pfifo_preempt;
if (g->chip_id < NV_CHIP_ID_KEPLER)
return -EOPNOTSUPP;
pfifo_preempt.raw = 0;
pfifo_preempt.id = tsg_id;
pfifo_preempt.is_pending = 0;
pfifo_preempt.type = PREEMPT_TYPE_TSG;
// Actually trigger the preemption
nvdebug_writel(g, NV_PFIFO_PREEMPT, pfifo_preempt.raw);
return 0;
}
/* Trigger a preempt of the specified runlist
@param rl_id ID of runlist to preempt.
@return 0 or -errno on error
*/
int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id) {
runlist_preempt_t rl_preempt;
if (g->chip_id < NV_CHIP_ID_VOLTA)
return -EOPNOTSUPP;
// Overwrite, as the register contains nothing to preserve
rl_preempt.raw = BIT(rl_id);
nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
return 0;
}
// Read and write runlist configuration, triggering a resubmit
int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
if (g->chip_id < NV_CHIP_ID_TURING) {
eng_runlist_gf100_t rl;
if (rl_id > MAX_RUNLISTS_GF100)
return -EINVAL;
if ((rl.raw = nvdebug_readq(g, NV_PFIFO_ENG_RUNLIST_BASE_GF100(rl_id))) == -1)
return -EIO;
rl.id = rl_id;
nvdebug_writeq(g, NV_PFIFO_RUNLIST_BASE_GF100, rl.raw);
} else if (g->chip_id < NV_CHIP_ID_AMPERE) {
runlist_submit_tu102_t submit;
if (rl_id > MAX_RUNLISTS_TU102)
return -EINVAL;
if ((submit.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id))) == -1)
return -EIO;
nvdebug_writeq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id), submit.raw);
} else {
return -EOPNOTSUPP;
}
return 0;
}
|
