diff options
author | Joshua Bakita <jbakita@cs.unc.edu> | 2024-04-08 15:35:54 -0400 |
---|---|---|
committer | Joshua Bakita <jbakita@cs.unc.edu> | 2024-04-08 17:09:13 -0400 |
commit | 47506870790989b5e2d9a6128711d96c487f0d7b (patch) | |
tree | 98c09b9464af4c4a983f75b17568aa5ca919d886 /runlist.c | |
parent | 14cb76b1a7e93a5f3900ea7696071dcc281a3586 (diff) |
Heavily refactor runlist code for correctness and Turing support
- Support differently-formatted runlist registers on Turing
- Support different runlist register offsets on Turing
- Fix incorrect indenting when printing the runlist
- Fix `preempt_tsg` and `switch_to_tsg` API implementations to
correctly interface with the hardware (previously, they would try
to disable scheduling for the last-updated runlist pointer, which
was nonsense, and just an artifact of my early misunderstandings
of how the NV_PFIFO_RUNLIST* registers worked).
- Remove misused NV_PFIFO_RUNLIST and NV_PFIFO_RUNLIST_BASE registers
- Refactor `runlist.c` to use the APIs from `bus.c`
Diffstat (limited to 'runlist.c')
-rw-r--r-- | runlist.c | 212 |
1 files changed, 87 insertions, 125 deletions
@@ -1,172 +1,134 @@ | |||
1 | #include <linux/kernel.h> // Kernel types | 1 | /* Copyright 2024 Joshua Bakita |
2 | * Helpers for dealing with the runlist and other Host (PFIFO) registers | ||
3 | */ | ||
4 | #include <linux/printk.h> // For printk() | ||
5 | #include <asm/errno.h> // For error defines | ||
6 | #include <asm/io.h> // For phys_to_virt() | ||
2 | 7 | ||
3 | #include "nvdebug.h" | 8 | #include "nvdebug.h" |
4 | 9 | ||
10 | // Uncomment to, upon BAR2 access failure, return a PRAMIN-based runlist pointer | ||
11 | // **If enabled, PRAMIN may not be otherwise used while walking the runlist!** | ||
5 | #define FALLBACK_TO_PRAMIN | 12 | #define FALLBACK_TO_PRAMIN |
6 | 13 | ||
7 | /* Get runlist head and info (incl. length) | 14 | /* Get runlist head and info (incl. length) |
8 | @param rl_iter Location at which to store output | 15 | @param rl_id Which runlist to obtain? |
9 | @param rl_id Which runlist to obtain? | 16 | @param rl_iter Location at which to store output |
17 | @return 0 or -errno on error | ||
10 | */ | 18 | */ |
11 | int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl_iter) { | 19 | int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl_iter) { |
12 | runlist_base_t rl_base; | 20 | uint64_t runlist_iova; |
13 | runlist_info_t rl_info; | 21 | enum INST_TARGET runlist_target; |
14 | u64 runlist_iova; | 22 | uint16_t runlist_len; |
23 | #ifdef FALLBACK_TO_PRAMIN | ||
24 | int off; | ||
25 | #endif // FALLBACK_TO_PRAMIN | ||
26 | // Zero-initialize the runlist iterator | ||
15 | *rl_iter = (struct runlist_iter){0}; | 27 | *rl_iter = (struct runlist_iter){0}; |
16 | rl_base.raw = nvdebug_readl(g, NV_PFIFO_ENG_RUNLIST_BASE(rl_id)); | 28 | |
17 | // Check that reads are working | 29 | // Get runlist location and length using architecture-dependent logic |
18 | if (rl_base.raw == -1) | 30 | if (g->chip_id < NV_CHIP_ID_TURING) { |
19 | return -EIO; | 31 | eng_runlist_gf100_t rl; |
20 | // The address pointed to `regs` + NV_PFIFO_RUNLIST_BASE seems to not be: | 32 | if ((rl.raw = nvdebug_readq(g, NV_PFIFO_ENG_RUNLIST_BASE_GF100(rl_id))) == -1) |
21 | // - A GPU address (type is sysmem_coherent) | 33 | return -EIO; |
22 | // - A physical address (dereferencing after ioremap crashes) | 34 | runlist_iova = ((uint64_t)rl.ptr) << 12; |
23 | // - A kernel virtual address (dereferencing segfaults) | 35 | runlist_target = rl.target; |
24 | // So maybe it's some sort of custom thing? This is an address that the GPU | 36 | printk(KERN_INFO "[nvdebug] Runlist %d: %d entries @ %llx in %s (config raw: %#018llx)\n", |
25 | // can use, so it would make most sense for it to be a physical address. | 37 | rl_id, rl.len, runlist_iova, target_to_text(rl.target), rl.raw); |
26 | // | 38 | runlist_len = rl.len; |
27 | // BUT, it can't possibly be a physical address, as it would refer to an | 39 | } else if (g->chip_id < NV_CHIP_ID_AMPERE) { |
28 | // address greater than the maximum one on our system (by a lot!). | 40 | runlist_base_tu102_t base; |
29 | // Maybe I'm reading the runlist base wrong? | 41 | runlist_submit_tu102_t submit; |
30 | // Aha, the driver calls it runlist_iova. Sounds like runlist I/O virtual | 42 | if ((base.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_BASE_TU102(rl_id))) == -1) |
31 | // address! So, what's this I/O address space? All I know is that it's what | 43 | return -EIO; |
32 | // nvgpu_mem_get_addr() returns. That function returns the result of either: | 44 | if ((submit.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id))) == -1) |
33 | // - gpu_phys_addr which is __nvgpu_sgl_phys on our platform which (?) | 45 | return -EIO; |
34 | // converts an IPA to a PA? | 46 | runlist_iova = ((uint64_t)base.ptr) << 12; |
35 | // - nvgpu_mem_iommu_translate | 47 | runlist_target = base.target; |
36 | // | 48 | runlist_len = submit.len; |
37 | // The original memory is allocated with nvgpu_dma_alloc_flags_sys(), which | 49 | } |
38 | // returns SYSMEM. | ||
39 | // | ||
40 | // To convert a physical address to a IOMMU address, we add a bit | ||
41 | // | ||
42 | // BUT, it turns out that it IS JUST A PHYSICAL ADDRESS! It wasn't working | ||
43 | // before because the GPU had simply gone to sleep and invalidated its | ||
44 | // register state, so nvgpu_readl() was simply returning garbage. | ||
45 | rl_info.raw = nvdebug_readl(g, NV_PFIFO_ENG_RUNLIST(rl_id)); | ||
46 | if (rl_info.raw == -1) | ||
47 | return -EIO; | ||
48 | runlist_iova = ((u64)rl_base.ptr) << 12; | ||
49 | printk(KERN_INFO "[nvdebug] Runlist %d @ %llx in %s (config raw: %x)\n", | ||
50 | rl_id, runlist_iova, target_to_text(rl_base.target), rl_base.raw); | ||
51 | printk(KERN_INFO "[nvdebug] Runlist length %d, ID %d\n", rl_info.len, rl_info.id); | ||
52 | // Return early on an empty runlist | 50 | // Return early on an empty runlist |
53 | if (!rl_info.len) | 51 | if (!runlist_len) |
54 | return 0; | 52 | return 0; |
53 | |||
55 | // If the runlist is in VID_MEM, search the BAR2/3 page tables for a mapping | 54 | // If the runlist is in VID_MEM, search the BAR2/3 page tables for a mapping |
56 | if (rl_base.target == TARGET_VID_MEM) { | 55 | if (runlist_target == TARGET_VID_MEM) { |
57 | printk(KERN_WARNING "[nvdebug] Runlist is located in video memory. Access to video memory is experimental."); | 56 | void __iomem *bar2_page_dir; |
58 | bar_config_block_t bar1_block, bar2_block; | 57 | bool pdb_is_ver2; |
59 | bar1_block.raw = nvdebug_readl(g, NV_PBUS_BAR1_BLOCK); | ||
60 | printk(KERN_INFO "[nvdebug] BAR1 inst block @ %llx in %s's %s address space.\n", ((u64)bar1_block.ptr) << 12, target_to_text(bar1_block.target), bar1_block.is_virtual ? "virtual" : "physical"); | ||
61 | bar2_block.raw = nvdebug_readl(g, NV_PBUS_BAR2_BLOCK); | ||
62 | printk(KERN_INFO "[nvdebug] BAR2 inst block @ %llx in %s's %s address space.\n", ((u64)bar2_block.ptr) << 12, target_to_text(bar2_block.target), bar1_block.is_virtual ? "virtual" : "physical"); | ||
63 | uint32_t bar_inst_pramin_offset = vram2PRAMIN(g, (uint64_t)bar2_block.ptr << 12); | ||
64 | if (!bar_inst_pramin_offset) { | ||
65 | printk(KERN_WARNING "[nvdebug] Unable to find instance block for BAR2/3 in the current NV_PRAMIN window. VRAM inaccessible.\n"); | ||
66 | goto attempt_pramin_access; | ||
67 | } | ||
68 | /* TODO: Support BAR1? | ||
69 | bar_inst_pramin_offset = vram2PRAMIN(g, bar1_block.ptr << 12); | ||
70 | if (!bar_inst_pramin_offset) { | ||
71 | printk(KERN_WARNING "[nvdebug] Unable to find instance block for BAR1 in the current NV_PRAMIN window. VRAM inaccessible.\n"); | ||
72 | return -EOPNOTSUPP; | ||
73 | }*/ | ||
74 | // Instance blocks (size == 1kb) contain many things, but we only care about | ||
75 | // the section which describes the location of the page directory (page table) | ||
76 | uint32_t bar_pdb_config_pramin_offset = bar_inst_pramin_offset + NV_PRAMIN_PDB_CONFIG_OFF; | ||
77 | page_dir_config_t pd_config; | ||
78 | pd_config.raw = nvdebug_readq(g, bar_pdb_config_pramin_offset + NV_PRAMIN); | ||
79 | uint64_t bar_pdb_vram_addr = pd_config.page_dir_hi; | ||
80 | bar_pdb_vram_addr <<= 20; | ||
81 | bar_pdb_vram_addr |= pd_config.page_dir_lo; | ||
82 | bar_pdb_vram_addr <<= 12; | ||
83 | printk(KERN_INFO "[nvdebug] BAR2 PDB @ %llx in %s of version %s (config raw: %llx)\n", bar_pdb_vram_addr, target_to_text(pd_config.target), pd_config.is_ver2 ? "2" : "1", pd_config.raw); | ||
84 | // TODO: SYSMEM support for page table location | ||
85 | if (pd_config.target != TARGET_VID_MEM) { | ||
86 | printk(KERN_WARNING "[nvdebug] BAR2 PDB is in an unsupported location.\n"); | ||
87 | goto attempt_pramin_access; | ||
88 | } | ||
89 | uint32_t bar_pdb_pramin_offset = vram2PRAMIN(g, bar_pdb_vram_addr); | ||
90 | if (!bar_pdb_pramin_offset) { | ||
91 | printk(KERN_WARNING "[nvdebug] Unable to find page directory BAR2/3 in the current NV_PRAMIN window. VRAM inaccessible.\n"); | ||
92 | goto attempt_pramin_access; | ||
93 | } | ||
94 | uint64_t runlist_bar_vaddr; | 58 | uint64_t runlist_bar_vaddr; |
95 | if (pd_config.is_ver2) | 59 | |
96 | runlist_bar_vaddr = search_page_directory(g, g->regs + NV_PRAMIN + bar_pdb_pramin_offset, phy2PRAMIN, runlist_iova); | 60 | if (get_bar2_pdb(g, &bar2_page_dir, &pdb_is_ver2) < 0) |
61 | return -EIO; | ||
62 | |||
63 | if (pdb_is_ver2) | ||
64 | runlist_bar_vaddr = search_page_directory(g, bar2_page_dir, phy2PRAMIN, runlist_iova); | ||
97 | else | 65 | else |
98 | runlist_bar_vaddr = search_v1_page_directory(g, g->regs + NV_PRAMIN + bar_pdb_pramin_offset, phy2PRAMIN, runlist_iova); | 66 | runlist_bar_vaddr = search_v1_page_directory(g, bar2_page_dir, phy2PRAMIN, runlist_iova); |
99 | if (!runlist_bar_vaddr) { | 67 | if (!runlist_bar_vaddr) { |
100 | printk(KERN_WARNING "[nvdebug] Unable to find runlist mapping in BAR2/3 page tables.\n"); | 68 | printk(KERN_WARNING "[nvdebug] Unable to find runlist mapping in BAR2/3 page tables.\n"); |
101 | goto attempt_pramin_access; | 69 | goto attempt_pramin_access; |
102 | } | 70 | } |
103 | printk(KERN_INFO "[nvdebug] Runlist @ %llx in BAR2 virtual address space.\n", runlist_bar_vaddr); | 71 | printk(KERN_INFO "[nvdebug] Runlist @ %llx in BAR2 virtual address space.\n", runlist_bar_vaddr); |
104 | /* XXX: Old test code | 72 | if (!g->bar2) { |
105 | uint32_t bar2_pd_pramin_offset = vram_to_pramin_off(bar2_pd); | ||
106 | //walk_pd_subtree(bar2_pd_pramin_offset); | ||
107 | uint64_t runlist_bar2_vaddr = search_pd_subtree(bar2_pd_pramin_offset, runlist_iova); | ||
108 | page_dir_entry_t pde_0; | ||
109 | pde_0.raw = nvdebug_readl(g, NV_PRAMIN + bar2_pd_pramin_offset); | ||
110 | uint32_t pde_1 = nvdebug_readl(g, NV_PRAMIN + vram_to_pramin_off(((u64)pde_0.addr) << 12)); | ||
111 | uint64_t pde_bar2_vaddr = search_pd_subtree(bar2_pd_pramin_offset, ((u64)pde_0.addr) << 12); | ||
112 | uint32_t pde_2 = readl(g->bar3 + pde_bar2_vaddr); | ||
113 | printk(KERN_INFO "[nvdebug] PDE0 via PRAMIN: %x, via BAR3: %x\n", pde_1, pde_2); | ||
114 | */ | ||
115 | if (!g->bar3) { | ||
116 | printk(KERN_WARNING "[nvdebug] BAR2/3 not mapped.\n"); | 73 | printk(KERN_WARNING "[nvdebug] BAR2/3 not mapped.\n"); |
117 | return -ENODEV; | 74 | return -ENODEV; |
118 | } | 75 | } |
119 | rl_iter->curr_entry = g->bar2 + runlist_bar_vaddr; | 76 | rl_iter->curr_entry = g->bar2 + runlist_bar_vaddr; |
120 | } else { | 77 | } else { |
121 | // Directly access the runlist if stored in SYS_MEM (physically addressed) | 78 | // Directly access the runlist if stored in SYS_MEM (physically addressed) |
122 | rl_iter->curr_entry = phys_to_virt(runlist_iova); | 79 | // XXX: SYS_MEM is an IOMMU address on some platforms, causing this to crash |
80 | rl_iter->curr_entry = (void*)phys_to_virt(runlist_iova); | ||
123 | } | 81 | } |
124 | rl_iter->rl_info = rl_info; | 82 | rl_iter->len = runlist_len; |
125 | return 0; | 83 | return 0; |
84 | |||
126 | attempt_pramin_access: | 85 | attempt_pramin_access: |
127 | #ifdef FALLBACK_TO_PRAMIN | 86 | #ifdef FALLBACK_TO_PRAMIN |
128 | printk(KERN_INFO "[nvdebug] Attempting to move PRAMIN window to runlist as BAR2/3-based access failed [DANGEROUS SIDE EFFECTS]!\n"); | 87 | printk(KERN_INFO "[nvdebug] Attempting to move PRAMIN window to runlist as BAR2/3-based access failed [DANGEROUS SIDE EFFECTS]!\n"); |
129 | bar0_window_t win; | 88 | if ((off = addr_to_pramin_mut(g, runlist_iova, runlist_target)) == -1) |
130 | win.base = (runlist_iova >> 16); | 89 | return off; |
131 | win.target = TARGET_VID_MEM; | ||
132 | // Shift PRAMIN window. This will cause problems if it races with driver code | ||
133 | // that tries to do the same, or expects the window not to move. | ||
134 | nvdebug_writel(g, NV_PBUS_BAR0_WINDOW, win.raw); | ||
135 | uint32_t off = vram2PRAMIN(g, runlist_iova); | ||
136 | // Workaround bug for if `off` should be zero (vram2PRAMIN normally returns | ||
137 | // this on error) | ||
138 | if (!off && (runlist_iova & 0xffffull != runlist_iova)) { | ||
139 | printk(KERN_INFO "[nvdebug] Unable to shift PRAMIN to runlist. Aborting...\n"); | ||
140 | return -EOPNOTSUPP; | ||
141 | } | ||
142 | rl_iter->curr_entry = g->regs + NV_PRAMIN + off; | 90 | rl_iter->curr_entry = g->regs + NV_PRAMIN + off; |
143 | rl_iter->rl_info = rl_info; | 91 | rl_iter->len = runlist_len; |
144 | return 0; | 92 | return 0; |
145 | #else | 93 | #else |
146 | return -EOPNOTSUPP; | 94 | return -EOPNOTSUPP; |
147 | #endif // FALLBACK_TO_PRAMIN | 95 | #endif // FALLBACK_TO_PRAMIN |
148 | } | 96 | } |
149 | 97 | ||
98 | /* Trigger a preempt of the specified TSG | ||
99 | @param tsg_id ID of TSG to preempt. | ||
100 | @return 0 or -errno on error | ||
101 | |||
102 | Note: If no other TSGs exist in the associated runlist, this TSG may | ||
103 | continue executing, unless NV_PFIFO_SCHED_DISABLE is set, or all the | ||
104 | channels of the TSG to be preempted are disabled. | ||
105 | */ | ||
150 | int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) { | 106 | int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) { |
151 | runlist_info_t rl_info; | ||
152 | pfifo_preempt_t pfifo_preempt; | 107 | pfifo_preempt_t pfifo_preempt; |
153 | runlist_disable_t rl_disable; | 108 | if (g->chip_id < NV_CHIP_ID_KEPLER) |
154 | if (!g) | 109 | return -EOPNOTSUPP; |
155 | return -EIO; | 110 | |
156 | rl_info.raw = nvdebug_readl(g, NV_PFIFO_RUNLIST); | 111 | pfifo_preempt.raw = 0; |
157 | pfifo_preempt.id = tsg_id; | 112 | pfifo_preempt.id = tsg_id; |
158 | pfifo_preempt.is_pending = 0; | 113 | pfifo_preempt.is_pending = 0; |
159 | pfifo_preempt.type = PREEMPT_TYPE_TSG; | 114 | pfifo_preempt.type = PREEMPT_TYPE_TSG; |
160 | // There may be a bug (?) that requires us to disable scheduling before preempting | 115 | |
161 | rl_disable.raw = nvdebug_readl(g, NV_PFIFO_SCHED_DISABLE); | ||
162 | rl_disable.raw |= BIT(rl_info.id); // Disable runlist rl_info.id | ||
163 | nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw); | ||
164 | // Actually trigger the preemption | 116 | // Actually trigger the preemption |
165 | nvdebug_writel(g, NV_PFIFO_PREEMPT, pfifo_preempt.raw); | 117 | nvdebug_writel(g, NV_PFIFO_PREEMPT, pfifo_preempt.raw); |
166 | // Renable scheduling | 118 | return 0; |
167 | rl_disable.raw &= ~BIT(rl_info.id); // Enable runlist rl_info.id | 119 | } |
168 | nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw); | 120 | |
121 | /* Trigger a preempt of the specified runlist | ||
122 | @param rl_id ID of runlist to preempt. | ||
123 | @return 0 or -errno on error | ||
124 | */ | ||
125 | int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id) { | ||
126 | runlist_preempt_t rl_preempt; | ||
127 | if (g->chip_id < NV_CHIP_ID_VOLTA) | ||
128 | return -EOPNOTSUPP; | ||
169 | 129 | ||
170 | printk(KERN_INFO "[nvdebug] TSG %d preempted (runlist %d)\n", tsg_id, rl_info.id); | 130 | // Overwrite, as the register contains nothing to preserve |
131 | rl_preempt.raw = BIT(rl_id); | ||
132 | nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw); | ||
171 | return 0; | 133 | return 0; |
172 | } | 134 | } |