aboutsummaryrefslogtreecommitdiffstats
path: root/runlist.c
diff options
context:
space:
mode:
Diffstat (limited to 'runlist.c')
-rw-r--r--runlist.c212
1 files changed, 87 insertions, 125 deletions
diff --git a/runlist.c b/runlist.c
index ed35c7e..c725e77 100644
--- a/runlist.c
+++ b/runlist.c
@@ -1,172 +1,134 @@
1#include <linux/kernel.h> // Kernel types 1/* Copyright 2024 Joshua Bakita
2 * Helpers for dealing with the runlist and other Host (PFIFO) registers
3 */
4#include <linux/printk.h> // For printk()
5#include <asm/errno.h> // For error defines
6#include <asm/io.h> // For phys_to_virt()
2 7
3#include "nvdebug.h" 8#include "nvdebug.h"
4 9
10// Uncomment to, upon BAR2 access failure, return a PRAMIN-based runlist pointer
11// **If enabled, PRAMIN may not be otherwise used while walking the runlist!**
5#define FALLBACK_TO_PRAMIN 12#define FALLBACK_TO_PRAMIN
6 13
7/* Get runlist head and info (incl. length) 14/* Get runlist head and info (incl. length)
8 @param rl_iter Location at which to store output 15 @param rl_id Which runlist to obtain?
9 @param rl_id Which runlist to obtain? 16 @param rl_iter Location at which to store output
17 @return 0 or -errno on error
10*/ 18*/
11int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl_iter) { 19int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl_iter) {
12 runlist_base_t rl_base; 20 uint64_t runlist_iova;
13 runlist_info_t rl_info; 21 enum INST_TARGET runlist_target;
14 u64 runlist_iova; 22 uint16_t runlist_len;
23#ifdef FALLBACK_TO_PRAMIN
24 int off;
25#endif // FALLBACK_TO_PRAMIN
26 // Zero-initialize the runlist iterator
15 *rl_iter = (struct runlist_iter){0}; 27 *rl_iter = (struct runlist_iter){0};
16 rl_base.raw = nvdebug_readl(g, NV_PFIFO_ENG_RUNLIST_BASE(rl_id)); 28
17 // Check that reads are working 29 // Get runlist location and length using architecture-dependent logic
18 if (rl_base.raw == -1) 30 if (g->chip_id < NV_CHIP_ID_TURING) {
19 return -EIO; 31 eng_runlist_gf100_t rl;
20 // The address pointed to `regs` + NV_PFIFO_RUNLIST_BASE seems to not be: 32 if ((rl.raw = nvdebug_readq(g, NV_PFIFO_ENG_RUNLIST_BASE_GF100(rl_id))) == -1)
21 // - A GPU address (type is sysmem_coherent) 33 return -EIO;
22 // - A physical address (dereferencing after ioremap crashes) 34 runlist_iova = ((uint64_t)rl.ptr) << 12;
23 // - A kernel virtual address (dereferencing segfaults) 35 runlist_target = rl.target;
24 // So maybe it's some sort of custom thing? This is an address that the GPU 36 printk(KERN_INFO "[nvdebug] Runlist %d: %d entries @ %llx in %s (config raw: %#018llx)\n",
25 // can use, so it would make most sense for it to be a physical address. 37 rl_id, rl.len, runlist_iova, target_to_text(rl.target), rl.raw);
26 // 38 runlist_len = rl.len;
27 // BUT, it can't possibly be a physical address, as it would refer to an 39 } else if (g->chip_id < NV_CHIP_ID_AMPERE) {
28 // address greater than the maximum one on our system (by a lot!). 40 runlist_base_tu102_t base;
29 // Maybe I'm reading the runlist base wrong? 41 runlist_submit_tu102_t submit;
30 // Aha, the driver calls it runlist_iova. Sounds like runlist I/O virtual 42 if ((base.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_BASE_TU102(rl_id))) == -1)
31 // address! So, what's this I/O address space? All I know is that it's what 43 return -EIO;
32 // nvgpu_mem_get_addr() returns. That function returns the result of either: 44 if ((submit.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id))) == -1)
33 // - gpu_phys_addr which is __nvgpu_sgl_phys on our platform which (?) 45 return -EIO;
34 // converts an IPA to a PA? 46 runlist_iova = ((uint64_t)base.ptr) << 12;
35 // - nvgpu_mem_iommu_translate 47 runlist_target = base.target;
36 // 48 runlist_len = submit.len;
37 // The original memory is allocated with nvgpu_dma_alloc_flags_sys(), which 49 }
38 // returns SYSMEM.
39 //
40 // To convert a physical address to a IOMMU address, we add a bit
41 //
42 // BUT, it turns out that it IS JUST A PHYSICAL ADDRESS! It wasn't working
43 // before because the GPU had simply gone to sleep and invalidated its
44 // register state, so nvgpu_readl() was simply returning garbage.
45 rl_info.raw = nvdebug_readl(g, NV_PFIFO_ENG_RUNLIST(rl_id));
46 if (rl_info.raw == -1)
47 return -EIO;
48 runlist_iova = ((u64)rl_base.ptr) << 12;
49 printk(KERN_INFO "[nvdebug] Runlist %d @ %llx in %s (config raw: %x)\n",
50 rl_id, runlist_iova, target_to_text(rl_base.target), rl_base.raw);
51 printk(KERN_INFO "[nvdebug] Runlist length %d, ID %d\n", rl_info.len, rl_info.id);
52 // Return early on an empty runlist 50 // Return early on an empty runlist
53 if (!rl_info.len) 51 if (!runlist_len)
54 return 0; 52 return 0;
53
55 // If the runlist is in VID_MEM, search the BAR2/3 page tables for a mapping 54 // If the runlist is in VID_MEM, search the BAR2/3 page tables for a mapping
56 if (rl_base.target == TARGET_VID_MEM) { 55 if (runlist_target == TARGET_VID_MEM) {
57 printk(KERN_WARNING "[nvdebug] Runlist is located in video memory. Access to video memory is experimental."); 56 void __iomem *bar2_page_dir;
58 bar_config_block_t bar1_block, bar2_block; 57 bool pdb_is_ver2;
59 bar1_block.raw = nvdebug_readl(g, NV_PBUS_BAR1_BLOCK);
60 printk(KERN_INFO "[nvdebug] BAR1 inst block @ %llx in %s's %s address space.\n", ((u64)bar1_block.ptr) << 12, target_to_text(bar1_block.target), bar1_block.is_virtual ? "virtual" : "physical");
61 bar2_block.raw = nvdebug_readl(g, NV_PBUS_BAR2_BLOCK);
62 printk(KERN_INFO "[nvdebug] BAR2 inst block @ %llx in %s's %s address space.\n", ((u64)bar2_block.ptr) << 12, target_to_text(bar2_block.target), bar1_block.is_virtual ? "virtual" : "physical");
63 uint32_t bar_inst_pramin_offset = vram2PRAMIN(g, (uint64_t)bar2_block.ptr << 12);
64 if (!bar_inst_pramin_offset) {
65 printk(KERN_WARNING "[nvdebug] Unable to find instance block for BAR2/3 in the current NV_PRAMIN window. VRAM inaccessible.\n");
66 goto attempt_pramin_access;
67 }
68 /* TODO: Support BAR1?
69 bar_inst_pramin_offset = vram2PRAMIN(g, bar1_block.ptr << 12);
70 if (!bar_inst_pramin_offset) {
71 printk(KERN_WARNING "[nvdebug] Unable to find instance block for BAR1 in the current NV_PRAMIN window. VRAM inaccessible.\n");
72 return -EOPNOTSUPP;
73 }*/
74 // Instance blocks (size == 1kb) contain many things, but we only care about
75 // the section which describes the location of the page directory (page table)
76 uint32_t bar_pdb_config_pramin_offset = bar_inst_pramin_offset + NV_PRAMIN_PDB_CONFIG_OFF;
77 page_dir_config_t pd_config;
78 pd_config.raw = nvdebug_readq(g, bar_pdb_config_pramin_offset + NV_PRAMIN);
79 uint64_t bar_pdb_vram_addr = pd_config.page_dir_hi;
80 bar_pdb_vram_addr <<= 20;
81 bar_pdb_vram_addr |= pd_config.page_dir_lo;
82 bar_pdb_vram_addr <<= 12;
83 printk(KERN_INFO "[nvdebug] BAR2 PDB @ %llx in %s of version %s (config raw: %llx)\n", bar_pdb_vram_addr, target_to_text(pd_config.target), pd_config.is_ver2 ? "2" : "1", pd_config.raw);
84 // TODO: SYSMEM support for page table location
85 if (pd_config.target != TARGET_VID_MEM) {
86 printk(KERN_WARNING "[nvdebug] BAR2 PDB is in an unsupported location.\n");
87 goto attempt_pramin_access;
88 }
89 uint32_t bar_pdb_pramin_offset = vram2PRAMIN(g, bar_pdb_vram_addr);
90 if (!bar_pdb_pramin_offset) {
91 printk(KERN_WARNING "[nvdebug] Unable to find page directory BAR2/3 in the current NV_PRAMIN window. VRAM inaccessible.\n");
92 goto attempt_pramin_access;
93 }
94 uint64_t runlist_bar_vaddr; 58 uint64_t runlist_bar_vaddr;
95 if (pd_config.is_ver2) 59
96 runlist_bar_vaddr = search_page_directory(g, g->regs + NV_PRAMIN + bar_pdb_pramin_offset, phy2PRAMIN, runlist_iova); 60 if (get_bar2_pdb(g, &bar2_page_dir, &pdb_is_ver2) < 0)
61 return -EIO;
62
63 if (pdb_is_ver2)
64 runlist_bar_vaddr = search_page_directory(g, bar2_page_dir, phy2PRAMIN, runlist_iova);
97 else 65 else
98 runlist_bar_vaddr = search_v1_page_directory(g, g->regs + NV_PRAMIN + bar_pdb_pramin_offset, phy2PRAMIN, runlist_iova); 66 runlist_bar_vaddr = search_v1_page_directory(g, bar2_page_dir, phy2PRAMIN, runlist_iova);
99 if (!runlist_bar_vaddr) { 67 if (!runlist_bar_vaddr) {
100 printk(KERN_WARNING "[nvdebug] Unable to find runlist mapping in BAR2/3 page tables.\n"); 68 printk(KERN_WARNING "[nvdebug] Unable to find runlist mapping in BAR2/3 page tables.\n");
101 goto attempt_pramin_access; 69 goto attempt_pramin_access;
102 } 70 }
103 printk(KERN_INFO "[nvdebug] Runlist @ %llx in BAR2 virtual address space.\n", runlist_bar_vaddr); 71 printk(KERN_INFO "[nvdebug] Runlist @ %llx in BAR2 virtual address space.\n", runlist_bar_vaddr);
104 /* XXX: Old test code 72 if (!g->bar2) {
105 uint32_t bar2_pd_pramin_offset = vram_to_pramin_off(bar2_pd);
106 //walk_pd_subtree(bar2_pd_pramin_offset);
107 uint64_t runlist_bar2_vaddr = search_pd_subtree(bar2_pd_pramin_offset, runlist_iova);
108 page_dir_entry_t pde_0;
109 pde_0.raw = nvdebug_readl(g, NV_PRAMIN + bar2_pd_pramin_offset);
110 uint32_t pde_1 = nvdebug_readl(g, NV_PRAMIN + vram_to_pramin_off(((u64)pde_0.addr) << 12));
111 uint64_t pde_bar2_vaddr = search_pd_subtree(bar2_pd_pramin_offset, ((u64)pde_0.addr) << 12);
112 uint32_t pde_2 = readl(g->bar3 + pde_bar2_vaddr);
113 printk(KERN_INFO "[nvdebug] PDE0 via PRAMIN: %x, via BAR3: %x\n", pde_1, pde_2);
114 */
115 if (!g->bar3) {
116 printk(KERN_WARNING "[nvdebug] BAR2/3 not mapped.\n"); 73 printk(KERN_WARNING "[nvdebug] BAR2/3 not mapped.\n");
117 return -ENODEV; 74 return -ENODEV;
118 } 75 }
119 rl_iter->curr_entry = g->bar2 + runlist_bar_vaddr; 76 rl_iter->curr_entry = g->bar2 + runlist_bar_vaddr;
120 } else { 77 } else {
121 // Directly access the runlist if stored in SYS_MEM (physically addressed) 78 // Directly access the runlist if stored in SYS_MEM (physically addressed)
122 rl_iter->curr_entry = phys_to_virt(runlist_iova); 79 // XXX: SYS_MEM is an IOMMU address on some platforms, causing this to crash
80 rl_iter->curr_entry = (void*)phys_to_virt(runlist_iova);
123 } 81 }
124 rl_iter->rl_info = rl_info; 82 rl_iter->len = runlist_len;
125 return 0; 83 return 0;
84
126attempt_pramin_access: 85attempt_pramin_access:
127#ifdef FALLBACK_TO_PRAMIN 86#ifdef FALLBACK_TO_PRAMIN
128 printk(KERN_INFO "[nvdebug] Attempting to move PRAMIN window to runlist as BAR2/3-based access failed [DANGEROUS SIDE EFFECTS]!\n"); 87 printk(KERN_INFO "[nvdebug] Attempting to move PRAMIN window to runlist as BAR2/3-based access failed [DANGEROUS SIDE EFFECTS]!\n");
129 bar0_window_t win; 88 if ((off = addr_to_pramin_mut(g, runlist_iova, runlist_target)) == -1)
130 win.base = (runlist_iova >> 16); 89 return off;
131 win.target = TARGET_VID_MEM;
132 // Shift PRAMIN window. This will cause problems if it races with driver code
133 // that tries to do the same, or expects the window not to move.
134 nvdebug_writel(g, NV_PBUS_BAR0_WINDOW, win.raw);
135 uint32_t off = vram2PRAMIN(g, runlist_iova);
136 // Workaround bug for if `off` should be zero (vram2PRAMIN normally returns
137 // this on error)
138 if (!off && (runlist_iova & 0xffffull != runlist_iova)) {
139 printk(KERN_INFO "[nvdebug] Unable to shift PRAMIN to runlist. Aborting...\n");
140 return -EOPNOTSUPP;
141 }
142 rl_iter->curr_entry = g->regs + NV_PRAMIN + off; 90 rl_iter->curr_entry = g->regs + NV_PRAMIN + off;
143 rl_iter->rl_info = rl_info; 91 rl_iter->len = runlist_len;
144 return 0; 92 return 0;
145#else 93#else
146 return -EOPNOTSUPP; 94 return -EOPNOTSUPP;
147#endif // FALLBACK_TO_PRAMIN 95#endif // FALLBACK_TO_PRAMIN
148} 96}
149 97
98/* Trigger a preempt of the specified TSG
99 @param tsg_id ID of TSG to preempt.
100 @return 0 or -errno on error
101
102 Note: If no other TSGs exist in the associ