aboutsummaryrefslogtreecommitdiffstats
path: root/mmu.c
diff options
context:
space:
mode:
authorJoshua Bakita <bakitajoshua@gmail.com>2024-04-22 18:46:26 -0400
committerJoshua Bakita <bakitajoshua@gmail.com>2024-04-22 18:46:26 -0400
commit968db9791dd8efd3526d7c47a751a54e1fa95eb4 (patch)
tree63d4c058ed87a8bc53df8cb89be1c6076ce6ee3b /mmu.c
parent091c242c9ef7cbd8d88d3beae936b14f5b907286 (diff)
Fix page-table traversal for version 1 page tables
- Correct V1 page table defines using information from kern_gmmu_fmt_gm10x.c in NVIDIA's open-gpu-kernel-modules repo. - Verify page table format in search_v1_page_directory() - Better, more controllable logging in mmu.c Tested on GM204 (GTX 970).
Diffstat (limited to 'mmu.c')
-rw-r--r--mmu.c44
1 files changed, 30 insertions, 14 deletions
diff --git a/mmu.c b/mmu.c
index eaf7d5f..6784b9f 100644
--- a/mmu.c
+++ b/mmu.c
@@ -7,13 +7,13 @@
7 7
8#include "nvdebug.h" 8#include "nvdebug.h"
9 9
10// Uncomment to print every PDE and PTE walked for debugging 10/* Set logging level for MMU operations
11//#define DEBUG 11 g_verbose >= 1: Log a single message describing the MMU operation
12#ifdef DEBUG 12 g_verbose >= 2: Log every PDE and PTE traversed
13#define printk_debug printk 13*/
14#else 14int g_verbose = 0;
15#define printk_debug(...) 15#define printk_debug if (g_verbose >= 2) printk
16#endif 16#define printk_info if (g_verbose >= 1) printk
17 17
18/* Convert a page directory (PD) pointer and aperture to be kernel-accessible 18/* Convert a page directory (PD) pointer and aperture to be kernel-accessible
19 19
@@ -24,7 +24,7 @@
24 @param pd_ap PD-type aperture (target address space) for `addr` 24 @param pd_ap PD-type aperture (target address space) for `addr`
25 @return A dereferencable kernel address, or an ERR_PTR-wrapped error 25 @return A dereferencable kernel address, or an ERR_PTR-wrapped error
26 */ 26 */
27void __iomem *pd_deref(struct nvdebug_state *g, uintptr_t addr, enum PD_TARGET pd_ap) { 27static void __iomem *pd_deref(struct nvdebug_state *g, uintptr_t addr, enum PD_TARGET pd_ap) {
28 struct iommu_domain *dom; 28 struct iommu_domain *dom;
29 phys_addr_t phys; 29 phys_addr_t phys;
30 30
@@ -55,7 +55,7 @@ void __iomem *pd_deref(struct nvdebug_state *g, uintptr_t addr, enum PD_TARGET p
55 // Check for, and translate through, the I/O MMU (if any) 55 // Check for, and translate through, the I/O MMU (if any)
56 if ((dom = iommu_get_domain_for_dev(g->dev))) { 56 if ((dom = iommu_get_domain_for_dev(g->dev))) {
57 phys = iommu_iova_to_phys(dom, addr); 57 phys = iommu_iova_to_phys(dom, addr);
58 printk(KERN_ERR "[nvdebug] I/O MMU translated SYS_MEM I/O VA %#lx to physical address %llx.\n", addr, phys); 58 printk_debug(KERN_DEBUG "[nvdebug] I/O MMU translated SYS_MEM I/O VA %#lx to physical address %#llx.\n", addr, phys);
59 } else 59 } else
60 phys = addr; 60 phys = addr;
61 61
@@ -94,13 +94,14 @@ uint64_t search_page_directory_subtree(struct nvdebug_state *g,
94 // Succeed when we reach a PTE with the address we want 94 // Succeed when we reach a PTE with the address we want
95 if (entry.is_pte) { 95 if (entry.is_pte) {
96 // TODO: Handle huge pages here 96 // TODO: Handle huge pages here
97 printk_debug(KERN_INFO "[nvdebug] PTE for phy addr %#018llx, ap '%s', vol '%d', priv '%d', ro '%d', no_atomics '%d' (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, entry.is_privileged, entry.is_readonly, entry.atomics_disabled, entry.raw_w); 97 printk_debug(KERN_DEBUG "[nvdebug] PTE for phy addr %#018llx, ap '%s', vol '%d', priv '%d', ro '%d', no_atomics '%d' (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, entry.is_privileged, entry.is_readonly, entry.atomics_disabled, entry.raw_w);
98 return (uint64_t)entry.addr << 12 == addr_to_find && entry.aperture == addr_to_find_aperture; 98 return (uint64_t)entry.addr << 12 == addr_to_find && entry.aperture == addr_to_find_aperture;
99 } 99 }
100 printk_debug(KERN_INFO "[nvdebug] Found PDE pointing to %#018llx in ap '%s' vol '%d' at lvl %d (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, level, entry.raw_w); 100 printk_debug(KERN_DEBUG "[nvdebug] Found PDE pointing to %#018llx in ap '%s' vol '%d' at lvl %d (raw: %#018llx)\n", ((u64)entry.addr_w) << 12, pd_target_to_text(entry.target), entry.is_volatile, level, entry.raw_w);
101 // Depth-first search of the page table 101 // Depth-first search of the page table
102 for (i = 0; i < NV_MMU_PT_V2_SZ[level + 1]; i++) { 102 for (i = 0; i < NV_MMU_PT_V2_SZ[level + 1]; i++) {
103 uint64_t next = ((uint64_t)entry.addr << 12) + NV_MMU_PT_V2_ENTRY_SZ[level + 1] * i; 103 uint64_t next = ((uint64_t)entry.addr << 12) + NV_MMU_PT_V2_ENTRY_SZ[level + 1] * i;
104 printk_debug(KERN_DEBUG "[nvdebug] Searching index %llu in lvl %d\n", i, level + 1);
104 res = search_page_directory_subtree(g, next, entry.target, addr_to_find, addr_to_find_aperture, level + 1); 105 res = search_page_directory_subtree(g, next, entry.target, addr_to_find, addr_to_find_aperture, level + 1);
105 if (res) 106 if (res)
106 return res | (i << NV_MMU_PT_V2_LSB[level + 1]); 107 return res | (i << NV_MMU_PT_V2_LSB[level + 1]);
@@ -134,7 +135,7 @@ uint64_t search_page_directory(struct nvdebug_state *g,
134 printk(KERN_WARNING "[nvdebug] Attempting to search for unaligned address %llx in search_page_directory()!\n", addr_to_find); 135 printk(KERN_WARNING "[nvdebug] Attempting to search for unaligned address %llx in search_page_directory()!\n", addr_to_find);
135 return 0; 136 return 0;
136 } 137 }
137 printk(KERN_INFO "[nvdebug] Searching for addr %#018llx in page table with base %#018lx\n", addr_to_find, (uintptr_t)pd_config.page_dir << 12); 138 printk_info(KERN_INFO "[nvdebug] Searching for addr %#018llx in page table with base %#018lx\n", addr_to_find, (uintptr_t)pd_config.page_dir << 12);
138 // Search the top-level page directory (PDE3) 139 // Search the top-level page directory (PDE3)
139 for (i = 0; i < NV_MMU_PT_V2_SZ[0]; i++) 140 for (i = 0; i < NV_MMU_PT_V2_SZ[0]; i++)
140 if ((res = search_page_directory_subtree(g, ((uintptr_t)pd_config.page_dir << 12) + NV_MMU_PT_V2_ENTRY_SZ[0] * i, INST2PD_TARGET(pd_config.target), addr_to_find, addr_to_find_aperture, 0))) 141 if ((res = search_page_directory_subtree(g, ((uintptr_t)pd_config.page_dir << 12) + NV_MMU_PT_V2_ENTRY_SZ[0] * i, INST2PD_TARGET(pd_config.target), addr_to_find, addr_to_find_aperture, 0)))
@@ -154,6 +155,21 @@ uint64_t search_v1_page_directory(struct nvdebug_state *g,
154 page_tbl_entry_v1_t pte; 155 page_tbl_entry_v1_t pte;
155 uintptr_t pte_offset, pde_offset; 156 uintptr_t pte_offset, pde_offset;
156 void __iomem *pte_addr, *pde_addr; 157 void __iomem *pte_addr, *pde_addr;
158
159 // This function only understands the Page Table Version 1 format
160 if (pd_config.is_ver2) {
161 printk(KERN_ERR "[nvdebug] Passed a Version 2 page table at %#018llx to translate_v1_page_directory()!\n", (uint64_t)pd_config.page_dir << 12);
162 return 0;
163 }
164
165 // We only understand the Version 1 format when 128 KiB huge pages are in-use
166 if (pd_config.is_64k_big_page) {
167 printk(KERN_ERR "[nvdebug] Page Table Version 1 with 64 KiB huge pages is unsupported!\n");
168 return 0;
169 }
170
171 printk_info(KERN_INFO "[nvdebug] Searching V1 page table at %#018lx in %s for addr %#018llx\n", (uintptr_t)pd_config.page_dir << 12, target_to_text(pd_config.target), addr_to_find);
172
157 // For each PDE 173 // For each PDE
158 do { 174 do {
159 // Index the list of page directory entries 175 // Index the list of page directory entries
@@ -172,7 +188,7 @@ uint64_t search_v1_page_directory(struct nvdebug_state *g,
172 if (pde.target == PD_TARGET_INVALID && pde.alt_target == PD_TARGET_INVALID) 188 if (pde.target == PD_TARGET_INVALID && pde.alt_target == PD_TARGET_INVALID)
173 continue; 189 continue;
174// printk(KERN_INFO "[nvdebug] Found %s PDE pointing to PTEs @ %llx in ap '%d' (raw: %llx)\n", pde.is_volatile ? "volatile" : "non-volatile", ((u64)pde.addr) << 12, pde.target, pde.raw); 190// printk(KERN_INFO "[nvdebug] Found %s PDE pointing to PTEs @ %llx in ap '%d' (raw: %llx)\n", pde.is_volatile ? "volatile" : "non-volatile", ((u64)pde.addr) << 12, pde.target, pde.raw);
175 printk_debug(KERN_INFO "[nvdebug] Found %s PDE pointing to PTEs @ %llx in ap '%d' (raw: %llx)\n", pde.alt_is_volatile ? "volatile" : "non-volatile", ((u64)pde.alt_addr) << 12, pde.alt_target, pde.raw); 191 printk_debug(KERN_DEBUG "[nvdebug] Found %s PDE at index %lld pointing to PTEs @ %#018llx in ap '%d' (raw: %#018llx)\n", pde.alt_is_volatile ? "volatile" : "non-volatile", i, ((u64)pde.alt_addr) << 12, pde.alt_target, pde.raw);
176 // For each PTE 192 // For each PTE
177 for (j = 0; j < NV_MMU_PT_V1_SZ[1]; j++) { 193 for (j = 0; j < NV_MMU_PT_V1_SZ[1]; j++) {
178 // Index the list of page table entries starting at pde.alt_addr 194 // Index the list of page table entries starting at pde.alt_addr
@@ -190,7 +206,7 @@ uint64_t search_v1_page_directory(struct nvdebug_state *g,
190 // Skip non-present PTEs 206 // Skip non-present PTEs
191 if (!pte.is_present) 207 if (!pte.is_present)
192 continue; 208 continue;
193 printk_debug(KERN_INFO "[nvdebug] PTE for phy addr %llx %s (raw: %llx)\n", ((u64)pte.addr) << 12, pte.is_present ? "present" : "non-present", pte.raw); 209 printk_debug(KERN_DEBUG "[nvdebug] PTE for phy addr %#018llx, ap '%s', vol '%d', priv '%d', ro '%d', no_atomics '%d' (raw: %#018llx)\n", ((u64)pte.addr) << 12, target_to_text(pte.target), pte.is_volatile, pte.is_privileged, pte.is_readonly, pte.atomics_disabled, pte.raw);
194 // If we find a matching PTE, return its virtual address 210 // If we find a matching PTE, return its virtual address
195 if ((uint64_t)pte.addr << 12 == addr_to_find && pte.target == addr_to_find_aperture) 211 if ((uint64_t)pte.addr << 12 == addr_to_find && pte.target == addr_to_find_aperture)
196 return i << NV_MMU_PT_V1_LSB[0] | j << NV_MMU_PT_V1_LSB[1]; 212 return i << NV_MMU_PT_V1_LSB[0] | j << NV_MMU_PT_V1_LSB[1];