1 files changed, 124 insertions, 34 deletions

diff --git a/nvdebug.h b/nvdebug.h
index 567806d..eff1470 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -818,6 +818,14 @@ typedef union {
 } bar_config_block_t;
 
 /* BAR0 PRAMIN (Private RAM Instance) window configuration
+  One of the oldest ways to access video memory on NVIDIA GPUs is by using
+  a configurable 1MB window into VRAM which is mapped into BAR0 (register)
+  space starting at offset NV_PRAMIN. This is still supported on NVIDIA GPUs
+  and appear to be used today to bootstrap page table configuration.
+
+  Why is it mapped at a location called NVIDIA Private RAM Instance? Because
+  this used to point to the entirety of intance RAM, which was seperate from
+  VRAM on older NVIDIA GPUs.
 
   BASE    : Base of window >> 16 in [TARGET] virtual address space
   TARGET  : Which address space BASE points into
@@ -843,7 +851,7 @@ typedef union {
 typedef union {
         struct {
                 uint32_t target:2;
-                uint32_t vol:1;
+                uint32_t is_volatile:1;
                  uint32_t padding0:1;
                 uint32_t fault_replay_tex:1;
                 uint32_t fault_replay_gcc:1;
@@ -853,6 +861,10 @@ typedef union {
                 uint32_t page_dir_lo:20;
                 uint32_t page_dir_hi:32;
         } __attribute__((packed));
+        struct {
+                uint32_t pad:12;
+                uint64_t page_dir:52; // Confirmed working on Xavier and tama
+        } __attribute__((packed));
         uint64_t raw;
 } page_dir_config_t;
 
@@ -888,6 +900,14 @@ typedef union {
   The following arrays merely represent different projections of Fig. 1, and
   only one is strictly needed to reconstruct all the others. However, due to
   the complexity of page tables, we include all of these to aid in readability.
+
+  Support: Pascal, Volta, Turing, Ampere, Ada, Ampere, Hopper*, Blackwell*
+  Note: *Hopper introduces Version 3 Page Tables, but is backwards-compatible.
+         The newer version adds a PD4 level to support 57-bit virtual
+         addresses, and slightly shifts the PDE and PTE fields.
+
+  See also: gp100-mmu-format.pdf in open-gpu-doc. In open-gpu-kernel-modules
+            this is synonymously the "NEW" and "VER2" layout.
 */
 // How many nodes/entries per level in V2 of NVIDIA's page table format
 static const int NV_MMU_PT_V2_SZ[5] = {4, 512, 512, 256, 512};
@@ -907,6 +927,12 @@ enum PD_TARGET {
         PTE_AND_TARGET_SYS_MEM_COHERENT = 5,  // b101
         PTE_AND_TARGET_SYS_MEM_NONCOHERENT = 7,  // b111
 };
+// The low bit is unset on page directory (PD) targets
+#define IS_PD_TARGET(target) (!(target & 0x1u))
+// Convert from an enum INST_TARGET to an enum PD_TARGET
+#define INST2PD_TARGET(target) ((target & 0x2) ? (target << 1) : (!target) << 1)
+// Convert from an enum V1_PD_TARGET to an enum PD_TARGET
+#define V12PD_TARGET(target) (target << 1)
 static inline const char *pd_target_to_text(enum PD_TARGET t) {
         switch (t) {
                 case PD_AND_TARGET_INVALID:
@@ -928,13 +954,10 @@ static inline const char *pd_target_to_text(enum PD_TARGET t) {
 }
 
 // Page Directory Entry/Page Table Entry V2 type
-// Note: As the meaning of target (bits 2:1) changes depending on if the entry
-//       is a PTE or not, this combines them into a single target field to
-//       simplify comparisons.
-// Support: Pascal, Volta, Turing, Ampere, Ada
-//
-// V3 introduced with Hopper, but Hopper and Blackwell also support V2
-//
+// Note: As the meaning of target (bits 2:1) at a PDE-level changes if the
+//       entry is a large-page PTE or not. To simply the logic, we combine them
+//       into a single target field to simplify comparisons.
+#define TARGET_PEER 1
 typedef union {
         // Page Directory Entry (PDE)
         struct {
@@ -965,21 +988,74 @@ typedef union {
         uint64_t raw_w;
 } page_dir_entry_t;
 
-// Page Directory Entry/Page Table Entry V1 type
-// Support: Fermi, Kepler, Maxwell
+/* GMMU Page Tables Version 1
+  These page tables contain 2 levels and are used in the Fermi, Kepler, and
+  Maxwell architectures to support a 40-bit virtual address space.
+
+  Version 1 Page Tables may be configured to support either 64 KiB or 128 KiB
+  large pages. Table addressing differs between the modes---even if the table
+  contains no large pages. The format for 4 KiB pages in each mode is shown
+  below.
+
+  V1 of NVIDIA's page table format uses 1 level of PDEs and a level of PTEs.
+  How the virtual address is sliced to yield an index into each level and a
+  page offset is shown by Fig 1 and Fig 2 (for 64 KiB and 128 KiB large page
+  modes respectively).
+
+  == Figure 1: 64 KiB mode ==
+  Page Offset (12 bits) <----------------------------------+
+  Page Table Entry (PTE) (13 bits) <--------------+        |
+  Page Directory Entry (PDE) (13 bits) <-+        |        |
+                                         ^        ^        ^
+                     Virtual address: [39, 25] [24, 12] [11, 0]
+
+  == Figure 2: 128 KiB mode ==
+  Page Offset (12 bits) <----------------------------------+
+  Page Table Entry (PTE) (14 bits) <--------------+        |
+  Page Directory Entry (PDE) (12 bits) <-+        |        |
+                                         ^        ^        ^
+                     Virtual address: [39, 26] [25, 12] [11, 0]
+
+
+  Support: Fermi, Kepler, Maxwell, Pascal*
+  Note: *Pascal introduces Version 2 Page Tables, but is backwards-compatible.
+  Note: We only implement the 64-KiB-large-page mode in nvdebug.
+
+  See also: mm_gk20a.c in nvgpu (Jetson GPU driver) and kern_gmmu_fmt_gm10x.c
+            in open-gpu-kernel-modules (open-source NVRM variant). This is
+            synonymously the "VER1" and unversioned layout in
+            open-gpu-kernel-modules, with some differences noted in Appdx 1.
+
+  == Appdx 1 ==
+  In open-gpu-kernel-modules, the unversioned MMU layout adds:
+  - Bit 35: NV_MMU_PTE_LOCK synonym for NV_MMU_PTE_ATOMIC_DISABLE
+  - Bit 62: NV_MMU_PTE_READ_DISABLE overlapping NV_MMU_PTE_COMPTAGLINE
+  - Bit 63: NV_MMU_PTE_WRITE_DISABLE overlapping NV_MMU_PTE_COMPTAGLINE
+  And removes:
+  - Bit 40, 41, 42, 43 from NV_MMU_PTE_KIND
+  The PDE layouts are identical. Given that the unversioned defines seem to
+  predate renaming and/or field extension/relocation, they are likely artifacts
+  from the page table development process, and have no meaning now.
+*/
+// Number of entries in the PDE and PTE levels
+static const int NV_MMU_PT_V1_SZ[2] = {8192, 8192};
+// Which bit index is the least significant in indexing each page level
+static const int NV_MMU_PT_V1_LSB[2] = {25, 12};
+
+// V1 Page Directory Entry target
 enum V1_PD_TARGET {
         PD_TARGET_INVALID = 0,
         PD_TARGET_VID_MEM = 1,
         PD_TARGET_SYS_MEM_COHERENT = 2,
         PD_TARGET_SYS_MEM_NONCOHERENT = 3,
 };
-// Page Directory Entry (PDE)
+// V1 Page Directory Entry (PDE)
 typedef union {
 // Large page fields
         struct {
 // 0:32
                 enum V1_PD_TARGET target:2;
-                 uint32_t padding0:2;
+                 uint32_t padding0:2; // Documented as "PDE_SIZE"?
                 uint64_t addr:28;  // May be wider?
 // 32:63
                  uint32_t padding2:3;
@@ -998,45 +1074,58 @@ typedef union {
         } __attribute__((packed));
         uint64_t raw;
 } page_dir_entry_v1_t;
-// Page Table Entry (PTE)
-// Reconstructed from info in Jetson nvgpu driver
+
+// V1 Page Table Entry (PTE)
 typedef union {
         struct {
 // 0:32
                 bool is_present:1;
                 bool is_privileged:1;
                 bool is_readonly:1;
-                 uint32_t padding0:1;
+                bool is_encrypted:1;
                 uint64_t addr:28;
 // 32:63
                 bool is_volatile:1;
                 enum INST_TARGET:2;
-                 uint32_t padding1:1;
+                bool atomics_disabled:1;
                 uint32_t kind:8;
-                uint32_t comptag:17;
-                 uint32_t padding2:1;
-                bool is_read_disabled:1;
-                bool is_write_disabled:1;
+                uint32_t comptag:20;
         } __attribute__((packed));
         uint64_t raw;
 } page_tbl_entry_v1_t;
-//enum V0_PDE_TYPE {NOT_PRESENT = 0, PAGE_64K = 1, PAGE_16K = 2, PAGE_4K = 3};
-//enum V0_PDE_SIZE {PDE_SZ_128K = 0, PDE_SZ_32K = 1, PDE_SZ_16K = 2, PDE_SZ_8K = 3};
-//static const int V0_PDE_SIZE2NUM[4] = {128*1024, 32*1024, 16*1024, 8*1024};
-/* PDE V0 (nv50/Tesla)
+
+/* GMMU Page Tables Version 0
+  This page table contains 2 levels to support a 40-bit virtual address space,
+  and is used in the Tesla (2.0?) architecture.
+
+  It is unclear what NVIDIA calls this page table layout. It predates V1, so we
+  call it V0.
+
+  See also: https://envytools.readthedocs.io/en/latest/hw/memory/g80-vm.html
+ */
+/*
+// What size pages are in the pointed-to page table?
+enum V0_PDE_TYPE {NOT_PRESENT = 0, PAGE_64K = 1, PAGE_16K = 2, PAGE_4K = 3};
+// How large is the pointed-to page table?
+enum V0_PDE_SIZE {PDE_SZ_128K = 0, PDE_SZ_32K = 1, PDE_SZ_16K = 2, PDE_SZ_8K = 3};
+// Given a page table size, how many entries does it have?
+static const int V0_PDE_SIZE2NUM[4] = {128*1024, 32*1024, 16*1024, 8*1024};
+
+// PDE V0 (nv50/Tesla)
 typedef union {
         struct {
-                enum V1_PDE_TYPE type:2;
+                enum V0_PDE_TYPE type:2;
                 enum INST_TARGET target:2;
                  uint32_t padding0:1;
-                enum V1_PDE_SIZE sublevel_size:2;
+                enum V0_PDE_SIZE sublevel_size:2;
                  uint32_t padding1:5;
                 uint32_t addr:28;
                  uint32_t padding2:24;
         } __attribute__((packed));
         uint64_t raw;
-} page_dir_entry_v1_t;*/
-/* PTE V0 (nv50)
+} page_dir_entry_v0_t;
+
+// PTE V0 (nv50) for small pages
 typedef union {
         struct {
                 bool is_present:1;
@@ -1055,7 +1144,8 @@ typedef union {
                  uint32_t padding5:1;
         } __attribute__((packed));
         uint64_t raw;
-} page_tbl_entry_v1_t;*/
+} page_tbl_entry_v0_t;
+*/
 
 // TODO(jbakita): Maybe put the above GPU types in a different file.
 
@@ -1077,6 +1167,8 @@ struct nvdebug_state {
         struct gk20a *g;
         // Pointer to PCI device needed for pci_iounmap
         struct pci_dev *pcid;
+        // Pointer to generic device struct (both platform and pcie devices)
+        struct device *dev;
 };
 
 /*const struct runlist_funcs {
@@ -1152,13 +1244,11 @@ int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id);
 void __iomem *phy2PRAMIN(struct nvdebug_state* g, uint64_t phy);
 uint64_t search_page_directory(
         struct nvdebug_state *g,
-        void __iomem *pde_offset,
-        void __iomem *(*off2addr)(struct nvdebug_state*, uint64_t),
+        page_dir_config_t pd_config,
         uint64_t addr_to_find);
 uint64_t search_v1_page_directory(
         struct nvdebug_state *g,
-        void __iomem *pde_offset,
-        void __iomem *(*off2addr)(struct nvdebug_state*, uint64_t),
+        page_dir_config_t pd_config,
         uint64_t addr_to_find);
 
 
@@ -1252,4 +1342,4 @@ static inline void nvdebug_writeq(struct nvdebug_state *s, u32 r, u64 v) {
 }
 // Defined in bus.c
 int addr_to_pramin_mut(struct nvdebug_state *g, uint64_t addr, enum INST_TARGET target);
-int get_bar2_pdb(struct nvdebug_state *g, void **pdb, bool *is_v2_pdb);
+int get_bar2_pdb(struct nvdebug_state *g, page_dir_config_t* pd);