1 files changed, 116 insertions, 86 deletions

diff --git a/nvdebug.h b/nvdebug.h
index 409b013..fd88b2e 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -10,6 +10,9 @@
  *
  * This function should not depend on any Linux-internal headers, and may be
  * included outside of nvdebug.
+ *
+ * Style: This file uses up to 82-character lines to accomodate 2-character
+ * indented quotes from open-gpu-doc without reflowing.
  */
 #include <linux/types.h>
 
@@ -53,7 +56,7 @@ struct gk20a;
                           on either runqueue.)"
 
   INST_PTR_LO           : lower 20 bits of the 4k-aligned instance block pointer
-  INST_PTR_HI           : upper 32 bit of instance block pointer
+  INST_PTR_HI           : upper 32 bits of instance block pointer
   INST_TARGET (TGI)     : aperture of the instance block
 
   USERD_PTR_LO          : upper 24 bits of the low 32 bits, of the 512-byte-aligned USERD pointer
@@ -78,14 +81,14 @@ static inline const char *target_to_text(enum INST_TARGET t) {
         }
 }
 
-// Support: Volta, Ampere, Turing, Ampere
+// Support: Volta, Ampere, Turing, Ampere, Hopper, Ada
 struct gv100_runlist_chan {
 // 0:63
         enum ENTRY_TYPE entry_type:1;
         uint32_t runqueue_selector:1;
-         uint32_t padding:2;
+         uint32_t :2;
         enum INST_TARGET inst_target:2;
-         uint32_t padding2:2;
+         uint32_t :2;
         uint32_t userd_ptr_lo:24;
         uint32_t userd_ptr_hi:32;
 // 64:128
@@ -95,15 +98,15 @@ struct gv100_runlist_chan {
 } __attribute__((packed));
 
 // Support: Fermi, Kepler*, Maxwell, Pascal
-// *In Kepler, inst fields may be unpopulated?
+// *On Kepler (and older?), inst fields are unpopulated (ex. gk104)
 struct gm107_runlist_chan {
         uint32_t chid:12;
-         uint32_t padding0:1;
+         uint32_t :1;
         enum ENTRY_TYPE entry_type:1;
-         uint32_t padding1:18;
+         uint32_t :18;
         uint32_t inst_ptr_lo:20;
         enum INST_TARGET inst_target:2;  // Totally guessing on this
-         uint32_t padding2:10;
+         uint32_t :10;
 } __attribute__((packed));
 
 #define gk110_runlist_chan gm107_runlist_chan
@@ -131,20 +134,20 @@ struct gm107_runlist_chan {
   than 1GHz."
 */
 
-// Support: Volta, Turing*, Ampere*
+// Support: Volta, Turing*, Ampere*, Hopper, Ada
 // *These treat bits 4:11 (8 bits) as GFID (unused)
 struct gv100_runlist_tsg {
 // 0:63
         enum ENTRY_TYPE entry_type:1;
-         uint64_t padding:15;
+         uint64_t :15;
         uint32_t timeslice_scale:4;
-         uint64_t padding2:4;
+         uint64_t :4;
         uint32_t timeslice_timeout:8;
         uint32_t tsg_length:8;
-         uint32_t padding3:24;
+         uint32_t :24;
 // 64:128
         uint32_t tsgid:12;
-         uint64_t padding4:52;
+         uint64_t :52;
 } __attribute__((packed));
 #define MAX_TSGID (1 << 12)
 
@@ -153,12 +156,12 @@ struct gv100_runlist_tsg {
 // Last 32 bits appear to contain an undocumented inst ptr
 struct gk110_runlist_tsg {
         uint32_t tsgid:12;
-         uint32_t padding0:1;
+         uint32_t :1;
         enum ENTRY_TYPE entry_type:1;
         uint32_t timeslice_scale:4;
         uint32_t timeslice_timeout:8;
         uint32_t tsg_length:6;
-         uint32_t padding1:32;
+         uint32_t :32;
 } __attribute__((packed));
 
 
@@ -175,11 +178,11 @@ enum PREEMPT_TYPE {PREEMPT_TYPE_CHANNEL = 0, PREEMPT_TYPE_TSG = 1};
 typedef union {
         struct {
                 uint32_t id:12;
-                 uint32_t padding:8;
+                 uint32_t :8;
                 bool is_pending:1;
-                 uint32_t padding2:3;
+                 uint32_t :3;
                 enum PREEMPT_TYPE type:2;
-                 uint32_t padding3:6;
+                 uint32_t :6;
         } __attribute__((packed));
         uint32_t raw;
 } pfifo_preempt_t;
@@ -220,7 +223,7 @@ typedef union {
                 bool runlist_11:1;
                 bool runlist_12:1;
                 bool runlist_13:1;
-                 uint32_t padding:18;
+                 uint32_t :18;
         } __attribute__((packed));
         uint32_t raw;
 } runlist_preempt_t;
@@ -313,7 +316,8 @@ typedef union {
   of this, the layout was modified to allow for larger runlist pointers (upper
   52 of 64 bits).
 
-  Support: Turing, Ampere, Lovelace?, Hopper?
+  Support: Turing, Ampere*, Hopper*, Ada*
+  *Only the register layout
 */
 // Support: Turing
 #define NV_PFIFO_RUNLIST_BASE_TU102(i) (0x00002B00+(i)*16) // Read/write
@@ -324,9 +328,9 @@ typedef union {
 typedef union {
         struct {
                 enum INST_TARGET target:2;
-                 uint32_t padding:10;
+                 uint32_t :10;
                 uint64_t ptr:28;
-                 uint32_t padding2:24;
+                 uint32_t :24;
         } __attribute__((packed));
         uint64_t raw;
 } runlist_base_tu102_t;
@@ -453,7 +457,7 @@ typedef union {
                 bool runlist_8:1;
                 bool runlist_9:1;
                 bool runlist_10:1;
-                 uint32_t padding:21;
+                 uint32_t :21;
         } __attribute__((packed));
         uint32_t raw;
 } runlist_disable_t;
@@ -546,17 +550,17 @@ typedef union {
                 uint32_t minor_revision:4;
                 uint32_t major_revision:4;
                  uint32_t reserved:4;
-                 uint32_t padding0:8;
+                 uint32_t :8;
                 uint32_t implementation:4;
                 uint32_t architecture:5;
-                 uint32_t padding1:3;
+                 uint32_t :3;
         } __attribute__((packed));
         uint32_t raw;
         // Arch << 4 + impl is also often used
         struct {
-                 uint32_t padding2:20;
+                 uint32_t :20;
                 uint32_t chip_id:9;
-                 uint32_t padding3:3;
+                 uint32_t :3;
         } __attribute__((packed));
 } mc_boot_0_t;
 
@@ -679,35 +683,35 @@ typedef union {
                 enum DEVICE_INFO_TYPE info_type:2;
                 bool fault_id_is_valid:1;
                 uint32_t fault_id:7;
-                 uint32_t padding0:2;
+                 uint32_t :2;
                 uint32_t pri_base:12;
-                 uint32_t padding1:2;
+                 uint32_t :2;
                 uint32_t inst_id:4;
                 uint32_t is_not_enum2:1;
                 bool has_next_entry:1;
         } __attribute__((packed));
         // ENUM type fields
         struct {
-                 uint32_t padding2:2;
+                 uint32_t :2;
                 bool reset_is_valid:1;
                 bool intr_is_valid:1;
                 bool runlist_is_valid:1;
                 bool engine_is_valid:1;
-                 uint32_t padding3:3;
+                 uint32_t :3;
                 uint32_t reset_enum:5;
-                 uint32_t padding4:1;
+                 uint32_t :1;
                 uint32_t intr_enum:5;
-                 uint32_t padding5:1;
+                 uint32_t :1;
                 uint32_t runlist_enum:4;
-                 uint32_t padding6:1;
+                 uint32_t :1;
                 uint32_t engine_enum:4;
-                 uint32_t padding7:2;
+                 uint32_t :2;
         } __attribute__((packed));
         // ENGINE_TYPE type fields
         struct {
-                 uint32_t padding8:2;
+                 uint32_t :2;
                 enum ENGINE_TYPES engine_type:29;
-                 uint32_t padding9:1;
+                 uint32_t :1;
         } __attribute__((packed));
         uint32_t raw;
 } ptop_device_info_gk104_t;
@@ -744,17 +748,19 @@ typedef union {
                      runlist of this engine.
   RLENG_ID         : What is the per-runlist ID of this engine?
 
-  Support: Ampere, Ada, Hopper, (and newer likely)
+  Support: Ampere, Hopper, Ada (and newer likely)
   See also: hw_top_ga100.h in nvgpu (NVIDIA's open-source Jetson GPU driver)
 */
 #define NV_PTOP_DEVICE_INFO_GA100(i) (0x00022800+(i)*4)
-#define NV_PTOP_DEVICE_INFO__SIZE_1_GA100(g) (nvdebug_readl(g, 0x0224fc) >> 20)
+#define NV_PTOP_DEVICE_INFO_LEN_GA100 0x000224fc
+#define NV_PTOP_DEVICE_INFO_LEN_SHIFT_GA100 20
+#define NV_PTOP_DEVICE_INFO__SIZE_1_GA100(g) (nvdebug_readl(g, NV_PTOP_DEVICE_INFO_LEN_GA100) >> NV_PTOP_DEVICE_INFO_LEN_SHIFT_GA100)
 
 typedef union {
         // _info type fields
         struct {
                 uint32_t fault_id:11;
-                 uint32_t padding0:5;
+                 uint32_t :5;
                 uint32_t inst_id:8;
                 enum ENGINE_TYPES engine_type:7; // "type_enum"
                 bool has_next_entry:1;
@@ -763,15 +769,15 @@ typedef union {
         struct {
                 uint32_t reset_id:8;
                 uint32_t pri_base:18; // "device_pri_base"
-                 uint32_t padding1:4;
+                 uint32_t :4;
                 uint32_t is_engine:1;
-                 uint32_t padding2:1;
+                 uint32_t :1;
         } __attribute__((packed));
         struct {
                 uint32_t rleng_id:2;
-                 uint32_t padding3:8;
+                 uint32_t :8;
                 uint32_t runlist_pri_base:16;
-                 uint32_t padding4:6;
+                 uint32_t :6;
         } __attribute__((packed));
         uint32_t raw;
 } ptop_device_info_ga100_t;
@@ -970,24 +976,31 @@ typedef union {
 #define NV_PRAMIN 0x00700000  // Goes until 0x00800000 (1MB window)
 #define NV_PRAMIN_LEN 0x00100000
 
-// Support: Tesla 2.0, Fermi, Kepler, Maxwell, Pascal, Turing, Ampere
+/* Page Directory Base (PDB) configuration for an instance block
+
+  Note: "Volta only supports [the] new page table format [V2] and [a] 64KB big
+        page size" (kern_gmmu_gv100.c in open-gpu-kernel-modules).
+  Support: Tesla 2.0* through Ampere, Ada
+   *FAULT_REPLAY_* fields are Pascal+ only
+  See also: dev_ram.h (open-gpu-kernel-modules) or dev_ram.ref.txt (open-gpu-doc)
+*/
 #define NV_PRAMIN_PDB_CONFIG_OFF 0x200
 typedef union {
         struct {
-                uint32_t target:2;
+                enum INST_TARGET target:2;
                 uint32_t is_volatile:1;
-                 uint32_t padding0:1;
-                uint32_t fault_replay_tex:1;
-                uint32_t fault_replay_gcc:1;
-                 uint32_t padding1:4;
+                 uint32_t :1;
+                bool fault_replay_tex:1;
+                bool fault_replay_gcc:1;
+                 uint32_t :4;
                 bool is_ver2:1;
                 bool is_64k_big_page:1;  // 128Kb otherwise
                 uint32_t page_dir_lo:20;
                 uint32_t page_dir_hi:32;
         } __attribute__((packed));
         struct {
-                uint32_t pad:12;
-                uint64_t page_dir:52; // Confirmed working on Xavier and tama
+                 uint32_t :12;
+                uint64_t page_dir:52;
         } __attribute__((packed));
         uint64_t raw;
 } page_dir_config_t;
@@ -1025,7 +1038,7 @@ typedef union {
   only one is strictly needed to reconstruct all the others. However, due to
   the complexity of page tables, we include all of these to aid in readability.
 
-  Support: Pascal, Volta, Turing, Ampere, Ada, Ampere, Hopper*, Blackwell*
+  Support: Pascal, Volta, Turing, Ampere, Hopper*, Ada, Blackwell*
   Note: *Hopper introduces Version 3 Page Tables, but is backwards-compatible.
          The newer version adds a PD4 level to support 57-bit virtual
          addresses, and slightly shifts the PDE and PTE fields.
@@ -1077,17 +1090,46 @@ static inline const char *pd_target_to_text(enum PD_TARGET t) {
         }
 }
 
-// Page Directory Entry/Page Table Entry V2 type
-// Note: As the meaning of target (bits 2:1) at a PDE-level changes if the
-//       entry is a large-page PTE or not. To simply the logic, we combine them
-//       into a single target field to simplify comparisons.
+/* Page Directory Entry/Page Table Entry V2 type
+  We consider the least-significant bit to be 0, and use interval notation.
+  Example: The first 8 bits of an address could be identically described as
+  (8, 0], [7, 0], [7, -1), or (8, -1).
+
+  ADDR   : Bits [35, 12] of the physical address; bits [11, 0] are 0. This is the
+           full 36-bit address for VID_MEM or PEER targets. For SYS_MEM targets,
+           use ADDR_W to include bits [57, 36] (per gp100-mmu-format.pdf, Pascal
+           only uses bits [46, 0]---a 2^47 = 128 TiB physical address space).
+           Points to first entry of the next level of the page table (in a PDE),
+           or the start of the physical frame (in a PTE).
+  ADDR_W : Bits [57, 12] of a SYS_MEM address. Only necessary for physical
+           addresses over 128 TiB. See ADDR.
+  IS_VOL : If set, the pointed-to frame should not be cached in the GPU L2 cache.
+           This applies to PDEs (then the pointed-to page table/directory frame
+           will not be cached), and to PTEs (then the pointed to data frame will
+           not be cached). This **does not apply to VID_MEM**, except on Tegra.
+  NO_ATS : "GPUs which support ATS [Volta+] perform a parallel lookup on both
+           ATS and GMMU page tables. The ATS lookup can be disabled by setting a
+           bit in the GMMU page tables. All GPUs which support ATS use the same
+           mechanism (a bit in PDE1), and have the same PDE1 coverage (512MB)."
+           (nvidia-uvm/uvm_mmu.h)
+           Other parts of the nvidia-uvm documentation note that disabling the
+           ATS lookup helps performance.
+
+  Note: As the meaning of target (bits 2:1) at a PDE-level changes if the
+        entry is a large-page PTE or not. To simply the logic, we combine them
+        into a single target field to simplify comparisons.
+
+  See also: gp100-mmu-format.pdf in open-gpu-doc.
+*/
 #define TARGET_PEER 1
 typedef union {
         // Page Directory Entry (PDE)
         struct {
                 enum PD_TARGET target:3;
                 bool is_volatile:1;
-                 uint32_t padding1:4;
+                 uint32_t :1;
+                bool no_ats:1; // Set to disable PCIe (?) Address Translation Services
+                 uint32_t :2;
                 uint32_t addr:24;
                  uint32_t __unused1;
         } __attribute__((packed));
@@ -1179,21 +1221,21 @@ typedef union {
         struct {
 // 0:32
                 enum V1_PD_TARGET target:2;
-                 uint32_t padding0:2; // Documented as "PDE_SIZE"?
+                 uint32_t :2; // Documented as "PDE_SIZE"?
                 uint64_t addr:28;  // May be wider?
 // 32:63
-                 uint32_t padding2:3;
+                 uint32_t :3;
                 uint32_t is_volatile:1; // Might have counted wrong?
-                 uint32_t padding3:28;
+                 uint32_t :28;
         } __attribute__((packed));
 // Small page fields
         struct {
 // 0:32
-                 uint32_t padding00:32;
+                 uint32_t :32;
 // 32:63
                 enum V1_PD_TARGET alt_target:2;
                 uint32_t alt_is_volatile:1; // Might have counted wrong?
-                 uint32_t padding03:1;
+                 uint32_t :1;
                 uint64_t alt_addr:28;
         } __attribute__((packed));
         uint64_t raw;
@@ -1240,11 +1282,11 @@ typedef union {
         struct {
                 enum V0_PDE_TYPE type:2;
                 enum INST_TARGET target:2;
-                 uint32_t padding0:1;
+                 uint32_t :1;
                 enum V0_PDE_SIZE sublevel_size:2;
-                 uint32_t padding1:5;
-                uint32_t addr:28;
-                 uint32_t padding2:24;
+                 uint32_t :5;
+                uint32_t addr:28; // Bits [12, 39] of the 40-bit page table address
+                 uint32_t :24;
         } __attribute__((packed));
         uint64_t raw;
 } page_dir_entry_v0_t;
@@ -1253,19 +1295,19 @@ typedef union {
 typedef union {
         struct {
                 bool is_present:1;
-                 uint32_t padding3:2;
+                 uint32_t :2;
                 bool is_readonly:1;
                 enum INST_TARGET target:2;
                 bool is_privileged:1;
                 uint32_t contig_blk_sz:3;
-                 uint32_t padding4:2;
-                uint32_t addr:28;
+                 uint32_t :2;
+                uint32_t addr:28; // Bits [12, 39] of the 40-bit frame address
                 uint32_t storage_type:7;  // ???
                 uint32_t compression_mode:2;  // ???
                 uint32_t compression_tag:12;  // ???
                 bool is_long_partition_cycle:1;  // ???
                 bool is_encrypted:1;
-                 uint32_t padding5:1;
+                 uint32_t :1;
         } __attribute__((packed));
         uint64_t raw;
 } page_tbl_entry_v0_t;
@@ -1316,27 +1358,14 @@ struct nvdebug_state {
                 void __iomem *bar3;
         };
         int chip_id;
-        // Additional state from the built-in driver. Only set iff
-        // chip_id == NV_CHIP_ID_GV11B
+        // Additional state from the built-in driver. Only set on Jetson boards
         struct gk20a *g;
-        // Pointer to PCI device needed for pci_iounmap
+        // Pointer to PCI device needed for pci_iounmap and pci_resource_start
         struct pci_dev *pcid;
         // Pointer to generic device struct (both platform and pcie devices)
         struct device *dev;
 };
 
-/*const struct runlist_funcs {
-        u8 size;
-        enum ENTRY_TYPE (*entry_type)(struct nvdebug_state *, void *);
-        uint32_t (*chid)(struct nvdebug_state *, void *);
-        uint32_t (*inst_ptr_lo)(struct nvdebug_state *, void *);
-        enum INST_TARGET (*inst_target)(struct nvdebug_state *, void *):
-        uint32_t (*tsgid)(struct nvdebug_state *, void *);
-        uint32_t (*timeslice_scale)(struct nvdebug_state *, void *);
-        uint32_t (*timeslice_timeout)(struct nvdebug_state *, void *);
-        uint32_t (*tsg_length)(struct nvdebug_state *, void *);
-};*/
-
 // This disgusting macro is a crutch to work around the fact that runlists were
 // different prior to Volta.
 #define VERSIONED_RL_ACCESSOR(_ENTRY_TYPE, type, prop) \
@@ -1366,6 +1395,7 @@ VERSIONED_RL_ACCESSOR(tsg, uint32_t, tsg_length);
 #define NV_RL_ENTRY_SIZE(g) \
          ((g)->chip_id >= NV_CHIP_ID_VOLTA ? sizeof(struct gv100_runlist_tsg) : sizeof(struct gk110_runlist_tsg))
 
+// chan and tsg should be pointers
 #define for_chan_in_tsg(g, chan, tsg) \
         for (chan = (typeof(chan))(((u8*)tsg) + NV_RL_ENTRY_SIZE(g)); \
              (u8*)chan < ((u8*)tsg) + (1 + tsg_length(g, tsg)) * NV_RL_ENTRY_SIZE(g); \