3 files changed, 192 insertions, 188 deletions

diff --git a/nvdebug.h b/nvdebug.h
index 2fc8c63..f65b403 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -131,8 +131,8 @@ struct gm107_runlist_chan {
   GPU instance addresses with Volta.
 */
 
-// Support: Volta, Ampere*, Turing*
-// *These treat the top 8 bits of TSGID as GFID (unused)
+// Support: Volta, Turing*, Ampere*
+// *These treat bits 4:11 (8 bits) as GFID (unused)
 struct gv100_runlist_tsg {
 // 0:63
         enum ENTRY_TYPE entry_type:1;
@@ -166,7 +166,7 @@ enum PREEMPT_TYPE {PREEMPT_TYPE_CHANNEL = 0, PREEMPT_TYPE_TSG = 1};
 
 /* Preempt a TSG or Channel by ID
   ID/CHID     : Id of TSG or channel to preempt
-  IS_PENDING  : Is a context switch pending?
+  IS_PENDING  : Is a context switch pending? (read-only)
   TYPE        : PREEMPT_TYPE_CHANNEL or PREEMPT_TYPE_TSG
 
   Support: Kepler, Maxwell, Pascal, Volta, Turing
@@ -201,7 +201,7 @@ typedef union {
     rl_preempt.raw |= BIT(nr);
     nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
 
-  Support: Volta
+  Support: Volta, Turing
 */
 #define NV_PFIFO_RUNLIST_PREEMPT 0x00002638
 typedef union {
@@ -255,39 +255,83 @@ typedef union {
  * cause a system to hang/stop responding."
  */
 
-// Note: This is different with Turing
-// Support: Fermi, Kepler, Maxwell, Pascal, Volta
-#define NV_PFIFO_RUNLIST_BASE 0x00002270
-#define NV_PFIFO_ENG_RUNLIST_BASE(i) (0x00002280+(i)*8)
+/* Runlist Metadata (up through Volta)
+  "Software specifies the GPU contexts that hardware should "run" by writing a
+  list of entries (known as a "runlist") to a 4k-aligned area of memory (beginning
+  at NV_PFIFO_RUNLIST_BASE), and by notifying Host that a new list is available
+  (by writing to NV_PFIFO_RUNLIST).
+
+  Submission of a new runlist causes Host to expire the timeslice of all work
+  scheduled by the previous runlist, allowing it to schedule the channels present
+  in the new runlist once they are fetched. SW can check the status of the runlist
+  by polling NV_PFIFO_ENG_RUNLIST_PENDING. (see dev_fifo.ref NV_PFIFO_RUNLIST for
+  a full description of the runlist submit mechanism).
+
+  Runlists can be stored in system memory or video memory (as specified by
+  NV_PFIFO_RUNLIST_BASE_TARGET). If a runlist is stored in video memory, software
+  will have to execute flush or read the last entry written before submitting the
+  runlist to Host to guarantee coherency." (volta/dev_ram.ref.txt)
+
+  We only document the *_PFIFO_ENG_RUNLIST_*(i) read-only registers here (where
+  i is a runlist index). Runlists are configured via the seperate, writable
+  *_PFIFO_RUNLIST_* register; see open-gpu-doc for more on that.
+
+  LEN         : Number of entries in runlist
+  IS_PENDING  : Is runlist committed?
+  PTR         : Pointer to start of 4k-aligned runlist (upper 28 of 40 bits)
+  TARGET      : Aperture of runlist (video or system memory)
+
+  Support: Fermi*, Kepler, Maxwell, Pascal, Volta
+  *Fermi may expose this information 8 bytes earlier, starting at 0x227C?
+*/
+#define NV_PFIFO_ENG_RUNLIST_BASE_GF100(i) (0x00002280+(i)*8) // Read-only
 typedef union {
         struct {
+                // NV_PFIFO_ENG_RUNLIST_BASE_* fields
                 uint32_t ptr:28;
                 enum INST_TARGET target:2;
-                 uint32_t padding:2;
+                 uint32_t padding1:2;
+                // NV_PFIFO_ENG_RUNLIST_* fields
+                uint16_t len:16;
+                 uint32_t padding2:4;
+                bool is_pending:1;
+                 uint32_t padding3:11;
         } __attribute__((packed));
-        uint32_t raw;
-} runlist_base_t;
+        uint64_t raw;
+} eng_runlist_gf100_t;
 
-// Support: Kepler, Maxwell, Pascal, Volta
-// Works on Fermi, but id is one bit longer and is b11111
-#define NV_PFIFO_RUNLIST 0x00002274
-#define NV_PFIFO_ENG_RUNLIST(i) (0x00002284+(i)*8)
+/*
+  Starting with Turing, the seperate registers for reading and writing runlist
+  configuration were dropped in favor of read/write indexed registers. As part
+  of this, the layout was modified to allow for larger runlist pointers (upper
+  52 of 64 bits).
+
+  Support: Turing, Ampere, Lovelace?, Hopper?
+*/
+// Support: Turing
+#define NV_PFIFO_RUNLIST_BASE_TU102(i) (0x00002B00+(i)*16) // Read/write
+#define NV_PFIFO_RUNLIST_SUBMIT_TU102(i) (0x00002B08+(i)*16) // Read/write
 typedef union {
-        // RUNLIST fields
         struct {
-                uint32_t len:16;
-                 uint32_t padding:4;
-                uint32_t id:4; // Runlist ID (each engine may have a seperate runlist)
-                 uint32_t padding2:8;
+                enum INST_TARGET target:2;
+                 uint32_t padding:10;
+                uint64_t ptr:28;
+                 uint32_t padding2:24;
         } __attribute__((packed));
-        // ENG_RUNLIST fields that differ
+        uint64_t raw;
+} runlist_base_tu102_t;
+
+typedef union {
         struct {
-                 uint32_t padding3:20;
-                bool is_pending:1; // Is runlist not yet committed?
-                 uint32_t padding4:11;
+                uint16_t len:16;
+                uint16_t offset:16;
+                uint32_t preempted_tsgid:14;
+                bool valid_preempted_tsgid:1;
+                bool is_pending:1;
+                uint32_t preempted_offset:16;
         } __attribute__((packed));
-        uint32_t raw;
-} runlist_info_t;
+        uint64_t raw;
+} runlist_submit_tu102_t;
 
 enum CHANNEL_STATUS {
         CHANNEL_STATUS_IDLE = 0,
@@ -307,8 +351,13 @@ enum CHANNEL_STATUS {
         CHANNEL_STATUS_ON_ENG_PENDING_ACQ_CTX_RELOAD = 14,
 };
 
+/* Programmable Channel Control System RAM (PCCSR)
+
+  512-entry array of channel control and status data structures.
+
+  Support: Fermi, Maxwell, Pascal, Volta, Turing, [more?]
+*/
 #define NV_PCCSR_CHANNEL_INST(i) (0x00800000+(i)*8)
-// There are a total of 512 possible channels
 #define MAX_CHID 512
 typedef union {
         struct {
@@ -1023,12 +1072,12 @@ VERSIONED_RL_ACCESSOR(tsg, uint32_t, tsg_length);
 struct runlist_iter {
         // Pointer to either a TSG or channel entry (they're the same size)
         void *curr_entry;
-        // This should be set to tsg_length when a TSG is reached, and
-        // decremented as each subsequent channel is printed. This allows us to
-        // track which channel are and are not part of the TSG.
-        int channels_left_in_tsg;
-        // Total runlist length, etc
-        runlist_info_t rl_info;
+        // This should be set to tsg_length + 1 when a TSG is reached, and
+        // decremented each time _next() is called. This allows us to
+        // track which channels are and are not part of the TSG.
+        int entries_left_in_tsg;
+        // Number of entries in runlist
+        int len;
 };
 
 #define NVDEBUG_MAX_DEVICES 8
@@ -1037,6 +1086,7 @@ extern struct nvdebug_state g_nvdebug_state[NVDEBUG_MAX_DEVICES];
 // Defined in runlist.c
 int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl_iter);
 int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id);
+int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id);
 
 // Defined in mmu.c
 uint32_t vram2PRAMIN(struct nvdebug_state *g, uint64_t addr);
diff --git a/runlist.c b/runlist.c
index ed35c7e..c725e77 100644
--- a/runlist.c
+++ b/runlist.c
@@ -1,172 +1,134 @@
-#include <linux/kernel.h>  // Kernel types
+/* Copyright 2024 Joshua Bakita
+ * Helpers for dealing with the runlist and other Host (PFIFO) registers
+ */
+#include <linux/printk.h> // For printk()
+#include <asm/errno.h> // For error defines
+#include <asm/io.h> // For phys_to_virt()
 
 #include "nvdebug.h"
 
+// Uncomment to, upon BAR2 access failure, return a PRAMIN-based runlist pointer
+// **If enabled, PRAMIN may not be otherwise used while walking the runlist!**
 #define FALLBACK_TO_PRAMIN
 
 /* Get runlist head and info (incl. length)
-   @param rl_iter Location at which to store output
-   @param rl_id   Which runlist to obtain?
+  @param rl_id   Which runlist to obtain?
+  @param rl_iter Location at which to store output
+  @return 0 or -errno on error
 */
 int get_runlist_iter(struct nvdebug_state *g, int rl_id, struct runlist_iter *rl_iter) {
-        runlist_base_t rl_base;
-        runlist_info_t rl_info;
-        u64 runlist_iova;
+        uint64_t runlist_iova;
+        enum INST_TARGET runlist_target;
+        uint16_t runlist_len;
+#ifdef FALLBACK_TO_PRAMIN
+        int off;
+#endif // FALLBACK_TO_PRAMIN
+        // Zero-initialize the runlist iterator
         *rl_iter = (struct runlist_iter){0};