4 files changed, 254 insertions, 40 deletions

diff --git a/nvdebug.h b/nvdebug.h
index fd88b2e..26689d9 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -365,6 +365,37 @@ enum CHANNEL_STATUS {
         CHANNEL_STATUS_ON_ENG_PENDING_ACQ_CTX_RELOAD = 14,
 };
 
+/* RunList RAM (RLRAM)
+  Starting with Ampere, the PFIFO register region no longer exists, and each
+  engine has seperate runlist RAM and channel RAM. The register (BAR0) offset for
+  Runlist RAM for each engine must be pulled from the runlist_pri_base field
+  (RUNLIST Private Register BASE address) provided by PTOP.
+
+ See get_runlist_ram() in runlist.c
+
+ Support: Ampere+
+*/
+#define NV_RUNLIST_BASE_GA100 0x080
+#define NV_RUNLIST_SUBMIT_GA100 0x088
+#define NV_RUNLIST_CHANNEL_CONFIG_GA100 0x004
+
+/* Channel RAM configuration, as contained in Runlist RAM
+
+  NUM_CHANNELS_LOG2 : 1 << NUM_CHANNELS_LOG2 is the number of channel_ctrl_ga100_t
+                      entries in the described Channel RAM region.
+  BAR0_OFFSET       : BAR0_OFFSET << 4 is the register offset (off BAR0) for the
+                      Channel RAM region.
+
+  Support: Ampere+
+*/
+typedef union {
+        struct {
+                uint8_t num_channels_log2:4;
+                uint32_t bar0_offset:28;
+        }__attribute__((packed));
+        uint32_t raw;
+} runlist_channel_config_t;
+
 /* Programmable Channel Control System RAM (PCCSR)
   512-entry array of channel control and status data structures.
 
@@ -425,6 +456,50 @@ typedef union {
         uint64_t raw;
 } channel_ctrl_t;
 
+/* CHannel RAM (CHRAM) (PCCSR replacement on Ampere+)
+  Starting with Ampere, channel IDs are no longer unique indexes into the
+  global channel RAM region (PCCSR), but are indexes into per-runlist channel
+  RAMs.
+
+  As Channel RAM entries are now subsidiary to a runlist, they do not contain
+  duplicate information, such as the instance pointer (to "result in smaller
+  hardware" per ga100/dev_ram.ref.txt in open-gpu-doc).
+
+  The new format retains and adds to the status information available about a
+  channel, but does so via bit flags rather than an enum. Some bit flags are
+  writable to trigger behavior previously dedicated to a bit (eg. writing to
+  `ctx_reload` triggers the same behavior as writing to `force_ctx_reload` did).
+
+  When the first bit (`is_write_one_clears_bits`) is set in this structure,
+  writing a 1 to any field will clear, rather than set, it. Writing a 0 to any
+  field is a no-op.
+
+  All fields read/write, except the following are read-only: BUSY, ON_PBDMA,
+  ON_ENG, PBDMA_BUSY, ENG_BUSY.
+
+  Support: Ampere, Hopper, Ada (and newer likely)
+  See also: manuals/ampere/ga100/dev_runlist.ref.txt in NVIDIA's open-gpu-doc
+*/
+typedef union {
+        struct {
+                bool is_write_one_clears_bits:1; // new
+                bool enable:1;
+                bool next:1;
+                bool busy:1;
+                bool pbdma_faulted:1; // write to force_pbdma_faulted
+                bool eng_faulted:1; // write to force_eng_faulted
+                bool on_pbdma:1; // breakout
+                bool on_eng:1; // breakout
+                bool pending:1; // breakout
+                bool ctx_reload:1; // breakout; write to force_ctx_reload
+                bool pbdma_busy:1; // breakout
+                bool eng_busy:1; // new
+                bool acquire_fail:1; // breakout
+                 uint32_t :19;
+        } __attribute__((packed));
+        uint32_t raw;
+} channel_ctrl_ga100_t;
+
 /* Control word for runlist enable/disable.
 
   RUNLIST_N           : Is runlist n disabled? (1 == disabled, 0 == enabled)
@@ -1413,14 +1488,19 @@ struct runlist_iter {
         int entries_left_in_tsg;
         // Number of entries in runlist
         int len;
-        // Offset to start of Channel RAM (as this is per-runlist on Ampere+)
-        uint32_t channel_ram;
+        // (Ampere+ only) Offset to the per-runlist "Runlist RAM" register region.
+        // This includes the offset for Channel RAM (per-runlist on Ampere+).
+        uint32_t runlist_pri_base;
 };
 
 #define NVDEBUG_MAX_DEVICES 8
 extern struct nvdebug_state g_nvdebug_state[NVDEBUG_MAX_DEVICES];
 
 // Defined in runlist.c
+int get_runlist_ram(
+        struct nvdebug_state *g,
+        int rl_id,
+        uint32_t *rl_ram_off /* out */);
 int get_runlist_iter(
         struct nvdebug_state *g,
         int rl_id,
diff --git a/nvdebug_entry.c b/nvdebug_entry.c
index eee7351..1f9e1c9 100644
--- a/nvdebug_entry.c
+++ b/nvdebug_entry.c
@@ -159,35 +159,53 @@ int probe_and_cache_devices(void) {
         return -ENODEV;
 }
 
-// Create files `/proc/gpu#/runlist#`, world readable
 // Support: Fermi, Maxwell, Pascal, Volta, Turing
-int create_runlist_files(int device_id, struct proc_dir_entry *dir) {
+int get_last_runlist_id_gk104(struct nvdebug_state *g) {
         ptop_device_info_gk104_t info;
-        struct proc_dir_entry *rl_entry;
-        int i, rl_id;
-        char runlist_name[12];
-        int max_rl_id = 0; // Always at least one runlist
+        int i, max_rl_id = 0; // Always at least one runlist
         // Figure out how many runlists there are by checking the device info
         // registers. Runlists are always numbered sequentially, so we just have
         // to find the highest-valued one and add 1 to get the number of runlists.
         for (i = 0; i < NV_PTOP_DEVICE_INFO__SIZE_1_GK104; i++) {
-                info.raw = nvdebug_readl(&g_nvdebug_state[device_id], NV_PTOP_DEVICE_INFO_GK104(i));
+                if ((info.raw = nvdebug_readl(g, NV_PTOP_DEVICE_INFO_GK104(i))) == -1)
+                        return -EIO;
                 if (info.info_type != INFO_TYPE_ENUM || !info.runlist_is_valid)
                         continue;
                 if (info.runlist_enum > max_rl_id)
                         max_rl_id = info.runlist_enum;
         }
-        // Create files to read each runlist. The read handling code looks at the
-        // `pde_data` associated with the file to determine what the runlist ID is.
-        for (rl_id = 0; rl_id <= max_rl_id; rl_id++) {
-                snprintf(runlist_name, 12, "runlist%d", rl_id);
-                rl_entry = proc_create_data(
-                        runlist_name, 0444, dir, compat_ops(&runlist_file_ops),
-                        (void*)(uintptr_t)rl_id);
-                if (!rl_entry)
-                        return -ENOMEM;
+        return max_rl_id;
+}
+
+// Support: Ampere, Hopper, Ada (and newer likely)
+// Identical structure to get_runlist_ram() in runlist.c. See comments there.
+int get_last_runlist_id_ga100(struct nvdebug_state *g) {
+        ptop_device_info_ga100_t ptop_entry;
+        int i, runlist_count = 0;
+        int ptop_size = NV_PTOP_DEVICE_INFO__SIZE_1_GA100(g);
+        int ptop_entry_subrow = 0;
+        for (i = 0; i < ptop_size; i++) {
+                if ((ptop_entry.raw = nvdebug_readl(g, NV_PTOP_DEVICE_INFO_GA100(i))) == -1)
+                        return -EIO;
+                if (!ptop_entry.raw)
+                        continue;
+                if (ptop_entry_subrow == 2 && ptop_entry.rleng_id == 0)
+                        runlist_count++;
+                if (ptop_entry.has_next_entry)
+                        ptop_entry_subrow += 1;
+                else
+                        ptop_entry_subrow = 0;
         }
-        return 0;
+        return runlist_count - 1;
+}
+
+// Return the maximum runlist ID. For a two-runlist GPU, this would return 1.
+int get_last_runlist_id(int device_id) {
+        struct nvdebug_state* g = &g_nvdebug_state[device_id];
+        if (g->chip_id >= NV_CHIP_ID_AMPERE)
+                return get_last_runlist_id_ga100(g);
+        else
+                return get_last_runlist_id_gk104(g);
 }
 
 // Create files `/proc/gpu#/gpc#_tpc_mask`, world readable
@@ -238,6 +256,7 @@ int __init nvdebug_init(void) {
         g_nvdebug_devices = res;
         // Create seperate ProcFS directories for each gpu
         while (res--) {
+                uintptr_t last_runlist = 0;
                 char device_id_str[7];
                 // Create a wider copy of the GPU ID to allow us to abuse the *data
                 // field of proc_dir_entry to store the GPU ID.
@@ -248,10 +267,24 @@ int __init nvdebug_init(void) {
                 snprintf(device_id_str, 7, "gpu%ld", device_id);
                 if (!(dir = proc_mkdir_data(device_id_str, 0555, NULL, (void*)device_id)))
                         goto out_nomem;
-                // Create files `/proc/gpu#/runlist#`, world readable
-                if (g_nvdebug_state[device_id].chip_id < NV_CHIP_ID_AMPERE)
-                        if ((err = create_runlist_files(device_id, dir)))
-                                goto out_err;
+                // Create files in the `/proc/gpu#/runlist#/` directory
+                // The read handling code looks at the `pde_data` associated with the parent
+                // directory to determine what the runlist ID is.
+                if ((last_runlist = get_last_runlist_id(device_id)) < 0)
+                        return last_runlist;
+                do {
+                        char runli