Ampere: disable/enable_channel, preempt/switch_to_tsg, and resubmit_runlist

**Modifes the user API from `echo 1 > /proc/gpuX/switch_to_tsg` to
`echo 1 > /proc/gpuX/runlist0/switch_to_tsg` to switch to TSG 1 on
runlist 0 on GPU X for pre-Ampere GPUs (for example).**

Feature changes:
- switch_to_tsg only makes sense on a per-runlist level. Before, this
  always operated on runlist0; this commit allows operating on any
  runlist by moving the API to the per-runlist paths.
- On Ampere+, channel and TSG IDs are per-runlist, and no longer
  GPU-global. Consequently, the disable/enable_channel and
  preempt_tsg APIs have been moved from GPU-global to per-runlist
  paths on Ampere+.

Bug fixes:
- `preempt_runlist()` is now supported on Maxwell and Pascal.
- `resubmit_runlist()` detects too-old GPUs.
- MAX_CHID corrected from 512 to 511 and documented.
- switch_to_tsg now includes a runlist resubmit, which appears to be
  necessary on Turing+ GPUs.

Tested on GK104 (Quadro K5000), GM204 (GTX 970), GP106 (GTX 1060 3GB),
GP104 (GTX 1080 Ti), GP10B (Jetson TX2), GV11B (Jetson Xavier), GV100
(Titan V), TU102 (RTX 2080 Ti), and AD102 (RTX 6000 Ada).

4 files changed, 260 insertions, 97 deletions

diff --git a/nvdebug.h b/nvdebug.h
index 26689d9..26167a7 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -172,7 +172,8 @@ enum PREEMPT_TYPE {PREEMPT_TYPE_CHANNEL = 0, PREEMPT_TYPE_TSG = 1};
   IS_PENDING  : Is a context switch pending? (read-only)
   TYPE        : PREEMPT_TYPE_CHANNEL or PREEMPT_TYPE_TSG
 
-  Support: Kepler, Maxwell, Pascal, Volta, Turing
+  Support: Fermi*, Kepler, Maxwell, Pascal, Volta, Turing
+  *Fermi only supports PREEMPT_TYPE_CHANNEL.
 */
 #define NV_PFIFO_PREEMPT 0x00002634
 typedef union {
@@ -187,10 +188,26 @@ typedef union {
         uint32_t raw;
 } pfifo_preempt_t;
 
+/* Preempt a TSG or Runlist by ID
+  Similar as on older GPUs (see above), but located at an offset in Runlist RAM.
+  This means that there's one instance of this register for each runlist.
+
+  IS_PENDING is now IS_TSG_PREEMPT_PENDING and IS_RUNLIST_PREEMPT_PENDING was
+  added in the following bit (bit 22). As these fields are unused in nvdebug,
+  we use the old structure for simplicity.
+
+  TYPE is now better described as IS_TSG_PREEMPT. TYPE == 0 requests a preempt
+  of the runlist (rather than a channel preemption, as on older GPUs).
+
+  Support: Ampere, Hopper, Ada, [newer untested]
+*/
+#define NV_RUNLIST_PREEMPT_GA100 0x098
+#define PREEMPT_TYPE_RUNLIST 0
+
 /*
   "Initiate a preempt of the engine by writing the bit associated with its
   runlist to NV_PFIFO_RUNLIST_PREEMPT...  Do not poll NV_PFIFO_RUNLIST_PREEMPT
-  for the preempt to complete."
+  for the preempt to complete." (open-gpu-doc)
 
   Useful for preempting multiple runlists at once.
 
@@ -204,7 +221,10 @@ typedef union {
     rl_preempt.raw |= BIT(nr);
     nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
 
-  Support: Volta, Turing
+  Support: Maxwell, Pascal, Volta, Turing
+
+  This register was deleted starting with Ampere, with functionality subsumed by
+  the NV_RUNLIST_PREEMPT register.
 */
 #define NV_PFIFO_RUNLIST_PREEMPT 0x00002638
 typedef union {
@@ -285,7 +305,7 @@ typedef union {
   TARGET      : Aperture of runlist (video or system memory)
 
   Support: Fermi*, Kepler, Maxwell, Pascal, Volta
-  *Fermi may expose this information 8 bytes earlier, starting at 0x227C?
+  *Fermi may expose ENG_RUNLING_* 8 bytes earlier, starting at 0x227C?
 */
 #define NV_PFIFO_RUNLIST_BASE_GF100 0x00002270 // Write-only
 #define NV_PFIFO_ENG_RUNLIST_BASE_GF100(i) (0x00002280+(i)*8) // Read-only
@@ -428,7 +448,11 @@ typedef union {
   See also: manuals/turing/tu104/dev_fifo.ref.txt in NVIDIA's open-gpu-doc
 */
 #define NV_PCCSR_CHANNEL_INST(i) (0x00800000+(i)*8)
-#define MAX_CHID 512
+// Maximum valid channel index in the PCCSR region
+// Channel IDs start at 0, and there are 4096 bytes of 8-byte CCSR entries (per
+// NV_PCCSR_CHANNEL_INST__SIZE_1 in at least Volta and Turing), yielding a total
+// of 512 channel IDs, with a maximum ID of 511.
+#define MAX_CHID 511
 typedef union {
         struct {
 // 0:31
@@ -554,6 +578,7 @@ typedef union {
 #define NV_CHIP_ID_GP106 0x136 // Discrete GeForce GTX 1060
 #define NV_CHIP_ID_GV11B 0x15B // Jetson Xavier embedded GPU
 
+#define NV_CHIP_ID_FERMI 0x0C0
 #define NV_CHIP_ID_KEPLER 0x0E0
 #define NV_CHIP_ID_MAXWELL 0x120
 #define NV_CHIP_ID_PASCAL 0x130
@@ -1505,7 +1530,7 @@ int get_runlist_iter(
         struct nvdebug_state *g,
         int rl_id,
         struct runlist_iter *rl_iter /* out */);
-int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id);
+int preempt_tsg(struct nvdebug_state *g, uint32_t rl_id, uint32_t tsg_id);
 int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id);
 int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id);
 
diff --git a/nvdebug_entry.c b/nvdebug_entry.c
index 5f99976..d5df7db 100644
--- a/nvdebug_entry.c
+++ b/nvdebug_entry.c
@@ -289,17 +289,45 @@ int __init nvdebug_init(void) {
                         snprintf(runlist_name, 12, "runlist%lu", last_runlist);
                         if (!(rl_dir = proc_mkdir_data(runlist_name, 0555, dir, (void*)device_id)))
                                 goto out_nomem;
+                        // Create one file for each runlist on Ampere+, or one file for each GPU on older
+                        if (g_nvdebug_state[res].chip_id >= NV_CHIP_ID_AMPERE || last_runlist == 0) {
+                                struct proc_dir_entry *chram_scope;
+                                // preempt_tsg, enable_channel, and disable_channel refer to a GPU-global channel
+                                // RAM on pre-Ampere GPUs
+                                if (g_nvdebug_state[res].chip_id >= NV_CHIP_ID_AMPERE)
+                                        chram_scope = rl_dir;
+                                else
+                                        chram_scope = dir;
+                                // Create file `/proc/gpu#/runlist#/preempt_tsg`, world writable
+                                // On Turing and older, `/proc/gpu#/preempt_tsg`
+                                if (!proc_create_data(
+                                                "preempt_tsg", 0222, chram_scope, compat_ops(&preempt_tsg_file_ops),
+                                                (void*)last_runlist))
+                                        goto out_nomem;
+                                // Create file `/proc/gpu#/runlist#/disable_channel`, world writable
+                                // On Turing and older, `/proc/gpu#/disable_channel`
+                                if (!proc_create_data(
+                                                "disable_channel", 0222, chram_scope, compat_ops(&disable_channel_file_ops),
+                                                (void*)last_runlist))
+                                        goto out_nomem;
+                                // Create file `/proc/gpu#/runlist#/enable_channel`, world writable
+                                // On Turing and older, `/proc/gpu#/enable_channel`
+                                if (!proc_create_data(
+                                                "enable_channel", 0222, chram_scope, compat_ops(&enable_channel_file_ops),
+                                                (void*)last_runlist))
+                                        goto out_nomem;
+                        }
                         // Create file `/proc/gpu#/runlist#/runlist`, world readable
                         if (!proc_create_data(
                                         "runlist", 0444, rl_dir, compat_ops(&runlist_file_ops),
                                         (void*)last_runlist))
                                 goto out_nomem;
+                        // Create file `/proc/gpu#/runlist#/switch_to_tsg`, world writable
+                        if (!proc_create_data(
+                                        "switch_to_tsg", 0222, rl_dir, compat_ops(&switch_to_tsg_file_ops),
+                                        (void*)last_runlist))
+                                goto out_nomem;
                 } while (last_runlist-- > 0);
-                // Create file `/proc/gpu#/preempt_tsg`, world writable
-                if (!proc_create_data(
-                                "preempt_tsg", 0222, dir, compat_ops(&preempt_tsg_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
                 /* On the TU104, the context scheduler (contained in the Host, aka
                  * PFIFO, unit) has been observed to sometimes to fail to schedule TSGs
                  * containing re-enabled channels. Resubmitting the runlist
@@ -311,21 +339,6 @@ int __init nvdebug_init(void) {
                                 "resubmit_runlist", 0222, dir, compat_ops(&resubmit_runlist_file_ops),
                                 (void*)device_id))
                         goto out_nomem;
-                // Create file `/proc/gpu#/disable_channel`, world writable
-                if (!proc_create_data(
-                                "disable_channel", 0222, dir, compat_ops(&disable_channel_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
-                // Create file `/proc/gpu#/enable_channel`, world writable
-                if (!proc_create_data(
-                                "enable_channel", 0222, dir, compat_ops(&enable_channel_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
-                // Create file `/proc/gpu#/switch_to_tsg`, world writable
-                if (!proc_create_data(
-                                "switch_to_tsg", 0222, dir, compat_ops(&switch_to_tsg_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
                 // Create file `/proc/gpu#/device_info`, world readable
                 if (!proc_create_data(
                                 "device_info", 0444, dir, compat_ops(&device_info_file_ops),
diff --git a/runlist.c b/runlist.c
index 7e6d292..7bb2ee4 100644
--- a/runlist.c
+++ b/runlist.c
@@ -169,24 +169,35 @@ attempt_pramin_access:
 
 /* Trigger a preempt of the specified TSG
   @param tsg_id ID of TSG to preempt.
+  @param rl_id  Which channel RAM address space to search?
   @return 0 or -errno on error
 
   Note: If no other TSGs exist in the associated runlist, this TSG may
         continue executing, unless NV_PFIFO_SCHED_DISABLE is set, or all the
         channels of the TSG to be preempted are disabled.
 */
-int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) {
-        pfifo_preempt_t pfifo_preempt;
+int preempt_tsg(struct nvdebug_state *g, uint32_t rl_id, uint32_t tsg_id) {
+        pfifo_preempt_t preempt;
+        // Fermi does not support time-slice groups
         if (g->chip_id < NV_CHIP_ID_KEPLER)
                 return -EOPNOTSUPP;
 
-        pfifo_preempt.raw = 0;
-        pfifo_preempt.id = tsg_id;
-        pfifo_preempt.is_pending = 0;
-        pfifo_preempt.type = PREEMPT_TYPE_TSG;
+        preempt.raw = 0;
+        preempt.id = tsg_id;
+        preempt.type = PREEMPT_TYPE_TSG;
 
         // Actually trigger the preemption
-        nvdebug_writel(g, NV_PFIFO_PREEMPT, pfifo_preempt.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                nvdebug_writel(g, NV_PFIFO_PREEMPT, preempt.raw);
+        } else {
+                uint32_t runlist_reg_base;
+                int err;
+                // As TSG and channel IDs are namespaced per-runlist starting with
+                // Ampere, the PREEMPT register is also per-runlist.
+                if ((err = get_runlist_ram(g, rl_id, &runlist_reg_base)))
+                        return err;
+                nvdebug_writel(g, runlist_reg_base + NV_RUNLIST_PREEMPT_GA100, preempt.raw);
+        }
         return 0;
 }
 
@@ -195,18 +206,38 @@ int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) {
   @return 0 or -errno on error
 */
 int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id) {
-        runlist_preempt_t rl_preempt;
-        if (g->chip_id < NV_CHIP_ID_VOLTA)
+        // The runlist preempt register does not exist on Kepler (tested gk104)
+        if (g->chip_id < NV_CHIP_ID_MAXWELL)
                 return -EOPNOTSUPP;
 
-        // Overwrite, as the register contains nothing to preserve
-        rl_preempt.raw = BIT(rl_id);
-        nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
+        // Write to trigger the preemption (the register contains nothing to
+        // preserve, and can thus just be overwritten)
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                runlist_preempt_t rl_preempt;
+                rl_preempt.raw = BIT(rl_id);
+                nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
+        } else {
+                int err;
+                uint32_t runlist_regs_base;
+                pfifo_preempt_t preempt;
+                // The RUNLIST_PREEMPT register was deleted, and the _PREEMPT register
+                // was extended to support runlist-level preemptions starting on Ampere
+                preempt.id = rl_id;
+                preempt.type = PREEMPT_TYPE_RUNLIST;
+                // The preempt register is scoped per-runlist on Ampere+
+                if ((err = get_runlist_ram(g, rl_id, &runlist_regs_base)))
+                        return err;
+                nvdebug_writel(g, runlist_regs_base + NV_RUNLIST_PREEMPT_GA100, preempt.raw);
+        }
         return 0;
 }
 
 // Read and write runlist configuration, triggering a resubmit
 int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
+        // Necessary registers do not exist pre-Fermi
+        if (g->chip_id < NV_CHIP_ID_FERMI)
+                return -EOPNOTSUPP;
+
         if (g->chip_id < NV_CHIP_ID_TURING) {
                 eng_runlist_gf100_t rl;
                 if (rl_id > MAX_RUNLISTS_GF100)
@@ -223,7 +254,19 @@ int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
                         return -EIO;
                 nvdebug_writeq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id), submit.raw);
         } else {
-                return -EOPNOTSUPP;
+                int err;
+                uint32_t runlist_pri_base;
+                runlist_submit_tu102_t submit;
+                if ((err = get_runlist_ram(g, rl_id, &runlist_pri_base)) < 0)
+                        return err;
+                if ((submit.raw = nvdebug_readq(g, runlist_pri_base + NV_RUNLIST_SUBMIT_GA100)) == -1)
+                        return -EIO;
+                // On Ampere, this does not appear to trigger a preempt of the
+                // currently-running channel (even if the currently running channel
+                // becomes disabled), but will cause newly re-enabled channels
+                // (at least if nothing else is pending) to become ready (tested on
+                // Jetson Orin).
+                nvdebug_writeq(g, runlist_pri_base + NV_RUNLIST_SUBMIT_GA100, submit.raw);
         }
         return 0;
 }
diff --git a/runlist_procfs.c b/runlist_procfs.c
index c1cfc87..b2159f6 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -199,11 +199,11 @@ struct file_operations runlist_file_ops = {
 };
 
 ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
-                               size_t count, loff_t *off) {
-        uint32_t target_tsgid;
+                               size_t count, loff_t *off) {
+        uint32_t target_tsgid, target_runlist_ram;
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         if (err)
                 return err;
 
@@ -211,8 +211,15 @@ ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
         if (target_tsgid > MAX_TSGID)
                 return -ERANGE;
 
+        // (Ab)use the PDE_DATA field for the index into which Runlist RAM this TSG
+        // ID is scoped to (only applicable on Ampere+)
+        if (g->chip_id >= NV_CHIP_ID_AMPERE)
+                target_runlist_ram = file2gpuidx(f);
+        else
+                target_runlist_ram = 0;
+
         // Execute preemption
-        if ((err = preempt_tsg(g, target_tsgid)))
+        if ((err = preempt_tsg(g, target_runlist_ram, target_tsgid)))
                 return err;
 
         return count;
@@ -223,13 +230,12 @@ struct file_operations preempt_tsg_file_ops = {
         .llseek = default_llseek,
 };
 
-
 ssize_t resubmit_runlist_file_write(struct file *f, const char __user *buffer,
-                               size_t count, loff_t *off) {
+                                    size_t count, loff_t *off) {
         uint32_t target_runlist;
+        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         int err = kstrtou32_from_user(buffer, count, 0, &target_runlist);
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         if (err)
                 return err;
 
@@ -245,26 +251,48 @@ struct file_operations resubmit_runlist_file_ops = {
         .llseek = default_llseek,
 };
 
+
 ssize_t disable_channel_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
+                                   size_t count, loff_t *off) {
         uint32_t target_channel;
-        channel_ctrl_t chan;
-        int err;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
                 return err;
 
-        if (target_channel > MAX_CHID)
-                return -ERANGE;
-
-        // Read current configuration
-        if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
-                return -EIO;
-        // Request disablement
-        chan.enable_clear = true;
-        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                channel_ctrl_t chan;
+                if (target_channel > MAX_CHID)
+                        return -ERANGE;
+                // Read current configuration
+                if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
+                        return -EIO;
+                // Request disablement
+                chan.enable_clear = true;
+                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        } else {
+                uint32_t runlist_reg_base, chram_base, channel_max;
+                runlist_channel_config_t channel_config;
+                channel_ctrl_ga100_t chan;
+                // (Ab)use the PDE_DATA field for the runlist ID
+                if ((err = get_runlist_ram(g, file2gpuidx(f), &runlist_reg_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((channel_config.raw = nvdebug_readl(g, runlist_reg_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_max = 1u << channel_config.num_channels_log2;
+                if (target_channel >= channel_max)
+                        return -ERANGE;
+                chram_base = (uint32_t)channel_config.bar0_offset << 4;
+                // Writing zeros to any field of the Ampere+ channel control structure
+                // does nothing, so don't bother to read the structure first, and just
+                // write zeros to all the fields we don't care about.
+                chan.raw = 0;
+                chan.is_write_one_clears_bits = 1; // Invert meaning of writing 1
+                chan.enable = 1;
+                nvdebug_writel(g, chram_base + sizeof(channel_ctrl_ga100_t) * target_channel, chan.raw);
+        }
 
         return count;
 }
@@ -275,23 +303,45 @@ struct file_operations disable_channel_file_ops = {
 };
 
 ssize_t enable_channel_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
+                                  size_t count, loff_t *off) {
         uint32_t target_channel;
-        channel_ctrl_t chan;
-        int err;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
                 return err;
 
-        if (target_channel > MAX_CHID)
-                return -ERANGE;
-
-        // Disable channel
-        chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel));
-        chan.enable_set = true;
-        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                channel_ctrl_t chan;
+                if (target_channel > MAX_CHID)
+                        return -ERANGE;
+                // Read current configuration
+                if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
+                        return -EIO;
+                // Disable channel
+                chan.enable_set = true;
+                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        } else {
+                uint32_t runlist_reg_base, chram_base, channel_max;
+                runlist_channel_config_t channel_config;
+                channel_ctrl_ga100_t chan;
+                // (Ab)use the PDE_DATA field for the runlist ID
+                if ((err = get_runlist_ram(g, file2gpuidx(f), &runlist_reg_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((channel_config.raw = nvdebug_readl(g, runlist_reg_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_max = 1u << channel_config.num_channels_log2;
+                if (target_channel >= channel_max)
+                        return -ERANGE;
+                chram_base = (uint32_t)channel_config.bar0_offset << 4;
+                // Writing zeros to any field of the Ampere+ channel control structure
+                // does nothing, so don't bother to read the structure first, and just
+                // write zeros to all the fields we don't care about.
+                chan.raw = 0;
+                chan.enable = 1;
+                nvdebug_writel(g, chram_base + sizeof(channel_ctrl_ga100_t) * target_channel, chan.raw);
+        }
 
         return count;
 }
@@ -301,52 +351,84 @@ struct file_operations enable_channel_file_ops = {
         .llseek = default_llseek,
 };
 
-// Note: Operates only on runlist 0 (Compute/Graphics)
+// Tested working on Pascal (gp106) through Ada (ad102)
 ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
-        uint32_t target_tsgid;
+                                 size_t count, loff_t *off) {
+        uint32_t target_tsgid, target_runlist, channel_regs_base;
         struct gv100_runlist_chan* chan;
         channel_ctrl_t chan_ctl;
+        channel_ctrl_ga100_t chan_ctl_ga100;
         struct runlist_iter rl_iter;
-        int err;
         loff_t pos = 0;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
         if (err)
                 return err;
 
         if (target_tsgid > MAX_TSGID)
                 return -ERANGE;
 
-        err = get_runlist_iter(g, 0, &rl_iter);
-        if (err)
+        // (Ab)use the PDE_DATA field for the runlist ID
+        target_runlist = file2gpuidx(f);
+
+        if ((err = get_runlist_iter(g, target_runlist, &rl_iter)))
                 return err;
 
+        // On Ampere, TSG and Channel IDs are only unique per-runlist, so we need
+        // to pull the per-runlist copy of Channel RAM.
+        if (g->chip_id >= NV_CHIP_ID_AMPERE) {
+                uint32_t runlist_regs_base;
+                runlist_channel_config_t chan_config;
+                if ((err = get_runlist_ram(g, target_runlist, &runlist_regs_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((chan_config.raw = nvdebug_readl(g, runlist_regs_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_regs_base = (uint32_t)chan_config.bar0_offset << 4;
+        }
+
         // Iterate through all TSGs
         while (pos < rl_iter.len) {
-                if (tsgid(g, rl_iter.curr_entry) == target_tsgid) {
-                        // Enable channels of target TSG
-                        for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
-                                chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
-                                chan_ctl.enable_set = true;
-                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
-                        }
-                } else {
-                        // XXX: Fix for bare channels. Maybe a "for_chan_until_tsg" macro?
-                        // Disable all other channels
-                        // (This is how the Jetson nvgpu driver disables TSGs)
-                        for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
-                                chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
-                                chan_ctl.enable_clear = true;
-                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
+                bool enable = false;
+                if (tsgid(g, rl_iter.curr_entry) == target_tsgid)
+                        enable = true;
+
+                // Either enable or disable all channels of each TSG, dependent on if
+                // they are contained within the target TSG or not.
+                for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
+                        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                                // Read, update, write for PCCSR
+                                if ((chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chid(g, chan)))) == -1)
+                                        return -EIO;
+                                if (enable)
+                                        chan_ctl.enable_set = true;
+                                else
+                                        chan_ctl.enable_clear = true;
+                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chid(g, chan)), chan_ctl.raw);
+                        } else {
+                                // Writing a 0 does nothing on Ampere+, so we can just write
+                                chan_ctl_ga100.raw = 0;
+                                chan_ctl_ga100.is_write_one_clears_bits = !enable;
+                                chan_ctl_ga100.enable = true;
+                                nvdebug_writel(g, channel_regs_base + sizeof(chan_ctl_ga100) * chid(g, chan), chan_ctl_ga100.raw);
                         }
                 }
                 pos += 1 + tsg_length(g, rl_iter.curr_entry);
                 rl_iter.curr_entry = next_tsg(g, rl_iter.curr_entry);
+
+                // TODO: Fix the above for bare channels. Add "for_chan_until_tsg"?
         }
-        // Trigger a runlist-level preempt to switch to `target_tsgid`
-        if ((err = preempt_runlist(g, 0)))
+
+        // Resubmit the runlist to ensure that changes to channel enablement are
+        // picked up on Turing+ GPUs (channel enablements may not be otherwise).
+        if (g->chip_id >= NV_CHIP_ID_TURING)
+                if ((err = resubmit_runlist(g, target_runlist)))
+                        return err;
+
+        // Trigger a runlist-level preempt to stop whatever was running, triggering
+        // the runlist scheduler to select and run the next-enabled channel.
+        if ((err = preempt_runlist(g, target_runlist)))
                 return err;
 
         return count;