4 files changed, 260 insertions, 97 deletions

diff --git a/nvdebug.h b/nvdebug.h
index 26689d9..26167a7 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -172,7 +172,8 @@ enum PREEMPT_TYPE {PREEMPT_TYPE_CHANNEL = 0, PREEMPT_TYPE_TSG = 1};
   IS_PENDING  : Is a context switch pending? (read-only)
   TYPE        : PREEMPT_TYPE_CHANNEL or PREEMPT_TYPE_TSG
 
-  Support: Kepler, Maxwell, Pascal, Volta, Turing
+  Support: Fermi*, Kepler, Maxwell, Pascal, Volta, Turing
+  *Fermi only supports PREEMPT_TYPE_CHANNEL.
 */
 #define NV_PFIFO_PREEMPT 0x00002634
 typedef union {
@@ -187,10 +188,26 @@ typedef union {
         uint32_t raw;
 } pfifo_preempt_t;
 
+/* Preempt a TSG or Runlist by ID
+  Similar as on older GPUs (see above), but located at an offset in Runlist RAM.
+  This means that there's one instance of this register for each runlist.
+
+  IS_PENDING is now IS_TSG_PREEMPT_PENDING and IS_RUNLIST_PREEMPT_PENDING was
+  added in the following bit (bit 22). As these fields are unused in nvdebug,
+  we use the old structure for simplicity.
+
+  TYPE is now better described as IS_TSG_PREEMPT. TYPE == 0 requests a preempt
+  of the runlist (rather than a channel preemption, as on older GPUs).
+
+  Support: Ampere, Hopper, Ada, [newer untested]
+*/
+#define NV_RUNLIST_PREEMPT_GA100 0x098
+#define PREEMPT_TYPE_RUNLIST 0
+
 /*
   "Initiate a preempt of the engine by writing the bit associated with its
   runlist to NV_PFIFO_RUNLIST_PREEMPT...  Do not poll NV_PFIFO_RUNLIST_PREEMPT
-  for the preempt to complete."
+  for the preempt to complete." (open-gpu-doc)
 
   Useful for preempting multiple runlists at once.
 
@@ -204,7 +221,10 @@ typedef union {
     rl_preempt.raw |= BIT(nr);
     nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
 
-  Support: Volta, Turing
+  Support: Maxwell, Pascal, Volta, Turing
+
+  This register was deleted starting with Ampere, with functionality subsumed by
+  the NV_RUNLIST_PREEMPT register.
 */
 #define NV_PFIFO_RUNLIST_PREEMPT 0x00002638
 typedef union {
@@ -285,7 +305,7 @@ typedef union {
   TARGET      : Aperture of runlist (video or system memory)
 
   Support: Fermi*, Kepler, Maxwell, Pascal, Volta
-  *Fermi may expose this information 8 bytes earlier, starting at 0x227C?
+  *Fermi may expose ENG_RUNLING_* 8 bytes earlier, starting at 0x227C?
 */
 #define NV_PFIFO_RUNLIST_BASE_GF100 0x00002270 // Write-only
 #define NV_PFIFO_ENG_RUNLIST_BASE_GF100(i) (0x00002280+(i)*8) // Read-only
@@ -428,7 +448,11 @@ typedef union {
   See also: manuals/turing/tu104/dev_fifo.ref.txt in NVIDIA's open-gpu-doc
 */
 #define NV_PCCSR_CHANNEL_INST(i) (0x00800000+(i)*8)
-#define MAX_CHID 512
+// Maximum valid channel index in the PCCSR region
+// Channel IDs start at 0, and there are 4096 bytes of 8-byte CCSR entries (per
+// NV_PCCSR_CHANNEL_INST__SIZE_1 in at least Volta and Turing), yielding a total
+// of 512 channel IDs, with a maximum ID of 511.
+#define MAX_CHID 511
 typedef union {
         struct {
 // 0:31
@@ -554,6 +578,7 @@ typedef union {
 #define NV_CHIP_ID_GP106 0x136 // Discrete GeForce GTX 1060
 #define NV_CHIP_ID_GV11B 0x15B // Jetson Xavier embedded GPU
 
+#define NV_CHIP_ID_FERMI 0x0C0
 #define NV_CHIP_ID_KEPLER 0x0E0
 #define NV_CHIP_ID_MAXWELL 0x120
 #define NV_CHIP_ID_PASCAL 0x130
@@ -1505,7 +1530,7 @@ int get_runlist_iter(
         struct nvdebug_state *g,
         int rl_id,
         struct runlist_iter *rl_iter /* out */);
-int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id);
+int preempt_tsg(struct nvdebug_state *g, uint32_t rl_id, uint32_t tsg_id);
 int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id);
 int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id);
 
diff --git a/nvdebug_entry.c b/nvdebug_entry.c
index 5f99976..d5df7db 100644
--- a/nvdebug_entry.c
+++ b/nvdebug_entry.c
@@ -289,17 +289,45 @@ int __init nvdebug_init(void) {
                         snprintf(runlist_name, 12, "runlist%lu", last_runlist);
                         if (!(rl_dir = proc_mkdir_data(runlist_name, 0555, dir, (void*)device_id)))
                                 goto out_nomem;
+                        // Create one file for each runlist on Ampere+, or one file for each GPU on older
+                        if (g_nvdebug_state[res].chip_id >= NV_CHIP_ID_AMPERE || last_runlist == 0) {
+                                struct proc_dir_entry *chram_scope;
+                                // preempt_tsg, enable_channel, and disable_channel refer to a GPU-global channel
+                                // RAM on pre-Ampere GPUs
+                                if (g_nvdebug_state[res].chip_id >= NV_CHIP_ID_AMPERE)
+                                        chram_scope = rl_dir;
+                                else
+                                        chram_scope = dir;
+                                // Create file `/proc/gpu#/runlist#/preempt_tsg`, world writable
+                                // On Turing and older, `/proc/gpu#/preempt_tsg`
+                                if (!proc_create_data(
+                                                "preempt_tsg", 0222, chram_scope, compat_ops(&preempt_tsg_file_ops),
+                                                (void*)last_runlist))
+                                        goto out_nomem;
+                                // Create file `/proc/gpu#/runlist#/disable_channel`, world writable
+                                // On Turing and older, `/proc/gpu#/disable_channel`
+                                if (!proc_create_data(
+                                                "disable_channel", 0222, chram_scope, compat_ops(&disable_channel_file_ops),
+                                                (void*)last_runlist))
+                                        goto out_nomem;
+                                // Create file `/proc/gpu#/runlist#/enable_channel`, world writable
+                                // On Turing and older, `/proc/gpu#/enable_channel`
+                                if (!proc_create_data(
+                                                "enable_channel", 0222, chram_scope, compat_ops(&enable_channel_file_ops),
+                                                (void*)last_runlist))
+                                        goto out_nomem;
+                        }
                         // Create file `/proc/gpu#/runlist#/runlist`, world readable
                         if (!proc_create_data(
                                         "runlist", 0444, rl_dir, compat_ops(&runlist_file_ops),
                                         (void*)last_runlist))
                                 goto out_nomem;
+                        // Create file `/proc/gpu#/runlist#/switch_to_tsg`, world writable
+                        if (!proc_create_data(
+                                        "switch_to_tsg", 0222, rl_dir, compat_ops(&switch_to_tsg_file_ops),
+                                        (void*)last_runlist))
+                                goto out_nomem;
                 } while (last_runlist-- > 0);
-                // Create file `/proc/gpu#/preempt_tsg`, world writable
-                if (!proc_create_data(
-                                "preempt_tsg", 0222, dir, compat_ops(&preempt_tsg_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
                 /* On the TU104, the context scheduler (contained in the Host, aka
                  * PFIFO, unit) has been observed to sometimes to fail to schedule TSGs
                  * containing re-enabled channels. Resubmitting the runlist
@@ -311,21 +339,6 @@ int __init nvdebug_init(void) {
                                 "resubmit_runlist", 0222, dir, compat_ops(&resubmit_runlist_file_ops),
                                 (void*)device_id))
                         goto out_nomem;
-                // Create file `/proc/gpu#/disable_channel`, world writable
-                if (!proc_create_data(
-                                "disable_channel", 0222, dir, compat_ops(&disable_channel_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
-                // Create file `/proc/gpu#/enable_channel`, world writable
-                if (!proc_create_data(
-                                "enable_channel", 0222, dir, compat_ops(&enable_channel_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
-                // Create file `/proc/gpu#/switch_to_tsg`, world writable
-                if (!proc_create_data(
-                                "switch_to_tsg", 0222, dir, compat_ops(&switch_to_tsg_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
                 // Create file `/proc/gpu#/device_info`, world readable
                 if (!proc_create_data(
                                 "device_info", 0444, dir, compat_ops(&device_info_file_ops),
diff --git a/runlist.c b/runlist.c
index 7e6d292..7bb2ee4 100644
--- a/runlist.c
+++ b/runlist.c
@@ -169,24 +169,35 @@ attempt_pramin_access:
 
 /* Trigger a preempt of the specified TSG
   @param tsg_id ID of TSG to preempt.
+  @param rl_id  Which channel RAM address space to search?
   @return 0 or -errno on error
 
   Note: If no other TSGs exist in the associated runlist, this TSG may
         continue executing, unless NV_PFIFO_SCHED_DISABLE is set, or all the
         channels of the TSG to be preempted are disabled.
 */
-int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) {
-        pfifo_preempt_t pfifo_preempt;
+int preempt_tsg(struct nvdebug_state *g, uint32_t rl_id, uint32_t tsg_id) {
+        pfifo_preempt_t preempt;
+        // Fermi does not support time-slice groups
         if (g->chip_id < NV_CHIP_ID_KEPLER)
                 return -EOPNOTSUPP;
 
-        pfifo_preempt.raw = 0;
-        pfifo_preempt.id = tsg_id;
-        pfifo_preempt.is_pending = 0;
-        pfifo_preempt.type = PREEMPT_TYPE_TSG;
+        preempt.raw = 0;
+        preempt.id = tsg_id;
+        preempt.type = PREEMPT_TYPE_TSG;
 
         // Actually trigger the preemption
-        nvdebug_writel(g, NV_PFIFO_PREEMPT, pfifo_preempt.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                nvdebug_writel(g, NV_PFIFO_PREEMPT, preempt.raw);
+        } else {
+                uint32_t runlist_reg_base;
+                int err;
+                // As TSG and channel IDs are namespaced per-runlist starting with
+                // Ampere, the PREEMPT register is also per-runlist.
+                if ((err = get_runlist_ram(g, rl_id, &runlist_reg_base)))
+                        return err;
+                nvdebug_writel(g, runlist_reg_base + NV_RUNLIST_PREEMPT_GA100, preempt.raw);
+        }
         return 0;
 }
 
@@ -195,18 +206,38 @@ int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) {
   @return 0 or -errno on error
 */
 int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id) {
-        runlist_preempt_t rl_preempt;
-        if (g->chip_id < NV_CHIP_ID_VOLTA)
+        // The runlist preempt register does not exist on Kepler (tested gk104)
+        if (g->chip_id < NV_CHIP_ID_MAXWELL)
                 return -EOPNOTSUPP;
 
-        // Overwrite, as the register contains nothing to preserve
-        rl_preempt.raw = BIT(rl_id);
-        nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
+        // Write to trigger the preemption (the register contains nothing to
+        // preserve, and can thus just be overwritten)
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                runlist_preempt_t rl_preempt;
+                rl_preempt.raw = BIT(rl_id);
+                nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
+        } else {
+                int err;
+                uint32_t runlist_regs_base;
+                pfifo_preempt_t preempt;
+                // The RUNLIST_PREEMPT register was deleted, and the _PREEMPT register
+                // was extended to support runlist-level preemptions starting on Ampere
+                preempt.id = rl_id;
+                preempt.type = PREEMPT_TYPE_RUNLIST;
+                // The preempt register is scoped per-runlist on Ampere+
+                if ((err = get_runlist_ram(g, rl_id, &runlist_regs_base)))
+                        return err;
+                nvdebug_writel(g, runlist_regs_base + NV_RUNLIST_PREEMPT_GA100, preempt.raw);
+        }
         return 0;
 }
 
 // Read and write runlist configuration, triggering a resubmit
 int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
+        // Necessary registers do not exist pre-Fermi
+        if (g->chip_id < NV_CHIP_ID_FERMI)
+                return -EOPNOTSUPP;
+
         if (g->chip_id < NV_CHIP_ID_TURING) {
                 eng_runlist_gf100_t rl;
                 if (rl_id > MAX_RUNLISTS_GF100)
@@ -223,7 +254,19 @@ int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
                         return -EIO;
                 nvdebug_writeq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id), submit.raw);
         } else {
-                return -EOPNOTSUPP;
+                int err;
+                uint32_t runlist_pri_base;
+                runlist_submit_tu102_t submit;
+                if ((err = get_runlist_ram(g, rl_id, &runlist_pri_base)) < 0)
+                        return err;
+                if ((submit.raw = nvdebug_readq(g, runlist_pri_base + NV_RUNLIST_SUBMIT_GA100)) == -1)
+                        return -EIO;
+                // On Ampere, this does not appear to trigger a preempt of the
+                // currently-running channel (even if the currently running channel
+                // becomes disabled), but will cause newly re-enabled channels
+                // (at least if nothing else is pending) to become ready (tested on
+                // Jetson Orin).
+                nvdebug_writeq(g, runlist_pri_base + NV_RUNLIST_SUBMIT_GA100, submit.raw);
         }
         return 0;
 }
diff --git a/runlist_procfs.c b/runlist_procfs.c
index c1cfc87..b2159f6 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -199,11 +199,11 @@ struct file_operations runlist_file_ops = {
 };
 
 ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
-                               size_t count, loff_t *off) {
-        uint32_t target_tsgid;
+                               size_t count, loff_t *off) {
+        uint32_t target_tsgid, target_runlist_ram;
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         if (err)
                 return err;
 
@@ -211,8 +211,15 @@ ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
         if (target_tsgid > MAX_TSGID)
                 return -ERANGE;
 
+        // (Ab)use the PDE_DATA field for the index into which Runlist RAM this TSG
+        // ID is scoped to (only applicable on Ampere+)
+        if (g->chip_id >= NV_CHIP_ID_AMPERE)
+                target_runlist_ram = file2gpuidx(f);
+        else
+                target_runlist_ram = 0;
+
         // Execute preemption
-        if ((err = preempt_tsg(g, target_tsgid)))
+        if ((err = preempt_tsg(g, target_runlist_ram, target_tsgid)))
                 return err;
 
         return count;
@@ -223,13 +230,12 @@ struct file_operations preempt_tsg_file_ops = {
         .llseek = default_llseek,
 };
 
-
 ssize_t resubmit_runlist_file_write(struct file *f, const char __user *buffer,
-                               size_t count, loff_t *off) {
+                                    size_t count, loff_t *off) {
         uint32_t target_runlist;
+        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         int err = kstrtou32_from_user(buffer, count, 0, &target_runlist);
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         if (err)
                 return err;
 
@@ -245,26 +251,48 @@ struct file_operations resubmit_runlist_file_ops = {
         .llseek = default_llseek,
 };
 
+
 ssize_t disable_channel_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
+                                   size_t count, loff_t *off) {
         uint32_t target_channel;
-        channel_ctrl_t chan;
-        int err;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
                 return err;
 
-        if (target_channel > MAX_CHID)
-                return -ERANGE;
-
-        // Read current configuration
-        if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
-                return -EIO;
-        // Request disablement
-        chan.enable_clear = true;
-        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                channel_ctrl_t chan;
+                if (target_channel > MAX_CHID)
+                        return -ERANGE;
+                // Read current configuration
+                if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
+                        return -EIO;
+                // Request disablement
+                chan.enable_clear = true;
+                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        } else {
+                uint32_t runlist_reg_base, chram_base, channel_max;
+                runlist_channel_config_t channel_config;
+                channel_ctrl_ga100_t chan;
+                // (Ab)use the PDE_DATA field for the runlist ID
+                if ((err = get_runlist_ram(g, file2gpuidx(f), &runlist_reg_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((channel_config.raw = nvdebug_readl(g, runlist_reg_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_max = 1u << channel_config.num_channels_log2;
+                if (target_channel >= channel_max)
+                        return -ERANGE;
+                chram_base = (uint32_t)channel_config.bar0_offset << 4;
+                // Writing zeros to any field of the Ampere+ channel control structure
+                // does nothing, so don't bother to read the structure first, and just
+                // write zeros to all the fields we don't care about.
+                chan.raw = 0;
+                chan.is_write_one_clears_bits = 1; // Invert meaning of writing 1
+                chan.enable = 1;
+                nvdebug_writel(g, chram_base + sizeof(channel_ctrl_ga100_t) * target_channel, chan.raw);
+        }
 
         return count;
 }
@@ -275,23 +303,45 @@ struct file_operations disable_channel_file_ops = {
 };
 
 ssize_t enable_channel_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
+                                  size_t count, loff_t *off) {
         uint32_t target_channel;
-        channel_ctrl_t chan;
-        int err;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
                 return err;
 
-        if (target_channel > MAX_CHID)
-                return -ERANGE;
-
-        // Disable channel
-        chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel));
-        chan.enable_set = true;
-        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                channel_ctrl_t chan;
+                if (target_channel > MAX_CHID)
+                        return -ERANGE;
+                // Read current configuration
+                if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
+                        return -EIO;
+                // Disable channel
+                chan.enable_set = true;
+                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        } else {
+                uint32_t runlist_reg_base, chram_base, channel_max;
+                runlist_channel_config_t channel_config;
+                channel_ctrl_ga100_t chan;
+                // (Ab)use the PDE_DATA field for the runlist ID
+                if ((err = get_runlist_ram(g, file2gpuidx(f), &runlist_reg_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((channel_config.raw = nvdebug_readl(g, runlist_reg_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_max = 1u << channel_config.num_channels_log2;
+                if (target_channel >= channel_max)
+                        return -ERANGE;
+                chram_base = (uint32_t)channel_config.bar0_offset << 4;
+                // Writing zeros to any field of the Ampere+ channel control structure
+                // does nothing, so don't bother to read the structure first, and just
+                // write zeros to all the fields we don't care about.
+                chan.raw = 0;
+                chan.enable = 1;
+                nvdebug_writel(g, chram_base + sizeof(channel_ctrl_ga100_t) * target_channel, chan.raw);
+        }
 
         return count;
 }
@@ -301,52 +351,84 @@ struct file_operations enable_channel_file_ops = {
         .llseek = default_llseek,
 };
 
-// Note: Operates only on runlist 0 (Compute/Graphics)
+// Tested working on Pascal (gp106) through Ada (ad102)
 ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
-        uint32_t target_tsgid;
+                                 size_t count, loff_t *off) {
+        uint32_t target_tsgid, target_runlist, channel_regs_base;
         struct gv100_runlist_chan* chan;
         channel_ctrl_t chan_ctl;
+        channel_ctrl_ga100_t chan_ctl_ga100;
         struct runlist_iter rl_iter;
-        int err;
         loff_t pos = 0;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
         if (err)
                 return err;
 
         if (target_tsgid > MAX_TSGID)
                 return -ERANGE;
 
-        err = get_runlist_iter(g, 0, &rl_iter);
-        if (err)
+        // (Ab)use the PDE_DATA field for the runlist ID
+        target_runlist = file2gpuidx(f);
+
+        if ((err = get_runlist_iter(g, target_runlist, &rl_iter)))
                 return err;
 
+        // On Ampere, TSG and Channel IDs are only unique per-runlist, so we need
+        // to pull the per-runlist copy of Channel RAM.
+        if (g->chip_id >= NV_CHIP_ID_AMPERE) {
+                uint32_t runlist_regs_base;
+                runlist_channel_config_t chan_config;
+                if ((err = get_runlist_ram(g, target_runlist, &runlist_regs_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((chan_config.raw = nvdebug_readl(g, runlist_regs_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_regs_base = (uint32_t)chan_config.bar0_offset << 4;
+        }
+
         // Iterate through all TSGs
         while (pos < rl_iter.len) {
-                if (tsgid(g, rl_iter.curr_entry) == target_tsgid) {
-                        // Enable channels of target TSG
-                        for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
-                                chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
-                                chan_ctl.enable_set = true;
-                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
-                        }
-                } else {
-                        // XXX: Fix for bare channels. Maybe a "for_chan_until_tsg" macro?
-                        // Disable all other channels
-                        // (This is how the Jetson nvgpu driver disables TSGs)
-                        for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
-                                chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
-                                chan_ctl.enable_clear = true;
-                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
+                bool enable = false;
+                if (tsgid(g, rl_iter.curr_entry) == target_tsgid)
+                        enable = true;
+
+                // Either enable or disable all channels of each TSG, dependent on if
+                // they are contained within the target TSG or not.
+                for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
+                        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                                // Read, update, write for PCCSR
+                                if ((chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chid(g, chan)))) == -1)
+                                        return -EIO;
+                                if (enable)
+                                        chan_ctl.enable_set = true;
+                                else
+                                        chan_ctl.enable_clear = true;
+                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chid(g, chan)), chan_ctl.raw);
+                        } else {
+                                // Writing a 0 does nothing on Ampere+, so we can just write
+                                chan_ctl_ga100.raw = 0;
+                                chan_ctl_ga100.is_write_one_clears_bits = !enable;
+                                chan_ctl_ga100.enable = true;
+                                nvdebug_writel(g, channel_regs_base + sizeof(chan_ctl_ga100) * chid(g, chan), chan_ctl_ga100.raw);
                         }
                 }
                 pos += 1 + tsg_length(g, rl_iter.curr_entry);
                 rl_iter.curr_entry = next_tsg(g, rl_iter.curr_entry);
+
+                // TODO: Fix the above for bare channels. Add "for_chan_until_tsg"?
         }
-        // Trigger a runlist-level preempt to switch to `target_tsgid`
-        if ((err = preempt_runlist(g, 0)))
+
+        // Resubmit the runlist to ensure that changes to channel enablement are
+        // picked up on Turing+ GPUs (channel enablements may not be otherwise).
+        if (g->chip_id >= NV_CHIP_ID_TURING)
+                if ((err = resubmit_runlist(g, target_runlist)))
+                        return err;
+
+        // Trigger a runlist-level preempt to stop whatever was running, triggering
+        // the runlist scheduler to select and run the next-enabled channel.
+        if ((err = preempt_runlist(g, target_runlist)))
                 return err;
 
         return count;