Ampere: disable/enable_channel, preempt/switch_to_tsg, and resubmit_runlist

**Modifes the user API from `echo 1 > /proc/gpuX/switch_to_tsg` to
`echo 1 > /proc/gpuX/runlist0/switch_to_tsg` to switch to TSG 1 on
runlist 0 on GPU X for pre-Ampere GPUs (for example).**

Feature changes:
- switch_to_tsg only makes sense on a per-runlist level. Before, this
  always operated on runlist0; this commit allows operating on any
  runlist by moving the API to the per-runlist paths.
- On Ampere+, channel and TSG IDs are per-runlist, and no longer
  GPU-global. Consequently, the disable/enable_channel and
  preempt_tsg APIs have been moved from GPU-global to per-runlist
  paths on Ampere+.

Bug fixes:
- `preempt_runlist()` is now supported on Maxwell and Pascal.
- `resubmit_runlist()` detects too-old GPUs.
- MAX_CHID corrected from 512 to 511 and documented.
- switch_to_tsg now includes a runlist resubmit, which appears to be
  necessary on Turing+ GPUs.

Tested on GK104 (Quadro K5000), GM204 (GTX 970), GP106 (GTX 1060 3GB),
GP104 (GTX 1080 Ti), GP10B (Jetson TX2), GV11B (Jetson Xavier), GV100
(Titan V), TU102 (RTX 2080 Ti), and AD102 (RTX 6000 Ada).

1 files changed, 140 insertions, 58 deletions

diff --git a/runlist_procfs.c b/runlist_procfs.c
index c1cfc87..b2159f6 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -199,11 +199,11 @@ struct file_operations runlist_file_ops = {
 };
 
 ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
-                               size_t count, loff_t *off) {
-        uint32_t target_tsgid;
+                               size_t count, loff_t *off) {
+        uint32_t target_tsgid, target_runlist_ram;
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         if (err)
                 return err;
 
@@ -211,8 +211,15 @@ ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
         if (target_tsgid > MAX_TSGID)
                 return -ERANGE;
 
+        // (Ab)use the PDE_DATA field for the index into which Runlist RAM this TSG
+        // ID is scoped to (only applicable on Ampere+)
+        if (g->chip_id >= NV_CHIP_ID_AMPERE)
+                target_runlist_ram = file2gpuidx(f);
+        else
+                target_runlist_ram = 0;
+
         // Execute preemption
-        if ((err = preempt_tsg(g, target_tsgid)))
+        if ((err = preempt_tsg(g, target_runlist_ram, target_tsgid)))
                 return err;
 
         return count;
@@ -223,13 +230,12 @@ struct file_operations preempt_tsg_file_ops = {
         .llseek = default_llseek,
 };
 
-
 ssize_t resubmit_runlist_file_write(struct file *f, const char __user *buffer,
-                               size_t count, loff_t *off) {
+                                    size_t count, loff_t *off) {
         uint32_t target_runlist;
+        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         int err = kstrtou32_from_user(buffer, count, 0, &target_runlist);
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         if (err)
                 return err;
 
@@ -245,26 +251,48 @@ struct file_operations resubmit_runlist_file_ops = {
         .llseek = default_llseek,
 };
 
+
 ssize_t disable_channel_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
+                                   size_t count, loff_t *off) {
         uint32_t target_channel;
-        channel_ctrl_t chan;
-        int err;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
                 return err;
 
-        if (target_channel > MAX_CHID)
-                return -ERANGE;
-
-        // Read current configuration
-        if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
-                return -EIO;
-        // Request disablement
-        chan.enable_clear = true;
-        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                channel_ctrl_t chan;
+                if (target_channel > MAX_CHID)
+                        return -ERANGE;
+                // Read current configuration
+                if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
+                        return -EIO;
+                // Request disablement
+                chan.enable_clear = true;
+                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        } else {
+                uint32_t runlist_reg_base, chram_base, channel_max;
+                runlist_channel_config_t channel_config;
+                channel_ctrl_ga100_t chan;
+                // (Ab)use the PDE_DATA field for the runlist ID
+                if ((err = get_runlist_ram(g, file2gpuidx(f), &runlist_reg_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((channel_config.raw = nvdebug_readl(g, runlist_reg_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_max = 1u << channel_config.num_channels_log2;
+                if (target_channel >= channel_max)
+                        return -ERANGE;
+                chram_base = (uint32_t)channel_config.bar0_offset << 4;
+                // Writing zeros to any field of the Ampere+ channel control structure
+                // does nothing, so don't bother to read the structure first, and just
+                // write zeros to all the fields we don't care about.
+                chan.raw = 0;
+                chan.is_write_one_clears_bits = 1; // Invert meaning of writing 1
+                chan.enable = 1;
+                nvdebug_writel(g, chram_base + sizeof(channel_ctrl_ga100_t) * target_channel, chan.raw);
+        }
 
         return count;
 }
@@ -275,23 +303,45 @@ struct file_operations disable_channel_file_ops = {
 };
 
 ssize_t enable_channel_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
+                                  size_t count, loff_t *off) {
         uint32_t target_channel;
-        channel_ctrl_t chan;
-        int err;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
                 return err;
 
-        if (target_channel > MAX_CHID)
-                return -ERANGE;
-
-        // Disable channel
-        chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel));
-        chan.enable_set = true;
-        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                channel_ctrl_t chan;
+                if (target_channel > MAX_CHID)
+                        return -ERANGE;
+                // Read current configuration
+                if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
+                        return -EIO;
+                // Disable channel
+                chan.enable_set = true;
+                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        } else {
+                uint32_t runlist_reg_base, chram_base, channel_max;
+                runlist_channel_config_t channel_config;
+                channel_ctrl_ga100_t chan;
+                // (Ab)use the PDE_DATA field for the runlist ID
+                if ((err = get_runlist_ram(g, file2gpuidx(f), &runlist_reg_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((channel_config.raw = nvdebug_readl(g, runlist_reg_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_max = 1u << channel_config.num_channels_log2;
+                if (target_channel >= channel_max)
+                        return -ERANGE;
+                chram_base = (uint32_t)channel_config.bar0_offset << 4;
+                // Writing zeros to any field of the Ampere+ channel control structure
+                // does nothing, so don't bother to read the structure first, and just
+                // write zeros to all the fields we don't care about.
+                chan.raw = 0;
+                chan.enable = 1;
+                nvdebug_writel(g, chram_base + sizeof(channel_ctrl_ga100_t) * target_channel, chan.raw);
+        }
 
         return count;
 }
@@ -301,52 +351,84 @@ struct file_operations enable_channel_file_ops = {
         .llseek = default_llseek,
 };
 
-// Note: Operates only on runlist 0 (Compute/Graphics)
+// Tested working on Pascal (gp106) through Ada (ad102)
 ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
-        uint32_t target_tsgid;
+                                 size_t count, loff_t *off) {
+        uint32_t target_tsgid, target_runlist, channel_regs_base;
         struct gv100_runlist_chan* chan;
         channel_ctrl_t chan_ctl;
+        channel_ctrl_ga100_t chan_ctl_ga100;
         struct runlist_iter rl_iter;
-        int err;
         loff_t pos = 0;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
         if (err)
                 return err;
 
         if (target_tsgid > MAX_TSGID)
                 return -ERANGE;
 
-        err = get_runlist_iter(g, 0, &rl_iter);
-        if (err)
+        // (Ab)use the PDE_DATA field for the runlist ID
+        target_runlist = file2gpuidx(f);
+
+        if ((err = get_runlist_iter(g, target_runlist, &rl_iter)))
                 return err;
 
+        // On Ampere, TSG and Channel IDs are only unique per-runlist, so we need
+        // to pull the per-runlist copy of Channel RAM.
+        if (g->chip_id >= NV_CHIP_ID_AMPERE) {
+                uint32_t runlist_regs_base;
+                runlist_channel_config_t chan_config;
+                if ((err = get_runlist_ram(g, target_runlist, &runlist_regs_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((chan_config.raw = nvdebug_readl(g, runlist_regs_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_regs_base = (uint32_t)chan_config.bar0_offset << 4;
+        }
+
         // Iterate through all TSGs
         while (pos < rl_iter.len) {
-                if (tsgid(g, rl_iter.curr_entry) == target_tsgid) {
-                        // Enable channels of target TSG
-                        for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
-                                chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
-                                chan_ctl.enable_set = true;
-                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
-                        }
-                } else {
-                        // XXX: Fix for bare channels. Maybe a "for_chan_until_tsg" macro?
-                        // Disable all other channels
-                        // (This is how the Jetson nvgpu driver disables TSGs)
-                        for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
-                                chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
-                                chan_ctl.enable_clear = true;
-                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
+                bool enable = false;
+                if (tsgid(g, rl_iter.curr_entry) == target_tsgid)
+                        enable = true;
+
+                // Either enable or disable all channels of each TSG, dependent on if
+                // they are contained within the target TSG or not.
+                for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
+                        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                                // Read, update, write for PCCSR
+                                if ((chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chid(g, chan)))) == -1)
+                                        return -EIO;
+                                if (enable)
+                                        chan_ctl.enable_set = true;
+                                else
+                                        chan_ctl.enable_clear = true;
+                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chid(g, chan)), chan_ctl.raw);
+                        } else {
+                                // Writing a 0 does nothing on Ampere+, so we can just write
+                                chan_ctl_ga100.raw = 0;
+                                chan_ctl_ga100.is_write_one_clears_bits = !enable;
+                                chan_ctl_ga100.enable = true;
+                                nvdebug_writel(g, channel_regs_base + sizeof(chan_ctl_ga100) * chid(g, chan), chan_ctl_ga100.raw);
                         }
                 }
                 pos += 1 + tsg_length(g, rl_iter.curr_entry);
                 rl_iter.curr_entry = next_tsg(g, rl_iter.curr_entry);
+
+                // TODO: Fix the above for bare channels. Add "for_chan_until_tsg"?
         }
-        // Trigger a runlist-level preempt to switch to `target_tsgid`
-        if ((err = preempt_runlist(g, 0)))
+
+        // Resubmit the runlist to ensure that changes to channel enablement are
+        // picked up on Turing+ GPUs (channel enablements may not be otherwise).
+        if (g->chip_id >= NV_CHIP_ID_TURING)
+                if ((err = resubmit_runlist(g, target_runlist)))
+                        return err;
+
+        // Trigger a runlist-level preempt to stop whatever was running, triggering
+        // the runlist scheduler to select and run the next-enabled channel.
+        if ((err = preempt_runlist(g, target_runlist)))
                 return err;
 
         return count;