1 files changed, 140 insertions, 58 deletions

diff --git a/runlist_procfs.c b/runlist_procfs.c
index c1cfc87..b2159f6 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -199,11 +199,11 @@ struct file_operations runlist_file_ops = {
 };
 
 ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
-                               size_t count, loff_t *off) {
-        uint32_t target_tsgid;
+                               size_t count, loff_t *off) {
+        uint32_t target_tsgid, target_runlist_ram;
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         if (err)
                 return err;
 
@@ -211,8 +211,15 @@ ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
         if (target_tsgid > MAX_TSGID)
                 return -ERANGE;
 
+        // (Ab)use the PDE_DATA field for the index into which Runlist RAM this TSG
+        // ID is scoped to (only applicable on Ampere+)
+        if (g->chip_id >= NV_CHIP_ID_AMPERE)
+                target_runlist_ram = file2gpuidx(f);
+        else
+                target_runlist_ram = 0;
+
         // Execute preemption
-        if ((err = preempt_tsg(g, target_tsgid)))
+        if ((err = preempt_tsg(g, target_runlist_ram, target_tsgid)))
                 return err;
 
         return count;
@@ -223,13 +230,12 @@ struct file_operations preempt_tsg_file_ops = {
         .llseek = default_llseek,
 };
 
-
 ssize_t resubmit_runlist_file_write(struct file *f, const char __user *buffer,
-                               size_t count, loff_t *off) {
+                                    size_t count, loff_t *off) {
         uint32_t target_runlist;
+        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         int err = kstrtou32_from_user(buffer, count, 0, &target_runlist);
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         if (err)
                 return err;
 
@@ -245,26 +251,48 @@ struct file_operations resubmit_runlist_file_ops = {
         .llseek = default_llseek,
 };
 
+
 ssize_t disable_channel_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
+                                   size_t count, loff_t *off) {
         uint32_t target_channel;
-        channel_ctrl_t chan;
-        int err;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
                 return err;
 
-        if (target_channel > MAX_CHID)
-                return -ERANGE;
-
-        // Read current configuration
-        if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
-                return -EIO;
-        // Request disablement
-        chan.enable_clear = true;
-        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                channel_ctrl_t chan;
+                if (target_channel > MAX_CHID)
+                        return -ERANGE;
+                // Read current configuration
+                if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
+                        return -EIO;
+                // Request disablement
+                chan.enable_clear = true;
+                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        } else {
+                uint32_t runlist_reg_base, chram_base, channel_max;
+                runlist_channel_config_t channel_config;
+                channel_ctrl_ga100_t chan;
+                // (Ab)use the PDE_DATA field for the runlist ID
+                if ((err = get_runlist_ram(g, file2gpuidx(f), &runlist_reg_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((channel_config.raw = nvdebug_readl(g, runlist_reg_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_max = 1u << channel_config.num_channels_log2;
+                if (target_channel >= channel_max)
+                        return -ERANGE;
+                chram_base = (uint32_t)channel_config.bar0_offset << 4;
+                // Writing zeros to any field of the Ampere+ channel control structure
+                // does nothing, so don't bother to read the structure first, and just
+                // write zeros to all the fields we don't care about.
+                chan.raw = 0;
+                chan.is_write_one_clears_bits = 1; // Invert meaning of writing 1
+                chan.enable = 1;
+                nvdebug_writel(g, chram_base + sizeof(channel_ctrl_ga100_t) * target_channel, chan.raw);
+        }
 
         return count;
 }
@@ -275,23 +303,45 @@ struct file_operations disable_channel_file_ops = {
 };
 
 ssize_t enable_channel_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
+                                  size_t count, loff_t *off) {
         uint32_t target_channel;
-        channel_ctrl_t chan;
-        int err;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
                 return err;
 
-        if (target_channel > MAX_CHID)
-                return -ERANGE;
-
-        // Disable channel
-        chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel));
-        chan.enable_set = true;
-        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                channel_ctrl_t chan;
+                if (target_channel > MAX_CHID)
+                        return -ERANGE;
+                // Read current configuration
+                if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
+                        return -EIO;
+                // Disable channel
+                chan.enable_set = true;
+                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        } else {
+                uint32_t runlist_reg_base, chram_base, channel_max;
+                runlist_channel_config_t channel_config;
+                channel_ctrl_ga100_t chan;
+                // (Ab)use the PDE_DATA field for the runlist ID
+                if ((err = get_runlist_ram(g, file2gpuidx(f), &runlist_reg_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((channel_config.raw = nvdebug_readl(g, runlist_reg_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_max = 1u << channel_config.num_channels_log2;
+                if (target_channel >= channel_max)
+                        return -ERANGE;
+                chram_base = (uint32_t)channel_config.bar0_offset << 4;
+                // Writing zeros to any field of the Ampere+ channel control structure
+                // does nothing, so don't bother to read the structure first, and just
+                // write zeros to all the fields we don't care about.
+                chan.raw = 0;
+                chan.enable = 1;
+                nvdebug_writel(g, chram_base + sizeof(channel_ctrl_ga100_t) * target_channel, chan.raw);
+        }
 
         return count;
 }
@@ -301,52 +351,84 @@ struct file_operations enable_channel_file_ops = {
         .llseek = default_llseek,
 };
 
-// Note: Operates only on runlist 0 (Compute/Graphics)
+// Tested working on Pascal (gp106) through Ada (ad102)
 ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
-        uint32_t target_tsgid;
+                                 size_t count, loff_t *off) {
+        uint32_t target_tsgid, target_runlist, channel_regs_base;
         struct gv100_runlist_chan* chan;
         channel_ctrl_t chan_ctl;
+        channel_ctrl_ga100_t chan_ctl_ga100;
         struct runlist_iter rl_iter;
-        int err;
         loff_t pos = 0;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
         if (err)
                 return err;
 
         if (target_tsgid > MAX_TSGID)
                 return -ERANGE;
 
-        err = get_runlist_iter(g, 0, &rl_iter);
-        if (err)
+        // (Ab)use the PDE_DATA field for the runlist ID
+        target_runlist = file2gpuidx(f);
+
+        if ((err = get_runlist_iter(g, target_runlist, &rl_iter)))
                 return err;
 
+        // On Ampere, TSG and Channel IDs are only unique per-runlist, so we need
+        // to pull the per-runlist copy of Channel RAM.
+        if (g->chip_id >= NV_CHIP_ID_AMPERE) {
+                uint32_t runlist_regs_base;
+                runlist_channel_config_t chan_config;
+                if ((err = get_runlist_ram(g, target_runlist, &runlist_regs_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((chan_config.raw = nvdebug_readl(g, runlist_regs_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_regs_base = (uint32_t)chan_config.bar0_offset << 4;
+        }
+
         // Iterate through all TSGs
         while (pos < rl_iter.len) {
-                if (tsgid(g, rl_iter.curr_entry) == target_tsgid) {
-                        // Enable channels of target TSG
-                        for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
-                                chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
-                                chan_ctl.enable_set = true;
-                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
-                        }
-                } else {
-                        // XXX: Fix for bare channels. Maybe a "for_chan_until_tsg" macro?
-                        // Disable all other channels
-                        // (This is how the Jetson nvgpu driver disables TSGs)
-                        for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
-                                chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
-                                chan_ctl.enable_clear = true;
-                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
+                bool enable = false;
+                if (tsgid(g, rl_iter.curr_entry) == target_tsgid)
+                        enable = true;
+
+                // Either enable or disable all channels of each TSG, dependent on if
+                // they are contained within the target TSG or not.
+                for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
+                        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                                // Read, update, write for PCCSR
+                                if ((chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chid(g, chan)))) == -1)
+                                        return -EIO;
+                                if (enable)
+                                        chan_ctl.enable_set = true;
+                                else
+                                        chan_ctl.enable_clear = true;
+                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chid(g, chan)), chan_ctl.raw);
+                        } else {
+                                // Writing a 0 does nothing on Ampere+, so we can just write
+                                chan_ctl_ga100.raw = 0;
+                                chan_ctl_ga100.is_write_one_clears_bits = !enable;
+                                chan_ctl_ga100.enable = true;
+                                nvdebug_writel(g, channel_regs_base + sizeof(chan_ctl_ga100) * chid(g, chan), chan_ctl_ga100.raw);
                         }
                 }
                 pos += 1 + tsg_length(g, rl_iter.curr_entry);
                 rl_iter.curr_entry = next_tsg(g, rl_iter.curr_entry);
+
+                // TODO: Fix the above for bare channels. Add "for_chan_until_tsg"?
         }
-        // Trigger a runlist-level preempt to switch to `target_tsgid`
-        if ((err = preempt_runlist(g, 0)))
+
+        // Resubmit the runlist to ensure that changes to channel enablement are
+        // picked up on Turing+ GPUs (channel enablements may not be otherwise).
+        if (g->chip_id >= NV_CHIP_ID_TURING)
+                if ((err = resubmit_runlist(g, target_runlist)))
+                        return err;
+
+        // Trigger a runlist-level preempt to stop whatever was running, triggering
+        // the runlist scheduler to select and run the next-enabled channel.
+        if ((err = preempt_runlist(g, target_runlist)))
                 return err;
 
         return count;