1 files changed, 140 insertions, 58 deletions

diff --git a/runlist_procfs.c b/runlist_procfs.c
index c1cfc87..b2159f6 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -199,11 +199,11 @@ struct file_operations runlist_file_ops = {
 };
 
 ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
-                               size_t count, loff_t *off) {
-        uint32_t target_tsgid;
+                               size_t count, loff_t *off) {
+        uint32_t target_tsgid, target_runlist_ram;
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         if (err)
                 return err;
 
@@ -211,8 +211,15 @@ ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
         if (target_tsgid > MAX_TSGID)
                 return -ERANGE;
 
+        // (Ab)use the PDE_DATA field for the index into which Runlist RAM this TSG
+        // ID is scoped to (only applicable on Ampere+)
+        if (g->chip_id >= NV_CHIP_ID_AMPERE)
+                target_runlist_ram = file2gpuidx(f);
+        else
+                target_runlist_ram = 0;
+
         // Execute preemption
-        if ((err = preempt_tsg(g, target_tsgid)))
+        if ((err = preempt_tsg(g, target_runlist_ram, target_tsgid)))
                 return err;
 
         return count;
@@ -223,13 +230,12 @@ struct file_operations preempt_tsg_file_ops = {
         .llseek = default_llseek,
 };
 
-
 ssize_t resubmit_runlist_file_write(struct file *f, const char __user *buffer,
-                               size_t count, loff_t *off) {
+                                    size_t count, loff_t *off) {
         uint32_t target_runlist;
+        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         int err = kstrtou32_from_user(buffer, count, 0, &target_runlist);
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         if (err)
                 return err;
 
@@ -245,26 +251,48 @@ struct file_operations resubmit_runlist_file_ops = {
         .llseek = default_llseek,
 };
 
+
 ssize_t disable_channel_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
+                                   size_t count, loff_t *off) {
         uint32_t target_channel;
-        channel_ctrl_t chan;
-        int err;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
                 return err;
 
-        if (target_channel > MAX_CHID)
-                return -ERANGE;
-
-        // Read current configuration
-        if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
-                return -EIO;
-        // Request disablement
-        chan.enable_clear = true;
-        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                channel_ctrl_t chan;
+                if (target_channel > MAX_CHID)
+                        return -ERANGE;
+                // Read current configuration
+                if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
+                        return -EIO;
+                // Request disablement
+                chan.enable_clear = true;
+                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        } else {
+                uint32_t runlist_reg_base, chram_base, channel_max;
+                runlist_channel_config_t channel_config;
+                channel_ctrl_ga100_t chan;
+                // (Ab)use the PDE_DATA field for the runlist ID
+                if ((err = get_runlist_ram(g, file2gpuidx(f), &runlist_reg_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((channel_config.raw = nvdebug_readl(g, runlist_reg_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_max = 1u << channel_config.num_channels_log2;
+                if (target_channel >= channel_max)
+                        return -ERANGE;
+                chram_base = (uint32_t)channel_config.bar0_offset << 4;
+                // Writing zeros to any field of the Ampere+ channel control structure
+                // does nothing, so don't bother to read the structure first, and just
+                // write zeros to all the fields we don't care about.
+                chan.raw = 0;
+                chan.is_write_one_clears_bits = 1; // Invert meaning of writing 1
+                chan.enable = 1;
+                nvdebug_writel(g, chram_base + sizeof(channel_ctrl_ga100_t) * target_channel, chan.raw);
+        }
 
         return count;
 }
@@ -275,23 +303,45 @@ struct file_operations disable_channel_file_ops = {
 };
 
 ssize_t enable_channel_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
+                                  size_t count, loff_t *off) {
         uint32_t target_channel;
-        channel_ctrl_t chan;
-        int err;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
                 return err;
 
-        if (target_channel > MAX_CHID)
-                return -ERANGE;
-
-        // Disable channel
-        chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel));
-        chan.enable_set = true;
-        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                channel_ctrl_t chan;
+                if (target_channel > MAX_CHID)
+                        return -ERANGE;
+                // Read current configuration
+                if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
+                        return -EIO;
+                // Disable channel
+                chan.enable_set = true;
+                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        } else {
+                uint32_t runlist_reg_base, chram_base, channel_max;
+                runlist_channel_config_t channel_config;
+                channel_ctrl_ga100_t chan;
+                // (Ab)use the PDE_DATA field for the runlist ID
+                if ((err = get_runlist_ram(g, file2gpuidx(f), &runlist_reg_base)))
+                        return err;
+                // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere
+                if ((channel_config.raw = nvdebug_readl(g, runlist_reg_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1)
+                        return -EIO;
+                channel_max = 1u << channel_config.num_channels_log2;
+                if (target_channel >= channel_max)
+                        return -ERANGE;
+                chram_base = (uint32_t)channel_config.bar0_offset << 4;
+                // Writing zeros to any field of the Ampere+ channel control structure
+                // does nothing, so don't bother to read the structure first, and just
+                // write zeros to all the fields we don't care about.
+                chan.raw = 0;
+                chan.enable = 1;
+                nvdebug_writel(g, chram_base + sizeof(channel_ctrl_ga100_t) * target_channel, chan.raw);
+        }
 
         return count;
 }
@@ -301,52 +351,84 @@ struct file_operations enable_channel_file_ops = {
         .llseek = default_llseek,
 };
 
-// Note: Operates only on runlist 0 (Compute/Graphics)
+// Tested working on Pascal (gp106) through Ada (ad102)
 ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer,
-                                   size_t count, loff_t *off) {
-        uint32_t target_tsgid;
+                                 size_t count, loff_t *off) {
+        uint32_t target_tsgid, target_runlist, channel_regs_base;
         struct gv100_runlist_chan* chan;
         channel_ctrl_t chan_ctl;
+        channel_ctrl_ga100_t chan_ctl_ga100;
         struct runlist_iter rl_iter;
-        int err;
         loff_t pos = 0;
-        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
         if (err)
                 return err;
 
         if (target_tsgid > MAX_TSGID)
                 return -ERANGE;
 
-        err = get_runlist_iter(g, 0, &rl_iter);
-        if (err)
+        // (Ab)use the PDE_DATA field for the runlist ID
+        target_runlist = file2gpuidx(f);