Heavily refactor runlist code for correctness and Turing support

- Support differently-formatted runlist registers on Turing
- Support different runlist register offsets on Turing
- Fix incorrect indenting when printing the runlist
- Fix `preempt_tsg` and `switch_to_tsg` API implementations to
  correctly interface with the hardware (previously, they would try
  to disable scheduling for the last-updated runlist pointer, which
  was nonsense, and just an artifact of my early misunderstandings
  of how the NV_PFIFO_RUNLIST* registers worked).
- Remove misused NV_PFIFO_RUNLIST and NV_PFIFO_RUNLIST_BASE registers
- Refactor `runlist.c` to use the APIs from `bus.c`

1 files changed, 22 insertions, 30 deletions

diff --git a/runlist_procfs.c b/runlist_procfs.c
index f7f937d..7dedee3 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -69,12 +69,12 @@ static void *runlist_file_seq_start(struct seq_file *s, loff_t *pos) {
                 if (err)
                         return ERR_PTR(err);
                 // Don't try to print an empty runlist
-                if (rl_iter.rl_info.len <= 0)
+                if (rl_iter.len <= 0)
                         return NULL;
                 return &rl_iter;
         }
         // If we're resuming an earlier print
-        if (*pos < rl_iter.rl_info.len) {
+        if (*pos < rl_iter.len) {
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0)
                 // There's a nasty bug prior to 4.19-rc1 that if the buffer overflows, the
                 // last update to `pos` is not saved. Work around that here by reloading a
@@ -98,14 +98,16 @@ static void* runlist_file_seq_next(struct seq_file *s, void *raw_rl_iter,
         (*pos)++;
         rl_iter->curr_entry += NV_RL_ENTRY_SIZE(g);
         // Verify we haven't reached the end of the runlist
-        // rl_info.len is the num of tsg entries + total num of channel entries
-        if (*pos < rl_iter->rl_info.len) {
+        // len is the num of tsg entries + total num of channel entries
+        if (*pos < rl_iter->len) {
                 ret = rl_iter;
         }
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0)
         // Bug workaround. See comment in runlist_file_seq_start()
         pos_fixup = ret ? *pos : 0;
 #endif
+        if (rl_iter->entries_left_in_tsg)
+                rl_iter->entries_left_in_tsg--;
         return ret;
 }
 
@@ -113,17 +115,19 @@ static void runlist_file_seq_stop(struct seq_file *s, void *raw_rl_iter) {
         // No cleanup needed
 }
 
+// _show() must be idempotent. This function will be rerun if the seq_printf
+// buffer was too small.
 static int runlist_file_seq_show(struct seq_file *s, void *raw_rl_iter) {
         struct runlist_iter *rl_iter = raw_rl_iter;
         void *entry = rl_iter->curr_entry;
         struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(s->file)];
         if (entry_type(g, entry) == ENTRY_TYPE_TSG) {
-                if (rl_iter->channels_left_in_tsg) {
-                        printk(KERN_WARNING "[nvdebug] Found TSG ID%d @ %px when %d channels were still expected under the previous TSG in the runlist!\n", tsgid(g, entry), entry, rl_iter->channels_left_in_tsg);
-                        while (rl_iter->channels_left_in_tsg--)
+                if (rl_iter->entries_left_in_tsg) {
+                        printk(KERN_WARNING "[nvdebug] Found TSG ID%d @ %px when %d channels were still expected under the previous TSG in the runlist!\n", tsgid(g, entry), entry, rl_iter->entries_left_in_tsg);
+                        while (rl_iter->entries_left_in_tsg--)
                                 seq_printf(s, "[missing channel]\n");
                 }
-                rl_iter->channels_left_in_tsg = tsg_length(g, entry);
+                rl_iter->entries_left_in_tsg = tsg_length(g, entry) + 1;
                 seq_printf(s, "+---- TSG Entry %-3d---+\n", tsgid(g, entry));
                 seq_printf(s, "| Scale: %-13d|\n", timeslice_scale(g, entry));
                 seq_printf(s, "| Timeout: %-11d|\n", timeslice_timeout(g, entry));
@@ -134,10 +138,8 @@ static int runlist_file_seq_show(struct seq_file *s, void *raw_rl_iter) {
 #ifndef DETAILED_CHANNEL_INFO
                 u64 instance_ptr = 0;
 #endif
-                if (rl_iter->channels_left_in_tsg) {
+                if (rl_iter->entries_left_in_tsg)
                         indt = "  ";
-                        rl_iter->channels_left_in_tsg--;
-                }
 #ifdef DETAILED_CHANNEL_INFO
                 runlist_detail_seq_show_chan(s, g, chid(g, entry), indt);
 #else
@@ -193,8 +195,7 @@ ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
                 return -ERANGE;
 
         // Execute preemption
-        err = preempt_tsg(g, target_tsgid);
-        if (err)
+        if ((err = preempt_tsg(g, target_tsgid)))
                 return err;
 
         return count;
@@ -210,8 +211,6 @@ ssize_t disable_channel_file_write(struct file *f, const char __user *buffer,
         uint32_t target_channel;
         channel_ctrl_t chan;
         int err;
-        runlist_info_t rl_info;
-        runlist_disable_t rl_disable;
         struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         err = kstrtou32_from_user(buffer, count, 0, &target_channel);
@@ -221,19 +220,12 @@ ssize_t disable_channel_file_write(struct file *f, const char __user *buffer,
         if (target_channel > MAX_CHID)
                 return -ERANGE;
 
-        // Disable channel
-        chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel));
+        // Read current configuration
+        if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1)
+                return -EIO;
+        // Request disablement
         chan.enable_clear = true;
-        // disable sched
-        rl_info.raw = nvdebug_readl(g, NV_PFIFO_RUNLIST);
-        rl_disable.raw = nvdebug_readl(g, NV_PFIFO_SCHED_DISABLE);
-        rl_disable.raw |= BIT(rl_info.id);
-        nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw);
-        // disable chan
         nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
-        // enable sched
-        rl_disable.raw &= ~BIT(rl_info.id);
-        nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw);
 
         return count;
 }
@@ -270,6 +262,7 @@ struct file_operations enable_channel_file_ops = {
         .llseek = default_llseek,
 };
 
+// Note: Operates only on runlist 0 (Compute/Graphics)
 ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer,
                                    size_t count, loff_t *off) {
         uint32_t target_tsgid;
@@ -292,7 +285,7 @@ ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer,
                 return err;
 
         // Iterate through all TSGs
-        while (pos < rl_iter.rl_info.len) {
+        while (pos < rl_iter.len) {
                 if (tsgid(g, rl_iter.curr_entry) == target_tsgid) {
                         // Enable channels of target TSG
                         for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
@@ -313,9 +306,8 @@ ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer,
                 pos += 1 + tsg_length(g, rl_iter.curr_entry);
                 rl_iter.curr_entry = next_tsg(g, rl_iter.curr_entry);
         }
-        // Switch to next TSG with active channels (should be our TSG)
-        err = preempt_tsg(g, target_tsgid);
-        if (err)
+        // Trigger a runlist-level preempt to switch to `target_tsgid`
+        if ((err = preempt_runlist(g, 0)))
                 return err;
 
         return count;