1 files changed, 111 insertions, 77 deletions

diff --git a/runlist_procfs.c b/runlist_procfs.c
index 411f844..a6b0d94 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -6,7 +6,14 @@
 #define RUNLIST_PROCFS_NAME "runlist"
 #define DETAILED_CHANNEL_INFO
 
-static int runlist_detail_seq_show_chan(struct seq_file *s, struct gk20a *g, uint32_t chid) {
+/* Print channel details using PCCSR (Programmable Channel Control System RAM?)
+ * @param s      Pointer to state from seq_file subsystem to pass to seq_printf
+ * @param g      Pointer to our internal GPU state
+ * @param chid   ID of channel to print details on, range [0, 512)
+ * @param prefix Text string to prefix each line with, or empty string
+ */
+#ifdef DETAILED_CHANNEL_INFO
+static int runlist_detail_seq_show_chan(struct seq_file *s, struct nvdebug_state *g, uint32_t chid, char *prefix) {
         channel_ctrl_t chan;
         char *loc_txt;
         u64 instance_ptr;
@@ -16,23 +23,37 @@ static int runlist_detail_seq_show_chan(struct seq_file *s, struct gk20a *g, uin
                 return -EIO;
         instance_ptr = chan.inst_ptr;
         instance_ptr <<= 12;
-        seq_printf(s, "  +- Channel Info %-4d -+\n", chid);
-        seq_printf(s, "  | Enabled:           %d|\n", chan.enable);
-        seq_printf(s, "  | Next:              %d|\n", chan.next);
-        seq_printf(s, "  | Force CTX Reload:  %d|\n", chan.force_ctx_reload);
-        seq_printf(s, "  | Enable set:        %d|\n", chan.enable_set);
-        seq_printf(s, "  | Enable clear:      %d|\n", chan.enable_clear);
-        seq_printf(s, "  | PBDMA Faulted:     %d|\n", chan.pbdma_faulted);
-        seq_printf(s, "  | ENG Faulted:       %d|\n", chan.eng_faulted);
-        seq_printf(s, "  | Status:           %2d|\n", chan.status);
-        seq_printf(s, "  | Busy:              %d|\n", chan.busy);
-        seq_printf(s, "  | Instance PTR:       |\n");
-        seq_printf(s, "  | %#018llx  |\n", instance_ptr);
-        seq_printf(s, "  | %-20s|\n", loc_txt);
-        seq_printf(s, "  | Instance bound:    %d|\n", chan.inst_bind);
-        seq_printf(s, "  +---------------------+\n");
+        seq_printf(s, "%s+- Channel Info %-4d -+\n", prefix, chid);
+        seq_printf(s, "%s| Enabled:           %d|\n", prefix, chan.enable);
+        seq_printf(s, "%s| Next:              %d|\n", prefix, chan.next);
+        seq_printf(s, "%s| Force CTX Reload:  %d|\n", prefix, chan.force_ctx_reload);
+        seq_printf(s, "%s| Enable set:        %d|\n", prefix, chan.enable_set);
+        seq_printf(s, "%s| Enable clear:      %d|\n", prefix, chan.enable_clear);
+        seq_printf(s, "%s| PBDMA Faulted:     %d|\n", prefix, chan.pbdma_faulted);
+        seq_printf(s, "%s| ENG Faulted:       %d|\n", prefix, chan.eng_faulted);
+        seq_printf(s, "%s| Status:           %2d|\n", prefix, chan.status);
+        seq_printf(s, "%s| Busy:              %d|\n", prefix, chan.busy);
+        seq_printf(s, "%s| Instance PTR:       |\n", prefix);
+        seq_printf(s, "%s| %#018llx  |\n", prefix, instance_ptr);
+        seq_printf(s, "%s| %-20s|\n", prefix, loc_txt);
+        seq_printf(s, "%s| Instance bound:    %d|\n", prefix, chan.inst_bind);
+        // START TEMP
+        // "runlist_id -1 is synonym for the ENGINE_GR_GK20A runlist id"
+        // GR, GRCE, and ASYNC_CE
+        // Note that this appears to be broken??
+        // Peek into the channel instance RAM
+        if (chan.inst_target == TARGET_SYS_MEM_COHERENT) {
+                seq_printf(s, "%s| Target Engine:    %2d|\n", prefix, *(uint32_t*)phys_to_virt(instance_ptr + 4/*bytes for 32bits*/*43/*NV_RAMFC_TARGET*/) & 0x1f);
+                seq_printf(s, "%s| PDB LO:   %#08x|\n", prefix, *(uint32_t*)phys_to_virt(instance_ptr + 4/*bytes for 32bits*/*128/*NV_RAMIN_PAGE_DIR_BASE_LO*/) & 0xfffff000);
+                seq_printf(s, "%s| Num subcontexts:  %2ld|\n", prefix, hweight64(*(uint64_t*)phys_to_virt(instance_ptr + 4/*bytes for 32bits*/*166/*NV_RAMIN_SC_PDB_VALID*/)));
+                // This appears to be unset on Xavier
+                //seq_printf(s, "%s| PAS ID:     %8ld|\n", prefix, *(uint32_t*)phys_to_virt(instance_ptr + 4/*bytes for 32bits*/*135/*NV_RAMIN_PASID*/) & 0xfffff);
+        }
+        // END TEMP
+        seq_printf(s, "%s+---------------------+\n", prefix);
         return 0;
 }
+#endif
 
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0)
 // Bug workaround. See comment in runlist_file_seq_start()
@@ -41,10 +62,14 @@ static loff_t pos_fixup;
 
 static void *runlist_file_seq_start(struct seq_file *s, loff_t *pos) {
         static struct runlist_iter rl_iter;
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(s->file)];
         // *pos == 0 for first call after read of file
         if (*pos == 0) {
-                int err = get_runlist_iter(&rl_iter);
+                int err = get_runlist_iter(g, seq2gpuidx(s), &rl_iter);
                 if (err)
+                        return ERR_PTR(err);
+                // Don't try to print an empty runlist
+                if (rl_iter.rl_info.len <= 0)
                         return NULL;
                 return &rl_iter;
         }
@@ -68,12 +93,13 @@ static void* runlist_file_seq_next(struct seq_file *s, void *raw_rl_iter,
                                    loff_t *pos) {
         struct runlist_iter* rl_iter = raw_rl_iter;
         void *ret = NULL;
-        // Advance by one TSG + channels under last TSG
-        *pos += 1 + rl_iter->curr_tsg->tsg_length;
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(s->file)];
+        // Advance by one TSG or channel
+        (*pos)++;
+        rl_iter->curr_entry += NV_RL_ENTRY_SIZE(g);
         // Verify we haven't reached the end of the runlist
         // rl_info.len is the num of tsg entries + total num of channel entries
         if (*pos < rl_iter->rl_info.len) {
-                rl_iter->curr_tsg = next_tsg(rl_iter->curr_tsg);
                 ret = rl_iter;
         }
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0)
@@ -88,57 +114,57 @@ static void runlist_file_seq_stop(struct seq_file *s, void *raw_rl_iter) {
 }
 
 static int runlist_file_seq_show(struct seq_file *s, void *raw_rl_iter) {
-        struct entry_tsg* tsg = ((struct runlist_iter*)raw_rl_iter)->curr_tsg;
-        struct runlist_chan* chan;
-        struct gk20a *g = get_live_gk20a();
-        if (!g)
-                return -EIO;
-        if (tsg->entry_type != ENTRY_TYPE_TSG) {
-                printk(KERN_WARNING "[nvdebug] Attempted to print non-TSG in tsg print logic!\n");
-                return -EIO;
-        }
-        seq_printf(s, "+---- TSG Entry %-2d----+\n", tsg->tsgid);
-        seq_printf(s, "| Scale: %-13d|\n", tsg->timeslice_scale);
-        seq_printf(s, "| Timeout: %-11d|\n", tsg->timeslice_timeout);
-        seq_printf(s, "+---------------------+\n");
-        for_chan_in_tsg(chan, tsg) {
+        struct runlist_iter *rl_iter = raw_rl_iter;
+        void *entry = rl_iter->curr_entry;
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(s->file)];
+        if (entry_type(g, entry) == ENTRY_TYPE_TSG) {
+                if (rl_iter->channels_left_in_tsg) {
+                        printk(KERN_WARNING "[nvdebug] Found a TSG @ %px when %d channels were still expected under the previous TSG in the runlist!\n", entry, rl_iter->channels_left_in_tsg);
+                        return -EIO;
+                }
+                rl_iter->channels_left_in_tsg = tsg_length(g, entry);
+                seq_printf(s, "+---- TSG Entry %-3d---+\n", tsgid(g, entry));
+                seq_printf(s, "| Scale: %-13d|\n", timeslice_scale(g, entry));
+                seq_printf(s, "| Timeout: %-11d|\n", timeslice_timeout(g, entry));
+                seq_printf(s, "| Length: %-12d|\n", tsg_length(g, entry));
+                seq_printf(s, "+---------------------+\n");
+        } else {
+                char *indt = "";
 #ifndef DETAILED_CHANNEL_INFO
-                char* loc_txt;
-                u64 instance_ptr;
+                u64 instance_ptr = 0;
 #endif
-                if (chan->entry_type != ENTRY_TYPE_CHAN) {
-                        printk(KERN_WARNING "[nvdebug] Attempted to print non-channel in channel print logic!\n");
-                        return -EIO;
+                if (rl_iter->channels_left_in_tsg) {
+                        indt = "  ";
+                        rl_iter->channels_left_in_tsg--;
                 }
 #ifdef DETAILED_CHANNEL_INFO
-                runlist_detail_seq_show_chan(s, g, chan->chid);
+                runlist_detail_seq_show_chan(s, g, chid(g, entry), indt);
 #else
-                loc_txt = target_to_text(chan->inst_target);
-                if (!loc_txt) {
-                        printk(KERN_WARNING "[nvdebug] Invalid apature in channel print logic!\n");
-                        return -EIO;
-                }
                 // Reconstruct pointer to channel instance block
-                instance_ptr = chan->inst_ptr_hi;
-                instance_ptr <<= 32;
-                instance_ptr |= chan->inst_ptr_lo << 12;
-
-                seq_printf(s, "  +- Channel Entry %-4d-+\n", chan->chid);
-                seq_printf(s, "  | Runqueue Selector: %d|\n", chan->runqueue_selector);
-                seq_printf(s, "  | Instance PTR:       |\n");
-                seq_printf(s, "  | %#018llx  |\n", instance_ptr);
-                seq_printf(s, "  | %-20s|\n", loc_txt);
-                seq_printf(s, "  +---------------------+\n");
+                if (g->chip_id >= NV_CHIP_ID_VOLTA) {
+                        instance_ptr = ((struct gv100_runlist_chan*)entry)->inst_ptr_hi;
+                        instance_ptr <<= 32;
+                }
+                instance_ptr |= inst_ptr_lo(g, entry) << 12;
+
+                seq_printf(s, "%s+- Channel Entry %-4d-+\n", indt, chid(g, entry));
+                if (g->chip_id >= NV_CHIP_ID_VOLTA)
+                        seq_printf(s, "%s| Runqueue Selector: %d|\n", indt,
+                                   ((struct gv100_runlist_chan*)entry)->runqueue_selector);
+                seq_printf(s, "%s| Instance PTR:       |\n", indt);
+                seq_printf(s, "%s| %#018llx  |\n", indt, instance_ptr);
+                seq_printf(s, "%s| %-20s|\n", indt, target_to_text(inst_target(g, entry)));
+                seq_printf(s, "%s+---------------------+\n", indt);
 #endif
         }
         return 0;
 }
 
 static const struct seq_operations runlist_file_seq_ops = {
-        .start = runlist_file_seq_start,
-        .next = runlist_file_seq_next,
-        .stop = runlist_file_seq_stop,
-        .show = runlist_file_seq_show,
+        .start = runlist_file_seq_start,
+        .next = runlist_file_seq_next,
+        .stop = runlist_file_seq_stop,
+        .show = runlist_file_seq_show,
 };
 
 static int runlist_file_open(struct inode *inode, struct file *f) {
@@ -157,6 +183,7 @@ ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
         uint32_t target_tsgid;
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
+        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         if (err)
                 return err;
 
@@ -165,7 +192,7 @@ ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
                 return -ERANGE;
 
         // Execute preemption
-        err = preempt_tsg(target_tsgid);
+        err = preempt_tsg(g, target_tsgid);
         if (err)
                 return err;
 
@@ -181,9 +208,9 @@ ssize_t disable_channel_file_write(struct file *f, const char __user *buffer,
         uint32_t target_channel;
         channel_ctrl_t chan;
         int err;
-        struct gk20a *g = get_live_gk20a();
-        if (!g)
-                return -EIO;
+        runlist_info_t rl_info;
+        runlist_disable_t rl_disable;
+        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
@@ -195,7 +222,16 @@ ssize_t disable_channel_file_write(struct file *f, const char __user *buffer,
         // Disable channel
         chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel));
         chan.enable_clear = true;
+        // disable sched
+        rl_info.raw = nvdebug_readl(g, NV_PFIFO_RUNLIST);
+        rl_disable.raw = nvdebug_readl(g, NV_PFIFO_SCHED_DISABLE);
+        rl_disable.raw |= BIT(rl_info.id);
+        nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw);
+        // disable chan
         nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+        // enable sched
+        rl_disable.raw &= ~BIT(rl_info.id);
+        nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw);
 
         return count;
 }
@@ -209,9 +245,7 @@ ssize_t enable_channel_file_write(struct file *f, const char __user *buffer,
         uint32_t target_channel;
         channel_ctrl_t chan;
         int err;
-        struct gk20a *g = get_live_gk20a();
-        if (!g)
-                return -EIO;
+        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         err = kstrtou32_from_user(buffer, count, 0, &target_channel);
         if (err)
@@ -235,14 +269,12 @@ const struct file_operations enable_channel_file_ops = {
 ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer,
                                    size_t count, loff_t *off) {
         uint32_t target_tsgid;
-        struct runlist_chan* chan;
+        struct gv100_runlist_chan* chan;
         channel_ctrl_t chan_ctl;
         struct runlist_iter rl_iter;
         int err;
         loff_t pos = 0;
-        struct gk20a *g = get_live_gk20a();
-        if (!g)
-                return -EIO;
+        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
         err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
         if (err)
@@ -251,32 +283,34 @@ ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer,
         if (target_tsgid > MAX_TSGID)
                 return -ERANGE;
 
-        err = get_runlist_iter(&rl_iter);
+        err = get_runlist_iter(g, 0, &rl_iter);
         if (err)
                 return err;
 
         // Iterate through all TSGs
         while (pos < rl_iter.rl_info.len) {
-                if (rl_iter.curr_tsg->tsgid == target_tsgid) {
+                if (tsgid(g, rl_iter.curr_entry) == target_tsgid) {
                         // Enable channels of target TSG
-                        for_chan_in_tsg(chan, rl_iter.curr_tsg) {
+                        for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
                                 chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
                                 chan_ctl.enable_set = true;
                                 nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
                         }
                 } else {
+                        // XXX: Fix for bare channels. Maybe a "for_chan_until_tsg" macro?
                         // Disable all other channels
-                        for_chan_in_tsg(chan, rl_iter.curr_tsg) {
+                        // (This is how the Jetson nvgpu driver disables TSGs)
+                        for_chan_in_tsg(g, chan, rl_iter.curr_entry) {
                                 chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
                                 chan_ctl.enable_clear = true;
                                 nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
                         }
                 }
-                pos += 1 + rl_iter.curr_tsg->tsg_length;
-                rl_iter.curr_tsg = next_tsg(rl_iter.curr_tsg);
+                pos += 1 + tsg_length(g, rl_iter.curr_entry);
+                rl_iter.curr_entry = next_tsg(g, rl_iter.curr_entry);
         }
         // Switch to next TSG with active channels (should be our TSG)
-        err = preempt_tsg(target_tsgid);
+        err = preempt_tsg(g, target_tsgid);
         if (err)
                 return err;