3 files changed, 179 insertions, 36 deletions

diff --git a/nvdebug.h b/nvdebug.h
index b4ff0a4..cd0dc90 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -12,7 +12,20 @@
 
   `INST_PTR` points to a GPU Instance Block which contains pointers to the GPU
   virtual address space for this context. All channels in a TSG point to the
-  same GPU Instance Block.
+  same GPU Instance Block (?).
+
+  "RUNQUEUE_SELECTOR determines to which runqueue the channel belongs, and
+  thereby which PBDMA will run the channel.  Increasing values select
+  increasingly numbered PBDMA IDs serving the runlist.  If the selector value
+  exceeds the number of PBDMAs on the runlist, the hardware will silently
+  reassign the channel to run on the first PBDMA as though RUNQUEUE_SELECTOR had
+  been set to 0.  (In current hardware, this is used by SCG on the graphics
+  runlist only to determine which FE pipe should service a given channel.  A
+  value of 0 targets the first FE pipe, which can process all FE driven engines:
+  Graphics, Compute, Inline2Memory, and TwoD.  A value of 1 targets the second
+  FE pipe, which can only process Compute work.  Note that GRCE work is allowed
+  on either runqueue." (NVIDIA) Note that it appears runqueue 1 is the default
+  for CUDA work on the Jetson Xavier.
 
   ENTRY_TYPE (T)        : type of this entry: ENTRY_TYPE_CHAN
   CHID (ID)             : identifier of the channel to run (overlays ENTRY_ID)
@@ -29,6 +42,19 @@
 */
 enum ENTRY_TYPE {ENTRY_TYPE_CHAN = 0, ENTRY_TYPE_TSG = 1};
 enum INST_TARGET {TARGET_VID_MEM = 0, TARGET_SYS_MEM_COHERENT = 2, TARGET_SYS_MEM_NONCOHERENT = 3};
+static inline char* target_to_text(enum INST_TARGET t) {
+        switch (t) {
+                case TARGET_VID_MEM:
+                        return "VID_MEM";
+                case TARGET_SYS_MEM_COHERENT:
+                        return "SYS_MEM_COHERENT";
+                case TARGET_SYS_MEM_NONCOHERENT:
+                        return "SYS_MEM_NONCOHERENT";
+                default:
+                        printk(KERN_WARNING "[nvdebug] Invalid aperture!\n");
+                        return NULL;
+        }
+}
 
 struct runlist_chan {
 // 0:63
@@ -55,10 +81,10 @@ struct runlist_chan {
   timeslice = (TSG_TIMESLICE_TIMEOUT << TSG_TIMESLICE_SCALE) * 1024 nanoseconds
 
   ENTRY_TYPE (T)      : type of this entry: ENTRY_TYPE_TSG
-  TSGID               : identifier of the Timeslice group (overlays ENTRY_ID)
-  TSG_LENGTH          : number of channels that are part of this timeslice group
   TIMESLICE_SCALE     : scale factor for the TSG's timeslice
   TIMESLICE_TIMEOUT   : timeout amount for the TSG's timeslice
+  TSG_LENGTH          : number of channels that are part of this timeslice group
+  TSGID               : identifier of the Timeslice group (overlays ENTRY_ID)
 */
 struct entry_tsg {
 // 0:63
@@ -130,6 +156,52 @@ typedef union {
         uint32_t raw;
 } runlist_info_t;
 
+enum CHANNEL_STATUS {
+        CHANNEL_STATUS_IDLE = 0,
+        CHANNEL_STATUS_PENDING = 1,
+        CHANNEL_STATUS_PENDING_CTX_RELOAD = 2,
+        CHANNEL_STATUS_PENDING_ACQUIRE = 3,
+        CHANNEL_STATUS_PENDING_ACQ_CTX_RELOAD = 4,
+        CHANNEL_STATUS_ON_PBDMA = 5,
+        CHANNEL_STATUS_ON_PBDMA_AND_ENG = 6,
+        CHANNEL_STATUS_ON_ENG = 7,
+        CHANNEL_STATUS_ON_ENG_PENDING_ACQUIRE = 8,
+        CHANNEL_STATUS_ON_ENG_PENDING = 9,
+        CHANNEL_STATUS_ON_PBDMA_CTX_RELOAD = 10,
+        CHANNEL_STATUS_ON_PBDMA_AND_ENG_CTX_RELOAD = 11,
+        CHANNEL_STATUS_ON_ENG_CTX_RELOAD = 12,
+        CHANNEL_STATUS_ON_ENG_PENDING_CTX_RELOAD = 13,
+        CHANNEL_STATUS_ON_ENG_PENDING_ACQ_CTX_RELOAD = 14,
+};
+
+#define NV_PCCSR_CHANNEL_INST(i) (0x00800000+(i)*8)
+#define MAX_CHID 512  // TODO: Double-check this is right
+// There are a total of 512 possible channels
+typedef union {
+        struct {
+// 0:31
+                uint32_t inst_ptr:28;
+                enum INST_TARGET inst_target:2;
+                 uint32_t padding0:1;
+                bool inst_bind:1;
+// 32:64
+                bool enable:1;
+                bool next:1;
+                 uint32_t padding:6;
+                bool force_ctx_reload:1;
+                 uint32_t padding2:1;
+                bool enable_set:1;
+                bool enable_clear:1;
+                 uint32_t padding3:10;
+                bool pbdma_faulted:1;
+                bool eng_faulted:1;
+                enum CHANNEL_STATUS status:4;
+                bool busy:1;
+                 uint32_t padding4:3;
+        } __attribute__((packed));
+        uint64_t raw;
+} channel_ctrl_t;
+
 // TODO(jbakita): Maybe put the above GPU types in a different file.
 
 #define for_chan_in_tsg(chan, tsg) \
@@ -146,6 +218,7 @@ struct runlist_iter {
 };
 
 // Defined in runlist.c
+struct gk20a* get_live_gk20a(void);
 int get_runlist_iter(struct runlist_iter *rl_iter);
 
 static inline struct gk20a *get_gk20a(struct device *dev) {
@@ -164,6 +237,20 @@ static inline u32 nvdebug_readl(struct gk20a* g, u32 r) {
         return readl(g_os->regs + r);
 }
 
+// quadword version of nvdebug_readl()
+static inline u64 nvdebug_readq(struct gk20a* g, u32 r) {
+        struct nvgpu_os_linux* g_os = container_of(g, struct nvgpu_os_linux, g);
+        u64 ret;
+        if (unlikely(!g_os->regs)) {
+                printk(KERN_ERR "[nvdebug] Attempted nvgpu_readl on non-existent registers!\n");
+                return -1;
+        }
+        // readq seems to always return the uppermost 32 bits as 0, so workaround with readl
+        ret = readl(g_os->regs + r);
+        ret |= ((u64)readl(g_os->regs + r + 4)) << 32;
+        return ret;
+}
+
 // Functionally identical to nvgpu_writel()
 static inline void nvdebug_writel(struct gk20a* g, u32 r, u32 v) {
         struct nvgpu_os_linux* g_os = container_of(g, struct nvgpu_os_linux, g);
diff --git a/runlist.c b/runlist.c
index 8691b51..8dfa1c7 100644
--- a/runlist.c
+++ b/runlist.c
@@ -8,23 +8,20 @@
 // Bus types are global symbols in the kernel
 extern struct bus_type platform_bus_type;
 
-int get_runlist_iter(struct runlist_iter *rl_iter) {
+struct gk20a* get_live_gk20a(void) {
         struct device *dev = NULL;
         struct device *temp_dev;
         struct gk20a *g;
-        struct entry_tsg head;
-        runlist_base_t rl_base;
-        runlist_info_t rl_info;
-        u64 runlist_iova;
+        struct nvgpu_os_linux *l;
         // Get the last device that matches our name
         while ((temp_dev = bus_find_device_by_name(&platform_bus_type, dev, "17000000.gv11b"))) {
                 dev = temp_dev;
                 printk(KERN_INFO "[nvdebug] Found a matching device %s\n", dev_name(dev));
         }
         if (!dev)
-                return -EIO;
+                return NULL;
         g = get_gk20a(dev);
-        // This address seems to not be:
+        // The address pointed to `regs` + NV_PFIFO_RUNLIST_BASE seems to not be:
         // - A GPU address (type is sysmem_coherent)
         // - A physical address (dereferencing after ioremap crashes)
         // - A kernel virtual address (dereferencing segfaults)
@@ -49,15 +46,23 @@ int get_runlist_iter(struct runlist_iter *rl_iter) {
         // BUT, it turns out that it IS JUST A PHYSICAL ADDRESS! It wasn't working
         // before because the GPU had simply gone to sleep and invalidated its
         // register state, so nvgpu_readl() was simply returning garbage.
-
-        printk(KERN_INFO "[nvdebug] Pulling runlist base address from %x\n", NV_PFIFO_RUNLIST_BASE);
-        printk(KERN_INFO "[nvdebug] Using struct gk20a* of %px\n", g);
-        printk(KERN_INFO "[nvdebug] g->name: %s, g->power_on: %d, g->sw_ready: %d, g->is_virtual %d\n",
-                g->name, g->power_on, g->sw_ready, g->is_virtual);
-        struct nvgpu_os_linux *l = container_of(g, struct nvgpu_os_linux, g);
-        printk(KERN_INFO "[nvdebug] l->regs %px, l->regs_saved %px\n", l->regs, l->regs_saved);
+        l = container_of(g, struct nvgpu_os_linux, g);
         if (!l->regs)
-                return -EIO;
+                return NULL;
+        return g;
+}
+
+/* Get runlist head and info (incl. length)
+   @param rl_iter Location at which to store output
+*/
+int get_runlist_iter(struct runlist_iter *rl_iter) {
+        struct entry_tsg head;
+        runlist_base_t rl_base;
+        runlist_info_t rl_info;
+        u64 runlist_iova;
+        struct gk20a *g = get_live_gk20a();
+        if (!g)
+                return -EIO;
         rl_base.raw = nvdebug_readl(g, NV_PFIFO_RUNLIST_BASE);
         rl_info.raw = nvdebug_readl(g, NV_PFIFO_RUNLIST);
         runlist_iova = ((u64)rl_base.ptr) << 12;
diff --git a/runlist_procfs.c b/runlist_procfs.c
index 2107bd4..183eab6 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -1,12 +1,46 @@
 #include <linux/seq_file.h> // For seq_* functions and types
+#include <linux/version.h>  // Macros to detect kernel version
 
 #include "nvdebug.h"
 
 #define RUNLIST_PROCFS_NAME "runlist"
+#define DETAILED_CHANNEL_INFO
+
+static int runlist_detail_seq_show_chan(struct seq_file *s, struct gk20a *g, uint32_t chid) {
+        channel_ctrl_t chan;
+        char *loc_txt;
+        u64 instance_ptr;
+        chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chid));
+        loc_txt = target_to_text(chan.inst_target);
+        if (!loc_txt)
+                return -EIO;
+        instance_ptr = chan.inst_ptr;
+        instance_ptr <<= 12;
+        seq_printf(s, "  +- Channel Info %-4d -+\n", chid);
+        seq_printf(s, "  | Enabled:           %d|\n", chan.enable);
+        seq_printf(s, "  | Next:              %d|\n", chan.next);
+        seq_printf(s, "  | Force CTX Reload:  %d|\n", chan.force_ctx_reload);
+        seq_printf(s, "  | Enable set:        %d|\n", chan.enable_set);
+        seq_printf(s, "  | Enable clear:      %d|\n", chan.enable_clear);
+        seq_printf(s, "  | PBDMA Faulted:     %d|\n", chan.pbdma_faulted);
+        seq_printf(s, "  | ENG Faulted:       %d|\n", chan.eng_faulted);
+        seq_printf(s, "  | Status:           %2d|\n", chan.status);
+        seq_printf(s, "  | Busy:              %d|\n", chan.busy);
+        seq_printf(s, "  | Instance PTR:       |\n");
+        seq_printf(s, "  | %#018llx  |\n", instance_ptr);
+        seq_printf(s, "  | %-20s|\n", loc_txt);
+        seq_printf(s, "  | Instance bound:    %d|\n", chan.inst_bind);
+        seq_printf(s, "  +---------------------+\n");
+        return 0;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0)
+// Bug workaround. See comment in runlist_file_seq_start()
+static loff_t pos_fixup;
+#endif
 
 static void *runlist_file_seq_start(struct seq_file *s, loff_t *pos) {
         static struct runlist_iter rl_iter;
-
         // *pos == 0 for first call after read of file
         if (*pos == 0) {
                 int err = get_runlist_iter(&rl_iter);
@@ -14,6 +48,18 @@ static void *runlist_file_seq_start(struct seq_file *s, loff_t *pos) {
                         return NULL;
                 return &rl_iter;
         }
+        // If we're resuming an earlier print
+        if (*pos < rl_iter.rl_info.len) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0)
+                // There's a nasty bug prior to 4.19-rc1 that if the buffer overflows, the
+                // last update to `pos` is not saved. Work around that here by reloading a
+                // saved copy of `pos`.
+                if (!pos_fixup)
+                        return NULL;
+                *pos = pos_fixup;
+#endif
+                return &rl_iter;
+        }
         // When called with *pos != 0, we already traversed the runlist
         return NULL;
 }
@@ -21,15 +67,20 @@ static void *runlist_file_seq_start(struct seq_file *s, loff_t *pos) {
 static void* runlist_file_seq_next(struct seq_file *s, void *raw_rl_iter,
                                    loff_t *pos) {
         struct runlist_iter* rl_iter = raw_rl_iter;
+        void *ret = NULL;
         // Advance by one TSG + channels under last TSG
         *pos += 1 + rl_iter->curr_tsg->tsg_length;
         // Verify we haven't reached the end of the runlist
         // rl_info.len is the num of tsg entries + total num of channel entries
         if (*pos < rl_iter->rl_info.len) {
                 rl_iter->curr_tsg = next_tsg(rl_iter->curr_tsg);
-                return rl_iter;
+                ret = rl_iter;
         }
-        return NULL;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0)
+        // Bug workaround. See comment in runlist_file_seq_start()
+        pos_fixup = ret ? *pos : 0;
+#endif
+        return ret;
 }
 
 static void runlist_file_seq_stop(struct seq_file *s, void *raw_rl_iter) {
@@ -39,8 +90,11 @@ static void runlist_file_seq_stop(struct seq_file *s, void *raw_rl_iter) {
 static int runlist_file_seq_show(struct seq_file *s, void *raw_rl_iter) {
         struct entry_tsg* tsg = ((struct runlist_iter*)raw_rl_iter)->curr_tsg;
         struct runlist_chan* chan;
+        struct gk20a *g = get_live_gk20a();
+        if (!g)
+                return -EIO;
         if (tsg->entry_type != ENTRY_TYPE_TSG) {
-                printk(KERN_WARNING "[nvdebug] Attempted to print non-TSG in nvdebug_print_tsg()!\n");
+                printk(KERN_WARNING "[nvdebug] Attempted to print non-TSG in tsg print logic!\n");
                 return -EIO;
         }
         seq_printf(s, "+---- TSG Entry %-2d----+\n", tsg->tsgid);
@@ -48,25 +102,21 @@ static int runlist_file_seq_show(struct seq_file *s, void *raw_rl_iter) {
         seq_printf(s, "| Timeout: %-11d|\n", tsg->timeslice_timeout);
         seq_printf(s, "+---------------------+\n");
         for_chan_in_tsg(chan, tsg) {
+#ifndef DETAILED_CHANNEL_INFO
                 char* loc_txt;
                 u64 instance_ptr;
+#endif
                 if (chan->entry_type != ENTRY_TYPE_CHAN) {
-                        printk(KERN_WARNING "[nvdebug] Attempted to print non-channel in nvdebug_print_channel()!\n");
+                        printk(KERN_WARNING "[nvdebug] Attempted to print non-channel in channel print logic!\n");
                         return -EIO;
                 }
-                switch (chan->inst_target) {
-                        case TARGET_VID_MEM:
-                                loc_txt = "VID_MEM";
-                                break;
-                        case TARGET_SYS_MEM_COHERENT:
-                                loc_txt = "SYS_MEM_COHERENT";
-                                break;
-                        case TARGET_SYS_MEM_NONCOHERENT:
-                                loc_txt = "SYS_MEM_NONCOHERENT";
-                                break;
-                        default:
-                                printk(KERN_WARNING "[nvdebug] Invalid aperture in runlist channel!\n");
-                                return -EIO;
+#ifdef DETAILED_CHANNEL_INFO
+                runlist_detail_seq_show_chan(s, g, chan->chid);
+#else
+                loc_txt = target_to_text(chan->inst_target);
+                if (!loc_txt) {
+                        printk(KERN_WARNING "[nvdebug] Invalid apature in channel print logic!\n");
+                        return -EIO;
                 }
                 // Reconstruct pointer to channel instance block
                 instance_ptr = chan->inst_ptr_hi;
@@ -79,6 +129,7 @@ static int runlist_file_seq_show(struct seq_file *s, void *raw_rl_iter) {
                 seq_printf(s, "  | %#018llx  |\n", instance_ptr);
                 seq_printf(s, "  | %-20s|\n", loc_txt);
                 seq_printf(s, "  +---------------------+\n");
+#endif
         }
         return 0;
 }