3 files changed, 179 insertions, 36 deletions

diff --git a/nvdebug.h b/nvdebug.h
index b4ff0a4..cd0dc90 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -12,7 +12,20 @@
 
   `INST_PTR` points to a GPU Instance Block which contains pointers to the GPU
   virtual address space for this context. All channels in a TSG point to the
-  same GPU Instance Block.
+  same GPU Instance Block (?).
+
+  "RUNQUEUE_SELECTOR determines to which runqueue the channel belongs, and
+  thereby which PBDMA will run the channel.  Increasing values select
+  increasingly numbered PBDMA IDs serving the runlist.  If the selector value
+  exceeds the number of PBDMAs on the runlist, the hardware will silently
+  reassign the channel to run on the first PBDMA as though RUNQUEUE_SELECTOR had
+  been set to 0.  (In current hardware, this is used by SCG on the graphics
+  runlist only to determine which FE pipe should service a given channel.  A
+  value of 0 targets the first FE pipe, which can process all FE driven engines:
+  Graphics, Compute, Inline2Memory, and TwoD.  A value of 1 targets the second
+  FE pipe, which can only process Compute work.  Note that GRCE work is allowed
+  on either runqueue." (NVIDIA) Note that it appears runqueue 1 is the default
+  for CUDA work on the Jetson Xavier.
 
   ENTRY_TYPE (T)        : type of this entry: ENTRY_TYPE_CHAN
   CHID (ID)             : identifier of the channel to run (overlays ENTRY_ID)
@@ -29,6 +42,19 @@
 */
 enum ENTRY_TYPE {ENTRY_TYPE_CHAN = 0, ENTRY_TYPE_TSG = 1};
 enum INST_TARGET {TARGET_VID_MEM = 0, TARGET_SYS_MEM_COHERENT = 2, TARGET_SYS_MEM_NONCOHERENT = 3};
+static inline char* target_to_text(enum INST_TARGET t) {
+        switch (t) {
+                case TARGET_VID_MEM:
+                        return "VID_MEM";
+                case TARGET_SYS_MEM_COHERENT:
+                        return "SYS_MEM_COHERENT";
+                case TARGET_SYS_MEM_NONCOHERENT:
+                        return "SYS_MEM_NONCOHERENT";
+                default:
+                        printk(KERN_WARNING "[nvdebug] Invalid aperture!\n");
+                        return NULL;
+        }
+}
 
 struct runlist_chan {
 // 0:63
@@ -55,10 +81,10 @@ struct runlist_chan {
   timeslice = (TSG_TIMESLICE_TIMEOUT << TSG_TIMESLICE_SCALE) * 1024 nanoseconds
 
   ENTRY_TYPE (T)      : type of this entry: ENTRY_TYPE_TSG
-  TSGID               : identifier of the Timeslice group (overlays ENTRY_ID)
-  TSG_LENGTH          : number of channels that are part of this timeslice group
   TIMESLICE_SCALE     : scale factor for the TSG's timeslice
   TIMESLICE_TIMEOUT   : timeout amount for the TSG's timeslice
+  TSG_LENGTH          : number of channels that are part of this timeslice group
+  TSGID               : identifier of the Timeslice group (overlays ENTRY_ID)
 */
 struct entry_tsg {
 // 0:63
@@ -130,6 +156,52 @@ typedef union {
         uint32_t raw;
 } runlist_info_t;
 
+enum CHANNEL_STATUS {
+        CHANNEL_STATUS_IDLE = 0,
+        CHANNEL_STATUS_PENDING = 1,
+        CHANNEL_STATUS_PENDING_CTX_RELOAD = 2,
+        CHANNEL_STATUS_PENDING_ACQUIRE = 3,
+        CHANNEL_STATUS_PENDING_ACQ_CTX_RELOAD = 4,
+        CHANNEL_STATUS_ON_PBDMA = 5,
+        CHANNEL_STATUS_ON_PBDMA_AND_ENG = 6,
+        CHANNEL_STATUS_ON_ENG = 7,
+        CHANNEL_STATUS_ON_ENG_PENDING_ACQUIRE = 8,
+        CHANNEL_STATUS_ON_ENG_PENDING = 9,
+        CHANNEL_STATUS_ON_PBDMA_CTX_RELOAD = 10,
+        CHANNEL_STATUS_ON_PBDMA_AND_ENG_CTX_RELOAD = 11,
+        CHANNEL_STATUS_ON_ENG_CTX_RELOAD = 12,
+        CHANNEL_STATUS_ON_ENG_PENDING_CTX_RELOAD = 13,
+        CHANNEL_STATUS_ON_ENG_PENDING_ACQ_CTX_RELOAD = 14,
+};
+
+#define NV_PCCSR_CHANNEL_INST(i) (0x00800000+(i)*8)
+#define MAX_CHID 512  // TODO: Double-check this is right
+// There are a total of 512 possible channels
+typedef union {
+        struct {
+// 0:31
+                uint32_t inst_ptr:28;
+                enum INST_TARGET inst_target:2;
+                 uint32_t padding0:1;
+                bool inst_bind:1;
+// 32:64
+                bool enable:1;
+                bool next:1;
+                 uint32_t padding:6;
+                bool force_ctx_reload:1;
+                 uint32_t padding2:1;
+                bool enable_set:1;
+                bool enable_clear:1;
+                 uint32_t padding3:10;
+                bool pbdma_faulted:1;
+                bool eng_faulted:1;
+                enum CHANNEL_STATUS status:4;
+                bool busy:1;
+                 uint32_t padding4:3;
+        } __attribute__((packed));
+        uint64_t raw;
+} channel_ctrl_t;
+
 // TODO(jbakita): Maybe put the above GPU types in a different file.
 
 #define for_chan_in_tsg(chan, tsg) \
@@ -146,6 +218,7 @@ struct runlist_iter {
 };
 
 // Defined in runlist.c
+struct gk20a* get_live_gk20a(void);
 int get_runlist_iter(struct runlist_iter *rl_iter);
 
 static inline struct gk20a *get_gk20a(struct device *dev) {
@@ -164,6 +237,20 @@ static inline u32 nvdebug_readl(struct gk20a* g, u32 r) {
         return readl(g_os->regs + r);
 }
 
+// quadword version of nvdebug_readl()
+static inline u64 nvdebug_readq(struct gk20a* g, u32 r) {
+        struct nvgpu_os_linux* g_os = container_of(g, struct nvgpu_os_linux, g);
+        u64 ret;
+        if (unlikely(!g_os->regs)) {
+                printk(KERN_ERR "[nvdebug] Attempted nvgpu_readl on non-existent registers!\n");
+                return -1;
+        }
+        // readq seems to always return the uppermost 32 bits as 0, so workaround with readl
+        ret = readl(g_os->regs + r);
+        ret |= ((u64)readl(g_os->regs + r + 4)) << 32;
+        return ret;
+}
+
 // Functionally identical to nvgpu_writel()
 static inline void nvdebug_writel(struct gk20a* g, u32 r, u32 v) {
         struct nvgpu_os_linux* g_os = container_of(g, struct nvgpu_os_linux, g);
diff --git a/runlist.c b/runlist.c
index 8691b51..8dfa1c7 100644
--- a/runlist.c
+++ b/runlist.c
@@ -8,23 +8,20 @@
 // Bus types are global symbols in the kernel
 extern struct bus_type platform_bus_type;
 
-int get_runlist_iter(struct runlist_iter *rl_iter) {
+struct gk20a* get_live_gk20a(void) {
         struct device *dev = NULL;
         struct device *temp_dev;
         struct gk20a *g;
-        struct entry_tsg head;
-        runlist_base_t rl_base;
-        runlist_info_t rl_info;
-        u64 runlist_iova;
+        struct nvgpu_os_linux *l;
         // Get the last device that matches our name
         while ((temp_dev = bus_find_device_by_name(&platform_bus_type, dev, "17000000.gv11b"))) {
                 dev = temp_dev;
                 printk(KERN_INFO "[nvdebug] Found a matching device %s\n", dev_name(dev));
         }
         if (!dev)
-                return -EIO;
+                return NULL;
         g = get_gk20a(dev);
-        // This address seems to not be:
+        // The address pointed to `regs` + NV_PFIFO_RUNLIST_BASE seems to not be:
         // - A GPU address (type is sysmem_coherent)
         // - A physical address (dereferencing after ioremap crashes)
         // - A kernel virtual address (dereferencing segfaults)
@@ -49,15 +46,23 @@ int get_runlist_iter(struct runlist_iter *rl_iter) {
         // BUT, it turns out that it IS JUST A PHYSICAL ADDRESS! It wasn't working
         // before because the GPU had simply gone to sleep and invalidated its
         // register state, so nvgpu_readl() was simply returning garbage.
-
-        printk(KERN_INFO "[nvdebug] Pulling runlist base address from %x\n", NV_PFIFO_RUNLIST_BASE);
-        printk(KERN_INFO "[nvdebug] Using struct gk20a* of %px\n", g);
-        printk(KERN_INFO "[nvdebug] g->name: %s, g->power_on: %d, g->sw_ready: %d, g->is_virtual %d\n",
-                g->name, g->power_on, g->sw_ready, g->is_virtual);
-        struct nvgpu_os_linux *l = container_of(g, struct nvgpu_os_linux, g);
-        printk(KERN_INFO "[nvdebug] l->regs %px, l->regs_saved %px\n", l->regs, l->regs_saved);
+        l = container_of(g, struct nvgpu_os_linux, g);
         if (!l->regs)
-                return -EIO;
+                return NULL;
+        return g;
+}
+
+/* Get runlist head and info (incl. length)
+   @param rl_iter Location at which to store output
+*/
+int get_runlist_iter(struct runlist_iter *rl_iter) {
+        struct entry_tsg head;
+        runlist_base_t rl_base;
+        runlist_info_t rl_info;
+        u64 runlist_iova;
+        struct gk20a *g = get_live_gk20a();
+        if (!g)
+                return -EIO;
         rl_base.raw = nvdebug_readl(g, NV_PFIFO_RUNLIST_BASE);
         rl_info.raw = nvdebug_readl(g, NV_PFIFO_RUNLIST);
         runlist_iova = ((u64)rl_base.ptr) << 12;
diff --git a/runlist_procfs.c b/runlist_procfs.c
index 2107bd4..183eab6 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -1,12 +1,46 @@
 #include <linux/seq_file.h> // For seq_* functions and types
+#include <linux/version.h>  // Macros to detect kernel version