Add /proc/gpu#/resubmit_runlist API

Resubmits the runlist in an identical configuration. Causes the
runlist scheduler to:
1. Reload and cache timeslice and scale values from TSGs.
2. Restart scheduling from the head of the runlist [may cause a
   preempt to be scheduled for the currently-running task (?)].
3. Address (?) an errata on Turing where re-enabled channels are
   not always detected.

Above behavior tested on GV100 and partially tested on TU102.

4 files changed, 73 insertions, 5 deletions

diff --git a/nvdebug.h b/nvdebug.h
index 6a2383e..7564f8c 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -274,7 +274,7 @@ typedef union {
 
   We only document the *_PFIFO_ENG_RUNLIST_*(i) read-only registers here (where
   i is a runlist index). Runlists are configured via the seperate, writable
-  *_PFIFO_RUNLIST_* register; see open-gpu-doc for more on that.
+  *_PFIFO_RUNLIST_* register; see open-gpu-doc for more.
 
   LEN         : Number of entries in runlist
   IS_PENDING  : Is runlist committed?
@@ -284,18 +284,25 @@ typedef union {
   Support: Fermi*, Kepler, Maxwell, Pascal, Volta
   *Fermi may expose this information 8 bytes earlier, starting at 0x227C?
 */
+#define NV_PFIFO_RUNLIST_BASE_GF100 0x00002270 // Write-only
 #define NV_PFIFO_ENG_RUNLIST_BASE_GF100(i) (0x00002280+(i)*8) // Read-only
 typedef union {
         struct {
                 // NV_PFIFO_ENG_RUNLIST_BASE_* fields
                 uint32_t ptr:28;
                 enum INST_TARGET target:2;
-                 uint32_t padding1:2;
+                 uint32_t :2;
                 // NV_PFIFO_ENG_RUNLIST_* fields
                 uint16_t len:16;
-                 uint32_t padding2:4;
-                bool is_pending:1;
-                 uint32_t padding3:11;
+                 uint32_t :4;
+                bool is_pending:1; // Read-only from NV_PFIFO_ENG_RUNLIST...
+                 uint32_t :11;
+        } __attribute__((packed));
+        struct {
+                // NV_PFIFO_RUNLIST_* fields that differ from NV_PFIFO_ENG_RUNLIST_*
+                 uint64_t :52;
+                uint32_t id:4; // Write-only to NV_PFIFO_RUNLIST...
+                 uint32_t :8;
         } __attribute__((packed));
         uint64_t raw;
 } eng_runlist_gf100_t;
@@ -311,6 +318,9 @@ typedef union {
 // Support: Turing
 #define NV_PFIFO_RUNLIST_BASE_TU102(i) (0x00002B00+(i)*16) // Read/write
 #define NV_PFIFO_RUNLIST_SUBMIT_TU102(i) (0x00002B08+(i)*16) // Read/write
+// Derived absolute maximum number of runlists
+#define MAX_RUNLISTS_TU102 80 // On Turing; another register is at 0x00003000
+#define MAX_RUNLISTS_GF100 34 // On Volta-; another register is at 0x00002390
 typedef union {
         struct {
                 enum INST_TARGET target:2;
diff --git a/nvdebug_entry.c b/nvdebug_entry.c
index 24fcd32..eee7351 100644
--- a/nvdebug_entry.c
+++ b/nvdebug_entry.c
@@ -25,6 +25,7 @@ MODULE_DESCRIPTION("A scheduling debugging module for NVIDIA GPUs");
 // runlist_procfs.c
 extern struct file_operations runlist_file_ops;
 extern struct file_operations preempt_tsg_file_ops;
+extern struct file_operations resubmit_runlist_file_ops;
 extern struct file_operations disable_channel_file_ops;
 extern struct file_operations enable_channel_file_ops;
 extern struct file_operations switch_to_tsg_file_ops;
@@ -256,6 +257,17 @@ int __init nvdebug_init(void) {
                                 "preempt_tsg", 0222, dir, compat_ops(&preempt_tsg_file_ops),
                                 (void*)device_id))
                         goto out_nomem;
+                /* On the TU104, the context scheduler (contained in the Host, aka
+                 * PFIFO, unit) has been observed to sometimes to fail to schedule TSGs
+                 * containing re-enabled channels. Resubmitting the runlist
+                 * configuration appears to remediate this condition, and so this API
+                 * is exposed to help reset GPU scheduling as necessary.
+                 */
+                // Create file `/proc/gpu#/resubmit_runlist`, world writable
+                if (!proc_create_data(
+                                "resubmit_runlist", 0222, dir, compat_ops(&resubmit_runlist_file_ops),
+                                (void*)device_id))
+                        goto out_nomem;
                 // Create file `/proc/gpu#/disable_channel`, world writable
                 if (!proc_create_data(
                                 "disable_channel", 0222, dir, compat_ops(&disable_channel_file_ops),
diff --git a/runlist.c b/runlist.c
index 22f47ff..91fca82 100644
--- a/runlist.c
+++ b/runlist.c
@@ -137,3 +137,22 @@ int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id) {
         nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
         return 0;
 }
+
+// Read and write runlist configuration, triggering a resubmit
+int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
+        if (g->chip_id < NV_CHIP_ID_TURING) {
+                eng_runlist_gf100_t rl;
+                if ((rl.raw = nvdebug_readq(g, NV_PFIFO_ENG_RUNLIST_BASE_GF100(rl_id))) == -1)
+                        return -EIO;
+                rl.id = rl_id;
+                nvdebug_writeq(g, NV_PFIFO_RUNLIST_BASE_GF100, rl.raw);
+        } else if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                runlist_submit_tu102_t submit;
+                if ((submit.raw = nvdebug_readq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id))) == -1)
+                        return -EIO;
+                nvdebug_writeq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id), submit.raw);
+        } else {
+                return -EOPNOTSUPP;
+        }
+        return 0;
+}
diff --git a/runlist_procfs.c b/runlist_procfs.c
index 0087d90..f569c77 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -207,6 +207,33 @@ struct file_operations preempt_tsg_file_ops = {
         .llseek = default_llseek,
 };
 
+
+ssize_t resubmit_runlist_file_write(struct file *f, const char __user *buffer,
+                               size_t count, loff_t *off) {
+        uint32_t target_runlist;
+        // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
+        int err = kstrtou32_from_user(buffer, count, 0, &target_runlist);
+        struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
+        if (err)
+                return err;
+
+        // Verify valid runlist (in terms of absolute maximums)
+        if (g->chip_id < NV_CHIP_ID_TURING && target_runlist > MAX_RUNLISTS_GF100)
+                return -ERANGE;
+        else if (g->chip_id < NV_CHIP_ID_AMPERE && target_runlist > MAX_RUNLISTS_TU102)
+                return -ERANGE;
+
+        if ((err = resubmit_runlist(g, target_runlist)))
+                return err;
+
+        return count;
+}
+
+struct file_operations resubmit_runlist_file_ops = {
+        .write = resubmit_runlist_file_write,
+        .llseek = default_llseek,
+};
+
 ssize_t disable_channel_file_write(struct file *f, const char __user *buffer,
                                    size_t count, loff_t *off) {
         uint32_t target_channel;