4 files changed, 313 insertions, 29 deletions

diff --git a/nvdebug.h b/nvdebug.h
index cd0dc90..9ac71da 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -99,42 +99,103 @@ struct entry_tsg {
         uint32_t tsgid:12;
          uint64_t padding4:52;
 } __attribute__((packed));
+#define MAX_TSGID (1 << 12)
 
 enum PREEMPT_TYPE {PREEMPT_TYPE_CHANNEL = 0, PREEMPT_TYPE_TSG = 1};
 
-/* Preempt
+/* Preempt a TSG or Channel by ID
   ID/CHID             : Id of TSG or channel to preempt
+  IS_PENDING          : ????
+  TYPE                : PREEMPT_TYPE_CHANNEL or PREEMPT_TYPE_TSG
+
+  Support: Kepler, Maxwell, Pascal, Volta
 */
 #define NV_PFIFO_PREEMPT 0x00002634
-struct pfifo_preempt {
-        uint32_t id:12;
-         uint32_t padding:8;
-        bool is_pending:1;
-         uint32_t padding2:3;
-        enum PREEMPT_TYPE type:2;
-         uint32_t padding3:6;
-} __attribute__((packed));
+typedef union {
+        struct {
+                uint32_t id:12;
+                 uint32_t padding:8;
+                bool is_pending:1;
+                 uint32_t padding2:3;
+                enum PREEMPT_TYPE type:2;
+                 uint32_t padding3:6;
+        } __attribute__((packed));
+        uint32_t raw;
+} pfifo_preempt_t;
+
+/*
+  "Initiate a preempt of the engine by writing the bit associated with its
+  runlist to NV_PFIFO_RUNLIST_PREEMPT...  Do not poll NV_PFIFO_RUNLIST_PREEMPT
+  for the preempt to complete."
 
+  Useful for preempting multiple runlists at once.
+
+  Appears to trigger an interrupt or some other side-effect on the Jetson
+  Xavier, as the built-in nvgpu driver seems to be disturbed by writing to this.
+
+  To select the runlist dynamically, use the BIT(nr) kernel macro.
+  Example:
+    runlist_preempt_t rl_preempt;
+    rl_preempt.raw = nvdebug_readl(g, NV_PFIFO_RUNLIST_PREEMPT);
+    rl_preempt.raw |= BIT(nr);
+    nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
+
+  Support: Volta
+*/
 #define NV_PFIFO_RUNLIST_PREEMPT 0x00002638
-struct runlist_preempt {
-        bool runlist_0:1;
-        bool runlist_1:1;
-        bool runlist_2:1;
-        bool runlist_3:1;
-        bool runlist_4:1;
-        bool runlist_5:1;
-        bool runlist_6:1;
-        bool runlist_7:1;
-        bool runlist_8:1;
-        bool runlist_9:1;
-        bool runlist_10:1;
-        bool runlist_11:1;
-        bool runlist_12:1;
-        bool runlist_13:1;
-         uint32_t padding:28;
-} __attribute__((packed));
+typedef union {
+        struct {
+                bool runlist_0:1;
+                bool runlist_1:1;
+                bool runlist_2:1;
+                bool runlist_3:1;
+                bool runlist_4:1;
+                bool runlist_5:1;
+                bool runlist_6:1;
+                bool runlist_7:1;
+                bool runlist_8:1;
+                bool runlist_9:1;
+                bool runlist_10:1;
+                bool runlist_11:1;
+                bool runlist_12:1;
+                bool runlist_13:1;
+                 uint32_t padding:18;
+        } __attribute__((packed));
+        uint32_t raw;
+} runlist_preempt_t;
+
+/* Additional information on preempting from NVIDIA's driver (commit b1d0d8ece)
+ * "From h/w team
+ * Engine save can be blocked by eng  stalling interrupts.
+ * FIFO interrupts shouldn’t block an engine save from
+ * finishing, but could block FIFO from reporting preempt done.
+ * No immediate reason to reset the engine if FIFO interrupt is
+ * pending.
+ * The hub, priv_ring, and ltc interrupts could block context
+ * switch (or memory), but doesn’t necessarily have to.
+ * For Hub interrupts they just report access counters and page
+ * faults. Neither of these necessarily block context switch
+ * or preemption, but they could.
+ * For example a page fault for graphics would prevent graphics
+ * from saving out. An access counter interrupt is a
+ * notification and has no effect.
+ * SW should handle page faults though for preempt to complete.
+ * PRI interrupt (due to a failed PRI transaction) will result
+ * in ctxsw failure reported to HOST.
+ * LTC interrupts are generally ECC related and if so,
+ * certainly don’t block preemption/ctxsw but they could.
+ * Bus interrupts shouldn’t have anything to do with preemption
+ * state as they are part of the Host EXT pipe, though they may
+ * exhibit a symptom that indicates that GPU is in a bad state.
+ * To be completely fair, when an engine is preempting SW
+ * really should just handle other interrupts as they come in.
+ * It’s generally bad to just poll and wait on a preempt
+ * to complete since there are many things in the GPU which may
+ * cause a system to hang/stop responding."
+ */
 
 // Note: This is different with Turing
+// Support: Kepler, Maxwell, Pascal, Volta
 #define NV_PFIFO_RUNLIST_BASE 0x00002270
 typedef union {
         struct {
@@ -145,6 +206,7 @@ typedef union {
         uint32_t raw;
 } runlist_base_t;
 
+// Support: Kepler, Maxwell, Pascal, Volta
 #define NV_PFIFO_RUNLIST 0x00002274
 typedef union {
         struct {
@@ -175,8 +237,8 @@ enum CHANNEL_STATUS {
 };
 
 #define NV_PCCSR_CHANNEL_INST(i) (0x00800000+(i)*8)
-#define MAX_CHID 512  // TODO: Double-check this is right
 // There are a total of 512 possible channels
+#define MAX_CHID 512
 typedef union {
         struct {
 // 0:31
@@ -202,6 +264,43 @@ typedef union {
         uint64_t raw;
 } channel_ctrl_t;
 
+/* Control word for runlist enable/disable.
+
+  RUNLIST_N           : Is runlist n disabled? (1 == disabled, 0 == enabled)
+
+  To select the runlist dynamically, use the BIT(nr) kernel macro.
+  Disabling example:
+    runlist_disable_t rl_disable;
+    rl_disable.raw = nvdebug_readl(g, NV_PFIFO_SCHED_DISABLE);
+    rl_disable.raw |= BIT(nr);
+    nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw);
+  Enabling example:
+    runlist_disable_t rl_disable;
+    rl_disable.raw = nvdebug_readl(g, NV_PFIFO_SCHED_DISABLE);
+    rl_disable.raw &= ~BIT(nr);
+    nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw);
+
+  Support: Fermi, Kepler, Maxwell, Pascal, Volta, Turing
+*/
+#define NV_PFIFO_SCHED_DISABLE 0x00002630
+typedef union {
+        struct {
+                bool runlist_0:1;
+                bool runlist_1:1;
+                bool runlist_2:1;
+                bool runlist_3:1;
+                bool runlist_4:1;
+                bool runlist_5:1;
+                bool runlist_6:1;
+                bool runlist_7:1;
+                bool runlist_8:1;
+                bool runlist_9:1;
+                bool runlist_10:1;
+                 uint32_t padding:21;
+        } __attribute__((packed));
+        uint32_t raw;
+} runlist_disable_t;
+
 // TODO(jbakita): Maybe put the above GPU types in a different file.
 
 #define for_chan_in_tsg(chan, tsg) \
@@ -220,6 +319,7 @@ struct runlist_iter {
 // Defined in runlist.c
 struct gk20a* get_live_gk20a(void);
 int get_runlist_iter(struct runlist_iter *rl_iter);
+int preempt_tsg(uint32_t tsg_id);
 
 static inline struct gk20a *get_gk20a(struct device *dev) {
         // XXX: Only works because gk20a* is the first member of gk20a_platform
diff --git a/nvdebug_entry.c b/nvdebug_entry.c
index 6346659..14ad6e9 100644
--- a/nvdebug_entry.c
+++ b/nvdebug_entry.c
@@ -20,11 +20,31 @@ MODULE_DESCRIPTION("A scheduling debugging module for NVIDIA GPUs");
 MODULE_SOFTDEP("pre: nvgpu"); // We only support the Jetson boards for now
 
 extern const struct file_operations runlist_file_ops;
+extern const struct file_operations preempt_tsg_file_ops;
+extern const struct file_operations disable_channel_file_ops;
+extern const struct file_operations enable_channel_file_ops;
+extern const struct file_operations switch_to_tsg_file_ops;
 
 int __init nvdebug_init(void) {
-        struct proc_dir_entry *entry = proc_create("runlist", 0444, NULL, &runlist_file_ops);
-        if (!entry) {
+        struct proc_dir_entry *rl_entry, *preempt_entry, *disable_channel_entry,
+                              *enable_channel_entry, *switch_to_tsg_entry;
+        // Create file `/proc/preempt_tsg`, world readable
+        rl_entry = proc_create("runlist", 0444, NULL, &runlist_file_ops);
+        // Create file `/proc/preempt_tsg`, world writable
+        preempt_entry = proc_create("preempt_tsg", 0222, NULL, &preempt_tsg_file_ops);
+        // Create file `/proc/disable_channel`, world writable
+        disable_channel_entry = proc_create("disable_channel", 0222, NULL, &disable_channel_file_ops);
+        // Create file `/proc/enable_channel`, world writable
+        enable_channel_entry = proc_create("enable_channel", 0222, NULL, &enable_channel_file_ops);
+        // Create file `/proc/switch_to_tsg`, world writable
+        switch_to_tsg_entry = proc_create("switch_to_tsg", 0222, NULL, &switch_to_tsg_file_ops);
+        // ProcFS entry creation only fails if out of memory
+        if (!rl_entry || !preempt_entry || !disable_channel_entry || !enable_channel_entry || !switch_to_tsg_entry) {
                 remove_proc_entry("runlist", NULL);
+                remove_proc_entry("preempt_tsg", NULL);
+                remove_proc_entry("disable_channel", NULL);
+                remove_proc_entry("enable_channel", NULL);
+                remove_proc_entry("switch_to_tsg", NULL);
                 printk(KERN_ERR "[nvdebug] Unable to initialize procfs entries!\n");
                 return -ENOMEM;
         }
@@ -34,6 +54,10 @@ int __init nvdebug_init(void) {
 
 static void __exit nvdebug_exit(void) {
         remove_proc_entry("runlist", NULL);
+        remove_proc_entry("preempt_tsg", NULL);
+        remove_proc_entry("disable_channel", NULL);
+        remove_proc_entry("enable_channel", NULL);
+        remove_proc_entry("switch_to_tsg", NULL);
         printk(KERN_INFO "[nvdebug] Exiting...\n");
 }
 
diff --git a/runlist.c b/runlist.c
index 8dfa1c7..03528af 100644
--- a/runlist.c
+++ b/runlist.c
@@ -109,3 +109,28 @@ int get_runlist_iter(struct runlist_iter *rl_iter) {
         printk(KERN_INFO "[nvdebug] tsg_length: %d\n", head.tsg_length);
         printk(KERN_INFO "[nvdebug] tsgid: %d\n", head.tsgid); */
 }
+
+int preempt_tsg(uint32_t tsg_id) {
+        struct gk20a *g = get_live_gk20a();
+        runlist_info_t rl_info;
+        pfifo_preempt_t pfifo_preempt;
+        runlist_disable_t rl_disable;
+        if (!g)
+                return -EIO;
+        rl_info.raw = nvdebug_readl(g, NV_PFIFO_RUNLIST);
+        pfifo_preempt.id = tsg_id;
+        pfifo_preempt.is_pending = 0;
+        pfifo_preempt.type = PREEMPT_TYPE_TSG;
+        // There may be a bug (?) that requires us to disable scheduling before preempting
+        rl_disable.raw = nvdebug_readl(g, NV_PFIFO_SCHED_DISABLE);
+        rl_disable.raw |= BIT(rl_info.id);  // Disable runlist rl_info.id
+        nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw);
+        // Actually trigger the preemption
+        nvdebug_writel(g, NV_PFIFO_PREEMPT, pfifo_preempt.raw);
+        // Renable scheduling
+        rl_disable.raw &= ~BIT(rl_info.id);  // Enable runlist rl_info.id
+        nvdebug_writel(g, NV_PFIFO_SCHED_DISABLE, rl_disable.raw);
+
+        printk(KERN_INFO "[nvdebug] TSG %d preempted (runlist %d)\n", tsg_id, rl_info.id);
+        return 0;
+}
diff --git a/runlist_procfs.c b/runlist_procfs.c
index 183eab6..411f844 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -151,3 +151,138 @@ const struct file_operations runlist_file_ops = {
         .llseek = seq_lseek,
         .release = seq_release,
 };
+
+ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer,
+                               size_t count, loff_t *off) {
+        uint32_t target_tsgid;
+        // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
+        int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
+        if (err)
+                return err;
+
+        // TSG IDs are a 12-bit field, so make sure the request is in-range
+        if (target_tsgid > MAX_TSGID)
+                return -ERANGE;
+
+        // Execute preemption
+        err = preempt_tsg(target_tsgid);
+        if (err)
+                return err;
+
+        return count;
+}
+
+const struct file_operations preempt_tsg_file_ops = {
+        .write = preempt_tsg_file_write,
+};
+
+ssize_t disable_channel_file_write(struct file *f, const char __user *buffer,
+                                   size_t count, loff_t *off) {
+        uint32_t target_channel;
+        channel_ctrl_t chan;
+        int err;
+        struct gk20a *g = get_live_gk20a();
+        if (!g)
+                return -EIO;
+        // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
+        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        if (err)
+                return err;
+
+        if (target_channel > MAX_CHID)
+                return -ERANGE;
+
+        // Disable channel
+        chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel));
+        chan.enable_clear = true;
+        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+
+        return count;
+}
+
+const struct file_operations disable_channel_file_ops = {
+        .write = disable_channel_file_write,
+};
+
+ssize_t enable_channel_file_write(struct file *f, const char __user *buffer,
+                                   size_t count, loff_t *off) {
+        uint32_t target_channel;
+        channel_ctrl_t chan;
+        int err;
+        struct gk20a *g = get_live_gk20a();
+        if (!g)
+                return -EIO;
+        // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
+        err = kstrtou32_from_user(buffer, count, 0, &target_channel);
+        if (err)
+                return err;
+
+        if (target_channel > MAX_CHID)
+                return -ERANGE;
+
+        // Disable channel
+        chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel));
+        chan.enable_set = true;
+        nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw);
+
+        return count;
+}
+
+const struct file_operations enable_channel_file_ops = {
+        .write = enable_channel_file_write,
+};
+
+ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer,
+                                   size_t count, loff_t *off) {
+        uint32_t target_tsgid;
+        struct runlist_chan* chan;
+        channel_ctrl_t chan_ctl;
+        struct runlist_iter rl_iter;
+        int err;
+        loff_t pos = 0;
+        struct gk20a *g = get_live_gk20a();
+        if (!g)
+                return -EIO;
+        // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
+        err = kstrtou32_from_user(buffer, count, 0, &target_tsgid);
+        if (err)
+                return err;
+
+        if (target_tsgid > MAX_TSGID)
+                return -ERANGE;
+
+        err = get_runlist_iter(&rl_iter);
+        if (err)
+                return err;
+
+        // Iterate through all TSGs
+        while (pos < rl_iter.rl_info.len) {
+                if (rl_iter.curr_tsg->tsgid == target_tsgid) {
+                        // Enable channels of target TSG
+                        for_chan_in_tsg(chan, rl_iter.curr_tsg) {
+                                chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
+                                chan_ctl.enable_set = true;
+                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
+                        }
+                } else {
+                        // Disable all other channels
+                        for_chan_in_tsg(chan, rl_iter.curr_tsg) {
+                                chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid));
+                                chan_ctl.enable_clear = true;
+                                nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw);
+                        }
+                }
+                pos += 1 + rl_iter.curr_tsg->tsg_length;
+                rl_iter.curr_tsg = next_tsg(rl_iter.curr_tsg);
+        }
+        // Switch to next TSG with active channels (should be our TSG)
+        err = preempt_tsg(target_tsgid);
+        if (err)
+                return err;
+
+        return count;
+}
+
+const struct file_operations switch_to_tsg_file_ops = {
+        .write = switch_to_tsg_file_write,
+};