4 files changed, 260 insertions, 97 deletions

diff --git a/nvdebug.h b/nvdebug.h
index 26689d9..26167a7 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -172,7 +172,8 @@ enum PREEMPT_TYPE {PREEMPT_TYPE_CHANNEL = 0, PREEMPT_TYPE_TSG = 1};
   IS_PENDING  : Is a context switch pending? (read-only)
   TYPE        : PREEMPT_TYPE_CHANNEL or PREEMPT_TYPE_TSG
 
-  Support: Kepler, Maxwell, Pascal, Volta, Turing
+  Support: Fermi*, Kepler, Maxwell, Pascal, Volta, Turing
+  *Fermi only supports PREEMPT_TYPE_CHANNEL.
 */
 #define NV_PFIFO_PREEMPT 0x00002634
 typedef union {
@@ -187,10 +188,26 @@ typedef union {
         uint32_t raw;
 } pfifo_preempt_t;
 
+/* Preempt a TSG or Runlist by ID
+  Similar as on older GPUs (see above), but located at an offset in Runlist RAM.
+  This means that there's one instance of this register for each runlist.
+
+  IS_PENDING is now IS_TSG_PREEMPT_PENDING and IS_RUNLIST_PREEMPT_PENDING was
+  added in the following bit (bit 22). As these fields are unused in nvdebug,
+  we use the old structure for simplicity.
+
+  TYPE is now better described as IS_TSG_PREEMPT. TYPE == 0 requests a preempt
+  of the runlist (rather than a channel preemption, as on older GPUs).
+
+  Support: Ampere, Hopper, Ada, [newer untested]
+*/
+#define NV_RUNLIST_PREEMPT_GA100 0x098
+#define PREEMPT_TYPE_RUNLIST 0
+
 /*
   "Initiate a preempt of the engine by writing the bit associated with its
   runlist to NV_PFIFO_RUNLIST_PREEMPT...  Do not poll NV_PFIFO_RUNLIST_PREEMPT
-  for the preempt to complete."
+  for the preempt to complete." (open-gpu-doc)
 
   Useful for preempting multiple runlists at once.
 
@@ -204,7 +221,10 @@ typedef union {
     rl_preempt.raw |= BIT(nr);
     nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
 
-  Support: Volta, Turing
+  Support: Maxwell, Pascal, Volta, Turing
+
+  This register was deleted starting with Ampere, with functionality subsumed by
+  the NV_RUNLIST_PREEMPT register.
 */
 #define NV_PFIFO_RUNLIST_PREEMPT 0x00002638
 typedef union {
@@ -285,7 +305,7 @@ typedef union {
   TARGET      : Aperture of runlist (video or system memory)
 
   Support: Fermi*, Kepler, Maxwell, Pascal, Volta
-  *Fermi may expose this information 8 bytes earlier, starting at 0x227C?
+  *Fermi may expose ENG_RUNLING_* 8 bytes earlier, starting at 0x227C?
 */
 #define NV_PFIFO_RUNLIST_BASE_GF100 0x00002270 // Write-only
 #define NV_PFIFO_ENG_RUNLIST_BASE_GF100(i) (0x00002280+(i)*8) // Read-only
@@ -428,7 +448,11 @@ typedef union {
   See also: manuals/turing/tu104/dev_fifo.ref.txt in NVIDIA's open-gpu-doc
 */
 #define NV_PCCSR_CHANNEL_INST(i) (0x00800000+(i)*8)
-#define MAX_CHID 512
+// Maximum valid channel index in the PCCSR region
+// Channel IDs start at 0, and there are 4096 bytes of 8-byte CCSR entries (per
+// NV_PCCSR_CHANNEL_INST__SIZE_1 in at least Volta and Turing), yielding a total
+// of 512 channel IDs, with a maximum ID of 511.
+#define MAX_CHID 511
 typedef union {
         struct {
 // 0:31
@@ -554,6 +578,7 @@ typedef union {
 #define NV_CHIP_ID_GP106 0x136 // Discrete GeForce GTX 1060
 #define NV_CHIP_ID_GV11B 0x15B // Jetson Xavier embedded GPU
 
+#define NV_CHIP_ID_FERMI 0x0C0
 #define NV_CHIP_ID_KEPLER 0x0E0
 #define NV_CHIP_ID_MAXWELL 0x120
 #define NV_CHIP_ID_PASCAL 0x130
@@ -1505,7 +1530,7 @@ int get_runlist_iter(
         struct nvdebug_state *g,
         int rl_id,
         struct runlist_iter *rl_iter /* out */);
-int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id);
+int preempt_tsg(struct nvdebug_state *g, uint32_t rl_id, uint32_t tsg_id);
 int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id);
 int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id);
 
diff --git a/nvdebug_entry.c b/nvdebug_entry.c
index 5f99976..d5df7db 100644
--- a/nvdebug_entry.c
+++ b/nvdebug_entry.c
@@ -289,17 +289,45 @@ int __init nvdebug_init(void) {
                         snprintf(runlist_name, 12, "runlist%lu", last_runlist);
                         if (!(rl_dir = proc_mkdir_data(runlist_name, 0555, dir, (void*)device_id)))
                                 goto out_nomem;
+                        // Create one file for each runlist on Ampere+, or one file for each GPU on older
+                        if (g_nvdebug_state[res].chip_id >= NV_CHIP_ID_AMPERE || last_runlist == 0) {
+                                struct proc_dir_entry *chram_scope;
+                                // preempt_tsg, enable_channel, and disable_channel refer to a GPU-global channel
+                                // RAM on pre-Ampere GPUs
+                                if (g_nvdebug_state[res].chip_id >= NV_CHIP_ID_AMPERE)
+                                        chram_scope = rl_dir;
+                                else
+                                        chram_scope = dir;
+                                // Create file `/proc/gpu#/runlist#/preempt_tsg`, world writable
+                                // On Turing and older, `/proc/gpu#/preempt_tsg`
+                                if (!proc_create_data(
+                                                "preempt_tsg", 0222, chram_scope, compat_ops(&preempt_tsg_file_ops),
+                                                (void*)last_runlist))
+                                        goto out_nomem;
+                                // Create file `/proc/gpu#/runlist#/disable_channel`, world writable
+                                // On Turing and older, `/proc/gpu#/disable_channel`
+                                if (!proc_create_data(
+                                                "disable_channel", 0222, chram_scope, compat_ops(&disable_channel_file_ops),
+                                                (void*)last_runlist))
+                                        goto out_nomem;
+                                // Create file `/proc/gpu#/runlist#/enable_channel`, world writable
+                                // On Turing and older, `/proc/gpu#/enable_channel`
+                                if (!proc_create_data(
+                                                "enable_channel", 0222, chram_scope, compat_ops(&enable_channel_file_ops),
+                                                (void*)last_runlist))
+                                        goto out_nomem;
+                        }
                         // Create file `/proc/gpu#/runlist#/runlist`, world readable
                         if (!proc_create_data(
                                         "runlist", 0444, rl_dir, compat_ops(&runlist_file_ops),
                                         (void*)last_runlist))
                                 goto out_nomem;
+                        // Create file `/proc/gpu#/runlist#/switch_to_tsg`, world writable
+                        if (!proc_create_data(
+                                        "switch_to_tsg", 0222, rl_dir, compat_ops(&switch_to_tsg_file_ops),
+                                        (void*)last_runlist))
+                                goto out_nomem;
                 } while (last_runlist-- > 0);
-                // Create file `/proc/gpu#/preempt_tsg`, world writable
-                if (!proc_create_data(
-                                "preempt_tsg", 0222, dir, compat_ops(&preempt_tsg_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
                 /* On the TU104, the context scheduler (contained in the Host, aka
                  * PFIFO, unit) has been observed to sometimes to fail to schedule TSGs
                  * containing re-enabled channels. Resubmitting the runlist
@@ -311,21 +339,6 @@ int __init nvdebug_init(void) {
                                 "resubmit_runlist", 0222, dir, compat_ops(&resubmit_runlist_file_ops),
                                 (void*)device_id))
                         goto out_nomem;
-                // Create file `/proc/gpu#/disable_channel`, world writable
-                if (!proc_create_data(
-                                "disable_channel", 0222, dir, compat_ops(&disable_channel_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
-                // Create file `/proc/gpu#/enable_channel`, world writable
-                if (!proc_create_data(
-                                "enable_channel", 0222, dir, compat_ops(&enable_channel_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
-                // Create file `/proc/gpu#/switch_to_tsg`, world writable
-                if (!proc_create_data(
-                                "switch_to_tsg", 0222, dir, compat_ops(&switch_to_tsg_file_ops),
-                                (void*)device_id))
-                        goto out_nomem;
                 // Create file `/proc/gpu#/device_info`, world readable
                 if (!proc_create_data(
                                 "device_info", 0444, dir, compat_ops(&device_info_file_ops),
diff --git a/runlist.c b/runlist.c
index 7e6d292..7bb2ee4 100644
--- a/runlist.c
+++ b/runlist.c
@@ -169,24 +169,35 @@ attempt_pramin_access:
 
 /* Trigger a preempt of the specified TSG
   @param tsg_id ID of TSG to preempt.
+  @param rl_id  Which channel RAM address space to search?
   @return 0 or -errno on error
 
   Note: If no other TSGs exist in the associated runlist, this TSG may
         continue executing, unless NV_PFIFO_SCHED_DISABLE is set, or all the
         channels of the TSG to be preempted are disabled.
 */
-int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) {
-        pfifo_preempt_t pfifo_preempt;
+int preempt_tsg(struct nvdebug_state *g, uint32_t rl_id, uint32_t tsg_id) {
+        pfifo_preempt_t preempt;
+        // Fermi does not support time-slice groups
         if (g->chip_id < NV_CHIP_ID_KEPLER)
                 return -EOPNOTSUPP;