Ampere: disable/enable_channel, preempt/switch_to_tsg, and resubmit_runlist

**Modifes the user API from `echo 1 > /proc/gpuX/switch_to_tsg` to
`echo 1 > /proc/gpuX/runlist0/switch_to_tsg` to switch to TSG 1 on
runlist 0 on GPU X for pre-Ampere GPUs (for example).**

Feature changes:
- switch_to_tsg only makes sense on a per-runlist level. Before, this
  always operated on runlist0; this commit allows operating on any
  runlist by moving the API to the per-runlist paths.
- On Ampere+, channel and TSG IDs are per-runlist, and no longer
  GPU-global. Consequently, the disable/enable_channel and
  preempt_tsg APIs have been moved from GPU-global to per-runlist
  paths on Ampere+.

Bug fixes:
- `preempt_runlist()` is now supported on Maxwell and Pascal.
- `resubmit_runlist()` detects too-old GPUs.
- MAX_CHID corrected from 512 to 511 and documented.
- switch_to_tsg now includes a runlist resubmit, which appears to be
  necessary on Turing+ GPUs.

Tested on GK104 (Quadro K5000), GM204 (GTX 970), GP106 (GTX 1060 3GB),
GP104 (GTX 1080 Ti), GP10B (Jetson TX2), GV11B (Jetson Xavier), GV100
(Titan V), TU102 (RTX 2080 Ti), and AD102 (RTX 6000 Ada).

1 files changed, 56 insertions, 13 deletions

diff --git a/runlist.c b/runlist.c
index 7e6d292..7bb2ee4 100644
--- a/runlist.c
+++ b/runlist.c
@@ -169,24 +169,35 @@ attempt_pramin_access:
 
 /* Trigger a preempt of the specified TSG
   @param tsg_id ID of TSG to preempt.
+  @param rl_id  Which channel RAM address space to search?
   @return 0 or -errno on error
 
   Note: If no other TSGs exist in the associated runlist, this TSG may
         continue executing, unless NV_PFIFO_SCHED_DISABLE is set, or all the
         channels of the TSG to be preempted are disabled.
 */
-int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) {
-        pfifo_preempt_t pfifo_preempt;
+int preempt_tsg(struct nvdebug_state *g, uint32_t rl_id, uint32_t tsg_id) {
+        pfifo_preempt_t preempt;
+        // Fermi does not support time-slice groups
         if (g->chip_id < NV_CHIP_ID_KEPLER)
                 return -EOPNOTSUPP;
 
-        pfifo_preempt.raw = 0;
-        pfifo_preempt.id = tsg_id;
-        pfifo_preempt.is_pending = 0;
-        pfifo_preempt.type = PREEMPT_TYPE_TSG;
+        preempt.raw = 0;
+        preempt.id = tsg_id;
+        preempt.type = PREEMPT_TYPE_TSG;
 
         // Actually trigger the preemption
-        nvdebug_writel(g, NV_PFIFO_PREEMPT, pfifo_preempt.raw);
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                nvdebug_writel(g, NV_PFIFO_PREEMPT, preempt.raw);
+        } else {
+                uint32_t runlist_reg_base;
+                int err;
+                // As TSG and channel IDs are namespaced per-runlist starting with
+                // Ampere, the PREEMPT register is also per-runlist.
+                if ((err = get_runlist_ram(g, rl_id, &runlist_reg_base)))
+                        return err;
+                nvdebug_writel(g, runlist_reg_base + NV_RUNLIST_PREEMPT_GA100, preempt.raw);
+        }
         return 0;
 }
 
@@ -195,18 +206,38 @@ int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id) {
   @return 0 or -errno on error
 */
 int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id) {
-        runlist_preempt_t rl_preempt;
-        if (g->chip_id < NV_CHIP_ID_VOLTA)
+        // The runlist preempt register does not exist on Kepler (tested gk104)
+        if (g->chip_id < NV_CHIP_ID_MAXWELL)
                 return -EOPNOTSUPP;
 
-        // Overwrite, as the register contains nothing to preserve
-        rl_preempt.raw = BIT(rl_id);
-        nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
+        // Write to trigger the preemption (the register contains nothing to
+        // preserve, and can thus just be overwritten)
+        if (g->chip_id < NV_CHIP_ID_AMPERE) {
+                runlist_preempt_t rl_preempt;
+                rl_preempt.raw = BIT(rl_id);
+                nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
+        } else {
+                int err;
+                uint32_t runlist_regs_base;
+                pfifo_preempt_t preempt;
+                // The RUNLIST_PREEMPT register was deleted, and the _PREEMPT register
+                // was extended to support runlist-level preemptions starting on Ampere
+                preempt.id = rl_id;
+                preempt.type = PREEMPT_TYPE_RUNLIST;
+                // The preempt register is scoped per-runlist on Ampere+
+                if ((err = get_runlist_ram(g, rl_id, &runlist_regs_base)))
+                        return err;
+                nvdebug_writel(g, runlist_regs_base + NV_RUNLIST_PREEMPT_GA100, preempt.raw);
+        }
         return 0;
 }
 
 // Read and write runlist configuration, triggering a resubmit
 int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
+        // Necessary registers do not exist pre-Fermi
+        if (g->chip_id < NV_CHIP_ID_FERMI)
+                return -EOPNOTSUPP;
+
         if (g->chip_id < NV_CHIP_ID_TURING) {
                 eng_runlist_gf100_t rl;
                 if (rl_id > MAX_RUNLISTS_GF100)
@@ -223,7 +254,19 @@ int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id) {
                         return -EIO;
                 nvdebug_writeq(g, NV_PFIFO_RUNLIST_SUBMIT_TU102(rl_id), submit.raw);
         } else {
-                return -EOPNOTSUPP;
+                int err;
+                uint32_t runlist_pri_base;
+                runlist_submit_tu102_t submit;
+                if ((err = get_runlist_ram(g, rl_id, &runlist_pri_base)) < 0)
+                        return err;
+                if ((submit.raw = nvdebug_readq(g, runlist_pri_base + NV_RUNLIST_SUBMIT_GA100)) == -1)
+                        return -EIO;
+                // On Ampere, this does not appear to trigger a preempt of the
+                // currently-running channel (even if the currently running channel
+                // becomes disabled), but will cause newly re-enabled channels
+                // (at least if nothing else is pending) to become ready (tested on
+                // Jetson Orin).
+                nvdebug_writeq(g, runlist_pri_base + NV_RUNLIST_SUBMIT_GA100, submit.raw);
         }
         return 0;
 }