Ampere: disable/enable_channel, preempt/switch_to_tsg, and resubmit_runlist

**Modifes the user API from `echo 1 > /proc/gpuX/switch_to_tsg` to
`echo 1 > /proc/gpuX/runlist0/switch_to_tsg` to switch to TSG 1 on
runlist 0 on GPU X for pre-Ampere GPUs (for example).**

Feature changes:
- switch_to_tsg only makes sense on a per-runlist level. Before, this
  always operated on runlist0; this commit allows operating on any
  runlist by moving the API to the per-runlist paths.
- On Ampere+, channel and TSG IDs are per-runlist, and no longer
  GPU-global. Consequently, the disable/enable_channel and
  preempt_tsg APIs have been moved from GPU-global to per-runlist
  paths on Ampere+.

Bug fixes:
- `preempt_runlist()` is now supported on Maxwell and Pascal.
- `resubmit_runlist()` detects too-old GPUs.
- MAX_CHID corrected from 512 to 511 and documented.
- switch_to_tsg now includes a runlist resubmit, which appears to be
  necessary on Turing+ GPUs.

Tested on GK104 (Quadro K5000), GM204 (GTX 970), GP106 (GTX 1060 3GB),
GP104 (GTX 1080 Ti), GP10B (Jetson TX2), GV11B (Jetson Xavier), GV100
(Titan V), TU102 (RTX 2080 Ti), and AD102 (RTX 6000 Ada).

1 files changed, 31 insertions, 6 deletions

diff --git a/nvdebug.h b/nvdebug.h
index 26689d9..26167a7 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -172,7 +172,8 @@ enum PREEMPT_TYPE {PREEMPT_TYPE_CHANNEL = 0, PREEMPT_TYPE_TSG = 1};
   IS_PENDING  : Is a context switch pending? (read-only)
   TYPE        : PREEMPT_TYPE_CHANNEL or PREEMPT_TYPE_TSG
 
-  Support: Kepler, Maxwell, Pascal, Volta, Turing
+  Support: Fermi*, Kepler, Maxwell, Pascal, Volta, Turing
+  *Fermi only supports PREEMPT_TYPE_CHANNEL.
 */
 #define NV_PFIFO_PREEMPT 0x00002634
 typedef union {
@@ -187,10 +188,26 @@ typedef union {
         uint32_t raw;
 } pfifo_preempt_t;
 
+/* Preempt a TSG or Runlist by ID
+  Similar as on older GPUs (see above), but located at an offset in Runlist RAM.
+  This means that there's one instance of this register for each runlist.
+
+  IS_PENDING is now IS_TSG_PREEMPT_PENDING and IS_RUNLIST_PREEMPT_PENDING was
+  added in the following bit (bit 22). As these fields are unused in nvdebug,
+  we use the old structure for simplicity.
+
+  TYPE is now better described as IS_TSG_PREEMPT. TYPE == 0 requests a preempt
+  of the runlist (rather than a channel preemption, as on older GPUs).
+
+  Support: Ampere, Hopper, Ada, [newer untested]
+*/
+#define NV_RUNLIST_PREEMPT_GA100 0x098
+#define PREEMPT_TYPE_RUNLIST 0
+
 /*
   "Initiate a preempt of the engine by writing the bit associated with its
   runlist to NV_PFIFO_RUNLIST_PREEMPT...  Do not poll NV_PFIFO_RUNLIST_PREEMPT
-  for the preempt to complete."
+  for the preempt to complete." (open-gpu-doc)
 
   Useful for preempting multiple runlists at once.
 
@@ -204,7 +221,10 @@ typedef union {
     rl_preempt.raw |= BIT(nr);
     nvdebug_writel(g, NV_PFIFO_RUNLIST_PREEMPT, rl_preempt.raw);
 
-  Support: Volta, Turing
+  Support: Maxwell, Pascal, Volta, Turing
+
+  This register was deleted starting with Ampere, with functionality subsumed by
+  the NV_RUNLIST_PREEMPT register.
 */
 #define NV_PFIFO_RUNLIST_PREEMPT 0x00002638
 typedef union {
@@ -285,7 +305,7 @@ typedef union {
   TARGET      : Aperture of runlist (video or system memory)
 
   Support: Fermi*, Kepler, Maxwell, Pascal, Volta
-  *Fermi may expose this information 8 bytes earlier, starting at 0x227C?
+  *Fermi may expose ENG_RUNLING_* 8 bytes earlier, starting at 0x227C?
 */
 #define NV_PFIFO_RUNLIST_BASE_GF100 0x00002270 // Write-only
 #define NV_PFIFO_ENG_RUNLIST_BASE_GF100(i) (0x00002280+(i)*8) // Read-only
@@ -428,7 +448,11 @@ typedef union {
   See also: manuals/turing/tu104/dev_fifo.ref.txt in NVIDIA's open-gpu-doc
 */
 #define NV_PCCSR_CHANNEL_INST(i) (0x00800000+(i)*8)
-#define MAX_CHID 512
+// Maximum valid channel index in the PCCSR region
+// Channel IDs start at 0, and there are 4096 bytes of 8-byte CCSR entries (per
+// NV_PCCSR_CHANNEL_INST__SIZE_1 in at least Volta and Turing), yielding a total
+// of 512 channel IDs, with a maximum ID of 511.
+#define MAX_CHID 511
 typedef union {
         struct {
 // 0:31
@@ -554,6 +578,7 @@ typedef union {
 #define NV_CHIP_ID_GP106 0x136 // Discrete GeForce GTX 1060
 #define NV_CHIP_ID_GV11B 0x15B // Jetson Xavier embedded GPU
 
+#define NV_CHIP_ID_FERMI 0x0C0
 #define NV_CHIP_ID_KEPLER 0x0E0
 #define NV_CHIP_ID_MAXWELL 0x120
 #define NV_CHIP_ID_PASCAL 0x130
@@ -1505,7 +1530,7 @@ int get_runlist_iter(
         struct nvdebug_state *g,
         int rl_id,
         struct runlist_iter *rl_iter /* out */);
-int preempt_tsg(struct nvdebug_state *g, uint32_t tsg_id);
+int preempt_tsg(struct nvdebug_state *g, uint32_t rl_id, uint32_t tsg_id);
 int preempt_runlist(struct nvdebug_state *g, uint32_t rl_id);
 int resubmit_runlist(struct nvdebug_state *g, uint32_t rl_id);