Support printing device info on Ampere+ GPUs. By Benjamin Hadad IV

commit c3d6f2c852eb046e9d4f4f1e6527b52c746b2693
Author: Joshua Bakita <bakitajoshua@gmail.com>
Date:   Sun Oct 29 14:37:51 2023 -0400

    Print Ampere+ device_info fields with correct offsets/widths

    Everything now has been checked against how nvgpu handles it

commit b70849d1ce67a58f9f69b37dc62122f789f4cdf7
Author: Joshua Bakita <jbakita@cs.unc.edu>
Date:   Wed Sep 20 14:27:38 2023 -0400

    Rearrange, fix an off-by-one error, and remove an unused define

    The code in nvdebug.h has been rearranged to enable an easier merge
    against the jbakita-wip branch.

commit 51f808e092846a60ea6c88ea3a1d2e349c92977b
Author: Joshua Bakita <jbakita@cs.unc.edu>
Date:   Wed Sep 20 13:09:17 2023 -0400

    Bug fixes and cleanup for new device_info logic

    - Update comments to match new structure
    - Make show() function idempotent
    - Skip empty table entries without aborting
    - Include names for new engine types
    - Add warning log messages for skipped table entries
    - Remove non-functional runlist file creation logic for Ampere+

commit 1d7adc3be1aef5ac9c144bb24008fd8cc5d688a5
Author: Benjamin Hadad IV <bh4@unc.edu>
Date:   Sat Aug 19 12:47:18 2023 -0400

    Debugging changes made to restore functionality following refactoring.
     - Debugged data display errors.
     - Debugged crash bugs.
     - Debugged memory issue.

commit 9e6cc03cdf736fbd817ed53fa9a7f506bc91a244
Author: Benjamin Hadad IV <bh4@unc.edu>
Date:   Wed Aug 16 22:00:20 2023 -0400

    A variety of changes have been made as part of the code review.
     - Functions have been consolidated.
     - Code was clarified and tidied up overall.
     - Unnecessary elements were removed.

commit 845960fc1b15995fdbd6d61c384567652a150bc4
Author: Benjamin Hadad IV <bh4@unc.edu>
Date:   Fri Jul 28 11:39:28 2023 -0400

    Refactored various systems and debugged minor issues
     - Added device_info_iter
     - Merged functions in device_info_procfs.c
     - Separated device_info data structs by version in nvdebug.h
     - Fixed issue with device_info runlist ID data

commit 8a57aaeba41c43233c323d7e0fc8bf1a81ebc65e
Author: Benjamin Hadad IV <bh4@unc.edu>
Date:   Fri Jul 21 11:32:51 2023 -0400

    I have updated the ptop_device_info_t comment in nvdebug.h.

commit 33c915f08f5dc63674b158ecc18897494256a6d0
Author: Benjamin Hadad IV <bh4@unc.edu>
Date:   Wed Jul 19 13:02:52 2023 -0400

    Debugged device_info functionality
     - Fixed device_info crash bugs
     - Made further edits to display functionality
     - Refactored code to enhance readability

commit bfb4dcf0e78954c0163f3a06a5a088c4d1b437a8
Author: Benjamin Hadad IV <bh4@unc.edu>
Date:   Thu Jul 13 12:13:17 2023 -0400

    This commit is to update the repo for display during a meeting.
    - Added an Ampere version of the device info data.
    - Added Ampere versions of auxillary functions.
    - Modified display functions to accommodate Ampere data.
    - Made other various small modifications.

commit 068e7f4e7208d6c9250ad72208e0b36fd9fdf2f6
Merge: 3725b15 073e897
Author: Benjamin Hadad IV <bh4@unc.edu>
Date:   Mon Jul 10 12:39:12 2023 -0400

    Merge branch 'jbakita-wip' of ssh://rtsrv.cs.unc.edu/public/nvdebug into wip
    I am merging Mr. Bakita's changes (046d7d2) into this repository.

commit 3725b15d5da3e06ef202045d710aa5f15eb72fcc
Author: Benjamin Hadad IV <bh4@unc.edu>
Date:   Mon Jul 3 04:30:54 2023 -0400

    I modified nvdebug.h for Ampere.

3 files changed, 167 insertions, 35 deletions

diff --git a/device_info_procfs.c b/device_info_procfs.c
index 5fc417f..b139c36 100644
--- a/device_info_procfs.c
+++ b/device_info_procfs.c
@@ -27,38 +27,65 @@ struct file_operations nvdebug_read_reg32_file_ops = {
         .llseek = default_llseek,
 };
 
+typedef struct {
+        int idx; // Current index in the device_info table
+        int length; // Length of device_info table (including unpopulated entries)
+        int type_of_next_entry; // Only used on Ampere+ GPUs
+        bool has_next_entry; // Only used on Ampere+ GPUs for show() idempotence
+} device_info_iter;
+
 //// ==v== PTOP_DEVICE_INFO ==v== ////
 
 // Called to start or resume a sequence. Prior to 4.19, *pos is unreliable.
-// Initializes iterator `idx` state and returns it. Ends sequence on NULL.
+// Initializes iterator `iter` state and returns it. Ends sequence on NULL.
 static void* device_info_file_seq_start(struct seq_file *s, loff_t *pos) {
-        static int idx;
-        // If start of sequence, reset `idx`
-        if (*pos == 0)
-                idx = 0;
-        // Number of possible info entries is fixed, and list is sparse
-        if (idx >= NV_PTOP_DEVICE_INFO__SIZE_1)
+        static device_info_iter iter;
+        // If freshly starting a sequence, reset the iterator
+        if (*pos == 0) {
+                struct nvdebug_state *g = &g_nvdebug_state[seq2gpuidx(s)];
+                iter.idx = 0;
+                iter.type_of_next_entry = 0;
+                iter.has_next_entry = 0;
+                // On Ampere+, the device_info table length can vary
+                if (g->chip_id >= NV_CHIP_ID_AMPERE)
+                        iter.length = NV_PTOP_DEVICE_INFO__SIZE_1_GA100(g);
+                else
+                        iter.length = NV_PTOP_DEVICE_INFO__SIZE_1_GK104;
+        }
+        // Number of possible info entries is fixed, and list is sparse, so stop
+        // iterating only when all entries have been checked, rather than on the first
+        // empty entry.
+        if (iter.idx >= iter.length)
                 return NULL;
-        return &idx;
+        return &iter;
 }
 
-// Steps to next record. Returns new value of `idx`.
+// Steps to next record. Returns `&iter` (address should not change)
 // Calls show() on non-NULL return
-static void* device_info_file_seq_next(struct seq_file *s, void *idx,
+static void* device_info_file_seq_next(struct seq_file *s, void *iter_raw,
                                        loff_t *pos) {
+        device_info_iter *iter = (device_info_iter*)iter_raw;
         (*pos)++; // Required by seq interface
-        // Number of possible info entries is fixed, and list is sparse
-        if ((*(int*)idx)++ >= NV_PTOP_DEVICE_INFO__SIZE_1)
+        // Number of possible info entries is fixed, and list is sparse, so stop
+        // iterating only when all entries have been checked, rather than on the first
+        // empty entry.
+        if (++iter->idx >= iter->length)
                 return NULL;
-        return idx;
+        // The info_type field is not available in the Ampere device_info data, so
+        // it must be inferred. NOP for older devices (cheaper than another branch).
+        // This has to be here (rather than in the show function) to support the
+        // idempotence requirements of show() in the seq_file interface.
+        iter->type_of_next_entry = iter->has_next_entry ? iter->type_of_next_entry + 1 : 0;
+        return iter;
 }
 
-// Print info at index *idx. Returns non-zero on error.
-static int device_info_file_seq_show(struct seq_file *s, void *idx) {
-        ptop_device_info_t curr_info;
+// Print info at iter->idx for Kepler--Turing GPUs. Returns non-zero on error.
+// Implementation of this function must be idempotent
+static int device_info_file_seq_show_gk104(struct seq_file *s, void *iter_raw) {
+        device_info_iter *iter = (device_info_iter*)iter_raw;
+        ptop_device_info_gk104_t curr_info;
         struct nvdebug_state *g = &g_nvdebug_state[seq2gpuidx(s)];
-
-        curr_info.raw = nvdebug_readl(g, NV_PTOP_DEVICE_INFO(*(int*)idx));
+        curr_info.raw = nvdebug_readl(g, NV_PTOP_DEVICE_INFO_GK104(iter->idx));
         // Check for read errors
         if (curr_info.raw == -1)
                 return -EIO;
@@ -90,7 +117,7 @@ static int device_info_file_seq_show(struct seq_file *s, void *idx) {
                 if (curr_info.engine_type < ENGINE_TYPES_LEN)
                         seq_printf(s, "%s)\n", ENGINE_TYPES_NAMES[curr_info.engine_type]);
                 else
-                        seq_printf(s, "Unknown Engine, introduced post-Ampere)\n");
+                        seq_printf(s, "Unknown Historical)\n");
                 break;
         case INFO_TYPE_NOT_VALID:
         default:
@@ -104,18 +131,81 @@ static int device_info_file_seq_show(struct seq_file *s, void *idx) {
         return 0;
 }
 
+// Print info at iter->idx for Ampere+ GPUs. Returns non-zero on error.
+// Implementation of this function must be idempotent
+static int device_info_file_seq_show_ga100(struct seq_file *s, void *iter_raw) {
+        device_info_iter *iter = (device_info_iter*)iter_raw;
+        ptop_device_info_ga100_t curr_info;
+        struct nvdebug_state *g = &g_nvdebug_state[seq2gpuidx(s)];
+        curr_info.raw = nvdebug_readl(g, NV_PTOP_DEVICE_INFO_GA100(iter->idx));
+        // Check for read errors
+        if (curr_info.raw == -1)
+                return -EIO;
+
+        // Update tracking data only used by next(); allows preserving idempotence
+        iter->has_next_entry = curr_info.has_next_entry;
+        // Silently skip empty entries
+        if (curr_info.raw == 0)
+                return 0;
+        // In nvdebug, an entry is considered invalid if it does not consist of at
+        // least two rows. So, if this is the first row of an entry, but another row
+        // is not indicated, this entry is invalid and should be skipped.
+        if (iter->type_of_next_entry == 0 && !curr_info.has_next_entry) {
+                printk(KERN_WARNING "[nvdebug] Skipping seemingly-invalid device_info entry (idx: %d, raw: %#0x)\n", iter->idx, curr_info.raw);
+                return 0;
+        }
+
+        // Parse and print the data
+        // Note: The goal of this interface is to present useful information to
+        // a human user, NOT to provide a stable format for scripts to parse.
+        // Because of this, we favor accurately printing the data in each entry,
+        // rather than providing stable (if imperfectly correct) field names
+        switch(iter->type_of_next_entry) {
+        case 0:
+                seq_printf(s, "| Engine Type:     %3d (", curr_info.engine_type);
+                if (curr_info.engine_type < ENGINE_TYPES_LEN)
+                        seq_printf(s, "%s)\n", ENGINE_TYPES_NAMES[curr_info.engine_type]);
+                else
+                        seq_printf(s, "Unknown, introduced post-Lovelace)\n");
+                seq_printf(s, "|           instance %d\n", curr_info.inst_id);
+                seq_printf(s, "| Fault ID:       %4d\n", curr_info.fault_id);
+                break;
+        case 1:
+                seq_printf(s, "| BAR0 Base %#.8x\n", curr_info.pri_base << 8);
+                seq_printf(s, "| Reset ID:        %3d\n", curr_info.reset_id);
+                seq_printf(s, "| Is Engine:         %1d\n", curr_info.is_engine);
+
+                break;
+        case 2:
+                seq_printf(s, "| Runlist Eng. ID:   %1d\n", curr_info.rleng_id);
+                // Theoretically, we could extract an ID from the runlist RAM
+                seq_printf(s, "| RL Base:  %#.8x\n", curr_info.runlist_pri_base << 10);
+                break;
+        default:
+                printk(KERN_WARNING "[nvdebug] Skipping unexpected continuation of device_info entry (idx: %d, raw: %#0x)\n", iter->idx, curr_info.raw);
+        }
+
+        // Draw a line between each device entry
+        if (!curr_info.has_next_entry)
+                seq_printf(s, "+---------------------+\n");
+        return 0;
+}
+
 static void device_info_file_seq_stop(struct seq_file *s, void *idx) {
-        // No cleanup needed
+        // No cleanup needed
 }
 
-static const struct seq_operations device_info_file_seq_ops = {
+static struct seq_operations device_info_file_seq_ops = {
         .start = device_info_file_seq_start,
         .next = device_info_file_seq_next,
         .stop = device_info_file_seq_stop,
-        .show = device_info_file_seq_show,
 };
 
 static int device_info_file_open(struct inode *inode, struct file *f) {
+        if (g_nvdebug_state[file2parentgpuidx(f)].chip_id >= NV_CHIP_ID_AMPERE)
+                device_info_file_seq_ops.show = device_info_file_seq_show_ga100;
+        else
+                device_info_file_seq_ops.show = device_info_file_seq_show_gk104;
         return seq_open(f, &device_info_file_seq_ops);
 }
 
diff --git a/nvdebug.h b/nvdebug.h
index 3860c2e..b79ede1 100644
--- a/nvdebug.h
+++ b/nvdebug.h
@@ -390,6 +390,7 @@ typedef union {
 #define NV_CHIP_ID_GV11B 0x15B // Jetson Xavier embedded GPU
 #define NV_CHIP_ID_KEPLER 0x0E0