1 files changed, 636 insertions, 9 deletions

diff --git a/runlist_procfs.c b/runlist_procfs.c
index b2159f6..a3a6df3 100644
--- a/runlist_procfs.c
+++ b/runlist_procfs.c
@@ -1,12 +1,117 @@
 #include <linux/seq_file.h> // For seq_* functions and types
 #include <linux/version.h>  // Macros to detect kernel version
+#include <linux/platform_device.h> // For platform_get_resource()
+#include <linux/pci.h> // For pci_resource_start()
+#include <linux/iommu.h> // For iommu_ functions
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,10,0)
+#include <linux/dma-map-ops.h> // For get_dma_ops()
+#endif
 
 #include "nvdebug_linux.h"
 
-// Uncomment to expand channel status information when printing the runlist
+// We cannot touch PRAMIN (via page table operations or ctxsw access) if we're
+// using it to walk the runlist
+//#ifndef FALLBACK_TO_PRAMIN
+// Uncomment to expand channel status, instance, and context information when
+// printing the runlist
 #define DETAILED_CHANNEL_INFO
+//#endif
 
 #ifdef DETAILED_CHANNEL_INFO
+// Print the channel instance and context swtich blocks
+// XXX: THIS IS UNSAFE ON KEPLER!
+// instance_deref() will call into the page table logic, which may move PRAMIN
+// PRAMIN appears heavily utilized by the driver on Bonham (at least), and
+// moving it causes problems.
+static int runlist_detail_seq_show_inst(struct seq_file *s, struct nvdebug_state *g, char *prefix, uint64_t instance_ptr, enum INST_TARGET instance_target) {
+        instance_ctrl_t *inst = NULL;
+        context_switch_ctrl_t *ctxsw = NULL;
+        int i;
+
+#ifdef FALLBACK_TO_PRAMIN
+        bar0_window_t win;
+        win.raw = nvdebug_readl(g, NV_XAL_EP_BAR0_WINDOW_BASE);
+        inst = g->regs + NV_PRAMIN + addr_to_pramin_mut(g, instance_ptr, instance_target);
+#else
+        if (IS_ERR(inst = instance_deref(g, instance_ptr, instance_target)))
+                return PTR_ERR(ctxsw);
+#endif // FALLBACK_TO_PRAMIN
+        // If unable to access instance block, skip
+        if (!inst)
+                return 0;
+
+        // Print the channel instance block
+        // As an ID, use upper 52 bits of the instance address (lower 12 are zero)
+        //seq_printf(s, "%s+- Inst %-13llx-+\n", prefix, instance_ptr >> 12);
+        seq_printf(s, "%s|= Instance Block ====|\n", prefix);
+        seq_printf(s, "%s| Target Engine:    %2d|\n", prefix, inst->fc_target);
+        seq_printf(s, "%s| Privileged:        %1d|\n", prefix, inst->fc_config_is_priv);
+        seq_printf(s, "%s| Channel VEID:     %2d|\n", prefix, inst->fc_chan_info_veid);
+        seq_printf(s, "%s| WFI PTR:            |\n", prefix);
+        seq_printf(s, "%s|   %#018llx|\n", prefix, (uint64_t)inst->engine_wfi_ptr << 12);
+        seq_printf(s, "%s| %20s|\n", prefix, target_to_text(inst->engine_wfi_target));
+        seq_printf(s, "%s| Virtual address?   %d|\n", prefix, inst->engine_wfi_is_virtual);
+        seq_printf(s, "%s| WFI VEID:         %2d|\n", prefix, inst->engine_wfi_veid);
+        seq_printf(s, "%s| All PDB PTR:        |\n", prefix);
+        seq_printf(s, "%s|   %#018llx|\n", prefix,  (u64)inst->pdb.page_dir << 12);
+        seq_printf(s, "%s| %20s|\n", prefix, target_to_text(inst->pdb.target));
+        seq_printf(s, "%s| %20s|\n", prefix, inst->pdb.is_volatile ? "volatile" : "non-volatile");
+//      seq_printf(s, "%s|raw:       %0#10lx|\n", prefix, inst->pdb.raw);
+        seq_printf(s, "%s| Num subcontexts:  %2ld|\n", prefix, hweight64(inst->subcontext_pdb_valid));
+        // Print configuration of every enabled subcontext
+        for (i = 0; i < 64; i++) {
+                // Skip subcontexts without their enable bit set
+                if (!(1 & (inst->subcontext_pdb_valid >> i)))
+                        continue;
+                seq_printf(s, "%s| CPU SC%02d ASID%7d|\n", prefix, i, inst->subcontext[i].pasid);
+                seq_printf(s, "%s| SC%02d PDB PTR:       |\n", prefix, i);
+                seq_printf(s, "%s|   %#018llx|\n", prefix,  ((u64)inst->subcontext[i].pdb.page_dir_hi << 32) | ((u64)inst->subcontext[i].pdb.page_dir_lo << 12));
+                seq_printf(s, "%s| %20s|\n", prefix, target_to_text(inst->subcontext[i].pdb.target));
+                seq_printf(s, "%s| %20s|\n", prefix, inst->subcontext[i].pdb.is_volatile ? "volatile" : "non-volatile");
+//              seq_printf(s, "%s|raw:       %0#10lx|\n", prefix, inst->subcontext[i].pdb.raw);
+        }
+
+        // XXX: CTXSW is only accessible via PRAMIN. Accessing PRAMIN appears to
+        // either be broken, or race with the driver on Kepler (gk104 tested). So,
+        // do not attempt to touch the CTXSW block on Kepler.
+        // TODO: This check should be moved into addr_to_pramin_mut().
+        if (g->chip_id < NV_CHIP_ID_MAXWELL)
+                return 0;
+        // End XXX
+
+        if (IS_ERR(ctxsw = get_ctxsw(g, inst))) {
+#ifdef FALLBACK_TO_PRAMIN
+                nvdebug_writel(g, NV_XAL_EP_BAR0_WINDOW_BASE, win.raw);
+#endif
+                return PTR_ERR(ctxsw);
+        }
+        // If unable to access CTXSW block, skip
+        if (!ctxsw) {
+#ifdef FALLBACK_TO_PRAMIN
+                nvdebug_writel(g, NV_XAL_EP_BAR0_WINDOW_BASE, win.raw);
+#endif
+                return 0;
+        }
+        // Access and print the preemption mode and context ID
+        seq_printf(s, "%s|= Context State =====|\n", prefix);
+        seq_printf(s, "%s| Ctx. ID:  %#10x|\n", prefix, ctxsw->context_id);
+        // No other CTXSW fields are supported pre-Pascal
+        if (g->chip_id < NV_CHIP_ID_PASCAL)
+                return 0;
+        seq_printf(s, "%s| Gfx. Preemption:%4s|\n", prefix,
+                   graphics_preempt_type_to_text(ctxsw->graphics_preemption_options));
+        seq_printf(s, "%s| Cmp. Preemption:%4s|\n", prefix,
+                   compute_preempt_type_to_text(ctxsw->compute_preemption_options));
+        seq_printf(s, "%s| #WFI Saves:%9d|\n", prefix, ctxsw->num_wfi_save_operations);
+        seq_printf(s, "%s| #CTA Saves:%9d|\n", prefix, ctxsw->num_cta_save_operations);
+        seq_printf(s, "%s| #GFXP Saves:%8d|\n", prefix, ctxsw->num_gfxp_save_operations);
+        seq_printf(s, "%s| #CILP Saves:%8d|\n", prefix, ctxsw->num_cilp_save_operations);
+#ifdef FALLBACK_TO_PRAMIN
+        nvdebug_writel(g, NV_XAL_EP_BAR0_WINDOW_BASE, win.raw);
+#endif
+        return 0;
+}
+
 /* Print channel details using PCCSR (Programmable Channel Control System RAM?)
   @param s      Pointer to state from seq_file subsystem to pass to seq_printf
   @param g      Pointer to our internal GPU state
@@ -32,16 +137,19 @@ static int runlist_detail_seq_show_chan(struct seq_file *s, struct nvdebug_state
         seq_printf(s, "%s|   %#018llx|\n", prefix, instance_ptr);
         seq_printf(s, "%s| %20s|\n", prefix, target_to_text(chan.inst_target));
         seq_printf(s, "%s| Instance bound:    %d|\n", prefix, chan.inst_bind);
-        return 0;
+        // Print instance block
+        return runlist_detail_seq_show_inst(s, g, prefix, instance_ptr, chan.inst_target);
 }
 
 /* `runlist_detail_seq_show_chan()`, but for Ampere+
+  @param instance_ptr     Address for the channel instance block
+  @param instance_target  Aperture of `instance_ptr`
   @param runlist_pri_base Base of the RLRAM region for this runlist
 
   `runlist_pri_base` is necessary, since Channel RAM is now per-runlist on
   Ampere+, and its location is configured in Runlist RAM.
 */
-static int runlist_detail_seq_show_chan_ga100(struct seq_file *s, struct nvdebug_state *g, uint32_t chid, char *prefix, uint32_t runlist_pri_base) {
+static int runlist_detail_seq_show_chan_ga100(struct seq_file *s, struct nvdebug_state *g, uint32_t chid, char *prefix, uint32_t runlist_pri_base, uint64_t instance_ptr, enum INST_TARGET instance_target) {
         runlist_channel_config_t channel_config;
         channel_ctrl_ga100_t chan;
 
@@ -63,7 +171,7 @@ static int runlist_detail_seq_show_chan_ga100(struct seq_file *s, struct nvdebug
         seq_printf(s, "%s| PBDMA Busy:        %d|\n", prefix, chan.pbdma_busy);
         seq_printf(s, "%s| ENG Busy:          %d|\n", prefix, chan.eng_busy);
         seq_printf(s, "%s| Acquire Fail:      %d|\n", prefix, chan.acquire_fail);
-        return 0;
+        return runlist_detail_seq_show_inst(s, g, prefix, instance_ptr, instance_target);
 }
 #endif
 
@@ -173,7 +281,7 @@ static int runlist_file_seq_show(struct seq_file *s, void *raw_rl_iter) {
                 if (g->chip_id < NV_CHIP_ID_AMPERE)
                         runlist_detail_seq_show_chan(s, g, chid(g, entry), indt);
                 else
-                        runlist_detail_seq_show_chan_ga100(s, g, chid(g, entry), indt, rl_iter->runlist_pri_base);
+                        runlist_detail_seq_show_chan_ga100(s, g, chid(g, entry), indt, rl_iter->runlist_pri_base, instance_ptr, inst_target(g, entry));
 #endif
                 seq_printf(s, "%s+---------------------+\n", indt);
         }
@@ -232,15 +340,17 @@ struct file_operations preempt_tsg_file_ops = {
 
 ssize_t resubmit_runlist_file_write(struct file *f, const char __user *buffer,
                                     size_t count, loff_t *off) {
-        uint32_t target_runlist;
+        uint32_t target_runlist, target_offset;
         struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)];
         // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
-        int err = kstrtou32_from_user(buffer, count, 0, &target_runlist);
+        int err = kstrtou32_from_user(buffer, count, 0, &target_offset);
         if (err)
                 return err;
+        // (Ab)use the PDE_DATA field for the runlist ID
+        target_runlist = file2gpuidx(f);
 
         // resubmit_runlist() checks that target_runlist is valid
-        if ((err = resubmit_runlist(g, target_runlist)))
+        if ((err = resubmit_runlist(g, target_runlist, target_offset)))
                 return err;
 
         return count;
@@ -351,6 +461,54 @@ struct file_operations enable_channel_file_ops = {
         .llseek = default_llseek,
 };
 
+ssize_t comm_preempt_channel_file_write(struct file *f, const char __user *buf,
+                                        size_t count, loff_t *off,
+                                        enum COMPUTE_PREEMPT_TYPE mode) {
+        uint32_t target_channel, target_runlist;
+        struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(f)];
+        // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec
+        int err = kstrtou32_from_user(buf, count, 0, &target_channel);
+        if (err)
+                return err;
+        // (Ab)use the PDE_DATA field used by file2gpuidx() for the runlist ID
+        target_runlist = file2gpuidx(f);
+        // Set preemption mode for the context of this channel