#include // For seq_* functions and types #include // Macros to detect kernel version #include "nvdebug_linux.h" // Uncomment to expand channel status information when printing the runlist #define DETAILED_CHANNEL_INFO #ifdef DETAILED_CHANNEL_INFO /* Print channel details using PCCSR (Programmable Channel Control System RAM?) @param s Pointer to state from seq_file subsystem to pass to seq_printf @param g Pointer to our internal GPU state @param chid ID of channel to print details on, range [0, 512) @param prefix Text string to prefix each line with, or empty string */ static int runlist_detail_seq_show_chan(struct seq_file *s, struct nvdebug_state *g, uint32_t chid, char *prefix) { channel_ctrl_t chan; uint64_t instance_ptr; if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chid))) == -1) return -EIO; instance_ptr = (uint64_t)chan.inst_ptr << 12; // Don't print write-only fields seq_printf(s, "%s|= Channel Info ======|\n", prefix); seq_printf(s, "%s| Enabled: %d|\n", prefix, chan.enable); seq_printf(s, "%s| Next: %d|\n", prefix, chan.next); seq_printf(s, "%s| PBDMA Faulted: %d|\n", prefix, chan.pbdma_faulted); seq_printf(s, "%s| ENG Faulted: %d|\n", prefix, chan.eng_faulted); seq_printf(s, "%s| Status: %2d|\n", prefix, chan.status); seq_printf(s, "%s| Busy: %d|\n", prefix, chan.busy); seq_printf(s, "%s| Instance PTR: |\n", prefix); seq_printf(s, "%s| %#018llx|\n", prefix, instance_ptr); seq_printf(s, "%s| %20s|\n", prefix, target_to_text(chan.inst_target)); seq_printf(s, "%s| Instance bound: %d|\n", prefix, chan.inst_bind); return 0; } /* `runlist_detail_seq_show_chan()`, but for Ampere+ @param runlist_pri_base Base of the RLRAM region for this runlist `runlist_pri_base` is necessary, since Channel RAM is now per-runlist on Ampere+, and its location is configured in Runlist RAM. */ static int runlist_detail_seq_show_chan_ga100(struct seq_file *s, struct nvdebug_state *g, uint32_t chid, char *prefix, uint32_t runlist_pri_base) { runlist_channel_config_t channel_config; channel_ctrl_ga100_t chan; // Channel RAM is subsidiary to Runlist RAM (ie. per-runlist) on Ampere+ if ((channel_config.raw = nvdebug_readl(g, runlist_pri_base + NV_RUNLIST_CHANNEL_CONFIG_GA100)) == -1) return -EIO; if ((chan.raw = nvdebug_readl(g, (((uint32_t)channel_config.bar0_offset << 4) + chid * 4))) == -1) return -EIO; seq_printf(s, "%s|= Channel Info ======|\n", prefix); seq_printf(s, "%s| Enabled: %d|\n", prefix, chan.enable); seq_printf(s, "%s| Next: %d|\n", prefix, chan.next); seq_printf(s, "%s| Busy: %d|\n", prefix, chan.busy); seq_printf(s, "%s| PBDMA Faulted: %d|\n", prefix, chan.pbdma_faulted); seq_printf(s, "%s| ENG Faulted: %d|\n", prefix, chan.eng_faulted); seq_printf(s, "%s| On PBDMA: %d|\n", prefix, chan.on_pbdma); seq_printf(s, "%s| On ENG: %d|\n", prefix, chan.on_eng); seq_printf(s, "%s| Pending: %d|\n", prefix, chan.pending); seq_printf(s, "%s| CTX Reload: %d|\n", prefix, chan.ctx_reload); seq_printf(s, "%s| PBDMA Busy: %d|\n", prefix, chan.pbdma_busy); seq_printf(s, "%s| ENG Busy: %d|\n", prefix, chan.eng_busy); seq_printf(s, "%s| Acquire Fail: %d|\n", prefix, chan.acquire_fail); return 0; } #endif #if LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0) // Bug workaround. See comment in runlist_file_seq_start() static loff_t pos_fixup; #endif static void *runlist_file_seq_start(struct seq_file *s, loff_t *pos) { static struct runlist_iter rl_iter; struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(s->file)]; // *pos == 0 for first call after read of file if (*pos == 0) { int err = get_runlist_iter(g, seq2gpuidx(s), &rl_iter); if (err) return ERR_PTR(err); // Don't try to print an empty runlist if (rl_iter.len <= 0) return NULL; return &rl_iter; } // If we're resuming an earlier print if (*pos < rl_iter.len) { #if LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0) // There's a nasty bug prior to 4.19-rc1 that if the buffer overflows, the // last update to `pos` is not saved. Work around that here by reloading a // saved copy of `pos`. if (!pos_fixup) return NULL; *pos = pos_fixup; #endif return &rl_iter; } // When called with *pos != 0, we already traversed the runlist return NULL; } static void* runlist_file_seq_next(struct seq_file *s, void *raw_rl_iter, loff_t *pos) { struct runlist_iter* rl_iter = raw_rl_iter; void *ret = NULL; struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(s->file)]; // Advance by one TSG or channel (*pos)++; rl_iter->curr_entry += NV_RL_ENTRY_SIZE(g); // Verify we haven't reached the end of the runlist // len is the num of tsg entries + total num of channel entries if (*pos < rl_iter->len) { ret = rl_iter; } #if LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0) // Bug workaround. See comment in runlist_file_seq_start() pos_fixup = ret ? *pos : 0; #endif if (rl_iter->entries_left_in_tsg) rl_iter->entries_left_in_tsg--; return ret; } static void runlist_file_seq_stop(struct seq_file *s, void *raw_rl_iter) { // No cleanup needed } // _show() must be idempotent. This function will be rerun if the seq_printf // buffer was too small. static int runlist_file_seq_show(struct seq_file *s, void *raw_rl_iter) { struct runlist_iter *rl_iter = raw_rl_iter; void *entry = rl_iter->curr_entry; struct nvdebug_state *g = &g_nvdebug_state[file2parentgpuidx(s->file)]; if (entry_type(g, entry) == ENTRY_TYPE_TSG) { if (rl_iter->entries_left_in_tsg) { printk(KERN_WARNING "[nvdebug] Found TSG ID%d @ %px when %d channels were still expected under the previous TSG in the runlist!\n", tsgid(g, entry), entry, rl_iter->entries_left_in_tsg); while (rl_iter->entries_left_in_tsg--) seq_printf(s, "[missing channel]\n"); } rl_iter->entries_left_in_tsg = tsg_length(g, entry) + 1; seq_printf(s, "+---- TSG Entry %-3d---+\n", tsgid(g, entry)); seq_printf(s, "| Scale: %-13d|\n", timeslice_scale(g, entry)); seq_printf(s, "| Timeout: %-11d|\n", timeslice_timeout(g, entry)); seq_printf(s, "| Length: %-12d|\n", tsg_length(g, entry)); seq_printf(s, "+---------------------+\n"); } else { char *indt = ""; u64 instance_ptr = 0; if (rl_iter->entries_left_in_tsg) indt = " "; // Reconstruct pointer to channel instance block if (g->chip_id >= NV_CHIP_ID_VOLTA) { instance_ptr = ((struct gv100_runlist_chan*)entry)->inst_ptr_hi; instance_ptr <<= 32; } instance_ptr |= inst_ptr_lo(g, entry) << 12; // Print channel information from runlist seq_printf(s, "%s+- Channel Entry %-4d-+\n", indt, chid(g, entry)); if (g->chip_id >= NV_CHIP_ID_VOLTA) seq_printf(s, "%s| Runqueue Selector: %d|\n", indt, ((struct gv100_runlist_chan*)entry)->runqueue_selector); // Not populated on Kepler [ex: gk104 in Bonham (Quadro K5000)], and // populated but unused on Pascal [ex: gp104 in Bonham (GTX 1080 Ti)]. // (The aperture field may be incorrectly populated as INVALID, but the // context still works on the aformentioned Pascal GPU.) seq_printf(s, "%s| Instance PTR: |\n", indt); seq_printf(s, "%s| %#018llx|\n", indt, instance_ptr); seq_printf(s, "%s| %20s|\n", indt, target_to_text(inst_target(g, entry))); #ifdef DETAILED_CHANNEL_INFO // Print channel info from PCCSR/Channel RAM and the instance block if (g->chip_id < NV_CHIP_ID_AMPERE) runlist_detail_seq_show_chan(s, g, chid(g, entry), indt); else runlist_detail_seq_show_chan_ga100(s, g, chid(g, entry), indt, rl_iter->runlist_pri_base); #endif seq_printf(s, "%s+---------------------+\n", indt); } return 0; } static const struct seq_operations runlist_file_seq_ops = { .start = runlist_file_seq_start, .next = runlist_file_seq_next, .stop = runlist_file_seq_stop, .show = runlist_file_seq_show, }; static int runlist_file_open(struct inode *inode, struct file *f) { return seq_open(f, &runlist_file_seq_ops); } struct file_operations runlist_file_ops = { .open = runlist_file_open, .read = seq_read, .llseek = seq_lseek, .release = seq_release, }; ssize_t preempt_tsg_file_write(struct file *f, const char __user *buffer, size_t count, loff_t *off) { uint32_t target_tsgid; // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec int err = kstrtou32_from_user(buffer, count, 0, &target_tsgid); struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)]; if (err) return err; // TSG IDs are a 12-bit field, so make sure the request is in-range if (target_tsgid > MAX_TSGID) return -ERANGE; // Execute preemption if ((err = preempt_tsg(g, target_tsgid))) return err; return count; } struct file_operations preempt_tsg_file_ops = { .write = preempt_tsg_file_write, .llseek = default_llseek, }; ssize_t resubmit_runlist_file_write(struct file *f, const char __user *buffer, size_t count, loff_t *off) { uint32_t target_runlist; // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec int err = kstrtou32_from_user(buffer, count, 0, &target_runlist); struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)]; if (err) return err; // resubmit_runlist() checks that target_runlist is valid if ((err = resubmit_runlist(g, target_runlist))) return err; return count; } struct file_operations resubmit_runlist_file_ops = { .write = resubmit_runlist_file_write, .llseek = default_llseek, }; ssize_t disable_channel_file_write(struct file *f, const char __user *buffer, size_t count, loff_t *off) { uint32_t target_channel; channel_ctrl_t chan; int err; struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)]; // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec err = kstrtou32_from_user(buffer, count, 0, &target_channel); if (err) return err; if (target_channel > MAX_CHID) return -ERANGE; // Read current configuration if ((chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel))) == -1) return -EIO; // Request disablement chan.enable_clear = true; nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw); return count; } struct file_operations disable_channel_file_ops = { .write = disable_channel_file_write, .llseek = default_llseek, }; ssize_t enable_channel_file_write(struct file *f, const char __user *buffer, size_t count, loff_t *off) { uint32_t target_channel; channel_ctrl_t chan; int err; struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)]; // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec err = kstrtou32_from_user(buffer, count, 0, &target_channel); if (err) return err; if (target_channel > MAX_CHID) return -ERANGE; // Disable channel chan.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(target_channel)); chan.enable_set = true; nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(target_channel), chan.raw); return count; } struct file_operations enable_channel_file_ops = { .write = enable_channel_file_write, .llseek = default_llseek, }; // Note: Operates only on runlist 0 (Compute/Graphics) ssize_t switch_to_tsg_file_write(struct file *f, const char __user *buffer, size_t count, loff_t *off) { uint32_t target_tsgid; struct gv100_runlist_chan* chan; channel_ctrl_t chan_ctl; struct runlist_iter rl_iter; int err; loff_t pos = 0; struct nvdebug_state *g = &g_nvdebug_state[file2gpuidx(f)]; // Passing 0 as the base to kstrtou32 indicates autodetect hex/octal/dec err = kstrtou32_from_user(buffer, count, 0, &target_tsgid); if (err) return err; if (target_tsgid > MAX_TSGID) return -ERANGE; err = get_runlist_iter(g, 0, &rl_iter); if (err) return err; // Iterate through all TSGs while (pos < rl_iter.len) { if (tsgid(g, rl_iter.curr_entry) == target_tsgid) { // Enable channels of target TSG for_chan_in_tsg(g, chan, rl_iter.curr_entry) { chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid)); chan_ctl.enable_set = true; nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw); } } else { // XXX: Fix for bare channels. Maybe a "for_chan_until_tsg" macro? // Disable all other channels // (This is how the Jetson nvgpu driver disables TSGs) for_chan_in_tsg(g, chan, rl_iter.curr_entry) { chan_ctl.raw = nvdebug_readq(g, NV_PCCSR_CHANNEL_INST(chan->chid)); chan_ctl.enable_clear = true; nvdebug_writeq(g, NV_PCCSR_CHANNEL_INST(chan->chid), chan_ctl.raw); } } pos += 1 + tsg_length(g, rl_iter.curr_entry); rl_iter.curr_entry = next_tsg(g, rl_iter.curr_entry); } // Trigger a runlist-level preempt to switch to `target_tsgid` if ((err = preempt_runlist(g, 0))) return err; return count; } struct file_operations switch_to_tsg_file_ops = { .write = switch_to_tsg_file_write, .llseek = default_llseek, };