/* Copyright 2021 Joshua Bakita * SPDX-License-Identifier: MIT */ // TODO(jbakita): Don't depend on these. #include // For struct gk20a #include // For struct nvgpu_os_linux /* Runlist Channel A timeslice group (TSG) is composed of channels. Each channel is a FIFO queue of GPU commands. These commands are typically queued from userspace. `INST_PTR` points to a GPU Instance Block which contains pointers to the GPU virtual address space for this context. All channels in a TSG point to the same GPU Instance Block. ENTRY_TYPE (T) : type of this entry: ENTRY_TYPE_CHAN CHID (ID) : identifier of the channel to run (overlays ENTRY_ID) RUNQUEUE_SELECTOR (Q) : selects which PBDMA should run this channel if more than one PBDMA is supported by the runlist INST_PTR_LO : lower 20 bits of the 4k-aligned instance block pointer INST_PTR_HI : upper 32 bit of instance block pointer INST_TARGET (TGI) : aperture of the instance block USERD_PTR_LO : upper 24 bits of the low 32 bits, of the 512-byte-aligned USERD pointer USERD_PTR_HI : upper 32 bits of USERD pointer USERD_TARGET (TGU) : aperture of the USERD data structure */ enum ENTRY_TYPE {ENTRY_TYPE_CHAN = 0, ENTRY_TYPE_TSG = 1}; enum INST_TARGET {TARGET_VID_MEM = 0, TARGET_SYS_MEM_COHERENT = 2, TARGET_SYS_MEM_NONCOHERENT = 3}; struct runlist_chan { // 0:63 enum ENTRY_TYPE entry_type:1; uint32_t runqueue_selector:1; uint32_t padding:2; enum INST_TARGET inst_target:2; uint32_t padding2:2; uint32_t userd_ptr_lo:24; uint32_t userd_ptr_hi:32; // 64:128 uint32_t chid:12; uint32_t inst_ptr_lo:20; uint32_t inst_ptr_hi:32; } __attribute__((packed)); /* Runlist TSG (TimeSlice Group) The runlist is composed of timeslice groups (TSG). Each TSG corresponds to a single virtual address space on the GPU and contains `TSG_LENGTH` channels. These channels and virtual address space are accessible to the GPU host unit for use until the timeslice expires or a TSG switch is forcibly initiated via a write to `NV_PFIFO_PREEMPT`. timeslice = (TSG_TIMESLICE_TIMEOUT << TSG_TIMESLICE_SCALE) * 1024 nanoseconds ENTRY_TYPE (T) : type of this entry: ENTRY_TYPE_TSG TSGID : identifier of the Timeslice group (overlays ENTRY_ID) TSG_LENGTH : number of channels that are part of this timeslice group TIMESLICE_SCALE : scale factor for the TSG's timeslice TIMESLICE_TIMEOUT : timeout amount for the TSG's timeslice */ struct entry_tsg { // 0:63 enum ENTRY_TYPE entry_type:1; uint64_t padding:15; uint32_t timeslice_scale:4; uint64_t padding2:4; uint32_t timeslice_timeout:8; uint32_t tsg_length:8; uint32_t padding3:24; // 64:128 uint32_t tsgid:12; uint64_t padding4:52; } __attribute__((packed)); enum PREEMPT_TYPE {PREEMPT_TYPE_CHANNEL = 0, PREEMPT_TYPE_TSG = 1}; /* Preempt ID/CHID : Id of TSG or channel to preempt */ #define NV_PFIFO_PREEMPT 0x00002634 struct pfifo_preempt { uint32_t id:12; uint32_t padding:8; bool is_pending:1; uint32_t padding2:3; enum PREEMPT_TYPE type:2; uint32_t padding3:6; } __attribute__((packed)); #define NV_PFIFO_RUNLIST_PREEMPT 0x00002638 struct runlist_preempt { bool runlist_0:1; bool runlist_1:1; bool runlist_2:1; bool runlist_3:1; bool runlist_4:1; bool runlist_5:1; bool runlist_6:1; bool runlist_7:1; bool runlist_8:1; bool runlist_9:1; bool runlist_10:1; bool runlist_11:1; bool runlist_12:1; bool runlist_13:1; uint32_t padding:28; } __attribute__((packed)); // Note: This is different with Turing #define NV_PFIFO_RUNLIST_BASE 0x00002270 typedef union { struct { uint32_t ptr:28; uint32_t type:2; uint32_t padding:2; } __attribute__((packed)); uint32_t raw; } runlist_base_t; #define NV_PFIFO_RUNLIST 0x00002274 typedef union { struct { uint32_t len:16; uint32_t padding:4; uint32_t id:4; uint32_t padding2:8; } __attribute__((packed)); uint32_t raw; } runlist_info_t; // TODO(jbakita): Maybe put the above GPU types in a different file. #define for_chan_in_tsg(chan, tsg) \ for (chan = (struct runlist_chan*)(tsg + 1); \ (void*)chan < (void*)(tsg + 1) + sizeof(struct runlist_chan) * tsg->tsg_length; \ chan++) #define next_tsg(tsg) \ (void*)(tsg + 1) + sizeof(struct runlist_chan) * tsg->tsg_length struct runlist_iter { struct entry_tsg *curr_tsg; runlist_info_t rl_info; }; // Defined in runlist.c int get_runlist_iter(struct runlist_iter *rl_iter); static inline struct gk20a *get_gk20a(struct device *dev) { // XXX: Only works because gk20a* is the first member of gk20a_platform return *((struct gk20a**)dev_get_drvdata(dev)); } // Functionally identical to nvgpu_readl() // (except we don't try to resolve situations where regs is NULL) static inline u32 nvdebug_readl(struct gk20a* g, u32 r) { struct nvgpu_os_linux* g_os = container_of(g, struct nvgpu_os_linux, g); if (unlikely(!g_os->regs)) { printk(KERN_ERR "[nvdebug] Attempted nvgpu_readl on non-existent registers!\n"); return -1; } return readl(g_os->regs + r); } // Functionally identical to nvgpu_writel() static inline void nvdebug_writel(struct gk20a* g, u32 r, u32 v) { struct nvgpu_os_linux* g_os = container_of(g, struct nvgpu_os_linux, g); if (unlikely(!g_os->regs)) { printk(KERN_ERR "[nvdebug] Attempted nvgpu_writel on non-existent registers!\n"); return; } writel_relaxed(v, g_os->regs + r); wmb(); }