1 files changed, 720 insertions, 0 deletions

diff --git a/drivers/gpu/nvgpu/common/linux/ctxsw_trace.c b/drivers/gpu/nvgpu/common/linux/ctxsw_trace.c
new file mode 100644
index 00000000..8268bf60
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/ctxsw_trace.c
@@ -0,0 +1,720 @@
+/*
+ * Copyright (c) 2016-2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/wait.h>
+#include <linux/ktime.h>
+#include <linux/uaccess.h>
+#include <linux/poll.h>
+#include <trace/events/gk20a.h>
+#include <uapi/linux/nvgpu.h>
+
+#include "gk20a/gk20a.h"
+#include "gk20a/gr_gk20a.h"
+
+#include <nvgpu/kmem.h>
+#include <nvgpu/log.h>
+#include <nvgpu/atomic.h>
+#include <nvgpu/barrier.h>
+
+#include "platform_gk20a.h"
+#include "os_linux.h"
+#include "ctxsw_trace.h"
+
+#include <nvgpu/hw/gk20a/hw_ctxsw_prog_gk20a.h>
+#include <nvgpu/hw/gk20a/hw_gr_gk20a.h>
+
+#define GK20A_CTXSW_TRACE_MAX_VM_RING_SIZE      (128*PAGE_SIZE)
+
+/* Userland-facing FIFO (one global + eventually one per VM) */
+struct gk20a_ctxsw_dev {
+        struct gk20a *g;
+
+        struct nvgpu_ctxsw_ring_header *hdr;
+        struct nvgpu_ctxsw_trace_entry *ents;
+        struct nvgpu_ctxsw_trace_filter filter;
+        bool write_enabled;
+        struct nvgpu_cond readout_wq;
+        size_t size;
+        u32 num_ents;
+
+        nvgpu_atomic_t vma_ref;
+
+        struct nvgpu_mutex write_lock;
+};
+
+
+struct gk20a_ctxsw_trace {
+        struct gk20a_ctxsw_dev devs[GK20A_CTXSW_TRACE_NUM_DEVS];
+};
+
+static inline int ring_is_empty(struct nvgpu_ctxsw_ring_header *hdr)
+{
+        return (hdr->write_idx == hdr->read_idx);
+}
+
+static inline int ring_is_full(struct nvgpu_ctxsw_ring_header *hdr)
+{
+        return ((hdr->write_idx + 1) % hdr->num_ents) == hdr->read_idx;
+}
+
+static inline int ring_len(struct nvgpu_ctxsw_ring_header *hdr)
+{
+        return (hdr->write_idx - hdr->read_idx) % hdr->num_ents;
+}
+
+ssize_t gk20a_ctxsw_dev_read(struct file *filp, char __user *buf, size_t size,
+        loff_t *off)
+{
+        struct gk20a_ctxsw_dev *dev = filp->private_data;
+        struct nvgpu_ctxsw_ring_header *hdr = dev->hdr;
+        struct nvgpu_ctxsw_trace_entry __user *entry =
+                (struct nvgpu_ctxsw_trace_entry *) buf;
+        size_t copied = 0;
+        int err;
+
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw,
+                "filp=%p buf=%p size=%zu", filp, buf, size);
+
+        nvgpu_mutex_acquire(&dev->write_lock);
+        while (ring_is_empty(hdr)) {
+                nvgpu_mutex_release(&dev->write_lock);
+                if (filp->f_flags & O_NONBLOCK)
+                        return -EAGAIN;
+                err = NVGPU_COND_WAIT_INTERRUPTIBLE(&dev->readout_wq,
+                        !ring_is_empty(hdr), 0);
+                if (err)
+                        return err;
+                nvgpu_mutex_acquire(&dev->write_lock);
+        }
+
+        while (size >= sizeof(struct nvgpu_ctxsw_trace_entry)) {
+                if (ring_is_empty(hdr))
+                        break;
+
+                if (copy_to_user(entry, &dev->ents[hdr->read_idx],
+                        sizeof(*entry))) {
+                        nvgpu_mutex_release(&dev->write_lock);
+                        return -EFAULT;
+                }
+
+                hdr->read_idx++;
+                if (hdr->read_idx >= hdr->num_ents)
+                        hdr->read_idx = 0;
+
+                entry++;
+                copied += sizeof(*entry);
+                size -= sizeof(*entry);
+        }
+
+        gk20a_dbg(gpu_dbg_ctxsw, "copied=%zu read_idx=%d", copied,
+                hdr->read_idx);
+
+        *off = hdr->read_idx;
+        nvgpu_mutex_release(&dev->write_lock);
+
+        return copied;
+}
+
+static int gk20a_ctxsw_dev_ioctl_trace_enable(struct gk20a_ctxsw_dev *dev)
+{
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "trace enabled");
+        nvgpu_mutex_acquire(&dev->write_lock);
+        dev->write_enabled = true;
+        nvgpu_mutex_release(&dev->write_lock);
+        dev->g->ops.fecs_trace.enable(dev->g);
+        return 0;
+}
+
+static int gk20a_ctxsw_dev_ioctl_trace_disable(struct gk20a_ctxsw_dev *dev)
+{
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "trace disabled");
+        dev->g->ops.fecs_trace.disable(dev->g);
+        nvgpu_mutex_acquire(&dev->write_lock);
+        dev->write_enabled = false;
+        nvgpu_mutex_release(&dev->write_lock);
+        return 0;
+}
+
+static int gk20a_ctxsw_dev_alloc_buffer(struct gk20a_ctxsw_dev *dev,
+                                        size_t size)
+{
+        struct gk20a *g = dev->g;
+        void *buf;
+        int err;
+
+        if ((dev->write_enabled) || (nvgpu_atomic_read(&dev->vma_ref)))
+                return -EBUSY;
+
+        err = g->ops.fecs_trace.alloc_user_buffer(g, &buf, &size);
+        if (err)
+                return err;
+
+
+        dev->hdr = buf;
+        dev->ents = (struct nvgpu_ctxsw_trace_entry *) (dev->hdr + 1);
+        dev->size = size;
+        dev->num_ents = dev->hdr->num_ents;
+
+        gk20a_dbg(gpu_dbg_ctxsw, "size=%zu hdr=%p ents=%p num_ents=%d",
+                dev->size, dev->hdr, dev->ents, dev->hdr->num_ents);
+        return 0;
+}
+
+int gk20a_ctxsw_dev_ring_alloc(struct gk20a *g,
+                void **buf, size_t *size)
+{
+        struct nvgpu_ctxsw_ring_header *hdr;
+
+        *size = roundup(*size, PAGE_SIZE);
+        hdr = vmalloc_user(*size);
+        if (!hdr)
+                return -ENOMEM;
+
+        hdr->magic = NVGPU_CTXSW_RING_HEADER_MAGIC;
+        hdr->version = NVGPU_CTXSW_RING_HEADER_VERSION;
+        hdr->num_ents = (*size - sizeof(struct nvgpu_ctxsw_ring_header))
+                / sizeof(struct nvgpu_ctxsw_trace_entry);
+        hdr->ent_size = sizeof(struct nvgpu_ctxsw_trace_entry);
+        hdr->drop_count = 0;
+        hdr->read_idx = 0;
+        hdr->write_idx = 0;
+        hdr->write_seqno = 0;
+
+        *buf = hdr;
+        return 0;
+}
+
+int gk20a_ctxsw_dev_ring_free(struct gk20a *g)
+{
+        struct gk20a_ctxsw_dev *dev = &g->ctxsw_trace->devs[0];
+
+        nvgpu_vfree(g, dev->hdr);
+        return 0;
+}
+
+static int gk20a_ctxsw_dev_ioctl_ring_setup(struct gk20a_ctxsw_dev *dev,
+        struct nvgpu_ctxsw_ring_setup_args *args)
+{
+        size_t size = args->size;
+        int ret;
+
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "size=%zu", size);
+
+        if (size > GK20A_CTXSW_TRACE_MAX_VM_RING_SIZE)
+                return -EINVAL;
+
+        nvgpu_mutex_acquire(&dev->write_lock);
+        ret = gk20a_ctxsw_dev_alloc_buffer(dev, size);
+        nvgpu_mutex_release(&dev->write_lock);
+
+        return ret;
+}
+
+static int gk20a_ctxsw_dev_ioctl_set_filter(struct gk20a_ctxsw_dev *dev,
+        struct nvgpu_ctxsw_trace_filter_args *args)
+{
+        struct gk20a *g = dev->g;
+
+        nvgpu_mutex_acquire(&dev->write_lock);
+        dev->filter = args->filter;
+        nvgpu_mutex_release(&dev->write_lock);
+
+        if (g->ops.fecs_trace.set_filter)
+                g->ops.fecs_trace.set_filter(g, &dev->filter);
+        return 0;
+}
+
+static int gk20a_ctxsw_dev_ioctl_get_filter(struct gk20a_ctxsw_dev *dev,
+        struct nvgpu_ctxsw_trace_filter_args *args)
+{
+        nvgpu_mutex_acquire(&dev->write_lock);
+        args->filter = dev->filter;
+        nvgpu_mutex_release(&dev->write_lock);
+
+        return 0;
+}
+
+static int gk20a_ctxsw_dev_ioctl_poll(struct gk20a_ctxsw_dev *dev)
+{
+        struct gk20a *g = dev->g;
+        int err;
+
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "");
+
+        err = gk20a_busy(g);
+        if (err)
+                return err;
+
+        if (g->ops.fecs_trace.flush)
+                err = g->ops.fecs_trace.flush(g);
+
+        if (likely(!err))
+                err = g->ops.fecs_trace.poll(g);
+
+        gk20a_idle(g);
+        return err;
+}
+
+int gk20a_ctxsw_dev_open(struct inode *inode, struct file *filp)
+{
+        struct nvgpu_os_linux *l;
+        struct gk20a *g;
+        struct gk20a_ctxsw_trace *trace;
+        struct gk20a_ctxsw_dev *dev;
+        int err;
+        size_t size;
+        u32 n;
+
+        /* only one VM for now */
+        const int vmid = 0;
+
+        l = container_of(inode->i_cdev, struct nvgpu_os_linux, ctxsw.cdev);
+        g = gk20a_get(&l->g);
+        if (!g)
+                return -ENODEV;
+
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "g=%p", g);
+
+        if (!capable(CAP_SYS_ADMIN)) {
+                err = -EPERM;
+                goto free_ref;
+        }
+
+        err = gk20a_busy(g);
+        if (err)
+                goto free_ref;
+
+        trace = g->ctxsw_trace;
+        if (!trace) {
+                err = -ENODEV;
+                goto idle;
+        }
+
+        /* Allow only one user for this device */
+        dev = &trace->devs[vmid];
+        nvgpu_mutex_acquire(&dev->write_lock);
+        if (dev->hdr) {
+                err = -EBUSY;
+                goto done;
+        }
+
+        /* By default, allocate ring buffer big enough to accommodate
+         * FECS records with default event filter */
+
+        /* enable all traces by default */
+        NVGPU_CTXSW_FILTER_SET_ALL(&dev->filter);
+
+        /* compute max number of entries generated with this filter */
+        n = g->ops.fecs_trace.max_entries(g, &dev->filter);
+
+        size = sizeof(struct nvgpu_ctxsw_ring_header) +
+                        n * sizeof(struct nvgpu_ctxsw_trace_entry);
+        gk20a_dbg(gpu_dbg_ctxsw, "size=%zu entries=%d ent_size=%zu",
+                size, n, sizeof(struct nvgpu_ctxsw_trace_entry));
+
+        err = gk20a_ctxsw_dev_alloc_buffer(dev, size);
+        if (!err) {
+                filp->private_data = dev;
+                gk20a_dbg(gpu_dbg_ctxsw, "filp=%p dev=%p size=%zu",
+                        filp, dev, size);
+        }
+
+done:
+        nvgpu_mutex_release(&dev->write_lock);
+
+idle:
+        gk20a_idle(g);
+free_ref:
+        if (err)
+                gk20a_put(g);
+        return err;
+}
+
+int gk20a_ctxsw_dev_release(struct inode *inode, struct file *filp)
+{
+        struct gk20a_ctxsw_dev *dev = filp->private_data;
+        struct gk20a *g = dev->g;
+
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "dev: %p", dev);
+
+        g->ops.fecs_trace.disable(g);
+
+        nvgpu_mutex_acquire(&dev->write_lock);
+        dev->write_enabled = false;
+        nvgpu_mutex_release(&dev->write_lock);
+
+        if (dev->hdr) {
+                dev->g->ops.fecs_trace.free_user_buffer(dev->g);
+                dev->hdr = NULL;
+        }
+        gk20a_put(g);
+        return 0;
+}
+
+long gk20a_ctxsw_dev_ioctl(struct file *filp, unsigned int cmd,
+        unsigned long arg)
+{
+        struct gk20a_ctxsw_dev *dev = filp->private_data;
+        struct gk20a *g = dev->g;
+        u8 buf[NVGPU_CTXSW_IOCTL_MAX_ARG_SIZE];
+        int err = 0;
+
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "nr=%d", _IOC_NR(cmd));
+
+        if ((_IOC_TYPE(cmd) != NVGPU_CTXSW_IOCTL_MAGIC) ||
+                (_IOC_NR(cmd) == 0) ||
+                (_IOC_NR(cmd) > NVGPU_CTXSW_IOCTL_LAST) ||
+                (_IOC_SIZE(cmd) > NVGPU_CTXSW_IOCTL_MAX_ARG_SIZE))
+                return -EINVAL;
+
+        memset(buf, 0, sizeof(buf));
+        if (_IOC_DIR(cmd) & _IOC_WRITE) {
+                if (copy_from_user(buf, (void __user *) arg, _IOC_SIZE(cmd)))
+                        return -EFAULT;
+        }
+
+        switch (cmd) {
+        case NVGPU_CTXSW_IOCTL_TRACE_ENABLE:
+                err = gk20a_ctxsw_dev_ioctl_trace_enable(dev);
+                break;
+        case NVGPU_CTXSW_IOCTL_TRACE_DISABLE:
+                err = gk20a_ctxsw_dev_ioctl_trace_disable(dev);
+                break;
+        case NVGPU_CTXSW_IOCTL_RING_SETUP:
+                err = gk20a_ctxsw_dev_ioctl_ring_setup(dev,
+                        (struct nvgpu_ctxsw_ring_setup_args *) buf);
+                break;
+        case NVGPU_CTXSW_IOCTL_SET_FILTER:
+                err = gk20a_ctxsw_dev_ioctl_set_filter(dev,
+                        (struct nvgpu_ctxsw_trace_filter_args *) buf);
+                break;
+        case NVGPU_CTXSW_IOCTL_GET_FILTER:
+                err = gk20a_ctxsw_dev_ioctl_get_filter(dev,
+                        (struct nvgpu_ctxsw_trace_filter_args *) buf);
+                break;
+        case NVGPU_CTXSW_IOCTL_POLL:
+                err = gk20a_ctxsw_dev_ioctl_poll(dev);
+                break;
+        default:
+                dev_dbg(dev_from_gk20a(g), "unrecognized gpu ioctl cmd: 0x%x",
+                        cmd);
+                err = -ENOTTY;
+        }
+
+        if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
+                err = copy_to_user((void __user *) arg, buf, _IOC_SIZE(cmd));
+
+        return err;
+}
+
+unsigned int gk20a_ctxsw_dev_poll(struct file *filp, poll_table *wait)
+{
+        struct gk20a_ctxsw_dev *dev = filp->private_data;
+        struct nvgpu_ctxsw_ring_header *hdr = dev->hdr;
+        unsigned int mask = 0;
+
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "");
+
+        nvgpu_mutex_acquire(&dev->write_lock);
+        poll_wait(filp, &dev->readout_wq.wq, wait);
+        if (!ring_is_empty(hdr))
+                mask |= POLLIN | POLLRDNORM;
+        nvgpu_mutex_release(&dev->write_lock);
+
+        return mask;
+}
+
+static void gk20a_ctxsw_dev_vma_open(struct vm_area_struct *vma)
+{
+        struct gk20a_ctxsw_dev *dev = vma->vm_private_data;
+
+        nvgpu_atomic_inc(&dev->vma_ref);
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "vma_ref=%d",
+                nvgpu_atomic_read(&dev->vma_ref));
+}
+
+static void gk20a_ctxsw_dev_vma_close(struct vm_area_struct *vma)
+{
+        struct gk20a_ctxsw_dev *dev = vma->vm_private_data;
+
+        nvgpu_atomic_dec(&dev->vma_ref);
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "vma_ref=%d",
+                nvgpu_atomic_read(&dev->vma_ref));
+}
+
+static struct vm_operations_struct gk20a_ctxsw_dev_vma_ops = {
+        .open = gk20a_ctxsw_dev_vma_open,
+        .close = gk20a_ctxsw_dev_vma_close,
+};
+
+int gk20a_ctxsw_dev_mmap_buffer(struct gk20a *g,
+                                struct vm_area_struct *vma)
+{
+        return remap_vmalloc_range(vma, g->ctxsw_trace->devs[0].hdr, 0);
+}
+
+int gk20a_ctxsw_dev_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+        struct gk20a_ctxsw_dev *dev = filp->private_data;
+        int ret;
+
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "vm_start=%lx vm_end=%lx",
+                vma->vm_start, vma->vm_end);
+
+        ret = dev->g->ops.fecs_trace.mmap_user_buffer(dev->g, vma);
+        if (likely(!ret)) {
+                vma->vm_private_data = dev;
+                vma->vm_ops = &gk20a_ctxsw_dev_vma_ops;
+                vma->vm_ops->open(vma);
+        }
+
+        return ret;
+}
+
+#ifdef CONFIG_GK20A_CTXSW_TRACE
+static int gk20a_ctxsw_init_devs(struct gk20a *g)
+{
+        struct gk20a_ctxsw_trace *trace = g->ctxsw_trace;
+        struct gk20a_ctxsw_dev *dev = trace->devs;
+        int err;
+        int i;
+
+        for (i = 0; i < GK20A_CTXSW_TRACE_NUM_DEVS; i++) {
+                dev->g = g;
+                dev->hdr = NULL;
+                dev->write_enabled = false;
+                nvgpu_cond_init(&dev->readout_wq);
+                err = nvgpu_mutex_init(&dev->write_lock);
+                if (err)
+                        return err;
+                nvgpu_atomic_set(&dev->vma_ref, 0);
+                dev++;
+        }
+        return 0;
+}
+#endif
+
+int gk20a_ctxsw_trace_init(struct gk20a *g)
+{
+#ifdef CONFIG_GK20A_CTXSW_TRACE
+        struct gk20a_ctxsw_trace *trace = g->ctxsw_trace;
+        int err;
+
+        gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "g=%p trace=%p", g, trace);
+
+        /* if tracing is not supported, skip this */
+        if (!g->ops.fecs_trace.init)
+                return 0;
+
+        if (likely(trace))
+                return 0;
+
+        trace = nvgpu_kzalloc(g, sizeof(*trace));
+        if (unlikely(!trace))
+                return -ENOMEM;
+        g->ctxsw_trace = trace;
+
+        err = gk20a_ctxsw_init_devs(g);
+        if (err)
+                goto fail;
+
+        err = g->ops.fecs_trace.init(g);
+        if (unlikely(err))
+                goto fail;
+
+        return 0;
+
+fail:
+        memset(&g->ops.fecs_trace, 0, sizeof(g->ops.fecs_trace));
+        nvgpu_kfree(g, trace);
+        g->ctxsw_trace = NULL;
+        return err;
+#else
+        return 0;
+#endif
+}
+
+void gk20a_ctxsw_trace_cleanup(struct gk20a *g)
+{
+#ifdef CONFIG_GK20A_CTXSW_TRACE
+        struct gk20a_ctxsw_trace *trace;
+        struct gk20a_ctxsw_dev *dev;
+        int i;
+
+        if (!g->ctxsw_trace)
+                return;
+
+        trace = g->ctxsw_trace;
+        dev = trace->devs;
+
+        for (i = 0; i < GK20A_CTXSW_TRACE_NUM_DEVS; i++) {
+                nvgpu_mutex_destroy(&dev->write_lock);
+                dev++;
+        }
+
+        nvgpu_kfree(g, g->ctxsw_trace);
+        g->ctxsw_trace = NULL;
+
+        g->ops.fecs_trace.deinit(g);
+#endif
+}
+
+int gk20a_ctxsw_trace_write(struct gk20a *g,
+                struct nvgpu_ctxsw_trace_entry *entry)
+{
+        struct nvgpu_ctxsw_ring_header *hdr;
+        struct gk20a_ctxsw_dev *dev;
+        int ret = 0;
+        const char *reason;
+        u32 write_idx;
+
+        if (!g->ctxsw_trace)
+                return 0;
+
+        if (unlikely(entry->vmid >= GK20A_CTXSW_TRACE_NUM_DEVS))
+                return -ENODEV;
+
+        dev = &g->ctxsw_trace->devs[entry->vmid];
+        hdr = dev->hdr;
+
+        gk20a_dbg(gpu_dbg_fn | gpu_dbg_ctxsw,
+                "dev=%p hdr=%p", dev, hdr);
+
+        nvgpu_mutex_acquire(&dev->write_lock);
+
+        if (unlikely(!hdr)) {
+                /* device has been released */
+                ret = -ENODEV;
+                goto done;
+        }
+
+        write_idx = hdr->write_idx;
+        if (write_idx >= dev->num_ents) {
+                nvgpu_err(dev->g,
+                        "write_idx=%u out of range [0..%u]",
+                        write_idx, dev->num_ents);
+                ret = -ENOSPC;
+                reason = "write_idx out of range";
+                goto disable;
+        }
+
+        entry->seqno = hdr->write_seqno++;
+
+        if (!dev->write_enabled) {
+                ret = -EBUSY;
+                reason = "write disabled";
+                goto drop;
+        }
+
+        if (unlikely(ring_is_full(hdr))) {
+                ret = -ENOSPC;
+                reason = "user fifo full";
+                goto drop;
+        }
+
+        if (!NVGPU_CTXSW_FILTER_ISSET(entry->tag, &dev->filter)) {
+                reason = "filtered out";
+                goto filter;
+        }
+
+        gk20a_dbg(gpu_dbg_ctxsw,
+                "seqno=%d context_id=%08x pid=%lld tag=%x timestamp=%llx",
+                entry->seqno, entry->context_id, entry->pid,
+                entry->tag, entry->timestamp);
+
+        dev->ents[write_idx] = *entry;
+
+        /* ensure record is written before updating write index */
+        nvgpu_smp_wmb();
+
+        write_idx++;
+        if (unlikely(write_idx >= hdr->num_ents))
+                write_idx = 0;
+        hdr->write_idx = write_idx;
+        gk20a_dbg(gpu_dbg_ctxsw, "added: read=%d write=%d len=%d",
+                hdr->read_idx, hdr->write_idx, ring_len(hdr));
+
+        nvgpu_mutex_release(&dev->write_lock);
+        return ret;
+
+disable:
+        g->ops.fecs_trace.disable(g);
+
+drop:
+        hdr->drop_count++;
+
+filter:
+        gk20a_dbg(gpu_dbg_ctxsw,
+                        "dropping seqno=%d context_id=%08x pid=%lld "
+                        "tag=%x time=%llx (%s)",
+                        entry->seqno, entry->context_id, entry->pid,
+                        entry->tag, entry->timestamp, reason);
+
+done:
+        nvgpu_mutex_release(&dev->write_lock);
+        return ret;
+}
+
+void gk20a_ctxsw_trace_wake_up(struct gk20a *g, int vmid)
+{
+        struct gk20a_ctxsw_dev *dev;
+
+        if (!g->ctxsw_trace)
+                return;
+
+        dev = &g->ctxsw_trace->devs[vmid];
+        nvgpu_cond_signal_interruptible(&dev->readout_wq);
+}
+
+void gk20a_ctxsw_trace_channel_reset(struct gk20a *g, struct channel_gk20a *ch)
+{
+#ifdef CONFIG_GK20A_CTXSW_TRACE
+        struct nvgpu_ctxsw_trace_entry entry = {
+                .vmid = 0,
+                .tag = NVGPU_CTXSW_TAG_ENGINE_RESET,
+                .context_id = 0,
+                .pid = ch->tgid,
+        };
+
+        if (!g->ctxsw_trace)
+                return;
+
+        g->ops.bus.read_ptimer(g, &entry.timestamp);
+        gk20a_ctxsw_trace_write(g, &entry);
+        gk20a_ctxsw_trace_wake_up(g, 0);
+#endif
+        trace_gk20a_channel_reset(ch->chid, ch->tsgid);
+}
+
+void gk20a_ctxsw_trace_tsg_reset(struct gk20a *g, struct tsg_gk20a *tsg)
+{
+#ifdef CONFIG_GK20A_CTXSW_TRACE
+        struct nvgpu_ctxsw_trace_entry entry = {
+                .vmid = 0,
+                .tag = NVGPU_CTXSW_TAG_ENGINE_RESET,
+                .context_id = 0,
+                .pid = tsg->tgid,
+        };
+
+        if (!g->ctxsw_trace)
+                return;
+
+        g->ops.bus.read_ptimer(g, &entry.timestamp);
+        gk20a_ctxsw_trace_write(g, &entry);
+        gk20a_ctxsw_trace_wake_up(g, 0);
+#endif
+        trace_gk20a_channel_reset(~0, tsg->tsgid);
+}