1 files changed, 376 insertions, 0 deletions

diff --git a/include/os/linux/debug_fifo.c b/include/os/linux/debug_fifo.c
new file mode 100644
index 0000000..98da8bc
--- /dev/null
+++ b/include/os/linux/debug_fifo.c
@@ -0,0 +1,376 @@
+/*
+ * Copyright (C) 2017-2020 NVIDIA Corporation.  All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include "debug_fifo.h"
+#include "os_linux.h"
+
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+
+#include <nvgpu/sort.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/channel.h>
+
+void __gk20a_fifo_profile_free(struct nvgpu_ref *ref);
+
+static void *gk20a_fifo_sched_debugfs_seq_start(
+                struct seq_file *s, loff_t *pos)
+{
+        struct gk20a *g = s->private;
+        struct fifo_gk20a *f = &g->fifo;
+
+        if (*pos >= f->num_channels)
+                return NULL;
+
+        return &f->channel[*pos];
+}
+
+static void *gk20a_fifo_sched_debugfs_seq_next(
+                struct seq_file *s, void *v, loff_t *pos)
+{
+        struct gk20a *g = s->private;
+        struct fifo_gk20a *f = &g->fifo;
+
+        ++(*pos);
+        if (*pos >= f->num_channels)
+                return NULL;
+
+        return &f->channel[*pos];
+}
+
+static void gk20a_fifo_sched_debugfs_seq_stop(
+                struct seq_file *s, void *v)
+{
+}
+
+static int gk20a_fifo_sched_debugfs_seq_show(
+                struct seq_file *s, void *v)
+{
+        struct gk20a *g = s->private;
+        struct fifo_gk20a *f = &g->fifo;
+        struct channel_gk20a *ch = v;
+        struct tsg_gk20a *tsg = NULL;
+
+        struct fifo_engine_info_gk20a *engine_info;
+        struct fifo_runlist_info_gk20a *runlist;
+        u32 runlist_id;
+        int ret = SEQ_SKIP;
+        u32 engine_id;
+
+        engine_id = gk20a_fifo_get_gr_engine_id(g);
+        engine_info = (f->engine_info + engine_id);
+        runlist_id = engine_info->runlist_id;
+        runlist = &f->runlist_info[runlist_id];
+
+        if (ch == f->channel) {
+                seq_puts(s, "chid     tsgid    pid      timeslice  timeout  interleave graphics_preempt compute_preempt\n");
+                seq_puts(s, "                            (usecs)   (msecs)\n");
+                ret = 0;
+        }
+
+        if (!test_bit(ch->chid, runlist->active_channels))
+                return ret;
+
+        if (gk20a_channel_get(ch)) {
+                tsg = tsg_gk20a_from_ch(ch);
+
+                if (tsg)
+                        seq_printf(s, "%-8d %-8d %-8d %-9d %-8d %-10d %-8d %-8d\n",
+                                ch->chid,
+                                ch->tsgid,
+                                ch->tgid,
+                                tsg->timeslice_us,
+                                ch->timeout_ms_max,
+                                tsg->interleave_level,
+                                tsg->gr_ctx.graphics_preempt_mode,
+                                tsg->gr_ctx.compute_preempt_mode);
+                gk20a_channel_put(ch);
+        }
+        return 0;
+}
+
+static const struct seq_operations gk20a_fifo_sched_debugfs_seq_ops = {
+        .start = gk20a_fifo_sched_debugfs_seq_start,
+        .next = gk20a_fifo_sched_debugfs_seq_next,
+        .stop = gk20a_fifo_sched_debugfs_seq_stop,
+        .show = gk20a_fifo_sched_debugfs_seq_show
+};
+
+static int gk20a_fifo_sched_debugfs_open(struct inode *inode,
+        struct file *file)
+{
+        struct gk20a *g = inode->i_private;
+        int err;
+
+        err = seq_open(file, &gk20a_fifo_sched_debugfs_seq_ops);
+        if (err)
+                return err;
+
+        nvgpu_log(g, gpu_dbg_info, "i_private=%p", inode->i_private);
+
+        ((struct seq_file *)file->private_data)->private = inode->i_private;
+        return 0;
+};
+
+/*
+ * The file operations structure contains our open function along with
+ * set of the canned seq_ ops.
+ */
+static const struct file_operations gk20a_fifo_sched_debugfs_fops = {
+        .owner = THIS_MODULE,
+        .open = gk20a_fifo_sched_debugfs_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = seq_release
+};
+
+static int gk20a_fifo_profile_enable(void *data, u64 val)
+{
+        struct gk20a *g = (struct gk20a *) data;
+        struct fifo_gk20a *f = &g->fifo;
+
+
+        nvgpu_mutex_acquire(&f->profile.lock);
+        if (val == 0) {
+                if (f->profile.enabled) {
+                        f->profile.enabled = false;
+                        nvgpu_ref_put(&f->profile.ref,
+                                __gk20a_fifo_profile_free);
+                }
+        } else {
+                if (!f->profile.enabled) {
+                        /* not kref init as it can have a running condition if
+                         * we enable/disable/enable while kickoff is happening
+                         */
+                        if (!nvgpu_ref_get_unless_zero(&f->profile.ref)) {
+                                f->profile.data = nvgpu_vzalloc(g,
+                                        FIFO_PROFILING_ENTRIES *
+                                        sizeof(struct fifo_profile_gk20a));
+                                f->profile.sorted  = nvgpu_vzalloc(g,
+                                        FIFO_PROFILING_ENTRIES *
+                                        sizeof(u64));
+                                if (!(f->profile.data && f->profile.sorted)) {
+                                        nvgpu_vfree(g, f->profile.data);
+                                        nvgpu_vfree(g, f->profile.sorted);
+                                        nvgpu_mutex_release(&f->profile.lock);
+                                        return -ENOMEM;
+                                }
+                                nvgpu_ref_init(&f->profile.ref);
+                        }
+                        atomic_set(&f->profile.get.atomic_var, 0);
+                        f->profile.enabled = true;
+                }
+        }
+        nvgpu_mutex_release(&f->profile.lock);
+
+        return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(
+        gk20a_fifo_profile_enable_debugfs_fops,
+        NULL,
+        gk20a_fifo_profile_enable,
+        "%llu\n"
+);
+
+static int __profile_cmp(const void *a, const void *b)
+{
+        return *((unsigned long long *) a) - *((unsigned long long *) b);
+}
+
+/*
+ * This uses about 800b in the stack, but the function using it is not part
+ * of a callstack where much memory is being used, so it is fine
+ */
+#define PERCENTILE_WIDTH        5
+#define PERCENTILE_RANGES       (100/PERCENTILE_WIDTH)
+
+static unsigned int __gk20a_fifo_create_stats(struct gk20a *g,
+                u64 *percentiles, u32 index_end, u32 index_start)
+{
+        unsigned int nelem = 0;
+        unsigned int index;
+        struct fifo_profile_gk20a *profile;
+
+        for (index = 0; index < FIFO_PROFILING_ENTRIES; index++) {
+                profile = &g->fifo.profile.data[index];
+
+                if (profile->timestamp[index_end] >
+                                profile->timestamp[index_start]) {
+                        /* This is a valid element */
+                        g->fifo.profile.sorted[nelem] =
+                                                profile->timestamp[index_end] -
+                                                profile->timestamp[index_start];
+                        nelem++;
+                }
+        }
+
+        /* sort it */
+        sort(g->fifo.profile.sorted, nelem, sizeof(unsigned long long),
+                __profile_cmp, NULL);
+
+        /* build ranges */
+        for (index = 0; index < PERCENTILE_RANGES; index++) {
+                percentiles[index] = nelem < PERCENTILE_RANGES ? 0 :
+                        g->fifo.profile.sorted[(PERCENTILE_WIDTH * (index + 1) *
+                                                nelem)/100 - 1];
+        }
+        return nelem;
+}
+
+static int gk20a_fifo_profile_stats(struct seq_file *s, void *unused)
+{
+        struct gk20a *g = s->private;
+        unsigned int get, nelem, index;
+        /*
+         * 800B in the stack, but function is declared statically and only
+         * called from debugfs handler
+         */
+        u64 percentiles_ioctl[PERCENTILE_RANGES];
+        u64 percentiles_kickoff[PERCENTILE_RANGES];
+        u64 percentiles_jobtracking[PERCENTILE_RANGES];
+        u64 percentiles_append[PERCENTILE_RANGES];
+        u64 percentiles_userd[PERCENTILE_RANGES];
+
+        if (!nvgpu_ref_get_unless_zero(&g->fifo.profile.ref)) {
+                seq_printf(s, "Profiling disabled\n");
+                return 0;
+        }
+
+        get = atomic_read(&g->fifo.profile.get.atomic_var);
+
+        __gk20a_fifo_create_stats(g, percentiles_ioctl,
+                PROFILE_IOCTL_EXIT, PROFILE_IOCTL_ENTRY);
+        __gk20a_fifo_create_stats(g, percentiles_kickoff,
+                PROFILE_END, PROFILE_ENTRY);
+        __gk20a_fifo_create_stats(g, percentiles_jobtracking,
+                PROFILE_JOB_TRACKING, PROFILE_IOCTL_ENTRY);
+        __gk20a_fifo_create_stats(g, percentiles_append,
+                PROFILE_APPEND, PROFILE_JOB_TRACKING);
+        nelem = __gk20a_fifo_create_stats(g, percentiles_userd,
+                PROFILE_END, PROFILE_APPEND);
+
+        seq_printf(s, "Number of kickoffs: %d\n", nelem);
+        seq_printf(s, "Perc \t ioctl(ns) \t kickoff(ns) \t pbcopy(ns) \t jobtrack(ns) \t userd(ns)\n");
+
+        for (index = 0; index < PERCENTILE_RANGES; index++)
+                seq_printf(s, "[%2dpc]\t%8lld\t%8lld\t%8lld\t%8lld\t%8lld\n",
+                        PERCENTILE_WIDTH * (index+1),
+                        percentiles_ioctl[index],
+                        percentiles_kickoff[index],
+                        percentiles_append[index],
+                        percentiles_jobtracking[index],
+                        percentiles_userd[index]);
+
+        nvgpu_ref_put(&g->fifo.profile.ref, __gk20a_fifo_profile_free);
+
+        return 0;
+}
+
+static int gk20a_fifo_profile_stats_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, gk20a_fifo_profile_stats, inode->i_private);
+}
+
+static const struct file_operations gk20a_fifo_profile_stats_debugfs_fops = {
+        .open           = gk20a_fifo_profile_stats_open,
+        .read           = seq_read,
+        .llseek         = seq_lseek,
+        .release        = single_release,
+};
+
+
+void gk20a_fifo_debugfs_init(struct gk20a *g)
+{
+        struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+        struct dentry *gpu_root = l->debugfs;
+        struct dentry *fifo_root;
+        struct dentry *profile_root;
+
+        fifo_root = debugfs_create_dir("fifo", gpu_root);
+        if (IS_ERR_OR_NULL(fifo_root))
+                return;
+
+        nvgpu_log(g, gpu_dbg_info, "g=%p", g);
+
+        debugfs_create_file("sched", 0600, fifo_root, g,
+                &gk20a_fifo_sched_debugfs_fops);
+
+        profile_root = debugfs_create_dir("profile", fifo_root);
+        if (IS_ERR_OR_NULL(profile_root))
+                return;
+
+        nvgpu_mutex_init(&g->fifo.profile.lock);
+        g->fifo.profile.enabled = false;
+        atomic_set(&g->fifo.profile.get.atomic_var, 0);
+        atomic_set(&g->fifo.profile.ref.refcount.atomic_var, 0);
+
+        debugfs_create_file("enable", 0600, profile_root, g,
+                &gk20a_fifo_profile_enable_debugfs_fops);
+
+        debugfs_create_file("stats", 0600, profile_root, g,
+                &gk20a_fifo_profile_stats_debugfs_fops);
+
+}
+
+void gk20a_fifo_profile_snapshot(struct fifo_profile_gk20a *profile, int idx)
+{
+        if (profile)
+                profile->timestamp[idx] = nvgpu_current_time_ns();
+}
+
+void __gk20a_fifo_profile_free(struct nvgpu_ref *ref)
+{
+        struct fifo_gk20a *f = container_of(ref, struct fifo_gk20a,
+                                                profile.ref);
+        nvgpu_vfree(f->g, f->profile.data);
+        nvgpu_vfree(f->g, f->profile.sorted);
+}
+
+/* Get the next element in the ring buffer of profile entries
+ * and grab a reference to the structure
+ */
+struct fifo_profile_gk20a *gk20a_fifo_profile_acquire(struct gk20a *g)
+{
+        struct fifo_gk20a *f = &g->fifo;
+        struct fifo_profile_gk20a *profile;
+        unsigned int index;
+
+        /* If kref is zero, profiling is not enabled */
+        if (!nvgpu_ref_get_unless_zero(&f->profile.ref))
+                return NULL;
+        index = atomic_inc_return(&f->profile.get.atomic_var);
+        profile = &f->profile.data[index % FIFO_PROFILING_ENTRIES];
+
+        return profile;
+}
+
+/* Free the reference to the structure. This allows deferred cleanups */
+void gk20a_fifo_profile_release(struct gk20a *g,
+                                        struct fifo_profile_gk20a *profile)
+{
+        nvgpu_ref_put(&g->fifo.profile.ref, __gk20a_fifo_profile_free);
+}
+
+void gk20a_fifo_debugfs_deinit(struct gk20a *g)
+{
+        struct fifo_gk20a *f = &g->fifo;
+
+        nvgpu_mutex_acquire(&f->profile.lock);
+        if (f->profile.enabled) {
+                f->profile.enabled = false;
+                nvgpu_ref_put(&f->profile.ref, __gk20a_fifo_profile_free);
+        }
+        nvgpu_mutex_release(&f->profile.lock);
+}