From d7fe2fbacb06e95f921f27f99f457904a0c3a57a Mon Sep 17 00:00:00 2001 From: Terje Bergstrom Date: Tue, 17 Oct 2017 10:44:51 -0700 Subject: gpu: nvgpu: Move sched to be Linux specific Move sched parameter APIs to be Linux specific implementation. At the same time the sched_ctrl fields were moved to nvgpu_os_linux. JIRA NVGPU-259 Change-Id: I2397e2602e1c4783f2bebf3aec462634b7f86d4a Signed-off-by: Terje Bergstrom Reviewed-on: https://git-master.nvidia.com/r/1580649 GVS: Gerrit_Virtual_Submit --- drivers/gpu/nvgpu/Makefile.nvgpu | 2 +- drivers/gpu/nvgpu/common/linux/debug_sched.c | 3 +- drivers/gpu/nvgpu/common/linux/ioctl_tsg.c | 23 +- drivers/gpu/nvgpu/common/linux/ioctl_tsg.h | 2 + drivers/gpu/nvgpu/common/linux/os_linux.h | 3 + drivers/gpu/nvgpu/common/linux/sched.c | 657 ++++++++++++++++++++++++++ drivers/gpu/nvgpu/common/linux/sched.h | 55 +++ drivers/gpu/nvgpu/gk20a/gk20a.h | 3 - drivers/gpu/nvgpu/gk20a/regops_gk20a.h | 2 - drivers/gpu/nvgpu/gk20a/sched_gk20a.c | 660 --------------------------- drivers/gpu/nvgpu/gk20a/sched_gk20a.h | 62 --- drivers/gpu/nvgpu/gk20a/tsg_gk20a.c | 4 - 12 files changed, 739 insertions(+), 737 deletions(-) create mode 100644 drivers/gpu/nvgpu/common/linux/sched.c create mode 100644 drivers/gpu/nvgpu/common/linux/sched.h delete mode 100644 drivers/gpu/nvgpu/gk20a/sched_gk20a.c delete mode 100644 drivers/gpu/nvgpu/gk20a/sched_gk20a.h diff --git a/drivers/gpu/nvgpu/Makefile.nvgpu b/drivers/gpu/nvgpu/Makefile.nvgpu index 81b0d1b8..ce4f67b0 100644 --- a/drivers/gpu/nvgpu/Makefile.nvgpu +++ b/drivers/gpu/nvgpu/Makefile.nvgpu @@ -53,6 +53,7 @@ nvgpu-y := \ common/linux/cde_gp10b.o \ common/linux/comptags.o \ common/linux/dmabuf.o \ + common/linux/sched.o \ common/mm/nvgpu_allocator.o \ common/mm/bitmap_allocator.o \ common/mm/buddy_allocator.o \ @@ -82,7 +83,6 @@ nvgpu-y := \ gk20a/gk20a.o \ gk20a/bus_gk20a.o \ gk20a/pramin_gk20a.o \ - gk20a/sched_gk20a.o \ gk20a/ce2_gk20a.o \ gk20a/fifo_gk20a.o \ gk20a/channel_gk20a.o \ diff --git a/drivers/gpu/nvgpu/common/linux/debug_sched.c b/drivers/gpu/nvgpu/common/linux/debug_sched.c index a42deb18..5b7cbddf 100644 --- a/drivers/gpu/nvgpu/common/linux/debug_sched.c +++ b/drivers/gpu/nvgpu/common/linux/debug_sched.c @@ -21,7 +21,8 @@ static int gk20a_sched_debugfs_show(struct seq_file *s, void *unused) { struct gk20a *g = s->private; - struct gk20a_sched_ctrl *sched = &g->sched_ctrl; + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct gk20a_sched_ctrl *sched = &l->sched_ctrl; bool sched_busy = true; int n = sched->bitmap_size / sizeof(u64); diff --git a/drivers/gpu/nvgpu/common/linux/ioctl_tsg.c b/drivers/gpu/nvgpu/common/linux/ioctl_tsg.c index 6d0439f3..ce7fa6af 100644 --- a/drivers/gpu/nvgpu/common/linux/ioctl_tsg.c +++ b/drivers/gpu/nvgpu/common/linux/ioctl_tsg.c @@ -237,6 +237,8 @@ int nvgpu_ioctl_tsg_open(struct gk20a *g, struct file *filp) priv->tsg = tsg; filp->private_data = priv; + gk20a_sched_ctrl_tsg_added(g, tsg); + return 0; free_ref: @@ -257,12 +259,22 @@ int nvgpu_ioctl_tsg_dev_open(struct inode *inode, struct file *filp) return ret; } +void nvgpu_ioctl_tsg_release(struct nvgpu_ref *ref) +{ + struct tsg_gk20a *tsg = container_of(ref, struct tsg_gk20a, refcount); + struct gk20a *g = tsg->g; + + gk20a_sched_ctrl_tsg_removed(g, tsg); + + gk20a_tsg_release(ref); +} + int nvgpu_ioctl_tsg_dev_release(struct inode *inode, struct file *filp) { struct tsg_private *priv = filp->private_data; struct tsg_gk20a *tsg = priv->tsg; - nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release); + nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); nvgpu_kfree(tsg->g, priv); return 0; } @@ -270,7 +282,8 @@ int nvgpu_ioctl_tsg_dev_release(struct inode *inode, struct file *filp) static int gk20a_tsg_ioctl_set_priority(struct gk20a *g, struct tsg_gk20a *tsg, struct nvgpu_set_priority_args *arg) { - struct gk20a_sched_ctrl *sched = &g->sched_ctrl; + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct gk20a_sched_ctrl *sched = &l->sched_ctrl; int err; nvgpu_mutex_acquire(&sched->control_lock); @@ -296,7 +309,8 @@ done: static int gk20a_tsg_ioctl_set_runlist_interleave(struct gk20a *g, struct tsg_gk20a *tsg, struct nvgpu_runlist_interleave_args *arg) { - struct gk20a_sched_ctrl *sched = &g->sched_ctrl; + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct gk20a_sched_ctrl *sched = &l->sched_ctrl; int err; gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsg->tsgid); @@ -323,7 +337,8 @@ done: static int gk20a_tsg_ioctl_set_timeslice(struct gk20a *g, struct tsg_gk20a *tsg, struct nvgpu_timeslice_args *arg) { - struct gk20a_sched_ctrl *sched = &g->sched_ctrl; + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct gk20a_sched_ctrl *sched = &l->sched_ctrl; int err; gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsg->tsgid); diff --git a/drivers/gpu/nvgpu/common/linux/ioctl_tsg.h b/drivers/gpu/nvgpu/common/linux/ioctl_tsg.h index 64d7a3da..67399fd4 100644 --- a/drivers/gpu/nvgpu/common/linux/ioctl_tsg.h +++ b/drivers/gpu/nvgpu/common/linux/ioctl_tsg.h @@ -16,11 +16,13 @@ struct inode; struct file; struct gk20a; +struct nvgpu_ref; int nvgpu_ioctl_tsg_dev_release(struct inode *inode, struct file *filp); int nvgpu_ioctl_tsg_dev_open(struct inode *inode, struct file *filp); int nvgpu_ioctl_tsg_open(struct gk20a *g, struct file *filp); long nvgpu_ioctl_tsg_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +void nvgpu_ioctl_tsg_release(struct nvgpu_ref *ref); #endif diff --git a/drivers/gpu/nvgpu/common/linux/os_linux.h b/drivers/gpu/nvgpu/common/linux/os_linux.h index 27433e32..9bb9e9f4 100644 --- a/drivers/gpu/nvgpu/common/linux/os_linux.h +++ b/drivers/gpu/nvgpu/common/linux/os_linux.h @@ -23,6 +23,7 @@ #endif #include "gk20a/gk20a.h" #include "cde.h" +#include "sched.h" struct nvgpu_os_linux_ops { struct { @@ -144,6 +145,8 @@ struct nvgpu_os_linux { struct gk20a_cde_app cde_app; struct rw_semaphore busy_lock; + + struct gk20a_sched_ctrl sched_ctrl; }; static inline struct nvgpu_os_linux *nvgpu_os_linux_from_gk20a(struct gk20a *g) diff --git a/drivers/gpu/nvgpu/common/linux/sched.c b/drivers/gpu/nvgpu/common/linux/sched.c new file mode 100644 index 00000000..fc3f6ed8 --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/sched.c @@ -0,0 +1,657 @@ +/* + * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "gk20a/gk20a.h" +#include "gk20a/gr_gk20a.h" +#include "sched.h" +#include "os_linux.h" +#include "ioctl_tsg.h" + +#include +#include + +ssize_t gk20a_sched_dev_read(struct file *filp, char __user *buf, + size_t size, loff_t *off) +{ + struct gk20a_sched_ctrl *sched = filp->private_data; + struct nvgpu_sched_event_arg event = { 0 }; + int err; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, + "filp=%p buf=%p size=%zu", filp, buf, size); + + if (size < sizeof(event)) + return -EINVAL; + size = sizeof(event); + + nvgpu_mutex_acquire(&sched->status_lock); + while (!sched->status) { + nvgpu_mutex_release(&sched->status_lock); + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; + err = NVGPU_COND_WAIT_INTERRUPTIBLE(&sched->readout_wq, + sched->status, 0); + if (err) + return err; + nvgpu_mutex_acquire(&sched->status_lock); + } + + event.reserved = 0; + event.status = sched->status; + + if (copy_to_user(buf, &event, size)) { + nvgpu_mutex_release(&sched->status_lock); + return -EFAULT; + } + + sched->status = 0; + + nvgpu_mutex_release(&sched->status_lock); + + return size; +} + +unsigned int gk20a_sched_dev_poll(struct file *filp, poll_table *wait) +{ + struct gk20a_sched_ctrl *sched = filp->private_data; + unsigned int mask = 0; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, ""); + + nvgpu_mutex_acquire(&sched->status_lock); + poll_wait(filp, &sched->readout_wq.wq, wait); + if (sched->status) + mask |= POLLIN | POLLRDNORM; + nvgpu_mutex_release(&sched->status_lock); + + return mask; +} + +static int gk20a_sched_dev_ioctl_get_tsgs(struct gk20a_sched_ctrl *sched, + struct nvgpu_sched_get_tsgs_args *arg) +{ + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "size=%u buffer=%llx", + arg->size, arg->buffer); + + if ((arg->size < sched->bitmap_size) || (!arg->buffer)) { + arg->size = sched->bitmap_size; + return -ENOSPC; + } + + nvgpu_mutex_acquire(&sched->status_lock); + if (copy_to_user((void __user *)(uintptr_t)arg->buffer, + sched->active_tsg_bitmap, sched->bitmap_size)) { + nvgpu_mutex_release(&sched->status_lock); + return -EFAULT; + } + nvgpu_mutex_release(&sched->status_lock); + + return 0; +} + +static int gk20a_sched_dev_ioctl_get_recent_tsgs(struct gk20a_sched_ctrl *sched, + struct nvgpu_sched_get_tsgs_args *arg) +{ + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "size=%u buffer=%llx", + arg->size, arg->buffer); + + if ((arg->size < sched->bitmap_size) || (!arg->buffer)) { + arg->size = sched->bitmap_size; + return -ENOSPC; + } + + nvgpu_mutex_acquire(&sched->status_lock); + if (copy_to_user((void __user *)(uintptr_t)arg->buffer, + sched->recent_tsg_bitmap, sched->bitmap_size)) { + nvgpu_mutex_release(&sched->status_lock); + return -EFAULT; + } + + memset(sched->recent_tsg_bitmap, 0, sched->bitmap_size); + nvgpu_mutex_release(&sched->status_lock); + + return 0; +} + +static int gk20a_sched_dev_ioctl_get_tsgs_by_pid(struct gk20a_sched_ctrl *sched, + struct nvgpu_sched_get_tsgs_by_pid_args *arg) +{ + struct fifo_gk20a *f = &sched->g->fifo; + struct tsg_gk20a *tsg; + u64 *bitmap; + unsigned int tsgid; + /* pid at user level corresponds to kernel tgid */ + pid_t tgid = (pid_t)arg->pid; + int err = 0; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "pid=%d size=%u buffer=%llx", + (pid_t)arg->pid, arg->size, arg->buffer); + + if ((arg->size < sched->bitmap_size) || (!arg->buffer)) { + arg->size = sched->bitmap_size; + return -ENOSPC; + } + + bitmap = nvgpu_kzalloc(sched->g, sched->bitmap_size); + if (!bitmap) + return -ENOMEM; + + nvgpu_mutex_acquire(&sched->status_lock); + for (tsgid = 0; tsgid < f->num_channels; tsgid++) { + if (NVGPU_SCHED_ISSET(tsgid, sched->active_tsg_bitmap)) { + tsg = &f->tsg[tsgid]; + if (tsg->tgid == tgid) + NVGPU_SCHED_SET(tsgid, bitmap); + } + } + nvgpu_mutex_release(&sched->status_lock); + + if (copy_to_user((void __user *)(uintptr_t)arg->buffer, + bitmap, sched->bitmap_size)) + err = -EFAULT; + + nvgpu_kfree(sched->g, bitmap); + + return err; +} + +static int gk20a_sched_dev_ioctl_get_params(struct gk20a_sched_ctrl *sched, + struct nvgpu_sched_tsg_get_params_args *arg) +{ + struct gk20a *g = sched->g; + struct fifo_gk20a *f = &g->fifo; + struct tsg_gk20a *tsg; + u32 tsgid = arg->tsgid; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); + + if (tsgid >= f->num_channels) + return -EINVAL; + + tsg = &f->tsg[tsgid]; + if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) + return -ENXIO; + + arg->pid = tsg->tgid; /* kernel tgid corresponds to user pid */ + arg->runlist_interleave = tsg->interleave_level; + arg->timeslice = tsg->timeslice_us; + + if (tsg->tsg_gr_ctx) { + arg->graphics_preempt_mode = + tsg->tsg_gr_ctx->graphics_preempt_mode; + arg->compute_preempt_mode = + tsg->tsg_gr_ctx->compute_preempt_mode; + } else { + arg->graphics_preempt_mode = 0; + arg->compute_preempt_mode = 0; + } + + nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); + + return 0; +} + +static int gk20a_sched_dev_ioctl_tsg_set_timeslice( + struct gk20a_sched_ctrl *sched, + struct nvgpu_sched_tsg_timeslice_args *arg) +{ + struct gk20a *g = sched->g; + struct fifo_gk20a *f = &g->fifo; + struct tsg_gk20a *tsg; + u32 tsgid = arg->tsgid; + int err; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); + + if (tsgid >= f->num_channels) + return -EINVAL; + + tsg = &f->tsg[tsgid]; + if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) + return -ENXIO; + + err = gk20a_busy(g); + if (err) + goto done; + + err = gk20a_tsg_set_timeslice(tsg, arg->timeslice); + + gk20a_idle(g); + +done: + nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); + + return err; +} + +static int gk20a_sched_dev_ioctl_tsg_set_runlist_interleave( + struct gk20a_sched_ctrl *sched, + struct nvgpu_sched_tsg_runlist_interleave_args *arg) +{ + struct gk20a *g = sched->g; + struct fifo_gk20a *f = &g->fifo; + struct tsg_gk20a *tsg; + u32 tsgid = arg->tsgid; + int err; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); + + if (tsgid >= f->num_channels) + return -EINVAL; + + tsg = &f->tsg[tsgid]; + if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) + return -ENXIO; + + err = gk20a_busy(g); + if (err) + goto done; + + err = gk20a_tsg_set_runlist_interleave(tsg, arg->runlist_interleave); + + gk20a_idle(g); + +done: + nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); + + return err; +} + +static int gk20a_sched_dev_ioctl_lock_control(struct gk20a_sched_ctrl *sched) +{ + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, ""); + + nvgpu_mutex_acquire(&sched->control_lock); + sched->control_locked = true; + nvgpu_mutex_release(&sched->control_lock); + return 0; +} + +static int gk20a_sched_dev_ioctl_unlock_control(struct gk20a_sched_ctrl *sched) +{ + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, ""); + + nvgpu_mutex_acquire(&sched->control_lock); + sched->control_locked = false; + nvgpu_mutex_release(&sched->control_lock); + return 0; +} + +static int gk20a_sched_dev_ioctl_get_api_version(struct gk20a_sched_ctrl *sched, + struct nvgpu_sched_api_version_args *args) +{ + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, ""); + + args->version = NVGPU_SCHED_API_VERSION; + return 0; +} + +static int gk20a_sched_dev_ioctl_get_tsg(struct gk20a_sched_ctrl *sched, + struct nvgpu_sched_tsg_refcount_args *arg) +{ + struct gk20a *g = sched->g; + struct fifo_gk20a *f = &g->fifo; + struct tsg_gk20a *tsg; + u32 tsgid = arg->tsgid; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); + + if (tsgid >= f->num_channels) + return -EINVAL; + + tsg = &f->tsg[tsgid]; + if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) + return -ENXIO; + + nvgpu_mutex_acquire(&sched->status_lock); + if (NVGPU_SCHED_ISSET(tsgid, sched->ref_tsg_bitmap)) { + nvgpu_warn(g, "tsgid=%d already referenced", tsgid); + /* unlock status_lock as nvgpu_ioctl_tsg_release locks it */ + nvgpu_mutex_release(&sched->status_lock); + nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); + return -ENXIO; + } + + /* keep reference on TSG, will be released on + * NVGPU_SCHED_IOCTL_PUT_TSG ioctl, or close + */ + NVGPU_SCHED_SET(tsgid, sched->ref_tsg_bitmap); + nvgpu_mutex_release(&sched->status_lock); + + return 0; +} + +static int gk20a_sched_dev_ioctl_put_tsg(struct gk20a_sched_ctrl *sched, + struct nvgpu_sched_tsg_refcount_args *arg) +{ + struct gk20a *g = sched->g; + struct fifo_gk20a *f = &g->fifo; + struct tsg_gk20a *tsg; + u32 tsgid = arg->tsgid; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); + + if (tsgid >= f->num_channels) + return -EINVAL; + + nvgpu_mutex_acquire(&sched->status_lock); + if (!NVGPU_SCHED_ISSET(tsgid, sched->ref_tsg_bitmap)) { + nvgpu_mutex_release(&sched->status_lock); + nvgpu_warn(g, "tsgid=%d not previously referenced", tsgid); + return -ENXIO; + } + NVGPU_SCHED_CLR(tsgid, sched->ref_tsg_bitmap); + nvgpu_mutex_release(&sched->status_lock); + + tsg = &f->tsg[tsgid]; + nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); + + return 0; +} + +int gk20a_sched_dev_open(struct inode *inode, struct file *filp) +{ + struct nvgpu_os_linux *l = container_of(inode->i_cdev, + struct nvgpu_os_linux, sched.cdev); + struct gk20a *g; + struct gk20a_sched_ctrl *sched; + int err = 0; + + g = gk20a_get(&l->g); + if (!g) + return -ENODEV; + sched = &l->sched_ctrl; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "g=%p", g); + + if (!sched->sw_ready) { + err = gk20a_busy(g); + if (err) + goto free_ref; + + gk20a_idle(g); + } + + if (!nvgpu_mutex_tryacquire(&sched->busy_lock)) { + err = -EBUSY; + goto free_ref; + } + + memcpy(sched->recent_tsg_bitmap, sched->active_tsg_bitmap, + sched->bitmap_size); + memset(sched->ref_tsg_bitmap, 0, sched->bitmap_size); + + filp->private_data = sched; + gk20a_dbg(gpu_dbg_sched, "filp=%p sched=%p", filp, sched); + +free_ref: + if (err) + gk20a_put(g); + return err; +} + +long gk20a_sched_dev_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + struct gk20a_sched_ctrl *sched = filp->private_data; + struct gk20a *g = sched->g; + u8 buf[NVGPU_CTXSW_IOCTL_MAX_ARG_SIZE]; + int err = 0; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "nr=%d", _IOC_NR(cmd)); + + if ((_IOC_TYPE(cmd) != NVGPU_SCHED_IOCTL_MAGIC) || + (_IOC_NR(cmd) == 0) || + (_IOC_NR(cmd) > NVGPU_SCHED_IOCTL_LAST) || + (_IOC_SIZE(cmd) > NVGPU_SCHED_IOCTL_MAX_ARG_SIZE)) + return -EINVAL; + + memset(buf, 0, sizeof(buf)); + if (_IOC_DIR(cmd) & _IOC_WRITE) { + if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd))) + return -EFAULT; + } + + switch (cmd) { + case NVGPU_SCHED_IOCTL_GET_TSGS: + err = gk20a_sched_dev_ioctl_get_tsgs(sched, + (struct nvgpu_sched_get_tsgs_args *)buf); + break; + case NVGPU_SCHED_IOCTL_GET_RECENT_TSGS: + err = gk20a_sched_dev_ioctl_get_recent_tsgs(sched, + (struct nvgpu_sched_get_tsgs_args *)buf); + break; + case NVGPU_SCHED_IOCTL_GET_TSGS_BY_PID: + err = gk20a_sched_dev_ioctl_get_tsgs_by_pid(sched, + (struct nvgpu_sched_get_tsgs_by_pid_args *)buf); + break; + case NVGPU_SCHED_IOCTL_TSG_GET_PARAMS: + err = gk20a_sched_dev_ioctl_get_params(sched, + (struct nvgpu_sched_tsg_get_params_args *)buf); + break; + case NVGPU_SCHED_IOCTL_TSG_SET_TIMESLICE: + err = gk20a_sched_dev_ioctl_tsg_set_timeslice(sched, + (struct nvgpu_sched_tsg_timeslice_args *)buf); + break; + case NVGPU_SCHED_IOCTL_TSG_SET_RUNLIST_INTERLEAVE: + err = gk20a_sched_dev_ioctl_tsg_set_runlist_interleave(sched, + (struct nvgpu_sched_tsg_runlist_interleave_args *)buf); + break; + case NVGPU_SCHED_IOCTL_LOCK_CONTROL: + err = gk20a_sched_dev_ioctl_lock_control(sched); + break; + case NVGPU_SCHED_IOCTL_UNLOCK_CONTROL: + err = gk20a_sched_dev_ioctl_unlock_control(sched); + break; + case NVGPU_SCHED_IOCTL_GET_API_VERSION: + err = gk20a_sched_dev_ioctl_get_api_version(sched, + (struct nvgpu_sched_api_version_args *)buf); + break; + case NVGPU_SCHED_IOCTL_GET_TSG: + err = gk20a_sched_dev_ioctl_get_tsg(sched, + (struct nvgpu_sched_tsg_refcount_args *)buf); + break; + case NVGPU_SCHED_IOCTL_PUT_TSG: + err = gk20a_sched_dev_ioctl_put_tsg(sched, + (struct nvgpu_sched_tsg_refcount_args *)buf); + break; + default: + nvgpu_log_info(g, "unrecognized gpu ioctl cmd: 0x%x", cmd); + err = -ENOTTY; + } + + /* Some ioctls like NVGPU_SCHED_IOCTL_GET_TSGS might be called on + * purpose with NULL buffer and/or zero size to discover TSG bitmap + * size. We need to update user arguments in this case too, even + * if we return an error. + */ + if ((!err || (err == -ENOSPC)) && (_IOC_DIR(cmd) & _IOC_READ)) { + if (copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd))) + err = -EFAULT; + } + + return err; +} + +int gk20a_sched_dev_release(struct inode *inode, struct file *filp) +{ + struct gk20a_sched_ctrl *sched = filp->private_data; + struct gk20a *g = sched->g; + struct fifo_gk20a *f = &g->fifo; + struct tsg_gk20a *tsg; + unsigned int tsgid; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "sched: %p", sched); + + /* release any reference to TSGs */ + for (tsgid = 0; tsgid < f->num_channels; tsgid++) { + if (NVGPU_SCHED_ISSET(tsgid, sched->ref_tsg_bitmap)) { + tsg = &f->tsg[tsgid]; + nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); + } + } + + /* unlock control */ + nvgpu_mutex_acquire(&sched->control_lock); + sched->control_locked = false; + nvgpu_mutex_release(&sched->control_lock); + + nvgpu_mutex_release(&sched->busy_lock); + gk20a_put(g); + return 0; +} + +void gk20a_sched_ctrl_tsg_added(struct gk20a *g, struct tsg_gk20a *tsg) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + int err; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsg->tsgid); + + if (!sched->sw_ready) { + err = gk20a_busy(g); + if (err) { + WARN_ON(err); + return; + } + + gk20a_idle(g); + } + + nvgpu_mutex_acquire(&sched->status_lock); + NVGPU_SCHED_SET(tsg->tsgid, sched->active_tsg_bitmap); + NVGPU_SCHED_SET(tsg->tsgid, sched->recent_tsg_bitmap); + sched->status |= NVGPU_SCHED_STATUS_TSG_OPEN; + nvgpu_mutex_release(&sched->status_lock); + nvgpu_cond_signal_interruptible(&sched->readout_wq); +} + +void gk20a_sched_ctrl_tsg_removed(struct gk20a *g, struct tsg_gk20a *tsg) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsg->tsgid); + + nvgpu_mutex_acquire(&sched->status_lock); + NVGPU_SCHED_CLR(tsg->tsgid, sched->active_tsg_bitmap); + + /* clear recent_tsg_bitmap as well: if app manager did not + * notice that TSG was previously added, no need to notify it + * if the TSG has been released in the meantime. If the + * TSG gets reallocated, app manager will be notified as usual. + */ + NVGPU_SCHED_CLR(tsg->tsgid, sched->recent_tsg_bitmap); + + /* do not set event_pending, we only want to notify app manager + * when TSGs are added, so that it can apply sched params + */ + nvgpu_mutex_release(&sched->status_lock); +} + +int gk20a_sched_ctrl_init(struct gk20a *g) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + struct fifo_gk20a *f = &g->fifo; + int err; + + if (sched->sw_ready) + return 0; + + sched->g = g; + sched->bitmap_size = roundup(f->num_channels, 64) / 8; + sched->status = 0; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "g=%p sched=%p size=%zu", + g, sched, sched->bitmap_size); + + sched->active_tsg_bitmap = nvgpu_kzalloc(g, sched->bitmap_size); + if (!sched->active_tsg_bitmap) + return -ENOMEM; + + sched->recent_tsg_bitmap = nvgpu_kzalloc(g, sched->bitmap_size); + if (!sched->recent_tsg_bitmap) { + err = -ENOMEM; + goto free_active; + } + + sched->ref_tsg_bitmap = nvgpu_kzalloc(g, sched->bitmap_size); + if (!sched->ref_tsg_bitmap) { + err = -ENOMEM; + goto free_recent; + } + + nvgpu_cond_init(&sched->readout_wq); + + err = nvgpu_mutex_init(&sched->status_lock); + if (err) + goto free_ref; + + err = nvgpu_mutex_init(&sched->control_lock); + if (err) + goto free_status_lock; + + err = nvgpu_mutex_init(&sched->busy_lock); + if (err) + goto free_control_lock; + + sched->sw_ready = true; + + return 0; + +free_control_lock: + nvgpu_mutex_destroy(&sched->control_lock); +free_status_lock: + nvgpu_mutex_destroy(&sched->status_lock); +free_ref: + nvgpu_kfree(g, sched->ref_tsg_bitmap); +free_recent: + nvgpu_kfree(g, sched->recent_tsg_bitmap); +free_active: + nvgpu_kfree(g, sched->active_tsg_bitmap); + + return err; +} + +void gk20a_sched_ctrl_cleanup(struct gk20a *g) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + + nvgpu_kfree(g, sched->active_tsg_bitmap); + nvgpu_kfree(g, sched->recent_tsg_bitmap); + nvgpu_kfree(g, sched->ref_tsg_bitmap); + sched->active_tsg_bitmap = NULL; + sched->recent_tsg_bitmap = NULL; + sched->ref_tsg_bitmap = NULL; + + nvgpu_mutex_destroy(&sched->status_lock); + nvgpu_mutex_destroy(&sched->control_lock); + nvgpu_mutex_destroy(&sched->busy_lock); + + sched->sw_ready = false; +} diff --git a/drivers/gpu/nvgpu/common/linux/sched.h b/drivers/gpu/nvgpu/common/linux/sched.h new file mode 100644 index 00000000..a699bbea --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/sched.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef __NVGPU_SCHED_H +#define __NVGPU_SCHED_H + +struct gk20a; +struct gpu_ops; +struct tsg_gk20a; +struct poll_table_struct; + +struct gk20a_sched_ctrl { + struct gk20a *g; + + struct nvgpu_mutex control_lock; + bool control_locked; + bool sw_ready; + struct nvgpu_mutex status_lock; + struct nvgpu_mutex busy_lock; + + u64 status; + + size_t bitmap_size; + u64 *active_tsg_bitmap; + u64 *recent_tsg_bitmap; + u64 *ref_tsg_bitmap; + + struct nvgpu_cond readout_wq; +}; + +int gk20a_sched_dev_release(struct inode *inode, struct file *filp); +int gk20a_sched_dev_open(struct inode *inode, struct file *filp); +long gk20a_sched_dev_ioctl(struct file *, unsigned int, unsigned long); +ssize_t gk20a_sched_dev_read(struct file *, char __user *, size_t, loff_t *); +unsigned int gk20a_sched_dev_poll(struct file *, struct poll_table_struct *); + +void gk20a_sched_ctrl_tsg_added(struct gk20a *, struct tsg_gk20a *); +void gk20a_sched_ctrl_tsg_removed(struct gk20a *, struct tsg_gk20a *); +int gk20a_sched_ctrl_init(struct gk20a *); + +void gk20a_sched_ctrl_cleanup(struct gk20a *g); + +#endif /* __NVGPU_SCHED_H */ diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index b55f4517..ba932df2 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h @@ -67,7 +67,6 @@ struct nvgpu_warpstate; #include "pmu_gk20a.h" #include "priv_ring_gk20a.h" #include "therm_gk20a.h" -#include "sched_gk20a.h" #ifdef CONFIG_ARCH_TEGRA_18x_SOC #include "clk/clk.h" #include "clk/clk_arb.h" @@ -1220,8 +1219,6 @@ struct gk20a { struct gk20a_ctxsw_trace *ctxsw_trace; struct gk20a_fecs_trace *fecs_trace; - struct gk20a_sched_ctrl sched_ctrl; - bool mmu_debug_ctrl; u32 tpc_fs_mask_user; diff --git a/drivers/gpu/nvgpu/gk20a/regops_gk20a.h b/drivers/gpu/nvgpu/gk20a/regops_gk20a.h index e2ef0e62..e0496a75 100644 --- a/drivers/gpu/nvgpu/gk20a/regops_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/regops_gk20a.h @@ -24,8 +24,6 @@ #ifndef REGOPS_GK20A_H #define REGOPS_GK20A_H -#include - struct regop_offset_range { u32 base:24; u32 count:8; diff --git a/drivers/gpu/nvgpu/gk20a/sched_gk20a.c b/drivers/gpu/nvgpu/gk20a/sched_gk20a.c deleted file mode 100644 index a77536af..00000000 --- a/drivers/gpu/nvgpu/gk20a/sched_gk20a.c +++ /dev/null @@ -1,660 +0,0 @@ -/* - * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "ctxsw_trace_gk20a.h" -#include "gk20a.h" -#include "gr_gk20a.h" -#include "sched_gk20a.h" -#include "common/linux/os_linux.h" - -#include -#include - -ssize_t gk20a_sched_dev_read(struct file *filp, char __user *buf, - size_t size, loff_t *off) -{ - struct gk20a_sched_ctrl *sched = filp->private_data; - struct nvgpu_sched_event_arg event = { 0 }; - int err; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, - "filp=%p buf=%p size=%zu", filp, buf, size); - - if (size < sizeof(event)) - return -EINVAL; - size = sizeof(event); - - nvgpu_mutex_acquire(&sched->status_lock); - while (!sched->status) { - nvgpu_mutex_release(&sched->status_lock); - if (filp->f_flags & O_NONBLOCK) - return -EAGAIN; - err = NVGPU_COND_WAIT_INTERRUPTIBLE(&sched->readout_wq, - sched->status, 0); - if (err) - return err; - nvgpu_mutex_acquire(&sched->status_lock); - } - - event.reserved = 0; - event.status = sched->status; - - if (copy_to_user(buf, &event, size)) { - nvgpu_mutex_release(&sched->status_lock); - return -EFAULT; - } - - sched->status = 0; - - nvgpu_mutex_release(&sched->status_lock); - - return size; -} - -unsigned int gk20a_sched_dev_poll(struct file *filp, poll_table *wait) -{ - struct gk20a_sched_ctrl *sched = filp->private_data; - unsigned int mask = 0; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, ""); - - nvgpu_mutex_acquire(&sched->status_lock); - poll_wait(filp, &sched->readout_wq.wq, wait); - if (sched->status) - mask |= POLLIN | POLLRDNORM; - nvgpu_mutex_release(&sched->status_lock); - - return mask; -} - -static int gk20a_sched_dev_ioctl_get_tsgs(struct gk20a_sched_ctrl *sched, - struct nvgpu_sched_get_tsgs_args *arg) -{ - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "size=%u buffer=%llx", - arg->size, arg->buffer); - - if ((arg->size < sched->bitmap_size) || (!arg->buffer)) { - arg->size = sched->bitmap_size; - return -ENOSPC; - } - - nvgpu_mutex_acquire(&sched->status_lock); - if (copy_to_user((void __user *)(uintptr_t)arg->buffer, - sched->active_tsg_bitmap, sched->bitmap_size)) { - nvgpu_mutex_release(&sched->status_lock); - return -EFAULT; - } - nvgpu_mutex_release(&sched->status_lock); - - return 0; -} - -static int gk20a_sched_dev_ioctl_get_recent_tsgs(struct gk20a_sched_ctrl *sched, - struct nvgpu_sched_get_tsgs_args *arg) -{ - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "size=%u buffer=%llx", - arg->size, arg->buffer); - - if ((arg->size < sched->bitmap_size) || (!arg->buffer)) { - arg->size = sched->bitmap_size; - return -ENOSPC; - } - - nvgpu_mutex_acquire(&sched->status_lock); - if (copy_to_user((void __user *)(uintptr_t)arg->buffer, - sched->recent_tsg_bitmap, sched->bitmap_size)) { - nvgpu_mutex_release(&sched->status_lock); - return -EFAULT; - } - - memset(sched->recent_tsg_bitmap, 0, sched->bitmap_size); - nvgpu_mutex_release(&sched->status_lock); - - return 0; -} - -static int gk20a_sched_dev_ioctl_get_tsgs_by_pid(struct gk20a_sched_ctrl *sched, - struct nvgpu_sched_get_tsgs_by_pid_args *arg) -{ - struct fifo_gk20a *f = &sched->g->fifo; - struct tsg_gk20a *tsg; - u64 *bitmap; - unsigned int tsgid; - /* pid at user level corresponds to kernel tgid */ - pid_t tgid = (pid_t)arg->pid; - int err = 0; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "pid=%d size=%u buffer=%llx", - (pid_t)arg->pid, arg->size, arg->buffer); - - if ((arg->size < sched->bitmap_size) || (!arg->buffer)) { - arg->size = sched->bitmap_size; - return -ENOSPC; - } - - bitmap = nvgpu_kzalloc(sched->g, sched->bitmap_size); - if (!bitmap) - return -ENOMEM; - - nvgpu_mutex_acquire(&sched->status_lock); - for (tsgid = 0; tsgid < f->num_channels; tsgid++) { - if (NVGPU_SCHED_ISSET(tsgid, sched->active_tsg_bitmap)) { - tsg = &f->tsg[tsgid]; - if (tsg->tgid == tgid) - NVGPU_SCHED_SET(tsgid, bitmap); - } - } - nvgpu_mutex_release(&sched->status_lock); - - if (copy_to_user((void __user *)(uintptr_t)arg->buffer, - bitmap, sched->bitmap_size)) - err = -EFAULT; - - nvgpu_kfree(sched->g, bitmap); - - return err; -} - -static int gk20a_sched_dev_ioctl_get_params(struct gk20a_sched_ctrl *sched, - struct nvgpu_sched_tsg_get_params_args *arg) -{ - struct gk20a *g = sched->g; - struct fifo_gk20a *f = &g->fifo; - struct tsg_gk20a *tsg; - u32 tsgid = arg->tsgid; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); - - if (tsgid >= f->num_channels) - return -EINVAL; - - tsg = &f->tsg[tsgid]; - if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) - return -ENXIO; - - arg->pid = tsg->tgid; /* kernel tgid corresponds to user pid */ - arg->runlist_interleave = tsg->interleave_level; - arg->timeslice = tsg->timeslice_us; - - if (tsg->tsg_gr_ctx) { - arg->graphics_preempt_mode = - tsg->tsg_gr_ctx->graphics_preempt_mode; - arg->compute_preempt_mode = - tsg->tsg_gr_ctx->compute_preempt_mode; - } else { - arg->graphics_preempt_mode = 0; - arg->compute_preempt_mode = 0; - } - - nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release); - - return 0; -} - -static int gk20a_sched_dev_ioctl_tsg_set_timeslice( - struct gk20a_sched_ctrl *sched, - struct nvgpu_sched_tsg_timeslice_args *arg) -{ - struct gk20a *g = sched->g; - struct fifo_gk20a *f = &g->fifo; - struct tsg_gk20a *tsg; - u32 tsgid = arg->tsgid; - int err; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); - - if (tsgid >= f->num_channels) - return -EINVAL; - - tsg = &f->tsg[tsgid]; - if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) - return -ENXIO; - - err = gk20a_busy(g); - if (err) - goto done; - - err = gk20a_tsg_set_timeslice(tsg, arg->timeslice); - - gk20a_idle(g); - -done: - nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release); - - return err; -} - -static int gk20a_sched_dev_ioctl_tsg_set_runlist_interleave( - struct gk20a_sched_ctrl *sched, - struct nvgpu_sched_tsg_runlist_interleave_args *arg) -{ - struct gk20a *g = sched->g; - struct fifo_gk20a *f = &g->fifo; - struct tsg_gk20a *tsg; - u32 tsgid = arg->tsgid; - int err; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); - - if (tsgid >= f->num_channels) - return -EINVAL; - - tsg = &f->tsg[tsgid]; - if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) - return -ENXIO; - - err = gk20a_busy(g); - if (err) - goto done; - - err = gk20a_tsg_set_runlist_interleave(tsg, arg->runlist_interleave); - - gk20a_idle(g); - -done: - nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release); - - return err; -} - -static int gk20a_sched_dev_ioctl_lock_control(struct gk20a_sched_ctrl *sched) -{ - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, ""); - - nvgpu_mutex_acquire(&sched->control_lock); - sched->control_locked = true; - nvgpu_mutex_release(&sched->control_lock); - return 0; -} - -static int gk20a_sched_dev_ioctl_unlock_control(struct gk20a_sched_ctrl *sched) -{ - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, ""); - - nvgpu_mutex_acquire(&sched->control_lock); - sched->control_locked = false; - nvgpu_mutex_release(&sched->control_lock); - return 0; -} - -static int gk20a_sched_dev_ioctl_get_api_version(struct gk20a_sched_ctrl *sched, - struct nvgpu_sched_api_version_args *args) -{ - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, ""); - - args->version = NVGPU_SCHED_API_VERSION; - return 0; -} - -static int gk20a_sched_dev_ioctl_get_tsg(struct gk20a_sched_ctrl *sched, - struct nvgpu_sched_tsg_refcount_args *arg) -{ - struct gk20a *g = sched->g; - struct fifo_gk20a *f = &g->fifo; - struct tsg_gk20a *tsg; - u32 tsgid = arg->tsgid; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); - - if (tsgid >= f->num_channels) - return -EINVAL; - - tsg = &f->tsg[tsgid]; - if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) - return -ENXIO; - - nvgpu_mutex_acquire(&sched->status_lock); - if (NVGPU_SCHED_ISSET(tsgid, sched->ref_tsg_bitmap)) { - nvgpu_warn(g, "tsgid=%d already referenced", tsgid); - /* unlock status_lock as gk20a_tsg_release locks it */ - nvgpu_mutex_release(&sched->status_lock); - nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release); - return -ENXIO; - } - - /* keep reference on TSG, will be released on - * NVGPU_SCHED_IOCTL_PUT_TSG ioctl, or close - */ - NVGPU_SCHED_SET(tsgid, sched->ref_tsg_bitmap); - nvgpu_mutex_release(&sched->status_lock); - - return 0; -} - -static int gk20a_sched_dev_ioctl_put_tsg(struct gk20a_sched_ctrl *sched, - struct nvgpu_sched_tsg_refcount_args *arg) -{ - struct gk20a *g = sched->g; - struct fifo_gk20a *f = &g->fifo; - struct tsg_gk20a *tsg; - u32 tsgid = arg->tsgid; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); - - if (tsgid >= f->num_channels) - return -EINVAL; - - nvgpu_mutex_acquire(&sched->status_lock); - if (!NVGPU_SCHED_ISSET(tsgid, sched->ref_tsg_bitmap)) { - nvgpu_mutex_release(&sched->status_lock); - nvgpu_warn(g, "tsgid=%d not previously referenced", tsgid); - return -ENXIO; - } - NVGPU_SCHED_CLR(tsgid, sched->ref_tsg_bitmap); - nvgpu_mutex_release(&sched->status_lock); - - tsg = &f->tsg[tsgid]; - nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release); - - return 0; -} - -int gk20a_sched_dev_open(struct inode *inode, struct file *filp) -{ - struct nvgpu_os_linux *l = container_of(inode->i_cdev, - struct nvgpu_os_linux, sched.cdev); - struct gk20a *g; - struct gk20a_sched_ctrl *sched; - int err = 0; - - g = gk20a_get(&l->g); - if (!g) - return -ENODEV; - sched = &g->sched_ctrl; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "g=%p", g); - - if (!sched->sw_ready) { - err = gk20a_busy(g); - if (err) - goto free_ref; - - gk20a_idle(g); - } - - if (!nvgpu_mutex_tryacquire(&sched->busy_lock)) { - err = -EBUSY; - goto free_ref; - } - - memcpy(sched->recent_tsg_bitmap, sched->active_tsg_bitmap, - sched->bitmap_size); - memset(sched->ref_tsg_bitmap, 0, sched->bitmap_size); - - filp->private_data = sched; - gk20a_dbg(gpu_dbg_sched, "filp=%p sched=%p", filp, sched); - -free_ref: - if (err) - gk20a_put(g); - return err; -} - -long gk20a_sched_dev_ioctl(struct file *filp, unsigned int cmd, - unsigned long arg) -{ - struct gk20a_sched_ctrl *sched = filp->private_data; - struct gk20a *g = sched->g; - u8 buf[NVGPU_CTXSW_IOCTL_MAX_ARG_SIZE]; - int err = 0; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "nr=%d", _IOC_NR(cmd)); - - if ((_IOC_TYPE(cmd) != NVGPU_SCHED_IOCTL_MAGIC) || - (_IOC_NR(cmd) == 0) || - (_IOC_NR(cmd) > NVGPU_SCHED_IOCTL_LAST) || - (_IOC_SIZE(cmd) > NVGPU_SCHED_IOCTL_MAX_ARG_SIZE)) - return -EINVAL; - - memset(buf, 0, sizeof(buf)); - if (_IOC_DIR(cmd) & _IOC_WRITE) { - if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd))) - return -EFAULT; - } - - switch (cmd) { - case NVGPU_SCHED_IOCTL_GET_TSGS: - err = gk20a_sched_dev_ioctl_get_tsgs(sched, - (struct nvgpu_sched_get_tsgs_args *)buf); - break; - case NVGPU_SCHED_IOCTL_GET_RECENT_TSGS: - err = gk20a_sched_dev_ioctl_get_recent_tsgs(sched, - (struct nvgpu_sched_get_tsgs_args *)buf); - break; - case NVGPU_SCHED_IOCTL_GET_TSGS_BY_PID: - err = gk20a_sched_dev_ioctl_get_tsgs_by_pid(sched, - (struct nvgpu_sched_get_tsgs_by_pid_args *)buf); - break; - case NVGPU_SCHED_IOCTL_TSG_GET_PARAMS: - err = gk20a_sched_dev_ioctl_get_params(sched, - (struct nvgpu_sched_tsg_get_params_args *)buf); - break; - case NVGPU_SCHED_IOCTL_TSG_SET_TIMESLICE: - err = gk20a_sched_dev_ioctl_tsg_set_timeslice(sched, - (struct nvgpu_sched_tsg_timeslice_args *)buf); - break; - case NVGPU_SCHED_IOCTL_TSG_SET_RUNLIST_INTERLEAVE: - err = gk20a_sched_dev_ioctl_tsg_set_runlist_interleave(sched, - (struct nvgpu_sched_tsg_runlist_interleave_args *)buf); - break; - case NVGPU_SCHED_IOCTL_LOCK_CONTROL: - err = gk20a_sched_dev_ioctl_lock_control(sched); - break; - case NVGPU_SCHED_IOCTL_UNLOCK_CONTROL: - err = gk20a_sched_dev_ioctl_unlock_control(sched); - break; - case NVGPU_SCHED_IOCTL_GET_API_VERSION: - err = gk20a_sched_dev_ioctl_get_api_version(sched, - (struct nvgpu_sched_api_version_args *)buf); - break; - case NVGPU_SCHED_IOCTL_GET_TSG: - err = gk20a_sched_dev_ioctl_get_tsg(sched, - (struct nvgpu_sched_tsg_refcount_args *)buf); - break; - case NVGPU_SCHED_IOCTL_PUT_TSG: - err = gk20a_sched_dev_ioctl_put_tsg(sched, - (struct nvgpu_sched_tsg_refcount_args *)buf); - break; - default: - nvgpu_log_info(g, "unrecognized gpu ioctl cmd: 0x%x", cmd); - err = -ENOTTY; - } - - /* Some ioctls like NVGPU_SCHED_IOCTL_GET_TSGS might be called on - * purpose with NULL buffer and/or zero size to discover TSG bitmap - * size. We need to update user arguments in this case too, even - * if we return an error. - */ - if ((!err || (err == -ENOSPC)) && (_IOC_DIR(cmd) & _IOC_READ)) { - if (copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd))) - err = -EFAULT; - } - - return err; -} - -int gk20a_sched_dev_release(struct inode *inode, struct file *filp) -{ - struct gk20a_sched_ctrl *sched = filp->private_data; - struct gk20a *g = sched->g; - struct fifo_gk20a *f = &g->fifo; - struct tsg_gk20a *tsg; - unsigned int tsgid; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "sched: %p", sched); - - /* release any reference to TSGs */ - for (tsgid = 0; tsgid < f->num_channels; tsgid++) { - if (NVGPU_SCHED_ISSET(tsgid, sched->ref_tsg_bitmap)) { - tsg = &f->tsg[tsgid]; - nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release); - } - } - - /* unlock control */ - nvgpu_mutex_acquire(&sched->control_lock); - sched->control_locked = false; - nvgpu_mutex_release(&sched->control_lock); - - nvgpu_mutex_release(&sched->busy_lock); - gk20a_put(g); - return 0; -} - -void gk20a_sched_ctrl_tsg_added(struct gk20a *g, struct tsg_gk20a *tsg) -{ - struct gk20a_sched_ctrl *sched = &g->sched_ctrl; - int err; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsg->tsgid); - - if (!sched->sw_ready) { - err = gk20a_busy(g); - if (err) { - WARN_ON(err); - return; - } - - gk20a_idle(g); - } - - nvgpu_mutex_acquire(&sched->status_lock); - NVGPU_SCHED_SET(tsg->tsgid, sched->active_tsg_bitmap); - NVGPU_SCHED_SET(tsg->tsgid, sched->recent_tsg_bitmap); - sched->status |= NVGPU_SCHED_STATUS_TSG_OPEN; - nvgpu_mutex_release(&sched->status_lock); - nvgpu_cond_signal_interruptible(&sched->readout_wq); -} - -void gk20a_sched_ctrl_tsg_removed(struct gk20a *g, struct tsg_gk20a *tsg) -{ - struct gk20a_sched_ctrl *sched = &g->sched_ctrl; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsg->tsgid); - - nvgpu_mutex_acquire(&sched->status_lock); - NVGPU_SCHED_CLR(tsg->tsgid, sched->active_tsg_bitmap); - - /* clear recent_tsg_bitmap as well: if app manager did not - * notice that TSG was previously added, no need to notify it - * if the TSG has been released in the meantime. If the - * TSG gets reallocated, app manager will be notified as usual. - */ - NVGPU_SCHED_CLR(tsg->tsgid, sched->recent_tsg_bitmap); - - /* do not set event_pending, we only want to notify app manager - * when TSGs are added, so that it can apply sched params - */ - nvgpu_mutex_release(&sched->status_lock); -} - -int gk20a_sched_ctrl_init(struct gk20a *g) -{ - struct gk20a_sched_ctrl *sched = &g->sched_ctrl; - struct fifo_gk20a *f = &g->fifo; - int err; - - if (sched->sw_ready) - return 0; - - sched->g = g; - sched->bitmap_size = roundup(f->num_channels, 64) / 8; - sched->status = 0; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_sched, "g=%p sched=%p size=%zu", - g, sched, sched->bitmap_size); - - sched->active_tsg_bitmap = nvgpu_kzalloc(g, sched->bitmap_size); - if (!sched->active_tsg_bitmap) - return -ENOMEM; - - sched->recent_tsg_bitmap = nvgpu_kzalloc(g, sched->bitmap_size); - if (!sched->recent_tsg_bitmap) { - err = -ENOMEM; - goto free_active; - } - - sched->ref_tsg_bitmap = nvgpu_kzalloc(g, sched->bitmap_size); - if (!sched->ref_tsg_bitmap) { - err = -ENOMEM; - goto free_recent; - } - - nvgpu_cond_init(&sched->readout_wq); - - err = nvgpu_mutex_init(&sched->status_lock); - if (err) - goto free_ref; - - err = nvgpu_mutex_init(&sched->control_lock); - if (err) - goto free_status_lock; - - err = nvgpu_mutex_init(&sched->busy_lock); - if (err) - goto free_control_lock; - - sched->sw_ready = true; - - return 0; - -free_control_lock: - nvgpu_mutex_destroy(&sched->control_lock); -free_status_lock: - nvgpu_mutex_destroy(&sched->status_lock); -free_ref: - nvgpu_kfree(g, sched->ref_tsg_bitmap); -free_recent: - nvgpu_kfree(g, sched->recent_tsg_bitmap); -free_active: - nvgpu_kfree(g, sched->active_tsg_bitmap); - - return err; -} - -void gk20a_sched_ctrl_cleanup(struct gk20a *g) -{ - struct gk20a_sched_ctrl *sched = &g->sched_ctrl; - - nvgpu_kfree(g, sched->active_tsg_bitmap); - nvgpu_kfree(g, sched->recent_tsg_bitmap); - nvgpu_kfree(g, sched->ref_tsg_bitmap); - sched->active_tsg_bitmap = NULL; - sched->recent_tsg_bitmap = NULL; - sched->ref_tsg_bitmap = NULL; - - nvgpu_mutex_destroy(&sched->status_lock); - nvgpu_mutex_destroy(&sched->control_lock); - nvgpu_mutex_destroy(&sched->busy_lock); - - sched->sw_ready = false; -} diff --git a/drivers/gpu/nvgpu/gk20a/sched_gk20a.h b/drivers/gpu/nvgpu/gk20a/sched_gk20a.h deleted file mode 100644 index 0cdb9914..00000000 --- a/drivers/gpu/nvgpu/gk20a/sched_gk20a.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#ifndef __SCHED_GK20A_H -#define __SCHED_GK20A_H - -struct gk20a; -struct gpu_ops; -struct tsg_gk20a; -struct poll_table_struct; - -struct gk20a_sched_ctrl { - struct gk20a *g; - - struct nvgpu_mutex control_lock; - bool control_locked; - bool sw_ready; - struct nvgpu_mutex status_lock; - struct nvgpu_mutex busy_lock; - - u64 status; - - size_t bitmap_size; - u64 *active_tsg_bitmap; - u64 *recent_tsg_bitmap; - u64 *ref_tsg_bitmap; - - struct nvgpu_cond readout_wq; -}; - -int gk20a_sched_dev_release(struct inode *inode, struct file *filp); -int gk20a_sched_dev_open(struct inode *inode, struct file *filp); -long gk20a_sched_dev_ioctl(struct file *, unsigned int, unsigned long); -ssize_t gk20a_sched_dev_read(struct file *, char __user *, size_t, loff_t *); -unsigned int gk20a_sched_dev_poll(struct file *, struct poll_table_struct *); - -void gk20a_sched_ctrl_tsg_added(struct gk20a *, struct tsg_gk20a *); -void gk20a_sched_ctrl_tsg_removed(struct gk20a *, struct tsg_gk20a *); -int gk20a_sched_ctrl_init(struct gk20a *); - -void gk20a_sched_ctrl_cleanup(struct gk20a *g); - -#endif /* __SCHED_GK20A_H */ diff --git a/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c b/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c index cde281ad..badc7ef9 100644 --- a/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c @@ -325,8 +325,6 @@ struct tsg_gk20a *gk20a_tsg_open(struct gk20a *g) gk20a_dbg(gpu_dbg_fn, "tsg opened %d\n", tsg->tsgid); - gk20a_sched_ctrl_tsg_added(g, tsg); - return tsg; clean_up: @@ -353,8 +351,6 @@ void gk20a_tsg_release(struct nvgpu_ref *ref) tsg->vm = NULL; } - gk20a_sched_ctrl_tsg_removed(g, tsg); - /* unhook all events created on this TSG */ nvgpu_mutex_acquire(&tsg->event_id_list_lock); nvgpu_list_for_each_entry_safe(event_id_data, event_id_data_temp, -- cgit v1.2.2