From a07e10f494c158ae31d6187e9be3db409528a507 Mon Sep 17 00:00:00 2001 From: Terje Bergstrom Date: Thu, 23 Mar 2017 14:19:01 -0700 Subject: gpu: nvgpu: Move channel IOCTL code to Linux module Move channel IOCTL specific code to Linux module. This clears some Linux dependencies from channel_gk20a.c. JIRA NVGPU-32 Change-Id: I41817d612b959709365bcabff9c8a15f2bfe4c60 Signed-off-by: Terje Bergstrom Reviewed-on: http://git-master/r/1330804 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/common/linux/ioctl_channel.c | 1179 ++++++++++++++++++++++++ 1 file changed, 1179 insertions(+) create mode 100644 drivers/gpu/nvgpu/common/linux/ioctl_channel.c (limited to 'drivers/gpu/nvgpu/common/linux/ioctl_channel.c') diff --git a/drivers/gpu/nvgpu/common/linux/ioctl_channel.c b/drivers/gpu/nvgpu/common/linux/ioctl_channel.c new file mode 100644 index 00000000..4a9531de --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/ioctl_channel.c @@ -0,0 +1,1179 @@ +/* + * GK20A Graphics channel + * + * Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "gk20a/gk20a.h" +#include "gk20a/debug_gk20a.h" +#include "gk20a/ctxsw_trace_gk20a.h" +#include "gk20a/dbg_gpu_gk20a.h" +#include "gk20a/fence_gk20a.h" + +/* + * Although channels do have pointers back to the gk20a struct that they were + * created under in cases where the driver is killed that pointer can be bad. + * The channel memory can be freed before the release() function for a given + * channel is called. This happens when the driver dies and userspace doesn't + * get a chance to call release() until after the entire gk20a driver data is + * unloaded and freed. + */ +struct channel_priv { + struct gk20a *g; + struct channel_gk20a *c; +}; + +#if defined(CONFIG_GK20A_CYCLE_STATS) + +static int gk20a_channel_cycle_stats(struct channel_gk20a *ch, + struct nvgpu_cycle_stats_args *args) +{ + struct dma_buf *dmabuf; + void *virtual_address; + + /* is it allowed to handle calls for current GPU? */ + if (0 == (ch->g->gpu_characteristics.flags & + NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS)) + return -ENOSYS; + + if (args->dmabuf_fd && !ch->cyclestate.cyclestate_buffer_handler) { + + /* set up new cyclestats buffer */ + dmabuf = dma_buf_get(args->dmabuf_fd); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + virtual_address = dma_buf_vmap(dmabuf); + if (!virtual_address) + return -ENOMEM; + + ch->cyclestate.cyclestate_buffer_handler = dmabuf; + ch->cyclestate.cyclestate_buffer = virtual_address; + ch->cyclestate.cyclestate_buffer_size = dmabuf->size; + return 0; + + } else if (!args->dmabuf_fd && + ch->cyclestate.cyclestate_buffer_handler) { + gk20a_channel_free_cycle_stats_buffer(ch); + return 0; + + } else if (!args->dmabuf_fd && + !ch->cyclestate.cyclestate_buffer_handler) { + /* no requst from GL */ + return 0; + + } else { + pr_err("channel already has cyclestats buffer\n"); + return -EINVAL; + } +} + +static int gk20a_flush_cycle_stats_snapshot(struct channel_gk20a *ch) +{ + int ret; + + nvgpu_mutex_acquire(&ch->cs_client_mutex); + if (ch->cs_client) + ret = gr_gk20a_css_flush(ch, ch->cs_client); + else + ret = -EBADF; + nvgpu_mutex_release(&ch->cs_client_mutex); + + return ret; +} + +static int gk20a_attach_cycle_stats_snapshot(struct channel_gk20a *ch, + u32 dmabuf_fd, + u32 perfmon_id_count, + u32 *perfmon_id_start) +{ + int ret; + + nvgpu_mutex_acquire(&ch->cs_client_mutex); + if (ch->cs_client) { + ret = -EEXIST; + } else { + ret = gr_gk20a_css_attach(ch, + dmabuf_fd, + perfmon_id_count, + perfmon_id_start, + &ch->cs_client); + } + nvgpu_mutex_release(&ch->cs_client_mutex); + + return ret; +} + +static int gk20a_channel_cycle_stats_snapshot(struct channel_gk20a *ch, + struct nvgpu_cycle_stats_snapshot_args *args) +{ + int ret; + + /* is it allowed to handle calls for current GPU? */ + if (0 == (ch->g->gpu_characteristics.flags & + NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS_SNAPSHOT)) + return -ENOSYS; + + if (!args->dmabuf_fd) + return -EINVAL; + + /* handle the command (most frequent cases first) */ + switch (args->cmd) { + case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_FLUSH: + ret = gk20a_flush_cycle_stats_snapshot(ch); + args->extra = 0; + break; + + case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_ATTACH: + ret = gk20a_attach_cycle_stats_snapshot(ch, + args->dmabuf_fd, + args->extra, + &args->extra); + break; + + case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_DETACH: + ret = gk20a_channel_free_cycle_stats_snapshot(ch); + args->extra = 0; + break; + + default: + pr_err("cyclestats: unknown command %u\n", args->cmd); + ret = -EINVAL; + break; + } + + return ret; +} +#endif + +static int gk20a_channel_set_wdt_status(struct channel_gk20a *ch, + struct nvgpu_channel_wdt_args *args) +{ + if (args->wdt_status == NVGPU_IOCTL_CHANNEL_DISABLE_WDT) + ch->wdt_enabled = false; + else if (args->wdt_status == NVGPU_IOCTL_CHANNEL_ENABLE_WDT) + ch->wdt_enabled = true; + + return 0; +} + +static int gk20a_init_error_notifier(struct channel_gk20a *ch, + struct nvgpu_set_error_notifier *args) +{ + struct device *dev = dev_from_gk20a(ch->g); + struct dma_buf *dmabuf; + void *va; + u64 end = args->offset + sizeof(struct nvgpu_notification); + + if (!args->mem) { + pr_err("gk20a_init_error_notifier: invalid memory handle\n"); + return -EINVAL; + } + + dmabuf = dma_buf_get(args->mem); + + gk20a_channel_free_error_notifiers(ch); + + if (IS_ERR(dmabuf)) { + pr_err("Invalid handle: %d\n", args->mem); + return -EINVAL; + } + + if (end > dmabuf->size || end < sizeof(struct nvgpu_notification)) { + dma_buf_put(dmabuf); + gk20a_err(dev, "gk20a_init_error_notifier: invalid offset\n"); + return -EINVAL; + } + + /* map handle */ + va = dma_buf_vmap(dmabuf); + if (!va) { + dma_buf_put(dmabuf); + pr_err("Cannot map notifier handle\n"); + return -ENOMEM; + } + + ch->error_notifier = va + args->offset; + ch->error_notifier_va = va; + memset(ch->error_notifier, 0, sizeof(struct nvgpu_notification)); + + /* set channel notifiers pointer */ + nvgpu_mutex_acquire(&ch->error_notifier_mutex); + ch->error_notifier_ref = dmabuf; + nvgpu_mutex_release(&ch->error_notifier_mutex); + + return 0; +} + +struct channel_gk20a *gk20a_get_channel_from_file(int fd) +{ + struct channel_priv *priv; + struct file *f = fget(fd); + + if (!f) + return NULL; + + if (f->f_op != &gk20a_channel_ops) { + fput(f); + return NULL; + } + + priv = (struct channel_priv *)f->private_data; + fput(f); + return priv->c; +} + +int gk20a_channel_release(struct inode *inode, struct file *filp) +{ + struct channel_priv *priv = filp->private_data; + struct channel_gk20a *ch = priv->c; + struct gk20a *g = priv->g; + + int err; + + err = gk20a_busy(g); + if (err) { + gk20a_err(dev_from_gk20a(g), "failed to release a channel!"); + goto channel_release; + } + + trace_gk20a_channel_release(dev_name(g->dev)); + + gk20a_channel_close(ch); + gk20a_idle(g); + +channel_release: + gk20a_put(g); + nvgpu_kfree(g, filp->private_data); + filp->private_data = NULL; + return 0; +} + +/* note: runlist_id -1 is synonym for the ENGINE_GR_GK20A runlist id */ +static int __gk20a_channel_open(struct gk20a *g, + struct file *filp, s32 runlist_id) +{ + int err; + struct channel_gk20a *ch; + struct channel_priv *priv; + + gk20a_dbg_fn(""); + + g = gk20a_get(g); + if (!g) + return -ENODEV; + + trace_gk20a_channel_open(dev_name(g->dev)); + + priv = nvgpu_kzalloc(g, sizeof(*priv)); + if (!priv) { + err = -ENOMEM; + goto free_ref; + } + + err = gk20a_busy(g); + if (err) { + gk20a_err(dev_from_gk20a(g), "failed to power on, %d", err); + goto fail_busy; + } + /* All the user space channel should be non privilege */ + ch = gk20a_open_new_channel(g, runlist_id, false); + gk20a_idle(g); + if (!ch) { + gk20a_err(dev_from_gk20a(g), + "failed to get f"); + err = -ENOMEM; + goto fail_busy; + } + + gk20a_channel_trace_sched_param( + trace_gk20a_channel_sched_defaults, ch); + + priv->g = g; + priv->c = ch; + + filp->private_data = priv; + return 0; + +fail_busy: + nvgpu_kfree(g, priv); +free_ref: + gk20a_put(g); + return err; +} + +int gk20a_channel_open(struct inode *inode, struct file *filp) +{ + struct gk20a *g = container_of(inode->i_cdev, + struct gk20a, channel.cdev); + int ret; + + gk20a_dbg_fn("start"); + ret = __gk20a_channel_open(g, filp, -1); + + gk20a_dbg_fn("end"); + return ret; +} + +int gk20a_channel_open_ioctl(struct gk20a *g, + struct nvgpu_channel_open_args *args) +{ + int err; + int fd; + struct file *file; + char name[64]; + s32 runlist_id = args->in.runlist_id; + + err = get_unused_fd_flags(O_RDWR); + if (err < 0) + return err; + fd = err; + + snprintf(name, sizeof(name), "nvhost-%s-fd%d", + dev_name(g->dev), fd); + + file = anon_inode_getfile(name, g->channel.cdev.ops, NULL, O_RDWR); + if (IS_ERR(file)) { + err = PTR_ERR(file); + goto clean_up; + } + + err = __gk20a_channel_open(g, file, runlist_id); + if (err) + goto clean_up_file; + + fd_install(fd, file); + args->out.channel_fd = fd; + return 0; + +clean_up_file: + fput(file); +clean_up: + put_unused_fd(fd); + return err; +} + +int nvgpu_channel_ioctl_alloc_gpfifo(struct channel_gk20a *c, + struct nvgpu_alloc_gpfifo_ex_args *args) +{ + return gk20a_channel_alloc_gpfifo(c, args->num_entries, + args->num_inflight_jobs, + args->flags); +} + + +static int gk20a_channel_wait_semaphore(struct channel_gk20a *ch, + ulong id, u32 offset, + u32 payload, long timeout) +{ + struct device *dev = ch->g->dev; + struct dma_buf *dmabuf; + void *data; + u32 *semaphore; + int ret = 0; + long remain; + + /* do not wait if channel has timed out */ + if (ch->has_timedout) + return -ETIMEDOUT; + + dmabuf = dma_buf_get(id); + if (IS_ERR(dmabuf)) { + gk20a_err(dev, "invalid notifier nvmap handle 0x%lx", id); + return -EINVAL; + } + + data = dma_buf_kmap(dmabuf, offset >> PAGE_SHIFT); + if (!data) { + gk20a_err(dev, "failed to map notifier memory"); + ret = -EINVAL; + goto cleanup_put; + } + + semaphore = data + (offset & ~PAGE_MASK); + + remain = wait_event_interruptible_timeout( + ch->semaphore_wq, + *semaphore == payload || ch->has_timedout, + timeout); + + if (remain == 0 && *semaphore != payload) + ret = -ETIMEDOUT; + else if (remain < 0) + ret = remain; + + dma_buf_kunmap(dmabuf, offset >> PAGE_SHIFT, data); +cleanup_put: + dma_buf_put(dmabuf); + return ret; +} + +static int gk20a_channel_wait(struct channel_gk20a *ch, + struct nvgpu_wait_args *args) +{ + struct device *d = dev_from_gk20a(ch->g); + struct dma_buf *dmabuf; + struct notification *notif; + struct timespec tv; + u64 jiffies; + ulong id; + u32 offset; + unsigned long timeout; + int remain, ret = 0; + u64 end; + + gk20a_dbg_fn(""); + + if (ch->has_timedout) + return -ETIMEDOUT; + + if (args->timeout == NVGPU_NO_TIMEOUT) + timeout = MAX_SCHEDULE_TIMEOUT; + else + timeout = (u32)msecs_to_jiffies(args->timeout); + + switch (args->type) { + case NVGPU_WAIT_TYPE_NOTIFIER: + id = args->condition.notifier.dmabuf_fd; + offset = args->condition.notifier.offset; + end = offset + sizeof(struct notification); + + dmabuf = dma_buf_get(id); + if (IS_ERR(dmabuf)) { + gk20a_err(d, "invalid notifier nvmap handle 0x%lx", + id); + return -EINVAL; + } + + if (end > dmabuf->size || end < sizeof(struct notification)) { + dma_buf_put(dmabuf); + gk20a_err(d, "invalid notifier offset\n"); + return -EINVAL; + } + + notif = dma_buf_vmap(dmabuf); + if (!notif) { + gk20a_err(d, "failed to map notifier memory"); + return -ENOMEM; + } + + notif = (struct notification *)((uintptr_t)notif + offset); + + /* user should set status pending before + * calling this ioctl */ + remain = wait_event_interruptible_timeout( + ch->notifier_wq, + notif->status == 0 || ch->has_timedout, + timeout); + + if (remain == 0 && notif->status != 0) { + ret = -ETIMEDOUT; + goto notif_clean_up; + } else if (remain < 0) { + ret = -EINTR; + goto notif_clean_up; + } + + /* TBD: fill in correct information */ + jiffies = get_jiffies_64(); + jiffies_to_timespec(jiffies, &tv); + notif->timestamp.nanoseconds[0] = tv.tv_nsec; + notif->timestamp.nanoseconds[1] = tv.tv_sec; + notif->info32 = 0xDEADBEEF; /* should be object name */ + notif->info16 = ch->hw_chid; /* should be method offset */ + +notif_clean_up: + dma_buf_vunmap(dmabuf, notif); + return ret; + + case NVGPU_WAIT_TYPE_SEMAPHORE: + ret = gk20a_channel_wait_semaphore(ch, + args->condition.semaphore.dmabuf_fd, + args->condition.semaphore.offset, + args->condition.semaphore.payload, + timeout); + + break; + + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static unsigned int gk20a_event_id_poll(struct file *filep, poll_table *wait) +{ + unsigned int mask = 0; + struct gk20a_event_id_data *event_id_data = filep->private_data; + struct gk20a *g = event_id_data->g; + u32 event_id = event_id_data->event_id; + + gk20a_dbg(gpu_dbg_fn | gpu_dbg_info, ""); + + poll_wait(filep, &event_id_data->event_id_wq, wait); + + nvgpu_mutex_acquire(&event_id_data->lock); + + if (event_id_data->is_tsg) { + struct tsg_gk20a *tsg = g->fifo.tsg + event_id_data->id; + + if (event_id_data->event_posted) { + gk20a_dbg_info( + "found pending event_id=%d on TSG=%d\n", + event_id, tsg->tsgid); + mask = (POLLPRI | POLLIN); + event_id_data->event_posted = false; + } + } else { + struct channel_gk20a *ch = g->fifo.channel + + event_id_data->id; + + if (event_id_data->event_posted) { + gk20a_dbg_info( + "found pending event_id=%d on chid=%d\n", + event_id, ch->hw_chid); + mask = (POLLPRI | POLLIN); + event_id_data->event_posted = false; + } + } + + nvgpu_mutex_release(&event_id_data->lock); + + return mask; +} + +static int gk20a_event_id_release(struct inode *inode, struct file *filp) +{ + struct gk20a_event_id_data *event_id_data = filp->private_data; + struct gk20a *g = event_id_data->g; + + if (event_id_data->is_tsg) { + struct tsg_gk20a *tsg = g->fifo.tsg + event_id_data->id; + + nvgpu_mutex_acquire(&tsg->event_id_list_lock); + list_del_init(&event_id_data->event_id_node); + nvgpu_mutex_release(&tsg->event_id_list_lock); + } else { + struct channel_gk20a *ch = g->fifo.channel + event_id_data->id; + + nvgpu_mutex_acquire(&ch->event_id_list_lock); + list_del_init(&event_id_data->event_id_node); + nvgpu_mutex_release(&ch->event_id_list_lock); + } + + nvgpu_mutex_destroy(&event_id_data->lock); + gk20a_put(g); + nvgpu_kfree(g, event_id_data); + filp->private_data = NULL; + + return 0; +} + +const struct file_operations gk20a_event_id_ops = { + .owner = THIS_MODULE, + .poll = gk20a_event_id_poll, + .release = gk20a_event_id_release, +}; + +static int gk20a_channel_get_event_data_from_id(struct channel_gk20a *ch, + u32 event_id, + struct gk20a_event_id_data **event_id_data) +{ + struct gk20a_event_id_data *local_event_id_data; + bool event_found = false; + + nvgpu_mutex_acquire(&ch->event_id_list_lock); + list_for_each_entry(local_event_id_data, &ch->event_id_list, + event_id_node) { + if (local_event_id_data->event_id == event_id) { + event_found = true; + break; + } + } + nvgpu_mutex_release(&ch->event_id_list_lock); + + if (event_found) { + *event_id_data = local_event_id_data; + return 0; + } else { + return -1; + } +} + +void gk20a_channel_event_id_post_event(struct channel_gk20a *ch, + u32 event_id) +{ + struct gk20a_event_id_data *event_id_data; + int err = 0; + + err = gk20a_channel_get_event_data_from_id(ch, event_id, + &event_id_data); + if (err) + return; + + nvgpu_mutex_acquire(&event_id_data->lock); + + gk20a_dbg_info( + "posting event for event_id=%d on ch=%d\n", + event_id, ch->hw_chid); + event_id_data->event_posted = true; + + wake_up_interruptible_all(&event_id_data->event_id_wq); + + nvgpu_mutex_release(&event_id_data->lock); +} + +static int gk20a_channel_event_id_enable(struct channel_gk20a *ch, + int event_id, + int *fd) +{ + struct gk20a *g; + int err = 0; + int local_fd; + struct file *file; + char name[64]; + struct gk20a_event_id_data *event_id_data; + + g = gk20a_get(ch->g); + if (!g) + return -ENODEV; + + err = gk20a_channel_get_event_data_from_id(ch, + event_id, &event_id_data); + if (err == 0) { + /* We already have event enabled */ + err = -EINVAL; + goto free_ref; + } + + err = get_unused_fd_flags(O_RDWR); + if (err < 0) + goto free_ref; + local_fd = err; + + snprintf(name, sizeof(name), "nvgpu-event%d-fd%d", + event_id, local_fd); + file = anon_inode_getfile(name, &gk20a_event_id_ops, + NULL, O_RDWR); + if (IS_ERR(file)) { + err = PTR_ERR(file); + goto clean_up; + } + + event_id_data = nvgpu_kzalloc(ch->g, sizeof(*event_id_data)); + if (!event_id_data) { + err = -ENOMEM; + goto clean_up_file; + } + event_id_data->g = g; + event_id_data->id = ch->hw_chid; + event_id_data->is_tsg = false; + event_id_data->event_id = event_id; + + init_waitqueue_head(&event_id_data->event_id_wq); + err = nvgpu_mutex_init(&event_id_data->lock); + if (err) + goto clean_up_free; + INIT_LIST_HEAD(&event_id_data->event_id_node); + + nvgpu_mutex_acquire(&ch->event_id_list_lock); + list_add_tail(&event_id_data->event_id_node, &ch->event_id_list); + nvgpu_mutex_release(&ch->event_id_list_lock); + + fd_install(local_fd, file); + file->private_data = event_id_data; + + *fd = local_fd; + + return 0; + +clean_up_free: + kfree(event_id_data); +clean_up_file: + fput(file); +clean_up: + put_unused_fd(local_fd); +free_ref: + gk20a_put(g); + return err; +} + +static int gk20a_channel_event_id_ctrl(struct channel_gk20a *ch, + struct nvgpu_event_id_ctrl_args *args) +{ + int err = 0; + int fd = -1; + + if (args->event_id >= NVGPU_IOCTL_CHANNEL_EVENT_ID_MAX) + return -EINVAL; + + if (gk20a_is_channel_marked_as_tsg(ch)) + return -EINVAL; + + switch (args->cmd) { + case NVGPU_IOCTL_CHANNEL_EVENT_ID_CMD_ENABLE: + err = gk20a_channel_event_id_enable(ch, args->event_id, &fd); + if (!err) + args->event_fd = fd; + break; + + default: + gk20a_err(dev_from_gk20a(ch->g), + "unrecognized channel event id cmd: 0x%x", + args->cmd); + err = -EINVAL; + break; + } + + return err; +} + +static int gk20a_channel_zcull_bind(struct channel_gk20a *ch, + struct nvgpu_zcull_bind_args *args) +{ + struct gk20a *g = ch->g; + struct gr_gk20a *gr = &g->gr; + + gk20a_dbg_fn(""); + + return g->ops.gr.bind_ctxsw_zcull(g, gr, ch, + args->gpu_va, args->mode); +} + +static int gk20a_ioctl_channel_submit_gpfifo( + struct channel_gk20a *ch, + struct nvgpu_submit_gpfifo_args *args) +{ + struct gk20a_fence *fence_out; + struct fifo_profile_gk20a *profile = NULL; + + int ret = 0; + gk20a_dbg_fn(""); + +#ifdef CONFIG_DEBUG_FS + profile = gk20a_fifo_profile_acquire(ch->g); + + if (profile) + profile->timestamp[PROFILE_IOCTL_ENTRY] = sched_clock(); +#endif + if (ch->has_timedout) + return -ETIMEDOUT; + ret = gk20a_submit_channel_gpfifo(ch, NULL, args, args->num_entries, + args->flags, &args->fence, + &fence_out, false, profile); + + if (ret) + goto clean_up; + + /* Convert fence_out to something we can pass back to user space. */ + if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) { + if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE) { + int fd = gk20a_fence_install_fd(fence_out); + if (fd < 0) + ret = fd; + else + args->fence.id = fd; + } else { + args->fence.id = fence_out->syncpt_id; + args->fence.value = fence_out->syncpt_value; + } + } + gk20a_fence_put(fence_out); +#ifdef CONFIG_DEBUG_FS + if (profile) { + profile->timestamp[PROFILE_IOCTL_EXIT] = sched_clock(); + gk20a_fifo_profile_release(ch->g, profile); + } +#endif +clean_up: + return ret; +} + +long gk20a_channel_ioctl(struct file *filp, + unsigned int cmd, unsigned long arg) +{ + struct channel_priv *priv = filp->private_data; + struct channel_gk20a *ch = priv->c; + struct device *dev = ch->g->dev; + u8 buf[NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE] = {0}; + int err = 0; + + gk20a_dbg_fn("start %d", _IOC_NR(cmd)); + + if ((_IOC_TYPE(cmd) != NVGPU_IOCTL_MAGIC) || + (_IOC_NR(cmd) == 0) || + (_IOC_NR(cmd) > NVGPU_IOCTL_CHANNEL_LAST) || + (_IOC_SIZE(cmd) > NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE)) + return -EINVAL; + + if (_IOC_DIR(cmd) & _IOC_WRITE) { + if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd))) + return -EFAULT; + } + + /* take a ref or return timeout if channel refs can't be taken */ + ch = gk20a_channel_get(ch); + if (!ch) + return -ETIMEDOUT; + + /* protect our sanity for threaded userspace - most of the channel is + * not thread safe */ + nvgpu_mutex_acquire(&ch->ioctl_lock); + + /* this ioctl call keeps a ref to the file which keeps a ref to the + * channel */ + + switch (cmd) { + case NVGPU_IOCTL_CHANNEL_OPEN: + err = gk20a_channel_open_ioctl(ch->g, + (struct nvgpu_channel_open_args *)buf); + break; + case NVGPU_IOCTL_CHANNEL_SET_NVMAP_FD: + break; + case NVGPU_IOCTL_CHANNEL_ALLOC_OBJ_CTX: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = ch->g->ops.gr.alloc_obj_ctx(ch, + (struct nvgpu_alloc_obj_ctx_args *)buf); + gk20a_idle(ch->g); + break; + case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO_EX: + { + struct nvgpu_alloc_gpfifo_ex_args *alloc_gpfifo_ex_args = + (struct nvgpu_alloc_gpfifo_ex_args *)buf; + + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + + if (!is_power_of_2(alloc_gpfifo_ex_args->num_entries)) { + err = -EINVAL; + gk20a_idle(ch->g); + break; + } + err = gk20a_channel_alloc_gpfifo(ch, + alloc_gpfifo_ex_args->num_entries, + alloc_gpfifo_ex_args->num_inflight_jobs, + alloc_gpfifo_ex_args->flags); + gk20a_idle(ch->g); + break; + } + case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO: + { + struct nvgpu_alloc_gpfifo_args *alloc_gpfifo_args = + (struct nvgpu_alloc_gpfifo_args *)buf; + + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + + /* + * Kernel can insert one extra gpfifo entry before user + * submitted gpfifos and another one after, for internal usage. + * Triple the requested size. + */ + err = gk20a_channel_alloc_gpfifo(ch, + alloc_gpfifo_args->num_entries * 3, + 0, + alloc_gpfifo_args->flags); + gk20a_idle(ch->g); + break; + } + case NVGPU_IOCTL_CHANNEL_SUBMIT_GPFIFO: + err = gk20a_ioctl_channel_submit_gpfifo(ch, + (struct nvgpu_submit_gpfifo_args *)buf); + break; + case NVGPU_IOCTL_CHANNEL_WAIT: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + + /* waiting is thread-safe, not dropping this mutex could + * deadlock in certain conditions */ + nvgpu_mutex_release(&ch->ioctl_lock); + + err = gk20a_channel_wait(ch, + (struct nvgpu_wait_args *)buf); + + nvgpu_mutex_acquire(&ch->ioctl_lock); + + gk20a_idle(ch->g); + break; + case NVGPU_IOCTL_CHANNEL_ZCULL_BIND: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = gk20a_channel_zcull_bind(ch, + (struct nvgpu_zcull_bind_args *)buf); + gk20a_idle(ch->g); + break; + case NVGPU_IOCTL_CHANNEL_SET_ERROR_NOTIFIER: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = gk20a_init_error_notifier(ch, + (struct nvgpu_set_error_notifier *)buf); + gk20a_idle(ch->g); + break; +#ifdef CONFIG_GK20A_CYCLE_STATS + case NVGPU_IOCTL_CHANNEL_CYCLE_STATS: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = gk20a_channel_cycle_stats(ch, + (struct nvgpu_cycle_stats_args *)buf); + gk20a_idle(ch->g); + break; +#endif + case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT: + { + u32 timeout = + (u32)((struct nvgpu_set_timeout_args *)buf)->timeout; + gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d", + timeout, ch->hw_chid); + ch->timeout_ms_max = timeout; + gk20a_channel_trace_sched_param( + trace_gk20a_channel_set_timeout, ch); + break; + } + case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT_EX: + { + u32 timeout = + (u32)((struct nvgpu_set_timeout_args *)buf)->timeout; + bool timeout_debug_dump = !((u32) + ((struct nvgpu_set_timeout_ex_args *)buf)->flags & + (1 << NVGPU_TIMEOUT_FLAG_DISABLE_DUMP)); + gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d", + timeout, ch->hw_chid); + ch->timeout_ms_max = timeout; + ch->timeout_debug_dump = timeout_debug_dump; + gk20a_channel_trace_sched_param( + trace_gk20a_channel_set_timeout, ch); + break; + } + case NVGPU_IOCTL_CHANNEL_GET_TIMEDOUT: + ((struct nvgpu_get_param_args *)buf)->value = + ch->has_timedout; + break; + case NVGPU_IOCTL_CHANNEL_SET_PRIORITY: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = ch->g->ops.fifo.channel_set_priority(ch, + ((struct nvgpu_set_priority_args *)buf)->priority); + + gk20a_idle(ch->g); + gk20a_channel_trace_sched_param( + trace_gk20a_channel_set_priority, ch); + break; + case NVGPU_IOCTL_CHANNEL_ENABLE: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + if (ch->g->ops.fifo.enable_channel) + ch->g->ops.fifo.enable_channel(ch); + else + err = -ENOSYS; + gk20a_idle(ch->g); + break; + case NVGPU_IOCTL_CHANNEL_DISABLE: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + if (ch->g->ops.fifo.disable_channel) + ch->g->ops.fifo.disable_channel(ch); + else + err = -ENOSYS; + gk20a_idle(ch->g); + break; + case NVGPU_IOCTL_CHANNEL_PREEMPT: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = gk20a_fifo_preempt(ch->g, ch); + gk20a_idle(ch->g); + break; + case NVGPU_IOCTL_CHANNEL_FORCE_RESET: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = ch->g->ops.fifo.force_reset_ch(ch, + NVGPU_CHANNEL_RESETCHANNEL_VERIF_ERROR, true); + gk20a_idle(ch->g); + break; + case NVGPU_IOCTL_CHANNEL_EVENT_ID_CTRL: + err = gk20a_channel_event_id_ctrl(ch, + (struct nvgpu_event_id_ctrl_args *)buf); + break; +#ifdef CONFIG_GK20A_CYCLE_STATS + case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = gk20a_channel_cycle_stats_snapshot(ch, + (struct nvgpu_cycle_stats_snapshot_args *)buf); + gk20a_idle(ch->g); + break; +#endif + case NVGPU_IOCTL_CHANNEL_WDT: + err = gk20a_channel_set_wdt_status(ch, + (struct nvgpu_channel_wdt_args *)buf); + break; + case NVGPU_IOCTL_CHANNEL_SET_RUNLIST_INTERLEAVE: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = gk20a_channel_set_runlist_interleave(ch, + ((struct nvgpu_runlist_interleave_args *)buf)->level); + + gk20a_idle(ch->g); + gk20a_channel_trace_sched_param( + trace_gk20a_channel_set_runlist_interleave, ch); + break; + case NVGPU_IOCTL_CHANNEL_SET_TIMESLICE: + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = ch->g->ops.fifo.channel_set_timeslice(ch, + ((struct nvgpu_timeslice_args *)buf)->timeslice_us); + + gk20a_idle(ch->g); + gk20a_channel_trace_sched_param( + trace_gk20a_channel_set_timeslice, ch); + break; + case NVGPU_IOCTL_CHANNEL_SET_PREEMPTION_MODE: + if (ch->g->ops.gr.set_preemption_mode) { + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = ch->g->ops.gr.set_preemption_mode(ch, + ((struct nvgpu_preemption_mode_args *)buf)->graphics_preempt_mode, + ((struct nvgpu_preemption_mode_args *)buf)->compute_preempt_mode); + gk20a_idle(ch->g); + } else { + err = -EINVAL; + } + break; + case NVGPU_IOCTL_CHANNEL_SET_BOOSTED_CTX: + if (ch->g->ops.gr.set_boosted_ctx) { + bool boost = + ((struct nvgpu_boosted_ctx_args *)buf)->boost; + + err = gk20a_busy(ch->g); + if (err) { + dev_err(dev, + "%s: failed to host gk20a for ioctl cmd: 0x%x", + __func__, cmd); + break; + } + err = ch->g->ops.gr.set_boosted_ctx(ch, boost); + gk20a_idle(ch->g); + } else { + err = -EINVAL; + } + break; + default: + dev_dbg(dev, "unrecognized ioctl cmd: 0x%x", cmd); + err = -ENOTTY; + break; + } + + if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ)) + err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd)); + + nvgpu_mutex_release(&ch->ioctl_lock); + + gk20a_channel_put(ch); + + gk20a_dbg_fn("end"); + + return err; +} -- cgit v1.2.2