From a07e10f494c158ae31d6187e9be3db409528a507 Mon Sep 17 00:00:00 2001 From: Terje Bergstrom Date: Thu, 23 Mar 2017 14:19:01 -0700 Subject: gpu: nvgpu: Move channel IOCTL code to Linux module Move channel IOCTL specific code to Linux module. This clears some Linux dependencies from channel_gk20a.c. JIRA NVGPU-32 Change-Id: I41817d612b959709365bcabff9c8a15f2bfe4c60 Signed-off-by: Terje Bergstrom Reviewed-on: http://git-master/r/1330804 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/gk20a/channel_gk20a.c | 1281 ++----------------------------- 1 file changed, 75 insertions(+), 1206 deletions(-) (limited to 'drivers/gpu/nvgpu/gk20a/channel_gk20a.c') diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c index cd49b4a9..b7306369 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c @@ -16,15 +16,10 @@ * along with this program. If not, see . */ -#include #include #include -#include /* need for nvmap.h*/ #include #include -#include -#include -#include #include #include @@ -40,8 +35,6 @@ #include -#define NVMAP_HANDLE_PARAM_SIZE 1 - /* * Although channels do have pointers back to the gk20a struct that they were * created under in cases where the driver is killed that pointer can be bad. @@ -55,7 +48,6 @@ struct channel_priv { struct channel_gk20a *c; }; -static struct channel_gk20a *allocate_channel(struct fifo_gk20a *f); static void free_channel(struct fifo_gk20a *f, struct channel_gk20a *c); static void gk20a_channel_dump_ref_actions(struct channel_gk20a *c); @@ -76,7 +68,6 @@ static struct channel_gk20a_job *channel_gk20a_joblist_peek( static int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add); -static void gk20a_free_error_notifiers(struct channel_gk20a *ch); static u32 gk20a_get_channel_watchdog_timeout(struct channel_gk20a *ch); @@ -323,171 +314,6 @@ void gk20a_disable_channel(struct channel_gk20a *ch) channel_gk20a_update_runlist(ch, false); } -#if defined(CONFIG_GK20A_CYCLE_STATS) - -static void gk20a_free_cycle_stats_buffer(struct channel_gk20a *ch) -{ - /* disable existing cyclestats buffer */ - nvgpu_mutex_acquire(&ch->cyclestate.cyclestate_buffer_mutex); - if (ch->cyclestate.cyclestate_buffer_handler) { - dma_buf_vunmap(ch->cyclestate.cyclestate_buffer_handler, - ch->cyclestate.cyclestate_buffer); - dma_buf_put(ch->cyclestate.cyclestate_buffer_handler); - ch->cyclestate.cyclestate_buffer_handler = NULL; - ch->cyclestate.cyclestate_buffer = NULL; - ch->cyclestate.cyclestate_buffer_size = 0; - } - nvgpu_mutex_release(&ch->cyclestate.cyclestate_buffer_mutex); -} - -static int gk20a_channel_cycle_stats(struct channel_gk20a *ch, - struct nvgpu_cycle_stats_args *args) -{ - struct dma_buf *dmabuf; - void *virtual_address; - - /* is it allowed to handle calls for current GPU? */ - if (0 == (ch->g->gpu_characteristics.flags & - NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS)) - return -ENOSYS; - - if (args->dmabuf_fd && !ch->cyclestate.cyclestate_buffer_handler) { - - /* set up new cyclestats buffer */ - dmabuf = dma_buf_get(args->dmabuf_fd); - if (IS_ERR(dmabuf)) - return PTR_ERR(dmabuf); - virtual_address = dma_buf_vmap(dmabuf); - if (!virtual_address) - return -ENOMEM; - - ch->cyclestate.cyclestate_buffer_handler = dmabuf; - ch->cyclestate.cyclestate_buffer = virtual_address; - ch->cyclestate.cyclestate_buffer_size = dmabuf->size; - return 0; - - } else if (!args->dmabuf_fd && - ch->cyclestate.cyclestate_buffer_handler) { - gk20a_free_cycle_stats_buffer(ch); - return 0; - - } else if (!args->dmabuf_fd && - !ch->cyclestate.cyclestate_buffer_handler) { - /* no requst from GL */ - return 0; - - } else { - pr_err("channel already has cyclestats buffer\n"); - return -EINVAL; - } -} - - -static int gk20a_flush_cycle_stats_snapshot(struct channel_gk20a *ch) -{ - int ret; - - nvgpu_mutex_acquire(&ch->cs_client_mutex); - if (ch->cs_client) - ret = gr_gk20a_css_flush(ch, ch->cs_client); - else - ret = -EBADF; - nvgpu_mutex_release(&ch->cs_client_mutex); - - return ret; -} - -static int gk20a_attach_cycle_stats_snapshot(struct channel_gk20a *ch, - u32 dmabuf_fd, - u32 perfmon_id_count, - u32 *perfmon_id_start) -{ - int ret; - - nvgpu_mutex_acquire(&ch->cs_client_mutex); - if (ch->cs_client) { - ret = -EEXIST; - } else { - ret = gr_gk20a_css_attach(ch, - dmabuf_fd, - perfmon_id_count, - perfmon_id_start, - &ch->cs_client); - } - nvgpu_mutex_release(&ch->cs_client_mutex); - - return ret; -} - -static int gk20a_free_cycle_stats_snapshot(struct channel_gk20a *ch) -{ - int ret; - - nvgpu_mutex_acquire(&ch->cs_client_mutex); - if (ch->cs_client) { - ret = gr_gk20a_css_detach(ch, ch->cs_client); - ch->cs_client = NULL; - } else { - ret = 0; - } - nvgpu_mutex_release(&ch->cs_client_mutex); - - return ret; -} - -static int gk20a_channel_cycle_stats_snapshot(struct channel_gk20a *ch, - struct nvgpu_cycle_stats_snapshot_args *args) -{ - int ret; - - /* is it allowed to handle calls for current GPU? */ - if (0 == (ch->g->gpu_characteristics.flags & - NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS_SNAPSHOT)) - return -ENOSYS; - - if (!args->dmabuf_fd) - return -EINVAL; - - /* handle the command (most frequent cases first) */ - switch (args->cmd) { - case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_FLUSH: - ret = gk20a_flush_cycle_stats_snapshot(ch); - args->extra = 0; - break; - - case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_ATTACH: - ret = gk20a_attach_cycle_stats_snapshot(ch, - args->dmabuf_fd, - args->extra, - &args->extra); - break; - - case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_DETACH: - ret = gk20a_free_cycle_stats_snapshot(ch); - args->extra = 0; - break; - - default: - pr_err("cyclestats: unknown command %u\n", args->cmd); - ret = -EINVAL; - break; - } - - return ret; -} -#endif - -static int gk20a_channel_set_wdt_status(struct channel_gk20a *ch, - struct nvgpu_channel_wdt_args *args) -{ - if (args->wdt_status == NVGPU_IOCTL_CHANNEL_DISABLE_WDT) - ch->wdt_enabled = false; - else if (args->wdt_status == NVGPU_IOCTL_CHANNEL_ENABLE_WDT) - ch->wdt_enabled = true; - - return 0; -} - int gk20a_channel_set_runlist_interleave(struct channel_gk20a *ch, u32 level) { @@ -516,54 +342,6 @@ int gk20a_channel_set_runlist_interleave(struct channel_gk20a *ch, return ret ? ret : g->ops.fifo.update_runlist(g, ch->runlist_id, ~0, true, true); } -static int gk20a_init_error_notifier(struct channel_gk20a *ch, - struct nvgpu_set_error_notifier *args) -{ - struct device *dev = dev_from_gk20a(ch->g); - struct dma_buf *dmabuf; - void *va; - u64 end = args->offset + sizeof(struct nvgpu_notification); - - if (!args->mem) { - pr_err("gk20a_init_error_notifier: invalid memory handle\n"); - return -EINVAL; - } - - dmabuf = dma_buf_get(args->mem); - - gk20a_free_error_notifiers(ch); - - if (IS_ERR(dmabuf)) { - pr_err("Invalid handle: %d\n", args->mem); - return -EINVAL; - } - - if (end > dmabuf->size || end < sizeof(struct nvgpu_notification)) { - dma_buf_put(dmabuf); - gk20a_err(dev, "gk20a_init_error_notifier: invalid offset\n"); - return -EINVAL; - } - - /* map handle */ - va = dma_buf_vmap(dmabuf); - if (!va) { - dma_buf_put(dmabuf); - pr_err("Cannot map notifier handle\n"); - return -ENOMEM; - } - - ch->error_notifier = va + args->offset; - ch->error_notifier_va = va; - memset(ch->error_notifier, 0, sizeof(struct nvgpu_notification)); - - /* set channel notifiers pointer */ - nvgpu_mutex_acquire(&ch->error_notifier_mutex); - ch->error_notifier_ref = dmabuf; - nvgpu_mutex_release(&ch->error_notifier_mutex); - - return 0; -} - /** * gk20a_set_error_notifier_locked() * Should be called with ch->error_notifier_mutex held @@ -595,7 +373,7 @@ void gk20a_set_error_notifier(struct channel_gk20a *ch, __u32 error) nvgpu_mutex_release(&ch->error_notifier_mutex); } -static void gk20a_free_error_notifiers(struct channel_gk20a *ch) +void gk20a_channel_free_error_notifiers(struct channel_gk20a *ch) { nvgpu_mutex_acquire(&ch->error_notifier_mutex); if (ch->error_notifier_ref) { @@ -628,6 +406,40 @@ static void gk20a_wait_until_counter_is_N( } } +#if defined(CONFIG_GK20A_CYCLE_STATS) +void gk20a_channel_free_cycle_stats_buffer(struct channel_gk20a *ch) +{ + /* disable existing cyclestats buffer */ + nvgpu_mutex_acquire(&ch->cyclestate.cyclestate_buffer_mutex); + if (ch->cyclestate.cyclestate_buffer_handler) { + dma_buf_vunmap(ch->cyclestate.cyclestate_buffer_handler, + ch->cyclestate.cyclestate_buffer); + dma_buf_put(ch->cyclestate.cyclestate_buffer_handler); + ch->cyclestate.cyclestate_buffer_handler = NULL; + ch->cyclestate.cyclestate_buffer = NULL; + ch->cyclestate.cyclestate_buffer_size = 0; + } + nvgpu_mutex_release(&ch->cyclestate.cyclestate_buffer_mutex); +} + +int gk20a_channel_free_cycle_stats_snapshot(struct channel_gk20a *ch) +{ + int ret; + + nvgpu_mutex_acquire(&ch->cs_client_mutex); + if (ch->cs_client) { + ret = gr_gk20a_css_detach(ch, ch->cs_client); + ch->cs_client = NULL; + } else { + ret = 0; + } + nvgpu_mutex_release(&ch->cs_client_mutex); + + return ret; +} + +#endif + /* call ONLY when no references to the channel exist: after the last put */ static void gk20a_free_channel(struct channel_gk20a *ch, bool force) { @@ -700,7 +512,7 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force) gk20a_dbg_info("freeing bound channel context, timeout=%ld", timeout); - gk20a_free_error_notifiers(ch); + gk20a_channel_free_error_notifiers(ch); if (g->ops.fecs_trace.unbind_channel && !ch->vpr) g->ops.fecs_trace.unbind_channel(g, ch); @@ -715,8 +527,8 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force) memset(&ch->gpfifo, 0, sizeof(struct gpfifo_desc)); #if defined(CONFIG_GK20A_CYCLE_STATS) - gk20a_free_cycle_stats_buffer(ch); - gk20a_free_cycle_stats_snapshot(ch); + gk20a_channel_free_cycle_stats_buffer(ch); + gk20a_channel_free_cycle_stats_snapshot(ch); #endif channel_gk20a_free_priv_cmdbuf(ch); @@ -930,50 +742,6 @@ void __gk20a_channel_kill(struct channel_gk20a *ch) gk20a_free_channel(ch, true); } -struct channel_gk20a *gk20a_get_channel_from_file(int fd) -{ - struct channel_priv *priv; - struct file *f = fget(fd); - - if (!f) - return NULL; - - if (f->f_op != &gk20a_channel_ops) { - fput(f); - return NULL; - } - - priv = (struct channel_priv *)f->private_data; - fput(f); - return priv->c; -} - -int gk20a_channel_release(struct inode *inode, struct file *filp) -{ - struct channel_priv *priv = filp->private_data; - struct channel_gk20a *ch = priv->c; - struct gk20a *g = priv->g; - - int err; - - err = gk20a_busy(g); - if (err) { - gk20a_err(dev_from_gk20a(g), "failed to release a channel!"); - goto channel_release; - } - - trace_gk20a_channel_release(g->name); - - gk20a_channel_close(ch); - gk20a_idle(g); - -channel_release: - gk20a_put(g); - nvgpu_kfree(g, filp->private_data); - filp->private_data = NULL; - return 0; -} - static void gk20a_channel_update_runcb_fn(struct work_struct *work) { struct channel_gk20a *ch = @@ -1100,109 +868,6 @@ struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g, return ch; } -/* note: runlist_id -1 is synonym for the ENGINE_GR_GK20A runlist id */ -static int __gk20a_channel_open(struct gk20a *g, struct file *filp, s32 runlist_id) -{ - int err; - struct channel_gk20a *ch; - struct channel_priv *priv; - - gk20a_dbg_fn(""); - - g = gk20a_get(g); - if (!g) - return -ENODEV; - - trace_gk20a_channel_open(g->name); - - priv = nvgpu_kzalloc(g, sizeof(*priv)); - if (!priv) { - err = -ENOMEM; - goto free_ref; - } - - err = gk20a_busy(g); - if (err) { - gk20a_err(dev_from_gk20a(g), "failed to power on, %d", err); - goto fail_busy; - } - /* All the user space channel should be non privilege */ - ch = gk20a_open_new_channel(g, runlist_id, false); - gk20a_idle(g); - if (!ch) { - gk20a_err(dev_from_gk20a(g), - "failed to get f"); - err = -ENOMEM; - goto fail_busy; - } - - gk20a_channel_trace_sched_param( - trace_gk20a_channel_sched_defaults, ch); - - priv->g = g; - priv->c = ch; - - filp->private_data = priv; - return 0; - -fail_busy: - nvgpu_kfree(g, priv); -free_ref: - gk20a_put(g); - return err; -} - -int gk20a_channel_open(struct inode *inode, struct file *filp) -{ - struct gk20a *g = container_of(inode->i_cdev, - struct gk20a, channel.cdev); - int ret; - - gk20a_dbg_fn("start"); - ret = __gk20a_channel_open(g, filp, -1); - - gk20a_dbg_fn("end"); - return ret; -} - -int gk20a_channel_open_ioctl(struct gk20a *g, - struct nvgpu_channel_open_args *args) -{ - int err; - int fd; - struct file *file; - char name[64]; - s32 runlist_id = args->in.runlist_id; - - err = get_unused_fd_flags(O_RDWR); - if (err < 0) - return err; - fd = err; - - snprintf(name, sizeof(name), "nvhost-%s-fd%d", - g->name, fd); - - file = anon_inode_getfile(name, g->channel.cdev.ops, NULL, O_RDWR); - if (IS_ERR(file)) { - err = PTR_ERR(file); - goto clean_up; - } - - err = __gk20a_channel_open(g, file, runlist_id); - if (err) - goto clean_up_file; - - fd_install(fd, file); - args->out.channel_fd = fd; - return 0; - -clean_up_file: - fput(file); -clean_up: - put_unused_fd(fd); - return err; -} - /* allocate private cmd buffer. used for inserting commands before/after user submitted buffers. */ static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c) @@ -1542,8 +1207,10 @@ static void channel_gk20a_free_prealloc_resources(struct channel_gk20a *c) c->joblist.pre_alloc.enabled = false; } -int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, - struct nvgpu_alloc_gpfifo_ex_args *args) +int gk20a_channel_alloc_gpfifo(struct channel_gk20a *c, + unsigned int num_entries, + unsigned int num_inflight_jobs, + u32 flags) { struct gk20a *g = c->g; struct device *d = dev_from_gk20a(g); @@ -1553,12 +1220,12 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, int err = 0; unsigned long acquire_timeout; - gpfifo_size = args->num_entries; + gpfifo_size = num_entries; - if (args->flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_VPR_ENABLED) + if (flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_VPR_ENABLED) c->vpr = true; - if (args->flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_DETERMINISTIC) + if (flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_DETERMINISTIC) c->deterministic = true; /* an address space needs to have been bound at this point. */ @@ -1625,15 +1292,15 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, err = g->ops.fifo.setup_ramfc(c, c->gpfifo.mem.gpu_va, c->gpfifo.entry_num, - acquire_timeout, args->flags); + acquire_timeout, flags); if (err) goto clean_up_sync; /* TBD: setup engine contexts */ - if (args->num_inflight_jobs) { + if (num_inflight_jobs) { err = channel_gk20a_prealloc_resources(c, - args->num_inflight_jobs); + num_inflight_jobs); if (err) goto clean_up_sync; } @@ -1654,7 +1321,7 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, clean_up_priv_cmd: channel_gk20a_free_priv_cmdbuf(c); clean_up_prealloc: - if (args->num_inflight_jobs) + if (num_inflight_jobs) channel_gk20a_free_prealloc_resources(c); clean_up_sync: if (c->sync) { @@ -3033,419 +2700,39 @@ fail_1: return err; } -static int gk20a_channel_wait_semaphore(struct channel_gk20a *ch, - ulong id, u32 offset, - u32 payload, long timeout) +/* in this context the "channel" is the host1x channel which + * maps to *all* gk20a channels */ +int gk20a_channel_suspend(struct gk20a *g) { - struct device *dev = ch->g->dev; - struct dma_buf *dmabuf; - void *data; - u32 *semaphore; - int ret = 0; - long remain; + struct fifo_gk20a *f = &g->fifo; + u32 chid; + bool channels_in_use = false; + u32 active_runlist_ids = 0; - /* do not wait if channel has timed out */ - if (ch->has_timedout) - return -ETIMEDOUT; + gk20a_dbg_fn(""); - dmabuf = dma_buf_get(id); - if (IS_ERR(dmabuf)) { - gk20a_err(dev, "invalid notifier nvmap handle 0x%lx", id); - return -EINVAL; - } + for (chid = 0; chid < f->num_channels; chid++) { + struct channel_gk20a *ch = &f->channel[chid]; + if (gk20a_channel_get(ch)) { + gk20a_dbg_info("suspend channel %d", chid); + /* disable channel */ + gk20a_disable_channel_tsg(g, ch); + /* preempt the channel */ + gk20a_fifo_preempt(g, ch); + /* wait for channel update notifiers */ + if (ch->update_fn) + cancel_work_sync(&ch->update_fn_work); - data = dma_buf_kmap(dmabuf, offset >> PAGE_SHIFT); - if (!data) { - gk20a_err(dev, "failed to map notifier memory"); - ret = -EINVAL; - goto cleanup_put; - } + channels_in_use = true; - semaphore = data + (offset & ~PAGE_MASK); + active_runlist_ids |= BIT(ch->runlist_id); - remain = wait_event_interruptible_timeout( - ch->semaphore_wq, - *semaphore == payload || ch->has_timedout, - timeout); + gk20a_channel_put(ch); + } + } - if (remain == 0 && *semaphore != payload) - ret = -ETIMEDOUT; - else if (remain < 0) - ret = remain; - - dma_buf_kunmap(dmabuf, offset >> PAGE_SHIFT, data); -cleanup_put: - dma_buf_put(dmabuf); - return ret; -} - -static int gk20a_channel_wait(struct channel_gk20a *ch, - struct nvgpu_wait_args *args) -{ - struct device *d = dev_from_gk20a(ch->g); - struct dma_buf *dmabuf; - struct notification *notif; - struct timespec tv; - u64 jiffies; - ulong id; - u32 offset; - unsigned long timeout; - int remain, ret = 0; - u64 end; - - gk20a_dbg_fn(""); - - if (ch->has_timedout) - return -ETIMEDOUT; - - if (args->timeout == NVGPU_NO_TIMEOUT) - timeout = MAX_SCHEDULE_TIMEOUT; - else - timeout = (u32)msecs_to_jiffies(args->timeout); - - switch (args->type) { - case NVGPU_WAIT_TYPE_NOTIFIER: - id = args->condition.notifier.dmabuf_fd; - offset = args->condition.notifier.offset; - end = offset + sizeof(struct notification); - - dmabuf = dma_buf_get(id); - if (IS_ERR(dmabuf)) { - gk20a_err(d, "invalid notifier nvmap handle 0x%lx", - id); - return -EINVAL; - } - - if (end > dmabuf->size || end < sizeof(struct notification)) { - dma_buf_put(dmabuf); - gk20a_err(d, "invalid notifier offset\n"); - return -EINVAL; - } - - notif = dma_buf_vmap(dmabuf); - if (!notif) { - gk20a_err(d, "failed to map notifier memory"); - return -ENOMEM; - } - - notif = (struct notification *)((uintptr_t)notif + offset); - - /* user should set status pending before - * calling this ioctl */ - remain = wait_event_interruptible_timeout( - ch->notifier_wq, - notif->status == 0 || ch->has_timedout, - timeout); - - if (remain == 0 && notif->status != 0) { - ret = -ETIMEDOUT; - goto notif_clean_up; - } else if (remain < 0) { - ret = -EINTR; - goto notif_clean_up; - } - - /* TBD: fill in correct information */ - jiffies = get_jiffies_64(); - jiffies_to_timespec(jiffies, &tv); - notif->timestamp.nanoseconds[0] = tv.tv_nsec; - notif->timestamp.nanoseconds[1] = tv.tv_sec; - notif->info32 = 0xDEADBEEF; /* should be object name */ - notif->info16 = ch->hw_chid; /* should be method offset */ - -notif_clean_up: - dma_buf_vunmap(dmabuf, notif); - return ret; - - case NVGPU_WAIT_TYPE_SEMAPHORE: - ret = gk20a_channel_wait_semaphore(ch, - args->condition.semaphore.dmabuf_fd, - args->condition.semaphore.offset, - args->condition.semaphore.payload, - timeout); - - break; - - default: - ret = -EINVAL; - break; - } - - return ret; -} - -static unsigned int gk20a_event_id_poll(struct file *filep, poll_table *wait) -{ - unsigned int mask = 0; - struct gk20a_event_id_data *event_id_data = filep->private_data; - struct gk20a *g = event_id_data->g; - u32 event_id = event_id_data->event_id; - - gk20a_dbg(gpu_dbg_fn | gpu_dbg_info, ""); - - poll_wait(filep, &event_id_data->event_id_wq, wait); - - nvgpu_mutex_acquire(&event_id_data->lock); - - if (event_id_data->is_tsg) { - struct tsg_gk20a *tsg = g->fifo.tsg + event_id_data->id; - - if (event_id_data->event_posted) { - gk20a_dbg_info( - "found pending event_id=%d on TSG=%d\n", - event_id, tsg->tsgid); - mask = (POLLPRI | POLLIN); - event_id_data->event_posted = false; - } - } else { - struct channel_gk20a *ch = g->fifo.channel - + event_id_data->id; - - if (event_id_data->event_posted) { - gk20a_dbg_info( - "found pending event_id=%d on chid=%d\n", - event_id, ch->hw_chid); - mask = (POLLPRI | POLLIN); - event_id_data->event_posted = false; - } - } - - nvgpu_mutex_release(&event_id_data->lock); - - return mask; -} - -static int gk20a_event_id_release(struct inode *inode, struct file *filp) -{ - struct gk20a_event_id_data *event_id_data = filp->private_data; - struct gk20a *g = event_id_data->g; - - if (event_id_data->is_tsg) { - struct tsg_gk20a *tsg = g->fifo.tsg + event_id_data->id; - - nvgpu_mutex_acquire(&tsg->event_id_list_lock); - list_del_init(&event_id_data->event_id_node); - nvgpu_mutex_release(&tsg->event_id_list_lock); - } else { - struct channel_gk20a *ch = g->fifo.channel + event_id_data->id; - - nvgpu_mutex_acquire(&ch->event_id_list_lock); - list_del_init(&event_id_data->event_id_node); - nvgpu_mutex_release(&ch->event_id_list_lock); - } - - nvgpu_mutex_destroy(&event_id_data->lock); - gk20a_put(g); - nvgpu_kfree(g, event_id_data); - filp->private_data = NULL; - - return 0; -} - -const struct file_operations gk20a_event_id_ops = { - .owner = THIS_MODULE, - .poll = gk20a_event_id_poll, - .release = gk20a_event_id_release, -}; - -static int gk20a_channel_get_event_data_from_id(struct channel_gk20a *ch, - u32 event_id, - struct gk20a_event_id_data **event_id_data) -{ - struct gk20a_event_id_data *local_event_id_data; - bool event_found = false; - - nvgpu_mutex_acquire(&ch->event_id_list_lock); - list_for_each_entry(local_event_id_data, &ch->event_id_list, - event_id_node) { - if (local_event_id_data->event_id == event_id) { - event_found = true; - break; - } - } - nvgpu_mutex_release(&ch->event_id_list_lock); - - if (event_found) { - *event_id_data = local_event_id_data; - return 0; - } else { - return -1; - } -} - -void gk20a_channel_event_id_post_event(struct channel_gk20a *ch, - u32 event_id) -{ - struct gk20a_event_id_data *event_id_data; - int err = 0; - - err = gk20a_channel_get_event_data_from_id(ch, event_id, - &event_id_data); - if (err) - return; - - nvgpu_mutex_acquire(&event_id_data->lock); - - gk20a_dbg_info( - "posting event for event_id=%d on ch=%d\n", - event_id, ch->hw_chid); - event_id_data->event_posted = true; - - wake_up_interruptible_all(&event_id_data->event_id_wq); - - nvgpu_mutex_release(&event_id_data->lock); -} - -static int gk20a_channel_event_id_enable(struct channel_gk20a *ch, - int event_id, - int *fd) -{ - struct gk20a *g; - int err = 0; - int local_fd; - struct file *file; - char name[64]; - struct gk20a_event_id_data *event_id_data; - - g = gk20a_get(ch->g); - if (!g) - return -ENODEV; - - err = gk20a_channel_get_event_data_from_id(ch, - event_id, &event_id_data); - if (err == 0) { - /* We already have event enabled */ - err = -EINVAL; - goto free_ref; - } - - err = get_unused_fd_flags(O_RDWR); - if (err < 0) - goto free_ref; - local_fd = err; - - snprintf(name, sizeof(name), "nvgpu-event%d-fd%d", - event_id, local_fd); - file = anon_inode_getfile(name, &gk20a_event_id_ops, - NULL, O_RDWR); - if (IS_ERR(file)) { - err = PTR_ERR(file); - goto clean_up; - } - - event_id_data = nvgpu_kzalloc(ch->g, sizeof(*event_id_data)); - if (!event_id_data) { - err = -ENOMEM; - goto clean_up_file; - } - event_id_data->g = g; - event_id_data->id = ch->hw_chid; - event_id_data->is_tsg = false; - event_id_data->event_id = event_id; - - init_waitqueue_head(&event_id_data->event_id_wq); - err = nvgpu_mutex_init(&event_id_data->lock); - if (err) - goto clean_up_free; - INIT_LIST_HEAD(&event_id_data->event_id_node); - - nvgpu_mutex_acquire(&ch->event_id_list_lock); - list_add_tail(&event_id_data->event_id_node, &ch->event_id_list); - nvgpu_mutex_release(&ch->event_id_list_lock); - - fd_install(local_fd, file); - file->private_data = event_id_data; - - *fd = local_fd; - - return 0; - -clean_up_free: - kfree(event_id_data); -clean_up_file: - fput(file); -clean_up: - put_unused_fd(local_fd); -free_ref: - gk20a_put(g); - return err; -} - -static int gk20a_channel_event_id_ctrl(struct channel_gk20a *ch, - struct nvgpu_event_id_ctrl_args *args) -{ - int err = 0; - int fd = -1; - - if (args->event_id >= NVGPU_IOCTL_CHANNEL_EVENT_ID_MAX) - return -EINVAL; - - if (gk20a_is_channel_marked_as_tsg(ch)) - return -EINVAL; - - switch (args->cmd) { - case NVGPU_IOCTL_CHANNEL_EVENT_ID_CMD_ENABLE: - err = gk20a_channel_event_id_enable(ch, args->event_id, &fd); - if (!err) - args->event_fd = fd; - break; - - default: - gk20a_err(dev_from_gk20a(ch->g), - "unrecognized channel event id cmd: 0x%x", - args->cmd); - err = -EINVAL; - break; - } - - return err; -} - -static int gk20a_channel_zcull_bind(struct channel_gk20a *ch, - struct nvgpu_zcull_bind_args *args) -{ - struct gk20a *g = ch->g; - struct gr_gk20a *gr = &g->gr; - - gk20a_dbg_fn(""); - - return g->ops.gr.bind_ctxsw_zcull(g, gr, ch, - args->gpu_va, args->mode); -} - -/* in this context the "channel" is the host1x channel which - * maps to *all* gk20a channels */ -int gk20a_channel_suspend(struct gk20a *g) -{ - struct fifo_gk20a *f = &g->fifo; - u32 chid; - bool channels_in_use = false; - u32 active_runlist_ids = 0; - - gk20a_dbg_fn(""); - - for (chid = 0; chid < f->num_channels; chid++) { - struct channel_gk20a *ch = &f->channel[chid]; - if (gk20a_channel_get(ch)) { - gk20a_dbg_info("suspend channel %d", chid); - /* disable channel */ - gk20a_disable_channel_tsg(g, ch); - /* preempt the channel */ - gk20a_fifo_preempt(g, ch); - /* wait for channel update notifiers */ - if (ch->update_fn) - cancel_work_sync(&ch->update_fn_work); - - channels_in_use = true; - - active_runlist_ids |= BIT(ch->runlist_id); - - gk20a_channel_put(ch); - } - } - - if (channels_in_use) { - gk20a_fifo_update_runlist_ids(g, active_runlist_ids, ~0, false, true); + if (channels_in_use) { + gk20a_fifo_update_runlist_ids(g, active_runlist_ids, ~0, false, true); for (chid = 0; chid < f->num_channels; chid++) { if (gk20a_channel_get(&f->channel[chid])) { @@ -3533,421 +2820,3 @@ void gk20a_channel_semaphore_wakeup(struct gk20a *g, bool post_events) } } } - -static int gk20a_ioctl_channel_submit_gpfifo( - struct channel_gk20a *ch, - struct nvgpu_submit_gpfifo_args *args) -{ - struct gk20a_fence *fence_out; - struct fifo_profile_gk20a *profile = NULL; - - int ret = 0; - gk20a_dbg_fn(""); - -#ifdef CONFIG_DEBUG_FS - profile = gk20a_fifo_profile_acquire(ch->g); - - if (profile) - profile->timestamp[PROFILE_IOCTL_ENTRY] = sched_clock(); -#endif - if (ch->has_timedout) - return -ETIMEDOUT; - ret = gk20a_submit_channel_gpfifo(ch, NULL, args, args->num_entries, - args->flags, &args->fence, - &fence_out, false, profile); - - if (ret) - goto clean_up; - - /* Convert fence_out to something we can pass back to user space. */ - if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) { - if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE) { - int fd = gk20a_fence_install_fd(fence_out); - if (fd < 0) - ret = fd; - else - args->fence.id = fd; - } else { - args->fence.id = fence_out->syncpt_id; - args->fence.value = fence_out->syncpt_value; - } - } - gk20a_fence_put(fence_out); -#ifdef CONFIG_DEBUG_FS - if (profile) { - profile->timestamp[PROFILE_IOCTL_EXIT] = sched_clock(); - gk20a_fifo_profile_release(ch->g, profile); - } -#endif -clean_up: - return ret; -} - -long gk20a_channel_ioctl(struct file *filp, - unsigned int cmd, unsigned long arg) -{ - struct channel_priv *priv = filp->private_data; - struct channel_gk20a *ch = priv->c; - struct gk20a *g = ch->g; - struct device *dev = g->dev; - u8 buf[NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE] = {0}; - int err = 0; - - gk20a_dbg_fn("start %d", _IOC_NR(cmd)); - - if ((_IOC_TYPE(cmd) != NVGPU_IOCTL_MAGIC) || - (_IOC_NR(cmd) == 0) || - (_IOC_NR(cmd) > NVGPU_IOCTL_CHANNEL_LAST) || - (_IOC_SIZE(cmd) > NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE)) - return -EINVAL; - - if (_IOC_DIR(cmd) & _IOC_WRITE) { - if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd))) - return -EFAULT; - } - - /* take a ref or return timeout if channel refs can't be taken */ - ch = gk20a_channel_get(ch); - if (!ch) - return -ETIMEDOUT; - - /* protect our sanity for threaded userspace - most of the channel is - * not thread safe */ - nvgpu_mutex_acquire(&ch->ioctl_lock); - - /* this ioctl call keeps a ref to the file which keeps a ref to the - * channel */ - - switch (cmd) { - case NVGPU_IOCTL_CHANNEL_OPEN: - err = gk20a_channel_open_ioctl(ch->g, - (struct nvgpu_channel_open_args *)buf); - break; - case NVGPU_IOCTL_CHANNEL_SET_NVMAP_FD: - break; - case NVGPU_IOCTL_CHANNEL_ALLOC_OBJ_CTX: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = ch->g->ops.gr.alloc_obj_ctx(ch, - (struct nvgpu_alloc_obj_ctx_args *)buf); - gk20a_idle(g); - break; - case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO_EX: - { - struct nvgpu_alloc_gpfifo_ex_args *alloc_gpfifo_ex_args = - (struct nvgpu_alloc_gpfifo_ex_args *)buf; - - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - - if (!is_power_of_2(alloc_gpfifo_ex_args->num_entries)) { - err = -EINVAL; - gk20a_idle(g); - break; - } - err = gk20a_alloc_channel_gpfifo(ch, alloc_gpfifo_ex_args); - gk20a_idle(g); - break; - } - case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO: - { - struct nvgpu_alloc_gpfifo_ex_args alloc_gpfifo_ex_args; - struct nvgpu_alloc_gpfifo_args *alloc_gpfifo_args = - (struct nvgpu_alloc_gpfifo_args *)buf; - - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - - /* prepare new args structure */ - memset(&alloc_gpfifo_ex_args, 0, - sizeof(struct nvgpu_alloc_gpfifo_ex_args)); - /* - * Kernel can insert one extra gpfifo entry before user - * submitted gpfifos and another one after, for internal usage. - * Triple the requested size. - */ - alloc_gpfifo_ex_args.num_entries = roundup_pow_of_two( - alloc_gpfifo_args->num_entries * 3); - alloc_gpfifo_ex_args.flags = alloc_gpfifo_args->flags; - - err = gk20a_alloc_channel_gpfifo(ch, &alloc_gpfifo_ex_args); - gk20a_idle(g); - break; - } - case NVGPU_IOCTL_CHANNEL_SUBMIT_GPFIFO: - err = gk20a_ioctl_channel_submit_gpfifo(ch, - (struct nvgpu_submit_gpfifo_args *)buf); - break; - case NVGPU_IOCTL_CHANNEL_WAIT: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - - /* waiting is thread-safe, not dropping this mutex could - * deadlock in certain conditions */ - nvgpu_mutex_release(&ch->ioctl_lock); - - err = gk20a_channel_wait(ch, - (struct nvgpu_wait_args *)buf); - - nvgpu_mutex_acquire(&ch->ioctl_lock); - - gk20a_idle(g); - break; - case NVGPU_IOCTL_CHANNEL_ZCULL_BIND: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = gk20a_channel_zcull_bind(ch, - (struct nvgpu_zcull_bind_args *)buf); - gk20a_idle(g); - break; - case NVGPU_IOCTL_CHANNEL_SET_ERROR_NOTIFIER: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = gk20a_init_error_notifier(ch, - (struct nvgpu_set_error_notifier *)buf); - gk20a_idle(g); - break; -#ifdef CONFIG_GK20A_CYCLE_STATS - case NVGPU_IOCTL_CHANNEL_CYCLE_STATS: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = gk20a_channel_cycle_stats(ch, - (struct nvgpu_cycle_stats_args *)buf); - gk20a_idle(g); - break; -#endif - case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT: - { - u32 timeout = - (u32)((struct nvgpu_set_timeout_args *)buf)->timeout; - gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d", - timeout, ch->hw_chid); - ch->timeout_ms_max = timeout; - gk20a_channel_trace_sched_param( - trace_gk20a_channel_set_timeout, ch); - break; - } - case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT_EX: - { - u32 timeout = - (u32)((struct nvgpu_set_timeout_args *)buf)->timeout; - bool timeout_debug_dump = !((u32) - ((struct nvgpu_set_timeout_ex_args *)buf)->flags & - (1 << NVGPU_TIMEOUT_FLAG_DISABLE_DUMP)); - gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d", - timeout, ch->hw_chid); - ch->timeout_ms_max = timeout; - ch->timeout_debug_dump = timeout_debug_dump; - gk20a_channel_trace_sched_param( - trace_gk20a_channel_set_timeout, ch); - break; - } - case NVGPU_IOCTL_CHANNEL_GET_TIMEDOUT: - ((struct nvgpu_get_param_args *)buf)->value = - ch->has_timedout; - break; - case NVGPU_IOCTL_CHANNEL_SET_PRIORITY: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = ch->g->ops.fifo.channel_set_priority(ch, - ((struct nvgpu_set_priority_args *)buf)->priority); - - gk20a_idle(g); - gk20a_channel_trace_sched_param( - trace_gk20a_channel_set_priority, ch); - break; - case NVGPU_IOCTL_CHANNEL_ENABLE: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - if (ch->g->ops.fifo.enable_channel) - ch->g->ops.fifo.enable_channel(ch); - else - err = -ENOSYS; - gk20a_idle(g); - break; - case NVGPU_IOCTL_CHANNEL_DISABLE: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - if (ch->g->ops.fifo.disable_channel) - ch->g->ops.fifo.disable_channel(ch); - else - err = -ENOSYS; - gk20a_idle(g); - break; - case NVGPU_IOCTL_CHANNEL_PREEMPT: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = gk20a_fifo_preempt(ch->g, ch); - gk20a_idle(g); - break; - case NVGPU_IOCTL_CHANNEL_FORCE_RESET: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = ch->g->ops.fifo.force_reset_ch(ch, - NVGPU_CHANNEL_RESETCHANNEL_VERIF_ERROR, true); - gk20a_idle(g); - break; - case NVGPU_IOCTL_CHANNEL_EVENT_ID_CTRL: - err = gk20a_channel_event_id_ctrl(ch, - (struct nvgpu_event_id_ctrl_args *)buf); - break; -#ifdef CONFIG_GK20A_CYCLE_STATS - case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = gk20a_channel_cycle_stats_snapshot(ch, - (struct nvgpu_cycle_stats_snapshot_args *)buf); - gk20a_idle(g); - break; -#endif - case NVGPU_IOCTL_CHANNEL_WDT: - err = gk20a_channel_set_wdt_status(ch, - (struct nvgpu_channel_wdt_args *)buf); - break; - case NVGPU_IOCTL_CHANNEL_SET_RUNLIST_INTERLEAVE: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = gk20a_channel_set_runlist_interleave(ch, - ((struct nvgpu_runlist_interleave_args *)buf)->level); - - gk20a_idle(g); - gk20a_channel_trace_sched_param( - trace_gk20a_channel_set_runlist_interleave, ch); - break; - case NVGPU_IOCTL_CHANNEL_SET_TIMESLICE: - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = ch->g->ops.fifo.channel_set_timeslice(ch, - ((struct nvgpu_timeslice_args *)buf)->timeslice_us); - - gk20a_idle(g); - gk20a_channel_trace_sched_param( - trace_gk20a_channel_set_timeslice, ch); - break; - case NVGPU_IOCTL_CHANNEL_SET_PREEMPTION_MODE: - if (ch->g->ops.gr.set_preemption_mode) { - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = ch->g->ops.gr.set_preemption_mode(ch, - ((struct nvgpu_preemption_mode_args *)buf)->graphics_preempt_mode, - ((struct nvgpu_preemption_mode_args *)buf)->compute_preempt_mode); - gk20a_idle(g); - } else { - err = -EINVAL; - } - break; - case NVGPU_IOCTL_CHANNEL_SET_BOOSTED_CTX: - if (ch->g->ops.gr.set_boosted_ctx) { - bool boost = - ((struct nvgpu_boosted_ctx_args *)buf)->boost; - - err = gk20a_busy(g); - if (err) { - dev_err(dev, - "%s: failed to host gk20a for ioctl cmd: 0x%x", - __func__, cmd); - break; - } - err = ch->g->ops.gr.set_boosted_ctx(ch, boost); - gk20a_idle(g); - } else { - err = -EINVAL; - } - break; - default: - dev_dbg(dev, "unrecognized ioctl cmd: 0x%x", cmd); - err = -ENOTTY; - break; - } - - if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ)) - err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd)); - - nvgpu_mutex_release(&ch->ioctl_lock); - - gk20a_channel_put(ch); - - gk20a_dbg_fn("end"); - - return err; -} -- cgit v1.2.2