From 23c7903eff6ee1ab184dfcc62c054de1557e5b1d Mon Sep 17 00:00:00 2001 From: Deepak Nibade Date: Thu, 26 Oct 2017 08:29:56 -0700 Subject: gpu: nvgpu: move submit path to linux Nvgpu submit path has a lot of dependency on Linux framework e.g. use of copy_from_user, use of structures defined in uapi/nvgpu headers, dma_buf_* calls for trace support etc Hence to keep common code independent of Linux code, move submit path to Linux directory Move below APIs to common/linux/channel.c trace_write_pushbuffer() trace_write_pushbuffer_range() gk20a_submit_prepare_syncs() gk20a_submit_append_priv_cmdbuf() gk20a_submit_append_gpfifo() gk20a_submit_channel_gpfifo() Move below APIs to common/linux/ce2.c gk20a_ce_execute_ops() Define gk20a_ce_execute_ops() in common/linux/ce2.c, and declare it in gk20a/ce2_gk20a.h since it is needed in common/mm code too Each OS needs to implement this API separately gk20a_channel_alloc_gpfifo() use sizeof(nvgpu_gpfifo) to get size of one gpfifo entry, but structure nvgpu_gpfifo is linux specific Define new nvgpu_get_gpfifo_entry_size() in linux specific code and use it in gk20a_channel_alloc_gpfifo() to get gpfifo entry size Each OS needs to implement this API separately Export some APIs from gk20a/ce2_gk20a.h and gk20a/channel_gk20a.h that are needed in linux code Jira NVGPU-259 Jira NVGPU-313 Change-Id: I360c6cb8ce4494b1e50c66af334a2a379f0d2dc4 Signed-off-by: Deepak Nibade Reviewed-on: https://git-master.nvidia.com/r/1586277 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/common/linux/channel.c | 648 +++++++++++++++++++++++++++++++ 1 file changed, 648 insertions(+) create mode 100644 drivers/gpu/nvgpu/common/linux/channel.c (limited to 'drivers/gpu/nvgpu/common/linux/channel.c') diff --git a/drivers/gpu/nvgpu/common/linux/channel.c b/drivers/gpu/nvgpu/common/linux/channel.c new file mode 100644 index 00000000..716c5820 --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/channel.c @@ -0,0 +1,648 @@ +/* + * Copyright (c) 2017, NVIDIA Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +/* + * This is required for nvgpu_vm_find_buf() which is used in the tracing + * code. Once we can get and access userspace buffers without requiring + * direct dma_buf usage this can be removed. + */ +#include + +#include "gk20a/gk20a.h" + +#include + +#include +#include +#include + +u32 nvgpu_get_gpfifo_entry_size(void) +{ + return sizeof(struct nvgpu_gpfifo); +} + +#ifdef CONFIG_DEBUG_FS +static void trace_write_pushbuffer(struct channel_gk20a *c, + struct nvgpu_gpfifo *g) +{ + void *mem = NULL; + unsigned int words; + u64 offset; + struct dma_buf *dmabuf = NULL; + + if (gk20a_debug_trace_cmdbuf) { + u64 gpu_va = (u64)g->entry0 | + (u64)((u64)pbdma_gp_entry1_get_hi_v(g->entry1) << 32); + int err; + + words = pbdma_gp_entry1_length_v(g->entry1); + err = nvgpu_vm_find_buf(c->vm, gpu_va, &dmabuf, &offset); + if (!err) + mem = dma_buf_vmap(dmabuf); + } + + if (mem) { + u32 i; + /* + * Write in batches of 128 as there seems to be a limit + * of how much you can output to ftrace at once. + */ + for (i = 0; i < words; i += 128U) { + trace_gk20a_push_cmdbuf( + c->g->name, + 0, + min(words - i, 128U), + offset + i * sizeof(u32), + mem); + } + dma_buf_vunmap(dmabuf, mem); + } +} +#endif + +static void trace_write_pushbuffer_range(struct channel_gk20a *c, + struct nvgpu_gpfifo *g, + struct nvgpu_gpfifo __user *user_gpfifo, + int offset, + int count) +{ +#ifdef CONFIG_DEBUG_FS + u32 size; + int i; + struct nvgpu_gpfifo *gp; + bool gpfifo_allocated = false; + + if (!gk20a_debug_trace_cmdbuf) + return; + + if (!g && !user_gpfifo) + return; + + if (!g) { + size = count * sizeof(struct nvgpu_gpfifo); + if (size) { + g = nvgpu_big_malloc(c->g, size); + if (!g) + return; + + if (copy_from_user(g, user_gpfifo, size)) { + nvgpu_big_free(c->g, g); + return; + } + } + gpfifo_allocated = true; + } + + gp = g + offset; + for (i = 0; i < count; i++, gp++) + trace_write_pushbuffer(c, gp); + + if (gpfifo_allocated) + nvgpu_big_free(c->g, g); +#endif +} + +/* + * Handle the submit synchronization - pre-fences and post-fences. + */ +static int gk20a_submit_prepare_syncs(struct channel_gk20a *c, + struct nvgpu_fence *fence, + struct channel_gk20a_job *job, + struct priv_cmd_entry **wait_cmd, + struct priv_cmd_entry **incr_cmd, + struct gk20a_fence **pre_fence, + struct gk20a_fence **post_fence, + bool force_need_sync_fence, + bool register_irq, + u32 flags) +{ + struct gk20a *g = c->g; + bool need_sync_fence = false; + bool new_sync_created = false; + int wait_fence_fd = -1; + int err = 0; + bool need_wfi = !(flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SUPPRESS_WFI); + bool pre_alloc_enabled = channel_gk20a_is_prealloc_enabled(c); + + /* + * If user wants to always allocate sync_fence_fds then respect that; + * otherwise, allocate sync_fence_fd based on user flags. + */ + if (force_need_sync_fence) + need_sync_fence = true; + + if (g->aggressive_sync_destroy_thresh) { + nvgpu_mutex_acquire(&c->sync_lock); + if (!c->sync) { + c->sync = gk20a_channel_sync_create(c); + if (!c->sync) { + err = -ENOMEM; + nvgpu_mutex_release(&c->sync_lock); + goto fail; + } + new_sync_created = true; + } + nvgpu_atomic_inc(&c->sync->refcount); + nvgpu_mutex_release(&c->sync_lock); + } + + if (g->ops.fifo.resetup_ramfc && new_sync_created) { + err = g->ops.fifo.resetup_ramfc(c); + if (err) + goto fail; + } + + /* + * Optionally insert syncpt wait in the beginning of gpfifo submission + * when user requested and the wait hasn't expired. Validate that the id + * makes sense, elide if not. The only reason this isn't being + * unceremoniously killed is to keep running some tests which trigger + * this condition. + */ + if (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT) { + job->pre_fence = gk20a_alloc_fence(c); + if (!job->pre_fence) { + err = -ENOMEM; + goto fail; + } + + if (!pre_alloc_enabled) + job->wait_cmd = nvgpu_kzalloc(g, + sizeof(struct priv_cmd_entry)); + + if (!job->wait_cmd) { + err = -ENOMEM; + goto clean_up_pre_fence; + } + + if (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE) { + wait_fence_fd = fence->id; + err = c->sync->wait_fd(c->sync, wait_fence_fd, + job->wait_cmd, job->pre_fence); + } else { + err = c->sync->wait_syncpt(c->sync, fence->id, + fence->value, job->wait_cmd, + job->pre_fence); + } + + if (!err) { + if (job->wait_cmd->valid) + *wait_cmd = job->wait_cmd; + *pre_fence = job->pre_fence; + } else + goto clean_up_wait_cmd; + } + + if ((flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) && + (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE)) + need_sync_fence = true; + + /* + * Always generate an increment at the end of a GPFIFO submission. This + * is used to keep track of method completion for idle railgating. The + * sync_pt/semaphore PB is added to the GPFIFO later on in submit. + */ + job->post_fence = gk20a_alloc_fence(c); + if (!job->post_fence) { + err = -ENOMEM; + goto clean_up_wait_cmd; + } + if (!pre_alloc_enabled) + job->incr_cmd = nvgpu_kzalloc(g, sizeof(struct priv_cmd_entry)); + + if (!job->incr_cmd) { + err = -ENOMEM; + goto clean_up_post_fence; + } + + if (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) + err = c->sync->incr_user(c->sync, wait_fence_fd, job->incr_cmd, + job->post_fence, need_wfi, need_sync_fence, + register_irq); + else + err = c->sync->incr(c->sync, job->incr_cmd, + job->post_fence, need_sync_fence, + register_irq); + if (!err) { + *incr_cmd = job->incr_cmd; + *post_fence = job->post_fence; + } else + goto clean_up_incr_cmd; + + return 0; + +clean_up_incr_cmd: + free_priv_cmdbuf(c, job->incr_cmd); + if (!pre_alloc_enabled) + job->incr_cmd = NULL; +clean_up_post_fence: + gk20a_fence_put(job->post_fence); + job->post_fence = NULL; +clean_up_wait_cmd: + free_priv_cmdbuf(c, job->wait_cmd); + if (!pre_alloc_enabled) + job->wait_cmd = NULL; +clean_up_pre_fence: + gk20a_fence_put(job->pre_fence); + job->pre_fence = NULL; +fail: + *wait_cmd = NULL; + *pre_fence = NULL; + return err; +} + +static void gk20a_submit_append_priv_cmdbuf(struct channel_gk20a *c, + struct priv_cmd_entry *cmd) +{ + struct gk20a *g = c->g; + struct nvgpu_mem *gpfifo_mem = &c->gpfifo.mem; + struct nvgpu_gpfifo x = { + .entry0 = u64_lo32(cmd->gva), + .entry1 = u64_hi32(cmd->gva) | + pbdma_gp_entry1_length_f(cmd->size) + }; + + nvgpu_mem_wr_n(g, gpfifo_mem, c->gpfifo.put * sizeof(x), + &x, sizeof(x)); + + if (cmd->mem->aperture == APERTURE_SYSMEM) + trace_gk20a_push_cmdbuf(g->name, 0, cmd->size, 0, + cmd->mem->cpu_va + cmd->off * sizeof(u32)); + + c->gpfifo.put = (c->gpfifo.put + 1) & (c->gpfifo.entry_num - 1); +} + +/* + * Copy source gpfifo entries into the gpfifo ring buffer, potentially + * splitting into two memcpys to handle wrap-around. + */ +static int gk20a_submit_append_gpfifo(struct channel_gk20a *c, + struct nvgpu_gpfifo *kern_gpfifo, + struct nvgpu_gpfifo __user *user_gpfifo, + u32 num_entries) +{ + /* byte offsets */ + u32 gpfifo_size = c->gpfifo.entry_num * sizeof(struct nvgpu_gpfifo); + u32 len = num_entries * sizeof(struct nvgpu_gpfifo); + u32 start = c->gpfifo.put * sizeof(struct nvgpu_gpfifo); + u32 end = start + len; /* exclusive */ + struct nvgpu_mem *gpfifo_mem = &c->gpfifo.mem; + struct nvgpu_gpfifo *cpu_src; + int err; + + if (user_gpfifo && !c->gpfifo.pipe) { + /* + * This path (from userspace to sysmem) is special in order to + * avoid two copies unnecessarily (from user to pipe, then from + * pipe to gpu sysmem buffer). + * + * As a special case, the pipe buffer exists if PRAMIN writes + * are forced, although the buffers may not be in vidmem in + * that case. + */ + if (end > gpfifo_size) { + /* wrap-around */ + int length0 = gpfifo_size - start; + int length1 = len - length0; + void __user *user2 = (u8 __user *)user_gpfifo + length0; + + err = copy_from_user(gpfifo_mem->cpu_va + start, + user_gpfifo, length0); + if (err) + return err; + + err = copy_from_user(gpfifo_mem->cpu_va, + user2, length1); + if (err) + return err; + } else { + err = copy_from_user(gpfifo_mem->cpu_va + start, + user_gpfifo, len); + if (err) + return err; + } + + trace_write_pushbuffer_range(c, NULL, user_gpfifo, + 0, num_entries); + goto out; + } else if (user_gpfifo) { + /* from userspace to vidmem or sysmem when pramin forced, use + * the common copy path below */ + err = copy_from_user(c->gpfifo.pipe, user_gpfifo, len); + if (err) + return err; + + cpu_src = c->gpfifo.pipe; + } else { + /* from kernel to either sysmem or vidmem, don't need + * copy_from_user so use the common path below */ + cpu_src = kern_gpfifo; + } + + if (end > gpfifo_size) { + /* wrap-around */ + int length0 = gpfifo_size - start; + int length1 = len - length0; + void *src2 = (u8 *)cpu_src + length0; + + nvgpu_mem_wr_n(c->g, gpfifo_mem, start, cpu_src, length0); + nvgpu_mem_wr_n(c->g, gpfifo_mem, 0, src2, length1); + } else { + nvgpu_mem_wr_n(c->g, gpfifo_mem, start, cpu_src, len); + + } + + trace_write_pushbuffer_range(c, cpu_src, NULL, 0, num_entries); + +out: + c->gpfifo.put = (c->gpfifo.put + num_entries) & + (c->gpfifo.entry_num - 1); + + return 0; +} + +int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, + struct nvgpu_gpfifo *gpfifo, + struct nvgpu_submit_gpfifo_args *args, + u32 num_entries, + u32 flags, + struct nvgpu_fence *fence, + struct gk20a_fence **fence_out, + bool force_need_sync_fence, + struct fifo_profile_gk20a *profile) +{ + struct gk20a *g = c->g; + struct priv_cmd_entry *wait_cmd = NULL; + struct priv_cmd_entry *incr_cmd = NULL; + struct gk20a_fence *pre_fence = NULL; + struct gk20a_fence *post_fence = NULL; + struct channel_gk20a_job *job = NULL; + /* we might need two extra gpfifo entries - one for pre fence + * and one for post fence. */ + const int extra_entries = 2; + bool skip_buffer_refcounting = (flags & + NVGPU_SUBMIT_GPFIFO_FLAGS_SKIP_BUFFER_REFCOUNTING); + int err = 0; + bool need_job_tracking; + bool need_deferred_cleanup = false; + struct nvgpu_gpfifo __user *user_gpfifo = args ? + (struct nvgpu_gpfifo __user *)(uintptr_t)args->gpfifo : NULL; + + if (nvgpu_is_enabled(g, NVGPU_DRIVER_IS_DYING)) + return -ENODEV; + + if (c->has_timedout) + return -ETIMEDOUT; + + if (!nvgpu_mem_is_valid(&c->gpfifo.mem)) + return -ENOMEM; + + /* fifo not large enough for request. Return error immediately. + * Kernel can insert gpfifo entries before and after user gpfifos. + * So, add extra_entries in user request. Also, HW with fifo size N + * can accept only N-1 entreis and so the below condition */ + if (c->gpfifo.entry_num - 1 < num_entries + extra_entries) { + nvgpu_err(g, "not enough gpfifo space allocated"); + return -ENOMEM; + } + + if (!gpfifo && !args) + return -EINVAL; + + if ((flags & (NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT | + NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET)) && + !fence) + return -EINVAL; + + /* an address space needs to have been bound at this point. */ + if (!gk20a_channel_as_bound(c)) { + nvgpu_err(g, + "not bound to an address space at time of gpfifo" + " submission."); + return -EINVAL; + } + + if (profile) + profile->timestamp[PROFILE_ENTRY] = sched_clock(); + + /* update debug settings */ + nvgpu_ltc_sync_enabled(g); + + gk20a_dbg_info("channel %d", c->chid); + + /* + * Job tracking is necessary for any of the following conditions: + * - pre- or post-fence functionality + * - channel wdt + * - GPU rail-gating with non-deterministic channels + * - buffer refcounting + * + * If none of the conditions are met, then job tracking is not + * required and a fast submit can be done (ie. only need to write + * out userspace GPFIFO entries and update GP_PUT). + */ + need_job_tracking = (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT) || + (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) || + c->wdt_enabled || + (g->can_railgate && !c->deterministic) || + !skip_buffer_refcounting; + + if (need_job_tracking) { + bool need_sync_framework = false; + + /* + * If the channel is to have deterministic latency and + * job tracking is required, the channel must have + * pre-allocated resources. Otherwise, we fail the submit here + */ + if (c->deterministic && !channel_gk20a_is_prealloc_enabled(c)) + return -EINVAL; + + need_sync_framework = force_need_sync_fence || + gk20a_channel_sync_needs_sync_framework(g) || + (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE && + (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT || + flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET)); + + /* + * Deferred clean-up is necessary for any of the following + * conditions: + * - channel's deterministic flag is not set + * - dependency on sync framework, which could make the + * behavior of the clean-up operation non-deterministic + * (should not be performed in the submit path) + * - channel wdt + * - GPU rail-gating with non-deterministic channels + * - buffer refcounting + * + * If none of the conditions are met, then deferred clean-up + * is not required, and we clean-up one job-tracking + * resource in the submit path. + */ + need_deferred_cleanup = !c->deterministic || + need_sync_framework || + c->wdt_enabled || + (g->can_railgate && + !c->deterministic) || + !skip_buffer_refcounting; + + /* + * For deterministic channels, we don't allow deferred clean_up + * processing to occur. In cases we hit this, we fail the submit + */ + if (c->deterministic && need_deferred_cleanup) + return -EINVAL; + + if (!c->deterministic) { + /* + * Get a power ref unless this is a deterministic + * channel that holds them during the channel lifetime. + * This one is released by gk20a_channel_clean_up_jobs, + * via syncpt or sema interrupt, whichever is used. + */ + err = gk20a_busy(g); + if (err) { + nvgpu_err(g, + "failed to host gk20a to submit gpfifo, process %s", + current->comm); + return err; + } + } + + if (!need_deferred_cleanup) { + /* clean up a single job */ + gk20a_channel_clean_up_jobs(c, false); + } + } + + + /* Grab access to HW to deal with do_idle */ + if (c->deterministic) + nvgpu_rwsem_down_read(&g->deterministic_busy); + + trace_gk20a_channel_submit_gpfifo(g->name, + c->chid, + num_entries, + flags, + fence ? fence->id : 0, + fence ? fence->value : 0); + + gk20a_dbg_info("pre-submit put %d, get %d, size %d", + c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num); + + /* + * Make sure we have enough space for gpfifo entries. Check cached + * values first and then read from HW. If no space, return EAGAIN + * and let userpace decide to re-try request or not. + */ + if (nvgpu_gp_free_count(c) < num_entries + extra_entries) { + if (nvgpu_get_gp_free_count(c) < num_entries + extra_entries) { + err = -EAGAIN; + goto clean_up; + } + } + + if (c->has_timedout) { + err = -ETIMEDOUT; + goto clean_up; + } + + if (need_job_tracking) { + err = channel_gk20a_alloc_job(c, &job); + if (err) + goto clean_up; + + err = gk20a_submit_prepare_syncs(c, fence, job, + &wait_cmd, &incr_cmd, + &pre_fence, &post_fence, + force_need_sync_fence, + need_deferred_cleanup, + flags); + if (err) + goto clean_up_job; + } + + if (profile) + profile->timestamp[PROFILE_JOB_TRACKING] = sched_clock(); + + if (wait_cmd) + gk20a_submit_append_priv_cmdbuf(c, wait_cmd); + + if (gpfifo || user_gpfifo) + err = gk20a_submit_append_gpfifo(c, gpfifo, user_gpfifo, + num_entries); + if (err) + goto clean_up_job; + + /* + * And here's where we add the incr_cmd we generated earlier. It should + * always run! + */ + if (incr_cmd) + gk20a_submit_append_priv_cmdbuf(c, incr_cmd); + + if (fence_out) + *fence_out = gk20a_fence_get(post_fence); + + if (need_job_tracking) + /* TODO! Check for errors... */ + gk20a_channel_add_job(c, job, skip_buffer_refcounting); + if (profile) + profile->timestamp[PROFILE_APPEND] = sched_clock(); + + g->ops.fifo.userd_gp_put(g, c); + + if ((NVGPU_SUBMIT_GPFIFO_FLAGS_RESCHEDULE_RUNLIST & flags) && + g->ops.fifo.reschedule_runlist) + g->ops.fifo.reschedule_runlist(g, c->runlist_id); + + /* No hw access beyond this point */ + if (c->deterministic) + nvgpu_rwsem_up_read(&g->deterministic_busy); + + trace_gk20a_channel_submitted_gpfifo(g->name, + c->chid, + num_entries, + flags, + post_fence ? post_fence->syncpt_id : 0, + post_fence ? post_fence->syncpt_value : 0); + + gk20a_dbg_info("post-submit put %d, get %d, size %d", + c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num); + + if (profile) + profile->timestamp[PROFILE_END] = sched_clock(); + gk20a_dbg_fn("done"); + return err; + +clean_up_job: + channel_gk20a_free_job(c, job); +clean_up: + gk20a_dbg_fn("fail"); + gk20a_fence_put(pre_fence); + gk20a_fence_put(post_fence); + if (c->deterministic) + nvgpu_rwsem_up_read(&g->deterministic_busy); + else if (need_deferred_cleanup) + gk20a_idle(g); + + return err; +} + -- cgit v1.2.2