From 2517d59be282426eec7a97745b76d745ff36c388 Mon Sep 17 00:00:00 2001 From: Debarshi Dutta Date: Tue, 4 Sep 2018 17:39:36 +0530 Subject: gpu: nvgpu: move channel_sync_gk20a.* to common directory 1) Move channel_sync_gk20a.* from gk20a/ to common/ directory as they donot program any hardware registers. Also as an add-on rename channel_sync_gk20a.* to channel_sync.* and update the headers in required files. 2) Rename the struct gk20a_channel_sync to struct nvgpu_channel_sync. Also, corresponding syncpt and semaphore versions of the struct alongwith related methods are renamed by removing "gk20a" from their names and adding "nvgpu". 3) Add misra-c cleanups Jira NVGPU-1086 Change-Id: I4e0e21803ca3858dd7a5fc4d2454dba1f1bfcecd Signed-off-by: Debarshi Dutta Reviewed-on: https://git-master.nvidia.com/r/1812594 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/Makefile | 4 +- drivers/gpu/nvgpu/Makefile.sources | 2 +- drivers/gpu/nvgpu/common/fifo/channel.c | 14 +- drivers/gpu/nvgpu/common/fifo/submit.c | 6 +- drivers/gpu/nvgpu/common/sync/channel_sync.c | 670 +++++++++++++++++++++ drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c | 662 -------------------- drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h | 112 ---- drivers/gpu/nvgpu/gk20a/gk20a.c | 4 +- drivers/gpu/nvgpu/gp10b/fifo_gp10b.c | 2 +- drivers/gpu/nvgpu/include/nvgpu/channel.h | 6 +- drivers/gpu/nvgpu/include/nvgpu/channel_sync.h | 113 ++++ drivers/gpu/nvgpu/os/linux/ioctl_channel.c | 4 +- drivers/gpu/nvgpu/os/linux/os_fence_android_sema.c | 4 +- .../gpu/nvgpu/os/linux/os_fence_android_syncpt.c | 4 +- 14 files changed, 808 insertions(+), 799 deletions(-) create mode 100644 drivers/gpu/nvgpu/common/sync/channel_sync.c delete mode 100644 drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c delete mode 100644 drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h create mode 100644 drivers/gpu/nvgpu/include/nvgpu/channel_sync.h (limited to 'drivers/gpu/nvgpu') diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index d0dd252e..d59c3f74 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -50,7 +50,8 @@ nvgpu-y += common/bus/bus_gk20a.o \ common/mc/mc_gm20b.o \ common/mc/mc_gp10b.o \ common/mc/mc_gv11b.o \ - common/mc/mc_gv100.o + common/mc/mc_gv100.o \ + common/sync/channel_sync.o # Linux specific parts of nvgpu. nvgpu-y += \ @@ -228,7 +229,6 @@ nvgpu-y += \ gk20a/gk20a.o \ gk20a/ce2_gk20a.o \ gk20a/fifo_gk20a.o \ - gk20a/channel_sync_gk20a.o \ gk20a/dbg_gpu_gk20a.o \ gk20a/regops_gk20a.o \ gk20a/gr_gk20a.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index a53548f4..fce8ea71 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -100,6 +100,7 @@ srcs := os/posix/nvgpu.c \ common/pmu/pmu_perfmon.c \ common/pmu/pmu_debug.c \ common/ptimer/ptimer.c \ + common/sync/channel_sync.c \ common/clock_gating/gm20b_gating_reglist.c \ common/clock_gating/gp10b_gating_reglist.c \ common/clock_gating/gv11b_gating_reglist.c \ @@ -147,7 +148,6 @@ srcs := os/posix/nvgpu.c \ common/ptimer/ptimer_gk20a.c \ gk20a/ce2_gk20a.c \ gk20a/fifo_gk20a.c \ - gk20a/channel_sync_gk20a.c \ gk20a/dbg_gpu_gk20a.c \ gk20a/regops_gk20a.c \ gk20a/gr_gk20a.c \ diff --git a/drivers/gpu/nvgpu/common/fifo/channel.c b/drivers/gpu/nvgpu/common/fifo/channel.c index 45f5b736..1613f5f6 100644 --- a/drivers/gpu/nvgpu/common/fifo/channel.c +++ b/drivers/gpu/nvgpu/common/fifo/channel.c @@ -44,11 +44,11 @@ #include #include #include +#include #include "gk20a/gk20a.h" #include "gk20a/dbg_gpu_gk20a.h" #include "gk20a/fence_gk20a.h" -#include "gk20a/channel_sync_gk20a.h" static void free_channel(struct fifo_gk20a *f, struct channel_gk20a *c); static void gk20a_channel_dump_ref_actions(struct channel_gk20a *c); @@ -416,7 +416,7 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force) /* sync must be destroyed before releasing channel vm */ nvgpu_mutex_acquire(&ch->sync_lock); if (ch->sync) { - gk20a_channel_sync_destroy(ch->sync, false); + nvgpu_channel_sync_destroy(ch->sync, false); ch->sync = NULL; } if (ch->user_sync) { @@ -425,9 +425,9 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force) * But it's already done if channel has timedout */ if (ch->has_timedout) { - gk20a_channel_sync_destroy(ch->user_sync, false); + nvgpu_channel_sync_destroy(ch->user_sync, false); } else { - gk20a_channel_sync_destroy(ch->user_sync, true); + nvgpu_channel_sync_destroy(ch->user_sync, true); } ch->user_sync = NULL; } @@ -1191,7 +1191,7 @@ int gk20a_channel_alloc_gpfifo(struct channel_gk20a *c, if (g->aggressive_sync_destroy_thresh == 0U) { nvgpu_mutex_acquire(&c->sync_lock); - c->sync = gk20a_channel_sync_create(c, false); + c->sync = nvgpu_channel_sync_create(c, false); if (c->sync == NULL) { err = -ENOMEM; nvgpu_mutex_release(&c->sync_lock); @@ -1253,7 +1253,7 @@ clean_up_prealloc: } clean_up_sync: if (c->sync) { - gk20a_channel_sync_destroy(c->sync, false); + nvgpu_channel_sync_destroy(c->sync, false); c->sync = NULL; } clean_up_unmap: @@ -1984,7 +1984,7 @@ void gk20a_channel_clean_up_jobs(struct channel_gk20a *c, if (nvgpu_atomic_dec_and_test( &c->sync->refcount) && g->aggressive_sync_destroy) { - gk20a_channel_sync_destroy(c->sync, + nvgpu_channel_sync_destroy(c->sync, false); c->sync = NULL; } diff --git a/drivers/gpu/nvgpu/common/fifo/submit.c b/drivers/gpu/nvgpu/common/fifo/submit.c index 1f7a04a2..5a0beea9 100644 --- a/drivers/gpu/nvgpu/common/fifo/submit.c +++ b/drivers/gpu/nvgpu/common/fifo/submit.c @@ -24,12 +24,12 @@ #include #include #include +#include #include #include "gk20a/gk20a.h" #include "gk20a/fence_gk20a.h" -#include "gk20a/channel_sync_gk20a.h" #include @@ -56,7 +56,7 @@ static int nvgpu_submit_prepare_syncs(struct channel_gk20a *c, if (g->aggressive_sync_destroy_thresh) { nvgpu_mutex_acquire(&c->sync_lock); if (!c->sync) { - c->sync = gk20a_channel_sync_create(c, false); + c->sync = nvgpu_channel_sync_create(c, false); if (!c->sync) { err = -ENOMEM; nvgpu_mutex_release(&c->sync_lock); @@ -409,7 +409,7 @@ static int nvgpu_submit_channel_gpfifo(struct channel_gk20a *c, } need_sync_framework = - gk20a_channel_sync_needs_sync_framework(g) || + nvgpu_channel_sync_needs_os_fence_framework(g) || (flags & NVGPU_SUBMIT_FLAGS_SYNC_FENCE && flags & NVGPU_SUBMIT_FLAGS_FENCE_GET); diff --git a/drivers/gpu/nvgpu/common/sync/channel_sync.c b/drivers/gpu/nvgpu/common/sync/channel_sync.c new file mode 100644 index 00000000..b4caab38 --- /dev/null +++ b/drivers/gpu/nvgpu/common/sync/channel_sync.c @@ -0,0 +1,670 @@ +/* + * GK20A Channel Synchronization Abstraction + * + * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gk20a/gk20a.h" +#include "gk20a/fence_gk20a.h" +#include "gk20a/mm_gk20a.h" + +#ifdef CONFIG_TEGRA_GK20A_NVHOST + +struct nvgpu_channel_sync_syncpt { + struct nvgpu_channel_sync ops; + struct channel_gk20a *c; + struct nvgpu_nvhost_dev *nvhost_dev; + u32 id; + struct nvgpu_mem syncpt_buf; +}; + +int channel_sync_syncpt_gen_wait_cmd(struct channel_gk20a *c, + u32 id, u32 thresh, struct priv_cmd_entry *wait_cmd, + u32 wait_cmd_size, int pos, bool preallocated) +{ + int err = 0; + bool is_expired = nvgpu_nvhost_syncpt_is_expired_ext( + c->g->nvhost_dev, id, thresh); + + if (is_expired) { + if (preallocated) { + nvgpu_memset(c->g, wait_cmd->mem, + (wait_cmd->off + (u32)pos * wait_cmd_size) * (u32)sizeof(u32), + 0, wait_cmd_size * (u32)sizeof(u32)); + } + } else { + if (!preallocated) { + err = gk20a_channel_alloc_priv_cmdbuf(c, + c->g->ops.fifo.get_syncpt_wait_cmd_size(), wait_cmd); + if (err != 0) { + nvgpu_err(c->g, "not enough priv cmd buffer space"); + return err; + } + } + nvgpu_log(c->g, gpu_dbg_info, "sp->id %d gpu va %llx", + id, c->vm->syncpt_ro_map_gpu_va); + c->g->ops.fifo.add_syncpt_wait_cmd(c->g, wait_cmd, + (u32)pos * wait_cmd_size, id, thresh, + c->vm->syncpt_ro_map_gpu_va); + } + + return 0; +} + +static int channel_sync_syncpt_wait_raw(struct nvgpu_channel_sync *s, + u32 id, u32 thresh, struct priv_cmd_entry *wait_cmd) +{ + struct nvgpu_channel_sync_syncpt *sp = + container_of(s, struct nvgpu_channel_sync_syncpt, ops); + struct channel_gk20a *c = sp->c; + int err = 0; + u32 wait_cmd_size = c->g->ops.fifo.get_syncpt_wait_cmd_size(); + + if (!nvgpu_nvhost_syncpt_is_valid_pt_ext(sp->nvhost_dev, id)) { + return -EINVAL; + } + + err = channel_sync_syncpt_gen_wait_cmd(c, id, thresh, + wait_cmd, wait_cmd_size, 0, false); + + return err; +} + +static int channel_sync_syncpt_wait_fd(struct nvgpu_channel_sync *s, int fd, + struct priv_cmd_entry *wait_cmd, int max_wait_cmds) +{ + struct nvgpu_os_fence os_fence = {0}; + struct nvgpu_channel_sync_syncpt *sp = + container_of(s, struct nvgpu_channel_sync_syncpt, ops); + struct channel_gk20a *c = sp->c; + int err = 0; + + err = nvgpu_os_fence_fdget(&os_fence, c, fd); + if (err != 0) { + return -EINVAL; + } + + err = os_fence.ops->program_waits(&os_fence, + wait_cmd, c, max_wait_cmds); + + os_fence.ops->drop_ref(&os_fence); + + return err; +} + +static void channel_sync_syncpt_update(void *priv, int nr_completed) +{ + struct channel_gk20a *ch = priv; + + gk20a_channel_update(ch); + + /* note: channel_get() is in channel_sync_syncpt_incr_common() */ + gk20a_channel_put(ch); +} + +static int channel_sync_syncpt_incr_common(struct nvgpu_channel_sync *s, + bool wfi_cmd, + bool register_irq, + struct priv_cmd_entry *incr_cmd, + struct gk20a_fence *fence, + bool need_sync_fence) +{ + u32 thresh; + int err; + struct nvgpu_channel_sync_syncpt *sp = + container_of(s, struct nvgpu_channel_sync_syncpt, ops); + struct channel_gk20a *c = sp->c; + struct nvgpu_os_fence os_fence = {0}; + + err = gk20a_channel_alloc_priv_cmdbuf(c, + c->g->ops.fifo.get_syncpt_incr_cmd_size(wfi_cmd), + incr_cmd); + if (err != 0) { + return err; + } + + nvgpu_log(c->g, gpu_dbg_info, "sp->id %d gpu va %llx", + sp->id, sp->syncpt_buf.gpu_va); + c->g->ops.fifo.add_syncpt_incr_cmd(c->g, wfi_cmd, + incr_cmd, sp->id, sp->syncpt_buf.gpu_va); + + thresh = nvgpu_nvhost_syncpt_incr_max_ext(sp->nvhost_dev, sp->id, + c->g->ops.fifo.get_syncpt_incr_per_release()); + + if (register_irq) { + struct channel_gk20a *referenced = gk20a_channel_get(c); + + WARN_ON(!referenced); + + if (referenced) { + /* note: channel_put() is in + * channel_sync_syncpt_update() */ + + err = nvgpu_nvhost_intr_register_notifier( + sp->nvhost_dev, + sp->id, thresh, + channel_sync_syncpt_update, c); + if (err != 0) { + gk20a_channel_put(referenced); + } + + /* Adding interrupt action should + * never fail. A proper error handling + * here would require us to decrement + * the syncpt max back to its original + * value. */ + WARN(err, + "failed to set submit complete interrupt"); + } + } + + if (need_sync_fence) { + err = nvgpu_os_fence_syncpt_create(&os_fence, c, sp->nvhost_dev, + sp->id, thresh); + + if (err != 0) { + goto clean_up_priv_cmd; + } + } + + err = gk20a_fence_from_syncpt(fence, sp->nvhost_dev, + sp->id, thresh, os_fence); + + if (err != 0) { + if (nvgpu_os_fence_is_initialized(&os_fence) != 0) { + os_fence.ops->drop_ref(&os_fence); + } + goto clean_up_priv_cmd; + } + + return 0; + +clean_up_priv_cmd: + gk20a_free_priv_cmdbuf(c, incr_cmd); + return err; +} + +static int channel_sync_syncpt_incr(struct nvgpu_channel_sync *s, + struct priv_cmd_entry *entry, + struct gk20a_fence *fence, + bool need_sync_fence, + bool register_irq) +{ + /* Don't put wfi cmd to this one since we're not returning + * a fence to user space. */ + return channel_sync_syncpt_incr_common(s, + false /* no wfi */, + register_irq /* register irq */, + entry, fence, need_sync_fence); +} + +static int channel_sync_syncpt_incr_user(struct nvgpu_channel_sync *s, + int wait_fence_fd, + struct priv_cmd_entry *entry, + struct gk20a_fence *fence, + bool wfi, + bool need_sync_fence, + bool register_irq) +{ + /* Need to do 'wfi + host incr' since we return the fence + * to user space. */ + return channel_sync_syncpt_incr_common(s, + wfi, + register_irq /* register irq */, + entry, fence, need_sync_fence); +} + +static void channel_sync_syncpt_set_min_eq_max(struct nvgpu_channel_sync *s) +{ + struct nvgpu_channel_sync_syncpt *sp = + container_of(s, struct nvgpu_channel_sync_syncpt, ops); + nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id); +} + +static void channel_sync_syncpt_set_safe_state(struct nvgpu_channel_sync *s) +{ + struct nvgpu_channel_sync_syncpt *sp = + container_of(s, struct nvgpu_channel_sync_syncpt, ops); + nvgpu_nvhost_syncpt_set_safe_state(sp->nvhost_dev, sp->id); +} + +static int syncpt_get_id(struct nvgpu_channel_sync *s) +{ + struct nvgpu_channel_sync_syncpt *sp = + container_of(s, struct nvgpu_channel_sync_syncpt, ops); + return sp->id; +} + +static u64 channel_sync_syncpt_get_address(struct nvgpu_channel_sync *s) +{ + struct nvgpu_channel_sync_syncpt *sp = + container_of(s, struct nvgpu_channel_sync_syncpt, ops); + return sp->syncpt_buf.gpu_va; +} + +static void channel_sync_syncpt_destroy(struct nvgpu_channel_sync *s) +{ + struct nvgpu_channel_sync_syncpt *sp = + container_of(s, struct nvgpu_channel_sync_syncpt, ops); + + + sp->c->g->ops.fifo.free_syncpt_buf(sp->c, &sp->syncpt_buf); + + nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id); + nvgpu_nvhost_syncpt_put_ref_ext(sp->nvhost_dev, sp->id); + nvgpu_kfree(sp->c->g, sp); +} + +static struct nvgpu_channel_sync * +channel_sync_syncpt_create(struct channel_gk20a *c, bool user_managed) +{ + struct nvgpu_channel_sync_syncpt *sp; + char syncpt_name[32]; + + sp = nvgpu_kzalloc(c->g, sizeof(*sp)); + if (sp == NULL) { + return NULL; + } + + sp->c = c; + sp->nvhost_dev = c->g->nvhost_dev; + + if (user_managed) { + snprintf(syncpt_name, sizeof(syncpt_name), + "%s_%d_user", c->g->name, c->chid); + + sp->id = nvgpu_nvhost_get_syncpt_client_managed(sp->nvhost_dev, + syncpt_name); + } else { + snprintf(syncpt_name, sizeof(syncpt_name), + "%s_%d", c->g->name, c->chid); + + sp->id = nvgpu_nvhost_get_syncpt_host_managed(sp->nvhost_dev, + c->chid, syncpt_name); + } + if (sp->id == 0) { + nvgpu_kfree(c->g, sp); + nvgpu_err(c->g, "failed to get free syncpt"); + return NULL; + } + + sp->c->g->ops.fifo.alloc_syncpt_buf(sp->c, sp->id, + &sp->syncpt_buf); + + nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id); + + nvgpu_atomic_set(&sp->ops.refcount, 0); + sp->ops.wait_syncpt = channel_sync_syncpt_wait_raw; + sp->ops.wait_fd = channel_sync_syncpt_wait_fd; + sp->ops.incr = channel_sync_syncpt_incr; + sp->ops.incr_user = channel_sync_syncpt_incr_user; + sp->ops.set_min_eq_max = channel_sync_syncpt_set_min_eq_max; + sp->ops.set_safe_state = channel_sync_syncpt_set_safe_state; + sp->ops.syncpt_id = syncpt_get_id; + sp->ops.syncpt_address = channel_sync_syncpt_get_address; + sp->ops.destroy = channel_sync_syncpt_destroy; + + return &sp->ops; +} +#endif /* CONFIG_TEGRA_GK20A_NVHOST */ + +struct nvgpu_channel_sync_semaphore { + struct nvgpu_channel_sync ops; + struct channel_gk20a *c; + + /* A semaphore pool owned by this channel. */ + struct nvgpu_semaphore_pool *pool; +}; + +static void add_sema_cmd(struct gk20a *g, struct channel_gk20a *c, + struct nvgpu_semaphore *s, struct priv_cmd_entry *cmd, + u32 offset, bool acquire, bool wfi) +{ + int ch = c->chid; + u32 ob, off = cmd->off + offset; + u64 va; + + ob = off; + + /* + * RO for acquire (since we just need to read the mem) and RW for + * release since we will need to write back to the semaphore memory. + */ + va = acquire ? nvgpu_semaphore_gpu_ro_va(s) : + nvgpu_semaphore_gpu_rw_va(s); + + /* + * If the op is not an acquire (so therefor a release) we should + * incr the underlying sema next_value. + */ + if (!acquire) { + nvgpu_semaphore_prepare(s, c->hw_sema); + } + + g->ops.fifo.add_sema_cmd(g, s, va, cmd, off, acquire, wfi); + + if (acquire) { + gpu_sema_verbose_dbg(g, "(A) c=%d ACQ_GE %-4u pool=%-3llu" + "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", + ch, nvgpu_semaphore_get_value(s), + s->location.pool->page_idx, va, cmd->gva, + cmd->mem->gpu_va, ob); + } else { + gpu_sema_verbose_dbg(g, "(R) c=%d INCR %u (%u) pool=%-3llu" + "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", + ch, nvgpu_semaphore_get_value(s), + nvgpu_semaphore_read(s), + s->location.pool->page_idx, + va, cmd->gva, cmd->mem->gpu_va, ob); + } +} + +void channel_sync_semaphore_gen_wait_cmd(struct channel_gk20a *c, + struct nvgpu_semaphore *sema, struct priv_cmd_entry *wait_cmd, + u32 wait_cmd_size, int pos) +{ + if (sema == NULL) { + /* expired */ + nvgpu_memset(c->g, wait_cmd->mem, + (wait_cmd->off + (u32)pos * wait_cmd_size) * (u32)sizeof(u32), + 0, wait_cmd_size * (u32)sizeof(u32)); + } else { + WARN_ON(!sema->incremented); + add_sema_cmd(c->g, c, sema, wait_cmd, + (u32)pos * wait_cmd_size, true, false); + nvgpu_semaphore_put(sema); + } +} + +static int channel_sync_semaphore_wait_raw_syncpt( + struct nvgpu_channel_sync *s, u32 id, + u32 thresh, struct priv_cmd_entry *entry) +{ + struct nvgpu_channel_sync_semaphore *sema = + container_of(s, struct nvgpu_channel_sync_semaphore, ops); + struct gk20a *g = sema->c->g; + nvgpu_err(g, "trying to use syncpoint synchronization"); + return -ENODEV; +} + +static int channel_sync_semaphore_wait_fd( + struct nvgpu_channel_sync *s, int fd, + struct priv_cmd_entry *entry, int max_wait_cmds) +{ + struct nvgpu_channel_sync_semaphore *sema = + container_of(s, struct nvgpu_channel_sync_semaphore, ops); + struct channel_gk20a *c = sema->c; + + struct nvgpu_os_fence os_fence = {0}; + int err; + + err = nvgpu_os_fence_fdget(&os_fence, c, fd); + if (err != 0) { + return err; + } + + err = os_fence.ops->program_waits(&os_fence, + entry, c, max_wait_cmds); + + os_fence.ops->drop_ref(&os_fence); + + return err; +} + +static int channel_sync_semaphore_incr_common( + struct nvgpu_channel_sync *s, bool wfi_cmd, + struct priv_cmd_entry *incr_cmd, + struct gk20a_fence *fence, + bool need_sync_fence) +{ + u32 incr_cmd_size; + struct nvgpu_channel_sync_semaphore *sp = + container_of(s, struct nvgpu_channel_sync_semaphore, ops); + struct channel_gk20a *c = sp->c; + struct nvgpu_semaphore *semaphore; + int err = 0; + struct nvgpu_os_fence os_fence = {0}; + + semaphore = nvgpu_semaphore_alloc(c); + if (semaphore == NULL) { + nvgpu_err(c->g, + "ran out of semaphores"); + return -ENOMEM; + } + + incr_cmd_size = c->g->ops.fifo.get_sema_incr_cmd_size(); + err = gk20a_channel_alloc_priv_cmdbuf(c, incr_cmd_size, incr_cmd); + if (err) { + nvgpu_err(c->g, + "not enough priv cmd buffer space"); + goto clean_up_sema; + } + + /* Release the completion semaphore. */ + add_sema_cmd(c->g, c, semaphore, incr_cmd, 0, false, wfi_cmd); + + if (need_sync_fence) { + err = nvgpu_os_fence_sema_create(&os_fence, c, + semaphore); + + if (err) { + goto clean_up_sema; + } + } + + err = gk20a_fence_from_semaphore(fence, + semaphore, + &c->semaphore_wq, + os_fence); + + if (err != 0) { + if (nvgpu_os_fence_is_initialized(&os_fence) != 0) { + os_fence.ops->drop_ref(&os_fence); + } + goto clean_up_sema; + } + + return 0; + +clean_up_sema: + nvgpu_semaphore_put(semaphore); + return err; +} + +static int channel_sync_semaphore_incr( + struct nvgpu_channel_sync *s, + struct priv_cmd_entry *entry, + struct gk20a_fence *fence, + bool need_sync_fence, + bool register_irq) +{ + /* Don't put wfi cmd to this one since we're not returning + * a fence to user space. */ + return channel_sync_semaphore_incr_common(s, + false /* no wfi */, + entry, fence, need_sync_fence); +} + +static int channel_sync_semaphore_incr_user( + struct nvgpu_channel_sync *s, + int wait_fence_fd, + struct priv_cmd_entry *entry, + struct gk20a_fence *fence, + bool wfi, + bool need_sync_fence, + bool register_irq) +{ +#ifdef CONFIG_SYNC + int err; + + err = channel_sync_semaphore_incr_common(s, wfi, entry, fence, + need_sync_fence); + if (err != 0) { + return err; + } + + return 0; +#else + struct nvgpu_channel_sync_semaphore *sema = + container_of(s, struct nvgpu_channel_sync_semaphore, ops); + nvgpu_err(sema->c->g, + "trying to use sync fds with CONFIG_SYNC disabled"); + return -ENODEV; +#endif +} + +static void channel_sync_semaphore_set_min_eq_max(struct nvgpu_channel_sync *s) +{ + struct nvgpu_channel_sync_semaphore *sp = + container_of(s, struct nvgpu_channel_sync_semaphore, ops); + struct channel_gk20a *c = sp->c; + bool updated; + + if (c->hw_sema == NULL) { + return; + } + + updated = nvgpu_semaphore_reset(c->hw_sema); + + if (updated) { + nvgpu_cond_broadcast_interruptible(&c->semaphore_wq); + } +} + +static void channel_sync_semaphore_set_safe_state(struct nvgpu_channel_sync *s) +{ + /* Nothing to do. */ +} + +static int channel_sync_semaphore_get_id(struct nvgpu_channel_sync *s) +{ + return -EINVAL; +} + +static u64 channel_sync_semaphore_get_address(struct nvgpu_channel_sync *s) +{ + return 0; +} + +static void channel_sync_semaphore_destroy(struct nvgpu_channel_sync *s) +{ + struct nvgpu_channel_sync_semaphore *sema = + container_of(s, struct nvgpu_channel_sync_semaphore, ops); + + struct channel_gk20a *c = sema->c; + struct gk20a *g = c->g; + + if (c->has_os_fence_framework_support && + g->os_channel.os_fence_framework_inst_exists(c)) { + g->os_channel.destroy_os_fence_framework(c); + } + + /* The sema pool is cleaned up by the VM destroy. */ + sema->pool = NULL; + + nvgpu_kfree(sema->c->g, sema); +} + +static struct nvgpu_channel_sync * +channel_sync_semaphore_create(struct channel_gk20a *c, bool user_managed) +{ + struct nvgpu_channel_sync_semaphore *sema; + struct gk20a *g = c->g; + char pool_name[20]; + int asid = -1; + int err; + + if (WARN_ON(c->vm == NULL)) { + return NULL; + } + + sema = nvgpu_kzalloc(c->g, sizeof(*sema)); + if (sema == NULL) { + return NULL; + } + sema->c = c; + + sprintf(pool_name, "semaphore_pool-%d", c->chid); + sema->pool = c->vm->sema_pool; + + if (c->vm->as_share != NULL) { + asid = c->vm->as_share->id; + } + + if (c->has_os_fence_framework_support) { + /*Init the sync_timeline for this channel */ + err = g->os_channel.init_os_fence_framework(c, + "gk20a_ch%d_as%d", c->chid, asid); + + if (err != 0) { + nvgpu_kfree(g, sema); + return NULL; + } + } + + nvgpu_atomic_set(&sema->ops.refcount, 0); + sema->ops.wait_syncpt = channel_sync_semaphore_wait_raw_syncpt; + sema->ops.wait_fd = channel_sync_semaphore_wait_fd; + sema->ops.incr = channel_sync_semaphore_incr; + sema->ops.incr_user = channel_sync_semaphore_incr_user; + sema->ops.set_min_eq_max = channel_sync_semaphore_set_min_eq_max; + sema->ops.set_safe_state = channel_sync_semaphore_set_safe_state; + sema->ops.syncpt_id = channel_sync_semaphore_get_id; + sema->ops.syncpt_address = channel_sync_semaphore_get_address; + sema->ops.destroy = channel_sync_semaphore_destroy; + + return &sema->ops; +} + +void nvgpu_channel_sync_destroy(struct nvgpu_channel_sync *sync, + bool set_safe_state) +{ + if (set_safe_state) { + sync->set_safe_state(sync); + } + sync->destroy(sync); +} + +struct nvgpu_channel_sync *nvgpu_channel_sync_create(struct channel_gk20a *c, + bool user_managed) +{ +#ifdef CONFIG_TEGRA_GK20A_NVHOST + if (gk20a_platform_has_syncpoints(c->g)) + return channel_sync_syncpt_create(c, user_managed); +#endif + return channel_sync_semaphore_create(c, user_managed); +} + +bool nvgpu_channel_sync_needs_os_fence_framework(struct gk20a *g) +{ + return !gk20a_platform_has_syncpoints(g); +} diff --git a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c deleted file mode 100644 index d7399403..00000000 --- a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c +++ /dev/null @@ -1,662 +0,0 @@ -/* - * GK20A Channel Synchronization Abstraction - * - * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "channel_sync_gk20a.h" -#include "gk20a.h" -#include "fence_gk20a.h" -#include "mm_gk20a.h" - -#ifdef CONFIG_TEGRA_GK20A_NVHOST - -struct gk20a_channel_syncpt { - struct gk20a_channel_sync ops; - struct channel_gk20a *c; - struct nvgpu_nvhost_dev *nvhost_dev; - u32 id; - struct nvgpu_mem syncpt_buf; -}; - -int gk20a_channel_gen_syncpt_wait_cmd(struct channel_gk20a *c, - u32 id, u32 thresh, struct priv_cmd_entry *wait_cmd, - u32 wait_cmd_size, int pos, bool preallocated) -{ - int err = 0; - bool is_expired = nvgpu_nvhost_syncpt_is_expired_ext( - c->g->nvhost_dev, id, thresh); - - if (is_expired) { - if (preallocated) { - nvgpu_memset(c->g, wait_cmd->mem, - (wait_cmd->off + pos * wait_cmd_size) * sizeof(u32), - 0, wait_cmd_size * sizeof(u32)); - } - } else { - if (!preallocated) { - err = gk20a_channel_alloc_priv_cmdbuf(c, - c->g->ops.fifo.get_syncpt_wait_cmd_size(), wait_cmd); - if (err) { - nvgpu_err(c->g, "not enough priv cmd buffer space"); - return err; - } - } - nvgpu_log(c->g, gpu_dbg_info, "sp->id %d gpu va %llx", - id, c->vm->syncpt_ro_map_gpu_va); - c->g->ops.fifo.add_syncpt_wait_cmd(c->g, wait_cmd, - pos * wait_cmd_size, id, thresh, - c->vm->syncpt_ro_map_gpu_va); - } - - return 0; -} - -static int gk20a_channel_syncpt_wait_syncpt(struct gk20a_channel_sync *s, - u32 id, u32 thresh, struct priv_cmd_entry *wait_cmd) -{ - struct gk20a_channel_syncpt *sp = - container_of(s, struct gk20a_channel_syncpt, ops); - struct channel_gk20a *c = sp->c; - int err = 0; - u32 wait_cmd_size = c->g->ops.fifo.get_syncpt_wait_cmd_size(); - - if (!nvgpu_nvhost_syncpt_is_valid_pt_ext(sp->nvhost_dev, id)) - return -EINVAL; - - err = gk20a_channel_gen_syncpt_wait_cmd(c, id, thresh, - wait_cmd, wait_cmd_size, 0, false); - - return err; -} - -static int gk20a_channel_syncpt_wait_fd(struct gk20a_channel_sync *s, int fd, - struct priv_cmd_entry *wait_cmd, int max_wait_cmds) -{ - struct nvgpu_os_fence os_fence = {0}; - struct gk20a_channel_syncpt *sp = - container_of(s, struct gk20a_channel_syncpt, ops); - struct channel_gk20a *c = sp->c; - int err = 0; - - err = nvgpu_os_fence_fdget(&os_fence, c, fd); - if (err) - return -EINVAL; - - err = os_fence.ops->program_waits(&os_fence, - wait_cmd, c, max_wait_cmds); - - os_fence.ops->drop_ref(&os_fence); - - return err; -} - -static void gk20a_channel_syncpt_update(void *priv, int nr_completed) -{ - struct channel_gk20a *ch = priv; - - gk20a_channel_update(ch); - - /* note: channel_get() is in __gk20a_channel_syncpt_incr() */ - gk20a_channel_put(ch); -} - -static int __gk20a_channel_syncpt_incr(struct gk20a_channel_sync *s, - bool wfi_cmd, - bool register_irq, - struct priv_cmd_entry *incr_cmd, - struct gk20a_fence *fence, - bool need_sync_fence) -{ - u32 thresh; - int err; - struct gk20a_channel_syncpt *sp = - container_of(s, struct gk20a_channel_syncpt, ops); - struct channel_gk20a *c = sp->c; - struct nvgpu_os_fence os_fence = {0}; - - err = gk20a_channel_alloc_priv_cmdbuf(c, - c->g->ops.fifo.get_syncpt_incr_cmd_size(wfi_cmd), - incr_cmd); - if (err) - return err; - - nvgpu_log(c->g, gpu_dbg_info, "sp->id %d gpu va %llx", - sp->id, sp->syncpt_buf.gpu_va); - c->g->ops.fifo.add_syncpt_incr_cmd(c->g, wfi_cmd, - incr_cmd, sp->id, sp->syncpt_buf.gpu_va); - - thresh = nvgpu_nvhost_syncpt_incr_max_ext(sp->nvhost_dev, sp->id, - c->g->ops.fifo.get_syncpt_incr_per_release()); - - if (register_irq) { - struct channel_gk20a *referenced = gk20a_channel_get(c); - - WARN_ON(!referenced); - - if (referenced) { - /* note: channel_put() is in - * gk20a_channel_syncpt_update() */ - - err = nvgpu_nvhost_intr_register_notifier( - sp->nvhost_dev, - sp->id, thresh, - gk20a_channel_syncpt_update, c); - if (err) - gk20a_channel_put(referenced); - - /* Adding interrupt action should - * never fail. A proper error handling - * here would require us to decrement - * the syncpt max back to its original - * value. */ - WARN(err, - "failed to set submit complete interrupt"); - } - } - - if (need_sync_fence) { - err = nvgpu_os_fence_syncpt_create(&os_fence, c, sp->nvhost_dev, - sp->id, thresh); - - if (err) - goto clean_up_priv_cmd; - } - - err = gk20a_fence_from_syncpt(fence, sp->nvhost_dev, - sp->id, thresh, os_fence); - - if (err) { - if (nvgpu_os_fence_is_initialized(&os_fence)) - os_fence.ops->drop_ref(&os_fence); - goto clean_up_priv_cmd; - } - - return 0; - -clean_up_priv_cmd: - gk20a_free_priv_cmdbuf(c, incr_cmd); - return err; -} - -static int gk20a_channel_syncpt_incr(struct gk20a_channel_sync *s, - struct priv_cmd_entry *entry, - struct gk20a_fence *fence, - bool need_sync_fence, - bool register_irq) -{ - /* Don't put wfi cmd to this one since we're not returning - * a fence to user space. */ - return __gk20a_channel_syncpt_incr(s, - false /* no wfi */, - register_irq /* register irq */, - entry, fence, need_sync_fence); -} - -static int gk20a_channel_syncpt_incr_user(struct gk20a_channel_sync *s, - int wait_fence_fd, - struct priv_cmd_entry *entry, - struct gk20a_fence *fence, - bool wfi, - bool need_sync_fence, - bool register_irq) -{ - /* Need to do 'wfi + host incr' since we return the fence - * to user space. */ - return __gk20a_channel_syncpt_incr(s, - wfi, - register_irq /* register irq */, - entry, fence, need_sync_fence); -} - -static void gk20a_channel_syncpt_set_min_eq_max(struct gk20a_channel_sync *s) -{ - struct gk20a_channel_syncpt *sp = - container_of(s, struct gk20a_channel_syncpt, ops); - nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id); -} - -static void gk20a_channel_syncpt_set_safe_state(struct gk20a_channel_sync *s) -{ - struct gk20a_channel_syncpt *sp = - container_of(s, struct gk20a_channel_syncpt, ops); - nvgpu_nvhost_syncpt_set_safe_state(sp->nvhost_dev, sp->id); -} - -static int gk20a_channel_syncpt_id(struct gk20a_channel_sync *s) -{ - struct gk20a_channel_syncpt *sp = - container_of(s, struct gk20a_channel_syncpt, ops); - return sp->id; -} - -static u64 gk20a_channel_syncpt_address(struct gk20a_channel_sync *s) -{ - struct gk20a_channel_syncpt *sp = - container_of(s, struct gk20a_channel_syncpt, ops); - return sp->syncpt_buf.gpu_va; -} - -static void gk20a_channel_syncpt_destroy(struct gk20a_channel_sync *s) -{ - struct gk20a_channel_syncpt *sp = - container_of(s, struct gk20a_channel_syncpt, ops); - - - sp->c->g->ops.fifo.free_syncpt_buf(sp->c, &sp->syncpt_buf); - - nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id); - nvgpu_nvhost_syncpt_put_ref_ext(sp->nvhost_dev, sp->id); - nvgpu_kfree(sp->c->g, sp); -} - -static struct gk20a_channel_sync * -gk20a_channel_syncpt_create(struct channel_gk20a *c, bool user_managed) -{ - struct gk20a_channel_syncpt *sp; - char syncpt_name[32]; - - sp = nvgpu_kzalloc(c->g, sizeof(*sp)); - if (!sp) - return NULL; - - sp->c = c; - sp->nvhost_dev = c->g->nvhost_dev; - - if (user_managed) { - snprintf(syncpt_name, sizeof(syncpt_name), - "%s_%d_user", c->g->name, c->chid); - - sp->id = nvgpu_nvhost_get_syncpt_client_managed(sp->nvhost_dev, - syncpt_name); - } else { - snprintf(syncpt_name, sizeof(syncpt_name), - "%s_%d", c->g->name, c->chid); - - sp->id = nvgpu_nvhost_get_syncpt_host_managed(sp->nvhost_dev, - c->chid, syncpt_name); - } - if (!sp->id) { - nvgpu_kfree(c->g, sp); - nvgpu_err(c->g, "failed to get free syncpt"); - return NULL; - } - - sp->c->g->ops.fifo.alloc_syncpt_buf(sp->c, sp->id, - &sp->syncpt_buf); - - nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id); - - nvgpu_atomic_set(&sp->ops.refcount, 0); - sp->ops.wait_syncpt = gk20a_channel_syncpt_wait_syncpt; - sp->ops.wait_fd = gk20a_channel_syncpt_wait_fd; - sp->ops.incr = gk20a_channel_syncpt_incr; - sp->ops.incr_user = gk20a_channel_syncpt_incr_user; - sp->ops.set_min_eq_max = gk20a_channel_syncpt_set_min_eq_max; - sp->ops.set_safe_state = gk20a_channel_syncpt_set_safe_state; - sp->ops.syncpt_id = gk20a_channel_syncpt_id; - sp->ops.syncpt_address = gk20a_channel_syncpt_address; - sp->ops.destroy = gk20a_channel_syncpt_destroy; - - return &sp->ops; -} -#endif /* CONFIG_TEGRA_GK20A_NVHOST */ - -struct gk20a_channel_semaphore { - struct gk20a_channel_sync ops; - struct channel_gk20a *c; - - /* A semaphore pool owned by this channel. */ - struct nvgpu_semaphore_pool *pool; -}; - -static void add_sema_cmd(struct gk20a *g, struct channel_gk20a *c, - struct nvgpu_semaphore *s, struct priv_cmd_entry *cmd, - u32 offset, bool acquire, bool wfi) -{ - int ch = c->chid; - u32 ob, off = cmd->off + offset; - u64 va; - - ob = off; - - /* - * RO for acquire (since we just need to read the mem) and RW for - * release since we will need to write back to the semaphore memory. - */ - va = acquire ? nvgpu_semaphore_gpu_ro_va(s) : - nvgpu_semaphore_gpu_rw_va(s); - - /* - * If the op is not an acquire (so therefor a release) we should - * incr the underlying sema next_value. - */ - if (!acquire) { - nvgpu_semaphore_prepare(s, c->hw_sema); - } - - g->ops.fifo.add_sema_cmd(g, s, va, cmd, off, acquire, wfi); - - if (acquire) { - gpu_sema_verbose_dbg(g, "(A) c=%d ACQ_GE %-4u pool=%-3llu" - "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", - ch, nvgpu_semaphore_get_value(s), - s->location.pool->page_idx, va, cmd->gva, - cmd->mem->gpu_va, ob); - } else { - gpu_sema_verbose_dbg(g, "(R) c=%d INCR %u (%u) pool=%-3llu" - "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", - ch, nvgpu_semaphore_get_value(s), - nvgpu_semaphore_read(s), - s->location.pool->page_idx, - va, cmd->gva, cmd->mem->gpu_va, ob); - } -} - -void gk20a_channel_gen_sema_wait_cmd(struct channel_gk20a *c, - struct nvgpu_semaphore *sema, struct priv_cmd_entry *wait_cmd, - u32 wait_cmd_size, int pos) -{ - if (!sema) { - /* expired */ - nvgpu_memset(c->g, wait_cmd->mem, - (wait_cmd->off + pos * wait_cmd_size) * sizeof(u32), - 0, wait_cmd_size * sizeof(u32)); - } else { - WARN_ON(!sema->incremented); - add_sema_cmd(c->g, c, sema, wait_cmd, - pos * wait_cmd_size, true, false); - nvgpu_semaphore_put(sema); - } -} - -static int gk20a_channel_semaphore_wait_syncpt( - struct gk20a_channel_sync *s, u32 id, - u32 thresh, struct priv_cmd_entry *entry) -{ - struct gk20a_channel_semaphore *sema = - container_of(s, struct gk20a_channel_semaphore, ops); - struct gk20a *g = sema->c->g; - nvgpu_err(g, "trying to use syncpoint synchronization"); - return -ENODEV; -} - -static int gk20a_channel_semaphore_wait_fd( - struct gk20a_channel_sync *s, int fd, - struct priv_cmd_entry *entry, int max_wait_cmds) -{ - struct gk20a_channel_semaphore *sema = - container_of(s, struct gk20a_channel_semaphore, ops); - struct channel_gk20a *c = sema->c; - - struct nvgpu_os_fence os_fence = {0}; - int err; - - err = nvgpu_os_fence_fdget(&os_fence, c, fd); - if (err) { - return err; - } - - err = os_fence.ops->program_waits(&os_fence, - entry, c, max_wait_cmds); - - os_fence.ops->drop_ref(&os_fence); - - return err; -} - -static int __gk20a_channel_semaphore_incr( - struct gk20a_channel_sync *s, bool wfi_cmd, - struct priv_cmd_entry *incr_cmd, - struct gk20a_fence *fence, - bool need_sync_fence) -{ - int incr_cmd_size; - struct gk20a_channel_semaphore *sp = - container_of(s, struct gk20a_channel_semaphore, ops); - struct channel_gk20a *c = sp->c; - struct nvgpu_semaphore *semaphore; - int err = 0; - struct nvgpu_os_fence os_fence = {0}; - - semaphore = nvgpu_semaphore_alloc(c); - if (!semaphore) { - nvgpu_err(c->g, - "ran out of semaphores"); - return -ENOMEM; - } - - incr_cmd_size = c->g->ops.fifo.get_sema_incr_cmd_size(); - err = gk20a_channel_alloc_priv_cmdbuf(c, incr_cmd_size, incr_cmd); - if (err) { - nvgpu_err(c->g, - "not enough priv cmd buffer space"); - goto clean_up_sema; - } - - /* Release the completion semaphore. */ - add_sema_cmd(c->g, c, semaphore, incr_cmd, 0, false, wfi_cmd); - - if (need_sync_fence) { - err = nvgpu_os_fence_sema_create(&os_fence, c, - semaphore); - - if (err) { - goto clean_up_sema; - } - } - - err = gk20a_fence_from_semaphore(fence, - semaphore, - &c->semaphore_wq, - os_fence); - - if (err) { - if (nvgpu_os_fence_is_initialized(&os_fence)) { - os_fence.ops->drop_ref(&os_fence); - } - goto clean_up_sema; - } - - return 0; - -clean_up_sema: - nvgpu_semaphore_put(semaphore); - return err; -} - -static int gk20a_channel_semaphore_incr( - struct gk20a_channel_sync *s, - struct priv_cmd_entry *entry, - struct gk20a_fence *fence, - bool need_sync_fence, - bool register_irq) -{ - /* Don't put wfi cmd to this one since we're not returning - * a fence to user space. */ - return __gk20a_channel_semaphore_incr(s, - false /* no wfi */, - entry, fence, need_sync_fence); -} - -static int gk20a_channel_semaphore_incr_user( - struct gk20a_channel_sync *s, - int wait_fence_fd, - struct priv_cmd_entry *entry, - struct gk20a_fence *fence, - bool wfi, - bool need_sync_fence, - bool register_irq) -{ -#ifdef CONFIG_SYNC - int err; - - err = __gk20a_channel_semaphore_incr(s, wfi, entry, fence, - need_sync_fence); - if (err) - return err; - - return 0; -#else - struct gk20a_channel_semaphore *sema = - container_of(s, struct gk20a_channel_semaphore, ops); - nvgpu_err(sema->c->g, - "trying to use sync fds with CONFIG_SYNC disabled"); - return -ENODEV; -#endif -} - -static void gk20a_channel_semaphore_set_min_eq_max(struct gk20a_channel_sync *s) -{ - struct gk20a_channel_semaphore *sp = - container_of(s, struct gk20a_channel_semaphore, ops); - struct channel_gk20a *c = sp->c; - bool updated; - - if (!c->hw_sema) { - return; - } - - updated = nvgpu_semaphore_reset(c->hw_sema); - - if (updated) { - nvgpu_cond_broadcast_interruptible(&c->semaphore_wq); - } -} - -static void gk20a_channel_semaphore_set_safe_state(struct gk20a_channel_sync *s) -{ - /* Nothing to do. */ -} - -static int gk20a_channel_semaphore_syncpt_id(struct gk20a_channel_sync *s) -{ - return -EINVAL; -} - -static u64 gk20a_channel_semaphore_syncpt_address(struct gk20a_channel_sync *s) -{ - return 0; -} - -static void gk20a_channel_semaphore_destroy(struct gk20a_channel_sync *s) -{ - struct gk20a_channel_semaphore *sema = - container_of(s, struct gk20a_channel_semaphore, ops); - - struct channel_gk20a *c = sema->c; - struct gk20a *g = c->g; - - if (c->has_os_fence_framework_support && - g->os_channel.os_fence_framework_inst_exists(c)) { - g->os_channel.destroy_os_fence_framework(c); - } - - /* The sema pool is cleaned up by the VM destroy. */ - sema->pool = NULL; - - nvgpu_kfree(sema->c->g, sema); -} - -static struct gk20a_channel_sync * -gk20a_channel_semaphore_create(struct channel_gk20a *c, bool user_managed) -{ - struct gk20a_channel_semaphore *sema; - struct gk20a *g = c->g; - char pool_name[20]; - int asid = -1; - int err; - - if (WARN_ON(!c->vm)) { - return NULL; - } - - sema = nvgpu_kzalloc(c->g, sizeof(*sema)); - if (!sema) { - return NULL; - } - sema->c = c; - - sprintf(pool_name, "semaphore_pool-%d", c->chid); - sema->pool = c->vm->sema_pool; - - if (c->vm->as_share) { - asid = c->vm->as_share->id; - } - - if (c->has_os_fence_framework_support) { - /*Init the sync_timeline for this channel */ - err = g->os_channel.init_os_fence_framework(c, - "gk20a_ch%d_as%d", c->chid, asid); - - if (err) { - nvgpu_kfree(g, sema); - return NULL; - } - } - - nvgpu_atomic_set(&sema->ops.refcount, 0); - sema->ops.wait_syncpt = gk20a_channel_semaphore_wait_syncpt; - sema->ops.wait_fd = gk20a_channel_semaphore_wait_fd; - sema->ops.incr = gk20a_channel_semaphore_incr; - sema->ops.incr_user = gk20a_channel_semaphore_incr_user; - sema->ops.set_min_eq_max = gk20a_channel_semaphore_set_min_eq_max; - sema->ops.set_safe_state = gk20a_channel_semaphore_set_safe_state; - sema->ops.syncpt_id = gk20a_channel_semaphore_syncpt_id; - sema->ops.syncpt_address = gk20a_channel_semaphore_syncpt_address; - sema->ops.destroy = gk20a_channel_semaphore_destroy; - - return &sema->ops; -} - -void gk20a_channel_sync_destroy(struct gk20a_channel_sync *sync, - bool set_safe_state) -{ - if (set_safe_state) { - sync->set_safe_state(sync); - } - sync->destroy(sync); -} - -struct gk20a_channel_sync *gk20a_channel_sync_create(struct channel_gk20a *c, - bool user_managed) -{ -#ifdef CONFIG_TEGRA_GK20A_NVHOST - if (gk20a_platform_has_syncpoints(c->g)) - return gk20a_channel_syncpt_create(c, user_managed); -#endif - return gk20a_channel_semaphore_create(c, user_managed); -} - -bool gk20a_channel_sync_needs_sync_framework(struct gk20a *g) -{ - return !gk20a_platform_has_syncpoints(g); -} diff --git a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h deleted file mode 100644 index e0e318d2..00000000 --- a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * drivers/video/tegra/host/gk20a/channel_sync_gk20a.h - * - * GK20A Channel Synchronization Abstraction - * - * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#ifndef NVGPU_GK20A_CHANNEL_SYNC_GK20A_H -#define NVGPU_GK20A_CHANNEL_SYNC_GK20A_H - -struct gk20a_channel_sync; -struct priv_cmd_entry; -struct channel_gk20a; -struct gk20a_fence; -struct gk20a; -struct nvgpu_semaphore; - -struct gk20a_channel_sync { - nvgpu_atomic_t refcount; - - /* Generate a gpu wait cmdbuf from syncpoint. - * Returns a gpu cmdbuf that performs the wait when executed - */ - int (*wait_syncpt)(struct gk20a_channel_sync *s, u32 id, u32 thresh, - struct priv_cmd_entry *entry); - - /* Generate a gpu wait cmdbuf from sync fd. - * Returns a gpu cmdbuf that performs the wait when executed - */ - int (*wait_fd)(struct gk20a_channel_sync *s, int fd, - struct priv_cmd_entry *entry, int max_wait_cmds); - - /* Increment syncpoint/semaphore. - * Returns - * - a gpu cmdbuf that performs the increment when executed, - * - a fence that can be passed to wait_cpu() and is_expired(). - */ - int (*incr)(struct gk20a_channel_sync *s, - struct priv_cmd_entry *entry, - struct gk20a_fence *fence, - bool need_sync_fence, - bool register_irq); - - /* Increment syncpoint/semaphore, so that the returned fence represents - * work completion (may need wfi) and can be returned to user space. - * Returns - * - a gpu cmdbuf that performs the increment when executed, - * - a fence that can be passed to wait_cpu() and is_expired(), - * - a gk20a_fence that signals when the incr has happened. - */ - int (*incr_user)(struct gk20a_channel_sync *s, - int wait_fence_fd, - struct priv_cmd_entry *entry, - struct gk20a_fence *fence, - bool wfi, - bool need_sync_fence, - bool register_irq); - - /* Reset the channel syncpoint/semaphore. */ - void (*set_min_eq_max)(struct gk20a_channel_sync *s); - - /* - * Set the channel syncpoint/semaphore to safe state - * This should be used to reset User managed syncpoint since we don't - * track threshold values for those syncpoints - */ - void (*set_safe_state)(struct gk20a_channel_sync *s); - - /* Returns the sync point id or negative number if no syncpt*/ - int (*syncpt_id)(struct gk20a_channel_sync *s); - - /* Returns the sync point address of sync point or 0 if not supported */ - u64 (*syncpt_address)(struct gk20a_channel_sync *s); - - /* Free the resources allocated by gk20a_channel_sync_create. */ - void (*destroy)(struct gk20a_channel_sync *s); -}; - -void gk20a_channel_gen_sema_wait_cmd(struct channel_gk20a *c, - struct nvgpu_semaphore *sema, struct priv_cmd_entry *wait_cmd, - u32 wait_cmd_size, int pos); - -int gk20a_channel_gen_syncpt_wait_cmd(struct channel_gk20a *c, - u32 id, u32 thresh, struct priv_cmd_entry *wait_cmd, - u32 wait_cmd_size, int pos, bool preallocated); - -void gk20a_channel_sync_destroy(struct gk20a_channel_sync *sync, - bool set_safe_state); -struct gk20a_channel_sync *gk20a_channel_sync_create(struct channel_gk20a *c, - bool user_managed); -bool gk20a_channel_sync_needs_sync_framework(struct gk20a *g); - -#endif /* NVGPU_GK20A_CHANNEL_SYNC_GK20A_H */ diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c index df16af85..74fc991d 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gk20a.c @@ -38,11 +38,11 @@ #include #include #include +#include #include #include "gk20a.h" -#include "channel_sync_gk20a.h" #include "dbg_gpu_gk20a.h" #include "hal.h" @@ -475,7 +475,7 @@ int gk20a_init_gpu_characteristics(struct gk20a *g) * supported otherwise, provided that the user doesn't request anything * that depends on deferred cleanup. */ - if (!gk20a_channel_sync_needs_sync_framework(g)) { + if (!nvgpu_channel_sync_needs_os_fence_framework(g)) { __nvgpu_set_enabled(g, NVGPU_SUPPORT_DETERMINISTIC_SUBMIT_FULL, true); diff --git a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c index 21fed4fc..f1b318c9 100644 --- a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c @@ -30,11 +30,11 @@ #include #include #include +#include #include "fifo_gp10b.h" #include "gm20b/fifo_gm20b.h" -#include "gk20a/channel_sync_gk20a.h" #include #include diff --git a/drivers/gpu/nvgpu/include/nvgpu/channel.h b/drivers/gpu/nvgpu/include/nvgpu/channel.h index 6cca843e..cd4fadf8 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/channel.h +++ b/drivers/gpu/nvgpu/include/nvgpu/channel.h @@ -35,7 +35,7 @@ struct gk20a; struct dbg_session_gk20a; struct gk20a_fence; struct fifo_profile_gk20a; -struct gk20a_channel_sync; +struct nvgpu_channel_sync; struct nvgpu_gpfifo_userdata; /* Flags to be passed to gk20a_channel_alloc_gpfifo() */ @@ -289,8 +289,8 @@ struct channel_gk20a { struct nvgpu_list_node dbg_s_list; struct nvgpu_mutex sync_lock; - struct gk20a_channel_sync *sync; - struct gk20a_channel_sync *user_sync; + struct nvgpu_channel_sync *sync; + struct nvgpu_channel_sync *user_sync; #ifdef CONFIG_TEGRA_GR_VIRTUALIZATION u64 virt_ctx; diff --git a/drivers/gpu/nvgpu/include/nvgpu/channel_sync.h b/drivers/gpu/nvgpu/include/nvgpu/channel_sync.h new file mode 100644 index 00000000..b5936edc --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/channel_sync.h @@ -0,0 +1,113 @@ +/* + * + * Nvgpu Channel Synchronization Abstraction + * + * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVGPU_CHANNEL_SYNC_H +#define NVGPU_CHANNEL_SYNC_H + +#include + +struct nvgpu_channel_sync; +struct priv_cmd_entry; +struct channel_gk20a; +struct gk20a_fence; +struct gk20a; +struct nvgpu_semaphore; + +struct nvgpu_channel_sync { + nvgpu_atomic_t refcount; + + /* Generate a gpu wait cmdbuf from syncpoint. + * Returns a gpu cmdbuf that performs the wait when executed + */ + int (*wait_syncpt)(struct nvgpu_channel_sync *s, u32 id, u32 thresh, + struct priv_cmd_entry *entry); + + /* Generate a gpu wait cmdbuf from sync fd. + * Returns a gpu cmdbuf that performs the wait when executed + */ + int (*wait_fd)(struct nvgpu_channel_sync *s, int fd, + struct priv_cmd_entry *entry, int max_wait_cmds); + + /* Increment syncpoint/semaphore. + * Returns + * - a gpu cmdbuf that performs the increment when executed, + * - a fence that can be passed to wait_cpu() and is_expired(). + */ + int (*incr)(struct nvgpu_channel_sync *s, + struct priv_cmd_entry *entry, + struct gk20a_fence *fence, + bool need_sync_fence, + bool register_irq); + + /* Increment syncpoint/semaphore, so that the returned fence represents + * work completion (may need wfi) and can be returned to user space. + * Returns + * - a gpu cmdbuf that performs the increment when executed, + * - a fence that can be passed to wait_cpu() and is_expired(), + * - a gk20a_fence that signals when the incr has happened. + */ + int (*incr_user)(struct nvgpu_channel_sync *s, + int wait_fence_fd, + struct priv_cmd_entry *entry, + struct gk20a_fence *fence, + bool wfi, + bool need_sync_fence, + bool register_irq); + + /* Reset the channel syncpoint/semaphore. */ + void (*set_min_eq_max)(struct nvgpu_channel_sync *s); + + /* + * Set the channel syncpoint/semaphore to safe state + * This should be used to reset User managed syncpoint since we don't + * track threshold values for those syncpoints + */ + void (*set_safe_state)(struct nvgpu_channel_sync *s); + + /* Returns the sync point id or negative number if no syncpt*/ + int (*syncpt_id)(struct nvgpu_channel_sync *s); + + /* Returns the sync point address of sync point or 0 if not supported */ + u64 (*syncpt_address)(struct nvgpu_channel_sync *s); + + /* Free the resources allocated by nvgpu_channel_sync_create. */ + void (*destroy)(struct nvgpu_channel_sync *s); +}; + +void channel_sync_semaphore_gen_wait_cmd(struct channel_gk20a *c, + struct nvgpu_semaphore *sema, struct priv_cmd_entry *wait_cmd, + u32 wait_cmd_size, int pos); + +int channel_sync_syncpt_gen_wait_cmd(struct channel_gk20a *c, + u32 id, u32 thresh, struct priv_cmd_entry *wait_cmd, + u32 wait_cmd_size, int pos, bool preallocated); + +void nvgpu_channel_sync_destroy(struct nvgpu_channel_sync *sync, + bool set_safe_state); +struct nvgpu_channel_sync *nvgpu_channel_sync_create(struct channel_gk20a *c, + bool user_managed); +bool nvgpu_channel_sync_needs_os_fence_framework(struct gk20a *g); + +#endif /* NVGPU_GK20A_CHANNEL_SYNC_GK20A_H */ diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_channel.c b/drivers/gpu/nvgpu/os/linux/ioctl_channel.c index e9a24923..0f6843df 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_channel.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_channel.c @@ -35,11 +35,11 @@ #include #include #include +#include #include "gk20a/gk20a.h" #include "gk20a/dbg_gpu_gk20a.h" #include "gk20a/fence_gk20a.h" -#include "gk20a/channel_sync_gk20a.h" #include "platform_gk20a.h" #include "ioctl_channel.h" @@ -1028,7 +1028,7 @@ static int nvgpu_ioctl_channel_get_user_syncpoint(struct channel_gk20a *ch, if (ch->user_sync) { nvgpu_mutex_release(&ch->sync_lock); } else { - ch->user_sync = gk20a_channel_sync_create(ch, true); + ch->user_sync = nvgpu_channel_sync_create(ch, true); if (!ch->user_sync) { nvgpu_mutex_release(&ch->sync_lock); return -ENOMEM; diff --git a/drivers/gpu/nvgpu/os/linux/os_fence_android_sema.c b/drivers/gpu/nvgpu/os/linux/os_fence_android_sema.c index ec3ccf0d..195da64a 100644 --- a/drivers/gpu/nvgpu/os/linux/os_fence_android_sema.c +++ b/drivers/gpu/nvgpu/os/linux/os_fence_android_sema.c @@ -21,8 +21,8 @@ #include #include #include +#include -#include "gk20a/channel_sync_gk20a.h" #include "gk20a/mm_gk20a.h" #include "sync_sema_android.h" @@ -63,7 +63,7 @@ int nvgpu_os_fence_sema_wait_gen_cmd(struct nvgpu_os_fence *s, sync_fence->cbs[i].sync_pt); sema = gk20a_sync_pt_sema(pt); - gk20a_channel_gen_sema_wait_cmd(c, sema, wait_cmd, + channel_sync_semaphore_gen_wait_cmd(c, sema, wait_cmd, wait_cmd_size, i); } diff --git a/drivers/gpu/nvgpu/os/linux/os_fence_android_syncpt.c b/drivers/gpu/nvgpu/os/linux/os_fence_android_syncpt.c index b15dba19..fe09db8f 100644 --- a/drivers/gpu/nvgpu/os/linux/os_fence_android_syncpt.c +++ b/drivers/gpu/nvgpu/os/linux/os_fence_android_syncpt.c @@ -23,9 +23,9 @@ #include #include #include +#include #include "gk20a/gk20a.h" -#include "gk20a/channel_sync_gk20a.h" #include "gk20a/mm_gk20a.h" #include "../drivers/staging/android/sync.h" @@ -76,7 +76,7 @@ int nvgpu_os_fence_syncpt_wait_gen_cmd(struct nvgpu_os_fence *s, u32 wait_id = nvgpu_nvhost_sync_pt_id(pt); u32 wait_value = nvgpu_nvhost_sync_pt_thresh(pt); - err = gk20a_channel_gen_syncpt_wait_cmd(c, wait_id, wait_value, + err = channel_sync_syncpt_gen_wait_cmd(c, wait_id, wait_value, wait_cmd, wait_cmd_size, i, true); } -- cgit v1.2.2