/*
* drivers/video/tegra/host/gk20a/channel_gk20a.c
*
* GK20A Graphics channel
*
* Copyright (c) 2011-2014, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include <linux/nvhost.h>
#include <linux/list.h>
#include <linux/delay.h>
#include <linux/highmem.h> /* need for nvmap.h*/
#include <trace/events/gk20a.h>
#include <linux/scatterlist.h>
#include <linux/file.h>
#include <linux/anon_inodes.h>
#include <linux/dma-buf.h>
#include "debug_gk20a.h"
#include "gk20a.h"
#include "dbg_gpu_gk20a.h"
#include "hw_ram_gk20a.h"
#include "hw_fifo_gk20a.h"
#include "hw_pbdma_gk20a.h"
#include "hw_ccsr_gk20a.h"
#include "hw_ltc_gk20a.h"
#define NVMAP_HANDLE_PARAM_SIZE 1
static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f);
static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c);
static void free_priv_cmdbuf(struct channel_gk20a *c,
struct priv_cmd_entry *e);
static void recycle_priv_cmdbuf(struct channel_gk20a *c);
static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c);
static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c);
static int channel_gk20a_commit_userd(struct channel_gk20a *c);
static int channel_gk20a_setup_userd(struct channel_gk20a *c);
static int channel_gk20a_setup_ramfc(struct channel_gk20a *c,
u64 gpfifo_base, u32 gpfifo_entries);
static void channel_gk20a_bind(struct channel_gk20a *ch_gk20a);
static void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a);
static int channel_gk20a_alloc_inst(struct gk20a *g,
struct channel_gk20a *ch);
static void channel_gk20a_free_inst(struct gk20a *g,
struct channel_gk20a *ch);
static int channel_gk20a_update_runlist(struct channel_gk20a *c,
bool add);
static void gk20a_free_error_notifiers(struct channel_gk20a *ch);
static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f)
{
struct channel_gk20a *ch = NULL;
int chid;
mutex_lock(&f->ch_inuse_mutex);
for (chid = 0; chid < f->num_channels; chid++) {
if (!f->channel[chid].in_use) {
f->channel[chid].in_use = true;
ch = &f->channel[chid];
break;
}
}
mutex_unlock(&f->ch_inuse_mutex);
return ch;
}
static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c)
{
mutex_lock(&f->ch_inuse_mutex);
f->channel[c->hw_chid].in_use = false;
mutex_unlock(&f->ch_inuse_mutex);
}
int channel_gk20a_commit_va(struct channel_gk20a *c)
{
u64 addr;
u32 addr_lo;
u32 addr_hi;
void *inst_ptr;
gk20a_dbg_fn("");
inst_ptr = c->inst_block.cpuva;
if (!inst_ptr)
return -ENOMEM;
addr = gk20a_mm_iova_addr(c->vm->pdes.sgt->sgl);
addr_lo = u64_lo32(addr >> 12);
addr_hi = u64_hi32(addr);
gk20a_dbg_info("pde pa=0x%llx addr_lo=0x%x addr_hi=0x%x",
(u64)addr, addr_lo, addr_hi);
gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
ram_in_page_dir_base_target_vid_mem_f() |
ram_in_page_dir_base_vol_true_f() |
ram_in_page_dir_base_lo_f(addr_lo));
gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
ram_in_page_dir_base_hi_f(addr_hi));
gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
u64_lo32(c->vm->va_limit) | 0xFFF);
gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
ram_in_adr_limit_hi_f(u64_hi32(c->vm->va_limit)));
gk20a_mm_l2_invalidate(c->g);
return 0;
}
static int channel_gk20a_commit_userd(struct channel_gk20a *c)
{
u32 addr_lo;
u32 addr_hi;
void *inst_ptr;
gk20a_dbg_fn("");
inst_ptr = c->inst_block.cpuva;
if (!inst_ptr)
return -ENOMEM;
addr_lo = u64_lo32(c->userd_iova >> ram_userd_base_shift_v());
addr_hi = u64_hi32(c->userd_iova);
gk20a_dbg_info("channel %d : set ramfc userd 0x%16llx",
c->hw_chid, (u64)c->userd_iova);
gk20a_mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_w(),
pbdma_userd_target_vid_mem_f() |
pbdma_userd_addr_f(addr_lo));
gk20a_mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_hi_w(),
pbdma_userd_target_vid_mem_f() |
pbdma_userd_hi_addr_f(addr_hi));
gk20a_mm_l2_invalidate(c->g);
return 0;
}
static int channel_gk20a_set_schedule_params(struct channel_gk20a *c,
u32 timeslice_timeout)
{
void *inst_ptr;
int shift = 3;
int value = timeslice_timeout;
inst_ptr = c->inst_block.cpuva;
if (!inst_ptr)
return -ENOMEM;
/* disable channel */
gk20a_writel(c->g, ccsr_channel_r(c->hw_chid),
gk20a_readl(c->g, ccsr_channel_r(c->hw_chid)) |
ccsr_channel_enable_clr_true_f());
/* preempt the channel */
WARN_ON(gk20a_fifo_preempt_channel(c->g, c->hw_chid));
/* flush GPU cache */
gk20a_mm_l2_flush(c->g, true);
/* value field is 8 bits long */
while (value >= 1 << 8) {
value >>= 1;
shift++;
}
/* time slice register is only 18bits long */
if ((value << shift) >= 1<<19) {
pr_err("Requested timeslice value is clamped to 18 bits\n");
value = 255;
shift = 10;
}
/* set new timeslice */
gk20a_mem_wr32(inst_ptr, ram_fc_eng_timeslice_w(),
value | (shift << 12) |
fifo_eng_timeslice_enable_true_f());
/* enable channel */
gk20a_writel(c->g, ccsr_channel_r(c->hw_chid),
gk20a_readl(c->g, ccsr_channel_r(c->hw_chid)) |
ccsr_channel_enable_set_true_f());
gk20a_mm_l2_invalidate(c->g);
return 0;
}
static int channel_gk20a_setup_ramfc(struct channel_gk20a *c,
u64 gpfifo_base, u32 gpfifo_entries)
{
void *inst_ptr;
gk20a_dbg_fn("");
inst_ptr = c->inst_block.cpuva;
if (!inst_ptr)
return -ENOMEM;
memset(inst_ptr, 0, ram_fc_size_val_v());
gk20a_mem_wr32(inst_ptr, ram_fc_gp_base_w(),
pbdma_gp_base_offset_f(
u64_lo32(gpfifo_base >> pbdma_gp_base_rsvd_s())));
gk20a_mem_wr32(inst_ptr, ram_fc_gp_base_hi_w(),
pbdma_gp_base_hi_offset_f(u64_hi32(gpfifo_base)) |
pbdma_gp_base_hi_limit2_f(ilog2(gpfifo_entries)));
gk20a_mem_wr32(inst_ptr, ram_fc_signature_w(),
pbdma_signature_hw_valid_f() | pbdma_signature_sw_zero_f());
gk20a_mem_wr32(inst_ptr, ram_fc_formats_w(),
pbdma_formats_gp_fermi0_f() |
pbdma_formats_pb_fermi1_f() |
pbdma_formats_mp_fermi0_f());
gk20a_mem_wr32(inst_ptr, ram_fc_pb_header_w(),
pbdma_pb_header_priv_user_f() |
pbdma_pb_header_method_zero_f() |
pbdma_pb_header_subchannel_zero_f() |
pbdma_pb_header_level_main_f() |
pbdma_pb_header_first_true_f() |
pbdma_pb_header_type_inc_f());
gk20a_mem_wr32(inst_ptr, ram_fc_subdevice_w(),
pbdma_subdevice_id_f(1) |
pbdma_subdevice_status_active_f() |
pbdma_subdevice_channel_dma_enable_f());
gk20a_mem_wr32(inst_ptr, ram_fc_target_w(), pbdma_target_engine_sw_f());
gk20a_mem_wr32(inst_ptr, ram_fc_acquire_w(),
pbdma_acquire_retry_man_2_f() |
pbdma_acquire_retry_exp_2_f() |
pbdma_acquire_timeout_exp_max_f() |
pbdma_acquire_timeout_man_max_f() |
pbdma_acquire_timeout_en_disable_f());
gk20a_mem_wr32(inst_ptr, ram_fc_eng_timeslice_w(),
fifo_eng_timeslice_timeout_128_f() |
fifo_eng_timeslice_timescale_3_f() |
fifo_eng_timeslice_enable_true_f());
gk20a_mem_wr32(inst_ptr, ram_fc_pb_timeslice_w(),
fifo_pb_timeslice_timeout_16_f() |
fifo_pb_timeslice_timescale_0_f() |
fifo_pb_timeslice_enable_true_f());
gk20a_mem_wr32(inst_ptr, ram_fc_chid_w(), ram_fc_chid_id_f(c->hw_chid));
/* TBD: alwasy priv mode? */
gk20a_mem_wr32(inst_ptr, ram_fc_hce_ctrl_w(),
pbdma_hce_ctrl_hce_priv_mode_yes_f());
gk20a_mm_l2_invalidate(c->g);
return 0;
}
static int channel_gk20a_setup_userd(struct channel_gk20a *c)
{
BUG_ON(!c->userd_cpu_va);
gk20a_dbg_fn("");
gk20a_mem_wr32(c->userd_cpu_va, ram_userd_put_w(), 0);
gk20a_mem_wr32(c->userd_cpu_va, ram_userd_get_w(), 0);
gk20a_mem_wr32(c->userd_cpu_va, ram_userd_ref_w(), 0);
gk20a_mem_wr32(c->userd_cpu_va, ram_userd_put_hi_w(), 0);
gk20a_mem_wr32(c->userd_cpu_va, ram_userd_ref_threshold_w(), 0);
gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_top_level_get_w(), 0);
gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_top_level_get_hi_w(), 0);
gk20a_mem_wr32(c->userd_cpu_va, ram_userd_get_hi_w(), 0);
gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_get_w(), 0);
gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_put_w(), 0);
gk20a_mm_l2_invalidate(c->g);
return 0;
}
static void channel_gk20a_bind(struct channel_gk20a *ch_gk20a)
{
struct gk20a *g = ch_gk20a->g;
struct fifo_gk20a *f = &g->fifo;
struct fifo_engine_info_gk20a *engine_info =
f->engine_info + ENGINE_GR_GK20A;
u32 inst_ptr = ch_gk20a->inst_block.cpu_pa
>> ram_in_base_shift_v();
gk20a_dbg_info("bind channel %d inst ptr 0x%08x",
ch_gk20a->hw_chid, inst_ptr);
ch_gk20a->bound = true;
gk20a_writel(g, ccsr_channel_r(ch_gk20a->hw_chid),
(gk20a_readl(g, ccsr_channel_r(ch_gk20a->hw_chid)) &
~ccsr_channel_runlist_f(~0)) |
ccsr_channel_runlist_f(engine_info->runlist_id));
gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid),
ccsr_channel_inst_ptr_f(inst_ptr) |
ccsr_channel_inst_target_vid_mem_f() |
ccsr_channel_inst_bind_true_f());
gk20a_writel(g, ccsr_channel_r(ch_gk20a->hw_chid),
(gk20a_readl(g, ccsr_channel_r(ch_gk20a->hw_chid)) &
~ccsr_channel_enable_set_f(~0)) |
ccsr_channel_enable_set_true_f());
}
static void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a)
{
struct gk20a *g = ch_gk20a->g;
gk20a_dbg_fn("");
if (ch_gk20a->bound)
gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid),
ccsr_channel_inst_ptr_f(0) |
ccsr_channel_inst_bind_false_f());
ch_gk20a->bound = false;
}
static int channel_gk20a_alloc_inst(struct gk20a *g,
struct channel_gk20a *ch)
{
struct device *d = dev_from_gk20a(g);
int err = 0;
dma_addr_t iova;
gk20a_dbg_fn("");
ch->inst_block.size = ram_in_alloc_size_v();
ch->inst_block.cpuva = dma_alloc_coherent(d,
ch->inst_block.size,
&iova,
GFP_KERNEL);
if (!ch->inst_block.cpuva) {
gk20a_err(d, "%s: memory allocation failed\n", __func__);
err = -ENOMEM;
goto clean_up;
}
ch->inst_block.iova = iova;
ch->inst_block.cpu_pa = gk20a_get_phys_from_iova(d,
ch->inst_block.iova);
if (!ch->inst_block.cpu_pa) {
gk20a_err(d, "%s: failed to get physical address\n", __func__);
err = -ENOMEM;
goto clean_up;
}
gk20a_dbg_info("channel %d inst block physical addr: 0x%16llx",
ch->hw_chid, (u64)ch->inst_block.cpu_pa);
gk20a_dbg_fn("done");
return 0;
clean_up:
gk20a_err(d, "fail");
channel_gk20a_free_inst(g, ch);
return err;
}
static void channel_gk20a_free_inst(struct gk20a *g,
struct channel_gk20a *ch)
{
struct device *d = dev_from_gk20a(g);
if (ch->inst_block.cpuva)
dma_free_coherent(d, ch->inst_block.size,
ch->inst_block.cpuva, ch->inst_block.iova);
ch->inst_block.cpuva = NULL;
ch->inst_block.iova = 0;
memset(&ch->inst_block, 0, sizeof(struct inst_desc));
}
static int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add)
{
return gk20a_fifo_update_runlist(c->g, 0, c->hw_chid, add, true);
}
void gk20a_disable_channel_no_update(struct channel_gk20a *ch)
{
/* ensure no fences are pending */
if (ch->sync)
ch->sync->set_min_eq_max(ch->sync);
/* disable channel */
gk20a_writel(ch->g, ccsr_channel_r(ch->hw_chid),
gk20a_readl(ch->g,
ccsr_channel_r(ch->hw_chid)) |
ccsr_channel_enable_clr_true_f());
}
static int gk20a_wait_channel_idle(struct channel_gk20a *ch)
{
bool channel_idle = false;
unsigned long end_jiffies = jiffies +
msecs_to_jiffies(gk20a_get_gr_idle_timeout(ch->g));
do {
mutex_lock(&ch->jobs_lock);
channel_idle = list_empty(&ch->jobs);
mutex_unlock(&ch->jobs_lock);
if (channel_idle)
break;
usleep_range(1000, 3000);
} while (time_before(jiffies, end_jiffies)
|| !tegra_platform_is_silicon());
if (!channel_idle)
gk20a_err(dev_from_gk20a(ch->g), "channel jobs not freed");
return 0;
}
void gk20a_disable_channel(struct channel_gk20a *ch,
bool finish,
unsigned long finish_timeout)
{
if (finish) {
int err = gk20a_channel_finish(ch, finish_timeout);
WARN_ON(err);
}
/* disable the channel from hw and increment syncpoints */
gk20a_disable_channel_no_update(ch);
gk20a_wait_channel_idle(ch);
/* preempt the channel */
gk20a_fifo_preempt_channel(ch->g, ch->hw_chid);
/* remove channel from runlist */
channel_gk20a_update_runlist(ch, false);
}
#if defined(CONFIG_GK20A_CYCLE_STATS)
static void gk20a_free_cycle_stats_buffer(struct channel_gk20a *ch)
{
/* disable existing cyclestats buffer */
mutex_lock(&ch->cyclestate.cyclestate_buffer_mutex);
if (ch->cyclestate.cyclestate_buffer_handler) {
dma_buf_vunmap(ch->cyclestate.cyclestate_buffer_handler,
ch->cyclestate.cyclestate_buffer);
dma_buf_put(ch->cyclestate.cyclestate_buffer_handler);
ch->cyclestate.cyclestate_buffer_handler = NULL;
ch->cyclestate.cyclestate_buffer = NULL;
ch->cyclestate.cyclestate_buffer_size = 0;
}
mutex_unlock(&ch->cyclestate.cyclestate_buffer_mutex);
}
static int gk20a_channel_cycle_stats(struct channel_gk20a *ch,
struct nvhost_cycle_stats_args *args)
{
struct dma_buf *dmabuf;
void *virtual_address;
if (args->nvmap_handle && !ch->cyclestate.cyclestate_buffer_handler) {
/* set up new cyclestats buffer */
dmabuf = dma_buf_get(args->nvmap_handle);
if (IS_ERR(dmabuf))
return PTR_ERR(dmabuf);
virtual_address = dma_buf_vmap(dmabuf);
if (!virtual_address)
return -ENOMEM;
ch->cyclestate.cyclestate_buffer_handler = dmabuf;
ch->cyclestate.cyclestate_buffer = virtual_address;
ch->cyclestate.cyclestate_buffer_size = dmabuf->size;
return 0;
} else if (!args->nvmap_handle &&
ch->cyclestate.cyclestate_buffer_handler) {
gk20a_free_cycle_stats_buffer(ch);
return 0;
} else if (!args->nvmap_handle &&
!ch->cyclestate.cyclestate_buffer_handler) {
/* no requst from GL */
return 0;
} else {
pr_err("channel already has cyclestats buffer\n");
return -EINVAL;
}
}
#endif
static int gk20a_init_error_notifier(struct channel_gk20a *ch,
struct nvhost_set_error_notifier *args) {
void *va;
struct dma_buf *dmabuf;
if (!args->mem) {
pr_err("gk20a_init_error_notifier: invalid memory handle\n");
return -EINVAL;
}
dmabuf = dma_buf_get(args->mem);
if (ch->error_notifier_ref)
gk20a_free_error_notifiers(ch);
if (IS_ERR(dmabuf)) {
pr_err("Invalid handle: %d\n", args->mem);
return -EINVAL;
}
/* map handle */
va = dma_buf_vmap(dmabuf);
if (!va) {
dma_buf_put(dmabuf);
pr_err("Cannot map notifier handle\n");
return -ENOMEM;
}
/* set channel notifiers pointer */
ch->error_notifier_ref = dmabuf;
ch->error_notifier = va + args->offset;
ch->error_notifier_va = va;
memset(ch->error_notifier, 0, sizeof(struct nvhost_notification));
return 0;
}
void gk20a_set_error_notifier(struct channel_gk20a *ch, __u32 error)
{
if (ch->error_notifier_ref) {
struct timespec time_data;
u64 nsec;
getnstimeofday(&time_data);
nsec = ((u64)time_data.tv_sec) * 1000000000u +
(u64)time_data.tv_nsec;
ch->error_notifier->time_stamp.nanoseconds[0] =
(u32)nsec;
ch->error_notifier->time_stamp.nanoseconds[1] =
(u32)(nsec >> 32);
ch->error_notifier->info32 = error;
ch->error_notifier->status = 0xffff;
gk20a_err(dev_from_gk20a(ch->g),
"error notifier set to %d\n", error);
}
}
static void gk20a_free_error_notifiers(struct channel_gk20a *ch)
{
if (ch->error_notifier_ref) {
dma_buf_vunmap(ch->error_notifier_ref, ch->error_notifier_va);
dma_buf_put(ch->error_notifier_ref);
ch->error_notifier_ref = 0;
ch->error_notifier = 0;
ch->error_notifier_va = 0;
}
}
void gk20a_free_channel(struct channel_gk20a *ch, bool finish)
{
struct gk20a *g = ch->g;
struct device *d = dev_from_gk20a(g);
struct fifo_gk20a *f = &g->fifo;
struct gr_gk20a *gr = &g->gr;
struct vm_gk20a *ch_vm = ch->vm;
unsigned long timeout = gk20a_get_gr_idle_timeout(g);
struct dbg_session_gk20a *dbg_s;
gk20a_dbg_fn("");
/* if engine reset was deferred, perform it now */
mutex_lock(&f->deferred_reset_mutex);
if (g->fifo.deferred_reset_pending) {
gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, "engine reset was"
" deferred, running now");
fifo_gk20a_finish_mmu_fault_handling(g, g->fifo.mmu_fault_engines);
g->fifo.mmu_fault_engines = 0;
g->fifo.deferred_reset_pending = false;
}
mutex_unlock(&f->deferred_reset_mutex);
if (!ch->bound)
return;
if (!gk20a_channel_as_bound(ch))
goto unbind;
gk20a_dbg_info("freeing bound channel context, timeout=%ld",
timeout);
gk20a_disable_channel(ch, finish && !ch->has_timedout, timeout);
gk20a_free_error_notifiers(ch);
/* release channel ctx */
gk20a_free_channel_ctx(ch);
gk20a_gr_flush_channel_tlb(gr);
memset(&ch->ramfc, 0, sizeof(struct mem_desc_sub));
/* free gpfifo */
if (ch->gpfifo.gpu_va)
gk20a_gmmu_unmap(ch_vm, ch->gpfifo.gpu_va,
ch->gpfifo.size, gk20a_mem_flag_none);
if (ch->gpfifo.cpu_va)
dma_free_coherent(d, ch->gpfifo.size,
ch->gpfifo.cpu_va, ch->gpfifo.iova);
ch->gpfifo.cpu_va = NULL;
ch->gpfifo.iova = 0;
gk20a_mm_l2_invalidate(ch->g);
memset(&ch->gpfifo, 0, sizeof(struct gpfifo_desc));
#if defined(CONFIG_GK20A_CYCLE_STATS)
gk20a_free_cycle_stats_buffer(ch);
#endif
channel_gk20a_free_priv_cmdbuf(ch);
if (ch->sync) {
ch->sync->destroy(ch->sync);
ch->sync = NULL;
}
/* release channel binding to the as_share */
gk20a_as_release_share(ch_vm->as_share);
unbind:
channel_gk20a_unbind(ch);
channel_gk20a_free_inst(g, ch);
ch->vpr = false;
ch->vm = NULL;
WARN_ON(ch->sync);
/* unlink all debug sessions */
mutex_lock(&ch->dbg_s_lock);
list_for_each_entry(dbg_s, &ch->dbg_s_list, dbg_s_list_node) {
dbg_s->ch = NULL;
list_del_init(&dbg_s->dbg_s_list_node);
}
mutex_unlock(&ch->dbg_s_lock);
/* ALWAYS last */
release_used_channel(f, ch);
}
int gk20a_channel_release(struct inode *inode, struct file *filp)
{
struct channel_gk20a *ch = (struct channel_gk20a *)filp->private_data;
struct gk20a *g = ch->g;
trace_gk20a_channel_release(dev_name(&g->dev->dev));
gk20a_channel_busy(ch->g->dev);
gk20a_free_channel(ch, true);
gk20a_channel_idle(ch->g->dev);
gk20a_put_client(g);
filp->private_data = NULL;
return 0;
}
static struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g)
{
struct fifo_gk20a *f = &g->fifo;
struct channel_gk20a *ch;
ch = acquire_unused_channel(f);
if (ch == NULL) {
/* TBD: we want to make this virtualizable */
gk20a_err(dev_from_gk20a(g), "out of hw chids");
return 0;
}
ch->g = g;
if (channel_gk20a_alloc_inst(g, ch)) {
ch->in_use = false;
gk20a_err(dev_from_gk20a(g),
"failed to open gk20a channel, out of inst mem");
return 0;
}
g->ops.fifo.bind_channel(ch);
ch->pid = current->pid;
/* reset timeout counter and update timestamp */
ch->timeout_accumulated_ms = 0;
ch->timeout_gpfifo_get = 0;
/* set gr host default timeout */
ch->timeout_ms_max = gk20a_get_gr_idle_timeout(g);
ch->timeout_debug_dump = true;
ch->has_timedout = false;
/* The channel is *not* runnable at this point. It still needs to have
* an address space bound and allocate a gpfifo and grctx. */
init_waitqueue_head(&ch->notifier_wq);
init_waitqueue_head(&ch->semaphore_wq);
init_waitqueue_head(&ch->submit_wq);
return ch;
}
static int __gk20a_channel_open(struct gk20a *g, struct file *filp)
{
int err;
struct channel_gk20a *ch;
trace_gk20a_channel_open(dev_name(&g->dev->dev));
err = gk20a_get_client(g);
if (err) {
gk20a_err(dev_from_gk20a(g),
"failed to get client ref");
return err;
}
err = gk20a_channel_busy(g->dev);
if (err) {
gk20a_put_client(g);
gk20a_err(dev_from_gk20a(g), "failed to power on, %d", err);
return err;
}
ch = gk20a_open_new_channel(g);
gk20a_channel_idle(g->dev);
if (!ch) {
gk20a_put_client(g);
gk20a_err(dev_from_gk20a(g),
"failed to get f");
return -ENOMEM;
}
filp->private_data = ch;
return 0;
}
int gk20a_channel_open(struct inode *inode, struct file *filp)
{
struct gk20a *g = container_of(inode->i_cdev,
struct gk20a, channel.cdev);
return __gk20a_channel_open(g, filp);
}
/* allocate private cmd buffer.
used for inserting commands before/after user submitted buffers. */
static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c)
{
struct device *d = dev_from_gk20a(c->g);
struct vm_gk20a *ch_vm = c->vm;
struct priv_cmd_queue *q = &c->priv_cmd_q;
struct priv_cmd_entry *e;
u32 i = 0, size;
int err = 0;
struct sg_table *sgt;
dma_addr_t iova;
/* Kernel can insert gpfifos before and after user gpfifos.
Before user gpfifos, kernel inserts fence_wait, which takes
syncpoint_a (2 dwords) + syncpoint_b (2 dwords) = 4 dwords.
After user gpfifos, kernel inserts fence_get, which takes
wfi (2 dwords) + syncpoint_a (2 dwords) + syncpoint_b (2 dwords)
= 6 dwords.
Worse case if kernel adds both of them for every user gpfifo,
max size of priv_cmdbuf is :
(gpfifo entry number * (2 / 3) * (4 + 6) * 4 bytes */
size = roundup_pow_of_two(
c->gpfifo.entry_num * 2 * 10 * sizeof(u32) / 3);
q->mem.base_cpuva = dma_alloc_coherent(d, size,
&iova,
GFP_KERNEL);
if (!q->mem.base_cpuva) {
gk20a_err(d, "%s: memory allocation failed\n", __func__);
err = -ENOMEM;
goto clean_up;
}
q->mem.base_iova = iova;
q->mem.size = size;
err = gk20a_get_sgtable(d, &sgt,
q->mem.base_cpuva, q->mem.base_iova, size);
if (err) {
gk20a_err(d, "%s: failed to create sg table\n", __func__);
goto clean_up;
}
memset(q->mem.base_cpuva, 0, size);
q->base_gpuva = gk20a_gmmu_map(ch_vm, &sgt,
size,
0, /* flags */
gk20a_mem_flag_none);
if (!q->base_gpuva) {
gk20a_err(d, "ch %d : failed to map gpu va"
"for priv cmd buffer", c->hw_chid);
err = -ENOMEM;
goto clean_up_sgt;
}
q->size = q->mem.size / sizeof (u32);
INIT_LIST_HEAD(&q->head);
INIT_LIST_HEAD(&q->free);
/* pre-alloc 25% of priv cmdbuf entries and put them on free list */
for (i = 0; i < q->size / 4; i++) {
e = kzalloc(sizeof(struct priv_cmd_entry), GFP_KERNEL);
if (!e) {
gk20a_err(d, "ch %d: fail to pre-alloc cmd entry",
c->hw_chid);
err = -ENOMEM;
goto clean_up_sgt;
}
e->pre_alloc = true;
list_add(&e->list, &q->free);
}
gk20a_free_sgtable(&sgt);
return 0;
clean_up_sgt:
gk20a_free_sgtable(&sgt);
clean_up:
channel_gk20a_free_priv_cmdbuf(c);
return err;
}
static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c)
{
struct device *d = dev_from_gk20a(c->g);
struct vm_gk20a *ch_vm = c->vm;
struct priv_cmd_queue *q = &c->priv_cmd_q;
struct priv_cmd_entry *e;
struct list_head *pos, *tmp, *head;
if (q->size == 0)
return;
if (q->base_gpuva)
gk20a_gmmu_unmap(ch_vm, q->base_gpuva,
q->mem.size, gk20a_mem_flag_none);
if (q->mem.base_cpuva)
dma_free_coherent(d, q->mem.size,
q->mem.base_cpuva, q->mem.base_iova);
q->mem.base_cpuva = NULL;
q->mem.base_iova = 0;
/* free used list */
head = &q->head;
list_for_each_safe(pos, tmp, head) {
e = container_of(pos, struct priv_cmd_entry, list);
free_priv_cmdbuf(c, e);
}
/* free free list */
head = &q->free;
list_for_each_safe(pos, tmp, head) {
e = container_of(pos, struct priv_cmd_entry, list);
e->pre_alloc = false;
free_priv_cmdbuf(c, e);
}
memset(q, 0, sizeof(struct priv_cmd_queue));
}
/* allocate a cmd buffer with given size. size is number of u32 entries */
int gk20a_channel_alloc_priv_cmdbuf(struct channel_gk20a *c, u32 orig_size,
struct priv_cmd_entry **entry)
{
struct priv_cmd_queue *q = &c->priv_cmd_q;
struct priv_cmd_entry *e;
struct list_head *node;
u32 free_count;
u32 size = orig_size;
bool no_retry = false;
gk20a_dbg_fn("size %d", orig_size);
*entry = NULL;
/* if free space in the end is less than requested, increase the size
* to make the real allocated space start from beginning. */
if (q->put + size > q->size)
size = orig_size + (q->size - q->put);
gk20a_dbg_info("ch %d: priv cmd queue get:put %d:%d",
c->hw_chid, q->get, q->put);
TRY_AGAIN:
free_count = (q->size - (q->put - q->get) - 1) % q->size;
if (size > free_count) {
if (!no_retry) {
recycle_priv_cmdbuf(c);
no_retry = true;
goto TRY_AGAIN;
} else
return -EAGAIN;
}
if (unlikely(list_empty(&q->free))) {
gk20a_dbg_info("ch %d: run out of pre-alloc entries",
c->hw_chid);
e = kzalloc(sizeof(struct priv_cmd_entry), GFP_KERNEL);
if (!e) {
gk20a_err(dev_from_gk20a(c->g),
"ch %d: fail to allocate priv cmd entry",
c->hw_chid);
return -ENOMEM;
}
} else {
node = q->free.next;
list_del(node);
e = container_of(node, struct priv_cmd_entry, list);
}
e->size = orig_size;
e->gp_get = c->gpfifo.get;
e->gp_put = c->gpfifo.put;
e->gp_wrap = c->gpfifo.wrap;
/* if we have increased size to skip free space in the end, set put
to beginning of cmd buffer (0) + size */
if (size != orig_size) {
e->ptr = q->mem.base_cpuva;
e->gva = q->base_gpuva;
q->put = orig_size;
} else {
e->ptr = q->mem.base_cpuva + q->put;
e->gva = q->base_gpuva + q->put * sizeof(u32);
q->put = (q->put + orig_size) & (q->size - 1);
}
/* we already handled q->put + size > q->size so BUG_ON this */
BUG_ON(q->put > q->size);
/* add new entry to head since we free from head */
list_add(&e->list, &q->head);
*entry = e;
gk20a_dbg_fn("done");
return 0;
}
/* Don't call this to free an explict cmd entry.
* It doesn't update priv_cmd_queue get/put */
static void free_priv_cmdbuf(struct channel_gk20a *c,
struct priv_cmd_entry *e)
{
struct priv_cmd_queue *q = &c->priv_cmd_q;
if (!e)
return;
list_del(&e->list);
if (unlikely(!e->pre_alloc))
kfree(e);
else {
memset(e, 0, sizeof(struct priv_cmd_entry));
e->pre_alloc = true;
list_add(&e->list, &q->free);
}
}
/* free entries if they're no longer being used */
static void recycle_priv_cmdbuf(struct channel_gk20a *c)
{
struct priv_cmd_queue *q = &c->priv_cmd_q;
struct priv_cmd_entry *e, *tmp;
struct list_head *head = &q->head;
bool wrap_around, found = false;
gk20a_dbg_fn("");
/* Find the most recent free entry. Free it and everything before it */
list_for_each_entry(e, head, list) {
gk20a_dbg_info("ch %d: cmd entry get:put:wrap %d:%d:%d "
"curr get:put:wrap %d:%d:%d",
c->hw_chid, e->gp_get, e->gp_put, e->gp_wrap,
c->gpfifo.get, c->gpfifo.put, c->gpfifo.wrap);
wrap_around = (c->gpfifo.wrap != e->gp_wrap);
if (e->gp_get < e->gp_put) {
if (c->gpfifo.get >= e->gp_put ||
wrap_around) {
found = true;
break;
} else
e->gp_get = c->gpfifo.get;
} else if (e->gp_get > e->gp_put) {
if (wrap_around &&
c->gpfifo.get >= e->gp_put) {
found = true;
break;
} else
e->gp_get = c->gpfifo.get;
}
}
if (found)
q->get = (e->ptr - q->mem.base_cpuva) + e->size;
else {
gk20a_dbg_info("no free entry recycled");
return;
}
list_for_each_entry_safe_continue(e, tmp, head, list) {
free_priv_cmdbuf(c, e);
}
gk20a_dbg_fn("done");
}
static int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c,
struct nvhost_alloc_gpfifo_args *args)
{
struct gk20a *g = c->g;
struct device *d = dev_from_gk20a(g);
struct vm_gk20a *ch_vm;
u32 gpfifo_size;
int err = 0;
struct sg_table *sgt;
dma_addr_t iova;
/* Kernel can insert one extra gpfifo entry before user submitted gpfifos
and another one after, for internal usage. Triple the requested size. */
gpfifo_size = roundup_pow_of_two(args->num_entries * 3);
if (args->flags & NVHOST_ALLOC_GPFIFO_FLAGS_VPR_ENABLED)
c->vpr = true;
/* an address space needs to have been bound at this point. */
if (!gk20a_channel_as_bound(c)) {
gk20a_err(d,
"not bound to an address space at time of gpfifo"
" allocation. Attempting to create and bind to"
" one...");
return -EINVAL;
}
ch_vm = c->vm;
c->cmds_pending = false;
c->last_submit_fence.valid = false;
c->ramfc.offset = 0;
c->ramfc.size = ram_in_ramfc_s() / 8;
if (c->gpfifo.cpu_va) {
gk20a_err(d, "channel %d :"
"gpfifo already allocated", c->hw_chid);
return -EEXIST;
}
c->gpfifo.size = gpfifo_size * sizeof(struct gpfifo);
c->gpfifo.cpu_va = (struct gpfifo *)dma_alloc_coherent(d,
c->gpfifo.size,
&iova,
GFP_KERNEL);
if (!c->gpfifo.cpu_va) {
gk20a_err(d, "%s: memory allocation failed\n", __func__);
err = -ENOMEM;
goto clean_up;
}
c->gpfifo.iova = iova;
c->gpfifo.entry_num = gpfifo_size;
c->gpfifo.get = c->gpfifo.put = 0;
err = gk20a_get_sgtable(d, &sgt,
c->gpfifo.cpu_va, c->gpfifo.iova, c->gpfifo.size);
if (err) {
gk20a_err(d, "%s: failed to allocate sg table\n", __func__);
goto clean_up;
}
c->gpfifo.gpu_va = gk20a_gmmu_map(ch_vm,
&sgt,
c->gpfifo.size,
0, /* flags */
gk20a_mem_flag_none);
if (!c->gpfifo.gpu_va) {
gk20a_err(d, "channel %d : failed to map"
" gpu_va for gpfifo", c->hw_chid);
err = -ENOMEM;
goto clean_up_sgt;
}
gk20a_dbg_info("channel %d : gpfifo_base 0x%016llx, size %d",
c->hw_chid, c->gpfifo.gpu_va, c->gpfifo.entry_num);
channel_gk20a_setup_ramfc(c, c->gpfifo.gpu_va, c->gpfifo.entry_num);
channel_gk20a_setup_userd(c);
channel_gk20a_commit_userd(c);
gk20a_mm_l2_invalidate(c->g);
/* TBD: setup engine contexts */
err = channel_gk20a_alloc_priv_cmdbuf(c);
if (err)
goto clean_up_unmap;
err = channel_gk20a_update_runlist(c, true);
if (err)
goto clean_up_unmap;
gk20a_free_sgtable(&sgt);
gk20a_dbg_fn("done");
return 0;
clean_up_unmap:
gk20a_gmmu_unmap(ch_vm, c->gpfifo.gpu_va,
c->gpfifo.size, gk20a_mem_flag_none);
clean_up_sgt:
gk20a_free_sgtable(&sgt);
clean_up:
dma_free_coherent(d, c->gpfifo.size,
c->gpfifo.cpu_va, c->gpfifo.iova);
c->gpfifo.cpu_va = NULL;
c->gpfifo.iova = 0;
memset(&c->gpfifo, 0, sizeof(struct gpfifo_desc));
gk20a_err(d, "fail");
return err;
}
static inline int wfi_cmd_size(void)
{
return 2;
}
void add_wfi_cmd(struct priv_cmd_entry *cmd, int *i)
{
/* wfi */
cmd->ptr[(*i)++] = 0x2001001E;
/* handle, ignored */
cmd->ptr[(*i)++] = 0x00000000;
}
static inline bool check_gp_put(struct gk20a *g,
struct channel_gk20a *c)
{
u32 put;
/* gp_put changed unexpectedly since last update? */
put = gk20a_bar1_readl(g,
c->userd_gpu_va + 4 * ram_userd_gp_put_w());
if (c->gpfifo.put != put) {
/*TBD: BUG_ON/teardown on this*/
gk20a_err(dev_from_gk20a(g), "gp_put changed unexpectedly "
"since last update");
c->gpfifo.put = put;
return false; /* surprise! */
}
return true; /* checked out ok */
}
/* Update with this periodically to determine how the gpfifo is draining. */
static inline u32 update_gp_get(struct gk20a *g,
struct channel_gk20a *c)
{
u32 new_get = gk20a_bar1_readl(g,
c->userd_gpu_va + sizeof(u32) * ram_userd_gp_get_w());
if (new_get < c->gpfifo.get)
c->gpfifo.wrap = !c->gpfifo.wrap;
c->gpfifo.get = new_get;
return new_get;
}
static inline u32 gp_free_count(struct channel_gk20a *c)
{
return (c->gpfifo.entry_num - (c->gpfifo.put - c->gpfifo.get) - 1) %
c->gpfifo.entry_num;
}
bool gk20a_channel_update_and_check_timeout(struct channel_gk20a *ch,
u32 timeout_delta_ms)
{
u32 gpfifo_get = update_gp_get(ch->g, ch);
/* Count consequent timeout isr */
if (gpfifo_get == ch->timeout_gpfifo_get) {
/* we didn't advance since previous channel timeout check */
ch->timeout_accumulated_ms += timeout_delta_ms;
} else {
/* first timeout isr encountered */
ch->timeout_accumulated_ms = timeout_delta_ms;
}
ch->timeout_gpfifo_get = gpfifo_get;
return ch->g->timeouts_enabled &&
ch->timeout_accumulated_ms > ch->timeout_ms_max;
}
/* Issue a syncpoint increment *preceded* by a wait-for-idle
* command. All commands on the channel will have been
* consumed at the time the fence syncpoint increment occurs.
*/
static int gk20a_channel_submit_wfi(struct channel_gk20a *c)
{
struct priv_cmd_entry *cmd = NULL;
struct gk20a *g = c->g;
u32 free_count;
int err;
if (c->has_timedout)
return -ETIMEDOUT;
if (!c->sync) {
c->sync = gk20a_channel_sync_create(c);
if (!c->sync)
return -ENOMEM;
}
update_gp_get(g, c);
free_count = gp_free_count(c);
if (unlikely(!free_count)) {
gk20a_err(dev_from_gk20a(g),
"not enough gpfifo space");
return -EAGAIN;
}
err = c->sync->incr_wfi(c->sync, &cmd, &c->last_submit_fence);
if (unlikely(err))
return err;
WARN_ON(!c->last_submit_fence.wfi);
c->gpfifo.cpu_va[c->gpfifo.put].entry0 = u64_lo32(cmd->gva);
c->gpfifo.cpu_va[c->gpfifo.put].entry1 = u64_hi32(cmd->gva) |
pbdma_gp_entry1_length_f(cmd->size);
c->gpfifo.put = (c->gpfifo.put + 1) & (c->gpfifo.entry_num - 1);
/* save gp_put */
cmd->gp_put = c->gpfifo.put;
gk20a_bar1_writel(g,
c->userd_gpu_va + 4 * ram_userd_gp_put_w(),
c->gpfifo.put);
gk20a_dbg_info("post-submit put %d, get %d, size %d",
c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
return 0;
}
static u32 get_gp_free_count(struct channel_gk20a *c)
{
update_gp_get(c->g, c);
return gp_free_count(c);
}
static void trace_write_pushbuffer(struct channel_gk20a *c, struct gpfifo *g)
{
void *mem = NULL;
unsigned int words;
u64 offset;
struct dma_buf *dmabuf = NULL;
if (gk20a_debug_trace_cmdbuf) {
u64 gpu_va = (u64)g->entry0 |
(u64)((u64)pbdma_gp_entry1_get_hi_v(g->entry1) << 32);
int err;
words = pbdma_gp_entry1_length_v(g->entry1);
err = gk20a_vm_find_buffer(c->vm, gpu_va, &dmabuf, &offset);
if (!err)
mem = dma_buf_vmap(dmabuf);
}
if (mem) {
u32 i;
/*
* Write in batches of 128 as there seems to be a limit
* of how much you can output to ftrace at once.
*/
for (i = 0; i < words; i += 128U) {
trace_gk20a_push_cmdbuf(
c->g->dev->name,
0,
min(words - i, 128U),
offset + i * sizeof(u32),
mem);
}
dma_buf_vunmap(dmabuf, mem);
}
}
static int gk20a_channel_add_job(struct channel_gk20a *c,
struct gk20a_channel_fence *fence)
{
struct vm_gk20a *vm = c->vm;
struct channel_gk20a_job *job = NULL;
struct mapped_buffer_node **mapped_buffers = NULL;
int err = 0, num_mapped_buffers;
/* job needs reference to this vm */
gk20a_vm_get(vm);
err = gk20a_vm_get_buffers(vm, &mapped_buffers, &num_mapped_buffers);
if (err) {
gk20a_vm_put(vm);
return err;
}
job = kzalloc(sizeof(*job), GFP_KERNEL);
if (!job) {
gk20a_vm_put_buffers(vm, mapped_buffers, num_mapped_buffers);
gk20a_vm_put(vm);
return -ENOMEM;
}
job->num_mapped_buffers = num_mapped_buffers;
job->mapped_buffers = mapped_buffers;
job->fence = *fence;
mutex_lock(&c->jobs_lock);
list_add_tail(&job->list, &c->jobs);
mutex_unlock(&c->jobs_lock);
return 0;
}
void gk20a_channel_update(struct channel_gk20a *c, int nr_completed)
{
struct gk20a *g = c->g;
struct vm_gk20a *vm = c->vm;
struct channel_gk20a_job *job, *n;
int i;
wake_up(&c->submit_wq);
mutex_lock(&c->jobs_lock);
list_for_each_entry_safe(job, n, &c->jobs, list) {
bool completed = WARN_ON(!c->sync) ||
c->sync->is_expired(c->sync, &job->fence);
if (!completed)
break;
gk20a_vm_put_buffers(vm, job->mapped_buffers,
job->num_mapped_buffers);
/* job is done. release its reference to vm */
gk20a_vm_put(vm);
list_del_init(&job->list);
kfree(job);
gk20a_channel_idle(g->dev);
}
mutex_unlock(&c->jobs_lock);
for (i = 0; i < nr_completed; i++)
gk20a_channel_idle(c->g->dev);
}
static int gk20a_submit_channel_gpfifo(struct channel_gk20a *c,
struct nvhost_gpfifo *gpfifo,
u32 num_entries,
struct nvhost_fence *fence,
u32 flags)
{
struct gk20a *g = c->g;
struct device *d = dev_from_gk20a(g);
u32 err = 0;
int i;
struct priv_cmd_entry *wait_cmd = NULL;
struct priv_cmd_entry *incr_cmd = NULL;
/* we might need two extra gpfifo entries - one for pre fence
* and one for post fence. */
const int extra_entries = 2;
if (c->has_timedout)
return -ETIMEDOUT;
if ((flags & (NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT |
NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET)) &&
!fence)
return -EINVAL;
if (!c->sync) {
c->sync = gk20a_channel_sync_create(c);
if (!c->sync)
return -ENOMEM;
}
#ifdef CONFIG_DEBUG_FS
/* update debug settings */
if (g->ops.ltc.sync_debugfs)
g->ops.ltc.sync_debugfs(g);
#endif
gk20a_dbg_info("channel %d", c->hw_chid);
/* gk20a_channel_update releases this ref. */
gk20a_channel_busy(g->dev);
trace_gk20a_channel_submit_gpfifo(c->g->dev->name,
c->hw_chid,
num_entries,
flags,
fence->syncpt_id, fence->value);
check_gp_put(g, c);
update_gp_get(g, c);
gk20a_dbg_info("pre-submit put %d, get %d, size %d",
c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
/* Invalidate tlb if it's dirty... */
/* TBD: this should be done in the cmd stream, not with PRIs. */
/* We don't know what context is currently running... */
/* Note also: there can be more than one context associated with the */
/* address space (vm). */
gk20a_mm_tlb_invalidate(c->vm);
/* Make sure we have enough space for gpfifo entries. If not,
* wait for signals from completed submits */
if (gp_free_count(c) < num_entries + extra_entries) {
err = wait_event_interruptible(c->submit_wq,
get_gp_free_count(c) >= num_entries + extra_entries ||
c->has_timedout);
}
if (c->has_timedout) {
err = -ETIMEDOUT;
goto clean_up;
}
if (err) {
gk20a_err(d, "not enough gpfifo space");
err = -EAGAIN;
goto clean_up;
}
/*
* optionally insert syncpt wait in the beginning of gpfifo submission
* when user requested and the wait hasn't expired.
* validate that the id makes sense, elide if not
* the only reason this isn't being unceremoniously killed is to
* keep running some tests which trigger this condition
*/
if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT) {
if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE)
err = c->sync->wait_fd(c->sync, fence->syncpt_id,
&wait_cmd);
else
err = c->sync->wait_syncpt(c->sync, fence->syncpt_id,
fence->value, &wait_cmd);
}
if (err)
goto clean_up;
/* always insert syncpt increment at end of gpfifo submission
to keep track of method completion for idle railgating */
if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET &&
flags & NVHOST_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE)
err = c->sync->incr_user_fd(c->sync, &incr_cmd,
&c->last_submit_fence,
&fence->syncpt_id);
else if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET)
err = c->sync->incr_user_syncpt(c->sync, &incr_cmd,
&c->last_submit_fence,
&fence->syncpt_id,
&fence->value);
else
err = c->sync->incr(c->sync, &incr_cmd,
&c->last_submit_fence);
if (err)
goto clean_up;
if (wait_cmd) {
c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
u64_lo32(wait_cmd->gva);
c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
u64_hi32(wait_cmd->gva) |
pbdma_gp_entry1_length_f(wait_cmd->size);
trace_write_pushbuffer(c, &c->gpfifo.cpu_va[c->gpfifo.put]);
c->gpfifo.put = (c->gpfifo.put + 1) &
(c->gpfifo.entry_num - 1);
/* save gp_put */
wait_cmd->gp_put = c->gpfifo.put;
}
for (i = 0; i < num_entries; i++) {
c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
gpfifo[i].entry0; /* cmd buf va low 32 */
c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
gpfifo[i].entry1; /* cmd buf va high 32 | words << 10 */
trace_write_pushbuffer(c, &c->gpfifo.cpu_va[c->gpfifo.put]);
c->gpfifo.put = (c->gpfifo.put + 1) &
(c->gpfifo.entry_num - 1);
}
if (incr_cmd) {
c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
u64_lo32(incr_cmd->gva);
c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
u64_hi32(incr_cmd->gva) |
pbdma_gp_entry1_length_f(incr_cmd->size);
trace_write_pushbuffer(c, &c->gpfifo.cpu_va[c->gpfifo.put]);
c->gpfifo.put = (c->gpfifo.put + 1) &
(c->gpfifo.entry_num - 1);
/* save gp_put */
incr_cmd->gp_put = c->gpfifo.put;
}
/* Invalidate tlb if it's dirty... */
/* TBD: this should be done in the cmd stream, not with PRIs. */
/* We don't know what context is currently running... */
/* Note also: there can be more than one context associated with the */
/* address space (vm). */
gk20a_mm_tlb_invalidate(c->vm);
trace_gk20a_channel_submitted_gpfifo(c->g->dev->name,
c->hw_chid,
num_entries,
flags,
fence->syncpt_id, fence->value);
/* TODO! Check for errors... */
gk20a_channel_add_job(c, &c->last_submit_fence);
c->cmds_pending = true;
gk20a_bar1_writel(g,
c->userd_gpu_va + 4 * ram_userd_gp_put_w(),
c->gpfifo.put);
gk20a_dbg_info("post-submit put %d, get %d, size %d",
c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
gk20a_dbg_fn("done");
return err;
clean_up:
gk20a_err(d, "fail");
free_priv_cmdbuf(c, wait_cmd);
free_priv_cmdbuf(c, incr_cmd);
gk20a_channel_idle(g->dev);
return err;
}
void gk20a_remove_channel_support(struct channel_gk20a *c)
{
}
int gk20a_init_channel_support(struct gk20a *g, u32 chid)
{
struct channel_gk20a *c = g->fifo.channel+chid;
c->g = g;
c->in_use = false;
c->hw_chid = chid;
c->bound = false;
c->remove_support = gk20a_remove_channel_support;
mutex_init(&c->jobs_lock);
INIT_LIST_HEAD(&c->jobs);
#if defined(CONFIG_GK20A_CYCLE_STATS)
mutex_init(&c->cyclestate.cyclestate_buffer_mutex);
#endif
INIT_LIST_HEAD(&c->dbg_s_list);
mutex_init(&c->dbg_s_lock);
return 0;
}
int gk20a_channel_finish(struct channel_gk20a *ch, unsigned long timeout)
{
int err = 0;
if (!ch->cmds_pending)
return 0;
/* Do not wait for a timedout channel */
if (ch->has_timedout)
return -ETIMEDOUT;
if (!(ch->last_submit_fence.valid && ch->last_submit_fence.wfi)) {
gk20a_dbg_fn("issuing wfi, incr to finish the channel");
err = gk20a_channel_submit_wfi(ch);
}
if (err)
return err;
BUG_ON(!(ch->last_submit_fence.valid && ch->last_submit_fence.wfi));
gk20a_dbg_fn("waiting for channel to finish thresh:%d",
ch->last_submit_fence.thresh);
err = ch->sync->wait_cpu(ch->sync, &ch->last_submit_fence, timeout);
if (WARN_ON(err))
dev_warn(dev_from_gk20a(ch->g),
"timed out waiting for gk20a channel to finish");
else
ch->cmds_pending = false;
return err;
}
static int gk20a_channel_wait_semaphore(struct channel_gk20a *ch,
ulong id, u32 offset,
u32 payload, long timeout)
{
struct platform_device *pdev = ch->g->dev;
struct dma_buf *dmabuf;
void *data;
u32 *semaphore;
int ret = 0;
long remain;
/* do not wait if channel has timed out */
if (ch->has_timedout)
return -ETIMEDOUT;
dmabuf = dma_buf_get(id);
if (IS_ERR(dmabuf)) {
gk20a_err(&pdev->dev, "invalid notifier nvmap handle 0x%lx",
id);
return -EINVAL;
}
data = dma_buf_kmap(dmabuf, offset >> PAGE_SHIFT);
if (!data) {
gk20a_err(&pdev->dev, "failed to map notifier memory");
ret = -EINVAL;
goto cleanup_put;
}
semaphore = data + (offset & ~PAGE_MASK);
remain = wait_event_interruptible_timeout(
ch->semaphore_wq,
*semaphore == payload || ch->has_timedout,
timeout);
if (remain == 0 && *semaphore != payload)
ret = -ETIMEDOUT;
else if (remain < 0)
ret = remain;
dma_buf_kunmap(dmabuf, offset >> PAGE_SHIFT, data);
cleanup_put:
dma_buf_put(dmabuf);
return ret;
}
static int gk20a_channel_wait(struct channel_gk20a *ch,
struct nvhost_wait_args *args)
{
struct device *d = dev_from_gk20a(ch->g);
struct dma_buf *dmabuf;
struct notification *notif;
struct timespec tv;
u64 jiffies;
ulong id;
u32 offset;
unsigned long timeout;
int remain, ret = 0;
gk20a_dbg_fn("");
if (ch->has_timedout)
return -ETIMEDOUT;
if (args->timeout == NVHOST_NO_TIMEOUT)
timeout = MAX_SCHEDULE_TIMEOUT;
else
timeout = (u32)msecs_to_jiffies(args->timeout);
switch (args->type) {
case NVHOST_WAIT_TYPE_NOTIFIER:
id = args->condition.notifier.nvmap_handle;
offset = args->condition.notifier.offset;
dmabuf = dma_buf_get(id);
if (IS_ERR(dmabuf)) {
gk20a_err(d, "invalid notifier nvmap handle 0x%lx",
id);
return -EINVAL;
}
notif = dma_buf_vmap(dmabuf);
if (!notif) {
gk20a_err(d, "failed to map notifier memory");
return -ENOMEM;
}
notif = (struct notification *)((uintptr_t)notif + offset);
/* user should set status pending before
* calling this ioctl */
remain = wait_event_interruptible_timeout(
ch->notifier_wq,
notif->status == 0 || ch->has_timedout,
timeout);
if (remain == 0 && notif->status != 0) {
ret = -ETIMEDOUT;
goto notif_clean_up;
} else if (remain < 0) {
ret = -EINTR;
goto notif_clean_up;
}
/* TBD: fill in correct information */
jiffies = get_jiffies_64();
jiffies_to_timespec(jiffies, &tv);
notif->timestamp.nanoseconds[0] = tv.tv_nsec;
notif->timestamp.nanoseconds[1] = tv.tv_sec;
notif->info32 = 0xDEADBEEF; /* should be object name */
notif->info16 = ch->hw_chid; /* should be method offset */
notif_clean_up:
dma_buf_vunmap(dmabuf, notif);
return ret;
case NVHOST_WAIT_TYPE_SEMAPHORE:
ret = gk20a_channel_wait_semaphore(ch,
args->condition.semaphore.nvmap_handle,
args->condition.semaphore.offset,
args->condition.semaphore.payload,
timeout);
break;
default:
ret = -EINVAL;
break;
}
return ret;
}
static int gk20a_channel_set_priority(struct channel_gk20a *ch,
u32 priority)
{
u32 timeslice_timeout;
/* set priority of graphics channel */
switch (priority) {
case NVHOST_PRIORITY_LOW:
/* 64 << 3 = 512us */
timeslice_timeout = 64;
break;
case NVHOST_PRIORITY_MEDIUM:
/* 128 << 3 = 1024us */
timeslice_timeout = 128;
break;
case NVHOST_PRIORITY_HIGH:
/* 255 << 3 = 2048us */
timeslice_timeout = 255;
break;
default:
pr_err("Unsupported priority");
return -EINVAL;
}
channel_gk20a_set_schedule_params(ch,
timeslice_timeout);
return 0;
}
static int gk20a_channel_zcull_bind(struct channel_gk20a *ch,
struct nvhost_zcull_bind_args *args)
{
struct gk20a *g = ch->g;
struct gr_gk20a *gr = &g->gr;
gk20a_dbg_fn("");
return gr_gk20a_bind_ctxsw_zcull(g, gr, ch,
args->gpu_va, args->mode);
}
/* in this context the "channel" is the host1x channel which
* maps to *all* gk20a channels */
int gk20a_channel_suspend(struct gk20a *g)
{
struct fifo_gk20a *f = &g->fifo;
u32 chid;
bool channels_in_use = false;
struct device *d = dev_from_gk20a(g);
int err;
gk20a_dbg_fn("");
/* idle the engine by submitting WFI on non-KEPLER_C channel */
for (chid = 0; chid < f->num_channels; chid++) {
struct channel_gk20a *c = &f->channel[chid];
if (c->in_use && c->obj_class != KEPLER_C) {
gk20a_platform_channel_busy(g->dev);
err = gk20a_channel_submit_wfi(c);
if (err) {
gk20a_err(d, "cannot idle channel %d\n",
chid);
return err;
}
c->sync->wait_cpu(c->sync, &c->last_submit_fence,
500000);
gk20a_platform_channel_idle(g->dev);
break;
}
}
for (chid = 0; chid < f->num_channels; chid++) {
if (f->channel[chid].in_use) {
gk20a_dbg_info("suspend channel %d", chid);
/* disable channel */
gk20a_writel(g, ccsr_channel_r(chid),
gk20a_readl(g, ccsr_channel_r(chid)) |
ccsr_channel_enable_clr_true_f());
/* preempt the channel */
gk20a_fifo_preempt_channel(g, chid);
channels_in_use = true;
}
}
if (channels_in_use) {
gk20a_fifo_update_runlist(g, 0, ~0, false, true);
for (chid = 0; chid < f->num_channels; chid++) {
if (f->channel[chid].in_use)
channel_gk20a_unbind(&f->channel[chid]);
}
}
gk20a_dbg_fn("done");
return 0;
}
/* in this context the "channel" is the host1x channel which
* maps to *all* gk20a channels */
int gk20a_channel_resume(struct gk20a *g)
{
struct fifo_gk20a *f = &g->fifo;
u32 chid;
bool channels_in_use = false;
gk20a_dbg_fn("");
for (chid = 0; chid < f->num_channels; chid++) {
if (f->channel[chid].in_use) {
gk20a_dbg_info("resume channel %d", chid);
g->ops.fifo.bind_channel(&f->channel[chid]);
channels_in_use = true;
}
}
if (channels_in_use)
gk20a_fifo_update_runlist(g, 0, ~0, true, true);
gk20a_dbg_fn("done");
return 0;
}
void gk20a_channel_semaphore_wakeup(struct gk20a *g)
{
struct fifo_gk20a *f = &g->fifo;
u32 chid;
gk20a_dbg_fn("");
for (chid = 0; chid < f->num_channels; chid++) {
struct channel_gk20a *c = g->fifo.channel+chid;
if (c->in_use)
wake_up_interruptible_all(&c->semaphore_wq);
}
}
static int gk20a_ioctl_channel_submit_gpfifo(
struct channel_gk20a *ch,
struct nvhost_submit_gpfifo_args *args)
{
void *gpfifo;
u32 size;
int ret = 0;
gk20a_dbg_fn("");
if (ch->has_timedout)
return -ETIMEDOUT;
size = args->num_entries * sizeof(struct nvhost_gpfifo);
gpfifo = kzalloc(size, GFP_KERNEL);
if (!gpfifo)
return -ENOMEM;
if (copy_from_user(gpfifo,
(void __user *)(uintptr_t)args->gpfifo, size)) {
ret = -EINVAL;
goto clean_up;
}
ret = gk20a_submit_channel_gpfifo(ch, gpfifo, args->num_entries,
&args->fence, args->flags);
clean_up:
kfree(gpfifo);
return ret;
}
void gk20a_init_channel(struct gpu_ops *gops)
{
gops->fifo.bind_channel = channel_gk20a_bind;
}
long gk20a_channel_ioctl(struct file *filp,
unsigned int cmd, unsigned long arg)
{
struct channel_gk20a *ch = filp->private_data;
struct platform_device *dev = ch->g->dev;
u8 buf[NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE];
int err = 0;
if ((_IOC_TYPE(cmd) != NVHOST_IOCTL_MAGIC) ||
(_IOC_NR(cmd) == 0) ||
(_IOC_NR(cmd) > NVHOST_IOCTL_CHANNEL_LAST) ||
(_IOC_SIZE(cmd) > NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE))
return -EFAULT;
if (_IOC_DIR(cmd) & _IOC_WRITE) {
if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
return -EFAULT;
}
switch (cmd) {
case NVHOST_IOCTL_CHANNEL_OPEN:
{
int fd;
struct file *file;
char *name;
err = get_unused_fd_flags(O_RDWR);
if (err < 0)
break;
fd = err;
name = kasprintf(GFP_KERNEL, "nvhost-%s-fd%d",
dev_name(&dev->dev), fd);
if (!name) {
err = -ENOMEM;
put_unused_fd(fd);
break;
}
file = anon_inode_getfile(name, filp->f_op, NULL, O_RDWR);
kfree(name);
if (IS_ERR(file)) {
err = PTR_ERR(file);
put_unused_fd(fd);
break;
}
fd_install(fd, file);
err = __gk20a_channel_open(ch->g, file);
if (err) {
put_unused_fd(fd);
fput(file);
break;
}
((struct nvhost_channel_open_args *)buf)->channel_fd = fd;
break;
}
case NVHOST_IOCTL_CHANNEL_SET_NVMAP_FD:
break;
case NVHOST_IOCTL_CHANNEL_ALLOC_OBJ_CTX:
gk20a_channel_busy(dev);
err = gk20a_alloc_obj_ctx(ch,
(struct nvhost_alloc_obj_ctx_args *)buf);
gk20a_channel_idle(dev);
break;
case NVHOST_IOCTL_CHANNEL_FREE_OBJ_CTX:
gk20a_channel_busy(dev);
err = gk20a_free_obj_ctx(ch,
(struct nvhost_free_obj_ctx_args *)buf);
gk20a_channel_idle(dev);
break;
case NVHOST_IOCTL_CHANNEL_ALLOC_GPFIFO:
gk20a_channel_busy(dev);
err = gk20a_alloc_channel_gpfifo(ch,
(struct nvhost_alloc_gpfifo_args *)buf);
gk20a_channel_idle(dev);
break;
case NVHOST_IOCTL_CHANNEL_SUBMIT_GPFIFO:
err = gk20a_ioctl_channel_submit_gpfifo(ch,
(struct nvhost_submit_gpfifo_args *)buf);
break;
case NVHOST_IOCTL_CHANNEL_WAIT:
gk20a_channel_busy(dev);
err = gk20a_channel_wait(ch,
(struct nvhost_wait_args *)buf);
gk20a_channel_idle(dev);
break;
case NVHOST_IOCTL_CHANNEL_ZCULL_BIND:
gk20a_channel_busy(dev);
err = gk20a_channel_zcull_bind(ch,
(struct nvhost_zcull_bind_args *)buf);
gk20a_channel_idle(dev);
break;
case NVHOST_IOCTL_CHANNEL_SET_ERROR_NOTIFIER:
gk20a_channel_busy(dev);
err = gk20a_init_error_notifier(ch,
(struct nvhost_set_error_notifier *)buf);
gk20a_channel_idle(dev);
break;
#ifdef CONFIG_GK20A_CYCLE_STATS
case NVHOST_IOCTL_CHANNEL_CYCLE_STATS:
gk20a_channel_busy(dev);
err = gk20a_channel_cycle_stats(ch,
(struct nvhost_cycle_stats_args *)buf);
gk20a_channel_idle(dev);
break;
#endif
case NVHOST_IOCTL_CHANNEL_SET_TIMEOUT:
{
u32 timeout =
(u32)((struct nvhost_set_timeout_args *)buf)->timeout;
gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
timeout, ch->hw_chid);
ch->timeout_ms_max = timeout;
break;
}
case NVHOST_IOCTL_CHANNEL_SET_TIMEOUT_EX:
{
u32 timeout =
(u32)((struct nvhost_set_timeout_args *)buf)->timeout;
bool timeout_debug_dump = !((u32)
((struct nvhost_set_timeout_ex_args *)buf)->flags &
(1 << NVHOST_TIMEOUT_FLAG_DISABLE_DUMP));
gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
timeout, ch->hw_chid);
ch->timeout_ms_max = timeout;
ch->timeout_debug_dump = timeout_debug_dump;
break;
}
case NVHOST_IOCTL_CHANNEL_GET_TIMEDOUT:
((struct nvhost_get_param_args *)buf)->value =
ch->has_timedout;
break;
case NVHOST_IOCTL_CHANNEL_SET_PRIORITY:
gk20a_channel_busy(dev);
gk20a_channel_set_priority(ch,
((struct nvhost_set_priority_args *)buf)->priority);
gk20a_channel_idle(dev);
break;
default:
dev_err(&dev->dev, "unrecognized ioctl cmd: 0x%x", cmd);
err = -ENOTTY;
break;
}
if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
return err;
}