/*
* GK20A Graphics FIFO (gr host)
*
* Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include <linux/delay.h>
#include <linux/scatterlist.h>
#include <trace/events/gk20a.h>
#include <linux/dma-mapping.h>
#ifdef CONFIG_TEGRA_GK20A_NVHOST
#include <linux/nvhost.h>
#endif
#include <linux/sort.h>
#include <nvgpu/dma.h>
#include <nvgpu/timers.h>
#include <nvgpu/semaphore.h>
#include <nvgpu/kmem.h>
#include <nvgpu/log.h>
#include <nvgpu/soc.h>
#include <nvgpu/atomic.h>
#include "gk20a.h"
#include "debug_gk20a.h"
#include "ctxsw_trace_gk20a.h"
#include <nvgpu/hw/gk20a/hw_fifo_gk20a.h>
#include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h>
#include <nvgpu/hw/gk20a/hw_ccsr_gk20a.h>
#include <nvgpu/hw/gk20a/hw_ram_gk20a.h>
#include <nvgpu/hw/gk20a/hw_top_gk20a.h>
#include <nvgpu/hw/gk20a/hw_mc_gk20a.h>
#include <nvgpu/hw/gk20a/hw_gr_gk20a.h>
#define FECS_METHOD_WFI_RESTORE 0x80000
static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
u32 hw_chid, bool add,
bool wait_for_finish);
static u32 gk20a_fifo_engines_on_id(struct gk20a *g, u32 id, bool is_tsg);
#ifdef CONFIG_DEBUG_FS
static void __gk20a_fifo_profile_free(struct kref *ref);
#endif
u32 gk20a_fifo_get_engine_ids(struct gk20a *g,
u32 engine_id[], u32 engine_id_sz,
u32 engine_enum)
{
struct fifo_gk20a *f = NULL;
u32 instance_cnt = 0;
u32 engine_id_idx;
u32 active_engine_id = 0;
struct fifo_engine_info_gk20a *info = NULL;
if (g && engine_id_sz && (engine_enum < ENGINE_INVAL_GK20A)) {
f = &g->fifo;
for (engine_id_idx = 0; engine_id_idx < f->num_engines; ++engine_id_idx) {
active_engine_id = f->active_engines_list[engine_id_idx];
info = &f->engine_info[active_engine_id];
if (info->engine_enum == engine_enum) {
if (instance_cnt < engine_id_sz) {
engine_id[instance_cnt] = active_engine_id;
++instance_cnt;
} else {
gk20a_dbg_info("warning engine_id table sz is small %d",
engine_id_sz);
}
}
}
}
return instance_cnt;
}
struct fifo_engine_info_gk20a *gk20a_fifo_get_engine_info(struct gk20a *g, u32 engine_id)
{
struct fifo_gk20a *f = NULL;
u32 engine_id_idx;
struct fifo_engine_info_gk20a *info = NULL;
if (!g)
return info;
f = &g->fifo;
if (engine_id < f->max_engines) {
for (engine_id_idx = 0; engine_id_idx < f->num_engines; ++engine_id_idx) {
if (engine_id == f->active_engines_list[engine_id_idx]) {
info = &f->engine_info[engine_id];
break;
}
}
}
if (!info)
nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id);
return info;
}
bool gk20a_fifo_is_valid_engine_id(struct gk20a *g, u32 engine_id)
{
struct fifo_gk20a *f = NULL;
u32 engine_id_idx;
bool valid = false;
if (!g)
return valid;
f = &g->fifo;
if (engine_id < f->max_engines) {
for (engine_id_idx = 0; engine_id_idx < f->num_engines; ++engine_id_idx) {
if (engine_id == f->active_engines_list[engine_id_idx]) {
valid = true;
break;
}
}
}
if (!valid)
nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id);
return valid;
}
u32 gk20a_fifo_get_gr_engine_id(struct gk20a *g)
{
u32 gr_engine_cnt = 0;
u32 gr_engine_id = FIFO_INVAL_ENGINE_ID;
/* Consider 1st available GR engine */
gr_engine_cnt = gk20a_fifo_get_engine_ids(g, &gr_engine_id,
1, ENGINE_GR_GK20A);
if (!gr_engine_cnt) {
nvgpu_err(g, "No GR engine available on this device!\n");
}
return gr_engine_id;
}
u32 gk20a_fifo_get_all_ce_engine_reset_mask(struct gk20a *g)
{
u32 reset_mask = 0;
u32 engine_enum = ENGINE_INVAL_GK20A;
struct fifo_gk20a *f = NULL;
u32 engine_id_idx;
struct fifo_engine_info_gk20a *engine_info;
u32 active_engine_id = 0;
if (!g)
return reset_mask;
f = &g->fifo;
for (engine_id_idx = 0; engine_id_idx < f->num_engines; ++engine_id_idx) {
active_engine_id = f->active_engines_list[engine_id_idx];
engine_info = &f->engine_info[active_engine_id];
engine_enum = engine_info->engine_enum;
if ((engine_enum == ENGINE_GRCE_GK20A) ||
(engine_enum == ENGINE_ASYNC_CE_GK20A))
reset_mask |= engine_info->reset_mask;
}
return reset_mask;
}
u32 gk20a_fifo_get_fast_ce_runlist_id(struct gk20a *g)
{
u32 ce_runlist_id = gk20a_fifo_get_gr_runlist_id(g);
u32 engine_enum = ENGINE_INVAL_GK20A;
struct fifo_gk20a *f = NULL;
u32 engine_id_idx;
struct fifo_engine_info_gk20a *engine_info;
u32 active_engine_id = 0;
if (!g)
return ce_runlist_id;
f = &g->fifo;
for (engine_id_idx = 0; engine_id_idx < f->num_engines; ++engine_id_idx) {
active_engine_id = f->active_engines_list[engine_id_idx];
engine_info = &f->engine_info[active_engine_id];
engine_enum = engine_info->engine_enum;
/* selecet last available ASYNC_CE if available */
if (engine_enum == ENGINE_ASYNC_CE_GK20A)
ce_runlist_id = engine_info->runlist_id;
}
return ce_runlist_id;
}
u32 gk20a_fifo_get_gr_runlist_id(struct gk20a *g)
{
u32 gr_engine_cnt = 0;
u32 gr_engine_id = FIFO_INVAL_ENGINE_ID;
struct fifo_engine_info_gk20a *engine_info;
u32 gr_runlist_id = ~0;
/* Consider 1st available GR engine */
gr_engine_cnt = gk20a_fifo_get_engine_ids(g, &gr_engine_id,
1, ENGINE_GR_GK20A);
if (!gr_engine_cnt) {
nvgpu_err(g,
"No GR engine available on this device!");
goto end;
}
engine_info = gk20a_fifo_get_engine_info(g, gr_engine_id);
if (engine_info) {
gr_runlist_id = engine_info->runlist_id;
} else {
nvgpu_err(g,
"gr_engine_id is not in active list/invalid %d", gr_engine_id);
}
end:
return gr_runlist_id;
}
bool gk20a_fifo_is_valid_runlist_id(struct gk20a *g, u32 runlist_id)
{
struct fifo_gk20a *f = NULL;
u32 engine_id_idx;
u32 active_engine_id;
struct fifo_engine_info_gk20a *engine_info;
if (!g)
return false;
f = &g->fifo;
for (engine_id_idx = 0; engine_id_idx < f->num_engines; ++engine_id_idx) {
active_engine_id = f->active_engines_list[engine_id_idx];
engine_info = gk20a_fifo_get_engine_info(g, active_engine_id);
if (engine_info && (engine_info->runlist_id == runlist_id)) {
return true;
}
}
return false;
}
/*
* Link engine IDs to MMU IDs and vice versa.
*/
static inline u32 gk20a_engine_id_to_mmu_id(struct gk20a *g, u32 engine_id)
{
u32 fault_id = FIFO_INVAL_ENGINE_ID;
struct fifo_engine_info_gk20a *engine_info;
engine_info = gk20a_fifo_get_engine_info(g, engine_id);
if (engine_info) {
fault_id = engine_info->fault_id;
} else {
nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id);
}
return fault_id;
}
static inline u32 gk20a_mmu_id_to_engine_id(struct gk20a *g, u32 fault_id)
{
u32 engine_id;
u32 active_engine_id;
struct fifo_engine_info_gk20a *engine_info;
struct fifo_gk20a *f = &g->fifo;
for (engine_id = 0; engine_id < f->num_engines; engine_id++) {
active_engine_id = f->active_engines_list[engine_id];
engine_info = &g->fifo.engine_info[active_engine_id];
if (engine_info->fault_id == fault_id)
break;
active_engine_id = FIFO_INVAL_ENGINE_ID;
}
return active_engine_id;
}
int gk20a_fifo_engine_enum_from_type(struct gk20a *g, u32 engine_type,
u32 *inst_id)
{
int ret = ENGINE_INVAL_GK20A;
gk20a_dbg_info("engine type %d", engine_type);
if (engine_type == top_device_info_type_enum_graphics_v())
ret = ENGINE_GR_GK20A;
else if ((engine_type >= top_device_info_type_enum_copy0_v()) &&
(engine_type <= top_device_info_type_enum_copy2_v())) {
/* Lets consider all the CE engine have separate runlist at this point
* We can identify the ENGINE_GRCE_GK20A type CE using runlist_id
* comparsion logic with GR runlist_id in init_engine_info() */
ret = ENGINE_ASYNC_CE_GK20A;
/* inst_id starts from CE0 to CE2 */
if (inst_id)
*inst_id = (engine_type - top_device_info_type_enum_copy0_v());
}
return ret;
}
int gk20a_fifo_init_engine_info(struct fifo_gk20a *f)
{
struct gk20a *g = f->g;
u32 i;
u32 max_info_entries = top_device_info__size_1_v();
u32 engine_enum = ENGINE_INVAL_GK20A;
u32 engine_id = FIFO_INVAL_ENGINE_ID;
u32 runlist_id = ~0;
u32 pbdma_id = ~0;
u32 intr_id = ~0;
u32 reset_id = ~0;
u32 inst_id = 0;
u32 pri_base = 0;
u32 fault_id = 0;
u32 gr_runlist_id = ~0;
bool found_pbdma_for_runlist = false;
gk20a_dbg_fn("");
f->num_engines = 0;
for (i = 0; i < max_info_entries; i++) {
u32 table_entry = gk20a_readl(f->g, top_device_info_r(i));
u32 entry = top_device_info_entry_v(table_entry);
u32 runlist_bit;
if (entry == top_device_info_entry_enum_v()) {
if (top_device_info_engine_v(table_entry)) {
engine_id =
top_device_info_engine_enum_v(table_entry);
gk20a_dbg_info("info: engine_id %d",
top_device_info_engine_enum_v(table_entry));
}
if (top_device_info_runlist_v(table_entry)) {
runlist_id =
top_device_info_runlist_enum_v(table_entry);
gk20a_dbg_info("gr info: runlist_id %d", runlist_id);
runlist_bit = BIT(runlist_id);
found_pbdma_for_runlist = false;
for (pbdma_id = 0; pbdma_id < f->num_pbdma;
pbdma_id++) {
if (f->pbdma_map[pbdma_id] &
runlist_bit) {
gk20a_dbg_info(
"gr info: pbdma_map[%d]=%d",
pbdma_id,
f->pbdma_map[pbdma_id]);
found_pbdma_for_runlist = true;
}
}
if (!found_pbdma_for_runlist) {
nvgpu_err(g, "busted pbdma map");
return -EINVAL;
}
}
if (top_device_info_intr_v(table_entry)) {
intr_id =
top_device_info_intr_enum_v(table_entry);
gk20a_dbg_info("gr info: intr_id %d", intr_id);
}
if (top_device_info_reset_v(table_entry)) {
reset_id =
top_device_info_reset_enum_v(table_entry);
gk20a_dbg_info("gr info: reset_id %d",
reset_id);
}
} else if (entry == top_device_info_entry_engine_type_v()) {
u32 engine_type =
top_device_info_type_enum_v(table_entry);
engine_enum =
g->ops.fifo.engine_enum_from_type(g,
engine_type, &inst_id);
} else if (entry == top_device_info_entry_data_v()) {
/* gk20a doesn't support device_info_data packet parsing */
if (g->ops.fifo.device_info_data_parse)
g->ops.fifo.device_info_data_parse(g,
table_entry, &inst_id, &pri_base,
&fault_id);
}
if (!top_device_info_chain_v(table_entry)) {
if (engine_enum < ENGINE_INVAL_GK20A) {
struct fifo_engine_info_gk20a *info =
&g->fifo.engine_info[engine_id];
info->intr_mask |= BIT(intr_id);
info->reset_mask |= BIT(reset_id);
info->runlist_id = runlist_id;
info->pbdma_id = pbdma_id;
info->inst_id = inst_id;
info->pri_base = pri_base;
if (engine_enum == ENGINE_GR_GK20A)
gr_runlist_id = runlist_id;
/* GR and GR_COPY shares same runlist_id */
if ((engine_enum == ENGINE_ASYNC_CE_GK20A) &&
(gr_runlist_id == runlist_id))
engine_enum = ENGINE_GRCE_GK20A;
info->engine_enum = engine_enum;
if (!fault_id && (engine_enum == ENGINE_GRCE_GK20A))
fault_id = 0x1b;
info->fault_id = fault_id;
/* engine_id starts from 0 to NV_HOST_NUM_ENGINES */
f->active_engines_list[f->num_engines] = engine_id;
++f->num_engines;
engine_enum = ENGINE_INVAL_GK20A;
}
}
}
return 0;
}
u32 gk20a_fifo_engine_interrupt_mask(struct gk20a *g)
{
u32 eng_intr_mask = 0;
unsigned int i;
u32 active_engine_id = 0;
u32 engine_enum = ENGINE_INVAL_GK20A;
for (i = 0; i < g->fifo.num_engines; i++) {
u32 intr_mask;
active_engine_id = g->fifo.active_engines_list[i];
intr_mask = g->fifo.engine_info[active_engine_id].intr_mask;
engine_enum = g->fifo.engine_info[active_engine_id].engine_enum;
if (((engine_enum == ENGINE_GRCE_GK20A) ||
(engine_enum == ENGINE_ASYNC_CE_GK20A)) &&
| 