From 40ca7cc573430ca4e21fdec4a44394c09d615846 Mon Sep 17 00:00:00 2001 From: Mahantesh Kumbar Date: Tue, 9 May 2017 15:49:43 +0530 Subject: gpu: nvgpu: reorganize PMU IPC - Moved PMU IPC related code to drivers/gpu/nvgpu/common/pmu/pmu_ipc.c file, -Below is the list which are moved seq mutex queue cmd/msg post & process event handling NVGPU-56 Change-Id: Ic380faa27de4e5574d5b22500125e86027fd4b5d Signed-off-by: Mahantesh Kumbar Reviewed-on: http://git-master/r/1478167 GVS: Gerrit_Virtual_Submit Reviewed-by: svccoveritychecker Reviewed-by: Terje Bergstrom --- drivers/gpu/nvgpu/Makefile.nvgpu | 1 + drivers/gpu/nvgpu/common/pmu/pmu_ipc.c | 893 +++++++++++++++++++++++++++++ drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | 20 +- drivers/gpu/nvgpu/gk20a/gk20a.h | 8 + drivers/gpu/nvgpu/gk20a/pmu_gk20a.c | 994 +++------------------------------ drivers/gpu/nvgpu/gk20a/pmu_gk20a.h | 27 +- drivers/gpu/nvgpu/gm20b/pmu_gm20b.c | 4 + drivers/gpu/nvgpu/gp106/pmu_gp106.c | 4 + drivers/gpu/nvgpu/gp10b/pmu_gp10b.c | 4 + drivers/gpu/nvgpu/include/nvgpu/pmu.h | 38 ++ 10 files changed, 1053 insertions(+), 940 deletions(-) create mode 100644 drivers/gpu/nvgpu/common/pmu/pmu_ipc.c diff --git a/drivers/gpu/nvgpu/Makefile.nvgpu b/drivers/gpu/nvgpu/Makefile.nvgpu index 5d06aad2..35e9d84a 100644 --- a/drivers/gpu/nvgpu/Makefile.nvgpu +++ b/drivers/gpu/nvgpu/Makefile.nvgpu @@ -58,6 +58,7 @@ nvgpu-y := \ common/rbtree.o \ common/vbios/bios.o \ common/falcon/falcon.o \ + common/pmu/pmu_ipc.o \ gk20a/gk20a.o \ gk20a/bus_gk20a.o \ gk20a/pramin_gk20a.o \ diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_ipc.c b/drivers/gpu/nvgpu/common/pmu/pmu_ipc.c new file mode 100644 index 00000000..74966f2d --- /dev/null +++ b/drivers/gpu/nvgpu/common/pmu/pmu_ipc.c @@ -0,0 +1,893 @@ +/* + * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include +#include +#include + +#include "gk20a/gk20a.h" + +void nvgpu_pmu_seq_init(struct nvgpu_pmu *pmu) +{ + u32 i; + + memset(pmu->seq, 0, + sizeof(struct pmu_sequence) * PMU_MAX_NUM_SEQUENCES); + memset(pmu->pmu_seq_tbl, 0, + sizeof(pmu->pmu_seq_tbl)); + + for (i = 0; i < PMU_MAX_NUM_SEQUENCES; i++) + pmu->seq[i].id = i; +} + +static int pmu_seq_acquire(struct nvgpu_pmu *pmu, + struct pmu_sequence **pseq) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + struct pmu_sequence *seq; + u32 index; + + nvgpu_mutex_acquire(&pmu->pmu_seq_lock); + index = find_first_zero_bit(pmu->pmu_seq_tbl, + sizeof(pmu->pmu_seq_tbl)); + if (index >= sizeof(pmu->pmu_seq_tbl)) { + nvgpu_err(g, "no free sequence available"); + nvgpu_mutex_release(&pmu->pmu_seq_lock); + return -EAGAIN; + } + set_bit(index, pmu->pmu_seq_tbl); + nvgpu_mutex_release(&pmu->pmu_seq_lock); + + seq = &pmu->seq[index]; + seq->state = PMU_SEQ_STATE_PENDING; + + *pseq = seq; + return 0; +} + +static void pmu_seq_release(struct nvgpu_pmu *pmu, + struct pmu_sequence *seq) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + + seq->state = PMU_SEQ_STATE_FREE; + seq->desc = PMU_INVALID_SEQ_DESC; + seq->callback = NULL; + seq->cb_params = NULL; + seq->msg = NULL; + seq->out_payload = NULL; + g->ops.pmu_ver.pmu_allocation_set_dmem_size(pmu, + g->ops.pmu_ver.get_pmu_seq_in_a_ptr(seq), 0); + g->ops.pmu_ver.pmu_allocation_set_dmem_size(pmu, + g->ops.pmu_ver.get_pmu_seq_out_a_ptr(seq), 0); + + clear_bit(seq->id, pmu->pmu_seq_tbl); +} +/* mutex */ +int nvgpu_pmu_mutex_acquire(struct nvgpu_pmu *pmu, u32 id, u32 *token) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + + return g->ops.pmu.pmu_mutex_acquire(pmu, id, token); +} + +int nvgpu_pmu_mutex_release(struct nvgpu_pmu *pmu, u32 id, u32 *token) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + + return g->ops.pmu.pmu_mutex_release(pmu, id, token); +} + +/* queue */ +int nvgpu_pmu_queue_init(struct nvgpu_pmu *pmu, + u32 id, union pmu_init_msg_pmu *init) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + struct pmu_queue *queue = &pmu->queue[id]; + int err; + + err = nvgpu_mutex_init(&queue->mutex); + if (err) + return err; + + queue->id = id; + g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params(queue, id, init); + queue->mutex_id = id; + + nvgpu_pmu_dbg(g, "queue %d: index %d, offset 0x%08x, size 0x%08x", + id, queue->index, queue->offset, queue->size); + + return 0; +} + +static int pmu_queue_head(struct nvgpu_pmu *pmu, struct pmu_queue *queue, + u32 *head, bool set) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + + return g->ops.pmu.pmu_queue_head(pmu, queue, head, set); +} + +static int pmu_queue_tail(struct nvgpu_pmu *pmu, struct pmu_queue *queue, + u32 *tail, bool set) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + + return g->ops.pmu.pmu_queue_tail(pmu, queue, tail, set); +} + +static inline void pmu_queue_read(struct nvgpu_pmu *pmu, + u32 offset, u8 *dst, u32 size) +{ + pmu_copy_from_dmem(pmu, offset, dst, size, 0); +} + +static inline void pmu_queue_write(struct nvgpu_pmu *pmu, + u32 offset, u8 *src, u32 size) +{ + pmu_copy_to_dmem(pmu, offset, src, size, 0); +} + + +static int pmu_queue_lock(struct nvgpu_pmu *pmu, + struct pmu_queue *queue) +{ + int err; + + if (PMU_IS_MESSAGE_QUEUE(queue->id)) + return 0; + + if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) { + nvgpu_mutex_acquire(&queue->mutex); + return 0; + } + + err = nvgpu_pmu_mutex_acquire(pmu, queue->mutex_id, &queue->mutex_lock); + return err; +} + +static int pmu_queue_unlock(struct nvgpu_pmu *pmu, + struct pmu_queue *queue) +{ + int err; + + if (PMU_IS_MESSAGE_QUEUE(queue->id)) + return 0; + + if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) { + nvgpu_mutex_release(&queue->mutex); + return 0; + } + + err = nvgpu_pmu_mutex_release(pmu, queue->mutex_id, &queue->mutex_lock); + return err; +} + +/* called by pmu_read_message, no lock */ +bool nvgpu_pmu_queue_is_empty(struct nvgpu_pmu *pmu, + struct pmu_queue *queue) +{ + u32 head, tail; + + pmu_queue_head(pmu, queue, &head, QUEUE_GET); + if (queue->opened && queue->oflag == OFLAG_READ) + tail = queue->position; + else + pmu_queue_tail(pmu, queue, &tail, QUEUE_GET); + + return head == tail; +} + +static bool pmu_queue_has_room(struct nvgpu_pmu *pmu, + struct pmu_queue *queue, u32 size, bool *need_rewind) +{ + u32 head, tail; + bool rewind = false; + unsigned int free; + + size = ALIGN(size, QUEUE_ALIGNMENT); + + pmu_queue_head(pmu, queue, &head, QUEUE_GET); + pmu_queue_tail(pmu, queue, &tail, QUEUE_GET); + if (head >= tail) { + free = queue->offset + queue->size - head; + free -= PMU_CMD_HDR_SIZE; + + if (size > free) { + rewind = true; + head = queue->offset; + } + } + + if (head < tail) + free = tail - head - 1; + + if (need_rewind) + *need_rewind = rewind; + + return size <= free; +} + +static int pmu_queue_push(struct nvgpu_pmu *pmu, + struct pmu_queue *queue, void *data, u32 size) +{ + + gk20a_dbg_fn(""); + + if (!queue->opened && queue->oflag == OFLAG_WRITE) { + nvgpu_err(gk20a_from_pmu(pmu), "queue not opened for write"); + return -EINVAL; + } + + pmu_queue_write(pmu, queue->position, data, size); + queue->position += ALIGN(size, QUEUE_ALIGNMENT); + return 0; +} + +static int pmu_queue_pop(struct nvgpu_pmu *pmu, + struct pmu_queue *queue, void *data, u32 size, + u32 *bytes_read) +{ + u32 head, tail, used; + + *bytes_read = 0; + + if (!queue->opened && queue->oflag == OFLAG_READ) { + nvgpu_err(gk20a_from_pmu(pmu), "queue not opened for read"); + return -EINVAL; + } + + pmu_queue_head(pmu, queue, &head, QUEUE_GET); + tail = queue->position; + + if (head == tail) + return 0; + + if (head > tail) + used = head - tail; + else + used = queue->offset + queue->size - tail; + + if (size > used) { + nvgpu_warn(gk20a_from_pmu(pmu), + "queue size smaller than request read"); + size = used; + } + + pmu_queue_read(pmu, tail, data, size); + queue->position += ALIGN(size, QUEUE_ALIGNMENT); + *bytes_read = size; + return 0; +} + +static void pmu_queue_rewind(struct nvgpu_pmu *pmu, + struct pmu_queue *queue) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + struct pmu_cmd cmd; + + gk20a_dbg_fn(""); + + if (!queue->opened) { + nvgpu_err(gk20a_from_pmu(pmu), "queue not opened"); + return; + } + + if (queue->oflag == OFLAG_WRITE) { + cmd.hdr.unit_id = PMU_UNIT_REWIND; + cmd.hdr.size = PMU_CMD_HDR_SIZE; + pmu_queue_push(pmu, queue, &cmd, cmd.hdr.size); + nvgpu_pmu_dbg(g, "queue %d rewinded", queue->id); + } + + queue->position = queue->offset; +} + +/* open for read and lock the queue */ +static int pmu_queue_open_read(struct nvgpu_pmu *pmu, + struct pmu_queue *queue) +{ + int err; + + err = pmu_queue_lock(pmu, queue); + if (err) + return err; + + if (queue->opened) + BUG(); + + pmu_queue_tail(pmu, queue, &queue->position, QUEUE_GET); + queue->oflag = OFLAG_READ; + queue->opened = true; + + return 0; +} + +/* open for write and lock the queue + * make sure there's enough free space for the write + * */ +static int pmu_queue_open_write(struct nvgpu_pmu *pmu, + struct pmu_queue *queue, u32 size) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + bool rewind = false; + int err; + + err = pmu_queue_lock(pmu, queue); + if (err) + return err; + + if (queue->opened) + BUG(); + + if (!pmu_queue_has_room(pmu, queue, size, &rewind)) { + nvgpu_pmu_dbg(g, "queue full: queue-id %d: index %d", + queue->id, queue->index); + pmu_queue_unlock(pmu, queue); + return -EAGAIN; + } + + pmu_queue_head(pmu, queue, &queue->position, QUEUE_GET); + queue->oflag = OFLAG_WRITE; + queue->opened = true; + + if (rewind) + pmu_queue_rewind(pmu, queue); + + return 0; +} + +/* close and unlock the queue */ +static int pmu_queue_close(struct nvgpu_pmu *pmu, + struct pmu_queue *queue, bool commit) +{ + if (!queue->opened) + return 0; + + if (commit) { + if (queue->oflag == OFLAG_READ) + pmu_queue_tail(pmu, queue, + &queue->position, QUEUE_SET); + else + pmu_queue_head(pmu, queue, + &queue->position, QUEUE_SET); + } + + queue->opened = false; + + pmu_queue_unlock(pmu, queue); + + return 0; +} + +static bool pmu_validate_cmd(struct nvgpu_pmu *pmu, struct pmu_cmd *cmd, + struct pmu_msg *msg, struct pmu_payload *payload, + u32 queue_id) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + struct pmu_queue *queue; + u32 in_size, out_size; + + if (!PMU_IS_SW_COMMAND_QUEUE(queue_id)) + goto invalid_cmd; + + queue = &pmu->queue[queue_id]; + if (cmd->hdr.size < PMU_CMD_HDR_SIZE) + goto invalid_cmd; + + if (cmd->hdr.size > (queue->size >> 1)) + goto invalid_cmd; + + if (msg != NULL && msg->hdr.size < PMU_MSG_HDR_SIZE) + goto invalid_cmd; + + if (!PMU_UNIT_ID_IS_VALID(cmd->hdr.unit_id)) + goto invalid_cmd; + + if (payload == NULL) + return true; + + if (payload->in.buf == NULL && payload->out.buf == NULL) + goto invalid_cmd; + + if ((payload->in.buf != NULL && payload->in.size == 0) || + (payload->out.buf != NULL && payload->out.size == 0)) + goto invalid_cmd; + + in_size = PMU_CMD_HDR_SIZE; + if (payload->in.buf) { + in_size += payload->in.offset; + in_size += g->ops.pmu_ver.get_pmu_allocation_struct_size(pmu); + } + + out_size = PMU_CMD_HDR_SIZE; + if (payload->out.buf) { + out_size += payload->out.offset; + out_size += g->ops.pmu_ver.get_pmu_allocation_struct_size(pmu); + } + + if (in_size > cmd->hdr.size || out_size > cmd->hdr.size) + goto invalid_cmd; + + + if ((payload->in.offset != 0 && payload->in.buf == NULL) || + (payload->out.offset != 0 && payload->out.buf == NULL)) + goto invalid_cmd; + + return true; + +invalid_cmd: + nvgpu_err(g, "invalid pmu cmd :\n" + "queue_id=%d,\n" + "cmd_size=%d, cmd_unit_id=%d, msg=%p, msg_size=%d,\n" + "payload in=%p, in_size=%d, in_offset=%d,\n" + "payload out=%p, out_size=%d, out_offset=%d", + queue_id, cmd->hdr.size, cmd->hdr.unit_id, + msg, msg ? msg->hdr.unit_id : ~0, + &payload->in, payload->in.size, payload->in.offset, + &payload->out, payload->out.size, payload->out.offset); + + return false; +} + +static int pmu_write_cmd(struct nvgpu_pmu *pmu, struct pmu_cmd *cmd, + u32 queue_id, unsigned long timeout_ms) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + struct pmu_queue *queue; + struct nvgpu_timeout timeout; + int err; + + gk20a_dbg_fn(""); + + queue = &pmu->queue[queue_id]; + nvgpu_timeout_init(g, &timeout, (int)timeout_ms, NVGPU_TIMER_CPU_TIMER); + + do { + err = pmu_queue_open_write(pmu, queue, cmd->hdr.size); + if (err == -EAGAIN && !nvgpu_timeout_expired(&timeout)) + nvgpu_usleep_range(1000, 2000); + else + break; + } while (1); + + if (err) + goto clean_up; + + pmu_queue_push(pmu, queue, cmd, cmd->hdr.size); + + + err = pmu_queue_close(pmu, queue, true); + +clean_up: + if (err) + nvgpu_err(g, "fail to write cmd to queue %d", queue_id); + else + gk20a_dbg_fn("done"); + + return err; +} + +int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd, + struct pmu_msg *msg, struct pmu_payload *payload, + u32 queue_id, pmu_callback callback, void *cb_param, + u32 *seq_desc, unsigned long timeout) +{ + struct nvgpu_pmu *pmu = &g->pmu; + struct pmu_v *pv = &g->ops.pmu_ver; + struct pmu_sequence *seq; + void *in = NULL, *out = NULL; + int err; + + gk20a_dbg_fn(""); + + if ((!cmd) || (!seq_desc) || (!pmu->pmu_ready)) { + if (!cmd) + nvgpu_warn(g, "%s(): PMU cmd buffer is NULL", __func__); + else if (!seq_desc) + nvgpu_warn(g, "%s(): Seq descriptor is NULL", __func__); + else + nvgpu_warn(g, "%s(): PMU is not ready", __func__); + + WARN_ON(1); + return -EINVAL; + } + + if (!pmu_validate_cmd(pmu, cmd, msg, payload, queue_id)) + return -EINVAL; + + err = pmu_seq_acquire(pmu, &seq); + if (err) + return err; + + cmd->hdr.seq_id = seq->id; + + cmd->hdr.ctrl_flags = 0; + cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_STATUS; + cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_INTR; + + seq->callback = callback; + seq->cb_params = cb_param; + seq->msg = msg; + seq->out_payload = NULL; + seq->desc = pmu->next_seq_desc++; + + if (payload) + seq->out_payload = payload->out.buf; + + *seq_desc = seq->desc; + + if (payload && payload->in.offset != 0) { + pv->set_pmu_allocation_ptr(pmu, &in, + ((u8 *)&cmd->cmd + payload->in.offset)); + + if (payload->in.buf != payload->out.buf) + pv->pmu_allocation_set_dmem_size(pmu, in, + (u16)payload->in.size); + else + pv->pmu_allocation_set_dmem_size(pmu, in, + (u16)max(payload->in.size, payload->out.size)); + + *(pv->pmu_allocation_get_dmem_offset_addr(pmu, in)) = + nvgpu_alloc(&pmu->dmem, + pv->pmu_allocation_get_dmem_size(pmu, in)); + if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu, in))) + goto clean_up; + + if (payload->in.fb_size != 0x0) { + seq->in_mem = nvgpu_kzalloc(g, + sizeof(struct nvgpu_mem)); + if (!seq->in_mem) { + err = -ENOMEM; + goto clean_up; + } + + gk20a_pmu_vidmem_surface_alloc(g, seq->in_mem, + payload->in.fb_size); + gk20a_pmu_surface_describe(g, seq->in_mem, + (struct flcn_mem_desc_v0 *) + pv->pmu_allocation_get_fb_addr(pmu, in)); + + nvgpu_mem_wr_n(g, seq->in_mem, 0, + payload->in.buf, payload->in.fb_size); + + } else { + pmu_copy_to_dmem(pmu, + (pv->pmu_allocation_get_dmem_offset(pmu, in)), + payload->in.buf, payload->in.size, 0); + } + pv->pmu_allocation_set_dmem_size(pmu, + pv->get_pmu_seq_in_a_ptr(seq), + pv->pmu_allocation_get_dmem_size(pmu, in)); + pv->pmu_allocation_set_dmem_offset(pmu, + pv->get_pmu_seq_in_a_ptr(seq), + pv->pmu_allocation_get_dmem_offset(pmu, in)); + } + + if (payload && payload->out.offset != 0) { + pv->set_pmu_allocation_ptr(pmu, &out, + ((u8 *)&cmd->cmd + payload->out.offset)); + pv->pmu_allocation_set_dmem_size(pmu, out, + (u16)payload->out.size); + + if (payload->in.buf != payload->out.buf) { + *(pv->pmu_allocation_get_dmem_offset_addr(pmu, out)) = + nvgpu_alloc(&pmu->dmem, + pv->pmu_allocation_get_dmem_size(pmu, out)); + if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu, + out))) + goto clean_up; + + if (payload->out.fb_size != 0x0) { + seq->out_mem = nvgpu_kzalloc(g, + sizeof(struct nvgpu_mem)); + if (!seq->out_mem) { + err = -ENOMEM; + goto clean_up; + } + gk20a_pmu_vidmem_surface_alloc(g, seq->out_mem, + payload->out.fb_size); + gk20a_pmu_surface_describe(g, seq->out_mem, + (struct flcn_mem_desc_v0 *) + pv->pmu_allocation_get_fb_addr(pmu, + out)); + } + } else { + BUG_ON(in == NULL); + seq->out_mem = seq->in_mem; + pv->pmu_allocation_set_dmem_offset(pmu, out, + pv->pmu_allocation_get_dmem_offset(pmu, in)); + } + pv->pmu_allocation_set_dmem_size(pmu, + pv->get_pmu_seq_out_a_ptr(seq), + pv->pmu_allocation_get_dmem_size(pmu, out)); + pv->pmu_allocation_set_dmem_offset(pmu, + pv->get_pmu_seq_out_a_ptr(seq), + pv->pmu_allocation_get_dmem_offset(pmu, out)); + + } + + + + seq->state = PMU_SEQ_STATE_USED; + + err = pmu_write_cmd(pmu, cmd, queue_id, timeout); + if (err) + seq->state = PMU_SEQ_STATE_PENDING; + + gk20a_dbg_fn("done"); + + return 0; + +clean_up: + gk20a_dbg_fn("fail"); + if (in) + nvgpu_free(&pmu->dmem, + pv->pmu_allocation_get_dmem_offset(pmu, in)); + if (out) + nvgpu_free(&pmu->dmem, + pv->pmu_allocation_get_dmem_offset(pmu, out)); + + pmu_seq_release(pmu, seq); + return err; +} + +static int pmu_response_handle(struct nvgpu_pmu *pmu, + struct pmu_msg *msg) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + struct pmu_sequence *seq; + struct pmu_v *pv = &g->ops.pmu_ver; + int ret = 0; + + gk20a_dbg_fn(""); + + seq = &pmu->seq[msg->hdr.seq_id]; + if (seq->state != PMU_SEQ_STATE_USED && + seq->state != PMU_SEQ_STATE_CANCELLED) { + nvgpu_err(g, "msg for an unknown sequence %d", seq->id); + return -EINVAL; + } + + if (msg->hdr.unit_id == PMU_UNIT_RC && + msg->msg.rc.msg_type == PMU_RC_MSG_TYPE_UNHANDLED_CMD) { + nvgpu_err(g, "unhandled cmd: seq %d", seq->id); + } else if (seq->state != PMU_SEQ_STATE_CANCELLED) { + if (seq->msg) { + if (seq->msg->hdr.size >= msg->hdr.size) { + memcpy(seq->msg, msg, msg->hdr.size); + } else { + nvgpu_err(g, "sequence %d msg buffer too small", + seq->id); + } + } + if (pv->pmu_allocation_get_dmem_size(pmu, + pv->get_pmu_seq_out_a_ptr(seq)) != 0) { + pmu_copy_from_dmem(pmu, + pv->pmu_allocation_get_dmem_offset(pmu, + pv->get_pmu_seq_out_a_ptr(seq)), + seq->out_payload, + pv->pmu_allocation_get_dmem_size(pmu, + pv->get_pmu_seq_out_a_ptr(seq)), 0); + } + } else + seq->callback = NULL; + if (pv->pmu_allocation_get_dmem_size(pmu, + pv->get_pmu_seq_in_a_ptr(seq)) != 0) + nvgpu_free(&pmu->dmem, + pv->pmu_allocation_get_dmem_offset(pmu, + pv->get_pmu_seq_in_a_ptr(seq))); + if (pv->pmu_allocation_get_dmem_size(pmu, + pv->get_pmu_seq_out_a_ptr(seq)) != 0) + nvgpu_free(&pmu->dmem, + pv->pmu_allocation_get_dmem_offset(pmu, + pv->get_pmu_seq_out_a_ptr(seq))); + + if (seq->out_mem != NULL) { + memset(pv->pmu_allocation_get_fb_addr(pmu, + pv->get_pmu_seq_out_a_ptr(seq)), 0x0, + pv->pmu_allocation_get_fb_size(pmu, + pv->get_pmu_seq_out_a_ptr(seq))); + + gk20a_pmu_surface_free(g, seq->out_mem); + if (seq->out_mem != seq->in_mem) + nvgpu_kfree(g, seq->out_mem); + else + seq->out_mem = NULL; + } + + if (seq->in_mem != NULL) { + memset(pv->pmu_allocation_get_fb_addr(pmu, + pv->get_pmu_seq_in_a_ptr(seq)), 0x0, + pv->pmu_allocation_get_fb_size(pmu, + pv->get_pmu_seq_in_a_ptr(seq))); + + gk20a_pmu_surface_free(g, seq->in_mem); + nvgpu_kfree(g, seq->in_mem); + seq->in_mem = NULL; + } + + if (seq->callback) + seq->callback(g, msg, seq->cb_params, seq->desc, ret); + + pmu_seq_release(pmu, seq); + + /* TBD: notify client waiting for available dmem */ + + gk20a_dbg_fn("done"); + + return 0; +} + +static int pmu_handle_event(struct nvgpu_pmu *pmu, struct pmu_msg *msg) +{ + int err = 0; + struct gk20a *g = gk20a_from_pmu(pmu); + + gk20a_dbg_fn(""); + switch (msg->hdr.unit_id) { + case PMU_UNIT_PERFMON: + case PMU_UNIT_PERFMON_T18X: + err = nvgpu_pmu_handle_perfmon_event(pmu, &msg->msg.perfmon); + break; + case PMU_UNIT_PERF: + if (g->ops.perf.handle_pmu_perf_event != NULL) { + err = g->ops.perf.handle_pmu_perf_event(g, + (void *)&msg->msg.perf); + } else { + WARN_ON(1); + } + break; + case PMU_UNIT_THERM: + err = nvgpu_pmu_handle_therm_event(pmu, &msg->msg.therm); + break; + default: + break; + } + + return err; +} + +static bool pmu_read_message(struct nvgpu_pmu *pmu, struct pmu_queue *queue, + struct pmu_msg *msg, int *status) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + u32 read_size, bytes_read; + int err; + + *status = 0; + + if (nvgpu_pmu_queue_is_empty(pmu, queue)) + return false; + + err = pmu_queue_open_read(pmu, queue); + if (err) { + nvgpu_err(g, "fail to open queue %d for read", queue->id); + *status = err; + return false; + } + + err = pmu_queue_pop(pmu, queue, &msg->hdr, + PMU_MSG_HDR_SIZE, &bytes_read); + if (err || bytes_read != PMU_MSG_HDR_SIZE) { + nvgpu_err(g, "fail to read msg from queue %d", queue->id); + *status = err | -EINVAL; + goto clean_up; + } + + if (msg->hdr.unit_id == PMU_UNIT_REWIND) { + pmu_queue_rewind(pmu, queue); + /* read again after rewind */ + err = pmu_queue_pop(pmu, queue, &msg->hdr, + PMU_MSG_HDR_SIZE, &bytes_read); + if (err || bytes_read != PMU_MSG_HDR_SIZE) { + nvgpu_err(g, + "fail to read msg from queue %d", queue->id); + *status = err | -EINVAL; + goto clean_up; + } + } + + if (!PMU_UNIT_ID_IS_VALID(msg->hdr.unit_id)) { + nvgpu_err(g, "read invalid unit_id %d from queue %d", + msg->hdr.unit_id, queue->id); + *status = -EINVAL; + goto clean_up; + } + + if (msg->hdr.size > PMU_MSG_HDR_SIZE) { + read_size = msg->hdr.size - PMU_MSG_HDR_SIZE; + err = pmu_queue_pop(pmu, queue, &msg->msg, + read_size, &bytes_read); + if (err || bytes_read != read_size) { + nvgpu_err(g, + "fail to read msg from queue %d", queue->id); + *status = err; + goto clean_up; + } + } + + err = pmu_queue_close(pmu, queue, true); + if (err) { + nvgpu_err(g, "fail to close queue %d", queue->id); + *status = err; + return false; + } + + return true; + +clean_up: + err = pmu_queue_close(pmu, queue, false); + if (err) + nvgpu_err(g, "fail to close queue %d", queue->id); + return false; +} + +int nvgpu_pmu_process_message(struct nvgpu_pmu *pmu) +{ + struct pmu_msg msg; + int status; + struct gk20a *g = gk20a_from_pmu(pmu); + + if (unlikely(!pmu->pmu_ready)) { + nvgpu_pmu_process_init_msg(pmu, &msg); + if (g->ops.pmu.init_wpr_region != NULL) + g->ops.pmu.init_wpr_region(g); + nvgpu_pmu_init_perfmon(pmu); + + return 0; + } + + while (pmu_read_message(pmu, + &pmu->queue[PMU_MESSAGE_QUEUE], &msg, &status)) { + + nvgpu_pmu_dbg(g, "read msg hdr: "); + nvgpu_pmu_dbg(g, "unit_id = 0x%08x, size = 0x%08x", + msg.hdr.unit_id, msg.hdr.size); + nvgpu_pmu_dbg(g, "ctrl_flags = 0x%08x, seq_id = 0x%08x", + msg.hdr.ctrl_flags, msg.hdr.seq_id); + + msg.hdr.ctrl_flags &= ~PMU_CMD_FLAGS_PMU_MASK; + + if (msg.hdr.ctrl_flags == PMU_CMD_FLAGS_EVENT) + pmu_handle_event(pmu, &msg); + else + pmu_response_handle(pmu, &msg); + } + + return 0; +} + +int pmu_wait_message_cond(struct nvgpu_pmu *pmu, u32 timeout_ms, + u32 *var, u32 val) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + struct nvgpu_timeout timeout; + unsigned long delay = GR_IDLE_CHECK_DEFAULT; + + nvgpu_timeout_init(g, &timeout, (int)timeout_ms, NVGPU_TIMER_CPU_TIMER); + + do { + if (*var == val) + return 0; + + if (gk20a_pmu_is_interrupted(pmu)) + gk20a_pmu_isr(g); + + nvgpu_usleep_range(delay, delay * 2); + delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); + } while (!nvgpu_timeout_expired(&timeout)); + + return -ETIMEDOUT; +} + diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c index 4b0667c5..00b26cf4 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c @@ -2629,12 +2629,12 @@ int gk20a_fifo_preempt_channel(struct gk20a *g, u32 hw_chid) for (i = 0; i < g->fifo.max_runlists; i++) nvgpu_mutex_acquire(&f->runlist_info[i].mutex); - mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); ret = __locked_fifo_preempt(g, hw_chid, false); if (!mutex_ret) - pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); for (i = 0; i < g->fifo.max_runlists; i++) nvgpu_mutex_release(&f->runlist_info[i].mutex); @@ -2656,12 +2656,12 @@ int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid) for (i = 0; i < g->fifo.max_runlists; i++) nvgpu_mutex_acquire(&f->runlist_info[i].mutex); - mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); ret = __locked_fifo_preempt(g, tsgid, true); if (!mutex_ret) - pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); for (i = 0; i < g->fifo.max_runlists; i++) nvgpu_mutex_release(&f->runlist_info[i].mutex); @@ -2718,12 +2718,12 @@ void gk20a_fifo_set_runlist_state(struct gk20a *g, u32 runlists_mask, } } - mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); gk20a_fifo_sched_disable_rw(g, runlists_mask, runlist_state); if (!mutex_ret) - pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); if (!is_runlist_info_mutex_locked) { gk20a_dbg_info("release runlist_info mutex"); @@ -2792,7 +2792,7 @@ int gk20a_fifo_disable_engine_activity(struct gk20a *g, fifo_engine_status_engine_busy_v() && !wait_for_idle) return -EBUSY; - mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); gk20a_fifo_set_runlist_state(g, fifo_sched_disable_runlist_m( eng_info->runlist_id), RUNLIST_DISABLED, @@ -2832,7 +2832,7 @@ int gk20a_fifo_disable_engine_activity(struct gk20a *g, clean_up: if (!mutex_ret) - pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); if (err) { gk20a_dbg_fn("failed"); @@ -3300,13 +3300,13 @@ int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 hw_chid, nvgpu_mutex_acquire(&runlist->mutex); - mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); ret = gk20a_fifo_update_runlist_locked(g, runlist_id, hw_chid, add, wait_for_finish); if (!mutex_ret) - pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); nvgpu_mutex_release(&runlist->mutex); return ret; diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index b4884af1..269f0d68 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h @@ -737,6 +737,14 @@ struct gpu_ops { u32 (*pmu_get_queue_head_size)(void); u32 (*pmu_get_queue_tail_size)(void); u32 (*pmu_get_queue_tail)(u32 i); + int (*pmu_queue_head)(struct nvgpu_pmu *pmu, + struct pmu_queue *queue, u32 *head, bool set); + int (*pmu_queue_tail)(struct nvgpu_pmu *pmu, + struct pmu_queue *queue, u32 *tail, bool set); + int (*pmu_mutex_acquire)(struct nvgpu_pmu *pmu, + u32 id, u32 *token); + int (*pmu_mutex_release)(struct nvgpu_pmu *pmu, + u32 id, u32 *token); int (*init_wpr_region)(struct gk20a *g); int (*load_lsfalcon_ucode)(struct gk20a *g, u32 falconidmask); void (*write_dmatrfbase)(struct gk20a *g, u32 addr); diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c index 4e416f67..deab46c8 100644 --- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c @@ -2598,171 +2598,7 @@ int pmu_bootstrap(struct nvgpu_pmu *pmu) return 0; } -void pmu_seq_init(struct nvgpu_pmu *pmu) -{ - u32 i; - - memset(pmu->seq, 0, - sizeof(struct pmu_sequence) * PMU_MAX_NUM_SEQUENCES); - memset(pmu->pmu_seq_tbl, 0, - sizeof(pmu->pmu_seq_tbl)); - - for (i = 0; i < PMU_MAX_NUM_SEQUENCES; i++) - pmu->seq[i].id = i; -} - -static int pmu_seq_acquire(struct nvgpu_pmu *pmu, - struct pmu_sequence **pseq) -{ - struct gk20a *g = gk20a_from_pmu(pmu); - struct pmu_sequence *seq; - u32 index; - - nvgpu_mutex_acquire(&pmu->pmu_seq_lock); - index = find_first_zero_bit(pmu->pmu_seq_tbl, - sizeof(pmu->pmu_seq_tbl)); - if (index >= sizeof(pmu->pmu_seq_tbl)) { - nvgpu_err(g, "no free sequence available"); - nvgpu_mutex_release(&pmu->pmu_seq_lock); - return -EAGAIN; - } - set_bit(index, pmu->pmu_seq_tbl); - nvgpu_mutex_release(&pmu->pmu_seq_lock); - - seq = &pmu->seq[index]; - seq->state = PMU_SEQ_STATE_PENDING; - - *pseq = seq; - return 0; -} - -static void pmu_seq_release(struct nvgpu_pmu *pmu, - struct pmu_sequence *seq) -{ - struct gk20a *g = gk20a_from_pmu(pmu); - seq->state = PMU_SEQ_STATE_FREE; - seq->desc = PMU_INVALID_SEQ_DESC; - seq->callback = NULL; - seq->cb_params = NULL; - seq->msg = NULL; - seq->out_payload = NULL; - g->ops.pmu_ver.pmu_allocation_set_dmem_size(pmu, - g->ops.pmu_ver.get_pmu_seq_in_a_ptr(seq), 0); - g->ops.pmu_ver.pmu_allocation_set_dmem_size(pmu, - g->ops.pmu_ver.get_pmu_seq_out_a_ptr(seq), 0); - - clear_bit(seq->id, pmu->pmu_seq_tbl); -} - -static int pmu_queue_init(struct nvgpu_pmu *pmu, - u32 id, union pmu_init_msg_pmu *init) -{ - struct gk20a *g = gk20a_from_pmu(pmu); - struct pmu_queue *queue = &pmu->queue[id]; - int err; - - err = nvgpu_mutex_init(&queue->mutex); - if (err) - return err; - - queue->id = id; - g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params(queue, id, init); - queue->mutex_id = id; - - gk20a_dbg_pmu("queue %d: index %d, offset 0x%08x, size 0x%08x", - id, queue->index, queue->offset, queue->size); - - return 0; -} - -static int pmu_queue_head(struct nvgpu_pmu *pmu, struct pmu_queue *queue, - u32 *head, bool set) -{ - struct gk20a *g = gk20a_from_pmu(pmu); - u32 queue_head_size = 0; - - if (g->ops.pmu.pmu_get_queue_head_size) - queue_head_size = g->ops.pmu.pmu_get_queue_head_size(); - - BUG_ON(!head || !queue_head_size); - - if (PMU_IS_COMMAND_QUEUE(queue->id)) { - - if (queue->index >= queue_head_size) - return -EINVAL; - - if (!set) - *head = pwr_pmu_queue_head_address_v( - gk20a_readl(g, - g->ops.pmu.pmu_get_queue_head(queue->index))); - else - gk20a_writel(g, - g->ops.pmu.pmu_get_queue_head(queue->index), - pwr_pmu_queue_head_address_f(*head)); - } else { - if (!set) - *head = pwr_pmu_msgq_head_val_v( - gk20a_readl(g, pwr_pmu_msgq_head_r())); - else - gk20a_writel(g, - pwr_pmu_msgq_head_r(), - pwr_pmu_msgq_head_val_f(*head)); - } - - return 0; -} - -static int pmu_queue_tail(struct nvgpu_pmu *pmu, struct pmu_queue *queue, - u32 *tail, bool set) -{ - struct gk20a *g = gk20a_from_pmu(pmu); - u32 queue_tail_size = 0; - - if (g->ops.pmu.pmu_get_queue_tail_size) - queue_tail_size = g->ops.pmu.pmu_get_queue_tail_size(); - - BUG_ON(!tail || !queue_tail_size); - - if (PMU_IS_COMMAND_QUEUE(queue->id)) { - - if (queue->index >= queue_tail_size) - return -EINVAL; - - if (!set) - *tail = pwr_pmu_queue_tail_address_v( - gk20a_readl(g, - g->ops.pmu.pmu_get_queue_tail(queue->index))); - else - gk20a_writel(g, - g->ops.pmu.pmu_get_queue_tail(queue->index), - pwr_pmu_queue_tail_address_f(*tail)); - - } else { - if (!set) - *tail = pwr_pmu_msgq_tail_val_v( - gk20a_readl(g, pwr_pmu_msgq_tail_r())); - else - gk20a_writel(g, - pwr_pmu_msgq_tail_r(), - pwr_pmu_msgq_tail_val_f(*tail)); - } - - return 0; -} - -static inline void pmu_queue_read(struct nvgpu_pmu *pmu, - u32 offset, u8 *dst, u32 size) -{ - pmu_copy_from_dmem(pmu, offset, dst, size, 0); -} - -static inline void pmu_queue_write(struct nvgpu_pmu *pmu, - u32 offset, u8 *src, u32 size) -{ - pmu_copy_to_dmem(pmu, offset, src, size, 0); -} - -int pmu_mutex_acquire(struct nvgpu_pmu *pmu, u32 id, u32 *token) +int gk20a_pmu_mutex_acquire(struct nvgpu_pmu *pmu, u32 id, u32 *token) { struct gk20a *g = gk20a_from_pmu(pmu); struct pmu_mutex *mutex; @@ -2831,7 +2667,7 @@ int pmu_mutex_acquire(struct nvgpu_pmu *pmu, u32 id, u32 *token) return -EBUSY; } -int pmu_mutex_release(struct nvgpu_pmu *pmu, u32 id, u32 *token) +int gk20a_pmu_mutex_release(struct nvgpu_pmu *pmu, u32 id, u32 *token) { struct gk20a *g = gk20a_from_pmu(pmu); struct pmu_mutex *mutex; @@ -2872,234 +2708,78 @@ int pmu_mutex_release(struct nvgpu_pmu *pmu, u32 id, u32 *token) return 0; } -static int pmu_queue_lock(struct nvgpu_pmu *pmu, - struct pmu_queue *queue) -{ - int err; - - if (PMU_IS_MESSAGE_QUEUE(queue->id)) - return 0; - - if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) { - nvgpu_mutex_acquire(&queue->mutex); - return 0; - } - - err = pmu_mutex_acquire(pmu, queue->mutex_id, &queue->mutex_lock); - return err; -} - -static int pmu_queue_unlock(struct nvgpu_pmu *pmu, - struct pmu_queue *queue) -{ - int err; - - if (PMU_IS_MESSAGE_QUEUE(queue->id)) - return 0; - - if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) { - nvgpu_mutex_release(&queue->mutex); - return 0; - } - - err = pmu_mutex_release(pmu, queue->mutex_id, &queue->mutex_lock); - return err; -} - -/* called by pmu_read_message, no lock */ -static bool pmu_queue_is_empty(struct nvgpu_pmu *pmu, - struct pmu_queue *queue) -{ - u32 head, tail; - - pmu_queue_head(pmu, queue, &head, QUEUE_GET); - if (queue->opened && queue->oflag == OFLAG_READ) - tail = queue->position; - else - pmu_queue_tail(pmu, queue, &tail, QUEUE_GET); - - return head == tail; -} - -static bool pmu_queue_has_room(struct nvgpu_pmu *pmu, - struct pmu_queue *queue, u32 size, bool *need_rewind) -{ - u32 head, tail; - bool rewind = false; - unsigned int free; - - size = ALIGN(size, QUEUE_ALIGNMENT); - - pmu_queue_head(pmu, queue, &head, QUEUE_GET); - pmu_queue_tail(pmu, queue, &tail, QUEUE_GET); - if (head >= tail) { - free = queue->offset + queue->size - head; - free -= PMU_CMD_HDR_SIZE; - - if (size > free) { - rewind = true; - head = queue->offset; - } - } - - if (head < tail) - free = tail - head - 1; - - if (need_rewind) - *need_rewind = rewind; - - return size <= free; -} - -static int pmu_queue_push(struct nvgpu_pmu *pmu, - struct pmu_queue *queue, void *data, u32 size) -{ - - gk20a_dbg_fn(""); - - if (!queue->opened && queue->oflag == OFLAG_WRITE){ - nvgpu_err(gk20a_from_pmu(pmu), "queue not opened for write"); - return -EINVAL; - } - - pmu_queue_write(pmu, queue->position, data, size); - queue->position += ALIGN(size, QUEUE_ALIGNMENT); - return 0; -} - -static int pmu_queue_pop(struct nvgpu_pmu *pmu, - struct pmu_queue *queue, void *data, u32 size, - u32 *bytes_read) +int gk20a_pmu_queue_head(struct nvgpu_pmu *pmu, struct pmu_queue *queue, + u32 *head, bool set) { - u32 head, tail, used; - - *bytes_read = 0; - - if (!queue->opened && queue->oflag == OFLAG_READ){ - nvgpu_err(gk20a_from_pmu(pmu), "queue not opened for read"); - return -EINVAL; - } - - pmu_queue_head(pmu, queue, &head, QUEUE_GET); - tail = queue->position; - - if (head == tail) - return 0; - - if (head > tail) - used = head - tail; - else - used = queue->offset + queue->size - tail; - - if (size > used) { - nvgpu_warn(gk20a_from_pmu(pmu), - "queue size smaller than request read"); - size = used; - } + struct gk20a *g = gk20a_from_pmu(pmu); + u32 queue_head_size = 0; - pmu_queue_read(pmu, tail, data, size); - queue->position += ALIGN(size, QUEUE_ALIGNMENT); - *bytes_read = size; - return 0; -} + if (g->ops.pmu.pmu_get_queue_head_size) + queue_head_size = g->ops.pmu.pmu_get_queue_head_size(); -static void pmu_queue_rewind(struct nvgpu_pmu *pmu, - struct pmu_queue *queue) -{ - struct pmu_cmd cmd; + BUG_ON(!head || !queue_head_size); - gk20a_dbg_fn(""); + if (PMU_IS_COMMAND_QUEUE(queue->id)) { - if (!queue->opened) { - nvgpu_err(gk20a_from_pmu(pmu), "queue not opened"); - return; - } + if (queue->index >= queue_head_size) + return -EINVAL; - if (queue->oflag == OFLAG_WRITE) { - cmd.hdr.unit_id = PMU_UNIT_REWIND; - cmd.hdr.size = PMU_CMD_HDR_SIZE; - pmu_queue_push(pmu, queue, &cmd, cmd.hdr.size); - gk20a_dbg_pmu("queue %d rewinded", queue->id); + if (!set) + *head = pwr_pmu_queue_head_address_v( + gk20a_readl(g, + g->ops.pmu.pmu_get_queue_head(queue->index))); + else + gk20a_writel(g, + g->ops.pmu.pmu_get_queue_head(queue->index), + pwr_pmu_queue_head_address_f(*head)); + } else { + if (!set) + *head = pwr_pmu_msgq_head_val_v( + gk20a_readl(g, pwr_pmu_msgq_head_r())); + else + gk20a_writel(g, + pwr_pmu_msgq_head_r(), + pwr_pmu_msgq_head_val_f(*head)); } - queue->position = queue->offset; - return; -} - -/* open for read and lock the queue */ -static int pmu_queue_open_read(struct nvgpu_pmu *pmu, - struct pmu_queue *queue) -{ - int err; - - err = pmu_queue_lock(pmu, queue); - if (err) - return err; - - if (queue->opened) - BUG(); - - pmu_queue_tail(pmu, queue, &queue->position, QUEUE_GET); - queue->oflag = OFLAG_READ; - queue->opened = true; - return 0; } -/* open for write and lock the queue - make sure there's enough free space for the write */ -static int pmu_queue_open_write(struct nvgpu_pmu *pmu, - struct pmu_queue *queue, u32 size) +int gk20a_pmu_queue_tail(struct nvgpu_pmu *pmu, struct pmu_queue *queue, + u32 *tail, bool set) { - bool rewind = false; - int err; - - err = pmu_queue_lock(pmu, queue); - if (err) - return err; - - if (queue->opened) - BUG(); + struct gk20a *g = gk20a_from_pmu(pmu); + u32 queue_tail_size = 0; - if (!pmu_queue_has_room(pmu, queue, size, &rewind)) { - gk20a_dbg_pmu("queue full: queue-id %d: index %d", - queue->id, queue->index); - pmu_queue_unlock(pmu, queue); - return -EAGAIN; - } + if (g->ops.pmu.pmu_get_queue_tail_size) + queue_tail_size = g->ops.pmu.pmu_get_queue_tail_size(); - pmu_queue_head(pmu, queue, &queue->position, QUEUE_GET); - queue->oflag = OFLAG_WRITE; - queue->opened = true; + BUG_ON(!tail || !queue_tail_size); - if (rewind) - pmu_queue_rewind(pmu, queue); + if (PMU_IS_COMMAND_QUEUE(queue->id)) { - return 0; -} + if (queue->index >= queue_tail_size) + return -EINVAL; -/* close and unlock the queue */ -static int pmu_queue_close(struct nvgpu_pmu *pmu, - struct pmu_queue *queue, bool commit) -{ - if (!queue->opened) - return 0; + if (!set) + *tail = pwr_pmu_queue_tail_address_v( + gk20a_readl(g, + g->ops.pmu.pmu_get_queue_tail(queue->index))); + else + gk20a_writel(g, + g->ops.pmu.pmu_get_queue_tail(queue->index), + pwr_pmu_queue_tail_address_f(*tail)); - if (commit) { - if (queue->oflag == OFLAG_READ) { - pmu_queue_tail(pmu, queue, - &queue->position, QUEUE_SET); - } - else { - pmu_queue_head(pmu, queue, - &queue->position, QUEUE_SET); - } + } else { + if (!set) + *tail = pwr_pmu_msgq_tail_val_v( + gk20a_readl(g, pwr_pmu_msgq_tail_r())); + else + gk20a_writel(g, + pwr_pmu_msgq_tail_r(), + pwr_pmu_msgq_tail_val_f(*tail)); } - queue->opened = false; - - pmu_queue_unlock(pmu, queue); - return 0; } @@ -3193,7 +2873,7 @@ static int gk20a_init_pmu_setup_sw(struct gk20a *g) pmu->mutex[i].id = i; pmu->mutex[i].index = i; } - pmu_seq_init(pmu); + nvgpu_pmu_seq_init(pmu); gk20a_dbg_fn("skip init"); goto skip_init; @@ -3226,7 +2906,7 @@ static int gk20a_init_pmu_setup_sw(struct gk20a *g) goto err_free_mutex; } - pmu_seq_init(pmu); + nvgpu_pmu_seq_init(pmu); err = nvgpu_dma_alloc_map_sys(vm, GK20A_PMU_SEQ_BUF_SIZE, &pmu->seq_buf); @@ -3572,6 +3252,10 @@ void gk20a_init_pmu_ops(struct gpu_ops *gops) gops->pmu.pmu_get_queue_head_size = pwr_pmu_queue_head__size_1_v; gops->pmu.pmu_get_queue_tail = pwr_pmu_queue_tail_r; gops->pmu.pmu_get_queue_tail_size = pwr_pmu_queue_tail__size_1_v; + gops->pmu.pmu_queue_head = gk20a_pmu_queue_head; + gops->pmu.pmu_queue_tail = gk20a_pmu_queue_tail; + gops->pmu.pmu_mutex_acquire = gk20a_pmu_mutex_acquire; + gops->pmu.pmu_mutex_release = gk20a_pmu_mutex_release; gops->pmu.pmu_setup_elpg = NULL; gops->pmu.init_wpr_region = NULL; gops->pmu.load_lsfalcon_ucode = NULL; @@ -3829,7 +3513,7 @@ static u8 get_perfmon_id(struct nvgpu_pmu *pmu) return unit_id; } -static int pmu_init_perfmon(struct nvgpu_pmu *pmu) +int nvgpu_pmu_init_perfmon(struct nvgpu_pmu *pmu) { struct gk20a *g = gk20a_from_pmu(pmu); struct pmu_v *pv = &g->ops.pmu_ver; @@ -3929,7 +3613,7 @@ static int pmu_init_perfmon(struct nvgpu_pmu *pmu) return 0; } -static int pmu_process_init_msg(struct nvgpu_pmu *pmu, +int nvgpu_pmu_process_init_msg(struct nvgpu_pmu *pmu, struct pmu_msg *msg) { struct gk20a *g = gk20a_from_pmu(pmu); @@ -3983,7 +3667,7 @@ static int pmu_process_init_msg(struct nvgpu_pmu *pmu, } for (i = 0; i < PMU_QUEUE_COUNT; i++) - pmu_queue_init(pmu, i, init); + nvgpu_pmu_queue_init(pmu, i, init); if (!nvgpu_alloc_initialized(&pmu->dmem)) { /* Align start and end addresses */ @@ -4007,169 +3691,6 @@ static int pmu_process_init_msg(struct nvgpu_pmu *pmu, return 0; } -static bool pmu_read_message(struct nvgpu_pmu *pmu, struct pmu_queue *queue, - struct pmu_msg *msg, int *status) -{ - struct gk20a *g = gk20a_from_pmu(pmu); - u32 read_size, bytes_read; - int err; - - *status = 0; - - if (pmu_queue_is_empty(pmu, queue)) - return false; - - err = pmu_queue_open_read(pmu, queue); - if (err) { - nvgpu_err(g, "fail to open queue %d for read", queue->id); - *status = err; - return false; - } - - err = pmu_queue_pop(pmu, queue, &msg->hdr, - PMU_MSG_HDR_SIZE, &bytes_read); - if (err || bytes_read != PMU_MSG_HDR_SIZE) { - nvgpu_err(g, "fail to read msg from queue %d", queue->id); - *status = err | -EINVAL; - goto clean_up; - } - - if (msg->hdr.unit_id == PMU_UNIT_REWIND) { - pmu_queue_rewind(pmu, queue); - /* read again after rewind */ - err = pmu_queue_pop(pmu, queue, &msg->hdr, - PMU_MSG_HDR_SIZE, &bytes_read); - if (err || bytes_read != PMU_MSG_HDR_SIZE) { - nvgpu_err(g, - "fail to read msg from queue %d", queue->id); - *status = err | -EINVAL; - goto clean_up; - } - } - - if (!PMU_UNIT_ID_IS_VALID(msg->hdr.unit_id)) { - nvgpu_err(g, "read invalid unit_id %d from queue %d", - msg->hdr.unit_id, queue->id); - *status = -EINVAL; - goto clean_up; - } - - if (msg->hdr.size > PMU_MSG_HDR_SIZE) { - read_size = msg->hdr.size - PMU_MSG_HDR_SIZE; - err = pmu_queue_pop(pmu, queue, &msg->msg, - read_size, &bytes_read); - if (err || bytes_read != read_size) { - nvgpu_err(g, - "fail to read msg from queue %d", queue->id); - *status = err; - goto clean_up; - } - } - - err = pmu_queue_close(pmu, queue, true); - if (err) { - nvgpu_err(g, "fail to close queue %d", queue->id); - *status = err; - return false; - } - - return true; - -clean_up: - err = pmu_queue_close(pmu, queue, false); - if (err) - nvgpu_err(g, "fail to close queue %d", queue->id); - return false; -} - -static int pmu_response_handle(struct nvgpu_pmu *pmu, - struct pmu_msg *msg) -{ - struct gk20a *g = gk20a_from_pmu(pmu); - struct pmu_sequence *seq; - struct pmu_v *pv = &g->ops.pmu_ver; - int ret = 0; - - gk20a_dbg_fn(""); - - seq = &pmu->seq[msg->hdr.seq_id]; - if (seq->state != PMU_SEQ_STATE_USED && - seq->state != PMU_SEQ_STATE_CANCELLED) { - nvgpu_err(g, "msg for an unknown sequence %d", seq->id); - return -EINVAL; - } - - if (msg->hdr.unit_id == PMU_UNIT_RC && - msg->msg.rc.msg_type == PMU_RC_MSG_TYPE_UNHANDLED_CMD) { - nvgpu_err(g, "unhandled cmd: seq %d", seq->id); - } - else if (seq->state != PMU_SEQ_STATE_CANCELLED) { - if (seq->msg) { - if (seq->msg->hdr.size >= msg->hdr.size) { - memcpy(seq->msg, msg, msg->hdr.size); - } else { - nvgpu_err(g, "sequence %d msg buffer too small", - seq->id); - } - } - if (pv->pmu_allocation_get_dmem_size(pmu, - pv->get_pmu_seq_out_a_ptr(seq)) != 0) { - pmu_copy_from_dmem(pmu, - pv->pmu_allocation_get_dmem_offset(pmu, - pv->get_pmu_seq_out_a_ptr(seq)), - seq->out_payload, - pv->pmu_allocation_get_dmem_size(pmu, - pv->get_pmu_seq_out_a_ptr(seq)), 0); - } - } else - seq->callback = NULL; - if (pv->pmu_allocation_get_dmem_size(pmu, - pv->get_pmu_seq_in_a_ptr(seq)) != 0) - nvgpu_free(&pmu->dmem, - pv->pmu_allocation_get_dmem_offset(pmu, - pv->get_pmu_seq_in_a_ptr(seq))); - if (pv->pmu_allocation_get_dmem_size(pmu, - pv->get_pmu_seq_out_a_ptr(seq)) != 0) - nvgpu_free(&pmu->dmem, - pv->pmu_allocation_get_dmem_offset(pmu, - pv->get_pmu_seq_out_a_ptr(seq))); - - if (seq->out_mem != NULL) { - memset(pv->pmu_allocation_get_fb_addr(pmu, - pv->get_pmu_seq_out_a_ptr(seq)), 0x0, - pv->pmu_allocation_get_fb_size(pmu, - pv->get_pmu_seq_out_a_ptr(seq))); - - gk20a_pmu_surface_free(g, seq->out_mem); - if (seq->out_mem != seq->in_mem) - nvgpu_kfree(g, seq->out_mem); - else - seq->out_mem = NULL; - } - - if (seq->in_mem != NULL) { - memset(pv->pmu_allocation_get_fb_addr(pmu, - pv->get_pmu_seq_in_a_ptr(seq)), 0x0, - pv->pmu_allocation_get_fb_size(pmu, - pv->get_pmu_seq_in_a_ptr(seq))); - - gk20a_pmu_surface_free(g, seq->in_mem); - nvgpu_kfree(g, seq->in_mem); - seq->in_mem = NULL; - } - - if (seq->callback) - seq->callback(g, msg, seq->cb_params, seq->desc, ret); - - pmu_seq_release(pmu, seq); - - /* TBD: notify client waiting for available dmem */ - - gk20a_dbg_fn("done"); - - return 0; -} - static void pmu_handle_zbc_msg(struct gk20a *g, struct pmu_msg *msg, void *param, u32 handle, u32 status) { @@ -4266,7 +3787,7 @@ int nvgpu_pmu_perfmon_stop_sampling(struct nvgpu_pmu *pmu) return 0; } -static int pmu_handle_perfmon_event(struct nvgpu_pmu *pmu, +int nvgpu_pmu_handle_perfmon_event(struct nvgpu_pmu *pmu, struct pmu_perfmon_msg *msg) { gk20a_dbg_fn(""); @@ -4298,8 +3819,7 @@ static int pmu_handle_perfmon_event(struct nvgpu_pmu *pmu, return 0; } - -static int pmu_handle_therm_event(struct nvgpu_pmu *pmu, +int nvgpu_pmu_handle_therm_event(struct nvgpu_pmu *pmu, struct nv_pmu_therm_msg *msg) { gk20a_dbg_fn(""); @@ -4323,99 +3843,6 @@ static int pmu_handle_therm_event(struct nvgpu_pmu *pmu, return 0; } -static int pmu_handle_event(struct nvgpu_pmu *pmu, struct pmu_msg *msg) -{ - int err = 0; - struct gk20a *g = gk20a_from_pmu(pmu); - - gk20a_dbg_fn(""); - switch (msg->hdr.unit_id) { - case PMU_UNIT_PERFMON: - case PMU_UNIT_PERFMON_T18X: - err = pmu_handle_perfmon_event(pmu, &msg->msg.perfmon); - break; - case PMU_UNIT_PERF: - if (g->ops.perf.handle_pmu_perf_event != NULL) { - err = g->ops.perf.handle_pmu_perf_event(g, - (void *)&msg->msg.perf); - } else { - WARN_ON(1); - } - break; - case PMU_UNIT_THERM: - err = pmu_handle_therm_event(pmu, &msg->msg.therm); - break; - default: - break; - } - - return err; -} - -static int pmu_process_message(struct nvgpu_pmu *pmu) -{ - struct pmu_msg msg; - int status; - struct gk20a *g = gk20a_from_pmu(pmu); - - if (unlikely(!pmu->pmu_ready)) { - pmu_process_init_msg(pmu, &msg); - if (g->ops.pmu.init_wpr_region != NULL) - g->ops.pmu.init_wpr_region(g); - pmu_init_perfmon(pmu); - - return 0; - } - - while (pmu_read_message(pmu, - &pmu->queue[PMU_MESSAGE_QUEUE], &msg, &status)) { - - gk20a_dbg_pmu("read msg hdr: " - "unit_id = 0x%08x, size = 0x%08x, " - "ctrl_flags = 0x%08x, seq_id = 0x%08x", - msg.hdr.unit_id, msg.hdr.size, - msg.hdr.ctrl_flags, msg.hdr.seq_id); - - msg.hdr.ctrl_flags &= ~PMU_CMD_FLAGS_PMU_MASK; - - if (msg.hdr.ctrl_flags == PMU_CMD_FLAGS_EVENT) { - pmu_handle_event(pmu, &msg); - } else { - pmu_response_handle(pmu, &msg); - } - } - - return 0; -} - -int pmu_wait_message_cond(struct nvgpu_pmu *pmu, u32 timeout_ms, - u32 *var, u32 val) -{ - struct gk20a *g = gk20a_from_pmu(pmu); - struct nvgpu_timeout timeout; - unsigned long delay = GR_IDLE_CHECK_DEFAULT; - u32 servicedpmuint; - - servicedpmuint = pwr_falcon_irqstat_halt_true_f() | - pwr_falcon_irqstat_exterr_true_f() | - pwr_falcon_irqstat_swgen0_true_f(); - - nvgpu_timeout_init(g, &timeout, (int)timeout_ms, NVGPU_TIMER_CPU_TIMER); - - do { - if (*var == val) - return 0; - - if (gk20a_readl(g, pwr_falcon_irqstat_r()) & servicedpmuint) - gk20a_pmu_isr(g); - - nvgpu_usleep_range(delay, delay * 2); - delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); - } while (!nvgpu_timeout_expired(&timeout)); - - return -ETIMEDOUT; -} - static void pmu_dump_elpg_stats(struct nvgpu_pmu *pmu) { struct gk20a *g = gk20a_from_pmu(pmu); @@ -4613,6 +4040,21 @@ void pmu_dump_falcon_stats(struct nvgpu_pmu *pmu) printtrace(pmu); } +bool gk20a_pmu_is_interrupted(struct nvgpu_pmu *pmu) +{ + struct gk20a *g = gk20a_from_pmu(pmu); + u32 servicedpmuint; + + servicedpmuint = pwr_falcon_irqstat_halt_true_f() | + pwr_falcon_irqstat_exterr_true_f() | + pwr_falcon_irqstat_swgen0_true_f(); + + if (gk20a_readl(g, pwr_falcon_irqstat_r()) & servicedpmuint) + return true; + + return false; +} + void gk20a_pmu_isr(struct gk20a *g) { struct nvgpu_pmu *pmu = &g->pmu; @@ -4661,7 +4103,7 @@ void gk20a_pmu_isr(struct gk20a *g) ~pwr_falcon_exterrstat_valid_m()); } if (intr & pwr_falcon_irqstat_swgen0_true_f()) { - pmu_process_message(pmu); + nvgpu_pmu_process_message(pmu); recheck = true; } @@ -4669,7 +4111,7 @@ void gk20a_pmu_isr(struct gk20a *g) if (recheck) { queue = &pmu->queue[PMU_MESSAGE_QUEUE]; - if (!pmu_queue_is_empty(pmu, queue)) + if (!nvgpu_pmu_queue_is_empty(pmu, queue)) gk20a_writel(g, pwr_falcon_irqsset_r(), pwr_falcon_irqsset_swgen0_set_f()); } @@ -4677,114 +4119,6 @@ void gk20a_pmu_isr(struct gk20a *g) nvgpu_mutex_release(&pmu->isr_mutex); } -static bool pmu_validate_cmd(struct nvgpu_pmu *pmu, struct pmu_cmd *cmd, - struct pmu_msg *msg, struct pmu_payload *payload, - u32 queue_id) -{ - struct gk20a *g = gk20a_from_pmu(pmu); - struct pmu_queue *queue; - u32 in_size, out_size; - - if (!PMU_IS_SW_COMMAND_QUEUE(queue_id)) - goto invalid_cmd; - - queue = &pmu->queue[queue_id]; - if (cmd->hdr.size < PMU_CMD_HDR_SIZE) - goto invalid_cmd; - - if (cmd->hdr.size > (queue->size >> 1)) - goto invalid_cmd; - - if (msg != NULL && msg->hdr.size < PMU_MSG_HDR_SIZE) - goto invalid_cmd; - - if (!PMU_UNIT_ID_IS_VALID(cmd->hdr.unit_id)) - goto invalid_cmd; - - if (payload == NULL) - return true; - - if (payload->in.buf == NULL && payload->out.buf == NULL) - goto invalid_cmd; - - if ((payload->in.buf != NULL && payload->in.size == 0) || - (payload->out.buf != NULL && payload->out.size == 0)) - goto invalid_cmd; - - in_size = PMU_CMD_HDR_SIZE; - if (payload->in.buf) { - in_size += payload->in.offset; - in_size += g->ops.pmu_ver.get_pmu_allocation_struct_size(pmu); - } - - out_size = PMU_CMD_HDR_SIZE; - if (payload->out.buf) { - out_size += payload->out.offset; - out_size += g->ops.pmu_ver.get_pmu_allocation_struct_size(pmu); - } - - if (in_size > cmd->hdr.size || out_size > cmd->hdr.size) - goto invalid_cmd; - - - if ((payload->in.offset != 0 && payload->in.buf == NULL) || - (payload->out.offset != 0 && payload->out.buf == NULL)) - goto invalid_cmd; - - return true; - -invalid_cmd: - nvgpu_err(g, "invalid pmu cmd :" - "queue_id=%d,\n" - "cmd_size=%d, cmd_unit_id=%d, msg=%p, msg_size=%d,\n" - "payload in=%p, in_size=%d, in_offset=%d,\n" - "payload out=%p, out_size=%d, out_offset=%d", - queue_id, cmd->hdr.size, cmd->hdr.unit_id, - msg, msg?msg->hdr.unit_id:~0, - &payload->in, payload->in.size, payload->in.offset, - &payload->out, payload->out.size, payload->out.offset); - - return false; -} - -static int pmu_write_cmd(struct nvgpu_pmu *pmu, struct pmu_cmd *cmd, - u32 queue_id, unsigned long timeout_ms) -{ - struct gk20a *g = gk20a_from_pmu(pmu); - struct pmu_queue *queue; - struct nvgpu_timeout timeout; - int err; - - gk20a_dbg_fn(""); - - queue = &pmu->queue[queue_id]; - nvgpu_timeout_init(g, &timeout, (int)timeout_ms, NVGPU_TIMER_CPU_TIMER); - - do { - err = pmu_queue_open_write(pmu, queue, cmd->hdr.size); - if (err == -EAGAIN && !nvgpu_timeout_expired(&timeout)) - nvgpu_usleep_range(1000, 2000); - else - break; - } while (1); - - if (err) - goto clean_up; - - pmu_queue_push(pmu, queue, cmd, cmd->hdr.size); - - - err = pmu_queue_close(pmu, queue, true); - -clean_up: - if (err) - nvgpu_err(g, "fail to write cmd to queue %d", queue_id); - else - gk20a_dbg_fn("done"); - - return err; -} - void gk20a_pmu_surface_describe(struct gk20a *g, struct nvgpu_mem *mem, struct flcn_mem_desc_v0 *fb) { @@ -4832,170 +4166,6 @@ void gk20a_pmu_surface_free(struct gk20a *g, struct nvgpu_mem *mem) memset(mem, 0, sizeof(struct nvgpu_mem)); } -int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd, - struct pmu_msg *msg, struct pmu_payload *payload, - u32 queue_id, pmu_callback callback, void* cb_param, - u32 *seq_desc, unsigned long timeout) -{ - struct nvgpu_pmu *pmu = &g->pmu; - struct pmu_v *pv = &g->ops.pmu_ver; - struct pmu_sequence *seq; - void *in = NULL, *out = NULL; - int err; - - gk20a_dbg_fn(""); - - if ((!cmd) || (!seq_desc) || (!pmu->pmu_ready)) { - if (!cmd) - nvgpu_warn(g, "%s(): PMU cmd buffer is NULL", __func__); - else if (!seq_desc) - nvgpu_warn(g, "%s(): Seq descriptor is NULL", __func__); - else - nvgpu_warn(g, "%s(): PMU is not ready", __func__); - - WARN_ON(1); - return -EINVAL; - } - - if (!pmu_validate_cmd(pmu, cmd, msg, payload, queue_id)) - return -EINVAL; - - err = pmu_seq_acquire(pmu, &seq); - if (err) - return err; - - cmd->hdr.seq_id = seq->id; - - cmd->hdr.ctrl_flags = 0; - cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_STATUS; - cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_INTR; - - seq->callback = callback; - seq->cb_params = cb_param; - seq->msg = msg; - seq->out_payload = NULL; - seq->desc = pmu->next_seq_desc++; - - if (payload) - seq->out_payload = payload->out.buf; - - *seq_desc = seq->desc; - - if (payload && payload->in.offset != 0) { - pv->set_pmu_allocation_ptr(pmu, &in, - ((u8 *)&cmd->cmd + payload->in.offset)); - - if (payload->in.buf != payload->out.buf) - pv->pmu_allocation_set_dmem_size(pmu, in, - (u16)payload->in.size); - else - pv->pmu_allocation_set_dmem_size(pmu, in, - (u16)max(payload->in.size, payload->out.size)); - - *(pv->pmu_allocation_get_dmem_offset_addr(pmu, in)) = - nvgpu_alloc(&pmu->dmem, - pv->pmu_allocation_get_dmem_size(pmu, in)); - if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu, in))) - goto clean_up; - - if (payload->in.fb_size != 0x0) { - seq->in_mem = nvgpu_kzalloc(g, - sizeof(struct nvgpu_mem)); - if (!seq->in_mem) { - err = -ENOMEM; - goto clean_up; - } - - gk20a_pmu_vidmem_surface_alloc(g, seq->in_mem, - payload->in.fb_size); - gk20a_pmu_surface_describe(g, seq->in_mem, - (struct flcn_mem_desc_v0 *) - pv->pmu_allocation_get_fb_addr(pmu, in)); - - nvgpu_mem_wr_n(g, seq->in_mem, 0, - payload->in.buf, payload->in.fb_size); - - } else { - pmu_copy_to_dmem(pmu, - (pv->pmu_allocation_get_dmem_offset(pmu, in)), - payload->in.buf, payload->in.size, 0); - } - pv->pmu_allocation_set_dmem_size(pmu, - pv->get_pmu_seq_in_a_ptr(seq), - pv->pmu_allocation_get_dmem_size(pmu, in)); - pv->pmu_allocation_set_dmem_offset(pmu, - pv->get_pmu_seq_in_a_ptr(seq), - pv->pmu_allocation_get_dmem_offset(pmu, in)); - } - - if (payload && payload->out.offset != 0) { - pv->set_pmu_allocation_ptr(pmu, &out, - ((u8 *)&cmd->cmd + payload->out.offset)); - pv->pmu_allocation_set_dmem_size(pmu, out, - (u16)payload->out.size); - - if (payload->in.buf != payload->out.buf) { - *(pv->pmu_allocation_get_dmem_offset_addr(pmu, out)) = - nvgpu_alloc(&pmu->dmem, - pv->pmu_allocation_get_dmem_size(pmu, out)); - if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu, - out))) - goto clean_up; - - if (payload->out.fb_size != 0x0) { - seq->out_mem = nvgpu_kzalloc(g, - sizeof(struct nvgpu_mem)); - if (!seq->out_mem) { - err = -ENOMEM; - goto clean_up; - } - gk20a_pmu_vidmem_surface_alloc(g, seq->out_mem, - payload->out.fb_size); - gk20a_pmu_surface_describe(g, seq->out_mem, - (struct flcn_mem_desc_v0 *) - pv->pmu_allocation_get_fb_addr(pmu, - out)); - } - } else { - BUG_ON(in == NULL); - seq->out_mem = seq->in_mem; - pv->pmu_allocation_set_dmem_offset(pmu, out, - pv->pmu_allocation_get_dmem_offset(pmu, in)); - } - pv->pmu_allocation_set_dmem_size(pmu, - pv->get_pmu_seq_out_a_ptr(seq), - pv->pmu_allocation_get_dmem_size(pmu, out)); - pv->pmu_allocation_set_dmem_offset(pmu, - pv->get_pmu_seq_out_a_ptr(seq), - pv->pmu_allocation_get_dmem_offset(pmu, out)); - - } - - - - seq->state = PMU_SEQ_STATE_USED; - - err = pmu_write_cmd(pmu, cmd, queue_id, timeout); - if (err) - seq->state = PMU_SEQ_STATE_PENDING; - - gk20a_dbg_fn("done"); - - return 0; - -clean_up: - gk20a_dbg_fn("fail"); - if (in) - nvgpu_free(&pmu->dmem, - pv->pmu_allocation_get_dmem_offset(pmu, in)); - if (out) - nvgpu_free(&pmu->dmem, - pv->pmu_allocation_get_dmem_offset(pmu, out)); - - pmu_seq_release(pmu, seq); - return err; -} - int gk20a_pmu_pg_global_enable(struct gk20a *g, u32 enable_pg) { u32 status = 0; diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h index 24bc5822..a53329b4 100644 --- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h @@ -57,15 +57,6 @@ struct nvgpu_firmware; #define PMU_PGENG_GR_BUFFER_IDX_ZBC (1) #define PMU_PGENG_GR_BUFFER_IDX_FECS (2) -struct pmu_payload { - struct { - void *buf; - u32 offset; - u32 size; - u32 fb_size; - } in, out; -}; - struct pmu_surface { struct nvgpu_mem vidmem_desc; struct nvgpu_mem sysmem_desc; @@ -119,14 +110,9 @@ struct pmu_pg_stats_data { int gk20a_init_pmu_support(struct gk20a *g); int gk20a_init_pmu_bind_fecs(struct gk20a *g); +bool gk20a_pmu_is_interrupted(struct nvgpu_pmu *pmu); void gk20a_pmu_isr(struct gk20a *g); -/* send a cmd to pmu */ -int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd, struct pmu_msg *msg, - struct pmu_payload *payload, u32 queue_id, - pmu_callback callback, void* cb_param, - u32 *seq_desc, unsigned long timeout); - int gk20a_pmu_enable_elpg(struct gk20a *g); int gk20a_pmu_disable_elpg(struct gk20a *g); int gk20a_pmu_pg_global_enable(struct gk20a *g, u32 enable_pg); @@ -138,8 +124,14 @@ void gk20a_pmu_save_zbc(struct gk20a *g, u32 entries); int gk20a_pmu_perfmon_enable(struct gk20a *g, bool enable); -int pmu_mutex_acquire(struct nvgpu_pmu *pmu, u32 id, u32 *token); -int pmu_mutex_release(struct nvgpu_pmu *pmu, u32 id, u32 *token); +int gk20a_pmu_mutex_acquire(struct nvgpu_pmu *pmu, u32 id, u32 *token); +int gk20a_pmu_mutex_release(struct nvgpu_pmu *pmu, u32 id, u32 *token); + +int gk20a_pmu_queue_head(struct nvgpu_pmu *pmu, struct pmu_queue *queue, + u32 *head, bool set); +int gk20a_pmu_queue_tail(struct nvgpu_pmu *pmu, struct pmu_queue *queue, + u32 *tail, bool set); + int gk20a_pmu_destroy(struct gk20a *g); int gk20a_pmu_load_norm(struct gk20a *g, u32 *load); int gk20a_pmu_load_update(struct gk20a *g); @@ -157,7 +149,6 @@ int pmu_bootstrap(struct nvgpu_pmu *pmu); int gk20a_init_pmu(struct nvgpu_pmu *pmu); void pmu_dump_falcon_stats(struct nvgpu_pmu *pmu); void gk20a_remove_pmu_support(struct nvgpu_pmu *pmu); -void pmu_seq_init(struct nvgpu_pmu *pmu); int gk20a_init_pmu(struct nvgpu_pmu *pmu); diff --git a/drivers/gpu/nvgpu/gm20b/pmu_gm20b.c b/drivers/gpu/nvgpu/gm20b/pmu_gm20b.c index 34c2c373..4fafc8f1 100644 --- a/drivers/gpu/nvgpu/gm20b/pmu_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/pmu_gm20b.c @@ -292,6 +292,10 @@ void gm20b_init_pmu_ops(struct gpu_ops *gops) gops->pmu.pmu_get_queue_head_size = pwr_pmu_queue_head__size_1_v; gops->pmu.pmu_get_queue_tail = pwr_pmu_queue_tail_r; gops->pmu.pmu_get_queue_tail_size = pwr_pmu_queue_tail__size_1_v; + gops->pmu.pmu_queue_head = gk20a_pmu_queue_head; + gops->pmu.pmu_queue_tail = gk20a_pmu_queue_tail; + gops->pmu.pmu_mutex_acquire = gk20a_pmu_mutex_acquire; + gops->pmu.pmu_mutex_release = gk20a_pmu_mutex_release; gops->pmu.lspmuwprinitdone = 0; gops->pmu.fecsbootstrapdone = false; gops->pmu.write_dmatrfbase = gm20b_write_dmatrfbase; diff --git a/drivers/gpu/nvgpu/gp106/pmu_gp106.c b/drivers/gpu/nvgpu/gp106/pmu_gp106.c index 89144c2a..a9b44019 100644 --- a/drivers/gpu/nvgpu/gp106/pmu_gp106.c +++ b/drivers/gpu/nvgpu/gp106/pmu_gp106.c @@ -422,6 +422,10 @@ void gp106_init_pmu_ops(struct gpu_ops *gops) gops->pmu.pmu_get_queue_head_size = pwr_pmu_queue_head__size_1_v; gops->pmu.pmu_get_queue_tail = pwr_pmu_queue_tail_r; gops->pmu.pmu_get_queue_tail_size = pwr_pmu_queue_tail__size_1_v; + gops->pmu.pmu_queue_head = gk20a_pmu_queue_head; + gops->pmu.pmu_queue_tail = gk20a_pmu_queue_tail; + gops->pmu.pmu_mutex_acquire = gk20a_pmu_mutex_acquire; + gops->pmu.pmu_mutex_release = gk20a_pmu_mutex_release; gops->pmu.lspmuwprinitdone = 0; gops->pmu.fecsbootstrapdone = false; gops->pmu.write_dmatrfbase = gp10b_write_dmatrfbase; diff --git a/drivers/gpu/nvgpu/gp10b/pmu_gp10b.c b/drivers/gpu/nvgpu/gp10b/pmu_gp10b.c index 1409d279..c89e910c 100644 --- a/drivers/gpu/nvgpu/gp10b/pmu_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/pmu_gp10b.c @@ -413,6 +413,10 @@ void gp10b_init_pmu_ops(struct gpu_ops *gops) gops->pmu.pmu_get_queue_head_size = pwr_pmu_queue_head__size_1_v; gops->pmu.pmu_get_queue_tail = pwr_pmu_queue_tail_r; gops->pmu.pmu_get_queue_tail_size = pwr_pmu_queue_tail__size_1_v; + gops->pmu.pmu_queue_head = gk20a_pmu_queue_head; + gops->pmu.pmu_queue_tail = gk20a_pmu_queue_tail; + gops->pmu.pmu_mutex_acquire = gk20a_pmu_mutex_acquire; + gops->pmu.pmu_mutex_release = gk20a_pmu_mutex_release; gops->pmu.lspmuwprinitdone = false; gops->pmu.fecsbootstrapdone = false; gops->pmu.write_dmatrfbase = gp10b_write_dmatrfbase; diff --git a/drivers/gpu/nvgpu/include/nvgpu/pmu.h b/drivers/gpu/nvgpu/include/nvgpu/pmu.h index 687e00ae..f447e910 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/pmu.h +++ b/drivers/gpu/nvgpu/include/nvgpu/pmu.h @@ -115,6 +115,15 @@ enum { typedef void (*pmu_callback)(struct gk20a *, struct pmu_msg *, void *, u32, u32); +struct pmu_payload { + struct { + void *buf; + u32 offset; + u32 size; + u32 fb_size; + } in, out; +}; + struct pmu_ucode_desc { u32 descriptor_size; u32 image_size; @@ -326,4 +335,33 @@ struct nvgpu_pmu { struct nvgpu_firmware *fw; }; +/* PMU IPC Methods */ +void nvgpu_pmu_seq_init(struct nvgpu_pmu *pmu); + +int nvgpu_pmu_mutex_acquire(struct nvgpu_pmu *pmu, u32 id, u32 *token); +int nvgpu_pmu_mutex_release(struct nvgpu_pmu *pmu, u32 id, u32 *token); + +int nvgpu_pmu_queue_init(struct nvgpu_pmu *pmu, u32 id, + union pmu_init_msg_pmu *init); +bool nvgpu_pmu_queue_is_empty(struct nvgpu_pmu *pmu, struct pmu_queue *queue); + +/* send a cmd to pmu */ +int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd, + struct pmu_msg *msg, struct pmu_payload *payload, + u32 queue_id, pmu_callback callback, void *cb_param, + u32 *seq_desc, unsigned long timeout); + +int nvgpu_pmu_process_message(struct nvgpu_pmu *pmu); + +/* perfmon */ +int nvgpu_pmu_init_perfmon(struct nvgpu_pmu *pmu); +int nvgpu_pmu_handle_perfmon_event(struct nvgpu_pmu *pmu, + struct pmu_perfmon_msg *msg); +int nvgpu_pmu_handle_therm_event(struct nvgpu_pmu *pmu, + struct nv_pmu_therm_msg *msg); + +/* PMU init */ +int nvgpu_pmu_process_init_msg(struct nvgpu_pmu *pmu, + struct pmu_msg *msg); + #endif /* __NVGPU_PMU_H__ */ -- cgit v1.2.2