gpu: nvgpu: reorganize PMU IPC

- Moved PMU IPC related code to
drivers/gpu/nvgpu/common/pmu/pmu_ipc.c file,
-Below is the list which are moved
  seq
  mutex
  queue
  cmd/msg post & process
  event handling

NVGPU-56

Change-Id: Ic380faa27de4e5574d5b22500125e86027fd4b5d
Signed-off-by: Mahantesh Kumbar <mkumbar@nvidia.com>
Reviewed-on: http://git-master/r/1478167
GVS: Gerrit_Virtual_Submit
Reviewed-by: svccoveritychecker <svccoveritychecker@nvidia.com>
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>

1 files changed, 893 insertions, 0 deletions

diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_ipc.c b/drivers/gpu/nvgpu/common/pmu/pmu_ipc.c
new file mode 100644
index 00000000..74966f2d
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/pmu/pmu_ipc.c
@@ -0,0 +1,893 @@
+/*
+ * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <nvgpu/pmu.h>
+#include <nvgpu/log.h>
+#include <nvgpu/pmuif/nvgpu_gpmu_cmdif.h>
+
+#include "gk20a/gk20a.h"
+
+void nvgpu_pmu_seq_init(struct nvgpu_pmu *pmu)
+{
+        u32 i;
+
+        memset(pmu->seq, 0,
+                sizeof(struct pmu_sequence) * PMU_MAX_NUM_SEQUENCES);
+        memset(pmu->pmu_seq_tbl, 0,
+                sizeof(pmu->pmu_seq_tbl));
+
+        for (i = 0; i < PMU_MAX_NUM_SEQUENCES; i++)
+                pmu->seq[i].id = i;
+}
+
+static int pmu_seq_acquire(struct nvgpu_pmu *pmu,
+                        struct pmu_sequence **pseq)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+        struct pmu_sequence *seq;
+        u32 index;
+
+        nvgpu_mutex_acquire(&pmu->pmu_seq_lock);
+        index = find_first_zero_bit(pmu->pmu_seq_tbl,
+                                sizeof(pmu->pmu_seq_tbl));
+        if (index >= sizeof(pmu->pmu_seq_tbl)) {
+                nvgpu_err(g, "no free sequence available");
+                nvgpu_mutex_release(&pmu->pmu_seq_lock);
+                return -EAGAIN;
+        }
+        set_bit(index, pmu->pmu_seq_tbl);
+        nvgpu_mutex_release(&pmu->pmu_seq_lock);
+
+        seq = &pmu->seq[index];
+        seq->state = PMU_SEQ_STATE_PENDING;
+
+        *pseq = seq;
+        return 0;
+}
+
+static void pmu_seq_release(struct nvgpu_pmu *pmu,
+                        struct pmu_sequence *seq)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+
+        seq->state      = PMU_SEQ_STATE_FREE;
+        seq->desc       = PMU_INVALID_SEQ_DESC;
+        seq->callback   = NULL;
+        seq->cb_params  = NULL;
+        seq->msg        = NULL;
+        seq->out_payload = NULL;
+        g->ops.pmu_ver.pmu_allocation_set_dmem_size(pmu,
+                g->ops.pmu_ver.get_pmu_seq_in_a_ptr(seq), 0);
+        g->ops.pmu_ver.pmu_allocation_set_dmem_size(pmu,
+                g->ops.pmu_ver.get_pmu_seq_out_a_ptr(seq), 0);
+
+        clear_bit(seq->id, pmu->pmu_seq_tbl);
+}
+/* mutex */
+int nvgpu_pmu_mutex_acquire(struct nvgpu_pmu *pmu, u32 id, u32 *token)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+
+        return g->ops.pmu.pmu_mutex_acquire(pmu, id, token);
+}
+
+int nvgpu_pmu_mutex_release(struct nvgpu_pmu *pmu, u32 id, u32 *token)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+
+        return g->ops.pmu.pmu_mutex_release(pmu, id, token);
+}
+
+/* queue */
+int nvgpu_pmu_queue_init(struct nvgpu_pmu *pmu,
+                u32 id, union pmu_init_msg_pmu *init)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+        struct pmu_queue *queue = &pmu->queue[id];
+        int err;
+
+        err = nvgpu_mutex_init(&queue->mutex);
+        if (err)
+                return err;
+
+        queue->id       = id;
+        g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params(queue, id, init);
+        queue->mutex_id = id;
+
+        nvgpu_pmu_dbg(g, "queue %d: index %d, offset 0x%08x, size 0x%08x",
+                id, queue->index, queue->offset, queue->size);
+
+        return 0;
+}
+
+static int pmu_queue_head(struct nvgpu_pmu *pmu, struct pmu_queue *queue,
+                        u32 *head, bool set)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+
+        return g->ops.pmu.pmu_queue_head(pmu, queue, head, set);
+}
+
+static int pmu_queue_tail(struct nvgpu_pmu *pmu, struct pmu_queue *queue,
+                        u32 *tail, bool set)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+
+        return g->ops.pmu.pmu_queue_tail(pmu, queue, tail, set);
+}
+
+static inline void pmu_queue_read(struct nvgpu_pmu *pmu,
+                        u32 offset, u8 *dst, u32 size)
+{
+        pmu_copy_from_dmem(pmu, offset, dst, size, 0);
+}
+
+static inline void pmu_queue_write(struct nvgpu_pmu *pmu,
+                        u32 offset, u8 *src, u32 size)
+{
+        pmu_copy_to_dmem(pmu, offset, src, size, 0);
+}
+
+
+static int pmu_queue_lock(struct nvgpu_pmu *pmu,
+                        struct pmu_queue *queue)
+{
+        int err;
+
+        if (PMU_IS_MESSAGE_QUEUE(queue->id))
+                return 0;
+
+        if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
+                nvgpu_mutex_acquire(&queue->mutex);
+                return 0;
+        }
+
+        err = nvgpu_pmu_mutex_acquire(pmu, queue->mutex_id, &queue->mutex_lock);
+        return err;
+}
+
+static int pmu_queue_unlock(struct nvgpu_pmu *pmu,
+                        struct pmu_queue *queue)
+{
+        int err;
+
+        if (PMU_IS_MESSAGE_QUEUE(queue->id))
+                return 0;
+
+        if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
+                nvgpu_mutex_release(&queue->mutex);
+                return 0;
+        }
+
+        err = nvgpu_pmu_mutex_release(pmu, queue->mutex_id, &queue->mutex_lock);
+        return err;
+}
+
+/* called by pmu_read_message, no lock */
+bool nvgpu_pmu_queue_is_empty(struct nvgpu_pmu *pmu,
+                        struct pmu_queue *queue)
+{
+        u32 head, tail;
+
+        pmu_queue_head(pmu, queue, &head, QUEUE_GET);
+        if (queue->opened && queue->oflag == OFLAG_READ)
+                tail = queue->position;
+        else
+                pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
+
+        return head == tail;
+}
+
+static bool pmu_queue_has_room(struct nvgpu_pmu *pmu,
+                        struct pmu_queue *queue, u32 size, bool *need_rewind)
+{
+        u32 head, tail;
+        bool rewind = false;
+        unsigned int free;
+
+        size = ALIGN(size, QUEUE_ALIGNMENT);
+
+        pmu_queue_head(pmu, queue, &head, QUEUE_GET);
+        pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
+        if (head >= tail) {
+                free = queue->offset + queue->size - head;
+                free -= PMU_CMD_HDR_SIZE;
+
+                if (size > free) {
+                        rewind = true;
+                        head = queue->offset;
+                }
+        }
+
+        if (head < tail)
+                free = tail - head - 1;
+
+        if (need_rewind)
+                *need_rewind = rewind;
+
+        return size <= free;
+}
+
+static int pmu_queue_push(struct nvgpu_pmu *pmu,
+                        struct pmu_queue *queue, void *data, u32 size)
+{
+
+        gk20a_dbg_fn("");
+
+        if (!queue->opened && queue->oflag == OFLAG_WRITE) {
+                nvgpu_err(gk20a_from_pmu(pmu), "queue not opened for write");
+                return -EINVAL;
+        }
+
+        pmu_queue_write(pmu, queue->position, data, size);
+        queue->position += ALIGN(size, QUEUE_ALIGNMENT);
+        return 0;
+}
+
+static int pmu_queue_pop(struct nvgpu_pmu *pmu,
+                        struct pmu_queue *queue, void *data, u32 size,
+                        u32 *bytes_read)
+{
+        u32 head, tail, used;
+
+        *bytes_read = 0;
+
+        if (!queue->opened && queue->oflag == OFLAG_READ) {
+                nvgpu_err(gk20a_from_pmu(pmu), "queue not opened for read");
+                return -EINVAL;
+        }
+
+        pmu_queue_head(pmu, queue, &head, QUEUE_GET);
+        tail = queue->position;
+
+        if (head == tail)
+                return 0;
+
+        if (head > tail)
+                used = head - tail;
+        else
+                used = queue->offset + queue->size - tail;
+
+        if (size > used) {
+                nvgpu_warn(gk20a_from_pmu(pmu),
+                        "queue size smaller than request read");
+                size = used;
+        }
+
+        pmu_queue_read(pmu, tail, data, size);
+        queue->position += ALIGN(size, QUEUE_ALIGNMENT);
+        *bytes_read = size;
+        return 0;
+}
+
+static void pmu_queue_rewind(struct nvgpu_pmu *pmu,
+                        struct pmu_queue *queue)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+        struct pmu_cmd cmd;
+
+        gk20a_dbg_fn("");
+
+        if (!queue->opened) {
+                nvgpu_err(gk20a_from_pmu(pmu), "queue not opened");
+                return;
+        }
+
+        if (queue->oflag == OFLAG_WRITE) {
+                cmd.hdr.unit_id = PMU_UNIT_REWIND;
+                cmd.hdr.size = PMU_CMD_HDR_SIZE;
+                pmu_queue_push(pmu, queue, &cmd, cmd.hdr.size);
+                nvgpu_pmu_dbg(g, "queue %d rewinded", queue->id);
+        }
+
+        queue->position = queue->offset;
+}
+
+/* open for read and lock the queue */
+static int pmu_queue_open_read(struct nvgpu_pmu *pmu,
+                        struct pmu_queue *queue)
+{
+        int err;
+
+        err = pmu_queue_lock(pmu, queue);
+        if (err)
+                return err;
+
+        if (queue->opened)
+                BUG();
+
+        pmu_queue_tail(pmu, queue, &queue->position, QUEUE_GET);
+        queue->oflag = OFLAG_READ;
+        queue->opened = true;
+
+        return 0;
+}
+
+/* open for write and lock the queue
+ * make sure there's enough free space for the write
+ * */
+static int pmu_queue_open_write(struct nvgpu_pmu *pmu,
+                        struct pmu_queue *queue, u32 size)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+        bool rewind = false;
+        int err;
+
+        err = pmu_queue_lock(pmu, queue);
+        if (err)
+                return err;
+
+        if (queue->opened)
+                BUG();
+
+        if (!pmu_queue_has_room(pmu, queue, size, &rewind)) {
+                nvgpu_pmu_dbg(g, "queue full: queue-id %d: index %d",
+                                queue->id, queue->index);
+                pmu_queue_unlock(pmu, queue);
+                return -EAGAIN;
+        }
+
+        pmu_queue_head(pmu, queue, &queue->position, QUEUE_GET);
+        queue->oflag = OFLAG_WRITE;
+        queue->opened = true;
+
+        if (rewind)
+                pmu_queue_rewind(pmu, queue);
+
+        return 0;
+}
+
+/* close and unlock the queue */
+static int pmu_queue_close(struct nvgpu_pmu *pmu,
+                        struct pmu_queue *queue, bool commit)
+{
+        if (!queue->opened)
+                return 0;
+
+        if (commit) {
+                if (queue->oflag == OFLAG_READ)
+                        pmu_queue_tail(pmu, queue,
+                                &queue->position, QUEUE_SET);
+                else
+                        pmu_queue_head(pmu, queue,
+                                &queue->position, QUEUE_SET);
+        }
+
+        queue->opened = false;
+
+        pmu_queue_unlock(pmu, queue);
+
+        return 0;
+}
+
+static bool pmu_validate_cmd(struct nvgpu_pmu *pmu, struct pmu_cmd *cmd,
+                        struct pmu_msg *msg, struct pmu_payload *payload,
+                        u32 queue_id)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+        struct pmu_queue *queue;
+        u32 in_size, out_size;
+
+        if (!PMU_IS_SW_COMMAND_QUEUE(queue_id))
+                goto invalid_cmd;
+
+        queue = &pmu->queue[queue_id];
+        if (cmd->hdr.size < PMU_CMD_HDR_SIZE)
+                goto invalid_cmd;
+
+        if (cmd->hdr.size > (queue->size >> 1))
+                goto invalid_cmd;
+
+        if (msg != NULL && msg->hdr.size < PMU_MSG_HDR_SIZE)
+                goto invalid_cmd;
+
+        if (!PMU_UNIT_ID_IS_VALID(cmd->hdr.unit_id))
+                goto invalid_cmd;
+
+        if (payload == NULL)
+                return true;
+
+        if (payload->in.buf == NULL && payload->out.buf == NULL)
+                goto invalid_cmd;
+
+        if ((payload->in.buf != NULL && payload->in.size == 0) ||
+            (payload->out.buf != NULL && payload->out.size == 0))
+                goto invalid_cmd;
+
+        in_size = PMU_CMD_HDR_SIZE;
+        if (payload->in.buf) {
+                in_size += payload->in.offset;
+                in_size += g->ops.pmu_ver.get_pmu_allocation_struct_size(pmu);
+        }
+
+        out_size = PMU_CMD_HDR_SIZE;
+        if (payload->out.buf) {
+                out_size += payload->out.offset;
+                out_size += g->ops.pmu_ver.get_pmu_allocation_struct_size(pmu);
+        }
+
+        if (in_size > cmd->hdr.size || out_size > cmd->hdr.size)
+                goto invalid_cmd;
+
+
+        if ((payload->in.offset != 0 && payload->in.buf == NULL) ||
+            (payload->out.offset != 0 && payload->out.buf == NULL))
+                goto invalid_cmd;
+
+        return true;
+
+invalid_cmd:
+        nvgpu_err(g, "invalid pmu cmd :\n"
+                "queue_id=%d,\n"
+                "cmd_size=%d, cmd_unit_id=%d, msg=%p, msg_size=%d,\n"
+                "payload in=%p, in_size=%d, in_offset=%d,\n"
+                "payload out=%p, out_size=%d, out_offset=%d",
+                queue_id, cmd->hdr.size, cmd->hdr.unit_id,
+                msg, msg ? msg->hdr.unit_id : ~0,
+                &payload->in, payload->in.size, payload->in.offset,
+                &payload->out, payload->out.size, payload->out.offset);
+
+        return false;
+}
+
+static int pmu_write_cmd(struct nvgpu_pmu *pmu, struct pmu_cmd *cmd,
+                        u32 queue_id, unsigned long timeout_ms)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+        struct pmu_queue *queue;
+        struct nvgpu_timeout timeout;
+        int err;
+
+        gk20a_dbg_fn("");
+
+        queue = &pmu->queue[queue_id];
+        nvgpu_timeout_init(g, &timeout, (int)timeout_ms, NVGPU_TIMER_CPU_TIMER);
+
+        do {
+                err = pmu_queue_open_write(pmu, queue, cmd->hdr.size);
+                if (err == -EAGAIN && !nvgpu_timeout_expired(&timeout))
+                        nvgpu_usleep_range(1000, 2000);
+                else
+                        break;
+        } while (1);
+
+        if (err)
+                goto clean_up;
+
+        pmu_queue_push(pmu, queue, cmd, cmd->hdr.size);
+
+
+        err = pmu_queue_close(pmu, queue, true);
+
+clean_up:
+        if (err)
+                nvgpu_err(g, "fail to write cmd to queue %d", queue_id);
+        else
+                gk20a_dbg_fn("done");
+
+        return err;
+}
+
+int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd,
+                struct pmu_msg *msg, struct pmu_payload *payload,
+                u32 queue_id, pmu_callback callback, void *cb_param,
+                u32 *seq_desc, unsigned long timeout)
+{
+        struct nvgpu_pmu *pmu = &g->pmu;
+        struct pmu_v *pv = &g->ops.pmu_ver;
+        struct pmu_sequence *seq;
+        void *in = NULL, *out = NULL;
+        int err;
+
+        gk20a_dbg_fn("");
+
+        if ((!cmd) || (!seq_desc) || (!pmu->pmu_ready)) {
+                if (!cmd)
+                        nvgpu_warn(g, "%s(): PMU cmd buffer is NULL", __func__);
+                else if (!seq_desc)
+                        nvgpu_warn(g, "%s(): Seq descriptor is NULL", __func__);
+                else
+                        nvgpu_warn(g, "%s(): PMU is not ready", __func__);
+
+                WARN_ON(1);
+                return -EINVAL;
+        }
+
+        if (!pmu_validate_cmd(pmu, cmd, msg, payload, queue_id))
+                return -EINVAL;
+
+        err = pmu_seq_acquire(pmu, &seq);
+        if (err)
+                return err;
+
+        cmd->hdr.seq_id = seq->id;
+
+        cmd->hdr.ctrl_flags = 0;
+        cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_STATUS;
+        cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_INTR;
+
+        seq->callback = callback;
+        seq->cb_params = cb_param;
+        seq->msg = msg;
+        seq->out_payload = NULL;
+        seq->desc = pmu->next_seq_desc++;
+
+        if (payload)
+                seq->out_payload = payload->out.buf;
+
+        *seq_desc = seq->desc;
+
+        if (payload && payload->in.offset != 0) {
+                pv->set_pmu_allocation_ptr(pmu, &in,
+                ((u8 *)&cmd->cmd + payload->in.offset));
+
+                if (payload->in.buf != payload->out.buf)
+                        pv->pmu_allocation_set_dmem_size(pmu, in,
+                        (u16)payload->in.size);
+                else
+                        pv->pmu_allocation_set_dmem_size(pmu, in,
+                        (u16)max(payload->in.size, payload->out.size));
+
+                *(pv->pmu_allocation_get_dmem_offset_addr(pmu, in)) =
+                        nvgpu_alloc(&pmu->dmem,
+                                     pv->pmu_allocation_get_dmem_size(pmu, in));
+                if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu, in)))
+                        goto clean_up;
+
+                if (payload->in.fb_size != 0x0) {
+                        seq->in_mem = nvgpu_kzalloc(g,
+                                        sizeof(struct nvgpu_mem));
+                        if (!seq->in_mem) {
+                                err = -ENOMEM;
+                                goto clean_up;
+                        }
+
+                        gk20a_pmu_vidmem_surface_alloc(g, seq->in_mem,
+                                payload->in.fb_size);
+                        gk20a_pmu_surface_describe(g, seq->in_mem,
+                                (struct flcn_mem_desc_v0 *)
+                                pv->pmu_allocation_get_fb_addr(pmu, in));
+
+                        nvgpu_mem_wr_n(g, seq->in_mem, 0,
+                                payload->in.buf, payload->in.fb_size);
+
+                } else {
+                        pmu_copy_to_dmem(pmu,
+                                (pv->pmu_allocation_get_dmem_offset(pmu, in)),
+                                payload->in.buf, payload->in.size, 0);
+                }
+                pv->pmu_allocation_set_dmem_size(pmu,
+                pv->get_pmu_seq_in_a_ptr(seq),
+                pv->pmu_allocation_get_dmem_size(pmu, in));
+                pv->pmu_allocation_set_dmem_offset(pmu,
+                pv->get_pmu_seq_in_a_ptr(seq),
+                pv->pmu_allocation_get_dmem_offset(pmu, in));
+        }
+
+        if (payload && payload->out.offset != 0) {
+                pv->set_pmu_allocation_ptr(pmu, &out,
+                ((u8 *)&cmd->cmd + payload->out.offset));
+                pv->pmu_allocation_set_dmem_size(pmu, out,
+                (u16)payload->out.size);
+
+                if (payload->in.buf != payload->out.buf) {
+                        *(pv->pmu_allocation_get_dmem_offset_addr(pmu, out)) =
+                                nvgpu_alloc(&pmu->dmem,
+                                    pv->pmu_allocation_get_dmem_size(pmu, out));
+                        if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu,
+                                        out)))
+                                goto clean_up;
+
+                        if (payload->out.fb_size != 0x0) {
+                                seq->out_mem = nvgpu_kzalloc(g,
+                                        sizeof(struct nvgpu_mem));
+                                if (!seq->out_mem) {
+                                        err = -ENOMEM;
+                                        goto clean_up;
+                                }
+                                gk20a_pmu_vidmem_surface_alloc(g, seq->out_mem,
+                                        payload->out.fb_size);
+                                gk20a_pmu_surface_describe(g, seq->out_mem,
+                                        (struct flcn_mem_desc_v0 *)
+                                        pv->pmu_allocation_get_fb_addr(pmu,
+                                        out));
+                        }
+                } else {
+                        BUG_ON(in == NULL);
+                        seq->out_mem = seq->in_mem;
+                        pv->pmu_allocation_set_dmem_offset(pmu, out,
+                        pv->pmu_allocation_get_dmem_offset(pmu, in));
+                }
+                pv->pmu_allocation_set_dmem_size(pmu,
+                pv->get_pmu_seq_out_a_ptr(seq),
+                pv->pmu_allocation_get_dmem_size(pmu, out));
+                pv->pmu_allocation_set_dmem_offset(pmu,
+                pv->get_pmu_seq_out_a_ptr(seq),
+                pv->pmu_allocation_get_dmem_offset(pmu, out));
+
+        }
+
+
+
+        seq->state = PMU_SEQ_STATE_USED;
+
+        err = pmu_write_cmd(pmu, cmd, queue_id, timeout);
+        if (err)
+                seq->state = PMU_SEQ_STATE_PENDING;
+
+        gk20a_dbg_fn("done");
+
+        return 0;
+
+clean_up:
+        gk20a_dbg_fn("fail");
+        if (in)
+                nvgpu_free(&pmu->dmem,
+                        pv->pmu_allocation_get_dmem_offset(pmu, in));
+        if (out)
+                nvgpu_free(&pmu->dmem,
+                        pv->pmu_allocation_get_dmem_offset(pmu, out));
+
+        pmu_seq_release(pmu, seq);
+        return err;
+}
+
+static int pmu_response_handle(struct nvgpu_pmu *pmu,
+                        struct pmu_msg *msg)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+        struct pmu_sequence *seq;
+        struct pmu_v *pv = &g->ops.pmu_ver;
+        int ret = 0;
+
+        gk20a_dbg_fn("");
+
+        seq = &pmu->seq[msg->hdr.seq_id];
+        if (seq->state != PMU_SEQ_STATE_USED &&
+            seq->state != PMU_SEQ_STATE_CANCELLED) {
+                nvgpu_err(g, "msg for an unknown sequence %d", seq->id);
+                return -EINVAL;
+        }
+
+        if (msg->hdr.unit_id == PMU_UNIT_RC &&
+            msg->msg.rc.msg_type == PMU_RC_MSG_TYPE_UNHANDLED_CMD) {
+                nvgpu_err(g, "unhandled cmd: seq %d", seq->id);
+        } else if (seq->state != PMU_SEQ_STATE_CANCELLED) {
+                if (seq->msg) {
+                        if (seq->msg->hdr.size >= msg->hdr.size) {
+                                memcpy(seq->msg, msg, msg->hdr.size);
+                        }  else {
+                                nvgpu_err(g, "sequence %d msg buffer too small",
+                                        seq->id);
+                        }
+                }
+                if (pv->pmu_allocation_get_dmem_size(pmu,
+                pv->get_pmu_seq_out_a_ptr(seq)) != 0) {
+                        pmu_copy_from_dmem(pmu,
+                        pv->pmu_allocation_get_dmem_offset(pmu,
+                        pv->get_pmu_seq_out_a_ptr(seq)),
+                        seq->out_payload,
+                        pv->pmu_allocation_get_dmem_size(pmu,
+                        pv->get_pmu_seq_out_a_ptr(seq)), 0);
+                }
+        } else
+                seq->callback = NULL;
+        if (pv->pmu_allocation_get_dmem_size(pmu,
+                        pv->get_pmu_seq_in_a_ptr(seq)) != 0)
+                nvgpu_free(&pmu->dmem,
+                        pv->pmu_allocation_get_dmem_offset(pmu,
+                        pv->get_pmu_seq_in_a_ptr(seq)));
+        if (pv->pmu_allocation_get_dmem_size(pmu,
+                        pv->get_pmu_seq_out_a_ptr(seq)) != 0)
+                nvgpu_free(&pmu->dmem,
+                        pv->pmu_allocation_get_dmem_offset(pmu,
+                        pv->get_pmu_seq_out_a_ptr(seq)));
+
+        if (seq->out_mem != NULL) {
+                memset(pv->pmu_allocation_get_fb_addr(pmu,
+                        pv->get_pmu_seq_out_a_ptr(seq)), 0x0,
+                        pv->pmu_allocation_get_fb_size(pmu,
+                                pv->get_pmu_seq_out_a_ptr(seq)));
+
+                gk20a_pmu_surface_free(g, seq->out_mem);
+                if (seq->out_mem != seq->in_mem)
+                        nvgpu_kfree(g, seq->out_mem);
+                else
+                        seq->out_mem = NULL;
+        }
+
+        if (seq->in_mem != NULL) {
+                memset(pv->pmu_allocation_get_fb_addr(pmu,
+                        pv->get_pmu_seq_in_a_ptr(seq)), 0x0,
+                        pv->pmu_allocation_get_fb_size(pmu,
+                                pv->get_pmu_seq_in_a_ptr(seq)));
+
+                gk20a_pmu_surface_free(g, seq->in_mem);
+                nvgpu_kfree(g, seq->in_mem);
+                seq->in_mem = NULL;
+        }
+
+        if (seq->callback)
+                seq->callback(g, msg, seq->cb_params, seq->desc, ret);
+
+        pmu_seq_release(pmu, seq);
+
+        /* TBD: notify client waiting for available dmem */
+
+        gk20a_dbg_fn("done");
+
+        return 0;
+}
+
+static int pmu_handle_event(struct nvgpu_pmu *pmu, struct pmu_msg *msg)
+{
+        int err = 0;
+        struct gk20a *g = gk20a_from_pmu(pmu);
+
+        gk20a_dbg_fn("");
+        switch (msg->hdr.unit_id) {
+        case PMU_UNIT_PERFMON:
+        case PMU_UNIT_PERFMON_T18X:
+                err = nvgpu_pmu_handle_perfmon_event(pmu, &msg->msg.perfmon);
+                break;
+        case PMU_UNIT_PERF:
+                if (g->ops.perf.handle_pmu_perf_event != NULL) {
+                        err = g->ops.perf.handle_pmu_perf_event(g,
+                                (void *)&msg->msg.perf);
+                } else {
+                        WARN_ON(1);
+                }
+                break;
+        case PMU_UNIT_THERM:
+                err = nvgpu_pmu_handle_therm_event(pmu, &msg->msg.therm);
+                break;
+        default:
+                break;
+        }
+
+        return err;
+}
+
+static bool pmu_read_message(struct nvgpu_pmu *pmu, struct pmu_queue *queue,
+                        struct pmu_msg *msg, int *status)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+        u32 read_size, bytes_read;
+        int err;
+
+        *status = 0;
+
+        if (nvgpu_pmu_queue_is_empty(pmu, queue))
+                return false;
+
+        err = pmu_queue_open_read(pmu, queue);
+        if (err) {
+                nvgpu_err(g, "fail to open queue %d for read", queue->id);
+                *status = err;
+                return false;
+        }
+
+        err = pmu_queue_pop(pmu, queue, &msg->hdr,
+                        PMU_MSG_HDR_SIZE, &bytes_read);
+        if (err || bytes_read != PMU_MSG_HDR_SIZE) {
+                nvgpu_err(g, "fail to read msg from queue %d", queue->id);
+                *status = err | -EINVAL;
+                goto clean_up;
+        }
+
+        if (msg->hdr.unit_id == PMU_UNIT_REWIND) {
+                pmu_queue_rewind(pmu, queue);
+                /* read again after rewind */
+                err = pmu_queue_pop(pmu, queue, &msg->hdr,
+                                PMU_MSG_HDR_SIZE, &bytes_read);
+                if (err || bytes_read != PMU_MSG_HDR_SIZE) {
+                        nvgpu_err(g,
+                                "fail to read msg from queue %d", queue->id);
+                        *status = err | -EINVAL;
+                        goto clean_up;
+                }
+        }
+
+        if (!PMU_UNIT_ID_IS_VALID(msg->hdr.unit_id)) {
+                nvgpu_err(g, "read invalid unit_id %d from queue %d",
+                        msg->hdr.unit_id, queue->id);
+                        *status = -EINVAL;
+                        goto clean_up;
+        }
+
+        if (msg->hdr.size > PMU_MSG_HDR_SIZE) {
+                read_size = msg->hdr.size - PMU_MSG_HDR_SIZE;
+                err = pmu_queue_pop(pmu, queue, &msg->msg,
+                        read_size, &bytes_read);
+                if (err || bytes_read != read_size) {
+                        nvgpu_err(g,
+                                "fail to read msg from queue %d", queue->id);
+                        *status = err;
+                        goto clean_up;
+                }
+        }
+
+        err = pmu_queue_close(pmu, queue, true);
+        if (err) {
+                nvgpu_err(g, "fail to close queue %d", queue->id);
+                *status = err;
+                return false;
+        }
+
+        return true;
+
+clean_up:
+        err = pmu_queue_close(pmu, queue, false);
+        if (err)
+                nvgpu_err(g, "fail to close queue %d", queue->id);
+        return false;
+}
+
+int nvgpu_pmu_process_message(struct nvgpu_pmu *pmu)
+{
+        struct pmu_msg msg;
+        int status;
+        struct gk20a *g = gk20a_from_pmu(pmu);
+
+        if (unlikely(!pmu->pmu_ready)) {
+                nvgpu_pmu_process_init_msg(pmu, &msg);
+                if (g->ops.pmu.init_wpr_region != NULL)
+                        g->ops.pmu.init_wpr_region(g);
+                nvgpu_pmu_init_perfmon(pmu);
+
+                return 0;
+        }
+
+        while (pmu_read_message(pmu,
+                &pmu->queue[PMU_MESSAGE_QUEUE], &msg, &status)) {
+
+                nvgpu_pmu_dbg(g, "read msg hdr: ");
+                nvgpu_pmu_dbg(g, "unit_id = 0x%08x, size = 0x%08x",
+                        msg.hdr.unit_id, msg.hdr.size);
+                nvgpu_pmu_dbg(g, "ctrl_flags = 0x%08x, seq_id = 0x%08x",
+                        msg.hdr.ctrl_flags, msg.hdr.seq_id);
+
+                msg.hdr.ctrl_flags &= ~PMU_CMD_FLAGS_PMU_MASK;
+
+                if (msg.hdr.ctrl_flags == PMU_CMD_FLAGS_EVENT)
+                        pmu_handle_event(pmu, &msg);
+                else
+                        pmu_response_handle(pmu, &msg);
+        }
+
+        return 0;
+}
+
+int pmu_wait_message_cond(struct nvgpu_pmu *pmu, u32 timeout_ms,
+                                 u32 *var, u32 val)
+{
+        struct gk20a *g = gk20a_from_pmu(pmu);
+        struct nvgpu_timeout timeout;
+        unsigned long delay = GR_IDLE_CHECK_DEFAULT;
+
+        nvgpu_timeout_init(g, &timeout, (int)timeout_ms, NVGPU_TIMER_CPU_TIMER);
+
+        do {
+                if (*var == val)
+                        return 0;
+
+                if (gk20a_pmu_is_interrupted(pmu))
+                        gk20a_pmu_isr(g);
+
+                nvgpu_usleep_range(delay, delay * 2);
+                delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
+        } while (!nvgpu_timeout_expired(&timeout));
+
+        return -ETIMEDOUT;
+}
+