gpu: nvgpu: PMU debug reorg

- Moved PMU debug related code to pmu_debug.c
  Print pmu trace buffer
  Moved PMU controller/engine status dump debug code
  Moved ELPG stats  dump code
- Removed PMU falcon controller status dump code & used
nvgpu_flcn_dump_stats() method,
- Method to print ELPG stats.
- PMU HAL to print PMU engine & ELPG debug info upon error

NVGPU JIRA-96

Change-Id: Iaa3d983f1d3b78a1b051beb6c109d3da8f8c90bc
Signed-off-by: Mahantesh Kumbar <mkumbar@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1516640
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
Reviewed-by: svccoveritychecker <svccoveritychecker@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Vijayakumar Subbu <vsubbu@nvidia.com>

6 files changed, 72 insertions, 155 deletions

diff --git a/drivers/gpu/nvgpu/Makefile.nvgpu b/drivers/gpu/nvgpu/Makefile.nvgpu
index 6e475fcb..35d5109b 100644
--- a/drivers/gpu/nvgpu/Makefile.nvgpu
+++ b/drivers/gpu/nvgpu/Makefile.nvgpu
@@ -69,6 +69,7 @@ nvgpu-y := \
         common/pmu/pmu_fw.o \
         common/pmu/pmu_pg.o \
         common/pmu/pmu_perfmon.o \
+        common/pmu/pmu_debug.o \
         common/ltc.o \
         gk20a/gk20a.o \
         gk20a/bus_gk20a.o \
diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_debug.c b/drivers/gpu/nvgpu/common/pmu/pmu_debug.c
new file mode 100644
index 00000000..744a618d
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/pmu/pmu_debug.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2016-2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <nvgpu/pmu.h>
+#include <nvgpu/log.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/pmuif/nvgpu_gpmu_cmdif.h>
+
+#include "gk20a/gk20a.h"
+
+void nvgpu_pmu_dump_elpg_stats(struct nvgpu_pmu *pmu)
+{
+        struct gk20a *g = pmu->g;
+
+        /* Print PG stats */
+        nvgpu_err(g, "Print PG stats");
+        nvgpu_flcn_print_dmem(pmu->flcn,
+                pmu->stat_dmem_offset[PMU_PG_ELPG_ENGINE_ID_GRAPHICS],
+                sizeof(struct pmu_pg_stats_v2));
+
+        gk20a_pmu_dump_elpg_stats(pmu);
+}
+
+void nvgpu_pmu_dump_falcon_stats(struct nvgpu_pmu *pmu)
+{
+        struct gk20a *g = pmu->g;
+
+        nvgpu_flcn_dump_stats(pmu->flcn);
+        gk20a_pmu_dump_falcon_stats(pmu);
+
+        nvgpu_err(g, "pmu state: %d", pmu->pmu_state);
+        nvgpu_err(g, "elpg state: %d", pmu->elpg_stat);
+
+        /* PMU may crash due to FECS crash. Dump FECS status */
+        gk20a_fecs_dump_falcon_stats(g);
+}
diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_pg.c b/drivers/gpu/nvgpu/common/pmu/pmu_pg.c
index b435f4a7..3f74b9f7 100644
--- a/drivers/gpu/nvgpu/common/pmu/pmu_pg.c
+++ b/drivers/gpu/nvgpu/common/pmu/pmu_pg.c
@@ -265,8 +265,8 @@ int nvgpu_pmu_disable_elpg(struct gk20a *g)
                 if (pmu->elpg_stat != PMU_ELPG_STAT_ON) {
                         nvgpu_err(g, "ELPG_ALLOW_ACK failed, elpg_stat=%d",
                                 pmu->elpg_stat);
-                        pmu_dump_elpg_stats(pmu);
-                        pmu_dump_falcon_stats(pmu);
+                        nvgpu_pmu_dump_elpg_stats(pmu);
+                        nvgpu_pmu_dump_falcon_stats(pmu);
                         ret = -EBUSY;
                         goto exit_unlock;
                 }
@@ -315,8 +315,8 @@ int nvgpu_pmu_disable_elpg(struct gk20a *g)
                                 ptr, PMU_ELPG_STAT_OFF);
                         if (*ptr != PMU_ELPG_STAT_OFF) {
                                 nvgpu_err(g, "ELPG_DISALLOW_ACK failed");
-                                        pmu_dump_elpg_stats(pmu);
-                                        pmu_dump_falcon_stats(pmu);
+                                        nvgpu_pmu_dump_elpg_stats(pmu);
+                                        nvgpu_pmu_dump_falcon_stats(pmu);
                                 ret = -EBUSY;
                                 goto exit_unlock;
                         }
diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
index 11de11de..ea0559aa 100644
--- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
@@ -39,7 +39,6 @@
 #define gk20a_dbg_pmu(fmt, arg...) \
         gk20a_dbg(gpu_dbg_pmu, fmt, ##arg)
 
-
 bool nvgpu_find_hex_in_string(char *strings, struct gk20a *g, u32 *hex_pos)
 {
         u32 i = 0, j = strlen(strings);
@@ -55,11 +54,11 @@ bool nvgpu_find_hex_in_string(char *strings, struct gk20a *g, u32 *hex_pos)
         return false;
 }
 
-static void printtrace(struct nvgpu_pmu *pmu)
+static void print_pmu_trace(struct nvgpu_pmu *pmu)
 {
+        struct gk20a *g = pmu->g;
         u32 i = 0, j = 0, k, l, m, count;
         char part_str[40], buf[0x40];
-        struct gk20a *g = gk20a_from_pmu(pmu);
         void *tracebuffer;
         char *trace;
         u32 *trace1;
@@ -70,13 +69,13 @@ static void printtrace(struct nvgpu_pmu *pmu)
                 return;
 
         /* read pmu traces into system memory buffer */
-        nvgpu_mem_rd_n(g, &pmu->trace_buf,
-                       0, tracebuffer, GK20A_PMU_TRACE_BUFSIZE);
+        nvgpu_mem_rd_n(g, &pmu->trace_buf, 0, tracebuffer,
+                GK20A_PMU_TRACE_BUFSIZE);
 
         trace = (char *)tracebuffer;
         trace1 = (u32 *)tracebuffer;
 
-        nvgpu_err(g, "Dump pmutrace");
+        nvgpu_err(g, "dump PMU trace buffer");
         for (i = 0; i < GK20A_PMU_TRACE_BUFSIZE; i += 0x40) {
                 for (j = 0; j < 0x40; j++)
                         if (trace1[(i / 4) + j])
@@ -100,6 +99,7 @@ static void printtrace(struct nvgpu_pmu *pmu)
                 scnprintf((buf + count), 0x40, "%s", (trace+i+20+m));
                 nvgpu_err(g, "%s", buf);
         }
+
         nvgpu_kfree(g, tracebuffer);
 }
 
@@ -597,51 +597,9 @@ int nvgpu_pmu_handle_therm_event(struct nvgpu_pmu *pmu,
         return 0;
 }
 
-void pmu_dump_elpg_stats(struct nvgpu_pmu *pmu)
+void gk20a_pmu_dump_elpg_stats(struct nvgpu_pmu *pmu)
 {
         struct gk20a *g = gk20a_from_pmu(pmu);
-        struct pmu_pg_stats stats;
-
-        nvgpu_flcn_copy_from_dmem(pmu->flcn,
-                pmu->stat_dmem_offset[PMU_PG_ELPG_ENGINE_ID_GRAPHICS],
-                (u8 *)&stats, sizeof(struct pmu_pg_stats), 0);
-
-        gk20a_dbg_pmu("pg_entry_start_timestamp : 0x%016llx",
-                stats.pg_entry_start_timestamp);
-        gk20a_dbg_pmu("pg_exit_start_timestamp : 0x%016llx",
-                stats.pg_exit_start_timestamp);
-        gk20a_dbg_pmu("pg_ingating_start_timestamp : 0x%016llx",
-                stats.pg_ingating_start_timestamp);
-        gk20a_dbg_pmu("pg_ungating_start_timestamp : 0x%016llx",
-                stats.pg_ungating_start_timestamp);
-        gk20a_dbg_pmu("pg_avg_entry_time_us : 0x%08x",
-                stats.pg_avg_entry_time_us);
-        gk20a_dbg_pmu("pg_avg_exit_time_us : 0x%08x",
-                stats.pg_avg_exit_time_us);
-        gk20a_dbg_pmu("pg_ingating_cnt : 0x%08x",
-                stats.pg_ingating_cnt);
-        gk20a_dbg_pmu("pg_ingating_time_us : 0x%08x",
-                stats.pg_ingating_time_us);
-        gk20a_dbg_pmu("pg_ungating_count : 0x%08x",
-                stats.pg_ungating_count);
-        gk20a_dbg_pmu("pg_ungating_time_us 0x%08x: ",
-                stats.pg_ungating_time_us);
-        gk20a_dbg_pmu("pg_gating_cnt : 0x%08x",
-                stats.pg_gating_cnt);
-        gk20a_dbg_pmu("pg_gating_deny_cnt : 0x%08x",
-                stats.pg_gating_deny_cnt);
-
-        /*
-           Turn on PG_DEBUG in ucode and locate symbol "ElpgLog" offset
-           in .nm file, e.g. 0x1000066c. use 0x66c.
-        u32 i, val[20];
-        nvgpu_flcn_copy_from_dmem(pmu->flcn, 0x66c,
-                (u8 *)val, sizeof(val), 0);
-        gk20a_dbg_pmu("elpg log begin");
-        for (i = 0; i < 20; i++)
-                gk20a_dbg_pmu("0x%08x", val[i]);
-        gk20a_dbg_pmu("elpg log end");
-        */
 
         gk20a_dbg_pmu("pwr_pmu_idle_mask_supp_r(3): 0x%08x",
                 gk20a_readl(g, pwr_pmu_idle_mask_supp_r(3)));
@@ -660,40 +618,13 @@ void pmu_dump_elpg_stats(struct nvgpu_pmu *pmu)
                 gk20a_readl(g, pwr_pmu_idle_count_r(4)));
         gk20a_dbg_pmu("pwr_pmu_idle_count_r(7): 0x%08x",
                 gk20a_readl(g, pwr_pmu_idle_count_r(7)));
-
-        /*
-         TBD: script can't generate those registers correctly
-        gk20a_dbg_pmu("pwr_pmu_idle_status_r(): 0x%08x",
-                gk20a_readl(g, pwr_pmu_idle_status_r()));
-        gk20a_dbg_pmu("pwr_pmu_pg_ctrl_r(): 0x%08x",
-                gk20a_readl(g, pwr_pmu_pg_ctrl_r()));
-        */
 }
 
-void pmu_dump_falcon_stats(struct nvgpu_pmu *pmu)
+void gk20a_pmu_dump_falcon_stats(struct nvgpu_pmu *pmu)
 {
         struct gk20a *g = gk20a_from_pmu(pmu);
         unsigned int i;
 
-        nvgpu_err(g, "pwr_falcon_os_r : %d",
-                gk20a_readl(g, pwr_falcon_os_r()));
-        nvgpu_err(g, "pwr_falcon_cpuctl_r : 0x%x",
-                gk20a_readl(g, pwr_falcon_cpuctl_r()));
-        nvgpu_err(g, "pwr_falcon_idlestate_r : 0x%x",
-                gk20a_readl(g, pwr_falcon_idlestate_r()));
-        nvgpu_err(g, "pwr_falcon_mailbox0_r : 0x%x",
-                gk20a_readl(g, pwr_falcon_mailbox0_r()));
-        nvgpu_err(g, "pwr_falcon_mailbox1_r : 0x%x",
-                gk20a_readl(g, pwr_falcon_mailbox1_r()));
-        nvgpu_err(g, "pwr_falcon_irqstat_r : 0x%x",
-                gk20a_readl(g, pwr_falcon_irqstat_r()));
-        nvgpu_err(g, "pwr_falcon_irqmode_r : 0x%x",
-                gk20a_readl(g, pwr_falcon_irqmode_r()));
-        nvgpu_err(g, "pwr_falcon_irqmask_r : 0x%x",
-                gk20a_readl(g, pwr_falcon_irqmask_r()));
-        nvgpu_err(g, "pwr_falcon_irqdest_r : 0x%x",
-                gk20a_readl(g, pwr_falcon_irqdest_r()));
-
         for (i = 0; i < pwr_pmu_mailbox__size_1_v(); i++)
                 nvgpu_err(g, "pwr_pmu_mailbox_r(%d) : 0x%x",
                         i, gk20a_readl(g, pwr_pmu_mailbox_r(i)));
@@ -702,14 +633,6 @@ void pmu_dump_falcon_stats(struct nvgpu_pmu *pmu)
                 nvgpu_err(g, "pwr_pmu_debug_r(%d) : 0x%x",
                         i, gk20a_readl(g, pwr_pmu_debug_r(i)));
 
-        for (i = 0; i < 6/*NV_PPWR_FALCON_ICD_IDX_RSTAT__SIZE_1*/; i++) {
-                gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
-                        pwr_pmu_falcon_icd_cmd_opc_rstat_f() |
-                        pwr_pmu_falcon_icd_cmd_idx_f(i));
-                nvgpu_err(g, "pmu_rstat (%d) : 0x%x",
-                        i, gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
-        }
-
         i = gk20a_readl(g, pwr_pmu_bar0_error_status_r());
         nvgpu_err(g, "pwr_pmu_bar0_error_status_r : 0x%x", i);
         if (i != 0) {
@@ -736,62 +659,8 @@ void pmu_dump_falcon_stats(struct nvgpu_pmu *pmu)
                         gk20a_readl(g, mc_enable_r()));
         }
 
-        nvgpu_err(g, "pwr_falcon_engctl_r : 0x%x",
-                gk20a_readl(g, pwr_falcon_engctl_r()));
-        nvgpu_err(g, "pwr_falcon_curctx_r : 0x%x",
-                gk20a_readl(g, pwr_falcon_curctx_r()));
-        nvgpu_err(g, "pwr_falcon_nxtctx_r : 0x%x",
-                gk20a_readl(g, pwr_falcon_nxtctx_r()));
-
-        gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
-                pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
-                pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_IMB));
-        nvgpu_err(g, "PMU_FALCON_REG_IMB : 0x%x",
-                gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
-
-        gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
-                pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
-                pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_DMB));
-        nvgpu_err(g, "PMU_FALCON_REG_DMB : 0x%x",
-                gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
-
-        gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
-                pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
-                pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_CSW));
-        nvgpu_err(g, "PMU_FALCON_REG_CSW : 0x%x",
-                gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
-
-        gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
-                pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
-                pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_CTX));
-        nvgpu_err(g, "PMU_FALCON_REG_CTX : 0x%x",
-                gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
-
-        gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
-                pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
-                pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_EXCI));
-        nvgpu_err(g, "PMU_FALCON_REG_EXCI : 0x%x",
-                gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
-
-        for (i = 0; i < 4; i++) {
-                gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
-                        pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
-                        pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_PC));
-                nvgpu_err(g, "PMU_FALCON_REG_PC : 0x%x",
-                        gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
-
-                gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
-                        pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
-                        pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_SP));
-                nvgpu_err(g, "PMU_FALCON_REG_SP : 0x%x",
-                        gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
-        }
-        nvgpu_err(g, "elpg stat: %d",
-                        pmu->elpg_stat);
-
-        /* PMU may crash due to FECS crash. Dump FECS status */
-        gk20a_fecs_dump_falcon_stats(g);
-        printtrace(pmu);
+        /* Print PMU F/W debug prints */
+        print_pmu_trace(pmu);
 }
 
 bool gk20a_pmu_is_interrupted(struct nvgpu_pmu *pmu)
@@ -840,7 +709,7 @@ void gk20a_pmu_isr(struct gk20a *g)
 
         if (intr & pwr_falcon_irqstat_halt_true_f()) {
                 nvgpu_err(g, "pmu halt intr not implemented");
-                pmu_dump_falcon_stats(pmu);
+                nvgpu_pmu_dump_falcon_stats(pmu);
                 if (gk20a_readl(g, pwr_pmu_mailbox_r
                                 (PMU_MODE_MISMATCH_STATUS_MAILBOX_R)) ==
                                 PMU_MODE_MISMATCH_STATUS_VAL)
@@ -850,7 +719,7 @@ void gk20a_pmu_isr(struct gk20a *g)
         if (intr & pwr_falcon_irqstat_exterr_true_f()) {
                 nvgpu_err(g,
                         "pmu exterr intr not implemented. Clearing interrupt.");
-                pmu_dump_falcon_stats(pmu);
+                nvgpu_pmu_dump_falcon_stats(pmu);
 
                 gk20a_writel(g, pwr_falcon_exterrstat_r(),
                         gk20a_readl(g, pwr_falcon_exterrstat_r()) &
diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h
index 3ad0c116..3992b029 100644
--- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h
@@ -57,12 +57,10 @@ int gk20a_init_pmu_setup_hw1(struct gk20a *g);
 void gk20a_write_dmatrfbase(struct gk20a *g, u32 addr);
 bool gk20a_is_pmu_supported(struct gk20a *g);
 
-void pmu_copy_to_dmem(struct nvgpu_pmu *pmu,
-                u32 dst, u8 *src, u32 size, u8 port);
 int pmu_bootstrap(struct nvgpu_pmu *pmu);
 
-void pmu_dump_elpg_stats(struct nvgpu_pmu *pmu);
-void pmu_dump_falcon_stats(struct nvgpu_pmu *pmu);
+void gk20a_pmu_dump_elpg_stats(struct nvgpu_pmu *pmu);
+void gk20a_pmu_dump_falcon_stats(struct nvgpu_pmu *pmu);
 
 void pmu_enable_irq(struct nvgpu_pmu *pmu, bool enable);
 int pmu_wait_message_cond(struct nvgpu_pmu *pmu, u32 timeout_ms,
@@ -74,8 +72,4 @@ void gk20a_pmu_elpg_statistics(struct gk20a *g, u32 pg_engine_id,
 bool gk20a_pmu_is_engine_in_reset(struct gk20a *g);
 int gk20a_pmu_engine_reset(struct gk20a *g, bool do_reset);
 
-int pmu_idle(struct nvgpu_pmu *pmu);
-
-bool nvgpu_find_hex_in_string(char *strings, struct gk20a *g, u32 *hex_pos);
-
 #endif /*__PMU_GK20A_H__*/
diff --git a/drivers/gpu/nvgpu/include/nvgpu/pmu.h b/drivers/gpu/nvgpu/include/nvgpu/pmu.h
index 556d9f39..60ce8b96 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/pmu.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/pmu.h
@@ -449,4 +449,9 @@ int nvgpu_aelpg_init_and_enable(struct gk20a *g, u8 ctrl_id);
 int nvgpu_pmu_ap_send_command(struct gk20a *g,
                 union pmu_ap_cmd *p_ap_cmd, bool b_block);
 
+/* PMU debug */
+void nvgpu_pmu_dump_falcon_stats(struct nvgpu_pmu *pmu);
+void nvgpu_pmu_dump_elpg_stats(struct nvgpu_pmu *pmu);
+bool nvgpu_find_hex_in_string(char *strings, struct gk20a *g, u32 *hex_pos);
+
 #endif /* __NVGPU_PMU_H__ */