gpu: nvgpu: add cg and pg function

Add new power/clock gating functions that can be called by
other units.

New clock_gating functions will reside in cg.c under
common/power_features/cg unit.

New power gating functions will reside in pg.c under
common/power_features/pg unit.

Use nvgpu_pg_elpg_disable and nvgpu_pg_elpg_enable to disable/enable
elpg and also in gr_gk20a_elpg_protected macro to access gr registers.

Add cg_pg_lock to make elpg_enabled, elcg_enabled, blcg_enabled
and slcg_enabled thread safe.

JIRA NVGPU-2014

Change-Id: I00d124c2ee16242c9a3ef82e7620fbb7f1297aff
Signed-off-by: Seema Khowala <seemaj@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/2025493
Signed-off-by: Debarshi Dutta <ddutta@nvidia.com>
(cherry-picked from c90585856567a547173a8b207365b3a4a3ccdd57 in
dev-kernel)
Reviewed-on: https://git-master.nvidia.com/r/2108406
GVS: Gerrit_Virtual_Submit
Reviewed-by: Bibek Basu <bbasu@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

5 files changed, 49 insertions, 186 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
index 6df8f6e4..5052fc35 100644
--- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
@@ -30,6 +30,7 @@
 #include <nvgpu/io.h>
 #include <nvgpu/utils.h>
 #include <nvgpu/channel.h>
+#include <nvgpu/power_features/cg.h>
 
 #include "gk20a.h"
 #include "gk20a/fence_gk20a.h"
@@ -339,14 +340,9 @@ int gk20a_init_ce_support(struct gk20a *g)
 
         g->ops.mc.reset(g, ce_reset_mask);
 
-        if (g->ops.clock_gating.slcg_ce2_load_gating_prod) {
-                g->ops.clock_gating.slcg_ce2_load_gating_prod(g,
-                                g->slcg_enabled);
-        }
-        if (g->ops.clock_gating.blcg_ce_load_gating_prod) {
-                g->ops.clock_gating.blcg_ce_load_gating_prod(g,
-                                g->blcg_enabled);
-        }
+        nvgpu_cg_slcg_ce2_load_enable(g);
+
+        nvgpu_cg_blcg_ce_load_enable(g);
 
         if (ce_app->initialised) {
                 /* assume this happen during poweron/poweroff GPU sequence */
diff --git a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
index adc13c3d..1686d01e 100644
--- a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
@@ -1,7 +1,7 @@
 /*
  * Tegra GK20A GPU Debugger/Profiler Driver
  *
- * Copyright (c) 2013-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2013-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -32,6 +32,7 @@
 #include <nvgpu/utils.h>
 #include <nvgpu/channel.h>
 #include <nvgpu/unit.h>
+#include <nvgpu/power_features/power_features.h>
 
 #include "gk20a.h"
 #include "gr_gk20a.h"
@@ -234,60 +235,28 @@ int dbg_set_powergate(struct dbg_session_gk20a *dbg_s, bool disable_powergate)
                         return err;
                 }
 
-                /*do elpg disable before clock gating */
-                nvgpu_pmu_pg_global_enable(g, false);
+                err = nvgpu_cg_pg_disable(g);
 
-                if (g->ops.clock_gating.slcg_gr_load_gating_prod) {
-                        g->ops.clock_gating.slcg_gr_load_gating_prod(g,
-                                false);
+                if (err == 0) {
+                        dbg_s->is_pg_disabled = true;
+                        nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn,
+                                        "pg disabled");
                 }
-                if (g->ops.clock_gating.slcg_perf_load_gating_prod) {
-                        g->ops.clock_gating.slcg_perf_load_gating_prod(g,
-                                false);
-                }
-                if (g->ops.clock_gating.slcg_ltc_load_gating_prod) {
-                        g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
-                                false);
-                }
-
-                gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_RUN);
-                gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN);
-
-                dbg_s->is_pg_disabled = true;
         } else {
                 /* restore (can) powergate, clk state */
                 /* release pending exceptions to fault/be handled as usual */
                 /*TBD: ordering of these? */
 
-                if (g->elcg_enabled) {
-                        gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_AUTO);
-                }
+                err = nvgpu_cg_pg_enable(g);
 
-                if (g->blcg_enabled) {
-                        gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_AUTO);
-                }
-
-                if (g->slcg_enabled) {
-                        if (g->ops.clock_gating.slcg_ltc_load_gating_prod) {
-                                g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
-                                        g->slcg_enabled);
-                        }
-                        if (g->ops.clock_gating.slcg_perf_load_gating_prod) {
-                                g->ops.clock_gating.slcg_perf_load_gating_prod(g,
-                                        g->slcg_enabled);
-                        }
-                        if (g->ops.clock_gating.slcg_gr_load_gating_prod) {
-                                g->ops.clock_gating.slcg_gr_load_gating_prod(g,
-                                        g->slcg_enabled);
-                        }
-                }
-                nvgpu_pmu_pg_global_enable(g, true);
-
-                nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn,
-                                        "module idle");
+                nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, "module idle");
                 gk20a_idle(g);
 
-                dbg_s->is_pg_disabled = false;
+                if (err == 0) {
+                        dbg_s->is_pg_disabled = false;
+                        nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn,
+                                        "pg enabled");
+                }
         }
 
         nvgpu_log(g, gpu_dbg_fn|gpu_dbg_gpu_dbg, "%s powergate mode = %s done",
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index 78f777ae..6d89940a 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -45,6 +45,8 @@
 #include <nvgpu/utils.h>
 #include <nvgpu/channel.h>
 #include <nvgpu/unit.h>
+#include <nvgpu/power_features/power_features.h>
+#include <nvgpu/power_features/cg.h>
 
 #include "gk20a.h"
 #include "mm_gk20a.h"
@@ -824,14 +826,9 @@ int gk20a_init_fifo_reset_enable_hw(struct gk20a *g)
         /* enable pmc pfifo */
         g->ops.mc.reset(g, g->ops.mc.reset_mask(g, NVGPU_UNIT_FIFO));
 
-        if (g->ops.clock_gating.slcg_fifo_load_gating_prod) {
-                g->ops.clock_gating.slcg_fifo_load_gating_prod(g,
-                                g->slcg_enabled);
-        }
-        if (g->ops.clock_gating.blcg_fifo_load_gating_prod) {
-                g->ops.clock_gating.blcg_fifo_load_gating_prod(g,
-                                g->blcg_enabled);
-        }
+        nvgpu_cg_slcg_fifo_load_enable(g);
+
+        nvgpu_cg_blcg_fifo_load_enable(g);
 
         timeout = gk20a_readl(g, fifo_fb_timeout_r());
         timeout = set_field(timeout, fifo_fb_timeout_period_m(),
@@ -1361,8 +1358,8 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
         }
 
         if (engine_enum == ENGINE_GR_GK20A) {
-                if (g->support_pmu && g->can_elpg) {
-                        if (nvgpu_pmu_disable_elpg(g)) {
+                if (g->support_pmu) {
+                        if (nvgpu_pg_elpg_disable(g) != 0 ) {
                                 nvgpu_err(g, "failed to set disable elpg");
                         }
                 }
@@ -1391,8 +1388,10 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
                                 "HALT gr pipe not supported and "
                                 "gr cannot be reset without halting gr pipe");
                 }
-                if (g->support_pmu && g->can_elpg) {
-                        nvgpu_pmu_enable_elpg(g);
+                if (g->support_pmu) {
+                        if (nvgpu_pg_elpg_enable(g) != 0 ) {
+                                nvgpu_err(g, "failed to set enable elpg");
+                        }
                 }
         }
         if ((engine_enum == ENGINE_GRCE_GK20A) ||
@@ -1638,25 +1637,11 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
         g->fifo.deferred_reset_pending = false;
 
         /* Disable power management */
-        if (g->support_pmu && g->can_elpg) {
-                if (nvgpu_pmu_disable_elpg(g)) {
-                        nvgpu_err(g, "failed to set disable elpg");
+        if (g->support_pmu) {
+                if (nvgpu_cg_pg_disable(g) != 0) {
+                        nvgpu_warn(g, "fail to disable power mgmt");
                 }
         }
-        if (g->ops.clock_gating.slcg_gr_load_gating_prod) {
-                g->ops.clock_gating.slcg_gr_load_gating_prod(g,
-                                false);
-        }
-        if (g->ops.clock_gating.slcg_perf_load_gating_prod) {
-                g->ops.clock_gating.slcg_perf_load_gating_prod(g,
-                                false);
-        }
-        if (g->ops.clock_gating.slcg_ltc_load_gating_prod) {
-                g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
-                                false);
-        }
-
-        gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN);
 
         /* Disable fifo access */
         grfifo_ctl = gk20a_readl(g, gr_gpfifo_ctl_r());
@@ -1842,8 +1827,10 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
                      gr_gpfifo_ctl_semaphore_access_enabled_f());
 
         /* It is safe to enable ELPG again. */
-        if (g->support_pmu && g->can_elpg) {
-                nvgpu_pmu_enable_elpg(g);
+        if (g->support_pmu) {
+                if (nvgpu_cg_pg_enable(g) != 0) {
+                        nvgpu_warn(g, "fail to enable power mgmt");
+                }
         }
 
         return verbose;
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
index 9e4d3c37..a4c1ce58 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
@@ -43,6 +43,8 @@
 #include <nvgpu/utils.h>
 #include <nvgpu/channel.h>
 #include <nvgpu/unit.h>
+#include <nvgpu/power_features/pg.h>
+#include <nvgpu/power_features/cg.h>
 
 #include "gk20a.h"
 #include "gr_gk20a.h"
@@ -91,8 +93,6 @@ static void gr_gk20a_free_channel_patch_ctx(struct gk20a *g,
 /* golden ctx image */
 static int gr_gk20a_init_golden_ctx_image(struct gk20a *g,
                                           struct channel_gk20a *c);
-/*elcg init */
-static void gr_gk20a_enable_elcg(struct gk20a *g);
 
 int gr_gk20a_get_ctx_id(struct gk20a *g,
                 struct channel_gk20a *c,
@@ -4227,33 +4227,6 @@ int gk20a_gr_zbc_set_table(struct gk20a *g, struct gr_gk20a *gr,
                 gr_gk20a_add_zbc(g, gr, zbc_val));
 }
 
-void gr_gk20a_init_cg_mode(struct gk20a *g, u32 cgmode, u32 mode_config)
-{
-        u32 engine_idx;
-        u32 active_engine_id = 0;
-        struct fifo_engine_info_gk20a *engine_info = NULL;
-        struct fifo_gk20a *f = &g->fifo;
-
-        for (engine_idx = 0; engine_idx < f->num_engines; ++engine_idx) {
-                active_engine_id = f->active_engines_list[engine_idx];
-                engine_info = &f->engine_info[active_engine_id];
-
-                /* gr_engine supports both BLCG and ELCG */
-                if ((cgmode == BLCG_MODE) &&
-                        (engine_info->engine_enum == ENGINE_GR_GK20A)) {
-                                g->ops.therm.init_blcg_mode(g, mode_config, active_engine_id);
-                                break;
-                } else if (cgmode == ELCG_MODE) {
-                        g->ops.therm.init_elcg_mode(g, mode_config,
-                                                active_engine_id);
-                } else {
-                        nvgpu_err(g, "invalid cg mode %d, config %d for "
-                                                        "act_eng_id %d",
-                                        cgmode, mode_config, active_engine_id);
-                }
-        }
-}
-
 void gr_gk20a_program_zcull_mapping(struct gk20a *g, u32 zcull_num_entries,
                                                 u32 *zcull_map_tiles)
 {
@@ -4655,60 +4628,6 @@ out:
         return err;
 }
 
-static void gr_gk20a_load_gating_prod(struct gk20a *g)
-{
-        nvgpu_log_fn(g, " ");
-
-        /* slcg prod values */
-        if (g->ops.clock_gating.slcg_bus_load_gating_prod) {
-                g->ops.clock_gating.slcg_bus_load_gating_prod(g,
-                                g->slcg_enabled);
-        }
-        if (g->ops.clock_gating.slcg_chiplet_load_gating_prod) {
-                g->ops.clock_gating.slcg_chiplet_load_gating_prod(g,
-                                g->slcg_enabled);
-        }
-        if (g->ops.clock_gating.slcg_gr_load_gating_prod) {
-                g->ops.clock_gating.slcg_gr_load_gating_prod(g,
-                                g->slcg_enabled);
-        }
-        if (g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod) {
-                g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod(g,
-                                g->slcg_enabled);
-        }
-        if (g->ops.clock_gating.slcg_perf_load_gating_prod) {
-                g->ops.clock_gating.slcg_perf_load_gating_prod(g,
-                                g->slcg_enabled);
-        }
-        if (g->ops.clock_gating.slcg_xbar_load_gating_prod) {
-                g->ops.clock_gating.slcg_xbar_load_gating_prod(g,
-                                g->slcg_enabled);
-        }
-
-        /* blcg prod values */
-        if (g->ops.clock_gating.blcg_bus_load_gating_prod) {
-                g->ops.clock_gating.blcg_bus_load_gating_prod(g,
-                                g->blcg_enabled);
-        }
-        if (g->ops.clock_gating.blcg_gr_load_gating_prod) {
-                g->ops.clock_gating.blcg_gr_load_gating_prod(g,
-                                g->blcg_enabled);
-        }
-        if (g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod) {
-                g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod(g,
-                                g->blcg_enabled);
-        }
-        if (g->ops.clock_gating.blcg_xbar_load_gating_prod) {
-                g->ops.clock_gating.blcg_xbar_load_gating_prod(g,
-                                g->blcg_enabled);
-        }
-        if (g->ops.clock_gating.pg_gr_load_gating_prod) {
-                g->ops.clock_gating.pg_gr_load_gating_prod(g, true);
-        }
-
-        nvgpu_log_fn(g, "done");
-}
-
 static int gk20a_init_gr_prepare(struct gk20a *g)
 {
         u32 err = 0;
@@ -4718,10 +4637,10 @@ static int gk20a_init_gr_prepare(struct gk20a *g)
                         g->ops.mc.reset_mask(g, NVGPU_UNIT_BLG) |
                         g->ops.mc.reset_mask(g, NVGPU_UNIT_PERFMON));
 
-        gr_gk20a_load_gating_prod(g);
+        nvgpu_cg_init_gr_load_gating_prod(g);
 
         /* Disable elcg until it gets enabled later in the init*/
-        gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN);
+        nvgpu_cg_elcg_disable(g);
 
         /* enable fifo access */
         gk20a_writel(g, gr_gpfifo_ctl_r(),
@@ -5041,7 +4960,7 @@ int gk20a_init_gr_support(struct gk20a *g)
                 }
         }
 
-        gr_gk20a_enable_elcg(g);
+        nvgpu_cg_elcg_enable(g);
         /* GR is inialized, signal possible waiters */
         g->gr.initialized = true;
         nvgpu_cond_signal(&g->gr.init_wq);
@@ -5128,15 +5047,6 @@ int gk20a_enable_gr_hw(struct gk20a *g)
         return 0;
 }
 
-static void gr_gk20a_enable_elcg(struct gk20a *g)
-{
-        if (g->elcg_enabled) {
-                gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_AUTO);
-        } else {
-                gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN);
-        }
-}
-
 int gk20a_gr_reset(struct gk20a *g)
 {
         int err;
@@ -5193,8 +5103,8 @@ int gk20a_gr_reset(struct gk20a *g)
                 return err;
         }
 
-        gr_gk20a_load_gating_prod(g);
-        gr_gk20a_enable_elcg(g);
+        nvgpu_cg_init_gr_load_gating_prod(g);
+        nvgpu_cg_elcg_enable(g);
 
         return err;
 }
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.h b/drivers/gpu/nvgpu/gk20a/gr_gk20a.h
index d795a3fc..8ff2cfd4 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.h
@@ -28,6 +28,7 @@
 
 #include "gr_ctx_gk20a.h"
 #include "mm_gk20a.h"
+#include <nvgpu/power_features/pg.h>
 
 #include <nvgpu/comptags.h>
 #include <nvgpu/cond.h>
@@ -598,16 +599,16 @@ u32 gk20a_gr_get_sm_no_lock_down_hww_global_esr_mask(struct gk20a *g);
 #define gr_gk20a_elpg_protected_call(g, func) \
         ({ \
                 int err = 0; \
-                if ((g->support_pmu) && (g->elpg_enabled)) {\
-                        err = nvgpu_pmu_disable_elpg(g); \
+                if (g->support_pmu) {\
+                        err = nvgpu_pg_elpg_disable(g);\
                         if (err != 0) {\
-                                nvgpu_pmu_enable_elpg(g); \
+                                err = nvgpu_pg_elpg_enable(g); \
                         } \
                 } \
                 if (err == 0) { \
                         err = func; \
-                        if ((g->support_pmu) && (g->elpg_enabled)) {\
-                                nvgpu_pmu_enable_elpg(g); \
+                        if (g->support_pmu) {\
+                                (void)nvgpu_pg_elpg_enable(g); \
                         } \
                 } \
                 err; \