12 files changed, 118 insertions, 9 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
index fb15b3da..5464b88a 100644
--- a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
@@ -582,6 +582,8 @@ static int dbg_set_powergate(struct dbg_session_gk20a *dbg_s,
                                         false);
                         g->ops.clock_gating.slcg_perf_load_gating_prod(g,
                                         false);
+                        g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
+                                        false);
                         gr_gk20a_init_blcg_mode(g, BLCG_RUN, ENGINE_GR_GK20A);
 
                         g->elcg_enabled = false;
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index e6b3fd5f..230e1722 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -385,6 +385,16 @@ int gk20a_init_fifo_reset_enable_hw(struct gk20a *g)
         gk20a_reset(g, mc_enable_pfifo_enabled_f()
                         | mc_enable_ce2_enabled_f());
 
+        if (g->ops.clock_gating.slcg_ce2_load_gating_prod)
+                g->ops.clock_gating.slcg_ce2_load_gating_prod(g,
+                                g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_fifo_load_gating_prod)
+                g->ops.clock_gating.slcg_fifo_load_gating_prod(g,
+                                g->slcg_enabled);
+        if (g->ops.clock_gating.blcg_fifo_load_gating_prod)
+                g->ops.clock_gating.blcg_fifo_load_gating_prod(g,
+                                g->blcg_enabled);
+
         /* enable pbdma */
         mask = 0;
         for (i = 0; i < proj_host_num_pbdma_v(); ++i)
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c
index a6a51de5..0caef967 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.c
@@ -907,6 +907,13 @@ static int gk20a_pm_finalize_poweron(struct device *dev)
         gk20a_writel(g, mc_intr_en_0_r(),
                 mc_intr_en_0_inta_hardware_f());
 
+        if (g->ops.clock_gating.slcg_bus_load_gating_prod)
+                g->ops.clock_gating.slcg_bus_load_gating_prod(g,
+                                g->slcg_enabled);
+        if (g->ops.clock_gating.blcg_bus_load_gating_prod)
+                g->ops.clock_gating.blcg_bus_load_gating_prod(g,
+                                g->blcg_enabled);
+
         if (!tegra_platform_is_silicon())
                 gk20a_writel(g, bus_intr_en_0_r(), 0x0);
         else
@@ -914,6 +921,7 @@ static int gk20a_pm_finalize_poweron(struct device *dev)
                                 bus_intr_en_0_pri_squash_m() |
                                 bus_intr_en_0_pri_fecserr_m() |
                                 bus_intr_en_0_pri_timeout_m());
+
         gk20a_reset_priv_ring(g);
 
         gk20a_detect_chip(g);
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index fc97fcb9..b6d73343 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -150,11 +150,28 @@ struct gpu_ops {
                 void (*init_kind_attr)(struct gk20a *g);
         } fb;
         struct {
+                void (*slcg_bus_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*slcg_ce2_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*slcg_chiplet_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*slcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*slcg_fb_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*slcg_fifo_load_gating_prod)(struct gk20a *g, bool prod);
                 void (*slcg_gr_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*slcg_ltc_load_gating_prod)(struct gk20a *g, bool prod);
                 void (*slcg_perf_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*slcg_priring_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*slcg_pmu_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*slcg_therm_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*slcg_xbar_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*blcg_bus_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*blcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*blcg_fb_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*blcg_fifo_load_gating_prod)(struct gk20a *g, bool prod);
                 void (*blcg_gr_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*blcg_ltc_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*blcg_pwr_csb_load_gating_prod)(struct gk20a *g, bool prod);
+                void (*blcg_pmu_load_gating_prod)(struct gk20a *g, bool prod);
                 void (*pg_gr_load_gating_prod)(struct gk20a *g, bool prod);
-                void (*slcg_therm_load_gating_prod)(struct gk20a *g, bool prod);
         } clock_gating;
         struct {
                 void (*bind_channel)(struct channel_gk20a *ch_gk20a);
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_gating_reglist.c b/drivers/gpu/nvgpu/gk20a/gk20a_gating_reglist.c
index c6478a5e..0e3b0cb3 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a_gating_reglist.c
+++ b/drivers/gpu/nvgpu/gk20a/gk20a_gating_reglist.c
@@ -311,6 +311,11 @@ void gr_gk20a_slcg_gr_load_gating_prod(struct gk20a *g,
         }
 }
 
+void ltc_gk20a_slcg_ltc_load_gating_prod(struct gk20a *g,
+        bool prod)
+{
+}
+
 void gr_gk20a_slcg_perf_load_gating_prod(struct gk20a *g,
         bool prod)
 {
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_gating_reglist.h b/drivers/gpu/nvgpu/gk20a/gk20a_gating_reglist.h
index 40a6c545..b2a02984 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a_gating_reglist.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a_gating_reglist.h
@@ -1,7 +1,7 @@
 /*
  * drivers/video/tegra/host/gk20a/gk20a_gating_reglist.h
  *
- * Copyright (c) 2012, NVIDIA Corporation.
+ * Copyright (c) 2012-2014, NVIDIA Corporation.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -27,6 +27,9 @@ void gr_gk20a_slcg_gr_load_gating_prod(struct gk20a *g,
 void gr_gk20a_slcg_perf_load_gating_prod(struct gk20a *g,
         bool prod);
 
+void ltc_gk20a_slcg_ltc_load_gating_prod(struct gk20a *g,
+        bool prod);
+
 void gr_gk20a_blcg_gr_load_gating_prod(struct gk20a *g,
         bool prod);
 
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_sysfs.c b/drivers/gpu/nvgpu/gk20a/gk20a_sysfs.c
index fceed5e9..687147ed 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a_sysfs.c
+++ b/drivers/gpu/nvgpu/gk20a/gk20a_sysfs.c
@@ -94,7 +94,19 @@ static ssize_t blcg_enable_store(struct device *device,
                 g->blcg_enabled = false;
 
         gk20a_busy(g->dev);
+        if (g->ops.clock_gating.blcg_bus_load_gating_prod)
+                g->ops.clock_gating.blcg_bus_load_gating_prod(g, g->blcg_enabled);
+        if (g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod)
+                g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod(g, g->blcg_enabled);
+        if (g->ops.clock_gating.blcg_fb_load_gating_prod)
+                g->ops.clock_gating.blcg_fb_load_gating_prod(g, g->blcg_enabled);
+        if (g->ops.clock_gating.blcg_fifo_load_gating_prod)
+                g->ops.clock_gating.blcg_fifo_load_gating_prod(g, g->blcg_enabled);
         g->ops.clock_gating.blcg_gr_load_gating_prod(g, g->blcg_enabled);
+        if (g->ops.clock_gating.blcg_ltc_load_gating_prod)
+                g->ops.clock_gating.blcg_ltc_load_gating_prod(g, g->blcg_enabled);
+        if (g->ops.clock_gating.blcg_pmu_load_gating_prod)
+                g->ops.clock_gating.blcg_pmu_load_gating_prod(g, g->blcg_enabled);
         gk20a_idle(g->dev);
 
         dev_info(device, "BLCG is %s.\n", g->blcg_enabled ? "enabled" :
@@ -136,8 +148,28 @@ static ssize_t slcg_enable_store(struct device *device,
          * it is added to init, we should add it here too.
          */
         gk20a_busy(g->dev);
+        if (g->ops.clock_gating.slcg_bus_load_gating_prod)
+                g->ops.clock_gating.slcg_bus_load_gating_prod(g, g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_ce2_load_gating_prod)
+                g->ops.clock_gating.slcg_ce2_load_gating_prod(g, g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_chiplet_load_gating_prod)
+                g->ops.clock_gating.slcg_chiplet_load_gating_prod(g, g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod)
+                g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod(g, g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_fb_load_gating_prod)
+                g->ops.clock_gating.slcg_fb_load_gating_prod(g, g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_fifo_load_gating_prod)
+                g->ops.clock_gating.slcg_fifo_load_gating_prod(g, g->slcg_enabled);
         g->ops.clock_gating.slcg_gr_load_gating_prod(g, g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_ltc_load_gating_prod)
+                g->ops.clock_gating.slcg_ltc_load_gating_prod(g, g->slcg_enabled);
         g->ops.clock_gating.slcg_perf_load_gating_prod(g, g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_priring_load_gating_prod)
+                g->ops.clock_gating.slcg_priring_load_gating_prod(g, g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_pmu_load_gating_prod)
+                g->ops.clock_gating.slcg_pmu_load_gating_prod(g, g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_xbar_load_gating_prod)
+                g->ops.clock_gating.slcg_xbar_load_gating_prod(g, g->slcg_enabled);
         gk20a_idle(g->dev);
 
         dev_info(device, "SLCG is %s.\n", g->slcg_enabled ? "enabled" :
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
index cbad1292..661a2ca3 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
@@ -4246,10 +4246,6 @@ static int gk20a_init_gr_setup_hw(struct gk20a *g)
         if (g->ops.gr.init_gpc_mmu)
                 g->ops.gr.init_gpc_mmu(g);
 
-        /* slcg prod values */
-        g->ops.clock_gating.slcg_gr_load_gating_prod(g, g->slcg_enabled);
-        g->ops.clock_gating.slcg_perf_load_gating_prod(g, g->slcg_enabled);
-
         /* init mmu debug buffer */
         addr = NV_MC_SMMU_VADDR_TRANSLATE(gr->mmu_wr_mem.iova);
         addr_lo = u64_lo32(addr);
@@ -4281,9 +4277,6 @@ static int gk20a_init_gr_setup_hw(struct gk20a *g)
 
         gr_gk20a_zcull_init_hw(g, gr);
 
-        g->ops.clock_gating.blcg_gr_load_gating_prod(g, g->blcg_enabled);
-        g->ops.clock_gating.pg_gr_load_gating_prod(g, true);
-
         if (g->elcg_enabled) {
                 gr_gk20a_init_elcg_mode(g, ELCG_AUTO, ENGINE_GR_GK20A);
                 gr_gk20a_init_elcg_mode(g, ELCG_AUTO, ENGINE_CE2_GK20A);
@@ -4426,6 +4419,19 @@ static int gk20a_init_gr_prepare(struct gk20a *g)
                         | mc_enable_blg_enabled_f()
                         | mc_enable_perfmon_enabled_f());
 
+        /* slcg prod values */
+        g->ops.clock_gating.slcg_gr_load_gating_prod(g, g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod)
+                g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod(g,
+                                g->slcg_enabled);
+        g->ops.clock_gating.slcg_perf_load_gating_prod(g, g->slcg_enabled);
+
+        g->ops.clock_gating.blcg_gr_load_gating_prod(g, g->blcg_enabled);
+        if (g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod)
+                g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod(g,
+                                g->blcg_enabled);
+        g->ops.clock_gating.pg_gr_load_gating_prod(g, true);
+
         /* enable fifo access */
         gk20a_writel(g, gr_gpfifo_ctl_r(),
                 gr_gpfifo_ctl_access_enabled_f() |
diff --git a/drivers/gpu/nvgpu/gk20a/hal_gk20a.c b/drivers/gpu/nvgpu/gk20a/hal_gk20a.c
index 218491ea..578b77bf 100644
--- a/drivers/gpu/nvgpu/gk20a/hal_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/hal_gk20a.c
@@ -33,6 +33,8 @@ struct gpu_ops gk20a_ops = {
                         gr_gk20a_slcg_gr_load_gating_prod,
                 .slcg_perf_load_gating_prod =
                         gr_gk20a_slcg_perf_load_gating_prod,
+                .slcg_ltc_load_gating_prod =
+                        ltc_gk20a_slcg_ltc_load_gating_prod,
                 .blcg_gr_load_gating_prod =
                         gr_gk20a_blcg_gr_load_gating_prod,
                 .pg_gr_load_gating_prod =
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
index 3feb675b..173776ff 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
@@ -251,6 +251,19 @@ static int gk20a_init_mm_reset_enable_hw(struct gk20a *g)
         if (g->ops.fb.reset)
                 g->ops.fb.reset(g);
 
+        if (g->ops.clock_gating.slcg_fb_load_gating_prod)
+                g->ops.clock_gating.slcg_fb_load_gating_prod(g,
+                                g->slcg_enabled);
+        if (g->ops.clock_gating.slcg_ltc_load_gating_prod)
+                g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
+                                g->slcg_enabled);
+        if (g->ops.clock_gating.blcg_fb_load_gating_prod)
+                g->ops.clock_gating.blcg_fb_load_gating_prod(g,
+                                g->blcg_enabled);
+        if (g->ops.clock_gating.blcg_ltc_load_gating_prod)
+                g->ops.clock_gating.blcg_ltc_load_gating_prod(g,
+                                g->blcg_enabled);
+
         if (g->ops.fb.init_fs_state)
                 g->ops.fb.init_fs_state(g);
 
diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
index eb62caaf..f77ad10b 100644
--- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
@@ -1124,6 +1124,13 @@ static int pmu_enable_hw(struct pmu_gk20a *pmu, bool enable)
                 int retries = GR_IDLE_CHECK_MAX / GR_IDLE_CHECK_DEFAULT;
                 gk20a_enable(g, mc_enable_pwr_enabled_f());
 
+                if (g->ops.clock_gating.slcg_pmu_load_gating_prod)
+                        g->ops.clock_gating.slcg_pmu_load_gating_prod(g,
+                                        g->slcg_enabled);
+                if (g->ops.clock_gating.blcg_pmu_load_gating_prod)
+                        g->ops.clock_gating.blcg_pmu_load_gating_prod(g,
+                                        g->blcg_enabled);
+
                 do {
                         u32 w = gk20a_readl(g, pwr_falcon_dmactl_r()) &
                                 (pwr_falcon_dmactl_dmem_scrubbing_m() |
diff --git a/drivers/gpu/nvgpu/gk20a/priv_ring_gk20a.c b/drivers/gpu/nvgpu/gk20a/priv_ring_gk20a.c
index aea1a80b..9d82a986 100644
--- a/drivers/gpu/nvgpu/gk20a/priv_ring_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/priv_ring_gk20a.c
@@ -39,6 +39,10 @@ void gk20a_reset_priv_ring(struct gk20a *g)
 
         gk20a_reset(g, mc_enable_priv_ring_enabled_f());
 
+        if (g->ops.clock_gating.slcg_priring_load_gating_prod)
+                g->ops.clock_gating.slcg_priring_load_gating_prod(g,
+                                g->slcg_enabled);
+
         gk20a_writel(g,pri_ringmaster_command_r(),
                         0x4);