10 files changed, 166 insertions, 35 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
index cd3ab0c2..f86a7377 100644
--- a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
@@ -1169,8 +1169,8 @@ static int dbg_set_powergate(struct dbg_session_gk20a *dbg_s, u32  powermode)
                                 return -EPERM;
 
                         /*do elpg disable before clock gating */
-                        if (support_gk20a_pmu(g->dev))
-                                gk20a_pmu_disable_elpg(g);
+                        gk20a_pmu_pg_global_enable(g, false);
+
                         if (g->ops.clock_gating.slcg_gr_load_gating_prod)
                                 g->ops.clock_gating.slcg_gr_load_gating_prod(g,
                                                 false);
@@ -1216,8 +1216,7 @@ static int dbg_set_powergate(struct dbg_session_gk20a *dbg_s, u32  powermode)
                                 g->ops.clock_gating.slcg_gr_load_gating_prod(g,
                                                 g->slcg_enabled);
 
-                        if (support_gk20a_pmu(g->dev))
-                                gk20a_pmu_enable_elpg(g);
+                        gk20a_pmu_pg_global_enable(g, true);
 
                         gk20a_dbg(gpu_dbg_gpu_dbg | gpu_dbg_fn, "module idle");
                         gk20a_idle(dbg_s->dev);
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index a1a8bf36..782469df 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -610,6 +610,9 @@ struct gpu_ops {
                 u32 (*pmu_pg_supported_engines_list)(struct gk20a *g);
                 u32 (*pmu_pg_engines_feature_list)(struct gk20a *g,
                         u32 pg_engine_id);
+                int (*pmu_lpwr_enable_pg)(struct gk20a *g, bool pstate_lock);
+                int (*pmu_lpwr_disable_pg)(struct gk20a *g, bool pstate_lock);
+                u32 (*pmu_pg_param_post_init)(struct gk20a *g);
                 int (*send_lrf_tex_ltc_dram_overide_en_dis_cmd)
                         (struct gk20a *g, u32 mask);
                 void (*dump_secure_fuses)(struct gk20a *g);
@@ -847,6 +850,7 @@ struct gk20a {
         bool elcg_enabled;
         bool elpg_enabled;
         bool aelpg_enabled;
+        bool mscg_enabled;
         bool forced_idle;
         bool forced_reset;
         bool allow_all;
diff --git a/drivers/gpu/nvgpu/gk20a/platform_gk20a.h b/drivers/gpu/nvgpu/gk20a/platform_gk20a.h
index 3d5cd1b2..2b17d32a 100644
--- a/drivers/gpu/nvgpu/gk20a/platform_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/platform_gk20a.h
@@ -93,6 +93,9 @@ struct gk20a_platform {
         /* Adaptative ELPG: true = enable flase = disable */
         bool enable_aelpg;
 
+        /* Memory System Clock Gating: true = enable flase = disable*/
+        bool enable_mscg;
+
         /* Timeout for per-channel watchdog (in mS) */
         u32 ch_wdt_timeout_ms;
 
diff --git a/drivers/gpu/nvgpu/gk20a/pmu_api.h b/drivers/gpu/nvgpu/gk20a/pmu_api.h
index 2fdd1333..def7bbea 100644
--- a/drivers/gpu/nvgpu/gk20a/pmu_api.h
+++ b/drivers/gpu/nvgpu/gk20a/pmu_api.h
@@ -526,17 +526,77 @@ enum {
 };
 
 #define PMU_PG_PARAM_CMD_GR_INIT_PARAM  0x0
+#define PMU_PG_PARAM_CMD_MS_INIT_PARAM  0x01
+#define PMU_PG_PARAM_CMD_MCLK_CHANGE  0x04
+#define PMU_PG_PARAM_CMD_POST_INIT  0x06
 
 #define PMU_PG_FEATURE_GR_SDIV_SLOWDOWN_ENABLED (1 << 0)
 #define PMU_PG_FEATURE_GR_POWER_GATING_ENABLED  (1 << 2)
 #define PMU_PG_FEATURE_GR_RPPG_ENABLED          (1 << 3)
 
+#define NVGPU_PMU_GR_FEATURE_MASK_RPPG  (1 << 3)
+#define NVGPU_PMU_GR_FEATURE_MASK_ALL   \
+        (       \
+                NVGPU_PMU_GR_FEATURE_MASK_RPPG   \
+        )
+
+#define NVGPU_PMU_MS_FEATURE_MASK_CLOCK_GATING  (1 << 0)
+#define NVGPU_PMU_MS_FEATURE_MASK_SW_ASR        (1 << 1)
+#define NVGPU_PMU_MS_FEATURE_MASK_RPPG          (1 << 8)
+#define NVGPU_PMU_MS_FEATURE_MASK_FB_TRAINING   (1 << 5)
+
+#define NVGPU_PMU_MS_FEATURE_MASK_ALL   \
+        (       \
+                NVGPU_PMU_MS_FEATURE_MASK_CLOCK_GATING  |\
+                NVGPU_PMU_MS_FEATURE_MASK_SW_ASR        |\
+                NVGPU_PMU_MS_FEATURE_MASK_RPPG          |\
+                NVGPU_PMU_MS_FEATURE_MASK_FB_TRAINING   \
+        )
+
+#define PG_REQUEST_TYPE_GLOBAL 0x0
+#define PG_REQUEST_TYPE_PSTATE 0x1
+
 struct pmu_pg_cmd_gr_init_param {
         u8 cmd_type;
         u16 sub_cmd_id;
         u8 featuremask;
 };
 
+struct pmu_pg_cmd_ms_init_param {
+        u8 cmd_type;
+        u16 cmd_id;
+        u8 psi;
+        u8 idle_flipped_test_enabled;
+        u16 psiSettleTimeUs;
+        u8 rsvd[2];
+        u32 support_mask;
+        u32 abort_timeout_us;
+};
+
+struct pmu_pg_cmd_mclk_change {
+        u8 cmd_type;
+        u16 cmd_id;
+        u8 rsvd;
+        u32 data;
+};
+
+#define PG_VOLT_RAIL_IDX_MAX 2
+
+struct pmu_pg_volt_rail {
+        u8    volt_rail_idx;
+        u8    sleep_volt_dev_idx;
+        u8    sleep_vfe_idx;
+        u32   sleep_voltage_uv;
+        u32   therm_vid0_cache;
+        u32   therm_vid1_cache;
+};
+
+struct pmu_pg_cmd_post_init_param {
+        u8 cmd_type;
+        u16 cmd_id;
+        struct pmu_pg_volt_rail pg_volt_rail[PG_VOLT_RAIL_IDX_MAX];
+};
+
 struct pmu_pg_cmd_stat {
         u8 cmd_type;
         u8 engine_id;
@@ -553,6 +613,9 @@ struct pmu_pg_cmd {
                 struct pmu_pg_cmd_eng_buf_load_v2 eng_buf_load_v2;
                 struct pmu_pg_cmd_stat stat;
                 struct pmu_pg_cmd_gr_init_param gr_init_param;
+                struct pmu_pg_cmd_ms_init_param ms_init_param;
+                struct pmu_pg_cmd_mclk_change mclk_change;
+                struct pmu_pg_cmd_post_init_param post_init;
                 /* TBD: other pg commands */
                 union pmu_ap_cmd ap_cmd;
                 struct nv_pmu_rppg_cmd rppg_cmd;
diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
index c70f9876..2b847008 100644
--- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
@@ -1364,6 +1364,7 @@ int gk20a_init_pmu(struct pmu_gk20a *pmu)
         struct pmu_v *pv = &g->ops.pmu_ver;
 
         mutex_init(&pmu->elpg_mutex);
+        mutex_init(&pmu->pg_mutex);
         mutex_init(&pmu->isr_mutex);
         mutex_init(&pmu->pmu_copy_lock);
         mutex_init(&pmu->pmu_seq_lock);
@@ -3298,6 +3299,9 @@ void gk20a_init_pmu_ops(struct gpu_ops *gops)
         gops->pmu.pmu_pg_init_param = NULL;
         gops->pmu.pmu_pg_supported_engines_list = gk20a_pmu_pg_engines_list;
         gops->pmu.pmu_pg_engines_feature_list = gk20a_pmu_pg_feature_list;
+        gops->pmu.pmu_lpwr_enable_pg = NULL;
+        gops->pmu.pmu_lpwr_disable_pg = NULL;
+        gops->pmu.pmu_pg_param_post_init = NULL;
         gops->pmu.send_lrf_tex_ltc_dram_overide_en_dis_cmd = NULL;
         gops->pmu.dump_secure_fuses = NULL;
         gops->pmu.is_lazy_bootstrap = NULL;
@@ -3378,6 +3382,7 @@ static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg,
                                 PMU_PG_FEATURE_GR_POWER_GATING_ENABLED) {
                                 pmu->initialized = true;
                                 pmu->pmu_state = PMU_STATE_STARTED;
+                                pmu->mscg_stat = PMU_MSCG_DISABLED;
                         } else
                                 schedule_work(&pmu->pg_init);
                 }
@@ -3506,6 +3511,9 @@ static int pmu_init_powergating(struct gk20a *g)
                 }
         }
 
+        if (g->ops.pmu.pmu_pg_param_post_init)
+                g->ops.pmu.pmu_pg_param_post_init(g);
+
         return 0;
 }
 
@@ -4693,44 +4701,62 @@ clean_up:
         return err;
 }
 
-static int gk20a_pmu_enable_elpg_locked(struct gk20a *g)
+int gk20a_pmu_pg_global_enable(struct gk20a *g, u32 enable_pg)
+{
+        u32 status = 0;
+
+        if (enable_pg == true) {
+                if (g->ops.pmu.pmu_pg_engines_feature_list &&
+                        g->ops.pmu.pmu_pg_engines_feature_list(g,
+                        PMU_PG_ELPG_ENGINE_ID_GRAPHICS) !=
+                        PMU_PG_FEATURE_GR_POWER_GATING_ENABLED) {
+                        if (g->ops.pmu.pmu_lpwr_enable_pg)
+                                status = g->ops.pmu.pmu_lpwr_enable_pg(g,
+                                                true);
+                } else if (support_gk20a_pmu(g->dev))
+                        status = gk20a_pmu_enable_elpg(g);
+        } else if (enable_pg == false) {
+                if (g->ops.pmu.pmu_pg_engines_feature_list &&
+                        g->ops.pmu.pmu_pg_engines_feature_list(g,
+                        PMU_PG_ELPG_ENGINE_ID_GRAPHICS) !=
+                        PMU_PG_FEATURE_GR_POWER_GATING_ENABLED) {
+                        if (g->ops.pmu.pmu_lpwr_disable_pg)
+                                status = g->ops.pmu.pmu_lpwr_disable_pg(g,
+                                                true);
+                } else if (support_gk20a_pmu(g->dev))
+                        status = gk20a_pmu_disable_elpg(g);
+        }
+
+        return status;
+}
+
+static int gk20a_pmu_enable_elpg_locked(struct gk20a *g, u32 pg_engine_id)
 {
         struct pmu_gk20a *pmu = &g->pmu;
         struct pmu_cmd cmd;
         u32 seq, status;
-        u32 pg_engine_id;
-        u32 pg_engine_id_list = 0;
 
         gk20a_dbg_fn("");
 
-        if (g->ops.pmu.pmu_pg_supported_engines_list)
-                pg_engine_id_list = g->ops.pmu.pmu_pg_supported_engines_list(g);
-        for (pg_engine_id = PMU_PG_ELPG_ENGINE_ID_GRAPHICS;
-                pg_engine_id < PMU_PG_ELPG_ENGINE_ID_INVALID_ENGINE;
-                pg_engine_id++) {
-
-                if (BIT(pg_engine_id) & pg_engine_id_list) {
-                        memset(&cmd, 0, sizeof(struct pmu_cmd));
-                        cmd.hdr.unit_id = PMU_UNIT_PG;
-                        cmd.hdr.size = PMU_CMD_HDR_SIZE +
-                                sizeof(struct pmu_pg_cmd_elpg_cmd);
-                        cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
-                        cmd.cmd.pg.elpg_cmd.engine_id = pg_engine_id;
-                        cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_ALLOW;
+        memset(&cmd, 0, sizeof(struct pmu_cmd));
+        cmd.hdr.unit_id = PMU_UNIT_PG;
+        cmd.hdr.size = PMU_CMD_HDR_SIZE +
+                sizeof(struct pmu_pg_cmd_elpg_cmd);
+        cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
+        cmd.cmd.pg.elpg_cmd.engine_id = pg_engine_id;
+        cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_ALLOW;
 
-                   /* no need to wait ack for ELPG enable but set
-                    * pending to sync with follow up ELPG disable
-                    */
-                        if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS)
-                                pmu->elpg_stat = PMU_ELPG_STAT_ON_PENDING;
+   /* no need to wait ack for ELPG enable but set
+        * pending to sync with follow up ELPG disable
+        */
+        if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS)
+                pmu->elpg_stat = PMU_ELPG_STAT_ON_PENDING;
 
-                        gk20a_dbg_pmu("cmd post PMU_PG_ELPG_CMD_ALLOW");
-                        status = gk20a_pmu_cmd_post(g, &cmd, NULL, NULL,
-                                PMU_COMMAND_QUEUE_HPQ, pmu_handle_pg_elpg_msg,
-                                pmu, &seq, ~0);
-                        WARN_ON(status != 0);
-                }
-        }
+        gk20a_dbg_pmu("cmd post PMU_PG_ELPG_CMD_ALLOW");
+        status = gk20a_pmu_cmd_post(g, &cmd, NULL, NULL,
+                PMU_COMMAND_QUEUE_HPQ, pmu_handle_pg_elpg_msg,
+                pmu, &seq, ~0);
+        WARN_ON(status != 0);
 
         gk20a_dbg_fn("done");
         return 0;
@@ -4740,12 +4766,13 @@ int gk20a_pmu_enable_elpg(struct gk20a *g)
 {
         struct pmu_gk20a *pmu = &g->pmu;
         struct gr_gk20a *gr = &g->gr;
+        u32 pg_engine_id;
+        u32 pg_engine_id_list = 0;
 
         int ret = 0;
 
         gk20a_dbg_fn("");
 
-
         if (!support_gk20a_pmu(g->dev))
                 return ret;
 
@@ -4772,7 +4799,20 @@ int gk20a_pmu_enable_elpg(struct gk20a *g)
         if (pmu->elpg_stat != PMU_ELPG_STAT_OFF)
                 goto exit_unlock;
 
-        ret = gk20a_pmu_enable_elpg_locked(g);
+        if (g->ops.pmu.pmu_pg_supported_engines_list)
+                pg_engine_id_list = g->ops.pmu.pmu_pg_supported_engines_list(g);
+
+        for (pg_engine_id = PMU_PG_ELPG_ENGINE_ID_GRAPHICS;
+                pg_engine_id < PMU_PG_ELPG_ENGINE_ID_INVALID_ENGINE;
+                pg_engine_id++) {
+
+                if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS &&
+                        pmu->mscg_stat == PMU_MSCG_DISABLED)
+                        continue;
+
+                if (BIT(pg_engine_id) & pg_engine_id_list)
+                        ret = gk20a_pmu_enable_elpg_locked(g, pg_engine_id);
+        }
 
 exit_unlock:
         mutex_unlock(&pmu->elpg_mutex);
@@ -4845,6 +4885,10 @@ int gk20a_pmu_disable_elpg(struct gk20a *g)
                 pg_engine_id < PMU_PG_ELPG_ENGINE_ID_INVALID_ENGINE;
                 pg_engine_id++) {
 
+                if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS &&
+                        pmu->mscg_stat == PMU_MSCG_DISABLED)
+                        continue;
+
                 if (BIT(pg_engine_id) & pg_engine_id_list) {
                         memset(&cmd, 0, sizeof(struct pmu_cmd));
                         cmd.hdr.unit_id = PMU_UNIT_PG;
diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h
index 78652bcb..56300dc8 100644
--- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h
@@ -629,6 +629,9 @@ struct pmu_pg_stats {
 #define PMU_ELPG_STAT_OFF_ON_PENDING    4   /* elpg is off, caller has requested on, but ALLOW
                                                cmd hasn't been sent due to ENABLE_ALLOW delay */
 
+#define PMU_MSCG_DISABLED 0
+#define PMU_MSCG_ENABLED 1
+
 /* Falcon Register index */
 #define PMU_FALCON_REG_R0               (0)
 #define PMU_FALCON_REG_R1               (1)
@@ -716,10 +719,13 @@ struct pmu_gk20a {
 
         u32 elpg_stat;
 
+        u32 mscg_stat;
+
         int pmu_state;
 
 #define PMU_ELPG_ENABLE_ALLOW_DELAY_MSEC        1 /* msec */
         struct work_struct pg_init;
+        struct mutex pg_mutex; /* protect pg-RPPG/MSCG enable/disable */
         struct mutex elpg_mutex; /* protect elpg enable/disable */
         int elpg_refcnt; /* disable -1, enable +1, <=0 elpg disabled, > 0 elpg enabled */
 
@@ -774,6 +780,7 @@ int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd, struct pmu_msg *msg
 
 int gk20a_pmu_enable_elpg(struct gk20a *g);
 int gk20a_pmu_disable_elpg(struct gk20a *g);
+int gk20a_pmu_pg_global_enable(struct gk20a *g, u32 enable_pg);
 
 u32 gk20a_pmu_pg_engines_list(struct gk20a *g);
 u32 gk20a_pmu_pg_feature_list(struct gk20a *g, u32 pg_engine_id);
diff --git a/drivers/gpu/nvgpu/gm206/bios_gm206.h b/drivers/gpu/nvgpu/gm206/bios_gm206.h
index 1d813df5..6fe19fb0 100644
--- a/drivers/gpu/nvgpu/gm206/bios_gm206.h
+++ b/drivers/gpu/nvgpu/gm206/bios_gm206.h
@@ -44,6 +44,9 @@ enum {
         VOLTAGE_RAIL_TABLE = 26,
         VOLTAGE_DEVICE_TABLE,
         VOLTAGE_POLICY_TABLE,
+        LOWPOWER_TABLE,
+        LOWPOWER_GR_TABLE = 32,
+        LOWPOWER_MS_TABLE = 33,
 };
 
 enum {
diff --git a/drivers/gpu/nvgpu/gm206/pmu_gm206.c b/drivers/gpu/nvgpu/gm206/pmu_gm206.c
index d109be97..1aff6ea6 100644
--- a/drivers/gpu/nvgpu/gm206/pmu_gm206.c
+++ b/drivers/gpu/nvgpu/gm206/pmu_gm206.c
@@ -159,6 +159,9 @@ void gm206_init_pmu_ops(struct gpu_ops *gops)
         gops->pmu.pmu_pg_init_param = NULL;
         gops->pmu.pmu_pg_supported_engines_list = NULL;
         gops->pmu.pmu_pg_engines_feature_list = NULL;
+        gops->pmu.pmu_lpwr_enable_pg = NULL;
+        gops->pmu.pmu_lpwr_disable_pg = NULL;
+        gops->pmu.pmu_pg_param_post_init = NULL;
         gops->pmu.send_lrf_tex_ltc_dram_overide_en_dis_cmd = NULL;
         gops->pmu.dump_secure_fuses = NULL;
         gops->pmu.reset = gk20a_pmu_reset;
diff --git a/drivers/gpu/nvgpu/gm20b/pmu_gm20b.c b/drivers/gpu/nvgpu/gm20b/pmu_gm20b.c
index 517c92a1..868e824a 100644
--- a/drivers/gpu/nvgpu/gm20b/pmu_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/pmu_gm20b.c
@@ -288,6 +288,9 @@ void gm20b_init_pmu_ops(struct gpu_ops *gops)
         gops->pmu.pmu_pg_init_param = NULL;
         gops->pmu.pmu_pg_supported_engines_list = gk20a_pmu_pg_engines_list;
         gops->pmu.pmu_pg_engines_feature_list = gk20a_pmu_pg_feature_list;
+        gops->pmu.pmu_lpwr_enable_pg = NULL;
+        gops->pmu.pmu_lpwr_disable_pg = NULL;
+        gops->pmu.pmu_pg_param_post_init = NULL;
         gops->pmu.send_lrf_tex_ltc_dram_overide_en_dis_cmd = NULL;
         gops->pmu.dump_secure_fuses = pmu_dump_security_fuses_gm20b;
         gops->pmu.reset = gk20a_pmu_reset;
diff --git a/drivers/gpu/nvgpu/nvgpu_common.c b/drivers/gpu/nvgpu/nvgpu_common.c
index 4f0e883f..179464d8 100644
--- a/drivers/gpu/nvgpu/nvgpu_common.c
+++ b/drivers/gpu/nvgpu/nvgpu_common.c
@@ -89,6 +89,8 @@ static void nvgpu_init_pm_vars(struct gk20a *g)
                 tegra_platform_is_silicon() ? platform->enable_elpg : false;
         g->aelpg_enabled =
                 tegra_platform_is_silicon() ? platform->enable_aelpg : false;
+        g->mscg_enabled =
+                tegra_platform_is_silicon() ? platform->enable_mscg : false;
 
         /* set default values to aelpg parameters */
         g->pmu.aelpg_param[0] = APCTRL_SAMPLING_PERIOD_PG_DEFAULT_US;