gpu: nvgpu: gv11b: enable RMW for gpu atomics

Separate HAL added in gv11b and gv100 for
init_gpc_mmu function.
In gv11b HAL, RMW is enabled for gpu atomics
as default.
In gv100 HAL, GPC atomic capability mode will
get set based on the FB MMU capability.
If GPU is connected through NVLINK then mmu
will be set to RMW mode, else it will be in
L2 mode.

Bug 200390336

Change-Id: I224934f83d1762ec864ef8da7265dd01d86893a0
Signed-off-by: Ashish Srivastava <assrivastava@nvidia.com>
Signed-off-by: Seema Khowala <seemaj@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1735137
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

6 files changed, 62 insertions, 1 deletions

diff --git a/drivers/gpu/nvgpu/gv100/gr_gv100.c b/drivers/gpu/nvgpu/gv100/gr_gv100.c
index 98e61eb0..2180fa1c 100644
--- a/drivers/gpu/nvgpu/gv100/gr_gv100.c
+++ b/drivers/gpu/nvgpu/gv100/gr_gv100.c
@@ -34,6 +34,7 @@
 #include "gv11b/subctx_gv11b.h"
 
 #include <nvgpu/hw/gv100/hw_gr_gv100.h>
+#include <nvgpu/hw/gv100/hw_fb_gv100.h>
 #include <nvgpu/hw/gv100/hw_proj_gv100.h>
 #include <nvgpu/hw/gv100/hw_fuse_gv100.h>
 
@@ -280,6 +281,43 @@ exit_build_table:
         return err;
 }
 
+void gr_gv100_init_gpc_mmu(struct gk20a *g)
+{
+        u32 temp;
+
+        nvgpu_log_info(g, "initialize gpc mmu");
+
+        if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) {
+                /* Bypass MMU check for non-secure boot. For
+                 * secure-boot,this register write has no-effect */
+                gk20a_writel(g, fb_priv_mmu_phy_secure_r(), 0xffffffff);
+        }
+        temp = gk20a_readl(g, fb_mmu_ctrl_r());
+        temp &= gr_gpcs_pri_mmu_ctrl_vm_pg_size_m() |
+                gr_gpcs_pri_mmu_ctrl_use_pdb_big_page_size_m() |
+                gr_gpcs_pri_mmu_ctrl_vol_fault_m() |
+                gr_gpcs_pri_mmu_ctrl_comp_fault_m() |
+                gr_gpcs_pri_mmu_ctrl_miss_gran_m() |
+                gr_gpcs_pri_mmu_ctrl_cache_mode_m() |
+                gr_gpcs_pri_mmu_ctrl_mmu_aperture_m() |
+                gr_gpcs_pri_mmu_ctrl_mmu_vol_m() |
+                gr_gpcs_pri_mmu_ctrl_mmu_disable_m()|
+                gr_gpcs_pri_mmu_ctrl_atomic_capability_mode_m();
+
+        gk20a_writel(g, gr_gpcs_pri_mmu_ctrl_r(), temp);
+        nvgpu_log_info(g, "mmu_ctrl_r = 0x%08x", temp);
+
+        gk20a_writel(g, gr_gpcs_pri_mmu_pm_unit_mask_r(), 0);
+        gk20a_writel(g, gr_gpcs_pri_mmu_pm_req_mask_r(), 0);
+
+        gk20a_writel(g, gr_gpcs_pri_mmu_debug_ctrl_r(),
+                        gk20a_readl(g, fb_mmu_debug_ctrl_r()));
+        gk20a_writel(g, gr_gpcs_pri_mmu_debug_wr_r(),
+                        gk20a_readl(g, fb_mmu_debug_wr_r()));
+        gk20a_writel(g, gr_gpcs_pri_mmu_debug_rd_r(),
+                        gk20a_readl(g, fb_mmu_debug_rd_r()));
+}
+
 u32 gr_gv100_get_patch_slots(struct gk20a *g)
 {
         struct gr_gk20a *gr = &g->gr;
diff --git a/drivers/gpu/nvgpu/gv100/gr_gv100.h b/drivers/gpu/nvgpu/gv100/gr_gv100.h
index ccc73e28..821659aa 100644
--- a/drivers/gpu/nvgpu/gv100/gr_gv100.h
+++ b/drivers/gpu/nvgpu/gv100/gr_gv100.h
@@ -46,4 +46,5 @@ int gr_gv100_add_ctxsw_reg_perf_pma(struct ctxsw_buf_offset_map_entry *map,
 void gr_gv100_split_fbpa_broadcast_addr(struct gk20a *g, u32 addr,
         u32 num_fbpas,
         u32 *priv_addr_table, u32 *t);
+void gr_gv100_init_gpc_mmu(struct gk20a *g);
 #endif
diff --git a/drivers/gpu/nvgpu/gv100/hal_gv100.c b/drivers/gpu/nvgpu/gv100/hal_gv100.c
index ae6c3d22..852fc1b7 100644
--- a/drivers/gpu/nvgpu/gv100/hal_gv100.c
+++ b/drivers/gpu/nvgpu/gv100/hal_gv100.c
@@ -290,7 +290,7 @@ static const struct gpu_ops gv100_ops = {
         },
         .gr = {
                 .get_patch_slots = gr_gv100_get_patch_slots,
-                .init_gpc_mmu = gr_gv11b_init_gpc_mmu,
+                .init_gpc_mmu = gr_gv100_init_gpc_mmu,
                 .bundle_cb_defaults = gr_gv100_bundle_cb_defaults,
                 .cb_size_default = gr_gv100_cb_size_default,
                 .calc_global_ctx_buffer_size =
diff --git a/drivers/gpu/nvgpu/gv11b/gr_gv11b.c b/drivers/gpu/nvgpu/gv11b/gr_gv11b.c
index aed45ceb..1336557a 100644
--- a/drivers/gpu/nvgpu/gv11b/gr_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/gr_gv11b.c
@@ -4263,7 +4263,13 @@ void gr_gv11b_init_gpc_mmu(struct gk20a *g)
                 gr_gpcs_pri_mmu_ctrl_mmu_aperture_m() |
                 gr_gpcs_pri_mmu_ctrl_mmu_vol_m() |
                 gr_gpcs_pri_mmu_ctrl_mmu_disable_m();
+
+        temp = set_field(temp, gr_gpcs_pri_mmu_ctrl_atomic_capability_mode_m(),
+                           gr_gpcs_pri_mmu_ctrl_atomic_capability_mode_rmw_f());
         gk20a_writel(g, gr_gpcs_pri_mmu_ctrl_r(), temp);
+        nvgpu_log_info(g, "mmu_ctrl_r = 0x%08x, atomic_capability_mode_rmw",
+                        temp);
+
         gk20a_writel(g, gr_gpcs_pri_mmu_pm_unit_mask_r(), 0);
         gk20a_writel(g, gr_gpcs_pri_mmu_pm_req_mask_r(), 0);
 
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_gr_gv100.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_gr_gv100.h
index 0f83d6ba..d2a73286 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_gr_gv100.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_gr_gv100.h
@@ -4028,6 +4028,14 @@ static inline u32 gr_gpcs_pri_mmu_ctrl_mmu_disable_m(void)
 {
         return 0x1U << 31U;
 }
+static inline u32 gr_gpcs_pri_mmu_ctrl_atomic_capability_mode_m(void)
+{
+        return 0x3U << 24U;
+}
+static inline u32 gr_gpcs_pri_mmu_ctrl_atomic_capability_mode_rmw_f(void)
+{
+        return 0x2000000U;
+}
 static inline u32 gr_gpcs_pri_mmu_pm_unit_mask_r(void)
 {
         return 0x00418890U;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gv11b/hw_gr_gv11b.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gv11b/hw_gr_gv11b.h
index 5de691a2..90994a53 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gv11b/hw_gr_gv11b.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gv11b/hw_gr_gv11b.h
@@ -4936,6 +4936,14 @@ static inline u32 gr_gpcs_pri_mmu_ctrl_mmu_disable_m(void)
 {
         return 0x1U << 31U;
 }
+static inline u32 gr_gpcs_pri_mmu_ctrl_atomic_capability_mode_m(void)
+{
+        return 0x3U << 24U;
+}
+static inline u32 gr_gpcs_pri_mmu_ctrl_atomic_capability_mode_rmw_f(void)
+{
+        return 0x2000000U;
+}
 static inline u32 gr_gpcs_pri_mmu_pm_unit_mask_r(void)
 {
         return 0x00418890U;