Revert "Revert "gpu: nvgpu: Get coherency on gv100 + NVLINK working""

This reverts commit 5a35a95654d561fce09a3b9abf6b82bb7a29d74b.

JIRA EVLR-2333

Change-Id: I923c32496c343d39d34f6d406c38a9f6ce7dc6e0
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1667167
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

7 files changed, 55 insertions, 35 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/bus_gk20a.c b/drivers/gpu/nvgpu/gk20a/bus_gk20a.c
index 7f0cfe58..b2800772 100644
--- a/drivers/gpu/nvgpu/gk20a/bus_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/bus_gk20a.c
@@ -21,6 +21,7 @@
  */
 
 #include <nvgpu/page_allocator.h>
+#include <nvgpu/enabled.h>
 #include <nvgpu/log.h>
 #include <nvgpu/soc.h>
 #include <nvgpu/bus.h>
@@ -155,8 +156,9 @@ int gk20a_bus_bar1_bind(struct gk20a *g, struct nvgpu_mem *bar1_inst)
 
         gk20a_writel(g, bus_bar1_block_r(),
                      nvgpu_aperture_mask(g, bar1_inst,
-                       bus_bar1_block_target_sys_mem_ncoh_f(),
-                       bus_bar1_block_target_vid_mem_f()) |
+                                         bus_bar1_block_target_sys_mem_ncoh_f(),
+                                         bus_bar1_block_target_sys_mem_coh_f(),
+                                         bus_bar1_block_target_vid_mem_f()) |
                      bus_bar1_block_mode_virtual_f() |
                      bus_bar1_block_ptr_f(ptr_v));
 
diff --git a/drivers/gpu/nvgpu/gk20a/fb_gk20a.c b/drivers/gpu/nvgpu/gk20a/fb_gk20a.c
index a5a2cb51..e3052701 100644
--- a/drivers/gpu/nvgpu/gk20a/fb_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fb_gk20a.c
@@ -98,8 +98,9 @@ void gk20a_fb_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb)
         gk20a_writel(g, fb_mmu_invalidate_pdb_r(),
                 fb_mmu_invalidate_pdb_addr_f(addr_lo) |
                 nvgpu_aperture_mask(g, pdb,
-                  fb_mmu_invalidate_pdb_aperture_sys_mem_f(),
-                  fb_mmu_invalidate_pdb_aperture_vid_mem_f()));
+                                fb_mmu_invalidate_pdb_aperture_sys_mem_f(),
+                                fb_mmu_invalidate_pdb_aperture_sys_mem_f(),
+                                fb_mmu_invalidate_pdb_aperture_vid_mem_f()));
 
         gk20a_writel(g, fb_mmu_invalidate_r(),
                 fb_mmu_invalidate_all_va_true_f() |
diff --git a/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c b/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c
index 409661fc..4fda0d2e 100644
--- a/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c
@@ -653,6 +653,7 @@ int gk20a_fecs_trace_bind_channel(struct gk20a *g,
                 return -ENOMEM;
         aperture = nvgpu_aperture_mask(g, &trace->trace_buf,
                         ctxsw_prog_main_image_context_timestamp_buffer_ptr_hi_target_sys_mem_noncoherent_f(),
+                        ctxsw_prog_main_image_context_timestamp_buffer_ptr_hi_target_sys_mem_coherent_f(),
                         ctxsw_prog_main_image_context_timestamp_buffer_ptr_hi_target_vid_mem_f());
 
         if (nvgpu_mem_begin(g, mem))
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index e12576d2..258006f9 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -28,6 +28,7 @@
 #include <nvgpu/dma.h>
 #include <nvgpu/timers.h>
 #include <nvgpu/semaphore.h>
+#include <nvgpu/enabled.h>
 #include <nvgpu/kmem.h>
 #include <nvgpu/log.h>
 #include <nvgpu/soc.h>
@@ -666,11 +667,13 @@ static void fifo_engine_exception_status(struct gk20a *g,
 static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
 {
         struct fifo_runlist_info_gk20a *runlist;
+        struct fifo_engine_info_gk20a *engine_info;
         unsigned int runlist_id;
         u32 i;
         size_t runlist_size;
         u32 active_engine_id, pbdma_id, engine_id;
-        struct fifo_engine_info_gk20a *engine_info;
+        int flags = nvgpu_is_enabled(g, NVGPU_MM_USE_PHYSICAL_SG) ?
+                NVGPU_DMA_FORCE_CONTIGUOUS : 0;
 
         nvgpu_log_fn(g, " ");
 
@@ -705,8 +708,9 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
                                 f->num_runlist_entries, runlist_size);
 
                 for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
-                        int err = nvgpu_dma_alloc_sys(g, runlist_size,
-                                        &runlist->mem[i]);
+                        int err = nvgpu_dma_alloc_flags_sys(g, flags,
+                                                            runlist_size,
+                                                            &runlist->mem[i]);
                         if (err) {
                                 nvgpu_err(g, "memory allocation failed");
                                 goto clean_up_runlist;
@@ -3240,8 +3244,9 @@ static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
                 gk20a_writel(g, fifo_runlist_base_r(),
                         fifo_runlist_base_ptr_f(u64_lo32(runlist_iova >> 12)) |
                         nvgpu_aperture_mask(g, &runlist->mem[new_buf],
-                          fifo_runlist_base_target_sys_mem_ncoh_f(),
-                          fifo_runlist_base_target_vid_mem_f()));
+                                fifo_runlist_base_target_sys_mem_ncoh_f(),
+                                fifo_runlist_base_target_sys_mem_coh_f(),
+                                fifo_runlist_base_target_vid_mem_f()));
         }
 
         gk20a_writel(g, fifo_runlist_r(),
@@ -3763,8 +3768,9 @@ static int gk20a_fifo_commit_userd(struct channel_gk20a *c)
         nvgpu_mem_wr32(g, &c->inst_block,
                        ram_in_ramfc_w() + ram_fc_userd_w(),
                        nvgpu_aperture_mask(g, &g->fifo.userd,
-                        pbdma_userd_target_sys_mem_ncoh_f(),
-                        pbdma_userd_target_vid_mem_f()) |
+                                           pbdma_userd_target_sys_mem_ncoh_f(),
+                                           pbdma_userd_target_sys_mem_coh_f(),
+                                           pbdma_userd_target_vid_mem_f()) |
                        pbdma_userd_addr_f(addr_lo));
 
         nvgpu_mem_wr32(g, &c->inst_block,
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
index 7160ab6f..bdb54325 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
@@ -742,13 +742,14 @@ void gr_gk20a_ctx_patch_write(struct gk20a *g,
 
 static u32 fecs_current_ctx_data(struct gk20a *g, struct nvgpu_mem *inst_block)
 {
-        u32 ptr = u64_lo32(nvgpu_inst_block_addr(g, inst_block)
-                        >> ram_in_base_shift_v());
+        u64 ptr = nvgpu_inst_block_addr(g, inst_block) >>
+                ram_in_base_shift_v();
         u32 aperture = nvgpu_aperture_mask(g, inst_block,
-                        gr_fecs_current_ctx_target_sys_mem_ncoh_f(),
-                        gr_fecs_current_ctx_target_vid_mem_f());
+                                gr_fecs_current_ctx_target_sys_mem_ncoh_f(),
+                                gr_fecs_current_ctx_target_sys_mem_coh_f(),
+                                gr_fecs_current_ctx_target_vid_mem_f());
 
-        return gr_fecs_current_ctx_ptr_f(ptr) | aperture |
+        return gr_fecs_current_ctx_ptr_f(u64_lo32(ptr)) | aperture |
                 gr_fecs_current_ctx_valid_f(1);
 }
 
@@ -2171,16 +2172,18 @@ void gr_gk20a_load_falcon_bind_instblk(struct gk20a *g)
 
         inst_ptr = nvgpu_inst_block_addr(g, &ucode_info->inst_blk_desc);
         gk20a_writel(g, gr_fecs_new_ctx_r(),
-                        gr_fecs_new_ctx_ptr_f(inst_ptr >> 12) |
-                        nvgpu_aperture_mask(g, &ucode_info->inst_blk_desc,
+                     gr_fecs_new_ctx_ptr_f(inst_ptr >> 12) |
+                     nvgpu_aperture_mask(g, &ucode_info->inst_blk_desc,
                                 gr_fecs_new_ctx_target_sys_mem_ncoh_f(),
+                                gr_fecs_new_ctx_target_sys_mem_coh_f(),
                                 gr_fecs_new_ctx_target_vid_mem_f()) |
-                        gr_fecs_new_ctx_valid_m());
+                     gr_fecs_new_ctx_valid_m());
 
         gk20a_writel(g, gr_fecs_arb_ctx_ptr_r(),
-                        gr_fecs_arb_ctx_ptr_ptr_f(inst_ptr >> 12) |
-                        nvgpu_aperture_mask(g, &ucode_info->inst_blk_desc,
+                     gr_fecs_arb_ctx_ptr_ptr_f(inst_ptr >> 12) |
+                     nvgpu_aperture_mask(g, &ucode_info->inst_blk_desc,
                                 gr_fecs_arb_ctx_ptr_target_sys_mem_ncoh_f(),
+                                gr_fecs_arb_ctx_ptr_target_sys_mem_coh_f(),
                                 gr_fecs_arb_ctx_ptr_target_vid_mem_f()));
 
         gk20a_writel(g, gr_fecs_arb_ctx_cmd_r(), 0x7);
@@ -4384,8 +4387,9 @@ static int gk20a_init_gr_setup_hw(struct gk20a *g)
 
         gk20a_writel(g, fb_mmu_debug_wr_r(),
                      nvgpu_aperture_mask(g, &gr->mmu_wr_mem,
-                       fb_mmu_debug_wr_aperture_sys_mem_ncoh_f(),
-                       fb_mmu_debug_wr_aperture_vid_mem_f()) |
+                                fb_mmu_debug_wr_aperture_sys_mem_ncoh_f(),
+                                fb_mmu_debug_wr_aperture_sys_mem_coh_f(),
+                                fb_mmu_debug_wr_aperture_vid_mem_f()) |
                      fb_mmu_debug_wr_vol_false_f() |
                      fb_mmu_debug_wr_addr_f(addr));
 
@@ -4394,8 +4398,9 @@ static int gk20a_init_gr_setup_hw(struct gk20a *g)
 
         gk20a_writel(g, fb_mmu_debug_rd_r(),
                      nvgpu_aperture_mask(g, &gr->mmu_rd_mem,
-                       fb_mmu_debug_wr_aperture_sys_mem_ncoh_f(),
-                       fb_mmu_debug_rd_aperture_vid_mem_f()) |
+                                fb_mmu_debug_wr_aperture_sys_mem_ncoh_f(),
+                                fb_mmu_debug_wr_aperture_sys_mem_coh_f(),
+                                fb_mmu_debug_rd_aperture_vid_mem_f()) |
                      fb_mmu_debug_rd_vol_false_f() |
                      fb_mmu_debug_rd_addr_f(addr));
 
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
index b27d1109..4ff6125b 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
@@ -122,8 +122,9 @@ static inline u32 big_valid_pde0_bits(struct gk20a *g,
 {
         u32 pde0_bits =
                 nvgpu_aperture_mask(g, pd->mem,
-                  gmmu_pde_aperture_big_sys_mem_ncoh_f(),
-                  gmmu_pde_aperture_big_video_memory_f()) |
+                                    gmmu_pde_aperture_big_sys_mem_ncoh_f(),
+                                    gmmu_pde_aperture_big_sys_mem_coh_f(),
+                                    gmmu_pde_aperture_big_video_memory_f()) |
                 gmmu_pde_address_big_sys_f(
                            (u32)(addr >> gmmu_pde_address_shift_v()));
 
@@ -135,8 +136,9 @@ static inline u32 small_valid_pde1_bits(struct gk20a *g,
 {
         u32 pde1_bits =
                 nvgpu_aperture_mask(g, pd->mem,
-                  gmmu_pde_aperture_small_sys_mem_ncoh_f(),
-                  gmmu_pde_aperture_small_video_memory_f()) |
+                                    gmmu_pde_aperture_small_sys_mem_ncoh_f(),
+                                    gmmu_pde_aperture_small_sys_mem_coh_f(),
+                                    gmmu_pde_aperture_small_video_memory_f()) |
                 gmmu_pde_vol_small_true_f() | /* tbd: why? */
                 gmmu_pde_address_small_sys_f(
                            (u32)(addr >> gmmu_pde_address_shift_v()));
@@ -215,6 +217,7 @@ static void __update_pte(struct vm_gk20a *vm,
 
         pte_w[1] = __nvgpu_aperture_mask(g, attrs->aperture,
                                          gmmu_pte_aperture_sys_mem_ncoh_f(),
+                                         gmmu_pte_aperture_sys_mem_coh_f(),
                                          gmmu_pte_aperture_video_memory_f()) |
                 gmmu_pte_kind_f(attrs->kind_v) |
                 gmmu_pte_comptagline_f((u32)(attrs->ctag >> ctag_shift));
@@ -268,7 +271,7 @@ static void update_gmmu_pte_locked(struct vm_gk20a *vm,
                 page_size >> 10,
                 nvgpu_gmmu_perm_str(attrs->rw_flag),
                 attrs->kind_v,
-                nvgpu_aperture_str(attrs->aperture),
+                nvgpu_aperture_str(g, attrs->aperture),
                 attrs->cacheable ? 'C' : '-',
                 attrs->sparse    ? 'S' : '-',
                 attrs->priv      ? 'P' : '-',
@@ -363,11 +366,12 @@ void gk20a_mm_init_pdb(struct gk20a *g, struct nvgpu_mem *inst_block,
         gk20a_dbg_info("pde pa=0x%llx", pdb_addr);
 
         nvgpu_mem_wr32(g, inst_block, ram_in_page_dir_base_lo_w(),
-                nvgpu_aperture_mask(g, vm->pdb.mem,
-                  ram_in_page_dir_base_target_sys_mem_ncoh_f(),
-                  ram_in_page_dir_base_target_vid_mem_f()) |
-                ram_in_page_dir_base_vol_true_f() |
-                ram_in_page_dir_base_lo_f(pdb_addr_lo));
+                       nvgpu_aperture_mask(g, vm->pdb.mem,
+                                ram_in_page_dir_base_target_sys_mem_ncoh_f(),
+                                ram_in_page_dir_base_target_sys_mem_coh_f(),
+                                ram_in_page_dir_base_target_vid_mem_f()) |
+                       ram_in_page_dir_base_vol_true_f() |
+                       ram_in_page_dir_base_lo_f(pdb_addr_lo));
 
         nvgpu_mem_wr32(g, inst_block, ram_in_page_dir_base_hi_w(),
                 ram_in_page_dir_base_hi_f(pdb_addr_hi));
diff --git a/drivers/gpu/nvgpu/gk20a/pramin_gk20a.c b/drivers/gpu/nvgpu/gk20a/pramin_gk20a.c
index bb8831e0..67fd2480 100644
--- a/drivers/gpu/nvgpu/gk20a/pramin_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/pramin_gk20a.c
@@ -41,6 +41,7 @@ u32 gk20a_pramin_enter(struct gk20a *g, struct nvgpu_mem *mem,
         u32 lo = (u32)(addr & 0xfffff);
         u32 win = nvgpu_aperture_mask(g, mem,
                         bus_bar0_window_target_sys_mem_noncoherent_f(),
+                        bus_bar0_window_target_sys_mem_coherent_f(),
                         bus_bar0_window_target_vid_mem_f()) |
                 bus_bar0_window_base_f(hi);