Revert "Revert "gpu: nvgpu: Get coherency on gv100 + NVLINK working""

This reverts commit 5a35a95654d561fce09a3b9abf6b82bb7a29d74b.

JIRA EVLR-2333

Change-Id: I923c32496c343d39d34f6d406c38a9f6ce7dc6e0
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1667167
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

2 files changed, 20 insertions, 14 deletions

diff --git a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
index c82fb1cc..1436a260 100644
--- a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
@@ -25,6 +25,7 @@
 #include <nvgpu/dma.h>
 #include <nvgpu/bug.h>
 #include <nvgpu/log2.h>
+#include <nvgpu/enabled.h>
 
 #include "fifo_gp10b.h"
 
@@ -78,8 +79,9 @@ int channel_gp10b_commit_userd(struct channel_gk20a *c)
         nvgpu_mem_wr32(g, &c->inst_block,
                        ram_in_ramfc_w() + ram_fc_userd_w(),
                        nvgpu_aperture_mask(g, &g->fifo.userd,
-                        pbdma_userd_target_sys_mem_ncoh_f(),
-                        pbdma_userd_target_vid_mem_f()) |
+                                           pbdma_userd_target_sys_mem_ncoh_f(),
+                                           pbdma_userd_target_sys_mem_coh_f(),
+                                           pbdma_userd_target_vid_mem_f()) |
                        pbdma_userd_addr_f(addr_lo));
 
         nvgpu_mem_wr32(g, &c->inst_block,
diff --git a/drivers/gpu/nvgpu/gp10b/mm_gp10b.c b/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
index 0439dda9..7ff5f6a6 100644
--- a/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
@@ -124,8 +124,9 @@ int gp10b_init_bar2_mm_hw_setup(struct gk20a *g)
 
         gk20a_writel(g, bus_bar2_block_r(),
                      nvgpu_aperture_mask(g, inst_block,
-                                bus_bar2_block_target_sys_mem_ncoh_f(),
-                                bus_bar2_block_target_vid_mem_f()) |
+                                         bus_bar2_block_target_sys_mem_ncoh_f(),
+                                         bus_bar2_block_target_sys_mem_coh_f(),
+                                         bus_bar2_block_target_vid_mem_f()) |
                      bus_bar2_block_mode_virtual_f() |
                      bus_bar2_block_ptr_f(inst_pa));
 
@@ -148,8 +149,9 @@ static void update_gmmu_pde3_locked(struct vm_gk20a *vm,
         phys_addr >>= gmmu_new_pde_address_shift_v();
 
         pde_v[0] |= nvgpu_aperture_mask(g, pd->mem,
-                        gmmu_new_pde_aperture_sys_mem_ncoh_f(),
-                        gmmu_new_pde_aperture_video_memory_f());
+                                        gmmu_new_pde_aperture_sys_mem_ncoh_f(),
+                                        gmmu_new_pde_aperture_sys_mem_coh_f(),
+                                        gmmu_new_pde_aperture_video_memory_f());
         pde_v[0] |= gmmu_new_pde_address_sys_f(u64_lo32(phys_addr));
         pde_v[0] |= gmmu_new_pde_vol_true_f();
         pde_v[1] |= phys_addr >> 24;
@@ -194,6 +196,7 @@ static void update_gmmu_pde0_locked(struct vm_gk20a *vm,
                         gmmu_new_dual_pde_address_small_sys_f(small_addr);
                 pde_v[2] |= nvgpu_aperture_mask(g, pd->mem,
                         gmmu_new_dual_pde_aperture_small_sys_mem_ncoh_f(),
+                        gmmu_new_dual_pde_aperture_small_sys_mem_coh_f(),
                         gmmu_new_dual_pde_aperture_small_video_memory_f());
                 pde_v[2] |= gmmu_new_dual_pde_vol_small_true_f();
                 pde_v[3] |= small_addr >> 24;
@@ -204,6 +207,7 @@ static void update_gmmu_pde0_locked(struct vm_gk20a *vm,
                 pde_v[0] |= gmmu_new_dual_pde_vol_big_true_f();
                 pde_v[0] |= nvgpu_aperture_mask(g, pd->mem,
                         gmmu_new_dual_pde_aperture_big_sys_mem_ncoh_f(),
+                        gmmu_new_dual_pde_aperture_big_sys_mem_coh_f(),
                         gmmu_new_dual_pde_aperture_big_video_memory_f());
                 pde_v[1] |= big_addr >> 28;
         }
@@ -240,11 +244,10 @@ static void __update_pte(struct vm_gk20a *vm,
                 gmmu_new_pte_address_sys_f(phys_shifted) :
                 gmmu_new_pte_address_vid_f(phys_shifted);
         u32 pte_tgt = __nvgpu_aperture_mask(g,
-                        attrs->aperture,
-                        attrs->coherent ?
-                                gmmu_new_pte_aperture_sys_mem_coh_f() :
-                                gmmu_new_pte_aperture_sys_mem_ncoh_f(),
-                        gmmu_new_pte_aperture_video_memory_f());
+                                        attrs->aperture,
+                                        gmmu_new_pte_aperture_sys_mem_ncoh_f(),
+                                        gmmu_new_pte_aperture_sys_mem_coh_f(),
+                                        gmmu_new_pte_aperture_video_memory_f());
 
         pte_w[0] = pte_valid | pte_addr | pte_tgt;
 
@@ -306,7 +309,7 @@ static void update_gmmu_pte_locked(struct vm_gk20a *vm,
                 page_size >> 10,
                 nvgpu_gmmu_perm_str(attrs->rw_flag),
                 attrs->kind_v,
-                nvgpu_aperture_str(attrs->aperture),
+                nvgpu_aperture_str(g, attrs->aperture),
                 attrs->cacheable ? 'C' : '-',
                 attrs->sparse    ? 'S' : '-',
                 attrs->priv      ? 'P' : '-',
@@ -428,8 +431,9 @@ void gp10b_mm_init_pdb(struct gk20a *g, struct nvgpu_mem *inst_block,
 
         nvgpu_mem_wr32(g, inst_block, ram_in_page_dir_base_lo_w(),
                 nvgpu_aperture_mask(g, vm->pdb.mem,
-                  ram_in_page_dir_base_target_sys_mem_ncoh_f(),
-                  ram_in_page_dir_base_target_vid_mem_f()) |
+                                ram_in_page_dir_base_target_sys_mem_ncoh_f(),
+                                ram_in_page_dir_base_target_sys_mem_coh_f(),
+                                ram_in_page_dir_base_target_vid_mem_f()) |
                 ram_in_page_dir_base_vol_true_f() |
                 ram_in_big_page_size_64kb_f() |
                 ram_in_page_dir_base_lo_f(pdb_addr_lo) |