gpu: nvgpu: Use coherent aperture flag

When using a coherent DMA API wee must make sure to program
any aperture fields with the coherent aperture setting. To
do this the nvgpu_aperture_mask() function was modified to
take a third aperture mask argument, a coherent setting, so
that code can use this function to generate coherent aperture
settings.

The aperture choice is some what tricky: the default version
of this function uses the state of the DMA API to determine
what aperture to use for SYSMEM: either coherent or
non-coherent internally. Thus a kernel user need only specify
the normal nvgpu_mem struct and the correct mask should be
chosen. Due to many uses of nvgpu_mem structs not created
directly from the DMA API wrapper it's easier to translate
SYSMEM to SYSMEM_COH after creation.

However, the GMMU mapping code, will encounter buffers from
userspace with difference coerency attributes than the DMA
API. Thus the __nvgpu_aperture_mask() really respects the
aperture setting passed in regardless of the DMA API state.
This aperture setting is pulled from NVGPU_VM_MAP_IO_COHERENT
since this is either passed in from userspace or set by the
kernel when using coherent DMA. The aperture field in attrs
is upgraded to coh if this flag is set.

This change also adds a coherent sysmem mask everywhere that
it can. There's a couple places that do not have a coherent
register field defined yet. These need to eventually be
defined and added.

Lastly the aperture mask code has been mvoed from the Linux
vm.c code to the general vm.c code since this function has
no Linux dependencies.

Note: depends on https://git-master.nvidia.com/r/1664536 for
new register fields.

JIRA EVLR-2333

Change-Id: I4b347911ecb7c511738563fe6c34d0e6aa380d71
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1655220
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

7 files changed, 33 insertions, 21 deletions

diff --git a/drivers/gpu/nvgpu/gv11b/acr_gv11b.c b/drivers/gpu/nvgpu/gv11b/acr_gv11b.c
index 799b2db4..4fa3f324 100644
--- a/drivers/gpu/nvgpu/gv11b/acr_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/acr_gv11b.c
@@ -27,9 +27,10 @@
 #include <nvgpu/nvgpu_common.h>
 #include <nvgpu/kmem.h>
 #include <nvgpu/nvgpu_mem.h>
-#include <nvgpu/acr/nvgpu_acr.h>
 #include <nvgpu/firmware.h>
 #include <nvgpu/mm.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/acr/nvgpu_acr.h>
 
 #include "gk20a/gk20a.h"
 #include "acr_gv11b.h"
@@ -220,7 +221,9 @@ static int bl_bootstrap(struct nvgpu_pmu *pmu,
                 pwr_pmu_new_instblk_ptr_f(
                 nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> 12) |
                 pwr_pmu_new_instblk_valid_f(1) |
-                pwr_pmu_new_instblk_target_sys_ncoh_f());
+                     (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) ?
+                      pwr_pmu_new_instblk_target_sys_coh_f() :
+                      pwr_pmu_new_instblk_target_sys_ncoh_f())) ;
 
         /*copy bootloader interface structure to dmem*/
         nvgpu_flcn_copy_to_dmem(pmu->flcn, 0, (u8 *)pbl_desc,
diff --git a/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c
index 617ea61d..86977bb3 100644
--- a/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c
@@ -31,14 +31,14 @@
 #include <nvgpu/dma.h>
 #include <nvgpu/mm.h>
 #include <nvgpu/sizes.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/log.h>
+#include <nvgpu/bug.h>
 
 #include "gk20a/gk20a.h"
 #include "gk20a/css_gr_gk20a.h"
 #include "css_gr_gv11b.h"
 
-#include <nvgpu/log.h>
-#include <nvgpu/bug.h>
-
 #include <nvgpu/hw/gv11b/hw_perf_gv11b.h>
 #include <nvgpu/hw/gv11b/hw_mc_gv11b.h>
 
@@ -144,6 +144,7 @@ int gv11b_css_hw_enable_snapshot(struct channel_gk20a *ch,
                         perf_pmasys_mem_block_valid_true_f() |
                         nvgpu_aperture_mask(g, &g->mm.hwpm.inst_block,
                                 perf_pmasys_mem_block_target_sys_ncoh_f(),
+                                perf_pmasys_mem_block_target_sys_coh_f(),
                                 perf_pmasys_mem_block_target_lfb_f()));
 
 
diff --git a/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c
index e5d88e8c..562476ca 100644
--- a/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c
@@ -59,11 +59,12 @@ int gv11b_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size)
         inst_pa_page = nvgpu_inst_block_addr(g, &mm->perfbuf.inst_block) >> 12;
 
         gk20a_writel(g, perf_pmasys_mem_block_r(),
-                        perf_pmasys_mem_block_base_f(inst_pa_page) |
-                        perf_pmasys_mem_block_valid_true_f() |
-                        nvgpu_aperture_mask(g, &mm->perfbuf.inst_block,
-+                               perf_pmasys_mem_block_target_sys_ncoh_f(),
-+                               perf_pmasys_mem_block_target_lfb_f()));
+                     perf_pmasys_mem_block_base_f(inst_pa_page) |
+                     perf_pmasys_mem_block_valid_true_f() |
+                     nvgpu_aperture_mask(g, &mm->perfbuf.inst_block,
+                                perf_pmasys_mem_block_target_sys_ncoh_f(),
+                                perf_pmasys_mem_block_target_sys_coh_f(),
+                                perf_pmasys_mem_block_target_lfb_f()));
 
         gk20a_idle(g);
         return 0;
diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
index 97ab7aab..6b4b07a6 100644
--- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
@@ -101,12 +101,14 @@ void gv11b_get_ch_runlist_entry(struct channel_gk20a *c, u32 *runlist)
                                                 c->runqueue_sel) |
                         ram_rl_entry_chan_userd_target_f(
                                 nvgpu_aperture_mask(g, &g->fifo.userd,
-                                ram_rl_entry_chan_userd_target_sys_mem_ncoh_v(),
-                                ram_rl_entry_chan_userd_target_vid_mem_v())) |
+                                        ram_rl_entry_chan_userd_target_sys_mem_ncoh_v(),
+                                        ram_rl_entry_chan_userd_target_sys_mem_coh_v(),
+                                        ram_rl_entry_chan_userd_target_vid_mem_v())) |
                         ram_rl_entry_chan_inst_target_f(
                                 nvgpu_aperture_mask(g, &c->inst_block,
-                                ram_rl_entry_chan_inst_target_sys_mem_ncoh_v(),
-                                ram_rl_entry_chan_inst_target_vid_mem_v()));
+                                        ram_rl_entry_chan_inst_target_sys_mem_ncoh_v(),
+                                        ram_rl_entry_chan_inst_target_sys_mem_coh_v(),
+                                        ram_rl_entry_chan_inst_target_vid_mem_v()));
 
         addr_lo = u64_lo32(c->userd_iova) >>
                         ram_rl_entry_chan_userd_ptr_align_shift_v();
diff --git a/drivers/gpu/nvgpu/gv11b/mm_gv11b.c b/drivers/gpu/nvgpu/gv11b/mm_gv11b.c
index ade1d9fe..b46ecb0a 100644
--- a/drivers/gpu/nvgpu/gv11b/mm_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/mm_gv11b.c
@@ -26,6 +26,7 @@
 #include <nvgpu/dma.h>
 #include <nvgpu/log.h>
 #include <nvgpu/mm.h>
+#include <nvgpu/enabled.h>
 
 #include "gk20a/gk20a.h"
 #include "gk20a/mm_gk20a.h"
@@ -292,8 +293,9 @@ int gv11b_init_bar2_mm_hw_setup(struct gk20a *g)
 
         gk20a_writel(g, bus_bar2_block_r(),
                      nvgpu_aperture_mask(g, inst_block,
-                                bus_bar2_block_target_sys_mem_ncoh_f(),
-                                bus_bar2_block_target_vid_mem_f()) |
+                                         bus_bar2_block_target_sys_mem_ncoh_f(),
+                                         bus_bar2_block_target_sys_mem_coh_f(),
+                                         bus_bar2_block_target_vid_mem_f()) |
                      bus_bar2_block_mode_virtual_f() |
                      bus_bar2_block_ptr_f(inst_pa));
 
diff --git a/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c
index 7dd4f8f4..13e70eca 100644
--- a/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c
@@ -195,9 +195,11 @@ int gv11b_pmu_bootstrap(struct nvgpu_pmu *pmu)
 
         gk20a_writel(g, pwr_pmu_new_instblk_r(),
                 pwr_pmu_new_instblk_ptr_f(
-                nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> ALIGN_4KB)
-                | pwr_pmu_new_instblk_valid_f(1)
-                | pwr_pmu_new_instblk_target_sys_ncoh_f());
+                nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> ALIGN_4KB) |
+                     pwr_pmu_new_instblk_valid_f(1) |
+                     (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) ?
+                      pwr_pmu_new_instblk_target_sys_coh_f() :
+                      pwr_pmu_new_instblk_target_sys_ncoh_f()));
 
         /* TBD: load all other surfaces */
         g->ops.pmu_ver.set_pmu_cmdline_args_trace_size(
diff --git a/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c
index 05d7dee0..bda4c8e4 100644
--- a/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c
@@ -177,8 +177,9 @@ void gv11b_subctx_commit_pdb(struct vm_gk20a *vm,
         u32 pdb_addr_lo, pdb_addr_hi;
         u64 pdb_addr;
         u32 aperture = nvgpu_aperture_mask(g, vm->pdb.mem,
-                        ram_in_sc_page_dir_base_target_sys_mem_ncoh_v(),
-                        ram_in_sc_page_dir_base_target_vid_mem_v());
+                                ram_in_sc_page_dir_base_target_sys_mem_ncoh_v(),
+                                ram_in_sc_page_dir_base_target_sys_mem_coh_v(),
+                                ram_in_sc_page_dir_base_target_vid_mem_v());
 
         pdb_addr = nvgpu_mem_get_addr(g, vm->pdb.mem);
         pdb_addr_lo = u64_lo32(pdb_addr >> ram_in_base_shift_v());