24 files changed, 184 insertions, 103 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c b/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c
index e441ec76..c859520d 100644
--- a/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c
+++ b/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c
@@ -34,30 +34,6 @@
 #include "gk20a/gk20a.h"
 #include "gk20a/mm_gk20a.h"
-u32 __nvgpu_aperture_mask(struct gk20a *g, enum nvgpu_aperture aperture,
-                u32 sysmem_mask, u32 vidmem_mask)
-{
-        switch (aperture) {
-        case APERTURE_SYSMEM:
-                /* some igpus consider system memory vidmem */
-                return nvgpu_is_enabled(g, NVGPU_MM_HONORS_APERTURE)
-                        ? sysmem_mask : vidmem_mask;
-        case APERTURE_VIDMEM:
-                /* for dgpus only */
-                return vidmem_mask;
-        case APERTURE_INVALID:
-                WARN_ON("Bad aperture");
-        }
-        return 0;
-}
-u32 nvgpu_aperture_mask(struct gk20a *g, struct nvgpu_mem *mem,
-                u32 sysmem_mask, u32 vidmem_mask)
-{
-        return __nvgpu_aperture_mask(g, mem->aperture,
-                        sysmem_mask, vidmem_mask);
-}
 int nvgpu_mem_begin(struct gk20a *g, struct nvgpu_mem *mem)
 {
        void *cpu_va;
diff --git a/drivers/gpu/nvgpu/common/linux/vm.c b/drivers/gpu/nvgpu/common/linux/vm.c
index e3ca4eda..52b2f30c 100644
--- a/drivers/gpu/nvgpu/common/linux/vm.c
+++ b/drivers/gpu/nvgpu/common/linux/vm.c
@@ -166,7 +166,8 @@ struct nvgpu_mapped_buf *nvgpu_vm_find_mapping(struct vm_gk20a *vm,
                  vm->gmmu_page_sizes[mapped_buffer->pgsz_idx] >> 10,
                  vm_aspace_id(vm),
                  mapped_buffer->flags,
-                  nvgpu_aperture_str(gk20a_dmabuf_aperture(g, os_buf->dmabuf)));
+                  nvgpu_aperture_str(g,
+                                     gk20a_dmabuf_aperture(g, os_buf->dmabuf)));
        return mapped_buffer;
 }
diff --git a/drivers/gpu/nvgpu/common/mm/gmmu.c b/drivers/gpu/nvgpu/common/mm/gmmu.c
index 5abf5951..41343718 100644
--- a/drivers/gpu/nvgpu/common/mm/gmmu.c
+++ b/drivers/gpu/nvgpu/common/mm/gmmu.c
@@ -634,7 +634,7 @@ static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm,
                   page_size >> 10,
                   nvgpu_gmmu_perm_str(attrs->rw_flag),
                   attrs->kind_v,
-                   nvgpu_aperture_str(attrs->aperture),
+                   nvgpu_aperture_str(g, attrs->aperture),
                   attrs->cacheable ? 'C' : '-',
                   attrs->sparse    ? 'S' : '-',
                   attrs->priv      ? 'P' : '-',
@@ -712,6 +712,13 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
        attrs.l3_alloc = (bool)(flags & NVGPU_VM_MAP_L3_ALLOC);
        /*
+         * Handle the IO coherency aperture: make sure the .aperture field is
+         * correct based on the IO coherency flag.
+         */
+        if (attrs.coherent && attrs.aperture == APERTURE_SYSMEM)
+                attrs.aperture = __APERTURE_SYSMEM_COH;
+        /*
         * Only allocate a new GPU VA range if we haven't already been passed a
         * GPU VA range. This facilitates fixed mappings.
         */
diff --git a/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
index 73b6b2a7..78a57b4e 100644
--- a/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
+++ b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
@@ -28,6 +28,52 @@
 #include "gk20a/gk20a.h"
+/*
+ * Make sure to use the right coherency aperture if you use this function! This
+ * will not add any checks. If you want to simply use the default coherency then
+ * use nvgpu_aperture_mask().
+ */
+u32 __nvgpu_aperture_mask(struct gk20a *g, enum nvgpu_aperture aperture,
+                          u32 sysmem_mask, u32 sysmem_coh_mask, u32 vidmem_mask)
+{
+        /*
+         * Some iGPUs treat sysmem (i.e SoC DRAM) as vidmem. In these cases the
+         * "sysmem" aperture should really be translated to VIDMEM.
+         */
+        if (!nvgpu_is_enabled(g, NVGPU_MM_HONORS_APERTURE))
+                aperture = APERTURE_VIDMEM;
+        switch (aperture) {
+        case __APERTURE_SYSMEM_COH:
+                return sysmem_coh_mask;
+        case APERTURE_SYSMEM:
+                return sysmem_mask;
+        case APERTURE_VIDMEM:
+                return vidmem_mask;
+        case APERTURE_INVALID:
+                WARN_ON("Bad aperture");
+        }
+        return 0;
+}
+u32 nvgpu_aperture_mask(struct gk20a *g, struct nvgpu_mem *mem,
+                        u32 sysmem_mask, u32 sysmem_coh_mask, u32 vidmem_mask)
+{
+        enum nvgpu_aperture ap = mem->aperture;
+        /*
+         * Handle the coherent aperture: ideally most of the driver is not
+         * aware of the difference between coherent and non-coherent sysmem so
+         * we add this translation step here.
+         */
+        if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) &&
+            ap == APERTURE_SYSMEM)
+                ap = __APERTURE_SYSMEM_COH;
+        return __nvgpu_aperture_mask(g, ap,
+                                     sysmem_mask, sysmem_coh_mask, vidmem_mask);
+}
 void *nvgpu_sgt_get_next(struct nvgpu_sgt *sgt, void *sgl)
 {
        return sgt->ops->sgl_next(sgl);
diff --git a/drivers/gpu/nvgpu/gk20a/bus_gk20a.c b/drivers/gpu/nvgpu/gk20a/bus_gk20a.c
index 7f0cfe58..b2800772 100644
--- a/drivers/gpu/nvgpu/gk20a/bus_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/bus_gk20a.c
@@ -21,6 +21,7 @@
 */
 #include <nvgpu/page_allocator.h>
+#include <nvgpu/enabled.h>
 #include <nvgpu/log.h>
 #include <nvgpu/soc.h>
 #include <nvgpu/bus.h>
@@ -155,8 +156,9 @@ int gk20a_bus_bar1_bind(struct gk20a *g, struct nvgpu_mem *bar1_inst)
        gk20a_writel(g, bus_bar1_block_r(),
                     nvgpu_aperture_mask(g, bar1_inst,
-                       bus_bar1_block_target_sys_mem_ncoh_f(),
+                                         bus_bar1_block_target_sys_mem_ncoh_f(),
-                       bus_bar1_block_target_vid_mem_f()) |
+                                         bus_bar1_block_target_sys_mem_coh_f(),
+                                         bus_bar1_block_target_vid_mem_f()) |
                     bus_bar1_block_mode_virtual_f() |
                     bus_bar1_block_ptr_f(ptr_v));
diff --git a/drivers/gpu/nvgpu/gk20a/fb_gk20a.c b/drivers/gpu/nvgpu/gk20a/fb_gk20a.c
index a5a2cb51..e3052701 100644
--- a/drivers/gpu/nvgpu/gk20a/fb_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fb_gk20a.c
@@ -98,8 +98,9 @@ void gk20a_fb_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb)
        gk20a_writel(g, fb_mmu_invalidate_pdb_r(),
                fb_mmu_invalidate_pdb_addr_f(addr_lo) |
                nvgpu_aperture_mask(g, pdb,
-                  fb_mmu_invalidate_pdb_aperture_sys_mem_f(),
+                                fb_mmu_invalidate_pdb_aperture_sys_mem_f(),
-                  fb_mmu_invalidate_pdb_aperture_vid_mem_f()));
+                                fb_mmu_invalidate_pdb_aperture_sys_mem_f(),
+                                fb_mmu_invalidate_pdb_aperture_vid_mem_f()));
        gk20a_writel(g, fb_mmu_invalidate_r(),
                fb_mmu_invalidate_all_va_true_f() |
diff --git a/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c b/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c
index 409661fc..4fda0d2e 100644
--- a/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c
@@ -653,6 +653,7 @@ int gk20a_fecs_trace_bind_channel(struct gk20a *g,
                return -ENOMEM;
        aperture = nvgpu_aperture_mask(g, &trace->trace_buf,
                        ctxsw_prog_main_image_context_timestamp_buffer_ptr_hi_target_sys_mem_noncoherent_f(),
+                        ctxsw_prog_main_image_context_timestamp_buffer_ptr_hi_target_sys_mem_coherent_f(),
                        ctxsw_prog_main_image_context_timestamp_buffer_ptr_hi_target_vid_mem_f());
        if (nvgpu_mem_begin(g, mem))
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index dd0b78c0..247557aa 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -3240,8 +3240,9 @@ static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
                gk20a_writel(g, fifo_runlist_base_r(),
                        fifo_runlist_base_ptr_f(u64_lo32(runlist_iova >> 12)) |
                        nvgpu_aperture_mask(g, &runlist->mem[new_buf],
-                          fifo_runlist_base_target_sys_mem_ncoh_f(),
+                                fifo_runlist_base_target_sys_mem_ncoh_f(),
-                          fifo_runlist_base_target_vid_mem_f()));
+                                fifo_runlist_base_target_sys_mem_coh_f(),
+                                fifo_runlist_base_target_vid_mem_f()));
        }
        gk20a_writel(g, fifo_runlist_r(),
@@ -3763,8 +3764,9 @@ static int gk20a_fifo_commit_userd(struct channel_gk20a *c)
        nvgpu_mem_wr32(g, &c->inst_block,
                       ram_in_ramfc_w() + ram_fc_userd_w(),
                       nvgpu_aperture_mask(g, &g->fifo.userd,
-                        pbdma_userd_target_sys_mem_ncoh_f(),
+                                           pbdma_userd_target_sys_mem_ncoh_f(),
-                        pbdma_userd_target_vid_mem_f()) |
+                                           pbdma_userd_target_sys_mem_coh_f(),
+                                           pbdma_userd_target_vid_mem_f()) |
                       pbdma_userd_addr_f(addr_lo));
        nvgpu_mem_wr32(g, &c->inst_block,
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
index 6ae743ef..2cde10ec 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
@@ -745,8 +745,9 @@ static u32 fecs_current_ctx_data(struct gk20a *g, struct nvgpu_mem *inst_block)
        u64 ptr = nvgpu_inst_block_addr(g, inst_block) >>
                ram_in_base_shift_v();
        u32 aperture = nvgpu_aperture_mask(g, inst_block,
-                        gr_fecs_current_ctx_target_sys_mem_ncoh_f(),
+                                gr_fecs_current_ctx_target_sys_mem_ncoh_f(),
-                        gr_fecs_current_ctx_target_vid_mem_f());
+                                gr_fecs_current_ctx_target_sys_mem_coh_f(),
+                                gr_fecs_current_ctx_target_vid_mem_f());
        return gr_fecs_current_ctx_ptr_f(u64_lo32(ptr)) | aperture |
                gr_fecs_current_ctx_valid_f(1);
@@ -2171,16 +2172,18 @@ void gr_gk20a_load_falcon_bind_instblk(struct gk20a *g)
        inst_ptr = nvgpu_inst_block_addr(g, &ucode_info->inst_blk_desc);
        gk20a_writel(g, gr_fecs_new_ctx_r(),
-                        gr_fecs_new_ctx_ptr_f(inst_ptr >> 12) |
+                     gr_fecs_new_ctx_ptr_f(inst_ptr >> 12) |
-                        nvgpu_aperture_mask(g, &ucode_info->inst_blk_desc,
+                     nvgpu_aperture_mask(g, &ucode_info->inst_blk_desc,
                                gr_fecs_new_ctx_target_sys_mem_ncoh_f(),
+                                gr_fecs_new_ctx_target_sys_mem_coh_f(),
                                gr_fecs_new_ctx_target_vid_mem_f()) |
-                        gr_fecs_new_ctx_valid_m());
+                     gr_fecs_new_ctx_valid_m());
        gk20a_writel(g, gr_fecs_arb_ctx_ptr_r(),
-                        gr_fecs_arb_ctx_ptr_ptr_f(inst_ptr >> 12) |
+                     gr_fecs_arb_ctx_ptr_ptr_f(inst_ptr >> 12) |
-                        nvgpu_aperture_mask(g, &ucode_info->inst_blk_desc,
+                     nvgpu_aperture_mask(g, &ucode_info->inst_blk_desc,
                                gr_fecs_arb_ctx_ptr_target_sys_mem_ncoh_f(),
+                                gr_fecs_arb_ctx_ptr_target_sys_mem_coh_f(),
                                gr_fecs_arb_ctx_ptr_target_vid_mem_f()));
        gk20a_writel(g, gr_fecs_arb_ctx_cmd_r(), 0x7);
@@ -4379,8 +4382,9 @@ static int gk20a_init_gr_setup_hw(struct gk20a *g)
        gk20a_writel(g, fb_mmu_debug_wr_r(),
                     nvgpu_aperture_mask(g, &gr->mmu_wr_mem,
-                       fb_mmu_debug_wr_aperture_sys_mem_ncoh_f(),
+                                fb_mmu_debug_wr_aperture_sys_mem_ncoh_f(),
-                       fb_mmu_debug_wr_aperture_vid_mem_f()) |
+                                fb_mmu_debug_wr_aperture_sys_mem_coh_f(),
+                                fb_mmu_debug_wr_aperture_vid_mem_f()) |
                     fb_mmu_debug_wr_vol_false_f() |
                     fb_mmu_debug_wr_addr_f(addr));
@@ -4389,8 +4393,9 @@ static int gk20a_init_gr_setup_hw(struct gk20a *g)
        gk20a_writel(g, fb_mmu_debug_rd_r(),
                     nvgpu_aperture_mask(g, &gr->mmu_rd_mem,
-                       fb_mmu_debug_wr_aperture_sys_mem_ncoh_f(),
+                                fb_mmu_debug_wr_aperture_sys_mem_ncoh_f(),
-                       fb_mmu_debug_rd_aperture_vid_mem_f()) |
+                                fb_mmu_debug_wr_aperture_sys_mem_coh_f(),
+                                fb_mmu_debug_rd_aperture_vid_mem_f()) |
                     fb_mmu_debug_rd_vol_false_f() |
                     fb_mmu_debug_rd_addr_f(addr));
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
index b27d1109..4ff6125b 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
@@ -122,8 +122,9 @@ static inline u32 big_valid_pde0_bits(struct gk20a *g,
 {
        u32 pde0_bits =
                nvgpu_aperture_mask(g, pd->mem,
-                  gmmu_pde_aperture_big_sys_mem_ncoh_f(),
+                                    gmmu_pde_aperture_big_sys_mem_ncoh_f(),
-                  gmmu_pde_aperture_big_video_memory_f()) |
+                                    gmmu_pde_aperture_big_sys_mem_coh_f(),
+                                    gmmu_pde_aperture_big_video_memory_f()) |
                gmmu_pde_address_big_sys_f(
                           (u32)(addr >> gmmu_pde_address_shift_v()));
@@ -135,8 +136,9 @@ static inline u32 small_valid_pde1_bits(struct gk20a *g,
 {
        u32 pde1_bits =
                nvgpu_aperture_mask(g, pd->mem,
-                  gmmu_pde_aperture_small_sys_mem_ncoh_f(),
+                                    gmmu_pde_aperture_small_sys_mem_ncoh_f(),
-                  gmmu_pde_aperture_small_video_memory_f()) |
+                                    gmmu_pde_aperture_small_sys_mem_coh_f(),
+                                    gmmu_pde_aperture_small_video_memory_f()) |
                gmmu_pde_vol_small_true_f() | /* tbd: why? */
                gmmu_pde_address_small_sys_f(
                           (u32)(addr >> gmmu_pde_address_shift_v()));
@@ -215,6 +217,7 @@ static void __update_pte(struct vm_gk20a *vm,
        pte_w[1] = __nvgpu_aperture_mask(g, attrs->aperture,
                                         gmmu_pte_aperture_sys_mem_ncoh_f(),
+                                         gmmu_pte_aperture_sys_mem_coh_f(),
                                         gmmu_pte_aperture_video_memory_f()) |
                gmmu_pte_kind_f(attrs->kind_v) |
                gmmu_pte_comptagline_f((u32)(attrs->ctag >> ctag_shift));
@@ -268,7 +271,7 @@ static void update_gmmu_pte_locked(struct vm_gk20a *vm,
                page_size >> 10,
                nvgpu_gmmu_perm_str(attrs->rw_flag),
                attrs->kind_v,
-                nvgpu_aperture_str(attrs->aperture),
+                nvgpu_aperture_str(g, attrs->aperture),
                attrs->cacheable ? 'C' : '-',
                attrs->sparse    ? 'S' : '-',
                attrs->priv      ? 'P' : '-',
@@ -363,11 +366,12 @@ void gk20a_mm_init_pdb(struct gk20a *g, struct nvgpu_mem *inst_block,
        gk20a_dbg_info("pde pa=0x%llx", pdb_addr);
        nvgpu_mem_wr32(g, inst_block, ram_in_page_dir_base_lo_w(),
-                nvgpu_aperture_mask(g, vm->pdb.mem,
+                       nvgpu_aperture_mask(g, vm->pdb.mem,
-                  ram_in_page_dir_base_target_sys_mem_ncoh_f(),
+                                ram_in_page_dir_base_target_sys_mem_ncoh_f(),
-                  ram_in_page_dir_base_target_vid_mem_f()) |
+                                ram_in_page_dir_base_target_sys_mem_coh_f(),
-                ram_in_page_dir_base_vol_true_f() |
+                                ram_in_page_dir_base_target_vid_mem_f()) |
-                ram_in_page_dir_base_lo_f(pdb_addr_lo));
+                       ram_in_page_dir_base_vol_true_f() |
+                       ram_in_page_dir_base_lo_f(pdb_addr_lo));
        nvgpu_mem_wr32(g, inst_block, ram_in_page_dir_base_hi_w(),
                ram_in_page_dir_base_hi_f(pdb_addr_hi));
diff --git a/drivers/gpu/nvgpu/gk20a/pramin_gk20a.c b/drivers/gpu/nvgpu/gk20a/pramin_gk20a.c
index 05d0473e..711aeb0d 100644
--- a/drivers/gpu/nvgpu/gk20a/pramin_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/pramin_gk20a.c
@@ -41,6 +41,7 @@ u32 gk20a_pramin_enter(struct gk20a *g, struct nvgpu_mem *mem,
        u32 lo = (u32)(addr & 0xfffff);
        u32 win = nvgpu_aperture_mask(g, mem,
                        bus_bar0_window_target_sys_mem_noncoherent_f(),
+                        bus_bar0_window_target_sys_mem_coherent_f(),
                        bus_bar0_window_target_vid_mem_f()) |
                bus_bar0_window_base_f(hi);
diff --git a/drivers/gpu/nvgpu/gm20b/bus_gm20b.c b/drivers/gpu/nvgpu/gm20b/bus_gm20b.c
index 34c8d4b7..cdd70d5b 100644
--- a/drivers/gpu/nvgpu/gm20b/bus_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/bus_gm20b.c
@@ -25,6 +25,7 @@
 #include <nvgpu/timers.h>
 #include <nvgpu/bus.h>
 #include <nvgpu/mm.h>
+#include <nvgpu/enabled.h>
 #include "bus_gm20b.h"
 #include "gk20a/gk20a.h"
@@ -43,8 +44,9 @@ int gm20b_bus_bar1_bind(struct gk20a *g, struct nvgpu_mem *bar1_inst)
        gk20a_writel(g, bus_bar1_block_r(),
                     nvgpu_aperture_mask(g, bar1_inst,
-                       bus_bar1_block_target_sys_mem_ncoh_f(),
+                                         bus_bar1_block_target_sys_mem_ncoh_f(),
-                       bus_bar1_block_target_vid_mem_f()) |
+                                         bus_bar1_block_target_sys_mem_coh_f(),
+                                         bus_bar1_block_target_vid_mem_f()) |
                     bus_bar1_block_mode_virtual_f() |
                     bus_bar1_block_ptr_f(ptr_v));
        nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER);
diff --git a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c
index 0762e8bd..15612995 100644
--- a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c
@@ -32,6 +32,7 @@
 #include <nvgpu/atomic.h>
 #include <nvgpu/barrier.h>
 #include <nvgpu/mm.h>
+#include <nvgpu/enabled.h>
 #include <nvgpu/hw/gm20b/hw_ccsr_gm20b.h>
 #include <nvgpu/hw/gm20b/hw_ram_gm20b.h>
@@ -51,11 +52,12 @@ void channel_gm20b_bind(struct channel_gk20a *c)
        gk20a_writel(g, ccsr_channel_inst_r(c->chid),
-                ccsr_channel_inst_ptr_f(inst_ptr) |
+                     ccsr_channel_inst_ptr_f(inst_ptr) |
-                nvgpu_aperture_mask(g, &c->inst_block,
+                     nvgpu_aperture_mask(g, &c->inst_block,
-                 ccsr_channel_inst_target_sys_mem_ncoh_f(),
+                                ccsr_channel_inst_target_sys_mem_ncoh_f(),
-                 ccsr_channel_inst_target_vid_mem_f()) |
+                                ccsr_channel_inst_target_sys_mem_coh_f(),
-                ccsr_channel_inst_bind_true_f());
+                                ccsr_channel_inst_target_vid_mem_f()) |
+                     ccsr_channel_inst_bind_true_f());
        gk20a_writel(g, ccsr_channel_r(c->chid),
                (gk20a_readl(g, ccsr_channel_r(c->chid)) &
diff --git a/drivers/gpu/nvgpu/gp106/sec2_gp106.c b/drivers/gpu/nvgpu/gp106/sec2_gp106.c
index 29aceb7c..8e4e5900 100644
--- a/drivers/gpu/nvgpu/gp106/sec2_gp106.c
+++ b/drivers/gpu/nvgpu/gp106/sec2_gp106.c
@@ -99,6 +99,7 @@ int bl_bootstrap_sec2(struct nvgpu_pmu *pmu,
                        nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> 12) |
                        pwr_pmu_new_instblk_valid_f(1) |
                        nvgpu_aperture_mask(g, &mm->pmu.inst_block,
+                                pwr_pmu_new_instblk_target_sys_ncoh_f(),
                                pwr_pmu_new_instblk_target_sys_coh_f(),
                                pwr_pmu_new_instblk_target_fb_f()));
@@ -165,6 +166,7 @@ void init_pmu_setup_hw1(struct gk20a *g)
                                        nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> 12) |
                                pwr_pmu_new_instblk_valid_f(1) |
                                nvgpu_aperture_mask(g, &mm->pmu.inst_block,
+                                        pwr_pmu_new_instblk_target_sys_ncoh_f(),
                                        pwr_pmu_new_instblk_target_sys_coh_f(),
                                        pwr_pmu_new_instblk_target_fb_f()));
diff --git a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
index c82fb1cc..1436a260 100644
--- a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
@@ -25,6 +25,7 @@
 #include <nvgpu/dma.h>
 #include <nvgpu/bug.h>
 #include <nvgpu/log2.h>
+#include <nvgpu/enabled.h>
 #include "fifo_gp10b.h"
@@ -78,8 +79,9 @@ int channel_gp10b_commit_userd(struct channel_gk20a *c)
        nvgpu_mem_wr32(g, &c->inst_block,
                       ram_in_ramfc_w() + ram_fc_userd_w(),
                       nvgpu_aperture_mask(g, &g->fifo.userd,
-                        pbdma_userd_target_sys_mem_ncoh_f(),
+                                           pbdma_userd_target_sys_mem_ncoh_f(),
-                        pbdma_userd_target_vid_mem_f()) |
+                                           pbdma_userd_target_sys_mem_coh_f(),
+                                           pbdma_userd_target_vid_mem_f()) |
                       pbdma_userd_addr_f(addr_lo));
        nvgpu_mem_wr32(g, &c->inst_block,
diff --git a/drivers/gpu/nvgpu/gp10b/mm_gp10b.c b/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
index 0439dda9..7ff5f6a6 100644
--- a/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
@@ -124,8 +124,9 @@ int gp10b_init_bar2_mm_hw_setup(struct gk20a *g)
        gk20a_writel(g, bus_bar2_block_r(),
                     nvgpu_aperture_mask(g, inst_block,
-                                bus_bar2_block_target_sys_mem_ncoh_f(),
+                                         bus_bar2_block_target_sys_mem_ncoh_f(),
-                                bus_bar2_block_target_vid_mem_f()) |
+                                         bus_bar2_block_target_sys_mem_coh_f(),
+                                         bus_bar2_block_target_vid_mem_f()) |
                     bus_bar2_block_mode_virtual_f() |
                     bus_bar2_block_ptr_f(inst_pa));
@@ -148,8 +149,9 @@ static void update_gmmu_pde3_locked(struct vm_gk20a *vm,
        phys_addr >>= gmmu_new_pde_address_shift_v();
        pde_v[0] |= nvgpu_aperture_mask(g, pd->mem,
-                        gmmu_new_pde_aperture_sys_mem_ncoh_f(),
+                                        gmmu_new_pde_aperture_sys_mem_ncoh_f(),
-                        gmmu_new_pde_aperture_video_memory_f());
+                                        gmmu_new_pde_aperture_sys_mem_coh_f(),
+                                        gmmu_new_pde_aperture_video_memory_f());
        pde_v[0] |= gmmu_new_pde_address_sys_f(u64_lo32(phys_addr));
        pde_v[0] |= gmmu_new_pde_vol_true_f();
        pde_v[1] |= phys_addr >> 24;
@@ -194,6 +196,7 @@ static void update_gmmu_pde0_locked(struct vm_gk20a *vm,
                        gmmu_new_dual_pde_address_small_sys_f(small_addr);
                pde_v[2] |= nvgpu_aperture_mask(g, pd->mem,
                        gmmu_new_dual_pde_aperture_small_sys_mem_ncoh_f(),
+                        gmmu_new_dual_pde_aperture_small_sys_mem_coh_f(),
                        gmmu_new_dual_pde_aperture_small_video_memory_f());
                pde_v[2] |= gmmu_new_dual_pde_vol_small_true_f();
                pde_v[3] |= small_addr >> 24;
@@ -204,6 +207,7 @@ static void update_gmmu_pde0_locked(struct vm_gk20a *vm,
                pde_v[0] |= gmmu_new_dual_pde_vol_big_true_f();
                pde_v[0] |= nvgpu_aperture_mask(g, pd->mem,
                        gmmu_new_dual_pde_aperture_big_sys_mem_ncoh_f(),
+                        gmmu_new_dual_pde_aperture_big_sys_mem_coh_f(),
                        gmmu_new_dual_pde_aperture_big_video_memory_f());
                pde_v[1] |= big_addr >> 28;
        }
@@ -240,11 +244,10 @@ static void __update_pte(struct vm_gk20a *vm,
                gmmu_new_pte_address_sys_f(phys_shifted) :
                gmmu_new_pte_address_vid_f(phys_shifted);
        u32 pte_tgt = __nvgpu_aperture_mask(g,
-                        attrs->aperture,
+                                        attrs->aperture,
-                        attrs->coherent ?
+                                        gmmu_new_pte_aperture_sys_mem_ncoh_f(),
-                                gmmu_new_pte_aperture_sys_mem_coh_f() :
+                                        gmmu_new_pte_aperture_sys_mem_coh_f(),
-                                gmmu_new_pte_aperture_sys_mem_ncoh_f(),
+                                        gmmu_new_pte_aperture_video_memory_f());
-                        gmmu_new_pte_aperture_video_memory_f());
        pte_w[0] = pte_valid | pte_addr | pte_tgt;
@@ -306,7 +309,7 @@ static void update_gmmu_pte_locked(struct vm_gk20a *vm,
                page_size >> 10,
                nvgpu_gmmu_perm_str(attrs->rw_flag),
                attrs->kind_v,
-                nvgpu_aperture_str(attrs->aperture),
+                nvgpu_aperture_str(g, attrs->aperture),
                attrs->cacheable ? 'C' : '-',
                attrs->sparse    ? 'S' : '-',
                attrs->priv      ? 'P' : '-',
@@ -428,8 +431,9 @@ void gp10b_mm_init_pdb(struct gk20a *g, struct nvgpu_mem *inst_block,
        nvgpu_mem_wr32(g, inst_block, ram_in_page_dir_base_lo_w(),
                nvgpu_aperture_mask(g, vm->pdb.mem,
-                  ram_in_page_dir_base_target_sys_mem_ncoh_f(),
+                                ram_in_page_dir_base_target_sys_mem_ncoh_f(),
-                  ram_in_page_dir_base_target_vid_mem_f()) |
+                                ram_in_page_dir_base_target_sys_mem_coh_f(),
+                                ram_in_page_dir_base_target_vid_mem_f()) |
                ram_in_page_dir_base_vol_true_f() |
                ram_in_big_page_size_64kb_f() |
                ram_in_page_dir_base_lo_f(pdb_addr_lo) |
diff --git a/drivers/gpu/nvgpu/gv11b/acr_gv11b.c b/drivers/gpu/nvgpu/gv11b/acr_gv11b.c
index 799b2db4..4fa3f324 100644
--- a/drivers/gpu/nvgpu/gv11b/acr_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/acr_gv11b.c
@@ -27,9 +27,10 @@
 #include <nvgpu/nvgpu_common.h>
 #include <nvgpu/kmem.h>
 #include <nvgpu/nvgpu_mem.h>
-#include <nvgpu/acr/nvgpu_acr.h>
 #include <nvgpu/firmware.h>
 #include <nvgpu/mm.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/acr/nvgpu_acr.h>
 #include "gk20a/gk20a.h"
 #include "acr_gv11b.h"
@@ -220,7 +221,9 @@ static int bl_bootstrap(struct nvgpu_pmu *pmu,
                pwr_pmu_new_instblk_ptr_f(
                nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> 12) |
                pwr_pmu_new_instblk_valid_f(1) |
-                pwr_pmu_new_instblk_target_sys_ncoh_f());
+                     (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) ?
+                      pwr_pmu_new_instblk_target_sys_coh_f() :
+                      pwr_pmu_new_instblk_target_sys_ncoh_f())) ;
        /*copy bootloader interface structure to dmem*/
        nvgpu_flcn_copy_to_dmem(pmu->flcn, 0, (u8 *)pbl_desc,
diff --git a/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c
index 617ea61d..86977bb3 100644
--- a/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c
@@ -31,14 +31,14 @@
 #include <nvgpu/dma.h>
 #include <nvgpu/mm.h>
 #include <nvgpu/sizes.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/log.h>
+#include <nvgpu/bug.h>
 #include "gk20a/gk20a.h"
 #include "gk20a/css_gr_gk20a.h"
 #include "css_gr_gv11b.h"
-#include <nvgpu/log.h>
-#include <nvgpu/bug.h>
 #include <nvgpu/hw/gv11b/hw_perf_gv11b.h>
 #include <nvgpu/hw/gv11b/hw_mc_gv11b.h>
@@ -144,6 +144,7 @@ int gv11b_css_hw_enable_snapshot(struct channel_gk20a *ch,
                        perf_pmasys_mem_block_valid_true_f() |
                        nvgpu_aperture_mask(g, &g->mm.hwpm.inst_block,
                                perf_pmasys_mem_block_target_sys_ncoh_f(),
+                                perf_pmasys_mem_block_target_sys_coh_f(),
                                perf_pmasys_mem_block_target_lfb_f()));
diff --git a/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c
index e5d88e8c..562476ca 100644
--- a/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c
@@ -59,11 +59,12 @@ int gv11b_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size)
        inst_pa_page = nvgpu_inst_block_addr(g, &mm->perfbuf.inst_block) >> 12;
        gk20a_writel(g, perf_pmasys_mem_block_r(),
-                        perf_pmasys_mem_block_base_f(inst_pa_page) |
+                     perf_pmasys_mem_block_base_f(inst_pa_page) |
-                        perf_pmasys_mem_block_valid_true_f() |
+                     perf_pmasys_mem_block_valid_true_f() |
-                        nvgpu_aperture_mask(g, &mm->perfbuf.inst_block,
+                     nvgpu_aperture_mask(g, &mm->perfbuf.inst_block,
-+                               perf_pmasys_mem_block_target_sys_ncoh_f(),
+                                perf_pmasys_mem_block_target_sys_ncoh_f(),
-+                               perf_pmasys_mem_block_target_lfb_f()));
+                                perf_pmasys_mem_block_target_sys_coh_f(),
+                                perf_pmasys_mem_block_target_lfb_f()));
        gk20a_idle(g);
        return 0;
diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
index 97ab7aab..6b4b07a6 100644
--- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
@@ -101,12 +101,14 @@ void gv11b_get_ch_runlist_entry(struct channel_gk20a *c, u32 *runlist)
                                                c->runqueue_sel) |
                        ram_rl_entry_chan_userd_target_f(
                                nvgpu_aperture_mask(g, &g->fifo.userd,
-                                ram_rl_entry_chan_userd_target_sys_mem_ncoh_v(),
+                                        ram_rl_entry_chan_userd_target_sys_mem_ncoh_v(),
-                                ram_rl_entry_chan_userd_target_vid_mem_v())) |
+                                        ram_rl_entry_chan_userd_target_sys_mem_coh_v(),
+                                        ram_rl_entry_chan_userd_target_vid_mem_v())) |
                        ram_rl_entry_chan_inst_target_f(
                                nvgpu_aperture_mask(g, &c->inst_block,
-                                ram_rl_entry_chan_inst_target_sys_mem_ncoh_v(),
+                                        ram_rl_entry_chan_inst_target_sys_mem_ncoh_v(),
-                                ram_rl_entry_chan_inst_target_vid_mem_v()));
+                                        ram_rl_entry_chan_inst_target_sys_mem_coh_v(),
+                                        ram_rl_entry_chan_inst_target_vid_mem_v()));
        addr_lo = u64_lo32(c->userd_iova) >>
                        ram_rl_entry_chan_userd_ptr_align_shift_v();
diff --git a/drivers/gpu/nvgpu/gv11b/mm_gv11b.c b/drivers/gpu/nvgpu/gv11b/mm_gv11b.c
index ade1d9fe..b46ecb0a 100644
--- a/drivers/gpu/nvgpu/gv11b/mm_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/mm_gv11b.c
@@ -26,6 +26,7 @@
 #include <nvgpu/dma.h>
 #include <nvgpu/log.h>
 #include <nvgpu/mm.h>
+#include <nvgpu/enabled.h>
 #include "gk20a/gk20a.h"
 #include "gk20a/mm_gk20a.h"
@@ -292,8 +293,9 @@ int gv11b_init_bar2_mm_hw_setup(struct gk20a *g)
        gk20a_writel(g, bus_bar2_block_r(),
                     nvgpu_aperture_mask(g, inst_block,
-                                bus_bar2_block_target_sys_mem_ncoh_f(),
+                                         bus_bar2_block_target_sys_mem_ncoh_f(),
-                                bus_bar2_block_target_vid_mem_f()) |
+                                         bus_bar2_block_target_sys_mem_coh_f(),
+                                         bus_bar2_block_target_vid_mem_f()) |
                     bus_bar2_block_mode_virtual_f() |
                     bus_bar2_block_ptr_f(inst_pa));
diff --git a/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c
index 7dd4f8f4..13e70eca 100644
--- a/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c
@@ -195,9 +195,11 @@ int gv11b_pmu_bootstrap(struct nvgpu_pmu *pmu)
        gk20a_writel(g, pwr_pmu_new_instblk_r(),
                pwr_pmu_new_instblk_ptr_f(
-                nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> ALIGN_4KB)
+                nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> ALIGN_4KB) |
-                | pwr_pmu_new_instblk_valid_f(1)
+                     pwr_pmu_new_instblk_valid_f(1) |
-                | pwr_pmu_new_instblk_target_sys_ncoh_f());
+                     (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) ?
+                      pwr_pmu_new_instblk_target_sys_coh_f() :
+                      pwr_pmu_new_instblk_target_sys_ncoh_f()));
        /* TBD: load all other surfaces */
        g->ops.pmu_ver.set_pmu_cmdline_args_trace_size(
diff --git a/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c
index 05d7dee0..bda4c8e4 100644
--- a/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c
@@ -177,8 +177,9 @@ void gv11b_subctx_commit_pdb(struct vm_gk20a *vm,
        u32 pdb_addr_lo, pdb_addr_hi;
        u64 pdb_addr;
        u32 aperture = nvgpu_aperture_mask(g, vm->pdb.mem,
-                        ram_in_sc_page_dir_base_target_sys_mem_ncoh_v(),
+                                ram_in_sc_page_dir_base_target_sys_mem_ncoh_v(),
-                        ram_in_sc_page_dir_base_target_vid_mem_v());
+                                ram_in_sc_page_dir_base_target_sys_mem_coh_v(),
+                                ram_in_sc_page_dir_base_target_vid_mem_v());
        pdb_addr = nvgpu_mem_get_addr(g, vm->pdb.mem);
        pdb_addr_lo = u64_lo32(pdb_addr >> ram_in_base_shift_v());
diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvgpu_mem.h b/drivers/gpu/nvgpu/include/nvgpu/nvgpu_mem.h
index 2b8b7015..f1ab8a6e 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/nvgpu_mem.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/nvgpu_mem.h
@@ -25,6 +25,7 @@
 #include <nvgpu/types.h>
 #include <nvgpu/list.h>
+#include <nvgpu/enabled.h>
 #ifdef __KERNEL__
 #include <nvgpu/linux/nvgpu_mem.h>
@@ -51,6 +52,10 @@ struct nvgpu_page_alloc;
 enum nvgpu_aperture {
        APERTURE_INVALID = 0, /* unallocated or N/A */
        APERTURE_SYSMEM,
+         /* Don't use directly. Use APERTURE_SYSMEM, this is used internally. */
+        __APERTURE_SYSMEM_COH,
        APERTURE_VIDMEM
 };
@@ -187,12 +192,18 @@ nvgpu_mem_from_clear_list_entry(struct nvgpu_list_node *node)
                                            clear_list_entry));
 };
-static inline const char *nvgpu_aperture_str(enum nvgpu_aperture aperture)
+static inline const char *nvgpu_aperture_str(struct gk20a *g,
+                                             enum nvgpu_aperture aperture)
 {
        switch (aperture) {
-                case APERTURE_INVALID: return "INVAL";
+        case APERTURE_INVALID:
-                case APERTURE_SYSMEM:  return "SYSMEM";
+                return "INVAL";
-                case APERTURE_VIDMEM:  return "VIDMEM";
+        case APERTURE_SYSMEM:
+                return "SYSMEM";
+        case __APERTURE_SYSMEM_COH:
+                return "SYSCOH";
+        case APERTURE_VIDMEM:
+                return "VIDMEM";
        };
        return "UNKNOWN";
 }
@@ -322,9 +333,9 @@ u64 nvgpu_mem_get_addr(struct gk20a *g, struct nvgpu_mem *mem);
 u64 nvgpu_mem_get_phys_addr(struct gk20a *g, struct nvgpu_mem *mem);
 u32 __nvgpu_aperture_mask(struct gk20a *g, enum nvgpu_aperture aperture,
-                u32 sysmem_mask, u32 vidmem_mask);
+                u32 sysmem_mask, u32 sysmem_coh_mask, u32 vidmem_mask);
 u32 nvgpu_aperture_mask(struct gk20a *g, struct nvgpu_mem *mem,
-                u32 sysmem_mask, u32 vidmem_mask);
+                u32 sysmem_mask, u32 sysmem_coh_mask, u32 vidmem_mask);
 u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys);