11 files changed, 149 insertions, 21 deletions

diff --git a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
index ade6ff0a..9cb26d3f 100644
--- a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
@@ -53,6 +53,7 @@ static int channel_gp10b_commit_userd(struct channel_gk20a *c)
         u32 addr_lo;
         u32 addr_hi;
         void *inst_ptr;
+        struct gk20a *g = c->g;
 
         gk20a_dbg_fn("");
 
@@ -67,12 +68,13 @@ static int channel_gp10b_commit_userd(struct channel_gk20a *c)
                 c->hw_chid, (u64)c->userd_iova);
 
         gk20a_mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_w(),
-                 pbdma_userd_target_vid_mem_f() |
-                 pbdma_userd_addr_f(addr_lo));
+                       (g->mm.vidmem_is_vidmem ?
+                        pbdma_userd_target_sys_mem_ncoh_f() :
+                        pbdma_userd_target_vid_mem_f()) |
+                       pbdma_userd_addr_f(addr_lo));
 
         gk20a_mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_hi_w(),
-                 pbdma_userd_target_vid_mem_f() |
-                 pbdma_userd_hi_addr_f(addr_hi));
+                       pbdma_userd_hi_addr_f(addr_hi));
 
         return 0;
 }
diff --git a/drivers/gpu/nvgpu/gp10b/hw_bus_gp10b.h b/drivers/gpu/nvgpu/gp10b/hw_bus_gp10b.h
index c04b01c1..bf251cd6 100644
--- a/drivers/gpu/nvgpu/gp10b/hw_bus_gp10b.h
+++ b/drivers/gpu/nvgpu/gp10b/hw_bus_gp10b.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -62,6 +62,14 @@ static inline u32 bus_bar1_block_target_vid_mem_f(void)
 {
         return 0x0;
 }
+static inline u32 bus_bar1_block_target_sys_mem_coh_f(void)
+{
+        return 0x20000000;
+}
+static inline u32 bus_bar1_block_target_sys_mem_ncoh_f(void)
+{
+        return 0x30000000;
+}
 static inline u32 bus_bar1_block_mode_virtual_f(void)
 {
         return 0x80000000;
@@ -78,6 +86,14 @@ static inline u32 bus_bar2_block_target_vid_mem_f(void)
 {
         return 0x0;
 }
+static inline u32 bus_bar2_block_target_sys_mem_coh_f(void)
+{
+        return 0x20000000;
+}
+static inline u32 bus_bar2_block_target_sys_mem_ncoh_f(void)
+{
+        return 0x30000000;
+}
 static inline u32 bus_bar2_block_mode_virtual_f(void)
 {
         return 0x80000000;
diff --git a/drivers/gpu/nvgpu/gp10b/hw_ccsr_gp10b.h b/drivers/gpu/nvgpu/gp10b/hw_ccsr_gp10b.h
index cd5265b3..99398961 100644
--- a/drivers/gpu/nvgpu/gp10b/hw_ccsr_gp10b.h
+++ b/drivers/gpu/nvgpu/gp10b/hw_ccsr_gp10b.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -66,6 +66,14 @@ static inline u32 ccsr_channel_inst_target_vid_mem_f(void)
 {
         return 0x0;
 }
+static inline u32 ccsr_channel_inst_target_sys_mem_coh_f(void)
+{
+        return 0x20000000;
+}
+static inline u32 ccsr_channel_inst_target_sys_mem_ncoh_f(void)
+{
+        return 0x30000000;
+}
 static inline u32 ccsr_channel_inst_bind_false_f(void)
 {
         return 0x0;
diff --git a/drivers/gpu/nvgpu/gp10b/hw_fb_gp10b.h b/drivers/gpu/nvgpu/gp10b/hw_fb_gp10b.h
index 1fee19b1..236c2f8a 100644
--- a/drivers/gpu/nvgpu/gp10b/hw_fb_gp10b.h
+++ b/drivers/gpu/nvgpu/gp10b/hw_fb_gp10b.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -106,6 +106,10 @@ static inline u32 fb_mmu_invalidate_pdb_aperture_vid_mem_f(void)
 {
         return 0x0;
 }
+static inline u32 fb_mmu_invalidate_pdb_aperture_sys_mem_f(void)
+{
+        return 0x2;
+}
 static inline u32 fb_mmu_invalidate_pdb_addr_f(u32 v)
 {
         return (v & 0xfffffff) << 4;
@@ -378,6 +382,14 @@ static inline u32 fb_mmu_debug_wr_aperture_vid_mem_f(void)
 {
         return 0x0;
 }
+static inline u32 fb_mmu_debug_wr_aperture_sys_mem_coh_f(void)
+{
+        return 0x2;
+}
+static inline u32 fb_mmu_debug_wr_aperture_sys_mem_ncoh_f(void)
+{
+        return 0x3;
+}
 static inline u32 fb_mmu_debug_wr_vol_false_f(void)
 {
         return 0x0;
@@ -406,6 +418,14 @@ static inline u32 fb_mmu_debug_rd_aperture_vid_mem_f(void)
 {
         return 0x0;
 }
+static inline u32 fb_mmu_debug_rd_aperture_sys_mem_coh_f(void)
+{
+        return 0x2;
+}
+static inline u32 fb_mmu_debug_rd_aperture_sys_mem_ncoh_f(void)
+{
+        return 0x3;
+}
 static inline u32 fb_mmu_debug_rd_vol_false_f(void)
 {
         return 0x0;
diff --git a/drivers/gpu/nvgpu/gp10b/hw_fifo_gp10b.h b/drivers/gpu/nvgpu/gp10b/hw_fifo_gp10b.h
index 34977523..4781ff85 100644
--- a/drivers/gpu/nvgpu/gp10b/hw_fifo_gp10b.h
+++ b/drivers/gpu/nvgpu/gp10b/hw_fifo_gp10b.h
@@ -82,6 +82,14 @@ static inline u32 fifo_runlist_base_target_vid_mem_f(void)
 {
         return 0x0;
 }
+static inline u32 fifo_runlist_base_target_sys_mem_coh_f(void)
+{
+        return 0x20000000;
+}
+static inline u32 fifo_runlist_base_target_sys_mem_ncoh_f(void)
+{
+        return 0x30000000;
+}
 static inline u32 fifo_runlist_r(void)
 {
         return 0x00002274;
diff --git a/drivers/gpu/nvgpu/gp10b/hw_gmmu_gp10b.h b/drivers/gpu/nvgpu/gp10b/hw_gmmu_gp10b.h
index 30e4307d..86870aea 100644
--- a/drivers/gpu/nvgpu/gp10b/hw_gmmu_gp10b.h
+++ b/drivers/gpu/nvgpu/gp10b/hw_gmmu_gp10b.h
@@ -70,6 +70,14 @@ static inline u32 gmmu_new_pde_aperture_video_memory_f(void)
 {
         return 0x2;
 }
+static inline u32 gmmu_new_pde_aperture_sys_mem_coh_f(void)
+{
+        return 0x4;
+}
+static inline u32 gmmu_new_pde_aperture_sys_mem_ncoh_f(void)
+{
+        return 0x6;
+}
 static inline u32 gmmu_new_pde_address_sys_f(u32 v)
 {
         return (v & 0xfffffff) << 8;
@@ -118,6 +126,14 @@ static inline u32 gmmu_new_dual_pde_aperture_big_video_memory_f(void)
 {
         return 0x2;
 }
+static inline u32 gmmu_new_dual_pde_aperture_big_sys_mem_coh_f(void)
+{
+        return 0x4;
+}
+static inline u32 gmmu_new_dual_pde_aperture_big_sys_mem_ncoh_f(void)
+{
+        return 0x6;
+}
 static inline u32 gmmu_new_dual_pde_address_big_sys_f(u32 v)
 {
         return (v & 0xfffffff) << 4;
@@ -138,6 +154,14 @@ static inline u32 gmmu_new_dual_pde_aperture_small_video_memory_f(void)
 {
         return 0x2;
 }
+static inline u32 gmmu_new_dual_pde_aperture_small_sys_mem_coh_f(void)
+{
+        return 0x4;
+}
+static inline u32 gmmu_new_dual_pde_aperture_small_sys_mem_ncoh_f(void)
+{
+        return 0x6;
+}
 static inline u32 gmmu_new_dual_pde_vol_small_w(void)
 {
         return 2;
@@ -238,6 +262,14 @@ static inline u32 gmmu_new_pte_aperture_video_memory_f(void)
 {
         return 0x0;
 }
+static inline u32 gmmu_new_pte_aperture_sys_mem_coh_f(void)
+{
+        return 0x4;
+}
+static inline u32 gmmu_new_pte_aperture_sys_mem_ncoh_f(void)
+{
+        return 0x6;
+}
 static inline u32 gmmu_new_pte_read_only_w(void)
 {
         return 0;
diff --git a/drivers/gpu/nvgpu/gp10b/hw_gr_gp10b.h b/drivers/gpu/nvgpu/gp10b/hw_gr_gp10b.h
index 78792f50..78304fb1 100644
--- a/drivers/gpu/nvgpu/gp10b/hw_gr_gp10b.h
+++ b/drivers/gpu/nvgpu/gp10b/hw_gr_gp10b.h
@@ -1010,6 +1010,14 @@ static inline u32 gr_fecs_current_ctx_target_vid_mem_f(void)
 {
         return 0x0;
 }
+static inline u32 gr_fecs_current_ctx_target_sys_mem_coh_f(void)
+{
+        return 0x20000000;
+}
+static inline u32 gr_fecs_current_ctx_target_sys_mem_ncoh_f(void)
+{
+        return 0x30000000;
+}
 static inline u32 gr_fecs_current_ctx_valid_s(void)
 {
         return 1;
diff --git a/drivers/gpu/nvgpu/gp10b/hw_pbdma_gp10b.h b/drivers/gpu/nvgpu/gp10b/hw_pbdma_gp10b.h
index f66fa934..0caffb22 100644
--- a/drivers/gpu/nvgpu/gp10b/hw_pbdma_gp10b.h
+++ b/drivers/gpu/nvgpu/gp10b/hw_pbdma_gp10b.h
@@ -322,6 +322,14 @@ static inline u32 pbdma_userd_target_vid_mem_f(void)
 {
         return 0x0;
 }
+static inline u32 pbdma_userd_target_sys_mem_coh_f(void)
+{
+        return 0x2;
+}
+static inline u32 pbdma_userd_target_sys_mem_ncoh_f(void)
+{
+        return 0x3;
+}
 static inline u32 pbdma_userd_addr_f(u32 v)
 {
         return (v & 0x7fffff) << 9;
diff --git a/drivers/gpu/nvgpu/gp10b/hw_pwr_gp10b.h b/drivers/gpu/nvgpu/gp10b/hw_pwr_gp10b.h
index 75bf59a3..9a3591c7 100644
--- a/drivers/gpu/nvgpu/gp10b/hw_pwr_gp10b.h
+++ b/drivers/gpu/nvgpu/gp10b/hw_pwr_gp10b.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -542,6 +542,10 @@ static inline u32 pwr_pmu_new_instblk_target_sys_coh_f(void)
 {
         return 0x20000000;
 }
+static inline u32 pwr_pmu_new_instblk_target_sys_ncoh_f(void)
+{
+        return 0x30000000;
+}
 static inline u32 pwr_pmu_new_instblk_valid_f(u32 v)
 {
         return (v & 0x1) << 30;
diff --git a/drivers/gpu/nvgpu/gp10b/hw_ram_gp10b.h b/drivers/gpu/nvgpu/gp10b/hw_ram_gp10b.h
index 863b15b8..55323579 100644
--- a/drivers/gpu/nvgpu/gp10b/hw_ram_gp10b.h
+++ b/drivers/gpu/nvgpu/gp10b/hw_ram_gp10b.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -70,6 +70,14 @@ static inline u32 ram_in_page_dir_base_target_vid_mem_f(void)
 {
         return 0x0;
 }
+static inline u32 ram_in_page_dir_base_target_sys_mem_coh_f(void)
+{
+        return 0x2;
+}
+static inline u32 ram_in_page_dir_base_target_sys_mem_ncoh_f(void)
+{
+        return 0x3;
+}
 static inline u32 ram_in_page_dir_base_vol_w(void)
 {
         return 128;
diff --git a/drivers/gpu/nvgpu/gp10b/mm_gp10b.c b/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
index deb8c138..0c00feb4 100644
--- a/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
@@ -43,9 +43,11 @@ static int gp10b_init_mm_setup_hw(struct gk20a *g)
         gk20a_dbg_info("bar1 inst block ptr: 0x%08x",  (u32)inst_pa);
 
         gk20a_writel(g, bus_bar1_block_r(),
-                        bus_bar1_block_target_vid_mem_f() |
-                        bus_bar1_block_mode_virtual_f() |
-                        bus_bar1_block_ptr_f(inst_pa));
+                     (g->mm.vidmem_is_vidmem ?
+                       bus_bar1_block_target_sys_mem_ncoh_f() :
+                       bus_bar1_block_target_vid_mem_f()) |
+                     bus_bar1_block_mode_virtual_f() |
+                     bus_bar1_block_ptr_f(inst_pa));
 
         if (g->ops.mm.init_bar2_mm_hw_setup) {
                 err = g->ops.mm.init_bar2_mm_hw_setup(g);
@@ -107,9 +109,11 @@ static int gb10b_init_bar2_mm_hw_setup(struct gk20a *g)
         gk20a_dbg_info("bar2 inst block ptr: 0x%08x",  (u32)inst_pa);
 
         gk20a_writel(g, bus_bar2_block_r(),
-                bus_bar2_block_target_vid_mem_f() |
-                bus_bar2_block_mode_virtual_f() |
-                bus_bar2_block_ptr_f(inst_pa));
+                     (g->mm.vidmem_is_vidmem ?
+                       bus_bar2_block_target_sys_mem_ncoh_f() :
+                       bus_bar2_block_target_vid_mem_f()) |
+                     bus_bar2_block_mode_virtual_f() |
+                     bus_bar2_block_ptr_f(inst_pa));
 
         gk20a_dbg_fn("done");
         return 0;
@@ -179,7 +183,9 @@ static int update_gmmu_pde3_locked(struct vm_gk20a *vm,
         pte_addr = entry_addr(g, pte) >> gmmu_new_pde_address_shift_v();
         pde_addr = entry_addr(g, parent);
 
-        pde_v[0] |= gmmu_new_pde_aperture_video_memory_f();
+        pde_v[0] |= g->mm.vidmem_is_vidmem ?
+                        gmmu_new_pde_aperture_sys_mem_ncoh_f() :
+                        gmmu_new_pde_aperture_video_memory_f();
         pde_v[0] |= gmmu_new_pde_address_sys_f(u64_lo32(pte_addr));
         pde_v[0] |= gmmu_new_pde_vol_true_f();
         pde_v[1] |= pte_addr >> 24;
@@ -232,7 +238,9 @@ static int update_gmmu_pde0_locked(struct vm_gk20a *vm,
 
         if (small_valid) {
                 pde_v[2] |= gmmu_new_dual_pde_address_small_sys_f(pte_addr_small);
-                pde_v[2] |= gmmu_new_dual_pde_aperture_small_video_memory_f();
+                pde_v[2] |= g->mm.vidmem_is_vidmem ?
+                        gmmu_new_dual_pde_aperture_small_sys_mem_ncoh_f() :
+                        gmmu_new_dual_pde_aperture_small_video_memory_f();
                 pde_v[2] |= gmmu_new_dual_pde_vol_small_true_f();
                 pde_v[3] |= pte_addr_small >> 24;
         }
@@ -240,7 +248,9 @@ static int update_gmmu_pde0_locked(struct vm_gk20a *vm,
         if (big_valid) {
                 pde_v[0] |= gmmu_new_dual_pde_address_big_sys_f(pte_addr_big);
                 pde_v[0] |= gmmu_new_dual_pde_vol_big_true_f();
-                pde_v[0] |= gmmu_new_dual_pde_aperture_big_video_memory_f();
+                pde_v[0] |= g->mm.vidmem_is_vidmem ?
+                        gmmu_new_dual_pde_aperture_big_sys_mem_ncoh_f() :
+                        gmmu_new_dual_pde_aperture_big_video_memory_f();
                 pde_v[1] |= pte_addr_big >> 28;
         }
 
@@ -279,8 +289,10 @@ static int update_gmmu_pte_locked(struct vm_gk20a *vm,
                         pte_w[0] = gmmu_new_pte_valid_false_f();
                 else
                         pte_w[0] = gmmu_new_pte_valid_true_f();
-                pte_w[0] |= gmmu_new_pte_aperture_video_memory_f() |
-                            gmmu_new_pte_address_sys_f(*iova
+                pte_w[0] |= g->mm.vidmem_is_vidmem ?
+                        gmmu_new_pte_aperture_sys_mem_ncoh_f() :
+                        gmmu_new_pte_aperture_video_memory_f();
+                pte_w[0] |= gmmu_new_pte_address_sys_f(*iova
                               >> gmmu_new_pte_address_shift_v());
 
                 if (priv)
@@ -373,7 +385,9 @@ static void gp10b_mm_init_pdb(struct gk20a *g, void *inst_ptr, u64 pdb_addr)
         u32 pdb_addr_hi = u64_hi32(pdb_addr);
 
         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
-                ram_in_page_dir_base_target_vid_mem_f() |
+                (g->mm.vidmem_is_vidmem ?
+                  ram_in_page_dir_base_target_sys_mem_ncoh_f() :
+                  ram_in_page_dir_base_target_vid_mem_f()) |
                 ram_in_page_dir_base_vol_true_f() |
                 ram_in_page_dir_base_lo_f(pdb_addr_lo) |
                 1 << 10);