gpu: nvgpu: Do not use G_ELPG_FLUSH

G_ELPG_FLUSH is protected in some chips. Use L2 flush operations
instead.

Bug 1698618

Change-Id: I984a8ace8bcd0ad2d4a4e2d63af75a342bdeb75a
Signed-off-by: Terje Bergstrom <tbergstrom@nvidia.com>
Reviewed-on: http://git-master/r/828656
(cherry picked from commit ba9075fa43975112a221d37d246f0b8f5af40fab)
Reviewed-on: http://git-master/r/829415

6 files changed, 129 insertions, 3 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index 58c8e9ad..e43e58a0 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -376,6 +376,7 @@ struct gpu_ops {
                 int (*fb_flush)(struct gk20a *g);
                 void (*l2_invalidate)(struct gk20a *g);
                 void (*l2_flush)(struct gk20a *g, bool invalidate);
+                void (*cbc_clean)(struct gk20a *g);
                 void (*tlb_invalidate)(struct vm_gk20a *vm);
                 void (*set_big_page_size)(struct gk20a *g,
                                           void *inst_ptr, int size);
diff --git a/drivers/gpu/nvgpu/gk20a/hw_flush_gk20a.h b/drivers/gpu/nvgpu/gk20a/hw_flush_gk20a.h
index 0aeb11f9..9cd91fad 100644
--- a/drivers/gpu/nvgpu/gk20a/hw_flush_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/hw_flush_gk20a.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012-2013, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2012-2015, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -114,6 +114,46 @@ static inline u32 flush_l2_flush_dirty_outstanding_true_v(void)
 {
         return 0x00000001;
 }
+static inline u32 flush_l2_clean_comptags_r(void)
+{
+        return 0x0007000c;
+}
+static inline u32 flush_l2_clean_comptags_pending_v(u32 r)
+{
+        return (r >> 0) & 0x1;
+}
+static inline u32 flush_l2_clean_comptags_pending_empty_v(void)
+{
+        return 0x00000000;
+}
+static inline u32 flush_l2_clean_comptags_pending_empty_f(void)
+{
+        return 0x0;
+}
+static inline u32 flush_l2_clean_comptags_pending_busy_v(void)
+{
+        return 0x00000001;
+}
+static inline u32 flush_l2_clean_comptags_pending_busy_f(void)
+{
+        return 0x1;
+}
+static inline u32 flush_l2_clean_comptags_outstanding_v(u32 r)
+{
+        return (r >> 1) & 0x1;
+}
+static inline u32 flush_l2_clean_comptags_outstanding_false_v(void)
+{
+        return 0x00000000;
+}
+static inline u32 flush_l2_clean_comptags_outstanding_false_f(void)
+{
+        return 0x0;
+}
+static inline u32 flush_l2_clean_comptags_outstanding_true_v(void)
+{
+        return 0x00000001;
+}
 static inline u32 flush_fb_flush_r(void)
 {
         return 0x00070000;
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
index 859e46fc..15fd32d3 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
@@ -3460,6 +3460,47 @@ hw_was_off:
         pm_runtime_put_noidle(&g->dev->dev);
 }
 
+void gk20a_mm_cbc_clean(struct gk20a *g)
+{
+        struct mm_gk20a *mm = &g->mm;
+        u32 data;
+        s32 retry = 200;
+
+        gk20a_dbg_fn("");
+
+        gk20a_busy_noresume(g->dev);
+        if (!g->power_on)
+                goto hw_was_off;
+
+        mutex_lock(&mm->l2_op_lock);
+
+        /* Flush all dirty lines from the CBC to L2 */
+        gk20a_writel(g, flush_l2_clean_comptags_r(),
+                flush_l2_clean_comptags_pending_busy_f());
+
+        do {
+                data = gk20a_readl(g, flush_l2_clean_comptags_r());
+
+                if (flush_l2_clean_comptags_outstanding_v(data) ==
+                        flush_l2_clean_comptags_outstanding_true_v() ||
+                    flush_l2_clean_comptags_pending_v(data) ==
+                        flush_l2_clean_comptags_pending_busy_v()) {
+                                gk20a_dbg_info("l2_clean_comptags 0x%x", data);
+                                retry--;
+                                udelay(5);
+                } else
+                        break;
+        } while (retry >= 0 || !tegra_platform_is_silicon());
+
+        if (tegra_platform_is_silicon() && retry < 0)
+                gk20a_warn(dev_from_gk20a(g),
+                        "l2_clean_comptags too many retries");
+
+        mutex_unlock(&mm->l2_op_lock);
+
+hw_was_off:
+        pm_runtime_put_noidle(&g->dev->dev);
+}
 
 int gk20a_vm_find_buffer(struct vm_gk20a *vm, u64 gpu_va,
                          struct dma_buf **dmabuf,
@@ -3555,7 +3596,8 @@ int gk20a_mm_suspend(struct gk20a *g)
 {
         gk20a_dbg_fn("");
 
-        g->ops.ltc.elpg_flush(g);
+        g->ops.mm.cbc_clean(g);
+        g->ops.mm.l2_flush(g, false);
 
         gk20a_dbg_fn("done");
         return 0;
@@ -3591,6 +3633,7 @@ void gk20a_init_mm(struct gpu_ops *gops)
         gops->mm.fb_flush = gk20a_mm_fb_flush;
         gops->mm.l2_invalidate = gk20a_mm_l2_invalidate;
         gops->mm.l2_flush = gk20a_mm_l2_flush;
+        gops->mm.cbc_clean = gk20a_mm_cbc_clean;
         gops->mm.tlb_invalidate = gk20a_mm_tlb_invalidate;
         gops->mm.get_iova_addr = gk20a_mm_iova_addr;
         gops->mm.get_physical_addr_bits = gk20a_mm_get_physical_addr_bits;
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
index e44ee631..9e373d8e 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
@@ -283,6 +283,7 @@ int gk20a_init_mm_setup_hw(struct gk20a *g);
 
 int gk20a_mm_fb_flush(struct gk20a *g);
 void gk20a_mm_l2_flush(struct gk20a *g, bool invalidate);
+void gk20a_mm_cbc_clean(struct gk20a *g);
 void gk20a_mm_l2_invalidate(struct gk20a *g);
 
 struct mm_gk20a {
diff --git a/drivers/gpu/nvgpu/gm20b/hw_flush_gm20b.h b/drivers/gpu/nvgpu/gm20b/hw_flush_gm20b.h
index a6d5e548..b77643be 100644
--- a/drivers/gpu/nvgpu/gm20b/hw_flush_gm20b.h
+++ b/drivers/gpu/nvgpu/gm20b/hw_flush_gm20b.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2015, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -114,6 +114,46 @@ static inline u32 flush_l2_flush_dirty_outstanding_true_v(void)
 {
         return 0x00000001;
 }
+static inline u32 flush_l2_clean_comptags_r(void)
+{
+        return 0x0007000c;
+}
+static inline u32 flush_l2_clean_comptags_pending_v(u32 r)
+{
+        return (r >> 0) & 0x1;
+}
+static inline u32 flush_l2_clean_comptags_pending_empty_v(void)
+{
+        return 0x00000000;
+}
+static inline u32 flush_l2_clean_comptags_pending_empty_f(void)
+{
+        return 0x0;
+}
+static inline u32 flush_l2_clean_comptags_pending_busy_v(void)
+{
+        return 0x00000001;
+}
+static inline u32 flush_l2_clean_comptags_pending_busy_f(void)
+{
+        return 0x1;
+}
+static inline u32 flush_l2_clean_comptags_outstanding_v(u32 r)
+{
+        return (r >> 1) & 0x1;
+}
+static inline u32 flush_l2_clean_comptags_outstanding_false_v(void)
+{
+        return 0x00000000;
+}
+static inline u32 flush_l2_clean_comptags_outstanding_false_f(void)
+{
+        return 0x0;
+}
+static inline u32 flush_l2_clean_comptags_outstanding_true_v(void)
+{
+        return 0x00000001;
+}
 static inline u32 flush_fb_flush_r(void)
 {
         return 0x00070000;
diff --git a/drivers/gpu/nvgpu/gm20b/mm_gm20b.c b/drivers/gpu/nvgpu/gm20b/mm_gm20b.c
index cf7ae46b..3d75a631 100644
--- a/drivers/gpu/nvgpu/gm20b/mm_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/mm_gm20b.c
@@ -120,6 +120,7 @@ void gm20b_init_mm(struct gpu_ops *gops)
         gops->mm.fb_flush = gk20a_mm_fb_flush;
         gops->mm.l2_invalidate = gk20a_mm_l2_invalidate;
         gops->mm.l2_flush = gk20a_mm_l2_flush;
+        gops->mm.cbc_clean = gk20a_mm_cbc_clean;
         gops->mm.tlb_invalidate = gk20a_mm_tlb_invalidate;
         gops->mm.set_big_page_size = gm20b_mm_set_big_page_size;
         gops->mm.get_big_page_sizes = gm20b_mm_get_big_page_sizes;