gpu: nvgpu: cache cde compbits buf mappings

don't unmap compbits_buf explicitly from system vm early but store it in
the dmabuf's private data, and unmap it later when all user mappings to
that buffer have been disappeared.

Bug 1546619

Change-Id: I333235a0ea74c48503608afac31f5e9f1eb4b99b
Signed-off-by: Konsta Holtta <kholtta@nvidia.com>
Reviewed-on: http://git-master/r/661949
(cherry picked from commit ed2177e25d9e5facfb38786b818330798a14b9bb)
Reviewed-on: http://git-master/r/661835
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Tested-by: Terje Bergstrom <tbergstrom@nvidia.com>

3 files changed, 91 insertions, 15 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/cde_gk20a.c b/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
index 57283343..49a1c1a8 100644
--- a/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
@@ -1,7 +1,7 @@
 /*
  * Color decompression engine support
  *
- * Copyright (c) 2014, NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2014-2015, NVIDIA Corporation.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -979,17 +979,23 @@ __releases(&cde_app->mutex)
                 map_size = compbits_buf->size - map_offset;
         }
 
-        /* map the destination buffer */
-        get_dma_buf(compbits_buf); /* a ref for gk20a_vm_map */
-        map_vaddr = gk20a_vm_map(cde_ctx->vm, compbits_buf, 0,
-                                 NVGPU_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
-                                 compbits_kind, NULL, true,
-                                 gk20a_mem_flag_none,
-                                 map_offset, map_size);
+        /* map the destination buffer, if not cached yet */
+        /* races protected by the cde app mutex above */
+        map_vaddr = gk20a_vm_cde_mapped(cde_ctx->vm, compbits_buf);
         if (!map_vaddr) {
-                dma_buf_put(compbits_buf);
-                err = -EINVAL;
-                goto exit_unlock;
+                /* take a ref for gk20a_vm_map, pair is in (cached) unmap */
+                get_dma_buf(compbits_buf);
+                map_vaddr = gk20a_vm_map(cde_ctx->vm, compbits_buf, 0,
+                                NVGPU_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
+                                compbits_kind, NULL, true,
+                                gk20a_mem_flag_none,
+                                map_offset, map_size);
+                if (!map_vaddr) {
+                        dma_buf_put(compbits_buf);
+                        err = -EINVAL;
+                        goto exit_unlock;
+                }
+                gk20a_vm_mark_cde_mapped(cde_ctx->vm, compbits_buf, map_vaddr);
         }
 
         /* store source buffer compression tags */
@@ -1052,9 +1058,7 @@ __releases(&cde_app->mutex)
 
 exit_unlock:
 
-        /* unmap the buffers - channel holds references to them now */
-        if (map_vaddr)
-                gk20a_vm_unmap(cde_ctx->vm, map_vaddr);
+        /* leave map_vaddr mapped - released when unmapped from userspace */
 
         mutex_unlock(&g->cde_app.mutex);
         return err;
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
index d8bd3e70..79bfb687 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
@@ -117,6 +117,11 @@ struct gk20a_dmabuf_priv {
         int pin_count;
 
         struct list_head states;
+
+        /* cached cde compbits buf */
+        struct vm_gk20a *cde_vm;
+        u64 cde_map_vaddr;
+        int map_count;
 };
 
 static void gk20a_vm_remove_support_nofree(struct vm_gk20a *vm);
@@ -198,6 +203,60 @@ void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf,
         mutex_unlock(&priv->lock);
 }
 
+/* CDE compbits buf caching: keep compbit buffer mapped during user mappings.
+ * Call these four only after dma_buf has a drvdata allocated */
+
+u64 gk20a_vm_cde_mapped(struct vm_gk20a *vm, struct dma_buf *dmabuf)
+{
+        struct device *dev = dev_from_vm(vm);
+        struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
+        u64 map_vaddr;
+
+        mutex_lock(&priv->lock);
+        map_vaddr = priv->cde_map_vaddr;
+        mutex_unlock(&priv->lock);
+
+        return map_vaddr;
+}
+
+void gk20a_vm_mark_cde_mapped(struct vm_gk20a *vm, struct dma_buf *dmabuf,
+                u64 map_vaddr)
+{
+        struct device *dev = dev_from_vm(vm);
+        struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
+
+        mutex_lock(&priv->lock);
+        priv->cde_vm = vm;
+        priv->cde_map_vaddr = map_vaddr;
+        mutex_unlock(&priv->lock);
+}
+
+static void gk20a_vm_inc_maps(struct vm_gk20a *vm, struct dma_buf *dmabuf)
+{
+        struct device *dev = dev_from_vm(vm);
+        struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
+
+        mutex_lock(&priv->lock);
+        priv->map_count++;
+        mutex_unlock(&priv->lock);
+}
+
+static void gk20a_vm_dec_maps(struct vm_gk20a *vm, struct dma_buf *dmabuf,
+                struct vm_gk20a **cde_vm, u64 *cde_map_vaddr)
+{
+        struct device *dev = dev_from_vm(vm);
+        struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
+
+        mutex_lock(&priv->lock);
+        if (--priv->map_count == 0) {
+                *cde_vm = priv->cde_vm;
+                *cde_map_vaddr = priv->cde_map_vaddr;
+                priv->cde_vm = NULL;
+                priv->cde_map_vaddr = 0;
+        }
+        mutex_unlock(&priv->lock);
+}
+
 void gk20a_get_comptags(struct device *dev, struct dma_buf *dmabuf,
                         struct gk20a_comptags *comptags)
 {
@@ -751,6 +810,8 @@ static void gk20a_vm_unmap_user(struct vm_gk20a *vm, u64 offset)
         struct device *d = dev_from_vm(vm);
         int retries;
         struct mapped_buffer_node *mapped_buffer;
+        struct vm_gk20a *cde_vm = NULL;
+        u64 cde_map_vaddr = 0;
 
         mutex_lock(&vm->update_gmmu_lock);
 
@@ -783,9 +844,15 @@ static void gk20a_vm_unmap_user(struct vm_gk20a *vm, u64 offset)
         mapped_buffer->user_mapped--;
         if (mapped_buffer->user_mapped == 0)
                 vm->num_user_mapped_buffers--;
+
+        gk20a_vm_dec_maps(vm, mapped_buffer->dmabuf, &cde_vm, &cde_map_vaddr);
+
         kref_put(&mapped_buffer->ref, gk20a_vm_unmap_locked_kref);
 
         mutex_unlock(&vm->update_gmmu_lock);
+
+        if (cde_map_vaddr)
+                gk20a_vm_unmap(cde_vm, cde_map_vaddr);
 }
 
 u64 gk20a_vm_alloc_va(struct vm_gk20a *vm,
@@ -2599,7 +2666,9 @@ int gk20a_vm_map_buffer(struct vm_gk20a *vm,
                         mapping_size);
 
         *offset_align = ret_va;
-        if (!ret_va) {
+        if (ret_va) {
+                gk20a_vm_inc_maps(vm, dmabuf);
+        } else {
                 dma_buf_put(dmabuf);
                 err = -EINVAL;
         }
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
index 42c164be..8f0f736b 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
@@ -554,6 +554,9 @@ void gk20a_deinit_vm(struct vm_gk20a *vm);
 int gk20a_vm_unmap_buffer(struct vm_gk20a *vm, u64 offset);
 void gk20a_get_comptags(struct device *dev, struct dma_buf *dmabuf,
                         struct gk20a_comptags *comptags);
+u64 gk20a_vm_cde_mapped(struct vm_gk20a *vm, struct dma_buf *dmabuf);
+void gk20a_vm_mark_cde_mapped(struct vm_gk20a *vm, struct dma_buf *dmabuf,
+                u64 map_vaddr);
 dma_addr_t gk20a_mm_gpuva_to_iova_base(struct vm_gk20a *vm, u64 gpu_vaddr);
 
 int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev);