4 files changed, 126 insertions, 19 deletions

diff --git a/drivers/gpu/nvgpu/common/linux/cde.c b/drivers/gpu/nvgpu/common/linux/cde.c
index 5b0fb910..0c52271a 100644
--- a/drivers/gpu/nvgpu/common/linux/cde.c
+++ b/drivers/gpu/nvgpu/common/linux/cde.c
@@ -975,7 +975,7 @@ __releases(&l->cde_app->mutex)
         u64 big_page_mask = 0;
         u32 flags;
         int err, i;
-        const s32 compbits_kind = 0;
+        const s16 compbits_kind = 0;
 
         gk20a_dbg(gpu_dbg_cde, "compbits_byte_offset=%llu scatterbuffer_byte_offset=%llu",
                   compbits_byte_offset, scatterbuffer_byte_offset);
@@ -1038,8 +1038,11 @@ __releases(&l->cde_app->mutex)
         /* map the destination buffer */
         get_dma_buf(compbits_scatter_buf); /* a ref for nvgpu_vm_map */
         map_vaddr = nvgpu_vm_map(cde_ctx->vm, compbits_scatter_buf, 0,
-                                 NVGPU_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
-                                 compbits_kind, true,
+                                 NVGPU_AS_MAP_BUFFER_FLAGS_CACHEABLE |
+                                 NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL,
+                                 NV_KIND_INVALID,
+                                 compbits_kind, /* incompressible kind */
+                                 true,
                                  gk20a_mem_flag_none,
                                  map_offset, map_size,
                                  NULL);
diff --git a/drivers/gpu/nvgpu/common/linux/ioctl_as.c b/drivers/gpu/nvgpu/common/linux/ioctl_as.c
index d4242955..cfc4e7ef 100644
--- a/drivers/gpu/nvgpu/common/linux/ioctl_as.c
+++ b/drivers/gpu/nvgpu/common/linux/ioctl_as.c
@@ -79,11 +79,22 @@ static int gk20a_as_ioctl_map_buffer_ex(
                 struct gk20a_as_share *as_share,
                 struct nvgpu_as_map_buffer_ex_args *args)
 {
+        s16 compressible_kind;
+        s16 incompressible_kind;
+
         gk20a_dbg_fn("");
 
+        if (args->flags & NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL) {
+                compressible_kind = args->compr_kind;
+                incompressible_kind = args->incompr_kind;
+        } else {
+                compressible_kind = args->kind;
+                incompressible_kind = NV_KIND_INVALID;
+        }
+
         return nvgpu_vm_map_buffer(as_share->vm, args->dmabuf_fd,
                                    &args->offset, args->flags,
-                                   args->kind,
+                                   compressible_kind, incompressible_kind,
                                    args->buffer_offset,
                                    args->mapping_size,
                                    NULL);
@@ -97,6 +108,7 @@ static int gk20a_as_ioctl_map_buffer(
         return nvgpu_vm_map_buffer(as_share->vm, args->dmabuf_fd,
                                    &args->o_a.offset,
                                    args->flags, NV_KIND_DEFAULT,
+                                   NV_KIND_DEFAULT,
                                    0, 0, NULL);
         /* args->o_a.offset will be set if !err */
 }
@@ -158,6 +170,9 @@ static int gk20a_as_ioctl_map_buffer_batch(
         }
 
         for (i = 0; i < args->num_maps; ++i) {
+                s16 compressible_kind;
+                s16 incompressible_kind;
+
                 struct nvgpu_as_map_buffer_ex_args map_args;
                 memset(&map_args, 0, sizeof(map_args));
 
@@ -167,10 +182,19 @@ static int gk20a_as_ioctl_map_buffer_batch(
                         break;
                 }
 
+                if (map_args.flags &
+                    NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL) {
+                        compressible_kind = map_args.compr_kind;
+                        incompressible_kind = map_args.incompr_kind;
+                } else {
+                        compressible_kind = map_args.kind;
+                        incompressible_kind = NV_KIND_INVALID;
+                }
+
                 err = nvgpu_vm_map_buffer(
                         as_share->vm, map_args.dmabuf_fd,
                         &map_args.offset, map_args.flags,
-                        map_args.kind,
+                        compressible_kind, incompressible_kind,
                         map_args.buffer_offset,
                         map_args.mapping_size,
                         &batch);
diff --git a/drivers/gpu/nvgpu/common/linux/vm.c b/drivers/gpu/nvgpu/common/linux/vm.c
index 58e2da13..86d8bec9 100644
--- a/drivers/gpu/nvgpu/common/linux/vm.c
+++ b/drivers/gpu/nvgpu/common/linux/vm.c
@@ -177,11 +177,46 @@ static u64 __nvgpu_vm_find_mapping(struct vm_gk20a *vm,
         return mapped_buffer->addr;
 }
 
+static int setup_bfr_kind_fields(struct buffer_attrs *bfr, s16 compr_kind,
+                                 s16 incompr_kind, u32 flags)
+{
+        if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL) {
+                /* were we supplied with a kind in either parameter? */
+                if ((compr_kind < 0 || compr_kind >= NV_KIND_ATTR_SIZE) &&
+                    (incompr_kind < 0 || incompr_kind >= NV_KIND_ATTR_SIZE))
+                        return -EINVAL;
+
+                if (compr_kind != NV_KIND_INVALID) {
+                        bfr->use_kind_v = true;
+                        bfr->kind_v = (u8)compr_kind;
+                }
+
+                if (incompr_kind != NV_KIND_INVALID) {
+                        bfr->use_uc_kind_v = true;
+                        bfr->uc_kind_v = (u8)incompr_kind;
+                }
+        } else {
+                if (compr_kind < 0 || compr_kind >= NV_KIND_ATTR_SIZE)
+                        return -EINVAL;
+
+                bfr->use_kind_v = true;
+                bfr->kind_v = (u8)compr_kind;
+
+                /*
+                 * Note: setup_buffer_kind_and_compression() will
+                 * figure out uc_kind_v or return an error
+                 */
+        }
+
+        return 0;
+}
+
 u64 nvgpu_vm_map(struct vm_gk20a *vm,
                  struct dma_buf *dmabuf,
                  u64 offset_align,
                  u32 flags,
-                 int kind,
+                 s16 compr_kind,
+                 s16 incompr_kind,
                  bool user_mapped,
                  int rw_flag,
                  u64 buffer_offset,
@@ -203,6 +238,22 @@ u64 nvgpu_vm_map(struct vm_gk20a *vm,
         u32 ctag_offset;
         enum nvgpu_aperture aperture;
 
+        /*
+         * The kind used as part of the key for map caching. HW may
+         * actually be programmed with the fallback kind in case the
+         * key kind is compressible but we're out of comptags.
+         */
+        s16 map_key_kind;
+
+        if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL) {
+                if (compr_kind != NV_KIND_INVALID)
+                        map_key_kind = compr_kind;
+                else
+                        map_key_kind = incompr_kind;
+        } else {
+                map_key_kind = compr_kind;
+        }
+
         if (user_mapped && vm->userspace_managed &&
             !(flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET)) {
                 nvgpu_err(g, "non-fixed-offset mapping not available on "
@@ -216,7 +267,7 @@ u64 nvgpu_vm_map(struct vm_gk20a *vm,
         if (!vm->userspace_managed) {
                 map_offset = __nvgpu_vm_find_mapping(
                         vm, dmabuf, offset_align,
-                        flags, kind,
+                        flags, map_key_kind,
                         user_mapped, rw_flag);
                 if (map_offset) {
                         nvgpu_mutex_release(&vm->update_gmmu_lock);
@@ -239,12 +290,10 @@ u64 nvgpu_vm_map(struct vm_gk20a *vm,
                 goto clean_up;
         }
 
-        if (kind >= NV_KIND_ATTR_SIZE) {
-                err = -EINVAL;
+        err = setup_bfr_kind_fields(&bfr, compr_kind, incompr_kind, flags);
+        if (err)
                 goto clean_up;
-        } else {
-                bfr.kind_v = (u8)kind;
-        }
+
         bfr.size = dmabuf->size;
         sgl = bfr.sgt->sgl;
 
@@ -306,10 +355,15 @@ u64 nvgpu_vm_map(struct vm_gk20a *vm,
                 err = gk20a_alloc_comptags(g, dev, dmabuf,
                                            ctag_allocator,
                                            bfr.ctag_lines);
-                if (err) {
-                        /* ok to fall back here if we ran out */
+                if (unlikely(err)) {
                         /* TBD: we can partially alloc ctags as well... */
-                        bfr.kind_v = bfr.uc_kind_v;
+                        if (bfr.use_uc_kind_v) {
+                                /* no comptags, but fallback kind available */
+                                bfr.kind_v = bfr.uc_kind_v;
+                        } else {
+                                nvgpu_err(g, "comptag alloc failed and no fallback kind specified");
+                                goto clean_up;
+                        }
                 } else {
                         gk20a_get_comptags(dev,
                                            dmabuf, &comptags);
@@ -371,7 +425,7 @@ u64 nvgpu_vm_map(struct vm_gk20a *vm,
         mapped_buffer->ctag_allocated_lines = bfr.ctag_allocated_lines;
         mapped_buffer->vm          = vm;
         mapped_buffer->flags       = flags;
-        mapped_buffer->kind        = kind;
+        mapped_buffer->kind        = map_key_kind;
         mapped_buffer->va_allocated = va_allocated;
         mapped_buffer->user_mapped = user_mapped ? 1 : 0;
         mapped_buffer->own_mem_ref = user_mapped;
diff --git a/drivers/gpu/nvgpu/common/linux/vm_priv.h b/drivers/gpu/nvgpu/common/linux/vm_priv.h
index 4f6b10bb..1eadf1d0 100644
--- a/drivers/gpu/nvgpu/common/linux/vm_priv.h
+++ b/drivers/gpu/nvgpu/common/linux/vm_priv.h
@@ -34,7 +34,9 @@ struct buffer_attrs {
         u32 ctag_allocated_lines;
         int pgsz_idx;
         u8 kind_v;
+        bool use_kind_v;
         u8 uc_kind_v;
+        bool use_uc_kind_v;
         bool ctag_user_mappable;
 };
 
@@ -42,19 +44,43 @@ u64 nvgpu_vm_map(struct vm_gk20a *vm,
                  struct dma_buf *dmabuf,
                  u64 offset_align,
                  u32 flags,
-                 int kind,
+
+                 /*
+                  * compressible kind if
+                  * NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL is
+                  * specified, otherwise just the kind
+                  */
+                 s16 compr_kind,
+
+                 /*
+                  * incompressible kind if
+                  * NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL is
+                  * specified, otherwise ignored
+                  */
+                 s16 incompr_kind,
+
                  bool user_mapped,
                  int rw_flag,
                  u64 buffer_offset,
                  u64 mapping_size,
                  struct vm_gk20a_mapping_batch *mapping_batch);
 
-/* Note: batch may be NULL if map op is not part of a batch */
+/*
+ * Notes:
+ * - Batch may be NULL if map op is not part of a batch.
+ * - If NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL is set,
+ *   compr_kind and incompr_kind work as explained in nvgpu.h.
+ * - If NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL is NOT set,
+ *   compr_kind holds the kind and kernel will figure out whether
+ *   it is a compressible or incompressible kind. If compressible, kernel will
+ *   also figure out the incompressible counterpart or return an error.
+ */
 int nvgpu_vm_map_buffer(struct vm_gk20a *vm,
                         int dmabuf_fd,
                         u64 *offset_align,
                         u32 flags, /* NVGPU_AS_MAP_BUFFER_FLAGS_ */
-                        int kind,
+                        s16 compr_kind,
+                        s16 incompr_kind,
                         u64 buffer_offset,
                         u64 mapping_size,
                         struct vm_gk20a_mapping_batch *batch);