7 files changed, 337 insertions, 5 deletions

diff --git a/drivers/gpu/nvgpu/common/linux/cde.c b/drivers/gpu/nvgpu/common/linux/cde.c
index 2832408d..c3a9b770 100644
--- a/drivers/gpu/nvgpu/common/linux/cde.c
+++ b/drivers/gpu/nvgpu/common/linux/cde.c
@@ -1068,8 +1068,8 @@ __releases(&l->cde_app->mutex)
         }
 
         if (scatterbuffer_byte_offset &&
-            g->ops.cde.need_scatter_buffer &&
-            g->ops.cde.need_scatter_buffer(g)) {
+            l->ops.cde.need_scatter_buffer &&
+            l->ops.cde.need_scatter_buffer(g)) {
                 struct sg_table *sgt;
                 void *scatter_buffer;
 
@@ -1092,7 +1092,7 @@ __releases(&l->cde_app->mutex)
                         err = -EINVAL;
                         goto exit_unmap_surface;
                 } else {
-                        err = g->ops.cde.populate_scatter_buffer(g, sgt,
+                        err = l->ops.cde.populate_scatter_buffer(g, sgt,
                                         compbits_byte_offset, scatter_buffer,
                                         scatterbuffer_size);
                         WARN_ON(err);
@@ -1463,8 +1463,8 @@ static int gk20a_buffer_convert_gpu_to_cde_v1(
         int hprog = -1;
         int vprog = -1;
 
-        if (g->ops.cde.get_program_numbers)
-                g->ops.cde.get_program_numbers(g, block_height_log2,
+        if (l->ops.cde.get_program_numbers)
+                l->ops.cde.get_program_numbers(g, block_height_log2,
                                                l->cde_app.shader_parameter,
                                                &hprog, &vprog);
         else {
diff --git a/drivers/gpu/nvgpu/common/linux/cde_gm20b.c b/drivers/gpu/nvgpu/common/linux/cde_gm20b.c
new file mode 100644
index 00000000..1cd15c54
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/cde_gm20b.c
@@ -0,0 +1,64 @@
+/*
+ * GM20B CDE
+ *
+ * Copyright (c) 2015-2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "gk20a/gk20a.h"
+#include "cde_gm20b.h"
+
+enum programs {
+        PROG_HPASS              = 0,
+        PROG_VPASS_LARGE        = 1,
+        PROG_VPASS_SMALL        = 2,
+        PROG_HPASS_DEBUG        = 3,
+        PROG_VPASS_LARGE_DEBUG  = 4,
+        PROG_VPASS_SMALL_DEBUG  = 5,
+        PROG_PASSTHROUGH        = 6,
+};
+
+static void gm20b_cde_get_program_numbers(struct gk20a *g,
+                                          u32 block_height_log2,
+                                          u32 shader_parameter,
+                                          int *hprog_out, int *vprog_out)
+{
+        int hprog = PROG_HPASS;
+        int vprog = (block_height_log2 >= 2) ?
+                PROG_VPASS_LARGE : PROG_VPASS_SMALL;
+        if (shader_parameter == 1) {
+                hprog = PROG_PASSTHROUGH;
+                vprog = PROG_PASSTHROUGH;
+        } else if (shader_parameter == 2) {
+                hprog = PROG_HPASS_DEBUG;
+                vprog = (block_height_log2 >= 2) ?
+                        PROG_VPASS_LARGE_DEBUG :
+                        PROG_VPASS_SMALL_DEBUG;
+        }
+
+        *hprog_out = hprog;
+        *vprog_out = vprog;
+}
+
+struct nvgpu_os_linux_ops gm20b_cde_ops = {
+        .cde = {
+                .get_program_numbers = gm20b_cde_get_program_numbers,
+        },
+};
diff --git a/drivers/gpu/nvgpu/common/linux/cde_gm20b.h b/drivers/gpu/nvgpu/common/linux/cde_gm20b.h
new file mode 100644
index 00000000..640d6ab6
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/cde_gm20b.h
@@ -0,0 +1,32 @@
+/*
+ * GM20B CDE
+ *
+ * Copyright (c) 2015-2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _NVHOST_GM20B_CDE
+#define _NVHOST_GM20B_CDE
+
+#include "os_linux.h"
+
+extern struct nvgpu_os_linux_ops gm20b_cde_ops;
+
+#endif
diff --git a/drivers/gpu/nvgpu/common/linux/cde_gp10b.c b/drivers/gpu/nvgpu/common/linux/cde_gp10b.c
new file mode 100644
index 00000000..ffae6e34
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/cde_gp10b.c
@@ -0,0 +1,160 @@
+/*
+ * GP10B CDE
+ *
+ * Copyright (c) 2015-2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "gk20a/gk20a.h"
+#include "cde_gp10b.h"
+
+#include <nvgpu/log.h>
+
+enum gp10b_programs {
+        GP10B_PROG_HPASS              = 0,
+        GP10B_PROG_HPASS_4K           = 1,
+        GP10B_PROG_VPASS              = 2,
+        GP10B_PROG_VPASS_4K           = 3,
+        GP10B_PROG_HPASS_DEBUG        = 4,
+        GP10B_PROG_HPASS_4K_DEBUG     = 5,
+        GP10B_PROG_VPASS_DEBUG        = 6,
+        GP10B_PROG_VPASS_4K_DEBUG     = 7,
+        GP10B_PROG_PASSTHROUGH        = 8,
+};
+
+void gp10b_cde_get_program_numbers(struct gk20a *g,
+                                          u32 block_height_log2,
+                                          u32 shader_parameter,
+                                          int *hprog_out, int *vprog_out)
+{
+        int hprog, vprog;
+
+        if (shader_parameter == 1) {
+                hprog = GP10B_PROG_PASSTHROUGH;
+                vprog = GP10B_PROG_PASSTHROUGH;
+        } else {
+                hprog = GP10B_PROG_HPASS;
+                vprog = GP10B_PROG_VPASS;
+                if (shader_parameter == 2) {
+                        hprog = GP10B_PROG_HPASS_DEBUG;
+                        vprog = GP10B_PROG_VPASS_DEBUG;
+                }
+                if (g->mm.bypass_smmu) {
+                        if (!g->mm.disable_bigpage) {
+                                nvgpu_warn(g,
+                                           "when bypass_smmu is 1, disable_bigpage must be 1 too");
+                        }
+                        hprog |= 1;
+                        vprog |= 1;
+                }
+        }
+
+        *hprog_out = hprog;
+        *vprog_out = vprog;
+}
+
+bool gp10b_need_scatter_buffer(struct gk20a *g)
+{
+        return g->mm.bypass_smmu;
+}
+
+static u8 parity(u32 a)
+{
+        a ^= a>>16u;
+        a ^= a>>8u;
+        a ^= a>>4u;
+        a &= 0xfu;
+        return (0x6996u >> a) & 1u;
+}
+
+int gp10b_populate_scatter_buffer(struct gk20a *g,
+                                         struct sg_table *sgt,
+                                         size_t surface_size,
+                                         void *scatter_buffer_ptr,
+                                         size_t scatter_buffer_size)
+{
+        /* map scatter buffer to CPU VA and fill it */
+        const u32 page_size_log2 = 12;
+        const u32 page_size = 1 << page_size_log2;
+        const u32 page_size_shift = page_size_log2 - 7u;
+
+        /* 0011 1111 1111 1111 1111 1110 0100 1000 */
+        const u32 getSliceMaskGP10B = 0x3ffffe48;
+        u8 *scatter_buffer = scatter_buffer_ptr;
+
+        size_t i;
+        struct scatterlist *sg = NULL;
+        u8 d = 0;
+        size_t page = 0;
+        size_t pages_left;
+
+        surface_size = round_up(surface_size, page_size);
+
+        pages_left = surface_size >> page_size_log2;
+        if ((pages_left >> 3) > scatter_buffer_size)
+            return -ENOMEM;
+
+        for_each_sg(sgt->sgl, sg, sgt->nents, i) {
+                unsigned int j;
+                u64 surf_pa = sg_phys(sg);
+                unsigned int n = (int)(sg->length >> page_size_log2);
+
+                gk20a_dbg(gpu_dbg_cde, "surfPA=0x%llx + %d pages", surf_pa, n);
+
+                for (j=0; j < n && pages_left > 0; j++, surf_pa += page_size) {
+                        u32 addr = (((u32)(surf_pa>>7)) & getSliceMaskGP10B) >> page_size_shift;
+                        u8 scatter_bit = parity(addr);
+                        u8 bit = page & 7;
+
+                        d |= scatter_bit << bit;
+                        if (bit == 7) {
+                                scatter_buffer[page >> 3] = d;
+                                d = 0;
+                        }
+
+                        ++page;
+                        --pages_left;
+                }
+
+                if (pages_left == 0)
+                        break;
+        }
+
+        /* write the last byte in case the number of pages is not divisible by 8 */
+        if ((page & 7) != 0)
+                scatter_buffer[page >> 3] = d;
+
+        if (nvgpu_log_mask_enabled(g, gpu_dbg_cde)) {
+                gk20a_dbg(gpu_dbg_cde, "scatterBuffer content:");
+                for (i = 0; i < page >> 3; i++) {
+                        gk20a_dbg(gpu_dbg_cde, " %x", scatter_buffer[i]);
+                }
+        }
+
+        return 0;
+}
+
+struct nvgpu_os_linux_ops gp10b_cde_ops = {
+        .cde = {
+                .get_program_numbers = gp10b_cde_get_program_numbers,
+                .need_scatter_buffer = gp10b_need_scatter_buffer,
+                .populate_scatter_buffer = gp10b_populate_scatter_buffer,
+        },
+};
diff --git a/drivers/gpu/nvgpu/common/linux/cde_gp10b.h b/drivers/gpu/nvgpu/common/linux/cde_gp10b.h
new file mode 100644
index 00000000..52e9f292
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/cde_gp10b.h
@@ -0,0 +1,32 @@
+/*
+ * GP10B CDE
+ *
+ * Copyright (c) 2015-2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _NVHOST_GP10B_CDE
+#define _NVHOST_GP10B_CDE
+
+#include "os_linux.h"
+
+extern struct nvgpu_os_linux_ops gp10b_cde_ops;
+
+#endif
diff --git a/drivers/gpu/nvgpu/common/linux/module.c b/drivers/gpu/nvgpu/common/linux/module.c
index fe3e4e6f..b7d13f05 100644
--- a/drivers/gpu/nvgpu/common/linux/module.c
+++ b/drivers/gpu/nvgpu/common/linux/module.c
@@ -49,6 +49,8 @@
 #endif
 #endif
 #include "os_linux.h"
+#include "cde_gm20b.h"
+#include "cde_gp10b.h"
 
 #define CLASS_NAME "nvidia-gpu"
 /* TODO: Change to e.g. "nvidia-gpu%s" once we have symlinks in place. */
@@ -154,6 +156,26 @@ static int gk20a_restore_registers(struct gk20a *g)
         return 0;
 }
 
+static int nvgpu_init_os_linux_ops(struct nvgpu_os_linux *l) {
+        struct gk20a *g = &l->g;
+        u32 ver = g->gpu_characteristics.arch + g->gpu_characteristics.impl;
+
+        switch (ver) {
+        case GK20A_GPUID_GM20B:
+        case GK20A_GPUID_GM20B_B:
+                l->ops.cde = gm20b_cde_ops.cde;
+                break;
+        case NVGPU_GPUID_GP10B:
+                l->ops.cde = gp10b_cde_ops.cde;
+                break;
+        default:
+                /* CDE is optional, so today ignoring unknown chip is fine */
+                break;
+        }
+
+        return 0;
+}
+
 int gk20a_pm_finalize_poweron(struct device *dev)
 {
         struct gk20a *g = get_gk20a(dev);
@@ -198,6 +220,10 @@ int gk20a_pm_finalize_poweron(struct device *dev)
 
         trace_gk20a_finalize_poweron_done(dev_name(dev));
 
+        err = nvgpu_init_os_linux_ops(l);
+        if (err)
+                goto done;
+
         enable_irq(g->irq_stall);
         if (g->irq_stall != g->irq_nonstall)
                 enable_irq(g->irq_nonstall);
diff --git a/drivers/gpu/nvgpu/common/linux/os_linux.h b/drivers/gpu/nvgpu/common/linux/os_linux.h
index c67cbbcc..27433e32 100644
--- a/drivers/gpu/nvgpu/common/linux/os_linux.h
+++ b/drivers/gpu/nvgpu/common/linux/os_linux.h
@@ -24,6 +24,21 @@
 #include "gk20a/gk20a.h"
 #include "cde.h"
 
+struct nvgpu_os_linux_ops {
+        struct {
+                void (*get_program_numbers)(struct gk20a *g,
+                                            u32 block_height_log2,
+                                            u32 shader_parameter,
+                                            int *hprog, int *vprog);
+                bool (*need_scatter_buffer)(struct gk20a *g);
+                int (*populate_scatter_buffer)(struct gk20a *g,
+                                               struct sg_table *sgt,
+                                               size_t surface_size,
+                                               void *scatter_buffer_ptr,
+                                               size_t scatter_buffer_size);
+        } cde;
+};
+
 struct nvgpu_os_linux {
         struct gk20a g;
         struct device *dev;
@@ -99,6 +114,9 @@ struct nvgpu_os_linux {
 #ifdef CONFIG_TEGRA_19x_GPU
         struct nvgpu_os_linux_t19x t19x;
 #endif
+
+        struct nvgpu_os_linux_ops ops;
+
 #ifdef CONFIG_DEBUG_FS
         struct dentry *debugfs;
         struct dentry *debugfs_alias;