gpu: nvgpu: Per-chip interrupt processing

Move accesses to MC registers under HAL so that they can be
reimplemented per chip.

Do chip detection and HAL initialization only once.

Bug 1567274

Change-Id: I20bf2f439d267d284bfd536f1a1dfb5d5a2dce4c
Signed-off-by: Terje Bergstrom <tbergstrom@nvidia.com>
Reviewed-on: http://git-master/r/590385

9 files changed, 244 insertions, 103 deletions

diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile
index cf5d65cc..391ccec7 100644
--- a/drivers/gpu/nvgpu/Makefile
+++ b/drivers/gpu/nvgpu/Makefile
@@ -38,6 +38,7 @@ nvgpu-y := \
         gk20a/cde_gk20a.o \
         gk20a/platform_gk20a_generic.o \
         gk20a/tsg_gk20a.o \
+        gk20a/mc_gk20a.o \
         gm20b/hal_gm20b.o \
         gm20b/ltc_gm20b.o \
         gm20b/gr_gm20b.o \
@@ -49,7 +50,8 @@ nvgpu-y := \
         gm20b/acr_gm20b.o \
         gm20b/pmu_gm20b.o \
         gm20b/mm_gm20b.o \
-        gm20b/regops_gm20b.o
+        gm20b/regops_gm20b.o \
+        gm20b/mc_gm20b.o
 
 nvgpu-$(CONFIG_TEGRA_GK20A) += gk20a/platform_gk20a_tegra.o
 nvgpu-$(CONFIG_SYNC) += gk20a/sync_gk20a.o
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c
index cea53e00..2e1bd003 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.c
@@ -524,48 +524,18 @@ int gk20a_sim_esc_read(struct gk20a *g, char *path, u32 index, u32 count, u32 *d
 static irqreturn_t gk20a_intr_isr_stall(int irq, void *dev_id)
 {
         struct gk20a *g = dev_id;
-        u32 mc_intr_0;
 
-        if (!g->power_on)
-                return IRQ_NONE;
-
-        /* not from gpu when sharing irq with others */
-        mc_intr_0 = gk20a_readl(g, mc_intr_0_r());
-        if (unlikely(!mc_intr_0))
-                return IRQ_NONE;
-
-        gk20a_writel(g, mc_intr_en_0_r(),
-                mc_intr_en_0_inta_disabled_f());
-
-        /* flush previous write */
-        gk20a_readl(g, mc_intr_en_0_r());
-
-        return IRQ_WAKE_THREAD;
+        return g->ops.mc.isr_stall(g);
 }
 
 static irqreturn_t gk20a_intr_isr_nonstall(int irq, void *dev_id)
 {
         struct gk20a *g = dev_id;
-        u32 mc_intr_1;
-
-        if (!g->power_on)
-                return IRQ_NONE;
-
-        /* not from gpu when sharing irq with others */
-        mc_intr_1 = gk20a_readl(g, mc_intr_1_r());
-        if (unlikely(!mc_intr_1))
-                return IRQ_NONE;
-
-        gk20a_writel(g, mc_intr_en_1_r(),
-                mc_intr_en_1_inta_disabled_f());
-
-        /* flush previous write */
-        gk20a_readl(g, mc_intr_en_1_r());
 
-        return IRQ_WAKE_THREAD;
+        return g->ops.mc.isr_nonstall(g);
 }
 
-static void gk20a_pbus_isr(struct gk20a *g)
+void gk20a_pbus_isr(struct gk20a *g)
 {
         u32 val;
         val = gk20a_readl(g, bus_intr_0_r());
@@ -595,59 +565,13 @@ static void gk20a_pbus_isr(struct gk20a *g)
 static irqreturn_t gk20a_intr_thread_stall(int irq, void *dev_id)
 {
         struct gk20a *g = dev_id;
-        u32 mc_intr_0;
-
-        gk20a_dbg(gpu_dbg_intr, "interrupt thread launched");
-
-        mc_intr_0 = gk20a_readl(g, mc_intr_0_r());
-
-        gk20a_dbg(gpu_dbg_intr, "stall intr %08x\n", mc_intr_0);
-
-        if (mc_intr_0 & mc_intr_0_pgraph_pending_f())
-                gr_gk20a_elpg_protected_call(g, gk20a_gr_isr(g));
-        if (mc_intr_0 & mc_intr_0_pfifo_pending_f())
-                gk20a_fifo_isr(g);
-        if (mc_intr_0 & mc_intr_0_pmu_pending_f())
-                gk20a_pmu_isr(g);
-        if (mc_intr_0 & mc_intr_0_priv_ring_pending_f())
-                gk20a_priv_ring_isr(g);
-        if (mc_intr_0 & mc_intr_0_ltc_pending_f())
-                g->ops.ltc.isr(g);
-        if (mc_intr_0 & mc_intr_0_pbus_pending_f())
-                gk20a_pbus_isr(g);
-
-        gk20a_writel(g, mc_intr_en_0_r(),
-                mc_intr_en_0_inta_hardware_f());
-
-        /* flush previous write */
-        gk20a_readl(g, mc_intr_en_0_r());
-
-        return IRQ_HANDLED;
+        return g->ops.mc.isr_thread_stall(g);
 }
 
 static irqreturn_t gk20a_intr_thread_nonstall(int irq, void *dev_id)
 {
         struct gk20a *g = dev_id;
-        u32 mc_intr_1;
-
-        gk20a_dbg(gpu_dbg_intr, "interrupt thread launched");
-
-        mc_intr_1 = gk20a_readl(g, mc_intr_1_r());
-
-        gk20a_dbg(gpu_dbg_intr, "non-stall intr %08x\n", mc_intr_1);
-
-        if (mc_intr_1 & mc_intr_0_pfifo_pending_f())
-                gk20a_fifo_nonstall_isr(g);
-        if (mc_intr_1 & mc_intr_0_pgraph_pending_f())
-                gk20a_gr_nonstall_isr(g);
-
-        gk20a_writel(g, mc_intr_en_1_r(),
-                mc_intr_en_1_inta_hardware_f());
-
-        /* flush previous write */
-        gk20a_readl(g, mc_intr_en_1_r());
-
-        return IRQ_HANDLED;
+        return g->ops.mc.isr_thread_nonstall(g);
 }
 
 static void gk20a_remove_support(struct platform_device *dev)
@@ -776,11 +700,15 @@ static int gk20a_pm_prepare_poweroff(struct device *dev)
         return ret;
 }
 
-static void gk20a_detect_chip(struct gk20a *g)
+static int gk20a_detect_chip(struct gk20a *g)
 {
         struct nvgpu_gpu_characteristics *gpu = &g->gpu_characteristics;
+        u32 mc_boot_0_value;
 
-        u32 mc_boot_0_value = gk20a_readl(g, mc_boot_0_r());
+        if (gpu->arch)
+                return 0;
+
+        mc_boot_0_value = gk20a_readl(g, mc_boot_0_r());
         gpu->arch = mc_boot_0_architecture_v(mc_boot_0_value) <<
                 NVGPU_GPU_ARCHITECTURE_SHIFT;
         gpu->impl = mc_boot_0_implementation_v(mc_boot_0_value);
@@ -792,6 +720,8 @@ static void gk20a_detect_chip(struct gk20a *g)
                         g->gpu_characteristics.arch,
                         g->gpu_characteristics.impl,
                         g->gpu_characteristics.rev);
+
+        return gpu_init_hal(g);
 }
 
 static int gk20a_pm_finalize_poweron(struct device *dev)
@@ -815,23 +745,16 @@ static int gk20a_pm_finalize_poweron(struct device *dev)
         nice_value = task_nice(current);
         set_user_nice(current, -20);
 
-        enable_irq(g->irq_stall);
-        enable_irq(g->irq_nonstall);
-
         g->power_on = true;
 
-        gk20a_writel(g, mc_intr_en_1_r(),
-                mc_intr_en_1_inta_hardware_f());
+        err = gk20a_detect_chip(g);
+        if (err)
+                goto done;
 
-        gk20a_writel(g, mc_intr_en_0_r(),
-                mc_intr_en_0_inta_hardware_f());
+        enable_irq(g->irq_stall);
+        enable_irq(g->irq_nonstall);
 
-        if (g->ops.clock_gating.slcg_bus_load_gating_prod)
-                g->ops.clock_gating.slcg_bus_load_gating_prod(g,
-                                g->slcg_enabled);
-        if (g->ops.clock_gating.blcg_bus_load_gating_prod)
-                g->ops.clock_gating.blcg_bus_load_gating_prod(g,
-                                g->blcg_enabled);
+        g->ops.mc.intr_enable(g);
 
         if (!tegra_platform_is_silicon())
                 gk20a_writel(g, bus_intr_en_0_r(), 0x0);
@@ -843,11 +766,6 @@ static int gk20a_pm_finalize_poweron(struct device *dev)
 
         gk20a_reset_priv_ring(g);
 
-        gk20a_detect_chip(g);
-        err = gpu_init_hal(g);
-        if (err)
-                goto done;
-
         /* TBD: move this after graphics init in which blcg/slcg is enabled.
            This function removes SlowdownOnBoot which applies 32x divider
            on gpcpll bypass path. The purpose of slowdown is to save power
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index 5669e1c5..a111a040 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -29,7 +29,8 @@ struct acr_gm20b;
 
 #include <linux/sched.h>
 #include <linux/spinlock.h>
-#include <uapi/linux/nvgpu.h>
+#include <linux/nvgpu.h>
+#include <linux/irqreturn.h>
 #include <linux/tegra-soc.h>
 
 #include "../../../arch/arm/mach-tegra/iomap.h"
@@ -335,6 +336,13 @@ struct gpu_ops {
                                 *get_qctl_whitelist_ranges)(void);
                 int (*get_qctl_whitelist_ranges_count)(void);
         } regops;
+        struct {
+                void (*intr_enable)(struct gk20a *g);
+                irqreturn_t (*isr_stall)(struct gk20a *g);
+                irqreturn_t (*isr_nonstall)(struct gk20a *g);
+                irqreturn_t (*isr_thread_stall)(struct gk20a *g);
+                irqreturn_t (*isr_thread_nonstall)(struct gk20a *g);
+        } mc;
 };
 
 struct gk20a {
@@ -734,6 +742,8 @@ gk20a_request_firmware(struct gk20a *g, const char *fw_name);
 
 int gk20a_init_gpu_characteristics(struct gk20a *g);
 
+void gk20a_pbus_isr(struct gk20a *g);
+
 int gk20a_user_init(struct platform_device *dev);
 void gk20a_user_deinit(struct platform_device *dev);
 
diff --git a/drivers/gpu/nvgpu/gk20a/hal_gk20a.c b/drivers/gpu/nvgpu/gk20a/hal_gk20a.c
index 578b77bf..208c1ef0 100644
--- a/drivers/gpu/nvgpu/gk20a/hal_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/hal_gk20a.c
@@ -23,6 +23,7 @@
 #include "channel_gk20a.h"
 #include "gr_ctx_gk20a.h"
 #include "mm_gk20a.h"
+#include "mc_gk20a.h"
 #include "pmu_gk20a.h"
 #include "clk_gk20a.h"
 #include "regops_gk20a.h"
@@ -48,6 +49,7 @@ int gk20a_init_hal(struct gpu_ops *gops)
 {
         *gops = gk20a_ops;
         gops->privsecurity = 0;
+        gk20a_init_mc(gops);
         gk20a_init_ltc(gops);
         gk20a_init_gr_ops(gops);
         gk20a_init_fb(gops);
diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c
new file mode 100644
index 00000000..53701605
--- /dev/null
+++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c
@@ -0,0 +1,136 @@
+/*
+ * GK20A memory interface
+ *
+ * Copyright (c) 2014, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/types.h>
+
+#include "gk20a.h"
+#include "mc_gk20a.h"
+#include "hw_mc_gk20a.h"
+
+irqreturn_t mc_gk20a_isr_stall(struct gk20a *g)
+{
+        u32 mc_intr_0;
+
+        if (!g->power_on)
+                return IRQ_NONE;
+
+        /* not from gpu when sharing irq with others */
+        mc_intr_0 = gk20a_readl(g, mc_intr_0_r());
+        if (unlikely(!mc_intr_0))
+                return IRQ_NONE;
+
+        gk20a_writel(g, mc_intr_en_0_r(),
+                mc_intr_en_0_inta_disabled_f());
+
+        /* flush previous write */
+        gk20a_readl(g, mc_intr_en_0_r());
+
+        return IRQ_WAKE_THREAD;
+}
+
+irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g)
+{
+        u32 mc_intr_1;
+
+        if (!g->power_on)
+                return IRQ_NONE;
+
+        /* not from gpu when sharing irq with others */
+        mc_intr_1 = gk20a_readl(g, mc_intr_1_r());
+        if (unlikely(!mc_intr_1))
+                return IRQ_NONE;
+
+        gk20a_writel(g, mc_intr_en_1_r(),
+                mc_intr_en_1_inta_disabled_f());
+
+        /* flush previous write */
+        gk20a_readl(g, mc_intr_en_1_r());
+
+        return IRQ_WAKE_THREAD;
+}
+
+irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g)
+{
+        u32 mc_intr_0;
+
+        gk20a_dbg(gpu_dbg_intr, "interrupt thread launched");
+
+        mc_intr_0 = gk20a_readl(g, mc_intr_0_r());
+
+        gk20a_dbg(gpu_dbg_intr, "stall intr %08x\n", mc_intr_0);
+
+        if (mc_intr_0 & mc_intr_0_pgraph_pending_f())
+                gr_gk20a_elpg_protected_call(g, gk20a_gr_isr(g));
+        if (mc_intr_0 & mc_intr_0_pfifo_pending_f())
+                gk20a_fifo_isr(g);
+        if (mc_intr_0 & mc_intr_0_pmu_pending_f())
+                gk20a_pmu_isr(g);
+        if (mc_intr_0 & mc_intr_0_priv_ring_pending_f())
+                gk20a_priv_ring_isr(g);
+        if (mc_intr_0 & mc_intr_0_ltc_pending_f())
+                g->ops.ltc.isr(g);
+        if (mc_intr_0 & mc_intr_0_pbus_pending_f())
+                gk20a_pbus_isr(g);
+
+        gk20a_writel(g, mc_intr_en_0_r(),
+                mc_intr_en_0_inta_hardware_f());
+
+        /* flush previous write */
+        gk20a_readl(g, mc_intr_en_0_r());
+
+        return IRQ_HANDLED;
+}
+
+irqreturn_t mc_gk20a_intr_thread_nonstall(struct gk20a *g)
+{
+        u32 mc_intr_1;
+
+        gk20a_dbg(gpu_dbg_intr, "interrupt thread launched");
+
+        mc_intr_1 = gk20a_readl(g, mc_intr_1_r());
+
+        gk20a_dbg(gpu_dbg_intr, "non-stall intr %08x\n", mc_intr_1);
+
+        if (mc_intr_1 & mc_intr_0_pfifo_pending_f())
+                gk20a_fifo_nonstall_isr(g);
+        if (mc_intr_1 & mc_intr_0_pgraph_pending_f())
+                gk20a_gr_nonstall_isr(g);
+
+        gk20a_writel(g, mc_intr_en_1_r(),
+                mc_intr_en_1_inta_hardware_f());
+
+        /* flush previous write */
+        gk20a_readl(g, mc_intr_en_1_r());
+
+        return IRQ_HANDLED;
+}
+
+void mc_gk20a_intr_enable(struct gk20a *g)
+{
+        gk20a_writel(g, mc_intr_en_1_r(),
+                mc_intr_en_1_inta_hardware_f());
+
+        gk20a_writel(g, mc_intr_en_0_r(),
+                mc_intr_en_0_inta_hardware_f());
+}
+
+void gk20a_init_mc(struct gpu_ops *gops)
+{
+        gops->mc.intr_enable = mc_gk20a_intr_enable;
+        gops->mc.isr_stall = mc_gk20a_isr_stall;
+        gops->mc.isr_nonstall = mc_gk20a_isr_nonstall;
+        gops->mc.isr_thread_stall = mc_gk20a_intr_thread_stall;
+        gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall;
+}
diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.h b/drivers/gpu/nvgpu/gk20a/mc_gk20a.h
new file mode 100644
index 00000000..7264ab41
--- /dev/null
+++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2014, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef MC_GK20A_H
+#define MC_GK20A_H
+struct gk20a;
+
+void gk20a_init_mc(struct gpu_ops *gops);
+void mc_gk20a_intr_enable(struct gk20a *g);
+irqreturn_t mc_gk20a_isr_stall(struct gk20a *g);
+irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g);
+irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g);
+irqreturn_t mc_gk20a_intr_thread_nonstall(struct gk20a *g);
+#endif
diff --git a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c
index ec786a44..574ad926 100644
--- a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c
@@ -27,6 +27,7 @@
 #include "mm_gm20b.h"
 #include "pmu_gm20b.h"
 #include "clk_gm20b.h"
+#include "mc_gm20b.h"
 #include <linux/tegra-fuse.h>
 #include "regops_gm20b.h"
 
@@ -113,6 +114,7 @@ int gm20b_init_hal(struct gpu_ops *gops)
         }
 #endif
 
+        gm20b_init_mc(gops);
         gm20b_init_ltc(gops);
         gm20b_init_gr(gops);
         gm20b_init_ltc(gops);
diff --git a/drivers/gpu/nvgpu/gm20b/mc_gm20b.c b/drivers/gpu/nvgpu/gm20b/mc_gm20b.c
new file mode 100644
index 00000000..22dce1e7
--- /dev/null
+++ b/drivers/gpu/nvgpu/gm20b/mc_gm20b.c
@@ -0,0 +1,29 @@
+/*
+ * GK20A memory interface
+ *
+ * Copyright (c) 2014, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/types.h>
+
+#include "gk20a/gk20a.h"
+#include "gk20a/mc_gk20a.h"
+#include "mc_gm20b.h"
+
+void gm20b_init_mc(struct gpu_ops *gops)
+{
+        gops->mc.intr_enable = mc_gk20a_intr_enable;
+        gops->mc.isr_stall = mc_gk20a_isr_stall;
+        gops->mc.isr_nonstall = mc_gk20a_isr_nonstall;
+        gops->mc.isr_thread_stall = mc_gk20a_intr_thread_stall;
+        gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall;
+}
diff --git a/drivers/gpu/nvgpu/gm20b/mc_gm20b.h b/drivers/gpu/nvgpu/gm20b/mc_gm20b.h
new file mode 100644
index 00000000..b19bf6fe
--- /dev/null
+++ b/drivers/gpu/nvgpu/gm20b/mc_gm20b.h
@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2014, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef MC_GM20B_H
+#define MC_GM20B_H
+
+void gm20b_init_mc(struct gpu_ops *gops);
+#endif