From 942029a433390f3385ed9d6fc35476bbf9eafd98 Mon Sep 17 00:00:00 2001 From: Terje Bergstrom Date: Mon, 5 Jun 2017 14:25:35 -0700 Subject: gpu: nvgpu: Split non-stall interrupt handling Split handling of stalling interrupt to Linux specific chip agnostic and OS independent chip specific parts. Linux specific chip independent part contains handler for ISR and passing the control to a bottom half worker. It uses the new MC HALs intr_nonstall (query interrupt status), intr_nonstall_pause (pause interrupts), intr_nonstall_resume (resume interrupts), and is_intr1_pending (query per-engine interrupt bit). MC HAL isr_nonstall is removed, because its work is now handled in chip independent code. JIRA NVGPU-26 Change-Id: I3e4c9905ef6eef7f1cc9f71b0278518ae663f87e Signed-off-by: Terje Bergstrom Reviewed-on: http://git-master/r/1497048 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/common/linux/intr.c | 84 +++++++++++++++++++++ drivers/gpu/nvgpu/common/linux/intr.h | 2 + drivers/gpu/nvgpu/common/linux/module.c | 9 ++- drivers/gpu/nvgpu/common/linux/pci.c | 2 +- drivers/gpu/nvgpu/gk20a/gk20a.c | 7 -- drivers/gpu/nvgpu/gk20a/gk20a.h | 7 +- drivers/gpu/nvgpu/gk20a/mc_gk20a.c | 128 +++++++------------------------- drivers/gpu/nvgpu/gk20a/mc_gk20a.h | 7 +- drivers/gpu/nvgpu/gm20b/mc_gm20b.c | 6 +- drivers/gpu/nvgpu/gp10b/mc_gp10b.c | 54 +++++--------- drivers/gpu/nvgpu/gp10b/mc_gp10b.h | 2 - 11 files changed, 150 insertions(+), 158 deletions(-) diff --git a/drivers/gpu/nvgpu/common/linux/intr.c b/drivers/gpu/nvgpu/common/linux/intr.c index 77e44dd9..7d699dee 100644 --- a/drivers/gpu/nvgpu/common/linux/intr.c +++ b/drivers/gpu/nvgpu/common/linux/intr.c @@ -12,10 +12,12 @@ */ #include +#include #include "gk20a/gk20a.h" #include +#include irqreturn_t nvgpu_intr_stall(struct gk20a *g) { @@ -56,3 +58,85 @@ irqreturn_t nvgpu_intr_thread_stall(struct gk20a *g) return IRQ_HANDLED; } +irqreturn_t nvgpu_intr_nonstall(struct gk20a *g) +{ + u32 mc_intr_1; + u32 hw_irq_count; + u32 engine_id_idx; + u32 active_engine_id = 0; + u32 engine_enum = ENGINE_INVAL_GK20A; + int ops_old, ops_new, ops = 0; + if (!g->power_on) + return IRQ_NONE; + + /* not from gpu when sharing irq with others */ + mc_intr_1 = g->ops.mc.intr_nonstall(g); + if (unlikely(!mc_intr_1)) + return IRQ_NONE; + + g->ops.mc.intr_nonstall_pause(g); + + if (g->ops.mc.is_intr1_pending(g, NVGPU_UNIT_FIFO, mc_intr_1)) + ops |= gk20a_fifo_nonstall_isr(g); + + for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines; + engine_id_idx++) { + struct fifo_engine_info_gk20a *engine_info; + + active_engine_id = g->fifo.active_engines_list[engine_id_idx]; + engine_info = &g->fifo.engine_info[active_engine_id]; + + if (mc_intr_1 & engine_info->intr_mask) { + engine_enum = engine_info->engine_enum; + /* GR Engine */ + if (engine_enum == ENGINE_GR_GK20A) + ops |= gk20a_gr_nonstall_isr(g); + + /* CE Engine */ + if (((engine_enum == ENGINE_GRCE_GK20A) || + (engine_enum == ENGINE_ASYNC_CE_GK20A)) && + g->ops.ce2.isr_nonstall) + ops |= g->ops.ce2.isr_nonstall(g, + engine_info->inst_id, + engine_info->pri_base); + } + } + if (ops) { + do { + ops_old = atomic_read(&g->nonstall_ops); + ops_new = ops_old | ops; + } while (ops_old != atomic_cmpxchg(&g->nonstall_ops, + ops_old, ops_new)); + + queue_work(g->nonstall_work_queue, &g->nonstall_fn_work); + } + + hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count); + + /* sync handled irq counter before re-enabling interrupts */ + atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count); + + g->ops.mc.intr_nonstall_resume(g); + + wake_up_all(&g->sw_irq_nonstall_last_handled_wq); + + return IRQ_HANDLED; +} + +void nvgpu_intr_nonstall_cb(struct work_struct *work) +{ + struct gk20a *g = container_of(work, struct gk20a, nonstall_fn_work); + u32 ops; + bool semaphore_wakeup, post_events; + + do { + ops = atomic_xchg(&g->nonstall_ops, 0); + + semaphore_wakeup = ops & gk20a_nonstall_ops_wakeup_semaphore; + post_events = ops & gk20a_nonstall_ops_post_events; + + if (semaphore_wakeup) + gk20a_channel_semaphore_wakeup(g, post_events); + + } while (atomic_read(&g->nonstall_ops) != 0); +} diff --git a/drivers/gpu/nvgpu/common/linux/intr.h b/drivers/gpu/nvgpu/common/linux/intr.h index 243d8f51..d43cdccb 100644 --- a/drivers/gpu/nvgpu/common/linux/intr.h +++ b/drivers/gpu/nvgpu/common/linux/intr.h @@ -17,4 +17,6 @@ struct gk20a; irqreturn_t nvgpu_intr_stall(struct gk20a *g); irqreturn_t nvgpu_intr_thread_stall(struct gk20a *g); +irqreturn_t nvgpu_intr_nonstall(struct gk20a *g); +void nvgpu_intr_nonstall_cb(struct work_struct *work); #endif diff --git a/drivers/gpu/nvgpu/common/linux/module.c b/drivers/gpu/nvgpu/common/linux/module.c index 4f7fc3fa..34a0ded6 100644 --- a/drivers/gpu/nvgpu/common/linux/module.c +++ b/drivers/gpu/nvgpu/common/linux/module.c @@ -162,6 +162,13 @@ int gk20a_pm_finalize_poweron(struct device *dev) nice_value = task_nice(current); set_user_nice(current, -20); + /* Enable interrupt workqueue */ + if (!g->nonstall_work_queue) { + g->nonstall_work_queue = alloc_workqueue("%s", + WQ_HIGHPRI, 1, "mc_nonstall"); + INIT_WORK(&g->nonstall_fn_work, nvgpu_intr_nonstall_cb); + } + err = gk20a_finalize_poweron(g); set_user_nice(current, nice_value); if (err) @@ -492,7 +499,7 @@ static irqreturn_t gk20a_intr_isr_nonstall(int irq, void *dev_id) { struct gk20a *g = dev_id; - return g->ops.mc.isr_nonstall(g); + return nvgpu_intr_nonstall(g); } static irqreturn_t gk20a_intr_thread_stall(int irq, void *dev_id) diff --git a/drivers/gpu/nvgpu/common/linux/pci.c b/drivers/gpu/nvgpu/common/linux/pci.c index b66a6658..4351ba5b 100644 --- a/drivers/gpu/nvgpu/common/linux/pci.c +++ b/drivers/gpu/nvgpu/common/linux/pci.c @@ -235,7 +235,7 @@ static irqreturn_t nvgpu_pci_isr(int irq, void *dev_id) irqreturn_t ret_nonstall; ret_stall = nvgpu_intr_stall(g); - ret_nonstall = g->ops.mc.isr_nonstall(g); + ret_nonstall = nvgpu_intr_nonstall(g); #if defined(CONFIG_PCI_MSI) /* Send MSI EOI */ diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c index e4d454fe..b038cd81 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gk20a.c @@ -187,13 +187,6 @@ int gk20a_finalize_poweron(struct gk20a *g) if (g->ops.clk.disable_slowboot) g->ops.clk.disable_slowboot(g); - /* Enable interrupt workqueue */ - if (!g->nonstall_work_queue) { - g->nonstall_work_queue = alloc_workqueue("%s", - WQ_HIGHPRI, 1, "mc_nonstall"); - INIT_WORK(&g->nonstall_fn_work, g->ops.mc.isr_nonstall_cb); - } - gk20a_enable_priv_ring(g); /* TBD: move this after graphics init in which blcg/slcg is enabled. diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index bd93cc33..60494050 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h @@ -36,7 +36,6 @@ struct nvgpu_clk_pll_debug_data; #include #include #include -#include #include #include #ifdef CONFIG_DEBUG_FS @@ -841,14 +840,14 @@ struct gpu_ops { void (*intr_unit_config)(struct gk20a *g, bool enable, bool is_stalling, u32 unit); void (*isr_stall)(struct gk20a *g); - irqreturn_t (*isr_nonstall)(struct gk20a *g); - void (*isr_thread_nonstall)(struct gk20a *g, u32 intr); - void (*isr_nonstall_cb)(struct work_struct *work); bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr); u32 intr_mask_restore[4]; u32 (*intr_stall)(struct gk20a *g); void (*intr_stall_pause)(struct gk20a *g); void (*intr_stall_resume)(struct gk20a *g); + u32 (*intr_nonstall)(struct gk20a *g); + void (*intr_nonstall_pause)(struct gk20a *g); + void (*intr_nonstall_resume)(struct gk20a *g); void (*enable)(struct gk20a *g, u32 units); void (*disable)(struct gk20a *g, u32 units); void (*reset)(struct gk20a *g, u32 units); diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c index 2cdcaaeb..111872a2 100644 --- a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c @@ -24,62 +24,6 @@ #include -void mc_gk20a_nonstall_cb(struct work_struct *work) -{ - struct gk20a *g = container_of(work, struct gk20a, nonstall_fn_work); - u32 ops; - bool semaphore_wakeup, post_events; - - do { - ops = atomic_xchg(&g->nonstall_ops, 0); - - semaphore_wakeup = ops & gk20a_nonstall_ops_wakeup_semaphore; - post_events = ops & gk20a_nonstall_ops_post_events; - - if (semaphore_wakeup) - gk20a_channel_semaphore_wakeup(g, post_events); - - } while (atomic_read(&g->nonstall_ops) != 0); -} - -irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g) -{ - u32 mc_intr_1; - u32 hw_irq_count; - - if (!g->power_on) - return IRQ_NONE; - - /* not from gpu when sharing irq with others */ - mc_intr_1 = gk20a_readl(g, mc_intr_1_r()); - if (unlikely(!mc_intr_1)) - return IRQ_NONE; - - gk20a_writel(g, mc_intr_en_1_r(), - mc_intr_en_1_inta_disabled_f()); - - /* flush previous write */ - gk20a_readl(g, mc_intr_en_1_r()); - - if (g->ops.mc.isr_thread_nonstall) - g->ops.mc.isr_thread_nonstall(g, mc_intr_1); - - hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count); - - /* sync handled irq counter before re-enabling interrupts */ - atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count); - - gk20a_writel(g, mc_intr_en_1_r(), - mc_intr_en_1_inta_hardware_f()); - - /* flush previous write */ - gk20a_readl(g, mc_intr_en_1_r()); - - wake_up_all(&g->sw_irq_nonstall_last_handled_wq); - - return IRQ_HANDLED; -} - void mc_gk20a_isr_stall(struct gk20a *g) { u32 mc_intr_0; @@ -128,49 +72,6 @@ void mc_gk20a_isr_stall(struct gk20a *g) atomic_set(&g->sw_irq_stall_last_handled, hw_irq_count); } -void mc_gk20a_intr_thread_nonstall(struct gk20a *g, u32 mc_intr_1) -{ - u32 engine_id_idx; - u32 active_engine_id = 0; - u32 engine_enum = ENGINE_INVAL_GK20A; - int ops_old, ops_new, ops = 0; - - if (g->ops.mc.is_intr1_pending(g, NVGPU_UNIT_FIFO, mc_intr_1)) - ops |= gk20a_fifo_nonstall_isr(g); - - for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines; - engine_id_idx++) { - active_engine_id = g->fifo.active_engines_list[engine_id_idx]; - - if (mc_intr_1 & - g->fifo.engine_info[active_engine_id].intr_mask) { - engine_enum = g->fifo.engine_info[active_engine_id].engine_enum; - /* GR Engine */ - if (engine_enum == ENGINE_GR_GK20A) - ops |= gk20a_gr_nonstall_isr(g); - - /* CE Engine */ - if (((engine_enum == ENGINE_GRCE_GK20A) || - (engine_enum == ENGINE_ASYNC_CE_GK20A)) && - g->ops.ce2.isr_nonstall) - ops |= g->ops.ce2.isr_nonstall(g, - g->fifo.engine_info[active_engine_id]. - inst_id, - g->fifo.engine_info[active_engine_id]. - pri_base); - } - } - if (ops) { - do { - ops_old = atomic_read(&g->nonstall_ops); - ops_new = ops_old | ops; - } while (ops_old != atomic_cmpxchg(&g->nonstall_ops, - ops_old, ops_new)); - - queue_work(g->nonstall_work_queue, &g->nonstall_fn_work); - } -} - void mc_gk20a_intr_enable(struct gk20a *g) { u32 eng_intr_mask = gk20a_fifo_engine_interrupt_mask(g); @@ -226,11 +127,34 @@ void mc_gk20a_intr_stall_resume(struct gk20a *g) gk20a_readl(g, mc_intr_en_0_r()); } +void mc_gk20a_intr_nonstall_pause(struct gk20a *g) +{ + gk20a_writel(g, mc_intr_en_1_r(), + mc_intr_en_0_inta_disabled_f()); + + /* flush previous write */ + gk20a_readl(g, mc_intr_en_1_r()); +} + +void mc_gk20a_intr_nonstall_resume(struct gk20a *g) +{ + gk20a_writel(g, mc_intr_en_1_r(), + mc_intr_en_0_inta_hardware_f()); + + /* flush previous write */ + gk20a_readl(g, mc_intr_en_1_r()); +} + u32 mc_gk20a_intr_stall(struct gk20a *g) { return gk20a_readl(g, mc_intr_0_r()); } +u32 mc_gk20a_intr_nonstall(struct gk20a *g) +{ + return gk20a_readl(g, mc_intr_1_r()); +} + void gk20a_mc_disable(struct gk20a *g, u32 units) { u32 pmc; @@ -320,9 +244,9 @@ void gk20a_init_mc(struct gpu_ops *gops) gops->mc.intr_stall = mc_gk20a_intr_stall; gops->mc.intr_stall_pause = mc_gk20a_intr_stall_pause; gops->mc.intr_stall_resume = mc_gk20a_intr_stall_resume; - gops->mc.isr_nonstall = mc_gk20a_isr_nonstall; - gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall; - gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb; + gops->mc.intr_nonstall = mc_gk20a_intr_nonstall; + gops->mc.intr_nonstall_pause = mc_gk20a_intr_nonstall_pause; + gops->mc.intr_nonstall_resume = mc_gk20a_intr_nonstall_resume; gops->mc.enable = gk20a_mc_enable; gops->mc.disable = gk20a_mc_disable; gops->mc.reset = gk20a_mc_reset; diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.h b/drivers/gpu/nvgpu/gk20a/mc_gk20a.h index 6c0d159a..7ecd27bf 100644 --- a/drivers/gpu/nvgpu/gk20a/mc_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.h @@ -23,10 +23,9 @@ void mc_gk20a_isr_stall(struct gk20a *g); u32 mc_gk20a_intr_stall(struct gk20a *g); void mc_gk20a_intr_stall_pause(struct gk20a *g); void mc_gk20a_intr_stall_resume(struct gk20a *g); -irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g); -irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g); -void mc_gk20a_intr_thread_nonstall(struct gk20a *g, u32 intr); -void mc_gk20a_nonstall_cb(struct work_struct *work); +u32 mc_gk20a_intr_nonstall(struct gk20a *g); +void mc_gk20a_intr_nonstall_pause(struct gk20a *g); +void mc_gk20a_intr_nonstall_resume(struct gk20a *g); void gk20a_mc_enable(struct gk20a *g, u32 units); void gk20a_mc_disable(struct gk20a *g, u32 units); void gk20a_mc_reset(struct gk20a *g, u32 units); diff --git a/drivers/gpu/nvgpu/gm20b/mc_gm20b.c b/drivers/gpu/nvgpu/gm20b/mc_gm20b.c index 7330a027..005ec729 100644 --- a/drivers/gpu/nvgpu/gm20b/mc_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/mc_gm20b.c @@ -25,9 +25,9 @@ void gm20b_init_mc(struct gpu_ops *gops) gops->mc.intr_stall = mc_gk20a_intr_stall; gops->mc.intr_stall_pause = mc_gk20a_intr_stall_pause; gops->mc.intr_stall_resume = mc_gk20a_intr_stall_resume; - gops->mc.isr_nonstall = mc_gk20a_isr_nonstall; - gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall; - gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb; + gops->mc.intr_nonstall = mc_gk20a_intr_nonstall; + gops->mc.intr_nonstall_pause = mc_gk20a_intr_nonstall_pause; + gops->mc.intr_nonstall_resume = mc_gk20a_intr_nonstall_resume; gops->mc.enable = gk20a_mc_enable; gops->mc.disable = gk20a_mc_disable; gops->mc.reset = gk20a_mc_reset; diff --git a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c index 7ccea370..39ad8f9b 100644 --- a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c @@ -68,37 +68,6 @@ void mc_gp10b_intr_unit_config(struct gk20a *g, bool enable, gk20a_writel(g, reg, mask); } -irqreturn_t mc_gp10b_isr_nonstall(struct gk20a *g) -{ - u32 mc_intr_1; - u32 hw_irq_count; - - if (!g->power_on) - return IRQ_NONE; - - /* not from gpu when sharing irq with others */ - mc_intr_1 = gk20a_readl(g, mc_intr_r(1)); - if (unlikely(!mc_intr_1)) - return IRQ_NONE; - - gk20a_writel(g, mc_intr_en_clear_r(1), 0xffffffff); - - if (g->ops.mc.isr_thread_nonstall) - g->ops.mc.isr_thread_nonstall(g, mc_intr_1); - - hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count); - - gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), - g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); - - /* sync handled irq counter before re-enabling interrupts */ - atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count); - - wake_up_all(&g->sw_irq_nonstall_last_handled_wq); - - return IRQ_HANDLED; -} - void mc_gp10b_isr_stall(struct gk20a *g) { u32 mc_intr_0; @@ -170,6 +139,23 @@ void mc_gp10b_intr_stall_resume(struct gk20a *g) g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_STALLING]); } +u32 mc_gp10b_intr_nonstall(struct gk20a *g) +{ + return gk20a_readl(g, mc_intr_r(NVGPU_MC_INTR_NONSTALLING)); +} + +void mc_gp10b_intr_nonstall_pause(struct gk20a *g) +{ + gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), + 0xffffffff); +} + +void mc_gp10b_intr_nonstall_resume(struct gk20a *g) +{ + gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), + g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); +} + static bool mc_gp10b_is_intr1_pending(struct gk20a *g, enum nvgpu_unit unit, u32 mc_intr_1) { @@ -202,9 +188,9 @@ void gp10b_init_mc(struct gpu_ops *gops) gops->mc.intr_stall = mc_gp10b_intr_stall; gops->mc.intr_stall_pause = mc_gp10b_intr_stall_pause; gops->mc.intr_stall_resume = mc_gp10b_intr_stall_resume; - gops->mc.isr_nonstall = mc_gp10b_isr_nonstall; - gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall; - gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb; + gops->mc.intr_nonstall = mc_gp10b_intr_nonstall; + gops->mc.intr_nonstall_pause = mc_gp10b_intr_nonstall_pause; + gops->mc.intr_nonstall_resume = mc_gp10b_intr_nonstall_resume; gops->mc.enable = gk20a_mc_enable; gops->mc.disable = gk20a_mc_disable; gops->mc.reset = gk20a_mc_reset; diff --git a/drivers/gpu/nvgpu/gp10b/mc_gp10b.h b/drivers/gpu/nvgpu/gp10b/mc_gp10b.h index 31867a88..ceba0b39 100644 --- a/drivers/gpu/nvgpu/gp10b/mc_gp10b.h +++ b/drivers/gpu/nvgpu/gp10b/mc_gp10b.h @@ -25,6 +25,4 @@ void mc_gp10b_intr_enable(struct gk20a *g); void mc_gp10b_intr_unit_config(struct gk20a *g, bool enable, bool is_stalling, u32 mask); void mc_gp10b_isr_stall(struct gk20a *g); -irqreturn_t mc_gp10b_isr_nonstall(struct gk20a *g); -irqreturn_t mc_gp10b_intr_thread_nonstall(struct gk20a *g); #endif -- cgit v1.2.2