gpu: nvgpu: refactor interrupt handling

JIRA: EVLR-1004

(*) Refactor the non-stalling interrupt path to execute clear on the
top half, so on dGPU case processing of stalling interrupts does not
block non-stalling one.
(*) Use a worker thread to do semaphore wakeups and allow batching of
the non-stalling operations.
(*) Fix a bug where some gpus will not properly track the completion
of interrupts, preventing safe driver unloads

Change-Id: Icc90a3acba544c97ec6a9285ab235d337ab9eefa
Signed-off-by: David Nieto <dmartineznie@nvidia.com>
Reviewed-on: http://git-master/r/1312796
Reviewed-by: svccoveritychecker <svccoveritychecker@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Lakshmanan M <lm@nvidia.com>
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Reviewed-by: Navneet Kumar <navneetk@nvidia.com>

13 files changed, 140 insertions, 127 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
index db1ac539..3fed937e 100644
--- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
@@ -76,8 +76,9 @@ void gk20a_ce2_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
         return;
 }
 
-void gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
+int gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
 {
+        int ops = 0;
         u32 ce2_intr = gk20a_readl(g, ce2_intr_status_r());
 
         gk20a_dbg(gpu_dbg_intr, "ce2 nonstall isr %08x\n", ce2_intr);
@@ -85,12 +86,10 @@ void gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
         if (ce2_intr & ce2_intr_status_nonblockpipe_pending_f()) {
                 gk20a_writel(g, ce2_intr_status_r(),
                         ce2_nonblockpipe_isr(g, ce2_intr));
-
-                /* wake threads waiting in this channel */
-                gk20a_channel_semaphore_wakeup(g, true);
+                ops |= (gk20a_nonstall_ops_wakeup_semaphore |
+                        gk20a_nonstall_ops_post_events);
         }
-
-        return;
+        return ops;
 }
 void gk20a_init_ce2(struct gpu_ops *gops)
 {
diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h
index 1bb25dd1..5cdd233e 100644
--- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h
@@ -26,7 +26,7 @@
 
 void gk20a_init_ce2(struct gpu_ops *gops);
 void gk20a_ce2_isr(struct gk20a *g, u32 inst_id, u32 pri_base);
-void gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base);
+int gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base);
 
 /* CE command utility macros */
 #define NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK 0xffffffff
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index b2efc1fa..04d68872 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -2377,7 +2377,7 @@ void gk20a_fifo_isr(struct gk20a *g)
         return;
 }
 
-void gk20a_fifo_nonstall_isr(struct gk20a *g)
+int gk20a_fifo_nonstall_isr(struct gk20a *g)
 {
         u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r());
         u32 clear_intr = 0;
@@ -2389,9 +2389,7 @@ void gk20a_fifo_nonstall_isr(struct gk20a *g)
 
         gk20a_writel(g, fifo_intr_0_r(), clear_intr);
 
-        gk20a_channel_semaphore_wakeup(g, false);
-
-        return;
+        return gk20a_nonstall_ops_wakeup_semaphore;
 }
 
 void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg)
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
index 37808928..c67ab456 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
@@ -214,7 +214,7 @@ int gk20a_init_fifo_support(struct gk20a *g);
 int gk20a_init_fifo_setup_hw(struct gk20a *g);
 
 void gk20a_fifo_isr(struct gk20a *g);
-void gk20a_fifo_nonstall_isr(struct gk20a *g);
+int gk20a_fifo_nonstall_isr(struct gk20a *g);
 
 int gk20a_fifo_preempt_channel(struct gk20a *g, u32 hw_chid);
 int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid);
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c
index d07918b0..e995dcbf 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.c
@@ -697,12 +697,6 @@ static irqreturn_t gk20a_intr_thread_stall(int irq, void *dev_id)
         return g->ops.mc.isr_thread_stall(g);
 }
 
-static irqreturn_t gk20a_intr_thread_nonstall(int irq, void *dev_id)
-{
-        struct gk20a *g = dev_id;
-        return g->ops.mc.isr_thread_nonstall(g);
-}
-
 void gk20a_remove_support(struct device *dev)
 {
         struct gk20a *g = get_gk20a(dev);
@@ -717,6 +711,12 @@ void gk20a_remove_support(struct device *dev)
 
         gk20a_channel_cancel_pending_sema_waits(g);
 
+        if (g->nonstall_work_queue) {
+                cancel_work_sync(&g->nonstall_fn_work);
+                destroy_workqueue(g->nonstall_work_queue);
+                g->nonstall_work_queue = NULL;
+        }
+
         if (g->pmu.remove_support)
                 g->pmu.remove_support(&g->pmu);
 
@@ -932,6 +932,13 @@ int gk20a_pm_finalize_poweron(struct device *dev)
         if (g->ops.clk.disable_slowboot)
                 g->ops.clk.disable_slowboot(g);
 
+        /* Enable interrupt workqueue */
+        if (!g->nonstall_work_queue) {
+                g->nonstall_work_queue = alloc_workqueue("%s",
+                                                WQ_HIGHPRI, 1, "mc_nonstall");
+                INIT_WORK(&g->nonstall_fn_work, g->ops.mc.isr_nonstall_cb);
+        }
+
         gk20a_enable_priv_ring(g);
 
         /* TBD: move this after graphics init in which blcg/slcg is enabled.
@@ -1617,10 +1624,9 @@ static int gk20a_probe(struct platform_device *dev)
                                 gk20a->irq_stall);
                 return err;
         }
-        err = devm_request_threaded_irq(&dev->dev,
+        err = devm_request_irq(&dev->dev,
                         gk20a->irq_nonstall,
                         gk20a_intr_isr_nonstall,
-                        gk20a_intr_thread_nonstall,
                         0, "gk20a_nonstall", gk20a);
         if (err) {
                 dev_err(&dev->dev,
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index 12c9a3ea..092bf7ae 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -155,7 +155,7 @@ struct gpu_ops {
         } ltc;
         struct {
                 void (*isr_stall)(struct gk20a *g, u32 inst_id, u32 pri_base);
-                void (*isr_nonstall)(struct gk20a *g, u32 inst_id, u32 pri_base);
+                int (*isr_nonstall)(struct gk20a *g, u32 inst_id, u32 pri_base);
         } ce2;
         struct {
                 int (*init_fs_state)(struct gk20a *g);
@@ -735,7 +735,8 @@ struct gpu_ops {
                 irqreturn_t (*isr_stall)(struct gk20a *g);
                 irqreturn_t (*isr_nonstall)(struct gk20a *g);
                 irqreturn_t (*isr_thread_stall)(struct gk20a *g);
-                irqreturn_t (*isr_thread_nonstall)(struct gk20a *g);
+                void (*isr_thread_nonstall)(struct gk20a *g, u32 intr);
+                void (*isr_nonstall_cb)(struct work_struct *work);
                 u32 intr_mask_restore[4];
         } mc;
         struct {
@@ -848,6 +849,10 @@ struct gk20a {
         atomic_t usage_count;
         int driver_is_dying;
 
+        atomic_t nonstall_ops;
+        struct work_struct nonstall_fn_work;
+        struct workqueue_struct *nonstall_work_queue;
+
         struct resource *reg_mem;
         void __iomem *regs;
         void __iomem *regs_saved;
@@ -1151,6 +1156,12 @@ enum gk20a_dbg_categories {
         gpu_dbg_mem     = BIT(31), /* memory accesses, very verbose */
 };
 
+/* operations that will need to be executed on non stall workqueue */
+enum gk20a_nonstall_ops {
+        gk20a_nonstall_ops_wakeup_semaphore = BIT(0), /* wake up semaphore */
+        gk20a_nonstall_ops_post_events = BIT(1),
+};
+
 extern u32 gk20a_dbg_mask;
 #ifdef CONFIG_GK20A_TRACE_PRINTK
 extern u32 gk20a_dbg_ftrace;
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
index 3ab63862..afa665ab 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
@@ -6582,6 +6582,7 @@ int gk20a_gr_isr(struct gk20a *g)
 
 int gk20a_gr_nonstall_isr(struct gk20a *g)
 {
+        int ops = 0;
         u32 gr_intr = gk20a_readl(g, gr_intr_nonstall_r());
 
         gk20a_dbg(gpu_dbg_intr, "pgraph nonstall intr %08x", gr_intr);
@@ -6590,11 +6591,10 @@ int gk20a_gr_nonstall_isr(struct gk20a *g)
                 /* Clear the interrupt */
                 gk20a_writel(g, gr_intr_nonstall_r(),
                         gr_intr_nonstall_trap_pending_f());
-                /* Wakeup all the waiting channels */
-                gk20a_channel_semaphore_wakeup(g, true);
+                ops |= (gk20a_nonstall_ops_wakeup_semaphore |
+                        gk20a_nonstall_ops_post_events);
         }
-
-        return 0;
+        return ops;
 }
 
 int gr_gk20a_fecs_get_reglist_img_size(struct gk20a *g, u32 *size)
diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c
index 65f1a119..caab6b5e 100644
--- a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c
@@ -21,6 +21,24 @@
 
 #include <nvgpu/hw/gk20a/hw_mc_gk20a.h>
 
+void mc_gk20a_nonstall_cb(struct work_struct *work)
+{
+        struct gk20a *g = container_of(work, struct gk20a, nonstall_fn_work);
+        u32 ops;
+        bool semaphore_wakeup, post_events;
+
+        do {
+                ops = atomic_xchg(&g->nonstall_ops, 0);
+
+                semaphore_wakeup = ops & gk20a_nonstall_ops_wakeup_semaphore;
+                post_events = ops & gk20a_nonstall_ops_post_events;
+
+                if (semaphore_wakeup)
+                        gk20a_channel_semaphore_wakeup(g, post_events);
+
+        } while (atomic_read(&g->nonstall_ops) != 0);
+}
+
 irqreturn_t mc_gk20a_isr_stall(struct gk20a *g)
 {
         u32 mc_intr_0;
@@ -51,6 +69,7 @@ irqreturn_t mc_gk20a_isr_stall(struct gk20a *g)
 irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g)
 {
         u32 mc_intr_1;
+        u32 hw_irq_count;
 
         if (!g->power_on)
                 return IRQ_NONE;
@@ -66,9 +85,23 @@ irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g)
         /* flush previous write */
         gk20a_readl(g, mc_intr_en_1_r());
 
-        atomic_inc(&g->hw_irq_nonstall_count);
+        if (g->ops.mc.isr_thread_nonstall)
+                g->ops.mc.isr_thread_nonstall(g, mc_intr_1);
 
-        return IRQ_WAKE_THREAD;
+        hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count);
+
+        /* sync handled irq counter before re-enabling interrupts */
+        atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count);
+
+        gk20a_writel(g, mc_intr_en_1_r(),
+                mc_intr_en_1_inta_hardware_f());
+
+        /* flush previous write */
+        gk20a_readl(g, mc_intr_en_1_r());
+
+        wake_up_all(&g->sw_irq_nonstall_last_handled_wq);
+
+        return IRQ_HANDLED;
 }
 
 irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g)
@@ -137,59 +170,47 @@ irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g)
         return IRQ_HANDLED;
 }
 
-irqreturn_t mc_gk20a_intr_thread_nonstall(struct gk20a *g)
+void mc_gk20a_intr_thread_nonstall(struct gk20a *g, u32 mc_intr_1)
 {
-        u32 mc_intr_1;
-        int hw_irq_count;
         u32 engine_id_idx;
         u32 active_engine_id = 0;
         u32 engine_enum = ENGINE_INVAL_GK20A;
-
-        gk20a_dbg(gpu_dbg_intr, "interrupt thread launched");
-
-        mc_intr_1 = gk20a_readl(g, mc_intr_1_r());
-        hw_irq_count = atomic_read(&g->hw_irq_nonstall_count);
-
-        gk20a_dbg(gpu_dbg_intr, "non-stall intr %08x\n", mc_intr_1);
+        int ops_old, ops_new, ops = 0;
 
         if (mc_intr_1 & mc_intr_0_pfifo_pending_f())
-                gk20a_fifo_nonstall_isr(g);
-        if (mc_intr_1 & mc_intr_0_priv_ring_pending_f())
-                gk20a_priv_ring_isr(g);
+                ops |= gk20a_fifo_nonstall_isr(g);
 
-        for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines; engine_id_idx++) {
+        for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines;
+                                                        engine_id_idx++) {
                 active_engine_id = g->fifo.active_engines_list[engine_id_idx];
 
-                if (mc_intr_1 & g->fifo.engine_info[active_engine_id].intr_mask) {
+                if (mc_intr_1 &
+                        g->fifo.engine_info[active_engine_id].intr_mask) {
                         engine_enum = g->fifo.engine_info[active_engine_id].engine_enum;
                         /* GR Engine */
-                        if (engine_enum == ENGINE_GR_GK20A) {
-                                gk20a_gr_nonstall_isr(g);
-                        }
+                        if (engine_enum == ENGINE_GR_GK20A)
+                                ops |= gk20a_gr_nonstall_isr(g);
 
                         /* CE Engine */
                         if (((engine_enum == ENGINE_GRCE_GK20A) ||
                                 (engine_enum == ENGINE_ASYNC_CE_GK20A)) &&
-                                g->ops.ce2.isr_nonstall) {
-                                        g->ops.ce2.isr_nonstall(g,
-                                        g->fifo.engine_info[active_engine_id].inst_id,
-                                        g->fifo.engine_info[active_engine_id].pri_base);
-                        }
+                                g->ops.ce2.isr_nonstall)
+                                        ops |= g->ops.ce2.isr_nonstall(g,
+                                        g->fifo.engine_info[active_engine_id].
+                                                                inst_id,
+                                        g->fifo.engine_info[active_engine_id].
+                                                                pri_base);
                 }
         }
-
-        /* sync handled irq counter before re-enabling interrupts */
-        atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count);
-
-        gk20a_writel(g, mc_intr_en_1_r(),
-                mc_intr_en_1_inta_hardware_f());
-
-        /* flush previous write */
-        gk20a_readl(g, mc_intr_en_1_r());
-
-        wake_up_all(&g->sw_irq_nonstall_last_handled_wq);
-
-        return IRQ_HANDLED;
+        if (ops) {
+                do {
+                        ops_old = atomic_read(&g->nonstall_ops);
+                        ops_new  = ops_old | ops;
+                } while (ops_old != atomic_cmpxchg(&g->nonstall_ops,
+                                                ops_old, ops_new));
+
+                queue_work(g->nonstall_work_queue, &g->nonstall_fn_work);
+        }
 }
 
 void mc_gk20a_intr_enable(struct gk20a *g)
@@ -237,4 +258,5 @@ void gk20a_init_mc(struct gpu_ops *gops)
         gops->mc.isr_nonstall = mc_gk20a_isr_nonstall;
         gops->mc.isr_thread_stall = mc_gk20a_intr_thread_stall;
         gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall;
+        gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb;
 }
diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.h b/drivers/gpu/nvgpu/gk20a/mc_gk20a.h
index 4bb3e118..1aad1a0b 100644
--- a/drivers/gpu/nvgpu/gk20a/mc_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2017, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -22,5 +22,6 @@ void mc_gk20a_intr_unit_config(struct gk20a *g, bool enable,
 irqreturn_t mc_gk20a_isr_stall(struct gk20a *g);
 irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g);
 irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g);
-irqreturn_t mc_gk20a_intr_thread_nonstall(struct gk20a *g);
+void mc_gk20a_intr_thread_nonstall(struct gk20a *g, u32 intr);
+void mc_gk20a_nonstall_cb(struct work_struct *work);
 #endif
diff --git a/drivers/gpu/nvgpu/gm20b/mc_gm20b.c b/drivers/gpu/nvgpu/gm20b/mc_gm20b.c
index 1d2d78e3..c8a42cd7 100644
--- a/drivers/gpu/nvgpu/gm20b/mc_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/mc_gm20b.c
@@ -1,7 +1,7 @@
 /*
  * GK20A memory interface
  *
- * Copyright (c) 2014, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2017, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -27,4 +27,5 @@ void gm20b_init_mc(struct gpu_ops *gops)
         gops->mc.isr_nonstall = mc_gk20a_isr_nonstall;
         gops->mc.isr_thread_stall = mc_gk20a_intr_thread_stall;
         gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall;
+        gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb;
 }
diff --git a/drivers/gpu/nvgpu/gp10b/ce_gp10b.c b/drivers/gpu/nvgpu/gp10b/ce_gp10b.c
index f19a4b0f..169309fa 100644
--- a/drivers/gpu/nvgpu/gp10b/ce_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/ce_gp10b.c
@@ -1,7 +1,7 @@
 /*
  * Pascal GPU series Copy Engine.
  *
- * Copyright (c) 2011-2016, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2011-2017, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -22,15 +22,6 @@
 
 #include <nvgpu/hw/gp10b/hw_ce_gp10b.h>
 
-static void ce_nonblockpipe_isr(struct gk20a *g, u32 fifo_intr)
-{
-        gk20a_dbg(gpu_dbg_intr, "ce non-blocking pipe interrupt\n");
-
-        /* wake theads waiting in this channel */
-        gk20a_channel_semaphore_wakeup(g, true);
-        return;
-}
-
 static u32 ce_blockpipe_isr(struct gk20a *g, u32 fifo_intr)
 {
         gk20a_dbg(gpu_dbg_intr, "ce blocking pipe interrupt\n");
@@ -63,8 +54,9 @@ static void gp10b_ce_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
         return;
 }
 
-static void gp10b_ce_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
+static int gp10b_ce_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
 {
+        int ops = 0;
         u32 ce_intr = gk20a_readl(g, ce_intr_status_r(inst_id));
 
         gk20a_dbg(gpu_dbg_intr, "ce nonstall isr %08x %08x\n", ce_intr, inst_id);
@@ -72,10 +64,11 @@ static void gp10b_ce_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
         if (ce_intr & ce_intr_status_nonblockpipe_pending_f()) {
                 gk20a_writel(g, ce_intr_status_r(inst_id),
                         ce_intr_status_nonblockpipe_pending_f());
-                ce_nonblockpipe_isr(g, ce_intr);
+                ops |= (gk20a_nonstall_ops_wakeup_semaphore |
+                        gk20a_nonstall_ops_post_events);
         }
 
-        return;
+        return ops;
 }
 void gp10b_init_ce(struct gpu_ops *gops)
 {
diff --git a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c
index 3f066c37..dfcbe398 100644
--- a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 
 #include "gk20a/gk20a.h"
+#include "gk20a/mc_gk20a.h"
 
 #include "mc_gp10b.h"
 
@@ -80,12 +81,15 @@ irqreturn_t mc_gp10b_isr_stall(struct gk20a *g)
 
         gk20a_writel(g, mc_intr_en_clear_r(0), 0xffffffff);
 
+        atomic_inc(&g->hw_irq_stall_count);
+
         return IRQ_WAKE_THREAD;
 }
 
 irqreturn_t mc_gp10b_isr_nonstall(struct gk20a *g)
 {
         u32 mc_intr_1;
+        u32 hw_irq_count;
 
         if (!g->power_on)
                 return IRQ_NONE;
@@ -97,12 +101,27 @@ irqreturn_t mc_gp10b_isr_nonstall(struct gk20a *g)
 
         gk20a_writel(g, mc_intr_en_clear_r(1), 0xffffffff);
 
-        return IRQ_WAKE_THREAD;
+        if (g->ops.mc.isr_thread_nonstall)
+                g->ops.mc.isr_thread_nonstall(g, mc_intr_1);
+
+        hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count);
+
+        gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING),
+                        g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]);
+
+        /* sync handled irq counter before re-enabling interrupts */
+        atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count);
+
+        wake_up_all(&g->sw_irq_nonstall_last_handled_wq);
+
+        return IRQ_HANDLED;
 }
 
 irqreturn_t mc_gp10b_intr_thread_stall(struct gk20a *g)
 {
         u32 mc_intr_0;
+        int hw_irq_count;
+
         u32 engine_id_idx;
         u32 active_engine_id = 0;
         u32 engine_enum = ENGINE_INVAL_GK20A;
@@ -110,6 +129,7 @@ irqreturn_t mc_gp10b_intr_thread_stall(struct gk20a *g)
         gk20a_dbg(gpu_dbg_intr, "interrupt thread launched");
 
         mc_intr_0 = gk20a_readl(g, mc_intr_r(0));
+        hw_irq_count = atomic_read(&g->hw_irq_stall_count);
 
         gk20a_dbg(gpu_dbg_intr, "stall intr %08x\n", mc_intr_0);
 
@@ -146,51 +166,13 @@ irqreturn_t mc_gp10b_intr_thread_stall(struct gk20a *g)
         if (mc_intr_0 & mc_intr_ltc_pending_f())
                 g->ops.ltc.isr(g);
 
+        /* sync handled irq counter before re-enabling interrupts */
+        atomic_set(&g->sw_irq_stall_last_handled, hw_irq_count);
+
         gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING),
                         g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_STALLING]);
 
-        return IRQ_HANDLED;
-}
-
-irqreturn_t mc_gp10b_intr_thread_nonstall(struct gk20a *g)
-{
-        u32 mc_intr_1;
-        u32 engine_id_idx;
-        u32 active_engine_id = 0;
-        u32 engine_enum = ENGINE_INVAL_GK20A;
-
-        gk20a_dbg(gpu_dbg_intr, "interrupt thread launched");
-
-        mc_intr_1 = gk20a_readl(g, mc_intr_r(1));
-
-        gk20a_dbg(gpu_dbg_intr, "non-stall intr %08x\n", mc_intr_1);
-
-        if (mc_intr_1 & mc_intr_pfifo_pending_f())
-                gk20a_fifo_nonstall_isr(g);
-
-        for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines; engine_id_idx++) {
-                active_engine_id = g->fifo.active_engines_list[engine_id_idx];
-
-                if (mc_intr_1 & g->fifo.engine_info[active_engine_id].intr_mask) {
-                        engine_enum = g->fifo.engine_info[active_engine_id].engine_enum;
-                        /* GR Engine */
-                        if (engine_enum == ENGINE_GR_GK20A) {
-                                gk20a_gr_nonstall_isr(g);
-                        }
-
-                        /* CE Engine */
-                        if (((engine_enum == ENGINE_GRCE_GK20A) ||
-                                (engine_enum == ENGINE_ASYNC_CE_GK20A)) &&
-                                g->ops.ce2.isr_nonstall) {
-                                        g->ops.ce2.isr_nonstall(g,
-                                        g->fifo.engine_info[active_engine_id].inst_id,
-                                        g->fifo.engine_info[active_engine_id].pri_base);
-                        }
-                }
-        }
-
-        gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING),
-                        g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]);
+        wake_up_all(&g->sw_irq_stall_last_handled_wq);
 
         return IRQ_HANDLED;
 }
@@ -202,5 +184,7 @@ void gp10b_init_mc(struct gpu_ops *gops)
         gops->mc.isr_stall = mc_gp10b_isr_stall;
         gops->mc.isr_nonstall = mc_gp10b_isr_nonstall;
         gops->mc.isr_thread_stall = mc_gp10b_intr_thread_stall;
-        gops->mc.isr_thread_nonstall = mc_gp10b_intr_thread_nonstall;
+        gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall;
+        gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb;
+
 }
diff --git a/drivers/gpu/nvgpu/pci.c b/drivers/gpu/nvgpu/pci.c
index 39559dac..69e16267 100644
--- a/drivers/gpu/nvgpu/pci.c
+++ b/drivers/gpu/nvgpu/pci.c
@@ -236,8 +236,7 @@ static irqreturn_t nvgpu_pci_isr(int irq, void *dev_id)
                 g->ops.xve.rearm_msi(g);
 #endif
 
-        return (ret_stall == IRQ_NONE && ret_nonstall == IRQ_NONE) ?
-                IRQ_NONE : IRQ_WAKE_THREAD;
+        return (ret_stall == IRQ_NONE) ? ret_nonstall : IRQ_WAKE_THREAD;
 }
 
 static irqreturn_t nvgpu_pci_intr_thread(int irq, void *dev_id)
@@ -245,7 +244,6 @@ static irqreturn_t nvgpu_pci_intr_thread(int irq, void *dev_id)
         struct gk20a *g = dev_id;
 
         g->ops.mc.isr_thread_stall(g);
-        g->ops.mc.isr_thread_nonstall(g);
 
         return IRQ_HANDLED;
 }