9 files changed, 79 insertions, 72 deletions

diff --git a/drivers/gpu/nvgpu/common/linux/driver_common.c b/drivers/gpu/nvgpu/common/linux/driver_common.c
index 4e2cb2b4..bd1b7611 100644
--- a/drivers/gpu/nvgpu/common/linux/driver_common.c
+++ b/drivers/gpu/nvgpu/common/linux/driver_common.c
@@ -38,8 +38,8 @@ static void nvgpu_init_vars(struct gk20a *g)
         struct device *dev = dev_from_gk20a(g);
         struct gk20a_platform *platform = dev_get_drvdata(dev);
 
-        init_waitqueue_head(&g->sw_irq_stall_last_handled_wq);
-        init_waitqueue_head(&g->sw_irq_nonstall_last_handled_wq);
+        init_waitqueue_head(&l->sw_irq_stall_last_handled_wq);
+        init_waitqueue_head(&l->sw_irq_nonstall_last_handled_wq);
         gk20a_init_gr(g);
 
         init_rwsem(&g->busy_lock);
@@ -236,18 +236,19 @@ static int cyclic_delta(int a, int b)
  */
 void nvgpu_wait_for_deferred_interrupts(struct gk20a *g)
 {
-        int stall_irq_threshold = atomic_read(&g->hw_irq_stall_count);
-        int nonstall_irq_threshold = atomic_read(&g->hw_irq_nonstall_count);
+        struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+        int stall_irq_threshold = atomic_read(&l->hw_irq_stall_count);
+        int nonstall_irq_threshold = atomic_read(&l->hw_irq_nonstall_count);
 
         /* wait until all stalling irqs are handled */
-        wait_event(g->sw_irq_stall_last_handled_wq,
+        wait_event(l->sw_irq_stall_last_handled_wq,
                    cyclic_delta(stall_irq_threshold,
-                                atomic_read(&g->sw_irq_stall_last_handled))
+                                atomic_read(&l->sw_irq_stall_last_handled))
                    <= 0);
 
         /* wait until all non-stalling irqs are handled */
-        wait_event(g->sw_irq_nonstall_last_handled_wq,
+        wait_event(l->sw_irq_nonstall_last_handled_wq,
                    cyclic_delta(nonstall_irq_threshold,
-                                atomic_read(&g->sw_irq_nonstall_last_handled))
+                                atomic_read(&l->sw_irq_nonstall_last_handled))
                    <= 0);
 }
diff --git a/drivers/gpu/nvgpu/common/linux/intr.c b/drivers/gpu/nvgpu/common/linux/intr.c
index 7d699dee..da177b55 100644
--- a/drivers/gpu/nvgpu/common/linux/intr.c
+++ b/drivers/gpu/nvgpu/common/linux/intr.c
@@ -18,9 +18,11 @@
 
 #include <nvgpu/atomic.h>
 #include <nvgpu/unit.h>
+#include "os_linux.h"
 
 irqreturn_t nvgpu_intr_stall(struct gk20a *g)
 {
+        struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
         u32 mc_intr_0;
 
         trace_mc_gk20a_intr_stall(g->name);
@@ -35,7 +37,7 @@ irqreturn_t nvgpu_intr_stall(struct gk20a *g)
 
         g->ops.mc.intr_stall_pause(g);
 
-        atomic_inc(&g->hw_irq_stall_count);
+        atomic_inc(&l->hw_irq_stall_count);
 
         trace_mc_gk20a_intr_stall_done(g->name);
 
@@ -44,14 +46,20 @@ irqreturn_t nvgpu_intr_stall(struct gk20a *g)
 
 irqreturn_t nvgpu_intr_thread_stall(struct gk20a *g)
 {
+        struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+        int hw_irq_count;
+
         gk20a_dbg(gpu_dbg_intr, "interrupt thread launched");
 
         trace_mc_gk20a_intr_thread_stall(g->name);
 
+        hw_irq_count = atomic_read(&l->hw_irq_stall_count);
         g->ops.mc.isr_stall(g);
         g->ops.mc.intr_stall_resume(g);
+        /* sync handled irq counter before re-enabling interrupts */
+        atomic_set(&l->sw_irq_stall_last_handled, hw_irq_count);
 
-        wake_up_all(&g->sw_irq_stall_last_handled_wq);
+        wake_up_all(&l->sw_irq_stall_last_handled_wq);
 
         trace_mc_gk20a_intr_thread_stall_done(g->name);
 
@@ -66,6 +74,8 @@ irqreturn_t nvgpu_intr_nonstall(struct gk20a *g)
         u32 active_engine_id = 0;
         u32 engine_enum = ENGINE_INVAL_GK20A;
         int ops_old, ops_new, ops = 0;
+        struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+
         if (!g->power_on)
                 return IRQ_NONE;
 
@@ -103,34 +113,36 @@ irqreturn_t nvgpu_intr_nonstall(struct gk20a *g)
         }
         if (ops) {
                 do {
-                        ops_old = atomic_read(&g->nonstall_ops);
+                        ops_old = atomic_read(&l->nonstall_ops);
                         ops_new  = ops_old | ops;
-                } while (ops_old != atomic_cmpxchg(&g->nonstall_ops,
+                } while (ops_old != atomic_cmpxchg(&l->nonstall_ops,
                                                 ops_old, ops_new));
 
-                queue_work(g->nonstall_work_queue, &g->nonstall_fn_work);
+                queue_work(l->nonstall_work_queue, &l->nonstall_fn_work);
         }
 
-        hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count);
+        hw_irq_count = atomic_inc_return(&l->hw_irq_nonstall_count);
 
         /* sync handled irq counter before re-enabling interrupts */
-        atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count);
+        atomic_set(&l->sw_irq_nonstall_last_handled, hw_irq_count);
 
         g->ops.mc.intr_nonstall_resume(g);
 
-        wake_up_all(&g->sw_irq_nonstall_last_handled_wq);
+        wake_up_all(&l->sw_irq_nonstall_last_handled_wq);
 
         return IRQ_HANDLED;
 }
 
 void nvgpu_intr_nonstall_cb(struct work_struct *work)
 {
-        struct gk20a *g = container_of(work, struct gk20a, nonstall_fn_work);
+        struct nvgpu_os_linux *l =
+                container_of(work, struct nvgpu_os_linux, nonstall_fn_work);
+        struct gk20a *g = &l->g;
         u32 ops;
         bool semaphore_wakeup, post_events;
 
         do {
-                ops = atomic_xchg(&g->nonstall_ops, 0);
+                ops = atomic_xchg(&l->nonstall_ops, 0);
 
                 semaphore_wakeup = ops & gk20a_nonstall_ops_wakeup_semaphore;
                 post_events = ops & gk20a_nonstall_ops_post_events;
@@ -138,5 +150,5 @@ void nvgpu_intr_nonstall_cb(struct work_struct *work)
                 if (semaphore_wakeup)
                         gk20a_channel_semaphore_wakeup(g, post_events);
 
-        } while (atomic_read(&g->nonstall_ops) != 0);
+        } while (atomic_read(&l->nonstall_ops) != 0);
 }
diff --git a/drivers/gpu/nvgpu/common/linux/module.c b/drivers/gpu/nvgpu/common/linux/module.c
index d19a7a45..cbfe6ad7 100644
--- a/drivers/gpu/nvgpu/common/linux/module.c
+++ b/drivers/gpu/nvgpu/common/linux/module.c
@@ -136,6 +136,7 @@ void gk20a_idle(struct gk20a *g)
 int gk20a_pm_finalize_poweron(struct device *dev)
 {
         struct gk20a *g = get_gk20a(dev);
+        struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
         struct gk20a_platform *platform = gk20a_get_platform(dev);
         int err, nice_value;
 
@@ -163,10 +164,10 @@ int gk20a_pm_finalize_poweron(struct device *dev)
         set_user_nice(current, -20);
 
         /* Enable interrupt workqueue */
-        if (!g->nonstall_work_queue) {
-                g->nonstall_work_queue = alloc_workqueue("%s",
+        if (!l->nonstall_work_queue) {
+                l->nonstall_work_queue = alloc_workqueue("%s",
                                                 WQ_HIGHPRI, 1, "mc_nonstall");
-                INIT_WORK(&g->nonstall_fn_work, nvgpu_intr_nonstall_cb);
+                INIT_WORK(&l->nonstall_fn_work, nvgpu_intr_nonstall_cb);
         }
 
         err = gk20a_finalize_poweron(g);
@@ -827,6 +828,34 @@ static int gk20a_pm_init(struct device *dev)
         return err;
 }
 
+/*
+ * Start the process for unloading the driver. Set NVGPU_DRIVER_IS_DYING.
+ */
+void gk20a_driver_start_unload(struct gk20a *g)
+{
+        struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+
+        gk20a_dbg(gpu_dbg_shutdown, "Driver is now going down!\n");
+
+        down_write(&g->busy_lock);
+        __nvgpu_set_enabled(g, NVGPU_DRIVER_IS_DYING, true);
+        up_write(&g->busy_lock);
+
+        if (g->is_virtual)
+                return;
+
+        gk20a_wait_for_idle(dev_from_gk20a(g));
+
+        nvgpu_wait_for_deferred_interrupts(g);
+        gk20a_channel_cancel_pending_sema_waits(g);
+
+        if (l->nonstall_work_queue) {
+                cancel_work_sync(&l->nonstall_fn_work);
+                destroy_workqueue(l->nonstall_work_queue);
+                l->nonstall_work_queue = NULL;
+        }
+}
+
 static inline void set_gk20a(struct platform_device *pdev, struct gk20a *gk20a)
 {
         gk20a_get_platform(&pdev->dev)->g = gk20a;
diff --git a/drivers/gpu/nvgpu/common/linux/module.h b/drivers/gpu/nvgpu/common/linux/module.h
index 0fde0b41..cfbbc0c7 100644
--- a/drivers/gpu/nvgpu/common/linux/module.h
+++ b/drivers/gpu/nvgpu/common/linux/module.h
@@ -18,6 +18,7 @@ struct device;
 
 int gk20a_pm_finalize_poweron(struct device *dev);
 void gk20a_remove_support(struct gk20a *g);
+void gk20a_driver_start_unload(struct gk20a *g);
 
 extern struct class nvgpu_class;
 
diff --git a/drivers/gpu/nvgpu/common/linux/os_linux.h b/drivers/gpu/nvgpu/common/linux/os_linux.h
index cf012acc..8f304fe3 100644
--- a/drivers/gpu/nvgpu/common/linux/os_linux.h
+++ b/drivers/gpu/nvgpu/common/linux/os_linux.h
@@ -69,6 +69,20 @@ struct nvgpu_os_linux {
         struct devfreq *devfreq;
 
         struct device_dma_parameters dma_parms;
+
+        atomic_t hw_irq_stall_count;
+        atomic_t hw_irq_nonstall_count;
+
+        wait_queue_head_t sw_irq_stall_last_handled_wq;
+        atomic_t sw_irq_stall_last_handled;
+
+        atomic_t nonstall_ops;
+
+        wait_queue_head_t sw_irq_nonstall_last_handled_wq;
+        atomic_t sw_irq_nonstall_last_handled;
+
+        struct work_struct nonstall_fn_work;
+        struct workqueue_struct *nonstall_work_queue;
 };
 
 static inline struct nvgpu_os_linux *nvgpu_os_linux_from_gk20a(struct gk20a *g)
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c
index 21e861fe..6350bcf5 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.c
@@ -358,32 +358,6 @@ int gk20a_can_busy(struct gk20a *g)
         return 1;
 }
 
-/*
- * Start the process for unloading the driver. Set NVGPU_DRIVER_IS_DYING.
- */
-void gk20a_driver_start_unload(struct gk20a *g)
-{
-        gk20a_dbg(gpu_dbg_shutdown, "Driver is now going down!\n");
-
-        down_write(&g->busy_lock);
-        __nvgpu_set_enabled(g, NVGPU_DRIVER_IS_DYING, true);
-        up_write(&g->busy_lock);
-
-        if (g->is_virtual)
-                return;
-
-        gk20a_wait_for_idle(dev_from_gk20a(g));
-
-        nvgpu_wait_for_deferred_interrupts(g);
-        gk20a_channel_cancel_pending_sema_waits(g);
-
-        if (g->nonstall_work_queue) {
-                cancel_work_sync(&g->nonstall_fn_work);
-                destroy_workqueue(g->nonstall_work_queue);
-                g->nonstall_work_queue = NULL;
-        }
-}
-
 int gk20a_wait_for_idle(struct device *dev)
 {
         struct gk20a *g = get_gk20a(dev);
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index 90c35a7b..8d9318b2 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -1042,10 +1042,6 @@ struct gk20a {
 
         atomic_t usage_count;
 
-        atomic_t nonstall_ops;
-        struct work_struct nonstall_fn_work;
-        struct workqueue_struct *nonstall_work_queue;
-
         struct kref refcount;
 
         struct resource *reg_mem;
@@ -1224,15 +1220,6 @@ struct gk20a {
         u32 max_ltc_count;
         u32 ltc_count;
 
-        atomic_t hw_irq_stall_count;
-        atomic_t hw_irq_nonstall_count;
-
-        atomic_t sw_irq_stall_last_handled;
-        wait_queue_head_t sw_irq_stall_last_handled_wq;
-
-        atomic_t sw_irq_nonstall_last_handled;
-        wait_queue_head_t sw_irq_nonstall_last_handled_wq;
-
         struct gk20a_channel_worker {
                 struct nvgpu_thread poll_task;
                 atomic_t put;
@@ -1485,7 +1472,6 @@ int __gk20a_do_idle(struct gk20a *g, bool force_reset);
 int __gk20a_do_unidle(struct gk20a *g);
 
 int gk20a_can_busy(struct gk20a *g);
-void gk20a_driver_start_unload(struct gk20a *g);
 int gk20a_wait_for_idle(struct device *dev);
 
 #define NVGPU_GPU_ARCHITECTURE_SHIFT 4
diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c
index accda972..e25fcfc3 100644
--- a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c
@@ -27,13 +27,11 @@
 void mc_gk20a_isr_stall(struct gk20a *g)
 {
         u32 mc_intr_0;
-        int hw_irq_count;
         u32 engine_id_idx;
         u32 active_engine_id = 0;
         u32 engine_enum = ENGINE_INVAL_GK20A;
 
         mc_intr_0 = g->ops.mc.intr_stall(g);
-        hw_irq_count = atomic_read(&g->hw_irq_stall_count);
 
         gk20a_dbg(gpu_dbg_intr, "stall intr %08x\n", mc_intr_0);
 
@@ -67,9 +65,6 @@ void mc_gk20a_isr_stall(struct gk20a *g)
                 g->ops.ltc.isr(g);
         if (mc_intr_0 & mc_intr_0_pbus_pending_f())
                 g->ops.bus.isr(g);
-
-        /* sync handled irq counter before re-enabling interrupts */
-        atomic_set(&g->sw_irq_stall_last_handled, hw_irq_count);
 }
 
 void mc_gk20a_intr_enable(struct gk20a *g)
diff --git a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c
index 718869f6..b7a52349 100644
--- a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c
@@ -71,14 +71,12 @@ void mc_gp10b_intr_unit_config(struct gk20a *g, bool enable,
 void mc_gp10b_isr_stall(struct gk20a *g)
 {
         u32 mc_intr_0;
-        int hw_irq_count;
 
         u32 engine_id_idx;
         u32 active_engine_id = 0;
         u32 engine_enum = ENGINE_INVAL_GK20A;
 
         mc_intr_0 = gk20a_readl(g, mc_intr_r(0));
-        hw_irq_count = atomic_read(&g->hw_irq_stall_count);
 
         gk20a_dbg(gpu_dbg_intr, "stall intr 0x%08x\n", mc_intr_0);
 
@@ -116,9 +114,6 @@ void mc_gp10b_isr_stall(struct gk20a *g)
         if (mc_intr_0 & mc_intr_pbus_pending_f())
                 g->ops.bus.isr(g);
 
-        /* sync handled irq counter before re-enabling interrupts */
-        atomic_set(&g->sw_irq_stall_last_handled, hw_irq_count);
-
         gk20a_dbg(gpu_dbg_intr, "stall intr done 0x%08x\n", mc_intr_0);
 
 }