diff options
Diffstat (limited to 'drivers/gpu')
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/ce2_gk20a.c | 11 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/ce2_gk20a.h | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | 6 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/fifo_gk20a.h | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/gk20a.c | 22 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/gk20a.h | 15 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/gr_gk20a.c | 8 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/mc_gk20a.c | 98 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/mc_gk20a.h | 5 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gm20b/mc_gm20b.c | 3 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gp10b/ce_gp10b.c | 19 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gp10b/mc_gp10b.c | 72 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/pci.c | 4 |
13 files changed, 140 insertions, 127 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c index db1ac539..3fed937e 100644 --- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c | |||
@@ -76,8 +76,9 @@ void gk20a_ce2_isr(struct gk20a *g, u32 inst_id, u32 pri_base) | |||
76 | return; | 76 | return; |
77 | } | 77 | } |
78 | 78 | ||
79 | void gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base) | 79 | int gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base) |
80 | { | 80 | { |
81 | int ops = 0; | ||
81 | u32 ce2_intr = gk20a_readl(g, ce2_intr_status_r()); | 82 | u32 ce2_intr = gk20a_readl(g, ce2_intr_status_r()); |
82 | 83 | ||
83 | gk20a_dbg(gpu_dbg_intr, "ce2 nonstall isr %08x\n", ce2_intr); | 84 | gk20a_dbg(gpu_dbg_intr, "ce2 nonstall isr %08x\n", ce2_intr); |
@@ -85,12 +86,10 @@ void gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base) | |||
85 | if (ce2_intr & ce2_intr_status_nonblockpipe_pending_f()) { | 86 | if (ce2_intr & ce2_intr_status_nonblockpipe_pending_f()) { |
86 | gk20a_writel(g, ce2_intr_status_r(), | 87 | gk20a_writel(g, ce2_intr_status_r(), |
87 | ce2_nonblockpipe_isr(g, ce2_intr)); | 88 | ce2_nonblockpipe_isr(g, ce2_intr)); |
88 | 89 | ops |= (gk20a_nonstall_ops_wakeup_semaphore | | |
89 | /* wake threads waiting in this channel */ | 90 | gk20a_nonstall_ops_post_events); |
90 | gk20a_channel_semaphore_wakeup(g, true); | ||
91 | } | 91 | } |
92 | 92 | return ops; | |
93 | return; | ||
94 | } | 93 | } |
95 | void gk20a_init_ce2(struct gpu_ops *gops) | 94 | void gk20a_init_ce2(struct gpu_ops *gops) |
96 | { | 95 | { |
diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h index 1bb25dd1..5cdd233e 100644 --- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h | |||
@@ -26,7 +26,7 @@ | |||
26 | 26 | ||
27 | void gk20a_init_ce2(struct gpu_ops *gops); | 27 | void gk20a_init_ce2(struct gpu_ops *gops); |
28 | void gk20a_ce2_isr(struct gk20a *g, u32 inst_id, u32 pri_base); | 28 | void gk20a_ce2_isr(struct gk20a *g, u32 inst_id, u32 pri_base); |
29 | void gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base); | 29 | int gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base); |
30 | 30 | ||
31 | /* CE command utility macros */ | 31 | /* CE command utility macros */ |
32 | #define NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK 0xffffffff | 32 | #define NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK 0xffffffff |
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c index b2efc1fa..04d68872 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | |||
@@ -2377,7 +2377,7 @@ void gk20a_fifo_isr(struct gk20a *g) | |||
2377 | return; | 2377 | return; |
2378 | } | 2378 | } |
2379 | 2379 | ||
2380 | void gk20a_fifo_nonstall_isr(struct gk20a *g) | 2380 | int gk20a_fifo_nonstall_isr(struct gk20a *g) |
2381 | { | 2381 | { |
2382 | u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r()); | 2382 | u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r()); |
2383 | u32 clear_intr = 0; | 2383 | u32 clear_intr = 0; |
@@ -2389,9 +2389,7 @@ void gk20a_fifo_nonstall_isr(struct gk20a *g) | |||
2389 | 2389 | ||
2390 | gk20a_writel(g, fifo_intr_0_r(), clear_intr); | 2390 | gk20a_writel(g, fifo_intr_0_r(), clear_intr); |
2391 | 2391 | ||
2392 | gk20a_channel_semaphore_wakeup(g, false); | 2392 | return gk20a_nonstall_ops_wakeup_semaphore; |
2393 | |||
2394 | return; | ||
2395 | } | 2393 | } |
2396 | 2394 | ||
2397 | void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg) | 2395 | void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg) |
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h index 37808928..c67ab456 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h | |||
@@ -214,7 +214,7 @@ int gk20a_init_fifo_support(struct gk20a *g); | |||
214 | int gk20a_init_fifo_setup_hw(struct gk20a *g); | 214 | int gk20a_init_fifo_setup_hw(struct gk20a *g); |
215 | 215 | ||
216 | void gk20a_fifo_isr(struct gk20a *g); | 216 | void gk20a_fifo_isr(struct gk20a *g); |
217 | void gk20a_fifo_nonstall_isr(struct gk20a *g); | 217 | int gk20a_fifo_nonstall_isr(struct gk20a *g); |
218 | 218 | ||
219 | int gk20a_fifo_preempt_channel(struct gk20a *g, u32 hw_chid); | 219 | int gk20a_fifo_preempt_channel(struct gk20a *g, u32 hw_chid); |
220 | int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid); | 220 | int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid); |
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c index d07918b0..e995dcbf 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gk20a.c | |||
@@ -697,12 +697,6 @@ static irqreturn_t gk20a_intr_thread_stall(int irq, void *dev_id) | |||
697 | return g->ops.mc.isr_thread_stall(g); | 697 | return g->ops.mc.isr_thread_stall(g); |
698 | } | 698 | } |
699 | 699 | ||
700 | static irqreturn_t gk20a_intr_thread_nonstall(int irq, void *dev_id) | ||
701 | { | ||
702 | struct gk20a *g = dev_id; | ||
703 | return g->ops.mc.isr_thread_nonstall(g); | ||
704 | } | ||
705 | |||
706 | void gk20a_remove_support(struct device *dev) | 700 | void gk20a_remove_support(struct device *dev) |
707 | { | 701 | { |
708 | struct gk20a *g = get_gk20a(dev); | 702 | struct gk20a *g = get_gk20a(dev); |
@@ -717,6 +711,12 @@ void gk20a_remove_support(struct device *dev) | |||
717 | 711 | ||
718 | gk20a_channel_cancel_pending_sema_waits(g); | 712 | gk20a_channel_cancel_pending_sema_waits(g); |
719 | 713 | ||
714 | if (g->nonstall_work_queue) { | ||
715 | cancel_work_sync(&g->nonstall_fn_work); | ||
716 | destroy_workqueue(g->nonstall_work_queue); | ||
717 | g->nonstall_work_queue = NULL; | ||
718 | } | ||
719 | |||
720 | if (g->pmu.remove_support) | 720 | if (g->pmu.remove_support) |
721 | g->pmu.remove_support(&g->pmu); | 721 | g->pmu.remove_support(&g->pmu); |
722 | 722 | ||
@@ -932,6 +932,13 @@ int gk20a_pm_finalize_poweron(struct device *dev) | |||
932 | if (g->ops.clk.disable_slowboot) | 932 | if (g->ops.clk.disable_slowboot) |
933 | g->ops.clk.disable_slowboot(g); | 933 | g->ops.clk.disable_slowboot(g); |
934 | 934 | ||
935 | /* Enable interrupt workqueue */ | ||
936 | if (!g->nonstall_work_queue) { | ||
937 | g->nonstall_work_queue = alloc_workqueue("%s", | ||
938 | WQ_HIGHPRI, 1, "mc_nonstall"); | ||
939 | INIT_WORK(&g->nonstall_fn_work, g->ops.mc.isr_nonstall_cb); | ||
940 | } | ||
941 | |||
935 | gk20a_enable_priv_ring(g); | 942 | gk20a_enable_priv_ring(g); |
936 | 943 | ||
937 | /* TBD: move this after graphics init in which blcg/slcg is enabled. | 944 | /* TBD: move this after graphics init in which blcg/slcg is enabled. |
@@ -1617,10 +1624,9 @@ static int gk20a_probe(struct platform_device *dev) | |||
1617 | gk20a->irq_stall); | 1624 | gk20a->irq_stall); |
1618 | return err; | 1625 | return err; |
1619 | } | 1626 | } |
1620 | err = devm_request_threaded_irq(&dev->dev, | 1627 | err = devm_request_irq(&dev->dev, |
1621 | gk20a->irq_nonstall, | 1628 | gk20a->irq_nonstall, |
1622 | gk20a_intr_isr_nonstall, | 1629 | gk20a_intr_isr_nonstall, |
1623 | gk20a_intr_thread_nonstall, | ||
1624 | 0, "gk20a_nonstall", gk20a); | 1630 | 0, "gk20a_nonstall", gk20a); |
1625 | if (err) { | 1631 | if (err) { |
1626 | dev_err(&dev->dev, | 1632 | dev_err(&dev->dev, |
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index 12c9a3ea..092bf7ae 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h | |||
@@ -155,7 +155,7 @@ struct gpu_ops { | |||
155 | } ltc; | 155 | } ltc; |
156 | struct { | 156 | struct { |
157 | void (*isr_stall)(struct gk20a *g, u32 inst_id, u32 pri_base); | 157 | void (*isr_stall)(struct gk20a *g, u32 inst_id, u32 pri_base); |
158 | void (*isr_nonstall)(struct gk20a *g, u32 inst_id, u32 pri_base); | 158 | int (*isr_nonstall)(struct gk20a *g, u32 inst_id, u32 pri_base); |
159 | } ce2; | 159 | } ce2; |
160 | struct { | 160 | struct { |
161 | int (*init_fs_state)(struct gk20a *g); | 161 | int (*init_fs_state)(struct gk20a *g); |
@@ -735,7 +735,8 @@ struct gpu_ops { | |||
735 | irqreturn_t (*isr_stall)(struct gk20a *g); | 735 | irqreturn_t (*isr_stall)(struct gk20a *g); |
736 | irqreturn_t (*isr_nonstall)(struct gk20a *g); | 736 | irqreturn_t (*isr_nonstall)(struct gk20a *g); |
737 | irqreturn_t (*isr_thread_stall)(struct gk20a *g); | 737 | irqreturn_t (*isr_thread_stall)(struct gk20a *g); |
738 | irqreturn_t (*isr_thread_nonstall)(struct gk20a *g); | 738 | void (*isr_thread_nonstall)(struct gk20a *g, u32 intr); |
739 | void (*isr_nonstall_cb)(struct work_struct *work); | ||
739 | u32 intr_mask_restore[4]; | 740 | u32 intr_mask_restore[4]; |
740 | } mc; | 741 | } mc; |
741 | struct { | 742 | struct { |
@@ -848,6 +849,10 @@ struct gk20a { | |||
848 | atomic_t usage_count; | 849 | atomic_t usage_count; |
849 | int driver_is_dying; | 850 | int driver_is_dying; |
850 | 851 | ||
852 | atomic_t nonstall_ops; | ||
853 | struct work_struct nonstall_fn_work; | ||
854 | struct workqueue_struct *nonstall_work_queue; | ||
855 | |||
851 | struct resource *reg_mem; | 856 | struct resource *reg_mem; |
852 | void __iomem *regs; | 857 | void __iomem *regs; |
853 | void __iomem *regs_saved; | 858 | void __iomem *regs_saved; |
@@ -1151,6 +1156,12 @@ enum gk20a_dbg_categories { | |||
1151 | gpu_dbg_mem = BIT(31), /* memory accesses, very verbose */ | 1156 | gpu_dbg_mem = BIT(31), /* memory accesses, very verbose */ |
1152 | }; | 1157 | }; |
1153 | 1158 | ||
1159 | /* operations that will need to be executed on non stall workqueue */ | ||
1160 | enum gk20a_nonstall_ops { | ||
1161 | gk20a_nonstall_ops_wakeup_semaphore = BIT(0), /* wake up semaphore */ | ||
1162 | gk20a_nonstall_ops_post_events = BIT(1), | ||
1163 | }; | ||
1164 | |||
1154 | extern u32 gk20a_dbg_mask; | 1165 | extern u32 gk20a_dbg_mask; |
1155 | #ifdef CONFIG_GK20A_TRACE_PRINTK | 1166 | #ifdef CONFIG_GK20A_TRACE_PRINTK |
1156 | extern u32 gk20a_dbg_ftrace; | 1167 | extern u32 gk20a_dbg_ftrace; |
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c index 3ab63862..afa665ab 100644 --- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c | |||
@@ -6582,6 +6582,7 @@ int gk20a_gr_isr(struct gk20a *g) | |||
6582 | 6582 | ||
6583 | int gk20a_gr_nonstall_isr(struct gk20a *g) | 6583 | int gk20a_gr_nonstall_isr(struct gk20a *g) |
6584 | { | 6584 | { |
6585 | int ops = 0; | ||
6585 | u32 gr_intr = gk20a_readl(g, gr_intr_nonstall_r()); | 6586 | u32 gr_intr = gk20a_readl(g, gr_intr_nonstall_r()); |
6586 | 6587 | ||
6587 | gk20a_dbg(gpu_dbg_intr, "pgraph nonstall intr %08x", gr_intr); | 6588 | gk20a_dbg(gpu_dbg_intr, "pgraph nonstall intr %08x", gr_intr); |
@@ -6590,11 +6591,10 @@ int gk20a_gr_nonstall_isr(struct gk20a *g) | |||
6590 | /* Clear the interrupt */ | 6591 | /* Clear the interrupt */ |
6591 | gk20a_writel(g, gr_intr_nonstall_r(), | 6592 | gk20a_writel(g, gr_intr_nonstall_r(), |
6592 | gr_intr_nonstall_trap_pending_f()); | 6593 | gr_intr_nonstall_trap_pending_f()); |
6593 | /* Wakeup all the waiting channels */ | 6594 | ops |= (gk20a_nonstall_ops_wakeup_semaphore | |
6594 | gk20a_channel_semaphore_wakeup(g, true); | 6595 | gk20a_nonstall_ops_post_events); |
6595 | } | 6596 | } |
6596 | 6597 | return ops; | |
6597 | return 0; | ||
6598 | } | 6598 | } |
6599 | 6599 | ||
6600 | int gr_gk20a_fecs_get_reglist_img_size(struct gk20a *g, u32 *size) | 6600 | int gr_gk20a_fecs_get_reglist_img_size(struct gk20a *g, u32 *size) |
diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c index 65f1a119..caab6b5e 100644 --- a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c | |||
@@ -21,6 +21,24 @@ | |||
21 | 21 | ||
22 | #include <nvgpu/hw/gk20a/hw_mc_gk20a.h> | 22 | #include <nvgpu/hw/gk20a/hw_mc_gk20a.h> |
23 | 23 | ||
24 | void mc_gk20a_nonstall_cb(struct work_struct *work) | ||
25 | { | ||
26 | struct gk20a *g = container_of(work, struct gk20a, nonstall_fn_work); | ||
27 | u32 ops; | ||
28 | bool semaphore_wakeup, post_events; | ||
29 | |||
30 | do { | ||
31 | ops = atomic_xchg(&g->nonstall_ops, 0); | ||
32 | |||
33 | semaphore_wakeup = ops & gk20a_nonstall_ops_wakeup_semaphore; | ||
34 | post_events = ops & gk20a_nonstall_ops_post_events; | ||
35 | |||
36 | if (semaphore_wakeup) | ||
37 | gk20a_channel_semaphore_wakeup(g, post_events); | ||
38 | |||
39 | } while (atomic_read(&g->nonstall_ops) != 0); | ||
40 | } | ||
41 | |||
24 | irqreturn_t mc_gk20a_isr_stall(struct gk20a *g) | 42 | irqreturn_t mc_gk20a_isr_stall(struct gk20a *g) |
25 | { | 43 | { |
26 | u32 mc_intr_0; | 44 | u32 mc_intr_0; |
@@ -51,6 +69,7 @@ irqreturn_t mc_gk20a_isr_stall(struct gk20a *g) | |||
51 | irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g) | 69 | irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g) |
52 | { | 70 | { |
53 | u32 mc_intr_1; | 71 | u32 mc_intr_1; |
72 | u32 hw_irq_count; | ||
54 | 73 | ||
55 | if (!g->power_on) | 74 | if (!g->power_on) |
56 | return IRQ_NONE; | 75 | return IRQ_NONE; |
@@ -66,9 +85,23 @@ irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g) | |||
66 | /* flush previous write */ | 85 | /* flush previous write */ |
67 | gk20a_readl(g, mc_intr_en_1_r()); | 86 | gk20a_readl(g, mc_intr_en_1_r()); |
68 | 87 | ||
69 | atomic_inc(&g->hw_irq_nonstall_count); | 88 | if (g->ops.mc.isr_thread_nonstall) |
89 | g->ops.mc.isr_thread_nonstall(g, mc_intr_1); | ||
70 | 90 | ||
71 | return IRQ_WAKE_THREAD; | 91 | hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count); |
92 | |||
93 | /* sync handled irq counter before re-enabling interrupts */ | ||
94 | atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count); | ||
95 | |||
96 | gk20a_writel(g, mc_intr_en_1_r(), | ||
97 | mc_intr_en_1_inta_hardware_f()); | ||
98 | |||
99 | /* flush previous write */ | ||
100 | gk20a_readl(g, mc_intr_en_1_r()); | ||
101 | |||
102 | wake_up_all(&g->sw_irq_nonstall_last_handled_wq); | ||
103 | |||
104 | return IRQ_HANDLED; | ||
72 | } | 105 | } |
73 | 106 | ||
74 | irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g) | 107 | irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g) |
@@ -137,59 +170,47 @@ irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g) | |||
137 | return IRQ_HANDLED; | 170 | return IRQ_HANDLED; |
138 | } | 171 | } |
139 | 172 | ||
140 | irqreturn_t mc_gk20a_intr_thread_nonstall(struct gk20a *g) | 173 | void mc_gk20a_intr_thread_nonstall(struct gk20a *g, u32 mc_intr_1) |
141 | { | 174 | { |
142 | u32 mc_intr_1; | ||
143 | int hw_irq_count; | ||
144 | u32 engine_id_idx; | 175 | u32 engine_id_idx; |
145 | u32 active_engine_id = 0; | 176 | u32 active_engine_id = 0; |
146 | u32 engine_enum = ENGINE_INVAL_GK20A; | 177 | u32 engine_enum = ENGINE_INVAL_GK20A; |
147 | 178 | int ops_old, ops_new, ops = 0; | |
148 | gk20a_dbg(gpu_dbg_intr, "interrupt thread launched"); | ||
149 | |||
150 | mc_intr_1 = gk20a_readl(g, mc_intr_1_r()); | ||
151 | hw_irq_count = atomic_read(&g->hw_irq_nonstall_count); | ||
152 | |||
153 | gk20a_dbg(gpu_dbg_intr, "non-stall intr %08x\n", mc_intr_1); | ||
154 | 179 | ||
155 | if (mc_intr_1 & mc_intr_0_pfifo_pending_f()) | 180 | if (mc_intr_1 & mc_intr_0_pfifo_pending_f()) |
156 | gk20a_fifo_nonstall_isr(g); | 181 | ops |= gk20a_fifo_nonstall_isr(g); |
157 | if (mc_intr_1 & mc_intr_0_priv_ring_pending_f()) | ||
158 | gk20a_priv_ring_isr(g); | ||
159 | 182 | ||
160 | for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines; engine_id_idx++) { | 183 | for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines; |
184 | engine_id_idx++) { | ||
161 | active_engine_id = g->fifo.active_engines_list[engine_id_idx]; | 185 | active_engine_id = g->fifo.active_engines_list[engine_id_idx]; |
162 | 186 | ||
163 | if (mc_intr_1 & g->fifo.engine_info[active_engine_id].intr_mask) { | 187 | if (mc_intr_1 & |
188 | g->fifo.engine_info[active_engine_id].intr_mask) { | ||
164 | engine_enum = g->fifo.engine_info[active_engine_id].engine_enum; | 189 | engine_enum = g->fifo.engine_info[active_engine_id].engine_enum; |
165 | /* GR Engine */ | 190 | /* GR Engine */ |
166 | if (engine_enum == ENGINE_GR_GK20A) { | 191 | if (engine_enum == ENGINE_GR_GK20A) |
167 | gk20a_gr_nonstall_isr(g); | 192 | ops |= gk20a_gr_nonstall_isr(g); |
168 | } | ||
169 | 193 | ||
170 | /* CE Engine */ | 194 | /* CE Engine */ |
171 | if (((engine_enum == ENGINE_GRCE_GK20A) || | 195 | if (((engine_enum == ENGINE_GRCE_GK20A) || |
172 | (engine_enum == ENGINE_ASYNC_CE_GK20A)) && | 196 | (engine_enum == ENGINE_ASYNC_CE_GK20A)) && |
173 | g->ops.ce2.isr_nonstall) { | 197 | g->ops.ce2.isr_nonstall) |
174 | g->ops.ce2.isr_nonstall(g, | 198 | ops |= g->ops.ce2.isr_nonstall(g, |
175 | g->fifo.engine_info[active_engine_id].inst_id, | 199 | g->fifo.engine_info[active_engine_id]. |
176 | g->fifo.engine_info[active_engine_id].pri_base); | 200 | inst_id, |
177 | } | 201 | g->fifo.engine_info[active_engine_id]. |
202 | pri_base); | ||
178 | } | 203 | } |
179 | } | 204 | } |
180 | 205 | if (ops) { | |
181 | /* sync handled irq counter before re-enabling interrupts */ | 206 | do { |
182 | atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count); | 207 | ops_old = atomic_read(&g->nonstall_ops); |
183 | 208 | ops_new = ops_old | ops; | |
184 | gk20a_writel(g, mc_intr_en_1_r(), | 209 | } while (ops_old != atomic_cmpxchg(&g->nonstall_ops, |
185 | mc_intr_en_1_inta_hardware_f()); | 210 | ops_old, ops_new)); |
186 | 211 | ||
187 | /* flush previous write */ | 212 | queue_work(g->nonstall_work_queue, &g->nonstall_fn_work); |
188 | gk20a_readl(g, mc_intr_en_1_r()); | 213 | } |
189 | |||
190 | wake_up_all(&g->sw_irq_nonstall_last_handled_wq); | ||
191 | |||
192 | return IRQ_HANDLED; | ||
193 | } | 214 | } |
194 | 215 | ||
195 | void mc_gk20a_intr_enable(struct gk20a *g) | 216 | void mc_gk20a_intr_enable(struct gk20a *g) |
@@ -237,4 +258,5 @@ void gk20a_init_mc(struct gpu_ops *gops) | |||
237 | gops->mc.isr_nonstall = mc_gk20a_isr_nonstall; | 258 | gops->mc.isr_nonstall = mc_gk20a_isr_nonstall; |
238 | gops->mc.isr_thread_stall = mc_gk20a_intr_thread_stall; | 259 | gops->mc.isr_thread_stall = mc_gk20a_intr_thread_stall; |
239 | gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall; | 260 | gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall; |
261 | gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb; | ||
240 | } | 262 | } |
diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.h b/drivers/gpu/nvgpu/gk20a/mc_gk20a.h index 4bb3e118..1aad1a0b 100644 --- a/drivers/gpu/nvgpu/gk20a/mc_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.h | |||
@@ -1,5 +1,5 @@ | |||
1 | /* | 1 | /* |
2 | * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved. | 2 | * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. |
3 | * | 3 | * |
4 | * This program is free software; you can redistribute it and/or modify it | 4 | * This program is free software; you can redistribute it and/or modify it |
5 | * under the terms and conditions of the GNU General Public License, | 5 | * under the terms and conditions of the GNU General Public License, |
@@ -22,5 +22,6 @@ void mc_gk20a_intr_unit_config(struct gk20a *g, bool enable, | |||
22 | irqreturn_t mc_gk20a_isr_stall(struct gk20a *g); | 22 | irqreturn_t mc_gk20a_isr_stall(struct gk20a *g); |
23 | irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g); | 23 | irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g); |
24 | irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g); | 24 | irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g); |
25 | irqreturn_t mc_gk20a_intr_thread_nonstall(struct gk20a *g); | 25 | void mc_gk20a_intr_thread_nonstall(struct gk20a *g, u32 intr); |
26 | void mc_gk20a_nonstall_cb(struct work_struct *work); | ||
26 | #endif | 27 | #endif |
diff --git a/drivers/gpu/nvgpu/gm20b/mc_gm20b.c b/drivers/gpu/nvgpu/gm20b/mc_gm20b.c index 1d2d78e3..c8a42cd7 100644 --- a/drivers/gpu/nvgpu/gm20b/mc_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/mc_gm20b.c | |||
@@ -1,7 +1,7 @@ | |||
1 | /* | 1 | /* |
2 | * GK20A memory interface | 2 | * GK20A memory interface |
3 | * | 3 | * |
4 | * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved. | 4 | * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. |
5 | * | 5 | * |
6 | * This program is free software; you can redistribute it and/or modify it | 6 | * This program is free software; you can redistribute it and/or modify it |
7 | * under the terms and conditions of the GNU General Public License, | 7 | * under the terms and conditions of the GNU General Public License, |
@@ -27,4 +27,5 @@ void gm20b_init_mc(struct gpu_ops *gops) | |||
27 | gops->mc.isr_nonstall = mc_gk20a_isr_nonstall; | 27 | gops->mc.isr_nonstall = mc_gk20a_isr_nonstall; |
28 | gops->mc.isr_thread_stall = mc_gk20a_intr_thread_stall; | 28 | gops->mc.isr_thread_stall = mc_gk20a_intr_thread_stall; |
29 | gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall; | 29 | gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall; |
30 | gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb; | ||
30 | } | 31 | } |
diff --git a/drivers/gpu/nvgpu/gp10b/ce_gp10b.c b/drivers/gpu/nvgpu/gp10b/ce_gp10b.c index f19a4b0f..169309fa 100644 --- a/drivers/gpu/nvgpu/gp10b/ce_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/ce_gp10b.c | |||
@@ -1,7 +1,7 @@ | |||
1 | /* | 1 | /* |
2 | * Pascal GPU series Copy Engine. | 2 | * Pascal GPU series Copy Engine. |
3 | * | 3 | * |
4 | * Copyright (c) 2011-2016, NVIDIA CORPORATION. All rights reserved. | 4 | * Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved. |
5 | * | 5 | * |
6 | * This program is free software; you can redistribute it and/or modify it | 6 | * This program is free software; you can redistribute it and/or modify it |
7 | * under the terms and conditions of the GNU General Public License, | 7 | * under the terms and conditions of the GNU General Public License, |
@@ -22,15 +22,6 @@ | |||
22 | 22 | ||
23 | #include <nvgpu/hw/gp10b/hw_ce_gp10b.h> | 23 | #include <nvgpu/hw/gp10b/hw_ce_gp10b.h> |
24 | 24 | ||
25 | static void ce_nonblockpipe_isr(struct gk20a *g, u32 fifo_intr) | ||
26 | { | ||
27 | gk20a_dbg(gpu_dbg_intr, "ce non-blocking pipe interrupt\n"); | ||
28 | |||
29 | /* wake theads waiting in this channel */ | ||
30 | gk20a_channel_semaphore_wakeup(g, true); | ||
31 | return; | ||
32 | } | ||
33 | |||
34 | static u32 ce_blockpipe_isr(struct gk20a *g, u32 fifo_intr) | 25 | static u32 ce_blockpipe_isr(struct gk20a *g, u32 fifo_intr) |
35 | { | 26 | { |
36 | gk20a_dbg(gpu_dbg_intr, "ce blocking pipe interrupt\n"); | 27 | gk20a_dbg(gpu_dbg_intr, "ce blocking pipe interrupt\n"); |
@@ -63,8 +54,9 @@ static void gp10b_ce_isr(struct gk20a *g, u32 inst_id, u32 pri_base) | |||
63 | return; | 54 | return; |
64 | } | 55 | } |
65 | 56 | ||
66 | static void gp10b_ce_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base) | 57 | static int gp10b_ce_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base) |
67 | { | 58 | { |
59 | int ops = 0; | ||
68 | u32 ce_intr = gk20a_readl(g, ce_intr_status_r(inst_id)); | 60 | u32 ce_intr = gk20a_readl(g, ce_intr_status_r(inst_id)); |
69 | 61 | ||
70 | gk20a_dbg(gpu_dbg_intr, "ce nonstall isr %08x %08x\n", ce_intr, inst_id); | 62 | gk20a_dbg(gpu_dbg_intr, "ce nonstall isr %08x %08x\n", ce_intr, inst_id); |
@@ -72,10 +64,11 @@ static void gp10b_ce_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base) | |||
72 | if (ce_intr & ce_intr_status_nonblockpipe_pending_f()) { | 64 | if (ce_intr & ce_intr_status_nonblockpipe_pending_f()) { |
73 | gk20a_writel(g, ce_intr_status_r(inst_id), | 65 | gk20a_writel(g, ce_intr_status_r(inst_id), |
74 | ce_intr_status_nonblockpipe_pending_f()); | 66 | ce_intr_status_nonblockpipe_pending_f()); |
75 | ce_nonblockpipe_isr(g, ce_intr); | 67 | ops |= (gk20a_nonstall_ops_wakeup_semaphore | |
68 | gk20a_nonstall_ops_post_events); | ||
76 | } | 69 | } |
77 | 70 | ||
78 | return; | 71 | return ops; |
79 | } | 72 | } |
80 | void gp10b_init_ce(struct gpu_ops *gops) | 73 | void gp10b_init_ce(struct gpu_ops *gops) |
81 | { | 74 | { |
diff --git a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c index 3f066c37..dfcbe398 100644 --- a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c | |||
@@ -16,6 +16,7 @@ | |||
16 | #include <linux/types.h> | 16 | #include <linux/types.h> |
17 | 17 | ||
18 | #include "gk20a/gk20a.h" | 18 | #include "gk20a/gk20a.h" |
19 | #include "gk20a/mc_gk20a.h" | ||
19 | 20 | ||
20 | #include "mc_gp10b.h" | 21 | #include "mc_gp10b.h" |
21 | 22 | ||
@@ -80,12 +81,15 @@ irqreturn_t mc_gp10b_isr_stall(struct gk20a *g) | |||
80 | 81 | ||
81 | gk20a_writel(g, mc_intr_en_clear_r(0), 0xffffffff); | 82 | gk20a_writel(g, mc_intr_en_clear_r(0), 0xffffffff); |
82 | 83 | ||
84 | atomic_inc(&g->hw_irq_stall_count); | ||
85 | |||
83 | return IRQ_WAKE_THREAD; | 86 | return IRQ_WAKE_THREAD; |
84 | } | 87 | } |
85 | 88 | ||
86 | irqreturn_t mc_gp10b_isr_nonstall(struct gk20a *g) | 89 | irqreturn_t mc_gp10b_isr_nonstall(struct gk20a *g) |
87 | { | 90 | { |
88 | u32 mc_intr_1; | 91 | u32 mc_intr_1; |
92 | u32 hw_irq_count; | ||
89 | 93 | ||
90 | if (!g->power_on) | 94 | if (!g->power_on) |
91 | return IRQ_NONE; | 95 | return IRQ_NONE; |
@@ -97,12 +101,27 @@ irqreturn_t mc_gp10b_isr_nonstall(struct gk20a *g) | |||
97 | 101 | ||
98 | gk20a_writel(g, mc_intr_en_clear_r(1), 0xffffffff); | 102 | gk20a_writel(g, mc_intr_en_clear_r(1), 0xffffffff); |
99 | 103 | ||
100 | return IRQ_WAKE_THREAD; | 104 | if (g->ops.mc.isr_thread_nonstall) |
105 | g->ops.mc.isr_thread_nonstall(g, mc_intr_1); | ||
106 | |||
107 | hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count); | ||
108 | |||
109 | gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), | ||
110 | g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); | ||
111 | |||
112 | /* sync handled irq counter before re-enabling interrupts */ | ||
113 | atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count); | ||
114 | |||
115 | wake_up_all(&g->sw_irq_nonstall_last_handled_wq); | ||
116 | |||
117 | return IRQ_HANDLED; | ||
101 | } | 118 | } |
102 | 119 | ||
103 | irqreturn_t mc_gp10b_intr_thread_stall(struct gk20a *g) | 120 | irqreturn_t mc_gp10b_intr_thread_stall(struct gk20a *g) |
104 | { | 121 | { |
105 | u32 mc_intr_0; | 122 | u32 mc_intr_0; |
123 | int hw_irq_count; | ||
124 | |||
106 | u32 engine_id_idx; | 125 | u32 engine_id_idx; |
107 | u32 active_engine_id = 0; | 126 | u32 active_engine_id = 0; |
108 | u32 engine_enum = ENGINE_INVAL_GK20A; | 127 | u32 engine_enum = ENGINE_INVAL_GK20A; |
@@ -110,6 +129,7 @@ irqreturn_t mc_gp10b_intr_thread_stall(struct gk20a *g) | |||
110 | gk20a_dbg(gpu_dbg_intr, "interrupt thread launched"); | 129 | gk20a_dbg(gpu_dbg_intr, "interrupt thread launched"); |
111 | 130 | ||
112 | mc_intr_0 = gk20a_readl(g, mc_intr_r(0)); | 131 | mc_intr_0 = gk20a_readl(g, mc_intr_r(0)); |
132 | hw_irq_count = atomic_read(&g->hw_irq_stall_count); | ||
113 | 133 | ||
114 | gk20a_dbg(gpu_dbg_intr, "stall intr %08x\n", mc_intr_0); | 134 | gk20a_dbg(gpu_dbg_intr, "stall intr %08x\n", mc_intr_0); |
115 | 135 | ||
@@ -146,51 +166,13 @@ irqreturn_t mc_gp10b_intr_thread_stall(struct gk20a *g) | |||
146 | if (mc_intr_0 & mc_intr_ltc_pending_f()) | 166 | if (mc_intr_0 & mc_intr_ltc_pending_f()) |
147 | g->ops.ltc.isr(g); | 167 | g->ops.ltc.isr(g); |
148 | 168 | ||
169 | /* sync handled irq counter before re-enabling interrupts */ | ||
170 | atomic_set(&g->sw_irq_stall_last_handled, hw_irq_count); | ||
171 | |||
149 | gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING), | 172 | gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING), |
150 | g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_STALLING]); | 173 | g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_STALLING]); |
151 | 174 | ||
152 | return IRQ_HANDLED; | 175 | wake_up_all(&g->sw_irq_stall_last_handled_wq); |
153 | } | ||
154 | |||
155 | irqreturn_t mc_gp10b_intr_thread_nonstall(struct gk20a *g) | ||
156 | { | ||
157 | u32 mc_intr_1; | ||
158 | u32 engine_id_idx; | ||
159 | u32 active_engine_id = 0; | ||
160 | u32 engine_enum = ENGINE_INVAL_GK20A; | ||
161 | |||
162 | gk20a_dbg(gpu_dbg_intr, "interrupt thread launched"); | ||
163 | |||
164 | mc_intr_1 = gk20a_readl(g, mc_intr_r(1)); | ||
165 | |||
166 | gk20a_dbg(gpu_dbg_intr, "non-stall intr %08x\n", mc_intr_1); | ||
167 | |||
168 | if (mc_intr_1 & mc_intr_pfifo_pending_f()) | ||
169 | gk20a_fifo_nonstall_isr(g); | ||
170 | |||
171 | for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines; engine_id_idx++) { | ||
172 | active_engine_id = g->fifo.active_engines_list[engine_id_idx]; | ||
173 | |||
174 | if (mc_intr_1 & g->fifo.engine_info[active_engine_id].intr_mask) { | ||
175 | engine_enum = g->fifo.engine_info[active_engine_id].engine_enum; | ||
176 | /* GR Engine */ | ||
177 | if (engine_enum == ENGINE_GR_GK20A) { | ||
178 | gk20a_gr_nonstall_isr(g); | ||
179 | } | ||
180 | |||
181 | /* CE Engine */ | ||
182 | if (((engine_enum == ENGINE_GRCE_GK20A) || | ||
183 | (engine_enum == ENGINE_ASYNC_CE_GK20A)) && | ||
184 | g->ops.ce2.isr_nonstall) { | ||
185 | g->ops.ce2.isr_nonstall(g, | ||
186 | g->fifo.engine_info[active_engine_id].inst_id, | ||
187 | g->fifo.engine_info[active_engine_id].pri_base); | ||
188 | } | ||
189 | } | ||
190 | } | ||
191 | |||
192 | gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), | ||
193 | g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); | ||
194 | 176 | ||
195 | return IRQ_HANDLED; | 177 | return IRQ_HANDLED; |
196 | } | 178 | } |
@@ -202,5 +184,7 @@ void gp10b_init_mc(struct gpu_ops *gops) | |||
202 | gops->mc.isr_stall = mc_gp10b_isr_stall; | 184 | gops->mc.isr_stall = mc_gp10b_isr_stall; |
203 | gops->mc.isr_nonstall = mc_gp10b_isr_nonstall; | 185 | gops->mc.isr_nonstall = mc_gp10b_isr_nonstall; |
204 | gops->mc.isr_thread_stall = mc_gp10b_intr_thread_stall; | 186 | gops->mc.isr_thread_stall = mc_gp10b_intr_thread_stall; |
205 | gops->mc.isr_thread_nonstall = mc_gp10b_intr_thread_nonstall; | 187 | gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall; |
188 | gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb; | ||
189 | |||
206 | } | 190 | } |
diff --git a/drivers/gpu/nvgpu/pci.c b/drivers/gpu/nvgpu/pci.c index 39559dac..69e16267 100644 --- a/drivers/gpu/nvgpu/pci.c +++ b/drivers/gpu/nvgpu/pci.c | |||
@@ -236,8 +236,7 @@ static irqreturn_t nvgpu_pci_isr(int irq, void *dev_id) | |||
236 | g->ops.xve.rearm_msi(g); | 236 | g->ops.xve.rearm_msi(g); |
237 | #endif | 237 | #endif |
238 | 238 | ||
239 | return (ret_stall == IRQ_NONE && ret_nonstall == IRQ_NONE) ? | 239 | return (ret_stall == IRQ_NONE) ? ret_nonstall : IRQ_WAKE_THREAD; |
240 | IRQ_NONE : IRQ_WAKE_THREAD; | ||
241 | } | 240 | } |
242 | 241 | ||
243 | static irqreturn_t nvgpu_pci_intr_thread(int irq, void *dev_id) | 242 | static irqreturn_t nvgpu_pci_intr_thread(int irq, void *dev_id) |
@@ -245,7 +244,6 @@ static irqreturn_t nvgpu_pci_intr_thread(int irq, void *dev_id) | |||
245 | struct gk20a *g = dev_id; | 244 | struct gk20a *g = dev_id; |
246 | 245 | ||
247 | g->ops.mc.isr_thread_stall(g); | 246 | g->ops.mc.isr_thread_stall(g); |
248 | g->ops.mc.isr_thread_nonstall(g); | ||
249 | 247 | ||
250 | return IRQ_HANDLED; | 248 | return IRQ_HANDLED; |
251 | } | 249 | } |