diff options
author | Arto Merilainen <amerilainen@nvidia.com> | 2014-03-19 03:38:25 -0400 |
---|---|---|
committer | Dan Willemsen <dwillemsen@nvidia.com> | 2015-03-18 15:08:53 -0400 |
commit | a9785995d5f22aaeb659285f8aeb64d8b56982e0 (patch) | |
tree | cc75f75bcf43db316a002a7a240b81f299bf6d7f /drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | |
parent | 61efaf843c22b85424036ec98015121c08f5f16c (diff) |
gpu: nvgpu: Add NVIDIA GPU Driver
This patch moves the NVIDIA GPU driver to a new location.
Bug 1482562
Change-Id: I24293810b9d0f1504fd9be00135e21dad656ccb6
Signed-off-by: Arto Merilainen <amerilainen@nvidia.com>
Reviewed-on: http://git-master/r/383722
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a/fifo_gk20a.c')
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | 1836 |
1 files changed, 1836 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c new file mode 100644 index 00000000..5575b995 --- /dev/null +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | |||
@@ -0,0 +1,1836 @@ | |||
1 | /* | ||
2 | * drivers/video/tegra/host/gk20a/fifo_gk20a.c | ||
3 | * | ||
4 | * GK20A Graphics FIFO (gr host) | ||
5 | * | ||
6 | * Copyright (c) 2011-2014, NVIDIA CORPORATION. All rights reserved. | ||
7 | * | ||
8 | * This program is free software; you can redistribute it and/or modify it | ||
9 | * under the terms and conditions of the GNU General Public License, | ||
10 | * version 2, as published by the Free Software Foundation. | ||
11 | * | ||
12 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
13 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
14 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
15 | * more details. | ||
16 | * | ||
17 | * You should have received a copy of the GNU General Public License along with | ||
18 | * this program; if not, write to the Free Software Foundation, Inc., | ||
19 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | ||
20 | */ | ||
21 | #include <linux/delay.h> | ||
22 | #include <linux/slab.h> | ||
23 | #include <linux/scatterlist.h> | ||
24 | #include <trace/events/gk20a.h> | ||
25 | #include <linux/dma-mapping.h> | ||
26 | #include <linux/nvhost.h> | ||
27 | |||
28 | #include "gk20a.h" | ||
29 | #include "debug_gk20a.h" | ||
30 | #include "hw_fifo_gk20a.h" | ||
31 | #include "hw_pbdma_gk20a.h" | ||
32 | #include "hw_ccsr_gk20a.h" | ||
33 | #include "hw_ram_gk20a.h" | ||
34 | #include "hw_proj_gk20a.h" | ||
35 | #include "hw_top_gk20a.h" | ||
36 | #include "hw_mc_gk20a.h" | ||
37 | #include "hw_gr_gk20a.h" | ||
38 | |||
39 | static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, | ||
40 | u32 hw_chid, bool add, | ||
41 | bool wait_for_finish); | ||
42 | static void gk20a_fifo_handle_mmu_fault_thread(struct work_struct *work); | ||
43 | |||
44 | /* | ||
45 | * Link engine IDs to MMU IDs and vice versa. | ||
46 | */ | ||
47 | |||
48 | static inline u32 gk20a_engine_id_to_mmu_id(u32 engine_id) | ||
49 | { | ||
50 | switch (engine_id) { | ||
51 | case ENGINE_GR_GK20A: | ||
52 | return 0x00; | ||
53 | case ENGINE_CE2_GK20A: | ||
54 | return 0x1b; | ||
55 | default: | ||
56 | return ~0; | ||
57 | } | ||
58 | } | ||
59 | |||
60 | static inline u32 gk20a_mmu_id_to_engine_id(u32 engine_id) | ||
61 | { | ||
62 | switch (engine_id) { | ||
63 | case 0x00: | ||
64 | return ENGINE_GR_GK20A; | ||
65 | case 0x1b: | ||
66 | return ENGINE_CE2_GK20A; | ||
67 | default: | ||
68 | return ~0; | ||
69 | } | ||
70 | } | ||
71 | |||
72 | |||
73 | static int init_engine_info(struct fifo_gk20a *f) | ||
74 | { | ||
75 | struct gk20a *g = f->g; | ||
76 | struct device *d = dev_from_gk20a(g); | ||
77 | struct fifo_engine_info_gk20a *gr_info; | ||
78 | const u32 gr_sw_id = ENGINE_GR_GK20A; | ||
79 | u32 i; | ||
80 | u32 max_info_entries = top_device_info__size_1_v(); | ||
81 | |||
82 | gk20a_dbg_fn(""); | ||
83 | |||
84 | /* all we really care about finding is the graphics entry */ | ||
85 | /* especially early on in sim it probably thinks it has more */ | ||
86 | f->num_engines = 1; | ||
87 | |||
88 | gr_info = f->engine_info + gr_sw_id; | ||
89 | |||
90 | gr_info->sw_id = gr_sw_id; | ||
91 | gr_info->name = "gr"; | ||
92 | gr_info->dev_info_id = top_device_info_type_enum_graphics_v(); | ||
93 | gr_info->mmu_fault_id = fifo_intr_mmu_fault_eng_id_graphics_v(); | ||
94 | gr_info->runlist_id = ~0; | ||
95 | gr_info->pbdma_id = ~0; | ||
96 | gr_info->engine_id = ~0; | ||
97 | |||
98 | for (i = 0; i < max_info_entries; i++) { | ||
99 | u32 table_entry = gk20a_readl(f->g, top_device_info_r(i)); | ||
100 | u32 entry = top_device_info_entry_v(table_entry); | ||
101 | u32 engine_enum = top_device_info_type_enum_v(table_entry); | ||
102 | u32 table_entry2 = 0; | ||
103 | |||
104 | if (entry == top_device_info_entry_not_valid_v()) | ||
105 | continue; | ||
106 | |||
107 | if (top_device_info_chain_v(table_entry) == | ||
108 | top_device_info_chain_enable_v()) { | ||
109 | |||
110 | table_entry2 = gk20a_readl(f->g, | ||
111 | top_device_info_r(++i)); | ||
112 | |||
113 | engine_enum = top_device_info_type_enum_v(table_entry2); | ||
114 | } | ||
115 | |||
116 | /* we only care about GR engine here */ | ||
117 | if (entry == top_device_info_entry_enum_v() && | ||
118 | engine_enum == gr_info->dev_info_id) { | ||
119 | int pbdma_id; | ||
120 | u32 runlist_bit; | ||
121 | |||
122 | gr_info->runlist_id = | ||
123 | top_device_info_runlist_enum_v(table_entry); | ||
124 | gk20a_dbg_info("gr info: runlist_id %d", gr_info->runlist_id); | ||
125 | |||
126 | gr_info->engine_id = | ||
127 | top_device_info_engine_enum_v(table_entry); | ||
128 | gk20a_dbg_info("gr info: engine_id %d", gr_info->engine_id); | ||
129 | |||
130 | runlist_bit = 1 << gr_info->runlist_id; | ||
131 | |||
132 | for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) { | ||
133 | gk20a_dbg_info("gr info: pbdma_map[%d]=%d", | ||
134 | pbdma_id, f->pbdma_map[pbdma_id]); | ||
135 | if (f->pbdma_map[pbdma_id] & runlist_bit) | ||
136 | break; | ||
137 | } | ||
138 | |||
139 | if (pbdma_id == f->num_pbdma) { | ||
140 | gk20a_err(d, "busted pbmda map"); | ||
141 | return -EINVAL; | ||
142 | } | ||
143 | gr_info->pbdma_id = pbdma_id; | ||
144 | |||
145 | break; | ||
146 | } | ||
147 | } | ||
148 | |||
149 | if (gr_info->runlist_id == ~0) { | ||
150 | gk20a_err(d, "busted device info"); | ||
151 | return -EINVAL; | ||
152 | } | ||
153 | |||
154 | return 0; | ||
155 | } | ||
156 | |||
157 | void gk20a_remove_fifo_support(struct fifo_gk20a *f) | ||
158 | { | ||
159 | struct gk20a *g = f->g; | ||
160 | struct device *d = dev_from_gk20a(g); | ||
161 | struct fifo_engine_info_gk20a *engine_info; | ||
162 | struct fifo_runlist_info_gk20a *runlist; | ||
163 | u32 runlist_id; | ||
164 | u32 i; | ||
165 | |||
166 | gk20a_dbg_fn(""); | ||
167 | |||
168 | if (f->channel) { | ||
169 | int c; | ||
170 | for (c = 0; c < f->num_channels; c++) { | ||
171 | if (f->channel[c].remove_support) | ||
172 | f->channel[c].remove_support(f->channel+c); | ||
173 | } | ||
174 | kfree(f->channel); | ||
175 | } | ||
176 | if (f->userd.gpu_va) | ||
177 | gk20a_gmmu_unmap(&g->mm.bar1.vm, | ||
178 | f->userd.gpu_va, | ||
179 | f->userd.size, | ||
180 | gk20a_mem_flag_none); | ||
181 | |||
182 | if (f->userd.sgt) | ||
183 | gk20a_free_sgtable(&f->userd.sgt); | ||
184 | |||
185 | if (f->userd.cpuva) | ||
186 | dma_free_coherent(d, | ||
187 | f->userd_total_size, | ||
188 | f->userd.cpuva, | ||
189 | f->userd.iova); | ||
190 | f->userd.cpuva = NULL; | ||
191 | f->userd.iova = 0; | ||
192 | |||
193 | engine_info = f->engine_info + ENGINE_GR_GK20A; | ||
194 | runlist_id = engine_info->runlist_id; | ||
195 | runlist = &f->runlist_info[runlist_id]; | ||
196 | |||
197 | for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) { | ||
198 | if (runlist->mem[i].cpuva) | ||
199 | dma_free_coherent(d, | ||
200 | runlist->mem[i].size, | ||
201 | runlist->mem[i].cpuva, | ||
202 | runlist->mem[i].iova); | ||
203 | runlist->mem[i].cpuva = NULL; | ||
204 | runlist->mem[i].iova = 0; | ||
205 | } | ||
206 | |||
207 | kfree(runlist->active_channels); | ||
208 | |||
209 | kfree(f->runlist_info); | ||
210 | kfree(f->pbdma_map); | ||
211 | kfree(f->engine_info); | ||
212 | } | ||
213 | |||
214 | /* reads info from hardware and fills in pbmda exception info record */ | ||
215 | static inline void get_exception_pbdma_info( | ||
216 | struct gk20a *g, | ||
217 | struct fifo_engine_info_gk20a *eng_info) | ||
218 | { | ||
219 | struct fifo_pbdma_exception_info_gk20a *e = | ||
220 | &eng_info->pbdma_exception_info; | ||
221 | |||
222 | u32 pbdma_status_r = e->status_r = gk20a_readl(g, | ||
223 | fifo_pbdma_status_r(eng_info->pbdma_id)); | ||
224 | e->id = fifo_pbdma_status_id_v(pbdma_status_r); /* vs. id_hw_v()? */ | ||
225 | e->id_is_chid = fifo_pbdma_status_id_type_v(pbdma_status_r) == | ||
226 | fifo_pbdma_status_id_type_chid_v(); | ||
227 | e->chan_status_v = fifo_pbdma_status_chan_status_v(pbdma_status_r); | ||
228 | e->next_id_is_chid = | ||
229 | fifo_pbdma_status_next_id_type_v(pbdma_status_r) == | ||
230 | fifo_pbdma_status_next_id_type_chid_v(); | ||
231 | e->next_id = fifo_pbdma_status_next_id_v(pbdma_status_r); | ||
232 | e->chsw_in_progress = | ||
233 | fifo_pbdma_status_chsw_v(pbdma_status_r) == | ||
234 | fifo_pbdma_status_chsw_in_progress_v(); | ||
235 | } | ||
236 | |||
237 | static void fifo_pbdma_exception_status(struct gk20a *g, | ||
238 | struct fifo_engine_info_gk20a *eng_info) | ||
239 | { | ||
240 | struct fifo_pbdma_exception_info_gk20a *e; | ||
241 | get_exception_pbdma_info(g, eng_info); | ||
242 | e = &eng_info->pbdma_exception_info; | ||
243 | |||
244 | gk20a_dbg_fn("pbdma_id %d, " | ||
245 | "id_type %s, id %d, chan_status %d, " | ||
246 | "next_id_type %s, next_id %d, " | ||
247 | "chsw_in_progress %d", | ||
248 | eng_info->pbdma_id, | ||
249 | e->id_is_chid ? "chid" : "tsgid", e->id, e->chan_status_v, | ||
250 | e->next_id_is_chid ? "chid" : "tsgid", e->next_id, | ||
251 | e->chsw_in_progress); | ||
252 | } | ||
253 | |||
254 | /* reads info from hardware and fills in pbmda exception info record */ | ||
255 | static inline void get_exception_engine_info( | ||
256 | struct gk20a *g, | ||
257 | struct fifo_engine_info_gk20a *eng_info) | ||
258 | { | ||
259 | struct fifo_engine_exception_info_gk20a *e = | ||
260 | &eng_info->engine_exception_info; | ||
261 | u32 engine_status_r = e->status_r = | ||
262 | gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id)); | ||
263 | e->id = fifo_engine_status_id_v(engine_status_r); /* vs. id_hw_v()? */ | ||
264 | e->id_is_chid = fifo_engine_status_id_type_v(engine_status_r) == | ||
265 | fifo_engine_status_id_type_chid_v(); | ||
266 | e->ctx_status_v = fifo_engine_status_ctx_status_v(engine_status_r); | ||
267 | e->faulted = | ||
268 | fifo_engine_status_faulted_v(engine_status_r) == | ||
269 | fifo_engine_status_faulted_true_v(); | ||
270 | e->idle = | ||
271 | fifo_engine_status_engine_v(engine_status_r) == | ||
272 | fifo_engine_status_engine_idle_v(); | ||
273 | e->ctxsw_in_progress = | ||
274 | fifo_engine_status_ctxsw_v(engine_status_r) == | ||
275 | fifo_engine_status_ctxsw_in_progress_v(); | ||
276 | } | ||
277 | |||
278 | static void fifo_engine_exception_status(struct gk20a *g, | ||
279 | struct fifo_engine_info_gk20a *eng_info) | ||
280 | { | ||
281 | struct fifo_engine_exception_info_gk20a *e; | ||
282 | get_exception_engine_info(g, eng_info); | ||
283 | e = &eng_info->engine_exception_info; | ||
284 | |||
285 | gk20a_dbg_fn("engine_id %d, id_type %s, id %d, ctx_status %d, " | ||
286 | "faulted %d, idle %d, ctxsw_in_progress %d, ", | ||
287 | eng_info->engine_id, e->id_is_chid ? "chid" : "tsgid", | ||
288 | e->id, e->ctx_status_v, | ||
289 | e->faulted, e->idle, e->ctxsw_in_progress); | ||
290 | } | ||
291 | |||
292 | static int init_runlist(struct gk20a *g, struct fifo_gk20a *f) | ||
293 | { | ||
294 | struct fifo_engine_info_gk20a *engine_info; | ||
295 | struct fifo_runlist_info_gk20a *runlist; | ||
296 | struct device *d = dev_from_gk20a(g); | ||
297 | u32 runlist_id; | ||
298 | u32 i; | ||
299 | u64 runlist_size; | ||
300 | |||
301 | gk20a_dbg_fn(""); | ||
302 | |||
303 | f->max_runlists = fifo_eng_runlist_base__size_1_v(); | ||
304 | f->runlist_info = kzalloc(sizeof(struct fifo_runlist_info_gk20a) * | ||
305 | f->max_runlists, GFP_KERNEL); | ||
306 | if (!f->runlist_info) | ||
307 | goto clean_up; | ||
308 | |||
309 | engine_info = f->engine_info + ENGINE_GR_GK20A; | ||
310 | runlist_id = engine_info->runlist_id; | ||
311 | runlist = &f->runlist_info[runlist_id]; | ||
312 | |||
313 | runlist->active_channels = | ||
314 | kzalloc(DIV_ROUND_UP(f->num_channels, BITS_PER_BYTE), | ||
315 | GFP_KERNEL); | ||
316 | if (!runlist->active_channels) | ||
317 | goto clean_up_runlist_info; | ||
318 | |||
319 | runlist_size = ram_rl_entry_size_v() * f->num_channels; | ||
320 | for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) { | ||
321 | dma_addr_t iova; | ||
322 | |||
323 | runlist->mem[i].cpuva = | ||
324 | dma_alloc_coherent(d, | ||
325 | runlist_size, | ||
326 | &iova, | ||
327 | GFP_KERNEL); | ||
328 | if (!runlist->mem[i].cpuva) { | ||
329 | dev_err(d, "memory allocation failed\n"); | ||
330 | goto clean_up_runlist; | ||
331 | } | ||
332 | runlist->mem[i].iova = iova; | ||
333 | runlist->mem[i].size = runlist_size; | ||
334 | } | ||
335 | mutex_init(&runlist->mutex); | ||
336 | init_waitqueue_head(&runlist->runlist_wq); | ||
337 | |||
338 | /* None of buffers is pinned if this value doesn't change. | ||
339 | Otherwise, one of them (cur_buffer) must have been pinned. */ | ||
340 | runlist->cur_buffer = MAX_RUNLIST_BUFFERS; | ||
341 | |||
342 | gk20a_dbg_fn("done"); | ||
343 | return 0; | ||
344 | |||
345 | clean_up_runlist: | ||
346 | for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) { | ||
347 | if (runlist->mem[i].cpuva) | ||
348 | dma_free_coherent(d, | ||
349 | runlist->mem[i].size, | ||
350 | runlist->mem[i].cpuva, | ||
351 | runlist->mem[i].iova); | ||
352 | runlist->mem[i].cpuva = NULL; | ||
353 | runlist->mem[i].iova = 0; | ||
354 | } | ||
355 | |||
356 | kfree(runlist->active_channels); | ||
357 | runlist->active_channels = NULL; | ||
358 | |||
359 | clean_up_runlist_info: | ||
360 | kfree(f->runlist_info); | ||
361 | f->runlist_info = NULL; | ||
362 | |||
363 | clean_up: | ||
364 | gk20a_dbg_fn("fail"); | ||
365 | return -ENOMEM; | ||
366 | } | ||
367 | |||
368 | #define GRFIFO_TIMEOUT_CHECK_PERIOD_US 100000 | ||
369 | |||
370 | int gk20a_init_fifo_reset_enable_hw(struct gk20a *g) | ||
371 | { | ||
372 | u32 intr_stall; | ||
373 | u32 mask; | ||
374 | u32 timeout; | ||
375 | int i; | ||
376 | |||
377 | gk20a_dbg_fn(""); | ||
378 | /* enable pmc pfifo */ | ||
379 | gk20a_reset(g, mc_enable_pfifo_enabled_f() | ||
380 | | mc_enable_ce2_enabled_f()); | ||
381 | |||
382 | /* enable pbdma */ | ||
383 | mask = 0; | ||
384 | for (i = 0; i < proj_host_num_pbdma_v(); ++i) | ||
385 | mask |= mc_enable_pb_sel_f(mc_enable_pb_0_enabled_v(), i); | ||
386 | gk20a_writel(g, mc_enable_pb_r(), mask); | ||
387 | |||
388 | /* enable pfifo interrupt */ | ||
389 | gk20a_writel(g, fifo_intr_0_r(), 0xFFFFFFFF); | ||
390 | gk20a_writel(g, fifo_intr_en_0_r(), 0x7FFFFFFF); | ||
391 | gk20a_writel(g, fifo_intr_en_1_r(), 0x80000000); | ||
392 | |||
393 | /* enable pbdma interrupt */ | ||
394 | mask = 0; | ||
395 | for (i = 0; i < proj_host_num_pbdma_v(); i++) { | ||
396 | intr_stall = gk20a_readl(g, pbdma_intr_stall_r(i)); | ||
397 | intr_stall &= ~pbdma_intr_stall_lbreq_enabled_f(); | ||
398 | gk20a_writel(g, pbdma_intr_stall_r(i), intr_stall); | ||
399 | gk20a_writel(g, pbdma_intr_0_r(i), 0xFFFFFFFF); | ||
400 | gk20a_writel(g, pbdma_intr_en_0_r(i), | ||
401 | (~0) & ~pbdma_intr_en_0_lbreq_enabled_f()); | ||
402 | gk20a_writel(g, pbdma_intr_1_r(i), 0xFFFFFFFF); | ||
403 | gk20a_writel(g, pbdma_intr_en_1_r(i), 0xFFFFFFFF); | ||
404 | } | ||
405 | |||
406 | /* TBD: apply overrides */ | ||
407 | |||
408 | /* TBD: BLCG prod */ | ||
409 | |||
410 | /* reset runlist interrupts */ | ||
411 | gk20a_writel(g, fifo_intr_runlist_r(), ~0); | ||
412 | |||
413 | /* TBD: do we need those? */ | ||
414 | timeout = gk20a_readl(g, fifo_fb_timeout_r()); | ||
415 | timeout = set_field(timeout, fifo_fb_timeout_period_m(), | ||
416 | fifo_fb_timeout_period_max_f()); | ||
417 | gk20a_writel(g, fifo_fb_timeout_r(), timeout); | ||
418 | |||
419 | if (tegra_platform_is_silicon()) { | ||
420 | timeout = gk20a_readl(g, fifo_pb_timeout_r()); | ||
421 | timeout &= ~fifo_pb_timeout_detection_enabled_f(); | ||
422 | gk20a_writel(g, fifo_pb_timeout_r(), timeout); | ||
423 | } | ||
424 | |||
425 | timeout = GRFIFO_TIMEOUT_CHECK_PERIOD_US | | ||
426 | fifo_eng_timeout_detection_enabled_f(); | ||
427 | gk20a_writel(g, fifo_eng_timeout_r(), timeout); | ||
428 | |||
429 | gk20a_dbg_fn("done"); | ||
430 | |||
431 | return 0; | ||
432 | } | ||
433 | |||
434 | static void gk20a_init_fifo_pbdma_intr_descs(struct fifo_gk20a *f) | ||
435 | { | ||
436 | /* These are all errors which indicate something really wrong | ||
437 | * going on in the device. */ | ||
438 | f->intr.pbdma.device_fatal_0 = | ||
439 | pbdma_intr_0_memreq_pending_f() | | ||
440 | pbdma_intr_0_memack_timeout_pending_f() | | ||
441 | pbdma_intr_0_memack_extra_pending_f() | | ||
442 | pbdma_intr_0_memdat_timeout_pending_f() | | ||
443 | pbdma_intr_0_memdat_extra_pending_f() | | ||
444 | pbdma_intr_0_memflush_pending_f() | | ||
445 | pbdma_intr_0_memop_pending_f() | | ||
446 | pbdma_intr_0_lbconnect_pending_f() | | ||
447 | pbdma_intr_0_lbreq_pending_f() | | ||
448 | pbdma_intr_0_lback_timeout_pending_f() | | ||
449 | pbdma_intr_0_lback_extra_pending_f() | | ||
450 | pbdma_intr_0_lbdat_timeout_pending_f() | | ||
451 | pbdma_intr_0_lbdat_extra_pending_f() | | ||
452 | pbdma_intr_0_xbarconnect_pending_f() | | ||
453 | pbdma_intr_0_pri_pending_f(); | ||
454 | |||
455 | /* These are data parsing, framing errors or others which can be | ||
456 | * recovered from with intervention... or just resetting the | ||
457 | * channel. */ | ||
458 | f->intr.pbdma.channel_fatal_0 = | ||
459 | pbdma_intr_0_gpfifo_pending_f() | | ||
460 | pbdma_intr_0_gpptr_pending_f() | | ||
461 | pbdma_intr_0_gpentry_pending_f() | | ||
462 | pbdma_intr_0_gpcrc_pending_f() | | ||
463 | pbdma_intr_0_pbptr_pending_f() | | ||
464 | pbdma_intr_0_pbentry_pending_f() | | ||
465 | pbdma_intr_0_pbcrc_pending_f() | | ||
466 | pbdma_intr_0_method_pending_f() | | ||
467 | pbdma_intr_0_methodcrc_pending_f() | | ||
468 | pbdma_intr_0_pbseg_pending_f() | | ||
469 | pbdma_intr_0_signature_pending_f(); | ||
470 | |||
471 | /* Can be used for sw-methods, or represents | ||
472 | * a recoverable timeout. */ | ||
473 | f->intr.pbdma.restartable_0 = | ||
474 | pbdma_intr_0_device_pending_f() | | ||
475 | pbdma_intr_0_acquire_pending_f(); | ||
476 | } | ||
477 | |||
478 | static int gk20a_init_fifo_setup_sw(struct gk20a *g) | ||
479 | { | ||
480 | struct fifo_gk20a *f = &g->fifo; | ||
481 | struct device *d = dev_from_gk20a(g); | ||
482 | int chid, i, err = 0; | ||
483 | dma_addr_t iova; | ||
484 | |||
485 | gk20a_dbg_fn(""); | ||
486 | |||
487 | if (f->sw_ready) { | ||
488 | gk20a_dbg_fn("skip init"); | ||
489 | return 0; | ||
490 | } | ||
491 | |||
492 | f->g = g; | ||
493 | |||
494 | INIT_WORK(&f->fault_restore_thread, | ||
495 | gk20a_fifo_handle_mmu_fault_thread); | ||
496 | mutex_init(&f->intr.isr.mutex); | ||
497 | gk20a_init_fifo_pbdma_intr_descs(f); /* just filling in data/tables */ | ||
498 | |||
499 | f->num_channels = ccsr_channel__size_1_v(); | ||
500 | f->num_pbdma = proj_host_num_pbdma_v(); | ||
501 | f->max_engines = ENGINE_INVAL_GK20A; | ||
502 | |||
503 | f->userd_entry_size = 1 << ram_userd_base_shift_v(); | ||
504 | f->userd_total_size = f->userd_entry_size * f->num_channels; | ||
505 | |||
506 | f->userd.cpuva = dma_alloc_coherent(d, | ||
507 | f->userd_total_size, | ||
508 | &iova, | ||
509 | GFP_KERNEL); | ||
510 | if (!f->userd.cpuva) { | ||
511 | dev_err(d, "memory allocation failed\n"); | ||
512 | goto clean_up; | ||
513 | } | ||
514 | |||
515 | f->userd.iova = iova; | ||
516 | err = gk20a_get_sgtable(d, &f->userd.sgt, | ||
517 | f->userd.cpuva, f->userd.iova, | ||
518 | f->userd_total_size); | ||
519 | if (err) { | ||
520 | dev_err(d, "failed to create sg table\n"); | ||
521 | goto clean_up; | ||
522 | } | ||
523 | |||
524 | /* bar1 va */ | ||
525 | f->userd.gpu_va = gk20a_gmmu_map(&g->mm.bar1.vm, | ||
526 | &f->userd.sgt, | ||
527 | f->userd_total_size, | ||
528 | 0, /* flags */ | ||
529 | gk20a_mem_flag_none); | ||
530 | if (!f->userd.gpu_va) { | ||
531 | dev_err(d, "gmmu mapping failed\n"); | ||
532 | goto clean_up; | ||
533 | } | ||
534 | |||
535 | gk20a_dbg(gpu_dbg_map, "userd bar1 va = 0x%llx", f->userd.gpu_va); | ||
536 | |||
537 | f->userd.size = f->userd_total_size; | ||
538 | |||
539 | f->channel = kzalloc(f->num_channels * sizeof(*f->channel), | ||
540 | GFP_KERNEL); | ||
541 | f->pbdma_map = kzalloc(f->num_pbdma * sizeof(*f->pbdma_map), | ||
542 | GFP_KERNEL); | ||
543 | f->engine_info = kzalloc(f->max_engines * sizeof(*f->engine_info), | ||
544 | GFP_KERNEL); | ||
545 | |||
546 | if (!(f->channel && f->pbdma_map && f->engine_info)) { | ||
547 | err = -ENOMEM; | ||
548 | goto clean_up; | ||
549 | } | ||
550 | |||
551 | /* pbdma map needs to be in place before calling engine info init */ | ||
552 | for (i = 0; i < f->num_pbdma; ++i) | ||
553 | f->pbdma_map[i] = gk20a_readl(g, fifo_pbdma_map_r(i)); | ||
554 | |||
555 | init_engine_info(f); | ||
556 | |||
557 | init_runlist(g, f); | ||
558 | |||
559 | for (chid = 0; chid < f->num_channels; chid++) { | ||
560 | f->channel[chid].userd_cpu_va = | ||
561 | f->userd.cpuva + chid * f->userd_entry_size; | ||
562 | f->channel[chid].userd_iova = | ||
563 | NV_MC_SMMU_VADDR_TRANSLATE(f->userd.iova) | ||
564 | + chid * f->userd_entry_size; | ||
565 | f->channel[chid].userd_gpu_va = | ||
566 | f->userd.gpu_va + chid * f->userd_entry_size; | ||
567 | |||
568 | gk20a_init_channel_support(g, chid); | ||
569 | } | ||
570 | mutex_init(&f->ch_inuse_mutex); | ||
571 | |||
572 | f->remove_support = gk20a_remove_fifo_support; | ||
573 | |||
574 | f->deferred_reset_pending = false; | ||
575 | mutex_init(&f->deferred_reset_mutex); | ||
576 | |||
577 | f->sw_ready = true; | ||
578 | |||
579 | gk20a_dbg_fn("done"); | ||
580 | return 0; | ||
581 | |||
582 | clean_up: | ||
583 | gk20a_dbg_fn("fail"); | ||
584 | if (f->userd.gpu_va) | ||
585 | gk20a_gmmu_unmap(&g->mm.bar1.vm, | ||
586 | f->userd.gpu_va, | ||
587 | f->userd.size, | ||
588 | gk20a_mem_flag_none); | ||
589 | if (f->userd.sgt) | ||
590 | gk20a_free_sgtable(&f->userd.sgt); | ||
591 | if (f->userd.cpuva) | ||
592 | dma_free_coherent(d, | ||
593 | f->userd_total_size, | ||
594 | f->userd.cpuva, | ||
595 | f->userd.iova); | ||
596 | f->userd.cpuva = NULL; | ||
597 | f->userd.iova = 0; | ||
598 | |||
599 | memset(&f->userd, 0, sizeof(struct userd_desc)); | ||
600 | |||
601 | kfree(f->channel); | ||
602 | f->channel = NULL; | ||
603 | kfree(f->pbdma_map); | ||
604 | f->pbdma_map = NULL; | ||
605 | kfree(f->engine_info); | ||
606 | f->engine_info = NULL; | ||
607 | |||
608 | return err; | ||
609 | } | ||
610 | |||
611 | static void gk20a_fifo_handle_runlist_event(struct gk20a *g) | ||
612 | { | ||
613 | struct fifo_gk20a *f = &g->fifo; | ||
614 | struct fifo_runlist_info_gk20a *runlist; | ||
615 | unsigned long runlist_event; | ||
616 | u32 runlist_id; | ||
617 | |||
618 | runlist_event = gk20a_readl(g, fifo_intr_runlist_r()); | ||
619 | gk20a_writel(g, fifo_intr_runlist_r(), runlist_event); | ||
620 | |||
621 | for_each_set_bit(runlist_id, &runlist_event, f->max_runlists) { | ||
622 | runlist = &f->runlist_info[runlist_id]; | ||
623 | wake_up(&runlist->runlist_wq); | ||
624 | } | ||
625 | |||
626 | } | ||
627 | |||
628 | static int gk20a_init_fifo_setup_hw(struct gk20a *g) | ||
629 | { | ||
630 | struct fifo_gk20a *f = &g->fifo; | ||
631 | |||
632 | gk20a_dbg_fn(""); | ||
633 | |||
634 | /* test write, read through bar1 @ userd region before | ||
635 | * turning on the snooping */ | ||
636 | { | ||
637 | struct fifo_gk20a *f = &g->fifo; | ||
638 | u32 v, v1 = 0x33, v2 = 0x55; | ||
639 | |||
640 | u32 bar1_vaddr = f->userd.gpu_va; | ||
641 | volatile u32 *cpu_vaddr = f->userd.cpuva; | ||
642 | |||
643 | gk20a_dbg_info("test bar1 @ vaddr 0x%x", | ||
644 | bar1_vaddr); | ||
645 | |||
646 | v = gk20a_bar1_readl(g, bar1_vaddr); | ||
647 | |||
648 | *cpu_vaddr = v1; | ||
649 | smp_mb(); | ||
650 | |||
651 | if (v1 != gk20a_bar1_readl(g, bar1_vaddr)) { | ||
652 | gk20a_err(dev_from_gk20a(g), "bar1 broken @ gk20a!"); | ||
653 | return -EINVAL; | ||
654 | } | ||
655 | |||
656 | gk20a_bar1_writel(g, bar1_vaddr, v2); | ||
657 | |||
658 | if (v2 != gk20a_bar1_readl(g, bar1_vaddr)) { | ||
659 | gk20a_err(dev_from_gk20a(g), "bar1 broken @ gk20a!"); | ||
660 | return -EINVAL; | ||
661 | } | ||
662 | |||
663 | /* is it visible to the cpu? */ | ||
664 | if (*cpu_vaddr != v2) { | ||
665 | gk20a_err(dev_from_gk20a(g), | ||
666 | "cpu didn't see bar1 write @ %p!", | ||
667 | cpu_vaddr); | ||
668 | } | ||
669 | |||
670 | /* put it back */ | ||
671 | gk20a_bar1_writel(g, bar1_vaddr, v); | ||
672 | } | ||
673 | |||
674 | /*XXX all manner of flushes and caching worries, etc */ | ||
675 | |||
676 | /* set the base for the userd region now */ | ||
677 | gk20a_writel(g, fifo_bar1_base_r(), | ||
678 | fifo_bar1_base_ptr_f(f->userd.gpu_va >> 12) | | ||
679 | fifo_bar1_base_valid_true_f()); | ||
680 | |||
681 | gk20a_dbg_fn("done"); | ||
682 | |||
683 | return 0; | ||
684 | } | ||
685 | |||
686 | int gk20a_init_fifo_support(struct gk20a *g) | ||
687 | { | ||
688 | u32 err; | ||
689 | |||
690 | err = gk20a_init_fifo_setup_sw(g); | ||
691 | if (err) | ||
692 | return err; | ||
693 | |||
694 | err = gk20a_init_fifo_setup_hw(g); | ||
695 | if (err) | ||
696 | return err; | ||
697 | |||
698 | return err; | ||
699 | } | ||
700 | |||
701 | static struct channel_gk20a * | ||
702 | channel_from_inst_ptr(struct fifo_gk20a *f, u64 inst_ptr) | ||
703 | { | ||
704 | int ci; | ||
705 | if (unlikely(!f->channel)) | ||
706 | return NULL; | ||
707 | for (ci = 0; ci < f->num_channels; ci++) { | ||
708 | struct channel_gk20a *c = f->channel+ci; | ||
709 | if (c->inst_block.cpuva && | ||
710 | (inst_ptr == c->inst_block.cpu_pa)) | ||
711 | return f->channel+ci; | ||
712 | } | ||
713 | return NULL; | ||
714 | } | ||
715 | |||
716 | /* fault info/descriptions. | ||
717 | * tbd: move to setup | ||
718 | * */ | ||
719 | static const char * const fault_type_descs[] = { | ||
720 | "pde", /*fifo_intr_mmu_fault_info_type_pde_v() == 0 */ | ||
721 | "pde size", | ||
722 | "pte", | ||
723 | "va limit viol", | ||
724 | "unbound inst", | ||
725 | "priv viol", | ||
726 | "ro viol", | ||
727 | "wo viol", | ||
728 | "pitch mask", | ||
729 | "work creation", | ||
730 | "bad aperture", | ||
731 | "compression failure", | ||
732 | "bad kind", | ||
733 | "region viol", | ||
734 | "dual ptes", | ||
735 | "poisoned", | ||
736 | }; | ||
737 | /* engine descriptions */ | ||
738 | static const char * const engine_subid_descs[] = { | ||
739 | "gpc", | ||
740 | "hub", | ||
741 | }; | ||
742 | |||
743 | static const char * const hub_client_descs[] = { | ||
744 | "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu", | ||
745 | "host cpu nb", "iso", "mmu", "mspdec", "msppp", "msvld", | ||
746 | "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc", | ||
747 | "scc nb", "sec", "ssync", "gr copy", "ce2", "xv", "mmu nb", | ||
748 | "msenc", "d falcon", "sked", "a falcon", "n/a", | ||
749 | }; | ||
750 | |||
751 | static const char * const gpc_client_descs[] = { | ||
752 | "l1 0", "t1 0", "pe 0", | ||
753 | "l1 1", "t1 1", "pe 1", | ||
754 | "l1 2", "t1 2", "pe 2", | ||
755 | "l1 3", "t1 3", "pe 3", | ||
756 | "rast", "gcc", "gpccs", | ||
757 | "prop 0", "prop 1", "prop 2", "prop 3", | ||
758 | "l1 4", "t1 4", "pe 4", | ||
759 | "l1 5", "t1 5", "pe 5", | ||
760 | "l1 6", "t1 6", "pe 6", | ||
761 | "l1 7", "t1 7", "pe 7", | ||
762 | "gpm", | ||
763 | "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3", | ||
764 | "rgg utlb", | ||
765 | }; | ||
766 | |||
767 | /* reads info from hardware and fills in mmu fault info record */ | ||
768 | static inline void get_exception_mmu_fault_info( | ||
769 | struct gk20a *g, u32 engine_id, | ||
770 | struct fifo_mmu_fault_info_gk20a *f) | ||
771 | { | ||
772 | u32 fault_info_v; | ||
773 | |||
774 | gk20a_dbg_fn("engine_id %d", engine_id); | ||
775 | |||
776 | memset(f, 0, sizeof(*f)); | ||
777 | |||
778 | f->fault_info_v = fault_info_v = gk20a_readl(g, | ||
779 | fifo_intr_mmu_fault_info_r(engine_id)); | ||
780 | f->fault_type_v = | ||
781 | fifo_intr_mmu_fault_info_type_v(fault_info_v); | ||
782 | f->engine_subid_v = | ||
783 | fifo_intr_mmu_fault_info_engine_subid_v(fault_info_v); | ||
784 | f->client_v = fifo_intr_mmu_fault_info_client_v(fault_info_v); | ||
785 | |||
786 | BUG_ON(f->fault_type_v >= ARRAY_SIZE(fault_type_descs)); | ||
787 | f->fault_type_desc = fault_type_descs[f->fault_type_v]; | ||
788 | |||
789 | BUG_ON(f->engine_subid_v >= ARRAY_SIZE(engine_subid_descs)); | ||
790 | f->engine_subid_desc = engine_subid_descs[f->engine_subid_v]; | ||
791 | |||
792 | if (f->engine_subid_v == | ||
793 | fifo_intr_mmu_fault_info_engine_subid_hub_v()) { | ||
794 | |||
795 | BUG_ON(f->client_v >= ARRAY_SIZE(hub_client_descs)); | ||
796 | f->client_desc = hub_client_descs[f->client_v]; | ||
797 | } else if (f->engine_subid_v == | ||
798 | fifo_intr_mmu_fault_info_engine_subid_gpc_v()) { | ||
799 | BUG_ON(f->client_v >= ARRAY_SIZE(gpc_client_descs)); | ||
800 | f->client_desc = gpc_client_descs[f->client_v]; | ||
801 | } else { | ||
802 | BUG_ON(1); | ||
803 | } | ||
804 | |||
805 | f->fault_hi_v = gk20a_readl(g, fifo_intr_mmu_fault_hi_r(engine_id)); | ||
806 | f->fault_lo_v = gk20a_readl(g, fifo_intr_mmu_fault_lo_r(engine_id)); | ||
807 | /* note:ignoring aperture on gk20a... */ | ||
808 | f->inst_ptr = fifo_intr_mmu_fault_inst_ptr_v( | ||
809 | gk20a_readl(g, fifo_intr_mmu_fault_inst_r(engine_id))); | ||
810 | /* note: inst_ptr is a 40b phys addr. */ | ||
811 | f->inst_ptr <<= fifo_intr_mmu_fault_inst_ptr_align_shift_v(); | ||
812 | } | ||
813 | |||
814 | static void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id) | ||
815 | { | ||
816 | gk20a_dbg_fn(""); | ||
817 | |||
818 | if (engine_id == top_device_info_type_enum_graphics_v()) { | ||
819 | /* resetting engine using mc_enable_r() is not enough, | ||
820 | * we do full init sequence */ | ||
821 | gk20a_gr_reset(g); | ||
822 | } | ||
823 | if (engine_id == top_device_info_type_enum_copy0_v()) | ||
824 | gk20a_reset(g, mc_enable_ce2_m()); | ||
825 | } | ||
826 | |||
827 | static void gk20a_fifo_handle_mmu_fault_thread(struct work_struct *work) | ||
828 | { | ||
829 | struct fifo_gk20a *f = container_of(work, struct fifo_gk20a, | ||
830 | fault_restore_thread); | ||
831 | struct gk20a *g = f->g; | ||
832 | int i; | ||
833 | |||
834 | /* Reinitialise FECS and GR */ | ||
835 | gk20a_init_pmu_setup_hw2(g); | ||
836 | |||
837 | /* It is safe to enable ELPG again. */ | ||
838 | gk20a_pmu_enable_elpg(g); | ||
839 | |||
840 | /* Restore the runlist */ | ||
841 | for (i = 0; i < g->fifo.max_runlists; i++) | ||
842 | gk20a_fifo_update_runlist_locked(g, i, ~0, true, true); | ||
843 | |||
844 | /* unlock all runlists */ | ||
845 | for (i = 0; i < g->fifo.max_runlists; i++) | ||
846 | mutex_unlock(&g->fifo.runlist_info[i].mutex); | ||
847 | |||
848 | } | ||
849 | |||
850 | static void gk20a_fifo_handle_chsw_fault(struct gk20a *g) | ||
851 | { | ||
852 | u32 intr; | ||
853 | |||
854 | intr = gk20a_readl(g, fifo_intr_chsw_error_r()); | ||
855 | gk20a_err(dev_from_gk20a(g), "chsw: %08x\n", intr); | ||
856 | gk20a_fecs_dump_falcon_stats(g); | ||
857 | gk20a_writel(g, fifo_intr_chsw_error_r(), intr); | ||
858 | } | ||
859 | |||
860 | static void gk20a_fifo_handle_dropped_mmu_fault(struct gk20a *g) | ||
861 | { | ||
862 | struct device *dev = dev_from_gk20a(g); | ||
863 | u32 fault_id = gk20a_readl(g, fifo_intr_mmu_fault_id_r()); | ||
864 | gk20a_err(dev, "dropped mmu fault (0x%08x)", fault_id); | ||
865 | } | ||
866 | |||
867 | static bool gk20a_fifo_should_defer_engine_reset(struct gk20a *g, u32 engine_id, | ||
868 | struct fifo_mmu_fault_info_gk20a *f, bool fake_fault) | ||
869 | { | ||
870 | /* channel recovery is only deferred if an sm debugger | ||
871 | is attached and has MMU debug mode is enabled */ | ||
872 | if (!gk20a_gr_sm_debugger_attached(g) || | ||
873 | !gk20a_mm_mmu_debug_mode_enabled(g)) | ||
874 | return false; | ||
875 | |||
876 | /* if this fault is fake (due to RC recovery), don't defer recovery */ | ||
877 | if (fake_fault) | ||
878 | return false; | ||
879 | |||
880 | if (engine_id != ENGINE_GR_GK20A || | ||
881 | f->engine_subid_v != fifo_intr_mmu_fault_info_engine_subid_gpc_v()) | ||
882 | return false; | ||
883 | |||
884 | return true; | ||
885 | } | ||
886 | |||
887 | void fifo_gk20a_finish_mmu_fault_handling(struct gk20a *g, | ||
888 | unsigned long fault_id) { | ||
889 | u32 engine_mmu_id; | ||
890 | int i; | ||
891 | |||
892 | /* reset engines */ | ||
893 | for_each_set_bit(engine_mmu_id, &fault_id, 32) { | ||
894 | u32 engine_id = gk20a_mmu_id_to_engine_id(engine_mmu_id); | ||
895 | if (engine_id != ~0) | ||
896 | gk20a_fifo_reset_engine(g, engine_id); | ||
897 | } | ||
898 | |||
899 | /* CLEAR the runlists. Do not wait for runlist to start as | ||
900 | * some engines may not be available right now */ | ||
901 | for (i = 0; i < g->fifo.max_runlists; i++) | ||
902 | gk20a_fifo_update_runlist_locked(g, i, ~0, false, false); | ||
903 | |||
904 | /* clear interrupt */ | ||
905 | gk20a_writel(g, fifo_intr_mmu_fault_id_r(), fault_id); | ||
906 | |||
907 | /* resume scheduler */ | ||
908 | gk20a_writel(g, fifo_error_sched_disable_r(), | ||
909 | gk20a_readl(g, fifo_error_sched_disable_r())); | ||
910 | |||
911 | /* Spawn a work to enable PMU and restore runlists */ | ||
912 | schedule_work(&g->fifo.fault_restore_thread); | ||
913 | } | ||
914 | |||
915 | static bool gk20a_fifo_set_ctx_mmu_error(struct gk20a *g, | ||
916 | struct channel_gk20a *ch) { | ||
917 | bool verbose = true; | ||
918 | if (!ch) | ||
919 | return verbose; | ||
920 | |||
921 | gk20a_err(dev_from_gk20a(g), | ||
922 | "channel %d generated a mmu fault", | ||
923 | ch->hw_chid); | ||
924 | if (ch->error_notifier) { | ||
925 | u32 err = ch->error_notifier->info32; | ||
926 | if (ch->error_notifier->status == 0xffff) { | ||
927 | /* If error code is already set, this mmu fault | ||
928 | * was triggered as part of recovery from other | ||
929 | * error condition. | ||
930 | * Don't overwrite error flag. */ | ||
931 | /* Fifo timeout debug spew is controlled by user */ | ||
932 | if (err == NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT) | ||
933 | verbose = ch->timeout_debug_dump; | ||
934 | } else { | ||
935 | gk20a_set_error_notifier(ch, | ||
936 | NVHOST_CHANNEL_FIFO_ERROR_MMU_ERR_FLT); | ||
937 | } | ||
938 | } | ||
939 | /* mark channel as faulted */ | ||
940 | ch->has_timedout = true; | ||
941 | wmb(); | ||
942 | /* unblock pending waits */ | ||
943 | wake_up(&ch->semaphore_wq); | ||
944 | wake_up(&ch->notifier_wq); | ||
945 | wake_up(&ch->submit_wq); | ||
946 | return verbose; | ||
947 | } | ||
948 | |||
949 | |||
950 | static bool gk20a_fifo_handle_mmu_fault(struct gk20a *g) | ||
951 | { | ||
952 | bool fake_fault; | ||
953 | unsigned long fault_id; | ||
954 | unsigned long engine_mmu_id; | ||
955 | int i; | ||
956 | bool verbose = true; | ||
957 | gk20a_dbg_fn(""); | ||
958 | |||
959 | g->fifo.deferred_reset_pending = false; | ||
960 | |||
961 | /* Disable ELPG */ | ||
962 | gk20a_pmu_disable_elpg(g); | ||
963 | |||
964 | /* If we have recovery in progress, MMU fault id is invalid */ | ||
965 | if (g->fifo.mmu_fault_engines) { | ||
966 | fault_id = g->fifo.mmu_fault_engines; | ||
967 | g->fifo.mmu_fault_engines = 0; | ||
968 | fake_fault = true; | ||
969 | } else { | ||
970 | fault_id = gk20a_readl(g, fifo_intr_mmu_fault_id_r()); | ||
971 | fake_fault = false; | ||
972 | gk20a_debug_dump(g->dev); | ||
973 | } | ||
974 | |||
975 | /* lock all runlists. Note that locks are are released in | ||
976 | * gk20a_fifo_handle_mmu_fault_thread() */ | ||
977 | for (i = 0; i < g->fifo.max_runlists; i++) | ||
978 | mutex_lock(&g->fifo.runlist_info[i].mutex); | ||
979 | |||
980 | /* go through all faulted engines */ | ||
981 | for_each_set_bit(engine_mmu_id, &fault_id, 32) { | ||
982 | /* bits in fifo_intr_mmu_fault_id_r do not correspond 1:1 to | ||
983 | * engines. Convert engine_mmu_id to engine_id */ | ||
984 | u32 engine_id = gk20a_mmu_id_to_engine_id(engine_mmu_id); | ||
985 | struct fifo_runlist_info_gk20a *runlist = g->fifo.runlist_info; | ||
986 | struct fifo_mmu_fault_info_gk20a f; | ||
987 | struct channel_gk20a *ch = NULL; | ||
988 | |||
989 | get_exception_mmu_fault_info(g, engine_mmu_id, &f); | ||
990 | trace_gk20a_mmu_fault(f.fault_hi_v, | ||
991 | f.fault_lo_v, | ||
992 | f.fault_info_v, | ||
993 | f.inst_ptr, | ||
994 | engine_id, | ||
995 | f.engine_subid_desc, | ||
996 | f.client_desc, | ||
997 | f.fault_type_desc); | ||
998 | gk20a_err(dev_from_gk20a(g), "mmu fault on engine %d, " | ||
999 | "engine subid %d (%s), client %d (%s), " | ||
1000 | "addr 0x%08x:0x%08x, type %d (%s), info 0x%08x," | ||
1001 | "inst_ptr 0x%llx\n", | ||
1002 | engine_id, | ||
1003 | f.engine_subid_v, f.engine_subid_desc, | ||
1004 | f.client_v, f.client_desc, | ||
1005 | f.fault_hi_v, f.fault_lo_v, | ||
1006 | f.fault_type_v, f.fault_type_desc, | ||
1007 | f.fault_info_v, f.inst_ptr); | ||
1008 | |||
1009 | /* get the channel */ | ||
1010 | if (fake_fault) { | ||
1011 | /* read and parse engine status */ | ||
1012 | u32 status = gk20a_readl(g, | ||
1013 | fifo_engine_status_r(engine_id)); | ||
1014 | u32 ctx_status = | ||
1015 | fifo_engine_status_ctx_status_v(status); | ||
1016 | bool type_ch = fifo_pbdma_status_id_type_v(status) == | ||
1017 | fifo_pbdma_status_id_type_chid_v(); | ||
1018 | |||
1019 | /* use next_id if context load is failing */ | ||
1020 | u32 id = (ctx_status == | ||
1021 | fifo_engine_status_ctx_status_ctxsw_load_v()) ? | ||
1022 | fifo_engine_status_next_id_v(status) : | ||
1023 | fifo_engine_status_id_v(status); | ||
1024 | |||
1025 | if (type_ch) { | ||
1026 | ch = g->fifo.channel + id; | ||
1027 | } else { | ||
1028 | gk20a_err(dev_from_gk20a(g), "non-chid type not supported"); | ||
1029 | WARN_ON(1); | ||
1030 | } | ||
1031 | } else { | ||
1032 | /* read channel based on instruction pointer */ | ||
1033 | ch = channel_from_inst_ptr(&g->fifo, f.inst_ptr); | ||
1034 | } | ||
1035 | |||
1036 | if (ch) { | ||
1037 | if (ch->in_use) { | ||
1038 | /* disable the channel from hw and increment | ||
1039 | * syncpoints */ | ||
1040 | gk20a_disable_channel_no_update(ch); | ||
1041 | |||
1042 | /* remove the channel from runlist */ | ||
1043 | clear_bit(ch->hw_chid, | ||
1044 | runlist->active_channels); | ||
1045 | } | ||
1046 | |||
1047 | /* check if engine reset should be deferred */ | ||
1048 | if (gk20a_fifo_should_defer_engine_reset(g, engine_id, &f, fake_fault)) { | ||
1049 | g->fifo.mmu_fault_engines = fault_id; | ||
1050 | |||
1051 | /* handled during channel free */ | ||
1052 | g->fifo.deferred_reset_pending = true; | ||
1053 | } else | ||
1054 | verbose = gk20a_fifo_set_ctx_mmu_error(g, ch); | ||
1055 | |||
1056 | } else if (f.inst_ptr == | ||
1057 | g->mm.bar1.inst_block.cpu_pa) { | ||
1058 | gk20a_err(dev_from_gk20a(g), "mmu fault from bar1"); | ||
1059 | } else if (f.inst_ptr == | ||
1060 | g->mm.pmu.inst_block.cpu_pa) { | ||
1061 | gk20a_err(dev_from_gk20a(g), "mmu fault from pmu"); | ||
1062 | } else | ||
1063 | gk20a_err(dev_from_gk20a(g), "couldn't locate channel for mmu fault"); | ||
1064 | } | ||
1065 | |||
1066 | if (g->fifo.deferred_reset_pending) { | ||
1067 | gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, "sm debugger attached," | ||
1068 | " deferring channel recovery to channel free"); | ||
1069 | /* clear interrupt */ | ||
1070 | gk20a_writel(g, fifo_intr_mmu_fault_id_r(), fault_id); | ||
1071 | return verbose; | ||
1072 | } | ||
1073 | |||
1074 | /* resetting the engines and clearing the runlists is done in | ||
1075 | a separate function to allow deferred reset. */ | ||
1076 | fifo_gk20a_finish_mmu_fault_handling(g, fault_id); | ||
1077 | return verbose; | ||
1078 | } | ||
1079 | |||
1080 | static void gk20a_fifo_get_faulty_channel(struct gk20a *g, int engine_id, | ||
1081 | u32 *chid, bool *type_ch) | ||
1082 | { | ||
1083 | u32 status = gk20a_readl(g, fifo_engine_status_r(engine_id)); | ||
1084 | u32 ctx_status = fifo_engine_status_ctx_status_v(status); | ||
1085 | |||
1086 | *type_ch = fifo_pbdma_status_id_type_v(status) == | ||
1087 | fifo_pbdma_status_id_type_chid_v(); | ||
1088 | /* use next_id if context load is failing */ | ||
1089 | *chid = (ctx_status == | ||
1090 | fifo_engine_status_ctx_status_ctxsw_load_v()) ? | ||
1091 | fifo_engine_status_next_id_v(status) : | ||
1092 | fifo_engine_status_id_v(status); | ||
1093 | } | ||
1094 | |||
1095 | void gk20a_fifo_recover(struct gk20a *g, u32 __engine_ids, | ||
1096 | bool verbose) | ||
1097 | { | ||
1098 | unsigned long end_jiffies = jiffies + | ||
1099 | msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)); | ||
1100 | unsigned long delay = GR_IDLE_CHECK_DEFAULT; | ||
1101 | unsigned long engine_id, i; | ||
1102 | unsigned long _engine_ids = __engine_ids; | ||
1103 | unsigned long engine_ids = 0; | ||
1104 | int ret; | ||
1105 | |||
1106 | if (verbose) | ||
1107 | gk20a_debug_dump(g->dev); | ||
1108 | |||
1109 | /* store faulted engines in advance */ | ||
1110 | g->fifo.mmu_fault_engines = 0; | ||
1111 | for_each_set_bit(engine_id, &_engine_ids, 32) { | ||
1112 | bool ref_type_ch; | ||
1113 | int ref_chid; | ||
1114 | gk20a_fifo_get_faulty_channel(g, engine_id, &ref_chid, | ||
1115 | &ref_type_ch); | ||
1116 | |||
1117 | /* Reset *all* engines that use the | ||
1118 | * same channel as faulty engine */ | ||
1119 | for (i = 0; i < g->fifo.max_engines; i++) { | ||
1120 | bool type_ch; | ||
1121 | u32 chid; | ||
1122 | gk20a_fifo_get_faulty_channel(g, i, &chid, &type_ch); | ||
1123 | if (ref_type_ch == type_ch && ref_chid == chid) { | ||
1124 | engine_ids |= BIT(i); | ||
1125 | g->fifo.mmu_fault_engines |= | ||
1126 | BIT(gk20a_engine_id_to_mmu_id(i)); | ||
1127 | } | ||
1128 | } | ||
1129 | |||
1130 | } | ||
1131 | |||
1132 | /* trigger faults for all bad engines */ | ||
1133 | for_each_set_bit(engine_id, &engine_ids, 32) { | ||
1134 | if (engine_id > g->fifo.max_engines) { | ||
1135 | WARN_ON(true); | ||
1136 | break; | ||
1137 | } | ||
1138 | |||
1139 | gk20a_writel(g, fifo_trigger_mmu_fault_r(engine_id), | ||
1140 | fifo_trigger_mmu_fault_id_f( | ||
1141 | gk20a_engine_id_to_mmu_id(engine_id)) | | ||
1142 | fifo_trigger_mmu_fault_enable_f(1)); | ||
1143 | } | ||
1144 | |||
1145 | /* Wait for MMU fault to trigger */ | ||
1146 | ret = -EBUSY; | ||
1147 | do { | ||
1148 | if (gk20a_readl(g, fifo_intr_0_r()) & | ||
1149 | fifo_intr_0_mmu_fault_pending_f()) { | ||
1150 | ret = 0; | ||
1151 | break; | ||
1152 | } | ||
1153 | |||
1154 | usleep_range(delay, delay * 2); | ||
1155 | delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); | ||
1156 | } while (time_before(jiffies, end_jiffies) || | ||
1157 | !tegra_platform_is_silicon()); | ||
1158 | |||
1159 | if (ret) | ||
1160 | gk20a_err(dev_from_gk20a(g), "mmu fault timeout"); | ||
1161 | |||
1162 | /* release mmu fault trigger */ | ||
1163 | for_each_set_bit(engine_id, &engine_ids, 32) | ||
1164 | gk20a_writel(g, fifo_trigger_mmu_fault_r(engine_id), 0); | ||
1165 | } | ||
1166 | |||
1167 | |||
1168 | static bool gk20a_fifo_handle_sched_error(struct gk20a *g) | ||
1169 | { | ||
1170 | u32 sched_error; | ||
1171 | u32 engine_id; | ||
1172 | int id = -1; | ||
1173 | bool non_chid = false; | ||
1174 | |||
1175 | /* read and reset the scheduler error register */ | ||
1176 | sched_error = gk20a_readl(g, fifo_intr_sched_error_r()); | ||
1177 | gk20a_writel(g, fifo_intr_0_r(), fifo_intr_0_sched_error_reset_f()); | ||
1178 | |||
1179 | for (engine_id = 0; engine_id < g->fifo.max_engines; engine_id++) { | ||
1180 | u32 status = gk20a_readl(g, fifo_engine_status_r(engine_id)); | ||
1181 | u32 ctx_status = fifo_engine_status_ctx_status_v(status); | ||
1182 | bool failing_engine; | ||
1183 | |||
1184 | /* we are interested in busy engines */ | ||
1185 | failing_engine = fifo_engine_status_engine_v(status) == | ||
1186 | fifo_engine_status_engine_busy_v(); | ||
1187 | |||
1188 | /* ..that are doing context switch */ | ||
1189 | failing_engine = failing_engine && | ||
1190 | (ctx_status == | ||
1191 | fifo_engine_status_ctx_status_ctxsw_switch_v() | ||
1192 | || ctx_status == | ||
1193 | fifo_engine_status_ctx_status_ctxsw_save_v() | ||
1194 | || ctx_status == | ||
1195 | fifo_engine_status_ctx_status_ctxsw_load_v()); | ||
1196 | |||
1197 | if (failing_engine) { | ||
1198 | id = (ctx_status == | ||
1199 | fifo_engine_status_ctx_status_ctxsw_load_v()) ? | ||
1200 | fifo_engine_status_next_id_v(status) : | ||
1201 | fifo_engine_status_id_v(status); | ||
1202 | non_chid = fifo_pbdma_status_id_type_v(status) != | ||
1203 | fifo_pbdma_status_id_type_chid_v(); | ||
1204 | break; | ||
1205 | } | ||
1206 | } | ||
1207 | |||
1208 | /* could not find the engine - should never happen */ | ||
1209 | if (unlikely(engine_id >= g->fifo.max_engines)) | ||
1210 | goto err; | ||
1211 | |||
1212 | if (fifo_intr_sched_error_code_f(sched_error) == | ||
1213 | fifo_intr_sched_error_code_ctxsw_timeout_v()) { | ||
1214 | struct fifo_gk20a *f = &g->fifo; | ||
1215 | struct channel_gk20a *ch = &f->channel[id]; | ||
1216 | |||
1217 | if (non_chid) { | ||
1218 | gk20a_fifo_recover(g, BIT(engine_id), true); | ||
1219 | goto err; | ||
1220 | } | ||
1221 | |||
1222 | if (gk20a_channel_update_and_check_timeout(ch, | ||
1223 | GRFIFO_TIMEOUT_CHECK_PERIOD_US / 1000)) { | ||
1224 | gk20a_set_error_notifier(ch, | ||
1225 | NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT); | ||
1226 | gk20a_err(dev_from_gk20a(g), | ||
1227 | "fifo sched ctxsw timeout error:" | ||
1228 | "engine = %u, ch = %d", engine_id, id); | ||
1229 | gk20a_fifo_recover(g, BIT(engine_id), | ||
1230 | ch->timeout_debug_dump); | ||
1231 | } else { | ||
1232 | gk20a_warn(dev_from_gk20a(g), | ||
1233 | "fifo is waiting for ctx switch for %d ms," | ||
1234 | "ch = %d\n", | ||
1235 | ch->timeout_accumulated_ms, | ||
1236 | id); | ||
1237 | } | ||
1238 | return ch->timeout_debug_dump; | ||
1239 | } | ||
1240 | err: | ||
1241 | gk20a_err(dev_from_gk20a(g), "fifo sched error : 0x%08x, engine=%u, %s=%d", | ||
1242 | sched_error, engine_id, non_chid ? "non-ch" : "ch", id); | ||
1243 | |||
1244 | return true; | ||
1245 | } | ||
1246 | |||
1247 | static u32 fifo_error_isr(struct gk20a *g, u32 fifo_intr) | ||
1248 | { | ||
1249 | bool print_channel_reset_log = false, reset_engine = false; | ||
1250 | struct device *dev = dev_from_gk20a(g); | ||
1251 | u32 handled = 0; | ||
1252 | |||
1253 | gk20a_dbg_fn(""); | ||
1254 | |||
1255 | if (fifo_intr & fifo_intr_0_pio_error_pending_f()) { | ||
1256 | /* pio mode is unused. this shouldn't happen, ever. */ | ||
1257 | /* should we clear it or just leave it pending? */ | ||
1258 | gk20a_err(dev, "fifo pio error!\n"); | ||
1259 | BUG_ON(1); | ||
1260 | } | ||
1261 | |||
1262 | if (fifo_intr & fifo_intr_0_bind_error_pending_f()) { | ||
1263 | u32 bind_error = gk20a_readl(g, fifo_intr_bind_error_r()); | ||
1264 | gk20a_err(dev, "fifo bind error: 0x%08x", bind_error); | ||
1265 | print_channel_reset_log = true; | ||
1266 | handled |= fifo_intr_0_bind_error_pending_f(); | ||
1267 | } | ||
1268 | |||
1269 | if (fifo_intr & fifo_intr_0_sched_error_pending_f()) { | ||
1270 | print_channel_reset_log = gk20a_fifo_handle_sched_error(g); | ||
1271 | handled |= fifo_intr_0_sched_error_pending_f(); | ||
1272 | } | ||
1273 | |||
1274 | if (fifo_intr & fifo_intr_0_chsw_error_pending_f()) { | ||
1275 | gk20a_fifo_handle_chsw_fault(g); | ||
1276 | handled |= fifo_intr_0_chsw_error_pending_f(); | ||
1277 | } | ||
1278 | |||
1279 | if (fifo_intr & fifo_intr_0_mmu_fault_pending_f()) { | ||
1280 | print_channel_reset_log = gk20a_fifo_handle_mmu_fault(g); | ||
1281 | reset_engine = true; | ||
1282 | handled |= fifo_intr_0_mmu_fault_pending_f(); | ||
1283 | } | ||
1284 | |||
1285 | if (fifo_intr & fifo_intr_0_dropped_mmu_fault_pending_f()) { | ||
1286 | gk20a_fifo_handle_dropped_mmu_fault(g); | ||
1287 | handled |= fifo_intr_0_dropped_mmu_fault_pending_f(); | ||
1288 | } | ||
1289 | |||
1290 | print_channel_reset_log = !g->fifo.deferred_reset_pending | ||
1291 | && print_channel_reset_log; | ||
1292 | |||
1293 | if (print_channel_reset_log) { | ||
1294 | int engine_id; | ||
1295 | gk20a_err(dev_from_gk20a(g), | ||
1296 | "channel reset initated from %s", __func__); | ||
1297 | for (engine_id = 0; | ||
1298 | engine_id < g->fifo.max_engines; | ||
1299 | engine_id++) { | ||
1300 | gk20a_dbg_fn("enum:%d -> engine_id:%d", engine_id, | ||
1301 | g->fifo.engine_info[engine_id].engine_id); | ||
1302 | fifo_pbdma_exception_status(g, | ||
1303 | &g->fifo.engine_info[engine_id]); | ||
1304 | fifo_engine_exception_status(g, | ||
1305 | &g->fifo.engine_info[engine_id]); | ||
1306 | } | ||
1307 | } | ||
1308 | |||
1309 | return handled; | ||
1310 | } | ||
1311 | |||
1312 | |||
1313 | static u32 gk20a_fifo_handle_pbdma_intr(struct device *dev, | ||
1314 | struct gk20a *g, | ||
1315 | struct fifo_gk20a *f, | ||
1316 | u32 pbdma_id) | ||
1317 | { | ||
1318 | u32 pbdma_intr_0 = gk20a_readl(g, pbdma_intr_0_r(pbdma_id)); | ||
1319 | u32 pbdma_intr_1 = gk20a_readl(g, pbdma_intr_1_r(pbdma_id)); | ||
1320 | u32 handled = 0; | ||
1321 | bool reset_device = false; | ||
1322 | bool reset_channel = false; | ||
1323 | |||
1324 | gk20a_dbg_fn(""); | ||
1325 | |||
1326 | gk20a_dbg(gpu_dbg_intr, "pbdma id intr pending %d %08x %08x", pbdma_id, | ||
1327 | pbdma_intr_0, pbdma_intr_1); | ||
1328 | if (pbdma_intr_0) { | ||
1329 | if (f->intr.pbdma.device_fatal_0 & pbdma_intr_0) { | ||
1330 | dev_err(dev, "unrecoverable device error: " | ||
1331 | "pbdma_intr_0(%d):0x%08x", pbdma_id, pbdma_intr_0); | ||
1332 | reset_device = true; | ||
1333 | /* TODO: disable pbdma intrs */ | ||
1334 | handled |= f->intr.pbdma.device_fatal_0 & pbdma_intr_0; | ||
1335 | } | ||
1336 | if (f->intr.pbdma.channel_fatal_0 & pbdma_intr_0) { | ||
1337 | dev_warn(dev, "channel error: " | ||
1338 | "pbdma_intr_0(%d):0x%08x", pbdma_id, pbdma_intr_0); | ||
1339 | reset_channel = true; | ||
1340 | /* TODO: clear pbdma channel errors */ | ||
1341 | handled |= f->intr.pbdma.channel_fatal_0 & pbdma_intr_0; | ||
1342 | } | ||
1343 | if (f->intr.pbdma.restartable_0 & pbdma_intr_0) { | ||
1344 | dev_warn(dev, "sw method: %08x %08x", | ||
1345 | gk20a_readl(g, pbdma_method0_r(0)), | ||
1346 | gk20a_readl(g, pbdma_method0_r(0)+4)); | ||
1347 | gk20a_writel(g, pbdma_method0_r(0), 0); | ||
1348 | gk20a_writel(g, pbdma_method0_r(0)+4, 0); | ||
1349 | handled |= f->intr.pbdma.restartable_0 & pbdma_intr_0; | ||
1350 | } | ||
1351 | |||
1352 | gk20a_writel(g, pbdma_intr_0_r(pbdma_id), pbdma_intr_0); | ||
1353 | } | ||
1354 | |||
1355 | /* all intrs in _intr_1 are "host copy engine" related, | ||
1356 | * which gk20a doesn't have. for now just make them channel fatal. */ | ||
1357 | if (pbdma_intr_1) { | ||
1358 | dev_err(dev, "channel hce error: pbdma_intr_1(%d): 0x%08x", | ||
1359 | pbdma_id, pbdma_intr_1); | ||
1360 | reset_channel = true; | ||
1361 | gk20a_writel(g, pbdma_intr_1_r(pbdma_id), pbdma_intr_1); | ||
1362 | } | ||
1363 | |||
1364 | |||
1365 | |||
1366 | return handled; | ||
1367 | } | ||
1368 | |||
1369 | static u32 fifo_channel_isr(struct gk20a *g, u32 fifo_intr) | ||
1370 | { | ||
1371 | gk20a_channel_semaphore_wakeup(g); | ||
1372 | return fifo_intr_0_channel_intr_pending_f(); | ||
1373 | } | ||
1374 | |||
1375 | |||
1376 | static u32 fifo_pbdma_isr(struct gk20a *g, u32 fifo_intr) | ||
1377 | { | ||
1378 | struct device *dev = dev_from_gk20a(g); | ||
1379 | struct fifo_gk20a *f = &g->fifo; | ||
1380 | u32 clear_intr = 0, i; | ||
1381 | u32 pbdma_pending = gk20a_readl(g, fifo_intr_pbdma_id_r()); | ||
1382 | |||
1383 | for (i = 0; i < fifo_intr_pbdma_id_status__size_1_v(); i++) { | ||
1384 | if (fifo_intr_pbdma_id_status_f(pbdma_pending, i)) { | ||
1385 | gk20a_dbg(gpu_dbg_intr, "pbdma id %d intr pending", i); | ||
1386 | clear_intr |= | ||
1387 | gk20a_fifo_handle_pbdma_intr(dev, g, f, i); | ||
1388 | } | ||
1389 | } | ||
1390 | return fifo_intr_0_pbdma_intr_pending_f(); | ||
1391 | } | ||
1392 | |||
1393 | void gk20a_fifo_isr(struct gk20a *g) | ||
1394 | { | ||
1395 | u32 error_intr_mask = | ||
1396 | fifo_intr_0_bind_error_pending_f() | | ||
1397 | fifo_intr_0_sched_error_pending_f() | | ||
1398 | fifo_intr_0_chsw_error_pending_f() | | ||
1399 | fifo_intr_0_fb_flush_timeout_pending_f() | | ||
1400 | fifo_intr_0_dropped_mmu_fault_pending_f() | | ||
1401 | fifo_intr_0_mmu_fault_pending_f() | | ||
1402 | fifo_intr_0_lb_error_pending_f() | | ||
1403 | fifo_intr_0_pio_error_pending_f(); | ||
1404 | |||
1405 | u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r()); | ||
1406 | u32 clear_intr = 0; | ||
1407 | |||
1408 | /* note we're not actually in an "isr", but rather | ||
1409 | * in a threaded interrupt context... */ | ||
1410 | mutex_lock(&g->fifo.intr.isr.mutex); | ||
1411 | |||
1412 | gk20a_dbg(gpu_dbg_intr, "fifo isr %08x\n", fifo_intr); | ||
1413 | |||
1414 | /* handle runlist update */ | ||
1415 | if (fifo_intr & fifo_intr_0_runlist_event_pending_f()) { | ||
1416 | gk20a_fifo_handle_runlist_event(g); | ||
1417 | clear_intr |= fifo_intr_0_runlist_event_pending_f(); | ||
1418 | } | ||
1419 | if (fifo_intr & fifo_intr_0_pbdma_intr_pending_f()) | ||
1420 | clear_intr |= fifo_pbdma_isr(g, fifo_intr); | ||
1421 | |||
1422 | if (unlikely(fifo_intr & error_intr_mask)) | ||
1423 | clear_intr = fifo_error_isr(g, fifo_intr); | ||
1424 | |||
1425 | gk20a_writel(g, fifo_intr_0_r(), clear_intr); | ||
1426 | |||
1427 | mutex_unlock(&g->fifo.intr.isr.mutex); | ||
1428 | |||
1429 | return; | ||
1430 | } | ||
1431 | |||
1432 | void gk20a_fifo_nonstall_isr(struct gk20a *g) | ||
1433 | { | ||
1434 | u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r()); | ||
1435 | u32 clear_intr = 0; | ||
1436 | |||
1437 | gk20a_dbg(gpu_dbg_intr, "fifo nonstall isr %08x\n", fifo_intr); | ||
1438 | |||
1439 | if (fifo_intr & fifo_intr_0_channel_intr_pending_f()) | ||
1440 | clear_intr |= fifo_channel_isr(g, fifo_intr); | ||
1441 | |||
1442 | gk20a_writel(g, fifo_intr_0_r(), clear_intr); | ||
1443 | |||
1444 | return; | ||
1445 | } | ||
1446 | |||
1447 | int gk20a_fifo_preempt_channel(struct gk20a *g, u32 hw_chid) | ||
1448 | { | ||
1449 | struct fifo_gk20a *f = &g->fifo; | ||
1450 | unsigned long end_jiffies = jiffies | ||
1451 | + msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)); | ||
1452 | u32 delay = GR_IDLE_CHECK_DEFAULT; | ||
1453 | u32 ret = 0; | ||
1454 | u32 token = PMU_INVALID_MUTEX_OWNER_ID; | ||
1455 | u32 elpg_off = 0; | ||
1456 | u32 i; | ||
1457 | |||
1458 | gk20a_dbg_fn("%d", hw_chid); | ||
1459 | |||
1460 | /* we have no idea which runlist we are using. lock all */ | ||
1461 | for (i = 0; i < g->fifo.max_runlists; i++) | ||
1462 | mutex_lock(&f->runlist_info[i].mutex); | ||
1463 | |||
1464 | /* disable elpg if failed to acquire pmu mutex */ | ||
1465 | elpg_off = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
1466 | if (elpg_off) | ||
1467 | gk20a_pmu_disable_elpg(g); | ||
1468 | |||
1469 | /* issue preempt */ | ||
1470 | gk20a_writel(g, fifo_preempt_r(), | ||
1471 | fifo_preempt_chid_f(hw_chid) | | ||
1472 | fifo_preempt_type_channel_f()); | ||
1473 | |||
1474 | /* wait for preempt */ | ||
1475 | ret = -EBUSY; | ||
1476 | do { | ||
1477 | if (!(gk20a_readl(g, fifo_preempt_r()) & | ||
1478 | fifo_preempt_pending_true_f())) { | ||
1479 | ret = 0; | ||
1480 | break; | ||
1481 | } | ||
1482 | |||
1483 | usleep_range(delay, delay * 2); | ||
1484 | delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); | ||
1485 | } while (time_before(jiffies, end_jiffies) || | ||
1486 | !tegra_platform_is_silicon()); | ||
1487 | |||
1488 | if (ret) { | ||
1489 | int i; | ||
1490 | u32 engines = 0; | ||
1491 | struct fifo_gk20a *f = &g->fifo; | ||
1492 | struct channel_gk20a *ch = &f->channel[hw_chid]; | ||
1493 | |||
1494 | gk20a_err(dev_from_gk20a(g), "preempt channel %d timeout\n", | ||
1495 | hw_chid); | ||
1496 | |||
1497 | /* forcefully reset all busy engines using this channel */ | ||
1498 | for (i = 0; i < g->fifo.max_engines; i++) { | ||
1499 | u32 status = gk20a_readl(g, fifo_engine_status_r(i)); | ||
1500 | u32 ctx_status = | ||
1501 | fifo_engine_status_ctx_status_v(status); | ||
1502 | bool type_ch = fifo_pbdma_status_id_type_v(status) == | ||
1503 | fifo_pbdma_status_id_type_chid_v(); | ||
1504 | bool busy = fifo_engine_status_engine_v(status) == | ||
1505 | fifo_engine_status_engine_busy_v(); | ||
1506 | u32 id = (ctx_status == | ||
1507 | fifo_engine_status_ctx_status_ctxsw_load_v()) ? | ||
1508 | fifo_engine_status_next_id_v(status) : | ||
1509 | fifo_engine_status_id_v(status); | ||
1510 | |||
1511 | if (type_ch && busy && id == hw_chid) | ||
1512 | engines |= BIT(i); | ||
1513 | } | ||
1514 | gk20a_set_error_notifier(ch, | ||
1515 | NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT); | ||
1516 | gk20a_fifo_recover(g, engines, true); | ||
1517 | } | ||
1518 | |||
1519 | /* re-enable elpg or release pmu mutex */ | ||
1520 | if (elpg_off) | ||
1521 | gk20a_pmu_enable_elpg(g); | ||
1522 | else | ||
1523 | pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
1524 | |||
1525 | for (i = 0; i < g->fifo.max_runlists; i++) | ||
1526 | mutex_unlock(&f->runlist_info[i].mutex); | ||
1527 | |||
1528 | return ret; | ||
1529 | } | ||
1530 | |||
1531 | int gk20a_fifo_enable_engine_activity(struct gk20a *g, | ||
1532 | struct fifo_engine_info_gk20a *eng_info) | ||
1533 | { | ||
1534 | u32 token = PMU_INVALID_MUTEX_OWNER_ID; | ||
1535 | u32 elpg_off; | ||
1536 | u32 enable; | ||
1537 | |||
1538 | gk20a_dbg_fn(""); | ||
1539 | |||
1540 | /* disable elpg if failed to acquire pmu mutex */ | ||
1541 | elpg_off = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
1542 | if (elpg_off) | ||
1543 | gk20a_pmu_disable_elpg(g); | ||
1544 | |||
1545 | enable = gk20a_readl(g, fifo_sched_disable_r()); | ||
1546 | enable &= ~(fifo_sched_disable_true_v() >> eng_info->runlist_id); | ||
1547 | gk20a_writel(g, fifo_sched_disable_r(), enable); | ||
1548 | |||
1549 | /* re-enable elpg or release pmu mutex */ | ||
1550 | if (elpg_off) | ||
1551 | gk20a_pmu_enable_elpg(g); | ||
1552 | else | ||
1553 | pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
1554 | |||
1555 | gk20a_dbg_fn("done"); | ||
1556 | return 0; | ||
1557 | } | ||
1558 | |||
1559 | int gk20a_fifo_disable_engine_activity(struct gk20a *g, | ||
1560 | struct fifo_engine_info_gk20a *eng_info, | ||
1561 | bool wait_for_idle) | ||
1562 | { | ||
1563 | u32 gr_stat, pbdma_stat, chan_stat, eng_stat, ctx_stat; | ||
1564 | u32 pbdma_chid = ~0, engine_chid = ~0, disable; | ||
1565 | u32 token = PMU_INVALID_MUTEX_OWNER_ID; | ||
1566 | u32 elpg_off; | ||
1567 | u32 err = 0; | ||
1568 | |||
1569 | gk20a_dbg_fn(""); | ||
1570 | |||
1571 | gr_stat = | ||
1572 | gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id)); | ||
1573 | if (fifo_engine_status_engine_v(gr_stat) == | ||
1574 | fifo_engine_status_engine_busy_v() && !wait_for_idle) | ||
1575 | return -EBUSY; | ||
1576 | |||
1577 | /* disable elpg if failed to acquire pmu mutex */ | ||
1578 | elpg_off = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
1579 | if (elpg_off) | ||
1580 | gk20a_pmu_disable_elpg(g); | ||
1581 | |||
1582 | disable = gk20a_readl(g, fifo_sched_disable_r()); | ||
1583 | disable = set_field(disable, | ||
1584 | fifo_sched_disable_runlist_m(eng_info->runlist_id), | ||
1585 | fifo_sched_disable_runlist_f(fifo_sched_disable_true_v(), | ||
1586 | eng_info->runlist_id)); | ||
1587 | gk20a_writel(g, fifo_sched_disable_r(), disable); | ||
1588 | |||
1589 | /* chid from pbdma status */ | ||
1590 | pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(eng_info->pbdma_id)); | ||
1591 | chan_stat = fifo_pbdma_status_chan_status_v(pbdma_stat); | ||
1592 | if (chan_stat == fifo_pbdma_status_chan_status_valid_v() || | ||
1593 | chan_stat == fifo_pbdma_status_chan_status_chsw_save_v()) | ||
1594 | pbdma_chid = fifo_pbdma_status_id_v(pbdma_stat); | ||
1595 | else if (chan_stat == fifo_pbdma_status_chan_status_chsw_load_v() || | ||
1596 | chan_stat == fifo_pbdma_status_chan_status_chsw_switch_v()) | ||
1597 | pbdma_chid = fifo_pbdma_status_next_id_v(pbdma_stat); | ||
1598 | |||
1599 | if (pbdma_chid != ~0) { | ||
1600 | err = gk20a_fifo_preempt_channel(g, pbdma_chid); | ||
1601 | if (err) | ||
1602 | goto clean_up; | ||
1603 | } | ||
1604 | |||
1605 | /* chid from engine status */ | ||
1606 | eng_stat = gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id)); | ||
1607 | ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); | ||
1608 | if (ctx_stat == fifo_engine_status_ctx_status_valid_v() || | ||
1609 | ctx_stat == fifo_engine_status_ctx_status_ctxsw_save_v()) | ||
1610 | engine_chid = fifo_engine_status_id_v(eng_stat); | ||
1611 | else if (ctx_stat == fifo_engine_status_ctx_status_ctxsw_load_v() || | ||
1612 | ctx_stat == fifo_engine_status_ctx_status_ctxsw_switch_v()) | ||
1613 | engine_chid = fifo_engine_status_next_id_v(eng_stat); | ||
1614 | |||
1615 | if (engine_chid != ~0 && engine_chid != pbdma_chid) { | ||
1616 | err = gk20a_fifo_preempt_channel(g, engine_chid); | ||
1617 | if (err) | ||
1618 | goto clean_up; | ||
1619 | } | ||
1620 | |||
1621 | clean_up: | ||
1622 | /* re-enable elpg or release pmu mutex */ | ||
1623 | if (elpg_off) | ||
1624 | gk20a_pmu_enable_elpg(g); | ||
1625 | else | ||
1626 | pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
1627 | |||
1628 | if (err) { | ||
1629 | gk20a_dbg_fn("failed"); | ||
1630 | if (gk20a_fifo_enable_engine_activity(g, eng_info)) | ||
1631 | gk20a_err(dev_from_gk20a(g), | ||
1632 | "failed to enable gr engine activity\n"); | ||
1633 | } else { | ||
1634 | gk20a_dbg_fn("done"); | ||
1635 | } | ||
1636 | return err; | ||
1637 | } | ||
1638 | |||
1639 | static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id) | ||
1640 | { | ||
1641 | struct fifo_gk20a *f = &g->fifo; | ||
1642 | u32 engines = 0; | ||
1643 | int i; | ||
1644 | |||
1645 | for (i = 0; i < f->max_engines; i++) { | ||
1646 | u32 status = gk20a_readl(g, fifo_engine_status_r(i)); | ||
1647 | bool engine_busy = fifo_engine_status_engine_v(status) == | ||
1648 | fifo_engine_status_engine_busy_v(); | ||
1649 | |||
1650 | if (engine_busy && | ||
1651 | (f->engine_info[i].runlist_id == runlist_id)) | ||
1652 | engines |= BIT(i); | ||
1653 | } | ||
1654 | gk20a_fifo_recover(g, engines, true); | ||
1655 | } | ||
1656 | |||
1657 | static int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id) | ||
1658 | { | ||
1659 | struct fifo_runlist_info_gk20a *runlist; | ||
1660 | u32 remain; | ||
1661 | bool pending; | ||
1662 | |||
1663 | runlist = &g->fifo.runlist_info[runlist_id]; | ||
1664 | remain = wait_event_timeout(runlist->runlist_wq, | ||
1665 | ((pending = gk20a_readl(g, fifo_eng_runlist_r(runlist_id)) & | ||
1666 | fifo_eng_runlist_pending_true_f()) == 0), | ||
1667 | msecs_to_jiffies(gk20a_get_gr_idle_timeout(g))); | ||
1668 | |||
1669 | if (remain == 0 && pending != 0) | ||
1670 | return -ETIMEDOUT; | ||
1671 | |||
1672 | return 0; | ||
1673 | } | ||
1674 | |||
1675 | static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, | ||
1676 | u32 hw_chid, bool add, | ||
1677 | bool wait_for_finish) | ||
1678 | { | ||
1679 | u32 ret = 0; | ||
1680 | struct device *d = dev_from_gk20a(g); | ||
1681 | struct fifo_gk20a *f = &g->fifo; | ||
1682 | struct fifo_runlist_info_gk20a *runlist = NULL; | ||
1683 | u32 *runlist_entry_base = NULL; | ||
1684 | u32 *runlist_entry = NULL; | ||
1685 | phys_addr_t runlist_pa; | ||
1686 | u32 old_buf, new_buf; | ||
1687 | u32 chid; | ||
1688 | u32 count = 0; | ||
1689 | runlist = &f->runlist_info[runlist_id]; | ||
1690 | |||
1691 | /* valid channel, add/remove it from active list. | ||
1692 | Otherwise, keep active list untouched for suspend/resume. */ | ||
1693 | if (hw_chid != ~0) { | ||
1694 | if (add) { | ||
1695 | if (test_and_set_bit(hw_chid, | ||
1696 | runlist->active_channels) == 1) | ||
1697 | return 0; | ||
1698 | } else { | ||
1699 | if (test_and_clear_bit(hw_chid, | ||
1700 | runlist->active_channels) == 0) | ||
1701 | return 0; | ||
1702 | } | ||
1703 | } | ||
1704 | |||
1705 | old_buf = runlist->cur_buffer; | ||
1706 | new_buf = !runlist->cur_buffer; | ||
1707 | |||
1708 | gk20a_dbg_info("runlist_id : %d, switch to new buffer 0x%16llx", | ||
1709 | runlist_id, runlist->mem[new_buf].iova); | ||
1710 | |||
1711 | runlist_pa = gk20a_get_phys_from_iova(d, runlist->mem[new_buf].iova); | ||
1712 | if (!runlist_pa) { | ||
1713 | ret = -EINVAL; | ||
1714 | goto clean_up; | ||
1715 | } | ||
1716 | |||
1717 | runlist_entry_base = runlist->mem[new_buf].cpuva; | ||
1718 | if (!runlist_entry_base) { | ||
1719 | ret = -ENOMEM; | ||
1720 | goto clean_up; | ||
1721 | } | ||
1722 | |||
1723 | if (hw_chid != ~0 || /* add/remove a valid channel */ | ||
1724 | add /* resume to add all channels back */) { | ||
1725 | runlist_entry = runlist_entry_base; | ||
1726 | for_each_set_bit(chid, | ||
1727 | runlist->active_channels, f->num_channels) { | ||
1728 | gk20a_dbg_info("add channel %d to runlist", chid); | ||
1729 | runlist_entry[0] = chid; | ||
1730 | runlist_entry[1] = 0; | ||
1731 | runlist_entry += 2; | ||
1732 | count++; | ||
1733 | } | ||
1734 | } else /* suspend to remove all channels */ | ||
1735 | count = 0; | ||
1736 | |||
1737 | if (count != 0) { | ||
1738 | gk20a_writel(g, fifo_runlist_base_r(), | ||
1739 | fifo_runlist_base_ptr_f(u64_lo32(runlist_pa >> 12)) | | ||
1740 | fifo_runlist_base_target_vid_mem_f()); | ||
1741 | } | ||
1742 | |||
1743 | gk20a_writel(g, fifo_runlist_r(), | ||
1744 | fifo_runlist_engine_f(runlist_id) | | ||
1745 | fifo_eng_runlist_length_f(count)); | ||
1746 | |||
1747 | if (wait_for_finish) { | ||
1748 | ret = gk20a_fifo_runlist_wait_pending(g, runlist_id); | ||
1749 | |||
1750 | if (ret == -ETIMEDOUT) { | ||
1751 | gk20a_err(dev_from_gk20a(g), | ||
1752 | "runlist update timeout"); | ||
1753 | |||
1754 | gk20a_fifo_runlist_reset_engines(g, runlist_id); | ||
1755 | |||
1756 | /* engine reset needs the lock. drop it */ | ||
1757 | mutex_unlock(&runlist->mutex); | ||
1758 | /* wait until the runlist is active again */ | ||
1759 | ret = gk20a_fifo_runlist_wait_pending(g, runlist_id); | ||
1760 | /* get the lock back. at this point everything should | ||
1761 | * should be fine */ | ||
1762 | mutex_lock(&runlist->mutex); | ||
1763 | |||
1764 | if (ret) | ||
1765 | gk20a_err(dev_from_gk20a(g), | ||
1766 | "runlist update failed: %d", ret); | ||
1767 | } else if (ret == -EINTR) | ||
1768 | gk20a_err(dev_from_gk20a(g), | ||
1769 | "runlist update interrupted"); | ||
1770 | } | ||
1771 | |||
1772 | runlist->cur_buffer = new_buf; | ||
1773 | |||
1774 | clean_up: | ||
1775 | return ret; | ||
1776 | } | ||
1777 | |||
1778 | /* add/remove a channel from runlist | ||
1779 | special cases below: runlist->active_channels will NOT be changed. | ||
1780 | (hw_chid == ~0 && !add) means remove all active channels from runlist. | ||
1781 | (hw_chid == ~0 && add) means restore all active channels on runlist. */ | ||
1782 | int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 hw_chid, | ||
1783 | bool add, bool wait_for_finish) | ||
1784 | { | ||
1785 | struct fifo_runlist_info_gk20a *runlist = NULL; | ||
1786 | struct fifo_gk20a *f = &g->fifo; | ||
1787 | u32 token = PMU_INVALID_MUTEX_OWNER_ID; | ||
1788 | u32 elpg_off; | ||
1789 | u32 ret = 0; | ||
1790 | |||
1791 | runlist = &f->runlist_info[runlist_id]; | ||
1792 | |||
1793 | mutex_lock(&runlist->mutex); | ||
1794 | |||
1795 | /* disable elpg if failed to acquire pmu mutex */ | ||
1796 | elpg_off = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
1797 | if (elpg_off) | ||
1798 | gk20a_pmu_disable_elpg(g); | ||
1799 | |||
1800 | ret = gk20a_fifo_update_runlist_locked(g, runlist_id, hw_chid, add, | ||
1801 | wait_for_finish); | ||
1802 | |||
1803 | /* re-enable elpg or release pmu mutex */ | ||
1804 | if (elpg_off) | ||
1805 | gk20a_pmu_enable_elpg(g); | ||
1806 | else | ||
1807 | pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
1808 | |||
1809 | mutex_unlock(&runlist->mutex); | ||
1810 | return ret; | ||
1811 | } | ||
1812 | |||
1813 | int gk20a_fifo_suspend(struct gk20a *g) | ||
1814 | { | ||
1815 | gk20a_dbg_fn(""); | ||
1816 | |||
1817 | /* stop bar1 snooping */ | ||
1818 | gk20a_writel(g, fifo_bar1_base_r(), | ||
1819 | fifo_bar1_base_valid_false_f()); | ||
1820 | |||
1821 | /* disable fifo intr */ | ||
1822 | gk20a_writel(g, fifo_intr_en_0_r(), 0); | ||
1823 | gk20a_writel(g, fifo_intr_en_1_r(), 0); | ||
1824 | |||
1825 | gk20a_dbg_fn("done"); | ||
1826 | return 0; | ||
1827 | } | ||
1828 | |||
1829 | bool gk20a_fifo_mmu_fault_pending(struct gk20a *g) | ||
1830 | { | ||
1831 | if (gk20a_readl(g, fifo_intr_0_r()) & | ||
1832 | fifo_intr_0_mmu_fault_pending_f()) | ||
1833 | return true; | ||
1834 | else | ||
1835 | return false; | ||
1836 | } | ||