summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
diff options
context:
space:
mode:
authorArto Merilainen <amerilainen@nvidia.com>2014-03-19 03:38:25 -0400
committerDan Willemsen <dwillemsen@nvidia.com>2015-03-18 15:08:53 -0400
commita9785995d5f22aaeb659285f8aeb64d8b56982e0 (patch)
treecc75f75bcf43db316a002a7a240b81f299bf6d7f /drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
parent61efaf843c22b85424036ec98015121c08f5f16c (diff)
gpu: nvgpu: Add NVIDIA GPU Driver
This patch moves the NVIDIA GPU driver to a new location. Bug 1482562 Change-Id: I24293810b9d0f1504fd9be00135e21dad656ccb6 Signed-off-by: Arto Merilainen <amerilainen@nvidia.com> Reviewed-on: http://git-master/r/383722 Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a/fifo_gk20a.c')
-rw-r--r--drivers/gpu/nvgpu/gk20a/fifo_gk20a.c1836
1 files changed, 1836 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
new file mode 100644
index 00000000..5575b995
--- /dev/null
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -0,0 +1,1836 @@
1/*
2 * drivers/video/tegra/host/gk20a/fifo_gk20a.c
3 *
4 * GK20A Graphics FIFO (gr host)
5 *
6 * Copyright (c) 2011-2014, NVIDIA CORPORATION. All rights reserved.
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License,
10 * version 2, as published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * more details.
16 *
17 * You should have received a copy of the GNU General Public License along with
18 * this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20 */
21#include <linux/delay.h>
22#include <linux/slab.h>
23#include <linux/scatterlist.h>
24#include <trace/events/gk20a.h>
25#include <linux/dma-mapping.h>
26#include <linux/nvhost.h>
27
28#include "gk20a.h"
29#include "debug_gk20a.h"
30#include "hw_fifo_gk20a.h"
31#include "hw_pbdma_gk20a.h"
32#include "hw_ccsr_gk20a.h"
33#include "hw_ram_gk20a.h"
34#include "hw_proj_gk20a.h"
35#include "hw_top_gk20a.h"
36#include "hw_mc_gk20a.h"
37#include "hw_gr_gk20a.h"
38
39static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
40 u32 hw_chid, bool add,
41 bool wait_for_finish);
42static void gk20a_fifo_handle_mmu_fault_thread(struct work_struct *work);
43
44/*
45 * Link engine IDs to MMU IDs and vice versa.
46 */
47
48static inline u32 gk20a_engine_id_to_mmu_id(u32 engine_id)
49{
50 switch (engine_id) {
51 case ENGINE_GR_GK20A:
52 return 0x00;
53 case ENGINE_CE2_GK20A:
54 return 0x1b;
55 default:
56 return ~0;
57 }
58}
59
60static inline u32 gk20a_mmu_id_to_engine_id(u32 engine_id)
61{
62 switch (engine_id) {
63 case 0x00:
64 return ENGINE_GR_GK20A;
65 case 0x1b:
66 return ENGINE_CE2_GK20A;
67 default:
68 return ~0;
69 }
70}
71
72
73static int init_engine_info(struct fifo_gk20a *f)
74{
75 struct gk20a *g = f->g;
76 struct device *d = dev_from_gk20a(g);
77 struct fifo_engine_info_gk20a *gr_info;
78 const u32 gr_sw_id = ENGINE_GR_GK20A;
79 u32 i;
80 u32 max_info_entries = top_device_info__size_1_v();
81
82 gk20a_dbg_fn("");
83
84 /* all we really care about finding is the graphics entry */
85 /* especially early on in sim it probably thinks it has more */
86 f->num_engines = 1;
87
88 gr_info = f->engine_info + gr_sw_id;
89
90 gr_info->sw_id = gr_sw_id;
91 gr_info->name = "gr";
92 gr_info->dev_info_id = top_device_info_type_enum_graphics_v();
93 gr_info->mmu_fault_id = fifo_intr_mmu_fault_eng_id_graphics_v();
94 gr_info->runlist_id = ~0;
95 gr_info->pbdma_id = ~0;
96 gr_info->engine_id = ~0;
97
98 for (i = 0; i < max_info_entries; i++) {
99 u32 table_entry = gk20a_readl(f->g, top_device_info_r(i));
100 u32 entry = top_device_info_entry_v(table_entry);
101 u32 engine_enum = top_device_info_type_enum_v(table_entry);
102 u32 table_entry2 = 0;
103
104 if (entry == top_device_info_entry_not_valid_v())
105 continue;
106
107 if (top_device_info_chain_v(table_entry) ==
108 top_device_info_chain_enable_v()) {
109
110 table_entry2 = gk20a_readl(f->g,
111 top_device_info_r(++i));
112
113 engine_enum = top_device_info_type_enum_v(table_entry2);
114 }
115
116 /* we only care about GR engine here */
117 if (entry == top_device_info_entry_enum_v() &&
118 engine_enum == gr_info->dev_info_id) {
119 int pbdma_id;
120 u32 runlist_bit;
121
122 gr_info->runlist_id =
123 top_device_info_runlist_enum_v(table_entry);
124 gk20a_dbg_info("gr info: runlist_id %d", gr_info->runlist_id);
125
126 gr_info->engine_id =
127 top_device_info_engine_enum_v(table_entry);
128 gk20a_dbg_info("gr info: engine_id %d", gr_info->engine_id);
129
130 runlist_bit = 1 << gr_info->runlist_id;
131
132 for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) {
133 gk20a_dbg_info("gr info: pbdma_map[%d]=%d",
134 pbdma_id, f->pbdma_map[pbdma_id]);
135 if (f->pbdma_map[pbdma_id] & runlist_bit)
136 break;
137 }
138
139 if (pbdma_id == f->num_pbdma) {
140 gk20a_err(d, "busted pbmda map");
141 return -EINVAL;
142 }
143 gr_info->pbdma_id = pbdma_id;
144
145 break;
146 }
147 }
148
149 if (gr_info->runlist_id == ~0) {
150 gk20a_err(d, "busted device info");
151 return -EINVAL;
152 }
153
154 return 0;
155}
156
157void gk20a_remove_fifo_support(struct fifo_gk20a *f)
158{
159 struct gk20a *g = f->g;
160 struct device *d = dev_from_gk20a(g);
161 struct fifo_engine_info_gk20a *engine_info;
162 struct fifo_runlist_info_gk20a *runlist;
163 u32 runlist_id;
164 u32 i;
165
166 gk20a_dbg_fn("");
167
168 if (f->channel) {
169 int c;
170 for (c = 0; c < f->num_channels; c++) {
171 if (f->channel[c].remove_support)
172 f->channel[c].remove_support(f->channel+c);
173 }
174 kfree(f->channel);
175 }
176 if (f->userd.gpu_va)
177 gk20a_gmmu_unmap(&g->mm.bar1.vm,
178 f->userd.gpu_va,
179 f->userd.size,
180 gk20a_mem_flag_none);
181
182 if (f->userd.sgt)
183 gk20a_free_sgtable(&f->userd.sgt);
184
185 if (f->userd.cpuva)
186 dma_free_coherent(d,
187 f->userd_total_size,
188 f->userd.cpuva,
189 f->userd.iova);
190 f->userd.cpuva = NULL;
191 f->userd.iova = 0;
192
193 engine_info = f->engine_info + ENGINE_GR_GK20A;
194 runlist_id = engine_info->runlist_id;
195 runlist = &f->runlist_info[runlist_id];
196
197 for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
198 if (runlist->mem[i].cpuva)
199 dma_free_coherent(d,
200 runlist->mem[i].size,
201 runlist->mem[i].cpuva,
202 runlist->mem[i].iova);
203 runlist->mem[i].cpuva = NULL;
204 runlist->mem[i].iova = 0;
205 }
206
207 kfree(runlist->active_channels);
208
209 kfree(f->runlist_info);
210 kfree(f->pbdma_map);
211 kfree(f->engine_info);
212}
213
214/* reads info from hardware and fills in pbmda exception info record */
215static inline void get_exception_pbdma_info(
216 struct gk20a *g,
217 struct fifo_engine_info_gk20a *eng_info)
218{
219 struct fifo_pbdma_exception_info_gk20a *e =
220 &eng_info->pbdma_exception_info;
221
222 u32 pbdma_status_r = e->status_r = gk20a_readl(g,
223 fifo_pbdma_status_r(eng_info->pbdma_id));
224 e->id = fifo_pbdma_status_id_v(pbdma_status_r); /* vs. id_hw_v()? */
225 e->id_is_chid = fifo_pbdma_status_id_type_v(pbdma_status_r) ==
226 fifo_pbdma_status_id_type_chid_v();
227 e->chan_status_v = fifo_pbdma_status_chan_status_v(pbdma_status_r);
228 e->next_id_is_chid =
229 fifo_pbdma_status_next_id_type_v(pbdma_status_r) ==
230 fifo_pbdma_status_next_id_type_chid_v();
231 e->next_id = fifo_pbdma_status_next_id_v(pbdma_status_r);
232 e->chsw_in_progress =
233 fifo_pbdma_status_chsw_v(pbdma_status_r) ==
234 fifo_pbdma_status_chsw_in_progress_v();
235}
236
237static void fifo_pbdma_exception_status(struct gk20a *g,
238 struct fifo_engine_info_gk20a *eng_info)
239{
240 struct fifo_pbdma_exception_info_gk20a *e;
241 get_exception_pbdma_info(g, eng_info);
242 e = &eng_info->pbdma_exception_info;
243
244 gk20a_dbg_fn("pbdma_id %d, "
245 "id_type %s, id %d, chan_status %d, "
246 "next_id_type %s, next_id %d, "
247 "chsw_in_progress %d",
248 eng_info->pbdma_id,
249 e->id_is_chid ? "chid" : "tsgid", e->id, e->chan_status_v,
250 e->next_id_is_chid ? "chid" : "tsgid", e->next_id,
251 e->chsw_in_progress);
252}
253
254/* reads info from hardware and fills in pbmda exception info record */
255static inline void get_exception_engine_info(
256 struct gk20a *g,
257 struct fifo_engine_info_gk20a *eng_info)
258{
259 struct fifo_engine_exception_info_gk20a *e =
260 &eng_info->engine_exception_info;
261 u32 engine_status_r = e->status_r =
262 gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
263 e->id = fifo_engine_status_id_v(engine_status_r); /* vs. id_hw_v()? */
264 e->id_is_chid = fifo_engine_status_id_type_v(engine_status_r) ==
265 fifo_engine_status_id_type_chid_v();
266 e->ctx_status_v = fifo_engine_status_ctx_status_v(engine_status_r);
267 e->faulted =
268 fifo_engine_status_faulted_v(engine_status_r) ==
269 fifo_engine_status_faulted_true_v();
270 e->idle =
271 fifo_engine_status_engine_v(engine_status_r) ==
272 fifo_engine_status_engine_idle_v();
273 e->ctxsw_in_progress =
274 fifo_engine_status_ctxsw_v(engine_status_r) ==
275 fifo_engine_status_ctxsw_in_progress_v();
276}
277
278static void fifo_engine_exception_status(struct gk20a *g,
279 struct fifo_engine_info_gk20a *eng_info)
280{
281 struct fifo_engine_exception_info_gk20a *e;
282 get_exception_engine_info(g, eng_info);
283 e = &eng_info->engine_exception_info;
284
285 gk20a_dbg_fn("engine_id %d, id_type %s, id %d, ctx_status %d, "
286 "faulted %d, idle %d, ctxsw_in_progress %d, ",
287 eng_info->engine_id, e->id_is_chid ? "chid" : "tsgid",
288 e->id, e->ctx_status_v,
289 e->faulted, e->idle, e->ctxsw_in_progress);
290}
291
292static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
293{
294 struct fifo_engine_info_gk20a *engine_info;
295 struct fifo_runlist_info_gk20a *runlist;
296 struct device *d = dev_from_gk20a(g);
297 u32 runlist_id;
298 u32 i;
299 u64 runlist_size;
300
301 gk20a_dbg_fn("");
302
303 f->max_runlists = fifo_eng_runlist_base__size_1_v();
304 f->runlist_info = kzalloc(sizeof(struct fifo_runlist_info_gk20a) *
305 f->max_runlists, GFP_KERNEL);
306 if (!f->runlist_info)
307 goto clean_up;
308
309 engine_info = f->engine_info + ENGINE_GR_GK20A;
310 runlist_id = engine_info->runlist_id;
311 runlist = &f->runlist_info[runlist_id];
312
313 runlist->active_channels =
314 kzalloc(DIV_ROUND_UP(f->num_channels, BITS_PER_BYTE),
315 GFP_KERNEL);
316 if (!runlist->active_channels)
317 goto clean_up_runlist_info;
318
319 runlist_size = ram_rl_entry_size_v() * f->num_channels;
320 for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
321 dma_addr_t iova;
322
323 runlist->mem[i].cpuva =
324 dma_alloc_coherent(d,
325 runlist_size,
326 &iova,
327 GFP_KERNEL);
328 if (!runlist->mem[i].cpuva) {
329 dev_err(d, "memory allocation failed\n");
330 goto clean_up_runlist;
331 }
332 runlist->mem[i].iova = iova;
333 runlist->mem[i].size = runlist_size;
334 }
335 mutex_init(&runlist->mutex);
336 init_waitqueue_head(&runlist->runlist_wq);
337
338 /* None of buffers is pinned if this value doesn't change.
339 Otherwise, one of them (cur_buffer) must have been pinned. */
340 runlist->cur_buffer = MAX_RUNLIST_BUFFERS;
341
342 gk20a_dbg_fn("done");
343 return 0;
344
345clean_up_runlist:
346 for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
347 if (runlist->mem[i].cpuva)
348 dma_free_coherent(d,
349 runlist->mem[i].size,
350 runlist->mem[i].cpuva,
351 runlist->mem[i].iova);
352 runlist->mem[i].cpuva = NULL;
353 runlist->mem[i].iova = 0;
354 }
355
356 kfree(runlist->active_channels);
357 runlist->active_channels = NULL;
358
359clean_up_runlist_info:
360 kfree(f->runlist_info);
361 f->runlist_info = NULL;
362
363clean_up:
364 gk20a_dbg_fn("fail");
365 return -ENOMEM;
366}
367
368#define GRFIFO_TIMEOUT_CHECK_PERIOD_US 100000
369
370int gk20a_init_fifo_reset_enable_hw(struct gk20a *g)
371{
372 u32 intr_stall;
373 u32 mask;
374 u32 timeout;
375 int i;
376
377 gk20a_dbg_fn("");
378 /* enable pmc pfifo */
379 gk20a_reset(g, mc_enable_pfifo_enabled_f()
380 | mc_enable_ce2_enabled_f());
381
382 /* enable pbdma */
383 mask = 0;
384 for (i = 0; i < proj_host_num_pbdma_v(); ++i)
385 mask |= mc_enable_pb_sel_f(mc_enable_pb_0_enabled_v(), i);
386 gk20a_writel(g, mc_enable_pb_r(), mask);
387
388 /* enable pfifo interrupt */
389 gk20a_writel(g, fifo_intr_0_r(), 0xFFFFFFFF);
390 gk20a_writel(g, fifo_intr_en_0_r(), 0x7FFFFFFF);
391 gk20a_writel(g, fifo_intr_en_1_r(), 0x80000000);
392
393 /* enable pbdma interrupt */
394 mask = 0;
395 for (i = 0; i < proj_host_num_pbdma_v(); i++) {
396 intr_stall = gk20a_readl(g, pbdma_intr_stall_r(i));
397 intr_stall &= ~pbdma_intr_stall_lbreq_enabled_f();
398 gk20a_writel(g, pbdma_intr_stall_r(i), intr_stall);
399 gk20a_writel(g, pbdma_intr_0_r(i), 0xFFFFFFFF);
400 gk20a_writel(g, pbdma_intr_en_0_r(i),
401 (~0) & ~pbdma_intr_en_0_lbreq_enabled_f());
402 gk20a_writel(g, pbdma_intr_1_r(i), 0xFFFFFFFF);
403 gk20a_writel(g, pbdma_intr_en_1_r(i), 0xFFFFFFFF);
404 }
405
406 /* TBD: apply overrides */
407
408 /* TBD: BLCG prod */
409
410 /* reset runlist interrupts */
411 gk20a_writel(g, fifo_intr_runlist_r(), ~0);
412
413 /* TBD: do we need those? */
414 timeout = gk20a_readl(g, fifo_fb_timeout_r());
415 timeout = set_field(timeout, fifo_fb_timeout_period_m(),
416 fifo_fb_timeout_period_max_f());
417 gk20a_writel(g, fifo_fb_timeout_r(), timeout);
418
419 if (tegra_platform_is_silicon()) {
420 timeout = gk20a_readl(g, fifo_pb_timeout_r());
421 timeout &= ~fifo_pb_timeout_detection_enabled_f();
422 gk20a_writel(g, fifo_pb_timeout_r(), timeout);
423 }
424
425 timeout = GRFIFO_TIMEOUT_CHECK_PERIOD_US |
426 fifo_eng_timeout_detection_enabled_f();
427 gk20a_writel(g, fifo_eng_timeout_r(), timeout);
428
429 gk20a_dbg_fn("done");
430
431 return 0;
432}
433
434static void gk20a_init_fifo_pbdma_intr_descs(struct fifo_gk20a *f)
435{
436 /* These are all errors which indicate something really wrong
437 * going on in the device. */
438 f->intr.pbdma.device_fatal_0 =
439 pbdma_intr_0_memreq_pending_f() |
440 pbdma_intr_0_memack_timeout_pending_f() |
441 pbdma_intr_0_memack_extra_pending_f() |
442 pbdma_intr_0_memdat_timeout_pending_f() |
443 pbdma_intr_0_memdat_extra_pending_f() |
444 pbdma_intr_0_memflush_pending_f() |
445 pbdma_intr_0_memop_pending_f() |
446 pbdma_intr_0_lbconnect_pending_f() |
447 pbdma_intr_0_lbreq_pending_f() |
448 pbdma_intr_0_lback_timeout_pending_f() |
449 pbdma_intr_0_lback_extra_pending_f() |
450 pbdma_intr_0_lbdat_timeout_pending_f() |
451 pbdma_intr_0_lbdat_extra_pending_f() |
452 pbdma_intr_0_xbarconnect_pending_f() |
453 pbdma_intr_0_pri_pending_f();
454
455 /* These are data parsing, framing errors or others which can be
456 * recovered from with intervention... or just resetting the
457 * channel. */
458 f->intr.pbdma.channel_fatal_0 =
459 pbdma_intr_0_gpfifo_pending_f() |
460 pbdma_intr_0_gpptr_pending_f() |
461 pbdma_intr_0_gpentry_pending_f() |
462 pbdma_intr_0_gpcrc_pending_f() |
463 pbdma_intr_0_pbptr_pending_f() |
464 pbdma_intr_0_pbentry_pending_f() |
465 pbdma_intr_0_pbcrc_pending_f() |
466 pbdma_intr_0_method_pending_f() |
467 pbdma_intr_0_methodcrc_pending_f() |
468 pbdma_intr_0_pbseg_pending_f() |
469 pbdma_intr_0_signature_pending_f();
470
471 /* Can be used for sw-methods, or represents
472 * a recoverable timeout. */
473 f->intr.pbdma.restartable_0 =
474 pbdma_intr_0_device_pending_f() |
475 pbdma_intr_0_acquire_pending_f();
476}
477
478static int gk20a_init_fifo_setup_sw(struct gk20a *g)
479{
480 struct fifo_gk20a *f = &g->fifo;
481 struct device *d = dev_from_gk20a(g);
482 int chid, i, err = 0;
483 dma_addr_t iova;
484
485 gk20a_dbg_fn("");
486
487 if (f->sw_ready) {
488 gk20a_dbg_fn("skip init");
489 return 0;
490 }
491
492 f->g = g;
493
494 INIT_WORK(&f->fault_restore_thread,
495 gk20a_fifo_handle_mmu_fault_thread);
496 mutex_init(&f->intr.isr.mutex);
497 gk20a_init_fifo_pbdma_intr_descs(f); /* just filling in data/tables */
498
499 f->num_channels = ccsr_channel__size_1_v();
500 f->num_pbdma = proj_host_num_pbdma_v();
501 f->max_engines = ENGINE_INVAL_GK20A;
502
503 f->userd_entry_size = 1 << ram_userd_base_shift_v();
504 f->userd_total_size = f->userd_entry_size * f->num_channels;
505
506 f->userd.cpuva = dma_alloc_coherent(d,
507 f->userd_total_size,
508 &iova,
509 GFP_KERNEL);
510 if (!f->userd.cpuva) {
511 dev_err(d, "memory allocation failed\n");
512 goto clean_up;
513 }
514
515 f->userd.iova = iova;
516 err = gk20a_get_sgtable(d, &f->userd.sgt,
517 f->userd.cpuva, f->userd.iova,
518 f->userd_total_size);
519 if (err) {
520 dev_err(d, "failed to create sg table\n");
521 goto clean_up;
522 }
523
524 /* bar1 va */
525 f->userd.gpu_va = gk20a_gmmu_map(&g->mm.bar1.vm,
526 &f->userd.sgt,
527 f->userd_total_size,
528 0, /* flags */
529 gk20a_mem_flag_none);
530 if (!f->userd.gpu_va) {
531 dev_err(d, "gmmu mapping failed\n");
532 goto clean_up;
533 }
534
535 gk20a_dbg(gpu_dbg_map, "userd bar1 va = 0x%llx", f->userd.gpu_va);
536
537 f->userd.size = f->userd_total_size;
538
539 f->channel = kzalloc(f->num_channels * sizeof(*f->channel),
540 GFP_KERNEL);
541 f->pbdma_map = kzalloc(f->num_pbdma * sizeof(*f->pbdma_map),
542 GFP_KERNEL);
543 f->engine_info = kzalloc(f->max_engines * sizeof(*f->engine_info),
544 GFP_KERNEL);
545
546 if (!(f->channel && f->pbdma_map && f->engine_info)) {
547 err = -ENOMEM;
548 goto clean_up;
549 }
550
551 /* pbdma map needs to be in place before calling engine info init */
552 for (i = 0; i < f->num_pbdma; ++i)
553 f->pbdma_map[i] = gk20a_readl(g, fifo_pbdma_map_r(i));
554
555 init_engine_info(f);
556
557 init_runlist(g, f);
558
559 for (chid = 0; chid < f->num_channels; chid++) {
560 f->channel[chid].userd_cpu_va =
561 f->userd.cpuva + chid * f->userd_entry_size;
562 f->channel[chid].userd_iova =
563 NV_MC_SMMU_VADDR_TRANSLATE(f->userd.iova)
564 + chid * f->userd_entry_size;
565 f->channel[chid].userd_gpu_va =
566 f->userd.gpu_va + chid * f->userd_entry_size;
567
568 gk20a_init_channel_support(g, chid);
569 }
570 mutex_init(&f->ch_inuse_mutex);
571
572 f->remove_support = gk20a_remove_fifo_support;
573
574 f->deferred_reset_pending = false;
575 mutex_init(&f->deferred_reset_mutex);
576
577 f->sw_ready = true;
578
579 gk20a_dbg_fn("done");
580 return 0;
581
582clean_up:
583 gk20a_dbg_fn("fail");
584 if (f->userd.gpu_va)
585 gk20a_gmmu_unmap(&g->mm.bar1.vm,
586 f->userd.gpu_va,
587 f->userd.size,
588 gk20a_mem_flag_none);
589 if (f->userd.sgt)
590 gk20a_free_sgtable(&f->userd.sgt);
591 if (f->userd.cpuva)
592 dma_free_coherent(d,
593 f->userd_total_size,
594 f->userd.cpuva,
595 f->userd.iova);
596 f->userd.cpuva = NULL;
597 f->userd.iova = 0;
598
599 memset(&f->userd, 0, sizeof(struct userd_desc));
600
601 kfree(f->channel);
602 f->channel = NULL;
603 kfree(f->pbdma_map);
604 f->pbdma_map = NULL;
605 kfree(f->engine_info);
606 f->engine_info = NULL;
607
608 return err;
609}
610
611static void gk20a_fifo_handle_runlist_event(struct gk20a *g)
612{
613 struct fifo_gk20a *f = &g->fifo;
614 struct fifo_runlist_info_gk20a *runlist;
615 unsigned long runlist_event;
616 u32 runlist_id;
617
618 runlist_event = gk20a_readl(g, fifo_intr_runlist_r());
619 gk20a_writel(g, fifo_intr_runlist_r(), runlist_event);
620
621 for_each_set_bit(runlist_id, &runlist_event, f->max_runlists) {
622 runlist = &f->runlist_info[runlist_id];
623 wake_up(&runlist->runlist_wq);
624 }
625
626}
627
628static int gk20a_init_fifo_setup_hw(struct gk20a *g)
629{
630 struct fifo_gk20a *f = &g->fifo;
631
632 gk20a_dbg_fn("");
633
634 /* test write, read through bar1 @ userd region before
635 * turning on the snooping */
636 {
637 struct fifo_gk20a *f = &g->fifo;
638 u32 v, v1 = 0x33, v2 = 0x55;
639
640 u32 bar1_vaddr = f->userd.gpu_va;
641 volatile u32 *cpu_vaddr = f->userd.cpuva;
642
643 gk20a_dbg_info("test bar1 @ vaddr 0x%x",
644 bar1_vaddr);
645
646 v = gk20a_bar1_readl(g, bar1_vaddr);
647
648 *cpu_vaddr = v1;
649 smp_mb();
650
651 if (v1 != gk20a_bar1_readl(g, bar1_vaddr)) {
652 gk20a_err(dev_from_gk20a(g), "bar1 broken @ gk20a!");
653 return -EINVAL;
654 }
655
656 gk20a_bar1_writel(g, bar1_vaddr, v2);
657
658 if (v2 != gk20a_bar1_readl(g, bar1_vaddr)) {
659 gk20a_err(dev_from_gk20a(g), "bar1 broken @ gk20a!");
660 return -EINVAL;
661 }
662
663 /* is it visible to the cpu? */
664 if (*cpu_vaddr != v2) {
665 gk20a_err(dev_from_gk20a(g),
666 "cpu didn't see bar1 write @ %p!",
667 cpu_vaddr);
668 }
669
670 /* put it back */
671 gk20a_bar1_writel(g, bar1_vaddr, v);
672 }
673
674 /*XXX all manner of flushes and caching worries, etc */
675
676 /* set the base for the userd region now */
677 gk20a_writel(g, fifo_bar1_base_r(),
678 fifo_bar1_base_ptr_f(f->userd.gpu_va >> 12) |
679 fifo_bar1_base_valid_true_f());
680
681 gk20a_dbg_fn("done");
682
683 return 0;
684}
685
686int gk20a_init_fifo_support(struct gk20a *g)
687{
688 u32 err;
689
690 err = gk20a_init_fifo_setup_sw(g);
691 if (err)
692 return err;
693
694 err = gk20a_init_fifo_setup_hw(g);
695 if (err)
696 return err;
697
698 return err;
699}
700
701static struct channel_gk20a *
702channel_from_inst_ptr(struct fifo_gk20a *f, u64 inst_ptr)
703{
704 int ci;
705 if (unlikely(!f->channel))
706 return NULL;
707 for (ci = 0; ci < f->num_channels; ci++) {
708 struct channel_gk20a *c = f->channel+ci;
709 if (c->inst_block.cpuva &&
710 (inst_ptr == c->inst_block.cpu_pa))
711 return f->channel+ci;
712 }
713 return NULL;
714}
715
716/* fault info/descriptions.
717 * tbd: move to setup
718 * */
719static const char * const fault_type_descs[] = {
720 "pde", /*fifo_intr_mmu_fault_info_type_pde_v() == 0 */
721 "pde size",
722 "pte",
723 "va limit viol",
724 "unbound inst",
725 "priv viol",
726 "ro viol",
727 "wo viol",
728 "pitch mask",
729 "work creation",
730 "bad aperture",
731 "compression failure",
732 "bad kind",
733 "region viol",
734 "dual ptes",
735 "poisoned",
736};
737/* engine descriptions */
738static const char * const engine_subid_descs[] = {
739 "gpc",
740 "hub",
741};
742
743static const char * const hub_client_descs[] = {
744 "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu",
745 "host cpu nb", "iso", "mmu", "mspdec", "msppp", "msvld",
746 "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc",
747 "scc nb", "sec", "ssync", "gr copy", "ce2", "xv", "mmu nb",
748 "msenc", "d falcon", "sked", "a falcon", "n/a",
749};
750
751static const char * const gpc_client_descs[] = {
752 "l1 0", "t1 0", "pe 0",
753 "l1 1", "t1 1", "pe 1",
754 "l1 2", "t1 2", "pe 2",
755 "l1 3", "t1 3", "pe 3",
756 "rast", "gcc", "gpccs",
757 "prop 0", "prop 1", "prop 2", "prop 3",
758 "l1 4", "t1 4", "pe 4",
759 "l1 5", "t1 5", "pe 5",
760 "l1 6", "t1 6", "pe 6",
761 "l1 7", "t1 7", "pe 7",
762 "gpm",
763 "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3",
764 "rgg utlb",
765};
766
767/* reads info from hardware and fills in mmu fault info record */
768static inline void get_exception_mmu_fault_info(
769 struct gk20a *g, u32 engine_id,
770 struct fifo_mmu_fault_info_gk20a *f)
771{
772 u32 fault_info_v;
773
774 gk20a_dbg_fn("engine_id %d", engine_id);
775
776 memset(f, 0, sizeof(*f));
777
778 f->fault_info_v = fault_info_v = gk20a_readl(g,
779 fifo_intr_mmu_fault_info_r(engine_id));
780 f->fault_type_v =
781 fifo_intr_mmu_fault_info_type_v(fault_info_v);
782 f->engine_subid_v =
783 fifo_intr_mmu_fault_info_engine_subid_v(fault_info_v);
784 f->client_v = fifo_intr_mmu_fault_info_client_v(fault_info_v);
785
786 BUG_ON(f->fault_type_v >= ARRAY_SIZE(fault_type_descs));
787 f->fault_type_desc = fault_type_descs[f->fault_type_v];
788
789 BUG_ON(f->engine_subid_v >= ARRAY_SIZE(engine_subid_descs));
790 f->engine_subid_desc = engine_subid_descs[f->engine_subid_v];
791
792 if (f->engine_subid_v ==
793 fifo_intr_mmu_fault_info_engine_subid_hub_v()) {
794
795 BUG_ON(f->client_v >= ARRAY_SIZE(hub_client_descs));
796 f->client_desc = hub_client_descs[f->client_v];
797 } else if (f->engine_subid_v ==
798 fifo_intr_mmu_fault_info_engine_subid_gpc_v()) {
799 BUG_ON(f->client_v >= ARRAY_SIZE(gpc_client_descs));
800 f->client_desc = gpc_client_descs[f->client_v];
801 } else {
802 BUG_ON(1);
803 }
804
805 f->fault_hi_v = gk20a_readl(g, fifo_intr_mmu_fault_hi_r(engine_id));
806 f->fault_lo_v = gk20a_readl(g, fifo_intr_mmu_fault_lo_r(engine_id));
807 /* note:ignoring aperture on gk20a... */
808 f->inst_ptr = fifo_intr_mmu_fault_inst_ptr_v(
809 gk20a_readl(g, fifo_intr_mmu_fault_inst_r(engine_id)));
810 /* note: inst_ptr is a 40b phys addr. */
811 f->inst_ptr <<= fifo_intr_mmu_fault_inst_ptr_align_shift_v();
812}
813
814static void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
815{
816 gk20a_dbg_fn("");
817
818 if (engine_id == top_device_info_type_enum_graphics_v()) {
819 /* resetting engine using mc_enable_r() is not enough,
820 * we do full init sequence */
821 gk20a_gr_reset(g);
822 }
823 if (engine_id == top_device_info_type_enum_copy0_v())
824 gk20a_reset(g, mc_enable_ce2_m());
825}
826
827static void gk20a_fifo_handle_mmu_fault_thread(struct work_struct *work)
828{
829 struct fifo_gk20a *f = container_of(work, struct fifo_gk20a,
830 fault_restore_thread);
831 struct gk20a *g = f->g;
832 int i;
833
834 /* Reinitialise FECS and GR */
835 gk20a_init_pmu_setup_hw2(g);
836
837 /* It is safe to enable ELPG again. */
838 gk20a_pmu_enable_elpg(g);
839
840 /* Restore the runlist */
841 for (i = 0; i < g->fifo.max_runlists; i++)
842 gk20a_fifo_update_runlist_locked(g, i, ~0, true, true);
843
844 /* unlock all runlists */
845 for (i = 0; i < g->fifo.max_runlists; i++)
846 mutex_unlock(&g->fifo.runlist_info[i].mutex);
847
848}
849
850static void gk20a_fifo_handle_chsw_fault(struct gk20a *g)
851{
852 u32 intr;
853
854 intr = gk20a_readl(g, fifo_intr_chsw_error_r());
855 gk20a_err(dev_from_gk20a(g), "chsw: %08x\n", intr);
856 gk20a_fecs_dump_falcon_stats(g);
857 gk20a_writel(g, fifo_intr_chsw_error_r(), intr);
858}
859
860static void gk20a_fifo_handle_dropped_mmu_fault(struct gk20a *g)
861{
862 struct device *dev = dev_from_gk20a(g);
863 u32 fault_id = gk20a_readl(g, fifo_intr_mmu_fault_id_r());
864 gk20a_err(dev, "dropped mmu fault (0x%08x)", fault_id);
865}
866
867static bool gk20a_fifo_should_defer_engine_reset(struct gk20a *g, u32 engine_id,
868 struct fifo_mmu_fault_info_gk20a *f, bool fake_fault)
869{
870 /* channel recovery is only deferred if an sm debugger
871 is attached and has MMU debug mode is enabled */
872 if (!gk20a_gr_sm_debugger_attached(g) ||
873 !gk20a_mm_mmu_debug_mode_enabled(g))
874 return false;
875
876 /* if this fault is fake (due to RC recovery), don't defer recovery */
877 if (fake_fault)
878 return false;
879
880 if (engine_id != ENGINE_GR_GK20A ||
881 f->engine_subid_v != fifo_intr_mmu_fault_info_engine_subid_gpc_v())
882 return false;
883
884 return true;
885}
886
887void fifo_gk20a_finish_mmu_fault_handling(struct gk20a *g,
888 unsigned long fault_id) {
889 u32 engine_mmu_id;
890 int i;
891
892 /* reset engines */
893 for_each_set_bit(engine_mmu_id, &fault_id, 32) {
894 u32 engine_id = gk20a_mmu_id_to_engine_id(engine_mmu_id);
895 if (engine_id != ~0)
896 gk20a_fifo_reset_engine(g, engine_id);
897 }
898
899 /* CLEAR the runlists. Do not wait for runlist to start as
900 * some engines may not be available right now */
901 for (i = 0; i < g->fifo.max_runlists; i++)
902 gk20a_fifo_update_runlist_locked(g, i, ~0, false, false);
903
904 /* clear interrupt */
905 gk20a_writel(g, fifo_intr_mmu_fault_id_r(), fault_id);
906
907 /* resume scheduler */
908 gk20a_writel(g, fifo_error_sched_disable_r(),
909 gk20a_readl(g, fifo_error_sched_disable_r()));
910
911 /* Spawn a work to enable PMU and restore runlists */
912 schedule_work(&g->fifo.fault_restore_thread);
913}
914
915static bool gk20a_fifo_set_ctx_mmu_error(struct gk20a *g,
916 struct channel_gk20a *ch) {
917 bool verbose = true;
918 if (!ch)
919 return verbose;
920
921 gk20a_err(dev_from_gk20a(g),
922 "channel %d generated a mmu fault",
923 ch->hw_chid);
924 if (ch->error_notifier) {
925 u32 err = ch->error_notifier->info32;
926 if (ch->error_notifier->status == 0xffff) {
927 /* If error code is already set, this mmu fault
928 * was triggered as part of recovery from other
929 * error condition.
930 * Don't overwrite error flag. */
931 /* Fifo timeout debug spew is controlled by user */
932 if (err == NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT)
933 verbose = ch->timeout_debug_dump;
934 } else {
935 gk20a_set_error_notifier(ch,
936 NVHOST_CHANNEL_FIFO_ERROR_MMU_ERR_FLT);
937 }
938 }
939 /* mark channel as faulted */
940 ch->has_timedout = true;
941 wmb();
942 /* unblock pending waits */
943 wake_up(&ch->semaphore_wq);
944 wake_up(&ch->notifier_wq);
945 wake_up(&ch->submit_wq);
946 return verbose;
947}
948
949
950static bool gk20a_fifo_handle_mmu_fault(struct gk20a *g)
951{
952 bool fake_fault;
953 unsigned long fault_id;
954 unsigned long engine_mmu_id;
955 int i;
956 bool verbose = true;
957 gk20a_dbg_fn("");
958
959 g->fifo.deferred_reset_pending = false;
960
961 /* Disable ELPG */
962 gk20a_pmu_disable_elpg(g);
963
964 /* If we have recovery in progress, MMU fault id is invalid */
965 if (g->fifo.mmu_fault_engines) {
966 fault_id = g->fifo.mmu_fault_engines;
967 g->fifo.mmu_fault_engines = 0;
968 fake_fault = true;
969 } else {
970 fault_id = gk20a_readl(g, fifo_intr_mmu_fault_id_r());
971 fake_fault = false;
972 gk20a_debug_dump(g->dev);
973 }
974
975 /* lock all runlists. Note that locks are are released in
976 * gk20a_fifo_handle_mmu_fault_thread() */
977 for (i = 0; i < g->fifo.max_runlists; i++)
978 mutex_lock(&g->fifo.runlist_info[i].mutex);
979
980 /* go through all faulted engines */
981 for_each_set_bit(engine_mmu_id, &fault_id, 32) {
982 /* bits in fifo_intr_mmu_fault_id_r do not correspond 1:1 to
983 * engines. Convert engine_mmu_id to engine_id */
984 u32 engine_id = gk20a_mmu_id_to_engine_id(engine_mmu_id);
985 struct fifo_runlist_info_gk20a *runlist = g->fifo.runlist_info;
986 struct fifo_mmu_fault_info_gk20a f;
987 struct channel_gk20a *ch = NULL;
988
989 get_exception_mmu_fault_info(g, engine_mmu_id, &f);
990 trace_gk20a_mmu_fault(f.fault_hi_v,
991 f.fault_lo_v,
992 f.fault_info_v,
993 f.inst_ptr,
994 engine_id,
995 f.engine_subid_desc,
996 f.client_desc,
997 f.fault_type_desc);
998 gk20a_err(dev_from_gk20a(g), "mmu fault on engine %d, "
999 "engine subid %d (%s), client %d (%s), "
1000 "addr 0x%08x:0x%08x, type %d (%s), info 0x%08x,"
1001 "inst_ptr 0x%llx\n",
1002 engine_id,
1003 f.engine_subid_v, f.engine_subid_desc,
1004 f.client_v, f.client_desc,
1005 f.fault_hi_v, f.fault_lo_v,
1006 f.fault_type_v, f.fault_type_desc,
1007 f.fault_info_v, f.inst_ptr);
1008
1009 /* get the channel */
1010 if (fake_fault) {
1011 /* read and parse engine status */
1012 u32 status = gk20a_readl(g,
1013 fifo_engine_status_r(engine_id));
1014 u32 ctx_status =
1015 fifo_engine_status_ctx_status_v(status);
1016 bool type_ch = fifo_pbdma_status_id_type_v(status) ==
1017 fifo_pbdma_status_id_type_chid_v();
1018
1019 /* use next_id if context load is failing */
1020 u32 id = (ctx_status ==
1021 fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1022 fifo_engine_status_next_id_v(status) :
1023 fifo_engine_status_id_v(status);
1024
1025 if (type_ch) {
1026 ch = g->fifo.channel + id;
1027 } else {
1028 gk20a_err(dev_from_gk20a(g), "non-chid type not supported");
1029 WARN_ON(1);
1030 }
1031 } else {
1032 /* read channel based on instruction pointer */
1033 ch = channel_from_inst_ptr(&g->fifo, f.inst_ptr);
1034 }
1035
1036 if (ch) {
1037 if (ch->in_use) {
1038 /* disable the channel from hw and increment
1039 * syncpoints */
1040 gk20a_disable_channel_no_update(ch);
1041
1042 /* remove the channel from runlist */
1043 clear_bit(ch->hw_chid,
1044 runlist->active_channels);
1045 }
1046
1047 /* check if engine reset should be deferred */
1048 if (gk20a_fifo_should_defer_engine_reset(g, engine_id, &f, fake_fault)) {
1049 g->fifo.mmu_fault_engines = fault_id;
1050
1051 /* handled during channel free */
1052 g->fifo.deferred_reset_pending = true;
1053 } else
1054 verbose = gk20a_fifo_set_ctx_mmu_error(g, ch);
1055
1056 } else if (f.inst_ptr ==
1057 g->mm.bar1.inst_block.cpu_pa) {
1058 gk20a_err(dev_from_gk20a(g), "mmu fault from bar1");
1059 } else if (f.inst_ptr ==
1060 g->mm.pmu.inst_block.cpu_pa) {
1061 gk20a_err(dev_from_gk20a(g), "mmu fault from pmu");
1062 } else
1063 gk20a_err(dev_from_gk20a(g), "couldn't locate channel for mmu fault");
1064 }
1065
1066 if (g->fifo.deferred_reset_pending) {
1067 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, "sm debugger attached,"
1068 " deferring channel recovery to channel free");
1069 /* clear interrupt */
1070 gk20a_writel(g, fifo_intr_mmu_fault_id_r(), fault_id);
1071 return verbose;
1072 }
1073
1074 /* resetting the engines and clearing the runlists is done in
1075 a separate function to allow deferred reset. */
1076 fifo_gk20a_finish_mmu_fault_handling(g, fault_id);
1077 return verbose;
1078}
1079
1080static void gk20a_fifo_get_faulty_channel(struct gk20a *g, int engine_id,
1081 u32 *chid, bool *type_ch)
1082{
1083 u32 status = gk20a_readl(g, fifo_engine_status_r(engine_id));
1084 u32 ctx_status = fifo_engine_status_ctx_status_v(status);
1085
1086 *type_ch = fifo_pbdma_status_id_type_v(status) ==
1087 fifo_pbdma_status_id_type_chid_v();
1088 /* use next_id if context load is failing */
1089 *chid = (ctx_status ==
1090 fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1091 fifo_engine_status_next_id_v(status) :
1092 fifo_engine_status_id_v(status);
1093}
1094
1095void gk20a_fifo_recover(struct gk20a *g, u32 __engine_ids,
1096 bool verbose)
1097{
1098 unsigned long end_jiffies = jiffies +
1099 msecs_to_jiffies(gk20a_get_gr_idle_timeout(g));
1100 unsigned long delay = GR_IDLE_CHECK_DEFAULT;
1101 unsigned long engine_id, i;
1102 unsigned long _engine_ids = __engine_ids;
1103 unsigned long engine_ids = 0;
1104 int ret;
1105
1106 if (verbose)
1107 gk20a_debug_dump(g->dev);
1108
1109 /* store faulted engines in advance */
1110 g->fifo.mmu_fault_engines = 0;
1111 for_each_set_bit(engine_id, &_engine_ids, 32) {
1112 bool ref_type_ch;
1113 int ref_chid;
1114 gk20a_fifo_get_faulty_channel(g, engine_id, &ref_chid,
1115 &ref_type_ch);
1116
1117 /* Reset *all* engines that use the
1118 * same channel as faulty engine */
1119 for (i = 0; i < g->fifo.max_engines; i++) {
1120 bool type_ch;
1121 u32 chid;
1122 gk20a_fifo_get_faulty_channel(g, i, &chid, &type_ch);
1123 if (ref_type_ch == type_ch && ref_chid == chid) {
1124 engine_ids |= BIT(i);
1125 g->fifo.mmu_fault_engines |=
1126 BIT(gk20a_engine_id_to_mmu_id(i));
1127 }
1128 }
1129
1130 }
1131
1132 /* trigger faults for all bad engines */
1133 for_each_set_bit(engine_id, &engine_ids, 32) {
1134 if (engine_id > g->fifo.max_engines) {
1135 WARN_ON(true);
1136 break;
1137 }
1138
1139 gk20a_writel(g, fifo_trigger_mmu_fault_r(engine_id),
1140 fifo_trigger_mmu_fault_id_f(
1141 gk20a_engine_id_to_mmu_id(engine_id)) |
1142 fifo_trigger_mmu_fault_enable_f(1));
1143 }
1144
1145 /* Wait for MMU fault to trigger */
1146 ret = -EBUSY;
1147 do {
1148 if (gk20a_readl(g, fifo_intr_0_r()) &
1149 fifo_intr_0_mmu_fault_pending_f()) {
1150 ret = 0;
1151 break;
1152 }
1153
1154 usleep_range(delay, delay * 2);
1155 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
1156 } while (time_before(jiffies, end_jiffies) ||
1157 !tegra_platform_is_silicon());
1158
1159 if (ret)
1160 gk20a_err(dev_from_gk20a(g), "mmu fault timeout");
1161
1162 /* release mmu fault trigger */
1163 for_each_set_bit(engine_id, &engine_ids, 32)
1164 gk20a_writel(g, fifo_trigger_mmu_fault_r(engine_id), 0);
1165}
1166
1167
1168static bool gk20a_fifo_handle_sched_error(struct gk20a *g)
1169{
1170 u32 sched_error;
1171 u32 engine_id;
1172 int id = -1;
1173 bool non_chid = false;
1174
1175 /* read and reset the scheduler error register */
1176 sched_error = gk20a_readl(g, fifo_intr_sched_error_r());
1177 gk20a_writel(g, fifo_intr_0_r(), fifo_intr_0_sched_error_reset_f());
1178
1179 for (engine_id = 0; engine_id < g->fifo.max_engines; engine_id++) {
1180 u32 status = gk20a_readl(g, fifo_engine_status_r(engine_id));
1181 u32 ctx_status = fifo_engine_status_ctx_status_v(status);
1182 bool failing_engine;
1183
1184 /* we are interested in busy engines */
1185 failing_engine = fifo_engine_status_engine_v(status) ==
1186 fifo_engine_status_engine_busy_v();
1187
1188 /* ..that are doing context switch */
1189 failing_engine = failing_engine &&
1190 (ctx_status ==
1191 fifo_engine_status_ctx_status_ctxsw_switch_v()
1192 || ctx_status ==
1193 fifo_engine_status_ctx_status_ctxsw_save_v()
1194 || ctx_status ==
1195 fifo_engine_status_ctx_status_ctxsw_load_v());
1196
1197 if (failing_engine) {
1198 id = (ctx_status ==
1199 fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1200 fifo_engine_status_next_id_v(status) :
1201 fifo_engine_status_id_v(status);
1202 non_chid = fifo_pbdma_status_id_type_v(status) !=
1203 fifo_pbdma_status_id_type_chid_v();
1204 break;
1205 }
1206 }
1207
1208 /* could not find the engine - should never happen */
1209 if (unlikely(engine_id >= g->fifo.max_engines))
1210 goto err;
1211
1212 if (fifo_intr_sched_error_code_f(sched_error) ==
1213 fifo_intr_sched_error_code_ctxsw_timeout_v()) {
1214 struct fifo_gk20a *f = &g->fifo;
1215 struct channel_gk20a *ch = &f->channel[id];
1216
1217 if (non_chid) {
1218 gk20a_fifo_recover(g, BIT(engine_id), true);
1219 goto err;
1220 }
1221
1222 if (gk20a_channel_update_and_check_timeout(ch,
1223 GRFIFO_TIMEOUT_CHECK_PERIOD_US / 1000)) {
1224 gk20a_set_error_notifier(ch,
1225 NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT);
1226 gk20a_err(dev_from_gk20a(g),
1227 "fifo sched ctxsw timeout error:"
1228 "engine = %u, ch = %d", engine_id, id);
1229 gk20a_fifo_recover(g, BIT(engine_id),
1230 ch->timeout_debug_dump);
1231 } else {
1232 gk20a_warn(dev_from_gk20a(g),
1233 "fifo is waiting for ctx switch for %d ms,"
1234 "ch = %d\n",
1235 ch->timeout_accumulated_ms,
1236 id);
1237 }
1238 return ch->timeout_debug_dump;
1239 }
1240err:
1241 gk20a_err(dev_from_gk20a(g), "fifo sched error : 0x%08x, engine=%u, %s=%d",
1242 sched_error, engine_id, non_chid ? "non-ch" : "ch", id);
1243
1244 return true;
1245}
1246
1247static u32 fifo_error_isr(struct gk20a *g, u32 fifo_intr)
1248{
1249 bool print_channel_reset_log = false, reset_engine = false;
1250 struct device *dev = dev_from_gk20a(g);
1251 u32 handled = 0;
1252
1253 gk20a_dbg_fn("");
1254
1255 if (fifo_intr & fifo_intr_0_pio_error_pending_f()) {
1256 /* pio mode is unused. this shouldn't happen, ever. */
1257 /* should we clear it or just leave it pending? */
1258 gk20a_err(dev, "fifo pio error!\n");
1259 BUG_ON(1);
1260 }
1261
1262 if (fifo_intr & fifo_intr_0_bind_error_pending_f()) {
1263 u32 bind_error = gk20a_readl(g, fifo_intr_bind_error_r());
1264 gk20a_err(dev, "fifo bind error: 0x%08x", bind_error);
1265 print_channel_reset_log = true;
1266 handled |= fifo_intr_0_bind_error_pending_f();
1267 }
1268
1269 if (fifo_intr & fifo_intr_0_sched_error_pending_f()) {
1270 print_channel_reset_log = gk20a_fifo_handle_sched_error(g);
1271 handled |= fifo_intr_0_sched_error_pending_f();
1272 }
1273
1274 if (fifo_intr & fifo_intr_0_chsw_error_pending_f()) {
1275 gk20a_fifo_handle_chsw_fault(g);
1276 handled |= fifo_intr_0_chsw_error_pending_f();
1277 }
1278
1279 if (fifo_intr & fifo_intr_0_mmu_fault_pending_f()) {
1280 print_channel_reset_log = gk20a_fifo_handle_mmu_fault(g);
1281 reset_engine = true;
1282 handled |= fifo_intr_0_mmu_fault_pending_f();
1283 }
1284
1285 if (fifo_intr & fifo_intr_0_dropped_mmu_fault_pending_f()) {
1286 gk20a_fifo_handle_dropped_mmu_fault(g);
1287 handled |= fifo_intr_0_dropped_mmu_fault_pending_f();
1288 }
1289
1290 print_channel_reset_log = !g->fifo.deferred_reset_pending
1291 && print_channel_reset_log;
1292
1293 if (print_channel_reset_log) {
1294 int engine_id;
1295 gk20a_err(dev_from_gk20a(g),
1296 "channel reset initated from %s", __func__);
1297 for (engine_id = 0;
1298 engine_id < g->fifo.max_engines;
1299 engine_id++) {
1300 gk20a_dbg_fn("enum:%d -> engine_id:%d", engine_id,
1301 g->fifo.engine_info[engine_id].engine_id);
1302 fifo_pbdma_exception_status(g,
1303 &g->fifo.engine_info[engine_id]);
1304 fifo_engine_exception_status(g,
1305 &g->fifo.engine_info[engine_id]);
1306 }
1307 }
1308
1309 return handled;
1310}
1311
1312
1313static u32 gk20a_fifo_handle_pbdma_intr(struct device *dev,
1314 struct gk20a *g,
1315 struct fifo_gk20a *f,
1316 u32 pbdma_id)
1317{
1318 u32 pbdma_intr_0 = gk20a_readl(g, pbdma_intr_0_r(pbdma_id));
1319 u32 pbdma_intr_1 = gk20a_readl(g, pbdma_intr_1_r(pbdma_id));
1320 u32 handled = 0;
1321 bool reset_device = false;
1322 bool reset_channel = false;
1323
1324 gk20a_dbg_fn("");
1325
1326 gk20a_dbg(gpu_dbg_intr, "pbdma id intr pending %d %08x %08x", pbdma_id,
1327 pbdma_intr_0, pbdma_intr_1);
1328 if (pbdma_intr_0) {
1329 if (f->intr.pbdma.device_fatal_0 & pbdma_intr_0) {
1330 dev_err(dev, "unrecoverable device error: "
1331 "pbdma_intr_0(%d):0x%08x", pbdma_id, pbdma_intr_0);
1332 reset_device = true;
1333 /* TODO: disable pbdma intrs */
1334 handled |= f->intr.pbdma.device_fatal_0 & pbdma_intr_0;
1335 }
1336 if (f->intr.pbdma.channel_fatal_0 & pbdma_intr_0) {
1337 dev_warn(dev, "channel error: "
1338 "pbdma_intr_0(%d):0x%08x", pbdma_id, pbdma_intr_0);
1339 reset_channel = true;
1340 /* TODO: clear pbdma channel errors */
1341 handled |= f->intr.pbdma.channel_fatal_0 & pbdma_intr_0;
1342 }
1343 if (f->intr.pbdma.restartable_0 & pbdma_intr_0) {
1344 dev_warn(dev, "sw method: %08x %08x",
1345 gk20a_readl(g, pbdma_method0_r(0)),
1346 gk20a_readl(g, pbdma_method0_r(0)+4));
1347 gk20a_writel(g, pbdma_method0_r(0), 0);
1348 gk20a_writel(g, pbdma_method0_r(0)+4, 0);
1349 handled |= f->intr.pbdma.restartable_0 & pbdma_intr_0;
1350 }
1351
1352 gk20a_writel(g, pbdma_intr_0_r(pbdma_id), pbdma_intr_0);
1353 }
1354
1355 /* all intrs in _intr_1 are "host copy engine" related,
1356 * which gk20a doesn't have. for now just make them channel fatal. */
1357 if (pbdma_intr_1) {
1358 dev_err(dev, "channel hce error: pbdma_intr_1(%d): 0x%08x",
1359 pbdma_id, pbdma_intr_1);
1360 reset_channel = true;
1361 gk20a_writel(g, pbdma_intr_1_r(pbdma_id), pbdma_intr_1);
1362 }
1363
1364
1365
1366 return handled;
1367}
1368
1369static u32 fifo_channel_isr(struct gk20a *g, u32 fifo_intr)
1370{
1371 gk20a_channel_semaphore_wakeup(g);
1372 return fifo_intr_0_channel_intr_pending_f();
1373}
1374
1375
1376static u32 fifo_pbdma_isr(struct gk20a *g, u32 fifo_intr)
1377{
1378 struct device *dev = dev_from_gk20a(g);
1379 struct fifo_gk20a *f = &g->fifo;
1380 u32 clear_intr = 0, i;
1381 u32 pbdma_pending = gk20a_readl(g, fifo_intr_pbdma_id_r());
1382
1383 for (i = 0; i < fifo_intr_pbdma_id_status__size_1_v(); i++) {
1384 if (fifo_intr_pbdma_id_status_f(pbdma_pending, i)) {
1385 gk20a_dbg(gpu_dbg_intr, "pbdma id %d intr pending", i);
1386 clear_intr |=
1387 gk20a_fifo_handle_pbdma_intr(dev, g, f, i);
1388 }
1389 }
1390 return fifo_intr_0_pbdma_intr_pending_f();
1391}
1392
1393void gk20a_fifo_isr(struct gk20a *g)
1394{
1395 u32 error_intr_mask =
1396 fifo_intr_0_bind_error_pending_f() |
1397 fifo_intr_0_sched_error_pending_f() |
1398 fifo_intr_0_chsw_error_pending_f() |
1399 fifo_intr_0_fb_flush_timeout_pending_f() |
1400 fifo_intr_0_dropped_mmu_fault_pending_f() |
1401 fifo_intr_0_mmu_fault_pending_f() |
1402 fifo_intr_0_lb_error_pending_f() |
1403 fifo_intr_0_pio_error_pending_f();
1404
1405 u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r());
1406 u32 clear_intr = 0;
1407
1408 /* note we're not actually in an "isr", but rather
1409 * in a threaded interrupt context... */
1410 mutex_lock(&g->fifo.intr.isr.mutex);
1411
1412 gk20a_dbg(gpu_dbg_intr, "fifo isr %08x\n", fifo_intr);
1413
1414 /* handle runlist update */
1415 if (fifo_intr & fifo_intr_0_runlist_event_pending_f()) {
1416 gk20a_fifo_handle_runlist_event(g);
1417 clear_intr |= fifo_intr_0_runlist_event_pending_f();
1418 }
1419 if (fifo_intr & fifo_intr_0_pbdma_intr_pending_f())
1420 clear_intr |= fifo_pbdma_isr(g, fifo_intr);
1421
1422 if (unlikely(fifo_intr & error_intr_mask))
1423 clear_intr = fifo_error_isr(g, fifo_intr);
1424
1425 gk20a_writel(g, fifo_intr_0_r(), clear_intr);
1426
1427 mutex_unlock(&g->fifo.intr.isr.mutex);
1428
1429 return;
1430}
1431
1432void gk20a_fifo_nonstall_isr(struct gk20a *g)
1433{
1434 u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r());
1435 u32 clear_intr = 0;
1436
1437 gk20a_dbg(gpu_dbg_intr, "fifo nonstall isr %08x\n", fifo_intr);
1438
1439 if (fifo_intr & fifo_intr_0_channel_intr_pending_f())
1440 clear_intr |= fifo_channel_isr(g, fifo_intr);
1441
1442 gk20a_writel(g, fifo_intr_0_r(), clear_intr);
1443
1444 return;
1445}
1446
1447int gk20a_fifo_preempt_channel(struct gk20a *g, u32 hw_chid)
1448{
1449 struct fifo_gk20a *f = &g->fifo;
1450 unsigned long end_jiffies = jiffies
1451 + msecs_to_jiffies(gk20a_get_gr_idle_timeout(g));
1452 u32 delay = GR_IDLE_CHECK_DEFAULT;
1453 u32 ret = 0;
1454 u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1455 u32 elpg_off = 0;
1456 u32 i;
1457
1458 gk20a_dbg_fn("%d", hw_chid);
1459
1460 /* we have no idea which runlist we are using. lock all */
1461 for (i = 0; i < g->fifo.max_runlists; i++)
1462 mutex_lock(&f->runlist_info[i].mutex);
1463
1464 /* disable elpg if failed to acquire pmu mutex */
1465 elpg_off = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1466 if (elpg_off)
1467 gk20a_pmu_disable_elpg(g);
1468
1469 /* issue preempt */
1470 gk20a_writel(g, fifo_preempt_r(),
1471 fifo_preempt_chid_f(hw_chid) |
1472 fifo_preempt_type_channel_f());
1473
1474 /* wait for preempt */
1475 ret = -EBUSY;
1476 do {
1477 if (!(gk20a_readl(g, fifo_preempt_r()) &
1478 fifo_preempt_pending_true_f())) {
1479 ret = 0;
1480 break;
1481 }
1482
1483 usleep_range(delay, delay * 2);
1484 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
1485 } while (time_before(jiffies, end_jiffies) ||
1486 !tegra_platform_is_silicon());
1487
1488 if (ret) {
1489 int i;
1490 u32 engines = 0;
1491 struct fifo_gk20a *f = &g->fifo;
1492 struct channel_gk20a *ch = &f->channel[hw_chid];
1493
1494 gk20a_err(dev_from_gk20a(g), "preempt channel %d timeout\n",
1495 hw_chid);
1496
1497 /* forcefully reset all busy engines using this channel */
1498 for (i = 0; i < g->fifo.max_engines; i++) {
1499 u32 status = gk20a_readl(g, fifo_engine_status_r(i));
1500 u32 ctx_status =
1501 fifo_engine_status_ctx_status_v(status);
1502 bool type_ch = fifo_pbdma_status_id_type_v(status) ==
1503 fifo_pbdma_status_id_type_chid_v();
1504 bool busy = fifo_engine_status_engine_v(status) ==
1505 fifo_engine_status_engine_busy_v();
1506 u32 id = (ctx_status ==
1507 fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1508 fifo_engine_status_next_id_v(status) :
1509 fifo_engine_status_id_v(status);
1510
1511 if (type_ch && busy && id == hw_chid)
1512 engines |= BIT(i);
1513 }
1514 gk20a_set_error_notifier(ch,
1515 NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT);
1516 gk20a_fifo_recover(g, engines, true);
1517 }
1518
1519 /* re-enable elpg or release pmu mutex */
1520 if (elpg_off)
1521 gk20a_pmu_enable_elpg(g);
1522 else
1523 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1524
1525 for (i = 0; i < g->fifo.max_runlists; i++)
1526 mutex_unlock(&f->runlist_info[i].mutex);
1527
1528 return ret;
1529}
1530
1531int gk20a_fifo_enable_engine_activity(struct gk20a *g,
1532 struct fifo_engine_info_gk20a *eng_info)
1533{
1534 u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1535 u32 elpg_off;
1536 u32 enable;
1537
1538 gk20a_dbg_fn("");
1539
1540 /* disable elpg if failed to acquire pmu mutex */
1541 elpg_off = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1542 if (elpg_off)
1543 gk20a_pmu_disable_elpg(g);
1544
1545 enable = gk20a_readl(g, fifo_sched_disable_r());
1546 enable &= ~(fifo_sched_disable_true_v() >> eng_info->runlist_id);
1547 gk20a_writel(g, fifo_sched_disable_r(), enable);
1548
1549 /* re-enable elpg or release pmu mutex */
1550 if (elpg_off)
1551 gk20a_pmu_enable_elpg(g);
1552 else
1553 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1554
1555 gk20a_dbg_fn("done");
1556 return 0;
1557}
1558
1559int gk20a_fifo_disable_engine_activity(struct gk20a *g,
1560 struct fifo_engine_info_gk20a *eng_info,
1561 bool wait_for_idle)
1562{
1563 u32 gr_stat, pbdma_stat, chan_stat, eng_stat, ctx_stat;
1564 u32 pbdma_chid = ~0, engine_chid = ~0, disable;
1565 u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1566 u32 elpg_off;
1567 u32 err = 0;
1568
1569 gk20a_dbg_fn("");
1570
1571 gr_stat =
1572 gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
1573 if (fifo_engine_status_engine_v(gr_stat) ==
1574 fifo_engine_status_engine_busy_v() && !wait_for_idle)
1575 return -EBUSY;
1576
1577 /* disable elpg if failed to acquire pmu mutex */
1578 elpg_off = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1579 if (elpg_off)
1580 gk20a_pmu_disable_elpg(g);
1581
1582 disable = gk20a_readl(g, fifo_sched_disable_r());
1583 disable = set_field(disable,
1584 fifo_sched_disable_runlist_m(eng_info->runlist_id),
1585 fifo_sched_disable_runlist_f(fifo_sched_disable_true_v(),
1586 eng_info->runlist_id));
1587 gk20a_writel(g, fifo_sched_disable_r(), disable);
1588
1589 /* chid from pbdma status */
1590 pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(eng_info->pbdma_id));
1591 chan_stat = fifo_pbdma_status_chan_status_v(pbdma_stat);
1592 if (chan_stat == fifo_pbdma_status_chan_status_valid_v() ||
1593 chan_stat == fifo_pbdma_status_chan_status_chsw_save_v())
1594 pbdma_chid = fifo_pbdma_status_id_v(pbdma_stat);
1595 else if (chan_stat == fifo_pbdma_status_chan_status_chsw_load_v() ||
1596 chan_stat == fifo_pbdma_status_chan_status_chsw_switch_v())
1597 pbdma_chid = fifo_pbdma_status_next_id_v(pbdma_stat);
1598
1599 if (pbdma_chid != ~0) {
1600 err = gk20a_fifo_preempt_channel(g, pbdma_chid);
1601 if (err)
1602 goto clean_up;
1603 }
1604
1605 /* chid from engine status */
1606 eng_stat = gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
1607 ctx_stat = fifo_engine_status_ctx_status_v(eng_stat);
1608 if (ctx_stat == fifo_engine_status_ctx_status_valid_v() ||
1609 ctx_stat == fifo_engine_status_ctx_status_ctxsw_save_v())
1610 engine_chid = fifo_engine_status_id_v(eng_stat);
1611 else if (ctx_stat == fifo_engine_status_ctx_status_ctxsw_load_v() ||
1612 ctx_stat == fifo_engine_status_ctx_status_ctxsw_switch_v())
1613 engine_chid = fifo_engine_status_next_id_v(eng_stat);
1614
1615 if (engine_chid != ~0 && engine_chid != pbdma_chid) {
1616 err = gk20a_fifo_preempt_channel(g, engine_chid);
1617 if (err)
1618 goto clean_up;
1619 }
1620
1621clean_up:
1622 /* re-enable elpg or release pmu mutex */
1623 if (elpg_off)
1624 gk20a_pmu_enable_elpg(g);
1625 else
1626 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1627
1628 if (err) {
1629 gk20a_dbg_fn("failed");
1630 if (gk20a_fifo_enable_engine_activity(g, eng_info))
1631 gk20a_err(dev_from_gk20a(g),
1632 "failed to enable gr engine activity\n");
1633 } else {
1634 gk20a_dbg_fn("done");
1635 }
1636 return err;
1637}
1638
1639static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id)
1640{
1641 struct fifo_gk20a *f = &g->fifo;
1642 u32 engines = 0;
1643 int i;
1644
1645 for (i = 0; i < f->max_engines; i++) {
1646 u32 status = gk20a_readl(g, fifo_engine_status_r(i));
1647 bool engine_busy = fifo_engine_status_engine_v(status) ==
1648 fifo_engine_status_engine_busy_v();
1649
1650 if (engine_busy &&
1651 (f->engine_info[i].runlist_id == runlist_id))
1652 engines |= BIT(i);
1653 }
1654 gk20a_fifo_recover(g, engines, true);
1655}
1656
1657static int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id)
1658{
1659 struct fifo_runlist_info_gk20a *runlist;
1660 u32 remain;
1661 bool pending;
1662
1663 runlist = &g->fifo.runlist_info[runlist_id];
1664 remain = wait_event_timeout(runlist->runlist_wq,
1665 ((pending = gk20a_readl(g, fifo_eng_runlist_r(runlist_id)) &
1666 fifo_eng_runlist_pending_true_f()) == 0),
1667 msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
1668
1669 if (remain == 0 && pending != 0)
1670 return -ETIMEDOUT;
1671
1672 return 0;
1673}
1674
1675static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
1676 u32 hw_chid, bool add,
1677 bool wait_for_finish)
1678{
1679 u32 ret = 0;
1680 struct device *d = dev_from_gk20a(g);
1681 struct fifo_gk20a *f = &g->fifo;
1682 struct fifo_runlist_info_gk20a *runlist = NULL;
1683 u32 *runlist_entry_base = NULL;
1684 u32 *runlist_entry = NULL;
1685 phys_addr_t runlist_pa;
1686 u32 old_buf, new_buf;
1687 u32 chid;
1688 u32 count = 0;
1689 runlist = &f->runlist_info[runlist_id];
1690
1691 /* valid channel, add/remove it from active list.
1692 Otherwise, keep active list untouched for suspend/resume. */
1693 if (hw_chid != ~0) {
1694 if (add) {
1695 if (test_and_set_bit(hw_chid,
1696 runlist->active_channels) == 1)
1697 return 0;
1698 } else {
1699 if (test_and_clear_bit(hw_chid,
1700 runlist->active_channels) == 0)
1701 return 0;
1702 }
1703 }
1704
1705 old_buf = runlist->cur_buffer;
1706 new_buf = !runlist->cur_buffer;
1707
1708 gk20a_dbg_info("runlist_id : %d, switch to new buffer 0x%16llx",
1709 runlist_id, runlist->mem[new_buf].iova);
1710
1711 runlist_pa = gk20a_get_phys_from_iova(d, runlist->mem[new_buf].iova);
1712 if (!runlist_pa) {
1713 ret = -EINVAL;
1714 goto clean_up;
1715 }
1716
1717 runlist_entry_base = runlist->mem[new_buf].cpuva;
1718 if (!runlist_entry_base) {
1719 ret = -ENOMEM;
1720 goto clean_up;
1721 }
1722
1723 if (hw_chid != ~0 || /* add/remove a valid channel */
1724 add /* resume to add all channels back */) {
1725 runlist_entry = runlist_entry_base;
1726 for_each_set_bit(chid,
1727 runlist->active_channels, f->num_channels) {
1728 gk20a_dbg_info("add channel %d to runlist", chid);
1729 runlist_entry[0] = chid;
1730 runlist_entry[1] = 0;
1731 runlist_entry += 2;
1732 count++;
1733 }
1734 } else /* suspend to remove all channels */
1735 count = 0;
1736
1737 if (count != 0) {
1738 gk20a_writel(g, fifo_runlist_base_r(),
1739 fifo_runlist_base_ptr_f(u64_lo32(runlist_pa >> 12)) |
1740 fifo_runlist_base_target_vid_mem_f());
1741 }
1742
1743 gk20a_writel(g, fifo_runlist_r(),
1744 fifo_runlist_engine_f(runlist_id) |
1745 fifo_eng_runlist_length_f(count));
1746
1747 if (wait_for_finish) {
1748 ret = gk20a_fifo_runlist_wait_pending(g, runlist_id);
1749
1750 if (ret == -ETIMEDOUT) {
1751 gk20a_err(dev_from_gk20a(g),
1752 "runlist update timeout");
1753
1754 gk20a_fifo_runlist_reset_engines(g, runlist_id);
1755
1756 /* engine reset needs the lock. drop it */
1757 mutex_unlock(&runlist->mutex);
1758 /* wait until the runlist is active again */
1759 ret = gk20a_fifo_runlist_wait_pending(g, runlist_id);
1760 /* get the lock back. at this point everything should
1761 * should be fine */
1762 mutex_lock(&runlist->mutex);
1763
1764 if (ret)
1765 gk20a_err(dev_from_gk20a(g),
1766 "runlist update failed: %d", ret);
1767 } else if (ret == -EINTR)
1768 gk20a_err(dev_from_gk20a(g),
1769 "runlist update interrupted");
1770 }
1771
1772 runlist->cur_buffer = new_buf;
1773
1774clean_up:
1775 return ret;
1776}
1777
1778/* add/remove a channel from runlist
1779 special cases below: runlist->active_channels will NOT be changed.
1780 (hw_chid == ~0 && !add) means remove all active channels from runlist.
1781 (hw_chid == ~0 && add) means restore all active channels on runlist. */
1782int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 hw_chid,
1783 bool add, bool wait_for_finish)
1784{
1785 struct fifo_runlist_info_gk20a *runlist = NULL;
1786 struct fifo_gk20a *f = &g->fifo;
1787 u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1788 u32 elpg_off;
1789 u32 ret = 0;
1790
1791 runlist = &f->runlist_info[runlist_id];
1792
1793 mutex_lock(&runlist->mutex);
1794
1795 /* disable elpg if failed to acquire pmu mutex */
1796 elpg_off = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1797 if (elpg_off)
1798 gk20a_pmu_disable_elpg(g);
1799
1800 ret = gk20a_fifo_update_runlist_locked(g, runlist_id, hw_chid, add,
1801 wait_for_finish);
1802
1803 /* re-enable elpg or release pmu mutex */
1804 if (elpg_off)
1805 gk20a_pmu_enable_elpg(g);
1806 else
1807 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1808
1809 mutex_unlock(&runlist->mutex);
1810 return ret;
1811}
1812
1813int gk20a_fifo_suspend(struct gk20a *g)
1814{
1815 gk20a_dbg_fn("");
1816
1817 /* stop bar1 snooping */
1818 gk20a_writel(g, fifo_bar1_base_r(),
1819 fifo_bar1_base_valid_false_f());
1820
1821 /* disable fifo intr */
1822 gk20a_writel(g, fifo_intr_en_0_r(), 0);
1823 gk20a_writel(g, fifo_intr_en_1_r(), 0);
1824
1825 gk20a_dbg_fn("done");
1826 return 0;
1827}
1828
1829bool gk20a_fifo_mmu_fault_pending(struct gk20a *g)
1830{
1831 if (gk20a_readl(g, fifo_intr_0_r()) &
1832 fifo_intr_0_mmu_fault_pending_f())
1833 return true;
1834 else
1835 return false;
1836}
generated by cgit v1.2.2 (git 2.25.0) at 2024-06-09 00:03:24 -0400