summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a/fifo_gk20a.c')
-rw-r--r--drivers/gpu/nvgpu/gk20a/fifo_gk20a.c554
1 files changed, 358 insertions, 196 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index e03c5da8..f06bf1c5 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -111,8 +111,9 @@ struct fifo_engine_info_gk20a *gk20a_fifo_get_engine_info(struct gk20a *g, u32 e
111 u32 engine_id_idx; 111 u32 engine_id_idx;
112 struct fifo_engine_info_gk20a *info = NULL; 112 struct fifo_engine_info_gk20a *info = NULL;
113 113
114 if (!g) 114 if (!g) {
115 return info; 115 return info;
116 }
116 117
117 f = &g->fifo; 118 f = &g->fifo;
118 119
@@ -125,8 +126,9 @@ struct fifo_engine_info_gk20a *gk20a_fifo_get_engine_info(struct gk20a *g, u32 e
125 } 126 }
126 } 127 }
127 128
128 if (!info) 129 if (!info) {
129 nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id); 130 nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id);
131 }
130 132
131 return info; 133 return info;
132} 134}
@@ -137,8 +139,9 @@ bool gk20a_fifo_is_valid_engine_id(struct gk20a *g, u32 engine_id)
137 u32 engine_id_idx; 139 u32 engine_id_idx;
138 bool valid = false; 140 bool valid = false;
139 141
140 if (!g) 142 if (!g) {
141 return valid; 143 return valid;
144 }
142 145
143 f = &g->fifo; 146 f = &g->fifo;
144 147
@@ -151,8 +154,9 @@ bool gk20a_fifo_is_valid_engine_id(struct gk20a *g, u32 engine_id)
151 } 154 }
152 } 155 }
153 156
154 if (!valid) 157 if (!valid) {
155 nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id); 158 nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id);
159 }
156 160
157 return valid; 161 return valid;
158} 162}
@@ -182,8 +186,9 @@ u32 gk20a_fifo_get_all_ce_engine_reset_mask(struct gk20a *g)
182 struct fifo_engine_info_gk20a *engine_info; 186 struct fifo_engine_info_gk20a *engine_info;
183 u32 active_engine_id = 0; 187 u32 active_engine_id = 0;
184 188
185 if (!g) 189 if (!g) {
186 return reset_mask; 190 return reset_mask;
191 }
187 192
188 f = &g->fifo; 193 f = &g->fifo;
189 194
@@ -193,8 +198,9 @@ u32 gk20a_fifo_get_all_ce_engine_reset_mask(struct gk20a *g)
193 engine_enum = engine_info->engine_enum; 198 engine_enum = engine_info->engine_enum;
194 199
195 if ((engine_enum == ENGINE_GRCE_GK20A) || 200 if ((engine_enum == ENGINE_GRCE_GK20A) ||
196 (engine_enum == ENGINE_ASYNC_CE_GK20A)) 201 (engine_enum == ENGINE_ASYNC_CE_GK20A)) {
197 reset_mask |= engine_info->reset_mask; 202 reset_mask |= engine_info->reset_mask;
203 }
198 } 204 }
199 205
200 return reset_mask; 206 return reset_mask;
@@ -209,8 +215,9 @@ u32 gk20a_fifo_get_fast_ce_runlist_id(struct gk20a *g)
209 struct fifo_engine_info_gk20a *engine_info; 215 struct fifo_engine_info_gk20a *engine_info;
210 u32 active_engine_id = 0; 216 u32 active_engine_id = 0;
211 217
212 if (!g) 218 if (!g) {
213 return ce_runlist_id; 219 return ce_runlist_id;
220 }
214 221
215 f = &g->fifo; 222 f = &g->fifo;
216 223
@@ -220,8 +227,9 @@ u32 gk20a_fifo_get_fast_ce_runlist_id(struct gk20a *g)
220 engine_enum = engine_info->engine_enum; 227 engine_enum = engine_info->engine_enum;
221 228
222 /* selecet last available ASYNC_CE if available */ 229 /* selecet last available ASYNC_CE if available */
223 if (engine_enum == ENGINE_ASYNC_CE_GK20A) 230 if (engine_enum == ENGINE_ASYNC_CE_GK20A) {
224 ce_runlist_id = engine_info->runlist_id; 231 ce_runlist_id = engine_info->runlist_id;
232 }
225 } 233 }
226 234
227 return ce_runlist_id; 235 return ce_runlist_id;
@@ -264,8 +272,9 @@ bool gk20a_fifo_is_valid_runlist_id(struct gk20a *g, u32 runlist_id)
264 u32 active_engine_id; 272 u32 active_engine_id;
265 struct fifo_engine_info_gk20a *engine_info; 273 struct fifo_engine_info_gk20a *engine_info;
266 274
267 if (!g) 275 if (!g) {
268 return false; 276 return false;
277 }
269 278
270 f = &g->fifo; 279 f = &g->fifo;
271 280
@@ -310,8 +319,9 @@ static inline u32 gk20a_mmu_id_to_engine_id(struct gk20a *g, u32 fault_id)
310 active_engine_id = f->active_engines_list[engine_id]; 319 active_engine_id = f->active_engines_list[engine_id];
311 engine_info = &g->fifo.engine_info[active_engine_id]; 320 engine_info = &g->fifo.engine_info[active_engine_id];
312 321
313 if (engine_info->fault_id == fault_id) 322 if (engine_info->fault_id == fault_id) {
314 break; 323 break;
324 }
315 active_engine_id = FIFO_INVAL_ENGINE_ID; 325 active_engine_id = FIFO_INVAL_ENGINE_ID;
316 } 326 }
317 return active_engine_id; 327 return active_engine_id;
@@ -323,17 +333,18 @@ int gk20a_fifo_engine_enum_from_type(struct gk20a *g, u32 engine_type,
323 int ret = ENGINE_INVAL_GK20A; 333 int ret = ENGINE_INVAL_GK20A;
324 334
325 nvgpu_log_info(g, "engine type %d", engine_type); 335 nvgpu_log_info(g, "engine type %d", engine_type);
326 if (engine_type == top_device_info_type_enum_graphics_v()) 336 if (engine_type == top_device_info_type_enum_graphics_v()) {
327 ret = ENGINE_GR_GK20A; 337 ret = ENGINE_GR_GK20A;
328 else if ((engine_type >= top_device_info_type_enum_copy0_v()) && 338 } else if ((engine_type >= top_device_info_type_enum_copy0_v()) &&
329 (engine_type <= top_device_info_type_enum_copy2_v())) { 339 (engine_type <= top_device_info_type_enum_copy2_v())) {
330 /* Lets consider all the CE engine have separate runlist at this point 340 /* Lets consider all the CE engine have separate runlist at this point
331 * We can identify the ENGINE_GRCE_GK20A type CE using runlist_id 341 * We can identify the ENGINE_GRCE_GK20A type CE using runlist_id
332 * comparsion logic with GR runlist_id in init_engine_info() */ 342 * comparsion logic with GR runlist_id in init_engine_info() */
333 ret = ENGINE_ASYNC_CE_GK20A; 343 ret = ENGINE_ASYNC_CE_GK20A;
334 /* inst_id starts from CE0 to CE2 */ 344 /* inst_id starts from CE0 to CE2 */
335 if (inst_id) 345 if (inst_id) {
336 *inst_id = (engine_type - top_device_info_type_enum_copy0_v()); 346 *inst_id = (engine_type - top_device_info_type_enum_copy0_v());
347 }
337 } 348 }
338 349
339 return ret; 350 return ret;
@@ -421,10 +432,11 @@ int gk20a_fifo_init_engine_info(struct fifo_gk20a *f)
421 engine_type, &inst_id); 432 engine_type, &inst_id);
422 } else if (entry == top_device_info_entry_data_v()) { 433 } else if (entry == top_device_info_entry_data_v()) {
423 /* gk20a doesn't support device_info_data packet parsing */ 434 /* gk20a doesn't support device_info_data packet parsing */
424 if (g->ops.fifo.device_info_data_parse) 435 if (g->ops.fifo.device_info_data_parse) {
425 g->ops.fifo.device_info_data_parse(g, 436 g->ops.fifo.device_info_data_parse(g,
426 table_entry, &inst_id, &pri_base, 437 table_entry, &inst_id, &pri_base,
427 &fault_id); 438 &fault_id);
439 }
428 } 440 }
429 441
430 if (!top_device_info_chain_v(table_entry)) { 442 if (!top_device_info_chain_v(table_entry)) {
@@ -439,18 +451,21 @@ int gk20a_fifo_init_engine_info(struct fifo_gk20a *f)
439 info->inst_id = inst_id; 451 info->inst_id = inst_id;
440 info->pri_base = pri_base; 452 info->pri_base = pri_base;
441 453
442 if (engine_enum == ENGINE_GR_GK20A) 454 if (engine_enum == ENGINE_GR_GK20A) {
443 gr_runlist_id = runlist_id; 455 gr_runlist_id = runlist_id;
456 }
444 457
445 /* GR and GR_COPY shares same runlist_id */ 458 /* GR and GR_COPY shares same runlist_id */
446 if ((engine_enum == ENGINE_ASYNC_CE_GK20A) && 459 if ((engine_enum == ENGINE_ASYNC_CE_GK20A) &&
447 (gr_runlist_id == runlist_id)) 460 (gr_runlist_id == runlist_id)) {
448 engine_enum = ENGINE_GRCE_GK20A; 461 engine_enum = ENGINE_GRCE_GK20A;
462 }
449 463
450 info->engine_enum = engine_enum; 464 info->engine_enum = engine_enum;
451 465
452 if (!fault_id && (engine_enum == ENGINE_GRCE_GK20A)) 466 if (!fault_id && (engine_enum == ENGINE_GRCE_GK20A)) {
453 fault_id = 0x1b; 467 fault_id = 0x1b;
468 }
454 info->fault_id = fault_id; 469 info->fault_id = fault_id;
455 470
456 /* engine_id starts from 0 to NV_HOST_NUM_ENGINES */ 471 /* engine_id starts from 0 to NV_HOST_NUM_ENGINES */
@@ -471,8 +486,9 @@ u32 gk20a_fifo_act_eng_interrupt_mask(struct gk20a *g, u32 act_eng_id)
471 struct fifo_engine_info_gk20a *engine_info = NULL; 486 struct fifo_engine_info_gk20a *engine_info = NULL;
472 487
473 engine_info = gk20a_fifo_get_engine_info(g, act_eng_id); 488 engine_info = gk20a_fifo_get_engine_info(g, act_eng_id);
474 if (engine_info) 489 if (engine_info) {
475 return engine_info->intr_mask; 490 return engine_info->intr_mask;
491 }
476 492
477 return 0; 493 return 0;
478} 494}
@@ -491,8 +507,9 @@ u32 gk20a_fifo_engine_interrupt_mask(struct gk20a *g)
491 engine_enum = g->fifo.engine_info[active_engine_id].engine_enum; 507 engine_enum = g->fifo.engine_info[active_engine_id].engine_enum;
492 if (((engine_enum == ENGINE_GRCE_GK20A) || 508 if (((engine_enum == ENGINE_GRCE_GK20A) ||
493 (engine_enum == ENGINE_ASYNC_CE_GK20A)) && 509 (engine_enum == ENGINE_ASYNC_CE_GK20A)) &&
494 (!g->ops.ce2.isr_stall || !g->ops.ce2.isr_nonstall)) 510 (!g->ops.ce2.isr_stall || !g->ops.ce2.isr_nonstall)) {
495 continue; 511 continue;
512 }
496 513
497 eng_intr_mask |= intr_mask; 514 eng_intr_mask |= intr_mask;
498 } 515 }
@@ -507,8 +524,9 @@ void gk20a_fifo_delete_runlist(struct fifo_gk20a *f)
507 struct fifo_runlist_info_gk20a *runlist; 524 struct fifo_runlist_info_gk20a *runlist;
508 struct gk20a *g = NULL; 525 struct gk20a *g = NULL;
509 526
510 if (!f || !f->runlist_info) 527 if (!f || !f->runlist_info) {
511 return; 528 return;
529 }
512 530
513 g = f->g; 531 g = f->g;
514 532
@@ -554,8 +572,9 @@ static void gk20a_remove_fifo_support(struct fifo_gk20a *f)
554 * Could race but worst that happens is we get an error message 572 * Could race but worst that happens is we get an error message
555 * from gk20a_free_channel() complaining about multiple closes. 573 * from gk20a_free_channel() complaining about multiple closes.
556 */ 574 */
557 if (c->referenceable) 575 if (c->referenceable) {
558 __gk20a_channel_kill(c); 576 __gk20a_channel_kill(c);
577 }
559 578
560 nvgpu_mutex_destroy(&tsg->event_id_list_lock); 579 nvgpu_mutex_destroy(&tsg->event_id_list_lock);
561 580
@@ -573,10 +592,11 @@ static void gk20a_remove_fifo_support(struct fifo_gk20a *f)
573 592
574 nvgpu_vfree(g, f->channel); 593 nvgpu_vfree(g, f->channel);
575 nvgpu_vfree(g, f->tsg); 594 nvgpu_vfree(g, f->tsg);
576 if (g->ops.mm.is_bar1_supported(g)) 595 if (g->ops.mm.is_bar1_supported(g)) {
577 nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd); 596 nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd);
578 else 597 } else {
579 nvgpu_dma_free(g, &f->userd); 598 nvgpu_dma_free(g, &f->userd);
599 }
580 600
581 gk20a_fifo_delete_runlist(f); 601 gk20a_fifo_delete_runlist(f);
582 602
@@ -683,8 +703,9 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
683 f->runlist_info = nvgpu_kzalloc(g, 703 f->runlist_info = nvgpu_kzalloc(g,
684 sizeof(struct fifo_runlist_info_gk20a) * 704 sizeof(struct fifo_runlist_info_gk20a) *
685 f->max_runlists); 705 f->max_runlists);
686 if (!f->runlist_info) 706 if (!f->runlist_info) {
687 goto clean_up_runlist; 707 goto clean_up_runlist;
708 }
688 709
689 memset(f->runlist_info, 0, (sizeof(struct fifo_runlist_info_gk20a) * 710 memset(f->runlist_info, 0, (sizeof(struct fifo_runlist_info_gk20a) *
690 f->max_runlists)); 711 f->max_runlists));
@@ -695,14 +716,16 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
695 runlist->active_channels = 716 runlist->active_channels =
696 nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, 717 nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
697 BITS_PER_BYTE)); 718 BITS_PER_BYTE));
698 if (!runlist->active_channels) 719 if (!runlist->active_channels) {
699 goto clean_up_runlist; 720 goto clean_up_runlist;
721 }
700 722
701 runlist->active_tsgs = 723 runlist->active_tsgs =
702 nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, 724 nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
703 BITS_PER_BYTE)); 725 BITS_PER_BYTE));
704 if (!runlist->active_tsgs) 726 if (!runlist->active_tsgs) {
705 goto clean_up_runlist; 727 goto clean_up_runlist;
728 }
706 729
707 runlist_size = f->runlist_entry_size * f->num_runlist_entries; 730 runlist_size = f->runlist_entry_size * f->num_runlist_entries;
708 nvgpu_log(g, gpu_dbg_info, 731 nvgpu_log(g, gpu_dbg_info,
@@ -725,8 +748,9 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
725 runlist->cur_buffer = MAX_RUNLIST_BUFFERS; 748 runlist->cur_buffer = MAX_RUNLIST_BUFFERS;
726 749
727 for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) { 750 for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) {
728 if (f->pbdma_map[pbdma_id] & BIT(runlist_id)) 751 if (f->pbdma_map[pbdma_id] & BIT(runlist_id)) {
729 runlist->pbdma_bitmask |= BIT(pbdma_id); 752 runlist->pbdma_bitmask |= BIT(pbdma_id);
753 }
730 } 754 }
731 nvgpu_log(g, gpu_dbg_info, "runlist %d : pbdma bitmask 0x%x", 755 nvgpu_log(g, gpu_dbg_info, "runlist %d : pbdma bitmask 0x%x",
732 runlist_id, runlist->pbdma_bitmask); 756 runlist_id, runlist->pbdma_bitmask);
@@ -735,8 +759,9 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
735 active_engine_id = f->active_engines_list[engine_id]; 759 active_engine_id = f->active_engines_list[engine_id];
736 engine_info = &f->engine_info[active_engine_id]; 760 engine_info = &f->engine_info[active_engine_id];
737 761
738 if (engine_info && engine_info->runlist_id == runlist_id) 762 if (engine_info && engine_info->runlist_id == runlist_id) {
739 runlist->eng_bitmask |= BIT(active_engine_id); 763 runlist->eng_bitmask |= BIT(active_engine_id);
764 }
740 } 765 }
741 nvgpu_log(g, gpu_dbg_info, "runlist %d : act eng bitmask 0x%x", 766 nvgpu_log(g, gpu_dbg_info, "runlist %d : act eng bitmask 0x%x",
742 runlist_id, runlist->eng_bitmask); 767 runlist_id, runlist->eng_bitmask);
@@ -791,12 +816,14 @@ int gk20a_init_fifo_reset_enable_hw(struct gk20a *g)
791 /* enable pmc pfifo */ 816 /* enable pmc pfifo */
792 g->ops.mc.reset(g, mc_enable_pfifo_enabled_f()); 817 g->ops.mc.reset(g, mc_enable_pfifo_enabled_f());
793 818
794 if (g->ops.clock_gating.slcg_fifo_load_gating_prod) 819 if (g->ops.clock_gating.slcg_fifo_load_gating_prod) {
795 g->ops.clock_gating.slcg_fifo_load_gating_prod(g, 820 g->ops.clock_gating.slcg_fifo_load_gating_prod(g,
796 g->slcg_enabled); 821 g->slcg_enabled);
797 if (g->ops.clock_gating.blcg_fifo_load_gating_prod) 822 }
823 if (g->ops.clock_gating.blcg_fifo_load_gating_prod) {
798 g->ops.clock_gating.blcg_fifo_load_gating_prod(g, 824 g->ops.clock_gating.blcg_fifo_load_gating_prod(g,
799 g->blcg_enabled); 825 g->blcg_enabled);
826 }
800 827
801 timeout = gk20a_readl(g, fifo_fb_timeout_r()); 828 timeout = gk20a_readl(g, fifo_fb_timeout_r());
802 timeout = set_field(timeout, fifo_fb_timeout_period_m(), 829 timeout = set_field(timeout, fifo_fb_timeout_period_m(),
@@ -812,12 +839,13 @@ int gk20a_init_fifo_reset_enable_hw(struct gk20a *g)
812 nvgpu_log_info(g, "pbdma_timeout reg val = 0x%08x", timeout); 839 nvgpu_log_info(g, "pbdma_timeout reg val = 0x%08x", timeout);
813 gk20a_writel(g, pbdma_timeout_r(i), timeout); 840 gk20a_writel(g, pbdma_timeout_r(i), timeout);
814 } 841 }
815 if (g->ops.fifo.apply_pb_timeout) 842 if (g->ops.fifo.apply_pb_timeout) {
816 g->ops.fifo.apply_pb_timeout(g); 843 g->ops.fifo.apply_pb_timeout(g);
844 }
817 845
818 if (g->ops.fifo.apply_ctxsw_timeout_intr) 846 if (g->ops.fifo.apply_ctxsw_timeout_intr) {
819 g->ops.fifo.apply_ctxsw_timeout_intr(g); 847 g->ops.fifo.apply_ctxsw_timeout_intr(g);
820 else { 848 } else {
821 timeout = g->fifo_eng_timeout_us; 849 timeout = g->fifo_eng_timeout_us;
822 timeout = scale_ptimer(timeout, 850 timeout = scale_ptimer(timeout,
823 ptimer_scalingfactor10x(g->ptimer_src_freq)); 851 ptimer_scalingfactor10x(g->ptimer_src_freq));
@@ -989,14 +1017,14 @@ int gk20a_init_fifo_setup_sw(struct gk20a *g)
989 return err; 1017 return err;
990 } 1018 }
991 1019
992 if (g->ops.mm.is_bar1_supported(g)) 1020 if (g->ops.mm.is_bar1_supported(g)) {
993 err = nvgpu_dma_alloc_map_sys(g->mm.bar1.vm, 1021 err = nvgpu_dma_alloc_map_sys(g->mm.bar1.vm,
994 f->userd_entry_size * f->num_channels, 1022 f->userd_entry_size * f->num_channels,
995 &f->userd); 1023 &f->userd);
996 1024 } else {
997 else
998 err = nvgpu_dma_alloc_sys(g, f->userd_entry_size * 1025 err = nvgpu_dma_alloc_sys(g, f->userd_entry_size *
999 f->num_channels, &f->userd); 1026 f->num_channels, &f->userd);
1027 }
1000 if (err) { 1028 if (err) {
1001 nvgpu_err(g, "userd memory allocation failed"); 1029 nvgpu_err(g, "userd memory allocation failed");
1002 goto clean_up; 1030 goto clean_up;
@@ -1012,8 +1040,9 @@ int gk20a_init_fifo_setup_sw(struct gk20a *g)
1012 } 1040 }
1013 1041
1014 err = nvgpu_channel_worker_init(g); 1042 err = nvgpu_channel_worker_init(g);
1015 if (err) 1043 if (err) {
1016 goto clean_up; 1044 goto clean_up;
1045 }
1017 1046
1018 f->sw_ready = true; 1047 f->sw_ready = true;
1019 1048
@@ -1023,10 +1052,11 @@ int gk20a_init_fifo_setup_sw(struct gk20a *g)
1023clean_up: 1052clean_up:
1024 nvgpu_log_fn(g, "fail"); 1053 nvgpu_log_fn(g, "fail");
1025 if (nvgpu_mem_is_valid(&f->userd)) { 1054 if (nvgpu_mem_is_valid(&f->userd)) {
1026 if (g->ops.mm.is_bar1_supported(g)) 1055 if (g->ops.mm.is_bar1_supported(g)) {
1027 nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd); 1056 nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd);
1028 else 1057 } else {
1029 nvgpu_dma_free(g, &f->userd); 1058 nvgpu_dma_free(g, &f->userd);
1059 }
1030 } 1060 }
1031 1061
1032 return err; 1062 return err;
@@ -1107,13 +1137,16 @@ int gk20a_init_fifo_support(struct gk20a *g)
1107 u32 err; 1137 u32 err;
1108 1138
1109 err = g->ops.fifo.setup_sw(g); 1139 err = g->ops.fifo.setup_sw(g);
1110 if (err) 1140 if (err) {
1111 return err; 1141 return err;
1142 }
1112 1143
1113 if (g->ops.fifo.init_fifo_setup_hw) 1144 if (g->ops.fifo.init_fifo_setup_hw) {
1114 err = g->ops.fifo.init_fifo_setup_hw(g); 1145 err = g->ops.fifo.init_fifo_setup_hw(g);
1115 if (err) 1146 }
1147 if (err) {
1116 return err; 1148 return err;
1149 }
1117 1150
1118 return err; 1151 return err;
1119} 1152}
@@ -1124,20 +1157,23 @@ gk20a_refch_from_inst_ptr(struct gk20a *g, u64 inst_ptr)
1124{ 1157{
1125 struct fifo_gk20a *f = &g->fifo; 1158 struct fifo_gk20a *f = &g->fifo;
1126 unsigned int ci; 1159 unsigned int ci;
1127 if (unlikely(!f->channel)) 1160 if (unlikely(!f->channel)) {
1128 return NULL; 1161 return NULL;
1162 }
1129 for (ci = 0; ci < f->num_channels; ci++) { 1163 for (ci = 0; ci < f->num_channels; ci++) {
1130 struct channel_gk20a *ch; 1164 struct channel_gk20a *ch;
1131 u64 ch_inst_ptr; 1165 u64 ch_inst_ptr;
1132 1166
1133 ch = gk20a_channel_get(&f->channel[ci]); 1167 ch = gk20a_channel_get(&f->channel[ci]);
1134 /* only alive channels are searched */ 1168 /* only alive channels are searched */
1135 if (!ch) 1169 if (!ch) {
1136 continue; 1170 continue;
1171 }
1137 1172
1138 ch_inst_ptr = nvgpu_inst_block_addr(g, &ch->inst_block); 1173 ch_inst_ptr = nvgpu_inst_block_addr(g, &ch->inst_block);
1139 if (inst_ptr == ch_inst_ptr) 1174 if (inst_ptr == ch_inst_ptr) {
1140 return ch; 1175 return ch;
1176 }
1141 1177
1142 gk20a_channel_put(ch); 1178 gk20a_channel_put(ch);
1143 } 1179 }
@@ -1199,34 +1235,37 @@ static const char * const does_not_exist[] = {
1199/* fill in mmu fault desc */ 1235/* fill in mmu fault desc */
1200void gk20a_fifo_get_mmu_fault_desc(struct mmu_fault_info *mmfault) 1236void gk20a_fifo_get_mmu_fault_desc(struct mmu_fault_info *mmfault)
1201{ 1237{
1202 if (mmfault->fault_type >= ARRAY_SIZE(gk20a_fault_type_descs)) 1238 if (mmfault->fault_type >= ARRAY_SIZE(gk20a_fault_type_descs)) {
1203 WARN_ON(mmfault->fault_type >= 1239 WARN_ON(mmfault->fault_type >=
1204 ARRAY_SIZE(gk20a_fault_type_descs)); 1240 ARRAY_SIZE(gk20a_fault_type_descs));
1205 else 1241 } else {
1206 mmfault->fault_type_desc = 1242 mmfault->fault_type_desc =
1207 gk20a_fault_type_descs[mmfault->fault_type]; 1243 gk20a_fault_type_descs[mmfault->fault_type];
1244 }
1208} 1245}
1209 1246
1210/* fill in mmu fault client description */ 1247/* fill in mmu fault client description */
1211void gk20a_fifo_get_mmu_fault_client_desc(struct mmu_fault_info *mmfault) 1248void gk20a_fifo_get_mmu_fault_client_desc(struct mmu_fault_info *mmfault)
1212{ 1249{
1213 if (mmfault->client_id >= ARRAY_SIZE(gk20a_hub_client_descs)) 1250 if (mmfault->client_id >= ARRAY_SIZE(gk20a_hub_client_descs)) {
1214 WARN_ON(mmfault->client_id >= 1251 WARN_ON(mmfault->client_id >=
1215 ARRAY_SIZE(gk20a_hub_client_descs)); 1252 ARRAY_SIZE(gk20a_hub_client_descs));
1216 else 1253 } else {
1217 mmfault->client_id_desc = 1254 mmfault->client_id_desc =
1218 gk20a_hub_client_descs[mmfault->client_id]; 1255 gk20a_hub_client_descs[mmfault->client_id];
1256 }
1219} 1257}
1220 1258
1221/* fill in mmu fault gpc description */ 1259/* fill in mmu fault gpc description */
1222void gk20a_fifo_get_mmu_fault_gpc_desc(struct mmu_fault_info *mmfault) 1260void gk20a_fifo_get_mmu_fault_gpc_desc(struct mmu_fault_info *mmfault)
1223{ 1261{
1224 if (mmfault->client_id >= ARRAY_SIZE(gk20a_gpc_client_descs)) 1262 if (mmfault->client_id >= ARRAY_SIZE(gk20a_gpc_client_descs)) {
1225 WARN_ON(mmfault->client_id >= 1263 WARN_ON(mmfault->client_id >=
1226 ARRAY_SIZE(gk20a_gpc_client_descs)); 1264 ARRAY_SIZE(gk20a_gpc_client_descs));
1227 else 1265 } else {
1228 mmfault->client_id_desc = 1266 mmfault->client_id_desc =
1229 gk20a_gpc_client_descs[mmfault->client_id]; 1267 gk20a_gpc_client_descs[mmfault->client_id];
1268 }
1230} 1269}
1231 1270
1232static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id, 1271static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id,
@@ -1236,8 +1275,9 @@ static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id,
1236 1275
1237 /* parse info */ 1276 /* parse info */
1238 mmfault->fault_type_desc = does_not_exist[0]; 1277 mmfault->fault_type_desc = does_not_exist[0];
1239 if (g->ops.fifo.get_mmu_fault_desc) 1278 if (g->ops.fifo.get_mmu_fault_desc) {
1240 g->ops.fifo.get_mmu_fault_desc(mmfault); 1279 g->ops.fifo.get_mmu_fault_desc(mmfault);
1280 }
1241 1281
1242 if (mmfault->client_type >= ARRAY_SIZE(engine_subid_descs)) { 1282 if (mmfault->client_type >= ARRAY_SIZE(engine_subid_descs)) {
1243 WARN_ON(mmfault->client_type >= ARRAY_SIZE(engine_subid_descs)); 1283 WARN_ON(mmfault->client_type >= ARRAY_SIZE(engine_subid_descs));
@@ -1250,12 +1290,13 @@ static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id,
1250 mmfault->client_id_desc = does_not_exist[0]; 1290 mmfault->client_id_desc = does_not_exist[0];
1251 if ((mmfault->client_type == 1291 if ((mmfault->client_type ==
1252 fifo_intr_mmu_fault_info_engine_subid_hub_v()) 1292 fifo_intr_mmu_fault_info_engine_subid_hub_v())
1253 && g->ops.fifo.get_mmu_fault_client_desc) 1293 && g->ops.fifo.get_mmu_fault_client_desc) {
1254 g->ops.fifo.get_mmu_fault_client_desc(mmfault); 1294 g->ops.fifo.get_mmu_fault_client_desc(mmfault);
1255 else if ((mmfault->client_type == 1295 } else if ((mmfault->client_type ==
1256 fifo_intr_mmu_fault_info_engine_subid_gpc_v()) 1296 fifo_intr_mmu_fault_info_engine_subid_gpc_v())
1257 && g->ops.fifo.get_mmu_fault_gpc_desc) 1297 && g->ops.fifo.get_mmu_fault_gpc_desc) {
1258 g->ops.fifo.get_mmu_fault_gpc_desc(mmfault); 1298 g->ops.fifo.get_mmu_fault_gpc_desc(mmfault);
1299 }
1259} 1300}
1260 1301
1261/* reads info from hardware and fills in mmu fault info record */ 1302/* reads info from hardware and fills in mmu fault info record */
@@ -1297,21 +1338,25 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
1297 1338
1298 nvgpu_log_fn(g, " "); 1339 nvgpu_log_fn(g, " ");
1299 1340
1300 if (!g) 1341 if (!g) {
1301 return; 1342 return;
1343 }
1302 1344
1303 engine_info = gk20a_fifo_get_engine_info(g, engine_id); 1345 engine_info = gk20a_fifo_get_engine_info(g, engine_id);
1304 1346
1305 if (engine_info) 1347 if (engine_info) {
1306 engine_enum = engine_info->engine_enum; 1348 engine_enum = engine_info->engine_enum;
1349 }
1307 1350
1308 if (engine_enum == ENGINE_INVAL_GK20A) 1351 if (engine_enum == ENGINE_INVAL_GK20A) {
1309 nvgpu_err(g, "unsupported engine_id %d", engine_id); 1352 nvgpu_err(g, "unsupported engine_id %d", engine_id);
1353 }
1310 1354
1311 if (engine_enum == ENGINE_GR_GK20A) { 1355 if (engine_enum == ENGINE_GR_GK20A) {
1312 if (g->support_pmu && g->can_elpg) { 1356 if (g->support_pmu && g->can_elpg) {
1313 if (nvgpu_pmu_disable_elpg(g)) 1357 if (nvgpu_pmu_disable_elpg(g)) {
1314 nvgpu_err(g, "failed to set disable elpg"); 1358 nvgpu_err(g, "failed to set disable elpg");
1359 }
1315 } 1360 }
1316 1361
1317#ifdef CONFIG_GK20A_CTXSW_TRACE 1362#ifdef CONFIG_GK20A_CTXSW_TRACE
@@ -1324,8 +1369,9 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
1324#endif 1369#endif
1325 if (!nvgpu_platform_is_simulation(g)) { 1370 if (!nvgpu_platform_is_simulation(g)) {
1326 /*HALT_PIPELINE method, halt GR engine*/ 1371 /*HALT_PIPELINE method, halt GR engine*/
1327 if (gr_gk20a_halt_pipe(g)) 1372 if (gr_gk20a_halt_pipe(g)) {
1328 nvgpu_err(g, "failed to HALT gr pipe"); 1373 nvgpu_err(g, "failed to HALT gr pipe");
1374 }
1329 /* 1375 /*
1330 * resetting engine using mc_enable_r() is not 1376 * resetting engine using mc_enable_r() is not
1331 * enough, we do full init sequence 1377 * enough, we do full init sequence
@@ -1337,8 +1383,9 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
1337 "HALT gr pipe not supported and " 1383 "HALT gr pipe not supported and "
1338 "gr cannot be reset without halting gr pipe"); 1384 "gr cannot be reset without halting gr pipe");
1339 } 1385 }
1340 if (g->support_pmu && g->can_elpg) 1386 if (g->support_pmu && g->can_elpg) {
1341 nvgpu_pmu_enable_elpg(g); 1387 nvgpu_pmu_enable_elpg(g);
1388 }
1342 } 1389 }
1343 if ((engine_enum == ENGINE_GRCE_GK20A) || 1390 if ((engine_enum == ENGINE_GRCE_GK20A) ||
1344 (engine_enum == ENGINE_ASYNC_CE_GK20A)) { 1391 (engine_enum == ENGINE_ASYNC_CE_GK20A)) {
@@ -1373,29 +1420,35 @@ bool gk20a_fifo_should_defer_engine_reset(struct gk20a *g, u32 engine_id,
1373 u32 engine_enum = ENGINE_INVAL_GK20A; 1420 u32 engine_enum = ENGINE_INVAL_GK20A;
1374 struct fifo_engine_info_gk20a *engine_info; 1421 struct fifo_engine_info_gk20a *engine_info;
1375 1422
1376 if (!g) 1423 if (!g) {
1377 return false; 1424 return false;
1425 }
1378 1426
1379 engine_info = gk20a_fifo_get_engine_info(g, engine_id); 1427 engine_info = gk20a_fifo_get_engine_info(g, engine_id);
1380 1428
1381 if (engine_info) 1429 if (engine_info) {
1382 engine_enum = engine_info->engine_enum; 1430 engine_enum = engine_info->engine_enum;
1431 }
1383 1432
1384 if (engine_enum == ENGINE_INVAL_GK20A) 1433 if (engine_enum == ENGINE_INVAL_GK20A) {
1385 return false; 1434 return false;
1435 }
1386 1436
1387 /* channel recovery is only deferred if an sm debugger 1437 /* channel recovery is only deferred if an sm debugger
1388 is attached and has MMU debug mode is enabled */ 1438 is attached and has MMU debug mode is enabled */
1389 if (!g->ops.gr.sm_debugger_attached(g) || 1439 if (!g->ops.gr.sm_debugger_attached(g) ||
1390 !g->ops.fb.is_debug_mode_enabled(g)) 1440 !g->ops.fb.is_debug_mode_enabled(g)) {
1391 return false; 1441 return false;
1442 }
1392 1443
1393 /* if this fault is fake (due to RC recovery), don't defer recovery */ 1444 /* if this fault is fake (due to RC recovery), don't defer recovery */
1394 if (fake_fault) 1445 if (fake_fault) {
1395 return false; 1446 return false;
1447 }
1396 1448
1397 if (engine_enum != ENGINE_GR_GK20A) 1449 if (engine_enum != ENGINE_GR_GK20A) {
1398 return false; 1450 return false;
1451 }
1399 1452
1400 return g->ops.fifo.is_fault_engine_subid_gpc(g, engine_subid); 1453 return g->ops.fifo.is_fault_engine_subid_gpc(g, engine_subid);
1401} 1454}
@@ -1405,12 +1458,14 @@ static bool gk20a_fifo_ch_timeout_debug_dump_state(struct gk20a *g,
1405 struct channel_gk20a *refch) 1458 struct channel_gk20a *refch)
1406{ 1459{
1407 bool verbose = true; 1460 bool verbose = true;
1408 if (!refch) 1461 if (!refch) {
1409 return verbose; 1462 return verbose;
1463 }
1410 1464
1411 if (nvgpu_is_error_notifier_set(refch, 1465 if (nvgpu_is_error_notifier_set(refch,
1412 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT)) 1466 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT)) {
1413 verbose = refch->timeout_debug_dump; 1467 verbose = refch->timeout_debug_dump;
1468 }
1414 1469
1415 return verbose; 1470 return verbose;
1416} 1471}
@@ -1499,15 +1554,17 @@ void gk20a_fifo_abort_tsg(struct gk20a *g, u32 tsgid, bool preempt)
1499 1554
1500 g->ops.fifo.disable_tsg(tsg); 1555 g->ops.fifo.disable_tsg(tsg);
1501 1556
1502 if (preempt) 1557 if (preempt) {
1503 g->ops.fifo.preempt_tsg(g, tsgid); 1558 g->ops.fifo.preempt_tsg(g, tsgid);
1559 }
1504 1560
1505 nvgpu_rwsem_down_read(&tsg->ch_list_lock); 1561 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
1506 nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { 1562 nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) {
1507 if (gk20a_channel_get(ch)) { 1563 if (gk20a_channel_get(ch)) {
1508 ch->has_timedout = true; 1564 ch->has_timedout = true;
1509 if (ch->g->ops.fifo.ch_abort_clean_up) 1565 if (ch->g->ops.fifo.ch_abort_clean_up) {
1510 ch->g->ops.fifo.ch_abort_clean_up(ch); 1566 ch->g->ops.fifo.ch_abort_clean_up(ch);
1567 }
1511 gk20a_channel_put(ch); 1568 gk20a_channel_put(ch);
1512 } 1569 }
1513 } 1570 }
@@ -1521,23 +1578,27 @@ int gk20a_fifo_deferred_reset(struct gk20a *g, struct channel_gk20a *ch)
1521 nvgpu_mutex_acquire(&g->dbg_sessions_lock); 1578 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1522 gr_gk20a_disable_ctxsw(g); 1579 gr_gk20a_disable_ctxsw(g);
1523 1580
1524 if (!g->fifo.deferred_reset_pending) 1581 if (!g->fifo.deferred_reset_pending) {
1525 goto clean_up; 1582 goto clean_up;
1583 }
1526 1584
1527 if (gk20a_is_channel_marked_as_tsg(ch)) 1585 if (gk20a_is_channel_marked_as_tsg(ch)) {
1528 engines = gk20a_fifo_engines_on_id(g, ch->tsgid, true); 1586 engines = gk20a_fifo_engines_on_id(g, ch->tsgid, true);
1529 else 1587 } else {
1530 engines = gk20a_fifo_engines_on_id(g, ch->chid, false); 1588 engines = gk20a_fifo_engines_on_id(g, ch->chid, false);
1531 if (!engines) 1589 }
1590 if (!engines) {
1532 goto clean_up; 1591 goto clean_up;
1592 }
1533 1593
1534 /* 1594 /*
1535 * If deferred reset is set for an engine, and channel is running 1595 * If deferred reset is set for an engine, and channel is running
1536 * on that engine, reset it 1596 * on that engine, reset it
1537 */ 1597 */
1538 for_each_set_bit(engine_id, &g->fifo.deferred_fault_engines, 32) { 1598 for_each_set_bit(engine_id, &g->fifo.deferred_fault_engines, 32) {
1539 if (BIT(engine_id) & engines) 1599 if (BIT(engine_id) & engines) {
1540 gk20a_fifo_reset_engine(g, engine_id); 1600 gk20a_fifo_reset_engine(g, engine_id);
1601 }
1541 } 1602 }
1542 1603
1543 g->fifo.deferred_fault_engines = 0; 1604 g->fifo.deferred_fault_engines = 0;
@@ -1568,18 +1629,22 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1568 1629
1569 /* Disable power management */ 1630 /* Disable power management */
1570 if (g->support_pmu && g->can_elpg) { 1631 if (g->support_pmu && g->can_elpg) {
1571 if (nvgpu_pmu_disable_elpg(g)) 1632 if (nvgpu_pmu_disable_elpg(g)) {
1572 nvgpu_err(g, "failed to set disable elpg"); 1633 nvgpu_err(g, "failed to set disable elpg");
1634 }
1573 } 1635 }
1574 if (g->ops.clock_gating.slcg_gr_load_gating_prod) 1636 if (g->ops.clock_gating.slcg_gr_load_gating_prod) {
1575 g->ops.clock_gating.slcg_gr_load_gating_prod(g, 1637 g->ops.clock_gating.slcg_gr_load_gating_prod(g,
1576 false); 1638 false);
1577 if (g->ops.clock_gating.slcg_perf_load_gating_prod) 1639 }
1640 if (g->ops.clock_gating.slcg_perf_load_gating_prod) {
1578 g->ops.clock_gating.slcg_perf_load_gating_prod(g, 1641 g->ops.clock_gating.slcg_perf_load_gating_prod(g,
1579 false); 1642 false);
1580 if (g->ops.clock_gating.slcg_ltc_load_gating_prod) 1643 }
1644 if (g->ops.clock_gating.slcg_ltc_load_gating_prod) {
1581 g->ops.clock_gating.slcg_ltc_load_gating_prod(g, 1645 g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
1582 false); 1646 false);
1647 }
1583 1648
1584 gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); 1649 gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN);
1585 1650
@@ -1673,9 +1738,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1673 fifo_engine_status_id_type_chid_v(); 1738 fifo_engine_status_id_type_chid_v();
1674 } 1739 }
1675 1740
1676 if (type == fifo_engine_status_id_type_tsgid_v()) 1741 if (type == fifo_engine_status_id_type_tsgid_v()) {
1677 tsg = &g->fifo.tsg[id]; 1742 tsg = &g->fifo.tsg[id];
1678 else if (type == fifo_engine_status_id_type_chid_v()) { 1743 } else if (type == fifo_engine_status_id_type_chid_v()) {
1679 ch = &g->fifo.channel[id]; 1744 ch = &g->fifo.channel[id];
1680 refch = gk20a_channel_get(ch); 1745 refch = gk20a_channel_get(ch);
1681 } 1746 }
@@ -1686,8 +1751,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1686 refch = ch; 1751 refch = ch;
1687 } 1752 }
1688 1753
1689 if (ch && gk20a_is_channel_marked_as_tsg(ch)) 1754 if (ch && gk20a_is_channel_marked_as_tsg(ch)) {
1690 tsg = &g->fifo.tsg[ch->tsgid]; 1755 tsg = &g->fifo.tsg[ch->tsgid];
1756 }
1691 1757
1692 /* check if engine reset should be deferred */ 1758 /* check if engine reset should be deferred */
1693 if (engine_id != FIFO_INVAL_ENGINE_ID) { 1759 if (engine_id != FIFO_INVAL_ENGINE_ID) {
@@ -1730,24 +1796,27 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1730 if (g->fifo.deferred_reset_pending) { 1796 if (g->fifo.deferred_reset_pending) {
1731 gk20a_disable_tsg(tsg); 1797 gk20a_disable_tsg(tsg);
1732 } else { 1798 } else {
1733 if (!fake_fault) 1799 if (!fake_fault) {
1734 gk20a_fifo_set_ctx_mmu_error_tsg(g, 1800 gk20a_fifo_set_ctx_mmu_error_tsg(g,
1735 tsg); 1801 tsg);
1802 }
1736 verbose = gk20a_fifo_error_tsg(g, tsg); 1803 verbose = gk20a_fifo_error_tsg(g, tsg);
1737 gk20a_fifo_abort_tsg(g, tsg->tsgid, false); 1804 gk20a_fifo_abort_tsg(g, tsg->tsgid, false);
1738 } 1805 }
1739 1806
1740 /* put back the ref taken early above */ 1807 /* put back the ref taken early above */
1741 if (refch) 1808 if (refch) {
1742 gk20a_channel_put(ch); 1809 gk20a_channel_put(ch);
1810 }
1743 } else if (ch) { 1811 } else if (ch) {
1744 if (refch) { 1812 if (refch) {
1745 if (g->fifo.deferred_reset_pending) { 1813 if (g->fifo.deferred_reset_pending) {
1746 g->ops.fifo.disable_channel(ch); 1814 g->ops.fifo.disable_channel(ch);
1747 } else { 1815 } else {
1748 if (!fake_fault) 1816 if (!fake_fault) {
1749 gk20a_fifo_set_ctx_mmu_error_ch( 1817 gk20a_fifo_set_ctx_mmu_error_ch(
1750 g, refch); 1818 g, refch);
1819 }
1751 1820
1752 verbose = gk20a_fifo_error_ch(g, 1821 verbose = gk20a_fifo_error_ch(g,
1753 refch); 1822 refch);
@@ -1765,8 +1834,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1765 } else if (mmfault_info.inst_ptr == 1834 } else if (mmfault_info.inst_ptr ==
1766 nvgpu_inst_block_addr(g, &g->mm.pmu.inst_block)) { 1835 nvgpu_inst_block_addr(g, &g->mm.pmu.inst_block)) {
1767 nvgpu_err(g, "mmu fault from pmu"); 1836 nvgpu_err(g, "mmu fault from pmu");
1768 } else 1837 } else {
1769 nvgpu_err(g, "couldn't locate channel for mmu fault"); 1838 nvgpu_err(g, "couldn't locate channel for mmu fault");
1839 }
1770 } 1840 }
1771 1841
1772 /* clear interrupt */ 1842 /* clear interrupt */
@@ -1782,8 +1852,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1782 gr_gpfifo_ctl_semaphore_access_enabled_f()); 1852 gr_gpfifo_ctl_semaphore_access_enabled_f());
1783 1853
1784 /* It is safe to enable ELPG again. */ 1854 /* It is safe to enable ELPG again. */
1785 if (g->support_pmu && g->can_elpg) 1855 if (g->support_pmu && g->can_elpg) {
1786 nvgpu_pmu_enable_elpg(g); 1856 nvgpu_pmu_enable_elpg(g);
1857 }
1787 1858
1788 return verbose; 1859 return verbose;
1789} 1860}
@@ -1856,8 +1927,9 @@ static u32 gk20a_fifo_engines_on_id(struct gk20a *g, u32 id, bool is_tsg)
1856 if ((is_tsg && type == 1927 if ((is_tsg && type ==
1857 fifo_engine_status_id_type_tsgid_v()) || 1928 fifo_engine_status_id_type_tsgid_v()) ||
1858 (!is_tsg && type == 1929 (!is_tsg && type ==
1859 fifo_engine_status_id_type_chid_v())) 1930 fifo_engine_status_id_type_chid_v())) {
1860 engines |= BIT(active_engine_id); 1931 engines |= BIT(active_engine_id);
1932 }
1861 } 1933 }
1862 } 1934 }
1863 1935
@@ -1875,17 +1947,18 @@ void gk20a_fifo_recover_ch(struct gk20a *g, u32 chid, bool verbose, int rc_type)
1875 1947
1876 engines = gk20a_fifo_engines_on_id(g, chid, false); 1948 engines = gk20a_fifo_engines_on_id(g, chid, false);
1877 1949
1878 if (engines) 1950 if (engines) {
1879 gk20a_fifo_recover(g, engines, chid, false, true, verbose, 1951 gk20a_fifo_recover(g, engines, chid, false, true, verbose,
1880 rc_type); 1952 rc_type);
1881 else { 1953 } else {
1882 struct channel_gk20a *ch = &g->fifo.channel[chid]; 1954 struct channel_gk20a *ch = &g->fifo.channel[chid];
1883 1955
1884 if (gk20a_channel_get(ch)) { 1956 if (gk20a_channel_get(ch)) {
1885 gk20a_channel_abort(ch, false); 1957 gk20a_channel_abort(ch, false);
1886 1958
1887 if (gk20a_fifo_error_ch(g, ch)) 1959 if (gk20a_fifo_error_ch(g, ch)) {
1888 gk20a_debug_dump(g); 1960 gk20a_debug_dump(g);
1961 }
1889 1962
1890 gk20a_channel_put(ch); 1963 gk20a_channel_put(ch);
1891 } 1964 }
@@ -1907,14 +1980,15 @@ void gk20a_fifo_recover_tsg(struct gk20a *g, u32 tsgid, bool verbose,
1907 1980
1908 engines = gk20a_fifo_engines_on_id(g, tsgid, true); 1981 engines = gk20a_fifo_engines_on_id(g, tsgid, true);
1909 1982
1910 if (engines) 1983 if (engines) {
1911 gk20a_fifo_recover(g, engines, tsgid, true, true, verbose, 1984 gk20a_fifo_recover(g, engines, tsgid, true, true, verbose,
1912 rc_type); 1985 rc_type);
1913 else { 1986 } else {
1914 struct tsg_gk20a *tsg = &g->fifo.tsg[tsgid]; 1987 struct tsg_gk20a *tsg = &g->fifo.tsg[tsgid];
1915 1988
1916 if (gk20a_fifo_error_tsg(g, tsg) && verbose) 1989 if (gk20a_fifo_error_tsg(g, tsg) && verbose) {
1917 gk20a_debug_dump(g); 1990 gk20a_debug_dump(g);
1991 }
1918 1992
1919 gk20a_fifo_abort_tsg(g, tsgid, false); 1993 gk20a_fifo_abort_tsg(g, tsgid, false);
1920 } 1994 }
@@ -1956,18 +2030,20 @@ void gk20a_fifo_teardown_ch_tsg(struct gk20a *g, u32 __engine_ids,
1956 for_each_set_bit(engine_id, &engine_ids, 32) { 2030 for_each_set_bit(engine_id, &engine_ids, 32) {
1957 u32 mmu_id = gk20a_engine_id_to_mmu_id(g, engine_id); 2031 u32 mmu_id = gk20a_engine_id_to_mmu_id(g, engine_id);
1958 2032
1959 if (mmu_id != FIFO_INVAL_ENGINE_ID) 2033 if (mmu_id != FIFO_INVAL_ENGINE_ID) {
1960 mmu_fault_engines |= BIT(mmu_id); 2034 mmu_fault_engines |= BIT(mmu_id);
2035 }
1961 } 2036 }
1962 } else { 2037 } else {
1963 /* store faulted engines in advance */ 2038 /* store faulted engines in advance */
1964 for_each_set_bit(engine_id, &_engine_ids, 32) { 2039 for_each_set_bit(engine_id, &_engine_ids, 32) {
1965 gk20a_fifo_get_faulty_id_type(g, engine_id, &ref_id, 2040 gk20a_fifo_get_faulty_id_type(g, engine_id, &ref_id,
1966 &ref_type); 2041 &ref_type);
1967 if (ref_type == fifo_engine_status_id_type_tsgid_v()) 2042 if (ref_type == fifo_engine_status_id_type_tsgid_v()) {
1968 ref_id_is_tsg = true; 2043 ref_id_is_tsg = true;
1969 else 2044 } else {
1970 ref_id_is_tsg = false; 2045 ref_id_is_tsg = false;
2046 }
1971 /* Reset *all* engines that use the 2047 /* Reset *all* engines that use the
1972 * same channel as faulty engine */ 2048 * same channel as faulty engine */
1973 for (i = 0; i < g->fifo.num_engines; i++) { 2049 for (i = 0; i < g->fifo.num_engines; i++) {
@@ -1980,8 +2056,9 @@ void gk20a_fifo_teardown_ch_tsg(struct gk20a *g, u32 __engine_ids,
1980 u32 mmu_id = gk20a_engine_id_to_mmu_id(g, active_engine_id); 2056 u32 mmu_id = gk20a_engine_id_to_mmu_id(g, active_engine_id);
1981 2057
1982 engine_ids |= BIT(active_engine_id); 2058 engine_ids |= BIT(active_engine_id);
1983 if (mmu_id != FIFO_INVAL_ENGINE_ID) 2059 if (mmu_id != FIFO_INVAL_ENGINE_ID) {
1984 mmu_fault_engines |= BIT(mmu_id); 2060 mmu_fault_engines |= BIT(mmu_id);
2061 }
1985 } 2062 }
1986 } 2063 }
1987 } 2064 }
@@ -2021,16 +2098,19 @@ void gk20a_fifo_recover(struct gk20a *g, u32 __engine_ids,
2021{ 2098{
2022 unsigned int id_type; 2099 unsigned int id_type;
2023 2100
2024 if (verbose) 2101 if (verbose) {
2025 gk20a_debug_dump(g); 2102 gk20a_debug_dump(g);
2103 }
2026 2104
2027 if (g->ops.ltc.flush) 2105 if (g->ops.ltc.flush) {
2028 g->ops.ltc.flush(g); 2106 g->ops.ltc.flush(g);
2107 }
2029 2108
2030 if (id_is_known) 2109 if (id_is_known) {
2031 id_type = id_is_tsg ? ID_TYPE_TSG : ID_TYPE_CHANNEL; 2110 id_type = id_is_tsg ? ID_TYPE_TSG : ID_TYPE_CHANNEL;
2032 else 2111 } else {
2033 id_type = ID_TYPE_UNKNOWN; 2112 id_type = ID_TYPE_UNKNOWN;
2113 }
2034 2114
2035 g->ops.fifo.teardown_ch_tsg(g, __engine_ids, hw_id, id_type, 2115 g->ops.fifo.teardown_ch_tsg(g, __engine_ids, hw_id, id_type,
2036 rc_type, NULL); 2116 rc_type, NULL);
@@ -2080,11 +2160,13 @@ int gk20a_fifo_tsg_unbind_channel_verify_status(struct channel_gk20a *ch)
2080 return -EINVAL; 2160 return -EINVAL;
2081 } 2161 }
2082 2162
2083 if (g->ops.fifo.tsg_verify_status_ctx_reload) 2163 if (g->ops.fifo.tsg_verify_status_ctx_reload) {
2084 g->ops.fifo.tsg_verify_status_ctx_reload(ch); 2164 g->ops.fifo.tsg_verify_status_ctx_reload(ch);
2165 }
2085 2166
2086 if (g->ops.fifo.tsg_verify_status_faulted) 2167 if (g->ops.fifo.tsg_verify_status_faulted) {
2087 g->ops.fifo.tsg_verify_status_faulted(ch); 2168 g->ops.fifo.tsg_verify_status_faulted(ch);
2169 }
2088 2170
2089 return 0; 2171 return 0;
2090} 2172}
@@ -2106,19 +2188,22 @@ int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch)
2106 g->ops.fifo.disable_tsg(tsg); 2188 g->ops.fifo.disable_tsg(tsg);
2107 2189
2108 err = g->ops.fifo.preempt_tsg(g, tsg->tsgid); 2190 err = g->ops.fifo.preempt_tsg(g, tsg->tsgid);
2109 if (err) 2191 if (err) {
2110 goto fail_enable_tsg; 2192 goto fail_enable_tsg;
2193 }
2111 2194
2112 if (g->ops.fifo.tsg_verify_channel_status && !tsg_timedout) { 2195 if (g->ops.fifo.tsg_verify_channel_status && !tsg_timedout) {
2113 err = g->ops.fifo.tsg_verify_channel_status(ch); 2196 err = g->ops.fifo.tsg_verify_channel_status(ch);
2114 if (err) 2197 if (err) {
2115 goto fail_enable_tsg; 2198 goto fail_enable_tsg;
2199 }
2116 } 2200 }
2117 2201
2118 /* Channel should be seen as TSG channel while updating runlist */ 2202 /* Channel should be seen as TSG channel while updating runlist */
2119 err = channel_gk20a_update_runlist(ch, false); 2203 err = channel_gk20a_update_runlist(ch, false);
2120 if (err) 2204 if (err) {
2121 goto fail_enable_tsg; 2205 goto fail_enable_tsg;
2206 }
2122 2207
2123 /* Remove channel from TSG and re-enable rest of the channels */ 2208 /* Remove channel from TSG and re-enable rest of the channels */
2124 nvgpu_rwsem_down_write(&tsg->ch_list_lock); 2209 nvgpu_rwsem_down_write(&tsg->ch_list_lock);
@@ -2131,17 +2216,20 @@ int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch)
2131 * Note that we can skip disabling and preempting TSG too in case of 2216 * Note that we can skip disabling and preempting TSG too in case of
2132 * time out, but we keep that to ensure TSG is kicked out 2217 * time out, but we keep that to ensure TSG is kicked out
2133 */ 2218 */
2134 if (!tsg_timedout) 2219 if (!tsg_timedout) {
2135 g->ops.fifo.enable_tsg(tsg); 2220 g->ops.fifo.enable_tsg(tsg);
2221 }
2136 2222
2137 if (ch->g->ops.fifo.ch_abort_clean_up) 2223 if (ch->g->ops.fifo.ch_abort_clean_up) {
2138 ch->g->ops.fifo.ch_abort_clean_up(ch); 2224 ch->g->ops.fifo.ch_abort_clean_up(ch);
2225 }
2139 2226
2140 return 0; 2227 return 0;
2141 2228
2142fail_enable_tsg: 2229fail_enable_tsg:
2143 if (!tsg_timedout) 2230 if (!tsg_timedout) {
2144 g->ops.fifo.enable_tsg(tsg); 2231 g->ops.fifo.enable_tsg(tsg);
2232 }
2145 return err; 2233 return err;
2146} 2234}
2147 2235
@@ -2225,9 +2313,10 @@ bool gk20a_fifo_check_ch_ctxsw_timeout(struct channel_gk20a *ch,
2225 &progress); 2313 &progress);
2226 *verbose = ch->timeout_debug_dump; 2314 *verbose = ch->timeout_debug_dump;
2227 *ms = ch->timeout_accumulated_ms; 2315 *ms = ch->timeout_accumulated_ms;
2228 if (recover) 2316 if (recover) {
2229 g->ops.fifo.set_error_notifier(ch, 2317 g->ops.fifo.set_error_notifier(ch,
2230 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT); 2318 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT);
2319 }
2231 2320
2232 gk20a_channel_put(ch); 2321 gk20a_channel_put(ch);
2233 } 2322 }
@@ -2255,8 +2344,9 @@ bool gk20a_fifo_check_tsg_ctxsw_timeout(struct tsg_gk20a *tsg,
2255 if (gk20a_channel_get(ch)) { 2344 if (gk20a_channel_get(ch)) {
2256 recover = gk20a_channel_update_and_check_timeout(ch, 2345 recover = gk20a_channel_update_and_check_timeout(ch,
2257 *ms, &progress); 2346 *ms, &progress);
2258 if (progress || recover) 2347 if (progress || recover) {
2259 break; 2348 break;
2349 }
2260 gk20a_channel_put(ch); 2350 gk20a_channel_put(ch);
2261 } 2351 }
2262 } 2352 }
@@ -2481,9 +2571,11 @@ static bool gk20a_fifo_is_sw_method_subch(struct gk20a *g, int pbdma_id,
2481 pbdma_method_subch = pbdma_method0_subch_v( 2571 pbdma_method_subch = pbdma_method0_subch_v(
2482 gk20a_readl(g, pbdma_method_reg)); 2572 gk20a_readl(g, pbdma_method_reg));
2483 2573
2484 if (pbdma_method_subch == 5 || pbdma_method_subch == 6 || 2574 if (pbdma_method_subch == 5 ||
2485 pbdma_method_subch == 7) 2575 pbdma_method_subch == 6 ||
2576 pbdma_method_subch == 7) {
2486 return true; 2577 return true;
2578 }
2487 2579
2488 return false; 2580 return false;
2489} 2581}
@@ -2565,9 +2657,10 @@ unsigned int gk20a_fifo_handle_pbdma_intr_0(struct gk20a *g, u32 pbdma_id,
2565 2657
2566 for (i = 0; i < 4; i++) { 2658 for (i = 0; i < 4; i++) {
2567 if (gk20a_fifo_is_sw_method_subch(g, 2659 if (gk20a_fifo_is_sw_method_subch(g,
2568 pbdma_id, i)) 2660 pbdma_id, i)) {
2569 gk20a_fifo_reset_pbdma_method(g, 2661 gk20a_fifo_reset_pbdma_method(g,
2570 pbdma_id, i); 2662 pbdma_id, i);
2663 }
2571 } 2664 }
2572 rc_type = RC_TYPE_PBDMA_FAULT; 2665 rc_type = RC_TYPE_PBDMA_FAULT;
2573 } 2666 }
@@ -2649,8 +2742,9 @@ u32 gk20a_fifo_handle_pbdma_intr(struct gk20a *g, struct fifo_gk20a *f,
2649 pbdma_id, pbdma_intr_0); 2742 pbdma_id, pbdma_intr_0);
2650 2743
2651 if (g->ops.fifo.handle_pbdma_intr_0(g, pbdma_id, pbdma_intr_0, 2744 if (g->ops.fifo.handle_pbdma_intr_0(g, pbdma_id, pbdma_intr_0,
2652 &handled, &error_notifier) != RC_TYPE_NO_RC) 2745 &handled, &error_notifier) != RC_TYPE_NO_RC) {
2653 rc_type = RC_TYPE_PBDMA_FAULT; 2746 rc_type = RC_TYPE_PBDMA_FAULT;
2747 }
2654 gk20a_writel(g, pbdma_intr_0_r(pbdma_id), pbdma_intr_0); 2748 gk20a_writel(g, pbdma_intr_0_r(pbdma_id), pbdma_intr_0);
2655 } 2749 }
2656 2750
@@ -2660,13 +2754,15 @@ u32 gk20a_fifo_handle_pbdma_intr(struct gk20a *g, struct fifo_gk20a *f,
2660 pbdma_id, pbdma_intr_1); 2754 pbdma_id, pbdma_intr_1);
2661 2755
2662 if (g->ops.fifo.handle_pbdma_intr_1(g, pbdma_id, pbdma_intr_1, 2756 if (g->ops.fifo.handle_pbdma_intr_1(g, pbdma_id, pbdma_intr_1,
2663 &handled, &error_notifier) != RC_TYPE_NO_RC) 2757 &handled, &error_notifier) != RC_TYPE_NO_RC) {
2664 rc_type = RC_TYPE_PBDMA_FAULT; 2758 rc_type = RC_TYPE_PBDMA_FAULT;
2759 }
2665 gk20a_writel(g, pbdma_intr_1_r(pbdma_id), pbdma_intr_1); 2760 gk20a_writel(g, pbdma_intr_1_r(pbdma_id), pbdma_intr_1);
2666 } 2761 }
2667 2762
2668 if (rc == RC_YES && rc_type == RC_TYPE_PBDMA_FAULT) 2763 if (rc == RC_YES && rc_type == RC_TYPE_PBDMA_FAULT) {
2669 gk20a_fifo_pbdma_fault_rc(g, f, pbdma_id, error_notifier); 2764 gk20a_fifo_pbdma_fault_rc(g, f, pbdma_id, error_notifier);
2765 }
2670 2766
2671 return handled; 2767 return handled;
2672} 2768}
@@ -2708,14 +2804,17 @@ void gk20a_fifo_isr(struct gk20a *g)
2708 gk20a_fifo_handle_runlist_event(g); 2804 gk20a_fifo_handle_runlist_event(g);
2709 clear_intr |= fifo_intr_0_runlist_event_pending_f(); 2805 clear_intr |= fifo_intr_0_runlist_event_pending_f();
2710 } 2806 }
2711 if (fifo_intr & fifo_intr_0_pbdma_intr_pending_f()) 2807 if (fifo_intr & fifo_intr_0_pbdma_intr_pending_f()) {
2712 clear_intr |= fifo_pbdma_isr(g, fifo_intr); 2808 clear_intr |= fifo_pbdma_isr(g, fifo_intr);
2809 }
2713 2810
2714 if (g->ops.fifo.handle_ctxsw_timeout) 2811 if (g->ops.fifo.handle_ctxsw_timeout) {
2715 g->ops.fifo.handle_ctxsw_timeout(g, fifo_intr); 2812 g->ops.fifo.handle_ctxsw_timeout(g, fifo_intr);
2813 }
2716 2814
2717 if (unlikely(fifo_intr & error_intr_mask)) 2815 if (unlikely(fifo_intr & error_intr_mask)) {
2718 clear_intr = fifo_error_isr(g, fifo_intr); 2816 clear_intr = fifo_error_isr(g, fifo_intr);
2817 }
2719 2818
2720 nvgpu_mutex_release(&g->fifo.intr.isr.mutex); 2819 nvgpu_mutex_release(&g->fifo.intr.isr.mutex);
2721 } 2820 }
@@ -2731,8 +2830,9 @@ u32 gk20a_fifo_nonstall_isr(struct gk20a *g)
2731 2830
2732 nvgpu_log(g, gpu_dbg_intr, "fifo nonstall isr %08x\n", fifo_intr); 2831 nvgpu_log(g, gpu_dbg_intr, "fifo nonstall isr %08x\n", fifo_intr);
2733 2832
2734 if (fifo_intr & fifo_intr_0_channel_intr_pending_f()) 2833 if (fifo_intr & fifo_intr_0_channel_intr_pending_f()) {
2735 clear_intr = fifo_intr_0_channel_intr_pending_f(); 2834 clear_intr = fifo_intr_0_channel_intr_pending_f();
2835 }
2736 2836
2737 gk20a_writel(g, fifo_intr_0_r(), clear_intr); 2837 gk20a_writel(g, fifo_intr_0_r(), clear_intr);
2738 2838
@@ -2741,14 +2841,15 @@ u32 gk20a_fifo_nonstall_isr(struct gk20a *g)
2741 2841
2742void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg) 2842void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg)
2743{ 2843{
2744 if (is_tsg) 2844 if (is_tsg) {
2745 gk20a_writel(g, fifo_preempt_r(), 2845 gk20a_writel(g, fifo_preempt_r(),
2746 fifo_preempt_id_f(id) | 2846 fifo_preempt_id_f(id) |
2747 fifo_preempt_type_tsg_f()); 2847 fifo_preempt_type_tsg_f());
2748 else 2848 } else {
2749 gk20a_writel(g, fifo_preempt_r(), 2849 gk20a_writel(g, fifo_preempt_r(),
2750 fifo_preempt_chid_f(id) | 2850 fifo_preempt_chid_f(id) |
2751 fifo_preempt_type_channel_f()); 2851 fifo_preempt_type_channel_f());
2852 }
2752} 2853}
2753 2854
2754static u32 gk20a_fifo_get_preempt_timeout(struct gk20a *g) 2855static u32 gk20a_fifo_get_preempt_timeout(struct gk20a *g)
@@ -2802,8 +2903,9 @@ void gk20a_fifo_preempt_timeout_rc(struct gk20a *g, u32 id,
2802 nvgpu_rwsem_down_read(&tsg->ch_list_lock); 2903 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
2803 nvgpu_list_for_each_entry(ch, &tsg->ch_list, 2904 nvgpu_list_for_each_entry(ch, &tsg->ch_list,
2804 channel_gk20a, ch_entry) { 2905 channel_gk20a, ch_entry) {
2805 if (!gk20a_channel_get(ch)) 2906 if (!gk20a_channel_get(ch)) {
2806 continue; 2907 continue;
2908 }
2807 g->ops.fifo.set_error_notifier(ch, 2909 g->ops.fifo.set_error_notifier(ch,
2808 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT); 2910 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT);
2809 gk20a_channel_put(ch); 2911 gk20a_channel_put(ch);
@@ -2854,8 +2956,9 @@ int gk20a_fifo_preempt_channel(struct gk20a *g, u32 chid)
2854 u32 i; 2956 u32 i;
2855 2957
2856 nvgpu_log_fn(g, "chid: %d", chid); 2958 nvgpu_log_fn(g, "chid: %d", chid);
2857 if (chid == FIFO_INVAL_CHANNEL_ID) 2959 if (chid == FIFO_INVAL_CHANNEL_ID) {
2858 return 0; 2960 return 0;
2961 }
2859 2962
2860 /* we have no idea which runlist we are using. lock all */ 2963 /* we have no idea which runlist we are using. lock all */
2861 for (i = 0; i < g->fifo.max_runlists; i++) { 2964 for (i = 0; i < g->fifo.max_runlists; i++) {
@@ -2866,8 +2969,9 @@ int gk20a_fifo_preempt_channel(struct gk20a *g, u32 chid)
2866 2969
2867 ret = __locked_fifo_preempt(g, chid, false); 2970 ret = __locked_fifo_preempt(g, chid, false);
2868 2971
2869 if (!mutex_ret) 2972 if (!mutex_ret) {
2870 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 2973 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
2974 }
2871 2975
2872 for (i = 0; i < g->fifo.max_runlists; i++) { 2976 for (i = 0; i < g->fifo.max_runlists; i++) {
2873 nvgpu_mutex_release(&f->runlist_info[i].runlist_lock); 2977 nvgpu_mutex_release(&f->runlist_info[i].runlist_lock);
@@ -2896,8 +3000,9 @@ int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid)
2896 u32 i; 3000 u32 i;
2897 3001
2898 nvgpu_log_fn(g, "tsgid: %d", tsgid); 3002 nvgpu_log_fn(g, "tsgid: %d", tsgid);
2899 if (tsgid == FIFO_INVAL_TSG_ID) 3003 if (tsgid == FIFO_INVAL_TSG_ID) {
2900 return 0; 3004 return 0;
3005 }
2901 3006
2902 /* we have no idea which runlist we are using. lock all */ 3007 /* we have no idea which runlist we are using. lock all */
2903 for (i = 0; i < g->fifo.max_runlists; i++) { 3008 for (i = 0; i < g->fifo.max_runlists; i++) {
@@ -2908,8 +3013,9 @@ int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid)
2908 3013
2909 ret = __locked_fifo_preempt(g, tsgid, true); 3014 ret = __locked_fifo_preempt(g, tsgid, true);
2910 3015
2911 if (!mutex_ret) 3016 if (!mutex_ret) {
2912 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 3017 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
3018 }
2913 3019
2914 for (i = 0; i < g->fifo.max_runlists; i++) { 3020 for (i = 0; i < g->fifo.max_runlists; i++) {
2915 nvgpu_mutex_release(&f->runlist_info[i].runlist_lock); 3021 nvgpu_mutex_release(&f->runlist_info[i].runlist_lock);
@@ -2931,10 +3037,11 @@ int gk20a_fifo_preempt(struct gk20a *g, struct channel_gk20a *ch)
2931{ 3037{
2932 int err; 3038 int err;
2933 3039
2934 if (gk20a_is_channel_marked_as_tsg(ch)) 3040 if (gk20a_is_channel_marked_as_tsg(ch)) {
2935 err = g->ops.fifo.preempt_tsg(ch->g, ch->tsgid); 3041 err = g->ops.fifo.preempt_tsg(ch->g, ch->tsgid);
2936 else 3042 } else {
2937 err = g->ops.fifo.preempt_channel(ch->g, ch->chid); 3043 err = g->ops.fifo.preempt_channel(ch->g, ch->chid);
3044 }
2938 3045
2939 return err; 3046 return err;
2940} 3047}
@@ -2946,10 +3053,11 @@ static void gk20a_fifo_sched_disable_rw(struct gk20a *g, u32 runlists_mask,
2946 3053
2947 reg_val = gk20a_readl(g, fifo_sched_disable_r()); 3054 reg_val = gk20a_readl(g, fifo_sched_disable_r());
2948 3055
2949 if (runlist_state == RUNLIST_DISABLED) 3056 if (runlist_state == RUNLIST_DISABLED) {
2950 reg_val |= runlists_mask; 3057 reg_val |= runlists_mask;
2951 else 3058 } else {
2952 reg_val &= (~runlists_mask); 3059 reg_val &= (~runlists_mask);
3060 }
2953 3061
2954 gk20a_writel(g, fifo_sched_disable_r(), reg_val); 3062 gk20a_writel(g, fifo_sched_disable_r(), reg_val);
2955 3063
@@ -2968,8 +3076,9 @@ void gk20a_fifo_set_runlist_state(struct gk20a *g, u32 runlists_mask,
2968 3076
2969 gk20a_fifo_sched_disable_rw(g, runlists_mask, runlist_state); 3077 gk20a_fifo_sched_disable_rw(g, runlists_mask, runlist_state);
2970 3078
2971 if (!mutex_ret) 3079 if (!mutex_ret) {
2972 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 3080 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
3081 }
2973} 3082}
2974 3083
2975void gk20a_fifo_enable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg) 3084void gk20a_fifo_enable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg)
@@ -3030,8 +3139,9 @@ int gk20a_fifo_disable_engine_activity(struct gk20a *g,
3030 gr_stat = 3139 gr_stat =
3031 gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id)); 3140 gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
3032 if (fifo_engine_status_engine_v(gr_stat) == 3141 if (fifo_engine_status_engine_v(gr_stat) ==
3033 fifo_engine_status_engine_busy_v() && !wait_for_idle) 3142 fifo_engine_status_engine_busy_v() && !wait_for_idle) {
3034 return -EBUSY; 3143 return -EBUSY;
3144 }
3035 3145
3036 mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 3146 mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
3037 3147
@@ -3042,43 +3152,49 @@ int gk20a_fifo_disable_engine_activity(struct gk20a *g,
3042 pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(eng_info->pbdma_id)); 3152 pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(eng_info->pbdma_id));
3043 chan_stat = fifo_pbdma_status_chan_status_v(pbdma_stat); 3153 chan_stat = fifo_pbdma_status_chan_status_v(pbdma_stat);
3044 if (chan_stat == fifo_pbdma_status_chan_status_valid_v() || 3154 if (chan_stat == fifo_pbdma_status_chan_status_valid_v() ||
3045 chan_stat == fifo_pbdma_status_chan_status_chsw_save_v()) 3155 chan_stat == fifo_pbdma_status_chan_status_chsw_save_v()) {
3046 pbdma_chid = fifo_pbdma_status_id_v(pbdma_stat); 3156 pbdma_chid = fifo_pbdma_status_id_v(pbdma_stat);
3047 else if (chan_stat == fifo_pbdma_status_chan_status_chsw_load_v() || 3157 } else if (chan_stat == fifo_pbdma_status_chan_status_chsw_load_v() ||
3048 chan_stat == fifo_pbdma_status_chan_status_chsw_switch_v()) 3158 chan_stat == fifo_pbdma_status_chan_status_chsw_switch_v()) {
3049 pbdma_chid = fifo_pbdma_status_next_id_v(pbdma_stat); 3159 pbdma_chid = fifo_pbdma_status_next_id_v(pbdma_stat);
3160 }
3050 3161
3051 if (pbdma_chid != FIFO_INVAL_CHANNEL_ID) { 3162 if (pbdma_chid != FIFO_INVAL_CHANNEL_ID) {
3052 err = g->ops.fifo.preempt_channel(g, pbdma_chid); 3163 err = g->ops.fifo.preempt_channel(g, pbdma_chid);
3053 if (err) 3164 if (err) {
3054 goto clean_up; 3165 goto clean_up;
3166 }
3055 } 3167 }
3056 3168
3057 /* chid from engine status */ 3169 /* chid from engine status */
3058 eng_stat = gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id)); 3170 eng_stat = gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
3059 ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); 3171 ctx_stat = fifo_engine_status_ctx_status_v(eng_stat);
3060 if (ctx_stat == fifo_engine_status_ctx_status_valid_v() || 3172 if (ctx_stat == fifo_engine_status_ctx_status_valid_v() ||
3061 ctx_stat == fifo_engine_status_ctx_status_ctxsw_save_v()) 3173 ctx_stat == fifo_engine_status_ctx_status_ctxsw_save_v()) {
3062 engine_chid = fifo_engine_status_id_v(eng_stat); 3174 engine_chid = fifo_engine_status_id_v(eng_stat);
3063 else if (ctx_stat == fifo_engine_status_ctx_status_ctxsw_load_v() || 3175 } else if (ctx_stat == fifo_engine_status_ctx_status_ctxsw_load_v() ||
3064 ctx_stat == fifo_engine_status_ctx_status_ctxsw_switch_v()) 3176 ctx_stat == fifo_engine_status_ctx_status_ctxsw_switch_v()) {
3065 engine_chid = fifo_engine_status_next_id_v(eng_stat); 3177 engine_chid = fifo_engine_status_next_id_v(eng_stat);
3178 }
3066 3179
3067 if (engine_chid != FIFO_INVAL_ENGINE_ID && engine_chid != pbdma_chid) { 3180 if (engine_chid != FIFO_INVAL_ENGINE_ID && engine_chid != pbdma_chid) {
3068 err = g->ops.fifo.preempt_channel(g, engine_chid); 3181 err = g->ops.fifo.preempt_channel(g, engine_chid);
3069 if (err) 3182 if (err) {
3070 goto clean_up; 3183 goto clean_up;
3184 }
3071 } 3185 }
3072 3186
3073clean_up: 3187clean_up:
3074 if (!mutex_ret) 3188 if (!mutex_ret) {
3075 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 3189 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
3190 }
3076 3191
3077 if (err) { 3192 if (err) {
3078 nvgpu_log_fn(g, "failed"); 3193 nvgpu_log_fn(g, "failed");
3079 if (gk20a_fifo_enable_engine_activity(g, eng_info)) 3194 if (gk20a_fifo_enable_engine_activity(g, eng_info)) {
3080 nvgpu_err(g, 3195 nvgpu_err(g,
3081 "failed to enable gr engine activity"); 3196 "failed to enable gr engine activity");
3197 }
3082 } else { 3198 } else {
3083 nvgpu_log_fn(g, "done"); 3199 nvgpu_log_fn(g, "done");
3084 } 3200 }
@@ -3110,10 +3226,11 @@ int gk20a_fifo_disable_all_engine_activity(struct gk20a *g,
3110 active_engine_id = g->fifo.active_engines_list[i]; 3226 active_engine_id = g->fifo.active_engines_list[i];
3111 err = gk20a_fifo_enable_engine_activity(g, 3227 err = gk20a_fifo_enable_engine_activity(g,
3112 &g->fifo.engine_info[active_engine_id]); 3228 &g->fifo.engine_info[active_engine_id]);
3113 if (err) 3229 if (err) {
3114 nvgpu_err(g, 3230 nvgpu_err(g,
3115 "failed to re-enable engine %d activity", 3231 "failed to re-enable engine %d activity",
3116 active_engine_id); 3232 active_engine_id);
3233 }
3117 } 3234 }
3118 } 3235 }
3119 3236
@@ -3133,13 +3250,15 @@ static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id)
3133 fifo_engine_status_engine_busy_v(); 3250 fifo_engine_status_engine_busy_v();
3134 3251
3135 if (engine_busy && 3252 if (engine_busy &&
3136 (f->engine_info[active_engine_id].runlist_id == runlist_id)) 3253 (f->engine_info[active_engine_id].runlist_id == runlist_id)) {
3137 engines |= BIT(active_engine_id); 3254 engines |= BIT(active_engine_id);
3255 }
3138 } 3256 }
3139 3257
3140 if (engines) 3258 if (engines) {
3141 gk20a_fifo_recover(g, engines, ~(u32)0, false, false, true, 3259 gk20a_fifo_recover(g, engines, ~(u32)0, false, false, true,
3142 RC_TYPE_RUNLIST_UPDATE_TIMEOUT); 3260 RC_TYPE_RUNLIST_UPDATE_TIMEOUT);
3261 }
3143} 3262}
3144 3263
3145int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id) 3264int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id)
@@ -3177,16 +3296,17 @@ void gk20a_get_tsg_runlist_entry(struct tsg_gk20a *tsg, u32 *runlist)
3177 ram_rl_entry_type_tsg_f() | 3296 ram_rl_entry_type_tsg_f() |
3178 ram_rl_entry_tsg_length_f(tsg->num_active_channels); 3297 ram_rl_entry_tsg_length_f(tsg->num_active_channels);
3179 3298
3180 if (tsg->timeslice_timeout) 3299 if (tsg->timeslice_timeout) {
3181 runlist_entry_0 |= 3300 runlist_entry_0 |=
3182 ram_rl_entry_timeslice_scale_f(tsg->timeslice_scale) | 3301 ram_rl_entry_timeslice_scale_f(tsg->timeslice_scale) |
3183 ram_rl_entry_timeslice_timeout_f(tsg->timeslice_timeout); 3302 ram_rl_entry_timeslice_timeout_f(tsg->timeslice_timeout);
3184 else 3303 } else {
3185 runlist_entry_0 |= 3304 runlist_entry_0 |=
3186 ram_rl_entry_timeslice_scale_f( 3305 ram_rl_entry_timeslice_scale_f(
3187 NVGPU_FIFO_DEFAULT_TIMESLICE_SCALE) | 3306 NVGPU_FIFO_DEFAULT_TIMESLICE_SCALE) |
3188 ram_rl_entry_timeslice_timeout_f( 3307 ram_rl_entry_timeslice_timeout_f(
3189 NVGPU_FIFO_DEFAULT_TIMESLICE_TIMEOUT); 3308 NVGPU_FIFO_DEFAULT_TIMESLICE_TIMEOUT);
3309 }
3190 3310
3191 runlist[0] = runlist_entry_0; 3311 runlist[0] = runlist_entry_0;
3192 runlist[1] = 0; 3312 runlist[1] = 0;
@@ -3230,8 +3350,9 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3230 for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) { 3350 for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) {
3231 struct tsg_gk20a *tsg = &f->tsg[tsgid]; 3351 struct tsg_gk20a *tsg = &f->tsg[tsgid];
3232 3352
3233 if (tsg->interleave_level != cur_level) 3353 if (tsg->interleave_level != cur_level) {
3234 continue; 3354 continue;
3355 }
3235 3356
3236 if (!last_level && !skip_next) { 3357 if (!last_level && !skip_next) {
3237 runlist_entry = gk20a_runlist_construct_locked(f, 3358 runlist_entry = gk20a_runlist_construct_locked(f,
@@ -3241,8 +3362,9 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3241 interleave_enabled, 3362 interleave_enabled,
3242 false, 3363 false,
3243 entries_left); 3364 entries_left);
3244 if (!interleave_enabled) 3365 if (!interleave_enabled) {
3245 skip_next = true; 3366 skip_next = true;
3367 }
3246 } 3368 }
3247 3369
3248 if (*entries_left == 0U) { 3370 if (*entries_left == 0U) {
@@ -3263,8 +3385,9 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3263 nvgpu_list_for_each_entry(ch, &tsg->ch_list, 3385 nvgpu_list_for_each_entry(ch, &tsg->ch_list,
3264 channel_gk20a, ch_entry) { 3386 channel_gk20a, ch_entry) {
3265 if (!test_bit(ch->chid, 3387 if (!test_bit(ch->chid,
3266 runlist->active_channels)) 3388 runlist->active_channels)) {
3267 continue; 3389 continue;
3390 }
3268 3391
3269 if (*entries_left == 0U) { 3392 if (*entries_left == 0U) {
3270 nvgpu_rwsem_up_read(&tsg->ch_list_lock); 3393 nvgpu_rwsem_up_read(&tsg->ch_list_lock);
@@ -3285,7 +3408,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3285 } 3408 }
3286 3409
3287 /* append entries from higher level if this level is empty */ 3410 /* append entries from higher level if this level is empty */
3288 if (!count && !last_level) 3411 if (!count && !last_level) {
3289 runlist_entry = gk20a_runlist_construct_locked(f, 3412 runlist_entry = gk20a_runlist_construct_locked(f,
3290 runlist, 3413 runlist,
3291 cur_level + 1, 3414 cur_level + 1,
@@ -3293,6 +3416,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3293 interleave_enabled, 3416 interleave_enabled,
3294 true, 3417 true,
3295 entries_left); 3418 entries_left);
3419 }
3296 3420
3297 /* 3421 /*
3298 * if previous and this level have entries, append 3422 * if previous and this level have entries, append
@@ -3300,7 +3424,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3300 * 3424 *
3301 * ex. dropping from MEDIUM to LOW, need to insert HIGH 3425 * ex. dropping from MEDIUM to LOW, need to insert HIGH
3302 */ 3426 */
3303 if (interleave_enabled && count && !prev_empty && !last_level) 3427 if (interleave_enabled && count && !prev_empty && !last_level) {
3304 runlist_entry = gk20a_runlist_construct_locked(f, 3428 runlist_entry = gk20a_runlist_construct_locked(f,
3305 runlist, 3429 runlist,
3306 cur_level + 1, 3430 cur_level + 1,
@@ -3308,6 +3432,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3308 interleave_enabled, 3432 interleave_enabled,
3309 false, 3433 false,
3310 entries_left); 3434 entries_left);
3435 }
3311 return runlist_entry; 3436 return runlist_entry;
3312} 3437}
3313 3438
@@ -3328,8 +3453,9 @@ int gk20a_fifo_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice)
3328 struct gk20a *g = tsg->g; 3453 struct gk20a *g = tsg->g;
3329 3454
3330 if (timeslice < g->min_timeslice_us || 3455 if (timeslice < g->min_timeslice_us ||
3331 timeslice > g->max_timeslice_us) 3456 timeslice > g->max_timeslice_us) {
3332 return -EINVAL; 3457 return -EINVAL;
3458 }
3333 3459
3334 gk20a_channel_get_timescale_from_timeslice(g, timeslice, 3460 gk20a_channel_get_timescale_from_timeslice(g, timeslice,
3335 &tsg->timeslice_timeout, &tsg->timeslice_scale); 3461 &tsg->timeslice_timeout, &tsg->timeslice_scale);
@@ -3382,23 +3508,28 @@ int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
3382 Otherwise, keep active list untouched for suspend/resume. */ 3508 Otherwise, keep active list untouched for suspend/resume. */
3383 if (chid != FIFO_INVAL_CHANNEL_ID) { 3509 if (chid != FIFO_INVAL_CHANNEL_ID) {
3384 ch = &f->channel[chid]; 3510 ch = &f->channel[chid];
3385 if (gk20a_is_channel_marked_as_tsg(ch)) 3511 if (gk20a_is_channel_marked_as_tsg(ch)) {
3386 tsg = &f->tsg[ch->tsgid]; 3512 tsg = &f->tsg[ch->tsgid];
3513 }
3387 3514
3388 if (add) { 3515 if (add) {
3389 if (test_and_set_bit(chid, 3516 if (test_and_set_bit(chid,
3390 runlist->active_channels) == 1) 3517 runlist->active_channels) == 1) {
3391 return 0; 3518 return 0;
3392 if (tsg && ++tsg->num_active_channels) 3519 }
3520 if (tsg && ++tsg->num_active_channels) {
3393 set_bit(f->channel[chid].tsgid, 3521 set_bit(f->channel[chid].tsgid,
3394 runlist->active_tsgs); 3522 runlist->active_tsgs);
3523 }
3395 } else { 3524 } else {
3396 if (test_and_clear_bit(chid, 3525 if (test_and_clear_bit(chid,
3397 runlist->active_channels) == 0) 3526 runlist->active_channels) == 0) {
3398 return 0; 3527 return 0;
3399 if (tsg && --tsg->num_active_channels == 0) 3528 }
3529 if (tsg && --tsg->num_active_channels == 0) {
3400 clear_bit(f->channel[chid].tsgid, 3530 clear_bit(f->channel[chid].tsgid,
3401 runlist->active_tsgs); 3531 runlist->active_tsgs);
3532 }
3402 } 3533 }
3403 } 3534 }
3404 3535
@@ -3439,8 +3570,10 @@ int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
3439 runlist->count = (runlist_end - runlist_entry_base) / 3570 runlist->count = (runlist_end - runlist_entry_base) /
3440 runlist_entry_words; 3571 runlist_entry_words;
3441 WARN_ON(runlist->count > f->num_runlist_entries); 3572 WARN_ON(runlist->count > f->num_runlist_entries);
3442 } else /* suspend to remove all channels */ 3573 } else {
3574 /* suspend to remove all channels */
3443 runlist->count = 0; 3575 runlist->count = 0;
3576 }
3444 3577
3445 g->ops.fifo.runlist_hw_submit(g, runlist_id, runlist->count, new_buf); 3578 g->ops.fifo.runlist_hw_submit(g, runlist_id, runlist->count, new_buf);
3446 3579
@@ -3452,8 +3585,9 @@ int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
3452 /* trigger runlist update timeout recovery */ 3585 /* trigger runlist update timeout recovery */
3453 return ret; 3586 return ret;
3454 3587
3455 } else if (ret == -EINTR) 3588 } else if (ret == -EINTR) {
3456 nvgpu_err(g, "runlist update interrupted"); 3589 nvgpu_err(g, "runlist update interrupted");
3590 }
3457 } 3591 }
3458 3592
3459 runlist->cur_buffer = new_buf; 3593 runlist->cur_buffer = new_buf;
@@ -3470,8 +3604,9 @@ int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid,
3470 u32 errcode; 3604 u32 errcode;
3471 unsigned long ulong_runlist_ids = (unsigned long)runlist_ids; 3605 unsigned long ulong_runlist_ids = (unsigned long)runlist_ids;
3472 3606
3473 if (!g) 3607 if (!g) {
3474 goto end; 3608 goto end;
3609 }
3475 3610
3476 ret = 0; 3611 ret = 0;
3477 for_each_set_bit(runlist_id, &ulong_runlist_ids, 32) { 3612 for_each_set_bit(runlist_id, &ulong_runlist_ids, 32) {
@@ -3501,14 +3636,17 @@ static int __locked_fifo_reschedule_preempt_next(struct channel_gk20a *ch,
3501 u32 preempt_type = 0; 3636 u32 preempt_type = 0;
3502 3637
3503 if (1 != gk20a_fifo_get_engine_ids( 3638 if (1 != gk20a_fifo_get_engine_ids(
3504 g, &gr_eng_id, 1, ENGINE_GR_GK20A)) 3639 g, &gr_eng_id, 1, ENGINE_GR_GK20A)) {
3505 return ret; 3640 return ret;
3506 if (!(runlist->eng_bitmask & (1 << gr_eng_id))) 3641 }
3642 if (!(runlist->eng_bitmask & (1 << gr_eng_id))) {
3507 return ret; 3643 return ret;
3644 }
3508 3645
3509 if (wait_preempt && gk20a_readl(g, fifo_preempt_r()) & 3646 if (wait_preempt && gk20a_readl(g, fifo_preempt_r()) &
3510 fifo_preempt_pending_true_f()) 3647 fifo_preempt_pending_true_f()) {
3511 return ret; 3648 return ret;
3649 }
3512 3650
3513 fecsstat0 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0)); 3651 fecsstat0 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0));
3514 engstat = gk20a_readl(g, fifo_engine_status_r(gr_eng_id)); 3652 engstat = gk20a_readl(g, fifo_engine_status_r(gr_eng_id));
@@ -3517,10 +3655,12 @@ static int __locked_fifo_reschedule_preempt_next(struct channel_gk20a *ch,
3517 /* host switching to next context, preempt that if needed */ 3655 /* host switching to next context, preempt that if needed */
3518 preempt_id = fifo_engine_status_next_id_v(engstat); 3656 preempt_id = fifo_engine_status_next_id_v(engstat);
3519 preempt_type = fifo_engine_status_next_id_type_v(engstat); 3657 preempt_type = fifo_engine_status_next_id_type_v(engstat);
3520 } else 3658 } else {
3521 return ret; 3659 return ret;
3522 if (preempt_id == ch->tsgid && preempt_type) 3660 }
3661 if (preempt_id == ch->tsgid && preempt_type) {
3523 return ret; 3662 return ret;
3663 }
3524 fecsstat1 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0)); 3664 fecsstat1 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0));
3525 if (fecsstat0 != FECS_MAILBOX_0_ACK_RESTORE || 3665 if (fecsstat0 != FECS_MAILBOX_0_ACK_RESTORE ||
3526 fecsstat1 != FECS_MAILBOX_0_ACK_RESTORE) { 3666 fecsstat1 != FECS_MAILBOX_0_ACK_RESTORE) {
@@ -3559,8 +3699,9 @@ int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next,
3559 int ret = 0; 3699 int ret = 0;
3560 3700
3561 runlist = &g->fifo.runlist_info[ch->runlist_id]; 3701 runlist = &g->fifo.runlist_info[ch->runlist_id];
3562 if (!nvgpu_mutex_tryacquire(&runlist->runlist_lock)) 3702 if (!nvgpu_mutex_tryacquire(&runlist->runlist_lock)) {
3563 return -EBUSY; 3703 return -EBUSY;
3704 }
3564 3705
3565 mutex_ret = nvgpu_pmu_mutex_acquire( 3706 mutex_ret = nvgpu_pmu_mutex_acquire(
3566 &g->pmu, PMU_MUTEX_ID_FIFO, &token); 3707 &g->pmu, PMU_MUTEX_ID_FIFO, &token);
@@ -3568,14 +3709,16 @@ int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next,
3568 g->ops.fifo.runlist_hw_submit( 3709 g->ops.fifo.runlist_hw_submit(
3569 g, ch->runlist_id, runlist->count, runlist->cur_buffer); 3710 g, ch->runlist_id, runlist->count, runlist->cur_buffer);
3570 3711
3571 if (preempt_next) 3712 if (preempt_next) {
3572 __locked_fifo_reschedule_preempt_next(ch, wait_preempt); 3713 __locked_fifo_reschedule_preempt_next(ch, wait_preempt);
3714 }
3573 3715
3574 gk20a_fifo_runlist_wait_pending(g, ch->runlist_id); 3716 gk20a_fifo_runlist_wait_pending(g, ch->runlist_id);
3575 3717
3576 if (!mutex_ret) 3718 if (!mutex_ret) {
3577 nvgpu_pmu_mutex_release( 3719 nvgpu_pmu_mutex_release(
3578 &g->pmu, PMU_MUTEX_ID_FIFO, &token); 3720 &g->pmu, PMU_MUTEX_ID_FIFO, &token);
3721 }
3579 nvgpu_mutex_release(&runlist->runlist_lock); 3722 nvgpu_mutex_release(&runlist->runlist_lock);
3580 3723
3581 return ret; 3724 return ret;
@@ -3605,13 +3748,15 @@ int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 chid,
3605 ret = gk20a_fifo_update_runlist_locked(g, runlist_id, chid, add, 3748 ret = gk20a_fifo_update_runlist_locked(g, runlist_id, chid, add,
3606 wait_for_finish); 3749 wait_for_finish);
3607 3750
3608 if (!mutex_ret) 3751 if (!mutex_ret) {
3609 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 3752 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
3753 }
3610 3754
3611 nvgpu_mutex_release(&runlist->runlist_lock); 3755 nvgpu_mutex_release(&runlist->runlist_lock);
3612 3756
3613 if (ret == -ETIMEDOUT) 3757 if (ret == -ETIMEDOUT) {
3614 gk20a_fifo_runlist_reset_engines(g, runlist_id); 3758 gk20a_fifo_runlist_reset_engines(g, runlist_id);
3759 }
3615 3760
3616 return ret; 3761 return ret;
3617} 3762}
@@ -3621,9 +3766,10 @@ int gk20a_fifo_suspend(struct gk20a *g)
3621 nvgpu_log_fn(g, " "); 3766 nvgpu_log_fn(g, " ");
3622 3767
3623 /* stop bar1 snooping */ 3768 /* stop bar1 snooping */
3624 if (g->ops.mm.is_bar1_supported(g)) 3769 if (g->ops.mm.is_bar1_supported(g)) {
3625 gk20a_writel(g, fifo_bar1_base_r(), 3770 gk20a_writel(g, fifo_bar1_base_r(),
3626 fifo_bar1_base_valid_false_f()); 3771 fifo_bar1_base_valid_false_f());
3772 }
3627 3773
3628 /* disable fifo intr */ 3774 /* disable fifo intr */
3629 gk20a_writel(g, fifo_intr_en_0_r(), 0); 3775 gk20a_writel(g, fifo_intr_en_0_r(), 0);
@@ -3636,10 +3782,11 @@ int gk20a_fifo_suspend(struct gk20a *g)
3636bool gk20a_fifo_mmu_fault_pending(struct gk20a *g) 3782bool gk20a_fifo_mmu_fault_pending(struct gk20a *g)
3637{ 3783{
3638 if (gk20a_readl(g, fifo_intr_0_r()) & 3784 if (gk20a_readl(g, fifo_intr_0_r()) &
3639 fifo_intr_0_mmu_fault_pending_f()) 3785 fifo_intr_0_mmu_fault_pending_f()) {
3640 return true; 3786 return true;
3641 else 3787 } else {
3642 return false; 3788 return false;
3789 }
3643} 3790}
3644 3791
3645bool gk20a_fifo_is_engine_busy(struct gk20a *g) 3792bool gk20a_fifo_is_engine_busy(struct gk20a *g)
@@ -3651,8 +3798,9 @@ bool gk20a_fifo_is_engine_busy(struct gk20a *g)
3651 for (i = 0; i < host_num_engines; i++) { 3798 for (i = 0; i < host_num_engines; i++) {
3652 u32 status = gk20a_readl(g, fifo_engine_status_r(i)); 3799 u32 status = gk20a_readl(g, fifo_engine_status_r(i));
3653 if (fifo_engine_status_engine_v(status) == 3800 if (fifo_engine_status_engine_v(status) ==
3654 fifo_engine_status_engine_busy_v()) 3801 fifo_engine_status_engine_busy_v()) {
3655 return true; 3802 return true;
3803 }
3656 } 3804 }
3657 return false; 3805 return false;
3658} 3806}
@@ -3704,10 +3852,11 @@ u32 gk20a_fifo_get_pbdma_signature(struct gk20a *g)
3704struct channel_gk20a *gk20a_fifo_channel_from_chid(struct gk20a *g, 3852struct channel_gk20a *gk20a_fifo_channel_from_chid(struct gk20a *g,
3705 u32 chid) 3853 u32 chid)
3706{ 3854{
3707 if (chid != FIFO_INVAL_CHANNEL_ID) 3855 if (chid != FIFO_INVAL_CHANNEL_ID) {
3708 return g->fifo.channel + chid; 3856 return g->fifo.channel + chid;
3709 else 3857 } else {
3710 return NULL; 3858 return NULL;
3859 }
3711} 3860}
3712 3861
3713static const char * const ccsr_chan_status_str[] = { 3862static const char * const ccsr_chan_status_str[] = {
@@ -3745,18 +3894,20 @@ static const char * const not_found_str[] = {
3745 3894
3746const char *gk20a_decode_ccsr_chan_status(u32 index) 3895const char *gk20a_decode_ccsr_chan_status(u32 index)
3747{ 3896{
3748 if (index >= ARRAY_SIZE(ccsr_chan_status_str)) 3897 if (index >= ARRAY_SIZE(ccsr_chan_status_str)) {
3749 return not_found_str[0]; 3898 return not_found_str[0];
3750 else 3899 } else {
3751 return ccsr_chan_status_str[index]; 3900 return ccsr_chan_status_str[index];
3901 }
3752} 3902}
3753 3903
3754const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index) 3904const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index)
3755{ 3905{
3756 if (index >= ARRAY_SIZE(pbdma_chan_eng_ctx_status_str)) 3906 if (index >= ARRAY_SIZE(pbdma_chan_eng_ctx_status_str)) {
3757 return not_found_str[0]; 3907 return not_found_str[0];
3758 else 3908 } else {
3759 return pbdma_chan_eng_ctx_status_str[index]; 3909 return pbdma_chan_eng_ctx_status_str[index];
3910 }
3760} 3911}
3761 3912
3762bool gk20a_fifo_channel_status_is_next(struct gk20a *g, u32 chid) 3913bool gk20a_fifo_channel_status_is_next(struct gk20a *g, u32 chid)
@@ -3792,11 +3943,13 @@ void gk20a_dump_channel_status_ramfc(struct gk20a *g,
3792 struct channel_gk20a *c = g->fifo.channel + chid; 3943 struct channel_gk20a *c = g->fifo.channel + chid;
3793 struct nvgpu_semaphore_int *hw_sema = NULL; 3944 struct nvgpu_semaphore_int *hw_sema = NULL;
3794 3945
3795 if (c->hw_sema) 3946 if (c->hw_sema) {
3796 hw_sema = c->hw_sema; 3947 hw_sema = c->hw_sema;
3948 }
3797 3949
3798 if (!ch_state) 3950 if (!ch_state) {
3799 return; 3951 return;
3952 }
3800 3953
3801 inst_mem = &ch_state->inst_block[0]; 3954 inst_mem = &ch_state->inst_block[0];
3802 3955
@@ -3831,12 +3984,13 @@ void gk20a_dump_channel_status_ramfc(struct gk20a *g,
3831 inst_mem[ram_fc_semaphoreb_w()], 3984 inst_mem[ram_fc_semaphoreb_w()],
3832 inst_mem[ram_fc_semaphorec_w()], 3985 inst_mem[ram_fc_semaphorec_w()],
3833 inst_mem[ram_fc_semaphored_w()]); 3986 inst_mem[ram_fc_semaphored_w()]);
3834 if (hw_sema) 3987 if (hw_sema) {
3835 gk20a_debug_output(o, "SEMA STATE: value: 0x%08x " 3988 gk20a_debug_output(o, "SEMA STATE: value: 0x%08x "
3836 "next_val: 0x%08x addr: 0x%010llx\n", 3989 "next_val: 0x%08x addr: 0x%010llx\n",
3837 __nvgpu_semaphore_read(hw_sema), 3990 __nvgpu_semaphore_read(hw_sema),
3838 nvgpu_atomic_read(&hw_sema->next_value), 3991 nvgpu_atomic_read(&hw_sema->next_value),
3839 nvgpu_hw_sema_addr(hw_sema)); 3992 nvgpu_hw_sema_addr(hw_sema));
3993 }
3840 3994
3841#ifdef CONFIG_TEGRA_GK20A_NVHOST 3995#ifdef CONFIG_TEGRA_GK20A_NVHOST
3842 if ((pbdma_syncpointb_op_v(syncpointb) == pbdma_syncpointb_op_wait_v()) 3996 if ((pbdma_syncpointb_op_v(syncpointb) == pbdma_syncpointb_op_wait_v())
@@ -3874,15 +4028,17 @@ void gk20a_debug_dump_all_channel_status_ramfc(struct gk20a *g,
3874 ram_in_alloc_size_v()); 4028 ram_in_alloc_size_v());
3875 /* ref taken stays to below loop with 4029 /* ref taken stays to below loop with
3876 * successful allocs */ 4030 * successful allocs */
3877 if (!ch_state[chid]) 4031 if (!ch_state[chid]) {
3878 gk20a_channel_put(ch); 4032 gk20a_channel_put(ch);
4033 }
3879 } 4034 }
3880 } 4035 }
3881 4036
3882 for (chid = 0; chid < f->num_channels; chid++) { 4037 for (chid = 0; chid < f->num_channels; chid++) {
3883 struct channel_gk20a *ch = &f->channel[chid]; 4038 struct channel_gk20a *ch = &f->channel[chid];
3884 if (!ch_state[chid]) 4039 if (!ch_state[chid]) {
3885 continue; 4040 continue;
4041 }
3886 4042
3887 ch_state[chid]->pid = ch->pid; 4043 ch_state[chid]->pid = ch->pid;
3888 ch_state[chid]->refs = nvgpu_atomic_read(&ch->ref_count); 4044 ch_state[chid]->refs = nvgpu_atomic_read(&ch->ref_count);
@@ -3964,10 +4120,12 @@ void gk20a_dump_eng_status(struct gk20a *g,
3964 "tsg" : "channel", 4120 "tsg" : "channel",
3965 gk20a_decode_pbdma_chan_eng_ctx_status(ctx_status)); 4121 gk20a_decode_pbdma_chan_eng_ctx_status(ctx_status));
3966 4122
3967 if (fifo_engine_status_faulted_v(status)) 4123 if (fifo_engine_status_faulted_v(status)) {
3968 gk20a_debug_output(o, "faulted "); 4124 gk20a_debug_output(o, "faulted ");
3969 if (fifo_engine_status_engine_v(status)) 4125 }
4126 if (fifo_engine_status_engine_v(status)) {
3970 gk20a_debug_output(o, "busy "); 4127 gk20a_debug_output(o, "busy ");
4128 }
3971 gk20a_debug_output(o, "\n"); 4129 gk20a_debug_output(o, "\n");
3972 } 4130 }
3973 gk20a_debug_output(o, "\n"); 4131 gk20a_debug_output(o, "\n");
@@ -4088,8 +4246,9 @@ int gk20a_fifo_setup_ramfc(struct channel_gk20a *c,
4088 4246
4089 nvgpu_mem_wr32(g, mem, ram_fc_chid_w(), ram_fc_chid_id_f(c->chid)); 4247 nvgpu_mem_wr32(g, mem, ram_fc_chid_w(), ram_fc_chid_id_f(c->chid));
4090 4248
4091 if (c->is_privileged_channel) 4249 if (c->is_privileged_channel) {
4092 gk20a_fifo_setup_ramfc_for_privileged_channel(c); 4250 gk20a_fifo_setup_ramfc_for_privileged_channel(c);
4251 }
4093 4252
4094 return gk20a_fifo_commit_userd(c); 4253 return gk20a_fifo_commit_userd(c);
4095} 4254}
@@ -4143,8 +4302,9 @@ int gk20a_fifo_alloc_inst(struct gk20a *g, struct channel_gk20a *ch)
4143 nvgpu_log_fn(g, " "); 4302 nvgpu_log_fn(g, " ");
4144 4303
4145 err = g->ops.mm.alloc_inst_block(g, &ch->inst_block); 4304 err = g->ops.mm.alloc_inst_block(g, &ch->inst_block);
4146 if (err) 4305 if (err) {
4147 return err; 4306 return err;
4307 }
4148 4308
4149 nvgpu_log_info(g, "channel %d inst block physical addr: 0x%16llx", 4309 nvgpu_log_info(g, "channel %d inst block physical addr: 0x%16llx",
4150 ch->chid, nvgpu_inst_block_addr(g, &ch->inst_block)); 4310 ch->chid, nvgpu_inst_block_addr(g, &ch->inst_block));
@@ -4189,16 +4349,18 @@ u32 gk20a_fifo_pbdma_acquire_val(u64 timeout)
4189 val = pbdma_acquire_retry_man_2_f() | 4349 val = pbdma_acquire_retry_man_2_f() |
4190 pbdma_acquire_retry_exp_2_f(); 4350 pbdma_acquire_retry_exp_2_f();
4191 4351
4192 if (!timeout) 4352 if (!timeout) {
4193 return val; 4353 return val;
4354 }
4194 4355
4195 timeout *= 80UL; 4356 timeout *= 80UL;
4196 do_div(timeout, 100); /* set acquire timeout to 80% of channel wdt */ 4357 do_div(timeout, 100); /* set acquire timeout to 80% of channel wdt */
4197 timeout *= 1000000UL; /* ms -> ns */ 4358 timeout *= 1000000UL; /* ms -> ns */
4198 do_div(timeout, 1024); /* in unit of 1024ns */ 4359 do_div(timeout, 1024); /* in unit of 1024ns */
4199 val_len = fls(timeout >> 32) + 32; 4360 val_len = fls(timeout >> 32) + 32;
4200 if (val_len == 32) 4361 if (val_len == 32) {
4201 val_len = fls(timeout); 4362 val_len = fls(timeout);
4363 }
4202 if (val_len > 16U + pbdma_acquire_timeout_exp_max_v()) { /* man: 16bits */ 4364 if (val_len > 16U + pbdma_acquire_timeout_exp_max_v()) { /* man: 16bits */
4203 exp = pbdma_acquire_timeout_exp_max_v(); 4365 exp = pbdma_acquire_timeout_exp_max_v();
4204 man = pbdma_acquire_timeout_man_max_v(); 4366 man = pbdma_acquire_timeout_man_max_v();
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-07 16:31:04 -0400