summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSrirangan <smadhavan@nvidia.com>2018-08-31 03:50:52 -0400
committermobile promotions <svcmobile_promotions@nvidia.com>2018-09-05 07:51:32 -0400
commit43851d41b187c92f5ea9c2f503a882277f661d7e (patch)
tree964a76c136c8c0dc14ec95358d27f930532b7dcb
parent0f97bd4d44c8bcedf298f725fe0b6cfc70fa81ff (diff)
gpu: nvgpu: gk20a: Fix MISRA 15.6 violations
MISRA Rule-15.6 requires that all if-else blocks be enclosed in braces, including single statement blocks. Fix errors due to single statement if blocks without braces by introducing the braces. JIRA NVGPU-671 Change-Id: Iedac7d50aa2ebd409434eea5fda902b16d9c6fea Signed-off-by: Srirangan <smadhavan@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/1797695 Reviewed-by: svc-misra-checker <svc-misra-checker@nvidia.com> GVS: Gerrit_Virtual_Submit Reviewed-by: Vijayakumar Subbu <vsubbu@nvidia.com> Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com> Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
-rw-r--r--drivers/gpu/nvgpu/gk20a/ce2_gk20a.c72
-rw-r--r--drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c38
-rw-r--r--drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c54
-rw-r--r--drivers/gpu/nvgpu/gk20a/fence_gk20a.c47
-rw-r--r--drivers/gpu/nvgpu/gk20a/fifo_gk20a.c554
-rw-r--r--drivers/gpu/nvgpu/gk20a/flcn_gk20a.c38
-rw-r--r--drivers/gpu/nvgpu/gk20a/gk20a.c82
-rw-r--r--drivers/gpu/nvgpu/gk20a/gr_ctx_gk20a.c113
8 files changed, 641 insertions, 357 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
index f905243e..4cc6c8ca 100644
--- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
@@ -79,11 +79,13 @@ void gk20a_ce2_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
79 nvgpu_log(g, gpu_dbg_intr, "ce2 isr %08x\n", ce2_intr); 79 nvgpu_log(g, gpu_dbg_intr, "ce2 isr %08x\n", ce2_intr);
80 80
81 /* clear blocking interrupts: they exibit broken behavior */ 81 /* clear blocking interrupts: they exibit broken behavior */
82 if (ce2_intr & ce2_intr_status_blockpipe_pending_f()) 82 if (ce2_intr & ce2_intr_status_blockpipe_pending_f()) {
83 clear_intr |= ce2_blockpipe_isr(g, ce2_intr); 83 clear_intr |= ce2_blockpipe_isr(g, ce2_intr);
84 }
84 85
85 if (ce2_intr & ce2_intr_status_launcherr_pending_f()) 86 if (ce2_intr & ce2_intr_status_launcherr_pending_f()) {
86 clear_intr |= ce2_launcherr_isr(g, ce2_intr); 87 clear_intr |= ce2_launcherr_isr(g, ce2_intr);
88 }
87 89
88 gk20a_writel(g, ce2_intr_status_r(), clear_intr); 90 gk20a_writel(g, ce2_intr_status_r(), clear_intr);
89 return; 91 return;
@@ -112,8 +114,9 @@ static void gk20a_ce_put_fences(struct gk20a_gpu_ctx *ce_ctx)
112 114
113 for (i = 0; i < NVGPU_CE_MAX_INFLIGHT_JOBS; i++) { 115 for (i = 0; i < NVGPU_CE_MAX_INFLIGHT_JOBS; i++) {
114 struct gk20a_fence **fence = &ce_ctx->postfences[i]; 116 struct gk20a_fence **fence = &ce_ctx->postfences[i];
115 if (*fence) 117 if (*fence) {
116 gk20a_fence_put(*fence); 118 gk20a_fence_put(*fence);
119 }
117 *fence = NULL; 120 *fence = NULL;
118 } 121 }
119} 122}
@@ -140,8 +143,9 @@ static void gk20a_ce_delete_gpu_context(struct gk20a_gpu_ctx *ce_ctx)
140 nvgpu_ref_put(&ce_ctx->tsg->refcount, gk20a_tsg_release); 143 nvgpu_ref_put(&ce_ctx->tsg->refcount, gk20a_tsg_release);
141 144
142 /* housekeeping on app */ 145 /* housekeeping on app */
143 if (list->prev && list->next) 146 if (list->prev && list->next) {
144 nvgpu_list_del(list); 147 nvgpu_list_del(list);
148 }
145 149
146 nvgpu_mutex_release(&ce_ctx->gpu_ctx_mutex); 150 nvgpu_mutex_release(&ce_ctx->gpu_ctx_mutex);
147 nvgpu_mutex_destroy(&ce_ctx->gpu_ctx_mutex); 151 nvgpu_mutex_destroy(&ce_ctx->gpu_ctx_mutex);
@@ -171,10 +175,11 @@ static inline unsigned int gk20a_ce_get_method_size(int request_operation,
171 chunk -= (u64) height * width; 175 chunk -= (u64) height * width;
172 } 176 }
173 177
174 if (request_operation & NVGPU_CE_PHYS_MODE_TRANSFER) 178 if (request_operation & NVGPU_CE_PHYS_MODE_TRANSFER) {
175 methodsize = (2 + (16 * iterations)) * sizeof(u32); 179 methodsize = (2 + (16 * iterations)) * sizeof(u32);
176 else if (request_operation & NVGPU_CE_MEMSET) 180 } else if (request_operation & NVGPU_CE_MEMSET) {
177 methodsize = (2 + (15 * iterations)) * sizeof(u32); 181 methodsize = (2 + (15 * iterations)) * sizeof(u32);
182 }
178 183
179 return methodsize; 184 return methodsize;
180} 185}
@@ -198,8 +203,9 @@ int gk20a_ce_prepare_submit(u64 src_buf,
198 /* failure case handling */ 203 /* failure case handling */
199 if ((gk20a_ce_get_method_size(request_operation, size) > 204 if ((gk20a_ce_get_method_size(request_operation, size) >
200 max_cmd_buf_size) || (!size) || 205 max_cmd_buf_size) || (!size) ||
201 (request_operation > NVGPU_CE_MEMSET)) 206 (request_operation > NVGPU_CE_MEMSET)) {
202 return 0; 207 return 0;
208 }
203 209
204 /* set the channel object */ 210 /* set the channel object */
205 cmd_buf_cpu_va[methodSize++] = 0x20018000; 211 cmd_buf_cpu_va[methodSize++] = 0x20018000;
@@ -252,13 +258,14 @@ int gk20a_ce_prepare_submit(u64 src_buf,
252 offset) & NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK); 258 offset) & NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK);
253 259
254 cmd_buf_cpu_va[methodSize++] = 0x20018098; 260 cmd_buf_cpu_va[methodSize++] = 0x20018098;
255 if (launch_flags & NVGPU_CE_SRC_LOCATION_LOCAL_FB) 261 if (launch_flags & NVGPU_CE_SRC_LOCATION_LOCAL_FB) {
256 cmd_buf_cpu_va[methodSize++] = 0x00000000; 262 cmd_buf_cpu_va[methodSize++] = 0x00000000;
257 else if (launch_flags & 263 } else if (launch_flags &
258 NVGPU_CE_SRC_LOCATION_NONCOHERENT_SYSMEM) 264 NVGPU_CE_SRC_LOCATION_NONCOHERENT_SYSMEM) {
259 cmd_buf_cpu_va[methodSize++] = 0x00000002; 265 cmd_buf_cpu_va[methodSize++] = 0x00000002;
260 else 266 } else {
261 cmd_buf_cpu_va[methodSize++] = 0x00000001; 267 cmd_buf_cpu_va[methodSize++] = 0x00000001;
268 }
262 269
263 launch |= 0x00001000; 270 launch |= 0x00001000;
264 } else if (request_operation & NVGPU_CE_MEMSET) { 271 } else if (request_operation & NVGPU_CE_MEMSET) {
@@ -289,25 +296,28 @@ int gk20a_ce_prepare_submit(u64 src_buf,
289 cmd_buf_cpu_va[methodSize++] = height; 296 cmd_buf_cpu_va[methodSize++] = height;
290 297
291 cmd_buf_cpu_va[methodSize++] = 0x20018099; 298 cmd_buf_cpu_va[methodSize++] = 0x20018099;
292 if (launch_flags & NVGPU_CE_DST_LOCATION_LOCAL_FB) 299 if (launch_flags & NVGPU_CE_DST_LOCATION_LOCAL_FB) {
293 cmd_buf_cpu_va[methodSize++] = 0x00000000; 300 cmd_buf_cpu_va[methodSize++] = 0x00000000;
294 else if (launch_flags & 301 } else if (launch_flags &
295 NVGPU_CE_DST_LOCATION_NONCOHERENT_SYSMEM) 302 NVGPU_CE_DST_LOCATION_NONCOHERENT_SYSMEM) {
296 cmd_buf_cpu_va[methodSize++] = 0x00000002; 303 cmd_buf_cpu_va[methodSize++] = 0x00000002;
297 else 304 } else {
298 cmd_buf_cpu_va[methodSize++] = 0x00000001; 305 cmd_buf_cpu_va[methodSize++] = 0x00000001;
306 }
299 307
300 launch |= 0x00002005; 308 launch |= 0x00002005;
301 309
302 if (launch_flags & NVGPU_CE_SRC_MEMORY_LAYOUT_BLOCKLINEAR) 310 if (launch_flags & NVGPU_CE_SRC_MEMORY_LAYOUT_BLOCKLINEAR) {
303 launch |= 0x00000000; 311 launch |= 0x00000000;
304 else 312 } else {
305 launch |= 0x00000080; 313 launch |= 0x00000080;
314 }
306 315
307 if (launch_flags & NVGPU_CE_DST_MEMORY_LAYOUT_BLOCKLINEAR) 316 if (launch_flags & NVGPU_CE_DST_MEMORY_LAYOUT_BLOCKLINEAR) {
308 launch |= 0x00000000; 317 launch |= 0x00000000;
309 else 318 } else {
310 launch |= 0x00000100; 319 launch |= 0x00000100;
320 }
311 321
312 cmd_buf_cpu_va[methodSize++] = 0x200180c0; 322 cmd_buf_cpu_va[methodSize++] = 0x200180c0;
313 cmd_buf_cpu_va[methodSize++] = launch; 323 cmd_buf_cpu_va[methodSize++] = launch;
@@ -329,12 +339,14 @@ int gk20a_init_ce_support(struct gk20a *g)
329 339
330 g->ops.mc.reset(g, ce_reset_mask); 340 g->ops.mc.reset(g, ce_reset_mask);
331 341
332 if (g->ops.clock_gating.slcg_ce2_load_gating_prod) 342 if (g->ops.clock_gating.slcg_ce2_load_gating_prod) {
333 g->ops.clock_gating.slcg_ce2_load_gating_prod(g, 343 g->ops.clock_gating.slcg_ce2_load_gating_prod(g,
334 g->slcg_enabled); 344 g->slcg_enabled);
335 if (g->ops.clock_gating.blcg_ce_load_gating_prod) 345 }
346 if (g->ops.clock_gating.blcg_ce_load_gating_prod) {
336 g->ops.clock_gating.blcg_ce_load_gating_prod(g, 347 g->ops.clock_gating.blcg_ce_load_gating_prod(g,
337 g->blcg_enabled); 348 g->blcg_enabled);
349 }
338 350
339 if (ce_app->initialised) { 351 if (ce_app->initialised) {
340 /* assume this happen during poweron/poweroff GPU sequence */ 352 /* assume this happen during poweron/poweroff GPU sequence */
@@ -345,8 +357,9 @@ int gk20a_init_ce_support(struct gk20a *g)
345 nvgpu_log(g, gpu_dbg_fn, "ce: init"); 357 nvgpu_log(g, gpu_dbg_fn, "ce: init");
346 358
347 err = nvgpu_mutex_init(&ce_app->app_mutex); 359 err = nvgpu_mutex_init(&ce_app->app_mutex);
348 if (err) 360 if (err) {
349 return err; 361 return err;
362 }
350 363
351 nvgpu_mutex_acquire(&ce_app->app_mutex); 364 nvgpu_mutex_acquire(&ce_app->app_mutex);
352 365
@@ -367,8 +380,9 @@ void gk20a_ce_destroy(struct gk20a *g)
367 struct gk20a_ce_app *ce_app = &g->ce_app; 380 struct gk20a_ce_app *ce_app = &g->ce_app;
368 struct gk20a_gpu_ctx *ce_ctx, *ce_ctx_save; 381 struct gk20a_gpu_ctx *ce_ctx, *ce_ctx_save;
369 382
370 if (!ce_app->initialised) 383 if (!ce_app->initialised) {
371 return; 384 return;
385 }
372 386
373 ce_app->app_state = NVGPU_CE_SUSPEND; 387 ce_app->app_state = NVGPU_CE_SUSPEND;
374 ce_app->initialised = false; 388 ce_app->initialised = false;
@@ -393,8 +407,9 @@ void gk20a_ce_suspend(struct gk20a *g)
393{ 407{
394 struct gk20a_ce_app *ce_app = &g->ce_app; 408 struct gk20a_ce_app *ce_app = &g->ce_app;
395 409
396 if (!ce_app->initialised) 410 if (!ce_app->initialised) {
397 return; 411 return;
412 }
398 413
399 ce_app->app_state = NVGPU_CE_SUSPEND; 414 ce_app->app_state = NVGPU_CE_SUSPEND;
400 415
@@ -413,12 +428,14 @@ u32 gk20a_ce_create_context(struct gk20a *g,
413 u32 ctx_id = ~0; 428 u32 ctx_id = ~0;
414 int err = 0; 429 int err = 0;
415 430
416 if (!ce_app->initialised || ce_app->app_state != NVGPU_CE_ACTIVE) 431 if (!ce_app->initialised || ce_app->app_state != NVGPU_CE_ACTIVE) {
417 return ctx_id; 432 return ctx_id;
433 }
418 434
419 ce_ctx = nvgpu_kzalloc(g, sizeof(*ce_ctx)); 435 ce_ctx = nvgpu_kzalloc(g, sizeof(*ce_ctx));
420 if (!ce_ctx) 436 if (!ce_ctx) {
421 return ctx_id; 437 return ctx_id;
438 }
422 439
423 err = nvgpu_mutex_init(&ce_ctx->gpu_ctx_mutex); 440 err = nvgpu_mutex_init(&ce_ctx->gpu_ctx_mutex);
424 if (err) { 441 if (err) {
@@ -538,8 +555,9 @@ void gk20a_ce_delete_context_priv(struct gk20a *g,
538 struct gk20a_ce_app *ce_app = &g->ce_app; 555 struct gk20a_ce_app *ce_app = &g->ce_app;
539 struct gk20a_gpu_ctx *ce_ctx, *ce_ctx_save; 556 struct gk20a_gpu_ctx *ce_ctx, *ce_ctx_save;
540 557
541 if (!ce_app->initialised ||ce_app->app_state != NVGPU_CE_ACTIVE) 558 if (!ce_app->initialised || ce_app->app_state != NVGPU_CE_ACTIVE) {
542 return; 559 return;
560 }
543 561
544 nvgpu_mutex_acquire(&ce_app->app_mutex); 562 nvgpu_mutex_acquire(&ce_app->app_mutex);
545 563
diff --git a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c
index 09668d49..f6134460 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c
@@ -358,24 +358,26 @@ static void add_sema_cmd(struct gk20a *g, struct channel_gk20a *c,
358 * If the op is not an acquire (so therefor a release) we should 358 * If the op is not an acquire (so therefor a release) we should
359 * incr the underlying sema next_value. 359 * incr the underlying sema next_value.
360 */ 360 */
361 if (!acquire) 361 if (!acquire) {
362 nvgpu_semaphore_prepare(s, c->hw_sema); 362 nvgpu_semaphore_prepare(s, c->hw_sema);
363 }
363 364
364 g->ops.fifo.add_sema_cmd(g, s, va, cmd, off, acquire, wfi); 365 g->ops.fifo.add_sema_cmd(g, s, va, cmd, off, acquire, wfi);
365 366
366 if (acquire) 367 if (acquire) {
367 gpu_sema_verbose_dbg(g, "(A) c=%d ACQ_GE %-4u pool=%-3d" 368 gpu_sema_verbose_dbg(g, "(A) c=%d ACQ_GE %-4u pool=%-3d"
368 "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", 369 "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u",
369 ch, nvgpu_semaphore_get_value(s), 370 ch, nvgpu_semaphore_get_value(s),
370 s->location.pool->page_idx, va, cmd->gva, 371 s->location.pool->page_idx, va, cmd->gva,
371 cmd->mem->gpu_va, ob); 372 cmd->mem->gpu_va, ob);
372 else 373 } else {
373 gpu_sema_verbose_dbg(g, "(R) c=%d INCR %u (%u) pool=%-3d" 374 gpu_sema_verbose_dbg(g, "(R) c=%d INCR %u (%u) pool=%-3d"
374 "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", 375 "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u",
375 ch, nvgpu_semaphore_get_value(s), 376 ch, nvgpu_semaphore_get_value(s),
376 nvgpu_semaphore_read(s), 377 nvgpu_semaphore_read(s),
377 s->location.pool->page_idx, 378 s->location.pool->page_idx,
378 va, cmd->gva, cmd->mem->gpu_va, ob); 379 va, cmd->gva, cmd->mem->gpu_va, ob);
380 }
379} 381}
380 382
381void gk20a_channel_gen_sema_wait_cmd(struct channel_gk20a *c, 383void gk20a_channel_gen_sema_wait_cmd(struct channel_gk20a *c,
@@ -418,8 +420,9 @@ static int gk20a_channel_semaphore_wait_fd(
418 int err; 420 int err;
419 421
420 err = nvgpu_os_fence_fdget(&os_fence, c, fd); 422 err = nvgpu_os_fence_fdget(&os_fence, c, fd);
421 if (err) 423 if (err) {
422 return err; 424 return err;
425 }
423 426
424 err = os_fence.ops->program_waits(&os_fence, 427 err = os_fence.ops->program_waits(&os_fence,
425 entry, c, max_wait_cmds); 428 entry, c, max_wait_cmds);
@@ -465,8 +468,9 @@ static int __gk20a_channel_semaphore_incr(
465 err = nvgpu_os_fence_sema_create(&os_fence, c, 468 err = nvgpu_os_fence_sema_create(&os_fence, c,
466 semaphore); 469 semaphore);
467 470
468 if (err) 471 if (err) {
469 goto clean_up_sema; 472 goto clean_up_sema;
473 }
470 } 474 }
471 475
472 err = gk20a_fence_from_semaphore(fence, 476 err = gk20a_fence_from_semaphore(fence,
@@ -475,8 +479,9 @@ static int __gk20a_channel_semaphore_incr(
475 os_fence); 479 os_fence);
476 480
477 if (err) { 481 if (err) {
478 if (nvgpu_os_fence_is_initialized(&os_fence)) 482 if (nvgpu_os_fence_is_initialized(&os_fence)) {
479 os_fence.ops->drop_ref(&os_fence); 483 os_fence.ops->drop_ref(&os_fence);
484 }
480 goto clean_up_sema; 485 goto clean_up_sema;
481 } 486 }
482 487
@@ -535,13 +540,15 @@ static void gk20a_channel_semaphore_set_min_eq_max(struct gk20a_channel_sync *s)
535 struct channel_gk20a *c = sp->c; 540 struct channel_gk20a *c = sp->c;
536 bool updated; 541 bool updated;
537 542
538 if (!c->hw_sema) 543 if (!c->hw_sema) {
539 return; 544 return;
545 }
540 546
541 updated = nvgpu_semaphore_reset(c->hw_sema); 547 updated = nvgpu_semaphore_reset(c->hw_sema);
542 548
543 if (updated) 549 if (updated) {
544 nvgpu_cond_broadcast_interruptible(&c->semaphore_wq); 550 nvgpu_cond_broadcast_interruptible(&c->semaphore_wq);
551 }
545} 552}
546 553
547static void gk20a_channel_semaphore_set_safe_state(struct gk20a_channel_sync *s) 554static void gk20a_channel_semaphore_set_safe_state(struct gk20a_channel_sync *s)
@@ -568,8 +575,9 @@ static void gk20a_channel_semaphore_destroy(struct gk20a_channel_sync *s)
568 struct gk20a *g = c->g; 575 struct gk20a *g = c->g;
569 576
570 if (c->has_os_fence_framework_support && 577 if (c->has_os_fence_framework_support &&
571 g->os_channel.os_fence_framework_inst_exists(c)) 578 g->os_channel.os_fence_framework_inst_exists(c)) {
572 g->os_channel.destroy_os_fence_framework(c); 579 g->os_channel.destroy_os_fence_framework(c);
580 }
573 581
574 /* The sema pool is cleaned up by the VM destroy. */ 582 /* The sema pool is cleaned up by the VM destroy. */
575 sema->pool = NULL; 583 sema->pool = NULL;
@@ -586,19 +594,22 @@ gk20a_channel_semaphore_create(struct channel_gk20a *c, bool user_managed)
586 int asid = -1; 594 int asid = -1;
587 int err; 595 int err;
588 596
589 if (WARN_ON(!c->vm)) 597 if (WARN_ON(!c->vm)) {
590 return NULL; 598 return NULL;
599 }
591 600
592 sema = nvgpu_kzalloc(c->g, sizeof(*sema)); 601 sema = nvgpu_kzalloc(c->g, sizeof(*sema));
593 if (!sema) 602 if (!sema) {
594 return NULL; 603 return NULL;
604 }
595 sema->c = c; 605 sema->c = c;
596 606
597 sprintf(pool_name, "semaphore_pool-%d", c->chid); 607 sprintf(pool_name, "semaphore_pool-%d", c->chid);
598 sema->pool = c->vm->sema_pool; 608 sema->pool = c->vm->sema_pool;
599 609
600 if (c->vm->as_share) 610 if (c->vm->as_share) {
601 asid = c->vm->as_share->id; 611 asid = c->vm->as_share->id;
612 }
602 613
603 if (c->has_os_fence_framework_support) { 614 if (c->has_os_fence_framework_support) {
604 /*Init the sync_timeline for this channel */ 615 /*Init the sync_timeline for this channel */
@@ -628,8 +639,9 @@ gk20a_channel_semaphore_create(struct channel_gk20a *c, bool user_managed)
628void gk20a_channel_sync_destroy(struct gk20a_channel_sync *sync, 639void gk20a_channel_sync_destroy(struct gk20a_channel_sync *sync,
629 bool set_safe_state) 640 bool set_safe_state)
630{ 641{
631 if (set_safe_state) 642 if (set_safe_state) {
632 sync->set_safe_state(sync); 643 sync->set_safe_state(sync);
644 }
633 sync->destroy(sync); 645 sync->destroy(sync);
634} 646}
635 647
diff --git a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
index 3be08865..e7f95245 100644
--- a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
@@ -229,21 +229,25 @@ int dbg_set_powergate(struct dbg_session_gk20a *dbg_s, bool disable_powergate)
229 nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, 229 nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn,
230 "module busy"); 230 "module busy");
231 err = gk20a_busy(g); 231 err = gk20a_busy(g);
232 if (err) 232 if (err) {
233 return err; 233 return err;
234 }
234 235
235 /*do elpg disable before clock gating */ 236 /*do elpg disable before clock gating */
236 nvgpu_pmu_pg_global_enable(g, false); 237 nvgpu_pmu_pg_global_enable(g, false);
237 238
238 if (g->ops.clock_gating.slcg_gr_load_gating_prod) 239 if (g->ops.clock_gating.slcg_gr_load_gating_prod) {
239 g->ops.clock_gating.slcg_gr_load_gating_prod(g, 240 g->ops.clock_gating.slcg_gr_load_gating_prod(g,
240 false); 241 false);
241 if (g->ops.clock_gating.slcg_perf_load_gating_prod) 242 }
243 if (g->ops.clock_gating.slcg_perf_load_gating_prod) {
242 g->ops.clock_gating.slcg_perf_load_gating_prod(g, 244 g->ops.clock_gating.slcg_perf_load_gating_prod(g,
243 false); 245 false);
244 if (g->ops.clock_gating.slcg_ltc_load_gating_prod) 246 }
247 if (g->ops.clock_gating.slcg_ltc_load_gating_prod) {
245 g->ops.clock_gating.slcg_ltc_load_gating_prod(g, 248 g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
246 false); 249 false);
250 }
247 251
248 gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_RUN); 252 gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_RUN);
249 gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); 253 gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN);
@@ -254,28 +258,27 @@ int dbg_set_powergate(struct dbg_session_gk20a *dbg_s, bool disable_powergate)
254 /* release pending exceptions to fault/be handled as usual */ 258 /* release pending exceptions to fault/be handled as usual */
255 /*TBD: ordering of these? */ 259 /*TBD: ordering of these? */
256 260
257 if (g->elcg_enabled) 261 if (g->elcg_enabled) {
258 gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_AUTO); 262 gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_AUTO);
263 }
259 264
260 if (g->blcg_enabled) 265 if (g->blcg_enabled) {
261 gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_AUTO); 266 gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_AUTO);
267 }
262 268
263 if (g->slcg_enabled) { 269 if (g->slcg_enabled) {
264 if (g->ops.clock_gating. 270 if (g->ops.clock_gating.slcg_ltc_load_gating_prod) {
265 slcg_ltc_load_gating_prod) 271 g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
266 g->ops.clock_gating. 272 g->slcg_enabled);
267 slcg_ltc_load_gating_prod(g, 273 }
268 g->slcg_enabled); 274 if (g->ops.clock_gating.slcg_perf_load_gating_prod) {
269 if (g->ops.clock_gating. 275 g->ops.clock_gating.slcg_perf_load_gating_prod(g,
270 slcg_perf_load_gating_prod) 276 g->slcg_enabled);
271 g->ops.clock_gating. 277 }
272 slcg_perf_load_gating_prod(g, 278 if (g->ops.clock_gating.slcg_gr_load_gating_prod) {
273 g->slcg_enabled); 279 g->ops.clock_gating.slcg_gr_load_gating_prod(g,
274 if (g->ops.clock_gating. 280 g->slcg_enabled);
275 slcg_gr_load_gating_prod) 281 }
276 g->ops.clock_gating.
277 slcg_gr_load_gating_prod(g,
278 g->slcg_enabled);
279 } 282 }
280 nvgpu_pmu_pg_global_enable(g, true); 283 nvgpu_pmu_pg_global_enable(g, true);
281 284
@@ -328,12 +331,14 @@ void nvgpu_release_profiler_reservation(struct dbg_session_gk20a *dbg_s,
328 struct gk20a *g = dbg_s->g; 331 struct gk20a *g = dbg_s->g;
329 332
330 g->profiler_reservation_count--; 333 g->profiler_reservation_count--;
331 if (g->profiler_reservation_count < 0) 334 if (g->profiler_reservation_count < 0) {
332 nvgpu_err(g, "Negative reservation count!"); 335 nvgpu_err(g, "Negative reservation count!");
336 }
333 dbg_s->has_profiler_reservation = false; 337 dbg_s->has_profiler_reservation = false;
334 prof_obj->has_reservation = false; 338 prof_obj->has_reservation = false;
335 if (prof_obj->ch == NULL) 339 if (prof_obj->ch == NULL) {
336 g->global_profiler_reservation_held = false; 340 g->global_profiler_reservation_held = false;
341 }
337} 342}
338 343
339int gk20a_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size) 344int gk20a_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size)
@@ -351,8 +356,9 @@ int gk20a_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size)
351 } 356 }
352 357
353 err = g->ops.mm.alloc_inst_block(g, &mm->perfbuf.inst_block); 358 err = g->ops.mm.alloc_inst_block(g, &mm->perfbuf.inst_block);
354 if (err) 359 if (err) {
355 return err; 360 return err;
361 }
356 362
357 g->ops.mm.init_inst_block(&mm->perfbuf.inst_block, mm->perfbuf.vm, 0); 363 g->ops.mm.init_inst_block(&mm->perfbuf.inst_block, mm->perfbuf.vm, 0);
358 364
diff --git a/drivers/gpu/nvgpu/gk20a/fence_gk20a.c b/drivers/gpu/nvgpu/gk20a/fence_gk20a.c
index 8f585afd..0df73278 100644
--- a/drivers/gpu/nvgpu/gk20a/fence_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fence_gk20a.c
@@ -44,29 +44,35 @@ static void gk20a_fence_free(struct nvgpu_ref *ref)
44 container_of(ref, struct gk20a_fence, ref); 44 container_of(ref, struct gk20a_fence, ref);
45 struct gk20a *g = f->g; 45 struct gk20a *g = f->g;
46 46
47 if (nvgpu_os_fence_is_initialized(&f->os_fence)) 47 if (nvgpu_os_fence_is_initialized(&f->os_fence)) {
48 f->os_fence.ops->drop_ref(&f->os_fence); 48 f->os_fence.ops->drop_ref(&f->os_fence);
49 }
49 50
50 if (f->semaphore) 51 if (f->semaphore) {
51 nvgpu_semaphore_put(f->semaphore); 52 nvgpu_semaphore_put(f->semaphore);
53 }
52 54
53 if (f->allocator) { 55 if (f->allocator) {
54 if (nvgpu_alloc_initialized(f->allocator)) 56 if (nvgpu_alloc_initialized(f->allocator)) {
55 nvgpu_free(f->allocator, (u64)(uintptr_t)f); 57 nvgpu_free(f->allocator, (u64)(uintptr_t)f);
56 } else 58 }
59 } else {
57 nvgpu_kfree(g, f); 60 nvgpu_kfree(g, f);
61 }
58} 62}
59 63
60void gk20a_fence_put(struct gk20a_fence *f) 64void gk20a_fence_put(struct gk20a_fence *f)
61{ 65{
62 if (f) 66 if (f) {
63 nvgpu_ref_put(&f->ref, gk20a_fence_free); 67 nvgpu_ref_put(&f->ref, gk20a_fence_free);
68 }
64} 69}
65 70
66struct gk20a_fence *gk20a_fence_get(struct gk20a_fence *f) 71struct gk20a_fence *gk20a_fence_get(struct gk20a_fence *f)
67{ 72{
68 if (f) 73 if (f) {
69 nvgpu_ref_get(&f->ref); 74 nvgpu_ref_get(&f->ref);
75 }
70 return f; 76 return f;
71} 77}
72 78
@@ -81,8 +87,9 @@ inline bool gk20a_fence_is_valid(struct gk20a_fence *f)
81int gk20a_fence_install_fd(struct gk20a_fence *f, int fd) 87int gk20a_fence_install_fd(struct gk20a_fence *f, int fd)
82{ 88{
83 if (!f || !gk20a_fence_is_valid(f) || 89 if (!f || !gk20a_fence_is_valid(f) ||
84 !nvgpu_os_fence_is_initialized(&f->os_fence)) 90 !nvgpu_os_fence_is_initialized(&f->os_fence)) {
85 return -EINVAL; 91 return -EINVAL;
92 }
86 93
87 f->os_fence.ops->install_fence(&f->os_fence, fd); 94 f->os_fence.ops->install_fence(&f->os_fence, fd);
88 95
@@ -93,8 +100,9 @@ int gk20a_fence_wait(struct gk20a *g, struct gk20a_fence *f,
93 unsigned long timeout) 100 unsigned long timeout)
94{ 101{
95 if (f && gk20a_fence_is_valid(f)) { 102 if (f && gk20a_fence_is_valid(f)) {
96 if (!nvgpu_platform_is_silicon(g)) 103 if (!nvgpu_platform_is_silicon(g)) {
97 timeout = MAX_SCHEDULE_TIMEOUT; 104 timeout = MAX_SCHEDULE_TIMEOUT;
105 }
98 return f->ops->wait(f, timeout); 106 return f->ops->wait(f, timeout);
99 } 107 }
100 return 0; 108 return 0;
@@ -102,10 +110,11 @@ int gk20a_fence_wait(struct gk20a *g, struct gk20a_fence *f,
102 110
103bool gk20a_fence_is_expired(struct gk20a_fence *f) 111bool gk20a_fence_is_expired(struct gk20a_fence *f)
104{ 112{
105 if (f && gk20a_fence_is_valid(f) && f->ops) 113 if (f && gk20a_fence_is_valid(f) && f->ops) {
106 return f->ops->is_expired(f); 114 return f->ops->is_expired(f);
107 else 115 } else {
108 return true; 116 return true;
117 }
109} 118}
110 119
111int gk20a_alloc_fence_pool(struct channel_gk20a *c, unsigned int count) 120int gk20a_alloc_fence_pool(struct channel_gk20a *c, unsigned int count)
@@ -120,14 +129,16 @@ int gk20a_alloc_fence_pool(struct channel_gk20a *c, unsigned int count)
120 fence_pool = nvgpu_vzalloc(c->g, size); 129 fence_pool = nvgpu_vzalloc(c->g, size);
121 } 130 }
122 131
123 if (!fence_pool) 132 if (!fence_pool) {
124 return -ENOMEM; 133 return -ENOMEM;
134 }
125 135
126 err = nvgpu_lockless_allocator_init(c->g, &c->fence_allocator, 136 err = nvgpu_lockless_allocator_init(c->g, &c->fence_allocator,
127 "fence_pool", (size_t)fence_pool, size, 137 "fence_pool", (size_t)fence_pool, size,
128 sizeof(struct gk20a_fence), 0); 138 sizeof(struct gk20a_fence), 0);
129 if (err) 139 if (err) {
130 goto fail; 140 goto fail;
141 }
131 142
132 return 0; 143 return 0;
133 144
@@ -163,8 +174,9 @@ struct gk20a_fence *gk20a_alloc_fence(struct channel_gk20a *c)
163 fence->allocator = &c->fence_allocator; 174 fence->allocator = &c->fence_allocator;
164 } 175 }
165 } 176 }
166 } else 177 } else {
167 fence = nvgpu_kzalloc(c->g, sizeof(struct gk20a_fence)); 178 fence = nvgpu_kzalloc(c->g, sizeof(struct gk20a_fence));
179 }
168 180
169 if (fence) { 181 if (fence) {
170 nvgpu_ref_init(&fence->ref); 182 nvgpu_ref_init(&fence->ref);
@@ -178,8 +190,9 @@ void gk20a_init_fence(struct gk20a_fence *f,
178 const struct gk20a_fence_ops *ops, 190 const struct gk20a_fence_ops *ops,
179 struct nvgpu_os_fence os_fence) 191 struct nvgpu_os_fence os_fence)
180{ 192{
181 if (!f) 193 if (!f) {
182 return; 194 return;
195 }
183 f->ops = ops; 196 f->ops = ops;
184 f->syncpt_id = -1; 197 f->syncpt_id = -1;
185 f->semaphore = NULL; 198 f->semaphore = NULL;
@@ -190,8 +203,9 @@ void gk20a_init_fence(struct gk20a_fence *f,
190 203
191static int nvgpu_semaphore_fence_wait(struct gk20a_fence *f, long timeout) 204static int nvgpu_semaphore_fence_wait(struct gk20a_fence *f, long timeout)
192{ 205{
193 if (!nvgpu_semaphore_is_acquired(f->semaphore)) 206 if (!nvgpu_semaphore_is_acquired(f->semaphore)) {
194 return 0; 207 return 0;
208 }
195 209
196 return NVGPU_COND_WAIT_INTERRUPTIBLE( 210 return NVGPU_COND_WAIT_INTERRUPTIBLE(
197 f->semaphore_wq, 211 f->semaphore_wq,
@@ -219,8 +233,9 @@ int gk20a_fence_from_semaphore(
219 struct gk20a_fence *f = fence_out; 233 struct gk20a_fence *f = fence_out;
220 234
221 gk20a_init_fence(f, &nvgpu_semaphore_fence_ops, os_fence); 235 gk20a_init_fence(f, &nvgpu_semaphore_fence_ops, os_fence);
222 if (!f) 236 if (!f) {
223 return -EINVAL; 237 return -EINVAL;
238 }
224 239
225 240
226 f->semaphore = semaphore; 241 f->semaphore = semaphore;
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index e03c5da8..f06bf1c5 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -111,8 +111,9 @@ struct fifo_engine_info_gk20a *gk20a_fifo_get_engine_info(struct gk20a *g, u32 e
111 u32 engine_id_idx; 111 u32 engine_id_idx;
112 struct fifo_engine_info_gk20a *info = NULL; 112 struct fifo_engine_info_gk20a *info = NULL;
113 113
114 if (!g) 114 if (!g) {
115 return info; 115 return info;
116 }
116 117
117 f = &g->fifo; 118 f = &g->fifo;
118 119
@@ -125,8 +126,9 @@ struct fifo_engine_info_gk20a *gk20a_fifo_get_engine_info(struct gk20a *g, u32 e
125 } 126 }
126 } 127 }
127 128
128 if (!info) 129 if (!info) {
129 nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id); 130 nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id);
131 }
130 132
131 return info; 133 return info;
132} 134}
@@ -137,8 +139,9 @@ bool gk20a_fifo_is_valid_engine_id(struct gk20a *g, u32 engine_id)
137 u32 engine_id_idx; 139 u32 engine_id_idx;
138 bool valid = false; 140 bool valid = false;
139 141
140 if (!g) 142 if (!g) {
141 return valid; 143 return valid;
144 }
142 145
143 f = &g->fifo; 146 f = &g->fifo;
144 147
@@ -151,8 +154,9 @@ bool gk20a_fifo_is_valid_engine_id(struct gk20a *g, u32 engine_id)
151 } 154 }
152 } 155 }
153 156
154 if (!valid) 157 if (!valid) {
155 nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id); 158 nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id);
159 }
156 160
157 return valid; 161 return valid;
158} 162}
@@ -182,8 +186,9 @@ u32 gk20a_fifo_get_all_ce_engine_reset_mask(struct gk20a *g)
182 struct fifo_engine_info_gk20a *engine_info; 186 struct fifo_engine_info_gk20a *engine_info;
183 u32 active_engine_id = 0; 187 u32 active_engine_id = 0;
184 188
185 if (!g) 189 if (!g) {
186 return reset_mask; 190 return reset_mask;
191 }
187 192
188 f = &g->fifo; 193 f = &g->fifo;
189 194
@@ -193,8 +198,9 @@ u32 gk20a_fifo_get_all_ce_engine_reset_mask(struct gk20a *g)
193 engine_enum = engine_info->engine_enum; 198 engine_enum = engine_info->engine_enum;
194 199
195 if ((engine_enum == ENGINE_GRCE_GK20A) || 200 if ((engine_enum == ENGINE_GRCE_GK20A) ||
196 (engine_enum == ENGINE_ASYNC_CE_GK20A)) 201 (engine_enum == ENGINE_ASYNC_CE_GK20A)) {
197 reset_mask |= engine_info->reset_mask; 202 reset_mask |= engine_info->reset_mask;
203 }
198 } 204 }
199 205
200 return reset_mask; 206 return reset_mask;
@@ -209,8 +215,9 @@ u32 gk20a_fifo_get_fast_ce_runlist_id(struct gk20a *g)
209 struct fifo_engine_info_gk20a *engine_info; 215 struct fifo_engine_info_gk20a *engine_info;
210 u32 active_engine_id = 0; 216 u32 active_engine_id = 0;
211 217
212 if (!g) 218 if (!g) {
213 return ce_runlist_id; 219 return ce_runlist_id;
220 }
214 221
215 f = &g->fifo; 222 f = &g->fifo;
216 223
@@ -220,8 +227,9 @@ u32 gk20a_fifo_get_fast_ce_runlist_id(struct gk20a *g)
220 engine_enum = engine_info->engine_enum; 227 engine_enum = engine_info->engine_enum;
221 228
222 /* selecet last available ASYNC_CE if available */ 229 /* selecet last available ASYNC_CE if available */
223 if (engine_enum == ENGINE_ASYNC_CE_GK20A) 230 if (engine_enum == ENGINE_ASYNC_CE_GK20A) {
224 ce_runlist_id = engine_info->runlist_id; 231 ce_runlist_id = engine_info->runlist_id;
232 }
225 } 233 }
226 234
227 return ce_runlist_id; 235 return ce_runlist_id;
@@ -264,8 +272,9 @@ bool gk20a_fifo_is_valid_runlist_id(struct gk20a *g, u32 runlist_id)
264 u32 active_engine_id; 272 u32 active_engine_id;
265 struct fifo_engine_info_gk20a *engine_info; 273 struct fifo_engine_info_gk20a *engine_info;
266 274
267 if (!g) 275 if (!g) {
268 return false; 276 return false;
277 }
269 278
270 f = &g->fifo; 279 f = &g->fifo;
271 280
@@ -310,8 +319,9 @@ static inline u32 gk20a_mmu_id_to_engine_id(struct gk20a *g, u32 fault_id)
310 active_engine_id = f->active_engines_list[engine_id]; 319 active_engine_id = f->active_engines_list[engine_id];
311 engine_info = &g->fifo.engine_info[active_engine_id]; 320 engine_info = &g->fifo.engine_info[active_engine_id];
312 321
313 if (engine_info->fault_id == fault_id) 322 if (engine_info->fault_id == fault_id) {
314 break; 323 break;
324 }
315 active_engine_id = FIFO_INVAL_ENGINE_ID; 325 active_engine_id = FIFO_INVAL_ENGINE_ID;
316 } 326 }
317 return active_engine_id; 327 return active_engine_id;
@@ -323,17 +333,18 @@ int gk20a_fifo_engine_enum_from_type(struct gk20a *g, u32 engine_type,
323 int ret = ENGINE_INVAL_GK20A; 333 int ret = ENGINE_INVAL_GK20A;
324 334
325 nvgpu_log_info(g, "engine type %d", engine_type); 335 nvgpu_log_info(g, "engine type %d", engine_type);
326 if (engine_type == top_device_info_type_enum_graphics_v()) 336 if (engine_type == top_device_info_type_enum_graphics_v()) {
327 ret = ENGINE_GR_GK20A; 337 ret = ENGINE_GR_GK20A;
328 else if ((engine_type >= top_device_info_type_enum_copy0_v()) && 338 } else if ((engine_type >= top_device_info_type_enum_copy0_v()) &&
329 (engine_type <= top_device_info_type_enum_copy2_v())) { 339 (engine_type <= top_device_info_type_enum_copy2_v())) {
330 /* Lets consider all the CE engine have separate runlist at this point 340 /* Lets consider all the CE engine have separate runlist at this point
331 * We can identify the ENGINE_GRCE_GK20A type CE using runlist_id 341 * We can identify the ENGINE_GRCE_GK20A type CE using runlist_id
332 * comparsion logic with GR runlist_id in init_engine_info() */ 342 * comparsion logic with GR runlist_id in init_engine_info() */
333 ret = ENGINE_ASYNC_CE_GK20A; 343 ret = ENGINE_ASYNC_CE_GK20A;
334 /* inst_id starts from CE0 to CE2 */ 344 /* inst_id starts from CE0 to CE2 */
335 if (inst_id) 345 if (inst_id) {
336 *inst_id = (engine_type - top_device_info_type_enum_copy0_v()); 346 *inst_id = (engine_type - top_device_info_type_enum_copy0_v());
347 }
337 } 348 }
338 349
339 return ret; 350 return ret;
@@ -421,10 +432,11 @@ int gk20a_fifo_init_engine_info(struct fifo_gk20a *f)
421 engine_type, &inst_id); 432 engine_type, &inst_id);
422 } else if (entry == top_device_info_entry_data_v()) { 433 } else if (entry == top_device_info_entry_data_v()) {
423 /* gk20a doesn't support device_info_data packet parsing */ 434 /* gk20a doesn't support device_info_data packet parsing */
424 if (g->ops.fifo.device_info_data_parse) 435 if (g->ops.fifo.device_info_data_parse) {
425 g->ops.fifo.device_info_data_parse(g, 436 g->ops.fifo.device_info_data_parse(g,
426 table_entry, &inst_id, &pri_base, 437 table_entry, &inst_id, &pri_base,
427 &fault_id); 438 &fault_id);
439 }
428 } 440 }
429 441
430 if (!top_device_info_chain_v(table_entry)) { 442 if (!top_device_info_chain_v(table_entry)) {
@@ -439,18 +451,21 @@ int gk20a_fifo_init_engine_info(struct fifo_gk20a *f)
439 info->inst_id = inst_id; 451 info->inst_id = inst_id;
440 info->pri_base = pri_base; 452 info->pri_base = pri_base;
441 453
442 if (engine_enum == ENGINE_GR_GK20A) 454 if (engine_enum == ENGINE_GR_GK20A) {
443 gr_runlist_id = runlist_id; 455 gr_runlist_id = runlist_id;
456 }
444 457
445 /* GR and GR_COPY shares same runlist_id */ 458 /* GR and GR_COPY shares same runlist_id */
446 if ((engine_enum == ENGINE_ASYNC_CE_GK20A) && 459 if ((engine_enum == ENGINE_ASYNC_CE_GK20A) &&
447 (gr_runlist_id == runlist_id)) 460 (gr_runlist_id == runlist_id)) {
448 engine_enum = ENGINE_GRCE_GK20A; 461 engine_enum = ENGINE_GRCE_GK20A;
462 }
449 463
450 info->engine_enum = engine_enum; 464 info->engine_enum = engine_enum;
451 465
452 if (!fault_id && (engine_enum == ENGINE_GRCE_GK20A)) 466 if (!fault_id && (engine_enum == ENGINE_GRCE_GK20A)) {
453 fault_id = 0x1b; 467 fault_id = 0x1b;
468 }
454 info->fault_id = fault_id; 469 info->fault_id = fault_id;
455 470
456 /* engine_id starts from 0 to NV_HOST_NUM_ENGINES */ 471 /* engine_id starts from 0 to NV_HOST_NUM_ENGINES */
@@ -471,8 +486,9 @@ u32 gk20a_fifo_act_eng_interrupt_mask(struct gk20a *g, u32 act_eng_id)
471 struct fifo_engine_info_gk20a *engine_info = NULL; 486 struct fifo_engine_info_gk20a *engine_info = NULL;
472 487
473 engine_info = gk20a_fifo_get_engine_info(g, act_eng_id); 488 engine_info = gk20a_fifo_get_engine_info(g, act_eng_id);
474 if (engine_info) 489 if (engine_info) {
475 return engine_info->intr_mask; 490 return engine_info->intr_mask;
491 }
476 492
477 return 0; 493 return 0;
478} 494}
@@ -491,8 +507,9 @@ u32 gk20a_fifo_engine_interrupt_mask(struct gk20a *g)
491 engine_enum = g->fifo.engine_info[active_engine_id].engine_enum; 507 engine_enum = g->fifo.engine_info[active_engine_id].engine_enum;
492 if (((engine_enum == ENGINE_GRCE_GK20A) || 508 if (((engine_enum == ENGINE_GRCE_GK20A) ||
493 (engine_enum == ENGINE_ASYNC_CE_GK20A)) && 509 (engine_enum == ENGINE_ASYNC_CE_GK20A)) &&
494 (!g->ops.ce2.isr_stall || !g->ops.ce2.isr_nonstall)) 510 (!g->ops.ce2.isr_stall || !g->ops.ce2.isr_nonstall)) {
495 continue; 511 continue;
512 }
496 513
497 eng_intr_mask |= intr_mask; 514 eng_intr_mask |= intr_mask;
498 } 515 }
@@ -507,8 +524,9 @@ void gk20a_fifo_delete_runlist(struct fifo_gk20a *f)
507 struct fifo_runlist_info_gk20a *runlist; 524 struct fifo_runlist_info_gk20a *runlist;
508 struct gk20a *g = NULL; 525 struct gk20a *g = NULL;
509 526
510 if (!f || !f->runlist_info) 527 if (!f || !f->runlist_info) {
511 return; 528 return;
529 }
512 530
513 g = f->g; 531 g = f->g;
514 532
@@ -554,8 +572,9 @@ static void gk20a_remove_fifo_support(struct fifo_gk20a *f)
554 * Could race but worst that happens is we get an error message 572 * Could race but worst that happens is we get an error message
555 * from gk20a_free_channel() complaining about multiple closes. 573 * from gk20a_free_channel() complaining about multiple closes.
556 */ 574 */
557 if (c->referenceable) 575 if (c->referenceable) {
558 __gk20a_channel_kill(c); 576 __gk20a_channel_kill(c);
577 }
559 578
560 nvgpu_mutex_destroy(&tsg->event_id_list_lock); 579 nvgpu_mutex_destroy(&tsg->event_id_list_lock);
561 580
@@ -573,10 +592,11 @@ static void gk20a_remove_fifo_support(struct fifo_gk20a *f)
573 592
574 nvgpu_vfree(g, f->channel); 593 nvgpu_vfree(g, f->channel);
575 nvgpu_vfree(g, f->tsg); 594 nvgpu_vfree(g, f->tsg);
576 if (g->ops.mm.is_bar1_supported(g)) 595 if (g->ops.mm.is_bar1_supported(g)) {
577 nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd); 596 nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd);
578 else 597 } else {
579 nvgpu_dma_free(g, &f->userd); 598 nvgpu_dma_free(g, &f->userd);
599 }
580 600
581 gk20a_fifo_delete_runlist(f); 601 gk20a_fifo_delete_runlist(f);
582 602
@@ -683,8 +703,9 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
683 f->runlist_info = nvgpu_kzalloc(g, 703 f->runlist_info = nvgpu_kzalloc(g,
684 sizeof(struct fifo_runlist_info_gk20a) * 704 sizeof(struct fifo_runlist_info_gk20a) *
685 f->max_runlists); 705 f->max_runlists);
686 if (!f->runlist_info) 706 if (!f->runlist_info) {
687 goto clean_up_runlist; 707 goto clean_up_runlist;
708 }
688 709
689 memset(f->runlist_info, 0, (sizeof(struct fifo_runlist_info_gk20a) * 710 memset(f->runlist_info, 0, (sizeof(struct fifo_runlist_info_gk20a) *
690 f->max_runlists)); 711 f->max_runlists));
@@ -695,14 +716,16 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
695 runlist->active_channels = 716 runlist->active_channels =
696 nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, 717 nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
697 BITS_PER_BYTE)); 718 BITS_PER_BYTE));
698 if (!runlist->active_channels) 719 if (!runlist->active_channels) {
699 goto clean_up_runlist; 720 goto clean_up_runlist;
721 }
700 722
701 runlist->active_tsgs = 723 runlist->active_tsgs =
702 nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, 724 nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
703 BITS_PER_BYTE)); 725 BITS_PER_BYTE));
704 if (!runlist->active_tsgs) 726 if (!runlist->active_tsgs) {
705 goto clean_up_runlist; 727 goto clean_up_runlist;
728 }
706 729
707 runlist_size = f->runlist_entry_size * f->num_runlist_entries; 730 runlist_size = f->runlist_entry_size * f->num_runlist_entries;
708 nvgpu_log(g, gpu_dbg_info, 731 nvgpu_log(g, gpu_dbg_info,
@@ -725,8 +748,9 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
725 runlist->cur_buffer = MAX_RUNLIST_BUFFERS; 748 runlist->cur_buffer = MAX_RUNLIST_BUFFERS;
726 749
727 for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) { 750 for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) {
728 if (f->pbdma_map[pbdma_id] & BIT(runlist_id)) 751 if (f->pbdma_map[pbdma_id] & BIT(runlist_id)) {
729 runlist->pbdma_bitmask |= BIT(pbdma_id); 752 runlist->pbdma_bitmask |= BIT(pbdma_id);
753 }
730 } 754 }
731 nvgpu_log(g, gpu_dbg_info, "runlist %d : pbdma bitmask 0x%x", 755 nvgpu_log(g, gpu_dbg_info, "runlist %d : pbdma bitmask 0x%x",
732 runlist_id, runlist->pbdma_bitmask); 756 runlist_id, runlist->pbdma_bitmask);
@@ -735,8 +759,9 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
735 active_engine_id = f->active_engines_list[engine_id]; 759 active_engine_id = f->active_engines_list[engine_id];
736 engine_info = &f->engine_info[active_engine_id]; 760 engine_info = &f->engine_info[active_engine_id];
737 761
738 if (engine_info && engine_info->runlist_id == runlist_id) 762 if (engine_info && engine_info->runlist_id == runlist_id) {
739 runlist->eng_bitmask |= BIT(active_engine_id); 763 runlist->eng_bitmask |= BIT(active_engine_id);
764 }
740 } 765 }
741 nvgpu_log(g, gpu_dbg_info, "runlist %d : act eng bitmask 0x%x", 766 nvgpu_log(g, gpu_dbg_info, "runlist %d : act eng bitmask 0x%x",
742 runlist_id, runlist->eng_bitmask); 767 runlist_id, runlist->eng_bitmask);
@@ -791,12 +816,14 @@ int gk20a_init_fifo_reset_enable_hw(struct gk20a *g)
791 /* enable pmc pfifo */ 816 /* enable pmc pfifo */
792 g->ops.mc.reset(g, mc_enable_pfifo_enabled_f()); 817 g->ops.mc.reset(g, mc_enable_pfifo_enabled_f());
793 818
794 if (g->ops.clock_gating.slcg_fifo_load_gating_prod) 819 if (g->ops.clock_gating.slcg_fifo_load_gating_prod) {
795 g->ops.clock_gating.slcg_fifo_load_gating_prod(g, 820 g->ops.clock_gating.slcg_fifo_load_gating_prod(g,
796 g->slcg_enabled); 821 g->slcg_enabled);
797 if (g->ops.clock_gating.blcg_fifo_load_gating_prod) 822 }
823 if (g->ops.clock_gating.blcg_fifo_load_gating_prod) {
798 g->ops.clock_gating.blcg_fifo_load_gating_prod(g, 824 g->ops.clock_gating.blcg_fifo_load_gating_prod(g,
799 g->blcg_enabled); 825 g->blcg_enabled);
826 }
800 827
801 timeout = gk20a_readl(g, fifo_fb_timeout_r()); 828 timeout = gk20a_readl(g, fifo_fb_timeout_r());
802 timeout = set_field(timeout, fifo_fb_timeout_period_m(), 829 timeout = set_field(timeout, fifo_fb_timeout_period_m(),
@@ -812,12 +839,13 @@ int gk20a_init_fifo_reset_enable_hw(struct gk20a *g)
812 nvgpu_log_info(g, "pbdma_timeout reg val = 0x%08x", timeout); 839 nvgpu_log_info(g, "pbdma_timeout reg val = 0x%08x", timeout);
813 gk20a_writel(g, pbdma_timeout_r(i), timeout); 840 gk20a_writel(g, pbdma_timeout_r(i), timeout);
814 } 841 }
815 if (g->ops.fifo.apply_pb_timeout) 842 if (g->ops.fifo.apply_pb_timeout) {
816 g->ops.fifo.apply_pb_timeout(g); 843 g->ops.fifo.apply_pb_timeout(g);
844 }
817 845
818 if (g->ops.fifo.apply_ctxsw_timeout_intr) 846 if (g->ops.fifo.apply_ctxsw_timeout_intr) {
819 g->ops.fifo.apply_ctxsw_timeout_intr(g); 847 g->ops.fifo.apply_ctxsw_timeout_intr(g);
820 else { 848 } else {
821 timeout = g->fifo_eng_timeout_us; 849 timeout = g->fifo_eng_timeout_us;
822 timeout = scale_ptimer(timeout, 850 timeout = scale_ptimer(timeout,
823 ptimer_scalingfactor10x(g->ptimer_src_freq)); 851 ptimer_scalingfactor10x(g->ptimer_src_freq));
@@ -989,14 +1017,14 @@ int gk20a_init_fifo_setup_sw(struct gk20a *g)
989 return err; 1017 return err;
990 } 1018 }
991 1019
992 if (g->ops.mm.is_bar1_supported(g)) 1020 if (g->ops.mm.is_bar1_supported(g)) {
993 err = nvgpu_dma_alloc_map_sys(g->mm.bar1.vm, 1021 err = nvgpu_dma_alloc_map_sys(g->mm.bar1.vm,
994 f->userd_entry_size * f->num_channels, 1022 f->userd_entry_size * f->num_channels,
995 &f->userd); 1023 &f->userd);
996 1024 } else {
997 else
998 err = nvgpu_dma_alloc_sys(g, f->userd_entry_size * 1025 err = nvgpu_dma_alloc_sys(g, f->userd_entry_size *
999 f->num_channels, &f->userd); 1026 f->num_channels, &f->userd);
1027 }
1000 if (err) { 1028 if (err) {
1001 nvgpu_err(g, "userd memory allocation failed"); 1029 nvgpu_err(g, "userd memory allocation failed");
1002 goto clean_up; 1030 goto clean_up;
@@ -1012,8 +1040,9 @@ int gk20a_init_fifo_setup_sw(struct gk20a *g)
1012 } 1040 }
1013 1041
1014 err = nvgpu_channel_worker_init(g); 1042 err = nvgpu_channel_worker_init(g);
1015 if (err) 1043 if (err) {
1016 goto clean_up; 1044 goto clean_up;
1045 }
1017 1046
1018 f->sw_ready = true; 1047 f->sw_ready = true;
1019 1048
@@ -1023,10 +1052,11 @@ int gk20a_init_fifo_setup_sw(struct gk20a *g)
1023clean_up: 1052clean_up:
1024 nvgpu_log_fn(g, "fail"); 1053 nvgpu_log_fn(g, "fail");
1025 if (nvgpu_mem_is_valid(&f->userd)) { 1054 if (nvgpu_mem_is_valid(&f->userd)) {
1026 if (g->ops.mm.is_bar1_supported(g)) 1055 if (g->ops.mm.is_bar1_supported(g)) {
1027 nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd); 1056 nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd);
1028 else 1057 } else {
1029 nvgpu_dma_free(g, &f->userd); 1058 nvgpu_dma_free(g, &f->userd);
1059 }
1030 } 1060 }
1031 1061
1032 return err; 1062 return err;
@@ -1107,13 +1137,16 @@ int gk20a_init_fifo_support(struct gk20a *g)
1107 u32 err; 1137 u32 err;
1108 1138
1109 err = g->ops.fifo.setup_sw(g); 1139 err = g->ops.fifo.setup_sw(g);
1110 if (err) 1140 if (err) {
1111 return err; 1141 return err;
1142 }
1112 1143
1113 if (g->ops.fifo.init_fifo_setup_hw) 1144 if (g->ops.fifo.init_fifo_setup_hw) {
1114 err = g->ops.fifo.init_fifo_setup_hw(g); 1145 err = g->ops.fifo.init_fifo_setup_hw(g);
1115 if (err) 1146 }
1147 if (err) {
1116 return err; 1148 return err;
1149 }
1117 1150
1118 return err; 1151 return err;
1119} 1152}
@@ -1124,20 +1157,23 @@ gk20a_refch_from_inst_ptr(struct gk20a *g, u64 inst_ptr)
1124{ 1157{
1125 struct fifo_gk20a *f = &g->fifo; 1158 struct fifo_gk20a *f = &g->fifo;
1126 unsigned int ci; 1159 unsigned int ci;
1127 if (unlikely(!f->channel)) 1160 if (unlikely(!f->channel)) {
1128 return NULL; 1161 return NULL;
1162 }
1129 for (ci = 0; ci < f->num_channels; ci++) { 1163 for (ci = 0; ci < f->num_channels; ci++) {
1130 struct channel_gk20a *ch; 1164 struct channel_gk20a *ch;
1131 u64 ch_inst_ptr; 1165 u64 ch_inst_ptr;
1132 1166
1133 ch = gk20a_channel_get(&f->channel[ci]); 1167 ch = gk20a_channel_get(&f->channel[ci]);
1134 /* only alive channels are searched */ 1168 /* only alive channels are searched */
1135 if (!ch) 1169 if (!ch) {
1136 continue; 1170 continue;
1171 }
1137 1172
1138 ch_inst_ptr = nvgpu_inst_block_addr(g, &ch->inst_block); 1173 ch_inst_ptr = nvgpu_inst_block_addr(g, &ch->inst_block);
1139 if (inst_ptr == ch_inst_ptr) 1174 if (inst_ptr == ch_inst_ptr) {
1140 return ch; 1175 return ch;
1176 }
1141 1177
1142 gk20a_channel_put(ch); 1178 gk20a_channel_put(ch);
1143 } 1179 }
@@ -1199,34 +1235,37 @@ static const char * const does_not_exist[] = {
1199/* fill in mmu fault desc */ 1235/* fill in mmu fault desc */
1200void gk20a_fifo_get_mmu_fault_desc(struct mmu_fault_info *mmfault) 1236void gk20a_fifo_get_mmu_fault_desc(struct mmu_fault_info *mmfault)
1201{ 1237{
1202 if (mmfault->fault_type >= ARRAY_SIZE(gk20a_fault_type_descs)) 1238 if (mmfault->fault_type >= ARRAY_SIZE(gk20a_fault_type_descs)) {
1203 WARN_ON(mmfault->fault_type >= 1239 WARN_ON(mmfault->fault_type >=
1204 ARRAY_SIZE(gk20a_fault_type_descs)); 1240 ARRAY_SIZE(gk20a_fault_type_descs));
1205 else 1241 } else {
1206 mmfault->fault_type_desc = 1242 mmfault->fault_type_desc =
1207 gk20a_fault_type_descs[mmfault->fault_type]; 1243 gk20a_fault_type_descs[mmfault->fault_type];
1244 }
1208} 1245}
1209 1246
1210/* fill in mmu fault client description */ 1247/* fill in mmu fault client description */
1211void gk20a_fifo_get_mmu_fault_client_desc(struct mmu_fault_info *mmfault) 1248void gk20a_fifo_get_mmu_fault_client_desc(struct mmu_fault_info *mmfault)
1212{ 1249{
1213 if (mmfault->client_id >= ARRAY_SIZE(gk20a_hub_client_descs)) 1250 if (mmfault->client_id >= ARRAY_SIZE(gk20a_hub_client_descs)) {
1214 WARN_ON(mmfault->client_id >= 1251 WARN_ON(mmfault->client_id >=
1215 ARRAY_SIZE(gk20a_hub_client_descs)); 1252 ARRAY_SIZE(gk20a_hub_client_descs));
1216 else 1253 } else {
1217 mmfault->client_id_desc = 1254 mmfault->client_id_desc =
1218 gk20a_hub_client_descs[mmfault->client_id]; 1255 gk20a_hub_client_descs[mmfault->client_id];
1256 }
1219} 1257}
1220 1258
1221/* fill in mmu fault gpc description */ 1259/* fill in mmu fault gpc description */
1222void gk20a_fifo_get_mmu_fault_gpc_desc(struct mmu_fault_info *mmfault) 1260void gk20a_fifo_get_mmu_fault_gpc_desc(struct mmu_fault_info *mmfault)
1223{ 1261{
1224 if (mmfault->client_id >= ARRAY_SIZE(gk20a_gpc_client_descs)) 1262 if (mmfault->client_id >= ARRAY_SIZE(gk20a_gpc_client_descs)) {
1225 WARN_ON(mmfault->client_id >= 1263 WARN_ON(mmfault->client_id >=
1226 ARRAY_SIZE(gk20a_gpc_client_descs)); 1264 ARRAY_SIZE(gk20a_gpc_client_descs));
1227 else 1265 } else {
1228 mmfault->client_id_desc = 1266 mmfault->client_id_desc =
1229 gk20a_gpc_client_descs[mmfault->client_id]; 1267 gk20a_gpc_client_descs[mmfault->client_id];
1268 }
1230} 1269}
1231 1270
1232static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id, 1271static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id,
@@ -1236,8 +1275,9 @@ static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id,
1236 1275
1237 /* parse info */ 1276 /* parse info */
1238 mmfault->fault_type_desc = does_not_exist[0]; 1277 mmfault->fault_type_desc = does_not_exist[0];
1239 if (g->ops.fifo.get_mmu_fault_desc) 1278 if (g->ops.fifo.get_mmu_fault_desc) {
1240 g->ops.fifo.get_mmu_fault_desc(mmfault); 1279 g->ops.fifo.get_mmu_fault_desc(mmfault);
1280 }
1241 1281
1242 if (mmfault->client_type >= ARRAY_SIZE(engine_subid_descs)) { 1282 if (mmfault->client_type >= ARRAY_SIZE(engine_subid_descs)) {
1243 WARN_ON(mmfault->client_type >= ARRAY_SIZE(engine_subid_descs)); 1283 WARN_ON(mmfault->client_type >= ARRAY_SIZE(engine_subid_descs));
@@ -1250,12 +1290,13 @@ static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id,
1250 mmfault->client_id_desc = does_not_exist[0]; 1290 mmfault->client_id_desc = does_not_exist[0];
1251 if ((mmfault->client_type == 1291 if ((mmfault->client_type ==
1252 fifo_intr_mmu_fault_info_engine_subid_hub_v()) 1292 fifo_intr_mmu_fault_info_engine_subid_hub_v())
1253 && g->ops.fifo.get_mmu_fault_client_desc) 1293 && g->ops.fifo.get_mmu_fault_client_desc) {
1254 g->ops.fifo.get_mmu_fault_client_desc(mmfault); 1294 g->ops.fifo.get_mmu_fault_client_desc(mmfault);
1255 else if ((mmfault->client_type == 1295 } else if ((mmfault->client_type ==
1256 fifo_intr_mmu_fault_info_engine_subid_gpc_v()) 1296 fifo_intr_mmu_fault_info_engine_subid_gpc_v())
1257 && g->ops.fifo.get_mmu_fault_gpc_desc) 1297 && g->ops.fifo.get_mmu_fault_gpc_desc) {
1258 g->ops.fifo.get_mmu_fault_gpc_desc(mmfault); 1298 g->ops.fifo.get_mmu_fault_gpc_desc(mmfault);
1299 }
1259} 1300}
1260 1301
1261/* reads info from hardware and fills in mmu fault info record */ 1302/* reads info from hardware and fills in mmu fault info record */
@@ -1297,21 +1338,25 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
1297 1338
1298 nvgpu_log_fn(g, " "); 1339 nvgpu_log_fn(g, " ");
1299 1340
1300 if (!g) 1341 if (!g) {
1301 return; 1342 return;
1343 }
1302 1344
1303 engine_info = gk20a_fifo_get_engine_info(g, engine_id); 1345 engine_info = gk20a_fifo_get_engine_info(g, engine_id);
1304 1346
1305 if (engine_info) 1347 if (engine_info) {
1306 engine_enum = engine_info->engine_enum; 1348 engine_enum = engine_info->engine_enum;
1349 }
1307 1350
1308 if (engine_enum == ENGINE_INVAL_GK20A) 1351 if (engine_enum == ENGINE_INVAL_GK20A) {
1309 nvgpu_err(g, "unsupported engine_id %d", engine_id); 1352 nvgpu_err(g, "unsupported engine_id %d", engine_id);
1353 }
1310 1354
1311 if (engine_enum == ENGINE_GR_GK20A) { 1355 if (engine_enum == ENGINE_GR_GK20A) {
1312 if (g->support_pmu && g->can_elpg) { 1356 if (g->support_pmu && g->can_elpg) {
1313 if (nvgpu_pmu_disable_elpg(g)) 1357 if (nvgpu_pmu_disable_elpg(g)) {
1314 nvgpu_err(g, "failed to set disable elpg"); 1358 nvgpu_err(g, "failed to set disable elpg");
1359 }
1315 } 1360 }
1316 1361
1317#ifdef CONFIG_GK20A_CTXSW_TRACE 1362#ifdef CONFIG_GK20A_CTXSW_TRACE
@@ -1324,8 +1369,9 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
1324#endif 1369#endif
1325 if (!nvgpu_platform_is_simulation(g)) { 1370 if (!nvgpu_platform_is_simulation(g)) {
1326 /*HALT_PIPELINE method, halt GR engine*/ 1371 /*HALT_PIPELINE method, halt GR engine*/
1327 if (gr_gk20a_halt_pipe(g)) 1372 if (gr_gk20a_halt_pipe(g)) {
1328 nvgpu_err(g, "failed to HALT gr pipe"); 1373 nvgpu_err(g, "failed to HALT gr pipe");
1374 }
1329 /* 1375 /*
1330 * resetting engine using mc_enable_r() is not 1376 * resetting engine using mc_enable_r() is not
1331 * enough, we do full init sequence 1377 * enough, we do full init sequence
@@ -1337,8 +1383,9 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
1337 "HALT gr pipe not supported and " 1383 "HALT gr pipe not supported and "
1338 "gr cannot be reset without halting gr pipe"); 1384 "gr cannot be reset without halting gr pipe");
1339 } 1385 }
1340 if (g->support_pmu && g->can_elpg) 1386 if (g->support_pmu && g->can_elpg) {
1341 nvgpu_pmu_enable_elpg(g); 1387 nvgpu_pmu_enable_elpg(g);
1388 }
1342 } 1389 }
1343 if ((engine_enum == ENGINE_GRCE_GK20A) || 1390 if ((engine_enum == ENGINE_GRCE_GK20A) ||
1344 (engine_enum == ENGINE_ASYNC_CE_GK20A)) { 1391 (engine_enum == ENGINE_ASYNC_CE_GK20A)) {
@@ -1373,29 +1420,35 @@ bool gk20a_fifo_should_defer_engine_reset(struct gk20a *g, u32 engine_id,
1373 u32 engine_enum = ENGINE_INVAL_GK20A; 1420 u32 engine_enum = ENGINE_INVAL_GK20A;
1374 struct fifo_engine_info_gk20a *engine_info; 1421 struct fifo_engine_info_gk20a *engine_info;
1375 1422
1376 if (!g) 1423 if (!g) {
1377 return false; 1424 return false;
1425 }
1378 1426
1379 engine_info = gk20a_fifo_get_engine_info(g, engine_id); 1427 engine_info = gk20a_fifo_get_engine_info(g, engine_id);
1380 1428
1381 if (engine_info) 1429 if (engine_info) {
1382 engine_enum = engine_info->engine_enum; 1430 engine_enum = engine_info->engine_enum;
1431 }
1383 1432
1384 if (engine_enum == ENGINE_INVAL_GK20A) 1433 if (engine_enum == ENGINE_INVAL_GK20A) {
1385 return false; 1434 return false;
1435 }
1386 1436
1387 /* channel recovery is only deferred if an sm debugger 1437 /* channel recovery is only deferred if an sm debugger
1388 is attached and has MMU debug mode is enabled */ 1438 is attached and has MMU debug mode is enabled */
1389 if (!g->ops.gr.sm_debugger_attached(g) || 1439 if (!g->ops.gr.sm_debugger_attached(g) ||
1390 !g->ops.fb.is_debug_mode_enabled(g)) 1440 !g->ops.fb.is_debug_mode_enabled(g)) {
1391 return false; 1441 return false;
1442 }
1392 1443
1393 /* if this fault is fake (due to RC recovery), don't defer recovery */ 1444 /* if this fault is fake (due to RC recovery), don't defer recovery */
1394 if (fake_fault) 1445 if (fake_fault) {
1395 return false; 1446 return false;
1447 }
1396 1448
1397 if (engine_enum != ENGINE_GR_GK20A) 1449 if (engine_enum != ENGINE_GR_GK20A) {
1398 return false; 1450 return false;
1451 }
1399 1452
1400 return g->ops.fifo.is_fault_engine_subid_gpc(g, engine_subid); 1453 return g->ops.fifo.is_fault_engine_subid_gpc(g, engine_subid);
1401} 1454}
@@ -1405,12 +1458,14 @@ static bool gk20a_fifo_ch_timeout_debug_dump_state(struct gk20a *g,
1405 struct channel_gk20a *refch) 1458 struct channel_gk20a *refch)
1406{ 1459{
1407 bool verbose = true; 1460 bool verbose = true;
1408 if (!refch) 1461 if (!refch) {
1409 return verbose; 1462 return verbose;
1463 }
1410 1464
1411 if (nvgpu_is_error_notifier_set(refch, 1465 if (nvgpu_is_error_notifier_set(refch,
1412 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT)) 1466 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT)) {
1413 verbose = refch->timeout_debug_dump; 1467 verbose = refch->timeout_debug_dump;
1468 }
1414 1469
1415 return verbose; 1470 return verbose;
1416} 1471}
@@ -1499,15 +1554,17 @@ void gk20a_fifo_abort_tsg(struct gk20a *g, u32 tsgid, bool preempt)
1499 1554
1500 g->ops.fifo.disable_tsg(tsg); 1555 g->ops.fifo.disable_tsg(tsg);
1501 1556
1502 if (preempt) 1557 if (preempt) {
1503 g->ops.fifo.preempt_tsg(g, tsgid); 1558 g->ops.fifo.preempt_tsg(g, tsgid);
1559 }
1504 1560
1505 nvgpu_rwsem_down_read(&tsg->ch_list_lock); 1561 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
1506 nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { 1562 nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) {
1507 if (gk20a_channel_get(ch)) { 1563 if (gk20a_channel_get(ch)) {
1508 ch->has_timedout = true; 1564 ch->has_timedout = true;
1509 if (ch->g->ops.fifo.ch_abort_clean_up) 1565 if (ch->g->ops.fifo.ch_abort_clean_up) {
1510 ch->g->ops.fifo.ch_abort_clean_up(ch); 1566 ch->g->ops.fifo.ch_abort_clean_up(ch);
1567 }
1511 gk20a_channel_put(ch); 1568 gk20a_channel_put(ch);
1512 } 1569 }
1513 } 1570 }
@@ -1521,23 +1578,27 @@ int gk20a_fifo_deferred_reset(struct gk20a *g, struct channel_gk20a *ch)
1521 nvgpu_mutex_acquire(&g->dbg_sessions_lock); 1578 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1522 gr_gk20a_disable_ctxsw(g); 1579 gr_gk20a_disable_ctxsw(g);
1523 1580
1524 if (!g->fifo.deferred_reset_pending) 1581 if (!g->fifo.deferred_reset_pending) {
1525 goto clean_up; 1582 goto clean_up;
1583 }
1526 1584
1527 if (gk20a_is_channel_marked_as_tsg(ch)) 1585 if (gk20a_is_channel_marked_as_tsg(ch)) {
1528 engines = gk20a_fifo_engines_on_id(g, ch->tsgid, true); 1586 engines = gk20a_fifo_engines_on_id(g, ch->tsgid, true);
1529 else 1587 } else {
1530 engines = gk20a_fifo_engines_on_id(g, ch->chid, false); 1588 engines = gk20a_fifo_engines_on_id(g, ch->chid, false);
1531 if (!engines) 1589 }
1590 if (!engines) {
1532 goto clean_up; 1591 goto clean_up;
1592 }
1533 1593
1534 /* 1594 /*
1535 * If deferred reset is set for an engine, and channel is running 1595 * If deferred reset is set for an engine, and channel is running
1536 * on that engine, reset it 1596 * on that engine, reset it
1537 */ 1597 */
1538 for_each_set_bit(engine_id, &g->fifo.deferred_fault_engines, 32) { 1598 for_each_set_bit(engine_id, &g->fifo.deferred_fault_engines, 32) {
1539 if (BIT(engine_id) & engines) 1599 if (BIT(engine_id) & engines) {
1540 gk20a_fifo_reset_engine(g, engine_id); 1600 gk20a_fifo_reset_engine(g, engine_id);
1601 }
1541 } 1602 }
1542 1603
1543 g->fifo.deferred_fault_engines = 0; 1604 g->fifo.deferred_fault_engines = 0;
@@ -1568,18 +1629,22 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1568 1629
1569 /* Disable power management */ 1630 /* Disable power management */
1570 if (g->support_pmu && g->can_elpg) { 1631 if (g->support_pmu && g->can_elpg) {
1571 if (nvgpu_pmu_disable_elpg(g)) 1632 if (nvgpu_pmu_disable_elpg(g)) {
1572 nvgpu_err(g, "failed to set disable elpg"); 1633 nvgpu_err(g, "failed to set disable elpg");
1634 }
1573 } 1635 }
1574 if (g->ops.clock_gating.slcg_gr_load_gating_prod) 1636 if (g->ops.clock_gating.slcg_gr_load_gating_prod) {
1575 g->ops.clock_gating.slcg_gr_load_gating_prod(g, 1637 g->ops.clock_gating.slcg_gr_load_gating_prod(g,
1576 false); 1638 false);
1577 if (g->ops.clock_gating.slcg_perf_load_gating_prod) 1639 }
1640 if (g->ops.clock_gating.slcg_perf_load_gating_prod) {
1578 g->ops.clock_gating.slcg_perf_load_gating_prod(g, 1641 g->ops.clock_gating.slcg_perf_load_gating_prod(g,
1579 false); 1642 false);
1580 if (g->ops.clock_gating.slcg_ltc_load_gating_prod) 1643 }
1644 if (g->ops.clock_gating.slcg_ltc_load_gating_prod) {
1581 g->ops.clock_gating.slcg_ltc_load_gating_prod(g, 1645 g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
1582 false); 1646 false);
1647 }
1583 1648
1584 gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); 1649 gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN);
1585 1650
@@ -1673,9 +1738,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1673 fifo_engine_status_id_type_chid_v(); 1738 fifo_engine_status_id_type_chid_v();
1674 } 1739 }
1675 1740
1676 if (type == fifo_engine_status_id_type_tsgid_v()) 1741 if (type == fifo_engine_status_id_type_tsgid_v()) {
1677 tsg = &g->fifo.tsg[id]; 1742 tsg = &g->fifo.tsg[id];
1678 else if (type == fifo_engine_status_id_type_chid_v()) { 1743 } else if (type == fifo_engine_status_id_type_chid_v()) {
1679 ch = &g->fifo.channel[id]; 1744 ch = &g->fifo.channel[id];
1680 refch = gk20a_channel_get(ch); 1745 refch = gk20a_channel_get(ch);
1681 } 1746 }
@@ -1686,8 +1751,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1686 refch = ch; 1751 refch = ch;
1687 } 1752 }
1688 1753
1689 if (ch && gk20a_is_channel_marked_as_tsg(ch)) 1754 if (ch && gk20a_is_channel_marked_as_tsg(ch)) {
1690 tsg = &g->fifo.tsg[ch->tsgid]; 1755 tsg = &g->fifo.tsg[ch->tsgid];
1756 }
1691 1757
1692 /* check if engine reset should be deferred */ 1758 /* check if engine reset should be deferred */
1693 if (engine_id != FIFO_INVAL_ENGINE_ID) { 1759 if (engine_id != FIFO_INVAL_ENGINE_ID) {
@@ -1730,24 +1796,27 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1730 if (g->fifo.deferred_reset_pending) { 1796 if (g->fifo.deferred_reset_pending) {
1731 gk20a_disable_tsg(tsg); 1797 gk20a_disable_tsg(tsg);
1732 } else { 1798 } else {
1733 if (!fake_fault) 1799 if (!fake_fault) {
1734 gk20a_fifo_set_ctx_mmu_error_tsg(g, 1800 gk20a_fifo_set_ctx_mmu_error_tsg(g,
1735 tsg); 1801 tsg);
1802 }
1736 verbose = gk20a_fifo_error_tsg(g, tsg); 1803 verbose = gk20a_fifo_error_tsg(g, tsg);
1737 gk20a_fifo_abort_tsg(g, tsg->tsgid, false); 1804 gk20a_fifo_abort_tsg(g, tsg->tsgid, false);
1738 } 1805 }
1739 1806
1740 /* put back the ref taken early above */ 1807 /* put back the ref taken early above */
1741 if (refch) 1808 if (refch) {
1742 gk20a_channel_put(ch); 1809 gk20a_channel_put(ch);
1810 }
1743 } else if (ch) { 1811 } else if (ch) {
1744 if (refch) { 1812 if (refch) {
1745 if (g->fifo.deferred_reset_pending) { 1813 if (g->fifo.deferred_reset_pending) {
1746 g->ops.fifo.disable_channel(ch); 1814 g->ops.fifo.disable_channel(ch);
1747 } else { 1815 } else {
1748 if (!fake_fault) 1816 if (!fake_fault) {
1749 gk20a_fifo_set_ctx_mmu_error_ch( 1817 gk20a_fifo_set_ctx_mmu_error_ch(
1750 g, refch); 1818 g, refch);
1819 }
1751 1820
1752 verbose = gk20a_fifo_error_ch(g, 1821 verbose = gk20a_fifo_error_ch(g,
1753 refch); 1822 refch);
@@ -1765,8 +1834,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1765 } else if (mmfault_info.inst_ptr == 1834 } else if (mmfault_info.inst_ptr ==
1766 nvgpu_inst_block_addr(g, &g->mm.pmu.inst_block)) { 1835 nvgpu_inst_block_addr(g, &g->mm.pmu.inst_block)) {
1767 nvgpu_err(g, "mmu fault from pmu"); 1836 nvgpu_err(g, "mmu fault from pmu");
1768 } else 1837 } else {
1769 nvgpu_err(g, "couldn't locate channel for mmu fault"); 1838 nvgpu_err(g, "couldn't locate channel for mmu fault");
1839 }
1770 } 1840 }
1771 1841
1772 /* clear interrupt */ 1842 /* clear interrupt */
@@ -1782,8 +1852,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked(
1782 gr_gpfifo_ctl_semaphore_access_enabled_f()); 1852 gr_gpfifo_ctl_semaphore_access_enabled_f());
1783 1853
1784 /* It is safe to enable ELPG again. */ 1854 /* It is safe to enable ELPG again. */
1785 if (g->support_pmu && g->can_elpg) 1855 if (g->support_pmu && g->can_elpg) {
1786 nvgpu_pmu_enable_elpg(g); 1856 nvgpu_pmu_enable_elpg(g);
1857 }
1787 1858
1788 return verbose; 1859 return verbose;
1789} 1860}
@@ -1856,8 +1927,9 @@ static u32 gk20a_fifo_engines_on_id(struct gk20a *g, u32 id, bool is_tsg)
1856 if ((is_tsg && type == 1927 if ((is_tsg && type ==
1857 fifo_engine_status_id_type_tsgid_v()) || 1928 fifo_engine_status_id_type_tsgid_v()) ||
1858 (!is_tsg && type == 1929 (!is_tsg && type ==
1859 fifo_engine_status_id_type_chid_v())) 1930 fifo_engine_status_id_type_chid_v())) {
1860 engines |= BIT(active_engine_id); 1931 engines |= BIT(active_engine_id);
1932 }
1861 } 1933 }
1862 } 1934 }
1863 1935
@@ -1875,17 +1947,18 @@ void gk20a_fifo_recover_ch(struct gk20a *g, u32 chid, bool verbose, int rc_type)
1875 1947
1876 engines = gk20a_fifo_engines_on_id(g, chid, false); 1948 engines = gk20a_fifo_engines_on_id(g, chid, false);
1877 1949
1878 if (engines) 1950 if (engines) {
1879 gk20a_fifo_recover(g, engines, chid, false, true, verbose, 1951 gk20a_fifo_recover(g, engines, chid, false, true, verbose,
1880 rc_type); 1952 rc_type);
1881 else { 1953 } else {
1882 struct channel_gk20a *ch = &g->fifo.channel[chid]; 1954 struct channel_gk20a *ch = &g->fifo.channel[chid];
1883 1955
1884 if (gk20a_channel_get(ch)) { 1956 if (gk20a_channel_get(ch)) {
1885 gk20a_channel_abort(ch, false); 1957 gk20a_channel_abort(ch, false);
1886 1958
1887 if (gk20a_fifo_error_ch(g, ch)) 1959 if (gk20a_fifo_error_ch(g, ch)) {
1888 gk20a_debug_dump(g); 1960 gk20a_debug_dump(g);
1961 }
1889 1962
1890 gk20a_channel_put(ch); 1963 gk20a_channel_put(ch);
1891 } 1964 }
@@ -1907,14 +1980,15 @@ void gk20a_fifo_recover_tsg(struct gk20a *g, u32 tsgid, bool verbose,
1907 1980
1908 engines = gk20a_fifo_engines_on_id(g, tsgid, true); 1981 engines = gk20a_fifo_engines_on_id(g, tsgid, true);
1909 1982
1910 if (engines) 1983 if (engines) {
1911 gk20a_fifo_recover(g, engines, tsgid, true, true, verbose, 1984 gk20a_fifo_recover(g, engines, tsgid, true, true, verbose,
1912 rc_type); 1985 rc_type);
1913 else { 1986 } else {
1914 struct tsg_gk20a *tsg = &g->fifo.tsg[tsgid]; 1987 struct tsg_gk20a *tsg = &g->fifo.tsg[tsgid];
1915 1988
1916 if (gk20a_fifo_error_tsg(g, tsg) && verbose) 1989 if (gk20a_fifo_error_tsg(g, tsg) && verbose) {
1917 gk20a_debug_dump(g); 1990 gk20a_debug_dump(g);
1991 }
1918 1992
1919 gk20a_fifo_abort_tsg(g, tsgid, false); 1993 gk20a_fifo_abort_tsg(g, tsgid, false);
1920 } 1994 }
@@ -1956,18 +2030,20 @@ void gk20a_fifo_teardown_ch_tsg(struct gk20a *g, u32 __engine_ids,
1956 for_each_set_bit(engine_id, &engine_ids, 32) { 2030 for_each_set_bit(engine_id, &engine_ids, 32) {
1957 u32 mmu_id = gk20a_engine_id_to_mmu_id(g, engine_id); 2031 u32 mmu_id = gk20a_engine_id_to_mmu_id(g, engine_id);
1958 2032
1959 if (mmu_id != FIFO_INVAL_ENGINE_ID) 2033 if (mmu_id != FIFO_INVAL_ENGINE_ID) {
1960 mmu_fault_engines |= BIT(mmu_id); 2034 mmu_fault_engines |= BIT(mmu_id);
2035 }
1961 } 2036 }
1962 } else { 2037 } else {
1963 /* store faulted engines in advance */ 2038 /* store faulted engines in advance */
1964 for_each_set_bit(engine_id, &_engine_ids, 32) { 2039 for_each_set_bit(engine_id, &_engine_ids, 32) {
1965 gk20a_fifo_get_faulty_id_type(g, engine_id, &ref_id, 2040 gk20a_fifo_get_faulty_id_type(g, engine_id, &ref_id,
1966 &ref_type); 2041 &ref_type);
1967 if (ref_type == fifo_engine_status_id_type_tsgid_v()) 2042 if (ref_type == fifo_engine_status_id_type_tsgid_v()) {
1968 ref_id_is_tsg = true; 2043 ref_id_is_tsg = true;
1969 else 2044 } else {
1970 ref_id_is_tsg = false; 2045 ref_id_is_tsg = false;
2046 }
1971 /* Reset *all* engines that use the 2047 /* Reset *all* engines that use the
1972 * same channel as faulty engine */ 2048 * same channel as faulty engine */
1973 for (i = 0; i < g->fifo.num_engines; i++) { 2049 for (i = 0; i < g->fifo.num_engines; i++) {
@@ -1980,8 +2056,9 @@ void gk20a_fifo_teardown_ch_tsg(struct gk20a *g, u32 __engine_ids,
1980 u32 mmu_id = gk20a_engine_id_to_mmu_id(g, active_engine_id); 2056 u32 mmu_id = gk20a_engine_id_to_mmu_id(g, active_engine_id);
1981 2057
1982 engine_ids |= BIT(active_engine_id); 2058 engine_ids |= BIT(active_engine_id);
1983 if (mmu_id != FIFO_INVAL_ENGINE_ID) 2059 if (mmu_id != FIFO_INVAL_ENGINE_ID) {
1984 mmu_fault_engines |= BIT(mmu_id); 2060 mmu_fault_engines |= BIT(mmu_id);
2061 }
1985 } 2062 }
1986 } 2063 }
1987 } 2064 }
@@ -2021,16 +2098,19 @@ void gk20a_fifo_recover(struct gk20a *g, u32 __engine_ids,
2021{ 2098{
2022 unsigned int id_type; 2099 unsigned int id_type;
2023 2100
2024 if (verbose) 2101 if (verbose) {
2025 gk20a_debug_dump(g); 2102 gk20a_debug_dump(g);
2103 }
2026 2104
2027 if (g->ops.ltc.flush) 2105 if (g->ops.ltc.flush) {
2028 g->ops.ltc.flush(g); 2106 g->ops.ltc.flush(g);
2107 }
2029 2108
2030 if (id_is_known) 2109 if (id_is_known) {
2031 id_type = id_is_tsg ? ID_TYPE_TSG : ID_TYPE_CHANNEL; 2110 id_type = id_is_tsg ? ID_TYPE_TSG : ID_TYPE_CHANNEL;
2032 else 2111 } else {
2033 id_type = ID_TYPE_UNKNOWN; 2112 id_type = ID_TYPE_UNKNOWN;
2113 }
2034 2114
2035 g->ops.fifo.teardown_ch_tsg(g, __engine_ids, hw_id, id_type, 2115 g->ops.fifo.teardown_ch_tsg(g, __engine_ids, hw_id, id_type,
2036 rc_type, NULL); 2116 rc_type, NULL);
@@ -2080,11 +2160,13 @@ int gk20a_fifo_tsg_unbind_channel_verify_status(struct channel_gk20a *ch)
2080 return -EINVAL; 2160 return -EINVAL;
2081 } 2161 }
2082 2162
2083 if (g->ops.fifo.tsg_verify_status_ctx_reload) 2163 if (g->ops.fifo.tsg_verify_status_ctx_reload) {
2084 g->ops.fifo.tsg_verify_status_ctx_reload(ch); 2164 g->ops.fifo.tsg_verify_status_ctx_reload(ch);
2165 }
2085 2166
2086 if (g->ops.fifo.tsg_verify_status_faulted) 2167 if (g->ops.fifo.tsg_verify_status_faulted) {
2087 g->ops.fifo.tsg_verify_status_faulted(ch); 2168 g->ops.fifo.tsg_verify_status_faulted(ch);
2169 }
2088 2170
2089 return 0; 2171 return 0;
2090} 2172}
@@ -2106,19 +2188,22 @@ int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch)
2106 g->ops.fifo.disable_tsg(tsg); 2188 g->ops.fifo.disable_tsg(tsg);
2107 2189
2108 err = g->ops.fifo.preempt_tsg(g, tsg->tsgid); 2190 err = g->ops.fifo.preempt_tsg(g, tsg->tsgid);
2109 if (err) 2191 if (err) {
2110 goto fail_enable_tsg; 2192 goto fail_enable_tsg;
2193 }
2111 2194
2112 if (g->ops.fifo.tsg_verify_channel_status && !tsg_timedout) { 2195 if (g->ops.fifo.tsg_verify_channel_status && !tsg_timedout) {
2113 err = g->ops.fifo.tsg_verify_channel_status(ch); 2196 err = g->ops.fifo.tsg_verify_channel_status(ch);
2114 if (err) 2197 if (err) {
2115 goto fail_enable_tsg; 2198 goto fail_enable_tsg;
2199 }
2116 } 2200 }
2117 2201
2118 /* Channel should be seen as TSG channel while updating runlist */ 2202 /* Channel should be seen as TSG channel while updating runlist */
2119 err = channel_gk20a_update_runlist(ch, false); 2203 err = channel_gk20a_update_runlist(ch, false);
2120 if (err) 2204 if (err) {
2121 goto fail_enable_tsg; 2205 goto fail_enable_tsg;
2206 }
2122 2207
2123 /* Remove channel from TSG and re-enable rest of the channels */ 2208 /* Remove channel from TSG and re-enable rest of the channels */
2124 nvgpu_rwsem_down_write(&tsg->ch_list_lock); 2209 nvgpu_rwsem_down_write(&tsg->ch_list_lock);
@@ -2131,17 +2216,20 @@ int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch)
2131 * Note that we can skip disabling and preempting TSG too in case of 2216 * Note that we can skip disabling and preempting TSG too in case of
2132 * time out, but we keep that to ensure TSG is kicked out 2217 * time out, but we keep that to ensure TSG is kicked out
2133 */ 2218 */
2134 if (!tsg_timedout) 2219 if (!tsg_timedout) {
2135 g->ops.fifo.enable_tsg(tsg); 2220 g->ops.fifo.enable_tsg(tsg);
2221 }
2136 2222
2137 if (ch->g->ops.fifo.ch_abort_clean_up) 2223 if (ch->g->ops.fifo.ch_abort_clean_up) {
2138 ch->g->ops.fifo.ch_abort_clean_up(ch); 2224 ch->g->ops.fifo.ch_abort_clean_up(ch);
2225 }
2139 2226
2140 return 0; 2227 return 0;
2141 2228
2142fail_enable_tsg: 2229fail_enable_tsg:
2143 if (!tsg_timedout) 2230 if (!tsg_timedout) {
2144 g->ops.fifo.enable_tsg(tsg); 2231 g->ops.fifo.enable_tsg(tsg);
2232 }
2145 return err; 2233 return err;
2146} 2234}
2147 2235
@@ -2225,9 +2313,10 @@ bool gk20a_fifo_check_ch_ctxsw_timeout(struct channel_gk20a *ch,
2225 &progress); 2313 &progress);
2226 *verbose = ch->timeout_debug_dump; 2314 *verbose = ch->timeout_debug_dump;
2227 *ms = ch->timeout_accumulated_ms; 2315 *ms = ch->timeout_accumulated_ms;
2228 if (recover) 2316 if (recover) {
2229 g->ops.fifo.set_error_notifier(ch, 2317 g->ops.fifo.set_error_notifier(ch,
2230 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT); 2318 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT);
2319 }
2231 2320
2232 gk20a_channel_put(ch); 2321 gk20a_channel_put(ch);
2233 } 2322 }
@@ -2255,8 +2344,9 @@ bool gk20a_fifo_check_tsg_ctxsw_timeout(struct tsg_gk20a *tsg,
2255 if (gk20a_channel_get(ch)) { 2344 if (gk20a_channel_get(ch)) {
2256 recover = gk20a_channel_update_and_check_timeout(ch, 2345 recover = gk20a_channel_update_and_check_timeout(ch,
2257 *ms, &progress); 2346 *ms, &progress);
2258 if (progress || recover) 2347 if (progress || recover) {
2259 break; 2348 break;
2349 }
2260 gk20a_channel_put(ch); 2350 gk20a_channel_put(ch);
2261 } 2351 }
2262 } 2352 }
@@ -2481,9 +2571,11 @@ static bool gk20a_fifo_is_sw_method_subch(struct gk20a *g, int pbdma_id,
2481 pbdma_method_subch = pbdma_method0_subch_v( 2571 pbdma_method_subch = pbdma_method0_subch_v(
2482 gk20a_readl(g, pbdma_method_reg)); 2572 gk20a_readl(g, pbdma_method_reg));
2483 2573
2484 if (pbdma_method_subch == 5 || pbdma_method_subch == 6 || 2574 if (pbdma_method_subch == 5 ||
2485 pbdma_method_subch == 7) 2575 pbdma_method_subch == 6 ||
2576 pbdma_method_subch == 7) {
2486 return true; 2577 return true;
2578 }
2487 2579
2488 return false; 2580 return false;
2489} 2581}
@@ -2565,9 +2657,10 @@ unsigned int gk20a_fifo_handle_pbdma_intr_0(struct gk20a *g, u32 pbdma_id,
2565 2657
2566 for (i = 0; i < 4; i++) { 2658 for (i = 0; i < 4; i++) {
2567 if (gk20a_fifo_is_sw_method_subch(g, 2659 if (gk20a_fifo_is_sw_method_subch(g,
2568 pbdma_id, i)) 2660 pbdma_id, i)) {
2569 gk20a_fifo_reset_pbdma_method(g, 2661 gk20a_fifo_reset_pbdma_method(g,
2570 pbdma_id, i); 2662 pbdma_id, i);
2663 }
2571 } 2664 }
2572 rc_type = RC_TYPE_PBDMA_FAULT; 2665 rc_type = RC_TYPE_PBDMA_FAULT;
2573 } 2666 }
@@ -2649,8 +2742,9 @@ u32 gk20a_fifo_handle_pbdma_intr(struct gk20a *g, struct fifo_gk20a *f,
2649 pbdma_id, pbdma_intr_0); 2742 pbdma_id, pbdma_intr_0);
2650 2743
2651 if (g->ops.fifo.handle_pbdma_intr_0(g, pbdma_id, pbdma_intr_0, 2744 if (g->ops.fifo.handle_pbdma_intr_0(g, pbdma_id, pbdma_intr_0,
2652 &handled, &error_notifier) != RC_TYPE_NO_RC) 2745 &handled, &error_notifier) != RC_TYPE_NO_RC) {
2653 rc_type = RC_TYPE_PBDMA_FAULT; 2746 rc_type = RC_TYPE_PBDMA_FAULT;
2747 }
2654 gk20a_writel(g, pbdma_intr_0_r(pbdma_id), pbdma_intr_0); 2748 gk20a_writel(g, pbdma_intr_0_r(pbdma_id), pbdma_intr_0);
2655 } 2749 }
2656 2750
@@ -2660,13 +2754,15 @@ u32 gk20a_fifo_handle_pbdma_intr(struct gk20a *g, struct fifo_gk20a *f,
2660 pbdma_id, pbdma_intr_1); 2754 pbdma_id, pbdma_intr_1);
2661 2755
2662 if (g->ops.fifo.handle_pbdma_intr_1(g, pbdma_id, pbdma_intr_1, 2756 if (g->ops.fifo.handle_pbdma_intr_1(g, pbdma_id, pbdma_intr_1,
2663 &handled, &error_notifier) != RC_TYPE_NO_RC) 2757 &handled, &error_notifier) != RC_TYPE_NO_RC) {
2664 rc_type = RC_TYPE_PBDMA_FAULT; 2758 rc_type = RC_TYPE_PBDMA_FAULT;
2759 }
2665 gk20a_writel(g, pbdma_intr_1_r(pbdma_id), pbdma_intr_1); 2760 gk20a_writel(g, pbdma_intr_1_r(pbdma_id), pbdma_intr_1);
2666 } 2761 }
2667 2762
2668 if (rc == RC_YES && rc_type == RC_TYPE_PBDMA_FAULT) 2763 if (rc == RC_YES && rc_type == RC_TYPE_PBDMA_FAULT) {
2669 gk20a_fifo_pbdma_fault_rc(g, f, pbdma_id, error_notifier); 2764 gk20a_fifo_pbdma_fault_rc(g, f, pbdma_id, error_notifier);
2765 }
2670 2766
2671 return handled; 2767 return handled;
2672} 2768}
@@ -2708,14 +2804,17 @@ void gk20a_fifo_isr(struct gk20a *g)
2708 gk20a_fifo_handle_runlist_event(g); 2804 gk20a_fifo_handle_runlist_event(g);
2709 clear_intr |= fifo_intr_0_runlist_event_pending_f(); 2805 clear_intr |= fifo_intr_0_runlist_event_pending_f();
2710 } 2806 }
2711 if (fifo_intr & fifo_intr_0_pbdma_intr_pending_f()) 2807 if (fifo_intr & fifo_intr_0_pbdma_intr_pending_f()) {
2712 clear_intr |= fifo_pbdma_isr(g, fifo_intr); 2808 clear_intr |= fifo_pbdma_isr(g, fifo_intr);
2809 }
2713 2810
2714 if (g->ops.fifo.handle_ctxsw_timeout) 2811 if (g->ops.fifo.handle_ctxsw_timeout) {
2715 g->ops.fifo.handle_ctxsw_timeout(g, fifo_intr); 2812 g->ops.fifo.handle_ctxsw_timeout(g, fifo_intr);
2813 }
2716 2814
2717 if (unlikely(fifo_intr & error_intr_mask)) 2815 if (unlikely(fifo_intr & error_intr_mask)) {
2718 clear_intr = fifo_error_isr(g, fifo_intr); 2816 clear_intr = fifo_error_isr(g, fifo_intr);
2817 }
2719 2818
2720 nvgpu_mutex_release(&g->fifo.intr.isr.mutex); 2819 nvgpu_mutex_release(&g->fifo.intr.isr.mutex);
2721 } 2820 }
@@ -2731,8 +2830,9 @@ u32 gk20a_fifo_nonstall_isr(struct gk20a *g)
2731 2830
2732 nvgpu_log(g, gpu_dbg_intr, "fifo nonstall isr %08x\n", fifo_intr); 2831 nvgpu_log(g, gpu_dbg_intr, "fifo nonstall isr %08x\n", fifo_intr);
2733 2832
2734 if (fifo_intr & fifo_intr_0_channel_intr_pending_f()) 2833 if (fifo_intr & fifo_intr_0_channel_intr_pending_f()) {
2735 clear_intr = fifo_intr_0_channel_intr_pending_f(); 2834 clear_intr = fifo_intr_0_channel_intr_pending_f();
2835 }
2736 2836
2737 gk20a_writel(g, fifo_intr_0_r(), clear_intr); 2837 gk20a_writel(g, fifo_intr_0_r(), clear_intr);
2738 2838
@@ -2741,14 +2841,15 @@ u32 gk20a_fifo_nonstall_isr(struct gk20a *g)
2741 2841
2742void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg) 2842void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg)
2743{ 2843{
2744 if (is_tsg) 2844 if (is_tsg) {
2745 gk20a_writel(g, fifo_preempt_r(), 2845 gk20a_writel(g, fifo_preempt_r(),
2746 fifo_preempt_id_f(id) | 2846 fifo_preempt_id_f(id) |
2747 fifo_preempt_type_tsg_f()); 2847 fifo_preempt_type_tsg_f());
2748 else 2848 } else {
2749 gk20a_writel(g, fifo_preempt_r(), 2849 gk20a_writel(g, fifo_preempt_r(),
2750 fifo_preempt_chid_f(id) | 2850 fifo_preempt_chid_f(id) |
2751 fifo_preempt_type_channel_f()); 2851 fifo_preempt_type_channel_f());
2852 }
2752} 2853}
2753 2854
2754static u32 gk20a_fifo_get_preempt_timeout(struct gk20a *g) 2855static u32 gk20a_fifo_get_preempt_timeout(struct gk20a *g)
@@ -2802,8 +2903,9 @@ void gk20a_fifo_preempt_timeout_rc(struct gk20a *g, u32 id,
2802 nvgpu_rwsem_down_read(&tsg->ch_list_lock); 2903 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
2803 nvgpu_list_for_each_entry(ch, &tsg->ch_list, 2904 nvgpu_list_for_each_entry(ch, &tsg->ch_list,
2804 channel_gk20a, ch_entry) { 2905 channel_gk20a, ch_entry) {
2805 if (!gk20a_channel_get(ch)) 2906 if (!gk20a_channel_get(ch)) {
2806 continue; 2907 continue;
2908 }
2807 g->ops.fifo.set_error_notifier(ch, 2909 g->ops.fifo.set_error_notifier(ch,
2808 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT); 2910 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT);
2809 gk20a_channel_put(ch); 2911 gk20a_channel_put(ch);
@@ -2854,8 +2956,9 @@ int gk20a_fifo_preempt_channel(struct gk20a *g, u32 chid)
2854 u32 i; 2956 u32 i;
2855 2957
2856 nvgpu_log_fn(g, "chid: %d", chid); 2958 nvgpu_log_fn(g, "chid: %d", chid);
2857 if (chid == FIFO_INVAL_CHANNEL_ID) 2959 if (chid == FIFO_INVAL_CHANNEL_ID) {
2858 return 0; 2960 return 0;
2961 }
2859 2962
2860 /* we have no idea which runlist we are using. lock all */ 2963 /* we have no idea which runlist we are using. lock all */
2861 for (i = 0; i < g->fifo.max_runlists; i++) { 2964 for (i = 0; i < g->fifo.max_runlists; i++) {
@@ -2866,8 +2969,9 @@ int gk20a_fifo_preempt_channel(struct gk20a *g, u32 chid)
2866 2969
2867 ret = __locked_fifo_preempt(g, chid, false); 2970 ret = __locked_fifo_preempt(g, chid, false);
2868 2971
2869 if (!mutex_ret) 2972 if (!mutex_ret) {
2870 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 2973 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
2974 }
2871 2975
2872 for (i = 0; i < g->fifo.max_runlists; i++) { 2976 for (i = 0; i < g->fifo.max_runlists; i++) {
2873 nvgpu_mutex_release(&f->runlist_info[i].runlist_lock); 2977 nvgpu_mutex_release(&f->runlist_info[i].runlist_lock);
@@ -2896,8 +3000,9 @@ int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid)
2896 u32 i; 3000 u32 i;
2897 3001
2898 nvgpu_log_fn(g, "tsgid: %d", tsgid); 3002 nvgpu_log_fn(g, "tsgid: %d", tsgid);
2899 if (tsgid == FIFO_INVAL_TSG_ID) 3003 if (tsgid == FIFO_INVAL_TSG_ID) {
2900 return 0; 3004 return 0;
3005 }
2901 3006
2902 /* we have no idea which runlist we are using. lock all */ 3007 /* we have no idea which runlist we are using. lock all */
2903 for (i = 0; i < g->fifo.max_runlists; i++) { 3008 for (i = 0; i < g->fifo.max_runlists; i++) {
@@ -2908,8 +3013,9 @@ int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid)
2908 3013
2909 ret = __locked_fifo_preempt(g, tsgid, true); 3014 ret = __locked_fifo_preempt(g, tsgid, true);
2910 3015
2911 if (!mutex_ret) 3016 if (!mutex_ret) {
2912 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 3017 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
3018 }
2913 3019
2914 for (i = 0; i < g->fifo.max_runlists; i++) { 3020 for (i = 0; i < g->fifo.max_runlists; i++) {
2915 nvgpu_mutex_release(&f->runlist_info[i].runlist_lock); 3021 nvgpu_mutex_release(&f->runlist_info[i].runlist_lock);
@@ -2931,10 +3037,11 @@ int gk20a_fifo_preempt(struct gk20a *g, struct channel_gk20a *ch)
2931{ 3037{
2932 int err; 3038 int err;
2933 3039
2934 if (gk20a_is_channel_marked_as_tsg(ch)) 3040 if (gk20a_is_channel_marked_as_tsg(ch)) {
2935 err = g->ops.fifo.preempt_tsg(ch->g, ch->tsgid); 3041 err = g->ops.fifo.preempt_tsg(ch->g, ch->tsgid);
2936 else 3042 } else {
2937 err = g->ops.fifo.preempt_channel(ch->g, ch->chid); 3043 err = g->ops.fifo.preempt_channel(ch->g, ch->chid);
3044 }
2938 3045
2939 return err; 3046 return err;
2940} 3047}
@@ -2946,10 +3053,11 @@ static void gk20a_fifo_sched_disable_rw(struct gk20a *g, u32 runlists_mask,
2946 3053
2947 reg_val = gk20a_readl(g, fifo_sched_disable_r()); 3054 reg_val = gk20a_readl(g, fifo_sched_disable_r());
2948 3055
2949 if (runlist_state == RUNLIST_DISABLED) 3056 if (runlist_state == RUNLIST_DISABLED) {
2950 reg_val |= runlists_mask; 3057 reg_val |= runlists_mask;
2951 else 3058 } else {
2952 reg_val &= (~runlists_mask); 3059 reg_val &= (~runlists_mask);
3060 }
2953 3061
2954 gk20a_writel(g, fifo_sched_disable_r(), reg_val); 3062 gk20a_writel(g, fifo_sched_disable_r(), reg_val);
2955 3063
@@ -2968,8 +3076,9 @@ void gk20a_fifo_set_runlist_state(struct gk20a *g, u32 runlists_mask,
2968 3076
2969 gk20a_fifo_sched_disable_rw(g, runlists_mask, runlist_state); 3077 gk20a_fifo_sched_disable_rw(g, runlists_mask, runlist_state);
2970 3078
2971 if (!mutex_ret) 3079 if (!mutex_ret) {
2972 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 3080 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
3081 }
2973} 3082}
2974 3083
2975void gk20a_fifo_enable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg) 3084void gk20a_fifo_enable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg)
@@ -3030,8 +3139,9 @@ int gk20a_fifo_disable_engine_activity(struct gk20a *g,
3030 gr_stat = 3139 gr_stat =
3031 gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id)); 3140 gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
3032 if (fifo_engine_status_engine_v(gr_stat) == 3141 if (fifo_engine_status_engine_v(gr_stat) ==
3033 fifo_engine_status_engine_busy_v() && !wait_for_idle) 3142 fifo_engine_status_engine_busy_v() && !wait_for_idle) {
3034 return -EBUSY; 3143 return -EBUSY;
3144 }
3035 3145
3036 mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 3146 mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
3037 3147
@@ -3042,43 +3152,49 @@ int gk20a_fifo_disable_engine_activity(struct gk20a *g,
3042 pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(eng_info->pbdma_id)); 3152 pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(eng_info->pbdma_id));
3043 chan_stat = fifo_pbdma_status_chan_status_v(pbdma_stat); 3153 chan_stat = fifo_pbdma_status_chan_status_v(pbdma_stat);
3044 if (chan_stat == fifo_pbdma_status_chan_status_valid_v() || 3154 if (chan_stat == fifo_pbdma_status_chan_status_valid_v() ||
3045 chan_stat == fifo_pbdma_status_chan_status_chsw_save_v()) 3155 chan_stat == fifo_pbdma_status_chan_status_chsw_save_v()) {
3046 pbdma_chid = fifo_pbdma_status_id_v(pbdma_stat); 3156 pbdma_chid = fifo_pbdma_status_id_v(pbdma_stat);
3047 else if (chan_stat == fifo_pbdma_status_chan_status_chsw_load_v() || 3157 } else if (chan_stat == fifo_pbdma_status_chan_status_chsw_load_v() ||
3048 chan_stat == fifo_pbdma_status_chan_status_chsw_switch_v()) 3158 chan_stat == fifo_pbdma_status_chan_status_chsw_switch_v()) {
3049 pbdma_chid = fifo_pbdma_status_next_id_v(pbdma_stat); 3159 pbdma_chid = fifo_pbdma_status_next_id_v(pbdma_stat);
3160 }
3050 3161
3051 if (pbdma_chid != FIFO_INVAL_CHANNEL_ID) { 3162 if (pbdma_chid != FIFO_INVAL_CHANNEL_ID) {
3052 err = g->ops.fifo.preempt_channel(g, pbdma_chid); 3163 err = g->ops.fifo.preempt_channel(g, pbdma_chid);
3053 if (err) 3164 if (err) {
3054 goto clean_up; 3165 goto clean_up;
3166 }
3055 } 3167 }
3056 3168
3057 /* chid from engine status */ 3169 /* chid from engine status */
3058 eng_stat = gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id)); 3170 eng_stat = gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
3059 ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); 3171 ctx_stat = fifo_engine_status_ctx_status_v(eng_stat);
3060 if (ctx_stat == fifo_engine_status_ctx_status_valid_v() || 3172 if (ctx_stat == fifo_engine_status_ctx_status_valid_v() ||
3061 ctx_stat == fifo_engine_status_ctx_status_ctxsw_save_v()) 3173 ctx_stat == fifo_engine_status_ctx_status_ctxsw_save_v()) {
3062 engine_chid = fifo_engine_status_id_v(eng_stat); 3174 engine_chid = fifo_engine_status_id_v(eng_stat);
3063 else if (ctx_stat == fifo_engine_status_ctx_status_ctxsw_load_v() || 3175 } else if (ctx_stat == fifo_engine_status_ctx_status_ctxsw_load_v() ||
3064 ctx_stat == fifo_engine_status_ctx_status_ctxsw_switch_v()) 3176 ctx_stat == fifo_engine_status_ctx_status_ctxsw_switch_v()) {
3065 engine_chid = fifo_engine_status_next_id_v(eng_stat); 3177 engine_chid = fifo_engine_status_next_id_v(eng_stat);
3178 }
3066 3179
3067 if (engine_chid != FIFO_INVAL_ENGINE_ID && engine_chid != pbdma_chid) { 3180 if (engine_chid != FIFO_INVAL_ENGINE_ID && engine_chid != pbdma_chid) {
3068 err = g->ops.fifo.preempt_channel(g, engine_chid); 3181 err = g->ops.fifo.preempt_channel(g, engine_chid);
3069 if (err) 3182 if (err) {
3070 goto clean_up; 3183 goto clean_up;
3184 }
3071 } 3185 }
3072 3186
3073clean_up: 3187clean_up:
3074 if (!mutex_ret) 3188 if (!mutex_ret) {
3075 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 3189 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
3190 }
3076 3191
3077 if (err) { 3192 if (err) {
3078 nvgpu_log_fn(g, "failed"); 3193 nvgpu_log_fn(g, "failed");
3079 if (gk20a_fifo_enable_engine_activity(g, eng_info)) 3194 if (gk20a_fifo_enable_engine_activity(g, eng_info)) {
3080 nvgpu_err(g, 3195 nvgpu_err(g,
3081 "failed to enable gr engine activity"); 3196 "failed to enable gr engine activity");
3197 }
3082 } else { 3198 } else {
3083 nvgpu_log_fn(g, "done"); 3199 nvgpu_log_fn(g, "done");
3084 } 3200 }
@@ -3110,10 +3226,11 @@ int gk20a_fifo_disable_all_engine_activity(struct gk20a *g,
3110 active_engine_id = g->fifo.active_engines_list[i]; 3226 active_engine_id = g->fifo.active_engines_list[i];
3111 err = gk20a_fifo_enable_engine_activity(g, 3227 err = gk20a_fifo_enable_engine_activity(g,
3112 &g->fifo.engine_info[active_engine_id]); 3228 &g->fifo.engine_info[active_engine_id]);
3113 if (err) 3229 if (err) {
3114 nvgpu_err(g, 3230 nvgpu_err(g,
3115 "failed to re-enable engine %d activity", 3231 "failed to re-enable engine %d activity",
3116 active_engine_id); 3232 active_engine_id);
3233 }
3117 } 3234 }
3118 } 3235 }
3119 3236
@@ -3133,13 +3250,15 @@ static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id)
3133 fifo_engine_status_engine_busy_v(); 3250 fifo_engine_status_engine_busy_v();
3134 3251
3135 if (engine_busy && 3252 if (engine_busy &&
3136 (f->engine_info[active_engine_id].runlist_id == runlist_id)) 3253 (f->engine_info[active_engine_id].runlist_id == runlist_id)) {
3137 engines |= BIT(active_engine_id); 3254 engines |= BIT(active_engine_id);
3255 }
3138 } 3256 }
3139 3257
3140 if (engines) 3258 if (engines) {
3141 gk20a_fifo_recover(g, engines, ~(u32)0, false, false, true, 3259 gk20a_fifo_recover(g, engines, ~(u32)0, false, false, true,
3142 RC_TYPE_RUNLIST_UPDATE_TIMEOUT); 3260 RC_TYPE_RUNLIST_UPDATE_TIMEOUT);
3261 }
3143} 3262}
3144 3263
3145int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id) 3264int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id)
@@ -3177,16 +3296,17 @@ void gk20a_get_tsg_runlist_entry(struct tsg_gk20a *tsg, u32 *runlist)
3177 ram_rl_entry_type_tsg_f() | 3296 ram_rl_entry_type_tsg_f() |
3178 ram_rl_entry_tsg_length_f(tsg->num_active_channels); 3297 ram_rl_entry_tsg_length_f(tsg->num_active_channels);
3179 3298
3180 if (tsg->timeslice_timeout) 3299 if (tsg->timeslice_timeout) {
3181 runlist_entry_0 |= 3300 runlist_entry_0 |=
3182 ram_rl_entry_timeslice_scale_f(tsg->timeslice_scale) | 3301 ram_rl_entry_timeslice_scale_f(tsg->timeslice_scale) |
3183 ram_rl_entry_timeslice_timeout_f(tsg->timeslice_timeout); 3302 ram_rl_entry_timeslice_timeout_f(tsg->timeslice_timeout);
3184 else 3303 } else {
3185 runlist_entry_0 |= 3304 runlist_entry_0 |=
3186 ram_rl_entry_timeslice_scale_f( 3305 ram_rl_entry_timeslice_scale_f(
3187 NVGPU_FIFO_DEFAULT_TIMESLICE_SCALE) | 3306 NVGPU_FIFO_DEFAULT_TIMESLICE_SCALE) |
3188 ram_rl_entry_timeslice_timeout_f( 3307 ram_rl_entry_timeslice_timeout_f(
3189 NVGPU_FIFO_DEFAULT_TIMESLICE_TIMEOUT); 3308 NVGPU_FIFO_DEFAULT_TIMESLICE_TIMEOUT);
3309 }
3190 3310
3191 runlist[0] = runlist_entry_0; 3311 runlist[0] = runlist_entry_0;
3192 runlist[1] = 0; 3312 runlist[1] = 0;
@@ -3230,8 +3350,9 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3230 for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) { 3350 for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) {
3231 struct tsg_gk20a *tsg = &f->tsg[tsgid]; 3351 struct tsg_gk20a *tsg = &f->tsg[tsgid];
3232 3352
3233 if (tsg->interleave_level != cur_level) 3353 if (tsg->interleave_level != cur_level) {
3234 continue; 3354 continue;
3355 }
3235 3356
3236 if (!last_level && !skip_next) { 3357 if (!last_level && !skip_next) {
3237 runlist_entry = gk20a_runlist_construct_locked(f, 3358 runlist_entry = gk20a_runlist_construct_locked(f,
@@ -3241,8 +3362,9 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3241 interleave_enabled, 3362 interleave_enabled,
3242 false, 3363 false,
3243 entries_left); 3364 entries_left);
3244 if (!interleave_enabled) 3365 if (!interleave_enabled) {
3245 skip_next = true; 3366 skip_next = true;
3367 }
3246 } 3368 }
3247 3369
3248 if (*entries_left == 0U) { 3370 if (*entries_left == 0U) {
@@ -3263,8 +3385,9 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3263 nvgpu_list_for_each_entry(ch, &tsg->ch_list, 3385 nvgpu_list_for_each_entry(ch, &tsg->ch_list,
3264 channel_gk20a, ch_entry) { 3386 channel_gk20a, ch_entry) {
3265 if (!test_bit(ch->chid, 3387 if (!test_bit(ch->chid,
3266 runlist->active_channels)) 3388 runlist->active_channels)) {
3267 continue; 3389 continue;
3390 }
3268 3391
3269 if (*entries_left == 0U) { 3392 if (*entries_left == 0U) {
3270 nvgpu_rwsem_up_read(&tsg->ch_list_lock); 3393 nvgpu_rwsem_up_read(&tsg->ch_list_lock);
@@ -3285,7 +3408,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3285 } 3408 }
3286 3409
3287 /* append entries from higher level if this level is empty */ 3410 /* append entries from higher level if this level is empty */
3288 if (!count && !last_level) 3411 if (!count && !last_level) {
3289 runlist_entry = gk20a_runlist_construct_locked(f, 3412 runlist_entry = gk20a_runlist_construct_locked(f,
3290 runlist, 3413 runlist,
3291 cur_level + 1, 3414 cur_level + 1,
@@ -3293,6 +3416,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3293 interleave_enabled, 3416 interleave_enabled,
3294 true, 3417 true,
3295 entries_left); 3418 entries_left);
3419 }
3296 3420
3297 /* 3421 /*
3298 * if previous and this level have entries, append 3422 * if previous and this level have entries, append
@@ -3300,7 +3424,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3300 * 3424 *
3301 * ex. dropping from MEDIUM to LOW, need to insert HIGH 3425 * ex. dropping from MEDIUM to LOW, need to insert HIGH
3302 */ 3426 */
3303 if (interleave_enabled && count && !prev_empty && !last_level) 3427 if (interleave_enabled && count && !prev_empty && !last_level) {
3304 runlist_entry = gk20a_runlist_construct_locked(f, 3428 runlist_entry = gk20a_runlist_construct_locked(f,
3305 runlist, 3429 runlist,
3306 cur_level + 1, 3430 cur_level + 1,
@@ -3308,6 +3432,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f,
3308 interleave_enabled, 3432 interleave_enabled,
3309 false, 3433 false,
3310 entries_left); 3434 entries_left);
3435 }
3311 return runlist_entry; 3436 return runlist_entry;
3312} 3437}
3313 3438
@@ -3328,8 +3453,9 @@ int gk20a_fifo_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice)
3328 struct gk20a *g = tsg->g; 3453 struct gk20a *g = tsg->g;
3329 3454
3330 if (timeslice < g->min_timeslice_us || 3455 if (timeslice < g->min_timeslice_us ||
3331 timeslice > g->max_timeslice_us) 3456 timeslice > g->max_timeslice_us) {
3332 return -EINVAL; 3457 return -EINVAL;
3458 }
3333 3459
3334 gk20a_channel_get_timescale_from_timeslice(g, timeslice, 3460 gk20a_channel_get_timescale_from_timeslice(g, timeslice,
3335 &tsg->timeslice_timeout, &tsg->timeslice_scale); 3461 &tsg->timeslice_timeout, &tsg->timeslice_scale);
@@ -3382,23 +3508,28 @@ int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
3382 Otherwise, keep active list untouched for suspend/resume. */ 3508 Otherwise, keep active list untouched for suspend/resume. */
3383 if (chid != FIFO_INVAL_CHANNEL_ID) { 3509 if (chid != FIFO_INVAL_CHANNEL_ID) {
3384 ch = &f->channel[chid]; 3510 ch = &f->channel[chid];
3385 if (gk20a_is_channel_marked_as_tsg(ch)) 3511 if (gk20a_is_channel_marked_as_tsg(ch)) {
3386 tsg = &f->tsg[ch->tsgid]; 3512 tsg = &f->tsg[ch->tsgid];
3513 }
3387 3514
3388 if (add) { 3515 if (add) {
3389 if (test_and_set_bit(chid, 3516 if (test_and_set_bit(chid,
3390 runlist->active_channels) == 1) 3517 runlist->active_channels) == 1) {
3391 return 0; 3518 return 0;
3392 if (tsg && ++tsg->num_active_channels) 3519 }
3520 if (tsg && ++tsg->num_active_channels) {
3393 set_bit(f->channel[chid].tsgid, 3521 set_bit(f->channel[chid].tsgid,
3394 runlist->active_tsgs); 3522 runlist->active_tsgs);
3523 }
3395 } else { 3524 } else {
3396 if (test_and_clear_bit(chid, 3525 if (test_and_clear_bit(chid,
3397 runlist->active_channels) == 0) 3526 runlist->active_channels) == 0) {
3398 return 0; 3527 return 0;
3399 if (tsg && --tsg->num_active_channels == 0) 3528 }
3529 if (tsg && --tsg->num_active_channels == 0) {
3400 clear_bit(f->channel[chid].tsgid, 3530 clear_bit(f->channel[chid].tsgid,
3401 runlist->active_tsgs); 3531 runlist->active_tsgs);
3532 }
3402 } 3533 }
3403 } 3534 }
3404 3535
@@ -3439,8 +3570,10 @@ int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
3439 runlist->count = (runlist_end - runlist_entry_base) / 3570 runlist->count = (runlist_end - runlist_entry_base) /
3440 runlist_entry_words; 3571 runlist_entry_words;
3441 WARN_ON(runlist->count > f->num_runlist_entries); 3572 WARN_ON(runlist->count > f->num_runlist_entries);
3442 } else /* suspend to remove all channels */ 3573 } else {
3574 /* suspend to remove all channels */
3443 runlist->count = 0; 3575 runlist->count = 0;
3576 }
3444 3577
3445 g->ops.fifo.runlist_hw_submit(g, runlist_id, runlist->count, new_buf); 3578 g->ops.fifo.runlist_hw_submit(g, runlist_id, runlist->count, new_buf);
3446 3579
@@ -3452,8 +3585,9 @@ int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
3452 /* trigger runlist update timeout recovery */ 3585 /* trigger runlist update timeout recovery */
3453 return ret; 3586 return ret;
3454 3587
3455 } else if (ret == -EINTR) 3588 } else if (ret == -EINTR) {
3456 nvgpu_err(g, "runlist update interrupted"); 3589 nvgpu_err(g, "runlist update interrupted");
3590 }
3457 } 3591 }
3458 3592
3459 runlist->cur_buffer = new_buf; 3593 runlist->cur_buffer = new_buf;
@@ -3470,8 +3604,9 @@ int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid,
3470 u32 errcode; 3604 u32 errcode;
3471 unsigned long ulong_runlist_ids = (unsigned long)runlist_ids; 3605 unsigned long ulong_runlist_ids = (unsigned long)runlist_ids;
3472 3606
3473 if (!g) 3607 if (!g) {
3474 goto end; 3608 goto end;
3609 }
3475 3610
3476 ret = 0; 3611 ret = 0;
3477 for_each_set_bit(runlist_id, &ulong_runlist_ids, 32) { 3612 for_each_set_bit(runlist_id, &ulong_runlist_ids, 32) {
@@ -3501,14 +3636,17 @@ static int __locked_fifo_reschedule_preempt_next(struct channel_gk20a *ch,
3501 u32 preempt_type = 0; 3636 u32 preempt_type = 0;
3502 3637
3503 if (1 != gk20a_fifo_get_engine_ids( 3638 if (1 != gk20a_fifo_get_engine_ids(
3504 g, &gr_eng_id, 1, ENGINE_GR_GK20A)) 3639 g, &gr_eng_id, 1, ENGINE_GR_GK20A)) {
3505 return ret; 3640 return ret;
3506 if (!(runlist->eng_bitmask & (1 << gr_eng_id))) 3641 }
3642 if (!(runlist->eng_bitmask & (1 << gr_eng_id))) {
3507 return ret; 3643 return ret;
3644 }
3508 3645
3509 if (wait_preempt && gk20a_readl(g, fifo_preempt_r()) & 3646 if (wait_preempt && gk20a_readl(g, fifo_preempt_r()) &
3510 fifo_preempt_pending_true_f()) 3647 fifo_preempt_pending_true_f()) {
3511 return ret; 3648 return ret;
3649 }
3512 3650
3513 fecsstat0 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0)); 3651 fecsstat0 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0));
3514 engstat = gk20a_readl(g, fifo_engine_status_r(gr_eng_id)); 3652 engstat = gk20a_readl(g, fifo_engine_status_r(gr_eng_id));
@@ -3517,10 +3655,12 @@ static int __locked_fifo_reschedule_preempt_next(struct channel_gk20a *ch,
3517 /* host switching to next context, preempt that if needed */ 3655 /* host switching to next context, preempt that if needed */
3518 preempt_id = fifo_engine_status_next_id_v(engstat); 3656 preempt_id = fifo_engine_status_next_id_v(engstat);
3519 preempt_type = fifo_engine_status_next_id_type_v(engstat); 3657 preempt_type = fifo_engine_status_next_id_type_v(engstat);
3520 } else 3658 } else {
3521 return ret; 3659 return ret;
3522 if (preempt_id == ch->tsgid && preempt_type) 3660 }
3661 if (preempt_id == ch->tsgid && preempt_type) {
3523 return ret; 3662 return ret;
3663 }
3524 fecsstat1 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0)); 3664 fecsstat1 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0));
3525 if (fecsstat0 != FECS_MAILBOX_0_ACK_RESTORE || 3665 if (fecsstat0 != FECS_MAILBOX_0_ACK_RESTORE ||
3526 fecsstat1 != FECS_MAILBOX_0_ACK_RESTORE) { 3666 fecsstat1 != FECS_MAILBOX_0_ACK_RESTORE) {
@@ -3559,8 +3699,9 @@ int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next,
3559 int ret = 0; 3699 int ret = 0;
3560 3700
3561 runlist = &g->fifo.runlist_info[ch->runlist_id]; 3701 runlist = &g->fifo.runlist_info[ch->runlist_id];
3562 if (!nvgpu_mutex_tryacquire(&runlist->runlist_lock)) 3702 if (!nvgpu_mutex_tryacquire(&runlist->runlist_lock)) {
3563 return -EBUSY; 3703 return -EBUSY;
3704 }
3564 3705
3565 mutex_ret = nvgpu_pmu_mutex_acquire( 3706 mutex_ret = nvgpu_pmu_mutex_acquire(
3566 &g->pmu, PMU_MUTEX_ID_FIFO, &token); 3707 &g->pmu, PMU_MUTEX_ID_FIFO, &token);
@@ -3568,14 +3709,16 @@ int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next,
3568 g->ops.fifo.runlist_hw_submit( 3709 g->ops.fifo.runlist_hw_submit(
3569 g, ch->runlist_id, runlist->count, runlist->cur_buffer); 3710 g, ch->runlist_id, runlist->count, runlist->cur_buffer);
3570 3711
3571 if (preempt_next) 3712 if (preempt_next) {
3572 __locked_fifo_reschedule_preempt_next(ch, wait_preempt); 3713 __locked_fifo_reschedule_preempt_next(ch, wait_preempt);
3714 }
3573 3715
3574 gk20a_fifo_runlist_wait_pending(g, ch->runlist_id); 3716 gk20a_fifo_runlist_wait_pending(g, ch->runlist_id);
3575 3717
3576 if (!mutex_ret) 3718 if (!mutex_ret) {
3577 nvgpu_pmu_mutex_release( 3719 nvgpu_pmu_mutex_release(
3578 &g->pmu, PMU_MUTEX_ID_FIFO, &token); 3720 &g->pmu, PMU_MUTEX_ID_FIFO, &token);
3721 }
3579 nvgpu_mutex_release(&runlist->runlist_lock); 3722 nvgpu_mutex_release(&runlist->runlist_lock);
3580 3723
3581 return ret; 3724 return ret;
@@ -3605,13 +3748,15 @@ int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 chid,
3605 ret = gk20a_fifo_update_runlist_locked(g, runlist_id, chid, add, 3748 ret = gk20a_fifo_update_runlist_locked(g, runlist_id, chid, add,
3606 wait_for_finish); 3749 wait_for_finish);
3607 3750
3608 if (!mutex_ret) 3751 if (!mutex_ret) {
3609 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); 3752 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
3753 }
3610 3754
3611 nvgpu_mutex_release(&runlist->runlist_lock); 3755 nvgpu_mutex_release(&runlist->runlist_lock);
3612 3756
3613 if (ret == -ETIMEDOUT) 3757 if (ret == -ETIMEDOUT) {
3614 gk20a_fifo_runlist_reset_engines(g, runlist_id); 3758 gk20a_fifo_runlist_reset_engines(g, runlist_id);
3759 }
3615 3760
3616 return ret; 3761 return ret;
3617} 3762}
@@ -3621,9 +3766,10 @@ int gk20a_fifo_suspend(struct gk20a *g)
3621 nvgpu_log_fn(g, " "); 3766 nvgpu_log_fn(g, " ");
3622 3767
3623 /* stop bar1 snooping */ 3768 /* stop bar1 snooping */
3624 if (g->ops.mm.is_bar1_supported(g)) 3769 if (g->ops.mm.is_bar1_supported(g)) {
3625 gk20a_writel(g, fifo_bar1_base_r(), 3770 gk20a_writel(g, fifo_bar1_base_r(),
3626 fifo_bar1_base_valid_false_f()); 3771 fifo_bar1_base_valid_false_f());
3772 }
3627 3773
3628 /* disable fifo intr */ 3774 /* disable fifo intr */
3629 gk20a_writel(g, fifo_intr_en_0_r(), 0); 3775 gk20a_writel(g, fifo_intr_en_0_r(), 0);
@@ -3636,10 +3782,11 @@ int gk20a_fifo_suspend(struct gk20a *g)
3636bool gk20a_fifo_mmu_fault_pending(struct gk20a *g) 3782bool gk20a_fifo_mmu_fault_pending(struct gk20a *g)
3637{ 3783{
3638 if (gk20a_readl(g, fifo_intr_0_r()) & 3784 if (gk20a_readl(g, fifo_intr_0_r()) &
3639 fifo_intr_0_mmu_fault_pending_f()) 3785 fifo_intr_0_mmu_fault_pending_f()) {
3640 return true; 3786 return true;
3641 else 3787 } else {
3642 return false; 3788 return false;
3789 }
3643} 3790}
3644 3791
3645bool gk20a_fifo_is_engine_busy(struct gk20a *g) 3792bool gk20a_fifo_is_engine_busy(struct gk20a *g)
@@ -3651,8 +3798,9 @@ bool gk20a_fifo_is_engine_busy(struct gk20a *g)
3651 for (i = 0; i < host_num_engines; i++) { 3798 for (i = 0; i < host_num_engines; i++) {
3652 u32 status = gk20a_readl(g, fifo_engine_status_r(i)); 3799 u32 status = gk20a_readl(g, fifo_engine_status_r(i));
3653 if (fifo_engine_status_engine_v(status) == 3800 if (fifo_engine_status_engine_v(status) ==
3654 fifo_engine_status_engine_busy_v()) 3801 fifo_engine_status_engine_busy_v()) {
3655 return true; 3802 return true;
3803 }
3656 } 3804 }
3657 return false; 3805 return false;
3658} 3806}
@@ -3704,10 +3852,11 @@ u32 gk20a_fifo_get_pbdma_signature(struct gk20a *g)
3704struct channel_gk20a *gk20a_fifo_channel_from_chid(struct gk20a *g, 3852struct channel_gk20a *gk20a_fifo_channel_from_chid(struct gk20a *g,
3705 u32 chid) 3853 u32 chid)
3706{ 3854{
3707 if (chid != FIFO_INVAL_CHANNEL_ID) 3855 if (chid != FIFO_INVAL_CHANNEL_ID) {
3708 return g->fifo.channel + chid; 3856 return g->fifo.channel + chid;
3709 else 3857 } else {
3710 return NULL; 3858 return NULL;
3859 }
3711} 3860}
3712 3861
3713static const char * const ccsr_chan_status_str[] = { 3862static const char * const ccsr_chan_status_str[] = {
@@ -3745,18 +3894,20 @@ static const char * const not_found_str[] = {
3745 3894
3746const char *gk20a_decode_ccsr_chan_status(u32 index) 3895const char *gk20a_decode_ccsr_chan_status(u32 index)
3747{ 3896{
3748 if (index >= ARRAY_SIZE(ccsr_chan_status_str)) 3897 if (index >= ARRAY_SIZE(ccsr_chan_status_str)) {
3749 return not_found_str[0]; 3898 return not_found_str[0];
3750 else 3899 } else {
3751 return ccsr_chan_status_str[index]; 3900 return ccsr_chan_status_str[index];
3901 }
3752} 3902}
3753 3903
3754const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index) 3904const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index)
3755{ 3905{
3756 if (index >= ARRAY_SIZE(pbdma_chan_eng_ctx_status_str)) 3906 if (index >= ARRAY_SIZE(pbdma_chan_eng_ctx_status_str)) {
3757 return not_found_str[0]; 3907 return not_found_str[0];
3758 else 3908 } else {
3759 return pbdma_chan_eng_ctx_status_str[index]; 3909 return pbdma_chan_eng_ctx_status_str[index];
3910 }
3760} 3911}
3761 3912
3762bool gk20a_fifo_channel_status_is_next(struct gk20a *g, u32 chid) 3913bool gk20a_fifo_channel_status_is_next(struct gk20a *g, u32 chid)
@@ -3792,11 +3943,13 @@ void gk20a_dump_channel_status_ramfc(struct gk20a *g,
3792 struct channel_gk20a *c = g->fifo.channel + chid; 3943 struct channel_gk20a *c = g->fifo.channel + chid;
3793 struct nvgpu_semaphore_int *hw_sema = NULL; 3944 struct nvgpu_semaphore_int *hw_sema = NULL;
3794 3945
3795 if (c->hw_sema) 3946 if (c->hw_sema) {
3796 hw_sema = c->hw_sema; 3947 hw_sema = c->hw_sema;
3948 }
3797 3949
3798 if (!ch_state) 3950 if (!ch_state) {
3799 return; 3951 return;
3952 }
3800 3953
3801 inst_mem = &ch_state->inst_block[0]; 3954 inst_mem = &ch_state->inst_block[0];
3802 3955
@@ -3831,12 +3984,13 @@ void gk20a_dump_channel_status_ramfc(struct gk20a *g,
3831 inst_mem[ram_fc_semaphoreb_w()], 3984 inst_mem[ram_fc_semaphoreb_w()],
3832 inst_mem[ram_fc_semaphorec_w()], 3985 inst_mem[ram_fc_semaphorec_w()],
3833 inst_mem[ram_fc_semaphored_w()]); 3986 inst_mem[ram_fc_semaphored_w()]);
3834 if (hw_sema) 3987 if (hw_sema) {
3835 gk20a_debug_output(o, "SEMA STATE: value: 0x%08x " 3988 gk20a_debug_output(o, "SEMA STATE: value: 0x%08x "
3836 "next_val: 0x%08x addr: 0x%010llx\n", 3989 "next_val: 0x%08x addr: 0x%010llx\n",
3837 __nvgpu_semaphore_read(hw_sema), 3990 __nvgpu_semaphore_read(hw_sema),
3838 nvgpu_atomic_read(&hw_sema->next_value), 3991 nvgpu_atomic_read(&hw_sema->next_value),
3839 nvgpu_hw_sema_addr(hw_sema)); 3992 nvgpu_hw_sema_addr(hw_sema));
3993 }
3840 3994
3841#ifdef CONFIG_TEGRA_GK20A_NVHOST 3995#ifdef CONFIG_TEGRA_GK20A_NVHOST
3842 if ((pbdma_syncpointb_op_v(syncpointb) == pbdma_syncpointb_op_wait_v()) 3996 if ((pbdma_syncpointb_op_v(syncpointb) == pbdma_syncpointb_op_wait_v())
@@ -3874,15 +4028,17 @@ void gk20a_debug_dump_all_channel_status_ramfc(struct gk20a *g,
3874 ram_in_alloc_size_v()); 4028 ram_in_alloc_size_v());
3875 /* ref taken stays to below loop with 4029 /* ref taken stays to below loop with
3876 * successful allocs */ 4030 * successful allocs */
3877 if (!ch_state[chid]) 4031 if (!ch_state[chid]) {
3878 gk20a_channel_put(ch); 4032 gk20a_channel_put(ch);
4033 }
3879 } 4034 }
3880 } 4035 }
3881 4036
3882 for (chid = 0; chid < f->num_channels; chid++) { 4037 for (chid = 0; chid < f->num_channels; chid++) {
3883 struct channel_gk20a *ch = &f->channel[chid]; 4038 struct channel_gk20a *ch = &f->channel[chid];
3884 if (!ch_state[chid]) 4039 if (!ch_state[chid]) {
3885 continue; 4040 continue;
4041 }
3886 4042
3887 ch_state[chid]->pid = ch->pid; 4043 ch_state[chid]->pid = ch->pid;
3888 ch_state[chid]->refs = nvgpu_atomic_read(&ch->ref_count); 4044 ch_state[chid]->refs = nvgpu_atomic_read(&ch->ref_count);
@@ -3964,10 +4120,12 @@ void gk20a_dump_eng_status(struct gk20a *g,
3964 "tsg" : "channel", 4120 "tsg" : "channel",
3965 gk20a_decode_pbdma_chan_eng_ctx_status(ctx_status)); 4121 gk20a_decode_pbdma_chan_eng_ctx_status(ctx_status));
3966 4122
3967 if (fifo_engine_status_faulted_v(status)) 4123 if (fifo_engine_status_faulted_v(status)) {
3968 gk20a_debug_output(o, "faulted "); 4124 gk20a_debug_output(o, "faulted ");
3969 if (fifo_engine_status_engine_v(status)) 4125 }
4126 if (fifo_engine_status_engine_v(status)) {
3970 gk20a_debug_output(o, "busy "); 4127 gk20a_debug_output(o, "busy ");
4128 }
3971 gk20a_debug_output(o, "\n"); 4129 gk20a_debug_output(o, "\n");
3972 } 4130 }
3973 gk20a_debug_output(o, "\n"); 4131 gk20a_debug_output(o, "\n");
@@ -4088,8 +4246,9 @@ int gk20a_fifo_setup_ramfc(struct channel_gk20a *c,
4088 4246
4089 nvgpu_mem_wr32(g, mem, ram_fc_chid_w(), ram_fc_chid_id_f(c->chid)); 4247 nvgpu_mem_wr32(g, mem, ram_fc_chid_w(), ram_fc_chid_id_f(c->chid));
4090 4248
4091 if (c->is_privileged_channel) 4249 if (c->is_privileged_channel) {
4092 gk20a_fifo_setup_ramfc_for_privileged_channel(c); 4250 gk20a_fifo_setup_ramfc_for_privileged_channel(c);
4251 }
4093 4252
4094 return gk20a_fifo_commit_userd(c); 4253 return gk20a_fifo_commit_userd(c);
4095} 4254}
@@ -4143,8 +4302,9 @@ int gk20a_fifo_alloc_inst(struct gk20a *g, struct channel_gk20a *ch)
4143 nvgpu_log_fn(g, " "); 4302 nvgpu_log_fn(g, " ");
4144 4303
4145 err = g->ops.mm.alloc_inst_block(g, &ch->inst_block); 4304 err = g->ops.mm.alloc_inst_block(g, &ch->inst_block);
4146 if (err) 4305 if (err) {
4147 return err; 4306 return err;
4307 }
4148 4308
4149 nvgpu_log_info(g, "channel %d inst block physical addr: 0x%16llx", 4309 nvgpu_log_info(g, "channel %d inst block physical addr: 0x%16llx",
4150 ch->chid, nvgpu_inst_block_addr(g, &ch->inst_block)); 4310 ch->chid, nvgpu_inst_block_addr(g, &ch->inst_block));
@@ -4189,16 +4349,18 @@ u32 gk20a_fifo_pbdma_acquire_val(u64 timeout)
4189 val = pbdma_acquire_retry_man_2_f() | 4349 val = pbdma_acquire_retry_man_2_f() |
4190 pbdma_acquire_retry_exp_2_f(); 4350 pbdma_acquire_retry_exp_2_f();
4191 4351
4192 if (!timeout) 4352 if (!timeout) {
4193 return val; 4353 return val;
4354 }
4194 4355
4195 timeout *= 80UL; 4356 timeout *= 80UL;
4196 do_div(timeout, 100); /* set acquire timeout to 80% of channel wdt */ 4357 do_div(timeout, 100); /* set acquire timeout to 80% of channel wdt */
4197 timeout *= 1000000UL; /* ms -> ns */ 4358 timeout *= 1000000UL; /* ms -> ns */
4198 do_div(timeout, 1024); /* in unit of 1024ns */ 4359 do_div(timeout, 1024); /* in unit of 1024ns */
4199 val_len = fls(timeout >> 32) + 32; 4360 val_len = fls(timeout >> 32) + 32;
4200 if (val_len == 32) 4361 if (val_len == 32) {
4201 val_len = fls(timeout); 4362 val_len = fls(timeout);
4363 }
4202 if (val_len > 16U + pbdma_acquire_timeout_exp_max_v()) { /* man: 16bits */ 4364 if (val_len > 16U + pbdma_acquire_timeout_exp_max_v()) { /* man: 16bits */
4203 exp = pbdma_acquire_timeout_exp_max_v(); 4365 exp = pbdma_acquire_timeout_exp_max_v();
4204 man = pbdma_acquire_timeout_man_max_v(); 4366 man = pbdma_acquire_timeout_man_max_v();
diff --git a/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c b/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c
index e6e16511..2f715ae1 100644
--- a/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c
@@ -35,10 +35,10 @@ static int gk20a_flcn_reset(struct nvgpu_falcon *flcn)
35 u32 unit_status = 0; 35 u32 unit_status = 0;
36 int status = 0; 36 int status = 0;
37 37
38 if (flcn->flcn_engine_dep_ops.reset_eng) 38 if (flcn->flcn_engine_dep_ops.reset_eng) {
39 /* falcon & engine reset */ 39 /* falcon & engine reset */
40 status = flcn->flcn_engine_dep_ops.reset_eng(g); 40 status = flcn->flcn_engine_dep_ops.reset_eng(g);
41 else { 41 } else {
42 /* do falcon CPU hard reset */ 42 /* do falcon CPU hard reset */
43 unit_status = gk20a_readl(g, base_addr + 43 unit_status = gk20a_readl(g, base_addr +
44 falcon_falcon_cpuctl_r()); 44 falcon_falcon_cpuctl_r());
@@ -62,9 +62,10 @@ static bool gk20a_flcn_clear_halt_interrupt_status(struct nvgpu_falcon *flcn)
62 data = gk20a_readl(g, (base_addr + falcon_falcon_irqstat_r())); 62 data = gk20a_readl(g, (base_addr + falcon_falcon_irqstat_r()));
63 63
64 if ((data & falcon_falcon_irqstat_halt_true_f()) != 64 if ((data & falcon_falcon_irqstat_halt_true_f()) !=
65 falcon_falcon_irqstat_halt_true_f()) 65 falcon_falcon_irqstat_halt_true_f()) {
66 /*halt irq is clear*/ 66 /*halt irq is clear*/
67 status = true; 67 status = true;
68 }
68 69
69 return status; 70 return status;
70} 71}
@@ -86,9 +87,10 @@ static void gk20a_flcn_set_irq(struct nvgpu_falcon *flcn, bool enable)
86 flcn->intr_mask); 87 flcn->intr_mask);
87 gk20a_writel(g, base_addr + falcon_falcon_irqdest_r(), 88 gk20a_writel(g, base_addr + falcon_falcon_irqdest_r(),
88 flcn->intr_dest); 89 flcn->intr_dest);
89 } else 90 } else {
90 gk20a_writel(g, base_addr + falcon_falcon_irqmclr_r(), 91 gk20a_writel(g, base_addr + falcon_falcon_irqmclr_r(),
91 0xffffffff); 92 0xffffffff);
93 }
92} 94}
93 95
94static bool gk20a_is_falcon_cpu_halted(struct nvgpu_falcon *flcn) 96static bool gk20a_is_falcon_cpu_halted(struct nvgpu_falcon *flcn)
@@ -112,10 +114,11 @@ static bool gk20a_is_falcon_idle(struct nvgpu_falcon *flcn)
112 base_addr + falcon_falcon_idlestate_r()); 114 base_addr + falcon_falcon_idlestate_r());
113 115
114 if (falcon_falcon_idlestate_falcon_busy_v(unit_status) == 0 && 116 if (falcon_falcon_idlestate_falcon_busy_v(unit_status) == 0 &&
115 falcon_falcon_idlestate_ext_busy_v(unit_status) == 0) 117 falcon_falcon_idlestate_ext_busy_v(unit_status) == 0) {
116 status = true; 118 status = true;
117 else 119 } else {
118 status = false; 120 status = false;
121 }
119 122
120 return status; 123 return status;
121} 124}
@@ -131,10 +134,11 @@ static bool gk20a_is_falcon_scrubbing_done(struct nvgpu_falcon *flcn)
131 base_addr + falcon_falcon_dmactl_r()); 134 base_addr + falcon_falcon_dmactl_r());
132 135
133 if (unit_status & (falcon_falcon_dmactl_dmem_scrubbing_m() | 136 if (unit_status & (falcon_falcon_dmactl_dmem_scrubbing_m() |
134 falcon_falcon_dmactl_imem_scrubbing_m())) 137 falcon_falcon_dmactl_imem_scrubbing_m())) {
135 status = false; 138 status = false;
136 else 139 } else {
137 status = true; 140 status = true;
141 }
138 142
139 return status; 143 return status;
140} 144}
@@ -147,12 +151,13 @@ static u32 gk20a_falcon_get_mem_size(struct nvgpu_falcon *flcn,
147 u32 hw_cfg_reg = gk20a_readl(g, 151 u32 hw_cfg_reg = gk20a_readl(g,
148 flcn->flcn_base + falcon_falcon_hwcfg_r()); 152 flcn->flcn_base + falcon_falcon_hwcfg_r());
149 153
150 if (mem_type == MEM_DMEM) 154 if (mem_type == MEM_DMEM) {
151 mem_size = falcon_falcon_hwcfg_dmem_size_v(hw_cfg_reg) 155 mem_size = falcon_falcon_hwcfg_dmem_size_v(hw_cfg_reg)
152 << GK20A_PMU_DMEM_BLKSIZE2; 156 << GK20A_PMU_DMEM_BLKSIZE2;
153 else 157 } else {
154 mem_size = falcon_falcon_hwcfg_imem_size_v(hw_cfg_reg) 158 mem_size = falcon_falcon_hwcfg_imem_size_v(hw_cfg_reg)
155 << GK20A_PMU_DMEM_BLKSIZE2; 159 << GK20A_PMU_DMEM_BLKSIZE2;
160 }
156 161
157 return mem_size; 162 return mem_size;
158} 163}
@@ -416,12 +421,13 @@ static u32 gk20a_falcon_mailbox_read(struct nvgpu_falcon *flcn,
416 struct gk20a *g = flcn->g; 421 struct gk20a *g = flcn->g;
417 u32 data = 0; 422 u32 data = 0;
418 423
419 if (mailbox_index < FALCON_MAILBOX_COUNT) 424 if (mailbox_index < FALCON_MAILBOX_COUNT) {
420 data = gk20a_readl(g, flcn->flcn_base + (mailbox_index ? 425 data = gk20a_readl(g, flcn->flcn_base + (mailbox_index ?
421 falcon_falcon_mailbox1_r() : 426 falcon_falcon_mailbox1_r() :
422 falcon_falcon_mailbox0_r())); 427 falcon_falcon_mailbox0_r()));
423 else 428 } else {
424 nvgpu_err(g, "incorrect mailbox id %d", mailbox_index); 429 nvgpu_err(g, "incorrect mailbox id %d", mailbox_index);
430 }
425 431
426 return data; 432 return data;
427} 433}
@@ -431,13 +437,14 @@ static void gk20a_falcon_mailbox_write(struct nvgpu_falcon *flcn,
431{ 437{
432 struct gk20a *g = flcn->g; 438 struct gk20a *g = flcn->g;
433 439
434 if (mailbox_index < FALCON_MAILBOX_COUNT) 440 if (mailbox_index < FALCON_MAILBOX_COUNT) {
435 gk20a_writel(g, flcn->flcn_base + (mailbox_index ? 441 gk20a_writel(g, flcn->flcn_base + (mailbox_index ?
436 falcon_falcon_mailbox1_r() : 442 falcon_falcon_mailbox1_r() :
437 falcon_falcon_mailbox0_r()), 443 falcon_falcon_mailbox0_r()),
438 data); 444 data);
439 else 445 } else {
440 nvgpu_err(g, "incorrect mailbox id %d", mailbox_index); 446 nvgpu_err(g, "incorrect mailbox id %d", mailbox_index);
447 }
441} 448}
442 449
443static int gk20a_falcon_bl_bootstrap(struct nvgpu_falcon *flcn, 450static int gk20a_falcon_bl_bootstrap(struct nvgpu_falcon *flcn,
@@ -739,7 +746,8 @@ void gk20a_falcon_hal_sw_init(struct nvgpu_falcon *flcn)
739 if (flcn->is_falcon_supported) { 746 if (flcn->is_falcon_supported) {
740 nvgpu_mutex_init(&flcn->copy_lock); 747 nvgpu_mutex_init(&flcn->copy_lock);
741 gk20a_falcon_ops(flcn); 748 gk20a_falcon_ops(flcn);
742 } else 749 } else {
743 nvgpu_log_info(g, "falcon 0x%x not supported on %s", 750 nvgpu_log_info(g, "falcon 0x%x not supported on %s",
744 flcn->flcn_id, g->name); 751 flcn->flcn_id, g->name);
752 }
745} 753}
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c
index 859a7b6a..f5e35927 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.c
@@ -74,16 +74,18 @@ int gk20a_detect_chip(struct gk20a *g)
74{ 74{
75 struct nvgpu_gpu_params *p = &g->params; 75 struct nvgpu_gpu_params *p = &g->params;
76 76
77 if (p->gpu_arch) 77 if (p->gpu_arch) {
78 return 0; 78 return 0;
79 }
79 80
80 gk20a_mc_boot_0(g, &p->gpu_arch, &p->gpu_impl, &p->gpu_rev); 81 gk20a_mc_boot_0(g, &p->gpu_arch, &p->gpu_impl, &p->gpu_rev);
81 82
82 if ((p->gpu_arch + p->gpu_impl) == NVGPU_GPUID_GV11B) { 83 if ((p->gpu_arch + p->gpu_impl) == NVGPU_GPUID_GV11B) {
83 84
84 /* overwrite gpu revison for A02 */ 85 /* overwrite gpu revison for A02 */
85 if (!nvgpu_is_soc_t194_a01(g)) 86 if (!nvgpu_is_soc_t194_a01(g)) {
86 p->gpu_rev = 0xa2; 87 p->gpu_rev = 0xa2;
88 }
87 89
88 } 90 }
89 91
@@ -114,13 +116,15 @@ int gk20a_prepare_poweroff(struct gk20a *g)
114 116
115 if (g->ops.fifo.channel_suspend) { 117 if (g->ops.fifo.channel_suspend) {
116 ret = g->ops.fifo.channel_suspend(g); 118 ret = g->ops.fifo.channel_suspend(g);
117 if (ret) 119 if (ret) {
118 return ret; 120 return ret;
121 }
119 } 122 }
120 123
121 /* disable elpg before gr or fifo suspend */ 124 /* disable elpg before gr or fifo suspend */
122 if (g->ops.pmu.is_pmu_supported(g)) 125 if (g->ops.pmu.is_pmu_supported(g)) {
123 ret |= nvgpu_pmu_destroy(g); 126 ret |= nvgpu_pmu_destroy(g);
127 }
124 128
125 ret |= gk20a_gr_suspend(g); 129 ret |= gk20a_gr_suspend(g);
126 ret |= nvgpu_mm_suspend(g); 130 ret |= nvgpu_mm_suspend(g);
@@ -129,11 +133,13 @@ int gk20a_prepare_poweroff(struct gk20a *g)
129 gk20a_ce_suspend(g); 133 gk20a_ce_suspend(g);
130 134
131 /* Disable GPCPLL */ 135 /* Disable GPCPLL */
132 if (g->ops.clk.suspend_clk_support) 136 if (g->ops.clk.suspend_clk_support) {
133 ret |= g->ops.clk.suspend_clk_support(g); 137 ret |= g->ops.clk.suspend_clk_support(g);
138 }
134 139
135 if (nvgpu_is_enabled(g, NVGPU_PMU_PSTATE)) 140 if (nvgpu_is_enabled(g, NVGPU_PMU_PSTATE)) {
136 gk20a_deinit_pstate_support(g); 141 gk20a_deinit_pstate_support(g);
142 }
137 143
138 gk20a_mask_interrupts(g); 144 gk20a_mask_interrupts(g);
139 145
@@ -151,8 +157,9 @@ int gk20a_finalize_poweron(struct gk20a *g)
151 157
152 nvgpu_log_fn(g, " "); 158 nvgpu_log_fn(g, " ");
153 159
154 if (g->power_on) 160 if (g->power_on) {
155 return 0; 161 return 0;
162 }
156 163
157 g->power_on = true; 164 g->power_on = true;
158 165
@@ -170,23 +177,27 @@ int gk20a_finalize_poweron(struct gk20a *g)
170 * buffers. 177 * buffers.
171 */ 178 */
172 err = nvgpu_pd_cache_init(g); 179 err = nvgpu_pd_cache_init(g);
173 if (err) 180 if (err) {
174 return err; 181 return err;
182 }
175 183
176 /* init interface layer support for PMU falcon */ 184 /* init interface layer support for PMU falcon */
177 nvgpu_flcn_sw_init(g, FALCON_ID_PMU); 185 nvgpu_flcn_sw_init(g, FALCON_ID_PMU);
178 nvgpu_flcn_sw_init(g, FALCON_ID_SEC2); 186 nvgpu_flcn_sw_init(g, FALCON_ID_SEC2);
179 nvgpu_flcn_sw_init(g, FALCON_ID_NVDEC); 187 nvgpu_flcn_sw_init(g, FALCON_ID_NVDEC);
180 188
181 if (g->ops.bios.init) 189 if (g->ops.bios.init) {
182 err = g->ops.bios.init(g); 190 err = g->ops.bios.init(g);
183 if (err) 191 }
192 if (err) {
184 goto done; 193 goto done;
194 }
185 195
186 g->ops.bus.init_hw(g); 196 g->ops.bus.init_hw(g);
187 197
188 if (g->ops.clk.disable_slowboot) 198 if (g->ops.clk.disable_slowboot) {
189 g->ops.clk.disable_slowboot(g); 199 g->ops.clk.disable_slowboot(g);
200 }
190 201
191 g->ops.priv_ring.enable_priv_ring(g); 202 g->ops.priv_ring.enable_priv_ring(g);
192 203
@@ -253,8 +264,9 @@ int gk20a_finalize_poweron(struct gk20a *g)
253 goto done; 264 goto done;
254 } 265 }
255 266
256 if (g->ops.therm.elcg_init_idle_filters) 267 if (g->ops.therm.elcg_init_idle_filters) {
257 g->ops.therm.elcg_init_idle_filters(g); 268 g->ops.therm.elcg_init_idle_filters(g);
269 }
258 270
259 g->ops.mc.intr_enable(g); 271 g->ops.mc.intr_enable(g);
260 272
@@ -265,8 +277,9 @@ int gk20a_finalize_poweron(struct gk20a *g)
265 } 277 }
266 278
267 if (g->ops.pmu.is_pmu_supported(g)) { 279 if (g->ops.pmu.is_pmu_supported(g)) {
268 if (g->ops.pmu.prepare_ucode) 280 if (g->ops.pmu.prepare_ucode) {
269 err = g->ops.pmu.prepare_ucode(g); 281 err = g->ops.pmu.prepare_ucode(g);
282 }
270 if (err) { 283 if (err) {
271 nvgpu_err(g, "failed to init pmu ucode"); 284 nvgpu_err(g, "failed to init pmu ucode");
272 goto done; 285 goto done;
@@ -314,9 +327,9 @@ int gk20a_finalize_poweron(struct gk20a *g)
314 } 327 }
315 } 328 }
316 329
317 if (g->ops.pmu_ver.clk.clk_set_boot_clk && nvgpu_is_enabled(g, NVGPU_PMU_PSTATE)) 330 if (g->ops.pmu_ver.clk.clk_set_boot_clk && nvgpu_is_enabled(g, NVGPU_PMU_PSTATE)) {
318 g->ops.pmu_ver.clk.clk_set_boot_clk(g); 331 g->ops.pmu_ver.clk.clk_set_boot_clk(g);
319 else { 332 } else {
320 err = nvgpu_clk_arb_init_arbiter(g); 333 err = nvgpu_clk_arb_init_arbiter(g);
321 if (err) { 334 if (err) {
322 nvgpu_err(g, "failed to init clk arb"); 335 nvgpu_err(g, "failed to init clk arb");
@@ -350,8 +363,9 @@ int gk20a_finalize_poweron(struct gk20a *g)
350 if (g->ops.xve.available_speeds) { 363 if (g->ops.xve.available_speeds) {
351 u32 speed; 364 u32 speed;
352 365
353 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_ASPM) && g->ops.xve.disable_aspm) 366 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_ASPM) && g->ops.xve.disable_aspm) {
354 g->ops.xve.disable_aspm(g); 367 g->ops.xve.disable_aspm(g);
368 }
355 369
356 g->ops.xve.available_speeds(g, &speed); 370 g->ops.xve.available_speeds(g, &speed);
357 371
@@ -374,12 +388,14 @@ int gk20a_finalize_poweron(struct gk20a *g)
374 } 388 }
375#endif 389#endif
376 390
377 if (g->ops.fifo.channel_resume) 391 if (g->ops.fifo.channel_resume) {
378 g->ops.fifo.channel_resume(g); 392 g->ops.fifo.channel_resume(g);
393 }
379 394
380done: 395done:
381 if (err) 396 if (err) {
382 g->power_on = false; 397 g->power_on = false;
398 }
383 399
384 return err; 400 return err;
385} 401}
@@ -390,8 +406,9 @@ done:
390 */ 406 */
391int gk20a_can_busy(struct gk20a *g) 407int gk20a_can_busy(struct gk20a *g)
392{ 408{
393 if (nvgpu_is_enabled(g, NVGPU_DRIVER_IS_DYING)) 409 if (nvgpu_is_enabled(g, NVGPU_DRIVER_IS_DYING)) {
394 return 0; 410 return 0;
411 }
395 return 1; 412 return 1;
396} 413}
397 414
@@ -400,8 +417,9 @@ int gk20a_wait_for_idle(struct gk20a *g)
400 int wait_length = 150; /* 3 second overall max wait. */ 417 int wait_length = 150; /* 3 second overall max wait. */
401 int target_usage_count = 0; 418 int target_usage_count = 0;
402 419
403 if (!g) 420 if (!g) {
404 return -ENODEV; 421 return -ENODEV;
422 }
405 423
406 while ((nvgpu_atomic_read(&g->usage_count) != target_usage_count) 424 while ((nvgpu_atomic_read(&g->usage_count) != target_usage_count)
407 && (wait_length-- >= 0)) { 425 && (wait_length-- >= 0)) {
@@ -423,14 +441,17 @@ int gk20a_init_gpu_characteristics(struct gk20a *g)
423 __nvgpu_set_enabled(g, NVGPU_SUPPORT_MAP_DIRECT_KIND_CTRL, true); 441 __nvgpu_set_enabled(g, NVGPU_SUPPORT_MAP_DIRECT_KIND_CTRL, true);
424 __nvgpu_set_enabled(g, NVGPU_SUPPORT_MAP_BUFFER_BATCH, true); 442 __nvgpu_set_enabled(g, NVGPU_SUPPORT_MAP_BUFFER_BATCH, true);
425 443
426 if (IS_ENABLED(CONFIG_SYNC)) 444 if (IS_ENABLED(CONFIG_SYNC)) {
427 __nvgpu_set_enabled(g, NVGPU_SUPPORT_SYNC_FENCE_FDS, true); 445 __nvgpu_set_enabled(g, NVGPU_SUPPORT_SYNC_FENCE_FDS, true);
446 }
428 447
429 if (g->ops.mm.support_sparse && g->ops.mm.support_sparse(g)) 448 if (g->ops.mm.support_sparse && g->ops.mm.support_sparse(g)) {
430 __nvgpu_set_enabled(g, NVGPU_SUPPORT_SPARSE_ALLOCS, true); 449 __nvgpu_set_enabled(g, NVGPU_SUPPORT_SPARSE_ALLOCS, true);
450 }
431 451
432 if (gk20a_platform_has_syncpoints(g)) 452 if (gk20a_platform_has_syncpoints(g)) {
433 __nvgpu_set_enabled(g, NVGPU_HAS_SYNCPOINTS, true); 453 __nvgpu_set_enabled(g, NVGPU_HAS_SYNCPOINTS, true);
454 }
434 455
435 /* 456 /*
436 * Fast submits are supported as long as the user doesn't request 457 * Fast submits are supported as long as the user doesn't request
@@ -447,23 +468,26 @@ int gk20a_init_gpu_characteristics(struct gk20a *g)
447 * supported otherwise, provided that the user doesn't request anything 468 * supported otherwise, provided that the user doesn't request anything
448 * that depends on deferred cleanup. 469 * that depends on deferred cleanup.
449 */ 470 */
450 if (!gk20a_channel_sync_needs_sync_framework(g)) 471 if (!gk20a_channel_sync_needs_sync_framework(g)) {
451 __nvgpu_set_enabled(g, 472 __nvgpu_set_enabled(g,
452 NVGPU_SUPPORT_DETERMINISTIC_SUBMIT_FULL, 473 NVGPU_SUPPORT_DETERMINISTIC_SUBMIT_FULL,
453 true); 474 true);
475 }
454 476
455 __nvgpu_set_enabled(g, NVGPU_SUPPORT_DETERMINISTIC_OPTS, true); 477 __nvgpu_set_enabled(g, NVGPU_SUPPORT_DETERMINISTIC_OPTS, true);
456 478
457 __nvgpu_set_enabled(g, NVGPU_SUPPORT_USERSPACE_MANAGED_AS, true); 479 __nvgpu_set_enabled(g, NVGPU_SUPPORT_USERSPACE_MANAGED_AS, true);
458 __nvgpu_set_enabled(g, NVGPU_SUPPORT_TSG, true); 480 __nvgpu_set_enabled(g, NVGPU_SUPPORT_TSG, true);
459 481
460 if (g->ops.clk_arb.get_arbiter_clk_domains) 482 if (g->ops.clk_arb.get_arbiter_clk_domains) {
461 __nvgpu_set_enabled(g, NVGPU_SUPPORT_CLOCK_CONTROLS, true); 483 __nvgpu_set_enabled(g, NVGPU_SUPPORT_CLOCK_CONTROLS, true);
484 }
462 485
463 g->ops.gr.detect_sm_arch(g); 486 g->ops.gr.detect_sm_arch(g);
464 487
465 if (g->ops.gr.init_cyclestats) 488 if (g->ops.gr.init_cyclestats) {
466 g->ops.gr.init_cyclestats(g); 489 g->ops.gr.init_cyclestats(g);
490 }
467 491
468 g->ops.gr.get_rop_l2_en_mask(g); 492 g->ops.gr.get_rop_l2_en_mask(g);
469 493
@@ -482,11 +506,13 @@ static void gk20a_free_cb(struct nvgpu_ref *refcount)
482 506
483 gk20a_ce_destroy(g); 507 gk20a_ce_destroy(g);
484 508
485 if (g->remove_support) 509 if (g->remove_support) {
486 g->remove_support(g); 510 g->remove_support(g);
511 }
487 512
488 if (g->free) 513 if (g->free) {
489 g->free(g); 514 g->free(g);
515 }
490} 516}
491 517
492/** 518/**
diff --git a/drivers/gpu/nvgpu/gk20a/gr_ctx_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_ctx_gk20a.c
index 956d0434..8b9ac326 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_ctx_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_ctx_gk20a.c
@@ -38,8 +38,9 @@ static int gr_gk20a_alloc_load_netlist_u32(struct gk20a *g, u32 *src, u32 len,
38 struct u32_list_gk20a *u32_list) 38 struct u32_list_gk20a *u32_list)
39{ 39{
40 u32_list->count = (len + sizeof(u32) - 1) / sizeof(u32); 40 u32_list->count = (len + sizeof(u32) - 1) / sizeof(u32);
41 if (!alloc_u32_list_gk20a(g, u32_list)) 41 if (!alloc_u32_list_gk20a(g, u32_list)) {
42 return -ENOMEM; 42 return -ENOMEM;
43 }
43 44
44 memcpy(u32_list->l, src, len); 45 memcpy(u32_list->l, src, len);
45 46
@@ -50,8 +51,9 @@ static int gr_gk20a_alloc_load_netlist_av(struct gk20a *g, u32 *src, u32 len,
50 struct av_list_gk20a *av_list) 51 struct av_list_gk20a *av_list)
51{ 52{
52 av_list->count = len / sizeof(struct av_gk20a); 53 av_list->count = len / sizeof(struct av_gk20a);
53 if (!alloc_av_list_gk20a(g, av_list)) 54 if (!alloc_av_list_gk20a(g, av_list)) {
54 return -ENOMEM; 55 return -ENOMEM;
56 }
55 57
56 memcpy(av_list->l, src, len); 58 memcpy(av_list->l, src, len);
57 59
@@ -62,8 +64,9 @@ static int gr_gk20a_alloc_load_netlist_av64(struct gk20a *g, u32 *src, u32 len,
62 struct av64_list_gk20a *av64_list) 64 struct av64_list_gk20a *av64_list)
63{ 65{
64 av64_list->count = len / sizeof(struct av64_gk20a); 66 av64_list->count = len / sizeof(struct av64_gk20a);
65 if (!alloc_av64_list_gk20a(g, av64_list)) 67 if (!alloc_av64_list_gk20a(g, av64_list)) {
66 return -ENOMEM; 68 return -ENOMEM;
69 }
67 70
68 memcpy(av64_list->l, src, len); 71 memcpy(av64_list->l, src, len);
69 72
@@ -74,8 +77,9 @@ static int gr_gk20a_alloc_load_netlist_aiv(struct gk20a *g, u32 *src, u32 len,
74 struct aiv_list_gk20a *aiv_list) 77 struct aiv_list_gk20a *aiv_list)
75{ 78{
76 aiv_list->count = len / sizeof(struct aiv_gk20a); 79 aiv_list->count = len / sizeof(struct aiv_gk20a);
77 if (!alloc_aiv_list_gk20a(g, aiv_list)) 80 if (!alloc_aiv_list_gk20a(g, aiv_list)) {
78 return -ENOMEM; 81 return -ENOMEM;
82 }
79 83
80 memcpy(aiv_list->l, src, len); 84 memcpy(aiv_list->l, src, len);
81 85
@@ -128,57 +132,65 @@ static int gr_gk20a_init_ctx_vars_fw(struct gk20a *g, struct gr_gk20a *gr)
128 nvgpu_log_info(g, "NETLIST_REGIONID_FECS_UCODE_DATA"); 132 nvgpu_log_info(g, "NETLIST_REGIONID_FECS_UCODE_DATA");
129 err = gr_gk20a_alloc_load_netlist_u32(g, 133 err = gr_gk20a_alloc_load_netlist_u32(g,
130 src, size, &g->gr.ctx_vars.ucode.fecs.data); 134 src, size, &g->gr.ctx_vars.ucode.fecs.data);
131 if (err) 135 if (err) {
132 goto clean_up; 136 goto clean_up;
137 }
133 break; 138 break;
134 case NETLIST_REGIONID_FECS_UCODE_INST: 139 case NETLIST_REGIONID_FECS_UCODE_INST:
135 nvgpu_log_info(g, "NETLIST_REGIONID_FECS_UCODE_INST"); 140 nvgpu_log_info(g, "NETLIST_REGIONID_FECS_UCODE_INST");
136 err = gr_gk20a_alloc_load_netlist_u32(g, 141 err = gr_gk20a_alloc_load_netlist_u32(g,
137 src, size, &g->gr.ctx_vars.ucode.fecs.inst); 142 src, size, &g->gr.ctx_vars.ucode.fecs.inst);
138 if (err) 143 if (err) {
139 goto clean_up; 144 goto clean_up;
145 }
140 break; 146 break;
141 case NETLIST_REGIONID_GPCCS_UCODE_DATA: 147 case NETLIST_REGIONID_GPCCS_UCODE_DATA:
142 nvgpu_log_info(g, "NETLIST_REGIONID_GPCCS_UCODE_DATA"); 148 nvgpu_log_info(g, "NETLIST_REGIONID_GPCCS_UCODE_DATA");
143 err = gr_gk20a_alloc_load_netlist_u32(g, 149 err = gr_gk20a_alloc_load_netlist_u32(g,
144 src, size, &g->gr.ctx_vars.ucode.gpccs.data); 150 src, size, &g->gr.ctx_vars.ucode.gpccs.data);
145 if (err) 151 if (err) {
146 goto clean_up; 152 goto clean_up;
153 }
147 break; 154 break;
148 case NETLIST_REGIONID_GPCCS_UCODE_INST: 155 case NETLIST_REGIONID_GPCCS_UCODE_INST:
149 nvgpu_log_info(g, "NETLIST_REGIONID_GPCCS_UCODE_INST"); 156 nvgpu_log_info(g, "NETLIST_REGIONID_GPCCS_UCODE_INST");
150 err = gr_gk20a_alloc_load_netlist_u32(g, 157 err = gr_gk20a_alloc_load_netlist_u32(g,
151 src, size, &g->gr.ctx_vars.ucode.gpccs.inst); 158 src, size, &g->gr.ctx_vars.ucode.gpccs.inst);
152 if (err) 159 if (err) {
153 goto clean_up; 160 goto clean_up;
161 }
154 break; 162 break;
155 case NETLIST_REGIONID_SW_BUNDLE_INIT: 163 case NETLIST_REGIONID_SW_BUNDLE_INIT:
156 nvgpu_log_info(g, "NETLIST_REGIONID_SW_BUNDLE_INIT"); 164 nvgpu_log_info(g, "NETLIST_REGIONID_SW_BUNDLE_INIT");
157 err = gr_gk20a_alloc_load_netlist_av(g, 165 err = gr_gk20a_alloc_load_netlist_av(g,
158 src, size, &g->gr.ctx_vars.sw_bundle_init); 166 src, size, &g->gr.ctx_vars.sw_bundle_init);
159 if (err) 167 if (err) {
160 goto clean_up; 168 goto clean_up;
169 }
161 break; 170 break;
162 case NETLIST_REGIONID_SW_METHOD_INIT: 171 case NETLIST_REGIONID_SW_METHOD_INIT:
163 nvgpu_log_info(g, "NETLIST_REGIONID_SW_METHOD_INIT"); 172 nvgpu_log_info(g, "NETLIST_REGIONID_SW_METHOD_INIT");
164 err = gr_gk20a_alloc_load_netlist_av(g, 173 err = gr_gk20a_alloc_load_netlist_av(g,
165 src, size, &g->gr.ctx_vars.sw_method_init); 174 src, size, &g->gr.ctx_vars.sw_method_init);
166 if (err) 175 if (err) {
167 goto clean_up; 176 goto clean_up;
177 }
168 break; 178 break;
169 case NETLIST_REGIONID_SW_CTX_LOAD: 179 case NETLIST_REGIONID_SW_CTX_LOAD:
170 nvgpu_log_info(g, "NETLIST_REGIONID_SW_CTX_LOAD"); 180 nvgpu_log_info(g, "NETLIST_REGIONID_SW_CTX_LOAD");
171 err = gr_gk20a_alloc_load_netlist_aiv(g, 181 err = gr_gk20a_alloc_load_netlist_aiv(g,
172 src, size, &g->gr.ctx_vars.sw_ctx_load); 182 src, size, &g->gr.ctx_vars.sw_ctx_load);
173 if (err) 183 if (err) {
174 goto clean_up; 184 goto clean_up;
185 }
175 break; 186 break;
176 case NETLIST_REGIONID_SW_NON_CTX_LOAD: 187 case NETLIST_REGIONID_SW_NON_CTX_LOAD:
177 nvgpu_log_info(g, "NETLIST_REGIONID_SW_NON_CTX_LOAD"); 188 nvgpu_log_info(g, "NETLIST_REGIONID_SW_NON_CTX_LOAD");
178 err = gr_gk20a_alloc_load_netlist_av(g, 189 err = gr_gk20a_alloc_load_netlist_av(g,
179 src, size, &g->gr.ctx_vars.sw_non_ctx_load); 190 src, size, &g->gr.ctx_vars.sw_non_ctx_load);
180 if (err) 191 if (err) {
181 goto clean_up; 192 goto clean_up;
193 }
182 break; 194 break;
183 case NETLIST_REGIONID_SWVEIDBUNDLEINIT: 195 case NETLIST_REGIONID_SWVEIDBUNDLEINIT:
184 nvgpu_log_info(g, 196 nvgpu_log_info(g,
@@ -186,64 +198,73 @@ static int gr_gk20a_init_ctx_vars_fw(struct gk20a *g, struct gr_gk20a *gr)
186 err = gr_gk20a_alloc_load_netlist_av(g, 198 err = gr_gk20a_alloc_load_netlist_av(g,
187 src, size, 199 src, size,
188 &g->gr.ctx_vars.sw_veid_bundle_init); 200 &g->gr.ctx_vars.sw_veid_bundle_init);
189 if (err) 201 if (err) {
190 goto clean_up; 202 goto clean_up;
203 }
191 break; 204 break;
192 case NETLIST_REGIONID_CTXREG_SYS: 205 case NETLIST_REGIONID_CTXREG_SYS:
193 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_SYS"); 206 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_SYS");
194 err = gr_gk20a_alloc_load_netlist_aiv(g, 207 err = gr_gk20a_alloc_load_netlist_aiv(g,
195 src, size, &g->gr.ctx_vars.ctxsw_regs.sys); 208 src, size, &g->gr.ctx_vars.ctxsw_regs.sys);
196 if (err) 209 if (err) {
197 goto clean_up; 210 goto clean_up;
211 }
198 break; 212 break;
199 case NETLIST_REGIONID_CTXREG_GPC: 213 case NETLIST_REGIONID_CTXREG_GPC:
200 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_GPC"); 214 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_GPC");
201 err = gr_gk20a_alloc_load_netlist_aiv(g, 215 err = gr_gk20a_alloc_load_netlist_aiv(g,
202 src, size, &g->gr.ctx_vars.ctxsw_regs.gpc); 216 src, size, &g->gr.ctx_vars.ctxsw_regs.gpc);
203 if (err) 217 if (err) {
204 goto clean_up; 218 goto clean_up;
219 }
205 break; 220 break;
206 case NETLIST_REGIONID_CTXREG_TPC: 221 case NETLIST_REGIONID_CTXREG_TPC:
207 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_TPC"); 222 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_TPC");
208 err = gr_gk20a_alloc_load_netlist_aiv(g, 223 err = gr_gk20a_alloc_load_netlist_aiv(g,
209 src, size, &g->gr.ctx_vars.ctxsw_regs.tpc); 224 src, size, &g->gr.ctx_vars.ctxsw_regs.tpc);
210 if (err) 225 if (err) {
211 goto clean_up; 226 goto clean_up;
227 }
212 break; 228 break;
213 case NETLIST_REGIONID_CTXREG_ZCULL_GPC: 229 case NETLIST_REGIONID_CTXREG_ZCULL_GPC:
214 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_ZCULL_GPC"); 230 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_ZCULL_GPC");
215 err = gr_gk20a_alloc_load_netlist_aiv(g, 231 err = gr_gk20a_alloc_load_netlist_aiv(g,
216 src, size, &g->gr.ctx_vars.ctxsw_regs.zcull_gpc); 232 src, size, &g->gr.ctx_vars.ctxsw_regs.zcull_gpc);
217 if (err) 233 if (err) {
218 goto clean_up; 234 goto clean_up;
235 }
219 break; 236 break;
220 case NETLIST_REGIONID_CTXREG_PPC: 237 case NETLIST_REGIONID_CTXREG_PPC:
221 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PPC"); 238 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PPC");
222 err = gr_gk20a_alloc_load_netlist_aiv(g, 239 err = gr_gk20a_alloc_load_netlist_aiv(g,
223 src, size, &g->gr.ctx_vars.ctxsw_regs.ppc); 240 src, size, &g->gr.ctx_vars.ctxsw_regs.ppc);
224 if (err) 241 if (err) {
225 goto clean_up; 242 goto clean_up;
243 }
226 break; 244 break;
227 case NETLIST_REGIONID_CTXREG_PM_SYS: 245 case NETLIST_REGIONID_CTXREG_PM_SYS:
228 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_SYS"); 246 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_SYS");
229 err = gr_gk20a_alloc_load_netlist_aiv(g, 247 err = gr_gk20a_alloc_load_netlist_aiv(g,
230 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_sys); 248 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_sys);
231 if (err) 249 if (err) {
232 goto clean_up; 250 goto clean_up;
251 }
233 break; 252 break;
234 case NETLIST_REGIONID_CTXREG_PM_GPC: 253 case NETLIST_REGIONID_CTXREG_PM_GPC:
235 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_GPC"); 254 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_GPC");
236 err = gr_gk20a_alloc_load_netlist_aiv(g, 255 err = gr_gk20a_alloc_load_netlist_aiv(g,
237 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_gpc); 256 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_gpc);
238 if (err) 257 if (err) {
239 goto clean_up; 258 goto clean_up;
259 }
240 break; 260 break;
241 case NETLIST_REGIONID_CTXREG_PM_TPC: 261 case NETLIST_REGIONID_CTXREG_PM_TPC:
242 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_TPC"); 262 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_TPC");
243 err = gr_gk20a_alloc_load_netlist_aiv(g, 263 err = gr_gk20a_alloc_load_netlist_aiv(g,
244 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_tpc); 264 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_tpc);
245 if (err) 265 if (err) {
246 goto clean_up; 266 goto clean_up;
267 }
247 break; 268 break;
248 case NETLIST_REGIONID_BUFFER_SIZE: 269 case NETLIST_REGIONID_BUFFER_SIZE:
249 g->gr.ctx_vars.buffer_size = *src; 270 g->gr.ctx_vars.buffer_size = *src;
@@ -269,108 +290,123 @@ static int gr_gk20a_init_ctx_vars_fw(struct gk20a *g, struct gr_gk20a *gr)
269 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMPPC"); 290 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMPPC");
270 err = gr_gk20a_alloc_load_netlist_aiv(g, 291 err = gr_gk20a_alloc_load_netlist_aiv(g,
271 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ppc); 292 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ppc);
272 if (err) 293 if (err) {
273 goto clean_up; 294 goto clean_up;
295 }
274 break; 296 break;
275 case NETLIST_REGIONID_NVPERF_CTXREG_SYS: 297 case NETLIST_REGIONID_NVPERF_CTXREG_SYS:
276 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_CTXREG_SYS"); 298 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_CTXREG_SYS");
277 err = gr_gk20a_alloc_load_netlist_aiv(g, 299 err = gr_gk20a_alloc_load_netlist_aiv(g,
278 src, size, &g->gr.ctx_vars.ctxsw_regs.perf_sys); 300 src, size, &g->gr.ctx_vars.ctxsw_regs.perf_sys);
279 if (err) 301 if (err) {
280 goto clean_up; 302 goto clean_up;
303 }
281 break; 304 break;
282 case NETLIST_REGIONID_NVPERF_FBP_CTXREGS: 305 case NETLIST_REGIONID_NVPERF_FBP_CTXREGS:
283 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_FBP_CTXREGS"); 306 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_FBP_CTXREGS");
284 err = gr_gk20a_alloc_load_netlist_aiv(g, 307 err = gr_gk20a_alloc_load_netlist_aiv(g,
285 src, size, &g->gr.ctx_vars.ctxsw_regs.fbp); 308 src, size, &g->gr.ctx_vars.ctxsw_regs.fbp);
286 if (err) 309 if (err) {
287 goto clean_up; 310 goto clean_up;
311 }
288 break; 312 break;
289 case NETLIST_REGIONID_NVPERF_CTXREG_GPC: 313 case NETLIST_REGIONID_NVPERF_CTXREG_GPC:
290 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_CTXREG_GPC"); 314 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_CTXREG_GPC");
291 err = gr_gk20a_alloc_load_netlist_aiv(g, 315 err = gr_gk20a_alloc_load_netlist_aiv(g,
292 src, size, &g->gr.ctx_vars.ctxsw_regs.perf_gpc); 316 src, size, &g->gr.ctx_vars.ctxsw_regs.perf_gpc);
293 if (err) 317 if (err) {
294 goto clean_up; 318 goto clean_up;
319 }
295 break; 320 break;
296 case NETLIST_REGIONID_NVPERF_FBP_ROUTER: 321 case NETLIST_REGIONID_NVPERF_FBP_ROUTER:
297 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_FBP_ROUTER"); 322 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_FBP_ROUTER");
298 err = gr_gk20a_alloc_load_netlist_aiv(g, 323 err = gr_gk20a_alloc_load_netlist_aiv(g,
299 src, size, &g->gr.ctx_vars.ctxsw_regs.fbp_router); 324 src, size, &g->gr.ctx_vars.ctxsw_regs.fbp_router);
300 if (err) 325 if (err) {
301 goto clean_up; 326 goto clean_up;
327 }
302 break; 328 break;
303 case NETLIST_REGIONID_NVPERF_GPC_ROUTER: 329 case NETLIST_REGIONID_NVPERF_GPC_ROUTER:
304 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_GPC_ROUTER"); 330 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_GPC_ROUTER");
305 err = gr_gk20a_alloc_load_netlist_aiv(g, 331 err = gr_gk20a_alloc_load_netlist_aiv(g,
306 src, size, &g->gr.ctx_vars.ctxsw_regs.gpc_router); 332 src, size, &g->gr.ctx_vars.ctxsw_regs.gpc_router);
307 if (err) 333 if (err) {
308 goto clean_up; 334 goto clean_up;
335 }
309 break; 336 break;
310 case NETLIST_REGIONID_CTXREG_PMLTC: 337 case NETLIST_REGIONID_CTXREG_PMLTC:
311 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMLTC"); 338 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMLTC");
312 err = gr_gk20a_alloc_load_netlist_aiv(g, 339 err = gr_gk20a_alloc_load_netlist_aiv(g,
313 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ltc); 340 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ltc);
314 if (err) 341 if (err) {
315 goto clean_up; 342 goto clean_up;
343 }
316 break; 344 break;
317 case NETLIST_REGIONID_CTXREG_PMFBPA: 345 case NETLIST_REGIONID_CTXREG_PMFBPA:
318 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMFBPA"); 346 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMFBPA");
319 err = gr_gk20a_alloc_load_netlist_aiv(g, 347 err = gr_gk20a_alloc_load_netlist_aiv(g,
320 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_fbpa); 348 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_fbpa);
321 if (err) 349 if (err) {
322 goto clean_up; 350 goto clean_up;
351 }
323 break; 352 break;
324 case NETLIST_REGIONID_NVPERF_SYS_ROUTER: 353 case NETLIST_REGIONID_NVPERF_SYS_ROUTER:
325 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_SYS_ROUTER"); 354 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_SYS_ROUTER");
326 err = gr_gk20a_alloc_load_netlist_aiv(g, 355 err = gr_gk20a_alloc_load_netlist_aiv(g,
327 src, size, &g->gr.ctx_vars.ctxsw_regs.perf_sys_router); 356 src, size, &g->gr.ctx_vars.ctxsw_regs.perf_sys_router);
328 if (err) 357 if (err) {
329 goto clean_up; 358 goto clean_up;
359 }
330 break; 360 break;
331 case NETLIST_REGIONID_NVPERF_PMA: 361 case NETLIST_REGIONID_NVPERF_PMA:
332 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_PMA"); 362 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_PMA");
333 err = gr_gk20a_alloc_load_netlist_aiv(g, 363 err = gr_gk20a_alloc_load_netlist_aiv(g,
334 src, size, &g->gr.ctx_vars.ctxsw_regs.perf_pma); 364 src, size, &g->gr.ctx_vars.ctxsw_regs.perf_pma);
335 if (err) 365 if (err) {
336 goto clean_up; 366 goto clean_up;
367 }
337 break; 368 break;
338 case NETLIST_REGIONID_CTXREG_PMROP: 369 case NETLIST_REGIONID_CTXREG_PMROP:
339 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMROP"); 370 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMROP");
340 err = gr_gk20a_alloc_load_netlist_aiv(g, 371 err = gr_gk20a_alloc_load_netlist_aiv(g,
341 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_rop); 372 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_rop);
342 if (err) 373 if (err) {
343 goto clean_up; 374 goto clean_up;
375 }
344 break; 376 break;
345 case NETLIST_REGIONID_CTXREG_PMUCGPC: 377 case NETLIST_REGIONID_CTXREG_PMUCGPC:
346 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMUCGPC"); 378 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMUCGPC");
347 err = gr_gk20a_alloc_load_netlist_aiv(g, 379 err = gr_gk20a_alloc_load_netlist_aiv(g,
348 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ucgpc); 380 src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ucgpc);
349 if (err) 381 if (err) {
350 goto clean_up; 382 goto clean_up;
383 }
351 break; 384 break;
352 case NETLIST_REGIONID_CTXREG_ETPC: 385 case NETLIST_REGIONID_CTXREG_ETPC:
353 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_ETPC"); 386 nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_ETPC");
354 err = gr_gk20a_alloc_load_netlist_aiv(g, 387 err = gr_gk20a_alloc_load_netlist_aiv(g,
355 src, size, &g->gr.ctx_vars.ctxsw_regs.etpc); 388 src, size, &g->gr.ctx_vars.ctxsw_regs.etpc);
356 if (err) 389 if (err) {
357 goto clean_up; 390 goto clean_up;
391 }
358 break; 392 break;
359 case NETLIST_REGIONID_SW_BUNDLE64_INIT: 393 case NETLIST_REGIONID_SW_BUNDLE64_INIT:
360 nvgpu_log_info(g, "NETLIST_REGIONID_SW_BUNDLE64_INIT"); 394 nvgpu_log_info(g, "NETLIST_REGIONID_SW_BUNDLE64_INIT");
361 err = gr_gk20a_alloc_load_netlist_av64(g, 395 err = gr_gk20a_alloc_load_netlist_av64(g,
362 src, size, 396 src, size,
363 &g->gr.ctx_vars.sw_bundle64_init); 397 &g->gr.ctx_vars.sw_bundle64_init);
364 if (err) 398 if (err) {
365 goto clean_up; 399 goto clean_up;
400 }
366 break; 401 break;
367 case NETLIST_REGIONID_NVPERF_PMCAU: 402 case NETLIST_REGIONID_NVPERF_PMCAU:
368 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_PMCAU"); 403 nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_PMCAU");
369 err = gr_gk20a_alloc_load_netlist_aiv(g, 404 err = gr_gk20a_alloc_load_netlist_aiv(g,
370 src, size, 405 src, size,
371 &g->gr.ctx_vars.ctxsw_regs.pm_cau); 406 &g->gr.ctx_vars.ctxsw_regs.pm_cau);
372 if (err) 407 if (err) {
373 goto clean_up; 408 goto clean_up;
409 }
374 break; 410 break;
375 411
376 default: 412 default:
@@ -442,8 +478,9 @@ done:
442 478
443int gr_gk20a_init_ctx_vars(struct gk20a *g, struct gr_gk20a *gr) 479int gr_gk20a_init_ctx_vars(struct gk20a *g, struct gr_gk20a *gr)
444{ 480{
445 if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) 481 if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) {
446 return gr_gk20a_init_ctx_vars_sim(g, gr); 482 return gr_gk20a_init_ctx_vars_sim(g, gr);
447 else 483 } else {
448 return gr_gk20a_init_ctx_vars_fw(g, gr); 484 return gr_gk20a_init_ctx_vars_fw(g, gr);
485 }
449} 486}