diff options
Diffstat (limited to 'drivers/gpu')
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/ce2_gk20a.c | 72 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c | 38 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c | 54 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/fence_gk20a.c | 47 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | 554 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/flcn_gk20a.c | 38 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/gk20a.c | 82 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/gr_ctx_gk20a.c | 113 |
8 files changed, 641 insertions, 357 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c index f905243e..4cc6c8ca 100644 --- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c | |||
@@ -79,11 +79,13 @@ void gk20a_ce2_isr(struct gk20a *g, u32 inst_id, u32 pri_base) | |||
79 | nvgpu_log(g, gpu_dbg_intr, "ce2 isr %08x\n", ce2_intr); | 79 | nvgpu_log(g, gpu_dbg_intr, "ce2 isr %08x\n", ce2_intr); |
80 | 80 | ||
81 | /* clear blocking interrupts: they exibit broken behavior */ | 81 | /* clear blocking interrupts: they exibit broken behavior */ |
82 | if (ce2_intr & ce2_intr_status_blockpipe_pending_f()) | 82 | if (ce2_intr & ce2_intr_status_blockpipe_pending_f()) { |
83 | clear_intr |= ce2_blockpipe_isr(g, ce2_intr); | 83 | clear_intr |= ce2_blockpipe_isr(g, ce2_intr); |
84 | } | ||
84 | 85 | ||
85 | if (ce2_intr & ce2_intr_status_launcherr_pending_f()) | 86 | if (ce2_intr & ce2_intr_status_launcherr_pending_f()) { |
86 | clear_intr |= ce2_launcherr_isr(g, ce2_intr); | 87 | clear_intr |= ce2_launcherr_isr(g, ce2_intr); |
88 | } | ||
87 | 89 | ||
88 | gk20a_writel(g, ce2_intr_status_r(), clear_intr); | 90 | gk20a_writel(g, ce2_intr_status_r(), clear_intr); |
89 | return; | 91 | return; |
@@ -112,8 +114,9 @@ static void gk20a_ce_put_fences(struct gk20a_gpu_ctx *ce_ctx) | |||
112 | 114 | ||
113 | for (i = 0; i < NVGPU_CE_MAX_INFLIGHT_JOBS; i++) { | 115 | for (i = 0; i < NVGPU_CE_MAX_INFLIGHT_JOBS; i++) { |
114 | struct gk20a_fence **fence = &ce_ctx->postfences[i]; | 116 | struct gk20a_fence **fence = &ce_ctx->postfences[i]; |
115 | if (*fence) | 117 | if (*fence) { |
116 | gk20a_fence_put(*fence); | 118 | gk20a_fence_put(*fence); |
119 | } | ||
117 | *fence = NULL; | 120 | *fence = NULL; |
118 | } | 121 | } |
119 | } | 122 | } |
@@ -140,8 +143,9 @@ static void gk20a_ce_delete_gpu_context(struct gk20a_gpu_ctx *ce_ctx) | |||
140 | nvgpu_ref_put(&ce_ctx->tsg->refcount, gk20a_tsg_release); | 143 | nvgpu_ref_put(&ce_ctx->tsg->refcount, gk20a_tsg_release); |
141 | 144 | ||
142 | /* housekeeping on app */ | 145 | /* housekeeping on app */ |
143 | if (list->prev && list->next) | 146 | if (list->prev && list->next) { |
144 | nvgpu_list_del(list); | 147 | nvgpu_list_del(list); |
148 | } | ||
145 | 149 | ||
146 | nvgpu_mutex_release(&ce_ctx->gpu_ctx_mutex); | 150 | nvgpu_mutex_release(&ce_ctx->gpu_ctx_mutex); |
147 | nvgpu_mutex_destroy(&ce_ctx->gpu_ctx_mutex); | 151 | nvgpu_mutex_destroy(&ce_ctx->gpu_ctx_mutex); |
@@ -171,10 +175,11 @@ static inline unsigned int gk20a_ce_get_method_size(int request_operation, | |||
171 | chunk -= (u64) height * width; | 175 | chunk -= (u64) height * width; |
172 | } | 176 | } |
173 | 177 | ||
174 | if (request_operation & NVGPU_CE_PHYS_MODE_TRANSFER) | 178 | if (request_operation & NVGPU_CE_PHYS_MODE_TRANSFER) { |
175 | methodsize = (2 + (16 * iterations)) * sizeof(u32); | 179 | methodsize = (2 + (16 * iterations)) * sizeof(u32); |
176 | else if (request_operation & NVGPU_CE_MEMSET) | 180 | } else if (request_operation & NVGPU_CE_MEMSET) { |
177 | methodsize = (2 + (15 * iterations)) * sizeof(u32); | 181 | methodsize = (2 + (15 * iterations)) * sizeof(u32); |
182 | } | ||
178 | 183 | ||
179 | return methodsize; | 184 | return methodsize; |
180 | } | 185 | } |
@@ -198,8 +203,9 @@ int gk20a_ce_prepare_submit(u64 src_buf, | |||
198 | /* failure case handling */ | 203 | /* failure case handling */ |
199 | if ((gk20a_ce_get_method_size(request_operation, size) > | 204 | if ((gk20a_ce_get_method_size(request_operation, size) > |
200 | max_cmd_buf_size) || (!size) || | 205 | max_cmd_buf_size) || (!size) || |
201 | (request_operation > NVGPU_CE_MEMSET)) | 206 | (request_operation > NVGPU_CE_MEMSET)) { |
202 | return 0; | 207 | return 0; |
208 | } | ||
203 | 209 | ||
204 | /* set the channel object */ | 210 | /* set the channel object */ |
205 | cmd_buf_cpu_va[methodSize++] = 0x20018000; | 211 | cmd_buf_cpu_va[methodSize++] = 0x20018000; |
@@ -252,13 +258,14 @@ int gk20a_ce_prepare_submit(u64 src_buf, | |||
252 | offset) & NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK); | 258 | offset) & NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK); |
253 | 259 | ||
254 | cmd_buf_cpu_va[methodSize++] = 0x20018098; | 260 | cmd_buf_cpu_va[methodSize++] = 0x20018098; |
255 | if (launch_flags & NVGPU_CE_SRC_LOCATION_LOCAL_FB) | 261 | if (launch_flags & NVGPU_CE_SRC_LOCATION_LOCAL_FB) { |
256 | cmd_buf_cpu_va[methodSize++] = 0x00000000; | 262 | cmd_buf_cpu_va[methodSize++] = 0x00000000; |
257 | else if (launch_flags & | 263 | } else if (launch_flags & |
258 | NVGPU_CE_SRC_LOCATION_NONCOHERENT_SYSMEM) | 264 | NVGPU_CE_SRC_LOCATION_NONCOHERENT_SYSMEM) { |
259 | cmd_buf_cpu_va[methodSize++] = 0x00000002; | 265 | cmd_buf_cpu_va[methodSize++] = 0x00000002; |
260 | else | 266 | } else { |
261 | cmd_buf_cpu_va[methodSize++] = 0x00000001; | 267 | cmd_buf_cpu_va[methodSize++] = 0x00000001; |
268 | } | ||
262 | 269 | ||
263 | launch |= 0x00001000; | 270 | launch |= 0x00001000; |
264 | } else if (request_operation & NVGPU_CE_MEMSET) { | 271 | } else if (request_operation & NVGPU_CE_MEMSET) { |
@@ -289,25 +296,28 @@ int gk20a_ce_prepare_submit(u64 src_buf, | |||
289 | cmd_buf_cpu_va[methodSize++] = height; | 296 | cmd_buf_cpu_va[methodSize++] = height; |
290 | 297 | ||
291 | cmd_buf_cpu_va[methodSize++] = 0x20018099; | 298 | cmd_buf_cpu_va[methodSize++] = 0x20018099; |
292 | if (launch_flags & NVGPU_CE_DST_LOCATION_LOCAL_FB) | 299 | if (launch_flags & NVGPU_CE_DST_LOCATION_LOCAL_FB) { |
293 | cmd_buf_cpu_va[methodSize++] = 0x00000000; | 300 | cmd_buf_cpu_va[methodSize++] = 0x00000000; |
294 | else if (launch_flags & | 301 | } else if (launch_flags & |
295 | NVGPU_CE_DST_LOCATION_NONCOHERENT_SYSMEM) | 302 | NVGPU_CE_DST_LOCATION_NONCOHERENT_SYSMEM) { |
296 | cmd_buf_cpu_va[methodSize++] = 0x00000002; | 303 | cmd_buf_cpu_va[methodSize++] = 0x00000002; |
297 | else | 304 | } else { |
298 | cmd_buf_cpu_va[methodSize++] = 0x00000001; | 305 | cmd_buf_cpu_va[methodSize++] = 0x00000001; |
306 | } | ||
299 | 307 | ||
300 | launch |= 0x00002005; | 308 | launch |= 0x00002005; |
301 | 309 | ||
302 | if (launch_flags & NVGPU_CE_SRC_MEMORY_LAYOUT_BLOCKLINEAR) | 310 | if (launch_flags & NVGPU_CE_SRC_MEMORY_LAYOUT_BLOCKLINEAR) { |
303 | launch |= 0x00000000; | 311 | launch |= 0x00000000; |
304 | else | 312 | } else { |
305 | launch |= 0x00000080; | 313 | launch |= 0x00000080; |
314 | } | ||
306 | 315 | ||
307 | if (launch_flags & NVGPU_CE_DST_MEMORY_LAYOUT_BLOCKLINEAR) | 316 | if (launch_flags & NVGPU_CE_DST_MEMORY_LAYOUT_BLOCKLINEAR) { |
308 | launch |= 0x00000000; | 317 | launch |= 0x00000000; |
309 | else | 318 | } else { |
310 | launch |= 0x00000100; | 319 | launch |= 0x00000100; |
320 | } | ||
311 | 321 | ||
312 | cmd_buf_cpu_va[methodSize++] = 0x200180c0; | 322 | cmd_buf_cpu_va[methodSize++] = 0x200180c0; |
313 | cmd_buf_cpu_va[methodSize++] = launch; | 323 | cmd_buf_cpu_va[methodSize++] = launch; |
@@ -329,12 +339,14 @@ int gk20a_init_ce_support(struct gk20a *g) | |||
329 | 339 | ||
330 | g->ops.mc.reset(g, ce_reset_mask); | 340 | g->ops.mc.reset(g, ce_reset_mask); |
331 | 341 | ||
332 | if (g->ops.clock_gating.slcg_ce2_load_gating_prod) | 342 | if (g->ops.clock_gating.slcg_ce2_load_gating_prod) { |
333 | g->ops.clock_gating.slcg_ce2_load_gating_prod(g, | 343 | g->ops.clock_gating.slcg_ce2_load_gating_prod(g, |
334 | g->slcg_enabled); | 344 | g->slcg_enabled); |
335 | if (g->ops.clock_gating.blcg_ce_load_gating_prod) | 345 | } |
346 | if (g->ops.clock_gating.blcg_ce_load_gating_prod) { | ||
336 | g->ops.clock_gating.blcg_ce_load_gating_prod(g, | 347 | g->ops.clock_gating.blcg_ce_load_gating_prod(g, |
337 | g->blcg_enabled); | 348 | g->blcg_enabled); |
349 | } | ||
338 | 350 | ||
339 | if (ce_app->initialised) { | 351 | if (ce_app->initialised) { |
340 | /* assume this happen during poweron/poweroff GPU sequence */ | 352 | /* assume this happen during poweron/poweroff GPU sequence */ |
@@ -345,8 +357,9 @@ int gk20a_init_ce_support(struct gk20a *g) | |||
345 | nvgpu_log(g, gpu_dbg_fn, "ce: init"); | 357 | nvgpu_log(g, gpu_dbg_fn, "ce: init"); |
346 | 358 | ||
347 | err = nvgpu_mutex_init(&ce_app->app_mutex); | 359 | err = nvgpu_mutex_init(&ce_app->app_mutex); |
348 | if (err) | 360 | if (err) { |
349 | return err; | 361 | return err; |
362 | } | ||
350 | 363 | ||
351 | nvgpu_mutex_acquire(&ce_app->app_mutex); | 364 | nvgpu_mutex_acquire(&ce_app->app_mutex); |
352 | 365 | ||
@@ -367,8 +380,9 @@ void gk20a_ce_destroy(struct gk20a *g) | |||
367 | struct gk20a_ce_app *ce_app = &g->ce_app; | 380 | struct gk20a_ce_app *ce_app = &g->ce_app; |
368 | struct gk20a_gpu_ctx *ce_ctx, *ce_ctx_save; | 381 | struct gk20a_gpu_ctx *ce_ctx, *ce_ctx_save; |
369 | 382 | ||
370 | if (!ce_app->initialised) | 383 | if (!ce_app->initialised) { |
371 | return; | 384 | return; |
385 | } | ||
372 | 386 | ||
373 | ce_app->app_state = NVGPU_CE_SUSPEND; | 387 | ce_app->app_state = NVGPU_CE_SUSPEND; |
374 | ce_app->initialised = false; | 388 | ce_app->initialised = false; |
@@ -393,8 +407,9 @@ void gk20a_ce_suspend(struct gk20a *g) | |||
393 | { | 407 | { |
394 | struct gk20a_ce_app *ce_app = &g->ce_app; | 408 | struct gk20a_ce_app *ce_app = &g->ce_app; |
395 | 409 | ||
396 | if (!ce_app->initialised) | 410 | if (!ce_app->initialised) { |
397 | return; | 411 | return; |
412 | } | ||
398 | 413 | ||
399 | ce_app->app_state = NVGPU_CE_SUSPEND; | 414 | ce_app->app_state = NVGPU_CE_SUSPEND; |
400 | 415 | ||
@@ -413,12 +428,14 @@ u32 gk20a_ce_create_context(struct gk20a *g, | |||
413 | u32 ctx_id = ~0; | 428 | u32 ctx_id = ~0; |
414 | int err = 0; | 429 | int err = 0; |
415 | 430 | ||
416 | if (!ce_app->initialised || ce_app->app_state != NVGPU_CE_ACTIVE) | 431 | if (!ce_app->initialised || ce_app->app_state != NVGPU_CE_ACTIVE) { |
417 | return ctx_id; | 432 | return ctx_id; |
433 | } | ||
418 | 434 | ||
419 | ce_ctx = nvgpu_kzalloc(g, sizeof(*ce_ctx)); | 435 | ce_ctx = nvgpu_kzalloc(g, sizeof(*ce_ctx)); |
420 | if (!ce_ctx) | 436 | if (!ce_ctx) { |
421 | return ctx_id; | 437 | return ctx_id; |
438 | } | ||
422 | 439 | ||
423 | err = nvgpu_mutex_init(&ce_ctx->gpu_ctx_mutex); | 440 | err = nvgpu_mutex_init(&ce_ctx->gpu_ctx_mutex); |
424 | if (err) { | 441 | if (err) { |
@@ -538,8 +555,9 @@ void gk20a_ce_delete_context_priv(struct gk20a *g, | |||
538 | struct gk20a_ce_app *ce_app = &g->ce_app; | 555 | struct gk20a_ce_app *ce_app = &g->ce_app; |
539 | struct gk20a_gpu_ctx *ce_ctx, *ce_ctx_save; | 556 | struct gk20a_gpu_ctx *ce_ctx, *ce_ctx_save; |
540 | 557 | ||
541 | if (!ce_app->initialised ||ce_app->app_state != NVGPU_CE_ACTIVE) | 558 | if (!ce_app->initialised || ce_app->app_state != NVGPU_CE_ACTIVE) { |
542 | return; | 559 | return; |
560 | } | ||
543 | 561 | ||
544 | nvgpu_mutex_acquire(&ce_app->app_mutex); | 562 | nvgpu_mutex_acquire(&ce_app->app_mutex); |
545 | 563 | ||
diff --git a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c index 09668d49..f6134460 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c | |||
@@ -358,24 +358,26 @@ static void add_sema_cmd(struct gk20a *g, struct channel_gk20a *c, | |||
358 | * If the op is not an acquire (so therefor a release) we should | 358 | * If the op is not an acquire (so therefor a release) we should |
359 | * incr the underlying sema next_value. | 359 | * incr the underlying sema next_value. |
360 | */ | 360 | */ |
361 | if (!acquire) | 361 | if (!acquire) { |
362 | nvgpu_semaphore_prepare(s, c->hw_sema); | 362 | nvgpu_semaphore_prepare(s, c->hw_sema); |
363 | } | ||
363 | 364 | ||
364 | g->ops.fifo.add_sema_cmd(g, s, va, cmd, off, acquire, wfi); | 365 | g->ops.fifo.add_sema_cmd(g, s, va, cmd, off, acquire, wfi); |
365 | 366 | ||
366 | if (acquire) | 367 | if (acquire) { |
367 | gpu_sema_verbose_dbg(g, "(A) c=%d ACQ_GE %-4u pool=%-3d" | 368 | gpu_sema_verbose_dbg(g, "(A) c=%d ACQ_GE %-4u pool=%-3d" |
368 | "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", | 369 | "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", |
369 | ch, nvgpu_semaphore_get_value(s), | 370 | ch, nvgpu_semaphore_get_value(s), |
370 | s->location.pool->page_idx, va, cmd->gva, | 371 | s->location.pool->page_idx, va, cmd->gva, |
371 | cmd->mem->gpu_va, ob); | 372 | cmd->mem->gpu_va, ob); |
372 | else | 373 | } else { |
373 | gpu_sema_verbose_dbg(g, "(R) c=%d INCR %u (%u) pool=%-3d" | 374 | gpu_sema_verbose_dbg(g, "(R) c=%d INCR %u (%u) pool=%-3d" |
374 | "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", | 375 | "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", |
375 | ch, nvgpu_semaphore_get_value(s), | 376 | ch, nvgpu_semaphore_get_value(s), |
376 | nvgpu_semaphore_read(s), | 377 | nvgpu_semaphore_read(s), |
377 | s->location.pool->page_idx, | 378 | s->location.pool->page_idx, |
378 | va, cmd->gva, cmd->mem->gpu_va, ob); | 379 | va, cmd->gva, cmd->mem->gpu_va, ob); |
380 | } | ||
379 | } | 381 | } |
380 | 382 | ||
381 | void gk20a_channel_gen_sema_wait_cmd(struct channel_gk20a *c, | 383 | void gk20a_channel_gen_sema_wait_cmd(struct channel_gk20a *c, |
@@ -418,8 +420,9 @@ static int gk20a_channel_semaphore_wait_fd( | |||
418 | int err; | 420 | int err; |
419 | 421 | ||
420 | err = nvgpu_os_fence_fdget(&os_fence, c, fd); | 422 | err = nvgpu_os_fence_fdget(&os_fence, c, fd); |
421 | if (err) | 423 | if (err) { |
422 | return err; | 424 | return err; |
425 | } | ||
423 | 426 | ||
424 | err = os_fence.ops->program_waits(&os_fence, | 427 | err = os_fence.ops->program_waits(&os_fence, |
425 | entry, c, max_wait_cmds); | 428 | entry, c, max_wait_cmds); |
@@ -465,8 +468,9 @@ static int __gk20a_channel_semaphore_incr( | |||
465 | err = nvgpu_os_fence_sema_create(&os_fence, c, | 468 | err = nvgpu_os_fence_sema_create(&os_fence, c, |
466 | semaphore); | 469 | semaphore); |
467 | 470 | ||
468 | if (err) | 471 | if (err) { |
469 | goto clean_up_sema; | 472 | goto clean_up_sema; |
473 | } | ||
470 | } | 474 | } |
471 | 475 | ||
472 | err = gk20a_fence_from_semaphore(fence, | 476 | err = gk20a_fence_from_semaphore(fence, |
@@ -475,8 +479,9 @@ static int __gk20a_channel_semaphore_incr( | |||
475 | os_fence); | 479 | os_fence); |
476 | 480 | ||
477 | if (err) { | 481 | if (err) { |
478 | if (nvgpu_os_fence_is_initialized(&os_fence)) | 482 | if (nvgpu_os_fence_is_initialized(&os_fence)) { |
479 | os_fence.ops->drop_ref(&os_fence); | 483 | os_fence.ops->drop_ref(&os_fence); |
484 | } | ||
480 | goto clean_up_sema; | 485 | goto clean_up_sema; |
481 | } | 486 | } |
482 | 487 | ||
@@ -535,13 +540,15 @@ static void gk20a_channel_semaphore_set_min_eq_max(struct gk20a_channel_sync *s) | |||
535 | struct channel_gk20a *c = sp->c; | 540 | struct channel_gk20a *c = sp->c; |
536 | bool updated; | 541 | bool updated; |
537 | 542 | ||
538 | if (!c->hw_sema) | 543 | if (!c->hw_sema) { |
539 | return; | 544 | return; |
545 | } | ||
540 | 546 | ||
541 | updated = nvgpu_semaphore_reset(c->hw_sema); | 547 | updated = nvgpu_semaphore_reset(c->hw_sema); |
542 | 548 | ||
543 | if (updated) | 549 | if (updated) { |
544 | nvgpu_cond_broadcast_interruptible(&c->semaphore_wq); | 550 | nvgpu_cond_broadcast_interruptible(&c->semaphore_wq); |
551 | } | ||
545 | } | 552 | } |
546 | 553 | ||
547 | static void gk20a_channel_semaphore_set_safe_state(struct gk20a_channel_sync *s) | 554 | static void gk20a_channel_semaphore_set_safe_state(struct gk20a_channel_sync *s) |
@@ -568,8 +575,9 @@ static void gk20a_channel_semaphore_destroy(struct gk20a_channel_sync *s) | |||
568 | struct gk20a *g = c->g; | 575 | struct gk20a *g = c->g; |
569 | 576 | ||
570 | if (c->has_os_fence_framework_support && | 577 | if (c->has_os_fence_framework_support && |
571 | g->os_channel.os_fence_framework_inst_exists(c)) | 578 | g->os_channel.os_fence_framework_inst_exists(c)) { |
572 | g->os_channel.destroy_os_fence_framework(c); | 579 | g->os_channel.destroy_os_fence_framework(c); |
580 | } | ||
573 | 581 | ||
574 | /* The sema pool is cleaned up by the VM destroy. */ | 582 | /* The sema pool is cleaned up by the VM destroy. */ |
575 | sema->pool = NULL; | 583 | sema->pool = NULL; |
@@ -586,19 +594,22 @@ gk20a_channel_semaphore_create(struct channel_gk20a *c, bool user_managed) | |||
586 | int asid = -1; | 594 | int asid = -1; |
587 | int err; | 595 | int err; |
588 | 596 | ||
589 | if (WARN_ON(!c->vm)) | 597 | if (WARN_ON(!c->vm)) { |
590 | return NULL; | 598 | return NULL; |
599 | } | ||
591 | 600 | ||
592 | sema = nvgpu_kzalloc(c->g, sizeof(*sema)); | 601 | sema = nvgpu_kzalloc(c->g, sizeof(*sema)); |
593 | if (!sema) | 602 | if (!sema) { |
594 | return NULL; | 603 | return NULL; |
604 | } | ||
595 | sema->c = c; | 605 | sema->c = c; |
596 | 606 | ||
597 | sprintf(pool_name, "semaphore_pool-%d", c->chid); | 607 | sprintf(pool_name, "semaphore_pool-%d", c->chid); |
598 | sema->pool = c->vm->sema_pool; | 608 | sema->pool = c->vm->sema_pool; |
599 | 609 | ||
600 | if (c->vm->as_share) | 610 | if (c->vm->as_share) { |
601 | asid = c->vm->as_share->id; | 611 | asid = c->vm->as_share->id; |
612 | } | ||
602 | 613 | ||
603 | if (c->has_os_fence_framework_support) { | 614 | if (c->has_os_fence_framework_support) { |
604 | /*Init the sync_timeline for this channel */ | 615 | /*Init the sync_timeline for this channel */ |
@@ -628,8 +639,9 @@ gk20a_channel_semaphore_create(struct channel_gk20a *c, bool user_managed) | |||
628 | void gk20a_channel_sync_destroy(struct gk20a_channel_sync *sync, | 639 | void gk20a_channel_sync_destroy(struct gk20a_channel_sync *sync, |
629 | bool set_safe_state) | 640 | bool set_safe_state) |
630 | { | 641 | { |
631 | if (set_safe_state) | 642 | if (set_safe_state) { |
632 | sync->set_safe_state(sync); | 643 | sync->set_safe_state(sync); |
644 | } | ||
633 | sync->destroy(sync); | 645 | sync->destroy(sync); |
634 | } | 646 | } |
635 | 647 | ||
diff --git a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c index 3be08865..e7f95245 100644 --- a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c | |||
@@ -229,21 +229,25 @@ int dbg_set_powergate(struct dbg_session_gk20a *dbg_s, bool disable_powergate) | |||
229 | nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, | 229 | nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, |
230 | "module busy"); | 230 | "module busy"); |
231 | err = gk20a_busy(g); | 231 | err = gk20a_busy(g); |
232 | if (err) | 232 | if (err) { |
233 | return err; | 233 | return err; |
234 | } | ||
234 | 235 | ||
235 | /*do elpg disable before clock gating */ | 236 | /*do elpg disable before clock gating */ |
236 | nvgpu_pmu_pg_global_enable(g, false); | 237 | nvgpu_pmu_pg_global_enable(g, false); |
237 | 238 | ||
238 | if (g->ops.clock_gating.slcg_gr_load_gating_prod) | 239 | if (g->ops.clock_gating.slcg_gr_load_gating_prod) { |
239 | g->ops.clock_gating.slcg_gr_load_gating_prod(g, | 240 | g->ops.clock_gating.slcg_gr_load_gating_prod(g, |
240 | false); | 241 | false); |
241 | if (g->ops.clock_gating.slcg_perf_load_gating_prod) | 242 | } |
243 | if (g->ops.clock_gating.slcg_perf_load_gating_prod) { | ||
242 | g->ops.clock_gating.slcg_perf_load_gating_prod(g, | 244 | g->ops.clock_gating.slcg_perf_load_gating_prod(g, |
243 | false); | 245 | false); |
244 | if (g->ops.clock_gating.slcg_ltc_load_gating_prod) | 246 | } |
247 | if (g->ops.clock_gating.slcg_ltc_load_gating_prod) { | ||
245 | g->ops.clock_gating.slcg_ltc_load_gating_prod(g, | 248 | g->ops.clock_gating.slcg_ltc_load_gating_prod(g, |
246 | false); | 249 | false); |
250 | } | ||
247 | 251 | ||
248 | gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_RUN); | 252 | gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_RUN); |
249 | gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); | 253 | gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); |
@@ -254,28 +258,27 @@ int dbg_set_powergate(struct dbg_session_gk20a *dbg_s, bool disable_powergate) | |||
254 | /* release pending exceptions to fault/be handled as usual */ | 258 | /* release pending exceptions to fault/be handled as usual */ |
255 | /*TBD: ordering of these? */ | 259 | /*TBD: ordering of these? */ |
256 | 260 | ||
257 | if (g->elcg_enabled) | 261 | if (g->elcg_enabled) { |
258 | gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_AUTO); | 262 | gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_AUTO); |
263 | } | ||
259 | 264 | ||
260 | if (g->blcg_enabled) | 265 | if (g->blcg_enabled) { |
261 | gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_AUTO); | 266 | gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_AUTO); |
267 | } | ||
262 | 268 | ||
263 | if (g->slcg_enabled) { | 269 | if (g->slcg_enabled) { |
264 | if (g->ops.clock_gating. | 270 | if (g->ops.clock_gating.slcg_ltc_load_gating_prod) { |
265 | slcg_ltc_load_gating_prod) | 271 | g->ops.clock_gating.slcg_ltc_load_gating_prod(g, |
266 | g->ops.clock_gating. | 272 | g->slcg_enabled); |
267 | slcg_ltc_load_gating_prod(g, | 273 | } |
268 | g->slcg_enabled); | 274 | if (g->ops.clock_gating.slcg_perf_load_gating_prod) { |
269 | if (g->ops.clock_gating. | 275 | g->ops.clock_gating.slcg_perf_load_gating_prod(g, |
270 | slcg_perf_load_gating_prod) | 276 | g->slcg_enabled); |
271 | g->ops.clock_gating. | 277 | } |
272 | slcg_perf_load_gating_prod(g, | 278 | if (g->ops.clock_gating.slcg_gr_load_gating_prod) { |
273 | g->slcg_enabled); | 279 | g->ops.clock_gating.slcg_gr_load_gating_prod(g, |
274 | if (g->ops.clock_gating. | 280 | g->slcg_enabled); |
275 | slcg_gr_load_gating_prod) | 281 | } |
276 | g->ops.clock_gating. | ||
277 | slcg_gr_load_gating_prod(g, | ||
278 | g->slcg_enabled); | ||
279 | } | 282 | } |
280 | nvgpu_pmu_pg_global_enable(g, true); | 283 | nvgpu_pmu_pg_global_enable(g, true); |
281 | 284 | ||
@@ -328,12 +331,14 @@ void nvgpu_release_profiler_reservation(struct dbg_session_gk20a *dbg_s, | |||
328 | struct gk20a *g = dbg_s->g; | 331 | struct gk20a *g = dbg_s->g; |
329 | 332 | ||
330 | g->profiler_reservation_count--; | 333 | g->profiler_reservation_count--; |
331 | if (g->profiler_reservation_count < 0) | 334 | if (g->profiler_reservation_count < 0) { |
332 | nvgpu_err(g, "Negative reservation count!"); | 335 | nvgpu_err(g, "Negative reservation count!"); |
336 | } | ||
333 | dbg_s->has_profiler_reservation = false; | 337 | dbg_s->has_profiler_reservation = false; |
334 | prof_obj->has_reservation = false; | 338 | prof_obj->has_reservation = false; |
335 | if (prof_obj->ch == NULL) | 339 | if (prof_obj->ch == NULL) { |
336 | g->global_profiler_reservation_held = false; | 340 | g->global_profiler_reservation_held = false; |
341 | } | ||
337 | } | 342 | } |
338 | 343 | ||
339 | int gk20a_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size) | 344 | int gk20a_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size) |
@@ -351,8 +356,9 @@ int gk20a_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size) | |||
351 | } | 356 | } |
352 | 357 | ||
353 | err = g->ops.mm.alloc_inst_block(g, &mm->perfbuf.inst_block); | 358 | err = g->ops.mm.alloc_inst_block(g, &mm->perfbuf.inst_block); |
354 | if (err) | 359 | if (err) { |
355 | return err; | 360 | return err; |
361 | } | ||
356 | 362 | ||
357 | g->ops.mm.init_inst_block(&mm->perfbuf.inst_block, mm->perfbuf.vm, 0); | 363 | g->ops.mm.init_inst_block(&mm->perfbuf.inst_block, mm->perfbuf.vm, 0); |
358 | 364 | ||
diff --git a/drivers/gpu/nvgpu/gk20a/fence_gk20a.c b/drivers/gpu/nvgpu/gk20a/fence_gk20a.c index 8f585afd..0df73278 100644 --- a/drivers/gpu/nvgpu/gk20a/fence_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fence_gk20a.c | |||
@@ -44,29 +44,35 @@ static void gk20a_fence_free(struct nvgpu_ref *ref) | |||
44 | container_of(ref, struct gk20a_fence, ref); | 44 | container_of(ref, struct gk20a_fence, ref); |
45 | struct gk20a *g = f->g; | 45 | struct gk20a *g = f->g; |
46 | 46 | ||
47 | if (nvgpu_os_fence_is_initialized(&f->os_fence)) | 47 | if (nvgpu_os_fence_is_initialized(&f->os_fence)) { |
48 | f->os_fence.ops->drop_ref(&f->os_fence); | 48 | f->os_fence.ops->drop_ref(&f->os_fence); |
49 | } | ||
49 | 50 | ||
50 | if (f->semaphore) | 51 | if (f->semaphore) { |
51 | nvgpu_semaphore_put(f->semaphore); | 52 | nvgpu_semaphore_put(f->semaphore); |
53 | } | ||
52 | 54 | ||
53 | if (f->allocator) { | 55 | if (f->allocator) { |
54 | if (nvgpu_alloc_initialized(f->allocator)) | 56 | if (nvgpu_alloc_initialized(f->allocator)) { |
55 | nvgpu_free(f->allocator, (u64)(uintptr_t)f); | 57 | nvgpu_free(f->allocator, (u64)(uintptr_t)f); |
56 | } else | 58 | } |
59 | } else { | ||
57 | nvgpu_kfree(g, f); | 60 | nvgpu_kfree(g, f); |
61 | } | ||
58 | } | 62 | } |
59 | 63 | ||
60 | void gk20a_fence_put(struct gk20a_fence *f) | 64 | void gk20a_fence_put(struct gk20a_fence *f) |
61 | { | 65 | { |
62 | if (f) | 66 | if (f) { |
63 | nvgpu_ref_put(&f->ref, gk20a_fence_free); | 67 | nvgpu_ref_put(&f->ref, gk20a_fence_free); |
68 | } | ||
64 | } | 69 | } |
65 | 70 | ||
66 | struct gk20a_fence *gk20a_fence_get(struct gk20a_fence *f) | 71 | struct gk20a_fence *gk20a_fence_get(struct gk20a_fence *f) |
67 | { | 72 | { |
68 | if (f) | 73 | if (f) { |
69 | nvgpu_ref_get(&f->ref); | 74 | nvgpu_ref_get(&f->ref); |
75 | } | ||
70 | return f; | 76 | return f; |
71 | } | 77 | } |
72 | 78 | ||
@@ -81,8 +87,9 @@ inline bool gk20a_fence_is_valid(struct gk20a_fence *f) | |||
81 | int gk20a_fence_install_fd(struct gk20a_fence *f, int fd) | 87 | int gk20a_fence_install_fd(struct gk20a_fence *f, int fd) |
82 | { | 88 | { |
83 | if (!f || !gk20a_fence_is_valid(f) || | 89 | if (!f || !gk20a_fence_is_valid(f) || |
84 | !nvgpu_os_fence_is_initialized(&f->os_fence)) | 90 | !nvgpu_os_fence_is_initialized(&f->os_fence)) { |
85 | return -EINVAL; | 91 | return -EINVAL; |
92 | } | ||
86 | 93 | ||
87 | f->os_fence.ops->install_fence(&f->os_fence, fd); | 94 | f->os_fence.ops->install_fence(&f->os_fence, fd); |
88 | 95 | ||
@@ -93,8 +100,9 @@ int gk20a_fence_wait(struct gk20a *g, struct gk20a_fence *f, | |||
93 | unsigned long timeout) | 100 | unsigned long timeout) |
94 | { | 101 | { |
95 | if (f && gk20a_fence_is_valid(f)) { | 102 | if (f && gk20a_fence_is_valid(f)) { |
96 | if (!nvgpu_platform_is_silicon(g)) | 103 | if (!nvgpu_platform_is_silicon(g)) { |
97 | timeout = MAX_SCHEDULE_TIMEOUT; | 104 | timeout = MAX_SCHEDULE_TIMEOUT; |
105 | } | ||
98 | return f->ops->wait(f, timeout); | 106 | return f->ops->wait(f, timeout); |
99 | } | 107 | } |
100 | return 0; | 108 | return 0; |
@@ -102,10 +110,11 @@ int gk20a_fence_wait(struct gk20a *g, struct gk20a_fence *f, | |||
102 | 110 | ||
103 | bool gk20a_fence_is_expired(struct gk20a_fence *f) | 111 | bool gk20a_fence_is_expired(struct gk20a_fence *f) |
104 | { | 112 | { |
105 | if (f && gk20a_fence_is_valid(f) && f->ops) | 113 | if (f && gk20a_fence_is_valid(f) && f->ops) { |
106 | return f->ops->is_expired(f); | 114 | return f->ops->is_expired(f); |
107 | else | 115 | } else { |
108 | return true; | 116 | return true; |
117 | } | ||
109 | } | 118 | } |
110 | 119 | ||
111 | int gk20a_alloc_fence_pool(struct channel_gk20a *c, unsigned int count) | 120 | int gk20a_alloc_fence_pool(struct channel_gk20a *c, unsigned int count) |
@@ -120,14 +129,16 @@ int gk20a_alloc_fence_pool(struct channel_gk20a *c, unsigned int count) | |||
120 | fence_pool = nvgpu_vzalloc(c->g, size); | 129 | fence_pool = nvgpu_vzalloc(c->g, size); |
121 | } | 130 | } |
122 | 131 | ||
123 | if (!fence_pool) | 132 | if (!fence_pool) { |
124 | return -ENOMEM; | 133 | return -ENOMEM; |
134 | } | ||
125 | 135 | ||
126 | err = nvgpu_lockless_allocator_init(c->g, &c->fence_allocator, | 136 | err = nvgpu_lockless_allocator_init(c->g, &c->fence_allocator, |
127 | "fence_pool", (size_t)fence_pool, size, | 137 | "fence_pool", (size_t)fence_pool, size, |
128 | sizeof(struct gk20a_fence), 0); | 138 | sizeof(struct gk20a_fence), 0); |
129 | if (err) | 139 | if (err) { |
130 | goto fail; | 140 | goto fail; |
141 | } | ||
131 | 142 | ||
132 | return 0; | 143 | return 0; |
133 | 144 | ||
@@ -163,8 +174,9 @@ struct gk20a_fence *gk20a_alloc_fence(struct channel_gk20a *c) | |||
163 | fence->allocator = &c->fence_allocator; | 174 | fence->allocator = &c->fence_allocator; |
164 | } | 175 | } |
165 | } | 176 | } |
166 | } else | 177 | } else { |
167 | fence = nvgpu_kzalloc(c->g, sizeof(struct gk20a_fence)); | 178 | fence = nvgpu_kzalloc(c->g, sizeof(struct gk20a_fence)); |
179 | } | ||
168 | 180 | ||
169 | if (fence) { | 181 | if (fence) { |
170 | nvgpu_ref_init(&fence->ref); | 182 | nvgpu_ref_init(&fence->ref); |
@@ -178,8 +190,9 @@ void gk20a_init_fence(struct gk20a_fence *f, | |||
178 | const struct gk20a_fence_ops *ops, | 190 | const struct gk20a_fence_ops *ops, |
179 | struct nvgpu_os_fence os_fence) | 191 | struct nvgpu_os_fence os_fence) |
180 | { | 192 | { |
181 | if (!f) | 193 | if (!f) { |
182 | return; | 194 | return; |
195 | } | ||
183 | f->ops = ops; | 196 | f->ops = ops; |
184 | f->syncpt_id = -1; | 197 | f->syncpt_id = -1; |
185 | f->semaphore = NULL; | 198 | f->semaphore = NULL; |
@@ -190,8 +203,9 @@ void gk20a_init_fence(struct gk20a_fence *f, | |||
190 | 203 | ||
191 | static int nvgpu_semaphore_fence_wait(struct gk20a_fence *f, long timeout) | 204 | static int nvgpu_semaphore_fence_wait(struct gk20a_fence *f, long timeout) |
192 | { | 205 | { |
193 | if (!nvgpu_semaphore_is_acquired(f->semaphore)) | 206 | if (!nvgpu_semaphore_is_acquired(f->semaphore)) { |
194 | return 0; | 207 | return 0; |
208 | } | ||
195 | 209 | ||
196 | return NVGPU_COND_WAIT_INTERRUPTIBLE( | 210 | return NVGPU_COND_WAIT_INTERRUPTIBLE( |
197 | f->semaphore_wq, | 211 | f->semaphore_wq, |
@@ -219,8 +233,9 @@ int gk20a_fence_from_semaphore( | |||
219 | struct gk20a_fence *f = fence_out; | 233 | struct gk20a_fence *f = fence_out; |
220 | 234 | ||
221 | gk20a_init_fence(f, &nvgpu_semaphore_fence_ops, os_fence); | 235 | gk20a_init_fence(f, &nvgpu_semaphore_fence_ops, os_fence); |
222 | if (!f) | 236 | if (!f) { |
223 | return -EINVAL; | 237 | return -EINVAL; |
238 | } | ||
224 | 239 | ||
225 | 240 | ||
226 | f->semaphore = semaphore; | 241 | f->semaphore = semaphore; |
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c index e03c5da8..f06bf1c5 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | |||
@@ -111,8 +111,9 @@ struct fifo_engine_info_gk20a *gk20a_fifo_get_engine_info(struct gk20a *g, u32 e | |||
111 | u32 engine_id_idx; | 111 | u32 engine_id_idx; |
112 | struct fifo_engine_info_gk20a *info = NULL; | 112 | struct fifo_engine_info_gk20a *info = NULL; |
113 | 113 | ||
114 | if (!g) | 114 | if (!g) { |
115 | return info; | 115 | return info; |
116 | } | ||
116 | 117 | ||
117 | f = &g->fifo; | 118 | f = &g->fifo; |
118 | 119 | ||
@@ -125,8 +126,9 @@ struct fifo_engine_info_gk20a *gk20a_fifo_get_engine_info(struct gk20a *g, u32 e | |||
125 | } | 126 | } |
126 | } | 127 | } |
127 | 128 | ||
128 | if (!info) | 129 | if (!info) { |
129 | nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id); | 130 | nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id); |
131 | } | ||
130 | 132 | ||
131 | return info; | 133 | return info; |
132 | } | 134 | } |
@@ -137,8 +139,9 @@ bool gk20a_fifo_is_valid_engine_id(struct gk20a *g, u32 engine_id) | |||
137 | u32 engine_id_idx; | 139 | u32 engine_id_idx; |
138 | bool valid = false; | 140 | bool valid = false; |
139 | 141 | ||
140 | if (!g) | 142 | if (!g) { |
141 | return valid; | 143 | return valid; |
144 | } | ||
142 | 145 | ||
143 | f = &g->fifo; | 146 | f = &g->fifo; |
144 | 147 | ||
@@ -151,8 +154,9 @@ bool gk20a_fifo_is_valid_engine_id(struct gk20a *g, u32 engine_id) | |||
151 | } | 154 | } |
152 | } | 155 | } |
153 | 156 | ||
154 | if (!valid) | 157 | if (!valid) { |
155 | nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id); | 158 | nvgpu_err(g, "engine_id is not in active list/invalid %d", engine_id); |
159 | } | ||
156 | 160 | ||
157 | return valid; | 161 | return valid; |
158 | } | 162 | } |
@@ -182,8 +186,9 @@ u32 gk20a_fifo_get_all_ce_engine_reset_mask(struct gk20a *g) | |||
182 | struct fifo_engine_info_gk20a *engine_info; | 186 | struct fifo_engine_info_gk20a *engine_info; |
183 | u32 active_engine_id = 0; | 187 | u32 active_engine_id = 0; |
184 | 188 | ||
185 | if (!g) | 189 | if (!g) { |
186 | return reset_mask; | 190 | return reset_mask; |
191 | } | ||
187 | 192 | ||
188 | f = &g->fifo; | 193 | f = &g->fifo; |
189 | 194 | ||
@@ -193,8 +198,9 @@ u32 gk20a_fifo_get_all_ce_engine_reset_mask(struct gk20a *g) | |||
193 | engine_enum = engine_info->engine_enum; | 198 | engine_enum = engine_info->engine_enum; |
194 | 199 | ||
195 | if ((engine_enum == ENGINE_GRCE_GK20A) || | 200 | if ((engine_enum == ENGINE_GRCE_GK20A) || |
196 | (engine_enum == ENGINE_ASYNC_CE_GK20A)) | 201 | (engine_enum == ENGINE_ASYNC_CE_GK20A)) { |
197 | reset_mask |= engine_info->reset_mask; | 202 | reset_mask |= engine_info->reset_mask; |
203 | } | ||
198 | } | 204 | } |
199 | 205 | ||
200 | return reset_mask; | 206 | return reset_mask; |
@@ -209,8 +215,9 @@ u32 gk20a_fifo_get_fast_ce_runlist_id(struct gk20a *g) | |||
209 | struct fifo_engine_info_gk20a *engine_info; | 215 | struct fifo_engine_info_gk20a *engine_info; |
210 | u32 active_engine_id = 0; | 216 | u32 active_engine_id = 0; |
211 | 217 | ||
212 | if (!g) | 218 | if (!g) { |
213 | return ce_runlist_id; | 219 | return ce_runlist_id; |
220 | } | ||
214 | 221 | ||
215 | f = &g->fifo; | 222 | f = &g->fifo; |
216 | 223 | ||
@@ -220,8 +227,9 @@ u32 gk20a_fifo_get_fast_ce_runlist_id(struct gk20a *g) | |||
220 | engine_enum = engine_info->engine_enum; | 227 | engine_enum = engine_info->engine_enum; |
221 | 228 | ||
222 | /* selecet last available ASYNC_CE if available */ | 229 | /* selecet last available ASYNC_CE if available */ |
223 | if (engine_enum == ENGINE_ASYNC_CE_GK20A) | 230 | if (engine_enum == ENGINE_ASYNC_CE_GK20A) { |
224 | ce_runlist_id = engine_info->runlist_id; | 231 | ce_runlist_id = engine_info->runlist_id; |
232 | } | ||
225 | } | 233 | } |
226 | 234 | ||
227 | return ce_runlist_id; | 235 | return ce_runlist_id; |
@@ -264,8 +272,9 @@ bool gk20a_fifo_is_valid_runlist_id(struct gk20a *g, u32 runlist_id) | |||
264 | u32 active_engine_id; | 272 | u32 active_engine_id; |
265 | struct fifo_engine_info_gk20a *engine_info; | 273 | struct fifo_engine_info_gk20a *engine_info; |
266 | 274 | ||
267 | if (!g) | 275 | if (!g) { |
268 | return false; | 276 | return false; |
277 | } | ||
269 | 278 | ||
270 | f = &g->fifo; | 279 | f = &g->fifo; |
271 | 280 | ||
@@ -310,8 +319,9 @@ static inline u32 gk20a_mmu_id_to_engine_id(struct gk20a *g, u32 fault_id) | |||
310 | active_engine_id = f->active_engines_list[engine_id]; | 319 | active_engine_id = f->active_engines_list[engine_id]; |
311 | engine_info = &g->fifo.engine_info[active_engine_id]; | 320 | engine_info = &g->fifo.engine_info[active_engine_id]; |
312 | 321 | ||
313 | if (engine_info->fault_id == fault_id) | 322 | if (engine_info->fault_id == fault_id) { |
314 | break; | 323 | break; |
324 | } | ||
315 | active_engine_id = FIFO_INVAL_ENGINE_ID; | 325 | active_engine_id = FIFO_INVAL_ENGINE_ID; |
316 | } | 326 | } |
317 | return active_engine_id; | 327 | return active_engine_id; |
@@ -323,17 +333,18 @@ int gk20a_fifo_engine_enum_from_type(struct gk20a *g, u32 engine_type, | |||
323 | int ret = ENGINE_INVAL_GK20A; | 333 | int ret = ENGINE_INVAL_GK20A; |
324 | 334 | ||
325 | nvgpu_log_info(g, "engine type %d", engine_type); | 335 | nvgpu_log_info(g, "engine type %d", engine_type); |
326 | if (engine_type == top_device_info_type_enum_graphics_v()) | 336 | if (engine_type == top_device_info_type_enum_graphics_v()) { |
327 | ret = ENGINE_GR_GK20A; | 337 | ret = ENGINE_GR_GK20A; |
328 | else if ((engine_type >= top_device_info_type_enum_copy0_v()) && | 338 | } else if ((engine_type >= top_device_info_type_enum_copy0_v()) && |
329 | (engine_type <= top_device_info_type_enum_copy2_v())) { | 339 | (engine_type <= top_device_info_type_enum_copy2_v())) { |
330 | /* Lets consider all the CE engine have separate runlist at this point | 340 | /* Lets consider all the CE engine have separate runlist at this point |
331 | * We can identify the ENGINE_GRCE_GK20A type CE using runlist_id | 341 | * We can identify the ENGINE_GRCE_GK20A type CE using runlist_id |
332 | * comparsion logic with GR runlist_id in init_engine_info() */ | 342 | * comparsion logic with GR runlist_id in init_engine_info() */ |
333 | ret = ENGINE_ASYNC_CE_GK20A; | 343 | ret = ENGINE_ASYNC_CE_GK20A; |
334 | /* inst_id starts from CE0 to CE2 */ | 344 | /* inst_id starts from CE0 to CE2 */ |
335 | if (inst_id) | 345 | if (inst_id) { |
336 | *inst_id = (engine_type - top_device_info_type_enum_copy0_v()); | 346 | *inst_id = (engine_type - top_device_info_type_enum_copy0_v()); |
347 | } | ||
337 | } | 348 | } |
338 | 349 | ||
339 | return ret; | 350 | return ret; |
@@ -421,10 +432,11 @@ int gk20a_fifo_init_engine_info(struct fifo_gk20a *f) | |||
421 | engine_type, &inst_id); | 432 | engine_type, &inst_id); |
422 | } else if (entry == top_device_info_entry_data_v()) { | 433 | } else if (entry == top_device_info_entry_data_v()) { |
423 | /* gk20a doesn't support device_info_data packet parsing */ | 434 | /* gk20a doesn't support device_info_data packet parsing */ |
424 | if (g->ops.fifo.device_info_data_parse) | 435 | if (g->ops.fifo.device_info_data_parse) { |
425 | g->ops.fifo.device_info_data_parse(g, | 436 | g->ops.fifo.device_info_data_parse(g, |
426 | table_entry, &inst_id, &pri_base, | 437 | table_entry, &inst_id, &pri_base, |
427 | &fault_id); | 438 | &fault_id); |
439 | } | ||
428 | } | 440 | } |
429 | 441 | ||
430 | if (!top_device_info_chain_v(table_entry)) { | 442 | if (!top_device_info_chain_v(table_entry)) { |
@@ -439,18 +451,21 @@ int gk20a_fifo_init_engine_info(struct fifo_gk20a *f) | |||
439 | info->inst_id = inst_id; | 451 | info->inst_id = inst_id; |
440 | info->pri_base = pri_base; | 452 | info->pri_base = pri_base; |
441 | 453 | ||
442 | if (engine_enum == ENGINE_GR_GK20A) | 454 | if (engine_enum == ENGINE_GR_GK20A) { |
443 | gr_runlist_id = runlist_id; | 455 | gr_runlist_id = runlist_id; |
456 | } | ||
444 | 457 | ||
445 | /* GR and GR_COPY shares same runlist_id */ | 458 | /* GR and GR_COPY shares same runlist_id */ |
446 | if ((engine_enum == ENGINE_ASYNC_CE_GK20A) && | 459 | if ((engine_enum == ENGINE_ASYNC_CE_GK20A) && |
447 | (gr_runlist_id == runlist_id)) | 460 | (gr_runlist_id == runlist_id)) { |
448 | engine_enum = ENGINE_GRCE_GK20A; | 461 | engine_enum = ENGINE_GRCE_GK20A; |
462 | } | ||
449 | 463 | ||
450 | info->engine_enum = engine_enum; | 464 | info->engine_enum = engine_enum; |
451 | 465 | ||
452 | if (!fault_id && (engine_enum == ENGINE_GRCE_GK20A)) | 466 | if (!fault_id && (engine_enum == ENGINE_GRCE_GK20A)) { |
453 | fault_id = 0x1b; | 467 | fault_id = 0x1b; |
468 | } | ||
454 | info->fault_id = fault_id; | 469 | info->fault_id = fault_id; |
455 | 470 | ||
456 | /* engine_id starts from 0 to NV_HOST_NUM_ENGINES */ | 471 | /* engine_id starts from 0 to NV_HOST_NUM_ENGINES */ |
@@ -471,8 +486,9 @@ u32 gk20a_fifo_act_eng_interrupt_mask(struct gk20a *g, u32 act_eng_id) | |||
471 | struct fifo_engine_info_gk20a *engine_info = NULL; | 486 | struct fifo_engine_info_gk20a *engine_info = NULL; |
472 | 487 | ||
473 | engine_info = gk20a_fifo_get_engine_info(g, act_eng_id); | 488 | engine_info = gk20a_fifo_get_engine_info(g, act_eng_id); |
474 | if (engine_info) | 489 | if (engine_info) { |
475 | return engine_info->intr_mask; | 490 | return engine_info->intr_mask; |
491 | } | ||
476 | 492 | ||
477 | return 0; | 493 | return 0; |
478 | } | 494 | } |
@@ -491,8 +507,9 @@ u32 gk20a_fifo_engine_interrupt_mask(struct gk20a *g) | |||
491 | engine_enum = g->fifo.engine_info[active_engine_id].engine_enum; | 507 | engine_enum = g->fifo.engine_info[active_engine_id].engine_enum; |
492 | if (((engine_enum == ENGINE_GRCE_GK20A) || | 508 | if (((engine_enum == ENGINE_GRCE_GK20A) || |
493 | (engine_enum == ENGINE_ASYNC_CE_GK20A)) && | 509 | (engine_enum == ENGINE_ASYNC_CE_GK20A)) && |
494 | (!g->ops.ce2.isr_stall || !g->ops.ce2.isr_nonstall)) | 510 | (!g->ops.ce2.isr_stall || !g->ops.ce2.isr_nonstall)) { |
495 | continue; | 511 | continue; |
512 | } | ||
496 | 513 | ||
497 | eng_intr_mask |= intr_mask; | 514 | eng_intr_mask |= intr_mask; |
498 | } | 515 | } |
@@ -507,8 +524,9 @@ void gk20a_fifo_delete_runlist(struct fifo_gk20a *f) | |||
507 | struct fifo_runlist_info_gk20a *runlist; | 524 | struct fifo_runlist_info_gk20a *runlist; |
508 | struct gk20a *g = NULL; | 525 | struct gk20a *g = NULL; |
509 | 526 | ||
510 | if (!f || !f->runlist_info) | 527 | if (!f || !f->runlist_info) { |
511 | return; | 528 | return; |
529 | } | ||
512 | 530 | ||
513 | g = f->g; | 531 | g = f->g; |
514 | 532 | ||
@@ -554,8 +572,9 @@ static void gk20a_remove_fifo_support(struct fifo_gk20a *f) | |||
554 | * Could race but worst that happens is we get an error message | 572 | * Could race but worst that happens is we get an error message |
555 | * from gk20a_free_channel() complaining about multiple closes. | 573 | * from gk20a_free_channel() complaining about multiple closes. |
556 | */ | 574 | */ |
557 | if (c->referenceable) | 575 | if (c->referenceable) { |
558 | __gk20a_channel_kill(c); | 576 | __gk20a_channel_kill(c); |
577 | } | ||
559 | 578 | ||
560 | nvgpu_mutex_destroy(&tsg->event_id_list_lock); | 579 | nvgpu_mutex_destroy(&tsg->event_id_list_lock); |
561 | 580 | ||
@@ -573,10 +592,11 @@ static void gk20a_remove_fifo_support(struct fifo_gk20a *f) | |||
573 | 592 | ||
574 | nvgpu_vfree(g, f->channel); | 593 | nvgpu_vfree(g, f->channel); |
575 | nvgpu_vfree(g, f->tsg); | 594 | nvgpu_vfree(g, f->tsg); |
576 | if (g->ops.mm.is_bar1_supported(g)) | 595 | if (g->ops.mm.is_bar1_supported(g)) { |
577 | nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd); | 596 | nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd); |
578 | else | 597 | } else { |
579 | nvgpu_dma_free(g, &f->userd); | 598 | nvgpu_dma_free(g, &f->userd); |
599 | } | ||
580 | 600 | ||
581 | gk20a_fifo_delete_runlist(f); | 601 | gk20a_fifo_delete_runlist(f); |
582 | 602 | ||
@@ -683,8 +703,9 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f) | |||
683 | f->runlist_info = nvgpu_kzalloc(g, | 703 | f->runlist_info = nvgpu_kzalloc(g, |
684 | sizeof(struct fifo_runlist_info_gk20a) * | 704 | sizeof(struct fifo_runlist_info_gk20a) * |
685 | f->max_runlists); | 705 | f->max_runlists); |
686 | if (!f->runlist_info) | 706 | if (!f->runlist_info) { |
687 | goto clean_up_runlist; | 707 | goto clean_up_runlist; |
708 | } | ||
688 | 709 | ||
689 | memset(f->runlist_info, 0, (sizeof(struct fifo_runlist_info_gk20a) * | 710 | memset(f->runlist_info, 0, (sizeof(struct fifo_runlist_info_gk20a) * |
690 | f->max_runlists)); | 711 | f->max_runlists)); |
@@ -695,14 +716,16 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f) | |||
695 | runlist->active_channels = | 716 | runlist->active_channels = |
696 | nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, | 717 | nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, |
697 | BITS_PER_BYTE)); | 718 | BITS_PER_BYTE)); |
698 | if (!runlist->active_channels) | 719 | if (!runlist->active_channels) { |
699 | goto clean_up_runlist; | 720 | goto clean_up_runlist; |
721 | } | ||
700 | 722 | ||
701 | runlist->active_tsgs = | 723 | runlist->active_tsgs = |
702 | nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, | 724 | nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, |
703 | BITS_PER_BYTE)); | 725 | BITS_PER_BYTE)); |
704 | if (!runlist->active_tsgs) | 726 | if (!runlist->active_tsgs) { |
705 | goto clean_up_runlist; | 727 | goto clean_up_runlist; |
728 | } | ||
706 | 729 | ||
707 | runlist_size = f->runlist_entry_size * f->num_runlist_entries; | 730 | runlist_size = f->runlist_entry_size * f->num_runlist_entries; |
708 | nvgpu_log(g, gpu_dbg_info, | 731 | nvgpu_log(g, gpu_dbg_info, |
@@ -725,8 +748,9 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f) | |||
725 | runlist->cur_buffer = MAX_RUNLIST_BUFFERS; | 748 | runlist->cur_buffer = MAX_RUNLIST_BUFFERS; |
726 | 749 | ||
727 | for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) { | 750 | for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) { |
728 | if (f->pbdma_map[pbdma_id] & BIT(runlist_id)) | 751 | if (f->pbdma_map[pbdma_id] & BIT(runlist_id)) { |
729 | runlist->pbdma_bitmask |= BIT(pbdma_id); | 752 | runlist->pbdma_bitmask |= BIT(pbdma_id); |
753 | } | ||
730 | } | 754 | } |
731 | nvgpu_log(g, gpu_dbg_info, "runlist %d : pbdma bitmask 0x%x", | 755 | nvgpu_log(g, gpu_dbg_info, "runlist %d : pbdma bitmask 0x%x", |
732 | runlist_id, runlist->pbdma_bitmask); | 756 | runlist_id, runlist->pbdma_bitmask); |
@@ -735,8 +759,9 @@ static int init_runlist(struct gk20a *g, struct fifo_gk20a *f) | |||
735 | active_engine_id = f->active_engines_list[engine_id]; | 759 | active_engine_id = f->active_engines_list[engine_id]; |
736 | engine_info = &f->engine_info[active_engine_id]; | 760 | engine_info = &f->engine_info[active_engine_id]; |
737 | 761 | ||
738 | if (engine_info && engine_info->runlist_id == runlist_id) | 762 | if (engine_info && engine_info->runlist_id == runlist_id) { |
739 | runlist->eng_bitmask |= BIT(active_engine_id); | 763 | runlist->eng_bitmask |= BIT(active_engine_id); |
764 | } | ||
740 | } | 765 | } |
741 | nvgpu_log(g, gpu_dbg_info, "runlist %d : act eng bitmask 0x%x", | 766 | nvgpu_log(g, gpu_dbg_info, "runlist %d : act eng bitmask 0x%x", |
742 | runlist_id, runlist->eng_bitmask); | 767 | runlist_id, runlist->eng_bitmask); |
@@ -791,12 +816,14 @@ int gk20a_init_fifo_reset_enable_hw(struct gk20a *g) | |||
791 | /* enable pmc pfifo */ | 816 | /* enable pmc pfifo */ |
792 | g->ops.mc.reset(g, mc_enable_pfifo_enabled_f()); | 817 | g->ops.mc.reset(g, mc_enable_pfifo_enabled_f()); |
793 | 818 | ||
794 | if (g->ops.clock_gating.slcg_fifo_load_gating_prod) | 819 | if (g->ops.clock_gating.slcg_fifo_load_gating_prod) { |
795 | g->ops.clock_gating.slcg_fifo_load_gating_prod(g, | 820 | g->ops.clock_gating.slcg_fifo_load_gating_prod(g, |
796 | g->slcg_enabled); | 821 | g->slcg_enabled); |
797 | if (g->ops.clock_gating.blcg_fifo_load_gating_prod) | 822 | } |
823 | if (g->ops.clock_gating.blcg_fifo_load_gating_prod) { | ||
798 | g->ops.clock_gating.blcg_fifo_load_gating_prod(g, | 824 | g->ops.clock_gating.blcg_fifo_load_gating_prod(g, |
799 | g->blcg_enabled); | 825 | g->blcg_enabled); |
826 | } | ||
800 | 827 | ||
801 | timeout = gk20a_readl(g, fifo_fb_timeout_r()); | 828 | timeout = gk20a_readl(g, fifo_fb_timeout_r()); |
802 | timeout = set_field(timeout, fifo_fb_timeout_period_m(), | 829 | timeout = set_field(timeout, fifo_fb_timeout_period_m(), |
@@ -812,12 +839,13 @@ int gk20a_init_fifo_reset_enable_hw(struct gk20a *g) | |||
812 | nvgpu_log_info(g, "pbdma_timeout reg val = 0x%08x", timeout); | 839 | nvgpu_log_info(g, "pbdma_timeout reg val = 0x%08x", timeout); |
813 | gk20a_writel(g, pbdma_timeout_r(i), timeout); | 840 | gk20a_writel(g, pbdma_timeout_r(i), timeout); |
814 | } | 841 | } |
815 | if (g->ops.fifo.apply_pb_timeout) | 842 | if (g->ops.fifo.apply_pb_timeout) { |
816 | g->ops.fifo.apply_pb_timeout(g); | 843 | g->ops.fifo.apply_pb_timeout(g); |
844 | } | ||
817 | 845 | ||
818 | if (g->ops.fifo.apply_ctxsw_timeout_intr) | 846 | if (g->ops.fifo.apply_ctxsw_timeout_intr) { |
819 | g->ops.fifo.apply_ctxsw_timeout_intr(g); | 847 | g->ops.fifo.apply_ctxsw_timeout_intr(g); |
820 | else { | 848 | } else { |
821 | timeout = g->fifo_eng_timeout_us; | 849 | timeout = g->fifo_eng_timeout_us; |
822 | timeout = scale_ptimer(timeout, | 850 | timeout = scale_ptimer(timeout, |
823 | ptimer_scalingfactor10x(g->ptimer_src_freq)); | 851 | ptimer_scalingfactor10x(g->ptimer_src_freq)); |
@@ -989,14 +1017,14 @@ int gk20a_init_fifo_setup_sw(struct gk20a *g) | |||
989 | return err; | 1017 | return err; |
990 | } | 1018 | } |
991 | 1019 | ||
992 | if (g->ops.mm.is_bar1_supported(g)) | 1020 | if (g->ops.mm.is_bar1_supported(g)) { |
993 | err = nvgpu_dma_alloc_map_sys(g->mm.bar1.vm, | 1021 | err = nvgpu_dma_alloc_map_sys(g->mm.bar1.vm, |
994 | f->userd_entry_size * f->num_channels, | 1022 | f->userd_entry_size * f->num_channels, |
995 | &f->userd); | 1023 | &f->userd); |
996 | 1024 | } else { | |
997 | else | ||
998 | err = nvgpu_dma_alloc_sys(g, f->userd_entry_size * | 1025 | err = nvgpu_dma_alloc_sys(g, f->userd_entry_size * |
999 | f->num_channels, &f->userd); | 1026 | f->num_channels, &f->userd); |
1027 | } | ||
1000 | if (err) { | 1028 | if (err) { |
1001 | nvgpu_err(g, "userd memory allocation failed"); | 1029 | nvgpu_err(g, "userd memory allocation failed"); |
1002 | goto clean_up; | 1030 | goto clean_up; |
@@ -1012,8 +1040,9 @@ int gk20a_init_fifo_setup_sw(struct gk20a *g) | |||
1012 | } | 1040 | } |
1013 | 1041 | ||
1014 | err = nvgpu_channel_worker_init(g); | 1042 | err = nvgpu_channel_worker_init(g); |
1015 | if (err) | 1043 | if (err) { |
1016 | goto clean_up; | 1044 | goto clean_up; |
1045 | } | ||
1017 | 1046 | ||
1018 | f->sw_ready = true; | 1047 | f->sw_ready = true; |
1019 | 1048 | ||
@@ -1023,10 +1052,11 @@ int gk20a_init_fifo_setup_sw(struct gk20a *g) | |||
1023 | clean_up: | 1052 | clean_up: |
1024 | nvgpu_log_fn(g, "fail"); | 1053 | nvgpu_log_fn(g, "fail"); |
1025 | if (nvgpu_mem_is_valid(&f->userd)) { | 1054 | if (nvgpu_mem_is_valid(&f->userd)) { |
1026 | if (g->ops.mm.is_bar1_supported(g)) | 1055 | if (g->ops.mm.is_bar1_supported(g)) { |
1027 | nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd); | 1056 | nvgpu_dma_unmap_free(g->mm.bar1.vm, &f->userd); |
1028 | else | 1057 | } else { |
1029 | nvgpu_dma_free(g, &f->userd); | 1058 | nvgpu_dma_free(g, &f->userd); |
1059 | } | ||
1030 | } | 1060 | } |
1031 | 1061 | ||
1032 | return err; | 1062 | return err; |
@@ -1107,13 +1137,16 @@ int gk20a_init_fifo_support(struct gk20a *g) | |||
1107 | u32 err; | 1137 | u32 err; |
1108 | 1138 | ||
1109 | err = g->ops.fifo.setup_sw(g); | 1139 | err = g->ops.fifo.setup_sw(g); |
1110 | if (err) | 1140 | if (err) { |
1111 | return err; | 1141 | return err; |
1142 | } | ||
1112 | 1143 | ||
1113 | if (g->ops.fifo.init_fifo_setup_hw) | 1144 | if (g->ops.fifo.init_fifo_setup_hw) { |
1114 | err = g->ops.fifo.init_fifo_setup_hw(g); | 1145 | err = g->ops.fifo.init_fifo_setup_hw(g); |
1115 | if (err) | 1146 | } |
1147 | if (err) { | ||
1116 | return err; | 1148 | return err; |
1149 | } | ||
1117 | 1150 | ||
1118 | return err; | 1151 | return err; |
1119 | } | 1152 | } |
@@ -1124,20 +1157,23 @@ gk20a_refch_from_inst_ptr(struct gk20a *g, u64 inst_ptr) | |||
1124 | { | 1157 | { |
1125 | struct fifo_gk20a *f = &g->fifo; | 1158 | struct fifo_gk20a *f = &g->fifo; |
1126 | unsigned int ci; | 1159 | unsigned int ci; |
1127 | if (unlikely(!f->channel)) | 1160 | if (unlikely(!f->channel)) { |
1128 | return NULL; | 1161 | return NULL; |
1162 | } | ||
1129 | for (ci = 0; ci < f->num_channels; ci++) { | 1163 | for (ci = 0; ci < f->num_channels; ci++) { |
1130 | struct channel_gk20a *ch; | 1164 | struct channel_gk20a *ch; |
1131 | u64 ch_inst_ptr; | 1165 | u64 ch_inst_ptr; |
1132 | 1166 | ||
1133 | ch = gk20a_channel_get(&f->channel[ci]); | 1167 | ch = gk20a_channel_get(&f->channel[ci]); |
1134 | /* only alive channels are searched */ | 1168 | /* only alive channels are searched */ |
1135 | if (!ch) | 1169 | if (!ch) { |
1136 | continue; | 1170 | continue; |
1171 | } | ||
1137 | 1172 | ||
1138 | ch_inst_ptr = nvgpu_inst_block_addr(g, &ch->inst_block); | 1173 | ch_inst_ptr = nvgpu_inst_block_addr(g, &ch->inst_block); |
1139 | if (inst_ptr == ch_inst_ptr) | 1174 | if (inst_ptr == ch_inst_ptr) { |
1140 | return ch; | 1175 | return ch; |
1176 | } | ||
1141 | 1177 | ||
1142 | gk20a_channel_put(ch); | 1178 | gk20a_channel_put(ch); |
1143 | } | 1179 | } |
@@ -1199,34 +1235,37 @@ static const char * const does_not_exist[] = { | |||
1199 | /* fill in mmu fault desc */ | 1235 | /* fill in mmu fault desc */ |
1200 | void gk20a_fifo_get_mmu_fault_desc(struct mmu_fault_info *mmfault) | 1236 | void gk20a_fifo_get_mmu_fault_desc(struct mmu_fault_info *mmfault) |
1201 | { | 1237 | { |
1202 | if (mmfault->fault_type >= ARRAY_SIZE(gk20a_fault_type_descs)) | 1238 | if (mmfault->fault_type >= ARRAY_SIZE(gk20a_fault_type_descs)) { |
1203 | WARN_ON(mmfault->fault_type >= | 1239 | WARN_ON(mmfault->fault_type >= |
1204 | ARRAY_SIZE(gk20a_fault_type_descs)); | 1240 | ARRAY_SIZE(gk20a_fault_type_descs)); |
1205 | else | 1241 | } else { |
1206 | mmfault->fault_type_desc = | 1242 | mmfault->fault_type_desc = |
1207 | gk20a_fault_type_descs[mmfault->fault_type]; | 1243 | gk20a_fault_type_descs[mmfault->fault_type]; |
1244 | } | ||
1208 | } | 1245 | } |
1209 | 1246 | ||
1210 | /* fill in mmu fault client description */ | 1247 | /* fill in mmu fault client description */ |
1211 | void gk20a_fifo_get_mmu_fault_client_desc(struct mmu_fault_info *mmfault) | 1248 | void gk20a_fifo_get_mmu_fault_client_desc(struct mmu_fault_info *mmfault) |
1212 | { | 1249 | { |
1213 | if (mmfault->client_id >= ARRAY_SIZE(gk20a_hub_client_descs)) | 1250 | if (mmfault->client_id >= ARRAY_SIZE(gk20a_hub_client_descs)) { |
1214 | WARN_ON(mmfault->client_id >= | 1251 | WARN_ON(mmfault->client_id >= |
1215 | ARRAY_SIZE(gk20a_hub_client_descs)); | 1252 | ARRAY_SIZE(gk20a_hub_client_descs)); |
1216 | else | 1253 | } else { |
1217 | mmfault->client_id_desc = | 1254 | mmfault->client_id_desc = |
1218 | gk20a_hub_client_descs[mmfault->client_id]; | 1255 | gk20a_hub_client_descs[mmfault->client_id]; |
1256 | } | ||
1219 | } | 1257 | } |
1220 | 1258 | ||
1221 | /* fill in mmu fault gpc description */ | 1259 | /* fill in mmu fault gpc description */ |
1222 | void gk20a_fifo_get_mmu_fault_gpc_desc(struct mmu_fault_info *mmfault) | 1260 | void gk20a_fifo_get_mmu_fault_gpc_desc(struct mmu_fault_info *mmfault) |
1223 | { | 1261 | { |
1224 | if (mmfault->client_id >= ARRAY_SIZE(gk20a_gpc_client_descs)) | 1262 | if (mmfault->client_id >= ARRAY_SIZE(gk20a_gpc_client_descs)) { |
1225 | WARN_ON(mmfault->client_id >= | 1263 | WARN_ON(mmfault->client_id >= |
1226 | ARRAY_SIZE(gk20a_gpc_client_descs)); | 1264 | ARRAY_SIZE(gk20a_gpc_client_descs)); |
1227 | else | 1265 | } else { |
1228 | mmfault->client_id_desc = | 1266 | mmfault->client_id_desc = |
1229 | gk20a_gpc_client_descs[mmfault->client_id]; | 1267 | gk20a_gpc_client_descs[mmfault->client_id]; |
1268 | } | ||
1230 | } | 1269 | } |
1231 | 1270 | ||
1232 | static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id, | 1271 | static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id, |
@@ -1236,8 +1275,9 @@ static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id, | |||
1236 | 1275 | ||
1237 | /* parse info */ | 1276 | /* parse info */ |
1238 | mmfault->fault_type_desc = does_not_exist[0]; | 1277 | mmfault->fault_type_desc = does_not_exist[0]; |
1239 | if (g->ops.fifo.get_mmu_fault_desc) | 1278 | if (g->ops.fifo.get_mmu_fault_desc) { |
1240 | g->ops.fifo.get_mmu_fault_desc(mmfault); | 1279 | g->ops.fifo.get_mmu_fault_desc(mmfault); |
1280 | } | ||
1241 | 1281 | ||
1242 | if (mmfault->client_type >= ARRAY_SIZE(engine_subid_descs)) { | 1282 | if (mmfault->client_type >= ARRAY_SIZE(engine_subid_descs)) { |
1243 | WARN_ON(mmfault->client_type >= ARRAY_SIZE(engine_subid_descs)); | 1283 | WARN_ON(mmfault->client_type >= ARRAY_SIZE(engine_subid_descs)); |
@@ -1250,12 +1290,13 @@ static void get_exception_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id, | |||
1250 | mmfault->client_id_desc = does_not_exist[0]; | 1290 | mmfault->client_id_desc = does_not_exist[0]; |
1251 | if ((mmfault->client_type == | 1291 | if ((mmfault->client_type == |
1252 | fifo_intr_mmu_fault_info_engine_subid_hub_v()) | 1292 | fifo_intr_mmu_fault_info_engine_subid_hub_v()) |
1253 | && g->ops.fifo.get_mmu_fault_client_desc) | 1293 | && g->ops.fifo.get_mmu_fault_client_desc) { |
1254 | g->ops.fifo.get_mmu_fault_client_desc(mmfault); | 1294 | g->ops.fifo.get_mmu_fault_client_desc(mmfault); |
1255 | else if ((mmfault->client_type == | 1295 | } else if ((mmfault->client_type == |
1256 | fifo_intr_mmu_fault_info_engine_subid_gpc_v()) | 1296 | fifo_intr_mmu_fault_info_engine_subid_gpc_v()) |
1257 | && g->ops.fifo.get_mmu_fault_gpc_desc) | 1297 | && g->ops.fifo.get_mmu_fault_gpc_desc) { |
1258 | g->ops.fifo.get_mmu_fault_gpc_desc(mmfault); | 1298 | g->ops.fifo.get_mmu_fault_gpc_desc(mmfault); |
1299 | } | ||
1259 | } | 1300 | } |
1260 | 1301 | ||
1261 | /* reads info from hardware and fills in mmu fault info record */ | 1302 | /* reads info from hardware and fills in mmu fault info record */ |
@@ -1297,21 +1338,25 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id) | |||
1297 | 1338 | ||
1298 | nvgpu_log_fn(g, " "); | 1339 | nvgpu_log_fn(g, " "); |
1299 | 1340 | ||
1300 | if (!g) | 1341 | if (!g) { |
1301 | return; | 1342 | return; |
1343 | } | ||
1302 | 1344 | ||
1303 | engine_info = gk20a_fifo_get_engine_info(g, engine_id); | 1345 | engine_info = gk20a_fifo_get_engine_info(g, engine_id); |
1304 | 1346 | ||
1305 | if (engine_info) | 1347 | if (engine_info) { |
1306 | engine_enum = engine_info->engine_enum; | 1348 | engine_enum = engine_info->engine_enum; |
1349 | } | ||
1307 | 1350 | ||
1308 | if (engine_enum == ENGINE_INVAL_GK20A) | 1351 | if (engine_enum == ENGINE_INVAL_GK20A) { |
1309 | nvgpu_err(g, "unsupported engine_id %d", engine_id); | 1352 | nvgpu_err(g, "unsupported engine_id %d", engine_id); |
1353 | } | ||
1310 | 1354 | ||
1311 | if (engine_enum == ENGINE_GR_GK20A) { | 1355 | if (engine_enum == ENGINE_GR_GK20A) { |
1312 | if (g->support_pmu && g->can_elpg) { | 1356 | if (g->support_pmu && g->can_elpg) { |
1313 | if (nvgpu_pmu_disable_elpg(g)) | 1357 | if (nvgpu_pmu_disable_elpg(g)) { |
1314 | nvgpu_err(g, "failed to set disable elpg"); | 1358 | nvgpu_err(g, "failed to set disable elpg"); |
1359 | } | ||
1315 | } | 1360 | } |
1316 | 1361 | ||
1317 | #ifdef CONFIG_GK20A_CTXSW_TRACE | 1362 | #ifdef CONFIG_GK20A_CTXSW_TRACE |
@@ -1324,8 +1369,9 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id) | |||
1324 | #endif | 1369 | #endif |
1325 | if (!nvgpu_platform_is_simulation(g)) { | 1370 | if (!nvgpu_platform_is_simulation(g)) { |
1326 | /*HALT_PIPELINE method, halt GR engine*/ | 1371 | /*HALT_PIPELINE method, halt GR engine*/ |
1327 | if (gr_gk20a_halt_pipe(g)) | 1372 | if (gr_gk20a_halt_pipe(g)) { |
1328 | nvgpu_err(g, "failed to HALT gr pipe"); | 1373 | nvgpu_err(g, "failed to HALT gr pipe"); |
1374 | } | ||
1329 | /* | 1375 | /* |
1330 | * resetting engine using mc_enable_r() is not | 1376 | * resetting engine using mc_enable_r() is not |
1331 | * enough, we do full init sequence | 1377 | * enough, we do full init sequence |
@@ -1337,8 +1383,9 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id) | |||
1337 | "HALT gr pipe not supported and " | 1383 | "HALT gr pipe not supported and " |
1338 | "gr cannot be reset without halting gr pipe"); | 1384 | "gr cannot be reset without halting gr pipe"); |
1339 | } | 1385 | } |
1340 | if (g->support_pmu && g->can_elpg) | 1386 | if (g->support_pmu && g->can_elpg) { |
1341 | nvgpu_pmu_enable_elpg(g); | 1387 | nvgpu_pmu_enable_elpg(g); |
1388 | } | ||
1342 | } | 1389 | } |
1343 | if ((engine_enum == ENGINE_GRCE_GK20A) || | 1390 | if ((engine_enum == ENGINE_GRCE_GK20A) || |
1344 | (engine_enum == ENGINE_ASYNC_CE_GK20A)) { | 1391 | (engine_enum == ENGINE_ASYNC_CE_GK20A)) { |
@@ -1373,29 +1420,35 @@ bool gk20a_fifo_should_defer_engine_reset(struct gk20a *g, u32 engine_id, | |||
1373 | u32 engine_enum = ENGINE_INVAL_GK20A; | 1420 | u32 engine_enum = ENGINE_INVAL_GK20A; |
1374 | struct fifo_engine_info_gk20a *engine_info; | 1421 | struct fifo_engine_info_gk20a *engine_info; |
1375 | 1422 | ||
1376 | if (!g) | 1423 | if (!g) { |
1377 | return false; | 1424 | return false; |
1425 | } | ||
1378 | 1426 | ||
1379 | engine_info = gk20a_fifo_get_engine_info(g, engine_id); | 1427 | engine_info = gk20a_fifo_get_engine_info(g, engine_id); |
1380 | 1428 | ||
1381 | if (engine_info) | 1429 | if (engine_info) { |
1382 | engine_enum = engine_info->engine_enum; | 1430 | engine_enum = engine_info->engine_enum; |
1431 | } | ||
1383 | 1432 | ||
1384 | if (engine_enum == ENGINE_INVAL_GK20A) | 1433 | if (engine_enum == ENGINE_INVAL_GK20A) { |
1385 | return false; | 1434 | return false; |
1435 | } | ||
1386 | 1436 | ||
1387 | /* channel recovery is only deferred if an sm debugger | 1437 | /* channel recovery is only deferred if an sm debugger |
1388 | is attached and has MMU debug mode is enabled */ | 1438 | is attached and has MMU debug mode is enabled */ |
1389 | if (!g->ops.gr.sm_debugger_attached(g) || | 1439 | if (!g->ops.gr.sm_debugger_attached(g) || |
1390 | !g->ops.fb.is_debug_mode_enabled(g)) | 1440 | !g->ops.fb.is_debug_mode_enabled(g)) { |
1391 | return false; | 1441 | return false; |
1442 | } | ||
1392 | 1443 | ||
1393 | /* if this fault is fake (due to RC recovery), don't defer recovery */ | 1444 | /* if this fault is fake (due to RC recovery), don't defer recovery */ |
1394 | if (fake_fault) | 1445 | if (fake_fault) { |
1395 | return false; | 1446 | return false; |
1447 | } | ||
1396 | 1448 | ||
1397 | if (engine_enum != ENGINE_GR_GK20A) | 1449 | if (engine_enum != ENGINE_GR_GK20A) { |
1398 | return false; | 1450 | return false; |
1451 | } | ||
1399 | 1452 | ||
1400 | return g->ops.fifo.is_fault_engine_subid_gpc(g, engine_subid); | 1453 | return g->ops.fifo.is_fault_engine_subid_gpc(g, engine_subid); |
1401 | } | 1454 | } |
@@ -1405,12 +1458,14 @@ static bool gk20a_fifo_ch_timeout_debug_dump_state(struct gk20a *g, | |||
1405 | struct channel_gk20a *refch) | 1458 | struct channel_gk20a *refch) |
1406 | { | 1459 | { |
1407 | bool verbose = true; | 1460 | bool verbose = true; |
1408 | if (!refch) | 1461 | if (!refch) { |
1409 | return verbose; | 1462 | return verbose; |
1463 | } | ||
1410 | 1464 | ||
1411 | if (nvgpu_is_error_notifier_set(refch, | 1465 | if (nvgpu_is_error_notifier_set(refch, |
1412 | NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT)) | 1466 | NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT)) { |
1413 | verbose = refch->timeout_debug_dump; | 1467 | verbose = refch->timeout_debug_dump; |
1468 | } | ||
1414 | 1469 | ||
1415 | return verbose; | 1470 | return verbose; |
1416 | } | 1471 | } |
@@ -1499,15 +1554,17 @@ void gk20a_fifo_abort_tsg(struct gk20a *g, u32 tsgid, bool preempt) | |||
1499 | 1554 | ||
1500 | g->ops.fifo.disable_tsg(tsg); | 1555 | g->ops.fifo.disable_tsg(tsg); |
1501 | 1556 | ||
1502 | if (preempt) | 1557 | if (preempt) { |
1503 | g->ops.fifo.preempt_tsg(g, tsgid); | 1558 | g->ops.fifo.preempt_tsg(g, tsgid); |
1559 | } | ||
1504 | 1560 | ||
1505 | nvgpu_rwsem_down_read(&tsg->ch_list_lock); | 1561 | nvgpu_rwsem_down_read(&tsg->ch_list_lock); |
1506 | nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { | 1562 | nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { |
1507 | if (gk20a_channel_get(ch)) { | 1563 | if (gk20a_channel_get(ch)) { |
1508 | ch->has_timedout = true; | 1564 | ch->has_timedout = true; |
1509 | if (ch->g->ops.fifo.ch_abort_clean_up) | 1565 | if (ch->g->ops.fifo.ch_abort_clean_up) { |
1510 | ch->g->ops.fifo.ch_abort_clean_up(ch); | 1566 | ch->g->ops.fifo.ch_abort_clean_up(ch); |
1567 | } | ||
1511 | gk20a_channel_put(ch); | 1568 | gk20a_channel_put(ch); |
1512 | } | 1569 | } |
1513 | } | 1570 | } |
@@ -1521,23 +1578,27 @@ int gk20a_fifo_deferred_reset(struct gk20a *g, struct channel_gk20a *ch) | |||
1521 | nvgpu_mutex_acquire(&g->dbg_sessions_lock); | 1578 | nvgpu_mutex_acquire(&g->dbg_sessions_lock); |
1522 | gr_gk20a_disable_ctxsw(g); | 1579 | gr_gk20a_disable_ctxsw(g); |
1523 | 1580 | ||
1524 | if (!g->fifo.deferred_reset_pending) | 1581 | if (!g->fifo.deferred_reset_pending) { |
1525 | goto clean_up; | 1582 | goto clean_up; |
1583 | } | ||
1526 | 1584 | ||
1527 | if (gk20a_is_channel_marked_as_tsg(ch)) | 1585 | if (gk20a_is_channel_marked_as_tsg(ch)) { |
1528 | engines = gk20a_fifo_engines_on_id(g, ch->tsgid, true); | 1586 | engines = gk20a_fifo_engines_on_id(g, ch->tsgid, true); |
1529 | else | 1587 | } else { |
1530 | engines = gk20a_fifo_engines_on_id(g, ch->chid, false); | 1588 | engines = gk20a_fifo_engines_on_id(g, ch->chid, false); |
1531 | if (!engines) | 1589 | } |
1590 | if (!engines) { | ||
1532 | goto clean_up; | 1591 | goto clean_up; |
1592 | } | ||
1533 | 1593 | ||
1534 | /* | 1594 | /* |
1535 | * If deferred reset is set for an engine, and channel is running | 1595 | * If deferred reset is set for an engine, and channel is running |
1536 | * on that engine, reset it | 1596 | * on that engine, reset it |
1537 | */ | 1597 | */ |
1538 | for_each_set_bit(engine_id, &g->fifo.deferred_fault_engines, 32) { | 1598 | for_each_set_bit(engine_id, &g->fifo.deferred_fault_engines, 32) { |
1539 | if (BIT(engine_id) & engines) | 1599 | if (BIT(engine_id) & engines) { |
1540 | gk20a_fifo_reset_engine(g, engine_id); | 1600 | gk20a_fifo_reset_engine(g, engine_id); |
1601 | } | ||
1541 | } | 1602 | } |
1542 | 1603 | ||
1543 | g->fifo.deferred_fault_engines = 0; | 1604 | g->fifo.deferred_fault_engines = 0; |
@@ -1568,18 +1629,22 @@ static bool gk20a_fifo_handle_mmu_fault_locked( | |||
1568 | 1629 | ||
1569 | /* Disable power management */ | 1630 | /* Disable power management */ |
1570 | if (g->support_pmu && g->can_elpg) { | 1631 | if (g->support_pmu && g->can_elpg) { |
1571 | if (nvgpu_pmu_disable_elpg(g)) | 1632 | if (nvgpu_pmu_disable_elpg(g)) { |
1572 | nvgpu_err(g, "failed to set disable elpg"); | 1633 | nvgpu_err(g, "failed to set disable elpg"); |
1634 | } | ||
1573 | } | 1635 | } |
1574 | if (g->ops.clock_gating.slcg_gr_load_gating_prod) | 1636 | if (g->ops.clock_gating.slcg_gr_load_gating_prod) { |
1575 | g->ops.clock_gating.slcg_gr_load_gating_prod(g, | 1637 | g->ops.clock_gating.slcg_gr_load_gating_prod(g, |
1576 | false); | 1638 | false); |
1577 | if (g->ops.clock_gating.slcg_perf_load_gating_prod) | 1639 | } |
1640 | if (g->ops.clock_gating.slcg_perf_load_gating_prod) { | ||
1578 | g->ops.clock_gating.slcg_perf_load_gating_prod(g, | 1641 | g->ops.clock_gating.slcg_perf_load_gating_prod(g, |
1579 | false); | 1642 | false); |
1580 | if (g->ops.clock_gating.slcg_ltc_load_gating_prod) | 1643 | } |
1644 | if (g->ops.clock_gating.slcg_ltc_load_gating_prod) { | ||
1581 | g->ops.clock_gating.slcg_ltc_load_gating_prod(g, | 1645 | g->ops.clock_gating.slcg_ltc_load_gating_prod(g, |
1582 | false); | 1646 | false); |
1647 | } | ||
1583 | 1648 | ||
1584 | gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); | 1649 | gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); |
1585 | 1650 | ||
@@ -1673,9 +1738,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked( | |||
1673 | fifo_engine_status_id_type_chid_v(); | 1738 | fifo_engine_status_id_type_chid_v(); |
1674 | } | 1739 | } |
1675 | 1740 | ||
1676 | if (type == fifo_engine_status_id_type_tsgid_v()) | 1741 | if (type == fifo_engine_status_id_type_tsgid_v()) { |
1677 | tsg = &g->fifo.tsg[id]; | 1742 | tsg = &g->fifo.tsg[id]; |
1678 | else if (type == fifo_engine_status_id_type_chid_v()) { | 1743 | } else if (type == fifo_engine_status_id_type_chid_v()) { |
1679 | ch = &g->fifo.channel[id]; | 1744 | ch = &g->fifo.channel[id]; |
1680 | refch = gk20a_channel_get(ch); | 1745 | refch = gk20a_channel_get(ch); |
1681 | } | 1746 | } |
@@ -1686,8 +1751,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked( | |||
1686 | refch = ch; | 1751 | refch = ch; |
1687 | } | 1752 | } |
1688 | 1753 | ||
1689 | if (ch && gk20a_is_channel_marked_as_tsg(ch)) | 1754 | if (ch && gk20a_is_channel_marked_as_tsg(ch)) { |
1690 | tsg = &g->fifo.tsg[ch->tsgid]; | 1755 | tsg = &g->fifo.tsg[ch->tsgid]; |
1756 | } | ||
1691 | 1757 | ||
1692 | /* check if engine reset should be deferred */ | 1758 | /* check if engine reset should be deferred */ |
1693 | if (engine_id != FIFO_INVAL_ENGINE_ID) { | 1759 | if (engine_id != FIFO_INVAL_ENGINE_ID) { |
@@ -1730,24 +1796,27 @@ static bool gk20a_fifo_handle_mmu_fault_locked( | |||
1730 | if (g->fifo.deferred_reset_pending) { | 1796 | if (g->fifo.deferred_reset_pending) { |
1731 | gk20a_disable_tsg(tsg); | 1797 | gk20a_disable_tsg(tsg); |
1732 | } else { | 1798 | } else { |
1733 | if (!fake_fault) | 1799 | if (!fake_fault) { |
1734 | gk20a_fifo_set_ctx_mmu_error_tsg(g, | 1800 | gk20a_fifo_set_ctx_mmu_error_tsg(g, |
1735 | tsg); | 1801 | tsg); |
1802 | } | ||
1736 | verbose = gk20a_fifo_error_tsg(g, tsg); | 1803 | verbose = gk20a_fifo_error_tsg(g, tsg); |
1737 | gk20a_fifo_abort_tsg(g, tsg->tsgid, false); | 1804 | gk20a_fifo_abort_tsg(g, tsg->tsgid, false); |
1738 | } | 1805 | } |
1739 | 1806 | ||
1740 | /* put back the ref taken early above */ | 1807 | /* put back the ref taken early above */ |
1741 | if (refch) | 1808 | if (refch) { |
1742 | gk20a_channel_put(ch); | 1809 | gk20a_channel_put(ch); |
1810 | } | ||
1743 | } else if (ch) { | 1811 | } else if (ch) { |
1744 | if (refch) { | 1812 | if (refch) { |
1745 | if (g->fifo.deferred_reset_pending) { | 1813 | if (g->fifo.deferred_reset_pending) { |
1746 | g->ops.fifo.disable_channel(ch); | 1814 | g->ops.fifo.disable_channel(ch); |
1747 | } else { | 1815 | } else { |
1748 | if (!fake_fault) | 1816 | if (!fake_fault) { |
1749 | gk20a_fifo_set_ctx_mmu_error_ch( | 1817 | gk20a_fifo_set_ctx_mmu_error_ch( |
1750 | g, refch); | 1818 | g, refch); |
1819 | } | ||
1751 | 1820 | ||
1752 | verbose = gk20a_fifo_error_ch(g, | 1821 | verbose = gk20a_fifo_error_ch(g, |
1753 | refch); | 1822 | refch); |
@@ -1765,8 +1834,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked( | |||
1765 | } else if (mmfault_info.inst_ptr == | 1834 | } else if (mmfault_info.inst_ptr == |
1766 | nvgpu_inst_block_addr(g, &g->mm.pmu.inst_block)) { | 1835 | nvgpu_inst_block_addr(g, &g->mm.pmu.inst_block)) { |
1767 | nvgpu_err(g, "mmu fault from pmu"); | 1836 | nvgpu_err(g, "mmu fault from pmu"); |
1768 | } else | 1837 | } else { |
1769 | nvgpu_err(g, "couldn't locate channel for mmu fault"); | 1838 | nvgpu_err(g, "couldn't locate channel for mmu fault"); |
1839 | } | ||
1770 | } | 1840 | } |
1771 | 1841 | ||
1772 | /* clear interrupt */ | 1842 | /* clear interrupt */ |
@@ -1782,8 +1852,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked( | |||
1782 | gr_gpfifo_ctl_semaphore_access_enabled_f()); | 1852 | gr_gpfifo_ctl_semaphore_access_enabled_f()); |
1783 | 1853 | ||
1784 | /* It is safe to enable ELPG again. */ | 1854 | /* It is safe to enable ELPG again. */ |
1785 | if (g->support_pmu && g->can_elpg) | 1855 | if (g->support_pmu && g->can_elpg) { |
1786 | nvgpu_pmu_enable_elpg(g); | 1856 | nvgpu_pmu_enable_elpg(g); |
1857 | } | ||
1787 | 1858 | ||
1788 | return verbose; | 1859 | return verbose; |
1789 | } | 1860 | } |
@@ -1856,8 +1927,9 @@ static u32 gk20a_fifo_engines_on_id(struct gk20a *g, u32 id, bool is_tsg) | |||
1856 | if ((is_tsg && type == | 1927 | if ((is_tsg && type == |
1857 | fifo_engine_status_id_type_tsgid_v()) || | 1928 | fifo_engine_status_id_type_tsgid_v()) || |
1858 | (!is_tsg && type == | 1929 | (!is_tsg && type == |
1859 | fifo_engine_status_id_type_chid_v())) | 1930 | fifo_engine_status_id_type_chid_v())) { |
1860 | engines |= BIT(active_engine_id); | 1931 | engines |= BIT(active_engine_id); |
1932 | } | ||
1861 | } | 1933 | } |
1862 | } | 1934 | } |
1863 | 1935 | ||
@@ -1875,17 +1947,18 @@ void gk20a_fifo_recover_ch(struct gk20a *g, u32 chid, bool verbose, int rc_type) | |||
1875 | 1947 | ||
1876 | engines = gk20a_fifo_engines_on_id(g, chid, false); | 1948 | engines = gk20a_fifo_engines_on_id(g, chid, false); |
1877 | 1949 | ||
1878 | if (engines) | 1950 | if (engines) { |
1879 | gk20a_fifo_recover(g, engines, chid, false, true, verbose, | 1951 | gk20a_fifo_recover(g, engines, chid, false, true, verbose, |
1880 | rc_type); | 1952 | rc_type); |
1881 | else { | 1953 | } else { |
1882 | struct channel_gk20a *ch = &g->fifo.channel[chid]; | 1954 | struct channel_gk20a *ch = &g->fifo.channel[chid]; |
1883 | 1955 | ||
1884 | if (gk20a_channel_get(ch)) { | 1956 | if (gk20a_channel_get(ch)) { |
1885 | gk20a_channel_abort(ch, false); | 1957 | gk20a_channel_abort(ch, false); |
1886 | 1958 | ||
1887 | if (gk20a_fifo_error_ch(g, ch)) | 1959 | if (gk20a_fifo_error_ch(g, ch)) { |
1888 | gk20a_debug_dump(g); | 1960 | gk20a_debug_dump(g); |
1961 | } | ||
1889 | 1962 | ||
1890 | gk20a_channel_put(ch); | 1963 | gk20a_channel_put(ch); |
1891 | } | 1964 | } |
@@ -1907,14 +1980,15 @@ void gk20a_fifo_recover_tsg(struct gk20a *g, u32 tsgid, bool verbose, | |||
1907 | 1980 | ||
1908 | engines = gk20a_fifo_engines_on_id(g, tsgid, true); | 1981 | engines = gk20a_fifo_engines_on_id(g, tsgid, true); |
1909 | 1982 | ||
1910 | if (engines) | 1983 | if (engines) { |
1911 | gk20a_fifo_recover(g, engines, tsgid, true, true, verbose, | 1984 | gk20a_fifo_recover(g, engines, tsgid, true, true, verbose, |
1912 | rc_type); | 1985 | rc_type); |
1913 | else { | 1986 | } else { |
1914 | struct tsg_gk20a *tsg = &g->fifo.tsg[tsgid]; | 1987 | struct tsg_gk20a *tsg = &g->fifo.tsg[tsgid]; |
1915 | 1988 | ||
1916 | if (gk20a_fifo_error_tsg(g, tsg) && verbose) | 1989 | if (gk20a_fifo_error_tsg(g, tsg) && verbose) { |
1917 | gk20a_debug_dump(g); | 1990 | gk20a_debug_dump(g); |
1991 | } | ||
1918 | 1992 | ||
1919 | gk20a_fifo_abort_tsg(g, tsgid, false); | 1993 | gk20a_fifo_abort_tsg(g, tsgid, false); |
1920 | } | 1994 | } |
@@ -1956,18 +2030,20 @@ void gk20a_fifo_teardown_ch_tsg(struct gk20a *g, u32 __engine_ids, | |||
1956 | for_each_set_bit(engine_id, &engine_ids, 32) { | 2030 | for_each_set_bit(engine_id, &engine_ids, 32) { |
1957 | u32 mmu_id = gk20a_engine_id_to_mmu_id(g, engine_id); | 2031 | u32 mmu_id = gk20a_engine_id_to_mmu_id(g, engine_id); |
1958 | 2032 | ||
1959 | if (mmu_id != FIFO_INVAL_ENGINE_ID) | 2033 | if (mmu_id != FIFO_INVAL_ENGINE_ID) { |
1960 | mmu_fault_engines |= BIT(mmu_id); | 2034 | mmu_fault_engines |= BIT(mmu_id); |
2035 | } | ||
1961 | } | 2036 | } |
1962 | } else { | 2037 | } else { |
1963 | /* store faulted engines in advance */ | 2038 | /* store faulted engines in advance */ |
1964 | for_each_set_bit(engine_id, &_engine_ids, 32) { | 2039 | for_each_set_bit(engine_id, &_engine_ids, 32) { |
1965 | gk20a_fifo_get_faulty_id_type(g, engine_id, &ref_id, | 2040 | gk20a_fifo_get_faulty_id_type(g, engine_id, &ref_id, |
1966 | &ref_type); | 2041 | &ref_type); |
1967 | if (ref_type == fifo_engine_status_id_type_tsgid_v()) | 2042 | if (ref_type == fifo_engine_status_id_type_tsgid_v()) { |
1968 | ref_id_is_tsg = true; | 2043 | ref_id_is_tsg = true; |
1969 | else | 2044 | } else { |
1970 | ref_id_is_tsg = false; | 2045 | ref_id_is_tsg = false; |
2046 | } | ||
1971 | /* Reset *all* engines that use the | 2047 | /* Reset *all* engines that use the |
1972 | * same channel as faulty engine */ | 2048 | * same channel as faulty engine */ |
1973 | for (i = 0; i < g->fifo.num_engines; i++) { | 2049 | for (i = 0; i < g->fifo.num_engines; i++) { |
@@ -1980,8 +2056,9 @@ void gk20a_fifo_teardown_ch_tsg(struct gk20a *g, u32 __engine_ids, | |||
1980 | u32 mmu_id = gk20a_engine_id_to_mmu_id(g, active_engine_id); | 2056 | u32 mmu_id = gk20a_engine_id_to_mmu_id(g, active_engine_id); |
1981 | 2057 | ||
1982 | engine_ids |= BIT(active_engine_id); | 2058 | engine_ids |= BIT(active_engine_id); |
1983 | if (mmu_id != FIFO_INVAL_ENGINE_ID) | 2059 | if (mmu_id != FIFO_INVAL_ENGINE_ID) { |
1984 | mmu_fault_engines |= BIT(mmu_id); | 2060 | mmu_fault_engines |= BIT(mmu_id); |
2061 | } | ||
1985 | } | 2062 | } |
1986 | } | 2063 | } |
1987 | } | 2064 | } |
@@ -2021,16 +2098,19 @@ void gk20a_fifo_recover(struct gk20a *g, u32 __engine_ids, | |||
2021 | { | 2098 | { |
2022 | unsigned int id_type; | 2099 | unsigned int id_type; |
2023 | 2100 | ||
2024 | if (verbose) | 2101 | if (verbose) { |
2025 | gk20a_debug_dump(g); | 2102 | gk20a_debug_dump(g); |
2103 | } | ||
2026 | 2104 | ||
2027 | if (g->ops.ltc.flush) | 2105 | if (g->ops.ltc.flush) { |
2028 | g->ops.ltc.flush(g); | 2106 | g->ops.ltc.flush(g); |
2107 | } | ||
2029 | 2108 | ||
2030 | if (id_is_known) | 2109 | if (id_is_known) { |
2031 | id_type = id_is_tsg ? ID_TYPE_TSG : ID_TYPE_CHANNEL; | 2110 | id_type = id_is_tsg ? ID_TYPE_TSG : ID_TYPE_CHANNEL; |
2032 | else | 2111 | } else { |
2033 | id_type = ID_TYPE_UNKNOWN; | 2112 | id_type = ID_TYPE_UNKNOWN; |
2113 | } | ||
2034 | 2114 | ||
2035 | g->ops.fifo.teardown_ch_tsg(g, __engine_ids, hw_id, id_type, | 2115 | g->ops.fifo.teardown_ch_tsg(g, __engine_ids, hw_id, id_type, |
2036 | rc_type, NULL); | 2116 | rc_type, NULL); |
@@ -2080,11 +2160,13 @@ int gk20a_fifo_tsg_unbind_channel_verify_status(struct channel_gk20a *ch) | |||
2080 | return -EINVAL; | 2160 | return -EINVAL; |
2081 | } | 2161 | } |
2082 | 2162 | ||
2083 | if (g->ops.fifo.tsg_verify_status_ctx_reload) | 2163 | if (g->ops.fifo.tsg_verify_status_ctx_reload) { |
2084 | g->ops.fifo.tsg_verify_status_ctx_reload(ch); | 2164 | g->ops.fifo.tsg_verify_status_ctx_reload(ch); |
2165 | } | ||
2085 | 2166 | ||
2086 | if (g->ops.fifo.tsg_verify_status_faulted) | 2167 | if (g->ops.fifo.tsg_verify_status_faulted) { |
2087 | g->ops.fifo.tsg_verify_status_faulted(ch); | 2168 | g->ops.fifo.tsg_verify_status_faulted(ch); |
2169 | } | ||
2088 | 2170 | ||
2089 | return 0; | 2171 | return 0; |
2090 | } | 2172 | } |
@@ -2106,19 +2188,22 @@ int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch) | |||
2106 | g->ops.fifo.disable_tsg(tsg); | 2188 | g->ops.fifo.disable_tsg(tsg); |
2107 | 2189 | ||
2108 | err = g->ops.fifo.preempt_tsg(g, tsg->tsgid); | 2190 | err = g->ops.fifo.preempt_tsg(g, tsg->tsgid); |
2109 | if (err) | 2191 | if (err) { |
2110 | goto fail_enable_tsg; | 2192 | goto fail_enable_tsg; |
2193 | } | ||
2111 | 2194 | ||
2112 | if (g->ops.fifo.tsg_verify_channel_status && !tsg_timedout) { | 2195 | if (g->ops.fifo.tsg_verify_channel_status && !tsg_timedout) { |
2113 | err = g->ops.fifo.tsg_verify_channel_status(ch); | 2196 | err = g->ops.fifo.tsg_verify_channel_status(ch); |
2114 | if (err) | 2197 | if (err) { |
2115 | goto fail_enable_tsg; | 2198 | goto fail_enable_tsg; |
2199 | } | ||
2116 | } | 2200 | } |
2117 | 2201 | ||
2118 | /* Channel should be seen as TSG channel while updating runlist */ | 2202 | /* Channel should be seen as TSG channel while updating runlist */ |
2119 | err = channel_gk20a_update_runlist(ch, false); | 2203 | err = channel_gk20a_update_runlist(ch, false); |
2120 | if (err) | 2204 | if (err) { |
2121 | goto fail_enable_tsg; | 2205 | goto fail_enable_tsg; |
2206 | } | ||
2122 | 2207 | ||
2123 | /* Remove channel from TSG and re-enable rest of the channels */ | 2208 | /* Remove channel from TSG and re-enable rest of the channels */ |
2124 | nvgpu_rwsem_down_write(&tsg->ch_list_lock); | 2209 | nvgpu_rwsem_down_write(&tsg->ch_list_lock); |
@@ -2131,17 +2216,20 @@ int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch) | |||
2131 | * Note that we can skip disabling and preempting TSG too in case of | 2216 | * Note that we can skip disabling and preempting TSG too in case of |
2132 | * time out, but we keep that to ensure TSG is kicked out | 2217 | * time out, but we keep that to ensure TSG is kicked out |
2133 | */ | 2218 | */ |
2134 | if (!tsg_timedout) | 2219 | if (!tsg_timedout) { |
2135 | g->ops.fifo.enable_tsg(tsg); | 2220 | g->ops.fifo.enable_tsg(tsg); |
2221 | } | ||
2136 | 2222 | ||
2137 | if (ch->g->ops.fifo.ch_abort_clean_up) | 2223 | if (ch->g->ops.fifo.ch_abort_clean_up) { |
2138 | ch->g->ops.fifo.ch_abort_clean_up(ch); | 2224 | ch->g->ops.fifo.ch_abort_clean_up(ch); |
2225 | } | ||
2139 | 2226 | ||
2140 | return 0; | 2227 | return 0; |
2141 | 2228 | ||
2142 | fail_enable_tsg: | 2229 | fail_enable_tsg: |
2143 | if (!tsg_timedout) | 2230 | if (!tsg_timedout) { |
2144 | g->ops.fifo.enable_tsg(tsg); | 2231 | g->ops.fifo.enable_tsg(tsg); |
2232 | } | ||
2145 | return err; | 2233 | return err; |
2146 | } | 2234 | } |
2147 | 2235 | ||
@@ -2225,9 +2313,10 @@ bool gk20a_fifo_check_ch_ctxsw_timeout(struct channel_gk20a *ch, | |||
2225 | &progress); | 2313 | &progress); |
2226 | *verbose = ch->timeout_debug_dump; | 2314 | *verbose = ch->timeout_debug_dump; |
2227 | *ms = ch->timeout_accumulated_ms; | 2315 | *ms = ch->timeout_accumulated_ms; |
2228 | if (recover) | 2316 | if (recover) { |
2229 | g->ops.fifo.set_error_notifier(ch, | 2317 | g->ops.fifo.set_error_notifier(ch, |
2230 | NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT); | 2318 | NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT); |
2319 | } | ||
2231 | 2320 | ||
2232 | gk20a_channel_put(ch); | 2321 | gk20a_channel_put(ch); |
2233 | } | 2322 | } |
@@ -2255,8 +2344,9 @@ bool gk20a_fifo_check_tsg_ctxsw_timeout(struct tsg_gk20a *tsg, | |||
2255 | if (gk20a_channel_get(ch)) { | 2344 | if (gk20a_channel_get(ch)) { |
2256 | recover = gk20a_channel_update_and_check_timeout(ch, | 2345 | recover = gk20a_channel_update_and_check_timeout(ch, |
2257 | *ms, &progress); | 2346 | *ms, &progress); |
2258 | if (progress || recover) | 2347 | if (progress || recover) { |
2259 | break; | 2348 | break; |
2349 | } | ||
2260 | gk20a_channel_put(ch); | 2350 | gk20a_channel_put(ch); |
2261 | } | 2351 | } |
2262 | } | 2352 | } |
@@ -2481,9 +2571,11 @@ static bool gk20a_fifo_is_sw_method_subch(struct gk20a *g, int pbdma_id, | |||
2481 | pbdma_method_subch = pbdma_method0_subch_v( | 2571 | pbdma_method_subch = pbdma_method0_subch_v( |
2482 | gk20a_readl(g, pbdma_method_reg)); | 2572 | gk20a_readl(g, pbdma_method_reg)); |
2483 | 2573 | ||
2484 | if (pbdma_method_subch == 5 || pbdma_method_subch == 6 || | 2574 | if (pbdma_method_subch == 5 || |
2485 | pbdma_method_subch == 7) | 2575 | pbdma_method_subch == 6 || |
2576 | pbdma_method_subch == 7) { | ||
2486 | return true; | 2577 | return true; |
2578 | } | ||
2487 | 2579 | ||
2488 | return false; | 2580 | return false; |
2489 | } | 2581 | } |
@@ -2565,9 +2657,10 @@ unsigned int gk20a_fifo_handle_pbdma_intr_0(struct gk20a *g, u32 pbdma_id, | |||
2565 | 2657 | ||
2566 | for (i = 0; i < 4; i++) { | 2658 | for (i = 0; i < 4; i++) { |
2567 | if (gk20a_fifo_is_sw_method_subch(g, | 2659 | if (gk20a_fifo_is_sw_method_subch(g, |
2568 | pbdma_id, i)) | 2660 | pbdma_id, i)) { |
2569 | gk20a_fifo_reset_pbdma_method(g, | 2661 | gk20a_fifo_reset_pbdma_method(g, |
2570 | pbdma_id, i); | 2662 | pbdma_id, i); |
2663 | } | ||
2571 | } | 2664 | } |
2572 | rc_type = RC_TYPE_PBDMA_FAULT; | 2665 | rc_type = RC_TYPE_PBDMA_FAULT; |
2573 | } | 2666 | } |
@@ -2649,8 +2742,9 @@ u32 gk20a_fifo_handle_pbdma_intr(struct gk20a *g, struct fifo_gk20a *f, | |||
2649 | pbdma_id, pbdma_intr_0); | 2742 | pbdma_id, pbdma_intr_0); |
2650 | 2743 | ||
2651 | if (g->ops.fifo.handle_pbdma_intr_0(g, pbdma_id, pbdma_intr_0, | 2744 | if (g->ops.fifo.handle_pbdma_intr_0(g, pbdma_id, pbdma_intr_0, |
2652 | &handled, &error_notifier) != RC_TYPE_NO_RC) | 2745 | &handled, &error_notifier) != RC_TYPE_NO_RC) { |
2653 | rc_type = RC_TYPE_PBDMA_FAULT; | 2746 | rc_type = RC_TYPE_PBDMA_FAULT; |
2747 | } | ||
2654 | gk20a_writel(g, pbdma_intr_0_r(pbdma_id), pbdma_intr_0); | 2748 | gk20a_writel(g, pbdma_intr_0_r(pbdma_id), pbdma_intr_0); |
2655 | } | 2749 | } |
2656 | 2750 | ||
@@ -2660,13 +2754,15 @@ u32 gk20a_fifo_handle_pbdma_intr(struct gk20a *g, struct fifo_gk20a *f, | |||
2660 | pbdma_id, pbdma_intr_1); | 2754 | pbdma_id, pbdma_intr_1); |
2661 | 2755 | ||
2662 | if (g->ops.fifo.handle_pbdma_intr_1(g, pbdma_id, pbdma_intr_1, | 2756 | if (g->ops.fifo.handle_pbdma_intr_1(g, pbdma_id, pbdma_intr_1, |
2663 | &handled, &error_notifier) != RC_TYPE_NO_RC) | 2757 | &handled, &error_notifier) != RC_TYPE_NO_RC) { |
2664 | rc_type = RC_TYPE_PBDMA_FAULT; | 2758 | rc_type = RC_TYPE_PBDMA_FAULT; |
2759 | } | ||
2665 | gk20a_writel(g, pbdma_intr_1_r(pbdma_id), pbdma_intr_1); | 2760 | gk20a_writel(g, pbdma_intr_1_r(pbdma_id), pbdma_intr_1); |
2666 | } | 2761 | } |
2667 | 2762 | ||
2668 | if (rc == RC_YES && rc_type == RC_TYPE_PBDMA_FAULT) | 2763 | if (rc == RC_YES && rc_type == RC_TYPE_PBDMA_FAULT) { |
2669 | gk20a_fifo_pbdma_fault_rc(g, f, pbdma_id, error_notifier); | 2764 | gk20a_fifo_pbdma_fault_rc(g, f, pbdma_id, error_notifier); |
2765 | } | ||
2670 | 2766 | ||
2671 | return handled; | 2767 | return handled; |
2672 | } | 2768 | } |
@@ -2708,14 +2804,17 @@ void gk20a_fifo_isr(struct gk20a *g) | |||
2708 | gk20a_fifo_handle_runlist_event(g); | 2804 | gk20a_fifo_handle_runlist_event(g); |
2709 | clear_intr |= fifo_intr_0_runlist_event_pending_f(); | 2805 | clear_intr |= fifo_intr_0_runlist_event_pending_f(); |
2710 | } | 2806 | } |
2711 | if (fifo_intr & fifo_intr_0_pbdma_intr_pending_f()) | 2807 | if (fifo_intr & fifo_intr_0_pbdma_intr_pending_f()) { |
2712 | clear_intr |= fifo_pbdma_isr(g, fifo_intr); | 2808 | clear_intr |= fifo_pbdma_isr(g, fifo_intr); |
2809 | } | ||
2713 | 2810 | ||
2714 | if (g->ops.fifo.handle_ctxsw_timeout) | 2811 | if (g->ops.fifo.handle_ctxsw_timeout) { |
2715 | g->ops.fifo.handle_ctxsw_timeout(g, fifo_intr); | 2812 | g->ops.fifo.handle_ctxsw_timeout(g, fifo_intr); |
2813 | } | ||
2716 | 2814 | ||
2717 | if (unlikely(fifo_intr & error_intr_mask)) | 2815 | if (unlikely(fifo_intr & error_intr_mask)) { |
2718 | clear_intr = fifo_error_isr(g, fifo_intr); | 2816 | clear_intr = fifo_error_isr(g, fifo_intr); |
2817 | } | ||
2719 | 2818 | ||
2720 | nvgpu_mutex_release(&g->fifo.intr.isr.mutex); | 2819 | nvgpu_mutex_release(&g->fifo.intr.isr.mutex); |
2721 | } | 2820 | } |
@@ -2731,8 +2830,9 @@ u32 gk20a_fifo_nonstall_isr(struct gk20a *g) | |||
2731 | 2830 | ||
2732 | nvgpu_log(g, gpu_dbg_intr, "fifo nonstall isr %08x\n", fifo_intr); | 2831 | nvgpu_log(g, gpu_dbg_intr, "fifo nonstall isr %08x\n", fifo_intr); |
2733 | 2832 | ||
2734 | if (fifo_intr & fifo_intr_0_channel_intr_pending_f()) | 2833 | if (fifo_intr & fifo_intr_0_channel_intr_pending_f()) { |
2735 | clear_intr = fifo_intr_0_channel_intr_pending_f(); | 2834 | clear_intr = fifo_intr_0_channel_intr_pending_f(); |
2835 | } | ||
2736 | 2836 | ||
2737 | gk20a_writel(g, fifo_intr_0_r(), clear_intr); | 2837 | gk20a_writel(g, fifo_intr_0_r(), clear_intr); |
2738 | 2838 | ||
@@ -2741,14 +2841,15 @@ u32 gk20a_fifo_nonstall_isr(struct gk20a *g) | |||
2741 | 2841 | ||
2742 | void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg) | 2842 | void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg) |
2743 | { | 2843 | { |
2744 | if (is_tsg) | 2844 | if (is_tsg) { |
2745 | gk20a_writel(g, fifo_preempt_r(), | 2845 | gk20a_writel(g, fifo_preempt_r(), |
2746 | fifo_preempt_id_f(id) | | 2846 | fifo_preempt_id_f(id) | |
2747 | fifo_preempt_type_tsg_f()); | 2847 | fifo_preempt_type_tsg_f()); |
2748 | else | 2848 | } else { |
2749 | gk20a_writel(g, fifo_preempt_r(), | 2849 | gk20a_writel(g, fifo_preempt_r(), |
2750 | fifo_preempt_chid_f(id) | | 2850 | fifo_preempt_chid_f(id) | |
2751 | fifo_preempt_type_channel_f()); | 2851 | fifo_preempt_type_channel_f()); |
2852 | } | ||
2752 | } | 2853 | } |
2753 | 2854 | ||
2754 | static u32 gk20a_fifo_get_preempt_timeout(struct gk20a *g) | 2855 | static u32 gk20a_fifo_get_preempt_timeout(struct gk20a *g) |
@@ -2802,8 +2903,9 @@ void gk20a_fifo_preempt_timeout_rc(struct gk20a *g, u32 id, | |||
2802 | nvgpu_rwsem_down_read(&tsg->ch_list_lock); | 2903 | nvgpu_rwsem_down_read(&tsg->ch_list_lock); |
2803 | nvgpu_list_for_each_entry(ch, &tsg->ch_list, | 2904 | nvgpu_list_for_each_entry(ch, &tsg->ch_list, |
2804 | channel_gk20a, ch_entry) { | 2905 | channel_gk20a, ch_entry) { |
2805 | if (!gk20a_channel_get(ch)) | 2906 | if (!gk20a_channel_get(ch)) { |
2806 | continue; | 2907 | continue; |
2908 | } | ||
2807 | g->ops.fifo.set_error_notifier(ch, | 2909 | g->ops.fifo.set_error_notifier(ch, |
2808 | NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT); | 2910 | NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT); |
2809 | gk20a_channel_put(ch); | 2911 | gk20a_channel_put(ch); |
@@ -2854,8 +2956,9 @@ int gk20a_fifo_preempt_channel(struct gk20a *g, u32 chid) | |||
2854 | u32 i; | 2956 | u32 i; |
2855 | 2957 | ||
2856 | nvgpu_log_fn(g, "chid: %d", chid); | 2958 | nvgpu_log_fn(g, "chid: %d", chid); |
2857 | if (chid == FIFO_INVAL_CHANNEL_ID) | 2959 | if (chid == FIFO_INVAL_CHANNEL_ID) { |
2858 | return 0; | 2960 | return 0; |
2961 | } | ||
2859 | 2962 | ||
2860 | /* we have no idea which runlist we are using. lock all */ | 2963 | /* we have no idea which runlist we are using. lock all */ |
2861 | for (i = 0; i < g->fifo.max_runlists; i++) { | 2964 | for (i = 0; i < g->fifo.max_runlists; i++) { |
@@ -2866,8 +2969,9 @@ int gk20a_fifo_preempt_channel(struct gk20a *g, u32 chid) | |||
2866 | 2969 | ||
2867 | ret = __locked_fifo_preempt(g, chid, false); | 2970 | ret = __locked_fifo_preempt(g, chid, false); |
2868 | 2971 | ||
2869 | if (!mutex_ret) | 2972 | if (!mutex_ret) { |
2870 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | 2973 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); |
2974 | } | ||
2871 | 2975 | ||
2872 | for (i = 0; i < g->fifo.max_runlists; i++) { | 2976 | for (i = 0; i < g->fifo.max_runlists; i++) { |
2873 | nvgpu_mutex_release(&f->runlist_info[i].runlist_lock); | 2977 | nvgpu_mutex_release(&f->runlist_info[i].runlist_lock); |
@@ -2896,8 +3000,9 @@ int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid) | |||
2896 | u32 i; | 3000 | u32 i; |
2897 | 3001 | ||
2898 | nvgpu_log_fn(g, "tsgid: %d", tsgid); | 3002 | nvgpu_log_fn(g, "tsgid: %d", tsgid); |
2899 | if (tsgid == FIFO_INVAL_TSG_ID) | 3003 | if (tsgid == FIFO_INVAL_TSG_ID) { |
2900 | return 0; | 3004 | return 0; |
3005 | } | ||
2901 | 3006 | ||
2902 | /* we have no idea which runlist we are using. lock all */ | 3007 | /* we have no idea which runlist we are using. lock all */ |
2903 | for (i = 0; i < g->fifo.max_runlists; i++) { | 3008 | for (i = 0; i < g->fifo.max_runlists; i++) { |
@@ -2908,8 +3013,9 @@ int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid) | |||
2908 | 3013 | ||
2909 | ret = __locked_fifo_preempt(g, tsgid, true); | 3014 | ret = __locked_fifo_preempt(g, tsgid, true); |
2910 | 3015 | ||
2911 | if (!mutex_ret) | 3016 | if (!mutex_ret) { |
2912 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | 3017 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); |
3018 | } | ||
2913 | 3019 | ||
2914 | for (i = 0; i < g->fifo.max_runlists; i++) { | 3020 | for (i = 0; i < g->fifo.max_runlists; i++) { |
2915 | nvgpu_mutex_release(&f->runlist_info[i].runlist_lock); | 3021 | nvgpu_mutex_release(&f->runlist_info[i].runlist_lock); |
@@ -2931,10 +3037,11 @@ int gk20a_fifo_preempt(struct gk20a *g, struct channel_gk20a *ch) | |||
2931 | { | 3037 | { |
2932 | int err; | 3038 | int err; |
2933 | 3039 | ||
2934 | if (gk20a_is_channel_marked_as_tsg(ch)) | 3040 | if (gk20a_is_channel_marked_as_tsg(ch)) { |
2935 | err = g->ops.fifo.preempt_tsg(ch->g, ch->tsgid); | 3041 | err = g->ops.fifo.preempt_tsg(ch->g, ch->tsgid); |
2936 | else | 3042 | } else { |
2937 | err = g->ops.fifo.preempt_channel(ch->g, ch->chid); | 3043 | err = g->ops.fifo.preempt_channel(ch->g, ch->chid); |
3044 | } | ||
2938 | 3045 | ||
2939 | return err; | 3046 | return err; |
2940 | } | 3047 | } |
@@ -2946,10 +3053,11 @@ static void gk20a_fifo_sched_disable_rw(struct gk20a *g, u32 runlists_mask, | |||
2946 | 3053 | ||
2947 | reg_val = gk20a_readl(g, fifo_sched_disable_r()); | 3054 | reg_val = gk20a_readl(g, fifo_sched_disable_r()); |
2948 | 3055 | ||
2949 | if (runlist_state == RUNLIST_DISABLED) | 3056 | if (runlist_state == RUNLIST_DISABLED) { |
2950 | reg_val |= runlists_mask; | 3057 | reg_val |= runlists_mask; |
2951 | else | 3058 | } else { |
2952 | reg_val &= (~runlists_mask); | 3059 | reg_val &= (~runlists_mask); |
3060 | } | ||
2953 | 3061 | ||
2954 | gk20a_writel(g, fifo_sched_disable_r(), reg_val); | 3062 | gk20a_writel(g, fifo_sched_disable_r(), reg_val); |
2955 | 3063 | ||
@@ -2968,8 +3076,9 @@ void gk20a_fifo_set_runlist_state(struct gk20a *g, u32 runlists_mask, | |||
2968 | 3076 | ||
2969 | gk20a_fifo_sched_disable_rw(g, runlists_mask, runlist_state); | 3077 | gk20a_fifo_sched_disable_rw(g, runlists_mask, runlist_state); |
2970 | 3078 | ||
2971 | if (!mutex_ret) | 3079 | if (!mutex_ret) { |
2972 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | 3080 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); |
3081 | } | ||
2973 | } | 3082 | } |
2974 | 3083 | ||
2975 | void gk20a_fifo_enable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg) | 3084 | void gk20a_fifo_enable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg) |
@@ -3030,8 +3139,9 @@ int gk20a_fifo_disable_engine_activity(struct gk20a *g, | |||
3030 | gr_stat = | 3139 | gr_stat = |
3031 | gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id)); | 3140 | gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id)); |
3032 | if (fifo_engine_status_engine_v(gr_stat) == | 3141 | if (fifo_engine_status_engine_v(gr_stat) == |
3033 | fifo_engine_status_engine_busy_v() && !wait_for_idle) | 3142 | fifo_engine_status_engine_busy_v() && !wait_for_idle) { |
3034 | return -EBUSY; | 3143 | return -EBUSY; |
3144 | } | ||
3035 | 3145 | ||
3036 | mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | 3146 | mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); |
3037 | 3147 | ||
@@ -3042,43 +3152,49 @@ int gk20a_fifo_disable_engine_activity(struct gk20a *g, | |||
3042 | pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(eng_info->pbdma_id)); | 3152 | pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(eng_info->pbdma_id)); |
3043 | chan_stat = fifo_pbdma_status_chan_status_v(pbdma_stat); | 3153 | chan_stat = fifo_pbdma_status_chan_status_v(pbdma_stat); |
3044 | if (chan_stat == fifo_pbdma_status_chan_status_valid_v() || | 3154 | if (chan_stat == fifo_pbdma_status_chan_status_valid_v() || |
3045 | chan_stat == fifo_pbdma_status_chan_status_chsw_save_v()) | 3155 | chan_stat == fifo_pbdma_status_chan_status_chsw_save_v()) { |
3046 | pbdma_chid = fifo_pbdma_status_id_v(pbdma_stat); | 3156 | pbdma_chid = fifo_pbdma_status_id_v(pbdma_stat); |
3047 | else if (chan_stat == fifo_pbdma_status_chan_status_chsw_load_v() || | 3157 | } else if (chan_stat == fifo_pbdma_status_chan_status_chsw_load_v() || |
3048 | chan_stat == fifo_pbdma_status_chan_status_chsw_switch_v()) | 3158 | chan_stat == fifo_pbdma_status_chan_status_chsw_switch_v()) { |
3049 | pbdma_chid = fifo_pbdma_status_next_id_v(pbdma_stat); | 3159 | pbdma_chid = fifo_pbdma_status_next_id_v(pbdma_stat); |
3160 | } | ||
3050 | 3161 | ||
3051 | if (pbdma_chid != FIFO_INVAL_CHANNEL_ID) { | 3162 | if (pbdma_chid != FIFO_INVAL_CHANNEL_ID) { |
3052 | err = g->ops.fifo.preempt_channel(g, pbdma_chid); | 3163 | err = g->ops.fifo.preempt_channel(g, pbdma_chid); |
3053 | if (err) | 3164 | if (err) { |
3054 | goto clean_up; | 3165 | goto clean_up; |
3166 | } | ||
3055 | } | 3167 | } |
3056 | 3168 | ||
3057 | /* chid from engine status */ | 3169 | /* chid from engine status */ |
3058 | eng_stat = gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id)); | 3170 | eng_stat = gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id)); |
3059 | ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); | 3171 | ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); |
3060 | if (ctx_stat == fifo_engine_status_ctx_status_valid_v() || | 3172 | if (ctx_stat == fifo_engine_status_ctx_status_valid_v() || |
3061 | ctx_stat == fifo_engine_status_ctx_status_ctxsw_save_v()) | 3173 | ctx_stat == fifo_engine_status_ctx_status_ctxsw_save_v()) { |
3062 | engine_chid = fifo_engine_status_id_v(eng_stat); | 3174 | engine_chid = fifo_engine_status_id_v(eng_stat); |
3063 | else if (ctx_stat == fifo_engine_status_ctx_status_ctxsw_load_v() || | 3175 | } else if (ctx_stat == fifo_engine_status_ctx_status_ctxsw_load_v() || |
3064 | ctx_stat == fifo_engine_status_ctx_status_ctxsw_switch_v()) | 3176 | ctx_stat == fifo_engine_status_ctx_status_ctxsw_switch_v()) { |
3065 | engine_chid = fifo_engine_status_next_id_v(eng_stat); | 3177 | engine_chid = fifo_engine_status_next_id_v(eng_stat); |
3178 | } | ||
3066 | 3179 | ||
3067 | if (engine_chid != FIFO_INVAL_ENGINE_ID && engine_chid != pbdma_chid) { | 3180 | if (engine_chid != FIFO_INVAL_ENGINE_ID && engine_chid != pbdma_chid) { |
3068 | err = g->ops.fifo.preempt_channel(g, engine_chid); | 3181 | err = g->ops.fifo.preempt_channel(g, engine_chid); |
3069 | if (err) | 3182 | if (err) { |
3070 | goto clean_up; | 3183 | goto clean_up; |
3184 | } | ||
3071 | } | 3185 | } |
3072 | 3186 | ||
3073 | clean_up: | 3187 | clean_up: |
3074 | if (!mutex_ret) | 3188 | if (!mutex_ret) { |
3075 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | 3189 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); |
3190 | } | ||
3076 | 3191 | ||
3077 | if (err) { | 3192 | if (err) { |
3078 | nvgpu_log_fn(g, "failed"); | 3193 | nvgpu_log_fn(g, "failed"); |
3079 | if (gk20a_fifo_enable_engine_activity(g, eng_info)) | 3194 | if (gk20a_fifo_enable_engine_activity(g, eng_info)) { |
3080 | nvgpu_err(g, | 3195 | nvgpu_err(g, |
3081 | "failed to enable gr engine activity"); | 3196 | "failed to enable gr engine activity"); |
3197 | } | ||
3082 | } else { | 3198 | } else { |
3083 | nvgpu_log_fn(g, "done"); | 3199 | nvgpu_log_fn(g, "done"); |
3084 | } | 3200 | } |
@@ -3110,10 +3226,11 @@ int gk20a_fifo_disable_all_engine_activity(struct gk20a *g, | |||
3110 | active_engine_id = g->fifo.active_engines_list[i]; | 3226 | active_engine_id = g->fifo.active_engines_list[i]; |
3111 | err = gk20a_fifo_enable_engine_activity(g, | 3227 | err = gk20a_fifo_enable_engine_activity(g, |
3112 | &g->fifo.engine_info[active_engine_id]); | 3228 | &g->fifo.engine_info[active_engine_id]); |
3113 | if (err) | 3229 | if (err) { |
3114 | nvgpu_err(g, | 3230 | nvgpu_err(g, |
3115 | "failed to re-enable engine %d activity", | 3231 | "failed to re-enable engine %d activity", |
3116 | active_engine_id); | 3232 | active_engine_id); |
3233 | } | ||
3117 | } | 3234 | } |
3118 | } | 3235 | } |
3119 | 3236 | ||
@@ -3133,13 +3250,15 @@ static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id) | |||
3133 | fifo_engine_status_engine_busy_v(); | 3250 | fifo_engine_status_engine_busy_v(); |
3134 | 3251 | ||
3135 | if (engine_busy && | 3252 | if (engine_busy && |
3136 | (f->engine_info[active_engine_id].runlist_id == runlist_id)) | 3253 | (f->engine_info[active_engine_id].runlist_id == runlist_id)) { |
3137 | engines |= BIT(active_engine_id); | 3254 | engines |= BIT(active_engine_id); |
3255 | } | ||
3138 | } | 3256 | } |
3139 | 3257 | ||
3140 | if (engines) | 3258 | if (engines) { |
3141 | gk20a_fifo_recover(g, engines, ~(u32)0, false, false, true, | 3259 | gk20a_fifo_recover(g, engines, ~(u32)0, false, false, true, |
3142 | RC_TYPE_RUNLIST_UPDATE_TIMEOUT); | 3260 | RC_TYPE_RUNLIST_UPDATE_TIMEOUT); |
3261 | } | ||
3143 | } | 3262 | } |
3144 | 3263 | ||
3145 | int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id) | 3264 | int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id) |
@@ -3177,16 +3296,17 @@ void gk20a_get_tsg_runlist_entry(struct tsg_gk20a *tsg, u32 *runlist) | |||
3177 | ram_rl_entry_type_tsg_f() | | 3296 | ram_rl_entry_type_tsg_f() | |
3178 | ram_rl_entry_tsg_length_f(tsg->num_active_channels); | 3297 | ram_rl_entry_tsg_length_f(tsg->num_active_channels); |
3179 | 3298 | ||
3180 | if (tsg->timeslice_timeout) | 3299 | if (tsg->timeslice_timeout) { |
3181 | runlist_entry_0 |= | 3300 | runlist_entry_0 |= |
3182 | ram_rl_entry_timeslice_scale_f(tsg->timeslice_scale) | | 3301 | ram_rl_entry_timeslice_scale_f(tsg->timeslice_scale) | |
3183 | ram_rl_entry_timeslice_timeout_f(tsg->timeslice_timeout); | 3302 | ram_rl_entry_timeslice_timeout_f(tsg->timeslice_timeout); |
3184 | else | 3303 | } else { |
3185 | runlist_entry_0 |= | 3304 | runlist_entry_0 |= |
3186 | ram_rl_entry_timeslice_scale_f( | 3305 | ram_rl_entry_timeslice_scale_f( |
3187 | NVGPU_FIFO_DEFAULT_TIMESLICE_SCALE) | | 3306 | NVGPU_FIFO_DEFAULT_TIMESLICE_SCALE) | |
3188 | ram_rl_entry_timeslice_timeout_f( | 3307 | ram_rl_entry_timeslice_timeout_f( |
3189 | NVGPU_FIFO_DEFAULT_TIMESLICE_TIMEOUT); | 3308 | NVGPU_FIFO_DEFAULT_TIMESLICE_TIMEOUT); |
3309 | } | ||
3190 | 3310 | ||
3191 | runlist[0] = runlist_entry_0; | 3311 | runlist[0] = runlist_entry_0; |
3192 | runlist[1] = 0; | 3312 | runlist[1] = 0; |
@@ -3230,8 +3350,9 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f, | |||
3230 | for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) { | 3350 | for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) { |
3231 | struct tsg_gk20a *tsg = &f->tsg[tsgid]; | 3351 | struct tsg_gk20a *tsg = &f->tsg[tsgid]; |
3232 | 3352 | ||
3233 | if (tsg->interleave_level != cur_level) | 3353 | if (tsg->interleave_level != cur_level) { |
3234 | continue; | 3354 | continue; |
3355 | } | ||
3235 | 3356 | ||
3236 | if (!last_level && !skip_next) { | 3357 | if (!last_level && !skip_next) { |
3237 | runlist_entry = gk20a_runlist_construct_locked(f, | 3358 | runlist_entry = gk20a_runlist_construct_locked(f, |
@@ -3241,8 +3362,9 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f, | |||
3241 | interleave_enabled, | 3362 | interleave_enabled, |
3242 | false, | 3363 | false, |
3243 | entries_left); | 3364 | entries_left); |
3244 | if (!interleave_enabled) | 3365 | if (!interleave_enabled) { |
3245 | skip_next = true; | 3366 | skip_next = true; |
3367 | } | ||
3246 | } | 3368 | } |
3247 | 3369 | ||
3248 | if (*entries_left == 0U) { | 3370 | if (*entries_left == 0U) { |
@@ -3263,8 +3385,9 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f, | |||
3263 | nvgpu_list_for_each_entry(ch, &tsg->ch_list, | 3385 | nvgpu_list_for_each_entry(ch, &tsg->ch_list, |
3264 | channel_gk20a, ch_entry) { | 3386 | channel_gk20a, ch_entry) { |
3265 | if (!test_bit(ch->chid, | 3387 | if (!test_bit(ch->chid, |
3266 | runlist->active_channels)) | 3388 | runlist->active_channels)) { |
3267 | continue; | 3389 | continue; |
3390 | } | ||
3268 | 3391 | ||
3269 | if (*entries_left == 0U) { | 3392 | if (*entries_left == 0U) { |
3270 | nvgpu_rwsem_up_read(&tsg->ch_list_lock); | 3393 | nvgpu_rwsem_up_read(&tsg->ch_list_lock); |
@@ -3285,7 +3408,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f, | |||
3285 | } | 3408 | } |
3286 | 3409 | ||
3287 | /* append entries from higher level if this level is empty */ | 3410 | /* append entries from higher level if this level is empty */ |
3288 | if (!count && !last_level) | 3411 | if (!count && !last_level) { |
3289 | runlist_entry = gk20a_runlist_construct_locked(f, | 3412 | runlist_entry = gk20a_runlist_construct_locked(f, |
3290 | runlist, | 3413 | runlist, |
3291 | cur_level + 1, | 3414 | cur_level + 1, |
@@ -3293,6 +3416,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f, | |||
3293 | interleave_enabled, | 3416 | interleave_enabled, |
3294 | true, | 3417 | true, |
3295 | entries_left); | 3418 | entries_left); |
3419 | } | ||
3296 | 3420 | ||
3297 | /* | 3421 | /* |
3298 | * if previous and this level have entries, append | 3422 | * if previous and this level have entries, append |
@@ -3300,7 +3424,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f, | |||
3300 | * | 3424 | * |
3301 | * ex. dropping from MEDIUM to LOW, need to insert HIGH | 3425 | * ex. dropping from MEDIUM to LOW, need to insert HIGH |
3302 | */ | 3426 | */ |
3303 | if (interleave_enabled && count && !prev_empty && !last_level) | 3427 | if (interleave_enabled && count && !prev_empty && !last_level) { |
3304 | runlist_entry = gk20a_runlist_construct_locked(f, | 3428 | runlist_entry = gk20a_runlist_construct_locked(f, |
3305 | runlist, | 3429 | runlist, |
3306 | cur_level + 1, | 3430 | cur_level + 1, |
@@ -3308,6 +3432,7 @@ u32 *gk20a_runlist_construct_locked(struct fifo_gk20a *f, | |||
3308 | interleave_enabled, | 3432 | interleave_enabled, |
3309 | false, | 3433 | false, |
3310 | entries_left); | 3434 | entries_left); |
3435 | } | ||
3311 | return runlist_entry; | 3436 | return runlist_entry; |
3312 | } | 3437 | } |
3313 | 3438 | ||
@@ -3328,8 +3453,9 @@ int gk20a_fifo_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice) | |||
3328 | struct gk20a *g = tsg->g; | 3453 | struct gk20a *g = tsg->g; |
3329 | 3454 | ||
3330 | if (timeslice < g->min_timeslice_us || | 3455 | if (timeslice < g->min_timeslice_us || |
3331 | timeslice > g->max_timeslice_us) | 3456 | timeslice > g->max_timeslice_us) { |
3332 | return -EINVAL; | 3457 | return -EINVAL; |
3458 | } | ||
3333 | 3459 | ||
3334 | gk20a_channel_get_timescale_from_timeslice(g, timeslice, | 3460 | gk20a_channel_get_timescale_from_timeslice(g, timeslice, |
3335 | &tsg->timeslice_timeout, &tsg->timeslice_scale); | 3461 | &tsg->timeslice_timeout, &tsg->timeslice_scale); |
@@ -3382,23 +3508,28 @@ int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, | |||
3382 | Otherwise, keep active list untouched for suspend/resume. */ | 3508 | Otherwise, keep active list untouched for suspend/resume. */ |
3383 | if (chid != FIFO_INVAL_CHANNEL_ID) { | 3509 | if (chid != FIFO_INVAL_CHANNEL_ID) { |
3384 | ch = &f->channel[chid]; | 3510 | ch = &f->channel[chid]; |
3385 | if (gk20a_is_channel_marked_as_tsg(ch)) | 3511 | if (gk20a_is_channel_marked_as_tsg(ch)) { |
3386 | tsg = &f->tsg[ch->tsgid]; | 3512 | tsg = &f->tsg[ch->tsgid]; |
3513 | } | ||
3387 | 3514 | ||
3388 | if (add) { | 3515 | if (add) { |
3389 | if (test_and_set_bit(chid, | 3516 | if (test_and_set_bit(chid, |
3390 | runlist->active_channels) == 1) | 3517 | runlist->active_channels) == 1) { |
3391 | return 0; | 3518 | return 0; |
3392 | if (tsg && ++tsg->num_active_channels) | 3519 | } |
3520 | if (tsg && ++tsg->num_active_channels) { | ||
3393 | set_bit(f->channel[chid].tsgid, | 3521 | set_bit(f->channel[chid].tsgid, |
3394 | runlist->active_tsgs); | 3522 | runlist->active_tsgs); |
3523 | } | ||
3395 | } else { | 3524 | } else { |
3396 | if (test_and_clear_bit(chid, | 3525 | if (test_and_clear_bit(chid, |
3397 | runlist->active_channels) == 0) | 3526 | runlist->active_channels) == 0) { |
3398 | return 0; | 3527 | return 0; |
3399 | if (tsg && --tsg->num_active_channels == 0) | 3528 | } |
3529 | if (tsg && --tsg->num_active_channels == 0) { | ||
3400 | clear_bit(f->channel[chid].tsgid, | 3530 | clear_bit(f->channel[chid].tsgid, |
3401 | runlist->active_tsgs); | 3531 | runlist->active_tsgs); |
3532 | } | ||
3402 | } | 3533 | } |
3403 | } | 3534 | } |
3404 | 3535 | ||
@@ -3439,8 +3570,10 @@ int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, | |||
3439 | runlist->count = (runlist_end - runlist_entry_base) / | 3570 | runlist->count = (runlist_end - runlist_entry_base) / |
3440 | runlist_entry_words; | 3571 | runlist_entry_words; |
3441 | WARN_ON(runlist->count > f->num_runlist_entries); | 3572 | WARN_ON(runlist->count > f->num_runlist_entries); |
3442 | } else /* suspend to remove all channels */ | 3573 | } else { |
3574 | /* suspend to remove all channels */ | ||
3443 | runlist->count = 0; | 3575 | runlist->count = 0; |
3576 | } | ||
3444 | 3577 | ||
3445 | g->ops.fifo.runlist_hw_submit(g, runlist_id, runlist->count, new_buf); | 3578 | g->ops.fifo.runlist_hw_submit(g, runlist_id, runlist->count, new_buf); |
3446 | 3579 | ||
@@ -3452,8 +3585,9 @@ int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, | |||
3452 | /* trigger runlist update timeout recovery */ | 3585 | /* trigger runlist update timeout recovery */ |
3453 | return ret; | 3586 | return ret; |
3454 | 3587 | ||
3455 | } else if (ret == -EINTR) | 3588 | } else if (ret == -EINTR) { |
3456 | nvgpu_err(g, "runlist update interrupted"); | 3589 | nvgpu_err(g, "runlist update interrupted"); |
3590 | } | ||
3457 | } | 3591 | } |
3458 | 3592 | ||
3459 | runlist->cur_buffer = new_buf; | 3593 | runlist->cur_buffer = new_buf; |
@@ -3470,8 +3604,9 @@ int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid, | |||
3470 | u32 errcode; | 3604 | u32 errcode; |
3471 | unsigned long ulong_runlist_ids = (unsigned long)runlist_ids; | 3605 | unsigned long ulong_runlist_ids = (unsigned long)runlist_ids; |
3472 | 3606 | ||
3473 | if (!g) | 3607 | if (!g) { |
3474 | goto end; | 3608 | goto end; |
3609 | } | ||
3475 | 3610 | ||
3476 | ret = 0; | 3611 | ret = 0; |
3477 | for_each_set_bit(runlist_id, &ulong_runlist_ids, 32) { | 3612 | for_each_set_bit(runlist_id, &ulong_runlist_ids, 32) { |
@@ -3501,14 +3636,17 @@ static int __locked_fifo_reschedule_preempt_next(struct channel_gk20a *ch, | |||
3501 | u32 preempt_type = 0; | 3636 | u32 preempt_type = 0; |
3502 | 3637 | ||
3503 | if (1 != gk20a_fifo_get_engine_ids( | 3638 | if (1 != gk20a_fifo_get_engine_ids( |
3504 | g, &gr_eng_id, 1, ENGINE_GR_GK20A)) | 3639 | g, &gr_eng_id, 1, ENGINE_GR_GK20A)) { |
3505 | return ret; | 3640 | return ret; |
3506 | if (!(runlist->eng_bitmask & (1 << gr_eng_id))) | 3641 | } |
3642 | if (!(runlist->eng_bitmask & (1 << gr_eng_id))) { | ||
3507 | return ret; | 3643 | return ret; |
3644 | } | ||
3508 | 3645 | ||
3509 | if (wait_preempt && gk20a_readl(g, fifo_preempt_r()) & | 3646 | if (wait_preempt && gk20a_readl(g, fifo_preempt_r()) & |
3510 | fifo_preempt_pending_true_f()) | 3647 | fifo_preempt_pending_true_f()) { |
3511 | return ret; | 3648 | return ret; |
3649 | } | ||
3512 | 3650 | ||
3513 | fecsstat0 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0)); | 3651 | fecsstat0 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0)); |
3514 | engstat = gk20a_readl(g, fifo_engine_status_r(gr_eng_id)); | 3652 | engstat = gk20a_readl(g, fifo_engine_status_r(gr_eng_id)); |
@@ -3517,10 +3655,12 @@ static int __locked_fifo_reschedule_preempt_next(struct channel_gk20a *ch, | |||
3517 | /* host switching to next context, preempt that if needed */ | 3655 | /* host switching to next context, preempt that if needed */ |
3518 | preempt_id = fifo_engine_status_next_id_v(engstat); | 3656 | preempt_id = fifo_engine_status_next_id_v(engstat); |
3519 | preempt_type = fifo_engine_status_next_id_type_v(engstat); | 3657 | preempt_type = fifo_engine_status_next_id_type_v(engstat); |
3520 | } else | 3658 | } else { |
3521 | return ret; | 3659 | return ret; |
3522 | if (preempt_id == ch->tsgid && preempt_type) | 3660 | } |
3661 | if (preempt_id == ch->tsgid && preempt_type) { | ||
3523 | return ret; | 3662 | return ret; |
3663 | } | ||
3524 | fecsstat1 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0)); | 3664 | fecsstat1 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(0)); |
3525 | if (fecsstat0 != FECS_MAILBOX_0_ACK_RESTORE || | 3665 | if (fecsstat0 != FECS_MAILBOX_0_ACK_RESTORE || |
3526 | fecsstat1 != FECS_MAILBOX_0_ACK_RESTORE) { | 3666 | fecsstat1 != FECS_MAILBOX_0_ACK_RESTORE) { |
@@ -3559,8 +3699,9 @@ int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next, | |||
3559 | int ret = 0; | 3699 | int ret = 0; |
3560 | 3700 | ||
3561 | runlist = &g->fifo.runlist_info[ch->runlist_id]; | 3701 | runlist = &g->fifo.runlist_info[ch->runlist_id]; |
3562 | if (!nvgpu_mutex_tryacquire(&runlist->runlist_lock)) | 3702 | if (!nvgpu_mutex_tryacquire(&runlist->runlist_lock)) { |
3563 | return -EBUSY; | 3703 | return -EBUSY; |
3704 | } | ||
3564 | 3705 | ||
3565 | mutex_ret = nvgpu_pmu_mutex_acquire( | 3706 | mutex_ret = nvgpu_pmu_mutex_acquire( |
3566 | &g->pmu, PMU_MUTEX_ID_FIFO, &token); | 3707 | &g->pmu, PMU_MUTEX_ID_FIFO, &token); |
@@ -3568,14 +3709,16 @@ int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next, | |||
3568 | g->ops.fifo.runlist_hw_submit( | 3709 | g->ops.fifo.runlist_hw_submit( |
3569 | g, ch->runlist_id, runlist->count, runlist->cur_buffer); | 3710 | g, ch->runlist_id, runlist->count, runlist->cur_buffer); |
3570 | 3711 | ||
3571 | if (preempt_next) | 3712 | if (preempt_next) { |
3572 | __locked_fifo_reschedule_preempt_next(ch, wait_preempt); | 3713 | __locked_fifo_reschedule_preempt_next(ch, wait_preempt); |
3714 | } | ||
3573 | 3715 | ||
3574 | gk20a_fifo_runlist_wait_pending(g, ch->runlist_id); | 3716 | gk20a_fifo_runlist_wait_pending(g, ch->runlist_id); |
3575 | 3717 | ||
3576 | if (!mutex_ret) | 3718 | if (!mutex_ret) { |
3577 | nvgpu_pmu_mutex_release( | 3719 | nvgpu_pmu_mutex_release( |
3578 | &g->pmu, PMU_MUTEX_ID_FIFO, &token); | 3720 | &g->pmu, PMU_MUTEX_ID_FIFO, &token); |
3721 | } | ||
3579 | nvgpu_mutex_release(&runlist->runlist_lock); | 3722 | nvgpu_mutex_release(&runlist->runlist_lock); |
3580 | 3723 | ||
3581 | return ret; | 3724 | return ret; |
@@ -3605,13 +3748,15 @@ int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 chid, | |||
3605 | ret = gk20a_fifo_update_runlist_locked(g, runlist_id, chid, add, | 3748 | ret = gk20a_fifo_update_runlist_locked(g, runlist_id, chid, add, |
3606 | wait_for_finish); | 3749 | wait_for_finish); |
3607 | 3750 | ||
3608 | if (!mutex_ret) | 3751 | if (!mutex_ret) { |
3609 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | 3752 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); |
3753 | } | ||
3610 | 3754 | ||
3611 | nvgpu_mutex_release(&runlist->runlist_lock); | 3755 | nvgpu_mutex_release(&runlist->runlist_lock); |
3612 | 3756 | ||
3613 | if (ret == -ETIMEDOUT) | 3757 | if (ret == -ETIMEDOUT) { |
3614 | gk20a_fifo_runlist_reset_engines(g, runlist_id); | 3758 | gk20a_fifo_runlist_reset_engines(g, runlist_id); |
3759 | } | ||
3615 | 3760 | ||
3616 | return ret; | 3761 | return ret; |
3617 | } | 3762 | } |
@@ -3621,9 +3766,10 @@ int gk20a_fifo_suspend(struct gk20a *g) | |||
3621 | nvgpu_log_fn(g, " "); | 3766 | nvgpu_log_fn(g, " "); |
3622 | 3767 | ||
3623 | /* stop bar1 snooping */ | 3768 | /* stop bar1 snooping */ |
3624 | if (g->ops.mm.is_bar1_supported(g)) | 3769 | if (g->ops.mm.is_bar1_supported(g)) { |
3625 | gk20a_writel(g, fifo_bar1_base_r(), | 3770 | gk20a_writel(g, fifo_bar1_base_r(), |
3626 | fifo_bar1_base_valid_false_f()); | 3771 | fifo_bar1_base_valid_false_f()); |
3772 | } | ||
3627 | 3773 | ||
3628 | /* disable fifo intr */ | 3774 | /* disable fifo intr */ |
3629 | gk20a_writel(g, fifo_intr_en_0_r(), 0); | 3775 | gk20a_writel(g, fifo_intr_en_0_r(), 0); |
@@ -3636,10 +3782,11 @@ int gk20a_fifo_suspend(struct gk20a *g) | |||
3636 | bool gk20a_fifo_mmu_fault_pending(struct gk20a *g) | 3782 | bool gk20a_fifo_mmu_fault_pending(struct gk20a *g) |
3637 | { | 3783 | { |
3638 | if (gk20a_readl(g, fifo_intr_0_r()) & | 3784 | if (gk20a_readl(g, fifo_intr_0_r()) & |
3639 | fifo_intr_0_mmu_fault_pending_f()) | 3785 | fifo_intr_0_mmu_fault_pending_f()) { |
3640 | return true; | 3786 | return true; |
3641 | else | 3787 | } else { |
3642 | return false; | 3788 | return false; |
3789 | } | ||
3643 | } | 3790 | } |
3644 | 3791 | ||
3645 | bool gk20a_fifo_is_engine_busy(struct gk20a *g) | 3792 | bool gk20a_fifo_is_engine_busy(struct gk20a *g) |
@@ -3651,8 +3798,9 @@ bool gk20a_fifo_is_engine_busy(struct gk20a *g) | |||
3651 | for (i = 0; i < host_num_engines; i++) { | 3798 | for (i = 0; i < host_num_engines; i++) { |
3652 | u32 status = gk20a_readl(g, fifo_engine_status_r(i)); | 3799 | u32 status = gk20a_readl(g, fifo_engine_status_r(i)); |
3653 | if (fifo_engine_status_engine_v(status) == | 3800 | if (fifo_engine_status_engine_v(status) == |
3654 | fifo_engine_status_engine_busy_v()) | 3801 | fifo_engine_status_engine_busy_v()) { |
3655 | return true; | 3802 | return true; |
3803 | } | ||
3656 | } | 3804 | } |
3657 | return false; | 3805 | return false; |
3658 | } | 3806 | } |
@@ -3704,10 +3852,11 @@ u32 gk20a_fifo_get_pbdma_signature(struct gk20a *g) | |||
3704 | struct channel_gk20a *gk20a_fifo_channel_from_chid(struct gk20a *g, | 3852 | struct channel_gk20a *gk20a_fifo_channel_from_chid(struct gk20a *g, |
3705 | u32 chid) | 3853 | u32 chid) |
3706 | { | 3854 | { |
3707 | if (chid != FIFO_INVAL_CHANNEL_ID) | 3855 | if (chid != FIFO_INVAL_CHANNEL_ID) { |
3708 | return g->fifo.channel + chid; | 3856 | return g->fifo.channel + chid; |
3709 | else | 3857 | } else { |
3710 | return NULL; | 3858 | return NULL; |
3859 | } | ||
3711 | } | 3860 | } |
3712 | 3861 | ||
3713 | static const char * const ccsr_chan_status_str[] = { | 3862 | static const char * const ccsr_chan_status_str[] = { |
@@ -3745,18 +3894,20 @@ static const char * const not_found_str[] = { | |||
3745 | 3894 | ||
3746 | const char *gk20a_decode_ccsr_chan_status(u32 index) | 3895 | const char *gk20a_decode_ccsr_chan_status(u32 index) |
3747 | { | 3896 | { |
3748 | if (index >= ARRAY_SIZE(ccsr_chan_status_str)) | 3897 | if (index >= ARRAY_SIZE(ccsr_chan_status_str)) { |
3749 | return not_found_str[0]; | 3898 | return not_found_str[0]; |
3750 | else | 3899 | } else { |
3751 | return ccsr_chan_status_str[index]; | 3900 | return ccsr_chan_status_str[index]; |
3901 | } | ||
3752 | } | 3902 | } |
3753 | 3903 | ||
3754 | const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index) | 3904 | const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index) |
3755 | { | 3905 | { |
3756 | if (index >= ARRAY_SIZE(pbdma_chan_eng_ctx_status_str)) | 3906 | if (index >= ARRAY_SIZE(pbdma_chan_eng_ctx_status_str)) { |
3757 | return not_found_str[0]; | 3907 | return not_found_str[0]; |
3758 | else | 3908 | } else { |
3759 | return pbdma_chan_eng_ctx_status_str[index]; | 3909 | return pbdma_chan_eng_ctx_status_str[index]; |
3910 | } | ||
3760 | } | 3911 | } |
3761 | 3912 | ||
3762 | bool gk20a_fifo_channel_status_is_next(struct gk20a *g, u32 chid) | 3913 | bool gk20a_fifo_channel_status_is_next(struct gk20a *g, u32 chid) |
@@ -3792,11 +3943,13 @@ void gk20a_dump_channel_status_ramfc(struct gk20a *g, | |||
3792 | struct channel_gk20a *c = g->fifo.channel + chid; | 3943 | struct channel_gk20a *c = g->fifo.channel + chid; |
3793 | struct nvgpu_semaphore_int *hw_sema = NULL; | 3944 | struct nvgpu_semaphore_int *hw_sema = NULL; |
3794 | 3945 | ||
3795 | if (c->hw_sema) | 3946 | if (c->hw_sema) { |
3796 | hw_sema = c->hw_sema; | 3947 | hw_sema = c->hw_sema; |
3948 | } | ||
3797 | 3949 | ||
3798 | if (!ch_state) | 3950 | if (!ch_state) { |
3799 | return; | 3951 | return; |
3952 | } | ||
3800 | 3953 | ||
3801 | inst_mem = &ch_state->inst_block[0]; | 3954 | inst_mem = &ch_state->inst_block[0]; |
3802 | 3955 | ||
@@ -3831,12 +3984,13 @@ void gk20a_dump_channel_status_ramfc(struct gk20a *g, | |||
3831 | inst_mem[ram_fc_semaphoreb_w()], | 3984 | inst_mem[ram_fc_semaphoreb_w()], |
3832 | inst_mem[ram_fc_semaphorec_w()], | 3985 | inst_mem[ram_fc_semaphorec_w()], |
3833 | inst_mem[ram_fc_semaphored_w()]); | 3986 | inst_mem[ram_fc_semaphored_w()]); |
3834 | if (hw_sema) | 3987 | if (hw_sema) { |
3835 | gk20a_debug_output(o, "SEMA STATE: value: 0x%08x " | 3988 | gk20a_debug_output(o, "SEMA STATE: value: 0x%08x " |
3836 | "next_val: 0x%08x addr: 0x%010llx\n", | 3989 | "next_val: 0x%08x addr: 0x%010llx\n", |
3837 | __nvgpu_semaphore_read(hw_sema), | 3990 | __nvgpu_semaphore_read(hw_sema), |
3838 | nvgpu_atomic_read(&hw_sema->next_value), | 3991 | nvgpu_atomic_read(&hw_sema->next_value), |
3839 | nvgpu_hw_sema_addr(hw_sema)); | 3992 | nvgpu_hw_sema_addr(hw_sema)); |
3993 | } | ||
3840 | 3994 | ||
3841 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | 3995 | #ifdef CONFIG_TEGRA_GK20A_NVHOST |
3842 | if ((pbdma_syncpointb_op_v(syncpointb) == pbdma_syncpointb_op_wait_v()) | 3996 | if ((pbdma_syncpointb_op_v(syncpointb) == pbdma_syncpointb_op_wait_v()) |
@@ -3874,15 +4028,17 @@ void gk20a_debug_dump_all_channel_status_ramfc(struct gk20a *g, | |||
3874 | ram_in_alloc_size_v()); | 4028 | ram_in_alloc_size_v()); |
3875 | /* ref taken stays to below loop with | 4029 | /* ref taken stays to below loop with |
3876 | * successful allocs */ | 4030 | * successful allocs */ |
3877 | if (!ch_state[chid]) | 4031 | if (!ch_state[chid]) { |
3878 | gk20a_channel_put(ch); | 4032 | gk20a_channel_put(ch); |
4033 | } | ||
3879 | } | 4034 | } |
3880 | } | 4035 | } |
3881 | 4036 | ||
3882 | for (chid = 0; chid < f->num_channels; chid++) { | 4037 | for (chid = 0; chid < f->num_channels; chid++) { |
3883 | struct channel_gk20a *ch = &f->channel[chid]; | 4038 | struct channel_gk20a *ch = &f->channel[chid]; |
3884 | if (!ch_state[chid]) | 4039 | if (!ch_state[chid]) { |
3885 | continue; | 4040 | continue; |
4041 | } | ||
3886 | 4042 | ||
3887 | ch_state[chid]->pid = ch->pid; | 4043 | ch_state[chid]->pid = ch->pid; |
3888 | ch_state[chid]->refs = nvgpu_atomic_read(&ch->ref_count); | 4044 | ch_state[chid]->refs = nvgpu_atomic_read(&ch->ref_count); |
@@ -3964,10 +4120,12 @@ void gk20a_dump_eng_status(struct gk20a *g, | |||
3964 | "tsg" : "channel", | 4120 | "tsg" : "channel", |
3965 | gk20a_decode_pbdma_chan_eng_ctx_status(ctx_status)); | 4121 | gk20a_decode_pbdma_chan_eng_ctx_status(ctx_status)); |
3966 | 4122 | ||
3967 | if (fifo_engine_status_faulted_v(status)) | 4123 | if (fifo_engine_status_faulted_v(status)) { |
3968 | gk20a_debug_output(o, "faulted "); | 4124 | gk20a_debug_output(o, "faulted "); |
3969 | if (fifo_engine_status_engine_v(status)) | 4125 | } |
4126 | if (fifo_engine_status_engine_v(status)) { | ||
3970 | gk20a_debug_output(o, "busy "); | 4127 | gk20a_debug_output(o, "busy "); |
4128 | } | ||
3971 | gk20a_debug_output(o, "\n"); | 4129 | gk20a_debug_output(o, "\n"); |
3972 | } | 4130 | } |
3973 | gk20a_debug_output(o, "\n"); | 4131 | gk20a_debug_output(o, "\n"); |
@@ -4088,8 +4246,9 @@ int gk20a_fifo_setup_ramfc(struct channel_gk20a *c, | |||
4088 | 4246 | ||
4089 | nvgpu_mem_wr32(g, mem, ram_fc_chid_w(), ram_fc_chid_id_f(c->chid)); | 4247 | nvgpu_mem_wr32(g, mem, ram_fc_chid_w(), ram_fc_chid_id_f(c->chid)); |
4090 | 4248 | ||
4091 | if (c->is_privileged_channel) | 4249 | if (c->is_privileged_channel) { |
4092 | gk20a_fifo_setup_ramfc_for_privileged_channel(c); | 4250 | gk20a_fifo_setup_ramfc_for_privileged_channel(c); |
4251 | } | ||
4093 | 4252 | ||
4094 | return gk20a_fifo_commit_userd(c); | 4253 | return gk20a_fifo_commit_userd(c); |
4095 | } | 4254 | } |
@@ -4143,8 +4302,9 @@ int gk20a_fifo_alloc_inst(struct gk20a *g, struct channel_gk20a *ch) | |||
4143 | nvgpu_log_fn(g, " "); | 4302 | nvgpu_log_fn(g, " "); |
4144 | 4303 | ||
4145 | err = g->ops.mm.alloc_inst_block(g, &ch->inst_block); | 4304 | err = g->ops.mm.alloc_inst_block(g, &ch->inst_block); |
4146 | if (err) | 4305 | if (err) { |
4147 | return err; | 4306 | return err; |
4307 | } | ||
4148 | 4308 | ||
4149 | nvgpu_log_info(g, "channel %d inst block physical addr: 0x%16llx", | 4309 | nvgpu_log_info(g, "channel %d inst block physical addr: 0x%16llx", |
4150 | ch->chid, nvgpu_inst_block_addr(g, &ch->inst_block)); | 4310 | ch->chid, nvgpu_inst_block_addr(g, &ch->inst_block)); |
@@ -4189,16 +4349,18 @@ u32 gk20a_fifo_pbdma_acquire_val(u64 timeout) | |||
4189 | val = pbdma_acquire_retry_man_2_f() | | 4349 | val = pbdma_acquire_retry_man_2_f() | |
4190 | pbdma_acquire_retry_exp_2_f(); | 4350 | pbdma_acquire_retry_exp_2_f(); |
4191 | 4351 | ||
4192 | if (!timeout) | 4352 | if (!timeout) { |
4193 | return val; | 4353 | return val; |
4354 | } | ||
4194 | 4355 | ||
4195 | timeout *= 80UL; | 4356 | timeout *= 80UL; |
4196 | do_div(timeout, 100); /* set acquire timeout to 80% of channel wdt */ | 4357 | do_div(timeout, 100); /* set acquire timeout to 80% of channel wdt */ |
4197 | timeout *= 1000000UL; /* ms -> ns */ | 4358 | timeout *= 1000000UL; /* ms -> ns */ |
4198 | do_div(timeout, 1024); /* in unit of 1024ns */ | 4359 | do_div(timeout, 1024); /* in unit of 1024ns */ |
4199 | val_len = fls(timeout >> 32) + 32; | 4360 | val_len = fls(timeout >> 32) + 32; |
4200 | if (val_len == 32) | 4361 | if (val_len == 32) { |
4201 | val_len = fls(timeout); | 4362 | val_len = fls(timeout); |
4363 | } | ||
4202 | if (val_len > 16U + pbdma_acquire_timeout_exp_max_v()) { /* man: 16bits */ | 4364 | if (val_len > 16U + pbdma_acquire_timeout_exp_max_v()) { /* man: 16bits */ |
4203 | exp = pbdma_acquire_timeout_exp_max_v(); | 4365 | exp = pbdma_acquire_timeout_exp_max_v(); |
4204 | man = pbdma_acquire_timeout_man_max_v(); | 4366 | man = pbdma_acquire_timeout_man_max_v(); |
diff --git a/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c b/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c index e6e16511..2f715ae1 100644 --- a/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c | |||
@@ -35,10 +35,10 @@ static int gk20a_flcn_reset(struct nvgpu_falcon *flcn) | |||
35 | u32 unit_status = 0; | 35 | u32 unit_status = 0; |
36 | int status = 0; | 36 | int status = 0; |
37 | 37 | ||
38 | if (flcn->flcn_engine_dep_ops.reset_eng) | 38 | if (flcn->flcn_engine_dep_ops.reset_eng) { |
39 | /* falcon & engine reset */ | 39 | /* falcon & engine reset */ |
40 | status = flcn->flcn_engine_dep_ops.reset_eng(g); | 40 | status = flcn->flcn_engine_dep_ops.reset_eng(g); |
41 | else { | 41 | } else { |
42 | /* do falcon CPU hard reset */ | 42 | /* do falcon CPU hard reset */ |
43 | unit_status = gk20a_readl(g, base_addr + | 43 | unit_status = gk20a_readl(g, base_addr + |
44 | falcon_falcon_cpuctl_r()); | 44 | falcon_falcon_cpuctl_r()); |
@@ -62,9 +62,10 @@ static bool gk20a_flcn_clear_halt_interrupt_status(struct nvgpu_falcon *flcn) | |||
62 | data = gk20a_readl(g, (base_addr + falcon_falcon_irqstat_r())); | 62 | data = gk20a_readl(g, (base_addr + falcon_falcon_irqstat_r())); |
63 | 63 | ||
64 | if ((data & falcon_falcon_irqstat_halt_true_f()) != | 64 | if ((data & falcon_falcon_irqstat_halt_true_f()) != |
65 | falcon_falcon_irqstat_halt_true_f()) | 65 | falcon_falcon_irqstat_halt_true_f()) { |
66 | /*halt irq is clear*/ | 66 | /*halt irq is clear*/ |
67 | status = true; | 67 | status = true; |
68 | } | ||
68 | 69 | ||
69 | return status; | 70 | return status; |
70 | } | 71 | } |
@@ -86,9 +87,10 @@ static void gk20a_flcn_set_irq(struct nvgpu_falcon *flcn, bool enable) | |||
86 | flcn->intr_mask); | 87 | flcn->intr_mask); |
87 | gk20a_writel(g, base_addr + falcon_falcon_irqdest_r(), | 88 | gk20a_writel(g, base_addr + falcon_falcon_irqdest_r(), |
88 | flcn->intr_dest); | 89 | flcn->intr_dest); |
89 | } else | 90 | } else { |
90 | gk20a_writel(g, base_addr + falcon_falcon_irqmclr_r(), | 91 | gk20a_writel(g, base_addr + falcon_falcon_irqmclr_r(), |
91 | 0xffffffff); | 92 | 0xffffffff); |
93 | } | ||
92 | } | 94 | } |
93 | 95 | ||
94 | static bool gk20a_is_falcon_cpu_halted(struct nvgpu_falcon *flcn) | 96 | static bool gk20a_is_falcon_cpu_halted(struct nvgpu_falcon *flcn) |
@@ -112,10 +114,11 @@ static bool gk20a_is_falcon_idle(struct nvgpu_falcon *flcn) | |||
112 | base_addr + falcon_falcon_idlestate_r()); | 114 | base_addr + falcon_falcon_idlestate_r()); |
113 | 115 | ||
114 | if (falcon_falcon_idlestate_falcon_busy_v(unit_status) == 0 && | 116 | if (falcon_falcon_idlestate_falcon_busy_v(unit_status) == 0 && |
115 | falcon_falcon_idlestate_ext_busy_v(unit_status) == 0) | 117 | falcon_falcon_idlestate_ext_busy_v(unit_status) == 0) { |
116 | status = true; | 118 | status = true; |
117 | else | 119 | } else { |
118 | status = false; | 120 | status = false; |
121 | } | ||
119 | 122 | ||
120 | return status; | 123 | return status; |
121 | } | 124 | } |
@@ -131,10 +134,11 @@ static bool gk20a_is_falcon_scrubbing_done(struct nvgpu_falcon *flcn) | |||
131 | base_addr + falcon_falcon_dmactl_r()); | 134 | base_addr + falcon_falcon_dmactl_r()); |
132 | 135 | ||
133 | if (unit_status & (falcon_falcon_dmactl_dmem_scrubbing_m() | | 136 | if (unit_status & (falcon_falcon_dmactl_dmem_scrubbing_m() | |
134 | falcon_falcon_dmactl_imem_scrubbing_m())) | 137 | falcon_falcon_dmactl_imem_scrubbing_m())) { |
135 | status = false; | 138 | status = false; |
136 | else | 139 | } else { |
137 | status = true; | 140 | status = true; |
141 | } | ||
138 | 142 | ||
139 | return status; | 143 | return status; |
140 | } | 144 | } |
@@ -147,12 +151,13 @@ static u32 gk20a_falcon_get_mem_size(struct nvgpu_falcon *flcn, | |||
147 | u32 hw_cfg_reg = gk20a_readl(g, | 151 | u32 hw_cfg_reg = gk20a_readl(g, |
148 | flcn->flcn_base + falcon_falcon_hwcfg_r()); | 152 | flcn->flcn_base + falcon_falcon_hwcfg_r()); |
149 | 153 | ||
150 | if (mem_type == MEM_DMEM) | 154 | if (mem_type == MEM_DMEM) { |
151 | mem_size = falcon_falcon_hwcfg_dmem_size_v(hw_cfg_reg) | 155 | mem_size = falcon_falcon_hwcfg_dmem_size_v(hw_cfg_reg) |
152 | << GK20A_PMU_DMEM_BLKSIZE2; | 156 | << GK20A_PMU_DMEM_BLKSIZE2; |
153 | else | 157 | } else { |
154 | mem_size = falcon_falcon_hwcfg_imem_size_v(hw_cfg_reg) | 158 | mem_size = falcon_falcon_hwcfg_imem_size_v(hw_cfg_reg) |
155 | << GK20A_PMU_DMEM_BLKSIZE2; | 159 | << GK20A_PMU_DMEM_BLKSIZE2; |
160 | } | ||
156 | 161 | ||
157 | return mem_size; | 162 | return mem_size; |
158 | } | 163 | } |
@@ -416,12 +421,13 @@ static u32 gk20a_falcon_mailbox_read(struct nvgpu_falcon *flcn, | |||
416 | struct gk20a *g = flcn->g; | 421 | struct gk20a *g = flcn->g; |
417 | u32 data = 0; | 422 | u32 data = 0; |
418 | 423 | ||
419 | if (mailbox_index < FALCON_MAILBOX_COUNT) | 424 | if (mailbox_index < FALCON_MAILBOX_COUNT) { |
420 | data = gk20a_readl(g, flcn->flcn_base + (mailbox_index ? | 425 | data = gk20a_readl(g, flcn->flcn_base + (mailbox_index ? |
421 | falcon_falcon_mailbox1_r() : | 426 | falcon_falcon_mailbox1_r() : |
422 | falcon_falcon_mailbox0_r())); | 427 | falcon_falcon_mailbox0_r())); |
423 | else | 428 | } else { |
424 | nvgpu_err(g, "incorrect mailbox id %d", mailbox_index); | 429 | nvgpu_err(g, "incorrect mailbox id %d", mailbox_index); |
430 | } | ||
425 | 431 | ||
426 | return data; | 432 | return data; |
427 | } | 433 | } |
@@ -431,13 +437,14 @@ static void gk20a_falcon_mailbox_write(struct nvgpu_falcon *flcn, | |||
431 | { | 437 | { |
432 | struct gk20a *g = flcn->g; | 438 | struct gk20a *g = flcn->g; |
433 | 439 | ||
434 | if (mailbox_index < FALCON_MAILBOX_COUNT) | 440 | if (mailbox_index < FALCON_MAILBOX_COUNT) { |
435 | gk20a_writel(g, flcn->flcn_base + (mailbox_index ? | 441 | gk20a_writel(g, flcn->flcn_base + (mailbox_index ? |
436 | falcon_falcon_mailbox1_r() : | 442 | falcon_falcon_mailbox1_r() : |
437 | falcon_falcon_mailbox0_r()), | 443 | falcon_falcon_mailbox0_r()), |
438 | data); | 444 | data); |
439 | else | 445 | } else { |
440 | nvgpu_err(g, "incorrect mailbox id %d", mailbox_index); | 446 | nvgpu_err(g, "incorrect mailbox id %d", mailbox_index); |
447 | } | ||
441 | } | 448 | } |
442 | 449 | ||
443 | static int gk20a_falcon_bl_bootstrap(struct nvgpu_falcon *flcn, | 450 | static int gk20a_falcon_bl_bootstrap(struct nvgpu_falcon *flcn, |
@@ -739,7 +746,8 @@ void gk20a_falcon_hal_sw_init(struct nvgpu_falcon *flcn) | |||
739 | if (flcn->is_falcon_supported) { | 746 | if (flcn->is_falcon_supported) { |
740 | nvgpu_mutex_init(&flcn->copy_lock); | 747 | nvgpu_mutex_init(&flcn->copy_lock); |
741 | gk20a_falcon_ops(flcn); | 748 | gk20a_falcon_ops(flcn); |
742 | } else | 749 | } else { |
743 | nvgpu_log_info(g, "falcon 0x%x not supported on %s", | 750 | nvgpu_log_info(g, "falcon 0x%x not supported on %s", |
744 | flcn->flcn_id, g->name); | 751 | flcn->flcn_id, g->name); |
752 | } | ||
745 | } | 753 | } |
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c index 859a7b6a..f5e35927 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gk20a.c | |||
@@ -74,16 +74,18 @@ int gk20a_detect_chip(struct gk20a *g) | |||
74 | { | 74 | { |
75 | struct nvgpu_gpu_params *p = &g->params; | 75 | struct nvgpu_gpu_params *p = &g->params; |
76 | 76 | ||
77 | if (p->gpu_arch) | 77 | if (p->gpu_arch) { |
78 | return 0; | 78 | return 0; |
79 | } | ||
79 | 80 | ||
80 | gk20a_mc_boot_0(g, &p->gpu_arch, &p->gpu_impl, &p->gpu_rev); | 81 | gk20a_mc_boot_0(g, &p->gpu_arch, &p->gpu_impl, &p->gpu_rev); |
81 | 82 | ||
82 | if ((p->gpu_arch + p->gpu_impl) == NVGPU_GPUID_GV11B) { | 83 | if ((p->gpu_arch + p->gpu_impl) == NVGPU_GPUID_GV11B) { |
83 | 84 | ||
84 | /* overwrite gpu revison for A02 */ | 85 | /* overwrite gpu revison for A02 */ |
85 | if (!nvgpu_is_soc_t194_a01(g)) | 86 | if (!nvgpu_is_soc_t194_a01(g)) { |
86 | p->gpu_rev = 0xa2; | 87 | p->gpu_rev = 0xa2; |
88 | } | ||
87 | 89 | ||
88 | } | 90 | } |
89 | 91 | ||
@@ -114,13 +116,15 @@ int gk20a_prepare_poweroff(struct gk20a *g) | |||
114 | 116 | ||
115 | if (g->ops.fifo.channel_suspend) { | 117 | if (g->ops.fifo.channel_suspend) { |
116 | ret = g->ops.fifo.channel_suspend(g); | 118 | ret = g->ops.fifo.channel_suspend(g); |
117 | if (ret) | 119 | if (ret) { |
118 | return ret; | 120 | return ret; |
121 | } | ||
119 | } | 122 | } |
120 | 123 | ||
121 | /* disable elpg before gr or fifo suspend */ | 124 | /* disable elpg before gr or fifo suspend */ |
122 | if (g->ops.pmu.is_pmu_supported(g)) | 125 | if (g->ops.pmu.is_pmu_supported(g)) { |
123 | ret |= nvgpu_pmu_destroy(g); | 126 | ret |= nvgpu_pmu_destroy(g); |
127 | } | ||
124 | 128 | ||
125 | ret |= gk20a_gr_suspend(g); | 129 | ret |= gk20a_gr_suspend(g); |
126 | ret |= nvgpu_mm_suspend(g); | 130 | ret |= nvgpu_mm_suspend(g); |
@@ -129,11 +133,13 @@ int gk20a_prepare_poweroff(struct gk20a *g) | |||
129 | gk20a_ce_suspend(g); | 133 | gk20a_ce_suspend(g); |
130 | 134 | ||
131 | /* Disable GPCPLL */ | 135 | /* Disable GPCPLL */ |
132 | if (g->ops.clk.suspend_clk_support) | 136 | if (g->ops.clk.suspend_clk_support) { |
133 | ret |= g->ops.clk.suspend_clk_support(g); | 137 | ret |= g->ops.clk.suspend_clk_support(g); |
138 | } | ||
134 | 139 | ||
135 | if (nvgpu_is_enabled(g, NVGPU_PMU_PSTATE)) | 140 | if (nvgpu_is_enabled(g, NVGPU_PMU_PSTATE)) { |
136 | gk20a_deinit_pstate_support(g); | 141 | gk20a_deinit_pstate_support(g); |
142 | } | ||
137 | 143 | ||
138 | gk20a_mask_interrupts(g); | 144 | gk20a_mask_interrupts(g); |
139 | 145 | ||
@@ -151,8 +157,9 @@ int gk20a_finalize_poweron(struct gk20a *g) | |||
151 | 157 | ||
152 | nvgpu_log_fn(g, " "); | 158 | nvgpu_log_fn(g, " "); |
153 | 159 | ||
154 | if (g->power_on) | 160 | if (g->power_on) { |
155 | return 0; | 161 | return 0; |
162 | } | ||
156 | 163 | ||
157 | g->power_on = true; | 164 | g->power_on = true; |
158 | 165 | ||
@@ -170,23 +177,27 @@ int gk20a_finalize_poweron(struct gk20a *g) | |||
170 | * buffers. | 177 | * buffers. |
171 | */ | 178 | */ |
172 | err = nvgpu_pd_cache_init(g); | 179 | err = nvgpu_pd_cache_init(g); |
173 | if (err) | 180 | if (err) { |
174 | return err; | 181 | return err; |
182 | } | ||
175 | 183 | ||
176 | /* init interface layer support for PMU falcon */ | 184 | /* init interface layer support for PMU falcon */ |
177 | nvgpu_flcn_sw_init(g, FALCON_ID_PMU); | 185 | nvgpu_flcn_sw_init(g, FALCON_ID_PMU); |
178 | nvgpu_flcn_sw_init(g, FALCON_ID_SEC2); | 186 | nvgpu_flcn_sw_init(g, FALCON_ID_SEC2); |
179 | nvgpu_flcn_sw_init(g, FALCON_ID_NVDEC); | 187 | nvgpu_flcn_sw_init(g, FALCON_ID_NVDEC); |
180 | 188 | ||
181 | if (g->ops.bios.init) | 189 | if (g->ops.bios.init) { |
182 | err = g->ops.bios.init(g); | 190 | err = g->ops.bios.init(g); |
183 | if (err) | 191 | } |
192 | if (err) { | ||
184 | goto done; | 193 | goto done; |
194 | } | ||
185 | 195 | ||
186 | g->ops.bus.init_hw(g); | 196 | g->ops.bus.init_hw(g); |
187 | 197 | ||
188 | if (g->ops.clk.disable_slowboot) | 198 | if (g->ops.clk.disable_slowboot) { |
189 | g->ops.clk.disable_slowboot(g); | 199 | g->ops.clk.disable_slowboot(g); |
200 | } | ||
190 | 201 | ||
191 | g->ops.priv_ring.enable_priv_ring(g); | 202 | g->ops.priv_ring.enable_priv_ring(g); |
192 | 203 | ||
@@ -253,8 +264,9 @@ int gk20a_finalize_poweron(struct gk20a *g) | |||
253 | goto done; | 264 | goto done; |
254 | } | 265 | } |
255 | 266 | ||
256 | if (g->ops.therm.elcg_init_idle_filters) | 267 | if (g->ops.therm.elcg_init_idle_filters) { |
257 | g->ops.therm.elcg_init_idle_filters(g); | 268 | g->ops.therm.elcg_init_idle_filters(g); |
269 | } | ||
258 | 270 | ||
259 | g->ops.mc.intr_enable(g); | 271 | g->ops.mc.intr_enable(g); |
260 | 272 | ||
@@ -265,8 +277,9 @@ int gk20a_finalize_poweron(struct gk20a *g) | |||
265 | } | 277 | } |
266 | 278 | ||
267 | if (g->ops.pmu.is_pmu_supported(g)) { | 279 | if (g->ops.pmu.is_pmu_supported(g)) { |
268 | if (g->ops.pmu.prepare_ucode) | 280 | if (g->ops.pmu.prepare_ucode) { |
269 | err = g->ops.pmu.prepare_ucode(g); | 281 | err = g->ops.pmu.prepare_ucode(g); |
282 | } | ||
270 | if (err) { | 283 | if (err) { |
271 | nvgpu_err(g, "failed to init pmu ucode"); | 284 | nvgpu_err(g, "failed to init pmu ucode"); |
272 | goto done; | 285 | goto done; |
@@ -314,9 +327,9 @@ int gk20a_finalize_poweron(struct gk20a *g) | |||
314 | } | 327 | } |
315 | } | 328 | } |
316 | 329 | ||
317 | if (g->ops.pmu_ver.clk.clk_set_boot_clk && nvgpu_is_enabled(g, NVGPU_PMU_PSTATE)) | 330 | if (g->ops.pmu_ver.clk.clk_set_boot_clk && nvgpu_is_enabled(g, NVGPU_PMU_PSTATE)) { |
318 | g->ops.pmu_ver.clk.clk_set_boot_clk(g); | 331 | g->ops.pmu_ver.clk.clk_set_boot_clk(g); |
319 | else { | 332 | } else { |
320 | err = nvgpu_clk_arb_init_arbiter(g); | 333 | err = nvgpu_clk_arb_init_arbiter(g); |
321 | if (err) { | 334 | if (err) { |
322 | nvgpu_err(g, "failed to init clk arb"); | 335 | nvgpu_err(g, "failed to init clk arb"); |
@@ -350,8 +363,9 @@ int gk20a_finalize_poweron(struct gk20a *g) | |||
350 | if (g->ops.xve.available_speeds) { | 363 | if (g->ops.xve.available_speeds) { |
351 | u32 speed; | 364 | u32 speed; |
352 | 365 | ||
353 | if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_ASPM) && g->ops.xve.disable_aspm) | 366 | if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_ASPM) && g->ops.xve.disable_aspm) { |
354 | g->ops.xve.disable_aspm(g); | 367 | g->ops.xve.disable_aspm(g); |
368 | } | ||
355 | 369 | ||
356 | g->ops.xve.available_speeds(g, &speed); | 370 | g->ops.xve.available_speeds(g, &speed); |
357 | 371 | ||
@@ -374,12 +388,14 @@ int gk20a_finalize_poweron(struct gk20a *g) | |||
374 | } | 388 | } |
375 | #endif | 389 | #endif |
376 | 390 | ||
377 | if (g->ops.fifo.channel_resume) | 391 | if (g->ops.fifo.channel_resume) { |
378 | g->ops.fifo.channel_resume(g); | 392 | g->ops.fifo.channel_resume(g); |
393 | } | ||
379 | 394 | ||
380 | done: | 395 | done: |
381 | if (err) | 396 | if (err) { |
382 | g->power_on = false; | 397 | g->power_on = false; |
398 | } | ||
383 | 399 | ||
384 | return err; | 400 | return err; |
385 | } | 401 | } |
@@ -390,8 +406,9 @@ done: | |||
390 | */ | 406 | */ |
391 | int gk20a_can_busy(struct gk20a *g) | 407 | int gk20a_can_busy(struct gk20a *g) |
392 | { | 408 | { |
393 | if (nvgpu_is_enabled(g, NVGPU_DRIVER_IS_DYING)) | 409 | if (nvgpu_is_enabled(g, NVGPU_DRIVER_IS_DYING)) { |
394 | return 0; | 410 | return 0; |
411 | } | ||
395 | return 1; | 412 | return 1; |
396 | } | 413 | } |
397 | 414 | ||
@@ -400,8 +417,9 @@ int gk20a_wait_for_idle(struct gk20a *g) | |||
400 | int wait_length = 150; /* 3 second overall max wait. */ | 417 | int wait_length = 150; /* 3 second overall max wait. */ |
401 | int target_usage_count = 0; | 418 | int target_usage_count = 0; |
402 | 419 | ||
403 | if (!g) | 420 | if (!g) { |
404 | return -ENODEV; | 421 | return -ENODEV; |
422 | } | ||
405 | 423 | ||
406 | while ((nvgpu_atomic_read(&g->usage_count) != target_usage_count) | 424 | while ((nvgpu_atomic_read(&g->usage_count) != target_usage_count) |
407 | && (wait_length-- >= 0)) { | 425 | && (wait_length-- >= 0)) { |
@@ -423,14 +441,17 @@ int gk20a_init_gpu_characteristics(struct gk20a *g) | |||
423 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_MAP_DIRECT_KIND_CTRL, true); | 441 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_MAP_DIRECT_KIND_CTRL, true); |
424 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_MAP_BUFFER_BATCH, true); | 442 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_MAP_BUFFER_BATCH, true); |
425 | 443 | ||
426 | if (IS_ENABLED(CONFIG_SYNC)) | 444 | if (IS_ENABLED(CONFIG_SYNC)) { |
427 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_SYNC_FENCE_FDS, true); | 445 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_SYNC_FENCE_FDS, true); |
446 | } | ||
428 | 447 | ||
429 | if (g->ops.mm.support_sparse && g->ops.mm.support_sparse(g)) | 448 | if (g->ops.mm.support_sparse && g->ops.mm.support_sparse(g)) { |
430 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_SPARSE_ALLOCS, true); | 449 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_SPARSE_ALLOCS, true); |
450 | } | ||
431 | 451 | ||
432 | if (gk20a_platform_has_syncpoints(g)) | 452 | if (gk20a_platform_has_syncpoints(g)) { |
433 | __nvgpu_set_enabled(g, NVGPU_HAS_SYNCPOINTS, true); | 453 | __nvgpu_set_enabled(g, NVGPU_HAS_SYNCPOINTS, true); |
454 | } | ||
434 | 455 | ||
435 | /* | 456 | /* |
436 | * Fast submits are supported as long as the user doesn't request | 457 | * Fast submits are supported as long as the user doesn't request |
@@ -447,23 +468,26 @@ int gk20a_init_gpu_characteristics(struct gk20a *g) | |||
447 | * supported otherwise, provided that the user doesn't request anything | 468 | * supported otherwise, provided that the user doesn't request anything |
448 | * that depends on deferred cleanup. | 469 | * that depends on deferred cleanup. |
449 | */ | 470 | */ |
450 | if (!gk20a_channel_sync_needs_sync_framework(g)) | 471 | if (!gk20a_channel_sync_needs_sync_framework(g)) { |
451 | __nvgpu_set_enabled(g, | 472 | __nvgpu_set_enabled(g, |
452 | NVGPU_SUPPORT_DETERMINISTIC_SUBMIT_FULL, | 473 | NVGPU_SUPPORT_DETERMINISTIC_SUBMIT_FULL, |
453 | true); | 474 | true); |
475 | } | ||
454 | 476 | ||
455 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_DETERMINISTIC_OPTS, true); | 477 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_DETERMINISTIC_OPTS, true); |
456 | 478 | ||
457 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_USERSPACE_MANAGED_AS, true); | 479 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_USERSPACE_MANAGED_AS, true); |
458 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_TSG, true); | 480 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_TSG, true); |
459 | 481 | ||
460 | if (g->ops.clk_arb.get_arbiter_clk_domains) | 482 | if (g->ops.clk_arb.get_arbiter_clk_domains) { |
461 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_CLOCK_CONTROLS, true); | 483 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_CLOCK_CONTROLS, true); |
484 | } | ||
462 | 485 | ||
463 | g->ops.gr.detect_sm_arch(g); | 486 | g->ops.gr.detect_sm_arch(g); |
464 | 487 | ||
465 | if (g->ops.gr.init_cyclestats) | 488 | if (g->ops.gr.init_cyclestats) { |
466 | g->ops.gr.init_cyclestats(g); | 489 | g->ops.gr.init_cyclestats(g); |
490 | } | ||
467 | 491 | ||
468 | g->ops.gr.get_rop_l2_en_mask(g); | 492 | g->ops.gr.get_rop_l2_en_mask(g); |
469 | 493 | ||
@@ -482,11 +506,13 @@ static void gk20a_free_cb(struct nvgpu_ref *refcount) | |||
482 | 506 | ||
483 | gk20a_ce_destroy(g); | 507 | gk20a_ce_destroy(g); |
484 | 508 | ||
485 | if (g->remove_support) | 509 | if (g->remove_support) { |
486 | g->remove_support(g); | 510 | g->remove_support(g); |
511 | } | ||
487 | 512 | ||
488 | if (g->free) | 513 | if (g->free) { |
489 | g->free(g); | 514 | g->free(g); |
515 | } | ||
490 | } | 516 | } |
491 | 517 | ||
492 | /** | 518 | /** |
diff --git a/drivers/gpu/nvgpu/gk20a/gr_ctx_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_ctx_gk20a.c index 956d0434..8b9ac326 100644 --- a/drivers/gpu/nvgpu/gk20a/gr_ctx_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gr_ctx_gk20a.c | |||
@@ -38,8 +38,9 @@ static int gr_gk20a_alloc_load_netlist_u32(struct gk20a *g, u32 *src, u32 len, | |||
38 | struct u32_list_gk20a *u32_list) | 38 | struct u32_list_gk20a *u32_list) |
39 | { | 39 | { |
40 | u32_list->count = (len + sizeof(u32) - 1) / sizeof(u32); | 40 | u32_list->count = (len + sizeof(u32) - 1) / sizeof(u32); |
41 | if (!alloc_u32_list_gk20a(g, u32_list)) | 41 | if (!alloc_u32_list_gk20a(g, u32_list)) { |
42 | return -ENOMEM; | 42 | return -ENOMEM; |
43 | } | ||
43 | 44 | ||
44 | memcpy(u32_list->l, src, len); | 45 | memcpy(u32_list->l, src, len); |
45 | 46 | ||
@@ -50,8 +51,9 @@ static int gr_gk20a_alloc_load_netlist_av(struct gk20a *g, u32 *src, u32 len, | |||
50 | struct av_list_gk20a *av_list) | 51 | struct av_list_gk20a *av_list) |
51 | { | 52 | { |
52 | av_list->count = len / sizeof(struct av_gk20a); | 53 | av_list->count = len / sizeof(struct av_gk20a); |
53 | if (!alloc_av_list_gk20a(g, av_list)) | 54 | if (!alloc_av_list_gk20a(g, av_list)) { |
54 | return -ENOMEM; | 55 | return -ENOMEM; |
56 | } | ||
55 | 57 | ||
56 | memcpy(av_list->l, src, len); | 58 | memcpy(av_list->l, src, len); |
57 | 59 | ||
@@ -62,8 +64,9 @@ static int gr_gk20a_alloc_load_netlist_av64(struct gk20a *g, u32 *src, u32 len, | |||
62 | struct av64_list_gk20a *av64_list) | 64 | struct av64_list_gk20a *av64_list) |
63 | { | 65 | { |
64 | av64_list->count = len / sizeof(struct av64_gk20a); | 66 | av64_list->count = len / sizeof(struct av64_gk20a); |
65 | if (!alloc_av64_list_gk20a(g, av64_list)) | 67 | if (!alloc_av64_list_gk20a(g, av64_list)) { |
66 | return -ENOMEM; | 68 | return -ENOMEM; |
69 | } | ||
67 | 70 | ||
68 | memcpy(av64_list->l, src, len); | 71 | memcpy(av64_list->l, src, len); |
69 | 72 | ||
@@ -74,8 +77,9 @@ static int gr_gk20a_alloc_load_netlist_aiv(struct gk20a *g, u32 *src, u32 len, | |||
74 | struct aiv_list_gk20a *aiv_list) | 77 | struct aiv_list_gk20a *aiv_list) |
75 | { | 78 | { |
76 | aiv_list->count = len / sizeof(struct aiv_gk20a); | 79 | aiv_list->count = len / sizeof(struct aiv_gk20a); |
77 | if (!alloc_aiv_list_gk20a(g, aiv_list)) | 80 | if (!alloc_aiv_list_gk20a(g, aiv_list)) { |
78 | return -ENOMEM; | 81 | return -ENOMEM; |
82 | } | ||
79 | 83 | ||
80 | memcpy(aiv_list->l, src, len); | 84 | memcpy(aiv_list->l, src, len); |
81 | 85 | ||
@@ -128,57 +132,65 @@ static int gr_gk20a_init_ctx_vars_fw(struct gk20a *g, struct gr_gk20a *gr) | |||
128 | nvgpu_log_info(g, "NETLIST_REGIONID_FECS_UCODE_DATA"); | 132 | nvgpu_log_info(g, "NETLIST_REGIONID_FECS_UCODE_DATA"); |
129 | err = gr_gk20a_alloc_load_netlist_u32(g, | 133 | err = gr_gk20a_alloc_load_netlist_u32(g, |
130 | src, size, &g->gr.ctx_vars.ucode.fecs.data); | 134 | src, size, &g->gr.ctx_vars.ucode.fecs.data); |
131 | if (err) | 135 | if (err) { |
132 | goto clean_up; | 136 | goto clean_up; |
137 | } | ||
133 | break; | 138 | break; |
134 | case NETLIST_REGIONID_FECS_UCODE_INST: | 139 | case NETLIST_REGIONID_FECS_UCODE_INST: |
135 | nvgpu_log_info(g, "NETLIST_REGIONID_FECS_UCODE_INST"); | 140 | nvgpu_log_info(g, "NETLIST_REGIONID_FECS_UCODE_INST"); |
136 | err = gr_gk20a_alloc_load_netlist_u32(g, | 141 | err = gr_gk20a_alloc_load_netlist_u32(g, |
137 | src, size, &g->gr.ctx_vars.ucode.fecs.inst); | 142 | src, size, &g->gr.ctx_vars.ucode.fecs.inst); |
138 | if (err) | 143 | if (err) { |
139 | goto clean_up; | 144 | goto clean_up; |
145 | } | ||
140 | break; | 146 | break; |
141 | case NETLIST_REGIONID_GPCCS_UCODE_DATA: | 147 | case NETLIST_REGIONID_GPCCS_UCODE_DATA: |
142 | nvgpu_log_info(g, "NETLIST_REGIONID_GPCCS_UCODE_DATA"); | 148 | nvgpu_log_info(g, "NETLIST_REGIONID_GPCCS_UCODE_DATA"); |
143 | err = gr_gk20a_alloc_load_netlist_u32(g, | 149 | err = gr_gk20a_alloc_load_netlist_u32(g, |
144 | src, size, &g->gr.ctx_vars.ucode.gpccs.data); | 150 | src, size, &g->gr.ctx_vars.ucode.gpccs.data); |
145 | if (err) | 151 | if (err) { |
146 | goto clean_up; | 152 | goto clean_up; |
153 | } | ||
147 | break; | 154 | break; |
148 | case NETLIST_REGIONID_GPCCS_UCODE_INST: | 155 | case NETLIST_REGIONID_GPCCS_UCODE_INST: |
149 | nvgpu_log_info(g, "NETLIST_REGIONID_GPCCS_UCODE_INST"); | 156 | nvgpu_log_info(g, "NETLIST_REGIONID_GPCCS_UCODE_INST"); |
150 | err = gr_gk20a_alloc_load_netlist_u32(g, | 157 | err = gr_gk20a_alloc_load_netlist_u32(g, |
151 | src, size, &g->gr.ctx_vars.ucode.gpccs.inst); | 158 | src, size, &g->gr.ctx_vars.ucode.gpccs.inst); |
152 | if (err) | 159 | if (err) { |
153 | goto clean_up; | 160 | goto clean_up; |
161 | } | ||
154 | break; | 162 | break; |
155 | case NETLIST_REGIONID_SW_BUNDLE_INIT: | 163 | case NETLIST_REGIONID_SW_BUNDLE_INIT: |
156 | nvgpu_log_info(g, "NETLIST_REGIONID_SW_BUNDLE_INIT"); | 164 | nvgpu_log_info(g, "NETLIST_REGIONID_SW_BUNDLE_INIT"); |
157 | err = gr_gk20a_alloc_load_netlist_av(g, | 165 | err = gr_gk20a_alloc_load_netlist_av(g, |
158 | src, size, &g->gr.ctx_vars.sw_bundle_init); | 166 | src, size, &g->gr.ctx_vars.sw_bundle_init); |
159 | if (err) | 167 | if (err) { |
160 | goto clean_up; | 168 | goto clean_up; |
169 | } | ||
161 | break; | 170 | break; |
162 | case NETLIST_REGIONID_SW_METHOD_INIT: | 171 | case NETLIST_REGIONID_SW_METHOD_INIT: |
163 | nvgpu_log_info(g, "NETLIST_REGIONID_SW_METHOD_INIT"); | 172 | nvgpu_log_info(g, "NETLIST_REGIONID_SW_METHOD_INIT"); |
164 | err = gr_gk20a_alloc_load_netlist_av(g, | 173 | err = gr_gk20a_alloc_load_netlist_av(g, |
165 | src, size, &g->gr.ctx_vars.sw_method_init); | 174 | src, size, &g->gr.ctx_vars.sw_method_init); |
166 | if (err) | 175 | if (err) { |
167 | goto clean_up; | 176 | goto clean_up; |
177 | } | ||
168 | break; | 178 | break; |
169 | case NETLIST_REGIONID_SW_CTX_LOAD: | 179 | case NETLIST_REGIONID_SW_CTX_LOAD: |
170 | nvgpu_log_info(g, "NETLIST_REGIONID_SW_CTX_LOAD"); | 180 | nvgpu_log_info(g, "NETLIST_REGIONID_SW_CTX_LOAD"); |
171 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 181 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
172 | src, size, &g->gr.ctx_vars.sw_ctx_load); | 182 | src, size, &g->gr.ctx_vars.sw_ctx_load); |
173 | if (err) | 183 | if (err) { |
174 | goto clean_up; | 184 | goto clean_up; |
185 | } | ||
175 | break; | 186 | break; |
176 | case NETLIST_REGIONID_SW_NON_CTX_LOAD: | 187 | case NETLIST_REGIONID_SW_NON_CTX_LOAD: |
177 | nvgpu_log_info(g, "NETLIST_REGIONID_SW_NON_CTX_LOAD"); | 188 | nvgpu_log_info(g, "NETLIST_REGIONID_SW_NON_CTX_LOAD"); |
178 | err = gr_gk20a_alloc_load_netlist_av(g, | 189 | err = gr_gk20a_alloc_load_netlist_av(g, |
179 | src, size, &g->gr.ctx_vars.sw_non_ctx_load); | 190 | src, size, &g->gr.ctx_vars.sw_non_ctx_load); |
180 | if (err) | 191 | if (err) { |
181 | goto clean_up; | 192 | goto clean_up; |
193 | } | ||
182 | break; | 194 | break; |
183 | case NETLIST_REGIONID_SWVEIDBUNDLEINIT: | 195 | case NETLIST_REGIONID_SWVEIDBUNDLEINIT: |
184 | nvgpu_log_info(g, | 196 | nvgpu_log_info(g, |
@@ -186,64 +198,73 @@ static int gr_gk20a_init_ctx_vars_fw(struct gk20a *g, struct gr_gk20a *gr) | |||
186 | err = gr_gk20a_alloc_load_netlist_av(g, | 198 | err = gr_gk20a_alloc_load_netlist_av(g, |
187 | src, size, | 199 | src, size, |
188 | &g->gr.ctx_vars.sw_veid_bundle_init); | 200 | &g->gr.ctx_vars.sw_veid_bundle_init); |
189 | if (err) | 201 | if (err) { |
190 | goto clean_up; | 202 | goto clean_up; |
203 | } | ||
191 | break; | 204 | break; |
192 | case NETLIST_REGIONID_CTXREG_SYS: | 205 | case NETLIST_REGIONID_CTXREG_SYS: |
193 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_SYS"); | 206 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_SYS"); |
194 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 207 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
195 | src, size, &g->gr.ctx_vars.ctxsw_regs.sys); | 208 | src, size, &g->gr.ctx_vars.ctxsw_regs.sys); |
196 | if (err) | 209 | if (err) { |
197 | goto clean_up; | 210 | goto clean_up; |
211 | } | ||
198 | break; | 212 | break; |
199 | case NETLIST_REGIONID_CTXREG_GPC: | 213 | case NETLIST_REGIONID_CTXREG_GPC: |
200 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_GPC"); | 214 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_GPC"); |
201 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 215 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
202 | src, size, &g->gr.ctx_vars.ctxsw_regs.gpc); | 216 | src, size, &g->gr.ctx_vars.ctxsw_regs.gpc); |
203 | if (err) | 217 | if (err) { |
204 | goto clean_up; | 218 | goto clean_up; |
219 | } | ||
205 | break; | 220 | break; |
206 | case NETLIST_REGIONID_CTXREG_TPC: | 221 | case NETLIST_REGIONID_CTXREG_TPC: |
207 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_TPC"); | 222 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_TPC"); |
208 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 223 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
209 | src, size, &g->gr.ctx_vars.ctxsw_regs.tpc); | 224 | src, size, &g->gr.ctx_vars.ctxsw_regs.tpc); |
210 | if (err) | 225 | if (err) { |
211 | goto clean_up; | 226 | goto clean_up; |
227 | } | ||
212 | break; | 228 | break; |
213 | case NETLIST_REGIONID_CTXREG_ZCULL_GPC: | 229 | case NETLIST_REGIONID_CTXREG_ZCULL_GPC: |
214 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_ZCULL_GPC"); | 230 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_ZCULL_GPC"); |
215 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 231 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
216 | src, size, &g->gr.ctx_vars.ctxsw_regs.zcull_gpc); | 232 | src, size, &g->gr.ctx_vars.ctxsw_regs.zcull_gpc); |
217 | if (err) | 233 | if (err) { |
218 | goto clean_up; | 234 | goto clean_up; |
235 | } | ||
219 | break; | 236 | break; |
220 | case NETLIST_REGIONID_CTXREG_PPC: | 237 | case NETLIST_REGIONID_CTXREG_PPC: |
221 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PPC"); | 238 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PPC"); |
222 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 239 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
223 | src, size, &g->gr.ctx_vars.ctxsw_regs.ppc); | 240 | src, size, &g->gr.ctx_vars.ctxsw_regs.ppc); |
224 | if (err) | 241 | if (err) { |
225 | goto clean_up; | 242 | goto clean_up; |
243 | } | ||
226 | break; | 244 | break; |
227 | case NETLIST_REGIONID_CTXREG_PM_SYS: | 245 | case NETLIST_REGIONID_CTXREG_PM_SYS: |
228 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_SYS"); | 246 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_SYS"); |
229 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 247 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
230 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_sys); | 248 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_sys); |
231 | if (err) | 249 | if (err) { |
232 | goto clean_up; | 250 | goto clean_up; |
251 | } | ||
233 | break; | 252 | break; |
234 | case NETLIST_REGIONID_CTXREG_PM_GPC: | 253 | case NETLIST_REGIONID_CTXREG_PM_GPC: |
235 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_GPC"); | 254 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_GPC"); |
236 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 255 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
237 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_gpc); | 256 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_gpc); |
238 | if (err) | 257 | if (err) { |
239 | goto clean_up; | 258 | goto clean_up; |
259 | } | ||
240 | break; | 260 | break; |
241 | case NETLIST_REGIONID_CTXREG_PM_TPC: | 261 | case NETLIST_REGIONID_CTXREG_PM_TPC: |
242 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_TPC"); | 262 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PM_TPC"); |
243 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 263 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
244 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_tpc); | 264 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_tpc); |
245 | if (err) | 265 | if (err) { |
246 | goto clean_up; | 266 | goto clean_up; |
267 | } | ||
247 | break; | 268 | break; |
248 | case NETLIST_REGIONID_BUFFER_SIZE: | 269 | case NETLIST_REGIONID_BUFFER_SIZE: |
249 | g->gr.ctx_vars.buffer_size = *src; | 270 | g->gr.ctx_vars.buffer_size = *src; |
@@ -269,108 +290,123 @@ static int gr_gk20a_init_ctx_vars_fw(struct gk20a *g, struct gr_gk20a *gr) | |||
269 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMPPC"); | 290 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMPPC"); |
270 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 291 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
271 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ppc); | 292 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ppc); |
272 | if (err) | 293 | if (err) { |
273 | goto clean_up; | 294 | goto clean_up; |
295 | } | ||
274 | break; | 296 | break; |
275 | case NETLIST_REGIONID_NVPERF_CTXREG_SYS: | 297 | case NETLIST_REGIONID_NVPERF_CTXREG_SYS: |
276 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_CTXREG_SYS"); | 298 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_CTXREG_SYS"); |
277 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 299 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
278 | src, size, &g->gr.ctx_vars.ctxsw_regs.perf_sys); | 300 | src, size, &g->gr.ctx_vars.ctxsw_regs.perf_sys); |
279 | if (err) | 301 | if (err) { |
280 | goto clean_up; | 302 | goto clean_up; |
303 | } | ||
281 | break; | 304 | break; |
282 | case NETLIST_REGIONID_NVPERF_FBP_CTXREGS: | 305 | case NETLIST_REGIONID_NVPERF_FBP_CTXREGS: |
283 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_FBP_CTXREGS"); | 306 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_FBP_CTXREGS"); |
284 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 307 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
285 | src, size, &g->gr.ctx_vars.ctxsw_regs.fbp); | 308 | src, size, &g->gr.ctx_vars.ctxsw_regs.fbp); |
286 | if (err) | 309 | if (err) { |
287 | goto clean_up; | 310 | goto clean_up; |
311 | } | ||
288 | break; | 312 | break; |
289 | case NETLIST_REGIONID_NVPERF_CTXREG_GPC: | 313 | case NETLIST_REGIONID_NVPERF_CTXREG_GPC: |
290 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_CTXREG_GPC"); | 314 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_CTXREG_GPC"); |
291 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 315 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
292 | src, size, &g->gr.ctx_vars.ctxsw_regs.perf_gpc); | 316 | src, size, &g->gr.ctx_vars.ctxsw_regs.perf_gpc); |
293 | if (err) | 317 | if (err) { |
294 | goto clean_up; | 318 | goto clean_up; |
319 | } | ||
295 | break; | 320 | break; |
296 | case NETLIST_REGIONID_NVPERF_FBP_ROUTER: | 321 | case NETLIST_REGIONID_NVPERF_FBP_ROUTER: |
297 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_FBP_ROUTER"); | 322 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_FBP_ROUTER"); |
298 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 323 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
299 | src, size, &g->gr.ctx_vars.ctxsw_regs.fbp_router); | 324 | src, size, &g->gr.ctx_vars.ctxsw_regs.fbp_router); |
300 | if (err) | 325 | if (err) { |
301 | goto clean_up; | 326 | goto clean_up; |
327 | } | ||
302 | break; | 328 | break; |
303 | case NETLIST_REGIONID_NVPERF_GPC_ROUTER: | 329 | case NETLIST_REGIONID_NVPERF_GPC_ROUTER: |
304 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_GPC_ROUTER"); | 330 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_GPC_ROUTER"); |
305 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 331 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
306 | src, size, &g->gr.ctx_vars.ctxsw_regs.gpc_router); | 332 | src, size, &g->gr.ctx_vars.ctxsw_regs.gpc_router); |
307 | if (err) | 333 | if (err) { |
308 | goto clean_up; | 334 | goto clean_up; |
335 | } | ||
309 | break; | 336 | break; |
310 | case NETLIST_REGIONID_CTXREG_PMLTC: | 337 | case NETLIST_REGIONID_CTXREG_PMLTC: |
311 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMLTC"); | 338 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMLTC"); |
312 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 339 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
313 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ltc); | 340 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ltc); |
314 | if (err) | 341 | if (err) { |
315 | goto clean_up; | 342 | goto clean_up; |
343 | } | ||
316 | break; | 344 | break; |
317 | case NETLIST_REGIONID_CTXREG_PMFBPA: | 345 | case NETLIST_REGIONID_CTXREG_PMFBPA: |
318 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMFBPA"); | 346 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMFBPA"); |
319 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 347 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
320 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_fbpa); | 348 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_fbpa); |
321 | if (err) | 349 | if (err) { |
322 | goto clean_up; | 350 | goto clean_up; |
351 | } | ||
323 | break; | 352 | break; |
324 | case NETLIST_REGIONID_NVPERF_SYS_ROUTER: | 353 | case NETLIST_REGIONID_NVPERF_SYS_ROUTER: |
325 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_SYS_ROUTER"); | 354 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_SYS_ROUTER"); |
326 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 355 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
327 | src, size, &g->gr.ctx_vars.ctxsw_regs.perf_sys_router); | 356 | src, size, &g->gr.ctx_vars.ctxsw_regs.perf_sys_router); |
328 | if (err) | 357 | if (err) { |
329 | goto clean_up; | 358 | goto clean_up; |
359 | } | ||
330 | break; | 360 | break; |
331 | case NETLIST_REGIONID_NVPERF_PMA: | 361 | case NETLIST_REGIONID_NVPERF_PMA: |
332 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_PMA"); | 362 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_PMA"); |
333 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 363 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
334 | src, size, &g->gr.ctx_vars.ctxsw_regs.perf_pma); | 364 | src, size, &g->gr.ctx_vars.ctxsw_regs.perf_pma); |
335 | if (err) | 365 | if (err) { |
336 | goto clean_up; | 366 | goto clean_up; |
367 | } | ||
337 | break; | 368 | break; |
338 | case NETLIST_REGIONID_CTXREG_PMROP: | 369 | case NETLIST_REGIONID_CTXREG_PMROP: |
339 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMROP"); | 370 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMROP"); |
340 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 371 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
341 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_rop); | 372 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_rop); |
342 | if (err) | 373 | if (err) { |
343 | goto clean_up; | 374 | goto clean_up; |
375 | } | ||
344 | break; | 376 | break; |
345 | case NETLIST_REGIONID_CTXREG_PMUCGPC: | 377 | case NETLIST_REGIONID_CTXREG_PMUCGPC: |
346 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMUCGPC"); | 378 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_PMUCGPC"); |
347 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 379 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
348 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ucgpc); | 380 | src, size, &g->gr.ctx_vars.ctxsw_regs.pm_ucgpc); |
349 | if (err) | 381 | if (err) { |
350 | goto clean_up; | 382 | goto clean_up; |
383 | } | ||
351 | break; | 384 | break; |
352 | case NETLIST_REGIONID_CTXREG_ETPC: | 385 | case NETLIST_REGIONID_CTXREG_ETPC: |
353 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_ETPC"); | 386 | nvgpu_log_info(g, "NETLIST_REGIONID_CTXREG_ETPC"); |
354 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 387 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
355 | src, size, &g->gr.ctx_vars.ctxsw_regs.etpc); | 388 | src, size, &g->gr.ctx_vars.ctxsw_regs.etpc); |
356 | if (err) | 389 | if (err) { |
357 | goto clean_up; | 390 | goto clean_up; |
391 | } | ||
358 | break; | 392 | break; |
359 | case NETLIST_REGIONID_SW_BUNDLE64_INIT: | 393 | case NETLIST_REGIONID_SW_BUNDLE64_INIT: |
360 | nvgpu_log_info(g, "NETLIST_REGIONID_SW_BUNDLE64_INIT"); | 394 | nvgpu_log_info(g, "NETLIST_REGIONID_SW_BUNDLE64_INIT"); |
361 | err = gr_gk20a_alloc_load_netlist_av64(g, | 395 | err = gr_gk20a_alloc_load_netlist_av64(g, |
362 | src, size, | 396 | src, size, |
363 | &g->gr.ctx_vars.sw_bundle64_init); | 397 | &g->gr.ctx_vars.sw_bundle64_init); |
364 | if (err) | 398 | if (err) { |
365 | goto clean_up; | 399 | goto clean_up; |
400 | } | ||
366 | break; | 401 | break; |
367 | case NETLIST_REGIONID_NVPERF_PMCAU: | 402 | case NETLIST_REGIONID_NVPERF_PMCAU: |
368 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_PMCAU"); | 403 | nvgpu_log_info(g, "NETLIST_REGIONID_NVPERF_PMCAU"); |
369 | err = gr_gk20a_alloc_load_netlist_aiv(g, | 404 | err = gr_gk20a_alloc_load_netlist_aiv(g, |
370 | src, size, | 405 | src, size, |
371 | &g->gr.ctx_vars.ctxsw_regs.pm_cau); | 406 | &g->gr.ctx_vars.ctxsw_regs.pm_cau); |
372 | if (err) | 407 | if (err) { |
373 | goto clean_up; | 408 | goto clean_up; |
409 | } | ||
374 | break; | 410 | break; |
375 | 411 | ||
376 | default: | 412 | default: |
@@ -442,8 +478,9 @@ done: | |||
442 | 478 | ||
443 | int gr_gk20a_init_ctx_vars(struct gk20a *g, struct gr_gk20a *gr) | 479 | int gr_gk20a_init_ctx_vars(struct gk20a *g, struct gr_gk20a *gr) |
444 | { | 480 | { |
445 | if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) | 481 | if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) { |
446 | return gr_gk20a_init_ctx_vars_sim(g, gr); | 482 | return gr_gk20a_init_ctx_vars_sim(g, gr); |
447 | else | 483 | } else { |
448 | return gr_gk20a_init_ctx_vars_fw(g, gr); | 484 | return gr_gk20a_init_ctx_vars_fw(g, gr); |
485 | } | ||
449 | } | 486 | } |