diff options
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a')
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/ce2_gk20a.c | 164 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/ce2_gk20a.h | 10 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/channel_gk20a.c | 666 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/channel_gk20a.h | 29 |
4 files changed, 42 insertions, 827 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c index 5314a1be..9ff6c792 100644 --- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c | |||
@@ -249,18 +249,7 @@ static inline unsigned int gk20a_ce_get_method_size(int request_operation, | |||
249 | return methodsize; | 249 | return methodsize; |
250 | } | 250 | } |
251 | 251 | ||
252 | static inline int gk20a_get_valid_launch_flags(struct gk20a *g, int launch_flags) | 252 | int gk20a_ce_prepare_submit(u64 src_buf, |
253 | { | ||
254 | /* there is no local memory available, | ||
255 | don't allow local memory related CE flags */ | ||
256 | if (!g->mm.vidmem.size) { | ||
257 | launch_flags &= ~(NVGPU_CE_SRC_LOCATION_LOCAL_FB | | ||
258 | NVGPU_CE_DST_LOCATION_LOCAL_FB); | ||
259 | } | ||
260 | return launch_flags; | ||
261 | } | ||
262 | |||
263 | static int gk20a_ce_prepare_submit(u64 src_buf, | ||
264 | u64 dst_buf, | 253 | u64 dst_buf, |
265 | u64 size, | 254 | u64 size, |
266 | u32 *cmd_buf_cpu_va, | 255 | u32 *cmd_buf_cpu_va, |
@@ -626,157 +615,6 @@ end: | |||
626 | } | 615 | } |
627 | EXPORT_SYMBOL(gk20a_ce_create_context_with_cb); | 616 | EXPORT_SYMBOL(gk20a_ce_create_context_with_cb); |
628 | 617 | ||
629 | int gk20a_ce_execute_ops(struct gk20a *g, | ||
630 | u32 ce_ctx_id, | ||
631 | u64 src_buf, | ||
632 | u64 dst_buf, | ||
633 | u64 size, | ||
634 | unsigned int payload, | ||
635 | int launch_flags, | ||
636 | int request_operation, | ||
637 | struct gk20a_fence *gk20a_fence_in, | ||
638 | u32 submit_flags, | ||
639 | struct gk20a_fence **gk20a_fence_out) | ||
640 | { | ||
641 | int ret = -EPERM; | ||
642 | struct gk20a_ce_app *ce_app = &g->ce_app; | ||
643 | struct gk20a_gpu_ctx *ce_ctx, *ce_ctx_save; | ||
644 | bool found = false; | ||
645 | u32 *cmd_buf_cpu_va; | ||
646 | u64 cmd_buf_gpu_va = 0; | ||
647 | u32 methodSize; | ||
648 | u32 cmd_buf_read_offset; | ||
649 | u32 fence_index; | ||
650 | struct nvgpu_gpfifo gpfifo; | ||
651 | struct nvgpu_fence fence = {0,0}; | ||
652 | struct gk20a_fence *ce_cmd_buf_fence_out = NULL; | ||
653 | struct nvgpu_gpu_characteristics *gpu_capability = &g->gpu_characteristics; | ||
654 | |||
655 | if (!ce_app->initialised ||ce_app->app_state != NVGPU_CE_ACTIVE) | ||
656 | goto end; | ||
657 | |||
658 | nvgpu_mutex_acquire(&ce_app->app_mutex); | ||
659 | |||
660 | nvgpu_list_for_each_entry_safe(ce_ctx, ce_ctx_save, | ||
661 | &ce_app->allocated_contexts, gk20a_gpu_ctx, list) { | ||
662 | if (ce_ctx->ctx_id == ce_ctx_id) { | ||
663 | found = true; | ||
664 | break; | ||
665 | } | ||
666 | } | ||
667 | |||
668 | nvgpu_mutex_release(&ce_app->app_mutex); | ||
669 | |||
670 | if (!found) { | ||
671 | ret = -EINVAL; | ||
672 | goto end; | ||
673 | } | ||
674 | |||
675 | if (ce_ctx->gpu_ctx_state != NVGPU_CE_GPU_CTX_ALLOCATED) { | ||
676 | ret = -ENODEV; | ||
677 | goto end; | ||
678 | } | ||
679 | |||
680 | nvgpu_mutex_acquire(&ce_ctx->gpu_ctx_mutex); | ||
681 | |||
682 | ce_ctx->cmd_buf_read_queue_offset %= ce_ctx->cmd_buf_end_queue_offset; | ||
683 | |||
684 | cmd_buf_read_offset = (ce_ctx->cmd_buf_read_queue_offset * | ||
685 | (NVGPU_CE_MAX_COMMAND_BUFF_SIZE_PER_KICKOFF / sizeof(u32))); | ||
686 | |||
687 | /* at end of command buffer has gk20a_fence for command buffer sync */ | ||
688 | fence_index = (cmd_buf_read_offset + | ||
689 | ((NVGPU_CE_MAX_COMMAND_BUFF_SIZE_PER_KICKOFF / sizeof(u32)) - | ||
690 | (NVGPU_CE_MAX_COMMAND_BUFF_SIZE_FOR_TRACING / sizeof(u32)))); | ||
691 | |||
692 | if (sizeof(struct gk20a_fence *) > NVGPU_CE_MAX_COMMAND_BUFF_SIZE_FOR_TRACING) { | ||
693 | ret = -ENOMEM; | ||
694 | goto noop; | ||
695 | } | ||
696 | |||
697 | cmd_buf_cpu_va = (u32 *)ce_ctx->cmd_buf_mem.cpu_va; | ||
698 | |||
699 | /* 0 is treated as invalid pre-sync */ | ||
700 | if (cmd_buf_cpu_va[fence_index]) { | ||
701 | struct gk20a_fence * ce_cmd_buf_fence_in = NULL; | ||
702 | |||
703 | memcpy((void *)&ce_cmd_buf_fence_in, | ||
704 | (void *)(cmd_buf_cpu_va + fence_index), | ||
705 | sizeof(struct gk20a_fence *)); | ||
706 | ret = gk20a_fence_wait(g, ce_cmd_buf_fence_in, | ||
707 | gk20a_get_gr_idle_timeout(g)); | ||
708 | |||
709 | gk20a_fence_put(ce_cmd_buf_fence_in); | ||
710 | /* Reset the stored last pre-sync */ | ||
711 | memset((void *)(cmd_buf_cpu_va + fence_index), | ||
712 | 0, | ||
713 | NVGPU_CE_MAX_COMMAND_BUFF_SIZE_FOR_TRACING); | ||
714 | if (ret) | ||
715 | goto noop; | ||
716 | } | ||
717 | |||
718 | cmd_buf_gpu_va = (ce_ctx->cmd_buf_mem.gpu_va + (u64)(cmd_buf_read_offset *sizeof(u32))); | ||
719 | |||
720 | methodSize = gk20a_ce_prepare_submit(src_buf, | ||
721 | dst_buf, | ||
722 | size, | ||
723 | &cmd_buf_cpu_va[cmd_buf_read_offset], | ||
724 | NVGPU_CE_MAX_COMMAND_BUFF_SIZE_PER_KICKOFF, | ||
725 | payload, | ||
726 | gk20a_get_valid_launch_flags(g, launch_flags), | ||
727 | request_operation, | ||
728 | gpu_capability->dma_copy_class, | ||
729 | gk20a_fence_in); | ||
730 | |||
731 | if (methodSize) { | ||
732 | /* TODO: Remove CPU pre-fence wait */ | ||
733 | if (gk20a_fence_in) { | ||
734 | ret = gk20a_fence_wait(g, gk20a_fence_in, | ||
735 | gk20a_get_gr_idle_timeout(g)); | ||
736 | gk20a_fence_put(gk20a_fence_in); | ||
737 | if (ret) | ||
738 | goto noop; | ||
739 | } | ||
740 | |||
741 | /* store the element into gpfifo */ | ||
742 | gpfifo.entry0 = | ||
743 | u64_lo32(cmd_buf_gpu_va); | ||
744 | gpfifo.entry1 = | ||
745 | (u64_hi32(cmd_buf_gpu_va) | | ||
746 | pbdma_gp_entry1_length_f(methodSize)); | ||
747 | |||
748 | /* take always the postfence as it is needed for protecting the ce context */ | ||
749 | submit_flags |= NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET; | ||
750 | |||
751 | nvgpu_smp_wmb(); | ||
752 | |||
753 | ret = gk20a_submit_channel_gpfifo(ce_ctx->ch, &gpfifo, NULL, | ||
754 | 1, submit_flags, &fence, | ||
755 | &ce_cmd_buf_fence_out, false, NULL); | ||
756 | |||
757 | if (!ret) { | ||
758 | memcpy((void *)(cmd_buf_cpu_va + fence_index), | ||
759 | (void *)&ce_cmd_buf_fence_out, | ||
760 | sizeof(struct gk20a_fence *)); | ||
761 | |||
762 | if (gk20a_fence_out) { | ||
763 | gk20a_fence_get(ce_cmd_buf_fence_out); | ||
764 | *gk20a_fence_out = ce_cmd_buf_fence_out; | ||
765 | } | ||
766 | |||
767 | /* Next available command buffer queue Index */ | ||
768 | ++ce_ctx->cmd_buf_read_queue_offset; | ||
769 | ++ce_ctx->submitted_seq_number; | ||
770 | } | ||
771 | } else | ||
772 | ret = -ENOMEM; | ||
773 | noop: | ||
774 | nvgpu_mutex_release(&ce_ctx->gpu_ctx_mutex); | ||
775 | end: | ||
776 | return ret; | ||
777 | } | ||
778 | EXPORT_SYMBOL(gk20a_ce_execute_ops); | ||
779 | |||
780 | void gk20a_ce_delete_context(struct gk20a *g, | 618 | void gk20a_ce_delete_context(struct gk20a *g, |
781 | u32 ce_ctx_id) | 619 | u32 ce_ctx_id) |
782 | { | 620 | { |
diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h index 1dad8952..8d3a4ca3 100644 --- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.h | |||
@@ -161,5 +161,15 @@ void gk20a_ce_delete_context_priv(struct gk20a *g, | |||
161 | u32 ce_ctx_id); | 161 | u32 ce_ctx_id); |
162 | void gk20a_ce_delete_context(struct gk20a *g, | 162 | void gk20a_ce_delete_context(struct gk20a *g, |
163 | u32 ce_ctx_id); | 163 | u32 ce_ctx_id); |
164 | int gk20a_ce_prepare_submit(u64 src_buf, | ||
165 | u64 dst_buf, | ||
166 | u64 size, | ||
167 | u32 *cmd_buf_cpu_va, | ||
168 | u32 max_cmd_buf_size, | ||
169 | unsigned int payload, | ||
170 | int launch_flags, | ||
171 | int request_operation, | ||
172 | u32 dma_copy_class, | ||
173 | struct gk20a_fence *gk20a_fence_in); | ||
164 | 174 | ||
165 | #endif /*__CE2_GK20A_H__*/ | 175 | #endif /*__CE2_GK20A_H__*/ |
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c index 00d20357..c938ba6b 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c | |||
@@ -44,45 +44,13 @@ | |||
44 | #include <nvgpu/barrier.h> | 44 | #include <nvgpu/barrier.h> |
45 | #include <nvgpu/ctxsw_trace.h> | 45 | #include <nvgpu/ctxsw_trace.h> |
46 | 46 | ||
47 | /* | ||
48 | * This is required for nvgpu_vm_find_buf() which is used in the tracing | ||
49 | * code. Once we can get and access userspace buffers without requiring | ||
50 | * direct dma_buf usage this can be removed. | ||
51 | */ | ||
52 | #include <nvgpu/linux/vm.h> | ||
53 | |||
54 | #include "gk20a.h" | 47 | #include "gk20a.h" |
55 | #include "dbg_gpu_gk20a.h" | 48 | #include "dbg_gpu_gk20a.h" |
56 | #include "fence_gk20a.h" | 49 | #include "fence_gk20a.h" |
57 | 50 | ||
58 | #include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h> | ||
59 | |||
60 | /* | ||
61 | * Note | ||
62 | * This is added for all the copy_from_user methods in this file which needs to | ||
63 | * be moved lated to reduce depenedency on Linux | ||
64 | */ | ||
65 | #include <linux/uaccess.h> | ||
66 | |||
67 | /* | ||
68 | * Although channels do have pointers back to the gk20a struct that they were | ||
69 | * created under in cases where the driver is killed that pointer can be bad. | ||
70 | * The channel memory can be freed before the release() function for a given | ||
71 | * channel is called. This happens when the driver dies and userspace doesn't | ||
72 | * get a chance to call release() until after the entire gk20a driver data is | ||
73 | * unloaded and freed. | ||
74 | */ | ||
75 | struct channel_priv { | ||
76 | struct gk20a *g; | ||
77 | struct channel_gk20a *c; | ||
78 | }; | ||
79 | |||
80 | static void free_channel(struct fifo_gk20a *f, struct channel_gk20a *c); | 51 | static void free_channel(struct fifo_gk20a *f, struct channel_gk20a *c); |
81 | static void gk20a_channel_dump_ref_actions(struct channel_gk20a *c); | 52 | static void gk20a_channel_dump_ref_actions(struct channel_gk20a *c); |
82 | 53 | ||
83 | static void free_priv_cmdbuf(struct channel_gk20a *c, | ||
84 | struct priv_cmd_entry *e); | ||
85 | |||
86 | static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c); | 54 | static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c); |
87 | static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c); | 55 | static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c); |
88 | 56 | ||
@@ -97,9 +65,6 @@ static struct channel_gk20a_job *channel_gk20a_joblist_peek( | |||
97 | 65 | ||
98 | static u32 gk20a_get_channel_watchdog_timeout(struct channel_gk20a *ch); | 66 | static u32 gk20a_get_channel_watchdog_timeout(struct channel_gk20a *ch); |
99 | 67 | ||
100 | static void gk20a_channel_clean_up_jobs(struct channel_gk20a *c, | ||
101 | bool clean_all); | ||
102 | |||
103 | /* allocate GPU channel */ | 68 | /* allocate GPU channel */ |
104 | static struct channel_gk20a *allocate_channel(struct fifo_gk20a *f) | 69 | static struct channel_gk20a *allocate_channel(struct fifo_gk20a *f) |
105 | { | 70 | { |
@@ -1038,7 +1003,7 @@ int gk20a_channel_alloc_priv_cmdbuf(struct channel_gk20a *c, u32 orig_size, | |||
1038 | 1003 | ||
1039 | /* Don't call this to free an explict cmd entry. | 1004 | /* Don't call this to free an explict cmd entry. |
1040 | * It doesn't update priv_cmd_queue get/put */ | 1005 | * It doesn't update priv_cmd_queue get/put */ |
1041 | static void free_priv_cmdbuf(struct channel_gk20a *c, | 1006 | void free_priv_cmdbuf(struct channel_gk20a *c, |
1042 | struct priv_cmd_entry *e) | 1007 | struct priv_cmd_entry *e) |
1043 | { | 1008 | { |
1044 | if (channel_gk20a_is_prealloc_enabled(c)) | 1009 | if (channel_gk20a_is_prealloc_enabled(c)) |
@@ -1047,7 +1012,7 @@ static void free_priv_cmdbuf(struct channel_gk20a *c, | |||
1047 | nvgpu_kfree(c->g, e); | 1012 | nvgpu_kfree(c->g, e); |
1048 | } | 1013 | } |
1049 | 1014 | ||
1050 | static int channel_gk20a_alloc_job(struct channel_gk20a *c, | 1015 | int channel_gk20a_alloc_job(struct channel_gk20a *c, |
1051 | struct channel_gk20a_job **job_out) | 1016 | struct channel_gk20a_job **job_out) |
1052 | { | 1017 | { |
1053 | int err = 0; | 1018 | int err = 0; |
@@ -1080,7 +1045,7 @@ static int channel_gk20a_alloc_job(struct channel_gk20a *c, | |||
1080 | return err; | 1045 | return err; |
1081 | } | 1046 | } |
1082 | 1047 | ||
1083 | static void channel_gk20a_free_job(struct channel_gk20a *c, | 1048 | void channel_gk20a_free_job(struct channel_gk20a *c, |
1084 | struct channel_gk20a_job *job) | 1049 | struct channel_gk20a_job *job) |
1085 | { | 1050 | { |
1086 | /* | 1051 | /* |
@@ -1267,11 +1232,12 @@ int gk20a_channel_alloc_gpfifo(struct channel_gk20a *c, | |||
1267 | { | 1232 | { |
1268 | struct gk20a *g = c->g; | 1233 | struct gk20a *g = c->g; |
1269 | struct vm_gk20a *ch_vm; | 1234 | struct vm_gk20a *ch_vm; |
1270 | u32 gpfifo_size; | 1235 | u32 gpfifo_size, gpfifo_entry_size; |
1271 | int err = 0; | 1236 | int err = 0; |
1272 | unsigned long acquire_timeout; | 1237 | unsigned long acquire_timeout; |
1273 | 1238 | ||
1274 | gpfifo_size = num_entries; | 1239 | gpfifo_size = num_entries; |
1240 | gpfifo_entry_size = nvgpu_get_gpfifo_entry_size(); | ||
1275 | 1241 | ||
1276 | if (flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_VPR_ENABLED) | 1242 | if (flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_VPR_ENABLED) |
1277 | c->vpr = true; | 1243 | c->vpr = true; |
@@ -1315,7 +1281,7 @@ int gk20a_channel_alloc_gpfifo(struct channel_gk20a *c, | |||
1315 | } | 1281 | } |
1316 | 1282 | ||
1317 | err = nvgpu_dma_alloc_map_sys(ch_vm, | 1283 | err = nvgpu_dma_alloc_map_sys(ch_vm, |
1318 | gpfifo_size * sizeof(struct nvgpu_gpfifo), | 1284 | gpfifo_size * gpfifo_entry_size, |
1319 | &c->gpfifo.mem); | 1285 | &c->gpfifo.mem); |
1320 | if (err) { | 1286 | if (err) { |
1321 | nvgpu_err(g, "%s: memory allocation failed", __func__); | 1287 | nvgpu_err(g, "%s: memory allocation failed", __func__); |
@@ -1324,7 +1290,7 @@ int gk20a_channel_alloc_gpfifo(struct channel_gk20a *c, | |||
1324 | 1290 | ||
1325 | if (c->gpfifo.mem.aperture == APERTURE_VIDMEM || g->mm.force_pramin) { | 1291 | if (c->gpfifo.mem.aperture == APERTURE_VIDMEM || g->mm.force_pramin) { |
1326 | c->gpfifo.pipe = nvgpu_big_malloc(g, | 1292 | c->gpfifo.pipe = nvgpu_big_malloc(g, |
1327 | gpfifo_size * sizeof(struct nvgpu_gpfifo)); | 1293 | gpfifo_size * gpfifo_entry_size); |
1328 | if (!c->gpfifo.pipe) { | 1294 | if (!c->gpfifo.pipe) { |
1329 | err = -ENOMEM; | 1295 | err = -ENOMEM; |
1330 | goto clean_up_unmap; | 1296 | goto clean_up_unmap; |
@@ -1427,7 +1393,7 @@ static inline u32 update_gp_get(struct gk20a *g, | |||
1427 | return new_get; | 1393 | return new_get; |
1428 | } | 1394 | } |
1429 | 1395 | ||
1430 | static inline u32 gp_free_count(struct channel_gk20a *c) | 1396 | u32 nvgpu_gp_free_count(struct channel_gk20a *c) |
1431 | { | 1397 | { |
1432 | return (c->gpfifo.entry_num - (c->gpfifo.put - c->gpfifo.get) - 1) % | 1398 | return (c->gpfifo.entry_num - (c->gpfifo.put - c->gpfifo.get) - 1) % |
1433 | c->gpfifo.entry_num; | 1399 | c->gpfifo.entry_num; |
@@ -1460,91 +1426,10 @@ static u32 gk20a_get_channel_watchdog_timeout(struct channel_gk20a *ch) | |||
1460 | return ch->g->ch_wdt_timeout_ms; | 1426 | return ch->g->ch_wdt_timeout_ms; |
1461 | } | 1427 | } |
1462 | 1428 | ||
1463 | static u32 get_gp_free_count(struct channel_gk20a *c) | 1429 | u32 nvgpu_get_gp_free_count(struct channel_gk20a *c) |
1464 | { | 1430 | { |
1465 | update_gp_get(c->g, c); | 1431 | update_gp_get(c->g, c); |
1466 | return gp_free_count(c); | 1432 | return nvgpu_gp_free_count(c); |
1467 | } | ||
1468 | |||
1469 | #ifdef CONFIG_DEBUG_FS | ||
1470 | static void trace_write_pushbuffer(struct channel_gk20a *c, | ||
1471 | struct nvgpu_gpfifo *g) | ||
1472 | { | ||
1473 | void *mem = NULL; | ||
1474 | unsigned int words; | ||
1475 | u64 offset; | ||
1476 | struct dma_buf *dmabuf = NULL; | ||
1477 | |||
1478 | if (gk20a_debug_trace_cmdbuf) { | ||
1479 | u64 gpu_va = (u64)g->entry0 | | ||
1480 | (u64)((u64)pbdma_gp_entry1_get_hi_v(g->entry1) << 32); | ||
1481 | int err; | ||
1482 | |||
1483 | words = pbdma_gp_entry1_length_v(g->entry1); | ||
1484 | err = nvgpu_vm_find_buf(c->vm, gpu_va, &dmabuf, &offset); | ||
1485 | if (!err) | ||
1486 | mem = dma_buf_vmap(dmabuf); | ||
1487 | } | ||
1488 | |||
1489 | if (mem) { | ||
1490 | u32 i; | ||
1491 | /* | ||
1492 | * Write in batches of 128 as there seems to be a limit | ||
1493 | * of how much you can output to ftrace at once. | ||
1494 | */ | ||
1495 | for (i = 0; i < words; i += 128U) { | ||
1496 | trace_gk20a_push_cmdbuf( | ||
1497 | c->g->name, | ||
1498 | 0, | ||
1499 | min(words - i, 128U), | ||
1500 | offset + i * sizeof(u32), | ||
1501 | mem); | ||
1502 | } | ||
1503 | dma_buf_vunmap(dmabuf, mem); | ||
1504 | } | ||
1505 | } | ||
1506 | #endif | ||
1507 | |||
1508 | static void trace_write_pushbuffer_range(struct channel_gk20a *c, | ||
1509 | struct nvgpu_gpfifo *g, | ||
1510 | struct nvgpu_gpfifo __user *user_gpfifo, | ||
1511 | int offset, | ||
1512 | int count) | ||
1513 | { | ||
1514 | #ifdef CONFIG_DEBUG_FS | ||
1515 | u32 size; | ||
1516 | int i; | ||
1517 | struct nvgpu_gpfifo *gp; | ||
1518 | bool gpfifo_allocated = false; | ||
1519 | |||
1520 | if (!gk20a_debug_trace_cmdbuf) | ||
1521 | return; | ||
1522 | |||
1523 | if (!g && !user_gpfifo) | ||
1524 | return; | ||
1525 | |||
1526 | if (!g) { | ||
1527 | size = count * sizeof(struct nvgpu_gpfifo); | ||
1528 | if (size) { | ||
1529 | g = nvgpu_big_malloc(c->g, size); | ||
1530 | if (!g) | ||
1531 | return; | ||
1532 | |||
1533 | if (copy_from_user(g, user_gpfifo, size)) { | ||
1534 | nvgpu_big_free(c->g, g); | ||
1535 | return; | ||
1536 | } | ||
1537 | } | ||
1538 | gpfifo_allocated = true; | ||
1539 | } | ||
1540 | |||
1541 | gp = g + offset; | ||
1542 | for (i = 0; i < count; i++, gp++) | ||
1543 | trace_write_pushbuffer(c, gp); | ||
1544 | |||
1545 | if (gpfifo_allocated) | ||
1546 | nvgpu_big_free(c->g, g); | ||
1547 | #endif | ||
1548 | } | 1433 | } |
1549 | 1434 | ||
1550 | static void __gk20a_channel_timeout_start(struct channel_gk20a *ch) | 1435 | static void __gk20a_channel_timeout_start(struct channel_gk20a *ch) |
@@ -2032,7 +1917,7 @@ int gk20a_free_priv_cmdbuf(struct channel_gk20a *c, struct priv_cmd_entry *e) | |||
2032 | return 0; | 1917 | return 0; |
2033 | } | 1918 | } |
2034 | 1919 | ||
2035 | static int gk20a_channel_add_job(struct channel_gk20a *c, | 1920 | int gk20a_channel_add_job(struct channel_gk20a *c, |
2036 | struct channel_gk20a_job *job, | 1921 | struct channel_gk20a_job *job, |
2037 | bool skip_buffer_refcounting) | 1922 | bool skip_buffer_refcounting) |
2038 | { | 1923 | { |
@@ -2097,7 +1982,7 @@ err_put_buffers: | |||
2097 | * per-job memory for completed jobs; in case of preallocated resources, this | 1982 | * per-job memory for completed jobs; in case of preallocated resources, this |
2098 | * opens up slots for new jobs to be submitted. | 1983 | * opens up slots for new jobs to be submitted. |
2099 | */ | 1984 | */ |
2100 | static void gk20a_channel_clean_up_jobs(struct channel_gk20a *c, | 1985 | void gk20a_channel_clean_up_jobs(struct channel_gk20a *c, |
2101 | bool clean_all) | 1986 | bool clean_all) |
2102 | { | 1987 | { |
2103 | struct vm_gk20a *vm; | 1988 | struct vm_gk20a *vm; |
@@ -2257,533 +2142,6 @@ void gk20a_channel_update(struct channel_gk20a *c) | |||
2257 | gk20a_channel_worker_enqueue(c); | 2142 | gk20a_channel_worker_enqueue(c); |
2258 | } | 2143 | } |
2259 | 2144 | ||
2260 | static void gk20a_submit_append_priv_cmdbuf(struct channel_gk20a *c, | ||
2261 | struct priv_cmd_entry *cmd) | ||
2262 | { | ||
2263 | struct gk20a *g = c->g; | ||
2264 | struct nvgpu_mem *gpfifo_mem = &c->gpfifo.mem; | ||
2265 | struct nvgpu_gpfifo x = { | ||
2266 | .entry0 = u64_lo32(cmd->gva), | ||
2267 | .entry1 = u64_hi32(cmd->gva) | | ||
2268 | pbdma_gp_entry1_length_f(cmd->size) | ||
2269 | }; | ||
2270 | |||
2271 | nvgpu_mem_wr_n(g, gpfifo_mem, c->gpfifo.put * sizeof(x), | ||
2272 | &x, sizeof(x)); | ||
2273 | |||
2274 | if (cmd->mem->aperture == APERTURE_SYSMEM) | ||
2275 | trace_gk20a_push_cmdbuf(g->name, 0, cmd->size, 0, | ||
2276 | cmd->mem->cpu_va + cmd->off * sizeof(u32)); | ||
2277 | |||
2278 | c->gpfifo.put = (c->gpfifo.put + 1) & (c->gpfifo.entry_num - 1); | ||
2279 | } | ||
2280 | |||
2281 | /* | ||
2282 | * Copy source gpfifo entries into the gpfifo ring buffer, potentially | ||
2283 | * splitting into two memcpys to handle wrap-around. | ||
2284 | */ | ||
2285 | static int gk20a_submit_append_gpfifo(struct channel_gk20a *c, | ||
2286 | struct nvgpu_gpfifo *kern_gpfifo, | ||
2287 | struct nvgpu_gpfifo __user *user_gpfifo, | ||
2288 | u32 num_entries) | ||
2289 | { | ||
2290 | /* byte offsets */ | ||
2291 | u32 gpfifo_size = c->gpfifo.entry_num * sizeof(struct nvgpu_gpfifo); | ||
2292 | u32 len = num_entries * sizeof(struct nvgpu_gpfifo); | ||
2293 | u32 start = c->gpfifo.put * sizeof(struct nvgpu_gpfifo); | ||
2294 | u32 end = start + len; /* exclusive */ | ||
2295 | struct nvgpu_mem *gpfifo_mem = &c->gpfifo.mem; | ||
2296 | struct nvgpu_gpfifo *cpu_src; | ||
2297 | int err; | ||
2298 | |||
2299 | if (user_gpfifo && !c->gpfifo.pipe) { | ||
2300 | /* | ||
2301 | * This path (from userspace to sysmem) is special in order to | ||
2302 | * avoid two copies unnecessarily (from user to pipe, then from | ||
2303 | * pipe to gpu sysmem buffer). | ||
2304 | * | ||
2305 | * As a special case, the pipe buffer exists if PRAMIN writes | ||
2306 | * are forced, although the buffers may not be in vidmem in | ||
2307 | * that case. | ||
2308 | */ | ||
2309 | if (end > gpfifo_size) { | ||
2310 | /* wrap-around */ | ||
2311 | int length0 = gpfifo_size - start; | ||
2312 | int length1 = len - length0; | ||
2313 | void __user *user2 = (u8 __user *)user_gpfifo + length0; | ||
2314 | |||
2315 | err = copy_from_user(gpfifo_mem->cpu_va + start, | ||
2316 | user_gpfifo, length0); | ||
2317 | if (err) | ||
2318 | return err; | ||
2319 | |||
2320 | err = copy_from_user(gpfifo_mem->cpu_va, | ||
2321 | user2, length1); | ||
2322 | if (err) | ||
2323 | return err; | ||
2324 | } else { | ||
2325 | err = copy_from_user(gpfifo_mem->cpu_va + start, | ||
2326 | user_gpfifo, len); | ||
2327 | if (err) | ||
2328 | return err; | ||
2329 | } | ||
2330 | |||
2331 | trace_write_pushbuffer_range(c, NULL, user_gpfifo, | ||
2332 | 0, num_entries); | ||
2333 | goto out; | ||
2334 | } else if (user_gpfifo) { | ||
2335 | /* from userspace to vidmem or sysmem when pramin forced, use | ||
2336 | * the common copy path below */ | ||
2337 | err = copy_from_user(c->gpfifo.pipe, user_gpfifo, len); | ||
2338 | if (err) | ||
2339 | return err; | ||
2340 | |||
2341 | cpu_src = c->gpfifo.pipe; | ||
2342 | } else { | ||
2343 | /* from kernel to either sysmem or vidmem, don't need | ||
2344 | * copy_from_user so use the common path below */ | ||
2345 | cpu_src = kern_gpfifo; | ||
2346 | } | ||
2347 | |||
2348 | if (end > gpfifo_size) { | ||
2349 | /* wrap-around */ | ||
2350 | int length0 = gpfifo_size - start; | ||
2351 | int length1 = len - length0; | ||
2352 | void *src2 = (u8 *)cpu_src + length0; | ||
2353 | |||
2354 | nvgpu_mem_wr_n(c->g, gpfifo_mem, start, cpu_src, length0); | ||
2355 | nvgpu_mem_wr_n(c->g, gpfifo_mem, 0, src2, length1); | ||
2356 | } else { | ||
2357 | nvgpu_mem_wr_n(c->g, gpfifo_mem, start, cpu_src, len); | ||
2358 | |||
2359 | } | ||
2360 | |||
2361 | trace_write_pushbuffer_range(c, cpu_src, NULL, 0, num_entries); | ||
2362 | |||
2363 | out: | ||
2364 | c->gpfifo.put = (c->gpfifo.put + num_entries) & | ||
2365 | (c->gpfifo.entry_num - 1); | ||
2366 | |||
2367 | return 0; | ||
2368 | } | ||
2369 | |||
2370 | /* | ||
2371 | * Handle the submit synchronization - pre-fences and post-fences. | ||
2372 | */ | ||
2373 | static int gk20a_submit_prepare_syncs(struct channel_gk20a *c, | ||
2374 | struct nvgpu_fence *fence, | ||
2375 | struct channel_gk20a_job *job, | ||
2376 | struct priv_cmd_entry **wait_cmd, | ||
2377 | struct priv_cmd_entry **incr_cmd, | ||
2378 | struct gk20a_fence **pre_fence, | ||
2379 | struct gk20a_fence **post_fence, | ||
2380 | bool force_need_sync_fence, | ||
2381 | bool register_irq, | ||
2382 | u32 flags) | ||
2383 | { | ||
2384 | struct gk20a *g = c->g; | ||
2385 | bool need_sync_fence = false; | ||
2386 | bool new_sync_created = false; | ||
2387 | int wait_fence_fd = -1; | ||
2388 | int err = 0; | ||
2389 | bool need_wfi = !(flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SUPPRESS_WFI); | ||
2390 | bool pre_alloc_enabled = channel_gk20a_is_prealloc_enabled(c); | ||
2391 | |||
2392 | /* | ||
2393 | * If user wants to always allocate sync_fence_fds then respect that; | ||
2394 | * otherwise, allocate sync_fence_fd based on user flags. | ||
2395 | */ | ||
2396 | if (force_need_sync_fence) | ||
2397 | need_sync_fence = true; | ||
2398 | |||
2399 | if (g->aggressive_sync_destroy_thresh) { | ||
2400 | nvgpu_mutex_acquire(&c->sync_lock); | ||
2401 | if (!c->sync) { | ||
2402 | c->sync = gk20a_channel_sync_create(c); | ||
2403 | if (!c->sync) { | ||
2404 | err = -ENOMEM; | ||
2405 | nvgpu_mutex_release(&c->sync_lock); | ||
2406 | goto fail; | ||
2407 | } | ||
2408 | new_sync_created = true; | ||
2409 | } | ||
2410 | nvgpu_atomic_inc(&c->sync->refcount); | ||
2411 | nvgpu_mutex_release(&c->sync_lock); | ||
2412 | } | ||
2413 | |||
2414 | if (g->ops.fifo.resetup_ramfc && new_sync_created) { | ||
2415 | err = g->ops.fifo.resetup_ramfc(c); | ||
2416 | if (err) | ||
2417 | goto fail; | ||
2418 | } | ||
2419 | |||
2420 | /* | ||
2421 | * Optionally insert syncpt wait in the beginning of gpfifo submission | ||
2422 | * when user requested and the wait hasn't expired. Validate that the id | ||
2423 | * makes sense, elide if not. The only reason this isn't being | ||
2424 | * unceremoniously killed is to keep running some tests which trigger | ||
2425 | * this condition. | ||
2426 | */ | ||
2427 | if (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT) { | ||
2428 | job->pre_fence = gk20a_alloc_fence(c); | ||
2429 | if (!job->pre_fence) { | ||
2430 | err = -ENOMEM; | ||
2431 | goto fail; | ||
2432 | } | ||
2433 | |||
2434 | if (!pre_alloc_enabled) | ||
2435 | job->wait_cmd = nvgpu_kzalloc(g, | ||
2436 | sizeof(struct priv_cmd_entry)); | ||
2437 | |||
2438 | if (!job->wait_cmd) { | ||
2439 | err = -ENOMEM; | ||
2440 | goto clean_up_pre_fence; | ||
2441 | } | ||
2442 | |||
2443 | if (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE) { | ||
2444 | wait_fence_fd = fence->id; | ||
2445 | err = c->sync->wait_fd(c->sync, wait_fence_fd, | ||
2446 | job->wait_cmd, job->pre_fence); | ||
2447 | } else { | ||
2448 | err = c->sync->wait_syncpt(c->sync, fence->id, | ||
2449 | fence->value, job->wait_cmd, | ||
2450 | job->pre_fence); | ||
2451 | } | ||
2452 | |||
2453 | if (!err) { | ||
2454 | if (job->wait_cmd->valid) | ||
2455 | *wait_cmd = job->wait_cmd; | ||
2456 | *pre_fence = job->pre_fence; | ||
2457 | } else | ||
2458 | goto clean_up_wait_cmd; | ||
2459 | } | ||
2460 | |||
2461 | if ((flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) && | ||
2462 | (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE)) | ||
2463 | need_sync_fence = true; | ||
2464 | |||
2465 | /* | ||
2466 | * Always generate an increment at the end of a GPFIFO submission. This | ||
2467 | * is used to keep track of method completion for idle railgating. The | ||
2468 | * sync_pt/semaphore PB is added to the GPFIFO later on in submit. | ||
2469 | */ | ||
2470 | job->post_fence = gk20a_alloc_fence(c); | ||
2471 | if (!job->post_fence) { | ||
2472 | err = -ENOMEM; | ||
2473 | goto clean_up_wait_cmd; | ||
2474 | } | ||
2475 | if (!pre_alloc_enabled) | ||
2476 | job->incr_cmd = nvgpu_kzalloc(g, sizeof(struct priv_cmd_entry)); | ||
2477 | |||
2478 | if (!job->incr_cmd) { | ||
2479 | err = -ENOMEM; | ||
2480 | goto clean_up_post_fence; | ||
2481 | } | ||
2482 | |||
2483 | if (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) | ||
2484 | err = c->sync->incr_user(c->sync, wait_fence_fd, job->incr_cmd, | ||
2485 | job->post_fence, need_wfi, need_sync_fence, | ||
2486 | register_irq); | ||
2487 | else | ||
2488 | err = c->sync->incr(c->sync, job->incr_cmd, | ||
2489 | job->post_fence, need_sync_fence, | ||
2490 | register_irq); | ||
2491 | if (!err) { | ||
2492 | *incr_cmd = job->incr_cmd; | ||
2493 | *post_fence = job->post_fence; | ||
2494 | } else | ||
2495 | goto clean_up_incr_cmd; | ||
2496 | |||
2497 | return 0; | ||
2498 | |||
2499 | clean_up_incr_cmd: | ||
2500 | free_priv_cmdbuf(c, job->incr_cmd); | ||
2501 | if (!pre_alloc_enabled) | ||
2502 | job->incr_cmd = NULL; | ||
2503 | clean_up_post_fence: | ||
2504 | gk20a_fence_put(job->post_fence); | ||
2505 | job->post_fence = NULL; | ||
2506 | clean_up_wait_cmd: | ||
2507 | free_priv_cmdbuf(c, job->wait_cmd); | ||
2508 | if (!pre_alloc_enabled) | ||
2509 | job->wait_cmd = NULL; | ||
2510 | clean_up_pre_fence: | ||
2511 | gk20a_fence_put(job->pre_fence); | ||
2512 | job->pre_fence = NULL; | ||
2513 | fail: | ||
2514 | *wait_cmd = NULL; | ||
2515 | *pre_fence = NULL; | ||
2516 | return err; | ||
2517 | } | ||
2518 | |||
2519 | int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, | ||
2520 | struct nvgpu_gpfifo *gpfifo, | ||
2521 | struct nvgpu_submit_gpfifo_args *args, | ||
2522 | u32 num_entries, | ||
2523 | u32 flags, | ||
2524 | struct nvgpu_fence *fence, | ||
2525 | struct gk20a_fence **fence_out, | ||
2526 | bool force_need_sync_fence, | ||
2527 | struct fifo_profile_gk20a *profile) | ||
2528 | { | ||
2529 | struct gk20a *g = c->g; | ||
2530 | struct priv_cmd_entry *wait_cmd = NULL; | ||
2531 | struct priv_cmd_entry *incr_cmd = NULL; | ||
2532 | struct gk20a_fence *pre_fence = NULL; | ||
2533 | struct gk20a_fence *post_fence = NULL; | ||
2534 | struct channel_gk20a_job *job = NULL; | ||
2535 | /* we might need two extra gpfifo entries - one for pre fence | ||
2536 | * and one for post fence. */ | ||
2537 | const int extra_entries = 2; | ||
2538 | bool skip_buffer_refcounting = (flags & | ||
2539 | NVGPU_SUBMIT_GPFIFO_FLAGS_SKIP_BUFFER_REFCOUNTING); | ||
2540 | int err = 0; | ||
2541 | bool need_job_tracking; | ||
2542 | bool need_deferred_cleanup = false; | ||
2543 | struct nvgpu_gpfifo __user *user_gpfifo = args ? | ||
2544 | (struct nvgpu_gpfifo __user *)(uintptr_t)args->gpfifo : NULL; | ||
2545 | |||
2546 | if (nvgpu_is_enabled(g, NVGPU_DRIVER_IS_DYING)) | ||
2547 | return -ENODEV; | ||
2548 | |||
2549 | if (c->has_timedout) | ||
2550 | return -ETIMEDOUT; | ||
2551 | |||
2552 | if (!nvgpu_mem_is_valid(&c->gpfifo.mem)) | ||
2553 | return -ENOMEM; | ||
2554 | |||
2555 | /* fifo not large enough for request. Return error immediately. | ||
2556 | * Kernel can insert gpfifo entries before and after user gpfifos. | ||
2557 | * So, add extra_entries in user request. Also, HW with fifo size N | ||
2558 | * can accept only N-1 entreis and so the below condition */ | ||
2559 | if (c->gpfifo.entry_num - 1 < num_entries + extra_entries) { | ||
2560 | nvgpu_err(g, "not enough gpfifo space allocated"); | ||
2561 | return -ENOMEM; | ||
2562 | } | ||
2563 | |||
2564 | if (!gpfifo && !args) | ||
2565 | return -EINVAL; | ||
2566 | |||
2567 | if ((flags & (NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT | | ||
2568 | NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET)) && | ||
2569 | !fence) | ||
2570 | return -EINVAL; | ||
2571 | |||
2572 | /* an address space needs to have been bound at this point. */ | ||
2573 | if (!gk20a_channel_as_bound(c)) { | ||
2574 | nvgpu_err(g, | ||
2575 | "not bound to an address space at time of gpfifo" | ||
2576 | " submission."); | ||
2577 | return -EINVAL; | ||
2578 | } | ||
2579 | |||
2580 | if (profile) | ||
2581 | profile->timestamp[PROFILE_ENTRY] = sched_clock(); | ||
2582 | |||
2583 | /* update debug settings */ | ||
2584 | nvgpu_ltc_sync_enabled(g); | ||
2585 | |||
2586 | gk20a_dbg_info("channel %d", c->chid); | ||
2587 | |||
2588 | /* | ||
2589 | * Job tracking is necessary for any of the following conditions: | ||
2590 | * - pre- or post-fence functionality | ||
2591 | * - channel wdt | ||
2592 | * - GPU rail-gating with non-deterministic channels | ||
2593 | * - buffer refcounting | ||
2594 | * | ||
2595 | * If none of the conditions are met, then job tracking is not | ||
2596 | * required and a fast submit can be done (ie. only need to write | ||
2597 | * out userspace GPFIFO entries and update GP_PUT). | ||
2598 | */ | ||
2599 | need_job_tracking = (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT) || | ||
2600 | (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) || | ||
2601 | c->wdt_enabled || | ||
2602 | (g->can_railgate && !c->deterministic) || | ||
2603 | !skip_buffer_refcounting; | ||
2604 | |||
2605 | if (need_job_tracking) { | ||
2606 | bool need_sync_framework = false; | ||
2607 | |||
2608 | /* | ||
2609 | * If the channel is to have deterministic latency and | ||
2610 | * job tracking is required, the channel must have | ||
2611 | * pre-allocated resources. Otherwise, we fail the submit here | ||
2612 | */ | ||
2613 | if (c->deterministic && !channel_gk20a_is_prealloc_enabled(c)) | ||
2614 | return -EINVAL; | ||
2615 | |||
2616 | need_sync_framework = force_need_sync_fence || | ||
2617 | gk20a_channel_sync_needs_sync_framework(g) || | ||
2618 | (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE && | ||
2619 | (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT || | ||
2620 | flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET)); | ||
2621 | |||
2622 | /* | ||
2623 | * Deferred clean-up is necessary for any of the following | ||
2624 | * conditions: | ||
2625 | * - channel's deterministic flag is not set | ||
2626 | * - dependency on sync framework, which could make the | ||
2627 | * behavior of the clean-up operation non-deterministic | ||
2628 | * (should not be performed in the submit path) | ||
2629 | * - channel wdt | ||
2630 | * - GPU rail-gating with non-deterministic channels | ||
2631 | * - buffer refcounting | ||
2632 | * | ||
2633 | * If none of the conditions are met, then deferred clean-up | ||
2634 | * is not required, and we clean-up one job-tracking | ||
2635 | * resource in the submit path. | ||
2636 | */ | ||
2637 | need_deferred_cleanup = !c->deterministic || | ||
2638 | need_sync_framework || | ||
2639 | c->wdt_enabled || | ||
2640 | (g->can_railgate && | ||
2641 | !c->deterministic) || | ||
2642 | !skip_buffer_refcounting; | ||
2643 | |||
2644 | /* | ||
2645 | * For deterministic channels, we don't allow deferred clean_up | ||
2646 | * processing to occur. In cases we hit this, we fail the submit | ||
2647 | */ | ||
2648 | if (c->deterministic && need_deferred_cleanup) | ||
2649 | return -EINVAL; | ||
2650 | |||
2651 | if (!c->deterministic) { | ||
2652 | /* | ||
2653 | * Get a power ref unless this is a deterministic | ||
2654 | * channel that holds them during the channel lifetime. | ||
2655 | * This one is released by gk20a_channel_clean_up_jobs, | ||
2656 | * via syncpt or sema interrupt, whichever is used. | ||
2657 | */ | ||
2658 | err = gk20a_busy(g); | ||
2659 | if (err) { | ||
2660 | nvgpu_err(g, | ||
2661 | "failed to host gk20a to submit gpfifo, process %s", | ||
2662 | current->comm); | ||
2663 | return err; | ||
2664 | } | ||
2665 | } | ||
2666 | |||
2667 | if (!need_deferred_cleanup) { | ||
2668 | /* clean up a single job */ | ||
2669 | gk20a_channel_clean_up_jobs(c, false); | ||
2670 | } | ||
2671 | } | ||
2672 | |||
2673 | |||
2674 | /* Grab access to HW to deal with do_idle */ | ||
2675 | if (c->deterministic) | ||
2676 | nvgpu_rwsem_down_read(&g->deterministic_busy); | ||
2677 | |||
2678 | trace_gk20a_channel_submit_gpfifo(g->name, | ||
2679 | c->chid, | ||
2680 | num_entries, | ||
2681 | flags, | ||
2682 | fence ? fence->id : 0, | ||
2683 | fence ? fence->value : 0); | ||
2684 | |||
2685 | gk20a_dbg_info("pre-submit put %d, get %d, size %d", | ||
2686 | c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num); | ||
2687 | |||
2688 | /* | ||
2689 | * Make sure we have enough space for gpfifo entries. Check cached | ||
2690 | * values first and then read from HW. If no space, return EAGAIN | ||
2691 | * and let userpace decide to re-try request or not. | ||
2692 | */ | ||
2693 | if (gp_free_count(c) < num_entries + extra_entries) { | ||
2694 | if (get_gp_free_count(c) < num_entries + extra_entries) { | ||
2695 | err = -EAGAIN; | ||
2696 | goto clean_up; | ||
2697 | } | ||
2698 | } | ||
2699 | |||
2700 | if (c->has_timedout) { | ||
2701 | err = -ETIMEDOUT; | ||
2702 | goto clean_up; | ||
2703 | } | ||
2704 | |||
2705 | if (need_job_tracking) { | ||
2706 | err = channel_gk20a_alloc_job(c, &job); | ||
2707 | if (err) | ||
2708 | goto clean_up; | ||
2709 | |||
2710 | err = gk20a_submit_prepare_syncs(c, fence, job, | ||
2711 | &wait_cmd, &incr_cmd, | ||
2712 | &pre_fence, &post_fence, | ||
2713 | force_need_sync_fence, | ||
2714 | need_deferred_cleanup, | ||
2715 | flags); | ||
2716 | if (err) | ||
2717 | goto clean_up_job; | ||
2718 | } | ||
2719 | |||
2720 | if (profile) | ||
2721 | profile->timestamp[PROFILE_JOB_TRACKING] = sched_clock(); | ||
2722 | |||
2723 | if (wait_cmd) | ||
2724 | gk20a_submit_append_priv_cmdbuf(c, wait_cmd); | ||
2725 | |||
2726 | if (gpfifo || user_gpfifo) | ||
2727 | err = gk20a_submit_append_gpfifo(c, gpfifo, user_gpfifo, | ||
2728 | num_entries); | ||
2729 | if (err) | ||
2730 | goto clean_up_job; | ||
2731 | |||
2732 | /* | ||
2733 | * And here's where we add the incr_cmd we generated earlier. It should | ||
2734 | * always run! | ||
2735 | */ | ||
2736 | if (incr_cmd) | ||
2737 | gk20a_submit_append_priv_cmdbuf(c, incr_cmd); | ||
2738 | |||
2739 | if (fence_out) | ||
2740 | *fence_out = gk20a_fence_get(post_fence); | ||
2741 | |||
2742 | if (need_job_tracking) | ||
2743 | /* TODO! Check for errors... */ | ||
2744 | gk20a_channel_add_job(c, job, skip_buffer_refcounting); | ||
2745 | if (profile) | ||
2746 | profile->timestamp[PROFILE_APPEND] = sched_clock(); | ||
2747 | |||
2748 | g->ops.fifo.userd_gp_put(g, c); | ||
2749 | |||
2750 | if ((NVGPU_SUBMIT_GPFIFO_FLAGS_RESCHEDULE_RUNLIST & flags) && | ||
2751 | g->ops.fifo.reschedule_runlist) | ||
2752 | g->ops.fifo.reschedule_runlist(g, c->runlist_id); | ||
2753 | |||
2754 | /* No hw access beyond this point */ | ||
2755 | if (c->deterministic) | ||
2756 | nvgpu_rwsem_up_read(&g->deterministic_busy); | ||
2757 | |||
2758 | trace_gk20a_channel_submitted_gpfifo(g->name, | ||
2759 | c->chid, | ||
2760 | num_entries, | ||
2761 | flags, | ||
2762 | post_fence ? post_fence->syncpt_id : 0, | ||
2763 | post_fence ? post_fence->syncpt_value : 0); | ||
2764 | |||
2765 | gk20a_dbg_info("post-submit put %d, get %d, size %d", | ||
2766 | c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num); | ||
2767 | |||
2768 | if (profile) | ||
2769 | profile->timestamp[PROFILE_END] = sched_clock(); | ||
2770 | gk20a_dbg_fn("done"); | ||
2771 | return err; | ||
2772 | |||
2773 | clean_up_job: | ||
2774 | channel_gk20a_free_job(c, job); | ||
2775 | clean_up: | ||
2776 | gk20a_dbg_fn("fail"); | ||
2777 | gk20a_fence_put(pre_fence); | ||
2778 | gk20a_fence_put(post_fence); | ||
2779 | if (c->deterministic) | ||
2780 | nvgpu_rwsem_up_read(&g->deterministic_busy); | ||
2781 | else if (need_deferred_cleanup) | ||
2782 | gk20a_idle(g); | ||
2783 | |||
2784 | return err; | ||
2785 | } | ||
2786 | |||
2787 | /* | 2145 | /* |
2788 | * Stop deterministic channel activity for do_idle() when power needs to go off | 2146 | * Stop deterministic channel activity for do_idle() when power needs to go off |
2789 | * momentarily but deterministic channels keep power refs for potentially a | 2147 | * momentarily but deterministic channels keep power refs for potentially a |
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h index 4b1cb351..cdf75a9a 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h | |||
@@ -24,6 +24,9 @@ | |||
24 | #ifndef CHANNEL_GK20A_H | 24 | #ifndef CHANNEL_GK20A_H |
25 | #define CHANNEL_GK20A_H | 25 | #define CHANNEL_GK20A_H |
26 | 26 | ||
27 | /* TODO: To be removed when work_struct update_fn_work is moved out of common code */ | ||
28 | #include <linux/workqueue.h> | ||
29 | |||
27 | #include <linux/stacktrace.h> | 30 | #include <linux/stacktrace.h> |
28 | #include <nvgpu/list.h> | 31 | #include <nvgpu/list.h> |
29 | 32 | ||
@@ -374,16 +377,6 @@ struct channel_gk20a *gk20a_open_new_channel_with_cb(struct gk20a *g, | |||
374 | int runlist_id, | 377 | int runlist_id, |
375 | bool is_privileged_channel); | 378 | bool is_privileged_channel); |
376 | 379 | ||
377 | int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, | ||
378 | struct nvgpu_gpfifo *gpfifo, | ||
379 | struct nvgpu_submit_gpfifo_args *args, | ||
380 | u32 num_entries, | ||
381 | u32 flags, | ||
382 | struct nvgpu_fence *fence, | ||
383 | struct gk20a_fence **fence_out, | ||
384 | bool force_need_sync_fence, | ||
385 | struct fifo_profile_gk20a *profile); | ||
386 | |||
387 | int gk20a_channel_alloc_gpfifo(struct channel_gk20a *c, | 380 | int gk20a_channel_alloc_gpfifo(struct channel_gk20a *c, |
388 | unsigned int num_entries, | 381 | unsigned int num_entries, |
389 | unsigned int num_inflight_jobs, | 382 | unsigned int num_inflight_jobs, |
@@ -408,4 +401,20 @@ int gk20a_channel_set_runlist_interleave(struct channel_gk20a *ch, | |||
408 | void gk20a_channel_event_id_post_event(struct channel_gk20a *ch, | 401 | void gk20a_channel_event_id_post_event(struct channel_gk20a *ch, |
409 | u32 event_id); | 402 | u32 event_id); |
410 | 403 | ||
404 | int channel_gk20a_alloc_job(struct channel_gk20a *c, | ||
405 | struct channel_gk20a_job **job_out); | ||
406 | void channel_gk20a_free_job(struct channel_gk20a *c, | ||
407 | struct channel_gk20a_job *job); | ||
408 | u32 nvgpu_get_gp_free_count(struct channel_gk20a *c); | ||
409 | u32 nvgpu_gp_free_count(struct channel_gk20a *c); | ||
410 | int gk20a_channel_add_job(struct channel_gk20a *c, | ||
411 | struct channel_gk20a_job *job, | ||
412 | bool skip_buffer_refcounting); | ||
413 | void free_priv_cmdbuf(struct channel_gk20a *c, | ||
414 | struct priv_cmd_entry *e); | ||
415 | void gk20a_channel_clean_up_jobs(struct channel_gk20a *c, | ||
416 | bool clean_all); | ||
417 | |||
418 | u32 nvgpu_get_gpfifo_entry_size(void); | ||
419 | |||
411 | #endif /* CHANNEL_GK20A_H */ | 420 | #endif /* CHANNEL_GK20A_H */ |