diff options
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a/channel_gk20a.c')
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/channel_gk20a.c | 131 |
1 files changed, 78 insertions, 53 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c index de82be36..9f8876c3 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c | |||
@@ -33,6 +33,7 @@ | |||
33 | 33 | ||
34 | #include "gk20a.h" | 34 | #include "gk20a.h" |
35 | #include "dbg_gpu_gk20a.h" | 35 | #include "dbg_gpu_gk20a.h" |
36 | #include "fence_gk20a.h" | ||
36 | #include "semaphore_gk20a.h" | 37 | #include "semaphore_gk20a.h" |
37 | 38 | ||
38 | #include "hw_ram_gk20a.h" | 39 | #include "hw_ram_gk20a.h" |
@@ -418,8 +419,8 @@ void gk20a_channel_abort(struct channel_gk20a *ch) | |||
418 | semaphore synchronization) */ | 419 | semaphore synchronization) */ |
419 | mutex_lock(&ch->jobs_lock); | 420 | mutex_lock(&ch->jobs_lock); |
420 | list_for_each_entry_safe(job, n, &ch->jobs, list) { | 421 | list_for_each_entry_safe(job, n, &ch->jobs, list) { |
421 | if (job->post_fence.semaphore) { | 422 | if (job->post_fence->semaphore) { |
422 | gk20a_semaphore_release(job->post_fence.semaphore); | 423 | gk20a_semaphore_release(job->post_fence->semaphore); |
423 | released_job_semaphore = true; | 424 | released_job_semaphore = true; |
424 | } | 425 | } |
425 | } | 426 | } |
@@ -685,8 +686,12 @@ unbind: | |||
685 | ch->vpr = false; | 686 | ch->vpr = false; |
686 | ch->vm = NULL; | 687 | ch->vm = NULL; |
687 | 688 | ||
688 | gk20a_channel_fence_close(&ch->last_submit.pre_fence); | 689 | mutex_lock(&ch->submit_lock); |
689 | gk20a_channel_fence_close(&ch->last_submit.post_fence); | 690 | gk20a_fence_put(ch->last_submit.pre_fence); |
691 | gk20a_fence_put(ch->last_submit.post_fence); | ||
692 | ch->last_submit.pre_fence = NULL; | ||
693 | ch->last_submit.post_fence = NULL; | ||
694 | mutex_unlock(&ch->submit_lock); | ||
690 | WARN_ON(ch->sync); | 695 | WARN_ON(ch->sync); |
691 | 696 | ||
692 | /* unlink all debug sessions */ | 697 | /* unlink all debug sessions */ |
@@ -1119,8 +1124,12 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, | |||
1119 | ch_vm = c->vm; | 1124 | ch_vm = c->vm; |
1120 | 1125 | ||
1121 | c->cmds_pending = false; | 1126 | c->cmds_pending = false; |
1122 | gk20a_channel_fence_close(&c->last_submit.pre_fence); | 1127 | mutex_lock(&c->submit_lock); |
1123 | gk20a_channel_fence_close(&c->last_submit.post_fence); | 1128 | gk20a_fence_put(c->last_submit.pre_fence); |
1129 | gk20a_fence_put(c->last_submit.post_fence); | ||
1130 | c->last_submit.pre_fence = NULL; | ||
1131 | c->last_submit.post_fence = NULL; | ||
1132 | mutex_unlock(&c->submit_lock); | ||
1124 | 1133 | ||
1125 | c->ramfc.offset = 0; | 1134 | c->ramfc.offset = 0; |
1126 | c->ramfc.size = ram_in_ramfc_s() / 8; | 1135 | c->ramfc.size = ram_in_ramfc_s() / 8; |
@@ -1303,8 +1312,10 @@ static int gk20a_channel_submit_wfi(struct channel_gk20a *c) | |||
1303 | } | 1312 | } |
1304 | } | 1313 | } |
1305 | 1314 | ||
1306 | gk20a_channel_fence_close(&c->last_submit.pre_fence); | 1315 | gk20a_fence_put(c->last_submit.pre_fence); |
1307 | gk20a_channel_fence_close(&c->last_submit.post_fence); | 1316 | gk20a_fence_put(c->last_submit.post_fence); |
1317 | c->last_submit.pre_fence = NULL; | ||
1318 | c->last_submit.post_fence = NULL; | ||
1308 | 1319 | ||
1309 | err = c->sync->incr_wfi(c->sync, &cmd, &c->last_submit.post_fence); | 1320 | err = c->sync->incr_wfi(c->sync, &cmd, &c->last_submit.post_fence); |
1310 | if (unlikely(err)) { | 1321 | if (unlikely(err)) { |
@@ -1312,7 +1323,7 @@ static int gk20a_channel_submit_wfi(struct channel_gk20a *c) | |||
1312 | return err; | 1323 | return err; |
1313 | } | 1324 | } |
1314 | 1325 | ||
1315 | WARN_ON(!c->last_submit.post_fence.wfi); | 1326 | WARN_ON(!c->last_submit.post_fence->wfi); |
1316 | 1327 | ||
1317 | c->gpfifo.cpu_va[c->gpfifo.put].entry0 = u64_lo32(cmd->gva); | 1328 | c->gpfifo.cpu_va[c->gpfifo.put].entry0 = u64_lo32(cmd->gva); |
1318 | c->gpfifo.cpu_va[c->gpfifo.put].entry1 = u64_hi32(cmd->gva) | | 1329 | c->gpfifo.cpu_va[c->gpfifo.put].entry1 = u64_hi32(cmd->gva) | |
@@ -1378,8 +1389,8 @@ static void trace_write_pushbuffer(struct channel_gk20a *c, struct gpfifo *g) | |||
1378 | } | 1389 | } |
1379 | 1390 | ||
1380 | static int gk20a_channel_add_job(struct channel_gk20a *c, | 1391 | static int gk20a_channel_add_job(struct channel_gk20a *c, |
1381 | struct gk20a_channel_fence *pre_fence, | 1392 | struct gk20a_fence *pre_fence, |
1382 | struct gk20a_channel_fence *post_fence) | 1393 | struct gk20a_fence *post_fence) |
1383 | { | 1394 | { |
1384 | struct vm_gk20a *vm = c->vm; | 1395 | struct vm_gk20a *vm = c->vm; |
1385 | struct channel_gk20a_job *job = NULL; | 1396 | struct channel_gk20a_job *job = NULL; |
@@ -1404,8 +1415,8 @@ static int gk20a_channel_add_job(struct channel_gk20a *c, | |||
1404 | 1415 | ||
1405 | job->num_mapped_buffers = num_mapped_buffers; | 1416 | job->num_mapped_buffers = num_mapped_buffers; |
1406 | job->mapped_buffers = mapped_buffers; | 1417 | job->mapped_buffers = mapped_buffers; |
1407 | gk20a_channel_fence_dup(pre_fence, &job->pre_fence); | 1418 | job->pre_fence = gk20a_fence_get(pre_fence); |
1408 | gk20a_channel_fence_dup(post_fence, &job->post_fence); | 1419 | job->post_fence = gk20a_fence_get(post_fence); |
1409 | 1420 | ||
1410 | mutex_lock(&c->jobs_lock); | 1421 | mutex_lock(&c->jobs_lock); |
1411 | list_add_tail(&job->list, &c->jobs); | 1422 | list_add_tail(&job->list, &c->jobs); |
@@ -1424,18 +1435,19 @@ void gk20a_channel_update(struct channel_gk20a *c, int nr_completed) | |||
1424 | mutex_lock(&c->submit_lock); | 1435 | mutex_lock(&c->submit_lock); |
1425 | mutex_lock(&c->jobs_lock); | 1436 | mutex_lock(&c->jobs_lock); |
1426 | list_for_each_entry_safe(job, n, &c->jobs, list) { | 1437 | list_for_each_entry_safe(job, n, &c->jobs, list) { |
1427 | bool completed = WARN_ON(!c->sync) || | 1438 | bool completed = gk20a_fence_is_expired(job->post_fence); |
1428 | c->sync->is_expired(c->sync, &job->post_fence); | ||
1429 | if (!completed) | 1439 | if (!completed) |
1430 | break; | 1440 | break; |
1431 | 1441 | ||
1442 | c->sync->signal_timeline(c->sync); | ||
1443 | |||
1432 | gk20a_vm_put_buffers(vm, job->mapped_buffers, | 1444 | gk20a_vm_put_buffers(vm, job->mapped_buffers, |
1433 | job->num_mapped_buffers); | 1445 | job->num_mapped_buffers); |
1434 | 1446 | ||
1435 | /* Close the fences (this will unref the semaphores and release | 1447 | /* Close the fences (this will unref the semaphores and release |
1436 | * them to the pool). */ | 1448 | * them to the pool). */ |
1437 | gk20a_channel_fence_close(&job->pre_fence); | 1449 | gk20a_fence_put(job->pre_fence); |
1438 | gk20a_channel_fence_close(&job->post_fence); | 1450 | gk20a_fence_put(job->post_fence); |
1439 | 1451 | ||
1440 | /* job is done. release its reference to vm */ | 1452 | /* job is done. release its reference to vm */ |
1441 | gk20a_vm_put(vm); | 1453 | gk20a_vm_put(vm); |
@@ -1453,7 +1465,7 @@ void gk20a_channel_update(struct channel_gk20a *c, int nr_completed) | |||
1453 | */ | 1465 | */ |
1454 | if (list_empty(&c->jobs)) { | 1466 | if (list_empty(&c->jobs)) { |
1455 | if (c->sync && c->sync->aggressive_destroy && | 1467 | if (c->sync && c->sync->aggressive_destroy && |
1456 | c->sync->is_expired(c->sync, &c->last_submit.post_fence)) { | 1468 | gk20a_fence_is_expired(c->last_submit.post_fence)) { |
1457 | c->sync->destroy(c->sync); | 1469 | c->sync->destroy(c->sync); |
1458 | c->sync = NULL; | 1470 | c->sync = NULL; |
1459 | } | 1471 | } |
@@ -1477,8 +1489,9 @@ void add_wait_cmd(u32 *ptr, u32 id, u32 thresh) | |||
1477 | int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, | 1489 | int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, |
1478 | struct nvhost_gpfifo *gpfifo, | 1490 | struct nvhost_gpfifo *gpfifo, |
1479 | u32 num_entries, | 1491 | u32 num_entries, |
1492 | u32 flags, | ||
1480 | struct nvhost_fence *fence, | 1493 | struct nvhost_fence *fence, |
1481 | u32 flags) | 1494 | struct gk20a_fence **fence_out) |
1482 | { | 1495 | { |
1483 | struct gk20a *g = c->g; | 1496 | struct gk20a *g = c->g; |
1484 | struct device *d = dev_from_gk20a(g); | 1497 | struct device *d = dev_from_gk20a(g); |
@@ -1487,8 +1500,8 @@ int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, | |||
1487 | int wait_fence_fd = -1; | 1500 | int wait_fence_fd = -1; |
1488 | struct priv_cmd_entry *wait_cmd = NULL; | 1501 | struct priv_cmd_entry *wait_cmd = NULL; |
1489 | struct priv_cmd_entry *incr_cmd = NULL; | 1502 | struct priv_cmd_entry *incr_cmd = NULL; |
1490 | struct gk20a_channel_fence pre_fence = { 0 }; | 1503 | struct gk20a_fence *pre_fence = NULL; |
1491 | struct gk20a_channel_fence post_fence = { 0 }; | 1504 | struct gk20a_fence *post_fence = NULL; |
1492 | /* we might need two extra gpfifo entries - one for pre fence | 1505 | /* we might need two extra gpfifo entries - one for pre fence |
1493 | * and one for post fence. */ | 1506 | * and one for post fence. */ |
1494 | const int extra_entries = 2; | 1507 | const int extra_entries = 2; |
@@ -1591,18 +1604,9 @@ int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, | |||
1591 | 1604 | ||
1592 | /* always insert syncpt increment at end of gpfifo submission | 1605 | /* always insert syncpt increment at end of gpfifo submission |
1593 | to keep track of method completion for idle railgating */ | 1606 | to keep track of method completion for idle railgating */ |
1594 | if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET && | 1607 | if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET) |
1595 | flags & NVHOST_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE) | 1608 | err = c->sync->incr_user(c->sync, wait_fence_fd, &incr_cmd, |
1596 | err = c->sync->incr_user_fd(c->sync, wait_fence_fd, &incr_cmd, | 1609 | &post_fence, need_wfi); |
1597 | &post_fence, | ||
1598 | need_wfi, | ||
1599 | &fence->syncpt_id); | ||
1600 | else if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET) | ||
1601 | err = c->sync->incr_user_syncpt(c->sync, &incr_cmd, | ||
1602 | &post_fence, | ||
1603 | need_wfi, | ||
1604 | &fence->syncpt_id, | ||
1605 | &fence->value); | ||
1606 | else | 1610 | else |
1607 | err = c->sync->incr(c->sync, &incr_cmd, | 1611 | err = c->sync->incr(c->sync, &incr_cmd, |
1608 | &post_fence); | 1612 | &post_fence); |
@@ -1653,13 +1657,15 @@ int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, | |||
1653 | incr_cmd->gp_put = c->gpfifo.put; | 1657 | incr_cmd->gp_put = c->gpfifo.put; |
1654 | } | 1658 | } |
1655 | 1659 | ||
1656 | gk20a_channel_fence_close(&c->last_submit.pre_fence); | 1660 | gk20a_fence_put(c->last_submit.pre_fence); |
1657 | gk20a_channel_fence_close(&c->last_submit.post_fence); | 1661 | gk20a_fence_put(c->last_submit.post_fence); |
1658 | c->last_submit.pre_fence = pre_fence; | 1662 | c->last_submit.pre_fence = pre_fence; |
1659 | c->last_submit.post_fence = post_fence; | 1663 | c->last_submit.post_fence = post_fence; |
1664 | if (fence_out) | ||
1665 | *fence_out = gk20a_fence_get(post_fence); | ||
1660 | 1666 | ||
1661 | /* TODO! Check for errors... */ | 1667 | /* TODO! Check for errors... */ |
1662 | gk20a_channel_add_job(c, &pre_fence, &post_fence); | 1668 | gk20a_channel_add_job(c, pre_fence, post_fence); |
1663 | 1669 | ||
1664 | c->cmds_pending = true; | 1670 | c->cmds_pending = true; |
1665 | gk20a_bar1_writel(g, | 1671 | gk20a_bar1_writel(g, |
@@ -1672,8 +1678,8 @@ int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, | |||
1672 | c->hw_chid, | 1678 | c->hw_chid, |
1673 | num_entries, | 1679 | num_entries, |
1674 | flags, | 1680 | flags, |
1675 | fence ? fence->syncpt_id : 0, | 1681 | post_fence->syncpt_id, |
1676 | fence ? fence->value : 0); | 1682 | post_fence->syncpt_value); |
1677 | 1683 | ||
1678 | gk20a_dbg_info("post-submit put %d, get %d, size %d", | 1684 | gk20a_dbg_info("post-submit put %d, get %d, size %d", |
1679 | c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num); | 1685 | c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num); |
@@ -1685,8 +1691,8 @@ clean_up: | |||
1685 | gk20a_err(d, "fail"); | 1691 | gk20a_err(d, "fail"); |
1686 | free_priv_cmdbuf(c, wait_cmd); | 1692 | free_priv_cmdbuf(c, wait_cmd); |
1687 | free_priv_cmdbuf(c, incr_cmd); | 1693 | free_priv_cmdbuf(c, incr_cmd); |
1688 | gk20a_channel_fence_close(&pre_fence); | 1694 | gk20a_fence_put(pre_fence); |
1689 | gk20a_channel_fence_close(&post_fence); | 1695 | gk20a_fence_put(post_fence); |
1690 | gk20a_idle(g->dev); | 1696 | gk20a_idle(g->dev); |
1691 | return err; | 1697 | return err; |
1692 | } | 1698 | } |
@@ -1719,7 +1725,7 @@ int gk20a_init_channel_support(struct gk20a *g, u32 chid) | |||
1719 | int gk20a_channel_finish(struct channel_gk20a *ch, unsigned long timeout) | 1725 | int gk20a_channel_finish(struct channel_gk20a *ch, unsigned long timeout) |
1720 | { | 1726 | { |
1721 | int err = 0; | 1727 | int err = 0; |
1722 | struct gk20a_channel_fence *fence = &ch->last_submit.post_fence; | 1728 | struct gk20a_fence *fence = ch->last_submit.post_fence; |
1723 | 1729 | ||
1724 | if (!ch->cmds_pending) | 1730 | if (!ch->cmds_pending) |
1725 | return 0; | 1731 | return 0; |
@@ -1728,26 +1734,25 @@ int gk20a_channel_finish(struct channel_gk20a *ch, unsigned long timeout) | |||
1728 | if (ch->has_timedout) | 1734 | if (ch->has_timedout) |
1729 | return -ETIMEDOUT; | 1735 | return -ETIMEDOUT; |
1730 | 1736 | ||
1731 | if (!(fence->valid && fence->wfi) && ch->obj_class != KEPLER_C) { | 1737 | if (!(fence && fence->wfi) && ch->obj_class != KEPLER_C) { |
1732 | gk20a_dbg_fn("issuing wfi, incr to finish the channel"); | 1738 | gk20a_dbg_fn("issuing wfi, incr to finish the channel"); |
1733 | err = gk20a_channel_submit_wfi(ch); | 1739 | err = gk20a_channel_submit_wfi(ch); |
1740 | fence = ch->last_submit.post_fence; | ||
1734 | } | 1741 | } |
1735 | if (err) | 1742 | if (err) |
1736 | return err; | 1743 | return err; |
1737 | 1744 | ||
1738 | BUG_ON(!(fence->valid && fence->wfi) && ch->obj_class != KEPLER_C); | 1745 | BUG_ON(!(fence && fence->wfi) && ch->obj_class != KEPLER_C); |
1739 | 1746 | ||
1740 | gk20a_dbg_fn("waiting for channel to finish thresh:%d sema:%p", | 1747 | gk20a_dbg_fn("waiting for channel to finish thresh:%d sema:%p", |
1741 | fence->thresh, fence->semaphore); | 1748 | fence->syncpt_value, fence->semaphore); |
1742 | 1749 | ||
1743 | if (ch->sync) { | 1750 | err = gk20a_fence_wait(fence, timeout); |
1744 | err = ch->sync->wait_cpu(ch->sync, fence, timeout); | 1751 | if (WARN_ON(err)) |
1745 | if (WARN_ON(err)) | 1752 | dev_warn(dev_from_gk20a(ch->g), |
1746 | dev_warn(dev_from_gk20a(ch->g), | 1753 | "timed out waiting for gk20a channel to finish"); |
1747 | "timed out waiting for gk20a channel to finish"); | 1754 | else |
1748 | else | 1755 | ch->cmds_pending = false; |
1749 | ch->cmds_pending = false; | ||
1750 | } | ||
1751 | 1756 | ||
1752 | return err; | 1757 | return err; |
1753 | } | 1758 | } |
@@ -2014,6 +2019,7 @@ static int gk20a_ioctl_channel_submit_gpfifo( | |||
2014 | struct channel_gk20a *ch, | 2019 | struct channel_gk20a *ch, |
2015 | struct nvhost_submit_gpfifo_args *args) | 2020 | struct nvhost_submit_gpfifo_args *args) |
2016 | { | 2021 | { |
2022 | struct gk20a_fence *fence_out; | ||
2017 | void *gpfifo; | 2023 | void *gpfifo; |
2018 | u32 size; | 2024 | u32 size; |
2019 | int ret = 0; | 2025 | int ret = 0; |
@@ -2036,7 +2042,26 @@ static int gk20a_ioctl_channel_submit_gpfifo( | |||
2036 | } | 2042 | } |
2037 | 2043 | ||
2038 | ret = gk20a_submit_channel_gpfifo(ch, gpfifo, args->num_entries, | 2044 | ret = gk20a_submit_channel_gpfifo(ch, gpfifo, args->num_entries, |
2039 | &args->fence, args->flags); | 2045 | args->flags, &args->fence, |
2046 | &fence_out); | ||
2047 | |||
2048 | if (ret) | ||
2049 | goto clean_up; | ||
2050 | |||
2051 | /* Convert fence_out to something we can pass back to user space. */ | ||
2052 | if (args->flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET) { | ||
2053 | if (args->flags & NVHOST_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE) { | ||
2054 | int fd = gk20a_fence_install_fd(fence_out); | ||
2055 | if (fd < 0) | ||
2056 | ret = fd; | ||
2057 | else | ||
2058 | args->fence.syncpt_id = fd; | ||
2059 | } else { | ||
2060 | args->fence.syncpt_id = fence_out->syncpt_id; | ||
2061 | args->fence.value = fence_out->syncpt_value; | ||
2062 | } | ||
2063 | } | ||
2064 | gk20a_fence_put(fence_out); | ||
2040 | 2065 | ||
2041 | clean_up: | 2066 | clean_up: |
2042 | kfree(gpfifo); | 2067 | kfree(gpfifo); |