diff options
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a/channel_gk20a.c')
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/channel_gk20a.c | 302 |
1 files changed, 252 insertions, 50 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c index c12f196d..5a71e874 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c | |||
@@ -42,8 +42,8 @@ | |||
42 | 42 | ||
43 | #define NVMAP_HANDLE_PARAM_SIZE 1 | 43 | #define NVMAP_HANDLE_PARAM_SIZE 1 |
44 | 44 | ||
45 | static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f); | 45 | static struct channel_gk20a *allocate_channel(struct fifo_gk20a *f); |
46 | static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c); | 46 | static void free_channel(struct fifo_gk20a *f, struct channel_gk20a *c); |
47 | 47 | ||
48 | static void free_priv_cmdbuf(struct channel_gk20a *c, | 48 | static void free_priv_cmdbuf(struct channel_gk20a *c, |
49 | struct priv_cmd_entry *e); | 49 | struct priv_cmd_entry *e); |
@@ -61,29 +61,33 @@ static int channel_gk20a_update_runlist(struct channel_gk20a *c, | |||
61 | bool add); | 61 | bool add); |
62 | static void gk20a_free_error_notifiers(struct channel_gk20a *ch); | 62 | static void gk20a_free_error_notifiers(struct channel_gk20a *ch); |
63 | 63 | ||
64 | static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f) | 64 | /* allocate GPU channel */ |
65 | static struct channel_gk20a *allocate_channel(struct fifo_gk20a *f) | ||
65 | { | 66 | { |
66 | struct channel_gk20a *ch = NULL; | 67 | struct channel_gk20a *ch = NULL; |
67 | int chid; | ||
68 | 68 | ||
69 | mutex_lock(&f->ch_inuse_mutex); | 69 | mutex_lock(&f->free_chs_mutex); |
70 | for (chid = 0; chid < f->num_channels; chid++) { | 70 | if (!list_empty(&f->free_chs)) { |
71 | if (!f->channel[chid].in_use) { | 71 | ch = list_first_entry(&f->free_chs, struct channel_gk20a, |
72 | f->channel[chid].in_use = true; | 72 | free_chs); |
73 | ch = &f->channel[chid]; | 73 | list_del(&ch->free_chs); |
74 | break; | 74 | WARN_ON(atomic_read(&ch->ref_count)); |
75 | } | 75 | WARN_ON(ch->referenceable); |
76 | } | 76 | } |
77 | mutex_unlock(&f->ch_inuse_mutex); | 77 | mutex_unlock(&f->free_chs_mutex); |
78 | 78 | ||
79 | return ch; | 79 | return ch; |
80 | } | 80 | } |
81 | 81 | ||
82 | static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c) | 82 | static void free_channel(struct fifo_gk20a *f, |
83 | struct channel_gk20a *ch) | ||
83 | { | 84 | { |
84 | mutex_lock(&f->ch_inuse_mutex); | 85 | trace_gk20a_release_used_channel(ch->hw_chid); |
85 | f->channel[c->hw_chid].in_use = false; | 86 | /* refcount is zero here and channel is in a freed/dead state */ |
86 | mutex_unlock(&f->ch_inuse_mutex); | 87 | mutex_lock(&f->free_chs_mutex); |
88 | /* add to head to increase visibility of timing-related bugs */ | ||
89 | list_add(&ch->free_chs, &f->free_chs); | ||
90 | mutex_unlock(&f->free_chs_mutex); | ||
87 | } | 91 | } |
88 | 92 | ||
89 | int channel_gk20a_commit_va(struct channel_gk20a *c) | 93 | int channel_gk20a_commit_va(struct channel_gk20a *c) |
@@ -361,6 +365,11 @@ void gk20a_channel_abort(struct channel_gk20a *ch) | |||
361 | struct channel_gk20a_job *job, *n; | 365 | struct channel_gk20a_job *job, *n; |
362 | bool released_job_semaphore = false; | 366 | bool released_job_semaphore = false; |
363 | 367 | ||
368 | gk20a_dbg_fn(""); | ||
369 | |||
370 | /* make sure new kickoffs are prevented */ | ||
371 | ch->has_timedout = true; | ||
372 | |||
364 | /* ensure no fences are pending */ | 373 | /* ensure no fences are pending */ |
365 | mutex_lock(&ch->submit_lock); | 374 | mutex_lock(&ch->submit_lock); |
366 | if (ch->sync) | 375 | if (ch->sync) |
@@ -416,6 +425,8 @@ void gk20a_disable_channel(struct channel_gk20a *ch, | |||
416 | bool finish, | 425 | bool finish, |
417 | unsigned long finish_timeout) | 426 | unsigned long finish_timeout) |
418 | { | 427 | { |
428 | gk20a_dbg_fn(""); | ||
429 | |||
419 | if (finish) { | 430 | if (finish) { |
420 | int err = gk20a_channel_finish(ch, finish_timeout); | 431 | int err = gk20a_channel_finish(ch, finish_timeout); |
421 | WARN_ON(err); | 432 | WARN_ON(err); |
@@ -627,8 +638,9 @@ void gk20a_set_error_notifier(struct channel_gk20a *ch, __u32 error) | |||
627 | (u32)(nsec >> 32); | 638 | (u32)(nsec >> 32); |
628 | ch->error_notifier->info32 = error; | 639 | ch->error_notifier->info32 = error; |
629 | ch->error_notifier->status = 0xffff; | 640 | ch->error_notifier->status = 0xffff; |
641 | |||
630 | gk20a_err(dev_from_gk20a(ch->g), | 642 | gk20a_err(dev_from_gk20a(ch->g), |
631 | "error notifier set to %d for ch %d\n", error, ch->hw_chid); | 643 | "error notifier set to %d for ch %d", error, ch->hw_chid); |
632 | } | 644 | } |
633 | } | 645 | } |
634 | 646 | ||
@@ -643,7 +655,53 @@ static void gk20a_free_error_notifiers(struct channel_gk20a *ch) | |||
643 | } | 655 | } |
644 | } | 656 | } |
645 | 657 | ||
646 | void gk20a_free_channel(struct channel_gk20a *ch, bool finish) | 658 | /* Returns delta of cyclic integers a and b. If a is ahead of b, delta |
659 | * is positive */ | ||
660 | static int cyclic_delta(int a, int b) | ||
661 | { | ||
662 | return a - b; | ||
663 | } | ||
664 | |||
665 | static void gk20a_wait_for_deferred_interrupts(struct gk20a *g) | ||
666 | { | ||
667 | int stall_irq_threshold = atomic_read(&g->hw_irq_stall_count); | ||
668 | int nonstall_irq_threshold = atomic_read(&g->hw_irq_nonstall_count); | ||
669 | |||
670 | /* wait until all stalling irqs are handled */ | ||
671 | wait_event(g->sw_irq_stall_last_handled_wq, | ||
672 | cyclic_delta(stall_irq_threshold, | ||
673 | atomic_read(&g->sw_irq_stall_last_handled)) | ||
674 | <= 0); | ||
675 | |||
676 | /* wait until all non-stalling irqs are handled */ | ||
677 | wait_event(g->sw_irq_nonstall_last_handled_wq, | ||
678 | cyclic_delta(nonstall_irq_threshold, | ||
679 | atomic_read(&g->sw_irq_nonstall_last_handled)) | ||
680 | <= 0); | ||
681 | } | ||
682 | |||
683 | static void gk20a_wait_until_counter_is_N( | ||
684 | struct channel_gk20a *ch, atomic_t *counter, int wait_value, | ||
685 | wait_queue_head_t *wq, const char *caller, const char *counter_name) | ||
686 | { | ||
687 | while (true) { | ||
688 | if (wait_event_timeout( | ||
689 | *wq, | ||
690 | atomic_read(counter) == wait_value, | ||
691 | msecs_to_jiffies(5000)) > 0) | ||
692 | break; | ||
693 | |||
694 | gk20a_warn(dev_from_gk20a(ch->g), | ||
695 | "%s: channel %d, still waiting, %s left: %d, waiting for: %d", | ||
696 | caller, ch->hw_chid, counter_name, | ||
697 | atomic_read(counter), wait_value); | ||
698 | } | ||
699 | } | ||
700 | |||
701 | |||
702 | |||
703 | /* call ONLY when no references to the channel exist: after the last put */ | ||
704 | static void gk20a_free_channel(struct channel_gk20a *ch) | ||
647 | { | 705 | { |
648 | struct gk20a *g = ch->g; | 706 | struct gk20a *g = ch->g; |
649 | struct fifo_gk20a *f = &g->fifo; | 707 | struct fifo_gk20a *f = &g->fifo; |
@@ -654,13 +712,50 @@ void gk20a_free_channel(struct channel_gk20a *ch, bool finish) | |||
654 | 712 | ||
655 | gk20a_dbg_fn(""); | 713 | gk20a_dbg_fn(""); |
656 | 714 | ||
715 | WARN_ON(ch->g == NULL); | ||
716 | |||
717 | trace_gk20a_free_channel(ch->hw_chid); | ||
718 | |||
719 | /* prevent new kickoffs */ | ||
720 | ch->has_timedout = true; | ||
721 | wmb(); | ||
722 | |||
723 | /* wait until there's only our ref to the channel */ | ||
724 | gk20a_wait_until_counter_is_N( | ||
725 | ch, &ch->ref_count, 1, &ch->ref_count_dec_wq, | ||
726 | __func__, "references"); | ||
727 | |||
728 | /* wait until all pending interrupts for recently completed | ||
729 | * jobs are handled */ | ||
730 | gk20a_wait_for_deferred_interrupts(g); | ||
731 | |||
732 | /* prevent new refs */ | ||
733 | spin_lock(&ch->ref_obtain_lock); | ||
734 | if (!ch->referenceable) { | ||
735 | spin_unlock(&ch->ref_obtain_lock); | ||
736 | gk20a_err(dev_from_gk20a(ch->g), | ||
737 | "Extra %s() called to channel %u", | ||
738 | __func__, ch->hw_chid); | ||
739 | return; | ||
740 | } | ||
741 | ch->referenceable = false; | ||
742 | spin_unlock(&ch->ref_obtain_lock); | ||
743 | |||
744 | /* matches with the initial reference in gk20a_open_new_channel() */ | ||
745 | atomic_dec(&ch->ref_count); | ||
746 | |||
747 | /* wait until no more refs to the channel */ | ||
748 | gk20a_wait_until_counter_is_N( | ||
749 | ch, &ch->ref_count, 0, &ch->ref_count_dec_wq, | ||
750 | __func__, "references"); | ||
751 | |||
657 | /* if engine reset was deferred, perform it now */ | 752 | /* if engine reset was deferred, perform it now */ |
658 | mutex_lock(&f->deferred_reset_mutex); | 753 | mutex_lock(&f->deferred_reset_mutex); |
659 | if (g->fifo.deferred_reset_pending) { | 754 | if (g->fifo.deferred_reset_pending) { |
660 | gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, "engine reset was" | 755 | gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, "engine reset was" |
661 | " deferred, running now"); | 756 | " deferred, running now"); |
662 | gk20a_fifo_reset_engine(g, g->fifo.mmu_fault_engines); | 757 | gk20a_fifo_reset_engine(g, g->fifo.deferred_fault_engines); |
663 | g->fifo.mmu_fault_engines = 0; | 758 | g->fifo.deferred_fault_engines = 0; |
664 | g->fifo.deferred_reset_pending = false; | 759 | g->fifo.deferred_reset_pending = false; |
665 | } | 760 | } |
666 | mutex_unlock(&f->deferred_reset_mutex); | 761 | mutex_unlock(&f->deferred_reset_mutex); |
@@ -674,7 +769,7 @@ void gk20a_free_channel(struct channel_gk20a *ch, bool finish) | |||
674 | gk20a_dbg_info("freeing bound channel context, timeout=%ld", | 769 | gk20a_dbg_info("freeing bound channel context, timeout=%ld", |
675 | timeout); | 770 | timeout); |
676 | 771 | ||
677 | gk20a_disable_channel(ch, finish && !ch->has_timedout, timeout); | 772 | gk20a_disable_channel(ch, !ch->has_timedout, timeout); |
678 | 773 | ||
679 | gk20a_free_error_notifiers(ch); | 774 | gk20a_free_error_notifiers(ch); |
680 | 775 | ||
@@ -714,6 +809,10 @@ void gk20a_free_channel(struct channel_gk20a *ch, bool finish) | |||
714 | spin_unlock(&ch->update_fn_lock); | 809 | spin_unlock(&ch->update_fn_lock); |
715 | cancel_work_sync(&ch->update_fn_work); | 810 | cancel_work_sync(&ch->update_fn_work); |
716 | 811 | ||
812 | /* make sure we don't have deferred interrupts pending that | ||
813 | * could still touch the channel */ | ||
814 | gk20a_wait_for_deferred_interrupts(g); | ||
815 | |||
717 | unbind: | 816 | unbind: |
718 | if (gk20a_is_channel_marked_as_tsg(ch)) | 817 | if (gk20a_is_channel_marked_as_tsg(ch)) |
719 | gk20a_tsg_unbind_channel(ch); | 818 | gk20a_tsg_unbind_channel(ch); |
@@ -743,8 +842,66 @@ unbind: | |||
743 | mutex_unlock(&ch->dbg_s_lock); | 842 | mutex_unlock(&ch->dbg_s_lock); |
744 | 843 | ||
745 | release: | 844 | release: |
845 | /* make sure we catch accesses of unopened channels in case | ||
846 | * there's non-refcounted channel pointers hanging around */ | ||
847 | ch->g = NULL; | ||
848 | wmb(); | ||
849 | |||
746 | /* ALWAYS last */ | 850 | /* ALWAYS last */ |
747 | release_used_channel(f, ch); | 851 | free_channel(f, ch); |
852 | } | ||
853 | |||
854 | /* Try to get a reference to the channel. Return nonzero on success. If fails, | ||
855 | * the channel is dead or being freed elsewhere and you must not touch it. | ||
856 | * | ||
857 | * Always when a channel_gk20a pointer is seen and about to be used, a | ||
858 | * reference must be held to it - either by you or the caller, which should be | ||
859 | * documented well or otherwise clearly seen. This usually boils down to the | ||
860 | * file from ioctls directly, or an explicit get in exception handlers when the | ||
861 | * channel is found by a hw_chid. | ||
862 | * | ||
863 | * Most global functions in this file require a reference to be held by the | ||
864 | * caller. | ||
865 | */ | ||
866 | struct channel_gk20a *_gk20a_channel_get(struct channel_gk20a *ch, | ||
867 | const char *caller) { | ||
868 | struct channel_gk20a *ret; | ||
869 | |||
870 | spin_lock(&ch->ref_obtain_lock); | ||
871 | |||
872 | if (likely(ch->referenceable)) { | ||
873 | atomic_inc(&ch->ref_count); | ||
874 | ret = ch; | ||
875 | } else | ||
876 | ret = NULL; | ||
877 | |||
878 | spin_unlock(&ch->ref_obtain_lock); | ||
879 | |||
880 | if (ret) | ||
881 | trace_gk20a_channel_get(ch->hw_chid, caller); | ||
882 | |||
883 | return ret; | ||
884 | } | ||
885 | |||
886 | void _gk20a_channel_put(struct channel_gk20a *ch, const char *caller) | ||
887 | { | ||
888 | trace_gk20a_channel_put(ch->hw_chid, caller); | ||
889 | atomic_dec(&ch->ref_count); | ||
890 | wake_up_all(&ch->ref_count_dec_wq); | ||
891 | |||
892 | /* More puts than gets. Channel is probably going to get | ||
893 | * stuck. */ | ||
894 | WARN_ON(atomic_read(&ch->ref_count) < 0); | ||
895 | |||
896 | /* Also, more puts than gets. ref_count can go to 0 only if | ||
897 | * the channel is closing. Channel is probably going to get | ||
898 | * stuck. */ | ||
899 | WARN_ON(atomic_read(&ch->ref_count) == 0 && ch->referenceable); | ||
900 | } | ||
901 | |||
902 | void gk20a_channel_close(struct channel_gk20a *ch) | ||
903 | { | ||
904 | gk20a_free_channel(ch); | ||
748 | } | 905 | } |
749 | 906 | ||
750 | int gk20a_channel_release(struct inode *inode, struct file *filp) | 907 | int gk20a_channel_release(struct inode *inode, struct file *filp) |
@@ -758,14 +915,14 @@ int gk20a_channel_release(struct inode *inode, struct file *filp) | |||
758 | 915 | ||
759 | trace_gk20a_channel_release(dev_name(&g->dev->dev)); | 916 | trace_gk20a_channel_release(dev_name(&g->dev->dev)); |
760 | 917 | ||
761 | err = gk20a_busy(ch->g->dev); | 918 | err = gk20a_busy(g->dev); |
762 | if (err) { | 919 | if (err) { |
763 | gk20a_err(dev_from_gk20a(g), "failed to release channel %d", | 920 | gk20a_err(dev_from_gk20a(g), "failed to release channel %d", |
764 | ch->hw_chid); | 921 | ch->hw_chid); |
765 | return err; | 922 | return err; |
766 | } | 923 | } |
767 | gk20a_free_channel(ch, true); | 924 | gk20a_channel_close(ch); |
768 | gk20a_idle(ch->g->dev); | 925 | gk20a_idle(g->dev); |
769 | 926 | ||
770 | filp->private_data = NULL; | 927 | filp->private_data = NULL; |
771 | return 0; | 928 | return 0; |
@@ -808,22 +965,31 @@ struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g) | |||
808 | struct fifo_gk20a *f = &g->fifo; | 965 | struct fifo_gk20a *f = &g->fifo; |
809 | struct channel_gk20a *ch; | 966 | struct channel_gk20a *ch; |
810 | 967 | ||
811 | ch = acquire_unused_channel(f); | 968 | gk20a_dbg_fn(""); |
969 | |||
970 | ch = allocate_channel(f); | ||
812 | if (ch == NULL) { | 971 | if (ch == NULL) { |
813 | /* TBD: we want to make this virtualizable */ | 972 | /* TBD: we want to make this virtualizable */ |
814 | gk20a_err(dev_from_gk20a(g), "out of hw chids"); | 973 | gk20a_err(dev_from_gk20a(g), "out of hw chids"); |
815 | return NULL; | 974 | return NULL; |
816 | } | 975 | } |
817 | 976 | ||
977 | trace_gk20a_open_new_channel(ch->hw_chid); | ||
978 | |||
979 | BUG_ON(ch->g); | ||
818 | ch->g = g; | 980 | ch->g = g; |
819 | 981 | ||
820 | if (g->ops.fifo.alloc_inst(g, ch)) { | 982 | if (g->ops.fifo.alloc_inst(g, ch)) { |
821 | ch->in_use = false; | 983 | ch->g = NULL; |
984 | free_channel(f, ch); | ||
822 | gk20a_err(dev_from_gk20a(g), | 985 | gk20a_err(dev_from_gk20a(g), |
823 | "failed to open gk20a channel, out of inst mem"); | 986 | "failed to open gk20a channel, out of inst mem"); |
824 | |||
825 | return NULL; | 987 | return NULL; |
826 | } | 988 | } |
989 | |||
990 | /* now the channel is in a limbo out of the free list but not marked as | ||
991 | * alive and used (i.e. get-able) yet */ | ||
992 | |||
827 | ch->pid = current->pid; | 993 | ch->pid = current->pid; |
828 | 994 | ||
829 | /* By default, channel is regular (non-TSG) channel */ | 995 | /* By default, channel is regular (non-TSG) channel */ |
@@ -854,6 +1020,13 @@ struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g) | |||
854 | spin_lock_init(&ch->update_fn_lock); | 1020 | spin_lock_init(&ch->update_fn_lock); |
855 | INIT_WORK(&ch->update_fn_work, gk20a_channel_update_runcb_fn); | 1021 | INIT_WORK(&ch->update_fn_work, gk20a_channel_update_runcb_fn); |
856 | 1022 | ||
1023 | /* Mark the channel alive, get-able, with 1 initial use | ||
1024 | * references. The initial reference will be decreased in | ||
1025 | * gk20a_free_channel() */ | ||
1026 | ch->referenceable = true; | ||
1027 | atomic_set(&ch->ref_count, 1); | ||
1028 | wmb(); | ||
1029 | |||
857 | return ch; | 1030 | return ch; |
858 | } | 1031 | } |
859 | 1032 | ||
@@ -1379,7 +1552,7 @@ static int gk20a_channel_add_job(struct channel_gk20a *c, | |||
1379 | struct mapped_buffer_node **mapped_buffers = NULL; | 1552 | struct mapped_buffer_node **mapped_buffers = NULL; |
1380 | int err = 0, num_mapped_buffers; | 1553 | int err = 0, num_mapped_buffers; |
1381 | 1554 | ||
1382 | /* job needs reference to this vm */ | 1555 | /* job needs reference to this vm (released in channel_update) */ |
1383 | gk20a_vm_get(vm); | 1556 | gk20a_vm_get(vm); |
1384 | 1557 | ||
1385 | err = gk20a_vm_get_buffers(vm, &mapped_buffers, &num_mapped_buffers); | 1558 | err = gk20a_vm_get_buffers(vm, &mapped_buffers, &num_mapped_buffers); |
@@ -1395,14 +1568,21 @@ static int gk20a_channel_add_job(struct channel_gk20a *c, | |||
1395 | return -ENOMEM; | 1568 | return -ENOMEM; |
1396 | } | 1569 | } |
1397 | 1570 | ||
1398 | job->num_mapped_buffers = num_mapped_buffers; | 1571 | /* put() is done in gk20a_channel_update() when the job is done */ |
1399 | job->mapped_buffers = mapped_buffers; | 1572 | c = gk20a_channel_get(c); |
1400 | job->pre_fence = gk20a_fence_get(pre_fence); | ||
1401 | job->post_fence = gk20a_fence_get(post_fence); | ||
1402 | 1573 | ||
1403 | mutex_lock(&c->jobs_lock); | 1574 | if (c) { |
1404 | list_add_tail(&job->list, &c->jobs); | 1575 | job->num_mapped_buffers = num_mapped_buffers; |
1405 | mutex_unlock(&c->jobs_lock); | 1576 | job->mapped_buffers = mapped_buffers; |
1577 | job->pre_fence = gk20a_fence_get(pre_fence); | ||
1578 | job->post_fence = gk20a_fence_get(post_fence); | ||
1579 | |||
1580 | mutex_lock(&c->jobs_lock); | ||
1581 | list_add_tail(&job->list, &c->jobs); | ||
1582 | mutex_unlock(&c->jobs_lock); | ||
1583 | } else { | ||
1584 | return -ETIMEDOUT; | ||
1585 | } | ||
1406 | 1586 | ||
1407 | return 0; | 1587 | return 0; |
1408 | } | 1588 | } |
@@ -1412,13 +1592,15 @@ void gk20a_channel_update(struct channel_gk20a *c, int nr_completed) | |||
1412 | struct vm_gk20a *vm = c->vm; | 1592 | struct vm_gk20a *vm = c->vm; |
1413 | struct channel_gk20a_job *job, *n; | 1593 | struct channel_gk20a_job *job, *n; |
1414 | 1594 | ||
1415 | trace_gk20a_channel_update(c); | 1595 | trace_gk20a_channel_update(c->hw_chid); |
1416 | 1596 | ||
1417 | wake_up(&c->submit_wq); | 1597 | wake_up(&c->submit_wq); |
1418 | 1598 | ||
1419 | mutex_lock(&c->submit_lock); | 1599 | mutex_lock(&c->submit_lock); |
1420 | mutex_lock(&c->jobs_lock); | 1600 | mutex_lock(&c->jobs_lock); |
1421 | list_for_each_entry_safe(job, n, &c->jobs, list) { | 1601 | list_for_each_entry_safe(job, n, &c->jobs, list) { |
1602 | struct gk20a *g = c->g; | ||
1603 | |||
1422 | bool completed = gk20a_fence_is_expired(job->post_fence); | 1604 | bool completed = gk20a_fence_is_expired(job->post_fence); |
1423 | if (!completed) | 1605 | if (!completed) |
1424 | break; | 1606 | break; |
@@ -1434,12 +1616,15 @@ void gk20a_channel_update(struct channel_gk20a *c, int nr_completed) | |||
1434 | gk20a_fence_put(job->pre_fence); | 1616 | gk20a_fence_put(job->pre_fence); |
1435 | gk20a_fence_put(job->post_fence); | 1617 | gk20a_fence_put(job->post_fence); |
1436 | 1618 | ||
1437 | /* job is done. release its reference to vm */ | 1619 | /* job is done. release its vm reference (taken in add_job) */ |
1438 | gk20a_vm_put(vm); | 1620 | gk20a_vm_put(vm); |
1621 | /* another bookkeeping taken in add_job. caller must hold a ref | ||
1622 | * so this wouldn't get freed here. */ | ||
1623 | gk20a_channel_put(c); | ||
1439 | 1624 | ||
1440 | list_del_init(&job->list); | 1625 | list_del_init(&job->list); |
1441 | kfree(job); | 1626 | kfree(job); |
1442 | gk20a_idle(c->g->dev); | 1627 | gk20a_idle(g->dev); |
1443 | } | 1628 | } |
1444 | 1629 | ||
1445 | /* | 1630 | /* |
@@ -1719,10 +1904,13 @@ clean_up: | |||
1719 | int gk20a_init_channel_support(struct gk20a *g, u32 chid) | 1904 | int gk20a_init_channel_support(struct gk20a *g, u32 chid) |
1720 | { | 1905 | { |
1721 | struct channel_gk20a *c = g->fifo.channel+chid; | 1906 | struct channel_gk20a *c = g->fifo.channel+chid; |
1722 | c->g = g; | 1907 | c->g = NULL; |
1723 | c->in_use = false; | ||
1724 | c->hw_chid = chid; | 1908 | c->hw_chid = chid; |
1725 | c->bound = false; | 1909 | c->bound = false; |
1910 | spin_lock_init(&c->ref_obtain_lock); | ||
1911 | atomic_set(&c->ref_count, 0); | ||
1912 | c->referenceable = false; | ||
1913 | init_waitqueue_head(&c->ref_count_dec_wq); | ||
1726 | mutex_init(&c->ioctl_lock); | 1914 | mutex_init(&c->ioctl_lock); |
1727 | mutex_init(&c->jobs_lock); | 1915 | mutex_init(&c->jobs_lock); |
1728 | mutex_init(&c->submit_lock); | 1916 | mutex_init(&c->submit_lock); |
@@ -1733,6 +1921,7 @@ int gk20a_init_channel_support(struct gk20a *g, u32 chid) | |||
1733 | #endif | 1921 | #endif |
1734 | INIT_LIST_HEAD(&c->dbg_s_list); | 1922 | INIT_LIST_HEAD(&c->dbg_s_list); |
1735 | mutex_init(&c->dbg_s_lock); | 1923 | mutex_init(&c->dbg_s_lock); |
1924 | list_add(&c->free_chs, &g->fifo.free_chs); | ||
1736 | 1925 | ||
1737 | return 0; | 1926 | return 0; |
1738 | } | 1927 | } |
@@ -2066,8 +2255,7 @@ int gk20a_channel_suspend(struct gk20a *g) | |||
2066 | 2255 | ||
2067 | for (chid = 0; chid < f->num_channels; chid++) { | 2256 | for (chid = 0; chid < f->num_channels; chid++) { |
2068 | struct channel_gk20a *ch = &f->channel[chid]; | 2257 | struct channel_gk20a *ch = &f->channel[chid]; |
2069 | if (ch->in_use) { | 2258 | if (gk20a_channel_get(ch)) { |
2070 | |||
2071 | gk20a_dbg_info("suspend channel %d", chid); | 2259 | gk20a_dbg_info("suspend channel %d", chid); |
2072 | /* disable channel */ | 2260 | /* disable channel */ |
2073 | g->ops.fifo.disable_channel(ch); | 2261 | g->ops.fifo.disable_channel(ch); |
@@ -2079,6 +2267,8 @@ int gk20a_channel_suspend(struct gk20a *g) | |||
2079 | flush_work(&ch->update_fn_work); | 2267 | flush_work(&ch->update_fn_work); |
2080 | 2268 | ||
2081 | channels_in_use = true; | 2269 | channels_in_use = true; |
2270 | |||
2271 | gk20a_channel_put(ch); | ||
2082 | } | 2272 | } |
2083 | } | 2273 | } |
2084 | 2274 | ||
@@ -2086,8 +2276,10 @@ int gk20a_channel_suspend(struct gk20a *g) | |||
2086 | g->ops.fifo.update_runlist(g, 0, ~0, false, true); | 2276 | g->ops.fifo.update_runlist(g, 0, ~0, false, true); |
2087 | 2277 | ||
2088 | for (chid = 0; chid < f->num_channels; chid++) { | 2278 | for (chid = 0; chid < f->num_channels; chid++) { |
2089 | if (f->channel[chid].in_use) | 2279 | if (gk20a_channel_get(&f->channel[chid])) { |
2090 | g->ops.fifo.unbind_channel(&f->channel[chid]); | 2280 | g->ops.fifo.unbind_channel(&f->channel[chid]); |
2281 | gk20a_channel_put(&f->channel[chid]); | ||
2282 | } | ||
2091 | } | 2283 | } |
2092 | } | 2284 | } |
2093 | 2285 | ||
@@ -2095,8 +2287,6 @@ int gk20a_channel_suspend(struct gk20a *g) | |||
2095 | return 0; | 2287 | return 0; |
2096 | } | 2288 | } |
2097 | 2289 | ||
2098 | /* in this context the "channel" is the host1x channel which | ||
2099 | * maps to *all* gk20a channels */ | ||
2100 | int gk20a_channel_resume(struct gk20a *g) | 2290 | int gk20a_channel_resume(struct gk20a *g) |
2101 | { | 2291 | { |
2102 | struct fifo_gk20a *f = &g->fifo; | 2292 | struct fifo_gk20a *f = &g->fifo; |
@@ -2106,10 +2296,11 @@ int gk20a_channel_resume(struct gk20a *g) | |||
2106 | gk20a_dbg_fn(""); | 2296 | gk20a_dbg_fn(""); |
2107 | 2297 | ||
2108 | for (chid = 0; chid < f->num_channels; chid++) { | 2298 | for (chid = 0; chid < f->num_channels; chid++) { |
2109 | if (f->channel[chid].in_use) { | 2299 | if (gk20a_channel_get(&f->channel[chid])) { |
2110 | gk20a_dbg_info("resume channel %d", chid); | 2300 | gk20a_dbg_info("resume channel %d", chid); |
2111 | g->ops.fifo.bind_channel(&f->channel[chid]); | 2301 | g->ops.fifo.bind_channel(&f->channel[chid]); |
2112 | channels_in_use = true; | 2302 | channels_in_use = true; |
2303 | gk20a_channel_put(&f->channel[chid]); | ||
2113 | } | 2304 | } |
2114 | } | 2305 | } |
2115 | 2306 | ||
@@ -2129,10 +2320,11 @@ void gk20a_channel_semaphore_wakeup(struct gk20a *g) | |||
2129 | 2320 | ||
2130 | for (chid = 0; chid < f->num_channels; chid++) { | 2321 | for (chid = 0; chid < f->num_channels; chid++) { |
2131 | struct channel_gk20a *c = g->fifo.channel+chid; | 2322 | struct channel_gk20a *c = g->fifo.channel+chid; |
2132 | if (c->in_use) { | 2323 | if (gk20a_channel_get(c)) { |
2133 | gk20a_channel_event(c); | 2324 | gk20a_channel_event(c); |
2134 | wake_up_interruptible_all(&c->semaphore_wq); | 2325 | wake_up_interruptible_all(&c->semaphore_wq); |
2135 | gk20a_channel_update(c, 0); | 2326 | gk20a_channel_update(c, 0); |
2327 | gk20a_channel_put(c); | ||
2136 | } | 2328 | } |
2137 | } | 2329 | } |
2138 | } | 2330 | } |
@@ -2225,10 +2417,18 @@ long gk20a_channel_ioctl(struct file *filp, | |||
2225 | return -EFAULT; | 2417 | return -EFAULT; |
2226 | } | 2418 | } |
2227 | 2419 | ||
2420 | /* take a ref or return timeout if channel refs can't be taken */ | ||
2421 | ch = gk20a_channel_get(ch); | ||
2422 | if (!ch) | ||
2423 | return -ETIMEDOUT; | ||
2424 | |||
2228 | /* protect our sanity for threaded userspace - most of the channel is | 2425 | /* protect our sanity for threaded userspace - most of the channel is |
2229 | * not thread safe */ | 2426 | * not thread safe */ |
2230 | mutex_lock(&ch->ioctl_lock); | 2427 | mutex_lock(&ch->ioctl_lock); |
2231 | 2428 | ||
2429 | /* this ioctl call keeps a ref to the file which keeps a ref to the | ||
2430 | * channel */ | ||
2431 | |||
2232 | switch (cmd) { | 2432 | switch (cmd) { |
2233 | case NVGPU_IOCTL_CHANNEL_OPEN: | 2433 | case NVGPU_IOCTL_CHANNEL_OPEN: |
2234 | err = gk20a_channel_open_ioctl(ch->g, | 2434 | err = gk20a_channel_open_ioctl(ch->g, |
@@ -2449,9 +2649,11 @@ long gk20a_channel_ioctl(struct file *filp, | |||
2449 | if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ)) | 2649 | if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ)) |
2450 | err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd)); | 2650 | err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd)); |
2451 | 2651 | ||
2452 | gk20a_dbg_fn("end"); | ||
2453 | |||
2454 | mutex_unlock(&ch->ioctl_lock); | 2652 | mutex_unlock(&ch->ioctl_lock); |
2455 | 2653 | ||
2654 | gk20a_channel_put(ch); | ||
2655 | |||
2656 | gk20a_dbg_fn("end"); | ||
2657 | |||
2456 | return err; | 2658 | return err; |
2457 | } | 2659 | } |