diff options
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a/channel_gk20a.c')
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/channel_gk20a.c | 158 |
1 files changed, 108 insertions, 50 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c index 1f63bbd8..20976992 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c | |||
@@ -28,6 +28,7 @@ | |||
28 | #include <linux/vmalloc.h> | 28 | #include <linux/vmalloc.h> |
29 | 29 | ||
30 | #include "debug_gk20a.h" | 30 | #include "debug_gk20a.h" |
31 | #include "ctxsw_trace_gk20a.h" | ||
31 | 32 | ||
32 | #include "gk20a.h" | 33 | #include "gk20a.h" |
33 | #include "dbg_gpu_gk20a.h" | 34 | #include "dbg_gpu_gk20a.h" |
@@ -44,6 +45,9 @@ | |||
44 | 45 | ||
45 | #define NVGPU_BEGIN_AGGRESSIVE_SYNC_DESTROY_LIMIT 64 /* channels */ | 46 | #define NVGPU_BEGIN_AGGRESSIVE_SYNC_DESTROY_LIMIT 64 /* channels */ |
46 | 47 | ||
48 | #define NVGPU_CHANNEL_MIN_TIMESLICE_US 1000 | ||
49 | #define NVGPU_CHANNEL_MAX_TIMESLICE_US 50000 | ||
50 | |||
47 | static struct channel_gk20a *allocate_channel(struct fifo_gk20a *f); | 51 | static struct channel_gk20a *allocate_channel(struct fifo_gk20a *f); |
48 | static void free_channel(struct fifo_gk20a *f, struct channel_gk20a *c); | 52 | static void free_channel(struct fifo_gk20a *f, struct channel_gk20a *c); |
49 | 53 | ||
@@ -177,7 +181,7 @@ int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g, | |||
177 | } | 181 | } |
178 | 182 | ||
179 | static int channel_gk20a_set_schedule_params(struct channel_gk20a *c, | 183 | static int channel_gk20a_set_schedule_params(struct channel_gk20a *c, |
180 | u32 timeslice_period, bool interleave) | 184 | u32 timeslice_period) |
181 | { | 185 | { |
182 | void *inst_ptr; | 186 | void *inst_ptr; |
183 | int shift = 0, value = 0; | 187 | int shift = 0, value = 0; |
@@ -205,30 +209,6 @@ static int channel_gk20a_set_schedule_params(struct channel_gk20a *c, | |||
205 | gk20a_readl(c->g, ccsr_channel_r(c->hw_chid)) | | 209 | gk20a_readl(c->g, ccsr_channel_r(c->hw_chid)) | |
206 | ccsr_channel_enable_set_true_f()); | 210 | ccsr_channel_enable_set_true_f()); |
207 | 211 | ||
208 | if (c->interleave != interleave) { | ||
209 | mutex_lock(&c->g->interleave_lock); | ||
210 | c->interleave = interleave; | ||
211 | if (interleave) | ||
212 | if (c->g->num_interleaved_channels >= | ||
213 | MAX_INTERLEAVED_CHANNELS) { | ||
214 | gk20a_err(dev_from_gk20a(c->g), | ||
215 | "Change of priority would exceed runlist length, only changing timeslice\n"); | ||
216 | c->interleave = false; | ||
217 | } else | ||
218 | c->g->num_interleaved_channels += 1; | ||
219 | else | ||
220 | c->g->num_interleaved_channels -= 1; | ||
221 | |||
222 | mutex_unlock(&c->g->interleave_lock); | ||
223 | gk20a_dbg_info("Set channel %d to interleave %d", | ||
224 | c->hw_chid, c->interleave); | ||
225 | |||
226 | gk20a_fifo_set_channel_priority( | ||
227 | c->g, 0, c->hw_chid, c->interleave); | ||
228 | c->g->ops.fifo.update_runlist( | ||
229 | c->g, 0, ~0, true, false); | ||
230 | } | ||
231 | |||
232 | return 0; | 212 | return 0; |
233 | } | 213 | } |
234 | 214 | ||
@@ -238,6 +218,12 @@ u32 channel_gk20a_pbdma_acquire_val(struct channel_gk20a *c) | |||
238 | u64 timeout; | 218 | u64 timeout; |
239 | int val_len; | 219 | int val_len; |
240 | 220 | ||
221 | val = pbdma_acquire_retry_man_2_f() | | ||
222 | pbdma_acquire_retry_exp_2_f(); | ||
223 | |||
224 | if (!c->g->timeouts_enabled) | ||
225 | return val; | ||
226 | |||
241 | timeout = gk20a_get_channel_watchdog_timeout(c); | 227 | timeout = gk20a_get_channel_watchdog_timeout(c); |
242 | do_div(timeout, 2); /* set acquire timeout to half of channel wdt */ | 228 | do_div(timeout, 2); /* set acquire timeout to half of channel wdt */ |
243 | timeout *= 1000000UL; /* ms -> ns */ | 229 | timeout *= 1000000UL; /* ms -> ns */ |
@@ -256,11 +242,10 @@ u32 channel_gk20a_pbdma_acquire_val(struct channel_gk20a *c) | |||
256 | man = timeout; | 242 | man = timeout; |
257 | } | 243 | } |
258 | 244 | ||
259 | val = pbdma_acquire_retry_man_2_f() | | 245 | val |= pbdma_acquire_timeout_exp_f(exp) | |
260 | pbdma_acquire_retry_exp_2_f() | | ||
261 | pbdma_acquire_timeout_exp_f(exp) | | ||
262 | pbdma_acquire_timeout_man_f(man) | | 246 | pbdma_acquire_timeout_man_f(man) | |
263 | pbdma_acquire_timeout_en_enable_f(); | 247 | pbdma_acquire_timeout_en_enable_f(); |
248 | |||
264 | return val; | 249 | return val; |
265 | } | 250 | } |
266 | 251 | ||
@@ -711,11 +696,39 @@ static int gk20a_channel_set_wdt_status(struct channel_gk20a *ch, | |||
711 | return 0; | 696 | return 0; |
712 | } | 697 | } |
713 | 698 | ||
714 | static int gk20a_init_error_notifier(struct channel_gk20a *ch, | 699 | static int gk20a_channel_set_runlist_interleave(struct channel_gk20a *ch, |
715 | struct nvgpu_set_error_notifier *args) { | 700 | u32 level) |
716 | void *va; | 701 | { |
702 | struct gk20a *g = ch->g; | ||
703 | int ret; | ||
704 | |||
705 | if (gk20a_is_channel_marked_as_tsg(ch)) { | ||
706 | gk20a_err(dev_from_gk20a(g), "invalid operation for TSG!\n"); | ||
707 | return -EINVAL; | ||
708 | } | ||
709 | |||
710 | switch (level) { | ||
711 | case NVGPU_RUNLIST_INTERLEAVE_LEVEL_LOW: | ||
712 | case NVGPU_RUNLIST_INTERLEAVE_LEVEL_MEDIUM: | ||
713 | case NVGPU_RUNLIST_INTERLEAVE_LEVEL_HIGH: | ||
714 | ret = g->ops.fifo.set_runlist_interleave(g, ch->hw_chid, | ||
715 | false, 0, level); | ||
716 | break; | ||
717 | default: | ||
718 | ret = -EINVAL; | ||
719 | break; | ||
720 | } | ||
717 | 721 | ||
722 | return ret ? ret : g->ops.fifo.update_runlist(g, 0, ~0, true, true); | ||
723 | } | ||
724 | |||
725 | static int gk20a_init_error_notifier(struct channel_gk20a *ch, | ||
726 | struct nvgpu_set_error_notifier *args) | ||
727 | { | ||
728 | struct device *dev = dev_from_gk20a(ch->g); | ||
718 | struct dma_buf *dmabuf; | 729 | struct dma_buf *dmabuf; |
730 | void *va; | ||
731 | u64 end = args->offset + sizeof(struct nvgpu_notification); | ||
719 | 732 | ||
720 | if (!args->mem) { | 733 | if (!args->mem) { |
721 | pr_err("gk20a_init_error_notifier: invalid memory handle\n"); | 734 | pr_err("gk20a_init_error_notifier: invalid memory handle\n"); |
@@ -731,6 +744,13 @@ static int gk20a_init_error_notifier(struct channel_gk20a *ch, | |||
731 | pr_err("Invalid handle: %d\n", args->mem); | 744 | pr_err("Invalid handle: %d\n", args->mem); |
732 | return -EINVAL; | 745 | return -EINVAL; |
733 | } | 746 | } |
747 | |||
748 | if (end > dmabuf->size || end < sizeof(struct nvgpu_notification)) { | ||
749 | dma_buf_put(dmabuf); | ||
750 | gk20a_err(dev, "gk20a_init_error_notifier: invalid offset\n"); | ||
751 | return -EINVAL; | ||
752 | } | ||
753 | |||
734 | /* map handle */ | 754 | /* map handle */ |
735 | va = dma_buf_vmap(dmabuf); | 755 | va = dma_buf_vmap(dmabuf); |
736 | if (!va) { | 756 | if (!va) { |
@@ -890,17 +910,6 @@ static void gk20a_free_channel(struct channel_gk20a *ch) | |||
890 | } | 910 | } |
891 | mutex_unlock(&f->deferred_reset_mutex); | 911 | mutex_unlock(&f->deferred_reset_mutex); |
892 | 912 | ||
893 | if (ch->interleave) { | ||
894 | ch->interleave = false; | ||
895 | gk20a_fifo_set_channel_priority( | ||
896 | ch->g, 0, ch->hw_chid, ch->interleave); | ||
897 | |||
898 | mutex_lock(&f->g->interleave_lock); | ||
899 | WARN_ON(f->g->num_interleaved_channels == 0); | ||
900 | f->g->num_interleaved_channels -= 1; | ||
901 | mutex_unlock(&f->g->interleave_lock); | ||
902 | } | ||
903 | |||
904 | if (!ch->bound) | 913 | if (!ch->bound) |
905 | goto release; | 914 | goto release; |
906 | 915 | ||
@@ -912,6 +921,9 @@ static void gk20a_free_channel(struct channel_gk20a *ch) | |||
912 | 921 | ||
913 | gk20a_free_error_notifiers(ch); | 922 | gk20a_free_error_notifiers(ch); |
914 | 923 | ||
924 | if (g->ops.fecs_trace.unbind_channel) | ||
925 | g->ops.fecs_trace.unbind_channel(g, ch); | ||
926 | |||
915 | /* release channel ctx */ | 927 | /* release channel ctx */ |
916 | g->ops.gr.free_channel_ctx(ch); | 928 | g->ops.gr.free_channel_ctx(ch); |
917 | 929 | ||
@@ -1145,11 +1157,8 @@ struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g) | |||
1145 | ch->has_timedout = false; | 1157 | ch->has_timedout = false; |
1146 | ch->wdt_enabled = true; | 1158 | ch->wdt_enabled = true; |
1147 | ch->obj_class = 0; | 1159 | ch->obj_class = 0; |
1148 | ch->interleave = false; | ||
1149 | ch->clean_up.scheduled = false; | 1160 | ch->clean_up.scheduled = false; |
1150 | gk20a_fifo_set_channel_priority( | 1161 | ch->interleave_level = NVGPU_RUNLIST_INTERLEAVE_LEVEL_LOW; |
1151 | ch->g, 0, ch->hw_chid, ch->interleave); | ||
1152 | |||
1153 | 1162 | ||
1154 | /* The channel is *not* runnable at this point. It still needs to have | 1163 | /* The channel is *not* runnable at this point. It still needs to have |
1155 | * an address space bound and allocate a gpfifo and grctx. */ | 1164 | * an address space bound and allocate a gpfifo and grctx. */ |
@@ -1697,6 +1706,10 @@ static void gk20a_channel_timeout_handler(struct work_struct *work) | |||
1697 | /* Need global lock since multiple channels can timeout at a time */ | 1706 | /* Need global lock since multiple channels can timeout at a time */ |
1698 | mutex_lock(&g->ch_wdt_lock); | 1707 | mutex_lock(&g->ch_wdt_lock); |
1699 | 1708 | ||
1709 | gk20a_debug_dump(g->dev); | ||
1710 | gk20a_gr_debug_dump(g->dev); | ||
1711 | |||
1712 | |||
1700 | /* Get timed out job and reset the timer */ | 1713 | /* Get timed out job and reset the timer */ |
1701 | mutex_lock(&ch->timeout.lock); | 1714 | mutex_lock(&ch->timeout.lock); |
1702 | job = ch->timeout.job; | 1715 | job = ch->timeout.job; |
@@ -2399,6 +2412,7 @@ static int gk20a_channel_wait(struct channel_gk20a *ch, | |||
2399 | u32 offset; | 2412 | u32 offset; |
2400 | unsigned long timeout; | 2413 | unsigned long timeout; |
2401 | int remain, ret = 0; | 2414 | int remain, ret = 0; |
2415 | u64 end; | ||
2402 | 2416 | ||
2403 | gk20a_dbg_fn(""); | 2417 | gk20a_dbg_fn(""); |
2404 | 2418 | ||
@@ -2414,6 +2428,7 @@ static int gk20a_channel_wait(struct channel_gk20a *ch, | |||
2414 | case NVGPU_WAIT_TYPE_NOTIFIER: | 2428 | case NVGPU_WAIT_TYPE_NOTIFIER: |
2415 | id = args->condition.notifier.dmabuf_fd; | 2429 | id = args->condition.notifier.dmabuf_fd; |
2416 | offset = args->condition.notifier.offset; | 2430 | offset = args->condition.notifier.offset; |
2431 | end = offset + sizeof(struct notification); | ||
2417 | 2432 | ||
2418 | dmabuf = dma_buf_get(id); | 2433 | dmabuf = dma_buf_get(id); |
2419 | if (IS_ERR(dmabuf)) { | 2434 | if (IS_ERR(dmabuf)) { |
@@ -2422,6 +2437,12 @@ static int gk20a_channel_wait(struct channel_gk20a *ch, | |||
2422 | return -EINVAL; | 2437 | return -EINVAL; |
2423 | } | 2438 | } |
2424 | 2439 | ||
2440 | if (end > dmabuf->size || end < sizeof(struct notification)) { | ||
2441 | dma_buf_put(dmabuf); | ||
2442 | gk20a_err(d, "invalid notifier offset\n"); | ||
2443 | return -EINVAL; | ||
2444 | } | ||
2445 | |||
2425 | notif = dma_buf_vmap(dmabuf); | 2446 | notif = dma_buf_vmap(dmabuf); |
2426 | if (!notif) { | 2447 | if (!notif) { |
2427 | gk20a_err(d, "failed to map notifier memory"); | 2448 | gk20a_err(d, "failed to map notifier memory"); |
@@ -2596,7 +2617,6 @@ unsigned int gk20a_channel_poll(struct file *filep, poll_table *wait) | |||
2596 | int gk20a_channel_set_priority(struct channel_gk20a *ch, u32 priority) | 2617 | int gk20a_channel_set_priority(struct channel_gk20a *ch, u32 priority) |
2597 | { | 2618 | { |
2598 | u32 timeslice_timeout; | 2619 | u32 timeslice_timeout; |
2599 | bool interleave = false; | ||
2600 | 2620 | ||
2601 | if (gk20a_is_channel_marked_as_tsg(ch)) { | 2621 | if (gk20a_is_channel_marked_as_tsg(ch)) { |
2602 | gk20a_err(dev_from_gk20a(ch->g), | 2622 | gk20a_err(dev_from_gk20a(ch->g), |
@@ -2613,8 +2633,6 @@ int gk20a_channel_set_priority(struct channel_gk20a *ch, u32 priority) | |||
2613 | timeslice_timeout = ch->g->timeslice_medium_priority_us; | 2633 | timeslice_timeout = ch->g->timeslice_medium_priority_us; |
2614 | break; | 2634 | break; |
2615 | case NVGPU_PRIORITY_HIGH: | 2635 | case NVGPU_PRIORITY_HIGH: |
2616 | if (ch->g->interleave_high_priority) | ||
2617 | interleave = true; | ||
2618 | timeslice_timeout = ch->g->timeslice_high_priority_us; | 2636 | timeslice_timeout = ch->g->timeslice_high_priority_us; |
2619 | break; | 2637 | break; |
2620 | default: | 2638 | default: |
@@ -2623,7 +2641,22 @@ int gk20a_channel_set_priority(struct channel_gk20a *ch, u32 priority) | |||
2623 | } | 2641 | } |
2624 | 2642 | ||
2625 | return channel_gk20a_set_schedule_params(ch, | 2643 | return channel_gk20a_set_schedule_params(ch, |
2626 | timeslice_timeout, interleave); | 2644 | timeslice_timeout); |
2645 | } | ||
2646 | |||
2647 | int gk20a_channel_set_timeslice(struct channel_gk20a *ch, u32 timeslice) | ||
2648 | { | ||
2649 | if (gk20a_is_channel_marked_as_tsg(ch)) { | ||
2650 | gk20a_err(dev_from_gk20a(ch->g), | ||
2651 | "invalid operation for TSG!\n"); | ||
2652 | return -EINVAL; | ||
2653 | } | ||
2654 | |||
2655 | if (timeslice < NVGPU_CHANNEL_MIN_TIMESLICE_US || | ||
2656 | timeslice > NVGPU_CHANNEL_MAX_TIMESLICE_US) | ||
2657 | return -EINVAL; | ||
2658 | |||
2659 | return channel_gk20a_set_schedule_params(ch, timeslice); | ||
2627 | } | 2660 | } |
2628 | 2661 | ||
2629 | static int gk20a_channel_zcull_bind(struct channel_gk20a *ch, | 2662 | static int gk20a_channel_zcull_bind(struct channel_gk20a *ch, |
@@ -2778,6 +2811,7 @@ void gk20a_init_channel(struct gpu_ops *gops) | |||
2778 | gops->fifo.free_inst = channel_gk20a_free_inst; | 2811 | gops->fifo.free_inst = channel_gk20a_free_inst; |
2779 | gops->fifo.setup_ramfc = channel_gk20a_setup_ramfc; | 2812 | gops->fifo.setup_ramfc = channel_gk20a_setup_ramfc; |
2780 | gops->fifo.channel_set_priority = gk20a_channel_set_priority; | 2813 | gops->fifo.channel_set_priority = gk20a_channel_set_priority; |
2814 | gops->fifo.channel_set_timeslice = gk20a_channel_set_timeslice; | ||
2781 | } | 2815 | } |
2782 | 2816 | ||
2783 | long gk20a_channel_ioctl(struct file *filp, | 2817 | long gk20a_channel_ioctl(struct file *filp, |
@@ -3028,6 +3062,30 @@ long gk20a_channel_ioctl(struct file *filp, | |||
3028 | err = gk20a_channel_set_wdt_status(ch, | 3062 | err = gk20a_channel_set_wdt_status(ch, |
3029 | (struct nvgpu_channel_wdt_args *)buf); | 3063 | (struct nvgpu_channel_wdt_args *)buf); |
3030 | break; | 3064 | break; |
3065 | case NVGPU_IOCTL_CHANNEL_SET_RUNLIST_INTERLEAVE: | ||
3066 | err = gk20a_busy(dev); | ||
3067 | if (err) { | ||
3068 | dev_err(&dev->dev, | ||
3069 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
3070 | __func__, cmd); | ||
3071 | break; | ||
3072 | } | ||
3073 | err = gk20a_channel_set_runlist_interleave(ch, | ||
3074 | ((struct nvgpu_runlist_interleave_args *)buf)->level); | ||
3075 | gk20a_idle(dev); | ||
3076 | break; | ||
3077 | case NVGPU_IOCTL_CHANNEL_SET_TIMESLICE: | ||
3078 | err = gk20a_busy(dev); | ||
3079 | if (err) { | ||
3080 | dev_err(&dev->dev, | ||
3081 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
3082 | __func__, cmd); | ||
3083 | break; | ||
3084 | } | ||
3085 | err = ch->g->ops.fifo.channel_set_timeslice(ch, | ||
3086 | ((struct nvgpu_timeslice_args *)buf)->timeslice_us); | ||
3087 | gk20a_idle(dev); | ||
3088 | break; | ||
3031 | default: | 3089 | default: |
3032 | dev_dbg(&dev->dev, "unrecognized ioctl cmd: 0x%x", cmd); | 3090 | dev_dbg(&dev->dev, "unrecognized ioctl cmd: 0x%x", cmd); |
3033 | err = -ENOTTY; | 3091 | err = -ENOTTY; |