summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a/channel_gk20a.c')
-rw-r--r--drivers/gpu/nvgpu/gk20a/channel_gk20a.c2111
1 files changed, 2111 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
new file mode 100644
index 00000000..6056f558
--- /dev/null
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
@@ -0,0 +1,2111 @@
1/*
2 * drivers/video/tegra/host/gk20a/channel_gk20a.c
3 *
4 * GK20A Graphics channel
5 *
6 * Copyright (c) 2011-2014, NVIDIA CORPORATION. All rights reserved.
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License,
10 * version 2, as published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * more details.
16 *
17 * You should have received a copy of the GNU General Public License along with
18 * this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20 */
21
22#include <linux/nvhost.h>
23#include <linux/list.h>
24#include <linux/delay.h>
25#include <linux/highmem.h> /* need for nvmap.h*/
26#include <trace/events/gk20a.h>
27#include <linux/scatterlist.h>
28#include <linux/file.h>
29#include <linux/anon_inodes.h>
30#include <linux/dma-buf.h>
31
32#include "debug_gk20a.h"
33
34#include "gk20a.h"
35#include "dbg_gpu_gk20a.h"
36
37#include "hw_ram_gk20a.h"
38#include "hw_fifo_gk20a.h"
39#include "hw_pbdma_gk20a.h"
40#include "hw_ccsr_gk20a.h"
41#include "hw_ltc_gk20a.h"
42
43#define NVMAP_HANDLE_PARAM_SIZE 1
44
45static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f);
46static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c);
47
48static void free_priv_cmdbuf(struct channel_gk20a *c,
49 struct priv_cmd_entry *e);
50static void recycle_priv_cmdbuf(struct channel_gk20a *c);
51
52static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c);
53static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c);
54
55static int channel_gk20a_commit_userd(struct channel_gk20a *c);
56static int channel_gk20a_setup_userd(struct channel_gk20a *c);
57static int channel_gk20a_setup_ramfc(struct channel_gk20a *c,
58 u64 gpfifo_base, u32 gpfifo_entries);
59
60static void channel_gk20a_bind(struct channel_gk20a *ch_gk20a);
61static void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a);
62
63static int channel_gk20a_alloc_inst(struct gk20a *g,
64 struct channel_gk20a *ch);
65static void channel_gk20a_free_inst(struct gk20a *g,
66 struct channel_gk20a *ch);
67
68static int channel_gk20a_update_runlist(struct channel_gk20a *c,
69 bool add);
70static void gk20a_free_error_notifiers(struct channel_gk20a *ch);
71
72static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f)
73{
74 struct channel_gk20a *ch = NULL;
75 int chid;
76
77 mutex_lock(&f->ch_inuse_mutex);
78 for (chid = 0; chid < f->num_channels; chid++) {
79 if (!f->channel[chid].in_use) {
80 f->channel[chid].in_use = true;
81 ch = &f->channel[chid];
82 break;
83 }
84 }
85 mutex_unlock(&f->ch_inuse_mutex);
86
87 return ch;
88}
89
90static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c)
91{
92 mutex_lock(&f->ch_inuse_mutex);
93 f->channel[c->hw_chid].in_use = false;
94 mutex_unlock(&f->ch_inuse_mutex);
95}
96
97int channel_gk20a_commit_va(struct channel_gk20a *c)
98{
99 u64 addr;
100 u32 addr_lo;
101 u32 addr_hi;
102 void *inst_ptr;
103
104 gk20a_dbg_fn("");
105
106 inst_ptr = c->inst_block.cpuva;
107 if (!inst_ptr)
108 return -ENOMEM;
109
110 addr = gk20a_mm_iova_addr(c->vm->pdes.sgt->sgl);
111 addr_lo = u64_lo32(addr >> 12);
112 addr_hi = u64_hi32(addr);
113
114 gk20a_dbg_info("pde pa=0x%llx addr_lo=0x%x addr_hi=0x%x",
115 (u64)addr, addr_lo, addr_hi);
116
117 gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
118 ram_in_page_dir_base_target_vid_mem_f() |
119 ram_in_page_dir_base_vol_true_f() |
120 ram_in_page_dir_base_lo_f(addr_lo));
121
122 gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
123 ram_in_page_dir_base_hi_f(addr_hi));
124
125 gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
126 u64_lo32(c->vm->va_limit) | 0xFFF);
127
128 gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
129 ram_in_adr_limit_hi_f(u64_hi32(c->vm->va_limit)));
130
131 gk20a_mm_l2_invalidate(c->g);
132
133 return 0;
134}
135
136static int channel_gk20a_commit_userd(struct channel_gk20a *c)
137{
138 u32 addr_lo;
139 u32 addr_hi;
140 void *inst_ptr;
141
142 gk20a_dbg_fn("");
143
144 inst_ptr = c->inst_block.cpuva;
145 if (!inst_ptr)
146 return -ENOMEM;
147
148 addr_lo = u64_lo32(c->userd_iova >> ram_userd_base_shift_v());
149 addr_hi = u64_hi32(c->userd_iova);
150
151 gk20a_dbg_info("channel %d : set ramfc userd 0x%16llx",
152 c->hw_chid, (u64)c->userd_iova);
153
154 gk20a_mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_w(),
155 pbdma_userd_target_vid_mem_f() |
156 pbdma_userd_addr_f(addr_lo));
157
158 gk20a_mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_hi_w(),
159 pbdma_userd_target_vid_mem_f() |
160 pbdma_userd_hi_addr_f(addr_hi));
161
162 gk20a_mm_l2_invalidate(c->g);
163
164 return 0;
165}
166
167static int channel_gk20a_set_schedule_params(struct channel_gk20a *c,
168 u32 timeslice_timeout)
169{
170 void *inst_ptr;
171 int shift = 3;
172 int value = timeslice_timeout;
173
174 inst_ptr = c->inst_block.cpuva;
175 if (!inst_ptr)
176 return -ENOMEM;
177
178 /* disable channel */
179 gk20a_writel(c->g, ccsr_channel_r(c->hw_chid),
180 gk20a_readl(c->g, ccsr_channel_r(c->hw_chid)) |
181 ccsr_channel_enable_clr_true_f());
182
183 /* preempt the channel */
184 WARN_ON(gk20a_fifo_preempt_channel(c->g, c->hw_chid));
185
186 /* flush GPU cache */
187 gk20a_mm_l2_flush(c->g, true);
188
189 /* value field is 8 bits long */
190 while (value >= 1 << 8) {
191 value >>= 1;
192 shift++;
193 }
194
195 /* time slice register is only 18bits long */
196 if ((value << shift) >= 1<<19) {
197 pr_err("Requested timeslice value is clamped to 18 bits\n");
198 value = 255;
199 shift = 10;
200 }
201
202 /* set new timeslice */
203 gk20a_mem_wr32(inst_ptr, ram_fc_eng_timeslice_w(),
204 value | (shift << 12) |
205 fifo_eng_timeslice_enable_true_f());
206
207 /* enable channel */
208 gk20a_writel(c->g, ccsr_channel_r(c->hw_chid),
209 gk20a_readl(c->g, ccsr_channel_r(c->hw_chid)) |
210 ccsr_channel_enable_set_true_f());
211
212 gk20a_mm_l2_invalidate(c->g);
213
214 return 0;
215}
216
217static int channel_gk20a_setup_ramfc(struct channel_gk20a *c,
218 u64 gpfifo_base, u32 gpfifo_entries)
219{
220 void *inst_ptr;
221
222 gk20a_dbg_fn("");
223
224 inst_ptr = c->inst_block.cpuva;
225 if (!inst_ptr)
226 return -ENOMEM;
227
228 memset(inst_ptr, 0, ram_fc_size_val_v());
229
230 gk20a_mem_wr32(inst_ptr, ram_fc_gp_base_w(),
231 pbdma_gp_base_offset_f(
232 u64_lo32(gpfifo_base >> pbdma_gp_base_rsvd_s())));
233
234 gk20a_mem_wr32(inst_ptr, ram_fc_gp_base_hi_w(),
235 pbdma_gp_base_hi_offset_f(u64_hi32(gpfifo_base)) |
236 pbdma_gp_base_hi_limit2_f(ilog2(gpfifo_entries)));
237
238 gk20a_mem_wr32(inst_ptr, ram_fc_signature_w(),
239 pbdma_signature_hw_valid_f() | pbdma_signature_sw_zero_f());
240
241 gk20a_mem_wr32(inst_ptr, ram_fc_formats_w(),
242 pbdma_formats_gp_fermi0_f() |
243 pbdma_formats_pb_fermi1_f() |
244 pbdma_formats_mp_fermi0_f());
245
246 gk20a_mem_wr32(inst_ptr, ram_fc_pb_header_w(),
247 pbdma_pb_header_priv_user_f() |
248 pbdma_pb_header_method_zero_f() |
249 pbdma_pb_header_subchannel_zero_f() |
250 pbdma_pb_header_level_main_f() |
251 pbdma_pb_header_first_true_f() |
252 pbdma_pb_header_type_inc_f());
253
254 gk20a_mem_wr32(inst_ptr, ram_fc_subdevice_w(),
255 pbdma_subdevice_id_f(1) |
256 pbdma_subdevice_status_active_f() |
257 pbdma_subdevice_channel_dma_enable_f());
258
259 gk20a_mem_wr32(inst_ptr, ram_fc_target_w(), pbdma_target_engine_sw_f());
260
261 gk20a_mem_wr32(inst_ptr, ram_fc_acquire_w(),
262 pbdma_acquire_retry_man_2_f() |
263 pbdma_acquire_retry_exp_2_f() |
264 pbdma_acquire_timeout_exp_max_f() |
265 pbdma_acquire_timeout_man_max_f() |
266 pbdma_acquire_timeout_en_disable_f());
267
268 gk20a_mem_wr32(inst_ptr, ram_fc_eng_timeslice_w(),
269 fifo_eng_timeslice_timeout_128_f() |
270 fifo_eng_timeslice_timescale_3_f() |
271 fifo_eng_timeslice_enable_true_f());
272
273 gk20a_mem_wr32(inst_ptr, ram_fc_pb_timeslice_w(),
274 fifo_pb_timeslice_timeout_16_f() |
275 fifo_pb_timeslice_timescale_0_f() |
276 fifo_pb_timeslice_enable_true_f());
277
278 gk20a_mem_wr32(inst_ptr, ram_fc_chid_w(), ram_fc_chid_id_f(c->hw_chid));
279
280 /* TBD: alwasy priv mode? */
281 gk20a_mem_wr32(inst_ptr, ram_fc_hce_ctrl_w(),
282 pbdma_hce_ctrl_hce_priv_mode_yes_f());
283
284 gk20a_mm_l2_invalidate(c->g);
285
286 return 0;
287}
288
289static int channel_gk20a_setup_userd(struct channel_gk20a *c)
290{
291 BUG_ON(!c->userd_cpu_va);
292
293 gk20a_dbg_fn("");
294
295 gk20a_mem_wr32(c->userd_cpu_va, ram_userd_put_w(), 0);
296 gk20a_mem_wr32(c->userd_cpu_va, ram_userd_get_w(), 0);
297 gk20a_mem_wr32(c->userd_cpu_va, ram_userd_ref_w(), 0);
298 gk20a_mem_wr32(c->userd_cpu_va, ram_userd_put_hi_w(), 0);
299 gk20a_mem_wr32(c->userd_cpu_va, ram_userd_ref_threshold_w(), 0);
300 gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_top_level_get_w(), 0);
301 gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_top_level_get_hi_w(), 0);
302 gk20a_mem_wr32(c->userd_cpu_va, ram_userd_get_hi_w(), 0);
303 gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_get_w(), 0);
304 gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_put_w(), 0);
305
306 gk20a_mm_l2_invalidate(c->g);
307
308 return 0;
309}
310
311static void channel_gk20a_bind(struct channel_gk20a *ch_gk20a)
312{
313 struct gk20a *g = ch_gk20a->g;
314 struct fifo_gk20a *f = &g->fifo;
315 struct fifo_engine_info_gk20a *engine_info =
316 f->engine_info + ENGINE_GR_GK20A;
317
318 u32 inst_ptr = ch_gk20a->inst_block.cpu_pa
319 >> ram_in_base_shift_v();
320
321 gk20a_dbg_info("bind channel %d inst ptr 0x%08x",
322 ch_gk20a->hw_chid, inst_ptr);
323
324 ch_gk20a->bound = true;
325
326 gk20a_writel(g, ccsr_channel_r(ch_gk20a->hw_chid),
327 (gk20a_readl(g, ccsr_channel_r(ch_gk20a->hw_chid)) &
328 ~ccsr_channel_runlist_f(~0)) |
329 ccsr_channel_runlist_f(engine_info->runlist_id));
330
331 gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid),
332 ccsr_channel_inst_ptr_f(inst_ptr) |
333 ccsr_channel_inst_target_vid_mem_f() |
334 ccsr_channel_inst_bind_true_f());
335
336 gk20a_writel(g, ccsr_channel_r(ch_gk20a->hw_chid),
337 (gk20a_readl(g, ccsr_channel_r(ch_gk20a->hw_chid)) &
338 ~ccsr_channel_enable_set_f(~0)) |
339 ccsr_channel_enable_set_true_f());
340}
341
342static void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a)
343{
344 struct gk20a *g = ch_gk20a->g;
345
346 gk20a_dbg_fn("");
347
348 if (ch_gk20a->bound)
349 gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid),
350 ccsr_channel_inst_ptr_f(0) |
351 ccsr_channel_inst_bind_false_f());
352
353 ch_gk20a->bound = false;
354}
355
356static int channel_gk20a_alloc_inst(struct gk20a *g,
357 struct channel_gk20a *ch)
358{
359 struct device *d = dev_from_gk20a(g);
360 int err = 0;
361 dma_addr_t iova;
362
363 gk20a_dbg_fn("");
364
365 ch->inst_block.size = ram_in_alloc_size_v();
366 ch->inst_block.cpuva = dma_alloc_coherent(d,
367 ch->inst_block.size,
368 &iova,
369 GFP_KERNEL);
370 if (!ch->inst_block.cpuva) {
371 gk20a_err(d, "%s: memory allocation failed\n", __func__);
372 err = -ENOMEM;
373 goto clean_up;
374 }
375
376 ch->inst_block.iova = iova;
377 ch->inst_block.cpu_pa = gk20a_get_phys_from_iova(d,
378 ch->inst_block.iova);
379 if (!ch->inst_block.cpu_pa) {
380 gk20a_err(d, "%s: failed to get physical address\n", __func__);
381 err = -ENOMEM;
382 goto clean_up;
383 }
384
385 gk20a_dbg_info("channel %d inst block physical addr: 0x%16llx",
386 ch->hw_chid, (u64)ch->inst_block.cpu_pa);
387
388 gk20a_dbg_fn("done");
389 return 0;
390
391clean_up:
392 gk20a_err(d, "fail");
393 channel_gk20a_free_inst(g, ch);
394 return err;
395}
396
397static void channel_gk20a_free_inst(struct gk20a *g,
398 struct channel_gk20a *ch)
399{
400 struct device *d = dev_from_gk20a(g);
401
402 if (ch->inst_block.cpuva)
403 dma_free_coherent(d, ch->inst_block.size,
404 ch->inst_block.cpuva, ch->inst_block.iova);
405 ch->inst_block.cpuva = NULL;
406 ch->inst_block.iova = 0;
407 memset(&ch->inst_block, 0, sizeof(struct inst_desc));
408}
409
410static int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add)
411{
412 return gk20a_fifo_update_runlist(c->g, 0, c->hw_chid, add, true);
413}
414
415void gk20a_disable_channel_no_update(struct channel_gk20a *ch)
416{
417 /* ensure no fences are pending */
418 if (ch->sync)
419 ch->sync->set_min_eq_max(ch->sync);
420
421 /* disable channel */
422 gk20a_writel(ch->g, ccsr_channel_r(ch->hw_chid),
423 gk20a_readl(ch->g,
424 ccsr_channel_r(ch->hw_chid)) |
425 ccsr_channel_enable_clr_true_f());
426}
427
428static int gk20a_wait_channel_idle(struct channel_gk20a *ch)
429{
430 bool channel_idle = false;
431 unsigned long end_jiffies = jiffies +
432 msecs_to_jiffies(gk20a_get_gr_idle_timeout(ch->g));
433
434 do {
435 mutex_lock(&ch->jobs_lock);
436 channel_idle = list_empty(&ch->jobs);
437 mutex_unlock(&ch->jobs_lock);
438 if (channel_idle)
439 break;
440
441 usleep_range(1000, 3000);
442 } while (time_before(jiffies, end_jiffies)
443 || !tegra_platform_is_silicon());
444
445 if (!channel_idle)
446 gk20a_err(dev_from_gk20a(ch->g), "channel jobs not freed");
447
448 return 0;
449}
450
451void gk20a_disable_channel(struct channel_gk20a *ch,
452 bool finish,
453 unsigned long finish_timeout)
454{
455 if (finish) {
456 int err = gk20a_channel_finish(ch, finish_timeout);
457 WARN_ON(err);
458 }
459
460 /* disable the channel from hw and increment syncpoints */
461 gk20a_disable_channel_no_update(ch);
462
463 gk20a_wait_channel_idle(ch);
464
465 /* preempt the channel */
466 gk20a_fifo_preempt_channel(ch->g, ch->hw_chid);
467
468 /* remove channel from runlist */
469 channel_gk20a_update_runlist(ch, false);
470}
471
472#if defined(CONFIG_GK20A_CYCLE_STATS)
473
474static void gk20a_free_cycle_stats_buffer(struct channel_gk20a *ch)
475{
476 /* disable existing cyclestats buffer */
477 mutex_lock(&ch->cyclestate.cyclestate_buffer_mutex);
478 if (ch->cyclestate.cyclestate_buffer_handler) {
479 dma_buf_vunmap(ch->cyclestate.cyclestate_buffer_handler,
480 ch->cyclestate.cyclestate_buffer);
481 dma_buf_put(ch->cyclestate.cyclestate_buffer_handler);
482 ch->cyclestate.cyclestate_buffer_handler = NULL;
483 ch->cyclestate.cyclestate_buffer = NULL;
484 ch->cyclestate.cyclestate_buffer_size = 0;
485 }
486 mutex_unlock(&ch->cyclestate.cyclestate_buffer_mutex);
487}
488
489static int gk20a_channel_cycle_stats(struct channel_gk20a *ch,
490 struct nvhost_cycle_stats_args *args)
491{
492 struct dma_buf *dmabuf;
493 void *virtual_address;
494
495 if (args->nvmap_handle && !ch->cyclestate.cyclestate_buffer_handler) {
496
497 /* set up new cyclestats buffer */
498 dmabuf = dma_buf_get(args->nvmap_handle);
499 if (IS_ERR(dmabuf))
500 return PTR_ERR(dmabuf);
501 virtual_address = dma_buf_vmap(dmabuf);
502 if (!virtual_address)
503 return -ENOMEM;
504
505 ch->cyclestate.cyclestate_buffer_handler = dmabuf;
506 ch->cyclestate.cyclestate_buffer = virtual_address;
507 ch->cyclestate.cyclestate_buffer_size = dmabuf->size;
508 return 0;
509
510 } else if (!args->nvmap_handle &&
511 ch->cyclestate.cyclestate_buffer_handler) {
512 gk20a_free_cycle_stats_buffer(ch);
513 return 0;
514
515 } else if (!args->nvmap_handle &&
516 !ch->cyclestate.cyclestate_buffer_handler) {
517 /* no requst from GL */
518 return 0;
519
520 } else {
521 pr_err("channel already has cyclestats buffer\n");
522 return -EINVAL;
523 }
524}
525#endif
526
527static int gk20a_init_error_notifier(struct channel_gk20a *ch,
528 struct nvhost_set_error_notifier *args) {
529 void *va;
530
531 struct dma_buf *dmabuf;
532
533 if (!args->mem) {
534 pr_err("gk20a_init_error_notifier: invalid memory handle\n");
535 return -EINVAL;
536 }
537
538 dmabuf = dma_buf_get(args->mem);
539
540 if (ch->error_notifier_ref)
541 gk20a_free_error_notifiers(ch);
542
543 if (IS_ERR(dmabuf)) {
544 pr_err("Invalid handle: %d\n", args->mem);
545 return -EINVAL;
546 }
547 /* map handle */
548 va = dma_buf_vmap(dmabuf);
549 if (!va) {
550 dma_buf_put(dmabuf);
551 pr_err("Cannot map notifier handle\n");
552 return -ENOMEM;
553 }
554
555 /* set channel notifiers pointer */
556 ch->error_notifier_ref = dmabuf;
557 ch->error_notifier = va + args->offset;
558 ch->error_notifier_va = va;
559 memset(ch->error_notifier, 0, sizeof(struct nvhost_notification));
560 return 0;
561}
562
563void gk20a_set_error_notifier(struct channel_gk20a *ch, __u32 error)
564{
565 if (ch->error_notifier_ref) {
566 struct timespec time_data;
567 u64 nsec;
568 getnstimeofday(&time_data);
569 nsec = ((u64)time_data.tv_sec) * 1000000000u +
570 (u64)time_data.tv_nsec;
571 ch->error_notifier->time_stamp.nanoseconds[0] =
572 (u32)nsec;
573 ch->error_notifier->time_stamp.nanoseconds[1] =
574 (u32)(nsec >> 32);
575 ch->error_notifier->info32 = error;
576 ch->error_notifier->status = 0xffff;
577 gk20a_err(dev_from_gk20a(ch->g),
578 "error notifier set to %d\n", error);
579 }
580}
581
582static void gk20a_free_error_notifiers(struct channel_gk20a *ch)
583{
584 if (ch->error_notifier_ref) {
585 dma_buf_vunmap(ch->error_notifier_ref, ch->error_notifier_va);
586 dma_buf_put(ch->error_notifier_ref);
587 ch->error_notifier_ref = 0;
588 ch->error_notifier = 0;
589 ch->error_notifier_va = 0;
590 }
591}
592
593void gk20a_free_channel(struct channel_gk20a *ch, bool finish)
594{
595 struct gk20a *g = ch->g;
596 struct device *d = dev_from_gk20a(g);
597 struct fifo_gk20a *f = &g->fifo;
598 struct gr_gk20a *gr = &g->gr;
599 struct vm_gk20a *ch_vm = ch->vm;
600 unsigned long timeout = gk20a_get_gr_idle_timeout(g);
601 struct dbg_session_gk20a *dbg_s;
602
603 gk20a_dbg_fn("");
604
605 /* if engine reset was deferred, perform it now */
606 mutex_lock(&f->deferred_reset_mutex);
607 if (g->fifo.deferred_reset_pending) {
608 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, "engine reset was"
609 " deferred, running now");
610 fifo_gk20a_finish_mmu_fault_handling(g, g->fifo.mmu_fault_engines);
611 g->fifo.mmu_fault_engines = 0;
612 g->fifo.deferred_reset_pending = false;
613 }
614 mutex_unlock(&f->deferred_reset_mutex);
615
616 if (!ch->bound)
617 return;
618
619 if (!gk20a_channel_as_bound(ch))
620 goto unbind;
621
622 gk20a_dbg_info("freeing bound channel context, timeout=%ld",
623 timeout);
624
625 gk20a_disable_channel(ch, finish && !ch->has_timedout, timeout);
626
627 gk20a_free_error_notifiers(ch);
628
629 /* release channel ctx */
630 gk20a_free_channel_ctx(ch);
631
632 gk20a_gr_flush_channel_tlb(gr);
633
634 memset(&ch->ramfc, 0, sizeof(struct mem_desc_sub));
635
636 /* free gpfifo */
637 if (ch->gpfifo.gpu_va)
638 gk20a_gmmu_unmap(ch_vm, ch->gpfifo.gpu_va,
639 ch->gpfifo.size, gk20a_mem_flag_none);
640 if (ch->gpfifo.cpu_va)
641 dma_free_coherent(d, ch->gpfifo.size,
642 ch->gpfifo.cpu_va, ch->gpfifo.iova);
643 ch->gpfifo.cpu_va = NULL;
644 ch->gpfifo.iova = 0;
645
646 gk20a_mm_l2_invalidate(ch->g);
647
648 memset(&ch->gpfifo, 0, sizeof(struct gpfifo_desc));
649
650#if defined(CONFIG_GK20A_CYCLE_STATS)
651 gk20a_free_cycle_stats_buffer(ch);
652#endif
653
654 channel_gk20a_free_priv_cmdbuf(ch);
655
656 if (ch->sync) {
657 ch->sync->destroy(ch->sync);
658 ch->sync = NULL;
659 }
660
661 /* release channel binding to the as_share */
662 gk20a_as_release_share(ch_vm->as_share);
663
664unbind:
665 channel_gk20a_unbind(ch);
666 channel_gk20a_free_inst(g, ch);
667
668 ch->vpr = false;
669 ch->vm = NULL;
670 WARN_ON(ch->sync);
671
672 /* unlink all debug sessions */
673 mutex_lock(&ch->dbg_s_lock);
674
675 list_for_each_entry(dbg_s, &ch->dbg_s_list, dbg_s_list_node) {
676 dbg_s->ch = NULL;
677 list_del_init(&dbg_s->dbg_s_list_node);
678 }
679
680 mutex_unlock(&ch->dbg_s_lock);
681
682 /* ALWAYS last */
683 release_used_channel(f, ch);
684}
685
686int gk20a_channel_release(struct inode *inode, struct file *filp)
687{
688 struct channel_gk20a *ch = (struct channel_gk20a *)filp->private_data;
689 struct gk20a *g = ch->g;
690
691 trace_gk20a_channel_release(dev_name(&g->dev->dev));
692
693 gk20a_channel_busy(ch->g->dev);
694 gk20a_free_channel(ch, true);
695 gk20a_channel_idle(ch->g->dev);
696
697 gk20a_put_client(g);
698 filp->private_data = NULL;
699 return 0;
700}
701
702static struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g)
703{
704 struct fifo_gk20a *f = &g->fifo;
705 struct channel_gk20a *ch;
706
707 ch = acquire_unused_channel(f);
708 if (ch == NULL) {
709 /* TBD: we want to make this virtualizable */
710 gk20a_err(dev_from_gk20a(g), "out of hw chids");
711 return 0;
712 }
713
714 ch->g = g;
715
716 if (channel_gk20a_alloc_inst(g, ch)) {
717 ch->in_use = false;
718 gk20a_err(dev_from_gk20a(g),
719 "failed to open gk20a channel, out of inst mem");
720
721 return 0;
722 }
723 g->ops.fifo.bind_channel(ch);
724 ch->pid = current->pid;
725
726 /* reset timeout counter and update timestamp */
727 ch->timeout_accumulated_ms = 0;
728 ch->timeout_gpfifo_get = 0;
729 /* set gr host default timeout */
730 ch->timeout_ms_max = gk20a_get_gr_idle_timeout(g);
731 ch->timeout_debug_dump = true;
732 ch->has_timedout = false;
733
734 /* The channel is *not* runnable at this point. It still needs to have
735 * an address space bound and allocate a gpfifo and grctx. */
736
737 init_waitqueue_head(&ch->notifier_wq);
738 init_waitqueue_head(&ch->semaphore_wq);
739 init_waitqueue_head(&ch->submit_wq);
740
741 return ch;
742}
743
744static int __gk20a_channel_open(struct gk20a *g, struct file *filp)
745{
746 int err;
747 struct channel_gk20a *ch;
748
749 trace_gk20a_channel_open(dev_name(&g->dev->dev));
750
751 err = gk20a_get_client(g);
752 if (err) {
753 gk20a_err(dev_from_gk20a(g),
754 "failed to get client ref");
755 return err;
756 }
757
758 err = gk20a_channel_busy(g->dev);
759 if (err) {
760 gk20a_put_client(g);
761 gk20a_err(dev_from_gk20a(g), "failed to power on, %d", err);
762 return err;
763 }
764 ch = gk20a_open_new_channel(g);
765 gk20a_channel_idle(g->dev);
766 if (!ch) {
767 gk20a_put_client(g);
768 gk20a_err(dev_from_gk20a(g),
769 "failed to get f");
770 return -ENOMEM;
771 }
772
773 filp->private_data = ch;
774 return 0;
775}
776
777int gk20a_channel_open(struct inode *inode, struct file *filp)
778{
779 struct gk20a *g = container_of(inode->i_cdev,
780 struct gk20a, channel.cdev);
781 return __gk20a_channel_open(g, filp);
782}
783
784/* allocate private cmd buffer.
785 used for inserting commands before/after user submitted buffers. */
786static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c)
787{
788 struct device *d = dev_from_gk20a(c->g);
789 struct vm_gk20a *ch_vm = c->vm;
790 struct priv_cmd_queue *q = &c->priv_cmd_q;
791 struct priv_cmd_entry *e;
792 u32 i = 0, size;
793 int err = 0;
794 struct sg_table *sgt;
795 dma_addr_t iova;
796
797 /* Kernel can insert gpfifos before and after user gpfifos.
798 Before user gpfifos, kernel inserts fence_wait, which takes
799 syncpoint_a (2 dwords) + syncpoint_b (2 dwords) = 4 dwords.
800 After user gpfifos, kernel inserts fence_get, which takes
801 wfi (2 dwords) + syncpoint_a (2 dwords) + syncpoint_b (2 dwords)
802 = 6 dwords.
803 Worse case if kernel adds both of them for every user gpfifo,
804 max size of priv_cmdbuf is :
805 (gpfifo entry number * (2 / 3) * (4 + 6) * 4 bytes */
806 size = roundup_pow_of_two(
807 c->gpfifo.entry_num * 2 * 10 * sizeof(u32) / 3);
808
809 q->mem.base_cpuva = dma_alloc_coherent(d, size,
810 &iova,
811 GFP_KERNEL);
812 if (!q->mem.base_cpuva) {
813 gk20a_err(d, "%s: memory allocation failed\n", __func__);
814 err = -ENOMEM;
815 goto clean_up;
816 }
817
818 q->mem.base_iova = iova;
819 q->mem.size = size;
820
821 err = gk20a_get_sgtable(d, &sgt,
822 q->mem.base_cpuva, q->mem.base_iova, size);
823 if (err) {
824 gk20a_err(d, "%s: failed to create sg table\n", __func__);
825 goto clean_up;
826 }
827
828 memset(q->mem.base_cpuva, 0, size);
829
830 q->base_gpuva = gk20a_gmmu_map(ch_vm, &sgt,
831 size,
832 0, /* flags */
833 gk20a_mem_flag_none);
834 if (!q->base_gpuva) {
835 gk20a_err(d, "ch %d : failed to map gpu va"
836 "for priv cmd buffer", c->hw_chid);
837 err = -ENOMEM;
838 goto clean_up_sgt;
839 }
840
841 q->size = q->mem.size / sizeof (u32);
842
843 INIT_LIST_HEAD(&q->head);
844 INIT_LIST_HEAD(&q->free);
845
846 /* pre-alloc 25% of priv cmdbuf entries and put them on free list */
847 for (i = 0; i < q->size / 4; i++) {
848 e = kzalloc(sizeof(struct priv_cmd_entry), GFP_KERNEL);
849 if (!e) {
850 gk20a_err(d, "ch %d: fail to pre-alloc cmd entry",
851 c->hw_chid);
852 err = -ENOMEM;
853 goto clean_up_sgt;
854 }
855 e->pre_alloc = true;
856 list_add(&e->list, &q->free);
857 }
858
859 gk20a_free_sgtable(&sgt);
860
861 return 0;
862
863clean_up_sgt:
864 gk20a_free_sgtable(&sgt);
865clean_up:
866 channel_gk20a_free_priv_cmdbuf(c);
867 return err;
868}
869
870static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c)
871{
872 struct device *d = dev_from_gk20a(c->g);
873 struct vm_gk20a *ch_vm = c->vm;
874 struct priv_cmd_queue *q = &c->priv_cmd_q;
875 struct priv_cmd_entry *e;
876 struct list_head *pos, *tmp, *head;
877
878 if (q->size == 0)
879 return;
880
881 if (q->base_gpuva)
882 gk20a_gmmu_unmap(ch_vm, q->base_gpuva,
883 q->mem.size, gk20a_mem_flag_none);
884 if (q->mem.base_cpuva)
885 dma_free_coherent(d, q->mem.size,
886 q->mem.base_cpuva, q->mem.base_iova);
887 q->mem.base_cpuva = NULL;
888 q->mem.base_iova = 0;
889
890 /* free used list */
891 head = &q->head;
892 list_for_each_safe(pos, tmp, head) {
893 e = container_of(pos, struct priv_cmd_entry, list);
894 free_priv_cmdbuf(c, e);
895 }
896
897 /* free free list */
898 head = &q->free;
899 list_for_each_safe(pos, tmp, head) {
900 e = container_of(pos, struct priv_cmd_entry, list);
901 e->pre_alloc = false;
902 free_priv_cmdbuf(c, e);
903 }
904
905 memset(q, 0, sizeof(struct priv_cmd_queue));
906}
907
908/* allocate a cmd buffer with given size. size is number of u32 entries */
909int gk20a_channel_alloc_priv_cmdbuf(struct channel_gk20a *c, u32 orig_size,
910 struct priv_cmd_entry **entry)
911{
912 struct priv_cmd_queue *q = &c->priv_cmd_q;
913 struct priv_cmd_entry *e;
914 struct list_head *node;
915 u32 free_count;
916 u32 size = orig_size;
917 bool no_retry = false;
918
919 gk20a_dbg_fn("size %d", orig_size);
920
921 *entry = NULL;
922
923 /* if free space in the end is less than requested, increase the size
924 * to make the real allocated space start from beginning. */
925 if (q->put + size > q->size)
926 size = orig_size + (q->size - q->put);
927
928 gk20a_dbg_info("ch %d: priv cmd queue get:put %d:%d",
929 c->hw_chid, q->get, q->put);
930
931TRY_AGAIN:
932 free_count = (q->size - (q->put - q->get) - 1) % q->size;
933
934 if (size > free_count) {
935 if (!no_retry) {
936 recycle_priv_cmdbuf(c);
937 no_retry = true;
938 goto TRY_AGAIN;
939 } else
940 return -EAGAIN;
941 }
942
943 if (unlikely(list_empty(&q->free))) {
944
945 gk20a_dbg_info("ch %d: run out of pre-alloc entries",
946 c->hw_chid);
947
948 e = kzalloc(sizeof(struct priv_cmd_entry), GFP_KERNEL);
949 if (!e) {
950 gk20a_err(dev_from_gk20a(c->g),
951 "ch %d: fail to allocate priv cmd entry",
952 c->hw_chid);
953 return -ENOMEM;
954 }
955 } else {
956 node = q->free.next;
957 list_del(node);
958 e = container_of(node, struct priv_cmd_entry, list);
959 }
960
961 e->size = orig_size;
962 e->gp_get = c->gpfifo.get;
963 e->gp_put = c->gpfifo.put;
964 e->gp_wrap = c->gpfifo.wrap;
965
966 /* if we have increased size to skip free space in the end, set put
967 to beginning of cmd buffer (0) + size */
968 if (size != orig_size) {
969 e->ptr = q->mem.base_cpuva;
970 e->gva = q->base_gpuva;
971 q->put = orig_size;
972 } else {
973 e->ptr = q->mem.base_cpuva + q->put;
974 e->gva = q->base_gpuva + q->put * sizeof(u32);
975 q->put = (q->put + orig_size) & (q->size - 1);
976 }
977
978 /* we already handled q->put + size > q->size so BUG_ON this */
979 BUG_ON(q->put > q->size);
980
981 /* add new entry to head since we free from head */
982 list_add(&e->list, &q->head);
983
984 *entry = e;
985
986 gk20a_dbg_fn("done");
987
988 return 0;
989}
990
991/* Don't call this to free an explict cmd entry.
992 * It doesn't update priv_cmd_queue get/put */
993static void free_priv_cmdbuf(struct channel_gk20a *c,
994 struct priv_cmd_entry *e)
995{
996 struct priv_cmd_queue *q = &c->priv_cmd_q;
997
998 if (!e)
999 return;
1000
1001 list_del(&e->list);
1002
1003 if (unlikely(!e->pre_alloc))
1004 kfree(e);
1005 else {
1006 memset(e, 0, sizeof(struct priv_cmd_entry));
1007 e->pre_alloc = true;
1008 list_add(&e->list, &q->free);
1009 }
1010}
1011
1012/* free entries if they're no longer being used */
1013static void recycle_priv_cmdbuf(struct channel_gk20a *c)
1014{
1015 struct priv_cmd_queue *q = &c->priv_cmd_q;
1016 struct priv_cmd_entry *e, *tmp;
1017 struct list_head *head = &q->head;
1018 bool wrap_around, found = false;
1019
1020 gk20a_dbg_fn("");
1021
1022 /* Find the most recent free entry. Free it and everything before it */
1023 list_for_each_entry(e, head, list) {
1024
1025 gk20a_dbg_info("ch %d: cmd entry get:put:wrap %d:%d:%d "
1026 "curr get:put:wrap %d:%d:%d",
1027 c->hw_chid, e->gp_get, e->gp_put, e->gp_wrap,
1028 c->gpfifo.get, c->gpfifo.put, c->gpfifo.wrap);
1029
1030 wrap_around = (c->gpfifo.wrap != e->gp_wrap);
1031 if (e->gp_get < e->gp_put) {
1032 if (c->gpfifo.get >= e->gp_put ||
1033 wrap_around) {
1034 found = true;
1035 break;
1036 } else
1037 e->gp_get = c->gpfifo.get;
1038 } else if (e->gp_get > e->gp_put) {
1039 if (wrap_around &&
1040 c->gpfifo.get >= e->gp_put) {
1041 found = true;
1042 break;
1043 } else
1044 e->gp_get = c->gpfifo.get;
1045 }
1046 }
1047
1048 if (found)
1049 q->get = (e->ptr - q->mem.base_cpuva) + e->size;
1050 else {
1051 gk20a_dbg_info("no free entry recycled");
1052 return;
1053 }
1054
1055 list_for_each_entry_safe_continue(e, tmp, head, list) {
1056 free_priv_cmdbuf(c, e);
1057 }
1058
1059 gk20a_dbg_fn("done");
1060}
1061
1062
1063static int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c,
1064 struct nvhost_alloc_gpfifo_args *args)
1065{
1066 struct gk20a *g = c->g;
1067 struct device *d = dev_from_gk20a(g);
1068 struct vm_gk20a *ch_vm;
1069 u32 gpfifo_size;
1070 int err = 0;
1071 struct sg_table *sgt;
1072 dma_addr_t iova;
1073
1074 /* Kernel can insert one extra gpfifo entry before user submitted gpfifos
1075 and another one after, for internal usage. Triple the requested size. */
1076 gpfifo_size = roundup_pow_of_two(args->num_entries * 3);
1077
1078 if (args->flags & NVHOST_ALLOC_GPFIFO_FLAGS_VPR_ENABLED)
1079 c->vpr = true;
1080
1081 /* an address space needs to have been bound at this point. */
1082 if (!gk20a_channel_as_bound(c)) {
1083 gk20a_err(d,
1084 "not bound to an address space at time of gpfifo"
1085 " allocation. Attempting to create and bind to"
1086 " one...");
1087 return -EINVAL;
1088 }
1089 ch_vm = c->vm;
1090
1091 c->cmds_pending = false;
1092 c->last_submit_fence.valid = false;
1093
1094 c->ramfc.offset = 0;
1095 c->ramfc.size = ram_in_ramfc_s() / 8;
1096
1097 if (c->gpfifo.cpu_va) {
1098 gk20a_err(d, "channel %d :"
1099 "gpfifo already allocated", c->hw_chid);
1100 return -EEXIST;
1101 }
1102
1103 c->gpfifo.size = gpfifo_size * sizeof(struct gpfifo);
1104 c->gpfifo.cpu_va = (struct gpfifo *)dma_alloc_coherent(d,
1105 c->gpfifo.size,
1106 &iova,
1107 GFP_KERNEL);
1108 if (!c->gpfifo.cpu_va) {
1109 gk20a_err(d, "%s: memory allocation failed\n", __func__);
1110 err = -ENOMEM;
1111 goto clean_up;
1112 }
1113
1114 c->gpfifo.iova = iova;
1115 c->gpfifo.entry_num = gpfifo_size;
1116
1117 c->gpfifo.get = c->gpfifo.put = 0;
1118
1119 err = gk20a_get_sgtable(d, &sgt,
1120 c->gpfifo.cpu_va, c->gpfifo.iova, c->gpfifo.size);
1121 if (err) {
1122 gk20a_err(d, "%s: failed to allocate sg table\n", __func__);
1123 goto clean_up;
1124 }
1125
1126 c->gpfifo.gpu_va = gk20a_gmmu_map(ch_vm,
1127 &sgt,
1128 c->gpfifo.size,
1129 0, /* flags */
1130 gk20a_mem_flag_none);
1131 if (!c->gpfifo.gpu_va) {
1132 gk20a_err(d, "channel %d : failed to map"
1133 " gpu_va for gpfifo", c->hw_chid);
1134 err = -ENOMEM;
1135 goto clean_up_sgt;
1136 }
1137
1138 gk20a_dbg_info("channel %d : gpfifo_base 0x%016llx, size %d",
1139 c->hw_chid, c->gpfifo.gpu_va, c->gpfifo.entry_num);
1140
1141 channel_gk20a_setup_ramfc(c, c->gpfifo.gpu_va, c->gpfifo.entry_num);
1142
1143 channel_gk20a_setup_userd(c);
1144 channel_gk20a_commit_userd(c);
1145
1146 gk20a_mm_l2_invalidate(c->g);
1147
1148 /* TBD: setup engine contexts */
1149
1150 err = channel_gk20a_alloc_priv_cmdbuf(c);
1151 if (err)
1152 goto clean_up_unmap;
1153
1154 err = channel_gk20a_update_runlist(c, true);
1155 if (err)
1156 goto clean_up_unmap;
1157
1158 gk20a_free_sgtable(&sgt);
1159
1160 gk20a_dbg_fn("done");
1161 return 0;
1162
1163clean_up_unmap:
1164 gk20a_gmmu_unmap(ch_vm, c->gpfifo.gpu_va,
1165 c->gpfifo.size, gk20a_mem_flag_none);
1166clean_up_sgt:
1167 gk20a_free_sgtable(&sgt);
1168clean_up:
1169 dma_free_coherent(d, c->gpfifo.size,
1170 c->gpfifo.cpu_va, c->gpfifo.iova);
1171 c->gpfifo.cpu_va = NULL;
1172 c->gpfifo.iova = 0;
1173 memset(&c->gpfifo, 0, sizeof(struct gpfifo_desc));
1174 gk20a_err(d, "fail");
1175 return err;
1176}
1177
1178static inline int wfi_cmd_size(void)
1179{
1180 return 2;
1181}
1182void add_wfi_cmd(struct priv_cmd_entry *cmd, int *i)
1183{
1184 /* wfi */
1185 cmd->ptr[(*i)++] = 0x2001001E;
1186 /* handle, ignored */
1187 cmd->ptr[(*i)++] = 0x00000000;
1188}
1189
1190static inline bool check_gp_put(struct gk20a *g,
1191 struct channel_gk20a *c)
1192{
1193 u32 put;
1194 /* gp_put changed unexpectedly since last update? */
1195 put = gk20a_bar1_readl(g,
1196 c->userd_gpu_va + 4 * ram_userd_gp_put_w());
1197 if (c->gpfifo.put != put) {
1198 /*TBD: BUG_ON/teardown on this*/
1199 gk20a_err(dev_from_gk20a(g), "gp_put changed unexpectedly "
1200 "since last update");
1201 c->gpfifo.put = put;
1202 return false; /* surprise! */
1203 }
1204 return true; /* checked out ok */
1205}
1206
1207/* Update with this periodically to determine how the gpfifo is draining. */
1208static inline u32 update_gp_get(struct gk20a *g,
1209 struct channel_gk20a *c)
1210{
1211 u32 new_get = gk20a_bar1_readl(g,
1212 c->userd_gpu_va + sizeof(u32) * ram_userd_gp_get_w());
1213 if (new_get < c->gpfifo.get)
1214 c->gpfifo.wrap = !c->gpfifo.wrap;
1215 c->gpfifo.get = new_get;
1216 return new_get;
1217}
1218
1219static inline u32 gp_free_count(struct channel_gk20a *c)
1220{
1221 return (c->gpfifo.entry_num - (c->gpfifo.put - c->gpfifo.get) - 1) %
1222 c->gpfifo.entry_num;
1223}
1224
1225bool gk20a_channel_update_and_check_timeout(struct channel_gk20a *ch,
1226 u32 timeout_delta_ms)
1227{
1228 u32 gpfifo_get = update_gp_get(ch->g, ch);
1229 /* Count consequent timeout isr */
1230 if (gpfifo_get == ch->timeout_gpfifo_get) {
1231 /* we didn't advance since previous channel timeout check */
1232 ch->timeout_accumulated_ms += timeout_delta_ms;
1233 } else {
1234 /* first timeout isr encountered */
1235 ch->timeout_accumulated_ms = timeout_delta_ms;
1236 }
1237
1238 ch->timeout_gpfifo_get = gpfifo_get;
1239
1240 return ch->g->timeouts_enabled &&
1241 ch->timeout_accumulated_ms > ch->timeout_ms_max;
1242}
1243
1244
1245/* Issue a syncpoint increment *preceded* by a wait-for-idle
1246 * command. All commands on the channel will have been
1247 * consumed at the time the fence syncpoint increment occurs.
1248 */
1249static int gk20a_channel_submit_wfi(struct channel_gk20a *c)
1250{
1251 struct priv_cmd_entry *cmd = NULL;
1252 struct gk20a *g = c->g;
1253 u32 free_count;
1254 int err;
1255
1256 if (c->has_timedout)
1257 return -ETIMEDOUT;
1258
1259 if (!c->sync) {
1260 c->sync = gk20a_channel_sync_create(c);
1261 if (!c->sync)
1262 return -ENOMEM;
1263 }
1264
1265 update_gp_get(g, c);
1266 free_count = gp_free_count(c);
1267 if (unlikely(!free_count)) {
1268 gk20a_err(dev_from_gk20a(g),
1269 "not enough gpfifo space");
1270 return -EAGAIN;
1271 }
1272
1273 err = c->sync->incr_wfi(c->sync, &cmd, &c->last_submit_fence);
1274 if (unlikely(err))
1275 return err;
1276
1277 WARN_ON(!c->last_submit_fence.wfi);
1278
1279 c->gpfifo.cpu_va[c->gpfifo.put].entry0 = u64_lo32(cmd->gva);
1280 c->gpfifo.cpu_va[c->gpfifo.put].entry1 = u64_hi32(cmd->gva) |
1281 pbdma_gp_entry1_length_f(cmd->size);
1282
1283 c->gpfifo.put = (c->gpfifo.put + 1) & (c->gpfifo.entry_num - 1);
1284
1285 /* save gp_put */
1286 cmd->gp_put = c->gpfifo.put;
1287
1288 gk20a_bar1_writel(g,
1289 c->userd_gpu_va + 4 * ram_userd_gp_put_w(),
1290 c->gpfifo.put);
1291
1292 gk20a_dbg_info("post-submit put %d, get %d, size %d",
1293 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1294
1295 return 0;
1296}
1297
1298static u32 get_gp_free_count(struct channel_gk20a *c)
1299{
1300 update_gp_get(c->g, c);
1301 return gp_free_count(c);
1302}
1303
1304static void trace_write_pushbuffer(struct channel_gk20a *c, struct gpfifo *g)
1305{
1306 void *mem = NULL;
1307 unsigned int words;
1308 u64 offset;
1309 struct dma_buf *dmabuf = NULL;
1310
1311 if (gk20a_debug_trace_cmdbuf) {
1312 u64 gpu_va = (u64)g->entry0 |
1313 (u64)((u64)pbdma_gp_entry1_get_hi_v(g->entry1) << 32);
1314 int err;
1315
1316 words = pbdma_gp_entry1_length_v(g->entry1);
1317 err = gk20a_vm_find_buffer(c->vm, gpu_va, &dmabuf, &offset);
1318 if (!err)
1319 mem = dma_buf_vmap(dmabuf);
1320 }
1321
1322 if (mem) {
1323 u32 i;
1324 /*
1325 * Write in batches of 128 as there seems to be a limit
1326 * of how much you can output to ftrace at once.
1327 */
1328 for (i = 0; i < words; i += 128U) {
1329 trace_gk20a_push_cmdbuf(
1330 c->g->dev->name,
1331 0,
1332 min(words - i, 128U),
1333 offset + i * sizeof(u32),
1334 mem);
1335 }
1336 dma_buf_vunmap(dmabuf, mem);
1337 }
1338}
1339
1340static int gk20a_channel_add_job(struct channel_gk20a *c,
1341 struct gk20a_channel_fence *fence)
1342{
1343 struct vm_gk20a *vm = c->vm;
1344 struct channel_gk20a_job *job = NULL;
1345 struct mapped_buffer_node **mapped_buffers = NULL;
1346 int err = 0, num_mapped_buffers;
1347
1348 /* job needs reference to this vm */
1349 gk20a_vm_get(vm);
1350
1351 err = gk20a_vm_get_buffers(vm, &mapped_buffers, &num_mapped_buffers);
1352 if (err) {
1353 gk20a_vm_put(vm);
1354 return err;
1355 }
1356
1357 job = kzalloc(sizeof(*job), GFP_KERNEL);
1358 if (!job) {
1359 gk20a_vm_put_buffers(vm, mapped_buffers, num_mapped_buffers);
1360 gk20a_vm_put(vm);
1361 return -ENOMEM;
1362 }
1363
1364 job->num_mapped_buffers = num_mapped_buffers;
1365 job->mapped_buffers = mapped_buffers;
1366 job->fence = *fence;
1367
1368 mutex_lock(&c->jobs_lock);
1369 list_add_tail(&job->list, &c->jobs);
1370 mutex_unlock(&c->jobs_lock);
1371
1372 return 0;
1373}
1374
1375void gk20a_channel_update(struct channel_gk20a *c, int nr_completed)
1376{
1377 struct gk20a *g = c->g;
1378 struct vm_gk20a *vm = c->vm;
1379 struct channel_gk20a_job *job, *n;
1380 int i;
1381
1382 wake_up(&c->submit_wq);
1383
1384 mutex_lock(&c->jobs_lock);
1385 list_for_each_entry_safe(job, n, &c->jobs, list) {
1386 bool completed = WARN_ON(!c->sync) ||
1387 c->sync->is_expired(c->sync, &job->fence);
1388 if (!completed)
1389 break;
1390
1391 gk20a_vm_put_buffers(vm, job->mapped_buffers,
1392 job->num_mapped_buffers);
1393
1394 /* job is done. release its reference to vm */
1395 gk20a_vm_put(vm);
1396
1397 list_del_init(&job->list);
1398 kfree(job);
1399 gk20a_channel_idle(g->dev);
1400 }
1401 mutex_unlock(&c->jobs_lock);
1402
1403 for (i = 0; i < nr_completed; i++)
1404 gk20a_channel_idle(c->g->dev);
1405}
1406
1407static int gk20a_submit_channel_gpfifo(struct channel_gk20a *c,
1408 struct nvhost_gpfifo *gpfifo,
1409 u32 num_entries,
1410 struct nvhost_fence *fence,
1411 u32 flags)
1412{
1413 struct gk20a *g = c->g;
1414 struct device *d = dev_from_gk20a(g);
1415 u32 err = 0;
1416 int i;
1417 struct priv_cmd_entry *wait_cmd = NULL;
1418 struct priv_cmd_entry *incr_cmd = NULL;
1419 /* we might need two extra gpfifo entries - one for pre fence
1420 * and one for post fence. */
1421 const int extra_entries = 2;
1422
1423 if (c->has_timedout)
1424 return -ETIMEDOUT;
1425
1426 if ((flags & (NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT |
1427 NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET)) &&
1428 !fence)
1429 return -EINVAL;
1430
1431 if (!c->sync) {
1432 c->sync = gk20a_channel_sync_create(c);
1433 if (!c->sync)
1434 return -ENOMEM;
1435 }
1436
1437#ifdef CONFIG_DEBUG_FS
1438 /* update debug settings */
1439 if (g->ops.ltc.sync_debugfs)
1440 g->ops.ltc.sync_debugfs(g);
1441#endif
1442
1443 gk20a_dbg_info("channel %d", c->hw_chid);
1444
1445 /* gk20a_channel_update releases this ref. */
1446 gk20a_channel_busy(g->dev);
1447
1448 trace_gk20a_channel_submit_gpfifo(c->g->dev->name,
1449 c->hw_chid,
1450 num_entries,
1451 flags,
1452 fence->syncpt_id, fence->value);
1453 check_gp_put(g, c);
1454 update_gp_get(g, c);
1455
1456 gk20a_dbg_info("pre-submit put %d, get %d, size %d",
1457 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1458
1459 /* Invalidate tlb if it's dirty... */
1460 /* TBD: this should be done in the cmd stream, not with PRIs. */
1461 /* We don't know what context is currently running... */
1462 /* Note also: there can be more than one context associated with the */
1463 /* address space (vm). */
1464 gk20a_mm_tlb_invalidate(c->vm);
1465
1466 /* Make sure we have enough space for gpfifo entries. If not,
1467 * wait for signals from completed submits */
1468 if (gp_free_count(c) < num_entries + extra_entries) {
1469 err = wait_event_interruptible(c->submit_wq,
1470 get_gp_free_count(c) >= num_entries + extra_entries ||
1471 c->has_timedout);
1472 }
1473
1474 if (c->has_timedout) {
1475 err = -ETIMEDOUT;
1476 goto clean_up;
1477 }
1478
1479 if (err) {
1480 gk20a_err(d, "not enough gpfifo space");
1481 err = -EAGAIN;
1482 goto clean_up;
1483 }
1484
1485 /*
1486 * optionally insert syncpt wait in the beginning of gpfifo submission
1487 * when user requested and the wait hasn't expired.
1488 * validate that the id makes sense, elide if not
1489 * the only reason this isn't being unceremoniously killed is to
1490 * keep running some tests which trigger this condition
1491 */
1492 if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT) {
1493 if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE)
1494 err = c->sync->wait_fd(c->sync, fence->syncpt_id,
1495 &wait_cmd);
1496 else
1497 err = c->sync->wait_syncpt(c->sync, fence->syncpt_id,
1498 fence->value, &wait_cmd);
1499 }
1500 if (err)
1501 goto clean_up;
1502
1503
1504 /* always insert syncpt increment at end of gpfifo submission
1505 to keep track of method completion for idle railgating */
1506 if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET &&
1507 flags & NVHOST_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE)
1508 err = c->sync->incr_user_fd(c->sync, &incr_cmd,
1509 &c->last_submit_fence,
1510 &fence->syncpt_id);
1511 else if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET)
1512 err = c->sync->incr_user_syncpt(c->sync, &incr_cmd,
1513 &c->last_submit_fence,
1514 &fence->syncpt_id,
1515 &fence->value);
1516 else
1517 err = c->sync->incr(c->sync, &incr_cmd,
1518 &c->last_submit_fence);
1519 if (err)
1520 goto clean_up;
1521
1522 if (wait_cmd) {
1523 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
1524 u64_lo32(wait_cmd->gva);
1525 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
1526 u64_hi32(wait_cmd->gva) |
1527 pbdma_gp_entry1_length_f(wait_cmd->size);
1528 trace_write_pushbuffer(c, &c->gpfifo.cpu_va[c->gpfifo.put]);
1529
1530 c->gpfifo.put = (c->gpfifo.put + 1) &
1531 (c->gpfifo.entry_num - 1);
1532
1533 /* save gp_put */
1534 wait_cmd->gp_put = c->gpfifo.put;
1535 }
1536
1537 for (i = 0; i < num_entries; i++) {
1538 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
1539 gpfifo[i].entry0; /* cmd buf va low 32 */
1540 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
1541 gpfifo[i].entry1; /* cmd buf va high 32 | words << 10 */
1542 trace_write_pushbuffer(c, &c->gpfifo.cpu_va[c->gpfifo.put]);
1543 c->gpfifo.put = (c->gpfifo.put + 1) &
1544 (c->gpfifo.entry_num - 1);
1545 }
1546
1547 if (incr_cmd) {
1548 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
1549 u64_lo32(incr_cmd->gva);
1550 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
1551 u64_hi32(incr_cmd->gva) |
1552 pbdma_gp_entry1_length_f(incr_cmd->size);
1553 trace_write_pushbuffer(c, &c->gpfifo.cpu_va[c->gpfifo.put]);
1554
1555 c->gpfifo.put = (c->gpfifo.put + 1) &
1556 (c->gpfifo.entry_num - 1);
1557
1558 /* save gp_put */
1559 incr_cmd->gp_put = c->gpfifo.put;
1560 }
1561
1562 /* Invalidate tlb if it's dirty... */
1563 /* TBD: this should be done in the cmd stream, not with PRIs. */
1564 /* We don't know what context is currently running... */
1565 /* Note also: there can be more than one context associated with the */
1566 /* address space (vm). */
1567 gk20a_mm_tlb_invalidate(c->vm);
1568
1569 trace_gk20a_channel_submitted_gpfifo(c->g->dev->name,
1570 c->hw_chid,
1571 num_entries,
1572 flags,
1573 fence->syncpt_id, fence->value);
1574
1575 /* TODO! Check for errors... */
1576 gk20a_channel_add_job(c, &c->last_submit_fence);
1577
1578 c->cmds_pending = true;
1579 gk20a_bar1_writel(g,
1580 c->userd_gpu_va + 4 * ram_userd_gp_put_w(),
1581 c->gpfifo.put);
1582
1583 gk20a_dbg_info("post-submit put %d, get %d, size %d",
1584 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1585
1586 gk20a_dbg_fn("done");
1587 return err;
1588
1589clean_up:
1590 gk20a_err(d, "fail");
1591 free_priv_cmdbuf(c, wait_cmd);
1592 free_priv_cmdbuf(c, incr_cmd);
1593 gk20a_channel_idle(g->dev);
1594 return err;
1595}
1596
1597void gk20a_remove_channel_support(struct channel_gk20a *c)
1598{
1599
1600}
1601
1602int gk20a_init_channel_support(struct gk20a *g, u32 chid)
1603{
1604 struct channel_gk20a *c = g->fifo.channel+chid;
1605 c->g = g;
1606 c->in_use = false;
1607 c->hw_chid = chid;
1608 c->bound = false;
1609 c->remove_support = gk20a_remove_channel_support;
1610 mutex_init(&c->jobs_lock);
1611 INIT_LIST_HEAD(&c->jobs);
1612#if defined(CONFIG_GK20A_CYCLE_STATS)
1613 mutex_init(&c->cyclestate.cyclestate_buffer_mutex);
1614#endif
1615 INIT_LIST_HEAD(&c->dbg_s_list);
1616 mutex_init(&c->dbg_s_lock);
1617
1618 return 0;
1619}
1620
1621int gk20a_channel_finish(struct channel_gk20a *ch, unsigned long timeout)
1622{
1623 int err = 0;
1624
1625 if (!ch->cmds_pending)
1626 return 0;
1627
1628 /* Do not wait for a timedout channel */
1629 if (ch->has_timedout)
1630 return -ETIMEDOUT;
1631
1632 if (!(ch->last_submit_fence.valid && ch->last_submit_fence.wfi)) {
1633 gk20a_dbg_fn("issuing wfi, incr to finish the channel");
1634 err = gk20a_channel_submit_wfi(ch);
1635 }
1636 if (err)
1637 return err;
1638
1639 BUG_ON(!(ch->last_submit_fence.valid && ch->last_submit_fence.wfi));
1640
1641 gk20a_dbg_fn("waiting for channel to finish thresh:%d",
1642 ch->last_submit_fence.thresh);
1643
1644 err = ch->sync->wait_cpu(ch->sync, &ch->last_submit_fence, timeout);
1645 if (WARN_ON(err))
1646 dev_warn(dev_from_gk20a(ch->g),
1647 "timed out waiting for gk20a channel to finish");
1648 else
1649 ch->cmds_pending = false;
1650
1651 return err;
1652}
1653
1654static int gk20a_channel_wait_semaphore(struct channel_gk20a *ch,
1655 ulong id, u32 offset,
1656 u32 payload, long timeout)
1657{
1658 struct platform_device *pdev = ch->g->dev;
1659 struct dma_buf *dmabuf;
1660 void *data;
1661 u32 *semaphore;
1662 int ret = 0;
1663 long remain;
1664
1665 /* do not wait if channel has timed out */
1666 if (ch->has_timedout)
1667 return -ETIMEDOUT;
1668
1669 dmabuf = dma_buf_get(id);
1670 if (IS_ERR(dmabuf)) {
1671 gk20a_err(&pdev->dev, "invalid notifier nvmap handle 0x%lx",
1672 id);
1673 return -EINVAL;
1674 }
1675
1676 data = dma_buf_kmap(dmabuf, offset >> PAGE_SHIFT);
1677 if (!data) {
1678 gk20a_err(&pdev->dev, "failed to map notifier memory");
1679 ret = -EINVAL;
1680 goto cleanup_put;
1681 }
1682
1683 semaphore = data + (offset & ~PAGE_MASK);
1684
1685 remain = wait_event_interruptible_timeout(
1686 ch->semaphore_wq,
1687 *semaphore == payload || ch->has_timedout,
1688 timeout);
1689
1690 if (remain == 0 && *semaphore != payload)
1691 ret = -ETIMEDOUT;
1692 else if (remain < 0)
1693 ret = remain;
1694
1695 dma_buf_kunmap(dmabuf, offset >> PAGE_SHIFT, data);
1696cleanup_put:
1697 dma_buf_put(dmabuf);
1698 return ret;
1699}
1700
1701static int gk20a_channel_wait(struct channel_gk20a *ch,
1702 struct nvhost_wait_args *args)
1703{
1704 struct device *d = dev_from_gk20a(ch->g);
1705 struct dma_buf *dmabuf;
1706 struct notification *notif;
1707 struct timespec tv;
1708 u64 jiffies;
1709 ulong id;
1710 u32 offset;
1711 unsigned long timeout;
1712 int remain, ret = 0;
1713
1714 gk20a_dbg_fn("");
1715
1716 if (ch->has_timedout)
1717 return -ETIMEDOUT;
1718
1719 if (args->timeout == NVHOST_NO_TIMEOUT)
1720 timeout = MAX_SCHEDULE_TIMEOUT;
1721 else
1722 timeout = (u32)msecs_to_jiffies(args->timeout);
1723
1724 switch (args->type) {
1725 case NVHOST_WAIT_TYPE_NOTIFIER:
1726 id = args->condition.notifier.nvmap_handle;
1727 offset = args->condition.notifier.offset;
1728
1729 dmabuf = dma_buf_get(id);
1730 if (IS_ERR(dmabuf)) {
1731 gk20a_err(d, "invalid notifier nvmap handle 0x%lx",
1732 id);
1733 return -EINVAL;
1734 }
1735
1736 notif = dma_buf_vmap(dmabuf);
1737 if (!notif) {
1738 gk20a_err(d, "failed to map notifier memory");
1739 return -ENOMEM;
1740 }
1741
1742 notif = (struct notification *)((uintptr_t)notif + offset);
1743
1744 /* user should set status pending before
1745 * calling this ioctl */
1746 remain = wait_event_interruptible_timeout(
1747 ch->notifier_wq,
1748 notif->status == 0 || ch->has_timedout,
1749 timeout);
1750
1751 if (remain == 0 && notif->status != 0) {
1752 ret = -ETIMEDOUT;
1753 goto notif_clean_up;
1754 } else if (remain < 0) {
1755 ret = -EINTR;
1756 goto notif_clean_up;
1757 }
1758
1759 /* TBD: fill in correct information */
1760 jiffies = get_jiffies_64();
1761 jiffies_to_timespec(jiffies, &tv);
1762 notif->timestamp.nanoseconds[0] = tv.tv_nsec;
1763 notif->timestamp.nanoseconds[1] = tv.tv_sec;
1764 notif->info32 = 0xDEADBEEF; /* should be object name */
1765 notif->info16 = ch->hw_chid; /* should be method offset */
1766
1767notif_clean_up:
1768 dma_buf_vunmap(dmabuf, notif);
1769 return ret;
1770
1771 case NVHOST_WAIT_TYPE_SEMAPHORE:
1772 ret = gk20a_channel_wait_semaphore(ch,
1773 args->condition.semaphore.nvmap_handle,
1774 args->condition.semaphore.offset,
1775 args->condition.semaphore.payload,
1776 timeout);
1777
1778 break;
1779
1780 default:
1781 ret = -EINVAL;
1782 break;
1783 }
1784
1785 return ret;
1786}
1787
1788static int gk20a_channel_set_priority(struct channel_gk20a *ch,
1789 u32 priority)
1790{
1791 u32 timeslice_timeout;
1792 /* set priority of graphics channel */
1793 switch (priority) {
1794 case NVHOST_PRIORITY_LOW:
1795 /* 64 << 3 = 512us */
1796 timeslice_timeout = 64;
1797 break;
1798 case NVHOST_PRIORITY_MEDIUM:
1799 /* 128 << 3 = 1024us */
1800 timeslice_timeout = 128;
1801 break;
1802 case NVHOST_PRIORITY_HIGH:
1803 /* 255 << 3 = 2048us */
1804 timeslice_timeout = 255;
1805 break;
1806 default:
1807 pr_err("Unsupported priority");
1808 return -EINVAL;
1809 }
1810 channel_gk20a_set_schedule_params(ch,
1811 timeslice_timeout);
1812 return 0;
1813}
1814
1815static int gk20a_channel_zcull_bind(struct channel_gk20a *ch,
1816 struct nvhost_zcull_bind_args *args)
1817{
1818 struct gk20a *g = ch->g;
1819 struct gr_gk20a *gr = &g->gr;
1820
1821 gk20a_dbg_fn("");
1822
1823 return gr_gk20a_bind_ctxsw_zcull(g, gr, ch,
1824 args->gpu_va, args->mode);
1825}
1826
1827/* in this context the "channel" is the host1x channel which
1828 * maps to *all* gk20a channels */
1829int gk20a_channel_suspend(struct gk20a *g)
1830{
1831 struct fifo_gk20a *f = &g->fifo;
1832 u32 chid;
1833 bool channels_in_use = false;
1834 struct device *d = dev_from_gk20a(g);
1835 int err;
1836
1837 gk20a_dbg_fn("");
1838
1839 /* idle the engine by submitting WFI on non-KEPLER_C channel */
1840 for (chid = 0; chid < f->num_channels; chid++) {
1841 struct channel_gk20a *c = &f->channel[chid];
1842 if (c->in_use && c->obj_class != KEPLER_C) {
1843 err = gk20a_channel_submit_wfi(c);
1844 if (err) {
1845 gk20a_err(d, "cannot idle channel %d\n",
1846 chid);
1847 return err;
1848 }
1849
1850 c->sync->wait_cpu(c->sync, &c->last_submit_fence,
1851 500000);
1852 break;
1853 }
1854 }
1855
1856 for (chid = 0; chid < f->num_channels; chid++) {
1857 if (f->channel[chid].in_use) {
1858
1859 gk20a_dbg_info("suspend channel %d", chid);
1860 /* disable channel */
1861 gk20a_writel(g, ccsr_channel_r(chid),
1862 gk20a_readl(g, ccsr_channel_r(chid)) |
1863 ccsr_channel_enable_clr_true_f());
1864 /* preempt the channel */
1865 gk20a_fifo_preempt_channel(g, chid);
1866
1867 channels_in_use = true;
1868 }
1869 }
1870
1871 if (channels_in_use) {
1872 gk20a_fifo_update_runlist(g, 0, ~0, false, true);
1873
1874 for (chid = 0; chid < f->num_channels; chid++) {
1875 if (f->channel[chid].in_use)
1876 channel_gk20a_unbind(&f->channel[chid]);
1877 }
1878 }
1879
1880 gk20a_dbg_fn("done");
1881 return 0;
1882}
1883
1884/* in this context the "channel" is the host1x channel which
1885 * maps to *all* gk20a channels */
1886int gk20a_channel_resume(struct gk20a *g)
1887{
1888 struct fifo_gk20a *f = &g->fifo;
1889 u32 chid;
1890 bool channels_in_use = false;
1891
1892 gk20a_dbg_fn("");
1893
1894 for (chid = 0; chid < f->num_channels; chid++) {
1895 if (f->channel[chid].in_use) {
1896 gk20a_dbg_info("resume channel %d", chid);
1897 g->ops.fifo.bind_channel(&f->channel[chid]);
1898 channels_in_use = true;
1899 }
1900 }
1901
1902 if (channels_in_use)
1903 gk20a_fifo_update_runlist(g, 0, ~0, true, true);
1904
1905 gk20a_dbg_fn("done");
1906 return 0;
1907}
1908
1909void gk20a_channel_semaphore_wakeup(struct gk20a *g)
1910{
1911 struct fifo_gk20a *f = &g->fifo;
1912 u32 chid;
1913
1914 gk20a_dbg_fn("");
1915
1916 for (chid = 0; chid < f->num_channels; chid++) {
1917 struct channel_gk20a *c = g->fifo.channel+chid;
1918 if (c->in_use)
1919 wake_up_interruptible_all(&c->semaphore_wq);
1920 }
1921}
1922
1923static int gk20a_ioctl_channel_submit_gpfifo(
1924 struct channel_gk20a *ch,
1925 struct nvhost_submit_gpfifo_args *args)
1926{
1927 void *gpfifo;
1928 u32 size;
1929 int ret = 0;
1930
1931 gk20a_dbg_fn("");
1932
1933 if (ch->has_timedout)
1934 return -ETIMEDOUT;
1935
1936 size = args->num_entries * sizeof(struct nvhost_gpfifo);
1937
1938 gpfifo = kzalloc(size, GFP_KERNEL);
1939 if (!gpfifo)
1940 return -ENOMEM;
1941
1942 if (copy_from_user(gpfifo,
1943 (void __user *)(uintptr_t)args->gpfifo, size)) {
1944 ret = -EINVAL;
1945 goto clean_up;
1946 }
1947
1948 ret = gk20a_submit_channel_gpfifo(ch, gpfifo, args->num_entries,
1949 &args->fence, args->flags);
1950
1951clean_up:
1952 kfree(gpfifo);
1953 return ret;
1954}
1955
1956void gk20a_init_fifo(struct gpu_ops *gops)
1957{
1958 gops->fifo.bind_channel = channel_gk20a_bind;
1959}
1960
1961long gk20a_channel_ioctl(struct file *filp,
1962 unsigned int cmd, unsigned long arg)
1963{
1964 struct channel_gk20a *ch = filp->private_data;
1965 struct platform_device *dev = ch->g->dev;
1966 u8 buf[NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE];
1967 int err = 0;
1968
1969 if ((_IOC_TYPE(cmd) != NVHOST_IOCTL_MAGIC) ||
1970 (_IOC_NR(cmd) == 0) ||
1971 (_IOC_NR(cmd) > NVHOST_IOCTL_CHANNEL_LAST) ||
1972 (_IOC_SIZE(cmd) > NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE))
1973 return -EFAULT;
1974
1975 if (_IOC_DIR(cmd) & _IOC_WRITE) {
1976 if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
1977 return -EFAULT;
1978 }
1979
1980 switch (cmd) {
1981 case NVHOST_IOCTL_CHANNEL_OPEN:
1982 {
1983 int fd;
1984 struct file *file;
1985 char *name;
1986
1987 err = get_unused_fd_flags(O_RDWR);
1988 if (err < 0)
1989 break;
1990 fd = err;
1991
1992 name = kasprintf(GFP_KERNEL, "nvhost-%s-fd%d",
1993 dev_name(&dev->dev), fd);
1994 if (!name) {
1995 err = -ENOMEM;
1996 put_unused_fd(fd);
1997 break;
1998 }
1999
2000 file = anon_inode_getfile(name, filp->f_op, NULL, O_RDWR);
2001 kfree(name);
2002 if (IS_ERR(file)) {
2003 err = PTR_ERR(file);
2004 put_unused_fd(fd);
2005 break;
2006 }
2007 fd_install(fd, file);
2008
2009 err = __gk20a_channel_open(ch->g, file);
2010 if (err) {
2011 put_unused_fd(fd);
2012 fput(file);
2013 break;
2014 }
2015
2016 ((struct nvhost_channel_open_args *)buf)->channel_fd = fd;
2017 break;
2018 }
2019 case NVHOST_IOCTL_CHANNEL_SET_NVMAP_FD:
2020 break;
2021 case NVHOST_IOCTL_CHANNEL_ALLOC_OBJ_CTX:
2022 gk20a_channel_busy(dev);
2023 err = gk20a_alloc_obj_ctx(ch,
2024 (struct nvhost_alloc_obj_ctx_args *)buf);
2025 gk20a_channel_idle(dev);
2026 break;
2027 case NVHOST_IOCTL_CHANNEL_FREE_OBJ_CTX:
2028 gk20a_channel_busy(dev);
2029 err = gk20a_free_obj_ctx(ch,
2030 (struct nvhost_free_obj_ctx_args *)buf);
2031 gk20a_channel_idle(dev);
2032 break;
2033 case NVHOST_IOCTL_CHANNEL_ALLOC_GPFIFO:
2034 gk20a_channel_busy(dev);
2035 err = gk20a_alloc_channel_gpfifo(ch,
2036 (struct nvhost_alloc_gpfifo_args *)buf);
2037 gk20a_channel_idle(dev);
2038 break;
2039 case NVHOST_IOCTL_CHANNEL_SUBMIT_GPFIFO:
2040 err = gk20a_ioctl_channel_submit_gpfifo(ch,
2041 (struct nvhost_submit_gpfifo_args *)buf);
2042 break;
2043 case NVHOST_IOCTL_CHANNEL_WAIT:
2044 gk20a_channel_busy(dev);
2045 err = gk20a_channel_wait(ch,
2046 (struct nvhost_wait_args *)buf);
2047 gk20a_channel_idle(dev);
2048 break;
2049 case NVHOST_IOCTL_CHANNEL_ZCULL_BIND:
2050 gk20a_channel_busy(dev);
2051 err = gk20a_channel_zcull_bind(ch,
2052 (struct nvhost_zcull_bind_args *)buf);
2053 gk20a_channel_idle(dev);
2054 break;
2055 case NVHOST_IOCTL_CHANNEL_SET_ERROR_NOTIFIER:
2056 gk20a_channel_busy(dev);
2057 err = gk20a_init_error_notifier(ch,
2058 (struct nvhost_set_error_notifier *)buf);
2059 gk20a_channel_idle(dev);
2060 break;
2061#ifdef CONFIG_GK20A_CYCLE_STATS
2062 case NVHOST_IOCTL_CHANNEL_CYCLE_STATS:
2063 gk20a_channel_busy(dev);
2064 err = gk20a_channel_cycle_stats(ch,
2065 (struct nvhost_cycle_stats_args *)buf);
2066 gk20a_channel_idle(dev);
2067 break;
2068#endif
2069 case NVHOST_IOCTL_CHANNEL_SET_TIMEOUT:
2070 {
2071 u32 timeout =
2072 (u32)((struct nvhost_set_timeout_args *)buf)->timeout;
2073 gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
2074 timeout, ch->hw_chid);
2075 ch->timeout_ms_max = timeout;
2076 break;
2077 }
2078 case NVHOST_IOCTL_CHANNEL_SET_TIMEOUT_EX:
2079 {
2080 u32 timeout =
2081 (u32)((struct nvhost_set_timeout_args *)buf)->timeout;
2082 bool timeout_debug_dump = !((u32)
2083 ((struct nvhost_set_timeout_ex_args *)buf)->flags &
2084 (1 << NVHOST_TIMEOUT_FLAG_DISABLE_DUMP));
2085 gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
2086 timeout, ch->hw_chid);
2087 ch->timeout_ms_max = timeout;
2088 ch->timeout_debug_dump = timeout_debug_dump;
2089 break;
2090 }
2091 case NVHOST_IOCTL_CHANNEL_GET_TIMEDOUT:
2092 ((struct nvhost_get_param_args *)buf)->value =
2093 ch->has_timedout;
2094 break;
2095 case NVHOST_IOCTL_CHANNEL_SET_PRIORITY:
2096 gk20a_channel_busy(dev);
2097 gk20a_channel_set_priority(ch,
2098 ((struct nvhost_set_priority_args *)buf)->priority);
2099 gk20a_channel_idle(dev);
2100 break;
2101 default:
2102 dev_err(&dev->dev, "unrecognized ioctl cmd: 0x%x", cmd);
2103 err = -ENOTTY;
2104 break;
2105 }
2106
2107 if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
2108 err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
2109
2110 return err;
2111}
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-07 16:07:07 -0400