summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/gpu/nvgpu/Makefile.nvgpu1
-rw-r--r--drivers/gpu/nvgpu/clk/clk_arb.c3
-rw-r--r--drivers/gpu/nvgpu/common/linux/ioctl.c2
-rw-r--r--drivers/gpu/nvgpu/common/linux/ioctl_channel.c1179
-rw-r--r--drivers/gpu/nvgpu/common/linux/ioctl_channel.h21
-rw-r--r--drivers/gpu/nvgpu/gk20a/cde_gk20a.c3
-rw-r--r--drivers/gpu/nvgpu/gk20a/ce2_gk20a.c3
-rw-r--r--drivers/gpu/nvgpu/gk20a/channel_gk20a.c1229
-rw-r--r--drivers/gpu/nvgpu/gk20a/channel_gk20a.h14
-rw-r--r--drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c2
-rw-r--r--drivers/gpu/nvgpu/gk20a/sched_gk20a.c2
-rw-r--r--drivers/gpu/nvgpu/gp106/xve_gp106.c1
12 files changed, 1268 insertions, 1192 deletions
diff --git a/drivers/gpu/nvgpu/Makefile.nvgpu b/drivers/gpu/nvgpu/Makefile.nvgpu
index e9e7581a..665076de 100644
--- a/drivers/gpu/nvgpu/Makefile.nvgpu
+++ b/drivers/gpu/nvgpu/Makefile.nvgpu
@@ -28,6 +28,7 @@ nvgpu-y := \
28 common/linux/ioctl.o \ 28 common/linux/ioctl.o \
29 common/linux/ioctl_ctrl.o \ 29 common/linux/ioctl_ctrl.o \
30 common/linux/ioctl_as.o \ 30 common/linux/ioctl_as.o \
31 common/linux/ioctl_channel.o \
31 common/mm/nvgpu_allocator.o \ 32 common/mm/nvgpu_allocator.o \
32 common/mm/bitmap_allocator.o \ 33 common/mm/bitmap_allocator.o \
33 common/mm/buddy_allocator.o \ 34 common/mm/buddy_allocator.o \
diff --git a/drivers/gpu/nvgpu/clk/clk_arb.c b/drivers/gpu/nvgpu/clk/clk_arb.c
index 0ccc7c53..dc063709 100644
--- a/drivers/gpu/nvgpu/clk/clk_arb.c
+++ b/drivers/gpu/nvgpu/clk/clk_arb.c
@@ -19,6 +19,9 @@
19#include <linux/bitops.h> 19#include <linux/bitops.h>
20#include <linux/rculist.h> 20#include <linux/rculist.h>
21#include <linux/llist.h> 21#include <linux/llist.h>
22#include <linux/uaccess.h>
23#include <linux/poll.h>
24#include <linux/atomic.h>
22 25
23#include <nvgpu/lock.h> 26#include <nvgpu/lock.h>
24#include <nvgpu/kmem.h> 27#include <nvgpu/kmem.h>
diff --git a/drivers/gpu/nvgpu/common/linux/ioctl.c b/drivers/gpu/nvgpu/common/linux/ioctl.c
index 53b9476e..db252aeb 100644
--- a/drivers/gpu/nvgpu/common/linux/ioctl.c
+++ b/drivers/gpu/nvgpu/common/linux/ioctl.c
@@ -23,8 +23,8 @@
23#include "gk20a/gk20a.h" 23#include "gk20a/gk20a.h"
24#include "gk20a/dbg_gpu_gk20a.h" 24#include "gk20a/dbg_gpu_gk20a.h"
25#include "gk20a/ctxsw_trace_gk20a.h" 25#include "gk20a/ctxsw_trace_gk20a.h"
26#include "gk20a/channel_gk20a.h"
27#include "gk20a/tsg_gk20a.h" 26#include "gk20a/tsg_gk20a.h"
27#include "ioctl_channel.h"
28#include "ioctl_ctrl.h" 28#include "ioctl_ctrl.h"
29#include "ioctl_as.h" 29#include "ioctl_as.h"
30 30
diff --git a/drivers/gpu/nvgpu/common/linux/ioctl_channel.c b/drivers/gpu/nvgpu/common/linux/ioctl_channel.c
new file mode 100644
index 00000000..4a9531de
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/ioctl_channel.c
@@ -0,0 +1,1179 @@
1/*
2 * GK20A Graphics channel
3 *
4 * Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <linux/list.h>
20#include <trace/events/gk20a.h>
21#include <linux/file.h>
22#include <linux/anon_inodes.h>
23#include <linux/dma-buf.h>
24
25#include <nvgpu/semaphore.h>
26#include <nvgpu/timers.h>
27#include <nvgpu/kmem.h>
28
29#include "gk20a/gk20a.h"
30#include "gk20a/debug_gk20a.h"
31#include "gk20a/ctxsw_trace_gk20a.h"
32#include "gk20a/dbg_gpu_gk20a.h"
33#include "gk20a/fence_gk20a.h"
34
35/*
36 * Although channels do have pointers back to the gk20a struct that they were
37 * created under in cases where the driver is killed that pointer can be bad.
38 * The channel memory can be freed before the release() function for a given
39 * channel is called. This happens when the driver dies and userspace doesn't
40 * get a chance to call release() until after the entire gk20a driver data is
41 * unloaded and freed.
42 */
43struct channel_priv {
44 struct gk20a *g;
45 struct channel_gk20a *c;
46};
47
48#if defined(CONFIG_GK20A_CYCLE_STATS)
49
50static int gk20a_channel_cycle_stats(struct channel_gk20a *ch,
51 struct nvgpu_cycle_stats_args *args)
52{
53 struct dma_buf *dmabuf;
54 void *virtual_address;
55
56 /* is it allowed to handle calls for current GPU? */
57 if (0 == (ch->g->gpu_characteristics.flags &
58 NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS))
59 return -ENOSYS;
60
61 if (args->dmabuf_fd && !ch->cyclestate.cyclestate_buffer_handler) {
62
63 /* set up new cyclestats buffer */
64 dmabuf = dma_buf_get(args->dmabuf_fd);
65 if (IS_ERR(dmabuf))
66 return PTR_ERR(dmabuf);
67 virtual_address = dma_buf_vmap(dmabuf);
68 if (!virtual_address)
69 return -ENOMEM;
70
71 ch->cyclestate.cyclestate_buffer_handler = dmabuf;
72 ch->cyclestate.cyclestate_buffer = virtual_address;
73 ch->cyclestate.cyclestate_buffer_size = dmabuf->size;
74 return 0;
75
76 } else if (!args->dmabuf_fd &&
77 ch->cyclestate.cyclestate_buffer_handler) {
78 gk20a_channel_free_cycle_stats_buffer(ch);
79 return 0;
80
81 } else if (!args->dmabuf_fd &&
82 !ch->cyclestate.cyclestate_buffer_handler) {
83 /* no requst from GL */
84 return 0;
85
86 } else {
87 pr_err("channel already has cyclestats buffer\n");
88 return -EINVAL;
89 }
90}
91
92static int gk20a_flush_cycle_stats_snapshot(struct channel_gk20a *ch)
93{
94 int ret;
95
96 nvgpu_mutex_acquire(&ch->cs_client_mutex);
97 if (ch->cs_client)
98 ret = gr_gk20a_css_flush(ch, ch->cs_client);
99 else
100 ret = -EBADF;
101 nvgpu_mutex_release(&ch->cs_client_mutex);
102
103 return ret;
104}
105
106static int gk20a_attach_cycle_stats_snapshot(struct channel_gk20a *ch,
107 u32 dmabuf_fd,
108 u32 perfmon_id_count,
109 u32 *perfmon_id_start)
110{
111 int ret;
112
113 nvgpu_mutex_acquire(&ch->cs_client_mutex);
114 if (ch->cs_client) {
115 ret = -EEXIST;
116 } else {
117 ret = gr_gk20a_css_attach(ch,
118 dmabuf_fd,
119 perfmon_id_count,
120 perfmon_id_start,
121 &ch->cs_client);
122 }
123 nvgpu_mutex_release(&ch->cs_client_mutex);
124
125 return ret;
126}
127
128static int gk20a_channel_cycle_stats_snapshot(struct channel_gk20a *ch,
129 struct nvgpu_cycle_stats_snapshot_args *args)
130{
131 int ret;
132
133 /* is it allowed to handle calls for current GPU? */
134 if (0 == (ch->g->gpu_characteristics.flags &
135 NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS_SNAPSHOT))
136 return -ENOSYS;
137
138 if (!args->dmabuf_fd)
139 return -EINVAL;
140
141 /* handle the command (most frequent cases first) */
142 switch (args->cmd) {
143 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_FLUSH:
144 ret = gk20a_flush_cycle_stats_snapshot(ch);
145 args->extra = 0;
146 break;
147
148 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_ATTACH:
149 ret = gk20a_attach_cycle_stats_snapshot(ch,
150 args->dmabuf_fd,
151 args->extra,
152 &args->extra);
153 break;
154
155 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_DETACH:
156 ret = gk20a_channel_free_cycle_stats_snapshot(ch);
157 args->extra = 0;
158 break;
159
160 default:
161 pr_err("cyclestats: unknown command %u\n", args->cmd);
162 ret = -EINVAL;
163 break;
164 }
165
166 return ret;
167}
168#endif
169
170static int gk20a_channel_set_wdt_status(struct channel_gk20a *ch,
171 struct nvgpu_channel_wdt_args *args)
172{
173 if (args->wdt_status == NVGPU_IOCTL_CHANNEL_DISABLE_WDT)
174 ch->wdt_enabled = false;
175 else if (args->wdt_status == NVGPU_IOCTL_CHANNEL_ENABLE_WDT)
176 ch->wdt_enabled = true;
177
178 return 0;
179}
180
181static int gk20a_init_error_notifier(struct channel_gk20a *ch,
182 struct nvgpu_set_error_notifier *args)
183{
184 struct device *dev = dev_from_gk20a(ch->g);
185 struct dma_buf *dmabuf;
186 void *va;
187 u64 end = args->offset + sizeof(struct nvgpu_notification);
188
189 if (!args->mem) {
190 pr_err("gk20a_init_error_notifier: invalid memory handle\n");
191 return -EINVAL;
192 }
193
194 dmabuf = dma_buf_get(args->mem);
195
196 gk20a_channel_free_error_notifiers(ch);
197
198 if (IS_ERR(dmabuf)) {
199 pr_err("Invalid handle: %d\n", args->mem);
200 return -EINVAL;
201 }
202
203 if (end > dmabuf->size || end < sizeof(struct nvgpu_notification)) {
204 dma_buf_put(dmabuf);
205 gk20a_err(dev, "gk20a_init_error_notifier: invalid offset\n");
206 return -EINVAL;
207 }
208
209 /* map handle */
210 va = dma_buf_vmap(dmabuf);
211 if (!va) {
212 dma_buf_put(dmabuf);
213 pr_err("Cannot map notifier handle\n");
214 return -ENOMEM;
215 }
216
217 ch->error_notifier = va + args->offset;
218 ch->error_notifier_va = va;
219 memset(ch->error_notifier, 0, sizeof(struct nvgpu_notification));
220
221 /* set channel notifiers pointer */
222 nvgpu_mutex_acquire(&ch->error_notifier_mutex);
223 ch->error_notifier_ref = dmabuf;
224 nvgpu_mutex_release(&ch->error_notifier_mutex);
225
226 return 0;
227}
228
229struct channel_gk20a *gk20a_get_channel_from_file(int fd)
230{
231 struct channel_priv *priv;
232 struct file *f = fget(fd);
233
234 if (!f)
235 return NULL;
236
237 if (f->f_op != &gk20a_channel_ops) {
238 fput(f);
239 return NULL;
240 }
241
242 priv = (struct channel_priv *)f->private_data;
243 fput(f);
244 return priv->c;
245}
246
247int gk20a_channel_release(struct inode *inode, struct file *filp)
248{
249 struct channel_priv *priv = filp->private_data;
250 struct channel_gk20a *ch = priv->c;
251 struct gk20a *g = priv->g;
252
253 int err;
254
255 err = gk20a_busy(g);
256 if (err) {
257 gk20a_err(dev_from_gk20a(g), "failed to release a channel!");
258 goto channel_release;
259 }
260
261 trace_gk20a_channel_release(dev_name(g->dev));
262
263 gk20a_channel_close(ch);
264 gk20a_idle(g);
265
266channel_release:
267 gk20a_put(g);
268 nvgpu_kfree(g, filp->private_data);
269 filp->private_data = NULL;
270 return 0;
271}
272
273/* note: runlist_id -1 is synonym for the ENGINE_GR_GK20A runlist id */
274static int __gk20a_channel_open(struct gk20a *g,
275 struct file *filp, s32 runlist_id)
276{
277 int err;
278 struct channel_gk20a *ch;
279 struct channel_priv *priv;
280
281 gk20a_dbg_fn("");
282
283 g = gk20a_get(g);
284 if (!g)
285 return -ENODEV;
286
287 trace_gk20a_channel_open(dev_name(g->dev));
288
289 priv = nvgpu_kzalloc(g, sizeof(*priv));
290 if (!priv) {
291 err = -ENOMEM;
292 goto free_ref;
293 }
294
295 err = gk20a_busy(g);
296 if (err) {
297 gk20a_err(dev_from_gk20a(g), "failed to power on, %d", err);
298 goto fail_busy;
299 }
300 /* All the user space channel should be non privilege */
301 ch = gk20a_open_new_channel(g, runlist_id, false);
302 gk20a_idle(g);
303 if (!ch) {
304 gk20a_err(dev_from_gk20a(g),
305 "failed to get f");
306 err = -ENOMEM;
307 goto fail_busy;
308 }
309
310 gk20a_channel_trace_sched_param(
311 trace_gk20a_channel_sched_defaults, ch);
312
313 priv->g = g;
314 priv->c = ch;
315
316 filp->private_data = priv;
317 return 0;
318
319fail_busy:
320 nvgpu_kfree(g, priv);
321free_ref:
322 gk20a_put(g);
323 return err;
324}
325
326int gk20a_channel_open(struct inode *inode, struct file *filp)
327{
328 struct gk20a *g = container_of(inode->i_cdev,
329 struct gk20a, channel.cdev);
330 int ret;
331
332 gk20a_dbg_fn("start");
333 ret = __gk20a_channel_open(g, filp, -1);
334
335 gk20a_dbg_fn("end");
336 return ret;
337}
338
339int gk20a_channel_open_ioctl(struct gk20a *g,
340 struct nvgpu_channel_open_args *args)
341{
342 int err;
343 int fd;
344 struct file *file;
345 char name[64];
346 s32 runlist_id = args->in.runlist_id;
347
348 err = get_unused_fd_flags(O_RDWR);
349 if (err < 0)
350 return err;
351 fd = err;
352
353 snprintf(name, sizeof(name), "nvhost-%s-fd%d",
354 dev_name(g->dev), fd);
355
356 file = anon_inode_getfile(name, g->channel.cdev.ops, NULL, O_RDWR);
357 if (IS_ERR(file)) {
358 err = PTR_ERR(file);
359 goto clean_up;
360 }
361
362 err = __gk20a_channel_open(g, file, runlist_id);
363 if (err)
364 goto clean_up_file;
365
366 fd_install(fd, file);
367 args->out.channel_fd = fd;
368 return 0;
369
370clean_up_file:
371 fput(file);
372clean_up:
373 put_unused_fd(fd);
374 return err;
375}
376
377int nvgpu_channel_ioctl_alloc_gpfifo(struct channel_gk20a *c,
378 struct nvgpu_alloc_gpfifo_ex_args *args)
379{
380 return gk20a_channel_alloc_gpfifo(c, args->num_entries,
381 args->num_inflight_jobs,
382 args->flags);
383}
384
385
386static int gk20a_channel_wait_semaphore(struct channel_gk20a *ch,
387 ulong id, u32 offset,
388 u32 payload, long timeout)
389{
390 struct device *dev = ch->g->dev;
391 struct dma_buf *dmabuf;
392 void *data;
393 u32 *semaphore;
394 int ret = 0;
395 long remain;
396
397 /* do not wait if channel has timed out */
398 if (ch->has_timedout)
399 return -ETIMEDOUT;
400
401 dmabuf = dma_buf_get(id);
402 if (IS_ERR(dmabuf)) {
403 gk20a_err(dev, "invalid notifier nvmap handle 0x%lx", id);
404 return -EINVAL;
405 }
406
407 data = dma_buf_kmap(dmabuf, offset >> PAGE_SHIFT);
408 if (!data) {
409 gk20a_err(dev, "failed to map notifier memory");
410 ret = -EINVAL;
411 goto cleanup_put;
412 }
413
414 semaphore = data + (offset & ~PAGE_MASK);
415
416 remain = wait_event_interruptible_timeout(
417 ch->semaphore_wq,
418 *semaphore == payload || ch->has_timedout,
419 timeout);
420
421 if (remain == 0 && *semaphore != payload)
422 ret = -ETIMEDOUT;
423 else if (remain < 0)
424 ret = remain;
425
426 dma_buf_kunmap(dmabuf, offset >> PAGE_SHIFT, data);
427cleanup_put:
428 dma_buf_put(dmabuf);
429 return ret;
430}
431
432static int gk20a_channel_wait(struct channel_gk20a *ch,
433 struct nvgpu_wait_args *args)
434{
435 struct device *d = dev_from_gk20a(ch->g);
436 struct dma_buf *dmabuf;
437 struct notification *notif;
438 struct timespec tv;
439 u64 jiffies;
440 ulong id;
441 u32 offset;
442 unsigned long timeout;
443 int remain, ret = 0;
444 u64 end;
445
446 gk20a_dbg_fn("");
447
448 if (ch->has_timedout)
449 return -ETIMEDOUT;
450
451 if (args->timeout == NVGPU_NO_TIMEOUT)
452 timeout = MAX_SCHEDULE_TIMEOUT;
453 else
454 timeout = (u32)msecs_to_jiffies(args->timeout);
455
456 switch (args->type) {
457 case NVGPU_WAIT_TYPE_NOTIFIER:
458 id = args->condition.notifier.dmabuf_fd;
459 offset = args->condition.notifier.offset;
460 end = offset + sizeof(struct notification);
461
462 dmabuf = dma_buf_get(id);
463 if (IS_ERR(dmabuf)) {
464 gk20a_err(d, "invalid notifier nvmap handle 0x%lx",
465 id);
466 return -EINVAL;
467 }
468
469 if (end > dmabuf->size || end < sizeof(struct notification)) {
470 dma_buf_put(dmabuf);
471 gk20a_err(d, "invalid notifier offset\n");
472 return -EINVAL;
473 }
474
475 notif = dma_buf_vmap(dmabuf);
476 if (!notif) {
477 gk20a_err(d, "failed to map notifier memory");
478 return -ENOMEM;
479 }
480
481 notif = (struct notification *)((uintptr_t)notif + offset);
482
483 /* user should set status pending before
484 * calling this ioctl */
485 remain = wait_event_interruptible_timeout(
486 ch->notifier_wq,
487 notif->status == 0 || ch->has_timedout,
488 timeout);
489
490 if (remain == 0 && notif->status != 0) {
491 ret = -ETIMEDOUT;
492 goto notif_clean_up;
493 } else if (remain < 0) {
494 ret = -EINTR;
495 goto notif_clean_up;
496 }
497
498 /* TBD: fill in correct information */
499 jiffies = get_jiffies_64();
500 jiffies_to_timespec(jiffies, &tv);
501 notif->timestamp.nanoseconds[0] = tv.tv_nsec;
502 notif->timestamp.nanoseconds[1] = tv.tv_sec;
503 notif->info32 = 0xDEADBEEF; /* should be object name */
504 notif->info16 = ch->hw_chid; /* should be method offset */
505
506notif_clean_up:
507 dma_buf_vunmap(dmabuf, notif);
508 return ret;
509
510 case NVGPU_WAIT_TYPE_SEMAPHORE:
511 ret = gk20a_channel_wait_semaphore(ch,
512 args->condition.semaphore.dmabuf_fd,
513 args->condition.semaphore.offset,
514 args->condition.semaphore.payload,
515 timeout);
516
517 break;
518
519 default:
520 ret = -EINVAL;
521 break;
522 }
523
524 return ret;
525}
526
527static unsigned int gk20a_event_id_poll(struct file *filep, poll_table *wait)
528{
529 unsigned int mask = 0;
530 struct gk20a_event_id_data *event_id_data = filep->private_data;
531 struct gk20a *g = event_id_data->g;
532 u32 event_id = event_id_data->event_id;
533
534 gk20a_dbg(gpu_dbg_fn | gpu_dbg_info, "");
535
536 poll_wait(filep, &event_id_data->event_id_wq, wait);
537
538 nvgpu_mutex_acquire(&event_id_data->lock);
539
540 if (event_id_data->is_tsg) {
541 struct tsg_gk20a *tsg = g->fifo.tsg + event_id_data->id;
542
543 if (event_id_data->event_posted) {
544 gk20a_dbg_info(
545 "found pending event_id=%d on TSG=%d\n",
546 event_id, tsg->tsgid);
547 mask = (POLLPRI | POLLIN);
548 event_id_data->event_posted = false;
549 }
550 } else {
551 struct channel_gk20a *ch = g->fifo.channel
552 + event_id_data->id;
553
554 if (event_id_data->event_posted) {
555 gk20a_dbg_info(
556 "found pending event_id=%d on chid=%d\n",
557 event_id, ch->hw_chid);
558 mask = (POLLPRI | POLLIN);
559 event_id_data->event_posted = false;
560 }
561 }
562
563 nvgpu_mutex_release(&event_id_data->lock);
564
565 return mask;
566}
567
568static int gk20a_event_id_release(struct inode *inode, struct file *filp)
569{
570 struct gk20a_event_id_data *event_id_data = filp->private_data;
571 struct gk20a *g = event_id_data->g;
572
573 if (event_id_data->is_tsg) {
574 struct tsg_gk20a *tsg = g->fifo.tsg + event_id_data->id;
575
576 nvgpu_mutex_acquire(&tsg->event_id_list_lock);
577 list_del_init(&event_id_data->event_id_node);
578 nvgpu_mutex_release(&tsg->event_id_list_lock);
579 } else {
580 struct channel_gk20a *ch = g->fifo.channel + event_id_data->id;
581
582 nvgpu_mutex_acquire(&ch->event_id_list_lock);
583 list_del_init(&event_id_data->event_id_node);
584 nvgpu_mutex_release(&ch->event_id_list_lock);
585 }
586
587 nvgpu_mutex_destroy(&event_id_data->lock);
588 gk20a_put(g);
589 nvgpu_kfree(g, event_id_data);
590 filp->private_data = NULL;
591
592 return 0;
593}
594
595const struct file_operations gk20a_event_id_ops = {
596 .owner = THIS_MODULE,
597 .poll = gk20a_event_id_poll,
598 .release = gk20a_event_id_release,
599};
600
601static int gk20a_channel_get_event_data_from_id(struct channel_gk20a *ch,
602 u32 event_id,
603 struct gk20a_event_id_data **event_id_data)
604{
605 struct gk20a_event_id_data *local_event_id_data;
606 bool event_found = false;
607
608 nvgpu_mutex_acquire(&ch->event_id_list_lock);
609 list_for_each_entry(local_event_id_data, &ch->event_id_list,
610 event_id_node) {
611 if (local_event_id_data->event_id == event_id) {
612 event_found = true;
613 break;
614 }
615 }
616 nvgpu_mutex_release(&ch->event_id_list_lock);
617
618 if (event_found) {
619 *event_id_data = local_event_id_data;
620 return 0;
621 } else {
622 return -1;
623 }
624}
625
626void gk20a_channel_event_id_post_event(struct channel_gk20a *ch,
627 u32 event_id)
628{
629 struct gk20a_event_id_data *event_id_data;
630 int err = 0;
631
632 err = gk20a_channel_get_event_data_from_id(ch, event_id,
633 &event_id_data);
634 if (err)
635 return;
636
637 nvgpu_mutex_acquire(&event_id_data->lock);
638
639 gk20a_dbg_info(
640 "posting event for event_id=%d on ch=%d\n",
641 event_id, ch->hw_chid);
642 event_id_data->event_posted = true;
643
644 wake_up_interruptible_all(&event_id_data->event_id_wq);
645
646 nvgpu_mutex_release(&event_id_data->lock);
647}
648
649static int gk20a_channel_event_id_enable(struct channel_gk20a *ch,
650 int event_id,
651 int *fd)
652{
653 struct gk20a *g;
654 int err = 0;
655 int local_fd;
656 struct file *file;
657 char name[64];
658 struct gk20a_event_id_data *event_id_data;
659
660 g = gk20a_get(ch->g);
661 if (!g)
662 return -ENODEV;
663
664 err = gk20a_channel_get_event_data_from_id(ch,
665 event_id, &event_id_data);
666 if (err == 0) {
667 /* We already have event enabled */
668 err = -EINVAL;
669 goto free_ref;
670 }
671
672 err = get_unused_fd_flags(O_RDWR);
673 if (err < 0)
674 goto free_ref;
675 local_fd = err;
676
677 snprintf(name, sizeof(name), "nvgpu-event%d-fd%d",
678 event_id, local_fd);
679 file = anon_inode_getfile(name, &gk20a_event_id_ops,
680 NULL, O_RDWR);
681 if (IS_ERR(file)) {
682 err = PTR_ERR(file);
683 goto clean_up;
684 }
685
686 event_id_data = nvgpu_kzalloc(ch->g, sizeof(*event_id_data));
687 if (!event_id_data) {
688 err = -ENOMEM;
689 goto clean_up_file;
690 }
691 event_id_data->g = g;
692 event_id_data->id = ch->hw_chid;
693 event_id_data->is_tsg = false;
694 event_id_data->event_id = event_id;
695
696 init_waitqueue_head(&event_id_data->event_id_wq);
697 err = nvgpu_mutex_init(&event_id_data->lock);
698 if (err)
699 goto clean_up_free;
700 INIT_LIST_HEAD(&event_id_data->event_id_node);
701
702 nvgpu_mutex_acquire(&ch->event_id_list_lock);
703 list_add_tail(&event_id_data->event_id_node, &ch->event_id_list);
704 nvgpu_mutex_release(&ch->event_id_list_lock);
705
706 fd_install(local_fd, file);
707 file->private_data = event_id_data;
708
709 *fd = local_fd;
710
711 return 0;
712
713clean_up_free:
714 kfree(event_id_data);
715clean_up_file:
716 fput(file);
717clean_up:
718 put_unused_fd(local_fd);
719free_ref:
720 gk20a_put(g);
721 return err;
722}
723
724static int gk20a_channel_event_id_ctrl(struct channel_gk20a *ch,
725 struct nvgpu_event_id_ctrl_args *args)
726{
727 int err = 0;
728 int fd = -1;
729
730 if (args->event_id >= NVGPU_IOCTL_CHANNEL_EVENT_ID_MAX)
731 return -EINVAL;
732
733 if (gk20a_is_channel_marked_as_tsg(ch))
734 return -EINVAL;
735
736 switch (args->cmd) {
737 case NVGPU_IOCTL_CHANNEL_EVENT_ID_CMD_ENABLE:
738 err = gk20a_channel_event_id_enable(ch, args->event_id, &fd);
739 if (!err)
740 args->event_fd = fd;
741 break;
742
743 default:
744 gk20a_err(dev_from_gk20a(ch->g),
745 "unrecognized channel event id cmd: 0x%x",
746 args->cmd);
747 err = -EINVAL;
748 break;
749 }
750
751 return err;
752}
753
754static int gk20a_channel_zcull_bind(struct channel_gk20a *ch,
755 struct nvgpu_zcull_bind_args *args)
756{
757 struct gk20a *g = ch->g;
758 struct gr_gk20a *gr = &g->gr;
759
760 gk20a_dbg_fn("");
761
762 return g->ops.gr.bind_ctxsw_zcull(g, gr, ch,
763 args->gpu_va, args->mode);
764}
765
766static int gk20a_ioctl_channel_submit_gpfifo(
767 struct channel_gk20a *ch,
768 struct nvgpu_submit_gpfifo_args *args)
769{
770 struct gk20a_fence *fence_out;
771 struct fifo_profile_gk20a *profile = NULL;
772
773 int ret = 0;
774 gk20a_dbg_fn("");
775
776#ifdef CONFIG_DEBUG_FS
777 profile = gk20a_fifo_profile_acquire(ch->g);
778
779 if (profile)
780 profile->timestamp[PROFILE_IOCTL_ENTRY] = sched_clock();
781#endif
782 if (ch->has_timedout)
783 return -ETIMEDOUT;
784 ret = gk20a_submit_channel_gpfifo(ch, NULL, args, args->num_entries,
785 args->flags, &args->fence,
786 &fence_out, false, profile);
787
788 if (ret)
789 goto clean_up;
790
791 /* Convert fence_out to something we can pass back to user space. */
792 if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) {
793 if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE) {
794 int fd = gk20a_fence_install_fd(fence_out);
795 if (fd < 0)
796 ret = fd;
797 else
798 args->fence.id = fd;
799 } else {
800 args->fence.id = fence_out->syncpt_id;
801 args->fence.value = fence_out->syncpt_value;
802 }
803 }
804 gk20a_fence_put(fence_out);
805#ifdef CONFIG_DEBUG_FS
806 if (profile) {
807 profile->timestamp[PROFILE_IOCTL_EXIT] = sched_clock();
808 gk20a_fifo_profile_release(ch->g, profile);
809 }
810#endif
811clean_up:
812 return ret;
813}
814
815long gk20a_channel_ioctl(struct file *filp,
816 unsigned int cmd, unsigned long arg)
817{
818 struct channel_priv *priv = filp->private_data;
819 struct channel_gk20a *ch = priv->c;
820 struct device *dev = ch->g->dev;
821 u8 buf[NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE] = {0};
822 int err = 0;
823
824 gk20a_dbg_fn("start %d", _IOC_NR(cmd));
825
826 if ((_IOC_TYPE(cmd) != NVGPU_IOCTL_MAGIC) ||
827 (_IOC_NR(cmd) == 0) ||
828 (_IOC_NR(cmd) > NVGPU_IOCTL_CHANNEL_LAST) ||
829 (_IOC_SIZE(cmd) > NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE))
830 return -EINVAL;
831
832 if (_IOC_DIR(cmd) & _IOC_WRITE) {
833 if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
834 return -EFAULT;
835 }
836
837 /* take a ref or return timeout if channel refs can't be taken */
838 ch = gk20a_channel_get(ch);
839 if (!ch)
840 return -ETIMEDOUT;
841
842 /* protect our sanity for threaded userspace - most of the channel is
843 * not thread safe */
844 nvgpu_mutex_acquire(&ch->ioctl_lock);
845
846 /* this ioctl call keeps a ref to the file which keeps a ref to the
847 * channel */
848
849 switch (cmd) {
850 case NVGPU_IOCTL_CHANNEL_OPEN:
851 err = gk20a_channel_open_ioctl(ch->g,
852 (struct nvgpu_channel_open_args *)buf);
853 break;
854 case NVGPU_IOCTL_CHANNEL_SET_NVMAP_FD:
855 break;
856 case NVGPU_IOCTL_CHANNEL_ALLOC_OBJ_CTX:
857 err = gk20a_busy(ch->g);
858 if (err) {
859 dev_err(dev,
860 "%s: failed to host gk20a for ioctl cmd: 0x%x",
861 __func__, cmd);
862 break;
863 }
864 err = ch->g->ops.gr.alloc_obj_ctx(ch,
865 (struct nvgpu_alloc_obj_ctx_args *)buf);
866 gk20a_idle(ch->g);
867 break;
868 case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO_EX:
869 {
870 struct nvgpu_alloc_gpfifo_ex_args *alloc_gpfifo_ex_args =
871 (struct nvgpu_alloc_gpfifo_ex_args *)buf;
872
873 err = gk20a_busy(ch->g);
874 if (err) {
875 dev_err(dev,
876 "%s: failed to host gk20a for ioctl cmd: 0x%x",
877 __func__, cmd);
878 break;
879 }
880
881 if (!is_power_of_2(alloc_gpfifo_ex_args->num_entries)) {
882 err = -EINVAL;
883 gk20a_idle(ch->g);
884 break;
885 }
886 err = gk20a_channel_alloc_gpfifo(ch,
887 alloc_gpfifo_ex_args->num_entries,
888 alloc_gpfifo_ex_args->num_inflight_jobs,
889 alloc_gpfifo_ex_args->flags);
890 gk20a_idle(ch->g);
891 break;
892 }
893 case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO:
894 {
895 struct nvgpu_alloc_gpfifo_args *alloc_gpfifo_args =
896 (struct nvgpu_alloc_gpfifo_args *)buf;
897
898 err = gk20a_busy(ch->g);
899 if (err) {
900 dev_err(dev,
901 "%s: failed to host gk20a for ioctl cmd: 0x%x",
902 __func__, cmd);
903 break;
904 }
905
906 /*
907 * Kernel can insert one extra gpfifo entry before user
908 * submitted gpfifos and another one after, for internal usage.
909 * Triple the requested size.
910 */
911 err = gk20a_channel_alloc_gpfifo(ch,
912 alloc_gpfifo_args->num_entries * 3,
913 0,
914 alloc_gpfifo_args->flags);
915 gk20a_idle(ch->g);
916 break;
917 }
918 case NVGPU_IOCTL_CHANNEL_SUBMIT_GPFIFO:
919 err = gk20a_ioctl_channel_submit_gpfifo(ch,
920 (struct nvgpu_submit_gpfifo_args *)buf);
921 break;
922 case NVGPU_IOCTL_CHANNEL_WAIT:
923 err = gk20a_busy(ch->g);
924 if (err) {
925 dev_err(dev,
926 "%s: failed to host gk20a for ioctl cmd: 0x%x",
927 __func__, cmd);
928 break;
929 }
930
931 /* waiting is thread-safe, not dropping this mutex could
932 * deadlock in certain conditions */
933 nvgpu_mutex_release(&ch->ioctl_lock);
934
935 err = gk20a_channel_wait(ch,
936 (struct nvgpu_wait_args *)buf);
937
938 nvgpu_mutex_acquire(&ch->ioctl_lock);
939
940 gk20a_idle(ch->g);
941 break;
942 case NVGPU_IOCTL_CHANNEL_ZCULL_BIND:
943 err = gk20a_busy(ch->g);
944 if (err) {
945 dev_err(dev,
946 "%s: failed to host gk20a for ioctl cmd: 0x%x",
947 __func__, cmd);
948 break;
949 }
950 err = gk20a_channel_zcull_bind(ch,
951 (struct nvgpu_zcull_bind_args *)buf);
952 gk20a_idle(ch->g);
953 break;
954 case NVGPU_IOCTL_CHANNEL_SET_ERROR_NOTIFIER:
955 err = gk20a_busy(ch->g);
956 if (err) {
957 dev_err(dev,
958 "%s: failed to host gk20a for ioctl cmd: 0x%x",
959 __func__, cmd);
960 break;
961 }
962 err = gk20a_init_error_notifier(ch,
963 (struct nvgpu_set_error_notifier *)buf);
964 gk20a_idle(ch->g);
965 break;
966#ifdef CONFIG_GK20A_CYCLE_STATS
967 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS:
968 err = gk20a_busy(ch->g);
969 if (err) {
970 dev_err(dev,
971 "%s: failed to host gk20a for ioctl cmd: 0x%x",
972 __func__, cmd);
973 break;
974 }
975 err = gk20a_channel_cycle_stats(ch,
976 (struct nvgpu_cycle_stats_args *)buf);
977 gk20a_idle(ch->g);
978 break;
979#endif
980 case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT:
981 {
982 u32 timeout =
983 (u32)((struct nvgpu_set_timeout_args *)buf)->timeout;
984 gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
985 timeout, ch->hw_chid);
986 ch->timeout_ms_max = timeout;
987 gk20a_channel_trace_sched_param(
988 trace_gk20a_channel_set_timeout, ch);
989 break;
990 }
991 case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT_EX:
992 {
993 u32 timeout =
994 (u32)((struct nvgpu_set_timeout_args *)buf)->timeout;
995 bool timeout_debug_dump = !((u32)
996 ((struct nvgpu_set_timeout_ex_args *)buf)->flags &
997 (1 << NVGPU_TIMEOUT_FLAG_DISABLE_DUMP));
998 gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
999 timeout, ch->hw_chid);
1000 ch->timeout_ms_max = timeout;
1001 ch->timeout_debug_dump = timeout_debug_dump;
1002 gk20a_channel_trace_sched_param(
1003 trace_gk20a_channel_set_timeout, ch);
1004 break;
1005 }
1006 case NVGPU_IOCTL_CHANNEL_GET_TIMEDOUT:
1007 ((struct nvgpu_get_param_args *)buf)->value =
1008 ch->has_timedout;
1009 break;
1010 case NVGPU_IOCTL_CHANNEL_SET_PRIORITY:
1011 err = gk20a_busy(ch->g);
1012 if (err) {
1013 dev_err(dev,
1014 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1015 __func__, cmd);
1016 break;
1017 }
1018 err = ch->g->ops.fifo.channel_set_priority(ch,
1019 ((struct nvgpu_set_priority_args *)buf)->priority);
1020
1021 gk20a_idle(ch->g);
1022 gk20a_channel_trace_sched_param(
1023 trace_gk20a_channel_set_priority, ch);
1024 break;
1025 case NVGPU_IOCTL_CHANNEL_ENABLE:
1026 err = gk20a_busy(ch->g);
1027 if (err) {
1028 dev_err(dev,
1029 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1030 __func__, cmd);
1031 break;
1032 }
1033 if (ch->g->ops.fifo.enable_channel)
1034 ch->g->ops.fifo.enable_channel(ch);
1035 else
1036 err = -ENOSYS;
1037 gk20a_idle(ch->g);
1038 break;
1039 case NVGPU_IOCTL_CHANNEL_DISABLE:
1040 err = gk20a_busy(ch->g);
1041 if (err) {
1042 dev_err(dev,
1043 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1044 __func__, cmd);
1045 break;
1046 }
1047 if (ch->g->ops.fifo.disable_channel)
1048 ch->g->ops.fifo.disable_channel(ch);
1049 else
1050 err = -ENOSYS;
1051 gk20a_idle(ch->g);
1052 break;
1053 case NVGPU_IOCTL_CHANNEL_PREEMPT:
1054 err = gk20a_busy(ch->g);
1055 if (err) {
1056 dev_err(dev,
1057 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1058 __func__, cmd);
1059 break;
1060 }
1061 err = gk20a_fifo_preempt(ch->g, ch);
1062 gk20a_idle(ch->g);
1063 break;
1064 case NVGPU_IOCTL_CHANNEL_FORCE_RESET:
1065 err = gk20a_busy(ch->g);
1066 if (err) {
1067 dev_err(dev,
1068 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1069 __func__, cmd);
1070 break;
1071 }
1072 err = ch->g->ops.fifo.force_reset_ch(ch,
1073 NVGPU_CHANNEL_RESETCHANNEL_VERIF_ERROR, true);
1074 gk20a_idle(ch->g);
1075 break;
1076 case NVGPU_IOCTL_CHANNEL_EVENT_ID_CTRL:
1077 err = gk20a_channel_event_id_ctrl(ch,
1078 (struct nvgpu_event_id_ctrl_args *)buf);
1079 break;
1080#ifdef CONFIG_GK20A_CYCLE_STATS
1081 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT:
1082 err = gk20a_busy(ch->g);
1083 if (err) {
1084 dev_err(dev,
1085 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1086 __func__, cmd);
1087 break;
1088 }
1089 err = gk20a_channel_cycle_stats_snapshot(ch,
1090 (struct nvgpu_cycle_stats_snapshot_args *)buf);
1091 gk20a_idle(ch->g);
1092 break;
1093#endif
1094 case NVGPU_IOCTL_CHANNEL_WDT:
1095 err = gk20a_channel_set_wdt_status(ch,
1096 (struct nvgpu_channel_wdt_args *)buf);
1097 break;
1098 case NVGPU_IOCTL_CHANNEL_SET_RUNLIST_INTERLEAVE:
1099 err = gk20a_busy(ch->g);
1100 if (err) {
1101 dev_err(dev,
1102 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1103 __func__, cmd);
1104 break;
1105 }
1106 err = gk20a_channel_set_runlist_interleave(ch,
1107 ((struct nvgpu_runlist_interleave_args *)buf)->level);
1108
1109 gk20a_idle(ch->g);
1110 gk20a_channel_trace_sched_param(
1111 trace_gk20a_channel_set_runlist_interleave, ch);
1112 break;
1113 case NVGPU_IOCTL_CHANNEL_SET_TIMESLICE:
1114 err = gk20a_busy(ch->g);
1115 if (err) {
1116 dev_err(dev,
1117 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1118 __func__, cmd);
1119 break;
1120 }
1121 err = ch->g->ops.fifo.channel_set_timeslice(ch,
1122 ((struct nvgpu_timeslice_args *)buf)->timeslice_us);
1123
1124 gk20a_idle(ch->g);
1125 gk20a_channel_trace_sched_param(
1126 trace_gk20a_channel_set_timeslice, ch);
1127 break;
1128 case NVGPU_IOCTL_CHANNEL_SET_PREEMPTION_MODE:
1129 if (ch->g->ops.gr.set_preemption_mode) {
1130 err = gk20a_busy(ch->g);
1131 if (err) {
1132 dev_err(dev,
1133 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1134 __func__, cmd);
1135 break;
1136 }
1137 err = ch->g->ops.gr.set_preemption_mode(ch,
1138 ((struct nvgpu_preemption_mode_args *)buf)->graphics_preempt_mode,
1139 ((struct nvgpu_preemption_mode_args *)buf)->compute_preempt_mode);
1140 gk20a_idle(ch->g);
1141 } else {
1142 err = -EINVAL;
1143 }
1144 break;
1145 case NVGPU_IOCTL_CHANNEL_SET_BOOSTED_CTX:
1146 if (ch->g->ops.gr.set_boosted_ctx) {
1147 bool boost =
1148 ((struct nvgpu_boosted_ctx_args *)buf)->boost;
1149
1150 err = gk20a_busy(ch->g);
1151 if (err) {
1152 dev_err(dev,
1153 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1154 __func__, cmd);
1155 break;
1156 }
1157 err = ch->g->ops.gr.set_boosted_ctx(ch, boost);
1158 gk20a_idle(ch->g);
1159 } else {
1160 err = -EINVAL;
1161 }
1162 break;
1163 default:
1164 dev_dbg(dev, "unrecognized ioctl cmd: 0x%x", cmd);
1165 err = -ENOTTY;
1166 break;
1167 }
1168
1169 if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
1170 err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
1171
1172 nvgpu_mutex_release(&ch->ioctl_lock);
1173
1174 gk20a_channel_put(ch);
1175
1176 gk20a_dbg_fn("end");
1177
1178 return err;
1179}
diff --git a/drivers/gpu/nvgpu/common/linux/ioctl_channel.h b/drivers/gpu/nvgpu/common/linux/ioctl_channel.h
new file mode 100644
index 00000000..f57739e2
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/ioctl_channel.h
@@ -0,0 +1,21 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 */
13#ifndef __NVGPU_IOCTL_CHANNEL_H__
14#define __NVGPU_IOCTL_CHANNEL_H__
15
16int gk20a_channel_open(struct inode *inode, struct file *filp);
17int gk20a_channel_release(struct inode *inode, struct file *filp);
18long gk20a_channel_ioctl(struct file *filp,
19 unsigned int cmd, unsigned long arg);
20
21#endif
diff --git a/drivers/gpu/nvgpu/gk20a/cde_gk20a.c b/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
index 42850588..926e21ef 100644
--- a/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
@@ -1224,8 +1224,7 @@ static int gk20a_cde_load(struct gk20a_cde_ctx *cde_ctx)
1224 } 1224 }
1225 1225
1226 /* allocate gpfifo (1024 should be more than enough) */ 1226 /* allocate gpfifo (1024 should be more than enough) */
1227 err = gk20a_alloc_channel_gpfifo(ch, 1227 err = gk20a_channel_alloc_gpfifo(ch, 1024, 0, 0);
1228 &(struct nvgpu_alloc_gpfifo_ex_args){1024, 0, 0, {}});
1229 if (err) { 1228 if (err) {
1230 gk20a_warn(cde_ctx->dev, "cde: unable to allocate gpfifo"); 1229 gk20a_warn(cde_ctx->dev, "cde: unable to allocate gpfifo");
1231 goto err_alloc_gpfifo; 1230 goto err_alloc_gpfifo;
diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
index 48f0cf8d..cce7a769 100644
--- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
@@ -471,8 +471,7 @@ u32 gk20a_ce_create_context_with_cb(struct device *dev,
471 } 471 }
472 472
473 /* allocate gpfifo (1024 should be more than enough) */ 473 /* allocate gpfifo (1024 should be more than enough) */
474 err = gk20a_alloc_channel_gpfifo(ce_ctx->ch, 474 err = gk20a_channel_alloc_gpfifo(ce_ctx->ch, 1024, 0, 0);
475 &(struct nvgpu_alloc_gpfifo_ex_args){1024, 0, 0, {}});
476 if (err) { 475 if (err) {
477 gk20a_err(ce_ctx->dev, "ce: unable to allocate gpfifo"); 476 gk20a_err(ce_ctx->dev, "ce: unable to allocate gpfifo");
478 goto end; 477 goto end;
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
index cd49b4a9..b7306369 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
@@ -16,15 +16,10 @@
16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */ 17 */
18 18
19#include <linux/nvhost.h>
20#include <linux/list.h> 19#include <linux/list.h>
21#include <linux/delay.h> 20#include <linux/delay.h>
22#include <linux/highmem.h> /* need for nvmap.h*/
23#include <linux/kthread.h> 21#include <linux/kthread.h>
24#include <trace/events/gk20a.h> 22#include <trace/events/gk20a.h>
25#include <linux/scatterlist.h>
26#include <linux/file.h>
27#include <linux/anon_inodes.h>
28#include <linux/dma-buf.h> 23#include <linux/dma-buf.h>
29#include <linux/circ_buf.h> 24#include <linux/circ_buf.h>
30 25
@@ -40,8 +35,6 @@
40 35
41#include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h> 36#include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h>
42 37
43#define NVMAP_HANDLE_PARAM_SIZE 1
44
45/* 38/*
46 * Although channels do have pointers back to the gk20a struct that they were 39 * Although channels do have pointers back to the gk20a struct that they were
47 * created under in cases where the driver is killed that pointer can be bad. 40 * created under in cases where the driver is killed that pointer can be bad.
@@ -55,7 +48,6 @@ struct channel_priv {
55 struct channel_gk20a *c; 48 struct channel_gk20a *c;
56}; 49};
57 50
58static struct channel_gk20a *allocate_channel(struct fifo_gk20a *f);
59static void free_channel(struct fifo_gk20a *f, struct channel_gk20a *c); 51static void free_channel(struct fifo_gk20a *f, struct channel_gk20a *c);
60static void gk20a_channel_dump_ref_actions(struct channel_gk20a *c); 52static void gk20a_channel_dump_ref_actions(struct channel_gk20a *c);
61 53
@@ -76,7 +68,6 @@ static struct channel_gk20a_job *channel_gk20a_joblist_peek(
76 68
77static int channel_gk20a_update_runlist(struct channel_gk20a *c, 69static int channel_gk20a_update_runlist(struct channel_gk20a *c,
78 bool add); 70 bool add);
79static void gk20a_free_error_notifiers(struct channel_gk20a *ch);
80 71
81static u32 gk20a_get_channel_watchdog_timeout(struct channel_gk20a *ch); 72static u32 gk20a_get_channel_watchdog_timeout(struct channel_gk20a *ch);
82 73
@@ -323,171 +314,6 @@ void gk20a_disable_channel(struct channel_gk20a *ch)
323 channel_gk20a_update_runlist(ch, false); 314 channel_gk20a_update_runlist(ch, false);
324} 315}
325 316
326#if defined(CONFIG_GK20A_CYCLE_STATS)
327
328static void gk20a_free_cycle_stats_buffer(struct channel_gk20a *ch)
329{
330 /* disable existing cyclestats buffer */
331 nvgpu_mutex_acquire(&ch->cyclestate.cyclestate_buffer_mutex);
332 if (ch->cyclestate.cyclestate_buffer_handler) {
333 dma_buf_vunmap(ch->cyclestate.cyclestate_buffer_handler,
334 ch->cyclestate.cyclestate_buffer);
335 dma_buf_put(ch->cyclestate.cyclestate_buffer_handler);
336 ch->cyclestate.cyclestate_buffer_handler = NULL;
337 ch->cyclestate.cyclestate_buffer = NULL;
338 ch->cyclestate.cyclestate_buffer_size = 0;
339 }
340 nvgpu_mutex_release(&ch->cyclestate.cyclestate_buffer_mutex);
341}
342
343static int gk20a_channel_cycle_stats(struct channel_gk20a *ch,
344 struct nvgpu_cycle_stats_args *args)
345{
346 struct dma_buf *dmabuf;
347 void *virtual_address;
348
349 /* is it allowed to handle calls for current GPU? */
350 if (0 == (ch->g->gpu_characteristics.flags &
351 NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS))
352 return -ENOSYS;
353
354 if (args->dmabuf_fd && !ch->cyclestate.cyclestate_buffer_handler) {
355
356 /* set up new cyclestats buffer */
357 dmabuf = dma_buf_get(args->dmabuf_fd);
358 if (IS_ERR(dmabuf))
359 return PTR_ERR(dmabuf);
360 virtual_address = dma_buf_vmap(dmabuf);
361 if (!virtual_address)
362 return -ENOMEM;
363
364 ch->cyclestate.cyclestate_buffer_handler = dmabuf;
365 ch->cyclestate.cyclestate_buffer = virtual_address;
366 ch->cyclestate.cyclestate_buffer_size = dmabuf->size;
367 return 0;
368
369 } else if (!args->dmabuf_fd &&
370 ch->cyclestate.cyclestate_buffer_handler) {
371 gk20a_free_cycle_stats_buffer(ch);
372 return 0;
373
374 } else if (!args->dmabuf_fd &&
375 !ch->cyclestate.cyclestate_buffer_handler) {
376 /* no requst from GL */
377 return 0;
378
379 } else {
380 pr_err("channel already has cyclestats buffer\n");
381 return -EINVAL;
382 }
383}
384
385
386static int gk20a_flush_cycle_stats_snapshot(struct channel_gk20a *ch)
387{
388 int ret;
389
390 nvgpu_mutex_acquire(&ch->cs_client_mutex);
391 if (ch->cs_client)
392 ret = gr_gk20a_css_flush(ch, ch->cs_client);
393 else
394 ret = -EBADF;
395 nvgpu_mutex_release(&ch->cs_client_mutex);
396
397 return ret;
398}
399
400static int gk20a_attach_cycle_stats_snapshot(struct channel_gk20a *ch,
401 u32 dmabuf_fd,
402 u32 perfmon_id_count,
403 u32 *perfmon_id_start)
404{
405 int ret;
406
407 nvgpu_mutex_acquire(&ch->cs_client_mutex);
408 if (ch->cs_client) {
409 ret = -EEXIST;
410 } else {
411 ret = gr_gk20a_css_attach(ch,
412 dmabuf_fd,
413 perfmon_id_count,
414 perfmon_id_start,
415 &ch->cs_client);
416 }
417 nvgpu_mutex_release(&ch->cs_client_mutex);
418
419 return ret;
420}
421
422static int gk20a_free_cycle_stats_snapshot(struct channel_gk20a *ch)
423{
424 int ret;
425
426 nvgpu_mutex_acquire(&ch->cs_client_mutex);
427 if (ch->cs_client) {
428 ret = gr_gk20a_css_detach(ch, ch->cs_client);
429 ch->cs_client = NULL;
430 } else {
431 ret = 0;
432 }
433 nvgpu_mutex_release(&ch->cs_client_mutex);
434
435 return ret;
436}
437
438static int gk20a_channel_cycle_stats_snapshot(struct channel_gk20a *ch,
439 struct nvgpu_cycle_stats_snapshot_args *args)
440{
441 int ret;
442
443 /* is it allowed to handle calls for current GPU? */
444 if (0 == (ch->g->gpu_characteristics.flags &
445 NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS_SNAPSHOT))
446 return -ENOSYS;
447
448 if (!args->dmabuf_fd)
449 return -EINVAL;
450
451 /* handle the command (most frequent cases first) */
452 switch (args->cmd) {
453 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_FLUSH:
454 ret = gk20a_flush_cycle_stats_snapshot(ch);
455 args->extra = 0;
456 break;
457
458 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_ATTACH:
459 ret = gk20a_attach_cycle_stats_snapshot(ch,
460 args->dmabuf_fd,
461 args->extra,
462 &args->extra);
463 break;
464
465 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_DETACH:
466 ret = gk20a_free_cycle_stats_snapshot(ch);
467 args->extra = 0;
468 break;
469
470 default:
471 pr_err("cyclestats: unknown command %u\n", args->cmd);
472 ret = -EINVAL;
473 break;
474 }
475
476 return ret;
477}
478#endif
479
480static int gk20a_channel_set_wdt_status(struct channel_gk20a *ch,
481 struct nvgpu_channel_wdt_args *args)
482{
483 if (args->wdt_status == NVGPU_IOCTL_CHANNEL_DISABLE_WDT)
484 ch->wdt_enabled = false;
485 else if (args->wdt_status == NVGPU_IOCTL_CHANNEL_ENABLE_WDT)
486 ch->wdt_enabled = true;
487
488 return 0;
489}
490
491int gk20a_channel_set_runlist_interleave(struct channel_gk20a *ch, 317int gk20a_channel_set_runlist_interleave(struct channel_gk20a *ch,
492 u32 level) 318 u32 level)
493{ 319{
@@ -516,54 +342,6 @@ int gk20a_channel_set_runlist_interleave(struct channel_gk20a *ch,
516 return ret ? ret : g->ops.fifo.update_runlist(g, ch->runlist_id, ~0, true, true); 342 return ret ? ret : g->ops.fifo.update_runlist(g, ch->runlist_id, ~0, true, true);
517} 343}
518 344
519static int gk20a_init_error_notifier(struct channel_gk20a *ch,
520 struct nvgpu_set_error_notifier *args)
521{
522 struct device *dev = dev_from_gk20a(ch->g);
523 struct dma_buf *dmabuf;
524 void *va;
525 u64 end = args->offset + sizeof(struct nvgpu_notification);
526
527 if (!args->mem) {
528 pr_err("gk20a_init_error_notifier: invalid memory handle\n");
529 return -EINVAL;
530 }
531
532 dmabuf = dma_buf_get(args->mem);
533
534 gk20a_free_error_notifiers(ch);
535
536 if (IS_ERR(dmabuf)) {
537 pr_err("Invalid handle: %d\n", args->mem);
538 return -EINVAL;
539 }
540
541 if (end > dmabuf->size || end < sizeof(struct nvgpu_notification)) {
542 dma_buf_put(dmabuf);
543 gk20a_err(dev, "gk20a_init_error_notifier: invalid offset\n");
544 return -EINVAL;
545 }
546
547 /* map handle */
548 va = dma_buf_vmap(dmabuf);
549 if (!va) {
550 dma_buf_put(dmabuf);
551 pr_err("Cannot map notifier handle\n");
552 return -ENOMEM;
553 }
554
555 ch->error_notifier = va + args->offset;
556 ch->error_notifier_va = va;
557 memset(ch->error_notifier, 0, sizeof(struct nvgpu_notification));
558
559 /* set channel notifiers pointer */
560 nvgpu_mutex_acquire(&ch->error_notifier_mutex);
561 ch->error_notifier_ref = dmabuf;
562 nvgpu_mutex_release(&ch->error_notifier_mutex);
563
564 return 0;
565}
566
567/** 345/**
568 * gk20a_set_error_notifier_locked() 346 * gk20a_set_error_notifier_locked()
569 * Should be called with ch->error_notifier_mutex held 347 * Should be called with ch->error_notifier_mutex held
@@ -595,7 +373,7 @@ void gk20a_set_error_notifier(struct channel_gk20a *ch, __u32 error)
595 nvgpu_mutex_release(&ch->error_notifier_mutex); 373 nvgpu_mutex_release(&ch->error_notifier_mutex);
596} 374}
597 375
598static void gk20a_free_error_notifiers(struct channel_gk20a *ch) 376void gk20a_channel_free_error_notifiers(struct channel_gk20a *ch)
599{ 377{
600 nvgpu_mutex_acquire(&ch->error_notifier_mutex); 378 nvgpu_mutex_acquire(&ch->error_notifier_mutex);
601 if (ch->error_notifier_ref) { 379 if (ch->error_notifier_ref) {
@@ -628,6 +406,40 @@ static void gk20a_wait_until_counter_is_N(
628 } 406 }
629} 407}
630 408
409#if defined(CONFIG_GK20A_CYCLE_STATS)
410void gk20a_channel_free_cycle_stats_buffer(struct channel_gk20a *ch)
411{
412 /* disable existing cyclestats buffer */
413 nvgpu_mutex_acquire(&ch->cyclestate.cyclestate_buffer_mutex);
414 if (ch->cyclestate.cyclestate_buffer_handler) {
415 dma_buf_vunmap(ch->cyclestate.cyclestate_buffer_handler,
416 ch->cyclestate.cyclestate_buffer);
417 dma_buf_put(ch->cyclestate.cyclestate_buffer_handler);
418 ch->cyclestate.cyclestate_buffer_handler = NULL;
419 ch->cyclestate.cyclestate_buffer = NULL;
420 ch->cyclestate.cyclestate_buffer_size = 0;
421 }
422 nvgpu_mutex_release(&ch->cyclestate.cyclestate_buffer_mutex);
423}
424
425int gk20a_channel_free_cycle_stats_snapshot(struct channel_gk20a *ch)
426{
427 int ret;
428
429 nvgpu_mutex_acquire(&ch->cs_client_mutex);
430 if (ch->cs_client) {
431 ret = gr_gk20a_css_detach(ch, ch->cs_client);
432 ch->cs_client = NULL;
433 } else {
434 ret = 0;
435 }
436 nvgpu_mutex_release(&ch->cs_client_mutex);
437
438 return ret;
439}
440
441#endif
442
631/* call ONLY when no references to the channel exist: after the last put */ 443/* call ONLY when no references to the channel exist: after the last put */
632static void gk20a_free_channel(struct channel_gk20a *ch, bool force) 444static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
633{ 445{
@@ -700,7 +512,7 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
700 gk20a_dbg_info("freeing bound channel context, timeout=%ld", 512 gk20a_dbg_info("freeing bound channel context, timeout=%ld",
701 timeout); 513 timeout);
702 514
703 gk20a_free_error_notifiers(ch); 515 gk20a_channel_free_error_notifiers(ch);
704 516
705 if (g->ops.fecs_trace.unbind_channel && !ch->vpr) 517 if (g->ops.fecs_trace.unbind_channel && !ch->vpr)
706 g->ops.fecs_trace.unbind_channel(g, ch); 518 g->ops.fecs_trace.unbind_channel(g, ch);
@@ -715,8 +527,8 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
715 memset(&ch->gpfifo, 0, sizeof(struct gpfifo_desc)); 527 memset(&ch->gpfifo, 0, sizeof(struct gpfifo_desc));
716 528
717#if defined(CONFIG_GK20A_CYCLE_STATS) 529#if defined(CONFIG_GK20A_CYCLE_STATS)
718 gk20a_free_cycle_stats_buffer(ch); 530 gk20a_channel_free_cycle_stats_buffer(ch);
719 gk20a_free_cycle_stats_snapshot(ch); 531 gk20a_channel_free_cycle_stats_snapshot(ch);
720#endif 532#endif
721 533
722 channel_gk20a_free_priv_cmdbuf(ch); 534 channel_gk20a_free_priv_cmdbuf(ch);
@@ -930,50 +742,6 @@ void __gk20a_channel_kill(struct channel_gk20a *ch)
930 gk20a_free_channel(ch, true); 742 gk20a_free_channel(ch, true);
931} 743}
932 744
933struct channel_gk20a *gk20a_get_channel_from_file(int fd)
934{
935 struct channel_priv *priv;
936 struct file *f = fget(fd);
937
938 if (!f)
939 return NULL;
940
941 if (f->f_op != &gk20a_channel_ops) {
942 fput(f);
943 return NULL;
944 }
945
946 priv = (struct channel_priv *)f->private_data;
947 fput(f);
948 return priv->c;
949}
950
951int gk20a_channel_release(struct inode *inode, struct file *filp)
952{
953 struct channel_priv *priv = filp->private_data;
954 struct channel_gk20a *ch = priv->c;
955 struct gk20a *g = priv->g;
956
957 int err;
958
959 err = gk20a_busy(g);
960 if (err) {
961 gk20a_err(dev_from_gk20a(g), "failed to release a channel!");
962 goto channel_release;
963 }
964
965 trace_gk20a_channel_release(g->name);
966
967 gk20a_channel_close(ch);
968 gk20a_idle(g);
969
970channel_release:
971 gk20a_put(g);
972 nvgpu_kfree(g, filp->private_data);
973 filp->private_data = NULL;
974 return 0;
975}
976
977static void gk20a_channel_update_runcb_fn(struct work_struct *work) 745static void gk20a_channel_update_runcb_fn(struct work_struct *work)
978{ 746{
979 struct channel_gk20a *ch = 747 struct channel_gk20a *ch =
@@ -1100,109 +868,6 @@ struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g,
1100 return ch; 868 return ch;
1101} 869}
1102 870
1103/* note: runlist_id -1 is synonym for the ENGINE_GR_GK20A runlist id */
1104static int __gk20a_channel_open(struct gk20a *g, struct file *filp, s32 runlist_id)
1105{
1106 int err;
1107 struct channel_gk20a *ch;
1108 struct channel_priv *priv;
1109
1110 gk20a_dbg_fn("");
1111
1112 g = gk20a_get(g);
1113 if (!g)
1114 return -ENODEV;
1115
1116 trace_gk20a_channel_open(g->name);
1117
1118 priv = nvgpu_kzalloc(g, sizeof(*priv));
1119 if (!priv) {
1120 err = -ENOMEM;
1121 goto free_ref;
1122 }
1123
1124 err = gk20a_busy(g);
1125 if (err) {
1126 gk20a_err(dev_from_gk20a(g), "failed to power on, %d", err);
1127 goto fail_busy;
1128 }
1129 /* All the user space channel should be non privilege */
1130 ch = gk20a_open_new_channel(g, runlist_id, false);
1131 gk20a_idle(g);
1132 if (!ch) {
1133 gk20a_err(dev_from_gk20a(g),
1134 "failed to get f");
1135 err = -ENOMEM;
1136 goto fail_busy;
1137 }
1138
1139 gk20a_channel_trace_sched_param(
1140 trace_gk20a_channel_sched_defaults, ch);
1141
1142 priv->g = g;
1143 priv->c = ch;
1144
1145 filp->private_data = priv;
1146 return 0;
1147
1148fail_busy:
1149 nvgpu_kfree(g, priv);
1150free_ref:
1151 gk20a_put(g);
1152 return err;
1153}
1154
1155int gk20a_channel_open(struct inode *inode, struct file *filp)
1156{
1157 struct gk20a *g = container_of(inode->i_cdev,
1158 struct gk20a, channel.cdev);
1159 int ret;
1160
1161 gk20a_dbg_fn("start");
1162 ret = __gk20a_channel_open(g, filp, -1);
1163
1164 gk20a_dbg_fn("end");
1165 return ret;
1166}
1167
1168int gk20a_channel_open_ioctl(struct gk20a *g,
1169 struct nvgpu_channel_open_args *args)
1170{
1171 int err;
1172 int fd;
1173 struct file *file;
1174 char name[64];
1175 s32 runlist_id = args->in.runlist_id;
1176
1177 err = get_unused_fd_flags(O_RDWR);
1178 if (err < 0)
1179 return err;
1180 fd = err;
1181
1182 snprintf(name, sizeof(name), "nvhost-%s-fd%d",
1183 g->name, fd);
1184
1185 file = anon_inode_getfile(name, g->channel.cdev.ops, NULL, O_RDWR);
1186 if (IS_ERR(file)) {
1187 err = PTR_ERR(file);
1188 goto clean_up;
1189 }
1190
1191 err = __gk20a_channel_open(g, file, runlist_id);
1192 if (err)
1193 goto clean_up_file;
1194
1195 fd_install(fd, file);
1196 args->out.channel_fd = fd;
1197 return 0;
1198
1199clean_up_file:
1200 fput(file);
1201clean_up:
1202 put_unused_fd(fd);
1203 return err;
1204}
1205
1206/* allocate private cmd buffer. 871/* allocate private cmd buffer.
1207 used for inserting commands before/after user submitted buffers. */ 872 used for inserting commands before/after user submitted buffers. */
1208static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c) 873static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c)
@@ -1542,8 +1207,10 @@ static void channel_gk20a_free_prealloc_resources(struct channel_gk20a *c)
1542 c->joblist.pre_alloc.enabled = false; 1207 c->joblist.pre_alloc.enabled = false;
1543} 1208}
1544 1209
1545int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, 1210int gk20a_channel_alloc_gpfifo(struct channel_gk20a *c,
1546 struct nvgpu_alloc_gpfifo_ex_args *args) 1211 unsigned int num_entries,
1212 unsigned int num_inflight_jobs,
1213 u32 flags)
1547{ 1214{
1548 struct gk20a *g = c->g; 1215 struct gk20a *g = c->g;
1549 struct device *d = dev_from_gk20a(g); 1216 struct device *d = dev_from_gk20a(g);
@@ -1553,12 +1220,12 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c,
1553 int err = 0; 1220 int err = 0;
1554 unsigned long acquire_timeout; 1221 unsigned long acquire_timeout;
1555 1222
1556 gpfifo_size = args->num_entries; 1223 gpfifo_size = num_entries;
1557 1224
1558 if (args->flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_VPR_ENABLED) 1225 if (flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_VPR_ENABLED)
1559 c->vpr = true; 1226 c->vpr = true;
1560 1227
1561 if (args->flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_DETERMINISTIC) 1228 if (flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_DETERMINISTIC)
1562 c->deterministic = true; 1229 c->deterministic = true;
1563 1230
1564 /* an address space needs to have been bound at this point. */ 1231 /* an address space needs to have been bound at this point. */
@@ -1625,15 +1292,15 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c,
1625 1292
1626 err = g->ops.fifo.setup_ramfc(c, c->gpfifo.mem.gpu_va, 1293 err = g->ops.fifo.setup_ramfc(c, c->gpfifo.mem.gpu_va,
1627 c->gpfifo.entry_num, 1294 c->gpfifo.entry_num,
1628 acquire_timeout, args->flags); 1295 acquire_timeout, flags);
1629 if (err) 1296 if (err)
1630 goto clean_up_sync; 1297 goto clean_up_sync;
1631 1298
1632 /* TBD: setup engine contexts */ 1299 /* TBD: setup engine contexts */
1633 1300
1634 if (args->num_inflight_jobs) { 1301 if (num_inflight_jobs) {
1635 err = channel_gk20a_prealloc_resources(c, 1302 err = channel_gk20a_prealloc_resources(c,
1636 args->num_inflight_jobs); 1303 num_inflight_jobs);
1637 if (err) 1304 if (err)
1638 goto clean_up_sync; 1305 goto clean_up_sync;
1639 } 1306 }
@@ -1654,7 +1321,7 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c,
1654clean_up_priv_cmd: 1321clean_up_priv_cmd:
1655 channel_gk20a_free_priv_cmdbuf(c); 1322 channel_gk20a_free_priv_cmdbuf(c);
1656clean_up_prealloc: 1323clean_up_prealloc:
1657 if (args->num_inflight_jobs) 1324 if (num_inflight_jobs)
1658 channel_gk20a_free_prealloc_resources(c); 1325 channel_gk20a_free_prealloc_resources(c);
1659clean_up_sync: 1326clean_up_sync:
1660 if (c->sync) { 1327 if (c->sync) {
@@ -3033,386 +2700,6 @@ fail_1:
3033 return err; 2700 return err;
3034} 2701}
3035 2702
3036static int gk20a_channel_wait_semaphore(struct channel_gk20a *ch,
3037 ulong id, u32 offset,
3038 u32 payload, long timeout)
3039{
3040 struct device *dev = ch->g->dev;
3041 struct dma_buf *dmabuf;
3042 void *data;
3043 u32 *semaphore;
3044 int ret = 0;
3045 long remain;
3046
3047 /* do not wait if channel has timed out */
3048 if (ch->has_timedout)
3049 return -ETIMEDOUT;
3050
3051 dmabuf = dma_buf_get(id);
3052 if (IS_ERR(dmabuf)) {
3053 gk20a_err(dev, "invalid notifier nvmap handle 0x%lx", id);
3054 return -EINVAL;
3055 }
3056
3057 data = dma_buf_kmap(dmabuf, offset >> PAGE_SHIFT);
3058 if (!data) {
3059 gk20a_err(dev, "failed to map notifier memory");
3060 ret = -EINVAL;
3061 goto cleanup_put;
3062 }
3063
3064 semaphore = data + (offset & ~PAGE_MASK);
3065
3066 remain = wait_event_interruptible_timeout(
3067 ch->semaphore_wq,
3068 *semaphore == payload || ch->has_timedout,
3069 timeout);
3070
3071 if (remain == 0 && *semaphore != payload)
3072 ret = -ETIMEDOUT;
3073 else if (remain < 0)
3074 ret = remain;
3075
3076 dma_buf_kunmap(dmabuf, offset >> PAGE_SHIFT, data);
3077cleanup_put:
3078 dma_buf_put(dmabuf);
3079 return ret;
3080}
3081
3082static int gk20a_channel_wait(struct channel_gk20a *ch,
3083 struct nvgpu_wait_args *args)
3084{
3085 struct device *d = dev_from_gk20a(ch->g);
3086 struct dma_buf *dmabuf;
3087 struct notification *notif;
3088 struct timespec tv;
3089 u64 jiffies;
3090 ulong id;
3091 u32 offset;
3092 unsigned long timeout;
3093 int remain, ret = 0;
3094 u64 end;
3095
3096 gk20a_dbg_fn("");
3097
3098 if (ch->has_timedout)
3099 return -ETIMEDOUT;
3100
3101 if (args->timeout == NVGPU_NO_TIMEOUT)
3102 timeout = MAX_SCHEDULE_TIMEOUT;
3103 else
3104 timeout = (u32)msecs_to_jiffies(args->timeout);
3105
3106 switch (args->type) {
3107 case NVGPU_WAIT_TYPE_NOTIFIER:
3108 id = args->condition.notifier.dmabuf_fd;
3109 offset = args->condition.notifier.offset;
3110 end = offset + sizeof(struct notification);
3111
3112 dmabuf = dma_buf_get(id);
3113 if (IS_ERR(dmabuf)) {
3114 gk20a_err(d, "invalid notifier nvmap handle 0x%lx",
3115 id);
3116 return -EINVAL;
3117 }
3118
3119 if (end > dmabuf->size || end < sizeof(struct notification)) {
3120 dma_buf_put(dmabuf);
3121 gk20a_err(d, "invalid notifier offset\n");
3122 return -EINVAL;
3123 }
3124
3125 notif = dma_buf_vmap(dmabuf);
3126 if (!notif) {
3127 gk20a_err(d, "failed to map notifier memory");
3128 return -ENOMEM;
3129 }
3130
3131 notif = (struct notification *)((uintptr_t)notif + offset);
3132
3133 /* user should set status pending before
3134 * calling this ioctl */
3135 remain = wait_event_interruptible_timeout(
3136 ch->notifier_wq,
3137 notif->status == 0 || ch->has_timedout,
3138 timeout);
3139
3140 if (remain == 0 && notif->status != 0) {
3141 ret = -ETIMEDOUT;
3142 goto notif_clean_up;
3143 } else if (remain < 0) {
3144 ret = -EINTR;
3145 goto notif_clean_up;
3146 }
3147
3148 /* TBD: fill in correct information */
3149 jiffies = get_jiffies_64();
3150 jiffies_to_timespec(jiffies, &tv);
3151 notif->timestamp.nanoseconds[0] = tv.tv_nsec;
3152 notif->timestamp.nanoseconds[1] = tv.tv_sec;
3153 notif->info32 = 0xDEADBEEF; /* should be object name */
3154 notif->info16 = ch->hw_chid; /* should be method offset */
3155
3156notif_clean_up:
3157 dma_buf_vunmap(dmabuf, notif);
3158 return ret;
3159
3160 case NVGPU_WAIT_TYPE_SEMAPHORE:
3161 ret = gk20a_channel_wait_semaphore(ch,
3162 args->condition.semaphore.dmabuf_fd,
3163 args->condition.semaphore.offset,
3164 args->condition.semaphore.payload,
3165 timeout);
3166
3167 break;
3168
3169 default:
3170 ret = -EINVAL;
3171 break;
3172 }
3173
3174 return ret;
3175}
3176
3177static unsigned int gk20a_event_id_poll(struct file *filep, poll_table *wait)
3178{
3179 unsigned int mask = 0;
3180 struct gk20a_event_id_data *event_id_data = filep->private_data;
3181 struct gk20a *g = event_id_data->g;
3182 u32 event_id = event_id_data->event_id;
3183
3184 gk20a_dbg(gpu_dbg_fn | gpu_dbg_info, "");
3185
3186 poll_wait(filep, &event_id_data->event_id_wq, wait);
3187
3188 nvgpu_mutex_acquire(&event_id_data->lock);
3189
3190 if (event_id_data->is_tsg) {
3191 struct tsg_gk20a *tsg = g->fifo.tsg + event_id_data->id;
3192
3193 if (event_id_data->event_posted) {
3194 gk20a_dbg_info(
3195 "found pending event_id=%d on TSG=%d\n",
3196 event_id, tsg->tsgid);
3197 mask = (POLLPRI | POLLIN);
3198 event_id_data->event_posted = false;
3199 }
3200 } else {
3201 struct channel_gk20a *ch = g->fifo.channel
3202 + event_id_data->id;
3203
3204 if (event_id_data->event_posted) {
3205 gk20a_dbg_info(
3206 "found pending event_id=%d on chid=%d\n",
3207 event_id, ch->hw_chid);
3208 mask = (POLLPRI | POLLIN);
3209 event_id_data->event_posted = false;
3210 }
3211 }
3212
3213 nvgpu_mutex_release(&event_id_data->lock);
3214
3215 return mask;
3216}
3217
3218static int gk20a_event_id_release(struct inode *inode, struct file *filp)
3219{
3220 struct gk20a_event_id_data *event_id_data = filp->private_data;
3221 struct gk20a *g = event_id_data->g;
3222
3223 if (event_id_data->is_tsg) {
3224 struct tsg_gk20a *tsg = g->fifo.tsg + event_id_data->id;
3225
3226 nvgpu_mutex_acquire(&tsg->event_id_list_lock);
3227 list_del_init(&event_id_data->event_id_node);
3228 nvgpu_mutex_release(&tsg->event_id_list_lock);
3229 } else {
3230 struct channel_gk20a *ch = g->fifo.channel + event_id_data->id;
3231
3232 nvgpu_mutex_acquire(&ch->event_id_list_lock);
3233 list_del_init(&event_id_data->event_id_node);
3234 nvgpu_mutex_release(&ch->event_id_list_lock);
3235 }
3236
3237 nvgpu_mutex_destroy(&event_id_data->lock);
3238 gk20a_put(g);
3239 nvgpu_kfree(g, event_id_data);
3240 filp->private_data = NULL;
3241
3242 return 0;
3243}
3244
3245const struct file_operations gk20a_event_id_ops = {
3246 .owner = THIS_MODULE,
3247 .poll = gk20a_event_id_poll,
3248 .release = gk20a_event_id_release,
3249};
3250
3251static int gk20a_channel_get_event_data_from_id(struct channel_gk20a *ch,
3252 u32 event_id,
3253 struct gk20a_event_id_data **event_id_data)
3254{
3255 struct gk20a_event_id_data *local_event_id_data;
3256 bool event_found = false;
3257
3258 nvgpu_mutex_acquire(&ch->event_id_list_lock);
3259 list_for_each_entry(local_event_id_data, &ch->event_id_list,
3260 event_id_node) {
3261 if (local_event_id_data->event_id == event_id) {
3262 event_found = true;
3263 break;
3264 }
3265 }
3266 nvgpu_mutex_release(&ch->event_id_list_lock);
3267
3268 if (event_found) {
3269 *event_id_data = local_event_id_data;
3270 return 0;
3271 } else {
3272 return -1;
3273 }
3274}
3275
3276void gk20a_channel_event_id_post_event(struct channel_gk20a *ch,
3277 u32 event_id)
3278{
3279 struct gk20a_event_id_data *event_id_data;
3280 int err = 0;
3281
3282 err = gk20a_channel_get_event_data_from_id(ch, event_id,
3283 &event_id_data);
3284 if (err)
3285 return;
3286
3287 nvgpu_mutex_acquire(&event_id_data->lock);
3288
3289 gk20a_dbg_info(
3290 "posting event for event_id=%d on ch=%d\n",
3291 event_id, ch->hw_chid);
3292 event_id_data->event_posted = true;
3293
3294 wake_up_interruptible_all(&event_id_data->event_id_wq);
3295
3296 nvgpu_mutex_release(&event_id_data->lock);
3297}
3298
3299static int gk20a_channel_event_id_enable(struct channel_gk20a *ch,
3300 int event_id,
3301 int *fd)
3302{
3303 struct gk20a *g;
3304 int err = 0;
3305 int local_fd;
3306 struct file *file;
3307 char name[64];
3308 struct gk20a_event_id_data *event_id_data;
3309
3310 g = gk20a_get(ch->g);
3311 if (!g)
3312 return -ENODEV;
3313
3314 err = gk20a_channel_get_event_data_from_id(ch,
3315 event_id, &event_id_data);
3316 if (err == 0) {
3317 /* We already have event enabled */
3318 err = -EINVAL;
3319 goto free_ref;
3320 }
3321
3322 err = get_unused_fd_flags(O_RDWR);
3323 if (err < 0)
3324 goto free_ref;
3325 local_fd = err;
3326
3327 snprintf(name, sizeof(name), "nvgpu-event%d-fd%d",
3328 event_id, local_fd);
3329 file = anon_inode_getfile(name, &gk20a_event_id_ops,
3330 NULL, O_RDWR);
3331 if (IS_ERR(file)) {
3332 err = PTR_ERR(file);
3333 goto clean_up;
3334 }
3335
3336 event_id_data = nvgpu_kzalloc(ch->g, sizeof(*event_id_data));
3337 if (!event_id_data) {
3338 err = -ENOMEM;
3339 goto clean_up_file;
3340 }
3341 event_id_data->g = g;
3342 event_id_data->id = ch->hw_chid;
3343 event_id_data->is_tsg = false;
3344 event_id_data->event_id = event_id;
3345
3346 init_waitqueue_head(&event_id_data->event_id_wq);
3347 err = nvgpu_mutex_init(&event_id_data->lock);
3348 if (err)
3349 goto clean_up_free;
3350 INIT_LIST_HEAD(&event_id_data->event_id_node);
3351
3352 nvgpu_mutex_acquire(&ch->event_id_list_lock);
3353 list_add_tail(&event_id_data->event_id_node, &ch->event_id_list);
3354 nvgpu_mutex_release(&ch->event_id_list_lock);
3355
3356 fd_install(local_fd, file);
3357 file->private_data = event_id_data;
3358
3359 *fd = local_fd;
3360
3361 return 0;
3362
3363clean_up_free:
3364 kfree(event_id_data);
3365clean_up_file:
3366 fput(file);
3367clean_up:
3368 put_unused_fd(local_fd);
3369free_ref:
3370 gk20a_put(g);
3371 return err;
3372}
3373
3374static int gk20a_channel_event_id_ctrl(struct channel_gk20a *ch,
3375 struct nvgpu_event_id_ctrl_args *args)
3376{
3377 int err = 0;
3378 int fd = -1;
3379
3380 if (args->event_id >= NVGPU_IOCTL_CHANNEL_EVENT_ID_MAX)
3381 return -EINVAL;
3382
3383 if (gk20a_is_channel_marked_as_tsg(ch))
3384 return -EINVAL;
3385
3386 switch (args->cmd) {
3387 case NVGPU_IOCTL_CHANNEL_EVENT_ID_CMD_ENABLE:
3388 err = gk20a_channel_event_id_enable(ch, args->event_id, &fd);
3389 if (!err)
3390 args->event_fd = fd;
3391 break;
3392
3393 default:
3394 gk20a_err(dev_from_gk20a(ch->g),
3395 "unrecognized channel event id cmd: 0x%x",
3396 args->cmd);
3397 err = -EINVAL;
3398 break;
3399 }
3400
3401 return err;
3402}
3403
3404static int gk20a_channel_zcull_bind(struct channel_gk20a *ch,
3405 struct nvgpu_zcull_bind_args *args)
3406{
3407 struct gk20a *g = ch->g;
3408 struct gr_gk20a *gr = &g->gr;
3409
3410 gk20a_dbg_fn("");
3411
3412 return g->ops.gr.bind_ctxsw_zcull(g, gr, ch,
3413 args->gpu_va, args->mode);
3414}
3415
3416/* in this context the "channel" is the host1x channel which 2703/* in this context the "channel" is the host1x channel which
3417 * maps to *all* gk20a channels */ 2704 * maps to *all* gk20a channels */
3418int gk20a_channel_suspend(struct gk20a *g) 2705int gk20a_channel_suspend(struct gk20a *g)
@@ -3533,421 +2820,3 @@ void gk20a_channel_semaphore_wakeup(struct gk20a *g, bool post_events)
3533 } 2820 }
3534 } 2821 }
3535} 2822}
3536
3537static int gk20a_ioctl_channel_submit_gpfifo(
3538 struct channel_gk20a *ch,
3539 struct nvgpu_submit_gpfifo_args *args)
3540{
3541 struct gk20a_fence *fence_out;
3542 struct fifo_profile_gk20a *profile = NULL;
3543
3544 int ret = 0;
3545 gk20a_dbg_fn("");
3546
3547#ifdef CONFIG_DEBUG_FS
3548 profile = gk20a_fifo_profile_acquire(ch->g);
3549
3550 if (profile)
3551 profile->timestamp[PROFILE_IOCTL_ENTRY] = sched_clock();
3552#endif
3553 if (ch->has_timedout)
3554 return -ETIMEDOUT;
3555 ret = gk20a_submit_channel_gpfifo(ch, NULL, args, args->num_entries,
3556 args->flags, &args->fence,
3557 &fence_out, false, profile);
3558
3559 if (ret)
3560 goto clean_up;
3561
3562 /* Convert fence_out to something we can pass back to user space. */
3563 if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) {
3564 if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE) {
3565 int fd = gk20a_fence_install_fd(fence_out);
3566 if (fd < 0)
3567 ret = fd;
3568 else
3569 args->fence.id = fd;
3570 } else {
3571 args->fence.id = fence_out->syncpt_id;
3572 args->fence.value = fence_out->syncpt_value;
3573 }
3574 }
3575 gk20a_fence_put(fence_out);
3576#ifdef CONFIG_DEBUG_FS
3577 if (profile) {
3578 profile->timestamp[PROFILE_IOCTL_EXIT] = sched_clock();
3579 gk20a_fifo_profile_release(ch->g, profile);
3580 }
3581#endif
3582clean_up:
3583 return ret;
3584}
3585
3586long gk20a_channel_ioctl(struct file *filp,
3587 unsigned int cmd, unsigned long arg)
3588{
3589 struct channel_priv *priv = filp->private_data;
3590 struct channel_gk20a *ch = priv->c;
3591 struct gk20a *g = ch->g;
3592 struct device *dev = g->dev;
3593 u8 buf[NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE] = {0};
3594 int err = 0;
3595
3596 gk20a_dbg_fn("start %d", _IOC_NR(cmd));
3597
3598 if ((_IOC_TYPE(cmd) != NVGPU_IOCTL_MAGIC) ||
3599 (_IOC_NR(cmd) == 0) ||
3600 (_IOC_NR(cmd) > NVGPU_IOCTL_CHANNEL_LAST) ||
3601 (_IOC_SIZE(cmd) > NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE))
3602 return -EINVAL;
3603
3604 if (_IOC_DIR(cmd) & _IOC_WRITE) {
3605 if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
3606 return -EFAULT;
3607 }
3608
3609 /* take a ref or return timeout if channel refs can't be taken */
3610 ch = gk20a_channel_get(ch);
3611 if (!ch)
3612 return -ETIMEDOUT;
3613
3614 /* protect our sanity for threaded userspace - most of the channel is
3615 * not thread safe */
3616 nvgpu_mutex_acquire(&ch->ioctl_lock);
3617
3618 /* this ioctl call keeps a ref to the file which keeps a ref to the
3619 * channel */
3620
3621 switch (cmd) {
3622 case NVGPU_IOCTL_CHANNEL_OPEN:
3623 err = gk20a_channel_open_ioctl(ch->g,
3624 (struct nvgpu_channel_open_args *)buf);
3625 break;
3626 case NVGPU_IOCTL_CHANNEL_SET_NVMAP_FD:
3627 break;
3628 case NVGPU_IOCTL_CHANNEL_ALLOC_OBJ_CTX:
3629 err = gk20a_busy(g);
3630 if (err) {
3631 dev_err(dev,
3632 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3633 __func__, cmd);
3634 break;
3635 }
3636 err = ch->g->ops.gr.alloc_obj_ctx(ch,
3637 (struct nvgpu_alloc_obj_ctx_args *)buf);
3638 gk20a_idle(g);
3639 break;
3640 case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO_EX:
3641 {
3642 struct nvgpu_alloc_gpfifo_ex_args *alloc_gpfifo_ex_args =
3643 (struct nvgpu_alloc_gpfifo_ex_args *)buf;
3644
3645 err = gk20a_busy(g);
3646 if (err) {
3647 dev_err(dev,
3648 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3649 __func__, cmd);
3650 break;
3651 }
3652
3653 if (!is_power_of_2(alloc_gpfifo_ex_args->num_entries)) {
3654 err = -EINVAL;
3655 gk20a_idle(g);
3656 break;
3657 }
3658 err = gk20a_alloc_channel_gpfifo(ch, alloc_gpfifo_ex_args);
3659 gk20a_idle(g);
3660 break;
3661 }
3662 case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO:
3663 {
3664 struct nvgpu_alloc_gpfifo_ex_args alloc_gpfifo_ex_args;
3665 struct nvgpu_alloc_gpfifo_args *alloc_gpfifo_args =
3666 (struct nvgpu_alloc_gpfifo_args *)buf;
3667
3668 err = gk20a_busy(g);
3669 if (err) {
3670 dev_err(dev,
3671 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3672 __func__, cmd);
3673 break;
3674 }
3675
3676 /* prepare new args structure */
3677 memset(&alloc_gpfifo_ex_args, 0,
3678 sizeof(struct nvgpu_alloc_gpfifo_ex_args));
3679 /*
3680 * Kernel can insert one extra gpfifo entry before user
3681 * submitted gpfifos and another one after, for internal usage.
3682 * Triple the requested size.
3683 */
3684 alloc_gpfifo_ex_args.num_entries = roundup_pow_of_two(
3685 alloc_gpfifo_args->num_entries * 3);
3686 alloc_gpfifo_ex_args.flags = alloc_gpfifo_args->flags;
3687
3688 err = gk20a_alloc_channel_gpfifo(ch, &alloc_gpfifo_ex_args);
3689 gk20a_idle(g);
3690 break;
3691 }
3692 case NVGPU_IOCTL_CHANNEL_SUBMIT_GPFIFO:
3693 err = gk20a_ioctl_channel_submit_gpfifo(ch,
3694 (struct nvgpu_submit_gpfifo_args *)buf);
3695 break;
3696 case NVGPU_IOCTL_CHANNEL_WAIT:
3697 err = gk20a_busy(g);
3698 if (err) {
3699 dev_err(dev,
3700 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3701 __func__, cmd);
3702 break;
3703 }
3704
3705 /* waiting is thread-safe, not dropping this mutex could
3706 * deadlock in certain conditions */
3707 nvgpu_mutex_release(&ch->ioctl_lock);
3708
3709 err = gk20a_channel_wait(ch,
3710 (struct nvgpu_wait_args *)buf);
3711
3712 nvgpu_mutex_acquire(&ch->ioctl_lock);
3713
3714 gk20a_idle(g);
3715 break;
3716 case NVGPU_IOCTL_CHANNEL_ZCULL_BIND:
3717 err = gk20a_busy(g);
3718 if (err) {
3719 dev_err(dev,
3720 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3721 __func__, cmd);
3722 break;
3723 }
3724 err = gk20a_channel_zcull_bind(ch,
3725 (struct nvgpu_zcull_bind_args *)buf);
3726 gk20a_idle(g);
3727 break;
3728 case NVGPU_IOCTL_CHANNEL_SET_ERROR_NOTIFIER:
3729 err = gk20a_busy(g);
3730 if (err) {
3731 dev_err(dev,
3732 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3733 __func__, cmd);
3734 break;
3735 }
3736 err = gk20a_init_error_notifier(ch,
3737 (struct nvgpu_set_error_notifier *)buf);
3738 gk20a_idle(g);
3739 break;
3740#ifdef CONFIG_GK20A_CYCLE_STATS
3741 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS:
3742 err = gk20a_busy(g);
3743 if (err) {
3744 dev_err(dev,
3745 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3746 __func__, cmd);
3747 break;
3748 }
3749 err = gk20a_channel_cycle_stats(ch,
3750 (struct nvgpu_cycle_stats_args *)buf);
3751 gk20a_idle(g);
3752 break;
3753#endif
3754 case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT:
3755 {
3756 u32 timeout =
3757 (u32)((struct nvgpu_set_timeout_args *)buf)->timeout;
3758 gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
3759 timeout, ch->hw_chid);
3760 ch->timeout_ms_max = timeout;
3761 gk20a_channel_trace_sched_param(
3762 trace_gk20a_channel_set_timeout, ch);
3763 break;
3764 }
3765 case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT_EX:
3766 {
3767 u32 timeout =
3768 (u32)((struct nvgpu_set_timeout_args *)buf)->timeout;
3769 bool timeout_debug_dump = !((u32)
3770 ((struct nvgpu_set_timeout_ex_args *)buf)->flags &
3771 (1 << NVGPU_TIMEOUT_FLAG_DISABLE_DUMP));
3772 gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
3773 timeout, ch->hw_chid);
3774 ch->timeout_ms_max = timeout;
3775 ch->timeout_debug_dump = timeout_debug_dump;
3776 gk20a_channel_trace_sched_param(
3777 trace_gk20a_channel_set_timeout, ch);
3778 break;
3779 }
3780 case NVGPU_IOCTL_CHANNEL_GET_TIMEDOUT:
3781 ((struct nvgpu_get_param_args *)buf)->value =
3782 ch->has_timedout;
3783 break;
3784 case NVGPU_IOCTL_CHANNEL_SET_PRIORITY:
3785 err = gk20a_busy(g);
3786 if (err) {
3787 dev_err(dev,
3788 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3789 __func__, cmd);
3790 break;
3791 }
3792 err = ch->g->ops.fifo.channel_set_priority(ch,
3793 ((struct nvgpu_set_priority_args *)buf)->priority);
3794
3795 gk20a_idle(g);
3796 gk20a_channel_trace_sched_param(
3797 trace_gk20a_channel_set_priority, ch);
3798 break;
3799 case NVGPU_IOCTL_CHANNEL_ENABLE:
3800 err = gk20a_busy(g);
3801 if (err) {
3802 dev_err(dev,
3803 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3804 __func__, cmd);
3805 break;
3806 }
3807 if (ch->g->ops.fifo.enable_channel)
3808 ch->g->ops.fifo.enable_channel(ch);
3809 else
3810 err = -ENOSYS;
3811 gk20a_idle(g);
3812 break;
3813 case NVGPU_IOCTL_CHANNEL_DISABLE:
3814 err = gk20a_busy(g);
3815 if (err) {
3816 dev_err(dev,
3817 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3818 __func__, cmd);
3819 break;
3820 }
3821 if (ch->g->ops.fifo.disable_channel)
3822 ch->g->ops.fifo.disable_channel(ch);
3823 else
3824 err = -ENOSYS;
3825 gk20a_idle(g);
3826 break;
3827 case NVGPU_IOCTL_CHANNEL_PREEMPT:
3828 err = gk20a_busy(g);
3829 if (err) {
3830 dev_err(dev,
3831 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3832 __func__, cmd);
3833 break;
3834 }
3835 err = gk20a_fifo_preempt(ch->g, ch);
3836 gk20a_idle(g);
3837 break;
3838 case NVGPU_IOCTL_CHANNEL_FORCE_RESET:
3839 err = gk20a_busy(g);
3840 if (err) {
3841 dev_err(dev,
3842 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3843 __func__, cmd);
3844 break;
3845 }
3846 err = ch->g->ops.fifo.force_reset_ch(ch,
3847 NVGPU_CHANNEL_RESETCHANNEL_VERIF_ERROR, true);
3848 gk20a_idle(g);
3849 break;
3850 case NVGPU_IOCTL_CHANNEL_EVENT_ID_CTRL:
3851 err = gk20a_channel_event_id_ctrl(ch,
3852 (struct nvgpu_event_id_ctrl_args *)buf);
3853 break;
3854#ifdef CONFIG_GK20A_CYCLE_STATS
3855 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT:
3856 err = gk20a_busy(g);
3857 if (err) {
3858 dev_err(dev,
3859 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3860 __func__, cmd);
3861 break;
3862 }
3863 err = gk20a_channel_cycle_stats_snapshot(ch,
3864 (struct nvgpu_cycle_stats_snapshot_args *)buf);
3865 gk20a_idle(g);
3866 break;
3867#endif
3868 case NVGPU_IOCTL_CHANNEL_WDT:
3869 err = gk20a_channel_set_wdt_status(ch,
3870 (struct nvgpu_channel_wdt_args *)buf);
3871 break;
3872 case NVGPU_IOCTL_CHANNEL_SET_RUNLIST_INTERLEAVE:
3873 err = gk20a_busy(g);
3874 if (err) {
3875 dev_err(dev,
3876 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3877 __func__, cmd);
3878 break;
3879 }
3880 err = gk20a_channel_set_runlist_interleave(ch,
3881 ((struct nvgpu_runlist_interleave_args *)buf)->level);
3882
3883 gk20a_idle(g);
3884 gk20a_channel_trace_sched_param(
3885 trace_gk20a_channel_set_runlist_interleave, ch);
3886 break;
3887 case NVGPU_IOCTL_CHANNEL_SET_TIMESLICE:
3888 err = gk20a_busy(g);
3889 if (err) {
3890 dev_err(dev,
3891 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3892 __func__, cmd);
3893 break;
3894 }
3895 err = ch->g->ops.fifo.channel_set_timeslice(ch,
3896 ((struct nvgpu_timeslice_args *)buf)->timeslice_us);
3897
3898 gk20a_idle(g);
3899 gk20a_channel_trace_sched_param(
3900 trace_gk20a_channel_set_timeslice, ch);
3901 break;
3902 case NVGPU_IOCTL_CHANNEL_SET_PREEMPTION_MODE:
3903 if (ch->g->ops.gr.set_preemption_mode) {
3904 err = gk20a_busy(g);
3905 if (err) {
3906 dev_err(dev,
3907 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3908 __func__, cmd);
3909 break;
3910 }
3911 err = ch->g->ops.gr.set_preemption_mode(ch,
3912 ((struct nvgpu_preemption_mode_args *)buf)->graphics_preempt_mode,
3913 ((struct nvgpu_preemption_mode_args *)buf)->compute_preempt_mode);
3914 gk20a_idle(g);
3915 } else {
3916 err = -EINVAL;
3917 }
3918 break;
3919 case NVGPU_IOCTL_CHANNEL_SET_BOOSTED_CTX:
3920 if (ch->g->ops.gr.set_boosted_ctx) {
3921 bool boost =
3922 ((struct nvgpu_boosted_ctx_args *)buf)->boost;
3923
3924 err = gk20a_busy(g);
3925 if (err) {
3926 dev_err(dev,
3927 "%s: failed to host gk20a for ioctl cmd: 0x%x",
3928 __func__, cmd);
3929 break;
3930 }
3931 err = ch->g->ops.gr.set_boosted_ctx(ch, boost);
3932 gk20a_idle(g);
3933 } else {
3934 err = -EINVAL;
3935 }
3936 break;
3937 default:
3938 dev_dbg(dev, "unrecognized ioctl cmd: 0x%x", cmd);
3939 err = -ENOTTY;
3940 break;
3941 }
3942
3943 if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
3944 err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
3945
3946 nvgpu_mutex_release(&ch->ioctl_lock);
3947
3948 gk20a_channel_put(ch);
3949
3950 gk20a_dbg_fn("end");
3951
3952 return err;
3953}
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
index 37b1d945..bcfdadec 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
@@ -18,13 +18,8 @@
18#ifndef CHANNEL_GK20A_H 18#ifndef CHANNEL_GK20A_H
19#define CHANNEL_GK20A_H 19#define CHANNEL_GK20A_H
20 20
21#include <linux/log2.h>
22#include <linux/poll.h>
23#include <linux/semaphore.h>
24#include <linux/slab.h>
25#include <linux/stacktrace.h> 21#include <linux/stacktrace.h>
26#include <linux/wait.h> 22#include <linux/wait.h>
27#include <uapi/linux/nvgpu.h>
28#include <nvgpu/list.h> 23#include <nvgpu/list.h>
29 24
30#include <nvgpu/lock.h> 25#include <nvgpu/lock.h>
@@ -359,8 +354,13 @@ int gk20a_submit_channel_gpfifo(struct channel_gk20a *c,
359 bool force_need_sync_fence, 354 bool force_need_sync_fence,
360 struct fifo_profile_gk20a *profile); 355 struct fifo_profile_gk20a *profile);
361 356
362int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, 357int gk20a_channel_alloc_gpfifo(struct channel_gk20a *c,
363 struct nvgpu_alloc_gpfifo_ex_args *args); 358 unsigned int num_entries,
359 unsigned int num_inflight_jobs,
360 u32 flags);
361void gk20a_channel_free_error_notifiers(struct channel_gk20a *ch);
362void gk20a_channel_free_cycle_stats_buffer(struct channel_gk20a *ch);
363int gk20a_channel_free_cycle_stats_snapshot(struct channel_gk20a *ch);
364 364
365void gk20a_channel_timeout_restart_all_channels(struct gk20a *g); 365void gk20a_channel_timeout_restart_all_channels(struct gk20a *g);
366 366
diff --git a/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c b/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c
index 1c3ba9c2..2f6a85a5 100644
--- a/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c
@@ -22,6 +22,8 @@
22#include <linux/hashtable.h> 22#include <linux/hashtable.h>
23#include <linux/debugfs.h> 23#include <linux/debugfs.h>
24#include <linux/log2.h> 24#include <linux/log2.h>
25#include <linux/uaccess.h>
26#include <linux/poll.h>
25#include <trace/events/gk20a.h> 27#include <trace/events/gk20a.h>
26#include <uapi/linux/nvgpu.h> 28#include <uapi/linux/nvgpu.h>
27 29
diff --git a/drivers/gpu/nvgpu/gk20a/sched_gk20a.c b/drivers/gpu/nvgpu/gk20a/sched_gk20a.c
index 49a09e99..a58de920 100644
--- a/drivers/gpu/nvgpu/gk20a/sched_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/sched_gk20a.c
@@ -21,6 +21,8 @@
21#include <linux/hashtable.h> 21#include <linux/hashtable.h>
22#include <linux/debugfs.h> 22#include <linux/debugfs.h>
23#include <linux/log2.h> 23#include <linux/log2.h>
24#include <linux/uaccess.h>
25#include <linux/poll.h>
24#include <uapi/linux/nvgpu.h> 26#include <uapi/linux/nvgpu.h>
25 27
26#include <nvgpu/kmem.h> 28#include <nvgpu/kmem.h>
diff --git a/drivers/gpu/nvgpu/gp106/xve_gp106.c b/drivers/gpu/nvgpu/gp106/xve_gp106.c
index ecc6de60..64d60794 100644
--- a/drivers/gpu/nvgpu/gp106/xve_gp106.c
+++ b/drivers/gpu/nvgpu/gp106/xve_gp106.c
@@ -17,6 +17,7 @@
17#include <linux/delay.h> 17#include <linux/delay.h>
18#include <linux/kernel.h> 18#include <linux/kernel.h>
19#include <linux/debugfs.h> 19#include <linux/debugfs.h>
20#include <linux/uaccess.h>
20 21
21#include "gk20a/gk20a.h" 22#include "gk20a/gk20a.h"
22#include "gm206/bios_gm206.h" 23#include "gm206/bios_gm206.h"
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-02 20:59:09 -0400