summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/linux/ioctl_channel.c
diff options
context:
space:
mode:
authorTerje Bergstrom <tbergstrom@nvidia.com>2018-04-18 15:59:00 -0400
committermobile promotions <svcmobile_promotions@nvidia.com>2018-06-15 20:47:31 -0400
commit2a2c16af5f9f1ccfc93a13e820d5381e5c881e92 (patch)
tree2e5d7b042270a649978e5bb540857012c85fb5b5 /drivers/gpu/nvgpu/common/linux/ioctl_channel.c
parent98d996f4ffb0137d119b5849cae46d7b7e5693e1 (diff)
gpu: nvgpu: Move Linux files away from common
Move all Linux source code files to drivers/gpu/nvgpu/os/linux from drivers/gpu/nvgpu/common/linux. This changes the meaning of common to be OS independent. JIRA NVGPU-598 JIRA NVGPU-601 Change-Id: Ib7f2a43d3688bb0d0b7dcc48469a6783fd988ce9 Signed-off-by: Terje Bergstrom <tbergstrom@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/1747714 Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com> Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/common/linux/ioctl_channel.c')
-rw-r--r--drivers/gpu/nvgpu/common/linux/ioctl_channel.c1388
1 files changed, 0 insertions, 1388 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/ioctl_channel.c b/drivers/gpu/nvgpu/common/linux/ioctl_channel.c
deleted file mode 100644
index b04bb9de..00000000
--- a/drivers/gpu/nvgpu/common/linux/ioctl_channel.c
+++ /dev/null
@@ -1,1388 +0,0 @@
1/*
2 * GK20A Graphics channel
3 *
4 * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <trace/events/gk20a.h>
20#include <linux/file.h>
21#include <linux/anon_inodes.h>
22#include <linux/dma-buf.h>
23#include <linux/poll.h>
24#include <uapi/linux/nvgpu.h>
25
26#include <nvgpu/semaphore.h>
27#include <nvgpu/timers.h>
28#include <nvgpu/kmem.h>
29#include <nvgpu/log.h>
30#include <nvgpu/list.h>
31#include <nvgpu/debug.h>
32#include <nvgpu/enabled.h>
33#include <nvgpu/error_notifier.h>
34#include <nvgpu/barrier.h>
35#include <nvgpu/nvhost.h>
36#include <nvgpu/os_sched.h>
37
38#include "gk20a/gk20a.h"
39#include "gk20a/dbg_gpu_gk20a.h"
40#include "gk20a/fence_gk20a.h"
41
42#include "platform_gk20a.h"
43#include "ioctl_channel.h"
44#include "channel.h"
45#include "os_linux.h"
46#include "ctxsw_trace.h"
47
48/* the minimal size of client buffer */
49#define CSS_MIN_CLIENT_SNAPSHOT_SIZE \
50 (sizeof(struct gk20a_cs_snapshot_fifo) + \
51 sizeof(struct gk20a_cs_snapshot_fifo_entry) * 256)
52
53static const char *gr_gk20a_graphics_preempt_mode_name(u32 graphics_preempt_mode)
54{
55 switch (graphics_preempt_mode) {
56 case NVGPU_PREEMPTION_MODE_GRAPHICS_WFI:
57 return "WFI";
58 default:
59 return "?";
60 }
61}
62
63static const char *gr_gk20a_compute_preempt_mode_name(u32 compute_preempt_mode)
64{
65 switch (compute_preempt_mode) {
66 case NVGPU_PREEMPTION_MODE_COMPUTE_WFI:
67 return "WFI";
68 case NVGPU_PREEMPTION_MODE_COMPUTE_CTA:
69 return "CTA";
70 default:
71 return "?";
72 }
73}
74
75static void gk20a_channel_trace_sched_param(
76 void (*trace)(int chid, int tsgid, pid_t pid, u32 timeslice,
77 u32 timeout, const char *interleave,
78 const char *graphics_preempt_mode,
79 const char *compute_preempt_mode),
80 struct channel_gk20a *ch)
81{
82 struct tsg_gk20a *tsg = tsg_gk20a_from_ch(ch);
83
84 if (!tsg)
85 return;
86
87 (trace)(ch->chid, ch->tsgid, ch->pid,
88 tsg_gk20a_from_ch(ch)->timeslice_us,
89 ch->timeout_ms_max,
90 gk20a_fifo_interleave_level_name(tsg->interleave_level),
91 gr_gk20a_graphics_preempt_mode_name(
92 tsg->gr_ctx.graphics_preempt_mode),
93 gr_gk20a_compute_preempt_mode_name(
94 tsg->gr_ctx.compute_preempt_mode));
95}
96
97/*
98 * Although channels do have pointers back to the gk20a struct that they were
99 * created under in cases where the driver is killed that pointer can be bad.
100 * The channel memory can be freed before the release() function for a given
101 * channel is called. This happens when the driver dies and userspace doesn't
102 * get a chance to call release() until after the entire gk20a driver data is
103 * unloaded and freed.
104 */
105struct channel_priv {
106 struct gk20a *g;
107 struct channel_gk20a *c;
108};
109
110#if defined(CONFIG_GK20A_CYCLE_STATS)
111
112void gk20a_channel_free_cycle_stats_buffer(struct channel_gk20a *ch)
113{
114 struct nvgpu_channel_linux *priv = ch->os_priv;
115
116 /* disable existing cyclestats buffer */
117 nvgpu_mutex_acquire(&ch->cyclestate.cyclestate_buffer_mutex);
118 if (priv->cyclestate_buffer_handler) {
119 dma_buf_vunmap(priv->cyclestate_buffer_handler,
120 ch->cyclestate.cyclestate_buffer);
121 dma_buf_put(priv->cyclestate_buffer_handler);
122 priv->cyclestate_buffer_handler = NULL;
123 ch->cyclestate.cyclestate_buffer = NULL;
124 ch->cyclestate.cyclestate_buffer_size = 0;
125 }
126 nvgpu_mutex_release(&ch->cyclestate.cyclestate_buffer_mutex);
127}
128
129static int gk20a_channel_cycle_stats(struct channel_gk20a *ch,
130 struct nvgpu_cycle_stats_args *args)
131{
132 struct dma_buf *dmabuf;
133 void *virtual_address;
134 struct nvgpu_channel_linux *priv = ch->os_priv;
135
136 /* is it allowed to handle calls for current GPU? */
137 if (!nvgpu_is_enabled(ch->g, NVGPU_SUPPORT_CYCLE_STATS))
138 return -ENOSYS;
139
140 if (args->dmabuf_fd && !priv->cyclestate_buffer_handler) {
141
142 /* set up new cyclestats buffer */
143 dmabuf = dma_buf_get(args->dmabuf_fd);
144 if (IS_ERR(dmabuf))
145 return PTR_ERR(dmabuf);
146 virtual_address = dma_buf_vmap(dmabuf);
147 if (!virtual_address)
148 return -ENOMEM;
149
150 priv->cyclestate_buffer_handler = dmabuf;
151 ch->cyclestate.cyclestate_buffer = virtual_address;
152 ch->cyclestate.cyclestate_buffer_size = dmabuf->size;
153 return 0;
154
155 } else if (!args->dmabuf_fd && priv->cyclestate_buffer_handler) {
156 gk20a_channel_free_cycle_stats_buffer(ch);
157 return 0;
158
159 } else if (!args->dmabuf_fd && !priv->cyclestate_buffer_handler) {
160 /* no requst from GL */
161 return 0;
162
163 } else {
164 pr_err("channel already has cyclestats buffer\n");
165 return -EINVAL;
166 }
167}
168
169static int gk20a_flush_cycle_stats_snapshot(struct channel_gk20a *ch)
170{
171 int ret;
172
173 nvgpu_mutex_acquire(&ch->cs_client_mutex);
174 if (ch->cs_client)
175 ret = gr_gk20a_css_flush(ch, ch->cs_client);
176 else
177 ret = -EBADF;
178 nvgpu_mutex_release(&ch->cs_client_mutex);
179
180 return ret;
181}
182
183static int gk20a_attach_cycle_stats_snapshot(struct channel_gk20a *ch,
184 u32 dmabuf_fd,
185 u32 perfmon_id_count,
186 u32 *perfmon_id_start)
187{
188 int ret = 0;
189 struct gk20a *g = ch->g;
190 struct gk20a_cs_snapshot_client_linux *client_linux;
191 struct gk20a_cs_snapshot_client *client;
192
193 nvgpu_mutex_acquire(&ch->cs_client_mutex);
194 if (ch->cs_client) {
195 nvgpu_mutex_release(&ch->cs_client_mutex);
196 return -EEXIST;
197 }
198
199 client_linux = nvgpu_kzalloc(g, sizeof(*client_linux));
200 if (!client_linux) {
201 ret = -ENOMEM;
202 goto err;
203 }
204
205 client_linux->dmabuf_fd = dmabuf_fd;
206 client_linux->dma_handler = dma_buf_get(client_linux->dmabuf_fd);
207 if (IS_ERR(client_linux->dma_handler)) {
208 ret = PTR_ERR(client_linux->dma_handler);
209 client_linux->dma_handler = NULL;
210 goto err_free;
211 }
212
213 client = &client_linux->cs_client;
214 client->snapshot_size = client_linux->dma_handler->size;
215 if (client->snapshot_size < CSS_MIN_CLIENT_SNAPSHOT_SIZE) {
216 ret = -ENOMEM;
217 goto err_put;
218 }
219
220 client->snapshot = (struct gk20a_cs_snapshot_fifo *)
221 dma_buf_vmap(client_linux->dma_handler);
222 if (!client->snapshot) {
223 ret = -ENOMEM;
224 goto err_put;
225 }
226
227 ch->cs_client = client;
228
229 ret = gr_gk20a_css_attach(ch,
230 perfmon_id_count,
231 perfmon_id_start,
232 ch->cs_client);
233
234 nvgpu_mutex_release(&ch->cs_client_mutex);
235
236 return ret;
237
238err_put:
239 dma_buf_put(client_linux->dma_handler);
240err_free:
241 nvgpu_kfree(g, client_linux);
242err:
243 nvgpu_mutex_release(&ch->cs_client_mutex);
244 return ret;
245}
246
247int gk20a_channel_free_cycle_stats_snapshot(struct channel_gk20a *ch)
248{
249 int ret;
250 struct gk20a_cs_snapshot_client_linux *client_linux;
251
252 nvgpu_mutex_acquire(&ch->cs_client_mutex);
253 if (!ch->cs_client) {
254 nvgpu_mutex_release(&ch->cs_client_mutex);
255 return 0;
256 }
257
258 client_linux = container_of(ch->cs_client,
259 struct gk20a_cs_snapshot_client_linux,
260 cs_client);
261
262 ret = gr_gk20a_css_detach(ch, ch->cs_client);
263
264 if (client_linux->dma_handler) {
265 if (ch->cs_client->snapshot)
266 dma_buf_vunmap(client_linux->dma_handler,
267 ch->cs_client->snapshot);
268 dma_buf_put(client_linux->dma_handler);
269 }
270
271 ch->cs_client = NULL;
272 nvgpu_kfree(ch->g, client_linux);
273
274 nvgpu_mutex_release(&ch->cs_client_mutex);
275
276 return ret;
277}
278
279static int gk20a_channel_cycle_stats_snapshot(struct channel_gk20a *ch,
280 struct nvgpu_cycle_stats_snapshot_args *args)
281{
282 int ret;
283
284 /* is it allowed to handle calls for current GPU? */
285 if (!nvgpu_is_enabled(ch->g, NVGPU_SUPPORT_CYCLE_STATS_SNAPSHOT))
286 return -ENOSYS;
287
288 if (!args->dmabuf_fd)
289 return -EINVAL;
290
291 /* handle the command (most frequent cases first) */
292 switch (args->cmd) {
293 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_FLUSH:
294 ret = gk20a_flush_cycle_stats_snapshot(ch);
295 args->extra = 0;
296 break;
297
298 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_ATTACH:
299 ret = gk20a_attach_cycle_stats_snapshot(ch,
300 args->dmabuf_fd,
301 args->extra,
302 &args->extra);
303 break;
304
305 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_DETACH:
306 ret = gk20a_channel_free_cycle_stats_snapshot(ch);
307 args->extra = 0;
308 break;
309
310 default:
311 pr_err("cyclestats: unknown command %u\n", args->cmd);
312 ret = -EINVAL;
313 break;
314 }
315
316 return ret;
317}
318#endif
319
320static int gk20a_channel_set_wdt_status(struct channel_gk20a *ch,
321 struct nvgpu_channel_wdt_args *args)
322{
323 u32 status = args->wdt_status & (NVGPU_IOCTL_CHANNEL_DISABLE_WDT |
324 NVGPU_IOCTL_CHANNEL_ENABLE_WDT);
325
326 if (status == NVGPU_IOCTL_CHANNEL_DISABLE_WDT)
327 ch->timeout.enabled = false;
328 else if (status == NVGPU_IOCTL_CHANNEL_ENABLE_WDT)
329 ch->timeout.enabled = true;
330 else
331 return -EINVAL;
332
333 if (args->wdt_status & NVGPU_IOCTL_CHANNEL_WDT_FLAG_SET_TIMEOUT)
334 ch->timeout.limit_ms = args->timeout_ms;
335
336 ch->timeout.debug_dump = (args->wdt_status &
337 NVGPU_IOCTL_CHANNEL_WDT_FLAG_DISABLE_DUMP) == 0;
338
339 return 0;
340}
341
342static void gk20a_channel_free_error_notifiers(struct channel_gk20a *ch)
343{
344 struct nvgpu_channel_linux *priv = ch->os_priv;
345
346 nvgpu_mutex_acquire(&priv->error_notifier.mutex);
347 if (priv->error_notifier.dmabuf) {
348 dma_buf_vunmap(priv->error_notifier.dmabuf, priv->error_notifier.vaddr);
349 dma_buf_put(priv->error_notifier.dmabuf);
350 priv->error_notifier.dmabuf = NULL;
351 priv->error_notifier.notification = NULL;
352 priv->error_notifier.vaddr = NULL;
353 }
354 nvgpu_mutex_release(&priv->error_notifier.mutex);
355}
356
357static int gk20a_init_error_notifier(struct channel_gk20a *ch,
358 struct nvgpu_set_error_notifier *args)
359{
360 struct dma_buf *dmabuf;
361 void *va;
362 u64 end = args->offset + sizeof(struct nvgpu_notification);
363 struct nvgpu_channel_linux *priv = ch->os_priv;
364
365 if (!args->mem) {
366 pr_err("gk20a_init_error_notifier: invalid memory handle\n");
367 return -EINVAL;
368 }
369
370 dmabuf = dma_buf_get(args->mem);
371
372 gk20a_channel_free_error_notifiers(ch);
373
374 if (IS_ERR(dmabuf)) {
375 pr_err("Invalid handle: %d\n", args->mem);
376 return -EINVAL;
377 }
378
379 if (end > dmabuf->size || end < sizeof(struct nvgpu_notification)) {
380 dma_buf_put(dmabuf);
381 nvgpu_err(ch->g, "gk20a_init_error_notifier: invalid offset");
382 return -EINVAL;
383 }
384
385 nvgpu_speculation_barrier();
386
387 /* map handle */
388 va = dma_buf_vmap(dmabuf);
389 if (!va) {
390 dma_buf_put(dmabuf);
391 pr_err("Cannot map notifier handle\n");
392 return -ENOMEM;
393 }
394
395 priv->error_notifier.notification = va + args->offset;
396 priv->error_notifier.vaddr = va;
397 memset(priv->error_notifier.notification, 0,
398 sizeof(struct nvgpu_notification));
399
400 /* set channel notifiers pointer */
401 nvgpu_mutex_acquire(&priv->error_notifier.mutex);
402 priv->error_notifier.dmabuf = dmabuf;
403 nvgpu_mutex_release(&priv->error_notifier.mutex);
404
405 return 0;
406}
407
408/*
409 * This returns the channel with a reference. The caller must
410 * gk20a_channel_put() the ref back after use.
411 *
412 * NULL is returned if the channel was not found.
413 */
414struct channel_gk20a *gk20a_get_channel_from_file(int fd)
415{
416 struct channel_gk20a *ch;
417 struct channel_priv *priv;
418 struct file *f = fget(fd);
419
420 if (!f)
421 return NULL;
422
423 if (f->f_op != &gk20a_channel_ops) {
424 fput(f);
425 return NULL;
426 }
427
428 priv = (struct channel_priv *)f->private_data;
429 ch = gk20a_channel_get(priv->c);
430 fput(f);
431 return ch;
432}
433
434int gk20a_channel_release(struct inode *inode, struct file *filp)
435{
436 struct channel_priv *priv = filp->private_data;
437 struct channel_gk20a *ch;
438 struct gk20a *g;
439
440 int err;
441
442 /* We could still end up here even if the channel_open failed, e.g.
443 * if we ran out of hw channel IDs.
444 */
445 if (!priv)
446 return 0;
447
448 ch = priv->c;
449 g = priv->g;
450
451 err = gk20a_busy(g);
452 if (err) {
453 nvgpu_err(g, "failed to release a channel!");
454 goto channel_release;
455 }
456
457 trace_gk20a_channel_release(dev_name(dev_from_gk20a(g)));
458
459 gk20a_channel_close(ch);
460 gk20a_channel_free_error_notifiers(ch);
461
462 gk20a_idle(g);
463
464channel_release:
465 gk20a_put(g);
466 nvgpu_kfree(g, filp->private_data);
467 filp->private_data = NULL;
468 return 0;
469}
470
471/* note: runlist_id -1 is synonym for the ENGINE_GR_GK20A runlist id */
472static int __gk20a_channel_open(struct gk20a *g,
473 struct file *filp, s32 runlist_id)
474{
475 int err;
476 struct channel_gk20a *ch;
477 struct channel_priv *priv;
478
479 nvgpu_log_fn(g, " ");
480
481 g = gk20a_get(g);
482 if (!g)
483 return -ENODEV;
484
485 trace_gk20a_channel_open(dev_name(dev_from_gk20a(g)));
486
487 priv = nvgpu_kzalloc(g, sizeof(*priv));
488 if (!priv) {
489 err = -ENOMEM;
490 goto free_ref;
491 }
492
493 err = gk20a_busy(g);
494 if (err) {
495 nvgpu_err(g, "failed to power on, %d", err);
496 goto fail_busy;
497 }
498 /* All the user space channel should be non privilege */
499 ch = gk20a_open_new_channel(g, runlist_id, false,
500 nvgpu_current_pid(g), nvgpu_current_tid(g));
501 gk20a_idle(g);
502 if (!ch) {
503 nvgpu_err(g,
504 "failed to get f");
505 err = -ENOMEM;
506 goto fail_busy;
507 }
508
509 gk20a_channel_trace_sched_param(
510 trace_gk20a_channel_sched_defaults, ch);
511
512 priv->g = g;
513 priv->c = ch;
514
515 filp->private_data = priv;
516 return 0;
517
518fail_busy:
519 nvgpu_kfree(g, priv);
520free_ref:
521 gk20a_put(g);
522 return err;
523}
524
525int gk20a_channel_open(struct inode *inode, struct file *filp)
526{
527 struct nvgpu_os_linux *l = container_of(inode->i_cdev,
528 struct nvgpu_os_linux, channel.cdev);
529 struct gk20a *g = &l->g;
530 int ret;
531
532 nvgpu_log_fn(g, "start");
533 ret = __gk20a_channel_open(g, filp, -1);
534
535 nvgpu_log_fn(g, "end");
536 return ret;
537}
538
539int gk20a_channel_open_ioctl(struct gk20a *g,
540 struct nvgpu_channel_open_args *args)
541{
542 int err;
543 int fd;
544 struct file *file;
545 char name[64];
546 s32 runlist_id = args->in.runlist_id;
547 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
548
549 err = get_unused_fd_flags(O_RDWR);
550 if (err < 0)
551 return err;
552 fd = err;
553
554 snprintf(name, sizeof(name), "nvhost-%s-fd%d",
555 dev_name(dev_from_gk20a(g)), fd);
556
557 file = anon_inode_getfile(name, l->channel.cdev.ops, NULL, O_RDWR);
558 if (IS_ERR(file)) {
559 err = PTR_ERR(file);
560 goto clean_up;
561 }
562
563 err = __gk20a_channel_open(g, file, runlist_id);
564 if (err)
565 goto clean_up_file;
566
567 fd_install(fd, file);
568 args->out.channel_fd = fd;
569 return 0;
570
571clean_up_file:
572 fput(file);
573clean_up:
574 put_unused_fd(fd);
575 return err;
576}
577
578static u32 nvgpu_gpfifo_user_flags_to_common_flags(u32 user_flags)
579{
580 u32 flags = 0;
581
582 if (user_flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_VPR_ENABLED)
583 flags |= NVGPU_GPFIFO_FLAGS_SUPPORT_VPR;
584
585 if (user_flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_DETERMINISTIC)
586 flags |= NVGPU_GPFIFO_FLAGS_SUPPORT_DETERMINISTIC;
587
588 if (user_flags & NVGPU_ALLOC_GPFIFO_FLAGS_REPLAYABLE_FAULTS_ENABLE)
589 flags |= NVGPU_GPFIFO_FLAGS_REPLAYABLE_FAULTS_ENABLE;
590
591 return flags;
592}
593
594static void nvgpu_get_gpfifo_ex_args(
595 struct nvgpu_alloc_gpfifo_ex_args *alloc_gpfifo_ex_args,
596 struct nvgpu_gpfifo_args *gpfifo_args)
597{
598 gpfifo_args->num_entries = alloc_gpfifo_ex_args->num_entries;
599 gpfifo_args->num_inflight_jobs = alloc_gpfifo_ex_args->num_inflight_jobs;
600 gpfifo_args->flags = nvgpu_gpfifo_user_flags_to_common_flags(
601 alloc_gpfifo_ex_args->flags);
602}
603
604static void nvgpu_get_gpfifo_args(
605 struct nvgpu_alloc_gpfifo_args *alloc_gpfifo_args,
606 struct nvgpu_gpfifo_args *gpfifo_args)
607{
608 /*
609 * Kernel can insert one extra gpfifo entry before user
610 * submitted gpfifos and another one after, for internal usage.
611 * Triple the requested size.
612 */
613 gpfifo_args->num_entries = alloc_gpfifo_args->num_entries * 3;
614 gpfifo_args->num_inflight_jobs = 0;
615 gpfifo_args->flags = nvgpu_gpfifo_user_flags_to_common_flags(
616 alloc_gpfifo_args->flags);
617}
618
619static void nvgpu_get_fence_args(
620 struct nvgpu_fence *fence_args_in,
621 struct nvgpu_channel_fence *fence_args_out)
622{
623 fence_args_out->id = fence_args_in->id;
624 fence_args_out->value = fence_args_in->value;
625}
626
627static int gk20a_channel_wait_semaphore(struct channel_gk20a *ch,
628 ulong id, u32 offset,
629 u32 payload, u32 timeout)
630{
631 struct dma_buf *dmabuf;
632 void *data;
633 u32 *semaphore;
634 int ret = 0;
635
636 /* do not wait if channel has timed out */
637 if (ch->has_timedout)
638 return -ETIMEDOUT;
639
640 dmabuf = dma_buf_get(id);
641 if (IS_ERR(dmabuf)) {
642 nvgpu_err(ch->g, "invalid notifier nvmap handle 0x%lx", id);
643 return -EINVAL;
644 }
645
646 data = dma_buf_kmap(dmabuf, offset >> PAGE_SHIFT);
647 if (!data) {
648 nvgpu_err(ch->g, "failed to map notifier memory");
649 ret = -EINVAL;
650 goto cleanup_put;
651 }
652
653 semaphore = data + (offset & ~PAGE_MASK);
654
655 ret = NVGPU_COND_WAIT_INTERRUPTIBLE(
656 &ch->semaphore_wq,
657 *semaphore == payload || ch->has_timedout,
658 timeout);
659
660 dma_buf_kunmap(dmabuf, offset >> PAGE_SHIFT, data);
661cleanup_put:
662 dma_buf_put(dmabuf);
663 return ret;
664}
665
666static int gk20a_channel_wait(struct channel_gk20a *ch,
667 struct nvgpu_wait_args *args)
668{
669 struct dma_buf *dmabuf;
670 struct gk20a *g = ch->g;
671 struct notification *notif;
672 struct timespec tv;
673 u64 jiffies;
674 ulong id;
675 u32 offset;
676 int remain, ret = 0;
677 u64 end;
678
679 nvgpu_log_fn(g, " ");
680
681 if (ch->has_timedout)
682 return -ETIMEDOUT;
683
684 switch (args->type) {
685 case NVGPU_WAIT_TYPE_NOTIFIER:
686 id = args->condition.notifier.dmabuf_fd;
687 offset = args->condition.notifier.offset;
688 end = offset + sizeof(struct notification);
689
690 dmabuf = dma_buf_get(id);
691 if (IS_ERR(dmabuf)) {
692 nvgpu_err(g, "invalid notifier nvmap handle 0x%lx",
693 id);
694 return -EINVAL;
695 }
696
697 if (end > dmabuf->size || end < sizeof(struct notification)) {
698 dma_buf_put(dmabuf);
699 nvgpu_err(g, "invalid notifier offset");
700 return -EINVAL;
701 }
702
703 nvgpu_speculation_barrier();
704
705 notif = dma_buf_vmap(dmabuf);
706 if (!notif) {
707 nvgpu_err(g, "failed to map notifier memory");
708 return -ENOMEM;
709 }
710
711 notif = (struct notification *)((uintptr_t)notif + offset);
712
713 /* user should set status pending before
714 * calling this ioctl */
715 remain = NVGPU_COND_WAIT_INTERRUPTIBLE(
716 &ch->notifier_wq,
717 notif->status == 0 || ch->has_timedout,
718 args->timeout);
719
720 if (remain == 0 && notif->status != 0) {
721 ret = -ETIMEDOUT;
722 goto notif_clean_up;
723 } else if (remain < 0) {
724 ret = -EINTR;
725 goto notif_clean_up;
726 }
727
728 /* TBD: fill in correct information */
729 jiffies = get_jiffies_64();
730 jiffies_to_timespec(jiffies, &tv);
731 notif->timestamp.nanoseconds[0] = tv.tv_nsec;
732 notif->timestamp.nanoseconds[1] = tv.tv_sec;
733 notif->info32 = 0xDEADBEEF; /* should be object name */
734 notif->info16 = ch->chid; /* should be method offset */
735
736notif_clean_up:
737 dma_buf_vunmap(dmabuf, notif);
738 return ret;
739
740 case NVGPU_WAIT_TYPE_SEMAPHORE:
741 ret = gk20a_channel_wait_semaphore(ch,
742 args->condition.semaphore.dmabuf_fd,
743 args->condition.semaphore.offset,
744 args->condition.semaphore.payload,
745 args->timeout);
746
747 break;
748
749 default:
750 ret = -EINVAL;
751 break;
752 }
753
754 return ret;
755}
756
757static int gk20a_channel_zcull_bind(struct channel_gk20a *ch,
758 struct nvgpu_zcull_bind_args *args)
759{
760 struct gk20a *g = ch->g;
761 struct gr_gk20a *gr = &g->gr;
762
763 nvgpu_log_fn(gr->g, " ");
764
765 return g->ops.gr.bind_ctxsw_zcull(g, gr, ch,
766 args->gpu_va, args->mode);
767}
768
769static int gk20a_ioctl_channel_submit_gpfifo(
770 struct channel_gk20a *ch,
771 struct nvgpu_submit_gpfifo_args *args)
772{
773 struct nvgpu_channel_fence fence;
774 struct gk20a_fence *fence_out;
775 struct fifo_profile_gk20a *profile = NULL;
776 u32 submit_flags = 0;
777 int fd = -1;
778 struct gk20a *g = ch->g;
779
780 int ret = 0;
781 nvgpu_log_fn(g, " ");
782
783 profile = gk20a_fifo_profile_acquire(ch->g);
784 gk20a_fifo_profile_snapshot(profile, PROFILE_IOCTL_ENTRY);
785
786 if (ch->has_timedout)
787 return -ETIMEDOUT;
788
789 nvgpu_get_fence_args(&args->fence, &fence);
790 submit_flags =
791 nvgpu_submit_gpfifo_user_flags_to_common_flags(args->flags);
792
793 /* Try and allocate an fd here*/
794 if ((args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET)
795 && (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE)) {
796 fd = get_unused_fd_flags(O_RDWR);
797 if (fd < 0)
798 return fd;
799 }
800
801 ret = gk20a_submit_channel_gpfifo(ch, NULL, args, args->num_entries,
802 submit_flags, &fence,
803 &fence_out, profile);
804
805 if (ret) {
806 if (fd != -1)
807 put_unused_fd(fd);
808 goto clean_up;
809 }
810
811 /* Convert fence_out to something we can pass back to user space. */
812 if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) {
813 if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE) {
814 ret = gk20a_fence_install_fd(fence_out, fd);
815 if (ret)
816 put_unused_fd(fd);
817 else
818 args->fence.id = fd;
819 } else {
820 args->fence.id = fence_out->syncpt_id;
821 args->fence.value = fence_out->syncpt_value;
822 }
823 }
824 gk20a_fence_put(fence_out);
825
826 gk20a_fifo_profile_snapshot(profile, PROFILE_IOCTL_EXIT);
827 if (profile)
828 gk20a_fifo_profile_release(ch->g, profile);
829
830clean_up:
831 return ret;
832}
833
834/*
835 * Convert linux specific runlist level of the form NVGPU_RUNLIST_INTERLEAVE_LEVEL_*
836 * to common runlist level of the form NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_*
837 */
838u32 nvgpu_get_common_runlist_level(u32 level)
839{
840 switch (level) {
841 case NVGPU_RUNLIST_INTERLEAVE_LEVEL_LOW:
842 return NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW;
843 case NVGPU_RUNLIST_INTERLEAVE_LEVEL_MEDIUM:
844 return NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM;
845 case NVGPU_RUNLIST_INTERLEAVE_LEVEL_HIGH:
846 return NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH;
847 default:
848 pr_err("%s: incorrect runlist level\n", __func__);
849 }
850
851 return level;
852}
853
854static u32 nvgpu_obj_ctx_user_flags_to_common_flags(u32 user_flags)
855{
856 u32 flags = 0;
857
858 if (user_flags & NVGPU_ALLOC_OBJ_FLAGS_GFXP)
859 flags |= NVGPU_OBJ_CTX_FLAGS_SUPPORT_GFXP;
860
861 if (user_flags & NVGPU_ALLOC_OBJ_FLAGS_CILP)
862 flags |= NVGPU_OBJ_CTX_FLAGS_SUPPORT_CILP;
863
864 return flags;
865}
866
867static int nvgpu_ioctl_channel_alloc_obj_ctx(struct channel_gk20a *ch,
868 u32 class_num, u32 user_flags)
869{
870 return ch->g->ops.gr.alloc_obj_ctx(ch, class_num,
871 nvgpu_obj_ctx_user_flags_to_common_flags(user_flags));
872}
873
874/*
875 * Convert common preemption mode flags of the form NVGPU_PREEMPTION_MODE_GRAPHICS_*
876 * into linux preemption mode flags of the form NVGPU_GRAPHICS_PREEMPTION_MODE_*
877 */
878u32 nvgpu_get_ioctl_graphics_preempt_mode_flags(u32 graphics_preempt_mode_flags)
879{
880 u32 flags = 0;
881
882 if (graphics_preempt_mode_flags & NVGPU_PREEMPTION_MODE_GRAPHICS_WFI)
883 flags |= NVGPU_GRAPHICS_PREEMPTION_MODE_WFI;
884 if (graphics_preempt_mode_flags & NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP)
885 flags |= NVGPU_GRAPHICS_PREEMPTION_MODE_GFXP;
886
887 return flags;
888}
889
890/*
891 * Convert common preemption mode flags of the form NVGPU_PREEMPTION_MODE_COMPUTE_*
892 * into linux preemption mode flags of the form NVGPU_COMPUTE_PREEMPTION_MODE_*
893 */
894u32 nvgpu_get_ioctl_compute_preempt_mode_flags(u32 compute_preempt_mode_flags)
895{
896 u32 flags = 0;
897
898 if (compute_preempt_mode_flags & NVGPU_PREEMPTION_MODE_COMPUTE_WFI)
899 flags |= NVGPU_COMPUTE_PREEMPTION_MODE_WFI;
900 if (compute_preempt_mode_flags & NVGPU_PREEMPTION_MODE_COMPUTE_CTA)
901 flags |= NVGPU_COMPUTE_PREEMPTION_MODE_CTA;
902 if (compute_preempt_mode_flags & NVGPU_PREEMPTION_MODE_COMPUTE_CILP)
903 flags |= NVGPU_COMPUTE_PREEMPTION_MODE_CILP;
904
905 return flags;
906}
907
908/*
909 * Convert common preemption modes of the form NVGPU_PREEMPTION_MODE_GRAPHICS_*
910 * into linux preemption modes of the form NVGPU_GRAPHICS_PREEMPTION_MODE_*
911 */
912u32 nvgpu_get_ioctl_graphics_preempt_mode(u32 graphics_preempt_mode)
913{
914 switch (graphics_preempt_mode) {
915 case NVGPU_PREEMPTION_MODE_GRAPHICS_WFI:
916 return NVGPU_GRAPHICS_PREEMPTION_MODE_WFI;
917 case NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP:
918 return NVGPU_GRAPHICS_PREEMPTION_MODE_GFXP;
919 }
920
921 return graphics_preempt_mode;
922}
923
924/*
925 * Convert common preemption modes of the form NVGPU_PREEMPTION_MODE_COMPUTE_*
926 * into linux preemption modes of the form NVGPU_COMPUTE_PREEMPTION_MODE_*
927 */
928u32 nvgpu_get_ioctl_compute_preempt_mode(u32 compute_preempt_mode)
929{
930 switch (compute_preempt_mode) {
931 case NVGPU_PREEMPTION_MODE_COMPUTE_WFI:
932 return NVGPU_COMPUTE_PREEMPTION_MODE_WFI;
933 case NVGPU_PREEMPTION_MODE_COMPUTE_CTA:
934 return NVGPU_COMPUTE_PREEMPTION_MODE_CTA;
935 case NVGPU_PREEMPTION_MODE_COMPUTE_CILP:
936 return NVGPU_COMPUTE_PREEMPTION_MODE_CILP;
937 }
938
939 return compute_preempt_mode;
940}
941
942/*
943 * Convert linux preemption modes of the form NVGPU_GRAPHICS_PREEMPTION_MODE_*
944 * into common preemption modes of the form NVGPU_PREEMPTION_MODE_GRAPHICS_*
945 */
946static u32 nvgpu_get_common_graphics_preempt_mode(u32 graphics_preempt_mode)
947{
948 switch (graphics_preempt_mode) {
949 case NVGPU_GRAPHICS_PREEMPTION_MODE_WFI:
950 return NVGPU_PREEMPTION_MODE_GRAPHICS_WFI;
951 case NVGPU_GRAPHICS_PREEMPTION_MODE_GFXP:
952 return NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP;
953 }
954
955 return graphics_preempt_mode;
956}
957
958/*
959 * Convert linux preemption modes of the form NVGPU_COMPUTE_PREEMPTION_MODE_*
960 * into common preemption modes of the form NVGPU_PREEMPTION_MODE_COMPUTE_*
961 */
962static u32 nvgpu_get_common_compute_preempt_mode(u32 compute_preempt_mode)
963{
964 switch (compute_preempt_mode) {
965 case NVGPU_COMPUTE_PREEMPTION_MODE_WFI:
966 return NVGPU_PREEMPTION_MODE_COMPUTE_WFI;
967 case NVGPU_COMPUTE_PREEMPTION_MODE_CTA:
968 return NVGPU_PREEMPTION_MODE_COMPUTE_CTA;
969 case NVGPU_COMPUTE_PREEMPTION_MODE_CILP:
970 return NVGPU_PREEMPTION_MODE_COMPUTE_CILP;
971 }
972
973 return compute_preempt_mode;
974}
975
976static int nvgpu_ioctl_channel_set_preemption_mode(struct channel_gk20a *ch,
977 u32 graphics_preempt_mode, u32 compute_preempt_mode)
978{
979 int err;
980
981 if (ch->g->ops.gr.set_preemption_mode) {
982 err = gk20a_busy(ch->g);
983 if (err) {
984 nvgpu_err(ch->g, "failed to power on, %d", err);
985 return err;
986 }
987 err = ch->g->ops.gr.set_preemption_mode(ch,
988 nvgpu_get_common_graphics_preempt_mode(graphics_preempt_mode),
989 nvgpu_get_common_compute_preempt_mode(compute_preempt_mode));
990 gk20a_idle(ch->g);
991 } else {
992 err = -EINVAL;
993 }
994
995 return err;
996}
997
998static int nvgpu_ioctl_channel_get_user_syncpoint(struct channel_gk20a *ch,
999 struct nvgpu_get_user_syncpoint_args *args)
1000{
1001#ifdef CONFIG_TEGRA_GK20A_NVHOST
1002 struct gk20a *g = ch->g;
1003 int err;
1004
1005 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_USER_SYNCPOINT)) {
1006 nvgpu_err(g, "user syncpoints not supported");
1007 return -EINVAL;
1008 }
1009
1010 if (!gk20a_platform_has_syncpoints(g)) {
1011 nvgpu_err(g, "syncpoints not supported");
1012 return -EINVAL;
1013 }
1014
1015 if (g->aggressive_sync_destroy_thresh) {
1016 nvgpu_err(g, "sufficient syncpoints not available");
1017 return -EINVAL;
1018 }
1019
1020 nvgpu_mutex_acquire(&ch->sync_lock);
1021 if (ch->user_sync) {
1022 nvgpu_mutex_release(&ch->sync_lock);
1023 } else {
1024 ch->user_sync = gk20a_channel_sync_create(ch, true);
1025 if (!ch->user_sync) {
1026 nvgpu_mutex_release(&ch->sync_lock);
1027 return -ENOMEM;
1028 }
1029 nvgpu_mutex_release(&ch->sync_lock);
1030
1031 if (g->ops.fifo.resetup_ramfc) {
1032 err = g->ops.fifo.resetup_ramfc(ch);
1033 if (err)
1034 return err;
1035 }
1036 }
1037
1038 args->syncpoint_id = ch->user_sync->syncpt_id(ch->user_sync);
1039 args->syncpoint_max = nvgpu_nvhost_syncpt_read_maxval(g->nvhost_dev,
1040 args->syncpoint_id);
1041 if (nvgpu_is_enabled(g, NVGPU_SUPPORT_SYNCPOINT_ADDRESS))
1042 args->gpu_va = ch->user_sync->syncpt_address(ch->user_sync);
1043 else
1044 args->gpu_va = 0;
1045
1046 return 0;
1047#else
1048 return -EINVAL;
1049#endif
1050}
1051
1052long gk20a_channel_ioctl(struct file *filp,
1053 unsigned int cmd, unsigned long arg)
1054{
1055 struct channel_priv *priv = filp->private_data;
1056 struct channel_gk20a *ch = priv->c;
1057 struct device *dev = dev_from_gk20a(ch->g);
1058 u8 buf[NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE] = {0};
1059 int err = 0;
1060 struct gk20a *g = ch->g;
1061
1062 nvgpu_log_fn(g, "start %d", _IOC_NR(cmd));
1063
1064 if ((_IOC_TYPE(cmd) != NVGPU_IOCTL_MAGIC) ||
1065 (_IOC_NR(cmd) == 0) ||
1066 (_IOC_NR(cmd) > NVGPU_IOCTL_CHANNEL_LAST) ||
1067 (_IOC_SIZE(cmd) > NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE))
1068 return -EINVAL;
1069
1070 if (_IOC_DIR(cmd) & _IOC_WRITE) {
1071 if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
1072 return -EFAULT;
1073 }
1074
1075 /* take a ref or return timeout if channel refs can't be taken */
1076 ch = gk20a_channel_get(ch);
1077 if (!ch)
1078 return -ETIMEDOUT;
1079
1080 /* protect our sanity for threaded userspace - most of the channel is
1081 * not thread safe */
1082 nvgpu_mutex_acquire(&ch->ioctl_lock);
1083
1084 /* this ioctl call keeps a ref to the file which keeps a ref to the
1085 * channel */
1086
1087 switch (cmd) {
1088 case NVGPU_IOCTL_CHANNEL_OPEN:
1089 err = gk20a_channel_open_ioctl(ch->g,
1090 (struct nvgpu_channel_open_args *)buf);
1091 break;
1092 case NVGPU_IOCTL_CHANNEL_SET_NVMAP_FD:
1093 break;
1094 case NVGPU_IOCTL_CHANNEL_ALLOC_OBJ_CTX:
1095 {
1096 struct nvgpu_alloc_obj_ctx_args *args =
1097 (struct nvgpu_alloc_obj_ctx_args *)buf;
1098
1099 err = gk20a_busy(ch->g);
1100 if (err) {
1101 dev_err(dev,
1102 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1103 __func__, cmd);
1104 break;
1105 }
1106 err = nvgpu_ioctl_channel_alloc_obj_ctx(ch, args->class_num, args->flags);
1107 gk20a_idle(ch->g);
1108 break;
1109 }
1110 case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO_EX:
1111 {
1112 struct nvgpu_alloc_gpfifo_ex_args *alloc_gpfifo_ex_args =
1113 (struct nvgpu_alloc_gpfifo_ex_args *)buf;
1114 struct nvgpu_gpfifo_args gpfifo_args;
1115
1116 nvgpu_get_gpfifo_ex_args(alloc_gpfifo_ex_args, &gpfifo_args);
1117
1118 err = gk20a_busy(ch->g);
1119 if (err) {
1120 dev_err(dev,
1121 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1122 __func__, cmd);
1123 break;
1124 }
1125
1126 if (!is_power_of_2(alloc_gpfifo_ex_args->num_entries)) {
1127 err = -EINVAL;
1128 gk20a_idle(ch->g);
1129 break;
1130 }
1131 err = gk20a_channel_alloc_gpfifo(ch, &gpfifo_args);
1132 gk20a_idle(ch->g);
1133 break;
1134 }
1135 case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO:
1136 {
1137 struct nvgpu_alloc_gpfifo_args *alloc_gpfifo_args =
1138 (struct nvgpu_alloc_gpfifo_args *)buf;
1139 struct nvgpu_gpfifo_args gpfifo_args;
1140
1141 nvgpu_get_gpfifo_args(alloc_gpfifo_args, &gpfifo_args);
1142
1143 err = gk20a_busy(ch->g);
1144 if (err) {
1145 dev_err(dev,
1146 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1147 __func__, cmd);
1148 break;
1149 }
1150
1151 err = gk20a_channel_alloc_gpfifo(ch, &gpfifo_args);
1152 gk20a_idle(ch->g);
1153 break;
1154 }
1155 case NVGPU_IOCTL_CHANNEL_SUBMIT_GPFIFO:
1156 err = gk20a_ioctl_channel_submit_gpfifo(ch,
1157 (struct nvgpu_submit_gpfifo_args *)buf);
1158 break;
1159 case NVGPU_IOCTL_CHANNEL_WAIT:
1160 err = gk20a_busy(ch->g);
1161 if (err) {
1162 dev_err(dev,
1163 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1164 __func__, cmd);
1165 break;
1166 }
1167
1168 /* waiting is thread-safe, not dropping this mutex could
1169 * deadlock in certain conditions */
1170 nvgpu_mutex_release(&ch->ioctl_lock);
1171
1172 err = gk20a_channel_wait(ch,
1173 (struct nvgpu_wait_args *)buf);
1174
1175 nvgpu_mutex_acquire(&ch->ioctl_lock);
1176
1177 gk20a_idle(ch->g);
1178 break;
1179 case NVGPU_IOCTL_CHANNEL_ZCULL_BIND:
1180 err = gk20a_busy(ch->g);
1181 if (err) {
1182 dev_err(dev,
1183 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1184 __func__, cmd);
1185 break;
1186 }
1187 err = gk20a_channel_zcull_bind(ch,
1188 (struct nvgpu_zcull_bind_args *)buf);
1189 gk20a_idle(ch->g);
1190 break;
1191 case NVGPU_IOCTL_CHANNEL_SET_ERROR_NOTIFIER:
1192 err = gk20a_busy(ch->g);
1193 if (err) {
1194 dev_err(dev,
1195 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1196 __func__, cmd);
1197 break;
1198 }
1199 err = gk20a_init_error_notifier(ch,
1200 (struct nvgpu_set_error_notifier *)buf);
1201 gk20a_idle(ch->g);
1202 break;
1203#ifdef CONFIG_GK20A_CYCLE_STATS
1204 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS:
1205 err = gk20a_busy(ch->g);
1206 if (err) {
1207 dev_err(dev,
1208 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1209 __func__, cmd);
1210 break;
1211 }
1212 err = gk20a_channel_cycle_stats(ch,
1213 (struct nvgpu_cycle_stats_args *)buf);
1214 gk20a_idle(ch->g);
1215 break;
1216#endif
1217 case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT:
1218 {
1219 u32 timeout =
1220 (u32)((struct nvgpu_set_timeout_args *)buf)->timeout;
1221 nvgpu_log(g, gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
1222 timeout, ch->chid);
1223 ch->timeout_ms_max = timeout;
1224 gk20a_channel_trace_sched_param(
1225 trace_gk20a_channel_set_timeout, ch);
1226 break;
1227 }
1228 case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT_EX:
1229 {
1230 u32 timeout =
1231 (u32)((struct nvgpu_set_timeout_args *)buf)->timeout;
1232 bool timeout_debug_dump = !((u32)
1233 ((struct nvgpu_set_timeout_ex_args *)buf)->flags &
1234 (1 << NVGPU_TIMEOUT_FLAG_DISABLE_DUMP));
1235 nvgpu_log(g, gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
1236 timeout, ch->chid);
1237 ch->timeout_ms_max = timeout;
1238 ch->timeout_debug_dump = timeout_debug_dump;
1239 gk20a_channel_trace_sched_param(
1240 trace_gk20a_channel_set_timeout, ch);
1241 break;
1242 }
1243 case NVGPU_IOCTL_CHANNEL_GET_TIMEDOUT:
1244 ((struct nvgpu_get_param_args *)buf)->value =
1245 ch->has_timedout;
1246 break;
1247 case NVGPU_IOCTL_CHANNEL_ENABLE:
1248 err = gk20a_busy(ch->g);
1249 if (err) {
1250 dev_err(dev,
1251 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1252 __func__, cmd);
1253 break;
1254 }
1255 if (ch->g->ops.fifo.enable_channel)
1256 ch->g->ops.fifo.enable_channel(ch);
1257 else
1258 err = -ENOSYS;
1259 gk20a_idle(ch->g);
1260 break;
1261 case NVGPU_IOCTL_CHANNEL_DISABLE:
1262 err = gk20a_busy(ch->g);
1263 if (err) {
1264 dev_err(dev,
1265 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1266 __func__, cmd);
1267 break;
1268 }
1269 if (ch->g->ops.fifo.disable_channel)
1270 ch->g->ops.fifo.disable_channel(ch);
1271 else
1272 err = -ENOSYS;
1273 gk20a_idle(ch->g);
1274 break;
1275 case NVGPU_IOCTL_CHANNEL_PREEMPT:
1276 err = gk20a_busy(ch->g);
1277 if (err) {
1278 dev_err(dev,
1279 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1280 __func__, cmd);
1281 break;
1282 }
1283 err = gk20a_fifo_preempt(ch->g, ch);
1284 gk20a_idle(ch->g);
1285 break;
1286 case NVGPU_IOCTL_CHANNEL_RESCHEDULE_RUNLIST:
1287 if (!capable(CAP_SYS_NICE)) {
1288 err = -EPERM;
1289 break;
1290 }
1291 if (!ch->g->ops.fifo.reschedule_runlist) {
1292 err = -ENOSYS;
1293 break;
1294 }
1295 err = gk20a_busy(ch->g);
1296 if (err) {
1297 dev_err(dev,
1298 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1299 __func__, cmd);
1300 break;
1301 }
1302 err = ch->g->ops.fifo.reschedule_runlist(ch,
1303 NVGPU_RESCHEDULE_RUNLIST_PREEMPT_NEXT &
1304 ((struct nvgpu_reschedule_runlist_args *)buf)->flags);
1305 gk20a_idle(ch->g);
1306 break;
1307 case NVGPU_IOCTL_CHANNEL_FORCE_RESET:
1308 err = gk20a_busy(ch->g);
1309 if (err) {
1310 dev_err(dev,
1311 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1312 __func__, cmd);
1313 break;
1314 }
1315 err = ch->g->ops.fifo.force_reset_ch(ch,
1316 NVGPU_ERR_NOTIFIER_RESETCHANNEL_VERIF_ERROR, true);
1317 gk20a_idle(ch->g);
1318 break;
1319#ifdef CONFIG_GK20A_CYCLE_STATS
1320 case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT:
1321 err = gk20a_busy(ch->g);
1322 if (err) {
1323 dev_err(dev,
1324 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1325 __func__, cmd);
1326 break;
1327 }
1328 err = gk20a_channel_cycle_stats_snapshot(ch,
1329 (struct nvgpu_cycle_stats_snapshot_args *)buf);
1330 gk20a_idle(ch->g);
1331 break;
1332#endif
1333 case NVGPU_IOCTL_CHANNEL_WDT:
1334 err = gk20a_channel_set_wdt_status(ch,
1335 (struct nvgpu_channel_wdt_args *)buf);
1336 break;
1337 case NVGPU_IOCTL_CHANNEL_SET_PREEMPTION_MODE:
1338 err = nvgpu_ioctl_channel_set_preemption_mode(ch,
1339 ((struct nvgpu_preemption_mode_args *)buf)->graphics_preempt_mode,
1340 ((struct nvgpu_preemption_mode_args *)buf)->compute_preempt_mode);
1341 break;
1342 case NVGPU_IOCTL_CHANNEL_SET_BOOSTED_CTX:
1343 if (ch->g->ops.gr.set_boosted_ctx) {
1344 bool boost =
1345 ((struct nvgpu_boosted_ctx_args *)buf)->boost;
1346
1347 err = gk20a_busy(ch->g);
1348 if (err) {
1349 dev_err(dev,
1350 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1351 __func__, cmd);
1352 break;
1353 }
1354 err = ch->g->ops.gr.set_boosted_ctx(ch, boost);
1355 gk20a_idle(ch->g);
1356 } else {
1357 err = -EINVAL;
1358 }
1359 break;
1360 case NVGPU_IOCTL_CHANNEL_GET_USER_SYNCPOINT:
1361 err = gk20a_busy(ch->g);
1362 if (err) {
1363 dev_err(dev,
1364 "%s: failed to host gk20a for ioctl cmd: 0x%x",
1365 __func__, cmd);
1366 break;
1367 }
1368 err = nvgpu_ioctl_channel_get_user_syncpoint(ch,
1369 (struct nvgpu_get_user_syncpoint_args *)buf);
1370 gk20a_idle(ch->g);
1371 break;
1372 default:
1373 dev_dbg(dev, "unrecognized ioctl cmd: 0x%x", cmd);
1374 err = -ENOTTY;
1375 break;
1376 }
1377
1378 if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
1379 err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
1380
1381 nvgpu_mutex_release(&ch->ioctl_lock);
1382
1383 gk20a_channel_put(ch);
1384
1385 nvgpu_log_fn(g, "end");
1386
1387 return err;
1388}
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-05 01:48:45 -0400