diff options
Diffstat (limited to 'drivers/gpu/nvgpu/os/linux/ioctl_channel.c')
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/ioctl_channel.c | 1388 |
1 files changed, 1388 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_channel.c b/drivers/gpu/nvgpu/os/linux/ioctl_channel.c new file mode 100644 index 00000000..b04bb9de --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/ioctl_channel.c | |||
@@ -0,0 +1,1388 @@ | |||
1 | /* | ||
2 | * GK20A Graphics channel | ||
3 | * | ||
4 | * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * This program is free software; you can redistribute it and/or modify it | ||
7 | * under the terms and conditions of the GNU General Public License, | ||
8 | * version 2, as published by the Free Software Foundation. | ||
9 | * | ||
10 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
13 | * more details. | ||
14 | * | ||
15 | * You should have received a copy of the GNU General Public License | ||
16 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
17 | */ | ||
18 | |||
19 | #include <trace/events/gk20a.h> | ||
20 | #include <linux/file.h> | ||
21 | #include <linux/anon_inodes.h> | ||
22 | #include <linux/dma-buf.h> | ||
23 | #include <linux/poll.h> | ||
24 | #include <uapi/linux/nvgpu.h> | ||
25 | |||
26 | #include <nvgpu/semaphore.h> | ||
27 | #include <nvgpu/timers.h> | ||
28 | #include <nvgpu/kmem.h> | ||
29 | #include <nvgpu/log.h> | ||
30 | #include <nvgpu/list.h> | ||
31 | #include <nvgpu/debug.h> | ||
32 | #include <nvgpu/enabled.h> | ||
33 | #include <nvgpu/error_notifier.h> | ||
34 | #include <nvgpu/barrier.h> | ||
35 | #include <nvgpu/nvhost.h> | ||
36 | #include <nvgpu/os_sched.h> | ||
37 | |||
38 | #include "gk20a/gk20a.h" | ||
39 | #include "gk20a/dbg_gpu_gk20a.h" | ||
40 | #include "gk20a/fence_gk20a.h" | ||
41 | |||
42 | #include "platform_gk20a.h" | ||
43 | #include "ioctl_channel.h" | ||
44 | #include "channel.h" | ||
45 | #include "os_linux.h" | ||
46 | #include "ctxsw_trace.h" | ||
47 | |||
48 | /* the minimal size of client buffer */ | ||
49 | #define CSS_MIN_CLIENT_SNAPSHOT_SIZE \ | ||
50 | (sizeof(struct gk20a_cs_snapshot_fifo) + \ | ||
51 | sizeof(struct gk20a_cs_snapshot_fifo_entry) * 256) | ||
52 | |||
53 | static const char *gr_gk20a_graphics_preempt_mode_name(u32 graphics_preempt_mode) | ||
54 | { | ||
55 | switch (graphics_preempt_mode) { | ||
56 | case NVGPU_PREEMPTION_MODE_GRAPHICS_WFI: | ||
57 | return "WFI"; | ||
58 | default: | ||
59 | return "?"; | ||
60 | } | ||
61 | } | ||
62 | |||
63 | static const char *gr_gk20a_compute_preempt_mode_name(u32 compute_preempt_mode) | ||
64 | { | ||
65 | switch (compute_preempt_mode) { | ||
66 | case NVGPU_PREEMPTION_MODE_COMPUTE_WFI: | ||
67 | return "WFI"; | ||
68 | case NVGPU_PREEMPTION_MODE_COMPUTE_CTA: | ||
69 | return "CTA"; | ||
70 | default: | ||
71 | return "?"; | ||
72 | } | ||
73 | } | ||
74 | |||
75 | static void gk20a_channel_trace_sched_param( | ||
76 | void (*trace)(int chid, int tsgid, pid_t pid, u32 timeslice, | ||
77 | u32 timeout, const char *interleave, | ||
78 | const char *graphics_preempt_mode, | ||
79 | const char *compute_preempt_mode), | ||
80 | struct channel_gk20a *ch) | ||
81 | { | ||
82 | struct tsg_gk20a *tsg = tsg_gk20a_from_ch(ch); | ||
83 | |||
84 | if (!tsg) | ||
85 | return; | ||
86 | |||
87 | (trace)(ch->chid, ch->tsgid, ch->pid, | ||
88 | tsg_gk20a_from_ch(ch)->timeslice_us, | ||
89 | ch->timeout_ms_max, | ||
90 | gk20a_fifo_interleave_level_name(tsg->interleave_level), | ||
91 | gr_gk20a_graphics_preempt_mode_name( | ||
92 | tsg->gr_ctx.graphics_preempt_mode), | ||
93 | gr_gk20a_compute_preempt_mode_name( | ||
94 | tsg->gr_ctx.compute_preempt_mode)); | ||
95 | } | ||
96 | |||
97 | /* | ||
98 | * Although channels do have pointers back to the gk20a struct that they were | ||
99 | * created under in cases where the driver is killed that pointer can be bad. | ||
100 | * The channel memory can be freed before the release() function for a given | ||
101 | * channel is called. This happens when the driver dies and userspace doesn't | ||
102 | * get a chance to call release() until after the entire gk20a driver data is | ||
103 | * unloaded and freed. | ||
104 | */ | ||
105 | struct channel_priv { | ||
106 | struct gk20a *g; | ||
107 | struct channel_gk20a *c; | ||
108 | }; | ||
109 | |||
110 | #if defined(CONFIG_GK20A_CYCLE_STATS) | ||
111 | |||
112 | void gk20a_channel_free_cycle_stats_buffer(struct channel_gk20a *ch) | ||
113 | { | ||
114 | struct nvgpu_channel_linux *priv = ch->os_priv; | ||
115 | |||
116 | /* disable existing cyclestats buffer */ | ||
117 | nvgpu_mutex_acquire(&ch->cyclestate.cyclestate_buffer_mutex); | ||
118 | if (priv->cyclestate_buffer_handler) { | ||
119 | dma_buf_vunmap(priv->cyclestate_buffer_handler, | ||
120 | ch->cyclestate.cyclestate_buffer); | ||
121 | dma_buf_put(priv->cyclestate_buffer_handler); | ||
122 | priv->cyclestate_buffer_handler = NULL; | ||
123 | ch->cyclestate.cyclestate_buffer = NULL; | ||
124 | ch->cyclestate.cyclestate_buffer_size = 0; | ||
125 | } | ||
126 | nvgpu_mutex_release(&ch->cyclestate.cyclestate_buffer_mutex); | ||
127 | } | ||
128 | |||
129 | static int gk20a_channel_cycle_stats(struct channel_gk20a *ch, | ||
130 | struct nvgpu_cycle_stats_args *args) | ||
131 | { | ||
132 | struct dma_buf *dmabuf; | ||
133 | void *virtual_address; | ||
134 | struct nvgpu_channel_linux *priv = ch->os_priv; | ||
135 | |||
136 | /* is it allowed to handle calls for current GPU? */ | ||
137 | if (!nvgpu_is_enabled(ch->g, NVGPU_SUPPORT_CYCLE_STATS)) | ||
138 | return -ENOSYS; | ||
139 | |||
140 | if (args->dmabuf_fd && !priv->cyclestate_buffer_handler) { | ||
141 | |||
142 | /* set up new cyclestats buffer */ | ||
143 | dmabuf = dma_buf_get(args->dmabuf_fd); | ||
144 | if (IS_ERR(dmabuf)) | ||
145 | return PTR_ERR(dmabuf); | ||
146 | virtual_address = dma_buf_vmap(dmabuf); | ||
147 | if (!virtual_address) | ||
148 | return -ENOMEM; | ||
149 | |||
150 | priv->cyclestate_buffer_handler = dmabuf; | ||
151 | ch->cyclestate.cyclestate_buffer = virtual_address; | ||
152 | ch->cyclestate.cyclestate_buffer_size = dmabuf->size; | ||
153 | return 0; | ||
154 | |||
155 | } else if (!args->dmabuf_fd && priv->cyclestate_buffer_handler) { | ||
156 | gk20a_channel_free_cycle_stats_buffer(ch); | ||
157 | return 0; | ||
158 | |||
159 | } else if (!args->dmabuf_fd && !priv->cyclestate_buffer_handler) { | ||
160 | /* no requst from GL */ | ||
161 | return 0; | ||
162 | |||
163 | } else { | ||
164 | pr_err("channel already has cyclestats buffer\n"); | ||
165 | return -EINVAL; | ||
166 | } | ||
167 | } | ||
168 | |||
169 | static int gk20a_flush_cycle_stats_snapshot(struct channel_gk20a *ch) | ||
170 | { | ||
171 | int ret; | ||
172 | |||
173 | nvgpu_mutex_acquire(&ch->cs_client_mutex); | ||
174 | if (ch->cs_client) | ||
175 | ret = gr_gk20a_css_flush(ch, ch->cs_client); | ||
176 | else | ||
177 | ret = -EBADF; | ||
178 | nvgpu_mutex_release(&ch->cs_client_mutex); | ||
179 | |||
180 | return ret; | ||
181 | } | ||
182 | |||
183 | static int gk20a_attach_cycle_stats_snapshot(struct channel_gk20a *ch, | ||
184 | u32 dmabuf_fd, | ||
185 | u32 perfmon_id_count, | ||
186 | u32 *perfmon_id_start) | ||
187 | { | ||
188 | int ret = 0; | ||
189 | struct gk20a *g = ch->g; | ||
190 | struct gk20a_cs_snapshot_client_linux *client_linux; | ||
191 | struct gk20a_cs_snapshot_client *client; | ||
192 | |||
193 | nvgpu_mutex_acquire(&ch->cs_client_mutex); | ||
194 | if (ch->cs_client) { | ||
195 | nvgpu_mutex_release(&ch->cs_client_mutex); | ||
196 | return -EEXIST; | ||
197 | } | ||
198 | |||
199 | client_linux = nvgpu_kzalloc(g, sizeof(*client_linux)); | ||
200 | if (!client_linux) { | ||
201 | ret = -ENOMEM; | ||
202 | goto err; | ||
203 | } | ||
204 | |||
205 | client_linux->dmabuf_fd = dmabuf_fd; | ||
206 | client_linux->dma_handler = dma_buf_get(client_linux->dmabuf_fd); | ||
207 | if (IS_ERR(client_linux->dma_handler)) { | ||
208 | ret = PTR_ERR(client_linux->dma_handler); | ||
209 | client_linux->dma_handler = NULL; | ||
210 | goto err_free; | ||
211 | } | ||
212 | |||
213 | client = &client_linux->cs_client; | ||
214 | client->snapshot_size = client_linux->dma_handler->size; | ||
215 | if (client->snapshot_size < CSS_MIN_CLIENT_SNAPSHOT_SIZE) { | ||
216 | ret = -ENOMEM; | ||
217 | goto err_put; | ||
218 | } | ||
219 | |||
220 | client->snapshot = (struct gk20a_cs_snapshot_fifo *) | ||
221 | dma_buf_vmap(client_linux->dma_handler); | ||
222 | if (!client->snapshot) { | ||
223 | ret = -ENOMEM; | ||
224 | goto err_put; | ||
225 | } | ||
226 | |||
227 | ch->cs_client = client; | ||
228 | |||
229 | ret = gr_gk20a_css_attach(ch, | ||
230 | perfmon_id_count, | ||
231 | perfmon_id_start, | ||
232 | ch->cs_client); | ||
233 | |||
234 | nvgpu_mutex_release(&ch->cs_client_mutex); | ||
235 | |||
236 | return ret; | ||
237 | |||
238 | err_put: | ||
239 | dma_buf_put(client_linux->dma_handler); | ||
240 | err_free: | ||
241 | nvgpu_kfree(g, client_linux); | ||
242 | err: | ||
243 | nvgpu_mutex_release(&ch->cs_client_mutex); | ||
244 | return ret; | ||
245 | } | ||
246 | |||
247 | int gk20a_channel_free_cycle_stats_snapshot(struct channel_gk20a *ch) | ||
248 | { | ||
249 | int ret; | ||
250 | struct gk20a_cs_snapshot_client_linux *client_linux; | ||
251 | |||
252 | nvgpu_mutex_acquire(&ch->cs_client_mutex); | ||
253 | if (!ch->cs_client) { | ||
254 | nvgpu_mutex_release(&ch->cs_client_mutex); | ||
255 | return 0; | ||
256 | } | ||
257 | |||
258 | client_linux = container_of(ch->cs_client, | ||
259 | struct gk20a_cs_snapshot_client_linux, | ||
260 | cs_client); | ||
261 | |||
262 | ret = gr_gk20a_css_detach(ch, ch->cs_client); | ||
263 | |||
264 | if (client_linux->dma_handler) { | ||
265 | if (ch->cs_client->snapshot) | ||
266 | dma_buf_vunmap(client_linux->dma_handler, | ||
267 | ch->cs_client->snapshot); | ||
268 | dma_buf_put(client_linux->dma_handler); | ||
269 | } | ||
270 | |||
271 | ch->cs_client = NULL; | ||
272 | nvgpu_kfree(ch->g, client_linux); | ||
273 | |||
274 | nvgpu_mutex_release(&ch->cs_client_mutex); | ||
275 | |||
276 | return ret; | ||
277 | } | ||
278 | |||
279 | static int gk20a_channel_cycle_stats_snapshot(struct channel_gk20a *ch, | ||
280 | struct nvgpu_cycle_stats_snapshot_args *args) | ||
281 | { | ||
282 | int ret; | ||
283 | |||
284 | /* is it allowed to handle calls for current GPU? */ | ||
285 | if (!nvgpu_is_enabled(ch->g, NVGPU_SUPPORT_CYCLE_STATS_SNAPSHOT)) | ||
286 | return -ENOSYS; | ||
287 | |||
288 | if (!args->dmabuf_fd) | ||
289 | return -EINVAL; | ||
290 | |||
291 | /* handle the command (most frequent cases first) */ | ||
292 | switch (args->cmd) { | ||
293 | case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_FLUSH: | ||
294 | ret = gk20a_flush_cycle_stats_snapshot(ch); | ||
295 | args->extra = 0; | ||
296 | break; | ||
297 | |||
298 | case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_ATTACH: | ||
299 | ret = gk20a_attach_cycle_stats_snapshot(ch, | ||
300 | args->dmabuf_fd, | ||
301 | args->extra, | ||
302 | &args->extra); | ||
303 | break; | ||
304 | |||
305 | case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT_CMD_DETACH: | ||
306 | ret = gk20a_channel_free_cycle_stats_snapshot(ch); | ||
307 | args->extra = 0; | ||
308 | break; | ||
309 | |||
310 | default: | ||
311 | pr_err("cyclestats: unknown command %u\n", args->cmd); | ||
312 | ret = -EINVAL; | ||
313 | break; | ||
314 | } | ||
315 | |||
316 | return ret; | ||
317 | } | ||
318 | #endif | ||
319 | |||
320 | static int gk20a_channel_set_wdt_status(struct channel_gk20a *ch, | ||
321 | struct nvgpu_channel_wdt_args *args) | ||
322 | { | ||
323 | u32 status = args->wdt_status & (NVGPU_IOCTL_CHANNEL_DISABLE_WDT | | ||
324 | NVGPU_IOCTL_CHANNEL_ENABLE_WDT); | ||
325 | |||
326 | if (status == NVGPU_IOCTL_CHANNEL_DISABLE_WDT) | ||
327 | ch->timeout.enabled = false; | ||
328 | else if (status == NVGPU_IOCTL_CHANNEL_ENABLE_WDT) | ||
329 | ch->timeout.enabled = true; | ||
330 | else | ||
331 | return -EINVAL; | ||
332 | |||
333 | if (args->wdt_status & NVGPU_IOCTL_CHANNEL_WDT_FLAG_SET_TIMEOUT) | ||
334 | ch->timeout.limit_ms = args->timeout_ms; | ||
335 | |||
336 | ch->timeout.debug_dump = (args->wdt_status & | ||
337 | NVGPU_IOCTL_CHANNEL_WDT_FLAG_DISABLE_DUMP) == 0; | ||
338 | |||
339 | return 0; | ||
340 | } | ||
341 | |||
342 | static void gk20a_channel_free_error_notifiers(struct channel_gk20a *ch) | ||
343 | { | ||
344 | struct nvgpu_channel_linux *priv = ch->os_priv; | ||
345 | |||
346 | nvgpu_mutex_acquire(&priv->error_notifier.mutex); | ||
347 | if (priv->error_notifier.dmabuf) { | ||
348 | dma_buf_vunmap(priv->error_notifier.dmabuf, priv->error_notifier.vaddr); | ||
349 | dma_buf_put(priv->error_notifier.dmabuf); | ||
350 | priv->error_notifier.dmabuf = NULL; | ||
351 | priv->error_notifier.notification = NULL; | ||
352 | priv->error_notifier.vaddr = NULL; | ||
353 | } | ||
354 | nvgpu_mutex_release(&priv->error_notifier.mutex); | ||
355 | } | ||
356 | |||
357 | static int gk20a_init_error_notifier(struct channel_gk20a *ch, | ||
358 | struct nvgpu_set_error_notifier *args) | ||
359 | { | ||
360 | struct dma_buf *dmabuf; | ||
361 | void *va; | ||
362 | u64 end = args->offset + sizeof(struct nvgpu_notification); | ||
363 | struct nvgpu_channel_linux *priv = ch->os_priv; | ||
364 | |||
365 | if (!args->mem) { | ||
366 | pr_err("gk20a_init_error_notifier: invalid memory handle\n"); | ||
367 | return -EINVAL; | ||
368 | } | ||
369 | |||
370 | dmabuf = dma_buf_get(args->mem); | ||
371 | |||
372 | gk20a_channel_free_error_notifiers(ch); | ||
373 | |||
374 | if (IS_ERR(dmabuf)) { | ||
375 | pr_err("Invalid handle: %d\n", args->mem); | ||
376 | return -EINVAL; | ||
377 | } | ||
378 | |||
379 | if (end > dmabuf->size || end < sizeof(struct nvgpu_notification)) { | ||
380 | dma_buf_put(dmabuf); | ||
381 | nvgpu_err(ch->g, "gk20a_init_error_notifier: invalid offset"); | ||
382 | return -EINVAL; | ||
383 | } | ||
384 | |||
385 | nvgpu_speculation_barrier(); | ||
386 | |||
387 | /* map handle */ | ||
388 | va = dma_buf_vmap(dmabuf); | ||
389 | if (!va) { | ||
390 | dma_buf_put(dmabuf); | ||
391 | pr_err("Cannot map notifier handle\n"); | ||
392 | return -ENOMEM; | ||
393 | } | ||
394 | |||
395 | priv->error_notifier.notification = va + args->offset; | ||
396 | priv->error_notifier.vaddr = va; | ||
397 | memset(priv->error_notifier.notification, 0, | ||
398 | sizeof(struct nvgpu_notification)); | ||
399 | |||
400 | /* set channel notifiers pointer */ | ||
401 | nvgpu_mutex_acquire(&priv->error_notifier.mutex); | ||
402 | priv->error_notifier.dmabuf = dmabuf; | ||
403 | nvgpu_mutex_release(&priv->error_notifier.mutex); | ||
404 | |||
405 | return 0; | ||
406 | } | ||
407 | |||
408 | /* | ||
409 | * This returns the channel with a reference. The caller must | ||
410 | * gk20a_channel_put() the ref back after use. | ||
411 | * | ||
412 | * NULL is returned if the channel was not found. | ||
413 | */ | ||
414 | struct channel_gk20a *gk20a_get_channel_from_file(int fd) | ||
415 | { | ||
416 | struct channel_gk20a *ch; | ||
417 | struct channel_priv *priv; | ||
418 | struct file *f = fget(fd); | ||
419 | |||
420 | if (!f) | ||
421 | return NULL; | ||
422 | |||
423 | if (f->f_op != &gk20a_channel_ops) { | ||
424 | fput(f); | ||
425 | return NULL; | ||
426 | } | ||
427 | |||
428 | priv = (struct channel_priv *)f->private_data; | ||
429 | ch = gk20a_channel_get(priv->c); | ||
430 | fput(f); | ||
431 | return ch; | ||
432 | } | ||
433 | |||
434 | int gk20a_channel_release(struct inode *inode, struct file *filp) | ||
435 | { | ||
436 | struct channel_priv *priv = filp->private_data; | ||
437 | struct channel_gk20a *ch; | ||
438 | struct gk20a *g; | ||
439 | |||
440 | int err; | ||
441 | |||
442 | /* We could still end up here even if the channel_open failed, e.g. | ||
443 | * if we ran out of hw channel IDs. | ||
444 | */ | ||
445 | if (!priv) | ||
446 | return 0; | ||
447 | |||
448 | ch = priv->c; | ||
449 | g = priv->g; | ||
450 | |||
451 | err = gk20a_busy(g); | ||
452 | if (err) { | ||
453 | nvgpu_err(g, "failed to release a channel!"); | ||
454 | goto channel_release; | ||
455 | } | ||
456 | |||
457 | trace_gk20a_channel_release(dev_name(dev_from_gk20a(g))); | ||
458 | |||
459 | gk20a_channel_close(ch); | ||
460 | gk20a_channel_free_error_notifiers(ch); | ||
461 | |||
462 | gk20a_idle(g); | ||
463 | |||
464 | channel_release: | ||
465 | gk20a_put(g); | ||
466 | nvgpu_kfree(g, filp->private_data); | ||
467 | filp->private_data = NULL; | ||
468 | return 0; | ||
469 | } | ||
470 | |||
471 | /* note: runlist_id -1 is synonym for the ENGINE_GR_GK20A runlist id */ | ||
472 | static int __gk20a_channel_open(struct gk20a *g, | ||
473 | struct file *filp, s32 runlist_id) | ||
474 | { | ||
475 | int err; | ||
476 | struct channel_gk20a *ch; | ||
477 | struct channel_priv *priv; | ||
478 | |||
479 | nvgpu_log_fn(g, " "); | ||
480 | |||
481 | g = gk20a_get(g); | ||
482 | if (!g) | ||
483 | return -ENODEV; | ||
484 | |||
485 | trace_gk20a_channel_open(dev_name(dev_from_gk20a(g))); | ||
486 | |||
487 | priv = nvgpu_kzalloc(g, sizeof(*priv)); | ||
488 | if (!priv) { | ||
489 | err = -ENOMEM; | ||
490 | goto free_ref; | ||
491 | } | ||
492 | |||
493 | err = gk20a_busy(g); | ||
494 | if (err) { | ||
495 | nvgpu_err(g, "failed to power on, %d", err); | ||
496 | goto fail_busy; | ||
497 | } | ||
498 | /* All the user space channel should be non privilege */ | ||
499 | ch = gk20a_open_new_channel(g, runlist_id, false, | ||
500 | nvgpu_current_pid(g), nvgpu_current_tid(g)); | ||
501 | gk20a_idle(g); | ||
502 | if (!ch) { | ||
503 | nvgpu_err(g, | ||
504 | "failed to get f"); | ||
505 | err = -ENOMEM; | ||
506 | goto fail_busy; | ||
507 | } | ||
508 | |||
509 | gk20a_channel_trace_sched_param( | ||
510 | trace_gk20a_channel_sched_defaults, ch); | ||
511 | |||
512 | priv->g = g; | ||
513 | priv->c = ch; | ||
514 | |||
515 | filp->private_data = priv; | ||
516 | return 0; | ||
517 | |||
518 | fail_busy: | ||
519 | nvgpu_kfree(g, priv); | ||
520 | free_ref: | ||
521 | gk20a_put(g); | ||
522 | return err; | ||
523 | } | ||
524 | |||
525 | int gk20a_channel_open(struct inode *inode, struct file *filp) | ||
526 | { | ||
527 | struct nvgpu_os_linux *l = container_of(inode->i_cdev, | ||
528 | struct nvgpu_os_linux, channel.cdev); | ||
529 | struct gk20a *g = &l->g; | ||
530 | int ret; | ||
531 | |||
532 | nvgpu_log_fn(g, "start"); | ||
533 | ret = __gk20a_channel_open(g, filp, -1); | ||
534 | |||
535 | nvgpu_log_fn(g, "end"); | ||
536 | return ret; | ||
537 | } | ||
538 | |||
539 | int gk20a_channel_open_ioctl(struct gk20a *g, | ||
540 | struct nvgpu_channel_open_args *args) | ||
541 | { | ||
542 | int err; | ||
543 | int fd; | ||
544 | struct file *file; | ||
545 | char name[64]; | ||
546 | s32 runlist_id = args->in.runlist_id; | ||
547 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
548 | |||
549 | err = get_unused_fd_flags(O_RDWR); | ||
550 | if (err < 0) | ||
551 | return err; | ||
552 | fd = err; | ||
553 | |||
554 | snprintf(name, sizeof(name), "nvhost-%s-fd%d", | ||
555 | dev_name(dev_from_gk20a(g)), fd); | ||
556 | |||
557 | file = anon_inode_getfile(name, l->channel.cdev.ops, NULL, O_RDWR); | ||
558 | if (IS_ERR(file)) { | ||
559 | err = PTR_ERR(file); | ||
560 | goto clean_up; | ||
561 | } | ||
562 | |||
563 | err = __gk20a_channel_open(g, file, runlist_id); | ||
564 | if (err) | ||
565 | goto clean_up_file; | ||
566 | |||
567 | fd_install(fd, file); | ||
568 | args->out.channel_fd = fd; | ||
569 | return 0; | ||
570 | |||
571 | clean_up_file: | ||
572 | fput(file); | ||
573 | clean_up: | ||
574 | put_unused_fd(fd); | ||
575 | return err; | ||
576 | } | ||
577 | |||
578 | static u32 nvgpu_gpfifo_user_flags_to_common_flags(u32 user_flags) | ||
579 | { | ||
580 | u32 flags = 0; | ||
581 | |||
582 | if (user_flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_VPR_ENABLED) | ||
583 | flags |= NVGPU_GPFIFO_FLAGS_SUPPORT_VPR; | ||
584 | |||
585 | if (user_flags & NVGPU_ALLOC_GPFIFO_EX_FLAGS_DETERMINISTIC) | ||
586 | flags |= NVGPU_GPFIFO_FLAGS_SUPPORT_DETERMINISTIC; | ||
587 | |||
588 | if (user_flags & NVGPU_ALLOC_GPFIFO_FLAGS_REPLAYABLE_FAULTS_ENABLE) | ||
589 | flags |= NVGPU_GPFIFO_FLAGS_REPLAYABLE_FAULTS_ENABLE; | ||
590 | |||
591 | return flags; | ||
592 | } | ||
593 | |||
594 | static void nvgpu_get_gpfifo_ex_args( | ||
595 | struct nvgpu_alloc_gpfifo_ex_args *alloc_gpfifo_ex_args, | ||
596 | struct nvgpu_gpfifo_args *gpfifo_args) | ||
597 | { | ||
598 | gpfifo_args->num_entries = alloc_gpfifo_ex_args->num_entries; | ||
599 | gpfifo_args->num_inflight_jobs = alloc_gpfifo_ex_args->num_inflight_jobs; | ||
600 | gpfifo_args->flags = nvgpu_gpfifo_user_flags_to_common_flags( | ||
601 | alloc_gpfifo_ex_args->flags); | ||
602 | } | ||
603 | |||
604 | static void nvgpu_get_gpfifo_args( | ||
605 | struct nvgpu_alloc_gpfifo_args *alloc_gpfifo_args, | ||
606 | struct nvgpu_gpfifo_args *gpfifo_args) | ||
607 | { | ||
608 | /* | ||
609 | * Kernel can insert one extra gpfifo entry before user | ||
610 | * submitted gpfifos and another one after, for internal usage. | ||
611 | * Triple the requested size. | ||
612 | */ | ||
613 | gpfifo_args->num_entries = alloc_gpfifo_args->num_entries * 3; | ||
614 | gpfifo_args->num_inflight_jobs = 0; | ||
615 | gpfifo_args->flags = nvgpu_gpfifo_user_flags_to_common_flags( | ||
616 | alloc_gpfifo_args->flags); | ||
617 | } | ||
618 | |||
619 | static void nvgpu_get_fence_args( | ||
620 | struct nvgpu_fence *fence_args_in, | ||
621 | struct nvgpu_channel_fence *fence_args_out) | ||
622 | { | ||
623 | fence_args_out->id = fence_args_in->id; | ||
624 | fence_args_out->value = fence_args_in->value; | ||
625 | } | ||
626 | |||
627 | static int gk20a_channel_wait_semaphore(struct channel_gk20a *ch, | ||
628 | ulong id, u32 offset, | ||
629 | u32 payload, u32 timeout) | ||
630 | { | ||
631 | struct dma_buf *dmabuf; | ||
632 | void *data; | ||
633 | u32 *semaphore; | ||
634 | int ret = 0; | ||
635 | |||
636 | /* do not wait if channel has timed out */ | ||
637 | if (ch->has_timedout) | ||
638 | return -ETIMEDOUT; | ||
639 | |||
640 | dmabuf = dma_buf_get(id); | ||
641 | if (IS_ERR(dmabuf)) { | ||
642 | nvgpu_err(ch->g, "invalid notifier nvmap handle 0x%lx", id); | ||
643 | return -EINVAL; | ||
644 | } | ||
645 | |||
646 | data = dma_buf_kmap(dmabuf, offset >> PAGE_SHIFT); | ||
647 | if (!data) { | ||
648 | nvgpu_err(ch->g, "failed to map notifier memory"); | ||
649 | ret = -EINVAL; | ||
650 | goto cleanup_put; | ||
651 | } | ||
652 | |||
653 | semaphore = data + (offset & ~PAGE_MASK); | ||
654 | |||
655 | ret = NVGPU_COND_WAIT_INTERRUPTIBLE( | ||
656 | &ch->semaphore_wq, | ||
657 | *semaphore == payload || ch->has_timedout, | ||
658 | timeout); | ||
659 | |||
660 | dma_buf_kunmap(dmabuf, offset >> PAGE_SHIFT, data); | ||
661 | cleanup_put: | ||
662 | dma_buf_put(dmabuf); | ||
663 | return ret; | ||
664 | } | ||
665 | |||
666 | static int gk20a_channel_wait(struct channel_gk20a *ch, | ||
667 | struct nvgpu_wait_args *args) | ||
668 | { | ||
669 | struct dma_buf *dmabuf; | ||
670 | struct gk20a *g = ch->g; | ||
671 | struct notification *notif; | ||
672 | struct timespec tv; | ||
673 | u64 jiffies; | ||
674 | ulong id; | ||
675 | u32 offset; | ||
676 | int remain, ret = 0; | ||
677 | u64 end; | ||
678 | |||
679 | nvgpu_log_fn(g, " "); | ||
680 | |||
681 | if (ch->has_timedout) | ||
682 | return -ETIMEDOUT; | ||
683 | |||
684 | switch (args->type) { | ||
685 | case NVGPU_WAIT_TYPE_NOTIFIER: | ||
686 | id = args->condition.notifier.dmabuf_fd; | ||
687 | offset = args->condition.notifier.offset; | ||
688 | end = offset + sizeof(struct notification); | ||
689 | |||
690 | dmabuf = dma_buf_get(id); | ||
691 | if (IS_ERR(dmabuf)) { | ||
692 | nvgpu_err(g, "invalid notifier nvmap handle 0x%lx", | ||
693 | id); | ||
694 | return -EINVAL; | ||
695 | } | ||
696 | |||
697 | if (end > dmabuf->size || end < sizeof(struct notification)) { | ||
698 | dma_buf_put(dmabuf); | ||
699 | nvgpu_err(g, "invalid notifier offset"); | ||
700 | return -EINVAL; | ||
701 | } | ||
702 | |||
703 | nvgpu_speculation_barrier(); | ||
704 | |||
705 | notif = dma_buf_vmap(dmabuf); | ||
706 | if (!notif) { | ||
707 | nvgpu_err(g, "failed to map notifier memory"); | ||
708 | return -ENOMEM; | ||
709 | } | ||
710 | |||
711 | notif = (struct notification *)((uintptr_t)notif + offset); | ||
712 | |||
713 | /* user should set status pending before | ||
714 | * calling this ioctl */ | ||
715 | remain = NVGPU_COND_WAIT_INTERRUPTIBLE( | ||
716 | &ch->notifier_wq, | ||
717 | notif->status == 0 || ch->has_timedout, | ||
718 | args->timeout); | ||
719 | |||
720 | if (remain == 0 && notif->status != 0) { | ||
721 | ret = -ETIMEDOUT; | ||
722 | goto notif_clean_up; | ||
723 | } else if (remain < 0) { | ||
724 | ret = -EINTR; | ||
725 | goto notif_clean_up; | ||
726 | } | ||
727 | |||
728 | /* TBD: fill in correct information */ | ||
729 | jiffies = get_jiffies_64(); | ||
730 | jiffies_to_timespec(jiffies, &tv); | ||
731 | notif->timestamp.nanoseconds[0] = tv.tv_nsec; | ||
732 | notif->timestamp.nanoseconds[1] = tv.tv_sec; | ||
733 | notif->info32 = 0xDEADBEEF; /* should be object name */ | ||
734 | notif->info16 = ch->chid; /* should be method offset */ | ||
735 | |||
736 | notif_clean_up: | ||
737 | dma_buf_vunmap(dmabuf, notif); | ||
738 | return ret; | ||
739 | |||
740 | case NVGPU_WAIT_TYPE_SEMAPHORE: | ||
741 | ret = gk20a_channel_wait_semaphore(ch, | ||
742 | args->condition.semaphore.dmabuf_fd, | ||
743 | args->condition.semaphore.offset, | ||
744 | args->condition.semaphore.payload, | ||
745 | args->timeout); | ||
746 | |||
747 | break; | ||
748 | |||
749 | default: | ||
750 | ret = -EINVAL; | ||
751 | break; | ||
752 | } | ||
753 | |||
754 | return ret; | ||
755 | } | ||
756 | |||
757 | static int gk20a_channel_zcull_bind(struct channel_gk20a *ch, | ||
758 | struct nvgpu_zcull_bind_args *args) | ||
759 | { | ||
760 | struct gk20a *g = ch->g; | ||
761 | struct gr_gk20a *gr = &g->gr; | ||
762 | |||
763 | nvgpu_log_fn(gr->g, " "); | ||
764 | |||
765 | return g->ops.gr.bind_ctxsw_zcull(g, gr, ch, | ||
766 | args->gpu_va, args->mode); | ||
767 | } | ||
768 | |||
769 | static int gk20a_ioctl_channel_submit_gpfifo( | ||
770 | struct channel_gk20a *ch, | ||
771 | struct nvgpu_submit_gpfifo_args *args) | ||
772 | { | ||
773 | struct nvgpu_channel_fence fence; | ||
774 | struct gk20a_fence *fence_out; | ||
775 | struct fifo_profile_gk20a *profile = NULL; | ||
776 | u32 submit_flags = 0; | ||
777 | int fd = -1; | ||
778 | struct gk20a *g = ch->g; | ||
779 | |||
780 | int ret = 0; | ||
781 | nvgpu_log_fn(g, " "); | ||
782 | |||
783 | profile = gk20a_fifo_profile_acquire(ch->g); | ||
784 | gk20a_fifo_profile_snapshot(profile, PROFILE_IOCTL_ENTRY); | ||
785 | |||
786 | if (ch->has_timedout) | ||
787 | return -ETIMEDOUT; | ||
788 | |||
789 | nvgpu_get_fence_args(&args->fence, &fence); | ||
790 | submit_flags = | ||
791 | nvgpu_submit_gpfifo_user_flags_to_common_flags(args->flags); | ||
792 | |||
793 | /* Try and allocate an fd here*/ | ||
794 | if ((args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) | ||
795 | && (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE)) { | ||
796 | fd = get_unused_fd_flags(O_RDWR); | ||
797 | if (fd < 0) | ||
798 | return fd; | ||
799 | } | ||
800 | |||
801 | ret = gk20a_submit_channel_gpfifo(ch, NULL, args, args->num_entries, | ||
802 | submit_flags, &fence, | ||
803 | &fence_out, profile); | ||
804 | |||
805 | if (ret) { | ||
806 | if (fd != -1) | ||
807 | put_unused_fd(fd); | ||
808 | goto clean_up; | ||
809 | } | ||
810 | |||
811 | /* Convert fence_out to something we can pass back to user space. */ | ||
812 | if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) { | ||
813 | if (args->flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE) { | ||
814 | ret = gk20a_fence_install_fd(fence_out, fd); | ||
815 | if (ret) | ||
816 | put_unused_fd(fd); | ||
817 | else | ||
818 | args->fence.id = fd; | ||
819 | } else { | ||
820 | args->fence.id = fence_out->syncpt_id; | ||
821 | args->fence.value = fence_out->syncpt_value; | ||
822 | } | ||
823 | } | ||
824 | gk20a_fence_put(fence_out); | ||
825 | |||
826 | gk20a_fifo_profile_snapshot(profile, PROFILE_IOCTL_EXIT); | ||
827 | if (profile) | ||
828 | gk20a_fifo_profile_release(ch->g, profile); | ||
829 | |||
830 | clean_up: | ||
831 | return ret; | ||
832 | } | ||
833 | |||
834 | /* | ||
835 | * Convert linux specific runlist level of the form NVGPU_RUNLIST_INTERLEAVE_LEVEL_* | ||
836 | * to common runlist level of the form NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_* | ||
837 | */ | ||
838 | u32 nvgpu_get_common_runlist_level(u32 level) | ||
839 | { | ||
840 | switch (level) { | ||
841 | case NVGPU_RUNLIST_INTERLEAVE_LEVEL_LOW: | ||
842 | return NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW; | ||
843 | case NVGPU_RUNLIST_INTERLEAVE_LEVEL_MEDIUM: | ||
844 | return NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM; | ||
845 | case NVGPU_RUNLIST_INTERLEAVE_LEVEL_HIGH: | ||
846 | return NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH; | ||
847 | default: | ||
848 | pr_err("%s: incorrect runlist level\n", __func__); | ||
849 | } | ||
850 | |||
851 | return level; | ||
852 | } | ||
853 | |||
854 | static u32 nvgpu_obj_ctx_user_flags_to_common_flags(u32 user_flags) | ||
855 | { | ||
856 | u32 flags = 0; | ||
857 | |||
858 | if (user_flags & NVGPU_ALLOC_OBJ_FLAGS_GFXP) | ||
859 | flags |= NVGPU_OBJ_CTX_FLAGS_SUPPORT_GFXP; | ||
860 | |||
861 | if (user_flags & NVGPU_ALLOC_OBJ_FLAGS_CILP) | ||
862 | flags |= NVGPU_OBJ_CTX_FLAGS_SUPPORT_CILP; | ||
863 | |||
864 | return flags; | ||
865 | } | ||
866 | |||
867 | static int nvgpu_ioctl_channel_alloc_obj_ctx(struct channel_gk20a *ch, | ||
868 | u32 class_num, u32 user_flags) | ||
869 | { | ||
870 | return ch->g->ops.gr.alloc_obj_ctx(ch, class_num, | ||
871 | nvgpu_obj_ctx_user_flags_to_common_flags(user_flags)); | ||
872 | } | ||
873 | |||
874 | /* | ||
875 | * Convert common preemption mode flags of the form NVGPU_PREEMPTION_MODE_GRAPHICS_* | ||
876 | * into linux preemption mode flags of the form NVGPU_GRAPHICS_PREEMPTION_MODE_* | ||
877 | */ | ||
878 | u32 nvgpu_get_ioctl_graphics_preempt_mode_flags(u32 graphics_preempt_mode_flags) | ||
879 | { | ||
880 | u32 flags = 0; | ||
881 | |||
882 | if (graphics_preempt_mode_flags & NVGPU_PREEMPTION_MODE_GRAPHICS_WFI) | ||
883 | flags |= NVGPU_GRAPHICS_PREEMPTION_MODE_WFI; | ||
884 | if (graphics_preempt_mode_flags & NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP) | ||
885 | flags |= NVGPU_GRAPHICS_PREEMPTION_MODE_GFXP; | ||
886 | |||
887 | return flags; | ||
888 | } | ||
889 | |||
890 | /* | ||
891 | * Convert common preemption mode flags of the form NVGPU_PREEMPTION_MODE_COMPUTE_* | ||
892 | * into linux preemption mode flags of the form NVGPU_COMPUTE_PREEMPTION_MODE_* | ||
893 | */ | ||
894 | u32 nvgpu_get_ioctl_compute_preempt_mode_flags(u32 compute_preempt_mode_flags) | ||
895 | { | ||
896 | u32 flags = 0; | ||
897 | |||
898 | if (compute_preempt_mode_flags & NVGPU_PREEMPTION_MODE_COMPUTE_WFI) | ||
899 | flags |= NVGPU_COMPUTE_PREEMPTION_MODE_WFI; | ||
900 | if (compute_preempt_mode_flags & NVGPU_PREEMPTION_MODE_COMPUTE_CTA) | ||
901 | flags |= NVGPU_COMPUTE_PREEMPTION_MODE_CTA; | ||
902 | if (compute_preempt_mode_flags & NVGPU_PREEMPTION_MODE_COMPUTE_CILP) | ||
903 | flags |= NVGPU_COMPUTE_PREEMPTION_MODE_CILP; | ||
904 | |||
905 | return flags; | ||
906 | } | ||
907 | |||
908 | /* | ||
909 | * Convert common preemption modes of the form NVGPU_PREEMPTION_MODE_GRAPHICS_* | ||
910 | * into linux preemption modes of the form NVGPU_GRAPHICS_PREEMPTION_MODE_* | ||
911 | */ | ||
912 | u32 nvgpu_get_ioctl_graphics_preempt_mode(u32 graphics_preempt_mode) | ||
913 | { | ||
914 | switch (graphics_preempt_mode) { | ||
915 | case NVGPU_PREEMPTION_MODE_GRAPHICS_WFI: | ||
916 | return NVGPU_GRAPHICS_PREEMPTION_MODE_WFI; | ||
917 | case NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP: | ||
918 | return NVGPU_GRAPHICS_PREEMPTION_MODE_GFXP; | ||
919 | } | ||
920 | |||
921 | return graphics_preempt_mode; | ||
922 | } | ||
923 | |||
924 | /* | ||
925 | * Convert common preemption modes of the form NVGPU_PREEMPTION_MODE_COMPUTE_* | ||
926 | * into linux preemption modes of the form NVGPU_COMPUTE_PREEMPTION_MODE_* | ||
927 | */ | ||
928 | u32 nvgpu_get_ioctl_compute_preempt_mode(u32 compute_preempt_mode) | ||
929 | { | ||
930 | switch (compute_preempt_mode) { | ||
931 | case NVGPU_PREEMPTION_MODE_COMPUTE_WFI: | ||
932 | return NVGPU_COMPUTE_PREEMPTION_MODE_WFI; | ||
933 | case NVGPU_PREEMPTION_MODE_COMPUTE_CTA: | ||
934 | return NVGPU_COMPUTE_PREEMPTION_MODE_CTA; | ||
935 | case NVGPU_PREEMPTION_MODE_COMPUTE_CILP: | ||
936 | return NVGPU_COMPUTE_PREEMPTION_MODE_CILP; | ||
937 | } | ||
938 | |||
939 | return compute_preempt_mode; | ||
940 | } | ||
941 | |||
942 | /* | ||
943 | * Convert linux preemption modes of the form NVGPU_GRAPHICS_PREEMPTION_MODE_* | ||
944 | * into common preemption modes of the form NVGPU_PREEMPTION_MODE_GRAPHICS_* | ||
945 | */ | ||
946 | static u32 nvgpu_get_common_graphics_preempt_mode(u32 graphics_preempt_mode) | ||
947 | { | ||
948 | switch (graphics_preempt_mode) { | ||
949 | case NVGPU_GRAPHICS_PREEMPTION_MODE_WFI: | ||
950 | return NVGPU_PREEMPTION_MODE_GRAPHICS_WFI; | ||
951 | case NVGPU_GRAPHICS_PREEMPTION_MODE_GFXP: | ||
952 | return NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP; | ||
953 | } | ||
954 | |||
955 | return graphics_preempt_mode; | ||
956 | } | ||
957 | |||
958 | /* | ||
959 | * Convert linux preemption modes of the form NVGPU_COMPUTE_PREEMPTION_MODE_* | ||
960 | * into common preemption modes of the form NVGPU_PREEMPTION_MODE_COMPUTE_* | ||
961 | */ | ||
962 | static u32 nvgpu_get_common_compute_preempt_mode(u32 compute_preempt_mode) | ||
963 | { | ||
964 | switch (compute_preempt_mode) { | ||
965 | case NVGPU_COMPUTE_PREEMPTION_MODE_WFI: | ||
966 | return NVGPU_PREEMPTION_MODE_COMPUTE_WFI; | ||
967 | case NVGPU_COMPUTE_PREEMPTION_MODE_CTA: | ||
968 | return NVGPU_PREEMPTION_MODE_COMPUTE_CTA; | ||
969 | case NVGPU_COMPUTE_PREEMPTION_MODE_CILP: | ||
970 | return NVGPU_PREEMPTION_MODE_COMPUTE_CILP; | ||
971 | } | ||
972 | |||
973 | return compute_preempt_mode; | ||
974 | } | ||
975 | |||
976 | static int nvgpu_ioctl_channel_set_preemption_mode(struct channel_gk20a *ch, | ||
977 | u32 graphics_preempt_mode, u32 compute_preempt_mode) | ||
978 | { | ||
979 | int err; | ||
980 | |||
981 | if (ch->g->ops.gr.set_preemption_mode) { | ||
982 | err = gk20a_busy(ch->g); | ||
983 | if (err) { | ||
984 | nvgpu_err(ch->g, "failed to power on, %d", err); | ||
985 | return err; | ||
986 | } | ||
987 | err = ch->g->ops.gr.set_preemption_mode(ch, | ||
988 | nvgpu_get_common_graphics_preempt_mode(graphics_preempt_mode), | ||
989 | nvgpu_get_common_compute_preempt_mode(compute_preempt_mode)); | ||
990 | gk20a_idle(ch->g); | ||
991 | } else { | ||
992 | err = -EINVAL; | ||
993 | } | ||
994 | |||
995 | return err; | ||
996 | } | ||
997 | |||
998 | static int nvgpu_ioctl_channel_get_user_syncpoint(struct channel_gk20a *ch, | ||
999 | struct nvgpu_get_user_syncpoint_args *args) | ||
1000 | { | ||
1001 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | ||
1002 | struct gk20a *g = ch->g; | ||
1003 | int err; | ||
1004 | |||
1005 | if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_USER_SYNCPOINT)) { | ||
1006 | nvgpu_err(g, "user syncpoints not supported"); | ||
1007 | return -EINVAL; | ||
1008 | } | ||
1009 | |||
1010 | if (!gk20a_platform_has_syncpoints(g)) { | ||
1011 | nvgpu_err(g, "syncpoints not supported"); | ||
1012 | return -EINVAL; | ||
1013 | } | ||
1014 | |||
1015 | if (g->aggressive_sync_destroy_thresh) { | ||
1016 | nvgpu_err(g, "sufficient syncpoints not available"); | ||
1017 | return -EINVAL; | ||
1018 | } | ||
1019 | |||
1020 | nvgpu_mutex_acquire(&ch->sync_lock); | ||
1021 | if (ch->user_sync) { | ||
1022 | nvgpu_mutex_release(&ch->sync_lock); | ||
1023 | } else { | ||
1024 | ch->user_sync = gk20a_channel_sync_create(ch, true); | ||
1025 | if (!ch->user_sync) { | ||
1026 | nvgpu_mutex_release(&ch->sync_lock); | ||
1027 | return -ENOMEM; | ||
1028 | } | ||
1029 | nvgpu_mutex_release(&ch->sync_lock); | ||
1030 | |||
1031 | if (g->ops.fifo.resetup_ramfc) { | ||
1032 | err = g->ops.fifo.resetup_ramfc(ch); | ||
1033 | if (err) | ||
1034 | return err; | ||
1035 | } | ||
1036 | } | ||
1037 | |||
1038 | args->syncpoint_id = ch->user_sync->syncpt_id(ch->user_sync); | ||
1039 | args->syncpoint_max = nvgpu_nvhost_syncpt_read_maxval(g->nvhost_dev, | ||
1040 | args->syncpoint_id); | ||
1041 | if (nvgpu_is_enabled(g, NVGPU_SUPPORT_SYNCPOINT_ADDRESS)) | ||
1042 | args->gpu_va = ch->user_sync->syncpt_address(ch->user_sync); | ||
1043 | else | ||
1044 | args->gpu_va = 0; | ||
1045 | |||
1046 | return 0; | ||
1047 | #else | ||
1048 | return -EINVAL; | ||
1049 | #endif | ||
1050 | } | ||
1051 | |||
1052 | long gk20a_channel_ioctl(struct file *filp, | ||
1053 | unsigned int cmd, unsigned long arg) | ||
1054 | { | ||
1055 | struct channel_priv *priv = filp->private_data; | ||
1056 | struct channel_gk20a *ch = priv->c; | ||
1057 | struct device *dev = dev_from_gk20a(ch->g); | ||
1058 | u8 buf[NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE] = {0}; | ||
1059 | int err = 0; | ||
1060 | struct gk20a *g = ch->g; | ||
1061 | |||
1062 | nvgpu_log_fn(g, "start %d", _IOC_NR(cmd)); | ||
1063 | |||
1064 | if ((_IOC_TYPE(cmd) != NVGPU_IOCTL_MAGIC) || | ||
1065 | (_IOC_NR(cmd) == 0) || | ||
1066 | (_IOC_NR(cmd) > NVGPU_IOCTL_CHANNEL_LAST) || | ||
1067 | (_IOC_SIZE(cmd) > NVGPU_IOCTL_CHANNEL_MAX_ARG_SIZE)) | ||
1068 | return -EINVAL; | ||
1069 | |||
1070 | if (_IOC_DIR(cmd) & _IOC_WRITE) { | ||
1071 | if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd))) | ||
1072 | return -EFAULT; | ||
1073 | } | ||
1074 | |||
1075 | /* take a ref or return timeout if channel refs can't be taken */ | ||
1076 | ch = gk20a_channel_get(ch); | ||
1077 | if (!ch) | ||
1078 | return -ETIMEDOUT; | ||
1079 | |||
1080 | /* protect our sanity for threaded userspace - most of the channel is | ||
1081 | * not thread safe */ | ||
1082 | nvgpu_mutex_acquire(&ch->ioctl_lock); | ||
1083 | |||
1084 | /* this ioctl call keeps a ref to the file which keeps a ref to the | ||
1085 | * channel */ | ||
1086 | |||
1087 | switch (cmd) { | ||
1088 | case NVGPU_IOCTL_CHANNEL_OPEN: | ||
1089 | err = gk20a_channel_open_ioctl(ch->g, | ||
1090 | (struct nvgpu_channel_open_args *)buf); | ||
1091 | break; | ||
1092 | case NVGPU_IOCTL_CHANNEL_SET_NVMAP_FD: | ||
1093 | break; | ||
1094 | case NVGPU_IOCTL_CHANNEL_ALLOC_OBJ_CTX: | ||
1095 | { | ||
1096 | struct nvgpu_alloc_obj_ctx_args *args = | ||
1097 | (struct nvgpu_alloc_obj_ctx_args *)buf; | ||
1098 | |||
1099 | err = gk20a_busy(ch->g); | ||
1100 | if (err) { | ||
1101 | dev_err(dev, | ||
1102 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1103 | __func__, cmd); | ||
1104 | break; | ||
1105 | } | ||
1106 | err = nvgpu_ioctl_channel_alloc_obj_ctx(ch, args->class_num, args->flags); | ||
1107 | gk20a_idle(ch->g); | ||
1108 | break; | ||
1109 | } | ||
1110 | case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO_EX: | ||
1111 | { | ||
1112 | struct nvgpu_alloc_gpfifo_ex_args *alloc_gpfifo_ex_args = | ||
1113 | (struct nvgpu_alloc_gpfifo_ex_args *)buf; | ||
1114 | struct nvgpu_gpfifo_args gpfifo_args; | ||
1115 | |||
1116 | nvgpu_get_gpfifo_ex_args(alloc_gpfifo_ex_args, &gpfifo_args); | ||
1117 | |||
1118 | err = gk20a_busy(ch->g); | ||
1119 | if (err) { | ||
1120 | dev_err(dev, | ||
1121 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1122 | __func__, cmd); | ||
1123 | break; | ||
1124 | } | ||
1125 | |||
1126 | if (!is_power_of_2(alloc_gpfifo_ex_args->num_entries)) { | ||
1127 | err = -EINVAL; | ||
1128 | gk20a_idle(ch->g); | ||
1129 | break; | ||
1130 | } | ||
1131 | err = gk20a_channel_alloc_gpfifo(ch, &gpfifo_args); | ||
1132 | gk20a_idle(ch->g); | ||
1133 | break; | ||
1134 | } | ||
1135 | case NVGPU_IOCTL_CHANNEL_ALLOC_GPFIFO: | ||
1136 | { | ||
1137 | struct nvgpu_alloc_gpfifo_args *alloc_gpfifo_args = | ||
1138 | (struct nvgpu_alloc_gpfifo_args *)buf; | ||
1139 | struct nvgpu_gpfifo_args gpfifo_args; | ||
1140 | |||
1141 | nvgpu_get_gpfifo_args(alloc_gpfifo_args, &gpfifo_args); | ||
1142 | |||
1143 | err = gk20a_busy(ch->g); | ||
1144 | if (err) { | ||
1145 | dev_err(dev, | ||
1146 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1147 | __func__, cmd); | ||
1148 | break; | ||
1149 | } | ||
1150 | |||
1151 | err = gk20a_channel_alloc_gpfifo(ch, &gpfifo_args); | ||
1152 | gk20a_idle(ch->g); | ||
1153 | break; | ||
1154 | } | ||
1155 | case NVGPU_IOCTL_CHANNEL_SUBMIT_GPFIFO: | ||
1156 | err = gk20a_ioctl_channel_submit_gpfifo(ch, | ||
1157 | (struct nvgpu_submit_gpfifo_args *)buf); | ||
1158 | break; | ||
1159 | case NVGPU_IOCTL_CHANNEL_WAIT: | ||
1160 | err = gk20a_busy(ch->g); | ||
1161 | if (err) { | ||
1162 | dev_err(dev, | ||
1163 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1164 | __func__, cmd); | ||
1165 | break; | ||
1166 | } | ||
1167 | |||
1168 | /* waiting is thread-safe, not dropping this mutex could | ||
1169 | * deadlock in certain conditions */ | ||
1170 | nvgpu_mutex_release(&ch->ioctl_lock); | ||
1171 | |||
1172 | err = gk20a_channel_wait(ch, | ||
1173 | (struct nvgpu_wait_args *)buf); | ||
1174 | |||
1175 | nvgpu_mutex_acquire(&ch->ioctl_lock); | ||
1176 | |||
1177 | gk20a_idle(ch->g); | ||
1178 | break; | ||
1179 | case NVGPU_IOCTL_CHANNEL_ZCULL_BIND: | ||
1180 | err = gk20a_busy(ch->g); | ||
1181 | if (err) { | ||
1182 | dev_err(dev, | ||
1183 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1184 | __func__, cmd); | ||
1185 | break; | ||
1186 | } | ||
1187 | err = gk20a_channel_zcull_bind(ch, | ||
1188 | (struct nvgpu_zcull_bind_args *)buf); | ||
1189 | gk20a_idle(ch->g); | ||
1190 | break; | ||
1191 | case NVGPU_IOCTL_CHANNEL_SET_ERROR_NOTIFIER: | ||
1192 | err = gk20a_busy(ch->g); | ||
1193 | if (err) { | ||
1194 | dev_err(dev, | ||
1195 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1196 | __func__, cmd); | ||
1197 | break; | ||
1198 | } | ||
1199 | err = gk20a_init_error_notifier(ch, | ||
1200 | (struct nvgpu_set_error_notifier *)buf); | ||
1201 | gk20a_idle(ch->g); | ||
1202 | break; | ||
1203 | #ifdef CONFIG_GK20A_CYCLE_STATS | ||
1204 | case NVGPU_IOCTL_CHANNEL_CYCLE_STATS: | ||
1205 | err = gk20a_busy(ch->g); | ||
1206 | if (err) { | ||
1207 | dev_err(dev, | ||
1208 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1209 | __func__, cmd); | ||
1210 | break; | ||
1211 | } | ||
1212 | err = gk20a_channel_cycle_stats(ch, | ||
1213 | (struct nvgpu_cycle_stats_args *)buf); | ||
1214 | gk20a_idle(ch->g); | ||
1215 | break; | ||
1216 | #endif | ||
1217 | case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT: | ||
1218 | { | ||
1219 | u32 timeout = | ||
1220 | (u32)((struct nvgpu_set_timeout_args *)buf)->timeout; | ||
1221 | nvgpu_log(g, gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d", | ||
1222 | timeout, ch->chid); | ||
1223 | ch->timeout_ms_max = timeout; | ||
1224 | gk20a_channel_trace_sched_param( | ||
1225 | trace_gk20a_channel_set_timeout, ch); | ||
1226 | break; | ||
1227 | } | ||
1228 | case NVGPU_IOCTL_CHANNEL_SET_TIMEOUT_EX: | ||
1229 | { | ||
1230 | u32 timeout = | ||
1231 | (u32)((struct nvgpu_set_timeout_args *)buf)->timeout; | ||
1232 | bool timeout_debug_dump = !((u32) | ||
1233 | ((struct nvgpu_set_timeout_ex_args *)buf)->flags & | ||
1234 | (1 << NVGPU_TIMEOUT_FLAG_DISABLE_DUMP)); | ||
1235 | nvgpu_log(g, gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d", | ||
1236 | timeout, ch->chid); | ||
1237 | ch->timeout_ms_max = timeout; | ||
1238 | ch->timeout_debug_dump = timeout_debug_dump; | ||
1239 | gk20a_channel_trace_sched_param( | ||
1240 | trace_gk20a_channel_set_timeout, ch); | ||
1241 | break; | ||
1242 | } | ||
1243 | case NVGPU_IOCTL_CHANNEL_GET_TIMEDOUT: | ||
1244 | ((struct nvgpu_get_param_args *)buf)->value = | ||
1245 | ch->has_timedout; | ||
1246 | break; | ||
1247 | case NVGPU_IOCTL_CHANNEL_ENABLE: | ||
1248 | err = gk20a_busy(ch->g); | ||
1249 | if (err) { | ||
1250 | dev_err(dev, | ||
1251 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1252 | __func__, cmd); | ||
1253 | break; | ||
1254 | } | ||
1255 | if (ch->g->ops.fifo.enable_channel) | ||
1256 | ch->g->ops.fifo.enable_channel(ch); | ||
1257 | else | ||
1258 | err = -ENOSYS; | ||
1259 | gk20a_idle(ch->g); | ||
1260 | break; | ||
1261 | case NVGPU_IOCTL_CHANNEL_DISABLE: | ||
1262 | err = gk20a_busy(ch->g); | ||
1263 | if (err) { | ||
1264 | dev_err(dev, | ||
1265 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1266 | __func__, cmd); | ||
1267 | break; | ||
1268 | } | ||
1269 | if (ch->g->ops.fifo.disable_channel) | ||
1270 | ch->g->ops.fifo.disable_channel(ch); | ||
1271 | else | ||
1272 | err = -ENOSYS; | ||
1273 | gk20a_idle(ch->g); | ||
1274 | break; | ||
1275 | case NVGPU_IOCTL_CHANNEL_PREEMPT: | ||
1276 | err = gk20a_busy(ch->g); | ||
1277 | if (err) { | ||
1278 | dev_err(dev, | ||
1279 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1280 | __func__, cmd); | ||
1281 | break; | ||
1282 | } | ||
1283 | err = gk20a_fifo_preempt(ch->g, ch); | ||
1284 | gk20a_idle(ch->g); | ||
1285 | break; | ||
1286 | case NVGPU_IOCTL_CHANNEL_RESCHEDULE_RUNLIST: | ||
1287 | if (!capable(CAP_SYS_NICE)) { | ||
1288 | err = -EPERM; | ||
1289 | break; | ||
1290 | } | ||
1291 | if (!ch->g->ops.fifo.reschedule_runlist) { | ||
1292 | err = -ENOSYS; | ||
1293 | break; | ||
1294 | } | ||
1295 | err = gk20a_busy(ch->g); | ||
1296 | if (err) { | ||
1297 | dev_err(dev, | ||
1298 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1299 | __func__, cmd); | ||
1300 | break; | ||
1301 | } | ||
1302 | err = ch->g->ops.fifo.reschedule_runlist(ch, | ||
1303 | NVGPU_RESCHEDULE_RUNLIST_PREEMPT_NEXT & | ||
1304 | ((struct nvgpu_reschedule_runlist_args *)buf)->flags); | ||
1305 | gk20a_idle(ch->g); | ||
1306 | break; | ||
1307 | case NVGPU_IOCTL_CHANNEL_FORCE_RESET: | ||
1308 | err = gk20a_busy(ch->g); | ||
1309 | if (err) { | ||
1310 | dev_err(dev, | ||
1311 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1312 | __func__, cmd); | ||
1313 | break; | ||
1314 | } | ||
1315 | err = ch->g->ops.fifo.force_reset_ch(ch, | ||
1316 | NVGPU_ERR_NOTIFIER_RESETCHANNEL_VERIF_ERROR, true); | ||
1317 | gk20a_idle(ch->g); | ||
1318 | break; | ||
1319 | #ifdef CONFIG_GK20A_CYCLE_STATS | ||
1320 | case NVGPU_IOCTL_CHANNEL_CYCLE_STATS_SNAPSHOT: | ||
1321 | err = gk20a_busy(ch->g); | ||
1322 | if (err) { | ||
1323 | dev_err(dev, | ||
1324 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1325 | __func__, cmd); | ||
1326 | break; | ||
1327 | } | ||
1328 | err = gk20a_channel_cycle_stats_snapshot(ch, | ||
1329 | (struct nvgpu_cycle_stats_snapshot_args *)buf); | ||
1330 | gk20a_idle(ch->g); | ||
1331 | break; | ||
1332 | #endif | ||
1333 | case NVGPU_IOCTL_CHANNEL_WDT: | ||
1334 | err = gk20a_channel_set_wdt_status(ch, | ||
1335 | (struct nvgpu_channel_wdt_args *)buf); | ||
1336 | break; | ||
1337 | case NVGPU_IOCTL_CHANNEL_SET_PREEMPTION_MODE: | ||
1338 | err = nvgpu_ioctl_channel_set_preemption_mode(ch, | ||
1339 | ((struct nvgpu_preemption_mode_args *)buf)->graphics_preempt_mode, | ||
1340 | ((struct nvgpu_preemption_mode_args *)buf)->compute_preempt_mode); | ||
1341 | break; | ||
1342 | case NVGPU_IOCTL_CHANNEL_SET_BOOSTED_CTX: | ||
1343 | if (ch->g->ops.gr.set_boosted_ctx) { | ||
1344 | bool boost = | ||
1345 | ((struct nvgpu_boosted_ctx_args *)buf)->boost; | ||
1346 | |||
1347 | err = gk20a_busy(ch->g); | ||
1348 | if (err) { | ||
1349 | dev_err(dev, | ||
1350 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1351 | __func__, cmd); | ||
1352 | break; | ||
1353 | } | ||
1354 | err = ch->g->ops.gr.set_boosted_ctx(ch, boost); | ||
1355 | gk20a_idle(ch->g); | ||
1356 | } else { | ||
1357 | err = -EINVAL; | ||
1358 | } | ||
1359 | break; | ||
1360 | case NVGPU_IOCTL_CHANNEL_GET_USER_SYNCPOINT: | ||
1361 | err = gk20a_busy(ch->g); | ||
1362 | if (err) { | ||
1363 | dev_err(dev, | ||
1364 | "%s: failed to host gk20a for ioctl cmd: 0x%x", | ||
1365 | __func__, cmd); | ||
1366 | break; | ||
1367 | } | ||
1368 | err = nvgpu_ioctl_channel_get_user_syncpoint(ch, | ||
1369 | (struct nvgpu_get_user_syncpoint_args *)buf); | ||
1370 | gk20a_idle(ch->g); | ||
1371 | break; | ||
1372 | default: | ||
1373 | dev_dbg(dev, "unrecognized ioctl cmd: 0x%x", cmd); | ||
1374 | err = -ENOTTY; | ||
1375 | break; | ||
1376 | } | ||
1377 | |||
1378 | if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ)) | ||
1379 | err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd)); | ||
1380 | |||
1381 | nvgpu_mutex_release(&ch->ioctl_lock); | ||
1382 | |||
1383 | gk20a_channel_put(ch); | ||
1384 | |||
1385 | nvgpu_log_fn(g, "end"); | ||
1386 | |||
1387 | return err; | ||
1388 | } | ||