aboutsummaryrefslogtreecommitdiffstats
path: root/include/os/linux/ioctl_ctrl.c
diff options
context:
space:
mode:
Diffstat (limited to 'include/os/linux/ioctl_ctrl.c')
-rw-r--r--include/os/linux/ioctl_ctrl.c2106
1 files changed, 2106 insertions, 0 deletions
diff --git a/include/os/linux/ioctl_ctrl.c b/include/os/linux/ioctl_ctrl.c
new file mode 100644
index 0000000..ee141ff
--- /dev/null
+++ b/include/os/linux/ioctl_ctrl.c
@@ -0,0 +1,2106 @@
1/*
2 * Copyright (c) 2011-2020, NVIDIA Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#include <linux/uaccess.h>
18#include <linux/cdev.h>
19#include <linux/file.h>
20#include <linux/anon_inodes.h>
21#include <linux/fs.h>
22#include <linux/pm_runtime.h>
23#include <uapi/linux/nvgpu.h>
24
25#include <nvgpu/bitops.h>
26#include <nvgpu/kmem.h>
27#include <nvgpu/bug.h>
28#include <nvgpu/ptimer.h>
29#include <nvgpu/vidmem.h>
30#include <nvgpu/log.h>
31#include <nvgpu/enabled.h>
32#include <nvgpu/sizes.h>
33#include <nvgpu/list.h>
34#include <nvgpu/clk_arb.h>
35#include <nvgpu/gk20a.h>
36#include <nvgpu/channel.h>
37
38#include "ioctl_ctrl.h"
39#include "ioctl_dbg.h"
40#include "ioctl_as.h"
41#include "ioctl_tsg.h"
42#include "ioctl_channel.h"
43#include "gk20a/fence_gk20a.h"
44
45#include "platform_gk20a.h"
46#include "os_linux.h"
47#include "dmabuf.h"
48#include "channel.h"
49#include "dmabuf_vidmem.h"
50
51#define HZ_TO_MHZ(a) ((a > 0xF414F9CD7ULL) ? 0xffff : (a >> 32) ? \
52 (u32) ((a * 0x10C8ULL) >> 32) : (u16) ((u32) a/MHZ))
53#define MHZ_TO_HZ(a) ((u64)a * MHZ)
54
55struct gk20a_ctrl_priv {
56 struct device *dev;
57 struct gk20a *g;
58 struct nvgpu_clk_session *clk_session;
59
60 struct nvgpu_list_node list;
61 struct {
62 struct vm_area_struct *vma;
63 unsigned long flags;
64 bool vma_mapped;
65 } usermode_vma;
66};
67
68static inline struct gk20a_ctrl_priv *
69gk20a_ctrl_priv_from_list(struct nvgpu_list_node *node)
70{
71 return (struct gk20a_ctrl_priv *)
72 ((uintptr_t)node - offsetof(struct gk20a_ctrl_priv, list));
73}
74
75static u32 gk20a_as_translate_as_alloc_flags(struct gk20a *g, u32 flags)
76{
77 u32 core_flags = 0;
78
79 if (flags & NVGPU_GPU_IOCTL_ALLOC_AS_FLAGS_USERSPACE_MANAGED)
80 core_flags |= NVGPU_AS_ALLOC_USERSPACE_MANAGED;
81
82 return core_flags;
83}
84
85int gk20a_ctrl_dev_open(struct inode *inode, struct file *filp)
86{
87 struct nvgpu_os_linux *l;
88 struct gk20a *g;
89 struct gk20a_ctrl_priv *priv;
90 int err = 0;
91
92 l = container_of(inode->i_cdev,
93 struct nvgpu_os_linux, ctrl.cdev);
94 g = gk20a_get(&l->g);
95 if (!g)
96 return -ENODEV;
97
98 nvgpu_log_fn(g, " ");
99
100 priv = nvgpu_kzalloc(g, sizeof(struct gk20a_ctrl_priv));
101 if (!priv) {
102 err = -ENOMEM;
103 goto free_ref;
104 }
105 filp->private_data = priv;
106 priv->dev = dev_from_gk20a(g);
107 /*
108 * We dont close the arbiter fd's after driver teardown to support
109 * GPU_LOST events, so we store g here, instead of dereferencing the
110 * dev structure on teardown
111 */
112 priv->g = g;
113
114 if (!g->sw_ready) {
115 err = gk20a_busy(g);
116 if (err)
117 goto free_ref;
118 gk20a_idle(g);
119 }
120
121 err = nvgpu_clk_arb_init_session(g, &priv->clk_session);
122free_ref:
123 if (err != 0) {
124 gk20a_put(g);
125 if (priv)
126 nvgpu_kfree(g, priv);
127 } else {
128 nvgpu_mutex_acquire(&l->ctrl.privs_lock);
129 nvgpu_list_add(&priv->list, &l->ctrl.privs);
130 nvgpu_mutex_release(&l->ctrl.privs_lock);
131 }
132
133 return err;
134}
135int gk20a_ctrl_dev_release(struct inode *inode, struct file *filp)
136{
137 struct gk20a_ctrl_priv *priv = filp->private_data;
138 struct gk20a *g = priv->g;
139 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
140
141 nvgpu_log_fn(g, " ");
142
143 nvgpu_mutex_acquire(&l->ctrl.privs_lock);
144 nvgpu_list_del(&priv->list);
145 nvgpu_mutex_release(&l->ctrl.privs_lock);
146
147 if (priv->clk_session)
148 nvgpu_clk_arb_release_session(g, priv->clk_session);
149
150 gk20a_put(g);
151 nvgpu_kfree(g, priv);
152
153 return 0;
154}
155
156struct nvgpu_flags_mapping {
157 u64 ioctl_flag;
158 int enabled_flag;
159};
160
161static struct nvgpu_flags_mapping flags_mapping[] = {
162 {NVGPU_GPU_FLAGS_CAN_RAILGATE,
163 NVGPU_CAN_RAILGATE},
164 {NVGPU_GPU_FLAGS_HAS_SYNCPOINTS,
165 NVGPU_HAS_SYNCPOINTS},
166 {NVGPU_GPU_FLAGS_SUPPORT_PARTIAL_MAPPINGS,
167 NVGPU_SUPPORT_PARTIAL_MAPPINGS},
168 {NVGPU_GPU_FLAGS_SUPPORT_SPARSE_ALLOCS,
169 NVGPU_SUPPORT_SPARSE_ALLOCS},
170 {NVGPU_GPU_FLAGS_SUPPORT_SYNC_FENCE_FDS,
171 NVGPU_SUPPORT_SYNC_FENCE_FDS},
172 {NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS,
173 NVGPU_SUPPORT_CYCLE_STATS},
174 {NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS_SNAPSHOT,
175 NVGPU_SUPPORT_CYCLE_STATS_SNAPSHOT},
176 {NVGPU_GPU_FLAGS_SUPPORT_USERSPACE_MANAGED_AS,
177 NVGPU_SUPPORT_USERSPACE_MANAGED_AS},
178 {NVGPU_GPU_FLAGS_SUPPORT_TSG,
179 NVGPU_SUPPORT_TSG},
180 {NVGPU_GPU_FLAGS_SUPPORT_CLOCK_CONTROLS,
181 NVGPU_SUPPORT_CLOCK_CONTROLS},
182 {NVGPU_GPU_FLAGS_SUPPORT_GET_VOLTAGE,
183 NVGPU_SUPPORT_GET_VOLTAGE},
184 {NVGPU_GPU_FLAGS_SUPPORT_GET_CURRENT,
185 NVGPU_SUPPORT_GET_CURRENT},
186 {NVGPU_GPU_FLAGS_SUPPORT_GET_POWER,
187 NVGPU_SUPPORT_GET_POWER},
188 {NVGPU_GPU_FLAGS_SUPPORT_GET_TEMPERATURE,
189 NVGPU_SUPPORT_GET_TEMPERATURE},
190 {NVGPU_GPU_FLAGS_SUPPORT_SET_THERM_ALERT_LIMIT,
191 NVGPU_SUPPORT_SET_THERM_ALERT_LIMIT},
192 {NVGPU_GPU_FLAGS_SUPPORT_DEVICE_EVENTS,
193 NVGPU_SUPPORT_DEVICE_EVENTS},
194 {NVGPU_GPU_FLAGS_SUPPORT_FECS_CTXSW_TRACE,
195 NVGPU_SUPPORT_FECS_CTXSW_TRACE},
196 {NVGPU_GPU_FLAGS_SUPPORT_DETERMINISTIC_SUBMIT_NO_JOBTRACKING,
197 NVGPU_SUPPORT_DETERMINISTIC_SUBMIT_NO_JOBTRACKING},
198 {NVGPU_GPU_FLAGS_SUPPORT_DETERMINISTIC_SUBMIT_FULL,
199 NVGPU_SUPPORT_DETERMINISTIC_SUBMIT_FULL},
200 {NVGPU_GPU_FLAGS_SUPPORT_DETERMINISTIC_OPTS,
201 NVGPU_SUPPORT_DETERMINISTIC_OPTS},
202 {NVGPU_GPU_FLAGS_SUPPORT_SYNCPOINT_ADDRESS,
203 NVGPU_SUPPORT_SYNCPOINT_ADDRESS},
204 {NVGPU_GPU_FLAGS_SUPPORT_USER_SYNCPOINT,
205 NVGPU_SUPPORT_USER_SYNCPOINT},
206 {NVGPU_GPU_FLAGS_SUPPORT_USERMODE_SUBMIT,
207 NVGPU_SUPPORT_USERMODE_SUBMIT},
208 {NVGPU_GPU_FLAGS_SUPPORT_IO_COHERENCE,
209 NVGPU_SUPPORT_IO_COHERENCE},
210 {NVGPU_GPU_FLAGS_SUPPORT_RESCHEDULE_RUNLIST,
211 NVGPU_SUPPORT_RESCHEDULE_RUNLIST},
212 {NVGPU_GPU_FLAGS_SUPPORT_MAP_DIRECT_KIND_CTRL,
213 NVGPU_SUPPORT_MAP_DIRECT_KIND_CTRL},
214 {NVGPU_GPU_FLAGS_ECC_ENABLED_SM_LRF,
215 NVGPU_ECC_ENABLED_SM_LRF},
216 {NVGPU_GPU_FLAGS_ECC_ENABLED_SM_SHM,
217 NVGPU_ECC_ENABLED_SM_SHM},
218 {NVGPU_GPU_FLAGS_ECC_ENABLED_TEX,
219 NVGPU_ECC_ENABLED_TEX},
220 {NVGPU_GPU_FLAGS_ECC_ENABLED_LTC,
221 NVGPU_ECC_ENABLED_LTC},
222 {NVGPU_GPU_FLAGS_SUPPORT_TSG_SUBCONTEXTS,
223 NVGPU_SUPPORT_TSG_SUBCONTEXTS},
224 {NVGPU_GPU_FLAGS_SUPPORT_SCG,
225 NVGPU_SUPPORT_SCG},
226 {NVGPU_GPU_FLAGS_SUPPORT_VPR,
227 NVGPU_SUPPORT_VPR},
228 {NVGPU_GPU_FLAGS_SUPPORT_SET_CTX_MMU_DEBUG_MODE,
229 NVGPU_SUPPORT_SET_CTX_MMU_DEBUG_MODE},
230};
231
232static u64 nvgpu_ctrl_ioctl_gpu_characteristics_flags(struct gk20a *g)
233{
234 unsigned int i;
235 u64 ioctl_flags = 0;
236
237 for (i = 0; i < sizeof(flags_mapping)/sizeof(*flags_mapping); i++) {
238 if (nvgpu_is_enabled(g, flags_mapping[i].enabled_flag))
239 ioctl_flags |= flags_mapping[i].ioctl_flag;
240 }
241
242 if (!capable(CAP_SYS_NICE)) {
243 ioctl_flags &= ~NVGPU_GPU_FLAGS_SUPPORT_RESCHEDULE_RUNLIST;
244 }
245
246 return ioctl_flags;
247}
248
249static void nvgpu_set_preemption_mode_flags(struct gk20a *g,
250 struct nvgpu_gpu_characteristics *gpu)
251{
252 struct nvgpu_preemption_modes_rec preemption_mode_rec;
253
254 g->ops.gr.get_preemption_mode_flags(g, &preemption_mode_rec);
255
256 gpu->graphics_preemption_mode_flags =
257 nvgpu_get_ioctl_graphics_preempt_mode_flags(
258 preemption_mode_rec.graphics_preemption_mode_flags);
259 gpu->compute_preemption_mode_flags =
260 nvgpu_get_ioctl_compute_preempt_mode_flags(
261 preemption_mode_rec.compute_preemption_mode_flags);
262
263 gpu->default_graphics_preempt_mode =
264 nvgpu_get_ioctl_graphics_preempt_mode(
265 preemption_mode_rec.default_graphics_preempt_mode);
266 gpu->default_compute_preempt_mode =
267 nvgpu_get_ioctl_compute_preempt_mode(
268 preemption_mode_rec.default_compute_preempt_mode);
269}
270
271static long
272gk20a_ctrl_ioctl_gpu_characteristics(
273 struct gk20a *g,
274 struct nvgpu_gpu_get_characteristics *request)
275{
276 struct nvgpu_gpu_characteristics gpu;
277 long err = 0;
278
279 if (gk20a_busy(g)) {
280 nvgpu_err(g, "failed to power on gpu");
281 return -EINVAL;
282 }
283
284 memset(&gpu, 0, sizeof(gpu));
285
286 gpu.L2_cache_size = g->ops.ltc.determine_L2_size_bytes(g);
287 gpu.on_board_video_memory_size = 0; /* integrated GPU */
288
289 gpu.num_gpc = g->gr.gpc_count;
290 gpu.max_gpc_count = g->gr.max_gpc_count;
291
292 gpu.num_tpc_per_gpc = g->gr.max_tpc_per_gpc_count;
293
294 gpu.bus_type = NVGPU_GPU_BUS_TYPE_AXI; /* always AXI for now */
295
296 gpu.compression_page_size = g->ops.fb.compression_page_size(g);
297
298 if (g->ops.gr.get_gpc_mask) {
299 gpu.gpc_mask = g->ops.gr.get_gpc_mask(g);
300 } else {
301 gpu.gpc_mask = BIT32(g->gr.gpc_count) - 1;
302 }
303
304 gpu.flags = nvgpu_ctrl_ioctl_gpu_characteristics_flags(g);
305
306 gpu.arch = g->params.gpu_arch;
307 gpu.impl = g->params.gpu_impl;
308 gpu.rev = g->params.gpu_rev;
309 gpu.reg_ops_limit = NVGPU_IOCTL_DBG_REG_OPS_LIMIT;
310 gpu.map_buffer_batch_limit = nvgpu_is_enabled(g, NVGPU_SUPPORT_MAP_BUFFER_BATCH) ?
311 NVGPU_IOCTL_AS_MAP_BUFFER_BATCH_LIMIT : 0;
312 gpu.twod_class = g->ops.get_litter_value(g, GPU_LIT_TWOD_CLASS);
313 gpu.threed_class = g->ops.get_litter_value(g, GPU_LIT_THREED_CLASS);
314 gpu.compute_class = g->ops.get_litter_value(g, GPU_LIT_COMPUTE_CLASS);
315 gpu.gpfifo_class = g->ops.get_litter_value(g, GPU_LIT_GPFIFO_CLASS);
316 gpu.inline_to_memory_class =
317 g->ops.get_litter_value(g, GPU_LIT_I2M_CLASS);
318 gpu.dma_copy_class =
319 g->ops.get_litter_value(g, GPU_LIT_DMA_COPY_CLASS);
320
321 gpu.vbios_version = g->bios.vbios_version;
322 gpu.vbios_oem_version = g->bios.vbios_oem_version;
323
324 gpu.big_page_size = nvgpu_mm_get_default_big_page_size(g);
325 gpu.pde_coverage_bit_count =
326 g->ops.mm.get_mmu_levels(g, gpu.big_page_size)[0].lo_bit[0];
327 gpu.available_big_page_sizes = nvgpu_mm_get_available_big_page_sizes(g);
328
329 gpu.sm_arch_sm_version = g->params.sm_arch_sm_version;
330 gpu.sm_arch_spa_version = g->params.sm_arch_spa_version;
331 gpu.sm_arch_warp_count = g->params.sm_arch_warp_count;
332
333 gpu.max_css_buffer_size = g->gr.max_css_buffer_size;
334
335 gpu.gpu_ioctl_nr_last = NVGPU_GPU_IOCTL_LAST;
336 gpu.tsg_ioctl_nr_last = NVGPU_TSG_IOCTL_LAST;
337 gpu.dbg_gpu_ioctl_nr_last = NVGPU_DBG_GPU_IOCTL_LAST;
338 gpu.ioctl_channel_nr_last = NVGPU_IOCTL_CHANNEL_LAST;
339 gpu.as_ioctl_nr_last = NVGPU_AS_IOCTL_LAST;
340 gpu.event_ioctl_nr_last = NVGPU_EVENT_IOCTL_LAST;
341 gpu.gpu_va_bit_count = 40;
342
343 strlcpy(gpu.chipname, g->name, sizeof(gpu.chipname));
344 gpu.max_fbps_count = g->ops.gr.get_max_fbps_count(g);
345 gpu.fbp_en_mask = g->ops.gr.get_fbp_en_mask(g);
346 gpu.max_ltc_per_fbp = g->ops.gr.get_max_ltc_per_fbp(g);
347 gpu.max_lts_per_ltc = g->ops.gr.get_max_lts_per_ltc(g);
348 gpu.gr_compbit_store_base_hw = g->gr.compbit_store.base_hw;
349 gpu.gr_gobs_per_comptagline_per_slice =
350 g->gr.gobs_per_comptagline_per_slice;
351 gpu.num_ltc = g->ltc_count;
352 gpu.lts_per_ltc = g->gr.slices_per_ltc;
353 gpu.cbc_cache_line_size = g->gr.cacheline_size;
354 gpu.cbc_comptags_per_line = g->gr.comptags_per_cacheline;
355
356 if (g->ops.clk.get_maxrate)
357 gpu.max_freq = g->ops.clk.get_maxrate(g, CTRL_CLK_DOMAIN_GPCCLK);
358
359 gpu.local_video_memory_size = g->mm.vidmem.size;
360
361 gpu.pci_vendor_id = g->pci_vendor_id;
362 gpu.pci_device_id = g->pci_device_id;
363 gpu.pci_subsystem_vendor_id = g->pci_subsystem_vendor_id;
364 gpu.pci_subsystem_device_id = g->pci_subsystem_device_id;
365 gpu.pci_class = g->pci_class;
366 gpu.pci_revision = g->pci_revision;
367
368 nvgpu_set_preemption_mode_flags(g, &gpu);
369
370 if (request->gpu_characteristics_buf_size > 0) {
371 size_t write_size = sizeof(gpu);
372
373 nvgpu_speculation_barrier();
374 if (write_size > request->gpu_characteristics_buf_size)
375 write_size = request->gpu_characteristics_buf_size;
376
377 err = copy_to_user((void __user *)(uintptr_t)
378 request->gpu_characteristics_buf_addr,
379 &gpu, write_size);
380 }
381
382 if (err == 0)
383 request->gpu_characteristics_buf_size = sizeof(gpu);
384
385 gk20a_idle(g);
386
387 return err;
388}
389
390static int gk20a_ctrl_prepare_compressible_read(
391 struct gk20a *g,
392 struct nvgpu_gpu_prepare_compressible_read_args *args)
393{
394 int ret = -ENOSYS;
395
396#ifdef CONFIG_NVGPU_SUPPORT_CDE
397 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
398 struct nvgpu_channel_fence fence;
399 struct gk20a_fence *fence_out = NULL;
400 int submit_flags = nvgpu_submit_gpfifo_user_flags_to_common_flags(
401 args->submit_flags);
402 int fd = -1;
403
404 fence.id = args->fence.syncpt_id;
405 fence.value = args->fence.syncpt_value;
406
407 /* Try and allocate an fd here*/
408 if ((submit_flags & NVGPU_SUBMIT_FLAGS_FENCE_GET)
409 && (submit_flags & NVGPU_SUBMIT_FLAGS_SYNC_FENCE)) {
410 fd = get_unused_fd_flags(O_RDWR);
411 if (fd < 0)
412 return fd;
413 }
414
415 ret = gk20a_prepare_compressible_read(l, args->handle,
416 args->request_compbits, args->offset,
417 args->compbits_hoffset, args->compbits_voffset,
418 args->scatterbuffer_offset,
419 args->width, args->height, args->block_height_log2,
420 submit_flags, &fence, &args->valid_compbits,
421 &args->zbc_color, &fence_out);
422
423 if (ret) {
424 if (fd != -1)
425 put_unused_fd(fd);
426 return ret;
427 }
428
429 /* Convert fence_out to something we can pass back to user space. */
430 if (submit_flags & NVGPU_SUBMIT_FLAGS_FENCE_GET) {
431 if (submit_flags & NVGPU_SUBMIT_FLAGS_SYNC_FENCE) {
432 if (fence_out) {
433 ret = gk20a_fence_install_fd(fence_out, fd);
434 if (ret)
435 put_unused_fd(fd);
436 else
437 args->fence.fd = fd;
438 } else {
439 args->fence.fd = -1;
440 put_unused_fd(fd);
441 }
442 } else {
443 if (fence_out) {
444 args->fence.syncpt_id = fence_out->syncpt_id;
445 args->fence.syncpt_value =
446 fence_out->syncpt_value;
447 } else {
448 args->fence.syncpt_id = -1;
449 args->fence.syncpt_value = 0;
450 }
451 }
452 }
453 gk20a_fence_put(fence_out);
454#endif
455
456 return ret;
457}
458
459static int gk20a_ctrl_mark_compressible_write(
460 struct gk20a *g,
461 struct nvgpu_gpu_mark_compressible_write_args *args)
462{
463 int ret = -ENOSYS;
464
465#ifdef CONFIG_NVGPU_SUPPORT_CDE
466 ret = gk20a_mark_compressible_write(g, args->handle,
467 args->valid_compbits, args->offset, args->zbc_color);
468#endif
469
470 return ret;
471}
472
473static int gk20a_ctrl_alloc_as(
474 struct gk20a *g,
475 struct nvgpu_alloc_as_args *args)
476{
477 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
478 struct gk20a_as_share *as_share;
479 int err;
480 int fd;
481 struct file *file;
482 char name[64];
483
484 err = get_unused_fd_flags(O_RDWR);
485 if (err < 0)
486 return err;
487 fd = err;
488
489 snprintf(name, sizeof(name), "nvhost-%s-fd%d", g->name, fd);
490
491 file = anon_inode_getfile(name, l->as_dev.cdev.ops, NULL, O_RDWR);
492 if (IS_ERR(file)) {
493 err = PTR_ERR(file);
494 goto clean_up;
495 }
496
497 err = gk20a_as_alloc_share(g, args->big_page_size,
498 gk20a_as_translate_as_alloc_flags(g,
499 args->flags),
500 &as_share);
501 if (err)
502 goto clean_up_file;
503
504 fd_install(fd, file);
505 file->private_data = as_share;
506
507 args->as_fd = fd;
508 return 0;
509
510clean_up_file:
511 fput(file);
512clean_up:
513 put_unused_fd(fd);
514 return err;
515}
516
517static int gk20a_ctrl_open_tsg(struct gk20a *g,
518 struct nvgpu_gpu_open_tsg_args *args)
519{
520 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
521 int err;
522 int fd;
523 struct file *file;
524 char name[64];
525
526 err = get_unused_fd_flags(O_RDWR);
527 if (err < 0)
528 return err;
529 fd = err;
530
531 snprintf(name, sizeof(name), "nvgpu-%s-tsg%d", g->name, fd);
532
533 file = anon_inode_getfile(name, l->tsg.cdev.ops, NULL, O_RDWR);
534 if (IS_ERR(file)) {
535 err = PTR_ERR(file);
536 goto clean_up;
537 }
538
539 err = nvgpu_ioctl_tsg_open(g, file);
540 if (err)
541 goto clean_up_file;
542
543 fd_install(fd, file);
544 args->tsg_fd = fd;
545 return 0;
546
547clean_up_file:
548 fput(file);
549clean_up:
550 put_unused_fd(fd);
551 return err;
552}
553
554static int gk20a_ctrl_get_tpc_masks(struct gk20a *g,
555 struct nvgpu_gpu_get_tpc_masks_args *args)
556{
557 struct gr_gk20a *gr = &g->gr;
558 int err = 0;
559 const u32 gpc_tpc_mask_size = sizeof(u32) * gr->max_gpc_count;
560
561 if (args->mask_buf_size > 0) {
562 size_t write_size = gpc_tpc_mask_size;
563
564 nvgpu_speculation_barrier();
565 if (write_size > args->mask_buf_size)
566 write_size = args->mask_buf_size;
567
568 err = copy_to_user((void __user *)(uintptr_t)
569 args->mask_buf_addr,
570 gr->gpc_tpc_mask, write_size);
571 }
572
573 if (err == 0)
574 args->mask_buf_size = gpc_tpc_mask_size;
575
576 return err;
577}
578
579static int gk20a_ctrl_get_fbp_l2_masks(
580 struct gk20a *g, struct nvgpu_gpu_get_fbp_l2_masks_args *args)
581{
582 struct gr_gk20a *gr = &g->gr;
583 int err = 0;
584 const u32 fbp_l2_mask_size = sizeof(u32) * gr->max_fbps_count;
585
586 if (args->mask_buf_size > 0) {
587 size_t write_size = fbp_l2_mask_size;
588
589 nvgpu_speculation_barrier();
590 if (write_size > args->mask_buf_size)
591 write_size = args->mask_buf_size;
592
593 err = copy_to_user((void __user *)(uintptr_t)
594 args->mask_buf_addr,
595 gr->fbp_rop_l2_en_mask, write_size);
596 }
597
598 if (err == 0)
599 args->mask_buf_size = fbp_l2_mask_size;
600
601 return err;
602}
603
604static int nvgpu_gpu_ioctl_l2_fb_ops(struct gk20a *g,
605 struct nvgpu_gpu_l2_fb_args *args)
606{
607 int ret;
608 bool always_poweron;
609
610 if ((!args->l2_flush && !args->fb_flush) ||
611 (!args->l2_flush && args->l2_invalidate))
612 return -EINVAL;
613
614 /* Handle this case for joint rails or DGPU */
615 always_poweron = (!nvgpu_is_enabled(g, NVGPU_CAN_RAILGATE) ||
616 !pm_runtime_enabled(dev_from_gk20a(g)));
617
618 /* In case of not always power_on, exit if g->power_on is false */
619 if (!always_poweron && !gk20a_check_poweron(g)) {
620 return 0;
621 }
622
623 /* There is a small window between a call to gk20a_idle() has occured
624 * and railgate being actually triggered(setting g->power_on = false),
625 * when l2_flush can race with railgate. Its better to take a busy_lock
626 * to prevent the gk20a_idle() from proceeding. There is a very small
627 * chance that gk20a_idle() might begin before gk20a_busy(). Having
628 * a locked access to g->power_on further reduces the probability of
629 * gk20a_idle() being triggered before gk20a_busy()
630 */
631 ret = gk20a_busy(g);
632
633 if (ret != 0) {
634 nvgpu_err(g, "failed to take power ref");
635 return ret;
636 }
637
638 if (args->l2_flush)
639 g->ops.mm.l2_flush(g, args->l2_invalidate ? true : false);
640
641 if (args->fb_flush)
642 g->ops.mm.fb_flush(g);
643
644 gk20a_idle(g);
645
646 return 0;
647}
648
649static int nvgpu_gpu_ioctl_set_mmu_debug_mode(
650 struct gk20a *g,
651 struct nvgpu_gpu_mmu_debug_mode_args *args)
652{
653 if (gk20a_busy(g)) {
654 nvgpu_err(g, "failed to power on gpu");
655 return -EINVAL;
656 }
657
658 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
659 g->ops.fb.set_debug_mode(g, args->state == 1);
660 nvgpu_mutex_release(&g->dbg_sessions_lock);
661
662 gk20a_idle(g);
663 return 0;
664}
665
666static int nvgpu_gpu_ioctl_set_debug_mode(
667 struct gk20a *g,
668 struct nvgpu_gpu_sm_debug_mode_args *args)
669{
670 struct channel_gk20a *ch;
671 int err;
672
673 ch = gk20a_get_channel_from_file(args->channel_fd);
674 if (!ch)
675 return -EINVAL;
676
677 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
678 if (g->ops.gr.set_sm_debug_mode)
679 err = g->ops.gr.set_sm_debug_mode(g, ch,
680 args->sms, !!args->enable);
681 else
682 err = -ENOSYS;
683 nvgpu_mutex_release(&g->dbg_sessions_lock);
684
685 gk20a_channel_put(ch);
686 return err;
687}
688
689static int nvgpu_gpu_ioctl_trigger_suspend(struct gk20a *g)
690{
691 int err;
692
693 err = gk20a_busy(g);
694 if (err)
695 return err;
696
697 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
698 err = gr_gk20a_elpg_protected_call(g,
699 g->ops.gr.trigger_suspend(g));
700 nvgpu_mutex_release(&g->dbg_sessions_lock);
701
702 gk20a_idle(g);
703
704 return err;
705}
706
707static int nvgpu_gpu_ioctl_wait_for_pause(struct gk20a *g,
708 struct nvgpu_gpu_wait_pause_args *args)
709{
710 int err;
711 struct warpstate *ioctl_w_state;
712 struct nvgpu_warpstate *w_state = NULL;
713 u32 sm_count, ioctl_size, size, sm_id;
714
715 sm_count = g->gr.gpc_count * g->gr.tpc_count;
716
717 ioctl_size = sm_count * sizeof(struct warpstate);
718 ioctl_w_state = nvgpu_kzalloc(g, ioctl_size);
719 if (!ioctl_w_state)
720 return -ENOMEM;
721
722 size = sm_count * sizeof(struct nvgpu_warpstate);
723 w_state = nvgpu_kzalloc(g, size);
724 if (!w_state) {
725 err = -ENOMEM;
726 goto out_free;
727 }
728
729 err = gk20a_busy(g);
730 if (err)
731 goto out_free;
732
733 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
734 (void)gr_gk20a_elpg_protected_call(g,
735 g->ops.gr.wait_for_pause(g, w_state));
736
737 for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) {
738 ioctl_w_state[sm_id].valid_warps[0] =
739 w_state[sm_id].valid_warps[0];
740 ioctl_w_state[sm_id].valid_warps[1] =
741 w_state[sm_id].valid_warps[1];
742 ioctl_w_state[sm_id].trapped_warps[0] =
743 w_state[sm_id].trapped_warps[0];
744 ioctl_w_state[sm_id].trapped_warps[1] =
745 w_state[sm_id].trapped_warps[1];
746 ioctl_w_state[sm_id].paused_warps[0] =
747 w_state[sm_id].paused_warps[0];
748 ioctl_w_state[sm_id].paused_warps[1] =
749 w_state[sm_id].paused_warps[1];
750 }
751 /* Copy to user space - pointed by "args->pwarpstate" */
752 if (copy_to_user((void __user *)(uintptr_t)args->pwarpstate,
753 w_state, ioctl_size)) {
754 nvgpu_log_fn(g, "copy_to_user failed!");
755 err = -EFAULT;
756 }
757
758 nvgpu_mutex_release(&g->dbg_sessions_lock);
759
760 gk20a_idle(g);
761
762out_free:
763 nvgpu_kfree(g, w_state);
764 nvgpu_kfree(g, ioctl_w_state);
765
766 return err;
767}
768
769static int nvgpu_gpu_ioctl_resume_from_pause(struct gk20a *g)
770{
771 int err;
772
773 err = gk20a_busy(g);
774 if (err)
775 return err;
776
777 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
778 err = gr_gk20a_elpg_protected_call(g,
779 g->ops.gr.resume_from_pause(g));
780 nvgpu_mutex_release(&g->dbg_sessions_lock);
781
782 gk20a_idle(g);
783
784 return err;
785}
786
787static int nvgpu_gpu_ioctl_clear_sm_errors(struct gk20a *g)
788{
789 int err;
790
791 err = gk20a_busy(g);
792 if (err)
793 return err;
794
795 err = gr_gk20a_elpg_protected_call(g,
796 g->ops.gr.clear_sm_errors(g));
797
798 gk20a_idle(g);
799
800 return err;
801}
802
803static int nvgpu_gpu_ioctl_has_any_exception(
804 struct gk20a *g,
805 struct nvgpu_gpu_tpc_exception_en_status_args *args)
806{
807 u32 tpc_exception_en;
808
809 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
810 tpc_exception_en = g->ops.gr.tpc_enabled_exceptions(g);
811 nvgpu_mutex_release(&g->dbg_sessions_lock);
812
813 args->tpc_exception_en_sm_mask = tpc_exception_en;
814
815 return 0;
816}
817
818static int gk20a_ctrl_get_num_vsms(struct gk20a *g,
819 struct nvgpu_gpu_num_vsms *args)
820{
821 struct gr_gk20a *gr = &g->gr;
822 args->num_vsms = gr->no_of_sm;
823 return 0;
824}
825
826static int gk20a_ctrl_vsm_mapping(struct gk20a *g,
827 struct nvgpu_gpu_vsms_mapping *args)
828{
829 int err = 0;
830 struct gr_gk20a *gr = &g->gr;
831 size_t write_size = gr->no_of_sm *
832 sizeof(struct nvgpu_gpu_vsms_mapping_entry);
833 struct nvgpu_gpu_vsms_mapping_entry *vsms_buf;
834 u32 i;
835
836 vsms_buf = nvgpu_kzalloc(g, write_size);
837 if (vsms_buf == NULL)
838 return -ENOMEM;
839
840 for (i = 0; i < gr->no_of_sm; i++) {
841 vsms_buf[i].gpc_index = gr->sm_to_cluster[i].gpc_index;
842 if (g->ops.gr.get_nonpes_aware_tpc)
843 vsms_buf[i].tpc_index =
844 g->ops.gr.get_nonpes_aware_tpc(g,
845 gr->sm_to_cluster[i].gpc_index,
846 gr->sm_to_cluster[i].tpc_index);
847 else
848 vsms_buf[i].tpc_index =
849 gr->sm_to_cluster[i].tpc_index;
850 }
851
852 err = copy_to_user((void __user *)(uintptr_t)
853 args->vsms_map_buf_addr,
854 vsms_buf, write_size);
855 nvgpu_kfree(g, vsms_buf);
856
857 return err;
858}
859
860static int nvgpu_gpu_get_cpu_time_correlation_info(
861 struct gk20a *g,
862 struct nvgpu_gpu_get_cpu_time_correlation_info_args *args)
863{
864 struct nvgpu_cpu_time_correlation_sample *samples;
865 int err;
866 u32 i;
867
868 if (args->count > NVGPU_GPU_GET_CPU_TIME_CORRELATION_INFO_MAX_COUNT ||
869 args->source_id != NVGPU_GPU_GET_CPU_TIME_CORRELATION_INFO_SRC_ID_TSC)
870 return -EINVAL;
871
872 samples = nvgpu_kzalloc(g, args->count *
873 sizeof(struct nvgpu_cpu_time_correlation_sample));
874 if (!samples) {
875 return -ENOMEM;
876 }
877
878 err = g->ops.ptimer.get_timestamps_zipper(g,
879 args->source_id, args->count, samples);
880 if (!err) {
881 for (i = 0; i < args->count; i++) {
882 args->samples[i].cpu_timestamp = samples[i].cpu_timestamp;
883 args->samples[i].gpu_timestamp = samples[i].gpu_timestamp;
884 }
885 }
886
887 nvgpu_kfree(g, samples);
888
889 return err;
890}
891
892static int nvgpu_gpu_get_gpu_time(
893 struct gk20a *g,
894 struct nvgpu_gpu_get_gpu_time_args *args)
895{
896 u64 time;
897 int err;
898
899 err = gk20a_busy(g);
900 if (err)
901 return err;
902
903 err = g->ops.ptimer.read_ptimer(g, &time);
904 if (!err)
905 args->gpu_timestamp = time;
906
907 gk20a_idle(g);
908 return err;
909}
910
911static int nvgpu_gpu_get_engine_info(
912 struct gk20a *g,
913 struct nvgpu_gpu_get_engine_info_args *args)
914{
915 int err = 0;
916 u32 engine_enum = ENGINE_INVAL_GK20A;
917 u32 report_index = 0;
918 u32 engine_id_idx;
919 const u32 max_buffer_engines = args->engine_info_buf_size /
920 sizeof(struct nvgpu_gpu_get_engine_info_item);
921 struct nvgpu_gpu_get_engine_info_item __user *dst_item_list =
922 (void __user *)(uintptr_t)args->engine_info_buf_addr;
923
924 for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines;
925 ++engine_id_idx) {
926 u32 active_engine_id = g->fifo.active_engines_list[engine_id_idx];
927 const struct fifo_engine_info_gk20a *src_info =
928 &g->fifo.engine_info[active_engine_id];
929 struct nvgpu_gpu_get_engine_info_item dst_info;
930
931 memset(&dst_info, 0, sizeof(dst_info));
932
933 engine_enum = src_info->engine_enum;
934
935 switch (engine_enum) {
936 case ENGINE_GR_GK20A:
937 dst_info.engine_id = NVGPU_GPU_ENGINE_ID_GR;
938 break;
939
940 case ENGINE_GRCE_GK20A:
941 dst_info.engine_id = NVGPU_GPU_ENGINE_ID_GR_COPY;
942 break;
943
944 case ENGINE_ASYNC_CE_GK20A:
945 dst_info.engine_id = NVGPU_GPU_ENGINE_ID_ASYNC_COPY;
946 break;
947
948 default:
949 nvgpu_err(g, "Unmapped engine enum %u",
950 engine_enum);
951 continue;
952 }
953
954 dst_info.engine_instance = src_info->inst_id;
955 dst_info.runlist_id = src_info->runlist_id;
956
957 if (report_index < max_buffer_engines) {
958 err = copy_to_user(&dst_item_list[report_index],
959 &dst_info, sizeof(dst_info));
960 if (err)
961 goto clean_up;
962 }
963
964 ++report_index;
965 }
966
967 args->engine_info_buf_size =
968 report_index * sizeof(struct nvgpu_gpu_get_engine_info_item);
969
970clean_up:
971 return err;
972}
973
974static int nvgpu_gpu_alloc_vidmem(struct gk20a *g,
975 struct nvgpu_gpu_alloc_vidmem_args *args)
976{
977 u32 align = args->in.alignment ? args->in.alignment : SZ_4K;
978 int fd;
979
980 nvgpu_log_fn(g, " ");
981
982 /* not yet supported */
983 if (WARN_ON(args->in.flags & NVGPU_GPU_ALLOC_VIDMEM_FLAG_CPU_MASK))
984 return -EINVAL;
985
986 /* not yet supported */
987 if (WARN_ON(args->in.flags & NVGPU_GPU_ALLOC_VIDMEM_FLAG_VPR))
988 return -EINVAL;
989
990 if (args->in.size & (SZ_4K - 1))
991 return -EINVAL;
992
993 if (!args->in.size)
994 return -EINVAL;
995
996 if (align & (align - 1))
997 return -EINVAL;
998
999 if (align > roundup_pow_of_two(args->in.size)) {
1000 /* log this special case, buddy allocator detail */
1001 nvgpu_warn(g,
1002 "alignment larger than buffer size rounded up to power of 2 is not supported");
1003 return -EINVAL;
1004 }
1005
1006 fd = nvgpu_vidmem_export_linux(g, args->in.size);
1007 if (fd < 0)
1008 return fd;
1009
1010 args->out.dmabuf_fd = fd;
1011
1012 nvgpu_log_fn(g, "done, fd=%d", fd);
1013
1014 return 0;
1015}
1016
1017static int nvgpu_gpu_get_memory_state(struct gk20a *g,
1018 struct nvgpu_gpu_get_memory_state_args *args)
1019{
1020 int err;
1021
1022 nvgpu_log_fn(g, " ");
1023
1024 if (args->reserved[0] || args->reserved[1] ||
1025 args->reserved[2] || args->reserved[3])
1026 return -EINVAL;
1027
1028 err = nvgpu_vidmem_get_space(g, &args->total_free_bytes);
1029
1030 nvgpu_log_fn(g, "done, err=%d, bytes=%lld", err, args->total_free_bytes);
1031
1032 return err;
1033}
1034
1035static u32 nvgpu_gpu_convert_clk_domain(u32 clk_domain)
1036{
1037 u32 domain = 0;
1038
1039 if (clk_domain == NVGPU_GPU_CLK_DOMAIN_MCLK)
1040 domain = NVGPU_CLK_DOMAIN_MCLK;
1041 else if (clk_domain == NVGPU_GPU_CLK_DOMAIN_GPCCLK)
1042 domain = NVGPU_CLK_DOMAIN_GPCCLK;
1043 else
1044 domain = NVGPU_CLK_DOMAIN_MAX + 1;
1045
1046 return domain;
1047}
1048
1049static int nvgpu_gpu_clk_get_vf_points(struct gk20a *g,
1050 struct gk20a_ctrl_priv *priv,
1051 struct nvgpu_gpu_clk_vf_points_args *args)
1052{
1053 struct nvgpu_gpu_clk_vf_point clk_point;
1054 struct nvgpu_gpu_clk_vf_point __user *entry;
1055 struct nvgpu_clk_session *session = priv->clk_session;
1056 u32 clk_domains = 0;
1057 int err;
1058 u16 last_mhz;
1059 u16 *fpoints;
1060 u32 i;
1061 u32 max_points = 0;
1062 u32 num_points = 0;
1063 u16 min_mhz;
1064 u16 max_mhz;
1065
1066 nvgpu_log_fn(g, " ");
1067
1068 if (!session || args->flags)
1069 return -EINVAL;
1070
1071 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
1072 args->num_entries = 0;
1073
1074 if (!nvgpu_clk_arb_is_valid_domain(g,
1075 nvgpu_gpu_convert_clk_domain(args->clk_domain)))
1076 return -EINVAL;
1077
1078 err = nvgpu_clk_arb_get_arbiter_clk_f_points(g,
1079 nvgpu_gpu_convert_clk_domain(args->clk_domain),
1080 &max_points, NULL);
1081 if (err)
1082 return err;
1083
1084 if (!args->max_entries) {
1085 args->max_entries = max_points;
1086 return 0;
1087 }
1088
1089 if (args->max_entries < max_points)
1090 return -EINVAL;
1091
1092 err = nvgpu_clk_arb_get_arbiter_clk_range(g,
1093 nvgpu_gpu_convert_clk_domain(args->clk_domain),
1094 &min_mhz, &max_mhz);
1095 if (err)
1096 return err;
1097
1098 fpoints = nvgpu_kcalloc(g, max_points, sizeof(u16));
1099 if (!fpoints)
1100 return -ENOMEM;
1101
1102 err = nvgpu_clk_arb_get_arbiter_clk_f_points(g,
1103 nvgpu_gpu_convert_clk_domain(args->clk_domain),
1104 &max_points, fpoints);
1105 if (err)
1106 goto fail;
1107
1108 entry = (struct nvgpu_gpu_clk_vf_point __user *)
1109 (uintptr_t)args->clk_vf_point_entries;
1110
1111 last_mhz = 0;
1112 num_points = 0;
1113 for (i = 0; (i < max_points) && !err; i++) {
1114
1115 /* filter out duplicate frequencies */
1116 if (fpoints[i] == last_mhz)
1117 continue;
1118
1119 /* filter out out-of-range frequencies */
1120 if ((fpoints[i] < min_mhz) || (fpoints[i] > max_mhz))
1121 continue;
1122
1123 last_mhz = fpoints[i];
1124 clk_point.freq_hz = MHZ_TO_HZ(fpoints[i]);
1125
1126 err = copy_to_user((void __user *)entry, &clk_point,
1127 sizeof(clk_point));
1128
1129 num_points++;
1130 entry++;
1131 }
1132
1133 args->num_entries = num_points;
1134
1135fail:
1136 nvgpu_kfree(g, fpoints);
1137 return err;
1138}
1139
1140static int nvgpu_gpu_clk_get_range(struct gk20a *g,
1141 struct gk20a_ctrl_priv *priv,
1142 struct nvgpu_gpu_clk_range_args *args)
1143{
1144 struct nvgpu_gpu_clk_range clk_range;
1145 struct nvgpu_gpu_clk_range __user *entry;
1146 struct nvgpu_clk_session *session = priv->clk_session;
1147
1148 u32 clk_domains = 0;
1149 u32 num_domains;
1150 u32 num_entries;
1151 u32 i;
1152 int bit;
1153 int err;
1154 u16 min_mhz, max_mhz;
1155
1156 nvgpu_log_fn(g, " ");
1157
1158 if (!session)
1159 return -EINVAL;
1160
1161 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
1162 num_domains = hweight_long(clk_domains);
1163
1164 if (!args->flags) {
1165 if (!args->num_entries) {
1166 args->num_entries = num_domains;
1167 return 0;
1168 }
1169
1170 if (args->num_entries < num_domains)
1171 return -EINVAL;
1172
1173 args->num_entries = 0;
1174 num_entries = num_domains;
1175
1176 } else {
1177 if (args->flags != NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS)
1178 return -EINVAL;
1179
1180 num_entries = args->num_entries;
1181 if (num_entries > num_domains)
1182 return -EINVAL;
1183 }
1184
1185 entry = (struct nvgpu_gpu_clk_range __user *)
1186 (uintptr_t)args->clk_range_entries;
1187
1188 for (i = 0; i < num_entries; i++, entry++) {
1189
1190 if (args->flags == NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS) {
1191 if (copy_from_user(&clk_range, (void __user *)entry,
1192 sizeof(clk_range)))
1193 return -EFAULT;
1194 } else {
1195 bit = ffs(clk_domains) - 1;
1196 clk_range.clk_domain = bit;
1197 clk_domains &= ~BIT(bit);
1198 }
1199
1200 clk_range.flags = 0;
1201 err = nvgpu_clk_arb_get_arbiter_clk_range(g,
1202 nvgpu_gpu_convert_clk_domain(clk_range.clk_domain),
1203 &min_mhz, &max_mhz);
1204 clk_range.min_hz = MHZ_TO_HZ(min_mhz);
1205 clk_range.max_hz = MHZ_TO_HZ(max_mhz);
1206
1207 if (err)
1208 return err;
1209
1210 err = copy_to_user(entry, &clk_range, sizeof(clk_range));
1211 if (err)
1212 return -EFAULT;
1213 }
1214
1215 args->num_entries = num_entries;
1216
1217 return 0;
1218}
1219
1220static int nvgpu_gpu_clk_set_info(struct gk20a *g,
1221 struct gk20a_ctrl_priv *priv,
1222 struct nvgpu_gpu_clk_set_info_args *args)
1223{
1224 struct nvgpu_gpu_clk_info clk_info;
1225 struct nvgpu_gpu_clk_info __user *entry;
1226 struct nvgpu_clk_session *session = priv->clk_session;
1227
1228 int fd;
1229 u32 clk_domains = 0;
1230 u16 freq_mhz;
1231 int i;
1232 int ret;
1233
1234 nvgpu_log_fn(g, " ");
1235
1236 if (!session || args->flags)
1237 return -EINVAL;
1238
1239 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
1240 if (!clk_domains)
1241 return -EINVAL;
1242
1243 entry = (struct nvgpu_gpu_clk_info __user *)
1244 (uintptr_t)args->clk_info_entries;
1245
1246 for (i = 0; i < args->num_entries; i++, entry++) {
1247
1248 if (copy_from_user(&clk_info, entry, sizeof(clk_info)))
1249 return -EFAULT;
1250
1251 if (!nvgpu_clk_arb_is_valid_domain(g,
1252 nvgpu_gpu_convert_clk_domain(clk_info.clk_domain)))
1253 return -EINVAL;
1254 }
1255 nvgpu_speculation_barrier();
1256
1257 entry = (struct nvgpu_gpu_clk_info __user *)
1258 (uintptr_t)args->clk_info_entries;
1259
1260 ret = nvgpu_clk_arb_install_request_fd(g, session, &fd);
1261 if (ret < 0)
1262 return ret;
1263
1264 for (i = 0; i < args->num_entries; i++, entry++) {
1265
1266 if (copy_from_user(&clk_info, (void __user *)entry,
1267 sizeof(clk_info)))
1268 return -EFAULT;
1269 freq_mhz = HZ_TO_MHZ(clk_info.freq_hz);
1270
1271 nvgpu_clk_arb_set_session_target_mhz(session, fd,
1272 nvgpu_gpu_convert_clk_domain(clk_info.clk_domain), freq_mhz);
1273 }
1274
1275 nvgpu_speculation_barrier();
1276 ret = nvgpu_clk_arb_commit_request_fd(g, session, fd);
1277 if (ret < 0)
1278 return ret;
1279
1280 args->completion_fd = fd;
1281
1282 return ret;
1283}
1284
1285static int nvgpu_gpu_clk_get_info(struct gk20a *g,
1286 struct gk20a_ctrl_priv *priv,
1287 struct nvgpu_gpu_clk_get_info_args *args)
1288{
1289 struct nvgpu_gpu_clk_info clk_info;
1290 struct nvgpu_gpu_clk_info __user *entry;
1291 struct nvgpu_clk_session *session = priv->clk_session;
1292 u32 clk_domains = 0;
1293 u32 num_domains;
1294 u32 num_entries;
1295 u32 i;
1296 u16 freq_mhz;
1297 int err;
1298 int bit;
1299
1300 nvgpu_log_fn(g, " ");
1301
1302 if (!session)
1303 return -EINVAL;
1304
1305 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
1306 num_domains = hweight_long(clk_domains);
1307
1308 if (!args->flags) {
1309 if (!args->num_entries) {
1310 args->num_entries = num_domains;
1311 return 0;
1312 }
1313
1314 if (args->num_entries < num_domains)
1315 return -EINVAL;
1316
1317 args->num_entries = 0;
1318 num_entries = num_domains;
1319
1320 } else {
1321 if (args->flags != NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS)
1322 return -EINVAL;
1323
1324 num_entries = args->num_entries;
1325 if (num_entries > num_domains * 3)
1326 return -EINVAL;
1327 }
1328
1329 entry = (struct nvgpu_gpu_clk_info __user *)
1330 (uintptr_t)args->clk_info_entries;
1331
1332 for (i = 0; i < num_entries; i++, entry++) {
1333
1334 if (args->flags == NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS) {
1335 if (copy_from_user(&clk_info, (void __user *)entry,
1336 sizeof(clk_info)))
1337 return -EFAULT;
1338 } else {
1339 bit = ffs(clk_domains) - 1;
1340 clk_info.clk_domain = bit;
1341 clk_domains &= ~BIT(bit);
1342 clk_info.clk_type = args->clk_type;
1343 }
1344
1345 nvgpu_speculation_barrier();
1346 switch (clk_info.clk_type) {
1347 case NVGPU_GPU_CLK_TYPE_TARGET:
1348 err = nvgpu_clk_arb_get_session_target_mhz(session,
1349 nvgpu_gpu_convert_clk_domain(clk_info.clk_domain),
1350 &freq_mhz);
1351 break;
1352 case NVGPU_GPU_CLK_TYPE_ACTUAL:
1353 err = nvgpu_clk_arb_get_arbiter_actual_mhz(g,
1354 nvgpu_gpu_convert_clk_domain(clk_info.clk_domain),
1355 &freq_mhz);
1356 break;
1357 case NVGPU_GPU_CLK_TYPE_EFFECTIVE:
1358 err = nvgpu_clk_arb_get_arbiter_effective_mhz(g,
1359 nvgpu_gpu_convert_clk_domain(clk_info.clk_domain),
1360 &freq_mhz);
1361 break;
1362 default:
1363 freq_mhz = 0;
1364 err = -EINVAL;
1365 break;
1366 }
1367 if (err)
1368 return err;
1369
1370 clk_info.flags = 0;
1371 clk_info.freq_hz = MHZ_TO_HZ(freq_mhz);
1372
1373 err = copy_to_user((void __user *)entry, &clk_info,
1374 sizeof(clk_info));
1375 if (err)
1376 return -EFAULT;
1377 }
1378
1379 nvgpu_speculation_barrier();
1380 args->num_entries = num_entries;
1381
1382 return 0;
1383}
1384
1385static int nvgpu_gpu_get_event_fd(struct gk20a *g,
1386 struct gk20a_ctrl_priv *priv,
1387 struct nvgpu_gpu_get_event_fd_args *args)
1388{
1389 struct nvgpu_clk_session *session = priv->clk_session;
1390
1391 nvgpu_log_fn(g, " ");
1392
1393 if (!session)
1394 return -EINVAL;
1395
1396 return nvgpu_clk_arb_install_event_fd(g, session, &args->event_fd,
1397 args->flags);
1398}
1399
1400static int nvgpu_gpu_get_voltage(struct gk20a *g,
1401 struct nvgpu_gpu_get_voltage_args *args)
1402{
1403 int err = -EINVAL;
1404
1405 nvgpu_log_fn(g, " ");
1406
1407 if (args->reserved)
1408 return -EINVAL;
1409
1410 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_GET_VOLTAGE))
1411 return -EINVAL;
1412
1413 err = gk20a_busy(g);
1414 if (err)
1415 return err;
1416
1417 nvgpu_speculation_barrier();
1418 switch (args->which) {
1419 case NVGPU_GPU_VOLTAGE_CORE:
1420 err = volt_get_voltage(g, CTRL_VOLT_DOMAIN_LOGIC, &args->voltage);
1421 break;
1422 case NVGPU_GPU_VOLTAGE_SRAM:
1423 err = volt_get_voltage(g, CTRL_VOLT_DOMAIN_SRAM, &args->voltage);
1424 break;
1425 case NVGPU_GPU_VOLTAGE_BUS:
1426 err = pmgr_pwr_devices_get_voltage(g, &args->voltage);
1427 break;
1428 default:
1429 err = -EINVAL;
1430 }
1431
1432 gk20a_idle(g);
1433
1434 return err;
1435}
1436
1437static int nvgpu_gpu_get_current(struct gk20a *g,
1438 struct nvgpu_gpu_get_current_args *args)
1439{
1440 int err;
1441
1442 nvgpu_log_fn(g, " ");
1443
1444 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1445 return -EINVAL;
1446
1447 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_GET_CURRENT))
1448 return -EINVAL;
1449
1450 err = gk20a_busy(g);
1451 if (err)
1452 return err;
1453
1454 err = pmgr_pwr_devices_get_current(g, &args->currnt);
1455
1456 gk20a_idle(g);
1457
1458 return err;
1459}
1460
1461static int nvgpu_gpu_get_power(struct gk20a *g,
1462 struct nvgpu_gpu_get_power_args *args)
1463{
1464 int err;
1465
1466 nvgpu_log_fn(g, " ");
1467
1468 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1469 return -EINVAL;
1470
1471 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_GET_POWER))
1472 return -EINVAL;
1473
1474 err = gk20a_busy(g);
1475 if (err)
1476 return err;
1477
1478 err = pmgr_pwr_devices_get_power(g, &args->power);
1479
1480 gk20a_idle(g);
1481
1482 return err;
1483}
1484
1485static int nvgpu_gpu_get_temperature(struct gk20a *g,
1486 struct nvgpu_gpu_get_temperature_args *args)
1487{
1488 int err;
1489 u32 temp_f24_8;
1490
1491 nvgpu_log_fn(g, " ");
1492
1493 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1494 return -EINVAL;
1495
1496 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_GET_TEMPERATURE))
1497 return -EINVAL;
1498
1499 if (!g->ops.therm.get_internal_sensor_curr_temp)
1500 return -EINVAL;
1501
1502 err = gk20a_busy(g);
1503 if (err)
1504 return err;
1505
1506 err = g->ops.therm.get_internal_sensor_curr_temp(g, &temp_f24_8);
1507
1508 gk20a_idle(g);
1509
1510 args->temp_f24_8 = (s32)temp_f24_8;
1511
1512 return err;
1513}
1514
1515static int nvgpu_gpu_set_therm_alert_limit(struct gk20a *g,
1516 struct nvgpu_gpu_set_therm_alert_limit_args *args)
1517{
1518 int err;
1519
1520 nvgpu_log_fn(g, " ");
1521
1522 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1523 return -EINVAL;
1524
1525 if (!g->ops.therm.configure_therm_alert)
1526 return -EINVAL;
1527
1528 err = gk20a_busy(g);
1529 if (err)
1530 return err;
1531
1532 err = g->ops.therm.configure_therm_alert(g, args->temp_f24_8);
1533
1534 gk20a_idle(g);
1535
1536 return err;
1537}
1538
1539static int nvgpu_gpu_set_deterministic_ch_railgate(struct channel_gk20a *ch,
1540 u32 flags)
1541{
1542 int err = 0;
1543 bool allow;
1544 bool disallow;
1545
1546 allow = flags &
1547 NVGPU_GPU_SET_DETERMINISTIC_OPTS_FLAGS_ALLOW_RAILGATING;
1548
1549 disallow = flags &
1550 NVGPU_GPU_SET_DETERMINISTIC_OPTS_FLAGS_DISALLOW_RAILGATING;
1551
1552 /* Can't be both at the same time */
1553 if (allow && disallow)
1554 return -EINVAL;
1555
1556 /* Nothing to do */
1557 if (!allow && !disallow)
1558 return 0;
1559
1560 /*
1561 * Moving into explicit idle or back from it? A call that doesn't
1562 * change the status is a no-op.
1563 */
1564 if (!ch->deterministic_railgate_allowed &&
1565 allow) {
1566 gk20a_idle(ch->g);
1567 } else if (ch->deterministic_railgate_allowed &&
1568 !allow) {
1569 err = gk20a_busy(ch->g);
1570 if (err) {
1571 nvgpu_warn(ch->g,
1572 "cannot busy to restore deterministic ch");
1573 return err;
1574 }
1575 }
1576 ch->deterministic_railgate_allowed = allow;
1577
1578 return err;
1579}
1580
1581static int nvgpu_gpu_set_deterministic_ch(struct channel_gk20a *ch, u32 flags)
1582{
1583 if (!ch->deterministic)
1584 return -EINVAL;
1585
1586 return nvgpu_gpu_set_deterministic_ch_railgate(ch, flags);
1587}
1588
1589static int nvgpu_gpu_set_deterministic_opts(struct gk20a *g,
1590 struct nvgpu_gpu_set_deterministic_opts_args *args)
1591{
1592 int __user *user_channels;
1593 u32 i = 0;
1594 int err = 0;
1595
1596 nvgpu_log_fn(g, " ");
1597
1598 user_channels = (int __user *)(uintptr_t)args->channels;
1599
1600 /* Upper limit; prevent holding deterministic_busy for long */
1601 if (args->num_channels > g->fifo.num_channels) {
1602 err = -EINVAL;
1603 goto out;
1604 }
1605
1606 /* Trivial sanity check first */
1607 if (!access_ok(VERIFY_READ, user_channels,
1608 args->num_channels * sizeof(int))) {
1609 err = -EFAULT;
1610 goto out;
1611 }
1612
1613 nvgpu_rwsem_down_read(&g->deterministic_busy);
1614
1615 /* note: we exit at the first failure */
1616 for (; i < args->num_channels; i++) {
1617 int ch_fd = 0;
1618 struct channel_gk20a *ch;
1619
1620 if (copy_from_user(&ch_fd, &user_channels[i], sizeof(int))) {
1621 /* User raced with above access_ok */
1622 err = -EFAULT;
1623 break;
1624 }
1625
1626 ch = gk20a_get_channel_from_file(ch_fd);
1627 if (!ch) {
1628 err = -EINVAL;
1629 break;
1630 }
1631
1632 err = nvgpu_gpu_set_deterministic_ch(ch, args->flags);
1633
1634 gk20a_channel_put(ch);
1635
1636 if (err)
1637 break;
1638 }
1639
1640 nvgpu_speculation_barrier();
1641 nvgpu_rwsem_up_read(&g->deterministic_busy);
1642
1643out:
1644 args->num_channels = i;
1645 return err;
1646}
1647
1648long gk20a_ctrl_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1649{
1650 struct gk20a_ctrl_priv *priv = filp->private_data;
1651 struct gk20a *g = priv->g;
1652 struct nvgpu_gpu_zcull_get_ctx_size_args *get_ctx_size_args;
1653 struct nvgpu_gpu_zcull_get_info_args *get_info_args;
1654 struct nvgpu_gpu_zbc_set_table_args *set_table_args;
1655 struct nvgpu_gpu_zbc_query_table_args *query_table_args;
1656 u8 buf[NVGPU_GPU_IOCTL_MAX_ARG_SIZE];
1657 struct gr_zcull_info *zcull_info;
1658 struct zbc_entry *zbc_val;
1659 struct zbc_query_params *zbc_tbl;
1660 int i, err = 0;
1661
1662 nvgpu_log_fn(g, "start %d", _IOC_NR(cmd));
1663
1664 if ((_IOC_TYPE(cmd) != NVGPU_GPU_IOCTL_MAGIC) ||
1665 (_IOC_NR(cmd) == 0) ||
1666 (_IOC_NR(cmd) > NVGPU_GPU_IOCTL_LAST) ||
1667 (_IOC_SIZE(cmd) > NVGPU_GPU_IOCTL_MAX_ARG_SIZE))
1668 return -EINVAL;
1669
1670 memset(buf, 0, sizeof(buf));
1671 if (_IOC_DIR(cmd) & _IOC_WRITE) {
1672 if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
1673 return -EFAULT;
1674 }
1675
1676 if (!g->sw_ready) {
1677 err = gk20a_busy(g);
1678 if (err)
1679 return err;
1680
1681 gk20a_idle(g);
1682 }
1683
1684 nvgpu_speculation_barrier();
1685 switch (cmd) {
1686 case NVGPU_GPU_IOCTL_ZCULL_GET_CTX_SIZE:
1687 get_ctx_size_args = (struct nvgpu_gpu_zcull_get_ctx_size_args *)buf;
1688
1689 get_ctx_size_args->size = gr_gk20a_get_ctxsw_zcull_size(g, &g->gr);
1690
1691 break;
1692 case NVGPU_GPU_IOCTL_ZCULL_GET_INFO:
1693 get_info_args = (struct nvgpu_gpu_zcull_get_info_args *)buf;
1694
1695 memset(get_info_args, 0, sizeof(struct nvgpu_gpu_zcull_get_info_args));
1696
1697 zcull_info = nvgpu_kzalloc(g, sizeof(struct gr_zcull_info));
1698 if (zcull_info == NULL)
1699 return -ENOMEM;
1700
1701 err = g->ops.gr.get_zcull_info(g, &g->gr, zcull_info);
1702 if (err) {
1703 nvgpu_kfree(g, zcull_info);
1704 break;
1705 }
1706
1707 get_info_args->width_align_pixels = zcull_info->width_align_pixels;
1708 get_info_args->height_align_pixels = zcull_info->height_align_pixels;
1709 get_info_args->pixel_squares_by_aliquots = zcull_info->pixel_squares_by_aliquots;
1710 get_info_args->aliquot_total = zcull_info->aliquot_total;
1711 get_info_args->region_byte_multiplier = zcull_info->region_byte_multiplier;
1712 get_info_args->region_header_size = zcull_info->region_header_size;
1713 get_info_args->subregion_header_size = zcull_info->subregion_header_size;
1714 get_info_args->subregion_width_align_pixels = zcull_info->subregion_width_align_pixels;
1715 get_info_args->subregion_height_align_pixels = zcull_info->subregion_height_align_pixels;
1716 get_info_args->subregion_count = zcull_info->subregion_count;
1717
1718 nvgpu_kfree(g, zcull_info);
1719 break;
1720 case NVGPU_GPU_IOCTL_ZBC_SET_TABLE:
1721 set_table_args = (struct nvgpu_gpu_zbc_set_table_args *)buf;
1722
1723 zbc_val = nvgpu_kzalloc(g, sizeof(struct zbc_entry));
1724 if (zbc_val == NULL)
1725 return -ENOMEM;
1726
1727 zbc_val->format = set_table_args->format;
1728 zbc_val->type = set_table_args->type;
1729
1730 nvgpu_speculation_barrier();
1731 switch (zbc_val->type) {
1732 case GK20A_ZBC_TYPE_COLOR:
1733 for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) {
1734 zbc_val->color_ds[i] = set_table_args->color_ds[i];
1735 zbc_val->color_l2[i] = set_table_args->color_l2[i];
1736 }
1737 break;
1738 case GK20A_ZBC_TYPE_DEPTH:
1739 case T19X_ZBC:
1740 zbc_val->depth = set_table_args->depth;
1741 break;
1742 default:
1743 err = -EINVAL;
1744 }
1745
1746 if (!err) {
1747 err = gk20a_busy(g);
1748 if (!err) {
1749 err = g->ops.gr.zbc_set_table(g, &g->gr,
1750 zbc_val);
1751 gk20a_idle(g);
1752 }
1753 }
1754
1755 if (zbc_val)
1756 nvgpu_kfree(g, zbc_val);
1757 break;
1758 case NVGPU_GPU_IOCTL_ZBC_QUERY_TABLE:
1759 query_table_args = (struct nvgpu_gpu_zbc_query_table_args *)buf;
1760
1761 zbc_tbl = nvgpu_kzalloc(g, sizeof(struct zbc_query_params));
1762 if (zbc_tbl == NULL)
1763 return -ENOMEM;
1764
1765 zbc_tbl->type = query_table_args->type;
1766 zbc_tbl->index_size = query_table_args->index_size;
1767
1768 err = g->ops.gr.zbc_query_table(g, &g->gr, zbc_tbl);
1769
1770 if (!err) {
1771 switch (zbc_tbl->type) {
1772 case GK20A_ZBC_TYPE_COLOR:
1773 for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) {
1774 query_table_args->color_ds[i] = zbc_tbl->color_ds[i];
1775 query_table_args->color_l2[i] = zbc_tbl->color_l2[i];
1776 }
1777 break;
1778 case GK20A_ZBC_TYPE_DEPTH:
1779 case T19X_ZBC:
1780 query_table_args->depth = zbc_tbl->depth;
1781 break;
1782 case GK20A_ZBC_TYPE_INVALID:
1783 query_table_args->index_size = zbc_tbl->index_size;
1784 break;
1785 default:
1786 err = -EINVAL;
1787 }
1788 if (!err) {
1789 query_table_args->format = zbc_tbl->format;
1790 query_table_args->ref_cnt = zbc_tbl->ref_cnt;
1791 }
1792 }
1793
1794 if (zbc_tbl)
1795 nvgpu_kfree(g, zbc_tbl);
1796 break;
1797
1798 case NVGPU_GPU_IOCTL_GET_CHARACTERISTICS:
1799 err = gk20a_ctrl_ioctl_gpu_characteristics(
1800 g, (struct nvgpu_gpu_get_characteristics *)buf);
1801 break;
1802 case NVGPU_GPU_IOCTL_PREPARE_COMPRESSIBLE_READ:
1803 err = gk20a_ctrl_prepare_compressible_read(g,
1804 (struct nvgpu_gpu_prepare_compressible_read_args *)buf);
1805 break;
1806 case NVGPU_GPU_IOCTL_MARK_COMPRESSIBLE_WRITE:
1807 err = gk20a_ctrl_mark_compressible_write(g,
1808 (struct nvgpu_gpu_mark_compressible_write_args *)buf);
1809 break;
1810 case NVGPU_GPU_IOCTL_ALLOC_AS:
1811 err = gk20a_ctrl_alloc_as(g,
1812 (struct nvgpu_alloc_as_args *)buf);
1813 break;
1814 case NVGPU_GPU_IOCTL_OPEN_TSG:
1815 err = gk20a_ctrl_open_tsg(g,
1816 (struct nvgpu_gpu_open_tsg_args *)buf);
1817 break;
1818 case NVGPU_GPU_IOCTL_GET_TPC_MASKS:
1819 err = gk20a_ctrl_get_tpc_masks(g,
1820 (struct nvgpu_gpu_get_tpc_masks_args *)buf);
1821 break;
1822 case NVGPU_GPU_IOCTL_GET_FBP_L2_MASKS:
1823 err = gk20a_ctrl_get_fbp_l2_masks(g,
1824 (struct nvgpu_gpu_get_fbp_l2_masks_args *)buf);
1825 break;
1826 case NVGPU_GPU_IOCTL_OPEN_CHANNEL:
1827 /* this arg type here, but ..gpu_open_channel_args in nvgpu.h
1828 * for consistency - they are the same */
1829 err = gk20a_channel_open_ioctl(g,
1830 (struct nvgpu_channel_open_args *)buf);
1831 break;
1832 case NVGPU_GPU_IOCTL_FLUSH_L2:
1833 err = nvgpu_gpu_ioctl_l2_fb_ops(g,
1834 (struct nvgpu_gpu_l2_fb_args *)buf);
1835 break;
1836
1837 case NVGPU_GPU_IOCTL_SET_MMUDEBUG_MODE:
1838 err = nvgpu_gpu_ioctl_set_mmu_debug_mode(g,
1839 (struct nvgpu_gpu_mmu_debug_mode_args *)buf);
1840 break;
1841
1842 case NVGPU_GPU_IOCTL_SET_SM_DEBUG_MODE:
1843 err = gr_gk20a_elpg_protected_call(g,
1844 nvgpu_gpu_ioctl_set_debug_mode(g, (struct nvgpu_gpu_sm_debug_mode_args *)buf));
1845 break;
1846
1847 case NVGPU_GPU_IOCTL_TRIGGER_SUSPEND:
1848 err = nvgpu_gpu_ioctl_trigger_suspend(g);
1849 break;
1850
1851 case NVGPU_GPU_IOCTL_WAIT_FOR_PAUSE:
1852 err = nvgpu_gpu_ioctl_wait_for_pause(g,
1853 (struct nvgpu_gpu_wait_pause_args *)buf);
1854 break;
1855
1856 case NVGPU_GPU_IOCTL_RESUME_FROM_PAUSE:
1857 err = nvgpu_gpu_ioctl_resume_from_pause(g);
1858 break;
1859
1860 case NVGPU_GPU_IOCTL_CLEAR_SM_ERRORS:
1861 err = nvgpu_gpu_ioctl_clear_sm_errors(g);
1862 break;
1863
1864 case NVGPU_GPU_IOCTL_GET_TPC_EXCEPTION_EN_STATUS:
1865 err = nvgpu_gpu_ioctl_has_any_exception(g,
1866 (struct nvgpu_gpu_tpc_exception_en_status_args *)buf);
1867 break;
1868
1869 case NVGPU_GPU_IOCTL_NUM_VSMS:
1870 err = gk20a_ctrl_get_num_vsms(g,
1871 (struct nvgpu_gpu_num_vsms *)buf);
1872 break;
1873 case NVGPU_GPU_IOCTL_VSMS_MAPPING:
1874 err = gk20a_ctrl_vsm_mapping(g,
1875 (struct nvgpu_gpu_vsms_mapping *)buf);
1876 break;
1877
1878 case NVGPU_GPU_IOCTL_GET_CPU_TIME_CORRELATION_INFO:
1879 err = nvgpu_gpu_get_cpu_time_correlation_info(g,
1880 (struct nvgpu_gpu_get_cpu_time_correlation_info_args *)buf);
1881 break;
1882
1883 case NVGPU_GPU_IOCTL_GET_GPU_TIME:
1884 err = nvgpu_gpu_get_gpu_time(g,
1885 (struct nvgpu_gpu_get_gpu_time_args *)buf);
1886 break;
1887
1888 case NVGPU_GPU_IOCTL_GET_ENGINE_INFO:
1889 err = nvgpu_gpu_get_engine_info(g,
1890 (struct nvgpu_gpu_get_engine_info_args *)buf);
1891 break;
1892
1893 case NVGPU_GPU_IOCTL_ALLOC_VIDMEM:
1894 err = nvgpu_gpu_alloc_vidmem(g,
1895 (struct nvgpu_gpu_alloc_vidmem_args *)buf);
1896 break;
1897
1898 case NVGPU_GPU_IOCTL_GET_MEMORY_STATE:
1899 err = nvgpu_gpu_get_memory_state(g,
1900 (struct nvgpu_gpu_get_memory_state_args *)buf);
1901 break;
1902
1903 case NVGPU_GPU_IOCTL_CLK_GET_RANGE:
1904 err = nvgpu_gpu_clk_get_range(g, priv,
1905 (struct nvgpu_gpu_clk_range_args *)buf);
1906 break;
1907
1908 case NVGPU_GPU_IOCTL_CLK_GET_VF_POINTS:
1909 err = nvgpu_gpu_clk_get_vf_points(g, priv,
1910 (struct nvgpu_gpu_clk_vf_points_args *)buf);
1911 break;
1912
1913 case NVGPU_GPU_IOCTL_CLK_SET_INFO:
1914 err = nvgpu_gpu_clk_set_info(g, priv,
1915 (struct nvgpu_gpu_clk_set_info_args *)buf);
1916 break;
1917
1918 case NVGPU_GPU_IOCTL_CLK_GET_INFO:
1919 err = nvgpu_gpu_clk_get_info(g, priv,
1920 (struct nvgpu_gpu_clk_get_info_args *)buf);
1921 break;
1922
1923 case NVGPU_GPU_IOCTL_GET_EVENT_FD:
1924 err = nvgpu_gpu_get_event_fd(g, priv,
1925 (struct nvgpu_gpu_get_event_fd_args *)buf);
1926 break;
1927
1928 case NVGPU_GPU_IOCTL_GET_VOLTAGE:
1929 err = nvgpu_gpu_get_voltage(g,
1930 (struct nvgpu_gpu_get_voltage_args *)buf);
1931 break;
1932
1933 case NVGPU_GPU_IOCTL_GET_CURRENT:
1934 err = nvgpu_gpu_get_current(g,
1935 (struct nvgpu_gpu_get_current_args *)buf);
1936 break;
1937
1938 case NVGPU_GPU_IOCTL_GET_POWER:
1939 err = nvgpu_gpu_get_power(g,
1940 (struct nvgpu_gpu_get_power_args *)buf);
1941 break;
1942
1943 case NVGPU_GPU_IOCTL_GET_TEMPERATURE:
1944 err = nvgpu_gpu_get_temperature(g,
1945 (struct nvgpu_gpu_get_temperature_args *)buf);
1946 break;
1947
1948 case NVGPU_GPU_IOCTL_SET_THERM_ALERT_LIMIT:
1949 err = nvgpu_gpu_set_therm_alert_limit(g,
1950 (struct nvgpu_gpu_set_therm_alert_limit_args *)buf);
1951 break;
1952
1953 case NVGPU_GPU_IOCTL_SET_DETERMINISTIC_OPTS:
1954 err = nvgpu_gpu_set_deterministic_opts(g,
1955 (struct nvgpu_gpu_set_deterministic_opts_args *)buf);
1956 break;
1957
1958 default:
1959 nvgpu_log_info(g, "unrecognized gpu ioctl cmd: 0x%x", cmd);
1960 err = -ENOTTY;
1961 break;
1962 }
1963
1964 if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
1965 err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
1966
1967 return err;
1968}
1969
1970static void usermode_vma_close(struct vm_area_struct *vma)
1971{
1972 struct gk20a_ctrl_priv *priv = vma->vm_private_data;
1973 struct gk20a *g = priv->g;
1974 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
1975
1976 nvgpu_mutex_acquire(&l->ctrl.privs_lock);
1977 priv->usermode_vma.vma = NULL;
1978 priv->usermode_vma.vma_mapped = false;
1979 nvgpu_mutex_release(&l->ctrl.privs_lock);
1980}
1981
1982struct vm_operations_struct usermode_vma_ops = {
1983 /* no .open - we use VM_DONTCOPY and don't support fork */
1984 .close = usermode_vma_close,
1985};
1986
1987int gk20a_ctrl_dev_mmap(struct file *filp, struct vm_area_struct *vma)
1988{
1989 struct gk20a_ctrl_priv *priv = filp->private_data;
1990 struct gk20a *g = priv->g;
1991 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
1992 u64 addr;
1993 int err;
1994
1995 if (g->ops.fifo.usermode_base == NULL)
1996 return -ENOSYS;
1997
1998 if (priv->usermode_vma.vma != NULL)
1999 return -EBUSY;
2000
2001 if (vma->vm_end - vma->vm_start != SZ_4K)
2002 return -EINVAL;
2003
2004 if (vma->vm_pgoff != 0UL)
2005 return -EINVAL;
2006
2007 addr = l->regs_bus_addr + g->ops.fifo.usermode_base(g);
2008
2009 /* Sync with poweron/poweroff, and require valid regs */
2010 err = gk20a_busy(g);
2011 if (err) {
2012 return err;
2013 }
2014
2015 nvgpu_mutex_acquire(&l->ctrl.privs_lock);
2016
2017 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
2018 VM_DONTDUMP | VM_PFNMAP;
2019 vma->vm_ops = &usermode_vma_ops;
2020 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
2021
2022 err = io_remap_pfn_range(vma, vma->vm_start, addr >> PAGE_SHIFT,
2023 vma->vm_end - vma->vm_start, vma->vm_page_prot);
2024 if (!err) {
2025 priv->usermode_vma.vma = vma;
2026 priv->usermode_vma.flags = vma->vm_flags;
2027 vma->vm_private_data = priv;
2028 priv->usermode_vma.vma_mapped = true;
2029 }
2030 nvgpu_mutex_release(&l->ctrl.privs_lock);
2031
2032 gk20a_idle(g);
2033
2034 return err;
2035}
2036
2037static void alter_usermode_mapping(struct gk20a *g,
2038 struct gk20a_ctrl_priv *priv,
2039 bool poweroff)
2040{
2041 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
2042 struct vm_area_struct *vma = priv->usermode_vma.vma;
2043 bool vma_mapped = priv->usermode_vma.vma_mapped;
2044 u64 addr;
2045 int err;
2046
2047 if (!vma) {
2048 /* Nothing to do - no mmap called */
2049 return;
2050 }
2051
2052 addr = l->regs_bus_addr + g->ops.fifo.usermode_base(g);
2053
2054 down_write(&vma->vm_mm->mmap_sem);
2055
2056 /*
2057 * This is a no-op for the below cases
2058 * a) poweroff and !vma_mapped - > do nothing as no map exists
2059 * b) !poweroff and vmap_mapped -> do nothing as already mapped
2060 */
2061 if (poweroff && vma_mapped) {
2062 err = zap_vma_ptes(vma, vma->vm_start, SZ_4K);
2063 if (err == 0) {
2064 vma->vm_flags = VM_NONE;
2065 priv->usermode_vma.vma_mapped = false;
2066 } else {
2067 nvgpu_err(g, "can't remove usermode mapping");
2068 }
2069 } else if (!poweroff && !vma_mapped) {
2070 vma->vm_flags = priv->usermode_vma.flags;
2071 err = io_remap_pfn_range(vma, vma->vm_start,
2072 addr >> PAGE_SHIFT,
2073 SZ_4K, vma->vm_page_prot);
2074 if (err != 0) {
2075 nvgpu_err(g, "can't restore usermode mapping");
2076 vma->vm_flags = VM_NONE;
2077 } else {
2078 priv->usermode_vma.vma_mapped = true;
2079 }
2080 }
2081
2082 up_write(&vma->vm_mm->mmap_sem);
2083}
2084
2085static void alter_usermode_mappings(struct gk20a *g, bool poweroff)
2086{
2087 struct gk20a_ctrl_priv *priv;
2088 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
2089
2090 nvgpu_mutex_acquire(&l->ctrl.privs_lock);
2091 nvgpu_list_for_each_entry(priv, &l->ctrl.privs,
2092 gk20a_ctrl_priv, list) {
2093 alter_usermode_mapping(g, priv, poweroff);
2094 }
2095 nvgpu_mutex_release(&l->ctrl.privs_lock);
2096}
2097
2098void nvgpu_hide_usermode_for_poweroff(struct gk20a *g)
2099{
2100 alter_usermode_mappings(g, true);
2101}
2102
2103void nvgpu_restore_usermode_for_poweron(struct gk20a *g)
2104{
2105 alter_usermode_mappings(g, false);
2106}
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-30 10:23:57 -0400