summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/linux/ioctl_ctrl.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common/linux/ioctl_ctrl.c')
-rw-r--r--drivers/gpu/nvgpu/common/linux/ioctl_ctrl.c1868
1 files changed, 1868 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/ioctl_ctrl.c b/drivers/gpu/nvgpu/common/linux/ioctl_ctrl.c
new file mode 100644
index 00000000..2a91b87d
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/ioctl_ctrl.c
@@ -0,0 +1,1868 @@
1/*
2 * Copyright (c) 2011-2017, NVIDIA Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#include <linux/uaccess.h>
18#include <linux/cdev.h>
19#include <linux/file.h>
20#include <linux/anon_inodes.h>
21#include <linux/fs.h>
22#include <uapi/linux/nvgpu.h>
23#include <uapi/linux/nvgpu-t18x.h>
24
25#include <nvgpu/bitops.h>
26#include <nvgpu/kmem.h>
27#include <nvgpu/bug.h>
28#include <nvgpu/bus.h>
29#include <nvgpu/vidmem.h>
30#include <nvgpu/log.h>
31#include <nvgpu/enabled.h>
32
33#include <nvgpu/linux/vidmem.h>
34
35#include "ioctl_ctrl.h"
36#include "ioctl_dbg.h"
37#include "ioctl_as.h"
38#ifdef CONFIG_TEGRA_19x_GPU
39#include "common/linux/ioctl_ctrl_t19x.h"
40#endif
41#include "ioctl_tsg.h"
42#include "ioctl_channel.h"
43#include "gk20a/gk20a.h"
44#include "gk20a/fence_gk20a.h"
45
46#include "platform_gk20a.h"
47#include "os_linux.h"
48#include "dmabuf.h"
49
50#define HZ_TO_MHZ(a) ((a > 0xF414F9CD7ULL) ? 0xffff : (a >> 32) ? \
51 (u32) ((a * 0x10C8ULL) >> 32) : (u16) ((u32) a/MHZ))
52#define MHZ_TO_HZ(a) ((u64)a * MHZ)
53
54struct gk20a_ctrl_priv {
55 struct device *dev;
56 struct gk20a *g;
57#ifdef CONFIG_ARCH_TEGRA_18x_SOC
58 struct nvgpu_clk_session *clk_session;
59#endif
60};
61
62int gk20a_ctrl_dev_open(struct inode *inode, struct file *filp)
63{
64 struct nvgpu_os_linux *l;
65 struct gk20a *g;
66 struct gk20a_ctrl_priv *priv;
67 int err = 0;
68
69 gk20a_dbg_fn("");
70
71 l = container_of(inode->i_cdev,
72 struct nvgpu_os_linux, ctrl.cdev);
73 g = gk20a_get(&l->g);
74 if (!g)
75 return -ENODEV;
76
77 priv = nvgpu_kzalloc(g, sizeof(struct gk20a_ctrl_priv));
78 if (!priv) {
79 err = -ENOMEM;
80 goto free_ref;
81 }
82 filp->private_data = priv;
83 priv->dev = dev_from_gk20a(g);
84 /*
85 * We dont close the arbiter fd's after driver teardown to support
86 * GPU_LOST events, so we store g here, instead of dereferencing the
87 * dev structure on teardown
88 */
89 priv->g = g;
90
91 if (!g->gr.sw_ready) {
92 err = gk20a_busy(g);
93 if (err)
94 goto free_ref;
95 gk20a_idle(g);
96 }
97
98#ifdef CONFIG_ARCH_TEGRA_18x_SOC
99 err = nvgpu_clk_arb_init_session(g, &priv->clk_session);
100#endif
101free_ref:
102 if (err)
103 gk20a_put(g);
104 return err;
105}
106int gk20a_ctrl_dev_release(struct inode *inode, struct file *filp)
107{
108 struct gk20a_ctrl_priv *priv = filp->private_data;
109 struct gk20a *g = priv->g;
110
111 gk20a_dbg_fn("");
112
113#ifdef CONFIG_ARCH_TEGRA_18x_SOC
114 if (priv->clk_session)
115 nvgpu_clk_arb_release_session(g, priv->clk_session);
116#endif
117
118 gk20a_put(g);
119 nvgpu_kfree(g, priv);
120
121 return 0;
122}
123
124struct nvgpu_flags_mapping {
125 u64 ioctl_flag;
126 int enabled_flag;
127};
128
129static struct nvgpu_flags_mapping flags_mapping[] = {
130 {NVGPU_GPU_FLAGS_HAS_SYNCPOINTS,
131 NVGPU_HAS_SYNCPOINTS},
132 {NVGPU_GPU_FLAGS_SUPPORT_PARTIAL_MAPPINGS,
133 NVGPU_SUPPORT_PARTIAL_MAPPINGS},
134 {NVGPU_GPU_FLAGS_SUPPORT_SPARSE_ALLOCS,
135 NVGPU_SUPPORT_SPARSE_ALLOCS},
136 {NVGPU_GPU_FLAGS_SUPPORT_SYNC_FENCE_FDS,
137 NVGPU_SUPPORT_SYNC_FENCE_FDS},
138 {NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS,
139 NVGPU_SUPPORT_CYCLE_STATS},
140 {NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS_SNAPSHOT,
141 NVGPU_SUPPORT_CYCLE_STATS_SNAPSHOT},
142 {NVGPU_GPU_FLAGS_SUPPORT_USERSPACE_MANAGED_AS,
143 NVGPU_SUPPORT_USERSPACE_MANAGED_AS},
144 {NVGPU_GPU_FLAGS_SUPPORT_TSG,
145 NVGPU_SUPPORT_TSG},
146 {NVGPU_GPU_FLAGS_SUPPORT_CLOCK_CONTROLS,
147 NVGPU_SUPPORT_CLOCK_CONTROLS},
148 {NVGPU_GPU_FLAGS_SUPPORT_GET_VOLTAGE,
149 NVGPU_SUPPORT_GET_VOLTAGE},
150 {NVGPU_GPU_FLAGS_SUPPORT_GET_CURRENT,
151 NVGPU_SUPPORT_GET_CURRENT},
152 {NVGPU_GPU_FLAGS_SUPPORT_GET_POWER,
153 NVGPU_SUPPORT_GET_POWER},
154 {NVGPU_GPU_FLAGS_SUPPORT_GET_TEMPERATURE,
155 NVGPU_SUPPORT_GET_TEMPERATURE},
156 {NVGPU_GPU_FLAGS_SUPPORT_SET_THERM_ALERT_LIMIT,
157 NVGPU_SUPPORT_SET_THERM_ALERT_LIMIT},
158 {NVGPU_GPU_FLAGS_SUPPORT_DEVICE_EVENTS,
159 NVGPU_SUPPORT_DEVICE_EVENTS},
160 {NVGPU_GPU_FLAGS_SUPPORT_FECS_CTXSW_TRACE,
161 NVGPU_SUPPORT_FECS_CTXSW_TRACE},
162 {NVGPU_GPU_FLAGS_SUPPORT_DETERMINISTIC_SUBMIT_NO_JOBTRACKING,
163 NVGPU_SUPPORT_DETERMINISTIC_SUBMIT_NO_JOBTRACKING},
164 {NVGPU_GPU_FLAGS_SUPPORT_DETERMINISTIC_SUBMIT_FULL,
165 NVGPU_SUPPORT_DETERMINISTIC_SUBMIT_FULL},
166 {NVGPU_GPU_FLAGS_SUPPORT_DETERMINISTIC_OPTS,
167 NVGPU_SUPPORT_DETERMINISTIC_OPTS},
168 {NVGPU_GPU_FLAGS_SUPPORT_IO_COHERENCE,
169 NVGPU_SUPPORT_IO_COHERENCE},
170 {NVGPU_GPU_FLAGS_SUPPORT_RESCHEDULE_RUNLIST,
171 NVGPU_SUPPORT_RESCHEDULE_RUNLIST},
172 {NVGPU_GPU_FLAGS_SUPPORT_MAP_DIRECT_KIND_CTRL,
173 NVGPU_SUPPORT_MAP_DIRECT_KIND_CTRL},
174 {NVGPU_GPU_FLAGS_ECC_ENABLED_SM_LRF,
175 NVGPU_ECC_ENABLED_SM_LRF},
176 {NVGPU_GPU_FLAGS_ECC_ENABLED_SM_SHM,
177 NVGPU_ECC_ENABLED_SM_SHM},
178 {NVGPU_GPU_FLAGS_ECC_ENABLED_TEX,
179 NVGPU_ECC_ENABLED_TEX},
180 {NVGPU_GPU_FLAGS_ECC_ENABLED_LTC,
181 NVGPU_ECC_ENABLED_LTC},
182};
183
184static u64 nvgpu_ctrl_ioctl_gpu_characteristics_flags(struct gk20a *g)
185{
186 unsigned int i;
187 u64 ioctl_flags = 0;
188
189 for (i = 0; i < sizeof(flags_mapping)/sizeof(*flags_mapping); i++) {
190 if (nvgpu_is_enabled(g, flags_mapping[i].enabled_flag))
191 ioctl_flags |= flags_mapping[i].ioctl_flag;
192 }
193
194 return ioctl_flags;
195}
196
197static void nvgpu_set_preemption_mode_flags(struct gk20a *g,
198 struct nvgpu_gpu_characteristics *gpu)
199{
200 struct nvgpu_preemption_modes_rec preemption_mode_rec;
201
202 g->ops.gr.get_preemption_mode_flags(g, &preemption_mode_rec);
203
204 gpu->graphics_preemption_mode_flags =
205 nvgpu_get_ioctl_graphics_preempt_mode_flags(
206 preemption_mode_rec.graphics_preemption_mode_flags);
207 gpu->compute_preemption_mode_flags =
208 nvgpu_get_ioctl_compute_preempt_mode_flags(
209 preemption_mode_rec.compute_preemption_mode_flags);
210
211 gpu->default_graphics_preempt_mode =
212 nvgpu_get_ioctl_graphics_preempt_mode(
213 preemption_mode_rec.default_graphics_preempt_mode);
214 gpu->default_compute_preempt_mode =
215 nvgpu_get_ioctl_compute_preempt_mode(
216 preemption_mode_rec.default_compute_preempt_mode);
217}
218
219static long
220gk20a_ctrl_ioctl_gpu_characteristics(
221 struct gk20a *g,
222 struct nvgpu_gpu_get_characteristics *request)
223{
224 struct nvgpu_gpu_characteristics gpu;
225 long err = 0;
226
227 if (gk20a_busy(g)) {
228 nvgpu_err(g, "failed to power on gpu");
229 return -EINVAL;
230 }
231
232 memset(&gpu, 0, sizeof(gpu));
233
234 gpu.L2_cache_size = g->ops.ltc.determine_L2_size_bytes(g);
235 gpu.on_board_video_memory_size = 0; /* integrated GPU */
236
237 gpu.num_gpc = g->gr.gpc_count;
238 gpu.max_gpc_count = g->gr.max_gpc_count;
239
240 gpu.num_tpc_per_gpc = g->gr.max_tpc_per_gpc_count;
241
242 gpu.bus_type = NVGPU_GPU_BUS_TYPE_AXI; /* always AXI for now */
243
244 gpu.compression_page_size = g->ops.fb.compression_page_size(g);
245
246 gpu.gpc_mask = (1 << g->gr.gpc_count)-1;
247
248 gpu.flags = nvgpu_ctrl_ioctl_gpu_characteristics_flags(g);
249#ifdef CONFIG_TEGRA_19x_GPU
250 gpu.flags |= nvgpu_ctrl_ioctl_gpu_characteristics_flags_t19x(g);
251#endif
252 gpu.arch = g->params.gpu_arch;
253 gpu.impl = g->params.gpu_impl;
254 gpu.rev = g->params.gpu_rev;
255 gpu.reg_ops_limit = NVGPU_IOCTL_DBG_REG_OPS_LIMIT;
256 gpu.map_buffer_batch_limit = nvgpu_is_enabled(g, NVGPU_SUPPORT_MAP_BUFFER_BATCH) ?
257 NVGPU_IOCTL_AS_MAP_BUFFER_BATCH_LIMIT : 0;
258 gpu.twod_class = g->ops.get_litter_value(g, GPU_LIT_TWOD_CLASS);
259 gpu.threed_class = g->ops.get_litter_value(g, GPU_LIT_THREED_CLASS);
260 gpu.compute_class = g->ops.get_litter_value(g, GPU_LIT_COMPUTE_CLASS);
261 gpu.gpfifo_class = g->ops.get_litter_value(g, GPU_LIT_GPFIFO_CLASS);
262 gpu.inline_to_memory_class =
263 g->ops.get_litter_value(g, GPU_LIT_I2M_CLASS);
264 gpu.dma_copy_class =
265 g->ops.get_litter_value(g, GPU_LIT_DMA_COPY_CLASS);
266
267 gpu.vbios_version = g->bios.vbios_version;
268 gpu.vbios_oem_version = g->bios.vbios_oem_version;
269
270 gpu.big_page_size = nvgpu_mm_get_default_big_page_size(g);
271 gpu.pde_coverage_bit_count =
272 g->ops.mm.get_mmu_levels(g, gpu.big_page_size)[0].lo_bit[0];
273 gpu.available_big_page_sizes = nvgpu_mm_get_available_big_page_sizes(g);
274
275 gpu.sm_arch_sm_version = g->params.sm_arch_sm_version;
276 gpu.sm_arch_spa_version = g->params.sm_arch_spa_version;
277 gpu.sm_arch_warp_count = g->params.sm_arch_warp_count;
278
279 gpu.max_css_buffer_size = g->gr.max_css_buffer_size;
280
281 gpu.gpu_ioctl_nr_last = NVGPU_GPU_IOCTL_LAST;
282 gpu.tsg_ioctl_nr_last = NVGPU_TSG_IOCTL_LAST;
283 gpu.dbg_gpu_ioctl_nr_last = NVGPU_DBG_GPU_IOCTL_LAST;
284 gpu.ioctl_channel_nr_last = NVGPU_IOCTL_CHANNEL_LAST;
285 gpu.as_ioctl_nr_last = NVGPU_AS_IOCTL_LAST;
286 gpu.event_ioctl_nr_last = NVGPU_EVENT_IOCTL_LAST;
287 gpu.gpu_va_bit_count = 40;
288
289 strlcpy(gpu.chipname, g->name, sizeof(gpu.chipname));
290 gpu.max_fbps_count = g->ops.gr.get_max_fbps_count(g);
291 gpu.fbp_en_mask = g->ops.gr.get_fbp_en_mask(g);
292 gpu.max_ltc_per_fbp = g->ops.gr.get_max_ltc_per_fbp(g);
293 gpu.max_lts_per_ltc = g->ops.gr.get_max_lts_per_ltc(g);
294 gpu.gr_compbit_store_base_hw = g->gr.compbit_store.base_hw;
295 gpu.gr_gobs_per_comptagline_per_slice =
296 g->gr.gobs_per_comptagline_per_slice;
297 gpu.num_ltc = g->ltc_count;
298 gpu.lts_per_ltc = g->gr.slices_per_ltc;
299 gpu.cbc_cache_line_size = g->gr.cacheline_size;
300 gpu.cbc_comptags_per_line = g->gr.comptags_per_cacheline;
301
302 if (g->ops.clk.get_maxrate)
303 gpu.max_freq = g->ops.clk.get_maxrate(g, CTRL_CLK_DOMAIN_GPCCLK);
304
305 gpu.local_video_memory_size = g->mm.vidmem.size;
306
307 gpu.pci_vendor_id = g->pci_vendor_id;
308 gpu.pci_device_id = g->pci_device_id;
309 gpu.pci_subsystem_vendor_id = g->pci_subsystem_vendor_id;
310 gpu.pci_subsystem_device_id = g->pci_subsystem_device_id;
311 gpu.pci_class = g->pci_class;
312 gpu.pci_revision = g->pci_revision;
313
314 nvgpu_set_preemption_mode_flags(g, &gpu);
315
316 if (request->gpu_characteristics_buf_size > 0) {
317 size_t write_size = sizeof(gpu);
318
319 if (write_size > request->gpu_characteristics_buf_size)
320 write_size = request->gpu_characteristics_buf_size;
321
322 err = copy_to_user((void __user *)(uintptr_t)
323 request->gpu_characteristics_buf_addr,
324 &gpu, write_size);
325 }
326
327 if (err == 0)
328 request->gpu_characteristics_buf_size = sizeof(gpu);
329
330 gk20a_idle(g);
331
332 return err;
333}
334
335static int gk20a_ctrl_prepare_compressible_read(
336 struct gk20a *g,
337 struct nvgpu_gpu_prepare_compressible_read_args *args)
338{
339 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
340 struct nvgpu_fence fence;
341 struct gk20a_fence *fence_out = NULL;
342 int ret = 0;
343 int flags = args->submit_flags;
344
345 fence.id = args->fence.syncpt_id;
346 fence.value = args->fence.syncpt_value;
347
348 ret = gk20a_prepare_compressible_read(l, args->handle,
349 args->request_compbits, args->offset,
350 args->compbits_hoffset, args->compbits_voffset,
351 args->scatterbuffer_offset,
352 args->width, args->height, args->block_height_log2,
353 flags, &fence, &args->valid_compbits,
354 &args->zbc_color, &fence_out);
355
356 if (ret)
357 return ret;
358
359 /* Convert fence_out to something we can pass back to user space. */
360 if (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_FENCE_GET) {
361 if (flags & NVGPU_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE) {
362 if (fence_out) {
363 int fd = gk20a_fence_install_fd(fence_out);
364 if (fd < 0)
365 ret = fd;
366 else
367 args->fence.fd = fd;
368 } else {
369 args->fence.fd = -1;
370 }
371 } else {
372 if (fence_out) {
373 args->fence.syncpt_id = fence_out->syncpt_id;
374 args->fence.syncpt_value =
375 fence_out->syncpt_value;
376 } else {
377 args->fence.syncpt_id = -1;
378 args->fence.syncpt_value = 0;
379 }
380 }
381 }
382 gk20a_fence_put(fence_out);
383
384 return 0;
385}
386
387static int gk20a_ctrl_mark_compressible_write(
388 struct gk20a *g,
389 struct nvgpu_gpu_mark_compressible_write_args *args)
390{
391 int ret;
392
393 ret = gk20a_mark_compressible_write(g, args->handle,
394 args->valid_compbits, args->offset, args->zbc_color);
395
396 return ret;
397}
398
399static int gk20a_ctrl_alloc_as(
400 struct gk20a *g,
401 struct nvgpu_alloc_as_args *args)
402{
403 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
404 struct gk20a_as_share *as_share;
405 int err;
406 int fd;
407 struct file *file;
408 char name[64];
409
410 err = get_unused_fd_flags(O_RDWR);
411 if (err < 0)
412 return err;
413 fd = err;
414
415 snprintf(name, sizeof(name), "nvhost-%s-fd%d", g->name, fd);
416
417 file = anon_inode_getfile(name, l->as_dev.cdev.ops, NULL, O_RDWR);
418 if (IS_ERR(file)) {
419 err = PTR_ERR(file);
420 goto clean_up;
421 }
422
423 err = gk20a_as_alloc_share(g, args->big_page_size, args->flags,
424 &as_share);
425 if (err)
426 goto clean_up_file;
427
428 fd_install(fd, file);
429 file->private_data = as_share;
430
431 args->as_fd = fd;
432 return 0;
433
434clean_up_file:
435 fput(file);
436clean_up:
437 put_unused_fd(fd);
438 return err;
439}
440
441static int gk20a_ctrl_open_tsg(struct gk20a *g,
442 struct nvgpu_gpu_open_tsg_args *args)
443{
444 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
445 int err;
446 int fd;
447 struct file *file;
448 char name[64];
449
450 err = get_unused_fd_flags(O_RDWR);
451 if (err < 0)
452 return err;
453 fd = err;
454
455 snprintf(name, sizeof(name), "nvgpu-%s-tsg%d", g->name, fd);
456
457 file = anon_inode_getfile(name, l->tsg.cdev.ops, NULL, O_RDWR);
458 if (IS_ERR(file)) {
459 err = PTR_ERR(file);
460 goto clean_up;
461 }
462
463 err = nvgpu_ioctl_tsg_open(g, file);
464 if (err)
465 goto clean_up_file;
466
467 fd_install(fd, file);
468 args->tsg_fd = fd;
469 return 0;
470
471clean_up_file:
472 fput(file);
473clean_up:
474 put_unused_fd(fd);
475 return err;
476}
477
478static int gk20a_ctrl_get_tpc_masks(struct gk20a *g,
479 struct nvgpu_gpu_get_tpc_masks_args *args)
480{
481 struct gr_gk20a *gr = &g->gr;
482 int err = 0;
483 const u32 gpc_tpc_mask_size = sizeof(u32) * gr->gpc_count;
484
485 if (args->mask_buf_size > 0) {
486 size_t write_size = gpc_tpc_mask_size;
487
488 if (write_size > args->mask_buf_size)
489 write_size = args->mask_buf_size;
490
491 err = copy_to_user((void __user *)(uintptr_t)
492 args->mask_buf_addr,
493 gr->gpc_tpc_mask, write_size);
494 }
495
496 if (err == 0)
497 args->mask_buf_size = gpc_tpc_mask_size;
498
499 return err;
500}
501
502static int gk20a_ctrl_get_fbp_l2_masks(
503 struct gk20a *g, struct nvgpu_gpu_get_fbp_l2_masks_args *args)
504{
505 struct gr_gk20a *gr = &g->gr;
506 int err = 0;
507 const u32 fbp_l2_mask_size = sizeof(u32) * gr->max_fbps_count;
508
509 if (args->mask_buf_size > 0) {
510 size_t write_size = fbp_l2_mask_size;
511
512 if (write_size > args->mask_buf_size)
513 write_size = args->mask_buf_size;
514
515 err = copy_to_user((void __user *)(uintptr_t)
516 args->mask_buf_addr,
517 gr->fbp_rop_l2_en_mask, write_size);
518 }
519
520 if (err == 0)
521 args->mask_buf_size = fbp_l2_mask_size;
522
523 return err;
524}
525
526static int nvgpu_gpu_ioctl_l2_fb_ops(struct gk20a *g,
527 struct nvgpu_gpu_l2_fb_args *args)
528{
529 int err = 0;
530
531 if ((!args->l2_flush && !args->fb_flush) ||
532 (!args->l2_flush && args->l2_invalidate))
533 return -EINVAL;
534
535 if (args->l2_flush)
536 g->ops.mm.l2_flush(g, args->l2_invalidate ? true : false);
537
538 if (args->fb_flush)
539 g->ops.mm.fb_flush(g);
540
541 return err;
542}
543
544/* Invalidate i-cache for kepler & maxwell */
545static int nvgpu_gpu_ioctl_inval_icache(
546 struct gk20a *g,
547 struct nvgpu_gpu_inval_icache_args *args)
548{
549 struct channel_gk20a *ch;
550 int err;
551
552 ch = gk20a_get_channel_from_file(args->channel_fd);
553 if (!ch)
554 return -EINVAL;
555
556 /* Take the global lock, since we'll be doing global regops */
557 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
558 err = g->ops.gr.inval_icache(g, ch);
559 nvgpu_mutex_release(&g->dbg_sessions_lock);
560
561 gk20a_channel_put(ch);
562 return err;
563}
564
565static int nvgpu_gpu_ioctl_set_mmu_debug_mode(
566 struct gk20a *g,
567 struct nvgpu_gpu_mmu_debug_mode_args *args)
568{
569 if (gk20a_busy(g)) {
570 nvgpu_err(g, "failed to power on gpu");
571 return -EINVAL;
572 }
573
574 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
575 g->ops.fb.set_debug_mode(g, args->state == 1);
576 nvgpu_mutex_release(&g->dbg_sessions_lock);
577
578 gk20a_idle(g);
579 return 0;
580}
581
582static int nvgpu_gpu_ioctl_set_debug_mode(
583 struct gk20a *g,
584 struct nvgpu_gpu_sm_debug_mode_args *args)
585{
586 struct channel_gk20a *ch;
587 int err;
588
589 ch = gk20a_get_channel_from_file(args->channel_fd);
590 if (!ch)
591 return -EINVAL;
592
593 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
594 if (g->ops.gr.set_sm_debug_mode)
595 err = g->ops.gr.set_sm_debug_mode(g, ch,
596 args->sms, !!args->enable);
597 else
598 err = -ENOSYS;
599 nvgpu_mutex_release(&g->dbg_sessions_lock);
600
601 gk20a_channel_put(ch);
602 return err;
603}
604
605static int nvgpu_gpu_ioctl_trigger_suspend(struct gk20a *g)
606{
607 int err;
608
609 err = gk20a_busy(g);
610 if (err)
611 return err;
612
613 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
614 err = g->ops.gr.trigger_suspend(g);
615 nvgpu_mutex_release(&g->dbg_sessions_lock);
616
617 gk20a_idle(g);
618
619 return err;
620}
621
622static int nvgpu_gpu_ioctl_wait_for_pause(struct gk20a *g,
623 struct nvgpu_gpu_wait_pause_args *args)
624{
625 int err;
626 struct warpstate *ioctl_w_state;
627 struct nvgpu_warpstate *w_state = NULL;
628 u32 sm_count, ioctl_size, size, sm_id;
629
630 sm_count = g->gr.gpc_count * g->gr.tpc_count;
631
632 ioctl_size = sm_count * sizeof(struct warpstate);
633 ioctl_w_state = nvgpu_kzalloc(g, ioctl_size);
634 if (!ioctl_w_state)
635 return -ENOMEM;
636
637 size = sm_count * sizeof(struct nvgpu_warpstate);
638 w_state = nvgpu_kzalloc(g, size);
639 if (!w_state) {
640 err = -ENOMEM;
641 goto out_free;
642 }
643
644 err = gk20a_busy(g);
645 if (err)
646 goto out_free;
647
648 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
649 g->ops.gr.wait_for_pause(g, w_state);
650
651 for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) {
652 ioctl_w_state[sm_id].valid_warps[0] =
653 w_state[sm_id].valid_warps[0];
654 ioctl_w_state[sm_id].valid_warps[1] =
655 w_state[sm_id].valid_warps[1];
656 ioctl_w_state[sm_id].trapped_warps[0] =
657 w_state[sm_id].trapped_warps[0];
658 ioctl_w_state[sm_id].trapped_warps[1] =
659 w_state[sm_id].trapped_warps[1];
660 ioctl_w_state[sm_id].paused_warps[0] =
661 w_state[sm_id].paused_warps[0];
662 ioctl_w_state[sm_id].paused_warps[1] =
663 w_state[sm_id].paused_warps[1];
664 }
665 /* Copy to user space - pointed by "args->pwarpstate" */
666 if (copy_to_user((void __user *)(uintptr_t)args->pwarpstate,
667 w_state, ioctl_size)) {
668 gk20a_dbg_fn("copy_to_user failed!");
669 err = -EFAULT;
670 }
671
672 nvgpu_mutex_release(&g->dbg_sessions_lock);
673
674 gk20a_idle(g);
675
676out_free:
677 nvgpu_kfree(g, w_state);
678 nvgpu_kfree(g, ioctl_w_state);
679
680 return err;
681}
682
683static int nvgpu_gpu_ioctl_resume_from_pause(struct gk20a *g)
684{
685 int err;
686
687 err = gk20a_busy(g);
688 if (err)
689 return err;
690
691 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
692 err = g->ops.gr.resume_from_pause(g);
693 nvgpu_mutex_release(&g->dbg_sessions_lock);
694
695 gk20a_idle(g);
696
697 return err;
698}
699
700static int nvgpu_gpu_ioctl_clear_sm_errors(struct gk20a *g)
701{
702 int err;
703
704 err = gk20a_busy(g);
705 if (err)
706 return err;
707
708 err = g->ops.gr.clear_sm_errors(g);
709
710 gk20a_idle(g);
711
712 return err;
713}
714
715static int nvgpu_gpu_ioctl_has_any_exception(
716 struct gk20a *g,
717 struct nvgpu_gpu_tpc_exception_en_status_args *args)
718{
719 u32 tpc_exception_en;
720
721 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
722 tpc_exception_en = g->ops.gr.tpc_enabled_exceptions(g);
723 nvgpu_mutex_release(&g->dbg_sessions_lock);
724
725 args->tpc_exception_en_sm_mask = tpc_exception_en;
726
727 return 0;
728}
729
730static int gk20a_ctrl_get_num_vsms(struct gk20a *g,
731 struct nvgpu_gpu_num_vsms *args)
732{
733 struct gr_gk20a *gr = &g->gr;
734 args->num_vsms = gr->no_of_sm;
735 return 0;
736}
737
738static int gk20a_ctrl_vsm_mapping(struct gk20a *g,
739 struct nvgpu_gpu_vsms_mapping *args)
740{
741 int err = 0;
742 struct gr_gk20a *gr = &g->gr;
743 size_t write_size = gr->no_of_sm *
744 sizeof(struct nvgpu_gpu_vsms_mapping_entry);
745 struct nvgpu_gpu_vsms_mapping_entry *vsms_buf;
746 u32 i;
747
748 vsms_buf = nvgpu_kzalloc(g, write_size);
749 if (vsms_buf == NULL)
750 return -ENOMEM;
751
752 for (i = 0; i < gr->no_of_sm; i++) {
753 vsms_buf[i].gpc_index = gr->sm_to_cluster[i].gpc_index;
754 vsms_buf[i].tpc_index = gr->sm_to_cluster[i].tpc_index;
755 }
756
757 err = copy_to_user((void __user *)(uintptr_t)
758 args->vsms_map_buf_addr,
759 vsms_buf, write_size);
760 nvgpu_kfree(g, vsms_buf);
761
762 return err;
763}
764
765static int gk20a_ctrl_get_buffer_info(
766 struct gk20a *g, struct nvgpu_gpu_get_buffer_info_args *args)
767{
768 return gk20a_mm_get_buffer_info(dev_from_gk20a(g), args->in.dmabuf_fd,
769 &args->out.id, &args->out.length);
770}
771
772static int nvgpu_gpu_get_cpu_time_correlation_info(
773 struct gk20a *g,
774 struct nvgpu_gpu_get_cpu_time_correlation_info_args *args)
775{
776 struct nvgpu_cpu_time_correlation_sample *samples;
777 int err;
778 u32 i;
779
780 if (args->count > NVGPU_GPU_GET_CPU_TIME_CORRELATION_INFO_MAX_COUNT ||
781 args->source_id != NVGPU_GPU_GET_CPU_TIME_CORRELATION_INFO_SRC_ID_TSC)
782 return -EINVAL;
783
784 samples = nvgpu_kzalloc(g, args->count *
785 sizeof(struct nvgpu_cpu_time_correlation_sample));
786 if (!samples) {
787 return -ENOMEM;
788 }
789
790 err = g->ops.bus.get_timestamps_zipper(g,
791 args->source_id, args->count, samples);
792 if (!err) {
793 for (i = 0; i < args->count; i++) {
794 args->samples[i].cpu_timestamp = samples[i].cpu_timestamp;
795 args->samples[i].gpu_timestamp = samples[i].gpu_timestamp;
796 }
797 }
798
799 nvgpu_kfree(g, samples);
800
801 return err;
802}
803
804static int nvgpu_gpu_get_gpu_time(
805 struct gk20a *g,
806 struct nvgpu_gpu_get_gpu_time_args *args)
807{
808 u64 time;
809 int err;
810
811 err = gk20a_busy(g);
812 if (err)
813 return err;
814
815 err = g->ops.bus.read_ptimer(g, &time);
816 if (!err)
817 args->gpu_timestamp = time;
818
819 gk20a_idle(g);
820 return err;
821}
822
823static int nvgpu_gpu_get_engine_info(
824 struct gk20a *g,
825 struct nvgpu_gpu_get_engine_info_args *args)
826{
827 int err = 0;
828 u32 engine_enum = ENGINE_INVAL_GK20A;
829 u32 report_index = 0;
830 u32 engine_id_idx;
831 const u32 max_buffer_engines = args->engine_info_buf_size /
832 sizeof(struct nvgpu_gpu_get_engine_info_item);
833 struct nvgpu_gpu_get_engine_info_item __user *dst_item_list =
834 (void __user *)(uintptr_t)args->engine_info_buf_addr;
835
836 for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines;
837 ++engine_id_idx) {
838 u32 active_engine_id = g->fifo.active_engines_list[engine_id_idx];
839 const struct fifo_engine_info_gk20a *src_info =
840 &g->fifo.engine_info[active_engine_id];
841 struct nvgpu_gpu_get_engine_info_item dst_info;
842
843 memset(&dst_info, 0, sizeof(dst_info));
844
845 engine_enum = src_info->engine_enum;
846
847 switch (engine_enum) {
848 case ENGINE_GR_GK20A:
849 dst_info.engine_id = NVGPU_GPU_ENGINE_ID_GR;
850 break;
851
852 case ENGINE_GRCE_GK20A:
853 dst_info.engine_id = NVGPU_GPU_ENGINE_ID_GR_COPY;
854 break;
855
856 case ENGINE_ASYNC_CE_GK20A:
857 dst_info.engine_id = NVGPU_GPU_ENGINE_ID_ASYNC_COPY;
858 break;
859
860 default:
861 nvgpu_err(g, "Unmapped engine enum %u",
862 engine_enum);
863 continue;
864 }
865
866 dst_info.engine_instance = src_info->inst_id;
867 dst_info.runlist_id = src_info->runlist_id;
868
869 if (report_index < max_buffer_engines) {
870 err = copy_to_user(&dst_item_list[report_index],
871 &dst_info, sizeof(dst_info));
872 if (err)
873 goto clean_up;
874 }
875
876 ++report_index;
877 }
878
879 args->engine_info_buf_size =
880 report_index * sizeof(struct nvgpu_gpu_get_engine_info_item);
881
882clean_up:
883 return err;
884}
885
886static int nvgpu_gpu_alloc_vidmem(struct gk20a *g,
887 struct nvgpu_gpu_alloc_vidmem_args *args)
888{
889 u32 align = args->in.alignment ? args->in.alignment : SZ_4K;
890 int fd;
891
892 gk20a_dbg_fn("");
893
894 /* not yet supported */
895 if (WARN_ON(args->in.flags & NVGPU_GPU_ALLOC_VIDMEM_FLAG_CPU_MASK))
896 return -EINVAL;
897
898 /* not yet supported */
899 if (WARN_ON(args->in.flags & NVGPU_GPU_ALLOC_VIDMEM_FLAG_VPR))
900 return -EINVAL;
901
902 if (args->in.size & (SZ_4K - 1))
903 return -EINVAL;
904
905 if (!args->in.size)
906 return -EINVAL;
907
908 if (align & (align - 1))
909 return -EINVAL;
910
911 if (align > roundup_pow_of_two(args->in.size)) {
912 /* log this special case, buddy allocator detail */
913 nvgpu_warn(g,
914 "alignment larger than buffer size rounded up to power of 2 is not supported");
915 return -EINVAL;
916 }
917
918 fd = nvgpu_vidmem_export_linux(g, args->in.size);
919 if (fd < 0)
920 return fd;
921
922 args->out.dmabuf_fd = fd;
923
924 gk20a_dbg_fn("done, fd=%d", fd);
925
926 return 0;
927}
928
929static int nvgpu_gpu_get_memory_state(struct gk20a *g,
930 struct nvgpu_gpu_get_memory_state_args *args)
931{
932 int err;
933
934 gk20a_dbg_fn("");
935
936 if (args->reserved[0] || args->reserved[1] ||
937 args->reserved[2] || args->reserved[3])
938 return -EINVAL;
939
940 err = nvgpu_vidmem_get_space(g, &args->total_free_bytes);
941
942 gk20a_dbg_fn("done, err=%d, bytes=%lld", err, args->total_free_bytes);
943
944 return err;
945}
946
947#ifdef CONFIG_ARCH_TEGRA_18x_SOC
948static int nvgpu_gpu_clk_get_vf_points(struct gk20a *g,
949 struct gk20a_ctrl_priv *priv,
950 struct nvgpu_gpu_clk_vf_points_args *args)
951{
952 struct nvgpu_gpu_clk_vf_point clk_point;
953 struct nvgpu_gpu_clk_vf_point __user *entry;
954 struct nvgpu_clk_session *session = priv->clk_session;
955 u32 clk_domains = 0;
956 int err;
957 u16 last_mhz;
958 u16 *fpoints;
959 u32 i;
960 u32 max_points = 0;
961 u32 num_points = 0;
962 u16 min_mhz;
963 u16 max_mhz;
964
965 gk20a_dbg_fn("");
966
967 if (!session || args->flags)
968 return -EINVAL;
969
970 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
971 args->num_entries = 0;
972
973 if (!nvgpu_clk_arb_is_valid_domain(g, args->clk_domain))
974 return -EINVAL;
975
976 err = nvgpu_clk_arb_get_arbiter_clk_f_points(g,
977 args->clk_domain, &max_points, NULL);
978 if (err)
979 return err;
980
981 if (!args->max_entries) {
982 args->max_entries = max_points;
983 return 0;
984 }
985
986 if (args->max_entries < max_points)
987 return -EINVAL;
988
989 err = nvgpu_clk_arb_get_arbiter_clk_range(g, args->clk_domain,
990 &min_mhz, &max_mhz);
991 if (err)
992 return err;
993
994 fpoints = nvgpu_kcalloc(g, max_points, sizeof(u16));
995 if (!fpoints)
996 return -ENOMEM;
997
998 err = nvgpu_clk_arb_get_arbiter_clk_f_points(g,
999 args->clk_domain, &max_points, fpoints);
1000 if (err)
1001 goto fail;
1002
1003 entry = (struct nvgpu_gpu_clk_vf_point __user *)
1004 (uintptr_t)args->clk_vf_point_entries;
1005
1006 last_mhz = 0;
1007 num_points = 0;
1008 for (i = 0; (i < max_points) && !err; i++) {
1009
1010 /* filter out duplicate frequencies */
1011 if (fpoints[i] == last_mhz)
1012 continue;
1013
1014 /* filter out out-of-range frequencies */
1015 if ((fpoints[i] < min_mhz) || (fpoints[i] > max_mhz))
1016 continue;
1017
1018 last_mhz = fpoints[i];
1019 clk_point.freq_hz = MHZ_TO_HZ(fpoints[i]);
1020
1021 err = copy_to_user((void __user *)entry, &clk_point,
1022 sizeof(clk_point));
1023
1024 num_points++;
1025 entry++;
1026 }
1027
1028 args->num_entries = num_points;
1029
1030fail:
1031 nvgpu_kfree(g, fpoints);
1032 return err;
1033}
1034
1035static int nvgpu_gpu_clk_get_range(struct gk20a *g,
1036 struct gk20a_ctrl_priv *priv,
1037 struct nvgpu_gpu_clk_range_args *args)
1038{
1039 struct nvgpu_gpu_clk_range clk_range;
1040 struct nvgpu_gpu_clk_range __user *entry;
1041 struct nvgpu_clk_session *session = priv->clk_session;
1042
1043 u32 clk_domains = 0;
1044 u32 num_domains;
1045 u32 num_entries;
1046 u32 i;
1047 int bit;
1048 int err;
1049 u16 min_mhz, max_mhz;
1050
1051 gk20a_dbg_fn("");
1052
1053 if (!session)
1054 return -EINVAL;
1055
1056 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
1057 num_domains = hweight_long(clk_domains);
1058
1059 if (!args->flags) {
1060 if (!args->num_entries) {
1061 args->num_entries = num_domains;
1062 return 0;
1063 }
1064
1065 if (args->num_entries < num_domains)
1066 return -EINVAL;
1067
1068 args->num_entries = 0;
1069 num_entries = num_domains;
1070
1071 } else {
1072 if (args->flags != NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS)
1073 return -EINVAL;
1074
1075 num_entries = args->num_entries;
1076 if (num_entries > num_domains)
1077 return -EINVAL;
1078 }
1079
1080 entry = (struct nvgpu_gpu_clk_range __user *)
1081 (uintptr_t)args->clk_range_entries;
1082
1083 for (i = 0; i < num_entries; i++, entry++) {
1084
1085 if (args->flags == NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS) {
1086 if (copy_from_user(&clk_range, (void __user *)entry,
1087 sizeof(clk_range)))
1088 return -EFAULT;
1089 } else {
1090 bit = ffs(clk_domains) - 1;
1091 clk_range.clk_domain = bit;
1092 clk_domains &= ~BIT(bit);
1093 }
1094
1095 clk_range.flags = 0;
1096 err = nvgpu_clk_arb_get_arbiter_clk_range(g,
1097 clk_range.clk_domain,
1098 &min_mhz, &max_mhz);
1099 clk_range.min_hz = MHZ_TO_HZ(min_mhz);
1100 clk_range.max_hz = MHZ_TO_HZ(max_mhz);
1101
1102 if (err)
1103 return err;
1104
1105 err = copy_to_user(entry, &clk_range, sizeof(clk_range));
1106 if (err)
1107 return -EFAULT;
1108 }
1109
1110 args->num_entries = num_entries;
1111
1112 return 0;
1113}
1114
1115
1116static int nvgpu_gpu_clk_set_info(struct gk20a *g,
1117 struct gk20a_ctrl_priv *priv,
1118 struct nvgpu_gpu_clk_set_info_args *args)
1119{
1120 struct nvgpu_gpu_clk_info clk_info;
1121 struct nvgpu_gpu_clk_info __user *entry;
1122 struct nvgpu_clk_session *session = priv->clk_session;
1123
1124 int fd;
1125 u32 clk_domains = 0;
1126 u16 freq_mhz;
1127 int i;
1128 int ret;
1129
1130 gk20a_dbg_fn("");
1131
1132 if (!session || args->flags)
1133 return -EINVAL;
1134
1135 gk20a_dbg_info("line=%d", __LINE__);
1136
1137 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
1138 if (!clk_domains)
1139 return -EINVAL;
1140
1141 entry = (struct nvgpu_gpu_clk_info __user *)
1142 (uintptr_t)args->clk_info_entries;
1143
1144 gk20a_dbg_info("line=%d", __LINE__);
1145
1146 for (i = 0; i < args->num_entries; i++, entry++) {
1147
1148 gk20a_dbg_info("line=%d", __LINE__);
1149 if (copy_from_user(&clk_info, entry, sizeof(clk_info)))
1150 return -EFAULT;
1151
1152 gk20a_dbg_info("i=%d domain=0x%08x", i, clk_info.clk_domain);
1153
1154 if (!nvgpu_clk_arb_is_valid_domain(g, clk_info.clk_domain))
1155 return -EINVAL;
1156 }
1157
1158 entry = (struct nvgpu_gpu_clk_info __user *)
1159 (uintptr_t)args->clk_info_entries;
1160
1161 ret = nvgpu_clk_arb_install_request_fd(g, session, &fd);
1162 if (ret < 0)
1163 return ret;
1164
1165 for (i = 0; i < args->num_entries; i++, entry++) {
1166
1167 if (copy_from_user(&clk_info, (void __user *)entry,
1168 sizeof(clk_info)))
1169 return -EFAULT;
1170 freq_mhz = HZ_TO_MHZ(clk_info.freq_hz);
1171
1172 nvgpu_clk_arb_set_session_target_mhz(session, fd,
1173 clk_info.clk_domain, freq_mhz);
1174 }
1175
1176 ret = nvgpu_clk_arb_commit_request_fd(g, session, fd);
1177 if (ret < 0)
1178 return ret;
1179
1180 args->completion_fd = fd;
1181
1182 return ret;
1183}
1184
1185static int nvgpu_gpu_clk_get_info(struct gk20a *g,
1186 struct gk20a_ctrl_priv *priv,
1187 struct nvgpu_gpu_clk_get_info_args *args)
1188{
1189 struct nvgpu_gpu_clk_info clk_info;
1190 struct nvgpu_gpu_clk_info __user *entry;
1191 struct nvgpu_clk_session *session = priv->clk_session;
1192 u32 clk_domains = 0;
1193 u32 num_domains;
1194 u32 num_entries;
1195 u32 i;
1196 u16 freq_mhz;
1197 int err;
1198 int bit;
1199
1200 gk20a_dbg_fn("");
1201
1202 if (!session)
1203 return -EINVAL;
1204
1205 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
1206 num_domains = hweight_long(clk_domains);
1207
1208 if (!args->flags) {
1209 if (!args->num_entries) {
1210 args->num_entries = num_domains;
1211 return 0;
1212 }
1213
1214 if (args->num_entries < num_domains)
1215 return -EINVAL;
1216
1217 args->num_entries = 0;
1218 num_entries = num_domains;
1219
1220 } else {
1221 if (args->flags != NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS)
1222 return -EINVAL;
1223
1224 num_entries = args->num_entries;
1225 if (num_entries > num_domains * 3)
1226 return -EINVAL;
1227 }
1228
1229 entry = (struct nvgpu_gpu_clk_info __user *)
1230 (uintptr_t)args->clk_info_entries;
1231
1232 for (i = 0; i < num_entries; i++, entry++) {
1233
1234 if (args->flags == NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS) {
1235 if (copy_from_user(&clk_info, (void __user *)entry,
1236 sizeof(clk_info)))
1237 return -EFAULT;
1238 } else {
1239 bit = ffs(clk_domains) - 1;
1240 clk_info.clk_domain = bit;
1241 clk_domains &= ~BIT(bit);
1242 clk_info.clk_type = args->clk_type;
1243 }
1244
1245 switch (clk_info.clk_type) {
1246 case NVGPU_GPU_CLK_TYPE_TARGET:
1247 err = nvgpu_clk_arb_get_session_target_mhz(session,
1248 clk_info.clk_domain, &freq_mhz);
1249 break;
1250 case NVGPU_GPU_CLK_TYPE_ACTUAL:
1251 err = nvgpu_clk_arb_get_arbiter_actual_mhz(g,
1252 clk_info.clk_domain, &freq_mhz);
1253 break;
1254 case NVGPU_GPU_CLK_TYPE_EFFECTIVE:
1255 err = nvgpu_clk_arb_get_arbiter_effective_mhz(g,
1256 clk_info.clk_domain, &freq_mhz);
1257 break;
1258 default:
1259 freq_mhz = 0;
1260 err = -EINVAL;
1261 break;
1262 }
1263 if (err)
1264 return err;
1265
1266 clk_info.flags = 0;
1267 clk_info.freq_hz = MHZ_TO_HZ(freq_mhz);
1268
1269 err = copy_to_user((void __user *)entry, &clk_info,
1270 sizeof(clk_info));
1271 if (err)
1272 return -EFAULT;
1273 }
1274
1275 args->num_entries = num_entries;
1276
1277 return 0;
1278}
1279
1280static int nvgpu_gpu_get_event_fd(struct gk20a *g,
1281 struct gk20a_ctrl_priv *priv,
1282 struct nvgpu_gpu_get_event_fd_args *args)
1283{
1284 struct nvgpu_clk_session *session = priv->clk_session;
1285
1286 gk20a_dbg_fn("");
1287
1288 if (!session)
1289 return -EINVAL;
1290
1291 return nvgpu_clk_arb_install_event_fd(g, session, &args->event_fd,
1292 args->flags);
1293}
1294
1295static int nvgpu_gpu_get_voltage(struct gk20a *g,
1296 struct nvgpu_gpu_get_voltage_args *args)
1297{
1298 int err = -EINVAL;
1299
1300 gk20a_dbg_fn("");
1301
1302 if (args->reserved)
1303 return -EINVAL;
1304
1305 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_GET_VOLTAGE))
1306 return -EINVAL;
1307
1308 err = gk20a_busy(g);
1309 if (err)
1310 return err;
1311
1312 switch (args->which) {
1313 case NVGPU_GPU_VOLTAGE_CORE:
1314 err = volt_get_voltage(g, CTRL_VOLT_DOMAIN_LOGIC, &args->voltage);
1315 break;
1316 case NVGPU_GPU_VOLTAGE_SRAM:
1317 err = volt_get_voltage(g, CTRL_VOLT_DOMAIN_SRAM, &args->voltage);
1318 break;
1319 case NVGPU_GPU_VOLTAGE_BUS:
1320 err = pmgr_pwr_devices_get_voltage(g, &args->voltage);
1321 break;
1322 default:
1323 err = -EINVAL;
1324 }
1325
1326 gk20a_idle(g);
1327
1328 return err;
1329}
1330
1331static int nvgpu_gpu_get_current(struct gk20a *g,
1332 struct nvgpu_gpu_get_current_args *args)
1333{
1334 int err;
1335
1336 gk20a_dbg_fn("");
1337
1338 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1339 return -EINVAL;
1340
1341 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_GET_CURRENT))
1342 return -EINVAL;
1343
1344 err = gk20a_busy(g);
1345 if (err)
1346 return err;
1347
1348 err = pmgr_pwr_devices_get_current(g, &args->currnt);
1349
1350 gk20a_idle(g);
1351
1352 return err;
1353}
1354
1355static int nvgpu_gpu_get_power(struct gk20a *g,
1356 struct nvgpu_gpu_get_power_args *args)
1357{
1358 int err;
1359
1360 gk20a_dbg_fn("");
1361
1362 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1363 return -EINVAL;
1364
1365 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_GET_POWER))
1366 return -EINVAL;
1367
1368 err = gk20a_busy(g);
1369 if (err)
1370 return err;
1371
1372 err = pmgr_pwr_devices_get_power(g, &args->power);
1373
1374 gk20a_idle(g);
1375
1376 return err;
1377}
1378
1379static int nvgpu_gpu_get_temperature(struct gk20a *g,
1380 struct nvgpu_gpu_get_temperature_args *args)
1381{
1382 int err;
1383 u32 temp_f24_8;
1384
1385 gk20a_dbg_fn("");
1386
1387 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1388 return -EINVAL;
1389
1390 if (!g->ops.therm.get_internal_sensor_curr_temp)
1391 return -EINVAL;
1392
1393 err = gk20a_busy(g);
1394 if (err)
1395 return err;
1396
1397 err = g->ops.therm.get_internal_sensor_curr_temp(g, &temp_f24_8);
1398
1399 gk20a_idle(g);
1400
1401 args->temp_f24_8 = (s32)temp_f24_8;
1402
1403 return err;
1404}
1405#endif
1406
1407static int nvgpu_gpu_set_therm_alert_limit(struct gk20a *g,
1408 struct nvgpu_gpu_set_therm_alert_limit_args *args)
1409{
1410 int err;
1411
1412 gk20a_dbg_fn("");
1413
1414 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1415 return -EINVAL;
1416
1417 if (!g->ops.therm.configure_therm_alert)
1418 return -EINVAL;
1419
1420 err = gk20a_busy(g);
1421 if (err)
1422 return err;
1423
1424 err = g->ops.therm.configure_therm_alert(g, args->temp_f24_8);
1425
1426 gk20a_idle(g);
1427
1428 return err;
1429}
1430
1431static int nvgpu_gpu_set_deterministic_ch_railgate(struct channel_gk20a *ch,
1432 u32 flags)
1433{
1434 int err = 0;
1435 bool allow;
1436 bool disallow;
1437
1438 allow = flags &
1439 NVGPU_GPU_SET_DETERMINISTIC_OPTS_FLAGS_ALLOW_RAILGATING;
1440
1441 disallow = flags &
1442 NVGPU_GPU_SET_DETERMINISTIC_OPTS_FLAGS_DISALLOW_RAILGATING;
1443
1444 /* Can't be both at the same time */
1445 if (allow && disallow)
1446 return -EINVAL;
1447
1448 /* Nothing to do */
1449 if (!allow && !disallow)
1450 return 0;
1451
1452 /*
1453 * Moving into explicit idle or back from it? A call that doesn't
1454 * change the status is a no-op.
1455 */
1456 if (!ch->deterministic_railgate_allowed &&
1457 allow) {
1458 gk20a_idle(ch->g);
1459 } else if (ch->deterministic_railgate_allowed &&
1460 !allow) {
1461 err = gk20a_busy(ch->g);
1462 if (err) {
1463 nvgpu_warn(ch->g,
1464 "cannot busy to restore deterministic ch");
1465 return err;
1466 }
1467 }
1468 ch->deterministic_railgate_allowed = allow;
1469
1470 return err;
1471}
1472
1473static int nvgpu_gpu_set_deterministic_ch(struct channel_gk20a *ch, u32 flags)
1474{
1475 if (!ch->deterministic)
1476 return -EINVAL;
1477
1478 return nvgpu_gpu_set_deterministic_ch_railgate(ch, flags);
1479}
1480
1481static int nvgpu_gpu_set_deterministic_opts(struct gk20a *g,
1482 struct nvgpu_gpu_set_deterministic_opts_args *args)
1483{
1484 int __user *user_channels;
1485 u32 i = 0;
1486 int err = 0;
1487
1488 gk20a_dbg_fn("");
1489
1490 user_channels = (int __user *)(uintptr_t)args->channels;
1491
1492 /* Upper limit; prevent holding deterministic_busy for long */
1493 if (args->num_channels > g->fifo.num_channels) {
1494 err = -EINVAL;
1495 goto out;
1496 }
1497
1498 /* Trivial sanity check first */
1499 if (!access_ok(VERIFY_READ, user_channels,
1500 args->num_channels * sizeof(int))) {
1501 err = -EFAULT;
1502 goto out;
1503 }
1504
1505 nvgpu_rwsem_down_read(&g->deterministic_busy);
1506
1507 /* note: we exit at the first failure */
1508 for (; i < args->num_channels; i++) {
1509 int ch_fd = 0;
1510 struct channel_gk20a *ch;
1511
1512 if (copy_from_user(&ch_fd, &user_channels[i], sizeof(int))) {
1513 /* User raced with above access_ok */
1514 err = -EFAULT;
1515 break;
1516 }
1517
1518 ch = gk20a_get_channel_from_file(ch_fd);
1519 if (!ch) {
1520 err = -EINVAL;
1521 break;
1522 }
1523
1524 err = nvgpu_gpu_set_deterministic_ch(ch, args->flags);
1525
1526 gk20a_channel_put(ch);
1527
1528 if (err)
1529 break;
1530 }
1531
1532 nvgpu_rwsem_up_read(&g->deterministic_busy);
1533
1534out:
1535 args->num_channels = i;
1536 return err;
1537}
1538
1539long gk20a_ctrl_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1540{
1541 struct gk20a_ctrl_priv *priv = filp->private_data;
1542 struct gk20a *g = priv->g;
1543 struct nvgpu_gpu_zcull_get_ctx_size_args *get_ctx_size_args;
1544 struct nvgpu_gpu_zcull_get_info_args *get_info_args;
1545 struct nvgpu_gpu_zbc_set_table_args *set_table_args;
1546 struct nvgpu_gpu_zbc_query_table_args *query_table_args;
1547 u8 buf[NVGPU_GPU_IOCTL_MAX_ARG_SIZE];
1548 struct gr_zcull_info *zcull_info;
1549 struct zbc_entry *zbc_val;
1550 struct zbc_query_params *zbc_tbl;
1551 int i, err = 0;
1552
1553 gk20a_dbg_fn("start %d", _IOC_NR(cmd));
1554
1555 if ((_IOC_TYPE(cmd) != NVGPU_GPU_IOCTL_MAGIC) ||
1556 (_IOC_NR(cmd) == 0) ||
1557 (_IOC_NR(cmd) > NVGPU_GPU_IOCTL_LAST) ||
1558 (_IOC_SIZE(cmd) > NVGPU_GPU_IOCTL_MAX_ARG_SIZE))
1559 return -EINVAL;
1560
1561 memset(buf, 0, sizeof(buf));
1562 if (_IOC_DIR(cmd) & _IOC_WRITE) {
1563 if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
1564 return -EFAULT;
1565 }
1566
1567 if (!g->gr.sw_ready) {
1568 err = gk20a_busy(g);
1569 if (err)
1570 return err;
1571
1572 gk20a_idle(g);
1573 }
1574
1575 switch (cmd) {
1576 case NVGPU_GPU_IOCTL_ZCULL_GET_CTX_SIZE:
1577 get_ctx_size_args = (struct nvgpu_gpu_zcull_get_ctx_size_args *)buf;
1578
1579 get_ctx_size_args->size = gr_gk20a_get_ctxsw_zcull_size(g, &g->gr);
1580
1581 break;
1582 case NVGPU_GPU_IOCTL_ZCULL_GET_INFO:
1583 get_info_args = (struct nvgpu_gpu_zcull_get_info_args *)buf;
1584
1585 memset(get_info_args, 0, sizeof(struct nvgpu_gpu_zcull_get_info_args));
1586
1587 zcull_info = nvgpu_kzalloc(g, sizeof(struct gr_zcull_info));
1588 if (zcull_info == NULL)
1589 return -ENOMEM;
1590
1591 err = g->ops.gr.get_zcull_info(g, &g->gr, zcull_info);
1592 if (err) {
1593 nvgpu_kfree(g, zcull_info);
1594 break;
1595 }
1596
1597 get_info_args->width_align_pixels = zcull_info->width_align_pixels;
1598 get_info_args->height_align_pixels = zcull_info->height_align_pixels;
1599 get_info_args->pixel_squares_by_aliquots = zcull_info->pixel_squares_by_aliquots;
1600 get_info_args->aliquot_total = zcull_info->aliquot_total;
1601 get_info_args->region_byte_multiplier = zcull_info->region_byte_multiplier;
1602 get_info_args->region_header_size = zcull_info->region_header_size;
1603 get_info_args->subregion_header_size = zcull_info->subregion_header_size;
1604 get_info_args->subregion_width_align_pixels = zcull_info->subregion_width_align_pixels;
1605 get_info_args->subregion_height_align_pixels = zcull_info->subregion_height_align_pixels;
1606 get_info_args->subregion_count = zcull_info->subregion_count;
1607
1608 nvgpu_kfree(g, zcull_info);
1609 break;
1610 case NVGPU_GPU_IOCTL_ZBC_SET_TABLE:
1611 set_table_args = (struct nvgpu_gpu_zbc_set_table_args *)buf;
1612
1613 zbc_val = nvgpu_kzalloc(g, sizeof(struct zbc_entry));
1614 if (zbc_val == NULL)
1615 return -ENOMEM;
1616
1617 zbc_val->format = set_table_args->format;
1618 zbc_val->type = set_table_args->type;
1619
1620 switch (zbc_val->type) {
1621 case GK20A_ZBC_TYPE_COLOR:
1622 for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) {
1623 zbc_val->color_ds[i] = set_table_args->color_ds[i];
1624 zbc_val->color_l2[i] = set_table_args->color_l2[i];
1625 }
1626 break;
1627 case GK20A_ZBC_TYPE_DEPTH:
1628 case T19X_ZBC:
1629 zbc_val->depth = set_table_args->depth;
1630 break;
1631 default:
1632 err = -EINVAL;
1633 }
1634
1635 if (!err) {
1636 err = gk20a_busy(g);
1637 if (!err) {
1638 err = g->ops.gr.zbc_set_table(g, &g->gr,
1639 zbc_val);
1640 gk20a_idle(g);
1641 }
1642 }
1643
1644 if (zbc_val)
1645 nvgpu_kfree(g, zbc_val);
1646 break;
1647 case NVGPU_GPU_IOCTL_ZBC_QUERY_TABLE:
1648 query_table_args = (struct nvgpu_gpu_zbc_query_table_args *)buf;
1649
1650 zbc_tbl = nvgpu_kzalloc(g, sizeof(struct zbc_query_params));
1651 if (zbc_tbl == NULL)
1652 return -ENOMEM;
1653
1654 zbc_tbl->type = query_table_args->type;
1655 zbc_tbl->index_size = query_table_args->index_size;
1656
1657 err = g->ops.gr.zbc_query_table(g, &g->gr, zbc_tbl);
1658
1659 if (!err) {
1660 switch (zbc_tbl->type) {
1661 case GK20A_ZBC_TYPE_COLOR:
1662 for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) {
1663 query_table_args->color_ds[i] = zbc_tbl->color_ds[i];
1664 query_table_args->color_l2[i] = zbc_tbl->color_l2[i];
1665 }
1666 break;
1667 case GK20A_ZBC_TYPE_DEPTH:
1668 case T19X_ZBC:
1669 query_table_args->depth = zbc_tbl->depth;
1670 break;
1671 case GK20A_ZBC_TYPE_INVALID:
1672 query_table_args->index_size = zbc_tbl->index_size;
1673 break;
1674 default:
1675 err = -EINVAL;
1676 }
1677 if (!err) {
1678 query_table_args->format = zbc_tbl->format;
1679 query_table_args->ref_cnt = zbc_tbl->ref_cnt;
1680 }
1681 }
1682
1683 if (zbc_tbl)
1684 nvgpu_kfree(g, zbc_tbl);
1685 break;
1686
1687 case NVGPU_GPU_IOCTL_GET_CHARACTERISTICS:
1688 err = gk20a_ctrl_ioctl_gpu_characteristics(
1689 g, (struct nvgpu_gpu_get_characteristics *)buf);
1690 break;
1691 case NVGPU_GPU_IOCTL_PREPARE_COMPRESSIBLE_READ:
1692 err = gk20a_ctrl_prepare_compressible_read(g,
1693 (struct nvgpu_gpu_prepare_compressible_read_args *)buf);
1694 break;
1695 case NVGPU_GPU_IOCTL_MARK_COMPRESSIBLE_WRITE:
1696 err = gk20a_ctrl_mark_compressible_write(g,
1697 (struct nvgpu_gpu_mark_compressible_write_args *)buf);
1698 break;
1699 case NVGPU_GPU_IOCTL_ALLOC_AS:
1700 err = gk20a_ctrl_alloc_as(g,
1701 (struct nvgpu_alloc_as_args *)buf);
1702 break;
1703 case NVGPU_GPU_IOCTL_OPEN_TSG:
1704 err = gk20a_ctrl_open_tsg(g,
1705 (struct nvgpu_gpu_open_tsg_args *)buf);
1706 break;
1707 case NVGPU_GPU_IOCTL_GET_TPC_MASKS:
1708 err = gk20a_ctrl_get_tpc_masks(g,
1709 (struct nvgpu_gpu_get_tpc_masks_args *)buf);
1710 break;
1711 case NVGPU_GPU_IOCTL_GET_FBP_L2_MASKS:
1712 err = gk20a_ctrl_get_fbp_l2_masks(g,
1713 (struct nvgpu_gpu_get_fbp_l2_masks_args *)buf);
1714 break;
1715 case NVGPU_GPU_IOCTL_OPEN_CHANNEL:
1716 /* this arg type here, but ..gpu_open_channel_args in nvgpu.h
1717 * for consistency - they are the same */
1718 err = gk20a_channel_open_ioctl(g,
1719 (struct nvgpu_channel_open_args *)buf);
1720 break;
1721 case NVGPU_GPU_IOCTL_FLUSH_L2:
1722 err = nvgpu_gpu_ioctl_l2_fb_ops(g,
1723 (struct nvgpu_gpu_l2_fb_args *)buf);
1724 break;
1725 case NVGPU_GPU_IOCTL_INVAL_ICACHE:
1726 err = gr_gk20a_elpg_protected_call(g,
1727 nvgpu_gpu_ioctl_inval_icache(g, (struct nvgpu_gpu_inval_icache_args *)buf));
1728 break;
1729
1730 case NVGPU_GPU_IOCTL_SET_MMUDEBUG_MODE:
1731 err = nvgpu_gpu_ioctl_set_mmu_debug_mode(g,
1732 (struct nvgpu_gpu_mmu_debug_mode_args *)buf);
1733 break;
1734
1735 case NVGPU_GPU_IOCTL_SET_SM_DEBUG_MODE:
1736 err = gr_gk20a_elpg_protected_call(g,
1737 nvgpu_gpu_ioctl_set_debug_mode(g, (struct nvgpu_gpu_sm_debug_mode_args *)buf));
1738 break;
1739
1740 case NVGPU_GPU_IOCTL_TRIGGER_SUSPEND:
1741 err = nvgpu_gpu_ioctl_trigger_suspend(g);
1742 break;
1743
1744 case NVGPU_GPU_IOCTL_WAIT_FOR_PAUSE:
1745 err = nvgpu_gpu_ioctl_wait_for_pause(g,
1746 (struct nvgpu_gpu_wait_pause_args *)buf);
1747 break;
1748
1749 case NVGPU_GPU_IOCTL_RESUME_FROM_PAUSE:
1750 err = nvgpu_gpu_ioctl_resume_from_pause(g);
1751 break;
1752
1753 case NVGPU_GPU_IOCTL_CLEAR_SM_ERRORS:
1754 err = nvgpu_gpu_ioctl_clear_sm_errors(g);
1755 break;
1756
1757 case NVGPU_GPU_IOCTL_GET_TPC_EXCEPTION_EN_STATUS:
1758 err = nvgpu_gpu_ioctl_has_any_exception(g,
1759 (struct nvgpu_gpu_tpc_exception_en_status_args *)buf);
1760 break;
1761
1762 case NVGPU_GPU_IOCTL_NUM_VSMS:
1763 err = gk20a_ctrl_get_num_vsms(g,
1764 (struct nvgpu_gpu_num_vsms *)buf);
1765 break;
1766 case NVGPU_GPU_IOCTL_VSMS_MAPPING:
1767 err = gk20a_ctrl_vsm_mapping(g,
1768 (struct nvgpu_gpu_vsms_mapping *)buf);
1769 break;
1770
1771 case NVGPU_GPU_IOCTL_GET_BUFFER_INFO:
1772 err = gk20a_ctrl_get_buffer_info(g,
1773 (struct nvgpu_gpu_get_buffer_info_args *)buf);
1774 break;
1775
1776 case NVGPU_GPU_IOCTL_GET_CPU_TIME_CORRELATION_INFO:
1777 err = nvgpu_gpu_get_cpu_time_correlation_info(g,
1778 (struct nvgpu_gpu_get_cpu_time_correlation_info_args *)buf);
1779 break;
1780
1781 case NVGPU_GPU_IOCTL_GET_GPU_TIME:
1782 err = nvgpu_gpu_get_gpu_time(g,
1783 (struct nvgpu_gpu_get_gpu_time_args *)buf);
1784 break;
1785
1786 case NVGPU_GPU_IOCTL_GET_ENGINE_INFO:
1787 err = nvgpu_gpu_get_engine_info(g,
1788 (struct nvgpu_gpu_get_engine_info_args *)buf);
1789 break;
1790
1791 case NVGPU_GPU_IOCTL_ALLOC_VIDMEM:
1792 err = nvgpu_gpu_alloc_vidmem(g,
1793 (struct nvgpu_gpu_alloc_vidmem_args *)buf);
1794 break;
1795
1796 case NVGPU_GPU_IOCTL_GET_MEMORY_STATE:
1797 err = nvgpu_gpu_get_memory_state(g,
1798 (struct nvgpu_gpu_get_memory_state_args *)buf);
1799 break;
1800
1801#ifdef CONFIG_ARCH_TEGRA_18x_SOC
1802 case NVGPU_GPU_IOCTL_CLK_GET_RANGE:
1803 err = nvgpu_gpu_clk_get_range(g, priv,
1804 (struct nvgpu_gpu_clk_range_args *)buf);
1805 break;
1806
1807 case NVGPU_GPU_IOCTL_CLK_GET_VF_POINTS:
1808 err = nvgpu_gpu_clk_get_vf_points(g, priv,
1809 (struct nvgpu_gpu_clk_vf_points_args *)buf);
1810 break;
1811
1812 case NVGPU_GPU_IOCTL_CLK_SET_INFO:
1813 err = nvgpu_gpu_clk_set_info(g, priv,
1814 (struct nvgpu_gpu_clk_set_info_args *)buf);
1815 break;
1816
1817 case NVGPU_GPU_IOCTL_CLK_GET_INFO:
1818 err = nvgpu_gpu_clk_get_info(g, priv,
1819 (struct nvgpu_gpu_clk_get_info_args *)buf);
1820 break;
1821
1822 case NVGPU_GPU_IOCTL_GET_EVENT_FD:
1823 err = nvgpu_gpu_get_event_fd(g, priv,
1824 (struct nvgpu_gpu_get_event_fd_args *)buf);
1825 break;
1826
1827 case NVGPU_GPU_IOCTL_GET_VOLTAGE:
1828 err = nvgpu_gpu_get_voltage(g,
1829 (struct nvgpu_gpu_get_voltage_args *)buf);
1830 break;
1831
1832 case NVGPU_GPU_IOCTL_GET_CURRENT:
1833 err = nvgpu_gpu_get_current(g,
1834 (struct nvgpu_gpu_get_current_args *)buf);
1835 break;
1836
1837 case NVGPU_GPU_IOCTL_GET_POWER:
1838 err = nvgpu_gpu_get_power(g,
1839 (struct nvgpu_gpu_get_power_args *)buf);
1840 break;
1841
1842 case NVGPU_GPU_IOCTL_GET_TEMPERATURE:
1843 err = nvgpu_gpu_get_temperature(g,
1844 (struct nvgpu_gpu_get_temperature_args *)buf);
1845 break;
1846#endif
1847
1848 case NVGPU_GPU_IOCTL_SET_THERM_ALERT_LIMIT:
1849 err = nvgpu_gpu_set_therm_alert_limit(g,
1850 (struct nvgpu_gpu_set_therm_alert_limit_args *)buf);
1851 break;
1852
1853 case NVGPU_GPU_IOCTL_SET_DETERMINISTIC_OPTS:
1854 err = nvgpu_gpu_set_deterministic_opts(g,
1855 (struct nvgpu_gpu_set_deterministic_opts_args *)buf);
1856 break;
1857
1858 default:
1859 gk20a_dbg_info("unrecognized gpu ioctl cmd: 0x%x", cmd);
1860 err = -ENOTTY;
1861 break;
1862 }
1863
1864 if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
1865 err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
1866
1867 return err;
1868}
generated by cgit v1.2.2 (git 2.25.0) at 2024-08-22 15:44:07 -0400