aboutsummaryrefslogtreecommitdiffstats
path: root/include/os/linux/ioctl_ctrl.c
diff options
context:
space:
mode:
Diffstat (limited to 'include/os/linux/ioctl_ctrl.c')
-rw-r--r--include/os/linux/ioctl_ctrl.c2144
1 files changed, 0 insertions, 2144 deletions
diff --git a/include/os/linux/ioctl_ctrl.c b/include/os/linux/ioctl_ctrl.c
deleted file mode 100644
index 841d345..0000000
--- a/include/os/linux/ioctl_ctrl.c
+++ /dev/null
@@ -1,2144 +0,0 @@
1/*
2 * Copyright (c) 2011-2021, NVIDIA Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#include <linux/uaccess.h>
18#include <linux/cdev.h>
19#include <linux/file.h>
20#include <linux/anon_inodes.h>
21#include <linux/fs.h>
22#include <linux/pm_runtime.h>
23#include <uapi/linux/nvgpu.h>
24
25#include <nvgpu/bitops.h>
26#include <nvgpu/kmem.h>
27#include <nvgpu/bug.h>
28#include <nvgpu/ptimer.h>
29#include <nvgpu/vidmem.h>
30#include <nvgpu/log.h>
31#include <nvgpu/enabled.h>
32#include <nvgpu/sizes.h>
33#include <nvgpu/list.h>
34#include <nvgpu/clk_arb.h>
35#include <nvgpu/gk20a.h>
36#include <nvgpu/channel.h>
37
38#include "ioctl_ctrl.h"
39#include "ioctl_dbg.h"
40#include "ioctl_as.h"
41#include "ioctl_tsg.h"
42#include "ioctl_channel.h"
43#include "gk20a/fence_gk20a.h"
44
45#include "platform_gk20a.h"
46#include "os_linux.h"
47#include "dmabuf.h"
48#include "channel.h"
49#include "dmabuf_vidmem.h"
50
51#define HZ_TO_MHZ(a) ((a > 0xF414F9CD7ULL) ? 0xffff : (a >> 32) ? \
52 (u32) ((a * 0x10C8ULL) >> 32) : (u16) ((u32) a/MHZ))
53#define MHZ_TO_HZ(a) ((u64)a * MHZ)
54
55struct gk20a_ctrl_priv {
56 struct device *dev;
57 struct gk20a *g;
58 struct nvgpu_clk_session *clk_session;
59
60 struct nvgpu_list_node list;
61 struct {
62 struct vm_area_struct *vma;
63 bool vma_mapped;
64 } usermode_vma;
65};
66
67static inline struct gk20a_ctrl_priv *
68gk20a_ctrl_priv_from_list(struct nvgpu_list_node *node)
69{
70 return (struct gk20a_ctrl_priv *)
71 ((uintptr_t)node - offsetof(struct gk20a_ctrl_priv, list));
72}
73
74static u32 gk20a_as_translate_as_alloc_flags(struct gk20a *g, u32 flags)
75{
76 u32 core_flags = 0;
77
78 if (flags & NVGPU_GPU_IOCTL_ALLOC_AS_FLAGS_USERSPACE_MANAGED)
79 core_flags |= NVGPU_AS_ALLOC_USERSPACE_MANAGED;
80
81 return core_flags;
82}
83
84int gk20a_ctrl_dev_open(struct inode *inode, struct file *filp)
85{
86 struct nvgpu_os_linux *l;
87 struct gk20a *g;
88 struct gk20a_ctrl_priv *priv;
89 int err = 0;
90
91 l = container_of(inode->i_cdev,
92 struct nvgpu_os_linux, ctrl.cdev);
93 g = gk20a_get(&l->g);
94 if (!g)
95 return -ENODEV;
96
97 nvgpu_log_fn(g, " ");
98
99 priv = nvgpu_kzalloc(g, sizeof(struct gk20a_ctrl_priv));
100 if (!priv) {
101 err = -ENOMEM;
102 goto free_ref;
103 }
104 filp->private_data = priv;
105 priv->dev = dev_from_gk20a(g);
106 /*
107 * We dont close the arbiter fd's after driver teardown to support
108 * GPU_LOST events, so we store g here, instead of dereferencing the
109 * dev structure on teardown
110 */
111 priv->g = g;
112
113 if (!g->sw_ready) {
114 err = gk20a_busy(g);
115 if (err)
116 goto free_ref;
117 gk20a_idle(g);
118 }
119
120 err = nvgpu_clk_arb_init_session(g, &priv->clk_session);
121free_ref:
122 if (err != 0) {
123 gk20a_put(g);
124 if (priv)
125 nvgpu_kfree(g, priv);
126 } else {
127 nvgpu_mutex_acquire(&l->ctrl.privs_lock);
128 nvgpu_list_add(&priv->list, &l->ctrl.privs);
129 nvgpu_mutex_release(&l->ctrl.privs_lock);
130 }
131
132 return err;
133}
134int gk20a_ctrl_dev_release(struct inode *inode, struct file *filp)
135{
136 struct gk20a_ctrl_priv *priv = filp->private_data;
137 struct gk20a *g = priv->g;
138 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
139
140 nvgpu_log_fn(g, " ");
141
142 nvgpu_mutex_acquire(&l->ctrl.privs_lock);
143 nvgpu_list_del(&priv->list);
144 nvgpu_mutex_release(&l->ctrl.privs_lock);
145
146 if (priv->clk_session)
147 nvgpu_clk_arb_release_session(g, priv->clk_session);
148
149 gk20a_put(g);
150 nvgpu_kfree(g, priv);
151
152 return 0;
153}
154
155struct nvgpu_flags_mapping {
156 u64 ioctl_flag;
157 int enabled_flag;
158};
159
160static struct nvgpu_flags_mapping flags_mapping[] = {
161 {NVGPU_GPU_FLAGS_CAN_RAILGATE,
162 NVGPU_CAN_RAILGATE},
163 {NVGPU_GPU_FLAGS_HAS_SYNCPOINTS,
164 NVGPU_HAS_SYNCPOINTS},
165 {NVGPU_GPU_FLAGS_SUPPORT_PARTIAL_MAPPINGS,
166 NVGPU_SUPPORT_PARTIAL_MAPPINGS},
167 {NVGPU_GPU_FLAGS_SUPPORT_SPARSE_ALLOCS,
168 NVGPU_SUPPORT_SPARSE_ALLOCS},
169 {NVGPU_GPU_FLAGS_SUPPORT_SYNC_FENCE_FDS,
170 NVGPU_SUPPORT_SYNC_FENCE_FDS},
171 {NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS,
172 NVGPU_SUPPORT_CYCLE_STATS},
173 {NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS_SNAPSHOT,
174 NVGPU_SUPPORT_CYCLE_STATS_SNAPSHOT},
175 {NVGPU_GPU_FLAGS_SUPPORT_USERSPACE_MANAGED_AS,
176 NVGPU_SUPPORT_USERSPACE_MANAGED_AS},
177 {NVGPU_GPU_FLAGS_SUPPORT_TSG,
178 NVGPU_SUPPORT_TSG},
179 {NVGPU_GPU_FLAGS_SUPPORT_CLOCK_CONTROLS,
180 NVGPU_SUPPORT_CLOCK_CONTROLS},
181 {NVGPU_GPU_FLAGS_SUPPORT_GET_VOLTAGE,
182 NVGPU_SUPPORT_GET_VOLTAGE},
183 {NVGPU_GPU_FLAGS_SUPPORT_GET_CURRENT,
184 NVGPU_SUPPORT_GET_CURRENT},
185 {NVGPU_GPU_FLAGS_SUPPORT_GET_POWER,
186 NVGPU_SUPPORT_GET_POWER},
187 {NVGPU_GPU_FLAGS_SUPPORT_GET_TEMPERATURE,
188 NVGPU_SUPPORT_GET_TEMPERATURE},
189 {NVGPU_GPU_FLAGS_SUPPORT_SET_THERM_ALERT_LIMIT,
190 NVGPU_SUPPORT_SET_THERM_ALERT_LIMIT},
191 {NVGPU_GPU_FLAGS_SUPPORT_DEVICE_EVENTS,
192 NVGPU_SUPPORT_DEVICE_EVENTS},
193 {NVGPU_GPU_FLAGS_SUPPORT_FECS_CTXSW_TRACE,
194 NVGPU_SUPPORT_FECS_CTXSW_TRACE},
195 {NVGPU_GPU_FLAGS_SUPPORT_DETERMINISTIC_SUBMIT_NO_JOBTRACKING,
196 NVGPU_SUPPORT_DETERMINISTIC_SUBMIT_NO_JOBTRACKING},
197 {NVGPU_GPU_FLAGS_SUPPORT_DETERMINISTIC_SUBMIT_FULL,
198 NVGPU_SUPPORT_DETERMINISTIC_SUBMIT_FULL},
199 {NVGPU_GPU_FLAGS_SUPPORT_DETERMINISTIC_OPTS,
200 NVGPU_SUPPORT_DETERMINISTIC_OPTS},
201 {NVGPU_GPU_FLAGS_SUPPORT_SYNCPOINT_ADDRESS,
202 NVGPU_SUPPORT_SYNCPOINT_ADDRESS},
203 {NVGPU_GPU_FLAGS_SUPPORT_USER_SYNCPOINT,
204 NVGPU_SUPPORT_USER_SYNCPOINT},
205 {NVGPU_GPU_FLAGS_SUPPORT_USERMODE_SUBMIT,
206 NVGPU_SUPPORT_USERMODE_SUBMIT},
207 {NVGPU_GPU_FLAGS_SUPPORT_IO_COHERENCE,
208 NVGPU_SUPPORT_IO_COHERENCE},
209 {NVGPU_GPU_FLAGS_SUPPORT_RESCHEDULE_RUNLIST,
210 NVGPU_SUPPORT_RESCHEDULE_RUNLIST},
211 {NVGPU_GPU_FLAGS_SUPPORT_MAP_DIRECT_KIND_CTRL,
212 NVGPU_SUPPORT_MAP_DIRECT_KIND_CTRL},
213 {NVGPU_GPU_FLAGS_ECC_ENABLED_SM_LRF,
214 NVGPU_ECC_ENABLED_SM_LRF},
215 {NVGPU_GPU_FLAGS_ECC_ENABLED_SM_SHM,
216 NVGPU_ECC_ENABLED_SM_SHM},
217 {NVGPU_GPU_FLAGS_ECC_ENABLED_TEX,
218 NVGPU_ECC_ENABLED_TEX},
219 {NVGPU_GPU_FLAGS_ECC_ENABLED_LTC,
220 NVGPU_ECC_ENABLED_LTC},
221 {NVGPU_GPU_FLAGS_SUPPORT_TSG_SUBCONTEXTS,
222 NVGPU_SUPPORT_TSG_SUBCONTEXTS},
223 {NVGPU_GPU_FLAGS_SUPPORT_SCG,
224 NVGPU_SUPPORT_SCG},
225 {NVGPU_GPU_FLAGS_SUPPORT_VPR,
226 NVGPU_SUPPORT_VPR},
227 {NVGPU_GPU_FLAGS_SUPPORT_SET_CTX_MMU_DEBUG_MODE,
228 NVGPU_SUPPORT_SET_CTX_MMU_DEBUG_MODE},
229};
230
231static u64 nvgpu_ctrl_ioctl_gpu_characteristics_flags(struct gk20a *g)
232{
233 unsigned int i;
234 u64 ioctl_flags = 0;
235
236 for (i = 0; i < sizeof(flags_mapping)/sizeof(*flags_mapping); i++) {
237 if (nvgpu_is_enabled(g, flags_mapping[i].enabled_flag))
238 ioctl_flags |= flags_mapping[i].ioctl_flag;
239 }
240
241 if (!capable(CAP_SYS_NICE)) {
242 ioctl_flags &= ~NVGPU_GPU_FLAGS_SUPPORT_RESCHEDULE_RUNLIST;
243 }
244
245 return ioctl_flags;
246}
247
248static void nvgpu_set_preemption_mode_flags(struct gk20a *g,
249 struct nvgpu_gpu_characteristics *gpu)
250{
251 struct nvgpu_preemption_modes_rec preemption_mode_rec;
252
253 g->ops.gr.get_preemption_mode_flags(g, &preemption_mode_rec);
254
255 gpu->graphics_preemption_mode_flags =
256 nvgpu_get_ioctl_graphics_preempt_mode_flags(
257 preemption_mode_rec.graphics_preemption_mode_flags);
258 gpu->compute_preemption_mode_flags =
259 nvgpu_get_ioctl_compute_preempt_mode_flags(
260 preemption_mode_rec.compute_preemption_mode_flags);
261
262 gpu->default_graphics_preempt_mode =
263 nvgpu_get_ioctl_graphics_preempt_mode(
264 preemption_mode_rec.default_graphics_preempt_mode);
265 gpu->default_compute_preempt_mode =
266 nvgpu_get_ioctl_compute_preempt_mode(
267 preemption_mode_rec.default_compute_preempt_mode);
268}
269
270static long
271gk20a_ctrl_ioctl_gpu_characteristics(
272 struct gk20a *g,
273 struct nvgpu_gpu_get_characteristics *request)
274{
275 struct nvgpu_gpu_characteristics gpu;
276 long err = 0;
277
278 if (gk20a_busy(g)) {
279 nvgpu_err(g, "failed to power on gpu");
280 return -EINVAL;
281 }
282
283 memset(&gpu, 0, sizeof(gpu));
284
285 gpu.L2_cache_size = g->ops.ltc.determine_L2_size_bytes(g);
286 gpu.on_board_video_memory_size = 0; /* integrated GPU */
287
288 gpu.num_gpc = g->gr.gpc_count;
289 gpu.max_gpc_count = g->gr.max_gpc_count;
290
291 gpu.num_tpc_per_gpc = g->gr.max_tpc_per_gpc_count;
292
293 gpu.bus_type = NVGPU_GPU_BUS_TYPE_AXI; /* always AXI for now */
294
295 gpu.compression_page_size = g->ops.fb.compression_page_size(g);
296
297 if (g->ops.gr.get_gpc_mask) {
298 gpu.gpc_mask = g->ops.gr.get_gpc_mask(g);
299 } else {
300 gpu.gpc_mask = BIT32(g->gr.gpc_count) - 1;
301 }
302
303 gpu.flags = nvgpu_ctrl_ioctl_gpu_characteristics_flags(g);
304
305 gpu.arch = g->params.gpu_arch;
306 gpu.impl = g->params.gpu_impl;
307 gpu.rev = g->params.gpu_rev;
308 gpu.reg_ops_limit = NVGPU_IOCTL_DBG_REG_OPS_LIMIT;
309 gpu.map_buffer_batch_limit = nvgpu_is_enabled(g, NVGPU_SUPPORT_MAP_BUFFER_BATCH) ?
310 NVGPU_IOCTL_AS_MAP_BUFFER_BATCH_LIMIT : 0;
311 gpu.twod_class = g->ops.get_litter_value(g, GPU_LIT_TWOD_CLASS);
312 gpu.threed_class = g->ops.get_litter_value(g, GPU_LIT_THREED_CLASS);
313 gpu.compute_class = g->ops.get_litter_value(g, GPU_LIT_COMPUTE_CLASS);
314 gpu.gpfifo_class = g->ops.get_litter_value(g, GPU_LIT_GPFIFO_CLASS);
315 gpu.inline_to_memory_class =
316 g->ops.get_litter_value(g, GPU_LIT_I2M_CLASS);
317 gpu.dma_copy_class =
318 g->ops.get_litter_value(g, GPU_LIT_DMA_COPY_CLASS);
319
320 gpu.vbios_version = g->bios.vbios_version;
321 gpu.vbios_oem_version = g->bios.vbios_oem_version;
322
323 gpu.big_page_size = nvgpu_mm_get_default_big_page_size(g);
324 gpu.pde_coverage_bit_count =
325 g->ops.mm.get_mmu_levels(g, gpu.big_page_size)[0].lo_bit[0];
326 gpu.available_big_page_sizes = nvgpu_mm_get_available_big_page_sizes(g);
327
328 gpu.sm_arch_sm_version = g->params.sm_arch_sm_version;
329 gpu.sm_arch_spa_version = g->params.sm_arch_spa_version;
330 gpu.sm_arch_warp_count = g->params.sm_arch_warp_count;
331
332 gpu.max_css_buffer_size = g->gr.max_css_buffer_size;
333
334 gpu.gpu_ioctl_nr_last = NVGPU_GPU_IOCTL_LAST;
335 gpu.tsg_ioctl_nr_last = NVGPU_TSG_IOCTL_LAST;
336 gpu.dbg_gpu_ioctl_nr_last = NVGPU_DBG_GPU_IOCTL_LAST;
337 gpu.ioctl_channel_nr_last = NVGPU_IOCTL_CHANNEL_LAST;
338 gpu.as_ioctl_nr_last = NVGPU_AS_IOCTL_LAST;
339 gpu.event_ioctl_nr_last = NVGPU_EVENT_IOCTL_LAST;
340 gpu.gpu_va_bit_count = 40;
341
342 strlcpy(gpu.chipname, g->name, sizeof(gpu.chipname));
343 gpu.max_fbps_count = g->ops.gr.get_max_fbps_count(g);
344 gpu.fbp_en_mask = g->ops.gr.get_fbp_en_mask(g);
345 gpu.max_ltc_per_fbp = g->ops.gr.get_max_ltc_per_fbp(g);
346 gpu.max_lts_per_ltc = g->ops.gr.get_max_lts_per_ltc(g);
347 gpu.gr_compbit_store_base_hw = g->gr.compbit_store.base_hw;
348 gpu.gr_gobs_per_comptagline_per_slice =
349 g->gr.gobs_per_comptagline_per_slice;
350 gpu.num_ltc = g->ltc_count;
351 gpu.lts_per_ltc = g->gr.slices_per_ltc;
352 gpu.cbc_cache_line_size = g->gr.cacheline_size;
353 gpu.cbc_comptags_per_line = g->gr.comptags_per_cacheline;
354
355 if (g->ops.clk.get_maxrate)
356 gpu.max_freq = g->ops.clk.get_maxrate(g, CTRL_CLK_DOMAIN_GPCCLK);
357
358 gpu.local_video_memory_size = g->mm.vidmem.size;
359
360 gpu.pci_vendor_id = g->pci_vendor_id;
361 gpu.pci_device_id = g->pci_device_id;
362 gpu.pci_subsystem_vendor_id = g->pci_subsystem_vendor_id;
363 gpu.pci_subsystem_device_id = g->pci_subsystem_device_id;
364 gpu.pci_class = g->pci_class;
365 gpu.pci_revision = g->pci_revision;
366
367 nvgpu_set_preemption_mode_flags(g, &gpu);
368
369 if (request->gpu_characteristics_buf_size > 0) {
370 size_t write_size = sizeof(gpu);
371
372 nvgpu_speculation_barrier();
373 if (write_size > request->gpu_characteristics_buf_size)
374 write_size = request->gpu_characteristics_buf_size;
375
376 err = copy_to_user((void __user *)(uintptr_t)
377 request->gpu_characteristics_buf_addr,
378 &gpu, write_size);
379 }
380
381 if (err == 0)
382 request->gpu_characteristics_buf_size = sizeof(gpu);
383
384 gk20a_idle(g);
385
386 return err;
387}
388
389static int gk20a_ctrl_prepare_compressible_read(
390 struct gk20a *g,
391 struct nvgpu_gpu_prepare_compressible_read_args *args)
392{
393 int ret = -ENOSYS;
394
395#ifdef CONFIG_NVGPU_SUPPORT_CDE
396 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
397 struct nvgpu_channel_fence fence;
398 struct gk20a_fence *fence_out = NULL;
399 int submit_flags = nvgpu_submit_gpfifo_user_flags_to_common_flags(
400 args->submit_flags);
401 int fd = -1;
402
403 fence.id = args->fence.syncpt_id;
404 fence.value = args->fence.syncpt_value;
405
406 /* Try and allocate an fd here*/
407 if ((submit_flags & NVGPU_SUBMIT_FLAGS_FENCE_GET)
408 && (submit_flags & NVGPU_SUBMIT_FLAGS_SYNC_FENCE)) {
409 fd = get_unused_fd_flags(O_RDWR);
410 if (fd < 0)
411 return fd;
412 }
413
414 ret = gk20a_prepare_compressible_read(l, args->handle,
415 args->request_compbits, args->offset,
416 args->compbits_hoffset, args->compbits_voffset,
417 args->scatterbuffer_offset,
418 args->width, args->height, args->block_height_log2,
419 submit_flags, &fence, &args->valid_compbits,
420 &args->zbc_color, &fence_out);
421
422 if (ret) {
423 if (fd != -1)
424 put_unused_fd(fd);
425 return ret;
426 }
427
428 /* Convert fence_out to something we can pass back to user space. */
429 if (submit_flags & NVGPU_SUBMIT_FLAGS_FENCE_GET) {
430 if (submit_flags & NVGPU_SUBMIT_FLAGS_SYNC_FENCE) {
431 if (fence_out) {
432 ret = gk20a_fence_install_fd(fence_out, fd);
433 if (ret)
434 put_unused_fd(fd);
435 else
436 args->fence.fd = fd;
437 } else {
438 args->fence.fd = -1;
439 put_unused_fd(fd);
440 }
441 } else {
442 if (fence_out) {
443 args->fence.syncpt_id = fence_out->syncpt_id;
444 args->fence.syncpt_value =
445 fence_out->syncpt_value;
446 } else {
447 args->fence.syncpt_id = -1;
448 args->fence.syncpt_value = 0;
449 }
450 }
451 }
452 gk20a_fence_put(fence_out);
453#endif
454
455 return ret;
456}
457
458static int gk20a_ctrl_mark_compressible_write(
459 struct gk20a *g,
460 struct nvgpu_gpu_mark_compressible_write_args *args)
461{
462 int ret = -ENOSYS;
463
464#ifdef CONFIG_NVGPU_SUPPORT_CDE
465 ret = gk20a_mark_compressible_write(g, args->handle,
466 args->valid_compbits, args->offset, args->zbc_color);
467#endif
468
469 return ret;
470}
471
472static int gk20a_ctrl_alloc_as(
473 struct gk20a *g,
474 struct nvgpu_alloc_as_args *args)
475{
476 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
477 struct gk20a_as_share *as_share;
478 int err;
479 int fd;
480 struct file *file;
481 char name[64];
482
483 err = get_unused_fd_flags(O_RDWR);
484 if (err < 0)
485 return err;
486 fd = err;
487
488 snprintf(name, sizeof(name), "nvhost-%s-fd%d", g->name, fd);
489
490 err = gk20a_as_alloc_share(g, args->big_page_size,
491 gk20a_as_translate_as_alloc_flags(g,
492 args->flags),
493 &as_share);
494 if (err)
495 goto clean_up;
496
497 file = anon_inode_getfile(name, l->as_dev.cdev.ops, as_share, O_RDWR);
498 if (IS_ERR(file)) {
499 err = PTR_ERR(file);
500 goto clean_up_as;
501 }
502
503 fd_install(fd, file);
504
505 args->as_fd = fd;
506 return 0;
507
508clean_up_as:
509 gk20a_as_release_share(as_share);
510clean_up:
511 put_unused_fd(fd);
512 return err;
513}
514
515static int gk20a_ctrl_open_tsg(struct gk20a *g,
516 struct nvgpu_gpu_open_tsg_args *args)
517{
518 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
519 int err;
520 int fd;
521 struct file *file;
522 char name[64];
523
524 err = get_unused_fd_flags(O_RDWR);
525 if (err < 0)
526 return err;
527 fd = err;
528
529 snprintf(name, sizeof(name), "nvgpu-%s-tsg%d", g->name, fd);
530
531 file = anon_inode_getfile(name, l->tsg.cdev.ops, NULL, O_RDWR);
532 if (IS_ERR(file)) {
533 err = PTR_ERR(file);
534 goto clean_up;
535 }
536
537 err = nvgpu_ioctl_tsg_open(g, file);
538 if (err)
539 goto clean_up_file;
540
541 fd_install(fd, file);
542 args->tsg_fd = fd;
543 return 0;
544
545clean_up_file:
546 fput(file);
547clean_up:
548 put_unused_fd(fd);
549 return err;
550}
551
552static int gk20a_ctrl_get_tpc_masks(struct gk20a *g,
553 struct nvgpu_gpu_get_tpc_masks_args *args)
554{
555 struct gr_gk20a *gr = &g->gr;
556 int err = 0;
557 const u32 gpc_tpc_mask_size = sizeof(u32) * gr->max_gpc_count;
558
559 if (args->mask_buf_size > 0) {
560 size_t write_size = gpc_tpc_mask_size;
561
562 nvgpu_speculation_barrier();
563 if (write_size > args->mask_buf_size)
564 write_size = args->mask_buf_size;
565
566 err = copy_to_user((void __user *)(uintptr_t)
567 args->mask_buf_addr,
568 gr->gpc_tpc_mask, write_size);
569 }
570
571 if (err == 0)
572 args->mask_buf_size = gpc_tpc_mask_size;
573
574 return err;
575}
576
577static int gk20a_ctrl_get_fbp_l2_masks(
578 struct gk20a *g, struct nvgpu_gpu_get_fbp_l2_masks_args *args)
579{
580 struct gr_gk20a *gr = &g->gr;
581 int err = 0;
582 const u32 fbp_l2_mask_size = sizeof(u32) * gr->max_fbps_count;
583
584 if (args->mask_buf_size > 0) {
585 size_t write_size = fbp_l2_mask_size;
586
587 nvgpu_speculation_barrier();
588 if (write_size > args->mask_buf_size)
589 write_size = args->mask_buf_size;
590
591 err = copy_to_user((void __user *)(uintptr_t)
592 args->mask_buf_addr,
593 gr->fbp_rop_l2_en_mask, write_size);
594 }
595
596 if (err == 0)
597 args->mask_buf_size = fbp_l2_mask_size;
598
599 return err;
600}
601
602static int nvgpu_gpu_ioctl_l2_fb_ops(struct gk20a *g,
603 struct nvgpu_gpu_l2_fb_args *args)
604{
605 int ret;
606 bool always_poweron;
607
608 if ((!args->l2_flush && !args->fb_flush) ||
609 (!args->l2_flush && args->l2_invalidate))
610 return -EINVAL;
611
612 /* Handle this case for joint rails or DGPU */
613 always_poweron = (!nvgpu_is_enabled(g, NVGPU_CAN_RAILGATE) ||
614 !pm_runtime_enabled(dev_from_gk20a(g)));
615
616 /* In case of not always power_on, exit if g->power_on is false */
617 if (!always_poweron && !gk20a_check_poweron(g)) {
618 return 0;
619 }
620
621 /* There is a small window between a call to gk20a_idle() has occured
622 * and railgate being actually triggered(setting g->power_on = false),
623 * when l2_flush can race with railgate. Its better to take a busy_lock
624 * to prevent the gk20a_idle() from proceeding. There is a very small
625 * chance that gk20a_idle() might begin before gk20a_busy(). Having
626 * a locked access to g->power_on further reduces the probability of
627 * gk20a_idle() being triggered before gk20a_busy()
628 */
629 ret = gk20a_busy(g);
630
631 if (ret != 0) {
632 nvgpu_err(g, "failed to take power ref");
633 return ret;
634 }
635
636 if (args->l2_flush)
637 g->ops.mm.l2_flush(g, args->l2_invalidate ? true : false);
638
639 if (args->fb_flush)
640 g->ops.mm.fb_flush(g);
641
642 gk20a_idle(g);
643
644 return 0;
645}
646
647static int nvgpu_gpu_ioctl_set_mmu_debug_mode(
648 struct gk20a *g,
649 struct nvgpu_gpu_mmu_debug_mode_args *args)
650{
651 if (gk20a_busy(g)) {
652 nvgpu_err(g, "failed to power on gpu");
653 return -EINVAL;
654 }
655
656 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
657 g->ops.fb.set_debug_mode(g, args->state == 1);
658 nvgpu_mutex_release(&g->dbg_sessions_lock);
659
660 gk20a_idle(g);
661 return 0;
662}
663
664static int nvgpu_gpu_ioctl_set_debug_mode(
665 struct gk20a *g,
666 struct nvgpu_gpu_sm_debug_mode_args *args)
667{
668 struct channel_gk20a *ch;
669 int err;
670
671 ch = gk20a_get_channel_from_file(args->channel_fd);
672 if (!ch)
673 return -EINVAL;
674
675 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
676 if (g->ops.gr.set_sm_debug_mode)
677 err = g->ops.gr.set_sm_debug_mode(g, ch,
678 args->sms, !!args->enable);
679 else
680 err = -ENOSYS;
681 nvgpu_mutex_release(&g->dbg_sessions_lock);
682
683 gk20a_channel_put(ch);
684 return err;
685}
686
687static int nvgpu_gpu_ioctl_trigger_suspend(struct gk20a *g)
688{
689 int err;
690
691 err = gk20a_busy(g);
692 if (err)
693 return err;
694
695 if (g->ops.gr.trigger_suspend) {
696 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
697 err = gr_gk20a_elpg_protected_call(g,
698 g->ops.gr.trigger_suspend(g));
699 nvgpu_mutex_release(&g->dbg_sessions_lock);
700 } else
701 err = -EINVAL;
702
703 gk20a_idle(g);
704
705 return err;
706}
707
708static int nvgpu_gpu_ioctl_wait_for_pause(struct gk20a *g,
709 struct nvgpu_gpu_wait_pause_args *args)
710{
711 int err;
712 struct warpstate *ioctl_w_state;
713 struct nvgpu_warpstate *w_state = NULL;
714 u32 sm_count, ioctl_size, size, sm_id;
715
716 sm_count = g->gr.gpc_count * g->gr.tpc_count;
717
718 ioctl_size = sm_count * sizeof(struct warpstate);
719 ioctl_w_state = nvgpu_kzalloc(g, ioctl_size);
720 if (!ioctl_w_state)
721 return -ENOMEM;
722
723 size = sm_count * sizeof(struct nvgpu_warpstate);
724 w_state = nvgpu_kzalloc(g, size);
725 if (!w_state) {
726 err = -ENOMEM;
727 goto out_free;
728 }
729
730 err = gk20a_busy(g);
731 if (err)
732 goto out_free;
733
734 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
735 if (g->ops.gr.wait_for_pause) {
736 (void)gr_gk20a_elpg_protected_call(g,
737 g->ops.gr.wait_for_pause(g, w_state));
738 } else {
739 err = -EINVAL;
740 goto out_idle;
741 }
742
743 for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) {
744 ioctl_w_state[sm_id].valid_warps[0] =
745 w_state[sm_id].valid_warps[0];
746 ioctl_w_state[sm_id].valid_warps[1] =
747 w_state[sm_id].valid_warps[1];
748 ioctl_w_state[sm_id].trapped_warps[0] =
749 w_state[sm_id].trapped_warps[0];
750 ioctl_w_state[sm_id].trapped_warps[1] =
751 w_state[sm_id].trapped_warps[1];
752 ioctl_w_state[sm_id].paused_warps[0] =
753 w_state[sm_id].paused_warps[0];
754 ioctl_w_state[sm_id].paused_warps[1] =
755 w_state[sm_id].paused_warps[1];
756 }
757 /* Copy to user space - pointed by "args->pwarpstate" */
758 if (copy_to_user((void __user *)(uintptr_t)args->pwarpstate,
759 w_state, ioctl_size)) {
760 nvgpu_log_fn(g, "copy_to_user failed!");
761 err = -EFAULT;
762 }
763
764out_idle:
765 nvgpu_mutex_release(&g->dbg_sessions_lock);
766
767 gk20a_idle(g);
768
769out_free:
770 nvgpu_kfree(g, w_state);
771 nvgpu_kfree(g, ioctl_w_state);
772
773 return err;
774}
775
776static int nvgpu_gpu_ioctl_resume_from_pause(struct gk20a *g)
777{
778 int err;
779
780 err = gk20a_busy(g);
781 if (err)
782 return err;
783
784 if (g->ops.gr.resume_from_pause) {
785 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
786 err = gr_gk20a_elpg_protected_call(g,
787 g->ops.gr.resume_from_pause(g));
788 nvgpu_mutex_release(&g->dbg_sessions_lock);
789 } else
790 err = -EINVAL;
791
792 gk20a_idle(g);
793
794 return err;
795}
796
797static int nvgpu_gpu_ioctl_clear_sm_errors(struct gk20a *g)
798{
799 int err;
800
801 err = gk20a_busy(g);
802 if (err)
803 return err;
804
805 if (g->ops.gr.clear_sm_errors) {
806 err = gr_gk20a_elpg_protected_call(g,
807 g->ops.gr.clear_sm_errors(g));
808 } else
809 err = -EINVAL;
810
811 gk20a_idle(g);
812
813 return err;
814}
815
816static int nvgpu_gpu_ioctl_has_any_exception(
817 struct gk20a *g,
818 struct nvgpu_gpu_tpc_exception_en_status_args *args)
819{
820 u32 tpc_exception_en;
821
822 if (g->ops.gr.tpc_enabled_exceptions) {
823 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
824 tpc_exception_en = g->ops.gr.tpc_enabled_exceptions(g);
825 nvgpu_mutex_release(&g->dbg_sessions_lock);
826 } else
827 return -EINVAL;
828
829 args->tpc_exception_en_sm_mask = tpc_exception_en;
830
831 return 0;
832}
833
834static int gk20a_ctrl_get_num_vsms(struct gk20a *g,
835 struct nvgpu_gpu_num_vsms *args)
836{
837 struct gr_gk20a *gr = &g->gr;
838 args->num_vsms = gr->no_of_sm;
839 return 0;
840}
841
842static int gk20a_ctrl_vsm_mapping(struct gk20a *g,
843 struct nvgpu_gpu_vsms_mapping *args)
844{
845 int err = 0;
846 struct gr_gk20a *gr = &g->gr;
847 size_t write_size = gr->no_of_sm *
848 sizeof(struct nvgpu_gpu_vsms_mapping_entry);
849 struct nvgpu_gpu_vsms_mapping_entry *vsms_buf;
850 u32 i;
851
852 vsms_buf = nvgpu_kzalloc(g, write_size);
853 if (vsms_buf == NULL)
854 return -ENOMEM;
855
856 for (i = 0; i < gr->no_of_sm; i++) {
857 vsms_buf[i].gpc_index = gr->sm_to_cluster[i].gpc_index;
858 if (g->ops.gr.get_nonpes_aware_tpc)
859 vsms_buf[i].tpc_index =
860 g->ops.gr.get_nonpes_aware_tpc(g,
861 gr->sm_to_cluster[i].gpc_index,
862 gr->sm_to_cluster[i].tpc_index);
863 else
864 vsms_buf[i].tpc_index =
865 gr->sm_to_cluster[i].tpc_index;
866 }
867
868 err = copy_to_user((void __user *)(uintptr_t)
869 args->vsms_map_buf_addr,
870 vsms_buf, write_size);
871 nvgpu_kfree(g, vsms_buf);
872
873 return err;
874}
875
876static int nvgpu_gpu_get_cpu_time_correlation_info(
877 struct gk20a *g,
878 struct nvgpu_gpu_get_cpu_time_correlation_info_args *args)
879{
880 struct nvgpu_cpu_time_correlation_sample *samples;
881 int err;
882 u32 i;
883
884 if (args->count > NVGPU_GPU_GET_CPU_TIME_CORRELATION_INFO_MAX_COUNT ||
885 args->source_id != NVGPU_GPU_GET_CPU_TIME_CORRELATION_INFO_SRC_ID_TSC)
886 return -EINVAL;
887
888 samples = nvgpu_kzalloc(g, args->count *
889 sizeof(struct nvgpu_cpu_time_correlation_sample));
890 if (!samples) {
891 return -ENOMEM;
892 }
893
894 err = g->ops.ptimer.get_timestamps_zipper(g,
895 args->source_id, args->count, samples);
896 if (!err) {
897 for (i = 0; i < args->count; i++) {
898 args->samples[i].cpu_timestamp = samples[i].cpu_timestamp;
899 args->samples[i].gpu_timestamp = samples[i].gpu_timestamp;
900 }
901 }
902
903 nvgpu_kfree(g, samples);
904
905 return err;
906}
907
908static int nvgpu_gpu_get_gpu_time(
909 struct gk20a *g,
910 struct nvgpu_gpu_get_gpu_time_args *args)
911{
912 u64 time;
913 int err;
914
915 err = gk20a_busy(g);
916 if (err)
917 return err;
918
919 err = g->ops.ptimer.read_ptimer(g, &time);
920 if (!err)
921 args->gpu_timestamp = time;
922
923 gk20a_idle(g);
924 return err;
925}
926
927static int nvgpu_gpu_get_engine_info(
928 struct gk20a *g,
929 struct nvgpu_gpu_get_engine_info_args *args)
930{
931 int err = 0;
932 u32 engine_enum = ENGINE_INVAL_GK20A;
933 u32 report_index = 0;
934 u32 engine_id_idx;
935 const u32 max_buffer_engines = args->engine_info_buf_size /
936 sizeof(struct nvgpu_gpu_get_engine_info_item);
937 struct nvgpu_gpu_get_engine_info_item __user *dst_item_list =
938 (void __user *)(uintptr_t)args->engine_info_buf_addr;
939
940 for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines;
941 ++engine_id_idx) {
942 u32 active_engine_id = g->fifo.active_engines_list[engine_id_idx];
943 const struct fifo_engine_info_gk20a *src_info =
944 &g->fifo.engine_info[active_engine_id];
945 struct nvgpu_gpu_get_engine_info_item dst_info;
946
947 memset(&dst_info, 0, sizeof(dst_info));
948
949 engine_enum = src_info->engine_enum;
950
951 switch (engine_enum) {
952 case ENGINE_GR_GK20A:
953 dst_info.engine_id = NVGPU_GPU_ENGINE_ID_GR;
954 break;
955
956 case ENGINE_GRCE_GK20A:
957 dst_info.engine_id = NVGPU_GPU_ENGINE_ID_GR_COPY;
958 break;
959
960 case ENGINE_ASYNC_CE_GK20A:
961 dst_info.engine_id = NVGPU_GPU_ENGINE_ID_ASYNC_COPY;
962 break;
963
964 default:
965 nvgpu_err(g, "Unmapped engine enum %u",
966 engine_enum);
967 continue;
968 }
969
970 dst_info.engine_instance = src_info->inst_id;
971 dst_info.runlist_id = src_info->runlist_id;
972
973 if (report_index < max_buffer_engines) {
974 err = copy_to_user(&dst_item_list[report_index],
975 &dst_info, sizeof(dst_info));
976 if (err)
977 goto clean_up;
978 }
979
980 ++report_index;
981 }
982
983 args->engine_info_buf_size =
984 report_index * sizeof(struct nvgpu_gpu_get_engine_info_item);
985
986clean_up:
987 return err;
988}
989
990static int nvgpu_gpu_alloc_vidmem(struct gk20a *g,
991 struct nvgpu_gpu_alloc_vidmem_args *args)
992{
993 u32 align = args->in.alignment ? args->in.alignment : SZ_4K;
994 int fd;
995
996 nvgpu_log_fn(g, " ");
997
998 /* not yet supported */
999 if (WARN_ON(args->in.flags & NVGPU_GPU_ALLOC_VIDMEM_FLAG_CPU_MASK))
1000 return -EINVAL;
1001
1002 /* not yet supported */
1003 if (WARN_ON(args->in.flags & NVGPU_GPU_ALLOC_VIDMEM_FLAG_VPR))
1004 return -EINVAL;
1005
1006 if (args->in.size & (SZ_4K - 1))
1007 return -EINVAL;
1008
1009 if (!args->in.size)
1010 return -EINVAL;
1011
1012 if (align & (align - 1))
1013 return -EINVAL;
1014
1015 if (align > roundup_pow_of_two(args->in.size)) {
1016 /* log this special case, buddy allocator detail */
1017 nvgpu_warn(g,
1018 "alignment larger than buffer size rounded up to power of 2 is not supported");
1019 return -EINVAL;
1020 }
1021
1022 fd = nvgpu_vidmem_export_linux(g, args->in.size);
1023 if (fd < 0)
1024 return fd;
1025
1026 args->out.dmabuf_fd = fd;
1027
1028 nvgpu_log_fn(g, "done, fd=%d", fd);
1029
1030 return 0;
1031}
1032
1033static int nvgpu_gpu_get_memory_state(struct gk20a *g,
1034 struct nvgpu_gpu_get_memory_state_args *args)
1035{
1036 int err;
1037
1038 nvgpu_log_fn(g, " ");
1039
1040 if (args->reserved[0] || args->reserved[1] ||
1041 args->reserved[2] || args->reserved[3])
1042 return -EINVAL;
1043
1044 err = nvgpu_vidmem_get_space(g, &args->total_free_bytes);
1045
1046 nvgpu_log_fn(g, "done, err=%d, bytes=%lld", err, args->total_free_bytes);
1047
1048 return err;
1049}
1050
1051static u32 nvgpu_gpu_convert_clk_domain(u32 clk_domain)
1052{
1053 u32 domain = 0;
1054
1055 if (clk_domain == NVGPU_GPU_CLK_DOMAIN_MCLK)
1056 domain = NVGPU_CLK_DOMAIN_MCLK;
1057 else if (clk_domain == NVGPU_GPU_CLK_DOMAIN_GPCCLK)
1058 domain = NVGPU_CLK_DOMAIN_GPCCLK;
1059 else
1060 domain = NVGPU_CLK_DOMAIN_MAX + 1;
1061
1062 return domain;
1063}
1064
1065static int nvgpu_gpu_clk_get_vf_points(struct gk20a *g,
1066 struct gk20a_ctrl_priv *priv,
1067 struct nvgpu_gpu_clk_vf_points_args *args)
1068{
1069 struct nvgpu_gpu_clk_vf_point clk_point;
1070 struct nvgpu_gpu_clk_vf_point __user *entry;
1071 struct nvgpu_clk_session *session = priv->clk_session;
1072 u32 clk_domains = 0;
1073 int err;
1074 u16 last_mhz;
1075 u16 *fpoints;
1076 u32 i;
1077 u32 max_points = 0;
1078 u32 num_points = 0;
1079 u16 min_mhz;
1080 u16 max_mhz;
1081
1082 nvgpu_log_fn(g, " ");
1083
1084 if (!session || args->flags)
1085 return -EINVAL;
1086
1087 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
1088 args->num_entries = 0;
1089
1090 if (!nvgpu_clk_arb_is_valid_domain(g,
1091 nvgpu_gpu_convert_clk_domain(args->clk_domain)))
1092 return -EINVAL;
1093
1094 err = nvgpu_clk_arb_get_arbiter_clk_f_points(g,
1095 nvgpu_gpu_convert_clk_domain(args->clk_domain),
1096 &max_points, NULL);
1097 if (err)
1098 return err;
1099
1100 if (!args->max_entries) {
1101 args->max_entries = max_points;
1102 return 0;
1103 }
1104
1105 if (args->max_entries < max_points)
1106 return -EINVAL;
1107
1108 err = nvgpu_clk_arb_get_arbiter_clk_range(g,
1109 nvgpu_gpu_convert_clk_domain(args->clk_domain),
1110 &min_mhz, &max_mhz);
1111 if (err)
1112 return err;
1113
1114 fpoints = nvgpu_kcalloc(g, max_points, sizeof(u16));
1115 if (!fpoints)
1116 return -ENOMEM;
1117
1118 err = nvgpu_clk_arb_get_arbiter_clk_f_points(g,
1119 nvgpu_gpu_convert_clk_domain(args->clk_domain),
1120 &max_points, fpoints);
1121 if (err)
1122 goto fail;
1123
1124 entry = (struct nvgpu_gpu_clk_vf_point __user *)
1125 (uintptr_t)args->clk_vf_point_entries;
1126
1127 last_mhz = 0;
1128 num_points = 0;
1129 for (i = 0; (i < max_points) && !err; i++) {
1130
1131 /* filter out duplicate frequencies */
1132 if (fpoints[i] == last_mhz)
1133 continue;
1134
1135 /* filter out out-of-range frequencies */
1136 if ((fpoints[i] < min_mhz) || (fpoints[i] > max_mhz))
1137 continue;
1138
1139 last_mhz = fpoints[i];
1140 clk_point.freq_hz = MHZ_TO_HZ(fpoints[i]);
1141
1142 err = copy_to_user((void __user *)entry, &clk_point,
1143 sizeof(clk_point));
1144
1145 num_points++;
1146 entry++;
1147 }
1148
1149 args->num_entries = num_points;
1150
1151fail:
1152 nvgpu_kfree(g, fpoints);
1153 return err;
1154}
1155
1156static int nvgpu_gpu_clk_get_range(struct gk20a *g,
1157 struct gk20a_ctrl_priv *priv,
1158 struct nvgpu_gpu_clk_range_args *args)
1159{
1160 struct nvgpu_gpu_clk_range clk_range;
1161 struct nvgpu_gpu_clk_range __user *entry;
1162 struct nvgpu_clk_session *session = priv->clk_session;
1163
1164 u32 clk_domains = 0;
1165 u32 num_domains;
1166 u32 num_entries;
1167 u32 i;
1168 int bit;
1169 int err;
1170 u16 min_mhz, max_mhz;
1171
1172 nvgpu_log_fn(g, " ");
1173
1174 if (!session)
1175 return -EINVAL;
1176
1177 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
1178 num_domains = hweight_long(clk_domains);
1179
1180 if (!args->flags) {
1181 if (!args->num_entries) {
1182 args->num_entries = num_domains;
1183 return 0;
1184 }
1185
1186 if (args->num_entries < num_domains)
1187 return -EINVAL;
1188
1189 args->num_entries = 0;
1190 num_entries = num_domains;
1191
1192 } else {
1193 if (args->flags != NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS)
1194 return -EINVAL;
1195
1196 num_entries = args->num_entries;
1197 if (num_entries > num_domains)
1198 return -EINVAL;
1199 }
1200
1201 entry = (struct nvgpu_gpu_clk_range __user *)
1202 (uintptr_t)args->clk_range_entries;
1203
1204 for (i = 0; i < num_entries; i++, entry++) {
1205
1206 if (args->flags == NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS) {
1207 if (copy_from_user(&clk_range, (void __user *)entry,
1208 sizeof(clk_range)))
1209 return -EFAULT;
1210 } else {
1211 bit = ffs(clk_domains) - 1;
1212 clk_range.clk_domain = bit;
1213 clk_domains &= ~BIT(bit);
1214 }
1215
1216 clk_range.flags = 0;
1217 err = nvgpu_clk_arb_get_arbiter_clk_range(g,
1218 nvgpu_gpu_convert_clk_domain(clk_range.clk_domain),
1219 &min_mhz, &max_mhz);
1220 clk_range.min_hz = MHZ_TO_HZ(min_mhz);
1221 clk_range.max_hz = MHZ_TO_HZ(max_mhz);
1222
1223 if (err)
1224 return err;
1225
1226 err = copy_to_user(entry, &clk_range, sizeof(clk_range));
1227 if (err)
1228 return -EFAULT;
1229 }
1230
1231 args->num_entries = num_entries;
1232
1233 return 0;
1234}
1235
1236static int nvgpu_gpu_clk_set_info(struct gk20a *g,
1237 struct gk20a_ctrl_priv *priv,
1238 struct nvgpu_gpu_clk_set_info_args *args)
1239{
1240 struct nvgpu_gpu_clk_info clk_info;
1241 struct nvgpu_gpu_clk_info __user *entry;
1242 struct nvgpu_clk_session *session = priv->clk_session;
1243
1244 int fd;
1245 u32 clk_domains = 0;
1246 u16 freq_mhz;
1247 int i;
1248 int ret;
1249
1250 nvgpu_log_fn(g, " ");
1251
1252 if (!session || args->flags)
1253 return -EINVAL;
1254
1255 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
1256 if (!clk_domains)
1257 return -EINVAL;
1258
1259 entry = (struct nvgpu_gpu_clk_info __user *)
1260 (uintptr_t)args->clk_info_entries;
1261
1262 for (i = 0; i < args->num_entries; i++, entry++) {
1263
1264 if (copy_from_user(&clk_info, entry, sizeof(clk_info)))
1265 return -EFAULT;
1266
1267 if (!nvgpu_clk_arb_is_valid_domain(g,
1268 nvgpu_gpu_convert_clk_domain(clk_info.clk_domain)))
1269 return -EINVAL;
1270 }
1271 nvgpu_speculation_barrier();
1272
1273 entry = (struct nvgpu_gpu_clk_info __user *)
1274 (uintptr_t)args->clk_info_entries;
1275
1276 ret = nvgpu_clk_arb_install_request_fd(g, session, &fd);
1277 if (ret < 0)
1278 return ret;
1279
1280 for (i = 0; i < args->num_entries; i++, entry++) {
1281
1282 if (copy_from_user(&clk_info, (void __user *)entry,
1283 sizeof(clk_info)))
1284 return -EFAULT;
1285 freq_mhz = HZ_TO_MHZ(clk_info.freq_hz);
1286
1287 nvgpu_clk_arb_set_session_target_mhz(session, fd,
1288 nvgpu_gpu_convert_clk_domain(clk_info.clk_domain), freq_mhz);
1289 }
1290
1291 nvgpu_speculation_barrier();
1292 ret = nvgpu_clk_arb_commit_request_fd(g, session, fd);
1293 if (ret < 0)
1294 return ret;
1295
1296 args->completion_fd = fd;
1297
1298 return ret;
1299}
1300
1301static int nvgpu_gpu_clk_get_info(struct gk20a *g,
1302 struct gk20a_ctrl_priv *priv,
1303 struct nvgpu_gpu_clk_get_info_args *args)
1304{
1305 struct nvgpu_gpu_clk_info clk_info;
1306 struct nvgpu_gpu_clk_info __user *entry;
1307 struct nvgpu_clk_session *session = priv->clk_session;
1308 u32 clk_domains = 0;
1309 u32 num_domains;
1310 u32 num_entries;
1311 u32 i;
1312 u16 freq_mhz;
1313 int err;
1314 int bit;
1315
1316 nvgpu_log_fn(g, " ");
1317
1318 if (!session)
1319 return -EINVAL;
1320
1321 clk_domains = nvgpu_clk_arb_get_arbiter_clk_domains(g);
1322 num_domains = hweight_long(clk_domains);
1323
1324 if (!args->flags) {
1325 if (!args->num_entries) {
1326 args->num_entries = num_domains;
1327 return 0;
1328 }
1329
1330 if (args->num_entries < num_domains)
1331 return -EINVAL;
1332
1333 args->num_entries = 0;
1334 num_entries = num_domains;
1335
1336 } else {
1337 if (args->flags != NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS)
1338 return -EINVAL;
1339
1340 num_entries = args->num_entries;
1341 if (num_entries > num_domains * 3)
1342 return -EINVAL;
1343 }
1344
1345 entry = (struct nvgpu_gpu_clk_info __user *)
1346 (uintptr_t)args->clk_info_entries;
1347
1348 for (i = 0; i < num_entries; i++, entry++) {
1349
1350 if (args->flags == NVGPU_GPU_CLK_FLAG_SPECIFIC_DOMAINS) {
1351 if (copy_from_user(&clk_info, (void __user *)entry,
1352 sizeof(clk_info)))
1353 return -EFAULT;
1354 } else {
1355 bit = ffs(clk_domains) - 1;
1356 clk_info.clk_domain = bit;
1357 clk_domains &= ~BIT(bit);
1358 clk_info.clk_type = args->clk_type;
1359 }
1360
1361 nvgpu_speculation_barrier();
1362 switch (clk_info.clk_type) {
1363 case NVGPU_GPU_CLK_TYPE_TARGET:
1364 err = nvgpu_clk_arb_get_session_target_mhz(session,
1365 nvgpu_gpu_convert_clk_domain(clk_info.clk_domain),
1366 &freq_mhz);
1367 break;
1368 case NVGPU_GPU_CLK_TYPE_ACTUAL:
1369 err = nvgpu_clk_arb_get_arbiter_actual_mhz(g,
1370 nvgpu_gpu_convert_clk_domain(clk_info.clk_domain),
1371 &freq_mhz);
1372 break;
1373 case NVGPU_GPU_CLK_TYPE_EFFECTIVE:
1374 err = nvgpu_clk_arb_get_arbiter_effective_mhz(g,
1375 nvgpu_gpu_convert_clk_domain(clk_info.clk_domain),
1376 &freq_mhz);
1377 break;
1378 default:
1379 freq_mhz = 0;
1380 err = -EINVAL;
1381 break;
1382 }
1383 if (err)
1384 return err;
1385
1386 clk_info.flags = 0;
1387 clk_info.freq_hz = MHZ_TO_HZ(freq_mhz);
1388
1389 err = copy_to_user((void __user *)entry, &clk_info,
1390 sizeof(clk_info));
1391 if (err)
1392 return -EFAULT;
1393 }
1394
1395 nvgpu_speculation_barrier();
1396 args->num_entries = num_entries;
1397
1398 return 0;
1399}
1400
1401static int nvgpu_gpu_get_event_fd(struct gk20a *g,
1402 struct gk20a_ctrl_priv *priv,
1403 struct nvgpu_gpu_get_event_fd_args *args)
1404{
1405 struct nvgpu_clk_session *session = priv->clk_session;
1406
1407 nvgpu_log_fn(g, " ");
1408
1409 if (!session)
1410 return -EINVAL;
1411
1412 return nvgpu_clk_arb_install_event_fd(g, session, &args->event_fd,
1413 args->flags);
1414}
1415
1416static int nvgpu_gpu_get_voltage(struct gk20a *g,
1417 struct nvgpu_gpu_get_voltage_args *args)
1418{
1419 int err = -EINVAL;
1420
1421 nvgpu_log_fn(g, " ");
1422
1423 if (args->reserved)
1424 return -EINVAL;
1425
1426 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_GET_VOLTAGE))
1427 return -EINVAL;
1428
1429 err = gk20a_busy(g);
1430 if (err)
1431 return err;
1432
1433 nvgpu_speculation_barrier();
1434 switch (args->which) {
1435 case NVGPU_GPU_VOLTAGE_CORE:
1436 err = volt_get_voltage(g, CTRL_VOLT_DOMAIN_LOGIC, &args->voltage);
1437 break;
1438 case NVGPU_GPU_VOLTAGE_SRAM:
1439 err = volt_get_voltage(g, CTRL_VOLT_DOMAIN_SRAM, &args->voltage);
1440 break;
1441 case NVGPU_GPU_VOLTAGE_BUS:
1442 err = pmgr_pwr_devices_get_voltage(g, &args->voltage);
1443 break;
1444 default:
1445 err = -EINVAL;
1446 }
1447
1448 gk20a_idle(g);
1449
1450 return err;
1451}
1452
1453static int nvgpu_gpu_get_current(struct gk20a *g,
1454 struct nvgpu_gpu_get_current_args *args)
1455{
1456 int err;
1457
1458 nvgpu_log_fn(g, " ");
1459
1460 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1461 return -EINVAL;
1462
1463 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_GET_CURRENT))
1464 return -EINVAL;
1465
1466 err = gk20a_busy(g);
1467 if (err)
1468 return err;
1469
1470 err = pmgr_pwr_devices_get_current(g, &args->currnt);
1471
1472 gk20a_idle(g);
1473
1474 return err;
1475}
1476
1477static int nvgpu_gpu_get_power(struct gk20a *g,
1478 struct nvgpu_gpu_get_power_args *args)
1479{
1480 int err;
1481
1482 nvgpu_log_fn(g, " ");
1483
1484 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1485 return -EINVAL;
1486
1487 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_GET_POWER))
1488 return -EINVAL;
1489
1490 err = gk20a_busy(g);
1491 if (err)
1492 return err;
1493
1494 err = pmgr_pwr_devices_get_power(g, &args->power);
1495
1496 gk20a_idle(g);
1497
1498 return err;
1499}
1500
1501static int nvgpu_gpu_get_temperature(struct gk20a *g,
1502 struct nvgpu_gpu_get_temperature_args *args)
1503{
1504 int err;
1505 u32 temp_f24_8;
1506
1507 nvgpu_log_fn(g, " ");
1508
1509 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1510 return -EINVAL;
1511
1512 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_GET_TEMPERATURE))
1513 return -EINVAL;
1514
1515 if (!g->ops.therm.get_internal_sensor_curr_temp)
1516 return -EINVAL;
1517
1518 err = gk20a_busy(g);
1519 if (err)
1520 return err;
1521
1522 err = g->ops.therm.get_internal_sensor_curr_temp(g, &temp_f24_8);
1523
1524 gk20a_idle(g);
1525
1526 args->temp_f24_8 = (s32)temp_f24_8;
1527
1528 return err;
1529}
1530
1531static int nvgpu_gpu_set_therm_alert_limit(struct gk20a *g,
1532 struct nvgpu_gpu_set_therm_alert_limit_args *args)
1533{
1534 int err;
1535
1536 nvgpu_log_fn(g, " ");
1537
1538 if (args->reserved[0] || args->reserved[1] || args->reserved[2])
1539 return -EINVAL;
1540
1541 if (!g->ops.therm.configure_therm_alert)
1542 return -EINVAL;
1543
1544 err = gk20a_busy(g);
1545 if (err)
1546 return err;
1547
1548 err = g->ops.therm.configure_therm_alert(g, args->temp_f24_8);
1549
1550 gk20a_idle(g);
1551
1552 return err;
1553}
1554
1555static int nvgpu_gpu_set_deterministic_ch_railgate(struct channel_gk20a *ch,
1556 u32 flags)
1557{
1558 int err = 0;
1559 bool allow;
1560 bool disallow;
1561
1562 allow = flags &
1563 NVGPU_GPU_SET_DETERMINISTIC_OPTS_FLAGS_ALLOW_RAILGATING;
1564
1565 disallow = flags &
1566 NVGPU_GPU_SET_DETERMINISTIC_OPTS_FLAGS_DISALLOW_RAILGATING;
1567
1568 /* Can't be both at the same time */
1569 if (allow && disallow)
1570 return -EINVAL;
1571
1572 /* Nothing to do */
1573 if (!allow && !disallow)
1574 return 0;
1575
1576 /*
1577 * Moving into explicit idle or back from it? A call that doesn't
1578 * change the status is a no-op.
1579 */
1580 if (!ch->deterministic_railgate_allowed &&
1581 allow) {
1582 gk20a_idle(ch->g);
1583 } else if (ch->deterministic_railgate_allowed &&
1584 !allow) {
1585 err = gk20a_busy(ch->g);
1586 if (err) {
1587 nvgpu_warn(ch->g,
1588 "cannot busy to restore deterministic ch");
1589 return err;
1590 }
1591 }
1592 ch->deterministic_railgate_allowed = allow;
1593
1594 return err;
1595}
1596
1597static int nvgpu_gpu_set_deterministic_ch(struct channel_gk20a *ch, u32 flags)
1598{
1599 if (!ch->deterministic)
1600 return -EINVAL;
1601
1602 return nvgpu_gpu_set_deterministic_ch_railgate(ch, flags);
1603}
1604
1605static int nvgpu_gpu_set_deterministic_opts(struct gk20a *g,
1606 struct nvgpu_gpu_set_deterministic_opts_args *args)
1607{
1608 int __user *user_channels;
1609 u32 i = 0;
1610 int err = 0;
1611
1612 nvgpu_log_fn(g, " ");
1613
1614 user_channels = (int __user *)(uintptr_t)args->channels;
1615
1616 /* Upper limit; prevent holding deterministic_busy for long */
1617 if (args->num_channels > g->fifo.num_channels) {
1618 err = -EINVAL;
1619 goto out;
1620 }
1621
1622 /* Trivial sanity check first */
1623 if (!access_ok(VERIFY_READ, user_channels,
1624 args->num_channels * sizeof(int))) {
1625 err = -EFAULT;
1626 goto out;
1627 }
1628
1629 nvgpu_rwsem_down_read(&g->deterministic_busy);
1630
1631 /* note: we exit at the first failure */
1632 for (; i < args->num_channels; i++) {
1633 int ch_fd = 0;
1634 struct channel_gk20a *ch;
1635
1636 if (copy_from_user(&ch_fd, &user_channels[i], sizeof(int))) {
1637 /* User raced with above access_ok */
1638 err = -EFAULT;
1639 break;
1640 }
1641
1642 ch = gk20a_get_channel_from_file(ch_fd);
1643 if (!ch) {
1644 err = -EINVAL;
1645 break;
1646 }
1647
1648 err = nvgpu_gpu_set_deterministic_ch(ch, args->flags);
1649
1650 gk20a_channel_put(ch);
1651
1652 if (err)
1653 break;
1654 }
1655
1656 nvgpu_speculation_barrier();
1657 nvgpu_rwsem_up_read(&g->deterministic_busy);
1658
1659out:
1660 args->num_channels = i;
1661 return err;
1662}
1663
1664long gk20a_ctrl_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1665{
1666 struct gk20a_ctrl_priv *priv = filp->private_data;
1667 struct gk20a *g = priv->g;
1668 struct nvgpu_gpu_zcull_get_ctx_size_args *get_ctx_size_args;
1669 struct nvgpu_gpu_zcull_get_info_args *get_info_args;
1670 struct nvgpu_gpu_zbc_set_table_args *set_table_args;
1671 struct nvgpu_gpu_zbc_query_table_args *query_table_args;
1672 u8 buf[NVGPU_GPU_IOCTL_MAX_ARG_SIZE];
1673 struct gr_zcull_info *zcull_info;
1674 struct zbc_entry *zbc_val;
1675 struct zbc_query_params *zbc_tbl;
1676 int i, err = 0;
1677
1678 nvgpu_log_fn(g, "start %d", _IOC_NR(cmd));
1679
1680 if ((_IOC_TYPE(cmd) != NVGPU_GPU_IOCTL_MAGIC) ||
1681 (_IOC_NR(cmd) == 0) ||
1682 (_IOC_NR(cmd) > NVGPU_GPU_IOCTL_LAST) ||
1683 (_IOC_SIZE(cmd) > NVGPU_GPU_IOCTL_MAX_ARG_SIZE))
1684 return -EINVAL;
1685
1686 memset(buf, 0, sizeof(buf));
1687 if (_IOC_DIR(cmd) & _IOC_WRITE) {
1688 if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
1689 return -EFAULT;
1690 }
1691
1692 if (!g->sw_ready) {
1693 err = gk20a_busy(g);
1694 if (err)
1695 return err;
1696
1697 gk20a_idle(g);
1698 }
1699
1700 nvgpu_speculation_barrier();
1701 switch (cmd) {
1702 case NVGPU_GPU_IOCTL_ZCULL_GET_CTX_SIZE:
1703 get_ctx_size_args = (struct nvgpu_gpu_zcull_get_ctx_size_args *)buf;
1704
1705 get_ctx_size_args->size = gr_gk20a_get_ctxsw_zcull_size(g, &g->gr);
1706
1707 break;
1708 case NVGPU_GPU_IOCTL_ZCULL_GET_INFO:
1709 get_info_args = (struct nvgpu_gpu_zcull_get_info_args *)buf;
1710
1711 memset(get_info_args, 0, sizeof(struct nvgpu_gpu_zcull_get_info_args));
1712
1713 zcull_info = nvgpu_kzalloc(g, sizeof(struct gr_zcull_info));
1714 if (zcull_info == NULL)
1715 return -ENOMEM;
1716
1717 err = g->ops.gr.get_zcull_info(g, &g->gr, zcull_info);
1718 if (err) {
1719 nvgpu_kfree(g, zcull_info);
1720 break;
1721 }
1722
1723 get_info_args->width_align_pixels = zcull_info->width_align_pixels;
1724 get_info_args->height_align_pixels = zcull_info->height_align_pixels;
1725 get_info_args->pixel_squares_by_aliquots = zcull_info->pixel_squares_by_aliquots;
1726 get_info_args->aliquot_total = zcull_info->aliquot_total;
1727 get_info_args->region_byte_multiplier = zcull_info->region_byte_multiplier;
1728 get_info_args->region_header_size = zcull_info->region_header_size;
1729 get_info_args->subregion_header_size = zcull_info->subregion_header_size;
1730 get_info_args->subregion_width_align_pixels = zcull_info->subregion_width_align_pixels;
1731 get_info_args->subregion_height_align_pixels = zcull_info->subregion_height_align_pixels;
1732 get_info_args->subregion_count = zcull_info->subregion_count;
1733
1734 nvgpu_kfree(g, zcull_info);
1735 break;
1736 case NVGPU_GPU_IOCTL_ZBC_SET_TABLE:
1737 set_table_args = (struct nvgpu_gpu_zbc_set_table_args *)buf;
1738
1739 zbc_val = nvgpu_kzalloc(g, sizeof(struct zbc_entry));
1740 if (zbc_val == NULL)
1741 return -ENOMEM;
1742
1743 zbc_val->format = set_table_args->format;
1744 zbc_val->type = set_table_args->type;
1745
1746 nvgpu_speculation_barrier();
1747 switch (zbc_val->type) {
1748 case GK20A_ZBC_TYPE_COLOR:
1749 for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) {
1750 zbc_val->color_ds[i] = set_table_args->color_ds[i];
1751 zbc_val->color_l2[i] = set_table_args->color_l2[i];
1752 }
1753 break;
1754 case GK20A_ZBC_TYPE_DEPTH:
1755 case T19X_ZBC:
1756 zbc_val->depth = set_table_args->depth;
1757 break;
1758 default:
1759 err = -EINVAL;
1760 }
1761
1762 if (!err) {
1763 err = gk20a_busy(g);
1764 if (!err) {
1765 err = g->ops.gr.zbc_set_table(g, &g->gr,
1766 zbc_val);
1767 gk20a_idle(g);
1768 }
1769 }
1770
1771 if (zbc_val)
1772 nvgpu_kfree(g, zbc_val);
1773 break;
1774 case NVGPU_GPU_IOCTL_ZBC_QUERY_TABLE:
1775 query_table_args = (struct nvgpu_gpu_zbc_query_table_args *)buf;
1776
1777 zbc_tbl = nvgpu_kzalloc(g, sizeof(struct zbc_query_params));
1778 if (zbc_tbl == NULL)
1779 return -ENOMEM;
1780
1781 zbc_tbl->type = query_table_args->type;
1782 zbc_tbl->index_size = query_table_args->index_size;
1783
1784 err = g->ops.gr.zbc_query_table(g, &g->gr, zbc_tbl);
1785
1786 if (!err) {
1787 switch (zbc_tbl->type) {
1788 case GK20A_ZBC_TYPE_COLOR:
1789 for (i = 0; i < GK20A_ZBC_COLOR_VALUE_SIZE; i++) {
1790 query_table_args->color_ds[i] = zbc_tbl->color_ds[i];
1791 query_table_args->color_l2[i] = zbc_tbl->color_l2[i];
1792 }
1793 break;
1794 case GK20A_ZBC_TYPE_DEPTH:
1795 case T19X_ZBC:
1796 query_table_args->depth = zbc_tbl->depth;
1797 break;
1798 case GK20A_ZBC_TYPE_INVALID:
1799 query_table_args->index_size = zbc_tbl->index_size;
1800 break;
1801 default:
1802 err = -EINVAL;
1803 }
1804 if (!err) {
1805 query_table_args->format = zbc_tbl->format;
1806 query_table_args->ref_cnt = zbc_tbl->ref_cnt;
1807 }
1808 }
1809
1810 if (zbc_tbl)
1811 nvgpu_kfree(g, zbc_tbl);
1812 break;
1813
1814 case NVGPU_GPU_IOCTL_GET_CHARACTERISTICS:
1815 err = gk20a_ctrl_ioctl_gpu_characteristics(
1816 g, (struct nvgpu_gpu_get_characteristics *)buf);
1817 break;
1818 case NVGPU_GPU_IOCTL_PREPARE_COMPRESSIBLE_READ:
1819 err = gk20a_ctrl_prepare_compressible_read(g,
1820 (struct nvgpu_gpu_prepare_compressible_read_args *)buf);
1821 break;
1822 case NVGPU_GPU_IOCTL_MARK_COMPRESSIBLE_WRITE:
1823 err = gk20a_ctrl_mark_compressible_write(g,
1824 (struct nvgpu_gpu_mark_compressible_write_args *)buf);
1825 break;
1826 case NVGPU_GPU_IOCTL_ALLOC_AS:
1827 err = gk20a_ctrl_alloc_as(g,
1828 (struct nvgpu_alloc_as_args *)buf);
1829 break;
1830 case NVGPU_GPU_IOCTL_OPEN_TSG:
1831 err = gk20a_ctrl_open_tsg(g,
1832 (struct nvgpu_gpu_open_tsg_args *)buf);
1833 break;
1834 case NVGPU_GPU_IOCTL_GET_TPC_MASKS:
1835 err = gk20a_ctrl_get_tpc_masks(g,
1836 (struct nvgpu_gpu_get_tpc_masks_args *)buf);
1837 break;
1838 case NVGPU_GPU_IOCTL_GET_FBP_L2_MASKS:
1839 err = gk20a_ctrl_get_fbp_l2_masks(g,
1840 (struct nvgpu_gpu_get_fbp_l2_masks_args *)buf);
1841 break;
1842 case NVGPU_GPU_IOCTL_OPEN_CHANNEL:
1843 /* this arg type here, but ..gpu_open_channel_args in nvgpu.h
1844 * for consistency - they are the same */
1845 err = gk20a_channel_open_ioctl(g,
1846 (struct nvgpu_channel_open_args *)buf);
1847 break;
1848 case NVGPU_GPU_IOCTL_FLUSH_L2:
1849 err = nvgpu_gpu_ioctl_l2_fb_ops(g,
1850 (struct nvgpu_gpu_l2_fb_args *)buf);
1851 break;
1852
1853 case NVGPU_GPU_IOCTL_SET_MMUDEBUG_MODE:
1854 err = nvgpu_gpu_ioctl_set_mmu_debug_mode(g,
1855 (struct nvgpu_gpu_mmu_debug_mode_args *)buf);
1856 break;
1857
1858 case NVGPU_GPU_IOCTL_SET_SM_DEBUG_MODE:
1859 err = gr_gk20a_elpg_protected_call(g,
1860 nvgpu_gpu_ioctl_set_debug_mode(g, (struct nvgpu_gpu_sm_debug_mode_args *)buf));
1861 break;
1862
1863 case NVGPU_GPU_IOCTL_TRIGGER_SUSPEND:
1864 err = nvgpu_gpu_ioctl_trigger_suspend(g);
1865 break;
1866
1867 case NVGPU_GPU_IOCTL_WAIT_FOR_PAUSE:
1868 err = nvgpu_gpu_ioctl_wait_for_pause(g,
1869 (struct nvgpu_gpu_wait_pause_args *)buf);
1870 break;
1871
1872 case NVGPU_GPU_IOCTL_RESUME_FROM_PAUSE:
1873 err = nvgpu_gpu_ioctl_resume_from_pause(g);
1874 break;
1875
1876 case NVGPU_GPU_IOCTL_CLEAR_SM_ERRORS:
1877 err = nvgpu_gpu_ioctl_clear_sm_errors(g);
1878 break;
1879
1880 case NVGPU_GPU_IOCTL_GET_TPC_EXCEPTION_EN_STATUS:
1881 err = nvgpu_gpu_ioctl_has_any_exception(g,
1882 (struct nvgpu_gpu_tpc_exception_en_status_args *)buf);
1883 break;
1884
1885 case NVGPU_GPU_IOCTL_NUM_VSMS:
1886 err = gk20a_ctrl_get_num_vsms(g,
1887 (struct nvgpu_gpu_num_vsms *)buf);
1888 break;
1889 case NVGPU_GPU_IOCTL_VSMS_MAPPING:
1890 err = gk20a_ctrl_vsm_mapping(g,
1891 (struct nvgpu_gpu_vsms_mapping *)buf);
1892 break;
1893
1894 case NVGPU_GPU_IOCTL_GET_CPU_TIME_CORRELATION_INFO:
1895 err = nvgpu_gpu_get_cpu_time_correlation_info(g,
1896 (struct nvgpu_gpu_get_cpu_time_correlation_info_args *)buf);
1897 break;
1898
1899 case NVGPU_GPU_IOCTL_GET_GPU_TIME:
1900 err = nvgpu_gpu_get_gpu_time(g,
1901 (struct nvgpu_gpu_get_gpu_time_args *)buf);
1902 break;
1903
1904 case NVGPU_GPU_IOCTL_GET_ENGINE_INFO:
1905 err = nvgpu_gpu_get_engine_info(g,
1906 (struct nvgpu_gpu_get_engine_info_args *)buf);
1907 break;
1908
1909 case NVGPU_GPU_IOCTL_ALLOC_VIDMEM:
1910 err = nvgpu_gpu_alloc_vidmem(g,
1911 (struct nvgpu_gpu_alloc_vidmem_args *)buf);
1912 break;
1913
1914 case NVGPU_GPU_IOCTL_GET_MEMORY_STATE:
1915 err = nvgpu_gpu_get_memory_state(g,
1916 (struct nvgpu_gpu_get_memory_state_args *)buf);
1917 break;
1918
1919 case NVGPU_GPU_IOCTL_CLK_GET_RANGE:
1920 err = nvgpu_gpu_clk_get_range(g, priv,
1921 (struct nvgpu_gpu_clk_range_args *)buf);
1922 break;
1923
1924 case NVGPU_GPU_IOCTL_CLK_GET_VF_POINTS:
1925 err = nvgpu_gpu_clk_get_vf_points(g, priv,
1926 (struct nvgpu_gpu_clk_vf_points_args *)buf);
1927 break;
1928
1929 case NVGPU_GPU_IOCTL_CLK_SET_INFO:
1930 err = nvgpu_gpu_clk_set_info(g, priv,
1931 (struct nvgpu_gpu_clk_set_info_args *)buf);
1932 break;
1933
1934 case NVGPU_GPU_IOCTL_CLK_GET_INFO:
1935 err = nvgpu_gpu_clk_get_info(g, priv,
1936 (struct nvgpu_gpu_clk_get_info_args *)buf);
1937 break;
1938
1939 case NVGPU_GPU_IOCTL_GET_EVENT_FD:
1940 err = nvgpu_gpu_get_event_fd(g, priv,
1941 (struct nvgpu_gpu_get_event_fd_args *)buf);
1942 break;
1943
1944 case NVGPU_GPU_IOCTL_GET_VOLTAGE:
1945 err = nvgpu_gpu_get_voltage(g,
1946 (struct nvgpu_gpu_get_voltage_args *)buf);
1947 break;
1948
1949 case NVGPU_GPU_IOCTL_GET_CURRENT:
1950 err = nvgpu_gpu_get_current(g,
1951 (struct nvgpu_gpu_get_current_args *)buf);
1952 break;
1953
1954 case NVGPU_GPU_IOCTL_GET_POWER:
1955 err = nvgpu_gpu_get_power(g,
1956 (struct nvgpu_gpu_get_power_args *)buf);
1957 break;
1958
1959 case NVGPU_GPU_IOCTL_GET_TEMPERATURE:
1960 err = nvgpu_gpu_get_temperature(g,
1961 (struct nvgpu_gpu_get_temperature_args *)buf);
1962 break;
1963
1964 case NVGPU_GPU_IOCTL_SET_THERM_ALERT_LIMIT:
1965 err = nvgpu_gpu_set_therm_alert_limit(g,
1966 (struct nvgpu_gpu_set_therm_alert_limit_args *)buf);
1967 break;
1968
1969 case NVGPU_GPU_IOCTL_SET_DETERMINISTIC_OPTS:
1970 err = nvgpu_gpu_set_deterministic_opts(g,
1971 (struct nvgpu_gpu_set_deterministic_opts_args *)buf);
1972 break;
1973
1974 default:
1975 nvgpu_log_info(g, "unrecognized gpu ioctl cmd: 0x%x", cmd);
1976 err = -ENOTTY;
1977 break;
1978 }
1979
1980 if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
1981 err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
1982
1983 return err;
1984}
1985
1986static void usermode_vma_close(struct vm_area_struct *vma)
1987{
1988 struct gk20a_ctrl_priv *priv = vma->vm_private_data;
1989 struct gk20a *g = priv->g;
1990 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
1991
1992 nvgpu_mutex_acquire(&l->ctrl.privs_lock);
1993 priv->usermode_vma.vma = NULL;
1994 priv->usermode_vma.vma_mapped = false;
1995 nvgpu_mutex_release(&l->ctrl.privs_lock);
1996}
1997
1998struct vm_operations_struct usermode_vma_ops = {
1999 /* no .open - we use VM_DONTCOPY and don't support fork */
2000 .close = usermode_vma_close,
2001};
2002
2003int gk20a_ctrl_dev_mmap(struct file *filp, struct vm_area_struct *vma)
2004{
2005 struct gk20a_ctrl_priv *priv = filp->private_data;
2006 struct gk20a *g = priv->g;
2007 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
2008 u64 addr;
2009 int err;
2010
2011 if (g->ops.fifo.usermode_base == NULL)
2012 return -ENOSYS;
2013
2014 if (priv->usermode_vma.vma != NULL)
2015 return -EBUSY;
2016
2017 if (vma->vm_end - vma->vm_start != SZ_4K)
2018 return -EINVAL;
2019
2020 if (vma->vm_pgoff != 0UL)
2021 return -EINVAL;
2022
2023 addr = l->regs_bus_addr + g->ops.fifo.usermode_base(g);
2024
2025 /* Sync with poweron/poweroff, and require valid regs */
2026 err = gk20a_busy(g);
2027 if (err) {
2028 return err;
2029 }
2030
2031 nvgpu_mutex_acquire(&l->ctrl.privs_lock);
2032
2033 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
2034 VM_DONTDUMP | VM_PFNMAP;
2035 vma->vm_ops = &usermode_vma_ops;
2036 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
2037
2038 err = io_remap_pfn_range(vma, vma->vm_start, addr >> PAGE_SHIFT,
2039 vma->vm_end - vma->vm_start, vma->vm_page_prot);
2040 if (!err) {
2041 priv->usermode_vma.vma = vma;
2042 vma->vm_private_data = priv;
2043 priv->usermode_vma.vma_mapped = true;
2044 }
2045 nvgpu_mutex_release(&l->ctrl.privs_lock);
2046
2047 gk20a_idle(g);
2048
2049 return err;
2050}
2051
2052static int alter_usermode_mapping(struct gk20a *g,
2053 struct gk20a_ctrl_priv *priv,
2054 bool poweroff)
2055{
2056 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
2057 struct vm_area_struct *vma = priv->usermode_vma.vma;
2058 bool vma_mapped = priv->usermode_vma.vma_mapped;
2059 u64 addr;
2060 int err = 0;
2061
2062 if (!vma) {
2063 /* Nothing to do - no mmap called */
2064 return 0;
2065 }
2066
2067 addr = l->regs_bus_addr + g->ops.fifo.usermode_base(g);
2068
2069 /*
2070 * This is a no-op for the below cases
2071 * a) poweroff and !vma_mapped - > do nothing as no map exists
2072 * b) !poweroff and vmap_mapped -> do nothing as already mapped
2073 */
2074 if (poweroff != vma_mapped) {
2075 return 0;
2076 }
2077
2078 /*
2079 * We use trylock due to lock inversion: we need to acquire
2080 * mmap_lock while holding ctrl_privs_lock. usermode_vma_close
2081 * does it in reverse order. Trylock is a way to avoid deadlock.
2082 */
2083 if (!down_write_trylock(&vma->vm_mm->mmap_sem)) {
2084 return -EBUSY;
2085 }
2086
2087 if (poweroff) {
2088 err = zap_vma_ptes(vma, vma->vm_start, SZ_4K);
2089 if (err == 0) {
2090 priv->usermode_vma.vma_mapped = false;
2091 } else {
2092 nvgpu_err(g, "can't remove usermode mapping");
2093 }
2094 } else {
2095 err = io_remap_pfn_range(vma, vma->vm_start,
2096 addr >> PAGE_SHIFT,
2097 SZ_4K, vma->vm_page_prot);
2098 if (err != 0) {
2099 nvgpu_err(g, "can't restore usermode mapping");
2100 } else {
2101 priv->usermode_vma.vma_mapped = true;
2102 }
2103 }
2104
2105 up_write(&vma->vm_mm->mmap_sem);
2106
2107 return err;
2108}
2109
2110static void alter_usermode_mappings(struct gk20a *g, bool poweroff)
2111{
2112 struct gk20a_ctrl_priv *priv;
2113 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
2114 int err = 0;
2115
2116 do {
2117 nvgpu_mutex_acquire(&l->ctrl.privs_lock);
2118 nvgpu_list_for_each_entry(priv, &l->ctrl.privs,
2119 gk20a_ctrl_priv, list) {
2120 err = alter_usermode_mapping(g, priv, poweroff);
2121 if (err != 0) {
2122 break;
2123 }
2124 }
2125 nvgpu_mutex_release(&l->ctrl.privs_lock);
2126
2127 if (err == -EBUSY) {
2128 nvgpu_log_info(g, "ctrl_privs_lock lock contended. retry altering usermode mappings");
2129 nvgpu_udelay(10);
2130 } else if (err != 0) {
2131 nvgpu_err(g, "can't alter usermode mapping. err = %d", err);
2132 }
2133 } while (err == -EBUSY);
2134}
2135
2136void nvgpu_hide_usermode_for_poweroff(struct gk20a *g)
2137{
2138 alter_usermode_mappings(g, true);
2139}
2140
2141void nvgpu_restore_usermode_for_poweron(struct gk20a *g)
2142{
2143 alter_usermode_mappings(g, false);
2144}
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-30 16:19:31 -0400