summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/os/linux/ioctl_dbg.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/os/linux/ioctl_dbg.c')
-rw-r--r--drivers/gpu/nvgpu/os/linux/ioctl_dbg.c2003
1 files changed, 2003 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_dbg.c b/drivers/gpu/nvgpu/os/linux/ioctl_dbg.c
new file mode 100644
index 00000000..31e7e2cb
--- /dev/null
+++ b/drivers/gpu/nvgpu/os/linux/ioctl_dbg.c
@@ -0,0 +1,2003 @@
1/*
2 * Tegra GK20A GPU Debugger/Profiler Driver
3 *
4 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <linux/fs.h>
20#include <linux/file.h>
21#include <linux/cdev.h>
22#include <linux/uaccess.h>
23#include <linux/dma-buf.h>
24#include <uapi/linux/nvgpu.h>
25
26#include <nvgpu/kmem.h>
27#include <nvgpu/log.h>
28#include <nvgpu/vm.h>
29#include <nvgpu/atomic.h>
30#include <nvgpu/cond.h>
31
32#include <nvgpu/linux/vidmem.h>
33#include <nvgpu/linux/vm.h>
34
35#include "gk20a/gk20a.h"
36#include "gk20a/gr_gk20a.h"
37#include "gk20a/regops_gk20a.h"
38#include "gk20a/dbg_gpu_gk20a.h"
39#include "os_linux.h"
40#include "platform_gk20a.h"
41#include "ioctl_dbg.h"
42
43/* turn seriously unwieldy names -> something shorter */
44#define REGOP_LINUX(x) NVGPU_DBG_GPU_REG_OP_##x
45
46/* silly allocator - just increment id */
47static nvgpu_atomic_t unique_id = NVGPU_ATOMIC_INIT(0);
48static int generate_unique_id(void)
49{
50 return nvgpu_atomic_add_return(1, &unique_id);
51}
52
53static int alloc_profiler(struct gk20a *g,
54 struct dbg_profiler_object_data **_prof)
55{
56 struct dbg_profiler_object_data *prof;
57 *_prof = NULL;
58
59 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
60
61 prof = nvgpu_kzalloc(g, sizeof(*prof));
62 if (!prof)
63 return -ENOMEM;
64
65 prof->prof_handle = generate_unique_id();
66 *_prof = prof;
67 return 0;
68}
69
70static int alloc_session(struct gk20a *g, struct dbg_session_gk20a_linux **_dbg_s_linux)
71{
72 struct dbg_session_gk20a_linux *dbg_s_linux;
73 *_dbg_s_linux = NULL;
74
75 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
76
77 dbg_s_linux = nvgpu_kzalloc(g, sizeof(*dbg_s_linux));
78 if (!dbg_s_linux)
79 return -ENOMEM;
80
81 dbg_s_linux->dbg_s.id = generate_unique_id();
82 *_dbg_s_linux = dbg_s_linux;
83 return 0;
84}
85
86static bool gr_context_info_available(struct dbg_session_gk20a *dbg_s,
87 struct gr_gk20a *gr);
88
89static int gk20a_perfbuf_release_locked(struct gk20a *g, u64 offset);
90
91static int nvgpu_ioctl_channel_reg_ops(struct dbg_session_gk20a *dbg_s,
92 struct nvgpu_dbg_gpu_exec_reg_ops_args *args);
93
94static int nvgpu_ioctl_powergate_gk20a(struct dbg_session_gk20a *dbg_s,
95 struct nvgpu_dbg_gpu_powergate_args *args);
96
97static int nvgpu_dbg_gpu_ioctl_smpc_ctxsw_mode(struct dbg_session_gk20a *dbg_s,
98 struct nvgpu_dbg_gpu_smpc_ctxsw_mode_args *args);
99
100static int nvgpu_dbg_gpu_ioctl_hwpm_ctxsw_mode(struct dbg_session_gk20a *dbg_s,
101 struct nvgpu_dbg_gpu_hwpm_ctxsw_mode_args *args);
102
103static int nvgpu_dbg_gpu_ioctl_suspend_resume_sm(
104 struct dbg_session_gk20a *dbg_s,
105 struct nvgpu_dbg_gpu_suspend_resume_all_sms_args *args);
106
107static int nvgpu_ioctl_allocate_profiler_object(struct dbg_session_gk20a_linux *dbg_s,
108 struct nvgpu_dbg_gpu_profiler_obj_mgt_args *args);
109
110static int nvgpu_ioctl_free_profiler_object(struct dbg_session_gk20a_linux *dbg_s_linux,
111 struct nvgpu_dbg_gpu_profiler_obj_mgt_args *args);
112
113static int nvgpu_ioctl_profiler_reserve(struct dbg_session_gk20a *dbg_s,
114 struct nvgpu_dbg_gpu_profiler_reserve_args *args);
115
116static int gk20a_perfbuf_map(struct dbg_session_gk20a *dbg_s,
117 struct nvgpu_dbg_gpu_perfbuf_map_args *args);
118
119static int gk20a_perfbuf_unmap(struct dbg_session_gk20a *dbg_s,
120 struct nvgpu_dbg_gpu_perfbuf_unmap_args *args);
121
122static int nvgpu_dbg_timeout_enable(struct dbg_session_gk20a *dbg_s,
123 int timeout_mode);
124
125static int nvgpu_profiler_reserve_acquire(struct dbg_session_gk20a *dbg_s,
126 u32 profiler_handle);
127
128static void gk20a_dbg_session_nvgpu_mutex_acquire(struct dbg_session_gk20a *dbg_s);
129
130static void gk20a_dbg_session_nvgpu_mutex_release(struct dbg_session_gk20a *dbg_s);
131
132static int nvgpu_profiler_reserve_release(struct dbg_session_gk20a *dbg_s,
133 u32 profiler_handle);
134
135static int dbg_unbind_all_channels_gk20a(struct dbg_session_gk20a *dbg_s);
136
137static int gk20a_dbg_gpu_do_dev_open(struct inode *inode,
138 struct file *filp, bool is_profiler);
139
140unsigned int gk20a_dbg_gpu_dev_poll(struct file *filep, poll_table *wait)
141{
142 unsigned int mask = 0;
143 struct dbg_session_gk20a_linux *dbg_session_linux = filep->private_data;
144 struct dbg_session_gk20a *dbg_s = &dbg_session_linux->dbg_s;
145 struct gk20a *g = dbg_s->g;
146
147 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
148
149 poll_wait(filep, &dbg_s->dbg_events.wait_queue.wq, wait);
150
151 gk20a_dbg_session_nvgpu_mutex_acquire(dbg_s);
152
153 if (dbg_s->dbg_events.events_enabled &&
154 dbg_s->dbg_events.num_pending_events > 0) {
155 nvgpu_log(g, gpu_dbg_gpu_dbg, "found pending event on session id %d",
156 dbg_s->id);
157 nvgpu_log(g, gpu_dbg_gpu_dbg, "%d events pending",
158 dbg_s->dbg_events.num_pending_events);
159 mask = (POLLPRI | POLLIN);
160 }
161
162 gk20a_dbg_session_nvgpu_mutex_release(dbg_s);
163
164 return mask;
165}
166
167int gk20a_dbg_gpu_dev_release(struct inode *inode, struct file *filp)
168{
169 struct dbg_session_gk20a_linux *dbg_session_linux = filp->private_data;
170 struct dbg_session_gk20a *dbg_s = &dbg_session_linux->dbg_s;
171 struct gk20a *g = dbg_s->g;
172 struct dbg_profiler_object_data *prof_obj, *tmp_obj;
173
174 nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, "%s", g->name);
175
176 /* unbind channels */
177 dbg_unbind_all_channels_gk20a(dbg_s);
178
179 /* Powergate/Timeout enable is called here as possibility of dbg_session
180 * which called powergate/timeout disable ioctl, to be killed without
181 * calling powergate/timeout enable ioctl
182 */
183 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
184 g->ops.dbg_session_ops.dbg_set_powergate(dbg_s, false);
185 nvgpu_dbg_timeout_enable(dbg_s, NVGPU_DBG_GPU_IOCTL_TIMEOUT_ENABLE);
186
187 /* If this session owned the perf buffer, release it */
188 if (g->perfbuf.owner == dbg_s)
189 gk20a_perfbuf_release_locked(g, g->perfbuf.offset);
190
191 /* Per-context profiler objects were released when we called
192 * dbg_unbind_all_channels. We could still have global ones.
193 */
194 nvgpu_list_for_each_entry_safe(prof_obj, tmp_obj, &g->profiler_objects,
195 dbg_profiler_object_data, prof_obj_entry) {
196 if (prof_obj->session_id == dbg_s->id) {
197 if (prof_obj->has_reservation)
198 g->ops.dbg_session_ops.
199 release_profiler_reservation(dbg_s, prof_obj);
200 nvgpu_list_del(&prof_obj->prof_obj_entry);
201 nvgpu_kfree(g, prof_obj);
202 }
203 }
204 nvgpu_mutex_release(&g->dbg_sessions_lock);
205
206 nvgpu_mutex_destroy(&dbg_s->ch_list_lock);
207 nvgpu_mutex_destroy(&dbg_s->ioctl_lock);
208
209 nvgpu_kfree(g, dbg_session_linux);
210 gk20a_put(g);
211
212 return 0;
213}
214
215int gk20a_prof_gpu_dev_open(struct inode *inode, struct file *filp)
216{
217 struct nvgpu_os_linux *l = container_of(inode->i_cdev,
218 struct nvgpu_os_linux, prof.cdev);
219 struct gk20a *g = &l->g;
220
221 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
222 return gk20a_dbg_gpu_do_dev_open(inode, filp, true /* is profiler */);
223}
224
225static int nvgpu_dbg_gpu_ioctl_timeout(struct dbg_session_gk20a *dbg_s,
226 struct nvgpu_dbg_gpu_timeout_args *args)
227{
228 int err;
229 struct gk20a *g = dbg_s->g;
230
231 nvgpu_log(g, gpu_dbg_fn, "timeout enable/disable = %d", args->enable);
232
233 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
234 err = nvgpu_dbg_timeout_enable(dbg_s, args->enable);
235 nvgpu_mutex_release(&g->dbg_sessions_lock);
236
237 return err;
238}
239
240static int nvgpu_dbg_gpu_ioctl_write_single_sm_error_state(
241 struct dbg_session_gk20a *dbg_s,
242 struct nvgpu_dbg_gpu_write_single_sm_error_state_args *args)
243{
244 struct gk20a *g = dbg_s->g;
245 struct gr_gk20a *gr = &g->gr;
246 u32 sm_id;
247 struct channel_gk20a *ch;
248 struct nvgpu_dbg_gpu_sm_error_state_record sm_error_state_record;
249 struct nvgpu_gr_sm_error_state sm_error_state;
250 int err = 0;
251
252 /* Not currently supported in the virtual case */
253 if (g->is_virtual)
254 return -ENOSYS;
255
256 ch = nvgpu_dbg_gpu_get_session_channel(dbg_s);
257 if (!ch)
258 return -EINVAL;
259
260 sm_id = args->sm_id;
261 if (sm_id >= gr->no_of_sm)
262 return -EINVAL;
263
264 nvgpu_speculation_barrier();
265
266 if (args->sm_error_state_record_size > 0) {
267 size_t read_size = sizeof(sm_error_state_record);
268
269 if (read_size > args->sm_error_state_record_size)
270 read_size = args->sm_error_state_record_size;
271
272 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
273 err = copy_from_user(&sm_error_state_record,
274 (void __user *)(uintptr_t)
275 args->sm_error_state_record_mem,
276 read_size);
277 nvgpu_mutex_release(&g->dbg_sessions_lock);
278 if (err)
279 return -ENOMEM;
280 }
281
282 err = gk20a_busy(g);
283 if (err)
284 return err;
285
286 sm_error_state.hww_global_esr =
287 sm_error_state_record.hww_global_esr;
288 sm_error_state.hww_warp_esr =
289 sm_error_state_record.hww_warp_esr;
290 sm_error_state.hww_warp_esr_pc =
291 sm_error_state_record.hww_warp_esr_pc;
292 sm_error_state.hww_global_esr_report_mask =
293 sm_error_state_record.hww_global_esr_report_mask;
294 sm_error_state.hww_warp_esr_report_mask =
295 sm_error_state_record.hww_warp_esr_report_mask;
296
297 err = gr_gk20a_elpg_protected_call(g,
298 g->ops.gr.update_sm_error_state(g, ch,
299 sm_id, &sm_error_state));
300
301 gk20a_idle(g);
302
303 return err;
304}
305
306
307static int nvgpu_dbg_gpu_ioctl_read_single_sm_error_state(
308 struct dbg_session_gk20a *dbg_s,
309 struct nvgpu_dbg_gpu_read_single_sm_error_state_args *args)
310{
311 struct gk20a *g = dbg_s->g;
312 struct gr_gk20a *gr = &g->gr;
313 struct nvgpu_gr_sm_error_state *sm_error_state;
314 struct nvgpu_dbg_gpu_sm_error_state_record sm_error_state_record;
315 u32 sm_id;
316 int err = 0;
317
318 sm_id = args->sm_id;
319 if (sm_id >= gr->no_of_sm)
320 return -EINVAL;
321
322 nvgpu_speculation_barrier();
323
324 sm_error_state = gr->sm_error_states + sm_id;
325 sm_error_state_record.hww_global_esr =
326 sm_error_state->hww_global_esr;
327 sm_error_state_record.hww_warp_esr =
328 sm_error_state->hww_warp_esr;
329 sm_error_state_record.hww_warp_esr_pc =
330 sm_error_state->hww_warp_esr_pc;
331 sm_error_state_record.hww_global_esr_report_mask =
332 sm_error_state->hww_global_esr_report_mask;
333 sm_error_state_record.hww_warp_esr_report_mask =
334 sm_error_state->hww_warp_esr_report_mask;
335
336 if (args->sm_error_state_record_size > 0) {
337 size_t write_size = sizeof(*sm_error_state);
338
339 if (write_size > args->sm_error_state_record_size)
340 write_size = args->sm_error_state_record_size;
341
342 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
343 err = copy_to_user((void __user *)(uintptr_t)
344 args->sm_error_state_record_mem,
345 &sm_error_state_record,
346 write_size);
347 nvgpu_mutex_release(&g->dbg_sessions_lock);
348 if (err) {
349 nvgpu_err(g, "copy_to_user failed!");
350 return err;
351 }
352
353 args->sm_error_state_record_size = write_size;
354 }
355
356 return 0;
357}
358
359
360static int nvgpu_dbg_gpu_ioctl_set_next_stop_trigger_type(
361 struct dbg_session_gk20a *dbg_s,
362 struct nvgpu_dbg_gpu_set_next_stop_trigger_type_args *args)
363{
364 struct gk20a *g = dbg_s->g;
365
366 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
367
368 gk20a_dbg_session_nvgpu_mutex_acquire(dbg_s);
369
370 dbg_s->broadcast_stop_trigger = (args->broadcast != 0);
371
372 gk20a_dbg_session_nvgpu_mutex_release(dbg_s);
373
374 return 0;
375}
376
377static int nvgpu_dbg_timeout_enable(struct dbg_session_gk20a *dbg_s,
378 int timeout_mode)
379{
380 struct gk20a *g = dbg_s->g;
381 int err = 0;
382
383 nvgpu_log(g, gpu_dbg_gpu_dbg, "Timeouts mode requested : %d",
384 timeout_mode);
385
386 switch (timeout_mode) {
387 case NVGPU_DBG_GPU_IOCTL_TIMEOUT_ENABLE:
388 if (dbg_s->is_timeout_disabled == true)
389 nvgpu_atomic_dec(&g->timeouts_disabled_refcount);
390 dbg_s->is_timeout_disabled = false;
391 break;
392
393 case NVGPU_DBG_GPU_IOCTL_TIMEOUT_DISABLE:
394 if (dbg_s->is_timeout_disabled == false)
395 nvgpu_atomic_inc(&g->timeouts_disabled_refcount);
396 dbg_s->is_timeout_disabled = true;
397 break;
398
399 default:
400 nvgpu_err(g,
401 "unrecognized dbg gpu timeout mode : 0x%x",
402 timeout_mode);
403 err = -EINVAL;
404 break;
405 }
406
407 if (!err)
408 nvgpu_log(g, gpu_dbg_gpu_dbg, "dbg is timeout disabled %s, "
409 "timeouts disabled refcount %d",
410 dbg_s->is_timeout_disabled ? "true" : "false",
411 nvgpu_atomic_read(&g->timeouts_disabled_refcount));
412 return err;
413}
414
415static int gk20a_dbg_gpu_do_dev_open(struct inode *inode,
416 struct file *filp, bool is_profiler)
417{
418 struct nvgpu_os_linux *l;
419 struct dbg_session_gk20a_linux *dbg_session_linux;
420 struct dbg_session_gk20a *dbg_s;
421 struct gk20a *g;
422
423 struct device *dev;
424
425 int err;
426
427 if (!is_profiler)
428 l = container_of(inode->i_cdev,
429 struct nvgpu_os_linux, dbg.cdev);
430 else
431 l = container_of(inode->i_cdev,
432 struct nvgpu_os_linux, prof.cdev);
433 g = gk20a_get(&l->g);
434 if (!g)
435 return -ENODEV;
436
437 dev = dev_from_gk20a(g);
438
439 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "dbg session: %s", g->name);
440
441 err = alloc_session(g, &dbg_session_linux);
442 if (err)
443 goto free_ref;
444
445 dbg_s = &dbg_session_linux->dbg_s;
446
447 filp->private_data = dbg_session_linux;
448 dbg_session_linux->dev = dev;
449 dbg_s->g = g;
450 dbg_s->is_profiler = is_profiler;
451 dbg_s->is_pg_disabled = false;
452 dbg_s->is_timeout_disabled = false;
453
454 nvgpu_cond_init(&dbg_s->dbg_events.wait_queue);
455 nvgpu_init_list_node(&dbg_s->ch_list);
456 err = nvgpu_mutex_init(&dbg_s->ch_list_lock);
457 if (err)
458 goto err_free_session;
459 err = nvgpu_mutex_init(&dbg_s->ioctl_lock);
460 if (err)
461 goto err_destroy_lock;
462 dbg_s->dbg_events.events_enabled = false;
463 dbg_s->dbg_events.num_pending_events = 0;
464
465 return 0;
466
467err_destroy_lock:
468 nvgpu_mutex_destroy(&dbg_s->ch_list_lock);
469err_free_session:
470 nvgpu_kfree(g, dbg_session_linux);
471free_ref:
472 gk20a_put(g);
473 return err;
474}
475
476void nvgpu_dbg_session_post_event(struct dbg_session_gk20a *dbg_s)
477{
478 nvgpu_cond_broadcast_interruptible(&dbg_s->dbg_events.wait_queue);
479}
480
481static int dbg_unbind_single_channel_gk20a(struct dbg_session_gk20a *dbg_s,
482 struct dbg_session_channel_data *ch_data)
483{
484 struct gk20a *g = dbg_s->g;
485 int chid;
486 struct dbg_session_data *session_data;
487 struct dbg_profiler_object_data *prof_obj, *tmp_obj;
488 struct dbg_session_channel_data_linux *ch_data_linux;
489
490 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
491
492 chid = ch_data->chid;
493
494 /* If there's a profiler ctx reservation record associated with this
495 * session/channel pair, release it.
496 */
497 nvgpu_list_for_each_entry_safe(prof_obj, tmp_obj, &g->profiler_objects,
498 dbg_profiler_object_data, prof_obj_entry) {
499 if ((prof_obj->session_id == dbg_s->id) &&
500 (prof_obj->ch->chid == chid)) {
501 if (prof_obj->has_reservation) {
502 g->ops.dbg_session_ops.
503 release_profiler_reservation(dbg_s, prof_obj);
504 }
505 nvgpu_list_del(&prof_obj->prof_obj_entry);
506 nvgpu_kfree(g, prof_obj);
507 }
508 }
509
510 nvgpu_list_del(&ch_data->ch_entry);
511
512 session_data = ch_data->session_data;
513 nvgpu_list_del(&session_data->dbg_s_entry);
514 nvgpu_kfree(dbg_s->g, session_data);
515
516 ch_data_linux = container_of(ch_data, struct dbg_session_channel_data_linux,
517 ch_data);
518
519 fput(ch_data_linux->ch_f);
520 nvgpu_kfree(dbg_s->g, ch_data_linux);
521
522 return 0;
523}
524
525static int dbg_bind_channel_gk20a(struct dbg_session_gk20a *dbg_s,
526 struct nvgpu_dbg_gpu_bind_channel_args *args)
527{
528 struct file *f;
529 struct gk20a *g = dbg_s->g;
530 struct channel_gk20a *ch;
531 struct dbg_session_channel_data_linux *ch_data_linux;
532 struct dbg_session_data *session_data;
533 int err = 0;
534
535 nvgpu_log(g, gpu_dbg_fn|gpu_dbg_gpu_dbg, "%s fd=%d",
536 g->name, args->channel_fd);
537
538 /*
539 * Although gk20a_get_channel_from_file gives us a channel ref, need to
540 * hold a ref to the file during the session lifetime. See comment in
541 * struct dbg_session_channel_data.
542 */
543 f = fget(args->channel_fd);
544 if (!f)
545 return -ENODEV;
546
547 ch = gk20a_get_channel_from_file(args->channel_fd);
548 if (!ch) {
549 nvgpu_log_fn(g, "no channel found for fd");
550 err = -EINVAL;
551 goto out_fput;
552 }
553
554 nvgpu_log_fn(g, "%s hwchid=%d", g->name, ch->chid);
555
556 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
557 nvgpu_mutex_acquire(&ch->dbg_s_lock);
558
559 ch_data_linux = nvgpu_kzalloc(g, sizeof(*ch_data_linux));
560 if (!ch_data_linux) {
561 err = -ENOMEM;
562 goto out_chput;
563 }
564 ch_data_linux->ch_f = f;
565 ch_data_linux->ch_data.channel_fd = args->channel_fd;
566 ch_data_linux->ch_data.chid = ch->chid;
567 ch_data_linux->ch_data.unbind_single_channel = dbg_unbind_single_channel_gk20a;
568 nvgpu_init_list_node(&ch_data_linux->ch_data.ch_entry);
569
570 session_data = nvgpu_kzalloc(g, sizeof(*session_data));
571 if (!session_data) {
572 err = -ENOMEM;
573 goto out_kfree;
574 }
575 session_data->dbg_s = dbg_s;
576 nvgpu_init_list_node(&session_data->dbg_s_entry);
577 ch_data_linux->ch_data.session_data = session_data;
578
579 nvgpu_list_add(&session_data->dbg_s_entry, &ch->dbg_s_list);
580
581 nvgpu_mutex_acquire(&dbg_s->ch_list_lock);
582 nvgpu_list_add_tail(&ch_data_linux->ch_data.ch_entry, &dbg_s->ch_list);
583 nvgpu_mutex_release(&dbg_s->ch_list_lock);
584
585 nvgpu_mutex_release(&ch->dbg_s_lock);
586 nvgpu_mutex_release(&g->dbg_sessions_lock);
587
588 gk20a_channel_put(ch);
589
590 return 0;
591
592out_kfree:
593 nvgpu_kfree(g, ch_data_linux);
594out_chput:
595 gk20a_channel_put(ch);
596 nvgpu_mutex_release(&ch->dbg_s_lock);
597 nvgpu_mutex_release(&g->dbg_sessions_lock);
598out_fput:
599 fput(f);
600 return err;
601}
602
603static int dbg_unbind_all_channels_gk20a(struct dbg_session_gk20a *dbg_s)
604{
605 struct dbg_session_channel_data *ch_data, *tmp;
606 struct gk20a *g = dbg_s->g;
607
608 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
609 nvgpu_mutex_acquire(&dbg_s->ch_list_lock);
610 nvgpu_list_for_each_entry_safe(ch_data, tmp, &dbg_s->ch_list,
611 dbg_session_channel_data, ch_entry)
612 ch_data->unbind_single_channel(dbg_s, ch_data);
613 nvgpu_mutex_release(&dbg_s->ch_list_lock);
614 nvgpu_mutex_release(&g->dbg_sessions_lock);
615
616 return 0;
617}
618
619/*
620 * Convert common regops op values of the form of NVGPU_DBG_REG_OP_*
621 * into linux regops op values of the form of NVGPU_DBG_GPU_REG_OP_*
622 */
623static u32 nvgpu_get_regops_op_values_linux(u32 regops_op)
624{
625 switch (regops_op) {
626 case REGOP(READ_32):
627 return REGOP_LINUX(READ_32);
628 case REGOP(WRITE_32):
629 return REGOP_LINUX(WRITE_32);
630 case REGOP(READ_64):
631 return REGOP_LINUX(READ_64);
632 case REGOP(WRITE_64):
633 return REGOP_LINUX(WRITE_64);
634 case REGOP(READ_08):
635 return REGOP_LINUX(READ_08);
636 case REGOP(WRITE_08):
637 return REGOP_LINUX(WRITE_08);
638 }
639
640 return regops_op;
641}
642
643/*
644 * Convert linux regops op values of the form of NVGPU_DBG_GPU_REG_OP_*
645 * into common regops op values of the form of NVGPU_DBG_REG_OP_*
646 */
647static u32 nvgpu_get_regops_op_values_common(u32 regops_op)
648{
649 switch (regops_op) {
650 case REGOP_LINUX(READ_32):
651 return REGOP(READ_32);
652 case REGOP_LINUX(WRITE_32):
653 return REGOP(WRITE_32);
654 case REGOP_LINUX(READ_64):
655 return REGOP(READ_64);
656 case REGOP_LINUX(WRITE_64):
657 return REGOP(WRITE_64);
658 case REGOP_LINUX(READ_08):
659 return REGOP(READ_08);
660 case REGOP_LINUX(WRITE_08):
661 return REGOP(WRITE_08);
662 }
663
664 return regops_op;
665}
666
667/*
668 * Convert common regops type values of the form of NVGPU_DBG_REG_OP_TYPE_*
669 * into linux regops type values of the form of NVGPU_DBG_GPU_REG_OP_TYPE_*
670 */
671static u32 nvgpu_get_regops_type_values_linux(u32 regops_type)
672{
673 switch (regops_type) {
674 case REGOP(TYPE_GLOBAL):
675 return REGOP_LINUX(TYPE_GLOBAL);
676 case REGOP(TYPE_GR_CTX):
677 return REGOP_LINUX(TYPE_GR_CTX);
678 case REGOP(TYPE_GR_CTX_TPC):
679 return REGOP_LINUX(TYPE_GR_CTX_TPC);
680 case REGOP(TYPE_GR_CTX_SM):
681 return REGOP_LINUX(TYPE_GR_CTX_SM);
682 case REGOP(TYPE_GR_CTX_CROP):
683 return REGOP_LINUX(TYPE_GR_CTX_CROP);
684 case REGOP(TYPE_GR_CTX_ZROP):
685 return REGOP_LINUX(TYPE_GR_CTX_ZROP);
686 case REGOP(TYPE_GR_CTX_QUAD):
687 return REGOP_LINUX(TYPE_GR_CTX_QUAD);
688 }
689
690 return regops_type;
691}
692
693/*
694 * Convert linux regops type values of the form of NVGPU_DBG_GPU_REG_OP_TYPE_*
695 * into common regops type values of the form of NVGPU_DBG_REG_OP_TYPE_*
696 */
697static u32 nvgpu_get_regops_type_values_common(u32 regops_type)
698{
699 switch (regops_type) {
700 case REGOP_LINUX(TYPE_GLOBAL):
701 return REGOP(TYPE_GLOBAL);
702 case REGOP_LINUX(TYPE_GR_CTX):
703 return REGOP(TYPE_GR_CTX);
704 case REGOP_LINUX(TYPE_GR_CTX_TPC):
705 return REGOP(TYPE_GR_CTX_TPC);
706 case REGOP_LINUX(TYPE_GR_CTX_SM):
707 return REGOP(TYPE_GR_CTX_SM);
708 case REGOP_LINUX(TYPE_GR_CTX_CROP):
709 return REGOP(TYPE_GR_CTX_CROP);
710 case REGOP_LINUX(TYPE_GR_CTX_ZROP):
711 return REGOP(TYPE_GR_CTX_ZROP);
712 case REGOP_LINUX(TYPE_GR_CTX_QUAD):
713 return REGOP(TYPE_GR_CTX_QUAD);
714 }
715
716 return regops_type;
717}
718
719/*
720 * Convert common regops status values of the form of NVGPU_DBG_REG_OP_STATUS_*
721 * into linux regops type values of the form of NVGPU_DBG_GPU_REG_OP_STATUS_*
722 */
723static u32 nvgpu_get_regops_status_values_linux(u32 regops_status)
724{
725 switch (regops_status) {
726 case REGOP(STATUS_SUCCESS):
727 return REGOP_LINUX(STATUS_SUCCESS);
728 case REGOP(STATUS_INVALID_OP):
729 return REGOP_LINUX(STATUS_INVALID_OP);
730 case REGOP(STATUS_INVALID_TYPE):
731 return REGOP_LINUX(STATUS_INVALID_TYPE);
732 case REGOP(STATUS_INVALID_OFFSET):
733 return REGOP_LINUX(STATUS_INVALID_OFFSET);
734 case REGOP(STATUS_UNSUPPORTED_OP):
735 return REGOP_LINUX(STATUS_UNSUPPORTED_OP);
736 case REGOP(STATUS_INVALID_MASK ):
737 return REGOP_LINUX(STATUS_INVALID_MASK);
738 }
739
740 return regops_status;
741}
742
743/*
744 * Convert linux regops status values of the form of NVGPU_DBG_GPU_REG_OP_STATUS_*
745 * into common regops type values of the form of NVGPU_DBG_REG_OP_STATUS_*
746 */
747static u32 nvgpu_get_regops_status_values_common(u32 regops_status)
748{
749 switch (regops_status) {
750 case REGOP_LINUX(STATUS_SUCCESS):
751 return REGOP(STATUS_SUCCESS);
752 case REGOP_LINUX(STATUS_INVALID_OP):
753 return REGOP(STATUS_INVALID_OP);
754 case REGOP_LINUX(STATUS_INVALID_TYPE):
755 return REGOP(STATUS_INVALID_TYPE);
756 case REGOP_LINUX(STATUS_INVALID_OFFSET):
757 return REGOP(STATUS_INVALID_OFFSET);
758 case REGOP_LINUX(STATUS_UNSUPPORTED_OP):
759 return REGOP(STATUS_UNSUPPORTED_OP);
760 case REGOP_LINUX(STATUS_INVALID_MASK ):
761 return REGOP(STATUS_INVALID_MASK);
762 }
763
764 return regops_status;
765}
766
767static int nvgpu_get_regops_data_common(struct nvgpu_dbg_gpu_reg_op *in,
768 struct nvgpu_dbg_reg_op *out, u32 num_ops)
769{
770 u32 i;
771
772 if(in == NULL || out == NULL)
773 return -ENOMEM;
774
775 for (i = 0; i < num_ops; i++) {
776 out[i].op = nvgpu_get_regops_op_values_common(in[i].op);
777 out[i].type = nvgpu_get_regops_type_values_common(in[i].type);
778 out[i].status = nvgpu_get_regops_status_values_common(in[i].status);
779 out[i].quad = in[i].quad;
780 out[i].group_mask = in[i].group_mask;
781 out[i].sub_group_mask = in[i].sub_group_mask;
782 out[i].offset = in[i].offset;
783 out[i].value_lo = in[i].value_lo;
784 out[i].value_hi = in[i].value_hi;
785 out[i].and_n_mask_lo = in[i].and_n_mask_lo;
786 out[i].and_n_mask_hi = in[i].and_n_mask_hi;
787 }
788
789 return 0;
790}
791
792static int nvgpu_get_regops_data_linux(struct nvgpu_dbg_reg_op *in,
793 struct nvgpu_dbg_gpu_reg_op *out, u32 num_ops)
794{
795 u32 i;
796
797 if(in == NULL || out == NULL)
798 return -ENOMEM;
799
800 for (i = 0; i < num_ops; i++) {
801 out[i].op = nvgpu_get_regops_op_values_linux(in[i].op);
802 out[i].type = nvgpu_get_regops_type_values_linux(in[i].type);
803 out[i].status = nvgpu_get_regops_status_values_linux(in[i].status);
804 out[i].quad = in[i].quad;
805 out[i].group_mask = in[i].group_mask;
806 out[i].sub_group_mask = in[i].sub_group_mask;
807 out[i].offset = in[i].offset;
808 out[i].value_lo = in[i].value_lo;
809 out[i].value_hi = in[i].value_hi;
810 out[i].and_n_mask_lo = in[i].and_n_mask_lo;
811 out[i].and_n_mask_hi = in[i].and_n_mask_hi;
812 }
813
814 return 0;
815}
816
817static int nvgpu_ioctl_channel_reg_ops(struct dbg_session_gk20a *dbg_s,
818 struct nvgpu_dbg_gpu_exec_reg_ops_args *args)
819{
820 int err = 0, powergate_err = 0;
821 bool is_pg_disabled = false;
822
823 struct gk20a *g = dbg_s->g;
824 struct channel_gk20a *ch;
825
826 nvgpu_log_fn(g, "%d ops, max fragment %d", args->num_ops, g->dbg_regops_tmp_buf_ops);
827
828 if (args->num_ops > NVGPU_IOCTL_DBG_REG_OPS_LIMIT) {
829 nvgpu_err(g, "regops limit exceeded");
830 return -EINVAL;
831 }
832
833 if (args->num_ops == 0) {
834 /* Nothing to do */
835 return 0;
836 }
837
838 if (g->dbg_regops_tmp_buf_ops == 0 || !g->dbg_regops_tmp_buf) {
839 nvgpu_err(g, "reg ops work buffer not allocated");
840 return -ENODEV;
841 }
842
843 if (!dbg_s->id) {
844 nvgpu_err(g, "can't call reg_ops on an unbound debugger session");
845 return -EINVAL;
846 }
847
848 ch = nvgpu_dbg_gpu_get_session_channel(dbg_s);
849 if (!dbg_s->is_profiler && !ch) {
850 nvgpu_err(g, "bind a channel before regops for a debugging session");
851 return -EINVAL;
852 }
853
854 /* be sure that ctx info is in place */
855 if (!g->is_virtual &&
856 !gr_context_info_available(dbg_s, &g->gr)) {
857 nvgpu_err(g, "gr context data not available");
858 return -ENODEV;
859 }
860
861 /* since exec_reg_ops sends methods to the ucode, it must take the
862 * global gpu lock to protect against mixing methods from debug sessions
863 * on other channels */
864 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
865
866 if (!dbg_s->is_pg_disabled && !g->is_virtual) {
867 /* In the virtual case, the server will handle
868 * disabling/enabling powergating when processing reg ops
869 */
870 powergate_err = g->ops.dbg_session_ops.dbg_set_powergate(dbg_s,
871 true);
872 is_pg_disabled = true;
873 }
874
875 if (!powergate_err) {
876 u64 ops_offset = 0; /* index offset */
877
878 struct nvgpu_dbg_gpu_reg_op *linux_fragment = NULL;
879
880 linux_fragment = nvgpu_kzalloc(g, g->dbg_regops_tmp_buf_ops *
881 sizeof(struct nvgpu_dbg_gpu_reg_op));
882
883 if (!linux_fragment)
884 return -ENOMEM;
885
886 while (ops_offset < args->num_ops && !err) {
887 const u64 num_ops =
888 min(args->num_ops - ops_offset,
889 (u64)(g->dbg_regops_tmp_buf_ops));
890 const u64 fragment_size =
891 num_ops * sizeof(struct nvgpu_dbg_gpu_reg_op);
892
893 void __user *const fragment =
894 (void __user *)(uintptr_t)
895 (args->ops +
896 ops_offset * sizeof(struct nvgpu_dbg_gpu_reg_op));
897
898 nvgpu_log_fn(g, "Regops fragment: start_op=%llu ops=%llu",
899 ops_offset, num_ops);
900
901 nvgpu_log_fn(g, "Copying regops from userspace");
902
903 if (copy_from_user(linux_fragment,
904 fragment, fragment_size)) {
905 nvgpu_err(g, "copy_from_user failed!");
906 err = -EFAULT;
907 break;
908 }
909
910 err = nvgpu_get_regops_data_common(linux_fragment,
911 g->dbg_regops_tmp_buf, num_ops);
912
913 if (err)
914 break;
915
916 err = g->ops.dbg_session_ops.exec_reg_ops(
917 dbg_s, g->dbg_regops_tmp_buf, num_ops);
918
919 err = nvgpu_get_regops_data_linux(g->dbg_regops_tmp_buf,
920 linux_fragment, num_ops);
921
922 if (err)
923 break;
924
925 nvgpu_log_fn(g, "Copying result to userspace");
926
927 if (copy_to_user(fragment, linux_fragment,
928 fragment_size)) {
929 nvgpu_err(g, "copy_to_user failed!");
930 err = -EFAULT;
931 break;
932 }
933
934 ops_offset += num_ops;
935 }
936
937 nvgpu_kfree(g, linux_fragment);
938
939 /* enable powergate, if previously disabled */
940 if (is_pg_disabled) {
941 powergate_err =
942 g->ops.dbg_session_ops.dbg_set_powergate(dbg_s,
943 false);
944 }
945 }
946
947 nvgpu_mutex_release(&g->dbg_sessions_lock);
948
949 if (!err && powergate_err)
950 err = powergate_err;
951
952 if (err)
953 nvgpu_err(g, "dbg regops failed");
954
955 return err;
956}
957
958static int nvgpu_ioctl_powergate_gk20a(struct dbg_session_gk20a *dbg_s,
959 struct nvgpu_dbg_gpu_powergate_args *args)
960{
961 int err;
962 struct gk20a *g = dbg_s->g;
963 nvgpu_log_fn(g, "%s powergate mode = %d",
964 g->name, args->mode);
965
966 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
967 if (args->mode == NVGPU_DBG_GPU_POWERGATE_MODE_DISABLE) {
968 err = g->ops.dbg_session_ops.dbg_set_powergate(dbg_s, true);
969 } else if (args->mode == NVGPU_DBG_GPU_POWERGATE_MODE_ENABLE) {
970 err = g->ops.dbg_session_ops.dbg_set_powergate(dbg_s, false);
971 } else {
972 nvgpu_err(g, "invalid powergate mode");
973 err = -EINVAL;
974 }
975 nvgpu_mutex_release(&g->dbg_sessions_lock);
976 return err;
977}
978
979static int nvgpu_dbg_gpu_ioctl_smpc_ctxsw_mode(struct dbg_session_gk20a *dbg_s,
980 struct nvgpu_dbg_gpu_smpc_ctxsw_mode_args *args)
981{
982 int err;
983 struct gk20a *g = dbg_s->g;
984 struct channel_gk20a *ch_gk20a;
985
986 nvgpu_log_fn(g, "%s smpc ctxsw mode = %d",
987 g->name, args->mode);
988
989 err = gk20a_busy(g);
990 if (err) {
991 nvgpu_err(g, "failed to poweron");
992 return err;
993 }
994
995 /* Take the global lock, since we'll be doing global regops */
996 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
997
998 ch_gk20a = nvgpu_dbg_gpu_get_session_channel(dbg_s);
999 if (!ch_gk20a) {
1000 nvgpu_err(g,
1001 "no bound channel for smpc ctxsw mode update");
1002 err = -EINVAL;
1003 goto clean_up;
1004 }
1005
1006 err = g->ops.gr.update_smpc_ctxsw_mode(g, ch_gk20a,
1007 args->mode == NVGPU_DBG_GPU_SMPC_CTXSW_MODE_CTXSW);
1008 if (err) {
1009 nvgpu_err(g,
1010 "error (%d) during smpc ctxsw mode update", err);
1011 goto clean_up;
1012 }
1013
1014 err = g->ops.regops.apply_smpc_war(dbg_s);
1015 clean_up:
1016 nvgpu_mutex_release(&g->dbg_sessions_lock);
1017 gk20a_idle(g);
1018 return err;
1019}
1020
1021static int nvgpu_dbg_gpu_ioctl_hwpm_ctxsw_mode(struct dbg_session_gk20a *dbg_s,
1022 struct nvgpu_dbg_gpu_hwpm_ctxsw_mode_args *args)
1023{
1024 int err;
1025 struct gk20a *g = dbg_s->g;
1026 struct channel_gk20a *ch_gk20a;
1027
1028 nvgpu_log_fn(g, "%s pm ctxsw mode = %d", g->name, args->mode);
1029
1030 /* Must have a valid reservation to enable/disable hwpm cxtsw.
1031 * Just print an error message for now, but eventually this should
1032 * return an error, at the point where all client sw has been
1033 * cleaned up.
1034 */
1035 if (!dbg_s->has_profiler_reservation) {
1036 nvgpu_err(g,
1037 "session doesn't have a valid reservation");
1038 }
1039
1040 err = gk20a_busy(g);
1041 if (err) {
1042 nvgpu_err(g, "failed to poweron");
1043 return err;
1044 }
1045
1046 /* Take the global lock, since we'll be doing global regops */
1047 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1048
1049 ch_gk20a = nvgpu_dbg_gpu_get_session_channel(dbg_s);
1050 if (!ch_gk20a) {
1051 nvgpu_err(g,
1052 "no bound channel for pm ctxsw mode update");
1053 err = -EINVAL;
1054 goto clean_up;
1055 }
1056 if (!dbg_s->is_pg_disabled) {
1057 nvgpu_err(g, "powergate is not disabled");
1058 err = -ENOSYS;
1059 goto clean_up;
1060 }
1061 err = g->ops.gr.update_hwpm_ctxsw_mode(g, ch_gk20a, 0,
1062 args->mode == NVGPU_DBG_GPU_HWPM_CTXSW_MODE_CTXSW);
1063 if (err)
1064 nvgpu_err(g,
1065 "error (%d) during pm ctxsw mode update", err);
1066 /* gk20a would require a WAR to set the core PM_ENABLE bit, not
1067 * added here with gk20a being deprecated
1068 */
1069 clean_up:
1070 nvgpu_mutex_release(&g->dbg_sessions_lock);
1071 gk20a_idle(g);
1072 return err;
1073}
1074
1075static int nvgpu_dbg_gpu_ioctl_suspend_resume_sm(
1076 struct dbg_session_gk20a *dbg_s,
1077 struct nvgpu_dbg_gpu_suspend_resume_all_sms_args *args)
1078{
1079 struct gk20a *g = dbg_s->g;
1080 struct channel_gk20a *ch;
1081 int err = 0, action = args->mode;
1082
1083 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "action: %d", args->mode);
1084
1085 ch = nvgpu_dbg_gpu_get_session_channel(dbg_s);
1086 if (!ch)
1087 return -EINVAL;
1088
1089 err = gk20a_busy(g);
1090 if (err) {
1091 nvgpu_err(g, "failed to poweron");
1092 return err;
1093 }
1094
1095 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1096
1097 /* Suspend GPU context switching */
1098 err = gr_gk20a_disable_ctxsw(g);
1099 if (err) {
1100 nvgpu_err(g, "unable to stop gr ctxsw");
1101 /* this should probably be ctx-fatal... */
1102 goto clean_up;
1103 }
1104
1105 switch (action) {
1106 case NVGPU_DBG_GPU_SUSPEND_ALL_SMS:
1107 gr_gk20a_suspend_context(ch);
1108 break;
1109
1110 case NVGPU_DBG_GPU_RESUME_ALL_SMS:
1111 gr_gk20a_resume_context(ch);
1112 break;
1113 }
1114
1115 err = gr_gk20a_enable_ctxsw(g);
1116 if (err)
1117 nvgpu_err(g, "unable to restart ctxsw!");
1118
1119clean_up:
1120 nvgpu_mutex_release(&g->dbg_sessions_lock);
1121 gk20a_idle(g);
1122
1123 return err;
1124}
1125
1126static int nvgpu_ioctl_allocate_profiler_object(
1127 struct dbg_session_gk20a_linux *dbg_session_linux,
1128 struct nvgpu_dbg_gpu_profiler_obj_mgt_args *args)
1129{
1130 int err = 0;
1131 struct dbg_session_gk20a *dbg_s = &dbg_session_linux->dbg_s;
1132 struct gk20a *g = get_gk20a(dbg_session_linux->dev);
1133 struct dbg_profiler_object_data *prof_obj;
1134
1135 nvgpu_log_fn(g, "%s", g->name);
1136
1137 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1138
1139 err = alloc_profiler(g, &prof_obj);
1140 if (err)
1141 goto clean_up;
1142
1143 prof_obj->session_id = dbg_s->id;
1144
1145 if (dbg_s->is_profiler)
1146 prof_obj->ch = NULL;
1147 else {
1148 prof_obj->ch = nvgpu_dbg_gpu_get_session_channel(dbg_s);
1149 if (prof_obj->ch == NULL) {
1150 nvgpu_err(g,
1151 "bind a channel for dbg session");
1152 nvgpu_kfree(g, prof_obj);
1153 err = -EINVAL;
1154 goto clean_up;
1155 }
1156 }
1157
1158 /* Return handle to client */
1159 args->profiler_handle = prof_obj->prof_handle;
1160
1161 nvgpu_init_list_node(&prof_obj->prof_obj_entry);
1162
1163 nvgpu_list_add(&prof_obj->prof_obj_entry, &g->profiler_objects);
1164clean_up:
1165 nvgpu_mutex_release(&g->dbg_sessions_lock);
1166 return err;
1167}
1168
1169static int nvgpu_ioctl_free_profiler_object(
1170 struct dbg_session_gk20a_linux *dbg_s_linux,
1171 struct nvgpu_dbg_gpu_profiler_obj_mgt_args *args)
1172{
1173 int err = 0;
1174 struct dbg_session_gk20a *dbg_s = &dbg_s_linux->dbg_s;
1175 struct gk20a *g = get_gk20a(dbg_s_linux->dev);
1176 struct dbg_profiler_object_data *prof_obj, *tmp_obj;
1177 bool obj_found = false;
1178
1179 nvgpu_log_fn(g, "%s session_id = %d profiler_handle = %x",
1180 g->name, dbg_s->id, args->profiler_handle);
1181
1182 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1183
1184 /* Remove profiler object from the list, if a match is found */
1185 nvgpu_list_for_each_entry_safe(prof_obj, tmp_obj, &g->profiler_objects,
1186 dbg_profiler_object_data, prof_obj_entry) {
1187 if (prof_obj->prof_handle == args->profiler_handle) {
1188 if (prof_obj->session_id != dbg_s->id) {
1189 nvgpu_err(g,
1190 "invalid handle %x",
1191 args->profiler_handle);
1192 err = -EINVAL;
1193 break;
1194 }
1195 if (prof_obj->has_reservation)
1196 g->ops.dbg_session_ops.
1197 release_profiler_reservation(dbg_s, prof_obj);
1198 nvgpu_list_del(&prof_obj->prof_obj_entry);
1199 nvgpu_kfree(g, prof_obj);
1200 obj_found = true;
1201 break;
1202 }
1203 }
1204 if (!obj_found) {
1205 nvgpu_err(g, "profiler %x not found",
1206 args->profiler_handle);
1207 err = -EINVAL;
1208 }
1209
1210 nvgpu_mutex_release(&g->dbg_sessions_lock);
1211 return err;
1212}
1213
1214static struct dbg_profiler_object_data *find_matching_prof_obj(
1215 struct dbg_session_gk20a *dbg_s,
1216 u32 profiler_handle)
1217{
1218 struct gk20a *g = dbg_s->g;
1219 struct dbg_profiler_object_data *prof_obj;
1220
1221 nvgpu_list_for_each_entry(prof_obj, &g->profiler_objects,
1222 dbg_profiler_object_data, prof_obj_entry) {
1223 if (prof_obj->prof_handle == profiler_handle) {
1224 if (prof_obj->session_id != dbg_s->id) {
1225 nvgpu_err(g,
1226 "invalid handle %x",
1227 profiler_handle);
1228 return NULL;
1229 }
1230 return prof_obj;
1231 }
1232 }
1233 return NULL;
1234}
1235
1236/* used in scenarios where the debugger session can take just the inter-session
1237 * lock for performance, but the profiler session must take the per-gpu lock
1238 * since it might not have an associated channel. */
1239static void gk20a_dbg_session_nvgpu_mutex_acquire(struct dbg_session_gk20a *dbg_s)
1240{
1241 struct channel_gk20a *ch = nvgpu_dbg_gpu_get_session_channel(dbg_s);
1242
1243 if (dbg_s->is_profiler || !ch)
1244 nvgpu_mutex_acquire(&dbg_s->g->dbg_sessions_lock);
1245 else
1246 nvgpu_mutex_acquire(&ch->dbg_s_lock);
1247}
1248
1249static void gk20a_dbg_session_nvgpu_mutex_release(struct dbg_session_gk20a *dbg_s)
1250{
1251 struct channel_gk20a *ch = nvgpu_dbg_gpu_get_session_channel(dbg_s);
1252
1253 if (dbg_s->is_profiler || !ch)
1254 nvgpu_mutex_release(&dbg_s->g->dbg_sessions_lock);
1255 else
1256 nvgpu_mutex_release(&ch->dbg_s_lock);
1257}
1258
1259static void gk20a_dbg_gpu_events_enable(struct dbg_session_gk20a *dbg_s)
1260{
1261 struct gk20a *g = dbg_s->g;
1262
1263 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
1264
1265 gk20a_dbg_session_nvgpu_mutex_acquire(dbg_s);
1266
1267 dbg_s->dbg_events.events_enabled = true;
1268 dbg_s->dbg_events.num_pending_events = 0;
1269
1270 gk20a_dbg_session_nvgpu_mutex_release(dbg_s);
1271}
1272
1273static void gk20a_dbg_gpu_events_disable(struct dbg_session_gk20a *dbg_s)
1274{
1275 struct gk20a *g = dbg_s->g;
1276
1277 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
1278
1279 gk20a_dbg_session_nvgpu_mutex_acquire(dbg_s);
1280
1281 dbg_s->dbg_events.events_enabled = false;
1282 dbg_s->dbg_events.num_pending_events = 0;
1283
1284 gk20a_dbg_session_nvgpu_mutex_release(dbg_s);
1285}
1286
1287static void gk20a_dbg_gpu_events_clear(struct dbg_session_gk20a *dbg_s)
1288{
1289 struct gk20a *g = dbg_s->g;
1290
1291 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
1292
1293 gk20a_dbg_session_nvgpu_mutex_acquire(dbg_s);
1294
1295 if (dbg_s->dbg_events.events_enabled &&
1296 dbg_s->dbg_events.num_pending_events > 0)
1297 dbg_s->dbg_events.num_pending_events--;
1298
1299 gk20a_dbg_session_nvgpu_mutex_release(dbg_s);
1300}
1301
1302
1303static int gk20a_dbg_gpu_events_ctrl(struct dbg_session_gk20a *dbg_s,
1304 struct nvgpu_dbg_gpu_events_ctrl_args *args)
1305{
1306 int ret = 0;
1307 struct channel_gk20a *ch;
1308 struct gk20a *g = dbg_s->g;
1309
1310 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "dbg events ctrl cmd %d", args->cmd);
1311
1312 ch = nvgpu_dbg_gpu_get_session_channel(dbg_s);
1313 if (!ch) {
1314 nvgpu_err(g, "no channel bound to dbg session");
1315 return -EINVAL;
1316 }
1317
1318 switch (args->cmd) {
1319 case NVGPU_DBG_GPU_EVENTS_CTRL_CMD_ENABLE:
1320 gk20a_dbg_gpu_events_enable(dbg_s);
1321 break;
1322
1323 case NVGPU_DBG_GPU_EVENTS_CTRL_CMD_DISABLE:
1324 gk20a_dbg_gpu_events_disable(dbg_s);
1325 break;
1326
1327 case NVGPU_DBG_GPU_EVENTS_CTRL_CMD_CLEAR:
1328 gk20a_dbg_gpu_events_clear(dbg_s);
1329 break;
1330
1331 default:
1332 nvgpu_err(g, "unrecognized dbg gpu events ctrl cmd: 0x%x",
1333 args->cmd);
1334 ret = -EINVAL;
1335 break;
1336 }
1337
1338 return ret;
1339}
1340
1341static int gk20a_perfbuf_map(struct dbg_session_gk20a *dbg_s,
1342 struct nvgpu_dbg_gpu_perfbuf_map_args *args)
1343{
1344 struct gk20a *g = dbg_s->g;
1345 struct mm_gk20a *mm = &g->mm;
1346 int err;
1347 u32 virt_size;
1348 u32 big_page_size = g->ops.mm.get_default_big_page_size();
1349
1350 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1351
1352 if (g->perfbuf.owner) {
1353 nvgpu_mutex_release(&g->dbg_sessions_lock);
1354 return -EBUSY;
1355 }
1356
1357 mm->perfbuf.vm = nvgpu_vm_init(g, big_page_size,
1358 big_page_size << 10,
1359 NV_MM_DEFAULT_KERNEL_SIZE,
1360 NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE,
1361 false, false, "perfbuf");
1362 if (!mm->perfbuf.vm) {
1363 nvgpu_mutex_release(&g->dbg_sessions_lock);
1364 return -ENOMEM;
1365 }
1366
1367 err = nvgpu_vm_map_buffer(mm->perfbuf.vm,
1368 args->dmabuf_fd,
1369 &args->offset,
1370 0,
1371 0,
1372 0,
1373 0,
1374 args->mapping_size,
1375 NULL);
1376 if (err)
1377 goto err_remove_vm;
1378
1379 /* perf output buffer may not cross a 4GB boundary */
1380 virt_size = u64_lo32(args->mapping_size);
1381 if (u64_hi32(args->offset) != u64_hi32(args->offset + virt_size)) {
1382 err = -EINVAL;
1383 goto err_unmap;
1384 }
1385
1386 err = g->ops.dbg_session_ops.perfbuffer_enable(g,
1387 args->offset, virt_size);
1388 if (err)
1389 goto err_unmap;
1390
1391 g->perfbuf.owner = dbg_s;
1392 g->perfbuf.offset = args->offset;
1393 nvgpu_mutex_release(&g->dbg_sessions_lock);
1394
1395 return 0;
1396
1397err_unmap:
1398 nvgpu_vm_unmap(mm->perfbuf.vm, args->offset, NULL);
1399err_remove_vm:
1400 nvgpu_vm_put(mm->perfbuf.vm);
1401 nvgpu_mutex_release(&g->dbg_sessions_lock);
1402 return err;
1403}
1404
1405static int gk20a_perfbuf_unmap(struct dbg_session_gk20a *dbg_s,
1406 struct nvgpu_dbg_gpu_perfbuf_unmap_args *args)
1407{
1408 struct gk20a *g = dbg_s->g;
1409 int err;
1410
1411 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1412 if ((g->perfbuf.owner != dbg_s) ||
1413 (g->perfbuf.offset != args->offset)) {
1414 nvgpu_mutex_release(&g->dbg_sessions_lock);
1415 return -EINVAL;
1416 }
1417
1418 err = gk20a_perfbuf_release_locked(g, args->offset);
1419
1420 nvgpu_mutex_release(&g->dbg_sessions_lock);
1421
1422 return err;
1423}
1424
1425static int gk20a_dbg_pc_sampling(struct dbg_session_gk20a *dbg_s,
1426 struct nvgpu_dbg_gpu_pc_sampling_args *args)
1427{
1428 struct channel_gk20a *ch;
1429 struct gk20a *g = dbg_s->g;
1430
1431 ch = nvgpu_dbg_gpu_get_session_channel(dbg_s);
1432 if (!ch)
1433 return -EINVAL;
1434
1435 nvgpu_log_fn(g, " ");
1436
1437 return g->ops.gr.update_pc_sampling ?
1438 g->ops.gr.update_pc_sampling(ch, args->enable) : -EINVAL;
1439}
1440
1441static int nvgpu_dbg_gpu_ioctl_clear_single_sm_error_state(
1442 struct dbg_session_gk20a *dbg_s,
1443 struct nvgpu_dbg_gpu_clear_single_sm_error_state_args *args)
1444{
1445 struct gk20a *g = dbg_s->g;
1446 struct gr_gk20a *gr = &g->gr;
1447 u32 sm_id;
1448 struct channel_gk20a *ch;
1449 int err = 0;
1450
1451 ch = nvgpu_dbg_gpu_get_session_channel(dbg_s);
1452 if (!ch)
1453 return -EINVAL;
1454
1455 sm_id = args->sm_id;
1456 if (sm_id >= gr->no_of_sm)
1457 return -EINVAL;
1458
1459 nvgpu_speculation_barrier();
1460
1461 err = gk20a_busy(g);
1462 if (err)
1463 return err;
1464
1465 err = gr_gk20a_elpg_protected_call(g,
1466 g->ops.gr.clear_sm_error_state(g, ch, sm_id));
1467
1468 gk20a_idle(g);
1469
1470 return err;
1471}
1472
1473static int
1474nvgpu_dbg_gpu_ioctl_suspend_resume_contexts(struct dbg_session_gk20a *dbg_s,
1475 struct nvgpu_dbg_gpu_suspend_resume_contexts_args *args)
1476{
1477 struct gk20a *g = dbg_s->g;
1478 int err = 0;
1479 int ctx_resident_ch_fd = -1;
1480
1481 err = gk20a_busy(g);
1482 if (err)
1483 return err;
1484
1485 switch (args->action) {
1486 case NVGPU_DBG_GPU_SUSPEND_ALL_CONTEXTS:
1487 err = g->ops.gr.suspend_contexts(g, dbg_s,
1488 &ctx_resident_ch_fd);
1489 break;
1490
1491 case NVGPU_DBG_GPU_RESUME_ALL_CONTEXTS:
1492 err = g->ops.gr.resume_contexts(g, dbg_s,
1493 &ctx_resident_ch_fd);
1494 break;
1495 }
1496
1497 if (ctx_resident_ch_fd < 0) {
1498 args->is_resident_context = 0;
1499 } else {
1500 args->is_resident_context = 1;
1501 args->resident_context_fd = ctx_resident_ch_fd;
1502 }
1503
1504 gk20a_idle(g);
1505
1506 return err;
1507}
1508
1509static int nvgpu_dbg_gpu_ioctl_access_fb_memory(struct dbg_session_gk20a *dbg_s,
1510 struct nvgpu_dbg_gpu_access_fb_memory_args *args)
1511{
1512 struct gk20a *g = dbg_s->g;
1513 struct dma_buf *dmabuf;
1514 void __user *user_buffer = (void __user *)(uintptr_t)args->buffer;
1515 void *buffer;
1516 u64 size, access_size, offset;
1517 u64 access_limit_size = SZ_4K;
1518 int err = 0;
1519
1520 if ((args->offset & 3) || (!args->size) || (args->size & 3))
1521 return -EINVAL;
1522
1523 dmabuf = dma_buf_get(args->dmabuf_fd);
1524 if (IS_ERR(dmabuf))
1525 return -EINVAL;
1526
1527 if ((args->offset > dmabuf->size) ||
1528 (args->size > dmabuf->size) ||
1529 (args->offset + args->size > dmabuf->size)) {
1530 err = -EINVAL;
1531 goto fail_dmabuf_put;
1532 }
1533
1534 buffer = nvgpu_big_zalloc(g, access_limit_size);
1535 if (!buffer) {
1536 err = -ENOMEM;
1537 goto fail_dmabuf_put;
1538 }
1539
1540 size = args->size;
1541 offset = 0;
1542
1543 err = gk20a_busy(g);
1544 if (err)
1545 goto fail_free_buffer;
1546
1547 while (size) {
1548 /* Max access size of access_limit_size in one loop */
1549 access_size = min(access_limit_size, size);
1550
1551 if (args->cmd ==
1552 NVGPU_DBG_GPU_IOCTL_ACCESS_FB_MEMORY_CMD_WRITE) {
1553 err = copy_from_user(buffer, user_buffer + offset,
1554 access_size);
1555 if (err)
1556 goto fail_idle;
1557 }
1558
1559 err = nvgpu_vidmem_buf_access_memory(g, dmabuf, buffer,
1560 args->offset + offset, access_size,
1561 args->cmd);
1562 if (err)
1563 goto fail_idle;
1564
1565 if (args->cmd ==
1566 NVGPU_DBG_GPU_IOCTL_ACCESS_FB_MEMORY_CMD_READ) {
1567 err = copy_to_user(user_buffer + offset,
1568 buffer, access_size);
1569 if (err)
1570 goto fail_idle;
1571 }
1572
1573 size -= access_size;
1574 offset += access_size;
1575 }
1576
1577fail_idle:
1578 gk20a_idle(g);
1579fail_free_buffer:
1580 nvgpu_big_free(g, buffer);
1581fail_dmabuf_put:
1582 dma_buf_put(dmabuf);
1583
1584 return err;
1585}
1586
1587static int nvgpu_ioctl_profiler_reserve(struct dbg_session_gk20a *dbg_s,
1588 struct nvgpu_dbg_gpu_profiler_reserve_args *args)
1589{
1590 if (args->acquire)
1591 return nvgpu_profiler_reserve_acquire(dbg_s, args->profiler_handle);
1592
1593 return nvgpu_profiler_reserve_release(dbg_s, args->profiler_handle);
1594}
1595
1596static void nvgpu_dbg_gpu_ioctl_get_timeout(struct dbg_session_gk20a *dbg_s,
1597 struct nvgpu_dbg_gpu_timeout_args *args)
1598{
1599 bool status;
1600 struct gk20a *g = dbg_s->g;
1601
1602 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1603 status = nvgpu_is_timeouts_enabled(g);
1604 nvgpu_mutex_release(&g->dbg_sessions_lock);
1605
1606 if (status)
1607 args->enable = NVGPU_DBG_GPU_IOCTL_TIMEOUT_ENABLE;
1608 else
1609 args->enable = NVGPU_DBG_GPU_IOCTL_TIMEOUT_DISABLE;
1610}
1611
1612/* In order to perform a context relative op the context has
1613 * to be created already... which would imply that the
1614 * context switch mechanism has already been put in place.
1615 * So by the time we perform such an opertation it should always
1616 * be possible to query for the appropriate context offsets, etc.
1617 *
1618 * But note: while the dbg_gpu bind requires the a channel fd,
1619 * it doesn't require an allocated gr/compute obj at that point...
1620 */
1621static bool gr_context_info_available(struct dbg_session_gk20a *dbg_s,
1622 struct gr_gk20a *gr)
1623{
1624 int err;
1625
1626 nvgpu_mutex_acquire(&gr->ctx_mutex);
1627 err = !gr->ctx_vars.golden_image_initialized;
1628 nvgpu_mutex_release(&gr->ctx_mutex);
1629 if (err)
1630 return false;
1631 return true;
1632
1633}
1634
1635static int gk20a_perfbuf_release_locked(struct gk20a *g, u64 offset)
1636{
1637 struct mm_gk20a *mm = &g->mm;
1638 struct vm_gk20a *vm = mm->perfbuf.vm;
1639 int err;
1640
1641 err = g->ops.dbg_session_ops.perfbuffer_disable(g);
1642
1643 nvgpu_vm_unmap(vm, offset, NULL);
1644 nvgpu_free_inst_block(g, &mm->perfbuf.inst_block);
1645 nvgpu_vm_put(vm);
1646
1647 g->perfbuf.owner = NULL;
1648 g->perfbuf.offset = 0;
1649 return err;
1650}
1651
1652static int nvgpu_profiler_reserve_release(struct dbg_session_gk20a *dbg_s,
1653 u32 profiler_handle)
1654{
1655 struct gk20a *g = dbg_s->g;
1656 struct dbg_profiler_object_data *prof_obj;
1657 int err = 0;
1658
1659 nvgpu_log_fn(g, "%s profiler_handle = %x", g->name, profiler_handle);
1660
1661 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1662
1663 /* Find matching object. */
1664 prof_obj = find_matching_prof_obj(dbg_s, profiler_handle);
1665
1666 if (!prof_obj) {
1667 nvgpu_err(g, "object not found");
1668 err = -EINVAL;
1669 goto exit;
1670 }
1671
1672 if (prof_obj->has_reservation)
1673 g->ops.dbg_session_ops.release_profiler_reservation(dbg_s, prof_obj);
1674 else {
1675 nvgpu_err(g, "No reservation found");
1676 err = -EINVAL;
1677 goto exit;
1678 }
1679exit:
1680 nvgpu_mutex_release(&g->dbg_sessions_lock);
1681 return err;
1682}
1683
1684static int nvgpu_profiler_reserve_acquire(struct dbg_session_gk20a *dbg_s,
1685 u32 profiler_handle)
1686{
1687 struct gk20a *g = dbg_s->g;
1688 struct dbg_profiler_object_data *prof_obj, *my_prof_obj;
1689 int err = 0;
1690
1691 nvgpu_log_fn(g, "%s profiler_handle = %x", g->name, profiler_handle);
1692
1693 if (g->profiler_reservation_count < 0) {
1694 nvgpu_err(g, "Negative reservation count!");
1695 return -EINVAL;
1696 }
1697
1698 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1699
1700 /* Find matching object. */
1701 my_prof_obj = find_matching_prof_obj(dbg_s, profiler_handle);
1702
1703 if (!my_prof_obj) {
1704 nvgpu_err(g, "object not found");
1705 err = -EINVAL;
1706 goto exit;
1707 }
1708
1709 /* If we already have the reservation, we're done */
1710 if (my_prof_obj->has_reservation) {
1711 err = 0;
1712 goto exit;
1713 }
1714
1715 if (my_prof_obj->ch == NULL) {
1716 /* Global reservations are only allowed if there are no other
1717 * global or per-context reservations currently held
1718 */
1719 if (!g->ops.dbg_session_ops.check_and_set_global_reservation(
1720 dbg_s, my_prof_obj)) {
1721 nvgpu_err(g,
1722 "global reserve: have existing reservation");
1723 err = -EBUSY;
1724 }
1725 } else if (g->global_profiler_reservation_held) {
1726 /* If there's a global reservation,
1727 * we can't take a per-context one.
1728 */
1729 nvgpu_err(g,
1730 "per-ctxt reserve: global reservation in effect");
1731 err = -EBUSY;
1732 } else if (gk20a_is_channel_marked_as_tsg(my_prof_obj->ch)) {
1733 /* TSG: check that another channel in the TSG
1734 * doesn't already have the reservation
1735 */
1736 int my_tsgid = my_prof_obj->ch->tsgid;
1737
1738 nvgpu_list_for_each_entry(prof_obj, &g->profiler_objects,
1739 dbg_profiler_object_data, prof_obj_entry) {
1740 if (prof_obj->has_reservation &&
1741 (prof_obj->ch->tsgid == my_tsgid)) {
1742 nvgpu_err(g,
1743 "per-ctxt reserve (tsg): already reserved");
1744 err = -EBUSY;
1745 goto exit;
1746 }
1747 }
1748
1749 if (!g->ops.dbg_session_ops.check_and_set_context_reservation(
1750 dbg_s, my_prof_obj)) {
1751 /* Another guest OS has the global reservation */
1752 nvgpu_err(g,
1753 "per-ctxt reserve: global reservation in effect");
1754 err = -EBUSY;
1755 }
1756 } else {
1757 /* channel: check that some other profiler object doesn't
1758 * already have the reservation.
1759 */
1760 struct channel_gk20a *my_ch = my_prof_obj->ch;
1761
1762 nvgpu_list_for_each_entry(prof_obj, &g->profiler_objects,
1763 dbg_profiler_object_data, prof_obj_entry) {
1764 if (prof_obj->has_reservation &&
1765 (prof_obj->ch == my_ch)) {
1766 nvgpu_err(g,
1767 "per-ctxt reserve (ch): already reserved");
1768 err = -EBUSY;
1769 goto exit;
1770 }
1771 }
1772
1773 if (!g->ops.dbg_session_ops.check_and_set_context_reservation(
1774 dbg_s, my_prof_obj)) {
1775 /* Another guest OS has the global reservation */
1776 nvgpu_err(g,
1777 "per-ctxt reserve: global reservation in effect");
1778 err = -EBUSY;
1779 }
1780 }
1781exit:
1782 nvgpu_mutex_release(&g->dbg_sessions_lock);
1783 return err;
1784}
1785
1786static int dbg_unbind_channel_gk20a(struct dbg_session_gk20a *dbg_s,
1787 struct nvgpu_dbg_gpu_unbind_channel_args *args)
1788{
1789 struct dbg_session_channel_data *ch_data;
1790 struct gk20a *g = dbg_s->g;
1791 bool channel_found = false;
1792 struct channel_gk20a *ch;
1793 int err;
1794
1795 nvgpu_log(g, gpu_dbg_fn|gpu_dbg_gpu_dbg, "%s fd=%d",
1796 g->name, args->channel_fd);
1797
1798 ch = gk20a_get_channel_from_file(args->channel_fd);
1799 if (!ch) {
1800 nvgpu_log_fn(g, "no channel found for fd");
1801 return -EINVAL;
1802 }
1803
1804 nvgpu_mutex_acquire(&dbg_s->ch_list_lock);
1805 nvgpu_list_for_each_entry(ch_data, &dbg_s->ch_list,
1806 dbg_session_channel_data, ch_entry) {
1807 if (ch->chid == ch_data->chid) {
1808 channel_found = true;
1809 break;
1810 }
1811 }
1812 nvgpu_mutex_release(&dbg_s->ch_list_lock);
1813
1814 if (!channel_found) {
1815 nvgpu_log_fn(g, "channel not bounded, fd=%d\n", args->channel_fd);
1816 err = -EINVAL;
1817 goto out;
1818 }
1819
1820 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1821 nvgpu_mutex_acquire(&dbg_s->ch_list_lock);
1822 err = dbg_unbind_single_channel_gk20a(dbg_s, ch_data);
1823 nvgpu_mutex_release(&dbg_s->ch_list_lock);
1824 nvgpu_mutex_release(&g->dbg_sessions_lock);
1825
1826out:
1827 gk20a_channel_put(ch);
1828 return err;
1829}
1830
1831int gk20a_dbg_gpu_dev_open(struct inode *inode, struct file *filp)
1832{
1833 struct nvgpu_os_linux *l = container_of(inode->i_cdev,
1834 struct nvgpu_os_linux, dbg.cdev);
1835 struct gk20a *g = &l->g;
1836
1837 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
1838 return gk20a_dbg_gpu_do_dev_open(inode, filp, false /* not profiler */);
1839}
1840
1841long gk20a_dbg_gpu_dev_ioctl(struct file *filp, unsigned int cmd,
1842 unsigned long arg)
1843{
1844 struct dbg_session_gk20a_linux *dbg_s_linux = filp->private_data;
1845 struct dbg_session_gk20a *dbg_s = &dbg_s_linux->dbg_s;
1846 struct gk20a *g = dbg_s->g;
1847 u8 buf[NVGPU_DBG_GPU_IOCTL_MAX_ARG_SIZE];
1848 int err = 0;
1849
1850 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
1851
1852 if ((_IOC_TYPE(cmd) != NVGPU_DBG_GPU_IOCTL_MAGIC) ||
1853 (_IOC_NR(cmd) == 0) ||
1854 (_IOC_NR(cmd) > NVGPU_DBG_GPU_IOCTL_LAST) ||
1855 (_IOC_SIZE(cmd) > NVGPU_DBG_GPU_IOCTL_MAX_ARG_SIZE))
1856 return -EINVAL;
1857
1858 memset(buf, 0, sizeof(buf));
1859 if (_IOC_DIR(cmd) & _IOC_WRITE) {
1860 if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
1861 return -EFAULT;
1862 }
1863
1864 if (!g->sw_ready) {
1865 err = gk20a_busy(g);
1866 if (err)
1867 return err;
1868
1869 gk20a_idle(g);
1870 }
1871
1872 /* protect from threaded user space calls */
1873 nvgpu_mutex_acquire(&dbg_s->ioctl_lock);
1874
1875 switch (cmd) {
1876 case NVGPU_DBG_GPU_IOCTL_BIND_CHANNEL:
1877 err = dbg_bind_channel_gk20a(dbg_s,
1878 (struct nvgpu_dbg_gpu_bind_channel_args *)buf);
1879 break;
1880
1881 case NVGPU_DBG_GPU_IOCTL_REG_OPS:
1882 err = nvgpu_ioctl_channel_reg_ops(dbg_s,
1883 (struct nvgpu_dbg_gpu_exec_reg_ops_args *)buf);
1884 break;
1885
1886 case NVGPU_DBG_GPU_IOCTL_POWERGATE:
1887 err = nvgpu_ioctl_powergate_gk20a(dbg_s,
1888 (struct nvgpu_dbg_gpu_powergate_args *)buf);
1889 break;
1890
1891 case NVGPU_DBG_GPU_IOCTL_EVENTS_CTRL:
1892 err = gk20a_dbg_gpu_events_ctrl(dbg_s,
1893 (struct nvgpu_dbg_gpu_events_ctrl_args *)buf);
1894 break;
1895
1896 case NVGPU_DBG_GPU_IOCTL_SMPC_CTXSW_MODE:
1897 err = nvgpu_dbg_gpu_ioctl_smpc_ctxsw_mode(dbg_s,
1898 (struct nvgpu_dbg_gpu_smpc_ctxsw_mode_args *)buf);
1899 break;
1900
1901 case NVGPU_DBG_GPU_IOCTL_HWPM_CTXSW_MODE:
1902 err = nvgpu_dbg_gpu_ioctl_hwpm_ctxsw_mode(dbg_s,
1903 (struct nvgpu_dbg_gpu_hwpm_ctxsw_mode_args *)buf);
1904 break;
1905
1906 case NVGPU_DBG_GPU_IOCTL_SUSPEND_RESUME_ALL_SMS:
1907 err = nvgpu_dbg_gpu_ioctl_suspend_resume_sm(dbg_s,
1908 (struct nvgpu_dbg_gpu_suspend_resume_all_sms_args *)buf);
1909 break;
1910
1911 case NVGPU_DBG_GPU_IOCTL_PERFBUF_MAP:
1912 err = gk20a_perfbuf_map(dbg_s,
1913 (struct nvgpu_dbg_gpu_perfbuf_map_args *)buf);
1914 break;
1915
1916 case NVGPU_DBG_GPU_IOCTL_PERFBUF_UNMAP:
1917 err = gk20a_perfbuf_unmap(dbg_s,
1918 (struct nvgpu_dbg_gpu_perfbuf_unmap_args *)buf);
1919 break;
1920
1921 case NVGPU_DBG_GPU_IOCTL_PC_SAMPLING:
1922 err = gk20a_dbg_pc_sampling(dbg_s,
1923 (struct nvgpu_dbg_gpu_pc_sampling_args *)buf);
1924 break;
1925
1926 case NVGPU_DBG_GPU_IOCTL_SET_NEXT_STOP_TRIGGER_TYPE:
1927 err = nvgpu_dbg_gpu_ioctl_set_next_stop_trigger_type(dbg_s,
1928 (struct nvgpu_dbg_gpu_set_next_stop_trigger_type_args *)buf);
1929 break;
1930
1931 case NVGPU_DBG_GPU_IOCTL_TIMEOUT:
1932 err = nvgpu_dbg_gpu_ioctl_timeout(dbg_s,
1933 (struct nvgpu_dbg_gpu_timeout_args *)buf);
1934 break;
1935
1936 case NVGPU_DBG_GPU_IOCTL_GET_TIMEOUT:
1937 nvgpu_dbg_gpu_ioctl_get_timeout(dbg_s,
1938 (struct nvgpu_dbg_gpu_timeout_args *)buf);
1939 break;
1940
1941 case NVGPU_DBG_GPU_IOCTL_READ_SINGLE_SM_ERROR_STATE:
1942 err = nvgpu_dbg_gpu_ioctl_read_single_sm_error_state(dbg_s,
1943 (struct nvgpu_dbg_gpu_read_single_sm_error_state_args *)buf);
1944 break;
1945
1946 case NVGPU_DBG_GPU_IOCTL_CLEAR_SINGLE_SM_ERROR_STATE:
1947 err = nvgpu_dbg_gpu_ioctl_clear_single_sm_error_state(dbg_s,
1948 (struct nvgpu_dbg_gpu_clear_single_sm_error_state_args *)buf);
1949 break;
1950
1951 case NVGPU_DBG_GPU_IOCTL_WRITE_SINGLE_SM_ERROR_STATE:
1952 err = nvgpu_dbg_gpu_ioctl_write_single_sm_error_state(dbg_s,
1953 (struct nvgpu_dbg_gpu_write_single_sm_error_state_args *)buf);
1954 break;
1955
1956 case NVGPU_DBG_GPU_IOCTL_UNBIND_CHANNEL:
1957 err = dbg_unbind_channel_gk20a(dbg_s,
1958 (struct nvgpu_dbg_gpu_unbind_channel_args *)buf);
1959 break;
1960
1961 case NVGPU_DBG_GPU_IOCTL_SUSPEND_RESUME_CONTEXTS:
1962 err = nvgpu_dbg_gpu_ioctl_suspend_resume_contexts(dbg_s,
1963 (struct nvgpu_dbg_gpu_suspend_resume_contexts_args *)buf);
1964 break;
1965
1966 case NVGPU_DBG_GPU_IOCTL_ACCESS_FB_MEMORY:
1967 err = nvgpu_dbg_gpu_ioctl_access_fb_memory(dbg_s,
1968 (struct nvgpu_dbg_gpu_access_fb_memory_args *)buf);
1969 break;
1970
1971 case NVGPU_DBG_GPU_IOCTL_PROFILER_ALLOCATE:
1972 err = nvgpu_ioctl_allocate_profiler_object(dbg_s_linux,
1973 (struct nvgpu_dbg_gpu_profiler_obj_mgt_args *)buf);
1974 break;
1975
1976 case NVGPU_DBG_GPU_IOCTL_PROFILER_FREE:
1977 err = nvgpu_ioctl_free_profiler_object(dbg_s_linux,
1978 (struct nvgpu_dbg_gpu_profiler_obj_mgt_args *)buf);
1979 break;
1980
1981 case NVGPU_DBG_GPU_IOCTL_PROFILER_RESERVE:
1982 err = nvgpu_ioctl_profiler_reserve(dbg_s,
1983 (struct nvgpu_dbg_gpu_profiler_reserve_args *)buf);
1984 break;
1985
1986 default:
1987 nvgpu_err(g,
1988 "unrecognized dbg gpu ioctl cmd: 0x%x",
1989 cmd);
1990 err = -ENOTTY;
1991 break;
1992 }
1993
1994 nvgpu_mutex_release(&dbg_s->ioctl_lock);
1995
1996 nvgpu_log(g, gpu_dbg_gpu_dbg, "ret=%d", err);
1997
1998 if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
1999 err = copy_to_user((void __user *)arg,
2000 buf, _IOC_SIZE(cmd));
2001
2002 return err;
2003}
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-07 15:44:17 -0400