summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/linux/vgpu/fifo_vgpu.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common/linux/vgpu/fifo_vgpu.c')
-rw-r--r--drivers/gpu/nvgpu/common/linux/vgpu/fifo_vgpu.c822
1 files changed, 822 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/vgpu/fifo_vgpu.c b/drivers/gpu/nvgpu/common/linux/vgpu/fifo_vgpu.c
new file mode 100644
index 00000000..cdcecca5
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/vgpu/fifo_vgpu.c
@@ -0,0 +1,822 @@
1/*
2 * Virtualized GPU Fifo
3 *
4 * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <linux/dma-mapping.h>
20#include <trace/events/gk20a.h>
21#include <uapi/linux/nvgpu.h>
22
23#include <nvgpu/kmem.h>
24#include <nvgpu/dma.h>
25#include <nvgpu/atomic.h>
26#include <nvgpu/bug.h>
27#include <nvgpu/barrier.h>
28
29#include "vgpu.h"
30#include "fifo_vgpu.h"
31
32#include <nvgpu/hw/gk20a/hw_fifo_gk20a.h>
33#include <nvgpu/hw/gk20a/hw_ram_gk20a.h>
34
35void vgpu_channel_bind(struct channel_gk20a *ch)
36{
37 struct tegra_vgpu_cmd_msg msg;
38 struct tegra_vgpu_channel_config_params *p =
39 &msg.params.channel_config;
40 int err;
41
42 gk20a_dbg_info("bind channel %d", ch->chid);
43
44 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_BIND;
45 msg.handle = vgpu_get_handle(ch->g);
46 p->handle = ch->virt_ctx;
47 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
48 WARN_ON(err || msg.ret);
49
50 nvgpu_smp_wmb();
51 nvgpu_atomic_set(&ch->bound, true);
52}
53
54void vgpu_channel_unbind(struct channel_gk20a *ch)
55{
56
57 gk20a_dbg_fn("");
58
59 if (nvgpu_atomic_cmpxchg(&ch->bound, true, false)) {
60 struct tegra_vgpu_cmd_msg msg;
61 struct tegra_vgpu_channel_config_params *p =
62 &msg.params.channel_config;
63 int err;
64
65 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_UNBIND;
66 msg.handle = vgpu_get_handle(ch->g);
67 p->handle = ch->virt_ctx;
68 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
69 WARN_ON(err || msg.ret);
70 }
71
72}
73
74int vgpu_channel_alloc_inst(struct gk20a *g, struct channel_gk20a *ch)
75{
76 struct tegra_vgpu_cmd_msg msg;
77 struct tegra_vgpu_channel_hwctx_params *p = &msg.params.channel_hwctx;
78 int err;
79
80 gk20a_dbg_fn("");
81
82 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_ALLOC_HWCTX;
83 msg.handle = vgpu_get_handle(g);
84 p->id = ch->chid;
85 p->pid = (u64)current->tgid;
86 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
87 if (err || msg.ret) {
88 nvgpu_err(g, "fail");
89 return -ENOMEM;
90 }
91
92 ch->virt_ctx = p->handle;
93 gk20a_dbg_fn("done");
94 return 0;
95}
96
97void vgpu_channel_free_inst(struct gk20a *g, struct channel_gk20a *ch)
98{
99 struct tegra_vgpu_cmd_msg msg;
100 struct tegra_vgpu_channel_hwctx_params *p = &msg.params.channel_hwctx;
101 int err;
102
103 gk20a_dbg_fn("");
104
105 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_FREE_HWCTX;
106 msg.handle = vgpu_get_handle(g);
107 p->handle = ch->virt_ctx;
108 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
109 WARN_ON(err || msg.ret);
110}
111
112void vgpu_channel_enable(struct channel_gk20a *ch)
113{
114 struct tegra_vgpu_cmd_msg msg;
115 struct tegra_vgpu_channel_config_params *p =
116 &msg.params.channel_config;
117 int err;
118
119 gk20a_dbg_fn("");
120
121 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_ENABLE;
122 msg.handle = vgpu_get_handle(ch->g);
123 p->handle = ch->virt_ctx;
124 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
125 WARN_ON(err || msg.ret);
126}
127
128void vgpu_channel_disable(struct channel_gk20a *ch)
129{
130 struct tegra_vgpu_cmd_msg msg;
131 struct tegra_vgpu_channel_config_params *p =
132 &msg.params.channel_config;
133 int err;
134
135 gk20a_dbg_fn("");
136
137 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_DISABLE;
138 msg.handle = vgpu_get_handle(ch->g);
139 p->handle = ch->virt_ctx;
140 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
141 WARN_ON(err || msg.ret);
142}
143
144int vgpu_channel_setup_ramfc(struct channel_gk20a *ch, u64 gpfifo_base,
145 u32 gpfifo_entries,
146 unsigned long acquire_timeout, u32 flags)
147{
148 struct device __maybe_unused *d = dev_from_gk20a(ch->g);
149 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(d);
150 struct tegra_vgpu_cmd_msg msg;
151 struct tegra_vgpu_ramfc_params *p = &msg.params.ramfc;
152 int err;
153
154 gk20a_dbg_fn("");
155
156 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SETUP_RAMFC;
157 msg.handle = vgpu_get_handle(ch->g);
158 p->handle = ch->virt_ctx;
159 p->gpfifo_va = gpfifo_base;
160 p->num_entries = gpfifo_entries;
161 p->userd_addr = ch->userd_iova;
162 p->iova = mapping ? 1 : 0;
163 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
164
165 return (err || msg.ret) ? -ENOMEM : 0;
166}
167
168int vgpu_fifo_init_engine_info(struct fifo_gk20a *f)
169{
170 struct vgpu_priv_data *priv = vgpu_get_priv_data(f->g);
171 struct tegra_vgpu_engines_info *engines = &priv->constants.engines_info;
172 u32 i;
173
174 gk20a_dbg_fn("");
175
176 if (engines->num_engines > TEGRA_VGPU_MAX_ENGINES) {
177 nvgpu_err(f->g, "num_engines %d larger than max %d",
178 engines->num_engines, TEGRA_VGPU_MAX_ENGINES);
179 return -EINVAL;
180 }
181
182 f->num_engines = engines->num_engines;
183 for (i = 0; i < f->num_engines; i++) {
184 struct fifo_engine_info_gk20a *info =
185 &f->engine_info[engines->info[i].engine_id];
186
187 if (engines->info[i].engine_id >= f->max_engines) {
188 nvgpu_err(f->g, "engine id %d larger than max %d",
189 engines->info[i].engine_id,
190 f->max_engines);
191 return -EINVAL;
192 }
193
194 info->intr_mask = engines->info[i].intr_mask;
195 info->reset_mask = engines->info[i].reset_mask;
196 info->runlist_id = engines->info[i].runlist_id;
197 info->pbdma_id = engines->info[i].pbdma_id;
198 info->inst_id = engines->info[i].inst_id;
199 info->pri_base = engines->info[i].pri_base;
200 info->engine_enum = engines->info[i].engine_enum;
201 info->fault_id = engines->info[i].fault_id;
202 f->active_engines_list[i] = engines->info[i].engine_id;
203 }
204
205 gk20a_dbg_fn("done");
206
207 return 0;
208}
209
210static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
211{
212 struct fifo_runlist_info_gk20a *runlist;
213 struct device *d = dev_from_gk20a(g);
214 unsigned int runlist_id = -1;
215 u32 i;
216 u64 runlist_size;
217
218 gk20a_dbg_fn("");
219
220 f->max_runlists = g->ops.fifo.eng_runlist_base_size();
221 f->runlist_info = nvgpu_kzalloc(g,
222 sizeof(struct fifo_runlist_info_gk20a) *
223 f->max_runlists);
224 if (!f->runlist_info)
225 goto clean_up_runlist;
226
227 memset(f->runlist_info, 0, (sizeof(struct fifo_runlist_info_gk20a) *
228 f->max_runlists));
229
230 for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) {
231 runlist = &f->runlist_info[runlist_id];
232
233 runlist->active_channels =
234 nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
235 BITS_PER_BYTE));
236 if (!runlist->active_channels)
237 goto clean_up_runlist;
238
239 runlist_size = sizeof(u16) * f->num_channels;
240 for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
241 int err = nvgpu_dma_alloc_sys(g, runlist_size,
242 &runlist->mem[i]);
243 if (err) {
244 dev_err(d, "memory allocation failed\n");
245 goto clean_up_runlist;
246 }
247 }
248 nvgpu_mutex_init(&runlist->mutex);
249
250 /* None of buffers is pinned if this value doesn't change.
251 Otherwise, one of them (cur_buffer) must have been pinned. */
252 runlist->cur_buffer = MAX_RUNLIST_BUFFERS;
253 }
254
255 gk20a_dbg_fn("done");
256 return 0;
257
258clean_up_runlist:
259 gk20a_fifo_delete_runlist(f);
260 gk20a_dbg_fn("fail");
261 return -ENOMEM;
262}
263
264static int vgpu_init_fifo_setup_sw(struct gk20a *g)
265{
266 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
267 struct fifo_gk20a *f = &g->fifo;
268 struct device *d = dev_from_gk20a(g);
269 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
270 unsigned int chid;
271 int err = 0;
272
273 gk20a_dbg_fn("");
274
275 if (f->sw_ready) {
276 gk20a_dbg_fn("skip init");
277 return 0;
278 }
279
280 f->g = g;
281 f->num_channels = priv->constants.num_channels;
282 f->max_engines = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_ENGINES);
283
284 f->userd_entry_size = 1 << ram_userd_base_shift_v();
285
286 err = nvgpu_dma_alloc_sys(g, f->userd_entry_size * f->num_channels,
287 &f->userd);
288 if (err) {
289 dev_err(d, "memory allocation failed\n");
290 goto clean_up;
291 }
292
293 /* bar1 va */
294 if (g->ops.mm.is_bar1_supported(g)) {
295 f->userd.gpu_va = vgpu_bar1_map(g, &f->userd.priv.sgt,
296 f->userd.size);
297 if (!f->userd.gpu_va) {
298 dev_err(d, "gmmu mapping failed\n");
299 goto clean_up;
300 }
301 /* if reduced BAR1 range is specified, use offset of 0
302 * (server returns offset assuming full BAR1 range)
303 */
304 if (resource_size(l->bar1_mem) ==
305 (resource_size_t)f->userd.size)
306 f->userd.gpu_va = 0;
307 }
308
309 gk20a_dbg(gpu_dbg_map_v, "userd bar1 va = 0x%llx", f->userd.gpu_va);
310
311 f->channel = nvgpu_vzalloc(g, f->num_channels * sizeof(*f->channel));
312 f->tsg = nvgpu_vzalloc(g, f->num_channels * sizeof(*f->tsg));
313 f->engine_info = nvgpu_kzalloc(g, f->max_engines *
314 sizeof(*f->engine_info));
315 f->active_engines_list = nvgpu_kzalloc(g, f->max_engines * sizeof(u32));
316
317 if (!(f->channel && f->tsg && f->engine_info && f->active_engines_list)) {
318 err = -ENOMEM;
319 goto clean_up;
320 }
321 memset(f->active_engines_list, 0xff, (f->max_engines * sizeof(u32)));
322
323 g->ops.fifo.init_engine_info(f);
324
325 init_runlist(g, f);
326
327 nvgpu_init_list_node(&f->free_chs);
328 nvgpu_mutex_init(&f->free_chs_mutex);
329
330 for (chid = 0; chid < f->num_channels; chid++) {
331 f->channel[chid].userd_iova =
332 nvgpu_mem_get_addr(g, &f->userd) +
333 chid * f->userd_entry_size;
334 f->channel[chid].userd_gpu_va =
335 f->userd.gpu_va + chid * f->userd_entry_size;
336
337 gk20a_init_channel_support(g, chid);
338 gk20a_init_tsg_support(g, chid);
339 }
340 nvgpu_mutex_init(&f->tsg_inuse_mutex);
341
342 err = nvgpu_channel_worker_init(g);
343 if (err)
344 goto clean_up;
345
346 f->deferred_reset_pending = false;
347 nvgpu_mutex_init(&f->deferred_reset_mutex);
348
349 f->channel_base = priv->constants.channel_base;
350
351 f->sw_ready = true;
352
353 gk20a_dbg_fn("done");
354 return 0;
355
356clean_up:
357 gk20a_dbg_fn("fail");
358 /* FIXME: unmap from bar1 */
359 nvgpu_dma_free(g, &f->userd);
360
361 memset(&f->userd, 0, sizeof(f->userd));
362
363 nvgpu_vfree(g, f->channel);
364 f->channel = NULL;
365 nvgpu_vfree(g, f->tsg);
366 f->tsg = NULL;
367 nvgpu_kfree(g, f->engine_info);
368 f->engine_info = NULL;
369 nvgpu_kfree(g, f->active_engines_list);
370 f->active_engines_list = NULL;
371
372 return err;
373}
374
375int vgpu_init_fifo_setup_hw(struct gk20a *g)
376{
377 gk20a_dbg_fn("");
378
379 /* test write, read through bar1 @ userd region before
380 * turning on the snooping */
381 {
382 struct fifo_gk20a *f = &g->fifo;
383 u32 v, v1 = 0x33, v2 = 0x55;
384
385 u32 bar1_vaddr = f->userd.gpu_va;
386 volatile u32 *cpu_vaddr = f->userd.cpu_va;
387
388 gk20a_dbg_info("test bar1 @ vaddr 0x%x",
389 bar1_vaddr);
390
391 v = gk20a_bar1_readl(g, bar1_vaddr);
392
393 *cpu_vaddr = v1;
394 nvgpu_mb();
395
396 if (v1 != gk20a_bar1_readl(g, bar1_vaddr)) {
397 nvgpu_err(g, "bar1 broken @ gk20a!");
398 return -EINVAL;
399 }
400
401 gk20a_bar1_writel(g, bar1_vaddr, v2);
402
403 if (v2 != gk20a_bar1_readl(g, bar1_vaddr)) {
404 nvgpu_err(g, "bar1 broken @ gk20a!");
405 return -EINVAL;
406 }
407
408 /* is it visible to the cpu? */
409 if (*cpu_vaddr != v2) {
410 nvgpu_err(g, "cpu didn't see bar1 write @ %p!",
411 cpu_vaddr);
412 }
413
414 /* put it back */
415 gk20a_bar1_writel(g, bar1_vaddr, v);
416 }
417
418 gk20a_dbg_fn("done");
419
420 return 0;
421}
422
423int vgpu_init_fifo_support(struct gk20a *g)
424{
425 u32 err;
426
427 gk20a_dbg_fn("");
428
429 err = vgpu_init_fifo_setup_sw(g);
430 if (err)
431 return err;
432
433 if (g->ops.fifo.init_fifo_setup_hw)
434 err = g->ops.fifo.init_fifo_setup_hw(g);
435 return err;
436}
437
438int vgpu_fifo_preempt_channel(struct gk20a *g, u32 chid)
439{
440 struct fifo_gk20a *f = &g->fifo;
441 struct channel_gk20a *ch = &f->channel[chid];
442 struct tegra_vgpu_cmd_msg msg;
443 struct tegra_vgpu_channel_config_params *p =
444 &msg.params.channel_config;
445 int err;
446
447 gk20a_dbg_fn("");
448
449 if (!nvgpu_atomic_read(&ch->bound))
450 return 0;
451
452 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_PREEMPT;
453 msg.handle = vgpu_get_handle(g);
454 p->handle = ch->virt_ctx;
455 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
456
457 if (err || msg.ret) {
458 nvgpu_err(g,
459 "preempt channel %d failed", chid);
460 err = -ENOMEM;
461 }
462
463 return err;
464}
465
466int vgpu_fifo_preempt_tsg(struct gk20a *g, u32 tsgid)
467{
468 struct tegra_vgpu_cmd_msg msg;
469 struct tegra_vgpu_tsg_preempt_params *p =
470 &msg.params.tsg_preempt;
471 int err;
472
473 gk20a_dbg_fn("");
474
475 msg.cmd = TEGRA_VGPU_CMD_TSG_PREEMPT;
476 msg.handle = vgpu_get_handle(g);
477 p->tsg_id = tsgid;
478 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
479 err = err ? err : msg.ret;
480
481 if (err) {
482 nvgpu_err(g,
483 "preempt tsg %u failed", tsgid);
484 }
485
486 return err;
487}
488
489static int vgpu_submit_runlist(struct gk20a *g, u64 handle, u8 runlist_id,
490 u16 *runlist, u32 num_entries)
491{
492 struct tegra_vgpu_cmd_msg msg;
493 struct tegra_vgpu_runlist_params *p;
494 int err;
495 void *oob_handle;
496 void *oob;
497 size_t size, oob_size;
498
499 oob_handle = tegra_gr_comm_oob_get_ptr(TEGRA_GR_COMM_CTX_CLIENT,
500 tegra_gr_comm_get_server_vmid(), TEGRA_VGPU_QUEUE_CMD,
501 &oob, &oob_size);
502 if (!oob_handle)
503 return -EINVAL;
504
505 size = sizeof(*runlist) * num_entries;
506 if (oob_size < size) {
507 err = -ENOMEM;
508 goto done;
509 }
510
511 msg.cmd = TEGRA_VGPU_CMD_SUBMIT_RUNLIST;
512 msg.handle = handle;
513 p = &msg.params.runlist;
514 p->runlist_id = runlist_id;
515 p->num_entries = num_entries;
516
517 memcpy(oob, runlist, size);
518 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
519
520 err = (err || msg.ret) ? -1 : 0;
521
522done:
523 tegra_gr_comm_oob_put_ptr(oob_handle);
524 return err;
525}
526
527static int vgpu_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
528 u32 chid, bool add,
529 bool wait_for_finish)
530{
531 struct fifo_gk20a *f = &g->fifo;
532 struct fifo_runlist_info_gk20a *runlist;
533 u16 *runlist_entry = NULL;
534 u32 count = 0;
535
536 gk20a_dbg_fn("");
537
538 runlist = &f->runlist_info[runlist_id];
539
540 /* valid channel, add/remove it from active list.
541 Otherwise, keep active list untouched for suspend/resume. */
542 if (chid != (u32)~0) {
543 if (add) {
544 if (test_and_set_bit(chid,
545 runlist->active_channels) == 1)
546 return 0;
547 } else {
548 if (test_and_clear_bit(chid,
549 runlist->active_channels) == 0)
550 return 0;
551 }
552 }
553
554 if (chid != (u32)~0 || /* add/remove a valid channel */
555 add /* resume to add all channels back */) {
556 u32 cid;
557
558 runlist_entry = runlist->mem[0].cpu_va;
559 for_each_set_bit(cid,
560 runlist->active_channels, f->num_channels) {
561 gk20a_dbg_info("add channel %d to runlist", cid);
562 runlist_entry[0] = cid;
563 runlist_entry++;
564 count++;
565 }
566 } else /* suspend to remove all channels */
567 count = 0;
568
569 return vgpu_submit_runlist(g, vgpu_get_handle(g), runlist_id,
570 runlist->mem[0].cpu_va, count);
571}
572
573/* add/remove a channel from runlist
574 special cases below: runlist->active_channels will NOT be changed.
575 (chid == ~0 && !add) means remove all active channels from runlist.
576 (chid == ~0 && add) means restore all active channels on runlist. */
577int vgpu_fifo_update_runlist(struct gk20a *g, u32 runlist_id,
578 u32 chid, bool add, bool wait_for_finish)
579{
580 struct fifo_runlist_info_gk20a *runlist = NULL;
581 struct fifo_gk20a *f = &g->fifo;
582 u32 ret = 0;
583
584 gk20a_dbg_fn("");
585
586 runlist = &f->runlist_info[runlist_id];
587
588 nvgpu_mutex_acquire(&runlist->mutex);
589
590 ret = vgpu_fifo_update_runlist_locked(g, runlist_id, chid, add,
591 wait_for_finish);
592
593 nvgpu_mutex_release(&runlist->mutex);
594 return ret;
595}
596
597int vgpu_fifo_wait_engine_idle(struct gk20a *g)
598{
599 gk20a_dbg_fn("");
600
601 return 0;
602}
603
604static int vgpu_fifo_tsg_set_runlist_interleave(struct gk20a *g,
605 u32 tsgid,
606 u32 runlist_id,
607 u32 new_level)
608{
609 struct tegra_vgpu_cmd_msg msg = {0};
610 struct tegra_vgpu_tsg_runlist_interleave_params *p =
611 &msg.params.tsg_interleave;
612 int err;
613
614 gk20a_dbg_fn("");
615
616 msg.cmd = TEGRA_VGPU_CMD_TSG_SET_RUNLIST_INTERLEAVE;
617 msg.handle = vgpu_get_handle(g);
618 p->tsg_id = tsgid;
619 p->level = new_level;
620 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
621 WARN_ON(err || msg.ret);
622 return err ? err : msg.ret;
623}
624
625int vgpu_fifo_set_runlist_interleave(struct gk20a *g,
626 u32 id,
627 bool is_tsg,
628 u32 runlist_id,
629 u32 new_level)
630{
631 struct tegra_vgpu_cmd_msg msg;
632 struct tegra_vgpu_channel_runlist_interleave_params *p =
633 &msg.params.channel_interleave;
634 struct channel_gk20a *ch;
635 int err;
636
637 gk20a_dbg_fn("");
638
639 if (is_tsg)
640 return vgpu_fifo_tsg_set_runlist_interleave(g, id,
641 runlist_id, new_level);
642
643 ch = &g->fifo.channel[id];
644 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SET_RUNLIST_INTERLEAVE;
645 msg.handle = vgpu_get_handle(ch->g);
646 p->handle = ch->virt_ctx;
647 p->level = new_level;
648 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
649 WARN_ON(err || msg.ret);
650 return err ? err : msg.ret;
651}
652
653int vgpu_channel_set_timeslice(struct channel_gk20a *ch, u32 timeslice)
654{
655 struct tegra_vgpu_cmd_msg msg;
656 struct tegra_vgpu_channel_timeslice_params *p =
657 &msg.params.channel_timeslice;
658 int err;
659
660 gk20a_dbg_fn("");
661
662 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SET_TIMESLICE;
663 msg.handle = vgpu_get_handle(ch->g);
664 p->handle = ch->virt_ctx;
665 p->timeslice_us = timeslice;
666 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
667 err = err ? err : msg.ret;
668 WARN_ON(err);
669 if (!err)
670 ch->timeslice_us = p->timeslice_us;
671 return err;
672}
673
674int vgpu_fifo_force_reset_ch(struct channel_gk20a *ch,
675 u32 err_code, bool verbose)
676{
677 struct tsg_gk20a *tsg = NULL;
678 struct channel_gk20a *ch_tsg = NULL;
679 struct gk20a *g = ch->g;
680 struct tegra_vgpu_cmd_msg msg = {0};
681 struct tegra_vgpu_channel_config_params *p =
682 &msg.params.channel_config;
683 int err;
684
685 gk20a_dbg_fn("");
686
687 if (gk20a_is_channel_marked_as_tsg(ch)) {
688 tsg = &g->fifo.tsg[ch->tsgid];
689
690 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
691
692 list_for_each_entry(ch_tsg, &tsg->ch_list, ch_entry) {
693 if (gk20a_channel_get(ch_tsg)) {
694 gk20a_set_error_notifier(ch_tsg, err_code);
695 ch_tsg->has_timedout = true;
696 gk20a_channel_put(ch_tsg);
697 }
698 }
699
700 nvgpu_rwsem_up_read(&tsg->ch_list_lock);
701 } else {
702 gk20a_set_error_notifier(ch, err_code);
703 ch->has_timedout = true;
704 }
705
706 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_FORCE_RESET;
707 msg.handle = vgpu_get_handle(ch->g);
708 p->handle = ch->virt_ctx;
709 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
710 WARN_ON(err || msg.ret);
711 if (!err)
712 gk20a_channel_abort(ch, false);
713 return err ? err : msg.ret;
714}
715
716static void vgpu_fifo_set_ctx_mmu_error_ch(struct gk20a *g,
717 struct channel_gk20a *ch)
718{
719 nvgpu_mutex_acquire(&ch->error_notifier_mutex);
720 if (ch->error_notifier_ref) {
721 if (ch->error_notifier->status == 0xffff) {
722 /* If error code is already set, this mmu fault
723 * was triggered as part of recovery from other
724 * error condition.
725 * Don't overwrite error flag. */
726 } else {
727 gk20a_set_error_notifier_locked(ch,
728 NVGPU_CHANNEL_FIFO_ERROR_MMU_ERR_FLT);
729 }
730 }
731 nvgpu_mutex_release(&ch->error_notifier_mutex);
732
733 /* mark channel as faulted */
734 ch->has_timedout = true;
735 nvgpu_smp_wmb();
736 /* unblock pending waits */
737 nvgpu_cond_broadcast_interruptible(&ch->semaphore_wq);
738 nvgpu_cond_broadcast_interruptible(&ch->notifier_wq);
739}
740
741static void vgpu_fifo_set_ctx_mmu_error_ch_tsg(struct gk20a *g,
742 struct channel_gk20a *ch)
743{
744 struct tsg_gk20a *tsg = NULL;
745 struct channel_gk20a *ch_tsg = NULL;
746
747 if (gk20a_is_channel_marked_as_tsg(ch)) {
748 tsg = &g->fifo.tsg[ch->tsgid];
749
750 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
751
752 list_for_each_entry(ch_tsg, &tsg->ch_list, ch_entry) {
753 if (gk20a_channel_get(ch_tsg)) {
754 vgpu_fifo_set_ctx_mmu_error_ch(g, ch_tsg);
755 gk20a_channel_put(ch_tsg);
756 }
757 }
758
759 nvgpu_rwsem_up_read(&tsg->ch_list_lock);
760 } else {
761 vgpu_fifo_set_ctx_mmu_error_ch(g, ch);
762 }
763}
764
765int vgpu_fifo_isr(struct gk20a *g, struct tegra_vgpu_fifo_intr_info *info)
766{
767 struct fifo_gk20a *f = &g->fifo;
768 struct channel_gk20a *ch = gk20a_channel_get(&f->channel[info->chid]);
769
770 gk20a_dbg_fn("");
771 if (!ch)
772 return 0;
773
774 nvgpu_err(g, "fifo intr (%d) on ch %u",
775 info->type, info->chid);
776
777 trace_gk20a_channel_reset(ch->chid, ch->tsgid);
778
779 switch (info->type) {
780 case TEGRA_VGPU_FIFO_INTR_PBDMA:
781 gk20a_set_error_notifier(ch, NVGPU_CHANNEL_PBDMA_ERROR);
782 break;
783 case TEGRA_VGPU_FIFO_INTR_CTXSW_TIMEOUT:
784 gk20a_set_error_notifier(ch,
785 NVGPU_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT);
786 break;
787 case TEGRA_VGPU_FIFO_INTR_MMU_FAULT:
788 vgpu_fifo_set_ctx_mmu_error_ch_tsg(g, ch);
789 gk20a_channel_abort(ch, false);
790 break;
791 default:
792 WARN_ON(1);
793 break;
794 }
795
796 gk20a_channel_put(ch);
797 return 0;
798}
799
800int vgpu_fifo_nonstall_isr(struct gk20a *g,
801 struct tegra_vgpu_fifo_nonstall_intr_info *info)
802{
803 gk20a_dbg_fn("");
804
805 switch (info->type) {
806 case TEGRA_VGPU_FIFO_NONSTALL_INTR_CHANNEL:
807 gk20a_channel_semaphore_wakeup(g, false);
808 break;
809 default:
810 WARN_ON(1);
811 break;
812 }
813
814 return 0;
815}
816
817u32 vgpu_fifo_default_timeslice_us(struct gk20a *g)
818{
819 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
820
821 return priv->constants.default_timeslice_us;
822}
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-07 15:25:03 -0400