diff options
Diffstat (limited to 'drivers/gpu/nvgpu/vgpu/fifo_vgpu.c')
-rw-r--r-- | drivers/gpu/nvgpu/vgpu/fifo_vgpu.c | 826 |
1 files changed, 826 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c new file mode 100644 index 00000000..66b2dbef --- /dev/null +++ b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c | |||
@@ -0,0 +1,826 @@ | |||
1 | /* | ||
2 | * Virtualized GPU Fifo | ||
3 | * | ||
4 | * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/dma-mapping.h> | ||
26 | #include <trace/events/gk20a.h> | ||
27 | #include <uapi/linux/nvgpu.h> | ||
28 | |||
29 | #include <nvgpu/kmem.h> | ||
30 | #include <nvgpu/dma.h> | ||
31 | #include <nvgpu/atomic.h> | ||
32 | #include <nvgpu/bug.h> | ||
33 | #include <nvgpu/barrier.h> | ||
34 | |||
35 | #include "vgpu/vgpu.h" | ||
36 | #include "vgpu/fifo_vgpu.h" | ||
37 | |||
38 | #include <nvgpu/hw/gk20a/hw_fifo_gk20a.h> | ||
39 | #include <nvgpu/hw/gk20a/hw_ram_gk20a.h> | ||
40 | |||
41 | void vgpu_channel_bind(struct channel_gk20a *ch) | ||
42 | { | ||
43 | struct tegra_vgpu_cmd_msg msg; | ||
44 | struct tegra_vgpu_channel_config_params *p = | ||
45 | &msg.params.channel_config; | ||
46 | int err; | ||
47 | |||
48 | gk20a_dbg_info("bind channel %d", ch->chid); | ||
49 | |||
50 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_BIND; | ||
51 | msg.handle = vgpu_get_handle(ch->g); | ||
52 | p->handle = ch->virt_ctx; | ||
53 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
54 | WARN_ON(err || msg.ret); | ||
55 | |||
56 | nvgpu_smp_wmb(); | ||
57 | nvgpu_atomic_set(&ch->bound, true); | ||
58 | } | ||
59 | |||
60 | void vgpu_channel_unbind(struct channel_gk20a *ch) | ||
61 | { | ||
62 | |||
63 | gk20a_dbg_fn(""); | ||
64 | |||
65 | if (nvgpu_atomic_cmpxchg(&ch->bound, true, false)) { | ||
66 | struct tegra_vgpu_cmd_msg msg; | ||
67 | struct tegra_vgpu_channel_config_params *p = | ||
68 | &msg.params.channel_config; | ||
69 | int err; | ||
70 | |||
71 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_UNBIND; | ||
72 | msg.handle = vgpu_get_handle(ch->g); | ||
73 | p->handle = ch->virt_ctx; | ||
74 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
75 | WARN_ON(err || msg.ret); | ||
76 | } | ||
77 | |||
78 | } | ||
79 | |||
80 | int vgpu_channel_alloc_inst(struct gk20a *g, struct channel_gk20a *ch) | ||
81 | { | ||
82 | struct tegra_vgpu_cmd_msg msg; | ||
83 | struct tegra_vgpu_channel_hwctx_params *p = &msg.params.channel_hwctx; | ||
84 | int err; | ||
85 | |||
86 | gk20a_dbg_fn(""); | ||
87 | |||
88 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_ALLOC_HWCTX; | ||
89 | msg.handle = vgpu_get_handle(g); | ||
90 | p->id = ch->chid; | ||
91 | p->pid = (u64)current->tgid; | ||
92 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
93 | if (err || msg.ret) { | ||
94 | nvgpu_err(g, "fail"); | ||
95 | return -ENOMEM; | ||
96 | } | ||
97 | |||
98 | ch->virt_ctx = p->handle; | ||
99 | gk20a_dbg_fn("done"); | ||
100 | return 0; | ||
101 | } | ||
102 | |||
103 | void vgpu_channel_free_inst(struct gk20a *g, struct channel_gk20a *ch) | ||
104 | { | ||
105 | struct tegra_vgpu_cmd_msg msg; | ||
106 | struct tegra_vgpu_channel_hwctx_params *p = &msg.params.channel_hwctx; | ||
107 | int err; | ||
108 | |||
109 | gk20a_dbg_fn(""); | ||
110 | |||
111 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_FREE_HWCTX; | ||
112 | msg.handle = vgpu_get_handle(g); | ||
113 | p->handle = ch->virt_ctx; | ||
114 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
115 | WARN_ON(err || msg.ret); | ||
116 | } | ||
117 | |||
118 | void vgpu_channel_enable(struct channel_gk20a *ch) | ||
119 | { | ||
120 | struct tegra_vgpu_cmd_msg msg; | ||
121 | struct tegra_vgpu_channel_config_params *p = | ||
122 | &msg.params.channel_config; | ||
123 | int err; | ||
124 | |||
125 | gk20a_dbg_fn(""); | ||
126 | |||
127 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_ENABLE; | ||
128 | msg.handle = vgpu_get_handle(ch->g); | ||
129 | p->handle = ch->virt_ctx; | ||
130 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
131 | WARN_ON(err || msg.ret); | ||
132 | } | ||
133 | |||
134 | void vgpu_channel_disable(struct channel_gk20a *ch) | ||
135 | { | ||
136 | struct tegra_vgpu_cmd_msg msg; | ||
137 | struct tegra_vgpu_channel_config_params *p = | ||
138 | &msg.params.channel_config; | ||
139 | int err; | ||
140 | |||
141 | gk20a_dbg_fn(""); | ||
142 | |||
143 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_DISABLE; | ||
144 | msg.handle = vgpu_get_handle(ch->g); | ||
145 | p->handle = ch->virt_ctx; | ||
146 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
147 | WARN_ON(err || msg.ret); | ||
148 | } | ||
149 | |||
150 | int vgpu_channel_setup_ramfc(struct channel_gk20a *ch, u64 gpfifo_base, | ||
151 | u32 gpfifo_entries, | ||
152 | unsigned long acquire_timeout, u32 flags) | ||
153 | { | ||
154 | struct device __maybe_unused *d = dev_from_gk20a(ch->g); | ||
155 | struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(d); | ||
156 | struct tegra_vgpu_cmd_msg msg; | ||
157 | struct tegra_vgpu_ramfc_params *p = &msg.params.ramfc; | ||
158 | int err; | ||
159 | |||
160 | gk20a_dbg_fn(""); | ||
161 | |||
162 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SETUP_RAMFC; | ||
163 | msg.handle = vgpu_get_handle(ch->g); | ||
164 | p->handle = ch->virt_ctx; | ||
165 | p->gpfifo_va = gpfifo_base; | ||
166 | p->num_entries = gpfifo_entries; | ||
167 | p->userd_addr = ch->userd_iova; | ||
168 | p->iova = mapping ? 1 : 0; | ||
169 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
170 | |||
171 | return (err || msg.ret) ? -ENOMEM : 0; | ||
172 | } | ||
173 | |||
174 | int vgpu_fifo_init_engine_info(struct fifo_gk20a *f) | ||
175 | { | ||
176 | struct vgpu_priv_data *priv = vgpu_get_priv_data(f->g); | ||
177 | struct tegra_vgpu_engines_info *engines = &priv->constants.engines_info; | ||
178 | u32 i; | ||
179 | |||
180 | gk20a_dbg_fn(""); | ||
181 | |||
182 | if (engines->num_engines > TEGRA_VGPU_MAX_ENGINES) { | ||
183 | nvgpu_err(f->g, "num_engines %d larger than max %d", | ||
184 | engines->num_engines, TEGRA_VGPU_MAX_ENGINES); | ||
185 | return -EINVAL; | ||
186 | } | ||
187 | |||
188 | f->num_engines = engines->num_engines; | ||
189 | for (i = 0; i < f->num_engines; i++) { | ||
190 | struct fifo_engine_info_gk20a *info = | ||
191 | &f->engine_info[engines->info[i].engine_id]; | ||
192 | |||
193 | if (engines->info[i].engine_id >= f->max_engines) { | ||
194 | nvgpu_err(f->g, "engine id %d larger than max %d", | ||
195 | engines->info[i].engine_id, | ||
196 | f->max_engines); | ||
197 | return -EINVAL; | ||
198 | } | ||
199 | |||
200 | info->intr_mask = engines->info[i].intr_mask; | ||
201 | info->reset_mask = engines->info[i].reset_mask; | ||
202 | info->runlist_id = engines->info[i].runlist_id; | ||
203 | info->pbdma_id = engines->info[i].pbdma_id; | ||
204 | info->inst_id = engines->info[i].inst_id; | ||
205 | info->pri_base = engines->info[i].pri_base; | ||
206 | info->engine_enum = engines->info[i].engine_enum; | ||
207 | info->fault_id = engines->info[i].fault_id; | ||
208 | f->active_engines_list[i] = engines->info[i].engine_id; | ||
209 | } | ||
210 | |||
211 | gk20a_dbg_fn("done"); | ||
212 | |||
213 | return 0; | ||
214 | } | ||
215 | |||
216 | static int init_runlist(struct gk20a *g, struct fifo_gk20a *f) | ||
217 | { | ||
218 | struct fifo_runlist_info_gk20a *runlist; | ||
219 | struct device *d = dev_from_gk20a(g); | ||
220 | unsigned int runlist_id = -1; | ||
221 | u32 i; | ||
222 | u64 runlist_size; | ||
223 | |||
224 | gk20a_dbg_fn(""); | ||
225 | |||
226 | f->max_runlists = g->ops.fifo.eng_runlist_base_size(); | ||
227 | f->runlist_info = nvgpu_kzalloc(g, | ||
228 | sizeof(struct fifo_runlist_info_gk20a) * | ||
229 | f->max_runlists); | ||
230 | if (!f->runlist_info) | ||
231 | goto clean_up_runlist; | ||
232 | |||
233 | memset(f->runlist_info, 0, (sizeof(struct fifo_runlist_info_gk20a) * | ||
234 | f->max_runlists)); | ||
235 | |||
236 | for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) { | ||
237 | runlist = &f->runlist_info[runlist_id]; | ||
238 | |||
239 | runlist->active_channels = | ||
240 | nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, | ||
241 | BITS_PER_BYTE)); | ||
242 | if (!runlist->active_channels) | ||
243 | goto clean_up_runlist; | ||
244 | |||
245 | runlist_size = sizeof(u16) * f->num_channels; | ||
246 | for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) { | ||
247 | int err = nvgpu_dma_alloc_sys(g, runlist_size, | ||
248 | &runlist->mem[i]); | ||
249 | if (err) { | ||
250 | dev_err(d, "memory allocation failed\n"); | ||
251 | goto clean_up_runlist; | ||
252 | } | ||
253 | } | ||
254 | nvgpu_mutex_init(&runlist->mutex); | ||
255 | |||
256 | /* None of buffers is pinned if this value doesn't change. | ||
257 | Otherwise, one of them (cur_buffer) must have been pinned. */ | ||
258 | runlist->cur_buffer = MAX_RUNLIST_BUFFERS; | ||
259 | } | ||
260 | |||
261 | gk20a_dbg_fn("done"); | ||
262 | return 0; | ||
263 | |||
264 | clean_up_runlist: | ||
265 | gk20a_fifo_delete_runlist(f); | ||
266 | gk20a_dbg_fn("fail"); | ||
267 | return -ENOMEM; | ||
268 | } | ||
269 | |||
270 | static int vgpu_init_fifo_setup_sw(struct gk20a *g) | ||
271 | { | ||
272 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
273 | struct fifo_gk20a *f = &g->fifo; | ||
274 | struct device *d = dev_from_gk20a(g); | ||
275 | struct vgpu_priv_data *priv = vgpu_get_priv_data(g); | ||
276 | unsigned int chid; | ||
277 | int err = 0; | ||
278 | |||
279 | gk20a_dbg_fn(""); | ||
280 | |||
281 | if (f->sw_ready) { | ||
282 | gk20a_dbg_fn("skip init"); | ||
283 | return 0; | ||
284 | } | ||
285 | |||
286 | f->g = g; | ||
287 | f->num_channels = priv->constants.num_channels; | ||
288 | f->max_engines = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_ENGINES); | ||
289 | |||
290 | f->userd_entry_size = 1 << ram_userd_base_shift_v(); | ||
291 | |||
292 | err = nvgpu_dma_alloc_sys(g, f->userd_entry_size * f->num_channels, | ||
293 | &f->userd); | ||
294 | if (err) { | ||
295 | dev_err(d, "memory allocation failed\n"); | ||
296 | goto clean_up; | ||
297 | } | ||
298 | |||
299 | /* bar1 va */ | ||
300 | if (g->ops.mm.is_bar1_supported(g)) { | ||
301 | f->userd.gpu_va = vgpu_bar1_map(g, &f->userd.priv.sgt, | ||
302 | f->userd.size); | ||
303 | if (!f->userd.gpu_va) { | ||
304 | dev_err(d, "gmmu mapping failed\n"); | ||
305 | goto clean_up; | ||
306 | } | ||
307 | /* if reduced BAR1 range is specified, use offset of 0 | ||
308 | * (server returns offset assuming full BAR1 range) | ||
309 | */ | ||
310 | if (resource_size(l->bar1_mem) == | ||
311 | (resource_size_t)f->userd.size) | ||
312 | f->userd.gpu_va = 0; | ||
313 | } | ||
314 | |||
315 | gk20a_dbg(gpu_dbg_map_v, "userd bar1 va = 0x%llx", f->userd.gpu_va); | ||
316 | |||
317 | f->channel = nvgpu_vzalloc(g, f->num_channels * sizeof(*f->channel)); | ||
318 | f->tsg = nvgpu_vzalloc(g, f->num_channels * sizeof(*f->tsg)); | ||
319 | f->engine_info = nvgpu_kzalloc(g, f->max_engines * | ||
320 | sizeof(*f->engine_info)); | ||
321 | f->active_engines_list = nvgpu_kzalloc(g, f->max_engines * sizeof(u32)); | ||
322 | |||
323 | if (!(f->channel && f->tsg && f->engine_info && f->active_engines_list)) { | ||
324 | err = -ENOMEM; | ||
325 | goto clean_up; | ||
326 | } | ||
327 | memset(f->active_engines_list, 0xff, (f->max_engines * sizeof(u32))); | ||
328 | |||
329 | g->ops.fifo.init_engine_info(f); | ||
330 | |||
331 | init_runlist(g, f); | ||
332 | |||
333 | nvgpu_init_list_node(&f->free_chs); | ||
334 | nvgpu_mutex_init(&f->free_chs_mutex); | ||
335 | |||
336 | for (chid = 0; chid < f->num_channels; chid++) { | ||
337 | f->channel[chid].userd_iova = | ||
338 | nvgpu_mem_get_addr(g, &f->userd) + | ||
339 | chid * f->userd_entry_size; | ||
340 | f->channel[chid].userd_gpu_va = | ||
341 | f->userd.gpu_va + chid * f->userd_entry_size; | ||
342 | |||
343 | gk20a_init_channel_support(g, chid); | ||
344 | gk20a_init_tsg_support(g, chid); | ||
345 | } | ||
346 | nvgpu_mutex_init(&f->tsg_inuse_mutex); | ||
347 | |||
348 | err = nvgpu_channel_worker_init(g); | ||
349 | if (err) | ||
350 | goto clean_up; | ||
351 | |||
352 | f->deferred_reset_pending = false; | ||
353 | nvgpu_mutex_init(&f->deferred_reset_mutex); | ||
354 | |||
355 | f->channel_base = priv->constants.channel_base; | ||
356 | |||
357 | f->sw_ready = true; | ||
358 | |||
359 | gk20a_dbg_fn("done"); | ||
360 | return 0; | ||
361 | |||
362 | clean_up: | ||
363 | gk20a_dbg_fn("fail"); | ||
364 | /* FIXME: unmap from bar1 */ | ||
365 | nvgpu_dma_free(g, &f->userd); | ||
366 | |||
367 | memset(&f->userd, 0, sizeof(f->userd)); | ||
368 | |||
369 | nvgpu_vfree(g, f->channel); | ||
370 | f->channel = NULL; | ||
371 | nvgpu_vfree(g, f->tsg); | ||
372 | f->tsg = NULL; | ||
373 | nvgpu_kfree(g, f->engine_info); | ||
374 | f->engine_info = NULL; | ||
375 | nvgpu_kfree(g, f->active_engines_list); | ||
376 | f->active_engines_list = NULL; | ||
377 | |||
378 | return err; | ||
379 | } | ||
380 | |||
381 | int vgpu_init_fifo_setup_hw(struct gk20a *g) | ||
382 | { | ||
383 | gk20a_dbg_fn(""); | ||
384 | |||
385 | /* test write, read through bar1 @ userd region before | ||
386 | * turning on the snooping */ | ||
387 | { | ||
388 | struct fifo_gk20a *f = &g->fifo; | ||
389 | u32 v, v1 = 0x33, v2 = 0x55; | ||
390 | |||
391 | u32 bar1_vaddr = f->userd.gpu_va; | ||
392 | volatile u32 *cpu_vaddr = f->userd.cpu_va; | ||
393 | |||
394 | gk20a_dbg_info("test bar1 @ vaddr 0x%x", | ||
395 | bar1_vaddr); | ||
396 | |||
397 | v = gk20a_bar1_readl(g, bar1_vaddr); | ||
398 | |||
399 | *cpu_vaddr = v1; | ||
400 | nvgpu_mb(); | ||
401 | |||
402 | if (v1 != gk20a_bar1_readl(g, bar1_vaddr)) { | ||
403 | nvgpu_err(g, "bar1 broken @ gk20a!"); | ||
404 | return -EINVAL; | ||
405 | } | ||
406 | |||
407 | gk20a_bar1_writel(g, bar1_vaddr, v2); | ||
408 | |||
409 | if (v2 != gk20a_bar1_readl(g, bar1_vaddr)) { | ||
410 | nvgpu_err(g, "bar1 broken @ gk20a!"); | ||
411 | return -EINVAL; | ||
412 | } | ||
413 | |||
414 | /* is it visible to the cpu? */ | ||
415 | if (*cpu_vaddr != v2) { | ||
416 | nvgpu_err(g, "cpu didn't see bar1 write @ %p!", | ||
417 | cpu_vaddr); | ||
418 | } | ||
419 | |||
420 | /* put it back */ | ||
421 | gk20a_bar1_writel(g, bar1_vaddr, v); | ||
422 | } | ||
423 | |||
424 | gk20a_dbg_fn("done"); | ||
425 | |||
426 | return 0; | ||
427 | } | ||
428 | |||
429 | int vgpu_init_fifo_support(struct gk20a *g) | ||
430 | { | ||
431 | u32 err; | ||
432 | |||
433 | gk20a_dbg_fn(""); | ||
434 | |||
435 | err = vgpu_init_fifo_setup_sw(g); | ||
436 | if (err) | ||
437 | return err; | ||
438 | |||
439 | if (g->ops.fifo.init_fifo_setup_hw) | ||
440 | err = g->ops.fifo.init_fifo_setup_hw(g); | ||
441 | return err; | ||
442 | } | ||
443 | |||
444 | int vgpu_fifo_preempt_channel(struct gk20a *g, u32 chid) | ||
445 | { | ||
446 | struct fifo_gk20a *f = &g->fifo; | ||
447 | struct channel_gk20a *ch = &f->channel[chid]; | ||
448 | struct tegra_vgpu_cmd_msg msg; | ||
449 | struct tegra_vgpu_channel_config_params *p = | ||
450 | &msg.params.channel_config; | ||
451 | int err; | ||
452 | |||
453 | gk20a_dbg_fn(""); | ||
454 | |||
455 | if (!nvgpu_atomic_read(&ch->bound)) | ||
456 | return 0; | ||
457 | |||
458 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_PREEMPT; | ||
459 | msg.handle = vgpu_get_handle(g); | ||
460 | p->handle = ch->virt_ctx; | ||
461 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
462 | |||
463 | if (err || msg.ret) { | ||
464 | nvgpu_err(g, | ||
465 | "preempt channel %d failed", chid); | ||
466 | err = -ENOMEM; | ||
467 | } | ||
468 | |||
469 | return err; | ||
470 | } | ||
471 | |||
472 | int vgpu_fifo_preempt_tsg(struct gk20a *g, u32 tsgid) | ||
473 | { | ||
474 | struct tegra_vgpu_cmd_msg msg; | ||
475 | struct tegra_vgpu_tsg_preempt_params *p = | ||
476 | &msg.params.tsg_preempt; | ||
477 | int err; | ||
478 | |||
479 | gk20a_dbg_fn(""); | ||
480 | |||
481 | msg.cmd = TEGRA_VGPU_CMD_TSG_PREEMPT; | ||
482 | msg.handle = vgpu_get_handle(g); | ||
483 | p->tsg_id = tsgid; | ||
484 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
485 | err = err ? err : msg.ret; | ||
486 | |||
487 | if (err) { | ||
488 | nvgpu_err(g, | ||
489 | "preempt tsg %u failed", tsgid); | ||
490 | } | ||
491 | |||
492 | return err; | ||
493 | } | ||
494 | |||
495 | static int vgpu_submit_runlist(struct gk20a *g, u64 handle, u8 runlist_id, | ||
496 | u16 *runlist, u32 num_entries) | ||
497 | { | ||
498 | struct tegra_vgpu_cmd_msg msg; | ||
499 | struct tegra_vgpu_runlist_params *p; | ||
500 | int err; | ||
501 | void *oob_handle; | ||
502 | void *oob; | ||
503 | size_t size, oob_size; | ||
504 | |||
505 | oob_handle = tegra_gr_comm_oob_get_ptr(TEGRA_GR_COMM_CTX_CLIENT, | ||
506 | tegra_gr_comm_get_server_vmid(), TEGRA_VGPU_QUEUE_CMD, | ||
507 | &oob, &oob_size); | ||
508 | if (!oob_handle) | ||
509 | return -EINVAL; | ||
510 | |||
511 | size = sizeof(*runlist) * num_entries; | ||
512 | if (oob_size < size) { | ||
513 | err = -ENOMEM; | ||
514 | goto done; | ||
515 | } | ||
516 | |||
517 | msg.cmd = TEGRA_VGPU_CMD_SUBMIT_RUNLIST; | ||
518 | msg.handle = handle; | ||
519 | p = &msg.params.runlist; | ||
520 | p->runlist_id = runlist_id; | ||
521 | p->num_entries = num_entries; | ||
522 | |||
523 | memcpy(oob, runlist, size); | ||
524 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
525 | |||
526 | err = (err || msg.ret) ? -1 : 0; | ||
527 | |||
528 | done: | ||
529 | tegra_gr_comm_oob_put_ptr(oob_handle); | ||
530 | return err; | ||
531 | } | ||
532 | |||
533 | static int vgpu_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, | ||
534 | u32 chid, bool add, | ||
535 | bool wait_for_finish) | ||
536 | { | ||
537 | struct fifo_gk20a *f = &g->fifo; | ||
538 | struct fifo_runlist_info_gk20a *runlist; | ||
539 | u16 *runlist_entry = NULL; | ||
540 | u32 count = 0; | ||
541 | |||
542 | gk20a_dbg_fn(""); | ||
543 | |||
544 | runlist = &f->runlist_info[runlist_id]; | ||
545 | |||
546 | /* valid channel, add/remove it from active list. | ||
547 | Otherwise, keep active list untouched for suspend/resume. */ | ||
548 | if (chid != (u32)~0) { | ||
549 | if (add) { | ||
550 | if (test_and_set_bit(chid, | ||
551 | runlist->active_channels) == 1) | ||
552 | return 0; | ||
553 | } else { | ||
554 | if (test_and_clear_bit(chid, | ||
555 | runlist->active_channels) == 0) | ||
556 | return 0; | ||
557 | } | ||
558 | } | ||
559 | |||
560 | if (chid != (u32)~0 || /* add/remove a valid channel */ | ||
561 | add /* resume to add all channels back */) { | ||
562 | u32 cid; | ||
563 | |||
564 | runlist_entry = runlist->mem[0].cpu_va; | ||
565 | for_each_set_bit(cid, | ||
566 | runlist->active_channels, f->num_channels) { | ||
567 | gk20a_dbg_info("add channel %d to runlist", cid); | ||
568 | runlist_entry[0] = cid; | ||
569 | runlist_entry++; | ||
570 | count++; | ||
571 | } | ||
572 | } else /* suspend to remove all channels */ | ||
573 | count = 0; | ||
574 | |||
575 | return vgpu_submit_runlist(g, vgpu_get_handle(g), runlist_id, | ||
576 | runlist->mem[0].cpu_va, count); | ||
577 | } | ||
578 | |||
579 | /* add/remove a channel from runlist | ||
580 | special cases below: runlist->active_channels will NOT be changed. | ||
581 | (chid == ~0 && !add) means remove all active channels from runlist. | ||
582 | (chid == ~0 && add) means restore all active channels on runlist. */ | ||
583 | int vgpu_fifo_update_runlist(struct gk20a *g, u32 runlist_id, | ||
584 | u32 chid, bool add, bool wait_for_finish) | ||
585 | { | ||
586 | struct fifo_runlist_info_gk20a *runlist = NULL; | ||
587 | struct fifo_gk20a *f = &g->fifo; | ||
588 | u32 ret = 0; | ||
589 | |||
590 | gk20a_dbg_fn(""); | ||
591 | |||
592 | runlist = &f->runlist_info[runlist_id]; | ||
593 | |||
594 | nvgpu_mutex_acquire(&runlist->mutex); | ||
595 | |||
596 | ret = vgpu_fifo_update_runlist_locked(g, runlist_id, chid, add, | ||
597 | wait_for_finish); | ||
598 | |||
599 | nvgpu_mutex_release(&runlist->mutex); | ||
600 | return ret; | ||
601 | } | ||
602 | |||
603 | int vgpu_fifo_wait_engine_idle(struct gk20a *g) | ||
604 | { | ||
605 | gk20a_dbg_fn(""); | ||
606 | |||
607 | return 0; | ||
608 | } | ||
609 | |||
610 | static int vgpu_fifo_tsg_set_runlist_interleave(struct gk20a *g, | ||
611 | u32 tsgid, | ||
612 | u32 runlist_id, | ||
613 | u32 new_level) | ||
614 | { | ||
615 | struct tegra_vgpu_cmd_msg msg = {0}; | ||
616 | struct tegra_vgpu_tsg_runlist_interleave_params *p = | ||
617 | &msg.params.tsg_interleave; | ||
618 | int err; | ||
619 | |||
620 | gk20a_dbg_fn(""); | ||
621 | |||
622 | msg.cmd = TEGRA_VGPU_CMD_TSG_SET_RUNLIST_INTERLEAVE; | ||
623 | msg.handle = vgpu_get_handle(g); | ||
624 | p->tsg_id = tsgid; | ||
625 | p->level = new_level; | ||
626 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
627 | WARN_ON(err || msg.ret); | ||
628 | return err ? err : msg.ret; | ||
629 | } | ||
630 | |||
631 | int vgpu_fifo_set_runlist_interleave(struct gk20a *g, | ||
632 | u32 id, | ||
633 | bool is_tsg, | ||
634 | u32 runlist_id, | ||
635 | u32 new_level) | ||
636 | { | ||
637 | struct tegra_vgpu_cmd_msg msg; | ||
638 | struct tegra_vgpu_channel_runlist_interleave_params *p = | ||
639 | &msg.params.channel_interleave; | ||
640 | struct channel_gk20a *ch; | ||
641 | int err; | ||
642 | |||
643 | gk20a_dbg_fn(""); | ||
644 | |||
645 | if (is_tsg) | ||
646 | return vgpu_fifo_tsg_set_runlist_interleave(g, id, | ||
647 | runlist_id, new_level); | ||
648 | |||
649 | ch = &g->fifo.channel[id]; | ||
650 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SET_RUNLIST_INTERLEAVE; | ||
651 | msg.handle = vgpu_get_handle(ch->g); | ||
652 | p->handle = ch->virt_ctx; | ||
653 | p->level = new_level; | ||
654 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
655 | WARN_ON(err || msg.ret); | ||
656 | return err ? err : msg.ret; | ||
657 | } | ||
658 | |||
659 | int vgpu_channel_set_timeslice(struct channel_gk20a *ch, u32 timeslice) | ||
660 | { | ||
661 | struct tegra_vgpu_cmd_msg msg; | ||
662 | struct tegra_vgpu_channel_timeslice_params *p = | ||
663 | &msg.params.channel_timeslice; | ||
664 | int err; | ||
665 | |||
666 | gk20a_dbg_fn(""); | ||
667 | |||
668 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SET_TIMESLICE; | ||
669 | msg.handle = vgpu_get_handle(ch->g); | ||
670 | p->handle = ch->virt_ctx; | ||
671 | p->timeslice_us = timeslice; | ||
672 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
673 | err = err ? err : msg.ret; | ||
674 | WARN_ON(err); | ||
675 | if (!err) | ||
676 | ch->timeslice_us = p->timeslice_us; | ||
677 | return err; | ||
678 | } | ||
679 | |||
680 | int vgpu_fifo_force_reset_ch(struct channel_gk20a *ch, | ||
681 | u32 err_code, bool verbose) | ||
682 | { | ||
683 | struct tsg_gk20a *tsg = NULL; | ||
684 | struct channel_gk20a *ch_tsg = NULL; | ||
685 | struct gk20a *g = ch->g; | ||
686 | struct tegra_vgpu_cmd_msg msg = {0}; | ||
687 | struct tegra_vgpu_channel_config_params *p = | ||
688 | &msg.params.channel_config; | ||
689 | int err; | ||
690 | |||
691 | gk20a_dbg_fn(""); | ||
692 | |||
693 | if (gk20a_is_channel_marked_as_tsg(ch)) { | ||
694 | tsg = &g->fifo.tsg[ch->tsgid]; | ||
695 | |||
696 | nvgpu_rwsem_down_read(&tsg->ch_list_lock); | ||
697 | |||
698 | list_for_each_entry(ch_tsg, &tsg->ch_list, ch_entry) { | ||
699 | if (gk20a_channel_get(ch_tsg)) { | ||
700 | gk20a_set_error_notifier(ch_tsg, err_code); | ||
701 | ch_tsg->has_timedout = true; | ||
702 | gk20a_channel_put(ch_tsg); | ||
703 | } | ||
704 | } | ||
705 | |||
706 | nvgpu_rwsem_up_read(&tsg->ch_list_lock); | ||
707 | } else { | ||
708 | gk20a_set_error_notifier(ch, err_code); | ||
709 | ch->has_timedout = true; | ||
710 | } | ||
711 | |||
712 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_FORCE_RESET; | ||
713 | msg.handle = vgpu_get_handle(ch->g); | ||
714 | p->handle = ch->virt_ctx; | ||
715 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
716 | WARN_ON(err || msg.ret); | ||
717 | return err ? err : msg.ret; | ||
718 | } | ||
719 | |||
720 | static void vgpu_fifo_set_ctx_mmu_error_ch(struct gk20a *g, | ||
721 | struct channel_gk20a *ch) | ||
722 | { | ||
723 | nvgpu_mutex_acquire(&ch->error_notifier_mutex); | ||
724 | if (ch->error_notifier_ref) { | ||
725 | if (ch->error_notifier->status == 0xffff) { | ||
726 | /* If error code is already set, this mmu fault | ||
727 | * was triggered as part of recovery from other | ||
728 | * error condition. | ||
729 | * Don't overwrite error flag. */ | ||
730 | } else { | ||
731 | gk20a_set_error_notifier_locked(ch, | ||
732 | NVGPU_CHANNEL_FIFO_ERROR_MMU_ERR_FLT); | ||
733 | } | ||
734 | } | ||
735 | nvgpu_mutex_release(&ch->error_notifier_mutex); | ||
736 | |||
737 | /* mark channel as faulted */ | ||
738 | ch->has_timedout = true; | ||
739 | nvgpu_smp_wmb(); | ||
740 | /* unblock pending waits */ | ||
741 | nvgpu_cond_broadcast_interruptible(&ch->semaphore_wq); | ||
742 | nvgpu_cond_broadcast_interruptible(&ch->notifier_wq); | ||
743 | } | ||
744 | |||
745 | static void vgpu_fifo_set_ctx_mmu_error_ch_tsg(struct gk20a *g, | ||
746 | struct channel_gk20a *ch) | ||
747 | { | ||
748 | struct tsg_gk20a *tsg = NULL; | ||
749 | struct channel_gk20a *ch_tsg = NULL; | ||
750 | |||
751 | if (gk20a_is_channel_marked_as_tsg(ch)) { | ||
752 | tsg = &g->fifo.tsg[ch->tsgid]; | ||
753 | |||
754 | nvgpu_rwsem_down_read(&tsg->ch_list_lock); | ||
755 | |||
756 | list_for_each_entry(ch_tsg, &tsg->ch_list, ch_entry) { | ||
757 | if (gk20a_channel_get(ch_tsg)) { | ||
758 | vgpu_fifo_set_ctx_mmu_error_ch(g, ch_tsg); | ||
759 | gk20a_channel_put(ch_tsg); | ||
760 | } | ||
761 | } | ||
762 | |||
763 | nvgpu_rwsem_up_read(&tsg->ch_list_lock); | ||
764 | } else { | ||
765 | vgpu_fifo_set_ctx_mmu_error_ch(g, ch); | ||
766 | } | ||
767 | } | ||
768 | |||
769 | int vgpu_fifo_isr(struct gk20a *g, struct tegra_vgpu_fifo_intr_info *info) | ||
770 | { | ||
771 | struct fifo_gk20a *f = &g->fifo; | ||
772 | struct channel_gk20a *ch = gk20a_channel_get(&f->channel[info->chid]); | ||
773 | |||
774 | gk20a_dbg_fn(""); | ||
775 | if (!ch) | ||
776 | return 0; | ||
777 | |||
778 | nvgpu_err(g, "fifo intr (%d) on ch %u", | ||
779 | info->type, info->chid); | ||
780 | |||
781 | trace_gk20a_channel_reset(ch->chid, ch->tsgid); | ||
782 | |||
783 | switch (info->type) { | ||
784 | case TEGRA_VGPU_FIFO_INTR_PBDMA: | ||
785 | gk20a_set_error_notifier(ch, NVGPU_CHANNEL_PBDMA_ERROR); | ||
786 | break; | ||
787 | case TEGRA_VGPU_FIFO_INTR_CTXSW_TIMEOUT: | ||
788 | gk20a_set_error_notifier(ch, | ||
789 | NVGPU_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT); | ||
790 | break; | ||
791 | case TEGRA_VGPU_FIFO_INTR_MMU_FAULT: | ||
792 | vgpu_fifo_set_ctx_mmu_error_ch_tsg(g, ch); | ||
793 | gk20a_channel_abort(ch, false); | ||
794 | break; | ||
795 | default: | ||
796 | WARN_ON(1); | ||
797 | break; | ||
798 | } | ||
799 | |||
800 | gk20a_channel_put(ch); | ||
801 | return 0; | ||
802 | } | ||
803 | |||
804 | int vgpu_fifo_nonstall_isr(struct gk20a *g, | ||
805 | struct tegra_vgpu_fifo_nonstall_intr_info *info) | ||
806 | { | ||
807 | gk20a_dbg_fn(""); | ||
808 | |||
809 | switch (info->type) { | ||
810 | case TEGRA_VGPU_FIFO_NONSTALL_INTR_CHANNEL: | ||
811 | gk20a_channel_semaphore_wakeup(g, false); | ||
812 | break; | ||
813 | default: | ||
814 | WARN_ON(1); | ||
815 | break; | ||
816 | } | ||
817 | |||
818 | return 0; | ||
819 | } | ||
820 | |||
821 | u32 vgpu_fifo_default_timeslice_us(struct gk20a *g) | ||
822 | { | ||
823 | struct vgpu_priv_data *priv = vgpu_get_priv_data(g); | ||
824 | |||
825 | return priv->constants.default_timeslice_us; | ||
826 | } | ||