diff options
author | Deepak Nibade <dnibade@nvidia.com> | 2017-11-14 09:43:28 -0500 |
---|---|---|
committer | mobile promotions <svcmobile_promotions@nvidia.com> | 2017-11-17 11:27:19 -0500 |
commit | b42fb7ba26b565f93118fbdd9e17b42ee6144c5e (patch) | |
tree | 26e2d919f019d15b51bba4d7b5c938f77ad5cff5 /drivers/gpu/nvgpu/common/linux/vgpu/gr_vgpu.c | |
parent | b7cc3a2aa6c92a09eed43513287c9062f22ad127 (diff) |
gpu: nvgpu: move vgpu code to linux
Most of VGPU code is linux specific but lies in common code
So until VGPU code is properly abstracted and made os-independent,
move all of VGPU code to linux specific directory
Handle corresponding Makefile changes
Update all #includes to reflect new paths
Add GPL license to newly added linux files
Jira NVGPU-387
Change-Id: Ic133e4c80e570bcc273f0dacf45283fefd678923
Signed-off-by: Deepak Nibade <dnibade@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1599472
GVS: Gerrit_Virtual_Submit
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/common/linux/vgpu/gr_vgpu.c')
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/vgpu/gr_vgpu.c | 1214 |
1 files changed, 1214 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/vgpu/gr_vgpu.c b/drivers/gpu/nvgpu/common/linux/vgpu/gr_vgpu.c new file mode 100644 index 00000000..dd2ae306 --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/vgpu/gr_vgpu.c | |||
@@ -0,0 +1,1214 @@ | |||
1 | /* | ||
2 | * Virtualized GPU Graphics | ||
3 | * | ||
4 | * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * This program is free software; you can redistribute it and/or modify it | ||
7 | * under the terms and conditions of the GNU General Public License, | ||
8 | * version 2, as published by the Free Software Foundation. | ||
9 | * | ||
10 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
13 | * more details. | ||
14 | * | ||
15 | * You should have received a copy of the GNU General Public License | ||
16 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
17 | */ | ||
18 | |||
19 | #include <uapi/linux/nvgpu.h> | ||
20 | |||
21 | #include <nvgpu/kmem.h> | ||
22 | #include <nvgpu/bug.h> | ||
23 | |||
24 | #include "vgpu.h" | ||
25 | #include "gr_vgpu.h" | ||
26 | #include "gk20a/dbg_gpu_gk20a.h" | ||
27 | |||
28 | #include <nvgpu/hw/gk20a/hw_gr_gk20a.h> | ||
29 | |||
30 | void vgpu_gr_detect_sm_arch(struct gk20a *g) | ||
31 | { | ||
32 | struct vgpu_priv_data *priv = vgpu_get_priv_data(g); | ||
33 | |||
34 | gk20a_dbg_fn(""); | ||
35 | |||
36 | g->params.sm_arch_sm_version = | ||
37 | priv->constants.sm_arch_sm_version; | ||
38 | g->params.sm_arch_spa_version = | ||
39 | priv->constants.sm_arch_spa_version; | ||
40 | g->params.sm_arch_warp_count = | ||
41 | priv->constants.sm_arch_warp_count; | ||
42 | } | ||
43 | |||
44 | int vgpu_gr_commit_inst(struct channel_gk20a *c, u64 gpu_va) | ||
45 | { | ||
46 | struct tegra_vgpu_cmd_msg msg; | ||
47 | struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx; | ||
48 | int err; | ||
49 | |||
50 | gk20a_dbg_fn(""); | ||
51 | |||
52 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_COMMIT_GR_CTX; | ||
53 | msg.handle = vgpu_get_handle(c->g); | ||
54 | p->handle = c->virt_ctx; | ||
55 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
56 | |||
57 | return (err || msg.ret) ? -1 : 0; | ||
58 | } | ||
59 | |||
60 | static int vgpu_gr_commit_global_ctx_buffers(struct gk20a *g, | ||
61 | struct channel_gk20a *c, bool patch) | ||
62 | { | ||
63 | struct tegra_vgpu_cmd_msg msg; | ||
64 | struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx; | ||
65 | int err; | ||
66 | |||
67 | gk20a_dbg_fn(""); | ||
68 | |||
69 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_COMMIT_GR_GLOBAL_CTX; | ||
70 | msg.handle = vgpu_get_handle(g); | ||
71 | p->handle = c->virt_ctx; | ||
72 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
73 | |||
74 | return (err || msg.ret) ? -1 : 0; | ||
75 | } | ||
76 | |||
77 | /* load saved fresh copy of gloden image into channel gr_ctx */ | ||
78 | static int vgpu_gr_load_golden_ctx_image(struct gk20a *g, | ||
79 | struct channel_gk20a *c) | ||
80 | { | ||
81 | struct tegra_vgpu_cmd_msg msg; | ||
82 | struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx; | ||
83 | int err; | ||
84 | |||
85 | gk20a_dbg_fn(""); | ||
86 | |||
87 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_LOAD_GR_GOLDEN_CTX; | ||
88 | msg.handle = vgpu_get_handle(g); | ||
89 | p->handle = c->virt_ctx; | ||
90 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
91 | |||
92 | return (err || msg.ret) ? -1 : 0; | ||
93 | } | ||
94 | |||
95 | int vgpu_gr_init_ctx_state(struct gk20a *g) | ||
96 | { | ||
97 | struct gr_gk20a *gr = &g->gr; | ||
98 | struct vgpu_priv_data *priv = vgpu_get_priv_data(g); | ||
99 | |||
100 | gk20a_dbg_fn(""); | ||
101 | |||
102 | g->gr.ctx_vars.golden_image_size = priv->constants.golden_ctx_size; | ||
103 | g->gr.ctx_vars.zcull_ctxsw_image_size = priv->constants.zcull_ctx_size; | ||
104 | g->gr.ctx_vars.pm_ctxsw_image_size = priv->constants.hwpm_ctx_size; | ||
105 | if (!g->gr.ctx_vars.golden_image_size || | ||
106 | !g->gr.ctx_vars.zcull_ctxsw_image_size || | ||
107 | !g->gr.ctx_vars.pm_ctxsw_image_size) | ||
108 | return -ENXIO; | ||
109 | |||
110 | gr->ctx_vars.buffer_size = g->gr.ctx_vars.golden_image_size; | ||
111 | g->gr.ctx_vars.priv_access_map_size = 512 * 1024; | ||
112 | return 0; | ||
113 | } | ||
114 | |||
115 | static int vgpu_gr_alloc_global_ctx_buffers(struct gk20a *g) | ||
116 | { | ||
117 | struct gr_gk20a *gr = &g->gr; | ||
118 | int attr_buffer_size; | ||
119 | |||
120 | u32 cb_buffer_size = gr->bundle_cb_default_size * | ||
121 | gr_scc_bundle_cb_size_div_256b_byte_granularity_v(); | ||
122 | |||
123 | u32 pagepool_buffer_size = g->ops.gr.pagepool_default_size(g) * | ||
124 | gr_scc_pagepool_total_pages_byte_granularity_v(); | ||
125 | |||
126 | gk20a_dbg_fn(""); | ||
127 | |||
128 | attr_buffer_size = g->ops.gr.calc_global_ctx_buffer_size(g); | ||
129 | |||
130 | gk20a_dbg_info("cb_buffer_size : %d", cb_buffer_size); | ||
131 | gr->global_ctx_buffer[CIRCULAR].mem.size = cb_buffer_size; | ||
132 | |||
133 | gk20a_dbg_info("pagepool_buffer_size : %d", pagepool_buffer_size); | ||
134 | gr->global_ctx_buffer[PAGEPOOL].mem.size = pagepool_buffer_size; | ||
135 | |||
136 | gk20a_dbg_info("attr_buffer_size : %d", attr_buffer_size); | ||
137 | gr->global_ctx_buffer[ATTRIBUTE].mem.size = attr_buffer_size; | ||
138 | |||
139 | gk20a_dbg_info("priv access map size : %d", | ||
140 | gr->ctx_vars.priv_access_map_size); | ||
141 | gr->global_ctx_buffer[PRIV_ACCESS_MAP].mem.size = | ||
142 | gr->ctx_vars.priv_access_map_size; | ||
143 | |||
144 | return 0; | ||
145 | } | ||
146 | |||
147 | static int vgpu_gr_map_global_ctx_buffers(struct gk20a *g, | ||
148 | struct channel_gk20a *c) | ||
149 | { | ||
150 | struct tegra_vgpu_cmd_msg msg; | ||
151 | struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx; | ||
152 | struct vm_gk20a *ch_vm = c->vm; | ||
153 | u64 *g_bfr_va = c->ch_ctx.global_ctx_buffer_va; | ||
154 | u64 *g_bfr_size = c->ch_ctx.global_ctx_buffer_size; | ||
155 | struct gr_gk20a *gr = &g->gr; | ||
156 | u64 gpu_va; | ||
157 | u32 i; | ||
158 | int err; | ||
159 | |||
160 | gk20a_dbg_fn(""); | ||
161 | |||
162 | /* FIXME: add VPR support */ | ||
163 | |||
164 | /* Circular Buffer */ | ||
165 | gpu_va = __nvgpu_vm_alloc_va(ch_vm, | ||
166 | gr->global_ctx_buffer[CIRCULAR].mem.size, | ||
167 | gmmu_page_size_kernel); | ||
168 | |||
169 | if (!gpu_va) | ||
170 | goto clean_up; | ||
171 | g_bfr_va[CIRCULAR_VA] = gpu_va; | ||
172 | g_bfr_size[CIRCULAR_VA] = gr->global_ctx_buffer[CIRCULAR].mem.size; | ||
173 | |||
174 | /* Attribute Buffer */ | ||
175 | gpu_va = __nvgpu_vm_alloc_va(ch_vm, | ||
176 | gr->global_ctx_buffer[ATTRIBUTE].mem.size, | ||
177 | gmmu_page_size_kernel); | ||
178 | |||
179 | if (!gpu_va) | ||
180 | goto clean_up; | ||
181 | g_bfr_va[ATTRIBUTE_VA] = gpu_va; | ||
182 | g_bfr_size[ATTRIBUTE_VA] = gr->global_ctx_buffer[ATTRIBUTE].mem.size; | ||
183 | |||
184 | /* Page Pool */ | ||
185 | gpu_va = __nvgpu_vm_alloc_va(ch_vm, | ||
186 | gr->global_ctx_buffer[PAGEPOOL].mem.size, | ||
187 | gmmu_page_size_kernel); | ||
188 | if (!gpu_va) | ||
189 | goto clean_up; | ||
190 | g_bfr_va[PAGEPOOL_VA] = gpu_va; | ||
191 | g_bfr_size[PAGEPOOL_VA] = gr->global_ctx_buffer[PAGEPOOL].mem.size; | ||
192 | |||
193 | /* Priv register Access Map */ | ||
194 | gpu_va = __nvgpu_vm_alloc_va(ch_vm, | ||
195 | gr->global_ctx_buffer[PRIV_ACCESS_MAP].mem.size, | ||
196 | gmmu_page_size_kernel); | ||
197 | if (!gpu_va) | ||
198 | goto clean_up; | ||
199 | g_bfr_va[PRIV_ACCESS_MAP_VA] = gpu_va; | ||
200 | g_bfr_size[PRIV_ACCESS_MAP_VA] = | ||
201 | gr->global_ctx_buffer[PRIV_ACCESS_MAP].mem.size; | ||
202 | |||
203 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_MAP_GR_GLOBAL_CTX; | ||
204 | msg.handle = vgpu_get_handle(g); | ||
205 | p->handle = c->virt_ctx; | ||
206 | p->cb_va = g_bfr_va[CIRCULAR_VA]; | ||
207 | p->attr_va = g_bfr_va[ATTRIBUTE_VA]; | ||
208 | p->page_pool_va = g_bfr_va[PAGEPOOL_VA]; | ||
209 | p->priv_access_map_va = g_bfr_va[PRIV_ACCESS_MAP_VA]; | ||
210 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
211 | if (err || msg.ret) | ||
212 | goto clean_up; | ||
213 | |||
214 | c->ch_ctx.global_ctx_buffer_mapped = true; | ||
215 | return 0; | ||
216 | |||
217 | clean_up: | ||
218 | for (i = 0; i < NR_GLOBAL_CTX_BUF_VA; i++) { | ||
219 | if (g_bfr_va[i]) { | ||
220 | __nvgpu_vm_free_va(ch_vm, g_bfr_va[i], | ||
221 | gmmu_page_size_kernel); | ||
222 | g_bfr_va[i] = 0; | ||
223 | } | ||
224 | } | ||
225 | return -ENOMEM; | ||
226 | } | ||
227 | |||
228 | static void vgpu_gr_unmap_global_ctx_buffers(struct channel_gk20a *c) | ||
229 | { | ||
230 | struct vm_gk20a *ch_vm = c->vm; | ||
231 | u64 *g_bfr_va = c->ch_ctx.global_ctx_buffer_va; | ||
232 | u64 *g_bfr_size = c->ch_ctx.global_ctx_buffer_size; | ||
233 | u32 i; | ||
234 | |||
235 | gk20a_dbg_fn(""); | ||
236 | |||
237 | if (c->ch_ctx.global_ctx_buffer_mapped) { | ||
238 | struct tegra_vgpu_cmd_msg msg; | ||
239 | struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx; | ||
240 | int err; | ||
241 | |||
242 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_UNMAP_GR_GLOBAL_CTX; | ||
243 | msg.handle = vgpu_get_handle(c->g); | ||
244 | p->handle = c->virt_ctx; | ||
245 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
246 | WARN_ON(err || msg.ret); | ||
247 | } | ||
248 | |||
249 | for (i = 0; i < NR_GLOBAL_CTX_BUF_VA; i++) { | ||
250 | if (g_bfr_va[i]) { | ||
251 | __nvgpu_vm_free_va(ch_vm, g_bfr_va[i], | ||
252 | gmmu_page_size_kernel); | ||
253 | g_bfr_va[i] = 0; | ||
254 | g_bfr_size[i] = 0; | ||
255 | } | ||
256 | } | ||
257 | c->ch_ctx.global_ctx_buffer_mapped = false; | ||
258 | } | ||
259 | |||
260 | int vgpu_gr_alloc_gr_ctx(struct gk20a *g, | ||
261 | struct gr_ctx_desc **__gr_ctx, | ||
262 | struct vm_gk20a *vm, | ||
263 | u32 class, | ||
264 | u32 flags) | ||
265 | { | ||
266 | struct tegra_vgpu_cmd_msg msg = {0}; | ||
267 | struct tegra_vgpu_gr_ctx_params *p = &msg.params.gr_ctx; | ||
268 | struct gr_gk20a *gr = &g->gr; | ||
269 | struct gr_ctx_desc *gr_ctx; | ||
270 | int err; | ||
271 | |||
272 | gk20a_dbg_fn(""); | ||
273 | |||
274 | if (gr->ctx_vars.buffer_size == 0) | ||
275 | return 0; | ||
276 | |||
277 | /* alloc channel gr ctx buffer */ | ||
278 | gr->ctx_vars.buffer_size = gr->ctx_vars.golden_image_size; | ||
279 | gr->ctx_vars.buffer_total_size = gr->ctx_vars.golden_image_size; | ||
280 | |||
281 | gr_ctx = nvgpu_kzalloc(g, sizeof(*gr_ctx)); | ||
282 | if (!gr_ctx) | ||
283 | return -ENOMEM; | ||
284 | |||
285 | gr_ctx->mem.size = gr->ctx_vars.buffer_total_size; | ||
286 | gr_ctx->mem.gpu_va = __nvgpu_vm_alloc_va(vm, | ||
287 | gr_ctx->mem.size, | ||
288 | gmmu_page_size_kernel); | ||
289 | |||
290 | if (!gr_ctx->mem.gpu_va) { | ||
291 | nvgpu_kfree(g, gr_ctx); | ||
292 | return -ENOMEM; | ||
293 | } | ||
294 | |||
295 | msg.cmd = TEGRA_VGPU_CMD_GR_CTX_ALLOC; | ||
296 | msg.handle = vgpu_get_handle(g); | ||
297 | p->as_handle = vm->handle; | ||
298 | p->gr_ctx_va = gr_ctx->mem.gpu_va; | ||
299 | p->class_num = class; | ||
300 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
301 | err = err ? err : msg.ret; | ||
302 | |||
303 | if (unlikely(err)) { | ||
304 | nvgpu_err(g, "fail to alloc gr_ctx"); | ||
305 | __nvgpu_vm_free_va(vm, gr_ctx->mem.gpu_va, | ||
306 | gmmu_page_size_kernel); | ||
307 | nvgpu_kfree(g, gr_ctx); | ||
308 | } else { | ||
309 | gr_ctx->virt_ctx = p->gr_ctx_handle; | ||
310 | *__gr_ctx = gr_ctx; | ||
311 | } | ||
312 | |||
313 | return err; | ||
314 | } | ||
315 | |||
316 | void vgpu_gr_free_gr_ctx(struct gk20a *g, struct vm_gk20a *vm, | ||
317 | struct gr_ctx_desc *gr_ctx) | ||
318 | { | ||
319 | gk20a_dbg_fn(""); | ||
320 | |||
321 | if (gr_ctx && gr_ctx->mem.gpu_va) { | ||
322 | struct tegra_vgpu_cmd_msg msg; | ||
323 | struct tegra_vgpu_gr_ctx_params *p = &msg.params.gr_ctx; | ||
324 | int err; | ||
325 | |||
326 | msg.cmd = TEGRA_VGPU_CMD_GR_CTX_FREE; | ||
327 | msg.handle = vgpu_get_handle(g); | ||
328 | p->gr_ctx_handle = gr_ctx->virt_ctx; | ||
329 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
330 | WARN_ON(err || msg.ret); | ||
331 | |||
332 | __nvgpu_vm_free_va(vm, gr_ctx->mem.gpu_va, | ||
333 | gmmu_page_size_kernel); | ||
334 | nvgpu_kfree(g, gr_ctx); | ||
335 | } | ||
336 | } | ||
337 | |||
338 | static void vgpu_gr_free_channel_gr_ctx(struct channel_gk20a *c) | ||
339 | { | ||
340 | gk20a_dbg_fn(""); | ||
341 | |||
342 | c->g->ops.gr.free_gr_ctx(c->g, c->vm, c->ch_ctx.gr_ctx); | ||
343 | c->ch_ctx.gr_ctx = NULL; | ||
344 | } | ||
345 | |||
346 | static int vgpu_gr_alloc_channel_patch_ctx(struct gk20a *g, | ||
347 | struct channel_gk20a *c) | ||
348 | { | ||
349 | struct patch_desc *patch_ctx = &c->ch_ctx.patch_ctx; | ||
350 | struct vm_gk20a *ch_vm = c->vm; | ||
351 | struct tegra_vgpu_cmd_msg msg; | ||
352 | struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx; | ||
353 | int err; | ||
354 | |||
355 | gk20a_dbg_fn(""); | ||
356 | |||
357 | patch_ctx->mem.size = 128 * sizeof(u32); | ||
358 | patch_ctx->mem.gpu_va = __nvgpu_vm_alloc_va(ch_vm, | ||
359 | patch_ctx->mem.size, | ||
360 | gmmu_page_size_kernel); | ||
361 | if (!patch_ctx->mem.gpu_va) | ||
362 | return -ENOMEM; | ||
363 | |||
364 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_ALLOC_GR_PATCH_CTX; | ||
365 | msg.handle = vgpu_get_handle(g); | ||
366 | p->handle = c->virt_ctx; | ||
367 | p->patch_ctx_va = patch_ctx->mem.gpu_va; | ||
368 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
369 | if (err || msg.ret) { | ||
370 | __nvgpu_vm_free_va(ch_vm, patch_ctx->mem.gpu_va, | ||
371 | gmmu_page_size_kernel); | ||
372 | err = -ENOMEM; | ||
373 | } | ||
374 | |||
375 | return err; | ||
376 | } | ||
377 | |||
378 | static void vgpu_gr_free_channel_patch_ctx(struct channel_gk20a *c) | ||
379 | { | ||
380 | struct patch_desc *patch_ctx = &c->ch_ctx.patch_ctx; | ||
381 | struct vm_gk20a *ch_vm = c->vm; | ||
382 | |||
383 | gk20a_dbg_fn(""); | ||
384 | |||
385 | if (patch_ctx->mem.gpu_va) { | ||
386 | struct tegra_vgpu_cmd_msg msg; | ||
387 | struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx; | ||
388 | int err; | ||
389 | |||
390 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_FREE_GR_PATCH_CTX; | ||
391 | msg.handle = vgpu_get_handle(c->g); | ||
392 | p->handle = c->virt_ctx; | ||
393 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
394 | WARN_ON(err || msg.ret); | ||
395 | |||
396 | __nvgpu_vm_free_va(ch_vm, patch_ctx->mem.gpu_va, | ||
397 | gmmu_page_size_kernel); | ||
398 | patch_ctx->mem.gpu_va = 0; | ||
399 | } | ||
400 | } | ||
401 | |||
402 | static void vgpu_gr_free_channel_pm_ctx(struct channel_gk20a *c) | ||
403 | { | ||
404 | struct tegra_vgpu_cmd_msg msg; | ||
405 | struct tegra_vgpu_channel_free_hwpm_ctx *p = &msg.params.free_hwpm_ctx; | ||
406 | struct channel_ctx_gk20a *ch_ctx = &c->ch_ctx; | ||
407 | struct pm_ctx_desc *pm_ctx = &ch_ctx->pm_ctx; | ||
408 | int err; | ||
409 | |||
410 | gk20a_dbg_fn(""); | ||
411 | |||
412 | /* check if hwpm was ever initialized. If not, nothing to do */ | ||
413 | if (pm_ctx->mem.gpu_va == 0) | ||
414 | return; | ||
415 | |||
416 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_FREE_HWPM_CTX; | ||
417 | msg.handle = vgpu_get_handle(c->g); | ||
418 | p->handle = c->virt_ctx; | ||
419 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
420 | WARN_ON(err || msg.ret); | ||
421 | |||
422 | __nvgpu_vm_free_va(c->vm, pm_ctx->mem.gpu_va, | ||
423 | gmmu_page_size_kernel); | ||
424 | pm_ctx->mem.gpu_va = 0; | ||
425 | } | ||
426 | |||
427 | void vgpu_gr_free_channel_ctx(struct channel_gk20a *c, bool is_tsg) | ||
428 | { | ||
429 | gk20a_dbg_fn(""); | ||
430 | |||
431 | if (c->g->ops.fifo.free_channel_ctx_header) | ||
432 | c->g->ops.fifo.free_channel_ctx_header(c); | ||
433 | vgpu_gr_unmap_global_ctx_buffers(c); | ||
434 | vgpu_gr_free_channel_patch_ctx(c); | ||
435 | vgpu_gr_free_channel_pm_ctx(c); | ||
436 | if (!is_tsg) | ||
437 | vgpu_gr_free_channel_gr_ctx(c); | ||
438 | |||
439 | /* zcull_ctx, pm_ctx */ | ||
440 | |||
441 | memset(&c->ch_ctx, 0, sizeof(struct channel_ctx_gk20a)); | ||
442 | |||
443 | c->first_init = false; | ||
444 | } | ||
445 | |||
446 | static int vgpu_gr_ch_bind_gr_ctx(struct channel_gk20a *c) | ||
447 | { | ||
448 | struct gr_ctx_desc *gr_ctx = c->ch_ctx.gr_ctx; | ||
449 | struct tegra_vgpu_cmd_msg msg = {0}; | ||
450 | struct tegra_vgpu_channel_bind_gr_ctx_params *p = | ||
451 | &msg.params.ch_bind_gr_ctx; | ||
452 | int err; | ||
453 | |||
454 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_BIND_GR_CTX; | ||
455 | msg.handle = vgpu_get_handle(c->g); | ||
456 | p->ch_handle = c->virt_ctx; | ||
457 | p->gr_ctx_handle = gr_ctx->virt_ctx; | ||
458 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
459 | err = err ? err : msg.ret; | ||
460 | WARN_ON(err); | ||
461 | |||
462 | return err; | ||
463 | } | ||
464 | |||
465 | static int vgpu_gr_tsg_bind_gr_ctx(struct tsg_gk20a *tsg) | ||
466 | { | ||
467 | struct gr_ctx_desc *gr_ctx = tsg->tsg_gr_ctx; | ||
468 | struct tegra_vgpu_cmd_msg msg = {0}; | ||
469 | struct tegra_vgpu_tsg_bind_gr_ctx_params *p = | ||
470 | &msg.params.tsg_bind_gr_ctx; | ||
471 | int err; | ||
472 | |||
473 | msg.cmd = TEGRA_VGPU_CMD_TSG_BIND_GR_CTX; | ||
474 | msg.handle = vgpu_get_handle(tsg->g); | ||
475 | p->tsg_id = tsg->tsgid; | ||
476 | p->gr_ctx_handle = gr_ctx->virt_ctx; | ||
477 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
478 | err = err ? err : msg.ret; | ||
479 | WARN_ON(err); | ||
480 | |||
481 | return err; | ||
482 | } | ||
483 | |||
484 | int vgpu_gr_alloc_obj_ctx(struct channel_gk20a *c, u32 class_num, u32 flags) | ||
485 | { | ||
486 | struct gk20a *g = c->g; | ||
487 | struct fifo_gk20a *f = &g->fifo; | ||
488 | struct channel_ctx_gk20a *ch_ctx = &c->ch_ctx; | ||
489 | struct tsg_gk20a *tsg = NULL; | ||
490 | int err = 0; | ||
491 | |||
492 | gk20a_dbg_fn(""); | ||
493 | |||
494 | /* an address space needs to have been bound at this point.*/ | ||
495 | if (!gk20a_channel_as_bound(c)) { | ||
496 | nvgpu_err(g, "not bound to address space at time" | ||
497 | " of grctx allocation"); | ||
498 | return -EINVAL; | ||
499 | } | ||
500 | |||
501 | if (!g->ops.gr.is_valid_class(g, class_num)) { | ||
502 | nvgpu_err(g, "invalid obj class 0x%x", class_num); | ||
503 | err = -EINVAL; | ||
504 | goto out; | ||
505 | } | ||
506 | c->obj_class = class_num; | ||
507 | |||
508 | if (gk20a_is_channel_marked_as_tsg(c)) | ||
509 | tsg = &f->tsg[c->tsgid]; | ||
510 | |||
511 | if (!tsg) { | ||
512 | /* allocate gr ctx buffer */ | ||
513 | if (!ch_ctx->gr_ctx) { | ||
514 | err = g->ops.gr.alloc_gr_ctx(g, &c->ch_ctx.gr_ctx, | ||
515 | c->vm, | ||
516 | class_num, | ||
517 | flags); | ||
518 | if (!err) | ||
519 | err = vgpu_gr_ch_bind_gr_ctx(c); | ||
520 | if (err) { | ||
521 | nvgpu_err(g, "fail to allocate gr ctx buffer"); | ||
522 | goto out; | ||
523 | } | ||
524 | } else { | ||
525 | /*TBD: needs to be more subtle about which is | ||
526 | * being allocated as some are allowed to be | ||
527 | * allocated along same channel */ | ||
528 | nvgpu_err(g, | ||
529 | "too many classes alloc'd on same channel"); | ||
530 | err = -EINVAL; | ||
531 | goto out; | ||
532 | } | ||
533 | } else { | ||
534 | if (!tsg->tsg_gr_ctx) { | ||
535 | tsg->vm = c->vm; | ||
536 | nvgpu_vm_get(tsg->vm); | ||
537 | err = g->ops.gr.alloc_gr_ctx(g, &tsg->tsg_gr_ctx, | ||
538 | c->vm, | ||
539 | class_num, | ||
540 | flags); | ||
541 | if (!err) | ||
542 | err = vgpu_gr_tsg_bind_gr_ctx(tsg); | ||
543 | if (err) { | ||
544 | nvgpu_err(g, | ||
545 | "fail to allocate TSG gr ctx buffer, err=%d", err); | ||
546 | nvgpu_vm_put(tsg->vm); | ||
547 | tsg->vm = NULL; | ||
548 | goto out; | ||
549 | } | ||
550 | } | ||
551 | |||
552 | ch_ctx->gr_ctx = tsg->tsg_gr_ctx; | ||
553 | err = vgpu_gr_ch_bind_gr_ctx(c); | ||
554 | if (err) { | ||
555 | nvgpu_err(g, "fail to bind gr ctx buffer"); | ||
556 | goto out; | ||
557 | } | ||
558 | } | ||
559 | |||
560 | /* commit gr ctx buffer */ | ||
561 | err = g->ops.gr.commit_inst(c, ch_ctx->gr_ctx->mem.gpu_va); | ||
562 | if (err) { | ||
563 | nvgpu_err(g, "fail to commit gr ctx buffer"); | ||
564 | goto out; | ||
565 | } | ||
566 | |||
567 | /* allocate patch buffer */ | ||
568 | if (ch_ctx->patch_ctx.mem.priv.pages == NULL) { | ||
569 | err = vgpu_gr_alloc_channel_patch_ctx(g, c); | ||
570 | if (err) { | ||
571 | nvgpu_err(g, "fail to allocate patch buffer"); | ||
572 | goto out; | ||
573 | } | ||
574 | } | ||
575 | |||
576 | /* map global buffer to channel gpu_va and commit */ | ||
577 | if (!ch_ctx->global_ctx_buffer_mapped) { | ||
578 | err = vgpu_gr_map_global_ctx_buffers(g, c); | ||
579 | if (err) { | ||
580 | nvgpu_err(g, "fail to map global ctx buffer"); | ||
581 | goto out; | ||
582 | } | ||
583 | gr_gk20a_elpg_protected_call(g, | ||
584 | vgpu_gr_commit_global_ctx_buffers(g, c, true)); | ||
585 | } | ||
586 | |||
587 | /* load golden image */ | ||
588 | if (!c->first_init) { | ||
589 | err = gr_gk20a_elpg_protected_call(g, | ||
590 | vgpu_gr_load_golden_ctx_image(g, c)); | ||
591 | if (err) { | ||
592 | nvgpu_err(g, "fail to load golden ctx image"); | ||
593 | goto out; | ||
594 | } | ||
595 | c->first_init = true; | ||
596 | } | ||
597 | |||
598 | gk20a_dbg_fn("done"); | ||
599 | return 0; | ||
600 | out: | ||
601 | /* 1. gr_ctx, patch_ctx and global ctx buffer mapping | ||
602 | can be reused so no need to release them. | ||
603 | 2. golden image load is a one time thing so if | ||
604 | they pass, no need to undo. */ | ||
605 | nvgpu_err(g, "fail"); | ||
606 | return err; | ||
607 | } | ||
608 | |||
609 | static int vgpu_gr_init_gr_config(struct gk20a *g, struct gr_gk20a *gr) | ||
610 | { | ||
611 | struct vgpu_priv_data *priv = vgpu_get_priv_data(g); | ||
612 | u32 gpc_index; | ||
613 | int err = -ENOMEM; | ||
614 | |||
615 | gk20a_dbg_fn(""); | ||
616 | |||
617 | gr->max_gpc_count = priv->constants.max_gpc_count; | ||
618 | gr->gpc_count = priv->constants.gpc_count; | ||
619 | gr->max_tpc_per_gpc_count = priv->constants.max_tpc_per_gpc_count; | ||
620 | |||
621 | gr->max_tpc_count = gr->max_gpc_count * gr->max_tpc_per_gpc_count; | ||
622 | |||
623 | gr->gpc_tpc_count = nvgpu_kzalloc(g, gr->gpc_count * sizeof(u32)); | ||
624 | if (!gr->gpc_tpc_count) | ||
625 | goto cleanup; | ||
626 | |||
627 | gr->gpc_tpc_mask = nvgpu_kzalloc(g, gr->gpc_count * sizeof(u32)); | ||
628 | if (!gr->gpc_tpc_mask) | ||
629 | goto cleanup; | ||
630 | |||
631 | gr->sm_to_cluster = nvgpu_kzalloc(g, gr->gpc_count * | ||
632 | gr->max_tpc_per_gpc_count * | ||
633 | sizeof(struct sm_info)); | ||
634 | if (!gr->sm_to_cluster) | ||
635 | goto cleanup; | ||
636 | |||
637 | gr->tpc_count = 0; | ||
638 | for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { | ||
639 | gr->gpc_tpc_count[gpc_index] = | ||
640 | priv->constants.gpc_tpc_count[gpc_index]; | ||
641 | |||
642 | gr->tpc_count += gr->gpc_tpc_count[gpc_index]; | ||
643 | |||
644 | if (g->ops.gr.get_gpc_tpc_mask) | ||
645 | gr->gpc_tpc_mask[gpc_index] = | ||
646 | g->ops.gr.get_gpc_tpc_mask(g, gpc_index); | ||
647 | } | ||
648 | |||
649 | g->ops.gr.bundle_cb_defaults(g); | ||
650 | g->ops.gr.cb_size_default(g); | ||
651 | g->ops.gr.calc_global_ctx_buffer_size(g); | ||
652 | err = g->ops.gr.init_fs_state(g); | ||
653 | if (err) | ||
654 | goto cleanup; | ||
655 | return 0; | ||
656 | cleanup: | ||
657 | nvgpu_err(g, "out of memory"); | ||
658 | |||
659 | nvgpu_kfree(g, gr->gpc_tpc_count); | ||
660 | gr->gpc_tpc_count = NULL; | ||
661 | |||
662 | nvgpu_kfree(g, gr->gpc_tpc_mask); | ||
663 | gr->gpc_tpc_mask = NULL; | ||
664 | |||
665 | return err; | ||
666 | } | ||
667 | |||
668 | int vgpu_gr_bind_ctxsw_zcull(struct gk20a *g, struct gr_gk20a *gr, | ||
669 | struct channel_gk20a *c, u64 zcull_va, | ||
670 | u32 mode) | ||
671 | { | ||
672 | struct tegra_vgpu_cmd_msg msg; | ||
673 | struct tegra_vgpu_zcull_bind_params *p = &msg.params.zcull_bind; | ||
674 | int err; | ||
675 | |||
676 | gk20a_dbg_fn(""); | ||
677 | |||
678 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_BIND_ZCULL; | ||
679 | msg.handle = vgpu_get_handle(g); | ||
680 | p->handle = c->virt_ctx; | ||
681 | p->zcull_va = zcull_va; | ||
682 | p->mode = mode; | ||
683 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
684 | |||
685 | return (err || msg.ret) ? -ENOMEM : 0; | ||
686 | } | ||
687 | |||
688 | int vgpu_gr_get_zcull_info(struct gk20a *g, struct gr_gk20a *gr, | ||
689 | struct gr_zcull_info *zcull_params) | ||
690 | { | ||
691 | struct tegra_vgpu_cmd_msg msg; | ||
692 | struct tegra_vgpu_zcull_info_params *p = &msg.params.zcull_info; | ||
693 | int err; | ||
694 | |||
695 | gk20a_dbg_fn(""); | ||
696 | |||
697 | msg.cmd = TEGRA_VGPU_CMD_GET_ZCULL_INFO; | ||
698 | msg.handle = vgpu_get_handle(g); | ||
699 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
700 | if (err || msg.ret) | ||
701 | return -ENOMEM; | ||
702 | |||
703 | zcull_params->width_align_pixels = p->width_align_pixels; | ||
704 | zcull_params->height_align_pixels = p->height_align_pixels; | ||
705 | zcull_params->pixel_squares_by_aliquots = p->pixel_squares_by_aliquots; | ||
706 | zcull_params->aliquot_total = p->aliquot_total; | ||
707 | zcull_params->region_byte_multiplier = p->region_byte_multiplier; | ||
708 | zcull_params->region_header_size = p->region_header_size; | ||
709 | zcull_params->subregion_header_size = p->subregion_header_size; | ||
710 | zcull_params->subregion_width_align_pixels = | ||
711 | p->subregion_width_align_pixels; | ||
712 | zcull_params->subregion_height_align_pixels = | ||
713 | p->subregion_height_align_pixels; | ||
714 | zcull_params->subregion_count = p->subregion_count; | ||
715 | |||
716 | return 0; | ||
717 | } | ||
718 | |||
719 | u32 vgpu_gr_get_gpc_tpc_mask(struct gk20a *g, u32 gpc_index) | ||
720 | { | ||
721 | struct vgpu_priv_data *priv = vgpu_get_priv_data(g); | ||
722 | |||
723 | return priv->constants.gpc_tpc_mask[gpc_index]; | ||
724 | } | ||
725 | |||
726 | u32 vgpu_gr_get_max_fbps_count(struct gk20a *g) | ||
727 | { | ||
728 | struct vgpu_priv_data *priv = vgpu_get_priv_data(g); | ||
729 | |||
730 | gk20a_dbg_fn(""); | ||
731 | |||
732 | return priv->constants.num_fbps; | ||
733 | } | ||
734 | |||
735 | u32 vgpu_gr_get_fbp_en_mask(struct gk20a *g) | ||
736 | { | ||
737 | struct vgpu_priv_data *priv = vgpu_get_priv_data(g); | ||
738 | |||
739 | gk20a_dbg_fn(""); | ||
740 | |||
741 | return priv->constants.fbp_en_mask; | ||
742 | } | ||
743 | |||
744 | u32 vgpu_gr_get_max_ltc_per_fbp(struct gk20a *g) | ||
745 | { | ||
746 | struct vgpu_priv_data *priv = vgpu_get_priv_data(g); | ||
747 | |||
748 | gk20a_dbg_fn(""); | ||
749 | |||
750 | return priv->constants.ltc_per_fbp; | ||
751 | } | ||
752 | |||
753 | u32 vgpu_gr_get_max_lts_per_ltc(struct gk20a *g) | ||
754 | { | ||
755 | struct vgpu_priv_data *priv = vgpu_get_priv_data(g); | ||
756 | |||
757 | gk20a_dbg_fn(""); | ||
758 | |||
759 | return priv->constants.max_lts_per_ltc; | ||
760 | } | ||
761 | |||
762 | u32 *vgpu_gr_rop_l2_en_mask(struct gk20a *g) | ||
763 | { | ||
764 | /* no one use it yet */ | ||
765 | return NULL; | ||
766 | } | ||
767 | |||
768 | int vgpu_gr_add_zbc(struct gk20a *g, struct gr_gk20a *gr, | ||
769 | struct zbc_entry *zbc_val) | ||
770 | { | ||
771 | struct tegra_vgpu_cmd_msg msg = {0}; | ||
772 | struct tegra_vgpu_zbc_set_table_params *p = &msg.params.zbc_set_table; | ||
773 | int err; | ||
774 | |||
775 | gk20a_dbg_fn(""); | ||
776 | |||
777 | msg.cmd = TEGRA_VGPU_CMD_ZBC_SET_TABLE; | ||
778 | msg.handle = vgpu_get_handle(g); | ||
779 | |||
780 | p->type = zbc_val->type; | ||
781 | p->format = zbc_val->format; | ||
782 | switch (p->type) { | ||
783 | case GK20A_ZBC_TYPE_COLOR: | ||
784 | memcpy(p->color_ds, zbc_val->color_ds, sizeof(p->color_ds)); | ||
785 | memcpy(p->color_l2, zbc_val->color_l2, sizeof(p->color_l2)); | ||
786 | break; | ||
787 | case GK20A_ZBC_TYPE_DEPTH: | ||
788 | p->depth = zbc_val->depth; | ||
789 | break; | ||
790 | default: | ||
791 | return -EINVAL; | ||
792 | } | ||
793 | |||
794 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
795 | |||
796 | return (err || msg.ret) ? -ENOMEM : 0; | ||
797 | } | ||
798 | |||
799 | int vgpu_gr_query_zbc(struct gk20a *g, struct gr_gk20a *gr, | ||
800 | struct zbc_query_params *query_params) | ||
801 | { | ||
802 | struct tegra_vgpu_cmd_msg msg = {0}; | ||
803 | struct tegra_vgpu_zbc_query_table_params *p = | ||
804 | &msg.params.zbc_query_table; | ||
805 | int err; | ||
806 | |||
807 | gk20a_dbg_fn(""); | ||
808 | |||
809 | msg.cmd = TEGRA_VGPU_CMD_ZBC_QUERY_TABLE; | ||
810 | msg.handle = vgpu_get_handle(g); | ||
811 | |||
812 | p->type = query_params->type; | ||
813 | p->index_size = query_params->index_size; | ||
814 | |||
815 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
816 | if (err || msg.ret) | ||
817 | return -ENOMEM; | ||
818 | |||
819 | switch (query_params->type) { | ||
820 | case GK20A_ZBC_TYPE_COLOR: | ||
821 | memcpy(query_params->color_ds, p->color_ds, | ||
822 | sizeof(query_params->color_ds)); | ||
823 | memcpy(query_params->color_l2, p->color_l2, | ||
824 | sizeof(query_params->color_l2)); | ||
825 | break; | ||
826 | case GK20A_ZBC_TYPE_DEPTH: | ||
827 | query_params->depth = p->depth; | ||
828 | break; | ||
829 | case GK20A_ZBC_TYPE_INVALID: | ||
830 | query_params->index_size = p->index_size; | ||
831 | break; | ||
832 | default: | ||
833 | return -EINVAL; | ||
834 | } | ||
835 | query_params->ref_cnt = p->ref_cnt; | ||
836 | query_params->format = p->format; | ||
837 | |||
838 | return 0; | ||
839 | } | ||
840 | |||
841 | static void vgpu_remove_gr_support(struct gr_gk20a *gr) | ||
842 | { | ||
843 | gk20a_dbg_fn(""); | ||
844 | |||
845 | gk20a_comptag_allocator_destroy(gr->g, &gr->comp_tags); | ||
846 | |||
847 | nvgpu_kfree(gr->g, gr->sm_error_states); | ||
848 | gr->sm_error_states = NULL; | ||
849 | |||
850 | nvgpu_kfree(gr->g, gr->gpc_tpc_mask); | ||
851 | gr->gpc_tpc_mask = NULL; | ||
852 | |||
853 | nvgpu_kfree(gr->g, gr->sm_to_cluster); | ||
854 | gr->sm_to_cluster = NULL; | ||
855 | |||
856 | nvgpu_kfree(gr->g, gr->gpc_tpc_count); | ||
857 | gr->gpc_tpc_count = NULL; | ||
858 | } | ||
859 | |||
860 | static int vgpu_gr_init_gr_setup_sw(struct gk20a *g) | ||
861 | { | ||
862 | struct gr_gk20a *gr = &g->gr; | ||
863 | int err; | ||
864 | |||
865 | gk20a_dbg_fn(""); | ||
866 | |||
867 | if (gr->sw_ready) { | ||
868 | gk20a_dbg_fn("skip init"); | ||
869 | return 0; | ||
870 | } | ||
871 | |||
872 | gr->g = g; | ||
873 | |||
874 | #if defined(CONFIG_GK20A_CYCLE_STATS) | ||
875 | nvgpu_mutex_init(&g->gr.cs_lock); | ||
876 | #endif | ||
877 | |||
878 | err = vgpu_gr_init_gr_config(g, gr); | ||
879 | if (err) | ||
880 | goto clean_up; | ||
881 | |||
882 | err = g->ops.gr.init_ctx_state(g); | ||
883 | if (err) | ||
884 | goto clean_up; | ||
885 | |||
886 | err = g->ops.ltc.init_comptags(g, gr); | ||
887 | if (err) | ||
888 | goto clean_up; | ||
889 | |||
890 | err = vgpu_gr_alloc_global_ctx_buffers(g); | ||
891 | if (err) | ||
892 | goto clean_up; | ||
893 | |||
894 | nvgpu_mutex_init(&gr->ctx_mutex); | ||
895 | |||
896 | gr->sm_error_states = nvgpu_kzalloc(g, | ||
897 | sizeof(struct nvgpu_gr_sm_error_state) * | ||
898 | gr->no_of_sm); | ||
899 | if (!gr->sm_error_states) { | ||
900 | err = -ENOMEM; | ||
901 | goto clean_up; | ||
902 | } | ||
903 | |||
904 | gr->remove_support = vgpu_remove_gr_support; | ||
905 | gr->sw_ready = true; | ||
906 | |||
907 | gk20a_dbg_fn("done"); | ||
908 | return 0; | ||
909 | |||
910 | clean_up: | ||
911 | nvgpu_err(g, "fail"); | ||
912 | vgpu_remove_gr_support(gr); | ||
913 | return err; | ||
914 | } | ||
915 | |||
916 | int vgpu_init_gr_support(struct gk20a *g) | ||
917 | { | ||
918 | gk20a_dbg_fn(""); | ||
919 | |||
920 | return vgpu_gr_init_gr_setup_sw(g); | ||
921 | } | ||
922 | |||
923 | int vgpu_gr_isr(struct gk20a *g, struct tegra_vgpu_gr_intr_info *info) | ||
924 | { | ||
925 | struct fifo_gk20a *f = &g->fifo; | ||
926 | struct channel_gk20a *ch = gk20a_channel_get(&f->channel[info->chid]); | ||
927 | |||
928 | gk20a_dbg_fn(""); | ||
929 | if (!ch) | ||
930 | return 0; | ||
931 | |||
932 | if (info->type != TEGRA_VGPU_GR_INTR_NOTIFY && | ||
933 | info->type != TEGRA_VGPU_GR_INTR_SEMAPHORE) | ||
934 | nvgpu_err(g, "gr intr (%d) on ch %u", info->type, info->chid); | ||
935 | |||
936 | switch (info->type) { | ||
937 | case TEGRA_VGPU_GR_INTR_NOTIFY: | ||
938 | nvgpu_cond_broadcast_interruptible(&ch->notifier_wq); | ||
939 | break; | ||
940 | case TEGRA_VGPU_GR_INTR_SEMAPHORE: | ||
941 | nvgpu_cond_broadcast_interruptible(&ch->semaphore_wq); | ||
942 | break; | ||
943 | case TEGRA_VGPU_GR_INTR_SEMAPHORE_TIMEOUT: | ||
944 | gk20a_set_error_notifier(ch, | ||
945 | NVGPU_CHANNEL_GR_SEMAPHORE_TIMEOUT); | ||
946 | break; | ||
947 | case TEGRA_VGPU_GR_INTR_ILLEGAL_NOTIFY: | ||
948 | gk20a_set_error_notifier(ch, | ||
949 | NVGPU_CHANNEL_GR_ILLEGAL_NOTIFY); | ||
950 | case TEGRA_VGPU_GR_INTR_ILLEGAL_METHOD: | ||
951 | break; | ||
952 | case TEGRA_VGPU_GR_INTR_ILLEGAL_CLASS: | ||
953 | gk20a_set_error_notifier(ch, | ||
954 | NVGPU_CHANNEL_GR_ERROR_SW_NOTIFY); | ||
955 | break; | ||
956 | case TEGRA_VGPU_GR_INTR_FECS_ERROR: | ||
957 | break; | ||
958 | case TEGRA_VGPU_GR_INTR_CLASS_ERROR: | ||
959 | gk20a_set_error_notifier(ch, | ||
960 | NVGPU_CHANNEL_GR_ERROR_SW_NOTIFY); | ||
961 | break; | ||
962 | case TEGRA_VGPU_GR_INTR_FIRMWARE_METHOD: | ||
963 | gk20a_set_error_notifier(ch, | ||
964 | NVGPU_CHANNEL_GR_ERROR_SW_NOTIFY); | ||
965 | break; | ||
966 | case TEGRA_VGPU_GR_INTR_EXCEPTION: | ||
967 | gk20a_set_error_notifier(ch, | ||
968 | NVGPU_CHANNEL_GR_ERROR_SW_NOTIFY); | ||
969 | break; | ||
970 | case TEGRA_VGPU_GR_INTR_SM_EXCEPTION: | ||
971 | gk20a_dbg_gpu_post_events(ch); | ||
972 | break; | ||
973 | default: | ||
974 | WARN_ON(1); | ||
975 | break; | ||
976 | } | ||
977 | |||
978 | gk20a_channel_put(ch); | ||
979 | return 0; | ||
980 | } | ||
981 | |||
982 | int vgpu_gr_nonstall_isr(struct gk20a *g, | ||
983 | struct tegra_vgpu_gr_nonstall_intr_info *info) | ||
984 | { | ||
985 | gk20a_dbg_fn(""); | ||
986 | |||
987 | switch (info->type) { | ||
988 | case TEGRA_VGPU_GR_NONSTALL_INTR_SEMAPHORE: | ||
989 | gk20a_channel_semaphore_wakeup(g, true); | ||
990 | break; | ||
991 | default: | ||
992 | WARN_ON(1); | ||
993 | break; | ||
994 | } | ||
995 | |||
996 | return 0; | ||
997 | } | ||
998 | |||
999 | int vgpu_gr_set_sm_debug_mode(struct gk20a *g, | ||
1000 | struct channel_gk20a *ch, u64 sms, bool enable) | ||
1001 | { | ||
1002 | struct tegra_vgpu_cmd_msg msg; | ||
1003 | struct tegra_vgpu_sm_debug_mode *p = &msg.params.sm_debug_mode; | ||
1004 | int err; | ||
1005 | |||
1006 | gk20a_dbg_fn(""); | ||
1007 | |||
1008 | msg.cmd = TEGRA_VGPU_CMD_SET_SM_DEBUG_MODE; | ||
1009 | msg.handle = vgpu_get_handle(g); | ||
1010 | p->handle = ch->virt_ctx; | ||
1011 | p->sms = sms; | ||
1012 | p->enable = (u32)enable; | ||
1013 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
1014 | WARN_ON(err || msg.ret); | ||
1015 | |||
1016 | return err ? err : msg.ret; | ||
1017 | } | ||
1018 | |||
1019 | int vgpu_gr_update_smpc_ctxsw_mode(struct gk20a *g, | ||
1020 | struct channel_gk20a *ch, bool enable) | ||
1021 | { | ||
1022 | struct tegra_vgpu_cmd_msg msg; | ||
1023 | struct tegra_vgpu_channel_set_ctxsw_mode *p = &msg.params.set_ctxsw_mode; | ||
1024 | int err; | ||
1025 | |||
1026 | gk20a_dbg_fn(""); | ||
1027 | |||
1028 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SET_SMPC_CTXSW_MODE; | ||
1029 | msg.handle = vgpu_get_handle(g); | ||
1030 | p->handle = ch->virt_ctx; | ||
1031 | |||
1032 | if (enable) | ||
1033 | p->mode = TEGRA_VGPU_CTXSW_MODE_CTXSW; | ||
1034 | else | ||
1035 | p->mode = TEGRA_VGPU_CTXSW_MODE_NO_CTXSW; | ||
1036 | |||
1037 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
1038 | WARN_ON(err || msg.ret); | ||
1039 | |||
1040 | return err ? err : msg.ret; | ||
1041 | } | ||
1042 | |||
1043 | int vgpu_gr_update_hwpm_ctxsw_mode(struct gk20a *g, | ||
1044 | struct channel_gk20a *ch, bool enable) | ||
1045 | { | ||
1046 | struct channel_ctx_gk20a *ch_ctx = &ch->ch_ctx; | ||
1047 | struct pm_ctx_desc *pm_ctx = &ch_ctx->pm_ctx; | ||
1048 | struct tegra_vgpu_cmd_msg msg; | ||
1049 | struct tegra_vgpu_channel_set_ctxsw_mode *p = &msg.params.set_ctxsw_mode; | ||
1050 | int err; | ||
1051 | |||
1052 | gk20a_dbg_fn(""); | ||
1053 | |||
1054 | if (enable) { | ||
1055 | p->mode = TEGRA_VGPU_CTXSW_MODE_CTXSW; | ||
1056 | |||
1057 | /* Allocate buffer if necessary */ | ||
1058 | if (pm_ctx->mem.gpu_va == 0) { | ||
1059 | pm_ctx->mem.gpu_va = __nvgpu_vm_alloc_va(ch->vm, | ||
1060 | g->gr.ctx_vars.pm_ctxsw_image_size, | ||
1061 | gmmu_page_size_kernel); | ||
1062 | |||
1063 | if (!pm_ctx->mem.gpu_va) | ||
1064 | return -ENOMEM; | ||
1065 | pm_ctx->mem.size = g->gr.ctx_vars.pm_ctxsw_image_size; | ||
1066 | } | ||
1067 | } else | ||
1068 | p->mode = TEGRA_VGPU_CTXSW_MODE_NO_CTXSW; | ||
1069 | |||
1070 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SET_HWPM_CTXSW_MODE; | ||
1071 | msg.handle = vgpu_get_handle(g); | ||
1072 | p->handle = ch->virt_ctx; | ||
1073 | p->gpu_va = pm_ctx->mem.gpu_va; | ||
1074 | |||
1075 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
1076 | WARN_ON(err || msg.ret); | ||
1077 | |||
1078 | return err ? err : msg.ret; | ||
1079 | } | ||
1080 | |||
1081 | int vgpu_gr_clear_sm_error_state(struct gk20a *g, | ||
1082 | struct channel_gk20a *ch, u32 sm_id) | ||
1083 | { | ||
1084 | struct gr_gk20a *gr = &g->gr; | ||
1085 | struct tegra_vgpu_cmd_msg msg; | ||
1086 | struct tegra_vgpu_clear_sm_error_state *p = | ||
1087 | &msg.params.clear_sm_error_state; | ||
1088 | int err; | ||
1089 | |||
1090 | nvgpu_mutex_acquire(&g->dbg_sessions_lock); | ||
1091 | msg.cmd = TEGRA_VGPU_CMD_CLEAR_SM_ERROR_STATE; | ||
1092 | msg.handle = vgpu_get_handle(g); | ||
1093 | p->handle = ch->virt_ctx; | ||
1094 | p->sm_id = sm_id; | ||
1095 | |||
1096 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
1097 | WARN_ON(err || msg.ret); | ||
1098 | |||
1099 | memset(&gr->sm_error_states[sm_id], 0, sizeof(*gr->sm_error_states)); | ||
1100 | nvgpu_mutex_release(&g->dbg_sessions_lock); | ||
1101 | |||
1102 | return err ? err : msg.ret; | ||
1103 | |||
1104 | |||
1105 | return 0; | ||
1106 | } | ||
1107 | |||
1108 | static int vgpu_gr_suspend_resume_contexts(struct gk20a *g, | ||
1109 | struct dbg_session_gk20a *dbg_s, | ||
1110 | int *ctx_resident_ch_fd, u32 cmd) | ||
1111 | { | ||
1112 | struct dbg_session_channel_data *ch_data; | ||
1113 | struct tegra_vgpu_cmd_msg msg; | ||
1114 | struct tegra_vgpu_suspend_resume_contexts *p; | ||
1115 | size_t n; | ||
1116 | int channel_fd = -1; | ||
1117 | int err = 0; | ||
1118 | void *handle = NULL; | ||
1119 | u16 *oob; | ||
1120 | size_t oob_size; | ||
1121 | |||
1122 | nvgpu_mutex_acquire(&g->dbg_sessions_lock); | ||
1123 | nvgpu_mutex_acquire(&dbg_s->ch_list_lock); | ||
1124 | |||
1125 | handle = tegra_gr_comm_oob_get_ptr(TEGRA_GR_COMM_CTX_CLIENT, | ||
1126 | tegra_gr_comm_get_server_vmid(), TEGRA_VGPU_QUEUE_CMD, | ||
1127 | (void **)&oob, &oob_size); | ||
1128 | if (!handle) { | ||
1129 | err = -EINVAL; | ||
1130 | goto done; | ||
1131 | } | ||
1132 | |||
1133 | n = 0; | ||
1134 | list_for_each_entry(ch_data, &dbg_s->ch_list, ch_entry) | ||
1135 | n++; | ||
1136 | |||
1137 | if (oob_size < n * sizeof(u16)) { | ||
1138 | err = -ENOMEM; | ||
1139 | goto done; | ||
1140 | } | ||
1141 | |||
1142 | msg.cmd = cmd; | ||
1143 | msg.handle = vgpu_get_handle(g); | ||
1144 | p = &msg.params.suspend_contexts; | ||
1145 | p->num_channels = n; | ||
1146 | n = 0; | ||
1147 | list_for_each_entry(ch_data, &dbg_s->ch_list, ch_entry) | ||
1148 | oob[n++] = (u16)ch_data->chid; | ||
1149 | |||
1150 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
1151 | if (err || msg.ret) { | ||
1152 | err = -ENOMEM; | ||
1153 | goto done; | ||
1154 | } | ||
1155 | |||
1156 | if (p->resident_chid != (u16)~0) { | ||
1157 | list_for_each_entry(ch_data, &dbg_s->ch_list, ch_entry) { | ||
1158 | if (ch_data->chid == p->resident_chid) { | ||
1159 | channel_fd = ch_data->channel_fd; | ||
1160 | break; | ||
1161 | } | ||
1162 | } | ||
1163 | } | ||
1164 | |||
1165 | done: | ||
1166 | if (handle) | ||
1167 | tegra_gr_comm_oob_put_ptr(handle); | ||
1168 | nvgpu_mutex_release(&dbg_s->ch_list_lock); | ||
1169 | nvgpu_mutex_release(&g->dbg_sessions_lock); | ||
1170 | *ctx_resident_ch_fd = channel_fd; | ||
1171 | return err; | ||
1172 | } | ||
1173 | |||
1174 | int vgpu_gr_suspend_contexts(struct gk20a *g, | ||
1175 | struct dbg_session_gk20a *dbg_s, | ||
1176 | int *ctx_resident_ch_fd) | ||
1177 | { | ||
1178 | return vgpu_gr_suspend_resume_contexts(g, dbg_s, | ||
1179 | ctx_resident_ch_fd, TEGRA_VGPU_CMD_SUSPEND_CONTEXTS); | ||
1180 | } | ||
1181 | |||
1182 | int vgpu_gr_resume_contexts(struct gk20a *g, | ||
1183 | struct dbg_session_gk20a *dbg_s, | ||
1184 | int *ctx_resident_ch_fd) | ||
1185 | { | ||
1186 | return vgpu_gr_suspend_resume_contexts(g, dbg_s, | ||
1187 | ctx_resident_ch_fd, TEGRA_VGPU_CMD_RESUME_CONTEXTS); | ||
1188 | } | ||
1189 | |||
1190 | void vgpu_gr_handle_sm_esr_event(struct gk20a *g, | ||
1191 | struct tegra_vgpu_sm_esr_info *info) | ||
1192 | { | ||
1193 | struct nvgpu_gr_sm_error_state *sm_error_states; | ||
1194 | |||
1195 | if (info->sm_id >= g->gr.no_of_sm) { | ||
1196 | nvgpu_err(g, "invalid smd_id %d / %d", | ||
1197 | info->sm_id, g->gr.no_of_sm); | ||
1198 | return; | ||
1199 | } | ||
1200 | |||
1201 | nvgpu_mutex_acquire(&g->dbg_sessions_lock); | ||
1202 | |||
1203 | sm_error_states = &g->gr.sm_error_states[info->sm_id]; | ||
1204 | |||
1205 | sm_error_states->hww_global_esr = info->hww_global_esr; | ||
1206 | sm_error_states->hww_warp_esr = info->hww_warp_esr; | ||
1207 | sm_error_states->hww_warp_esr_pc = info->hww_warp_esr_pc; | ||
1208 | sm_error_states->hww_global_esr_report_mask = | ||
1209 | info->hww_global_esr_report_mask; | ||
1210 | sm_error_states->hww_warp_esr_report_mask = | ||
1211 | info->hww_warp_esr_report_mask; | ||
1212 | |||
1213 | nvgpu_mutex_release(&g->dbg_sessions_lock); | ||
1214 | } | ||