summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/vgpu
diff options
context:
space:
mode:
authorRichard Zhao <rizhao@nvidia.com>2018-01-30 02:24:37 -0500
committermobile promotions <svcmobile_promotions@nvidia.com>2018-02-27 17:30:52 -0500
commit6393eddfa996fba03464f897b85aa5ec79860fed (patch)
tree557ebe9be93e2b0464118e7d8ec019d9d5dbae5f /drivers/gpu/nvgpu/vgpu
parent7932568b7fe9e16b2b83bc58b2b3686c0d5e52d4 (diff)
gpu: nvgpu: vgpu: move common files out of linux folder
Most of files have been moved out of linux folder. More code could be common as halifying going on. Jira EVLR-2364 Change-Id: Ia9dbdbc82f45ceefe5c788eac7517000cd455d5e Signed-off-by: Richard Zhao <rizhao@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/1649947 Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com> Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/vgpu')
-rw-r--r--drivers/gpu/nvgpu/vgpu/ce2_vgpu.c52
-rw-r--r--drivers/gpu/nvgpu/vgpu/css_vgpu.h40
-rw-r--r--drivers/gpu/nvgpu/vgpu/dbg_vgpu.c214
-rw-r--r--drivers/gpu/nvgpu/vgpu/dbg_vgpu.h47
-rw-r--r--drivers/gpu/nvgpu/vgpu/fecs_trace_vgpu.h47
-rw-r--r--drivers/gpu/nvgpu/vgpu/fifo_vgpu.c770
-rw-r--r--drivers/gpu/nvgpu/vgpu/fifo_vgpu.h65
-rw-r--r--drivers/gpu/nvgpu/vgpu/gm20b/vgpu_gr_gm20b.c48
-rw-r--r--drivers/gpu/nvgpu/vgpu/gm20b/vgpu_gr_gm20b.h30
-rw-r--r--drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fifo_gp10b.c30
-rw-r--r--drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fuse_gp10b.c38
-rw-r--r--drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fuse_gp10b.h30
-rw-r--r--drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.c308
-rw-r--r--drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.h43
-rw-r--r--drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c613
-rw-r--r--drivers/gpu/nvgpu/vgpu/gp10b/vgpu_mm_gp10b.c206
-rw-r--r--drivers/gpu/nvgpu/vgpu/gp10b/vgpu_mm_gp10b.h45
-rw-r--r--drivers/gpu/nvgpu/vgpu/gr_vgpu.c1283
-rw-r--r--drivers/gpu/nvgpu/vgpu/gr_vgpu.h74
-rw-r--r--drivers/gpu/nvgpu/vgpu/gv11b/vgpu_fifo_gv11b.c145
-rw-r--r--drivers/gpu/nvgpu/vgpu/gv11b/vgpu_fifo_gv11b.h33
-rw-r--r--drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gr_gv11b.c40
-rw-r--r--drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gr_gv11b.h30
-rw-r--r--drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gv11b.c47
-rw-r--r--drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gv11b.h30
-rw-r--r--drivers/gpu/nvgpu/vgpu/gv11b/vgpu_hal_gv11b.c603
-rw-r--r--drivers/gpu/nvgpu/vgpu/gv11b/vgpu_subctx_gv11b.c80
-rw-r--r--drivers/gpu/nvgpu/vgpu/gv11b/vgpu_subctx_gv11b.h31
-rw-r--r--drivers/gpu/nvgpu/vgpu/gv11b/vgpu_tsg_gv11b.c59
-rw-r--r--drivers/gpu/nvgpu/vgpu/gv11b/vgpu_tsg_gv11b.h29
-rw-r--r--drivers/gpu/nvgpu/vgpu/ltc_vgpu.c69
-rw-r--r--drivers/gpu/nvgpu/vgpu/ltc_vgpu.h33
-rw-r--r--drivers/gpu/nvgpu/vgpu/mm_vgpu.c278
-rw-r--r--drivers/gpu/nvgpu/vgpu/mm_vgpu.h47
-rw-r--r--drivers/gpu/nvgpu/vgpu/tsg_vgpu.c160
-rw-r--r--drivers/gpu/nvgpu/vgpu/vgpu.c350
36 files changed, 6047 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/vgpu/ce2_vgpu.c b/drivers/gpu/nvgpu/vgpu/ce2_vgpu.c
new file mode 100644
index 00000000..914041ff
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/ce2_vgpu.c
@@ -0,0 +1,52 @@
1/*
2 * Virtualized GPU CE2
3 *
4 * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include "gk20a/channel_gk20a.h"
26
27#include <nvgpu/bug.h>
28#include <nvgpu/vgpu/vgpu.h>
29
30int vgpu_ce2_nonstall_isr(struct gk20a *g,
31 struct tegra_vgpu_ce2_nonstall_intr_info *info)
32{
33 gk20a_dbg_fn("");
34
35 switch (info->type) {
36 case TEGRA_VGPU_CE2_NONSTALL_INTR_NONBLOCKPIPE:
37 gk20a_channel_semaphore_wakeup(g, true);
38 break;
39 default:
40 WARN_ON(1);
41 break;
42 }
43
44 return 0;
45}
46
47u32 vgpu_ce_get_num_pce(struct gk20a *g)
48{
49 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
50
51 return priv->constants.num_pce;
52}
diff --git a/drivers/gpu/nvgpu/vgpu/css_vgpu.h b/drivers/gpu/nvgpu/vgpu/css_vgpu.h
new file mode 100644
index 00000000..de4466f3
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/css_vgpu.h
@@ -0,0 +1,40 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _CSS_VGPU_H_
24#define _CSS_VGPU_H_
25
26#include <nvgpu/types.h>
27
28struct gr_gk20a;
29struct channel_gk20a;
30struct gk20a_cs_snapshot_client;
31
32void vgpu_css_release_snapshot_buffer(struct gr_gk20a *gr);
33int vgpu_css_flush_snapshots(struct channel_gk20a *ch,
34 u32 *pending, bool *hw_overflow);
35int vgpu_css_detach(struct channel_gk20a *ch,
36 struct gk20a_cs_snapshot_client *cs_client);
37int vgpu_css_enable_snapshot_buffer(struct channel_gk20a *ch,
38 struct gk20a_cs_snapshot_client *cs_client);
39u32 vgpu_css_get_buffer_size(struct gk20a *g);
40#endif
diff --git a/drivers/gpu/nvgpu/vgpu/dbg_vgpu.c b/drivers/gpu/nvgpu/vgpu/dbg_vgpu.c
new file mode 100644
index 00000000..092954ed
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/dbg_vgpu.c
@@ -0,0 +1,214 @@
1/*
2 * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/vgpu/vgpu_ivc.h>
24#include <nvgpu/vgpu/tegra_vgpu.h>
25#include <nvgpu/vgpu/vgpu.h>
26#include <nvgpu/bug.h>
27
28#include "gk20a/gk20a.h"
29#include "gk20a/channel_gk20a.h"
30#include "gk20a/dbg_gpu_gk20a.h"
31#include "gk20a/regops_gk20a.h"
32#include "dbg_vgpu.h"
33
34int vgpu_exec_regops(struct dbg_session_gk20a *dbg_s,
35 struct nvgpu_dbg_reg_op *ops,
36 u64 num_ops)
37{
38 struct channel_gk20a *ch;
39 struct tegra_vgpu_cmd_msg msg;
40 struct tegra_vgpu_reg_ops_params *p = &msg.params.reg_ops;
41 void *oob;
42 size_t oob_size, ops_size;
43 void *handle = NULL;
44 int err = 0;
45
46 gk20a_dbg_fn("");
47 BUG_ON(sizeof(*ops) != sizeof(struct tegra_vgpu_reg_op));
48
49 handle = vgpu_ivc_oob_get_ptr(vgpu_ivc_get_server_vmid(),
50 TEGRA_VGPU_QUEUE_CMD,
51 &oob, &oob_size);
52 if (!handle)
53 return -EINVAL;
54
55 ops_size = sizeof(*ops) * num_ops;
56 if (oob_size < ops_size) {
57 err = -ENOMEM;
58 goto fail;
59 }
60
61 memcpy(oob, ops, ops_size);
62
63 msg.cmd = TEGRA_VGPU_CMD_REG_OPS;
64 msg.handle = vgpu_get_handle(dbg_s->g);
65 ch = nvgpu_dbg_gpu_get_session_channel(dbg_s);
66 p->handle = ch ? ch->virt_ctx : 0;
67 p->num_ops = num_ops;
68 p->is_profiler = dbg_s->is_profiler;
69 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
70 err = err ? err : msg.ret;
71 if (!err)
72 memcpy(ops, oob, ops_size);
73
74fail:
75 vgpu_ivc_oob_put_ptr(handle);
76 return err;
77}
78
79int vgpu_dbg_set_powergate(struct dbg_session_gk20a *dbg_s, bool disable_powergate)
80{
81 struct tegra_vgpu_cmd_msg msg;
82 struct tegra_vgpu_set_powergate_params *p = &msg.params.set_powergate;
83 int err = 0;
84 u32 mode;
85
86 gk20a_dbg_fn("");
87
88 /* Just return if requested mode is the same as the session's mode */
89 if (disable_powergate) {
90 if (dbg_s->is_pg_disabled)
91 return 0;
92 dbg_s->is_pg_disabled = true;
93 mode = TEGRA_VGPU_POWERGATE_MODE_DISABLE;
94 } else {
95 if (!dbg_s->is_pg_disabled)
96 return 0;
97 dbg_s->is_pg_disabled = false;
98 mode = TEGRA_VGPU_POWERGATE_MODE_ENABLE;
99 }
100
101 msg.cmd = TEGRA_VGPU_CMD_SET_POWERGATE;
102 msg.handle = vgpu_get_handle(dbg_s->g);
103 p->mode = mode;
104 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
105 err = err ? err : msg.ret;
106 return err;
107}
108
109static int vgpu_sendrecv_prof_cmd(struct dbg_session_gk20a *dbg_s, u32 mode)
110{
111 struct tegra_vgpu_cmd_msg msg;
112 struct tegra_vgpu_prof_mgt_params *p = &msg.params.prof_management;
113 int err = 0;
114
115 msg.cmd = TEGRA_VGPU_CMD_PROF_MGT;
116 msg.handle = vgpu_get_handle(dbg_s->g);
117
118 p->mode = mode;
119
120 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
121 err = err ? err : msg.ret;
122 return err;
123}
124
125bool vgpu_check_and_set_global_reservation(
126 struct dbg_session_gk20a *dbg_s,
127 struct dbg_profiler_object_data *prof_obj)
128{
129 struct gk20a *g = dbg_s->g;
130
131 if (g->profiler_reservation_count > 0)
132 return false;
133
134 /* Check that another guest OS doesn't already have a reservation */
135 if (!vgpu_sendrecv_prof_cmd(dbg_s, TEGRA_VGPU_PROF_GET_GLOBAL)) {
136 g->global_profiler_reservation_held = true;
137 g->profiler_reservation_count = 1;
138 dbg_s->has_profiler_reservation = true;
139 prof_obj->has_reservation = true;
140 return true;
141 }
142 return false;
143}
144
145bool vgpu_check_and_set_context_reservation(
146 struct dbg_session_gk20a *dbg_s,
147 struct dbg_profiler_object_data *prof_obj)
148{
149 struct gk20a *g = dbg_s->g;
150
151 /* Assumes that we've already checked that no global reservation
152 * is in effect for this guest.
153 *
154 * If our reservation count is non-zero, then no other guest has the
155 * global reservation; if it is zero, need to check with RM server.
156 *
157 */
158 if ((g->profiler_reservation_count != 0) ||
159 !vgpu_sendrecv_prof_cmd(dbg_s, TEGRA_VGPU_PROF_GET_CONTEXT)) {
160 g->profiler_reservation_count++;
161 dbg_s->has_profiler_reservation = true;
162 prof_obj->has_reservation = true;
163 return true;
164 }
165 return false;
166}
167
168void vgpu_release_profiler_reservation(
169 struct dbg_session_gk20a *dbg_s,
170 struct dbg_profiler_object_data *prof_obj)
171{
172 struct gk20a *g = dbg_s->g;
173
174 dbg_s->has_profiler_reservation = false;
175 prof_obj->has_reservation = false;
176 if (prof_obj->ch == NULL)
177 g->global_profiler_reservation_held = false;
178
179 /* If new reservation count is zero, notify server */
180 g->profiler_reservation_count--;
181 if (g->profiler_reservation_count == 0)
182 vgpu_sendrecv_prof_cmd(dbg_s, TEGRA_VGPU_PROF_RELEASE);
183}
184
185static int vgpu_sendrecv_perfbuf_cmd(struct gk20a *g, u64 offset, u32 size)
186{
187 struct mm_gk20a *mm = &g->mm;
188 struct vm_gk20a *vm = mm->perfbuf.vm;
189 struct tegra_vgpu_cmd_msg msg;
190 struct tegra_vgpu_perfbuf_mgt_params *p =
191 &msg.params.perfbuf_management;
192 int err;
193
194 msg.cmd = TEGRA_VGPU_CMD_PERFBUF_MGT;
195 msg.handle = vgpu_get_handle(g);
196
197 p->vm_handle = vm->handle;
198 p->offset = offset;
199 p->size = size;
200
201 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
202 err = err ? err : msg.ret;
203 return err;
204}
205
206int vgpu_perfbuffer_enable(struct gk20a *g, u64 offset, u32 size)
207{
208 return vgpu_sendrecv_perfbuf_cmd(g, offset, size);
209}
210
211int vgpu_perfbuffer_disable(struct gk20a *g)
212{
213 return vgpu_sendrecv_perfbuf_cmd(g, 0, 0);
214}
diff --git a/drivers/gpu/nvgpu/vgpu/dbg_vgpu.h b/drivers/gpu/nvgpu/vgpu/dbg_vgpu.h
new file mode 100644
index 00000000..90645e59
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/dbg_vgpu.h
@@ -0,0 +1,47 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _DBG_VGPU_H_
24#define _DBG_VGPU_H_
25
26struct dbg_session_gk20a;
27struct nvgpu_dbg_reg_op;
28struct dbg_profiler_object_data;
29struct gk20a;
30
31int vgpu_exec_regops(struct dbg_session_gk20a *dbg_s,
32 struct nvgpu_dbg_reg_op *ops,
33 u64 num_ops);
34int vgpu_dbg_set_powergate(struct dbg_session_gk20a *dbg_s, bool disable_powergate);
35bool vgpu_check_and_set_global_reservation(
36 struct dbg_session_gk20a *dbg_s,
37 struct dbg_profiler_object_data *prof_obj);
38bool vgpu_check_and_set_context_reservation(
39 struct dbg_session_gk20a *dbg_s,
40 struct dbg_profiler_object_data *prof_obj);
41
42void vgpu_release_profiler_reservation(
43 struct dbg_session_gk20a *dbg_s,
44 struct dbg_profiler_object_data *prof_obj);
45int vgpu_perfbuffer_enable(struct gk20a *g, u64 offset, u32 size);
46int vgpu_perfbuffer_disable(struct gk20a *g);
47#endif
diff --git a/drivers/gpu/nvgpu/vgpu/fecs_trace_vgpu.h b/drivers/gpu/nvgpu/vgpu/fecs_trace_vgpu.h
new file mode 100644
index 00000000..b957a363
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/fecs_trace_vgpu.h
@@ -0,0 +1,47 @@
1/*
2 * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef __FECS_TRACE_VGPU_H
24#define __FECS_TRACE_VGPU_H
25
26#include <nvgpu/types.h>
27
28struct gk20a;
29struct vm_area_struct;
30struct nvgpu_ctxsw_trace_filter;
31
32void vgpu_fecs_trace_data_update(struct gk20a *g);
33int vgpu_fecs_trace_init(struct gk20a *g);
34int vgpu_fecs_trace_deinit(struct gk20a *g);
35int vgpu_fecs_trace_enable(struct gk20a *g);
36int vgpu_fecs_trace_disable(struct gk20a *g);
37bool vgpu_fecs_trace_is_enabled(struct gk20a *g);
38int vgpu_fecs_trace_poll(struct gk20a *g);
39int vgpu_alloc_user_buffer(struct gk20a *g, void **buf, size_t *size);
40int vgpu_free_user_buffer(struct gk20a *g);
41int vgpu_mmap_user_buffer(struct gk20a *g, struct vm_area_struct *vma);
42int vgpu_fecs_trace_max_entries(struct gk20a *g,
43 struct nvgpu_ctxsw_trace_filter *filter);
44int vgpu_fecs_trace_set_filter(struct gk20a *g,
45 struct nvgpu_ctxsw_trace_filter *filter);
46
47#endif /* __FECS_TRACE_VGPU_H */
diff --git a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c
new file mode 100644
index 00000000..580bfb60
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c
@@ -0,0 +1,770 @@
1/*
2 * Virtualized GPU Fifo
3 *
4 * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <trace/events/gk20a.h>
26
27#include <nvgpu/kmem.h>
28#include <nvgpu/dma.h>
29#include <nvgpu/atomic.h>
30#include <nvgpu/bug.h>
31#include <nvgpu/barrier.h>
32#include <nvgpu/error_notifier.h>
33#include <nvgpu/vgpu/vgpu_ivc.h>
34#include <nvgpu/vgpu/vgpu.h>
35
36#include "gk20a/gk20a.h"
37#include "fifo_vgpu.h"
38
39#include <nvgpu/hw/gk20a/hw_fifo_gk20a.h>
40#include <nvgpu/hw/gk20a/hw_ram_gk20a.h>
41
42void vgpu_channel_bind(struct channel_gk20a *ch)
43{
44 struct tegra_vgpu_cmd_msg msg;
45 struct tegra_vgpu_channel_config_params *p =
46 &msg.params.channel_config;
47 int err;
48
49 gk20a_dbg_info("bind channel %d", ch->chid);
50
51 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_BIND;
52 msg.handle = vgpu_get_handle(ch->g);
53 p->handle = ch->virt_ctx;
54 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
55 WARN_ON(err || msg.ret);
56
57 nvgpu_smp_wmb();
58 nvgpu_atomic_set(&ch->bound, true);
59}
60
61void vgpu_channel_unbind(struct channel_gk20a *ch)
62{
63
64 gk20a_dbg_fn("");
65
66 if (nvgpu_atomic_cmpxchg(&ch->bound, true, false)) {
67 struct tegra_vgpu_cmd_msg msg;
68 struct tegra_vgpu_channel_config_params *p =
69 &msg.params.channel_config;
70 int err;
71
72 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_UNBIND;
73 msg.handle = vgpu_get_handle(ch->g);
74 p->handle = ch->virt_ctx;
75 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
76 WARN_ON(err || msg.ret);
77 }
78
79}
80
81int vgpu_channel_alloc_inst(struct gk20a *g, struct channel_gk20a *ch)
82{
83 struct tegra_vgpu_cmd_msg msg;
84 struct tegra_vgpu_channel_hwctx_params *p = &msg.params.channel_hwctx;
85 int err;
86
87 gk20a_dbg_fn("");
88
89 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_ALLOC_HWCTX;
90 msg.handle = vgpu_get_handle(g);
91 p->id = ch->chid;
92 p->pid = (u64)current->tgid;
93 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
94 if (err || msg.ret) {
95 nvgpu_err(g, "fail");
96 return -ENOMEM;
97 }
98
99 ch->virt_ctx = p->handle;
100 gk20a_dbg_fn("done");
101 return 0;
102}
103
104void vgpu_channel_free_inst(struct gk20a *g, struct channel_gk20a *ch)
105{
106 struct tegra_vgpu_cmd_msg msg;
107 struct tegra_vgpu_channel_hwctx_params *p = &msg.params.channel_hwctx;
108 int err;
109
110 gk20a_dbg_fn("");
111
112 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_FREE_HWCTX;
113 msg.handle = vgpu_get_handle(g);
114 p->handle = ch->virt_ctx;
115 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
116 WARN_ON(err || msg.ret);
117}
118
119void vgpu_channel_enable(struct channel_gk20a *ch)
120{
121 struct tegra_vgpu_cmd_msg msg;
122 struct tegra_vgpu_channel_config_params *p =
123 &msg.params.channel_config;
124 int err;
125
126 gk20a_dbg_fn("");
127
128 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_ENABLE;
129 msg.handle = vgpu_get_handle(ch->g);
130 p->handle = ch->virt_ctx;
131 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
132 WARN_ON(err || msg.ret);
133}
134
135void vgpu_channel_disable(struct channel_gk20a *ch)
136{
137 struct tegra_vgpu_cmd_msg msg;
138 struct tegra_vgpu_channel_config_params *p =
139 &msg.params.channel_config;
140 int err;
141
142 gk20a_dbg_fn("");
143
144 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_DISABLE;
145 msg.handle = vgpu_get_handle(ch->g);
146 p->handle = ch->virt_ctx;
147 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
148 WARN_ON(err || msg.ret);
149}
150
151int vgpu_channel_setup_ramfc(struct channel_gk20a *ch, u64 gpfifo_base,
152 u32 gpfifo_entries,
153 unsigned long acquire_timeout, u32 flags)
154{
155 struct tegra_vgpu_cmd_msg msg;
156 struct tegra_vgpu_ramfc_params *p = &msg.params.ramfc;
157 int err;
158
159 gk20a_dbg_fn("");
160
161 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SETUP_RAMFC;
162 msg.handle = vgpu_get_handle(ch->g);
163 p->handle = ch->virt_ctx;
164 p->gpfifo_va = gpfifo_base;
165 p->num_entries = gpfifo_entries;
166 p->userd_addr = ch->userd_iova;
167 p->iova = 0;
168 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
169
170 return (err || msg.ret) ? -ENOMEM : 0;
171}
172
173int vgpu_fifo_init_engine_info(struct fifo_gk20a *f)
174{
175 struct vgpu_priv_data *priv = vgpu_get_priv_data(f->g);
176 struct tegra_vgpu_engines_info *engines = &priv->constants.engines_info;
177 u32 i;
178
179 gk20a_dbg_fn("");
180
181 if (engines->num_engines > TEGRA_VGPU_MAX_ENGINES) {
182 nvgpu_err(f->g, "num_engines %d larger than max %d",
183 engines->num_engines, TEGRA_VGPU_MAX_ENGINES);
184 return -EINVAL;
185 }
186
187 f->num_engines = engines->num_engines;
188 for (i = 0; i < f->num_engines; i++) {
189 struct fifo_engine_info_gk20a *info =
190 &f->engine_info[engines->info[i].engine_id];
191
192 if (engines->info[i].engine_id >= f->max_engines) {
193 nvgpu_err(f->g, "engine id %d larger than max %d",
194 engines->info[i].engine_id,
195 f->max_engines);
196 return -EINVAL;
197 }
198
199 info->intr_mask = engines->info[i].intr_mask;
200 info->reset_mask = engines->info[i].reset_mask;
201 info->runlist_id = engines->info[i].runlist_id;
202 info->pbdma_id = engines->info[i].pbdma_id;
203 info->inst_id = engines->info[i].inst_id;
204 info->pri_base = engines->info[i].pri_base;
205 info->engine_enum = engines->info[i].engine_enum;
206 info->fault_id = engines->info[i].fault_id;
207 f->active_engines_list[i] = engines->info[i].engine_id;
208 }
209
210 gk20a_dbg_fn("done");
211
212 return 0;
213}
214
215static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
216{
217 struct fifo_runlist_info_gk20a *runlist;
218 unsigned int runlist_id = -1;
219 u32 i;
220 u64 runlist_size;
221
222 gk20a_dbg_fn("");
223
224 f->max_runlists = g->ops.fifo.eng_runlist_base_size();
225 f->runlist_info = nvgpu_kzalloc(g,
226 sizeof(struct fifo_runlist_info_gk20a) *
227 f->max_runlists);
228 if (!f->runlist_info)
229 goto clean_up_runlist;
230
231 memset(f->runlist_info, 0, (sizeof(struct fifo_runlist_info_gk20a) *
232 f->max_runlists));
233
234 for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) {
235 runlist = &f->runlist_info[runlist_id];
236
237 runlist->active_channels =
238 nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
239 BITS_PER_BYTE));
240 if (!runlist->active_channels)
241 goto clean_up_runlist;
242
243 runlist_size = sizeof(u16) * f->num_channels;
244 for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
245 int err = nvgpu_dma_alloc_sys(g, runlist_size,
246 &runlist->mem[i]);
247 if (err) {
248 nvgpu_err(g, "memory allocation failed");
249 goto clean_up_runlist;
250 }
251 }
252 nvgpu_mutex_init(&runlist->mutex);
253
254 /* None of buffers is pinned if this value doesn't change.
255 Otherwise, one of them (cur_buffer) must have been pinned. */
256 runlist->cur_buffer = MAX_RUNLIST_BUFFERS;
257 }
258
259 gk20a_dbg_fn("done");
260 return 0;
261
262clean_up_runlist:
263 gk20a_fifo_delete_runlist(f);
264 gk20a_dbg_fn("fail");
265 return -ENOMEM;
266}
267
268static int vgpu_init_fifo_setup_sw(struct gk20a *g)
269{
270 struct fifo_gk20a *f = &g->fifo;
271 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
272 unsigned int chid;
273 int err = 0;
274
275 gk20a_dbg_fn("");
276
277 if (f->sw_ready) {
278 gk20a_dbg_fn("skip init");
279 return 0;
280 }
281
282 f->g = g;
283 f->num_channels = priv->constants.num_channels;
284 f->max_engines = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_ENGINES);
285
286 f->userd_entry_size = 1 << ram_userd_base_shift_v();
287
288 err = nvgpu_dma_alloc_sys(g, f->userd_entry_size * f->num_channels,
289 &f->userd);
290 if (err) {
291 nvgpu_err(g, "memory allocation failed");
292 goto clean_up;
293 }
294
295 /* bar1 va */
296 if (g->ops.mm.is_bar1_supported(g)) {
297 f->userd.gpu_va = vgpu_bar1_map(g, &f->userd);
298 if (!f->userd.gpu_va) {
299 nvgpu_err(g, "gmmu mapping failed");
300 goto clean_up;
301 }
302 /* if reduced BAR1 range is specified, use offset of 0
303 * (server returns offset assuming full BAR1 range)
304 */
305 if (vgpu_is_reduced_bar1(g))
306 f->userd.gpu_va = 0;
307 }
308
309 gk20a_dbg(gpu_dbg_map_v, "userd bar1 va = 0x%llx", f->userd.gpu_va);
310
311 f->channel = nvgpu_vzalloc(g, f->num_channels * sizeof(*f->channel));
312 f->tsg = nvgpu_vzalloc(g, f->num_channels * sizeof(*f->tsg));
313 f->engine_info = nvgpu_kzalloc(g, f->max_engines *
314 sizeof(*f->engine_info));
315 f->active_engines_list = nvgpu_kzalloc(g, f->max_engines * sizeof(u32));
316
317 if (!(f->channel && f->tsg && f->engine_info && f->active_engines_list)) {
318 err = -ENOMEM;
319 goto clean_up;
320 }
321 memset(f->active_engines_list, 0xff, (f->max_engines * sizeof(u32)));
322
323 g->ops.fifo.init_engine_info(f);
324
325 init_runlist(g, f);
326
327 nvgpu_init_list_node(&f->free_chs);
328 nvgpu_mutex_init(&f->free_chs_mutex);
329
330 for (chid = 0; chid < f->num_channels; chid++) {
331 f->channel[chid].userd_iova =
332 nvgpu_mem_get_addr(g, &f->userd) +
333 chid * f->userd_entry_size;
334 f->channel[chid].userd_gpu_va =
335 f->userd.gpu_va + chid * f->userd_entry_size;
336
337 gk20a_init_channel_support(g, chid);
338 gk20a_init_tsg_support(g, chid);
339 }
340 nvgpu_mutex_init(&f->tsg_inuse_mutex);
341
342 err = nvgpu_channel_worker_init(g);
343 if (err)
344 goto clean_up;
345
346 f->deferred_reset_pending = false;
347 nvgpu_mutex_init(&f->deferred_reset_mutex);
348
349 f->channel_base = priv->constants.channel_base;
350
351 f->sw_ready = true;
352
353 gk20a_dbg_fn("done");
354 return 0;
355
356clean_up:
357 gk20a_dbg_fn("fail");
358 /* FIXME: unmap from bar1 */
359 nvgpu_dma_free(g, &f->userd);
360
361 memset(&f->userd, 0, sizeof(f->userd));
362
363 nvgpu_vfree(g, f->channel);
364 f->channel = NULL;
365 nvgpu_vfree(g, f->tsg);
366 f->tsg = NULL;
367 nvgpu_kfree(g, f->engine_info);
368 f->engine_info = NULL;
369 nvgpu_kfree(g, f->active_engines_list);
370 f->active_engines_list = NULL;
371
372 return err;
373}
374
375int vgpu_init_fifo_setup_hw(struct gk20a *g)
376{
377 gk20a_dbg_fn("");
378
379 /* test write, read through bar1 @ userd region before
380 * turning on the snooping */
381 {
382 struct fifo_gk20a *f = &g->fifo;
383 u32 v, v1 = 0x33, v2 = 0x55;
384
385 u32 bar1_vaddr = f->userd.gpu_va;
386 volatile u32 *cpu_vaddr = f->userd.cpu_va;
387
388 gk20a_dbg_info("test bar1 @ vaddr 0x%x",
389 bar1_vaddr);
390
391 v = gk20a_bar1_readl(g, bar1_vaddr);
392
393 *cpu_vaddr = v1;
394 nvgpu_mb();
395
396 if (v1 != gk20a_bar1_readl(g, bar1_vaddr)) {
397 nvgpu_err(g, "bar1 broken @ gk20a!");
398 return -EINVAL;
399 }
400
401 gk20a_bar1_writel(g, bar1_vaddr, v2);
402
403 if (v2 != gk20a_bar1_readl(g, bar1_vaddr)) {
404 nvgpu_err(g, "bar1 broken @ gk20a!");
405 return -EINVAL;
406 }
407
408 /* is it visible to the cpu? */
409 if (*cpu_vaddr != v2) {
410 nvgpu_err(g, "cpu didn't see bar1 write @ %p!",
411 cpu_vaddr);
412 }
413
414 /* put it back */
415 gk20a_bar1_writel(g, bar1_vaddr, v);
416 }
417
418 gk20a_dbg_fn("done");
419
420 return 0;
421}
422
423int vgpu_init_fifo_support(struct gk20a *g)
424{
425 u32 err;
426
427 gk20a_dbg_fn("");
428
429 err = vgpu_init_fifo_setup_sw(g);
430 if (err)
431 return err;
432
433 if (g->ops.fifo.init_fifo_setup_hw)
434 err = g->ops.fifo.init_fifo_setup_hw(g);
435 return err;
436}
437
438int vgpu_fifo_preempt_channel(struct gk20a *g, u32 chid)
439{
440 struct fifo_gk20a *f = &g->fifo;
441 struct channel_gk20a *ch = &f->channel[chid];
442 struct tegra_vgpu_cmd_msg msg;
443 struct tegra_vgpu_channel_config_params *p =
444 &msg.params.channel_config;
445 int err;
446
447 gk20a_dbg_fn("");
448
449 if (!nvgpu_atomic_read(&ch->bound))
450 return 0;
451
452 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_PREEMPT;
453 msg.handle = vgpu_get_handle(g);
454 p->handle = ch->virt_ctx;
455 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
456
457 if (err || msg.ret) {
458 nvgpu_err(g,
459 "preempt channel %d failed", chid);
460 err = -ENOMEM;
461 }
462
463 return err;
464}
465
466int vgpu_fifo_preempt_tsg(struct gk20a *g, u32 tsgid)
467{
468 struct tegra_vgpu_cmd_msg msg;
469 struct tegra_vgpu_tsg_preempt_params *p =
470 &msg.params.tsg_preempt;
471 int err;
472
473 gk20a_dbg_fn("");
474
475 msg.cmd = TEGRA_VGPU_CMD_TSG_PREEMPT;
476 msg.handle = vgpu_get_handle(g);
477 p->tsg_id = tsgid;
478 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
479 err = err ? err : msg.ret;
480
481 if (err) {
482 nvgpu_err(g,
483 "preempt tsg %u failed", tsgid);
484 }
485
486 return err;
487}
488
489static int vgpu_submit_runlist(struct gk20a *g, u64 handle, u8 runlist_id,
490 u16 *runlist, u32 num_entries)
491{
492 struct tegra_vgpu_cmd_msg msg;
493 struct tegra_vgpu_runlist_params *p;
494 int err;
495 void *oob_handle;
496 void *oob;
497 size_t size, oob_size;
498
499 oob_handle = vgpu_ivc_oob_get_ptr(vgpu_ivc_get_server_vmid(),
500 TEGRA_VGPU_QUEUE_CMD,
501 &oob, &oob_size);
502 if (!oob_handle)
503 return -EINVAL;
504
505 size = sizeof(*runlist) * num_entries;
506 if (oob_size < size) {
507 err = -ENOMEM;
508 goto done;
509 }
510
511 msg.cmd = TEGRA_VGPU_CMD_SUBMIT_RUNLIST;
512 msg.handle = handle;
513 p = &msg.params.runlist;
514 p->runlist_id = runlist_id;
515 p->num_entries = num_entries;
516
517 memcpy(oob, runlist, size);
518 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
519
520 err = (err || msg.ret) ? -1 : 0;
521
522done:
523 vgpu_ivc_oob_put_ptr(oob_handle);
524 return err;
525}
526
527static int vgpu_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
528 u32 chid, bool add,
529 bool wait_for_finish)
530{
531 struct fifo_gk20a *f = &g->fifo;
532 struct fifo_runlist_info_gk20a *runlist;
533 u16 *runlist_entry = NULL;
534 u32 count = 0;
535
536 gk20a_dbg_fn("");
537
538 runlist = &f->runlist_info[runlist_id];
539
540 /* valid channel, add/remove it from active list.
541 Otherwise, keep active list untouched for suspend/resume. */
542 if (chid != (u32)~0) {
543 if (add) {
544 if (test_and_set_bit(chid,
545 runlist->active_channels) == 1)
546 return 0;
547 } else {
548 if (test_and_clear_bit(chid,
549 runlist->active_channels) == 0)
550 return 0;
551 }
552 }
553
554 if (chid != (u32)~0 || /* add/remove a valid channel */
555 add /* resume to add all channels back */) {
556 u32 cid;
557
558 runlist_entry = runlist->mem[0].cpu_va;
559 for_each_set_bit(cid,
560 runlist->active_channels, f->num_channels) {
561 gk20a_dbg_info("add channel %d to runlist", cid);
562 runlist_entry[0] = cid;
563 runlist_entry++;
564 count++;
565 }
566 } else /* suspend to remove all channels */
567 count = 0;
568
569 return vgpu_submit_runlist(g, vgpu_get_handle(g), runlist_id,
570 runlist->mem[0].cpu_va, count);
571}
572
573/* add/remove a channel from runlist
574 special cases below: runlist->active_channels will NOT be changed.
575 (chid == ~0 && !add) means remove all active channels from runlist.
576 (chid == ~0 && add) means restore all active channels on runlist. */
577int vgpu_fifo_update_runlist(struct gk20a *g, u32 runlist_id,
578 u32 chid, bool add, bool wait_for_finish)
579{
580 struct fifo_runlist_info_gk20a *runlist = NULL;
581 struct fifo_gk20a *f = &g->fifo;
582 u32 ret = 0;
583
584 gk20a_dbg_fn("");
585
586 runlist = &f->runlist_info[runlist_id];
587
588 nvgpu_mutex_acquire(&runlist->mutex);
589
590 ret = vgpu_fifo_update_runlist_locked(g, runlist_id, chid, add,
591 wait_for_finish);
592
593 nvgpu_mutex_release(&runlist->mutex);
594 return ret;
595}
596
597int vgpu_fifo_wait_engine_idle(struct gk20a *g)
598{
599 gk20a_dbg_fn("");
600
601 return 0;
602}
603
604int vgpu_fifo_set_runlist_interleave(struct gk20a *g,
605 u32 id,
606 u32 runlist_id,
607 u32 new_level)
608{
609 struct tegra_vgpu_cmd_msg msg = {0};
610 struct tegra_vgpu_tsg_runlist_interleave_params *p =
611 &msg.params.tsg_interleave;
612 int err;
613
614 gk20a_dbg_fn("");
615
616 msg.cmd = TEGRA_VGPU_CMD_TSG_SET_RUNLIST_INTERLEAVE;
617 msg.handle = vgpu_get_handle(g);
618 p->tsg_id = id;
619 p->level = new_level;
620 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
621 WARN_ON(err || msg.ret);
622 return err ? err : msg.ret;
623}
624
625int vgpu_fifo_force_reset_ch(struct channel_gk20a *ch,
626 u32 err_code, bool verbose)
627{
628 struct tsg_gk20a *tsg = NULL;
629 struct channel_gk20a *ch_tsg = NULL;
630 struct gk20a *g = ch->g;
631 struct tegra_vgpu_cmd_msg msg = {0};
632 struct tegra_vgpu_channel_config_params *p =
633 &msg.params.channel_config;
634 int err;
635
636 gk20a_dbg_fn("");
637
638 if (gk20a_is_channel_marked_as_tsg(ch)) {
639 tsg = &g->fifo.tsg[ch->tsgid];
640
641 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
642
643 nvgpu_list_for_each_entry(ch_tsg, &tsg->ch_list,
644 channel_gk20a, ch_entry) {
645 if (gk20a_channel_get(ch_tsg)) {
646 nvgpu_set_error_notifier(ch_tsg, err_code);
647 ch_tsg->has_timedout = true;
648 gk20a_channel_put(ch_tsg);
649 }
650 }
651
652 nvgpu_rwsem_up_read(&tsg->ch_list_lock);
653 } else {
654 nvgpu_set_error_notifier(ch, err_code);
655 ch->has_timedout = true;
656 }
657
658 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_FORCE_RESET;
659 msg.handle = vgpu_get_handle(ch->g);
660 p->handle = ch->virt_ctx;
661 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
662 WARN_ON(err || msg.ret);
663 if (!err)
664 gk20a_channel_abort(ch, false);
665 return err ? err : msg.ret;
666}
667
668static void vgpu_fifo_set_ctx_mmu_error_ch(struct gk20a *g,
669 struct channel_gk20a *ch)
670{
671 /*
672 * If error code is already set, this mmu fault
673 * was triggered as part of recovery from other
674 * error condition.
675 * Don't overwrite error flag.
676 */
677 nvgpu_set_error_notifier_if_empty(ch,
678 NVGPU_ERR_NOTIFIER_FIFO_ERROR_MMU_ERR_FLT);
679
680 /* mark channel as faulted */
681 ch->has_timedout = true;
682 nvgpu_smp_wmb();
683 /* unblock pending waits */
684 nvgpu_cond_broadcast_interruptible(&ch->semaphore_wq);
685 nvgpu_cond_broadcast_interruptible(&ch->notifier_wq);
686}
687
688static void vgpu_fifo_set_ctx_mmu_error_ch_tsg(struct gk20a *g,
689 struct channel_gk20a *ch)
690{
691 struct tsg_gk20a *tsg = NULL;
692 struct channel_gk20a *ch_tsg = NULL;
693
694 if (gk20a_is_channel_marked_as_tsg(ch)) {
695 tsg = &g->fifo.tsg[ch->tsgid];
696
697 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
698
699 nvgpu_list_for_each_entry(ch_tsg, &tsg->ch_list,
700 channel_gk20a, ch_entry) {
701 if (gk20a_channel_get(ch_tsg)) {
702 vgpu_fifo_set_ctx_mmu_error_ch(g, ch_tsg);
703 gk20a_channel_put(ch_tsg);
704 }
705 }
706
707 nvgpu_rwsem_up_read(&tsg->ch_list_lock);
708 } else {
709 vgpu_fifo_set_ctx_mmu_error_ch(g, ch);
710 }
711}
712
713int vgpu_fifo_isr(struct gk20a *g, struct tegra_vgpu_fifo_intr_info *info)
714{
715 struct fifo_gk20a *f = &g->fifo;
716 struct channel_gk20a *ch = gk20a_channel_get(&f->channel[info->chid]);
717
718 gk20a_dbg_fn("");
719 if (!ch)
720 return 0;
721
722 nvgpu_err(g, "fifo intr (%d) on ch %u",
723 info->type, info->chid);
724
725 trace_gk20a_channel_reset(ch->chid, ch->tsgid);
726
727 switch (info->type) {
728 case TEGRA_VGPU_FIFO_INTR_PBDMA:
729 nvgpu_set_error_notifier(ch, NVGPU_ERR_NOTIFIER_PBDMA_ERROR);
730 break;
731 case TEGRA_VGPU_FIFO_INTR_CTXSW_TIMEOUT:
732 nvgpu_set_error_notifier(ch,
733 NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT);
734 break;
735 case TEGRA_VGPU_FIFO_INTR_MMU_FAULT:
736 vgpu_fifo_set_ctx_mmu_error_ch_tsg(g, ch);
737 gk20a_channel_abort(ch, false);
738 break;
739 default:
740 WARN_ON(1);
741 break;
742 }
743
744 gk20a_channel_put(ch);
745 return 0;
746}
747
748int vgpu_fifo_nonstall_isr(struct gk20a *g,
749 struct tegra_vgpu_fifo_nonstall_intr_info *info)
750{
751 gk20a_dbg_fn("");
752
753 switch (info->type) {
754 case TEGRA_VGPU_FIFO_NONSTALL_INTR_CHANNEL:
755 gk20a_channel_semaphore_wakeup(g, false);
756 break;
757 default:
758 WARN_ON(1);
759 break;
760 }
761
762 return 0;
763}
764
765u32 vgpu_fifo_default_timeslice_us(struct gk20a *g)
766{
767 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
768
769 return priv->constants.default_timeslice_us;
770}
diff --git a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.h b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.h
new file mode 100644
index 00000000..20205d3c
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.h
@@ -0,0 +1,65 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _FIFO_VGPU_H_
24#define _FIFO_VGPU_H_
25
26#include <nvgpu/types.h>
27
28struct gk20a;
29struct channel_gk20a;
30struct fifo_gk20a;
31struct tsg_gk20a;
32
33int vgpu_init_fifo_setup_hw(struct gk20a *g);
34void vgpu_channel_bind(struct channel_gk20a *ch);
35void vgpu_channel_unbind(struct channel_gk20a *ch);
36int vgpu_channel_alloc_inst(struct gk20a *g, struct channel_gk20a *ch);
37void vgpu_channel_free_inst(struct gk20a *g, struct channel_gk20a *ch);
38void vgpu_channel_enable(struct channel_gk20a *ch);
39void vgpu_channel_disable(struct channel_gk20a *ch);
40int vgpu_channel_setup_ramfc(struct channel_gk20a *ch, u64 gpfifo_base,
41 u32 gpfifo_entries,
42 unsigned long acquire_timeout, u32 flags);
43int vgpu_fifo_init_engine_info(struct fifo_gk20a *f);
44int vgpu_fifo_preempt_channel(struct gk20a *g, u32 chid);
45int vgpu_fifo_preempt_tsg(struct gk20a *g, u32 tsgid);
46int vgpu_fifo_update_runlist(struct gk20a *g, u32 runlist_id,
47 u32 chid, bool add, bool wait_for_finish);
48int vgpu_fifo_wait_engine_idle(struct gk20a *g);
49int vgpu_fifo_set_runlist_interleave(struct gk20a *g,
50 u32 id,
51 u32 runlist_id,
52 u32 new_level);
53int vgpu_channel_set_timeslice(struct channel_gk20a *ch, u32 timeslice);
54int vgpu_fifo_force_reset_ch(struct channel_gk20a *ch,
55 u32 err_code, bool verbose);
56u32 vgpu_fifo_default_timeslice_us(struct gk20a *g);
57int vgpu_tsg_open(struct tsg_gk20a *tsg);
58void vgpu_tsg_release(struct tsg_gk20a *tsg);
59int vgpu_tsg_bind_channel(struct tsg_gk20a *tsg,
60 struct channel_gk20a *ch);
61int vgpu_tsg_unbind_channel(struct channel_gk20a *ch);
62int vgpu_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice);
63int vgpu_enable_tsg(struct tsg_gk20a *tsg);
64
65#endif
diff --git a/drivers/gpu/nvgpu/vgpu/gm20b/vgpu_gr_gm20b.c b/drivers/gpu/nvgpu/vgpu/gm20b/vgpu_gr_gm20b.c
new file mode 100644
index 00000000..e83980a6
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gm20b/vgpu_gr_gm20b.c
@@ -0,0 +1,48 @@
1/*
2 * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/enabled.h>
24
25#include "gk20a/gk20a.h"
26#include "gk20a/css_gr_gk20a.h"
27#include "vgpu/css_vgpu.h"
28#include "vgpu_gr_gm20b.h"
29
30void vgpu_gr_gm20b_init_cyclestats(struct gk20a *g)
31{
32#if defined(CONFIG_GK20A_CYCLE_STATS)
33 bool snapshots_supported = true;
34
35 /* cyclestats not supported on vgpu */
36 __nvgpu_set_enabled(g, NVGPU_SUPPORT_CYCLE_STATS, false);
37
38 g->gr.max_css_buffer_size = vgpu_css_get_buffer_size(g);
39
40 /* snapshots not supported if the buffer size is 0 */
41 if (g->gr.max_css_buffer_size == 0)
42 snapshots_supported = false;
43
44 __nvgpu_set_enabled(g, NVGPU_SUPPORT_CYCLE_STATS_SNAPSHOT,
45 snapshots_supported);
46#endif
47}
48
diff --git a/drivers/gpu/nvgpu/vgpu/gm20b/vgpu_gr_gm20b.h b/drivers/gpu/nvgpu/vgpu/gm20b/vgpu_gr_gm20b.h
new file mode 100644
index 00000000..00759433
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gm20b/vgpu_gr_gm20b.h
@@ -0,0 +1,30 @@
1/*
2 * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef __VGPU_GR_GM20B_H__
24#define __VGPU_GR_GM20B_H__
25
26#include "gk20a/gk20a.h"
27
28void vgpu_gr_gm20b_init_cyclestats(struct gk20a *g);
29
30#endif
diff --git a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fifo_gp10b.c b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fifo_gp10b.c
new file mode 100644
index 00000000..ac187227
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fifo_gp10b.c
@@ -0,0 +1,30 @@
1/*
2 * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include "vgpu_fifo_gp10b.h"
24
25void vgpu_gp10b_init_fifo_ops(struct gpu_ops *gops)
26{
27 /* syncpoint protection not supported yet */
28 gops->fifo.resetup_ramfc = NULL;
29 gops->fifo.reschedule_runlist = NULL;
30}
diff --git a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fuse_gp10b.c b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fuse_gp10b.c
new file mode 100644
index 00000000..52b2aee5
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fuse_gp10b.c
@@ -0,0 +1,38 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/enabled.h>
24
25#include "gk20a/gk20a.h"
26
27int vgpu_gp10b_fuse_check_priv_security(struct gk20a *g)
28{
29 if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) {
30 __nvgpu_set_enabled(g, NVGPU_SEC_PRIVSECURITY, false);
31 __nvgpu_set_enabled(g, NVGPU_SEC_SECUREGPCCS, false);
32 } else {
33 __nvgpu_set_enabled(g, NVGPU_SEC_PRIVSECURITY, true);
34 __nvgpu_set_enabled(g, NVGPU_SEC_SECUREGPCCS, true);
35 }
36
37 return 0;
38}
diff --git a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fuse_gp10b.h b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fuse_gp10b.h
new file mode 100644
index 00000000..12334f23
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fuse_gp10b.h
@@ -0,0 +1,30 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _VGPU_GP10B_FUSE
24#define _VGPU_GP10B_FUSE
25
26struct gk20a;
27
28int vgpu_gp10b_fuse_check_priv_security(struct gk20a *g);
29
30#endif
diff --git a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.c b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.c
new file mode 100644
index 00000000..ab35dc67
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.c
@@ -0,0 +1,308 @@
1/*
2 * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/kmem.h>
24#include <nvgpu/dma.h>
25#include <nvgpu/bug.h>
26#include <nvgpu/vgpu/vgpu.h>
27
28#include "vgpu/gm20b/vgpu_gr_gm20b.h"
29
30#include "gp10b/gr_gp10b.h"
31#include "vgpu_gr_gp10b.h"
32
33#include <nvgpu/hw/gp10b/hw_gr_gp10b.h>
34
35int vgpu_gr_gp10b_alloc_gr_ctx(struct gk20a *g,
36 struct nvgpu_gr_ctx *gr_ctx,
37 struct vm_gk20a *vm,
38 u32 class,
39 u32 flags)
40{
41 u32 graphics_preempt_mode = 0;
42 u32 compute_preempt_mode = 0;
43 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
44 int err;
45
46 gk20a_dbg_fn("");
47
48 err = vgpu_gr_alloc_gr_ctx(g, gr_ctx, vm, class, flags);
49 if (err)
50 return err;
51
52 if (flags & NVGPU_OBJ_CTX_FLAGS_SUPPORT_GFXP)
53 graphics_preempt_mode = NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP;
54 if (flags & NVGPU_OBJ_CTX_FLAGS_SUPPORT_CILP)
55 compute_preempt_mode = NVGPU_PREEMPTION_MODE_COMPUTE_CILP;
56
57 if (priv->constants.force_preempt_mode && !graphics_preempt_mode &&
58 !compute_preempt_mode) {
59 graphics_preempt_mode = g->ops.gr.is_valid_gfx_class(g, class) ?
60 NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP : 0;
61 compute_preempt_mode =
62 g->ops.gr.is_valid_compute_class(g, class) ?
63 NVGPU_PREEMPTION_MODE_COMPUTE_CTA : 0;
64 }
65
66 if (graphics_preempt_mode || compute_preempt_mode) {
67 if (g->ops.gr.set_ctxsw_preemption_mode) {
68 err = g->ops.gr.set_ctxsw_preemption_mode(g, gr_ctx, vm,
69 class, graphics_preempt_mode, compute_preempt_mode);
70 if (err) {
71 nvgpu_err(g,
72 "set_ctxsw_preemption_mode failed");
73 goto fail;
74 }
75 } else {
76 err = -ENOSYS;
77 goto fail;
78 }
79 }
80
81 gk20a_dbg_fn("done");
82 return err;
83
84fail:
85 vgpu_gr_free_gr_ctx(g, vm, gr_ctx);
86 return err;
87}
88
89int vgpu_gr_gp10b_set_ctxsw_preemption_mode(struct gk20a *g,
90 struct nvgpu_gr_ctx *gr_ctx,
91 struct vm_gk20a *vm, u32 class,
92 u32 graphics_preempt_mode,
93 u32 compute_preempt_mode)
94{
95 struct tegra_vgpu_cmd_msg msg = {};
96 struct tegra_vgpu_gr_bind_ctxsw_buffers_params *p =
97 &msg.params.gr_bind_ctxsw_buffers;
98 int err = 0;
99
100 if (g->ops.gr.is_valid_gfx_class(g, class) &&
101 g->gr.ctx_vars.force_preemption_gfxp)
102 graphics_preempt_mode = NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP;
103
104 if (g->ops.gr.is_valid_compute_class(g, class) &&
105 g->gr.ctx_vars.force_preemption_cilp)
106 compute_preempt_mode = NVGPU_PREEMPTION_MODE_COMPUTE_CILP;
107
108 /* check for invalid combinations */
109 if ((graphics_preempt_mode == 0) && (compute_preempt_mode == 0))
110 return -EINVAL;
111
112 if ((graphics_preempt_mode == NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP) &&
113 (compute_preempt_mode == NVGPU_PREEMPTION_MODE_COMPUTE_CILP))
114 return -EINVAL;
115
116 /* set preemption modes */
117 switch (graphics_preempt_mode) {
118 case NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP:
119 {
120 u32 spill_size =
121 gr_gpc0_swdx_rm_spill_buffer_size_256b_default_v() *
122 gr_gpc0_swdx_rm_spill_buffer_size_256b_byte_granularity_v();
123 u32 pagepool_size = g->ops.gr.pagepool_default_size(g) *
124 gr_scc_pagepool_total_pages_byte_granularity_v();
125 u32 betacb_size = g->gr.attrib_cb_default_size +
126 (gr_gpc0_ppc0_cbm_beta_cb_size_v_gfxp_v() -
127 gr_gpc0_ppc0_cbm_beta_cb_size_v_default_v());
128 u32 attrib_cb_size = (betacb_size + g->gr.alpha_cb_size) *
129 gr_gpc0_ppc0_cbm_beta_cb_size_v_granularity_v() *
130 g->gr.max_tpc_count;
131 struct nvgpu_mem *desc;
132
133 attrib_cb_size = ALIGN(attrib_cb_size, 128);
134
135 gk20a_dbg_info("gfxp context preempt size=%d",
136 g->gr.ctx_vars.preempt_image_size);
137 gk20a_dbg_info("gfxp context spill size=%d", spill_size);
138 gk20a_dbg_info("gfxp context pagepool size=%d", pagepool_size);
139 gk20a_dbg_info("gfxp context attrib cb size=%d",
140 attrib_cb_size);
141
142 err = gr_gp10b_alloc_buffer(vm,
143 g->gr.ctx_vars.preempt_image_size,
144 &gr_ctx->preempt_ctxsw_buffer);
145 if (err) {
146 err = -ENOMEM;
147 goto fail;
148 }
149 desc = &gr_ctx->preempt_ctxsw_buffer;
150 p->gpu_va[TEGRA_VGPU_GR_BIND_CTXSW_BUFFER_MAIN] = desc->gpu_va;
151 p->size[TEGRA_VGPU_GR_BIND_CTXSW_BUFFER_MAIN] = desc->size;
152
153 err = gr_gp10b_alloc_buffer(vm,
154 spill_size,
155 &gr_ctx->spill_ctxsw_buffer);
156 if (err) {
157 err = -ENOMEM;
158 goto fail;
159 }
160 desc = &gr_ctx->spill_ctxsw_buffer;
161 p->gpu_va[TEGRA_VGPU_GR_BIND_CTXSW_BUFFER_SPILL] = desc->gpu_va;
162 p->size[TEGRA_VGPU_GR_BIND_CTXSW_BUFFER_SPILL] = desc->size;
163
164 err = gr_gp10b_alloc_buffer(vm,
165 pagepool_size,
166 &gr_ctx->pagepool_ctxsw_buffer);
167 if (err) {
168 err = -ENOMEM;
169 goto fail;
170 }
171 desc = &gr_ctx->pagepool_ctxsw_buffer;
172 p->gpu_va[TEGRA_VGPU_GR_BIND_CTXSW_BUFFER_PAGEPOOL] =
173 desc->gpu_va;
174 p->size[TEGRA_VGPU_GR_BIND_CTXSW_BUFFER_PAGEPOOL] = desc->size;
175
176 err = gr_gp10b_alloc_buffer(vm,
177 attrib_cb_size,
178 &gr_ctx->betacb_ctxsw_buffer);
179 if (err) {
180 err = -ENOMEM;
181 goto fail;
182 }
183 desc = &gr_ctx->betacb_ctxsw_buffer;
184 p->gpu_va[TEGRA_VGPU_GR_BIND_CTXSW_BUFFER_BETACB] =
185 desc->gpu_va;
186 p->size[TEGRA_VGPU_GR_BIND_CTXSW_BUFFER_BETACB] = desc->size;
187
188 gr_ctx->graphics_preempt_mode = NVGPU_PREEMPTION_MODE_GRAPHICS_GFXP;
189 p->mode = TEGRA_VGPU_GR_CTXSW_PREEMPTION_MODE_GFX_GFXP;
190 break;
191 }
192 case NVGPU_PREEMPTION_MODE_GRAPHICS_WFI:
193 gr_ctx->graphics_preempt_mode = graphics_preempt_mode;
194 break;
195
196 default:
197 break;
198 }
199
200 if (g->ops.gr.is_valid_compute_class(g, class)) {
201 switch (compute_preempt_mode) {
202 case NVGPU_PREEMPTION_MODE_COMPUTE_WFI:
203 gr_ctx->compute_preempt_mode =
204 NVGPU_PREEMPTION_MODE_COMPUTE_WFI;
205 p->mode = TEGRA_VGPU_GR_CTXSW_PREEMPTION_MODE_WFI;
206 break;
207 case NVGPU_PREEMPTION_MODE_COMPUTE_CTA:
208 gr_ctx->compute_preempt_mode =
209 NVGPU_PREEMPTION_MODE_COMPUTE_CTA;
210 p->mode =
211 TEGRA_VGPU_GR_CTXSW_PREEMPTION_MODE_COMPUTE_CTA;
212 break;
213 case NVGPU_PREEMPTION_MODE_COMPUTE_CILP:
214 gr_ctx->compute_preempt_mode =
215 NVGPU_PREEMPTION_MODE_COMPUTE_CILP;
216 p->mode =
217 TEGRA_VGPU_GR_CTXSW_PREEMPTION_MODE_COMPUTE_CILP;
218 break;
219 default:
220 break;
221 }
222 }
223
224 if (gr_ctx->graphics_preempt_mode || gr_ctx->compute_preempt_mode) {
225 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_BIND_GR_CTXSW_BUFFERS;
226 msg.handle = vgpu_get_handle(g);
227 p->gr_ctx_handle = gr_ctx->virt_ctx;
228 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
229 if (err || msg.ret) {
230 err = -ENOMEM;
231 goto fail;
232 }
233 }
234
235 return err;
236
237fail:
238 nvgpu_err(g, "%s failed %d", __func__, err);
239 return err;
240}
241
242int vgpu_gr_gp10b_set_preemption_mode(struct channel_gk20a *ch,
243 u32 graphics_preempt_mode,
244 u32 compute_preempt_mode)
245{
246 struct nvgpu_gr_ctx *gr_ctx;
247 struct gk20a *g = ch->g;
248 struct tsg_gk20a *tsg;
249 struct vm_gk20a *vm;
250 u32 class;
251 int err;
252
253 class = ch->obj_class;
254 if (!class)
255 return -EINVAL;
256
257 tsg = tsg_gk20a_from_ch(ch);
258 if (!tsg)
259 return -EINVAL;
260
261 vm = tsg->vm;
262 gr_ctx = &tsg->gr_ctx;
263
264 /* skip setting anything if both modes are already set */
265 if (graphics_preempt_mode &&
266 (graphics_preempt_mode == gr_ctx->graphics_preempt_mode))
267 graphics_preempt_mode = 0;
268
269 if (compute_preempt_mode &&
270 (compute_preempt_mode == gr_ctx->compute_preempt_mode))
271 compute_preempt_mode = 0;
272
273 if (graphics_preempt_mode == 0 && compute_preempt_mode == 0)
274 return 0;
275
276 if (g->ops.gr.set_ctxsw_preemption_mode) {
277 err = g->ops.gr.set_ctxsw_preemption_mode(g, gr_ctx, vm, class,
278 graphics_preempt_mode,
279 compute_preempt_mode);
280 if (err) {
281 nvgpu_err(g, "set_ctxsw_preemption_mode failed");
282 return err;
283 }
284 } else {
285 err = -ENOSYS;
286 }
287
288 return err;
289}
290
291int vgpu_gr_gp10b_init_ctx_state(struct gk20a *g)
292{
293 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
294 int err;
295
296 gk20a_dbg_fn("");
297
298 err = vgpu_gr_init_ctx_state(g);
299 if (err)
300 return err;
301
302 g->gr.ctx_vars.preempt_image_size =
303 priv->constants.preempt_ctx_size;
304 if (!g->gr.ctx_vars.preempt_image_size)
305 return -EINVAL;
306
307 return 0;
308}
diff --git a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.h b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.h
new file mode 100644
index 00000000..0dc53982
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.h
@@ -0,0 +1,43 @@
1/*
2 * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef __VGPU_GR_GP10B_H__
24#define __VGPU_GR_GP10B_H__
25
26#include "gk20a/gk20a.h"
27
28int vgpu_gr_gp10b_alloc_gr_ctx(struct gk20a *g,
29 struct nvgpu_gr_ctx *gr_ctx,
30 struct vm_gk20a *vm,
31 u32 class,
32 u32 flags);
33int vgpu_gr_gp10b_set_ctxsw_preemption_mode(struct gk20a *g,
34 struct nvgpu_gr_ctx *gr_ctx,
35 struct vm_gk20a *vm, u32 class,
36 u32 graphics_preempt_mode,
37 u32 compute_preempt_mode);
38int vgpu_gr_gp10b_set_preemption_mode(struct channel_gk20a *ch,
39 u32 graphics_preempt_mode,
40 u32 compute_preempt_mode);
41int vgpu_gr_gp10b_init_ctx_state(struct gk20a *g);
42
43#endif
diff --git a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c
new file mode 100644
index 00000000..f1ced1c8
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_hal_gp10b.c
@@ -0,0 +1,613 @@
1/*
2 * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include "vgpu/fifo_vgpu.h"
24#include "vgpu/gr_vgpu.h"
25#include "vgpu/ltc_vgpu.h"
26#include "vgpu/mm_vgpu.h"
27#include "vgpu/dbg_vgpu.h"
28#include "vgpu/fecs_trace_vgpu.h"
29#include "vgpu/css_vgpu.h"
30#include "gp10b/gp10b.h"
31#include "gp10b/hal_gp10b.h"
32#include "vgpu/gm20b/vgpu_gr_gm20b.h"
33#include "vgpu_gr_gp10b.h"
34#include "vgpu_mm_gp10b.h"
35#include "vgpu_fuse_gp10b.h"
36
37#include "gk20a/bus_gk20a.h"
38#include "gk20a/pramin_gk20a.h"
39#include "gk20a/flcn_gk20a.h"
40#include "gk20a/mc_gk20a.h"
41#include "gk20a/fb_gk20a.h"
42
43#include "gp10b/mc_gp10b.h"
44#include "gp10b/ltc_gp10b.h"
45#include "gp10b/mm_gp10b.h"
46#include "gp10b/ce_gp10b.h"
47#include "gp10b/fb_gp10b.h"
48#include "gp10b/pmu_gp10b.h"
49#include "gp10b/gr_gp10b.h"
50#include "gp10b/gr_ctx_gp10b.h"
51#include "gp10b/fifo_gp10b.h"
52#include "gp10b/gp10b_gating_reglist.h"
53#include "gp10b/regops_gp10b.h"
54#include "gp10b/therm_gp10b.h"
55#include "gp10b/priv_ring_gp10b.h"
56
57#include "gm20b/ltc_gm20b.h"
58#include "gm20b/gr_gm20b.h"
59#include "gm20b/fifo_gm20b.h"
60#include "gm20b/acr_gm20b.h"
61#include "gm20b/pmu_gm20b.h"
62#include "gm20b/fb_gm20b.h"
63#include "gm20b/mm_gm20b.h"
64
65#include <nvgpu/enabled.h>
66#include <nvgpu/vgpu/vgpu.h>
67
68#include <nvgpu/hw/gp10b/hw_fuse_gp10b.h>
69#include <nvgpu/hw/gp10b/hw_fifo_gp10b.h>
70#include <nvgpu/hw/gp10b/hw_ram_gp10b.h>
71#include <nvgpu/hw/gp10b/hw_top_gp10b.h>
72#include <nvgpu/hw/gp10b/hw_pram_gp10b.h>
73#include <nvgpu/hw/gp10b/hw_pwr_gp10b.h>
74
75static const struct gpu_ops vgpu_gp10b_ops = {
76 .ltc = {
77 .determine_L2_size_bytes = vgpu_determine_L2_size_bytes,
78 .set_zbc_color_entry = gm20b_ltc_set_zbc_color_entry,
79 .set_zbc_depth_entry = gm20b_ltc_set_zbc_depth_entry,
80 .init_cbc = gm20b_ltc_init_cbc,
81 .init_fs_state = vgpu_ltc_init_fs_state,
82 .init_comptags = vgpu_ltc_init_comptags,
83 .cbc_ctrl = NULL,
84 .isr = gp10b_ltc_isr,
85 .cbc_fix_config = gm20b_ltc_cbc_fix_config,
86 .flush = gm20b_flush_ltc,
87 .set_enabled = gp10b_ltc_set_enabled,
88 },
89 .ce2 = {
90 .isr_stall = gp10b_ce_isr,
91 .isr_nonstall = gp10b_ce_nonstall_isr,
92 .get_num_pce = vgpu_ce_get_num_pce,
93 },
94 .gr = {
95 .get_patch_slots = gr_gk20a_get_patch_slots,
96 .init_gpc_mmu = gr_gm20b_init_gpc_mmu,
97 .bundle_cb_defaults = gr_gm20b_bundle_cb_defaults,
98 .cb_size_default = gr_gp10b_cb_size_default,
99 .calc_global_ctx_buffer_size =
100 gr_gp10b_calc_global_ctx_buffer_size,
101 .commit_global_attrib_cb = gr_gp10b_commit_global_attrib_cb,
102 .commit_global_bundle_cb = gr_gp10b_commit_global_bundle_cb,
103 .commit_global_cb_manager = gr_gp10b_commit_global_cb_manager,
104 .commit_global_pagepool = gr_gp10b_commit_global_pagepool,
105 .handle_sw_method = gr_gp10b_handle_sw_method,
106 .set_alpha_circular_buffer_size =
107 gr_gp10b_set_alpha_circular_buffer_size,
108 .set_circular_buffer_size = gr_gp10b_set_circular_buffer_size,
109 .enable_hww_exceptions = gr_gk20a_enable_hww_exceptions,
110 .is_valid_class = gr_gp10b_is_valid_class,
111 .is_valid_gfx_class = gr_gp10b_is_valid_gfx_class,
112 .is_valid_compute_class = gr_gp10b_is_valid_compute_class,
113 .get_sm_dsm_perf_regs = gr_gm20b_get_sm_dsm_perf_regs,
114 .get_sm_dsm_perf_ctrl_regs = gr_gm20b_get_sm_dsm_perf_ctrl_regs,
115 .init_fs_state = vgpu_gr_init_fs_state,
116 .set_hww_esr_report_mask = gr_gm20b_set_hww_esr_report_mask,
117 .falcon_load_ucode = gr_gm20b_load_ctxsw_ucode_segments,
118 .load_ctxsw_ucode = gr_gk20a_load_ctxsw_ucode,
119 .set_gpc_tpc_mask = gr_gp10b_set_gpc_tpc_mask,
120 .get_gpc_tpc_mask = vgpu_gr_get_gpc_tpc_mask,
121 .alloc_obj_ctx = vgpu_gr_alloc_obj_ctx,
122 .bind_ctxsw_zcull = vgpu_gr_bind_ctxsw_zcull,
123 .get_zcull_info = vgpu_gr_get_zcull_info,
124 .is_tpc_addr = gr_gm20b_is_tpc_addr,
125 .get_tpc_num = gr_gm20b_get_tpc_num,
126 .detect_sm_arch = vgpu_gr_detect_sm_arch,
127 .add_zbc_color = gr_gp10b_add_zbc_color,
128 .add_zbc_depth = gr_gp10b_add_zbc_depth,
129 .zbc_set_table = vgpu_gr_add_zbc,
130 .zbc_query_table = vgpu_gr_query_zbc,
131 .pmu_save_zbc = gk20a_pmu_save_zbc,
132 .add_zbc = gr_gk20a_add_zbc,
133 .pagepool_default_size = gr_gp10b_pagepool_default_size,
134 .init_ctx_state = vgpu_gr_gp10b_init_ctx_state,
135 .alloc_gr_ctx = vgpu_gr_gp10b_alloc_gr_ctx,
136 .free_gr_ctx = vgpu_gr_free_gr_ctx,
137 .update_ctxsw_preemption_mode =
138 gr_gp10b_update_ctxsw_preemption_mode,
139 .dump_gr_regs = NULL,
140 .update_pc_sampling = gr_gm20b_update_pc_sampling,
141 .get_fbp_en_mask = vgpu_gr_get_fbp_en_mask,
142 .get_max_ltc_per_fbp = vgpu_gr_get_max_ltc_per_fbp,
143 .get_max_lts_per_ltc = vgpu_gr_get_max_lts_per_ltc,
144 .get_rop_l2_en_mask = vgpu_gr_rop_l2_en_mask,
145 .get_max_fbps_count = vgpu_gr_get_max_fbps_count,
146 .init_sm_dsm_reg_info = gr_gm20b_init_sm_dsm_reg_info,
147 .wait_empty = gr_gp10b_wait_empty,
148 .init_cyclestats = vgpu_gr_gm20b_init_cyclestats,
149 .set_sm_debug_mode = vgpu_gr_set_sm_debug_mode,
150 .enable_cde_in_fecs = gr_gm20b_enable_cde_in_fecs,
151 .bpt_reg_info = gr_gm20b_bpt_reg_info,
152 .get_access_map = gr_gp10b_get_access_map,
153 .handle_fecs_error = gr_gp10b_handle_fecs_error,
154 .handle_sm_exception = gr_gp10b_handle_sm_exception,
155 .handle_tex_exception = gr_gp10b_handle_tex_exception,
156 .enable_gpc_exceptions = gk20a_gr_enable_gpc_exceptions,
157 .enable_exceptions = gk20a_gr_enable_exceptions,
158 .get_lrf_tex_ltc_dram_override = get_ecc_override_val,
159 .update_smpc_ctxsw_mode = vgpu_gr_update_smpc_ctxsw_mode,
160 .update_hwpm_ctxsw_mode = vgpu_gr_update_hwpm_ctxsw_mode,
161 .record_sm_error_state = gm20b_gr_record_sm_error_state,
162 .update_sm_error_state = gm20b_gr_update_sm_error_state,
163 .clear_sm_error_state = vgpu_gr_clear_sm_error_state,
164 .suspend_contexts = vgpu_gr_suspend_contexts,
165 .resume_contexts = vgpu_gr_resume_contexts,
166 .get_preemption_mode_flags = gr_gp10b_get_preemption_mode_flags,
167 .init_sm_id_table = vgpu_gr_init_sm_id_table,
168 .load_smid_config = gr_gp10b_load_smid_config,
169 .program_sm_id_numbering = gr_gm20b_program_sm_id_numbering,
170 .is_ltcs_ltss_addr = gr_gm20b_is_ltcs_ltss_addr,
171 .is_ltcn_ltss_addr = gr_gm20b_is_ltcn_ltss_addr,
172 .split_lts_broadcast_addr = gr_gm20b_split_lts_broadcast_addr,
173 .split_ltc_broadcast_addr = gr_gm20b_split_ltc_broadcast_addr,
174 .setup_rop_mapping = gr_gk20a_setup_rop_mapping,
175 .program_zcull_mapping = gr_gk20a_program_zcull_mapping,
176 .commit_global_timeslice = gr_gk20a_commit_global_timeslice,
177 .commit_inst = vgpu_gr_commit_inst,
178 .write_zcull_ptr = gr_gk20a_write_zcull_ptr,
179 .write_pm_ptr = gr_gk20a_write_pm_ptr,
180 .init_elcg_mode = gr_gk20a_init_elcg_mode,
181 .load_tpc_mask = gr_gm20b_load_tpc_mask,
182 .inval_icache = gr_gk20a_inval_icache,
183 .trigger_suspend = gr_gk20a_trigger_suspend,
184 .wait_for_pause = gr_gk20a_wait_for_pause,
185 .resume_from_pause = gr_gk20a_resume_from_pause,
186 .clear_sm_errors = gr_gk20a_clear_sm_errors,
187 .tpc_enabled_exceptions = gr_gk20a_tpc_enabled_exceptions,
188 .get_esr_sm_sel = gk20a_gr_get_esr_sm_sel,
189 .sm_debugger_attached = gk20a_gr_sm_debugger_attached,
190 .suspend_single_sm = gk20a_gr_suspend_single_sm,
191 .suspend_all_sms = gk20a_gr_suspend_all_sms,
192 .resume_single_sm = gk20a_gr_resume_single_sm,
193 .resume_all_sms = gk20a_gr_resume_all_sms,
194 .get_sm_hww_warp_esr = gp10b_gr_get_sm_hww_warp_esr,
195 .get_sm_hww_global_esr = gk20a_gr_get_sm_hww_global_esr,
196 .get_sm_no_lock_down_hww_global_esr_mask =
197 gk20a_gr_get_sm_no_lock_down_hww_global_esr_mask,
198 .lock_down_sm = gk20a_gr_lock_down_sm,
199 .wait_for_sm_lock_down = gk20a_gr_wait_for_sm_lock_down,
200 .clear_sm_hww = gm20b_gr_clear_sm_hww,
201 .init_ovr_sm_dsm_perf = gk20a_gr_init_ovr_sm_dsm_perf,
202 .get_ovr_perf_regs = gk20a_gr_get_ovr_perf_regs,
203 .disable_rd_coalesce = gm20a_gr_disable_rd_coalesce,
204 .set_boosted_ctx = NULL,
205 .set_preemption_mode = vgpu_gr_gp10b_set_preemption_mode,
206 .set_czf_bypass = gr_gp10b_set_czf_bypass,
207 .init_czf_bypass = gr_gp10b_init_czf_bypass,
208 .pre_process_sm_exception = gr_gp10b_pre_process_sm_exception,
209 .set_preemption_buffer_va = gr_gp10b_set_preemption_buffer_va,
210 .init_preemption_state = gr_gp10b_init_preemption_state,
211 .update_boosted_ctx = NULL,
212 .set_bes_crop_debug3 = gr_gp10b_set_bes_crop_debug3,
213 .set_bes_crop_debug4 = gr_gp10b_set_bes_crop_debug4,
214 .create_gr_sysfs = gr_gp10b_create_sysfs,
215 .set_ctxsw_preemption_mode =
216 vgpu_gr_gp10b_set_ctxsw_preemption_mode,
217 .init_ctxsw_hdr_data = gr_gp10b_init_ctxsw_hdr_data,
218 .init_gfxp_wfi_timeout_count =
219 gr_gp10b_init_gfxp_wfi_timeout_count,
220 .get_max_gfxp_wfi_timeout_count =
221 gr_gp10b_get_max_gfxp_wfi_timeout_count,
222 },
223 .fb = {
224 .reset = fb_gk20a_reset,
225 .init_hw = gk20a_fb_init_hw,
226 .init_fs_state = fb_gm20b_init_fs_state,
227 .set_mmu_page_size = gm20b_fb_set_mmu_page_size,
228 .set_use_full_comp_tag_line =
229 gm20b_fb_set_use_full_comp_tag_line,
230 .compression_page_size = gp10b_fb_compression_page_size,
231 .compressible_page_size = gp10b_fb_compressible_page_size,
232 .compression_align_mask = gm20b_fb_compression_align_mask,
233 .vpr_info_fetch = gm20b_fb_vpr_info_fetch,
234 .dump_vpr_wpr_info = gm20b_fb_dump_vpr_wpr_info,
235 .read_wpr_info = gm20b_fb_read_wpr_info,
236 .is_debug_mode_enabled = NULL,
237 .set_debug_mode = vgpu_mm_mmu_set_debug_mode,
238 .tlb_invalidate = vgpu_mm_tlb_invalidate,
239 },
240 .clock_gating = {
241 .slcg_bus_load_gating_prod =
242 gp10b_slcg_bus_load_gating_prod,
243 .slcg_ce2_load_gating_prod =
244 gp10b_slcg_ce2_load_gating_prod,
245 .slcg_chiplet_load_gating_prod =
246 gp10b_slcg_chiplet_load_gating_prod,
247 .slcg_ctxsw_firmware_load_gating_prod =
248 gp10b_slcg_ctxsw_firmware_load_gating_prod,
249 .slcg_fb_load_gating_prod =
250 gp10b_slcg_fb_load_gating_prod,
251 .slcg_fifo_load_gating_prod =
252 gp10b_slcg_fifo_load_gating_prod,
253 .slcg_gr_load_gating_prod =
254 gr_gp10b_slcg_gr_load_gating_prod,
255 .slcg_ltc_load_gating_prod =
256 ltc_gp10b_slcg_ltc_load_gating_prod,
257 .slcg_perf_load_gating_prod =
258 gp10b_slcg_perf_load_gating_prod,
259 .slcg_priring_load_gating_prod =
260 gp10b_slcg_priring_load_gating_prod,
261 .slcg_pmu_load_gating_prod =
262 gp10b_slcg_pmu_load_gating_prod,
263 .slcg_therm_load_gating_prod =
264 gp10b_slcg_therm_load_gating_prod,
265 .slcg_xbar_load_gating_prod =
266 gp10b_slcg_xbar_load_gating_prod,
267 .blcg_bus_load_gating_prod =
268 gp10b_blcg_bus_load_gating_prod,
269 .blcg_ce_load_gating_prod =
270 gp10b_blcg_ce_load_gating_prod,
271 .blcg_ctxsw_firmware_load_gating_prod =
272 gp10b_blcg_ctxsw_firmware_load_gating_prod,
273 .blcg_fb_load_gating_prod =
274 gp10b_blcg_fb_load_gating_prod,
275 .blcg_fifo_load_gating_prod =
276 gp10b_blcg_fifo_load_gating_prod,
277 .blcg_gr_load_gating_prod =
278 gp10b_blcg_gr_load_gating_prod,
279 .blcg_ltc_load_gating_prod =
280 gp10b_blcg_ltc_load_gating_prod,
281 .blcg_pwr_csb_load_gating_prod =
282 gp10b_blcg_pwr_csb_load_gating_prod,
283 .blcg_pmu_load_gating_prod =
284 gp10b_blcg_pmu_load_gating_prod,
285 .blcg_xbar_load_gating_prod =
286 gp10b_blcg_xbar_load_gating_prod,
287 .pg_gr_load_gating_prod =
288 gr_gp10b_pg_gr_load_gating_prod,
289 },
290 .fifo = {
291 .init_fifo_setup_hw = vgpu_init_fifo_setup_hw,
292 .bind_channel = vgpu_channel_bind,
293 .unbind_channel = vgpu_channel_unbind,
294 .disable_channel = vgpu_channel_disable,
295 .enable_channel = vgpu_channel_enable,
296 .alloc_inst = vgpu_channel_alloc_inst,
297 .free_inst = vgpu_channel_free_inst,
298 .setup_ramfc = vgpu_channel_setup_ramfc,
299 .default_timeslice_us = vgpu_fifo_default_timeslice_us,
300 .setup_userd = gk20a_fifo_setup_userd,
301 .userd_gp_get = gk20a_fifo_userd_gp_get,
302 .userd_gp_put = gk20a_fifo_userd_gp_put,
303 .userd_pb_get = gk20a_fifo_userd_pb_get,
304 .pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val,
305 .preempt_channel = vgpu_fifo_preempt_channel,
306 .preempt_tsg = vgpu_fifo_preempt_tsg,
307 .enable_tsg = vgpu_enable_tsg,
308 .disable_tsg = gk20a_disable_tsg,
309 .tsg_verify_channel_status = NULL,
310 .tsg_verify_status_ctx_reload = NULL,
311 .reschedule_runlist = NULL,
312 .update_runlist = vgpu_fifo_update_runlist,
313 .trigger_mmu_fault = gm20b_fifo_trigger_mmu_fault,
314 .get_mmu_fault_info = gp10b_fifo_get_mmu_fault_info,
315 .wait_engine_idle = vgpu_fifo_wait_engine_idle,
316 .get_num_fifos = gm20b_fifo_get_num_fifos,
317 .get_pbdma_signature = gp10b_fifo_get_pbdma_signature,
318 .set_runlist_interleave = vgpu_fifo_set_runlist_interleave,
319 .tsg_set_timeslice = vgpu_tsg_set_timeslice,
320 .tsg_open = vgpu_tsg_open,
321 .force_reset_ch = vgpu_fifo_force_reset_ch,
322 .engine_enum_from_type = gp10b_fifo_engine_enum_from_type,
323 .device_info_data_parse = gp10b_device_info_data_parse,
324 .eng_runlist_base_size = fifo_eng_runlist_base__size_1_v,
325 .init_engine_info = vgpu_fifo_init_engine_info,
326 .runlist_entry_size = ram_rl_entry_size_v,
327 .get_tsg_runlist_entry = gk20a_get_tsg_runlist_entry,
328 .get_ch_runlist_entry = gk20a_get_ch_runlist_entry,
329 .is_fault_engine_subid_gpc = gk20a_is_fault_engine_subid_gpc,
330 .dump_pbdma_status = gk20a_dump_pbdma_status,
331 .dump_eng_status = gk20a_dump_eng_status,
332 .dump_channel_status_ramfc = gk20a_dump_channel_status_ramfc,
333 .intr_0_error_mask = gk20a_fifo_intr_0_error_mask,
334 .is_preempt_pending = gk20a_fifo_is_preempt_pending,
335 .init_pbdma_intr_descs = gp10b_fifo_init_pbdma_intr_descs,
336 .reset_enable_hw = gk20a_init_fifo_reset_enable_hw,
337 .teardown_ch_tsg = gk20a_fifo_teardown_ch_tsg,
338 .handle_sched_error = gk20a_fifo_handle_sched_error,
339 .handle_pbdma_intr_0 = gk20a_fifo_handle_pbdma_intr_0,
340 .handle_pbdma_intr_1 = gk20a_fifo_handle_pbdma_intr_1,
341 .tsg_bind_channel = vgpu_tsg_bind_channel,
342 .tsg_unbind_channel = vgpu_tsg_unbind_channel,
343#ifdef CONFIG_TEGRA_GK20A_NVHOST
344 .alloc_syncpt_buf = gk20a_fifo_alloc_syncpt_buf,
345 .free_syncpt_buf = gk20a_fifo_free_syncpt_buf,
346 .add_syncpt_wait_cmd = gk20a_fifo_add_syncpt_wait_cmd,
347 .get_syncpt_wait_cmd_size = gk20a_fifo_get_syncpt_wait_cmd_size,
348 .add_syncpt_incr_cmd = gk20a_fifo_add_syncpt_incr_cmd,
349 .get_syncpt_incr_cmd_size = gk20a_fifo_get_syncpt_incr_cmd_size,
350 .get_sync_ro_map = NULL,
351#endif
352 .resetup_ramfc = NULL,
353 .device_info_fault_id = top_device_info_data_fault_id_enum_v,
354 },
355 .gr_ctx = {
356 .get_netlist_name = gr_gp10b_get_netlist_name,
357 .is_fw_defined = gr_gp10b_is_firmware_defined,
358 },
359#ifdef CONFIG_GK20A_CTXSW_TRACE
360 .fecs_trace = {
361 .alloc_user_buffer = vgpu_alloc_user_buffer,
362 .free_user_buffer = vgpu_free_user_buffer,
363 .mmap_user_buffer = vgpu_mmap_user_buffer,
364 .init = vgpu_fecs_trace_init,
365 .deinit = vgpu_fecs_trace_deinit,
366 .enable = vgpu_fecs_trace_enable,
367 .disable = vgpu_fecs_trace_disable,
368 .is_enabled = vgpu_fecs_trace_is_enabled,
369 .reset = NULL,
370 .flush = NULL,
371 .poll = vgpu_fecs_trace_poll,
372 .bind_channel = NULL,
373 .unbind_channel = NULL,
374 .max_entries = vgpu_fecs_trace_max_entries,
375 .set_filter = vgpu_fecs_trace_set_filter,
376 },
377#endif /* CONFIG_GK20A_CTXSW_TRACE */
378 .mm = {
379 /* FIXME: add support for sparse mappings */
380 .support_sparse = NULL,
381 .gmmu_map = vgpu_gp10b_locked_gmmu_map,
382 .gmmu_unmap = vgpu_locked_gmmu_unmap,
383 .vm_bind_channel = vgpu_vm_bind_channel,
384 .fb_flush = vgpu_mm_fb_flush,
385 .l2_invalidate = vgpu_mm_l2_invalidate,
386 .l2_flush = vgpu_mm_l2_flush,
387 .cbc_clean = gk20a_mm_cbc_clean,
388 .set_big_page_size = gm20b_mm_set_big_page_size,
389 .get_big_page_sizes = gm20b_mm_get_big_page_sizes,
390 .get_default_big_page_size = gp10b_mm_get_default_big_page_size,
391 .gpu_phys_addr = gm20b_gpu_phys_addr,
392 .get_iommu_bit = gk20a_mm_get_iommu_bit,
393 .get_mmu_levels = gp10b_mm_get_mmu_levels,
394 .init_pdb = gp10b_mm_init_pdb,
395 .init_mm_setup_hw = vgpu_gp10b_init_mm_setup_hw,
396 .is_bar1_supported = gm20b_mm_is_bar1_supported,
397 .init_inst_block = gk20a_init_inst_block,
398 .mmu_fault_pending = gk20a_fifo_mmu_fault_pending,
399 .init_bar2_vm = gp10b_init_bar2_vm,
400 .init_bar2_mm_hw_setup = gp10b_init_bar2_mm_hw_setup,
401 .remove_bar2_vm = gp10b_remove_bar2_vm,
402 .get_kind_invalid = gm20b_get_kind_invalid,
403 .get_kind_pitch = gm20b_get_kind_pitch,
404 },
405 .pramin = {
406 .enter = gk20a_pramin_enter,
407 .exit = gk20a_pramin_exit,
408 .data032_r = pram_data032_r,
409 },
410 .therm = {
411 .init_therm_setup_hw = gp10b_init_therm_setup_hw,
412 .elcg_init_idle_filters = gp10b_elcg_init_idle_filters,
413 },
414 .pmu = {
415 .pmu_setup_elpg = gp10b_pmu_setup_elpg,
416 .pmu_get_queue_head = pwr_pmu_queue_head_r,
417 .pmu_get_queue_head_size = pwr_pmu_queue_head__size_1_v,
418 .pmu_get_queue_tail = pwr_pmu_queue_tail_r,
419 .pmu_get_queue_tail_size = pwr_pmu_queue_tail__size_1_v,
420 .pmu_queue_head = gk20a_pmu_queue_head,
421 .pmu_queue_tail = gk20a_pmu_queue_tail,
422 .pmu_msgq_tail = gk20a_pmu_msgq_tail,
423 .pmu_mutex_size = pwr_pmu_mutex__size_1_v,
424 .pmu_mutex_acquire = gk20a_pmu_mutex_acquire,
425 .pmu_mutex_release = gk20a_pmu_mutex_release,
426 .write_dmatrfbase = gp10b_write_dmatrfbase,
427 .pmu_elpg_statistics = gp10b_pmu_elpg_statistics,
428 .pmu_init_perfmon = nvgpu_pmu_init_perfmon,
429 .pmu_perfmon_start_sampling = nvgpu_pmu_perfmon_start_sampling,
430 .pmu_perfmon_stop_sampling = nvgpu_pmu_perfmon_stop_sampling,
431 .pmu_pg_init_param = gp10b_pg_gr_init,
432 .pmu_pg_supported_engines_list = gk20a_pmu_pg_engines_list,
433 .pmu_pg_engines_feature_list = gk20a_pmu_pg_feature_list,
434 .dump_secure_fuses = pmu_dump_security_fuses_gp10b,
435 .reset_engine = gk20a_pmu_engine_reset,
436 .is_engine_in_reset = gk20a_pmu_is_engine_in_reset,
437 },
438 .regops = {
439 .get_global_whitelist_ranges =
440 gp10b_get_global_whitelist_ranges,
441 .get_global_whitelist_ranges_count =
442 gp10b_get_global_whitelist_ranges_count,
443 .get_context_whitelist_ranges =
444 gp10b_get_context_whitelist_ranges,
445 .get_context_whitelist_ranges_count =
446 gp10b_get_context_whitelist_ranges_count,
447 .get_runcontrol_whitelist = gp10b_get_runcontrol_whitelist,
448 .get_runcontrol_whitelist_count =
449 gp10b_get_runcontrol_whitelist_count,
450 .get_runcontrol_whitelist_ranges =
451 gp10b_get_runcontrol_whitelist_ranges,
452 .get_runcontrol_whitelist_ranges_count =
453 gp10b_get_runcontrol_whitelist_ranges_count,
454 .get_qctl_whitelist = gp10b_get_qctl_whitelist,
455 .get_qctl_whitelist_count = gp10b_get_qctl_whitelist_count,
456 .get_qctl_whitelist_ranges = gp10b_get_qctl_whitelist_ranges,
457 .get_qctl_whitelist_ranges_count =
458 gp10b_get_qctl_whitelist_ranges_count,
459 .apply_smpc_war = gp10b_apply_smpc_war,
460 },
461 .mc = {
462 .intr_enable = mc_gp10b_intr_enable,
463 .intr_unit_config = mc_gp10b_intr_unit_config,
464 .isr_stall = mc_gp10b_isr_stall,
465 .intr_stall = mc_gp10b_intr_stall,
466 .intr_stall_pause = mc_gp10b_intr_stall_pause,
467 .intr_stall_resume = mc_gp10b_intr_stall_resume,
468 .intr_nonstall = mc_gp10b_intr_nonstall,
469 .intr_nonstall_pause = mc_gp10b_intr_nonstall_pause,
470 .intr_nonstall_resume = mc_gp10b_intr_nonstall_resume,
471 .enable = gk20a_mc_enable,
472 .disable = gk20a_mc_disable,
473 .reset = gk20a_mc_reset,
474 .boot_0 = gk20a_mc_boot_0,
475 .is_intr1_pending = mc_gp10b_is_intr1_pending,
476 },
477 .debug = {
478 .show_dump = NULL,
479 },
480 .dbg_session_ops = {
481 .exec_reg_ops = vgpu_exec_regops,
482 .dbg_set_powergate = vgpu_dbg_set_powergate,
483 .check_and_set_global_reservation =
484 vgpu_check_and_set_global_reservation,
485 .check_and_set_context_reservation =
486 vgpu_check_and_set_context_reservation,
487 .release_profiler_reservation =
488 vgpu_release_profiler_reservation,
489 .perfbuffer_enable = vgpu_perfbuffer_enable,
490 .perfbuffer_disable = vgpu_perfbuffer_disable,
491 },
492 .bus = {
493 .init_hw = gk20a_bus_init_hw,
494 .isr = gk20a_bus_isr,
495 .read_ptimer = vgpu_read_ptimer,
496 .get_timestamps_zipper = vgpu_get_timestamps_zipper,
497 .bar1_bind = gk20a_bus_bar1_bind,
498 },
499#if defined(CONFIG_GK20A_CYCLE_STATS)
500 .css = {
501 .enable_snapshot = vgpu_css_enable_snapshot_buffer,
502 .disable_snapshot = vgpu_css_release_snapshot_buffer,
503 .check_data_available = vgpu_css_flush_snapshots,
504 .detach_snapshot = vgpu_css_detach,
505 .set_handled_snapshots = NULL,
506 .allocate_perfmon_ids = NULL,
507 .release_perfmon_ids = NULL,
508 },
509#endif
510 .falcon = {
511 .falcon_hal_sw_init = gk20a_falcon_hal_sw_init,
512 },
513 .priv_ring = {
514 .isr = gp10b_priv_ring_isr,
515 },
516 .fuse = {
517 .check_priv_security = vgpu_gp10b_fuse_check_priv_security,
518 },
519 .chip_init_gpu_characteristics = vgpu_init_gpu_characteristics,
520 .get_litter_value = gp10b_get_litter_value,
521};
522
523int vgpu_gp10b_init_hal(struct gk20a *g)
524{
525 struct gpu_ops *gops = &g->ops;
526
527 gops->ltc = vgpu_gp10b_ops.ltc;
528 gops->ce2 = vgpu_gp10b_ops.ce2;
529 gops->gr = vgpu_gp10b_ops.gr;
530 gops->fb = vgpu_gp10b_ops.fb;
531 gops->clock_gating = vgpu_gp10b_ops.clock_gating;
532 gops->fifo = vgpu_gp10b_ops.fifo;
533 gops->gr_ctx = vgpu_gp10b_ops.gr_ctx;
534#ifdef CONFIG_GK20A_CTXSW_TRACE
535 gops->fecs_trace = vgpu_gp10b_ops.fecs_trace;
536#endif
537 gops->mm = vgpu_gp10b_ops.mm;
538 gops->pramin = vgpu_gp10b_ops.pramin;
539 gops->therm = vgpu_gp10b_ops.therm;
540 gops->pmu = vgpu_gp10b_ops.pmu;
541 gops->regops = vgpu_gp10b_ops.regops;
542 gops->mc = vgpu_gp10b_ops.mc;
543 gops->debug = vgpu_gp10b_ops.debug;
544 gops->dbg_session_ops = vgpu_gp10b_ops.dbg_session_ops;
545 gops->bus = vgpu_gp10b_ops.bus;
546#if defined(CONFIG_GK20A_CYCLE_STATS)
547 gops->css = vgpu_gp10b_ops.css;
548#endif
549 gops->falcon = vgpu_gp10b_ops.falcon;
550
551 gops->priv_ring = vgpu_gp10b_ops.priv_ring;
552
553 gops->fuse = vgpu_gp10b_ops.fuse;
554
555 /* Lone Functions */
556 gops->chip_init_gpu_characteristics =
557 vgpu_gp10b_ops.chip_init_gpu_characteristics;
558 gops->get_litter_value = vgpu_gp10b_ops.get_litter_value;
559
560 __nvgpu_set_enabled(g, NVGPU_GR_USE_DMA_FOR_FW_BOOTSTRAP, true);
561 __nvgpu_set_enabled(g, NVGPU_PMU_PSTATE, false);
562
563 /* Read fuses to check if gpu needs to boot in secure/non-secure mode */
564 if (gops->fuse.check_priv_security(g))
565 return -EINVAL; /* Do not boot gpu */
566
567 /* priv security dependent ops */
568 if (nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) {
569 /* Add in ops from gm20b acr */
570 gops->pmu.is_pmu_supported = gm20b_is_pmu_supported,
571 gops->pmu.prepare_ucode = prepare_ucode_blob,
572 gops->pmu.pmu_setup_hw_and_bootstrap = gm20b_bootstrap_hs_flcn,
573 gops->pmu.is_lazy_bootstrap = gm20b_is_lazy_bootstrap,
574 gops->pmu.is_priv_load = gm20b_is_priv_load,
575 gops->pmu.get_wpr = gm20b_wpr_info,
576 gops->pmu.alloc_blob_space = gm20b_alloc_blob_space,
577 gops->pmu.pmu_populate_loader_cfg =
578 gm20b_pmu_populate_loader_cfg,
579 gops->pmu.flcn_populate_bl_dmem_desc =
580 gm20b_flcn_populate_bl_dmem_desc,
581 gops->pmu.falcon_wait_for_halt = pmu_wait_for_halt,
582 gops->pmu.falcon_clear_halt_interrupt_status =
583 clear_halt_interrupt_status,
584 gops->pmu.init_falcon_setup_hw = gm20b_init_pmu_setup_hw1,
585
586 gops->pmu.init_wpr_region = gm20b_pmu_init_acr;
587 gops->pmu.load_lsfalcon_ucode = gp10b_load_falcon_ucode;
588 gops->pmu.is_lazy_bootstrap = gp10b_is_lazy_bootstrap;
589 gops->pmu.is_priv_load = gp10b_is_priv_load;
590
591 gops->gr.load_ctxsw_ucode = gr_gm20b_load_ctxsw_ucode;
592 } else {
593 /* Inherit from gk20a */
594 gops->pmu.is_pmu_supported = gk20a_is_pmu_supported,
595 gops->pmu.prepare_ucode = nvgpu_pmu_prepare_ns_ucode_blob,
596 gops->pmu.pmu_setup_hw_and_bootstrap = gk20a_init_pmu_setup_hw1,
597 gops->pmu.pmu_nsbootstrap = pmu_bootstrap,
598
599 gops->pmu.load_lsfalcon_ucode = NULL;
600 gops->pmu.init_wpr_region = NULL;
601 gops->pmu.pmu_setup_hw_and_bootstrap = gp10b_init_pmu_setup_hw1;
602
603 gops->gr.load_ctxsw_ucode = gr_gk20a_load_ctxsw_ucode;
604 }
605
606 __nvgpu_set_enabled(g, NVGPU_PMU_FECS_BOOTSTRAP_DONE, false);
607 g->pmu_lsf_pmu_wpr_init_done = 0;
608 g->bootstrap_owner = LSF_BOOTSTRAP_OWNER_DEFAULT;
609
610 g->name = "gp10b";
611
612 return 0;
613}
diff --git a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_mm_gp10b.c b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_mm_gp10b.c
new file mode 100644
index 00000000..cf9a28c7
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_mm_gp10b.c
@@ -0,0 +1,206 @@
1/*
2 * Virtualized GPU Memory Management
3 *
4 * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include "vgpu_mm_gp10b.h"
26#include "gk20a/mm_gk20a.h"
27
28#include <nvgpu/bug.h>
29#include <nvgpu/dma.h>
30#include <nvgpu/vgpu/vgpu_ivc.h>
31#include <nvgpu/vgpu/vgpu.h>
32
33int vgpu_gp10b_init_mm_setup_hw(struct gk20a *g)
34{
35 g->mm.disable_bigpage = true;
36 return 0;
37}
38
39static inline int add_mem_desc(struct tegra_vgpu_mem_desc *mem_desc,
40 u64 addr, u64 size, size_t *oob_size)
41{
42 if (*oob_size < sizeof(*mem_desc))
43 return -ENOMEM;
44
45 mem_desc->addr = addr;
46 mem_desc->length = size;
47 *oob_size -= sizeof(*mem_desc);
48 return 0;
49}
50
51u64 vgpu_gp10b_locked_gmmu_map(struct vm_gk20a *vm,
52 u64 map_offset,
53 struct nvgpu_sgt *sgt,
54 u64 buffer_offset,
55 u64 size,
56 int pgsz_idx,
57 u8 kind_v,
58 u32 ctag_offset,
59 u32 flags,
60 int rw_flag,
61 bool clear_ctags,
62 bool sparse,
63 bool priv,
64 struct vm_gk20a_mapping_batch *batch,
65 enum nvgpu_aperture aperture)
66{
67 int err = 0;
68 struct gk20a *g = gk20a_from_vm(vm);
69 struct tegra_vgpu_cmd_msg msg;
70 struct tegra_vgpu_as_map_ex_params *p = &msg.params.as_map_ex;
71 struct tegra_vgpu_mem_desc *mem_desc;
72 u32 page_size = vm->gmmu_page_sizes[pgsz_idx];
73 u64 buffer_size = PAGE_ALIGN(size);
74 u64 space_to_skip = buffer_offset;
75 u32 mem_desc_count = 0, i;
76 void *handle = NULL;
77 size_t oob_size;
78 u8 prot;
79 void *sgl;
80
81 gk20a_dbg_fn("");
82
83 /* FIXME: add support for sparse mappings */
84
85 if (WARN_ON(!sgt) || WARN_ON(nvgpu_iommuable(g)))
86 return 0;
87
88 if (space_to_skip & (page_size - 1))
89 return 0;
90
91 memset(&msg, 0, sizeof(msg));
92
93 /* Allocate (or validate when map_offset != 0) the virtual address. */
94 if (!map_offset) {
95 map_offset = __nvgpu_vm_alloc_va(vm, size, pgsz_idx);
96 if (!map_offset) {
97 nvgpu_err(g, "failed to allocate va space");
98 err = -ENOMEM;
99 goto fail;
100 }
101 }
102
103 handle = vgpu_ivc_oob_get_ptr(vgpu_ivc_get_server_vmid(),
104 TEGRA_VGPU_QUEUE_CMD,
105 (void **)&mem_desc, &oob_size);
106 if (!handle) {
107 err = -EINVAL;
108 goto fail;
109 }
110 sgl = sgt->sgl;
111 while (sgl) {
112 u64 phys_addr;
113 u64 chunk_length;
114
115 /*
116 * Cut out sgl ents for space_to_skip.
117 */
118 if (space_to_skip &&
119 space_to_skip >= nvgpu_sgt_get_length(sgt, sgl)) {
120 space_to_skip -= nvgpu_sgt_get_length(sgt, sgl);
121 sgl = nvgpu_sgt_get_next(sgt, sgl);
122 continue;
123 }
124
125 phys_addr = nvgpu_sgt_get_phys(sgt, sgl) + space_to_skip;
126 chunk_length = min(size,
127 nvgpu_sgt_get_length(sgt, sgl) - space_to_skip);
128
129 if (add_mem_desc(&mem_desc[mem_desc_count++], phys_addr,
130 chunk_length, &oob_size)) {
131 err = -ENOMEM;
132 goto fail;
133 }
134
135 space_to_skip = 0;
136 size -= chunk_length;
137 sgl = nvgpu_sgt_get_next(sgt, sgl);
138
139 if (size == 0)
140 break;
141 }
142
143 if (rw_flag == gk20a_mem_flag_read_only)
144 prot = TEGRA_VGPU_MAP_PROT_READ_ONLY;
145 else if (rw_flag == gk20a_mem_flag_write_only)
146 prot = TEGRA_VGPU_MAP_PROT_WRITE_ONLY;
147 else
148 prot = TEGRA_VGPU_MAP_PROT_NONE;
149
150 if (pgsz_idx == gmmu_page_size_kernel) {
151 if (page_size == vm->gmmu_page_sizes[gmmu_page_size_small]) {
152 pgsz_idx = gmmu_page_size_small;
153 } else if (page_size ==
154 vm->gmmu_page_sizes[gmmu_page_size_big]) {
155 pgsz_idx = gmmu_page_size_big;
156 } else {
157 nvgpu_err(g, "invalid kernel page size %d",
158 page_size);
159 goto fail;
160 }
161 }
162
163 msg.cmd = TEGRA_VGPU_CMD_AS_MAP_EX;
164 msg.handle = vgpu_get_handle(g);
165 p->handle = vm->handle;
166 p->gpu_va = map_offset;
167 p->size = buffer_size;
168 p->mem_desc_count = mem_desc_count;
169 p->pgsz_idx = pgsz_idx;
170 p->iova = 0;
171 p->kind = kind_v;
172 if (flags & NVGPU_VM_MAP_CACHEABLE)
173 p->flags = TEGRA_VGPU_MAP_CACHEABLE;
174 if (flags & NVGPU_VM_MAP_IO_COHERENT)
175 p->flags |= TEGRA_VGPU_MAP_IO_COHERENT;
176 if (flags & NVGPU_VM_MAP_L3_ALLOC)
177 p->flags |= TEGRA_VGPU_MAP_L3_ALLOC;
178 p->prot = prot;
179 p->ctag_offset = ctag_offset;
180 p->clear_ctags = clear_ctags;
181 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
182 if (err || msg.ret)
183 goto fail;
184
185 /* TLB invalidate handled on server side */
186
187 vgpu_ivc_oob_put_ptr(handle);
188 return map_offset;
189fail:
190 if (handle)
191 vgpu_ivc_oob_put_ptr(handle);
192 nvgpu_err(g, "Failed: err=%d, msg.ret=%d", err, msg.ret);
193 nvgpu_err(g,
194 " Map: %-5s GPU virt %#-12llx +%#-9llx "
195 "phys offset: %#-4llx; pgsz: %3dkb perm=%-2s | "
196 "kind=%#02x APT=%-6s",
197 vm->name, map_offset, buffer_size, buffer_offset,
198 vm->gmmu_page_sizes[pgsz_idx] >> 10,
199 nvgpu_gmmu_perm_str(rw_flag),
200 kind_v, "SYSMEM");
201 for (i = 0; i < mem_desc_count; i++)
202 nvgpu_err(g, " > 0x%010llx + 0x%llx",
203 mem_desc[i].addr, mem_desc[i].length);
204
205 return 0;
206}
diff --git a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_mm_gp10b.h b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_mm_gp10b.h
new file mode 100644
index 00000000..44072dd6
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_mm_gp10b.h
@@ -0,0 +1,45 @@
1/*
2 * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef __VGPU_MM_GP10B_H__
24#define __VGPU_MM_GP10B_H__
25
26#include "gk20a/gk20a.h"
27
28u64 vgpu_gp10b_locked_gmmu_map(struct vm_gk20a *vm,
29 u64 map_offset,
30 struct nvgpu_sgt *sgt,
31 u64 buffer_offset,
32 u64 size,
33 int pgsz_idx,
34 u8 kind_v,
35 u32 ctag_offset,
36 u32 flags,
37 int rw_flag,
38 bool clear_ctags,
39 bool sparse,
40 bool priv,
41 struct vm_gk20a_mapping_batch *batch,
42 enum nvgpu_aperture aperture);
43int vgpu_gp10b_init_mm_setup_hw(struct gk20a *g);
44
45#endif
diff --git a/drivers/gpu/nvgpu/vgpu/gr_vgpu.c b/drivers/gpu/nvgpu/vgpu/gr_vgpu.c
new file mode 100644
index 00000000..ee5a5d36
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gr_vgpu.c
@@ -0,0 +1,1283 @@
1/*
2 * Virtualized GPU Graphics
3 *
4 * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <nvgpu/kmem.h>
26#include <nvgpu/bug.h>
27#include <nvgpu/dma.h>
28#include <nvgpu/error_notifier.h>
29#include <nvgpu/dma.h>
30#include <nvgpu/vgpu/vgpu_ivc.h>
31#include <nvgpu/vgpu/vgpu.h>
32
33#include "gr_vgpu.h"
34#include "gk20a/gk20a.h"
35#include "gk20a/dbg_gpu_gk20a.h"
36#include "gk20a/channel_gk20a.h"
37#include "gk20a/tsg_gk20a.h"
38
39#include <nvgpu/hw/gk20a/hw_gr_gk20a.h>
40#include <nvgpu/hw/gk20a/hw_ctxsw_prog_gk20a.h>
41
42void vgpu_gr_detect_sm_arch(struct gk20a *g)
43{
44 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
45
46 gk20a_dbg_fn("");
47
48 g->params.sm_arch_sm_version =
49 priv->constants.sm_arch_sm_version;
50 g->params.sm_arch_spa_version =
51 priv->constants.sm_arch_spa_version;
52 g->params.sm_arch_warp_count =
53 priv->constants.sm_arch_warp_count;
54}
55
56int vgpu_gr_commit_inst(struct channel_gk20a *c, u64 gpu_va)
57{
58 struct tegra_vgpu_cmd_msg msg;
59 struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx;
60 int err;
61
62 gk20a_dbg_fn("");
63
64 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_COMMIT_GR_CTX;
65 msg.handle = vgpu_get_handle(c->g);
66 p->handle = c->virt_ctx;
67 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
68
69 return (err || msg.ret) ? -1 : 0;
70}
71
72static int vgpu_gr_commit_global_ctx_buffers(struct gk20a *g,
73 struct channel_gk20a *c, bool patch)
74{
75 struct tegra_vgpu_cmd_msg msg;
76 struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx;
77 int err;
78
79 gk20a_dbg_fn("");
80
81 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_COMMIT_GR_GLOBAL_CTX;
82 msg.handle = vgpu_get_handle(g);
83 p->handle = c->virt_ctx;
84 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
85
86 return (err || msg.ret) ? -1 : 0;
87}
88
89/* load saved fresh copy of gloden image into channel gr_ctx */
90static int vgpu_gr_load_golden_ctx_image(struct gk20a *g,
91 struct channel_gk20a *c)
92{
93 struct tegra_vgpu_cmd_msg msg;
94 struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx;
95 int err;
96
97 gk20a_dbg_fn("");
98
99 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_LOAD_GR_GOLDEN_CTX;
100 msg.handle = vgpu_get_handle(g);
101 p->handle = c->virt_ctx;
102 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
103
104 return (err || msg.ret) ? -1 : 0;
105}
106
107int vgpu_gr_init_ctx_state(struct gk20a *g)
108{
109 struct gr_gk20a *gr = &g->gr;
110 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
111
112 gk20a_dbg_fn("");
113
114 g->gr.ctx_vars.golden_image_size = priv->constants.golden_ctx_size;
115 g->gr.ctx_vars.zcull_ctxsw_image_size = priv->constants.zcull_ctx_size;
116 g->gr.ctx_vars.pm_ctxsw_image_size = priv->constants.hwpm_ctx_size;
117 if (!g->gr.ctx_vars.golden_image_size ||
118 !g->gr.ctx_vars.zcull_ctxsw_image_size ||
119 !g->gr.ctx_vars.pm_ctxsw_image_size)
120 return -ENXIO;
121
122 gr->ctx_vars.buffer_size = g->gr.ctx_vars.golden_image_size;
123 g->gr.ctx_vars.priv_access_map_size = 512 * 1024;
124 return 0;
125}
126
127static int vgpu_gr_alloc_global_ctx_buffers(struct gk20a *g)
128{
129 struct gr_gk20a *gr = &g->gr;
130 int attr_buffer_size;
131
132 u32 cb_buffer_size = gr->bundle_cb_default_size *
133 gr_scc_bundle_cb_size_div_256b_byte_granularity_v();
134
135 u32 pagepool_buffer_size = g->ops.gr.pagepool_default_size(g) *
136 gr_scc_pagepool_total_pages_byte_granularity_v();
137
138 gk20a_dbg_fn("");
139
140 attr_buffer_size = g->ops.gr.calc_global_ctx_buffer_size(g);
141
142 gk20a_dbg_info("cb_buffer_size : %d", cb_buffer_size);
143 gr->global_ctx_buffer[CIRCULAR].mem.size = cb_buffer_size;
144
145 gk20a_dbg_info("pagepool_buffer_size : %d", pagepool_buffer_size);
146 gr->global_ctx_buffer[PAGEPOOL].mem.size = pagepool_buffer_size;
147
148 gk20a_dbg_info("attr_buffer_size : %d", attr_buffer_size);
149 gr->global_ctx_buffer[ATTRIBUTE].mem.size = attr_buffer_size;
150
151 gk20a_dbg_info("priv access map size : %d",
152 gr->ctx_vars.priv_access_map_size);
153 gr->global_ctx_buffer[PRIV_ACCESS_MAP].mem.size =
154 gr->ctx_vars.priv_access_map_size;
155
156 return 0;
157}
158
159static int vgpu_gr_map_global_ctx_buffers(struct gk20a *g,
160 struct channel_gk20a *c)
161{
162 struct tegra_vgpu_cmd_msg msg;
163 struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx;
164 struct vm_gk20a *ch_vm = c->vm;
165 struct tsg_gk20a *tsg;
166 u64 *g_bfr_va;
167 u64 *g_bfr_size;
168 struct gr_gk20a *gr = &g->gr;
169 u64 gpu_va;
170 u32 i;
171 int err;
172
173 gk20a_dbg_fn("");
174
175 tsg = tsg_gk20a_from_ch(c);
176 if (!tsg)
177 return -EINVAL;
178
179 g_bfr_va = tsg->gr_ctx.global_ctx_buffer_va;
180 g_bfr_size = tsg->gr_ctx.global_ctx_buffer_size;
181
182 /* Circular Buffer */
183 gpu_va = __nvgpu_vm_alloc_va(ch_vm,
184 gr->global_ctx_buffer[CIRCULAR].mem.size,
185 gmmu_page_size_kernel);
186
187 if (!gpu_va)
188 goto clean_up;
189 g_bfr_va[CIRCULAR_VA] = gpu_va;
190 g_bfr_size[CIRCULAR_VA] = gr->global_ctx_buffer[CIRCULAR].mem.size;
191
192 /* Attribute Buffer */
193 gpu_va = __nvgpu_vm_alloc_va(ch_vm,
194 gr->global_ctx_buffer[ATTRIBUTE].mem.size,
195 gmmu_page_size_kernel);
196
197 if (!gpu_va)
198 goto clean_up;
199 g_bfr_va[ATTRIBUTE_VA] = gpu_va;
200 g_bfr_size[ATTRIBUTE_VA] = gr->global_ctx_buffer[ATTRIBUTE].mem.size;
201
202 /* Page Pool */
203 gpu_va = __nvgpu_vm_alloc_va(ch_vm,
204 gr->global_ctx_buffer[PAGEPOOL].mem.size,
205 gmmu_page_size_kernel);
206 if (!gpu_va)
207 goto clean_up;
208 g_bfr_va[PAGEPOOL_VA] = gpu_va;
209 g_bfr_size[PAGEPOOL_VA] = gr->global_ctx_buffer[PAGEPOOL].mem.size;
210
211 /* Priv register Access Map */
212 gpu_va = __nvgpu_vm_alloc_va(ch_vm,
213 gr->global_ctx_buffer[PRIV_ACCESS_MAP].mem.size,
214 gmmu_page_size_kernel);
215 if (!gpu_va)
216 goto clean_up;
217 g_bfr_va[PRIV_ACCESS_MAP_VA] = gpu_va;
218 g_bfr_size[PRIV_ACCESS_MAP_VA] =
219 gr->global_ctx_buffer[PRIV_ACCESS_MAP].mem.size;
220
221 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_MAP_GR_GLOBAL_CTX;
222 msg.handle = vgpu_get_handle(g);
223 p->handle = c->virt_ctx;
224 p->cb_va = g_bfr_va[CIRCULAR_VA];
225 p->attr_va = g_bfr_va[ATTRIBUTE_VA];
226 p->page_pool_va = g_bfr_va[PAGEPOOL_VA];
227 p->priv_access_map_va = g_bfr_va[PRIV_ACCESS_MAP_VA];
228 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
229 if (err || msg.ret)
230 goto clean_up;
231
232 tsg->gr_ctx.global_ctx_buffer_mapped = true;
233 return 0;
234
235 clean_up:
236 for (i = 0; i < NR_GLOBAL_CTX_BUF_VA; i++) {
237 if (g_bfr_va[i]) {
238 __nvgpu_vm_free_va(ch_vm, g_bfr_va[i],
239 gmmu_page_size_kernel);
240 g_bfr_va[i] = 0;
241 }
242 }
243 return -ENOMEM;
244}
245
246static void vgpu_gr_unmap_global_ctx_buffers(struct tsg_gk20a *tsg)
247{
248 struct vm_gk20a *ch_vm = tsg->vm;
249 u64 *g_bfr_va = tsg->gr_ctx.global_ctx_buffer_va;
250 u64 *g_bfr_size = tsg->gr_ctx.global_ctx_buffer_size;
251 u32 i;
252
253 gk20a_dbg_fn("");
254
255 if (tsg->gr_ctx.global_ctx_buffer_mapped) {
256 /* server will unmap on channel close */
257
258 for (i = 0; i < NR_GLOBAL_CTX_BUF_VA; i++) {
259 if (g_bfr_va[i]) {
260 __nvgpu_vm_free_va(ch_vm, g_bfr_va[i],
261 gmmu_page_size_kernel);
262 g_bfr_va[i] = 0;
263 g_bfr_size[i] = 0;
264 }
265 }
266
267 tsg->gr_ctx.global_ctx_buffer_mapped = false;
268 }
269}
270
271int vgpu_gr_alloc_gr_ctx(struct gk20a *g,
272 struct nvgpu_gr_ctx *gr_ctx,
273 struct vm_gk20a *vm,
274 u32 class,
275 u32 flags)
276{
277 struct tegra_vgpu_cmd_msg msg = {0};
278 struct tegra_vgpu_gr_ctx_params *p = &msg.params.gr_ctx;
279 struct gr_gk20a *gr = &g->gr;
280 int err;
281
282 gk20a_dbg_fn("");
283
284 if (gr->ctx_vars.buffer_size == 0)
285 return 0;
286
287 /* alloc channel gr ctx buffer */
288 gr->ctx_vars.buffer_size = gr->ctx_vars.golden_image_size;
289 gr->ctx_vars.buffer_total_size = gr->ctx_vars.golden_image_size;
290
291 gr_ctx->mem.gpu_va = __nvgpu_vm_alloc_va(vm,
292 gr->ctx_vars.buffer_total_size,
293 gmmu_page_size_kernel);
294
295 if (!gr_ctx->mem.gpu_va)
296 return -ENOMEM;
297 gr_ctx->mem.size = gr->ctx_vars.buffer_total_size;
298 gr_ctx->mem.aperture = APERTURE_SYSMEM;
299
300 msg.cmd = TEGRA_VGPU_CMD_GR_CTX_ALLOC;
301 msg.handle = vgpu_get_handle(g);
302 p->as_handle = vm->handle;
303 p->gr_ctx_va = gr_ctx->mem.gpu_va;
304 p->class_num = class;
305 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
306 err = err ? err : msg.ret;
307
308 if (unlikely(err)) {
309 nvgpu_err(g, "fail to alloc gr_ctx");
310 __nvgpu_vm_free_va(vm, gr_ctx->mem.gpu_va,
311 gmmu_page_size_kernel);
312 gr_ctx->mem.aperture = APERTURE_INVALID;
313 } else {
314 gr_ctx->virt_ctx = p->gr_ctx_handle;
315 }
316
317 return err;
318}
319
320static int vgpu_gr_alloc_channel_patch_ctx(struct gk20a *g,
321 struct channel_gk20a *c)
322{
323 struct tsg_gk20a *tsg;
324 struct patch_desc *patch_ctx;
325 struct vm_gk20a *ch_vm = c->vm;
326 struct tegra_vgpu_cmd_msg msg;
327 struct tegra_vgpu_ch_ctx_params *p = &msg.params.ch_ctx;
328 int err;
329
330 gk20a_dbg_fn("");
331
332 tsg = tsg_gk20a_from_ch(c);
333 if (!tsg)
334 return -EINVAL;
335
336 patch_ctx = &tsg->gr_ctx.patch_ctx;
337 patch_ctx->mem.size = 128 * sizeof(u32);
338 patch_ctx->mem.gpu_va = __nvgpu_vm_alloc_va(ch_vm,
339 patch_ctx->mem.size,
340 gmmu_page_size_kernel);
341 if (!patch_ctx->mem.gpu_va)
342 return -ENOMEM;
343
344 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_ALLOC_GR_PATCH_CTX;
345 msg.handle = vgpu_get_handle(g);
346 p->handle = c->virt_ctx;
347 p->patch_ctx_va = patch_ctx->mem.gpu_va;
348 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
349 if (err || msg.ret) {
350 __nvgpu_vm_free_va(ch_vm, patch_ctx->mem.gpu_va,
351 gmmu_page_size_kernel);
352 err = -ENOMEM;
353 }
354
355 return err;
356}
357
358static void vgpu_gr_free_channel_patch_ctx(struct tsg_gk20a *tsg)
359{
360 struct patch_desc *patch_ctx = &tsg->gr_ctx.patch_ctx;
361
362 gk20a_dbg_fn("");
363
364 if (patch_ctx->mem.gpu_va) {
365 /* server will free on channel close */
366
367 __nvgpu_vm_free_va(tsg->vm, patch_ctx->mem.gpu_va,
368 gmmu_page_size_kernel);
369 patch_ctx->mem.gpu_va = 0;
370 }
371}
372
373static void vgpu_gr_free_channel_pm_ctx(struct tsg_gk20a *tsg)
374{
375 struct nvgpu_gr_ctx *ch_ctx = &tsg->gr_ctx;
376 struct pm_ctx_desc *pm_ctx = &ch_ctx->pm_ctx;
377
378 gk20a_dbg_fn("");
379
380 /* check if hwpm was ever initialized. If not, nothing to do */
381 if (pm_ctx->mem.gpu_va == 0)
382 return;
383
384 /* server will free on channel close */
385
386 __nvgpu_vm_free_va(tsg->vm, pm_ctx->mem.gpu_va,
387 gmmu_page_size_kernel);
388 pm_ctx->mem.gpu_va = 0;
389}
390
391void vgpu_gr_free_gr_ctx(struct gk20a *g,
392 struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx)
393{
394 struct tsg_gk20a *tsg;
395
396 gk20a_dbg_fn("");
397
398 if (gr_ctx->mem.gpu_va) {
399 struct tegra_vgpu_cmd_msg msg;
400 struct tegra_vgpu_gr_ctx_params *p = &msg.params.gr_ctx;
401 int err;
402
403 msg.cmd = TEGRA_VGPU_CMD_GR_CTX_FREE;
404 msg.handle = vgpu_get_handle(g);
405 p->gr_ctx_handle = gr_ctx->virt_ctx;
406 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
407 WARN_ON(err || msg.ret);
408
409 __nvgpu_vm_free_va(vm, gr_ctx->mem.gpu_va,
410 gmmu_page_size_kernel);
411
412 tsg = &g->fifo.tsg[gr_ctx->tsgid];
413 vgpu_gr_unmap_global_ctx_buffers(tsg);
414 vgpu_gr_free_channel_patch_ctx(tsg);
415 vgpu_gr_free_channel_pm_ctx(tsg);
416
417 nvgpu_dma_unmap_free(vm, &gr_ctx->pagepool_ctxsw_buffer);
418 nvgpu_dma_unmap_free(vm, &gr_ctx->betacb_ctxsw_buffer);
419 nvgpu_dma_unmap_free(vm, &gr_ctx->spill_ctxsw_buffer);
420 nvgpu_dma_unmap_free(vm, &gr_ctx->preempt_ctxsw_buffer);
421
422 memset(gr_ctx, 0, sizeof(*gr_ctx));
423 }
424}
425
426static int vgpu_gr_ch_bind_gr_ctx(struct channel_gk20a *c)
427{
428 struct tsg_gk20a *tsg;
429 struct nvgpu_gr_ctx *gr_ctx;
430 struct tegra_vgpu_cmd_msg msg = {0};
431 struct tegra_vgpu_channel_bind_gr_ctx_params *p =
432 &msg.params.ch_bind_gr_ctx;
433 int err;
434
435 tsg = tsg_gk20a_from_ch(c);
436 if (!tsg)
437 return -EINVAL;
438
439 gr_ctx = &tsg->gr_ctx;
440
441 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_BIND_GR_CTX;
442 msg.handle = vgpu_get_handle(c->g);
443 p->ch_handle = c->virt_ctx;
444 p->gr_ctx_handle = gr_ctx->virt_ctx;
445 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
446 err = err ? err : msg.ret;
447 WARN_ON(err);
448
449 return err;
450}
451
452static int vgpu_gr_tsg_bind_gr_ctx(struct tsg_gk20a *tsg)
453{
454 struct nvgpu_gr_ctx *gr_ctx = &tsg->gr_ctx;
455 struct tegra_vgpu_cmd_msg msg = {0};
456 struct tegra_vgpu_tsg_bind_gr_ctx_params *p =
457 &msg.params.tsg_bind_gr_ctx;
458 int err;
459
460 msg.cmd = TEGRA_VGPU_CMD_TSG_BIND_GR_CTX;
461 msg.handle = vgpu_get_handle(tsg->g);
462 p->tsg_id = tsg->tsgid;
463 p->gr_ctx_handle = gr_ctx->virt_ctx;
464 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
465 err = err ? err : msg.ret;
466 WARN_ON(err);
467
468 return err;
469}
470
471int vgpu_gr_alloc_obj_ctx(struct channel_gk20a *c, u32 class_num, u32 flags)
472{
473 struct gk20a *g = c->g;
474 struct fifo_gk20a *f = &g->fifo;
475 struct nvgpu_gr_ctx *gr_ctx = NULL;
476 struct tsg_gk20a *tsg = NULL;
477 int err = 0;
478
479 gk20a_dbg_fn("");
480
481 /* an address space needs to have been bound at this point.*/
482 if (!gk20a_channel_as_bound(c)) {
483 nvgpu_err(g, "not bound to address space at time"
484 " of grctx allocation");
485 return -EINVAL;
486 }
487
488 if (!g->ops.gr.is_valid_class(g, class_num)) {
489 nvgpu_err(g, "invalid obj class 0x%x", class_num);
490 err = -EINVAL;
491 goto out;
492 }
493 c->obj_class = class_num;
494
495 if (!gk20a_is_channel_marked_as_tsg(c))
496 return -EINVAL;
497
498 tsg = &f->tsg[c->tsgid];
499 gr_ctx = &tsg->gr_ctx;
500
501 if (!nvgpu_mem_is_valid(&gr_ctx->mem)) {
502 tsg->vm = c->vm;
503 nvgpu_vm_get(tsg->vm);
504 err = g->ops.gr.alloc_gr_ctx(g, gr_ctx,
505 c->vm,
506 class_num,
507 flags);
508 if (!err) {
509 gr_ctx->tsgid = tsg->tsgid;
510 err = vgpu_gr_tsg_bind_gr_ctx(tsg);
511 }
512 if (err) {
513 nvgpu_err(g,
514 "fail to allocate TSG gr ctx buffer, err=%d", err);
515 nvgpu_vm_put(tsg->vm);
516 tsg->vm = NULL;
517 goto out;
518 }
519
520 err = vgpu_gr_ch_bind_gr_ctx(c);
521 if (err) {
522 nvgpu_err(g, "fail to bind gr ctx buffer");
523 goto out;
524 }
525
526 /* commit gr ctx buffer */
527 err = g->ops.gr.commit_inst(c, gr_ctx->mem.gpu_va);
528 if (err) {
529 nvgpu_err(g, "fail to commit gr ctx buffer");
530 goto out;
531 }
532
533 /* allocate patch buffer */
534 err = vgpu_gr_alloc_channel_patch_ctx(g, c);
535 if (err) {
536 nvgpu_err(g, "fail to allocate patch buffer");
537 goto out;
538 }
539
540 /* map global buffer to channel gpu_va and commit */
541 err = vgpu_gr_map_global_ctx_buffers(g, c);
542 if (err) {
543 nvgpu_err(g, "fail to map global ctx buffer");
544 goto out;
545 }
546
547 err = vgpu_gr_commit_global_ctx_buffers(g, c, true);
548 if (err) {
549 nvgpu_err(g, "fail to commit global ctx buffers");
550 goto out;
551 }
552
553 /* load golden image */
554 err = gr_gk20a_elpg_protected_call(g,
555 vgpu_gr_load_golden_ctx_image(g, c));
556 if (err) {
557 nvgpu_err(g, "fail to load golden ctx image");
558 goto out;
559 }
560 } else {
561 err = vgpu_gr_ch_bind_gr_ctx(c);
562 if (err) {
563 nvgpu_err(g, "fail to bind gr ctx buffer");
564 goto out;
565 }
566
567 /* commit gr ctx buffer */
568 err = g->ops.gr.commit_inst(c, gr_ctx->mem.gpu_va);
569 if (err) {
570 nvgpu_err(g, "fail to commit gr ctx buffer");
571 goto out;
572 }
573 }
574
575 /* PM ctxt switch is off by default */
576 gr_ctx->pm_ctx.pm_mode = ctxsw_prog_main_image_pm_mode_no_ctxsw_f();
577
578 gk20a_dbg_fn("done");
579 return 0;
580out:
581 /* 1. gr_ctx, patch_ctx and global ctx buffer mapping
582 can be reused so no need to release them.
583 2. golden image load is a one time thing so if
584 they pass, no need to undo. */
585 nvgpu_err(g, "fail");
586 return err;
587}
588
589static int vgpu_gr_init_gr_config(struct gk20a *g, struct gr_gk20a *gr)
590{
591 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
592 u32 gpc_index;
593 u32 sm_per_tpc;
594 int err = -ENOMEM;
595
596 gk20a_dbg_fn("");
597
598 gr->max_gpc_count = priv->constants.max_gpc_count;
599 gr->gpc_count = priv->constants.gpc_count;
600 gr->max_tpc_per_gpc_count = priv->constants.max_tpc_per_gpc_count;
601
602 gr->max_tpc_count = gr->max_gpc_count * gr->max_tpc_per_gpc_count;
603
604 gr->gpc_tpc_count = nvgpu_kzalloc(g, gr->gpc_count * sizeof(u32));
605 if (!gr->gpc_tpc_count)
606 goto cleanup;
607
608 gr->gpc_tpc_mask = nvgpu_kzalloc(g, gr->gpc_count * sizeof(u32));
609 if (!gr->gpc_tpc_mask)
610 goto cleanup;
611
612 sm_per_tpc = priv->constants.sm_per_tpc;
613 gr->sm_to_cluster = nvgpu_kzalloc(g, gr->gpc_count *
614 gr->max_tpc_per_gpc_count *
615 sm_per_tpc *
616 sizeof(struct sm_info));
617 if (!gr->sm_to_cluster)
618 goto cleanup;
619
620 gr->tpc_count = 0;
621 for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) {
622 gr->gpc_tpc_count[gpc_index] =
623 priv->constants.gpc_tpc_count[gpc_index];
624
625 gr->tpc_count += gr->gpc_tpc_count[gpc_index];
626
627 if (g->ops.gr.get_gpc_tpc_mask)
628 gr->gpc_tpc_mask[gpc_index] =
629 g->ops.gr.get_gpc_tpc_mask(g, gpc_index);
630 }
631
632 g->ops.gr.bundle_cb_defaults(g);
633 g->ops.gr.cb_size_default(g);
634 g->ops.gr.calc_global_ctx_buffer_size(g);
635 err = g->ops.gr.init_fs_state(g);
636 if (err)
637 goto cleanup;
638 return 0;
639cleanup:
640 nvgpu_err(g, "out of memory");
641
642 nvgpu_kfree(g, gr->gpc_tpc_count);
643 gr->gpc_tpc_count = NULL;
644
645 nvgpu_kfree(g, gr->gpc_tpc_mask);
646 gr->gpc_tpc_mask = NULL;
647
648 return err;
649}
650
651int vgpu_gr_bind_ctxsw_zcull(struct gk20a *g, struct gr_gk20a *gr,
652 struct channel_gk20a *c, u64 zcull_va,
653 u32 mode)
654{
655 struct tegra_vgpu_cmd_msg msg;
656 struct tegra_vgpu_zcull_bind_params *p = &msg.params.zcull_bind;
657 int err;
658
659 gk20a_dbg_fn("");
660
661 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_BIND_ZCULL;
662 msg.handle = vgpu_get_handle(g);
663 p->handle = c->virt_ctx;
664 p->zcull_va = zcull_va;
665 p->mode = mode;
666 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
667
668 return (err || msg.ret) ? -ENOMEM : 0;
669}
670
671int vgpu_gr_get_zcull_info(struct gk20a *g, struct gr_gk20a *gr,
672 struct gr_zcull_info *zcull_params)
673{
674 struct tegra_vgpu_cmd_msg msg;
675 struct tegra_vgpu_zcull_info_params *p = &msg.params.zcull_info;
676 int err;
677
678 gk20a_dbg_fn("");
679
680 msg.cmd = TEGRA_VGPU_CMD_GET_ZCULL_INFO;
681 msg.handle = vgpu_get_handle(g);
682 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
683 if (err || msg.ret)
684 return -ENOMEM;
685
686 zcull_params->width_align_pixels = p->width_align_pixels;
687 zcull_params->height_align_pixels = p->height_align_pixels;
688 zcull_params->pixel_squares_by_aliquots = p->pixel_squares_by_aliquots;
689 zcull_params->aliquot_total = p->aliquot_total;
690 zcull_params->region_byte_multiplier = p->region_byte_multiplier;
691 zcull_params->region_header_size = p->region_header_size;
692 zcull_params->subregion_header_size = p->subregion_header_size;
693 zcull_params->subregion_width_align_pixels =
694 p->subregion_width_align_pixels;
695 zcull_params->subregion_height_align_pixels =
696 p->subregion_height_align_pixels;
697 zcull_params->subregion_count = p->subregion_count;
698
699 return 0;
700}
701
702u32 vgpu_gr_get_gpc_tpc_mask(struct gk20a *g, u32 gpc_index)
703{
704 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
705
706 return priv->constants.gpc_tpc_mask[gpc_index];
707}
708
709u32 vgpu_gr_get_max_fbps_count(struct gk20a *g)
710{
711 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
712
713 gk20a_dbg_fn("");
714
715 return priv->constants.num_fbps;
716}
717
718u32 vgpu_gr_get_fbp_en_mask(struct gk20a *g)
719{
720 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
721
722 gk20a_dbg_fn("");
723
724 return priv->constants.fbp_en_mask;
725}
726
727u32 vgpu_gr_get_max_ltc_per_fbp(struct gk20a *g)
728{
729 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
730
731 gk20a_dbg_fn("");
732
733 return priv->constants.ltc_per_fbp;
734}
735
736u32 vgpu_gr_get_max_lts_per_ltc(struct gk20a *g)
737{
738 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
739
740 gk20a_dbg_fn("");
741
742 return priv->constants.max_lts_per_ltc;
743}
744
745u32 *vgpu_gr_rop_l2_en_mask(struct gk20a *g)
746{
747 /* no one use it yet */
748 return NULL;
749}
750
751int vgpu_gr_add_zbc(struct gk20a *g, struct gr_gk20a *gr,
752 struct zbc_entry *zbc_val)
753{
754 struct tegra_vgpu_cmd_msg msg = {0};
755 struct tegra_vgpu_zbc_set_table_params *p = &msg.params.zbc_set_table;
756 int err;
757
758 gk20a_dbg_fn("");
759
760 msg.cmd = TEGRA_VGPU_CMD_ZBC_SET_TABLE;
761 msg.handle = vgpu_get_handle(g);
762
763 p->type = zbc_val->type;
764 p->format = zbc_val->format;
765 switch (p->type) {
766 case GK20A_ZBC_TYPE_COLOR:
767 memcpy(p->color_ds, zbc_val->color_ds, sizeof(p->color_ds));
768 memcpy(p->color_l2, zbc_val->color_l2, sizeof(p->color_l2));
769 break;
770 case GK20A_ZBC_TYPE_DEPTH:
771 p->depth = zbc_val->depth;
772 break;
773 default:
774 return -EINVAL;
775 }
776
777 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
778
779 return (err || msg.ret) ? -ENOMEM : 0;
780}
781
782int vgpu_gr_query_zbc(struct gk20a *g, struct gr_gk20a *gr,
783 struct zbc_query_params *query_params)
784{
785 struct tegra_vgpu_cmd_msg msg = {0};
786 struct tegra_vgpu_zbc_query_table_params *p =
787 &msg.params.zbc_query_table;
788 int err;
789
790 gk20a_dbg_fn("");
791
792 msg.cmd = TEGRA_VGPU_CMD_ZBC_QUERY_TABLE;
793 msg.handle = vgpu_get_handle(g);
794
795 p->type = query_params->type;
796 p->index_size = query_params->index_size;
797
798 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
799 if (err || msg.ret)
800 return -ENOMEM;
801
802 switch (query_params->type) {
803 case GK20A_ZBC_TYPE_COLOR:
804 memcpy(query_params->color_ds, p->color_ds,
805 sizeof(query_params->color_ds));
806 memcpy(query_params->color_l2, p->color_l2,
807 sizeof(query_params->color_l2));
808 break;
809 case GK20A_ZBC_TYPE_DEPTH:
810 query_params->depth = p->depth;
811 break;
812 case GK20A_ZBC_TYPE_INVALID:
813 query_params->index_size = p->index_size;
814 break;
815 default:
816 return -EINVAL;
817 }
818 query_params->ref_cnt = p->ref_cnt;
819 query_params->format = p->format;
820
821 return 0;
822}
823
824static void vgpu_remove_gr_support(struct gr_gk20a *gr)
825{
826 gk20a_dbg_fn("");
827
828 gk20a_comptag_allocator_destroy(gr->g, &gr->comp_tags);
829
830 nvgpu_kfree(gr->g, gr->sm_error_states);
831 gr->sm_error_states = NULL;
832
833 nvgpu_kfree(gr->g, gr->gpc_tpc_mask);
834 gr->gpc_tpc_mask = NULL;
835
836 nvgpu_kfree(gr->g, gr->sm_to_cluster);
837 gr->sm_to_cluster = NULL;
838
839 nvgpu_kfree(gr->g, gr->gpc_tpc_count);
840 gr->gpc_tpc_count = NULL;
841}
842
843static int vgpu_gr_init_gr_setup_sw(struct gk20a *g)
844{
845 struct gr_gk20a *gr = &g->gr;
846 int err;
847
848 gk20a_dbg_fn("");
849
850 if (gr->sw_ready) {
851 gk20a_dbg_fn("skip init");
852 return 0;
853 }
854
855 gr->g = g;
856
857#if defined(CONFIG_GK20A_CYCLE_STATS)
858 nvgpu_mutex_init(&g->gr.cs_lock);
859#endif
860
861 err = vgpu_gr_init_gr_config(g, gr);
862 if (err)
863 goto clean_up;
864
865 err = g->ops.gr.init_ctx_state(g);
866 if (err)
867 goto clean_up;
868
869 err = g->ops.ltc.init_comptags(g, gr);
870 if (err)
871 goto clean_up;
872
873 err = vgpu_gr_alloc_global_ctx_buffers(g);
874 if (err)
875 goto clean_up;
876
877 nvgpu_mutex_init(&gr->ctx_mutex);
878
879 gr->sm_error_states = nvgpu_kzalloc(g,
880 sizeof(struct nvgpu_gr_sm_error_state) *
881 gr->no_of_sm);
882 if (!gr->sm_error_states) {
883 err = -ENOMEM;
884 goto clean_up;
885 }
886
887 gr->remove_support = vgpu_remove_gr_support;
888 gr->sw_ready = true;
889
890 gk20a_dbg_fn("done");
891 return 0;
892
893clean_up:
894 nvgpu_err(g, "fail");
895 vgpu_remove_gr_support(gr);
896 return err;
897}
898
899int vgpu_init_gr_support(struct gk20a *g)
900{
901 gk20a_dbg_fn("");
902
903 return vgpu_gr_init_gr_setup_sw(g);
904}
905
906int vgpu_gr_isr(struct gk20a *g, struct tegra_vgpu_gr_intr_info *info)
907{
908 struct fifo_gk20a *f = &g->fifo;
909 struct channel_gk20a *ch = gk20a_channel_get(&f->channel[info->chid]);
910
911 gk20a_dbg_fn("");
912 if (!ch)
913 return 0;
914
915 if (info->type != TEGRA_VGPU_GR_INTR_NOTIFY &&
916 info->type != TEGRA_VGPU_GR_INTR_SEMAPHORE)
917 nvgpu_err(g, "gr intr (%d) on ch %u", info->type, info->chid);
918
919 switch (info->type) {
920 case TEGRA_VGPU_GR_INTR_NOTIFY:
921 nvgpu_cond_broadcast_interruptible(&ch->notifier_wq);
922 break;
923 case TEGRA_VGPU_GR_INTR_SEMAPHORE:
924 nvgpu_cond_broadcast_interruptible(&ch->semaphore_wq);
925 break;
926 case TEGRA_VGPU_GR_INTR_SEMAPHORE_TIMEOUT:
927 nvgpu_set_error_notifier(ch,
928 NVGPU_ERR_NOTIFIER_GR_SEMAPHORE_TIMEOUT);
929 break;
930 case TEGRA_VGPU_GR_INTR_ILLEGAL_NOTIFY:
931 nvgpu_set_error_notifier(ch,
932 NVGPU_ERR_NOTIFIER_GR_ILLEGAL_NOTIFY);
933 case TEGRA_VGPU_GR_INTR_ILLEGAL_METHOD:
934 break;
935 case TEGRA_VGPU_GR_INTR_ILLEGAL_CLASS:
936 nvgpu_set_error_notifier(ch,
937 NVGPU_ERR_NOTIFIER_GR_ERROR_SW_NOTIFY);
938 break;
939 case TEGRA_VGPU_GR_INTR_FECS_ERROR:
940 break;
941 case TEGRA_VGPU_GR_INTR_CLASS_ERROR:
942 nvgpu_set_error_notifier(ch,
943 NVGPU_ERR_NOTIFIER_GR_ERROR_SW_NOTIFY);
944 break;
945 case TEGRA_VGPU_GR_INTR_FIRMWARE_METHOD:
946 nvgpu_set_error_notifier(ch,
947 NVGPU_ERR_NOTIFIER_GR_ERROR_SW_NOTIFY);
948 break;
949 case TEGRA_VGPU_GR_INTR_EXCEPTION:
950 nvgpu_set_error_notifier(ch,
951 NVGPU_ERR_NOTIFIER_GR_ERROR_SW_NOTIFY);
952 break;
953 case TEGRA_VGPU_GR_INTR_SM_EXCEPTION:
954 gk20a_dbg_gpu_post_events(ch);
955 break;
956 default:
957 WARN_ON(1);
958 break;
959 }
960
961 gk20a_channel_put(ch);
962 return 0;
963}
964
965int vgpu_gr_nonstall_isr(struct gk20a *g,
966 struct tegra_vgpu_gr_nonstall_intr_info *info)
967{
968 gk20a_dbg_fn("");
969
970 switch (info->type) {
971 case TEGRA_VGPU_GR_NONSTALL_INTR_SEMAPHORE:
972 gk20a_channel_semaphore_wakeup(g, true);
973 break;
974 default:
975 WARN_ON(1);
976 break;
977 }
978
979 return 0;
980}
981
982int vgpu_gr_set_sm_debug_mode(struct gk20a *g,
983 struct channel_gk20a *ch, u64 sms, bool enable)
984{
985 struct tegra_vgpu_cmd_msg msg;
986 struct tegra_vgpu_sm_debug_mode *p = &msg.params.sm_debug_mode;
987 int err;
988
989 gk20a_dbg_fn("");
990
991 msg.cmd = TEGRA_VGPU_CMD_SET_SM_DEBUG_MODE;
992 msg.handle = vgpu_get_handle(g);
993 p->handle = ch->virt_ctx;
994 p->sms = sms;
995 p->enable = (u32)enable;
996 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
997 WARN_ON(err || msg.ret);
998
999 return err ? err : msg.ret;
1000}
1001
1002int vgpu_gr_update_smpc_ctxsw_mode(struct gk20a *g,
1003 struct channel_gk20a *ch, bool enable)
1004{
1005 struct tegra_vgpu_cmd_msg msg;
1006 struct tegra_vgpu_channel_set_ctxsw_mode *p = &msg.params.set_ctxsw_mode;
1007 int err;
1008
1009 gk20a_dbg_fn("");
1010
1011 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SET_SMPC_CTXSW_MODE;
1012 msg.handle = vgpu_get_handle(g);
1013 p->handle = ch->virt_ctx;
1014
1015 if (enable)
1016 p->mode = TEGRA_VGPU_CTXSW_MODE_CTXSW;
1017 else
1018 p->mode = TEGRA_VGPU_CTXSW_MODE_NO_CTXSW;
1019
1020 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
1021 WARN_ON(err || msg.ret);
1022
1023 return err ? err : msg.ret;
1024}
1025
1026int vgpu_gr_update_hwpm_ctxsw_mode(struct gk20a *g,
1027 struct channel_gk20a *ch, bool enable)
1028{
1029 struct tsg_gk20a *tsg;
1030 struct nvgpu_gr_ctx *ch_ctx;
1031 struct pm_ctx_desc *pm_ctx;
1032 struct tegra_vgpu_cmd_msg msg;
1033 struct tegra_vgpu_channel_set_ctxsw_mode *p = &msg.params.set_ctxsw_mode;
1034 int err;
1035
1036 gk20a_dbg_fn("");
1037
1038 tsg = tsg_gk20a_from_ch(ch);
1039 if (!tsg)
1040 return -EINVAL;
1041
1042 ch_ctx = &tsg->gr_ctx;
1043 pm_ctx = &ch_ctx->pm_ctx;
1044
1045 if (enable) {
1046 /*
1047 * send command to enable HWPM only once - otherwise server
1048 * will return an error due to using the same GPU VA twice.
1049 */
1050 if (pm_ctx->pm_mode == ctxsw_prog_main_image_pm_mode_ctxsw_f())
1051 return 0;
1052
1053 p->mode = TEGRA_VGPU_CTXSW_MODE_CTXSW;
1054
1055 /* Allocate buffer if necessary */
1056 if (pm_ctx->mem.gpu_va == 0) {
1057 pm_ctx->mem.gpu_va = __nvgpu_vm_alloc_va(ch->vm,
1058 g->gr.ctx_vars.pm_ctxsw_image_size,
1059 gmmu_page_size_kernel);
1060
1061 if (!pm_ctx->mem.gpu_va)
1062 return -ENOMEM;
1063 pm_ctx->mem.size = g->gr.ctx_vars.pm_ctxsw_image_size;
1064 }
1065 } else {
1066 if (pm_ctx->pm_mode == ctxsw_prog_main_image_pm_mode_no_ctxsw_f())
1067 return 0;
1068
1069 p->mode = TEGRA_VGPU_CTXSW_MODE_NO_CTXSW;
1070 }
1071
1072 msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SET_HWPM_CTXSW_MODE;
1073 msg.handle = vgpu_get_handle(g);
1074 p->handle = ch->virt_ctx;
1075 p->gpu_va = pm_ctx->mem.gpu_va;
1076
1077 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
1078 WARN_ON(err || msg.ret);
1079 err = err ? err : msg.ret;
1080 if (!err)
1081 pm_ctx->pm_mode = enable ?
1082 ctxsw_prog_main_image_pm_mode_ctxsw_f() :
1083 ctxsw_prog_main_image_pm_mode_no_ctxsw_f();
1084
1085 return err;
1086}
1087
1088int vgpu_gr_clear_sm_error_state(struct gk20a *g,
1089 struct channel_gk20a *ch, u32 sm_id)
1090{
1091 struct gr_gk20a *gr = &g->gr;
1092 struct tegra_vgpu_cmd_msg msg;
1093 struct tegra_vgpu_clear_sm_error_state *p =
1094 &msg.params.clear_sm_error_state;
1095 int err;
1096
1097 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1098 msg.cmd = TEGRA_VGPU_CMD_CLEAR_SM_ERROR_STATE;
1099 msg.handle = vgpu_get_handle(g);
1100 p->handle = ch->virt_ctx;
1101 p->sm_id = sm_id;
1102
1103 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
1104 WARN_ON(err || msg.ret);
1105
1106 memset(&gr->sm_error_states[sm_id], 0, sizeof(*gr->sm_error_states));
1107 nvgpu_mutex_release(&g->dbg_sessions_lock);
1108
1109 return err ? err : msg.ret;
1110
1111
1112 return 0;
1113}
1114
1115static int vgpu_gr_suspend_resume_contexts(struct gk20a *g,
1116 struct dbg_session_gk20a *dbg_s,
1117 int *ctx_resident_ch_fd, u32 cmd)
1118{
1119 struct dbg_session_channel_data *ch_data;
1120 struct tegra_vgpu_cmd_msg msg;
1121 struct tegra_vgpu_suspend_resume_contexts *p;
1122 size_t n;
1123 int channel_fd = -1;
1124 int err = 0;
1125 void *handle = NULL;
1126 u16 *oob;
1127 size_t oob_size;
1128
1129 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1130 nvgpu_mutex_acquire(&dbg_s->ch_list_lock);
1131
1132 handle = vgpu_ivc_oob_get_ptr(vgpu_ivc_get_server_vmid(),
1133 TEGRA_VGPU_QUEUE_CMD,
1134 (void **)&oob, &oob_size);
1135 if (!handle) {
1136 err = -EINVAL;
1137 goto done;
1138 }
1139
1140 n = 0;
1141 nvgpu_list_for_each_entry(ch_data, &dbg_s->ch_list,
1142 dbg_session_channel_data, ch_entry)
1143 n++;
1144
1145 if (oob_size < n * sizeof(u16)) {
1146 err = -ENOMEM;
1147 goto done;
1148 }
1149
1150 msg.cmd = cmd;
1151 msg.handle = vgpu_get_handle(g);
1152 p = &msg.params.suspend_contexts;
1153 p->num_channels = n;
1154 n = 0;
1155 nvgpu_list_for_each_entry(ch_data, &dbg_s->ch_list,
1156 dbg_session_channel_data, ch_entry)
1157 oob[n++] = (u16)ch_data->chid;
1158
1159 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
1160 if (err || msg.ret) {
1161 err = -ENOMEM;
1162 goto done;
1163 }
1164
1165 if (p->resident_chid != (u16)~0) {
1166 nvgpu_list_for_each_entry(ch_data, &dbg_s->ch_list,
1167 dbg_session_channel_data, ch_entry) {
1168 if (ch_data->chid == p->resident_chid) {
1169 channel_fd = ch_data->channel_fd;
1170 break;
1171 }
1172 }
1173 }
1174
1175done:
1176 if (handle)
1177 vgpu_ivc_oob_put_ptr(handle);
1178 nvgpu_mutex_release(&dbg_s->ch_list_lock);
1179 nvgpu_mutex_release(&g->dbg_sessions_lock);
1180 *ctx_resident_ch_fd = channel_fd;
1181 return err;
1182}
1183
1184int vgpu_gr_suspend_contexts(struct gk20a *g,
1185 struct dbg_session_gk20a *dbg_s,
1186 int *ctx_resident_ch_fd)
1187{
1188 return vgpu_gr_suspend_resume_contexts(g, dbg_s,
1189 ctx_resident_ch_fd, TEGRA_VGPU_CMD_SUSPEND_CONTEXTS);
1190}
1191
1192int vgpu_gr_resume_contexts(struct gk20a *g,
1193 struct dbg_session_gk20a *dbg_s,
1194 int *ctx_resident_ch_fd)
1195{
1196 return vgpu_gr_suspend_resume_contexts(g, dbg_s,
1197 ctx_resident_ch_fd, TEGRA_VGPU_CMD_RESUME_CONTEXTS);
1198}
1199
1200void vgpu_gr_handle_sm_esr_event(struct gk20a *g,
1201 struct tegra_vgpu_sm_esr_info *info)
1202{
1203 struct nvgpu_gr_sm_error_state *sm_error_states;
1204
1205 if (info->sm_id >= g->gr.no_of_sm) {
1206 nvgpu_err(g, "invalid smd_id %d / %d",
1207 info->sm_id, g->gr.no_of_sm);
1208 return;
1209 }
1210
1211 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
1212
1213 sm_error_states = &g->gr.sm_error_states[info->sm_id];
1214
1215 sm_error_states->hww_global_esr = info->hww_global_esr;
1216 sm_error_states->hww_warp_esr = info->hww_warp_esr;
1217 sm_error_states->hww_warp_esr_pc = info->hww_warp_esr_pc;
1218 sm_error_states->hww_global_esr_report_mask =
1219 info->hww_global_esr_report_mask;
1220 sm_error_states->hww_warp_esr_report_mask =
1221 info->hww_warp_esr_report_mask;
1222
1223 nvgpu_mutex_release(&g->dbg_sessions_lock);
1224}
1225
1226int vgpu_gr_init_sm_id_table(struct gk20a *g)
1227{
1228 struct tegra_vgpu_cmd_msg msg = {};
1229 struct tegra_vgpu_vsms_mapping_params *p = &msg.params.vsms_mapping;
1230 struct tegra_vgpu_vsms_mapping_entry *entry;
1231 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
1232 struct sm_info *sm_info;
1233 int err;
1234 struct gr_gk20a *gr = &g->gr;
1235 size_t oob_size;
1236 void *handle = NULL;
1237 u32 sm_id;
1238 u32 max_sm;
1239
1240 msg.cmd = TEGRA_VGPU_CMD_GET_VSMS_MAPPING;
1241 msg.handle = vgpu_get_handle(g);
1242 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
1243 err = err ? err : msg.ret;
1244 if (err) {
1245 nvgpu_err(g, "get vsms mapping failed err %d", err);
1246 return err;
1247 }
1248
1249 handle = vgpu_ivc_oob_get_ptr(vgpu_ivc_get_server_vmid(),
1250 TEGRA_VGPU_QUEUE_CMD,
1251 (void **)&entry, &oob_size);
1252 if (!handle)
1253 return -EINVAL;
1254
1255 max_sm = gr->gpc_count *
1256 gr->max_tpc_per_gpc_count *
1257 priv->constants.sm_per_tpc;
1258 if (p->num_sm > max_sm)
1259 return -EINVAL;
1260
1261 if ((p->num_sm * sizeof(*entry)) > oob_size)
1262 return -EINVAL;
1263
1264 gr->no_of_sm = p->num_sm;
1265 for (sm_id = 0; sm_id < p->num_sm; sm_id++, entry++) {
1266 sm_info = &gr->sm_to_cluster[sm_id];
1267 sm_info->tpc_index = entry->tpc_index;
1268 sm_info->gpc_index = entry->gpc_index;
1269 sm_info->sm_index = entry->sm_index;
1270 sm_info->global_tpc_index = entry->global_tpc_index;
1271 }
1272 vgpu_ivc_oob_put_ptr(handle);
1273
1274 return 0;
1275}
1276
1277int vgpu_gr_init_fs_state(struct gk20a *g)
1278{
1279 if (!g->ops.gr.init_sm_id_table)
1280 return -EINVAL;
1281
1282 return g->ops.gr.init_sm_id_table(g);
1283}
diff --git a/drivers/gpu/nvgpu/vgpu/gr_vgpu.h b/drivers/gpu/nvgpu/vgpu/gr_vgpu.h
new file mode 100644
index 00000000..1f55823c
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gr_vgpu.h
@@ -0,0 +1,74 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _GR_VGPU_H_
24#define _GR_VGPU_H_
25
26#include <nvgpu/types.h>
27
28struct gk20a;
29struct channel_gk20a;
30struct gr_gk20a;
31struct gr_zcull_info;
32struct zbc_entry;
33struct zbc_query_params;
34struct dbg_session_gk20a;
35struct tsg_gk20a;
36
37void vgpu_gr_detect_sm_arch(struct gk20a *g);
38void vgpu_gr_free_channel_ctx(struct channel_gk20a *c, bool is_tsg);
39void vgpu_gr_free_tsg_ctx(struct tsg_gk20a *tsg);
40int vgpu_gr_alloc_obj_ctx(struct channel_gk20a *c, u32 class_num, u32 flags);
41int vgpu_gr_bind_ctxsw_zcull(struct gk20a *g, struct gr_gk20a *gr,
42 struct channel_gk20a *c, u64 zcull_va,
43 u32 mode);
44int vgpu_gr_get_zcull_info(struct gk20a *g, struct gr_gk20a *gr,
45 struct gr_zcull_info *zcull_params);
46u32 vgpu_gr_get_gpc_tpc_mask(struct gk20a *g, u32 gpc_index);
47u32 vgpu_gr_get_max_fbps_count(struct gk20a *g);
48u32 vgpu_gr_get_fbp_en_mask(struct gk20a *g);
49u32 vgpu_gr_get_max_ltc_per_fbp(struct gk20a *g);
50u32 vgpu_gr_get_max_lts_per_ltc(struct gk20a *g);
51u32 *vgpu_gr_rop_l2_en_mask(struct gk20a *g);
52int vgpu_gr_add_zbc(struct gk20a *g, struct gr_gk20a *gr,
53 struct zbc_entry *zbc_val);
54int vgpu_gr_query_zbc(struct gk20a *g, struct gr_gk20a *gr,
55 struct zbc_query_params *query_params);
56int vgpu_gr_set_sm_debug_mode(struct gk20a *g,
57 struct channel_gk20a *ch, u64 sms, bool enable);
58int vgpu_gr_update_smpc_ctxsw_mode(struct gk20a *g,
59 struct channel_gk20a *ch, bool enable);
60int vgpu_gr_update_hwpm_ctxsw_mode(struct gk20a *g,
61 struct channel_gk20a *ch, bool enable);
62int vgpu_gr_clear_sm_error_state(struct gk20a *g,
63 struct channel_gk20a *ch, u32 sm_id);
64int vgpu_gr_suspend_contexts(struct gk20a *g,
65 struct dbg_session_gk20a *dbg_s,
66 int *ctx_resident_ch_fd);
67int vgpu_gr_resume_contexts(struct gk20a *g,
68 struct dbg_session_gk20a *dbg_s,
69 int *ctx_resident_ch_fd);
70int vgpu_gr_commit_inst(struct channel_gk20a *c, u64 gpu_va);
71int vgpu_gr_init_sm_id_table(struct gk20a *g);
72int vgpu_gr_init_fs_state(struct gk20a *g);
73
74#endif
diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_fifo_gv11b.c b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_fifo_gv11b.c
new file mode 100644
index 00000000..18d2de70
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_fifo_gv11b.c
@@ -0,0 +1,145 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <gk20a/gk20a.h>
24#include <nvgpu/vgpu/vgpu.h>
25#include <nvgpu/nvhost.h>
26#include <nvgpu/vgpu/tegra_vgpu.h>
27
28#include "gv11b/fifo_gv11b.h"
29
30#ifdef CONFIG_TEGRA_GK20A_NVHOST
31
32static int set_syncpt_ro_map_gpu_va_locked(struct vm_gk20a *vm)
33{
34 int err;
35 struct gk20a *g = gk20a_from_vm(vm);
36 struct tegra_vgpu_cmd_msg msg = {};
37 struct tegra_vgpu_map_syncpt_params *p = &msg.params.map_syncpt;
38
39 if (vm->syncpt_ro_map_gpu_va)
40 return 0;
41
42 vm->syncpt_ro_map_gpu_va = __nvgpu_vm_alloc_va(vm,
43 g->syncpt_unit_size,
44 gmmu_page_size_kernel);
45 if (!vm->syncpt_ro_map_gpu_va) {
46 nvgpu_err(g, "allocating read-only va space failed");
47 return -ENOMEM;
48 }
49
50 msg.cmd = TEGRA_VGPU_CMD_MAP_SYNCPT;
51 msg.handle = vgpu_get_handle(g);
52 p->as_handle = vm->handle;
53 p->gpu_va = vm->syncpt_ro_map_gpu_va;
54 p->len = g->syncpt_unit_size;
55 p->offset = 0;
56 p->prot = TEGRA_VGPU_MAP_PROT_READ_ONLY;
57 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
58 err = err ? err : msg.ret;
59 if (err) {
60 nvgpu_err(g,
61 "mapping read-only va space failed err %d",
62 err);
63 __nvgpu_vm_free_va(vm, vm->syncpt_ro_map_gpu_va,
64 gmmu_page_size_kernel);
65 vm->syncpt_ro_map_gpu_va = 0;
66 return err;
67 }
68
69 return 0;
70}
71
72int vgpu_gv11b_fifo_alloc_syncpt_buf(struct channel_gk20a *c,
73 u32 syncpt_id, struct nvgpu_mem *syncpt_buf)
74{
75 int err;
76 struct gk20a *g = c->g;
77 struct tegra_vgpu_cmd_msg msg = {};
78 struct tegra_vgpu_map_syncpt_params *p = &msg.params.map_syncpt;
79
80 /*
81 * Add ro map for complete sync point shim range in vm.
82 * All channels sharing same vm will share same ro mapping.
83 * Create rw map for current channel sync point.
84 */
85 nvgpu_mutex_acquire(&c->vm->syncpt_ro_map_lock);
86 err = set_syncpt_ro_map_gpu_va_locked(c->vm);
87 nvgpu_mutex_release(&c->vm->syncpt_ro_map_lock);
88 if (err)
89 return err;
90
91 syncpt_buf->gpu_va = __nvgpu_vm_alloc_va(c->vm, g->syncpt_size,
92 gmmu_page_size_kernel);
93 if (!syncpt_buf->gpu_va) {
94 nvgpu_err(g, "allocating syncpt va space failed");
95 return -ENOMEM;
96 }
97
98 msg.cmd = TEGRA_VGPU_CMD_MAP_SYNCPT;
99 msg.handle = vgpu_get_handle(g);
100 p->as_handle = c->vm->handle;
101 p->gpu_va = syncpt_buf->gpu_va;
102 p->len = g->syncpt_size;
103 p->offset =
104 nvgpu_nvhost_syncpt_unit_interface_get_byte_offset(syncpt_id);
105 p->prot = TEGRA_VGPU_MAP_PROT_NONE;
106 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
107 err = err ? err : msg.ret;
108 if (err) {
109 nvgpu_err(g, "mapping syncpt va space failed err %d", err);
110 __nvgpu_vm_free_va(c->vm, syncpt_buf->gpu_va,
111 gmmu_page_size_kernel);
112 return err;
113 }
114
115 return 0;
116}
117
118int vgpu_gv11b_fifo_get_sync_ro_map(struct vm_gk20a *vm,
119 u64 *base_gpuva, u32 *sync_size)
120{
121 struct gk20a *g = gk20a_from_vm(vm);
122 int err;
123
124 nvgpu_mutex_acquire(&vm->syncpt_ro_map_lock);
125 err = set_syncpt_ro_map_gpu_va_locked(vm);
126 nvgpu_mutex_release(&vm->syncpt_ro_map_lock);
127 if (err)
128 return err;
129
130 *base_gpuva = vm->syncpt_ro_map_gpu_va;
131 *sync_size = g->syncpt_size;
132
133 return 0;
134}
135#endif /* CONFIG_TEGRA_GK20A_NVHOST */
136
137int vgpu_gv11b_init_fifo_setup_hw(struct gk20a *g)
138{
139 struct fifo_gk20a *f = &g->fifo;
140 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
141
142 f->max_subctx_count = priv->constants.max_subctx_count;
143
144 return 0;
145}
diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_fifo_gv11b.h b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_fifo_gv11b.h
new file mode 100644
index 00000000..6d8f8f60
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_fifo_gv11b.h
@@ -0,0 +1,33 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _VGPU_FIFO_GV11B_H_
24#define _VGPU_FIFO_GV11B_H_
25
26struct gk20a;
27
28int vgpu_gv11b_init_fifo_setup_hw(struct gk20a *g);
29int vgpu_gv11b_fifo_alloc_syncpt_buf(struct channel_gk20a *c,
30 u32 syncpt_id, struct nvgpu_mem *syncpt_buf);
31int vgpu_gv11b_fifo_get_sync_ro_map(struct vm_gk20a *vm,
32 u64 *base_gpuva, u32 *sync_size);
33#endif
diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gr_gv11b.c b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gr_gv11b.c
new file mode 100644
index 00000000..3c93c581
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gr_gv11b.c
@@ -0,0 +1,40 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include "gk20a/gk20a.h"
24#include "vgpu/gr_vgpu.h"
25#include "vgpu_subctx_gv11b.h"
26
27int vgpu_gr_gv11b_commit_inst(struct channel_gk20a *c, u64 gpu_va)
28{
29 int err;
30
31 err = vgpu_gv11b_alloc_subctx_header(c);
32 if (err)
33 return err;
34
35 err = vgpu_gr_commit_inst(c, gpu_va);
36 if (err)
37 vgpu_gv11b_free_subctx_header(c);
38
39 return err;
40}
diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gr_gv11b.h b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gr_gv11b.h
new file mode 100644
index 00000000..2433dcd9
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gr_gv11b.h
@@ -0,0 +1,30 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _VGPU_GR_GV11B_H_
24#define _VGPU_GR_GV11B_H_
25
26struct channel_gk20a;
27
28int vgpu_gr_gv11b_commit_inst(struct channel_gk20a *c, u64 gpu_va);
29
30#endif
diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gv11b.c b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gv11b.c
new file mode 100644
index 00000000..a303a3a2
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gv11b.c
@@ -0,0 +1,47 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/enabled.h>
24#include <nvgpu/vgpu/vgpu.h>
25
26#include "gk20a/gk20a.h"
27#include "vgpu_gv11b.h"
28
29int vgpu_gv11b_init_gpu_characteristics(struct gk20a *g)
30{
31 int err;
32
33 gk20a_dbg_fn("");
34
35 err = vgpu_init_gpu_characteristics(g);
36 if (err) {
37 nvgpu_err(g, "vgpu_init_gpu_characteristics failed, err %d\n", err);
38 return err;
39 }
40
41 __nvgpu_set_enabled(g, NVGPU_SUPPORT_TSG_SUBCONTEXTS, true);
42 __nvgpu_set_enabled(g, NVGPU_SUPPORT_IO_COHERENCE, true);
43 __nvgpu_set_enabled(g, NVGPU_SUPPORT_SCG, true);
44 __nvgpu_set_enabled(g, NVGPU_SUPPORT_SYNCPOINT_ADDRESS, true);
45
46 return 0;
47}
diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gv11b.h b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gv11b.h
new file mode 100644
index 00000000..8b2175d9
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_gv11b.h
@@ -0,0 +1,30 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _VGPU_GV11B_H_
24#define _VGPU_GV11B_H_
25
26struct gk20a;
27
28int vgpu_gv11b_init_gpu_characteristics(struct gk20a *g);
29
30#endif
diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_hal_gv11b.c b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_hal_gv11b.c
new file mode 100644
index 00000000..76f7b389
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_hal_gv11b.c
@@ -0,0 +1,603 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <gk20a/gk20a.h>
24#include <gv11b/hal_gv11b.h>
25#include <nvgpu/vgpu/vgpu.h>
26
27#include "vgpu/fifo_vgpu.h"
28#include "vgpu/gr_vgpu.h"
29#include "vgpu/ltc_vgpu.h"
30#include "vgpu/mm_vgpu.h"
31#include "vgpu/dbg_vgpu.h"
32#include "vgpu/fecs_trace_vgpu.h"
33#include "vgpu/css_vgpu.h"
34#include "vgpu/gm20b/vgpu_gr_gm20b.h"
35#include "vgpu/gp10b/vgpu_mm_gp10b.h"
36#include "vgpu/gp10b/vgpu_gr_gp10b.h"
37
38#include <gk20a/fb_gk20a.h>
39#include <gk20a/flcn_gk20a.h>
40#include <gk20a/bus_gk20a.h>
41#include <gk20a/mc_gk20a.h>
42
43#include <gm20b/gr_gm20b.h>
44#include <gm20b/fb_gm20b.h>
45#include <gm20b/fifo_gm20b.h>
46#include <gm20b/pmu_gm20b.h>
47#include <gm20b/mm_gm20b.h>
48#include <gm20b/acr_gm20b.h>
49#include <gm20b/ltc_gm20b.h>
50
51#include <gp10b/fb_gp10b.h>
52#include <gp10b/pmu_gp10b.h>
53#include <gp10b/mm_gp10b.h>
54#include <gp10b/mc_gp10b.h>
55#include <gp10b/ce_gp10b.h>
56#include "gp10b/gr_gp10b.h"
57#include <gp10b/fifo_gp10b.h>
58#include <gp10b/therm_gp10b.h>
59#include <gp10b/priv_ring_gp10b.h>
60#include <gp10b/ltc_gp10b.h>
61
62#include <gp106/pmu_gp106.h>
63#include <gp106/acr_gp106.h>
64
65#include <gv11b/fb_gv11b.h>
66#include <gv11b/pmu_gv11b.h>
67#include <gv11b/acr_gv11b.h>
68#include <gv11b/mm_gv11b.h>
69#include <gv11b/mc_gv11b.h>
70#include <gv11b/ce_gv11b.h>
71#include <gv11b/fifo_gv11b.h>
72#include <gv11b/therm_gv11b.h>
73#include <gv11b/regops_gv11b.h>
74#include <gv11b/gr_ctx_gv11b.h>
75#include <gv11b/ltc_gv11b.h>
76#include <gv11b/gv11b_gating_reglist.h>
77#include <gv11b/gr_gv11b.h>
78
79#include <nvgpu/enabled.h>
80
81#include "vgpu_gv11b.h"
82#include "vgpu_gr_gv11b.h"
83#include "vgpu_fifo_gv11b.h"
84#include "vgpu_subctx_gv11b.h"
85#include "vgpu_tsg_gv11b.h"
86
87#include <nvgpu/hw/gv11b/hw_fuse_gv11b.h>
88#include <nvgpu/hw/gv11b/hw_fifo_gv11b.h>
89#include <nvgpu/hw/gv11b/hw_ram_gv11b.h>
90#include <nvgpu/hw/gv11b/hw_top_gv11b.h>
91#include <nvgpu/hw/gv11b/hw_pwr_gv11b.h>
92
93static const struct gpu_ops vgpu_gv11b_ops = {
94 .ltc = {
95 .determine_L2_size_bytes = vgpu_determine_L2_size_bytes,
96 .set_zbc_s_entry = gv11b_ltc_set_zbc_stencil_entry,
97 .set_zbc_color_entry = gm20b_ltc_set_zbc_color_entry,
98 .set_zbc_depth_entry = gm20b_ltc_set_zbc_depth_entry,
99 .init_cbc = NULL,
100 .init_fs_state = vgpu_ltc_init_fs_state,
101 .init_comptags = vgpu_ltc_init_comptags,
102 .cbc_ctrl = NULL,
103 .isr = gv11b_ltc_isr,
104 .flush = gm20b_flush_ltc,
105 .set_enabled = gp10b_ltc_set_enabled,
106 },
107 .ce2 = {
108 .isr_stall = gv11b_ce_isr,
109 .isr_nonstall = gp10b_ce_nonstall_isr,
110 .get_num_pce = vgpu_ce_get_num_pce,
111 },
112 .gr = {
113 .init_gpc_mmu = gr_gv11b_init_gpc_mmu,
114 .bundle_cb_defaults = gr_gv11b_bundle_cb_defaults,
115 .cb_size_default = gr_gv11b_cb_size_default,
116 .calc_global_ctx_buffer_size =
117 gr_gv11b_calc_global_ctx_buffer_size,
118 .commit_global_attrib_cb = gr_gv11b_commit_global_attrib_cb,
119 .commit_global_bundle_cb = gr_gp10b_commit_global_bundle_cb,
120 .commit_global_cb_manager = gr_gp10b_commit_global_cb_manager,
121 .commit_global_pagepool = gr_gp10b_commit_global_pagepool,
122 .handle_sw_method = gr_gv11b_handle_sw_method,
123 .set_alpha_circular_buffer_size =
124 gr_gv11b_set_alpha_circular_buffer_size,
125 .set_circular_buffer_size = gr_gv11b_set_circular_buffer_size,
126 .enable_hww_exceptions = gr_gv11b_enable_hww_exceptions,
127 .is_valid_class = gr_gv11b_is_valid_class,
128 .is_valid_gfx_class = gr_gv11b_is_valid_gfx_class,
129 .is_valid_compute_class = gr_gv11b_is_valid_compute_class,
130 .get_sm_dsm_perf_regs = gv11b_gr_get_sm_dsm_perf_regs,
131 .get_sm_dsm_perf_ctrl_regs = gv11b_gr_get_sm_dsm_perf_ctrl_regs,
132 .init_fs_state = vgpu_gr_init_fs_state,
133 .set_hww_esr_report_mask = gv11b_gr_set_hww_esr_report_mask,
134 .falcon_load_ucode = gr_gm20b_load_ctxsw_ucode_segments,
135 .load_ctxsw_ucode = gr_gk20a_load_ctxsw_ucode,
136 .set_gpc_tpc_mask = gr_gv11b_set_gpc_tpc_mask,
137 .get_gpc_tpc_mask = vgpu_gr_get_gpc_tpc_mask,
138 .alloc_obj_ctx = vgpu_gr_alloc_obj_ctx,
139 .bind_ctxsw_zcull = vgpu_gr_bind_ctxsw_zcull,
140 .get_zcull_info = vgpu_gr_get_zcull_info,
141 .is_tpc_addr = gr_gm20b_is_tpc_addr,
142 .get_tpc_num = gr_gm20b_get_tpc_num,
143 .detect_sm_arch = vgpu_gr_detect_sm_arch,
144 .add_zbc_color = gr_gp10b_add_zbc_color,
145 .add_zbc_depth = gr_gp10b_add_zbc_depth,
146 .zbc_set_table = vgpu_gr_add_zbc,
147 .zbc_query_table = vgpu_gr_query_zbc,
148 .pmu_save_zbc = gk20a_pmu_save_zbc,
149 .add_zbc = gr_gk20a_add_zbc,
150 .pagepool_default_size = gr_gv11b_pagepool_default_size,
151 .init_ctx_state = vgpu_gr_gp10b_init_ctx_state,
152 .alloc_gr_ctx = vgpu_gr_gp10b_alloc_gr_ctx,
153 .free_gr_ctx = vgpu_gr_free_gr_ctx,
154 .update_ctxsw_preemption_mode =
155 gr_gp10b_update_ctxsw_preemption_mode,
156 .dump_gr_regs = NULL,
157 .update_pc_sampling = gr_gm20b_update_pc_sampling,
158 .get_fbp_en_mask = vgpu_gr_get_fbp_en_mask,
159 .get_max_ltc_per_fbp = vgpu_gr_get_max_ltc_per_fbp,
160 .get_max_lts_per_ltc = vgpu_gr_get_max_lts_per_ltc,
161 .get_rop_l2_en_mask = vgpu_gr_rop_l2_en_mask,
162 .get_max_fbps_count = vgpu_gr_get_max_fbps_count,
163 .init_sm_dsm_reg_info = gv11b_gr_init_sm_dsm_reg_info,
164 .wait_empty = gr_gv11b_wait_empty,
165 .init_cyclestats = vgpu_gr_gm20b_init_cyclestats,
166 .set_sm_debug_mode = vgpu_gr_set_sm_debug_mode,
167 .enable_cde_in_fecs = gr_gm20b_enable_cde_in_fecs,
168 .bpt_reg_info = gv11b_gr_bpt_reg_info,
169 .get_access_map = gr_gv11b_get_access_map,
170 .handle_fecs_error = gr_gv11b_handle_fecs_error,
171 .handle_sm_exception = gr_gk20a_handle_sm_exception,
172 .handle_tex_exception = gr_gv11b_handle_tex_exception,
173 .enable_gpc_exceptions = gr_gv11b_enable_gpc_exceptions,
174 .enable_exceptions = gr_gv11b_enable_exceptions,
175 .get_lrf_tex_ltc_dram_override = get_ecc_override_val,
176 .update_smpc_ctxsw_mode = vgpu_gr_update_smpc_ctxsw_mode,
177 .update_hwpm_ctxsw_mode = vgpu_gr_update_hwpm_ctxsw_mode,
178 .record_sm_error_state = gv11b_gr_record_sm_error_state,
179 .update_sm_error_state = gv11b_gr_update_sm_error_state,
180 .clear_sm_error_state = vgpu_gr_clear_sm_error_state,
181 .suspend_contexts = vgpu_gr_suspend_contexts,
182 .resume_contexts = vgpu_gr_resume_contexts,
183 .get_preemption_mode_flags = gr_gp10b_get_preemption_mode_flags,
184 .init_sm_id_table = vgpu_gr_init_sm_id_table,
185 .load_smid_config = gr_gv11b_load_smid_config,
186 .program_sm_id_numbering = gr_gv11b_program_sm_id_numbering,
187 .is_ltcs_ltss_addr = gr_gm20b_is_ltcs_ltss_addr,
188 .is_ltcn_ltss_addr = gr_gm20b_is_ltcn_ltss_addr,
189 .split_lts_broadcast_addr = gr_gm20b_split_lts_broadcast_addr,
190 .split_ltc_broadcast_addr = gr_gm20b_split_ltc_broadcast_addr,
191 .setup_rop_mapping = gr_gv11b_setup_rop_mapping,
192 .program_zcull_mapping = gr_gv11b_program_zcull_mapping,
193 .commit_global_timeslice = gr_gv11b_commit_global_timeslice,
194 .commit_inst = vgpu_gr_gv11b_commit_inst,
195 .write_zcull_ptr = gr_gv11b_write_zcull_ptr,
196 .write_pm_ptr = gr_gv11b_write_pm_ptr,
197 .init_elcg_mode = gr_gv11b_init_elcg_mode,
198 .load_tpc_mask = gr_gv11b_load_tpc_mask,
199 .inval_icache = gr_gk20a_inval_icache,
200 .trigger_suspend = gv11b_gr_sm_trigger_suspend,
201 .wait_for_pause = gr_gk20a_wait_for_pause,
202 .resume_from_pause = gv11b_gr_resume_from_pause,
203 .clear_sm_errors = gr_gk20a_clear_sm_errors,
204 .tpc_enabled_exceptions = gr_gk20a_tpc_enabled_exceptions,
205 .get_esr_sm_sel = gv11b_gr_get_esr_sm_sel,
206 .sm_debugger_attached = gv11b_gr_sm_debugger_attached,
207 .suspend_single_sm = gv11b_gr_suspend_single_sm,
208 .suspend_all_sms = gv11b_gr_suspend_all_sms,
209 .resume_single_sm = gv11b_gr_resume_single_sm,
210 .resume_all_sms = gv11b_gr_resume_all_sms,
211 .get_sm_hww_warp_esr = gv11b_gr_get_sm_hww_warp_esr,
212 .get_sm_hww_global_esr = gv11b_gr_get_sm_hww_global_esr,
213 .get_sm_no_lock_down_hww_global_esr_mask =
214 gv11b_gr_get_sm_no_lock_down_hww_global_esr_mask,
215 .lock_down_sm = gv11b_gr_lock_down_sm,
216 .wait_for_sm_lock_down = gv11b_gr_wait_for_sm_lock_down,
217 .clear_sm_hww = gv11b_gr_clear_sm_hww,
218 .init_ovr_sm_dsm_perf = gv11b_gr_init_ovr_sm_dsm_perf,
219 .get_ovr_perf_regs = gv11b_gr_get_ovr_perf_regs,
220 .disable_rd_coalesce = gm20a_gr_disable_rd_coalesce,
221 .set_boosted_ctx = NULL,
222 .set_preemption_mode = vgpu_gr_gp10b_set_preemption_mode,
223 .set_czf_bypass = NULL,
224 .pre_process_sm_exception = gr_gv11b_pre_process_sm_exception,
225 .set_preemption_buffer_va = gr_gv11b_set_preemption_buffer_va,
226 .init_preemption_state = NULL,
227 .update_boosted_ctx = NULL,
228 .set_bes_crop_debug3 = gr_gp10b_set_bes_crop_debug3,
229 .set_bes_crop_debug4 = gr_gp10b_set_bes_crop_debug4,
230 .create_gr_sysfs = gr_gv11b_create_sysfs,
231 .set_ctxsw_preemption_mode = vgpu_gr_gp10b_set_ctxsw_preemption_mode,
232 .is_etpc_addr = gv11b_gr_pri_is_etpc_addr,
233 .egpc_etpc_priv_addr_table = gv11b_gr_egpc_etpc_priv_addr_table,
234 .handle_tpc_mpc_exception = gr_gv11b_handle_tpc_mpc_exception,
235 .zbc_s_query_table = gr_gv11b_zbc_s_query_table,
236 .load_zbc_s_default_tbl = gr_gv11b_load_stencil_default_tbl,
237 .handle_gpc_gpcmmu_exception =
238 gr_gv11b_handle_gpc_gpcmmu_exception,
239 .add_zbc_type_s = gr_gv11b_add_zbc_type_s,
240 .get_egpc_base = gv11b_gr_get_egpc_base,
241 .get_egpc_etpc_num = gv11b_gr_get_egpc_etpc_num,
242 .handle_gpc_gpccs_exception =
243 gr_gv11b_handle_gpc_gpccs_exception,
244 .load_zbc_s_tbl = gr_gv11b_load_stencil_tbl,
245 .access_smpc_reg = gv11b_gr_access_smpc_reg,
246 .is_egpc_addr = gv11b_gr_pri_is_egpc_addr,
247 .add_zbc_s = gr_gv11b_add_zbc_stencil,
248 .handle_gcc_exception = gr_gv11b_handle_gcc_exception,
249 .init_sw_veid_bundle = gr_gv11b_init_sw_veid_bundle,
250 .handle_tpc_sm_ecc_exception =
251 gr_gv11b_handle_tpc_sm_ecc_exception,
252 .decode_egpc_addr = gv11b_gr_decode_egpc_addr,
253 .init_ctxsw_hdr_data = gr_gp10b_init_ctxsw_hdr_data,
254 .init_gfxp_wfi_timeout_count =
255 gr_gv11b_init_gfxp_wfi_timeout_count,
256 .get_max_gfxp_wfi_timeout_count =
257 gr_gv11b_get_max_gfxp_wfi_timeout_count,
258 },
259 .fb = {
260 .reset = gv11b_fb_reset,
261 .init_hw = gk20a_fb_init_hw,
262 .init_fs_state = gv11b_fb_init_fs_state,
263 .init_cbc = gv11b_fb_init_cbc,
264 .set_mmu_page_size = gm20b_fb_set_mmu_page_size,
265 .set_use_full_comp_tag_line =
266 gm20b_fb_set_use_full_comp_tag_line,
267 .compression_page_size = gp10b_fb_compression_page_size,
268 .compressible_page_size = gp10b_fb_compressible_page_size,
269 .compression_align_mask = gm20b_fb_compression_align_mask,
270 .vpr_info_fetch = gm20b_fb_vpr_info_fetch,
271 .dump_vpr_wpr_info = gm20b_fb_dump_vpr_wpr_info,
272 .read_wpr_info = gm20b_fb_read_wpr_info,
273 .is_debug_mode_enabled = NULL,
274 .set_debug_mode = vgpu_mm_mmu_set_debug_mode,
275 .tlb_invalidate = vgpu_mm_tlb_invalidate,
276 .hub_isr = gv11b_fb_hub_isr,
277 },
278 .clock_gating = {
279 .slcg_bus_load_gating_prod =
280 gv11b_slcg_bus_load_gating_prod,
281 .slcg_ce2_load_gating_prod =
282 gv11b_slcg_ce2_load_gating_prod,
283 .slcg_chiplet_load_gating_prod =
284 gv11b_slcg_chiplet_load_gating_prod,
285 .slcg_ctxsw_firmware_load_gating_prod =
286 gv11b_slcg_ctxsw_firmware_load_gating_prod,
287 .slcg_fb_load_gating_prod =
288 gv11b_slcg_fb_load_gating_prod,
289 .slcg_fifo_load_gating_prod =
290 gv11b_slcg_fifo_load_gating_prod,
291 .slcg_gr_load_gating_prod =
292 gr_gv11b_slcg_gr_load_gating_prod,
293 .slcg_ltc_load_gating_prod =
294 ltc_gv11b_slcg_ltc_load_gating_prod,
295 .slcg_perf_load_gating_prod =
296 gv11b_slcg_perf_load_gating_prod,
297 .slcg_priring_load_gating_prod =
298 gv11b_slcg_priring_load_gating_prod,
299 .slcg_pmu_load_gating_prod =
300 gv11b_slcg_pmu_load_gating_prod,
301 .slcg_therm_load_gating_prod =
302 gv11b_slcg_therm_load_gating_prod,
303 .slcg_xbar_load_gating_prod =
304 gv11b_slcg_xbar_load_gating_prod,
305 .blcg_bus_load_gating_prod =
306 gv11b_blcg_bus_load_gating_prod,
307 .blcg_ce_load_gating_prod =
308 gv11b_blcg_ce_load_gating_prod,
309 .blcg_ctxsw_firmware_load_gating_prod =
310 gv11b_blcg_ctxsw_firmware_load_gating_prod,
311 .blcg_fb_load_gating_prod =
312 gv11b_blcg_fb_load_gating_prod,
313 .blcg_fifo_load_gating_prod =
314 gv11b_blcg_fifo_load_gating_prod,
315 .blcg_gr_load_gating_prod =
316 gv11b_blcg_gr_load_gating_prod,
317 .blcg_ltc_load_gating_prod =
318 gv11b_blcg_ltc_load_gating_prod,
319 .blcg_pwr_csb_load_gating_prod =
320 gv11b_blcg_pwr_csb_load_gating_prod,
321 .blcg_pmu_load_gating_prod =
322 gv11b_blcg_pmu_load_gating_prod,
323 .blcg_xbar_load_gating_prod =
324 gv11b_blcg_xbar_load_gating_prod,
325 .pg_gr_load_gating_prod =
326 gr_gv11b_pg_gr_load_gating_prod,
327 },
328 .fifo = {
329 .init_fifo_setup_hw = vgpu_gv11b_init_fifo_setup_hw,
330 .bind_channel = vgpu_channel_bind,
331 .unbind_channel = vgpu_channel_unbind,
332 .disable_channel = vgpu_channel_disable,
333 .enable_channel = vgpu_channel_enable,
334 .alloc_inst = vgpu_channel_alloc_inst,
335 .free_inst = vgpu_channel_free_inst,
336 .setup_ramfc = vgpu_channel_setup_ramfc,
337 .default_timeslice_us = vgpu_fifo_default_timeslice_us,
338 .setup_userd = gk20a_fifo_setup_userd,
339 .userd_gp_get = gv11b_userd_gp_get,
340 .userd_gp_put = gv11b_userd_gp_put,
341 .userd_pb_get = gv11b_userd_pb_get,
342 .pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val,
343 .preempt_channel = vgpu_fifo_preempt_channel,
344 .preempt_tsg = vgpu_fifo_preempt_tsg,
345 .enable_tsg = vgpu_enable_tsg,
346 .disable_tsg = gk20a_disable_tsg,
347 .tsg_verify_channel_status = NULL,
348 .tsg_verify_status_ctx_reload = NULL,
349 /* TODO: implement it for CE fault */
350 .tsg_verify_status_faulted = NULL,
351 .update_runlist = vgpu_fifo_update_runlist,
352 .trigger_mmu_fault = NULL,
353 .get_mmu_fault_info = NULL,
354 .wait_engine_idle = vgpu_fifo_wait_engine_idle,
355 .get_num_fifos = gv11b_fifo_get_num_fifos,
356 .get_pbdma_signature = gp10b_fifo_get_pbdma_signature,
357 .set_runlist_interleave = vgpu_fifo_set_runlist_interleave,
358 .tsg_set_timeslice = vgpu_tsg_set_timeslice,
359 .tsg_open = vgpu_tsg_open,
360 .tsg_release = vgpu_tsg_release,
361 .force_reset_ch = vgpu_fifo_force_reset_ch,
362 .engine_enum_from_type = gp10b_fifo_engine_enum_from_type,
363 .device_info_data_parse = gp10b_device_info_data_parse,
364 .eng_runlist_base_size = fifo_eng_runlist_base__size_1_v,
365 .init_engine_info = vgpu_fifo_init_engine_info,
366 .runlist_entry_size = ram_rl_entry_size_v,
367 .get_tsg_runlist_entry = gv11b_get_tsg_runlist_entry,
368 .get_ch_runlist_entry = gv11b_get_ch_runlist_entry,
369 .is_fault_engine_subid_gpc = gv11b_is_fault_engine_subid_gpc,
370 .dump_pbdma_status = gk20a_dump_pbdma_status,
371 .dump_eng_status = gv11b_dump_eng_status,
372 .dump_channel_status_ramfc = gv11b_dump_channel_status_ramfc,
373 .intr_0_error_mask = gv11b_fifo_intr_0_error_mask,
374 .is_preempt_pending = gv11b_fifo_is_preempt_pending,
375 .init_pbdma_intr_descs = gv11b_fifo_init_pbdma_intr_descs,
376 .reset_enable_hw = gv11b_init_fifo_reset_enable_hw,
377 .teardown_ch_tsg = gv11b_fifo_teardown_ch_tsg,
378 .handle_sched_error = gv11b_fifo_handle_sched_error,
379 .handle_pbdma_intr_0 = gv11b_fifo_handle_pbdma_intr_0,
380 .handle_pbdma_intr_1 = gv11b_fifo_handle_pbdma_intr_1,
381 .init_eng_method_buffers = gv11b_fifo_init_eng_method_buffers,
382 .deinit_eng_method_buffers =
383 gv11b_fifo_deinit_eng_method_buffers,
384 .tsg_bind_channel = vgpu_gv11b_tsg_bind_channel,
385 .tsg_unbind_channel = vgpu_tsg_unbind_channel,
386#ifdef CONFIG_TEGRA_GK20A_NVHOST
387 .alloc_syncpt_buf = vgpu_gv11b_fifo_alloc_syncpt_buf,
388 .free_syncpt_buf = gv11b_fifo_free_syncpt_buf,
389 .add_syncpt_wait_cmd = gv11b_fifo_add_syncpt_wait_cmd,
390 .get_syncpt_wait_cmd_size = gv11b_fifo_get_syncpt_wait_cmd_size,
391 .add_syncpt_incr_cmd = gv11b_fifo_add_syncpt_incr_cmd,
392 .get_syncpt_incr_cmd_size = gv11b_fifo_get_syncpt_incr_cmd_size,
393 .get_sync_ro_map = vgpu_gv11b_fifo_get_sync_ro_map,
394#endif
395 .resetup_ramfc = NULL,
396 .reschedule_runlist = NULL,
397 .device_info_fault_id = top_device_info_data_fault_id_enum_v,
398 .free_channel_ctx_header = vgpu_gv11b_free_subctx_header,
399 .preempt_ch_tsg = gv11b_fifo_preempt_ch_tsg,
400 .handle_ctxsw_timeout = gv11b_fifo_handle_ctxsw_timeout,
401 },
402 .gr_ctx = {
403 .get_netlist_name = gr_gv11b_get_netlist_name,
404 .is_fw_defined = gr_gv11b_is_firmware_defined,
405 },
406#ifdef CONFIG_GK20A_CTXSW_TRACE
407 .fecs_trace = {
408 .alloc_user_buffer = NULL,
409 .free_user_buffer = NULL,
410 .mmap_user_buffer = NULL,
411 .init = NULL,
412 .deinit = NULL,
413 .enable = NULL,
414 .disable = NULL,
415 .is_enabled = NULL,
416 .reset = NULL,
417 .flush = NULL,
418 .poll = NULL,
419 .bind_channel = NULL,
420 .unbind_channel = NULL,
421 .max_entries = NULL,
422 },
423#endif /* CONFIG_GK20A_CTXSW_TRACE */
424 .mm = {
425 /* FIXME: add support for sparse mappings */
426 .support_sparse = NULL,
427 .gmmu_map = vgpu_gp10b_locked_gmmu_map,
428 .gmmu_unmap = vgpu_locked_gmmu_unmap,
429 .vm_bind_channel = vgpu_vm_bind_channel,
430 .fb_flush = vgpu_mm_fb_flush,
431 .l2_invalidate = vgpu_mm_l2_invalidate,
432 .l2_flush = vgpu_mm_l2_flush,
433 .cbc_clean = gk20a_mm_cbc_clean,
434 .set_big_page_size = gm20b_mm_set_big_page_size,
435 .get_big_page_sizes = gm20b_mm_get_big_page_sizes,
436 .get_default_big_page_size = gp10b_mm_get_default_big_page_size,
437 .gpu_phys_addr = gm20b_gpu_phys_addr,
438 .get_iommu_bit = gk20a_mm_get_iommu_bit,
439 .get_mmu_levels = gp10b_mm_get_mmu_levels,
440 .init_pdb = gp10b_mm_init_pdb,
441 .init_mm_setup_hw = vgpu_gp10b_init_mm_setup_hw,
442 .is_bar1_supported = gv11b_mm_is_bar1_supported,
443 .init_inst_block = gv11b_init_inst_block,
444 .mmu_fault_pending = gv11b_mm_mmu_fault_pending,
445 .get_kind_invalid = gm20b_get_kind_invalid,
446 .get_kind_pitch = gm20b_get_kind_pitch,
447 .init_bar2_vm = gp10b_init_bar2_vm,
448 .init_bar2_mm_hw_setup = gv11b_init_bar2_mm_hw_setup,
449 .remove_bar2_vm = gv11b_mm_remove_bar2_vm,
450 .fault_info_mem_destroy = gv11b_mm_fault_info_mem_destroy,
451 },
452 .therm = {
453 .init_therm_setup_hw = gp10b_init_therm_setup_hw,
454 .elcg_init_idle_filters = gv11b_elcg_init_idle_filters,
455 },
456 .pmu = {
457 .pmu_setup_elpg = gp10b_pmu_setup_elpg,
458 .pmu_get_queue_head = pwr_pmu_queue_head_r,
459 .pmu_get_queue_head_size = pwr_pmu_queue_head__size_1_v,
460 .pmu_get_queue_tail = pwr_pmu_queue_tail_r,
461 .pmu_get_queue_tail_size = pwr_pmu_queue_tail__size_1_v,
462 .pmu_queue_head = gk20a_pmu_queue_head,
463 .pmu_queue_tail = gk20a_pmu_queue_tail,
464 .pmu_msgq_tail = gk20a_pmu_msgq_tail,
465 .pmu_mutex_size = pwr_pmu_mutex__size_1_v,
466 .pmu_mutex_acquire = gk20a_pmu_mutex_acquire,
467 .pmu_mutex_release = gk20a_pmu_mutex_release,
468 .write_dmatrfbase = gp10b_write_dmatrfbase,
469 .pmu_elpg_statistics = gp106_pmu_elpg_statistics,
470 .pmu_init_perfmon = nvgpu_pmu_init_perfmon_rpc,
471 .pmu_perfmon_start_sampling = nvgpu_pmu_perfmon_start_sampling_rpc,
472 .pmu_perfmon_stop_sampling = nvgpu_pmu_perfmon_stop_sampling_rpc,
473 .pmu_perfmon_get_samples_rpc = nvgpu_pmu_perfmon_get_samples_rpc,
474 .pmu_pg_init_param = gv11b_pg_gr_init,
475 .pmu_pg_supported_engines_list = gk20a_pmu_pg_engines_list,
476 .pmu_pg_engines_feature_list = gk20a_pmu_pg_feature_list,
477 .dump_secure_fuses = pmu_dump_security_fuses_gp10b,
478 .reset_engine = gp106_pmu_engine_reset,
479 .is_engine_in_reset = gp106_pmu_is_engine_in_reset,
480 .pmu_nsbootstrap = gv11b_pmu_bootstrap,
481 .pmu_pg_set_sub_feature_mask = gv11b_pg_set_subfeature_mask,
482 .is_pmu_supported = gv11b_is_pmu_supported,
483 },
484 .regops = {
485 .get_global_whitelist_ranges =
486 gv11b_get_global_whitelist_ranges,
487 .get_global_whitelist_ranges_count =
488 gv11b_get_global_whitelist_ranges_count,
489 .get_context_whitelist_ranges =
490 gv11b_get_context_whitelist_ranges,
491 .get_context_whitelist_ranges_count =
492 gv11b_get_context_whitelist_ranges_count,
493 .get_runcontrol_whitelist = gv11b_get_runcontrol_whitelist,
494 .get_runcontrol_whitelist_count =
495 gv11b_get_runcontrol_whitelist_count,
496 .get_runcontrol_whitelist_ranges =
497 gv11b_get_runcontrol_whitelist_ranges,
498 .get_runcontrol_whitelist_ranges_count =
499 gv11b_get_runcontrol_whitelist_ranges_count,
500 .get_qctl_whitelist = gv11b_get_qctl_whitelist,
501 .get_qctl_whitelist_count = gv11b_get_qctl_whitelist_count,
502 .get_qctl_whitelist_ranges = gv11b_get_qctl_whitelist_ranges,
503 .get_qctl_whitelist_ranges_count =
504 gv11b_get_qctl_whitelist_ranges_count,
505 .apply_smpc_war = gv11b_apply_smpc_war,
506 },
507 .mc = {
508 .intr_enable = mc_gv11b_intr_enable,
509 .intr_unit_config = mc_gp10b_intr_unit_config,
510 .isr_stall = mc_gp10b_isr_stall,
511 .intr_stall = mc_gp10b_intr_stall,
512 .intr_stall_pause = mc_gp10b_intr_stall_pause,
513 .intr_stall_resume = mc_gp10b_intr_stall_resume,
514 .intr_nonstall = mc_gp10b_intr_nonstall,
515 .intr_nonstall_pause = mc_gp10b_intr_nonstall_pause,
516 .intr_nonstall_resume = mc_gp10b_intr_nonstall_resume,
517 .enable = gk20a_mc_enable,
518 .disable = gk20a_mc_disable,
519 .reset = gk20a_mc_reset,
520 .boot_0 = gk20a_mc_boot_0,
521 .is_intr1_pending = mc_gp10b_is_intr1_pending,
522 .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending,
523 },
524 .debug = {
525 .show_dump = NULL,
526 },
527 .dbg_session_ops = {
528 .exec_reg_ops = vgpu_exec_regops,
529 .dbg_set_powergate = vgpu_dbg_set_powergate,
530 .check_and_set_global_reservation =
531 vgpu_check_and_set_global_reservation,
532 .check_and_set_context_reservation =
533 vgpu_check_and_set_context_reservation,
534 .release_profiler_reservation =
535 vgpu_release_profiler_reservation,
536 .perfbuffer_enable = vgpu_perfbuffer_enable,
537 .perfbuffer_disable = vgpu_perfbuffer_disable,
538 },
539 .bus = {
540 .init_hw = gk20a_bus_init_hw,
541 .isr = gk20a_bus_isr,
542 .read_ptimer = vgpu_read_ptimer,
543 .get_timestamps_zipper = vgpu_get_timestamps_zipper,
544 .bar1_bind = NULL,
545 },
546#if defined(CONFIG_GK20A_CYCLE_STATS)
547 .css = {
548 .enable_snapshot = vgpu_css_enable_snapshot_buffer,
549 .disable_snapshot = vgpu_css_release_snapshot_buffer,
550 .check_data_available = vgpu_css_flush_snapshots,
551 .detach_snapshot = vgpu_css_detach,
552 .set_handled_snapshots = NULL,
553 .allocate_perfmon_ids = NULL,
554 .release_perfmon_ids = NULL,
555 },
556#endif
557 .falcon = {
558 .falcon_hal_sw_init = gk20a_falcon_hal_sw_init,
559 },
560 .priv_ring = {
561 .isr = gp10b_priv_ring_isr,
562 },
563 .chip_init_gpu_characteristics = vgpu_gv11b_init_gpu_characteristics,
564 .get_litter_value = gv11b_get_litter_value,
565};
566
567int vgpu_gv11b_init_hal(struct gk20a *g)
568{
569 struct gpu_ops *gops = &g->ops;
570
571 gops->ltc = vgpu_gv11b_ops.ltc;
572 gops->ce2 = vgpu_gv11b_ops.ce2;
573 gops->gr = vgpu_gv11b_ops.gr;
574 gops->fb = vgpu_gv11b_ops.fb;
575 gops->clock_gating = vgpu_gv11b_ops.clock_gating;
576 gops->fifo = vgpu_gv11b_ops.fifo;
577 gops->gr_ctx = vgpu_gv11b_ops.gr_ctx;
578 gops->mm = vgpu_gv11b_ops.mm;
579#ifdef CONFIG_GK20A_CTXSW_TRACE
580 gops->fecs_trace = vgpu_gv11b_ops.fecs_trace;
581#endif
582 gops->therm = vgpu_gv11b_ops.therm;
583 gops->pmu = vgpu_gv11b_ops.pmu;
584 gops->regops = vgpu_gv11b_ops.regops;
585 gops->mc = vgpu_gv11b_ops.mc;
586 gops->debug = vgpu_gv11b_ops.debug;
587 gops->dbg_session_ops = vgpu_gv11b_ops.dbg_session_ops;
588 gops->bus = vgpu_gv11b_ops.bus;
589#if defined(CONFIG_GK20A_CYCLE_STATS)
590 gops->css = vgpu_gv11b_ops.css;
591#endif
592 gops->falcon = vgpu_gv11b_ops.falcon;
593 gops->priv_ring = vgpu_gv11b_ops.priv_ring;
594
595 /* Lone functions */
596 gops->chip_init_gpu_characteristics =
597 vgpu_gv11b_ops.chip_init_gpu_characteristics;
598 gops->get_litter_value = vgpu_gv11b_ops.get_litter_value;
599
600 g->name = "gv11b";
601
602 return 0;
603}
diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_subctx_gv11b.c b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_subctx_gv11b.c
new file mode 100644
index 00000000..ba92c8d5
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_subctx_gv11b.c
@@ -0,0 +1,80 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include "gk20a/gk20a.h"
24
25#include <nvgpu/vgpu/vgpu.h>
26#include <nvgpu/vgpu/tegra_vgpu.h>
27#include <nvgpu/hw/gv11b/hw_ctxsw_prog_gv11b.h>
28
29int vgpu_gv11b_alloc_subctx_header(struct channel_gk20a *c)
30{
31 struct ctx_header_desc *ctx = &c->ctx_header;
32 struct tegra_vgpu_cmd_msg msg = {};
33 struct tegra_vgpu_alloc_ctx_header_params *p =
34 &msg.params.alloc_ctx_header;
35 int err;
36
37 msg.cmd = TEGRA_VGPU_CMD_ALLOC_CTX_HEADER;
38 msg.handle = vgpu_get_handle(c->g);
39 p->ch_handle = c->virt_ctx;
40 p->ctx_header_va = __nvgpu_vm_alloc_va(c->vm,
41 ctxsw_prog_fecs_header_v(),
42 gmmu_page_size_kernel);
43 if (!p->ctx_header_va) {
44 nvgpu_err(c->g, "alloc va failed for ctx_header");
45 return -ENOMEM;
46 }
47 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
48 err = err ? err : msg.ret;
49 if (unlikely(err)) {
50 nvgpu_err(c->g, "alloc ctx_header failed err %d", err);
51 __nvgpu_vm_free_va(c->vm, p->ctx_header_va,
52 gmmu_page_size_kernel);
53 return err;
54 }
55 ctx->mem.gpu_va = p->ctx_header_va;
56
57 return err;
58}
59
60void vgpu_gv11b_free_subctx_header(struct channel_gk20a *c)
61{
62 struct ctx_header_desc *ctx = &c->ctx_header;
63 struct tegra_vgpu_cmd_msg msg = {};
64 struct tegra_vgpu_free_ctx_header_params *p =
65 &msg.params.free_ctx_header;
66 int err;
67
68 if (ctx->mem.gpu_va) {
69 msg.cmd = TEGRA_VGPU_CMD_FREE_CTX_HEADER;
70 msg.handle = vgpu_get_handle(c->g);
71 p->ch_handle = c->virt_ctx;
72 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
73 err = err ? err : msg.ret;
74 if (unlikely(err))
75 nvgpu_err(c->g, "free ctx_header failed err %d", err);
76 __nvgpu_vm_free_va(c->vm, ctx->mem.gpu_va,
77 gmmu_page_size_kernel);
78 ctx->mem.gpu_va = 0;
79 }
80}
diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_subctx_gv11b.h b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_subctx_gv11b.h
new file mode 100644
index 00000000..4bd0c2de
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_subctx_gv11b.h
@@ -0,0 +1,31 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _VGPU_SUBCTX_GV11B_H_
24#define _VGPU_SUBCTX_GV11B_H_
25
26struct channel_gk20a;
27
28int vgpu_gv11b_alloc_subctx_header(struct channel_gk20a *c);
29void vgpu_gv11b_free_subctx_header(struct channel_gk20a *c);
30
31#endif
diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_tsg_gv11b.c b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_tsg_gv11b.c
new file mode 100644
index 00000000..b249b5af
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_tsg_gv11b.c
@@ -0,0 +1,59 @@
1/*
2 * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/vgpu/tegra_vgpu.h>
24#include <nvgpu/vgpu/vgpu.h>
25
26#include "gk20a/gk20a.h"
27#include "vgpu_tsg_gv11b.h"
28
29int vgpu_gv11b_tsg_bind_channel(struct tsg_gk20a *tsg,
30 struct channel_gk20a *ch)
31{
32 struct tegra_vgpu_cmd_msg msg = {};
33 struct tegra_vgpu_tsg_bind_channel_ex_params *p =
34 &msg.params.tsg_bind_channel_ex;
35 int err;
36
37 gk20a_dbg_fn("");
38
39 err = gk20a_tsg_bind_channel(tsg, ch);
40 if (err)
41 return err;
42
43 msg.cmd = TEGRA_VGPU_CMD_TSG_BIND_CHANNEL_EX;
44 msg.handle = vgpu_get_handle(tsg->g);
45 p->tsg_id = tsg->tsgid;
46 p->ch_handle = ch->virt_ctx;
47 p->subctx_id = ch->subctx_id;
48 p->runqueue_sel = ch->runqueue_sel;
49 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
50 err = err ? err : msg.ret;
51 if (err) {
52 nvgpu_err(tsg->g,
53 "vgpu_gv11b_tsg_bind_channel failed, ch %d tsgid %d",
54 ch->chid, tsg->tsgid);
55 gk20a_tsg_unbind_channel(ch);
56 }
57
58 return err;
59}
diff --git a/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_tsg_gv11b.h b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_tsg_gv11b.h
new file mode 100644
index 00000000..9ce84170
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/gv11b/vgpu_tsg_gv11b.h
@@ -0,0 +1,29 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _VGPU_TSG_GV11B_H_
24#define _VGPU_TSG_GV11B_H_
25
26int vgpu_gv11b_tsg_bind_channel(struct tsg_gk20a *tsg,
27 struct channel_gk20a *ch);
28
29#endif
diff --git a/drivers/gpu/nvgpu/vgpu/ltc_vgpu.c b/drivers/gpu/nvgpu/vgpu/ltc_vgpu.c
new file mode 100644
index 00000000..7e86ba38
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/ltc_vgpu.c
@@ -0,0 +1,69 @@
1/*
2 * Virtualized GPU L2
3 *
4 * Copyright (c) 2014-2018 NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <nvgpu/vgpu/vgpu.h>
26
27#include "gk20a/gk20a.h"
28#include "ltc_vgpu.h"
29
30int vgpu_determine_L2_size_bytes(struct gk20a *g)
31{
32 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
33
34 gk20a_dbg_fn("");
35
36 return priv->constants.l2_size;
37}
38
39int vgpu_ltc_init_comptags(struct gk20a *g, struct gr_gk20a *gr)
40{
41 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
42 u32 max_comptag_lines = 0;
43 int err;
44
45 gk20a_dbg_fn("");
46
47 gr->cacheline_size = priv->constants.cacheline_size;
48 gr->comptags_per_cacheline = priv->constants.comptags_per_cacheline;
49 gr->slices_per_ltc = priv->constants.slices_per_ltc;
50 max_comptag_lines = priv->constants.comptag_lines;
51
52 if (max_comptag_lines < 2)
53 return -ENXIO;
54
55 err = gk20a_comptag_allocator_init(g, &gr->comp_tags, max_comptag_lines);
56 if (err)
57 return err;
58
59 return 0;
60}
61
62void vgpu_ltc_init_fs_state(struct gk20a *g)
63{
64 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
65
66 gk20a_dbg_fn("");
67
68 g->ltc_count = priv->constants.ltc_count;
69}
diff --git a/drivers/gpu/nvgpu/vgpu/ltc_vgpu.h b/drivers/gpu/nvgpu/vgpu/ltc_vgpu.h
new file mode 100644
index 00000000..adc76171
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/ltc_vgpu.h
@@ -0,0 +1,33 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _LTC_VGPU_H_
24#define _LTC_VGPU_H_
25
26struct gk20a;
27struct gr_gk20a;
28
29int vgpu_determine_L2_size_bytes(struct gk20a *g);
30int vgpu_ltc_init_comptags(struct gk20a *g, struct gr_gk20a *gr);
31void vgpu_ltc_init_fs_state(struct gk20a *g);
32
33#endif
diff --git a/drivers/gpu/nvgpu/vgpu/mm_vgpu.c b/drivers/gpu/nvgpu/vgpu/mm_vgpu.c
new file mode 100644
index 00000000..21496906
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/mm_vgpu.c
@@ -0,0 +1,278 @@
1/*
2 * Virtualized GPU Memory Management
3 *
4 * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <nvgpu/kmem.h>
26#include <nvgpu/dma.h>
27#include <nvgpu/bug.h>
28#include <nvgpu/vm.h>
29#include <nvgpu/vm_area.h>
30
31#include <nvgpu/vgpu/vm.h>
32#include <nvgpu/vgpu/vgpu.h>
33
34#include <nvgpu/linux/vm.h>
35#include <nvgpu/linux/nvgpu_mem.h>
36
37#include "mm_vgpu.h"
38#include "gk20a/gk20a.h"
39#include "gk20a/mm_gk20a.h"
40#include "gm20b/mm_gm20b.h"
41
42static int vgpu_init_mm_setup_sw(struct gk20a *g)
43{
44 struct mm_gk20a *mm = &g->mm;
45
46 gk20a_dbg_fn("");
47
48 if (mm->sw_ready) {
49 gk20a_dbg_fn("skip init");
50 return 0;
51 }
52
53 nvgpu_mutex_init(&mm->tlb_lock);
54 nvgpu_mutex_init(&mm->priv_lock);
55
56 mm->g = g;
57
58 /*TBD: make channel vm size configurable */
59 mm->channel.user_size = NV_MM_DEFAULT_USER_SIZE;
60 mm->channel.kernel_size = NV_MM_DEFAULT_KERNEL_SIZE;
61
62 gk20a_dbg_info("channel vm size: user %dMB kernel %dMB",
63 (int)(mm->channel.user_size >> 20),
64 (int)(mm->channel.kernel_size >> 20));
65
66 mm->sw_ready = true;
67
68 return 0;
69}
70
71int vgpu_init_mm_support(struct gk20a *g)
72{
73 int err;
74
75 gk20a_dbg_fn("");
76
77 err = vgpu_init_mm_setup_sw(g);
78 if (err)
79 return err;
80
81 if (g->ops.mm.init_mm_setup_hw)
82 err = g->ops.mm.init_mm_setup_hw(g);
83
84 return err;
85}
86
87void vgpu_locked_gmmu_unmap(struct vm_gk20a *vm,
88 u64 vaddr,
89 u64 size,
90 int pgsz_idx,
91 bool va_allocated,
92 int rw_flag,
93 bool sparse,
94 struct vm_gk20a_mapping_batch *batch)
95{
96 struct gk20a *g = gk20a_from_vm(vm);
97 struct tegra_vgpu_cmd_msg msg;
98 struct tegra_vgpu_as_map_params *p = &msg.params.as_map;
99 int err;
100
101 gk20a_dbg_fn("");
102
103 if (va_allocated) {
104 err = __nvgpu_vm_free_va(vm, vaddr, pgsz_idx);
105 if (err) {
106 nvgpu_err(g, "failed to free va");
107 return;
108 }
109 }
110
111 msg.cmd = TEGRA_VGPU_CMD_AS_UNMAP;
112 msg.handle = vgpu_get_handle(g);
113 p->handle = vm->handle;
114 p->gpu_va = vaddr;
115 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
116 if (err || msg.ret)
117 nvgpu_err(g, "failed to update gmmu ptes on unmap");
118
119 /* TLB invalidate handled on server side */
120}
121
122/*
123 * This is called by the common VM init routine to handle vGPU specifics of
124 * intializing a VM on a vGPU. This alone is not enough to init a VM. See
125 * nvgpu_vm_init().
126 */
127int vgpu_vm_init(struct gk20a *g, struct vm_gk20a *vm)
128{
129 struct tegra_vgpu_cmd_msg msg;
130 struct tegra_vgpu_as_share_params *p = &msg.params.as_share;
131 int err;
132
133 msg.cmd = TEGRA_VGPU_CMD_AS_ALLOC_SHARE;
134 msg.handle = vgpu_get_handle(g);
135 p->size = vm->va_limit;
136 p->big_page_size = vm->big_page_size;
137
138 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
139 if (err || msg.ret)
140 return -ENOMEM;
141
142 vm->handle = p->handle;
143
144 return 0;
145}
146
147/*
148 * Similar to vgpu_vm_init() this is called as part of the cleanup path for
149 * VMs. This alone is not enough to remove a VM - see nvgpu_vm_remove().
150 */
151void vgpu_vm_remove(struct vm_gk20a *vm)
152{
153 struct gk20a *g = gk20a_from_vm(vm);
154 struct tegra_vgpu_cmd_msg msg;
155 struct tegra_vgpu_as_share_params *p = &msg.params.as_share;
156 int err;
157
158 msg.cmd = TEGRA_VGPU_CMD_AS_FREE_SHARE;
159 msg.handle = vgpu_get_handle(g);
160 p->handle = vm->handle;
161 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
162 WARN_ON(err || msg.ret);
163}
164
165u64 vgpu_bar1_map(struct gk20a *g, struct nvgpu_mem *mem)
166{
167 u64 addr = nvgpu_mem_get_addr(g, mem);
168 struct tegra_vgpu_cmd_msg msg;
169 struct tegra_vgpu_as_map_params *p = &msg.params.as_map;
170 int err;
171
172 msg.cmd = TEGRA_VGPU_CMD_MAP_BAR1;
173 msg.handle = vgpu_get_handle(g);
174 p->addr = addr;
175 p->size = mem->size;
176 p->iova = 0;
177 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
178 if (err || msg.ret)
179 addr = 0;
180 else
181 addr = p->gpu_va;
182
183 return addr;
184}
185
186int vgpu_vm_bind_channel(struct gk20a_as_share *as_share,
187 struct channel_gk20a *ch)
188{
189 struct vm_gk20a *vm = as_share->vm;
190 struct tegra_vgpu_cmd_msg msg;
191 struct tegra_vgpu_as_bind_share_params *p = &msg.params.as_bind_share;
192 int err;
193
194 gk20a_dbg_fn("");
195
196 ch->vm = vm;
197 msg.cmd = TEGRA_VGPU_CMD_AS_BIND_SHARE;
198 msg.handle = vgpu_get_handle(ch->g);
199 p->as_handle = vm->handle;
200 p->chan_handle = ch->virt_ctx;
201 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
202
203 if (err || msg.ret) {
204 ch->vm = NULL;
205 err = -ENOMEM;
206 }
207
208 if (ch->vm)
209 nvgpu_vm_get(ch->vm);
210
211 return err;
212}
213
214static void vgpu_cache_maint(u64 handle, u8 op)
215{
216 struct tegra_vgpu_cmd_msg msg;
217 struct tegra_vgpu_cache_maint_params *p = &msg.params.cache_maint;
218 int err;
219
220 msg.cmd = TEGRA_VGPU_CMD_CACHE_MAINT;
221 msg.handle = handle;
222 p->op = op;
223 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
224 WARN_ON(err || msg.ret);
225}
226
227int vgpu_mm_fb_flush(struct gk20a *g)
228{
229
230 gk20a_dbg_fn("");
231
232 vgpu_cache_maint(vgpu_get_handle(g), TEGRA_VGPU_FB_FLUSH);
233 return 0;
234}
235
236void vgpu_mm_l2_invalidate(struct gk20a *g)
237{
238
239 gk20a_dbg_fn("");
240
241 vgpu_cache_maint(vgpu_get_handle(g), TEGRA_VGPU_L2_MAINT_INV);
242}
243
244void vgpu_mm_l2_flush(struct gk20a *g, bool invalidate)
245{
246 u8 op;
247
248 gk20a_dbg_fn("");
249
250 if (invalidate)
251 op = TEGRA_VGPU_L2_MAINT_FLUSH_INV;
252 else
253 op = TEGRA_VGPU_L2_MAINT_FLUSH;
254
255 vgpu_cache_maint(vgpu_get_handle(g), op);
256}
257
258void vgpu_mm_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb)
259{
260 gk20a_dbg_fn("");
261
262 nvgpu_err(g, "call to RM server not supported");
263}
264
265void vgpu_mm_mmu_set_debug_mode(struct gk20a *g, bool enable)
266{
267 struct tegra_vgpu_cmd_msg msg;
268 struct tegra_vgpu_mmu_debug_mode *p = &msg.params.mmu_debug_mode;
269 int err;
270
271 gk20a_dbg_fn("");
272
273 msg.cmd = TEGRA_VGPU_CMD_SET_MMU_DEBUG_MODE;
274 msg.handle = vgpu_get_handle(g);
275 p->enable = (u32)enable;
276 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
277 WARN_ON(err || msg.ret);
278}
diff --git a/drivers/gpu/nvgpu/vgpu/mm_vgpu.h b/drivers/gpu/nvgpu/vgpu/mm_vgpu.h
new file mode 100644
index 00000000..12265fc1
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/mm_vgpu.h
@@ -0,0 +1,47 @@
1/*
2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _MM_VGPU_H_
24#define _MM_VGPU_H_
25
26struct nvgpu_mem;
27struct channel_gk20a;
28struct vm_gk20a_mapping_batch;
29struct gk20a_as_share;
30struct vm_gk20a;
31
32void vgpu_locked_gmmu_unmap(struct vm_gk20a *vm,
33 u64 vaddr,
34 u64 size,
35 int pgsz_idx,
36 bool va_allocated,
37 int rw_flag,
38 bool sparse,
39 struct vm_gk20a_mapping_batch *batch);
40int vgpu_vm_bind_channel(struct gk20a_as_share *as_share,
41 struct channel_gk20a *ch);
42int vgpu_mm_fb_flush(struct gk20a *g);
43void vgpu_mm_l2_invalidate(struct gk20a *g);
44void vgpu_mm_l2_flush(struct gk20a *g, bool invalidate);
45void vgpu_mm_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb);
46void vgpu_mm_mmu_set_debug_mode(struct gk20a *g, bool enable);
47#endif
diff --git a/drivers/gpu/nvgpu/vgpu/tsg_vgpu.c b/drivers/gpu/nvgpu/vgpu/tsg_vgpu.c
new file mode 100644
index 00000000..a6e493d0
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/tsg_vgpu.c
@@ -0,0 +1,160 @@
1/*
2 * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include "gk20a/gk20a.h"
24#include "gk20a/channel_gk20a.h"
25#include "gk20a/tsg_gk20a.h"
26#include "fifo_vgpu.h"
27
28#include <nvgpu/bug.h>
29#include <nvgpu/vgpu/tegra_vgpu.h>
30#include <nvgpu/vgpu/vgpu.h>
31
32int vgpu_tsg_open(struct tsg_gk20a *tsg)
33{
34 struct tegra_vgpu_cmd_msg msg = {};
35 struct tegra_vgpu_tsg_open_rel_params *p =
36 &msg.params.tsg_open;
37 int err;
38
39 gk20a_dbg_fn("");
40
41 msg.cmd = TEGRA_VGPU_CMD_TSG_OPEN;
42 msg.handle = vgpu_get_handle(tsg->g);
43 p->tsg_id = tsg->tsgid;
44 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
45 err = err ? err : msg.ret;
46 if (err) {
47 nvgpu_err(tsg->g,
48 "vgpu_tsg_open failed, tsgid %d", tsg->tsgid);
49 }
50
51 return err;
52}
53
54void vgpu_tsg_release(struct tsg_gk20a *tsg)
55{
56 struct tegra_vgpu_cmd_msg msg = {};
57 struct tegra_vgpu_tsg_open_rel_params *p =
58 &msg.params.tsg_release;
59 int err;
60
61 gk20a_dbg_fn("");
62
63 msg.cmd = TEGRA_VGPU_CMD_TSG_RELEASE;
64 msg.handle = vgpu_get_handle(tsg->g);
65 p->tsg_id = tsg->tsgid;
66 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
67 err = err ? err : msg.ret;
68 if (err) {
69 nvgpu_err(tsg->g,
70 "vgpu_tsg_release failed, tsgid %d", tsg->tsgid);
71 }
72}
73
74int vgpu_enable_tsg(struct tsg_gk20a *tsg)
75{
76 struct gk20a *g = tsg->g;
77 struct channel_gk20a *ch;
78
79 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
80 nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry)
81 g->ops.fifo.enable_channel(ch);
82 nvgpu_rwsem_up_read(&tsg->ch_list_lock);
83
84 return 0;
85}
86
87int vgpu_tsg_bind_channel(struct tsg_gk20a *tsg,
88 struct channel_gk20a *ch)
89{
90 struct tegra_vgpu_cmd_msg msg = {};
91 struct tegra_vgpu_tsg_bind_unbind_channel_params *p =
92 &msg.params.tsg_bind_unbind_channel;
93 int err;
94
95 gk20a_dbg_fn("");
96
97 err = gk20a_tsg_bind_channel(tsg, ch);
98 if (err)
99 return err;
100
101 msg.cmd = TEGRA_VGPU_CMD_TSG_BIND_CHANNEL;
102 msg.handle = vgpu_get_handle(tsg->g);
103 p->tsg_id = tsg->tsgid;
104 p->ch_handle = ch->virt_ctx;
105 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
106 err = err ? err : msg.ret;
107 if (err) {
108 nvgpu_err(tsg->g,
109 "vgpu_tsg_bind_channel failed, ch %d tsgid %d",
110 ch->chid, tsg->tsgid);
111 gk20a_tsg_unbind_channel(ch);
112 }
113
114 return err;
115}
116
117int vgpu_tsg_unbind_channel(struct channel_gk20a *ch)
118{
119 struct tegra_vgpu_cmd_msg msg = {};
120 struct tegra_vgpu_tsg_bind_unbind_channel_params *p =
121 &msg.params.tsg_bind_unbind_channel;
122 int err;
123
124 gk20a_dbg_fn("");
125
126 err = gk20a_fifo_tsg_unbind_channel(ch);
127 if (err)
128 return err;
129
130 msg.cmd = TEGRA_VGPU_CMD_TSG_UNBIND_CHANNEL;
131 msg.handle = vgpu_get_handle(ch->g);
132 p->ch_handle = ch->virt_ctx;
133 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
134 err = err ? err : msg.ret;
135 WARN_ON(err);
136
137 return err;
138}
139
140int vgpu_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice)
141{
142 struct tegra_vgpu_cmd_msg msg = {0};
143 struct tegra_vgpu_tsg_timeslice_params *p =
144 &msg.params.tsg_timeslice;
145 int err;
146
147 gk20a_dbg_fn("");
148
149 msg.cmd = TEGRA_VGPU_CMD_TSG_SET_TIMESLICE;
150 msg.handle = vgpu_get_handle(tsg->g);
151 p->tsg_id = tsg->tsgid;
152 p->timeslice_us = timeslice;
153 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
154 err = err ? err : msg.ret;
155 WARN_ON(err);
156 if (!err)
157 tsg->timeslice_us = timeslice;
158
159 return err;
160}
diff --git a/drivers/gpu/nvgpu/vgpu/vgpu.c b/drivers/gpu/nvgpu/vgpu/vgpu.c
new file mode 100644
index 00000000..eb56d4f9
--- /dev/null
+++ b/drivers/gpu/nvgpu/vgpu/vgpu.c
@@ -0,0 +1,350 @@
1/*
2 * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/enabled.h>
24#include <nvgpu/bus.h>
25#include <nvgpu/vgpu/vgpu_ivc.h>
26#include <nvgpu/vgpu/vgpu.h>
27
28#include "gk20a/gk20a.h"
29#include "fecs_trace_vgpu.h"
30
31int vgpu_comm_init(struct gk20a *g)
32{
33 size_t queue_sizes[] = { TEGRA_VGPU_QUEUE_SIZES };
34
35 return vgpu_ivc_init(g, 3, queue_sizes, TEGRA_VGPU_QUEUE_CMD,
36 ARRAY_SIZE(queue_sizes));
37}
38
39void vgpu_comm_deinit(void)
40{
41 size_t queue_sizes[] = { TEGRA_VGPU_QUEUE_SIZES };
42
43 vgpu_ivc_deinit(TEGRA_VGPU_QUEUE_CMD, ARRAY_SIZE(queue_sizes));
44}
45
46int vgpu_comm_sendrecv(struct tegra_vgpu_cmd_msg *msg, size_t size_in,
47 size_t size_out)
48{
49 void *handle;
50 size_t size = size_in;
51 void *data = msg;
52 int err;
53
54 err = vgpu_ivc_sendrecv(vgpu_ivc_get_server_vmid(),
55 TEGRA_VGPU_QUEUE_CMD, &handle, &data, &size);
56 if (!err) {
57 WARN_ON(size < size_out);
58 memcpy(msg, data, size_out);
59 vgpu_ivc_release(handle);
60 }
61
62 return err;
63}
64
65u64 vgpu_connect(void)
66{
67 struct tegra_vgpu_cmd_msg msg;
68 struct tegra_vgpu_connect_params *p = &msg.params.connect;
69 int err;
70
71 msg.cmd = TEGRA_VGPU_CMD_CONNECT;
72 p->module = TEGRA_VGPU_MODULE_GPU;
73 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
74
75 return (err || msg.ret) ? 0 : p->handle;
76}
77
78int vgpu_get_attribute(u64 handle, u32 attrib, u32 *value)
79{
80 struct tegra_vgpu_cmd_msg msg;
81 struct tegra_vgpu_attrib_params *p = &msg.params.attrib;
82 int err;
83
84 msg.cmd = TEGRA_VGPU_CMD_GET_ATTRIBUTE;
85 msg.handle = handle;
86 p->attrib = attrib;
87 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
88
89 if (err || msg.ret)
90 return -1;
91
92 *value = p->value;
93 return 0;
94}
95
96static void vgpu_handle_channel_event(struct gk20a *g,
97 struct tegra_vgpu_channel_event_info *info)
98{
99 struct tsg_gk20a *tsg;
100
101 if (!info->is_tsg) {
102 nvgpu_err(g, "channel event posted");
103 return;
104 }
105
106 if (info->id >= g->fifo.num_channels ||
107 info->event_id >= TEGRA_VGPU_CHANNEL_EVENT_ID_MAX) {
108 nvgpu_err(g, "invalid channel event");
109 return;
110 }
111
112 tsg = &g->fifo.tsg[info->id];
113
114 gk20a_tsg_event_id_post_event(tsg, info->event_id);
115}
116
117int vgpu_intr_thread(void *dev_id)
118{
119 struct gk20a *g = dev_id;
120 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
121
122 while (true) {
123 struct tegra_vgpu_intr_msg *msg;
124 u32 sender;
125 void *handle;
126 size_t size;
127 int err;
128
129 err = vgpu_ivc_recv(TEGRA_VGPU_QUEUE_INTR, &handle,
130 (void **)&msg, &size, &sender);
131 if (err == -ETIME)
132 continue;
133 if (WARN_ON(err))
134 continue;
135
136 if (msg->event == TEGRA_VGPU_EVENT_ABORT) {
137 vgpu_ivc_release(handle);
138 break;
139 }
140
141 switch (msg->event) {
142 case TEGRA_VGPU_EVENT_INTR:
143 if (msg->unit == TEGRA_VGPU_INTR_GR)
144 vgpu_gr_isr(g, &msg->info.gr_intr);
145 else if (msg->unit == TEGRA_VGPU_NONSTALL_INTR_GR)
146 vgpu_gr_nonstall_isr(g,
147 &msg->info.gr_nonstall_intr);
148 else if (msg->unit == TEGRA_VGPU_INTR_FIFO)
149 vgpu_fifo_isr(g, &msg->info.fifo_intr);
150 else if (msg->unit == TEGRA_VGPU_NONSTALL_INTR_FIFO)
151 vgpu_fifo_nonstall_isr(g,
152 &msg->info.fifo_nonstall_intr);
153 else if (msg->unit == TEGRA_VGPU_NONSTALL_INTR_CE2)
154 vgpu_ce2_nonstall_isr(g,
155 &msg->info.ce2_nonstall_intr);
156 break;
157#ifdef CONFIG_GK20A_CTXSW_TRACE
158 case TEGRA_VGPU_EVENT_FECS_TRACE:
159 vgpu_fecs_trace_data_update(g);
160 break;
161#endif
162 case TEGRA_VGPU_EVENT_CHANNEL:
163 vgpu_handle_channel_event(g, &msg->info.channel_event);
164 break;
165 case TEGRA_VGPU_EVENT_SM_ESR:
166 vgpu_gr_handle_sm_esr_event(g, &msg->info.sm_esr);
167 break;
168 default:
169 nvgpu_err(g, "unknown event %u", msg->event);
170 break;
171 }
172
173 vgpu_ivc_release(handle);
174 }
175
176 while (!nvgpu_thread_should_stop(&priv->intr_handler))
177 nvgpu_msleep(10);
178 return 0;
179}
180
181void vgpu_remove_support_common(struct gk20a *g)
182{
183 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
184 struct tegra_vgpu_intr_msg msg;
185 int err;
186
187 if (g->dbg_regops_tmp_buf)
188 nvgpu_kfree(g, g->dbg_regops_tmp_buf);
189
190 if (g->pmu.remove_support)
191 g->pmu.remove_support(&g->pmu);
192
193 if (g->gr.remove_support)
194 g->gr.remove_support(&g->gr);
195
196 if (g->fifo.remove_support)
197 g->fifo.remove_support(&g->fifo);
198
199 if (g->mm.remove_support)
200 g->mm.remove_support(&g->mm);
201
202 msg.event = TEGRA_VGPU_EVENT_ABORT;
203 err = vgpu_ivc_send(vgpu_ivc_get_peer_self(), TEGRA_VGPU_QUEUE_INTR,
204 &msg, sizeof(msg));
205 WARN_ON(err);
206 nvgpu_thread_stop(&priv->intr_handler);
207}
208
209void vgpu_detect_chip(struct gk20a *g)
210{
211 struct nvgpu_gpu_params *p = &g->params;
212 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
213
214 p->gpu_arch = priv->constants.arch;
215 p->gpu_impl = priv->constants.impl;
216 p->gpu_rev = priv->constants.rev;
217
218 gk20a_dbg_info("arch: %x, impl: %x, rev: %x\n",
219 p->gpu_arch,
220 p->gpu_impl,
221 p->gpu_rev);
222}
223
224int vgpu_init_gpu_characteristics(struct gk20a *g)
225{
226 int err;
227
228 gk20a_dbg_fn("");
229
230 err = gk20a_init_gpu_characteristics(g);
231 if (err)
232 return err;
233
234 __nvgpu_set_enabled(g, NVGPU_SUPPORT_MAP_BUFFER_BATCH, false);
235
236 /* features vgpu does not support */
237 __nvgpu_set_enabled(g, NVGPU_SUPPORT_RESCHEDULE_RUNLIST, false);
238
239 return 0;
240}
241
242int vgpu_read_ptimer(struct gk20a *g, u64 *value)
243{
244 struct tegra_vgpu_cmd_msg msg = {0};
245 struct tegra_vgpu_read_ptimer_params *p = &msg.params.read_ptimer;
246 int err;
247
248 gk20a_dbg_fn("");
249
250 msg.cmd = TEGRA_VGPU_CMD_READ_PTIMER;
251 msg.handle = vgpu_get_handle(g);
252
253 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
254 err = err ? err : msg.ret;
255 if (!err)
256 *value = p->time;
257 else
258 nvgpu_err(g, "vgpu read ptimer failed, err=%d", err);
259
260 return err;
261}
262
263int vgpu_get_timestamps_zipper(struct gk20a *g,
264 u32 source_id, u32 count,
265 struct nvgpu_cpu_time_correlation_sample *samples)
266{
267 struct tegra_vgpu_cmd_msg msg = {0};
268 struct tegra_vgpu_get_timestamps_zipper_params *p =
269 &msg.params.get_timestamps_zipper;
270 int err;
271 u32 i;
272
273 gk20a_dbg_fn("");
274
275 if (count > TEGRA_VGPU_GET_TIMESTAMPS_ZIPPER_MAX_COUNT) {
276 nvgpu_err(g, "count %u overflow", count);
277 return -EINVAL;
278 }
279
280 msg.cmd = TEGRA_VGPU_CMD_GET_TIMESTAMPS_ZIPPER;
281 msg.handle = vgpu_get_handle(g);
282 p->source_id = TEGRA_VGPU_GET_TIMESTAMPS_ZIPPER_SRC_ID_TSC;
283 p->count = count;
284
285 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
286 err = err ? err : msg.ret;
287 if (err) {
288 nvgpu_err(g, "vgpu get timestamps zipper failed, err=%d", err);
289 return err;
290 }
291
292 for (i = 0; i < count; i++) {
293 samples[i].cpu_timestamp = p->samples[i].cpu_timestamp;
294 samples[i].gpu_timestamp = p->samples[i].gpu_timestamp;
295 }
296
297 return err;
298}
299
300int vgpu_init_hal(struct gk20a *g)
301{
302 u32 ver = g->params.gpu_arch + g->params.gpu_impl;
303 int err;
304
305 switch (ver) {
306 case NVGPU_GPUID_GP10B:
307 gk20a_dbg_info("gp10b detected");
308 err = vgpu_gp10b_init_hal(g);
309 break;
310 case NVGPU_GPUID_GV11B:
311 err = vgpu_gv11b_init_hal(g);
312 break;
313 default:
314 nvgpu_err(g, "no support for %x", ver);
315 err = -ENODEV;
316 break;
317 }
318
319 return err;
320}
321
322int vgpu_get_constants(struct gk20a *g)
323{
324 struct tegra_vgpu_cmd_msg msg = {};
325 struct tegra_vgpu_constants_params *p = &msg.params.constants;
326 struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
327 int err;
328
329 gk20a_dbg_fn("");
330
331 msg.cmd = TEGRA_VGPU_CMD_GET_CONSTANTS;
332 msg.handle = vgpu_get_handle(g);
333 err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
334 err = err ? err : msg.ret;
335
336 if (unlikely(err)) {
337 nvgpu_err(g, "%s failed, err=%d", __func__, err);
338 return err;
339 }
340
341 if (unlikely(p->gpc_count > TEGRA_VGPU_MAX_GPC_COUNT ||
342 p->max_tpc_per_gpc_count > TEGRA_VGPU_MAX_TPC_COUNT_PER_GPC)) {
343 nvgpu_err(g, "gpc_count %d max_tpc_per_gpc %d overflow",
344 (int)p->gpc_count, (int)p->max_tpc_per_gpc_count);
345 return -EINVAL;
346 }
347
348 priv->constants = *p;
349 return 0;
350}
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-07 15:25:12 -0400