diff options
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a/tsg_gk20a.c')
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/tsg_gk20a.c | 362 |
1 files changed, 362 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c b/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c new file mode 100644 index 00000000..636ee8a0 --- /dev/null +++ b/drivers/gpu/nvgpu/gk20a/tsg_gk20a.c | |||
@@ -0,0 +1,362 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #include <uapi/linux/nvgpu.h> | ||
24 | |||
25 | #include <nvgpu/kmem.h> | ||
26 | #include <nvgpu/log.h> | ||
27 | |||
28 | #include "gk20a.h" | ||
29 | #include "tsg_gk20a.h" | ||
30 | |||
31 | bool gk20a_is_channel_marked_as_tsg(struct channel_gk20a *ch) | ||
32 | { | ||
33 | return !(ch->tsgid == NVGPU_INVALID_TSG_ID); | ||
34 | } | ||
35 | |||
36 | int gk20a_enable_tsg(struct tsg_gk20a *tsg) | ||
37 | { | ||
38 | struct gk20a *g = tsg->g; | ||
39 | struct channel_gk20a *ch; | ||
40 | bool is_next, is_ctx_reload; | ||
41 | |||
42 | gk20a_fifo_disable_tsg_sched(g, tsg); | ||
43 | |||
44 | /* | ||
45 | * Due to h/w bug that exists in Maxwell and Pascal, | ||
46 | * we first need to enable all channels with NEXT and CTX_RELOAD set, | ||
47 | * and then rest of the channels should be enabled | ||
48 | */ | ||
49 | nvgpu_rwsem_down_read(&tsg->ch_list_lock); | ||
50 | nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { | ||
51 | is_next = gk20a_fifo_channel_status_is_next(g, ch->chid); | ||
52 | is_ctx_reload = gk20a_fifo_channel_status_is_ctx_reload(g, ch->chid); | ||
53 | |||
54 | if (is_next || is_ctx_reload) | ||
55 | g->ops.fifo.enable_channel(ch); | ||
56 | } | ||
57 | |||
58 | nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { | ||
59 | is_next = gk20a_fifo_channel_status_is_next(g, ch->chid); | ||
60 | is_ctx_reload = gk20a_fifo_channel_status_is_ctx_reload(g, ch->chid); | ||
61 | |||
62 | if (is_next || is_ctx_reload) | ||
63 | continue; | ||
64 | |||
65 | g->ops.fifo.enable_channel(ch); | ||
66 | } | ||
67 | nvgpu_rwsem_up_read(&tsg->ch_list_lock); | ||
68 | |||
69 | gk20a_fifo_enable_tsg_sched(g, tsg); | ||
70 | |||
71 | return 0; | ||
72 | } | ||
73 | |||
74 | int gk20a_disable_tsg(struct tsg_gk20a *tsg) | ||
75 | { | ||
76 | struct gk20a *g = tsg->g; | ||
77 | struct channel_gk20a *ch; | ||
78 | |||
79 | nvgpu_rwsem_down_read(&tsg->ch_list_lock); | ||
80 | nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { | ||
81 | g->ops.fifo.disable_channel(ch); | ||
82 | } | ||
83 | nvgpu_rwsem_up_read(&tsg->ch_list_lock); | ||
84 | |||
85 | return 0; | ||
86 | } | ||
87 | |||
88 | static bool gk20a_is_channel_active(struct gk20a *g, struct channel_gk20a *ch) | ||
89 | { | ||
90 | struct fifo_gk20a *f = &g->fifo; | ||
91 | struct fifo_runlist_info_gk20a *runlist; | ||
92 | unsigned int i; | ||
93 | |||
94 | for (i = 0; i < f->max_runlists; ++i) { | ||
95 | runlist = &f->runlist_info[i]; | ||
96 | if (test_bit(ch->chid, runlist->active_channels)) | ||
97 | return true; | ||
98 | } | ||
99 | |||
100 | return false; | ||
101 | } | ||
102 | |||
103 | /* | ||
104 | * API to mark channel as part of TSG | ||
105 | * | ||
106 | * Note that channel is not runnable when we bind it to TSG | ||
107 | */ | ||
108 | int gk20a_tsg_bind_channel(struct tsg_gk20a *tsg, | ||
109 | struct channel_gk20a *ch) | ||
110 | { | ||
111 | gk20a_dbg_fn(""); | ||
112 | |||
113 | /* check if channel is already bound to some TSG */ | ||
114 | if (gk20a_is_channel_marked_as_tsg(ch)) { | ||
115 | return -EINVAL; | ||
116 | } | ||
117 | |||
118 | /* channel cannot be bound to TSG if it is already active */ | ||
119 | if (gk20a_is_channel_active(tsg->g, ch)) { | ||
120 | return -EINVAL; | ||
121 | } | ||
122 | |||
123 | ch->tsgid = tsg->tsgid; | ||
124 | |||
125 | /* all the channel part of TSG should need to be same runlist_id */ | ||
126 | if (tsg->runlist_id == FIFO_INVAL_TSG_ID) | ||
127 | tsg->runlist_id = ch->runlist_id; | ||
128 | else if (tsg->runlist_id != ch->runlist_id) { | ||
129 | nvgpu_err(tsg->g, | ||
130 | "Error: TSG channel should be share same runlist ch[%d] tsg[%d]", | ||
131 | ch->runlist_id, tsg->runlist_id); | ||
132 | return -EINVAL; | ||
133 | } | ||
134 | |||
135 | nvgpu_rwsem_down_write(&tsg->ch_list_lock); | ||
136 | nvgpu_list_add_tail(&ch->ch_entry, &tsg->ch_list); | ||
137 | nvgpu_rwsem_up_write(&tsg->ch_list_lock); | ||
138 | |||
139 | nvgpu_ref_get(&tsg->refcount); | ||
140 | |||
141 | gk20a_dbg(gpu_dbg_fn, "BIND tsg:%d channel:%d\n", | ||
142 | tsg->tsgid, ch->chid); | ||
143 | |||
144 | gk20a_dbg_fn("done"); | ||
145 | return 0; | ||
146 | } | ||
147 | |||
148 | int gk20a_tsg_unbind_channel(struct channel_gk20a *ch) | ||
149 | { | ||
150 | struct gk20a *g = ch->g; | ||
151 | struct tsg_gk20a *tsg = &g->fifo.tsg[ch->tsgid]; | ||
152 | int err; | ||
153 | |||
154 | err = gk20a_fifo_tsg_unbind_channel(ch); | ||
155 | if (err) { | ||
156 | nvgpu_err(g, "Channel %d unbind failed, tearing down TSG %d", | ||
157 | ch->chid, tsg->tsgid); | ||
158 | |||
159 | gk20a_fifo_abort_tsg(ch->g, ch->tsgid, true); | ||
160 | /* If channel unbind fails, channel is still part of runlist */ | ||
161 | channel_gk20a_update_runlist(ch, false); | ||
162 | |||
163 | nvgpu_rwsem_down_write(&tsg->ch_list_lock); | ||
164 | nvgpu_list_del(&ch->ch_entry); | ||
165 | nvgpu_rwsem_up_write(&tsg->ch_list_lock); | ||
166 | } | ||
167 | |||
168 | nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release); | ||
169 | ch->tsgid = NVGPU_INVALID_TSG_ID; | ||
170 | |||
171 | gk20a_dbg(gpu_dbg_fn, "UNBIND tsg:%d channel:%d\n", | ||
172 | tsg->tsgid, ch->chid); | ||
173 | |||
174 | return 0; | ||
175 | } | ||
176 | |||
177 | int gk20a_init_tsg_support(struct gk20a *g, u32 tsgid) | ||
178 | { | ||
179 | struct tsg_gk20a *tsg = NULL; | ||
180 | int err; | ||
181 | |||
182 | if (tsgid >= g->fifo.num_channels) | ||
183 | return -EINVAL; | ||
184 | |||
185 | tsg = &g->fifo.tsg[tsgid]; | ||
186 | |||
187 | tsg->in_use = false; | ||
188 | tsg->tsgid = tsgid; | ||
189 | |||
190 | nvgpu_init_list_node(&tsg->ch_list); | ||
191 | nvgpu_rwsem_init(&tsg->ch_list_lock); | ||
192 | |||
193 | nvgpu_init_list_node(&tsg->event_id_list); | ||
194 | err = nvgpu_mutex_init(&tsg->event_id_list_lock); | ||
195 | if (err) { | ||
196 | tsg->in_use = true; /* make this TSG unusable */ | ||
197 | return err; | ||
198 | } | ||
199 | |||
200 | return 0; | ||
201 | } | ||
202 | |||
203 | int gk20a_tsg_set_runlist_interleave(struct tsg_gk20a *tsg, u32 level) | ||
204 | { | ||
205 | struct gk20a *g = tsg->g; | ||
206 | int ret; | ||
207 | |||
208 | gk20a_dbg(gpu_dbg_sched, "tsgid=%u interleave=%u", tsg->tsgid, level); | ||
209 | |||
210 | switch (level) { | ||
211 | case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW: | ||
212 | case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM: | ||
213 | case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH: | ||
214 | ret = g->ops.fifo.set_runlist_interleave(g, tsg->tsgid, | ||
215 | true, 0, level); | ||
216 | if (!ret) | ||
217 | tsg->interleave_level = level; | ||
218 | break; | ||
219 | default: | ||
220 | ret = -EINVAL; | ||
221 | break; | ||
222 | } | ||
223 | |||
224 | return ret ? ret : g->ops.fifo.update_runlist(g, tsg->runlist_id, ~0, true, true); | ||
225 | } | ||
226 | |||
227 | int gk20a_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice) | ||
228 | { | ||
229 | struct gk20a *g = tsg->g; | ||
230 | |||
231 | gk20a_dbg(gpu_dbg_sched, "tsgid=%u timeslice=%u us", tsg->tsgid, timeslice); | ||
232 | |||
233 | return g->ops.fifo.tsg_set_timeslice(tsg, timeslice); | ||
234 | } | ||
235 | |||
236 | u32 gk20a_tsg_get_timeslice(struct tsg_gk20a *tsg) | ||
237 | { | ||
238 | struct gk20a *g = tsg->g; | ||
239 | |||
240 | if (!tsg->timeslice_us) | ||
241 | return g->ops.fifo.default_timeslice_us(g); | ||
242 | |||
243 | return tsg->timeslice_us; | ||
244 | } | ||
245 | |||
246 | static void release_used_tsg(struct fifo_gk20a *f, struct tsg_gk20a *tsg) | ||
247 | { | ||
248 | nvgpu_mutex_acquire(&f->tsg_inuse_mutex); | ||
249 | f->tsg[tsg->tsgid].in_use = false; | ||
250 | nvgpu_mutex_release(&f->tsg_inuse_mutex); | ||
251 | } | ||
252 | |||
253 | static struct tsg_gk20a *gk20a_tsg_acquire_unused_tsg(struct fifo_gk20a *f) | ||
254 | { | ||
255 | struct tsg_gk20a *tsg = NULL; | ||
256 | unsigned int tsgid; | ||
257 | |||
258 | nvgpu_mutex_acquire(&f->tsg_inuse_mutex); | ||
259 | for (tsgid = 0; tsgid < f->num_channels; tsgid++) { | ||
260 | if (!f->tsg[tsgid].in_use) { | ||
261 | f->tsg[tsgid].in_use = true; | ||
262 | tsg = &f->tsg[tsgid]; | ||
263 | break; | ||
264 | } | ||
265 | } | ||
266 | nvgpu_mutex_release(&f->tsg_inuse_mutex); | ||
267 | |||
268 | return tsg; | ||
269 | } | ||
270 | |||
271 | struct tsg_gk20a *gk20a_tsg_open(struct gk20a *g) | ||
272 | { | ||
273 | struct tsg_gk20a *tsg; | ||
274 | int err; | ||
275 | |||
276 | tsg = gk20a_tsg_acquire_unused_tsg(&g->fifo); | ||
277 | if (!tsg) | ||
278 | return NULL; | ||
279 | |||
280 | tsg->g = g; | ||
281 | tsg->num_active_channels = 0; | ||
282 | nvgpu_ref_init(&tsg->refcount); | ||
283 | |||
284 | tsg->tsg_gr_ctx = NULL; | ||
285 | tsg->vm = NULL; | ||
286 | tsg->interleave_level = NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW; | ||
287 | tsg->timeslice_us = 0; | ||
288 | tsg->timeslice_timeout = 0; | ||
289 | tsg->timeslice_scale = 0; | ||
290 | tsg->runlist_id = ~0; | ||
291 | tsg->tgid = current->tgid; | ||
292 | |||
293 | if (g->ops.fifo.init_eng_method_buffers) | ||
294 | g->ops.fifo.init_eng_method_buffers(g, tsg); | ||
295 | |||
296 | if (g->ops.fifo.tsg_open) { | ||
297 | err = g->ops.fifo.tsg_open(tsg); | ||
298 | if (err) { | ||
299 | nvgpu_err(g, "tsg %d fifo open failed %d", | ||
300 | tsg->tsgid, err); | ||
301 | goto clean_up; | ||
302 | } | ||
303 | } | ||
304 | |||
305 | gk20a_dbg(gpu_dbg_fn, "tsg opened %d\n", tsg->tsgid); | ||
306 | |||
307 | return tsg; | ||
308 | |||
309 | clean_up: | ||
310 | nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release); | ||
311 | return NULL; | ||
312 | } | ||
313 | |||
314 | void gk20a_tsg_release(struct nvgpu_ref *ref) | ||
315 | { | ||
316 | struct tsg_gk20a *tsg = container_of(ref, struct tsg_gk20a, refcount); | ||
317 | struct gk20a *g = tsg->g; | ||
318 | struct gk20a_event_id_data *event_id_data, *event_id_data_temp; | ||
319 | |||
320 | if (tsg->tsg_gr_ctx) { | ||
321 | gr_gk20a_free_tsg_gr_ctx(tsg); | ||
322 | tsg->tsg_gr_ctx = NULL; | ||
323 | } | ||
324 | |||
325 | if (g->ops.fifo.deinit_eng_method_buffers) | ||
326 | g->ops.fifo.deinit_eng_method_buffers(g, tsg); | ||
327 | |||
328 | if (tsg->vm) { | ||
329 | nvgpu_vm_put(tsg->vm); | ||
330 | tsg->vm = NULL; | ||
331 | } | ||
332 | |||
333 | /* unhook all events created on this TSG */ | ||
334 | nvgpu_mutex_acquire(&tsg->event_id_list_lock); | ||
335 | nvgpu_list_for_each_entry_safe(event_id_data, event_id_data_temp, | ||
336 | &tsg->event_id_list, | ||
337 | gk20a_event_id_data, | ||
338 | event_id_node) { | ||
339 | nvgpu_list_del(&event_id_data->event_id_node); | ||
340 | } | ||
341 | nvgpu_mutex_release(&tsg->event_id_list_lock); | ||
342 | |||
343 | release_used_tsg(&g->fifo, tsg); | ||
344 | |||
345 | tsg->runlist_id = ~0; | ||
346 | |||
347 | gk20a_dbg(gpu_dbg_fn, "tsg released %d\n", tsg->tsgid); | ||
348 | gk20a_put(g); | ||
349 | } | ||
350 | |||
351 | struct tsg_gk20a *tsg_gk20a_from_ch(struct channel_gk20a *ch) | ||
352 | { | ||
353 | struct tsg_gk20a *tsg = NULL; | ||
354 | |||
355 | if (gk20a_is_channel_marked_as_tsg(ch)) { | ||
356 | struct gk20a *g = ch->g; | ||
357 | struct fifo_gk20a *f = &g->fifo; | ||
358 | tsg = &f->tsg[ch->tsgid]; | ||
359 | } | ||
360 | |||
361 | return tsg; | ||
362 | } | ||