diff options
Diffstat (limited to 'drivers/gpu/nvgpu/common/sync')
-rw-r--r-- | drivers/gpu/nvgpu/common/sync/channel_sync.c | 670 |
1 files changed, 670 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/sync/channel_sync.c b/drivers/gpu/nvgpu/common/sync/channel_sync.c new file mode 100644 index 00000000..b4caab38 --- /dev/null +++ b/drivers/gpu/nvgpu/common/sync/channel_sync.c | |||
@@ -0,0 +1,670 @@ | |||
1 | /* | ||
2 | * GK20A Channel Synchronization Abstraction | ||
3 | * | ||
4 | * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <nvgpu/semaphore.h> | ||
26 | #include <nvgpu/kmem.h> | ||
27 | #include <nvgpu/log.h> | ||
28 | #include <nvgpu/atomic.h> | ||
29 | #include <nvgpu/bug.h> | ||
30 | #include <nvgpu/list.h> | ||
31 | #include <nvgpu/nvhost.h> | ||
32 | #include <nvgpu/os_fence.h> | ||
33 | #include <nvgpu/channel.h> | ||
34 | #include <nvgpu/channel_sync.h> | ||
35 | |||
36 | #include "gk20a/gk20a.h" | ||
37 | #include "gk20a/fence_gk20a.h" | ||
38 | #include "gk20a/mm_gk20a.h" | ||
39 | |||
40 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | ||
41 | |||
42 | struct nvgpu_channel_sync_syncpt { | ||
43 | struct nvgpu_channel_sync ops; | ||
44 | struct channel_gk20a *c; | ||
45 | struct nvgpu_nvhost_dev *nvhost_dev; | ||
46 | u32 id; | ||
47 | struct nvgpu_mem syncpt_buf; | ||
48 | }; | ||
49 | |||
50 | int channel_sync_syncpt_gen_wait_cmd(struct channel_gk20a *c, | ||
51 | u32 id, u32 thresh, struct priv_cmd_entry *wait_cmd, | ||
52 | u32 wait_cmd_size, int pos, bool preallocated) | ||
53 | { | ||
54 | int err = 0; | ||
55 | bool is_expired = nvgpu_nvhost_syncpt_is_expired_ext( | ||
56 | c->g->nvhost_dev, id, thresh); | ||
57 | |||
58 | if (is_expired) { | ||
59 | if (preallocated) { | ||
60 | nvgpu_memset(c->g, wait_cmd->mem, | ||
61 | (wait_cmd->off + (u32)pos * wait_cmd_size) * (u32)sizeof(u32), | ||
62 | 0, wait_cmd_size * (u32)sizeof(u32)); | ||
63 | } | ||
64 | } else { | ||
65 | if (!preallocated) { | ||
66 | err = gk20a_channel_alloc_priv_cmdbuf(c, | ||
67 | c->g->ops.fifo.get_syncpt_wait_cmd_size(), wait_cmd); | ||
68 | if (err != 0) { | ||
69 | nvgpu_err(c->g, "not enough priv cmd buffer space"); | ||
70 | return err; | ||
71 | } | ||
72 | } | ||
73 | nvgpu_log(c->g, gpu_dbg_info, "sp->id %d gpu va %llx", | ||
74 | id, c->vm->syncpt_ro_map_gpu_va); | ||
75 | c->g->ops.fifo.add_syncpt_wait_cmd(c->g, wait_cmd, | ||
76 | (u32)pos * wait_cmd_size, id, thresh, | ||
77 | c->vm->syncpt_ro_map_gpu_va); | ||
78 | } | ||
79 | |||
80 | return 0; | ||
81 | } | ||
82 | |||
83 | static int channel_sync_syncpt_wait_raw(struct nvgpu_channel_sync *s, | ||
84 | u32 id, u32 thresh, struct priv_cmd_entry *wait_cmd) | ||
85 | { | ||
86 | struct nvgpu_channel_sync_syncpt *sp = | ||
87 | container_of(s, struct nvgpu_channel_sync_syncpt, ops); | ||
88 | struct channel_gk20a *c = sp->c; | ||
89 | int err = 0; | ||
90 | u32 wait_cmd_size = c->g->ops.fifo.get_syncpt_wait_cmd_size(); | ||
91 | |||
92 | if (!nvgpu_nvhost_syncpt_is_valid_pt_ext(sp->nvhost_dev, id)) { | ||
93 | return -EINVAL; | ||
94 | } | ||
95 | |||
96 | err = channel_sync_syncpt_gen_wait_cmd(c, id, thresh, | ||
97 | wait_cmd, wait_cmd_size, 0, false); | ||
98 | |||
99 | return err; | ||
100 | } | ||
101 | |||
102 | static int channel_sync_syncpt_wait_fd(struct nvgpu_channel_sync *s, int fd, | ||
103 | struct priv_cmd_entry *wait_cmd, int max_wait_cmds) | ||
104 | { | ||
105 | struct nvgpu_os_fence os_fence = {0}; | ||
106 | struct nvgpu_channel_sync_syncpt *sp = | ||
107 | container_of(s, struct nvgpu_channel_sync_syncpt, ops); | ||
108 | struct channel_gk20a *c = sp->c; | ||
109 | int err = 0; | ||
110 | |||
111 | err = nvgpu_os_fence_fdget(&os_fence, c, fd); | ||
112 | if (err != 0) { | ||
113 | return -EINVAL; | ||
114 | } | ||
115 | |||
116 | err = os_fence.ops->program_waits(&os_fence, | ||
117 | wait_cmd, c, max_wait_cmds); | ||
118 | |||
119 | os_fence.ops->drop_ref(&os_fence); | ||
120 | |||
121 | return err; | ||
122 | } | ||
123 | |||
124 | static void channel_sync_syncpt_update(void *priv, int nr_completed) | ||
125 | { | ||
126 | struct channel_gk20a *ch = priv; | ||
127 | |||
128 | gk20a_channel_update(ch); | ||
129 | |||
130 | /* note: channel_get() is in channel_sync_syncpt_incr_common() */ | ||
131 | gk20a_channel_put(ch); | ||
132 | } | ||
133 | |||
134 | static int channel_sync_syncpt_incr_common(struct nvgpu_channel_sync *s, | ||
135 | bool wfi_cmd, | ||
136 | bool register_irq, | ||
137 | struct priv_cmd_entry *incr_cmd, | ||
138 | struct gk20a_fence *fence, | ||
139 | bool need_sync_fence) | ||
140 | { | ||
141 | u32 thresh; | ||
142 | int err; | ||
143 | struct nvgpu_channel_sync_syncpt *sp = | ||
144 | container_of(s, struct nvgpu_channel_sync_syncpt, ops); | ||
145 | struct channel_gk20a *c = sp->c; | ||
146 | struct nvgpu_os_fence os_fence = {0}; | ||
147 | |||
148 | err = gk20a_channel_alloc_priv_cmdbuf(c, | ||
149 | c->g->ops.fifo.get_syncpt_incr_cmd_size(wfi_cmd), | ||
150 | incr_cmd); | ||
151 | if (err != 0) { | ||
152 | return err; | ||
153 | } | ||
154 | |||
155 | nvgpu_log(c->g, gpu_dbg_info, "sp->id %d gpu va %llx", | ||
156 | sp->id, sp->syncpt_buf.gpu_va); | ||
157 | c->g->ops.fifo.add_syncpt_incr_cmd(c->g, wfi_cmd, | ||
158 | incr_cmd, sp->id, sp->syncpt_buf.gpu_va); | ||
159 | |||
160 | thresh = nvgpu_nvhost_syncpt_incr_max_ext(sp->nvhost_dev, sp->id, | ||
161 | c->g->ops.fifo.get_syncpt_incr_per_release()); | ||
162 | |||
163 | if (register_irq) { | ||
164 | struct channel_gk20a *referenced = gk20a_channel_get(c); | ||
165 | |||
166 | WARN_ON(!referenced); | ||
167 | |||
168 | if (referenced) { | ||
169 | /* note: channel_put() is in | ||
170 | * channel_sync_syncpt_update() */ | ||
171 | |||
172 | err = nvgpu_nvhost_intr_register_notifier( | ||
173 | sp->nvhost_dev, | ||
174 | sp->id, thresh, | ||
175 | channel_sync_syncpt_update, c); | ||
176 | if (err != 0) { | ||
177 | gk20a_channel_put(referenced); | ||
178 | } | ||
179 | |||
180 | /* Adding interrupt action should | ||
181 | * never fail. A proper error handling | ||
182 | * here would require us to decrement | ||
183 | * the syncpt max back to its original | ||
184 | * value. */ | ||
185 | WARN(err, | ||
186 | "failed to set submit complete interrupt"); | ||
187 | } | ||
188 | } | ||
189 | |||
190 | if (need_sync_fence) { | ||
191 | err = nvgpu_os_fence_syncpt_create(&os_fence, c, sp->nvhost_dev, | ||
192 | sp->id, thresh); | ||
193 | |||
194 | if (err != 0) { | ||
195 | goto clean_up_priv_cmd; | ||
196 | } | ||
197 | } | ||
198 | |||
199 | err = gk20a_fence_from_syncpt(fence, sp->nvhost_dev, | ||
200 | sp->id, thresh, os_fence); | ||
201 | |||
202 | if (err != 0) { | ||
203 | if (nvgpu_os_fence_is_initialized(&os_fence) != 0) { | ||
204 | os_fence.ops->drop_ref(&os_fence); | ||
205 | } | ||
206 | goto clean_up_priv_cmd; | ||
207 | } | ||
208 | |||
209 | return 0; | ||
210 | |||
211 | clean_up_priv_cmd: | ||
212 | gk20a_free_priv_cmdbuf(c, incr_cmd); | ||
213 | return err; | ||
214 | } | ||
215 | |||
216 | static int channel_sync_syncpt_incr(struct nvgpu_channel_sync *s, | ||
217 | struct priv_cmd_entry *entry, | ||
218 | struct gk20a_fence *fence, | ||
219 | bool need_sync_fence, | ||
220 | bool register_irq) | ||
221 | { | ||
222 | /* Don't put wfi cmd to this one since we're not returning | ||
223 | * a fence to user space. */ | ||
224 | return channel_sync_syncpt_incr_common(s, | ||
225 | false /* no wfi */, | ||
226 | register_irq /* register irq */, | ||
227 | entry, fence, need_sync_fence); | ||
228 | } | ||
229 | |||
230 | static int channel_sync_syncpt_incr_user(struct nvgpu_channel_sync *s, | ||
231 | int wait_fence_fd, | ||
232 | struct priv_cmd_entry *entry, | ||
233 | struct gk20a_fence *fence, | ||
234 | bool wfi, | ||
235 | bool need_sync_fence, | ||
236 | bool register_irq) | ||
237 | { | ||
238 | /* Need to do 'wfi + host incr' since we return the fence | ||
239 | * to user space. */ | ||
240 | return channel_sync_syncpt_incr_common(s, | ||
241 | wfi, | ||
242 | register_irq /* register irq */, | ||
243 | entry, fence, need_sync_fence); | ||
244 | } | ||
245 | |||
246 | static void channel_sync_syncpt_set_min_eq_max(struct nvgpu_channel_sync *s) | ||
247 | { | ||
248 | struct nvgpu_channel_sync_syncpt *sp = | ||
249 | container_of(s, struct nvgpu_channel_sync_syncpt, ops); | ||
250 | nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id); | ||
251 | } | ||
252 | |||
253 | static void channel_sync_syncpt_set_safe_state(struct nvgpu_channel_sync *s) | ||
254 | { | ||
255 | struct nvgpu_channel_sync_syncpt *sp = | ||
256 | container_of(s, struct nvgpu_channel_sync_syncpt, ops); | ||
257 | nvgpu_nvhost_syncpt_set_safe_state(sp->nvhost_dev, sp->id); | ||
258 | } | ||
259 | |||
260 | static int syncpt_get_id(struct nvgpu_channel_sync *s) | ||
261 | { | ||
262 | struct nvgpu_channel_sync_syncpt *sp = | ||
263 | container_of(s, struct nvgpu_channel_sync_syncpt, ops); | ||
264 | return sp->id; | ||
265 | } | ||
266 | |||
267 | static u64 channel_sync_syncpt_get_address(struct nvgpu_channel_sync *s) | ||
268 | { | ||
269 | struct nvgpu_channel_sync_syncpt *sp = | ||
270 | container_of(s, struct nvgpu_channel_sync_syncpt, ops); | ||
271 | return sp->syncpt_buf.gpu_va; | ||
272 | } | ||
273 | |||
274 | static void channel_sync_syncpt_destroy(struct nvgpu_channel_sync *s) | ||
275 | { | ||
276 | struct nvgpu_channel_sync_syncpt *sp = | ||
277 | container_of(s, struct nvgpu_channel_sync_syncpt, ops); | ||
278 | |||
279 | |||
280 | sp->c->g->ops.fifo.free_syncpt_buf(sp->c, &sp->syncpt_buf); | ||
281 | |||
282 | nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id); | ||
283 | nvgpu_nvhost_syncpt_put_ref_ext(sp->nvhost_dev, sp->id); | ||
284 | nvgpu_kfree(sp->c->g, sp); | ||
285 | } | ||
286 | |||
287 | static struct nvgpu_channel_sync * | ||
288 | channel_sync_syncpt_create(struct channel_gk20a *c, bool user_managed) | ||
289 | { | ||
290 | struct nvgpu_channel_sync_syncpt *sp; | ||
291 | char syncpt_name[32]; | ||
292 | |||
293 | sp = nvgpu_kzalloc(c->g, sizeof(*sp)); | ||
294 | if (sp == NULL) { | ||
295 | return NULL; | ||
296 | } | ||
297 | |||
298 | sp->c = c; | ||
299 | sp->nvhost_dev = c->g->nvhost_dev; | ||
300 | |||
301 | if (user_managed) { | ||
302 | snprintf(syncpt_name, sizeof(syncpt_name), | ||
303 | "%s_%d_user", c->g->name, c->chid); | ||
304 | |||
305 | sp->id = nvgpu_nvhost_get_syncpt_client_managed(sp->nvhost_dev, | ||
306 | syncpt_name); | ||
307 | } else { | ||
308 | snprintf(syncpt_name, sizeof(syncpt_name), | ||
309 | "%s_%d", c->g->name, c->chid); | ||
310 | |||
311 | sp->id = nvgpu_nvhost_get_syncpt_host_managed(sp->nvhost_dev, | ||
312 | c->chid, syncpt_name); | ||
313 | } | ||
314 | if (sp->id == 0) { | ||
315 | nvgpu_kfree(c->g, sp); | ||
316 | nvgpu_err(c->g, "failed to get free syncpt"); | ||
317 | return NULL; | ||
318 | } | ||
319 | |||
320 | sp->c->g->ops.fifo.alloc_syncpt_buf(sp->c, sp->id, | ||
321 | &sp->syncpt_buf); | ||
322 | |||
323 | nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id); | ||
324 | |||
325 | nvgpu_atomic_set(&sp->ops.refcount, 0); | ||
326 | sp->ops.wait_syncpt = channel_sync_syncpt_wait_raw; | ||
327 | sp->ops.wait_fd = channel_sync_syncpt_wait_fd; | ||
328 | sp->ops.incr = channel_sync_syncpt_incr; | ||
329 | sp->ops.incr_user = channel_sync_syncpt_incr_user; | ||
330 | sp->ops.set_min_eq_max = channel_sync_syncpt_set_min_eq_max; | ||
331 | sp->ops.set_safe_state = channel_sync_syncpt_set_safe_state; | ||
332 | sp->ops.syncpt_id = syncpt_get_id; | ||
333 | sp->ops.syncpt_address = channel_sync_syncpt_get_address; | ||
334 | sp->ops.destroy = channel_sync_syncpt_destroy; | ||
335 | |||
336 | return &sp->ops; | ||
337 | } | ||
338 | #endif /* CONFIG_TEGRA_GK20A_NVHOST */ | ||
339 | |||
340 | struct nvgpu_channel_sync_semaphore { | ||
341 | struct nvgpu_channel_sync ops; | ||
342 | struct channel_gk20a *c; | ||
343 | |||
344 | /* A semaphore pool owned by this channel. */ | ||
345 | struct nvgpu_semaphore_pool *pool; | ||
346 | }; | ||
347 | |||
348 | static void add_sema_cmd(struct gk20a *g, struct channel_gk20a *c, | ||
349 | struct nvgpu_semaphore *s, struct priv_cmd_entry *cmd, | ||
350 | u32 offset, bool acquire, bool wfi) | ||
351 | { | ||
352 | int ch = c->chid; | ||
353 | u32 ob, off = cmd->off + offset; | ||
354 | u64 va; | ||
355 | |||
356 | ob = off; | ||
357 | |||
358 | /* | ||
359 | * RO for acquire (since we just need to read the mem) and RW for | ||
360 | * release since we will need to write back to the semaphore memory. | ||
361 | */ | ||
362 | va = acquire ? nvgpu_semaphore_gpu_ro_va(s) : | ||
363 | nvgpu_semaphore_gpu_rw_va(s); | ||
364 | |||
365 | /* | ||
366 | * If the op is not an acquire (so therefor a release) we should | ||
367 | * incr the underlying sema next_value. | ||
368 | */ | ||
369 | if (!acquire) { | ||
370 | nvgpu_semaphore_prepare(s, c->hw_sema); | ||
371 | } | ||
372 | |||
373 | g->ops.fifo.add_sema_cmd(g, s, va, cmd, off, acquire, wfi); | ||
374 | |||
375 | if (acquire) { | ||
376 | gpu_sema_verbose_dbg(g, "(A) c=%d ACQ_GE %-4u pool=%-3llu" | ||
377 | "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", | ||
378 | ch, nvgpu_semaphore_get_value(s), | ||
379 | s->location.pool->page_idx, va, cmd->gva, | ||
380 | cmd->mem->gpu_va, ob); | ||
381 | } else { | ||
382 | gpu_sema_verbose_dbg(g, "(R) c=%d INCR %u (%u) pool=%-3llu" | ||
383 | "va=0x%llx cmd_mem=0x%llx b=0x%llx off=%u", | ||
384 | ch, nvgpu_semaphore_get_value(s), | ||
385 | nvgpu_semaphore_read(s), | ||
386 | s->location.pool->page_idx, | ||
387 | va, cmd->gva, cmd->mem->gpu_va, ob); | ||
388 | } | ||
389 | } | ||
390 | |||
391 | void channel_sync_semaphore_gen_wait_cmd(struct channel_gk20a *c, | ||
392 | struct nvgpu_semaphore *sema, struct priv_cmd_entry *wait_cmd, | ||
393 | u32 wait_cmd_size, int pos) | ||
394 | { | ||
395 | if (sema == NULL) { | ||
396 | /* expired */ | ||
397 | nvgpu_memset(c->g, wait_cmd->mem, | ||
398 | (wait_cmd->off + (u32)pos * wait_cmd_size) * (u32)sizeof(u32), | ||
399 | 0, wait_cmd_size * (u32)sizeof(u32)); | ||
400 | } else { | ||
401 | WARN_ON(!sema->incremented); | ||
402 | add_sema_cmd(c->g, c, sema, wait_cmd, | ||
403 | (u32)pos * wait_cmd_size, true, false); | ||
404 | nvgpu_semaphore_put(sema); | ||
405 | } | ||
406 | } | ||
407 | |||
408 | static int channel_sync_semaphore_wait_raw_syncpt( | ||
409 | struct nvgpu_channel_sync *s, u32 id, | ||
410 | u32 thresh, struct priv_cmd_entry *entry) | ||
411 | { | ||
412 | struct nvgpu_channel_sync_semaphore *sema = | ||
413 | container_of(s, struct nvgpu_channel_sync_semaphore, ops); | ||
414 | struct gk20a *g = sema->c->g; | ||
415 | nvgpu_err(g, "trying to use syncpoint synchronization"); | ||
416 | return -ENODEV; | ||
417 | } | ||
418 | |||
419 | static int channel_sync_semaphore_wait_fd( | ||
420 | struct nvgpu_channel_sync *s, int fd, | ||
421 | struct priv_cmd_entry *entry, int max_wait_cmds) | ||
422 | { | ||
423 | struct nvgpu_channel_sync_semaphore *sema = | ||
424 | container_of(s, struct nvgpu_channel_sync_semaphore, ops); | ||
425 | struct channel_gk20a *c = sema->c; | ||
426 | |||
427 | struct nvgpu_os_fence os_fence = {0}; | ||
428 | int err; | ||
429 | |||
430 | err = nvgpu_os_fence_fdget(&os_fence, c, fd); | ||
431 | if (err != 0) { | ||
432 | return err; | ||
433 | } | ||
434 | |||
435 | err = os_fence.ops->program_waits(&os_fence, | ||
436 | entry, c, max_wait_cmds); | ||
437 | |||
438 | os_fence.ops->drop_ref(&os_fence); | ||
439 | |||
440 | return err; | ||
441 | } | ||
442 | |||
443 | static int channel_sync_semaphore_incr_common( | ||
444 | struct nvgpu_channel_sync *s, bool wfi_cmd, | ||
445 | struct priv_cmd_entry *incr_cmd, | ||
446 | struct gk20a_fence *fence, | ||
447 | bool need_sync_fence) | ||
448 | { | ||
449 | u32 incr_cmd_size; | ||
450 | struct nvgpu_channel_sync_semaphore *sp = | ||
451 | container_of(s, struct nvgpu_channel_sync_semaphore, ops); | ||
452 | struct channel_gk20a *c = sp->c; | ||
453 | struct nvgpu_semaphore *semaphore; | ||
454 | int err = 0; | ||
455 | struct nvgpu_os_fence os_fence = {0}; | ||
456 | |||
457 | semaphore = nvgpu_semaphore_alloc(c); | ||
458 | if (semaphore == NULL) { | ||
459 | nvgpu_err(c->g, | ||
460 | "ran out of semaphores"); | ||
461 | return -ENOMEM; | ||
462 | } | ||
463 | |||
464 | incr_cmd_size = c->g->ops.fifo.get_sema_incr_cmd_size(); | ||
465 | err = gk20a_channel_alloc_priv_cmdbuf(c, incr_cmd_size, incr_cmd); | ||
466 | if (err) { | ||
467 | nvgpu_err(c->g, | ||
468 | "not enough priv cmd buffer space"); | ||
469 | goto clean_up_sema; | ||
470 | } | ||
471 | |||
472 | /* Release the completion semaphore. */ | ||
473 | add_sema_cmd(c->g, c, semaphore, incr_cmd, 0, false, wfi_cmd); | ||
474 | |||
475 | if (need_sync_fence) { | ||
476 | err = nvgpu_os_fence_sema_create(&os_fence, c, | ||
477 | semaphore); | ||
478 | |||
479 | if (err) { | ||
480 | goto clean_up_sema; | ||
481 | } | ||
482 | } | ||
483 | |||
484 | err = gk20a_fence_from_semaphore(fence, | ||
485 | semaphore, | ||
486 | &c->semaphore_wq, | ||
487 | os_fence); | ||
488 | |||
489 | if (err != 0) { | ||
490 | if (nvgpu_os_fence_is_initialized(&os_fence) != 0) { | ||
491 | os_fence.ops->drop_ref(&os_fence); | ||
492 | } | ||
493 | goto clean_up_sema; | ||
494 | } | ||
495 | |||
496 | return 0; | ||
497 | |||
498 | clean_up_sema: | ||
499 | nvgpu_semaphore_put(semaphore); | ||
500 | return err; | ||
501 | } | ||
502 | |||
503 | static int channel_sync_semaphore_incr( | ||
504 | struct nvgpu_channel_sync *s, | ||
505 | struct priv_cmd_entry *entry, | ||
506 | struct gk20a_fence *fence, | ||
507 | bool need_sync_fence, | ||
508 | bool register_irq) | ||
509 | { | ||
510 | /* Don't put wfi cmd to this one since we're not returning | ||
511 | * a fence to user space. */ | ||
512 | return channel_sync_semaphore_incr_common(s, | ||
513 | false /* no wfi */, | ||
514 | entry, fence, need_sync_fence); | ||
515 | } | ||
516 | |||
517 | static int channel_sync_semaphore_incr_user( | ||
518 | struct nvgpu_channel_sync *s, | ||
519 | int wait_fence_fd, | ||
520 | struct priv_cmd_entry *entry, | ||
521 | struct gk20a_fence *fence, | ||
522 | bool wfi, | ||
523 | bool need_sync_fence, | ||
524 | bool register_irq) | ||
525 | { | ||
526 | #ifdef CONFIG_SYNC | ||
527 | int err; | ||
528 | |||
529 | err = channel_sync_semaphore_incr_common(s, wfi, entry, fence, | ||
530 | need_sync_fence); | ||
531 | if (err != 0) { | ||
532 | return err; | ||
533 | } | ||
534 | |||
535 | return 0; | ||
536 | #else | ||
537 | struct nvgpu_channel_sync_semaphore *sema = | ||
538 | container_of(s, struct nvgpu_channel_sync_semaphore, ops); | ||
539 | nvgpu_err(sema->c->g, | ||
540 | "trying to use sync fds with CONFIG_SYNC disabled"); | ||
541 | return -ENODEV; | ||
542 | #endif | ||
543 | } | ||
544 | |||
545 | static void channel_sync_semaphore_set_min_eq_max(struct nvgpu_channel_sync *s) | ||
546 | { | ||
547 | struct nvgpu_channel_sync_semaphore *sp = | ||
548 | container_of(s, struct nvgpu_channel_sync_semaphore, ops); | ||
549 | struct channel_gk20a *c = sp->c; | ||
550 | bool updated; | ||
551 | |||
552 | if (c->hw_sema == NULL) { | ||
553 | return; | ||
554 | } | ||
555 | |||
556 | updated = nvgpu_semaphore_reset(c->hw_sema); | ||
557 | |||
558 | if (updated) { | ||
559 | nvgpu_cond_broadcast_interruptible(&c->semaphore_wq); | ||
560 | } | ||
561 | } | ||
562 | |||
563 | static void channel_sync_semaphore_set_safe_state(struct nvgpu_channel_sync *s) | ||
564 | { | ||
565 | /* Nothing to do. */ | ||
566 | } | ||
567 | |||
568 | static int channel_sync_semaphore_get_id(struct nvgpu_channel_sync *s) | ||
569 | { | ||
570 | return -EINVAL; | ||
571 | } | ||
572 | |||
573 | static u64 channel_sync_semaphore_get_address(struct nvgpu_channel_sync *s) | ||
574 | { | ||
575 | return 0; | ||
576 | } | ||
577 | |||
578 | static void channel_sync_semaphore_destroy(struct nvgpu_channel_sync *s) | ||
579 | { | ||
580 | struct nvgpu_channel_sync_semaphore *sema = | ||
581 | container_of(s, struct nvgpu_channel_sync_semaphore, ops); | ||
582 | |||
583 | struct channel_gk20a *c = sema->c; | ||
584 | struct gk20a *g = c->g; | ||
585 | |||
586 | if (c->has_os_fence_framework_support && | ||
587 | g->os_channel.os_fence_framework_inst_exists(c)) { | ||
588 | g->os_channel.destroy_os_fence_framework(c); | ||
589 | } | ||
590 | |||
591 | /* The sema pool is cleaned up by the VM destroy. */ | ||
592 | sema->pool = NULL; | ||
593 | |||
594 | nvgpu_kfree(sema->c->g, sema); | ||
595 | } | ||
596 | |||
597 | static struct nvgpu_channel_sync * | ||
598 | channel_sync_semaphore_create(struct channel_gk20a *c, bool user_managed) | ||
599 | { | ||
600 | struct nvgpu_channel_sync_semaphore *sema; | ||
601 | struct gk20a *g = c->g; | ||
602 | char pool_name[20]; | ||
603 | int asid = -1; | ||
604 | int err; | ||
605 | |||
606 | if (WARN_ON(c->vm == NULL)) { | ||
607 | return NULL; | ||
608 | } | ||
609 | |||
610 | sema = nvgpu_kzalloc(c->g, sizeof(*sema)); | ||
611 | if (sema == NULL) { | ||
612 | return NULL; | ||
613 | } | ||
614 | sema->c = c; | ||
615 | |||
616 | sprintf(pool_name, "semaphore_pool-%d", c->chid); | ||
617 | sema->pool = c->vm->sema_pool; | ||
618 | |||
619 | if (c->vm->as_share != NULL) { | ||
620 | asid = c->vm->as_share->id; | ||
621 | } | ||
622 | |||
623 | if (c->has_os_fence_framework_support) { | ||
624 | /*Init the sync_timeline for this channel */ | ||
625 | err = g->os_channel.init_os_fence_framework(c, | ||
626 | "gk20a_ch%d_as%d", c->chid, asid); | ||
627 | |||
628 | if (err != 0) { | ||
629 | nvgpu_kfree(g, sema); | ||
630 | return NULL; | ||
631 | } | ||
632 | } | ||
633 | |||
634 | nvgpu_atomic_set(&sema->ops.refcount, 0); | ||
635 | sema->ops.wait_syncpt = channel_sync_semaphore_wait_raw_syncpt; | ||
636 | sema->ops.wait_fd = channel_sync_semaphore_wait_fd; | ||
637 | sema->ops.incr = channel_sync_semaphore_incr; | ||
638 | sema->ops.incr_user = channel_sync_semaphore_incr_user; | ||
639 | sema->ops.set_min_eq_max = channel_sync_semaphore_set_min_eq_max; | ||
640 | sema->ops.set_safe_state = channel_sync_semaphore_set_safe_state; | ||
641 | sema->ops.syncpt_id = channel_sync_semaphore_get_id; | ||
642 | sema->ops.syncpt_address = channel_sync_semaphore_get_address; | ||
643 | sema->ops.destroy = channel_sync_semaphore_destroy; | ||
644 | |||
645 | return &sema->ops; | ||
646 | } | ||
647 | |||
648 | void nvgpu_channel_sync_destroy(struct nvgpu_channel_sync *sync, | ||
649 | bool set_safe_state) | ||
650 | { | ||
651 | if (set_safe_state) { | ||
652 | sync->set_safe_state(sync); | ||
653 | } | ||
654 | sync->destroy(sync); | ||
655 | } | ||
656 | |||
657 | struct nvgpu_channel_sync *nvgpu_channel_sync_create(struct channel_gk20a *c, | ||
658 | bool user_managed) | ||
659 | { | ||
660 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | ||
661 | if (gk20a_platform_has_syncpoints(c->g)) | ||
662 | return channel_sync_syncpt_create(c, user_managed); | ||
663 | #endif | ||
664 | return channel_sync_semaphore_create(c, user_managed); | ||
665 | } | ||
666 | |||
667 | bool nvgpu_channel_sync_needs_os_fence_framework(struct gk20a *g) | ||
668 | { | ||
669 | return !gk20a_platform_has_syncpoints(g); | ||
670 | } | ||