diff options
Diffstat (limited to 'drivers/gpu/nvgpu/os/linux/sched.c')
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/sched.c | 676 |
1 files changed, 676 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/os/linux/sched.c b/drivers/gpu/nvgpu/os/linux/sched.c new file mode 100644 index 00000000..2ad5aabf --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/sched.c | |||
@@ -0,0 +1,676 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify it | ||
5 | * under the terms and conditions of the GNU General Public License, | ||
6 | * version 2, as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | * more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | #include <asm/barrier.h> | ||
17 | #include <linux/wait.h> | ||
18 | #include <linux/uaccess.h> | ||
19 | #include <linux/poll.h> | ||
20 | #include <uapi/linux/nvgpu.h> | ||
21 | |||
22 | #include <nvgpu/kmem.h> | ||
23 | #include <nvgpu/log.h> | ||
24 | #include <nvgpu/bug.h> | ||
25 | #include <nvgpu/barrier.h> | ||
26 | |||
27 | #include "gk20a/gk20a.h" | ||
28 | #include "gk20a/gr_gk20a.h" | ||
29 | #include "sched.h" | ||
30 | #include "os_linux.h" | ||
31 | #include "ioctl_tsg.h" | ||
32 | |||
33 | #include <nvgpu/hw/gk20a/hw_ctxsw_prog_gk20a.h> | ||
34 | #include <nvgpu/hw/gk20a/hw_gr_gk20a.h> | ||
35 | |||
36 | ssize_t gk20a_sched_dev_read(struct file *filp, char __user *buf, | ||
37 | size_t size, loff_t *off) | ||
38 | { | ||
39 | struct gk20a_sched_ctrl *sched = filp->private_data; | ||
40 | struct gk20a *g = sched->g; | ||
41 | struct nvgpu_sched_event_arg event = { 0 }; | ||
42 | int err; | ||
43 | |||
44 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, | ||
45 | "filp=%p buf=%p size=%zu", filp, buf, size); | ||
46 | |||
47 | if (size < sizeof(event)) | ||
48 | return -EINVAL; | ||
49 | size = sizeof(event); | ||
50 | |||
51 | nvgpu_mutex_acquire(&sched->status_lock); | ||
52 | while (!sched->status) { | ||
53 | nvgpu_mutex_release(&sched->status_lock); | ||
54 | if (filp->f_flags & O_NONBLOCK) | ||
55 | return -EAGAIN; | ||
56 | err = NVGPU_COND_WAIT_INTERRUPTIBLE(&sched->readout_wq, | ||
57 | sched->status, 0); | ||
58 | if (err) | ||
59 | return err; | ||
60 | nvgpu_mutex_acquire(&sched->status_lock); | ||
61 | } | ||
62 | |||
63 | event.reserved = 0; | ||
64 | event.status = sched->status; | ||
65 | |||
66 | if (copy_to_user(buf, &event, size)) { | ||
67 | nvgpu_mutex_release(&sched->status_lock); | ||
68 | return -EFAULT; | ||
69 | } | ||
70 | |||
71 | sched->status = 0; | ||
72 | |||
73 | nvgpu_mutex_release(&sched->status_lock); | ||
74 | |||
75 | return size; | ||
76 | } | ||
77 | |||
78 | unsigned int gk20a_sched_dev_poll(struct file *filp, poll_table *wait) | ||
79 | { | ||
80 | struct gk20a_sched_ctrl *sched = filp->private_data; | ||
81 | struct gk20a *g = sched->g; | ||
82 | unsigned int mask = 0; | ||
83 | |||
84 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, " "); | ||
85 | |||
86 | nvgpu_mutex_acquire(&sched->status_lock); | ||
87 | poll_wait(filp, &sched->readout_wq.wq, wait); | ||
88 | if (sched->status) | ||
89 | mask |= POLLIN | POLLRDNORM; | ||
90 | nvgpu_mutex_release(&sched->status_lock); | ||
91 | |||
92 | return mask; | ||
93 | } | ||
94 | |||
95 | static int gk20a_sched_dev_ioctl_get_tsgs(struct gk20a_sched_ctrl *sched, | ||
96 | struct nvgpu_sched_get_tsgs_args *arg) | ||
97 | { | ||
98 | struct gk20a *g = sched->g; | ||
99 | |||
100 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "size=%u buffer=%llx", | ||
101 | arg->size, arg->buffer); | ||
102 | |||
103 | if ((arg->size < sched->bitmap_size) || (!arg->buffer)) { | ||
104 | arg->size = sched->bitmap_size; | ||
105 | return -ENOSPC; | ||
106 | } | ||
107 | |||
108 | nvgpu_mutex_acquire(&sched->status_lock); | ||
109 | if (copy_to_user((void __user *)(uintptr_t)arg->buffer, | ||
110 | sched->active_tsg_bitmap, sched->bitmap_size)) { | ||
111 | nvgpu_mutex_release(&sched->status_lock); | ||
112 | return -EFAULT; | ||
113 | } | ||
114 | nvgpu_mutex_release(&sched->status_lock); | ||
115 | |||
116 | return 0; | ||
117 | } | ||
118 | |||
119 | static int gk20a_sched_dev_ioctl_get_recent_tsgs(struct gk20a_sched_ctrl *sched, | ||
120 | struct nvgpu_sched_get_tsgs_args *arg) | ||
121 | { | ||
122 | struct gk20a *g = sched->g; | ||
123 | |||
124 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "size=%u buffer=%llx", | ||
125 | arg->size, arg->buffer); | ||
126 | |||
127 | if ((arg->size < sched->bitmap_size) || (!arg->buffer)) { | ||
128 | arg->size = sched->bitmap_size; | ||
129 | return -ENOSPC; | ||
130 | } | ||
131 | |||
132 | nvgpu_mutex_acquire(&sched->status_lock); | ||
133 | if (copy_to_user((void __user *)(uintptr_t)arg->buffer, | ||
134 | sched->recent_tsg_bitmap, sched->bitmap_size)) { | ||
135 | nvgpu_mutex_release(&sched->status_lock); | ||
136 | return -EFAULT; | ||
137 | } | ||
138 | |||
139 | memset(sched->recent_tsg_bitmap, 0, sched->bitmap_size); | ||
140 | nvgpu_mutex_release(&sched->status_lock); | ||
141 | |||
142 | return 0; | ||
143 | } | ||
144 | |||
145 | static int gk20a_sched_dev_ioctl_get_tsgs_by_pid(struct gk20a_sched_ctrl *sched, | ||
146 | struct nvgpu_sched_get_tsgs_by_pid_args *arg) | ||
147 | { | ||
148 | struct gk20a *g = sched->g; | ||
149 | struct fifo_gk20a *f = &g->fifo; | ||
150 | struct tsg_gk20a *tsg; | ||
151 | u64 *bitmap; | ||
152 | unsigned int tsgid; | ||
153 | /* pid at user level corresponds to kernel tgid */ | ||
154 | pid_t tgid = (pid_t)arg->pid; | ||
155 | int err = 0; | ||
156 | |||
157 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "pid=%d size=%u buffer=%llx", | ||
158 | (pid_t)arg->pid, arg->size, arg->buffer); | ||
159 | |||
160 | if ((arg->size < sched->bitmap_size) || (!arg->buffer)) { | ||
161 | arg->size = sched->bitmap_size; | ||
162 | return -ENOSPC; | ||
163 | } | ||
164 | |||
165 | bitmap = nvgpu_kzalloc(sched->g, sched->bitmap_size); | ||
166 | if (!bitmap) | ||
167 | return -ENOMEM; | ||
168 | |||
169 | nvgpu_mutex_acquire(&sched->status_lock); | ||
170 | for (tsgid = 0; tsgid < f->num_channels; tsgid++) { | ||
171 | if (NVGPU_SCHED_ISSET(tsgid, sched->active_tsg_bitmap)) { | ||
172 | tsg = &f->tsg[tsgid]; | ||
173 | if (tsg->tgid == tgid) | ||
174 | NVGPU_SCHED_SET(tsgid, bitmap); | ||
175 | } | ||
176 | } | ||
177 | nvgpu_mutex_release(&sched->status_lock); | ||
178 | |||
179 | if (copy_to_user((void __user *)(uintptr_t)arg->buffer, | ||
180 | bitmap, sched->bitmap_size)) | ||
181 | err = -EFAULT; | ||
182 | |||
183 | nvgpu_kfree(sched->g, bitmap); | ||
184 | |||
185 | return err; | ||
186 | } | ||
187 | |||
188 | static int gk20a_sched_dev_ioctl_get_params(struct gk20a_sched_ctrl *sched, | ||
189 | struct nvgpu_sched_tsg_get_params_args *arg) | ||
190 | { | ||
191 | struct gk20a *g = sched->g; | ||
192 | struct fifo_gk20a *f = &g->fifo; | ||
193 | struct tsg_gk20a *tsg; | ||
194 | u32 tsgid = arg->tsgid; | ||
195 | |||
196 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); | ||
197 | |||
198 | if (tsgid >= f->num_channels) | ||
199 | return -EINVAL; | ||
200 | |||
201 | nvgpu_speculation_barrier(); | ||
202 | |||
203 | tsg = &f->tsg[tsgid]; | ||
204 | if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) | ||
205 | return -ENXIO; | ||
206 | |||
207 | arg->pid = tsg->tgid; /* kernel tgid corresponds to user pid */ | ||
208 | arg->runlist_interleave = tsg->interleave_level; | ||
209 | arg->timeslice = tsg->timeslice_us; | ||
210 | |||
211 | arg->graphics_preempt_mode = | ||
212 | tsg->gr_ctx.graphics_preempt_mode; | ||
213 | arg->compute_preempt_mode = | ||
214 | tsg->gr_ctx.compute_preempt_mode; | ||
215 | |||
216 | nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); | ||
217 | |||
218 | return 0; | ||
219 | } | ||
220 | |||
221 | static int gk20a_sched_dev_ioctl_tsg_set_timeslice( | ||
222 | struct gk20a_sched_ctrl *sched, | ||
223 | struct nvgpu_sched_tsg_timeslice_args *arg) | ||
224 | { | ||
225 | struct gk20a *g = sched->g; | ||
226 | struct fifo_gk20a *f = &g->fifo; | ||
227 | struct tsg_gk20a *tsg; | ||
228 | u32 tsgid = arg->tsgid; | ||
229 | int err; | ||
230 | |||
231 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); | ||
232 | |||
233 | if (tsgid >= f->num_channels) | ||
234 | return -EINVAL; | ||
235 | |||
236 | nvgpu_speculation_barrier(); | ||
237 | |||
238 | tsg = &f->tsg[tsgid]; | ||
239 | if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) | ||
240 | return -ENXIO; | ||
241 | |||
242 | err = gk20a_busy(g); | ||
243 | if (err) | ||
244 | goto done; | ||
245 | |||
246 | err = gk20a_tsg_set_timeslice(tsg, arg->timeslice); | ||
247 | |||
248 | gk20a_idle(g); | ||
249 | |||
250 | done: | ||
251 | nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); | ||
252 | |||
253 | return err; | ||
254 | } | ||
255 | |||
256 | static int gk20a_sched_dev_ioctl_tsg_set_runlist_interleave( | ||
257 | struct gk20a_sched_ctrl *sched, | ||
258 | struct nvgpu_sched_tsg_runlist_interleave_args *arg) | ||
259 | { | ||
260 | struct gk20a *g = sched->g; | ||
261 | struct fifo_gk20a *f = &g->fifo; | ||
262 | struct tsg_gk20a *tsg; | ||
263 | u32 tsgid = arg->tsgid; | ||
264 | int err; | ||
265 | |||
266 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); | ||
267 | |||
268 | if (tsgid >= f->num_channels) | ||
269 | return -EINVAL; | ||
270 | |||
271 | nvgpu_speculation_barrier(); | ||
272 | |||
273 | tsg = &f->tsg[tsgid]; | ||
274 | if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) | ||
275 | return -ENXIO; | ||
276 | |||
277 | err = gk20a_busy(g); | ||
278 | if (err) | ||
279 | goto done; | ||
280 | |||
281 | err = gk20a_tsg_set_runlist_interleave(tsg, arg->runlist_interleave); | ||
282 | |||
283 | gk20a_idle(g); | ||
284 | |||
285 | done: | ||
286 | nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); | ||
287 | |||
288 | return err; | ||
289 | } | ||
290 | |||
291 | static int gk20a_sched_dev_ioctl_lock_control(struct gk20a_sched_ctrl *sched) | ||
292 | { | ||
293 | struct gk20a *g = sched->g; | ||
294 | |||
295 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, " "); | ||
296 | |||
297 | nvgpu_mutex_acquire(&sched->control_lock); | ||
298 | sched->control_locked = true; | ||
299 | nvgpu_mutex_release(&sched->control_lock); | ||
300 | return 0; | ||
301 | } | ||
302 | |||
303 | static int gk20a_sched_dev_ioctl_unlock_control(struct gk20a_sched_ctrl *sched) | ||
304 | { | ||
305 | struct gk20a *g = sched->g; | ||
306 | |||
307 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, " "); | ||
308 | |||
309 | nvgpu_mutex_acquire(&sched->control_lock); | ||
310 | sched->control_locked = false; | ||
311 | nvgpu_mutex_release(&sched->control_lock); | ||
312 | return 0; | ||
313 | } | ||
314 | |||
315 | static int gk20a_sched_dev_ioctl_get_api_version(struct gk20a_sched_ctrl *sched, | ||
316 | struct nvgpu_sched_api_version_args *args) | ||
317 | { | ||
318 | struct gk20a *g = sched->g; | ||
319 | |||
320 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, " "); | ||
321 | |||
322 | args->version = NVGPU_SCHED_API_VERSION; | ||
323 | return 0; | ||
324 | } | ||
325 | |||
326 | static int gk20a_sched_dev_ioctl_get_tsg(struct gk20a_sched_ctrl *sched, | ||
327 | struct nvgpu_sched_tsg_refcount_args *arg) | ||
328 | { | ||
329 | struct gk20a *g = sched->g; | ||
330 | struct fifo_gk20a *f = &g->fifo; | ||
331 | struct tsg_gk20a *tsg; | ||
332 | u32 tsgid = arg->tsgid; | ||
333 | |||
334 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); | ||
335 | |||
336 | if (tsgid >= f->num_channels) | ||
337 | return -EINVAL; | ||
338 | |||
339 | nvgpu_speculation_barrier(); | ||
340 | |||
341 | tsg = &f->tsg[tsgid]; | ||
342 | if (!nvgpu_ref_get_unless_zero(&tsg->refcount)) | ||
343 | return -ENXIO; | ||
344 | |||
345 | nvgpu_mutex_acquire(&sched->status_lock); | ||
346 | if (NVGPU_SCHED_ISSET(tsgid, sched->ref_tsg_bitmap)) { | ||
347 | nvgpu_warn(g, "tsgid=%d already referenced", tsgid); | ||
348 | /* unlock status_lock as nvgpu_ioctl_tsg_release locks it */ | ||
349 | nvgpu_mutex_release(&sched->status_lock); | ||
350 | nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); | ||
351 | return -ENXIO; | ||
352 | } | ||
353 | |||
354 | /* keep reference on TSG, will be released on | ||
355 | * NVGPU_SCHED_IOCTL_PUT_TSG ioctl, or close | ||
356 | */ | ||
357 | NVGPU_SCHED_SET(tsgid, sched->ref_tsg_bitmap); | ||
358 | nvgpu_mutex_release(&sched->status_lock); | ||
359 | |||
360 | return 0; | ||
361 | } | ||
362 | |||
363 | static int gk20a_sched_dev_ioctl_put_tsg(struct gk20a_sched_ctrl *sched, | ||
364 | struct nvgpu_sched_tsg_refcount_args *arg) | ||
365 | { | ||
366 | struct gk20a *g = sched->g; | ||
367 | struct fifo_gk20a *f = &g->fifo; | ||
368 | struct tsg_gk20a *tsg; | ||
369 | u32 tsgid = arg->tsgid; | ||
370 | |||
371 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsgid); | ||
372 | |||
373 | if (tsgid >= f->num_channels) | ||
374 | return -EINVAL; | ||
375 | |||
376 | nvgpu_speculation_barrier(); | ||
377 | |||
378 | nvgpu_mutex_acquire(&sched->status_lock); | ||
379 | if (!NVGPU_SCHED_ISSET(tsgid, sched->ref_tsg_bitmap)) { | ||
380 | nvgpu_mutex_release(&sched->status_lock); | ||
381 | nvgpu_warn(g, "tsgid=%d not previously referenced", tsgid); | ||
382 | return -ENXIO; | ||
383 | } | ||
384 | NVGPU_SCHED_CLR(tsgid, sched->ref_tsg_bitmap); | ||
385 | nvgpu_mutex_release(&sched->status_lock); | ||
386 | |||
387 | tsg = &f->tsg[tsgid]; | ||
388 | nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); | ||
389 | |||
390 | return 0; | ||
391 | } | ||
392 | |||
393 | int gk20a_sched_dev_open(struct inode *inode, struct file *filp) | ||
394 | { | ||
395 | struct nvgpu_os_linux *l = container_of(inode->i_cdev, | ||
396 | struct nvgpu_os_linux, sched.cdev); | ||
397 | struct gk20a *g; | ||
398 | struct gk20a_sched_ctrl *sched; | ||
399 | int err = 0; | ||
400 | |||
401 | g = gk20a_get(&l->g); | ||
402 | if (!g) | ||
403 | return -ENODEV; | ||
404 | sched = &l->sched_ctrl; | ||
405 | |||
406 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "g=%p", g); | ||
407 | |||
408 | if (!sched->sw_ready) { | ||
409 | err = gk20a_busy(g); | ||
410 | if (err) | ||
411 | goto free_ref; | ||
412 | |||
413 | gk20a_idle(g); | ||
414 | } | ||
415 | |||
416 | if (!nvgpu_mutex_tryacquire(&sched->busy_lock)) { | ||
417 | err = -EBUSY; | ||
418 | goto free_ref; | ||
419 | } | ||
420 | |||
421 | memcpy(sched->recent_tsg_bitmap, sched->active_tsg_bitmap, | ||
422 | sched->bitmap_size); | ||
423 | memset(sched->ref_tsg_bitmap, 0, sched->bitmap_size); | ||
424 | |||
425 | filp->private_data = sched; | ||
426 | nvgpu_log(g, gpu_dbg_sched, "filp=%p sched=%p", filp, sched); | ||
427 | |||
428 | free_ref: | ||
429 | if (err) | ||
430 | gk20a_put(g); | ||
431 | return err; | ||
432 | } | ||
433 | |||
434 | long gk20a_sched_dev_ioctl(struct file *filp, unsigned int cmd, | ||
435 | unsigned long arg) | ||
436 | { | ||
437 | struct gk20a_sched_ctrl *sched = filp->private_data; | ||
438 | struct gk20a *g = sched->g; | ||
439 | u8 buf[NVGPU_CTXSW_IOCTL_MAX_ARG_SIZE]; | ||
440 | int err = 0; | ||
441 | |||
442 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "nr=%d", _IOC_NR(cmd)); | ||
443 | |||
444 | if ((_IOC_TYPE(cmd) != NVGPU_SCHED_IOCTL_MAGIC) || | ||
445 | (_IOC_NR(cmd) == 0) || | ||
446 | (_IOC_NR(cmd) > NVGPU_SCHED_IOCTL_LAST) || | ||
447 | (_IOC_SIZE(cmd) > NVGPU_SCHED_IOCTL_MAX_ARG_SIZE)) | ||
448 | return -EINVAL; | ||
449 | |||
450 | memset(buf, 0, sizeof(buf)); | ||
451 | if (_IOC_DIR(cmd) & _IOC_WRITE) { | ||
452 | if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd))) | ||
453 | return -EFAULT; | ||
454 | } | ||
455 | |||
456 | switch (cmd) { | ||
457 | case NVGPU_SCHED_IOCTL_GET_TSGS: | ||
458 | err = gk20a_sched_dev_ioctl_get_tsgs(sched, | ||
459 | (struct nvgpu_sched_get_tsgs_args *)buf); | ||
460 | break; | ||
461 | case NVGPU_SCHED_IOCTL_GET_RECENT_TSGS: | ||
462 | err = gk20a_sched_dev_ioctl_get_recent_tsgs(sched, | ||
463 | (struct nvgpu_sched_get_tsgs_args *)buf); | ||
464 | break; | ||
465 | case NVGPU_SCHED_IOCTL_GET_TSGS_BY_PID: | ||
466 | err = gk20a_sched_dev_ioctl_get_tsgs_by_pid(sched, | ||
467 | (struct nvgpu_sched_get_tsgs_by_pid_args *)buf); | ||
468 | break; | ||
469 | case NVGPU_SCHED_IOCTL_TSG_GET_PARAMS: | ||
470 | err = gk20a_sched_dev_ioctl_get_params(sched, | ||
471 | (struct nvgpu_sched_tsg_get_params_args *)buf); | ||
472 | break; | ||
473 | case NVGPU_SCHED_IOCTL_TSG_SET_TIMESLICE: | ||
474 | err = gk20a_sched_dev_ioctl_tsg_set_timeslice(sched, | ||
475 | (struct nvgpu_sched_tsg_timeslice_args *)buf); | ||
476 | break; | ||
477 | case NVGPU_SCHED_IOCTL_TSG_SET_RUNLIST_INTERLEAVE: | ||
478 | err = gk20a_sched_dev_ioctl_tsg_set_runlist_interleave(sched, | ||
479 | (struct nvgpu_sched_tsg_runlist_interleave_args *)buf); | ||
480 | break; | ||
481 | case NVGPU_SCHED_IOCTL_LOCK_CONTROL: | ||
482 | err = gk20a_sched_dev_ioctl_lock_control(sched); | ||
483 | break; | ||
484 | case NVGPU_SCHED_IOCTL_UNLOCK_CONTROL: | ||
485 | err = gk20a_sched_dev_ioctl_unlock_control(sched); | ||
486 | break; | ||
487 | case NVGPU_SCHED_IOCTL_GET_API_VERSION: | ||
488 | err = gk20a_sched_dev_ioctl_get_api_version(sched, | ||
489 | (struct nvgpu_sched_api_version_args *)buf); | ||
490 | break; | ||
491 | case NVGPU_SCHED_IOCTL_GET_TSG: | ||
492 | err = gk20a_sched_dev_ioctl_get_tsg(sched, | ||
493 | (struct nvgpu_sched_tsg_refcount_args *)buf); | ||
494 | break; | ||
495 | case NVGPU_SCHED_IOCTL_PUT_TSG: | ||
496 | err = gk20a_sched_dev_ioctl_put_tsg(sched, | ||
497 | (struct nvgpu_sched_tsg_refcount_args *)buf); | ||
498 | break; | ||
499 | default: | ||
500 | nvgpu_log_info(g, "unrecognized gpu ioctl cmd: 0x%x", cmd); | ||
501 | err = -ENOTTY; | ||
502 | } | ||
503 | |||
504 | /* Some ioctls like NVGPU_SCHED_IOCTL_GET_TSGS might be called on | ||
505 | * purpose with NULL buffer and/or zero size to discover TSG bitmap | ||
506 | * size. We need to update user arguments in this case too, even | ||
507 | * if we return an error. | ||
508 | */ | ||
509 | if ((!err || (err == -ENOSPC)) && (_IOC_DIR(cmd) & _IOC_READ)) { | ||
510 | if (copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd))) | ||
511 | err = -EFAULT; | ||
512 | } | ||
513 | |||
514 | return err; | ||
515 | } | ||
516 | |||
517 | int gk20a_sched_dev_release(struct inode *inode, struct file *filp) | ||
518 | { | ||
519 | struct gk20a_sched_ctrl *sched = filp->private_data; | ||
520 | struct gk20a *g = sched->g; | ||
521 | struct fifo_gk20a *f = &g->fifo; | ||
522 | struct tsg_gk20a *tsg; | ||
523 | unsigned int tsgid; | ||
524 | |||
525 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "sched: %p", sched); | ||
526 | |||
527 | /* release any reference to TSGs */ | ||
528 | for (tsgid = 0; tsgid < f->num_channels; tsgid++) { | ||
529 | if (NVGPU_SCHED_ISSET(tsgid, sched->ref_tsg_bitmap)) { | ||
530 | tsg = &f->tsg[tsgid]; | ||
531 | nvgpu_ref_put(&tsg->refcount, nvgpu_ioctl_tsg_release); | ||
532 | } | ||
533 | } | ||
534 | |||
535 | /* unlock control */ | ||
536 | nvgpu_mutex_acquire(&sched->control_lock); | ||
537 | sched->control_locked = false; | ||
538 | nvgpu_mutex_release(&sched->control_lock); | ||
539 | |||
540 | nvgpu_mutex_release(&sched->busy_lock); | ||
541 | gk20a_put(g); | ||
542 | return 0; | ||
543 | } | ||
544 | |||
545 | void gk20a_sched_ctrl_tsg_added(struct gk20a *g, struct tsg_gk20a *tsg) | ||
546 | { | ||
547 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
548 | struct gk20a_sched_ctrl *sched = &l->sched_ctrl; | ||
549 | int err; | ||
550 | |||
551 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsg->tsgid); | ||
552 | |||
553 | if (!sched->sw_ready) { | ||
554 | err = gk20a_busy(g); | ||
555 | if (err) { | ||
556 | WARN_ON(err); | ||
557 | return; | ||
558 | } | ||
559 | |||
560 | gk20a_idle(g); | ||
561 | } | ||
562 | |||
563 | nvgpu_mutex_acquire(&sched->status_lock); | ||
564 | NVGPU_SCHED_SET(tsg->tsgid, sched->active_tsg_bitmap); | ||
565 | NVGPU_SCHED_SET(tsg->tsgid, sched->recent_tsg_bitmap); | ||
566 | sched->status |= NVGPU_SCHED_STATUS_TSG_OPEN; | ||
567 | nvgpu_mutex_release(&sched->status_lock); | ||
568 | nvgpu_cond_signal_interruptible(&sched->readout_wq); | ||
569 | } | ||
570 | |||
571 | void gk20a_sched_ctrl_tsg_removed(struct gk20a *g, struct tsg_gk20a *tsg) | ||
572 | { | ||
573 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
574 | struct gk20a_sched_ctrl *sched = &l->sched_ctrl; | ||
575 | |||
576 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsg->tsgid); | ||
577 | |||
578 | nvgpu_mutex_acquire(&sched->status_lock); | ||
579 | NVGPU_SCHED_CLR(tsg->tsgid, sched->active_tsg_bitmap); | ||
580 | |||
581 | /* clear recent_tsg_bitmap as well: if app manager did not | ||
582 | * notice that TSG was previously added, no need to notify it | ||
583 | * if the TSG has been released in the meantime. If the | ||
584 | * TSG gets reallocated, app manager will be notified as usual. | ||
585 | */ | ||
586 | NVGPU_SCHED_CLR(tsg->tsgid, sched->recent_tsg_bitmap); | ||
587 | |||
588 | /* do not set event_pending, we only want to notify app manager | ||
589 | * when TSGs are added, so that it can apply sched params | ||
590 | */ | ||
591 | nvgpu_mutex_release(&sched->status_lock); | ||
592 | } | ||
593 | |||
594 | int gk20a_sched_ctrl_init(struct gk20a *g) | ||
595 | { | ||
596 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
597 | struct gk20a_sched_ctrl *sched = &l->sched_ctrl; | ||
598 | struct fifo_gk20a *f = &g->fifo; | ||
599 | int err; | ||
600 | |||
601 | if (sched->sw_ready) | ||
602 | return 0; | ||
603 | |||
604 | sched->g = g; | ||
605 | sched->bitmap_size = roundup(f->num_channels, 64) / 8; | ||
606 | sched->status = 0; | ||
607 | |||
608 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "g=%p sched=%p size=%zu", | ||
609 | g, sched, sched->bitmap_size); | ||
610 | |||
611 | sched->active_tsg_bitmap = nvgpu_kzalloc(g, sched->bitmap_size); | ||
612 | if (!sched->active_tsg_bitmap) | ||
613 | return -ENOMEM; | ||
614 | |||
615 | sched->recent_tsg_bitmap = nvgpu_kzalloc(g, sched->bitmap_size); | ||
616 | if (!sched->recent_tsg_bitmap) { | ||
617 | err = -ENOMEM; | ||
618 | goto free_active; | ||
619 | } | ||
620 | |||
621 | sched->ref_tsg_bitmap = nvgpu_kzalloc(g, sched->bitmap_size); | ||
622 | if (!sched->ref_tsg_bitmap) { | ||
623 | err = -ENOMEM; | ||
624 | goto free_recent; | ||
625 | } | ||
626 | |||
627 | nvgpu_cond_init(&sched->readout_wq); | ||
628 | |||
629 | err = nvgpu_mutex_init(&sched->status_lock); | ||
630 | if (err) | ||
631 | goto free_ref; | ||
632 | |||
633 | err = nvgpu_mutex_init(&sched->control_lock); | ||
634 | if (err) | ||
635 | goto free_status_lock; | ||
636 | |||
637 | err = nvgpu_mutex_init(&sched->busy_lock); | ||
638 | if (err) | ||
639 | goto free_control_lock; | ||
640 | |||
641 | sched->sw_ready = true; | ||
642 | |||
643 | return 0; | ||
644 | |||
645 | free_control_lock: | ||
646 | nvgpu_mutex_destroy(&sched->control_lock); | ||
647 | free_status_lock: | ||
648 | nvgpu_mutex_destroy(&sched->status_lock); | ||
649 | free_ref: | ||
650 | nvgpu_kfree(g, sched->ref_tsg_bitmap); | ||
651 | free_recent: | ||
652 | nvgpu_kfree(g, sched->recent_tsg_bitmap); | ||
653 | free_active: | ||
654 | nvgpu_kfree(g, sched->active_tsg_bitmap); | ||
655 | |||
656 | return err; | ||
657 | } | ||
658 | |||
659 | void gk20a_sched_ctrl_cleanup(struct gk20a *g) | ||
660 | { | ||
661 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
662 | struct gk20a_sched_ctrl *sched = &l->sched_ctrl; | ||
663 | |||
664 | nvgpu_kfree(g, sched->active_tsg_bitmap); | ||
665 | nvgpu_kfree(g, sched->recent_tsg_bitmap); | ||
666 | nvgpu_kfree(g, sched->ref_tsg_bitmap); | ||
667 | sched->active_tsg_bitmap = NULL; | ||
668 | sched->recent_tsg_bitmap = NULL; | ||
669 | sched->ref_tsg_bitmap = NULL; | ||
670 | |||
671 | nvgpu_mutex_destroy(&sched->status_lock); | ||
672 | nvgpu_mutex_destroy(&sched->control_lock); | ||
673 | nvgpu_mutex_destroy(&sched->busy_lock); | ||
674 | |||
675 | sched->sw_ready = false; | ||
676 | } | ||