diff options
Diffstat (limited to 'drivers/gpu/nvgpu/os/linux/module.c')
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/module.c | 1365 |
1 files changed, 1365 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/os/linux/module.c b/drivers/gpu/nvgpu/os/linux/module.c new file mode 100644 index 00000000..af71cc81 --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/module.c | |||
@@ -0,0 +1,1365 @@ | |||
1 | /* | ||
2 | * GK20A Graphics | ||
3 | * | ||
4 | * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * This program is free software; you can redistribute it and/or modify it | ||
7 | * under the terms and conditions of the GNU General Public License, | ||
8 | * version 2, as published by the Free Software Foundation. | ||
9 | * | ||
10 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
13 | * more details. | ||
14 | * | ||
15 | * You should have received a copy of the GNU General Public License | ||
16 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
17 | */ | ||
18 | |||
19 | #include <linux/module.h> | ||
20 | #include <linux/of.h> | ||
21 | #include <linux/of_device.h> | ||
22 | #include <linux/of_platform.h> | ||
23 | #include <linux/of_address.h> | ||
24 | #include <linux/interrupt.h> | ||
25 | #include <linux/pm_runtime.h> | ||
26 | #include <linux/reset.h> | ||
27 | #include <linux/platform/tegra/common.h> | ||
28 | #include <linux/pci.h> | ||
29 | |||
30 | #include <uapi/linux/nvgpu.h> | ||
31 | #include <dt-bindings/soc/gm20b-fuse.h> | ||
32 | #include <dt-bindings/soc/gp10b-fuse.h> | ||
33 | |||
34 | #include <soc/tegra/fuse.h> | ||
35 | |||
36 | #include <nvgpu/dma.h> | ||
37 | #include <nvgpu/kmem.h> | ||
38 | #include <nvgpu/nvgpu_common.h> | ||
39 | #include <nvgpu/soc.h> | ||
40 | #include <nvgpu/enabled.h> | ||
41 | #include <nvgpu/debug.h> | ||
42 | #include <nvgpu/ctxsw_trace.h> | ||
43 | #include <nvgpu/vidmem.h> | ||
44 | #include <nvgpu/sim.h> | ||
45 | |||
46 | #include "platform_gk20a.h" | ||
47 | #include "sysfs.h" | ||
48 | #include "vgpu/vgpu_linux.h" | ||
49 | #include "scale.h" | ||
50 | #include "pci.h" | ||
51 | #include "module.h" | ||
52 | #include "module_usermode.h" | ||
53 | #include "intr.h" | ||
54 | #include "ioctl.h" | ||
55 | |||
56 | #include "os_linux.h" | ||
57 | #include "ctxsw_trace.h" | ||
58 | #include "driver_common.h" | ||
59 | #include "channel.h" | ||
60 | |||
61 | #ifdef CONFIG_NVGPU_SUPPORT_CDE | ||
62 | #include "cde.h" | ||
63 | #endif | ||
64 | |||
65 | #define CLASS_NAME "nvidia-gpu" | ||
66 | /* TODO: Change to e.g. "nvidia-gpu%s" once we have symlinks in place. */ | ||
67 | |||
68 | #define GK20A_WAIT_FOR_IDLE_MS 2000 | ||
69 | |||
70 | #define CREATE_TRACE_POINTS | ||
71 | #include <trace/events/gk20a.h> | ||
72 | |||
73 | |||
74 | struct device_node *nvgpu_get_node(struct gk20a *g) | ||
75 | { | ||
76 | struct device *dev = dev_from_gk20a(g); | ||
77 | |||
78 | if (dev_is_pci(dev)) { | ||
79 | struct pci_bus *bus = to_pci_dev(dev)->bus; | ||
80 | |||
81 | while (!pci_is_root_bus(bus)) | ||
82 | bus = bus->parent; | ||
83 | |||
84 | return bus->bridge->parent->of_node; | ||
85 | } | ||
86 | |||
87 | return dev->of_node; | ||
88 | } | ||
89 | |||
90 | void gk20a_busy_noresume(struct gk20a *g) | ||
91 | { | ||
92 | pm_runtime_get_noresume(dev_from_gk20a(g)); | ||
93 | } | ||
94 | |||
95 | int gk20a_busy(struct gk20a *g) | ||
96 | { | ||
97 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
98 | int ret = 0; | ||
99 | struct device *dev; | ||
100 | |||
101 | if (!g) | ||
102 | return -ENODEV; | ||
103 | |||
104 | atomic_inc(&g->usage_count.atomic_var); | ||
105 | |||
106 | down_read(&l->busy_lock); | ||
107 | |||
108 | if (!gk20a_can_busy(g)) { | ||
109 | ret = -ENODEV; | ||
110 | atomic_dec(&g->usage_count.atomic_var); | ||
111 | goto fail; | ||
112 | } | ||
113 | |||
114 | dev = dev_from_gk20a(g); | ||
115 | |||
116 | if (pm_runtime_enabled(dev)) { | ||
117 | /* Increment usage count and attempt to resume device */ | ||
118 | ret = pm_runtime_get_sync(dev); | ||
119 | if (ret < 0) { | ||
120 | /* Mark suspended so runtime pm will retry later */ | ||
121 | pm_runtime_set_suspended(dev); | ||
122 | pm_runtime_put_noidle(dev); | ||
123 | atomic_dec(&g->usage_count.atomic_var); | ||
124 | goto fail; | ||
125 | } | ||
126 | } else { | ||
127 | nvgpu_mutex_acquire(&g->poweron_lock); | ||
128 | if (!g->power_on) { | ||
129 | ret = gk20a_gpu_is_virtual(dev) ? | ||
130 | vgpu_pm_finalize_poweron(dev) | ||
131 | : gk20a_pm_finalize_poweron(dev); | ||
132 | if (ret) { | ||
133 | atomic_dec(&g->usage_count.atomic_var); | ||
134 | nvgpu_mutex_release(&g->poweron_lock); | ||
135 | goto fail; | ||
136 | } | ||
137 | } | ||
138 | nvgpu_mutex_release(&g->poweron_lock); | ||
139 | } | ||
140 | |||
141 | fail: | ||
142 | up_read(&l->busy_lock); | ||
143 | |||
144 | return ret < 0 ? ret : 0; | ||
145 | } | ||
146 | |||
147 | void gk20a_idle_nosuspend(struct gk20a *g) | ||
148 | { | ||
149 | pm_runtime_put_noidle(dev_from_gk20a(g)); | ||
150 | } | ||
151 | |||
152 | void gk20a_idle(struct gk20a *g) | ||
153 | { | ||
154 | struct device *dev; | ||
155 | |||
156 | atomic_dec(&g->usage_count.atomic_var); | ||
157 | |||
158 | dev = dev_from_gk20a(g); | ||
159 | |||
160 | if (!(dev && gk20a_can_busy(g))) | ||
161 | return; | ||
162 | |||
163 | if (pm_runtime_enabled(dev)) { | ||
164 | pm_runtime_mark_last_busy(dev); | ||
165 | pm_runtime_put_sync_autosuspend(dev); | ||
166 | } | ||
167 | } | ||
168 | |||
169 | /* | ||
170 | * Undoes gk20a_lockout_registers(). | ||
171 | */ | ||
172 | static int gk20a_restore_registers(struct gk20a *g) | ||
173 | { | ||
174 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
175 | |||
176 | l->regs = l->regs_saved; | ||
177 | l->bar1 = l->bar1_saved; | ||
178 | |||
179 | nvgpu_restore_usermode_registers(g); | ||
180 | |||
181 | return 0; | ||
182 | } | ||
183 | |||
184 | static int nvgpu_init_os_linux_ops(struct nvgpu_os_linux *l) | ||
185 | { | ||
186 | int err = 0; | ||
187 | |||
188 | #ifdef CONFIG_NVGPU_SUPPORT_CDE | ||
189 | err = nvgpu_cde_init_ops(l); | ||
190 | #endif | ||
191 | |||
192 | return err; | ||
193 | } | ||
194 | |||
195 | int nvgpu_finalize_poweron_linux(struct nvgpu_os_linux *l) | ||
196 | { | ||
197 | struct gk20a *g = &l->g; | ||
198 | int err; | ||
199 | |||
200 | if (l->init_done) | ||
201 | return 0; | ||
202 | |||
203 | err = nvgpu_init_channel_support_linux(l); | ||
204 | if (err) { | ||
205 | nvgpu_err(g, "failed to init linux channel support"); | ||
206 | return err; | ||
207 | } | ||
208 | |||
209 | l->init_done = true; | ||
210 | |||
211 | return 0; | ||
212 | } | ||
213 | |||
214 | int gk20a_pm_finalize_poweron(struct device *dev) | ||
215 | { | ||
216 | struct gk20a *g = get_gk20a(dev); | ||
217 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
218 | struct gk20a_platform *platform = gk20a_get_platform(dev); | ||
219 | int err; | ||
220 | |||
221 | nvgpu_log_fn(g, " "); | ||
222 | |||
223 | if (g->power_on) | ||
224 | return 0; | ||
225 | |||
226 | trace_gk20a_finalize_poweron(dev_name(dev)); | ||
227 | |||
228 | /* Increment platform power refcount */ | ||
229 | if (platform->busy) { | ||
230 | err = platform->busy(dev); | ||
231 | if (err < 0) { | ||
232 | nvgpu_err(g, "failed to poweron platform dependency"); | ||
233 | return err; | ||
234 | } | ||
235 | } | ||
236 | |||
237 | err = gk20a_restore_registers(g); | ||
238 | if (err) | ||
239 | return err; | ||
240 | |||
241 | /* Enable interrupt workqueue */ | ||
242 | if (!l->nonstall_work_queue) { | ||
243 | l->nonstall_work_queue = alloc_workqueue("%s", | ||
244 | WQ_HIGHPRI, 1, "mc_nonstall"); | ||
245 | INIT_WORK(&l->nonstall_fn_work, nvgpu_intr_nonstall_cb); | ||
246 | } | ||
247 | |||
248 | err = gk20a_detect_chip(g); | ||
249 | if (err) | ||
250 | return err; | ||
251 | |||
252 | if (g->sim) { | ||
253 | if (g->sim->sim_init_late) | ||
254 | g->sim->sim_init_late(g); | ||
255 | } | ||
256 | |||
257 | err = gk20a_finalize_poweron(g); | ||
258 | if (err) | ||
259 | goto done; | ||
260 | |||
261 | err = nvgpu_finalize_poweron_linux(l); | ||
262 | if (err) | ||
263 | goto done; | ||
264 | |||
265 | nvgpu_init_mm_ce_context(g); | ||
266 | |||
267 | nvgpu_vidmem_thread_unpause(&g->mm); | ||
268 | |||
269 | /* Initialise scaling: it will initialize scaling drive only once */ | ||
270 | if (IS_ENABLED(CONFIG_GK20A_DEVFREQ) && | ||
271 | nvgpu_platform_is_silicon(g)) { | ||
272 | gk20a_scale_init(dev); | ||
273 | if (platform->initscale) | ||
274 | platform->initscale(dev); | ||
275 | } | ||
276 | |||
277 | trace_gk20a_finalize_poweron_done(dev_name(dev)); | ||
278 | |||
279 | err = nvgpu_init_os_linux_ops(l); | ||
280 | if (err) | ||
281 | goto done; | ||
282 | |||
283 | enable_irq(g->irq_stall); | ||
284 | if (g->irq_stall != g->irq_nonstall) | ||
285 | enable_irq(g->irq_nonstall); | ||
286 | g->irqs_enabled = 1; | ||
287 | |||
288 | gk20a_scale_resume(dev_from_gk20a(g)); | ||
289 | |||
290 | #ifdef CONFIG_NVGPU_SUPPORT_CDE | ||
291 | if (platform->has_cde) | ||
292 | gk20a_init_cde_support(l); | ||
293 | #endif | ||
294 | |||
295 | err = gk20a_sched_ctrl_init(g); | ||
296 | if (err) { | ||
297 | nvgpu_err(g, "failed to init sched control"); | ||
298 | return err; | ||
299 | } | ||
300 | |||
301 | g->sw_ready = true; | ||
302 | |||
303 | done: | ||
304 | if (err) | ||
305 | g->power_on = false; | ||
306 | |||
307 | return err; | ||
308 | } | ||
309 | |||
310 | /* | ||
311 | * Locks out the driver from accessing GPU registers. This prevents access to | ||
312 | * thse registers after the GPU has been clock or power gated. This should help | ||
313 | * find annoying bugs where register reads and writes are silently dropped | ||
314 | * after the GPU has been turned off. On older chips these reads and writes can | ||
315 | * also lock the entire CPU up. | ||
316 | */ | ||
317 | static int gk20a_lockout_registers(struct gk20a *g) | ||
318 | { | ||
319 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
320 | |||
321 | l->regs = NULL; | ||
322 | l->bar1 = NULL; | ||
323 | |||
324 | nvgpu_lockout_usermode_registers(g); | ||
325 | |||
326 | return 0; | ||
327 | } | ||
328 | |||
329 | static int gk20a_pm_prepare_poweroff(struct device *dev) | ||
330 | { | ||
331 | struct gk20a *g = get_gk20a(dev); | ||
332 | #ifdef CONFIG_NVGPU_SUPPORT_CDE | ||
333 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
334 | #endif | ||
335 | int ret = 0; | ||
336 | struct gk20a_platform *platform = gk20a_get_platform(dev); | ||
337 | bool irqs_enabled; | ||
338 | |||
339 | nvgpu_log_fn(g, " "); | ||
340 | |||
341 | nvgpu_mutex_acquire(&g->poweroff_lock); | ||
342 | |||
343 | if (!g->power_on) | ||
344 | goto done; | ||
345 | |||
346 | /* disable IRQs and wait for completion */ | ||
347 | irqs_enabled = g->irqs_enabled; | ||
348 | if (irqs_enabled) { | ||
349 | disable_irq(g->irq_stall); | ||
350 | if (g->irq_stall != g->irq_nonstall) | ||
351 | disable_irq(g->irq_nonstall); | ||
352 | g->irqs_enabled = 0; | ||
353 | } | ||
354 | |||
355 | gk20a_scale_suspend(dev); | ||
356 | |||
357 | #ifdef CONFIG_NVGPU_SUPPORT_CDE | ||
358 | gk20a_cde_suspend(l); | ||
359 | #endif | ||
360 | |||
361 | ret = gk20a_prepare_poweroff(g); | ||
362 | if (ret) | ||
363 | goto error; | ||
364 | |||
365 | /* Decrement platform power refcount */ | ||
366 | if (platform->idle) | ||
367 | platform->idle(dev); | ||
368 | |||
369 | /* Stop CPU from accessing the GPU registers. */ | ||
370 | gk20a_lockout_registers(g); | ||
371 | |||
372 | nvgpu_mutex_release(&g->poweroff_lock); | ||
373 | return 0; | ||
374 | |||
375 | error: | ||
376 | /* re-enabled IRQs if previously enabled */ | ||
377 | if (irqs_enabled) { | ||
378 | enable_irq(g->irq_stall); | ||
379 | if (g->irq_stall != g->irq_nonstall) | ||
380 | enable_irq(g->irq_nonstall); | ||
381 | g->irqs_enabled = 1; | ||
382 | } | ||
383 | |||
384 | gk20a_scale_resume(dev); | ||
385 | done: | ||
386 | nvgpu_mutex_release(&g->poweroff_lock); | ||
387 | |||
388 | return ret; | ||
389 | } | ||
390 | |||
391 | static struct of_device_id tegra_gk20a_of_match[] = { | ||
392 | #ifdef CONFIG_TEGRA_GK20A | ||
393 | { .compatible = "nvidia,tegra210-gm20b", | ||
394 | .data = &gm20b_tegra_platform }, | ||
395 | { .compatible = "nvidia,tegra186-gp10b", | ||
396 | .data = &gp10b_tegra_platform }, | ||
397 | { .compatible = "nvidia,gv11b", | ||
398 | .data = &gv11b_tegra_platform }, | ||
399 | #ifdef CONFIG_TEGRA_GR_VIRTUALIZATION | ||
400 | { .compatible = "nvidia,gv11b-vgpu", | ||
401 | .data = &gv11b_vgpu_tegra_platform}, | ||
402 | #endif | ||
403 | #ifdef CONFIG_TEGRA_GR_VIRTUALIZATION | ||
404 | { .compatible = "nvidia,tegra124-gk20a-vgpu", | ||
405 | .data = &vgpu_tegra_platform }, | ||
406 | #endif | ||
407 | #endif | ||
408 | |||
409 | { }, | ||
410 | }; | ||
411 | |||
412 | #ifdef CONFIG_PM | ||
413 | /** | ||
414 | * __gk20a_do_idle() - force the GPU to idle and railgate | ||
415 | * | ||
416 | * In success, this call MUST be balanced by caller with __gk20a_do_unidle() | ||
417 | * | ||
418 | * Acquires two locks : &l->busy_lock and &platform->railgate_lock | ||
419 | * In success, we hold these locks and return | ||
420 | * In failure, we release these locks and return | ||
421 | */ | ||
422 | int __gk20a_do_idle(struct gk20a *g, bool force_reset) | ||
423 | { | ||
424 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
425 | struct device *dev = dev_from_gk20a(g); | ||
426 | struct gk20a_platform *platform = dev_get_drvdata(dev); | ||
427 | struct nvgpu_timeout timeout; | ||
428 | int ref_cnt; | ||
429 | int target_ref_cnt = 0; | ||
430 | bool is_railgated; | ||
431 | int err = 0; | ||
432 | |||
433 | /* | ||
434 | * Hold back deterministic submits and changes to deterministic | ||
435 | * channels - this must be outside the power busy locks. | ||
436 | */ | ||
437 | gk20a_channel_deterministic_idle(g); | ||
438 | |||
439 | /* acquire busy lock to block other busy() calls */ | ||
440 | down_write(&l->busy_lock); | ||
441 | |||
442 | /* acquire railgate lock to prevent unrailgate in midst of do_idle() */ | ||
443 | nvgpu_mutex_acquire(&platform->railgate_lock); | ||
444 | |||
445 | /* check if it is already railgated ? */ | ||
446 | if (platform->is_railgated(dev)) | ||
447 | return 0; | ||
448 | |||
449 | /* | ||
450 | * release railgate_lock, prevent suspend by incrementing usage counter, | ||
451 | * re-acquire railgate_lock | ||
452 | */ | ||
453 | nvgpu_mutex_release(&platform->railgate_lock); | ||
454 | pm_runtime_get_sync(dev); | ||
455 | |||
456 | /* | ||
457 | * One refcount taken in this API | ||
458 | * If User disables rail gating, we take one more | ||
459 | * extra refcount | ||
460 | */ | ||
461 | if (g->can_railgate) | ||
462 | target_ref_cnt = 1; | ||
463 | else | ||
464 | target_ref_cnt = 2; | ||
465 | nvgpu_mutex_acquire(&platform->railgate_lock); | ||
466 | |||
467 | nvgpu_timeout_init(g, &timeout, GK20A_WAIT_FOR_IDLE_MS, | ||
468 | NVGPU_TIMER_CPU_TIMER); | ||
469 | |||
470 | /* check and wait until GPU is idle (with a timeout) */ | ||
471 | do { | ||
472 | nvgpu_usleep_range(1000, 1100); | ||
473 | ref_cnt = atomic_read(&dev->power.usage_count); | ||
474 | } while (ref_cnt != target_ref_cnt && !nvgpu_timeout_expired(&timeout)); | ||
475 | |||
476 | if (ref_cnt != target_ref_cnt) { | ||
477 | nvgpu_err(g, "failed to idle - refcount %d != target_ref_cnt", | ||
478 | ref_cnt); | ||
479 | goto fail_drop_usage_count; | ||
480 | } | ||
481 | |||
482 | /* check if global force_reset flag is set */ | ||
483 | force_reset |= platform->force_reset_in_do_idle; | ||
484 | |||
485 | nvgpu_timeout_init(g, &timeout, GK20A_WAIT_FOR_IDLE_MS, | ||
486 | NVGPU_TIMER_CPU_TIMER); | ||
487 | |||
488 | if (g->can_railgate && !force_reset) { | ||
489 | /* | ||
490 | * Case 1 : GPU railgate is supported | ||
491 | * | ||
492 | * if GPU is now idle, we will have only one ref count, | ||
493 | * drop this ref which will rail gate the GPU | ||
494 | */ | ||
495 | pm_runtime_put_sync(dev); | ||
496 | |||
497 | /* add sufficient delay to allow GPU to rail gate */ | ||
498 | nvgpu_msleep(g->railgate_delay); | ||
499 | |||
500 | /* check in loop if GPU is railgated or not */ | ||
501 | do { | ||
502 | nvgpu_usleep_range(1000, 1100); | ||
503 | is_railgated = platform->is_railgated(dev); | ||
504 | } while (!is_railgated && !nvgpu_timeout_expired(&timeout)); | ||
505 | |||
506 | if (is_railgated) { | ||
507 | return 0; | ||
508 | } else { | ||
509 | nvgpu_err(g, "failed to idle in timeout"); | ||
510 | goto fail_timeout; | ||
511 | } | ||
512 | } else { | ||
513 | /* | ||
514 | * Case 2 : GPU railgate is not supported or we explicitly | ||
515 | * do not want to depend on runtime PM | ||
516 | * | ||
517 | * if GPU is now idle, call prepare_poweroff() to save the | ||
518 | * state and then do explicit railgate | ||
519 | * | ||
520 | * __gk20a_do_unidle() needs to unrailgate, call | ||
521 | * finalize_poweron(), and then call pm_runtime_put_sync() | ||
522 | * to balance the GPU usage counter | ||
523 | */ | ||
524 | |||
525 | /* Save the GPU state */ | ||
526 | err = gk20a_pm_prepare_poweroff(dev); | ||
527 | if (err) | ||
528 | goto fail_drop_usage_count; | ||
529 | |||
530 | /* railgate GPU */ | ||
531 | platform->railgate(dev); | ||
532 | |||
533 | nvgpu_udelay(10); | ||
534 | |||
535 | g->forced_reset = true; | ||
536 | return 0; | ||
537 | } | ||
538 | |||
539 | fail_drop_usage_count: | ||
540 | pm_runtime_put_noidle(dev); | ||
541 | fail_timeout: | ||
542 | nvgpu_mutex_release(&platform->railgate_lock); | ||
543 | up_write(&l->busy_lock); | ||
544 | gk20a_channel_deterministic_unidle(g); | ||
545 | return -EBUSY; | ||
546 | } | ||
547 | |||
548 | /** | ||
549 | * gk20a_do_idle() - wrap up for __gk20a_do_idle() to be called | ||
550 | * from outside of GPU driver | ||
551 | * | ||
552 | * In success, this call MUST be balanced by caller with gk20a_do_unidle() | ||
553 | */ | ||
554 | static int gk20a_do_idle(void *_g) | ||
555 | { | ||
556 | struct gk20a *g = (struct gk20a *)_g; | ||
557 | |||
558 | return __gk20a_do_idle(g, true); | ||
559 | } | ||
560 | |||
561 | /** | ||
562 | * __gk20a_do_unidle() - unblock all the tasks blocked by __gk20a_do_idle() | ||
563 | */ | ||
564 | int __gk20a_do_unidle(struct gk20a *g) | ||
565 | { | ||
566 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
567 | struct device *dev = dev_from_gk20a(g); | ||
568 | struct gk20a_platform *platform = dev_get_drvdata(dev); | ||
569 | int err; | ||
570 | |||
571 | if (g->forced_reset) { | ||
572 | /* | ||
573 | * If we did a forced-reset/railgate | ||
574 | * then unrailgate the GPU here first | ||
575 | */ | ||
576 | platform->unrailgate(dev); | ||
577 | |||
578 | /* restore the GPU state */ | ||
579 | err = gk20a_pm_finalize_poweron(dev); | ||
580 | if (err) | ||
581 | return err; | ||
582 | |||
583 | /* balance GPU usage counter */ | ||
584 | pm_runtime_put_sync(dev); | ||
585 | |||
586 | g->forced_reset = false; | ||
587 | } | ||
588 | |||
589 | /* release the lock and open up all other busy() calls */ | ||
590 | nvgpu_mutex_release(&platform->railgate_lock); | ||
591 | up_write(&l->busy_lock); | ||
592 | |||
593 | gk20a_channel_deterministic_unidle(g); | ||
594 | |||
595 | return 0; | ||
596 | } | ||
597 | |||
598 | /** | ||
599 | * gk20a_do_unidle() - wrap up for __gk20a_do_unidle() | ||
600 | */ | ||
601 | static int gk20a_do_unidle(void *_g) | ||
602 | { | ||
603 | struct gk20a *g = (struct gk20a *)_g; | ||
604 | |||
605 | return __gk20a_do_unidle(g); | ||
606 | } | ||
607 | #endif | ||
608 | |||
609 | void __iomem *nvgpu_ioremap_resource(struct platform_device *dev, int i, | ||
610 | struct resource **out) | ||
611 | { | ||
612 | struct resource *r = platform_get_resource(dev, IORESOURCE_MEM, i); | ||
613 | |||
614 | if (!r) | ||
615 | return NULL; | ||
616 | if (out) | ||
617 | *out = r; | ||
618 | return devm_ioremap_resource(&dev->dev, r); | ||
619 | } | ||
620 | |||
621 | static irqreturn_t gk20a_intr_isr_stall(int irq, void *dev_id) | ||
622 | { | ||
623 | struct gk20a *g = dev_id; | ||
624 | |||
625 | return nvgpu_intr_stall(g); | ||
626 | } | ||
627 | |||
628 | static irqreturn_t gk20a_intr_isr_nonstall(int irq, void *dev_id) | ||
629 | { | ||
630 | struct gk20a *g = dev_id; | ||
631 | |||
632 | return nvgpu_intr_nonstall(g); | ||
633 | } | ||
634 | |||
635 | static irqreturn_t gk20a_intr_thread_stall(int irq, void *dev_id) | ||
636 | { | ||
637 | struct gk20a *g = dev_id; | ||
638 | |||
639 | return nvgpu_intr_thread_stall(g); | ||
640 | } | ||
641 | |||
642 | void gk20a_remove_support(struct gk20a *g) | ||
643 | { | ||
644 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
645 | struct sim_nvgpu_linux *sim_linux; | ||
646 | |||
647 | tegra_unregister_idle_unidle(gk20a_do_idle); | ||
648 | |||
649 | nvgpu_kfree(g, g->dbg_regops_tmp_buf); | ||
650 | |||
651 | nvgpu_remove_channel_support_linux(l); | ||
652 | |||
653 | if (g->pmu.remove_support) | ||
654 | g->pmu.remove_support(&g->pmu); | ||
655 | |||
656 | if (g->gr.remove_support) | ||
657 | g->gr.remove_support(&g->gr); | ||
658 | |||
659 | if (g->mm.remove_ce_support) | ||
660 | g->mm.remove_ce_support(&g->mm); | ||
661 | |||
662 | if (g->fifo.remove_support) | ||
663 | g->fifo.remove_support(&g->fifo); | ||
664 | |||
665 | if (g->mm.remove_support) | ||
666 | g->mm.remove_support(&g->mm); | ||
667 | |||
668 | if (g->sim) { | ||
669 | sim_linux = container_of(g->sim, struct sim_nvgpu_linux, sim); | ||
670 | if (g->sim->remove_support) | ||
671 | g->sim->remove_support(g); | ||
672 | if (sim_linux->remove_support_linux) | ||
673 | sim_linux->remove_support_linux(g); | ||
674 | } | ||
675 | |||
676 | /* free mappings to registers, etc */ | ||
677 | if (l->regs) { | ||
678 | iounmap(l->regs); | ||
679 | l->regs = NULL; | ||
680 | } | ||
681 | if (l->bar1) { | ||
682 | iounmap(l->bar1); | ||
683 | l->bar1 = NULL; | ||
684 | } | ||
685 | |||
686 | nvgpu_remove_usermode_support(g); | ||
687 | |||
688 | nvgpu_free_enabled_flags(g); | ||
689 | } | ||
690 | |||
691 | static int gk20a_init_support(struct platform_device *dev) | ||
692 | { | ||
693 | int err = -ENOMEM; | ||
694 | struct gk20a *g = get_gk20a(&dev->dev); | ||
695 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
696 | |||
697 | tegra_register_idle_unidle(gk20a_do_idle, gk20a_do_unidle, g); | ||
698 | |||
699 | l->regs = nvgpu_ioremap_resource(dev, GK20A_BAR0_IORESOURCE_MEM, | ||
700 | &l->reg_mem); | ||
701 | if (IS_ERR(l->regs)) { | ||
702 | nvgpu_err(g, "failed to remap gk20a registers"); | ||
703 | err = PTR_ERR(l->regs); | ||
704 | goto fail; | ||
705 | } | ||
706 | |||
707 | l->bar1 = nvgpu_ioremap_resource(dev, GK20A_BAR1_IORESOURCE_MEM, | ||
708 | &l->bar1_mem); | ||
709 | if (IS_ERR(l->bar1)) { | ||
710 | nvgpu_err(g, "failed to remap gk20a bar1"); | ||
711 | err = PTR_ERR(l->bar1); | ||
712 | goto fail; | ||
713 | } | ||
714 | |||
715 | err = nvgpu_init_sim_support_linux(g, dev); | ||
716 | if (err) | ||
717 | goto fail; | ||
718 | err = nvgpu_init_sim_support(g); | ||
719 | if (err) | ||
720 | goto fail_sim; | ||
721 | |||
722 | nvgpu_init_usermode_support(g); | ||
723 | return 0; | ||
724 | |||
725 | fail_sim: | ||
726 | nvgpu_remove_sim_support_linux(g); | ||
727 | fail: | ||
728 | if (l->regs) { | ||
729 | iounmap(l->regs); | ||
730 | l->regs = NULL; | ||
731 | } | ||
732 | if (l->bar1) { | ||
733 | iounmap(l->bar1); | ||
734 | l->bar1 = NULL; | ||
735 | } | ||
736 | |||
737 | return err; | ||
738 | } | ||
739 | |||
740 | static int gk20a_pm_railgate(struct device *dev) | ||
741 | { | ||
742 | struct gk20a_platform *platform = dev_get_drvdata(dev); | ||
743 | int ret = 0; | ||
744 | struct gk20a *g = get_gk20a(dev); | ||
745 | |||
746 | /* if platform is already railgated, then just return */ | ||
747 | if (platform->is_railgated && platform->is_railgated(dev)) | ||
748 | return ret; | ||
749 | |||
750 | #ifdef CONFIG_DEBUG_FS | ||
751 | g->pstats.last_rail_gate_start = jiffies; | ||
752 | |||
753 | if (g->pstats.railgating_cycle_count >= 1) | ||
754 | g->pstats.total_rail_ungate_time_ms = | ||
755 | g->pstats.total_rail_ungate_time_ms + | ||
756 | jiffies_to_msecs(g->pstats.last_rail_gate_start - | ||
757 | g->pstats.last_rail_ungate_complete); | ||
758 | #endif | ||
759 | |||
760 | if (platform->railgate) | ||
761 | ret = platform->railgate(dev); | ||
762 | if (ret) { | ||
763 | nvgpu_err(g, "failed to railgate platform, err=%d", ret); | ||
764 | return ret; | ||
765 | } | ||
766 | |||
767 | #ifdef CONFIG_DEBUG_FS | ||
768 | g->pstats.last_rail_gate_complete = jiffies; | ||
769 | #endif | ||
770 | ret = tegra_fuse_clock_disable(); | ||
771 | if (ret) | ||
772 | nvgpu_err(g, "failed to disable tegra fuse clock, err=%d", ret); | ||
773 | |||
774 | return ret; | ||
775 | } | ||
776 | |||
777 | static int gk20a_pm_unrailgate(struct device *dev) | ||
778 | { | ||
779 | struct gk20a_platform *platform = dev_get_drvdata(dev); | ||
780 | int ret = 0; | ||
781 | struct gk20a *g = get_gk20a(dev); | ||
782 | |||
783 | ret = tegra_fuse_clock_enable(); | ||
784 | if (ret) { | ||
785 | nvgpu_err(g, "failed to enable tegra fuse clock, err=%d", ret); | ||
786 | return ret; | ||
787 | } | ||
788 | #ifdef CONFIG_DEBUG_FS | ||
789 | g->pstats.last_rail_ungate_start = jiffies; | ||
790 | if (g->pstats.railgating_cycle_count >= 1) | ||
791 | g->pstats.total_rail_gate_time_ms = | ||
792 | g->pstats.total_rail_gate_time_ms + | ||
793 | jiffies_to_msecs(g->pstats.last_rail_ungate_start - | ||
794 | g->pstats.last_rail_gate_complete); | ||
795 | |||
796 | g->pstats.railgating_cycle_count++; | ||
797 | #endif | ||
798 | |||
799 | trace_gk20a_pm_unrailgate(dev_name(dev)); | ||
800 | |||
801 | if (platform->unrailgate) { | ||
802 | nvgpu_mutex_acquire(&platform->railgate_lock); | ||
803 | ret = platform->unrailgate(dev); | ||
804 | nvgpu_mutex_release(&platform->railgate_lock); | ||
805 | } | ||
806 | |||
807 | #ifdef CONFIG_DEBUG_FS | ||
808 | g->pstats.last_rail_ungate_complete = jiffies; | ||
809 | #endif | ||
810 | |||
811 | return ret; | ||
812 | } | ||
813 | |||
814 | /* | ||
815 | * Remove association of the driver with OS interrupt handler | ||
816 | */ | ||
817 | void nvgpu_free_irq(struct gk20a *g) | ||
818 | { | ||
819 | struct device *dev = dev_from_gk20a(g); | ||
820 | |||
821 | devm_free_irq(dev, g->irq_stall, g); | ||
822 | if (g->irq_stall != g->irq_nonstall) | ||
823 | devm_free_irq(dev, g->irq_nonstall, g); | ||
824 | } | ||
825 | |||
826 | /* | ||
827 | * Idle the GPU in preparation of shutdown/remove. | ||
828 | * gk20a_driver_start_unload() does not idle the GPU, but instead changes the SW | ||
829 | * state to prevent further activity on the driver SW side. | ||
830 | * On driver removal quiesce() should be called after start_unload() | ||
831 | */ | ||
832 | int nvgpu_quiesce(struct gk20a *g) | ||
833 | { | ||
834 | int err; | ||
835 | struct device *dev = dev_from_gk20a(g); | ||
836 | |||
837 | if (g->power_on) { | ||
838 | err = gk20a_wait_for_idle(g); | ||
839 | if (err) { | ||
840 | nvgpu_err(g, "failed to idle GPU, err=%d", err); | ||
841 | return err; | ||
842 | } | ||
843 | |||
844 | err = gk20a_fifo_disable_all_engine_activity(g, true); | ||
845 | if (err) { | ||
846 | nvgpu_err(g, | ||
847 | "failed to disable engine activity, err=%d", | ||
848 | err); | ||
849 | return err; | ||
850 | } | ||
851 | |||
852 | err = gk20a_fifo_wait_engine_idle(g); | ||
853 | if (err) { | ||
854 | nvgpu_err(g, "failed to idle engines, err=%d", | ||
855 | err); | ||
856 | return err; | ||
857 | } | ||
858 | } | ||
859 | |||
860 | if (gk20a_gpu_is_virtual(dev)) | ||
861 | err = vgpu_pm_prepare_poweroff(dev); | ||
862 | else | ||
863 | err = gk20a_pm_prepare_poweroff(dev); | ||
864 | |||
865 | if (err) | ||
866 | nvgpu_err(g, "failed to prepare for poweroff, err=%d", | ||
867 | err); | ||
868 | |||
869 | return err; | ||
870 | } | ||
871 | |||
872 | static void gk20a_pm_shutdown(struct platform_device *pdev) | ||
873 | { | ||
874 | struct gk20a_platform *platform = platform_get_drvdata(pdev); | ||
875 | struct gk20a *g = platform->g; | ||
876 | int err; | ||
877 | |||
878 | nvgpu_info(g, "shutting down"); | ||
879 | |||
880 | /* vgpu has nothing to clean up currently */ | ||
881 | if (gk20a_gpu_is_virtual(&pdev->dev)) | ||
882 | return; | ||
883 | |||
884 | if (!g->power_on) | ||
885 | goto finish; | ||
886 | |||
887 | gk20a_driver_start_unload(g); | ||
888 | |||
889 | /* If GPU is already railgated, | ||
890 | * just prevent more requests, and return */ | ||
891 | if (platform->is_railgated && platform->is_railgated(&pdev->dev)) { | ||
892 | __pm_runtime_disable(&pdev->dev, false); | ||
893 | nvgpu_info(g, "already railgated, shut down complete"); | ||
894 | return; | ||
895 | } | ||
896 | |||
897 | /* Prevent more requests by disabling Runtime PM */ | ||
898 | __pm_runtime_disable(&pdev->dev, false); | ||
899 | |||
900 | err = nvgpu_quiesce(g); | ||
901 | if (err) | ||
902 | goto finish; | ||
903 | |||
904 | err = gk20a_pm_railgate(&pdev->dev); | ||
905 | if (err) | ||
906 | nvgpu_err(g, "failed to railgate, err=%d", err); | ||
907 | |||
908 | finish: | ||
909 | nvgpu_info(g, "shut down complete"); | ||
910 | } | ||
911 | |||
912 | #ifdef CONFIG_PM | ||
913 | static int gk20a_pm_runtime_resume(struct device *dev) | ||
914 | { | ||
915 | int err = 0; | ||
916 | |||
917 | err = gk20a_pm_unrailgate(dev); | ||
918 | if (err) | ||
919 | goto fail; | ||
920 | |||
921 | if (gk20a_gpu_is_virtual(dev)) | ||
922 | err = vgpu_pm_finalize_poweron(dev); | ||
923 | else | ||
924 | err = gk20a_pm_finalize_poweron(dev); | ||
925 | if (err) | ||
926 | goto fail_poweron; | ||
927 | |||
928 | return 0; | ||
929 | |||
930 | fail_poweron: | ||
931 | gk20a_pm_railgate(dev); | ||
932 | fail: | ||
933 | return err; | ||
934 | } | ||
935 | |||
936 | static int gk20a_pm_runtime_suspend(struct device *dev) | ||
937 | { | ||
938 | int err = 0; | ||
939 | struct gk20a *g = get_gk20a(dev); | ||
940 | |||
941 | if (gk20a_gpu_is_virtual(dev)) | ||
942 | err = vgpu_pm_prepare_poweroff(dev); | ||
943 | else | ||
944 | err = gk20a_pm_prepare_poweroff(dev); | ||
945 | if (err) { | ||
946 | nvgpu_err(g, "failed to power off, err=%d", err); | ||
947 | goto fail; | ||
948 | } | ||
949 | |||
950 | err = gk20a_pm_railgate(dev); | ||
951 | if (err) | ||
952 | goto fail; | ||
953 | |||
954 | return 0; | ||
955 | |||
956 | fail: | ||
957 | gk20a_pm_finalize_poweron(dev); | ||
958 | pm_runtime_mark_last_busy(dev); | ||
959 | return err; | ||
960 | } | ||
961 | |||
962 | static int gk20a_pm_suspend(struct device *dev) | ||
963 | { | ||
964 | struct gk20a_platform *platform = dev_get_drvdata(dev); | ||
965 | struct gk20a *g = get_gk20a(dev); | ||
966 | int ret = 0; | ||
967 | int idle_usage_count = 0; | ||
968 | |||
969 | if (!g->power_on) { | ||
970 | if (!pm_runtime_enabled(dev)) | ||
971 | gk20a_pm_railgate(dev); | ||
972 | return 0; | ||
973 | } | ||
974 | |||
975 | if (nvgpu_atomic_read(&g->usage_count) > idle_usage_count) | ||
976 | return -EBUSY; | ||
977 | |||
978 | ret = gk20a_pm_runtime_suspend(dev); | ||
979 | if (ret) | ||
980 | return ret; | ||
981 | |||
982 | if (platform->suspend) | ||
983 | platform->suspend(dev); | ||
984 | |||
985 | g->suspended = true; | ||
986 | |||
987 | return 0; | ||
988 | } | ||
989 | |||
990 | static int gk20a_pm_resume(struct device *dev) | ||
991 | { | ||
992 | struct gk20a *g = get_gk20a(dev); | ||
993 | int ret = 0; | ||
994 | |||
995 | if (!g->suspended) { | ||
996 | if (!pm_runtime_enabled(dev)) | ||
997 | gk20a_pm_unrailgate(dev); | ||
998 | return 0; | ||
999 | } | ||
1000 | |||
1001 | ret = gk20a_pm_runtime_resume(dev); | ||
1002 | |||
1003 | g->suspended = false; | ||
1004 | |||
1005 | return ret; | ||
1006 | } | ||
1007 | |||
1008 | static const struct dev_pm_ops gk20a_pm_ops = { | ||
1009 | .runtime_resume = gk20a_pm_runtime_resume, | ||
1010 | .runtime_suspend = gk20a_pm_runtime_suspend, | ||
1011 | .resume = gk20a_pm_resume, | ||
1012 | .suspend = gk20a_pm_suspend, | ||
1013 | }; | ||
1014 | #endif | ||
1015 | |||
1016 | static int gk20a_pm_init(struct device *dev) | ||
1017 | { | ||
1018 | struct gk20a *g = get_gk20a(dev); | ||
1019 | int err = 0; | ||
1020 | |||
1021 | nvgpu_log_fn(g, " "); | ||
1022 | |||
1023 | /* | ||
1024 | * Initialise pm runtime. For railgate disable | ||
1025 | * case, set autosuspend delay to negative which | ||
1026 | * will suspend runtime pm | ||
1027 | */ | ||
1028 | if (g->railgate_delay && g->can_railgate) | ||
1029 | pm_runtime_set_autosuspend_delay(dev, | ||
1030 | g->railgate_delay); | ||
1031 | else | ||
1032 | pm_runtime_set_autosuspend_delay(dev, -1); | ||
1033 | |||
1034 | pm_runtime_use_autosuspend(dev); | ||
1035 | pm_runtime_enable(dev); | ||
1036 | |||
1037 | return err; | ||
1038 | } | ||
1039 | |||
1040 | /* | ||
1041 | * Start the process for unloading the driver. Set NVGPU_DRIVER_IS_DYING. | ||
1042 | */ | ||
1043 | void gk20a_driver_start_unload(struct gk20a *g) | ||
1044 | { | ||
1045 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
1046 | |||
1047 | nvgpu_log(g, gpu_dbg_shutdown, "Driver is now going down!\n"); | ||
1048 | |||
1049 | down_write(&l->busy_lock); | ||
1050 | __nvgpu_set_enabled(g, NVGPU_DRIVER_IS_DYING, true); | ||
1051 | /* GR SW ready needs to be invalidated at this time with the busy lock | ||
1052 | * held to prevent a racing condition on the gr/mm code */ | ||
1053 | g->gr.sw_ready = false; | ||
1054 | g->sw_ready = false; | ||
1055 | up_write(&l->busy_lock); | ||
1056 | |||
1057 | if (g->is_virtual) | ||
1058 | return; | ||
1059 | |||
1060 | gk20a_wait_for_idle(g); | ||
1061 | |||
1062 | nvgpu_wait_for_deferred_interrupts(g); | ||
1063 | |||
1064 | if (l->nonstall_work_queue) { | ||
1065 | cancel_work_sync(&l->nonstall_fn_work); | ||
1066 | destroy_workqueue(l->nonstall_work_queue); | ||
1067 | l->nonstall_work_queue = NULL; | ||
1068 | } | ||
1069 | } | ||
1070 | |||
1071 | static inline void set_gk20a(struct platform_device *pdev, struct gk20a *gk20a) | ||
1072 | { | ||
1073 | gk20a_get_platform(&pdev->dev)->g = gk20a; | ||
1074 | } | ||
1075 | |||
1076 | static int nvgpu_read_fuse_overrides(struct gk20a *g) | ||
1077 | { | ||
1078 | struct device_node *np = nvgpu_get_node(g); | ||
1079 | u32 *fuses; | ||
1080 | int count, i; | ||
1081 | |||
1082 | if (!np) /* may be pcie device */ | ||
1083 | return 0; | ||
1084 | |||
1085 | count = of_property_count_elems_of_size(np, "fuse-overrides", 8); | ||
1086 | if (count <= 0) | ||
1087 | return count; | ||
1088 | |||
1089 | fuses = nvgpu_kmalloc(g, sizeof(u32) * count * 2); | ||
1090 | if (!fuses) | ||
1091 | return -ENOMEM; | ||
1092 | of_property_read_u32_array(np, "fuse-overrides", fuses, count * 2); | ||
1093 | for (i = 0; i < count; i++) { | ||
1094 | u32 fuse, value; | ||
1095 | |||
1096 | fuse = fuses[2 * i]; | ||
1097 | value = fuses[2 * i + 1]; | ||
1098 | switch (fuse) { | ||
1099 | case GM20B_FUSE_OPT_TPC_DISABLE: | ||
1100 | g->tpc_fs_mask_user = ~value; | ||
1101 | break; | ||
1102 | case GP10B_FUSE_OPT_ECC_EN: | ||
1103 | g->gr.fecs_feature_override_ecc_val = value; | ||
1104 | break; | ||
1105 | default: | ||
1106 | nvgpu_err(g, "ignore unknown fuse override %08x", fuse); | ||
1107 | break; | ||
1108 | } | ||
1109 | } | ||
1110 | |||
1111 | nvgpu_kfree(g, fuses); | ||
1112 | |||
1113 | return 0; | ||
1114 | } | ||
1115 | |||
1116 | static int gk20a_probe(struct platform_device *dev) | ||
1117 | { | ||
1118 | struct nvgpu_os_linux *l = NULL; | ||
1119 | struct gk20a *gk20a; | ||
1120 | int err; | ||
1121 | struct gk20a_platform *platform = NULL; | ||
1122 | struct device_node *np; | ||
1123 | |||
1124 | if (dev->dev.of_node) { | ||
1125 | const struct of_device_id *match; | ||
1126 | |||
1127 | match = of_match_device(tegra_gk20a_of_match, &dev->dev); | ||
1128 | if (match) | ||
1129 | platform = (struct gk20a_platform *)match->data; | ||
1130 | } else | ||
1131 | platform = (struct gk20a_platform *)dev->dev.platform_data; | ||
1132 | |||
1133 | if (!platform) { | ||
1134 | dev_err(&dev->dev, "no platform data\n"); | ||
1135 | return -ENODATA; | ||
1136 | } | ||
1137 | |||
1138 | platform_set_drvdata(dev, platform); | ||
1139 | |||
1140 | if (gk20a_gpu_is_virtual(&dev->dev)) | ||
1141 | return vgpu_probe(dev); | ||
1142 | |||
1143 | l = kzalloc(sizeof(*l), GFP_KERNEL); | ||
1144 | if (!l) { | ||
1145 | dev_err(&dev->dev, "couldn't allocate gk20a support"); | ||
1146 | return -ENOMEM; | ||
1147 | } | ||
1148 | |||
1149 | hash_init(l->ecc_sysfs_stats_htable); | ||
1150 | |||
1151 | gk20a = &l->g; | ||
1152 | |||
1153 | nvgpu_log_fn(gk20a, " "); | ||
1154 | |||
1155 | nvgpu_init_gk20a(gk20a); | ||
1156 | set_gk20a(dev, gk20a); | ||
1157 | l->dev = &dev->dev; | ||
1158 | gk20a->log_mask = NVGPU_DEFAULT_DBG_MASK; | ||
1159 | |||
1160 | nvgpu_kmem_init(gk20a); | ||
1161 | |||
1162 | err = nvgpu_init_enabled_flags(gk20a); | ||
1163 | if (err) | ||
1164 | goto return_err; | ||
1165 | |||
1166 | np = nvgpu_get_node(gk20a); | ||
1167 | if (of_dma_is_coherent(np)) { | ||
1168 | __nvgpu_set_enabled(gk20a, NVGPU_USE_COHERENT_SYSMEM, true); | ||
1169 | __nvgpu_set_enabled(gk20a, NVGPU_SUPPORT_IO_COHERENCE, true); | ||
1170 | } | ||
1171 | |||
1172 | if (nvgpu_platform_is_simulation(gk20a)) | ||
1173 | __nvgpu_set_enabled(gk20a, NVGPU_IS_FMODEL, true); | ||
1174 | |||
1175 | gk20a->irq_stall = platform_get_irq(dev, 0); | ||
1176 | gk20a->irq_nonstall = platform_get_irq(dev, 1); | ||
1177 | if (gk20a->irq_stall < 0 || gk20a->irq_nonstall < 0) { | ||
1178 | err = -ENXIO; | ||
1179 | goto return_err; | ||
1180 | } | ||
1181 | |||
1182 | err = devm_request_threaded_irq(&dev->dev, | ||
1183 | gk20a->irq_stall, | ||
1184 | gk20a_intr_isr_stall, | ||
1185 | gk20a_intr_thread_stall, | ||
1186 | 0, "gk20a_stall", gk20a); | ||
1187 | if (err) { | ||
1188 | dev_err(&dev->dev, | ||
1189 | "failed to request stall intr irq @ %d\n", | ||
1190 | gk20a->irq_stall); | ||
1191 | goto return_err; | ||
1192 | } | ||
1193 | err = devm_request_irq(&dev->dev, | ||
1194 | gk20a->irq_nonstall, | ||
1195 | gk20a_intr_isr_nonstall, | ||
1196 | 0, "gk20a_nonstall", gk20a); | ||
1197 | if (err) { | ||
1198 | dev_err(&dev->dev, | ||
1199 | "failed to request non-stall intr irq @ %d\n", | ||
1200 | gk20a->irq_nonstall); | ||
1201 | goto return_err; | ||
1202 | } | ||
1203 | disable_irq(gk20a->irq_stall); | ||
1204 | if (gk20a->irq_stall != gk20a->irq_nonstall) | ||
1205 | disable_irq(gk20a->irq_nonstall); | ||
1206 | |||
1207 | err = gk20a_init_support(dev); | ||
1208 | if (err) | ||
1209 | goto return_err; | ||
1210 | |||
1211 | err = nvgpu_read_fuse_overrides(gk20a); | ||
1212 | |||
1213 | #ifdef CONFIG_RESET_CONTROLLER | ||
1214 | platform->reset_control = devm_reset_control_get(&dev->dev, NULL); | ||
1215 | if (IS_ERR(platform->reset_control)) | ||
1216 | platform->reset_control = NULL; | ||
1217 | #endif | ||
1218 | |||
1219 | err = nvgpu_probe(gk20a, "gpu.0", INTERFACE_NAME, &nvgpu_class); | ||
1220 | if (err) | ||
1221 | goto return_err; | ||
1222 | |||
1223 | err = gk20a_pm_init(&dev->dev); | ||
1224 | if (err) { | ||
1225 | dev_err(&dev->dev, "pm init failed"); | ||
1226 | goto return_err; | ||
1227 | } | ||
1228 | |||
1229 | gk20a->mm.has_physical_mode = !nvgpu_is_hypervisor_mode(gk20a); | ||
1230 | |||
1231 | return 0; | ||
1232 | |||
1233 | return_err: | ||
1234 | nvgpu_free_enabled_flags(gk20a); | ||
1235 | |||
1236 | /* | ||
1237 | * Last since the above allocs may use data structures in here. | ||
1238 | */ | ||
1239 | nvgpu_kmem_fini(gk20a, NVGPU_KMEM_FINI_FORCE_CLEANUP); | ||
1240 | |||
1241 | kfree(l); | ||
1242 | |||
1243 | return err; | ||
1244 | } | ||
1245 | |||
1246 | int nvgpu_remove(struct device *dev, struct class *class) | ||
1247 | { | ||
1248 | struct gk20a *g = get_gk20a(dev); | ||
1249 | #ifdef CONFIG_NVGPU_SUPPORT_CDE | ||
1250 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
1251 | #endif | ||
1252 | struct gk20a_platform *platform = gk20a_get_platform(dev); | ||
1253 | int err; | ||
1254 | |||
1255 | nvgpu_log_fn(g, " "); | ||
1256 | |||
1257 | err = nvgpu_quiesce(g); | ||
1258 | WARN(err, "gpu failed to idle during driver removal"); | ||
1259 | |||
1260 | if (nvgpu_mem_is_valid(&g->syncpt_mem)) | ||
1261 | nvgpu_dma_free(g, &g->syncpt_mem); | ||
1262 | |||
1263 | #ifdef CONFIG_NVGPU_SUPPORT_CDE | ||
1264 | if (platform->has_cde) | ||
1265 | gk20a_cde_destroy(l); | ||
1266 | #endif | ||
1267 | |||
1268 | #ifdef CONFIG_GK20A_CTXSW_TRACE | ||
1269 | gk20a_ctxsw_trace_cleanup(g); | ||
1270 | #endif | ||
1271 | |||
1272 | gk20a_sched_ctrl_cleanup(g); | ||
1273 | |||
1274 | if (IS_ENABLED(CONFIG_GK20A_DEVFREQ)) | ||
1275 | gk20a_scale_exit(dev); | ||
1276 | |||
1277 | nvgpu_clk_arb_cleanup_arbiter(g); | ||
1278 | |||
1279 | gk20a_user_deinit(dev, class); | ||
1280 | |||
1281 | gk20a_debug_deinit(g); | ||
1282 | |||
1283 | nvgpu_remove_sysfs(dev); | ||
1284 | |||
1285 | if (platform->secure_buffer.destroy) | ||
1286 | platform->secure_buffer.destroy(g, | ||
1287 | &platform->secure_buffer); | ||
1288 | |||
1289 | if (pm_runtime_enabled(dev)) | ||
1290 | pm_runtime_disable(dev); | ||
1291 | |||
1292 | if (platform->remove) | ||
1293 | platform->remove(dev); | ||
1294 | |||
1295 | nvgpu_log_fn(g, "removed"); | ||
1296 | |||
1297 | return err; | ||
1298 | } | ||
1299 | |||
1300 | static int __exit gk20a_remove(struct platform_device *pdev) | ||
1301 | { | ||
1302 | int err; | ||
1303 | struct device *dev = &pdev->dev; | ||
1304 | struct gk20a *g = get_gk20a(dev); | ||
1305 | |||
1306 | if (gk20a_gpu_is_virtual(dev)) | ||
1307 | return vgpu_remove(pdev); | ||
1308 | |||
1309 | err = nvgpu_remove(dev, &nvgpu_class); | ||
1310 | |||
1311 | set_gk20a(pdev, NULL); | ||
1312 | gk20a_put(g); | ||
1313 | |||
1314 | return err; | ||
1315 | } | ||
1316 | |||
1317 | static struct platform_driver gk20a_driver = { | ||
1318 | .probe = gk20a_probe, | ||
1319 | .remove = __exit_p(gk20a_remove), | ||
1320 | .shutdown = gk20a_pm_shutdown, | ||
1321 | .driver = { | ||
1322 | .owner = THIS_MODULE, | ||
1323 | .name = "gk20a", | ||
1324 | .probe_type = PROBE_PREFER_ASYNCHRONOUS, | ||
1325 | #ifdef CONFIG_OF | ||
1326 | .of_match_table = tegra_gk20a_of_match, | ||
1327 | #endif | ||
1328 | #ifdef CONFIG_PM | ||
1329 | .pm = &gk20a_pm_ops, | ||
1330 | #endif | ||
1331 | .suppress_bind_attrs = true, | ||
1332 | } | ||
1333 | }; | ||
1334 | |||
1335 | struct class nvgpu_class = { | ||
1336 | .owner = THIS_MODULE, | ||
1337 | .name = CLASS_NAME, | ||
1338 | }; | ||
1339 | |||
1340 | static int __init gk20a_init(void) | ||
1341 | { | ||
1342 | |||
1343 | int ret; | ||
1344 | |||
1345 | ret = class_register(&nvgpu_class); | ||
1346 | if (ret) | ||
1347 | return ret; | ||
1348 | |||
1349 | ret = nvgpu_pci_init(); | ||
1350 | if (ret) | ||
1351 | return ret; | ||
1352 | |||
1353 | return platform_driver_register(&gk20a_driver); | ||
1354 | } | ||
1355 | |||
1356 | static void __exit gk20a_exit(void) | ||
1357 | { | ||
1358 | nvgpu_pci_exit(); | ||
1359 | platform_driver_unregister(&gk20a_driver); | ||
1360 | class_unregister(&nvgpu_class); | ||
1361 | } | ||
1362 | |||
1363 | MODULE_LICENSE("GPL v2"); | ||
1364 | module_init(gk20a_init); | ||
1365 | module_exit(gk20a_exit); | ||