summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/linux/module.c
diff options
context:
space:
mode:
authorTerje Bergstrom <tbergstrom@nvidia.com>2018-04-18 15:59:00 -0400
committermobile promotions <svcmobile_promotions@nvidia.com>2018-06-15 20:47:31 -0400
commit2a2c16af5f9f1ccfc93a13e820d5381e5c881e92 (patch)
tree2e5d7b042270a649978e5bb540857012c85fb5b5 /drivers/gpu/nvgpu/common/linux/module.c
parent98d996f4ffb0137d119b5849cae46d7b7e5693e1 (diff)
gpu: nvgpu: Move Linux files away from common
Move all Linux source code files to drivers/gpu/nvgpu/os/linux from drivers/gpu/nvgpu/common/linux. This changes the meaning of common to be OS independent. JIRA NVGPU-598 JIRA NVGPU-601 Change-Id: Ib7f2a43d3688bb0d0b7dcc48469a6783fd988ce9 Signed-off-by: Terje Bergstrom <tbergstrom@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/1747714 Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com> Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/common/linux/module.c')
-rw-r--r--drivers/gpu/nvgpu/common/linux/module.c1365
1 files changed, 0 insertions, 1365 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/module.c b/drivers/gpu/nvgpu/common/linux/module.c
deleted file mode 100644
index af71cc81..00000000
--- a/drivers/gpu/nvgpu/common/linux/module.c
+++ /dev/null
@@ -1,1365 +0,0 @@
1/*
2 * GK20A Graphics
3 *
4 * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <linux/module.h>
20#include <linux/of.h>
21#include <linux/of_device.h>
22#include <linux/of_platform.h>
23#include <linux/of_address.h>
24#include <linux/interrupt.h>
25#include <linux/pm_runtime.h>
26#include <linux/reset.h>
27#include <linux/platform/tegra/common.h>
28#include <linux/pci.h>
29
30#include <uapi/linux/nvgpu.h>
31#include <dt-bindings/soc/gm20b-fuse.h>
32#include <dt-bindings/soc/gp10b-fuse.h>
33
34#include <soc/tegra/fuse.h>
35
36#include <nvgpu/dma.h>
37#include <nvgpu/kmem.h>
38#include <nvgpu/nvgpu_common.h>
39#include <nvgpu/soc.h>
40#include <nvgpu/enabled.h>
41#include <nvgpu/debug.h>
42#include <nvgpu/ctxsw_trace.h>
43#include <nvgpu/vidmem.h>
44#include <nvgpu/sim.h>
45
46#include "platform_gk20a.h"
47#include "sysfs.h"
48#include "vgpu/vgpu_linux.h"
49#include "scale.h"
50#include "pci.h"
51#include "module.h"
52#include "module_usermode.h"
53#include "intr.h"
54#include "ioctl.h"
55
56#include "os_linux.h"
57#include "ctxsw_trace.h"
58#include "driver_common.h"
59#include "channel.h"
60
61#ifdef CONFIG_NVGPU_SUPPORT_CDE
62#include "cde.h"
63#endif
64
65#define CLASS_NAME "nvidia-gpu"
66/* TODO: Change to e.g. "nvidia-gpu%s" once we have symlinks in place. */
67
68#define GK20A_WAIT_FOR_IDLE_MS 2000
69
70#define CREATE_TRACE_POINTS
71#include <trace/events/gk20a.h>
72
73
74struct device_node *nvgpu_get_node(struct gk20a *g)
75{
76 struct device *dev = dev_from_gk20a(g);
77
78 if (dev_is_pci(dev)) {
79 struct pci_bus *bus = to_pci_dev(dev)->bus;
80
81 while (!pci_is_root_bus(bus))
82 bus = bus->parent;
83
84 return bus->bridge->parent->of_node;
85 }
86
87 return dev->of_node;
88}
89
90void gk20a_busy_noresume(struct gk20a *g)
91{
92 pm_runtime_get_noresume(dev_from_gk20a(g));
93}
94
95int gk20a_busy(struct gk20a *g)
96{
97 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
98 int ret = 0;
99 struct device *dev;
100
101 if (!g)
102 return -ENODEV;
103
104 atomic_inc(&g->usage_count.atomic_var);
105
106 down_read(&l->busy_lock);
107
108 if (!gk20a_can_busy(g)) {
109 ret = -ENODEV;
110 atomic_dec(&g->usage_count.atomic_var);
111 goto fail;
112 }
113
114 dev = dev_from_gk20a(g);
115
116 if (pm_runtime_enabled(dev)) {
117 /* Increment usage count and attempt to resume device */
118 ret = pm_runtime_get_sync(dev);
119 if (ret < 0) {
120 /* Mark suspended so runtime pm will retry later */
121 pm_runtime_set_suspended(dev);
122 pm_runtime_put_noidle(dev);
123 atomic_dec(&g->usage_count.atomic_var);
124 goto fail;
125 }
126 } else {
127 nvgpu_mutex_acquire(&g->poweron_lock);
128 if (!g->power_on) {
129 ret = gk20a_gpu_is_virtual(dev) ?
130 vgpu_pm_finalize_poweron(dev)
131 : gk20a_pm_finalize_poweron(dev);
132 if (ret) {
133 atomic_dec(&g->usage_count.atomic_var);
134 nvgpu_mutex_release(&g->poweron_lock);
135 goto fail;
136 }
137 }
138 nvgpu_mutex_release(&g->poweron_lock);
139 }
140
141fail:
142 up_read(&l->busy_lock);
143
144 return ret < 0 ? ret : 0;
145}
146
147void gk20a_idle_nosuspend(struct gk20a *g)
148{
149 pm_runtime_put_noidle(dev_from_gk20a(g));
150}
151
152void gk20a_idle(struct gk20a *g)
153{
154 struct device *dev;
155
156 atomic_dec(&g->usage_count.atomic_var);
157
158 dev = dev_from_gk20a(g);
159
160 if (!(dev && gk20a_can_busy(g)))
161 return;
162
163 if (pm_runtime_enabled(dev)) {
164 pm_runtime_mark_last_busy(dev);
165 pm_runtime_put_sync_autosuspend(dev);
166 }
167}
168
169/*
170 * Undoes gk20a_lockout_registers().
171 */
172static int gk20a_restore_registers(struct gk20a *g)
173{
174 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
175
176 l->regs = l->regs_saved;
177 l->bar1 = l->bar1_saved;
178
179 nvgpu_restore_usermode_registers(g);
180
181 return 0;
182}
183
184static int nvgpu_init_os_linux_ops(struct nvgpu_os_linux *l)
185{
186 int err = 0;
187
188#ifdef CONFIG_NVGPU_SUPPORT_CDE
189 err = nvgpu_cde_init_ops(l);
190#endif
191
192 return err;
193}
194
195int nvgpu_finalize_poweron_linux(struct nvgpu_os_linux *l)
196{
197 struct gk20a *g = &l->g;
198 int err;
199
200 if (l->init_done)
201 return 0;
202
203 err = nvgpu_init_channel_support_linux(l);
204 if (err) {
205 nvgpu_err(g, "failed to init linux channel support");
206 return err;
207 }
208
209 l->init_done = true;
210
211 return 0;
212}
213
214int gk20a_pm_finalize_poweron(struct device *dev)
215{
216 struct gk20a *g = get_gk20a(dev);
217 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
218 struct gk20a_platform *platform = gk20a_get_platform(dev);
219 int err;
220
221 nvgpu_log_fn(g, " ");
222
223 if (g->power_on)
224 return 0;
225
226 trace_gk20a_finalize_poweron(dev_name(dev));
227
228 /* Increment platform power refcount */
229 if (platform->busy) {
230 err = platform->busy(dev);
231 if (err < 0) {
232 nvgpu_err(g, "failed to poweron platform dependency");
233 return err;
234 }
235 }
236
237 err = gk20a_restore_registers(g);
238 if (err)
239 return err;
240
241 /* Enable interrupt workqueue */
242 if (!l->nonstall_work_queue) {
243 l->nonstall_work_queue = alloc_workqueue("%s",
244 WQ_HIGHPRI, 1, "mc_nonstall");
245 INIT_WORK(&l->nonstall_fn_work, nvgpu_intr_nonstall_cb);
246 }
247
248 err = gk20a_detect_chip(g);
249 if (err)
250 return err;
251
252 if (g->sim) {
253 if (g->sim->sim_init_late)
254 g->sim->sim_init_late(g);
255 }
256
257 err = gk20a_finalize_poweron(g);
258 if (err)
259 goto done;
260
261 err = nvgpu_finalize_poweron_linux(l);
262 if (err)
263 goto done;
264
265 nvgpu_init_mm_ce_context(g);
266
267 nvgpu_vidmem_thread_unpause(&g->mm);
268
269 /* Initialise scaling: it will initialize scaling drive only once */
270 if (IS_ENABLED(CONFIG_GK20A_DEVFREQ) &&
271 nvgpu_platform_is_silicon(g)) {
272 gk20a_scale_init(dev);
273 if (platform->initscale)
274 platform->initscale(dev);
275 }
276
277 trace_gk20a_finalize_poweron_done(dev_name(dev));
278
279 err = nvgpu_init_os_linux_ops(l);
280 if (err)
281 goto done;
282
283 enable_irq(g->irq_stall);
284 if (g->irq_stall != g->irq_nonstall)
285 enable_irq(g->irq_nonstall);
286 g->irqs_enabled = 1;
287
288 gk20a_scale_resume(dev_from_gk20a(g));
289
290#ifdef CONFIG_NVGPU_SUPPORT_CDE
291 if (platform->has_cde)
292 gk20a_init_cde_support(l);
293#endif
294
295 err = gk20a_sched_ctrl_init(g);
296 if (err) {
297 nvgpu_err(g, "failed to init sched control");
298 return err;
299 }
300
301 g->sw_ready = true;
302
303done:
304 if (err)
305 g->power_on = false;
306
307 return err;
308}
309
310/*
311 * Locks out the driver from accessing GPU registers. This prevents access to
312 * thse registers after the GPU has been clock or power gated. This should help
313 * find annoying bugs where register reads and writes are silently dropped
314 * after the GPU has been turned off. On older chips these reads and writes can
315 * also lock the entire CPU up.
316 */
317static int gk20a_lockout_registers(struct gk20a *g)
318{
319 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
320
321 l->regs = NULL;
322 l->bar1 = NULL;
323
324 nvgpu_lockout_usermode_registers(g);
325
326 return 0;
327}
328
329static int gk20a_pm_prepare_poweroff(struct device *dev)
330{
331 struct gk20a *g = get_gk20a(dev);
332#ifdef CONFIG_NVGPU_SUPPORT_CDE
333 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
334#endif
335 int ret = 0;
336 struct gk20a_platform *platform = gk20a_get_platform(dev);
337 bool irqs_enabled;
338
339 nvgpu_log_fn(g, " ");
340
341 nvgpu_mutex_acquire(&g->poweroff_lock);
342
343 if (!g->power_on)
344 goto done;
345
346 /* disable IRQs and wait for completion */
347 irqs_enabled = g->irqs_enabled;
348 if (irqs_enabled) {
349 disable_irq(g->irq_stall);
350 if (g->irq_stall != g->irq_nonstall)
351 disable_irq(g->irq_nonstall);
352 g->irqs_enabled = 0;
353 }
354
355 gk20a_scale_suspend(dev);
356
357#ifdef CONFIG_NVGPU_SUPPORT_CDE
358 gk20a_cde_suspend(l);
359#endif
360
361 ret = gk20a_prepare_poweroff(g);
362 if (ret)
363 goto error;
364
365 /* Decrement platform power refcount */
366 if (platform->idle)
367 platform->idle(dev);
368
369 /* Stop CPU from accessing the GPU registers. */
370 gk20a_lockout_registers(g);
371
372 nvgpu_mutex_release(&g->poweroff_lock);
373 return 0;
374
375error:
376 /* re-enabled IRQs if previously enabled */
377 if (irqs_enabled) {
378 enable_irq(g->irq_stall);
379 if (g->irq_stall != g->irq_nonstall)
380 enable_irq(g->irq_nonstall);
381 g->irqs_enabled = 1;
382 }
383
384 gk20a_scale_resume(dev);
385done:
386 nvgpu_mutex_release(&g->poweroff_lock);
387
388 return ret;
389}
390
391static struct of_device_id tegra_gk20a_of_match[] = {
392#ifdef CONFIG_TEGRA_GK20A
393 { .compatible = "nvidia,tegra210-gm20b",
394 .data = &gm20b_tegra_platform },
395 { .compatible = "nvidia,tegra186-gp10b",
396 .data = &gp10b_tegra_platform },
397 { .compatible = "nvidia,gv11b",
398 .data = &gv11b_tegra_platform },
399#ifdef CONFIG_TEGRA_GR_VIRTUALIZATION
400 { .compatible = "nvidia,gv11b-vgpu",
401 .data = &gv11b_vgpu_tegra_platform},
402#endif
403#ifdef CONFIG_TEGRA_GR_VIRTUALIZATION
404 { .compatible = "nvidia,tegra124-gk20a-vgpu",
405 .data = &vgpu_tegra_platform },
406#endif
407#endif
408
409 { },
410};
411
412#ifdef CONFIG_PM
413/**
414 * __gk20a_do_idle() - force the GPU to idle and railgate
415 *
416 * In success, this call MUST be balanced by caller with __gk20a_do_unidle()
417 *
418 * Acquires two locks : &l->busy_lock and &platform->railgate_lock
419 * In success, we hold these locks and return
420 * In failure, we release these locks and return
421 */
422int __gk20a_do_idle(struct gk20a *g, bool force_reset)
423{
424 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
425 struct device *dev = dev_from_gk20a(g);
426 struct gk20a_platform *platform = dev_get_drvdata(dev);
427 struct nvgpu_timeout timeout;
428 int ref_cnt;
429 int target_ref_cnt = 0;
430 bool is_railgated;
431 int err = 0;
432
433 /*
434 * Hold back deterministic submits and changes to deterministic
435 * channels - this must be outside the power busy locks.
436 */
437 gk20a_channel_deterministic_idle(g);
438
439 /* acquire busy lock to block other busy() calls */
440 down_write(&l->busy_lock);
441
442 /* acquire railgate lock to prevent unrailgate in midst of do_idle() */
443 nvgpu_mutex_acquire(&platform->railgate_lock);
444
445 /* check if it is already railgated ? */
446 if (platform->is_railgated(dev))
447 return 0;
448
449 /*
450 * release railgate_lock, prevent suspend by incrementing usage counter,
451 * re-acquire railgate_lock
452 */
453 nvgpu_mutex_release(&platform->railgate_lock);
454 pm_runtime_get_sync(dev);
455
456 /*
457 * One refcount taken in this API
458 * If User disables rail gating, we take one more
459 * extra refcount
460 */
461 if (g->can_railgate)
462 target_ref_cnt = 1;
463 else
464 target_ref_cnt = 2;
465 nvgpu_mutex_acquire(&platform->railgate_lock);
466
467 nvgpu_timeout_init(g, &timeout, GK20A_WAIT_FOR_IDLE_MS,
468 NVGPU_TIMER_CPU_TIMER);
469
470 /* check and wait until GPU is idle (with a timeout) */
471 do {
472 nvgpu_usleep_range(1000, 1100);
473 ref_cnt = atomic_read(&dev->power.usage_count);
474 } while (ref_cnt != target_ref_cnt && !nvgpu_timeout_expired(&timeout));
475
476 if (ref_cnt != target_ref_cnt) {
477 nvgpu_err(g, "failed to idle - refcount %d != target_ref_cnt",
478 ref_cnt);
479 goto fail_drop_usage_count;
480 }
481
482 /* check if global force_reset flag is set */
483 force_reset |= platform->force_reset_in_do_idle;
484
485 nvgpu_timeout_init(g, &timeout, GK20A_WAIT_FOR_IDLE_MS,
486 NVGPU_TIMER_CPU_TIMER);
487
488 if (g->can_railgate && !force_reset) {
489 /*
490 * Case 1 : GPU railgate is supported
491 *
492 * if GPU is now idle, we will have only one ref count,
493 * drop this ref which will rail gate the GPU
494 */
495 pm_runtime_put_sync(dev);
496
497 /* add sufficient delay to allow GPU to rail gate */
498 nvgpu_msleep(g->railgate_delay);
499
500 /* check in loop if GPU is railgated or not */
501 do {
502 nvgpu_usleep_range(1000, 1100);
503 is_railgated = platform->is_railgated(dev);
504 } while (!is_railgated && !nvgpu_timeout_expired(&timeout));
505
506 if (is_railgated) {
507 return 0;
508 } else {
509 nvgpu_err(g, "failed to idle in timeout");
510 goto fail_timeout;
511 }
512 } else {
513 /*
514 * Case 2 : GPU railgate is not supported or we explicitly
515 * do not want to depend on runtime PM
516 *
517 * if GPU is now idle, call prepare_poweroff() to save the
518 * state and then do explicit railgate
519 *
520 * __gk20a_do_unidle() needs to unrailgate, call
521 * finalize_poweron(), and then call pm_runtime_put_sync()
522 * to balance the GPU usage counter
523 */
524
525 /* Save the GPU state */
526 err = gk20a_pm_prepare_poweroff(dev);
527 if (err)
528 goto fail_drop_usage_count;
529
530 /* railgate GPU */
531 platform->railgate(dev);
532
533 nvgpu_udelay(10);
534
535 g->forced_reset = true;
536 return 0;
537 }
538
539fail_drop_usage_count:
540 pm_runtime_put_noidle(dev);
541fail_timeout:
542 nvgpu_mutex_release(&platform->railgate_lock);
543 up_write(&l->busy_lock);
544 gk20a_channel_deterministic_unidle(g);
545 return -EBUSY;
546}
547
548/**
549 * gk20a_do_idle() - wrap up for __gk20a_do_idle() to be called
550 * from outside of GPU driver
551 *
552 * In success, this call MUST be balanced by caller with gk20a_do_unidle()
553 */
554static int gk20a_do_idle(void *_g)
555{
556 struct gk20a *g = (struct gk20a *)_g;
557
558 return __gk20a_do_idle(g, true);
559}
560
561/**
562 * __gk20a_do_unidle() - unblock all the tasks blocked by __gk20a_do_idle()
563 */
564int __gk20a_do_unidle(struct gk20a *g)
565{
566 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
567 struct device *dev = dev_from_gk20a(g);
568 struct gk20a_platform *platform = dev_get_drvdata(dev);
569 int err;
570
571 if (g->forced_reset) {
572 /*
573 * If we did a forced-reset/railgate
574 * then unrailgate the GPU here first
575 */
576 platform->unrailgate(dev);
577
578 /* restore the GPU state */
579 err = gk20a_pm_finalize_poweron(dev);
580 if (err)
581 return err;
582
583 /* balance GPU usage counter */
584 pm_runtime_put_sync(dev);
585
586 g->forced_reset = false;
587 }
588
589 /* release the lock and open up all other busy() calls */
590 nvgpu_mutex_release(&platform->railgate_lock);
591 up_write(&l->busy_lock);
592
593 gk20a_channel_deterministic_unidle(g);
594
595 return 0;
596}
597
598/**
599 * gk20a_do_unidle() - wrap up for __gk20a_do_unidle()
600 */
601static int gk20a_do_unidle(void *_g)
602{
603 struct gk20a *g = (struct gk20a *)_g;
604
605 return __gk20a_do_unidle(g);
606}
607#endif
608
609void __iomem *nvgpu_ioremap_resource(struct platform_device *dev, int i,
610 struct resource **out)
611{
612 struct resource *r = platform_get_resource(dev, IORESOURCE_MEM, i);
613
614 if (!r)
615 return NULL;
616 if (out)
617 *out = r;
618 return devm_ioremap_resource(&dev->dev, r);
619}
620
621static irqreturn_t gk20a_intr_isr_stall(int irq, void *dev_id)
622{
623 struct gk20a *g = dev_id;
624
625 return nvgpu_intr_stall(g);
626}
627
628static irqreturn_t gk20a_intr_isr_nonstall(int irq, void *dev_id)
629{
630 struct gk20a *g = dev_id;
631
632 return nvgpu_intr_nonstall(g);
633}
634
635static irqreturn_t gk20a_intr_thread_stall(int irq, void *dev_id)
636{
637 struct gk20a *g = dev_id;
638
639 return nvgpu_intr_thread_stall(g);
640}
641
642void gk20a_remove_support(struct gk20a *g)
643{
644 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
645 struct sim_nvgpu_linux *sim_linux;
646
647 tegra_unregister_idle_unidle(gk20a_do_idle);
648
649 nvgpu_kfree(g, g->dbg_regops_tmp_buf);
650
651 nvgpu_remove_channel_support_linux(l);
652
653 if (g->pmu.remove_support)
654 g->pmu.remove_support(&g->pmu);
655
656 if (g->gr.remove_support)
657 g->gr.remove_support(&g->gr);
658
659 if (g->mm.remove_ce_support)
660 g->mm.remove_ce_support(&g->mm);
661
662 if (g->fifo.remove_support)
663 g->fifo.remove_support(&g->fifo);
664
665 if (g->mm.remove_support)
666 g->mm.remove_support(&g->mm);
667
668 if (g->sim) {
669 sim_linux = container_of(g->sim, struct sim_nvgpu_linux, sim);
670 if (g->sim->remove_support)
671 g->sim->remove_support(g);
672 if (sim_linux->remove_support_linux)
673 sim_linux->remove_support_linux(g);
674 }
675
676 /* free mappings to registers, etc */
677 if (l->regs) {
678 iounmap(l->regs);
679 l->regs = NULL;
680 }
681 if (l->bar1) {
682 iounmap(l->bar1);
683 l->bar1 = NULL;
684 }
685
686 nvgpu_remove_usermode_support(g);
687
688 nvgpu_free_enabled_flags(g);
689}
690
691static int gk20a_init_support(struct platform_device *dev)
692{
693 int err = -ENOMEM;
694 struct gk20a *g = get_gk20a(&dev->dev);
695 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
696
697 tegra_register_idle_unidle(gk20a_do_idle, gk20a_do_unidle, g);
698
699 l->regs = nvgpu_ioremap_resource(dev, GK20A_BAR0_IORESOURCE_MEM,
700 &l->reg_mem);
701 if (IS_ERR(l->regs)) {
702 nvgpu_err(g, "failed to remap gk20a registers");
703 err = PTR_ERR(l->regs);
704 goto fail;
705 }
706
707 l->bar1 = nvgpu_ioremap_resource(dev, GK20A_BAR1_IORESOURCE_MEM,
708 &l->bar1_mem);
709 if (IS_ERR(l->bar1)) {
710 nvgpu_err(g, "failed to remap gk20a bar1");
711 err = PTR_ERR(l->bar1);
712 goto fail;
713 }
714
715 err = nvgpu_init_sim_support_linux(g, dev);
716 if (err)
717 goto fail;
718 err = nvgpu_init_sim_support(g);
719 if (err)
720 goto fail_sim;
721
722 nvgpu_init_usermode_support(g);
723 return 0;
724
725fail_sim:
726 nvgpu_remove_sim_support_linux(g);
727fail:
728 if (l->regs) {
729 iounmap(l->regs);
730 l->regs = NULL;
731 }
732 if (l->bar1) {
733 iounmap(l->bar1);
734 l->bar1 = NULL;
735 }
736
737 return err;
738}
739
740static int gk20a_pm_railgate(struct device *dev)
741{
742 struct gk20a_platform *platform = dev_get_drvdata(dev);
743 int ret = 0;
744 struct gk20a *g = get_gk20a(dev);
745
746 /* if platform is already railgated, then just return */
747 if (platform->is_railgated && platform->is_railgated(dev))
748 return ret;
749
750#ifdef CONFIG_DEBUG_FS
751 g->pstats.last_rail_gate_start = jiffies;
752
753 if (g->pstats.railgating_cycle_count >= 1)
754 g->pstats.total_rail_ungate_time_ms =
755 g->pstats.total_rail_ungate_time_ms +
756 jiffies_to_msecs(g->pstats.last_rail_gate_start -
757 g->pstats.last_rail_ungate_complete);
758#endif
759
760 if (platform->railgate)
761 ret = platform->railgate(dev);
762 if (ret) {
763 nvgpu_err(g, "failed to railgate platform, err=%d", ret);
764 return ret;
765 }
766
767#ifdef CONFIG_DEBUG_FS
768 g->pstats.last_rail_gate_complete = jiffies;
769#endif
770 ret = tegra_fuse_clock_disable();
771 if (ret)
772 nvgpu_err(g, "failed to disable tegra fuse clock, err=%d", ret);
773
774 return ret;
775}
776
777static int gk20a_pm_unrailgate(struct device *dev)
778{
779 struct gk20a_platform *platform = dev_get_drvdata(dev);
780 int ret = 0;
781 struct gk20a *g = get_gk20a(dev);
782
783 ret = tegra_fuse_clock_enable();
784 if (ret) {
785 nvgpu_err(g, "failed to enable tegra fuse clock, err=%d", ret);
786 return ret;
787 }
788#ifdef CONFIG_DEBUG_FS
789 g->pstats.last_rail_ungate_start = jiffies;
790 if (g->pstats.railgating_cycle_count >= 1)
791 g->pstats.total_rail_gate_time_ms =
792 g->pstats.total_rail_gate_time_ms +
793 jiffies_to_msecs(g->pstats.last_rail_ungate_start -
794 g->pstats.last_rail_gate_complete);
795
796 g->pstats.railgating_cycle_count++;
797#endif
798
799 trace_gk20a_pm_unrailgate(dev_name(dev));
800
801 if (platform->unrailgate) {
802 nvgpu_mutex_acquire(&platform->railgate_lock);
803 ret = platform->unrailgate(dev);
804 nvgpu_mutex_release(&platform->railgate_lock);
805 }
806
807#ifdef CONFIG_DEBUG_FS
808 g->pstats.last_rail_ungate_complete = jiffies;
809#endif
810
811 return ret;
812}
813
814/*
815 * Remove association of the driver with OS interrupt handler
816 */
817void nvgpu_free_irq(struct gk20a *g)
818{
819 struct device *dev = dev_from_gk20a(g);
820
821 devm_free_irq(dev, g->irq_stall, g);
822 if (g->irq_stall != g->irq_nonstall)
823 devm_free_irq(dev, g->irq_nonstall, g);
824}
825
826/*
827 * Idle the GPU in preparation of shutdown/remove.
828 * gk20a_driver_start_unload() does not idle the GPU, but instead changes the SW
829 * state to prevent further activity on the driver SW side.
830 * On driver removal quiesce() should be called after start_unload()
831 */
832int nvgpu_quiesce(struct gk20a *g)
833{
834 int err;
835 struct device *dev = dev_from_gk20a(g);
836
837 if (g->power_on) {
838 err = gk20a_wait_for_idle(g);
839 if (err) {
840 nvgpu_err(g, "failed to idle GPU, err=%d", err);
841 return err;
842 }
843
844 err = gk20a_fifo_disable_all_engine_activity(g, true);
845 if (err) {
846 nvgpu_err(g,
847 "failed to disable engine activity, err=%d",
848 err);
849 return err;
850 }
851
852 err = gk20a_fifo_wait_engine_idle(g);
853 if (err) {
854 nvgpu_err(g, "failed to idle engines, err=%d",
855 err);
856 return err;
857 }
858 }
859
860 if (gk20a_gpu_is_virtual(dev))
861 err = vgpu_pm_prepare_poweroff(dev);
862 else
863 err = gk20a_pm_prepare_poweroff(dev);
864
865 if (err)
866 nvgpu_err(g, "failed to prepare for poweroff, err=%d",
867 err);
868
869 return err;
870}
871
872static void gk20a_pm_shutdown(struct platform_device *pdev)
873{
874 struct gk20a_platform *platform = platform_get_drvdata(pdev);
875 struct gk20a *g = platform->g;
876 int err;
877
878 nvgpu_info(g, "shutting down");
879
880 /* vgpu has nothing to clean up currently */
881 if (gk20a_gpu_is_virtual(&pdev->dev))
882 return;
883
884 if (!g->power_on)
885 goto finish;
886
887 gk20a_driver_start_unload(g);
888
889 /* If GPU is already railgated,
890 * just prevent more requests, and return */
891 if (platform->is_railgated && platform->is_railgated(&pdev->dev)) {
892 __pm_runtime_disable(&pdev->dev, false);
893 nvgpu_info(g, "already railgated, shut down complete");
894 return;
895 }
896
897 /* Prevent more requests by disabling Runtime PM */
898 __pm_runtime_disable(&pdev->dev, false);
899
900 err = nvgpu_quiesce(g);
901 if (err)
902 goto finish;
903
904 err = gk20a_pm_railgate(&pdev->dev);
905 if (err)
906 nvgpu_err(g, "failed to railgate, err=%d", err);
907
908finish:
909 nvgpu_info(g, "shut down complete");
910}
911
912#ifdef CONFIG_PM
913static int gk20a_pm_runtime_resume(struct device *dev)
914{
915 int err = 0;
916
917 err = gk20a_pm_unrailgate(dev);
918 if (err)
919 goto fail;
920
921 if (gk20a_gpu_is_virtual(dev))
922 err = vgpu_pm_finalize_poweron(dev);
923 else
924 err = gk20a_pm_finalize_poweron(dev);
925 if (err)
926 goto fail_poweron;
927
928 return 0;
929
930fail_poweron:
931 gk20a_pm_railgate(dev);
932fail:
933 return err;
934}
935
936static int gk20a_pm_runtime_suspend(struct device *dev)
937{
938 int err = 0;
939 struct gk20a *g = get_gk20a(dev);
940
941 if (gk20a_gpu_is_virtual(dev))
942 err = vgpu_pm_prepare_poweroff(dev);
943 else
944 err = gk20a_pm_prepare_poweroff(dev);
945 if (err) {
946 nvgpu_err(g, "failed to power off, err=%d", err);
947 goto fail;
948 }
949
950 err = gk20a_pm_railgate(dev);
951 if (err)
952 goto fail;
953
954 return 0;
955
956fail:
957 gk20a_pm_finalize_poweron(dev);
958 pm_runtime_mark_last_busy(dev);
959 return err;
960}
961
962static int gk20a_pm_suspend(struct device *dev)
963{
964 struct gk20a_platform *platform = dev_get_drvdata(dev);
965 struct gk20a *g = get_gk20a(dev);
966 int ret = 0;
967 int idle_usage_count = 0;
968
969 if (!g->power_on) {
970 if (!pm_runtime_enabled(dev))
971 gk20a_pm_railgate(dev);
972 return 0;
973 }
974
975 if (nvgpu_atomic_read(&g->usage_count) > idle_usage_count)
976 return -EBUSY;
977
978 ret = gk20a_pm_runtime_suspend(dev);
979 if (ret)
980 return ret;
981
982 if (platform->suspend)
983 platform->suspend(dev);
984
985 g->suspended = true;
986
987 return 0;
988}
989
990static int gk20a_pm_resume(struct device *dev)
991{
992 struct gk20a *g = get_gk20a(dev);
993 int ret = 0;
994
995 if (!g->suspended) {
996 if (!pm_runtime_enabled(dev))
997 gk20a_pm_unrailgate(dev);
998 return 0;
999 }
1000
1001 ret = gk20a_pm_runtime_resume(dev);
1002
1003 g->suspended = false;
1004
1005 return ret;
1006}
1007
1008static const struct dev_pm_ops gk20a_pm_ops = {
1009 .runtime_resume = gk20a_pm_runtime_resume,
1010 .runtime_suspend = gk20a_pm_runtime_suspend,
1011 .resume = gk20a_pm_resume,
1012 .suspend = gk20a_pm_suspend,
1013};
1014#endif
1015
1016static int gk20a_pm_init(struct device *dev)
1017{
1018 struct gk20a *g = get_gk20a(dev);
1019 int err = 0;
1020
1021 nvgpu_log_fn(g, " ");
1022
1023 /*
1024 * Initialise pm runtime. For railgate disable
1025 * case, set autosuspend delay to negative which
1026 * will suspend runtime pm
1027 */
1028 if (g->railgate_delay && g->can_railgate)
1029 pm_runtime_set_autosuspend_delay(dev,
1030 g->railgate_delay);
1031 else
1032 pm_runtime_set_autosuspend_delay(dev, -1);
1033
1034 pm_runtime_use_autosuspend(dev);
1035 pm_runtime_enable(dev);
1036
1037 return err;
1038}
1039
1040/*
1041 * Start the process for unloading the driver. Set NVGPU_DRIVER_IS_DYING.
1042 */
1043void gk20a_driver_start_unload(struct gk20a *g)
1044{
1045 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
1046
1047 nvgpu_log(g, gpu_dbg_shutdown, "Driver is now going down!\n");
1048
1049 down_write(&l->busy_lock);
1050 __nvgpu_set_enabled(g, NVGPU_DRIVER_IS_DYING, true);
1051 /* GR SW ready needs to be invalidated at this time with the busy lock
1052 * held to prevent a racing condition on the gr/mm code */
1053 g->gr.sw_ready = false;
1054 g->sw_ready = false;
1055 up_write(&l->busy_lock);
1056
1057 if (g->is_virtual)
1058 return;
1059
1060 gk20a_wait_for_idle(g);
1061
1062 nvgpu_wait_for_deferred_interrupts(g);
1063
1064 if (l->nonstall_work_queue) {
1065 cancel_work_sync(&l->nonstall_fn_work);
1066 destroy_workqueue(l->nonstall_work_queue);
1067 l->nonstall_work_queue = NULL;
1068 }
1069}
1070
1071static inline void set_gk20a(struct platform_device *pdev, struct gk20a *gk20a)
1072{
1073 gk20a_get_platform(&pdev->dev)->g = gk20a;
1074}
1075
1076static int nvgpu_read_fuse_overrides(struct gk20a *g)
1077{
1078 struct device_node *np = nvgpu_get_node(g);
1079 u32 *fuses;
1080 int count, i;
1081
1082 if (!np) /* may be pcie device */
1083 return 0;
1084
1085 count = of_property_count_elems_of_size(np, "fuse-overrides", 8);
1086 if (count <= 0)
1087 return count;
1088
1089 fuses = nvgpu_kmalloc(g, sizeof(u32) * count * 2);
1090 if (!fuses)
1091 return -ENOMEM;
1092 of_property_read_u32_array(np, "fuse-overrides", fuses, count * 2);
1093 for (i = 0; i < count; i++) {
1094 u32 fuse, value;
1095
1096 fuse = fuses[2 * i];
1097 value = fuses[2 * i + 1];
1098 switch (fuse) {
1099 case GM20B_FUSE_OPT_TPC_DISABLE:
1100 g->tpc_fs_mask_user = ~value;
1101 break;
1102 case GP10B_FUSE_OPT_ECC_EN:
1103 g->gr.fecs_feature_override_ecc_val = value;
1104 break;
1105 default:
1106 nvgpu_err(g, "ignore unknown fuse override %08x", fuse);
1107 break;
1108 }
1109 }
1110
1111 nvgpu_kfree(g, fuses);
1112
1113 return 0;
1114}
1115
1116static int gk20a_probe(struct platform_device *dev)
1117{
1118 struct nvgpu_os_linux *l = NULL;
1119 struct gk20a *gk20a;
1120 int err;
1121 struct gk20a_platform *platform = NULL;
1122 struct device_node *np;
1123
1124 if (dev->dev.of_node) {
1125 const struct of_device_id *match;
1126
1127 match = of_match_device(tegra_gk20a_of_match, &dev->dev);
1128 if (match)
1129 platform = (struct gk20a_platform *)match->data;
1130 } else
1131 platform = (struct gk20a_platform *)dev->dev.platform_data;
1132
1133 if (!platform) {
1134 dev_err(&dev->dev, "no platform data\n");
1135 return -ENODATA;
1136 }
1137
1138 platform_set_drvdata(dev, platform);
1139
1140 if (gk20a_gpu_is_virtual(&dev->dev))
1141 return vgpu_probe(dev);
1142
1143 l = kzalloc(sizeof(*l), GFP_KERNEL);
1144 if (!l) {
1145 dev_err(&dev->dev, "couldn't allocate gk20a support");
1146 return -ENOMEM;
1147 }
1148
1149 hash_init(l->ecc_sysfs_stats_htable);
1150
1151 gk20a = &l->g;
1152
1153 nvgpu_log_fn(gk20a, " ");
1154
1155 nvgpu_init_gk20a(gk20a);
1156 set_gk20a(dev, gk20a);
1157 l->dev = &dev->dev;
1158 gk20a->log_mask = NVGPU_DEFAULT_DBG_MASK;
1159
1160 nvgpu_kmem_init(gk20a);
1161
1162 err = nvgpu_init_enabled_flags(gk20a);
1163 if (err)
1164 goto return_err;
1165
1166 np = nvgpu_get_node(gk20a);
1167 if (of_dma_is_coherent(np)) {
1168 __nvgpu_set_enabled(gk20a, NVGPU_USE_COHERENT_SYSMEM, true);
1169 __nvgpu_set_enabled(gk20a, NVGPU_SUPPORT_IO_COHERENCE, true);
1170 }
1171
1172 if (nvgpu_platform_is_simulation(gk20a))
1173 __nvgpu_set_enabled(gk20a, NVGPU_IS_FMODEL, true);
1174
1175 gk20a->irq_stall = platform_get_irq(dev, 0);
1176 gk20a->irq_nonstall = platform_get_irq(dev, 1);
1177 if (gk20a->irq_stall < 0 || gk20a->irq_nonstall < 0) {
1178 err = -ENXIO;
1179 goto return_err;
1180 }
1181
1182 err = devm_request_threaded_irq(&dev->dev,
1183 gk20a->irq_stall,
1184 gk20a_intr_isr_stall,
1185 gk20a_intr_thread_stall,
1186 0, "gk20a_stall", gk20a);
1187 if (err) {
1188 dev_err(&dev->dev,
1189 "failed to request stall intr irq @ %d\n",
1190 gk20a->irq_stall);
1191 goto return_err;
1192 }
1193 err = devm_request_irq(&dev->dev,
1194 gk20a->irq_nonstall,
1195 gk20a_intr_isr_nonstall,
1196 0, "gk20a_nonstall", gk20a);
1197 if (err) {
1198 dev_err(&dev->dev,
1199 "failed to request non-stall intr irq @ %d\n",
1200 gk20a->irq_nonstall);
1201 goto return_err;
1202 }
1203 disable_irq(gk20a->irq_stall);
1204 if (gk20a->irq_stall != gk20a->irq_nonstall)
1205 disable_irq(gk20a->irq_nonstall);
1206
1207 err = gk20a_init_support(dev);
1208 if (err)
1209 goto return_err;
1210
1211 err = nvgpu_read_fuse_overrides(gk20a);
1212
1213#ifdef CONFIG_RESET_CONTROLLER
1214 platform->reset_control = devm_reset_control_get(&dev->dev, NULL);
1215 if (IS_ERR(platform->reset_control))
1216 platform->reset_control = NULL;
1217#endif
1218
1219 err = nvgpu_probe(gk20a, "gpu.0", INTERFACE_NAME, &nvgpu_class);
1220 if (err)
1221 goto return_err;
1222
1223 err = gk20a_pm_init(&dev->dev);
1224 if (err) {
1225 dev_err(&dev->dev, "pm init failed");
1226 goto return_err;
1227 }
1228
1229 gk20a->mm.has_physical_mode = !nvgpu_is_hypervisor_mode(gk20a);
1230
1231 return 0;
1232
1233return_err:
1234 nvgpu_free_enabled_flags(gk20a);
1235
1236 /*
1237 * Last since the above allocs may use data structures in here.
1238 */
1239 nvgpu_kmem_fini(gk20a, NVGPU_KMEM_FINI_FORCE_CLEANUP);
1240
1241 kfree(l);
1242
1243 return err;
1244}
1245
1246int nvgpu_remove(struct device *dev, struct class *class)
1247{
1248 struct gk20a *g = get_gk20a(dev);
1249#ifdef CONFIG_NVGPU_SUPPORT_CDE
1250 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
1251#endif
1252 struct gk20a_platform *platform = gk20a_get_platform(dev);
1253 int err;
1254
1255 nvgpu_log_fn(g, " ");
1256
1257 err = nvgpu_quiesce(g);
1258 WARN(err, "gpu failed to idle during driver removal");
1259
1260 if (nvgpu_mem_is_valid(&g->syncpt_mem))
1261 nvgpu_dma_free(g, &g->syncpt_mem);
1262
1263#ifdef CONFIG_NVGPU_SUPPORT_CDE
1264 if (platform->has_cde)
1265 gk20a_cde_destroy(l);
1266#endif
1267
1268#ifdef CONFIG_GK20A_CTXSW_TRACE
1269 gk20a_ctxsw_trace_cleanup(g);
1270#endif
1271
1272 gk20a_sched_ctrl_cleanup(g);
1273
1274 if (IS_ENABLED(CONFIG_GK20A_DEVFREQ))
1275 gk20a_scale_exit(dev);
1276
1277 nvgpu_clk_arb_cleanup_arbiter(g);
1278
1279 gk20a_user_deinit(dev, class);
1280
1281 gk20a_debug_deinit(g);
1282
1283 nvgpu_remove_sysfs(dev);
1284
1285 if (platform->secure_buffer.destroy)
1286 platform->secure_buffer.destroy(g,
1287 &platform->secure_buffer);
1288
1289 if (pm_runtime_enabled(dev))
1290 pm_runtime_disable(dev);
1291
1292 if (platform->remove)
1293 platform->remove(dev);
1294
1295 nvgpu_log_fn(g, "removed");
1296
1297 return err;
1298}
1299
1300static int __exit gk20a_remove(struct platform_device *pdev)
1301{
1302 int err;
1303 struct device *dev = &pdev->dev;
1304 struct gk20a *g = get_gk20a(dev);
1305
1306 if (gk20a_gpu_is_virtual(dev))
1307 return vgpu_remove(pdev);
1308
1309 err = nvgpu_remove(dev, &nvgpu_class);
1310
1311 set_gk20a(pdev, NULL);
1312 gk20a_put(g);
1313
1314 return err;
1315}
1316
1317static struct platform_driver gk20a_driver = {
1318 .probe = gk20a_probe,
1319 .remove = __exit_p(gk20a_remove),
1320 .shutdown = gk20a_pm_shutdown,
1321 .driver = {
1322 .owner = THIS_MODULE,
1323 .name = "gk20a",
1324 .probe_type = PROBE_PREFER_ASYNCHRONOUS,
1325#ifdef CONFIG_OF
1326 .of_match_table = tegra_gk20a_of_match,
1327#endif
1328#ifdef CONFIG_PM
1329 .pm = &gk20a_pm_ops,
1330#endif
1331 .suppress_bind_attrs = true,
1332 }
1333};
1334
1335struct class nvgpu_class = {
1336 .owner = THIS_MODULE,
1337 .name = CLASS_NAME,
1338};
1339
1340static int __init gk20a_init(void)
1341{
1342
1343 int ret;
1344
1345 ret = class_register(&nvgpu_class);
1346 if (ret)
1347 return ret;
1348
1349 ret = nvgpu_pci_init();
1350 if (ret)
1351 return ret;
1352
1353 return platform_driver_register(&gk20a_driver);
1354}
1355
1356static void __exit gk20a_exit(void)
1357{
1358 nvgpu_pci_exit();
1359 platform_driver_unregister(&gk20a_driver);
1360 class_unregister(&nvgpu_class);
1361}
1362
1363MODULE_LICENSE("GPL v2");
1364module_init(gk20a_init);
1365module_exit(gk20a_exit);
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-05 02:48:13 -0400