aboutsummaryrefslogtreecommitdiffstats
path: root/include/os/linux
diff options
context:
space:
mode:
authorJoshua Bakita <bakitajoshua@gmail.com>2023-10-29 13:07:40 -0400
committerJoshua Bakita <bakitajoshua@gmail.com>2023-10-29 13:10:52 -0400
commit2c5337a24f7f2d02989dfb733c55d6d8c7e90493 (patch)
treeb9f1028cb443b03190b710c0d7ee640bf5958631 /include/os/linux
parentaa06f84f03cba7ad1aae5cd527355bb3d8c152a6 (diff)
Update includes to L4T r32.7.4 and drop nvgpu/gk20a.h dependency
Also add instructions for updating `include/`. These files are now only needed to build on Linux 4.9-based Tegra platforms.
Diffstat (limited to 'include/os/linux')
-rw-r--r--include/os/linux/debug.c4
-rw-r--r--include/os/linux/driver_common.c69
-rw-r--r--include/os/linux/ecc_linux.h49
-rw-r--r--include/os/linux/ioctl_clk_arb.c17
-rw-r--r--include/os/linux/ioctl_ctrl.c122
-rw-r--r--include/os/linux/ioctl_tsg.c26
-rw-r--r--include/os/linux/module.c22
-rw-r--r--include/os/linux/os_linux.h7
-rw-r--r--include/os/linux/platform_gv11b_tegra.c4
-rw-r--r--include/os/linux/scale.c29
-rw-r--r--include/os/linux/sdl.c341
-rw-r--r--include/os/linux/vm.c8
12 files changed, 601 insertions, 97 deletions
diff --git a/include/os/linux/debug.c b/include/os/linux/debug.c
index 5f0703c..b8c4596 100644
--- a/include/os/linux/debug.c
+++ b/include/os/linux/debug.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (C) 2017-2018 NVIDIA Corporation. All rights reserved. 2 * Copyright (C) 2017-2021 NVIDIA Corporation. All rights reserved.
3 * 3 *
4 * This software is licensed under the terms of the GNU General Public 4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and 5 * License version 2, as published by the Free Software Foundation, and
@@ -224,7 +224,7 @@ static int railgate_residency_show(struct seq_file *s, void *data)
224 unsigned long total_rail_gate_time_ms; 224 unsigned long total_rail_gate_time_ms;
225 unsigned long total_rail_ungate_time_ms; 225 unsigned long total_rail_ungate_time_ms;
226 226
227 if (platform->is_railgated(dev_from_gk20a(g))) { 227 if (platform && platform->is_railgated && platform->is_railgated(dev_from_gk20a(g))) {
228 time_since_last_state_transition_ms = 228 time_since_last_state_transition_ms =
229 jiffies_to_msecs(jiffies - 229 jiffies_to_msecs(jiffies -
230 g->pstats.last_rail_gate_complete); 230 g->pstats.last_rail_gate_complete);
diff --git a/include/os/linux/driver_common.c b/include/os/linux/driver_common.c
index c76dabe..8f5872d 100644
--- a/include/os/linux/driver_common.c
+++ b/include/os/linux/driver_common.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. 2 * Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify it 4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License, 5 * under the terms and conditions of the GNU General Public License,
@@ -18,6 +18,7 @@
18#include <linux/dma-mapping.h> 18#include <linux/dma-mapping.h>
19#include <linux/mm.h> 19#include <linux/mm.h>
20#include <linux/slab.h> 20#include <linux/slab.h>
21#include <linux/of_platform.h>
21#include <uapi/linux/nvgpu.h> 22#include <uapi/linux/nvgpu.h>
22 23
23#include <nvgpu/defaults.h> 24#include <nvgpu/defaults.h>
@@ -241,6 +242,8 @@ int nvgpu_probe(struct gk20a *g,
241 struct device *dev = dev_from_gk20a(g); 242 struct device *dev = dev_from_gk20a(g);
242 struct gk20a_platform *platform = dev_get_drvdata(dev); 243 struct gk20a_platform *platform = dev_get_drvdata(dev);
243 int err = 0; 244 int err = 0;
245 struct device_node *np = dev->of_node;
246 bool disable_l3_alloc = false;
244 247
245 nvgpu_init_vars(g); 248 nvgpu_init_vars(g);
246 nvgpu_init_gr_vars(g); 249 nvgpu_init_gr_vars(g);
@@ -265,6 +268,12 @@ int nvgpu_probe(struct gk20a *g,
265 return err; 268 return err;
266 } 269 }
267 270
271 disable_l3_alloc = of_property_read_bool(np, "disable_l3_alloc");
272 if (disable_l3_alloc) {
273 nvgpu_log_info(g, "L3 alloc is disabled\n");
274 __nvgpu_set_enabled(g, NVGPU_DISABLE_L3_SUPPORT, true);
275 }
276
268 nvgpu_init_mm_vars(g); 277 nvgpu_init_mm_vars(g);
269 278
270 /* platform probe can defer do user init only if probe succeeds */ 279 /* platform probe can defer do user init only if probe succeeds */
@@ -312,30 +321,70 @@ static int cyclic_delta(int a, int b)
312} 321}
313 322
314/** 323/**
315 * nvgpu_wait_for_deferred_interrupts - Wait for interrupts to complete 324 * nvgpu_wait_for_stall_interrupts - Wait for the stalling interrupts to
325 * complete.
316 * 326 *
317 * @g - The GPU to wait on. 327 * @g - The GPU to wait on.
328 * @timeout - maximum time period to wait for.
318 * 329 *
319 * Waits until all interrupt handlers that have been scheduled to run have 330 * Waits until all stalling interrupt handlers that have been scheduled to run
320 * completed. 331 * have completed.
321 */ 332 */
322void nvgpu_wait_for_deferred_interrupts(struct gk20a *g) 333int nvgpu_wait_for_stall_interrupts(struct gk20a *g, u32 timeout)
323{ 334{
324 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); 335 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
325 int stall_irq_threshold = atomic_read(&l->hw_irq_stall_count); 336 int stall_irq_threshold = atomic_read(&l->hw_irq_stall_count);
326 int nonstall_irq_threshold = atomic_read(&l->hw_irq_nonstall_count);
327 337
328 /* wait until all stalling irqs are handled */ 338 /* wait until all stalling irqs are handled */
329 NVGPU_COND_WAIT(&l->sw_irq_stall_last_handled_wq, 339 return NVGPU_COND_WAIT(&l->sw_irq_stall_last_handled_wq,
330 cyclic_delta(stall_irq_threshold, 340 cyclic_delta(stall_irq_threshold,
331 atomic_read(&l->sw_irq_stall_last_handled)) 341 atomic_read(&l->sw_irq_stall_last_handled))
332 <= 0, 0); 342 <= 0, timeout);
343}
344
345/**
346 * nvgpu_wait_for_nonstall_interrupts - Wait for the nonstalling interrupts to
347 * complete.
348 *
349 * @g - The GPU to wait on.
350 * @timeout - maximum time period to wait for.
351 *
352 * Waits until all non-stalling interrupt handlers that have been scheduled to
353 * run have completed.
354 */
355int nvgpu_wait_for_nonstall_interrupts(struct gk20a *g, u32 timeout)
356{
357 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
358 int nonstall_irq_threshold = atomic_read(&l->hw_irq_nonstall_count);
333 359
334 /* wait until all non-stalling irqs are handled */ 360 /* wait until all non-stalling irqs are handled */
335 NVGPU_COND_WAIT(&l->sw_irq_nonstall_last_handled_wq, 361 return NVGPU_COND_WAIT(&l->sw_irq_nonstall_last_handled_wq,
336 cyclic_delta(nonstall_irq_threshold, 362 cyclic_delta(nonstall_irq_threshold,
337 atomic_read(&l->sw_irq_nonstall_last_handled)) 363 atomic_read(&l->sw_irq_nonstall_last_handled))
338 <= 0, 0); 364 <= 0, timeout);
365}
366
367/**
368 * nvgpu_wait_for_deferred_interrupts - Wait for interrupts to complete
369 *
370 * @g - The GPU to wait on.
371 *
372 * Waits until all interrupt handlers that have been scheduled to run have
373 * completed.
374 */
375void nvgpu_wait_for_deferred_interrupts(struct gk20a *g)
376{
377 int ret;
378
379 ret = nvgpu_wait_for_stall_interrupts(g, 0U);
380 if (ret != 0) {
381 nvgpu_err(g, "wait for stall interrupts failed %d", ret);
382 }
383
384 ret = nvgpu_wait_for_nonstall_interrupts(g, 0U);
385 if (ret != 0) {
386 nvgpu_err(g, "wait for nonstall interrupts failed %d", ret);
387 }
339} 388}
340 389
341static void nvgpu_free_gk20a(struct gk20a *g) 390static void nvgpu_free_gk20a(struct gk20a *g)
diff --git a/include/os/linux/ecc_linux.h b/include/os/linux/ecc_linux.h
new file mode 100644
index 0000000..7e0f650
--- /dev/null
+++ b/include/os/linux/ecc_linux.h
@@ -0,0 +1,49 @@
1/*
2 *
3 * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24#ifndef NVGPU_OS_ECC_LINUX_H
25#define NVGPU_OS_ECC_LINUX_H
26
27#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING
28
29#include <linux/tegra_l1ss_kernel_interface.h>
30#include <linux/tegra_l1ss_ioctl.h>
31#include <linux/tegra_nv_guard_service_id.h>
32#include <linux/tegra_nv_guard_group_id.h>
33
34#include <nvgpu/nvgpu_err.h>
35
36struct nvgpu_ecc_reporting_linux {
37 struct nvgpu_ecc_reporting common;
38 client_param_t priv;
39};
40
41static inline struct nvgpu_ecc_reporting_linux *get_ecc_reporting_linux(
42 struct nvgpu_ecc_reporting *ecc_report)
43{
44 return container_of(ecc_report, struct nvgpu_ecc_reporting_linux, common);
45}
46
47#endif /* CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING */
48
49#endif \ No newline at end of file
diff --git a/include/os/linux/ioctl_clk_arb.c b/include/os/linux/ioctl_clk_arb.c
index 477222d..9f32102 100644
--- a/include/os/linux/ioctl_clk_arb.c
+++ b/include/os/linux/ioctl_clk_arb.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. 2 * Copyright (c) 2016-2021, NVIDIA CORPORATION. All rights reserved.
3 * 3 *
4 * This software is licensed under the terms of the GNU General Public 4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and 5 * License version 2, as published by the Free Software Foundation, and
@@ -51,19 +51,28 @@ static int nvgpu_clk_arb_release_completion_dev(struct inode *inode,
51{ 51{
52 struct nvgpu_clk_dev *dev = filp->private_data; 52 struct nvgpu_clk_dev *dev = filp->private_data;
53 struct nvgpu_clk_session *session = dev->session; 53 struct nvgpu_clk_session *session = dev->session;
54 struct gk20a *g = session->g;
55 struct nvgpu_clk_arb *arb = g->clk_arb;
54 56
57 clk_arb_dbg(g, " ");
55 58
56 clk_arb_dbg(session->g, " "); 59 nvgpu_spinlock_acquire(&session->session_lock);
60 nvgpu_spinlock_acquire(&arb->requests_lock);
61
62 nvgpu_list_del(&dev->node);
63
64 nvgpu_spinlock_release(&arb->requests_lock);
65 nvgpu_spinlock_release(&session->session_lock);
57 66
58 /* This is done to account for the extra refcount taken in 67 /* This is done to account for the extra refcount taken in
59 * nvgpu_clk_arb_commit_request_fd without events support in iGPU 68 * nvgpu_clk_arb_commit_request_fd without events support in iGPU
60 */ 69 */
61 if (!session->g->clk_arb->clk_arb_events_supported) { 70 if (!arb->clk_arb_events_supported) {
62 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd); 71 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
63 } 72 }
64 73
65 nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
66 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd); 74 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
75 nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
67 return 0; 76 return 0;
68} 77}
69 78
diff --git a/include/os/linux/ioctl_ctrl.c b/include/os/linux/ioctl_ctrl.c
index ee141ff..841d345 100644
--- a/include/os/linux/ioctl_ctrl.c
+++ b/include/os/linux/ioctl_ctrl.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (c) 2011-2020, NVIDIA Corporation. All rights reserved. 2 * Copyright (c) 2011-2021, NVIDIA Corporation. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify it 4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License, 5 * under the terms and conditions of the GNU General Public License,
@@ -60,7 +60,6 @@ struct gk20a_ctrl_priv {
60 struct nvgpu_list_node list; 60 struct nvgpu_list_node list;
61 struct { 61 struct {
62 struct vm_area_struct *vma; 62 struct vm_area_struct *vma;
63 unsigned long flags;
64 bool vma_mapped; 63 bool vma_mapped;
65 } usermode_vma; 64 } usermode_vma;
66}; 65};
@@ -488,27 +487,26 @@ static int gk20a_ctrl_alloc_as(
488 487
489 snprintf(name, sizeof(name), "nvhost-%s-fd%d", g->name, fd); 488 snprintf(name, sizeof(name), "nvhost-%s-fd%d", g->name, fd);
490 489
491 file = anon_inode_getfile(name, l->as_dev.cdev.ops, NULL, O_RDWR);
492 if (IS_ERR(file)) {
493 err = PTR_ERR(file);
494 goto clean_up;
495 }
496
497 err = gk20a_as_alloc_share(g, args->big_page_size, 490 err = gk20a_as_alloc_share(g, args->big_page_size,
498 gk20a_as_translate_as_alloc_flags(g, 491 gk20a_as_translate_as_alloc_flags(g,
499 args->flags), 492 args->flags),
500 &as_share); 493 &as_share);
501 if (err) 494 if (err)
502 goto clean_up_file; 495 goto clean_up;
496
497 file = anon_inode_getfile(name, l->as_dev.cdev.ops, as_share, O_RDWR);
498 if (IS_ERR(file)) {
499 err = PTR_ERR(file);
500 goto clean_up_as;
501 }
503 502
504 fd_install(fd, file); 503 fd_install(fd, file);
505 file->private_data = as_share;
506 504
507 args->as_fd = fd; 505 args->as_fd = fd;
508 return 0; 506 return 0;
509 507
510clean_up_file: 508clean_up_as:
511 fput(file); 509 gk20a_as_release_share(as_share);
512clean_up: 510clean_up:
513 put_unused_fd(fd); 511 put_unused_fd(fd);
514 return err; 512 return err;
@@ -692,12 +690,15 @@ static int nvgpu_gpu_ioctl_trigger_suspend(struct gk20a *g)
692 690
693 err = gk20a_busy(g); 691 err = gk20a_busy(g);
694 if (err) 692 if (err)
695 return err; 693 return err;
696 694
697 nvgpu_mutex_acquire(&g->dbg_sessions_lock); 695 if (g->ops.gr.trigger_suspend) {
698 err = gr_gk20a_elpg_protected_call(g, 696 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
697 err = gr_gk20a_elpg_protected_call(g,
699 g->ops.gr.trigger_suspend(g)); 698 g->ops.gr.trigger_suspend(g));
700 nvgpu_mutex_release(&g->dbg_sessions_lock); 699 nvgpu_mutex_release(&g->dbg_sessions_lock);
700 } else
701 err = -EINVAL;
701 702
702 gk20a_idle(g); 703 gk20a_idle(g);
703 704
@@ -731,8 +732,13 @@ static int nvgpu_gpu_ioctl_wait_for_pause(struct gk20a *g,
731 goto out_free; 732 goto out_free;
732 733
733 nvgpu_mutex_acquire(&g->dbg_sessions_lock); 734 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
734 (void)gr_gk20a_elpg_protected_call(g, 735 if (g->ops.gr.wait_for_pause) {
736 (void)gr_gk20a_elpg_protected_call(g,
735 g->ops.gr.wait_for_pause(g, w_state)); 737 g->ops.gr.wait_for_pause(g, w_state));
738 } else {
739 err = -EINVAL;
740 goto out_idle;
741 }
736 742
737 for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) { 743 for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) {
738 ioctl_w_state[sm_id].valid_warps[0] = 744 ioctl_w_state[sm_id].valid_warps[0] =
@@ -755,6 +761,7 @@ static int nvgpu_gpu_ioctl_wait_for_pause(struct gk20a *g,
755 err = -EFAULT; 761 err = -EFAULT;
756 } 762 }
757 763
764out_idle:
758 nvgpu_mutex_release(&g->dbg_sessions_lock); 765 nvgpu_mutex_release(&g->dbg_sessions_lock);
759 766
760 gk20a_idle(g); 767 gk20a_idle(g);
@@ -772,12 +779,15 @@ static int nvgpu_gpu_ioctl_resume_from_pause(struct gk20a *g)
772 779
773 err = gk20a_busy(g); 780 err = gk20a_busy(g);
774 if (err) 781 if (err)
775 return err; 782 return err;
776 783
777 nvgpu_mutex_acquire(&g->dbg_sessions_lock); 784 if (g->ops.gr.resume_from_pause) {
778 err = gr_gk20a_elpg_protected_call(g, 785 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
786 err = gr_gk20a_elpg_protected_call(g,
779 g->ops.gr.resume_from_pause(g)); 787 g->ops.gr.resume_from_pause(g));
780 nvgpu_mutex_release(&g->dbg_sessions_lock); 788 nvgpu_mutex_release(&g->dbg_sessions_lock);
789 } else
790 err = -EINVAL;
781 791
782 gk20a_idle(g); 792 gk20a_idle(g);
783 793
@@ -792,8 +802,11 @@ static int nvgpu_gpu_ioctl_clear_sm_errors(struct gk20a *g)
792 if (err) 802 if (err)
793 return err; 803 return err;
794 804
795 err = gr_gk20a_elpg_protected_call(g, 805 if (g->ops.gr.clear_sm_errors) {
806 err = gr_gk20a_elpg_protected_call(g,
796 g->ops.gr.clear_sm_errors(g)); 807 g->ops.gr.clear_sm_errors(g));
808 } else
809 err = -EINVAL;
797 810
798 gk20a_idle(g); 811 gk20a_idle(g);
799 812
@@ -806,9 +819,12 @@ static int nvgpu_gpu_ioctl_has_any_exception(
806{ 819{
807 u32 tpc_exception_en; 820 u32 tpc_exception_en;
808 821
809 nvgpu_mutex_acquire(&g->dbg_sessions_lock); 822 if (g->ops.gr.tpc_enabled_exceptions) {
810 tpc_exception_en = g->ops.gr.tpc_enabled_exceptions(g); 823 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
811 nvgpu_mutex_release(&g->dbg_sessions_lock); 824 tpc_exception_en = g->ops.gr.tpc_enabled_exceptions(g);
825 nvgpu_mutex_release(&g->dbg_sessions_lock);
826 } else
827 return -EINVAL;
812 828
813 args->tpc_exception_en_sm_mask = tpc_exception_en; 829 args->tpc_exception_en_sm_mask = tpc_exception_en;
814 830
@@ -2023,7 +2039,6 @@ int gk20a_ctrl_dev_mmap(struct file *filp, struct vm_area_struct *vma)
2023 vma->vm_end - vma->vm_start, vma->vm_page_prot); 2039 vma->vm_end - vma->vm_start, vma->vm_page_prot);
2024 if (!err) { 2040 if (!err) {
2025 priv->usermode_vma.vma = vma; 2041 priv->usermode_vma.vma = vma;
2026 priv->usermode_vma.flags = vma->vm_flags;
2027 vma->vm_private_data = priv; 2042 vma->vm_private_data = priv;
2028 priv->usermode_vma.vma_mapped = true; 2043 priv->usermode_vma.vma_mapped = true;
2029 } 2044 }
@@ -2034,7 +2049,7 @@ int gk20a_ctrl_dev_mmap(struct file *filp, struct vm_area_struct *vma)
2034 return err; 2049 return err;
2035} 2050}
2036 2051
2037static void alter_usermode_mapping(struct gk20a *g, 2052static int alter_usermode_mapping(struct gk20a *g,
2038 struct gk20a_ctrl_priv *priv, 2053 struct gk20a_ctrl_priv *priv,
2039 bool poweroff) 2054 bool poweroff)
2040{ 2055{
@@ -2042,57 +2057,80 @@ static void alter_usermode_mapping(struct gk20a *g,
2042 struct vm_area_struct *vma = priv->usermode_vma.vma; 2057 struct vm_area_struct *vma = priv->usermode_vma.vma;
2043 bool vma_mapped = priv->usermode_vma.vma_mapped; 2058 bool vma_mapped = priv->usermode_vma.vma_mapped;
2044 u64 addr; 2059 u64 addr;
2045 int err; 2060 int err = 0;
2046 2061
2047 if (!vma) { 2062 if (!vma) {
2048 /* Nothing to do - no mmap called */ 2063 /* Nothing to do - no mmap called */
2049 return; 2064 return 0;
2050 } 2065 }
2051 2066
2052 addr = l->regs_bus_addr + g->ops.fifo.usermode_base(g); 2067 addr = l->regs_bus_addr + g->ops.fifo.usermode_base(g);
2053 2068
2054 down_write(&vma->vm_mm->mmap_sem);
2055
2056 /* 2069 /*
2057 * This is a no-op for the below cases 2070 * This is a no-op for the below cases
2058 * a) poweroff and !vma_mapped - > do nothing as no map exists 2071 * a) poweroff and !vma_mapped - > do nothing as no map exists
2059 * b) !poweroff and vmap_mapped -> do nothing as already mapped 2072 * b) !poweroff and vmap_mapped -> do nothing as already mapped
2060 */ 2073 */
2061 if (poweroff && vma_mapped) { 2074 if (poweroff != vma_mapped) {
2075 return 0;
2076 }
2077
2078 /*
2079 * We use trylock due to lock inversion: we need to acquire
2080 * mmap_lock while holding ctrl_privs_lock. usermode_vma_close
2081 * does it in reverse order. Trylock is a way to avoid deadlock.
2082 */
2083 if (!down_write_trylock(&vma->vm_mm->mmap_sem)) {
2084 return -EBUSY;
2085 }
2086
2087 if (poweroff) {
2062 err = zap_vma_ptes(vma, vma->vm_start, SZ_4K); 2088 err = zap_vma_ptes(vma, vma->vm_start, SZ_4K);
2063 if (err == 0) { 2089 if (err == 0) {
2064 vma->vm_flags = VM_NONE;
2065 priv->usermode_vma.vma_mapped = false; 2090 priv->usermode_vma.vma_mapped = false;
2066 } else { 2091 } else {
2067 nvgpu_err(g, "can't remove usermode mapping"); 2092 nvgpu_err(g, "can't remove usermode mapping");
2068 } 2093 }
2069 } else if (!poweroff && !vma_mapped) { 2094 } else {
2070 vma->vm_flags = priv->usermode_vma.flags;
2071 err = io_remap_pfn_range(vma, vma->vm_start, 2095 err = io_remap_pfn_range(vma, vma->vm_start,
2072 addr >> PAGE_SHIFT, 2096 addr >> PAGE_SHIFT,
2073 SZ_4K, vma->vm_page_prot); 2097 SZ_4K, vma->vm_page_prot);
2074 if (err != 0) { 2098 if (err != 0) {
2075 nvgpu_err(g, "can't restore usermode mapping"); 2099 nvgpu_err(g, "can't restore usermode mapping");
2076 vma->vm_flags = VM_NONE;
2077 } else { 2100 } else {
2078 priv->usermode_vma.vma_mapped = true; 2101 priv->usermode_vma.vma_mapped = true;
2079 } 2102 }
2080 } 2103 }
2081 2104
2082 up_write(&vma->vm_mm->mmap_sem); 2105 up_write(&vma->vm_mm->mmap_sem);
2106
2107 return err;
2083} 2108}
2084 2109
2085static void alter_usermode_mappings(struct gk20a *g, bool poweroff) 2110static void alter_usermode_mappings(struct gk20a *g, bool poweroff)
2086{ 2111{
2087 struct gk20a_ctrl_priv *priv; 2112 struct gk20a_ctrl_priv *priv;
2088 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); 2113 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
2114 int err = 0;
2089 2115
2090 nvgpu_mutex_acquire(&l->ctrl.privs_lock); 2116 do {
2091 nvgpu_list_for_each_entry(priv, &l->ctrl.privs, 2117 nvgpu_mutex_acquire(&l->ctrl.privs_lock);
2092 gk20a_ctrl_priv, list) { 2118 nvgpu_list_for_each_entry(priv, &l->ctrl.privs,
2093 alter_usermode_mapping(g, priv, poweroff); 2119 gk20a_ctrl_priv, list) {
2094 } 2120 err = alter_usermode_mapping(g, priv, poweroff);
2095 nvgpu_mutex_release(&l->ctrl.privs_lock); 2121 if (err != 0) {
2122 break;
2123 }
2124 }
2125 nvgpu_mutex_release(&l->ctrl.privs_lock);
2126
2127 if (err == -EBUSY) {
2128 nvgpu_log_info(g, "ctrl_privs_lock lock contended. retry altering usermode mappings");
2129 nvgpu_udelay(10);
2130 } else if (err != 0) {
2131 nvgpu_err(g, "can't alter usermode mapping. err = %d", err);
2132 }
2133 } while (err == -EBUSY);
2096} 2134}
2097 2135
2098void nvgpu_hide_usermode_for_poweroff(struct gk20a *g) 2136void nvgpu_hide_usermode_for_poweroff(struct gk20a *g)
diff --git a/include/os/linux/ioctl_tsg.c b/include/os/linux/ioctl_tsg.c
index 2f8cb3a..296b02b 100644
--- a/include/os/linux/ioctl_tsg.c
+++ b/include/os/linux/ioctl_tsg.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved. 2 * Copyright (c) 2014-2021, NVIDIA CORPORATION. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify it 4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License, 5 * under the terms and conditions of the GNU General Public License,
@@ -134,7 +134,10 @@ static int gk20a_tsg_unbind_channel_fd(struct tsg_gk20a *tsg, int ch_fd)
134 goto out; 134 goto out;
135 } 135 }
136 136
137 err = gk20a_tsg_unbind_channel(ch); 137 err = gk20a_tsg_unbind_channel(ch, false);
138 if (err == -EAGAIN) {
139 goto out;
140 }
138 141
139 /* 142 /*
140 * Mark the channel timedout since channel unbound from TSG 143 * Mark the channel timedout since channel unbound from TSG
@@ -307,17 +310,10 @@ static int gk20a_tsg_event_id_enable(struct tsg_gk20a *tsg,
307 snprintf(name, sizeof(name), "nvgpu-event%d-fd%d", 310 snprintf(name, sizeof(name), "nvgpu-event%d-fd%d",
308 event_id, local_fd); 311 event_id, local_fd);
309 312
310 file = anon_inode_getfile(name, &gk20a_event_id_ops,
311 NULL, O_RDWR);
312 if (IS_ERR(file)) {
313 err = PTR_ERR(file);
314 goto clean_up;
315 }
316
317 event_id_data = nvgpu_kzalloc(tsg->g, sizeof(*event_id_data)); 313 event_id_data = nvgpu_kzalloc(tsg->g, sizeof(*event_id_data));
318 if (!event_id_data) { 314 if (!event_id_data) {
319 err = -ENOMEM; 315 err = -ENOMEM;
320 goto clean_up_file; 316 goto clean_up;
321 } 317 }
322 event_id_data->g = g; 318 event_id_data->g = g;
323 event_id_data->id = tsg->tsgid; 319 event_id_data->id = tsg->tsgid;
@@ -330,12 +326,18 @@ static int gk20a_tsg_event_id_enable(struct tsg_gk20a *tsg,
330 326
331 nvgpu_init_list_node(&event_id_data->event_id_node); 327 nvgpu_init_list_node(&event_id_data->event_id_node);
332 328
329 file = anon_inode_getfile(name, &gk20a_event_id_ops,
330 event_id_data, O_RDWR);
331 if (IS_ERR(file)) {
332 err = PTR_ERR(file);
333 goto clean_up_free;
334 }
335
333 nvgpu_mutex_acquire(&tsg->event_id_list_lock); 336 nvgpu_mutex_acquire(&tsg->event_id_list_lock);
334 nvgpu_list_add_tail(&event_id_data->event_id_node, &tsg->event_id_list); 337 nvgpu_list_add_tail(&event_id_data->event_id_node, &tsg->event_id_list);
335 nvgpu_mutex_release(&tsg->event_id_list_lock); 338 nvgpu_mutex_release(&tsg->event_id_list_lock);
336 339
337 fd_install(local_fd, file); 340 fd_install(local_fd, file);
338 file->private_data = event_id_data;
339 341
340 *fd = local_fd; 342 *fd = local_fd;
341 343
@@ -343,8 +345,6 @@ static int gk20a_tsg_event_id_enable(struct tsg_gk20a *tsg,
343 345
344clean_up_free: 346clean_up_free:
345 nvgpu_kfree(g, event_id_data); 347 nvgpu_kfree(g, event_id_data);
346clean_up_file:
347 fput(file);
348clean_up: 348clean_up:
349 put_unused_fd(local_fd); 349 put_unused_fd(local_fd);
350free_ref: 350free_ref:
diff --git a/include/os/linux/module.c b/include/os/linux/module.c
index 807df2c..fdbab46 100644
--- a/include/os/linux/module.c
+++ b/include/os/linux/module.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * GK20A Graphics 2 * GK20A Graphics
3 * 3 *
4 * Copyright (c) 2011-2020, NVIDIA CORPORATION. All rights reserved. 4 * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved.
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify it 6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License, 7 * under the terms and conditions of the GNU General Public License,
@@ -49,6 +49,7 @@
49#include <nvgpu/clk_arb.h> 49#include <nvgpu/clk_arb.h>
50#include <nvgpu/timers.h> 50#include <nvgpu/timers.h>
51#include <nvgpu/channel.h> 51#include <nvgpu/channel.h>
52#include <nvgpu/nvgpu_err.h>
52 53
53#include "platform_gk20a.h" 54#include "platform_gk20a.h"
54#include "sysfs.h" 55#include "sysfs.h"
@@ -355,6 +356,10 @@ int gk20a_pm_finalize_poweron(struct device *dev)
355 gk20a_init_cde_support(l); 356 gk20a_init_cde_support(l);
356#endif 357#endif
357 358
359#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING
360 nvgpu_enable_ecc_reporting(g);
361#endif
362
358 err = gk20a_sched_ctrl_init(g); 363 err = gk20a_sched_ctrl_init(g);
359 if (err) { 364 if (err) {
360 nvgpu_err(g, "failed to init sched control"); 365 nvgpu_err(g, "failed to init sched control");
@@ -364,9 +369,14 @@ int gk20a_pm_finalize_poweron(struct device *dev)
364 g->sw_ready = true; 369 g->sw_ready = true;
365 370
366done: 371done:
367 if (err) 372 if (err) {
368 g->power_on = false; 373 g->power_on = false;
369 374
375#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING
376 nvgpu_disable_ecc_reporting(g);
377#endif
378 }
379
370 nvgpu_mutex_release(&g->power_lock); 380 nvgpu_mutex_release(&g->power_lock);
371 return err; 381 return err;
372} 382}
@@ -433,6 +443,10 @@ static int gk20a_pm_prepare_poweroff(struct device *dev)
433 /* Stop CPU from accessing the GPU registers. */ 443 /* Stop CPU from accessing the GPU registers. */
434 gk20a_lockout_registers(g); 444 gk20a_lockout_registers(g);
435 445
446#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING
447 nvgpu_disable_ecc_reporting(g);
448#endif
449
436 nvgpu_hide_usermode_for_poweroff(g); 450 nvgpu_hide_usermode_for_poweroff(g);
437 nvgpu_mutex_release(&g->power_lock); 451 nvgpu_mutex_release(&g->power_lock);
438 return 0; 452 return 0;
@@ -1382,6 +1396,10 @@ static int gk20a_probe(struct platform_device *dev)
1382 goto return_err; 1396 goto return_err;
1383 } 1397 }
1384 1398
1399#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING
1400 nvgpu_init_ecc_reporting(gk20a);
1401#endif
1402
1385 gk20a->nvgpu_reboot_nb.notifier_call = 1403 gk20a->nvgpu_reboot_nb.notifier_call =
1386 nvgpu_kernel_shutdown_notification; 1404 nvgpu_kernel_shutdown_notification;
1387 err = register_reboot_notifier(&gk20a->nvgpu_reboot_nb); 1405 err = register_reboot_notifier(&gk20a->nvgpu_reboot_nb);
diff --git a/include/os/linux/os_linux.h b/include/os/linux/os_linux.h
index 25c6c03..adcfdb2 100644
--- a/include/os/linux/os_linux.h
+++ b/include/os/linux/os_linux.h
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. 2 * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify it 4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License, 5 * under the terms and conditions of the GNU General Public License,
@@ -25,6 +25,7 @@
25 25
26#include "cde.h" 26#include "cde.h"
27#include "sched.h" 27#include "sched.h"
28#include "ecc_linux.h"
28 29
29struct nvgpu_os_linux_ops { 30struct nvgpu_os_linux_ops {
30 struct { 31 struct {
@@ -134,6 +135,10 @@ struct nvgpu_os_linux {
134 135
135 u64 regs_bus_addr; 136 u64 regs_bus_addr;
136 137
138#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING
139 struct nvgpu_ecc_reporting_linux ecc_reporting_linux;
140#endif
141
137 struct nvgpu_os_linux_ops ops; 142 struct nvgpu_os_linux_ops ops;
138 143
139#ifdef CONFIG_DEBUG_FS 144#ifdef CONFIG_DEBUG_FS
diff --git a/include/os/linux/platform_gv11b_tegra.c b/include/os/linux/platform_gv11b_tegra.c
index 6c9d0f5..7900eaa 100644
--- a/include/os/linux/platform_gv11b_tegra.c
+++ b/include/os/linux/platform_gv11b_tegra.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * GV11B Tegra Platform Interface 2 * GV11B Tegra Platform Interface
3 * 3 *
4 * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. 4 * Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved.
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify it 6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License, 7 * under the terms and conditions of the GNU General Public License,
@@ -322,7 +322,7 @@ struct gk20a_platform gv11b_tegra_platform = {
322 322
323 .honors_aperture = true, 323 .honors_aperture = true,
324 .unified_memory = true, 324 .unified_memory = true,
325 .dma_mask = DMA_BIT_MASK(36), 325 .dma_mask = DMA_BIT_MASK(38),
326 326
327 .reset_assert = gp10b_tegra_reset_assert, 327 .reset_assert = gp10b_tegra_reset_assert,
328 .reset_deassert = gp10b_tegra_reset_deassert, 328 .reset_deassert = gp10b_tegra_reset_deassert,
diff --git a/include/os/linux/scale.c b/include/os/linux/scale.c
index 388e168..f8f0ef9 100644
--- a/include/os/linux/scale.c
+++ b/include/os/linux/scale.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * gk20a clock scaling profile 2 * gk20a clock scaling profile
3 * 3 *
4 * Copyright (c) 2013-2020, NVIDIA Corporation. All rights reserved. 4 * Copyright (c) 2013-2023, NVIDIA Corporation. All rights reserved.
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify it 6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License, 7 * under the terms and conditions of the GNU General Public License,
@@ -148,32 +148,24 @@ static int gk20a_scale_target(struct device *dev, unsigned long *freq,
148{ 148{
149 struct gk20a_platform *platform = dev_get_drvdata(dev); 149 struct gk20a_platform *platform = dev_get_drvdata(dev);
150 struct gk20a *g = platform->g; 150 struct gk20a *g = platform->g;
151 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
152 struct gk20a_scale_profile *profile = g->scale_profile; 151 struct gk20a_scale_profile *profile = g->scale_profile;
153 struct devfreq *devfreq = l->devfreq;
154 unsigned long local_freq = *freq; 152 unsigned long local_freq = *freq;
155 unsigned long rounded_rate; 153 unsigned long rounded_rate;
154#ifdef CONFIG_GK20A_PM_QOS
156 unsigned long min_freq = 0, max_freq = 0; 155 unsigned long min_freq = 0, max_freq = 0;
156#endif
157 157
158 if (nvgpu_clk_arb_has_active_req(g)) 158 if (nvgpu_clk_arb_has_active_req(g))
159 return 0; 159 return 0;
160
161#ifdef CONFIG_GK20A_PM_QOS
160 /* 162 /*
161 * Calculate floor and cap frequency values 163 * devfreq takes care of min/max freq clipping in update_devfreq() then
162 * 164 * invoked devfreq->profile->target(), thus we only need to do freq
163 * Policy : 165 * clipping based on pm_qos constraint
164 * We have two APIs to clip the frequency
165 * 1. devfreq
166 * 2. pm_qos
167 *
168 * To calculate floor (min) freq, we select MAX of floor frequencies
169 * requested from both APIs
170 * To get cap (max) freq, we select MIN of max frequencies
171 *
172 * In case we have conflict (min_freq > max_freq) after above
173 * steps, we ensure that max_freq wins over min_freq
174 */ 166 */
175 min_freq = max_t(u32, devfreq->min_freq, profile->qos_min_freq); 167 min_freq = profile->qos_min_freq;
176 max_freq = min_t(u32, devfreq->max_freq, profile->qos_max_freq); 168 max_freq = profile->qos_max_freq;
177 169
178 if (min_freq > max_freq) 170 if (min_freq > max_freq)
179 min_freq = max_freq; 171 min_freq = max_freq;
@@ -184,6 +176,7 @@ static int gk20a_scale_target(struct device *dev, unsigned long *freq,
184 176
185 if (local_freq > max_freq) 177 if (local_freq > max_freq)
186 local_freq = max_freq; 178 local_freq = max_freq;
179#endif
187 180
188 /* set the final frequency */ 181 /* set the final frequency */
189 rounded_rate = platform->clk_round_rate(dev, local_freq); 182 rounded_rate = platform->clk_round_rate(dev, local_freq);
diff --git a/include/os/linux/sdl.c b/include/os/linux/sdl.c
new file mode 100644
index 0000000..c4dccdc
--- /dev/null
+++ b/include/os/linux/sdl.c
@@ -0,0 +1,341 @@
1/*
2 * Copyright (c) 2021, NVIDIA Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#include <nvgpu/gk20a.h>
18#include <nvgpu/types.h>
19#include <nvgpu/nvgpu_err.h>
20#include <nvgpu/timers.h>
21#include <nvgpu/bug.h>
22
23#include "ecc_linux.h"
24#include "os_linux.h"
25#include "module.h"
26
27/* This look-up table initializes the list of hw units and their errors.
28 * It also specifies the error injection mechanism supported, for each error.
29 * In case of hw error injection support, this initialization will be overriden
30 * by the values provided from the hal layes of corresponding hw units.
31 */
32static struct nvgpu_err_hw_module gv11b_err_lut[] = {
33 {
34 .name = "sm",
35 .hw_unit = (u32)NVGPU_ERR_MODULE_SM,
36 .num_errs = 21U,
37 .base_ecc_service_id =
38 NVGUARD_SERVICE_IGPU_SM_SWERR_L1_TAG_ECC_CORRECTED,
39 .errs = (struct nvgpu_err_desc[]) {
40 GPU_NONCRITERR("l1_tag_ecc_corrected",
41 GPU_SM_L1_TAG_ECC_CORRECTED, 0, 0),
42 GPU_CRITERR("l1_tag_ecc_uncorrected",
43 GPU_SM_L1_TAG_ECC_UNCORRECTED, 0, 0),
44 GPU_NONCRITERR("cbu_ecc_corrected", 0, 0, 0),
45 GPU_CRITERR("cbu_ecc_uncorrected",
46 GPU_SM_CBU_ECC_UNCORRECTED, 0, 0),
47 GPU_NONCRITERR("lrf_ecc_corrected", 0, 0, 0),
48 GPU_CRITERR("lrf_ecc_uncorrected",
49 GPU_SM_LRF_ECC_UNCORRECTED, 0, 0),
50 GPU_NONCRITERR("l1_data_ecc_corrected", 0, 0, 0),
51 GPU_CRITERR("l1_data_ecc_uncorrected",
52 GPU_SM_L1_DATA_ECC_UNCORRECTED, 0, 0),
53 GPU_NONCRITERR("icache_l0_data_ecc_corrected", 0, 0, 0),
54 GPU_CRITERR("icache_l0_data_ecc_uncorrected",
55 GPU_SM_ICACHE_L0_DATA_ECC_UNCORRECTED, 0, 0),
56 GPU_NONCRITERR("icache_l1_data_ecc_corrected", 0, 0, 0),
57 GPU_CRITERR("icache_l1_data_ecc_uncorrected",
58 GPU_SM_ICACHE_L1_DATA_ECC_UNCORRECTED, 0, 0),
59 GPU_NONCRITERR("icache_l0_predecode_ecc_corrected", 0, 0, 0),
60 GPU_CRITERR("icache_l0_predecode_ecc_uncorrected",
61 GPU_SM_ICACHE_L0_PREDECODE_ECC_UNCORRECTED, 0, 0),
62 GPU_NONCRITERR("l1_tag_miss_fifo_ecc_corrected", 0, 0, 0),
63 GPU_CRITERR("l1_tag_miss_fifo_ecc_uncorrected",
64 GPU_SM_L1_TAG_MISS_FIFO_ECC_UNCORRECTED, 0, 0),
65 GPU_NONCRITERR("l1_tag_s2r_pixprf_ecc_corrected", 0, 0, 0),
66 GPU_CRITERR("l1_tag_s2r_pixprf_ecc_uncorrected",
67 GPU_SM_L1_TAG_S2R_PIXPRF_ECC_UNCORRECTED, 0, 0),
68 GPU_CRITERR("machine_check_error", 0, 0, 0),
69 GPU_NONCRITERR("icache_l1_predecode_ecc_corrected", 0, 0, 0),
70 GPU_CRITERR("icache_l1_predecode_ecc_uncorrected",
71 GPU_SM_ICACHE_L1_PREDECODE_ECC_UNCORRECTED, 0, 0),
72 },
73 },
74 {
75 .name = "fecs",
76 .hw_unit = (u32)NVGPU_ERR_MODULE_FECS,
77 .num_errs = 4U,
78 .base_ecc_service_id =
79 NVGUARD_SERVICE_IGPU_FECS_SWERR_FALCON_IMEM_ECC_CORRECTED,
80 .errs = (struct nvgpu_err_desc[]) {
81 GPU_NONCRITERR("falcon_imem_ecc_corrected",
82 GPU_FECS_FALCON_IMEM_ECC_CORRECTED, 0, 0),
83 GPU_CRITERR("falcon_imem_ecc_uncorrected",
84 GPU_FECS_FALCON_IMEM_ECC_UNCORRECTED, 0, 0),
85 GPU_NONCRITERR("falcon_dmem_ecc_corrected", 0, 0, 0),
86 GPU_CRITERR("falcon_dmem_ecc_uncorrected",
87 GPU_FECS_FALCON_DMEM_ECC_UNCORRECTED, 0, 0),
88 },
89 },
90 {
91 .name = "pmu",
92 .hw_unit = NVGPU_ERR_MODULE_PMU,
93 .num_errs = 4U,
94 .base_ecc_service_id =
95 NVGUARD_SERVICE_IGPU_PMU_SWERR_FALCON_IMEM_ECC_CORRECTED,
96 .errs = (struct nvgpu_err_desc[]) {
97 GPU_NONCRITERR("falcon_imem_ecc_corrected",
98 GPU_PMU_FALCON_IMEM_ECC_CORRECTED, 0, 0),
99 GPU_CRITERR("falcon_imem_ecc_uncorrected",
100 GPU_PMU_FALCON_IMEM_ECC_UNCORRECTED, 0, 0),
101 GPU_NONCRITERR("falcon_dmem_ecc_corrected", 0, 0, 0),
102 GPU_CRITERR("falcon_dmem_ecc_uncorrected",
103 GPU_PMU_FALCON_DMEM_ECC_UNCORRECTED, 0, 0),
104 },
105 },
106};
107
108static void nvgpu_init_err_msg_header(struct gpu_err_header *header)
109{
110 header->version.major = (u16)1U;
111 header->version.minor = (u16)0U;
112 header->sub_err_type = 0U;
113 header->sub_unit_id = 0UL;
114 header->address = 0UL;
115 header->timestamp_ns = 0UL;
116}
117
118static void nvgpu_init_ecc_err_msg(struct gpu_ecc_error_info *err_info)
119{
120 nvgpu_init_err_msg_header(&err_info->header);
121 err_info->err_cnt = 0UL;
122}
123
124static void nvgpu_report_ecc_error_linux(struct gk20a *g, u32 hw_unit, u32 inst,
125 u32 err_id, u64 err_addr, u64 err_count)
126{
127 int err = 0;
128 u32 s_id = 0;
129 u8 err_status = 0;
130 u8 err_info_size = 0;
131 u64 timestamp = 0ULL;
132 int err_threshold_counter = 0;
133 struct gpu_ecc_error_info err_pkt;
134 struct nvgpu_err_desc *err_desc = NULL;
135 struct nvgpu_err_hw_module *hw_module = NULL;
136 nv_guard_request_t req;
137
138 memset(&req, 0, sizeof(req));
139 nvgpu_init_ecc_err_msg(&err_pkt);
140 if (hw_unit >= sizeof(gv11b_err_lut)/sizeof(gv11b_err_lut[0])) {
141 err = -EINVAL;
142 goto done;
143 }
144
145 hw_module = &gv11b_err_lut[hw_unit];
146 if (err_id >= hw_module->num_errs) {
147 nvgpu_err(g, "invalid err_id (%u) for hw module (%u)",
148 err_id, hw_module->hw_unit);
149 err = -EINVAL;
150 goto done;
151 }
152 err_desc = &hw_module->errs[err_id];
153 timestamp = (u64)nvgpu_current_time_ns();
154
155 err_pkt.header.timestamp_ns = timestamp;
156 err_pkt.header.sub_unit_id = inst;
157 err_pkt.header.address = err_addr;
158 err_pkt.err_cnt = err_count;
159 err_info_size = sizeof(err_pkt);
160
161 s_id = hw_module->base_ecc_service_id + err_id;
162
163 if (err_desc->is_critical) {
164 err_status = NVGUARD_ERROR_DETECTED;
165 } else {
166 err_status = NVGUARD_NO_ERROR;
167 }
168
169 nvgpu_atomic_inc(&err_desc->err_count);
170 err_threshold_counter = nvgpu_atomic_cmpxchg(&err_desc->err_count,
171 err_desc->err_threshold + 1, 0);
172
173 if (unlikely(err_threshold_counter != err_desc->err_threshold + 1)) {
174 goto done;
175 }
176
177 nvgpu_log(g, gpu_dbg_ecc, "ECC reporting hw: %s, desc:%s, count:%llu",
178 hw_module->name, err_desc->name, err_count);
179
180 req.srv_id_cmd = NVGUARD_SERVICESTATUS_NOTIFICATION;
181 req.srv_status.srv_id = (nv_guard_service_id_t)s_id;
182 req.srv_status.status = err_status;
183 req.srv_status.timestamp = timestamp;
184 req.srv_status.error_info_size = err_info_size;
185 memcpy(req.srv_status.error_info, (u8*)&err_pkt, err_info_size);
186
187 /*
188 * l1ss_submit_rq may fail due to kmalloc failures but may pass in
189 * subsequent calls
190 */
191 err = l1ss_submit_rq(&req, true);
192 if (err != 0) {
193 nvgpu_err(g, "Error returned from L1SS submit %d", err);
194 }
195
196 if (err_desc->is_critical) {
197 nvgpu_quiesce(g);
198 }
199
200done:
201 return;
202}
203
204static void nvgpu_report_ecc_error_empty(struct gk20a *g, u32 hw_unit, u32 inst,
205 u32 err_id, u64 err_addr, u64 err_count) {
206 nvgpu_log(g, gpu_dbg_ecc, "ECC reporting empty");
207}
208
209const struct nvgpu_ecc_reporting_ops default_disabled_ecc_report_ops = {
210 .report_ecc_err = nvgpu_report_ecc_error_empty,
211};
212
213const struct nvgpu_ecc_reporting_ops ecc_enable_report_ops = {
214 .report_ecc_err = nvgpu_report_ecc_error_linux,
215};
216
217static int nvgpu_l1ss_callback(l1ss_cli_callback_param param, void *data)
218{
219 struct gk20a *g = (struct gk20a *)data;
220 struct nvgpu_os_linux *l = NULL;
221 struct nvgpu_ecc_reporting_linux *ecc_reporting_linux = NULL;
222 int err = 0;
223 /* Ensure we have a valid gk20a struct before proceeding */
224 if ((g == NULL) || (gk20a_get(g) == NULL)) {
225 return -ENODEV;
226 }
227
228 l = nvgpu_os_linux_from_gk20a(g);
229 ecc_reporting_linux = &l->ecc_reporting_linux;
230
231 nvgpu_spinlock_acquire(&ecc_reporting_linux->common.lock);
232 if (param == L1SS_READY) {
233 if (!ecc_reporting_linux->common.ecc_reporting_service_enabled) {
234 ecc_reporting_linux->common.ecc_reporting_service_enabled = true;
235 ecc_reporting_linux->common.ops = &ecc_enable_report_ops;
236 nvgpu_log(g, gpu_dbg_ecc, "ECC reporting is enabled");
237 }
238 } else if (param == L1SS_NOT_READY) {
239 if (ecc_reporting_linux->common.ecc_reporting_service_enabled) {
240 ecc_reporting_linux->common.ecc_reporting_service_enabled = false;
241 ecc_reporting_linux->common.ops = &default_disabled_ecc_report_ops;
242 nvgpu_log(g, gpu_dbg_ecc, "ECC reporting is disabled");
243 }
244 } else {
245 err = -EINVAL;
246 }
247 nvgpu_spinlock_release(&ecc_reporting_linux->common.lock);
248
249 gk20a_put(g);
250
251 return err;
252}
253
254void nvgpu_init_ecc_reporting(struct gk20a *g)
255{
256 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
257 struct nvgpu_ecc_reporting_linux *ecc_report_linux = &l->ecc_reporting_linux;
258 int err = 0;
259 /* This will invoke the registration API */
260 nvgpu_spinlock_init(&ecc_report_linux->common.lock);
261 ecc_report_linux->priv.id = (NVGUARD_GROUPID_IGPU & NVGUARD_GROUPINDEX_FIELDMASK);
262 ecc_report_linux->priv.cli_callback = nvgpu_l1ss_callback;
263 ecc_report_linux->priv.data = g;
264 ecc_report_linux->common.ops = &default_disabled_ecc_report_ops;
265
266 nvgpu_log(g, gpu_dbg_ecc, "ECC reporting Init");
267
268 /*
269 * err == 0 indicates service is available but not active yet.
270 * err == 1 indicates service is available and active
271 * error for other cases.
272 */
273 err = l1ss_register_client(&ecc_report_linux->priv);
274 if (err == 0) {
275 ecc_report_linux->common.ecc_reporting_service_enabled = false;
276 nvgpu_log(g, gpu_dbg_ecc, "ECC reporting init success");
277 } else if (err == 1) {
278 ecc_report_linux->common.ecc_reporting_service_enabled = true;
279 /* Actual Ops will be replaced during nvgpu_enable_ecc_reporting
280 * called as part of gk20a_busy()
281 */
282 } else {
283 nvgpu_log(g, gpu_dbg_ecc, "ECC reporting init failure %d", err);
284 }
285}
286
287void nvgpu_deinit_ecc_reporting(struct gk20a *g)
288{
289 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
290 struct nvgpu_ecc_reporting_linux *ecc_report_linux = &l->ecc_reporting_linux;
291
292 if (ecc_report_linux->common.ecc_reporting_service_enabled) {
293 ecc_report_linux->common.ecc_reporting_service_enabled = false;
294 l1ss_deregister_client(ecc_report_linux->priv.id);
295 memset(ecc_report_linux, 0, sizeof(*ecc_report_linux));
296 nvgpu_log(g, gpu_dbg_ecc, "ECC reporting de-init success");
297 }
298
299}
300
301void nvgpu_enable_ecc_reporting(struct gk20a *g)
302{
303 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
304 struct nvgpu_ecc_reporting_linux *ecc_report_linux = &l->ecc_reporting_linux;
305 struct nvgpu_ecc_reporting *error_reporting = &ecc_report_linux->common;
306
307 nvgpu_spinlock_acquire(&ecc_report_linux->common.lock);
308 if (error_reporting->ecc_reporting_service_enabled) {
309 error_reporting->ops = &ecc_enable_report_ops;
310 nvgpu_log(g, gpu_dbg_ecc, "ECC reporting is enabled");
311 }
312 nvgpu_spinlock_release(&ecc_report_linux->common.lock);
313}
314
315void nvgpu_disable_ecc_reporting(struct gk20a *g)
316{
317 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
318 struct nvgpu_ecc_reporting_linux *ecc_report_linux = &l->ecc_reporting_linux;
319 struct nvgpu_ecc_reporting *error_reporting = &ecc_report_linux->common;
320
321 nvgpu_spinlock_acquire(&ecc_report_linux->common.lock);
322 error_reporting->ops = &default_disabled_ecc_report_ops;
323 nvgpu_log(g, gpu_dbg_ecc, "ECC reporting is disabled");
324 nvgpu_spinlock_release(&ecc_report_linux->common.lock);
325}
326
327void nvgpu_report_ecc_err(struct gk20a *g, u32 hw_unit, u32 inst,
328 u32 err_id, u64 err_addr, u64 err_count)
329{
330 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
331 struct nvgpu_ecc_reporting_linux *ecc_report_linux = &l->ecc_reporting_linux;
332 struct nvgpu_ecc_reporting *error_reporting = &ecc_report_linux->common;
333 void (*report_ecc_err_func)(struct gk20a *g, u32 hw_unit, u32 inst,
334 u32 err_id, u64 err_addr, u64 err_count);
335
336 nvgpu_spinlock_acquire(&ecc_report_linux->common.lock);
337 report_ecc_err_func = error_reporting->ops->report_ecc_err;
338 nvgpu_spinlock_release(&ecc_report_linux->common.lock);
339
340 report_ecc_err_func(g, hw_unit, inst, err_id, err_addr, err_count);
341}
diff --git a/include/os/linux/vm.c b/include/os/linux/vm.c
index dc807ab..8956cce 100644
--- a/include/os/linux/vm.c
+++ b/include/os/linux/vm.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. 2 * Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify it 4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License, 5 * under the terms and conditions of the GNU General Public License,
@@ -50,8 +50,10 @@ static u32 nvgpu_vm_translate_linux_flags(struct gk20a *g, u32 flags)
50 core_flags |= NVGPU_VM_MAP_IO_COHERENT; 50 core_flags |= NVGPU_VM_MAP_IO_COHERENT;
51 if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_UNMAPPED_PTE) 51 if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_UNMAPPED_PTE)
52 core_flags |= NVGPU_VM_MAP_UNMAPPED_PTE; 52 core_flags |= NVGPU_VM_MAP_UNMAPPED_PTE;
53 if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_L3_ALLOC) 53 if (!nvgpu_is_enabled(g, NVGPU_DISABLE_L3_SUPPORT)) {
54 core_flags |= NVGPU_VM_MAP_L3_ALLOC; 54 if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_L3_ALLOC)
55 core_flags |= NVGPU_VM_MAP_L3_ALLOC;
56 }
55 if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL) 57 if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL)
56 core_flags |= NVGPU_VM_MAP_DIRECT_KIND_CTRL; 58 core_flags |= NVGPU_VM_MAP_DIRECT_KIND_CTRL;
57 if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_PLATFORM_ATOMIC) 59 if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_PLATFORM_ATOMIC)