From 2c5337a24f7f2d02989dfb733c55d6d8c7e90493 Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Sun, 29 Oct 2023 13:07:40 -0400 Subject: Update includes to L4T r32.7.4 and drop nvgpu/gk20a.h dependency Also add instructions for updating `include/`. These files are now only needed to build on Linux 4.9-based Tegra platforms. --- include/os/linux/debug.c | 4 +- include/os/linux/driver_common.c | 69 ++++++- include/os/linux/ecc_linux.h | 49 +++++ include/os/linux/ioctl_clk_arb.c | 17 +- include/os/linux/ioctl_ctrl.c | 122 ++++++++---- include/os/linux/ioctl_tsg.c | 26 +-- include/os/linux/module.c | 22 ++- include/os/linux/os_linux.h | 7 +- include/os/linux/platform_gv11b_tegra.c | 4 +- include/os/linux/scale.c | 29 ++- include/os/linux/sdl.c | 341 ++++++++++++++++++++++++++++++++ include/os/linux/vm.c | 8 +- include/os/posix/nvgpu.c | 18 +- 13 files changed, 618 insertions(+), 98 deletions(-) create mode 100644 include/os/linux/ecc_linux.h create mode 100644 include/os/linux/sdl.c (limited to 'include/os') diff --git a/include/os/linux/debug.c b/include/os/linux/debug.c index 5f0703c..b8c4596 100644 --- a/include/os/linux/debug.c +++ b/include/os/linux/debug.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017-2018 NVIDIA Corporation. All rights reserved. + * Copyright (C) 2017-2021 NVIDIA Corporation. All rights reserved. * * This software is licensed under the terms of the GNU General Public * License version 2, as published by the Free Software Foundation, and @@ -224,7 +224,7 @@ static int railgate_residency_show(struct seq_file *s, void *data) unsigned long total_rail_gate_time_ms; unsigned long total_rail_ungate_time_ms; - if (platform->is_railgated(dev_from_gk20a(g))) { + if (platform && platform->is_railgated && platform->is_railgated(dev_from_gk20a(g))) { time_since_last_state_transition_ms = jiffies_to_msecs(jiffies - g->pstats.last_rail_gate_complete); diff --git a/include/os/linux/driver_common.c b/include/os/linux/driver_common.c index c76dabe..8f5872d 100644 --- a/include/os/linux/driver_common.c +++ b/include/os/linux/driver_common.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -241,6 +242,8 @@ int nvgpu_probe(struct gk20a *g, struct device *dev = dev_from_gk20a(g); struct gk20a_platform *platform = dev_get_drvdata(dev); int err = 0; + struct device_node *np = dev->of_node; + bool disable_l3_alloc = false; nvgpu_init_vars(g); nvgpu_init_gr_vars(g); @@ -265,6 +268,12 @@ int nvgpu_probe(struct gk20a *g, return err; } + disable_l3_alloc = of_property_read_bool(np, "disable_l3_alloc"); + if (disable_l3_alloc) { + nvgpu_log_info(g, "L3 alloc is disabled\n"); + __nvgpu_set_enabled(g, NVGPU_DISABLE_L3_SUPPORT, true); + } + nvgpu_init_mm_vars(g); /* platform probe can defer do user init only if probe succeeds */ @@ -312,30 +321,70 @@ static int cyclic_delta(int a, int b) } /** - * nvgpu_wait_for_deferred_interrupts - Wait for interrupts to complete + * nvgpu_wait_for_stall_interrupts - Wait for the stalling interrupts to + * complete. * * @g - The GPU to wait on. + * @timeout - maximum time period to wait for. * - * Waits until all interrupt handlers that have been scheduled to run have - * completed. + * Waits until all stalling interrupt handlers that have been scheduled to run + * have completed. */ -void nvgpu_wait_for_deferred_interrupts(struct gk20a *g) +int nvgpu_wait_for_stall_interrupts(struct gk20a *g, u32 timeout) { struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); int stall_irq_threshold = atomic_read(&l->hw_irq_stall_count); - int nonstall_irq_threshold = atomic_read(&l->hw_irq_nonstall_count); /* wait until all stalling irqs are handled */ - NVGPU_COND_WAIT(&l->sw_irq_stall_last_handled_wq, + return NVGPU_COND_WAIT(&l->sw_irq_stall_last_handled_wq, cyclic_delta(stall_irq_threshold, atomic_read(&l->sw_irq_stall_last_handled)) - <= 0, 0); + <= 0, timeout); +} + +/** + * nvgpu_wait_for_nonstall_interrupts - Wait for the nonstalling interrupts to + * complete. + * + * @g - The GPU to wait on. + * @timeout - maximum time period to wait for. + * + * Waits until all non-stalling interrupt handlers that have been scheduled to + * run have completed. + */ +int nvgpu_wait_for_nonstall_interrupts(struct gk20a *g, u32 timeout) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + int nonstall_irq_threshold = atomic_read(&l->hw_irq_nonstall_count); /* wait until all non-stalling irqs are handled */ - NVGPU_COND_WAIT(&l->sw_irq_nonstall_last_handled_wq, + return NVGPU_COND_WAIT(&l->sw_irq_nonstall_last_handled_wq, cyclic_delta(nonstall_irq_threshold, atomic_read(&l->sw_irq_nonstall_last_handled)) - <= 0, 0); + <= 0, timeout); +} + +/** + * nvgpu_wait_for_deferred_interrupts - Wait for interrupts to complete + * + * @g - The GPU to wait on. + * + * Waits until all interrupt handlers that have been scheduled to run have + * completed. + */ +void nvgpu_wait_for_deferred_interrupts(struct gk20a *g) +{ + int ret; + + ret = nvgpu_wait_for_stall_interrupts(g, 0U); + if (ret != 0) { + nvgpu_err(g, "wait for stall interrupts failed %d", ret); + } + + ret = nvgpu_wait_for_nonstall_interrupts(g, 0U); + if (ret != 0) { + nvgpu_err(g, "wait for nonstall interrupts failed %d", ret); + } } static void nvgpu_free_gk20a(struct gk20a *g) diff --git a/include/os/linux/ecc_linux.h b/include/os/linux/ecc_linux.h new file mode 100644 index 0000000..7e0f650 --- /dev/null +++ b/include/os/linux/ecc_linux.h @@ -0,0 +1,49 @@ +/* + * + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVGPU_OS_ECC_LINUX_H +#define NVGPU_OS_ECC_LINUX_H + +#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING + +#include +#include +#include +#include + +#include + +struct nvgpu_ecc_reporting_linux { + struct nvgpu_ecc_reporting common; + client_param_t priv; +}; + +static inline struct nvgpu_ecc_reporting_linux *get_ecc_reporting_linux( + struct nvgpu_ecc_reporting *ecc_report) +{ + return container_of(ecc_report, struct nvgpu_ecc_reporting_linux, common); +} + +#endif /* CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING */ + +#endif \ No newline at end of file diff --git a/include/os/linux/ioctl_clk_arb.c b/include/os/linux/ioctl_clk_arb.c index 477222d..9f32102 100644 --- a/include/os/linux/ioctl_clk_arb.c +++ b/include/os/linux/ioctl_clk_arb.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2016-2021, NVIDIA CORPORATION. All rights reserved. * * This software is licensed under the terms of the GNU General Public * License version 2, as published by the Free Software Foundation, and @@ -51,19 +51,28 @@ static int nvgpu_clk_arb_release_completion_dev(struct inode *inode, { struct nvgpu_clk_dev *dev = filp->private_data; struct nvgpu_clk_session *session = dev->session; + struct gk20a *g = session->g; + struct nvgpu_clk_arb *arb = g->clk_arb; + clk_arb_dbg(g, " "); - clk_arb_dbg(session->g, " "); + nvgpu_spinlock_acquire(&session->session_lock); + nvgpu_spinlock_acquire(&arb->requests_lock); + + nvgpu_list_del(&dev->node); + + nvgpu_spinlock_release(&arb->requests_lock); + nvgpu_spinlock_release(&session->session_lock); /* This is done to account for the extra refcount taken in * nvgpu_clk_arb_commit_request_fd without events support in iGPU */ - if (!session->g->clk_arb->clk_arb_events_supported) { + if (!arb->clk_arb_events_supported) { nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd); } - nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session); nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd); + nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session); return 0; } diff --git a/include/os/linux/ioctl_ctrl.c b/include/os/linux/ioctl_ctrl.c index ee141ff..841d345 100644 --- a/include/os/linux/ioctl_ctrl.c +++ b/include/os/linux/ioctl_ctrl.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2020, NVIDIA Corporation. All rights reserved. + * Copyright (c) 2011-2021, NVIDIA Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -60,7 +60,6 @@ struct gk20a_ctrl_priv { struct nvgpu_list_node list; struct { struct vm_area_struct *vma; - unsigned long flags; bool vma_mapped; } usermode_vma; }; @@ -488,27 +487,26 @@ static int gk20a_ctrl_alloc_as( snprintf(name, sizeof(name), "nvhost-%s-fd%d", g->name, fd); - file = anon_inode_getfile(name, l->as_dev.cdev.ops, NULL, O_RDWR); - if (IS_ERR(file)) { - err = PTR_ERR(file); - goto clean_up; - } - err = gk20a_as_alloc_share(g, args->big_page_size, gk20a_as_translate_as_alloc_flags(g, args->flags), &as_share); if (err) - goto clean_up_file; + goto clean_up; + + file = anon_inode_getfile(name, l->as_dev.cdev.ops, as_share, O_RDWR); + if (IS_ERR(file)) { + err = PTR_ERR(file); + goto clean_up_as; + } fd_install(fd, file); - file->private_data = as_share; args->as_fd = fd; return 0; -clean_up_file: - fput(file); +clean_up_as: + gk20a_as_release_share(as_share); clean_up: put_unused_fd(fd); return err; @@ -692,12 +690,15 @@ static int nvgpu_gpu_ioctl_trigger_suspend(struct gk20a *g) err = gk20a_busy(g); if (err) - return err; + return err; - nvgpu_mutex_acquire(&g->dbg_sessions_lock); - err = gr_gk20a_elpg_protected_call(g, + if (g->ops.gr.trigger_suspend) { + nvgpu_mutex_acquire(&g->dbg_sessions_lock); + err = gr_gk20a_elpg_protected_call(g, g->ops.gr.trigger_suspend(g)); - nvgpu_mutex_release(&g->dbg_sessions_lock); + nvgpu_mutex_release(&g->dbg_sessions_lock); + } else + err = -EINVAL; gk20a_idle(g); @@ -731,8 +732,13 @@ static int nvgpu_gpu_ioctl_wait_for_pause(struct gk20a *g, goto out_free; nvgpu_mutex_acquire(&g->dbg_sessions_lock); - (void)gr_gk20a_elpg_protected_call(g, + if (g->ops.gr.wait_for_pause) { + (void)gr_gk20a_elpg_protected_call(g, g->ops.gr.wait_for_pause(g, w_state)); + } else { + err = -EINVAL; + goto out_idle; + } for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) { ioctl_w_state[sm_id].valid_warps[0] = @@ -755,6 +761,7 @@ static int nvgpu_gpu_ioctl_wait_for_pause(struct gk20a *g, err = -EFAULT; } +out_idle: nvgpu_mutex_release(&g->dbg_sessions_lock); gk20a_idle(g); @@ -772,12 +779,15 @@ static int nvgpu_gpu_ioctl_resume_from_pause(struct gk20a *g) err = gk20a_busy(g); if (err) - return err; + return err; - nvgpu_mutex_acquire(&g->dbg_sessions_lock); - err = gr_gk20a_elpg_protected_call(g, + if (g->ops.gr.resume_from_pause) { + nvgpu_mutex_acquire(&g->dbg_sessions_lock); + err = gr_gk20a_elpg_protected_call(g, g->ops.gr.resume_from_pause(g)); - nvgpu_mutex_release(&g->dbg_sessions_lock); + nvgpu_mutex_release(&g->dbg_sessions_lock); + } else + err = -EINVAL; gk20a_idle(g); @@ -792,8 +802,11 @@ static int nvgpu_gpu_ioctl_clear_sm_errors(struct gk20a *g) if (err) return err; - err = gr_gk20a_elpg_protected_call(g, + if (g->ops.gr.clear_sm_errors) { + err = gr_gk20a_elpg_protected_call(g, g->ops.gr.clear_sm_errors(g)); + } else + err = -EINVAL; gk20a_idle(g); @@ -806,9 +819,12 @@ static int nvgpu_gpu_ioctl_has_any_exception( { u32 tpc_exception_en; - nvgpu_mutex_acquire(&g->dbg_sessions_lock); - tpc_exception_en = g->ops.gr.tpc_enabled_exceptions(g); - nvgpu_mutex_release(&g->dbg_sessions_lock); + if (g->ops.gr.tpc_enabled_exceptions) { + nvgpu_mutex_acquire(&g->dbg_sessions_lock); + tpc_exception_en = g->ops.gr.tpc_enabled_exceptions(g); + nvgpu_mutex_release(&g->dbg_sessions_lock); + } else + return -EINVAL; args->tpc_exception_en_sm_mask = tpc_exception_en; @@ -2023,7 +2039,6 @@ int gk20a_ctrl_dev_mmap(struct file *filp, struct vm_area_struct *vma) vma->vm_end - vma->vm_start, vma->vm_page_prot); if (!err) { priv->usermode_vma.vma = vma; - priv->usermode_vma.flags = vma->vm_flags; vma->vm_private_data = priv; priv->usermode_vma.vma_mapped = true; } @@ -2034,7 +2049,7 @@ int gk20a_ctrl_dev_mmap(struct file *filp, struct vm_area_struct *vma) return err; } -static void alter_usermode_mapping(struct gk20a *g, +static int alter_usermode_mapping(struct gk20a *g, struct gk20a_ctrl_priv *priv, bool poweroff) { @@ -2042,57 +2057,80 @@ static void alter_usermode_mapping(struct gk20a *g, struct vm_area_struct *vma = priv->usermode_vma.vma; bool vma_mapped = priv->usermode_vma.vma_mapped; u64 addr; - int err; + int err = 0; if (!vma) { /* Nothing to do - no mmap called */ - return; + return 0; } addr = l->regs_bus_addr + g->ops.fifo.usermode_base(g); - down_write(&vma->vm_mm->mmap_sem); - /* * This is a no-op for the below cases * a) poweroff and !vma_mapped - > do nothing as no map exists * b) !poweroff and vmap_mapped -> do nothing as already mapped */ - if (poweroff && vma_mapped) { + if (poweroff != vma_mapped) { + return 0; + } + + /* + * We use trylock due to lock inversion: we need to acquire + * mmap_lock while holding ctrl_privs_lock. usermode_vma_close + * does it in reverse order. Trylock is a way to avoid deadlock. + */ + if (!down_write_trylock(&vma->vm_mm->mmap_sem)) { + return -EBUSY; + } + + if (poweroff) { err = zap_vma_ptes(vma, vma->vm_start, SZ_4K); if (err == 0) { - vma->vm_flags = VM_NONE; priv->usermode_vma.vma_mapped = false; } else { nvgpu_err(g, "can't remove usermode mapping"); } - } else if (!poweroff && !vma_mapped) { - vma->vm_flags = priv->usermode_vma.flags; + } else { err = io_remap_pfn_range(vma, vma->vm_start, addr >> PAGE_SHIFT, SZ_4K, vma->vm_page_prot); if (err != 0) { nvgpu_err(g, "can't restore usermode mapping"); - vma->vm_flags = VM_NONE; } else { priv->usermode_vma.vma_mapped = true; } } up_write(&vma->vm_mm->mmap_sem); + + return err; } static void alter_usermode_mappings(struct gk20a *g, bool poweroff) { struct gk20a_ctrl_priv *priv; struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + int err = 0; - nvgpu_mutex_acquire(&l->ctrl.privs_lock); - nvgpu_list_for_each_entry(priv, &l->ctrl.privs, - gk20a_ctrl_priv, list) { - alter_usermode_mapping(g, priv, poweroff); - } - nvgpu_mutex_release(&l->ctrl.privs_lock); + do { + nvgpu_mutex_acquire(&l->ctrl.privs_lock); + nvgpu_list_for_each_entry(priv, &l->ctrl.privs, + gk20a_ctrl_priv, list) { + err = alter_usermode_mapping(g, priv, poweroff); + if (err != 0) { + break; + } + } + nvgpu_mutex_release(&l->ctrl.privs_lock); + + if (err == -EBUSY) { + nvgpu_log_info(g, "ctrl_privs_lock lock contended. retry altering usermode mappings"); + nvgpu_udelay(10); + } else if (err != 0) { + nvgpu_err(g, "can't alter usermode mapping. err = %d", err); + } + } while (err == -EBUSY); } void nvgpu_hide_usermode_for_poweroff(struct gk20a *g) diff --git a/include/os/linux/ioctl_tsg.c b/include/os/linux/ioctl_tsg.c index 2f8cb3a..296b02b 100644 --- a/include/os/linux/ioctl_tsg.c +++ b/include/os/linux/ioctl_tsg.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2014-2021, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -134,7 +134,10 @@ static int gk20a_tsg_unbind_channel_fd(struct tsg_gk20a *tsg, int ch_fd) goto out; } - err = gk20a_tsg_unbind_channel(ch); + err = gk20a_tsg_unbind_channel(ch, false); + if (err == -EAGAIN) { + goto out; + } /* * Mark the channel timedout since channel unbound from TSG @@ -307,17 +310,10 @@ static int gk20a_tsg_event_id_enable(struct tsg_gk20a *tsg, snprintf(name, sizeof(name), "nvgpu-event%d-fd%d", event_id, local_fd); - file = anon_inode_getfile(name, &gk20a_event_id_ops, - NULL, O_RDWR); - if (IS_ERR(file)) { - err = PTR_ERR(file); - goto clean_up; - } - event_id_data = nvgpu_kzalloc(tsg->g, sizeof(*event_id_data)); if (!event_id_data) { err = -ENOMEM; - goto clean_up_file; + goto clean_up; } event_id_data->g = g; event_id_data->id = tsg->tsgid; @@ -330,12 +326,18 @@ static int gk20a_tsg_event_id_enable(struct tsg_gk20a *tsg, nvgpu_init_list_node(&event_id_data->event_id_node); + file = anon_inode_getfile(name, &gk20a_event_id_ops, + event_id_data, O_RDWR); + if (IS_ERR(file)) { + err = PTR_ERR(file); + goto clean_up_free; + } + nvgpu_mutex_acquire(&tsg->event_id_list_lock); nvgpu_list_add_tail(&event_id_data->event_id_node, &tsg->event_id_list); nvgpu_mutex_release(&tsg->event_id_list_lock); fd_install(local_fd, file); - file->private_data = event_id_data; *fd = local_fd; @@ -343,8 +345,6 @@ static int gk20a_tsg_event_id_enable(struct tsg_gk20a *tsg, clean_up_free: nvgpu_kfree(g, event_id_data); -clean_up_file: - fput(file); clean_up: put_unused_fd(local_fd); free_ref: diff --git a/include/os/linux/module.c b/include/os/linux/module.c index 807df2c..fdbab46 100644 --- a/include/os/linux/module.c +++ b/include/os/linux/module.c @@ -1,7 +1,7 @@ /* * GK20A Graphics * - * Copyright (c) 2011-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -49,6 +49,7 @@ #include #include #include +#include #include "platform_gk20a.h" #include "sysfs.h" @@ -355,6 +356,10 @@ int gk20a_pm_finalize_poweron(struct device *dev) gk20a_init_cde_support(l); #endif +#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING + nvgpu_enable_ecc_reporting(g); +#endif + err = gk20a_sched_ctrl_init(g); if (err) { nvgpu_err(g, "failed to init sched control"); @@ -364,9 +369,14 @@ int gk20a_pm_finalize_poweron(struct device *dev) g->sw_ready = true; done: - if (err) + if (err) { g->power_on = false; +#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING + nvgpu_disable_ecc_reporting(g); +#endif + } + nvgpu_mutex_release(&g->power_lock); return err; } @@ -433,6 +443,10 @@ static int gk20a_pm_prepare_poweroff(struct device *dev) /* Stop CPU from accessing the GPU registers. */ gk20a_lockout_registers(g); +#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING + nvgpu_disable_ecc_reporting(g); +#endif + nvgpu_hide_usermode_for_poweroff(g); nvgpu_mutex_release(&g->power_lock); return 0; @@ -1382,6 +1396,10 @@ static int gk20a_probe(struct platform_device *dev) goto return_err; } +#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING + nvgpu_init_ecc_reporting(gk20a); +#endif + gk20a->nvgpu_reboot_nb.notifier_call = nvgpu_kernel_shutdown_notification; err = register_reboot_notifier(&gk20a->nvgpu_reboot_nb); diff --git a/include/os/linux/os_linux.h b/include/os/linux/os_linux.h index 25c6c03..adcfdb2 100644 --- a/include/os/linux/os_linux.h +++ b/include/os/linux/os_linux.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -25,6 +25,7 @@ #include "cde.h" #include "sched.h" +#include "ecc_linux.h" struct nvgpu_os_linux_ops { struct { @@ -134,6 +135,10 @@ struct nvgpu_os_linux { u64 regs_bus_addr; +#ifdef CONFIG_NVGPU_SUPPORT_LINUX_ECC_ERROR_REPORTING + struct nvgpu_ecc_reporting_linux ecc_reporting_linux; +#endif + struct nvgpu_os_linux_ops ops; #ifdef CONFIG_DEBUG_FS diff --git a/include/os/linux/platform_gv11b_tegra.c b/include/os/linux/platform_gv11b_tegra.c index 6c9d0f5..7900eaa 100644 --- a/include/os/linux/platform_gv11b_tegra.c +++ b/include/os/linux/platform_gv11b_tegra.c @@ -1,7 +1,7 @@ /* * GV11B Tegra Platform Interface * - * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -322,7 +322,7 @@ struct gk20a_platform gv11b_tegra_platform = { .honors_aperture = true, .unified_memory = true, - .dma_mask = DMA_BIT_MASK(36), + .dma_mask = DMA_BIT_MASK(38), .reset_assert = gp10b_tegra_reset_assert, .reset_deassert = gp10b_tegra_reset_deassert, diff --git a/include/os/linux/scale.c b/include/os/linux/scale.c index 388e168..f8f0ef9 100644 --- a/include/os/linux/scale.c +++ b/include/os/linux/scale.c @@ -1,7 +1,7 @@ /* * gk20a clock scaling profile * - * Copyright (c) 2013-2020, NVIDIA Corporation. All rights reserved. + * Copyright (c) 2013-2023, NVIDIA Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -148,32 +148,24 @@ static int gk20a_scale_target(struct device *dev, unsigned long *freq, { struct gk20a_platform *platform = dev_get_drvdata(dev); struct gk20a *g = platform->g; - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); struct gk20a_scale_profile *profile = g->scale_profile; - struct devfreq *devfreq = l->devfreq; unsigned long local_freq = *freq; unsigned long rounded_rate; +#ifdef CONFIG_GK20A_PM_QOS unsigned long min_freq = 0, max_freq = 0; +#endif if (nvgpu_clk_arb_has_active_req(g)) return 0; + +#ifdef CONFIG_GK20A_PM_QOS /* - * Calculate floor and cap frequency values - * - * Policy : - * We have two APIs to clip the frequency - * 1. devfreq - * 2. pm_qos - * - * To calculate floor (min) freq, we select MAX of floor frequencies - * requested from both APIs - * To get cap (max) freq, we select MIN of max frequencies - * - * In case we have conflict (min_freq > max_freq) after above - * steps, we ensure that max_freq wins over min_freq + * devfreq takes care of min/max freq clipping in update_devfreq() then + * invoked devfreq->profile->target(), thus we only need to do freq + * clipping based on pm_qos constraint */ - min_freq = max_t(u32, devfreq->min_freq, profile->qos_min_freq); - max_freq = min_t(u32, devfreq->max_freq, profile->qos_max_freq); + min_freq = profile->qos_min_freq; + max_freq = profile->qos_max_freq; if (min_freq > max_freq) min_freq = max_freq; @@ -184,6 +176,7 @@ static int gk20a_scale_target(struct device *dev, unsigned long *freq, if (local_freq > max_freq) local_freq = max_freq; +#endif /* set the final frequency */ rounded_rate = platform->clk_round_rate(dev, local_freq); diff --git a/include/os/linux/sdl.c b/include/os/linux/sdl.c new file mode 100644 index 0000000..c4dccdc --- /dev/null +++ b/include/os/linux/sdl.c @@ -0,0 +1,341 @@ +/* + * Copyright (c) 2021, NVIDIA Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include + +#include "ecc_linux.h" +#include "os_linux.h" +#include "module.h" + +/* This look-up table initializes the list of hw units and their errors. + * It also specifies the error injection mechanism supported, for each error. + * In case of hw error injection support, this initialization will be overriden + * by the values provided from the hal layes of corresponding hw units. + */ +static struct nvgpu_err_hw_module gv11b_err_lut[] = { + { + .name = "sm", + .hw_unit = (u32)NVGPU_ERR_MODULE_SM, + .num_errs = 21U, + .base_ecc_service_id = + NVGUARD_SERVICE_IGPU_SM_SWERR_L1_TAG_ECC_CORRECTED, + .errs = (struct nvgpu_err_desc[]) { + GPU_NONCRITERR("l1_tag_ecc_corrected", + GPU_SM_L1_TAG_ECC_CORRECTED, 0, 0), + GPU_CRITERR("l1_tag_ecc_uncorrected", + GPU_SM_L1_TAG_ECC_UNCORRECTED, 0, 0), + GPU_NONCRITERR("cbu_ecc_corrected", 0, 0, 0), + GPU_CRITERR("cbu_ecc_uncorrected", + GPU_SM_CBU_ECC_UNCORRECTED, 0, 0), + GPU_NONCRITERR("lrf_ecc_corrected", 0, 0, 0), + GPU_CRITERR("lrf_ecc_uncorrected", + GPU_SM_LRF_ECC_UNCORRECTED, 0, 0), + GPU_NONCRITERR("l1_data_ecc_corrected", 0, 0, 0), + GPU_CRITERR("l1_data_ecc_uncorrected", + GPU_SM_L1_DATA_ECC_UNCORRECTED, 0, 0), + GPU_NONCRITERR("icache_l0_data_ecc_corrected", 0, 0, 0), + GPU_CRITERR("icache_l0_data_ecc_uncorrected", + GPU_SM_ICACHE_L0_DATA_ECC_UNCORRECTED, 0, 0), + GPU_NONCRITERR("icache_l1_data_ecc_corrected", 0, 0, 0), + GPU_CRITERR("icache_l1_data_ecc_uncorrected", + GPU_SM_ICACHE_L1_DATA_ECC_UNCORRECTED, 0, 0), + GPU_NONCRITERR("icache_l0_predecode_ecc_corrected", 0, 0, 0), + GPU_CRITERR("icache_l0_predecode_ecc_uncorrected", + GPU_SM_ICACHE_L0_PREDECODE_ECC_UNCORRECTED, 0, 0), + GPU_NONCRITERR("l1_tag_miss_fifo_ecc_corrected", 0, 0, 0), + GPU_CRITERR("l1_tag_miss_fifo_ecc_uncorrected", + GPU_SM_L1_TAG_MISS_FIFO_ECC_UNCORRECTED, 0, 0), + GPU_NONCRITERR("l1_tag_s2r_pixprf_ecc_corrected", 0, 0, 0), + GPU_CRITERR("l1_tag_s2r_pixprf_ecc_uncorrected", + GPU_SM_L1_TAG_S2R_PIXPRF_ECC_UNCORRECTED, 0, 0), + GPU_CRITERR("machine_check_error", 0, 0, 0), + GPU_NONCRITERR("icache_l1_predecode_ecc_corrected", 0, 0, 0), + GPU_CRITERR("icache_l1_predecode_ecc_uncorrected", + GPU_SM_ICACHE_L1_PREDECODE_ECC_UNCORRECTED, 0, 0), + }, + }, + { + .name = "fecs", + .hw_unit = (u32)NVGPU_ERR_MODULE_FECS, + .num_errs = 4U, + .base_ecc_service_id = + NVGUARD_SERVICE_IGPU_FECS_SWERR_FALCON_IMEM_ECC_CORRECTED, + .errs = (struct nvgpu_err_desc[]) { + GPU_NONCRITERR("falcon_imem_ecc_corrected", + GPU_FECS_FALCON_IMEM_ECC_CORRECTED, 0, 0), + GPU_CRITERR("falcon_imem_ecc_uncorrected", + GPU_FECS_FALCON_IMEM_ECC_UNCORRECTED, 0, 0), + GPU_NONCRITERR("falcon_dmem_ecc_corrected", 0, 0, 0), + GPU_CRITERR("falcon_dmem_ecc_uncorrected", + GPU_FECS_FALCON_DMEM_ECC_UNCORRECTED, 0, 0), + }, + }, + { + .name = "pmu", + .hw_unit = NVGPU_ERR_MODULE_PMU, + .num_errs = 4U, + .base_ecc_service_id = + NVGUARD_SERVICE_IGPU_PMU_SWERR_FALCON_IMEM_ECC_CORRECTED, + .errs = (struct nvgpu_err_desc[]) { + GPU_NONCRITERR("falcon_imem_ecc_corrected", + GPU_PMU_FALCON_IMEM_ECC_CORRECTED, 0, 0), + GPU_CRITERR("falcon_imem_ecc_uncorrected", + GPU_PMU_FALCON_IMEM_ECC_UNCORRECTED, 0, 0), + GPU_NONCRITERR("falcon_dmem_ecc_corrected", 0, 0, 0), + GPU_CRITERR("falcon_dmem_ecc_uncorrected", + GPU_PMU_FALCON_DMEM_ECC_UNCORRECTED, 0, 0), + }, + }, +}; + +static void nvgpu_init_err_msg_header(struct gpu_err_header *header) +{ + header->version.major = (u16)1U; + header->version.minor = (u16)0U; + header->sub_err_type = 0U; + header->sub_unit_id = 0UL; + header->address = 0UL; + header->timestamp_ns = 0UL; +} + +static void nvgpu_init_ecc_err_msg(struct gpu_ecc_error_info *err_info) +{ + nvgpu_init_err_msg_header(&err_info->header); + err_info->err_cnt = 0UL; +} + +static void nvgpu_report_ecc_error_linux(struct gk20a *g, u32 hw_unit, u32 inst, + u32 err_id, u64 err_addr, u64 err_count) +{ + int err = 0; + u32 s_id = 0; + u8 err_status = 0; + u8 err_info_size = 0; + u64 timestamp = 0ULL; + int err_threshold_counter = 0; + struct gpu_ecc_error_info err_pkt; + struct nvgpu_err_desc *err_desc = NULL; + struct nvgpu_err_hw_module *hw_module = NULL; + nv_guard_request_t req; + + memset(&req, 0, sizeof(req)); + nvgpu_init_ecc_err_msg(&err_pkt); + if (hw_unit >= sizeof(gv11b_err_lut)/sizeof(gv11b_err_lut[0])) { + err = -EINVAL; + goto done; + } + + hw_module = &gv11b_err_lut[hw_unit]; + if (err_id >= hw_module->num_errs) { + nvgpu_err(g, "invalid err_id (%u) for hw module (%u)", + err_id, hw_module->hw_unit); + err = -EINVAL; + goto done; + } + err_desc = &hw_module->errs[err_id]; + timestamp = (u64)nvgpu_current_time_ns(); + + err_pkt.header.timestamp_ns = timestamp; + err_pkt.header.sub_unit_id = inst; + err_pkt.header.address = err_addr; + err_pkt.err_cnt = err_count; + err_info_size = sizeof(err_pkt); + + s_id = hw_module->base_ecc_service_id + err_id; + + if (err_desc->is_critical) { + err_status = NVGUARD_ERROR_DETECTED; + } else { + err_status = NVGUARD_NO_ERROR; + } + + nvgpu_atomic_inc(&err_desc->err_count); + err_threshold_counter = nvgpu_atomic_cmpxchg(&err_desc->err_count, + err_desc->err_threshold + 1, 0); + + if (unlikely(err_threshold_counter != err_desc->err_threshold + 1)) { + goto done; + } + + nvgpu_log(g, gpu_dbg_ecc, "ECC reporting hw: %s, desc:%s, count:%llu", + hw_module->name, err_desc->name, err_count); + + req.srv_id_cmd = NVGUARD_SERVICESTATUS_NOTIFICATION; + req.srv_status.srv_id = (nv_guard_service_id_t)s_id; + req.srv_status.status = err_status; + req.srv_status.timestamp = timestamp; + req.srv_status.error_info_size = err_info_size; + memcpy(req.srv_status.error_info, (u8*)&err_pkt, err_info_size); + + /* + * l1ss_submit_rq may fail due to kmalloc failures but may pass in + * subsequent calls + */ + err = l1ss_submit_rq(&req, true); + if (err != 0) { + nvgpu_err(g, "Error returned from L1SS submit %d", err); + } + + if (err_desc->is_critical) { + nvgpu_quiesce(g); + } + +done: + return; +} + +static void nvgpu_report_ecc_error_empty(struct gk20a *g, u32 hw_unit, u32 inst, + u32 err_id, u64 err_addr, u64 err_count) { + nvgpu_log(g, gpu_dbg_ecc, "ECC reporting empty"); +} + +const struct nvgpu_ecc_reporting_ops default_disabled_ecc_report_ops = { + .report_ecc_err = nvgpu_report_ecc_error_empty, +}; + +const struct nvgpu_ecc_reporting_ops ecc_enable_report_ops = { + .report_ecc_err = nvgpu_report_ecc_error_linux, +}; + +static int nvgpu_l1ss_callback(l1ss_cli_callback_param param, void *data) +{ + struct gk20a *g = (struct gk20a *)data; + struct nvgpu_os_linux *l = NULL; + struct nvgpu_ecc_reporting_linux *ecc_reporting_linux = NULL; + int err = 0; + /* Ensure we have a valid gk20a struct before proceeding */ + if ((g == NULL) || (gk20a_get(g) == NULL)) { + return -ENODEV; + } + + l = nvgpu_os_linux_from_gk20a(g); + ecc_reporting_linux = &l->ecc_reporting_linux; + + nvgpu_spinlock_acquire(&ecc_reporting_linux->common.lock); + if (param == L1SS_READY) { + if (!ecc_reporting_linux->common.ecc_reporting_service_enabled) { + ecc_reporting_linux->common.ecc_reporting_service_enabled = true; + ecc_reporting_linux->common.ops = &ecc_enable_report_ops; + nvgpu_log(g, gpu_dbg_ecc, "ECC reporting is enabled"); + } + } else if (param == L1SS_NOT_READY) { + if (ecc_reporting_linux->common.ecc_reporting_service_enabled) { + ecc_reporting_linux->common.ecc_reporting_service_enabled = false; + ecc_reporting_linux->common.ops = &default_disabled_ecc_report_ops; + nvgpu_log(g, gpu_dbg_ecc, "ECC reporting is disabled"); + } + } else { + err = -EINVAL; + } + nvgpu_spinlock_release(&ecc_reporting_linux->common.lock); + + gk20a_put(g); + + return err; +} + +void nvgpu_init_ecc_reporting(struct gk20a *g) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct nvgpu_ecc_reporting_linux *ecc_report_linux = &l->ecc_reporting_linux; + int err = 0; + /* This will invoke the registration API */ + nvgpu_spinlock_init(&ecc_report_linux->common.lock); + ecc_report_linux->priv.id = (NVGUARD_GROUPID_IGPU & NVGUARD_GROUPINDEX_FIELDMASK); + ecc_report_linux->priv.cli_callback = nvgpu_l1ss_callback; + ecc_report_linux->priv.data = g; + ecc_report_linux->common.ops = &default_disabled_ecc_report_ops; + + nvgpu_log(g, gpu_dbg_ecc, "ECC reporting Init"); + + /* + * err == 0 indicates service is available but not active yet. + * err == 1 indicates service is available and active + * error for other cases. + */ + err = l1ss_register_client(&ecc_report_linux->priv); + if (err == 0) { + ecc_report_linux->common.ecc_reporting_service_enabled = false; + nvgpu_log(g, gpu_dbg_ecc, "ECC reporting init success"); + } else if (err == 1) { + ecc_report_linux->common.ecc_reporting_service_enabled = true; + /* Actual Ops will be replaced during nvgpu_enable_ecc_reporting + * called as part of gk20a_busy() + */ + } else { + nvgpu_log(g, gpu_dbg_ecc, "ECC reporting init failure %d", err); + } +} + +void nvgpu_deinit_ecc_reporting(struct gk20a *g) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct nvgpu_ecc_reporting_linux *ecc_report_linux = &l->ecc_reporting_linux; + + if (ecc_report_linux->common.ecc_reporting_service_enabled) { + ecc_report_linux->common.ecc_reporting_service_enabled = false; + l1ss_deregister_client(ecc_report_linux->priv.id); + memset(ecc_report_linux, 0, sizeof(*ecc_report_linux)); + nvgpu_log(g, gpu_dbg_ecc, "ECC reporting de-init success"); + } + +} + +void nvgpu_enable_ecc_reporting(struct gk20a *g) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct nvgpu_ecc_reporting_linux *ecc_report_linux = &l->ecc_reporting_linux; + struct nvgpu_ecc_reporting *error_reporting = &ecc_report_linux->common; + + nvgpu_spinlock_acquire(&ecc_report_linux->common.lock); + if (error_reporting->ecc_reporting_service_enabled) { + error_reporting->ops = &ecc_enable_report_ops; + nvgpu_log(g, gpu_dbg_ecc, "ECC reporting is enabled"); + } + nvgpu_spinlock_release(&ecc_report_linux->common.lock); +} + +void nvgpu_disable_ecc_reporting(struct gk20a *g) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct nvgpu_ecc_reporting_linux *ecc_report_linux = &l->ecc_reporting_linux; + struct nvgpu_ecc_reporting *error_reporting = &ecc_report_linux->common; + + nvgpu_spinlock_acquire(&ecc_report_linux->common.lock); + error_reporting->ops = &default_disabled_ecc_report_ops; + nvgpu_log(g, gpu_dbg_ecc, "ECC reporting is disabled"); + nvgpu_spinlock_release(&ecc_report_linux->common.lock); +} + +void nvgpu_report_ecc_err(struct gk20a *g, u32 hw_unit, u32 inst, + u32 err_id, u64 err_addr, u64 err_count) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + struct nvgpu_ecc_reporting_linux *ecc_report_linux = &l->ecc_reporting_linux; + struct nvgpu_ecc_reporting *error_reporting = &ecc_report_linux->common; + void (*report_ecc_err_func)(struct gk20a *g, u32 hw_unit, u32 inst, + u32 err_id, u64 err_addr, u64 err_count); + + nvgpu_spinlock_acquire(&ecc_report_linux->common.lock); + report_ecc_err_func = error_reporting->ops->report_ecc_err; + nvgpu_spinlock_release(&ecc_report_linux->common.lock); + + report_ecc_err_func(g, hw_unit, inst, err_id, err_addr, err_count); +} diff --git a/include/os/linux/vm.c b/include/os/linux/vm.c index dc807ab..8956cce 100644 --- a/include/os/linux/vm.c +++ b/include/os/linux/vm.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -50,8 +50,10 @@ static u32 nvgpu_vm_translate_linux_flags(struct gk20a *g, u32 flags) core_flags |= NVGPU_VM_MAP_IO_COHERENT; if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_UNMAPPED_PTE) core_flags |= NVGPU_VM_MAP_UNMAPPED_PTE; - if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_L3_ALLOC) - core_flags |= NVGPU_VM_MAP_L3_ALLOC; + if (!nvgpu_is_enabled(g, NVGPU_DISABLE_L3_SUPPORT)) { + if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_L3_ALLOC) + core_flags |= NVGPU_VM_MAP_L3_ALLOC; + } if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_DIRECT_KIND_CTRL) core_flags |= NVGPU_VM_MAP_DIRECT_KIND_CTRL; if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_PLATFORM_ATOMIC) diff --git a/include/os/posix/nvgpu.c b/include/os/posix/nvgpu.c index e485ed7..5b68113 100644 --- a/include/os/posix/nvgpu.c +++ b/include/os/posix/nvgpu.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -36,6 +36,22 @@ #include "os_posix.h" +int nvgpu_wait_for_stall_interrupts(struct gk20a *g, u32 timeout) +{ + /* + * No interrupts in userspace so nothing to wait for. + */ + return 0; +} + +int nvgpu_wait_for_nonstall_interrupts(struct gk20a *g, u32 timeout) +{ + /* + * No interrupts in userspace so nothing to wait for. + */ + return 0; +} + void nvgpu_wait_for_deferred_interrupts(struct gk20a *g) { /* -- cgit v1.2.2