From b4b1fb97bd095fa0de72550b04dc414b0194903e Mon Sep 17 00:00:00 2001 From: Nitin Kumbhar Date: Mon, 11 Jun 2018 15:31:33 +0530 Subject: gpu: nvgpu: shutdown nvlink in driver remove During driver remove, if nvlink is set up, gracefully shut it down so that it can be enumerated again. Bug 1987855 Change-Id: Ibd83a5e29364b22264e689aa879569a9cccf0f79 Signed-off-by: Nitin Kumbhar Reviewed-on: https://git-master.nvidia.com/r/1746073 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/common/nvlink.c | 11 +++++++++++ drivers/gpu/nvgpu/gk20a/gk20a.h | 1 + drivers/gpu/nvgpu/gv100/hal_gv100.c | 1 + drivers/gpu/nvgpu/gv100/nvlink_gv100.c | 5 +++++ drivers/gpu/nvgpu/gv100/nvlink_gv100.h | 1 + drivers/gpu/nvgpu/os/linux/nvlink.c | 26 ++++++++++++++++++++++++++ drivers/gpu/nvgpu/os/linux/nvlink.h | 22 ++++++++++++++++++++++ drivers/gpu/nvgpu/os/linux/pci.c | 8 +++++--- 8 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/nvgpu/os/linux/nvlink.h diff --git a/drivers/gpu/nvgpu/common/nvlink.c b/drivers/gpu/nvgpu/common/nvlink.c index 6bfc154c..b13e5942 100644 --- a/drivers/gpu/nvgpu/common/nvlink.c +++ b/drivers/gpu/nvgpu/common/nvlink.c @@ -92,6 +92,16 @@ static int nvgpu_nvlink_interface_init(struct nvlink_device *ndev) return err; } +static int nvgpu_nvlink_interface_disable(struct nvlink_device *ndev) +{ + int err = 0; + struct gk20a *g = (struct gk20a *) ndev->priv; + + if (g->ops.nvlink.interface_disable) + err = g->ops.nvlink.interface_disable(g); + return err; +} + static int nvgpu_nvlink_shutdown(struct nvlink_device *ndev) { int err; @@ -401,6 +411,7 @@ static int nvgpu_nvlink_init_ops(struct gk20a *g) ndev->dev_ops.dev_early_init = nvgpu_nvlink_early_init; ndev->dev_ops.dev_interface_init = nvgpu_nvlink_interface_init; ndev->dev_ops.dev_reg_init = nvgpu_nvlink_reg_init; + ndev->dev_ops.dev_interface_disable = nvgpu_nvlink_interface_disable; ndev->dev_ops.dev_shutdown = nvgpu_nvlink_shutdown; /* Fill in the link struct */ diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index 4fe93e3b..0df111c8 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h @@ -1267,6 +1267,7 @@ struct gpu_ops { int (*set_sublink_mode)(struct gk20a *g, u32 link_id, bool is_rx_sublink, u32 mode); int (*interface_init)(struct gk20a *g); + int (*interface_disable)(struct gk20a *g); int (*reg_init)(struct gk20a *g); int (*shutdown)(struct gk20a *g); int (*early_init)(struct gk20a *g); diff --git a/drivers/gpu/nvgpu/gv100/hal_gv100.c b/drivers/gpu/nvgpu/gv100/hal_gv100.c index 6ebf1675..6339110e 100644 --- a/drivers/gpu/nvgpu/gv100/hal_gv100.c +++ b/drivers/gpu/nvgpu/gv100/hal_gv100.c @@ -896,6 +896,7 @@ static const struct gpu_ops gv100_ops = { .get_rx_sublink_state = gv100_nvlink_link_get_rx_sublink_state, .set_sublink_mode = gv100_nvlink_link_set_sublink_mode, .interface_init = gv100_nvlink_interface_init, + .interface_disable = gv100_nvlink_interface_disable, .reg_init = gv100_nvlink_reg_init, .shutdown = gv100_nvlink_shutdown, .early_init = gv100_nvlink_early_init, diff --git a/drivers/gpu/nvgpu/gv100/nvlink_gv100.c b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c index c328dd70..41c2cd2b 100644 --- a/drivers/gpu/nvgpu/gv100/nvlink_gv100.c +++ b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c @@ -2269,6 +2269,11 @@ int gv100_nvlink_interface_init(struct gk20a *g) return 0; } +int gv100_nvlink_interface_disable(struct gk20a *g) +{ + return 0; +} + int gv100_nvlink_reg_init(struct gk20a *g) { u32 i = 0; diff --git a/drivers/gpu/nvgpu/gv100/nvlink_gv100.h b/drivers/gpu/nvgpu/gv100/nvlink_gv100.h index 6310af28..75595bca 100644 --- a/drivers/gpu/nvgpu/gv100/nvlink_gv100.h +++ b/drivers/gpu/nvgpu/gv100/nvlink_gv100.h @@ -50,6 +50,7 @@ u32 gv100_nvlink_link_get_rx_sublink_state(struct gk20a *g, u32 link_id); int gv100_nvlink_link_set_sublink_mode(struct gk20a *g, u32 link_id, bool is_rx_sublink, u32 mode); int gv100_nvlink_interface_init(struct gk20a *g); +int gv100_nvlink_interface_disable(struct gk20a *g); int gv100_nvlink_reg_init(struct gk20a *g); int gv100_nvlink_shutdown(struct gk20a *g); int gv100_nvlink_early_init(struct gk20a *g); diff --git a/drivers/gpu/nvgpu/os/linux/nvlink.c b/drivers/gpu/nvgpu/os/linux/nvlink.c index c93514c0..0db483d6 100644 --- a/drivers/gpu/nvgpu/os/linux/nvlink.c +++ b/drivers/gpu/nvgpu/os/linux/nvlink.c @@ -14,6 +14,10 @@ * along with this program. If not, see . */ +#ifdef CONFIG_TEGRA_NVLINK +#include +#endif + #include #include #include @@ -104,3 +108,25 @@ void nvgpu_mss_nvlink_init_credits(struct gk20a *g) val = readl_relaxed(soc4 + 4); writel_relaxed(val, soc4 + 4); } + +int nvgpu_nvlink_deinit(struct gk20a *g) +{ +#ifdef CONFIG_TEGRA_NVLINK + struct nvlink_device *ndev = g->nvlink.priv; + int err; + + if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) + return -ENODEV; + + err = nvlink_shutdown(ndev); + if (err) { + nvgpu_err(g, "failed to shut down nvlink"); + return err; + } + + nvgpu_nvlink_remove(g); + + return 0; +#endif + return -ENODEV; +} diff --git a/drivers/gpu/nvgpu/os/linux/nvlink.h b/drivers/gpu/nvgpu/os/linux/nvlink.h new file mode 100644 index 00000000..4dc54f6f --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/nvlink.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef NVGPU_OS_LINUX_NVLINK_H + +struct gk20a; +int nvgpu_nvlink_deinit(struct gk20a *g); + +#endif diff --git a/drivers/gpu/nvgpu/os/linux/pci.c b/drivers/gpu/nvgpu/os/linux/pci.c index 41fb69a0..ee4bcf8c 100644 --- a/drivers/gpu/nvgpu/os/linux/pci.c +++ b/drivers/gpu/nvgpu/os/linux/pci.c @@ -26,7 +26,9 @@ #include #include #include +#include +#include "nvlink.h" #include "gk20a/gk20a.h" #include "clk/clk.h" #include "clk/clk_mclk.h" @@ -35,12 +37,10 @@ #include "sysfs.h" #include "os_linux.h" #include "platform_gk20a.h" -#include #include "pci.h" #include "pci_usermode.h" -#include "os_linux.h" #include "driver_common.h" #define PCI_INTERFACE_NAME "card-%s%%s" @@ -828,9 +828,11 @@ static void nvgpu_pci_remove(struct pci_dev *pdev) if (gk20a_gpu_is_virtual(dev)) return; - nvgpu_nvlink_remove(g); + err = nvgpu_nvlink_deinit(g); + WARN(err, "gpu failed to remove nvlink"); gk20a_driver_start_unload(g); + err = nvgpu_quiesce(g); /* TODO: handle failure to idle */ WARN(err, "gpu failed to idle during driver removal"); -- cgit v1.2.2