From 686c3b701fb1db75a0b416a25c288c04a396df31 Mon Sep 17 00:00:00 2001 From: Terje Bergstrom Date: Thu, 2 Feb 2017 15:10:57 -0800 Subject: gpu: nvgpu: Move platform files to Tegra Linux Platform files are used for adding code to probe for Tegra Linux platform. Move the files to Tegra Linux directory to make this clear. Change-Id: Ida66af835688325f095260c618dad90395851267 Signed-off-by: Terje Bergstrom Reviewed-on: http://git-master/r/1300112 Reviewed-by: Alex Waterman GVS: Gerrit_Virtual_Submit --- drivers/gpu/nvgpu/Makefile.nvgpu | 4 +- drivers/gpu/nvgpu/gk20a/platform_gk20a_tegra.c | 1158 -------------------- drivers/gpu/nvgpu/gp10b/platform_gp10b_tegra.c | 793 -------------- .../gpu/nvgpu/tegra/linux/platform_gk20a_tegra.c | 1158 ++++++++++++++++++++ .../gpu/nvgpu/tegra/linux/platform_gp10b_tegra.c | 791 +++++++++++++ 5 files changed, 1951 insertions(+), 1953 deletions(-) delete mode 100644 drivers/gpu/nvgpu/gk20a/platform_gk20a_tegra.c delete mode 100644 drivers/gpu/nvgpu/gp10b/platform_gp10b_tegra.c create mode 100644 drivers/gpu/nvgpu/tegra/linux/platform_gk20a_tegra.c create mode 100644 drivers/gpu/nvgpu/tegra/linux/platform_gp10b_tegra.c diff --git a/drivers/gpu/nvgpu/Makefile.nvgpu b/drivers/gpu/nvgpu/Makefile.nvgpu index 6e6f79d9..00b4a291 100644 --- a/drivers/gpu/nvgpu/Makefile.nvgpu +++ b/drivers/gpu/nvgpu/Makefile.nvgpu @@ -90,7 +90,7 @@ nvgpu-y := \ boardobj/boardobjgrp_e255.o \ boardobj/boardobjgrp_e32.o -nvgpu-$(CONFIG_TEGRA_GK20A) += gk20a/platform_gk20a_tegra.o +nvgpu-$(CONFIG_TEGRA_GK20A) += tegra/linux/platform_gk20a_tegra.o nvgpu-$(CONFIG_SYNC) += gk20a/sync_gk20a.o nvgpu-$(CONFIG_GK20A_PCI) += pci.o @@ -192,7 +192,7 @@ nvgpu-y += \ lpwr/rppg.o \ lpwr/lpwr.o -nvgpu-$(CONFIG_TEGRA_GK20A) += gp10b/platform_gp10b_tegra.o +nvgpu-$(CONFIG_TEGRA_GK20A) += tegra/linux/platform_gp10b_tegra.o nvgpu-$(CONFIG_TEGRA_GR_VIRTUALIZATION) += \ vgpu/gp10b/vgpu_hal_gp10b.o \ diff --git a/drivers/gpu/nvgpu/gk20a/platform_gk20a_tegra.c b/drivers/gpu/nvgpu/gk20a/platform_gk20a_tegra.c deleted file mode 100644 index 225b98e4..00000000 --- a/drivers/gpu/nvgpu/gk20a/platform_gk20a_tegra.c +++ /dev/null @@ -1,1158 +0,0 @@ -/* - * GK20A Tegra Platform Interface - * - * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if defined(CONFIG_TEGRA_CLK_FRAMEWORK) -#include -#endif -#include -#include -#include -#if defined(CONFIG_COMMON_CLK) -#include -#endif -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) -#include -#endif -#ifdef CONFIG_TEGRA_BWMGR -#include -#endif - -#include - -#include "gk20a.h" -#include "hal_gk20a.h" -#include "platform_gk20a.h" -#include "gk20a_scale.h" -#include "gm20b/clk_gm20b.h" - -#define TEGRA_GK20A_BW_PER_FREQ 32 -#define TEGRA_GM20B_BW_PER_FREQ 64 -#define TEGRA_DDR3_BW_PER_FREQ 16 -#define TEGRA_DDR4_BW_PER_FREQ 16 -#define MC_CLIENT_GPU 34 -#define PMC_GPU_RG_CNTRL_0 0x2d4 - -#ifdef CONFIG_COMMON_CLK -#define GPU_RAIL_NAME "vdd-gpu" -#else -#define GPU_RAIL_NAME "vdd_gpu" -#endif - -extern struct device tegra_vpr_dev; - -#ifdef CONFIG_TEGRA_BWMGR -struct gk20a_emc_params { - unsigned long bw_ratio; - unsigned long freq_last_set; - struct tegra_bwmgr_client *bwmgr_cl; -}; -#else -struct gk20a_emc_params { - unsigned long bw_ratio; - unsigned long freq_last_set; -}; -#endif - -static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE); -static inline u32 __maybe_unused pmc_read(unsigned long reg) -{ - return readl(pmc + reg); -} - -static inline void __maybe_unused pmc_write(u32 val, unsigned long reg) -{ - writel_relaxed(val, pmc + reg); -} -#define MHZ_TO_HZ(x) ((x) * 1000000) -#define HZ_TO_MHZ(x) ((x) / 1000000) - -static void gk20a_tegra_secure_page_destroy(struct device *dev, - struct secure_page_buffer *secure_buffer) -{ - dma_free_attrs(&tegra_vpr_dev, secure_buffer->size, - (void *)(uintptr_t)secure_buffer->iova, - secure_buffer->iova, &secure_buffer->attrs); -} - -int gk20a_tegra_secure_page_alloc(struct device *dev) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - struct secure_page_buffer *secure_buffer = &platform->secure_buffer; - DEFINE_DMA_ATTRS(attrs); - dma_addr_t iova; - size_t size = PAGE_SIZE; - - if (platform->is_fmodel) - return -EINVAL; - - (void)dma_alloc_attrs(&tegra_vpr_dev, size, &iova, - DMA_MEMORY_NOMAP, &attrs); - if (dma_mapping_error(&tegra_vpr_dev, iova)) - return -ENOMEM; - - secure_buffer->size = size; - secure_buffer->iova = iova; - secure_buffer->attrs = attrs; - secure_buffer->destroy = gk20a_tegra_secure_page_destroy; - - return 0; -} - -static void gk20a_tegra_secure_destroy(struct gk20a *g, - struct gr_ctx_buffer_desc *desc) -{ - DEFINE_DMA_ATTRS(attrs); - - if (desc->mem.sgt) { - phys_addr_t pa = sg_phys(desc->mem.sgt->sgl); - dma_free_attrs(&tegra_vpr_dev, desc->mem.size, - (void *)(uintptr_t)pa, - pa, &attrs); - gk20a_free_sgtable(&desc->mem.sgt); - desc->mem.sgt = NULL; - } -} - -int gk20a_tegra_secure_alloc(struct device *dev, - struct gr_ctx_buffer_desc *desc, - size_t size) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - DEFINE_DMA_ATTRS(attrs); - dma_addr_t iova; - struct sg_table *sgt; - struct page *page; - int err = 0; - - if (!platform->secure_alloc_ready) - return -EINVAL; - - (void)dma_alloc_attrs(&tegra_vpr_dev, size, &iova, - DMA_MEMORY_NOMAP, &attrs); - if (dma_mapping_error(&tegra_vpr_dev, iova)) - return -ENOMEM; - - sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); - if (!sgt) { - gk20a_err(dev, "failed to allocate memory\n"); - goto fail; - } - err = sg_alloc_table(sgt, 1, GFP_KERNEL); - if (err) { - gk20a_err(dev, "failed to allocate sg_table\n"); - goto fail_sgt; - } - page = phys_to_page(iova); - sg_set_page(sgt->sgl, page, size, 0); - /* This bypasses SMMU for VPR during gmmu_map. */ - sg_dma_address(sgt->sgl) = 0; - - desc->destroy = gk20a_tegra_secure_destroy; - - desc->mem.sgt = sgt; - desc->mem.size = size; - desc->mem.aperture = APERTURE_SYSMEM; - - return err; - -fail_sgt: - kfree(sgt); -fail: - dma_free_attrs(&tegra_vpr_dev, desc->mem.size, - (void *)(uintptr_t)iova, iova, &attrs); - return err; -} - -/* - * gk20a_tegra_get_emc_rate() - * - * This function returns the minimum emc clock based on gpu frequency - */ - -static unsigned long gk20a_tegra_get_emc_rate(struct gk20a *g, - struct gk20a_emc_params *emc_params) -{ - unsigned long gpu_freq, gpu_fmax_at_vmin; - unsigned long emc_rate, emc_scale; - - gpu_freq = clk_get_rate(g->clk.tegra_clk); - gpu_fmax_at_vmin = tegra_dvfs_get_fmax_at_vmin_safe_t( - clk_get_parent(g->clk.tegra_clk)); - - /* When scaling emc, account for the gpu load when the - * gpu frequency is less than or equal to fmax@vmin. */ - if (gpu_freq <= gpu_fmax_at_vmin) - emc_scale = min(g->pmu.load_avg, g->emc3d_ratio); - else - emc_scale = g->emc3d_ratio; - - emc_rate = - (HZ_TO_MHZ(gpu_freq) * emc_params->bw_ratio * emc_scale) / 1000; - - return MHZ_TO_HZ(emc_rate); -} - -/* - * gk20a_tegra_postscale(profile, freq) - * - * This function sets emc frequency based on current gpu frequency - */ - -static void gk20a_tegra_postscale(struct device *dev, - unsigned long freq) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - struct gk20a_scale_profile *profile = platform->g->scale_profile; - struct gk20a_emc_params *emc_params = profile->private_data; - struct gk20a *g = get_gk20a(dev); - struct clk *emc_clk = platform->clk[2]; - enum tegra_chipid chip_id = tegra_get_chip_id(); - unsigned long emc_target; - unsigned long emc_freq_lower, emc_freq_upper, emc_freq_rounded; - - emc_target = gk20a_tegra_get_emc_rate(g, emc_params); - - switch (chip_id) { - case TEGRA124: - case TEGRA132: - /* T124 and T132 don't apply any rounding. The resulting - * emc frequency gets implicitly rounded up after issuing - * the clock_set_request. - * So explicitly round up the emc target here to achieve - * the same outcome. */ - emc_freq_rounded = - tegra_emc_round_rate_updown(emc_target, true); - break; - - case TEGRA210: - emc_freq_lower = (unsigned long) - tegra_emc_round_rate_updown(emc_target, false); - emc_freq_upper = (unsigned long) - tegra_emc_round_rate_updown(emc_target, true); - - /* round to the nearest frequency step */ - if (emc_target < (emc_freq_lower + emc_freq_upper) / 2) - emc_freq_rounded = emc_freq_lower; - else - emc_freq_rounded = emc_freq_upper; - break; - - default: - /* a proper rounding function needs to be implemented - * for emc in t18x */ - emc_freq_rounded = clk_round_rate(emc_clk, emc_target); - break; - } - - /* only change the emc clock if new rounded frequency is different - * from previously set emc rate */ - if (emc_freq_rounded != emc_params->freq_last_set) { - clk_set_rate(emc_clk, emc_freq_rounded); - emc_params->freq_last_set = emc_freq_rounded; - } -} - -/* - * gk20a_tegra_prescale(profile, freq) - * - * This function informs EDP about changed constraints. - */ - -static void gk20a_tegra_prescale(struct device *dev) -{ - struct gk20a *g = get_gk20a(dev); - u32 avg = 0; - - gk20a_pmu_load_norm(g, &avg); - tegra_edp_notify_gpu_load(avg, clk_get_rate(g->clk.tegra_clk)); -} - -/* - * gk20a_tegra_calibrate_emc() - * - */ - -static void gk20a_tegra_calibrate_emc(struct device *dev, - struct gk20a_emc_params *emc_params) -{ - enum tegra_chipid cid = tegra_get_chip_id(); - long gpu_bw, emc_bw; - - /* store gpu bw based on soc */ - switch (cid) { - case TEGRA210: - gpu_bw = TEGRA_GM20B_BW_PER_FREQ; - break; - case TEGRA124: - case TEGRA132: - gpu_bw = TEGRA_GK20A_BW_PER_FREQ; - break; - default: - gpu_bw = 0; - break; - } - - /* TODO detect DDR type. - * Okay for now since DDR3 and DDR4 have the same BW ratio */ - emc_bw = TEGRA_DDR3_BW_PER_FREQ; - - /* Calculate the bandwidth ratio of gpu_freq <-> emc_freq - * NOTE the ratio must come out as an integer */ - emc_params->bw_ratio = (gpu_bw / emc_bw); -} - -#ifdef CONFIG_TEGRA_BWMGR -void gm20b_bwmgr_set_rate(struct gk20a_platform *platform, bool enb) -{ - struct gk20a_scale_profile *profile = platform->g->scale_profile; - struct gk20a_emc_params *params; - unsigned long rate; - - if (!profile || !profile->private_data) - return; - - params = (struct gk20a_emc_params *)profile->private_data; - rate = (enb) ? params->freq_last_set : 0; - tegra_bwmgr_set_emc(params->bwmgr_cl, rate, TEGRA_BWMGR_SET_EMC_FLOOR); -} - -static void gm20b_tegra_postscale(struct device *dev, unsigned long freq) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - struct gk20a_scale_profile *profile = platform->g->scale_profile; - struct gk20a_emc_params *emc_params; - unsigned long emc_rate; - - if (!profile) - return; - - emc_params = profile->private_data; - emc_rate = gk20a_tegra_get_emc_rate(get_gk20a(dev), emc_params); - - if (emc_rate > tegra_bwmgr_get_max_emc_rate()) - emc_rate = tegra_bwmgr_get_max_emc_rate(); - - emc_params->freq_last_set = emc_rate; - nvgpu_mutex_acquire(&platform->railgate_lock); - if (platform->is_railgated && !platform->is_railgated(dev)) - goto done; - - tegra_bwmgr_set_emc(emc_params->bwmgr_cl, emc_rate, - TEGRA_BWMGR_SET_EMC_FLOOR); - -done: - nvgpu_mutex_release(&platform->railgate_lock); -} - -#endif - -#ifdef CONFIG_TEGRA_CLK_FRAMEWORK -/* - * gk20a_tegra_railgate() - * - * Gate (disable) gk20a power rail - */ - -static int gk20a_tegra_railgate(struct device *dev) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - int ret = 0; - - if (platform->is_fmodel || - !tegra_dvfs_is_rail_up(platform->gpu_rail)) - return 0; - - tegra_mc_flush(MC_CLIENT_GPU); - - udelay(10); - - /* enable clamp */ - pmc_write(0x1, PMC_GPU_RG_CNTRL_0); - pmc_read(PMC_GPU_RG_CNTRL_0); - - udelay(10); - - platform->reset_assert(dev); - - udelay(10); - - /* - * GPCPLL is already disabled before entering this function; reference - * clocks are enabled until now - disable them just before rail gating - */ - clk_disable(platform->clk[0]); - clk_disable(platform->clk[1]); - - udelay(10); - - if (tegra_dvfs_is_rail_up(platform->gpu_rail)) { - ret = tegra_dvfs_rail_power_down(platform->gpu_rail); - if (ret) - goto err_power_off; - } else - pr_info("No GPU regulator?\n"); - - return 0; - -err_power_off: - gk20a_err(dev, "Could not railgate GPU"); - return ret; -} - - -/* - * gk20a_tegra_unrailgate() - * - * Ungate (enable) gk20a power rail - */ - -static int gk20a_tegra_unrailgate(struct device *dev) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - int ret = 0; - bool first = false; - - if (platform->is_fmodel) - return 0; - - if (!platform->gpu_rail) { - platform->gpu_rail = tegra_dvfs_get_rail_by_name("vdd_gpu"); - if (IS_ERR_OR_NULL(platform->gpu_rail)) { - WARN(1, "No GPU regulator?\n"); - return -EINVAL; - } - first = true; - } - - ret = tegra_dvfs_rail_power_up(platform->gpu_rail); - if (ret) - return ret; - - if (!first) { - ret = clk_enable(platform->clk[0]); - if (ret) { - gk20a_err(dev, "could not turn on gpu pll"); - goto err_clk_on; - } - ret = clk_enable(platform->clk[1]); - if (ret) { - gk20a_err(dev, "could not turn on pwr clock"); - goto err_clk_on; - } - } - - udelay(10); - - platform->reset_assert(dev); - - udelay(10); - - pmc_write(0, PMC_GPU_RG_CNTRL_0); - pmc_read(PMC_GPU_RG_CNTRL_0); - - udelay(10); - - platform->reset_deassert(dev); - - /* Flush MC after boot/railgate/SC7 */ - tegra_mc_flush(MC_CLIENT_GPU); - - udelay(10); - - tegra_mc_flush_done(MC_CLIENT_GPU); - - udelay(10); - - return 0; - -err_clk_on: - tegra_dvfs_rail_power_down(platform->gpu_rail); - - return ret; -} - -#endif - - -#if defined(CONFIG_TEGRA_CLK_FRAMEWORK) || defined(CONFIG_TEGRA_DVFS) -/* - * gk20a_tegra_is_railgated() - * - * Check status of gk20a power rail - */ - -static bool gk20a_tegra_is_railgated(struct device *dev) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - bool ret = false; - - if (!platform->is_fmodel) - ret = !tegra_dvfs_is_rail_up(platform->gpu_rail); - - return ret; -} - -/* - * gm20b_tegra_railgate() - * - * Gate (disable) gm20b power rail - */ - -static int gm20b_tegra_railgate(struct device *dev) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - int ret = 0; - - if (platform->is_fmodel || - !tegra_dvfs_is_rail_up(platform->gpu_rail)) - return 0; - - tegra_mc_flush(MC_CLIENT_GPU); - - udelay(10); - - /* enable clamp */ - pmc_write(0x1, PMC_GPU_RG_CNTRL_0); - pmc_read(PMC_GPU_RG_CNTRL_0); - - udelay(10); - - platform->reset_assert(dev); - - udelay(10); - - /* - * GPCPLL is already disabled before entering this function; reference - * clocks are enabled until now - disable them just before rail gating - */ - clk_disable_unprepare(platform->clk_reset); - clk_disable_unprepare(platform->clk[0]); - clk_disable_unprepare(platform->clk[1]); - if (platform->clk[3]) - clk_disable_unprepare(platform->clk[3]); - - udelay(10); - - tegra_soctherm_gpu_tsens_invalidate(1); - - if (tegra_dvfs_is_rail_up(platform->gpu_rail)) { - ret = tegra_dvfs_rail_power_down(platform->gpu_rail); - if (ret) - goto err_power_off; - } else - pr_info("No GPU regulator?\n"); - -#ifdef CONFIG_TEGRA_BWMGR - gm20b_bwmgr_set_rate(platform, false); -#endif - - return 0; - -err_power_off: - gk20a_err(dev, "Could not railgate GPU"); - return ret; -} - - -/* - * gm20b_tegra_unrailgate() - * - * Ungate (enable) gm20b power rail - */ - -static int gm20b_tegra_unrailgate(struct device *dev) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - int ret = 0; - bool first = false; - - if (platform->is_fmodel) - return 0; - -#ifdef CONFIG_TEGRA_CLK_FRAMEWORK - if (!platform->gpu_rail) { - platform->gpu_rail = tegra_dvfs_get_rail_by_name(GPU_RAIL_NAME); - if (IS_ERR_OR_NULL(platform->gpu_rail)) { - WARN(1, "No GPU regulator?\n"); - return -EINVAL; - } - first = true; - } -#endif - - ret = tegra_dvfs_rail_power_up(platform->gpu_rail); - if (ret) - return ret; - -#ifdef CONFIG_TEGRA_BWMGR - gm20b_bwmgr_set_rate(platform, true); -#endif - - tegra_soctherm_gpu_tsens_invalidate(0); - - if (!platform->clk_reset) { - platform->clk_reset = clk_get(dev, "gpu_gate"); - if (IS_ERR(platform->clk_reset)) { - gk20a_err(dev, "fail to get gpu reset clk\n"); - goto err_clk_on; - } - } - - if (!first) { - ret = clk_prepare_enable(platform->clk_reset); - if (ret) { - gk20a_err(dev, "could not turn on gpu_gate"); - goto err_clk_on; - } - - ret = clk_prepare_enable(platform->clk[0]); - if (ret) { - gk20a_err(dev, "could not turn on gpu pll"); - goto err_clk_on; - } - ret = clk_prepare_enable(platform->clk[1]); - if (ret) { - gk20a_err(dev, "could not turn on pwr clock"); - goto err_clk_on; - } - - if (platform->clk[3]) { - ret = clk_prepare_enable(platform->clk[3]); - if (ret) { - gk20a_err(dev, "could not turn on fuse clock"); - goto err_clk_on; - } - } - } - - udelay(10); - - platform->reset_assert(dev); - - udelay(10); - - pmc_write(0, PMC_GPU_RG_CNTRL_0); - pmc_read(PMC_GPU_RG_CNTRL_0); - - udelay(10); - - clk_disable(platform->clk_reset); - platform->reset_deassert(dev); - clk_enable(platform->clk_reset); - - /* Flush MC after boot/railgate/SC7 */ - tegra_mc_flush(MC_CLIENT_GPU); - - udelay(10); - - tegra_mc_flush_done(MC_CLIENT_GPU); - - udelay(10); - - return 0; - -err_clk_on: - tegra_dvfs_rail_power_down(platform->gpu_rail); - - return ret; -} -#endif - - -static struct { - char *name; - unsigned long default_rate; -} tegra_gk20a_clocks[] = { -#ifdef CONFIG_TEGRA_CLK_FRAMEWORK - {"PLLG_ref", UINT_MAX}, - {"pwr", 204000000}, - {"emc", UINT_MAX}, -#elif defined(CONFIG_COMMON_CLK) - {"gpu_ref", UINT_MAX}, - {"pll_p_out5", 204000000}, - {"emc", UINT_MAX}, - {"fuse", UINT_MAX}, -#endif -}; - - - -/* - * gk20a_tegra_get_clocks() - * - * This function finds clocks in tegra platform and populates - * the clock information to gk20a platform data. - */ - -static int gk20a_tegra_get_clocks(struct device *dev) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - char devname[16]; - unsigned int i; - int ret = 0; - - BUG_ON(GK20A_CLKS_MAX < ARRAY_SIZE(tegra_gk20a_clocks)); - - snprintf(devname, sizeof(devname), "tegra_%s", dev_name(dev)); - - platform->num_clks = 0; - for (i = 0; i < ARRAY_SIZE(tegra_gk20a_clocks); i++) { - long rate = tegra_gk20a_clocks[i].default_rate; - struct clk *c; - - c = clk_get_sys(devname, tegra_gk20a_clocks[i].name); - if (IS_ERR(c)) { - ret = PTR_ERR(c); - goto err_get_clock; - } - rate = clk_round_rate(c, rate); - clk_set_rate(c, rate); - platform->clk[i] = c; - } - platform->num_clks = i; - - return 0; - -err_get_clock: - - while (i--) - clk_put(platform->clk[i]); - return ret; -} - -static int gk20a_tegra_reset_assert(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - - if (!platform->clk_reset) - platform->clk_reset = platform->clk[0]; - - tegra_periph_reset_assert(platform->clk_reset); - - return 0; -} - -static int gk20a_tegra_reset_deassert(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - - if (!platform->clk_reset) - return -EINVAL; - - tegra_periph_reset_deassert(platform->clk_reset); - - return 0; -} - -#if defined(CONFIG_RESET_CONTROLLER) && defined(CONFIG_COMMON_CLK) -static int gm20b_tegra_reset_assert(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - - if (!platform->reset_control) { - WARN(1, "Reset control not initialized\n"); - return -ENOSYS; - } - - return reset_control_assert(platform->reset_control); -} - -static int gm20b_tegra_reset_deassert(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - - if (!platform->reset_control) { - WARN(1, "Reset control not initialized\n"); - return -ENOSYS; - } - - return reset_control_deassert(platform->reset_control); -} -#endif - -static void gk20a_tegra_scale_init(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a_scale_profile *profile = platform->g->scale_profile; - struct gk20a_emc_params *emc_params; - - if (!profile) - return; - - emc_params = kzalloc(sizeof(*emc_params), GFP_KERNEL); - if (!emc_params) - return; - - emc_params->freq_last_set = -1; - gk20a_tegra_calibrate_emc(dev, emc_params); - -#ifdef CONFIG_TEGRA_BWMGR - emc_params->bwmgr_cl = tegra_bwmgr_register(TEGRA_BWMGR_CLIENT_GPU); - if (!emc_params->bwmgr_cl) { - gk20a_dbg_info("%s Missing GPU BWMGR client\n", __func__); - return; - } -#endif - - profile->private_data = emc_params; -} - -static void gk20a_tegra_scale_exit(struct device *dev) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - struct gk20a_scale_profile *profile = platform->g->scale_profile; - struct gk20a_emc_params *emc_params; - - if (!profile) - return; - - emc_params = profile->private_data; -#ifdef CONFIG_TEGRA_BWMGR - tegra_bwmgr_unregister(emc_params->bwmgr_cl); -#endif - - kfree(profile->private_data); -} - -void gk20a_tegra_debug_dump(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a *g = platform->g; - - if (g->host1x_dev) - nvhost_debug_dump_device(g->host1x_dev); -} - -int gk20a_tegra_busy(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a *g = platform->g; - - if (g->host1x_dev) - return nvhost_module_busy_ext(g->host1x_dev); - return 0; -} - -void gk20a_tegra_idle(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a *g = platform->g; - - if (g->host1x_dev) - nvhost_module_idle_ext(g->host1x_dev); -} - -static int gk20a_tegra_probe(struct device *dev) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - struct device_node *np = dev->of_node; - const __be32 *host1x_ptr; - struct platform_device *host1x_pdev = NULL; - bool joint_xpu_rail = false; - int ret; - -#ifdef CONFIG_COMMON_CLK - /* DVFS is not guaranteed to be initialized at the time of probe on - * kernels with Common Clock Framework enabled. - */ - if (!platform->gpu_rail) { - platform->gpu_rail = tegra_dvfs_get_rail_by_name(GPU_RAIL_NAME); - if (!platform->gpu_rail) { - gk20a_dbg_info("deferring probe no gpu_rail\n"); - return -EPROBE_DEFER; - } - } - - if (!tegra_dvfs_is_rail_ready(platform->gpu_rail)) { - gk20a_dbg_info("deferring probe gpu_rail not ready\n"); - return -EPROBE_DEFER; - } -#endif - - host1x_ptr = of_get_property(np, "nvidia,host1x", NULL); - if (host1x_ptr) { - struct device_node *host1x_node = - of_find_node_by_phandle(be32_to_cpup(host1x_ptr)); - - host1x_pdev = of_find_device_by_node(host1x_node); - if (!host1x_pdev) { - dev_warn(dev, "host1x device not available"); - return -EPROBE_DEFER; - } - - } else { - host1x_pdev = to_platform_device(dev->parent); - dev_warn(dev, "host1x reference not found. assuming host1x to be parent"); - } - - platform->g->host1x_dev = host1x_pdev; - -#ifdef CONFIG_OF - joint_xpu_rail = of_property_read_bool(of_chosen, - "nvidia,tegra-joint_xpu_rail"); -#endif - - if (joint_xpu_rail) { - gk20a_dbg_info("XPU rails are joint\n"); - platform->can_railgate = false; - } - - /* WAR for bug 1547668: Disable railgating and scaling irrespective of - * platform data if the rework has not been made. */ - - if (tegra_get_chip_id() == TEGRA210) { - np = of_find_node_by_path("/gpu-dvfs-rework"); - if (!(np && of_device_is_available(np))) { - platform->devfreq_governor = ""; - dev_warn(dev, "board does not support scaling"); - } - } - - if (tegra_get_chip_id() == TEGRA132) - platform->soc_name = "tegra13x"; - - platform->g->mm.vidmem_is_vidmem = platform->vidmem_is_vidmem; - - gk20a_tegra_get_clocks(dev); - - if (platform->clk_register) { - ret = platform->clk_register(platform->g); - if (ret) - return ret; - } - -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) - pmc = ioremap(TEGRA_PMC_BASE, 4096); -#endif - - return 0; -} - -static int gk20a_tegra_late_probe(struct device *dev) -{ - /* Initialise tegra specific scaling quirks */ - gk20a_tegra_scale_init(dev); - - return 0; -} - -static int gk20a_tegra_remove(struct device *dev) -{ - /* deinitialise tegra specific scaling quirks */ - gk20a_tegra_scale_exit(dev); - - return 0; -} - -static int gk20a_tegra_suspend(struct device *dev) -{ - tegra_edp_notify_gpu_load(0, 0); - return 0; -} - -#if defined(CONFIG_TEGRA_CLK_FRAMEWORK) || defined(CONFIG_COMMON_CLK) -static unsigned long gk20a_get_clk_rate(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a *g = platform->g; - - return gk20a_clk_get_rate(g); - -} - -static long gk20a_round_clk_rate(struct device *dev, unsigned long rate) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a *g = platform->g; - - return gk20a_clk_round_rate(g, rate); -} - -static int gk20a_set_clk_rate(struct device *dev, unsigned long rate) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a *g = platform->g; - - return gk20a_clk_set_rate(g, rate); -} - -static int gk20a_clk_get_freqs(struct device *dev, - unsigned long **freqs, int *num_freqs) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a *g = platform->g; - - /* make sure the clock is available */ - if (!gk20a_clk_get(g)) - return -ENOSYS; - - return tegra_dvfs_get_freqs(clk_get_parent(g->clk.tegra_clk), - freqs, num_freqs); -} -#endif - - -struct gk20a_platform gk20a_tegra_platform = { - .has_syncpoints = true, - .aggressive_sync_destroy_thresh = 64, - - /* power management configuration */ - .railgate_delay = 500, - .can_railgate = true, - .can_elpg = true, - .enable_slcg = true, - .enable_blcg = true, - .enable_elcg = true, - .enable_elpg = true, - .enable_aelpg = true, - .ptimer_src_freq = 12000000, - - .force_reset_in_do_idle = false, - - .default_big_page_size = SZ_128K, - - .ch_wdt_timeout_ms = 7000, - - .probe = gk20a_tegra_probe, - .late_probe = gk20a_tegra_late_probe, - .remove = gk20a_tegra_remove, - - /* power management callbacks */ - .suspend = gk20a_tegra_suspend, -#ifdef CONFIG_TEGRA_CLK_FRAMEWORK - .railgate = gk20a_tegra_railgate, - .unrailgate = gk20a_tegra_unrailgate, - .is_railgated = gk20a_tegra_is_railgated, -#endif - - .busy = gk20a_tegra_busy, - .idle = gk20a_tegra_idle, - - .reset_assert = gk20a_tegra_reset_assert, - .reset_deassert = gk20a_tegra_reset_deassert, - -#ifdef CONFIG_TEGRA_CLK_FRAMEWORK - .clk_get_rate = gk20a_get_clk_rate, - .clk_round_rate = gk20a_round_clk_rate, - .clk_set_rate = gk20a_set_clk_rate, - .get_clk_freqs = gk20a_clk_get_freqs, -#endif - - /* frequency scaling configuration */ - .prescale = gk20a_tegra_prescale, - .postscale = gk20a_tegra_postscale, - .devfreq_governor = "nvhost_podgov", - .qos_notify = gk20a_scale_qos_notify, - - .secure_alloc = gk20a_tegra_secure_alloc, - .secure_page_alloc = gk20a_tegra_secure_page_alloc, - .dump_platform_dependencies = gk20a_tegra_debug_dump, - - .has_ce = true, - - .soc_name = "tegra12x", - - .vidmem_is_vidmem = false, -}; - -struct gk20a_platform gm20b_tegra_platform = { - .has_syncpoints = true, - .aggressive_sync_destroy_thresh = 64, - - /* power management configuration */ - .railgate_delay = 500, - .can_railgate = true, - .can_elpg = true, - .enable_slcg = true, - .enable_blcg = true, - .enable_elcg = true, - .enable_elpg = true, - .enable_aelpg = true, - .ptimer_src_freq = 19200000, - - .force_reset_in_do_idle = false, - - .default_big_page_size = SZ_128K, - - .ch_wdt_timeout_ms = 5000, - - .probe = gk20a_tegra_probe, - .late_probe = gk20a_tegra_late_probe, - .remove = gk20a_tegra_remove, - /* power management callbacks */ - .suspend = gk20a_tegra_suspend, - -#if defined(CONFIG_TEGRA_CLK_FRAMEWORK) || defined(CONFIG_TEGRA_DVFS) - .railgate = gm20b_tegra_railgate, - .unrailgate = gm20b_tegra_unrailgate, - .is_railgated = gk20a_tegra_is_railgated, -#endif - - .busy = gk20a_tegra_busy, - .idle = gk20a_tegra_idle, - -#if defined(CONFIG_RESET_CONTROLLER) && defined(CONFIG_COMMON_CLK) - .reset_assert = gm20b_tegra_reset_assert, - .reset_deassert = gm20b_tegra_reset_deassert, -#else - .reset_assert = gk20a_tegra_reset_assert, - .reset_deassert = gk20a_tegra_reset_deassert, -#endif - -#if defined(CONFIG_TEGRA_CLK_FRAMEWORK) || defined(CONFIG_COMMON_CLK) - .clk_get_rate = gk20a_get_clk_rate, - .clk_round_rate = gk20a_round_clk_rate, - .clk_set_rate = gk20a_set_clk_rate, - .get_clk_freqs = gk20a_clk_get_freqs, -#endif - -#ifdef CONFIG_COMMON_CLK - .clk_register = gm20b_register_gpcclk, -#endif - - /* frequency scaling configuration */ - .prescale = gk20a_tegra_prescale, -#ifdef CONFIG_TEGRA_BWMGR - .postscale = gm20b_tegra_postscale, -#else - .postscale = gk20a_tegra_postscale, -#endif - .devfreq_governor = "nvhost_podgov", - .qos_notify = gk20a_scale_qos_notify, - - .secure_alloc = gk20a_tegra_secure_alloc, - .secure_page_alloc = gk20a_tegra_secure_page_alloc, - .dump_platform_dependencies = gk20a_tegra_debug_dump, - - .has_cde = true, - - .has_ce = true, - - .soc_name = "tegra21x", - - .vidmem_is_vidmem = false, -}; diff --git a/drivers/gpu/nvgpu/gp10b/platform_gp10b_tegra.c b/drivers/gpu/nvgpu/gp10b/platform_gp10b_tegra.c deleted file mode 100644 index 3a232402..00000000 --- a/drivers/gpu/nvgpu/gp10b/platform_gp10b_tegra.c +++ /dev/null @@ -1,793 +0,0 @@ -/* - * GP10B Tegra Platform Interface - * - * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include "gk20a/platform_gk20a.h" -#include "gk20a/gk20a.h" -#include "gk20a/gk20a_scale.h" - -#include "platform_tegra.h" -#include "gr_gp10b.h" -#include "ltc_gp10b.h" -#include "gp10b_sysfs.h" - -#include -#include - -/* Select every GP10B_FREQ_SELECT_STEP'th frequency from h/w table */ -#define GP10B_FREQ_SELECT_STEP 8 -/* Max number of freq supported in h/w */ -#define GP10B_MAX_SUPPORTED_FREQS 120 -static unsigned long -gp10b_freq_table[GP10B_MAX_SUPPORTED_FREQS / GP10B_FREQ_SELECT_STEP]; - -#define TEGRA_GP10B_BW_PER_FREQ 64 -#define TEGRA_DDR4_BW_PER_FREQ 16 - -#define EMC_BW_RATIO (TEGRA_GP10B_BW_PER_FREQ / TEGRA_DDR4_BW_PER_FREQ) - -static struct { - char *name; - unsigned long default_rate; -} tegra_gp10b_clocks[] = { - {"gpu", 1000000000}, - {"gpu_sys", 204000000} }; - -static void gr_gp10b_remove_sysfs(struct device *dev); - -/* - * gp10b_tegra_get_clocks() - * - * This function finds clocks in tegra platform and populates - * the clock information to gp10b platform data. - */ - -int gp10b_tegra_get_clocks(struct device *dev) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - unsigned int i; - - if (platform->is_fmodel) - return 0; - - platform->num_clks = 0; - for (i = 0; i < ARRAY_SIZE(tegra_gp10b_clocks); i++) { - long rate = tegra_gp10b_clocks[i].default_rate; - struct clk *c; - - c = clk_get(dev, tegra_gp10b_clocks[i].name); - if (IS_ERR(c)) { - gk20a_err(dev, "cannot get clock %s", - tegra_gp10b_clocks[i].name); - } else { - clk_set_rate(c, rate); - platform->clk[i] = c; - } - } - platform->num_clks = i; - - return 0; -} - -static void gp10b_tegra_scale_init(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a_scale_profile *profile = platform->g->scale_profile; - struct tegra_bwmgr_client *bwmgr_handle; - - if (!profile) - return; - - bwmgr_handle = tegra_bwmgr_register(TEGRA_BWMGR_CLIENT_GPU); - if (!bwmgr_handle) - return; - - profile->private_data = (void *)bwmgr_handle; -} - -static void gp10b_tegra_scale_exit(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a_scale_profile *profile = platform->g->scale_profile; - - if (profile) - tegra_bwmgr_unregister( - (struct tegra_bwmgr_client *)profile->private_data); -} - -static int gp10b_tegra_probe(struct device *dev) -{ - struct gk20a_platform *platform = dev_get_drvdata(dev); - struct device_node *np = dev->of_node; - struct device_node *host1x_node; - struct platform_device *host1x_pdev; - const __be32 *host1x_ptr; - - host1x_ptr = of_get_property(np, "nvidia,host1x", NULL); - if (!host1x_ptr) { - gk20a_err(dev, "host1x device not available"); - return -ENOSYS; - } - - host1x_node = of_find_node_by_phandle(be32_to_cpup(host1x_ptr)); - host1x_pdev = of_find_device_by_node(host1x_node); - if (!host1x_pdev) { - gk20a_err(dev, "host1x device not available"); - return -ENOSYS; - } - - platform->g->host1x_dev = host1x_pdev; - platform->bypass_smmu = !device_is_iommuable(dev); - platform->disable_bigpage = platform->bypass_smmu; - - platform->g->gr.t18x.ctx_vars.dump_ctxsw_stats_on_channel_close - = false; - platform->g->gr.t18x.ctx_vars.dump_ctxsw_stats_on_channel_close - = false; - - platform->g->gr.t18x.ctx_vars.force_preemption_gfxp = false; - platform->g->gr.t18x.ctx_vars.force_preemption_cilp = false; - - platform->g->gr.t18x.ctx_vars.debugfs_force_preemption_gfxp = - debugfs_create_bool("force_preemption_gfxp", S_IRUGO|S_IWUSR, - platform->debugfs, - &platform->g->gr.t18x.ctx_vars.force_preemption_gfxp); - - platform->g->gr.t18x.ctx_vars.debugfs_force_preemption_cilp = - debugfs_create_bool("force_preemption_cilp", S_IRUGO|S_IWUSR, - platform->debugfs, - &platform->g->gr.t18x.ctx_vars.force_preemption_cilp); - - platform->g->gr.t18x.ctx_vars.debugfs_dump_ctxsw_stats = - debugfs_create_bool("dump_ctxsw_stats_on_channel_close", - S_IRUGO|S_IWUSR, - platform->debugfs, - &platform->g->gr.t18x. - ctx_vars.dump_ctxsw_stats_on_channel_close); - - platform->g->mm.vidmem_is_vidmem = platform->vidmem_is_vidmem; - - gp10b_tegra_get_clocks(dev); - - return 0; -} - -static int gp10b_tegra_late_probe(struct device *dev) -{ - /*Create GP10B specific sysfs*/ - gp10b_create_sysfs(dev); - - /* Initialise tegra specific scaling quirks */ - gp10b_tegra_scale_init(dev); - return 0; -} - -static int gp10b_tegra_remove(struct device *dev) -{ - gr_gp10b_remove_sysfs(dev); - /*Remove GP10B specific sysfs*/ - gp10b_remove_sysfs(dev); - - /* deinitialise tegra specific scaling quirks */ - gp10b_tegra_scale_exit(dev); - - return 0; - -} - -static bool gp10b_tegra_is_railgated(struct device *dev) -{ - bool ret = false; - - if (tegra_bpmp_running()) - ret = !tegra_powergate_is_powered(TEGRA186_POWER_DOMAIN_GPU); - - return ret; -} - -static int gp10b_tegra_railgate(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a_scale_profile *profile = platform->g->scale_profile; - - /* remove emc frequency floor */ - if (profile) - tegra_bwmgr_set_emc( - (struct tegra_bwmgr_client *)profile->private_data, - 0, TEGRA_BWMGR_SET_EMC_FLOOR); - - if (tegra_bpmp_running() && - tegra_powergate_is_powered(TEGRA186_POWER_DOMAIN_GPU)) { - int i; - for (i = 0; i < platform->num_clks; i++) { - if (platform->clk[i]) - clk_disable_unprepare(platform->clk[i]); - } - tegra_powergate_partition(TEGRA186_POWER_DOMAIN_GPU); - } - return 0; -} - -static int gp10b_tegra_unrailgate(struct device *dev) -{ - int ret = 0; - struct gk20a_platform *platform = gk20a_get_platform(dev); - struct gk20a_scale_profile *profile = platform->g->scale_profile; - - if (tegra_bpmp_running()) { - int i; - ret = tegra_unpowergate_partition(TEGRA186_POWER_DOMAIN_GPU); - for (i = 0; i < platform->num_clks; i++) { - if (platform->clk[i]) - clk_prepare_enable(platform->clk[i]); - } - } - - /* to start with set emc frequency floor to max rate*/ - if (profile) - tegra_bwmgr_set_emc( - (struct tegra_bwmgr_client *)profile->private_data, - tegra_bwmgr_get_max_emc_rate(), - TEGRA_BWMGR_SET_EMC_FLOOR); - return ret; -} - -static int gp10b_tegra_suspend(struct device *dev) -{ - return 0; -} - -int gp10b_tegra_reset_assert(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - int ret = 0; - - if (!platform->reset_control) - return -EINVAL; - - ret = reset_control_assert(platform->reset_control); - - return ret; -} - -int gp10b_tegra_reset_deassert(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - int ret = 0; - - if (!platform->reset_control) - return -EINVAL; - - ret = reset_control_deassert(platform->reset_control); - - return ret; -} - -static void gp10b_tegra_prescale(struct device *dev) -{ - struct gk20a *g = get_gk20a(dev); - u32 avg = 0; - - gk20a_dbg_fn(""); - - gk20a_pmu_load_norm(g, &avg); - - gk20a_dbg_fn("done"); -} - -static void gp10b_tegra_postscale(struct device *pdev, - unsigned long freq) -{ - struct gk20a_platform *platform = gk20a_get_platform(pdev); - struct gk20a_scale_profile *profile = platform->g->scale_profile; - struct gk20a *g = get_gk20a(pdev); - unsigned long emc_rate; - - gk20a_dbg_fn(""); - if (profile && !gp10b_tegra_is_railgated(pdev)) { - unsigned long emc_scale; - - if (freq <= gp10b_freq_table[0]) - emc_scale = 0; - else - emc_scale = g->emc3d_ratio; - - emc_rate = (freq * EMC_BW_RATIO * emc_scale) / 1000; - - if (emc_rate > tegra_bwmgr_get_max_emc_rate()) - emc_rate = tegra_bwmgr_get_max_emc_rate(); - - tegra_bwmgr_set_emc( - (struct tegra_bwmgr_client *)profile->private_data, - emc_rate, TEGRA_BWMGR_SET_EMC_FLOOR); - } - gk20a_dbg_fn("done"); -} - -static unsigned long gp10b_get_clk_rate(struct device *dev) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - - return clk_get_rate(platform->clk[0]); - -} - -static long gp10b_round_clk_rate(struct device *dev, unsigned long rate) -{ - struct gk20a *g = get_gk20a(dev); - struct gk20a_scale_profile *profile = g->scale_profile; - unsigned long *freq_table = profile->devfreq_profile.freq_table; - int max_states = profile->devfreq_profile.max_state; - int i; - - for (i = 0; i < max_states; ++i) - if (freq_table[i] >= rate) - return freq_table[i]; - - return freq_table[max_states - 1]; -} - -static int gp10b_set_clk_rate(struct device *dev, unsigned long rate) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - - return clk_set_rate(platform->clk[0], rate); -} - -static int gp10b_clk_get_freqs(struct device *dev, - unsigned long **freqs, int *num_freqs) -{ - struct gk20a_platform *platform = gk20a_get_platform(dev); - unsigned long max_rate; - unsigned long new_rate = 0, prev_rate = 0; - int i = 0, freq_counter = 0; - - max_rate = clk_round_rate(platform->clk[0], (UINT_MAX - 1)); - - /* - * Walk the h/w frequency table and only select - * GP10B_FREQ_SELECT_STEP'th frequencies and - * add MAX freq to last - */ - for (; i < GP10B_MAX_SUPPORTED_FREQS; ++i) { - prev_rate = new_rate; - new_rate = clk_round_rate(platform->clk[0], prev_rate + 1); - - if (i % GP10B_FREQ_SELECT_STEP == 0 || - new_rate == max_rate) { - gp10b_freq_table[freq_counter++] = new_rate; - - if (new_rate == max_rate) - break; - } - } - - WARN_ON(i == GP10B_MAX_SUPPORTED_FREQS); - - /* Fill freq table */ - *freqs = gp10b_freq_table; - *num_freqs = freq_counter; - - gk20a_dbg_info("min rate: %ld max rate: %ld num_of_freq %d\n", - gp10b_freq_table[0], max_rate, *num_freqs); - - return 0; -} - -struct gk20a_platform gp10b_tegra_platform = { - .has_syncpoints = true, - - /* power management configuration */ - .railgate_delay = 500, - - /* power management configuration */ - .can_railgate = true, - .enable_elpg = true, - .can_elpg = true, - .enable_blcg = true, - .enable_slcg = true, - .enable_elcg = true, - .enable_aelpg = true, - - /* ptimer src frequency in hz*/ - .ptimer_src_freq = 31250000, - - .ch_wdt_timeout_ms = 5000, - - .probe = gp10b_tegra_probe, - .late_probe = gp10b_tegra_late_probe, - .remove = gp10b_tegra_remove, - - /* power management callbacks */ - .suspend = gp10b_tegra_suspend, - .railgate = gp10b_tegra_railgate, - .unrailgate = gp10b_tegra_unrailgate, - .is_railgated = gp10b_tegra_is_railgated, - - .busy = gk20a_tegra_busy, - .idle = gk20a_tegra_idle, - - .dump_platform_dependencies = gk20a_tegra_debug_dump, - - .default_big_page_size = SZ_64K, - - .has_cde = true, - - .has_ce = true, - - .clk_get_rate = gp10b_get_clk_rate, - .clk_round_rate = gp10b_round_clk_rate, - .clk_set_rate = gp10b_set_clk_rate, - .get_clk_freqs = gp10b_clk_get_freqs, - - /* frequency scaling configuration */ - .prescale = gp10b_tegra_prescale, - .postscale = gp10b_tegra_postscale, - .devfreq_governor = "nvhost_podgov", - - .qos_notify = gk20a_scale_qos_notify, - - .secure_alloc = gk20a_tegra_secure_alloc, - .secure_page_alloc = gk20a_tegra_secure_page_alloc, - - .reset_assert = gp10b_tegra_reset_assert, - .reset_deassert = gp10b_tegra_reset_deassert, - - .force_reset_in_do_idle = false, - - .soc_name = "tegra18x", - - .vidmem_is_vidmem = false, -}; - - -#define ECC_STAT_NAME_MAX_SIZE 100 - - -static DEFINE_HASHTABLE(ecc_hash_table, 5); - -static struct device_attribute *dev_attr_sm_lrf_ecc_single_err_count_array; -static struct device_attribute *dev_attr_sm_lrf_ecc_double_err_count_array; - -static struct device_attribute *dev_attr_sm_shm_ecc_sec_count_array; -static struct device_attribute *dev_attr_sm_shm_ecc_sed_count_array; -static struct device_attribute *dev_attr_sm_shm_ecc_ded_count_array; - -static struct device_attribute *dev_attr_tex_ecc_total_sec_pipe0_count_array; -static struct device_attribute *dev_attr_tex_ecc_total_ded_pipe0_count_array; -static struct device_attribute *dev_attr_tex_ecc_unique_sec_pipe0_count_array; -static struct device_attribute *dev_attr_tex_ecc_unique_ded_pipe0_count_array; -static struct device_attribute *dev_attr_tex_ecc_total_sec_pipe1_count_array; -static struct device_attribute *dev_attr_tex_ecc_total_ded_pipe1_count_array; -static struct device_attribute *dev_attr_tex_ecc_unique_sec_pipe1_count_array; -static struct device_attribute *dev_attr_tex_ecc_unique_ded_pipe1_count_array; - -static struct device_attribute *dev_attr_l2_ecc_sec_count_array; -static struct device_attribute *dev_attr_l2_ecc_ded_count_array; - - -static u32 gen_ecc_hash_key(char *str) -{ - int i = 0; - u32 hash_key = 0; - - while (str[i]) { - hash_key += (u32)(str[i]); - i++; - }; - - return hash_key; -} - -static ssize_t ecc_stat_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - const char *ecc_stat_full_name = attr->attr.name; - const char *ecc_stat_base_name; - unsigned int hw_unit; - struct ecc_stat *ecc_stat; - u32 hash_key; - - if (sscanf(ecc_stat_full_name, "ltc%u", &hw_unit) == 1) { - ecc_stat_base_name = &(ecc_stat_full_name[strlen("ltc0_")]); - } else if (sscanf(ecc_stat_full_name, "gpc0_tpc%u", &hw_unit) == 1) { - ecc_stat_base_name = &(ecc_stat_full_name[strlen("gpc0_tpc0_")]); - } else { - return snprintf(buf, - PAGE_SIZE, - "Error: Invalid ECC stat name!\n"); - } - - hash_key = gen_ecc_hash_key((char *)ecc_stat_base_name); - hash_for_each_possible(ecc_hash_table, - ecc_stat, - hash_node, - hash_key) { - if (!strcmp(ecc_stat_full_name, ecc_stat->names[hw_unit])) - return snprintf(buf, PAGE_SIZE, "%u\n", ecc_stat->counters[hw_unit]); - } - - return snprintf(buf, PAGE_SIZE, "Error: No ECC stat found!\n"); -} - -static int ecc_stat_create(struct device *dev, - int is_l2, - char *ecc_stat_name, - struct ecc_stat *ecc_stat, - struct device_attribute *dev_attr_array) -{ - int error = 0; - struct gk20a *g = get_gk20a(dev); - int num_hw_units = 0; - int hw_unit = 0; - u32 hash_key = 0; - - if (is_l2) - num_hw_units = g->ltc_count; - else - num_hw_units = g->gr.tpc_count; - - /* Allocate arrays */ - dev_attr_array = kzalloc(sizeof(struct device_attribute) * num_hw_units, GFP_KERNEL); - ecc_stat->counters = kzalloc(sizeof(u32) * num_hw_units, GFP_KERNEL); - ecc_stat->names = kzalloc(sizeof(char *) * num_hw_units, GFP_KERNEL); - for (hw_unit = 0; hw_unit < num_hw_units; hw_unit++) { - ecc_stat->names[hw_unit] = kzalloc(sizeof(char) * ECC_STAT_NAME_MAX_SIZE, GFP_KERNEL); - } - - for (hw_unit = 0; hw_unit < num_hw_units; hw_unit++) { - /* Fill in struct device_attribute members */ - if (is_l2) - snprintf(ecc_stat->names[hw_unit], - ECC_STAT_NAME_MAX_SIZE, - "ltc%d_%s", - hw_unit, - ecc_stat_name); - else - snprintf(ecc_stat->names[hw_unit], - ECC_STAT_NAME_MAX_SIZE, - "gpc0_tpc%d_%s", - hw_unit, - ecc_stat_name); - - sysfs_attr_init(&dev_attr_array[hw_unit].attr); - dev_attr_array[hw_unit].attr.name = ecc_stat->names[hw_unit]; - dev_attr_array[hw_unit].attr.mode = VERIFY_OCTAL_PERMISSIONS(S_IRUGO); - dev_attr_array[hw_unit].show = ecc_stat_show; - dev_attr_array[hw_unit].store = NULL; - - /* Create sysfs file */ - error |= device_create_file(dev, &dev_attr_array[hw_unit]); - } - - /* Add hash table entry */ - hash_key = gen_ecc_hash_key(ecc_stat_name); - hash_add(ecc_hash_table, - &ecc_stat->hash_node, - hash_key); - - return error; -} - -static void ecc_stat_remove(struct device *dev, - int is_l2, - struct ecc_stat *ecc_stat, - struct device_attribute *dev_attr_array) -{ - struct gk20a *g = get_gk20a(dev); - int num_hw_units = 0; - int hw_unit = 0; - - if (is_l2) - num_hw_units = g->ltc_count; - else - num_hw_units = g->gr.tpc_count; - - /* Remove sysfs files */ - for (hw_unit = 0; hw_unit < num_hw_units; hw_unit++) { - device_remove_file(dev, &dev_attr_array[hw_unit]); - } - - /* Remove hash table entry */ - hash_del(&ecc_stat->hash_node); - - /* Free arrays */ - kfree(ecc_stat->counters); - for (hw_unit = 0; hw_unit < num_hw_units; hw_unit++) { - kfree(ecc_stat->names[hw_unit]); - } - kfree(ecc_stat->names); - kfree(dev_attr_array); -} - -void gr_gp10b_create_sysfs(struct device *dev) -{ - int error = 0; - struct gk20a *g = get_gk20a(dev); - - /* This stat creation function is called on GR init. GR can get - initialized multiple times but we only need to create the ECC - stats once. Therefore, add the following check to avoid - creating duplicate stat sysfs nodes. */ - if (g->gr.t18x.ecc_stats.sm_lrf_single_err_count.counters != NULL) - return; - - error |= ecc_stat_create(dev, - 0, - "sm_lrf_ecc_single_err_count", - &g->gr.t18x.ecc_stats.sm_lrf_single_err_count, - dev_attr_sm_lrf_ecc_single_err_count_array); - error |= ecc_stat_create(dev, - 0, - "sm_lrf_ecc_double_err_count", - &g->gr.t18x.ecc_stats.sm_lrf_double_err_count, - dev_attr_sm_lrf_ecc_double_err_count_array); - - error |= ecc_stat_create(dev, - 0, - "sm_shm_ecc_sec_count", - &g->gr.t18x.ecc_stats.sm_shm_sec_count, - dev_attr_sm_shm_ecc_sec_count_array); - error |= ecc_stat_create(dev, - 0, - "sm_shm_ecc_sed_count", - &g->gr.t18x.ecc_stats.sm_shm_sed_count, - dev_attr_sm_shm_ecc_sed_count_array); - error |= ecc_stat_create(dev, - 0, - "sm_shm_ecc_ded_count", - &g->gr.t18x.ecc_stats.sm_shm_ded_count, - dev_attr_sm_shm_ecc_ded_count_array); - - error |= ecc_stat_create(dev, - 0, - "tex_ecc_total_sec_pipe0_count", - &g->gr.t18x.ecc_stats.tex_total_sec_pipe0_count, - dev_attr_tex_ecc_total_sec_pipe0_count_array); - error |= ecc_stat_create(dev, - 0, - "tex_ecc_total_ded_pipe0_count", - &g->gr.t18x.ecc_stats.tex_total_ded_pipe0_count, - dev_attr_tex_ecc_total_ded_pipe0_count_array); - error |= ecc_stat_create(dev, - 0, - "tex_ecc_unique_sec_pipe0_count", - &g->gr.t18x.ecc_stats.tex_unique_sec_pipe0_count, - dev_attr_tex_ecc_unique_sec_pipe0_count_array); - error |= ecc_stat_create(dev, - 0, - "tex_ecc_unique_ded_pipe0_count", - &g->gr.t18x.ecc_stats.tex_unique_ded_pipe0_count, - dev_attr_tex_ecc_unique_ded_pipe0_count_array); - error |= ecc_stat_create(dev, - 0, - "tex_ecc_total_sec_pipe1_count", - &g->gr.t18x.ecc_stats.tex_total_sec_pipe1_count, - dev_attr_tex_ecc_total_sec_pipe1_count_array); - error |= ecc_stat_create(dev, - 0, - "tex_ecc_total_ded_pipe1_count", - &g->gr.t18x.ecc_stats.tex_total_ded_pipe1_count, - dev_attr_tex_ecc_total_ded_pipe1_count_array); - error |= ecc_stat_create(dev, - 0, - "tex_ecc_unique_sec_pipe1_count", - &g->gr.t18x.ecc_stats.tex_unique_sec_pipe1_count, - dev_attr_tex_ecc_unique_sec_pipe1_count_array); - error |= ecc_stat_create(dev, - 0, - "tex_ecc_unique_ded_pipe1_count", - &g->gr.t18x.ecc_stats.tex_unique_ded_pipe1_count, - dev_attr_tex_ecc_unique_ded_pipe1_count_array); - - error |= ecc_stat_create(dev, - 1, - "lts0_ecc_sec_count", - &g->gr.t18x.ecc_stats.l2_sec_count, - dev_attr_l2_ecc_sec_count_array); - error |= ecc_stat_create(dev, - 1, - "lts0_ecc_ded_count", - &g->gr.t18x.ecc_stats.l2_ded_count, - dev_attr_l2_ecc_ded_count_array); - - if (error) - dev_err(dev, "Failed to create sysfs attributes!\n"); -} - -static void gr_gp10b_remove_sysfs(struct device *dev) -{ - struct gk20a *g = get_gk20a(dev); - - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.sm_lrf_single_err_count, - dev_attr_sm_lrf_ecc_single_err_count_array); - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.sm_lrf_double_err_count, - dev_attr_sm_lrf_ecc_double_err_count_array); - - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.sm_shm_sec_count, - dev_attr_sm_shm_ecc_sec_count_array); - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.sm_shm_sed_count, - dev_attr_sm_shm_ecc_sed_count_array); - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.sm_shm_ded_count, - dev_attr_sm_shm_ecc_ded_count_array); - - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.tex_total_sec_pipe0_count, - dev_attr_tex_ecc_total_sec_pipe0_count_array); - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.tex_total_ded_pipe0_count, - dev_attr_tex_ecc_total_ded_pipe0_count_array); - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.tex_unique_sec_pipe0_count, - dev_attr_tex_ecc_unique_sec_pipe0_count_array); - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.tex_unique_ded_pipe0_count, - dev_attr_tex_ecc_unique_ded_pipe0_count_array); - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.tex_total_sec_pipe1_count, - dev_attr_tex_ecc_total_sec_pipe1_count_array); - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.tex_total_ded_pipe1_count, - dev_attr_tex_ecc_total_ded_pipe1_count_array); - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.tex_unique_sec_pipe1_count, - dev_attr_tex_ecc_unique_sec_pipe1_count_array); - ecc_stat_remove(dev, - 0, - &g->gr.t18x.ecc_stats.tex_unique_ded_pipe1_count, - dev_attr_tex_ecc_unique_ded_pipe1_count_array); - - ecc_stat_remove(dev, - 1, - &g->gr.t18x.ecc_stats.l2_sec_count, - dev_attr_l2_ecc_sec_count_array); - ecc_stat_remove(dev, - 1, - &g->gr.t18x.ecc_stats.l2_ded_count, - dev_attr_l2_ecc_ded_count_array); -} diff --git a/drivers/gpu/nvgpu/tegra/linux/platform_gk20a_tegra.c b/drivers/gpu/nvgpu/tegra/linux/platform_gk20a_tegra.c new file mode 100644 index 00000000..56ba1ecd --- /dev/null +++ b/drivers/gpu/nvgpu/tegra/linux/platform_gk20a_tegra.c @@ -0,0 +1,1158 @@ +/* + * GK20A Tegra Platform Interface + * + * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(CONFIG_TEGRA_CLK_FRAMEWORK) +#include +#endif +#include +#include +#include +#if defined(CONFIG_COMMON_CLK) +#include +#endif +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) +#include +#endif +#ifdef CONFIG_TEGRA_BWMGR +#include +#endif + +#include + +#include "gk20a/gk20a.h" +#include "gk20a/hal_gk20a.h" +#include "gk20a/platform_gk20a.h" +#include "gk20a/gk20a_scale.h" +#include "gm20b/clk_gm20b.h" + +#define TEGRA_GK20A_BW_PER_FREQ 32 +#define TEGRA_GM20B_BW_PER_FREQ 64 +#define TEGRA_DDR3_BW_PER_FREQ 16 +#define TEGRA_DDR4_BW_PER_FREQ 16 +#define MC_CLIENT_GPU 34 +#define PMC_GPU_RG_CNTRL_0 0x2d4 + +#ifdef CONFIG_COMMON_CLK +#define GPU_RAIL_NAME "vdd-gpu" +#else +#define GPU_RAIL_NAME "vdd_gpu" +#endif + +extern struct device tegra_vpr_dev; + +#ifdef CONFIG_TEGRA_BWMGR +struct gk20a_emc_params { + unsigned long bw_ratio; + unsigned long freq_last_set; + struct tegra_bwmgr_client *bwmgr_cl; +}; +#else +struct gk20a_emc_params { + unsigned long bw_ratio; + unsigned long freq_last_set; +}; +#endif + +static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE); +static inline u32 __maybe_unused pmc_read(unsigned long reg) +{ + return readl(pmc + reg); +} + +static inline void __maybe_unused pmc_write(u32 val, unsigned long reg) +{ + writel_relaxed(val, pmc + reg); +} +#define MHZ_TO_HZ(x) ((x) * 1000000) +#define HZ_TO_MHZ(x) ((x) / 1000000) + +static void gk20a_tegra_secure_page_destroy(struct device *dev, + struct secure_page_buffer *secure_buffer) +{ + dma_free_attrs(&tegra_vpr_dev, secure_buffer->size, + (void *)(uintptr_t)secure_buffer->iova, + secure_buffer->iova, &secure_buffer->attrs); +} + +int gk20a_tegra_secure_page_alloc(struct device *dev) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + struct secure_page_buffer *secure_buffer = &platform->secure_buffer; + DEFINE_DMA_ATTRS(attrs); + dma_addr_t iova; + size_t size = PAGE_SIZE; + + if (platform->is_fmodel) + return -EINVAL; + + (void)dma_alloc_attrs(&tegra_vpr_dev, size, &iova, + DMA_MEMORY_NOMAP, &attrs); + if (dma_mapping_error(&tegra_vpr_dev, iova)) + return -ENOMEM; + + secure_buffer->size = size; + secure_buffer->iova = iova; + secure_buffer->attrs = attrs; + secure_buffer->destroy = gk20a_tegra_secure_page_destroy; + + return 0; +} + +static void gk20a_tegra_secure_destroy(struct gk20a *g, + struct gr_ctx_buffer_desc *desc) +{ + DEFINE_DMA_ATTRS(attrs); + + if (desc->mem.sgt) { + phys_addr_t pa = sg_phys(desc->mem.sgt->sgl); + dma_free_attrs(&tegra_vpr_dev, desc->mem.size, + (void *)(uintptr_t)pa, + pa, &attrs); + gk20a_free_sgtable(&desc->mem.sgt); + desc->mem.sgt = NULL; + } +} + +int gk20a_tegra_secure_alloc(struct device *dev, + struct gr_ctx_buffer_desc *desc, + size_t size) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + DEFINE_DMA_ATTRS(attrs); + dma_addr_t iova; + struct sg_table *sgt; + struct page *page; + int err = 0; + + if (!platform->secure_alloc_ready) + return -EINVAL; + + (void)dma_alloc_attrs(&tegra_vpr_dev, size, &iova, + DMA_MEMORY_NOMAP, &attrs); + if (dma_mapping_error(&tegra_vpr_dev, iova)) + return -ENOMEM; + + sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); + if (!sgt) { + gk20a_err(dev, "failed to allocate memory\n"); + goto fail; + } + err = sg_alloc_table(sgt, 1, GFP_KERNEL); + if (err) { + gk20a_err(dev, "failed to allocate sg_table\n"); + goto fail_sgt; + } + page = phys_to_page(iova); + sg_set_page(sgt->sgl, page, size, 0); + /* This bypasses SMMU for VPR during gmmu_map. */ + sg_dma_address(sgt->sgl) = 0; + + desc->destroy = gk20a_tegra_secure_destroy; + + desc->mem.sgt = sgt; + desc->mem.size = size; + desc->mem.aperture = APERTURE_SYSMEM; + + return err; + +fail_sgt: + kfree(sgt); +fail: + dma_free_attrs(&tegra_vpr_dev, desc->mem.size, + (void *)(uintptr_t)iova, iova, &attrs); + return err; +} + +/* + * gk20a_tegra_get_emc_rate() + * + * This function returns the minimum emc clock based on gpu frequency + */ + +static unsigned long gk20a_tegra_get_emc_rate(struct gk20a *g, + struct gk20a_emc_params *emc_params) +{ + unsigned long gpu_freq, gpu_fmax_at_vmin; + unsigned long emc_rate, emc_scale; + + gpu_freq = clk_get_rate(g->clk.tegra_clk); + gpu_fmax_at_vmin = tegra_dvfs_get_fmax_at_vmin_safe_t( + clk_get_parent(g->clk.tegra_clk)); + + /* When scaling emc, account for the gpu load when the + * gpu frequency is less than or equal to fmax@vmin. */ + if (gpu_freq <= gpu_fmax_at_vmin) + emc_scale = min(g->pmu.load_avg, g->emc3d_ratio); + else + emc_scale = g->emc3d_ratio; + + emc_rate = + (HZ_TO_MHZ(gpu_freq) * emc_params->bw_ratio * emc_scale) / 1000; + + return MHZ_TO_HZ(emc_rate); +} + +/* + * gk20a_tegra_postscale(profile, freq) + * + * This function sets emc frequency based on current gpu frequency + */ + +static void gk20a_tegra_postscale(struct device *dev, + unsigned long freq) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + struct gk20a_scale_profile *profile = platform->g->scale_profile; + struct gk20a_emc_params *emc_params = profile->private_data; + struct gk20a *g = get_gk20a(dev); + struct clk *emc_clk = platform->clk[2]; + enum tegra_chipid chip_id = tegra_get_chip_id(); + unsigned long emc_target; + unsigned long emc_freq_lower, emc_freq_upper, emc_freq_rounded; + + emc_target = gk20a_tegra_get_emc_rate(g, emc_params); + + switch (chip_id) { + case TEGRA124: + case TEGRA132: + /* T124 and T132 don't apply any rounding. The resulting + * emc frequency gets implicitly rounded up after issuing + * the clock_set_request. + * So explicitly round up the emc target here to achieve + * the same outcome. */ + emc_freq_rounded = + tegra_emc_round_rate_updown(emc_target, true); + break; + + case TEGRA210: + emc_freq_lower = (unsigned long) + tegra_emc_round_rate_updown(emc_target, false); + emc_freq_upper = (unsigned long) + tegra_emc_round_rate_updown(emc_target, true); + + /* round to the nearest frequency step */ + if (emc_target < (emc_freq_lower + emc_freq_upper) / 2) + emc_freq_rounded = emc_freq_lower; + else + emc_freq_rounded = emc_freq_upper; + break; + + default: + /* a proper rounding function needs to be implemented + * for emc in t18x */ + emc_freq_rounded = clk_round_rate(emc_clk, emc_target); + break; + } + + /* only change the emc clock if new rounded frequency is different + * from previously set emc rate */ + if (emc_freq_rounded != emc_params->freq_last_set) { + clk_set_rate(emc_clk, emc_freq_rounded); + emc_params->freq_last_set = emc_freq_rounded; + } +} + +/* + * gk20a_tegra_prescale(profile, freq) + * + * This function informs EDP about changed constraints. + */ + +static void gk20a_tegra_prescale(struct device *dev) +{ + struct gk20a *g = get_gk20a(dev); + u32 avg = 0; + + gk20a_pmu_load_norm(g, &avg); + tegra_edp_notify_gpu_load(avg, clk_get_rate(g->clk.tegra_clk)); +} + +/* + * gk20a_tegra_calibrate_emc() + * + */ + +static void gk20a_tegra_calibrate_emc(struct device *dev, + struct gk20a_emc_params *emc_params) +{ + enum tegra_chipid cid = tegra_get_chip_id(); + long gpu_bw, emc_bw; + + /* store gpu bw based on soc */ + switch (cid) { + case TEGRA210: + gpu_bw = TEGRA_GM20B_BW_PER_FREQ; + break; + case TEGRA124: + case TEGRA132: + gpu_bw = TEGRA_GK20A_BW_PER_FREQ; + break; + default: + gpu_bw = 0; + break; + } + + /* TODO detect DDR type. + * Okay for now since DDR3 and DDR4 have the same BW ratio */ + emc_bw = TEGRA_DDR3_BW_PER_FREQ; + + /* Calculate the bandwidth ratio of gpu_freq <-> emc_freq + * NOTE the ratio must come out as an integer */ + emc_params->bw_ratio = (gpu_bw / emc_bw); +} + +#ifdef CONFIG_TEGRA_BWMGR +void gm20b_bwmgr_set_rate(struct gk20a_platform *platform, bool enb) +{ + struct gk20a_scale_profile *profile = platform->g->scale_profile; + struct gk20a_emc_params *params; + unsigned long rate; + + if (!profile || !profile->private_data) + return; + + params = (struct gk20a_emc_params *)profile->private_data; + rate = (enb) ? params->freq_last_set : 0; + tegra_bwmgr_set_emc(params->bwmgr_cl, rate, TEGRA_BWMGR_SET_EMC_FLOOR); +} + +static void gm20b_tegra_postscale(struct device *dev, unsigned long freq) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + struct gk20a_scale_profile *profile = platform->g->scale_profile; + struct gk20a_emc_params *emc_params; + unsigned long emc_rate; + + if (!profile) + return; + + emc_params = profile->private_data; + emc_rate = gk20a_tegra_get_emc_rate(get_gk20a(dev), emc_params); + + if (emc_rate > tegra_bwmgr_get_max_emc_rate()) + emc_rate = tegra_bwmgr_get_max_emc_rate(); + + emc_params->freq_last_set = emc_rate; + nvgpu_mutex_acquire(&platform->railgate_lock); + if (platform->is_railgated && !platform->is_railgated(dev)) + goto done; + + tegra_bwmgr_set_emc(emc_params->bwmgr_cl, emc_rate, + TEGRA_BWMGR_SET_EMC_FLOOR); + +done: + nvgpu_mutex_release(&platform->railgate_lock); +} + +#endif + +#ifdef CONFIG_TEGRA_CLK_FRAMEWORK +/* + * gk20a_tegra_railgate() + * + * Gate (disable) gk20a power rail + */ + +static int gk20a_tegra_railgate(struct device *dev) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + int ret = 0; + + if (platform->is_fmodel || + !tegra_dvfs_is_rail_up(platform->gpu_rail)) + return 0; + + tegra_mc_flush(MC_CLIENT_GPU); + + udelay(10); + + /* enable clamp */ + pmc_write(0x1, PMC_GPU_RG_CNTRL_0); + pmc_read(PMC_GPU_RG_CNTRL_0); + + udelay(10); + + platform->reset_assert(dev); + + udelay(10); + + /* + * GPCPLL is already disabled before entering this function; reference + * clocks are enabled until now - disable them just before rail gating + */ + clk_disable(platform->clk[0]); + clk_disable(platform->clk[1]); + + udelay(10); + + if (tegra_dvfs_is_rail_up(platform->gpu_rail)) { + ret = tegra_dvfs_rail_power_down(platform->gpu_rail); + if (ret) + goto err_power_off; + } else + pr_info("No GPU regulator?\n"); + + return 0; + +err_power_off: + gk20a_err(dev, "Could not railgate GPU"); + return ret; +} + + +/* + * gk20a_tegra_unrailgate() + * + * Ungate (enable) gk20a power rail + */ + +static int gk20a_tegra_unrailgate(struct device *dev) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + int ret = 0; + bool first = false; + + if (platform->is_fmodel) + return 0; + + if (!platform->gpu_rail) { + platform->gpu_rail = tegra_dvfs_get_rail_by_name("vdd_gpu"); + if (IS_ERR_OR_NULL(platform->gpu_rail)) { + WARN(1, "No GPU regulator?\n"); + return -EINVAL; + } + first = true; + } + + ret = tegra_dvfs_rail_power_up(platform->gpu_rail); + if (ret) + return ret; + + if (!first) { + ret = clk_enable(platform->clk[0]); + if (ret) { + gk20a_err(dev, "could not turn on gpu pll"); + goto err_clk_on; + } + ret = clk_enable(platform->clk[1]); + if (ret) { + gk20a_err(dev, "could not turn on pwr clock"); + goto err_clk_on; + } + } + + udelay(10); + + platform->reset_assert(dev); + + udelay(10); + + pmc_write(0, PMC_GPU_RG_CNTRL_0); + pmc_read(PMC_GPU_RG_CNTRL_0); + + udelay(10); + + platform->reset_deassert(dev); + + /* Flush MC after boot/railgate/SC7 */ + tegra_mc_flush(MC_CLIENT_GPU); + + udelay(10); + + tegra_mc_flush_done(MC_CLIENT_GPU); + + udelay(10); + + return 0; + +err_clk_on: + tegra_dvfs_rail_power_down(platform->gpu_rail); + + return ret; +} + +#endif + + +#if defined(CONFIG_TEGRA_CLK_FRAMEWORK) || defined(CONFIG_TEGRA_DVFS) +/* + * gk20a_tegra_is_railgated() + * + * Check status of gk20a power rail + */ + +static bool gk20a_tegra_is_railgated(struct device *dev) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + bool ret = false; + + if (!platform->is_fmodel) + ret = !tegra_dvfs_is_rail_up(platform->gpu_rail); + + return ret; +} + +/* + * gm20b_tegra_railgate() + * + * Gate (disable) gm20b power rail + */ + +static int gm20b_tegra_railgate(struct device *dev) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + int ret = 0; + + if (platform->is_fmodel || + !tegra_dvfs_is_rail_up(platform->gpu_rail)) + return 0; + + tegra_mc_flush(MC_CLIENT_GPU); + + udelay(10); + + /* enable clamp */ + pmc_write(0x1, PMC_GPU_RG_CNTRL_0); + pmc_read(PMC_GPU_RG_CNTRL_0); + + udelay(10); + + platform->reset_assert(dev); + + udelay(10); + + /* + * GPCPLL is already disabled before entering this function; reference + * clocks are enabled until now - disable them just before rail gating + */ + clk_disable_unprepare(platform->clk_reset); + clk_disable_unprepare(platform->clk[0]); + clk_disable_unprepare(platform->clk[1]); + if (platform->clk[3]) + clk_disable_unprepare(platform->clk[3]); + + udelay(10); + + tegra_soctherm_gpu_tsens_invalidate(1); + + if (tegra_dvfs_is_rail_up(platform->gpu_rail)) { + ret = tegra_dvfs_rail_power_down(platform->gpu_rail); + if (ret) + goto err_power_off; + } else + pr_info("No GPU regulator?\n"); + +#ifdef CONFIG_TEGRA_BWMGR + gm20b_bwmgr_set_rate(platform, false); +#endif + + return 0; + +err_power_off: + gk20a_err(dev, "Could not railgate GPU"); + return ret; +} + + +/* + * gm20b_tegra_unrailgate() + * + * Ungate (enable) gm20b power rail + */ + +static int gm20b_tegra_unrailgate(struct device *dev) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + int ret = 0; + bool first = false; + + if (platform->is_fmodel) + return 0; + +#ifdef CONFIG_TEGRA_CLK_FRAMEWORK + if (!platform->gpu_rail) { + platform->gpu_rail = tegra_dvfs_get_rail_by_name(GPU_RAIL_NAME); + if (IS_ERR_OR_NULL(platform->gpu_rail)) { + WARN(1, "No GPU regulator?\n"); + return -EINVAL; + } + first = true; + } +#endif + + ret = tegra_dvfs_rail_power_up(platform->gpu_rail); + if (ret) + return ret; + +#ifdef CONFIG_TEGRA_BWMGR + gm20b_bwmgr_set_rate(platform, true); +#endif + + tegra_soctherm_gpu_tsens_invalidate(0); + + if (!platform->clk_reset) { + platform->clk_reset = clk_get(dev, "gpu_gate"); + if (IS_ERR(platform->clk_reset)) { + gk20a_err(dev, "fail to get gpu reset clk\n"); + goto err_clk_on; + } + } + + if (!first) { + ret = clk_prepare_enable(platform->clk_reset); + if (ret) { + gk20a_err(dev, "could not turn on gpu_gate"); + goto err_clk_on; + } + + ret = clk_prepare_enable(platform->clk[0]); + if (ret) { + gk20a_err(dev, "could not turn on gpu pll"); + goto err_clk_on; + } + ret = clk_prepare_enable(platform->clk[1]); + if (ret) { + gk20a_err(dev, "could not turn on pwr clock"); + goto err_clk_on; + } + + if (platform->clk[3]) { + ret = clk_prepare_enable(platform->clk[3]); + if (ret) { + gk20a_err(dev, "could not turn on fuse clock"); + goto err_clk_on; + } + } + } + + udelay(10); + + platform->reset_assert(dev); + + udelay(10); + + pmc_write(0, PMC_GPU_RG_CNTRL_0); + pmc_read(PMC_GPU_RG_CNTRL_0); + + udelay(10); + + clk_disable(platform->clk_reset); + platform->reset_deassert(dev); + clk_enable(platform->clk_reset); + + /* Flush MC after boot/railgate/SC7 */ + tegra_mc_flush(MC_CLIENT_GPU); + + udelay(10); + + tegra_mc_flush_done(MC_CLIENT_GPU); + + udelay(10); + + return 0; + +err_clk_on: + tegra_dvfs_rail_power_down(platform->gpu_rail); + + return ret; +} +#endif + + +static struct { + char *name; + unsigned long default_rate; +} tegra_gk20a_clocks[] = { +#ifdef CONFIG_TEGRA_CLK_FRAMEWORK + {"PLLG_ref", UINT_MAX}, + {"pwr", 204000000}, + {"emc", UINT_MAX}, +#elif defined(CONFIG_COMMON_CLK) + {"gpu_ref", UINT_MAX}, + {"pll_p_out5", 204000000}, + {"emc", UINT_MAX}, + {"fuse", UINT_MAX}, +#endif +}; + + + +/* + * gk20a_tegra_get_clocks() + * + * This function finds clocks in tegra platform and populates + * the clock information to gk20a platform data. + */ + +static int gk20a_tegra_get_clocks(struct device *dev) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + char devname[16]; + unsigned int i; + int ret = 0; + + BUG_ON(GK20A_CLKS_MAX < ARRAY_SIZE(tegra_gk20a_clocks)); + + snprintf(devname, sizeof(devname), "tegra_%s", dev_name(dev)); + + platform->num_clks = 0; + for (i = 0; i < ARRAY_SIZE(tegra_gk20a_clocks); i++) { + long rate = tegra_gk20a_clocks[i].default_rate; + struct clk *c; + + c = clk_get_sys(devname, tegra_gk20a_clocks[i].name); + if (IS_ERR(c)) { + ret = PTR_ERR(c); + goto err_get_clock; + } + rate = clk_round_rate(c, rate); + clk_set_rate(c, rate); + platform->clk[i] = c; + } + platform->num_clks = i; + + return 0; + +err_get_clock: + + while (i--) + clk_put(platform->clk[i]); + return ret; +} + +static int gk20a_tegra_reset_assert(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + + if (!platform->clk_reset) + platform->clk_reset = platform->clk[0]; + + tegra_periph_reset_assert(platform->clk_reset); + + return 0; +} + +static int gk20a_tegra_reset_deassert(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + + if (!platform->clk_reset) + return -EINVAL; + + tegra_periph_reset_deassert(platform->clk_reset); + + return 0; +} + +#if defined(CONFIG_RESET_CONTROLLER) && defined(CONFIG_COMMON_CLK) +static int gm20b_tegra_reset_assert(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + + if (!platform->reset_control) { + WARN(1, "Reset control not initialized\n"); + return -ENOSYS; + } + + return reset_control_assert(platform->reset_control); +} + +static int gm20b_tegra_reset_deassert(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + + if (!platform->reset_control) { + WARN(1, "Reset control not initialized\n"); + return -ENOSYS; + } + + return reset_control_deassert(platform->reset_control); +} +#endif + +static void gk20a_tegra_scale_init(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a_scale_profile *profile = platform->g->scale_profile; + struct gk20a_emc_params *emc_params; + + if (!profile) + return; + + emc_params = kzalloc(sizeof(*emc_params), GFP_KERNEL); + if (!emc_params) + return; + + emc_params->freq_last_set = -1; + gk20a_tegra_calibrate_emc(dev, emc_params); + +#ifdef CONFIG_TEGRA_BWMGR + emc_params->bwmgr_cl = tegra_bwmgr_register(TEGRA_BWMGR_CLIENT_GPU); + if (!emc_params->bwmgr_cl) { + gk20a_dbg_info("%s Missing GPU BWMGR client\n", __func__); + return; + } +#endif + + profile->private_data = emc_params; +} + +static void gk20a_tegra_scale_exit(struct device *dev) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + struct gk20a_scale_profile *profile = platform->g->scale_profile; + struct gk20a_emc_params *emc_params; + + if (!profile) + return; + + emc_params = profile->private_data; +#ifdef CONFIG_TEGRA_BWMGR + tegra_bwmgr_unregister(emc_params->bwmgr_cl); +#endif + + kfree(profile->private_data); +} + +void gk20a_tegra_debug_dump(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a *g = platform->g; + + if (g->host1x_dev) + nvhost_debug_dump_device(g->host1x_dev); +} + +int gk20a_tegra_busy(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a *g = platform->g; + + if (g->host1x_dev) + return nvhost_module_busy_ext(g->host1x_dev); + return 0; +} + +void gk20a_tegra_idle(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a *g = platform->g; + + if (g->host1x_dev) + nvhost_module_idle_ext(g->host1x_dev); +} + +static int gk20a_tegra_probe(struct device *dev) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + struct device_node *np = dev->of_node; + const __be32 *host1x_ptr; + struct platform_device *host1x_pdev = NULL; + bool joint_xpu_rail = false; + int ret; + +#ifdef CONFIG_COMMON_CLK + /* DVFS is not guaranteed to be initialized at the time of probe on + * kernels with Common Clock Framework enabled. + */ + if (!platform->gpu_rail) { + platform->gpu_rail = tegra_dvfs_get_rail_by_name(GPU_RAIL_NAME); + if (!platform->gpu_rail) { + gk20a_dbg_info("deferring probe no gpu_rail\n"); + return -EPROBE_DEFER; + } + } + + if (!tegra_dvfs_is_rail_ready(platform->gpu_rail)) { + gk20a_dbg_info("deferring probe gpu_rail not ready\n"); + return -EPROBE_DEFER; + } +#endif + + host1x_ptr = of_get_property(np, "nvidia,host1x", NULL); + if (host1x_ptr) { + struct device_node *host1x_node = + of_find_node_by_phandle(be32_to_cpup(host1x_ptr)); + + host1x_pdev = of_find_device_by_node(host1x_node); + if (!host1x_pdev) { + dev_warn(dev, "host1x device not available"); + return -EPROBE_DEFER; + } + + } else { + host1x_pdev = to_platform_device(dev->parent); + dev_warn(dev, "host1x reference not found. assuming host1x to be parent"); + } + + platform->g->host1x_dev = host1x_pdev; + +#ifdef CONFIG_OF + joint_xpu_rail = of_property_read_bool(of_chosen, + "nvidia,tegra-joint_xpu_rail"); +#endif + + if (joint_xpu_rail) { + gk20a_dbg_info("XPU rails are joint\n"); + platform->can_railgate = false; + } + + /* WAR for bug 1547668: Disable railgating and scaling irrespective of + * platform data if the rework has not been made. */ + + if (tegra_get_chip_id() == TEGRA210) { + np = of_find_node_by_path("/gpu-dvfs-rework"); + if (!(np && of_device_is_available(np))) { + platform->devfreq_governor = ""; + dev_warn(dev, "board does not support scaling"); + } + } + + if (tegra_get_chip_id() == TEGRA132) + platform->soc_name = "tegra13x"; + + platform->g->mm.vidmem_is_vidmem = platform->vidmem_is_vidmem; + + gk20a_tegra_get_clocks(dev); + + if (platform->clk_register) { + ret = platform->clk_register(platform->g); + if (ret) + return ret; + } + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) + pmc = ioremap(TEGRA_PMC_BASE, 4096); +#endif + + return 0; +} + +static int gk20a_tegra_late_probe(struct device *dev) +{ + /* Initialise tegra specific scaling quirks */ + gk20a_tegra_scale_init(dev); + + return 0; +} + +static int gk20a_tegra_remove(struct device *dev) +{ + /* deinitialise tegra specific scaling quirks */ + gk20a_tegra_scale_exit(dev); + + return 0; +} + +static int gk20a_tegra_suspend(struct device *dev) +{ + tegra_edp_notify_gpu_load(0, 0); + return 0; +} + +#if defined(CONFIG_TEGRA_CLK_FRAMEWORK) || defined(CONFIG_COMMON_CLK) +static unsigned long gk20a_get_clk_rate(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a *g = platform->g; + + return gk20a_clk_get_rate(g); + +} + +static long gk20a_round_clk_rate(struct device *dev, unsigned long rate) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a *g = platform->g; + + return gk20a_clk_round_rate(g, rate); +} + +static int gk20a_set_clk_rate(struct device *dev, unsigned long rate) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a *g = platform->g; + + return gk20a_clk_set_rate(g, rate); +} + +static int gk20a_clk_get_freqs(struct device *dev, + unsigned long **freqs, int *num_freqs) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a *g = platform->g; + + /* make sure the clock is available */ + if (!gk20a_clk_get(g)) + return -ENOSYS; + + return tegra_dvfs_get_freqs(clk_get_parent(g->clk.tegra_clk), + freqs, num_freqs); +} +#endif + + +struct gk20a_platform gk20a_tegra_platform = { + .has_syncpoints = true, + .aggressive_sync_destroy_thresh = 64, + + /* power management configuration */ + .railgate_delay = 500, + .can_railgate = true, + .can_elpg = true, + .enable_slcg = true, + .enable_blcg = true, + .enable_elcg = true, + .enable_elpg = true, + .enable_aelpg = true, + .ptimer_src_freq = 12000000, + + .force_reset_in_do_idle = false, + + .default_big_page_size = SZ_128K, + + .ch_wdt_timeout_ms = 7000, + + .probe = gk20a_tegra_probe, + .late_probe = gk20a_tegra_late_probe, + .remove = gk20a_tegra_remove, + + /* power management callbacks */ + .suspend = gk20a_tegra_suspend, +#ifdef CONFIG_TEGRA_CLK_FRAMEWORK + .railgate = gk20a_tegra_railgate, + .unrailgate = gk20a_tegra_unrailgate, + .is_railgated = gk20a_tegra_is_railgated, +#endif + + .busy = gk20a_tegra_busy, + .idle = gk20a_tegra_idle, + + .reset_assert = gk20a_tegra_reset_assert, + .reset_deassert = gk20a_tegra_reset_deassert, + +#ifdef CONFIG_TEGRA_CLK_FRAMEWORK + .clk_get_rate = gk20a_get_clk_rate, + .clk_round_rate = gk20a_round_clk_rate, + .clk_set_rate = gk20a_set_clk_rate, + .get_clk_freqs = gk20a_clk_get_freqs, +#endif + + /* frequency scaling configuration */ + .prescale = gk20a_tegra_prescale, + .postscale = gk20a_tegra_postscale, + .devfreq_governor = "nvhost_podgov", + .qos_notify = gk20a_scale_qos_notify, + + .secure_alloc = gk20a_tegra_secure_alloc, + .secure_page_alloc = gk20a_tegra_secure_page_alloc, + .dump_platform_dependencies = gk20a_tegra_debug_dump, + + .has_ce = true, + + .soc_name = "tegra12x", + + .vidmem_is_vidmem = false, +}; + +struct gk20a_platform gm20b_tegra_platform = { + .has_syncpoints = true, + .aggressive_sync_destroy_thresh = 64, + + /* power management configuration */ + .railgate_delay = 500, + .can_railgate = true, + .can_elpg = true, + .enable_slcg = true, + .enable_blcg = true, + .enable_elcg = true, + .enable_elpg = true, + .enable_aelpg = true, + .ptimer_src_freq = 19200000, + + .force_reset_in_do_idle = false, + + .default_big_page_size = SZ_128K, + + .ch_wdt_timeout_ms = 5000, + + .probe = gk20a_tegra_probe, + .late_probe = gk20a_tegra_late_probe, + .remove = gk20a_tegra_remove, + /* power management callbacks */ + .suspend = gk20a_tegra_suspend, + +#if defined(CONFIG_TEGRA_CLK_FRAMEWORK) || defined(CONFIG_TEGRA_DVFS) + .railgate = gm20b_tegra_railgate, + .unrailgate = gm20b_tegra_unrailgate, + .is_railgated = gk20a_tegra_is_railgated, +#endif + + .busy = gk20a_tegra_busy, + .idle = gk20a_tegra_idle, + +#if defined(CONFIG_RESET_CONTROLLER) && defined(CONFIG_COMMON_CLK) + .reset_assert = gm20b_tegra_reset_assert, + .reset_deassert = gm20b_tegra_reset_deassert, +#else + .reset_assert = gk20a_tegra_reset_assert, + .reset_deassert = gk20a_tegra_reset_deassert, +#endif + +#if defined(CONFIG_TEGRA_CLK_FRAMEWORK) || defined(CONFIG_COMMON_CLK) + .clk_get_rate = gk20a_get_clk_rate, + .clk_round_rate = gk20a_round_clk_rate, + .clk_set_rate = gk20a_set_clk_rate, + .get_clk_freqs = gk20a_clk_get_freqs, +#endif + +#ifdef CONFIG_COMMON_CLK + .clk_register = gm20b_register_gpcclk, +#endif + + /* frequency scaling configuration */ + .prescale = gk20a_tegra_prescale, +#ifdef CONFIG_TEGRA_BWMGR + .postscale = gm20b_tegra_postscale, +#else + .postscale = gk20a_tegra_postscale, +#endif + .devfreq_governor = "nvhost_podgov", + .qos_notify = gk20a_scale_qos_notify, + + .secure_alloc = gk20a_tegra_secure_alloc, + .secure_page_alloc = gk20a_tegra_secure_page_alloc, + .dump_platform_dependencies = gk20a_tegra_debug_dump, + + .has_cde = true, + + .has_ce = true, + + .soc_name = "tegra21x", + + .vidmem_is_vidmem = false, +}; diff --git a/drivers/gpu/nvgpu/tegra/linux/platform_gp10b_tegra.c b/drivers/gpu/nvgpu/tegra/linux/platform_gp10b_tegra.c new file mode 100644 index 00000000..f0c3640a --- /dev/null +++ b/drivers/gpu/nvgpu/tegra/linux/platform_gp10b_tegra.c @@ -0,0 +1,791 @@ +/* + * GP10B Tegra Platform Interface + * + * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#include "gk20a/platform_gk20a.h" +#include "gk20a/gk20a.h" +#include "gk20a/gk20a_scale.h" + +#include "platform_tegra.h" +#include "gp10b/gp10b_sysfs.h" + +#include +#include + +/* Select every GP10B_FREQ_SELECT_STEP'th frequency from h/w table */ +#define GP10B_FREQ_SELECT_STEP 8 +/* Max number of freq supported in h/w */ +#define GP10B_MAX_SUPPORTED_FREQS 120 +static unsigned long +gp10b_freq_table[GP10B_MAX_SUPPORTED_FREQS / GP10B_FREQ_SELECT_STEP]; + +#define TEGRA_GP10B_BW_PER_FREQ 64 +#define TEGRA_DDR4_BW_PER_FREQ 16 + +#define EMC_BW_RATIO (TEGRA_GP10B_BW_PER_FREQ / TEGRA_DDR4_BW_PER_FREQ) + +static struct { + char *name; + unsigned long default_rate; +} tegra_gp10b_clocks[] = { + {"gpu", 1000000000}, + {"gpu_sys", 204000000} }; + +static void gr_gp10b_remove_sysfs(struct device *dev); + +/* + * gp10b_tegra_get_clocks() + * + * This function finds clocks in tegra platform and populates + * the clock information to gp10b platform data. + */ + +int gp10b_tegra_get_clocks(struct device *dev) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + unsigned int i; + + if (platform->is_fmodel) + return 0; + + platform->num_clks = 0; + for (i = 0; i < ARRAY_SIZE(tegra_gp10b_clocks); i++) { + long rate = tegra_gp10b_clocks[i].default_rate; + struct clk *c; + + c = clk_get(dev, tegra_gp10b_clocks[i].name); + if (IS_ERR(c)) { + gk20a_err(dev, "cannot get clock %s", + tegra_gp10b_clocks[i].name); + } else { + clk_set_rate(c, rate); + platform->clk[i] = c; + } + } + platform->num_clks = i; + + return 0; +} + +static void gp10b_tegra_scale_init(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a_scale_profile *profile = platform->g->scale_profile; + struct tegra_bwmgr_client *bwmgr_handle; + + if (!profile) + return; + + bwmgr_handle = tegra_bwmgr_register(TEGRA_BWMGR_CLIENT_GPU); + if (!bwmgr_handle) + return; + + profile->private_data = (void *)bwmgr_handle; +} + +static void gp10b_tegra_scale_exit(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a_scale_profile *profile = platform->g->scale_profile; + + if (profile) + tegra_bwmgr_unregister( + (struct tegra_bwmgr_client *)profile->private_data); +} + +static int gp10b_tegra_probe(struct device *dev) +{ + struct gk20a_platform *platform = dev_get_drvdata(dev); + struct device_node *np = dev->of_node; + struct device_node *host1x_node; + struct platform_device *host1x_pdev; + const __be32 *host1x_ptr; + + host1x_ptr = of_get_property(np, "nvidia,host1x", NULL); + if (!host1x_ptr) { + gk20a_err(dev, "host1x device not available"); + return -ENOSYS; + } + + host1x_node = of_find_node_by_phandle(be32_to_cpup(host1x_ptr)); + host1x_pdev = of_find_device_by_node(host1x_node); + if (!host1x_pdev) { + gk20a_err(dev, "host1x device not available"); + return -ENOSYS; + } + + platform->g->host1x_dev = host1x_pdev; + platform->bypass_smmu = !device_is_iommuable(dev); + platform->disable_bigpage = platform->bypass_smmu; + + platform->g->gr.t18x.ctx_vars.dump_ctxsw_stats_on_channel_close + = false; + platform->g->gr.t18x.ctx_vars.dump_ctxsw_stats_on_channel_close + = false; + + platform->g->gr.t18x.ctx_vars.force_preemption_gfxp = false; + platform->g->gr.t18x.ctx_vars.force_preemption_cilp = false; + + platform->g->gr.t18x.ctx_vars.debugfs_force_preemption_gfxp = + debugfs_create_bool("force_preemption_gfxp", S_IRUGO|S_IWUSR, + platform->debugfs, + &platform->g->gr.t18x.ctx_vars.force_preemption_gfxp); + + platform->g->gr.t18x.ctx_vars.debugfs_force_preemption_cilp = + debugfs_create_bool("force_preemption_cilp", S_IRUGO|S_IWUSR, + platform->debugfs, + &platform->g->gr.t18x.ctx_vars.force_preemption_cilp); + + platform->g->gr.t18x.ctx_vars.debugfs_dump_ctxsw_stats = + debugfs_create_bool("dump_ctxsw_stats_on_channel_close", + S_IRUGO|S_IWUSR, + platform->debugfs, + &platform->g->gr.t18x. + ctx_vars.dump_ctxsw_stats_on_channel_close); + + platform->g->mm.vidmem_is_vidmem = platform->vidmem_is_vidmem; + + gp10b_tegra_get_clocks(dev); + + return 0; +} + +static int gp10b_tegra_late_probe(struct device *dev) +{ + /*Create GP10B specific sysfs*/ + gp10b_create_sysfs(dev); + + /* Initialise tegra specific scaling quirks */ + gp10b_tegra_scale_init(dev); + return 0; +} + +static int gp10b_tegra_remove(struct device *dev) +{ + gr_gp10b_remove_sysfs(dev); + /*Remove GP10B specific sysfs*/ + gp10b_remove_sysfs(dev); + + /* deinitialise tegra specific scaling quirks */ + gp10b_tegra_scale_exit(dev); + + return 0; + +} + +static bool gp10b_tegra_is_railgated(struct device *dev) +{ + bool ret = false; + + if (tegra_bpmp_running()) + ret = !tegra_powergate_is_powered(TEGRA186_POWER_DOMAIN_GPU); + + return ret; +} + +static int gp10b_tegra_railgate(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a_scale_profile *profile = platform->g->scale_profile; + + /* remove emc frequency floor */ + if (profile) + tegra_bwmgr_set_emc( + (struct tegra_bwmgr_client *)profile->private_data, + 0, TEGRA_BWMGR_SET_EMC_FLOOR); + + if (tegra_bpmp_running() && + tegra_powergate_is_powered(TEGRA186_POWER_DOMAIN_GPU)) { + int i; + for (i = 0; i < platform->num_clks; i++) { + if (platform->clk[i]) + clk_disable_unprepare(platform->clk[i]); + } + tegra_powergate_partition(TEGRA186_POWER_DOMAIN_GPU); + } + return 0; +} + +static int gp10b_tegra_unrailgate(struct device *dev) +{ + int ret = 0; + struct gk20a_platform *platform = gk20a_get_platform(dev); + struct gk20a_scale_profile *profile = platform->g->scale_profile; + + if (tegra_bpmp_running()) { + int i; + ret = tegra_unpowergate_partition(TEGRA186_POWER_DOMAIN_GPU); + for (i = 0; i < platform->num_clks; i++) { + if (platform->clk[i]) + clk_prepare_enable(platform->clk[i]); + } + } + + /* to start with set emc frequency floor to max rate*/ + if (profile) + tegra_bwmgr_set_emc( + (struct tegra_bwmgr_client *)profile->private_data, + tegra_bwmgr_get_max_emc_rate(), + TEGRA_BWMGR_SET_EMC_FLOOR); + return ret; +} + +static int gp10b_tegra_suspend(struct device *dev) +{ + return 0; +} + +int gp10b_tegra_reset_assert(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + int ret = 0; + + if (!platform->reset_control) + return -EINVAL; + + ret = reset_control_assert(platform->reset_control); + + return ret; +} + +int gp10b_tegra_reset_deassert(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + int ret = 0; + + if (!platform->reset_control) + return -EINVAL; + + ret = reset_control_deassert(platform->reset_control); + + return ret; +} + +static void gp10b_tegra_prescale(struct device *dev) +{ + struct gk20a *g = get_gk20a(dev); + u32 avg = 0; + + gk20a_dbg_fn(""); + + gk20a_pmu_load_norm(g, &avg); + + gk20a_dbg_fn("done"); +} + +static void gp10b_tegra_postscale(struct device *pdev, + unsigned long freq) +{ + struct gk20a_platform *platform = gk20a_get_platform(pdev); + struct gk20a_scale_profile *profile = platform->g->scale_profile; + struct gk20a *g = get_gk20a(pdev); + unsigned long emc_rate; + + gk20a_dbg_fn(""); + if (profile && !gp10b_tegra_is_railgated(pdev)) { + unsigned long emc_scale; + + if (freq <= gp10b_freq_table[0]) + emc_scale = 0; + else + emc_scale = g->emc3d_ratio; + + emc_rate = (freq * EMC_BW_RATIO * emc_scale) / 1000; + + if (emc_rate > tegra_bwmgr_get_max_emc_rate()) + emc_rate = tegra_bwmgr_get_max_emc_rate(); + + tegra_bwmgr_set_emc( + (struct tegra_bwmgr_client *)profile->private_data, + emc_rate, TEGRA_BWMGR_SET_EMC_FLOOR); + } + gk20a_dbg_fn("done"); +} + +static unsigned long gp10b_get_clk_rate(struct device *dev) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + + return clk_get_rate(platform->clk[0]); + +} + +static long gp10b_round_clk_rate(struct device *dev, unsigned long rate) +{ + struct gk20a *g = get_gk20a(dev); + struct gk20a_scale_profile *profile = g->scale_profile; + unsigned long *freq_table = profile->devfreq_profile.freq_table; + int max_states = profile->devfreq_profile.max_state; + int i; + + for (i = 0; i < max_states; ++i) + if (freq_table[i] >= rate) + return freq_table[i]; + + return freq_table[max_states - 1]; +} + +static int gp10b_set_clk_rate(struct device *dev, unsigned long rate) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + + return clk_set_rate(platform->clk[0], rate); +} + +static int gp10b_clk_get_freqs(struct device *dev, + unsigned long **freqs, int *num_freqs) +{ + struct gk20a_platform *platform = gk20a_get_platform(dev); + unsigned long max_rate; + unsigned long new_rate = 0, prev_rate = 0; + int i = 0, freq_counter = 0; + + max_rate = clk_round_rate(platform->clk[0], (UINT_MAX - 1)); + + /* + * Walk the h/w frequency table and only select + * GP10B_FREQ_SELECT_STEP'th frequencies and + * add MAX freq to last + */ + for (; i < GP10B_MAX_SUPPORTED_FREQS; ++i) { + prev_rate = new_rate; + new_rate = clk_round_rate(platform->clk[0], prev_rate + 1); + + if (i % GP10B_FREQ_SELECT_STEP == 0 || + new_rate == max_rate) { + gp10b_freq_table[freq_counter++] = new_rate; + + if (new_rate == max_rate) + break; + } + } + + WARN_ON(i == GP10B_MAX_SUPPORTED_FREQS); + + /* Fill freq table */ + *freqs = gp10b_freq_table; + *num_freqs = freq_counter; + + gk20a_dbg_info("min rate: %ld max rate: %ld num_of_freq %d\n", + gp10b_freq_table[0], max_rate, *num_freqs); + + return 0; +} + +struct gk20a_platform gp10b_tegra_platform = { + .has_syncpoints = true, + + /* power management configuration */ + .railgate_delay = 500, + + /* power management configuration */ + .can_railgate = true, + .enable_elpg = true, + .can_elpg = true, + .enable_blcg = true, + .enable_slcg = true, + .enable_elcg = true, + .enable_aelpg = true, + + /* ptimer src frequency in hz*/ + .ptimer_src_freq = 31250000, + + .ch_wdt_timeout_ms = 5000, + + .probe = gp10b_tegra_probe, + .late_probe = gp10b_tegra_late_probe, + .remove = gp10b_tegra_remove, + + /* power management callbacks */ + .suspend = gp10b_tegra_suspend, + .railgate = gp10b_tegra_railgate, + .unrailgate = gp10b_tegra_unrailgate, + .is_railgated = gp10b_tegra_is_railgated, + + .busy = gk20a_tegra_busy, + .idle = gk20a_tegra_idle, + + .dump_platform_dependencies = gk20a_tegra_debug_dump, + + .default_big_page_size = SZ_64K, + + .has_cde = true, + + .has_ce = true, + + .clk_get_rate = gp10b_get_clk_rate, + .clk_round_rate = gp10b_round_clk_rate, + .clk_set_rate = gp10b_set_clk_rate, + .get_clk_freqs = gp10b_clk_get_freqs, + + /* frequency scaling configuration */ + .prescale = gp10b_tegra_prescale, + .postscale = gp10b_tegra_postscale, + .devfreq_governor = "nvhost_podgov", + + .qos_notify = gk20a_scale_qos_notify, + + .secure_alloc = gk20a_tegra_secure_alloc, + .secure_page_alloc = gk20a_tegra_secure_page_alloc, + + .reset_assert = gp10b_tegra_reset_assert, + .reset_deassert = gp10b_tegra_reset_deassert, + + .force_reset_in_do_idle = false, + + .soc_name = "tegra18x", + + .vidmem_is_vidmem = false, +}; + + +#define ECC_STAT_NAME_MAX_SIZE 100 + + +static DEFINE_HASHTABLE(ecc_hash_table, 5); + +static struct device_attribute *dev_attr_sm_lrf_ecc_single_err_count_array; +static struct device_attribute *dev_attr_sm_lrf_ecc_double_err_count_array; + +static struct device_attribute *dev_attr_sm_shm_ecc_sec_count_array; +static struct device_attribute *dev_attr_sm_shm_ecc_sed_count_array; +static struct device_attribute *dev_attr_sm_shm_ecc_ded_count_array; + +static struct device_attribute *dev_attr_tex_ecc_total_sec_pipe0_count_array; +static struct device_attribute *dev_attr_tex_ecc_total_ded_pipe0_count_array; +static struct device_attribute *dev_attr_tex_ecc_unique_sec_pipe0_count_array; +static struct device_attribute *dev_attr_tex_ecc_unique_ded_pipe0_count_array; +static struct device_attribute *dev_attr_tex_ecc_total_sec_pipe1_count_array; +static struct device_attribute *dev_attr_tex_ecc_total_ded_pipe1_count_array; +static struct device_attribute *dev_attr_tex_ecc_unique_sec_pipe1_count_array; +static struct device_attribute *dev_attr_tex_ecc_unique_ded_pipe1_count_array; + +static struct device_attribute *dev_attr_l2_ecc_sec_count_array; +static struct device_attribute *dev_attr_l2_ecc_ded_count_array; + + +static u32 gen_ecc_hash_key(char *str) +{ + int i = 0; + u32 hash_key = 0; + + while (str[i]) { + hash_key += (u32)(str[i]); + i++; + }; + + return hash_key; +} + +static ssize_t ecc_stat_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + const char *ecc_stat_full_name = attr->attr.name; + const char *ecc_stat_base_name; + unsigned int hw_unit; + struct ecc_stat *ecc_stat; + u32 hash_key; + + if (sscanf(ecc_stat_full_name, "ltc%u", &hw_unit) == 1) { + ecc_stat_base_name = &(ecc_stat_full_name[strlen("ltc0_")]); + } else if (sscanf(ecc_stat_full_name, "gpc0_tpc%u", &hw_unit) == 1) { + ecc_stat_base_name = &(ecc_stat_full_name[strlen("gpc0_tpc0_")]); + } else { + return snprintf(buf, + PAGE_SIZE, + "Error: Invalid ECC stat name!\n"); + } + + hash_key = gen_ecc_hash_key((char *)ecc_stat_base_name); + hash_for_each_possible(ecc_hash_table, + ecc_stat, + hash_node, + hash_key) { + if (!strcmp(ecc_stat_full_name, ecc_stat->names[hw_unit])) + return snprintf(buf, PAGE_SIZE, "%u\n", ecc_stat->counters[hw_unit]); + } + + return snprintf(buf, PAGE_SIZE, "Error: No ECC stat found!\n"); +} + +static int ecc_stat_create(struct device *dev, + int is_l2, + char *ecc_stat_name, + struct ecc_stat *ecc_stat, + struct device_attribute *dev_attr_array) +{ + int error = 0; + struct gk20a *g = get_gk20a(dev); + int num_hw_units = 0; + int hw_unit = 0; + u32 hash_key = 0; + + if (is_l2) + num_hw_units = g->ltc_count; + else + num_hw_units = g->gr.tpc_count; + + /* Allocate arrays */ + dev_attr_array = kzalloc(sizeof(struct device_attribute) * num_hw_units, GFP_KERNEL); + ecc_stat->counters = kzalloc(sizeof(u32) * num_hw_units, GFP_KERNEL); + ecc_stat->names = kzalloc(sizeof(char *) * num_hw_units, GFP_KERNEL); + for (hw_unit = 0; hw_unit < num_hw_units; hw_unit++) { + ecc_stat->names[hw_unit] = kzalloc(sizeof(char) * ECC_STAT_NAME_MAX_SIZE, GFP_KERNEL); + } + + for (hw_unit = 0; hw_unit < num_hw_units; hw_unit++) { + /* Fill in struct device_attribute members */ + if (is_l2) + snprintf(ecc_stat->names[hw_unit], + ECC_STAT_NAME_MAX_SIZE, + "ltc%d_%s", + hw_unit, + ecc_stat_name); + else + snprintf(ecc_stat->names[hw_unit], + ECC_STAT_NAME_MAX_SIZE, + "gpc0_tpc%d_%s", + hw_unit, + ecc_stat_name); + + sysfs_attr_init(&dev_attr_array[hw_unit].attr); + dev_attr_array[hw_unit].attr.name = ecc_stat->names[hw_unit]; + dev_attr_array[hw_unit].attr.mode = VERIFY_OCTAL_PERMISSIONS(S_IRUGO); + dev_attr_array[hw_unit].show = ecc_stat_show; + dev_attr_array[hw_unit].store = NULL; + + /* Create sysfs file */ + error |= device_create_file(dev, &dev_attr_array[hw_unit]); + } + + /* Add hash table entry */ + hash_key = gen_ecc_hash_key(ecc_stat_name); + hash_add(ecc_hash_table, + &ecc_stat->hash_node, + hash_key); + + return error; +} + +static void ecc_stat_remove(struct device *dev, + int is_l2, + struct ecc_stat *ecc_stat, + struct device_attribute *dev_attr_array) +{ + struct gk20a *g = get_gk20a(dev); + int num_hw_units = 0; + int hw_unit = 0; + + if (is_l2) + num_hw_units = g->ltc_count; + else + num_hw_units = g->gr.tpc_count; + + /* Remove sysfs files */ + for (hw_unit = 0; hw_unit < num_hw_units; hw_unit++) { + device_remove_file(dev, &dev_attr_array[hw_unit]); + } + + /* Remove hash table entry */ + hash_del(&ecc_stat->hash_node); + + /* Free arrays */ + kfree(ecc_stat->counters); + for (hw_unit = 0; hw_unit < num_hw_units; hw_unit++) { + kfree(ecc_stat->names[hw_unit]); + } + kfree(ecc_stat->names); + kfree(dev_attr_array); +} + +void gr_gp10b_create_sysfs(struct device *dev) +{ + int error = 0; + struct gk20a *g = get_gk20a(dev); + + /* This stat creation function is called on GR init. GR can get + initialized multiple times but we only need to create the ECC + stats once. Therefore, add the following check to avoid + creating duplicate stat sysfs nodes. */ + if (g->gr.t18x.ecc_stats.sm_lrf_single_err_count.counters != NULL) + return; + + error |= ecc_stat_create(dev, + 0, + "sm_lrf_ecc_single_err_count", + &g->gr.t18x.ecc_stats.sm_lrf_single_err_count, + dev_attr_sm_lrf_ecc_single_err_count_array); + error |= ecc_stat_create(dev, + 0, + "sm_lrf_ecc_double_err_count", + &g->gr.t18x.ecc_stats.sm_lrf_double_err_count, + dev_attr_sm_lrf_ecc_double_err_count_array); + + error |= ecc_stat_create(dev, + 0, + "sm_shm_ecc_sec_count", + &g->gr.t18x.ecc_stats.sm_shm_sec_count, + dev_attr_sm_shm_ecc_sec_count_array); + error |= ecc_stat_create(dev, + 0, + "sm_shm_ecc_sed_count", + &g->gr.t18x.ecc_stats.sm_shm_sed_count, + dev_attr_sm_shm_ecc_sed_count_array); + error |= ecc_stat_create(dev, + 0, + "sm_shm_ecc_ded_count", + &g->gr.t18x.ecc_stats.sm_shm_ded_count, + dev_attr_sm_shm_ecc_ded_count_array); + + error |= ecc_stat_create(dev, + 0, + "tex_ecc_total_sec_pipe0_count", + &g->gr.t18x.ecc_stats.tex_total_sec_pipe0_count, + dev_attr_tex_ecc_total_sec_pipe0_count_array); + error |= ecc_stat_create(dev, + 0, + "tex_ecc_total_ded_pipe0_count", + &g->gr.t18x.ecc_stats.tex_total_ded_pipe0_count, + dev_attr_tex_ecc_total_ded_pipe0_count_array); + error |= ecc_stat_create(dev, + 0, + "tex_ecc_unique_sec_pipe0_count", + &g->gr.t18x.ecc_stats.tex_unique_sec_pipe0_count, + dev_attr_tex_ecc_unique_sec_pipe0_count_array); + error |= ecc_stat_create(dev, + 0, + "tex_ecc_unique_ded_pipe0_count", + &g->gr.t18x.ecc_stats.tex_unique_ded_pipe0_count, + dev_attr_tex_ecc_unique_ded_pipe0_count_array); + error |= ecc_stat_create(dev, + 0, + "tex_ecc_total_sec_pipe1_count", + &g->gr.t18x.ecc_stats.tex_total_sec_pipe1_count, + dev_attr_tex_ecc_total_sec_pipe1_count_array); + error |= ecc_stat_create(dev, + 0, + "tex_ecc_total_ded_pipe1_count", + &g->gr.t18x.ecc_stats.tex_total_ded_pipe1_count, + dev_attr_tex_ecc_total_ded_pipe1_count_array); + error |= ecc_stat_create(dev, + 0, + "tex_ecc_unique_sec_pipe1_count", + &g->gr.t18x.ecc_stats.tex_unique_sec_pipe1_count, + dev_attr_tex_ecc_unique_sec_pipe1_count_array); + error |= ecc_stat_create(dev, + 0, + "tex_ecc_unique_ded_pipe1_count", + &g->gr.t18x.ecc_stats.tex_unique_ded_pipe1_count, + dev_attr_tex_ecc_unique_ded_pipe1_count_array); + + error |= ecc_stat_create(dev, + 1, + "lts0_ecc_sec_count", + &g->gr.t18x.ecc_stats.l2_sec_count, + dev_attr_l2_ecc_sec_count_array); + error |= ecc_stat_create(dev, + 1, + "lts0_ecc_ded_count", + &g->gr.t18x.ecc_stats.l2_ded_count, + dev_attr_l2_ecc_ded_count_array); + + if (error) + dev_err(dev, "Failed to create sysfs attributes!\n"); +} + +static void gr_gp10b_remove_sysfs(struct device *dev) +{ + struct gk20a *g = get_gk20a(dev); + + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.sm_lrf_single_err_count, + dev_attr_sm_lrf_ecc_single_err_count_array); + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.sm_lrf_double_err_count, + dev_attr_sm_lrf_ecc_double_err_count_array); + + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.sm_shm_sec_count, + dev_attr_sm_shm_ecc_sec_count_array); + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.sm_shm_sed_count, + dev_attr_sm_shm_ecc_sed_count_array); + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.sm_shm_ded_count, + dev_attr_sm_shm_ecc_ded_count_array); + + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.tex_total_sec_pipe0_count, + dev_attr_tex_ecc_total_sec_pipe0_count_array); + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.tex_total_ded_pipe0_count, + dev_attr_tex_ecc_total_ded_pipe0_count_array); + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.tex_unique_sec_pipe0_count, + dev_attr_tex_ecc_unique_sec_pipe0_count_array); + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.tex_unique_ded_pipe0_count, + dev_attr_tex_ecc_unique_ded_pipe0_count_array); + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.tex_total_sec_pipe1_count, + dev_attr_tex_ecc_total_sec_pipe1_count_array); + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.tex_total_ded_pipe1_count, + dev_attr_tex_ecc_total_ded_pipe1_count_array); + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.tex_unique_sec_pipe1_count, + dev_attr_tex_ecc_unique_sec_pipe1_count_array); + ecc_stat_remove(dev, + 0, + &g->gr.t18x.ecc_stats.tex_unique_ded_pipe1_count, + dev_attr_tex_ecc_unique_ded_pipe1_count_array); + + ecc_stat_remove(dev, + 1, + &g->gr.t18x.ecc_stats.l2_sec_count, + dev_attr_l2_ecc_sec_count_array); + ecc_stat_remove(dev, + 1, + &g->gr.t18x.ecc_stats.l2_ded_count, + dev_attr_l2_ecc_ded_count_array); +} -- cgit v1.2.2