/*
 * Copyright (c) 2016-2017, NVIDIA CORPORATION.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>
#include <linux/uaccess.h>
#endif

#include "gk20a/gk20a.h"
#include "gm206/bios_gm206.h"
#include "gp106/xve_gp106.h"

#ifdef CONFIG_DEBUG_FS
#include "gk20a/platform_gk20a.h"
#endif

#include <nvgpu/bug.h>

#include <nvgpu/hw/gp106/hw_xp_gp106.h>
#include <nvgpu/hw/gp106/hw_xve_gp106.h>

#define NV_PCFG 0x88000

static void xve_xve_writel_gp106(struct gk20a *g, u32 reg, u32 val)
{
	gk20a_writel(g, NV_PCFG + reg, val);
}

static u32 xve_xve_readl_gp106(struct gk20a *g, u32 reg)
{
	return gk20a_readl(g, NV_PCFG + reg);
}

/**
 * Resets the GPU (except the XVE/XP).
 */
static void xve_reset_gpu_gp106(struct gk20a *g)
{
	u32 reset;

	/*
	 * This resets the GPU except for the XVE/XP (since then we would lose
	 * the dGPU from the bus). t18x has a HW limitation where once that
	 * happens the GPU is gone until the entire system is reset.
	 *
	 * We have to use the auto-deassert register since we won't be able to
	 * access the GPU after the GPU goes into reset. This appears like the
	 * GPU has dropped from the bus and causes nvgpu to reset the entire
	 * system. Whoops!
	 */
	reset = xve_reset_reset_m() |
		xve_reset_gpu_on_sw_reset_m() |
		xve_reset_counter_en_m() |
		xve_reset_counter_val_f(0x7ff) |
		xve_reset_clock_on_sw_reset_m() |
		xve_reset_clock_counter_en_m() |
		xve_reset_clock_counter_val_f(0x7ff);

	g->ops.xve.xve_writel(g, xve_reset_r(), reset | xve_reset_reset_m());

	/*
	 * Don't access GPU until _after_ it's back out of reset!
	 */
	nvgpu_msleep(100);
	g->ops.xve.xve_writel(g, xve_reset_r(), 0);
}

/**
 * Places one of:
 *
 *   %GPU_XVE_SPEED_2P5
 *   %GPU_XVE_SPEED_5P0
 *   %GPU_XVE_SPEED_8P0
 *
 * in the u32 pointed to by @xve_link_speed. If for some reason an unknown PCIe
 * bus speed is detected then *@xve_link_speed is not touched and -ENODEV is
 * returned.
 */
static int xve_get_speed_gp106(struct gk20a *g, u32 *xve_link_speed)
{
	u32 status;
	u32 link_speed, real_link_speed = 0;

	status = g->ops.xve.xve_readl(g, xve_link_control_status_r());

	link_speed = xve_link_control_status_link_speed_v(status);

	/*
	 * Can't use a switch statement becuase switch statements dont work with
	 * function calls.
	 */
	if (link_speed == xve_link_control_status_link_speed_link_speed_2p5_v())
		real_link_speed = GPU_XVE_SPEED_2P5;
	if (link_speed == xve_link_control_status_link_speed_link_speed_5p0_v())
		real_link_speed = GPU_XVE_SPEED_5P0;
	if (link_speed == xve_link_control_status_link_speed_link_speed_8p0_v())
		real_link_speed = GPU_XVE_SPEED_8P0;

	if (!real_link_speed) {
		pr_warn("%s: Unknown PCIe bus speed!\n", __func__);
		return -ENODEV;
	}

	*xve_link_speed = real_link_speed;
	return 0;
}

/**
 * Set the mask for L0s in the XVE.
 *
 * When @status is non-zero the mask for L0s is set which _disables_ L0s. When
 * @status is zero L0s is no longer masked and may be enabled.
 */
static void set_xve_l0s_mask(struct gk20a *g, bool status)
{
	u32 xve_priv;
	u32 status_bit = status ? 1 : 0;

	xve_priv = g->ops.xve.xve_readl(g, xve_priv_xv_r());

	xve_priv = set_field(xve_priv,
		  xve_priv_xv_cya_l0s_enable_m(),
		  xve_priv_xv_cya_l0s_enable_f(status_bit));

	g->ops.xve.xve_writel(g, xve_priv_xv_r(), xve_priv);
}

/**
 * Set the mask for L1 in the XVE.
 *
 * When @status is non-zero the mask for L1 is set which _disables_ L0s. When
 * @status is zero L1 is no longer masked and may be enabled.
 */
static void set_xve_l1_mask(struct gk20a *g, int status)
{
	u32 xve_priv;
	u32 status_bit = status ? 1 : 0;

	xve_priv = g->ops.xve.xve_readl(g, xve_priv_xv_r());

	xve_priv = set_field(xve_priv,
		  xve_priv_xv_cya_l1_enable_m(),
		  xve_priv_xv_cya_l1_enable_f(status_bit));

	g->ops.xve.xve_writel(g, xve_priv_xv_r(), xve_priv);
}

/**
 * Disable ASPM permanently.
 */
static void xve_disable_aspm_gp106(struct gk20a *g)
{
	u32 xve_priv;

	xve_priv = g->ops.xve.xve_readl(g, xve_priv_xv_r());

	set_xve_l0s_mask(g, true);
	set_xve_l1_mask(g, true);
}

/**
 * When doing the speed change disable power saving features.
 */
static void disable_aspm_gp106(struct gk20a *g)
{
	u32 xve_priv;

	xve_priv = g->ops.xve.xve_readl(g, xve_priv_xv_r());

	/*
	 * Store prior ASPM state so we can restore it later on.
	 */
	g->xve_l0s = xve_priv_xv_cya_l0s_enable_v(xve_priv);
	g->xve_l1  = xve_priv_xv_cya_l1_enable_v(xve_priv);

	set_xve_l0s_mask(g, true);
	set_xve_l1_mask(g, true);
}

/**
 * Restore the state saved by disable_aspm_gp106().
 */
static void enable_aspm_gp106(struct gk20a *g)
{
	set_xve_l0s_mask(g, g->xve_l0s);
	set_xve_l1_mask(g, g->xve_l1);
}

/*
 * Error checking is done in xve_set_speed_gp106.
 */
static int __do_xve_set_speed_gp106(struct gk20a *g, u32 next_link_speed)
{
	u32 current_link_speed, new_link_speed;
	u32 dl_mgr, saved_dl_mgr;
	u32 pl_link_config;
	u32 link_control_status, link_speed_setting, link_width;
	struct nvgpu_timeout timeout;
	int attempts = 10, err_status = 0;

	g->ops.xve.get_speed(g, &current_link_speed);
	xv_sc_dbg(PRE_CHANGE, "Executing PCIe link change.");
	xv_sc_dbg(PRE_CHANGE, "  Current speed:  %s",
		  xve_speed_to_str(current_link_speed));
	xv_sc_dbg(PRE_CHANGE, "  Next speed:     %s",
		  xve_speed_to_str(next_link_speed));
	xv_sc_dbg(PRE_CHANGE, "  PL_LINK_CONFIG: 0x%08x",
		  gk20a_readl(g, xp_pl_link_config_r(0)));

	xv_sc_dbg(DISABLE_ASPM, "Disabling ASPM...");
	disable_aspm_gp106(g);
	xv_sc_dbg(DISABLE_ASPM, "  Done!");

	xv_sc_dbg(DL_SAFE_MODE, "Putting DL in safe mode...");
	saved_dl_mgr = gk20a_readl(g, xp_dl_mgr_r(0));

	/*
	 * Put the DL in safe mode.
	 */
	dl_mgr = saved_dl_mgr;
	dl_mgr |= xp_dl_mgr_safe_timing_f(1);
	gk20a_writel(g, xp_dl_mgr_r(0), dl_mgr);
	xv_sc_dbg(DL_SAFE_MODE, "  Done!");

	nvgpu_timeout_init(g, &timeout, GPU_XVE_TIMEOUT_MS,
			NVGPU_TIMER_CPU_TIMER);

	xv_sc_dbg(CHECK_LINK, "Checking for link idle...");
	do {
		pl_link_config = gk20a_readl(g, xp_pl_link_config_r(0));
		if ((xp_pl_link_config_ltssm_status_f(pl_link_config) ==
		     xp_pl_link_config_ltssm_status_idle_v()) &&
		    (xp_pl_link_config_ltssm_directive_f(pl_link_config) ==
		     xp_pl_link_config_ltssm_directive_normal_operations_v()))
			break;
	} while (!nvgpu_timeout_expired(&timeout));

	if (nvgpu_timeout_peek_expired(&timeout)) {
		err_status = -ETIMEDOUT;
		goto done;
	}

	xv_sc_dbg(CHECK_LINK, "  Done");

	xv_sc_dbg(LINK_SETTINGS, "Preparing next link settings");
	pl_link_config &= ~xp_pl_link_config_max_link_rate_m();
	switch (next_link_speed) {
	case GPU_XVE_SPEED_2P5:
		link_speed_setting =
			xve_link_control_status_link_speed_link_speed_2p5_v();
		pl_link_config |= xp_pl_link_config_max_link_rate_f(
			xp_pl_link_config_max_link_rate_2500_mtps_v());
		break;
	case GPU_XVE_SPEED_5P0:
		link_speed_setting =
			xve_link_control_status_link_speed_link_speed_5p0_v();
		pl_link_config |= xp_pl_link_config_max_link_rate_f(
			xp_pl_link_config_max_link_rate_5000_mtps_v());
		break;
	case GPU_XVE_SPEED_8P0:
		link_speed_setting =
			xve_link_control_status_link_speed_link_speed_8p0_v();
		pl_link_config |= xp_pl_link_config_max_link_rate_f(
			xp_pl_link_config_max_link_rate_8000_mtps_v());
		break;
	default:
		BUG(); /* Should never be hit. */
	}

	link_control_status =
		g->ops.xve.xve_readl(g, xve_link_control_status_r());
	link_width = xve_link_control_status_link_width_v(link_control_status);

	pl_link_config &= ~xp_pl_link_config_target_tx_width_m();

	/* Can't use a switch due to oddities in register definitions. */
	if (link_width == xve_link_control_status_link_width_x1_v())
		pl_link_config |= xp_pl_link_config_target_tx_width_f(
			xp_pl_link_config_target_tx_width_x1_v());
	else if (link_width == xve_link_control_status_link_width_x2_v())
		pl_link_config |= xp_pl_link_config_target_tx_width_f(
			xp_pl_link_config_target_tx_width_x2_v());
	else if (link_width == xve_link_control_status_link_width_x4_v())
		pl_link_config |= xp_pl_link_config_target_tx_width_f(
			xp_pl_link_config_target_tx_width_x4_v());
	else if (link_width == xve_link_control_status_link_width_x8_v())
		pl_link_config |= xp_pl_link_config_target_tx_width_f(
			xp_pl_link_config_target_tx_width_x8_v());
	else if (link_width == xve_link_control_status_link_width_x16_v())
		pl_link_config |= xp_pl_link_config_target_tx_width_f(
			xp_pl_link_config_target_tx_width_x16_v());
	else
		BUG();

	xv_sc_dbg(LINK_SETTINGS, "  pl_link_config = 0x%08x", pl_link_config);
	xv_sc_dbg(LINK_SETTINGS, "  Done");

	xv_sc_dbg(EXEC_CHANGE, "Running link speed change...");

	nvgpu_timeout_init(g, &timeout, GPU_XVE_TIMEOUT_MS,
			NVGPU_TIMER_CPU_TIMER);
	do {
		gk20a_writel(g, xp_pl_link_config_r(0), pl_link_config);
		if (pl_link_config ==
		    gk20a_readl(g, xp_pl_link_config_r(0)))
			break;
	} while (!nvgpu_timeout_expired(&timeout));

	if (nvgpu_timeout_peek_expired(&timeout)) {
		err_status = -ETIMEDOUT;
		goto done;
	}

	xv_sc_dbg(EXEC_CHANGE, "  Wrote PL_LINK_CONFIG.");

	pl_link_config = gk20a_readl(g, xp_pl_link_config_r(0));

	do {
		pl_link_config = set_field(pl_link_config,
			  xp_pl_link_config_ltssm_directive_m(),
			  xp_pl_link_config_ltssm_directive_f(
			  xp_pl_link_config_ltssm_directive_change_speed_v()));

		xv_sc_dbg(EXEC_CHANGE, "  Executing change (0x%08x)!",
			  pl_link_config);
		gk20a_writel(g, xp_pl_link_config_r(0), pl_link_config);

		/*
		 * Read NV_XP_PL_LINK_CONFIG until the link has swapped to
		 * the target speed.
		 */
		nvgpu_timeout_init(g, &timeout, GPU_XVE_TIMEOUT_MS,
				NVGPU_TIMER_CPU_TIMER);
		do {
			pl_link_config = gk20a_readl(g, xp_pl_link_config_r(0));
			if (pl_link_config != 0xfffffff &&
			    (xp_pl_link_config_ltssm_status_f(pl_link_config) ==
			     xp_pl_link_config_ltssm_status_idle_v()) &&
			    (xp_pl_link_config_ltssm_directive_f(pl_link_config) ==
			     xp_pl_link_config_ltssm_directive_normal_operations_v()))
				break;
		} while (!nvgpu_timeout_expired(&timeout));

		if (nvgpu_timeout_peek_expired(&timeout)) {
			err_status = -ETIMEDOUT;
			xv_sc_dbg(EXEC_CHANGE, "  timeout; pl_link_config = 0x%x",
				pl_link_config);
			break;
		}

		xv_sc_dbg(EXEC_CHANGE, "  Change done... Checking status");

		if (pl_link_config == 0xffffffff) {
			WARN(1, "GPU fell of PCI bus!?");

			/*
			 * The rest of the driver is probably about to
			 * explode...
			 */
			BUG();
		}

		link_control_status =
			g->ops.xve.xve_readl(g, xve_link_control_status_r());
		xv_sc_dbg(EXEC_CHANGE, "  target %d vs current %d",
			  link_speed_setting,
			  xve_link_control_status_link_speed_v(link_control_status));

		if (err_status == -ETIMEDOUT)
			xv_sc_dbg(EXEC_CHANGE, "  Oops timed out?");
	} while (attempts-- > 0 &&
		 link_speed_setting !=
		 xve_link_control_status_link_speed_v(link_control_status));

	xv_sc_dbg(EXEC_VERIF, "Verifying speed change...");

	/*
	 * Check that the new link speed is actually active. If we failed to
	 * change to the new link speed then return to the link speed setting
	 * pre-speed change.
	 */
	new_link_speed = xve_link_control_status_link_speed_v(
		link_control_status);
	if (link_speed_setting != new_link_speed) {
		u32 link_config = gk20a_readl(g, xp_pl_link_config_r(0));

		xv_sc_dbg(EXEC_VERIF, "  Current and target speeds mismatch!");
		xv_sc_dbg(EXEC_VERIF, "    LINK_CONTROL_STATUS: 0x%08x",
			  g->ops.xve.xve_readl(g, xve_link_control_status_r()));
		xv_sc_dbg(EXEC_VERIF, "    Link speed is %s - should be %s",
			  xve_speed_to_str(new_link_speed),
			  xve_speed_to_str(link_speed_setting));

		link_config &= ~xp_pl_link_config_max_link_rate_m();
		if (new_link_speed ==
		    xve_link_control_status_link_speed_link_speed_2p5_v())
			link_config |= xp_pl_link_config_max_link_rate_f(
				xp_pl_link_config_max_link_rate_2500_mtps_v());
		else if (new_link_speed ==
			 xve_link_control_status_link_speed_link_speed_5p0_v())
			link_config |= xp_pl_link_config_max_link_rate_f(
				xp_pl_link_config_max_link_rate_5000_mtps_v());
		else if (new_link_speed ==
			 xve_link_control_status_link_speed_link_speed_8p0_v())
			link_config |= xp_pl_link_config_max_link_rate_f(
				xp_pl_link_config_max_link_rate_8000_mtps_v());
		else
			link_config |= xp_pl_link_config_max_link_rate_f(
				xp_pl_link_config_max_link_rate_2500_mtps_v());

		gk20a_writel(g, xp_pl_link_config_r(0), link_config);
		err_status = -ENODEV;
	} else {
		xv_sc_dbg(EXEC_VERIF, "  Current and target speeds match!");
		err_status = 0;
	}

done:
	/* Restore safe timings. */
	xv_sc_dbg(CLEANUP, "Restoring saved DL settings...");
	gk20a_writel(g, xp_dl_mgr_r(0), saved_dl_mgr);
	xv_sc_dbg(CLEANUP, "  Done");

	xv_sc_dbg(CLEANUP, "Re-enabling ASPM settings...");
	enable_aspm_gp106(g);
	xv_sc_dbg(CLEANUP, "  Done");

	return err_status;
}

/**
 * Sets the PCIe link speed to @xve_link_speed which must be one of:
 *
 *   %GPU_XVE_SPEED_2P5
 *   %GPU_XVE_SPEED_5P0
 *   %GPU_XVE_SPEED_8P0
 *
 * If an error is encountered an appropriate error will be returned.
 */
static int xve_set_speed_gp106(struct gk20a *g, u32 next_link_speed)
{
	u32 current_link_speed;
	int err;

	if ((next_link_speed & GPU_XVE_SPEED_MASK) == 0)
		return -EINVAL;

	err = g->ops.xve.get_speed(g, &current_link_speed);
	if (err)
		return err;

	/* No-op. */
	if (current_link_speed == next_link_speed)
		return 0;

	return __do_xve_set_speed_gp106(g, next_link_speed);
}

/**
 * Places a bitmask of available speeds for gp106 in @speed_mask.
 */
static void xve_available_speeds_gp106(struct gk20a *g, u32 *speed_mask)
{
	*speed_mask = GPU_XVE_SPEED_2P5 | GPU_XVE_SPEED_5P0;
}

#ifdef CONFIG_DEBUG_FS
static ssize_t xve_link_speed_write(struct file *filp,
				    const char __user *buff,
				    size_t len, loff_t *off)
{
	struct gk20a *g = ((struct seq_file *)filp->private_data)->private;
	char kbuff[16];
	u32 buff_size, check_len;
	u32 link_speed = 0;
	int ret;

	buff_size = min_t(size_t, 16, len);

	memset(kbuff, 0, 16);
	if (copy_from_user(kbuff, buff, buff_size))
		return -EFAULT;

	check_len = strlen("Gen1");
	if (strncmp(kbuff, "Gen1", check_len) == 0)
		link_speed = GPU_XVE_SPEED_2P5;
	else if (strncmp(kbuff, "Gen2", check_len) == 0)
		link_speed = GPU_XVE_SPEED_5P0;
	else if (strncmp(kbuff, "Gen3", check_len) == 0)
		link_speed = GPU_XVE_SPEED_8P0;
	else
		nvgpu_err(g, "%s: Unknown PCIe speed: %s",
			  __func__, kbuff);

	if (!link_speed)
		return -EINVAL;

	/* Brief pause... To help rate limit this. */
	nvgpu_msleep(250);

	/*
	 * And actually set the speed. Yay.
	 */
	ret = g->ops.xve.set_speed(g, link_speed);
	if (ret)
		return ret;

	return len;
}

static int xve_link_speed_show(struct seq_file *s, void *unused)
{
	struct gk20a *g = s->private;
	u32 speed;
	int err;

	err = g->ops.xve.get_speed(g, &speed);
	if (err)
		return err;

	seq_printf(s, "Current PCIe speed:\n  %s\n", xve_speed_to_str(speed));

	return 0;
}

static int xve_link_speed_open(struct inode *inode, struct file *file)
{
	return single_open(file, xve_link_speed_show, inode->i_private);
}

static const struct file_operations xve_link_speed_fops = {
	.open = xve_link_speed_open,
	.read = seq_read,
	.write = xve_link_speed_write,
	.llseek = seq_lseek,
	.release = single_release,
};

static int xve_available_speeds_show(struct seq_file *s, void *unused)
{
	struct gk20a *g = s->private;
	u32 available_speeds;

	g->ops.xve.available_speeds(g, &available_speeds);

	seq_puts(s, "Available PCIe bus speeds:\n");
	if (available_speeds & GPU_XVE_SPEED_2P5)
		seq_puts(s, "  Gen1\n");
	if (available_speeds & GPU_XVE_SPEED_5P0)
		seq_puts(s, "  Gen2\n");
	if (available_speeds & GPU_XVE_SPEED_8P0)
		seq_puts(s, "  Gen3\n");

	return 0;
}

static int xve_available_speeds_open(struct inode *inode, struct file *file)
{
	return single_open(file, xve_available_speeds_show, inode->i_private);
}

static const struct file_operations xve_available_speeds_fops = {
	.open = xve_available_speeds_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static int xve_link_control_status_show(struct seq_file *s, void *unused)
{
	struct gk20a *g = s->private;
	u32 link_status;

	link_status = g->ops.xve.xve_readl(g, xve_link_control_status_r());
	seq_printf(s, "0x%08x\n", link_status);

	return 0;
}

static int xve_link_control_status_open(struct inode *inode, struct file *file)
{
	return single_open(file, xve_link_control_status_show, inode->i_private);
}

static const struct file_operations xve_link_control_status_fops = {
	.open = xve_link_control_status_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};
#endif

static int xve_sw_init_gp106(struct device *dev)
{
	int err = -ENODEV;
#ifdef CONFIG_DEBUG_FS
	struct gk20a *g = get_gk20a(dev);
	struct gk20a_platform *plat = gk20a_get_platform(dev);
	struct dentry *gpu_root = plat->debugfs;

	g->debugfs_xve = debugfs_create_dir("xve", gpu_root);
	if (IS_ERR_OR_NULL(g->debugfs_xve))
		goto fail;

	/*
	 * These are just debug nodes. If they fail to get made it's not worth
	 * worrying the higher level SW.
	 */
	debugfs_create_file("link_speed", S_IRUGO,
			    g->debugfs_xve, g,
			    &xve_link_speed_fops);
	debugfs_create_file("available_speeds", S_IRUGO,
			    g->debugfs_xve, g,
			    &xve_available_speeds_fops);
	debugfs_create_file("link_control_status", S_IRUGO,
			    g->debugfs_xve, g,
			    &xve_link_control_status_fops);

	err = 0;
fail:
	return err;
#else
	return err;
#endif
}

#if defined(CONFIG_PCI_MSI)
static void xve_rearm_msi_gp106(struct gk20a *g)
{
	/* We just need to write a dummy val in the CYA_2 offset */
	g->ops.xve.xve_writel(g, xve_cya_2_r(), 0);
}
#endif

static void xve_enable_shadow_rom_gp106(struct gk20a *g)
{
	g->ops.xve.xve_writel(g, xve_rom_ctrl_r(),
			xve_rom_ctrl_rom_shadow_enabled_f());
}

static void xve_disable_shadow_rom_gp106(struct gk20a *g)
{
	g->ops.xve.xve_writel(g, xve_rom_ctrl_r(),
			xve_rom_ctrl_rom_shadow_disabled_f());
}

/*
 * Init the HAL functions and what not. xve_sw_init_gp106() is for initializing
 * all the other stuff like debugfs nodes, etc.
 */
int gp106_init_xve_ops(struct gpu_ops *gops)
{
	gops->xve.sw_init          = xve_sw_init_gp106;
	gops->xve.get_speed        = xve_get_speed_gp106;
	gops->xve.set_speed        = xve_set_speed_gp106;
	gops->xve.available_speeds = xve_available_speeds_gp106;
	gops->xve.xve_readl        = xve_xve_readl_gp106;
	gops->xve.xve_writel       = xve_xve_writel_gp106;
	gops->xve.disable_aspm     = xve_disable_aspm_gp106;
	gops->xve.reset_gpu        = xve_reset_gpu_gp106;
#if defined(CONFIG_PCI_MSI)
	gops->xve.rearm_msi        = xve_rearm_msi_gp106;
#endif
	gops->xve.enable_shadow_rom = xve_enable_shadow_rom_gp106;
	gops->xve.disable_shadow_rom = xve_disable_shadow_rom_gp106;

	return 0;
}