1 files changed, 623 insertions, 0 deletions

diff --git a/drivers/gpu/nvgpu/gp106/xve_gp106.c b/drivers/gpu/nvgpu/gp106/xve_gp106.c
new file mode 100644
index 00000000..23a02fbd
--- /dev/null
+++ b/drivers/gpu/nvgpu/gp106/xve_gp106.c
@@ -0,0 +1,623 @@
+/*
+ * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/debugfs.h>
+
+#include "gk20a/gk20a.h"
+#include "gm206/bios_gm206.h"
+#include "gp106/xve_gp106.h"
+
+#include "gp106/hw_xp_gp106.h"
+#include "gp106/hw_xve_gp106.h"
+
+/**
+ * Init a timer and place the timeout data in @timeout.
+ */
+static void init_timeout(u32 timeout_ms, u32 *timeout)
+{
+        *timeout = jiffies + msecs_to_jiffies(timeout_ms);
+}
+
+/**
+ * Returns 1 if the current time is after @timeout i.e: the timer timed
+ * out. Returns 0 if the timer still has time left.
+ */
+static int check_timeout(u32 *timeout)
+{
+        unsigned long now = jiffies;
+        unsigned long timeout_l = (unsigned long)*timeout;
+
+        if (time_after(now, timeout_l))
+                return 1;
+
+        return 0;
+}
+
+static void xve_xve_writel_gp106(struct gk20a *g, u32 reg, u32 val)
+{
+        gk20a_writel(g, NV_PCFG + reg, val);
+}
+
+static u32 xve_xve_readl_gp106(struct gk20a *g, u32 reg)
+{
+        return gk20a_readl(g, NV_PCFG + reg);
+}
+
+/**
+ * Places one of:
+ *
+ *   %GPU_XVE_SPEED_2P5
+ *   %GPU_XVE_SPEED_5P0
+ *   %GPU_XVE_SPEED_8P0
+ *
+ * in the u32 pointed to by @xve_link_speed. If for some reason an unknown PCIe
+ * bus speed is detected then *@xve_link_speed is not touched and -ENODEV is
+ * returned.
+ */
+static int xve_get_speed_gp106(struct gk20a *g, u32 *xve_link_speed)
+{
+        u32 status;
+        u32 link_speed, real_link_speed = 0;
+
+        status = g->ops.xve.xve_readl(g, xve_link_control_status_r());
+
+        link_speed = xve_link_control_status_link_speed_v(status);
+
+        /*
+         * Can't use a switch statement becuase switch statements dont work with
+         * function calls.
+         */
+        if (link_speed == xve_link_control_status_link_speed_link_speed_2p5_v())
+                real_link_speed = GPU_XVE_SPEED_2P5;
+        if (link_speed == xve_link_control_status_link_speed_link_speed_5p0_v())
+                real_link_speed = GPU_XVE_SPEED_5P0;
+        if (link_speed == xve_link_control_status_link_speed_link_speed_8p0_v())
+                real_link_speed = GPU_XVE_SPEED_8P0;
+
+        if (!real_link_speed) {
+                pr_warn("%s: Unknown PCIe bus speed!\n", __func__);
+                return -ENODEV;
+        }
+
+        *xve_link_speed = real_link_speed;
+        return 0;
+}
+
+/**
+ * Set the mask for L0s in the XVE.
+ *
+ * When @status is non-zero the mask for L0s is set which _disables_ L0s. When
+ * @status is zero L0s is no longer masked and may be enabled.
+ */
+static void set_xve_l0s_mask(struct gk20a *g, bool status)
+{
+        u32 xve_priv;
+        u32 status_bit = status ? 1 : 0;
+
+        xve_priv = g->ops.xve.xve_readl(g, xve_priv_xv_r());
+
+        xve_priv = set_field(xve_priv,
+                  xve_priv_xv_cya_l0s_enable_m(),
+                  xve_priv_xv_cya_l0s_enable_f(status_bit));
+
+        g->ops.xve.xve_writel(g, xve_priv_xv_r(), xve_priv);
+}
+
+/**
+ * Set the mask for L1 in the XVE.
+ *
+ * When @status is non-zero the mask for L1 is set which _disables_ L0s. When
+ * @status is zero L1 is no longer masked and may be enabled.
+ */
+static void set_xve_l1_mask(struct gk20a *g, int status)
+{
+        u32 xve_priv;
+        u32 status_bit = status ? 1 : 0;
+
+        xve_priv = g->ops.xve.xve_readl(g, xve_priv_xv_r());
+
+        xve_priv = set_field(xve_priv,
+                  xve_priv_xv_cya_l1_enable_m(),
+                  xve_priv_xv_cya_l1_enable_f(status_bit));
+
+        g->ops.xve.xve_writel(g, xve_priv_xv_r(), xve_priv);
+}
+
+/**
+ * When doing the speed change disable power saving features.
+ */
+static void disable_aspm_gp106(struct gk20a *g)
+{
+        u32 xve_priv;
+
+        xve_priv = g->ops.xve.xve_readl(g, xve_priv_xv_r());
+
+        /*
+         * Store prior ASPM state so we can restore it later on.
+         */
+        g->xve_l0s = xve_priv_xv_cya_l0s_enable_v(xve_priv);
+        g->xve_l1  = xve_priv_xv_cya_l1_enable_v(xve_priv);
+
+        set_xve_l0s_mask(g, true);
+        set_xve_l1_mask(g, true);
+}
+
+/**
+ * Restore the state saved by disable_aspm_gp106().
+ */
+static void enable_aspm_gp106(struct gk20a *g)
+{
+        set_xve_l0s_mask(g, g->xve_l0s);
+        set_xve_l1_mask(g, g->xve_l1);
+}
+
+/*
+ * Error checking is done in xve_set_speed_gp106.
+ */
+static int __do_xve_set_speed_gp106(struct gk20a *g, u32 next_link_speed)
+{
+        u32 current_link_speed, new_link_speed;
+        u32 dl_mgr, saved_dl_mgr;
+        u32 pl_link_config;
+        u32 link_control_status, link_speed_setting, link_width;
+        u32 timeout;
+        int attempts = 10, err_status = 0;
+
+        g->ops.xve.get_speed(g, &current_link_speed);
+        xv_sc_dbg(PRE_CHANGE, "Executing PCIe link change.");
+        xv_sc_dbg(PRE_CHANGE, "  Current speed:  %s",
+                  xve_speed_to_str(current_link_speed));
+        xv_sc_dbg(PRE_CHANGE, "  Next speed:     %s",
+                  xve_speed_to_str(next_link_speed));
+        xv_sc_dbg(PRE_CHANGE, "  PL_LINK_CONFIG: 0x%08x",
+                  gk20a_readl(g, xp_pl_link_config_r(0)));
+
+        xv_sc_dbg(DISABLE_ASPM, "Disabling ASPM...");
+        disable_aspm_gp106(g);
+        xv_sc_dbg(DISABLE_ASPM, "  Done!");
+
+        xv_sc_dbg(DL_SAFE_MODE, "Putting DL in safe mode...");
+        saved_dl_mgr = gk20a_readl(g, xp_dl_mgr_r(0));
+
+        /*
+         * Put the DL in safe mode.
+         */
+        dl_mgr = saved_dl_mgr;
+        dl_mgr |= xp_dl_mgr_safe_timing_f(1);
+        gk20a_writel(g, xp_dl_mgr_r(0), dl_mgr);
+        xv_sc_dbg(DL_SAFE_MODE, "  Done!");
+
+        init_timeout(GPU_XVE_TIMEOUT_MS, &timeout);
+
+        xv_sc_dbg(CHECK_LINK, "Checking for link idle...");
+        while (1) {
+                pl_link_config = gk20a_readl(g, xp_pl_link_config_r(0));
+                if ((xp_pl_link_config_ltssm_status_f(pl_link_config) ==
+                     xp_pl_link_config_ltssm_status_idle_v()) &&
+                    (xp_pl_link_config_ltssm_directive_f(pl_link_config) ==
+                     xp_pl_link_config_ltssm_directive_normal_operations_v()))
+                        break;
+
+                if (check_timeout(&timeout)) {
+                        err_status = -ETIMEDOUT;
+                        break;
+                }
+        }
+
+        if (err_status == -ETIMEDOUT)
+                /* TODO: debug message. */
+                goto done;
+
+        xv_sc_dbg(CHECK_LINK, "  Done");
+
+        xv_sc_dbg(LINK_SETTINGS, "Preparing next link settings");
+        pl_link_config &= ~xp_pl_link_config_max_link_rate_m();
+        switch (next_link_speed) {
+        case GPU_XVE_SPEED_2P5:
+                link_speed_setting =
+                        xve_link_control_status_link_speed_link_speed_2p5_v();
+                pl_link_config |= xp_pl_link_config_max_link_rate_f(
+                        xp_pl_link_config_max_link_rate_2500_mtps_v());
+                break;
+        case GPU_XVE_SPEED_5P0:
+                link_speed_setting =
+                        xve_link_control_status_link_speed_link_speed_5p0_v();
+                pl_link_config |= xp_pl_link_config_max_link_rate_f(
+                        xp_pl_link_config_max_link_rate_5000_mtps_v());
+                break;
+        case GPU_XVE_SPEED_8P0:
+                link_speed_setting =
+                        xve_link_control_status_link_speed_link_speed_8p0_v();
+                pl_link_config |= xp_pl_link_config_max_link_rate_f(
+                        xp_pl_link_config_max_link_rate_8000_mtps_v());
+                break;
+        default:
+                BUG(); /* Should never be hit. */
+        }
+
+        link_control_status =
+                g->ops.xve.xve_readl(g, xve_link_control_status_r());
+        link_width = xve_link_control_status_link_width_v(link_control_status);
+
+        pl_link_config &= ~xp_pl_link_config_target_tx_width_m();
+
+        /* Can't use a switch due to oddities in register definitions. */
+        if (link_width == xve_link_control_status_link_width_x1_v())
+                pl_link_config |= xp_pl_link_config_target_tx_width_f(
+                        xp_pl_link_config_target_tx_width_x1_v());
+        else if (link_width == xve_link_control_status_link_width_x2_v())
+                pl_link_config |= xp_pl_link_config_target_tx_width_f(
+                        xp_pl_link_config_target_tx_width_x2_v());
+        else if (link_width == xve_link_control_status_link_width_x4_v())
+                pl_link_config |= xp_pl_link_config_target_tx_width_f(
+                        xp_pl_link_config_target_tx_width_x4_v());
+        else if (link_width == xve_link_control_status_link_width_x8_v())
+                pl_link_config |= xp_pl_link_config_target_tx_width_f(
+                        xp_pl_link_config_target_tx_width_x8_v());
+        else if (link_width == xve_link_control_status_link_width_x16_v())
+                pl_link_config |= xp_pl_link_config_target_tx_width_f(
+                        xp_pl_link_config_target_tx_width_x16_v());
+        else
+                BUG();
+
+        xv_sc_dbg(LINK_SETTINGS, "  pl_link_config = 0x%08x", pl_link_config);
+        xv_sc_dbg(LINK_SETTINGS, "  Done");
+
+        xv_sc_dbg(EXEC_CHANGE, "Running link speed change...");
+
+        init_timeout(GPU_XVE_TIMEOUT_MS, &timeout);
+        while (1) {
+                gk20a_writel(g, xp_pl_link_config_r(0), pl_link_config);
+                if (pl_link_config ==
+                    gk20a_readl(g, xp_pl_link_config_r(0)))
+                        break;
+
+                if (check_timeout(&timeout)) {
+                        err_status = -ETIMEDOUT;
+                        break;
+                }
+        }
+
+        if (err_status == -ETIMEDOUT)
+                goto done;
+
+        xv_sc_dbg(EXEC_CHANGE, "  Wrote PL_LINK_CONFIG.");
+
+        pl_link_config = gk20a_readl(g, xp_pl_link_config_r(0));
+
+        do {
+                pl_link_config = set_field(pl_link_config,
+                          xp_pl_link_config_ltssm_directive_m(),
+                          xp_pl_link_config_ltssm_directive_f(
+                          xp_pl_link_config_ltssm_directive_change_speed_v()));
+
+                xv_sc_dbg(EXEC_CHANGE, "  Executing change (0x%08x)!",
+                          pl_link_config);
+                gk20a_writel(g, xp_pl_link_config_r(0), pl_link_config);
+
+                /*
+                 * Read NV_XP_PL_LINK_CONFIG until the link has swapped to
+                 * the target speed.
+                 */
+                init_timeout(GPU_XVE_TIMEOUT_MS, &timeout);
+                while (1) {
+                        pl_link_config = gk20a_readl(g, xp_pl_link_config_r(0));
+                        if (pl_link_config != 0xfffffff &&
+                            (xp_pl_link_config_ltssm_status_f(pl_link_config) ==
+                             xp_pl_link_config_ltssm_status_idle_v()) &&
+                            (xp_pl_link_config_ltssm_directive_f(pl_link_config) ==
+                             xp_pl_link_config_ltssm_directive_normal_operations_v()))
+                                break;
+
+                        if (check_timeout(&timeout)) {
+                                err_status = -ETIMEDOUT;
+                                xv_sc_dbg(EXEC_CHANGE, "  timeout; pl_link_config = 0x%x",
+                                        pl_link_config);
+                                break;
+                        }
+                }
+
+                xv_sc_dbg(EXEC_CHANGE, "  Change done... Checking status");
+
+                if (pl_link_config == 0xffffffff) {
+                        WARN(1, "GPU fell of PCI bus!?");
+
+                        /*
+                         * The rest of the driver is probably about to
+                         * explode...
+                         */
+                        BUG();
+                }
+
+                link_control_status =
+                        g->ops.xve.xve_readl(g, xve_link_control_status_r());
+                xv_sc_dbg(EXEC_CHANGE, "  target %d vs current %d",
+                          link_speed_setting,
+                          xve_link_control_status_link_speed_v(link_control_status));
+
+                if (err_status == -ETIMEDOUT)
+                        xv_sc_dbg(EXEC_CHANGE, "  Oops timed out?");
+        } while (attempts-- > 0 &&
+                 link_speed_setting !=
+                 xve_link_control_status_link_speed_v(link_control_status));
+
+        xv_sc_dbg(EXEC_VERIF, "Verifying speed change...");
+
+        /*
+         * Check that the new link speed is actually active. If we failed to
+         * change to the new link speed then return to the link speed setting
+         * pre-speed change.
+         */
+        new_link_speed = xve_link_control_status_link_speed_v(
+                link_control_status);
+        if (link_speed_setting != new_link_speed) {
+                u32 link_config = gk20a_readl(g, xp_pl_link_config_r(0));
+
+                xv_sc_dbg(EXEC_VERIF, "  Current and target speeds mismatch!");
+                xv_sc_dbg(EXEC_VERIF, "    LINK_CONTROL_STATUS: 0x%08x",
+                          g->ops.xve.xve_readl(g, xve_link_control_status_r()));
+                xv_sc_dbg(EXEC_VERIF, "    Link speed is %s - should be %s",
+                          xve_speed_to_str(new_link_speed),
+                          xve_speed_to_str(link_speed_setting));
+
+                link_config &= ~xp_pl_link_config_max_link_rate_m();
+                if (new_link_speed ==
+                    xve_link_control_status_link_speed_link_speed_2p5_v())
+                        link_config |= xp_pl_link_config_max_link_rate_f(
+                                xp_pl_link_config_max_link_rate_2500_mtps_v());
+                else if (new_link_speed ==
+                         xve_link_control_status_link_speed_link_speed_5p0_v())
+                        link_config |= xp_pl_link_config_max_link_rate_f(
+                                xp_pl_link_config_max_link_rate_5000_mtps_v());
+                else if (new_link_speed ==
+                         xve_link_control_status_link_speed_link_speed_8p0_v())
+                        link_config |= xp_pl_link_config_max_link_rate_f(
+                                xp_pl_link_config_max_link_rate_8000_mtps_v());
+                else
+                        link_config |= xp_pl_link_config_max_link_rate_f(
+                                xp_pl_link_config_max_link_rate_2500_mtps_v());
+
+                gk20a_writel(g, xp_pl_link_config_r(0), link_config);
+                err_status = -ENODEV;
+        } else {
+                xv_sc_dbg(EXEC_VERIF, "  Current and target speeds match!");
+                err_status = 0;
+        }
+
+done:
+        /* Restore safe timings. */
+        xv_sc_dbg(CLEANUP, "Restoring saved DL settings...");
+        gk20a_writel(g, xp_dl_mgr_r(0), saved_dl_mgr);
+        xv_sc_dbg(CLEANUP, "  Done");
+
+        xv_sc_dbg(CLEANUP, "Re-enabling ASPM settings...");
+        enable_aspm_gp106(g);
+        xv_sc_dbg(CLEANUP, "  Done");
+
+        return err_status;
+}
+
+/**
+ * Sets the PCIe link speed to @xve_link_speed which must be one of:
+ *
+ *   %GPU_XVE_SPEED_2P5
+ *   %GPU_XVE_SPEED_5P0
+ *   %GPU_XVE_SPEED_8P0
+ *
+ * If an error is encountered an appropriate error will be returned.
+ */
+static int xve_set_speed_gp106(struct gk20a *g, u32 next_link_speed)
+{
+        u32 current_link_speed;
+        int err;
+
+        if ((next_link_speed & GPU_XVE_SPEED_MASK) == 0)
+                return -EINVAL;
+
+        err = g->ops.xve.get_speed(g, &current_link_speed);
+        if (err)
+                return err;
+
+        /* No-op. */
+        if (current_link_speed == next_link_speed)
+                return 0;
+
+        return __do_xve_set_speed_gp106(g, next_link_speed);
+}
+
+/**
+ * Places a bitmask of available speeds for gp106 in @speed_mask.
+ */
+static void xve_available_speeds_gp106(struct gk20a *g, u32 *speed_mask)
+{
+        *speed_mask = GPU_XVE_SPEED_2P5 | GPU_XVE_SPEED_5P0;
+}
+
+static ssize_t xve_link_speed_write(struct file *filp,
+                                    const char __user *buff,
+                                    size_t len, loff_t *off)
+{
+        struct gk20a *g = ((struct seq_file *)filp->private_data)->private;
+        char kbuff[16];
+        u32 buff_size, check_len;
+        u32 link_speed = 0;
+        int ret;
+
+        buff_size = min_t(size_t, 16, len);
+
+        memset(kbuff, 0, 16);
+        if (copy_from_user(kbuff, buff, buff_size))
+                return -EFAULT;
+
+        check_len = strlen("Gen1");
+        if (strncmp(kbuff, "Gen1", check_len) == 0)
+                link_speed = GPU_XVE_SPEED_2P5;
+        else if (strncmp(kbuff, "Gen2", check_len) == 0)
+                link_speed = GPU_XVE_SPEED_5P0;
+        else if (strncmp(kbuff, "Gen3", check_len) == 0)
+                link_speed = GPU_XVE_SPEED_8P0;
+        else
+                gk20a_err(g->dev, "%s: Unknown PCIe speed: %s\n",
+                          __func__, kbuff);
+
+        if (!link_speed)
+                return -EINVAL;
+
+        /* Brief pause... To help rate limit this. */
+        msleep(250);
+
+        /*
+         * And actually set the speed. Yay.
+         */
+        ret = g->ops.xve.set_speed(g, link_speed);
+        if (ret)
+                return ret;
+
+        return len;
+}
+
+static int xve_link_speed_show(struct seq_file *s, void *unused)
+{
+        struct gk20a *g = s->private;
+        u32 speed;
+        int err;
+
+        err = g->ops.xve.get_speed(g, &speed);
+        if (err)
+                return err;
+
+        seq_printf(s, "Current PCIe speed:\n  %s\n", xve_speed_to_str(speed));
+
+        return 0;
+}
+
+static int xve_link_speed_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, xve_link_speed_show, inode->i_private);
+}
+
+static const struct file_operations xve_link_speed_fops = {
+        .open = xve_link_speed_open,
+        .read = seq_read,
+        .write = xve_link_speed_write,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+
+static int xve_available_speeds_show(struct seq_file *s, void *unused)
+{
+        struct gk20a *g = s->private;
+        u32 available_speeds;
+
+        g->ops.xve.available_speeds(g, &available_speeds);
+
+        seq_puts(s, "Available PCIe bus speeds:\n");
+        if (available_speeds & GPU_XVE_SPEED_2P5)
+                seq_puts(s, "  Gen1\n");
+        if (available_speeds & GPU_XVE_SPEED_5P0)
+                seq_puts(s, "  Gen2\n");
+        if (available_speeds & GPU_XVE_SPEED_8P0)
+                seq_puts(s, "  Gen3\n");
+
+        return 0;
+}
+
+static int xve_available_speeds_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, xve_available_speeds_show, inode->i_private);
+}
+
+static const struct file_operations xve_available_speeds_fops = {
+        .open = xve_available_speeds_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+
+static int xve_link_control_status_show(struct seq_file *s, void *unused)
+{
+        struct gk20a *g = s->private;
+        u32 link_status;
+
+        link_status = g->ops.xve.xve_readl(g, xve_link_control_status_r());
+        seq_printf(s, "0x%08x\n", link_status);
+
+        return 0;
+}
+
+static int xve_link_control_status_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, xve_link_control_status_show, inode->i_private);
+}
+
+static const struct file_operations xve_link_control_status_fops = {
+        .open = xve_link_control_status_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+
+static int xve_sw_init_gp106(struct device *dev)
+{
+        int err = -ENODEV;
+#ifdef CONFIG_DEBUG_FS
+        struct gk20a *g = get_gk20a(dev);
+        struct gk20a_platform *plat = gk20a_get_platform(dev);
+        struct dentry *gpu_root = plat->debugfs;
+
+        g->debugfs_xve = debugfs_create_dir("xve", gpu_root);
+        if (IS_ERR_OR_NULL(g->debugfs_xve))
+                goto fail;
+
+        /*
+         * These are just debug nodes. If they fail to get made it's not worth
+         * worrying the higher level SW.
+         */
+        debugfs_create_file("link_speed", S_IRUGO,
+                            g->debugfs_xve, g,
+                            &xve_link_speed_fops);
+        debugfs_create_file("available_speeds", S_IRUGO,
+                            g->debugfs_xve, g,
+                            &xve_available_speeds_fops);
+        debugfs_create_file("link_control_status", S_IRUGO,
+                            g->debugfs_xve, g,
+                            &xve_link_control_status_fops);
+
+        err = 0;
+fail:
+        return err;
+#else
+        return err;
+#endif
+}
+
+/*
+ * Init the HAL functions and what not. xve_sw_init_gp106() is for initializing
+ * all the other stuff like debugfs nodes, etc.
+ */
+int gp106_init_xve_ops(struct gpu_ops *gops)
+{
+        gops->xve.sw_init          = xve_sw_init_gp106;
+        gops->xve.get_speed        = xve_get_speed_gp106;
+        gops->xve.set_speed        = xve_set_speed_gp106;
+        gops->xve.available_speeds = xve_available_speeds_gp106;
+        gops->xve.xve_readl        = xve_xve_readl_gp106;
+        gops->xve.xve_writel       = xve_xve_writel_gp106;
+
+        return 0;
+}