memory: tegra: Introduce Tegra20 EMC driver

Introduce driver for the External Memory Controller (EMC) found on Tegra20
chips, which controls the external DRAM on the board. The purpose of this
driver is to program memory timing for external memory on the EMC clock
rate change.

Signed-off-by: Dmitry Osipenko <digetx@gmail.com>
Acked-by: Peter De Schrijver <pdeschrijver@nvidia.com>
Signed-off-by: Thierry Reding <treding@nvidia.com>

3 files changed, 602 insertions, 0 deletions

diff --git a/drivers/memory/tegra/Kconfig b/drivers/memory/tegra/Kconfig
index 6d74e499e18d..34e0b70f5c5f 100644
--- a/drivers/memory/tegra/Kconfig
+++ b/drivers/memory/tegra/Kconfig
@@ -6,6 +6,16 @@ config TEGRA_MC
           This driver supports the Memory Controller (MC) hardware found on
           NVIDIA Tegra SoCs.
 
+config TEGRA20_EMC
+        bool "NVIDIA Tegra20 External Memory Controller driver"
+        default y
+        depends on ARCH_TEGRA_2x_SOC
+        help
+          This driver is for the External Memory Controller (EMC) found on
+          Tegra20 chips. The EMC controls the external DRAM on the board.
+          This driver is required to change memory timings / clock rate for
+          external memory.
+
 config TEGRA124_EMC
         bool "NVIDIA Tegra124 External Memory Controller driver"
         default y
diff --git a/drivers/memory/tegra/Makefile b/drivers/memory/tegra/Makefile
index 94ab16ba075b..3971a6b7c487 100644
--- a/drivers/memory/tegra/Makefile
+++ b/drivers/memory/tegra/Makefile
@@ -10,5 +10,6 @@ tegra-mc-$(CONFIG_ARCH_TEGRA_210_SOC) += tegra210.o
 
 obj-$(CONFIG_TEGRA_MC) += tegra-mc.o
 
+obj-$(CONFIG_TEGRA20_EMC)  += tegra20-emc.o
 obj-$(CONFIG_TEGRA124_EMC) += tegra124-emc.o
 obj-$(CONFIG_ARCH_TEGRA_186_SOC) += tegra186.o
diff --git a/drivers/memory/tegra/tegra20-emc.c b/drivers/memory/tegra/tegra20-emc.c
new file mode 100644
index 000000000000..9ee5bef49e47
--- /dev/null
+++ b/drivers/memory/tegra/tegra20-emc.c
@@ -0,0 +1,591 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Tegra20 External Memory Controller driver
+ *
+ * Author: Dmitry Osipenko <digetx@gmail.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/sort.h>
+#include <linux/types.h>
+
+#include <soc/tegra/fuse.h>
+
+#define EMC_INTSTATUS                           0x000
+#define EMC_INTMASK                             0x004
+#define EMC_TIMING_CONTROL                      0x028
+#define EMC_RC                                  0x02c
+#define EMC_RFC                                 0x030
+#define EMC_RAS                                 0x034
+#define EMC_RP                                  0x038
+#define EMC_R2W                                 0x03c
+#define EMC_W2R                                 0x040
+#define EMC_R2P                                 0x044
+#define EMC_W2P                                 0x048
+#define EMC_RD_RCD                              0x04c
+#define EMC_WR_RCD                              0x050
+#define EMC_RRD                                 0x054
+#define EMC_REXT                                0x058
+#define EMC_WDV                                 0x05c
+#define EMC_QUSE                                0x060
+#define EMC_QRST                                0x064
+#define EMC_QSAFE                               0x068
+#define EMC_RDV                                 0x06c
+#define EMC_REFRESH                             0x070
+#define EMC_BURST_REFRESH_NUM                   0x074
+#define EMC_PDEX2WR                             0x078
+#define EMC_PDEX2RD                             0x07c
+#define EMC_PCHG2PDEN                           0x080
+#define EMC_ACT2PDEN                            0x084
+#define EMC_AR2PDEN                             0x088
+#define EMC_RW2PDEN                             0x08c
+#define EMC_TXSR                                0x090
+#define EMC_TCKE                                0x094
+#define EMC_TFAW                                0x098
+#define EMC_TRPAB                               0x09c
+#define EMC_TCLKSTABLE                          0x0a0
+#define EMC_TCLKSTOP                            0x0a4
+#define EMC_TREFBW                              0x0a8
+#define EMC_QUSE_EXTRA                          0x0ac
+#define EMC_ODT_WRITE                           0x0b0
+#define EMC_ODT_READ                            0x0b4
+#define EMC_FBIO_CFG5                           0x104
+#define EMC_FBIO_CFG6                           0x114
+#define EMC_AUTO_CAL_INTERVAL                   0x2a8
+#define EMC_CFG_2                               0x2b8
+#define EMC_CFG_DIG_DLL                         0x2bc
+#define EMC_DLL_XFORM_DQS                       0x2c0
+#define EMC_DLL_XFORM_QUSE                      0x2c4
+#define EMC_ZCAL_REF_CNT                        0x2e0
+#define EMC_ZCAL_WAIT_CNT                       0x2e4
+#define EMC_CFG_CLKTRIM_0                       0x2d0
+#define EMC_CFG_CLKTRIM_1                       0x2d4
+#define EMC_CFG_CLKTRIM_2                       0x2d8
+
+#define EMC_CLKCHANGE_REQ_ENABLE                BIT(0)
+#define EMC_CLKCHANGE_PD_ENABLE                 BIT(1)
+#define EMC_CLKCHANGE_SR_ENABLE                 BIT(2)
+
+#define EMC_TIMING_UPDATE                       BIT(0)
+
+#define EMC_REFRESH_OVERFLOW_INT                BIT(3)
+#define EMC_CLKCHANGE_COMPLETE_INT              BIT(4)
+
+static const u16 emc_timing_registers[] = {
+        EMC_RC,
+        EMC_RFC,
+        EMC_RAS,
+        EMC_RP,
+        EMC_R2W,
+        EMC_W2R,
+        EMC_R2P,
+        EMC_W2P,
+        EMC_RD_RCD,
+        EMC_WR_RCD,
+        EMC_RRD,
+        EMC_REXT,
+        EMC_WDV,
+        EMC_QUSE,
+        EMC_QRST,
+        EMC_QSAFE,
+        EMC_RDV,
+        EMC_REFRESH,
+        EMC_BURST_REFRESH_NUM,
+        EMC_PDEX2WR,
+        EMC_PDEX2RD,
+        EMC_PCHG2PDEN,
+        EMC_ACT2PDEN,
+        EMC_AR2PDEN,
+        EMC_RW2PDEN,
+        EMC_TXSR,
+        EMC_TCKE,
+        EMC_TFAW,
+        EMC_TRPAB,
+        EMC_TCLKSTABLE,
+        EMC_TCLKSTOP,
+        EMC_TREFBW,
+        EMC_QUSE_EXTRA,
+        EMC_FBIO_CFG6,
+        EMC_ODT_WRITE,
+        EMC_ODT_READ,
+        EMC_FBIO_CFG5,
+        EMC_CFG_DIG_DLL,
+        EMC_DLL_XFORM_DQS,
+        EMC_DLL_XFORM_QUSE,
+        EMC_ZCAL_REF_CNT,
+        EMC_ZCAL_WAIT_CNT,
+        EMC_AUTO_CAL_INTERVAL,
+        EMC_CFG_CLKTRIM_0,
+        EMC_CFG_CLKTRIM_1,
+        EMC_CFG_CLKTRIM_2,
+};
+
+struct emc_timing {
+        unsigned long rate;
+        u32 data[ARRAY_SIZE(emc_timing_registers)];
+};
+
+struct tegra_emc {
+        struct device *dev;
+        struct completion clk_handshake_complete;
+        struct notifier_block clk_nb;
+        struct clk *backup_clk;
+        struct clk *emc_mux;
+        struct clk *pll_m;
+        struct clk *clk;
+        void __iomem *regs;
+
+        struct emc_timing *timings;
+        unsigned int num_timings;
+};
+
+static irqreturn_t tegra_emc_isr(int irq, void *data)
+{
+        struct tegra_emc *emc = data;
+        u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT;
+        u32 status;
+
+        status = readl_relaxed(emc->regs + EMC_INTSTATUS) & intmask;
+        if (!status)
+                return IRQ_NONE;
+
+        /* notify about EMC-CAR handshake completion */
+        if (status & EMC_CLKCHANGE_COMPLETE_INT)
+                complete(&emc->clk_handshake_complete);
+
+        /* notify about HW problem */
+        if (status & EMC_REFRESH_OVERFLOW_INT)
+                dev_err_ratelimited(emc->dev,
+                                    "refresh request overflow timeout\n");
+
+        /* clear interrupts */
+        writel_relaxed(status, emc->regs + EMC_INTSTATUS);
+
+        return IRQ_HANDLED;
+}
+
+static struct emc_timing *tegra_emc_find_timing(struct tegra_emc *emc,
+                                                unsigned long rate)
+{
+        struct emc_timing *timing = NULL;
+        unsigned int i;
+
+        for (i = 0; i < emc->num_timings; i++) {
+                if (emc->timings[i].rate >= rate) {
+                        timing = &emc->timings[i];
+                        break;
+                }
+        }
+
+        if (!timing) {
+                dev_err(emc->dev, "no timing for rate %lu\n", rate);
+                return NULL;
+        }
+
+        return timing;
+}
+
+static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate)
+{
+        struct emc_timing *timing = tegra_emc_find_timing(emc, rate);
+        unsigned int i;
+
+        if (!timing)
+                return -EINVAL;
+
+        dev_dbg(emc->dev, "%s: using timing rate %lu for requested rate %lu\n",
+                __func__, timing->rate, rate);
+
+        /* program shadow registers */
+        for (i = 0; i < ARRAY_SIZE(timing->data); i++)
+                writel_relaxed(timing->data[i],
+                               emc->regs + emc_timing_registers[i]);
+
+        /* wait until programming has settled */
+        readl_relaxed(emc->regs + emc_timing_registers[i - 1]);
+
+        reinit_completion(&emc->clk_handshake_complete);
+
+        return 0;
+}
+
+static int emc_complete_timing_change(struct tegra_emc *emc, bool flush)
+{
+        long timeout;
+
+        dev_dbg(emc->dev, "%s: flush %d\n", __func__, flush);
+
+        if (flush) {
+                /* manually initiate memory timing update */
+                writel_relaxed(EMC_TIMING_UPDATE,
+                               emc->regs + EMC_TIMING_CONTROL);
+                return 0;
+        }
+
+        timeout = wait_for_completion_timeout(&emc->clk_handshake_complete,
+                                              usecs_to_jiffies(100));
+        if (timeout == 0) {
+                dev_err(emc->dev, "EMC-CAR handshake failed\n");
+                return -EIO;
+        } else if (timeout < 0) {
+                dev_err(emc->dev, "failed to wait for EMC-CAR handshake: %ld\n",
+                        timeout);
+                return timeout;
+        }
+
+        return 0;
+}
+
+static int tegra_emc_clk_change_notify(struct notifier_block *nb,
+                                       unsigned long msg, void *data)
+{
+        struct tegra_emc *emc = container_of(nb, struct tegra_emc, clk_nb);
+        struct clk_notifier_data *cnd = data;
+        int err;
+
+        switch (msg) {
+        case PRE_RATE_CHANGE:
+                err = emc_prepare_timing_change(emc, cnd->new_rate);
+                break;
+
+        case ABORT_RATE_CHANGE:
+                err = emc_prepare_timing_change(emc, cnd->old_rate);
+                if (err)
+                        break;
+
+                err = emc_complete_timing_change(emc, true);
+                break;
+
+        case POST_RATE_CHANGE:
+                err = emc_complete_timing_change(emc, false);
+                break;
+
+        default:
+                return NOTIFY_DONE;
+        }
+
+        return notifier_from_errno(err);
+}
+
+static int load_one_timing_from_dt(struct tegra_emc *emc,
+                                   struct emc_timing *timing,
+                                   struct device_node *node)
+{
+        u32 rate;
+        int err;
+
+        if (!of_device_is_compatible(node, "nvidia,tegra20-emc-table")) {
+                dev_err(emc->dev, "incompatible DT node: %pOF\n", node);
+                return -EINVAL;
+        }
+
+        err = of_property_read_u32(node, "clock-frequency", &rate);
+        if (err) {
+                dev_err(emc->dev, "timing %pOF: failed to read rate: %d\n",
+                        node, err);
+                return err;
+        }
+
+        err = of_property_read_u32_array(node, "nvidia,emc-registers",
+                                         timing->data,
+                                         ARRAY_SIZE(emc_timing_registers));
+        if (err) {
+                dev_err(emc->dev,
+                        "timing %pOF: failed to read emc timing data: %d\n",
+                        node, err);
+                return err;
+        }
+
+        /*
+         * The EMC clock rate is twice the bus rate, and the bus rate is
+         * measured in kHz.
+         */
+        timing->rate = rate * 2 * 1000;
+
+        dev_dbg(emc->dev, "%s: %pOF: EMC rate %lu\n",
+                __func__, node, timing->rate);
+
+        return 0;
+}
+
+static int cmp_timings(const void *_a, const void *_b)
+{
+        const struct emc_timing *a = _a;
+        const struct emc_timing *b = _b;
+
+        if (a->rate < b->rate)
+                return -1;
+
+        if (a->rate > b->rate)
+                return 1;
+
+        return 0;
+}
+
+static int tegra_emc_load_timings_from_dt(struct tegra_emc *emc,
+                                          struct device_node *node)
+{
+        struct device_node *child;
+        struct emc_timing *timing;
+        int child_count;
+        int err;
+
+        child_count = of_get_child_count(node);
+        if (!child_count) {
+                dev_err(emc->dev, "no memory timings in DT node: %pOF\n", node);
+                return -EINVAL;
+        }
+
+        emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing),
+                                    GFP_KERNEL);
+        if (!emc->timings)
+                return -ENOMEM;
+
+        emc->num_timings = child_count;
+        timing = emc->timings;
+
+        for_each_child_of_node(node, child) {
+                err = load_one_timing_from_dt(emc, timing++, child);
+                if (err) {
+                        of_node_put(child);
+                        return err;
+                }
+        }
+
+        sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings,
+             NULL);
+
+        return 0;
+}
+
+static struct device_node *
+tegra_emc_find_node_by_ram_code(struct device *dev)
+{
+        struct device_node *np;
+        u32 value, ram_code;
+        int err;
+
+        if (!of_property_read_bool(dev->of_node, "nvidia,use-ram-code"))
+                return of_node_get(dev->of_node);
+
+        ram_code = tegra_read_ram_code();
+
+        for (np = of_find_node_by_name(dev->of_node, "emc-tables"); np;
+             np = of_find_node_by_name(np, "emc-tables")) {
+                err = of_property_read_u32(np, "nvidia,ram-code", &value);
+                if (err || value != ram_code) {
+                        of_node_put(np);
+                        continue;
+                }
+
+                return np;
+        }
+
+        dev_err(dev, "no memory timings for RAM code %u found in device tree\n",
+                ram_code);
+
+        return NULL;
+}
+
+static int emc_setup_hw(struct tegra_emc *emc)
+{
+        u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT;
+        u32 emc_cfg;
+
+        emc_cfg = readl_relaxed(emc->regs + EMC_CFG_2);
+
+        /*
+         * Depending on a memory type, DRAM should enter either self-refresh
+         * or power-down state on EMC clock change.
+         */
+        if (!(emc_cfg & EMC_CLKCHANGE_PD_ENABLE) &&
+            !(emc_cfg & EMC_CLKCHANGE_SR_ENABLE)) {
+                dev_err(emc->dev,
+                        "bootloader didn't specify DRAM auto-suspend mode\n");
+                return -EINVAL;
+        }
+
+        /* enable EMC and CAR to handshake on PLL divider/source changes */
+        emc_cfg |= EMC_CLKCHANGE_REQ_ENABLE;
+        writel_relaxed(emc_cfg, emc->regs + EMC_CFG_2);
+
+        /* initialize interrupt */
+        writel_relaxed(intmask, emc->regs + EMC_INTMASK);
+        writel_relaxed(intmask, emc->regs + EMC_INTSTATUS);
+
+        return 0;
+}
+
+static int emc_init(struct tegra_emc *emc, unsigned long rate)
+{
+        int err;
+
+        err = clk_set_parent(emc->emc_mux, emc->backup_clk);
+        if (err) {
+                dev_err(emc->dev,
+                        "failed to reparent to backup source: %d\n", err);
+                return err;
+        }
+
+        err = clk_set_rate(emc->pll_m, rate);
+        if (err) {
+                dev_err(emc->dev,
+                        "failed to change pll_m rate: %d\n", err);
+                return err;
+        }
+
+        err = clk_set_parent(emc->emc_mux, emc->pll_m);
+        if (err) {
+                dev_err(emc->dev,
+                        "failed to reparent to pll_m: %d\n", err);
+                return err;
+        }
+
+        err = clk_set_rate(emc->clk, rate);
+        if (err) {
+                dev_err(emc->dev,
+                        "failed to change emc rate: %d\n", err);
+                return err;
+        }
+
+        return 0;
+}
+
+static int tegra_emc_probe(struct platform_device *pdev)
+{
+        struct device_node *np;
+        struct tegra_emc *emc;
+        struct resource *res;
+        int irq, err;
+
+        /* driver has nothing to do in a case of memory timing absence */
+        if (of_get_child_count(pdev->dev.of_node) == 0) {
+                dev_info(&pdev->dev,
+                         "EMC device tree node doesn't have memory timings\n");
+                return 0;
+        }
+
+        irq = platform_get_irq(pdev, 0);
+        if (irq < 0) {
+                dev_err(&pdev->dev, "interrupt not specified\n");
+                dev_err(&pdev->dev, "please update your device tree\n");
+                return irq;
+        }
+
+        np = tegra_emc_find_node_by_ram_code(&pdev->dev);
+        if (!np)
+                return -EINVAL;
+
+        emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);
+        if (!emc) {
+                of_node_put(np);
+                return -ENOMEM;
+        }
+
+        init_completion(&emc->clk_handshake_complete);
+        emc->clk_nb.notifier_call = tegra_emc_clk_change_notify;
+        emc->dev = &pdev->dev;
+
+        err = tegra_emc_load_timings_from_dt(emc, np);
+        of_node_put(np);
+        if (err)
+                return err;
+
+        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        emc->regs = devm_ioremap_resource(&pdev->dev, res);
+        if (IS_ERR(emc->regs))
+                return PTR_ERR(emc->regs);
+
+        err = emc_setup_hw(emc);
+        if (err)
+                return err;
+
+        err = devm_request_irq(&pdev->dev, irq, tegra_emc_isr, 0,
+                               dev_name(&pdev->dev), emc);
+        if (err) {
+                dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", irq, err);
+                return err;
+        }
+
+        emc->clk = devm_clk_get(&pdev->dev, "emc");
+        if (IS_ERR(emc->clk)) {
+                err = PTR_ERR(emc->clk);
+                dev_err(&pdev->dev, "failed to get emc clock: %d\n", err);
+                return err;
+        }
+
+        emc->pll_m = clk_get_sys(NULL, "pll_m");
+        if (IS_ERR(emc->pll_m)) {
+                err = PTR_ERR(emc->pll_m);
+                dev_err(&pdev->dev, "failed to get pll_m clock: %d\n", err);
+                return err;
+        }
+
+        emc->backup_clk = clk_get_sys(NULL, "pll_p");
+        if (IS_ERR(emc->backup_clk)) {
+                err = PTR_ERR(emc->backup_clk);
+                dev_err(&pdev->dev, "failed to get pll_p clock: %d\n", err);
+                goto put_pll_m;
+        }
+
+        emc->emc_mux = clk_get_parent(emc->clk);
+        if (IS_ERR(emc->emc_mux)) {
+                err = PTR_ERR(emc->emc_mux);
+                dev_err(&pdev->dev, "failed to get emc_mux clock: %d\n", err);
+                goto put_backup;
+        }
+
+        err = clk_notifier_register(emc->clk, &emc->clk_nb);
+        if (err) {
+                dev_err(&pdev->dev, "failed to register clk notifier: %d\n",
+                        err);
+                goto put_backup;
+        }
+
+        /* set DRAM clock rate to maximum */
+        err = emc_init(emc, emc->timings[emc->num_timings - 1].rate);
+        if (err) {
+                dev_err(&pdev->dev, "failed to initialize EMC clock rate: %d\n",
+                        err);
+                goto unreg_notifier;
+        }
+
+        return 0;
+
+unreg_notifier:
+        clk_notifier_unregister(emc->clk, &emc->clk_nb);
+put_backup:
+        clk_put(emc->backup_clk);
+put_pll_m:
+        clk_put(emc->pll_m);
+
+        return err;
+}
+
+static const struct of_device_id tegra_emc_of_match[] = {
+        { .compatible = "nvidia,tegra20-emc", },
+        {},
+};
+
+static struct platform_driver tegra_emc_driver = {
+        .probe = tegra_emc_probe,
+        .driver = {
+                .name = "tegra20-emc",
+                .of_match_table = tegra_emc_of_match,
+                .suppress_bind_attrs = true,
+        },
+};
+
+static int __init tegra_emc_init(void)
+{
+        return platform_driver_register(&tegra_emc_driver);
+}
+subsys_initcall(tegra_emc_init);