drm/nv50/pm: introduce hwsq-based memory reclocking

More work needs to be done on supporting the different memory types.

v2 (Ben Skeggs):
- fixed up conflicts from not having pausing patch first
- restructured code somewhat to fit with how all the other code works
- fixed bug where incorrect mpll_ctrl could get set sometimes
- removed stuff that's cargo-culted from the binary driver
- merged nv92+ display disable into hwsq
- fixed incorrect opcode 0x5f magic at end of ucode

Signed-off-by: Martin Peres <martin.peres@ensi-bourges.fr>
Signed-off-by: Ben Skeggs <bskeggs@redhat.com>

2 files changed, 243 insertions, 62 deletions

diff --git a/drivers/gpu/drm/nouveau/nouveau_hwsq.h b/drivers/gpu/drm/nouveau/nouveau_hwsq.h
new file mode 100644
index 000000000000..d59a3b3ff644
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_hwsq.h
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifndef __NOUVEAU_HWSQ_H__
+#define __NOUVEAU_HWSQ_H__
+
+struct hwsq_ucode {
+        u8 data[0x200];
+        union {
+                u8  *u08;
+                u16 *u16;
+                u32 *u32;
+        } ptr;
+        u16 len;
+
+        u32 reg;
+        u32 val;
+};
+
+static inline void
+hwsq_init(struct hwsq_ucode *hwsq)
+{
+        hwsq->ptr.u08 = hwsq->data;
+        hwsq->reg = 0xffffffff;
+        hwsq->val = 0xffffffff;
+}
+
+static inline void
+hwsq_fini(struct hwsq_ucode *hwsq)
+{
+        do {
+                *hwsq->ptr.u08++ = 0x7f;
+                hwsq->len = hwsq->ptr.u08 - hwsq->data;
+        } while (hwsq->len & 3);
+        hwsq->ptr.u08 = hwsq->data;
+}
+
+static inline void
+hwsq_unkn(struct hwsq_ucode *hwsq, u8 v0)
+{
+        *hwsq->ptr.u08++ = v0;
+}
+
+static inline void
+hwsq_op5f(struct hwsq_ucode *hwsq, u8 v0, u8 v1)
+{
+        *hwsq->ptr.u08++ = 0x5f;
+        *hwsq->ptr.u08++ = v0;
+        *hwsq->ptr.u08++ = v1;
+}
+
+static inline void
+hwsq_wr32(struct hwsq_ucode *hwsq, u32 reg, u32 val)
+{
+        if (val != hwsq->val) {
+                if ((val & 0xffff0000) == (hwsq->val & 0xffff0000)) {
+                        *hwsq->ptr.u08++ = 0x42;
+                        *hwsq->ptr.u16++ = (val & 0x0000ffff);
+                } else {
+                        *hwsq->ptr.u08++ = 0xe2;
+                        *hwsq->ptr.u32++ = val;
+                }
+
+                hwsq->val = val;
+        }
+
+        if ((reg & 0xffff0000) == (hwsq->reg & 0xffff0000)) {
+                *hwsq->ptr.u08++ = 0x40;
+                *hwsq->ptr.u16++ = (reg & 0x0000ffff);
+        } else {
+                *hwsq->ptr.u08++ = 0xe0;
+                *hwsq->ptr.u32++ = reg;
+        }
+        hwsq->reg = reg;
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nv50_pm.c b/drivers/gpu/drm/nouveau/nv50_pm.c
index e025cae5ef23..22789db48969 100644
--- a/drivers/gpu/drm/nouveau/nv50_pm.c
+++ b/drivers/gpu/drm/nouveau/nv50_pm.c
@@ -27,6 +27,7 @@
 #include "nouveau_bios.h"
 #include "nouveau_hw.h"
 #include "nouveau_pm.h"
+#include "nouveau_hwsq.h"
 
 enum clk_src {
         clk_src_crystal,
@@ -351,6 +352,9 @@ nv50_pm_clocks_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 }
 
 struct nv50_pm_state {
+        struct hwsq_ucode mclk_hwsq;
+        u32 mscript;
+
         u32 emast;
         u32 nctrl;
         u32 ncoef;
@@ -359,10 +363,6 @@ struct nv50_pm_state {
 
         u32 amast;
         u32 pdivs;
-
-        u32 mscript;
-        u32 mctrl;
-        u32 mcoef;
 };
 
 static u32
@@ -415,6 +415,80 @@ clk_same(u32 a, u32 b)
         return ((a / 1000) == (b / 1000));
 }
 
+static int
+calc_mclk(struct drm_device *dev, u32 freq, struct hwsq_ucode *hwsq)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct pll_lims pll;
+        u32 mast = nv_rd32(dev, 0x00c040);
+        u32 ctrl = nv_rd32(dev, 0x004008);
+        u32 coef = nv_rd32(dev, 0x00400c);
+        u32 orig = ctrl;
+        u32 crtc_mask = 0;
+        int N, M, P;
+        int ret, i;
+
+        /* use pcie refclock if possible, otherwise use mpll */
+        ctrl &= ~0x81ff0200;
+        if (clk_same(freq, read_clk(dev, clk_src_href))) {
+                ctrl |= 0x00000200 | (pll.log2p_bias << 19);
+        } else {
+                ret = calc_pll(dev, 0x4008, &pll, freq, &N, &M, &P);
+                if (ret == 0)
+                        return -EINVAL;
+
+                ctrl |= 0x80000000 | (P << 22) | (P << 16);
+                ctrl |= pll.log2p_bias << 19;
+                coef  = (N << 8) | M;
+        }
+
+        mast &= ~0xc0000000; /* get MCLK_2 from HREF */
+        mast |=  0x0000c000; /* use MCLK_2 as MPLL_BYPASS clock */
+
+        /* determine active crtcs */
+        for (i = 0; i < 2; i++) {
+                if (nv_rd32(dev, NV50_PDISPLAY_CRTC_C(i, CLOCK)))
+                        crtc_mask |= (1 << i);
+        }
+
+        /* build the ucode which will reclock the memory for us */
+        hwsq_init(hwsq);
+        if (crtc_mask) {
+                hwsq_op5f(hwsq, crtc_mask, 0x00); /* wait for scanout */
+                hwsq_op5f(hwsq, crtc_mask, 0x01); /* wait for vblank */
+        }
+        if (dev_priv->chipset >= 0x92)
+                hwsq_wr32(hwsq, 0x611200, 0x00003300); /* disable scanout */
+        hwsq_unkn(hwsq, 0xb0); /* disable bus access */
+        hwsq_op5f(hwsq, 0x00, 0x01); /* no idea :s */
+
+        /* prepare memory controller */
+        hwsq_wr32(hwsq, 0x1002d4, 0x00000001); /* precharge banks and idle */
+        hwsq_wr32(hwsq, 0x1002d0, 0x00000001); /* force refresh */
+        hwsq_wr32(hwsq, 0x100210, 0x00000000); /* stop the automatic refresh */
+        hwsq_wr32(hwsq, 0x1002dc, 0x00000001); /* start self refresh mode */
+
+        /* reclock memory */
+        hwsq_wr32(hwsq, 0xc040, mast);
+        hwsq_wr32(hwsq, 0x4008, orig | 0x00000200); /* bypass MPLL */
+        hwsq_wr32(hwsq, 0x400c, coef);
+        hwsq_wr32(hwsq, 0x4008, ctrl);
+
+        /* restart memory controller */
+        hwsq_wr32(hwsq, 0x1002d4, 0x00000001); /* precharge banks and idle */
+        hwsq_wr32(hwsq, 0x1002dc, 0x00000000); /* stop self refresh mode */
+        hwsq_wr32(hwsq, 0x100210, 0x80000000); /* restart automatic refresh */
+        hwsq_unkn(hwsq, 0x07); /* wait for the PLL to stabilize (12us) */
+
+        hwsq_unkn(hwsq, 0x0b); /* may be unnecessary: causes flickering */
+        hwsq_unkn(hwsq, 0xd0); /* enable bus access again */
+        hwsq_op5f(hwsq, 0x00, 0x00); /* no idea, reverse of 0x00, 0x01? */
+        if (dev_priv->chipset >= 0x92)
+                hwsq_wr32(hwsq, 0x611200, 0x00003330); /* enable scanout */
+        hwsq_fini(hwsq);
+        return 0;
+}
+
 void *
 nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 {
@@ -462,23 +536,16 @@ nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
                 info->scoef  = (N << 8) | M;
         }
 
-        /* memory: use pcie refclock if possible, otherwise use mpll */
-        info->mscript = perflvl->memscript;
-        if (clk_same(perflvl->memory, read_clk(dev, clk_src_href))) {
-                info->mctrl = 0x00000200 | (pll.log2p_bias << 19);
-                info->mcoef = nv_rd32(dev, 0x400c);
-        } else
+        /* memory: build hwsq ucode which we'll use to reclock memory */
+        info->mclk_hwsq.len = 0;
         if (perflvl->memory) {
-                clk = calc_pll(dev, 0x4008, &pll, perflvl->memory,
-                               &N, &M, &P1);
-                if (clk == 0)
+                clk = calc_mclk(dev, perflvl->memory, &info->mclk_hwsq);
+                if (clk < 0) {
+                        ret = clk;
                         goto error;
+                }
 
-                info->mctrl  = 0x80000000 | (P1 << 22) | (P1 << 16);
-                info->mctrl |= pll.log2p_bias << 19;
-                info->mcoef  = (N << 8) | M;
-        } else {
-                info->mctrl = 0x00000000;
+                info->mscript = perflvl->memscript;
         }
 
         /* vdec: avoid modifying xpll until we know exactly how the other
@@ -537,6 +604,44 @@ error:
         return ERR_PTR(ret);
 }
 
+static int
+prog_mclk(struct drm_device *dev, struct hwsq_ucode *hwsq)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        u32 hwsq_data, hwsq_kick;
+        int i;
+
+        if (dev_priv->chipset < 0x90) {
+                hwsq_data = 0x001400;
+                hwsq_kick = 0x00000003;
+        } else {
+                hwsq_data = 0x080000;
+                hwsq_kick = 0x00000001;
+        }
+
+        /* upload hwsq ucode */
+        nv_mask(dev, 0x001098, 0x00000008, 0x00000000);
+        nv_wr32(dev, 0x001304, 0x00000000);
+        for (i = 0; i < hwsq->len / 4; i++)
+                nv_wr32(dev, hwsq_data + (i * 4), hwsq->ptr.u32[i]);
+        nv_mask(dev, 0x001098, 0x00000018, 0x00000018);
+
+        /* launch, and wait for completion */
+        nv_wr32(dev, 0x00130c, hwsq_kick);
+        if (!nv_wait(dev, 0x001308, 0x00000100, 0x00000000)) {
+                NV_ERROR(dev, "hwsq ucode exec timed out\n");
+                NV_ERROR(dev, "0x001308: 0x%08x\n", nv_rd32(dev, 0x001308));
+                for (i = 0; i < hwsq->len / 4; i++) {
+                        NV_ERROR(dev, "0x%06x: 0x%08x\n", 0x1400 + (i * 4),
+                                 nv_rd32(dev, 0x001400 + (i * 4)));
+                }
+
+                return -EIO;
+        }
+
+        return 0;
+}
+
 int
 nv50_pm_clocks_set(struct drm_device *dev, void *data)
 {
@@ -550,6 +655,28 @@ nv50_pm_clocks_set(struct drm_device *dev, void *data)
         if (!nv_wait(dev, 0x002504, 0x00000010, 0x00000010))
                 goto error;
 
+        /* memory: it is *very* important we change this first, the ucode
+         * we build in pre() now has hardcoded 0xc040 values, which can't
+         * change before we execute it or the engine clocks may end up
+         * messed up.
+         */
+        if (info->mclk_hwsq.len) {
+                /* execute some scripts that do ??? from the vbios.. */
+                if (!bit_table(dev, 'M', &M) && M.version == 1) {
+                        if (M.length >= 6)
+                                nouveau_bios_init_exec(dev, ROM16(M.data[5]));
+                        if (M.length >= 8)
+                                nouveau_bios_init_exec(dev, ROM16(M.data[7]));
+                        if (M.length >= 10)
+                                nouveau_bios_init_exec(dev, ROM16(M.data[9]));
+                        nouveau_bios_init_exec(dev, info->mscript);
+                }
+
+                ret = prog_mclk(dev, &info->mclk_hwsq);
+                if (ret)
+                        goto resume;
+        }
+
         /* reclock vdec/dom6 */
         nv_mask(dev, 0x00c040, 0x00000c00, 0x00000000);
         switch (dev_priv->chipset) {
@@ -578,50 +705,6 @@ nv50_pm_clocks_set(struct drm_device *dev, void *data)
         nv_wr32(dev, 0x00402c, info->ncoef);
         nv_mask(dev, 0x00c040, 0x00100033, info->emast);
 
-        /* memory */
-        if (!info->mctrl)
-                goto resume;
-
-        /* execute some scripts that do ??? from the vbios.. */
-        if (!bit_table(dev, 'M', &M) && M.version == 1) {
-                if (M.length >= 6)
-                        nouveau_bios_init_exec(dev, ROM16(M.data[5]));
-                if (M.length >= 8)
-                        nouveau_bios_init_exec(dev, ROM16(M.data[7]));
-                if (M.length >= 10)
-                        nouveau_bios_init_exec(dev, ROM16(M.data[9]));
-                nouveau_bios_init_exec(dev, info->mscript);
-        }
-
-        /* disable display */
-        if (dev_priv->chipset >= 0x92) {
-                nv_wr32(dev, 0x611200, 0x00003300);
-                udelay(100);
-        }
-
-        /* prepare ram for reclocking */
-        nv_wr32(dev, 0x1002d4, 0x00000001); /* precharge */
-        nv_wr32(dev, 0x1002d0, 0x00000001); /* refresh */
-        nv_wr32(dev, 0x1002d0, 0x00000001); /* refresh */
-        nv_mask(dev, 0x100210, 0x80000000, 0x00000000); /* no auto-refresh */
-        nv_wr32(dev, 0x1002dc, 0x00000001); /* enable self-refresh */
-
-        /* modify mpll */
-        nv_mask(dev, 0x00c040, 0x0000c000, 0x0000c000);
-        nv_mask(dev, 0x004008, 0x01ff0200, 0x00000200 | info->mctrl);
-        nv_wr32(dev, 0x00400c, info->mcoef);
-        udelay(100);
-        nv_mask(dev, 0x004008, 0x81ff0200, info->mctrl);
-
-        /* re-enable normal operation of memory controller */
-        nv_wr32(dev, 0x1002dc, 0x00000000);
-        nv_mask(dev, 0x100210, 0x80000000, 0x80000000);
-        udelay(100);
-
-        /* re-enable display */
-        if (dev_priv->chipset >= 0x92)
-                nv_wr32(dev, 0x611200, 0x00003330);
-
         goto resume;
 error:
         ret = -EBUSY;