Merge branch 'drm-next' of git://people.freedesktop.org/~airlied/linux

Pull drm updates from Dave Airlie:
 "Highlights:

   - AMD KFD driver merge

     This is the AMD HSA interface for exposing a lowlevel interface for
     GPGPU use.  They have an open source userspace built on top of this
     interface, and the code looks as good as it was going to get out of
     tree.

   - Initial atomic modesetting work

     The need for an atomic modesetting interface to allow userspace to
     try and send a complete set of modesetting state to the driver has
     arisen, and been suffering from neglect this past year.  No more,
     the start of the common code and changes for msm driver to use it
     are in this tree.  Ongoing work to get the userspace ioctl finished
     and the code clean will probably wait until next kernel.

   - DisplayID 1.3 and tiled monitor exposed to userspace.

     Tiled monitor property is now exposed for userspace to make use of.

   - Rockchip drm driver merged.

   - imx gpu driver moved out of staging

  Other stuff:

   - core:
        panel - MIPI DSI + new panels.
        expose suggested x/y properties for virtual GPUs

   - i915:
        Initial Skylake (SKL) support
        gen3/4 reset work
        start of dri1/ums removal
        infoframe tracking
        fixes for lots of things.

   - nouveau:
        tegra k1 voltage support
        GM204 modesetting support
        GT21x memory reclocking work

   - radeon:
        CI dpm fixes
        GPUVM improvements
        Initial DPM fan control

   - rcar-du:
        HDMI support added
        removed some support for old boards
        slave encoder driver for Analog Devices adv7511

   - exynos:
        Exynos4415 SoC support

   - msm:
        a4xx gpu support
        atomic helper conversion

   - tegra:
        iommu support
        universal plane support
        ganged-mode DSI support

   - sti:
        HDMI i2c improvements

   - vmwgfx:
        some late fixes.

   - qxl:
        use suggested x/y properties"

* 'drm-next' of git://people.freedesktop.org/~airlied/linux: (969 commits)
  drm: sti: fix module compilation issue
  drm/i915: save/restore GMBUS freq across suspend/resume on gen4
  drm: sti: correctly cleanup CRTC and planes
  drm: sti: add HQVDP plane
  drm: sti: add cursor plane
  drm: sti: enable auxiliary CRTC
  drm: sti: fix delay in VTG programming
  drm: sti: prepare sti_tvout to support auxiliary crtc
  drm: sti: use drm_crtc_vblank_{on/off} instead of drm_vblank_{on/off}
  drm: sti: fix hdmi avi infoframe
  drm: sti: remove event lock while disabling vblank
  drm: sti: simplify gdp code
  drm: sti: clear all mixer control
  drm: sti: remove gpio for HDMI hot plug detection
  drm: sti: allow to change hdmi ddc i2c adapter
  drm/doc: Document drm_add_modes_noedid() usage
  drm/i915: Remove '& 0xffff' from the mask given to WA_REG()
  drm/i915: Invert the mask and val arguments in wa_add() and WA_REG()
  drm: Zero out DRM object memory upon cleanup
  drm/i915/bdw: Fix the write setting up the WIZ hashing mode
  ...

103 files changed, 5799 insertions, 2361 deletions

diff --git a/drivers/gpu/drm/nouveau/core/core/handle.c b/drivers/gpu/drm/nouveau/core/core/handle.c
index a490b805d7e3..13f816cb08bd 100644
--- a/drivers/gpu/drm/nouveau/core/core/handle.c
+++ b/drivers/gpu/drm/nouveau/core/core/handle.c
@@ -222,116 +222,3 @@ nouveau_handle_put(struct nouveau_handle *handle)
         if (handle)
                 nouveau_namedb_put(handle);
 }
-
-int
-nouveau_handle_new(struct nouveau_object *client, u32 _parent, u32 _handle,
-                   u16 _oclass, void *data, u32 size,
-                   struct nouveau_object **pobject)
-{
-        struct nouveau_object *parent = NULL;
-        struct nouveau_object *engctx = NULL;
-        struct nouveau_object *object = NULL;
-        struct nouveau_object *engine;
-        struct nouveau_oclass *oclass;
-        struct nouveau_handle *handle;
-        int ret;
-
-        /* lookup parent object and ensure it *is* a parent */
-        parent = nouveau_handle_ref(client, _parent);
-        if (!parent) {
-                nv_error(client, "parent 0x%08x not found\n", _parent);
-                return -ENOENT;
-        }
-
-        if (!nv_iclass(parent, NV_PARENT_CLASS)) {
-                nv_error(parent, "cannot have children\n");
-                ret = -EINVAL;
-                goto fail_class;
-        }
-
-        /* check that parent supports the requested subclass */
-        ret = nouveau_parent_sclass(parent, _oclass, &engine, &oclass);
-        if (ret) {
-                nv_debug(parent, "illegal class 0x%04x\n", _oclass);
-                goto fail_class;
-        }
-
-        /* make sure engine init has been completed *before* any objects
-         * it controls are created - the constructors may depend on
-         * state calculated at init (ie. default context construction)
-         */
-        if (engine) {
-                ret = nouveau_object_inc(engine);
-                if (ret)
-                        goto fail_class;
-        }
-
-        /* if engine requires it, create a context object to insert
-         * between the parent and its children (eg. PGRAPH context)
-         */
-        if (engine && nv_engine(engine)->cclass) {
-                ret = nouveau_object_ctor(parent, engine,
-                                          nv_engine(engine)->cclass,
-                                          data, size, &engctx);
-                if (ret)
-                        goto fail_engctx;
-        } else {
-                nouveau_object_ref(parent, &engctx);
-        }
-
-        /* finally, create new object and bind it to its handle */
-        ret = nouveau_object_ctor(engctx, engine, oclass, data, size, &object);
-        *pobject = object;
-        if (ret)
-                goto fail_ctor;
-
-        ret = nouveau_object_inc(object);
-        if (ret)
-                goto fail_init;
-
-        ret = nouveau_handle_create(parent, _parent, _handle, object, &handle);
-        if (ret)
-                goto fail_handle;
-
-        ret = nouveau_handle_init(handle);
-        if (ret)
-                nouveau_handle_destroy(handle);
-
-fail_handle:
-        nouveau_object_dec(object, false);
-fail_init:
-        nouveau_object_ref(NULL, &object);
-fail_ctor:
-        nouveau_object_ref(NULL, &engctx);
-fail_engctx:
-        if (engine)
-                nouveau_object_dec(engine, false);
-fail_class:
-        nouveau_object_ref(NULL, &parent);
-        return ret;
-}
-
-int
-nouveau_handle_del(struct nouveau_object *client, u32 _parent, u32 _handle)
-{
-        struct nouveau_object *parent = NULL;
-        struct nouveau_object *namedb = NULL;
-        struct nouveau_handle *handle = NULL;
-
-        parent = nouveau_handle_ref(client, _parent);
-        if (!parent)
-                return -ENOENT;
-
-        namedb = nv_pclass(parent, NV_NAMEDB_CLASS);
-        if (namedb) {
-                handle = nouveau_namedb_get(nv_namedb(namedb), _handle);
-                if (handle) {
-                        nouveau_namedb_put(handle);
-                        nouveau_handle_fini(handle, false);
-                        nouveau_handle_destroy(handle);
-                }
-        }
-
-        nouveau_object_ref(NULL, &parent);
-        return handle ? 0 : -EINVAL;
-}
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/base.c b/drivers/gpu/drm/nouveau/core/engine/device/base.c
index 0ef5a5713182..137e0b0faeae 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/base.c
@@ -29,6 +29,7 @@
 #include <nvif/unpack.h>
 #include <nvif/class.h>
 
+#include <subdev/bios.h>
 #include <subdev/fb.h>
 #include <subdev/instmem.h>
 
@@ -138,7 +139,7 @@ nouveau_devobj_info(struct nouveau_object *object, void *data, u32 size)
         }
 
         args->v0.chipset  = device->chipset;
-        args->v0.revision = device->chipset >= 0x10 ? nv_rd32(device, 0) : 0x00;
+        args->v0.revision = device->chiprev;
         if (pfb)  args->v0.ram_size = args->v0.ram_user = pfb->ram->size;
         else      args->v0.ram_size = args->v0.ram_user = 0;
         if (imem) args->v0.ram_user = args->v0.ram_user - imem->reserved;
@@ -222,6 +223,7 @@ static const u64 disable_map[] = {
         [NVDEV_SUBDEV_VOLT]     = NV_DEVICE_V0_DISABLE_CORE,
         [NVDEV_SUBDEV_THERM]    = NV_DEVICE_V0_DISABLE_CORE,
         [NVDEV_SUBDEV_PWR]      = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_FUSE]     = NV_DEVICE_V0_DISABLE_CORE,
         [NVDEV_ENGINE_DMAOBJ]   = NV_DEVICE_V0_DISABLE_CORE,
         [NVDEV_ENGINE_PERFMON]  = NV_DEVICE_V0_DISABLE_CORE,
         [NVDEV_ENGINE_FIFO]     = NV_DEVICE_V0_DISABLE_FIFO,
@@ -235,6 +237,7 @@ static const u64 disable_map[] = {
         [NVDEV_ENGINE_PPP]      = NV_DEVICE_V0_DISABLE_PPP,
         [NVDEV_ENGINE_COPY0]    = NV_DEVICE_V0_DISABLE_COPY0,
         [NVDEV_ENGINE_COPY1]    = NV_DEVICE_V0_DISABLE_COPY1,
+        [NVDEV_ENGINE_COPY2]    = NV_DEVICE_V0_DISABLE_COPY1,
         [NVDEV_ENGINE_VIC]      = NV_DEVICE_V0_DISABLE_VIC,
         [NVDEV_ENGINE_VENC]     = NV_DEVICE_V0_DISABLE_VENC,
         [NVDEV_ENGINE_DISP]     = NV_DEVICE_V0_DISABLE_DISP,
@@ -352,12 +355,14 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                 /* determine chipset and derive architecture from it */
                 if ((boot0 & 0x1f000000) > 0) {
                         device->chipset = (boot0 & 0x1ff00000) >> 20;
+                        device->chiprev = (boot0 & 0x000000ff);
                         switch (device->chipset & 0x1f0) {
                         case 0x010: {
                                 if (0x461 & (1 << (device->chipset & 0xf)))
                                         device->card_type = NV_10;
                                 else
                                         device->card_type = NV_11;
+                                device->chiprev = 0x00;
                                 break;
                         }
                         case 0x020: device->card_type = NV_20; break;
@@ -373,7 +378,8 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                         case 0x0e0:
                         case 0x0f0:
                         case 0x100: device->card_type = NV_E0; break;
-                        case 0x110: device->card_type = GM100; break;
+                        case 0x110:
+                        case 0x120: device->card_type = GM100; break;
                         default:
                                 break;
                         }
@@ -427,6 +433,10 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                 }
 
                 nv_debug(device, "crystal freq: %dKHz\n", device->crystal);
+        } else
+        if ( (args->v0.disable & NV_DEVICE_V0_DISABLE_IDENTIFY)) {
+                device->cname = "NULL";
+                device->oclass[NVDEV_SUBDEV_VBIOS] = &nouveau_bios_oclass;
         }
 
         if (!(args->v0.disable & NV_DEVICE_V0_DISABLE_MMIO) &&
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/gm100.c b/drivers/gpu/drm/nouveau/core/engine/device/gm100.c
index 6295668e29a5..4e74a3376de8 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/gm100.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/gm100.c
@@ -98,6 +98,49 @@ gm100_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
 #endif
                 break;
+        case 0x124:
+                device->cname = "GM204";
+                device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+                device->oclass[NVDEV_SUBDEV_GPIO   ] =  nve0_gpio_oclass;
+                device->oclass[NVDEV_SUBDEV_I2C    ] =  gm204_i2c_oclass;
+                device->oclass[NVDEV_SUBDEV_FUSE   ] = &gm107_fuse_oclass;
+#if 0
+                /* looks to be some non-trivial changes */
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nve0_clock_oclass;
+                /* priv ring says no to 0x10eb14 writes */
+                device->oclass[NVDEV_SUBDEV_THERM  ] = &gm107_therm_oclass;
+#endif
+                device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+                device->oclass[NVDEV_SUBDEV_DEVINIT] =  gm204_devinit_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  gk20a_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_TIMER  ] = &gk20a_timer_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  gm107_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gm107_ltc_oclass;
+                device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
+                device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
+                device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nv108_pwr_oclass;
+#if 0
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
+#endif
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
+#if 0
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv108_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_GR     ] =  gm107_graph_oclass;
+#endif
+                device->oclass[NVDEV_ENGINE_DISP   ] =  gm204_disp_oclass;
+#if 0
+                device->oclass[NVDEV_ENGINE_COPY0  ] = &gm204_copy0_oclass;
+                device->oclass[NVDEV_ENGINE_COPY1  ] = &gm204_copy1_oclass;
+                device->oclass[NVDEV_ENGINE_COPY2  ] = &gm204_copy2_oclass;
+                device->oclass[NVDEV_ENGINE_BSP    ] = &nve0_bsp_oclass;
+                device->oclass[NVDEV_ENGINE_VP     ] = &nve0_vp_oclass;
+                device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
+#endif
+                break;
         default:
                 nv_fatal(device, "unknown Maxwell chipset\n");
                 return -EINVAL;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nve0.c b/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
index b1b2e484ecfa..674da1f095b2 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
@@ -179,6 +179,7 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_GR     ] =  gk20a_graph_oclass;
                 device->oclass[NVDEV_ENGINE_COPY2  ] = &nve0_copy2_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] = &nve0_perfmon_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &gk20a_volt_oclass;
                 break;
         case 0xf0:
                 device->cname = "GK110";
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/dport.c b/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
index 39890221b91c..16db08dfba6e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
@@ -28,7 +28,7 @@
 #include <subdev/bios/init.h>
 #include <subdev/i2c.h>
 
-#include <engine/disp.h>
+#include "nv50.h"
 
 #include <nvif/class.h>
 
@@ -326,7 +326,7 @@ void
 nouveau_dp_train(struct work_struct *w)
 {
         struct nvkm_output_dp *outp = container_of(w, typeof(*outp), lt.work);
-        struct nouveau_disp *disp = nouveau_disp(outp);
+        struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
         const struct dp_rates *cfg = nouveau_dp_rates;
         struct dp_state _dp = {
                 .outp = outp,
@@ -334,8 +334,11 @@ nouveau_dp_train(struct work_struct *w)
         u32 datarate = 0;
         int ret;
 
+        if (!outp->base.info.location && priv->sor.magic)
+                priv->sor.magic(&outp->base);
+
         /* bring capabilities within encoder limits */
-        if (nv_mclass(disp) < GF110_DISP)
+        if (nv_mclass(priv) < GF110_DISP)
                 outp->dpcd[2] &= ~DPCD_RC02_TPS3_SUPPORTED;
         if ((outp->dpcd[2] & 0x1f) > outp->base.info.dpconf.link_nr) {
                 outp->dpcd[2] &= ~DPCD_RC02_MAX_LANE_COUNT;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c b/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c
index b3df3fe2dc09..e2ad0543fb31 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c
@@ -35,8 +35,8 @@
 
 static struct nouveau_oclass
 gm107_disp_sclass[] = {
-        { GM107_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
-        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GM107_DISP_CORE_CHANNEL_DMA, &nvd0_disp_core_ofuncs.base },
+        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_base_ofuncs.base },
         { GK104_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
         { GK104_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
         { GK104_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
@@ -44,8 +44,8 @@ gm107_disp_sclass[] = {
 };
 
 static struct nouveau_oclass
-gm107_disp_base_oclass[] = {
-        { GM107_DISP, &nvd0_disp_base_ofuncs },
+gm107_disp_main_oclass[] = {
+        { GM107_DISP, &nvd0_disp_main_ofuncs },
         {}
 };
 
@@ -72,7 +72,7 @@ gm107_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_engine(priv)->sclass = gm107_disp_base_oclass;
+        nv_engine(priv)->sclass = gm107_disp_main_oclass;
         nv_engine(priv)->cclass = &nv50_disp_cclass;
         nv_subdev(priv)->intr = nvd0_disp_intr;
         INIT_WORK(&priv->supervisor, nvd0_disp_intr_supervisor);
@@ -99,9 +99,9 @@ gm107_disp_oclass = &(struct nv50_disp_impl) {
         },
         .base.vblank = &nvd0_disp_vblank_func,
         .base.outp =  nvd0_disp_outp_sclass,
-        .mthd.core = &nve0_disp_mast_mthd_chan,
-        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.core = &nve0_disp_core_mthd_chan,
+        .mthd.base = &nvd0_disp_base_mthd_chan,
         .mthd.ovly = &nve0_disp_ovly_mthd_chan,
         .mthd.prev = -0x020000,
-        .head.scanoutpos = nvd0_disp_base_scanoutpos,
+        .head.scanoutpos = nvd0_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/gm204.c b/drivers/gpu/drm/nouveau/core/engine/disp/gm204.c
new file mode 100644
index 000000000000..672ded79b2a9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/gm204.c
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <engine/software.h>
+#include <engine/disp.h>
+
+#include <nvif/class.h>
+
+#include "nv50.h"
+
+/*******************************************************************************
+ * Base display object
+ ******************************************************************************/
+
+static struct nouveau_oclass
+gm204_disp_sclass[] = {
+        { GM204_DISP_CORE_CHANNEL_DMA, &nvd0_disp_core_ofuncs.base },
+        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_base_ofuncs.base },
+        { GK104_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
+        { GK104_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
+        { GK104_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
+        {}
+};
+
+static struct nouveau_oclass
+gm204_disp_main_oclass[] = {
+        { GM204_DISP, &nvd0_disp_main_ofuncs },
+        {}
+};
+
+/*******************************************************************************
+ * Display engine implementation
+ ******************************************************************************/
+
+static int
+gm204_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+               struct nouveau_oclass *oclass, void *data, u32 size,
+               struct nouveau_object **pobject)
+{
+        struct nv50_disp_priv *priv;
+        int heads = nv_rd32(parent, 0x022448);
+        int ret;
+
+        ret = nouveau_disp_create(parent, engine, oclass, heads,
+                                  "PDISP", "display", &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        ret = nvkm_event_init(&nvd0_disp_chan_uevent, 1, 17, &priv->uevent);
+        if (ret)
+                return ret;
+
+        nv_engine(priv)->sclass = gm204_disp_main_oclass;
+        nv_engine(priv)->cclass = &nv50_disp_cclass;
+        nv_subdev(priv)->intr = nvd0_disp_intr;
+        INIT_WORK(&priv->supervisor, nvd0_disp_intr_supervisor);
+        priv->sclass = gm204_disp_sclass;
+        priv->head.nr = heads;
+        priv->dac.nr = 3;
+        priv->sor.nr = 4;
+        priv->dac.power = nv50_dac_power;
+        priv->dac.sense = nv50_dac_sense;
+        priv->sor.power = nv50_sor_power;
+        priv->sor.hda_eld = nvd0_hda_eld;
+        priv->sor.hdmi = nvd0_hdmi_ctrl;
+        priv->sor.magic = gm204_sor_magic;
+        return 0;
+}
+
+struct nouveau_oclass *
+gm204_disp_outp_sclass[] = {
+        &gm204_sor_dp_impl.base.base,
+        NULL
+};
+
+struct nouveau_oclass *
+gm204_disp_oclass = &(struct nv50_disp_impl) {
+        .base.base.handle = NV_ENGINE(DISP, 0x07),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gm204_disp_ctor,
+                .dtor = _nouveau_disp_dtor,
+                .init = _nouveau_disp_init,
+                .fini = _nouveau_disp_fini,
+        },
+        .base.vblank = &nvd0_disp_vblank_func,
+        .base.outp =  gm204_disp_outp_sclass,
+        .mthd.core = &nve0_disp_core_mthd_chan,
+        .mthd.base = &nvd0_disp_base_mthd_chan,
+        .mthd.ovly = &nve0_disp_ovly_mthd_chan,
+        .mthd.prev = -0x020000,
+        .head.scanoutpos = nvd0_disp_main_scanoutpos,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
index 2df3a937037d..44a8290aaea5 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
@@ -88,12 +88,14 @@ nv50_disp_chan_uevent_fini(struct nvkm_event *event, int type, int index)
 {
         struct nv50_disp_priv *priv = container_of(event, typeof(*priv), uevent);
         nv_mask(priv, 0x610028, 0x00000001 << index, 0x00000000 << index);
+        nv_wr32(priv, 0x610020, 0x00000001 << index);
 }
 
 static void
 nv50_disp_chan_uevent_init(struct nvkm_event *event, int types, int index)
 {
         struct nv50_disp_priv *priv = container_of(event, typeof(*priv), uevent);
+        nv_wr32(priv, 0x610020, 0x00000001 << index);
         nv_mask(priv, 0x610028, 0x00000001 << index, 0x00000001 << index);
 }
 
@@ -374,7 +376,7 @@ nv50_disp_mthd_chan(struct nv50_disp_priv *priv, int debug, int head,
 }
 
 const struct nv50_disp_mthd_list
-nv50_disp_mast_mthd_base = {
+nv50_disp_core_mthd_base = {
         .mthd = 0x0000,
         .addr = 0x000000,
         .data = {
@@ -387,7 +389,7 @@ nv50_disp_mast_mthd_base = {
 };
 
 static const struct nv50_disp_mthd_list
-nv50_disp_mast_mthd_dac = {
+nv50_disp_core_mthd_dac = {
         .mthd = 0x0080,
         .addr = 0x000008,
         .data = {
@@ -399,7 +401,7 @@ nv50_disp_mast_mthd_dac = {
 };
 
 const struct nv50_disp_mthd_list
-nv50_disp_mast_mthd_sor = {
+nv50_disp_core_mthd_sor = {
         .mthd = 0x0040,
         .addr = 0x000008,
         .data = {
@@ -409,7 +411,7 @@ nv50_disp_mast_mthd_sor = {
 };
 
 const struct nv50_disp_mthd_list
-nv50_disp_mast_mthd_pior = {
+nv50_disp_core_mthd_pior = {
         .mthd = 0x0040,
         .addr = 0x000008,
         .data = {
@@ -419,7 +421,7 @@ nv50_disp_mast_mthd_pior = {
 };
 
 static const struct nv50_disp_mthd_list
-nv50_disp_mast_mthd_head = {
+nv50_disp_core_mthd_head = {
         .mthd = 0x0400,
         .addr = 0x000540,
         .data = {
@@ -466,21 +468,21 @@ nv50_disp_mast_mthd_head = {
 };
 
 static const struct nv50_disp_mthd_chan
-nv50_disp_mast_mthd_chan = {
+nv50_disp_core_mthd_chan = {
         .name = "Core",
         .addr = 0x000000,
         .data = {
-                { "Global", 1, &nv50_disp_mast_mthd_base },
-                {    "DAC", 3, &nv50_disp_mast_mthd_dac  },
-                {    "SOR", 2, &nv50_disp_mast_mthd_sor  },
-                {   "PIOR", 3, &nv50_disp_mast_mthd_pior },
-                {   "HEAD", 2, &nv50_disp_mast_mthd_head },
+                { "Global", 1, &nv50_disp_core_mthd_base },
+                {    "DAC", 3, &nv50_disp_core_mthd_dac  },
+                {    "SOR", 2, &nv50_disp_core_mthd_sor  },
+                {   "PIOR", 3, &nv50_disp_core_mthd_pior },
+                {   "HEAD", 2, &nv50_disp_core_mthd_head },
                 {}
         }
 };
 
 int
-nv50_disp_mast_ctor(struct nouveau_object *parent,
+nv50_disp_core_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
@@ -509,7 +511,7 @@ nv50_disp_mast_ctor(struct nouveau_object *parent,
 }
 
 static int
-nv50_disp_mast_init(struct nouveau_object *object)
+nv50_disp_core_init(struct nouveau_object *object)
 {
         struct nv50_disp_priv *priv = (void *)object->engine;
         struct nv50_disp_dmac *mast = (void *)object;
@@ -546,7 +548,7 @@ nv50_disp_mast_init(struct nouveau_object *object)
 }
 
 static int
-nv50_disp_mast_fini(struct nouveau_object *object, bool suspend)
+nv50_disp_core_fini(struct nouveau_object *object, bool suspend)
 {
         struct nv50_disp_priv *priv = (void *)object->engine;
         struct nv50_disp_dmac *mast = (void *)object;
@@ -567,11 +569,11 @@ nv50_disp_mast_fini(struct nouveau_object *object, bool suspend)
 }
 
 struct nv50_disp_chan_impl
-nv50_disp_mast_ofuncs = {
-        .base.ctor = nv50_disp_mast_ctor,
+nv50_disp_core_ofuncs = {
+        .base.ctor = nv50_disp_core_ctor,
         .base.dtor = nv50_disp_dmac_dtor,
-        .base.init = nv50_disp_mast_init,
-        .base.fini = nv50_disp_mast_fini,
+        .base.init = nv50_disp_core_init,
+        .base.fini = nv50_disp_core_fini,
         .base.map  = nv50_disp_chan_map,
         .base.ntfy = nv50_disp_chan_ntfy,
         .base.rd32 = nv50_disp_chan_rd32,
@@ -586,7 +588,7 @@ nv50_disp_mast_ofuncs = {
  ******************************************************************************/
 
 static const struct nv50_disp_mthd_list
-nv50_disp_sync_mthd_base = {
+nv50_disp_base_mthd_base = {
         .mthd = 0x0000,
         .addr = 0x000000,
         .data = {
@@ -611,7 +613,7 @@ nv50_disp_sync_mthd_base = {
 };
 
 const struct nv50_disp_mthd_list
-nv50_disp_sync_mthd_image = {
+nv50_disp_base_mthd_image = {
         .mthd = 0x0400,
         .addr = 0x000000,
         .data = {
@@ -625,18 +627,18 @@ nv50_disp_sync_mthd_image = {
 };
 
 static const struct nv50_disp_mthd_chan
-nv50_disp_sync_mthd_chan = {
+nv50_disp_base_mthd_chan = {
         .name = "Base",
         .addr = 0x000540,
         .data = {
-                { "Global", 1, &nv50_disp_sync_mthd_base },
-                {  "Image", 2, &nv50_disp_sync_mthd_image },
+                { "Global", 1, &nv50_disp_base_mthd_base },
+                {  "Image", 2, &nv50_disp_base_mthd_image },
                 {}
         }
 };
 
 int
-nv50_disp_sync_ctor(struct nouveau_object *parent,
+nv50_disp_base_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
@@ -669,8 +671,8 @@ nv50_disp_sync_ctor(struct nouveau_object *parent,
 }
 
 struct nv50_disp_chan_impl
-nv50_disp_sync_ofuncs = {
-        .base.ctor = nv50_disp_sync_ctor,
+nv50_disp_base_ofuncs = {
+        .base.ctor = nv50_disp_base_ctor,
         .base.dtor = nv50_disp_dmac_dtor,
         .base.init = nv50_disp_dmac_init,
         .base.fini = nv50_disp_dmac_fini,
@@ -942,7 +944,7 @@ nv50_disp_curs_ofuncs = {
  ******************************************************************************/
 
 int
-nv50_disp_base_scanoutpos(NV50_DISP_MTHD_V0)
+nv50_disp_main_scanoutpos(NV50_DISP_MTHD_V0)
 {
         const u32 blanke = nv_rd32(priv, 0x610aec + (head * 0x540));
         const u32 blanks = nv_rd32(priv, 0x610af4 + (head * 0x540));
@@ -974,7 +976,7 @@ nv50_disp_base_scanoutpos(NV50_DISP_MTHD_V0)
 }
 
 int
-nv50_disp_base_mthd(struct nouveau_object *object, u32 mthd,
+nv50_disp_main_mthd(struct nouveau_object *object, u32 mthd,
                     void *data, u32 size)
 {
         const struct nv50_disp_impl *impl = (void *)nv_oclass(object->engine);
@@ -1098,7 +1100,7 @@ nv50_disp_base_mthd(struct nouveau_object *object, u32 mthd,
 }
 
 int
-nv50_disp_base_ctor(struct nouveau_object *parent,
+nv50_disp_main_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
                     struct nouveau_oclass *oclass, void *data, u32 size,
                     struct nouveau_object **pobject)
@@ -1118,7 +1120,7 @@ nv50_disp_base_ctor(struct nouveau_object *parent,
 }
 
 void
-nv50_disp_base_dtor(struct nouveau_object *object)
+nv50_disp_main_dtor(struct nouveau_object *object)
 {
         struct nv50_disp_base *base = (void *)object;
         nouveau_ramht_ref(NULL, &base->ramht);
@@ -1126,7 +1128,7 @@ nv50_disp_base_dtor(struct nouveau_object *object)
 }
 
 static int
-nv50_disp_base_init(struct nouveau_object *object)
+nv50_disp_main_init(struct nouveau_object *object)
 {
         struct nv50_disp_priv *priv = (void *)object->engine;
         struct nv50_disp_base *base = (void *)object;
@@ -1194,7 +1196,7 @@ nv50_disp_base_init(struct nouveau_object *object)
 }
 
 static int
-nv50_disp_base_fini(struct nouveau_object *object, bool suspend)
+nv50_disp_main_fini(struct nouveau_object *object, bool suspend)
 {
         struct nv50_disp_priv *priv = (void *)object->engine;
         struct nv50_disp_base *base = (void *)object;
@@ -1207,25 +1209,25 @@ nv50_disp_base_fini(struct nouveau_object *object, bool suspend)
 }
 
 struct nouveau_ofuncs
-nv50_disp_base_ofuncs = {
-        .ctor = nv50_disp_base_ctor,
-        .dtor = nv50_disp_base_dtor,
-        .init = nv50_disp_base_init,
-        .fini = nv50_disp_base_fini,
-        .mthd = nv50_disp_base_mthd,
+nv50_disp_main_ofuncs = {
+        .ctor = nv50_disp_main_ctor,
+        .dtor = nv50_disp_main_dtor,
+        .init = nv50_disp_main_init,
+        .fini = nv50_disp_main_fini,
+        .mthd = nv50_disp_main_mthd,
         .ntfy = nouveau_disp_ntfy,
 };
 
 static struct nouveau_oclass
-nv50_disp_base_oclass[] = {
-        { NV50_DISP, &nv50_disp_base_ofuncs },
+nv50_disp_main_oclass[] = {
+        { NV50_DISP, &nv50_disp_main_ofuncs },
         {}
 };
 
 static struct nouveau_oclass
 nv50_disp_sclass[] = {
-        { NV50_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
-        { NV50_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { NV50_DISP_CORE_CHANNEL_DMA, &nv50_disp_core_ofuncs.base },
+        { NV50_DISP_BASE_CHANNEL_DMA, &nv50_disp_base_ofuncs.base },
         { NV50_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
         { NV50_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
         { NV50_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
@@ -1974,7 +1976,7 @@ nv50_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_engine(priv)->sclass = nv50_disp_base_oclass;
+        nv_engine(priv)->sclass = nv50_disp_main_oclass;
         nv_engine(priv)->cclass = &nv50_disp_cclass;
         nv_subdev(priv)->intr = nv50_disp_intr;
         INIT_WORK(&priv->supervisor, nv50_disp_intr_supervisor);
@@ -2007,9 +2009,9 @@ nv50_disp_oclass = &(struct nv50_disp_impl) {
         },
         .base.vblank = &nv50_disp_vblank_func,
         .base.outp =  nv50_disp_outp_sclass,
-        .mthd.core = &nv50_disp_mast_mthd_chan,
-        .mthd.base = &nv50_disp_sync_mthd_chan,
+        .mthd.core = &nv50_disp_core_mthd_chan,
+        .mthd.base = &nv50_disp_base_mthd_chan,
         .mthd.ovly = &nv50_disp_ovly_mthd_chan,
         .mthd.prev = 0x000004,
-        .head.scanoutpos = nv50_disp_base_scanoutpos,
+        .head.scanoutpos = nv50_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
index 5279feefec06..7f08078ee925 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
@@ -42,6 +42,7 @@ struct nv50_disp_priv {
                 int (*hda_eld)(NV50_DISP_MTHD_V1);
                 int (*hdmi)(NV50_DISP_MTHD_V1);
                 u32 lvdsconf;
+                void (*magic)(struct nvkm_output *);
         } sor;
         struct {
                 int nr;
@@ -63,10 +64,10 @@ struct nv50_disp_impl {
         } head;
 };
 
-int nv50_disp_base_scanoutpos(NV50_DISP_MTHD_V0);
-int nv50_disp_base_mthd(struct nouveau_object *, u32, void *, u32);
+int nv50_disp_main_scanoutpos(NV50_DISP_MTHD_V0);
+int nv50_disp_main_mthd(struct nouveau_object *, u32, void *, u32);
 
-int nvd0_disp_base_scanoutpos(NV50_DISP_MTHD_V0);
+int nvd0_disp_main_scanoutpos(NV50_DISP_MTHD_V0);
 
 int nv50_dac_power(NV50_DISP_MTHD_V1);
 int nv50_dac_sense(NV50_DISP_MTHD_V1);
@@ -169,18 +170,18 @@ struct nv50_disp_mthd_chan {
         } data[];
 };
 
-extern struct nv50_disp_chan_impl nv50_disp_mast_ofuncs;
-int nv50_disp_mast_ctor(struct nouveau_object *, struct nouveau_object *,
+extern struct nv50_disp_chan_impl nv50_disp_core_ofuncs;
+int nv50_disp_core_ctor(struct nouveau_object *, struct nouveau_object *,
                         struct nouveau_oclass *, void *, u32,
                         struct nouveau_object **);
-extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_base;
-extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_sor;
-extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_pior;
-extern struct nv50_disp_chan_impl nv50_disp_sync_ofuncs;
-int nv50_disp_sync_ctor(struct nouveau_object *, struct nouveau_object *,
+extern const struct nv50_disp_mthd_list nv50_disp_core_mthd_base;
+extern const struct nv50_disp_mthd_list nv50_disp_core_mthd_sor;
+extern const struct nv50_disp_mthd_list nv50_disp_core_mthd_pior;
+extern struct nv50_disp_chan_impl nv50_disp_base_ofuncs;
+int nv50_disp_base_ctor(struct nouveau_object *, struct nouveau_object *,
                         struct nouveau_oclass *, void *, u32,
                         struct nouveau_object **);
-extern const struct nv50_disp_mthd_list nv50_disp_sync_mthd_image;
+extern const struct nv50_disp_mthd_list nv50_disp_base_mthd_image;
 extern struct nv50_disp_chan_impl nv50_disp_ovly_ofuncs;
 int nv50_disp_ovly_ctor(struct nouveau_object *, struct nouveau_object *,
                         struct nouveau_oclass *, void *, u32,
@@ -194,12 +195,12 @@ extern struct nv50_disp_chan_impl nv50_disp_curs_ofuncs;
 int nv50_disp_curs_ctor(struct nouveau_object *, struct nouveau_object *,
                         struct nouveau_oclass *, void *, u32,
                         struct nouveau_object **);
-extern struct nouveau_ofuncs nv50_disp_base_ofuncs;
-int  nv50_disp_base_ctor(struct nouveau_object *, struct nouveau_object *,
+extern struct nouveau_ofuncs nv50_disp_main_ofuncs;
+int  nv50_disp_main_ctor(struct nouveau_object *, struct nouveau_object *,
                          struct nouveau_oclass *, void *, u32,
                          struct nouveau_object **);
-void nv50_disp_base_dtor(struct nouveau_object *);
-extern struct nouveau_omthds nv50_disp_base_omthds[];
+void nv50_disp_main_dtor(struct nouveau_object *);
+extern struct nouveau_omthds nv50_disp_main_omthds[];
 extern struct nouveau_oclass nv50_disp_cclass;
 void nv50_disp_mthd_chan(struct nv50_disp_priv *, int debug, int head,
                          const struct nv50_disp_mthd_chan *);
@@ -207,31 +208,31 @@ void nv50_disp_intr_supervisor(struct work_struct *);
 void nv50_disp_intr(struct nouveau_subdev *);
 extern const struct nvkm_event_func nv50_disp_vblank_func;
 
-extern const struct nv50_disp_mthd_chan nv84_disp_mast_mthd_chan;
-extern const struct nv50_disp_mthd_list nv84_disp_mast_mthd_dac;
-extern const struct nv50_disp_mthd_list nv84_disp_mast_mthd_head;
-extern const struct nv50_disp_mthd_chan nv84_disp_sync_mthd_chan;
+extern const struct nv50_disp_mthd_chan nv84_disp_core_mthd_chan;
+extern const struct nv50_disp_mthd_list nv84_disp_core_mthd_dac;
+extern const struct nv50_disp_mthd_list nv84_disp_core_mthd_head;
+extern const struct nv50_disp_mthd_chan nv84_disp_base_mthd_chan;
 extern const struct nv50_disp_mthd_chan nv84_disp_ovly_mthd_chan;
 
-extern const struct nv50_disp_mthd_chan nv94_disp_mast_mthd_chan;
+extern const struct nv50_disp_mthd_chan nv94_disp_core_mthd_chan;
 
-extern struct nv50_disp_chan_impl nvd0_disp_mast_ofuncs;
-extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_base;
-extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_dac;
-extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_sor;
-extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_pior;
-extern struct nv50_disp_chan_impl nvd0_disp_sync_ofuncs;
+extern struct nv50_disp_chan_impl nvd0_disp_core_ofuncs;
+extern const struct nv50_disp_mthd_list nvd0_disp_core_mthd_base;
+extern const struct nv50_disp_mthd_list nvd0_disp_core_mthd_dac;
+extern const struct nv50_disp_mthd_list nvd0_disp_core_mthd_sor;
+extern const struct nv50_disp_mthd_list nvd0_disp_core_mthd_pior;
+extern struct nv50_disp_chan_impl nvd0_disp_base_ofuncs;
 extern struct nv50_disp_chan_impl nvd0_disp_ovly_ofuncs;
-extern const struct nv50_disp_mthd_chan nvd0_disp_sync_mthd_chan;
+extern const struct nv50_disp_mthd_chan nvd0_disp_base_mthd_chan;
 extern struct nv50_disp_chan_impl nvd0_disp_oimm_ofuncs;
 extern struct nv50_disp_chan_impl nvd0_disp_curs_ofuncs;
-extern struct nouveau_ofuncs nvd0_disp_base_ofuncs;
+extern struct nouveau_ofuncs nvd0_disp_main_ofuncs;
 extern struct nouveau_oclass nvd0_disp_cclass;
 void nvd0_disp_intr_supervisor(struct work_struct *);
 void nvd0_disp_intr(struct nouveau_subdev *);
 extern const struct nvkm_event_func nvd0_disp_vblank_func;
 
-extern const struct nv50_disp_mthd_chan nve0_disp_mast_mthd_chan;
+extern const struct nv50_disp_mthd_chan nve0_disp_core_mthd_chan;
 extern const struct nv50_disp_mthd_chan nve0_disp_ovly_mthd_chan;
 
 extern struct nvkm_output_dp_impl nv50_pior_dp_impl;
@@ -242,6 +243,10 @@ int nv94_sor_dp_lnk_pwr(struct nvkm_output_dp *, int);
 extern struct nouveau_oclass *nv94_disp_outp_sclass[];
 
 extern struct nvkm_output_dp_impl nvd0_sor_dp_impl;
+int nvd0_sor_dp_lnk_ctl(struct nvkm_output_dp *, int, int, bool);
 extern struct nouveau_oclass *nvd0_disp_outp_sclass[];
 
+void gm204_sor_magic(struct nvkm_output *outp);
+extern struct nvkm_output_dp_impl gm204_sor_dp_impl;
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
index d36284715b2a..13eff5e4ee51 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
@@ -34,7 +34,7 @@
  ******************************************************************************/
 
 const struct nv50_disp_mthd_list
-nv84_disp_mast_mthd_dac = {
+nv84_disp_core_mthd_dac = {
         .mthd = 0x0080,
         .addr = 0x000008,
         .data = {
@@ -46,7 +46,7 @@ nv84_disp_mast_mthd_dac = {
 };
 
 const struct nv50_disp_mthd_list
-nv84_disp_mast_mthd_head = {
+nv84_disp_core_mthd_head = {
         .mthd = 0x0400,
         .addr = 0x000540,
         .data = {
@@ -98,15 +98,15 @@ nv84_disp_mast_mthd_head = {
 };
 
 const struct nv50_disp_mthd_chan
-nv84_disp_mast_mthd_chan = {
+nv84_disp_core_mthd_chan = {
         .name = "Core",
         .addr = 0x000000,
         .data = {
-                { "Global", 1, &nv50_disp_mast_mthd_base },
-                {    "DAC", 3, &nv84_disp_mast_mthd_dac  },
-                {    "SOR", 2, &nv50_disp_mast_mthd_sor  },
-                {   "PIOR", 3, &nv50_disp_mast_mthd_pior },
-                {   "HEAD", 2, &nv84_disp_mast_mthd_head },
+                { "Global", 1, &nv50_disp_core_mthd_base },
+                {    "DAC", 3, &nv84_disp_core_mthd_dac  },
+                {    "SOR", 2, &nv50_disp_core_mthd_sor  },
+                {   "PIOR", 3, &nv50_disp_core_mthd_pior },
+                {   "HEAD", 2, &nv84_disp_core_mthd_head },
                 {}
         }
 };
@@ -116,7 +116,7 @@ nv84_disp_mast_mthd_chan = {
  ******************************************************************************/
 
 static const struct nv50_disp_mthd_list
-nv84_disp_sync_mthd_base = {
+nv84_disp_base_mthd_base = {
         .mthd = 0x0000,
         .addr = 0x000000,
         .data = {
@@ -146,12 +146,12 @@ nv84_disp_sync_mthd_base = {
 };
 
 const struct nv50_disp_mthd_chan
-nv84_disp_sync_mthd_chan = {
+nv84_disp_base_mthd_chan = {
         .name = "Base",
         .addr = 0x000540,
         .data = {
-                { "Global", 1, &nv84_disp_sync_mthd_base },
-                {  "Image", 2, &nv50_disp_sync_mthd_image },
+                { "Global", 1, &nv84_disp_base_mthd_base },
+                {  "Image", 2, &nv50_disp_base_mthd_image },
                 {}
         }
 };
@@ -204,8 +204,8 @@ nv84_disp_ovly_mthd_chan = {
 
 static struct nouveau_oclass
 nv84_disp_sclass[] = {
-        { G82_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
-        { G82_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { G82_DISP_CORE_CHANNEL_DMA, &nv50_disp_core_ofuncs.base },
+        { G82_DISP_BASE_CHANNEL_DMA, &nv50_disp_base_ofuncs.base },
         { G82_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
         { G82_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
         { G82_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
@@ -213,8 +213,8 @@ nv84_disp_sclass[] = {
 };
 
 static struct nouveau_oclass
-nv84_disp_base_oclass[] = {
-        { G82_DISP, &nv50_disp_base_ofuncs },
+nv84_disp_main_oclass[] = {
+        { G82_DISP, &nv50_disp_main_ofuncs },
         {}
 };
 
@@ -240,7 +240,7 @@ nv84_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_engine(priv)->sclass = nv84_disp_base_oclass;
+        nv_engine(priv)->sclass = nv84_disp_main_oclass;
         nv_engine(priv)->cclass = &nv50_disp_cclass;
         nv_subdev(priv)->intr = nv50_disp_intr;
         INIT_WORK(&priv->supervisor, nv50_disp_intr_supervisor);
@@ -268,9 +268,9 @@ nv84_disp_oclass = &(struct nv50_disp_impl) {
         },
         .base.vblank = &nv50_disp_vblank_func,
         .base.outp =  nv50_disp_outp_sclass,
-        .mthd.core = &nv84_disp_mast_mthd_chan,
-        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.core = &nv84_disp_core_mthd_chan,
+        .mthd.base = &nv84_disp_base_mthd_chan,
         .mthd.ovly = &nv84_disp_ovly_mthd_chan,
         .mthd.prev = 0x000004,
-        .head.scanoutpos = nv50_disp_base_scanoutpos,
+        .head.scanoutpos = nv50_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
index a117064002b1..2bb7ac5cd0e6 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
@@ -34,7 +34,7 @@
  ******************************************************************************/
 
 const struct nv50_disp_mthd_list
-nv94_disp_mast_mthd_sor = {
+nv94_disp_core_mthd_sor = {
         .mthd = 0x0040,
         .addr = 0x000008,
         .data = {
@@ -44,15 +44,15 @@ nv94_disp_mast_mthd_sor = {
 };
 
 const struct nv50_disp_mthd_chan
-nv94_disp_mast_mthd_chan = {
+nv94_disp_core_mthd_chan = {
         .name = "Core",
         .addr = 0x000000,
         .data = {
-                { "Global", 1, &nv50_disp_mast_mthd_base },
-                {    "DAC", 3, &nv84_disp_mast_mthd_dac  },
-                {    "SOR", 4, &nv94_disp_mast_mthd_sor  },
-                {   "PIOR", 3, &nv50_disp_mast_mthd_pior },
-                {   "HEAD", 2, &nv84_disp_mast_mthd_head },
+                { "Global", 1, &nv50_disp_core_mthd_base },
+                {    "DAC", 3, &nv84_disp_core_mthd_dac  },
+                {    "SOR", 4, &nv94_disp_core_mthd_sor  },
+                {   "PIOR", 3, &nv50_disp_core_mthd_pior },
+                {   "HEAD", 2, &nv84_disp_core_mthd_head },
                 {}
         }
 };
@@ -63,8 +63,8 @@ nv94_disp_mast_mthd_chan = {
 
 static struct nouveau_oclass
 nv94_disp_sclass[] = {
-        { GT206_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
-        { GT200_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { GT206_DISP_CORE_CHANNEL_DMA, &nv50_disp_core_ofuncs.base },
+        { GT200_DISP_BASE_CHANNEL_DMA, &nv50_disp_base_ofuncs.base },
         { GT200_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
         { G82_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
         { G82_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
@@ -72,8 +72,8 @@ nv94_disp_sclass[] = {
 };
 
 static struct nouveau_oclass
-nv94_disp_base_oclass[] = {
-        { GT206_DISP, &nv50_disp_base_ofuncs },
+nv94_disp_main_oclass[] = {
+        { GT206_DISP, &nv50_disp_main_ofuncs },
         {}
 };
 
@@ -99,7 +99,7 @@ nv94_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_engine(priv)->sclass = nv94_disp_base_oclass;
+        nv_engine(priv)->sclass = nv94_disp_main_oclass;
         nv_engine(priv)->cclass = &nv50_disp_cclass;
         nv_subdev(priv)->intr = nv50_disp_intr;
         INIT_WORK(&priv->supervisor, nv50_disp_intr_supervisor);
@@ -134,9 +134,9 @@ nv94_disp_oclass = &(struct nv50_disp_impl) {
         },
         .base.vblank = &nv50_disp_vblank_func,
         .base.outp =  nv94_disp_outp_sclass,
-        .mthd.core = &nv94_disp_mast_mthd_chan,
-        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.core = &nv94_disp_core_mthd_chan,
+        .mthd.base = &nv84_disp_base_mthd_chan,
         .mthd.ovly = &nv84_disp_ovly_mthd_chan,
         .mthd.prev = 0x000004,
-        .head.scanoutpos = nv50_disp_base_scanoutpos,
+        .head.scanoutpos = nv50_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
index c67e68aadd45..b32456c9494f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
@@ -80,8 +80,8 @@ nva0_disp_ovly_mthd_chan = {
 
 static struct nouveau_oclass
 nva0_disp_sclass[] = {
-        { GT200_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
-        { GT200_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { GT200_DISP_CORE_CHANNEL_DMA, &nv50_disp_core_ofuncs.base },
+        { GT200_DISP_BASE_CHANNEL_DMA, &nv50_disp_base_ofuncs.base },
         { GT200_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
         { G82_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
         { G82_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
@@ -89,8 +89,8 @@ nva0_disp_sclass[] = {
 };
 
 static struct nouveau_oclass
-nva0_disp_base_oclass[] = {
-        { GT200_DISP, &nv50_disp_base_ofuncs },
+nva0_disp_main_oclass[] = {
+        { GT200_DISP, &nv50_disp_main_ofuncs },
         {}
 };
 
@@ -116,7 +116,7 @@ nva0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_engine(priv)->sclass = nva0_disp_base_oclass;
+        nv_engine(priv)->sclass = nva0_disp_main_oclass;
         nv_engine(priv)->cclass = &nv50_disp_cclass;
         nv_subdev(priv)->intr = nv50_disp_intr;
         INIT_WORK(&priv->supervisor, nv50_disp_intr_supervisor);
@@ -144,9 +144,9 @@ nva0_disp_oclass = &(struct nv50_disp_impl) {
         },
         .base.vblank = &nv50_disp_vblank_func,
         .base.outp =  nv50_disp_outp_sclass,
-        .mthd.core = &nv84_disp_mast_mthd_chan,
-        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.core = &nv84_disp_core_mthd_chan,
+        .mthd.base = &nv84_disp_base_mthd_chan,
         .mthd.ovly = &nva0_disp_ovly_mthd_chan,
         .mthd.prev = 0x000004,
-        .head.scanoutpos = nv50_disp_base_scanoutpos,
+        .head.scanoutpos = nv50_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c b/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
index 22969f355aae..951d79f9b781 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
@@ -35,8 +35,8 @@
 
 static struct nouveau_oclass
 nva3_disp_sclass[] = {
-        { GT214_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
-        { GT214_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { GT214_DISP_CORE_CHANNEL_DMA, &nv50_disp_core_ofuncs.base },
+        { GT214_DISP_BASE_CHANNEL_DMA, &nv50_disp_base_ofuncs.base },
         { GT214_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
         { GT214_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
         { GT214_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
@@ -44,8 +44,8 @@ nva3_disp_sclass[] = {
 };
 
 static struct nouveau_oclass
-nva3_disp_base_oclass[] = {
-        { GT214_DISP, &nv50_disp_base_ofuncs },
+nva3_disp_main_oclass[] = {
+        { GT214_DISP, &nv50_disp_main_ofuncs },
         {}
 };
 
@@ -71,7 +71,7 @@ nva3_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_engine(priv)->sclass = nva3_disp_base_oclass;
+        nv_engine(priv)->sclass = nva3_disp_main_oclass;
         nv_engine(priv)->cclass = &nv50_disp_cclass;
         nv_subdev(priv)->intr = nv50_disp_intr;
         INIT_WORK(&priv->supervisor, nv50_disp_intr_supervisor);
@@ -100,9 +100,9 @@ nva3_disp_oclass = &(struct nv50_disp_impl) {
         },
         .base.vblank = &nv50_disp_vblank_func,
         .base.outp =  nv94_disp_outp_sclass,
-        .mthd.core = &nv94_disp_mast_mthd_chan,
-        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.core = &nv94_disp_core_mthd_chan,
+        .mthd.base = &nv84_disp_base_mthd_chan,
         .mthd.ovly = &nv84_disp_ovly_mthd_chan,
         .mthd.prev = 0x000004,
-        .head.scanoutpos = nv50_disp_base_scanoutpos,
+        .head.scanoutpos = nv50_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
index 747e64bb9c06..181a2d57e356 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
@@ -51,12 +51,14 @@ nvd0_disp_chan_uevent_fini(struct nvkm_event *event, int type, int index)
 {
         struct nv50_disp_priv *priv = container_of(event, typeof(*priv), uevent);
         nv_mask(priv, 0x610090, 0x00000001 << index, 0x00000000 << index);
+        nv_wr32(priv, 0x61008c, 0x00000001 << index);
 }
 
 static void
 nvd0_disp_chan_uevent_init(struct nvkm_event *event, int types, int index)
 {
         struct nv50_disp_priv *priv = container_of(event, typeof(*priv), uevent);
+        nv_wr32(priv, 0x61008c, 0x00000001 << index);
         nv_mask(priv, 0x610090, 0x00000001 << index, 0x00000001 << index);
 }
 
@@ -151,7 +153,7 @@ nvd0_disp_dmac_fini(struct nouveau_object *object, bool suspend)
  ******************************************************************************/
 
 const struct nv50_disp_mthd_list
-nvd0_disp_mast_mthd_base = {
+nvd0_disp_core_mthd_base = {
         .mthd = 0x0000,
         .addr = 0x000000,
         .data = {
@@ -164,7 +166,7 @@ nvd0_disp_mast_mthd_base = {
 };
 
 const struct nv50_disp_mthd_list
-nvd0_disp_mast_mthd_dac = {
+nvd0_disp_core_mthd_dac = {
         .mthd = 0x0020,
         .addr = 0x000020,
         .data = {
@@ -177,7 +179,7 @@ nvd0_disp_mast_mthd_dac = {
 };
 
 const struct nv50_disp_mthd_list
-nvd0_disp_mast_mthd_sor = {
+nvd0_disp_core_mthd_sor = {
         .mthd = 0x0020,
         .addr = 0x000020,
         .data = {
@@ -190,7 +192,7 @@ nvd0_disp_mast_mthd_sor = {
 };
 
 const struct nv50_disp_mthd_list
-nvd0_disp_mast_mthd_pior = {
+nvd0_disp_core_mthd_pior = {
         .mthd = 0x0020,
         .addr = 0x000020,
         .data = {
@@ -203,7 +205,7 @@ nvd0_disp_mast_mthd_pior = {
 };
 
 static const struct nv50_disp_mthd_list
-nvd0_disp_mast_mthd_head = {
+nvd0_disp_core_mthd_head = {
         .mthd = 0x0300,
         .addr = 0x000300,
         .data = {
@@ -277,21 +279,21 @@ nvd0_disp_mast_mthd_head = {
 };
 
 static const struct nv50_disp_mthd_chan
-nvd0_disp_mast_mthd_chan = {
+nvd0_disp_core_mthd_chan = {
         .name = "Core",
         .addr = 0x000000,
         .data = {
-                { "Global", 1, &nvd0_disp_mast_mthd_base },
-                {    "DAC", 3, &nvd0_disp_mast_mthd_dac  },
-                {    "SOR", 8, &nvd0_disp_mast_mthd_sor  },
-                {   "PIOR", 4, &nvd0_disp_mast_mthd_pior },
-                {   "HEAD", 4, &nvd0_disp_mast_mthd_head },
+                { "Global", 1, &nvd0_disp_core_mthd_base },
+                {    "DAC", 3, &nvd0_disp_core_mthd_dac  },
+                {    "SOR", 8, &nvd0_disp_core_mthd_sor  },
+                {   "PIOR", 4, &nvd0_disp_core_mthd_pior },
+                {   "HEAD", 4, &nvd0_disp_core_mthd_head },
                 {}
         }
 };
 
 static int
-nvd0_disp_mast_init(struct nouveau_object *object)
+nvd0_disp_core_init(struct nouveau_object *object)
 {
         struct nv50_disp_priv *priv = (void *)object->engine;
         struct nv50_disp_dmac *mast = (void *)object;
@@ -322,7 +324,7 @@ nvd0_disp_mast_init(struct nouveau_object *object)
 }
 
 static int
-nvd0_disp_mast_fini(struct nouveau_object *object, bool suspend)
+nvd0_disp_core_fini(struct nouveau_object *object, bool suspend)
 {
         struct nv50_disp_priv *priv = (void *)object->engine;
         struct nv50_disp_dmac *mast = (void *)object;
@@ -344,11 +346,11 @@ nvd0_disp_mast_fini(struct nouveau_object *object, bool suspend)
 }
 
 struct nv50_disp_chan_impl
-nvd0_disp_mast_ofuncs = {
-        .base.ctor = nv50_disp_mast_ctor,
+nvd0_disp_core_ofuncs = {
+        .base.ctor = nv50_disp_core_ctor,
         .base.dtor = nv50_disp_dmac_dtor,
-        .base.init = nvd0_disp_mast_init,
-        .base.fini = nvd0_disp_mast_fini,
+        .base.init = nvd0_disp_core_init,
+        .base.fini = nvd0_disp_core_fini,
         .base.ntfy = nv50_disp_chan_ntfy,
         .base.map  = nv50_disp_chan_map,
         .base.rd32 = nv50_disp_chan_rd32,
@@ -363,7 +365,7 @@ nvd0_disp_mast_ofuncs = {
  ******************************************************************************/
 
 static const struct nv50_disp_mthd_list
-nvd0_disp_sync_mthd_base = {
+nvd0_disp_base_mthd_base = {
         .mthd = 0x0000,
         .addr = 0x000000,
         .data = {
@@ -413,7 +415,7 @@ nvd0_disp_sync_mthd_base = {
 };
 
 static const struct nv50_disp_mthd_list
-nvd0_disp_sync_mthd_image = {
+nvd0_disp_base_mthd_image = {
         .mthd = 0x0400,
         .addr = 0x000400,
         .data = {
@@ -427,19 +429,19 @@ nvd0_disp_sync_mthd_image = {
 };
 
 const struct nv50_disp_mthd_chan
-nvd0_disp_sync_mthd_chan = {
+nvd0_disp_base_mthd_chan = {
         .name = "Base",
         .addr = 0x001000,
         .data = {
-                { "Global", 1, &nvd0_disp_sync_mthd_base },
-                {  "Image", 2, &nvd0_disp_sync_mthd_image },
+                { "Global", 1, &nvd0_disp_base_mthd_base },
+                {  "Image", 2, &nvd0_disp_base_mthd_image },
                 {}
         }
 };
 
 struct nv50_disp_chan_impl
-nvd0_disp_sync_ofuncs = {
-        .base.ctor = nv50_disp_sync_ctor,
+nvd0_disp_base_ofuncs = {
+        .base.ctor = nv50_disp_base_ctor,
         .base.dtor = nv50_disp_dmac_dtor,
         .base.init = nvd0_disp_dmac_init,
         .base.fini = nvd0_disp_dmac_fini,
@@ -624,7 +626,7 @@ nvd0_disp_curs_ofuncs = {
  ******************************************************************************/
 
 int
-nvd0_disp_base_scanoutpos(NV50_DISP_MTHD_V0)
+nvd0_disp_main_scanoutpos(NV50_DISP_MTHD_V0)
 {
         const u32 total  = nv_rd32(priv, 0x640414 + (head * 0x300));
         const u32 blanke = nv_rd32(priv, 0x64041c + (head * 0x300));
@@ -656,7 +658,7 @@ nvd0_disp_base_scanoutpos(NV50_DISP_MTHD_V0)
 }
 
 static int
-nvd0_disp_base_init(struct nouveau_object *object)
+nvd0_disp_main_init(struct nouveau_object *object)
 {
         struct nv50_disp_priv *priv = (void *)object->engine;
         struct nv50_disp_base *base = (void *)object;
@@ -725,7 +727,7 @@ nvd0_disp_base_init(struct nouveau_object *object)
 }
 
 static int
-nvd0_disp_base_fini(struct nouveau_object *object, bool suspend)
+nvd0_disp_main_fini(struct nouveau_object *object, bool suspend)
 {
         struct nv50_disp_priv *priv = (void *)object->engine;
         struct nv50_disp_base *base = (void *)object;
@@ -737,25 +739,25 @@ nvd0_disp_base_fini(struct nouveau_object *object, bool suspend)
 }
 
 struct nouveau_ofuncs
-nvd0_disp_base_ofuncs = {
-        .ctor = nv50_disp_base_ctor,
-        .dtor = nv50_disp_base_dtor,
-        .init = nvd0_disp_base_init,
-        .fini = nvd0_disp_base_fini,
-        .mthd = nv50_disp_base_mthd,
+nvd0_disp_main_ofuncs = {
+        .ctor = nv50_disp_main_ctor,
+        .dtor = nv50_disp_main_dtor,
+        .init = nvd0_disp_main_init,
+        .fini = nvd0_disp_main_fini,
+        .mthd = nv50_disp_main_mthd,
         .ntfy = nouveau_disp_ntfy,
 };
 
 static struct nouveau_oclass
-nvd0_disp_base_oclass[] = {
-        { GF110_DISP, &nvd0_disp_base_ofuncs },
+nvd0_disp_main_oclass[] = {
+        { GF110_DISP, &nvd0_disp_main_ofuncs },
         {}
 };
 
 static struct nouveau_oclass
 nvd0_disp_sclass[] = {
-        { GF110_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
-        { GF110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GF110_DISP_CORE_CHANNEL_DMA, &nvd0_disp_core_ofuncs.base },
+        { GF110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_base_ofuncs.base },
         { GF110_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
         { GF110_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
         { GF110_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
@@ -1055,6 +1057,9 @@ nvd0_disp_intr_unk2_2(struct nv50_disp_priv *priv, int head)
 
                 if (nvkm_output_dp_train(outp, pclk, true))
                         ERR("link not trained before attach\n");
+        } else {
+                if (priv->sor.magic)
+                        priv->sor.magic(outp);
         }
 
         exec_clkcmp(priv, head, 0, pclk, &conf);
@@ -1063,10 +1068,18 @@ nvd0_disp_intr_unk2_2(struct nv50_disp_priv *priv, int head)
                 addr = 0x612280 + (ffs(outp->info.or) - 1) * 0x800;
                 data = 0x00000000;
         } else {
-                if (outp->info.type == DCB_OUTPUT_DP)
-                        nvd0_disp_intr_unk2_2_tu(priv, head, &outp->info);
                 addr = 0x612300 + (ffs(outp->info.or) - 1) * 0x800;
                 data = (conf & 0x0100) ? 0x00000101 : 0x00000000;
+                switch (outp->info.type) {
+                case DCB_OUTPUT_TMDS:
+                        nv_mask(priv, addr, 0x007c0000, 0x00280000);
+                        break;
+                case DCB_OUTPUT_DP:
+                        nvd0_disp_intr_unk2_2_tu(priv, head, &outp->info);
+                        break;
+                default:
+                        break;
+                }
         }
 
         nv_mask(priv, addr, 0x00000707, data);
@@ -1259,7 +1272,7 @@ nvd0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_engine(priv)->sclass = nvd0_disp_base_oclass;
+        nv_engine(priv)->sclass = nvd0_disp_main_oclass;
         nv_engine(priv)->cclass = &nv50_disp_cclass;
         nv_subdev(priv)->intr = nvd0_disp_intr;
         INIT_WORK(&priv->supervisor, nvd0_disp_intr_supervisor);
@@ -1292,9 +1305,9 @@ nvd0_disp_oclass = &(struct nv50_disp_impl) {
         },
         .base.vblank = &nvd0_disp_vblank_func,
         .base.outp =  nvd0_disp_outp_sclass,
-        .mthd.core = &nvd0_disp_mast_mthd_chan,
-        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.core = &nvd0_disp_core_mthd_chan,
+        .mthd.base = &nvd0_disp_base_mthd_chan,
         .mthd.ovly = &nvd0_disp_ovly_mthd_chan,
         .mthd.prev = -0x020000,
-        .head.scanoutpos = nvd0_disp_base_scanoutpos,
+        .head.scanoutpos = nvd0_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
index db144b2cf06b..55debec7e68f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
@@ -34,7 +34,7 @@
  ******************************************************************************/
 
 static const struct nv50_disp_mthd_list
-nve0_disp_mast_mthd_head = {
+nve0_disp_core_mthd_head = {
         .mthd = 0x0300,
         .addr = 0x000300,
         .data = {
@@ -113,15 +113,15 @@ nve0_disp_mast_mthd_head = {
 };
 
 const struct nv50_disp_mthd_chan
-nve0_disp_mast_mthd_chan = {
+nve0_disp_core_mthd_chan = {
         .name = "Core",
         .addr = 0x000000,
         .data = {
-                { "Global", 1, &nvd0_disp_mast_mthd_base },
-                {    "DAC", 3, &nvd0_disp_mast_mthd_dac  },
-                {    "SOR", 8, &nvd0_disp_mast_mthd_sor  },
-                {   "PIOR", 4, &nvd0_disp_mast_mthd_pior },
-                {   "HEAD", 4, &nve0_disp_mast_mthd_head },
+                { "Global", 1, &nvd0_disp_core_mthd_base },
+                {    "DAC", 3, &nvd0_disp_core_mthd_dac  },
+                {    "SOR", 8, &nvd0_disp_core_mthd_sor  },
+                {   "PIOR", 4, &nvd0_disp_core_mthd_pior },
+                {   "HEAD", 4, &nve0_disp_core_mthd_head },
                 {}
         }
 };
@@ -200,8 +200,8 @@ nve0_disp_ovly_mthd_chan = {
 
 static struct nouveau_oclass
 nve0_disp_sclass[] = {
-        { GK104_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
-        { GK104_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GK104_DISP_CORE_CHANNEL_DMA, &nvd0_disp_core_ofuncs.base },
+        { GK104_DISP_BASE_CHANNEL_DMA, &nvd0_disp_base_ofuncs.base },
         { GK104_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
         { GK104_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
         { GK104_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
@@ -209,8 +209,8 @@ nve0_disp_sclass[] = {
 };
 
 static struct nouveau_oclass
-nve0_disp_base_oclass[] = {
-        { GK104_DISP, &nvd0_disp_base_ofuncs },
+nve0_disp_main_oclass[] = {
+        { GK104_DISP, &nvd0_disp_main_ofuncs },
         {}
 };
 
@@ -237,7 +237,7 @@ nve0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_engine(priv)->sclass = nve0_disp_base_oclass;
+        nv_engine(priv)->sclass = nve0_disp_main_oclass;
         nv_engine(priv)->cclass = &nv50_disp_cclass;
         nv_subdev(priv)->intr = nvd0_disp_intr;
         INIT_WORK(&priv->supervisor, nvd0_disp_intr_supervisor);
@@ -264,9 +264,9 @@ nve0_disp_oclass = &(struct nv50_disp_impl) {
         },
         .base.vblank = &nvd0_disp_vblank_func,
         .base.outp =  nvd0_disp_outp_sclass,
-        .mthd.core = &nve0_disp_mast_mthd_chan,
-        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.core = &nve0_disp_core_mthd_chan,
+        .mthd.base = &nvd0_disp_base_mthd_chan,
         .mthd.ovly = &nve0_disp_ovly_mthd_chan,
         .mthd.prev = -0x020000,
-        .head.scanoutpos = nvd0_disp_base_scanoutpos,
+        .head.scanoutpos = nvd0_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
index 402d7d67d806..3e7e2d28744c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
@@ -35,8 +35,8 @@
 
 static struct nouveau_oclass
 nvf0_disp_sclass[] = {
-        { GK110_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
-        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GK110_DISP_CORE_CHANNEL_DMA, &nvd0_disp_core_ofuncs.base },
+        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_base_ofuncs.base },
         { GK104_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
         { GK104_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
         { GK104_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
@@ -44,8 +44,8 @@ nvf0_disp_sclass[] = {
 };
 
 static struct nouveau_oclass
-nvf0_disp_base_oclass[] = {
-        { GK110_DISP, &nvd0_disp_base_ofuncs },
+nvf0_disp_main_oclass[] = {
+        { GK110_DISP, &nvd0_disp_main_ofuncs },
         {}
 };
 
@@ -72,7 +72,7 @@ nvf0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_engine(priv)->sclass = nvf0_disp_base_oclass;
+        nv_engine(priv)->sclass = nvf0_disp_main_oclass;
         nv_engine(priv)->cclass = &nv50_disp_cclass;
         nv_subdev(priv)->intr = nvd0_disp_intr;
         INIT_WORK(&priv->supervisor, nvd0_disp_intr_supervisor);
@@ -99,9 +99,9 @@ nvf0_disp_oclass = &(struct nv50_disp_impl) {
         },
         .base.vblank = &nvd0_disp_vblank_func,
         .base.outp =  nvd0_disp_outp_sclass,
-        .mthd.core = &nve0_disp_mast_mthd_chan,
-        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.core = &nve0_disp_core_mthd_chan,
+        .mthd.base = &nvd0_disp_base_mthd_chan,
         .mthd.ovly = &nve0_disp_ovly_mthd_chan,
         .mthd.prev = -0x020000,
-        .head.scanoutpos = nvd0_disp_base_scanoutpos,
+        .head.scanoutpos = nvd0_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/outp.c b/drivers/gpu/drm/nouveau/core/engine/disp/outp.c
index a5ff00a9cedc..bbd9b6fdc90f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/outp.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/outp.c
@@ -85,7 +85,10 @@ nvkm_output_create_(struct nouveau_object *parent,
             dcbE->sorconf.link : 0, dcbE->connector, dcbE->i2c_index,
             dcbE->bus, dcbE->heads);
 
-        outp->port = i2c->find(i2c, outp->info.i2c_index);
+        if (outp->info.type != DCB_OUTPUT_DP)
+                outp->port = i2c->find(i2c, NV_I2C_PORT(outp->info.i2c_index));
+        else
+                outp->port = i2c->find(i2c, NV_I2C_AUX(outp->info.i2c_index));
         outp->edid = outp->port;
 
         data = nvbios_connEp(bios, outp->info.connector, &ver, &hdr, &connE);
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/sorgm204.c b/drivers/gpu/drm/nouveau/core/engine/disp/sorgm204.c
new file mode 100644
index 000000000000..0b4fad39e9a6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/sorgm204.c
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+
+#include <subdev/bios.h>
+#include <subdev/bios/dcb.h>
+#include <subdev/bios/dp.h>
+#include <subdev/bios/init.h>
+#include <subdev/timer.h>
+
+#include "nv50.h"
+
+static inline u32
+gm204_sor_soff(struct nvkm_output_dp *outp)
+{
+        return (ffs(outp->base.info.or) - 1) * 0x800;
+}
+
+static inline u32
+gm204_sor_loff(struct nvkm_output_dp *outp)
+{
+        return gm204_sor_soff(outp) + !(outp->base.info.sorconf.link & 1) * 0x80;
+}
+
+void
+gm204_sor_magic(struct nvkm_output *outp)
+{
+        struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
+        const u32 soff = outp->or * 0x100;
+        const u32 data = outp->or + 1;
+        if (outp->info.sorconf.link & 1)
+                nv_mask(priv, 0x612308 + soff, 0x0000001f, 0x00000000 | data);
+        if (outp->info.sorconf.link & 2)
+                nv_mask(priv, 0x612388 + soff, 0x0000001f, 0x00000010 | data);
+}
+
+static inline u32
+gm204_sor_dp_lane_map(struct nv50_disp_priv *priv, u8 lane)
+{
+        return lane * 0x08;
+}
+
+static int
+gm204_sor_dp_pattern(struct nvkm_output_dp *outp, int pattern)
+{
+        struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
+        const u32 soff = gm204_sor_soff(outp);
+        const u32 data = 0x01010101 * pattern;
+        if (outp->base.info.sorconf.link & 1)
+                nv_mask(priv, 0x61c110 + soff, 0x0f0f0f0f, data);
+        else
+                nv_mask(priv, 0x61c12c + soff, 0x0f0f0f0f, data);
+        return 0;
+}
+
+static int
+gm204_sor_dp_lnk_pwr(struct nvkm_output_dp *outp, int nr)
+{
+        struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
+        const u32 soff = gm204_sor_soff(outp);
+        const u32 loff = gm204_sor_loff(outp);
+        u32 mask = 0, i;
+
+        for (i = 0; i < nr; i++)
+                mask |= 1 << (gm204_sor_dp_lane_map(priv, i) >> 3);
+
+        nv_mask(priv, 0x61c130 + loff, 0x0000000f, mask);
+        nv_mask(priv, 0x61c034 + soff, 0x80000000, 0x80000000);
+        nv_wait(priv, 0x61c034 + soff, 0x80000000, 0x00000000);
+        return 0;
+}
+
+static int
+gm204_sor_dp_drv_ctl(struct nvkm_output_dp *outp, int ln, int vs, int pe, int pc)
+{
+        struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        const u32 shift = gm204_sor_dp_lane_map(priv, ln);
+        const u32 loff = gm204_sor_loff(outp);
+        u32 addr, data[4];
+        u8  ver, hdr, cnt, len;
+        struct nvbios_dpout info;
+        struct nvbios_dpcfg ocfg;
+
+        addr = nvbios_dpout_match(bios, outp->base.info.hasht,
+                                        outp->base.info.hashm,
+                                 &ver, &hdr, &cnt, &len, &info);
+        if (!addr)
+                return -ENODEV;
+
+        addr = nvbios_dpcfg_match(bios, addr, pc, vs, pe,
+                                 &ver, &hdr, &cnt, &len, &ocfg);
+        if (!addr)
+                return -EINVAL;
+
+        data[0] = nv_rd32(priv, 0x61c118 + loff) & ~(0x000000ff << shift);
+        data[1] = nv_rd32(priv, 0x61c120 + loff) & ~(0x000000ff << shift);
+        data[2] = nv_rd32(priv, 0x61c130 + loff);
+        if ((data[2] & 0x0000ff00) < (ocfg.tx_pu << 8) || ln == 0)
+                data[2] = (data[2] & ~0x0000ff00) | (ocfg.tx_pu << 8);
+        nv_wr32(priv, 0x61c118 + loff, data[0] | (ocfg.dc << shift));
+        nv_wr32(priv, 0x61c120 + loff, data[1] | (ocfg.pe << shift));
+        nv_wr32(priv, 0x61c130 + loff, data[2] | (ocfg.tx_pu << 8));
+        data[3] = nv_rd32(priv, 0x61c13c + loff) & ~(0x000000ff << shift);
+        nv_wr32(priv, 0x61c13c + loff, data[3] | (ocfg.pc << shift));
+        return 0;
+}
+
+struct nvkm_output_dp_impl
+gm204_sor_dp_impl = {
+        .base.base.handle = DCB_OUTPUT_DP,
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nvkm_output_dp_ctor,
+                .dtor = _nvkm_output_dp_dtor,
+                .init = _nvkm_output_dp_init,
+                .fini = _nvkm_output_dp_fini,
+        },
+        .pattern = gm204_sor_dp_pattern,
+        .lnk_pwr = gm204_sor_dp_lnk_pwr,
+        .lnk_ctl = nvd0_sor_dp_lnk_ctl,
+        .drv_ctl = gm204_sor_dp_drv_ctl,
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
index 7b7bbc3e459e..fdab2939070c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
@@ -60,7 +60,7 @@ nvd0_sor_dp_pattern(struct nvkm_output_dp *outp, int pattern)
         return 0;
 }
 
-static int
+int
 nvd0_sor_dp_lnk_ctl(struct nvkm_output_dp *outp, int nr, int bw, bool ef)
 {
         struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
index 3fc4f0b0eaca..19f5f6522962 100644
--- a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
@@ -51,6 +51,7 @@ nvd0_dmaobj_bind(struct nouveau_dmaobj *dmaobj,
                 case GK104_DISP_CORE_CHANNEL_DMA:
                 case GK110_DISP_CORE_CHANNEL_DMA:
                 case GM107_DISP_CORE_CHANNEL_DMA:
+                case GM204_DISP_CORE_CHANNEL_DMA:
                 case GF110_DISP_BASE_CHANNEL_DMA:
                 case GK104_DISP_BASE_CHANNEL_DMA:
                 case GK110_DISP_BASE_CHANNEL_DMA:
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
index f8734eb74eaa..6a8db7c80bd1 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
@@ -792,7 +792,7 @@ nve0_fifo_intr_fault(struct nve0_fifo_priv *priv, int unit)
         nouveau_engctx_put(engctx);
 }
 
-static const struct nouveau_bitfield nve0_fifo_pbdma_intr[] = {
+static const struct nouveau_bitfield nve0_fifo_pbdma_intr_0[] = {
         { 0x00000001, "MEMREQ" },
         { 0x00000002, "MEMACK_TIMEOUT" },
         { 0x00000004, "MEMACK_EXTRA" },
@@ -827,9 +827,10 @@ static const struct nouveau_bitfield nve0_fifo_pbdma_intr[] = {
 };
 
 static void
-nve0_fifo_intr_pbdma(struct nve0_fifo_priv *priv, int unit)
+nve0_fifo_intr_pbdma_0(struct nve0_fifo_priv *priv, int unit)
 {
-        u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000));
+        u32 mask = nv_rd32(priv, 0x04010c + (unit * 0x2000));
+        u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000)) & mask;
         u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000));
         u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000));
         u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff;
@@ -840,11 +841,12 @@ nve0_fifo_intr_pbdma(struct nve0_fifo_priv *priv, int unit)
         if (stat & 0x00800000) {
                 if (!nve0_fifo_swmthd(priv, chid, mthd, data))
                         show &= ~0x00800000;
+                nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008);
         }
 
         if (show) {
                 nv_error(priv, "PBDMA%d:", unit);
-                nouveau_bitfield_print(nve0_fifo_pbdma_intr, show);
+                nouveau_bitfield_print(nve0_fifo_pbdma_intr_0, show);
                 pr_cont("\n");
                 nv_error(priv,
                          "PBDMA%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n",
@@ -853,10 +855,37 @@ nve0_fifo_intr_pbdma(struct nve0_fifo_priv *priv, int unit)
                          subc, mthd, data);
         }
 
-        nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008);
         nv_wr32(priv, 0x040108 + (unit * 0x2000), stat);
 }
 
+static const struct nouveau_bitfield nve0_fifo_pbdma_intr_1[] = {
+        { 0x00000001, "HCE_RE_ILLEGAL_OP" },
+        { 0x00000002, "HCE_RE_ALIGNB" },
+        { 0x00000004, "HCE_PRIV" },
+        { 0x00000008, "HCE_ILLEGAL_MTHD" },
+        { 0x00000010, "HCE_ILLEGAL_CLASS" },
+        {}
+};
+
+static void
+nve0_fifo_intr_pbdma_1(struct nve0_fifo_priv *priv, int unit)
+{
+        u32 mask = nv_rd32(priv, 0x04014c + (unit * 0x2000));
+        u32 stat = nv_rd32(priv, 0x040148 + (unit * 0x2000)) & mask;
+        u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff;
+
+        if (stat) {
+                nv_error(priv, "PBDMA%d:", unit);
+                nouveau_bitfield_print(nve0_fifo_pbdma_intr_1, stat);
+                pr_cont("\n");
+                nv_error(priv, "PBDMA%d: ch %d %08x %08x\n", unit, chid,
+                         nv_rd32(priv, 0x040150 + (unit * 0x2000)),
+                         nv_rd32(priv, 0x040154 + (unit * 0x2000)));
+        }
+
+        nv_wr32(priv, 0x040148 + (unit * 0x2000), stat);
+}
+
 static void
 nve0_fifo_intr_runlist(struct nve0_fifo_priv *priv)
 {
@@ -939,7 +968,8 @@ nve0_fifo_intr(struct nouveau_subdev *subdev)
                 u32 mask = nv_rd32(priv, 0x0025a0);
                 while (mask) {
                         u32 unit = __ffs(mask);
-                        nve0_fifo_intr_pbdma(priv, unit);
+                        nve0_fifo_intr_pbdma_0(priv, unit);
+                        nve0_fifo_intr_pbdma_1(priv, unit);
                         nv_wr32(priv, 0x0025a0, (1 << unit));
                         mask &= ~(1 << unit);
                 }
@@ -1022,6 +1052,12 @@ nve0_fifo_init(struct nouveau_object *object)
                 nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */
         }
 
+        /* PBDMA[n].HCE */
+        for (i = 0; i < priv->spoon_nr; i++) {
+                nv_wr32(priv, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */
+                nv_wr32(priv, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */
+        }
+
         nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12);
 
         nv_wr32(priv, 0x002100, 0xffffffff);
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
index 30fd1dc64f93..17251e4b9e86 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
@@ -1557,7 +1557,7 @@ nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                     nvc0_graph_ctor_fw(priv, "fuc409d", &priv->fuc409d) ||
                     nvc0_graph_ctor_fw(priv, "fuc41ac", &priv->fuc41ac) ||
                     nvc0_graph_ctor_fw(priv, "fuc41ad", &priv->fuc41ad))
-                        return -EINVAL;
+                        return -ENODEV;
                 priv->firmware = true;
         }
 
diff --git a/drivers/gpu/drm/nouveau/core/include/core/device.h b/drivers/gpu/drm/nouveau/core/include/core/device.h
index 1d9d893929bb..2ec2e50d3676 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/device.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/device.h
@@ -16,6 +16,7 @@ enum nv_subdev_type {
          * to during POST.
          */
         NVDEV_SUBDEV_DEVINIT,
+        NVDEV_SUBDEV_IBUS,
         NVDEV_SUBDEV_GPIO,
         NVDEV_SUBDEV_I2C,
         NVDEV_SUBDEV_DEVINIT_LAST = NVDEV_SUBDEV_I2C,
@@ -31,7 +32,6 @@ enum nv_subdev_type {
         NVDEV_SUBDEV_TIMER,
         NVDEV_SUBDEV_FB,
         NVDEV_SUBDEV_LTC,
-        NVDEV_SUBDEV_IBUS,
         NVDEV_SUBDEV_INSTMEM,
         NVDEV_SUBDEV_VM,
         NVDEV_SUBDEV_BAR,
@@ -92,6 +92,7 @@ struct nouveau_device {
                 GM100    = 0x110,
         } card_type;
         u32 chipset;
+        u8  chiprev;
         u32 crystal;
 
         struct nouveau_oclass *oclass[NVDEV_SUBDEV_NR];
@@ -158,6 +159,12 @@ nv_device_is_pci(struct nouveau_device *device)
         return device->pdev != NULL;
 }
 
+static inline bool
+nv_device_is_cpu_coherent(struct nouveau_device *device)
+{
+        return (!IS_ENABLED(CONFIG_ARM) && nv_device_is_pci(device));
+}
+
 static inline struct device *
 nv_device_base(struct nouveau_device *device)
 {
diff --git a/drivers/gpu/drm/nouveau/core/include/core/handle.h b/drivers/gpu/drm/nouveau/core/include/core/handle.h
index ceb67d770875..d22a59138a9b 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/handle.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/handle.h
@@ -23,11 +23,6 @@ void nouveau_handle_destroy(struct nouveau_handle *);
 int  nouveau_handle_init(struct nouveau_handle *);
 int  nouveau_handle_fini(struct nouveau_handle *, bool suspend);
 
-int  nouveau_handle_new(struct nouveau_object *, u32 parent, u32 handle,
-                        u16 oclass, void *data, u32 size,
-                        struct nouveau_object **);
-int  nouveau_handle_del(struct nouveau_object *, u32 parent, u32 handle);
-
 struct nouveau_object *
 nouveau_handle_ref(struct nouveau_object *, u32 name);
 
diff --git a/drivers/gpu/drm/nouveau/core/include/core/object.h b/drivers/gpu/drm/nouveau/core/include/core/object.h
index d7039482d6fd..2e2afa502c99 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/object.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/object.h
@@ -203,21 +203,4 @@ nv_memcmp(void *obj, u32 addr, const char *str, u32 len)
         return 0;
 }
 
-#include <core/handle.h>
-
-static inline int
-nouveau_object_new(struct nouveau_object *client, u32 parent, u32 handle,
-                   u16 oclass, void *data, u32 size,
-                   struct nouveau_object **pobject)
-{
-        return nouveau_handle_new(client, parent, handle, oclass,
-                                  data, size, pobject);
-}
-
-static inline int
-nouveau_object_del(struct nouveau_object *client, u32 parent, u32 handle)
-{
-        return nouveau_handle_del(client, parent, handle);
-}
-
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/disp.h b/drivers/gpu/drm/nouveau/core/include/engine/disp.h
index 7a64f347b385..fc307f1317ff 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/disp.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/disp.h
@@ -31,5 +31,6 @@ extern struct nouveau_oclass *nvd0_disp_oclass;
 extern struct nouveau_oclass *nve0_disp_oclass;
 extern struct nouveau_oclass *nvf0_disp_oclass;
 extern struct nouveau_oclass *gm107_disp_oclass;
+extern struct nouveau_oclass *gm204_disp_oclass;
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/M0203.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/M0203.h
new file mode 100644
index 000000000000..1f84d3612dd8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/M0203.h
@@ -0,0 +1,31 @@
+#ifndef __NVBIOS_M0203_H__
+#define __NVBIOS_M0203_H__
+
+struct nvbios_M0203T {
+#define M0203T_TYPE_RAMCFG 0x00
+        u8  type;
+        u16 pointer;
+};
+
+u32 nvbios_M0203Te(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u32 nvbios_M0203Tp(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+                   struct nvbios_M0203T *);
+
+struct nvbios_M0203E {
+#define M0203E_TYPE_DDR2  0x0
+#define M0203E_TYPE_DDR3  0x1
+#define M0203E_TYPE_GDDR3 0x2
+#define M0203E_TYPE_GDDR5 0x3
+#define M0203E_TYPE_SKIP  0xf
+        u8 type;
+        u8 strap;
+        u8 group;
+};
+
+u32 nvbios_M0203Ee(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr);
+u32 nvbios_M0203Ep(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr,
+                   struct nvbios_M0203E *);
+u32 nvbios_M0203Em(struct nouveau_bios *, u8 ramcfg, u8 *ver, u8 *hdr,
+                   struct nvbios_M0203E *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/i2c.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/i2c.h
index 10b57a19a7de..c9bb112895af 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/bios/i2c.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/i2c.h
@@ -4,11 +4,14 @@
 struct nouveau_bios;
 
 enum dcb_i2c_type {
-        DCB_I2C_NV04_BIT = 0,
-        DCB_I2C_NV4E_BIT = 4,
-        DCB_I2C_NVIO_BIT = 5,
-        DCB_I2C_NVIO_AUX = 6,
-        DCB_I2C_UNUSED = 0xff
+        /* matches bios type field prior to ccb 4.1 */
+        DCB_I2C_NV04_BIT = 0x00,
+        DCB_I2C_NV4E_BIT = 0x04,
+        DCB_I2C_NVIO_BIT = 0x05,
+        DCB_I2C_NVIO_AUX = 0x06,
+        /* made up - mostly */
+        DCB_I2C_PMGR     = 0x80,
+        DCB_I2C_UNUSED   = 0xff
 };
 
 struct dcb_i2c_entry {
@@ -16,6 +19,7 @@ struct dcb_i2c_entry {
         u8 drive;
         u8 sense;
         u8 share;
+        u8 auxch;
 };
 
 u16 dcb_i2c_table(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/image.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/image.h
new file mode 100644
index 000000000000..3348b4580843
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/image.h
@@ -0,0 +1,13 @@
+#ifndef __NVBIOS_IMAGE_H__
+#define __NVBIOS_IMAGE_H__
+
+struct nvbios_image {
+        u32  base;
+        u32  size;
+        u8   type;
+        bool last;
+};
+
+bool nvbios_image(struct nouveau_bios *, int, struct nvbios_image *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/npde.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/npde.h
new file mode 100644
index 000000000000..b18413d951e5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/npde.h
@@ -0,0 +1,12 @@
+#ifndef __NVBIOS_NPDE_H__
+#define __NVBIOS_NPDE_H__
+
+struct nvbios_npdeT {
+        u32 image_size;
+        bool last;
+};
+
+u32 nvbios_npdeTe(struct nouveau_bios *, u32);
+u32 nvbios_npdeTp(struct nouveau_bios *, u32, struct nvbios_npdeT *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/pcir.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/pcir.h
new file mode 100644
index 000000000000..3d634a06dca1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/pcir.h
@@ -0,0 +1,18 @@
+#ifndef __NVBIOS_PCIR_H__
+#define __NVBIOS_PCIR_H__
+
+struct nvbios_pcirT {
+        u16 vendor_id;
+        u16 device_id;
+        u8  class_code[3];
+        u32 image_size;
+        u16 image_rev;
+        u8  image_type;
+        bool last;
+};
+
+u32 nvbios_pcirTe(struct nouveau_bios *, u32, u8 *ver, u16 *hdr);
+u32 nvbios_pcirTp(struct nouveau_bios *, u32, u8 *ver, u16 *hdr,
+                  struct nvbios_pcirT *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/pmu.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/pmu.h
new file mode 100644
index 000000000000..9de593deaea8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/pmu.h
@@ -0,0 +1,37 @@
+#ifndef __NVBIOS_PMU_H__
+#define __NVBIOS_PMU_H__
+
+struct nvbios_pmuT {
+};
+
+u32 nvbios_pmuTe(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u32 nvbios_pmuTp(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+                 struct nvbios_pmuT *);
+
+struct nvbios_pmuE {
+        u8  type;
+        u32 data;
+};
+
+u32 nvbios_pmuEe(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr);
+u32 nvbios_pmuEp(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr,
+                 struct nvbios_pmuE *);
+
+struct nvbios_pmuR {
+        u32 boot_addr_pmu;
+        u32 boot_addr;
+        u32 boot_size;
+        u32 code_addr_pmu;
+        u32 code_addr;
+        u32 code_size;
+        u32 init_addr_pmu;
+
+        u32 data_addr_pmu;
+        u32 data_addr;
+        u32 data_size;
+        u32 args_addr_pmu;
+};
+
+bool nvbios_pmuRm(struct nouveau_bios *, u8 type, struct nvbios_pmuR *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h
index a685bbd04568..4a0e0ceb41ba 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h
@@ -43,8 +43,9 @@ struct nvbios_ramcfg {
                         unsigned ramcfg_10_02_08:1;
                         unsigned ramcfg_10_02_10:1;
                         unsigned ramcfg_10_02_20:1;
-                        unsigned ramcfg_10_02_40:1;
+                        unsigned ramcfg_10_DLLoff:1;
                         unsigned ramcfg_10_03_0f:4;
+                        unsigned ramcfg_10_04_01:1;
                         unsigned ramcfg_10_05:8;
                         unsigned ramcfg_10_06:8;
                         unsigned ramcfg_10_07:8;
@@ -95,9 +96,29 @@ struct nvbios_ramcfg {
         union {
                 struct {
                         unsigned timing_10_WR:8;
+                        unsigned timing_10_WTR:8;
                         unsigned timing_10_CL:8;
+                        unsigned timing_10_RC:8;
+                        /*empty: 4 */
+                        unsigned timing_10_RFC:8;        /* Byte 5 */
+                        /*empty: 6 */
+                        unsigned timing_10_RAS:8;        /* Byte 7 */
+                        /*empty: 8 */
+                        unsigned timing_10_RP:8;         /* Byte 9 */
+                        unsigned timing_10_RCDRD:8;
+                        unsigned timing_10_RCDWR:8;
+                        unsigned timing_10_RRD:8;
+                        unsigned timing_10_13:8;
                         unsigned timing_10_ODT:3;
+                        /* empty: 15 */
+                        unsigned timing_10_16:8;
+                        /* empty: 17 */
+                        unsigned timing_10_18:8;
                         unsigned timing_10_CWL:8;
+                        unsigned timing_10_20:8;
+                        unsigned timing_10_21:8;
+                        /* empty: 22, 23 */
+                        unsigned timing_10_24:8;
                 };
                 struct {
                         unsigned timing_20_2e_03:2;
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h b/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h
index e292271a84e4..e007a9d44683 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h
@@ -30,5 +30,6 @@ extern struct nouveau_oclass *nva3_devinit_oclass;
 extern struct nouveau_oclass *nvaf_devinit_oclass;
 extern struct nouveau_oclass *nvc0_devinit_oclass;
 extern struct nouveau_oclass *gm107_devinit_oclass;
+extern struct nouveau_oclass *gm204_devinit_oclass;
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h b/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
index 1b937c2c25ae..d94ccacb40bf 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
@@ -8,6 +8,8 @@
 #include <subdev/bios/i2c.h>
 
 #define NV_I2C_PORT(n)    (0x00 + (n))
+#define NV_I2C_AUX(n)     (0x10 + (n))
+#define NV_I2C_EXT(n)     (0x20 + (n))
 #define NV_I2C_DEFAULT(n) (0x80 + (n))
 
 #define NV_I2C_TYPE_DCBI2C(n) (0x0000 | (n))
@@ -89,6 +91,7 @@ extern struct nouveau_oclass *nv94_i2c_oclass;
 extern struct nouveau_oclass *nvd0_i2c_oclass;
 extern struct nouveau_oclass *gf117_i2c_oclass;
 extern struct nouveau_oclass *nve0_i2c_oclass;
+extern struct nouveau_oclass *gm204_i2c_oclass;
 
 static inline int
 nv_rdi2cr(struct nouveau_i2c_port *port, u8 addr, u8 reg)
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h b/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h
index bf3d1f611333..f2427bf5aeed 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h
@@ -48,6 +48,8 @@ void nouveau_memx_wait(struct nouveau_memx *,
                        u32 addr, u32 mask, u32 data, u32 nsec);
 void nouveau_memx_nsec(struct nouveau_memx *, u32 nsec);
 void nouveau_memx_wait_vblank(struct nouveau_memx *);
+void nouveau_memx_train(struct nouveau_memx *);
+int  nouveau_memx_train_result(struct nouveau_pwr *, u32 *, int);
 void nouveau_memx_block(struct nouveau_memx *);
 void nouveau_memx_unblock(struct nouveau_memx *);
 
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/volt.h b/drivers/gpu/drm/nouveau/core/include/subdev/volt.h
index 820b62ffd75b..67db5e58880d 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/volt.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/volt.h
@@ -52,6 +52,7 @@ int  _nouveau_volt_init(struct nouveau_object *);
 #define _nouveau_volt_fini _nouveau_subdev_fini
 
 extern struct nouveau_oclass nv40_volt_oclass;
+extern struct nouveau_oclass gk20a_volt_oclass;
 
 int nouveau_voltgpio_init(struct nouveau_volt *);
 int nouveau_voltgpio_get(struct nouveau_volt *);
diff --git a/drivers/gpu/drm/nouveau/core/os.h b/drivers/gpu/drm/nouveau/core/os.h
index ccfa21d72ddc..bdd05ee7ec72 100644
--- a/drivers/gpu/drm/nouveau/core/os.h
+++ b/drivers/gpu/drm/nouveau/core/os.h
@@ -23,6 +23,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/power_supply.h>
 #include <linux/clk.h>
+#include <linux/regulator/consumer.h>
 
 #include <asm/unaligned.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/M0203.c b/drivers/gpu/drm/nouveau/core/subdev/bios/M0203.c
new file mode 100644
index 000000000000..28906b16d4e5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/M0203.c
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/M0203.h>
+
+u32
+nvbios_M0203Te(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+        struct bit_entry bit_M;
+        u32 data = 0x00000000;
+
+        if (!bit_entry(bios, 'M', &bit_M)) {
+                if (bit_M.version == 2 && bit_M.length > 0x04)
+                        data = nv_ro16(bios, bit_M.offset + 0x03);
+                if (data) {
+                        *ver = nv_ro08(bios, data + 0x00);
+                        switch (*ver) {
+                        case 0x10:
+                                *hdr = nv_ro08(bios, data + 0x01);
+                                *len = nv_ro08(bios, data + 0x02);
+                                *cnt = nv_ro08(bios, data + 0x03);
+                                return data;
+                        default:
+                                break;
+                        }
+                }
+        }
+
+        return 0x00000000;
+}
+
+u32
+nvbios_M0203Tp(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+               struct nvbios_M0203T *info)
+{
+        u32 data = nvbios_M0203Te(bios, ver, hdr, cnt, len);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!data * *ver) {
+        case 0x10:
+                info->type    = nv_ro08(bios, data + 0x04);
+                info->pointer = nv_ro16(bios, data + 0x05);
+                break;
+        default:
+                break;
+        }
+        return data;
+}
+
+u32
+nvbios_M0203Ee(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr)
+{
+        u8  cnt, len;
+        u32 data = nvbios_M0203Te(bios, ver, hdr, &cnt, &len);
+        if (data && idx < cnt) {
+                data = data + *hdr + idx * len;
+                *hdr = len;
+                return data;
+        }
+        return 0x00000000;
+}
+
+u32
+nvbios_M0203Ep(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr,
+               struct nvbios_M0203E *info)
+{
+        u32 data = nvbios_M0203Ee(bios, idx, ver, hdr);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!data * *ver) {
+        case 0x10:
+                info->type  = (nv_ro08(bios, data + 0x00) & 0x0f) >> 0;
+                info->strap = (nv_ro08(bios, data + 0x00) & 0xf0) >> 4;
+                info->group = (nv_ro08(bios, data + 0x01) & 0x0f) >> 0;
+                return data;
+        default:
+                break;
+        }
+        return 0x00000000;
+}
+
+u32
+nvbios_M0203Em(struct nouveau_bios *bios, u8 ramcfg, u8 *ver, u8 *hdr,
+               struct nvbios_M0203E *info)
+{
+        struct nvbios_M0203T M0203T;
+        u8  cnt, len, idx = 0xff;
+        u32 data;
+
+        if (!nvbios_M0203Tp(bios, ver, hdr, &cnt, &len, &M0203T)) {
+                nv_warn(bios, "M0203T not found\n");
+                return 0x00000000;
+        }
+
+        while ((data = nvbios_M0203Ep(bios, ++idx, ver, hdr, info))) {
+                switch (M0203T.type) {
+                case M0203T_TYPE_RAMCFG:
+                        if (info->strap != ramcfg)
+                                continue;
+                        return data;
+                default:
+                        nv_warn(bios, "M0203T type %02x\n", M0203T.type);
+                        return 0x00000000;
+                }
+        }
+
+        return data;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/base.c b/drivers/gpu/drm/nouveau/core/subdev/bios/base.c
index d45704a2c2df..7df3a273553d 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/base.c
@@ -31,6 +31,8 @@
 #include <subdev/bios/bmp.h>
 #include <subdev/bios/bit.h>
 
+#include "priv.h"
+
 u8
 nvbios_checksum(const u8 *data, int size)
 {
@@ -56,362 +58,21 @@ nvbios_findstr(const u8 *data, int size, const char *str, int len)
         return 0;
 }
 
-#if defined(__powerpc__)
-static void
-nouveau_bios_shadow_of(struct nouveau_bios *bios)
+int
+nvbios_extend(struct nouveau_bios *bios, u32 length)
 {
-        struct pci_dev *pdev = nv_device(bios)->pdev;
-        struct device_node *dn;
-        const u32 *data;
-        int size;
-
-        dn = pci_device_to_OF_node(pdev);
-        if (!dn) {
-                nv_info(bios, "Unable to get the OF node\n");
-                return;
-        }
-
-        data = of_get_property(dn, "NVDA,BMP", &size);
-        if (data && size) {
-                bios->size = size;
-                bios->data = kmalloc(bios->size, GFP_KERNEL);
-                if (bios->data)
-                        memcpy(bios->data, data, size);
-        }
-}
-#endif
-
-static void
-nouveau_bios_shadow_pramin(struct nouveau_bios *bios)
-{
-        struct nouveau_device *device = nv_device(bios);
-        u64 addr = 0;
-        u32 bar0 = 0;
-        int i;
-
-        if (device->card_type >= NV_50) {
-                if (device->card_type >= NV_C0 && device->card_type < GM100) {
-                        if (nv_rd32(bios, 0x022500) & 0x00000001)
-                                return;
-                } else
-                if (device->card_type >= GM100) {
-                        if (nv_rd32(bios, 0x021c04) & 0x00000001)
-                                return;
-                }
-
-                addr = nv_rd32(bios, 0x619f04);
-                if (!(addr & 0x00000008)) {
-                        nv_debug(bios, "... not enabled\n");
-                        return;
+        if (bios->size < length) {
+                u8 *prev = bios->data;
+                if (!(bios->data = kmalloc(length, GFP_KERNEL))) {
+                        bios->data = prev;
+                        return -ENOMEM;
                 }
-                if ( (addr & 0x00000003) != 1) {
-                        nv_debug(bios, "... not in vram\n");
-                        return;
-                }
-
-                addr = (addr & 0xffffff00) << 8;
-                if (!addr) {
-                        addr  = (u64)nv_rd32(bios, 0x001700) << 16;
-                        addr += 0xf0000;
-                }
-
-                bar0 = nv_mask(bios, 0x001700, 0xffffffff, addr >> 16);
-        }
-
-        /* bail if no rom signature */
-        if (nv_rd08(bios, 0x700000) != 0x55 ||
-            nv_rd08(bios, 0x700001) != 0xaa)
-                goto out;
-
-        bios->size = nv_rd08(bios, 0x700002) * 512;
-        if (!bios->size)
-                goto out;
-
-        bios->data = kmalloc(bios->size, GFP_KERNEL);
-        if (bios->data) {
-                for (i = 0; i < bios->size; i++)
-                        nv_wo08(bios, i, nv_rd08(bios, 0x700000 + i));
-        }
-
-out:
-        if (device->card_type >= NV_50)
-                nv_wr32(bios, 0x001700, bar0);
-}
-
-static void
-nouveau_bios_shadow_prom(struct nouveau_bios *bios)
-{
-        struct nouveau_device *device = nv_device(bios);
-        u32 pcireg, access;
-        u16 pcir;
-        int i;
-
-        /* there is no prom on nv4x IGP's */
-        if (device->card_type == NV_40 && device->chipset >= 0x4c)
-                return;
-
-        /* enable access to rom */
-        if (device->card_type >= NV_50)
-                pcireg = 0x088050;
-        else
-                pcireg = 0x001850;
-        access = nv_mask(bios, pcireg, 0x00000001, 0x00000000);
-
-        /* WARNING: PROM accesses should always be 32-bits aligned. Other
-         * accesses work on most chipset but do not on Kepler chipsets
-         */
-
-        /* bail if no rom signature, with a workaround for a PROM reading
-         * issue on some chipsets.  the first read after a period of
-         * inactivity returns the wrong result, so retry the first header
-         * byte a few times before giving up as a workaround
-         */
-        i = 16;
-        do {
-                u32 data = le32_to_cpu(nv_rd32(bios, 0x300000)) & 0xffff;
-                if (data == 0xaa55)
-                        break;
-        } while (i--);
-
-        if (!i)
-                goto out;
-
-        /* read entire bios image to system memory */
-        bios->size = (le32_to_cpu(nv_rd32(bios, 0x300000)) >> 16) & 0xff;
-        bios->size = bios->size * 512;
-        if (!bios->size)
-                goto out;
-
-        bios->data = kmalloc(bios->size, GFP_KERNEL);
-        if (!bios->data)
-                goto out;
-
-        for (i = 0; i < bios->size; i += 4)
-                ((u32 *)bios->data)[i/4] = nv_rd32(bios, 0x300000 + i);
-
-        /* check the PCI record header */
-        pcir = nv_ro16(bios, 0x0018);
-        if (bios->data[pcir + 0] != 'P' ||
-            bios->data[pcir + 1] != 'C' ||
-            bios->data[pcir + 2] != 'I' ||
-            bios->data[pcir + 3] != 'R') {
-                bios->size = 0;
-                kfree(bios->data);
-        }
-
-out:
-        /* disable access to rom */
-        nv_wr32(bios, pcireg, access);
-}
-
-#if defined(CONFIG_ACPI) && defined(CONFIG_X86)
-int nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len);
-bool nouveau_acpi_rom_supported(struct pci_dev *pdev);
-#else
-static inline bool
-nouveau_acpi_rom_supported(struct pci_dev *pdev) {
-        return false;
-}
-
-static inline int
-nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len) {
-        return -EINVAL;
-}
-#endif
-
-static void
-nouveau_bios_shadow_acpi(struct nouveau_bios *bios)
-{
-        struct pci_dev *pdev = nv_device(bios)->pdev;
-        int ret, cnt, i;
-
-        if (!nouveau_acpi_rom_supported(pdev)) {
-                bios->data = NULL;
-                return;
-        }
-
-        bios->size = 0;
-        bios->data = kmalloc(4096, GFP_KERNEL);
-        if (bios->data) {
-                if (nouveau_acpi_get_bios_chunk(bios->data, 0, 4096) == 4096)
-                        bios->size = bios->data[2] * 512;
-                kfree(bios->data);
+                memcpy(bios->data, prev, bios->size);
+                bios->size = length;
+                kfree(prev);
+                return 1;
         }
-
-        if (!bios->size)
-                return;
-
-        bios->data = kmalloc(bios->size, GFP_KERNEL);
-        if (bios->data) {
-                /* disobey the acpi spec - much faster on at least w530 ... */
-                ret = nouveau_acpi_get_bios_chunk(bios->data, 0, bios->size);
-                if (ret != bios->size ||
-                    nvbios_checksum(bios->data, bios->size)) {
-                        /* ... that didn't work, ok, i'll be good now */
-                        for (i = 0; i < bios->size; i += cnt) {
-                                cnt = min((bios->size - i), (u32)4096);
-                                ret = nouveau_acpi_get_bios_chunk(bios->data, i, cnt);
-                                if (ret != cnt)
-                                        break;
-                        }
-                }
-        }
-}
-
-static void
-nouveau_bios_shadow_pci(struct nouveau_bios *bios)
-{
-        struct pci_dev *pdev = nv_device(bios)->pdev;
-        size_t size;
-
-        if (!pci_enable_rom(pdev)) {
-                void __iomem *rom = pci_map_rom(pdev, &size);
-                if (rom && size) {
-                        bios->data = kmalloc(size, GFP_KERNEL);
-                        if (bios->data) {
-                                memcpy_fromio(bios->data, rom, size);
-                                bios->size = size;
-                        }
-                }
-                if (rom)
-                        pci_unmap_rom(pdev, rom);
-
-                pci_disable_rom(pdev);
-        }
-}
-
-static void
-nouveau_bios_shadow_platform(struct nouveau_bios *bios)
-{
-        struct pci_dev *pdev = nv_device(bios)->pdev;
-        size_t size;
-
-        void __iomem *rom = pci_platform_rom(pdev, &size);
-        if (rom && size) {
-                bios->data = kmalloc(size, GFP_KERNEL);
-                if (bios->data) {
-                        memcpy_fromio(bios->data, rom, size);
-                        bios->size = size;
-                }
-        }
-}
-
-static int
-nouveau_bios_score(struct nouveau_bios *bios, const bool writeable)
-{
-        if (bios->size < 3 || !bios->data || bios->data[0] != 0x55 ||
-                        bios->data[1] != 0xAA) {
-                nv_info(bios, "... signature not found\n");
-                return 0;
-        }
-
-        if (nvbios_checksum(bios->data,
-                        min_t(u32, bios->data[2] * 512, bios->size))) {
-                nv_info(bios, "... checksum invalid\n");
-                /* if a ro image is somewhat bad, it's probably all rubbish */
-                return writeable ? 2 : 1;
-        }
-
-        nv_info(bios, "... appears to be valid\n");
-        return 3;
-}
-
-struct methods {
-        const char desc[16];
-        void (*shadow)(struct nouveau_bios *);
-        const bool rw;
-        int score;
-        u32 size;
-        u8 *data;
-};
-
-static int
-nouveau_bios_shadow(struct nouveau_bios *bios)
-{
-        struct methods shadow_methods[] = {
-#if defined(__powerpc__)
-                { "OpenFirmware", nouveau_bios_shadow_of, true, 0, 0, NULL },
-#endif
-                { "PRAMIN", nouveau_bios_shadow_pramin, true, 0, 0, NULL },
-                { "PROM", nouveau_bios_shadow_prom, false, 0, 0, NULL },
-                { "ACPI", nouveau_bios_shadow_acpi, true, 0, 0, NULL },
-                { "PCIROM", nouveau_bios_shadow_pci, true, 0, 0, NULL },
-                { "PLATFORM", nouveau_bios_shadow_platform, true, 0, 0, NULL },
-                {}
-        };
-        struct methods *mthd, *best;
-        const struct firmware *fw;
-        const char *optarg;
-        int optlen, ret;
-        char *source;
-
-        optarg = nouveau_stropt(nv_device(bios)->cfgopt, "NvBios", &optlen);
-        source = optarg ? kstrndup(optarg, optlen, GFP_KERNEL) : NULL;
-        if (source) {
-                /* try to match one of the built-in methods */
-                mthd = shadow_methods;
-                do {
-                        if (strcasecmp(source, mthd->desc))
-                                continue;
-                        nv_info(bios, "source: %s\n", mthd->desc);
-
-                        mthd->shadow(bios);
-                        mthd->score = nouveau_bios_score(bios, mthd->rw);
-                        if (mthd->score) {
-                                kfree(source);
-                                return 0;
-                        }
-                } while ((++mthd)->shadow);
-
-                /* attempt to load firmware image */
-                ret = request_firmware(&fw, source, &nv_device(bios)->pdev->dev);
-                if (ret == 0) {
-                        bios->size = fw->size;
-                        bios->data = kmemdup(fw->data, fw->size, GFP_KERNEL);
-                        release_firmware(fw);
-
-                        nv_info(bios, "image: %s\n", source);
-                        if (nouveau_bios_score(bios, 1)) {
-                                kfree(source);
-                                return 0;
-                        }
-
-                        kfree(bios->data);
-                        bios->data = NULL;
-                }
-
-                nv_error(bios, "source \'%s\' invalid\n", source);
-                kfree(source);
-        }
-
-        mthd = shadow_methods;
-        do {
-                nv_info(bios, "checking %s for image...\n", mthd->desc);
-                mthd->shadow(bios);
-                mthd->score = nouveau_bios_score(bios, mthd->rw);
-                mthd->size = bios->size;
-                mthd->data = bios->data;
-                bios->data = NULL;
-        } while (mthd->score != 3 && (++mthd)->shadow);
-
-        mthd = shadow_methods;
-        best = mthd;
-        do {
-                if (mthd->score > best->score) {
-                        kfree(best->data);
-                        best = mthd;
-                }
-        } while ((++mthd)->shadow);
-
-        if (best->score) {
-                nv_info(bios, "using image from %s\n", best->desc);
-                bios->size = best->size;
-                bios->data = best->data;
-                return 0;
-        }
-
-        nv_error(bios, "unable to locate usable image\n");
-        return -EINVAL;
+        return 0;
 }
 
 static u8
@@ -472,7 +133,7 @@ nouveau_bios_ctor(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
-        ret = nouveau_bios_shadow(bios);
+        ret = nvbios_shadow(bios);
         if (ret)
                 return ret;
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c b/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c
index bd8d348385b3..96099aff8b41 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c
@@ -42,7 +42,7 @@ dcb_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 
         *ver = nv_ro08(bios, dcb);
 
-        if (*ver >= 0x41) {
+        if (*ver >= 0x42) {
                 nv_warn(bios, "DCB version 0x%02x unknown\n", *ver);
                 return 0x0000;
         } else
@@ -157,17 +157,20 @@ dcb_outp_parse(struct nouveau_bios *bios, u8 idx, u8 *ver, u8 *len,
                                         break;
                                 }
 
-                                switch (conf & 0x0f000000) {
-                                case 0x0f000000:
-                                        outp->dpconf.link_nr = 4;
-                                        break;
-                                case 0x03000000:
-                                        outp->dpconf.link_nr = 2;
-                                        break;
-                                case 0x01000000:
-                                default:
-                                        outp->dpconf.link_nr = 1;
-                                        break;
+                                outp->dpconf.link_nr = (conf & 0x0f000000) >> 24;
+                                if (*ver < 0x41) {
+                                        switch (outp->dpconf.link_nr) {
+                                        case 0x0f:
+                                                outp->dpconf.link_nr = 4;
+                                                break;
+                                        case 0x03:
+                                                outp->dpconf.link_nr = 2;
+                                                break;
+                                        case 0x01:
+                                        default:
+                                                outp->dpconf.link_nr = 1;
+                                                break;
+                                        }
                                 }
 
                                 /* fall-through... */
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/disp.c b/drivers/gpu/drm/nouveau/core/subdev/bios/disp.c
index 7f16e52d9bea..51f355599694 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/disp.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/disp.c
@@ -40,6 +40,7 @@ nvbios_disp_table(struct nouveau_bios *bios,
                                 switch (*ver) {
                                 case 0x20:
                                 case 0x21:
+                                case 0x22:
                                         *hdr = nv_ro08(bios, data + 0x01);
                                         *len = nv_ro08(bios, data + 0x02);
                                         *cnt = nv_ro08(bios, data + 0x03);
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c b/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c
index f309dd657250..cef53f81f12b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c
@@ -41,6 +41,7 @@ nvbios_dp_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
                                 case 0x21:
                                 case 0x30:
                                 case 0x40:
+                                case 0x41:
                                         *hdr = nv_ro08(bios, data + 0x01);
                                         *len = nv_ro08(bios, data + 0x02);
                                         *cnt = nv_ro08(bios, data + 0x03);
@@ -70,6 +71,7 @@ nvbios_dpout_entry(struct nouveau_bios *bios, u8 idx,
                         *cnt = nv_ro08(bios, outp + 0x04);
                         break;
                 case 0x40:
+                case 0x41:
                         *hdr = nv_ro08(bios, data + 0x04);
                         *cnt = 0;
                         *len = 0;
@@ -108,6 +110,7 @@ nvbios_dpout_parse(struct nouveau_bios *bios, u8 idx,
                                 info->script[4] = nv_ro16(bios, data + 0x10);
                         break;
                 case 0x40:
+                case 0x41:
                         info->flags     = nv_ro08(bios, data + 0x04);
                         info->script[0] = nv_ro16(bios, data + 0x05);
                         info->script[1] = nv_ro16(bios, data + 0x07);
@@ -172,10 +175,11 @@ nvbios_dpcfg_parse(struct nouveau_bios *bios, u16 outp, u8 idx,
                         break;
                 case 0x30:
                 case 0x40:
+                case 0x41:
                         info->pc    = nv_ro08(bios, data + 0x00);
                         info->dc    = nv_ro08(bios, data + 0x01);
                         info->pe    = nv_ro08(bios, data + 0x02);
-                        info->tx_pu = nv_ro08(bios, data + 0x03);
+                        info->tx_pu = nv_ro08(bios, data + 0x03) & 0x0f;
                         break;
                 default:
                         data = 0x0000;
@@ -194,6 +198,10 @@ nvbios_dpcfg_match(struct nouveau_bios *bios, u16 outp, u8 pc, u8 vs, u8 pe,
         u16 data;
 
         if (*ver >= 0x30) {
+                /*XXX: there's a second set of these on at least 4.1, that
+                 *     i've witnessed nvidia using instead of the first
+                 *     on gm204.  figure out what/why
+                 */
                 const u8 vsoff[] = { 0, 4, 7, 9 };
                 idx = (pc * 10) + vsoff[vs] + pe;
         } else {
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/extdev.c b/drivers/gpu/drm/nouveau/core/subdev/bios/extdev.c
index b2a676e53580..49285d4f7ca5 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/extdev.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/extdev.c
@@ -90,7 +90,7 @@ nvbios_extdev_find(struct nouveau_bios *bios, enum nvbios_extdev_type type,
         u16 entry;
 
         i = 0;
-        while (!(entry = nvbios_extdev_entry(bios, i++, &ver, &len))) {
+        while ((entry = nvbios_extdev_entry(bios, i++, &ver, &len))) {
                 extdev_parse_entry(bios, entry, func);
                 if (func->type == type)
                         return 0;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/i2c.c b/drivers/gpu/drm/nouveau/core/subdev/bios/i2c.c
index cfb9288c6d28..282320ba9264 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/i2c.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/i2c.c
@@ -39,6 +39,11 @@ dcb_i2c_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
                         i2c = nv_ro16(bios, dcb + 4);
         }
 
+        if (i2c && *ver >= 0x42) {
+                nv_warn(bios, "ccb %02x not supported\n", *ver);
+                return 0x0000;
+        }
+
         if (i2c && *ver >= 0x30) {
                 *ver = nv_ro08(bios, i2c + 0);
                 *hdr = nv_ro08(bios, i2c + 1);
@@ -70,14 +75,25 @@ dcb_i2c_parse(struct nouveau_bios *bios, u8 idx, struct dcb_i2c_entry *info)
         u8  ver, len;
         u16 ent = dcb_i2c_entry(bios, idx, &ver, &len);
         if (ent) {
-                info->type  = nv_ro08(bios, ent + 3);
-                info->share = DCB_I2C_UNUSED;
-                if (ver < 0x30) {
-                        info->type &= 0x07;
+                if (ver >= 0x41) {
+                        if (!(nv_ro32(bios, ent) & 0x80000000))
+                                info->type = DCB_I2C_UNUSED;
+                        else
+                                info->type = DCB_I2C_PMGR;
+                } else
+                if (ver >= 0x30) {
+                        info->type = nv_ro08(bios, ent + 0x03);
+                } else {
+                        info->type = nv_ro08(bios, ent + 0x03) & 0x07;
                         if (info->type == 0x07)
                                 info->type = DCB_I2C_UNUSED;
                 }
 
+                info->drive = DCB_I2C_UNUSED;
+                info->sense = DCB_I2C_UNUSED;
+                info->share = DCB_I2C_UNUSED;
+                info->auxch = DCB_I2C_UNUSED;
+
                 switch (info->type) {
                 case DCB_I2C_NV04_BIT:
                         info->drive = nv_ro08(bios, ent + 0);
@@ -87,12 +103,23 @@ dcb_i2c_parse(struct nouveau_bios *bios, u8 idx, struct dcb_i2c_entry *info)
                         info->drive = nv_ro08(bios, ent + 1);
                         return 0;
                 case DCB_I2C_NVIO_BIT:
-                case DCB_I2C_NVIO_AUX:
                         info->drive = nv_ro08(bios, ent + 0) & 0x0f;
-                        if (nv_ro08(bios, ent + 1) & 0x01) {
-                                info->share  = nv_ro08(bios, ent + 1) >> 1;
-                                info->share &= 0x0f;
-                        }
+                        if (nv_ro08(bios, ent + 1) & 0x01)
+                                info->share = nv_ro08(bios, ent + 1) >> 1;
+                        return 0;
+                case DCB_I2C_NVIO_AUX:
+                        info->auxch = nv_ro08(bios, ent + 0) & 0x0f;
+                        if (nv_ro08(bios, ent + 1) & 0x01)
+                                        info->share = info->auxch;
+                        return 0;
+                case DCB_I2C_PMGR:
+                        info->drive = (nv_ro16(bios, ent + 0) & 0x01f) >> 0;
+                        if (info->drive == 0x1f)
+                                info->drive = DCB_I2C_UNUSED;
+                        info->auxch = (nv_ro16(bios, ent + 0) & 0x3e0) >> 5;
+                        if (info->auxch == 0x1f)
+                                info->auxch = DCB_I2C_UNUSED;
+                        info->share = info->auxch;
                         return 0;
                 case DCB_I2C_UNUSED:
                         return 0;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/image.c b/drivers/gpu/drm/nouveau/core/subdev/bios/image.c
new file mode 100644
index 000000000000..373f9a564ac9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/image.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/image.h>
+#include <subdev/bios/pcir.h>
+#include <subdev/bios/npde.h>
+
+static bool
+nvbios_imagen(struct nouveau_bios *bios, struct nvbios_image *image)
+{
+        struct nvbios_pcirT pcir;
+        struct nvbios_npdeT npde;
+        u8  ver;
+        u16 hdr;
+        u32 data;
+
+        switch ((data = nv_ro16(bios, image->base + 0x00))) {
+        case 0xaa55:
+        case 0xbb77:
+        case 0x4e56: /* NV */
+                break;
+        default:
+                nv_debug(bios, "%08x: ROM signature (%04x) unknown\n",
+                         image->base, data);
+                return false;
+        }
+
+        if (!(data = nvbios_pcirTp(bios, image->base, &ver, &hdr, &pcir)))
+                return false;
+        image->size = pcir.image_size;
+        image->type = pcir.image_type;
+        image->last = pcir.last;
+
+        if (image->type != 0x70) {
+                if (!(data = nvbios_npdeTp(bios, image->base, &npde)))
+                        return true;
+                image->size = npde.image_size;
+                image->last = npde.last;
+        } else {
+                image->last = true;
+        }
+
+        return true;
+}
+
+bool
+nvbios_image(struct nouveau_bios *bios, int idx, struct nvbios_image *image)
+{
+        memset(image, 0x00, sizeof(*image));
+        do {
+                image->base += image->size;
+                if (image->last || !nvbios_imagen(bios, image))
+                        return false;
+        } while(idx--);
+        return true;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/init.c b/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
index 626380f9e4c0..c6579ef32cd1 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
@@ -255,6 +255,8 @@ init_i2c(struct nvbios_init *init, int index)
                 }
 
                 index = init->outp->i2c_index;
+                if (init->outp->type == DCB_OUTPUT_DP)
+                        index += NV_I2C_AUX(0);
         }
 
         return i2c->find(i2c, index);
@@ -278,7 +280,7 @@ init_wri2cr(struct nvbios_init *init, u8 index, u8 addr, u8 reg, u8 val)
         return -ENODEV;
 }
 
-static int
+static u8
 init_rdauxr(struct nvbios_init *init, u32 addr)
 {
         struct nouveau_i2c_port *port = init_i2c(init, -2);
@@ -286,20 +288,24 @@ init_rdauxr(struct nvbios_init *init, u32 addr)
 
         if (port && init_exec(init)) {
                 int ret = nv_rdaux(port, addr, &data, 1);
-                if (ret)
-                        return ret;
-                return data;
+                if (ret == 0)
+                        return data;
+                trace("auxch read failed with %d\n", ret);
         }
 
-        return -ENODEV;
+        return 0x00;
 }
 
 static int
 init_wrauxr(struct nvbios_init *init, u32 addr, u8 data)
 {
         struct nouveau_i2c_port *port = init_i2c(init, -2);
-        if (port && init_exec(init))
-                return nv_wraux(port, addr, &data, 1);
+        if (port && init_exec(init)) {
+                int ret = nv_wraux(port, addr, &data, 1);
+                if (ret)
+                        trace("auxch write failed with %d\n", ret);
+                return ret;
+        }
         return -ENODEV;
 }
 
@@ -838,6 +844,40 @@ init_io_or(struct nvbios_init *init)
 }
 
 /**
+ * INIT_ANDN_REG - opcode 0x47
+ *
+ */
+static void
+init_andn_reg(struct nvbios_init *init)
+{
+        struct nouveau_bios *bios = init->bios;
+        u32  reg = nv_ro32(bios, init->offset + 1);
+        u32 mask = nv_ro32(bios, init->offset + 5);
+
+        trace("ANDN_REG\tR[0x%06x] &= ~0x%08x\n", reg, mask);
+        init->offset += 9;
+
+        init_mask(init, reg, mask, 0);
+}
+
+/**
+ * INIT_OR_REG - opcode 0x48
+ *
+ */
+static void
+init_or_reg(struct nvbios_init *init)
+{
+        struct nouveau_bios *bios = init->bios;
+        u32  reg = nv_ro32(bios, init->offset + 1);
+        u32 mask = nv_ro32(bios, init->offset + 5);
+
+        trace("OR_REG\tR[0x%06x] |= 0x%08x\n", reg, mask);
+        init->offset += 9;
+
+        init_mask(init, reg, 0, mask);
+}
+
+/**
  * INIT_INDEX_ADDRESS_LATCHED - opcode 0x49
  *
  */
@@ -2068,6 +2108,8 @@ static struct nvbios_init_opcode {
         [0x3a] = { init_dp_condition },
         [0x3b] = { init_io_mask_or },
         [0x3c] = { init_io_or },
+        [0x47] = { init_andn_reg },
+        [0x48] = { init_or_reg },
         [0x49] = { init_idx_addr_latched },
         [0x4a] = { init_io_restrict_pll2 },
         [0x4b] = { init_pll2 },
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/npde.c b/drivers/gpu/drm/nouveau/core/subdev/bios/npde.c
new file mode 100644
index 000000000000..d694716a166c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/npde.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/npde.h>
+#include <subdev/bios/pcir.h>
+
+u32
+nvbios_npdeTe(struct nouveau_bios *bios, u32 base)
+{
+        struct nvbios_pcirT pcir;
+        u8  ver; u16 hdr;
+        u32 data = nvbios_pcirTp(bios, base, &ver, &hdr, &pcir);
+        if (data = (data + hdr + 0x0f) & ~0x0f, data) {
+                switch (nv_ro32(bios, data + 0x00)) {
+                case 0x4544504e: /* NPDE */
+                        break;
+                default:
+                        nv_debug(bios, "%08x: NPDE signature (%08x) unknown\n",
+                                 data, nv_ro32(bios, data + 0x00));
+                        data = 0;
+                        break;
+                }
+        }
+        return data;
+}
+
+u32
+nvbios_npdeTp(struct nouveau_bios *bios, u32 base, struct nvbios_npdeT *info)
+{
+        u32 data = nvbios_npdeTe(bios, base);
+        memset(info, 0x00, sizeof(*info));
+        if (data) {
+                info->image_size = nv_ro16(bios, data + 0x08) * 512;
+                info->last = nv_ro08(bios, data + 0x0a) & 0x80;
+        }
+        return data;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/pcir.c b/drivers/gpu/drm/nouveau/core/subdev/bios/pcir.c
new file mode 100644
index 000000000000..91dae26bc50f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/pcir.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/pcir.h>
+
+u32
+nvbios_pcirTe(struct nouveau_bios *bios, u32 base, u8 *ver, u16 *hdr)
+{
+        u32 data = nv_ro16(bios, base + 0x18);
+        if (data) {
+                data += base;
+                switch (nv_ro32(bios, data + 0x00)) {
+                case 0x52494350: /* PCIR */
+                case 0x53494752: /* RGIS */
+                case 0x5344504e: /* NPDS */
+                        *hdr = nv_ro16(bios, data + 0x0a);
+                        *ver = nv_ro08(bios, data + 0x0c);
+                        break;
+                default:
+                        nv_debug(bios, "%08x: PCIR signature (%08x) unknown\n",
+                                 data, nv_ro32(bios, data + 0x00));
+                        data = 0;
+                        break;
+                }
+        }
+        return data;
+}
+
+u32
+nvbios_pcirTp(struct nouveau_bios *bios, u32 base, u8 *ver, u16 *hdr,
+              struct nvbios_pcirT *info)
+{
+        u32 data = nvbios_pcirTe(bios, base, ver, hdr);
+        memset(info, 0x00, sizeof(*info));
+        if (data) {
+                info->vendor_id = nv_ro16(bios, data + 0x04);
+                info->device_id = nv_ro16(bios, data + 0x06);
+                info->class_code[0] = nv_ro08(bios, data + 0x0d);
+                info->class_code[1] = nv_ro08(bios, data + 0x0e);
+                info->class_code[2] = nv_ro08(bios, data + 0x0f);
+                info->image_size = nv_ro16(bios, data + 0x10) * 512;
+                info->image_rev = nv_ro16(bios, data + 0x12);
+                info->image_type = nv_ro08(bios, data + 0x14);
+                info->last = nv_ro08(bios, data + 0x15) & 0x80;
+        }
+        return data;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/pmu.c b/drivers/gpu/drm/nouveau/core/subdev/bios/pmu.c
new file mode 100644
index 000000000000..66c56ba07d1b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/pmu.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/image.h>
+#include <subdev/bios/pmu.h>
+
+static u32
+weirdo_pointer(struct nouveau_bios *bios, u32 data)
+{
+        struct nvbios_image image;
+        int idx = 0;
+        if (nvbios_image(bios, idx++, &image)) {
+                data -= image.size;
+                while (nvbios_image(bios, idx++, &image)) {
+                        if (image.type == 0xe0)
+                                return image.base + data;
+                }
+        }
+        return 0;
+}
+
+u32
+nvbios_pmuTe(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+        struct bit_entry bit_p;
+        u32 data = 0;
+
+        if (!bit_entry(bios, 'p', &bit_p)) {
+                if (bit_p.version == 2 && bit_p.length >= 4)
+                        data = nv_ro32(bios, bit_p.offset + 0x00);
+                if ((data = weirdo_pointer(bios, data))) {
+                        *ver = nv_ro08(bios, data + 0x00); /* maybe? */
+                        *hdr = nv_ro08(bios, data + 0x01);
+                        *len = nv_ro08(bios, data + 0x02);
+                        *cnt = nv_ro08(bios, data + 0x03);
+                }
+        }
+
+        return data;
+}
+
+u32
+nvbios_pmuTp(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+             struct nvbios_pmuT *info)
+{
+        u32 data = nvbios_pmuTe(bios, ver, hdr, cnt, len);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!data * *ver) {
+        default:
+                break;
+        }
+        return data;
+}
+
+u32
+nvbios_pmuEe(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr)
+{
+        u8  cnt, len;
+        u32 data = nvbios_pmuTe(bios, ver, hdr, &cnt, &len);
+        if (data && idx < cnt) {
+                data = data + *hdr + (idx * len);
+                *hdr = len;
+                return data;
+        }
+        return 0;
+}
+
+u32
+nvbios_pmuEp(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr,
+             struct nvbios_pmuE *info)
+{
+        u32 data = nvbios_pmuEe(bios, idx, ver, hdr);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!data * *ver) {
+        default:
+                info->type = nv_ro08(bios, data + 0x00);
+                info->data = nv_ro32(bios, data + 0x02);
+                break;
+        }
+        return data;
+}
+
+bool
+nvbios_pmuRm(struct nouveau_bios *bios, u8 type, struct nvbios_pmuR *info)
+{
+        struct nvbios_pmuE pmuE;
+        u8  ver, hdr, idx = 0;
+        u32 data;
+        memset(info, 0x00, sizeof(*info));
+        while ((data = nvbios_pmuEp(bios, idx++, &ver, &hdr, &pmuE))) {
+                if ( pmuE.type == type &&
+                    (data = weirdo_pointer(bios, pmuE.data))) {
+                        info->init_addr_pmu = nv_ro32(bios, data + 0x08);
+                        info->args_addr_pmu = nv_ro32(bios, data + 0x0c);
+                        info->boot_addr     = data + 0x30;
+                        info->boot_addr_pmu = nv_ro32(bios, data + 0x10) +
+                                              nv_ro32(bios, data + 0x18);
+                        info->boot_size     = nv_ro32(bios, data + 0x1c) -
+                                              nv_ro32(bios, data + 0x18);
+                        info->code_addr     = info->boot_addr + info->boot_size;
+                        info->code_addr_pmu = info->boot_addr_pmu +
+                                              info->boot_size;
+                        info->code_size     = nv_ro32(bios, data + 0x20);
+                        info->data_addr     = data + 0x30 +
+                                              nv_ro32(bios, data + 0x24);
+                        info->data_addr_pmu = nv_ro32(bios, data + 0x28);
+                        info->data_size     = nv_ro32(bios, data + 0x2c);
+                        return true;
+                }
+        }
+        return false;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/priv.h b/drivers/gpu/drm/nouveau/core/subdev/bios/priv.h
new file mode 100644
index 000000000000..187d225bd1e9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/priv.h
@@ -0,0 +1,25 @@
+#ifndef __NVKM_BIOS_PRIV_H__
+#define __NVKM_BIOS_PRIV_H__
+
+#include <subdev/bios.h>
+
+struct nvbios_source {
+        const char *name;
+        void *(*init)(struct nouveau_bios *, const char *);
+        void  (*fini)(void *);
+        u32   (*read)(void *, u32 offset, u32 length, struct nouveau_bios *);
+        bool rw;
+};
+
+int nvbios_extend(struct nouveau_bios *, u32 length);
+int nvbios_shadow(struct nouveau_bios *);
+
+extern const struct nvbios_source nvbios_rom;
+extern const struct nvbios_source nvbios_ramin;
+extern const struct nvbios_source nvbios_acpi_fast;
+extern const struct nvbios_source nvbios_acpi_slow;
+extern const struct nvbios_source nvbios_pcirom;
+extern const struct nvbios_source nvbios_platform;
+extern const struct nvbios_source nvbios_of;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c b/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c
index 6c401f70ab99..1623c8dfe797 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c
@@ -25,6 +25,7 @@
 #include <subdev/bios.h>
 #include <subdev/bios/bit.h>
 #include <subdev/bios/ramcfg.h>
+#include <subdev/bios/M0203.h>
 
 static u8
 nvbios_ramcfg_strap(struct nouveau_subdev *subdev)
@@ -54,12 +55,22 @@ nvbios_ramcfg_index(struct nouveau_subdev *subdev)
         u8 strap = nvbios_ramcfg_strap(subdev);
         u32 xlat = 0x00000000;
         struct bit_entry bit_M;
+        struct nvbios_M0203E M0203E;
+        u8 ver, hdr;
 
         if (!bit_entry(bios, 'M', &bit_M)) {
                 if (bit_M.version == 1 && bit_M.length >= 5)
                         xlat = nv_ro16(bios, bit_M.offset + 3);
-                if (bit_M.version == 2 && bit_M.length >= 3)
+                if (bit_M.version == 2 && bit_M.length >= 3) {
+                        /*XXX: is M ever shorter than this?
+                         *     if not - what is xlat used for now?
+                         *     also - sigh..
+                         */
+                        if (bit_M.length >= 7 &&
+                            nvbios_M0203Em(bios, strap, &ver, &hdr, &M0203E))
+                                return M0203E.group;
                         xlat = nv_ro16(bios, bit_M.offset + 1);
+                }
         }
 
         if (xlat)
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/rammap.c b/drivers/gpu/drm/nouveau/core/subdev/bios/rammap.c
index 585e69331ccc..c5685228c322 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/rammap.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/rammap.c
@@ -162,8 +162,9 @@ nvbios_rammapSp(struct nouveau_bios *bios, u32 data,
                 p->ramcfg_10_02_08 = (nv_ro08(bios, data + 0x02) & 0x08) >> 3;
                 p->ramcfg_10_02_10 = (nv_ro08(bios, data + 0x02) & 0x10) >> 4;
                 p->ramcfg_10_02_20 = (nv_ro08(bios, data + 0x02) & 0x20) >> 5;
-                p->ramcfg_10_02_40 = (nv_ro08(bios, data + 0x02) & 0x40) >> 6;
+                p->ramcfg_10_DLLoff = (nv_ro08(bios, data + 0x02) & 0x40) >> 6;
                 p->ramcfg_10_03_0f = (nv_ro08(bios, data + 0x03) & 0x0f) >> 0;
+                p->ramcfg_10_04_01 = (nv_ro08(bios, data + 0x04) & 0x01) >> 0;
                 p->ramcfg_10_05    = (nv_ro08(bios, data + 0x05) & 0xff) >> 0;
                 p->ramcfg_10_06    = (nv_ro08(bios, data + 0x06) & 0xff) >> 0;
                 p->ramcfg_10_07    = (nv_ro08(bios, data + 0x07) & 0xff) >> 0;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadow.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadow.c
new file mode 100644
index 000000000000..bb9e0018d936
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadow.c
@@ -0,0 +1,270 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include "priv.h"
+#include <core/option.h>
+#include <subdev/bios/image.h>
+
+struct shadow {
+        struct nouveau_oclass base;
+        u32 skip;
+        const struct nvbios_source *func;
+        void *data;
+        u32 size;
+        int score;
+};
+
+static bool
+shadow_fetch(struct nouveau_bios *bios, u32 upto)
+{
+        struct shadow *mthd = (void *)nv_object(bios)->oclass;
+        const u32 limit = (upto + 3) & ~3;
+        const u32 start = bios->size;
+        void *data = mthd->data;
+        if (nvbios_extend(bios, limit) > 0) {
+                u32 read = mthd->func->read(data, start, limit - start, bios);
+                bios->size = start + read;
+        }
+        return bios->size >= limit;
+}
+
+static u8
+shadow_rd08(struct nouveau_object *object, u64 addr)
+{
+        struct nouveau_bios *bios = (void *)object;
+        if (shadow_fetch(bios, addr + 1))
+                return bios->data[addr];
+        return 0x00;
+}
+
+static u16
+shadow_rd16(struct nouveau_object *object, u64 addr)
+{
+        struct nouveau_bios *bios = (void *)object;
+        if (shadow_fetch(bios, addr + 2))
+                return get_unaligned_le16(&bios->data[addr]);
+        return 0x0000;
+}
+
+static u32
+shadow_rd32(struct nouveau_object *object, u64 addr)
+{
+        struct nouveau_bios *bios = (void *)object;
+        if (shadow_fetch(bios, addr + 4))
+                return get_unaligned_le32(&bios->data[addr]);
+        return 0x00000000;
+}
+
+static struct nouveau_oclass
+shadow_class = {
+        .handle = NV_SUBDEV(VBIOS, 0x00),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .rd08 = shadow_rd08,
+                .rd16 = shadow_rd16,
+                .rd32 = shadow_rd32,
+        },
+};
+
+static int
+shadow_image(struct nouveau_bios *bios, int idx, struct shadow *mthd)
+{
+        struct nvbios_image image;
+        int score = 1;
+
+        if (!nvbios_image(bios, idx, &image)) {
+                nv_debug(bios, "image %d invalid\n", idx);
+                return 0;
+        }
+        nv_debug(bios, "%08x: type %02x, %d bytes\n",
+                 image.base, image.type, image.size);
+
+        if (!shadow_fetch(bios, image.size)) {
+                nv_debug(bios, "%08x: fetch failed\n", image.base);
+                return 0;
+        }
+
+        switch (image.type) {
+        case 0x00:
+                if (nvbios_checksum(&bios->data[image.base], image.size)) {
+                        nv_debug(bios, "%08x: checksum failed\n", image.base);
+                        if (mthd->func->rw)
+                                score += 1;
+                        score += 1;
+                } else {
+                        score += 3;
+                }
+                break;
+        default:
+                score += 3;
+                break;
+        }
+
+        if (!image.last)
+                score += shadow_image(bios, idx + 1, mthd);
+        return score;
+}
+
+static int
+shadow_score(struct nouveau_bios *bios, struct shadow *mthd)
+{
+        struct nouveau_oclass *oclass = nv_object(bios)->oclass;
+        int score;
+        nv_object(bios)->oclass = &mthd->base;
+        score = shadow_image(bios, 0, mthd);
+        nv_object(bios)->oclass = oclass;
+        return score;
+
+}
+
+static int
+shadow_method(struct nouveau_bios *bios, struct shadow *mthd, const char *name)
+{
+        const struct nvbios_source *func = mthd->func;
+        if (func->name) {
+                nv_debug(bios, "trying %s...\n", name ? name : func->name);
+                if (func->init) {
+                        mthd->data = func->init(bios, name);
+                        if (IS_ERR(mthd->data)) {
+                                mthd->data = NULL;
+                                return 0;
+                        }
+                }
+                mthd->score = shadow_score(bios, mthd);
+                if (func->fini)
+                        func->fini(mthd->data);
+                nv_debug(bios, "scored %d\n", mthd->score);
+                mthd->data = bios->data;
+                mthd->size = bios->size;
+                bios->data  = NULL;
+                bios->size  = 0;
+        }
+        return mthd->score;
+}
+
+static u32
+shadow_fw_read(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        const struct firmware *fw = data;
+        if (offset + length <= fw->size) {
+                memcpy(bios->data + offset, fw->data + offset, length);
+                return length;
+        }
+        return 0;
+}
+
+static void *
+shadow_fw_init(struct nouveau_bios *bios, const char *name)
+{
+        struct device *dev = &nv_device(bios)->pdev->dev;
+        const struct firmware *fw;
+        int ret = request_firmware(&fw, name, dev);
+        if (ret)
+                return ERR_PTR(-ENOENT);
+        return (void *)fw;
+}
+
+static const struct nvbios_source
+shadow_fw = {
+        .name = "firmware",
+        .init = shadow_fw_init,
+        .fini = (void(*)(void *))release_firmware,
+        .read = shadow_fw_read,
+        .rw = false,
+};
+
+int
+nvbios_shadow(struct nouveau_bios *bios)
+{
+        struct shadow mthds[] = {
+                { shadow_class, 0, &nvbios_of },
+                { shadow_class, 0, &nvbios_ramin },
+                { shadow_class, 0, &nvbios_rom },
+                { shadow_class, 0, &nvbios_acpi_fast },
+                { shadow_class, 4, &nvbios_acpi_slow },
+                { shadow_class, 1, &nvbios_pcirom },
+                { shadow_class, 1, &nvbios_platform },
+                { shadow_class }
+        }, *mthd = mthds, *best = NULL;
+        const char *optarg;
+        char *source;
+        int optlen;
+
+        /* handle user-specified bios source */
+        optarg = nouveau_stropt(nv_device(bios)->cfgopt, "NvBios", &optlen);
+        source = optarg ? kstrndup(optarg, optlen, GFP_KERNEL) : NULL;
+        if (source) {
+                /* try to match one of the built-in methods */
+                for (mthd = mthds; mthd->func; mthd++) {
+                        if (mthd->func->name &&
+                            !strcasecmp(source, mthd->func->name)) {
+                                best = mthd;
+                                if (shadow_method(bios, mthd, NULL))
+                                        break;
+                        }
+                }
+
+                /* otherwise, attempt to load as firmware */
+                if (!best && (best = mthd)) {
+                        mthd->func = &shadow_fw;
+                        shadow_method(bios, mthd, source);
+                        mthd->func = NULL;
+                }
+
+                if (!best->score) {
+                        nv_error(bios, "%s invalid\n", source);
+                        kfree(source);
+                        source = NULL;
+                }
+        }
+
+        /* scan all potential bios sources, looking for best image */
+        if (!best || !best->score) {
+                for (mthd = mthds, best = mthd; mthd->func; mthd++) {
+                        if (!mthd->skip || best->score < mthd->skip) {
+                                if (shadow_method(bios, mthd, NULL)) {
+                                        if (mthd->score > best->score)
+                                                best = mthd;
+                                }
+                        }
+                }
+        }
+
+        /* cleanup the ones we didn't use */
+        for (mthd = mthds; mthd->func; mthd++) {
+                if (mthd != best)
+                        kfree(mthd->data);
+        }
+
+        if (!best->score) {
+                nv_fatal(bios, "unable to locate usable image\n");
+                return -EINVAL;
+        }
+
+        nv_info(bios, "using image from %s\n", best->func ?
+                best->func->name : source);
+        bios->data = best->data;
+        bios->size = best->size;
+        kfree(source);
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadowacpi.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowacpi.c
new file mode 100644
index 000000000000..bc130c12ec06
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowacpi.c
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "priv.h"
+
+#if defined(CONFIG_ACPI) && defined(CONFIG_X86)
+int nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len);
+bool nouveau_acpi_rom_supported(struct pci_dev *pdev);
+#else
+static inline bool
+nouveau_acpi_rom_supported(struct pci_dev *pdev)
+{
+        return false;
+}
+
+static inline int
+nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len)
+{
+        return -EINVAL;
+}
+#endif
+
+/* This version of the shadow function disobeys the ACPI spec and tries
+ * to fetch in units of more than 4KiB at a time.  This is a LOT faster
+ * on some systems, such as Lenovo W530.
+ */
+static u32
+acpi_read_fast(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        u32 limit = (offset + length + 0xfff) & ~0xfff;
+        u32 start = offset & ~0x00000fff;
+        u32 fetch = limit - start;
+
+        if (nvbios_extend(bios, limit) > 0) {
+                int ret = nouveau_acpi_get_bios_chunk(bios->data, start, fetch);
+                if (ret == fetch)
+                        return fetch;
+        }
+
+        return 0;
+}
+
+/* Other systems, such as the one in fdo#55948, will report a success
+ * but only return 4KiB of data.  The common bios fetching logic will
+ * detect an invalid image, and fall back to this version of the read
+ * function.
+ */
+static u32
+acpi_read_slow(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        u32 limit = (offset + length + 0xfff) & ~0xfff;
+        u32 start = offset & ~0xfff;
+        u32 fetch = 0;
+
+        if (nvbios_extend(bios, limit) > 0) {
+                while (start + fetch < limit) {
+                        int ret = nouveau_acpi_get_bios_chunk(bios->data,
+                                                              start + fetch,
+                                                              0x1000);
+                        if (ret != 0x1000)
+                                break;
+                        fetch += 0x1000;
+                }
+        }
+
+        return fetch;
+}
+
+static void *
+acpi_init(struct nouveau_bios *bios, const char *name)
+{
+        if (!nouveau_acpi_rom_supported(nv_device(bios)->pdev))
+                return ERR_PTR(-ENODEV);
+        return NULL;
+}
+
+const struct nvbios_source
+nvbios_acpi_fast = {
+        .name = "ACPI",
+        .init = acpi_init,
+        .read = acpi_read_fast,
+        .rw = false,
+};
+
+const struct nvbios_source
+nvbios_acpi_slow = {
+        .name = "ACPI",
+        .init = acpi_init,
+        .read = acpi_read_slow,
+        .rw = false,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadowof.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowof.c
new file mode 100644
index 000000000000..3abe487a6025
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowof.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "priv.h"
+
+#if defined(__powerpc__)
+struct priv {
+        const void __iomem *data;
+        int size;
+};
+
+static u32
+of_read(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        struct priv *priv = data;
+        if (offset + length <= priv->size) {
+                memcpy_fromio(bios->data + offset, priv->data + offset, length);
+                return length;
+        }
+        return 0;
+}
+
+static void *
+of_init(struct nouveau_bios *bios, const char *name)
+{
+        struct pci_dev *pdev = nv_device(bios)->pdev;
+        struct device_node *dn;
+        struct priv *priv;
+        if (!(dn = pci_device_to_OF_node(pdev)))
+                return ERR_PTR(-ENODEV);
+        if (!(priv = kzalloc(sizeof(*priv), GFP_KERNEL)))
+                return ERR_PTR(-ENOMEM);
+        if ((priv->data = of_get_property(dn, "NVDA,BMP", &priv->size)))
+                return priv;
+        kfree(priv);
+        return ERR_PTR(-EINVAL);
+}
+
+const struct nvbios_source
+nvbios_of = {
+        .name = "OpenFirmware",
+        .init = of_init,
+        .fini = (void(*)(void *))kfree,
+        .read = of_read,
+        .rw = false,
+};
+#else
+const struct nvbios_source
+nvbios_of = {
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadowpci.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowpci.c
new file mode 100644
index 000000000000..1d0389c0abef
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowpci.c
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "priv.h"
+
+struct priv {
+        struct pci_dev *pdev;
+        void __iomem *rom;
+        size_t size;
+};
+
+static u32
+pcirom_read(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        struct priv *priv = data;
+        if (offset + length <= priv->size) {
+                memcpy_fromio(bios->data + offset, priv->rom + offset, length);
+                return length;
+        }
+        return 0;
+}
+
+static void
+pcirom_fini(void *data)
+{
+        struct priv *priv = data;
+        pci_unmap_rom(priv->pdev, priv->rom);
+        pci_disable_rom(priv->pdev);
+        kfree(priv);
+}
+
+static void *
+pcirom_init(struct nouveau_bios *bios, const char *name)
+{
+        struct pci_dev *pdev = nv_device(bios)->pdev;
+        struct priv *priv = NULL;
+        int ret;
+
+        if (!(ret = pci_enable_rom(pdev))) {
+                if (ret = -ENOMEM,
+                    (priv = kmalloc(sizeof(*priv), GFP_KERNEL))) {
+                        if (ret = -EFAULT,
+                            (priv->rom = pci_map_rom(pdev, &priv->size))) {
+                                priv->pdev = pdev;
+                                return priv;
+                        }
+                        kfree(priv);
+                }
+                pci_disable_rom(pdev);
+        }
+
+        return ERR_PTR(ret);
+}
+
+const struct nvbios_source
+nvbios_pcirom = {
+        .name = "PCIROM",
+        .init = pcirom_init,
+        .fini = pcirom_fini,
+        .read = pcirom_read,
+        .rw = true,
+};
+
+static void *
+platform_init(struct nouveau_bios *bios, const char *name)
+{
+        struct pci_dev *pdev = nv_device(bios)->pdev;
+        struct priv *priv;
+        int ret = -ENOMEM;
+
+        if ((priv = kmalloc(sizeof(*priv), GFP_KERNEL))) {
+                if (ret = -ENODEV,
+                    (priv->rom = pci_platform_rom(pdev, &priv->size)))
+                        return priv;
+                kfree(priv);
+        }
+
+        return ERR_PTR(ret);
+}
+
+const struct nvbios_source
+nvbios_platform = {
+        .name = "PLATFORM",
+        .init = platform_init,
+        .fini = (void(*)(void *))kfree,
+        .read = pcirom_read,
+        .rw = true,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadowramin.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowramin.c
new file mode 100644
index 000000000000..5e58bba0dd5c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowramin.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "priv.h"
+
+struct priv {
+        struct nouveau_bios *bios;
+        u32 bar0;
+};
+
+static u32
+pramin_read(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        u32 i;
+        if (offset + length <= 0x00100000) {
+                for (i = offset; i < offset + length; i += 4)
+                        *(u32 *)&bios->data[i] = nv_rd32(bios, 0x700000 + i);
+                return length;
+        }
+        return 0;
+}
+
+static void
+pramin_fini(void *data)
+{
+        struct priv *priv = data;
+        nv_wr32(priv->bios, 0x001700, priv->bar0);
+        kfree(priv);
+}
+
+static void *
+pramin_init(struct nouveau_bios *bios, const char *name)
+{
+        struct priv *priv = NULL;
+        u64 addr = 0;
+
+        /* PRAMIN always potentially available prior to nv50 */
+        if (nv_device(bios)->card_type < NV_50)
+                return NULL;
+
+        /* we can't get the bios image pointer without PDISP */
+        if (nv_device(bios)->card_type >= GM100)
+                addr = nv_rd32(bios, 0x021c04);
+        else
+        if (nv_device(bios)->card_type >= NV_C0)
+                addr = nv_rd32(bios, 0x022500);
+        if (addr & 0x00000001) {
+                nv_debug(bios, "... display disabled\n");
+                return ERR_PTR(-ENODEV);
+        }
+
+        /* check that the window is enabled and in vram, particularly
+         * important as we don't want to be touching vram on an
+         * uninitialised board
+         */
+        addr = nv_rd32(bios, 0x619f04);
+        if (!(addr & 0x00000008)) {
+                nv_debug(bios, "... not enabled\n");
+                return ERR_PTR(-ENODEV);
+        }
+        if ( (addr & 0x00000003) != 1) {
+                nv_debug(bios, "... not in vram\n");
+                return ERR_PTR(-ENODEV);
+        }
+
+        /* some alternate method inherited from xf86-video-nv... */
+        addr = (addr & 0xffffff00) << 8;
+        if (!addr) {
+                addr  = (u64)nv_rd32(bios, 0x001700) << 16;
+                addr += 0xf0000;
+        }
+
+        /* modify bar0 PRAMIN window to cover the bios image */
+        if (!(priv = kmalloc(sizeof(*priv), GFP_KERNEL))) {
+                nv_error(bios, "... out of memory\n");
+                return ERR_PTR(-ENOMEM);
+        }
+
+        priv->bios = bios;
+        priv->bar0 = nv_rd32(bios, 0x001700);
+        nv_wr32(bios, 0x001700, addr >> 16);
+        return priv;
+}
+
+const struct nvbios_source
+nvbios_ramin = {
+        .name = "PRAMIN",
+        .init = pramin_init,
+        .fini = pramin_fini,
+        .read = pramin_read,
+        .rw = true,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadowrom.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowrom.c
new file mode 100644
index 000000000000..b7992bc3ffa5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowrom.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "priv.h"
+
+static u32
+prom_read(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        u32 i;
+        if (offset + length <= 0x00100000) {
+                for (i = offset; i < offset + length; i += 4)
+                        *(u32 *)&bios->data[i] = nv_rd32(bios, 0x300000 + i);
+                return length;
+        }
+        return 0;
+}
+
+static void
+prom_fini(void *data)
+{
+        struct nouveau_bios *bios = data;
+        if (nv_device(bios)->card_type < NV_50)
+                nv_mask(bios, 0x001850, 0x00000001, 0x00000001);
+        else
+                nv_mask(bios, 0x088050, 0x00000001, 0x00000001);
+}
+
+static void *
+prom_init(struct nouveau_bios *bios, const char *name)
+{
+        if (nv_device(bios)->card_type < NV_50) {
+                if (nv_device(bios)->card_type == NV_40 &&
+                    nv_device(bios)->chipset >= 0x4c)
+                        return ERR_PTR(-ENODEV);
+                nv_mask(bios, 0x001850, 0x00000001, 0x00000000);
+        } else {
+                nv_mask(bios, 0x088050, 0x00000001, 0x00000000);
+        }
+        return bios;
+}
+
+const struct nvbios_source
+nvbios_rom = {
+        .name = "PROM",
+        .init = prom_init,
+        .fini = prom_fini,
+        .read = prom_read,
+        .rw = false,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/timing.c b/drivers/gpu/drm/nouveau/core/subdev/bios/timing.c
index 46d955eb51eb..8521eca1ed9c 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/timing.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/timing.c
@@ -93,10 +93,44 @@ nvbios_timingEp(struct nouveau_bios *bios, int idx,
         p->timing_hdr = *hdr;
         switch (!!data * *ver) {
         case 0x10:
-                p->timing_10_WR = nv_ro08(bios, data + 0x00);
-                p->timing_10_CL = nv_ro08(bios, data + 0x02);
-                p->timing_10_ODT = nv_ro08(bios, data + 0x0e) & 0x07;
-                p->timing_10_CWL = nv_ro08(bios, data + 0x13);
+                p->timing_10_WR    = nv_ro08(bios, data + 0x00);
+                p->timing_10_WTR   = nv_ro08(bios, data + 0x01);
+                p->timing_10_CL    = nv_ro08(bios, data + 0x02);
+                p->timing_10_RC    = nv_ro08(bios, data + 0x03);
+                p->timing_10_RFC   = nv_ro08(bios, data + 0x05);
+                p->timing_10_RAS   = nv_ro08(bios, data + 0x07);
+                p->timing_10_RP    = nv_ro08(bios, data + 0x09);
+                p->timing_10_RCDRD = nv_ro08(bios, data + 0x0a);
+                p->timing_10_RCDWR = nv_ro08(bios, data + 0x0b);
+                p->timing_10_RRD   = nv_ro08(bios, data + 0x0c);
+                p->timing_10_13    = nv_ro08(bios, data + 0x0d);
+                p->timing_10_ODT   = nv_ro08(bios, data + 0x0e) & 0x07;
+
+                p->timing_10_24  = 0xff;
+                p->timing_10_21  = 0;
+                p->timing_10_20  = 0;
+                p->timing_10_CWL = 0;
+                p->timing_10_18  = 0;
+                p->timing_10_16  = 0;
+
+                switch (min_t(u8, *hdr, 25)) {
+                case 25:
+                        p->timing_10_24  = nv_ro08(bios, data + 0x18);
+                case 24:
+                case 23:
+                case 22:
+                        p->timing_10_21  = nv_ro08(bios, data + 0x15);
+                case 21:
+                        p->timing_10_20  = nv_ro08(bios, data + 0x14);
+                case 20:
+                        p->timing_10_CWL = nv_ro08(bios, data + 0x13);
+                case 19:
+                        p->timing_10_18  = nv_ro08(bios, data + 0x12);
+                case 18:
+                case 17:
+                        p->timing_10_16  = nv_ro08(bios, data + 0x10);
+                }
+
                 break;
         case 0x20:
                 p->timing[0] = nv_ro32(bios, data + 0x00);
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/gk20a.c b/drivers/gpu/drm/nouveau/core/subdev/clock/gk20a.c
index 425a8d5e9129..fb4fad374bdd 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/gk20a.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/gk20a.c
@@ -109,7 +109,7 @@ struct gk20a_clk_pllg_params {
 };
 
 static const struct gk20a_clk_pllg_params gk20a_pllg_params = {
-        .min_vco = 1000, .max_vco = 1700,
+        .min_vco = 1000, .max_vco = 2064,
         .min_u = 12, .max_u = 38,
         .min_m = 1, .max_m = 255,
         .min_n = 8, .max_n = 255,
@@ -470,76 +470,91 @@ gk20a_pstates[] = {
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 72000,
+                        .voltage = 0,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 108000,
+                        .voltage = 1,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 180000,
+                        .voltage = 2,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 252000,
+                        .voltage = 3,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 324000,
+                        .voltage = 4,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 396000,
+                        .voltage = 5,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 468000,
+                        .voltage = 6,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 540000,
+                        .voltage = 7,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 612000,
+                        .voltage = 8,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 648000,
+                        .voltage = 9,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 684000,
+                        .voltage = 10,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 708000,
+                        .voltage = 11,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 756000,
+                        .voltage = 12,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 804000,
+                        .voltage = 13,
                 },
         },
         {
                 .base = {
                         .domain[nv_clk_src_gpc] = 852000,
+                        .voltage = 14,
                 },
         },
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
index 094551d8ad9b..07ad01247675 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
@@ -510,7 +510,7 @@ nva3_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         int ret;
 
         ret = nouveau_clock_create(parent, engine, oclass, nva3_domain, NULL, 0,
-                                   false, &priv);
+                                   true, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c
index 239acfe876c3..0e45cee82463 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c
@@ -24,8 +24,6 @@
 
 #include <core/option.h>
 
-#include <subdev/bios.h>
-#include <subdev/bios/init.h>
 #include <subdev/vga.h>
 
 #include "priv.h"
@@ -56,7 +54,7 @@ _nouveau_devinit_init(struct nouveau_object *object)
         if (ret)
                 return ret;
 
-        ret = nvbios_init(&devinit->base, devinit->post);
+        ret = impl->post(&devinit->base, devinit->post);
         if (ret)
                 return ret;
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/gm107.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/gm107.c
index c69bc7f54e37..4ba43d6a1ec8 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/gm107.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/gm107.c
@@ -24,7 +24,7 @@
 
 #include "nv50.h"
 
-static u64
+u64
 gm107_devinit_disable(struct nouveau_devinit *devinit)
 {
         struct nv50_devinit_priv *priv = (void *)devinit;
@@ -53,4 +53,5 @@ gm107_devinit_oclass = &(struct nouveau_devinit_impl) {
         },
         .pll_set = nvc0_devinit_pll_set,
         .disable = gm107_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/gm204.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/gm204.c
new file mode 100644
index 000000000000..e44a86662a2a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/gm204.c
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/pmu.h>
+
+#include "nv50.h"
+
+static void
+pmu_code(struct nv50_devinit_priv *priv, u32 pmu, u32 img, u32 len, bool sec)
+{
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        int i;
+
+        nv_wr32(priv, 0x10a180, 0x01000000 | (sec ? 0x10000000 : 0) | pmu);
+        for (i = 0; i < len; i += 4) {
+                if ((i & 0xff) == 0)
+                        nv_wr32(priv, 0x10a188, (pmu + i) >> 8);
+                nv_wr32(priv, 0x10a184, nv_ro32(bios, img + i));
+        }
+
+        while (i & 0xff) {
+                nv_wr32(priv, 0x10a184, 0x00000000);
+                i += 4;
+        }
+}
+
+static void
+pmu_data(struct nv50_devinit_priv *priv, u32 pmu, u32 img, u32 len)
+{
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        int i;
+
+        nv_wr32(priv, 0x10a1c0, 0x01000000 | pmu);
+        for (i = 0; i < len; i += 4)
+                nv_wr32(priv, 0x10a1c4, nv_ro32(bios, img + i));
+}
+
+static u32
+pmu_args(struct nv50_devinit_priv *priv, u32 argp, u32 argi)
+{
+        nv_wr32(priv, 0x10a1c0, argp);
+        nv_wr32(priv, 0x10a1c0, nv_rd32(priv, 0x10a1c4) + argi);
+        return nv_rd32(priv, 0x10a1c4);
+}
+
+static void
+pmu_exec(struct nv50_devinit_priv *priv, u32 init_addr)
+{
+        nv_wr32(priv, 0x10a104, init_addr);
+        nv_wr32(priv, 0x10a10c, 0x00000000);
+        nv_wr32(priv, 0x10a100, 0x00000002);
+}
+
+static int
+pmu_load(struct nv50_devinit_priv *priv, u8 type, bool post,
+         u32 *init_addr_pmu, u32 *args_addr_pmu)
+{
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        struct nvbios_pmuR pmu;
+
+        if (!nvbios_pmuRm(bios, type, &pmu)) {
+                nv_error(priv, "VBIOS PMU fuc %02x not found\n", type);
+                return -EINVAL;
+        }
+
+        if (!post)
+                return 0;
+
+        pmu_code(priv, pmu.boot_addr_pmu, pmu.boot_addr, pmu.boot_size, false);
+        pmu_code(priv, pmu.code_addr_pmu, pmu.code_addr, pmu.code_size, true);
+        pmu_data(priv, pmu.data_addr_pmu, pmu.data_addr, pmu.data_size);
+
+        if (init_addr_pmu) {
+                *init_addr_pmu = pmu.init_addr_pmu;
+                *args_addr_pmu = pmu.args_addr_pmu;
+                return 0;
+        }
+
+        return pmu_exec(priv, pmu.init_addr_pmu), 0;
+}
+
+static int
+gm204_devinit_post(struct nouveau_subdev *subdev, bool post)
+{
+        struct nv50_devinit_priv *priv = (void *)nouveau_devinit(subdev);
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        struct bit_entry bit_I;
+        u32 init, args;
+        int ret;
+
+        if (bit_entry(bios, 'I', &bit_I) || bit_I.version != 1 ||
+                                            bit_I.length < 0x1c) {
+                nv_error(priv, "VBIOS PMU init data not found\n");
+                return -EINVAL;
+        }
+
+        /* reset PMU and load init table parser ucode */
+        if (post) {
+                nv_mask(priv, 0x000200, 0x00002000, 0x00000000);
+                nv_mask(priv, 0x000200, 0x00002000, 0x00002000);
+                nv_rd32(priv, 0x000200);
+                while (nv_rd32(priv, 0x10a10c) & 0x00000006) {
+                }
+        }
+
+        ret = pmu_load(priv, 0x04, post, &init, &args);
+        if (ret)
+                return ret;
+
+        /* upload first chunk of init data */
+        if (post) {
+                u32 pmu = pmu_args(priv, args + 0x08, 0x08);
+                u32 img = nv_ro16(bios, bit_I.offset + 0x14);
+                u32 len = nv_ro16(bios, bit_I.offset + 0x16);
+                pmu_data(priv, pmu, img, len);
+        }
+
+        /* upload second chunk of init data */
+        if (post) {
+                u32 pmu = pmu_args(priv, args + 0x08, 0x10);
+                u32 img = nv_ro16(bios, bit_I.offset + 0x18);
+                u32 len = nv_ro16(bios, bit_I.offset + 0x1a);
+                pmu_data(priv, pmu, img, len);
+        }
+
+        /* execute init tables */
+        if (post) {
+                nv_wr32(priv, 0x10a040, 0x00005000);
+                pmu_exec(priv, init);
+                while (!(nv_rd32(priv, 0x10a040) & 0x00002000)) {
+                }
+        }
+
+        /* load and execute some other ucode image (bios therm?) */
+        return pmu_load(priv, 0x01, post, NULL, NULL);
+}
+
+struct nouveau_oclass *
+gm204_devinit_oclass = &(struct nouveau_devinit_impl) {
+        .base.handle = NV_SUBDEV(DEVINIT, 0x07),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv50_devinit_ctor,
+                .dtor = _nouveau_devinit_dtor,
+                .init = nv50_devinit_init,
+                .fini = _nouveau_devinit_fini,
+        },
+        .pll_set = nvc0_devinit_pll_set,
+        .disable = gm107_devinit_disable,
+        .post = gm204_devinit_post,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
index 052ad690b468..65651c50f6ea 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
@@ -464,4 +464,5 @@ nv04_devinit_oclass = &(struct nouveau_devinit_impl) {
         },
         .meminit = nv04_devinit_meminit,
         .pll_set = nv04_devinit_pll_set,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c
index 4a19c10e5178..a2007a3efc4d 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c
@@ -136,4 +136,5 @@ nv05_devinit_oclass = &(struct nouveau_devinit_impl) {
         },
         .meminit = nv05_devinit_meminit,
         .pll_set = nv04_devinit_pll_set,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
index 3b8d657da279..178b46f79b50 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
@@ -107,4 +107,5 @@ nv10_devinit_oclass = &(struct nouveau_devinit_impl) {
         },
         .meminit = nv10_devinit_meminit,
         .pll_set = nv04_devinit_pll_set,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv1a.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv1a.c
index 526d0c6faacd..995dd97af3e9 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv1a.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv1a.c
@@ -34,4 +34,5 @@ nv1a_devinit_oclass = &(struct nouveau_devinit_impl) {
                 .fini = nv04_devinit_fini,
         },
         .pll_set = nv04_devinit_pll_set,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c
index 04bc9732644c..915089fb46f7 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c
@@ -71,4 +71,5 @@ nv20_devinit_oclass = &(struct nouveau_devinit_impl) {
         },
         .meminit = nv20_devinit_meminit,
         .pll_set = nv04_devinit_pll_set,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.c
index b46c62a1d5d8..968334d1dca4 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.c
@@ -26,6 +26,7 @@
 #include <subdev/bios/dcb.h>
 #include <subdev/bios/disp.h>
 #include <subdev/bios/init.h>
+#include <subdev/ibus.h>
 #include <subdev/vga.h>
 
 #include "nv50.h"
@@ -91,6 +92,7 @@ int
 nv50_devinit_init(struct nouveau_object *object)
 {
         struct nouveau_bios *bios = nouveau_bios(object);
+        struct nouveau_ibus *ibus = nouveau_ibus(object);
         struct nv50_devinit_priv *priv = (void *)object;
         struct nvbios_outp info;
         struct dcb_output outp;
@@ -105,6 +107,13 @@ nv50_devinit_init(struct nouveau_object *object)
                 }
         }
 
+        /* some boards appear to require certain priv register timeouts
+         * to be bumped before runing devinit scripts.  not a clue why
+         * the vbios engineers didn't make the scripts just work...
+         */
+        if (priv->base.post && ibus)
+                nv_ofuncs(ibus)->init(nv_object(ibus));
+
         ret = nouveau_devinit_init(&priv->base);
         if (ret)
                 return ret;
@@ -160,4 +169,5 @@ nv50_devinit_oclass = &(struct nouveau_devinit_impl) {
         },
         .pll_set = nv50_devinit_pll_set,
         .disable = nv50_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h
index 51d5076333ec..f412bb7f780e 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h
@@ -18,4 +18,6 @@ int  nva3_devinit_pll_set(struct nouveau_devinit *, u32, u32);
 
 int  nvc0_devinit_pll_set(struct nouveau_devinit *, u32, u32);
 
+u64  gm107_devinit_disable(struct nouveau_devinit *);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv84.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv84.c
index 787422505d87..a7c80ded77cd 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv84.c
@@ -60,4 +60,5 @@ nv84_devinit_oclass = &(struct nouveau_devinit_impl) {
         },
         .pll_set = nv50_devinit_pll_set,
         .disable = nv84_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv98.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv98.c
index 2b0e963fc6f0..a773253a17f6 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv98.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv98.c
@@ -59,4 +59,5 @@ nv98_devinit_oclass = &(struct nouveau_devinit_impl) {
         },
         .pll_set = nv50_devinit_pll_set,
         .disable = nv98_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c
index 006cf348bda7..b9cd9e53f760 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c
@@ -142,4 +142,5 @@ nva3_devinit_oclass = &(struct nouveau_devinit_impl) {
         .pll_set = nva3_devinit_pll_set,
         .disable = nva3_devinit_disable,
         .mmio    = nva3_devinit_mmio,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nvaf.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nvaf.c
index 4fc68d27eff3..3729846a8e5c 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nvaf.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nvaf.c
@@ -60,4 +60,5 @@ nvaf_devinit_oclass = &(struct nouveau_devinit_impl) {
         },
         .pll_set = nva3_devinit_pll_set,
         .disable = nvaf_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c
index 30c765747eea..80bd7f5eda3d 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c
@@ -115,4 +115,5 @@ nvc0_devinit_oclass = &(struct nouveau_devinit_impl) {
         },
         .pll_set = nvc0_devinit_pll_set,
         .disable = nvc0_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h b/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h
index f0e8683ad840..cbcd51852472 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h
@@ -3,6 +3,7 @@
 
 #include <subdev/bios.h>
 #include <subdev/bios/pll.h>
+#include <subdev/bios/init.h>
 #include <subdev/clock/pll.h>
 #include <subdev/devinit.h>
 
@@ -12,6 +13,7 @@ struct nouveau_devinit_impl {
         int  (*pll_set)(struct nouveau_devinit *, u32 type, u32 freq);
         u64  (*disable)(struct nouveau_devinit *);
         u32  (*mmio)(struct nouveau_devinit *, u32);
+        int  (*post)(struct nouveau_subdev *, bool);
 };
 
 #define nouveau_devinit_create(p,e,o,d)                                        \
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/base.c b/drivers/gpu/drm/nouveau/core/subdev/fb/base.c
index f009d8a39d9d..c866148c440f 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/base.c
@@ -23,37 +23,30 @@
  */
 
 #include <subdev/bios.h>
-#include <subdev/bios/bit.h>
+#include <subdev/bios/M0203.h>
 
 #include "priv.h"
 
 int
 nouveau_fb_bios_memtype(struct nouveau_bios *bios)
 {
-        struct bit_entry M;
-        u8 ramcfg;
-
-        ramcfg = (nv_rd32(bios, 0x101000) & 0x0000003c) >> 2;
-        if (!bit_entry(bios, 'M', &M) && M.version == 2 && M.length >= 5) {
-                u16 table   = nv_ro16(bios, M.offset + 3);
-                u8  version = nv_ro08(bios, table + 0);
-                u8  header  = nv_ro08(bios, table + 1);
-                u8  record  = nv_ro08(bios, table + 2);
-                u8  entries = nv_ro08(bios, table + 3);
-                if (table && version == 0x10 && ramcfg < entries) {
-                        u16 entry = table + header + (ramcfg * record);
-                        switch (nv_ro08(bios, entry) & 0x0f) {
-                        case 0: return NV_MEM_TYPE_DDR2;
-                        case 1: return NV_MEM_TYPE_DDR3;
-                        case 2: return NV_MEM_TYPE_GDDR3;
-                        case 3: return NV_MEM_TYPE_GDDR5;
-                        default:
-                                break;
-                        }
-
+        const u8 ramcfg = (nv_rd32(bios, 0x101000) & 0x0000003c) >> 2;
+        struct nvbios_M0203E M0203E;
+        u8 ver, hdr;
+
+        if (nvbios_M0203Em(bios, ramcfg, &ver, &hdr, &M0203E)) {
+                switch (M0203E.type) {
+                case M0203E_TYPE_DDR2 : return NV_MEM_TYPE_DDR2;
+                case M0203E_TYPE_DDR3 : return NV_MEM_TYPE_DDR3;
+                case M0203E_TYPE_GDDR3: return NV_MEM_TYPE_GDDR3;
+                case M0203E_TYPE_GDDR5: return NV_MEM_TYPE_GDDR5;
+                default:
+                        nv_warn(bios, "M0203E type %02x\n", M0203E.type);
+                        return NV_MEM_TYPE_UNKNOWN;
                 }
         }
 
+        nv_warn(bios, "M0203E not matched!\n");
         return NV_MEM_TYPE_UNKNOWN;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/gddr3.c b/drivers/gpu/drm/nouveau/core/subdev/fb/gddr3.c
new file mode 100644
index 000000000000..d85a25d027ee
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/gddr3.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ *          Roy Spliet <rspliet@eclipso.eu>
+ */
+
+#include <subdev/bios.h>
+#include "priv.h"
+
+struct ramxlat {
+        int id;
+        u8 enc;
+};
+
+static inline int
+ramxlat(const struct ramxlat *xlat, int id)
+{
+        while (xlat->id >= 0) {
+                if (xlat->id == id)
+                        return xlat->enc;
+                xlat++;
+        }
+        return -EINVAL;
+}
+
+static const struct ramxlat
+ramgddr3_cl_lo[] = {
+        { 7, 7 }, { 8, 0 }, { 9, 1 }, { 10, 2 }, { 11, 3 },
+        /* the below are mentioned in some, but not all, gddr3 docs */
+        { 12, 4 }, { 13, 5 }, { 14, 6 },
+        /* XXX: Per Samsung docs, are these used? They overlap with Qimonda */
+        /* { 4, 4 }, { 5, 5 }, { 6, 6 }, { 12, 8 }, { 13, 9 }, { 14, 10 },
+         * { 15, 11 }, */
+        { -1 }
+};
+
+static const struct ramxlat
+ramgddr3_cl_hi[] = {
+        { 10, 2 }, { 11, 3 }, { 12, 4 }, { 13, 5 }, { 14, 6 }, { 15, 7 },
+        { 16, 0 }, { 17, 1 },
+        { -1 }
+};
+
+static const struct ramxlat
+ramgddr3_wr_lo[] = {
+        { 5, 2 }, { 7, 4 }, { 8, 5 }, { 9, 6 }, { 10, 7 },
+        { 11, 0 },
+        /* the below are mentioned in some, but not all, gddr3 docs */
+        { 4, 1 }, { 6, 3 }, { 12, 1 }, { 13 , 2 },
+        { -1 }
+};
+
+int
+nouveau_gddr3_calc(struct nouveau_ram *ram)
+{
+        int CL, WR, CWL, DLL = 0, ODT = 0, hi;
+
+        switch (ram->next->bios.timing_ver) {
+        case 0x10:
+                CWL = ram->next->bios.timing_10_CWL;
+                CL  = ram->next->bios.timing_10_CL;
+                WR  = ram->next->bios.timing_10_WR;
+                DLL = !ram->next->bios.ramcfg_10_DLLoff;
+                ODT = ram->next->bios.timing_10_ODT;
+                break;
+        case 0x20:
+                CWL = (ram->next->bios.timing[1] & 0x00000f80) >> 7;
+                CL  = (ram->next->bios.timing[1] & 0x0000001f) >> 0;
+                WR  = (ram->next->bios.timing[2] & 0x007f0000) >> 16;
+                /* XXX: Get these values from the VBIOS instead */
+                DLL = !(ram->mr[1] & 0x1);
+                ODT =  (ram->mr[1] & 0x004) >> 2 |
+                       (ram->mr[1] & 0x040) >> 5 |
+                       (ram->mr[1] & 0x200) >> 7;
+                break;
+        default:
+                return -ENOSYS;
+        }
+
+        hi = ram->mr[2] & 0x1;
+        CL  = ramxlat(hi ? ramgddr3_cl_hi : ramgddr3_cl_lo, CL);
+        WR  = ramxlat(ramgddr3_wr_lo, WR);
+        if (CL < 0 || CWL < 1 || CWL > 7 || WR < 0)
+                return -EINVAL;
+
+        ram->mr[0] &= ~0xf74;
+        ram->mr[0] |= (CWL & 0x07) << 9;
+        ram->mr[0] |= (CL & 0x07) << 4;
+        ram->mr[0] |= (CL & 0x08) >> 1;
+
+        ram->mr[1] &= ~0x3fc;
+        ram->mr[1] |= (ODT & 0x03) << 2;
+        ram->mr[1] |= (ODT & 0x03) << 8;
+        ram->mr[1] |= (WR  & 0x03) << 4;
+        ram->mr[1] |= (WR  & 0x04) << 5;
+        ram->mr[1] |= !DLL << 6;
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
index 60322e906dd4..283863f7aa9b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
@@ -37,6 +37,7 @@ extern struct nouveau_oclass gm107_ram_oclass;
 
 int nouveau_sddr2_calc(struct nouveau_ram *ram);
 int nouveau_sddr3_calc(struct nouveau_ram *ram);
+int nouveau_gddr3_calc(struct nouveau_ram *ram);
 int nouveau_gddr5_calc(struct nouveau_ram *ram, bool nuts);
 
 #define nouveau_fb_create(p,e,c,d)                                             \
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramfuc.h b/drivers/gpu/drm/nouveau/core/subdev/fb/ramfuc.h
index d1fbbe4b00a2..0ac7256443bb 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramfuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramfuc.h
@@ -141,6 +141,20 @@ ramfuc_wait_vblank(struct ramfuc *ram)
 }
 
 static inline void
+ramfuc_train(struct ramfuc *ram)
+{
+        nouveau_memx_train(ram->memx);
+}
+
+static inline int
+ramfuc_train_result(struct nouveau_fb *pfb, u32 *result, u32 rsize)
+{
+        struct nouveau_pwr *ppwr = nouveau_pwr(pfb);
+
+        return nouveau_memx_train_result(ppwr, result, rsize);
+}
+
+static inline void
 ramfuc_block(struct ramfuc *ram)
 {
         nouveau_memx_block(ram->memx);
@@ -162,6 +176,8 @@ ramfuc_unblock(struct ramfuc *ram)
 #define ram_wait(s,r,m,d,n)  ramfuc_wait(&(s)->base, (r), (m), (d), (n))
 #define ram_nsec(s,n)        ramfuc_nsec(&(s)->base, (n))
 #define ram_wait_vblank(s)   ramfuc_wait_vblank(&(s)->base)
+#define ram_train(s)         ramfuc_train(&(s)->base)
+#define ram_train_result(s,r,l) ramfuc_train_result((s), (r), (l))
 #define ram_block(s)         ramfuc_block(&(s)->base)
 #define ram_unblock(s)       ramfuc_unblock(&(s)->base)
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c
index 3601deca0bd5..3b38a538845d 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c
@@ -20,86 +20,512 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  * Authors: Ben Skeggs
+ *          Roy Spliet <rspliet@eclipso.eu>
  */
 
 #include <subdev/bios.h>
 #include <subdev/bios/bit.h>
 #include <subdev/bios/pll.h>
 #include <subdev/bios/rammap.h>
+#include <subdev/bios/M0205.h>
 #include <subdev/bios/timing.h>
 
 #include <subdev/clock/nva3.h>
 #include <subdev/clock/pll.h>
 
+#include <subdev/gpio.h>
+
+#include <subdev/timer.h>
+
+#include <engine/fifo.h>
+
 #include <core/option.h>
 
 #include "ramfuc.h"
 
 #include "nv50.h"
 
+/* XXX: Remove when memx gains GPIO support */
+extern int nv50_gpio_location(int line, u32 *reg, u32 *shift);
+
 struct nva3_ramfuc {
         struct ramfuc base;
+        struct ramfuc_reg r_0x001610;
+        struct ramfuc_reg r_0x001700;
+        struct ramfuc_reg r_0x002504;
         struct ramfuc_reg r_0x004000;
         struct ramfuc_reg r_0x004004;
         struct ramfuc_reg r_0x004018;
         struct ramfuc_reg r_0x004128;
         struct ramfuc_reg r_0x004168;
+        struct ramfuc_reg r_0x100080;
         struct ramfuc_reg r_0x100200;
         struct ramfuc_reg r_0x100210;
         struct ramfuc_reg r_0x100220[9];
+        struct ramfuc_reg r_0x100264;
         struct ramfuc_reg r_0x1002d0;
         struct ramfuc_reg r_0x1002d4;
         struct ramfuc_reg r_0x1002dc;
         struct ramfuc_reg r_0x10053c;
         struct ramfuc_reg r_0x1005a0;
         struct ramfuc_reg r_0x1005a4;
+        struct ramfuc_reg r_0x100700;
         struct ramfuc_reg r_0x100714;
         struct ramfuc_reg r_0x100718;
         struct ramfuc_reg r_0x10071c;
+        struct ramfuc_reg r_0x100720;
         struct ramfuc_reg r_0x100760;
         struct ramfuc_reg r_0x1007a0;
         struct ramfuc_reg r_0x1007e0;
+        struct ramfuc_reg r_0x100da0;
         struct ramfuc_reg r_0x10f804;
         struct ramfuc_reg r_0x1110e0;
         struct ramfuc_reg r_0x111100;
         struct ramfuc_reg r_0x111104;
+        struct ramfuc_reg r_0x1111e0;
+        struct ramfuc_reg r_0x111400;
         struct ramfuc_reg r_0x611200;
         struct ramfuc_reg r_mr[4];
+        struct ramfuc_reg r_gpioFBVREF;
+};
+
+struct nva3_ltrain {
+        enum {
+                NVA3_TRAIN_UNKNOWN,
+                NVA3_TRAIN_UNSUPPORTED,
+                NVA3_TRAIN_ONCE,
+                NVA3_TRAIN_EXEC,
+                NVA3_TRAIN_DONE
+        } state;
+        u32 r_100720;
+        u32 r_1111e0;
+        u32 r_111400;
+        struct nouveau_mem *mem;
 };
 
 struct nva3_ram {
         struct nouveau_ram base;
         struct nva3_ramfuc fuc;
+        struct nva3_ltrain ltrain;
 };
 
+void
+nva3_link_train_calc(u32 *vals, struct nva3_ltrain *train)
+{
+        int i, lo, hi;
+        u8 median[8], bins[4] = {0, 0, 0, 0}, bin = 0, qty = 0;
+
+        for (i = 0; i < 8; i++) {
+                for (lo = 0; lo < 0x40; lo++) {
+                        if (!(vals[lo] & 0x80000000))
+                                continue;
+                        if (vals[lo] & (0x101 << i))
+                                break;
+                }
+
+                if (lo == 0x40)
+                        return;
+
+                for (hi = lo + 1; hi < 0x40; hi++) {
+                        if (!(vals[lo] & 0x80000000))
+                                continue;
+                        if (!(vals[hi] & (0x101 << i))) {
+                                hi--;
+                                break;
+                        }
+                }
+
+                median[i] = ((hi - lo) >> 1) + lo;
+                bins[(median[i] & 0xf0) >> 4]++;
+                median[i] += 0x30;
+        }
+
+        /* Find the best value for 0x1111e0 */
+        for (i = 0; i < 4; i++) {
+                if (bins[i] > qty) {
+                        bin = i + 3;
+                        qty = bins[i];
+                }
+        }
+
+        train->r_100720 = 0;
+        for (i = 0; i < 8; i++) {
+                median[i] = max(median[i], (u8) (bin << 4));
+                median[i] = min(median[i], (u8) ((bin << 4) | 0xf));
+
+                train->r_100720 |= ((median[i] & 0x0f) << (i << 2));
+        }
+
+        train->r_1111e0 = 0x02000000 | (bin * 0x101);
+        train->r_111400 = 0x0;
+}
+
+/*
+ * Link training for (at least) DDR3
+ */
+int
+nva3_link_train(struct nouveau_fb *pfb)
+{
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nva3_ram *ram = (void *)pfb->ram;
+        struct nouveau_clock *clk = nouveau_clock(pfb);
+        struct nva3_ltrain *train = &ram->ltrain;
+        struct nouveau_device *device = nv_device(pfb);
+        struct nva3_ramfuc *fuc = &ram->fuc;
+        u32 *result, r1700;
+        int ret, i;
+        struct nvbios_M0205T M0205T = { 0 };
+        u8 ver, hdr, cnt, len, snr, ssz;
+        unsigned int clk_current;
+        unsigned long flags;
+        unsigned long *f = &flags;
+
+        if (nouveau_boolopt(device->cfgopt, "NvMemExec", true) != true)
+                return -ENOSYS;
+
+        /* XXX: Multiple partitions? */
+        result = kmalloc(64 * sizeof(u32), GFP_KERNEL);
+        if (!result)
+                return -ENOMEM;
+
+        train->state = NVA3_TRAIN_EXEC;
+
+        /* Clock speeds for training and back */
+        nvbios_M0205Tp(bios, &ver, &hdr, &cnt, &len, &snr, &ssz, &M0205T);
+        if (M0205T.freq == 0)
+                return -ENOENT;
+
+        clk_current = clk->read(clk, nv_clk_src_mem);
+
+        ret = nva3_clock_pre(clk, f);
+        if (ret)
+                goto out;
+
+        /* First: clock up/down */
+        ret = ram->base.calc(pfb, (u32) M0205T.freq * 1000);
+        if (ret)
+                goto out;
+
+        /* Do this *after* calc, eliminates write in script */
+        nv_wr32(pfb, 0x111400, 0x00000000);
+        /* XXX: Magic writes that improve train reliability? */
+        nv_mask(pfb, 0x100674, 0x0000ffff, 0x00000000);
+        nv_mask(pfb, 0x1005e4, 0x0000ffff, 0x00000000);
+        nv_mask(pfb, 0x100b0c, 0x000000ff, 0x00000000);
+        nv_wr32(pfb, 0x100c04, 0x00000400);
+
+        /* Now the training script */
+        r1700 = ram_rd32(fuc, 0x001700);
+
+        ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
+        ram_wr32(fuc, 0x611200, 0x3300);
+        ram_wait_vblank(fuc);
+        ram_wait(fuc, 0x611200, 0x00000003, 0x00000000, 500000);
+        ram_mask(fuc, 0x001610, 0x00000083, 0x00000003);
+        ram_mask(fuc, 0x100080, 0x00000020, 0x00000000);
+        ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);
+        ram_wr32(fuc, 0x001700, 0x00000000);
+
+        ram_train(fuc);
+
+        /* Reset */
+        ram_mask(fuc, 0x10f804, 0x80000000, 0x80000000);
+        ram_wr32(fuc, 0x10053c, 0x0);
+        ram_wr32(fuc, 0x100720, train->r_100720);
+        ram_wr32(fuc, 0x1111e0, train->r_1111e0);
+        ram_wr32(fuc, 0x111400, train->r_111400);
+        ram_nuke(fuc, 0x100080);
+        ram_mask(fuc, 0x100080, 0x00000020, 0x00000020);
+        ram_nsec(fuc, 1000);
+
+        ram_wr32(fuc, 0x001700, r1700);
+        ram_mask(fuc, 0x001610, 0x00000083, 0x00000080);
+        ram_wr32(fuc, 0x611200, 0x3330);
+        ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);
+
+        ram_exec(fuc, true);
+
+        ram->base.calc(pfb, clk_current);
+        ram_exec(fuc, true);
+
+        /* Post-processing, avoids flicker */
+        nv_mask(pfb, 0x616308, 0x10, 0x10);
+        nv_mask(pfb, 0x616b08, 0x10, 0x10);
+
+        nva3_clock_post(clk, f);
+
+        ram_train_result(pfb, result, 64);
+        for (i = 0; i < 64; i++)
+                nv_debug(pfb, "Train: %08x", result[i]);
+        nva3_link_train_calc(result, train);
+
+        nv_debug(pfb, "Train: %08x %08x %08x", train->r_100720,
+                        train->r_1111e0, train->r_111400);
+
+        kfree(result);
+
+        train->state = NVA3_TRAIN_DONE;
+
+        return ret;
+
+out:
+        if(ret == -EBUSY)
+                f = NULL;
+
+        train->state = NVA3_TRAIN_UNSUPPORTED;
+
+        nva3_clock_post(clk, f);
+        return ret;
+}
+
+int
+nva3_link_train_init(struct nouveau_fb *pfb)
+{
+        static const u32 pattern[16] = {
+                0xaaaaaaaa, 0xcccccccc, 0xdddddddd, 0xeeeeeeee,
+                0x00000000, 0x11111111, 0x44444444, 0xdddddddd,
+                0x33333333, 0x55555555, 0x77777777, 0x66666666,
+                0x99999999, 0x88888888, 0xeeeeeeee, 0xbbbbbbbb,
+        };
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nva3_ram *ram = (void *)pfb->ram;
+        struct nva3_ltrain *train = &ram->ltrain;
+        struct nouveau_mem *mem;
+        struct nvbios_M0205E M0205E;
+        u8 ver, hdr, cnt, len;
+        u32 r001700;
+        int ret, i = 0;
+
+        train->state = NVA3_TRAIN_UNSUPPORTED;
+
+        /* We support type "5"
+         * XXX: training pattern table appears to be unused for this routine */
+        if (!nvbios_M0205Ep(bios, i, &ver, &hdr, &cnt, &len, &M0205E))
+                return -ENOENT;
+
+        if (M0205E.type != 5)
+                return 0;
+
+        train->state = NVA3_TRAIN_ONCE;
+
+        ret = pfb->ram->get(pfb, 0x8000, 0x10000, 0, 0x800, &ram->ltrain.mem);
+        if (ret)
+                return ret;
+
+        mem = ram->ltrain.mem;
+
+        nv_wr32(pfb, 0x100538, 0x10000000 | (mem->offset >> 16));
+        nv_wr32(pfb, 0x1005a8, 0x0000ffff);
+        nv_mask(pfb, 0x10f800, 0x00000001, 0x00000001);
+
+        for (i = 0; i < 0x30; i++) {
+                nv_wr32(pfb, 0x10f8c0, (i << 8) | i);
+                nv_wr32(pfb, 0x10f900, pattern[i % 16]);
+        }
+
+        for (i = 0; i < 0x30; i++) {
+                nv_wr32(pfb, 0x10f8e0, (i << 8) | i);
+                nv_wr32(pfb, 0x10f920, pattern[i % 16]);
+        }
+
+        /* And upload the pattern */
+        r001700 = nv_rd32(pfb, 0x1700);
+        nv_wr32(pfb, 0x1700, mem->offset >> 16);
+        for (i = 0; i < 16; i++)
+                nv_wr32(pfb, 0x700000 + (i << 2), pattern[i]);
+        for (i = 0; i < 16; i++)
+                nv_wr32(pfb, 0x700100 + (i << 2), pattern[i]);
+        nv_wr32(pfb, 0x1700, r001700);
+
+        train->r_100720 = nv_rd32(pfb, 0x100720);
+        train->r_1111e0 = nv_rd32(pfb, 0x1111e0);
+        train->r_111400 = nv_rd32(pfb, 0x111400);
+
+        return 0;
+}
+
+void
+nva3_link_train_fini(struct nouveau_fb *pfb)
+{
+        struct nva3_ram *ram = (void *)pfb->ram;
+
+        if (ram->ltrain.mem)
+                pfb->ram->put(pfb, &ram->ltrain.mem);
+}
+
+/*
+ * RAM reclocking
+ */
+#define T(t) cfg->timing_10_##t
+static int
+nva3_ram_timing_calc(struct nouveau_fb *pfb, u32 *timing)
+{
+        struct nva3_ram *ram = (void *)pfb->ram;
+        struct nvbios_ramcfg *cfg = &ram->base.target.bios;
+        int tUNK_base, tUNK_40_0, prevCL;
+        u32 cur2, cur3, cur7, cur8;
+
+        cur2 = nv_rd32(pfb, 0x100228);
+        cur3 = nv_rd32(pfb, 0x10022c);
+        cur7 = nv_rd32(pfb, 0x10023c);
+        cur8 = nv_rd32(pfb, 0x100240);
+
+
+        switch ((!T(CWL)) * ram->base.type) {
+        case NV_MEM_TYPE_DDR2:
+                T(CWL) = T(CL) - 1;
+                break;
+        case NV_MEM_TYPE_GDDR3:
+                T(CWL) = ((cur2 & 0xff000000) >> 24) + 1;
+                break;
+        }
+
+        prevCL = (cur3 & 0x000000ff) + 1;
+        tUNK_base = ((cur7 & 0x00ff0000) >> 16) - prevCL;
+
+        timing[0] = (T(RP) << 24 | T(RAS) << 16 | T(RFC) << 8 | T(RC));
+        timing[1] = (T(WR) + 1 + T(CWL)) << 24 |
+                    max_t(u8,T(18), 1) << 16 |
+                    (T(WTR) + 1 + T(CWL)) << 8 |
+                    (5 + T(CL) - T(CWL));
+        timing[2] = (T(CWL) - 1) << 24 |
+                    (T(RRD) << 16) |
+                    (T(RCDWR) << 8) |
+                    T(RCDRD);
+        timing[3] = (cur3 & 0x00ff0000) |
+                    (0x30 + T(CL)) << 24 |
+                    (0xb + T(CL)) << 8 |
+                    (T(CL) - 1);
+        timing[4] = T(20) << 24 |
+                    T(21) << 16 |
+                    T(13) << 8 |
+                    T(13);
+        timing[5] = T(RFC) << 24 |
+                    max_t(u8,T(RCDRD), T(RCDWR)) << 16 |
+                    max_t(u8, (T(CWL) + 6), (T(CL) + 2)) << 8 |
+                    T(RP);
+        timing[6] = (0x5a + T(CL)) << 16 |
+                    max_t(u8, 1, (6 - T(CL) + T(CWL))) << 8 |
+                    (0x50 + T(CL) - T(CWL));
+        timing[7] = (cur7 & 0xff000000) |
+                    ((tUNK_base + T(CL)) << 16) |
+                    0x202;
+        timing[8] = cur8 & 0xffffff00;
+
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR2:
+        case NV_MEM_TYPE_GDDR3:
+                tUNK_40_0 = prevCL - (cur8 & 0xff);
+                if (tUNK_40_0 > 0)
+                        timing[8] |= T(CL);
+                break;
+        default:
+                break;
+        }
+
+        nv_debug(pfb, "Entry: 220: %08x %08x %08x %08x\n",
+                        timing[0], timing[1], timing[2], timing[3]);
+        nv_debug(pfb, "  230: %08x %08x %08x %08x\n",
+                        timing[4], timing[5], timing[6], timing[7]);
+        nv_debug(pfb, "  240: %08x\n", timing[8]);
+        return 0;
+}
+#undef T
+
+static void
+nouveau_sddr2_dll_reset(struct nva3_ramfuc *fuc)
+{
+        ram_mask(fuc, mr[0], 0x100, 0x100);
+        ram_nsec(fuc, 1000);
+        ram_mask(fuc, mr[0], 0x100, 0x000);
+        ram_nsec(fuc, 1000);
+}
+
+static void
+nouveau_sddr3_dll_disable(struct nva3_ramfuc *fuc, u32 *mr)
+{
+        u32 mr1_old = ram_rd32(fuc, mr[1]);
+
+        if (!(mr1_old & 0x1)) {
+                ram_wr32(fuc, 0x1002d4, 0x00000001);
+                ram_wr32(fuc, mr[1], mr[1]);
+                ram_nsec(fuc, 1000);
+        }
+}
+
+static void
+nouveau_gddr3_dll_disable(struct nva3_ramfuc *fuc, u32 *mr)
+{
+        u32 mr1_old = ram_rd32(fuc, mr[1]);
+
+        if (!(mr1_old & 0x40)) {
+                ram_wr32(fuc, mr[1], mr[1]);
+                ram_nsec(fuc, 1000);
+        }
+}
+
+static void
+nva3_ram_lock_pll(struct nva3_ramfuc *fuc, struct nva3_clock_info *mclk)
+{
+        ram_wr32(fuc, 0x004004, mclk->pll);
+        ram_mask(fuc, 0x004000, 0x00000001, 0x00000001);
+        ram_mask(fuc, 0x004000, 0x00000010, 0x00000000);
+        ram_wait(fuc, 0x004000, 0x00020000, 0x00020000, 64000);
+        ram_mask(fuc, 0x004000, 0x00000010, 0x00000010);
+}
+
+static void
+nva3_ram_fbvref(struct nva3_ramfuc *fuc, u32 val)
+{
+        struct nouveau_gpio *gpio = nouveau_gpio(fuc->base.pfb);
+        struct dcb_gpio_func func;
+        u32 reg, sh, gpio_val;
+        int ret;
+
+        if (gpio->get(gpio, 0, 0x2e, DCB_GPIO_UNUSED) != val) {
+                ret = gpio->find(gpio, 0, 0x2e, DCB_GPIO_UNUSED, &func);
+                if (ret)
+                        return;
+
+                nv50_gpio_location(func.line, &reg, &sh);
+                gpio_val = ram_rd32(fuc, gpioFBVREF);
+                if (gpio_val & (8 << sh))
+                        val = !val;
+
+                ram_mask(fuc, gpioFBVREF, (0x3 << sh), ((val | 0x2) << sh));
+                ram_nsec(fuc, 20000);
+        }
+}
+
 static int
 nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
 {
         struct nouveau_bios *bios = nouveau_bios(pfb);
         struct nva3_ram *ram = (void *)pfb->ram;
         struct nva3_ramfuc *fuc = &ram->fuc;
+        struct nva3_ltrain *train = &ram->ltrain;
         struct nva3_clock_info mclk;
         struct nouveau_ram_data *next;
         u8  ver, hdr, cnt, len, strap;
         u32 data;
-        u32 r004018, r100760, ctrl;
+        u32 r004018, r100760, r100da0, r111100, ctrl;
         u32 unk714, unk718, unk71c;
         int ret, i;
+        u32 timing[9];
+        bool pll2pll;
 
         next = &ram->base.target;
         next->freq = freq;
         ram->base.next = next;
 
+        if (ram->ltrain.state == NVA3_TRAIN_ONCE)
+                nva3_link_train(pfb);
+
         /* lookup memory config data relevant to the target frequency */
         i = 0;
-        while ((data = nvbios_rammapEp(bios, i++, &ver, &hdr, &cnt, &len,
-                                      &next->bios))) {
-                if (freq / 1000 >= next->bios.rammap_min &&
-                    freq / 1000 <= next->bios.rammap_max)
-                        break;
-        }
-
-        if (!data || ver != 0x10 || hdr < 0x0e) {
+        data = nvbios_rammapEm(bios, freq / 1000, &ver, &hdr, &cnt, &len,
+                                      &next->bios);
+        if (!data || ver != 0x10 || hdr < 0x05) {
                 nv_error(pfb, "invalid/missing rammap entry\n");
                 return -EINVAL;
         }
@@ -113,7 +539,7 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
 
         data = nvbios_rammapSp(bios, data, ver, hdr, cnt, len, strap,
                                &ver, &hdr, &next->bios);
-        if (!data || ver != 0x10 || hdr < 0x0e) {
+        if (!data || ver != 0x10 || hdr < 0x09) {
                 nv_error(pfb, "invalid/missing ramcfg entry\n");
                 return -EINVAL;
         }
@@ -123,7 +549,7 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
                 data = nvbios_timingEp(bios, next->bios.ramcfg_timing,
                                        &ver, &hdr, &cnt, &len,
                                        &next->bios);
-                if (!data || ver != 0x10 || hdr < 0x19) {
+                if (!data || ver != 0x10 || hdr < 0x17) {
                         nv_error(pfb, "invalid/missing timing entry\n");
                         return -EINVAL;
                 }
@@ -135,53 +561,99 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
                 return ret;
         }
 
+        nva3_ram_timing_calc(pfb, timing);
+
         ret = ram_init(fuc, pfb);
         if (ret)
                 return ret;
 
+        /* Determine ram-specific MR values */
+        ram->base.mr[0] = ram_rd32(fuc, mr[0]);
+        ram->base.mr[1] = ram_rd32(fuc, mr[1]);
+        ram->base.mr[2] = ram_rd32(fuc, mr[2]);
+
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR2:
+                ret = nouveau_sddr2_calc(&ram->base);
+                break;
+        case NV_MEM_TYPE_DDR3:
+                ret = nouveau_sddr3_calc(&ram->base);
+                break;
+        case NV_MEM_TYPE_GDDR3:
+                ret = nouveau_gddr3_calc(&ram->base);
+                break;
+        default:
+                ret = -ENOSYS;
+                break;
+        }
+
+        if (ret)
+                return ret;
+
         /* XXX: where the fuck does 750MHz come from? */
         if (freq <= 750000) {
                 r004018 = 0x10000000;
                 r100760 = 0x22222222;
+                r100da0 = 0x00000010;
         } else {
                 r004018 = 0x00000000;
                 r100760 = 0x00000000;
+                r100da0 = 0x00000000;
         }
 
+        if (!next->bios.ramcfg_10_DLLoff)
+                r004018 |= 0x00004000;
+
+        /* pll2pll requires to switch to a safe clock first */
         ctrl = ram_rd32(fuc, 0x004000);
-        if (ctrl & 0x00000008) {
-                if (mclk.pll) {
-                        ram_mask(fuc, 0x004128, 0x00000101, 0x00000101);
-                        ram_wr32(fuc, 0x004004, mclk.pll);
-                        ram_wr32(fuc, 0x004000, (ctrl |= 0x00000001));
-                        ram_wr32(fuc, 0x004000, (ctrl &= 0xffffffef));
-                        ram_wait(fuc, 0x004000, 0x00020000, 0x00020000, 64000);
-                        ram_wr32(fuc, 0x004000, (ctrl |= 0x00000010));
-                        ram_wr32(fuc, 0x004018, 0x00005000 | r004018);
-                        ram_wr32(fuc, 0x004000, (ctrl |= 0x00000004));
-                }
-        } else {
-                u32 ssel = 0x00000101;
-                if (mclk.clk)
-                        ssel |= mclk.clk;
-                else
-                        ssel |= 0x00080000; /* 324MHz, shouldn't matter... */
-                ram_mask(fuc, 0x004168, 0x003f3141, ctrl);
-        }
+        pll2pll = (!(ctrl & 0x00000008)) && mclk.pll;
 
+        /* Pre, NVIDIA does this outside the script */
         if (next->bios.ramcfg_10_02_10) {
                 ram_mask(fuc, 0x111104, 0x00000600, 0x00000000);
         } else {
                 ram_mask(fuc, 0x111100, 0x40000000, 0x40000000);
                 ram_mask(fuc, 0x111104, 0x00000180, 0x00000000);
         }
+        /* Always disable this bit during reclock */
+        ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
+
+        /* If switching from non-pll to pll, lock before disabling FB */
+        if (mclk.pll && !pll2pll) {
+                ram_mask(fuc, 0x004128, 0x003f3141, mclk.clk | 0x00000101);
+                nva3_ram_lock_pll(fuc, &mclk);
+        }
+
+        /* Start with disabling some CRTCs and PFIFO? */
+        ram_wait_vblank(fuc);
+        ram_wr32(fuc, 0x611200, 0x3300);
+        ram_mask(fuc, 0x002504, 0x1, 0x1);
+        ram_nsec(fuc, 10000);
+        ram_wait(fuc, 0x002504, 0x10, 0x10, 20000); /* XXX: or longer? */
+        ram_block(fuc);
+        ram_nsec(fuc, 2000);
+
+        if (!next->bios.ramcfg_10_02_10) {
+                if (ram->base.type == NV_MEM_TYPE_GDDR3)
+                        ram_mask(fuc, 0x111100, 0x04020000, 0x00020000);
+                else
+                        ram_mask(fuc, 0x111100, 0x04020000, 0x04020000);
+        }
+
+        /* If we're disabling the DLL, do it now */
+        switch (next->bios.ramcfg_10_DLLoff * ram->base.type) {
+        case NV_MEM_TYPE_DDR3:
+                nouveau_sddr3_dll_disable(fuc, ram->base.mr);
+                break;
+        case NV_MEM_TYPE_GDDR3:
+                nouveau_gddr3_dll_disable(fuc, ram->base.mr);
+                break;
+        }
 
-        if (!next->bios.rammap_10_04_02)
-                ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
-        ram_wr32(fuc, 0x611200, 0x00003300);
-        if (!next->bios.ramcfg_10_02_10)
-                ram_wr32(fuc, 0x111100, 0x4c020000); /*XXX*/
+        if (fuc->r_gpioFBVREF.addr && next->bios.timing_10_ODT)
+                nva3_ram_fbvref(fuc, 0);
 
+        /* Brace RAM for impact */
         ram_wr32(fuc, 0x1002d4, 0x00000001);
         ram_wr32(fuc, 0x1002d0, 0x00000001);
         ram_wr32(fuc, 0x1002d0, 0x00000001);
@@ -189,24 +661,38 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
         ram_wr32(fuc, 0x1002dc, 0x00000001);
         ram_nsec(fuc, 2000);
 
-        ctrl = ram_rd32(fuc, 0x004000);
-        if (!(ctrl & 0x00000008) && mclk.pll) {
-                ram_wr32(fuc, 0x004000, (ctrl |=  0x00000008));
+        if (nv_device(pfb)->chipset == 0xa3 && freq <= 500000)
+                ram_mask(fuc, 0x100700, 0x00000006, 0x00000006);
+
+        /* Fiddle with clocks */
+        /* There's 4 scenario's
+         * pll->pll: first switch to a 324MHz clock, set up new PLL, switch
+         * clk->pll: Set up new PLL, switch
+         * pll->clk: Set up clock, switch
+         * clk->clk: Overwrite ctrl and other bits, switch */
+
+        /* Switch to regular clock - 324MHz */
+        if (pll2pll) {
+                ram_mask(fuc, 0x004000, 0x00000004, 0x00000004);
+                ram_mask(fuc, 0x004168, 0x003f3141, 0x00083101);
+                ram_mask(fuc, 0x004000, 0x00000008, 0x00000008);
                 ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
                 ram_wr32(fuc, 0x004018, 0x00001000);
-                ram_wr32(fuc, 0x004000, (ctrl &= ~0x00000001));
-                ram_wr32(fuc, 0x004004, mclk.pll);
-                ram_wr32(fuc, 0x004000, (ctrl |=  0x00000001));
-                udelay(64);
-                ram_wr32(fuc, 0x004018, 0x00005000 | r004018);
-                udelay(20);
-        } else
-        if (!mclk.pll) {
-                ram_mask(fuc, 0x004168, 0x003f3040, mclk.clk);
-                ram_wr32(fuc, 0x004000, (ctrl |= 0x00000008));
+                nva3_ram_lock_pll(fuc, &mclk);
+        }
+
+        if (mclk.pll) {
+                ram_mask(fuc, 0x004000, 0x00000105, 0x00000105);
+                ram_wr32(fuc, 0x004018, 0x00001000 | r004018);
+                ram_wr32(fuc, 0x100da0, r100da0);
+        } else {
+                ram_mask(fuc, 0x004168, 0x003f3141, mclk.clk | 0x00000101);
+                ram_mask(fuc, 0x004000, 0x00000108, 0x00000008);
                 ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
-                ram_wr32(fuc, 0x004018, 0x0000d000 | r004018);
+                ram_wr32(fuc, 0x004018, 0x00009000 | r004018);
+                ram_wr32(fuc, 0x100da0, r100da0);
         }
+        ram_nsec(fuc, 20000);
 
         if (next->bios.rammap_10_04_08) {
                 ram_wr32(fuc, 0x1005a0, next->bios.ramcfg_10_06 << 16 |
@@ -220,6 +706,12 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
                                         0x80000000);
                 ram_mask(fuc, 0x10053c, 0x00001000, 0x00000000);
         } else {
+                if (train->state == NVA3_TRAIN_DONE) {
+                        ram_wr32(fuc, 0x100080, 0x1020);
+                        ram_mask(fuc, 0x111400, 0xffffffff, train->r_111400);
+                        ram_mask(fuc, 0x1111e0, 0xffffffff, train->r_1111e0);
+                        ram_mask(fuc, 0x100720, 0xffffffff, train->r_100720);
+                }
                 ram_mask(fuc, 0x10053c, 0x00001000, 0x00001000);
                 ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);
                 ram_mask(fuc, 0x100760, 0x22222222, r100760);
@@ -227,65 +719,131 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
                 ram_mask(fuc, 0x1007e0, 0x22222222, r100760);
         }
 
+        if (nv_device(pfb)->chipset == 0xa3 && freq > 500000) {
+                ram_mask(fuc, 0x100700, 0x00000006, 0x00000000);
+        }
+
+        /* Final switch */
         if (mclk.pll) {
                 ram_mask(fuc, 0x1110e0, 0x00088000, 0x00011000);
-                ram_wr32(fuc, 0x004000, (ctrl &= ~0x00000008));
+                ram_mask(fuc, 0x004000, 0x00000008, 0x00000000);
         }
 
-        /*XXX: LEAVE */
         ram_wr32(fuc, 0x1002dc, 0x00000000);
         ram_wr32(fuc, 0x1002d4, 0x00000001);
         ram_wr32(fuc, 0x100210, 0x80000000);
-        ram_nsec(fuc, 1000);
-        ram_nsec(fuc, 1000);
+        ram_nsec(fuc, 2000);
 
-        ram_mask(fuc, mr[2], 0x00000000, 0x00000000);
-        ram_nsec(fuc, 1000);
-        ram_nuke(fuc, mr[0]);
-        ram_mask(fuc, mr[0], 0x00000000, 0x00000000);
-        ram_nsec(fuc, 1000);
+        /* Set RAM MR parameters and timings */
+        for (i = 2; i >= 0; i--) {
+                if (ram_rd32(fuc, mr[i]) != ram->base.mr[i]) {
+                        ram_wr32(fuc, mr[i], ram->base.mr[i]);
+                        ram_nsec(fuc, 1000);
+                }
+        }
 
-        ram_mask(fuc, 0x100220[3], 0x00000000, 0x00000000);
-        ram_mask(fuc, 0x100220[1], 0x00000000, 0x00000000);
-        ram_mask(fuc, 0x100220[6], 0x00000000, 0x00000000);
-        ram_mask(fuc, 0x100220[7], 0x00000000, 0x00000000);
-        ram_mask(fuc, 0x100220[2], 0x00000000, 0x00000000);
-        ram_mask(fuc, 0x100220[4], 0x00000000, 0x00000000);
-        ram_mask(fuc, 0x100220[5], 0x00000000, 0x00000000);
-        ram_mask(fuc, 0x100220[0], 0x00000000, 0x00000000);
-        ram_mask(fuc, 0x100220[8], 0x00000000, 0x00000000);
+        ram_wr32(fuc, 0x100220[3], timing[3]);
+        ram_wr32(fuc, 0x100220[1], timing[1]);
+        ram_wr32(fuc, 0x100220[6], timing[6]);
+        ram_wr32(fuc, 0x100220[7], timing[7]);
+        ram_wr32(fuc, 0x100220[2], timing[2]);
+        ram_wr32(fuc, 0x100220[4], timing[4]);
+        ram_wr32(fuc, 0x100220[5], timing[5]);
+        ram_wr32(fuc, 0x100220[0], timing[0]);
+        ram_wr32(fuc, 0x100220[8], timing[8]);
 
+        /* Misc */
         ram_mask(fuc, 0x100200, 0x00001000, !next->bios.ramcfg_10_02_08 << 12);
 
-        unk714 = ram_rd32(fuc, 0x100714) & ~0xf0000010;
-        unk718 = ram_rd32(fuc, 0x100718) & ~0x00000100;
-        unk71c = ram_rd32(fuc, 0x10071c) & ~0x00000100;
+        /* XXX: A lot of "chipset"/"ram type" specific stuff...? */
+        unk714  = ram_rd32(fuc, 0x100714) & ~0xf0000130;
+        unk718  = ram_rd32(fuc, 0x100718) & ~0x00000100;
+        unk71c  = ram_rd32(fuc, 0x10071c) & ~0x00000100;
+        r111100 = ram_rd32(fuc, 0x111100) & ~0x3a800000;
+
+        if (next->bios.ramcfg_10_02_04) {
+                switch (ram->base.type) {
+                case NV_MEM_TYPE_DDR3:
+                        if (nv_device(pfb)->chipset != 0xa8)
+                                r111100 |= 0x00000004;
+                        /* no break */
+                case NV_MEM_TYPE_DDR2:
+                        r111100 |= 0x08000000;
+                        break;
+                default:
+                        break;
+                }
+        } else {
+                switch (ram->base.type) {
+                case NV_MEM_TYPE_DDR2:
+                        r111100 |= 0x1a800000;
+                        unk714  |= 0x00000010;
+                        break;
+                case NV_MEM_TYPE_DDR3:
+                        if (nv_device(pfb)->chipset == 0xa8) {
+                                r111100 |=  0x08000000;
+                        } else {
+                                r111100 &= ~0x00000004;
+                                r111100 |=  0x12800000;
+                        }
+                        unk714  |= 0x00000010;
+                        break;
+                case NV_MEM_TYPE_GDDR3:
+                        r111100 |= 0x30000000;
+                        unk714  |= 0x00000020;
+                        break;
+                default:
+                        break;
+                }
+        }
+
+        unk714 |= (next->bios.ramcfg_10_04_01) << 8;
+
         if (next->bios.ramcfg_10_02_20)
                 unk714 |= 0xf0000000;
-        if (!next->bios.ramcfg_10_02_04)
-                unk714 |= 0x00000010;
-        ram_wr32(fuc, 0x100714, unk714);
-
+        if (next->bios.ramcfg_10_02_02)
+                unk718 |= 0x00000100;
         if (next->bios.ramcfg_10_02_01)
                 unk71c |= 0x00000100;
-        ram_wr32(fuc, 0x10071c, unk71c);
+        if (next->bios.timing_10_24 != 0xff) {
+                unk718 &= ~0xf0000000;
+                unk718 |= next->bios.timing_10_24 << 28;
+        }
+        if (next->bios.ramcfg_10_02_10)
+                r111100 &= ~0x04020000;
 
-        if (next->bios.ramcfg_10_02_02)
-                unk718 |= 0x00000100;
-        ram_wr32(fuc, 0x100718, unk718);
+        ram_mask(fuc, 0x100714, 0xffffffff, unk714);
+        ram_mask(fuc, 0x10071c, 0xffffffff, unk71c);
+        ram_mask(fuc, 0x100718, 0xffffffff, unk718);
+        ram_mask(fuc, 0x111100, 0xffffffff, r111100);
 
-        if (next->bios.ramcfg_10_02_10)
-                ram_wr32(fuc, 0x111100, 0x48000000); /*XXX*/
+        if (fuc->r_gpioFBVREF.addr && !next->bios.timing_10_ODT)
+                nva3_ram_fbvref(fuc, 1);
 
-        ram_mask(fuc, mr[0], 0x100, 0x100);
-        ram_nsec(fuc, 1000);
-        ram_mask(fuc, mr[0], 0x100, 0x000);
-        ram_nsec(fuc, 1000);
+        /* Reset DLL */
+        if (!next->bios.ramcfg_10_DLLoff)
+                nouveau_sddr2_dll_reset(fuc);
 
-        ram_nsec(fuc, 2000);
-        ram_nsec(fuc, 12000);
+        if (ram->base.type == NV_MEM_TYPE_GDDR3) {
+                ram_nsec(fuc, 31000);
+        } else {
+                ram_nsec(fuc, 14000);
+        }
+
+        if (ram->base.type == NV_MEM_TYPE_DDR3) {
+                ram_wr32(fuc, 0x100264, 0x1);
+                ram_nsec(fuc, 2000);
+        }
 
-        ram_wr32(fuc, 0x611200, 0x00003330);
+        ram_nuke(fuc, 0x100700);
+        ram_mask(fuc, 0x100700, 0x01000000, 0x01000000);
+        ram_mask(fuc, 0x100700, 0x01000000, 0x00000000);
+
+        /* Re-enable FB */
+        ram_unblock(fuc);
+        ram_wr32(fuc, 0x611200, 0x3330);
+
+        /* Post fiddlings */
         if (next->bios.rammap_10_04_02)
                 ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);
         if (next->bios.ramcfg_10_02_10) {
@@ -313,7 +871,22 @@ nva3_ram_prog(struct nouveau_fb *pfb)
         struct nouveau_device *device = nv_device(pfb);
         struct nva3_ram *ram = (void *)pfb->ram;
         struct nva3_ramfuc *fuc = &ram->fuc;
-        ram_exec(fuc, nouveau_boolopt(device->cfgopt, "NvMemExec", true));
+        bool exec = nouveau_boolopt(device->cfgopt, "NvMemExec", true);
+
+        if (exec) {
+                nv_mask(pfb, 0x001534, 0x2, 0x2);
+
+                ram_exec(fuc, true);
+
+                /* Post-processing, avoids flicker */
+                nv_mask(pfb, 0x002504, 0x1, 0x0);
+                nv_mask(pfb, 0x001534, 0x2, 0x0);
+
+                nv_mask(pfb, 0x616308, 0x10, 0x10);
+                nv_mask(pfb, 0x616b08, 0x10, 0x10);
+        } else {
+                ram_exec(fuc, false);
+        }
         return 0;
 }
 
@@ -330,38 +903,24 @@ nva3_ram_init(struct nouveau_object *object)
 {
         struct nouveau_fb *pfb = (void *)object->parent;
         struct nva3_ram   *ram = (void *)object;
-        int ret, i;
+        int ret;
 
         ret = nouveau_ram_init(&ram->base);
         if (ret)
                 return ret;
 
-        /* prepare for ddr link training, and load training patterns */
-        switch (ram->base.type) {
-        case NV_MEM_TYPE_DDR3: {
-                if (nv_device(pfb)->chipset == 0xa8) {
-                        static const u32 pattern[16] = {
-                                0xaaaaaaaa, 0xcccccccc, 0xdddddddd, 0xeeeeeeee,
-                                0x00000000, 0x11111111, 0x44444444, 0xdddddddd,
-                                0x33333333, 0x55555555, 0x77777777, 0x66666666,
-                                0x99999999, 0x88888888, 0xeeeeeeee, 0xbbbbbbbb,
-                        };
-
-                        nv_wr32(pfb, 0x100538, 0x10001ff6); /*XXX*/
-                        nv_wr32(pfb, 0x1005a8, 0x0000ffff);
-                        nv_mask(pfb, 0x10f800, 0x00000001, 0x00000001);
-                        for (i = 0; i < 0x30; i++) {
-                                nv_wr32(pfb, 0x10f8c0, (i << 8) | i);
-                                nv_wr32(pfb, 0x10f8e0, (i << 8) | i);
-                                nv_wr32(pfb, 0x10f900, pattern[i % 16]);
-                                nv_wr32(pfb, 0x10f920, pattern[i % 16]);
-                        }
-                }
-        }
-                break;
-        default:
-                break;
-        }
+        nva3_link_train_init(pfb);
+
+        return 0;
+}
+
+static int
+nva3_ram_fini(struct nouveau_object *object, bool suspend)
+{
+        struct nouveau_fb *pfb = (void *)object->parent;
+
+        if (!suspend)
+                nva3_link_train_fini(pfb);
 
         return 0;
 }
@@ -371,8 +930,12 @@ nva3_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
               struct nouveau_oclass *oclass, void *data, u32 datasize,
               struct nouveau_object **pobject)
 {
+        struct nouveau_fb *pfb = nouveau_fb(parent);
+        struct nouveau_gpio *gpio = nouveau_gpio(pfb);
+        struct dcb_gpio_func func;
         struct nva3_ram *ram;
         int ret, i;
+        u32 reg, shift;
 
         ret = nv50_ram_create(parent, engine, oclass, &ram);
         *pobject = nv_object(ram);
@@ -380,7 +943,9 @@ nva3_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                 return ret;
 
         switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR2:
         case NV_MEM_TYPE_DDR3:
+        case NV_MEM_TYPE_GDDR3:
                 ram->base.calc = nva3_ram_calc;
                 ram->base.prog = nva3_ram_prog;
                 ram->base.tidy = nva3_ram_tidy;
@@ -390,31 +955,41 @@ nva3_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                 return 0;
         }
 
+        ram->fuc.r_0x001610 = ramfuc_reg(0x001610);
+        ram->fuc.r_0x001700 = ramfuc_reg(0x001700);
+        ram->fuc.r_0x002504 = ramfuc_reg(0x002504);
         ram->fuc.r_0x004000 = ramfuc_reg(0x004000);
         ram->fuc.r_0x004004 = ramfuc_reg(0x004004);
         ram->fuc.r_0x004018 = ramfuc_reg(0x004018);
         ram->fuc.r_0x004128 = ramfuc_reg(0x004128);
         ram->fuc.r_0x004168 = ramfuc_reg(0x004168);
+        ram->fuc.r_0x100080 = ramfuc_reg(0x100080);
         ram->fuc.r_0x100200 = ramfuc_reg(0x100200);
         ram->fuc.r_0x100210 = ramfuc_reg(0x100210);
         for (i = 0; i < 9; i++)
                 ram->fuc.r_0x100220[i] = ramfuc_reg(0x100220 + (i * 4));
+        ram->fuc.r_0x100264 = ramfuc_reg(0x100264);
         ram->fuc.r_0x1002d0 = ramfuc_reg(0x1002d0);
         ram->fuc.r_0x1002d4 = ramfuc_reg(0x1002d4);
         ram->fuc.r_0x1002dc = ramfuc_reg(0x1002dc);
         ram->fuc.r_0x10053c = ramfuc_reg(0x10053c);
         ram->fuc.r_0x1005a0 = ramfuc_reg(0x1005a0);
         ram->fuc.r_0x1005a4 = ramfuc_reg(0x1005a4);
+        ram->fuc.r_0x100700 = ramfuc_reg(0x100700);
         ram->fuc.r_0x100714 = ramfuc_reg(0x100714);
         ram->fuc.r_0x100718 = ramfuc_reg(0x100718);
         ram->fuc.r_0x10071c = ramfuc_reg(0x10071c);
+        ram->fuc.r_0x100720 = ramfuc_reg(0x100720);
         ram->fuc.r_0x100760 = ramfuc_stride(0x100760, 4, ram->base.part_mask);
         ram->fuc.r_0x1007a0 = ramfuc_stride(0x1007a0, 4, ram->base.part_mask);
         ram->fuc.r_0x1007e0 = ramfuc_stride(0x1007e0, 4, ram->base.part_mask);
+        ram->fuc.r_0x100da0 = ramfuc_stride(0x100da0, 4, ram->base.part_mask);
         ram->fuc.r_0x10f804 = ramfuc_reg(0x10f804);
         ram->fuc.r_0x1110e0 = ramfuc_stride(0x1110e0, 4, ram->base.part_mask);
         ram->fuc.r_0x111100 = ramfuc_reg(0x111100);
         ram->fuc.r_0x111104 = ramfuc_reg(0x111104);
+        ram->fuc.r_0x1111e0 = ramfuc_reg(0x1111e0);
+        ram->fuc.r_0x111400 = ramfuc_reg(0x111400);
         ram->fuc.r_0x611200 = ramfuc_reg(0x611200);
 
         if (ram->base.ranks > 1) {
@@ -429,6 +1004,12 @@ nva3_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                 ram->fuc.r_mr[3] = ramfuc_reg(0x1002e4);
         }
 
+        ret = gpio->find(gpio, 0, 0x2e, DCB_GPIO_UNUSED, &func);
+        if (ret == 0) {
+                nv50_gpio_location(func.line, &reg, &shift);
+                ram->fuc.r_gpioFBVREF = ramfuc_reg(reg);
+        }
+
         return 0;
 }
 
@@ -438,6 +1019,6 @@ nva3_ram_oclass = {
                 .ctor = nva3_ram_ctor,
                 .dtor = _nouveau_ram_dtor,
                 .init = nva3_ram_init,
-                .fini = _nouveau_ram_fini,
+                .fini = nva3_ram_fini,
         },
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/sddr2.c b/drivers/gpu/drm/nouveau/core/subdev/fb/sddr2.c
index bb1eb8f3e639..252575f3aa29 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/sddr2.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/sddr2.c
@@ -66,7 +66,7 @@ nouveau_sddr2_calc(struct nouveau_ram *ram)
         case 0x10:
                 CL  = ram->next->bios.timing_10_CL;
                 WR  = ram->next->bios.timing_10_WR;
-                DLL = !ram->next->bios.ramcfg_10_02_40;
+                DLL = !ram->next->bios.ramcfg_10_DLLoff;
                 ODT = ram->next->bios.timing_10_ODT & 3;
                 break;
         case 0x20:
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/sddr3.c b/drivers/gpu/drm/nouveau/core/subdev/fb/sddr3.c
index 83949b11833a..a2dca4869e52 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/sddr3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/sddr3.c
@@ -80,7 +80,7 @@ nouveau_sddr3_calc(struct nouveau_ram *ram)
                 CWL = ram->next->bios.timing_10_CWL;
                 CL  = ram->next->bios.timing_10_CL;
                 WR  = ram->next->bios.timing_10_WR;
-                DLL = !ram->next->bios.ramcfg_10_02_40;
+                DLL = !ram->next->bios.ramcfg_10_DLLoff;
                 ODT = ram->next->bios.timing_10_ODT;
                 break;
         case 0x20:
diff --git a/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c
index 1864fa98e6b1..2e30d5a62d6e 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c
@@ -54,7 +54,7 @@ nv50_gpio_reset(struct nouveau_gpio *gpio, u8 match)
         }
 }
 
-static int
+int
 nv50_gpio_location(int line, u32 *reg, u32 *shift)
 {
         const u32 nv50_gpio_reg[4] = { 0xe104, 0xe108, 0xe280, 0xe284 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
index 2b1bf545e488..0dc605db7ec8 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
@@ -473,18 +473,56 @@ nouveau_i2c_extdev_sclass[] = {
         nouveau_anx9805_sclass,
 };
 
+static void
+nouveau_i2c_create_port(struct nouveau_i2c *i2c, int index, u8 type,
+                        struct dcb_i2c_entry *info)
+{
+        const struct nouveau_i2c_impl *impl = (void *)nv_oclass(i2c);
+        struct nouveau_oclass *oclass;
+        struct nouveau_object *parent;
+        struct nouveau_object *object;
+        int ret, pad;
+
+        if (info->share != DCB_I2C_UNUSED) {
+                pad    = info->share;
+                oclass = impl->pad_s;
+        } else {
+                if (type != DCB_I2C_NVIO_AUX)
+                        pad = 0x100 + info->drive;
+                else
+                        pad = 0x100 + info->auxch;
+                oclass = impl->pad_x;
+        }
+
+        ret = nouveau_object_ctor(NULL, nv_object(i2c), oclass, NULL, pad,
+                                 &parent);
+        if (ret < 0)
+                return;
+
+        oclass = impl->sclass;
+        do {
+                ret = -EINVAL;
+                if (oclass->handle == type) {
+                        ret = nouveau_object_ctor(parent, nv_object(i2c),
+                                                  oclass, info, index,
+                                                 &object);
+                }
+        } while (ret && (++oclass)->handle);
+
+        nouveau_object_ref(NULL, &parent);
+}
+
 int
 nouveau_i2c_create_(struct nouveau_object *parent,
                     struct nouveau_object *engine,
                     struct nouveau_oclass *oclass,
                     int length, void **pobject)
 {
-        const struct nouveau_i2c_impl *impl = (void *)oclass;
         struct nouveau_bios *bios = nouveau_bios(parent);
         struct nouveau_i2c *i2c;
         struct nouveau_object *object;
         struct dcb_i2c_entry info;
-        int ret, i, j, index = -1, pad;
+        int ret, i, j, index = -1;
         struct dcb_output outp;
         u8  ver, hdr;
         u32 data;
@@ -507,43 +545,40 @@ nouveau_i2c_create_(struct nouveau_object *parent,
         INIT_LIST_HEAD(&i2c->ports);
 
         while (!dcb_i2c_parse(bios, ++index, &info)) {
-                if (info.type == DCB_I2C_UNUSED)
+                switch (info.type) {
+                case DCB_I2C_NV04_BIT:
+                case DCB_I2C_NV4E_BIT:
+                case DCB_I2C_NVIO_BIT:
+                        nouveau_i2c_create_port(i2c, NV_I2C_PORT(index),
+                                                info.type, &info);
+                        break;
+                case DCB_I2C_NVIO_AUX:
+                        nouveau_i2c_create_port(i2c, NV_I2C_AUX(index),
+                                                info.type, &info);
+                        break;
+                case DCB_I2C_PMGR:
+                        if (info.drive != DCB_I2C_UNUSED) {
+                                nouveau_i2c_create_port(i2c, NV_I2C_PORT(index),
+                                                        DCB_I2C_NVIO_BIT,
+                                                        &info);
+                        }
+                        if (info.auxch != DCB_I2C_UNUSED) {
+                                nouveau_i2c_create_port(i2c, NV_I2C_AUX(index),
+                                                        DCB_I2C_NVIO_AUX,
+                                                        &info);
+                        }
+                        break;
+                case DCB_I2C_UNUSED:
+                default:
                         continue;
-
-                if (info.share != DCB_I2C_UNUSED) {
-                        if (info.type == DCB_I2C_NVIO_AUX)
-                                pad = info.drive;
-                        else
-                                pad = info.share;
-                        oclass = impl->pad_s;
-                } else {
-                        pad = 0x100 + info.drive;
-                        oclass = impl->pad_x;
                 }
-
-                ret = nouveau_object_ctor(NULL, *pobject, oclass,
-                                          NULL, pad, &parent);
-                if (ret < 0)
-                        continue;
-
-                oclass = impl->sclass;
-                do {
-                        ret = -EINVAL;
-                        if (oclass->handle == info.type) {
-                                ret = nouveau_object_ctor(parent, *pobject,
-                                                          oclass, &info,
-                                                          index, &object);
-                        }
-                } while (ret && (++oclass)->handle);
-
-                nouveau_object_ref(NULL, &parent);
         }
 
         /* in addition to the busses specified in the i2c table, there
          * may be ddc/aux channels hiding behind external tmds/dp/etc
          * transmitters.
          */
-        index = ((index + 0x0f) / 0x10) * 0x10;
+        index = NV_I2C_EXT(0);
         i = -1;
         while ((data = dcb_outp_parse(bios, ++i, &ver, &hdr, &outp))) {
                 if (!outp.location || !outp.extdev)
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/gm204.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/gm204.c
new file mode 100644
index 000000000000..06a2b87ccbf1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/gm204.c
@@ -0,0 +1,221 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv50.h"
+
+#define AUX_DBG(fmt, args...) nv_debug(aux, "AUXCH(%d): " fmt, ch, ##args)
+#define AUX_ERR(fmt, args...) nv_error(aux, "AUXCH(%d): " fmt, ch, ##args)
+
+static void
+auxch_fini(struct nouveau_i2c *aux, int ch)
+{
+        nv_mask(aux, 0x00d954 + (ch * 0x50), 0x00310000, 0x00000000);
+}
+
+static int
+auxch_init(struct nouveau_i2c *aux, int ch)
+{
+        const u32 unksel = 1; /* nfi which to use, or if it matters.. */
+        const u32 ureq = unksel ? 0x00100000 : 0x00200000;
+        const u32 urep = unksel ? 0x01000000 : 0x02000000;
+        u32 ctrl, timeout;
+
+        /* wait up to 1ms for any previous transaction to be done... */
+        timeout = 1000;
+        do {
+                ctrl = nv_rd32(aux, 0x00d954 + (ch * 0x50));
+                udelay(1);
+                if (!timeout--) {
+                        AUX_ERR("begin idle timeout 0x%08x\n", ctrl);
+                        return -EBUSY;
+                }
+        } while (ctrl & 0x03010000);
+
+        /* set some magic, and wait up to 1ms for it to appear */
+        nv_mask(aux, 0x00d954 + (ch * 0x50), 0x00300000, ureq);
+        timeout = 1000;
+        do {
+                ctrl = nv_rd32(aux, 0x00d954 + (ch * 0x50));
+                udelay(1);
+                if (!timeout--) {
+                        AUX_ERR("magic wait 0x%08x\n", ctrl);
+                        auxch_fini(aux, ch);
+                        return -EBUSY;
+                }
+        } while ((ctrl & 0x03000000) != urep);
+
+        return 0;
+}
+
+int
+gm204_aux(struct nouveau_i2c_port *base, bool retry,
+         u8 type, u32 addr, u8 *data, u8 size)
+{
+        struct nouveau_i2c *aux = nouveau_i2c(base);
+        struct nv50_i2c_port *port = (void *)base;
+        u32 ctrl, stat, timeout, retries;
+        u32 xbuf[4] = {};
+        int ch = port->addr;
+        int ret, i;
+
+        AUX_DBG("%d: 0x%08x %d\n", type, addr, size);
+
+        ret = auxch_init(aux, ch);
+        if (ret)
+                goto out;
+
+        stat = nv_rd32(aux, 0x00d958 + (ch * 0x50));
+        if (!(stat & 0x10000000)) {
+                AUX_DBG("sink not detected\n");
+                ret = -ENXIO;
+                goto out;
+        }
+
+        if (!(type & 1)) {
+                memcpy(xbuf, data, size);
+                for (i = 0; i < 16; i += 4) {
+                        AUX_DBG("wr 0x%08x\n", xbuf[i / 4]);
+                        nv_wr32(aux, 0x00d930 + (ch * 0x50) + i, xbuf[i / 4]);
+                }
+        }
+
+        ctrl  = nv_rd32(aux, 0x00d954 + (ch * 0x50));
+        ctrl &= ~0x0001f0ff;
+        ctrl |= type << 12;
+        ctrl |= size - 1;
+        nv_wr32(aux, 0x00d950 + (ch * 0x50), addr);
+
+        /* (maybe) retry transaction a number of times on failure... */
+        for (retries = 0; !ret && retries < 32; retries++) {
+                /* reset, and delay a while if this is a retry */
+                nv_wr32(aux, 0x00d954 + (ch * 0x50), 0x80000000 | ctrl);
+                nv_wr32(aux, 0x00d954 + (ch * 0x50), 0x00000000 | ctrl);
+                if (retries)
+                        udelay(400);
+
+                /* transaction request, wait up to 1ms for it to complete */
+                nv_wr32(aux, 0x00d954 + (ch * 0x50), 0x00010000 | ctrl);
+
+                timeout = 1000;
+                do {
+                        ctrl = nv_rd32(aux, 0x00d954 + (ch * 0x50));
+                        udelay(1);
+                        if (!timeout--) {
+                                AUX_ERR("tx req timeout 0x%08x\n", ctrl);
+                                ret = -EIO;
+                                goto out;
+                        }
+                } while (ctrl & 0x00010000);
+                ret = 1;
+
+                /* read status, and check if transaction completed ok */
+                stat = nv_mask(aux, 0x00d958 + (ch * 0x50), 0, 0);
+                if ((stat & 0x000f0000) == 0x00080000 ||
+                    (stat & 0x000f0000) == 0x00020000)
+                        ret = retry ? 0 : 1;
+                if ((stat & 0x00000100))
+                        ret = -ETIMEDOUT;
+                if ((stat & 0x00000e00))
+                        ret = -EIO;
+
+                AUX_DBG("%02d 0x%08x 0x%08x\n", retries, ctrl, stat);
+        }
+
+        if (type & 1) {
+                for (i = 0; i < 16; i += 4) {
+                        xbuf[i / 4] = nv_rd32(aux, 0x00d940 + (ch * 0x50) + i);
+                        AUX_DBG("rd 0x%08x\n", xbuf[i / 4]);
+                }
+                memcpy(data, xbuf, size);
+        }
+
+out:
+        auxch_fini(aux, ch);
+        return ret < 0 ? ret : (stat & 0x000f0000) >> 16;
+}
+
+static const struct nouveau_i2c_func
+gm204_aux_func = {
+        .aux       = gm204_aux,
+};
+
+int
+gm204_aux_port_ctor(struct nouveau_object *parent,
+                    struct nouveau_object *engine,
+                    struct nouveau_oclass *oclass, void *data, u32 index,
+                    struct nouveau_object **pobject)
+{
+        struct dcb_i2c_entry *info = data;
+        struct nv50_i2c_port *port;
+        int ret;
+
+        ret = nouveau_i2c_port_create(parent, engine, oclass, index,
+                                      &nouveau_i2c_aux_algo, &gm204_aux_func,
+                                      &port);
+        *pobject = nv_object(port);
+        if (ret)
+                return ret;
+
+        port->base.aux = info->auxch;
+        port->addr = info->auxch;
+        return 0;
+}
+
+struct nouveau_oclass
+gm204_i2c_sclass[] = {
+        { .handle = NV_I2C_TYPE_DCBI2C(DCB_I2C_NVIO_BIT),
+          .ofuncs = &(struct nouveau_ofuncs) {
+                  .ctor = nvd0_i2c_port_ctor,
+                  .dtor = _nouveau_i2c_port_dtor,
+                  .init = nv50_i2c_port_init,
+                  .fini = _nouveau_i2c_port_fini,
+          },
+        },
+        { .handle = NV_I2C_TYPE_DCBI2C(DCB_I2C_NVIO_AUX),
+          .ofuncs = &(struct nouveau_ofuncs) {
+                  .ctor = gm204_aux_port_ctor,
+                  .dtor = _nouveau_i2c_port_dtor,
+                  .init = _nouveau_i2c_port_init,
+                  .fini = _nouveau_i2c_port_fini,
+          },
+        },
+        {}
+};
+
+struct nouveau_oclass *
+gm204_i2c_oclass = &(struct nouveau_i2c_impl) {
+        .base.handle = NV_SUBDEV(I2C, 0x24),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nouveau_i2c_ctor,
+                .dtor = _nouveau_i2c_dtor,
+                .init = _nouveau_i2c_init,
+                .fini = _nouveau_i2c_fini,
+        },
+        .sclass = gm204_i2c_sclass,
+        .pad_x = &nv04_i2c_pad_oclass,
+        .pad_s = &gm204_i2c_pad_oclass,
+        .aux = 8,
+        .aux_stat = nve0_aux_stat,
+        .aux_mask = nve0_aux_mask,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/nv50.h b/drivers/gpu/drm/nouveau/core/subdev/i2c/nv50.h
index 5d2a77421c74..9ef965692fb1 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/nv50.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/nv50.h
@@ -10,8 +10,6 @@ struct nv50_i2c_priv {
 struct nv50_i2c_port {
         struct nouveau_i2c_port base;
         u32 addr;
-        u32 ctrl;
-        u32 data;
         u32 state;
 };
 
@@ -29,4 +27,8 @@ int  nv94_aux_port_ctor(struct nouveau_object *, struct nouveau_object *,
 void nv94_i2c_acquire(struct nouveau_i2c_port *);
 void nv94_i2c_release(struct nouveau_i2c_port *);
 
+int  nvd0_i2c_port_ctor(struct nouveau_object *, struct nouveau_object *,
+                        struct nouveau_oclass *, void *, u32,
+                        struct nouveau_object **);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/nv94.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/nv94.c
index f59c3a255462..e383ee81f4d2 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/nv94.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/nv94.c
@@ -214,10 +214,6 @@ nv94_i2c_port_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         port->state = 7;
         port->addr = nv50_i2c_addr[info->drive];
-        if (info->share != DCB_I2C_UNUSED) {
-                port->ctrl = 0x00e500 + (info->share * 0x50);
-                port->data = 0x0000e001;
-        }
         return 0;
 }
 
@@ -242,13 +238,8 @@ nv94_aux_port_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        port->base.aux = info->drive;
-        port->addr = info->drive;
-        if (info->share != DCB_I2C_UNUSED) {
-                port->ctrl = 0x00e500 + (info->drive * 0x50);
-                port->data = 0x00002002;
-        }
-
+        port->base.aux = info->auxch;
+        port->addr = info->auxch;
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/nvd0.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/nvd0.c
index 364ddb1c5f03..fd99380502ec 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/nvd0.c
@@ -48,7 +48,7 @@ nvd0_i2c_func = {
         .sense_sda = nvd0_i2c_sense_sda,
 };
 
-static int
+int
 nvd0_i2c_port_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                    struct nouveau_oclass *oclass, void *data, u32 index,
                    struct nouveau_object **pobject)
@@ -66,10 +66,6 @@ nvd0_i2c_port_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         port->state = 0x00000007;
         port->addr = 0x00d014 + (info->drive * 0x20);
-        if (info->share != DCB_I2C_UNUSED) {
-                port->ctrl = 0x00e500 + (info->share * 0x50);
-                port->data = 0x0000e001;
-        }
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/nve0.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/nve0.c
index cae77e1ad8dc..25fe5c2d110e 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/nve0.c
@@ -24,7 +24,7 @@
 
 #include "nv50.h"
 
-static void
+void
 nve0_aux_stat(struct nouveau_i2c *i2c, u32 *hi, u32 *lo, u32 *rq, u32 *tx)
 {
         u32 intr = nv_rd32(i2c, 0x00dc60);
@@ -38,7 +38,7 @@ nve0_aux_stat(struct nouveau_i2c *i2c, u32 *hi, u32 *lo, u32 *rq, u32 *tx)
         nv_wr32(i2c, 0x00dc60, intr);
 }
 
-static void
+void
 nve0_aux_mask(struct nouveau_i2c *i2c, u32 type, u32 mask, u32 data)
 {
         u32 temp = nv_rd32(i2c, 0x00dc68), i;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/padgm204.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/padgm204.c
new file mode 100644
index 000000000000..f0e6fbbaa8cd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/padgm204.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "pad.h"
+
+struct gm204_i2c_pad {
+        struct nvkm_i2c_pad base;
+        int addr;
+};
+
+static int
+gm204_i2c_pad_fini(struct nouveau_object *object, bool suspend)
+{
+        struct nouveau_i2c *i2c = (void *)object->engine;
+        struct gm204_i2c_pad *pad = (void *)object;
+        nv_mask(i2c, 0x00d97c + pad->addr, 0x00000001, 0x00000001);
+        return nvkm_i2c_pad_fini(&pad->base, suspend);
+}
+
+static int
+gm204_i2c_pad_init(struct nouveau_object *object)
+{
+        struct nouveau_i2c *i2c = (void *)object->engine;
+        struct gm204_i2c_pad *pad = (void *)object;
+
+        switch (nv_oclass(pad->base.next)->handle) {
+        case NV_I2C_TYPE_DCBI2C(DCB_I2C_NVIO_AUX):
+                nv_mask(i2c, 0x00d970 + pad->addr, 0x0000c003, 0x00000002);
+                break;
+        case NV_I2C_TYPE_DCBI2C(DCB_I2C_NVIO_BIT):
+        default:
+                nv_mask(i2c, 0x00d970 + pad->addr, 0x0000c003, 0x0000c001);
+                break;
+        }
+
+        nv_mask(i2c, 0x00d97c + pad->addr, 0x00000001, 0x00000000);
+        return nvkm_i2c_pad_init(&pad->base);
+}
+
+static int
+gm204_i2c_pad_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                  struct nouveau_oclass *oclass, void *data, u32 index,
+                  struct nouveau_object **pobject)
+{
+        struct gm204_i2c_pad *pad;
+        int ret;
+
+        ret = nvkm_i2c_pad_create(parent, engine, oclass, index, &pad);
+        *pobject = nv_object(pad);
+        if (ret)
+                return ret;
+
+        pad->addr = index * 0x50;;
+        return 0;
+}
+
+struct nouveau_oclass
+gm204_i2c_pad_oclass = {
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gm204_i2c_pad_ctor,
+                .dtor = _nvkm_i2c_pad_dtor,
+                .init = gm204_i2c_pad_init,
+                .fini = gm204_i2c_pad_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/priv.h b/drivers/gpu/drm/nouveau/core/subdev/i2c/priv.h
index 780090b6425a..4fe7ae3fde4e 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/priv.h
@@ -5,6 +5,7 @@
 
 extern struct nouveau_oclass nv04_i2c_pad_oclass;
 extern struct nouveau_oclass nv94_i2c_pad_oclass;
+extern struct nouveau_oclass gm204_i2c_pad_oclass;
 
 #define nouveau_i2c_port_create(p,e,o,i,a,f,d)                                 \
         nouveau_i2c_port_create_((p), (e), (o), (i), (a), (f),                 \
@@ -82,4 +83,7 @@ struct nouveau_i2c_impl {
 void nv94_aux_stat(struct nouveau_i2c *, u32 *, u32 *, u32 *, u32 *);
 void nv94_aux_mask(struct nouveau_i2c *, u32, u32, u32);
 
+void nve0_aux_stat(struct nouveau_i2c *, u32 *, u32 *, u32 *, u32 *);
+void nve0_aux_mask(struct nouveau_i2c *, u32, u32, u32);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/memx.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/memx.fuc
index e89789a53b80..ec03f9a4290b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/memx.fuc
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/memx.fuc
@@ -50,6 +50,7 @@ handler(WR32  , 0x0000, 0x0002, #memx_func_wr32)
 handler(WAIT  , 0x0004, 0x0000, #memx_func_wait)
 handler(DELAY , 0x0001, 0x0000, #memx_func_delay)
 handler(VBLANK, 0x0001, 0x0000, #memx_func_wait_vblank)
+handler(TRAIN , 0x0000, 0x0000, #memx_func_train)
 memx_func_tail:
 
 .equ #memx_func_size #memx_func_next - #memx_func_head
@@ -63,6 +64,10 @@ memx_ts_end:
 memx_data_head:
 .skip 0x0800
 memx_data_tail:
+
+memx_train_head:
+.skip 0x0100
+memx_train_tail:
 #endif
 
 /******************************************************************************
@@ -260,6 +265,101 @@ memx_func_delay:
 // description
 //
 // $r15 - current (memx)
+// $r4  - packet length
+// $r3  - opcode desciption
+// $r0  - zero
+memx_func_train:
+#if NVKM_PPWR_CHIPSET == GT215
+// $r5 - outer loop counter
+// $r6 - inner loop counter
+// $r7 - entry counter (#memx_train_head + $r7)
+        movw $r5 0x3
+        movw $r7 0x0
+
+// Read random memory to wake up... things
+        imm32($r9, 0x700000)
+        nv_rd32($r8,$r9)
+        movw $r14 0x2710
+        call(nsec)
+
+        memx_func_train_loop_outer:
+                mulu $r8 $r5 0x101
+                sethi $r8 0x02000000
+                imm32($r9, 0x1111e0)
+                nv_wr32($r9, $r8)
+                push $r5
+
+                movw $r6 0x0
+                memx_func_train_loop_inner:
+                        movw $r8 0x1111
+                        mulu $r9 $r6 $r8
+                        shl b32 $r8 $r9 0x10
+                        or $r8 $r9
+                        imm32($r9, 0x100720)
+                        nv_wr32($r9, $r8)
+
+                        imm32($r9, 0x100080)
+                        nv_rd32($r8, $r9)
+                        or $r8 $r8 0x20
+                        nv_wr32($r9, $r8)
+
+                        imm32($r9, 0x10053c)
+                        imm32($r8, 0x80003002)
+                        nv_wr32($r9, $r8)
+
+                        imm32($r14, 0x100560)
+                        imm32($r13, 0x80000000)
+                        add b32 $r12 $r13 0
+                        imm32($r11, 0x001e8480)
+                        call(wait)
+
+                        // $r5 - inner inner loop counter
+                        // $r9 - result
+                        movw $r5 0
+                        imm32($r9, 0x8300ffff)
+                        memx_func_train_loop_4x:
+                                imm32($r10, 0x100080)
+                                nv_rd32($r8, $r10)
+                                imm32($r11, 0xffffffdf)
+                                and $r8 $r11
+                                nv_wr32($r10, $r8)
+
+                                imm32($r10, 0x10053c)
+                                imm32($r8, 0x80003002)
+                                nv_wr32($r10, $r8)
+
+                                imm32($r14, 0x100560)
+                                imm32($r13, 0x80000000)
+                                mov b32 $r12 $r13
+                                imm32($r11, 0x00002710)
+                                call(wait)
+
+                                nv_rd32($r13, $r14)
+                                and $r9 $r9 $r13
+
+                                add b32 $r5 1
+                                cmp b16 $r5 0x4
+                                bra l #memx_func_train_loop_4x
+
+                        add b32 $r10 $r7 #memx_train_head
+                        st b32 D[$r10 + 0] $r9
+                        add b32 $r6 1
+                        add b32 $r7 4
+
+                        cmp b16 $r6 0x10
+                        bra l #memx_func_train_loop_inner
+
+                pop $r5
+                add b32 $r5 1
+                cmp b16 $r5 7
+                bra l #memx_func_train_loop_outer
+
+#endif
+        ret
+
+// description
+//
+// $r15 - current (memx)
 // $r14 - sender process name
 // $r13 - message (exec)
 // $r12 - head of script
@@ -307,8 +407,19 @@ memx_exec:
 // $r11 - data1
 // $r0  - zero
 memx_info:
+        cmp b16 $r12 0x1
+        bra e #memx_info_train
+
+        memx_info_data:
         mov $r12 #memx_data_head
         mov $r11 #memx_data_tail - #memx_data_head
+        bra #memx_info_send
+
+        memx_info_train:
+        mov $r12 #memx_train_head
+        mov $r11 #memx_train_tail - #memx_train_head
+
+        memx_info_send:
         call(send)
         ret
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h
index 4d278a96b2bb..713e11e2953d 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h
@@ -46,8 +46,8 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x584d454d,
-        0x0000061c,
-        0x0000060e,
+        0x0000062d,
+        0x0000061f,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -68,8 +68,8 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x46524550,
-        0x00000620,
-        0x0000061e,
+        0x00000631,
+        0x0000062f,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -90,8 +90,8 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x5f433249,
-        0x00000a24,
-        0x000008cb,
+        0x00000a35,
+        0x000008dc,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -112,8 +112,8 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x54534554,
-        0x00000a45,
-        0x00000a26,
+        0x00000a56,
+        0x00000a37,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -134,8 +134,8 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x454c4449,
-        0x00000a50,
-        0x00000a4e,
+        0x00000a61,
+        0x00000a5f,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -246,13 +246,15 @@ uint32_t nv108_pwr_data[] = {
         0x00010006,
         0x00000000,
         0x0000057b,
-/* 0x03b8: memx_func_tail */
-/* 0x03b8: memx_ts_start */
+        0x00000007,
         0x00000000,
-/* 0x03bc: memx_ts_end */
+        0x000005c3,
+/* 0x03c4: memx_func_tail */
+/* 0x03c4: memx_ts_start */
         0x00000000,
-/* 0x03c0: memx_data_head */
+/* 0x03c8: memx_ts_end */
         0x00000000,
+/* 0x03cc: memx_data_head */
         0x00000000,
         0x00000000,
         0x00000000,
@@ -764,8 +766,75 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x00000000,
-/* 0x0bc0: memx_data_tail */
-/* 0x0bc0: i2c_scl_map */
+        0x00000000,
+/* 0x0bcc: memx_data_tail */
+/* 0x0bcc: memx_train_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0ccc: memx_train_tail */
+/* 0x0ccc: i2c_scl_map */
         0x00000400,
         0x00000800,
         0x00001000,
@@ -776,7 +845,7 @@ uint32_t nv108_pwr_data[] = {
         0x00020000,
         0x00040000,
         0x00080000,
-/* 0x0be8: i2c_sda_map */
+/* 0x0cf4: i2c_sda_map */
         0x00100000,
         0x00200000,
         0x00400000,
@@ -844,9 +913,6 @@ uint32_t nv108_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
 };
 
 uint32_t nv108_pwr_code[] = {
@@ -1215,10 +1281,10 @@ uint32_t nv108_pwr_code[] = {
         0xf40464f0,
         0x2c06f70b,
         0xb50066cf,
-        0x00f8ee06,
+        0x00f8f106,
 /* 0x0500: memx_func_leave */
         0x66cf2c06,
-        0xef06b500,
+        0xf206b500,
         0xe4400406,
         0x0006f607,
 /* 0x0512: memx_func_leave_wait */
@@ -1270,370 +1336,374 @@ uint32_t nv108_pwr_code[] = {
         0x9800f800,
         0x10b6001e,
         0x005d7e04,
-/* 0x05c3: memx_exec */
-        0xf900f800,
-        0xb2d0f9e0,
-/* 0x05cb: memx_exec_next */
-        0x98b2b2c1,
-        0x10b60013,
-        0xf034e704,
-        0xe033e701,
-        0x0132b601,
-        0x980c30f0,
-        0x55f9de35,
-        0x1ef412a6,
-        0xee0b98e5,
-        0xbbef0c98,
-        0xc44b02cb,
-        0x00bbcf07,
-        0xe0fcd0fc,
-        0x0002c27e,
-/* 0x0602: memx_info */
-        0xc04c00f8,
+/* 0x05c3: memx_func_train */
+        0xf800f800,
+/* 0x05c5: memx_exec */
+        0xf9e0f900,
+        0xb2c1b2d0,
+/* 0x05cd: memx_exec_next */
+        0x001398b2,
+        0xe70410b6,
+        0xe701f034,
+        0xb601e033,
+        0x30f00132,
+        0xde35980c,
+        0x12a655f9,
+        0x98e51ef4,
+        0x0c98f10b,
+        0x02cbbbf2,
+        0xcf07c44b,
+        0xd0fc00bb,
+        0xc27ee0fc,
+        0x00f80002,
+/* 0x0604: memx_info */
+        0xf401c670,
+/* 0x060a: memx_info_data */
+        0xcc4c0c0b,
         0x08004b03,
-        0x0002c27e,
-/* 0x060e: memx_recv */
-        0xd6b000f8,
-        0xb20bf401,
-        0xf400d6b0,
-        0x00f8eb0b,
-/* 0x061c: memx_init */
-/* 0x061e: perf_recv */
-        0x00f800f8,
-/* 0x0620: perf_init */
-/* 0x0622: i2c_drive_scl */
-        0x36b000f8,
-        0x0d0bf400,
-        0xf607e040,
-        0x04bd0001,
-/* 0x0632: i2c_drive_scl_lo */
-        0xe44000f8,
-        0x0001f607,
-        0x00f804bd,
-/* 0x063c: i2c_drive_sda */
-        0xf40036b0,
-        0xe0400d0b,
-        0x0002f607,
-        0x00f804bd,
-/* 0x064c: i2c_drive_sda_lo */
-        0xf607e440,
-        0x04bd0002,
-/* 0x0656: i2c_sense_scl */
-        0x32f400f8,
-        0x07c44301,
-        0xfd0033cf,
-        0x0bf40431,
-        0x0131f406,
-/* 0x0668: i2c_sense_scl_done */
-/* 0x066a: i2c_sense_sda */
-        0x32f400f8,
-        0x07c44301,
-        0xfd0033cf,
-        0x0bf40432,
-        0x0131f406,
-/* 0x067c: i2c_sense_sda_done */
-/* 0x067e: i2c_raise_scl */
-        0x40f900f8,
-        0x03089844,
-        0x06227e01,
-/* 0x0689: i2c_raise_scl_wait */
-        0x03e84e00,
-        0x00005d7e,
-        0x0006567e,
-        0xb60901f4,
-        0x1bf40142,
-/* 0x069d: i2c_raise_scl_done */
-        0xf840fcef,
-/* 0x06a1: i2c_start */
-        0x06567e00,
-        0x0d11f400,
-        0x00066a7e,
-        0xf40611f4,
-/* 0x06b2: i2c_start_rep */
-        0x00032e0e,
-        0x0006227e,
-        0x3c7e0103,
-        0x76bb0006,
-        0x0465b600,
-        0x659450f9,
-        0x0256bb04,
-        0x75fd50bd,
-        0x7e50fc04,
-        0xb600067e,
-        0x11f40464,
-/* 0x06dd: i2c_start_send */
-        0x7e00031d,
-        0x4e00063c,
-        0x5d7e1388,
-        0x00030000,
-        0x0006227e,
-        0x7e13884e,
-/* 0x06f7: i2c_start_out */
-        0xf800005d,
-/* 0x06f9: i2c_stop */
-        0x7e000300,
-        0x03000622,
-        0x063c7e00,
-        0x03e84e00,
-        0x00005d7e,
-        0x227e0103,
-        0x884e0006,
-        0x005d7e13,
+/* 0x0613: memx_info_train */
+        0x4c090ef4,
+        0x004b0bcc,
+/* 0x0619: memx_info_send */
+        0x02c27e01,
+/* 0x061f: memx_recv */
+        0xb000f800,
+        0x0bf401d6,
+        0x00d6b0a3,
+        0xf8dc0bf4,
+/* 0x062d: memx_init */
+/* 0x062f: perf_recv */
+        0xf800f800,
+/* 0x0631: perf_init */
+/* 0x0633: i2c_drive_scl */
+        0xb000f800,
+        0x0bf40036,
+        0x07e0400d,
+        0xbd0001f6,
+/* 0x0643: i2c_drive_scl_lo */
+        0x4000f804,
+        0x01f607e4,
+        0xf804bd00,
+/* 0x064d: i2c_drive_sda */
+        0x0036b000,
+        0x400d0bf4,
+        0x02f607e0,
+        0xf804bd00,
+/* 0x065d: i2c_drive_sda_lo */
+        0x07e44000,
+        0xbd0002f6,
+/* 0x0667: i2c_sense_scl */
+        0xf400f804,
+        0xc4430132,
+        0x0033cf07,
+        0xf40431fd,
+        0x31f4060b,
+/* 0x0679: i2c_sense_scl_done */
+/* 0x067b: i2c_sense_sda */
+        0xf400f801,
+        0xc4430132,
+        0x0033cf07,
+        0xf40432fd,
+        0x31f4060b,
+/* 0x068d: i2c_sense_sda_done */
+/* 0x068f: i2c_raise_scl */
+        0xf900f801,
+        0x08984440,
+        0x337e0103,
+/* 0x069a: i2c_raise_scl_wait */
+        0xe84e0006,
+        0x005d7e03,
+        0x06677e00,
+        0x0901f400,
+        0xf40142b6,
+/* 0x06ae: i2c_raise_scl_done */
+        0x40fcef1b,
+/* 0x06b2: i2c_start */
+        0x677e00f8,
+        0x11f40006,
+        0x067b7e0d,
+        0x0611f400,
+/* 0x06c3: i2c_start_rep */
+        0x032e0ef4,
+        0x06337e00,
         0x7e010300,
-        0x4e00063c,
-        0x5d7e1388,
-        0x00f80000,
-/* 0x0728: i2c_bitw */
-        0x00063c7e,
-        0x7e03e84e,
-        0xbb00005d,
+        0xbb00064d,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x00067e7e,
+        0x00068f7e,
         0xf40464b6,
-        0x884e1711,
-        0x005d7e13,
-        0x7e000300,
-        0x4e000622,
-        0x5d7e1388,
-/* 0x0766: i2c_bitw_out */
-        0x00f80000,
-/* 0x0768: i2c_bitr */
-        0x3c7e0103,
+/* 0x06ee: i2c_start_send */
+        0x00031d11,
+        0x00064d7e,
+        0x7e13884e,
+        0x0300005d,
+        0x06337e00,
+        0x13884e00,
+        0x00005d7e,
+/* 0x0708: i2c_start_out */
+/* 0x070a: i2c_stop */
+        0x000300f8,
+        0x0006337e,
+        0x4d7e0003,
         0xe84e0006,
         0x005d7e03,
-        0x0076bb00,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0x7e7e50fc,
-        0x64b60006,
-        0x1a11f404,
-        0x00066a7e,
-        0x227e0003,
-        0x884e0006,
-        0x005d7e13,
-        0x013cf000,
-/* 0x07ab: i2c_bitr_done */
-        0xf80131f4,
-/* 0x07ad: i2c_get_byte */
-        0x04000500,
-/* 0x07b1: i2c_get_byte_next */
-        0x0154b608,
+        0x7e010300,
+        0x4e000633,
+        0x5d7e1388,
+        0x01030000,
+        0x00064d7e,
+        0x7e13884e,
+        0xf800005d,
+/* 0x0739: i2c_bitw */
+        0x064d7e00,
+        0x03e84e00,
+        0x00005d7e,
         0xb60076bb,
         0x50f90465,
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0x07687e50,
+        0x068f7e50,
         0x0464b600,
-        0xfd2a11f4,
-        0x42b60553,
-        0xd81bf401,
-        0x76bb0103,
+        0x4e1711f4,
+        0x5d7e1388,
+        0x00030000,
+        0x0006337e,
+        0x7e13884e,
+/* 0x0777: i2c_bitw_out */
+        0xf800005d,
+/* 0x0779: i2c_bitr */
+        0x7e010300,
+        0x4e00064d,
+        0x5d7e03e8,
+        0x76bb0000,
         0x0465b600,
         0x659450f9,
         0x0256bb04,
         0x75fd50bd,
         0x7e50fc04,
-        0xb6000728,
-/* 0x07fa: i2c_get_byte_done */
-        0x00f80464,
-/* 0x07fc: i2c_put_byte */
-/* 0x07fe: i2c_put_byte_next */
-        0x42b60804,
-        0x3854ff01,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0x07287e50,
-        0x0464b600,
-        0xb03411f4,
-        0x1bf40046,
-        0x0076bbd8,
+        0xb600068f,
+        0x11f40464,
+        0x067b7e1a,
+        0x7e000300,
+        0x4e000633,
+        0x5d7e1388,
+        0x3cf00000,
+        0x0131f401,
+/* 0x07bc: i2c_bitr_done */
+/* 0x07be: i2c_get_byte */
+        0x000500f8,
+/* 0x07c2: i2c_get_byte_next */
+        0x54b60804,
+        0x0076bb01,
+        0xf90465b6,
+        0x04659450,
+        0xbd0256bb,
+        0x0475fd50,
+        0x797e50fc,
+        0x64b60007,
+        0x2a11f404,
+        0xb60553fd,
+        0x1bf40142,
+        0xbb0103d8,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x0007397e,
+/* 0x080b: i2c_get_byte_done */
+        0xf80464b6,
+/* 0x080d: i2c_put_byte */
+/* 0x080f: i2c_put_byte_next */
+        0xb6080400,
+        0x54ff0142,
+        0x0076bb38,
         0xf90465b6,
         0x04659450,
         0xbd0256bb,
         0x0475fd50,
-        0x687e50fc,
+        0x397e50fc,
         0x64b60007,
-        0x0f11f404,
-        0xb00076bb,
-        0x1bf40136,
-        0x0132f406,
-/* 0x0854: i2c_put_byte_done */
-/* 0x0856: i2c_addr */
-        0x76bb00f8,
+        0x3411f404,
+        0xf40046b0,
+        0x76bbd81b,
         0x0465b600,
         0x659450f9,
         0x0256bb04,
         0x75fd50bd,
         0x7e50fc04,
-        0xb60006a1,
+        0xb6000779,
         0x11f40464,
-        0x2ec3e729,
-        0x0134b601,
-        0xbb0553fd,
+        0x0076bb0f,
+        0xf40136b0,
+        0x32f4061b,
+/* 0x0865: i2c_put_byte_done */
+/* 0x0867: i2c_addr */
+        0xbb00f801,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x0007fc7e,
-/* 0x089b: i2c_addr_done */
-        0xf80464b6,
-/* 0x089d: i2c_acquire_addr */
-        0xf8cec700,
-        0xb705e4b6,
-        0xf8d014e0,
-/* 0x08a9: i2c_acquire */
-        0x089d7e00,
-        0x00047e00,
-        0x03d9f000,
-        0x00002e7e,
-/* 0x08ba: i2c_release */
-        0x9d7e00f8,
+        0x0006b27e,
+        0xf40464b6,
+        0xc3e72911,
+        0x34b6012e,
+        0x0553fd01,
+        0xb60076bb,
+        0x50f90465,
+        0xbb046594,
+        0x50bd0256,
+        0xfc0475fd,
+        0x080d7e50,
+        0x0464b600,
+/* 0x08ac: i2c_addr_done */
+/* 0x08ae: i2c_acquire_addr */
+        0xcec700f8,
+        0x05e4b6f8,
+        0xd014e0b7,
+/* 0x08ba: i2c_acquire */
+        0xae7e00f8,
         0x047e0008,
-        0xdaf00000,
+        0xd9f00000,
         0x002e7e03,
-/* 0x08cb: i2c_recv */
-        0xf400f800,
-        0xc1c70132,
-        0x0214b6f8,
-        0xf52816b0,
-        0xb801371f,
-        0x000be813,
-        0xb8003298,
-        0x000bc013,
-        0xf4003198,
-        0xd0f90231,
-        0xd0f9e0f9,
-        0x000067f1,
-        0x100063f1,
-        0xbb016792,
+/* 0x08cb: i2c_release */
+        0x7e00f800,
+        0x7e0008ae,
+        0xf0000004,
+        0x2e7e03da,
+        0x00f80000,
+/* 0x08dc: i2c_recv */
+        0xc70132f4,
+        0x14b6f8c1,
+        0x2816b002,
+        0x01371ff5,
+        0x0cf413b8,
+        0x00329800,
+        0x0ccc13b8,
+        0x00319800,
+        0xf90231f4,
+        0xf9e0f9d0,
+        0x0067f1d0,
+        0x0063f100,
+        0x01679210,
+        0xb60076bb,
+        0x50f90465,
+        0xbb046594,
+        0x50bd0256,
+        0xfc0475fd,
+        0x08ba7e50,
+        0x0464b600,
+        0xd6b0d0fc,
+        0xb01bf500,
+        0xbb000500,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x0008a97e,
-        0xfc0464b6,
-        0x00d6b0d0,
-        0x00b01bf5,
-        0x76bb0005,
+        0x0008677e,
+        0xf50464b6,
+        0xc700cc11,
+        0x76bbe0c5,
         0x0465b600,
         0x659450f9,
         0x0256bb04,
         0x75fd50bd,
         0x7e50fc04,
-        0xb6000856,
+        0xb600080d,
         0x11f50464,
-        0xc5c700cc,
-        0x0076bbe0,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0xfc7e50fc,
-        0x64b60007,
-        0xa911f504,
-        0xbb010500,
-        0x65b60076,
-        0x9450f904,
-        0x56bb0465,
-        0xfd50bd02,
-        0x50fc0475,
-        0x0008567e,
-        0xf50464b6,
-        0xbb008711,
-        0x65b60076,
-        0x9450f904,
-        0x56bb0465,
-        0xfd50bd02,
-        0x50fc0475,
-        0x0007ad7e,
-        0xf40464b6,
-        0x5bcb6711,
-        0x0076bbe0,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0xf97e50fc,
-        0x64b60006,
-        0xbd5bb204,
-        0x410ef474,
-/* 0x09d0: i2c_recv_not_rd08 */
-        0xf401d6b0,
-        0x00053b1b,
-        0x0008567e,
-        0xc73211f4,
-        0xfc7ee0c5,
-        0x11f40007,
-        0x7e000528,
-        0xf4000856,
-        0xb5c71f11,
-        0x07fc7ee0,
-        0x1511f400,
-        0x0006f97e,
-        0xc5c774bd,
-        0x091bf408,
-        0xf40232f4,
-/* 0x0a0e: i2c_recv_not_wr08 */
-/* 0x0a0e: i2c_recv_done */
-        0xcec7030e,
-        0x08ba7ef8,
-        0xfce0fc00,
-        0x0912f4d0,
-        0xc27e7cb2,
-/* 0x0a22: i2c_recv_exit */
-        0x00f80002,
-/* 0x0a24: i2c_init */
-/* 0x0a26: test_recv */
-        0x584100f8,
-        0x0011cf04,
-        0x400110b6,
-        0x01f60458,
-        0xf104bd00,
-        0xf1d900e7,
-        0x7e134fe3,
-        0xf8000201,
-/* 0x0a45: test_init */
-        0x08004e00,
-        0x0002017e,
-/* 0x0a4e: idle_recv */
-        0x00f800f8,
-/* 0x0a50: idle */
-        0x410031f4,
-        0x11cf0454,
+        0x010500a9,
+        0xb60076bb,
+        0x50f90465,
+        0xbb046594,
+        0x50bd0256,
+        0xfc0475fd,
+        0x08677e50,
+        0x0464b600,
+        0x008711f5,
+        0xb60076bb,
+        0x50f90465,
+        0xbb046594,
+        0x50bd0256,
+        0xfc0475fd,
+        0x07be7e50,
+        0x0464b600,
+        0xcb6711f4,
+        0x76bbe05b,
+        0x0465b600,
+        0x659450f9,
+        0x0256bb04,
+        0x75fd50bd,
+        0x7e50fc04,
+        0xb600070a,
+        0x5bb20464,
+        0x0ef474bd,
+/* 0x09e1: i2c_recv_not_rd08 */
+        0x01d6b041,
+        0x053b1bf4,
+        0x08677e00,
+        0x3211f400,
+        0x7ee0c5c7,
+        0xf400080d,
+        0x00052811,
+        0x0008677e,
+        0xc71f11f4,
+        0x0d7ee0b5,
+        0x11f40008,
+        0x070a7e15,
+        0xc774bd00,
+        0x1bf408c5,
+        0x0232f409,
+/* 0x0a1f: i2c_recv_not_wr08 */
+/* 0x0a1f: i2c_recv_done */
+        0xc7030ef4,
+        0xcb7ef8ce,
+        0xe0fc0008,
+        0x12f4d0fc,
+        0x7e7cb209,
+/* 0x0a33: i2c_recv_exit */
+        0xf80002c2,
+/* 0x0a35: i2c_init */
+/* 0x0a37: test_recv */
+        0x4100f800,
+        0x11cf0458,
         0x0110b600,
-        0xf6045440,
+        0xf6045840,
         0x04bd0001,
-/* 0x0a64: idle_loop */
-        0x32f45801,
-/* 0x0a69: idle_proc */
-/* 0x0a69: idle_proc_exec */
-        0xb210f902,
-        0x02cb7e1e,
-        0xf410fc00,
-        0x31f40911,
-        0xf00ef402,
-/* 0x0a7c: idle_proc_next */
-        0xa65810b6,
-        0xe81bf41f,
-        0xf4e002f4,
-        0x0ef40028,
-        0x000000c6,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
+        0xd900e7f1,
+        0x134fe3f1,
+        0x0002017e,
+/* 0x0a56: test_init */
+        0x004e00f8,
+        0x02017e08,
+/* 0x0a5f: idle_recv */
+        0xf800f800,
+/* 0x0a61: idle */
+        0x0031f400,
+        0xcf045441,
+        0x10b60011,
+        0x04544001,
+        0xbd0001f6,
+/* 0x0a75: idle_loop */
+        0xf4580104,
+/* 0x0a7a: idle_proc */
+/* 0x0a7a: idle_proc_exec */
+        0x10f90232,
+        0xcb7e1eb2,
+        0x10fc0002,
+        0xf40911f4,
+        0x0ef40231,
+/* 0x0a8d: idle_proc_next */
+        0x5810b6f0,
+        0x1bf41fa6,
+        0xe002f4e8,
+        0xf40028f4,
+        0x0000c60e,
         0x00000000,
         0x00000000,
         0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h
index 64e97baabc3c..d1f9b6cb66d7 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h
@@ -46,8 +46,8 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x584d454d,
-        0x000006e0,
-        0x000006d2,
+        0x00000842,
+        0x00000834,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -68,8 +68,8 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x46524550,
-        0x000006e4,
-        0x000006e2,
+        0x00000846,
+        0x00000844,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -90,8 +90,8 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x5f433249,
-        0x00000b14,
-        0x000009b7,
+        0x00000c76,
+        0x00000b19,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -112,8 +112,8 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x54534554,
-        0x00000b3d,
-        0x00000b16,
+        0x00000c9f,
+        0x00000c78,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -134,8 +134,8 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x454c4449,
-        0x00000b49,
-        0x00000b47,
+        0x00000cab,
+        0x00000ca9,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -246,13 +246,15 @@ uint32_t nva3_pwr_data[] = {
         0x00010006,
         0x00000000,
         0x000005f8,
-/* 0x03b8: memx_func_tail */
-/* 0x03b8: memx_ts_start */
+        0x00000007,
         0x00000000,
-/* 0x03bc: memx_ts_end */
+        0x0000067e,
+/* 0x03c4: memx_func_tail */
+/* 0x03c4: memx_ts_start */
         0x00000000,
-/* 0x03c0: memx_data_head */
+/* 0x03c8: memx_ts_end */
         0x00000000,
+/* 0x03cc: memx_data_head */
         0x00000000,
         0x00000000,
         0x00000000,
@@ -764,8 +766,75 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x00000000,
-/* 0x0bc0: memx_data_tail */
-/* 0x0bc0: i2c_scl_map */
+        0x00000000,
+/* 0x0bcc: memx_data_tail */
+/* 0x0bcc: memx_train_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0ccc: memx_train_tail */
+/* 0x0ccc: i2c_scl_map */
         0x00001000,
         0x00004000,
         0x00010000,
@@ -776,7 +845,7 @@ uint32_t nva3_pwr_data[] = {
         0x01000000,
         0x04000000,
         0x10000000,
-/* 0x0be8: i2c_sda_map */
+/* 0x0cf4: i2c_sda_map */
         0x00002000,
         0x00008000,
         0x00020000,
@@ -787,7 +856,7 @@ uint32_t nva3_pwr_data[] = {
         0x02000000,
         0x08000000,
         0x20000000,
-/* 0x0c10: i2c_ctrl */
+/* 0x0d1c: i2c_ctrl */
         0x0000e138,
         0x0000e150,
         0x0000e168,
@@ -845,9 +914,6 @@ uint32_t nva3_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
 };
 
 uint32_t nva3_pwr_code[] = {
@@ -1258,11 +1324,11 @@ uint32_t nva3_pwr_code[] = {
         0x67f0f30b,
         0x0664b62c,
         0x800066cf,
-        0x00f8ee06,
+        0x00f8f106,
 /* 0x05a8: memx_func_leave */
         0xb62c67f0,
         0x66cf0664,
-        0xef068000,
+        0xf2068000,
         0xf10467f0,
         0xb607e407,
         0x06d00604,
@@ -1323,408 +1389,479 @@ uint32_t nva3_pwr_code[] = {
         0x9800f8a4,
         0x10b6001e,
         0x7f21f404,
-/* 0x067e: memx_exec */
-        0xe0f900f8,
-        0xc1b9d0f9,
-        0x02b2b902,
-/* 0x0688: memx_exec_next */
-        0xb6001398,
-        0x34e70410,
-        0x33e701f0,
-        0x32b601e0,
-        0x0c30f001,
-        0xf9de3598,
-        0x0612b855,
-        0x98e41ef4,
-        0x0c98ee0b,
-        0x02cbbbef,
-        0x07c4b7f1,
-        0xcf06b4b6,
-        0xd0fc00bb,
-        0x21f5e0fc,
+/* 0x067e: memx_func_train */
+        0x57f100f8,
+        0x77f10003,
+        0x97f10000,
+        0x93f00000,
+        0x029eb970,
+        0xb90421f4,
+        0xe7f102d8,
+        0x21f42710,
+/* 0x069d: memx_func_train_loop_outer */
+        0x0158e07f,
+        0x0083f101,
+        0xe097f102,
+        0x1193f011,
+        0x80f990f9,
+        0xe0fcd0fc,
+        0xf93f21f4,
+        0x0067f150,
+/* 0x06bd: memx_func_train_loop_inner */
+        0x1187f100,
+        0x9068ff11,
+        0xfd109894,
+        0x97f10589,
+        0x93f00720,
+        0xf990f910,
+        0xfcd0fc80,
+        0x3f21f4e0,
+        0x008097f1,
+        0xb91093f0,
+        0x21f4029e,
+        0x02d8b904,
+        0xf92088c5,
+        0xfc80f990,
+        0xf4e0fcd0,
+        0x97f13f21,
+        0x93f0053c,
+        0x0287f110,
+        0x0083f130,
+        0xf990f980,
+        0xfcd0fc80,
+        0x3f21f4e0,
+        0x0560e7f1,
+        0xf110e3f0,
+        0xf10000d7,
+        0x908000d3,
+        0xb7f100dc,
+        0xb3f08480,
+        0xa421f41e,
+        0x000057f1,
+        0xffff97f1,
+        0x830093f1,
+/* 0x073c: memx_func_train_loop_4x */
+        0x0080a7f1,
+        0xb910a3f0,
+        0x21f402ae,
+        0x02d8b904,
+        0xffdfb7f1,
+        0xffffb3f1,
+        0xf9048bfd,
+        0xfc80f9a0,
+        0xf4e0fcd0,
+        0xa7f13f21,
+        0xa3f0053c,
+        0x0287f110,
+        0x0083f130,
+        0xf9a0f980,
+        0xfcd0fc80,
+        0x3f21f4e0,
+        0x0560e7f1,
+        0xf110e3f0,
+        0xf10000d7,
+        0xb98000d3,
+        0xb7f102dc,
+        0xb3f02710,
+        0xa421f400,
+        0xf402eeb9,
+        0xddb90421,
+        0x949dff02,
+        0x700150b6,
+        0x1ef40456,
+        0xcc7aa092,
+        0x00a9800b,
+        0xb60160b6,
+        0x66700470,
+        0x001ef510,
+        0xb650fcff,
+        0x56700150,
+        0xd41ef507,
+/* 0x07cf: memx_exec */
+        0xf900f8fe,
+        0xb9d0f9e0,
+        0xb2b902c1,
+/* 0x07d9: memx_exec_next */
+        0x00139802,
+        0xe70410b6,
+        0xe701f034,
+        0xb601e033,
+        0x30f00132,
+        0xde35980c,
+        0x12b855f9,
+        0xe41ef406,
+        0x98f10b98,
+        0xcbbbf20c,
+        0xc4b7f102,
+        0x06b4b607,
+        0xfc00bbcf,
+        0xf5e0fcd0,
+        0xf8034221,
+/* 0x0815: memx_info */
+        0x01c67000,
+/* 0x081b: memx_info_data */
+        0xf10e0bf4,
+        0xf103ccc7,
+        0xf40800b7,
+/* 0x0826: memx_info_train */
+        0xc7f10b0e,
+        0xb7f10bcc,
+/* 0x082e: memx_info_send */
+        0x21f50100,
         0x00f80342,
-/* 0x06c4: memx_info */
-        0x03c0c7f1,
-        0x0800b7f1,
-        0x034221f5,
-/* 0x06d2: memx_recv */
-        0xd6b000f8,
-        0xa90bf401,
-        0xf400d6b0,
-        0x00f8e90b,
-/* 0x06e0: memx_init */
-/* 0x06e2: perf_recv */
+/* 0x0834: memx_recv */
+        0xf401d6b0,
+        0xd6b0980b,
+        0xd80bf400,
+/* 0x0842: memx_init */
+        0x00f800f8,
+/* 0x0844: perf_recv */
+/* 0x0846: perf_init */
         0x00f800f8,
-/* 0x06e4: perf_init */
-/* 0x06e6: i2c_drive_scl */
+/* 0x0848: i2c_drive_scl */
+        0xf40036b0,
+        0x07f1110b,
+        0x04b607e0,
+        0x0001d006,
+        0x00f804bd,
+/* 0x085c: i2c_drive_scl_lo */
+        0x07e407f1,
+        0xd00604b6,
+        0x04bd0001,
+/* 0x086a: i2c_drive_sda */
         0x36b000f8,
         0x110bf400,
         0x07e007f1,
         0xd00604b6,
-        0x04bd0001,
-/* 0x06fa: i2c_drive_scl_lo */
+        0x04bd0002,
+/* 0x087e: i2c_drive_sda_lo */
         0x07f100f8,
         0x04b607e4,
-        0x0001d006,
-        0x00f804bd,
-/* 0x0708: i2c_drive_sda */
-        0xf40036b0,
-        0x07f1110b,
-        0x04b607e0,
         0x0002d006,
         0x00f804bd,
-/* 0x071c: i2c_drive_sda_lo */
-        0x07e407f1,
-        0xd00604b6,
-        0x04bd0002,
-/* 0x072a: i2c_sense_scl */
-        0x32f400f8,
-        0xc437f101,
-        0x0634b607,
-        0xfd0033cf,
-        0x0bf40431,
-        0x0131f406,
-/* 0x0740: i2c_sense_scl_done */
-/* 0x0742: i2c_sense_sda */
-        0x32f400f8,
-        0xc437f101,
-        0x0634b607,
-        0xfd0033cf,
-        0x0bf40432,
-        0x0131f406,
-/* 0x0758: i2c_sense_sda_done */
-/* 0x075a: i2c_raise_scl */
-        0x40f900f8,
-        0x089847f1,
-        0xf50137f0,
-/* 0x0767: i2c_raise_scl_wait */
-        0xf106e621,
-        0xf403e8e7,
-        0x21f57f21,
-        0x01f4072a,
-        0x0142b609,
-/* 0x077b: i2c_raise_scl_done */
-        0xfcef1bf4,
-/* 0x077f: i2c_start */
-        0xf500f840,
-        0xf4072a21,
-        0x21f50d11,
-        0x11f40742,
-        0x300ef406,
-/* 0x0790: i2c_start_rep */
-        0xf50037f0,
-        0xf006e621,
-        0x21f50137,
-        0x76bb0708,
-        0x0465b600,
-        0x659450f9,
-        0x0256bb04,
-        0x75fd50bd,
-        0xf550fc04,
-        0xb6075a21,
-        0x11f40464,
-/* 0x07bd: i2c_start_send */
-        0x0037f01f,
-        0x070821f5,
-        0x1388e7f1,
-        0xf07f21f4,
-        0x21f50037,
-        0xe7f106e6,
-        0x21f41388,
-/* 0x07d9: i2c_start_out */
-/* 0x07db: i2c_stop */
-        0xf000f87f,
-        0x21f50037,
-        0x37f006e6,
-        0x0821f500,
-        0xe8e7f107,
+/* 0x088c: i2c_sense_scl */
+        0xf10132f4,
+        0xb607c437,
+        0x33cf0634,
+        0x0431fd00,
+        0xf4060bf4,
+/* 0x08a2: i2c_sense_scl_done */
+        0x00f80131,
+/* 0x08a4: i2c_sense_sda */
+        0xf10132f4,
+        0xb607c437,
+        0x33cf0634,
+        0x0432fd00,
+        0xf4060bf4,
+/* 0x08ba: i2c_sense_sda_done */
+        0x00f80131,
+/* 0x08bc: i2c_raise_scl */
+        0x47f140f9,
+        0x37f00898,
+        0x4821f501,
+/* 0x08c9: i2c_raise_scl_wait */
+        0xe8e7f108,
         0x7f21f403,
-        0xf50137f0,
-        0xf106e621,
-        0xf41388e7,
-        0x37f07f21,
-        0x0821f501,
-        0x88e7f107,
-        0x7f21f413,
-/* 0x080e: i2c_bitw */
-        0x21f500f8,
-        0xe7f10708,
-        0x21f403e8,
-        0x0076bb7f,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0x21f550fc,
-        0x64b6075a,
-        0x1811f404,
-        0x1388e7f1,
-        0xf07f21f4,
+        0x088c21f5,
+        0xb60901f4,
+        0x1bf40142,
+/* 0x08dd: i2c_raise_scl_done */
+        0xf840fcef,
+/* 0x08e1: i2c_start */
+        0x8c21f500,
+        0x0d11f408,
+        0x08a421f5,
+        0xf40611f4,
+/* 0x08f2: i2c_start_rep */
+        0x37f0300e,
+        0x4821f500,
+        0x0137f008,
+        0x086a21f5,
+        0xb60076bb,
+        0x50f90465,
+        0xbb046594,
+        0x50bd0256,
+        0xfc0475fd,
+        0xbc21f550,
+        0x0464b608,
+/* 0x091f: i2c_start_send */
+        0xf01f11f4,
         0x21f50037,
-        0xe7f106e6,
+        0xe7f1086a,
         0x21f41388,
-/* 0x084d: i2c_bitw_out */
-/* 0x084f: i2c_bitr */
-        0xf000f87f,
-        0x21f50137,
-        0xe7f10708,
-        0x21f403e8,
-        0x0076bb7f,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0x21f550fc,
-        0x64b6075a,
-        0x1b11f404,
-        0x074221f5,
+        0x0037f07f,
+        0x084821f5,
+        0x1388e7f1,
+/* 0x093b: i2c_start_out */
+        0xf87f21f4,
+/* 0x093d: i2c_stop */
+        0x0037f000,
+        0x084821f5,
         0xf50037f0,
-        0xf106e621,
+        0xf1086a21,
+        0xf403e8e7,
+        0x37f07f21,
+        0x4821f501,
+        0x88e7f108,
+        0x7f21f413,
+        0xf50137f0,
+        0xf1086a21,
         0xf41388e7,
-        0x3cf07f21,
-        0x0131f401,
-/* 0x0894: i2c_bitr_done */
-/* 0x0896: i2c_get_byte */
-        0x57f000f8,
-        0x0847f000,
-/* 0x089c: i2c_get_byte_next */
-        0xbb0154b6,
+        0x00f87f21,
+/* 0x0970: i2c_bitw */
+        0x086a21f5,
+        0x03e8e7f1,
+        0xbb7f21f4,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x084f21f5,
+        0x08bc21f5,
         0xf40464b6,
-        0x53fd2b11,
-        0x0142b605,
-        0xf0d81bf4,
-        0x76bb0137,
-        0x0465b600,
-        0x659450f9,
-        0x0256bb04,
-        0x75fd50bd,
-        0xf550fc04,
-        0xb6080e21,
-/* 0x08e6: i2c_get_byte_done */
-        0x00f80464,
-/* 0x08e8: i2c_put_byte */
-/* 0x08eb: i2c_put_byte_next */
-        0xb60847f0,
-        0x54ff0142,
-        0x0076bb38,
+        0xe7f11811,
+        0x21f41388,
+        0x0037f07f,
+        0x084821f5,
+        0x1388e7f1,
+/* 0x09af: i2c_bitw_out */
+        0xf87f21f4,
+/* 0x09b1: i2c_bitr */
+        0x0137f000,
+        0x086a21f5,
+        0x03e8e7f1,
+        0xbb7f21f4,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x08bc21f5,
+        0xf40464b6,
+        0x21f51b11,
+        0x37f008a4,
+        0x4821f500,
+        0x88e7f108,
+        0x7f21f413,
+        0xf4013cf0,
+/* 0x09f6: i2c_bitr_done */
+        0x00f80131,
+/* 0x09f8: i2c_get_byte */
+        0xf00057f0,
+/* 0x09fe: i2c_get_byte_next */
+        0x54b60847,
+        0x0076bb01,
         0xf90465b6,
         0x04659450,
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b6080e,
-        0x3411f404,
-        0xf40046b0,
-        0x76bbd81b,
-        0x0465b600,
-        0x659450f9,
-        0x0256bb04,
-        0x75fd50bd,
-        0xf550fc04,
-        0xb6084f21,
-        0x11f40464,
-        0x0076bb0f,
-        0xf40136b0,
-        0x32f4061b,
-/* 0x0941: i2c_put_byte_done */
-/* 0x0943: i2c_addr */
-        0xbb00f801,
+        0x64b609b1,
+        0x2b11f404,
+        0xb60553fd,
+        0x1bf40142,
+        0x0137f0d8,
+        0xb60076bb,
+        0x50f90465,
+        0xbb046594,
+        0x50bd0256,
+        0xfc0475fd,
+        0x7021f550,
+        0x0464b609,
+/* 0x0a48: i2c_get_byte_done */
+/* 0x0a4a: i2c_put_byte */
+        0x47f000f8,
+/* 0x0a4d: i2c_put_byte_next */
+        0x0142b608,
+        0xbb3854ff,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x077f21f5,
+        0x097021f5,
         0xf40464b6,
-        0xc3e72911,
-        0x34b6012e,
-        0x0553fd01,
+        0x46b03411,
+        0xd81bf400,
         0xb60076bb,
         0x50f90465,
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0xe821f550,
-        0x0464b608,
-/* 0x0988: i2c_addr_done */
-/* 0x098a: i2c_acquire_addr */
-        0xcec700f8,
-        0x02e4b6f8,
-        0x0c10e0b7,
-        0xf800ee98,
-/* 0x0999: i2c_acquire */
-        0x8a21f500,
-        0x0421f409,
-        0xf403d9f0,
-        0x00f83f21,
-/* 0x09a8: i2c_release */
-        0x098a21f5,
-        0xf00421f4,
-        0x21f403da,
-/* 0x09b7: i2c_recv */
-        0xf400f83f,
-        0xc1c70132,
-        0x0214b6f8,
-        0xf52816b0,
-        0xa0013a1f,
-        0x980be813,
-        0x13a00032,
-        0x31980bc0,
-        0x0231f400,
-        0xe0f9d0f9,
-        0x67f1d0f9,
-        0x63f10000,
-        0x67921000,
-        0x0076bb01,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0x21f550fc,
-        0x64b60999,
-        0xb0d0fc04,
-        0x1bf500d6,
-        0x57f000b3,
+        0xb121f550,
+        0x0464b609,
+        0xbb0f11f4,
+        0x36b00076,
+        0x061bf401,
+/* 0x0aa3: i2c_put_byte_done */
+        0xf80132f4,
+/* 0x0aa5: i2c_addr */
         0x0076bb00,
         0xf90465b6,
         0x04659450,
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b60943,
-        0xd011f504,
-        0xe0c5c700,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0xe821f550,
-        0x0464b608,
-        0x00ad11f5,
-        0xbb0157f0,
+        0x64b608e1,
+        0x2911f404,
+        0x012ec3e7,
+        0xfd0134b6,
+        0x76bb0553,
+        0x0465b600,
+        0x659450f9,
+        0x0256bb04,
+        0x75fd50bd,
+        0xf550fc04,
+        0xb60a4a21,
+/* 0x0aea: i2c_addr_done */
+        0x00f80464,
+/* 0x0aec: i2c_acquire_addr */
+        0xb6f8cec7,
+        0xe0b702e4,
+        0xee980d1c,
+/* 0x0afb: i2c_acquire */
+        0xf500f800,
+        0xf40aec21,
+        0xd9f00421,
+        0x3f21f403,
+/* 0x0b0a: i2c_release */
+        0x21f500f8,
+        0x21f40aec,
+        0x03daf004,
+        0xf83f21f4,
+/* 0x0b19: i2c_recv */
+        0x0132f400,
+        0xb6f8c1c7,
+        0x16b00214,
+        0x3a1ff528,
+        0xf413a001,
+        0x0032980c,
+        0x0ccc13a0,
+        0xf4003198,
+        0xd0f90231,
+        0xd0f9e0f9,
+        0x000067f1,
+        0x100063f1,
+        0xbb016792,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x094321f5,
-        0xf50464b6,
-        0xbb008a11,
+        0x0afb21f5,
+        0xfc0464b6,
+        0x00d6b0d0,
+        0x00b31bf5,
+        0xbb0057f0,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x089621f5,
-        0xf40464b6,
-        0x5bcb6a11,
-        0x0076bbe0,
+        0x0aa521f5,
+        0xf50464b6,
+        0xc700d011,
+        0x76bbe0c5,
+        0x0465b600,
+        0x659450f9,
+        0x0256bb04,
+        0x75fd50bd,
+        0xf550fc04,
+        0xb60a4a21,
+        0x11f50464,
+        0x57f000ad,
+        0x0076bb01,
         0xf90465b6,
         0x04659450,
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b607db,
-        0x025bb904,
-        0x0ef474bd,
-/* 0x0abd: i2c_recv_not_rd08 */
-        0x01d6b043,
-        0xf03d1bf4,
-        0x21f50057,
-        0x11f40943,
-        0xe0c5c733,
-        0x08e821f5,
-        0xf02911f4,
-        0x21f50057,
-        0x11f40943,
-        0xe0b5c71f,
-        0x08e821f5,
-        0xf51511f4,
-        0xbd07db21,
-        0x08c5c774,
-        0xf4091bf4,
-        0x0ef40232,
-/* 0x0afd: i2c_recv_not_wr08 */
-/* 0x0afd: i2c_recv_done */
-        0xf8cec703,
-        0x09a821f5,
-        0xd0fce0fc,
-        0xb90a12f4,
-        0x21f5027c,
-/* 0x0b12: i2c_recv_exit */
-        0x00f80342,
-/* 0x0b14: i2c_init */
-/* 0x0b16: test_recv */
-        0x17f100f8,
-        0x14b605d8,
-        0x0011cf06,
-        0xf10110b6,
-        0xb605d807,
-        0x01d00604,
-        0xf104bd00,
-        0xf1d900e7,
-        0xf5134fe3,
-        0xf8026221,
-/* 0x0b3d: test_init */
-        0x00e7f100,
-        0x6221f508,
-/* 0x0b47: idle_recv */
-        0xf800f802,
-/* 0x0b49: idle */
-        0x0031f400,
-        0x05d417f1,
+        0x64b60aa5,
+        0x8a11f504,
+        0x0076bb00,
+        0xf90465b6,
+        0x04659450,
+        0xbd0256bb,
+        0x0475fd50,
+        0x21f550fc,
+        0x64b609f8,
+        0x6a11f404,
+        0xbbe05bcb,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x093d21f5,
+        0xb90464b6,
+        0x74bd025b,
+/* 0x0c1f: i2c_recv_not_rd08 */
+        0xb0430ef4,
+        0x1bf401d6,
+        0x0057f03d,
+        0x0aa521f5,
+        0xc73311f4,
+        0x21f5e0c5,
+        0x11f40a4a,
+        0x0057f029,
+        0x0aa521f5,
+        0xc71f11f4,
+        0x21f5e0b5,
+        0x11f40a4a,
+        0x3d21f515,
+        0xc774bd09,
+        0x1bf408c5,
+        0x0232f409,
+/* 0x0c5f: i2c_recv_not_wr08 */
+/* 0x0c5f: i2c_recv_done */
+        0xc7030ef4,
+        0x21f5f8ce,
+        0xe0fc0b0a,
+        0x12f4d0fc,
+        0x027cb90a,
+        0x034221f5,
+/* 0x0c74: i2c_recv_exit */
+/* 0x0c76: i2c_init */
+        0x00f800f8,
+/* 0x0c78: test_recv */
+        0x05d817f1,
         0xcf0614b6,
         0x10b60011,
-        0xd407f101,
+        0xd807f101,
         0x0604b605,
         0xbd0001d0,
-/* 0x0b65: idle_loop */
-        0x5817f004,
-/* 0x0b6b: idle_proc */
-/* 0x0b6b: idle_proc_exec */
-        0xf90232f4,
-        0x021eb910,
-        0x034b21f5,
-        0x11f410fc,
-        0x0231f409,
-/* 0x0b7f: idle_proc_next */
-        0xb6ef0ef4,
-        0x1fb85810,
-        0xe61bf406,
-        0xf4dd02f4,
-        0x0ef40028,
-        0x000000bb,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
+        0x00e7f104,
+        0x4fe3f1d9,
+        0x6221f513,
+/* 0x0c9f: test_init */
+        0xf100f802,
+        0xf50800e7,
+        0xf8026221,
+/* 0x0ca9: idle_recv */
+/* 0x0cab: idle */
+        0xf400f800,
+        0x17f10031,
+        0x14b605d4,
+        0x0011cf06,
+        0xf10110b6,
+        0xb605d407,
+        0x01d00604,
+/* 0x0cc7: idle_loop */
+        0xf004bd00,
+        0x32f45817,
+/* 0x0ccd: idle_proc */
+/* 0x0ccd: idle_proc_exec */
+        0xb910f902,
+        0x21f5021e,
+        0x10fc034b,
+        0xf40911f4,
+        0x0ef40231,
+/* 0x0ce1: idle_proc_next */
+        0x5810b6ef,
+        0xf4061fb8,
+        0x02f4e61b,
+        0x0028f4dd,
+        0x00bb0ef4,
         0x00000000,
         0x00000000,
         0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h
index ca30fa4011b5..90221d973f84 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h
@@ -46,8 +46,8 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x584d454d,
-        0x0000074b,
-        0x0000073d,
+        0x0000075e,
+        0x00000750,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -68,8 +68,8 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x46524550,
-        0x0000074f,
-        0x0000074d,
+        0x00000762,
+        0x00000760,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -90,8 +90,8 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x5f433249,
-        0x00000b7f,
-        0x00000a22,
+        0x00000b92,
+        0x00000a35,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -112,8 +112,8 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x54534554,
-        0x00000ba8,
-        0x00000b81,
+        0x00000bbb,
+        0x00000b94,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -134,8 +134,8 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x454c4449,
-        0x00000bb4,
-        0x00000bb2,
+        0x00000bc7,
+        0x00000bc5,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -246,13 +246,15 @@ uint32_t nvc0_pwr_data[] = {
         0x00010006,
         0x00000000,
         0x00000663,
-/* 0x03b8: memx_func_tail */
-/* 0x03b8: memx_ts_start */
+        0x00000007,
         0x00000000,
-/* 0x03bc: memx_ts_end */
+        0x000006e9,
+/* 0x03c4: memx_func_tail */
+/* 0x03c4: memx_ts_start */
         0x00000000,
-/* 0x03c0: memx_data_head */
+/* 0x03c8: memx_ts_end */
         0x00000000,
+/* 0x03cc: memx_data_head */
         0x00000000,
         0x00000000,
         0x00000000,
@@ -764,8 +766,75 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x00000000,
-/* 0x0bc0: memx_data_tail */
-/* 0x0bc0: i2c_scl_map */
+        0x00000000,
+/* 0x0bcc: memx_data_tail */
+/* 0x0bcc: memx_train_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0ccc: memx_train_tail */
+/* 0x0ccc: i2c_scl_map */
         0x00001000,
         0x00004000,
         0x00010000,
@@ -776,7 +845,7 @@ uint32_t nvc0_pwr_data[] = {
         0x01000000,
         0x04000000,
         0x10000000,
-/* 0x0be8: i2c_sda_map */
+/* 0x0cf4: i2c_sda_map */
         0x00002000,
         0x00008000,
         0x00020000,
@@ -787,7 +856,7 @@ uint32_t nvc0_pwr_data[] = {
         0x02000000,
         0x08000000,
         0x20000000,
-/* 0x0c10: i2c_ctrl */
+/* 0x0d1c: i2c_ctrl */
         0x0000e138,
         0x0000e150,
         0x0000e168,
@@ -845,9 +914,6 @@ uint32_t nvc0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
 };
 
 uint32_t nvc0_pwr_code[] = {
@@ -1272,10 +1338,10 @@ uint32_t nvc0_pwr_code[] = {
         0xcf0664b6,
         0x06800066,
 /* 0x05db: memx_func_leave */
-        0xf000f8ee,
+        0xf000f8f1,
         0x64b62c67,
         0x0066cf06,
-        0xf0ef0680,
+        0xf0f20680,
         0x07f10467,
         0x04b607e4,
         0x0006d006,
@@ -1350,382 +1416,450 @@ uint32_t nvc0_pwr_code[] = {
         0x1e9800f8,
         0x0410b600,
         0xf87f21f4,
-/* 0x06e9: memx_exec */
-        0xf9e0f900,
-        0x02c1b9d0,
-/* 0x06f3: memx_exec_next */
-        0x9802b2b9,
-        0x10b60013,
-        0xf034e704,
-        0xe033e701,
-        0x0132b601,
-        0x980c30f0,
-        0x55f9de35,
-        0xf40612b8,
-        0x0b98e41e,
-        0xef0c98ee,
-        0xf102cbbb,
-        0xb607c4b7,
-        0xbbcf06b4,
-        0xfcd0fc00,
-        0x4221f5e0,
-/* 0x072f: memx_info */
-        0xf100f803,
-        0xf103c0c7,
-        0xf50800b7,
+/* 0x06e9: memx_func_train */
+/* 0x06eb: memx_exec */
+        0xf900f800,
+        0xb9d0f9e0,
+        0xb2b902c1,
+/* 0x06f5: memx_exec_next */
+        0x00139802,
+        0xe70410b6,
+        0xe701f034,
+        0xb601e033,
+        0x30f00132,
+        0xde35980c,
+        0x12b855f9,
+        0xe41ef406,
+        0x98f10b98,
+        0xcbbbf20c,
+        0xc4b7f102,
+        0x06b4b607,
+        0xfc00bbcf,
+        0xf5e0fcd0,
         0xf8034221,
-/* 0x073d: memx_recv */
-        0x01d6b000,
-        0xb0a90bf4,
-        0x0bf400d6,
-/* 0x074b: memx_init */
-        0xf800f8e9,
-/* 0x074d: perf_recv */
-/* 0x074f: perf_init */
-        0xf800f800,
-/* 0x0751: i2c_drive_scl */
-        0x0036b000,
-        0xf1110bf4,
-        0xb607e007,
-        0x01d00604,
-        0xf804bd00,
-/* 0x0765: i2c_drive_scl_lo */
-        0xe407f100,
-        0x0604b607,
-        0xbd0001d0,
-/* 0x0773: i2c_drive_sda */
-        0xb000f804,
-        0x0bf40036,
-        0xe007f111,
-        0x0604b607,
-        0xbd0002d0,
-/* 0x0787: i2c_drive_sda_lo */
-        0xf100f804,
-        0xb607e407,
-        0x02d00604,
-        0xf804bd00,
-/* 0x0795: i2c_sense_scl */
-        0x0132f400,
-        0x07c437f1,
-        0xcf0634b6,
-        0x31fd0033,
-        0x060bf404,
-/* 0x07ab: i2c_sense_scl_done */
-        0xf80131f4,
-/* 0x07ad: i2c_sense_sda */
-        0x0132f400,
-        0x07c437f1,
-        0xcf0634b6,
-        0x32fd0033,
-        0x060bf404,
-/* 0x07c3: i2c_sense_sda_done */
-        0xf80131f4,
-/* 0x07c5: i2c_raise_scl */
-        0xf140f900,
-        0xf0089847,
-        0x21f50137,
-/* 0x07d2: i2c_raise_scl_wait */
-        0xe7f10751,
-        0x21f403e8,
-        0x9521f57f,
-        0x0901f407,
-        0xf40142b6,
-/* 0x07e6: i2c_raise_scl_done */
-        0x40fcef1b,
-/* 0x07ea: i2c_start */
-        0x21f500f8,
-        0x11f40795,
-        0xad21f50d,
-        0x0611f407,
-/* 0x07fb: i2c_start_rep */
-        0xf0300ef4,
-        0x21f50037,
-        0x37f00751,
-        0x7321f501,
-        0x0076bb07,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0x21f550fc,
-        0x64b607c5,
-        0x1f11f404,
-/* 0x0828: i2c_start_send */
-        0xf50037f0,
-        0xf1077321,
-        0xf41388e7,
-        0x37f07f21,
-        0x5121f500,
-        0x88e7f107,
-        0x7f21f413,
-/* 0x0844: i2c_start_out */
-/* 0x0846: i2c_stop */
-        0x37f000f8,
-        0x5121f500,
-        0x0037f007,
-        0x077321f5,
-        0x03e8e7f1,
-        0xf07f21f4,
-        0x21f50137,
-        0xe7f10751,
-        0x21f41388,
-        0x0137f07f,
-        0x077321f5,
-        0x1388e7f1,
-        0xf87f21f4,
-/* 0x0879: i2c_bitw */
-        0x7321f500,
+/* 0x0731: memx_info */
+        0x01c67000,
+/* 0x0737: memx_info_data */
+        0xf10e0bf4,
+        0xf103ccc7,
+        0xf40800b7,
+/* 0x0742: memx_info_train */
+        0xc7f10b0e,
+        0xb7f10bcc,
+/* 0x074a: memx_info_send */
+        0x21f50100,
+        0x00f80342,
+/* 0x0750: memx_recv */
+        0xf401d6b0,
+        0xd6b0980b,
+        0xd80bf400,
+/* 0x075e: memx_init */
+        0x00f800f8,
+/* 0x0760: perf_recv */
+/* 0x0762: perf_init */
+        0x00f800f8,
+/* 0x0764: i2c_drive_scl */
+        0xf40036b0,
+        0x07f1110b,
+        0x04b607e0,
+        0x0001d006,
+        0x00f804bd,
+/* 0x0778: i2c_drive_scl_lo */
+        0x07e407f1,
+        0xd00604b6,
+        0x04bd0001,
+/* 0x0786: i2c_drive_sda */
+        0x36b000f8,
+        0x110bf400,
+        0x07e007f1,
+        0xd00604b6,
+        0x04bd0002,
+/* 0x079a: i2c_drive_sda_lo */
+        0x07f100f8,
+        0x04b607e4,
+        0x0002d006,
+        0x00f804bd,
+/* 0x07a8: i2c_sense_scl */
+        0xf10132f4,
+        0xb607c437,
+        0x33cf0634,
+        0x0431fd00,
+        0xf4060bf4,
+/* 0x07be: i2c_sense_scl_done */
+        0x00f80131,
+/* 0x07c0: i2c_sense_sda */
+        0xf10132f4,
+        0xb607c437,
+        0x33cf0634,
+        0x0432fd00,
+        0xf4060bf4,
+/* 0x07d6: i2c_sense_sda_done */
+        0x00f80131,
+/* 0x07d8: i2c_raise_scl */
+        0x47f140f9,
+        0x37f00898,
+        0x6421f501,
+/* 0x07e5: i2c_raise_scl_wait */
         0xe8e7f107,
         0x7f21f403,
+        0x07a821f5,
+        0xb60901f4,
+        0x1bf40142,
+/* 0x07f9: i2c_raise_scl_done */
+        0xf840fcef,
+/* 0x07fd: i2c_start */
+        0xa821f500,
+        0x0d11f407,
+        0x07c021f5,
+        0xf40611f4,
+/* 0x080e: i2c_start_rep */
+        0x37f0300e,
+        0x6421f500,
+        0x0137f007,
+        0x078621f5,
         0xb60076bb,
         0x50f90465,
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0xc521f550,
+        0xd821f550,
         0x0464b607,
-        0xf11811f4,
-        0xf41388e7,
+/* 0x083b: i2c_start_send */
+        0xf01f11f4,
+        0x21f50037,
+        0xe7f10786,
+        0x21f41388,
+        0x0037f07f,
+        0x076421f5,
+        0x1388e7f1,
+/* 0x0857: i2c_start_out */
+        0xf87f21f4,
+/* 0x0859: i2c_stop */
+        0x0037f000,
+        0x076421f5,
+        0xf50037f0,
+        0xf1078621,
+        0xf403e8e7,
         0x37f07f21,
-        0x5121f500,
+        0x6421f501,
         0x88e7f107,
         0x7f21f413,
-/* 0x08b8: i2c_bitw_out */
-/* 0x08ba: i2c_bitr */
-        0x37f000f8,
-        0x7321f501,
-        0xe8e7f107,
-        0x7f21f403,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0xc521f550,
-        0x0464b607,
-        0xf51b11f4,
-        0xf007ad21,
-        0x21f50037,
-        0xe7f10751,
+        0xf50137f0,
+        0xf1078621,
+        0xf41388e7,
+        0x00f87f21,
+/* 0x088c: i2c_bitw */
+        0x078621f5,
+        0x03e8e7f1,
+        0xbb7f21f4,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x07d821f5,
+        0xf40464b6,
+        0xe7f11811,
         0x21f41388,
-        0x013cf07f,
-/* 0x08ff: i2c_bitr_done */
-        0xf80131f4,
-/* 0x0901: i2c_get_byte */
-        0x0057f000,
-/* 0x0907: i2c_get_byte_next */
-        0xb60847f0,
-        0x76bb0154,
-        0x0465b600,
-        0x659450f9,
-        0x0256bb04,
-        0x75fd50bd,
-        0xf550fc04,
-        0xb608ba21,
-        0x11f40464,
-        0x0553fd2b,
-        0xf40142b6,
-        0x37f0d81b,
+        0x0037f07f,
+        0x076421f5,
+        0x1388e7f1,
+/* 0x08cb: i2c_bitw_out */
+        0xf87f21f4,
+/* 0x08cd: i2c_bitr */
+        0x0137f000,
+        0x078621f5,
+        0x03e8e7f1,
+        0xbb7f21f4,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x07d821f5,
+        0xf40464b6,
+        0x21f51b11,
+        0x37f007c0,
+        0x6421f500,
+        0x88e7f107,
+        0x7f21f413,
+        0xf4013cf0,
+/* 0x0912: i2c_bitr_done */
+        0x00f80131,
+/* 0x0914: i2c_get_byte */
+        0xf00057f0,
+/* 0x091a: i2c_get_byte_next */
+        0x54b60847,
         0x0076bb01,
         0xf90465b6,
         0x04659450,
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b60879,
-/* 0x0951: i2c_get_byte_done */
-/* 0x0953: i2c_put_byte */
-        0xf000f804,
-/* 0x0956: i2c_put_byte_next */
-        0x42b60847,
-        0x3854ff01,
+        0x64b608cd,
+        0x2b11f404,
+        0xb60553fd,
+        0x1bf40142,
+        0x0137f0d8,
         0xb60076bb,
         0x50f90465,
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0x7921f550,
+        0x8c21f550,
         0x0464b608,
-        0xb03411f4,
-        0x1bf40046,
-        0x0076bbd8,
+/* 0x0964: i2c_get_byte_done */
+/* 0x0966: i2c_put_byte */
+        0x47f000f8,
+/* 0x0969: i2c_put_byte_next */
+        0x0142b608,
+        0xbb3854ff,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x088c21f5,
+        0xf40464b6,
+        0x46b03411,
+        0xd81bf400,
+        0xb60076bb,
+        0x50f90465,
+        0xbb046594,
+        0x50bd0256,
+        0xfc0475fd,
+        0xcd21f550,
+        0x0464b608,
+        0xbb0f11f4,
+        0x36b00076,
+        0x061bf401,
+/* 0x09bf: i2c_put_byte_done */
+        0xf80132f4,
+/* 0x09c1: i2c_addr */
+        0x0076bb00,
         0xf90465b6,
         0x04659450,
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b608ba,
-        0x0f11f404,
-        0xb00076bb,
-        0x1bf40136,
-        0x0132f406,
-/* 0x09ac: i2c_put_byte_done */
-/* 0x09ae: i2c_addr */
-        0x76bb00f8,
+        0x64b607fd,
+        0x2911f404,
+        0x012ec3e7,
+        0xfd0134b6,
+        0x76bb0553,
         0x0465b600,
         0x659450f9,
         0x0256bb04,
         0x75fd50bd,
         0xf550fc04,
-        0xb607ea21,
-        0x11f40464,
-        0x2ec3e729,
-        0x0134b601,
-        0xbb0553fd,
+        0xb6096621,
+/* 0x0a06: i2c_addr_done */
+        0x00f80464,
+/* 0x0a08: i2c_acquire_addr */
+        0xb6f8cec7,
+        0xe0b702e4,
+        0xee980d1c,
+/* 0x0a17: i2c_acquire */
+        0xf500f800,
+        0xf40a0821,
+        0xd9f00421,
+        0x3f21f403,
+/* 0x0a26: i2c_release */
+        0x21f500f8,
+        0x21f40a08,
+        0x03daf004,
+        0xf83f21f4,
+/* 0x0a35: i2c_recv */
+        0x0132f400,
+        0xb6f8c1c7,
+        0x16b00214,
+        0x3a1ff528,
+        0xf413a001,
+        0x0032980c,
+        0x0ccc13a0,
+        0xf4003198,
+        0xd0f90231,
+        0xd0f9e0f9,
+        0x000067f1,
+        0x100063f1,
+        0xbb016792,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x095321f5,
-/* 0x09f3: i2c_addr_done */
-        0xf80464b6,
-/* 0x09f5: i2c_acquire_addr */
-        0xf8cec700,
-        0xb702e4b6,
-        0x980c10e0,
-        0x00f800ee,
-/* 0x0a04: i2c_acquire */
-        0x09f521f5,
-        0xf00421f4,
-        0x21f403d9,
-/* 0x0a13: i2c_release */
-        0xf500f83f,
-        0xf409f521,
-        0xdaf00421,
-        0x3f21f403,
-/* 0x0a22: i2c_recv */
-        0x32f400f8,
-        0xf8c1c701,
-        0xb00214b6,
-        0x1ff52816,
-        0x13a0013a,
-        0x32980be8,
-        0xc013a000,
-        0x0031980b,
-        0xf90231f4,
-        0xf9e0f9d0,
-        0x0067f1d0,
-        0x0063f100,
-        0x01679210,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0x0421f550,
-        0x0464b60a,
-        0xd6b0d0fc,
-        0xb31bf500,
-        0x0057f000,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0xae21f550,
-        0x0464b609,
-        0x00d011f5,
-        0xbbe0c5c7,
+        0x0a1721f5,
+        0xfc0464b6,
+        0x00d6b0d0,
+        0x00b31bf5,
+        0xbb0057f0,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x095321f5,
+        0x09c121f5,
         0xf50464b6,
-        0xf000ad11,
-        0x76bb0157,
+        0xc700d011,
+        0x76bbe0c5,
         0x0465b600,
         0x659450f9,
         0x0256bb04,
         0x75fd50bd,
         0xf550fc04,
-        0xb609ae21,
+        0xb6096621,
         0x11f50464,
-        0x76bb008a,
-        0x0465b600,
-        0x659450f9,
-        0x0256bb04,
-        0x75fd50bd,
-        0xf550fc04,
-        0xb6090121,
-        0x11f40464,
-        0xe05bcb6a,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0x4621f550,
-        0x0464b608,
-        0xbd025bb9,
-        0x430ef474,
-/* 0x0b28: i2c_recv_not_rd08 */
-        0xf401d6b0,
-        0x57f03d1b,
-        0xae21f500,
-        0x3311f409,
-        0xf5e0c5c7,
-        0xf4095321,
-        0x57f02911,
-        0xae21f500,
-        0x1f11f409,
-        0xf5e0b5c7,
-        0xf4095321,
-        0x21f51511,
-        0x74bd0846,
-        0xf408c5c7,
-        0x32f4091b,
-        0x030ef402,
-/* 0x0b68: i2c_recv_not_wr08 */
-/* 0x0b68: i2c_recv_done */
-        0xf5f8cec7,
-        0xfc0a1321,
-        0xf4d0fce0,
-        0x7cb90a12,
-        0x4221f502,
-/* 0x0b7d: i2c_recv_exit */
-/* 0x0b7f: i2c_init */
-        0xf800f803,
-/* 0x0b81: test_recv */
-        0xd817f100,
-        0x0614b605,
-        0xb60011cf,
-        0x07f10110,
-        0x04b605d8,
-        0x0001d006,
-        0xe7f104bd,
-        0xe3f1d900,
-        0x21f5134f,
-        0x00f80262,
-/* 0x0ba8: test_init */
-        0x0800e7f1,
-        0x026221f5,
-/* 0x0bb2: idle_recv */
+        0x57f000ad,
+        0x0076bb01,
+        0xf90465b6,
+        0x04659450,
+        0xbd0256bb,
+        0x0475fd50,
+        0x21f550fc,
+        0x64b609c1,
+        0x8a11f504,
+        0x0076bb00,
+        0xf90465b6,
+        0x04659450,
+        0xbd0256bb,
+        0x0475fd50,
+        0x21f550fc,
+        0x64b60914,
+        0x6a11f404,
+        0xbbe05bcb,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x085921f5,
+        0xb90464b6,
+        0x74bd025b,
+/* 0x0b3b: i2c_recv_not_rd08 */
+        0xb0430ef4,
+        0x1bf401d6,
+        0x0057f03d,
+        0x09c121f5,
+        0xc73311f4,
+        0x21f5e0c5,
+        0x11f40966,
+        0x0057f029,
+        0x09c121f5,
+        0xc71f11f4,
+        0x21f5e0b5,
+        0x11f40966,
+        0x5921f515,
+        0xc774bd08,
+        0x1bf408c5,
+        0x0232f409,
+/* 0x0b7b: i2c_recv_not_wr08 */
+/* 0x0b7b: i2c_recv_done */
+        0xc7030ef4,
+        0x21f5f8ce,
+        0xe0fc0a26,
+        0x12f4d0fc,
+        0x027cb90a,
+        0x034221f5,
+/* 0x0b90: i2c_recv_exit */
+/* 0x0b92: i2c_init */
         0x00f800f8,
-/* 0x0bb4: idle */
-        0xf10031f4,
-        0xb605d417,
-        0x11cf0614,
-        0x0110b600,
-        0x05d407f1,
-        0xd00604b6,
-        0x04bd0001,
-/* 0x0bd0: idle_loop */
-        0xf45817f0,
-/* 0x0bd6: idle_proc */
-/* 0x0bd6: idle_proc_exec */
-        0x10f90232,
-        0xf5021eb9,
-        0xfc034b21,
-        0x0911f410,
-        0xf40231f4,
-/* 0x0bea: idle_proc_next */
-        0x10b6ef0e,
-        0x061fb858,
-        0xf4e61bf4,
-        0x28f4dd02,
-        0xbb0ef400,
+/* 0x0b94: test_recv */
+        0x05d817f1,
+        0xcf0614b6,
+        0x10b60011,
+        0xd807f101,
+        0x0604b605,
+        0xbd0001d0,
+        0x00e7f104,
+        0x4fe3f1d9,
+        0x6221f513,
+/* 0x0bbb: test_init */
+        0xf100f802,
+        0xf50800e7,
+        0xf8026221,
+/* 0x0bc5: idle_recv */
+/* 0x0bc7: idle */
+        0xf400f800,
+        0x17f10031,
+        0x14b605d4,
+        0x0011cf06,
+        0xf10110b6,
+        0xb605d407,
+        0x01d00604,
+/* 0x0be3: idle_loop */
+        0xf004bd00,
+        0x32f45817,
+/* 0x0be9: idle_proc */
+/* 0x0be9: idle_proc_exec */
+        0xb910f902,
+        0x21f5021e,
+        0x10fc034b,
+        0xf40911f4,
+        0x0ef40231,
+/* 0x0bfd: idle_proc_next */
+        0x5810b6ef,
+        0xf4061fb8,
+        0x02f4e61b,
+        0x0028f4dd,
+        0x00bb0ef4,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
         0x00000000,
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h
index 12d86f72ad10..7e16aab44d85 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h
@@ -46,8 +46,8 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x584d454d,
-        0x00000678,
-        0x0000066a,
+        0x0000068b,
+        0x0000067d,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -68,8 +68,8 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x46524550,
-        0x0000067c,
-        0x0000067a,
+        0x0000068f,
+        0x0000068d,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -90,8 +90,8 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x5f433249,
-        0x00000a97,
-        0x0000093a,
+        0x00000aaa,
+        0x0000094d,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -112,8 +112,8 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x54534554,
-        0x00000aba,
-        0x00000a99,
+        0x00000acd,
+        0x00000aac,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -134,8 +134,8 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x454c4449,
-        0x00000ac6,
-        0x00000ac4,
+        0x00000ad9,
+        0x00000ad7,
         0x00000000,
         0x00000000,
         0x00000000,
@@ -246,13 +246,15 @@ uint32_t nvd0_pwr_data[] = {
         0x00010006,
         0x00000000,
         0x000005d3,
-/* 0x03b8: memx_func_tail */
-/* 0x03b8: memx_ts_start */
+        0x00000007,
         0x00000000,
-/* 0x03bc: memx_ts_end */
+        0x00000619,
+/* 0x03c4: memx_func_tail */
+/* 0x03c4: memx_ts_start */
         0x00000000,
-/* 0x03c0: memx_data_head */
+/* 0x03c8: memx_ts_end */
         0x00000000,
+/* 0x03cc: memx_data_head */
         0x00000000,
         0x00000000,
         0x00000000,
@@ -764,8 +766,75 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x00000000,
-/* 0x0bc0: memx_data_tail */
-/* 0x0bc0: i2c_scl_map */
+        0x00000000,
+/* 0x0bcc: memx_data_tail */
+/* 0x0bcc: memx_train_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0ccc: memx_train_tail */
+/* 0x0ccc: i2c_scl_map */
         0x00000400,
         0x00000800,
         0x00001000,
@@ -776,7 +845,7 @@ uint32_t nvd0_pwr_data[] = {
         0x00020000,
         0x00040000,
         0x00080000,
-/* 0x0be8: i2c_sda_map */
+/* 0x0cf4: i2c_sda_map */
         0x00100000,
         0x00200000,
         0x00400000,
@@ -844,9 +913,6 @@ uint32_t nvd0_pwr_data[] = {
         0x00000000,
         0x00000000,
         0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
 };
 
 uint32_t nvd0_pwr_code[] = {
@@ -1236,11 +1302,11 @@ uint32_t nvd0_pwr_code[] = {
         0x0bf40464,
         0x2c67f0f6,
         0x800066cf,
-        0x00f8ee06,
+        0x00f8f106,
 /* 0x0554: memx_func_leave */
         0xcf2c67f0,
         0x06800066,
-        0x0467f0ef,
+        0x0467f0f2,
         0x07e407f1,
         0xbd0006d0,
 /* 0x0569: memx_func_leave_wait */
@@ -1292,379 +1358,383 @@ uint32_t nvd0_pwr_code[] = {
         0x1e9800f8,
         0x0410b600,
         0xf86721f4,
-/* 0x0619: memx_exec */
-        0xf9e0f900,
-        0x02c1b9d0,
-/* 0x0623: memx_exec_next */
-        0x9802b2b9,
-        0x10b60013,
-        0xf034e704,
-        0xe033e701,
-        0x0132b601,
-        0x980c30f0,
-        0x55f9de35,
-        0xf40612b8,
-        0x0b98e41e,
-        0xef0c98ee,
-        0xf102cbbb,
-        0xcf07c4b7,
-        0xd0fc00bb,
-        0x21f5e0fc,
-        0x00f802f1,
-/* 0x065c: memx_info */
-        0x03c0c7f1,
-        0x0800b7f1,
+/* 0x0619: memx_func_train */
+/* 0x061b: memx_exec */
+        0xf900f800,
+        0xb9d0f9e0,
+        0xb2b902c1,
+/* 0x0625: memx_exec_next */
+        0x00139802,
+        0xe70410b6,
+        0xe701f034,
+        0xb601e033,
+        0x30f00132,
+        0xde35980c,
+        0x12b855f9,
+        0xe41ef406,
+        0x98f10b98,
+        0xcbbbf20c,
+        0xc4b7f102,
+        0x00bbcf07,
+        0xe0fcd0fc,
         0x02f121f5,
-/* 0x066a: memx_recv */
-        0xd6b000f8,
-        0xac0bf401,
-        0xf400d6b0,
-        0x00f8e90b,
-/* 0x0678: memx_init */
-/* 0x067a: perf_recv */
-        0x00f800f8,
-/* 0x067c: perf_init */
-/* 0x067e: i2c_drive_scl */
-        0x36b000f8,
-        0x0e0bf400,
-        0x07e007f1,
-        0xbd0001d0,
-/* 0x068f: i2c_drive_scl_lo */
-        0xf100f804,
-        0xd007e407,
+/* 0x065e: memx_info */
+        0xc67000f8,
+        0x0e0bf401,
+/* 0x0664: memx_info_data */
+        0x03ccc7f1,
+        0x0800b7f1,
+/* 0x066f: memx_info_train */
+        0xf10b0ef4,
+        0xf10bccc7,
+/* 0x0677: memx_info_send */
+        0xf50100b7,
+        0xf802f121,
+/* 0x067d: memx_recv */
+        0x01d6b000,
+        0xb09b0bf4,
+        0x0bf400d6,
+/* 0x068b: memx_init */
+        0xf800f8d8,
+/* 0x068d: perf_recv */
+/* 0x068f: perf_init */
+        0xf800f800,
+/* 0x0691: i2c_drive_scl */
+        0x0036b000,
+        0xf10e0bf4,
+        0xd007e007,
         0x04bd0001,
-/* 0x069a: i2c_drive_sda */
-        0x36b000f8,
-        0x0e0bf400,
-        0x07e007f1,
-        0xbd0002d0,
-/* 0x06ab: i2c_drive_sda_lo */
-        0xf100f804,
-        0xd007e407,
+/* 0x06a2: i2c_drive_scl_lo */
+        0x07f100f8,
+        0x01d007e4,
+        0xf804bd00,
+/* 0x06ad: i2c_drive_sda */
+        0x0036b000,
+        0xf10e0bf4,
+        0xd007e007,
         0x04bd0002,
-/* 0x06b6: i2c_sense_scl */
+/* 0x06be: i2c_drive_sda_lo */
+        0x07f100f8,
+        0x02d007e4,
+        0xf804bd00,
+/* 0x06c9: i2c_sense_scl */
+        0x0132f400,
+        0x07c437f1,
+        0xfd0033cf,
+        0x0bf40431,
+        0x0131f406,
+/* 0x06dc: i2c_sense_scl_done */
+/* 0x06de: i2c_sense_sda */
         0x32f400f8,
         0xc437f101,
         0x0033cf07,
-        0xf40431fd,
+        0xf40432fd,
         0x31f4060b,
-/* 0x06c9: i2c_sense_scl_done */
-/* 0x06cb: i2c_sense_sda */
-        0xf400f801,
-        0x37f10132,
-        0x33cf07c4,
-        0x0432fd00,
-        0xf4060bf4,
-/* 0x06de: i2c_sense_sda_done */
-        0x00f80131,
-/* 0x06e0: i2c_raise_scl */
-        0x47f140f9,
-        0x37f00898,
-        0x7e21f501,
-/* 0x06ed: i2c_raise_scl_wait */
-        0xe8e7f106,
-        0x6721f403,
-        0x06b621f5,
-        0xb60901f4,
-        0x1bf40142,
-/* 0x0701: i2c_raise_scl_done */
-        0xf840fcef,
-/* 0x0705: i2c_start */
-        0xb621f500,
-        0x0d11f406,
-        0x06cb21f5,
-        0xf40611f4,
-/* 0x0716: i2c_start_rep */
-        0x37f0300e,
-        0x7e21f500,
-        0x0137f006,
-        0x069a21f5,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0xe021f550,
-        0x0464b606,
-/* 0x0743: i2c_start_send */
-        0xf01f11f4,
-        0x21f50037,
-        0xe7f1069a,
-        0x21f41388,
-        0x0037f067,
-        0x067e21f5,
-        0x1388e7f1,
-/* 0x075f: i2c_start_out */
-        0xf86721f4,
-/* 0x0761: i2c_stop */
-        0x0037f000,
-        0x067e21f5,
-        0xf50037f0,
-        0xf1069a21,
-        0xf403e8e7,
-        0x37f06721,
-        0x7e21f501,
-        0x88e7f106,
-        0x6721f413,
-        0xf50137f0,
-        0xf1069a21,
-        0xf41388e7,
-        0x00f86721,
-/* 0x0794: i2c_bitw */
-        0x069a21f5,
+/* 0x06f1: i2c_sense_sda_done */
+/* 0x06f3: i2c_raise_scl */
+        0xf900f801,
+        0x9847f140,
+        0x0137f008,
+        0x069121f5,
+/* 0x0700: i2c_raise_scl_wait */
         0x03e8e7f1,
-        0xbb6721f4,
-        0x65b60076,
-        0x9450f904,
-        0x56bb0465,
-        0xfd50bd02,
-        0x50fc0475,
-        0x06e021f5,
-        0xf40464b6,
-        0xe7f11811,
-        0x21f41388,
-        0x0037f067,
-        0x067e21f5,
-        0x1388e7f1,
-/* 0x07d3: i2c_bitw_out */
-        0xf86721f4,
-/* 0x07d5: i2c_bitr */
-        0x0137f000,
-        0x069a21f5,
-        0x03e8e7f1,
-        0xbb6721f4,
+        0xf56721f4,
+        0xf406c921,
+        0x42b60901,
+        0xef1bf401,
+/* 0x0714: i2c_raise_scl_done */
+        0x00f840fc,
+/* 0x0718: i2c_start */
+        0x06c921f5,
+        0xf50d11f4,
+        0xf406de21,
+        0x0ef40611,
+/* 0x0729: i2c_start_rep */
+        0x0037f030,
+        0x069121f5,
+        0xf50137f0,
+        0xbb06ad21,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x06e021f5,
+        0x06f321f5,
         0xf40464b6,
-        0x21f51b11,
-        0x37f006cb,
-        0x7e21f500,
+/* 0x0756: i2c_start_send */
+        0x37f01f11,
+        0xad21f500,
         0x88e7f106,
         0x6721f413,
-        0xf4013cf0,
-/* 0x081a: i2c_bitr_done */
-        0x00f80131,
-/* 0x081c: i2c_get_byte */
-        0xf00057f0,
-/* 0x0822: i2c_get_byte_next */
-        0x54b60847,
-        0x0076bb01,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0x21f550fc,
-        0x64b607d5,
-        0x2b11f404,
-        0xb60553fd,
-        0x1bf40142,
-        0x0137f0d8,
+        0xf50037f0,
+        0xf1069121,
+        0xf41388e7,
+/* 0x0772: i2c_start_out */
+        0x00f86721,
+/* 0x0774: i2c_stop */
+        0xf50037f0,
+        0xf0069121,
+        0x21f50037,
+        0xe7f106ad,
+        0x21f403e8,
+        0x0137f067,
+        0x069121f5,
+        0x1388e7f1,
+        0xf06721f4,
+        0x21f50137,
+        0xe7f106ad,
+        0x21f41388,
+/* 0x07a7: i2c_bitw */
+        0xf500f867,
+        0xf106ad21,
+        0xf403e8e7,
+        0x76bb6721,
+        0x0465b600,
+        0x659450f9,
+        0x0256bb04,
+        0x75fd50bd,
+        0xf550fc04,
+        0xb606f321,
+        0x11f40464,
+        0x88e7f118,
+        0x6721f413,
+        0xf50037f0,
+        0xf1069121,
+        0xf41388e7,
+/* 0x07e6: i2c_bitw_out */
+        0x00f86721,
+/* 0x07e8: i2c_bitr */
+        0xf50137f0,
+        0xf106ad21,
+        0xf403e8e7,
+        0x76bb6721,
+        0x0465b600,
+        0x659450f9,
+        0x0256bb04,
+        0x75fd50bd,
+        0xf550fc04,
+        0xb606f321,
+        0x11f40464,
+        0xde21f51b,
+        0x0037f006,
+        0x069121f5,
+        0x1388e7f1,
+        0xf06721f4,
+        0x31f4013c,
+/* 0x082d: i2c_bitr_done */
+/* 0x082f: i2c_get_byte */
+        0xf000f801,
+        0x47f00057,
+/* 0x0835: i2c_get_byte_next */
+        0x0154b608,
         0xb60076bb,
         0x50f90465,
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0x9421f550,
+        0xe821f550,
         0x0464b607,
-/* 0x086c: i2c_get_byte_done */
-/* 0x086e: i2c_put_byte */
-        0x47f000f8,
-/* 0x0871: i2c_put_byte_next */
-        0x0142b608,
-        0xbb3854ff,
+        0xfd2b11f4,
+        0x42b60553,
+        0xd81bf401,
+        0xbb0137f0,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x07a721f5,
+/* 0x087f: i2c_get_byte_done */
+        0xf80464b6,
+/* 0x0881: i2c_put_byte */
+        0x0847f000,
+/* 0x0884: i2c_put_byte_next */
+        0xff0142b6,
+        0x76bb3854,
+        0x0465b600,
+        0x659450f9,
+        0x0256bb04,
+        0x75fd50bd,
+        0xf550fc04,
+        0xb607a721,
+        0x11f40464,
+        0x0046b034,
+        0xbbd81bf4,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x079421f5,
+        0x07e821f5,
         0xf40464b6,
-        0x46b03411,
-        0xd81bf400,
+        0x76bb0f11,
+        0x0136b000,
+        0xf4061bf4,
+/* 0x08da: i2c_put_byte_done */
+        0x00f80132,
+/* 0x08dc: i2c_addr */
         0xb60076bb,
         0x50f90465,
         0xbb046594,
         0x50bd0256,
         0xfc0475fd,
-        0xd521f550,
+        0x1821f550,
         0x0464b607,
-        0xbb0f11f4,
-        0x36b00076,
-        0x061bf401,
-/* 0x08c7: i2c_put_byte_done */
-        0xf80132f4,
-/* 0x08c9: i2c_addr */
-        0x0076bb00,
+        0xe72911f4,
+        0xb6012ec3,
+        0x53fd0134,
+        0x0076bb05,
         0xf90465b6,
         0x04659450,
         0xbd0256bb,
         0x0475fd50,
         0x21f550fc,
-        0x64b60705,
-        0x2911f404,
-        0x012ec3e7,
-        0xfd0134b6,
-        0x76bb0553,
-        0x0465b600,
-        0x659450f9,
-        0x0256bb04,
-        0x75fd50bd,
-        0xf550fc04,
-        0xb6086e21,
-/* 0x090e: i2c_addr_done */
-        0x00f80464,
-/* 0x0910: i2c_acquire_addr */
-        0xb6f8cec7,
-        0xe0b705e4,
-        0x00f8d014,
-/* 0x091c: i2c_acquire */
-        0x091021f5,
-        0xf00421f4,
-        0x21f403d9,
-/* 0x092b: i2c_release */
-        0xf500f833,
-        0xf4091021,
-        0xdaf00421,
+        0x64b60881,
+/* 0x0921: i2c_addr_done */
+/* 0x0923: i2c_acquire_addr */
+        0xc700f804,
+        0xe4b6f8ce,
+        0x14e0b705,
+/* 0x092f: i2c_acquire */
+        0xf500f8d0,
+        0xf4092321,
+        0xd9f00421,
         0x3321f403,
-/* 0x093a: i2c_recv */
-        0x32f400f8,
-        0xf8c1c701,
-        0xb00214b6,
-        0x1ff52816,
-        0x13a0013a,
-        0x32980be8,
-        0xc013a000,
-        0x0031980b,
-        0xf90231f4,
-        0xf9e0f9d0,
-        0x0067f1d0,
-        0x0063f100,
-        0x01679210,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0x1c21f550,
-        0x0464b609,
-        0xd6b0d0fc,
-        0xb31bf500,
-        0x0057f000,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0xc921f550,
-        0x0464b608,
-        0x00d011f5,
-        0xbbe0c5c7,
+/* 0x093e: i2c_release */
+        0x21f500f8,
+        0x21f40923,
+        0x03daf004,
+        0xf83321f4,
+/* 0x094d: i2c_recv */
+        0x0132f400,
+        0xb6f8c1c7,
+        0x16b00214,
+        0x3a1ff528,
+        0xf413a001,
+        0x0032980c,
+        0x0ccc13a0,
+        0xf4003198,
+        0xd0f90231,
+        0xd0f9e0f9,
+        0x000067f1,
+        0x100063f1,
+        0xbb016792,
         0x65b60076,
         0x9450f904,
         0x56bb0465,
         0xfd50bd02,
         0x50fc0475,
-        0x086e21f5,
+        0x092f21f5,
+        0xfc0464b6,
+        0x00d6b0d0,
+        0x00b31bf5,
+        0xbb0057f0,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x08dc21f5,
         0xf50464b6,
-        0xf000ad11,
-        0x76bb0157,
+        0xc700d011,
+        0x76bbe0c5,
         0x0465b600,
         0x659450f9,
         0x0256bb04,
         0x75fd50bd,
         0xf550fc04,
-        0xb608c921,
+        0xb6088121,
         0x11f50464,
-        0x76bb008a,
-        0x0465b600,
-        0x659450f9,
-        0x0256bb04,
-        0x75fd50bd,
-        0xf550fc04,
-        0xb6081c21,
-        0x11f40464,
-        0xe05bcb6a,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0x6121f550,
-        0x0464b607,
-        0xbd025bb9,
-        0x430ef474,
-/* 0x0a40: i2c_recv_not_rd08 */
-        0xf401d6b0,
-        0x57f03d1b,
-        0xc921f500,
-        0x3311f408,
-        0xf5e0c5c7,
-        0xf4086e21,
-        0x57f02911,
-        0xc921f500,
-        0x1f11f408,
-        0xf5e0b5c7,
-        0xf4086e21,
-        0x21f51511,
-        0x74bd0761,
-        0xf408c5c7,
-        0x32f4091b,
-        0x030ef402,
-/* 0x0a80: i2c_recv_not_wr08 */
-/* 0x0a80: i2c_recv_done */
-        0xf5f8cec7,
-        0xfc092b21,
-        0xf4d0fce0,
-        0x7cb90a12,
-        0xf121f502,
-/* 0x0a95: i2c_recv_exit */
-/* 0x0a97: i2c_init */
+        0x57f000ad,
+        0x0076bb01,
+        0xf90465b6,
+        0x04659450,
+        0xbd0256bb,
+        0x0475fd50,
+        0x21f550fc,
+        0x64b608dc,
+        0x8a11f504,
+        0x0076bb00,
+        0xf90465b6,
+        0x04659450,
+        0xbd0256bb,
+        0x0475fd50,
+        0x21f550fc,
+        0x64b6082f,
+        0x6a11f404,
+        0xbbe05bcb,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x077421f5,
+        0xb90464b6,
+        0x74bd025b,
+/* 0x0a53: i2c_recv_not_rd08 */
+        0xb0430ef4,
+        0x1bf401d6,
+        0x0057f03d,
+        0x08dc21f5,
+        0xc73311f4,
+        0x21f5e0c5,
+        0x11f40881,
+        0x0057f029,
+        0x08dc21f5,
+        0xc71f11f4,
+        0x21f5e0b5,
+        0x11f40881,
+        0x7421f515,
+        0xc774bd07,
+        0x1bf408c5,
+        0x0232f409,
+/* 0x0a93: i2c_recv_not_wr08 */
+/* 0x0a93: i2c_recv_done */
+        0xc7030ef4,
+        0x21f5f8ce,
+        0xe0fc093e,
+        0x12f4d0fc,
+        0x027cb90a,
+        0x02f121f5,
+/* 0x0aa8: i2c_recv_exit */
+/* 0x0aaa: i2c_init */
+        0x00f800f8,
+/* 0x0aac: test_recv */
+        0x05d817f1,
+        0xb60011cf,
+        0x07f10110,
+        0x01d005d8,
+        0xf104bd00,
+        0xf1d900e7,
+        0xf5134fe3,
+        0xf8022321,
+/* 0x0acd: test_init */
+        0x00e7f100,
+        0x2321f508,
+/* 0x0ad7: idle_recv */
         0xf800f802,
-/* 0x0a99: test_recv */
-        0xd817f100,
-        0x0011cf05,
-        0xf10110b6,
-        0xd005d807,
-        0x04bd0001,
-        0xd900e7f1,
-        0x134fe3f1,
-        0x022321f5,
-/* 0x0aba: test_init */
-        0xe7f100f8,
-        0x21f50800,
-        0x00f80223,
-/* 0x0ac4: idle_recv */
-/* 0x0ac6: idle */
-        0x31f400f8,
-        0xd417f100,
-        0x0011cf05,
-        0xf10110b6,
-        0xd005d407,
-        0x04bd0001,
-/* 0x0adc: idle_loop */
-        0xf45817f0,
-/* 0x0ae2: idle_proc */
-/* 0x0ae2: idle_proc_exec */
-        0x10f90232,
-        0xf5021eb9,
-        0xfc02fa21,
-        0x0911f410,
-        0xf40231f4,
-/* 0x0af6: idle_proc_next */
-        0x10b6ef0e,
-        0x061fb858,
-        0xf4e61bf4,
-        0x28f4dd02,
-        0xc10ef400,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
+/* 0x0ad9: idle */
+        0x0031f400,
+        0x05d417f1,
+        0xb60011cf,
+        0x07f10110,
+        0x01d005d4,
+/* 0x0aef: idle_loop */
+        0xf004bd00,
+        0x32f45817,
+/* 0x0af5: idle_proc */
+/* 0x0af5: idle_proc_exec */
+        0xb910f902,
+        0x21f5021e,
+        0x10fc02fa,
+        0xf40911f4,
+        0x0ef40231,
+/* 0x0b09: idle_proc_next */
+        0x5810b6ef,
+        0xf4061fb8,
+        0x02f4e61b,
+        0x0028f4dd,
+        0x00c10ef4,
         0x00000000,
         0x00000000,
         0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/os.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/os.h
index 522e3079f824..c8b06cb77e72 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/os.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/os.h
@@ -18,6 +18,10 @@
 #define MEMX_MSG_INFO 0
 #define MEMX_MSG_EXEC 1
 
+/* MEMX: info types */
+#define MEMX_INFO_DATA  0
+#define MEMX_INFO_TRAIN 1
+
 /* MEMX: script opcode definitions */
 #define MEMX_ENTER  1
 #define MEMX_LEAVE  2
@@ -25,6 +29,7 @@
 #define MEMX_WAIT   4
 #define MEMX_DELAY  5
 #define MEMX_VBLANK 6
+#define MEMX_TRAIN  7
 
 /* I2C_: message identifiers */
 #define I2C__MSG_RD08 0
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c
index 65eaa2546cad..7a9299d7159f 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c
@@ -47,7 +47,8 @@ nouveau_memx_init(struct nouveau_pwr *ppwr, struct nouveau_memx **pmemx)
         u32 reply[2];
         int ret;
 
-        ret = ppwr->message(ppwr, reply, PROC_MEMX, MEMX_MSG_INFO, 0, 0);
+        ret = ppwr->message(ppwr, reply, PROC_MEMX, MEMX_MSG_INFO,
+                                        MEMX_INFO_DATA, 0);
         if (ret)
                 return ret;
 
@@ -106,7 +107,7 @@ nouveau_memx_wait(struct nouveau_memx *memx,
 {
         nv_debug(memx->ppwr, "R[%06x] & 0x%08x == 0x%08x, %d us\n",
                                 addr, mask, data, nsec);
-        memx_cmd(memx, MEMX_WAIT, 4, (u32[]){ addr, ~mask, data, nsec });
+        memx_cmd(memx, MEMX_WAIT, 4, (u32[]){ addr, mask, data, nsec });
         memx_out(memx); /* fuc can't handle multiple */
 }
 
@@ -152,6 +153,38 @@ nouveau_memx_wait_vblank(struct nouveau_memx *memx)
 }
 
 void
+nouveau_memx_train(struct nouveau_memx *memx)
+{
+        nv_debug(memx->ppwr, "   MEM TRAIN\n");
+        memx_cmd(memx, MEMX_TRAIN, 0, NULL);
+}
+
+int
+nouveau_memx_train_result(struct nouveau_pwr *ppwr, u32 *res, int rsize)
+{
+        u32 reply[2], base, size, i;
+        int ret;
+
+        ret = ppwr->message(ppwr, reply, PROC_MEMX, MEMX_MSG_INFO,
+                                        MEMX_INFO_TRAIN, 0);
+        if (ret)
+                return ret;
+
+        base = reply[0];
+        size = reply[1] >> 2;
+        if (size > rsize)
+                return -ENOMEM;
+
+        /* read the packet */
+        nv_wr32(ppwr, 0x10a1c0, 0x02000000 | base);
+
+        for (i = 0; i < size; i++)
+                res[i] = nv_rd32(ppwr, 0x10a1c4);
+
+        return 0;
+}
+
+void
 nouveau_memx_block(struct nouveau_memx *memx)
 {
         nv_debug(memx->ppwr, "   HOST BLOCKED\n");
diff --git a/drivers/gpu/drm/nouveau/core/subdev/volt/base.c b/drivers/gpu/drm/nouveau/core/subdev/volt/base.c
index 32794a999106..26ccd8df193f 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/volt/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/volt/base.c
@@ -101,6 +101,41 @@ nouveau_volt_set_id(struct nouveau_volt *volt, u8 id, int condition)
         return ret;
 }
 
+static void nouveau_volt_parse_bios(struct nouveau_bios *bios,
+                struct nouveau_volt *volt)
+{
+        struct nvbios_volt_entry ivid;
+        struct nvbios_volt info;
+        u8  ver, hdr, cnt, len;
+        u16 data;
+        int i;
+
+        data = nvbios_volt_parse(bios, &ver, &hdr, &cnt, &len, &info);
+        if (data && info.vidmask && info.base && info.step) {
+                for (i = 0; i < info.vidmask + 1; i++) {
+                        if (info.base >= info.min &&
+                                info.base <= info.max) {
+                                volt->vid[volt->vid_nr].uv = info.base;
+                                volt->vid[volt->vid_nr].vid = i;
+                                volt->vid_nr++;
+                        }
+                        info.base += info.step;
+                }
+                volt->vid_mask = info.vidmask;
+        } else if (data && info.vidmask) {
+                for (i = 0; i < cnt; i++) {
+                        data = nvbios_volt_entry_parse(bios, i, &ver, &hdr,
+                                                          &ivid);
+                        if (data) {
+                                volt->vid[volt->vid_nr].uv = ivid.voltage;
+                                volt->vid[volt->vid_nr].vid = ivid.vid;
+                                volt->vid_nr++;
+                        }
+                }
+                volt->vid_mask = info.vidmask;
+        }
+}
+
 int
 _nouveau_volt_init(struct nouveau_object *object)
 {
@@ -136,10 +171,6 @@ nouveau_volt_create_(struct nouveau_object *parent,
 {
         struct nouveau_bios *bios = nouveau_bios(parent);
         struct nouveau_volt *volt;
-        struct nvbios_volt_entry ivid;
-        struct nvbios_volt info;
-        u8  ver, hdr, cnt, len;
-        u16 data;
         int ret, i;
 
         ret = nouveau_subdev_create_(parent, engine, oclass, 0, "VOLT",
@@ -152,31 +183,9 @@ nouveau_volt_create_(struct nouveau_object *parent,
         volt->set = nouveau_volt_set;
         volt->set_id = nouveau_volt_set_id;
 
-        data = nvbios_volt_parse(bios, &ver, &hdr, &cnt, &len, &info);
-        if (data && info.vidmask && info.base && info.step) {
-                for (i = 0; i < info.vidmask + 1; i++) {
-                        if (info.base >= info.min &&
-                            info.base <= info.max) {
-                                volt->vid[volt->vid_nr].uv = info.base;
-                                volt->vid[volt->vid_nr].vid = i;
-                                volt->vid_nr++;
-                        }
-                        info.base += info.step;
-                }
-                volt->vid_mask = info.vidmask;
-        } else
-        if (data && info.vidmask) {
-                for (i = 0; i < cnt; i++) {
-                        data = nvbios_volt_entry_parse(bios, i, &ver, &hdr,
-                                                      &ivid);
-                        if (data) {
-                                volt->vid[volt->vid_nr].uv = ivid.voltage;
-                                volt->vid[volt->vid_nr].vid = ivid.vid;
-                                volt->vid_nr++;
-                        }
-                }
-                volt->vid_mask = info.vidmask;
-        }
+        /* Assuming the non-bios device should build the voltage table later */
+        if (bios)
+                nouveau_volt_parse_bios(bios, volt);
 
         if (volt->vid_nr) {
                 for (i = 0; i < volt->vid_nr; i++) {
diff --git a/drivers/gpu/drm/nouveau/core/subdev/volt/gk20a.c b/drivers/gpu/drm/nouveau/core/subdev/volt/gk20a.c
new file mode 100644
index 000000000000..717368ef31ac
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/volt/gk20a.c
@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifdef __KERNEL__
+#include <nouveau_platform.h>
+#endif
+#include <subdev/volt.h>
+
+struct cvb_coef {
+        int c0;
+        int c1;
+        int c2;
+        int c3;
+        int c4;
+        int c5;
+};
+
+struct gk20a_volt_priv {
+        struct nouveau_volt base;
+        struct regulator *vdd;
+};
+
+const struct cvb_coef gk20a_cvb_coef[] = {
+        /* MHz,        c0,     c1,   c2,    c3,     c4,   c5 */
+        /*  72 */ { 1209886, -36468,  515,   417, -13123,  203},
+        /* 108 */ { 1130804, -27659,  296,   298, -10834,  221},
+        /* 180 */ { 1162871, -27110,  247,   238, -10681,  268},
+        /* 252 */ { 1220458, -28654,  247,   179, -10376,  298},
+        /* 324 */ { 1280953, -30204,  247,   119,  -9766,  304},
+        /* 396 */ { 1344547, -31777,  247,   119,  -8545,  292},
+        /* 468 */ { 1420168, -34227,  269,    60,  -7172,  256},
+        /* 540 */ { 1490757, -35955,  274,    60,  -5188,  197},
+        /* 612 */ { 1599112, -42583,  398,     0,  -1831,  119},
+        /* 648 */ { 1366986, -16459, -274,     0,  -3204,   72},
+        /* 684 */ { 1391884, -17078, -274,   -60,  -1526,   30},
+        /* 708 */ { 1415522, -17497, -274,   -60,   -458,    0},
+        /* 756 */ { 1464061, -18331, -274,  -119,   1831,  -72},
+        /* 804 */ { 1524225, -20064, -254,  -119,   4272, -155},
+        /* 852 */ { 1608418, -21643, -269,     0,    763,  -48},
+};
+
+/**
+ * cvb_mv = ((c2 * speedo / s_scale + c1) * speedo / s_scale + c0)
+ */
+static inline int
+gk20a_volt_get_cvb_voltage(int speedo, int s_scale,
+                const struct cvb_coef *coef)
+{
+        int mv;
+
+        mv = DIV_ROUND_CLOSEST(coef->c2 * speedo, s_scale);
+        mv = DIV_ROUND_CLOSEST((mv + coef->c1) * speedo, s_scale) + coef->c0;
+        return mv;
+}
+
+/**
+ * cvb_t_mv =
+ * ((c2 * speedo / s_scale + c1) * speedo / s_scale + c0) +
+ * ((c3 * speedo / s_scale + c4 + c5 * T / t_scale) * T / t_scale)
+ */
+static inline int
+gk20a_volt_get_cvb_t_voltage(int speedo, int temp, int s_scale, int t_scale,
+                const struct cvb_coef *coef)
+{
+        int cvb_mv, mv;
+
+        cvb_mv = gk20a_volt_get_cvb_voltage(speedo, s_scale, coef);
+
+        mv = DIV_ROUND_CLOSEST(coef->c3 * speedo, s_scale) + coef->c4 +
+                DIV_ROUND_CLOSEST(coef->c5 * temp, t_scale);
+        mv = DIV_ROUND_CLOSEST(mv * temp, t_scale) + cvb_mv;
+        return mv;
+}
+
+static int
+gk20a_volt_calc_voltage(const struct cvb_coef *coef, int speedo)
+{
+        int mv;
+
+        mv = gk20a_volt_get_cvb_t_voltage(speedo, -10, 100, 10, coef);
+        mv = DIV_ROUND_UP(mv, 1000);
+
+        return mv * 1000;
+}
+
+static int
+gk20a_volt_vid_get(struct nouveau_volt *volt)
+{
+        struct gk20a_volt_priv *priv = (void *)volt;
+        int i, uv;
+
+        uv = regulator_get_voltage(priv->vdd);
+
+        for (i = 0; i < volt->vid_nr; i++)
+                if (volt->vid[i].uv >= uv)
+                        return i;
+
+        return -EINVAL;
+}
+
+static int
+gk20a_volt_vid_set(struct nouveau_volt *volt, u8 vid)
+{
+        struct gk20a_volt_priv *priv = (void *)volt;
+
+        nv_debug(volt, "set voltage as %duv\n", volt->vid[vid].uv);
+        return regulator_set_voltage(priv->vdd, volt->vid[vid].uv, 1200000);
+}
+
+static int
+gk20a_volt_set_id(struct nouveau_volt *volt, u8 id, int condition)
+{
+        struct gk20a_volt_priv *priv = (void *)volt;
+        int prev_uv = regulator_get_voltage(priv->vdd);
+        int target_uv = volt->vid[id].uv;
+        int ret;
+
+        nv_debug(volt, "prev=%d, target=%d, condition=%d\n",
+                        prev_uv, target_uv, condition);
+        if (!condition ||
+                (condition < 0 && target_uv < prev_uv) ||
+                (condition > 0 && target_uv > prev_uv)) {
+                ret = gk20a_volt_vid_set(volt, volt->vid[id].vid);
+        } else {
+                ret = 0;
+        }
+
+        return ret;
+}
+
+static int
+gk20a_volt_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+               struct nouveau_oclass *oclass, void *data, u32 size,
+               struct nouveau_object **pobject)
+{
+        struct gk20a_volt_priv *priv;
+        struct nouveau_volt *volt;
+        struct nouveau_platform_device *plat;
+        int i, ret, uv;
+
+        ret = nouveau_volt_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        volt = &priv->base;
+
+        plat = nv_device_to_platform(nv_device(parent));
+
+        uv = regulator_get_voltage(plat->gpu->vdd);
+        nv_info(priv, "The default voltage is %duV\n", uv);
+
+        priv->vdd = plat->gpu->vdd;
+        priv->base.vid_get = gk20a_volt_vid_get;
+        priv->base.vid_set = gk20a_volt_vid_set;
+        priv->base.set_id = gk20a_volt_set_id;
+
+        volt->vid_nr = ARRAY_SIZE(gk20a_cvb_coef);
+        nv_debug(priv, "%s - vid_nr = %d\n", __func__, volt->vid_nr);
+        for (i = 0; i < volt->vid_nr; i++) {
+                volt->vid[i].vid = i;
+                volt->vid[i].uv = gk20a_volt_calc_voltage(&gk20a_cvb_coef[i],
+                                        plat->gpu_speedo);
+                nv_debug(priv, "%2d: vid=%d, uv=%d\n", i, volt->vid[i].vid,
+                                        volt->vid[i].uv);
+        }
+
+        return 0;
+}
+
+struct nouveau_oclass
+gk20a_volt_oclass = {
+        .handle = NV_SUBDEV(VOLT, 0xea),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gk20a_volt_ctor,
+                .dtor = _nouveau_volt_dtor,
+                .init = _nouveau_volt_init,
+                .fini = _nouveau_volt_fini,
+        },
+};