126 files changed, 6261 insertions, 2636 deletions
diff --git a/drivers/gpu/drm/nouveau/Makefile b/drivers/gpu/drm/nouveau/Makefile
index 12c24c8abf7f..6461e3565afe 100644
--- a/drivers/gpu/drm/nouveau/Makefile
+++ b/drivers/gpu/drm/nouveau/Makefile
@@ -41,17 +41,28 @@ nouveau-y += core/subdev/bios/extdev.o
 nouveau-y += core/subdev/bios/fan.o
 nouveau-y += core/subdev/bios/gpio.o
 nouveau-y += core/subdev/bios/i2c.o
+nouveau-y += core/subdev/bios/image.o
 nouveau-y += core/subdev/bios/init.o
 nouveau-y += core/subdev/bios/mxm.o
+nouveau-y += core/subdev/bios/npde.o
+nouveau-y += core/subdev/bios/pcir.o
 nouveau-y += core/subdev/bios/perf.o
 nouveau-y += core/subdev/bios/pll.o
+nouveau-y += core/subdev/bios/pmu.o
 nouveau-y += core/subdev/bios/ramcfg.o
 nouveau-y += core/subdev/bios/rammap.o
+nouveau-y += core/subdev/bios/shadow.o
+nouveau-y += core/subdev/bios/shadowacpi.o
+nouveau-y += core/subdev/bios/shadowof.o
+nouveau-y += core/subdev/bios/shadowpci.o
+nouveau-y += core/subdev/bios/shadowramin.o
+nouveau-y += core/subdev/bios/shadowrom.o
 nouveau-y += core/subdev/bios/timing.o
 nouveau-y += core/subdev/bios/therm.o
 nouveau-y += core/subdev/bios/vmap.o
 nouveau-y += core/subdev/bios/volt.o
 nouveau-y += core/subdev/bios/xpio.o
+nouveau-y += core/subdev/bios/M0203.o
 nouveau-y += core/subdev/bios/M0205.o
 nouveau-y += core/subdev/bios/M0209.o
 nouveau-y += core/subdev/bios/P0260.o
@@ -86,6 +97,7 @@ nouveau-y += core/subdev/devinit/nva3.o
 nouveau-y += core/subdev/devinit/nvaf.o
 nouveau-y += core/subdev/devinit/nvc0.o
 nouveau-y += core/subdev/devinit/gm107.o
+nouveau-y += core/subdev/devinit/gm204.o
 nouveau-y += core/subdev/fb/base.o
 nouveau-y += core/subdev/fb/nv04.o
 nouveau-y += core/subdev/fb/nv10.o
@@ -129,6 +141,7 @@ nouveau-y += core/subdev/fb/ramgk20a.o
 nouveau-y += core/subdev/fb/ramgm107.o
 nouveau-y += core/subdev/fb/sddr2.o
 nouveau-y += core/subdev/fb/sddr3.o
+nouveau-y += core/subdev/fb/gddr3.o
 nouveau-y += core/subdev/fb/gddr5.o
 nouveau-y += core/subdev/fuse/base.o
 nouveau-y += core/subdev/fuse/g80.o
@@ -147,6 +160,7 @@ nouveau-y += core/subdev/i2c/bit.o
 nouveau-y += core/subdev/i2c/pad.o
 nouveau-y += core/subdev/i2c/padnv04.o
 nouveau-y += core/subdev/i2c/padnv94.o
+nouveau-y += core/subdev/i2c/padgm204.o
 nouveau-y += core/subdev/i2c/nv04.o
 nouveau-y += core/subdev/i2c/nv4e.o
 nouveau-y += core/subdev/i2c/nv50.o
@@ -154,6 +168,7 @@ nouveau-y += core/subdev/i2c/nv94.o
 nouveau-y += core/subdev/i2c/nvd0.o
 nouveau-y += core/subdev/i2c/gf117.o
 nouveau-y += core/subdev/i2c/nve0.o
+nouveau-y += core/subdev/i2c/gm204.o
 nouveau-y += core/subdev/ibus/nvc0.o
 nouveau-y += core/subdev/ibus/nve0.o
 nouveau-y += core/subdev/ibus/gk20a.o
@@ -211,6 +226,7 @@ nouveau-y += core/subdev/vm/nvc0.o
 nouveau-y += core/subdev/volt/base.o
 nouveau-y += core/subdev/volt/gpio.o
 nouveau-y += core/subdev/volt/nv40.o
+nouveau-y += core/subdev/volt/gk20a.o
 nouveau-y += core/engine/falcon.o
 nouveau-y += core/engine/xtensa.o
@@ -254,6 +270,7 @@ nouveau-y += core/engine/disp/nvd0.o
 nouveau-y += core/engine/disp/nve0.o
 nouveau-y += core/engine/disp/nvf0.o
 nouveau-y += core/engine/disp/gm107.o
+nouveau-y += core/engine/disp/gm204.o
 nouveau-y += core/engine/disp/dacnv50.o
 nouveau-y += core/engine/disp/dport.o
 nouveau-y += core/engine/disp/hdanva3.o
@@ -266,6 +283,7 @@ nouveau-y += core/engine/disp/piornv50.o
 nouveau-y += core/engine/disp/sornv50.o
 nouveau-y += core/engine/disp/sornv94.o
 nouveau-y += core/engine/disp/sornvd0.o
+nouveau-y += core/engine/disp/sorgm204.o
 nouveau-y += core/engine/disp/vga.o
 nouveau-y += core/engine/fifo/base.o
 nouveau-y += core/engine/fifo/nv04.o
diff --git a/drivers/gpu/drm/nouveau/core/core/handle.c b/drivers/gpu/drm/nouveau/core/core/handle.c
index a490b805d7e3..13f816cb08bd 100644
--- a/drivers/gpu/drm/nouveau/core/core/handle.c
+++ b/drivers/gpu/drm/nouveau/core/core/handle.c
@@ -222,116 +222,3 @@ nouveau_handle_put(struct nouveau_handle *handle)
        if (handle)
                nouveau_namedb_put(handle);
 }
-int
-nouveau_handle_new(struct nouveau_object *client, u32 _parent, u32 _handle,
-                   u16 _oclass, void *data, u32 size,
-                   struct nouveau_object **pobject)
-{
-        struct nouveau_object *parent = NULL;
-        struct nouveau_object *engctx = NULL;
-        struct nouveau_object *object = NULL;
-        struct nouveau_object *engine;
-        struct nouveau_oclass *oclass;
-        struct nouveau_handle *handle;
-        int ret;
-        /* lookup parent object and ensure it *is* a parent */
-        parent = nouveau_handle_ref(client, _parent);
-        if (!parent) {
-                nv_error(client, "parent 0x%08x not found\n", _parent);
-                return -ENOENT;
-        }
-        if (!nv_iclass(parent, NV_PARENT_CLASS)) {
-                nv_error(parent, "cannot have children\n");
-                ret = -EINVAL;
-                goto fail_class;
-        }
-        /* check that parent supports the requested subclass */
-        ret = nouveau_parent_sclass(parent, _oclass, &engine, &oclass);
-        if (ret) {
-                nv_debug(parent, "illegal class 0x%04x\n", _oclass);
-                goto fail_class;
-        }
-        /* make sure engine init has been completed *before* any objects
-         * it controls are created - the constructors may depend on
-         * state calculated at init (ie. default context construction)
-         */
-        if (engine) {
-                ret = nouveau_object_inc(engine);
-                if (ret)
-                        goto fail_class;
-        }
-        /* if engine requires it, create a context object to insert
-         * between the parent and its children (eg. PGRAPH context)
-         */
-        if (engine && nv_engine(engine)->cclass) {
-                ret = nouveau_object_ctor(parent, engine,
-                                          nv_engine(engine)->cclass,
-                                          data, size, &engctx);
-                if (ret)
-                        goto fail_engctx;
-        } else {
-                nouveau_object_ref(parent, &engctx);
-        }
-        /* finally, create new object and bind it to its handle */
-        ret = nouveau_object_ctor(engctx, engine, oclass, data, size, &object);
-        *pobject = object;
-        if (ret)
-                goto fail_ctor;
-        ret = nouveau_object_inc(object);
-        if (ret)
-                goto fail_init;
-        ret = nouveau_handle_create(parent, _parent, _handle, object, &handle);
-        if (ret)
-                goto fail_handle;
-        ret = nouveau_handle_init(handle);
-        if (ret)
-                nouveau_handle_destroy(handle);
-fail_handle:
-        nouveau_object_dec(object, false);
-fail_init:
-        nouveau_object_ref(NULL, &object);
-fail_ctor:
-        nouveau_object_ref(NULL, &engctx);
-fail_engctx:
-        if (engine)
-                nouveau_object_dec(engine, false);
-fail_class:
-        nouveau_object_ref(NULL, &parent);
-        return ret;
-}
-int
-nouveau_handle_del(struct nouveau_object *client, u32 _parent, u32 _handle)
-{
-        struct nouveau_object *parent = NULL;
-        struct nouveau_object *namedb = NULL;
-        struct nouveau_handle *handle = NULL;
-        parent = nouveau_handle_ref(client, _parent);
-        if (!parent)
-                return -ENOENT;
-        namedb = nv_pclass(parent, NV_NAMEDB_CLASS);
-        if (namedb) {
-                handle = nouveau_namedb_get(nv_namedb(namedb), _handle);
-                if (handle) {
-                        nouveau_namedb_put(handle);
-                        nouveau_handle_fini(handle, false);
-                        nouveau_handle_destroy(handle);
-                }
-        }
-        nouveau_object_ref(NULL, &parent);
-        return handle ? 0 : -EINVAL;
-}
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/base.c b/drivers/gpu/drm/nouveau/core/engine/device/base.c
index 0ef5a5713182..137e0b0faeae 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/base.c
@@ -29,6 +29,7 @@
 #include <nvif/unpack.h>
 #include <nvif/class.h>
+#include <subdev/bios.h>
 #include <subdev/fb.h>
 #include <subdev/instmem.h>
@@ -138,7 +139,7 @@ nouveau_devobj_info(struct nouveau_object *object, void *data, u32 size)
        }
        args->v0.chipset  = device->chipset;
-        args->v0.revision = device->chipset >= 0x10 ? nv_rd32(device, 0) : 0x00;
+        args->v0.revision = device->chiprev;
        if (pfb)  args->v0.ram_size = args->v0.ram_user = pfb->ram->size;
        else      args->v0.ram_size = args->v0.ram_user = 0;
        if (imem) args->v0.ram_user = args->v0.ram_user - imem->reserved;
@@ -222,6 +223,7 @@ static const u64 disable_map[] = {
        [NVDEV_SUBDEV_VOLT]     = NV_DEVICE_V0_DISABLE_CORE,
        [NVDEV_SUBDEV_THERM]    = NV_DEVICE_V0_DISABLE_CORE,
        [NVDEV_SUBDEV_PWR]      = NV_DEVICE_V0_DISABLE_CORE,
+        [NVDEV_SUBDEV_FUSE]     = NV_DEVICE_V0_DISABLE_CORE,
        [NVDEV_ENGINE_DMAOBJ]   = NV_DEVICE_V0_DISABLE_CORE,
        [NVDEV_ENGINE_PERFMON]  = NV_DEVICE_V0_DISABLE_CORE,
        [NVDEV_ENGINE_FIFO]     = NV_DEVICE_V0_DISABLE_FIFO,
@@ -235,6 +237,7 @@ static const u64 disable_map[] = {
        [NVDEV_ENGINE_PPP]      = NV_DEVICE_V0_DISABLE_PPP,
        [NVDEV_ENGINE_COPY0]    = NV_DEVICE_V0_DISABLE_COPY0,
        [NVDEV_ENGINE_COPY1]    = NV_DEVICE_V0_DISABLE_COPY1,
+        [NVDEV_ENGINE_COPY2]    = NV_DEVICE_V0_DISABLE_COPY1,
        [NVDEV_ENGINE_VIC]      = NV_DEVICE_V0_DISABLE_VIC,
        [NVDEV_ENGINE_VENC]     = NV_DEVICE_V0_DISABLE_VENC,
        [NVDEV_ENGINE_DISP]     = NV_DEVICE_V0_DISABLE_DISP,
@@ -352,12 +355,14 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                /* determine chipset and derive architecture from it */
                if ((boot0 & 0x1f000000) > 0) {
                        device->chipset = (boot0 & 0x1ff00000) >> 20;
+                        device->chiprev = (boot0 & 0x000000ff);
                        switch (device->chipset & 0x1f0) {
                        case 0x010: {
                                if (0x461 & (1 << (device->chipset & 0xf)))
                                        device->card_type = NV_10;
                                else
                                        device->card_type = NV_11;
+                                device->chiprev = 0x00;
                                break;
                        }
                        case 0x020: device->card_type = NV_20; break;
@@ -373,7 +378,8 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                        case 0x0e0:
                        case 0x0f0:
                        case 0x100: device->card_type = NV_E0; break;
-                        case 0x110: device->card_type = GM100; break;
+                        case 0x110:
+                        case 0x120: device->card_type = GM100; break;
                        default:
                                break;
                        }
@@ -427,6 +433,10 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                }
                nv_debug(device, "crystal freq: %dKHz\n", device->crystal);
+        } else
+        if ( (args->v0.disable & NV_DEVICE_V0_DISABLE_IDENTIFY)) {
+                device->cname = "NULL";
+                device->oclass[NVDEV_SUBDEV_VBIOS] = &nouveau_bios_oclass;
        }
        if (!(args->v0.disable & NV_DEVICE_V0_DISABLE_MMIO) &&
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/gm100.c b/drivers/gpu/drm/nouveau/core/engine/device/gm100.c
index 6295668e29a5..4e74a3376de8 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/gm100.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/gm100.c
@@ -98,6 +98,49 @@ gm100_identify(struct nouveau_device *device)
                device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
 #endif
                break;
+        case 0x124:
+                device->cname = "GM204";
+                device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+                device->oclass[NVDEV_SUBDEV_GPIO   ] =  nve0_gpio_oclass;
+                device->oclass[NVDEV_SUBDEV_I2C    ] =  gm204_i2c_oclass;
+                device->oclass[NVDEV_SUBDEV_FUSE   ] = &gm107_fuse_oclass;
+#if 0
+                /* looks to be some non-trivial changes */
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nve0_clock_oclass;
+                /* priv ring says no to 0x10eb14 writes */
+                device->oclass[NVDEV_SUBDEV_THERM  ] = &gm107_therm_oclass;
+#endif
+                device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+                device->oclass[NVDEV_SUBDEV_DEVINIT] =  gm204_devinit_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  gk20a_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_TIMER  ] = &gk20a_timer_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  gm107_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_LTC    ] =  gm107_ltc_oclass;
+                device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
+                device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
+                device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] =  nv108_pwr_oclass;
+#if 0
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
+#endif
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] =  nvd0_dmaeng_oclass;
+#if 0
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv108_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_GR     ] =  gm107_graph_oclass;
+#endif
+                device->oclass[NVDEV_ENGINE_DISP   ] =  gm204_disp_oclass;
+#if 0
+                device->oclass[NVDEV_ENGINE_COPY0  ] = &gm204_copy0_oclass;
+                device->oclass[NVDEV_ENGINE_COPY1  ] = &gm204_copy1_oclass;
+                device->oclass[NVDEV_ENGINE_COPY2  ] = &gm204_copy2_oclass;
+                device->oclass[NVDEV_ENGINE_BSP    ] = &nve0_bsp_oclass;
+                device->oclass[NVDEV_ENGINE_VP     ] = &nve0_vp_oclass;
+                device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
+#endif
+                break;
        default:
                nv_fatal(device, "unknown Maxwell chipset\n");
                return -EINVAL;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nve0.c b/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
index b1b2e484ecfa..674da1f095b2 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
@@ -179,6 +179,7 @@ nve0_identify(struct nouveau_device *device)
                device->oclass[NVDEV_ENGINE_GR     ] =  gk20a_graph_oclass;
                device->oclass[NVDEV_ENGINE_COPY2  ] = &nve0_copy2_oclass;
                device->oclass[NVDEV_ENGINE_PERFMON] = &nve0_perfmon_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &gk20a_volt_oclass;
                break;
        case 0xf0:
                device->cname = "GK110";
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/dport.c b/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
index 39890221b91c..16db08dfba6e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
@@ -28,7 +28,7 @@
 #include <subdev/bios/init.h>
 #include <subdev/i2c.h>
-#include <engine/disp.h>
+#include "nv50.h"
 #include <nvif/class.h>
@@ -326,7 +326,7 @@ void
 nouveau_dp_train(struct work_struct *w)
 {
        struct nvkm_output_dp *outp = container_of(w, typeof(*outp), lt.work);
-        struct nouveau_disp *disp = nouveau_disp(outp);
+        struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
        const struct dp_rates *cfg = nouveau_dp_rates;
        struct dp_state _dp = {
                .outp = outp,
@@ -334,8 +334,11 @@ nouveau_dp_train(struct work_struct *w)
        u32 datarate = 0;
        int ret;
+        if (!outp->base.info.location && priv->sor.magic)
+                priv->sor.magic(&outp->base);
        /* bring capabilities within encoder limits */
-        if (nv_mclass(disp) < GF110_DISP)
+        if (nv_mclass(priv) < GF110_DISP)
                outp->dpcd[2] &= ~DPCD_RC02_TPS3_SUPPORTED;
        if ((outp->dpcd[2] & 0x1f) > outp->base.info.dpconf.link_nr) {
                outp->dpcd[2] &= ~DPCD_RC02_MAX_LANE_COUNT;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c b/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c
index b3df3fe2dc09..e2ad0543fb31 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c
@@ -35,8 +35,8 @@
 static struct nouveau_oclass
 gm107_disp_sclass[] = {
-        { GM107_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
+        { GM107_DISP_CORE_CHANNEL_DMA, &nvd0_disp_core_ofuncs.base },
-        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_base_ofuncs.base },
        { GK104_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
        { GK104_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
        { GK104_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
@@ -44,8 +44,8 @@ gm107_disp_sclass[] = {
 };
 static struct nouveau_oclass
-gm107_disp_base_oclass[] = {
+gm107_disp_main_oclass[] = {
-        { GM107_DISP, &nvd0_disp_base_ofuncs },
+        { GM107_DISP, &nvd0_disp_main_ofuncs },
        {}
 };
@@ -72,7 +72,7 @@ gm107_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        if (ret)
                return ret;
-        nv_engine(priv)->sclass = gm107_disp_base_oclass;
+        nv_engine(priv)->sclass = gm107_disp_main_oclass;
        nv_engine(priv)->cclass = &nv50_disp_cclass;
        nv_subdev(priv)->intr = nvd0_disp_intr;
        INIT_WORK(&priv->supervisor, nvd0_disp_intr_supervisor);
@@ -99,9 +99,9 @@ gm107_disp_oclass = &(struct nv50_disp_impl) {
        },
        .base.vblank = &nvd0_disp_vblank_func,
        .base.outp =  nvd0_disp_outp_sclass,
-        .mthd.core = &nve0_disp_mast_mthd_chan,
+        .mthd.core = &nve0_disp_core_mthd_chan,
-        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.base = &nvd0_disp_base_mthd_chan,
        .mthd.ovly = &nve0_disp_ovly_mthd_chan,
        .mthd.prev = -0x020000,
-        .head.scanoutpos = nvd0_disp_base_scanoutpos,
+        .head.scanoutpos = nvd0_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/gm204.c b/drivers/gpu/drm/nouveau/core/engine/disp/gm204.c
new file mode 100644
index 000000000000..672ded79b2a9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/gm204.c
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include <engine/software.h>
+#include <engine/disp.h>
+#include <nvif/class.h>
+#include "nv50.h"
+/*******************************************************************************
+ * Base display object
+ ******************************************************************************/
+static struct nouveau_oclass
+gm204_disp_sclass[] = {
+        { GM204_DISP_CORE_CHANNEL_DMA, &nvd0_disp_core_ofuncs.base },
+        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_base_ofuncs.base },
+        { GK104_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
+        { GK104_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
+        { GK104_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
+        {}
+};
+static struct nouveau_oclass
+gm204_disp_main_oclass[] = {
+        { GM204_DISP, &nvd0_disp_main_ofuncs },
+        {}
+};
+/*******************************************************************************
+ * Display engine implementation
+ ******************************************************************************/
+static int
+gm204_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+               struct nouveau_oclass *oclass, void *data, u32 size,
+               struct nouveau_object **pobject)
+{
+        struct nv50_disp_priv *priv;
+        int heads = nv_rd32(parent, 0x022448);
+        int ret;
+        ret = nouveau_disp_create(parent, engine, oclass, heads,
+                                  "PDISP", "display", &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+        ret = nvkm_event_init(&nvd0_disp_chan_uevent, 1, 17, &priv->uevent);
+        if (ret)
+                return ret;
+        nv_engine(priv)->sclass = gm204_disp_main_oclass;
+        nv_engine(priv)->cclass = &nv50_disp_cclass;
+        nv_subdev(priv)->intr = nvd0_disp_intr;
+        INIT_WORK(&priv->supervisor, nvd0_disp_intr_supervisor);
+        priv->sclass = gm204_disp_sclass;
+        priv->head.nr = heads;
+        priv->dac.nr = 3;
+        priv->sor.nr = 4;
+        priv->dac.power = nv50_dac_power;
+        priv->dac.sense = nv50_dac_sense;
+        priv->sor.power = nv50_sor_power;
+        priv->sor.hda_eld = nvd0_hda_eld;
+        priv->sor.hdmi = nvd0_hdmi_ctrl;
+        priv->sor.magic = gm204_sor_magic;
+        return 0;
+}
+struct nouveau_oclass *
+gm204_disp_outp_sclass[] = {
+        &gm204_sor_dp_impl.base.base,
+        NULL
+};
+struct nouveau_oclass *
+gm204_disp_oclass = &(struct nv50_disp_impl) {
+        .base.base.handle = NV_ENGINE(DISP, 0x07),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gm204_disp_ctor,
+                .dtor = _nouveau_disp_dtor,
+                .init = _nouveau_disp_init,
+                .fini = _nouveau_disp_fini,
+        },
+        .base.vblank = &nvd0_disp_vblank_func,
+        .base.outp =  gm204_disp_outp_sclass,
+        .mthd.core = &nve0_disp_core_mthd_chan,
+        .mthd.base = &nvd0_disp_base_mthd_chan,
+        .mthd.ovly = &nve0_disp_ovly_mthd_chan,
+        .mthd.prev = -0x020000,
+        .head.scanoutpos = nvd0_disp_main_scanoutpos,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
index 2df3a937037d..44a8290aaea5 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
@@ -88,12 +88,14 @@ nv50_disp_chan_uevent_fini(struct nvkm_event *event, int type, int index)
 {
        struct nv50_disp_priv *priv = container_of(event, typeof(*priv), uevent);
        nv_mask(priv, 0x610028, 0x00000001 << index, 0x00000000 << index);
+        nv_wr32(priv, 0x610020, 0x00000001 << index);
 }
 static void
 nv50_disp_chan_uevent_init(struct nvkm_event *event, int types, int index)
 {
        struct nv50_disp_priv *priv = container_of(event, typeof(*priv), uevent);
+        nv_wr32(priv, 0x610020, 0x00000001 << index);
        nv_mask(priv, 0x610028, 0x00000001 << index, 0x00000001 << index);
 }
@@ -374,7 +376,7 @@ nv50_disp_mthd_chan(struct nv50_disp_priv *priv, int debug, int head,
 }
 const struct nv50_disp_mthd_list
-nv50_disp_mast_mthd_base = {
+nv50_disp_core_mthd_base = {
        .mthd = 0x0000,
        .addr = 0x000000,
        .data = {
@@ -387,7 +389,7 @@ nv50_disp_mast_mthd_base = {
 };
 static const struct nv50_disp_mthd_list
-nv50_disp_mast_mthd_dac = {
+nv50_disp_core_mthd_dac = {
        .mthd = 0x0080,
        .addr = 0x000008,
        .data = {
@@ -399,7 +401,7 @@ nv50_disp_mast_mthd_dac = {
 };
 const struct nv50_disp_mthd_list
-nv50_disp_mast_mthd_sor = {
+nv50_disp_core_mthd_sor = {
        .mthd = 0x0040,
        .addr = 0x000008,
        .data = {
@@ -409,7 +411,7 @@ nv50_disp_mast_mthd_sor = {
 };
 const struct nv50_disp_mthd_list
-nv50_disp_mast_mthd_pior = {
+nv50_disp_core_mthd_pior = {
        .mthd = 0x0040,
        .addr = 0x000008,
        .data = {
@@ -419,7 +421,7 @@ nv50_disp_mast_mthd_pior = {
 };
 static const struct nv50_disp_mthd_list
-nv50_disp_mast_mthd_head = {
+nv50_disp_core_mthd_head = {
        .mthd = 0x0400,
        .addr = 0x000540,
        .data = {
@@ -466,21 +468,21 @@ nv50_disp_mast_mthd_head = {
 };
 static const struct nv50_disp_mthd_chan
-nv50_disp_mast_mthd_chan = {
+nv50_disp_core_mthd_chan = {
        .name = "Core",
        .addr = 0x000000,
        .data = {
-                { "Global", 1, &nv50_disp_mast_mthd_base },
+                { "Global", 1, &nv50_disp_core_mthd_base },
-                {    "DAC", 3, &nv50_disp_mast_mthd_dac  },
+                {    "DAC", 3, &nv50_disp_core_mthd_dac  },
-                {    "SOR", 2, &nv50_disp_mast_mthd_sor  },
+                {    "SOR", 2, &nv50_disp_core_mthd_sor  },
-                {   "PIOR", 3, &nv50_disp_mast_mthd_pior },
+                {   "PIOR", 3, &nv50_disp_core_mthd_pior },
-                {   "HEAD", 2, &nv50_disp_mast_mthd_head },
+                {   "HEAD", 2, &nv50_disp_core_mthd_head },
                {}
        }
 };
 int
-nv50_disp_mast_ctor(struct nouveau_object *parent,
+nv50_disp_core_ctor(struct nouveau_object *parent,
                    struct nouveau_object *engine,
                    struct nouveau_oclass *oclass, void *data, u32 size,
                    struct nouveau_object **pobject)
@@ -509,7 +511,7 @@ nv50_disp_mast_ctor(struct nouveau_object *parent,
 }
 static int
-nv50_disp_mast_init(struct nouveau_object *object)
+nv50_disp_core_init(struct nouveau_object *object)
 {
        struct nv50_disp_priv *priv = (void *)object->engine;
        struct nv50_disp_dmac *mast = (void *)object;
@@ -546,7 +548,7 @@ nv50_disp_mast_init(struct nouveau_object *object)
 }
 static int
-nv50_disp_mast_fini(struct nouveau_object *object, bool suspend)
+nv50_disp_core_fini(struct nouveau_object *object, bool suspend)
 {
        struct nv50_disp_priv *priv = (void *)object->engine;
        struct nv50_disp_dmac *mast = (void *)object;
@@ -567,11 +569,11 @@ nv50_disp_mast_fini(struct nouveau_object *object, bool suspend)
 }
 struct nv50_disp_chan_impl
-nv50_disp_mast_ofuncs = {
+nv50_disp_core_ofuncs = {
-        .base.ctor = nv50_disp_mast_ctor,
+        .base.ctor = nv50_disp_core_ctor,
        .base.dtor = nv50_disp_dmac_dtor,
-        .base.init = nv50_disp_mast_init,
+        .base.init = nv50_disp_core_init,
-        .base.fini = nv50_disp_mast_fini,
+        .base.fini = nv50_disp_core_fini,
        .base.map  = nv50_disp_chan_map,
        .base.ntfy = nv50_disp_chan_ntfy,
        .base.rd32 = nv50_disp_chan_rd32,
@@ -586,7 +588,7 @@ nv50_disp_mast_ofuncs = {
 ******************************************************************************/
 static const struct nv50_disp_mthd_list
-nv50_disp_sync_mthd_base = {
+nv50_disp_base_mthd_base = {
        .mthd = 0x0000,
        .addr = 0x000000,
        .data = {
@@ -611,7 +613,7 @@ nv50_disp_sync_mthd_base = {
 };
 const struct nv50_disp_mthd_list
-nv50_disp_sync_mthd_image = {
+nv50_disp_base_mthd_image = {
        .mthd = 0x0400,
        .addr = 0x000000,
        .data = {
@@ -625,18 +627,18 @@ nv50_disp_sync_mthd_image = {
 };
 static const struct nv50_disp_mthd_chan
-nv50_disp_sync_mthd_chan = {
+nv50_disp_base_mthd_chan = {
        .name = "Base",
        .addr = 0x000540,
        .data = {
-                { "Global", 1, &nv50_disp_sync_mthd_base },
+                { "Global", 1, &nv50_disp_base_mthd_base },
-                {  "Image", 2, &nv50_disp_sync_mthd_image },
+                {  "Image", 2, &nv50_disp_base_mthd_image },
                {}
        }
 };
 int
-nv50_disp_sync_ctor(struct nouveau_object *parent,
+nv50_disp_base_ctor(struct nouveau_object *parent,
                    struct nouveau_object *engine,
                    struct nouveau_oclass *oclass, void *data, u32 size,
                    struct nouveau_object **pobject)
@@ -669,8 +671,8 @@ nv50_disp_sync_ctor(struct nouveau_object *parent,
 }
 struct nv50_disp_chan_impl
-nv50_disp_sync_ofuncs = {
+nv50_disp_base_ofuncs = {
-        .base.ctor = nv50_disp_sync_ctor,
+        .base.ctor = nv50_disp_base_ctor,
        .base.dtor = nv50_disp_dmac_dtor,
        .base.init = nv50_disp_dmac_init,
        .base.fini = nv50_disp_dmac_fini,
@@ -942,7 +944,7 @@ nv50_disp_curs_ofuncs = {
 ******************************************************************************/
 int
-nv50_disp_base_scanoutpos(NV50_DISP_MTHD_V0)
+nv50_disp_main_scanoutpos(NV50_DISP_MTHD_V0)
 {
        const u32 blanke = nv_rd32(priv, 0x610aec + (head * 0x540));
        const u32 blanks = nv_rd32(priv, 0x610af4 + (head * 0x540));
@@ -974,7 +976,7 @@ nv50_disp_base_scanoutpos(NV50_DISP_MTHD_V0)
 }
 int
-nv50_disp_base_mthd(struct nouveau_object *object, u32 mthd,
+nv50_disp_main_mthd(struct nouveau_object *object, u32 mthd,
                    void *data, u32 size)
 {
        const struct nv50_disp_impl *impl = (void *)nv_oclass(object->engine);
@@ -1098,7 +1100,7 @@ nv50_disp_base_mthd(struct nouveau_object *object, u32 mthd,
 }
 int
-nv50_disp_base_ctor(struct nouveau_object *parent,
+nv50_disp_main_ctor(struct nouveau_object *parent,
                    struct nouveau_object *engine,
                    struct nouveau_oclass *oclass, void *data, u32 size,
                    struct nouveau_object **pobject)
@@ -1118,7 +1120,7 @@ nv50_disp_base_ctor(struct nouveau_object *parent,
 }
 void
-nv50_disp_base_dtor(struct nouveau_object *object)
+nv50_disp_main_dtor(struct nouveau_object *object)
 {
        struct nv50_disp_base *base = (void *)object;
        nouveau_ramht_ref(NULL, &base->ramht);
@@ -1126,7 +1128,7 @@ nv50_disp_base_dtor(struct nouveau_object *object)
 }
 static int
-nv50_disp_base_init(struct nouveau_object *object)
+nv50_disp_main_init(struct nouveau_object *object)
 {
        struct nv50_disp_priv *priv = (void *)object->engine;
        struct nv50_disp_base *base = (void *)object;
@@ -1194,7 +1196,7 @@ nv50_disp_base_init(struct nouveau_object *object)
 }
 static int
-nv50_disp_base_fini(struct nouveau_object *object, bool suspend)
+nv50_disp_main_fini(struct nouveau_object *object, bool suspend)
 {
        struct nv50_disp_priv *priv = (void *)object->engine;
        struct nv50_disp_base *base = (void *)object;
@@ -1207,25 +1209,25 @@ nv50_disp_base_fini(struct nouveau_object *object, bool suspend)
 }
 struct nouveau_ofuncs
-nv50_disp_base_ofuncs = {
+nv50_disp_main_ofuncs = {
-        .ctor = nv50_disp_base_ctor,
+        .ctor = nv50_disp_main_ctor,
-        .dtor = nv50_disp_base_dtor,
+        .dtor = nv50_disp_main_dtor,
-        .init = nv50_disp_base_init,
+        .init = nv50_disp_main_init,
-        .fini = nv50_disp_base_fini,
+        .fini = nv50_disp_main_fini,
-        .mthd = nv50_disp_base_mthd,
+        .mthd = nv50_disp_main_mthd,
        .ntfy = nouveau_disp_ntfy,
 };
 static struct nouveau_oclass
-nv50_disp_base_oclass[] = {
+nv50_disp_main_oclass[] = {
-        { NV50_DISP, &nv50_disp_base_ofuncs },
+        { NV50_DISP, &nv50_disp_main_ofuncs },
        {}
 };
 static struct nouveau_oclass
 nv50_disp_sclass[] = {
-        { NV50_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
+        { NV50_DISP_CORE_CHANNEL_DMA, &nv50_disp_core_ofuncs.base },
-        { NV50_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { NV50_DISP_BASE_CHANNEL_DMA, &nv50_disp_base_ofuncs.base },
        { NV50_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
        { NV50_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
        { NV50_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
@@ -1974,7 +1976,7 @@ nv50_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        if (ret)
                return ret;
-        nv_engine(priv)->sclass = nv50_disp_base_oclass;
+        nv_engine(priv)->sclass = nv50_disp_main_oclass;
        nv_engine(priv)->cclass = &nv50_disp_cclass;
        nv_subdev(priv)->intr = nv50_disp_intr;
        INIT_WORK(&priv->supervisor, nv50_disp_intr_supervisor);
@@ -2007,9 +2009,9 @@ nv50_disp_oclass = &(struct nv50_disp_impl) {
        },
        .base.vblank = &nv50_disp_vblank_func,
        .base.outp =  nv50_disp_outp_sclass,
-        .mthd.core = &nv50_disp_mast_mthd_chan,
+        .mthd.core = &nv50_disp_core_mthd_chan,
-        .mthd.base = &nv50_disp_sync_mthd_chan,
+        .mthd.base = &nv50_disp_base_mthd_chan,
        .mthd.ovly = &nv50_disp_ovly_mthd_chan,
        .mthd.prev = 0x000004,
-        .head.scanoutpos = nv50_disp_base_scanoutpos,
+        .head.scanoutpos = nv50_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
index 5279feefec06..7f08078ee925 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
@@ -42,6 +42,7 @@ struct nv50_disp_priv {
                int (*hda_eld)(NV50_DISP_MTHD_V1);
                int (*hdmi)(NV50_DISP_MTHD_V1);
                u32 lvdsconf;
+                void (*magic)(struct nvkm_output *);
        } sor;
        struct {
                int nr;
@@ -63,10 +64,10 @@ struct nv50_disp_impl {
        } head;
 };
-int nv50_disp_base_scanoutpos(NV50_DISP_MTHD_V0);
+int nv50_disp_main_scanoutpos(NV50_DISP_MTHD_V0);
-int nv50_disp_base_mthd(struct nouveau_object *, u32, void *, u32);
+int nv50_disp_main_mthd(struct nouveau_object *, u32, void *, u32);
-int nvd0_disp_base_scanoutpos(NV50_DISP_MTHD_V0);
+int nvd0_disp_main_scanoutpos(NV50_DISP_MTHD_V0);
 int nv50_dac_power(NV50_DISP_MTHD_V1);
 int nv50_dac_sense(NV50_DISP_MTHD_V1);
@@ -169,18 +170,18 @@ struct nv50_disp_mthd_chan {
        } data[];
 };
-extern struct nv50_disp_chan_impl nv50_disp_mast_ofuncs;
+extern struct nv50_disp_chan_impl nv50_disp_core_ofuncs;
-int nv50_disp_mast_ctor(struct nouveau_object *, struct nouveau_object *,
+int nv50_disp_core_ctor(struct nouveau_object *, struct nouveau_object *,
                        struct nouveau_oclass *, void *, u32,
                        struct nouveau_object **);
-extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_base;
+extern const struct nv50_disp_mthd_list nv50_disp_core_mthd_base;
-extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_sor;
+extern const struct nv50_disp_mthd_list nv50_disp_core_mthd_sor;
-extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_pior;
+extern const struct nv50_disp_mthd_list nv50_disp_core_mthd_pior;
-extern struct nv50_disp_chan_impl nv50_disp_sync_ofuncs;
+extern struct nv50_disp_chan_impl nv50_disp_base_ofuncs;
-int nv50_disp_sync_ctor(struct nouveau_object *, struct nouveau_object *,
+int nv50_disp_base_ctor(struct nouveau_object *, struct nouveau_object *,
                        struct nouveau_oclass *, void *, u32,
                        struct nouveau_object **);
-extern const struct nv50_disp_mthd_list nv50_disp_sync_mthd_image;
+extern const struct nv50_disp_mthd_list nv50_disp_base_mthd_image;
 extern struct nv50_disp_chan_impl nv50_disp_ovly_ofuncs;
 int nv50_disp_ovly_ctor(struct nouveau_object *, struct nouveau_object *,
                        struct nouveau_oclass *, void *, u32,
@@ -194,12 +195,12 @@ extern struct nv50_disp_chan_impl nv50_disp_curs_ofuncs;
 int nv50_disp_curs_ctor(struct nouveau_object *, struct nouveau_object *,
                        struct nouveau_oclass *, void *, u32,
                        struct nouveau_object **);
-extern struct nouveau_ofuncs nv50_disp_base_ofuncs;
+extern struct nouveau_ofuncs nv50_disp_main_ofuncs;
-int  nv50_disp_base_ctor(struct nouveau_object *, struct nouveau_object *,
+int  nv50_disp_main_ctor(struct nouveau_object *, struct nouveau_object *,
                         struct nouveau_oclass *, void *, u32,
                         struct nouveau_object **);
-void nv50_disp_base_dtor(struct nouveau_object *);
+void nv50_disp_main_dtor(struct nouveau_object *);
-extern struct nouveau_omthds nv50_disp_base_omthds[];
+extern struct nouveau_omthds nv50_disp_main_omthds[];
 extern struct nouveau_oclass nv50_disp_cclass;
 void nv50_disp_mthd_chan(struct nv50_disp_priv *, int debug, int head,
                         const struct nv50_disp_mthd_chan *);
@@ -207,31 +208,31 @@ void nv50_disp_intr_supervisor(struct work_struct *);
 void nv50_disp_intr(struct nouveau_subdev *);
 extern const struct nvkm_event_func nv50_disp_vblank_func;
-extern const struct nv50_disp_mthd_chan nv84_disp_mast_mthd_chan;
+extern const struct nv50_disp_mthd_chan nv84_disp_core_mthd_chan;
-extern const struct nv50_disp_mthd_list nv84_disp_mast_mthd_dac;
+extern const struct nv50_disp_mthd_list nv84_disp_core_mthd_dac;
-extern const struct nv50_disp_mthd_list nv84_disp_mast_mthd_head;
+extern const struct nv50_disp_mthd_list nv84_disp_core_mthd_head;
-extern const struct nv50_disp_mthd_chan nv84_disp_sync_mthd_chan;
+extern const struct nv50_disp_mthd_chan nv84_disp_base_mthd_chan;
 extern const struct nv50_disp_mthd_chan nv84_disp_ovly_mthd_chan;
-extern const struct nv50_disp_mthd_chan nv94_disp_mast_mthd_chan;
+extern const struct nv50_disp_mthd_chan nv94_disp_core_mthd_chan;
-extern struct nv50_disp_chan_impl nvd0_disp_mast_ofuncs;
+extern struct nv50_disp_chan_impl nvd0_disp_core_ofuncs;
-extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_base;
+extern const struct nv50_disp_mthd_list nvd0_disp_core_mthd_base;
-extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_dac;
+extern const struct nv50_disp_mthd_list nvd0_disp_core_mthd_dac;
-extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_sor;
+extern const struct nv50_disp_mthd_list nvd0_disp_core_mthd_sor;
-extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_pior;
+extern const struct nv50_disp_mthd_list nvd0_disp_core_mthd_pior;
-extern struct nv50_disp_chan_impl nvd0_disp_sync_ofuncs;
+extern struct nv50_disp_chan_impl nvd0_disp_base_ofuncs;
 extern struct nv50_disp_chan_impl nvd0_disp_ovly_ofuncs;
-extern const struct nv50_disp_mthd_chan nvd0_disp_sync_mthd_chan;
+extern const struct nv50_disp_mthd_chan nvd0_disp_base_mthd_chan;
 extern struct nv50_disp_chan_impl nvd0_disp_oimm_ofuncs;
 extern struct nv50_disp_chan_impl nvd0_disp_curs_ofuncs;
-extern struct nouveau_ofuncs nvd0_disp_base_ofuncs;
+extern struct nouveau_ofuncs nvd0_disp_main_ofuncs;
 extern struct nouveau_oclass nvd0_disp_cclass;
 void nvd0_disp_intr_supervisor(struct work_struct *);
 void nvd0_disp_intr(struct nouveau_subdev *);
 extern const struct nvkm_event_func nvd0_disp_vblank_func;
-extern const struct nv50_disp_mthd_chan nve0_disp_mast_mthd_chan;
+extern const struct nv50_disp_mthd_chan nve0_disp_core_mthd_chan;
 extern const struct nv50_disp_mthd_chan nve0_disp_ovly_mthd_chan;
 extern struct nvkm_output_dp_impl nv50_pior_dp_impl;
@@ -242,6 +243,10 @@ int nv94_sor_dp_lnk_pwr(struct nvkm_output_dp *, int);
 extern struct nouveau_oclass *nv94_disp_outp_sclass[];
 extern struct nvkm_output_dp_impl nvd0_sor_dp_impl;
+int nvd0_sor_dp_lnk_ctl(struct nvkm_output_dp *, int, int, bool);
 extern struct nouveau_oclass *nvd0_disp_outp_sclass[];
+void gm204_sor_magic(struct nvkm_output *outp);
+extern struct nvkm_output_dp_impl gm204_sor_dp_impl;
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
index d36284715b2a..13eff5e4ee51 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
@@ -34,7 +34,7 @@
 ******************************************************************************/
 const struct nv50_disp_mthd_list
-nv84_disp_mast_mthd_dac = {
+nv84_disp_core_mthd_dac = {
        .mthd = 0x0080,
        .addr = 0x000008,
        .data = {
@@ -46,7 +46,7 @@ nv84_disp_mast_mthd_dac = {
 };
 const struct nv50_disp_mthd_list
-nv84_disp_mast_mthd_head = {
+nv84_disp_core_mthd_head = {
        .mthd = 0x0400,
        .addr = 0x000540,
        .data = {
@@ -98,15 +98,15 @@ nv84_disp_mast_mthd_head = {
 };
 const struct nv50_disp_mthd_chan
-nv84_disp_mast_mthd_chan = {
+nv84_disp_core_mthd_chan = {
        .name = "Core",
        .addr = 0x000000,
        .data = {
-                { "Global", 1, &nv50_disp_mast_mthd_base },
+                { "Global", 1, &nv50_disp_core_mthd_base },
-                {    "DAC", 3, &nv84_disp_mast_mthd_dac  },
+                {    "DAC", 3, &nv84_disp_core_mthd_dac  },
-                {    "SOR", 2, &nv50_disp_mast_mthd_sor  },
+                {    "SOR", 2, &nv50_disp_core_mthd_sor  },
-                {   "PIOR", 3, &nv50_disp_mast_mthd_pior },
+                {   "PIOR", 3, &nv50_disp_core_mthd_pior },
-                {   "HEAD", 2, &nv84_disp_mast_mthd_head },
+                {   "HEAD", 2, &nv84_disp_core_mthd_head },
                {}
        }
 };
@@ -116,7 +116,7 @@ nv84_disp_mast_mthd_chan = {
 ******************************************************************************/
 static const struct nv50_disp_mthd_list
-nv84_disp_sync_mthd_base = {
+nv84_disp_base_mthd_base = {
        .mthd = 0x0000,
        .addr = 0x000000,
        .data = {
@@ -146,12 +146,12 @@ nv84_disp_sync_mthd_base = {
 };
 const struct nv50_disp_mthd_chan
-nv84_disp_sync_mthd_chan = {
+nv84_disp_base_mthd_chan = {
        .name = "Base",
        .addr = 0x000540,
        .data = {
-                { "Global", 1, &nv84_disp_sync_mthd_base },
+                { "Global", 1, &nv84_disp_base_mthd_base },
-                {  "Image", 2, &nv50_disp_sync_mthd_image },
+                {  "Image", 2, &nv50_disp_base_mthd_image },
                {}
        }
 };
@@ -204,8 +204,8 @@ nv84_disp_ovly_mthd_chan = {
 static struct nouveau_oclass
 nv84_disp_sclass[] = {
-        { G82_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
+        { G82_DISP_CORE_CHANNEL_DMA, &nv50_disp_core_ofuncs.base },
-        { G82_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { G82_DISP_BASE_CHANNEL_DMA, &nv50_disp_base_ofuncs.base },
        { G82_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
        { G82_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
        { G82_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
@@ -213,8 +213,8 @@ nv84_disp_sclass[] = {
 };
 static struct nouveau_oclass
-nv84_disp_base_oclass[] = {
+nv84_disp_main_oclass[] = {
-        { G82_DISP, &nv50_disp_base_ofuncs },
+        { G82_DISP, &nv50_disp_main_ofuncs },
        {}
 };
@@ -240,7 +240,7 @@ nv84_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        if (ret)
                return ret;
-        nv_engine(priv)->sclass = nv84_disp_base_oclass;
+        nv_engine(priv)->sclass = nv84_disp_main_oclass;
        nv_engine(priv)->cclass = &nv50_disp_cclass;
        nv_subdev(priv)->intr = nv50_disp_intr;
        INIT_WORK(&priv->supervisor, nv50_disp_intr_supervisor);
@@ -268,9 +268,9 @@ nv84_disp_oclass = &(struct nv50_disp_impl) {
        },
        .base.vblank = &nv50_disp_vblank_func,
        .base.outp =  nv50_disp_outp_sclass,
-        .mthd.core = &nv84_disp_mast_mthd_chan,
+        .mthd.core = &nv84_disp_core_mthd_chan,
-        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.base = &nv84_disp_base_mthd_chan,
        .mthd.ovly = &nv84_disp_ovly_mthd_chan,
        .mthd.prev = 0x000004,
-        .head.scanoutpos = nv50_disp_base_scanoutpos,
+        .head.scanoutpos = nv50_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
index a117064002b1..2bb7ac5cd0e6 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
@@ -34,7 +34,7 @@
 ******************************************************************************/
 const struct nv50_disp_mthd_list
-nv94_disp_mast_mthd_sor = {
+nv94_disp_core_mthd_sor = {
        .mthd = 0x0040,
        .addr = 0x000008,
        .data = {
@@ -44,15 +44,15 @@ nv94_disp_mast_mthd_sor = {
 };
 const struct nv50_disp_mthd_chan
-nv94_disp_mast_mthd_chan = {
+nv94_disp_core_mthd_chan = {
        .name = "Core",
        .addr = 0x000000,
        .data = {
-                { "Global", 1, &nv50_disp_mast_mthd_base },
+                { "Global", 1, &nv50_disp_core_mthd_base },
-                {    "DAC", 3, &nv84_disp_mast_mthd_dac  },
+                {    "DAC", 3, &nv84_disp_core_mthd_dac  },
-                {    "SOR", 4, &nv94_disp_mast_mthd_sor  },
+                {    "SOR", 4, &nv94_disp_core_mthd_sor  },
-                {   "PIOR", 3, &nv50_disp_mast_mthd_pior },
+                {   "PIOR", 3, &nv50_disp_core_mthd_pior },
-                {   "HEAD", 2, &nv84_disp_mast_mthd_head },
+                {   "HEAD", 2, &nv84_disp_core_mthd_head },
                {}
        }
 };
@@ -63,8 +63,8 @@ nv94_disp_mast_mthd_chan = {
 static struct nouveau_oclass
 nv94_disp_sclass[] = {
-        { GT206_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
+        { GT206_DISP_CORE_CHANNEL_DMA, &nv50_disp_core_ofuncs.base },
-        { GT200_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { GT200_DISP_BASE_CHANNEL_DMA, &nv50_disp_base_ofuncs.base },
        { GT200_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
        { G82_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
        { G82_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
@@ -72,8 +72,8 @@ nv94_disp_sclass[] = {
 };
 static struct nouveau_oclass
-nv94_disp_base_oclass[] = {
+nv94_disp_main_oclass[] = {
-        { GT206_DISP, &nv50_disp_base_ofuncs },
+        { GT206_DISP, &nv50_disp_main_ofuncs },
        {}
 };
@@ -99,7 +99,7 @@ nv94_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        if (ret)
                return ret;
-        nv_engine(priv)->sclass = nv94_disp_base_oclass;
+        nv_engine(priv)->sclass = nv94_disp_main_oclass;
        nv_engine(priv)->cclass = &nv50_disp_cclass;
        nv_subdev(priv)->intr = nv50_disp_intr;
        INIT_WORK(&priv->supervisor, nv50_disp_intr_supervisor);
@@ -134,9 +134,9 @@ nv94_disp_oclass = &(struct nv50_disp_impl) {
        },
        .base.vblank = &nv50_disp_vblank_func,
        .base.outp =  nv94_disp_outp_sclass,
-        .mthd.core = &nv94_disp_mast_mthd_chan,
+        .mthd.core = &nv94_disp_core_mthd_chan,
-        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.base = &nv84_disp_base_mthd_chan,
        .mthd.ovly = &nv84_disp_ovly_mthd_chan,
        .mthd.prev = 0x000004,
-        .head.scanoutpos = nv50_disp_base_scanoutpos,
+        .head.scanoutpos = nv50_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
index c67e68aadd45..b32456c9494f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
@@ -80,8 +80,8 @@ nva0_disp_ovly_mthd_chan = {
 static struct nouveau_oclass
 nva0_disp_sclass[] = {
-        { GT200_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
+        { GT200_DISP_CORE_CHANNEL_DMA, &nv50_disp_core_ofuncs.base },
-        { GT200_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { GT200_DISP_BASE_CHANNEL_DMA, &nv50_disp_base_ofuncs.base },
        { GT200_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
        { G82_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
        { G82_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
@@ -89,8 +89,8 @@ nva0_disp_sclass[] = {
 };
 static struct nouveau_oclass
-nva0_disp_base_oclass[] = {
+nva0_disp_main_oclass[] = {
-        { GT200_DISP, &nv50_disp_base_ofuncs },
+        { GT200_DISP, &nv50_disp_main_ofuncs },
        {}
 };
@@ -116,7 +116,7 @@ nva0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        if (ret)
                return ret;
-        nv_engine(priv)->sclass = nva0_disp_base_oclass;
+        nv_engine(priv)->sclass = nva0_disp_main_oclass;
        nv_engine(priv)->cclass = &nv50_disp_cclass;
        nv_subdev(priv)->intr = nv50_disp_intr;
        INIT_WORK(&priv->supervisor, nv50_disp_intr_supervisor);
@@ -144,9 +144,9 @@ nva0_disp_oclass = &(struct nv50_disp_impl) {
        },
        .base.vblank = &nv50_disp_vblank_func,
        .base.outp =  nv50_disp_outp_sclass,
-        .mthd.core = &nv84_disp_mast_mthd_chan,
+        .mthd.core = &nv84_disp_core_mthd_chan,
-        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.base = &nv84_disp_base_mthd_chan,
        .mthd.ovly = &nva0_disp_ovly_mthd_chan,
        .mthd.prev = 0x000004,
-        .head.scanoutpos = nv50_disp_base_scanoutpos,
+        .head.scanoutpos = nv50_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c b/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
index 22969f355aae..951d79f9b781 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
@@ -35,8 +35,8 @@
 static struct nouveau_oclass
 nva3_disp_sclass[] = {
-        { GT214_DISP_CORE_CHANNEL_DMA, &nv50_disp_mast_ofuncs.base },
+        { GT214_DISP_CORE_CHANNEL_DMA, &nv50_disp_core_ofuncs.base },
-        { GT214_DISP_BASE_CHANNEL_DMA, &nv50_disp_sync_ofuncs.base },
+        { GT214_DISP_BASE_CHANNEL_DMA, &nv50_disp_base_ofuncs.base },
        { GT214_DISP_OVERLAY_CHANNEL_DMA, &nv50_disp_ovly_ofuncs.base },
        { GT214_DISP_OVERLAY, &nv50_disp_oimm_ofuncs.base },
        { GT214_DISP_CURSOR, &nv50_disp_curs_ofuncs.base },
@@ -44,8 +44,8 @@ nva3_disp_sclass[] = {
 };
 static struct nouveau_oclass
-nva3_disp_base_oclass[] = {
+nva3_disp_main_oclass[] = {
-        { GT214_DISP, &nv50_disp_base_ofuncs },
+        { GT214_DISP, &nv50_disp_main_ofuncs },
        {}
 };
@@ -71,7 +71,7 @@ nva3_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        if (ret)
                return ret;
-        nv_engine(priv)->sclass = nva3_disp_base_oclass;
+        nv_engine(priv)->sclass = nva3_disp_main_oclass;
        nv_engine(priv)->cclass = &nv50_disp_cclass;
        nv_subdev(priv)->intr = nv50_disp_intr;
        INIT_WORK(&priv->supervisor, nv50_disp_intr_supervisor);
@@ -100,9 +100,9 @@ nva3_disp_oclass = &(struct nv50_disp_impl) {
        },
        .base.vblank = &nv50_disp_vblank_func,
        .base.outp =  nv94_disp_outp_sclass,
-        .mthd.core = &nv94_disp_mast_mthd_chan,
+        .mthd.core = &nv94_disp_core_mthd_chan,
-        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.base = &nv84_disp_base_mthd_chan,
        .mthd.ovly = &nv84_disp_ovly_mthd_chan,
        .mthd.prev = 0x000004,
-        .head.scanoutpos = nv50_disp_base_scanoutpos,
+        .head.scanoutpos = nv50_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
index 747e64bb9c06..181a2d57e356 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
@@ -51,12 +51,14 @@ nvd0_disp_chan_uevent_fini(struct nvkm_event *event, int type, int index)
 {
        struct nv50_disp_priv *priv = container_of(event, typeof(*priv), uevent);
        nv_mask(priv, 0x610090, 0x00000001 << index, 0x00000000 << index);
+        nv_wr32(priv, 0x61008c, 0x00000001 << index);
 }
 static void
 nvd0_disp_chan_uevent_init(struct nvkm_event *event, int types, int index)
 {
        struct nv50_disp_priv *priv = container_of(event, typeof(*priv), uevent);
+        nv_wr32(priv, 0x61008c, 0x00000001 << index);
        nv_mask(priv, 0x610090, 0x00000001 << index, 0x00000001 << index);
 }
@@ -151,7 +153,7 @@ nvd0_disp_dmac_fini(struct nouveau_object *object, bool suspend)
 ******************************************************************************/
 const struct nv50_disp_mthd_list
-nvd0_disp_mast_mthd_base = {
+nvd0_disp_core_mthd_base = {
        .mthd = 0x0000,
        .addr = 0x000000,
        .data = {
@@ -164,7 +166,7 @@ nvd0_disp_mast_mthd_base = {
 };
 const struct nv50_disp_mthd_list
-nvd0_disp_mast_mthd_dac = {
+nvd0_disp_core_mthd_dac = {
        .mthd = 0x0020,
        .addr = 0x000020,
        .data = {
@@ -177,7 +179,7 @@ nvd0_disp_mast_mthd_dac = {
 };
 const struct nv50_disp_mthd_list
-nvd0_disp_mast_mthd_sor = {
+nvd0_disp_core_mthd_sor = {
        .mthd = 0x0020,
        .addr = 0x000020,
        .data = {
@@ -190,7 +192,7 @@ nvd0_disp_mast_mthd_sor = {
 };
 const struct nv50_disp_mthd_list
-nvd0_disp_mast_mthd_pior = {
+nvd0_disp_core_mthd_pior = {
        .mthd = 0x0020,
        .addr = 0x000020,
        .data = {
@@ -203,7 +205,7 @@ nvd0_disp_mast_mthd_pior = {
 };
 static const struct nv50_disp_mthd_list
-nvd0_disp_mast_mthd_head = {
+nvd0_disp_core_mthd_head = {
        .mthd = 0x0300,
        .addr = 0x000300,
        .data = {
@@ -277,21 +279,21 @@ nvd0_disp_mast_mthd_head = {
 };
 static const struct nv50_disp_mthd_chan
-nvd0_disp_mast_mthd_chan = {
+nvd0_disp_core_mthd_chan = {
        .name = "Core",
        .addr = 0x000000,
        .data = {
-                { "Global", 1, &nvd0_disp_mast_mthd_base },
+                { "Global", 1, &nvd0_disp_core_mthd_base },
-                {    "DAC", 3, &nvd0_disp_mast_mthd_dac  },
+                {    "DAC", 3, &nvd0_disp_core_mthd_dac  },
-                {    "SOR", 8, &nvd0_disp_mast_mthd_sor  },
+                {    "SOR", 8, &nvd0_disp_core_mthd_sor  },
-                {   "PIOR", 4, &nvd0_disp_mast_mthd_pior },
+                {   "PIOR", 4, &nvd0_disp_core_mthd_pior },
-                {   "HEAD", 4, &nvd0_disp_mast_mthd_head },
+                {   "HEAD", 4, &nvd0_disp_core_mthd_head },
                {}
        }
 };
 static int
-nvd0_disp_mast_init(struct nouveau_object *object)
+nvd0_disp_core_init(struct nouveau_object *object)
 {
        struct nv50_disp_priv *priv = (void *)object->engine;
        struct nv50_disp_dmac *mast = (void *)object;
@@ -322,7 +324,7 @@ nvd0_disp_mast_init(struct nouveau_object *object)
 }
 static int
-nvd0_disp_mast_fini(struct nouveau_object *object, bool suspend)
+nvd0_disp_core_fini(struct nouveau_object *object, bool suspend)
 {
        struct nv50_disp_priv *priv = (void *)object->engine;
        struct nv50_disp_dmac *mast = (void *)object;
@@ -344,11 +346,11 @@ nvd0_disp_mast_fini(struct nouveau_object *object, bool suspend)
 }
 struct nv50_disp_chan_impl
-nvd0_disp_mast_ofuncs = {
+nvd0_disp_core_ofuncs = {
-        .base.ctor = nv50_disp_mast_ctor,
+        .base.ctor = nv50_disp_core_ctor,
        .base.dtor = nv50_disp_dmac_dtor,
-        .base.init = nvd0_disp_mast_init,
+        .base.init = nvd0_disp_core_init,
-        .base.fini = nvd0_disp_mast_fini,
+        .base.fini = nvd0_disp_core_fini,
        .base.ntfy = nv50_disp_chan_ntfy,
        .base.map  = nv50_disp_chan_map,
        .base.rd32 = nv50_disp_chan_rd32,
@@ -363,7 +365,7 @@ nvd0_disp_mast_ofuncs = {
 ******************************************************************************/
 static const struct nv50_disp_mthd_list
-nvd0_disp_sync_mthd_base = {
+nvd0_disp_base_mthd_base = {
        .mthd = 0x0000,
        .addr = 0x000000,
        .data = {
@@ -413,7 +415,7 @@ nvd0_disp_sync_mthd_base = {
 };
 static const struct nv50_disp_mthd_list
-nvd0_disp_sync_mthd_image = {
+nvd0_disp_base_mthd_image = {
        .mthd = 0x0400,
        .addr = 0x000400,
        .data = {
@@ -427,19 +429,19 @@ nvd0_disp_sync_mthd_image = {
 };
 const struct nv50_disp_mthd_chan
-nvd0_disp_sync_mthd_chan = {
+nvd0_disp_base_mthd_chan = {
        .name = "Base",
        .addr = 0x001000,
        .data = {
-                { "Global", 1, &nvd0_disp_sync_mthd_base },
+                { "Global", 1, &nvd0_disp_base_mthd_base },
-                {  "Image", 2, &nvd0_disp_sync_mthd_image },
+                {  "Image", 2, &nvd0_disp_base_mthd_image },
                {}
        }
 };
 struct nv50_disp_chan_impl
-nvd0_disp_sync_ofuncs = {
+nvd0_disp_base_ofuncs = {
-        .base.ctor = nv50_disp_sync_ctor,
+        .base.ctor = nv50_disp_base_ctor,
        .base.dtor = nv50_disp_dmac_dtor,
        .base.init = nvd0_disp_dmac_init,
        .base.fini = nvd0_disp_dmac_fini,
@@ -624,7 +626,7 @@ nvd0_disp_curs_ofuncs = {
 ******************************************************************************/
 int
-nvd0_disp_base_scanoutpos(NV50_DISP_MTHD_V0)
+nvd0_disp_main_scanoutpos(NV50_DISP_MTHD_V0)
 {
        const u32 total  = nv_rd32(priv, 0x640414 + (head * 0x300));
        const u32 blanke = nv_rd32(priv, 0x64041c + (head * 0x300));
@@ -656,7 +658,7 @@ nvd0_disp_base_scanoutpos(NV50_DISP_MTHD_V0)
 }
 static int
-nvd0_disp_base_init(struct nouveau_object *object)
+nvd0_disp_main_init(struct nouveau_object *object)
 {
        struct nv50_disp_priv *priv = (void *)object->engine;
        struct nv50_disp_base *base = (void *)object;
@@ -725,7 +727,7 @@ nvd0_disp_base_init(struct nouveau_object *object)
 }
 static int
-nvd0_disp_base_fini(struct nouveau_object *object, bool suspend)
+nvd0_disp_main_fini(struct nouveau_object *object, bool suspend)
 {
        struct nv50_disp_priv *priv = (void *)object->engine;
        struct nv50_disp_base *base = (void *)object;
@@ -737,25 +739,25 @@ nvd0_disp_base_fini(struct nouveau_object *object, bool suspend)
 }
 struct nouveau_ofuncs
-nvd0_disp_base_ofuncs = {
+nvd0_disp_main_ofuncs = {
-        .ctor = nv50_disp_base_ctor,
+        .ctor = nv50_disp_main_ctor,
-        .dtor = nv50_disp_base_dtor,
+        .dtor = nv50_disp_main_dtor,
-        .init = nvd0_disp_base_init,
+        .init = nvd0_disp_main_init,
-        .fini = nvd0_disp_base_fini,
+        .fini = nvd0_disp_main_fini,
-        .mthd = nv50_disp_base_mthd,
+        .mthd = nv50_disp_main_mthd,
        .ntfy = nouveau_disp_ntfy,
 };
 static struct nouveau_oclass
-nvd0_disp_base_oclass[] = {
+nvd0_disp_main_oclass[] = {
-        { GF110_DISP, &nvd0_disp_base_ofuncs },
+        { GF110_DISP, &nvd0_disp_main_ofuncs },
        {}
 };
 static struct nouveau_oclass
 nvd0_disp_sclass[] = {
-        { GF110_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
+        { GF110_DISP_CORE_CHANNEL_DMA, &nvd0_disp_core_ofuncs.base },
-        { GF110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GF110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_base_ofuncs.base },
        { GF110_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
        { GF110_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
        { GF110_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
@@ -1055,6 +1057,9 @@ nvd0_disp_intr_unk2_2(struct nv50_disp_priv *priv, int head)
                if (nvkm_output_dp_train(outp, pclk, true))
                        ERR("link not trained before attach\n");
+        } else {
+                if (priv->sor.magic)
+                        priv->sor.magic(outp);
        }
        exec_clkcmp(priv, head, 0, pclk, &conf);
@@ -1063,10 +1068,18 @@ nvd0_disp_intr_unk2_2(struct nv50_disp_priv *priv, int head)
                addr = 0x612280 + (ffs(outp->info.or) - 1) * 0x800;
                data = 0x00000000;
        } else {
-                if (outp->info.type == DCB_OUTPUT_DP)
-                        nvd0_disp_intr_unk2_2_tu(priv, head, &outp->info);
                addr = 0x612300 + (ffs(outp->info.or) - 1) * 0x800;
                data = (conf & 0x0100) ? 0x00000101 : 0x00000000;
+                switch (outp->info.type) {
+                case DCB_OUTPUT_TMDS:
+                        nv_mask(priv, addr, 0x007c0000, 0x00280000);
+                        break;
+                case DCB_OUTPUT_DP:
+                        nvd0_disp_intr_unk2_2_tu(priv, head, &outp->info);
+                        break;
+                default:
+                        break;
+                }
        }
        nv_mask(priv, addr, 0x00000707, data);
@@ -1259,7 +1272,7 @@ nvd0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        if (ret)
                return ret;
-        nv_engine(priv)->sclass = nvd0_disp_base_oclass;
+        nv_engine(priv)->sclass = nvd0_disp_main_oclass;
        nv_engine(priv)->cclass = &nv50_disp_cclass;
        nv_subdev(priv)->intr = nvd0_disp_intr;
        INIT_WORK(&priv->supervisor, nvd0_disp_intr_supervisor);
@@ -1292,9 +1305,9 @@ nvd0_disp_oclass = &(struct nv50_disp_impl) {
        },
        .base.vblank = &nvd0_disp_vblank_func,
        .base.outp =  nvd0_disp_outp_sclass,
-        .mthd.core = &nvd0_disp_mast_mthd_chan,
+        .mthd.core = &nvd0_disp_core_mthd_chan,
-        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.base = &nvd0_disp_base_mthd_chan,
        .mthd.ovly = &nvd0_disp_ovly_mthd_chan,
        .mthd.prev = -0x020000,
-        .head.scanoutpos = nvd0_disp_base_scanoutpos,
+        .head.scanoutpos = nvd0_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
index db144b2cf06b..55debec7e68f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
@@ -34,7 +34,7 @@
 ******************************************************************************/
 static const struct nv50_disp_mthd_list
-nve0_disp_mast_mthd_head = {
+nve0_disp_core_mthd_head = {
        .mthd = 0x0300,
        .addr = 0x000300,
        .data = {
@@ -113,15 +113,15 @@ nve0_disp_mast_mthd_head = {
 };
 const struct nv50_disp_mthd_chan
-nve0_disp_mast_mthd_chan = {
+nve0_disp_core_mthd_chan = {
        .name = "Core",
        .addr = 0x000000,
        .data = {
-                { "Global", 1, &nvd0_disp_mast_mthd_base },
+                { "Global", 1, &nvd0_disp_core_mthd_base },
-                {    "DAC", 3, &nvd0_disp_mast_mthd_dac  },
+                {    "DAC", 3, &nvd0_disp_core_mthd_dac  },
-                {    "SOR", 8, &nvd0_disp_mast_mthd_sor  },
+                {    "SOR", 8, &nvd0_disp_core_mthd_sor  },
-                {   "PIOR", 4, &nvd0_disp_mast_mthd_pior },
+                {   "PIOR", 4, &nvd0_disp_core_mthd_pior },
-                {   "HEAD", 4, &nve0_disp_mast_mthd_head },
+                {   "HEAD", 4, &nve0_disp_core_mthd_head },
                {}
        }
 };
@@ -200,8 +200,8 @@ nve0_disp_ovly_mthd_chan = {
 static struct nouveau_oclass
 nve0_disp_sclass[] = {
-        { GK104_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
+        { GK104_DISP_CORE_CHANNEL_DMA, &nvd0_disp_core_ofuncs.base },
-        { GK104_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GK104_DISP_BASE_CHANNEL_DMA, &nvd0_disp_base_ofuncs.base },
        { GK104_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
        { GK104_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
        { GK104_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
@@ -209,8 +209,8 @@ nve0_disp_sclass[] = {
 };
 static struct nouveau_oclass
-nve0_disp_base_oclass[] = {
+nve0_disp_main_oclass[] = {
-        { GK104_DISP, &nvd0_disp_base_ofuncs },
+        { GK104_DISP, &nvd0_disp_main_ofuncs },
        {}
 };
@@ -237,7 +237,7 @@ nve0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        if (ret)
                return ret;
-        nv_engine(priv)->sclass = nve0_disp_base_oclass;
+        nv_engine(priv)->sclass = nve0_disp_main_oclass;
        nv_engine(priv)->cclass = &nv50_disp_cclass;
        nv_subdev(priv)->intr = nvd0_disp_intr;
        INIT_WORK(&priv->supervisor, nvd0_disp_intr_supervisor);
@@ -264,9 +264,9 @@ nve0_disp_oclass = &(struct nv50_disp_impl) {
        },
        .base.vblank = &nvd0_disp_vblank_func,
        .base.outp =  nvd0_disp_outp_sclass,
-        .mthd.core = &nve0_disp_mast_mthd_chan,
+        .mthd.core = &nve0_disp_core_mthd_chan,
-        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.base = &nvd0_disp_base_mthd_chan,
        .mthd.ovly = &nve0_disp_ovly_mthd_chan,
        .mthd.prev = -0x020000,
-        .head.scanoutpos = nvd0_disp_base_scanoutpos,
+        .head.scanoutpos = nvd0_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
index 402d7d67d806..3e7e2d28744c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
@@ -35,8 +35,8 @@
 static struct nouveau_oclass
 nvf0_disp_sclass[] = {
-        { GK110_DISP_CORE_CHANNEL_DMA, &nvd0_disp_mast_ofuncs.base },
+        { GK110_DISP_CORE_CHANNEL_DMA, &nvd0_disp_core_ofuncs.base },
-        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_sync_ofuncs.base },
+        { GK110_DISP_BASE_CHANNEL_DMA, &nvd0_disp_base_ofuncs.base },
        { GK104_DISP_OVERLAY_CONTROL_DMA, &nvd0_disp_ovly_ofuncs.base },
        { GK104_DISP_OVERLAY, &nvd0_disp_oimm_ofuncs.base },
        { GK104_DISP_CURSOR, &nvd0_disp_curs_ofuncs.base },
@@ -44,8 +44,8 @@ nvf0_disp_sclass[] = {
 };
 static struct nouveau_oclass
-nvf0_disp_base_oclass[] = {
+nvf0_disp_main_oclass[] = {
-        { GK110_DISP, &nvd0_disp_base_ofuncs },
+        { GK110_DISP, &nvd0_disp_main_ofuncs },
        {}
 };
@@ -72,7 +72,7 @@ nvf0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        if (ret)
                return ret;
-        nv_engine(priv)->sclass = nvf0_disp_base_oclass;
+        nv_engine(priv)->sclass = nvf0_disp_main_oclass;
        nv_engine(priv)->cclass = &nv50_disp_cclass;
        nv_subdev(priv)->intr = nvd0_disp_intr;
        INIT_WORK(&priv->supervisor, nvd0_disp_intr_supervisor);
@@ -99,9 +99,9 @@ nvf0_disp_oclass = &(struct nv50_disp_impl) {
        },
        .base.vblank = &nvd0_disp_vblank_func,
        .base.outp =  nvd0_disp_outp_sclass,
-        .mthd.core = &nve0_disp_mast_mthd_chan,
+        .mthd.core = &nve0_disp_core_mthd_chan,
-        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.base = &nvd0_disp_base_mthd_chan,
        .mthd.ovly = &nve0_disp_ovly_mthd_chan,
        .mthd.prev = -0x020000,
-        .head.scanoutpos = nvd0_disp_base_scanoutpos,
+        .head.scanoutpos = nvd0_disp_main_scanoutpos,
 }.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/outp.c b/drivers/gpu/drm/nouveau/core/engine/disp/outp.c
index a5ff00a9cedc..bbd9b6fdc90f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/outp.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/outp.c
@@ -85,7 +85,10 @@ nvkm_output_create_(struct nouveau_object *parent,
            dcbE->sorconf.link : 0, dcbE->connector, dcbE->i2c_index,
            dcbE->bus, dcbE->heads);
-        outp->port = i2c->find(i2c, outp->info.i2c_index);
+        if (outp->info.type != DCB_OUTPUT_DP)
+                outp->port = i2c->find(i2c, NV_I2C_PORT(outp->info.i2c_index));
+        else
+                outp->port = i2c->find(i2c, NV_I2C_AUX(outp->info.i2c_index));
        outp->edid = outp->port;
        data = nvbios_connEp(bios, outp->info.connector, &ver, &hdr, &connE);
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/sorgm204.c b/drivers/gpu/drm/nouveau/core/engine/disp/sorgm204.c
new file mode 100644
index 000000000000..0b4fad39e9a6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/sorgm204.c
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include <core/os.h>
+#include <subdev/bios.h>
+#include <subdev/bios/dcb.h>
+#include <subdev/bios/dp.h>
+#include <subdev/bios/init.h>
+#include <subdev/timer.h>
+#include "nv50.h"
+static inline u32
+gm204_sor_soff(struct nvkm_output_dp *outp)
+{
+        return (ffs(outp->base.info.or) - 1) * 0x800;
+}
+static inline u32
+gm204_sor_loff(struct nvkm_output_dp *outp)
+{
+        return gm204_sor_soff(outp) + !(outp->base.info.sorconf.link & 1) * 0x80;
+}
+void
+gm204_sor_magic(struct nvkm_output *outp)
+{
+        struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
+        const u32 soff = outp->or * 0x100;
+        const u32 data = outp->or + 1;
+        if (outp->info.sorconf.link & 1)
+                nv_mask(priv, 0x612308 + soff, 0x0000001f, 0x00000000 | data);
+        if (outp->info.sorconf.link & 2)
+                nv_mask(priv, 0x612388 + soff, 0x0000001f, 0x00000010 | data);
+}
+static inline u32
+gm204_sor_dp_lane_map(struct nv50_disp_priv *priv, u8 lane)
+{
+        return lane * 0x08;
+}
+static int
+gm204_sor_dp_pattern(struct nvkm_output_dp *outp, int pattern)
+{
+        struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
+        const u32 soff = gm204_sor_soff(outp);
+        const u32 data = 0x01010101 * pattern;
+        if (outp->base.info.sorconf.link & 1)
+                nv_mask(priv, 0x61c110 + soff, 0x0f0f0f0f, data);
+        else
+                nv_mask(priv, 0x61c12c + soff, 0x0f0f0f0f, data);
+        return 0;
+}
+static int
+gm204_sor_dp_lnk_pwr(struct nvkm_output_dp *outp, int nr)
+{
+        struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
+        const u32 soff = gm204_sor_soff(outp);
+        const u32 loff = gm204_sor_loff(outp);
+        u32 mask = 0, i;
+        for (i = 0; i < nr; i++)
+                mask |= 1 << (gm204_sor_dp_lane_map(priv, i) >> 3);
+        nv_mask(priv, 0x61c130 + loff, 0x0000000f, mask);
+        nv_mask(priv, 0x61c034 + soff, 0x80000000, 0x80000000);
+        nv_wait(priv, 0x61c034 + soff, 0x80000000, 0x00000000);
+        return 0;
+}
+static int
+gm204_sor_dp_drv_ctl(struct nvkm_output_dp *outp, int ln, int vs, int pe, int pc)
+{
+        struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        const u32 shift = gm204_sor_dp_lane_map(priv, ln);
+        const u32 loff = gm204_sor_loff(outp);
+        u32 addr, data[4];
+        u8  ver, hdr, cnt, len;
+        struct nvbios_dpout info;
+        struct nvbios_dpcfg ocfg;
+        addr = nvbios_dpout_match(bios, outp->base.info.hasht,
+                                        outp->base.info.hashm,
+                                 &ver, &hdr, &cnt, &len, &info);
+        if (!addr)
+                return -ENODEV;
+        addr = nvbios_dpcfg_match(bios, addr, pc, vs, pe,
+                                 &ver, &hdr, &cnt, &len, &ocfg);
+        if (!addr)
+                return -EINVAL;
+        data[0] = nv_rd32(priv, 0x61c118 + loff) & ~(0x000000ff << shift);
+        data[1] = nv_rd32(priv, 0x61c120 + loff) & ~(0x000000ff << shift);
+        data[2] = nv_rd32(priv, 0x61c130 + loff);
+        if ((data[2] & 0x0000ff00) < (ocfg.tx_pu << 8) || ln == 0)
+                data[2] = (data[2] & ~0x0000ff00) | (ocfg.tx_pu << 8);
+        nv_wr32(priv, 0x61c118 + loff, data[0] | (ocfg.dc << shift));
+        nv_wr32(priv, 0x61c120 + loff, data[1] | (ocfg.pe << shift));
+        nv_wr32(priv, 0x61c130 + loff, data[2] | (ocfg.tx_pu << 8));
+        data[3] = nv_rd32(priv, 0x61c13c + loff) & ~(0x000000ff << shift);
+        nv_wr32(priv, 0x61c13c + loff, data[3] | (ocfg.pc << shift));
+        return 0;
+}
+struct nvkm_output_dp_impl
+gm204_sor_dp_impl = {
+        .base.base.handle = DCB_OUTPUT_DP,
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nvkm_output_dp_ctor,
+                .dtor = _nvkm_output_dp_dtor,
+                .init = _nvkm_output_dp_init,
+                .fini = _nvkm_output_dp_fini,
+        },
+        .pattern = gm204_sor_dp_pattern,
+        .lnk_pwr = gm204_sor_dp_lnk_pwr,
+        .lnk_ctl = nvd0_sor_dp_lnk_ctl,
+        .drv_ctl = gm204_sor_dp_drv_ctl,
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
index 7b7bbc3e459e..fdab2939070c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
@@ -60,7 +60,7 @@ nvd0_sor_dp_pattern(struct nvkm_output_dp *outp, int pattern)
        return 0;
 }
-static int
+int
 nvd0_sor_dp_lnk_ctl(struct nvkm_output_dp *outp, int nr, int bw, bool ef)
 {
        struct nv50_disp_priv *priv = (void *)nouveau_disp(outp);
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
index 3fc4f0b0eaca..19f5f6522962 100644
--- a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
@@ -51,6 +51,7 @@ nvd0_dmaobj_bind(struct nouveau_dmaobj *dmaobj,
                case GK104_DISP_CORE_CHANNEL_DMA:
                case GK110_DISP_CORE_CHANNEL_DMA:
                case GM107_DISP_CORE_CHANNEL_DMA:
+                case GM204_DISP_CORE_CHANNEL_DMA:
                case GF110_DISP_BASE_CHANNEL_DMA:
                case GK104_DISP_BASE_CHANNEL_DMA:
                case GK110_DISP_BASE_CHANNEL_DMA:
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
index d2f0fd39c145..fc9ef663f25a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
@@ -792,7 +792,7 @@ nve0_fifo_intr_fault(struct nve0_fifo_priv *priv, int unit)
        nouveau_engctx_put(engctx);
 }
-static const struct nouveau_bitfield nve0_fifo_pbdma_intr[] = {
+static const struct nouveau_bitfield nve0_fifo_pbdma_intr_0[] = {
        { 0x00000001, "MEMREQ" },
        { 0x00000002, "MEMACK_TIMEOUT" },
        { 0x00000004, "MEMACK_EXTRA" },
@@ -827,9 +827,10 @@ static const struct nouveau_bitfield nve0_fifo_pbdma_intr[] = {
 };
 static void
-nve0_fifo_intr_pbdma(struct nve0_fifo_priv *priv, int unit)
+nve0_fifo_intr_pbdma_0(struct nve0_fifo_priv *priv, int unit)
 {
-        u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000));
+        u32 mask = nv_rd32(priv, 0x04010c + (unit * 0x2000));
+        u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000)) & mask;
        u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000));
        u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000));
        u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff;
@@ -840,11 +841,12 @@ nve0_fifo_intr_pbdma(struct nve0_fifo_priv *priv, int unit)
        if (stat & 0x00800000) {
                if (!nve0_fifo_swmthd(priv, chid, mthd, data))
                        show &= ~0x00800000;
+                nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008);
        }
        if (show) {
                nv_error(priv, "PBDMA%d:", unit);
-                nouveau_bitfield_print(nve0_fifo_pbdma_intr, show);
+                nouveau_bitfield_print(nve0_fifo_pbdma_intr_0, show);
                pr_cont("\n");
                nv_error(priv,
                         "PBDMA%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n",
@@ -853,10 +855,37 @@ nve0_fifo_intr_pbdma(struct nve0_fifo_priv *priv, int unit)
                         subc, mthd, data);
        }
-        nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008);
        nv_wr32(priv, 0x040108 + (unit * 0x2000), stat);
 }
+static const struct nouveau_bitfield nve0_fifo_pbdma_intr_1[] = {
+        { 0x00000001, "HCE_RE_ILLEGAL_OP" },
+        { 0x00000002, "HCE_RE_ALIGNB" },
+        { 0x00000004, "HCE_PRIV" },
+        { 0x00000008, "HCE_ILLEGAL_MTHD" },
+        { 0x00000010, "HCE_ILLEGAL_CLASS" },
+        {}
+};
+static void
+nve0_fifo_intr_pbdma_1(struct nve0_fifo_priv *priv, int unit)
+{
+        u32 mask = nv_rd32(priv, 0x04014c + (unit * 0x2000));
+        u32 stat = nv_rd32(priv, 0x040148 + (unit * 0x2000)) & mask;
+        u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff;
+        if (stat) {
+                nv_error(priv, "PBDMA%d:", unit);
+                nouveau_bitfield_print(nve0_fifo_pbdma_intr_1, stat);
+                pr_cont("\n");
+                nv_error(priv, "PBDMA%d: ch %d %08x %08x\n", unit, chid,
+                         nv_rd32(priv, 0x040150 + (unit * 0x2000)),
+                         nv_rd32(priv, 0x040154 + (unit * 0x2000)));
+        }
+        nv_wr32(priv, 0x040148 + (unit * 0x2000), stat);
+}
 static void
 nve0_fifo_intr_runlist(struct nve0_fifo_priv *priv)
 {
@@ -939,7 +968,8 @@ nve0_fifo_intr(struct nouveau_subdev *subdev)
                u32 mask = nv_rd32(priv, 0x0025a0);
                while (mask) {
                        u32 unit = __ffs(mask);
-                        nve0_fifo_intr_pbdma(priv, unit);
+                        nve0_fifo_intr_pbdma_0(priv, unit);
+                        nve0_fifo_intr_pbdma_1(priv, unit);
                        nv_wr32(priv, 0x0025a0, (1 << unit));
                        mask &= ~(1 << unit);
                }
@@ -1022,6 +1052,12 @@ nve0_fifo_init(struct nouveau_object *object)
                nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */
        }
+        /* PBDMA[n].HCE */
+        for (i = 0; i < priv->spoon_nr; i++) {
+                nv_wr32(priv, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */
+                nv_wr32(priv, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */
+        }
        nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12);
        nv_wr32(priv, 0x002100, 0xffffffff);
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
index 30fd1dc64f93..17251e4b9e86 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
@@ -1557,7 +1557,7 @@ nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                    nvc0_graph_ctor_fw(priv, "fuc409d", &priv->fuc409d) ||
                    nvc0_graph_ctor_fw(priv, "fuc41ac", &priv->fuc41ac) ||
                    nvc0_graph_ctor_fw(priv, "fuc41ad", &priv->fuc41ad))
-                        return -EINVAL;
+                        return -ENODEV;
                priv->firmware = true;
        }
diff --git a/drivers/gpu/drm/nouveau/core/include/core/device.h b/drivers/gpu/drm/nouveau/core/include/core/device.h
index 1d9d893929bb..2ec2e50d3676 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/device.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/device.h
@@ -16,6 +16,7 @@ enum nv_subdev_type {
         * to during POST.
         */
        NVDEV_SUBDEV_DEVINIT,
+        NVDEV_SUBDEV_IBUS,
        NVDEV_SUBDEV_GPIO,
        NVDEV_SUBDEV_I2C,
        NVDEV_SUBDEV_DEVINIT_LAST = NVDEV_SUBDEV_I2C,
@@ -31,7 +32,6 @@ enum nv_subdev_type {
        NVDEV_SUBDEV_TIMER,
        NVDEV_SUBDEV_FB,
        NVDEV_SUBDEV_LTC,
-        NVDEV_SUBDEV_IBUS,
        NVDEV_SUBDEV_INSTMEM,
        NVDEV_SUBDEV_VM,
        NVDEV_SUBDEV_BAR,
@@ -92,6 +92,7 @@ struct nouveau_device {
                GM100    = 0x110,
        } card_type;
        u32 chipset;
+        u8  chiprev;
        u32 crystal;
        struct nouveau_oclass *oclass[NVDEV_SUBDEV_NR];
@@ -158,6 +159,12 @@ nv_device_is_pci(struct nouveau_device *device)
        return device->pdev != NULL;
 }
+static inline bool
+nv_device_is_cpu_coherent(struct nouveau_device *device)
+{
+        return (!IS_ENABLED(CONFIG_ARM) && nv_device_is_pci(device));
+}
 static inline struct device *
 nv_device_base(struct nouveau_device *device)
 {
diff --git a/drivers/gpu/drm/nouveau/core/include/core/handle.h b/drivers/gpu/drm/nouveau/core/include/core/handle.h
index ceb67d770875..d22a59138a9b 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/handle.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/handle.h
@@ -23,11 +23,6 @@ void nouveau_handle_destroy(struct nouveau_handle *);
 int  nouveau_handle_init(struct nouveau_handle *);
 int  nouveau_handle_fini(struct nouveau_handle *, bool suspend);
-int  nouveau_handle_new(struct nouveau_object *, u32 parent, u32 handle,
-                        u16 oclass, void *data, u32 size,
-                        struct nouveau_object **);
-int  nouveau_handle_del(struct nouveau_object *, u32 parent, u32 handle);
 struct nouveau_object *
 nouveau_handle_ref(struct nouveau_object *, u32 name);
diff --git a/drivers/gpu/drm/nouveau/core/include/core/object.h b/drivers/gpu/drm/nouveau/core/include/core/object.h
index d7039482d6fd..2e2afa502c99 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/object.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/object.h
@@ -203,21 +203,4 @@ nv_memcmp(void *obj, u32 addr, const char *str, u32 len)
        return 0;
 }
-#include <core/handle.h>
-static inline int
-nouveau_object_new(struct nouveau_object *client, u32 parent, u32 handle,
-                   u16 oclass, void *data, u32 size,
-                   struct nouveau_object **pobject)
-{
-        return nouveau_handle_new(client, parent, handle, oclass,
-                                  data, size, pobject);
-}
-static inline int
-nouveau_object_del(struct nouveau_object *client, u32 parent, u32 handle)
-{
-        return nouveau_handle_del(client, parent, handle);
-}
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/disp.h b/drivers/gpu/drm/nouveau/core/include/engine/disp.h
index 7a64f347b385..fc307f1317ff 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/disp.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/disp.h
@@ -31,5 +31,6 @@ extern struct nouveau_oclass *nvd0_disp_oclass;
 extern struct nouveau_oclass *nve0_disp_oclass;
 extern struct nouveau_oclass *nvf0_disp_oclass;
 extern struct nouveau_oclass *gm107_disp_oclass;
+extern struct nouveau_oclass *gm204_disp_oclass;
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/M0203.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/M0203.h
new file mode 100644
index 000000000000..1f84d3612dd8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/M0203.h
@@ -0,0 +1,31 @@
+#ifndef __NVBIOS_M0203_H__
+#define __NVBIOS_M0203_H__
+struct nvbios_M0203T {
+#define M0203T_TYPE_RAMCFG 0x00
+        u8  type;
+        u16 pointer;
+};
+u32 nvbios_M0203Te(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u32 nvbios_M0203Tp(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+                   struct nvbios_M0203T *);
+struct nvbios_M0203E {
+#define M0203E_TYPE_DDR2  0x0
+#define M0203E_TYPE_DDR3  0x1
+#define M0203E_TYPE_GDDR3 0x2
+#define M0203E_TYPE_GDDR5 0x3
+#define M0203E_TYPE_SKIP  0xf
+        u8 type;
+        u8 strap;
+        u8 group;
+};
+u32 nvbios_M0203Ee(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr);
+u32 nvbios_M0203Ep(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr,
+                   struct nvbios_M0203E *);
+u32 nvbios_M0203Em(struct nouveau_bios *, u8 ramcfg, u8 *ver, u8 *hdr,
+                   struct nvbios_M0203E *);
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/i2c.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/i2c.h
index 10b57a19a7de..c9bb112895af 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/bios/i2c.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/i2c.h
@@ -4,11 +4,14 @@
 struct nouveau_bios;
 enum dcb_i2c_type {
-        DCB_I2C_NV04_BIT = 0,
+        /* matches bios type field prior to ccb 4.1 */
-        DCB_I2C_NV4E_BIT = 4,
+        DCB_I2C_NV04_BIT = 0x00,
-        DCB_I2C_NVIO_BIT = 5,
+        DCB_I2C_NV4E_BIT = 0x04,
-        DCB_I2C_NVIO_AUX = 6,
+        DCB_I2C_NVIO_BIT = 0x05,
-        DCB_I2C_UNUSED = 0xff
+        DCB_I2C_NVIO_AUX = 0x06,
+        /* made up - mostly */
+        DCB_I2C_PMGR     = 0x80,
+        DCB_I2C_UNUSED   = 0xff
 };
 struct dcb_i2c_entry {
@@ -16,6 +19,7 @@ struct dcb_i2c_entry {
        u8 drive;
        u8 sense;
        u8 share;
+        u8 auxch;
 };
 u16 dcb_i2c_table(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/image.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/image.h
new file mode 100644
index 000000000000..3348b4580843
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/image.h
@@ -0,0 +1,13 @@
+#ifndef __NVBIOS_IMAGE_H__
+#define __NVBIOS_IMAGE_H__
+struct nvbios_image {
+        u32  base;
+        u32  size;
+        u8   type;
+        bool last;
+};
+bool nvbios_image(struct nouveau_bios *, int, struct nvbios_image *);
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/npde.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/npde.h
new file mode 100644
index 000000000000..b18413d951e5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/npde.h
@@ -0,0 +1,12 @@
+#ifndef __NVBIOS_NPDE_H__
+#define __NVBIOS_NPDE_H__
+struct nvbios_npdeT {
+        u32 image_size;
+        bool last;
+};
+u32 nvbios_npdeTe(struct nouveau_bios *, u32);
+u32 nvbios_npdeTp(struct nouveau_bios *, u32, struct nvbios_npdeT *);
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/pcir.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/pcir.h
new file mode 100644
index 000000000000..3d634a06dca1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/pcir.h
@@ -0,0 +1,18 @@
+#ifndef __NVBIOS_PCIR_H__
+#define __NVBIOS_PCIR_H__
+struct nvbios_pcirT {
+        u16 vendor_id;
+        u16 device_id;
+        u8  class_code[3];
+        u32 image_size;
+        u16 image_rev;
+        u8  image_type;
+        bool last;
+};
+u32 nvbios_pcirTe(struct nouveau_bios *, u32, u8 *ver, u16 *hdr);
+u32 nvbios_pcirTp(struct nouveau_bios *, u32, u8 *ver, u16 *hdr,
+                  struct nvbios_pcirT *);
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/pmu.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/pmu.h
new file mode 100644
index 000000000000..9de593deaea8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/pmu.h
@@ -0,0 +1,37 @@
+#ifndef __NVBIOS_PMU_H__
+#define __NVBIOS_PMU_H__
+struct nvbios_pmuT {
+};
+u32 nvbios_pmuTe(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u32 nvbios_pmuTp(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+                 struct nvbios_pmuT *);
+struct nvbios_pmuE {
+        u8  type;
+        u32 data;
+};
+u32 nvbios_pmuEe(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr);
+u32 nvbios_pmuEp(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr,
+                 struct nvbios_pmuE *);
+struct nvbios_pmuR {
+        u32 boot_addr_pmu;
+        u32 boot_addr;
+        u32 boot_size;
+        u32 code_addr_pmu;
+        u32 code_addr;
+        u32 code_size;
+        u32 init_addr_pmu;
+        u32 data_addr_pmu;
+        u32 data_addr;
+        u32 data_size;
+        u32 args_addr_pmu;
+};
+bool nvbios_pmuRm(struct nouveau_bios *, u8 type, struct nvbios_pmuR *);
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h
index a685bbd04568..4a0e0ceb41ba 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h
@@ -43,8 +43,9 @@ struct nvbios_ramcfg {
                        unsigned ramcfg_10_02_08:1;
                        unsigned ramcfg_10_02_10:1;
                        unsigned ramcfg_10_02_20:1;
-                        unsigned ramcfg_10_02_40:1;
+                        unsigned ramcfg_10_DLLoff:1;
                        unsigned ramcfg_10_03_0f:4;
+                        unsigned ramcfg_10_04_01:1;
                        unsigned ramcfg_10_05:8;
                        unsigned ramcfg_10_06:8;
                        unsigned ramcfg_10_07:8;
@@ -95,9 +96,29 @@ struct nvbios_ramcfg {
        union {
                struct {
                        unsigned timing_10_WR:8;
+                        unsigned timing_10_WTR:8;
                        unsigned timing_10_CL:8;
+                        unsigned timing_10_RC:8;
+                        /*empty: 4 */
+                        unsigned timing_10_RFC:8;        /* Byte 5 */
+                        /*empty: 6 */
+                        unsigned timing_10_RAS:8;        /* Byte 7 */
+                        /*empty: 8 */
+                        unsigned timing_10_RP:8;         /* Byte 9 */
+                        unsigned timing_10_RCDRD:8;
+                        unsigned timing_10_RCDWR:8;
+                        unsigned timing_10_RRD:8;
+                        unsigned timing_10_13:8;
                        unsigned timing_10_ODT:3;
+                        /* empty: 15 */
+                        unsigned timing_10_16:8;
+                        /* empty: 17 */
+                        unsigned timing_10_18:8;
                        unsigned timing_10_CWL:8;
+                        unsigned timing_10_20:8;
+                        unsigned timing_10_21:8;
+                        /* empty: 22, 23 */
+                        unsigned timing_10_24:8;
                };
                struct {
                        unsigned timing_20_2e_03:2;
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h b/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h
index e292271a84e4..e007a9d44683 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h
@@ -30,5 +30,6 @@ extern struct nouveau_oclass *nva3_devinit_oclass;
 extern struct nouveau_oclass *nvaf_devinit_oclass;
 extern struct nouveau_oclass *nvc0_devinit_oclass;
 extern struct nouveau_oclass *gm107_devinit_oclass;
+extern struct nouveau_oclass *gm204_devinit_oclass;
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h b/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
index 1b937c2c25ae..d94ccacb40bf 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
@@ -8,6 +8,8 @@
 #include <subdev/bios/i2c.h>
 #define NV_I2C_PORT(n)    (0x00 + (n))
+#define NV_I2C_AUX(n)     (0x10 + (n))
+#define NV_I2C_EXT(n)     (0x20 + (n))
 #define NV_I2C_DEFAULT(n) (0x80 + (n))
 #define NV_I2C_TYPE_DCBI2C(n) (0x0000 | (n))
@@ -89,6 +91,7 @@ extern struct nouveau_oclass *nv94_i2c_oclass;
 extern struct nouveau_oclass *nvd0_i2c_oclass;
 extern struct nouveau_oclass *gf117_i2c_oclass;
 extern struct nouveau_oclass *nve0_i2c_oclass;
+extern struct nouveau_oclass *gm204_i2c_oclass;
 static inline int
 nv_rdi2cr(struct nouveau_i2c_port *port, u8 addr, u8 reg)
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h b/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h
index bf3d1f611333..f2427bf5aeed 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h
@@ -48,6 +48,8 @@ void nouveau_memx_wait(struct nouveau_memx *,
                       u32 addr, u32 mask, u32 data, u32 nsec);
 void nouveau_memx_nsec(struct nouveau_memx *, u32 nsec);
 void nouveau_memx_wait_vblank(struct nouveau_memx *);
+void nouveau_memx_train(struct nouveau_memx *);
+int  nouveau_memx_train_result(struct nouveau_pwr *, u32 *, int);
 void nouveau_memx_block(struct nouveau_memx *);
 void nouveau_memx_unblock(struct nouveau_memx *);
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/volt.h b/drivers/gpu/drm/nouveau/core/include/subdev/volt.h
index 820b62ffd75b..67db5e58880d 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/volt.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/volt.h
@@ -52,6 +52,7 @@ int  _nouveau_volt_init(struct nouveau_object *);
 #define _nouveau_volt_fini _nouveau_subdev_fini
 extern struct nouveau_oclass nv40_volt_oclass;
+extern struct nouveau_oclass gk20a_volt_oclass;
 int nouveau_voltgpio_init(struct nouveau_volt *);
 int nouveau_voltgpio_get(struct nouveau_volt *);
diff --git a/drivers/gpu/drm/nouveau/core/os.h b/drivers/gpu/drm/nouveau/core/os.h
index ccfa21d72ddc..bdd05ee7ec72 100644
--- a/drivers/gpu/drm/nouveau/core/os.h
+++ b/drivers/gpu/drm/nouveau/core/os.h
@@ -23,6 +23,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/power_supply.h>
 #include <linux/clk.h>
+#include <linux/regulator/consumer.h>
 #include <asm/unaligned.h>
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/M0203.c b/drivers/gpu/drm/nouveau/core/subdev/bios/M0203.c
new file mode 100644
index 000000000000..28906b16d4e5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/M0203.c
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/M0203.h>
+u32
+nvbios_M0203Te(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+        struct bit_entry bit_M;
+        u32 data = 0x00000000;
+        if (!bit_entry(bios, 'M', &bit_M)) {
+                if (bit_M.version == 2 && bit_M.length > 0x04)
+                        data = nv_ro16(bios, bit_M.offset + 0x03);
+                if (data) {
+                        *ver = nv_ro08(bios, data + 0x00);
+                        switch (*ver) {
+                        case 0x10:
+                                *hdr = nv_ro08(bios, data + 0x01);
+                                *len = nv_ro08(bios, data + 0x02);
+                                *cnt = nv_ro08(bios, data + 0x03);
+                                return data;
+                        default:
+                                break;
+                        }
+                }
+        }
+        return 0x00000000;
+}
+u32
+nvbios_M0203Tp(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+               struct nvbios_M0203T *info)
+{
+        u32 data = nvbios_M0203Te(bios, ver, hdr, cnt, len);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!data * *ver) {
+        case 0x10:
+                info->type    = nv_ro08(bios, data + 0x04);
+                info->pointer = nv_ro16(bios, data + 0x05);
+                break;
+        default:
+                break;
+        }
+        return data;
+}
+u32
+nvbios_M0203Ee(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr)
+{
+        u8  cnt, len;
+        u32 data = nvbios_M0203Te(bios, ver, hdr, &cnt, &len);
+        if (data && idx < cnt) {
+                data = data + *hdr + idx * len;
+                *hdr = len;
+                return data;
+        }
+        return 0x00000000;
+}
+u32
+nvbios_M0203Ep(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr,
+               struct nvbios_M0203E *info)
+{
+        u32 data = nvbios_M0203Ee(bios, idx, ver, hdr);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!data * *ver) {
+        case 0x10:
+                info->type  = (nv_ro08(bios, data + 0x00) & 0x0f) >> 0;
+                info->strap = (nv_ro08(bios, data + 0x00) & 0xf0) >> 4;
+                info->group = (nv_ro08(bios, data + 0x01) & 0x0f) >> 0;
+                return data;
+        default:
+                break;
+        }
+        return 0x00000000;
+}
+u32
+nvbios_M0203Em(struct nouveau_bios *bios, u8 ramcfg, u8 *ver, u8 *hdr,
+               struct nvbios_M0203E *info)
+{
+        struct nvbios_M0203T M0203T;
+        u8  cnt, len, idx = 0xff;
+        u32 data;
+        if (!nvbios_M0203Tp(bios, ver, hdr, &cnt, &len, &M0203T)) {
+                nv_warn(bios, "M0203T not found\n");
+                return 0x00000000;
+        }
+        while ((data = nvbios_M0203Ep(bios, ++idx, ver, hdr, info))) {
+                switch (M0203T.type) {
+                case M0203T_TYPE_RAMCFG:
+                        if (info->strap != ramcfg)
+                                continue;
+                        return data;
+                default:
+                        nv_warn(bios, "M0203T type %02x\n", M0203T.type);
+                        return 0x00000000;
+                }
+        }
+        return data;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/base.c b/drivers/gpu/drm/nouveau/core/subdev/bios/base.c
index d45704a2c2df..7df3a273553d 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/base.c
@@ -31,6 +31,8 @@
 #include <subdev/bios/bmp.h>
 #include <subdev/bios/bit.h>
+#include "priv.h"
 u8
 nvbios_checksum(const u8 *data, int size)
 {
@@ -56,362 +58,21 @@ nvbios_findstr(const u8 *data, int size, const char *str, int len)
        return 0;
 }
-#if defined(__powerpc__)
+int
-static void
+nvbios_extend(struct nouveau_bios *bios, u32 length)
-nouveau_bios_shadow_of(struct nouveau_bios *bios)
 {
-        struct pci_dev *pdev = nv_device(bios)->pdev;
+        if (bios->size < length) {
-        struct device_node *dn;
+                u8 *prev = bios->data;
-        const u32 *data;
+                if (!(bios->data = kmalloc(length, GFP_KERNEL))) {
-        int size;
+                        bios->data = prev;
+                        return -ENOMEM;
-        dn = pci_device_to_OF_node(pdev);
-        if (!dn) {
-                nv_info(bios, "Unable to get the OF node\n");
-                return;
-        }
-        data = of_get_property(dn, "NVDA,BMP", &size);
-        if (data && size) {
-                bios->size = size;
-                bios->data = kmalloc(bios->size, GFP_KERNEL);
-                if (bios->data)
-                        memcpy(bios->data, data, size);
-        }
-}
-#endif
-static void
-nouveau_bios_shadow_pramin(struct nouveau_bios *bios)
-{
-        struct nouveau_device *device = nv_device(bios);
-        u64 addr = 0;
-        u32 bar0 = 0;
-        int i;
-        if (device->card_type >= NV_50) {
-                if (device->card_type >= NV_C0 && device->card_type < GM100) {
-                        if (nv_rd32(bios, 0x022500) & 0x00000001)
-                                return;
-                } else
-                if (device->card_type >= GM100) {
-                        if (nv_rd32(bios, 0x021c04) & 0x00000001)
-                                return;
-                }
-                addr = nv_rd32(bios, 0x619f04);
-                if (!(addr & 0x00000008)) {
-                        nv_debug(bios, "... not enabled\n");
-                        return;
                }
-                if ( (addr & 0x00000003) != 1) {
+                memcpy(bios->data, prev, bios->size);
-                        nv_debug(bios, "... not in vram\n");
+                bios->size = length;
-                        return;
+                kfree(prev);
-                }
+                return 1;
-                addr = (addr & 0xffffff00) << 8;
-                if (!addr) {
-                        addr  = (u64)nv_rd32(bios, 0x001700) << 16;
-                        addr += 0xf0000;
-                }
-                bar0 = nv_mask(bios, 0x001700, 0xffffffff, addr >> 16);
-        }
-        /* bail if no rom signature */
-        if (nv_rd08(bios, 0x700000) != 0x55 ||
-            nv_rd08(bios, 0x700001) != 0xaa)
-                goto out;
-        bios->size = nv_rd08(bios, 0x700002) * 512;
-        if (!bios->size)
-                goto out;
-        bios->data = kmalloc(bios->size, GFP_KERNEL);
-        if (bios->data) {
-                for (i = 0; i < bios->size; i++)
-                        nv_wo08(bios, i, nv_rd08(bios, 0x700000 + i));
-        }
-out:
-        if (device->card_type >= NV_50)
-                nv_wr32(bios, 0x001700, bar0);
-}
-static void
-nouveau_bios_shadow_prom(struct nouveau_bios *bios)
-{
-        struct nouveau_device *device = nv_device(bios);
-        u32 pcireg, access;
-        u16 pcir;
-        int i;
-        /* there is no prom on nv4x IGP's */
-        if (device->card_type == NV_40 && device->chipset >= 0x4c)
-                return;
-        /* enable access to rom */
-        if (device->card_type >= NV_50)
-                pcireg = 0x088050;
-        else
-                pcireg = 0x001850;
-        access = nv_mask(bios, pcireg, 0x00000001, 0x00000000);
-        /* WARNING: PROM accesses should always be 32-bits aligned. Other
-         * accesses work on most chipset but do not on Kepler chipsets
-         */
-        /* bail if no rom signature, with a workaround for a PROM reading
-         * issue on some chipsets.  the first read after a period of
-         * inactivity returns the wrong result, so retry the first header
-         * byte a few times before giving up as a workaround
-         */
-        i = 16;
-        do {
-                u32 data = le32_to_cpu(nv_rd32(bios, 0x300000)) & 0xffff;
-                if (data == 0xaa55)
-                        break;
-        } while (i--);
-        if (!i)
-                goto out;
-        /* read entire bios image to system memory */
-        bios->size = (le32_to_cpu(nv_rd32(bios, 0x300000)) >> 16) & 0xff;
-        bios->size = bios->size * 512;
-        if (!bios->size)
-                goto out;
-        bios->data = kmalloc(bios->size, GFP_KERNEL);
-        if (!bios->data)
-                goto out;
-        for (i = 0; i < bios->size; i += 4)
-                ((u32 *)bios->data)[i/4] = nv_rd32(bios, 0x300000 + i);
-        /* check the PCI record header */
-        pcir = nv_ro16(bios, 0x0018);
-        if (bios->data[pcir + 0] != 'P' ||
-            bios->data[pcir + 1] != 'C' ||
-            bios->data[pcir + 2] != 'I' ||
-            bios->data[pcir + 3] != 'R') {
-                bios->size = 0;
-                kfree(bios->data);
-        }
-out:
-        /* disable access to rom */
-        nv_wr32(bios, pcireg, access);
-}
-#if defined(CONFIG_ACPI) && defined(CONFIG_X86)
-int nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len);
-bool nouveau_acpi_rom_supported(struct pci_dev *pdev);
-#else
-static inline bool
-nouveau_acpi_rom_supported(struct pci_dev *pdev) {
-        return false;
-}
-static inline int
-nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len) {
-        return -EINVAL;
-}
-#endif
-static void
-nouveau_bios_shadow_acpi(struct nouveau_bios *bios)
-{
-        struct pci_dev *pdev = nv_device(bios)->pdev;
-        int ret, cnt, i;
-        if (!nouveau_acpi_rom_supported(pdev)) {
-                bios->data = NULL;
-                return;
-        }
-        bios->size = 0;
-        bios->data = kmalloc(4096, GFP_KERNEL);
-        if (bios->data) {
-                if (nouveau_acpi_get_bios_chunk(bios->data, 0, 4096) == 4096)
-                        bios->size = bios->data[2] * 512;
-                kfree(bios->data);
        }
+        return 0;
-        if (!bios->size)
-                return;
-        bios->data = kmalloc(bios->size, GFP_KERNEL);
-        if (bios->data) {
-                /* disobey the acpi spec - much faster on at least w530 ... */
-                ret = nouveau_acpi_get_bios_chunk(bios->data, 0, bios->size);
-                if (ret != bios->size ||
-                    nvbios_checksum(bios->data, bios->size)) {
-                        /* ... that didn't work, ok, i'll be good now */
-                        for (i = 0; i < bios->size; i += cnt) {
-                                cnt = min((bios->size - i), (u32)4096);
-                                ret = nouveau_acpi_get_bios_chunk(bios->data, i, cnt);
-                                if (ret != cnt)
-                                        break;
-                        }
-                }
-        }
-}
-static void
-nouveau_bios_shadow_pci(struct nouveau_bios *bios)
-{
-        struct pci_dev *pdev = nv_device(bios)->pdev;
-        size_t size;
-        if (!pci_enable_rom(pdev)) {
-                void __iomem *rom = pci_map_rom(pdev, &size);
-                if (rom && size) {
-                        bios->data = kmalloc(size, GFP_KERNEL);
-                        if (bios->data) {
-                                memcpy_fromio(bios->data, rom, size);
-                                bios->size = size;
-                        }
-                }
-                if (rom)
-                        pci_unmap_rom(pdev, rom);
-                pci_disable_rom(pdev);
-        }
-}
-static void
-nouveau_bios_shadow_platform(struct nouveau_bios *bios)
-{
-        struct pci_dev *pdev = nv_device(bios)->pdev;
-        size_t size;
-        void __iomem *rom = pci_platform_rom(pdev, &size);
-        if (rom && size) {
-                bios->data = kmalloc(size, GFP_KERNEL);
-                if (bios->data) {
-                        memcpy_fromio(bios->data, rom, size);
-                        bios->size = size;
-                }
-        }
-}
-static int
-nouveau_bios_score(struct nouveau_bios *bios, const bool writeable)
-{
-        if (bios->size < 3 || !bios->data || bios->data[0] != 0x55 ||
-                        bios->data[1] != 0xAA) {
-                nv_info(bios, "... signature not found\n");
-                return 0;
-        }
-        if (nvbios_checksum(bios->data,
-                        min_t(u32, bios->data[2] * 512, bios->size))) {
-                nv_info(bios, "... checksum invalid\n");
-                /* if a ro image is somewhat bad, it's probably all rubbish */
-                return writeable ? 2 : 1;
-        }
-        nv_info(bios, "... appears to be valid\n");
-        return 3;
-}
-struct methods {
-        const char desc[16];
-        void (*shadow)(struct nouveau_bios *);
-        const bool rw;
-        int score;
-        u32 size;
-        u8 *data;
-};
-static int
-nouveau_bios_shadow(struct nouveau_bios *bios)
-{
-        struct methods shadow_methods[] = {
-#if defined(__powerpc__)
-                { "OpenFirmware", nouveau_bios_shadow_of, true, 0, 0, NULL },
-#endif
-                { "PRAMIN", nouveau_bios_shadow_pramin, true, 0, 0, NULL },
-                { "PROM", nouveau_bios_shadow_prom, false, 0, 0, NULL },
-                { "ACPI", nouveau_bios_shadow_acpi, true, 0, 0, NULL },
-                { "PCIROM", nouveau_bios_shadow_pci, true, 0, 0, NULL },
-                { "PLATFORM", nouveau_bios_shadow_platform, true, 0, 0, NULL },
-                {}
-        };
-        struct methods *mthd, *best;
-        const struct firmware *fw;
-        const char *optarg;
-        int optlen, ret;
-        char *source;
-        optarg = nouveau_stropt(nv_device(bios)->cfgopt, "NvBios", &optlen);
-        source = optarg ? kstrndup(optarg, optlen, GFP_KERNEL) : NULL;
-        if (source) {
-                /* try to match one of the built-in methods */
-                mthd = shadow_methods;
-                do {
-                        if (strcasecmp(source, mthd->desc))
-                                continue;
-                        nv_info(bios, "source: %s\n", mthd->desc);
-                        mthd->shadow(bios);
-                        mthd->score = nouveau_bios_score(bios, mthd->rw);
-                        if (mthd->score) {
-                                kfree(source);
-                                return 0;
-                        }
-                } while ((++mthd)->shadow);
-                /* attempt to load firmware image */
-                ret = request_firmware(&fw, source, &nv_device(bios)->pdev->dev);
-                if (ret == 0) {
-                        bios->size = fw->size;
-                        bios->data = kmemdup(fw->data, fw->size, GFP_KERNEL);
-                        release_firmware(fw);
-                        nv_info(bios, "image: %s\n", source);
-                        if (nouveau_bios_score(bios, 1)) {
-                                kfree(source);
-                                return 0;
-                        }
-                        kfree(bios->data);
-                        bios->data = NULL;
-                }
-                nv_error(bios, "source \'%s\' invalid\n", source);
-                kfree(source);
-        }
-        mthd = shadow_methods;
-        do {
-                nv_info(bios, "checking %s for image...\n", mthd->desc);
-                mthd->shadow(bios);
-                mthd->score = nouveau_bios_score(bios, mthd->rw);
-                mthd->size = bios->size;
-                mthd->data = bios->data;
-                bios->data = NULL;
-        } while (mthd->score != 3 && (++mthd)->shadow);
-        mthd = shadow_methods;
-        best = mthd;
-        do {
-                if (mthd->score > best->score) {
-                        kfree(best->data);
-                        best = mthd;
-                }
-        } while ((++mthd)->shadow);
-        if (best->score) {
-                nv_info(bios, "using image from %s\n", best->desc);
-                bios->size = best->size;
-                bios->data = best->data;
-                return 0;
-        }
-        nv_error(bios, "unable to locate usable image\n");
-        return -EINVAL;
 }
 static u8
@@ -472,7 +133,7 @@ nouveau_bios_ctor(struct nouveau_object *parent,
        if (ret)
                return ret;
-        ret = nouveau_bios_shadow(bios);
+        ret = nvbios_shadow(bios);
        if (ret)
                return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c b/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c
index bd8d348385b3..96099aff8b41 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c
@@ -42,7 +42,7 @@ dcb_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
        *ver = nv_ro08(bios, dcb);
-        if (*ver >= 0x41) {
+        if (*ver >= 0x42) {
                nv_warn(bios, "DCB version 0x%02x unknown\n", *ver);
                return 0x0000;
        } else
@@ -157,17 +157,20 @@ dcb_outp_parse(struct nouveau_bios *bios, u8 idx, u8 *ver, u8 *len,
                                        break;
                                }
-                                switch (conf & 0x0f000000) {
+                                outp->dpconf.link_nr = (conf & 0x0f000000) >> 24;
-                                case 0x0f000000:
+                                if (*ver < 0x41) {
-                                        outp->dpconf.link_nr = 4;
+                                        switch (outp->dpconf.link_nr) {
-                                        break;
+                                        case 0x0f:
-                                case 0x03000000:
+                                                outp->dpconf.link_nr = 4;
-                                        outp->dpconf.link_nr = 2;
+                                                break;
-                                        break;
+                                        case 0x03:
-                                case 0x01000000:
+                                                outp->dpconf.link_nr = 2;
-                                default:
+                                                break;
-                                        outp->dpconf.link_nr = 1;
+                                        case 0x01:
-                                        break;
+                                        default:
+                                                outp->dpconf.link_nr = 1;
+                                                break;
+                                        }
                                }
                                /* fall-through... */
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/disp.c b/drivers/gpu/drm/nouveau/core/subdev/bios/disp.c
index 7f16e52d9bea..51f355599694 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/disp.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/disp.c
@@ -40,6 +40,7 @@ nvbios_disp_table(struct nouveau_bios *bios,
                                switch (*ver) {
                                case 0x20:
                                case 0x21:
+                                case 0x22:
                                        *hdr = nv_ro08(bios, data + 0x01);
                                        *len = nv_ro08(bios, data + 0x02);
                                        *cnt = nv_ro08(bios, data + 0x03);
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c b/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c
index f309dd657250..cef53f81f12b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c
@@ -41,6 +41,7 @@ nvbios_dp_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
                                case 0x21:
                                case 0x30:
                                case 0x40:
+                                case 0x41:
                                        *hdr = nv_ro08(bios, data + 0x01);
                                        *len = nv_ro08(bios, data + 0x02);
                                        *cnt = nv_ro08(bios, data + 0x03);
@@ -70,6 +71,7 @@ nvbios_dpout_entry(struct nouveau_bios *bios, u8 idx,
                        *cnt = nv_ro08(bios, outp + 0x04);
                        break;
                case 0x40:
+                case 0x41:
                        *hdr = nv_ro08(bios, data + 0x04);
                        *cnt = 0;
                        *len = 0;
@@ -108,6 +110,7 @@ nvbios_dpout_parse(struct nouveau_bios *bios, u8 idx,
                                info->script[4] = nv_ro16(bios, data + 0x10);
                        break;
                case 0x40:
+                case 0x41:
                        info->flags     = nv_ro08(bios, data + 0x04);
                        info->script[0] = nv_ro16(bios, data + 0x05);
                        info->script[1] = nv_ro16(bios, data + 0x07);
@@ -172,10 +175,11 @@ nvbios_dpcfg_parse(struct nouveau_bios *bios, u16 outp, u8 idx,
                        break;
                case 0x30:
                case 0x40:
+                case 0x41:
                        info->pc    = nv_ro08(bios, data + 0x00);
                        info->dc    = nv_ro08(bios, data + 0x01);
                        info->pe    = nv_ro08(bios, data + 0x02);
-                        info->tx_pu = nv_ro08(bios, data + 0x03);
+                        info->tx_pu = nv_ro08(bios, data + 0x03) & 0x0f;
                        break;
                default:
                        data = 0x0000;
@@ -194,6 +198,10 @@ nvbios_dpcfg_match(struct nouveau_bios *bios, u16 outp, u8 pc, u8 vs, u8 pe,
        u16 data;
        if (*ver >= 0x30) {
+                /*XXX: there's a second set of these on at least 4.1, that
+                 *     i've witnessed nvidia using instead of the first
+                 *     on gm204.  figure out what/why
+                 */
                const u8 vsoff[] = { 0, 4, 7, 9 };
                idx = (pc * 10) + vsoff[vs] + pe;
        } else {
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/extdev.c b/drivers/gpu/drm/nouveau/core/subdev/bios/extdev.c
index b2a676e53580..49285d4f7ca5 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/extdev.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/extdev.c
@@ -90,7 +90,7 @@ nvbios_extdev_find(struct nouveau_bios *bios, enum nvbios_extdev_type type,
        u16 entry;
        i = 0;
-        while (!(entry = nvbios_extdev_entry(bios, i++, &ver, &len))) {
+        while ((entry = nvbios_extdev_entry(bios, i++, &ver, &len))) {
                extdev_parse_entry(bios, entry, func);
                if (func->type == type)
                        return 0;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/i2c.c b/drivers/gpu/drm/nouveau/core/subdev/bios/i2c.c
index cfb9288c6d28..282320ba9264 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/i2c.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/i2c.c
@@ -39,6 +39,11 @@ dcb_i2c_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
                        i2c = nv_ro16(bios, dcb + 4);
        }
+        if (i2c && *ver >= 0x42) {
+                nv_warn(bios, "ccb %02x not supported\n", *ver);
+                return 0x0000;
+        }
        if (i2c && *ver >= 0x30) {
                *ver = nv_ro08(bios, i2c + 0);
                *hdr = nv_ro08(bios, i2c + 1);
@@ -70,14 +75,25 @@ dcb_i2c_parse(struct nouveau_bios *bios, u8 idx, struct dcb_i2c_entry *info)
        u8  ver, len;
        u16 ent = dcb_i2c_entry(bios, idx, &ver, &len);
        if (ent) {
-                info->type  = nv_ro08(bios, ent + 3);
+                if (ver >= 0x41) {
-                info->share = DCB_I2C_UNUSED;
+                        if (!(nv_ro32(bios, ent) & 0x80000000))
-                if (ver < 0x30) {
+                                info->type = DCB_I2C_UNUSED;
-                        info->type &= 0x07;
+                        else
+                                info->type = DCB_I2C_PMGR;
+                } else
+                if (ver >= 0x30) {
+                        info->type = nv_ro08(bios, ent + 0x03);
+                } else {
+                        info->type = nv_ro08(bios, ent + 0x03) & 0x07;
                        if (info->type == 0x07)
                                info->type = DCB_I2C_UNUSED;
                }
+                info->drive = DCB_I2C_UNUSED;
+                info->sense = DCB_I2C_UNUSED;
+                info->share = DCB_I2C_UNUSED;
+                info->auxch = DCB_I2C_UNUSED;
                switch (info->type) {
                case DCB_I2C_NV04_BIT:
                        info->drive = nv_ro08(bios, ent + 0);
@@ -87,12 +103,23 @@ dcb_i2c_parse(struct nouveau_bios *bios, u8 idx, struct dcb_i2c_entry *info)
                        info->drive = nv_ro08(bios, ent + 1);
                        return 0;
                case DCB_I2C_NVIO_BIT:
-                case DCB_I2C_NVIO_AUX:
                        info->drive = nv_ro08(bios, ent + 0) & 0x0f;
-                        if (nv_ro08(bios, ent + 1) & 0x01) {
+                        if (nv_ro08(bios, ent + 1) & 0x01)
-                                info->share  = nv_ro08(bios, ent + 1) >> 1;
+                                info->share = nv_ro08(bios, ent + 1) >> 1;
-                                info->share &= 0x0f;
+                        return 0;
-                        }
+                case DCB_I2C_NVIO_AUX:
+                        info->auxch = nv_ro08(bios, ent + 0) & 0x0f;
+                        if (nv_ro08(bios, ent + 1) & 0x01)
+                                        info->share = info->auxch;
+                        return 0;
+                case DCB_I2C_PMGR:
+                        info->drive = (nv_ro16(bios, ent + 0) & 0x01f) >> 0;
+                        if (info->drive == 0x1f)
+                                info->drive = DCB_I2C_UNUSED;
+                        info->auxch = (nv_ro16(bios, ent + 0) & 0x3e0) >> 5;
+                        if (info->auxch == 0x1f)
+                                info->auxch = DCB_I2C_UNUSED;
+                        info->share = info->auxch;
                        return 0;
                case DCB_I2C_UNUSED:
                        return 0;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/image.c b/drivers/gpu/drm/nouveau/core/subdev/bios/image.c
new file mode 100644
index 000000000000..373f9a564ac9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/image.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include <subdev/bios.h>
+#include <subdev/bios/image.h>
+#include <subdev/bios/pcir.h>
+#include <subdev/bios/npde.h>
+static bool
+nvbios_imagen(struct nouveau_bios *bios, struct nvbios_image *image)
+{
+        struct nvbios_pcirT pcir;
+        struct nvbios_npdeT npde;
+        u8  ver;
+        u16 hdr;
+        u32 data;
+        switch ((data = nv_ro16(bios, image->base + 0x00))) {
+        case 0xaa55:
+        case 0xbb77:
+        case 0x4e56: /* NV */
+                break;
+        default:
+                nv_debug(bios, "%08x: ROM signature (%04x) unknown\n",
+                         image->base, data);
+                return false;
+        }
+        if (!(data = nvbios_pcirTp(bios, image->base, &ver, &hdr, &pcir)))
+                return false;
+        image->size = pcir.image_size;
+        image->type = pcir.image_type;
+        image->last = pcir.last;
+        if (image->type != 0x70) {
+                if (!(data = nvbios_npdeTp(bios, image->base, &npde)))
+                        return true;
+                image->size = npde.image_size;
+                image->last = npde.last;
+        } else {
+                image->last = true;
+        }
+        return true;
+}
+bool
+nvbios_image(struct nouveau_bios *bios, int idx, struct nvbios_image *image)
+{
+        memset(image, 0x00, sizeof(*image));
+        do {
+                image->base += image->size;
+                if (image->last || !nvbios_imagen(bios, image))
+                        return false;
+        } while(idx--);
+        return true;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/init.c b/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
index 626380f9e4c0..c6579ef32cd1 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
@@ -255,6 +255,8 @@ init_i2c(struct nvbios_init *init, int index)
                }
                index = init->outp->i2c_index;
+                if (init->outp->type == DCB_OUTPUT_DP)
+                        index += NV_I2C_AUX(0);
        }
        return i2c->find(i2c, index);
@@ -278,7 +280,7 @@ init_wri2cr(struct nvbios_init *init, u8 index, u8 addr, u8 reg, u8 val)
        return -ENODEV;
 }
-static int
+static u8
 init_rdauxr(struct nvbios_init *init, u32 addr)
 {
        struct nouveau_i2c_port *port = init_i2c(init, -2);
@@ -286,20 +288,24 @@ init_rdauxr(struct nvbios_init *init, u32 addr)
        if (port && init_exec(init)) {
                int ret = nv_rdaux(port, addr, &data, 1);
-                if (ret)
+                if (ret == 0)
-                        return ret;
+                        return data;
-                return data;
+                trace("auxch read failed with %d\n", ret);
        }
-        return -ENODEV;
+        return 0x00;
 }
 static int
 init_wrauxr(struct nvbios_init *init, u32 addr, u8 data)
 {
        struct nouveau_i2c_port *port = init_i2c(init, -2);
-        if (port && init_exec(init))
+        if (port && init_exec(init)) {
-                return nv_wraux(port, addr, &data, 1);
+                int ret = nv_wraux(port, addr, &data, 1);
+                if (ret)
+                        trace("auxch write failed with %d\n", ret);
+                return ret;
+        }
        return -ENODEV;
 }
@@ -838,6 +844,40 @@ init_io_or(struct nvbios_init *init)
 }
 /**
+ * INIT_ANDN_REG - opcode 0x47
+ *
+ */
+static void
+init_andn_reg(struct nvbios_init *init)
+{
+        struct nouveau_bios *bios = init->bios;
+        u32  reg = nv_ro32(bios, init->offset + 1);
+        u32 mask = nv_ro32(bios, init->offset + 5);
+        trace("ANDN_REG\tR[0x%06x] &= ~0x%08x\n", reg, mask);
+        init->offset += 9;
+        init_mask(init, reg, mask, 0);
+}
+/**
+ * INIT_OR_REG - opcode 0x48
+ *
+ */
+static void
+init_or_reg(struct nvbios_init *init)
+{
+        struct nouveau_bios *bios = init->bios;
+        u32  reg = nv_ro32(bios, init->offset + 1);
+        u32 mask = nv_ro32(bios, init->offset + 5);
+        trace("OR_REG\tR[0x%06x] |= 0x%08x\n", reg, mask);
+        init->offset += 9;
+        init_mask(init, reg, 0, mask);
+}
+/**
 * INIT_INDEX_ADDRESS_LATCHED - opcode 0x49
 *
 */
@@ -2068,6 +2108,8 @@ static struct nvbios_init_opcode {
        [0x3a] = { init_dp_condition },
        [0x3b] = { init_io_mask_or },
        [0x3c] = { init_io_or },
+        [0x47] = { init_andn_reg },
+        [0x48] = { init_or_reg },
        [0x49] = { init_idx_addr_latched },
        [0x4a] = { init_io_restrict_pll2 },
        [0x4b] = { init_pll2 },
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/npde.c b/drivers/gpu/drm/nouveau/core/subdev/bios/npde.c
new file mode 100644
index 000000000000..d694716a166c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/npde.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include <subdev/bios.h>
+#include <subdev/bios/npde.h>
+#include <subdev/bios/pcir.h>
+u32
+nvbios_npdeTe(struct nouveau_bios *bios, u32 base)
+{
+        struct nvbios_pcirT pcir;
+        u8  ver; u16 hdr;
+        u32 data = nvbios_pcirTp(bios, base, &ver, &hdr, &pcir);
+        if (data = (data + hdr + 0x0f) & ~0x0f, data) {
+                switch (nv_ro32(bios, data + 0x00)) {
+                case 0x4544504e: /* NPDE */
+                        break;
+                default:
+                        nv_debug(bios, "%08x: NPDE signature (%08x) unknown\n",
+                                 data, nv_ro32(bios, data + 0x00));
+                        data = 0;
+                        break;
+                }
+        }
+        return data;
+}
+u32
+nvbios_npdeTp(struct nouveau_bios *bios, u32 base, struct nvbios_npdeT *info)
+{
+        u32 data = nvbios_npdeTe(bios, base);
+        memset(info, 0x00, sizeof(*info));
+        if (data) {
+                info->image_size = nv_ro16(bios, data + 0x08) * 512;
+                info->last = nv_ro08(bios, data + 0x0a) & 0x80;
+        }
+        return data;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/pcir.c b/drivers/gpu/drm/nouveau/core/subdev/bios/pcir.c
new file mode 100644
index 000000000000..91dae26bc50f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/pcir.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include <subdev/bios.h>
+#include <subdev/bios/pcir.h>
+u32
+nvbios_pcirTe(struct nouveau_bios *bios, u32 base, u8 *ver, u16 *hdr)
+{
+        u32 data = nv_ro16(bios, base + 0x18);
+        if (data) {
+                data += base;
+                switch (nv_ro32(bios, data + 0x00)) {
+                case 0x52494350: /* PCIR */
+                case 0x53494752: /* RGIS */
+                case 0x5344504e: /* NPDS */
+                        *hdr = nv_ro16(bios, data + 0x0a);
+                        *ver = nv_ro08(bios, data + 0x0c);
+                        break;
+                default:
+                        nv_debug(bios, "%08x: PCIR signature (%08x) unknown\n",
+                                 data, nv_ro32(bios, data + 0x00));
+                        data = 0;
+                        break;
+                }
+        }
+        return data;
+}
+u32
+nvbios_pcirTp(struct nouveau_bios *bios, u32 base, u8 *ver, u16 *hdr,
+              struct nvbios_pcirT *info)
+{
+        u32 data = nvbios_pcirTe(bios, base, ver, hdr);
+        memset(info, 0x00, sizeof(*info));
+        if (data) {
+                info->vendor_id = nv_ro16(bios, data + 0x04);
+                info->device_id = nv_ro16(bios, data + 0x06);
+                info->class_code[0] = nv_ro08(bios, data + 0x0d);
+                info->class_code[1] = nv_ro08(bios, data + 0x0e);
+                info->class_code[2] = nv_ro08(bios, data + 0x0f);
+                info->image_size = nv_ro16(bios, data + 0x10) * 512;
+                info->image_rev = nv_ro16(bios, data + 0x12);
+                info->image_type = nv_ro08(bios, data + 0x14);
+                info->last = nv_ro08(bios, data + 0x15) & 0x80;
+        }
+        return data;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/pmu.c b/drivers/gpu/drm/nouveau/core/subdev/bios/pmu.c
new file mode 100644
index 000000000000..66c56ba07d1b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/pmu.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/image.h>
+#include <subdev/bios/pmu.h>
+static u32
+weirdo_pointer(struct nouveau_bios *bios, u32 data)
+{
+        struct nvbios_image image;
+        int idx = 0;
+        if (nvbios_image(bios, idx++, &image)) {
+                data -= image.size;
+                while (nvbios_image(bios, idx++, &image)) {
+                        if (image.type == 0xe0)
+                                return image.base + data;
+                }
+        }
+        return 0;
+}
+u32
+nvbios_pmuTe(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+        struct bit_entry bit_p;
+        u32 data = 0;
+        if (!bit_entry(bios, 'p', &bit_p)) {
+                if (bit_p.version == 2 && bit_p.length >= 4)
+                        data = nv_ro32(bios, bit_p.offset + 0x00);
+                if ((data = weirdo_pointer(bios, data))) {
+                        *ver = nv_ro08(bios, data + 0x00); /* maybe? */
+                        *hdr = nv_ro08(bios, data + 0x01);
+                        *len = nv_ro08(bios, data + 0x02);
+                        *cnt = nv_ro08(bios, data + 0x03);
+                }
+        }
+        return data;
+}
+u32
+nvbios_pmuTp(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+             struct nvbios_pmuT *info)
+{
+        u32 data = nvbios_pmuTe(bios, ver, hdr, cnt, len);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!data * *ver) {
+        default:
+                break;
+        }
+        return data;
+}
+u32
+nvbios_pmuEe(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr)
+{
+        u8  cnt, len;
+        u32 data = nvbios_pmuTe(bios, ver, hdr, &cnt, &len);
+        if (data && idx < cnt) {
+                data = data + *hdr + (idx * len);
+                *hdr = len;
+                return data;
+        }
+        return 0;
+}
+u32
+nvbios_pmuEp(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr,
+             struct nvbios_pmuE *info)
+{
+        u32 data = nvbios_pmuEe(bios, idx, ver, hdr);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!data * *ver) {
+        default:
+                info->type = nv_ro08(bios, data + 0x00);
+                info->data = nv_ro32(bios, data + 0x02);
+                break;
+        }
+        return data;
+}
+bool
+nvbios_pmuRm(struct nouveau_bios *bios, u8 type, struct nvbios_pmuR *info)
+{
+        struct nvbios_pmuE pmuE;
+        u8  ver, hdr, idx = 0;
+        u32 data;
+        memset(info, 0x00, sizeof(*info));
+        while ((data = nvbios_pmuEp(bios, idx++, &ver, &hdr, &pmuE))) {
+                if ( pmuE.type == type &&
+                    (data = weirdo_pointer(bios, pmuE.data))) {
+                        info->init_addr_pmu = nv_ro32(bios, data + 0x08);
+                        info->args_addr_pmu = nv_ro32(bios, data + 0x0c);
+                        info->boot_addr     = data + 0x30;
+                        info->boot_addr_pmu = nv_ro32(bios, data + 0x10) +
+                                              nv_ro32(bios, data + 0x18);
+                        info->boot_size     = nv_ro32(bios, data + 0x1c) -
+                                              nv_ro32(bios, data + 0x18);
+                        info->code_addr     = info->boot_addr + info->boot_size;
+                        info->code_addr_pmu = info->boot_addr_pmu +
+                                              info->boot_size;
+                        info->code_size     = nv_ro32(bios, data + 0x20);
+                        info->data_addr     = data + 0x30 +
+                                              nv_ro32(bios, data + 0x24);
+                        info->data_addr_pmu = nv_ro32(bios, data + 0x28);
+                        info->data_size     = nv_ro32(bios, data + 0x2c);
+                        return true;
+                }
+        }
+        return false;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/priv.h b/drivers/gpu/drm/nouveau/core/subdev/bios/priv.h
new file mode 100644
index 000000000000..187d225bd1e9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/priv.h
@@ -0,0 +1,25 @@
+#ifndef __NVKM_BIOS_PRIV_H__
+#define __NVKM_BIOS_PRIV_H__
+#include <subdev/bios.h>
+struct nvbios_source {
+        const char *name;
+        void *(*init)(struct nouveau_bios *, const char *);
+        void  (*fini)(void *);
+        u32   (*read)(void *, u32 offset, u32 length, struct nouveau_bios *);
+        bool rw;
+};
+int nvbios_extend(struct nouveau_bios *, u32 length);
+int nvbios_shadow(struct nouveau_bios *);
+extern const struct nvbios_source nvbios_rom;
+extern const struct nvbios_source nvbios_ramin;
+extern const struct nvbios_source nvbios_acpi_fast;
+extern const struct nvbios_source nvbios_acpi_slow;
+extern const struct nvbios_source nvbios_pcirom;
+extern const struct nvbios_source nvbios_platform;
+extern const struct nvbios_source nvbios_of;
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c b/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c
index 6c401f70ab99..1623c8dfe797 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c
@@ -25,6 +25,7 @@
 #include <subdev/bios.h>
 #include <subdev/bios/bit.h>
 #include <subdev/bios/ramcfg.h>
+#include <subdev/bios/M0203.h>
 static u8
 nvbios_ramcfg_strap(struct nouveau_subdev *subdev)
@@ -54,12 +55,22 @@ nvbios_ramcfg_index(struct nouveau_subdev *subdev)
        u8 strap = nvbios_ramcfg_strap(subdev);
        u32 xlat = 0x00000000;
        struct bit_entry bit_M;
+        struct nvbios_M0203E M0203E;
+        u8 ver, hdr;
        if (!bit_entry(bios, 'M', &bit_M)) {
                if (bit_M.version == 1 && bit_M.length >= 5)
                        xlat = nv_ro16(bios, bit_M.offset + 3);
-                if (bit_M.version == 2 && bit_M.length >= 3)
+                if (bit_M.version == 2 && bit_M.length >= 3) {
+                        /*XXX: is M ever shorter than this?
+                         *     if not - what is xlat used for now?
+                         *     also - sigh..
+                         */
+                        if (bit_M.length >= 7 &&
+                            nvbios_M0203Em(bios, strap, &ver, &hdr, &M0203E))
+                                return M0203E.group;
                        xlat = nv_ro16(bios, bit_M.offset + 1);
+                }
        }
        if (xlat)
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/rammap.c b/drivers/gpu/drm/nouveau/core/subdev/bios/rammap.c
index 585e69331ccc..c5685228c322 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/rammap.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/rammap.c
@@ -162,8 +162,9 @@ nvbios_rammapSp(struct nouveau_bios *bios, u32 data,
                p->ramcfg_10_02_08 = (nv_ro08(bios, data + 0x02) & 0x08) >> 3;
                p->ramcfg_10_02_10 = (nv_ro08(bios, data + 0x02) & 0x10) >> 4;
                p->ramcfg_10_02_20 = (nv_ro08(bios, data + 0x02) & 0x20) >> 5;
-                p->ramcfg_10_02_40 = (nv_ro08(bios, data + 0x02) & 0x40) >> 6;
+                p->ramcfg_10_DLLoff = (nv_ro08(bios, data + 0x02) & 0x40) >> 6;
                p->ramcfg_10_03_0f = (nv_ro08(bios, data + 0x03) & 0x0f) >> 0;
+                p->ramcfg_10_04_01 = (nv_ro08(bios, data + 0x04) & 0x01) >> 0;
                p->ramcfg_10_05    = (nv_ro08(bios, data + 0x05) & 0xff) >> 0;
                p->ramcfg_10_06    = (nv_ro08(bios, data + 0x06) & 0xff) >> 0;
                p->ramcfg_10_07    = (nv_ro08(bios, data + 0x07) & 0xff) >> 0;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadow.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadow.c
new file mode 100644
index 000000000000..bb9e0018d936
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadow.c
@@ -0,0 +1,270 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "priv.h"
+#include <core/option.h>
+#include <subdev/bios/image.h>
+struct shadow {
+        struct nouveau_oclass base;
+        u32 skip;
+        const struct nvbios_source *func;
+        void *data;
+        u32 size;
+        int score;
+};
+static bool
+shadow_fetch(struct nouveau_bios *bios, u32 upto)
+{
+        struct shadow *mthd = (void *)nv_object(bios)->oclass;
+        const u32 limit = (upto + 3) & ~3;
+        const u32 start = bios->size;
+        void *data = mthd->data;
+        if (nvbios_extend(bios, limit) > 0) {
+                u32 read = mthd->func->read(data, start, limit - start, bios);
+                bios->size = start + read;
+        }
+        return bios->size >= limit;
+}
+static u8
+shadow_rd08(struct nouveau_object *object, u64 addr)
+{
+        struct nouveau_bios *bios = (void *)object;
+        if (shadow_fetch(bios, addr + 1))
+                return bios->data[addr];
+        return 0x00;
+}
+static u16
+shadow_rd16(struct nouveau_object *object, u64 addr)
+{
+        struct nouveau_bios *bios = (void *)object;
+        if (shadow_fetch(bios, addr + 2))
+                return get_unaligned_le16(&bios->data[addr]);
+        return 0x0000;
+}
+static u32
+shadow_rd32(struct nouveau_object *object, u64 addr)
+{
+        struct nouveau_bios *bios = (void *)object;
+        if (shadow_fetch(bios, addr + 4))
+                return get_unaligned_le32(&bios->data[addr]);
+        return 0x00000000;
+}
+static struct nouveau_oclass
+shadow_class = {
+        .handle = NV_SUBDEV(VBIOS, 0x00),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .rd08 = shadow_rd08,
+                .rd16 = shadow_rd16,
+                .rd32 = shadow_rd32,
+        },
+};
+static int
+shadow_image(struct nouveau_bios *bios, int idx, struct shadow *mthd)
+{
+        struct nvbios_image image;
+        int score = 1;
+        if (!nvbios_image(bios, idx, &image)) {
+                nv_debug(bios, "image %d invalid\n", idx);
+                return 0;
+        }
+        nv_debug(bios, "%08x: type %02x, %d bytes\n",
+                 image.base, image.type, image.size);
+        if (!shadow_fetch(bios, image.size)) {
+                nv_debug(bios, "%08x: fetch failed\n", image.base);
+                return 0;
+        }
+        switch (image.type) {
+        case 0x00:
+                if (nvbios_checksum(&bios->data[image.base], image.size)) {
+                        nv_debug(bios, "%08x: checksum failed\n", image.base);
+                        if (mthd->func->rw)
+                                score += 1;
+                        score += 1;
+                } else {
+                        score += 3;
+                }
+                break;
+        default:
+                score += 3;
+                break;
+        }
+        if (!image.last)
+                score += shadow_image(bios, idx + 1, mthd);
+        return score;
+}
+static int
+shadow_score(struct nouveau_bios *bios, struct shadow *mthd)
+{
+        struct nouveau_oclass *oclass = nv_object(bios)->oclass;
+        int score;
+        nv_object(bios)->oclass = &mthd->base;
+        score = shadow_image(bios, 0, mthd);
+        nv_object(bios)->oclass = oclass;
+        return score;
+}
+static int
+shadow_method(struct nouveau_bios *bios, struct shadow *mthd, const char *name)
+{
+        const struct nvbios_source *func = mthd->func;
+        if (func->name) {
+                nv_debug(bios, "trying %s...\n", name ? name : func->name);
+                if (func->init) {
+                        mthd->data = func->init(bios, name);
+                        if (IS_ERR(mthd->data)) {
+                                mthd->data = NULL;
+                                return 0;
+                        }
+                }
+                mthd->score = shadow_score(bios, mthd);
+                if (func->fini)
+                        func->fini(mthd->data);
+                nv_debug(bios, "scored %d\n", mthd->score);
+                mthd->data = bios->data;
+                mthd->size = bios->size;
+                bios->data  = NULL;
+                bios->size  = 0;
+        }
+        return mthd->score;
+}
+static u32
+shadow_fw_read(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        const struct firmware *fw = data;
+        if (offset + length <= fw->size) {
+                memcpy(bios->data + offset, fw->data + offset, length);
+                return length;
+        }
+        return 0;
+}
+static void *
+shadow_fw_init(struct nouveau_bios *bios, const char *name)
+{
+        struct device *dev = &nv_device(bios)->pdev->dev;
+        const struct firmware *fw;
+        int ret = request_firmware(&fw, name, dev);
+        if (ret)
+                return ERR_PTR(-ENOENT);
+        return (void *)fw;
+}
+static const struct nvbios_source
+shadow_fw = {
+        .name = "firmware",
+        .init = shadow_fw_init,
+        .fini = (void(*)(void *))release_firmware,
+        .read = shadow_fw_read,
+        .rw = false,
+};
+int
+nvbios_shadow(struct nouveau_bios *bios)
+{
+        struct shadow mthds[] = {
+                { shadow_class, 0, &nvbios_of },
+                { shadow_class, 0, &nvbios_ramin },
+                { shadow_class, 0, &nvbios_rom },
+                { shadow_class, 0, &nvbios_acpi_fast },
+                { shadow_class, 4, &nvbios_acpi_slow },
+                { shadow_class, 1, &nvbios_pcirom },
+                { shadow_class, 1, &nvbios_platform },
+                { shadow_class }
+        }, *mthd = mthds, *best = NULL;
+        const char *optarg;
+        char *source;
+        int optlen;
+        /* handle user-specified bios source */
+        optarg = nouveau_stropt(nv_device(bios)->cfgopt, "NvBios", &optlen);
+        source = optarg ? kstrndup(optarg, optlen, GFP_KERNEL) : NULL;
+        if (source) {
+                /* try to match one of the built-in methods */
+                for (mthd = mthds; mthd->func; mthd++) {
+                        if (mthd->func->name &&
+                            !strcasecmp(source, mthd->func->name)) {
+                                best = mthd;
+                                if (shadow_method(bios, mthd, NULL))
+                                        break;
+                        }
+                }
+                /* otherwise, attempt to load as firmware */
+                if (!best && (best = mthd)) {
+                        mthd->func = &shadow_fw;
+                        shadow_method(bios, mthd, source);
+                        mthd->func = NULL;
+                }
+                if (!best->score) {
+                        nv_error(bios, "%s invalid\n", source);
+                        kfree(source);
+                        source = NULL;
+                }
+        }
+        /* scan all potential bios sources, looking for best image */
+        if (!best || !best->score) {
+                for (mthd = mthds, best = mthd; mthd->func; mthd++) {
+                        if (!mthd->skip || best->score < mthd->skip) {
+                                if (shadow_method(bios, mthd, NULL)) {
+                                        if (mthd->score > best->score)
+                                                best = mthd;
+                                }
+                        }
+                }
+        }
+        /* cleanup the ones we didn't use */
+        for (mthd = mthds; mthd->func; mthd++) {
+                if (mthd != best)
+                        kfree(mthd->data);
+        }
+        if (!best->score) {
+                nv_fatal(bios, "unable to locate usable image\n");
+                return -EINVAL;
+        }
+        nv_info(bios, "using image from %s\n", best->func ?
+                best->func->name : source);
+        bios->data = best->data;
+        bios->size = best->size;
+        kfree(source);
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadowacpi.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowacpi.c
new file mode 100644
index 000000000000..bc130c12ec06
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowacpi.c
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "priv.h"
+#if defined(CONFIG_ACPI) && defined(CONFIG_X86)
+int nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len);
+bool nouveau_acpi_rom_supported(struct pci_dev *pdev);
+#else
+static inline bool
+nouveau_acpi_rom_supported(struct pci_dev *pdev)
+{
+        return false;
+}
+static inline int
+nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len)
+{
+        return -EINVAL;
+}
+#endif
+/* This version of the shadow function disobeys the ACPI spec and tries
+ * to fetch in units of more than 4KiB at a time.  This is a LOT faster
+ * on some systems, such as Lenovo W530.
+ */
+static u32
+acpi_read_fast(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        u32 limit = (offset + length + 0xfff) & ~0xfff;
+        u32 start = offset & ~0x00000fff;
+        u32 fetch = limit - start;
+        if (nvbios_extend(bios, limit) > 0) {
+                int ret = nouveau_acpi_get_bios_chunk(bios->data, start, fetch);
+                if (ret == fetch)
+                        return fetch;
+        }
+        return 0;
+}
+/* Other systems, such as the one in fdo#55948, will report a success
+ * but only return 4KiB of data.  The common bios fetching logic will
+ * detect an invalid image, and fall back to this version of the read
+ * function.
+ */
+static u32
+acpi_read_slow(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        u32 limit = (offset + length + 0xfff) & ~0xfff;
+        u32 start = offset & ~0xfff;
+        u32 fetch = 0;
+        if (nvbios_extend(bios, limit) > 0) {
+                while (start + fetch < limit) {
+                        int ret = nouveau_acpi_get_bios_chunk(bios->data,
+                                                              start + fetch,
+                                                              0x1000);
+                        if (ret != 0x1000)
+                                break;
+                        fetch += 0x1000;
+                }
+        }
+        return fetch;
+}
+static void *
+acpi_init(struct nouveau_bios *bios, const char *name)
+{
+        if (!nouveau_acpi_rom_supported(nv_device(bios)->pdev))
+                return ERR_PTR(-ENODEV);
+        return NULL;
+}
+const struct nvbios_source
+nvbios_acpi_fast = {
+        .name = "ACPI",
+        .init = acpi_init,
+        .read = acpi_read_fast,
+        .rw = false,
+};
+const struct nvbios_source
+nvbios_acpi_slow = {
+        .name = "ACPI",
+        .init = acpi_init,
+        .read = acpi_read_slow,
+        .rw = false,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadowof.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowof.c
new file mode 100644
index 000000000000..3abe487a6025
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowof.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "priv.h"
+#if defined(__powerpc__)
+struct priv {
+        const void __iomem *data;
+        int size;
+};
+static u32
+of_read(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        struct priv *priv = data;
+        if (offset + length <= priv->size) {
+                memcpy_fromio(bios->data + offset, priv->data + offset, length);
+                return length;
+        }
+        return 0;
+}
+static void *
+of_init(struct nouveau_bios *bios, const char *name)
+{
+        struct pci_dev *pdev = nv_device(bios)->pdev;
+        struct device_node *dn;
+        struct priv *priv;
+        if (!(dn = pci_device_to_OF_node(pdev)))
+                return ERR_PTR(-ENODEV);
+        if (!(priv = kzalloc(sizeof(*priv), GFP_KERNEL)))
+                return ERR_PTR(-ENOMEM);
+        if ((priv->data = of_get_property(dn, "NVDA,BMP", &priv->size)))
+                return priv;
+        kfree(priv);
+        return ERR_PTR(-EINVAL);
+}
+const struct nvbios_source
+nvbios_of = {
+        .name = "OpenFirmware",
+        .init = of_init,
+        .fini = (void(*)(void *))kfree,
+        .read = of_read,
+        .rw = false,
+};
+#else
+const struct nvbios_source
+nvbios_of = {
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadowpci.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowpci.c
new file mode 100644
index 000000000000..1d0389c0abef
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowpci.c
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "priv.h"
+struct priv {
+        struct pci_dev *pdev;
+        void __iomem *rom;
+        size_t size;
+};
+static u32
+pcirom_read(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        struct priv *priv = data;
+        if (offset + length <= priv->size) {
+                memcpy_fromio(bios->data + offset, priv->rom + offset, length);
+                return length;
+        }
+        return 0;
+}
+static void
+pcirom_fini(void *data)
+{
+        struct priv *priv = data;
+        pci_unmap_rom(priv->pdev, priv->rom);
+        pci_disable_rom(priv->pdev);
+        kfree(priv);
+}
+static void *
+pcirom_init(struct nouveau_bios *bios, const char *name)
+{
+        struct pci_dev *pdev = nv_device(bios)->pdev;
+        struct priv *priv = NULL;
+        int ret;
+        if (!(ret = pci_enable_rom(pdev))) {
+                if (ret = -ENOMEM,
+                    (priv = kmalloc(sizeof(*priv), GFP_KERNEL))) {
+                        if (ret = -EFAULT,
+                            (priv->rom = pci_map_rom(pdev, &priv->size))) {
+                                priv->pdev = pdev;
+                                return priv;
+                        }
+                        kfree(priv);
+                }
+                pci_disable_rom(pdev);
+        }
+        return ERR_PTR(ret);
+}
+const struct nvbios_source
+nvbios_pcirom = {
+        .name = "PCIROM",
+        .init = pcirom_init,
+        .fini = pcirom_fini,
+        .read = pcirom_read,
+        .rw = true,
+};
+static void *
+platform_init(struct nouveau_bios *bios, const char *name)
+{
+        struct pci_dev *pdev = nv_device(bios)->pdev;
+        struct priv *priv;
+        int ret = -ENOMEM;
+        if ((priv = kmalloc(sizeof(*priv), GFP_KERNEL))) {
+                if (ret = -ENODEV,
+                    (priv->rom = pci_platform_rom(pdev, &priv->size)))
+                        return priv;
+                kfree(priv);
+        }
+        return ERR_PTR(ret);
+}
+const struct nvbios_source
+nvbios_platform = {
+        .name = "PLATFORM",
+        .init = platform_init,
+        .fini = (void(*)(void *))kfree,
+        .read = pcirom_read,
+        .rw = true,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadowramin.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowramin.c
new file mode 100644
index 000000000000..5e58bba0dd5c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowramin.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "priv.h"
+struct priv {
+        struct nouveau_bios *bios;
+        u32 bar0;
+};
+static u32
+pramin_read(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        u32 i;
+        if (offset + length <= 0x00100000) {
+                for (i = offset; i < offset + length; i += 4)
+                        *(u32 *)&bios->data[i] = nv_rd32(bios, 0x700000 + i);
+                return length;
+        }
+        return 0;
+}
+static void
+pramin_fini(void *data)
+{
+        struct priv *priv = data;
+        nv_wr32(priv->bios, 0x001700, priv->bar0);
+        kfree(priv);
+}
+static void *
+pramin_init(struct nouveau_bios *bios, const char *name)
+{
+        struct priv *priv = NULL;
+        u64 addr = 0;
+        /* PRAMIN always potentially available prior to nv50 */
+        if (nv_device(bios)->card_type < NV_50)
+                return NULL;
+        /* we can't get the bios image pointer without PDISP */
+        if (nv_device(bios)->card_type >= GM100)
+                addr = nv_rd32(bios, 0x021c04);
+        else
+        if (nv_device(bios)->card_type >= NV_C0)
+                addr = nv_rd32(bios, 0x022500);
+        if (addr & 0x00000001) {
+                nv_debug(bios, "... display disabled\n");
+                return ERR_PTR(-ENODEV);
+        }
+        /* check that the window is enabled and in vram, particularly
+         * important as we don't want to be touching vram on an
+         * uninitialised board
+         */
+        addr = nv_rd32(bios, 0x619f04);
+        if (!(addr & 0x00000008)) {
+                nv_debug(bios, "... not enabled\n");
+                return ERR_PTR(-ENODEV);
+        }
+        if ( (addr & 0x00000003) != 1) {
+                nv_debug(bios, "... not in vram\n");
+                return ERR_PTR(-ENODEV);
+        }
+        /* some alternate method inherited from xf86-video-nv... */
+        addr = (addr & 0xffffff00) << 8;
+        if (!addr) {
+                addr  = (u64)nv_rd32(bios, 0x001700) << 16;
+                addr += 0xf0000;
+        }
+        /* modify bar0 PRAMIN window to cover the bios image */
+        if (!(priv = kmalloc(sizeof(*priv), GFP_KERNEL))) {
+                nv_error(bios, "... out of memory\n");
+                return ERR_PTR(-ENOMEM);
+        }
+        priv->bios = bios;
+        priv->bar0 = nv_rd32(bios, 0x001700);
+        nv_wr32(bios, 0x001700, addr >> 16);
+        return priv;
+}
+const struct nvbios_source
+nvbios_ramin = {
+        .name = "PRAMIN",
+        .init = pramin_init,
+        .fini = pramin_fini,
+        .read = pramin_read,
+        .rw = true,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/shadowrom.c b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowrom.c
new file mode 100644
index 000000000000..b7992bc3ffa5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/shadowrom.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "priv.h"
+static u32
+prom_read(void *data, u32 offset, u32 length, struct nouveau_bios *bios)
+{
+        u32 i;
+        if (offset + length <= 0x00100000) {
+                for (i = offset; i < offset + length; i += 4)
+                        *(u32 *)&bios->data[i] = nv_rd32(bios, 0x300000 + i);
+                return length;
+        }
+        return 0;
+}
+static void
+prom_fini(void *data)
+{
+        struct nouveau_bios *bios = data;
+        if (nv_device(bios)->card_type < NV_50)
+                nv_mask(bios, 0x001850, 0x00000001, 0x00000001);
+        else
+                nv_mask(bios, 0x088050, 0x00000001, 0x00000001);
+}
+static void *
+prom_init(struct nouveau_bios *bios, const char *name)
+{
+        if (nv_device(bios)->card_type < NV_50) {
+                if (nv_device(bios)->card_type == NV_40 &&
+                    nv_device(bios)->chipset >= 0x4c)
+                        return ERR_PTR(-ENODEV);
+                nv_mask(bios, 0x001850, 0x00000001, 0x00000000);
+        } else {
+                nv_mask(bios, 0x088050, 0x00000001, 0x00000000);
+        }
+        return bios;
+}
+const struct nvbios_source
+nvbios_rom = {
+        .name = "PROM",
+        .init = prom_init,
+        .fini = prom_fini,
+        .read = prom_read,
+        .rw = false,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/timing.c b/drivers/gpu/drm/nouveau/core/subdev/bios/timing.c
index 46d955eb51eb..8521eca1ed9c 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/timing.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/timing.c
@@ -93,10 +93,44 @@ nvbios_timingEp(struct nouveau_bios *bios, int idx,
        p->timing_hdr = *hdr;
        switch (!!data * *ver) {
        case 0x10:
-                p->timing_10_WR = nv_ro08(bios, data + 0x00);
+                p->timing_10_WR    = nv_ro08(bios, data + 0x00);
-                p->timing_10_CL = nv_ro08(bios, data + 0x02);
+                p->timing_10_WTR   = nv_ro08(bios, data + 0x01);
-                p->timing_10_ODT = nv_ro08(bios, data + 0x0e) & 0x07;
+                p->timing_10_CL    = nv_ro08(bios, data + 0x02);
-                p->timing_10_CWL = nv_ro08(bios, data + 0x13);
+                p->timing_10_RC    = nv_ro08(bios, data + 0x03);
+                p->timing_10_RFC   = nv_ro08(bios, data + 0x05);
+                p->timing_10_RAS   = nv_ro08(bios, data + 0x07);
+                p->timing_10_RP    = nv_ro08(bios, data + 0x09);
+                p->timing_10_RCDRD = nv_ro08(bios, data + 0x0a);
+                p->timing_10_RCDWR = nv_ro08(bios, data + 0x0b);
+                p->timing_10_RRD   = nv_ro08(bios, data + 0x0c);
+                p->timing_10_13    = nv_ro08(bios, data + 0x0d);
+                p->timing_10_ODT   = nv_ro08(bios, data + 0x0e) & 0x07;
+                p->timing_10_24  = 0xff;
+                p->timing_10_21  = 0;
+                p->timing_10_20  = 0;
+                p->timing_10_CWL = 0;
+                p->timing_10_18  = 0;
+                p->timing_10_16  = 0;
+                switch (min_t(u8, *hdr, 25)) {
+                case 25:
+                        p->timing_10_24  = nv_ro08(bios, data + 0x18);
+                case 24:
+                case 23:
+                case 22:
+                        p->timing_10_21  = nv_ro08(bios, data + 0x15);
+                case 21:
+                        p->timing_10_20  = nv_ro08(bios, data + 0x14);
+                case 20:
+                        p->timing_10_CWL = nv_ro08(bios, data + 0x13);
+                case 19:
+                        p->timing_10_18  = nv_ro08(bios, data + 0x12);
+                case 18:
+                case 17:
+                        p->timing_10_16  = nv_ro08(bios, data + 0x10);
+                }
                break;
        case 0x20:
                p->timing[0] = nv_ro32(bios, data + 0x00);
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/gk20a.c b/drivers/gpu/drm/nouveau/core/subdev/clock/gk20a.c
index 425a8d5e9129..fb4fad374bdd 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/gk20a.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/gk20a.c
@@ -109,7 +109,7 @@ struct gk20a_clk_pllg_params {
 };
 static const struct gk20a_clk_pllg_params gk20a_pllg_params = {
-        .min_vco = 1000, .max_vco = 1700,
+        .min_vco = 1000, .max_vco = 2064,
        .min_u = 12, .max_u = 38,
        .min_m = 1, .max_m = 255,
        .min_n = 8, .max_n = 255,
@@ -470,76 +470,91 @@ gk20a_pstates[] = {
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 72000,
+                        .voltage = 0,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 108000,
+                        .voltage = 1,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 180000,
+                        .voltage = 2,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 252000,
+                        .voltage = 3,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 324000,
+                        .voltage = 4,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 396000,
+                        .voltage = 5,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 468000,
+                        .voltage = 6,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 540000,
+                        .voltage = 7,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 612000,
+                        .voltage = 8,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 648000,
+                        .voltage = 9,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 684000,
+                        .voltage = 10,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 708000,
+                        .voltage = 11,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 756000,
+                        .voltage = 12,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 804000,
+                        .voltage = 13,
                },
        },
        {
                .base = {
                        .domain[nv_clk_src_gpc] = 852000,
+                        .voltage = 14,
                },
        },
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
index 094551d8ad9b..07ad01247675 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
@@ -510,7 +510,7 @@ nva3_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        int ret;
        ret = nouveau_clock_create(parent, engine, oclass, nva3_domain, NULL, 0,
-                                   false, &priv);
+                                   true, &priv);
        *pobject = nv_object(priv);
        if (ret)
                return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c
index 239acfe876c3..0e45cee82463 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c
@@ -24,8 +24,6 @@
 #include <core/option.h>
-#include <subdev/bios.h>
-#include <subdev/bios/init.h>
 #include <subdev/vga.h>
 #include "priv.h"
@@ -56,7 +54,7 @@ _nouveau_devinit_init(struct nouveau_object *object)
        if (ret)
                return ret;
-        ret = nvbios_init(&devinit->base, devinit->post);
+        ret = impl->post(&devinit->base, devinit->post);
        if (ret)
                return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/gm107.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/gm107.c
index c69bc7f54e37..4ba43d6a1ec8 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/gm107.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/gm107.c
@@ -24,7 +24,7 @@
 #include "nv50.h"
-static u64
+u64
 gm107_devinit_disable(struct nouveau_devinit *devinit)
 {
        struct nv50_devinit_priv *priv = (void *)devinit;
@@ -53,4 +53,5 @@ gm107_devinit_oclass = &(struct nouveau_devinit_impl) {
        },
        .pll_set = nvc0_devinit_pll_set,
        .disable = gm107_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/gm204.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/gm204.c
new file mode 100644
index 000000000000..e44a86662a2a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/gm204.c
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/pmu.h>
+#include "nv50.h"
+static void
+pmu_code(struct nv50_devinit_priv *priv, u32 pmu, u32 img, u32 len, bool sec)
+{
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        int i;
+        nv_wr32(priv, 0x10a180, 0x01000000 | (sec ? 0x10000000 : 0) | pmu);
+        for (i = 0; i < len; i += 4) {
+                if ((i & 0xff) == 0)
+                        nv_wr32(priv, 0x10a188, (pmu + i) >> 8);
+                nv_wr32(priv, 0x10a184, nv_ro32(bios, img + i));
+        }
+        while (i & 0xff) {
+                nv_wr32(priv, 0x10a184, 0x00000000);
+                i += 4;
+        }
+}
+static void
+pmu_data(struct nv50_devinit_priv *priv, u32 pmu, u32 img, u32 len)
+{
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        int i;
+        nv_wr32(priv, 0x10a1c0, 0x01000000 | pmu);
+        for (i = 0; i < len; i += 4)
+                nv_wr32(priv, 0x10a1c4, nv_ro32(bios, img + i));
+}
+static u32
+pmu_args(struct nv50_devinit_priv *priv, u32 argp, u32 argi)
+{
+        nv_wr32(priv, 0x10a1c0, argp);
+        nv_wr32(priv, 0x10a1c0, nv_rd32(priv, 0x10a1c4) + argi);
+        return nv_rd32(priv, 0x10a1c4);
+}
+static void
+pmu_exec(struct nv50_devinit_priv *priv, u32 init_addr)
+{
+        nv_wr32(priv, 0x10a104, init_addr);
+        nv_wr32(priv, 0x10a10c, 0x00000000);
+        nv_wr32(priv, 0x10a100, 0x00000002);
+}
+static int
+pmu_load(struct nv50_devinit_priv *priv, u8 type, bool post,
+         u32 *init_addr_pmu, u32 *args_addr_pmu)
+{
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        struct nvbios_pmuR pmu;
+        if (!nvbios_pmuRm(bios, type, &pmu)) {
+                nv_error(priv, "VBIOS PMU fuc %02x not found\n", type);
+                return -EINVAL;
+        }
+        if (!post)
+                return 0;
+        pmu_code(priv, pmu.boot_addr_pmu, pmu.boot_addr, pmu.boot_size, false);
+        pmu_code(priv, pmu.code_addr_pmu, pmu.code_addr, pmu.code_size, true);
+        pmu_data(priv, pmu.data_addr_pmu, pmu.data_addr, pmu.data_size);
+        if (init_addr_pmu) {
+                *init_addr_pmu = pmu.init_addr_pmu;
+                *args_addr_pmu = pmu.args_addr_pmu;
+                return 0;
+        }
+        return pmu_exec(priv, pmu.init_addr_pmu), 0;
+}
+static int
+gm204_devinit_post(struct nouveau_subdev *subdev, bool post)
+{
+        struct nv50_devinit_priv *priv = (void *)nouveau_devinit(subdev);
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        struct bit_entry bit_I;
+        u32 init, args;
+        int ret;
+        if (bit_entry(bios, 'I', &bit_I) || bit_I.version != 1 ||
+                                            bit_I.length < 0x1c) {
+                nv_error(priv, "VBIOS PMU init data not found\n");
+                return -EINVAL;
+        }
+        /* reset PMU and load init table parser ucode */
+        if (post) {
+                nv_mask(priv, 0x000200, 0x00002000, 0x00000000);
+                nv_mask(priv, 0x000200, 0x00002000, 0x00002000);
+                nv_rd32(priv, 0x000200);
+                while (nv_rd32(priv, 0x10a10c) & 0x00000006) {
+                }
+        }
+        ret = pmu_load(priv, 0x04, post, &init, &args);
+        if (ret)
+                return ret;
+        /* upload first chunk of init data */
+        if (post) {
+                u32 pmu = pmu_args(priv, args + 0x08, 0x08);
+                u32 img = nv_ro16(bios, bit_I.offset + 0x14);
+                u32 len = nv_ro16(bios, bit_I.offset + 0x16);
+                pmu_data(priv, pmu, img, len);
+        }
+        /* upload second chunk of init data */
+        if (post) {
+                u32 pmu = pmu_args(priv, args + 0x08, 0x10);
+                u32 img = nv_ro16(bios, bit_I.offset + 0x18);
+                u32 len = nv_ro16(bios, bit_I.offset + 0x1a);
+                pmu_data(priv, pmu, img, len);
+        }
+        /* execute init tables */
+        if (post) {
+                nv_wr32(priv, 0x10a040, 0x00005000);
+                pmu_exec(priv, init);
+                while (!(nv_rd32(priv, 0x10a040) & 0x00002000)) {
+                }
+        }
+        /* load and execute some other ucode image (bios therm?) */
+        return pmu_load(priv, 0x01, post, NULL, NULL);
+}
+struct nouveau_oclass *
+gm204_devinit_oclass = &(struct nouveau_devinit_impl) {
+        .base.handle = NV_SUBDEV(DEVINIT, 0x07),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv50_devinit_ctor,
+                .dtor = _nouveau_devinit_dtor,
+                .init = nv50_devinit_init,
+                .fini = _nouveau_devinit_fini,
+        },
+        .pll_set = nvc0_devinit_pll_set,
+        .disable = gm107_devinit_disable,
+        .post = gm204_devinit_post,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
index 052ad690b468..65651c50f6ea 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
@@ -464,4 +464,5 @@ nv04_devinit_oclass = &(struct nouveau_devinit_impl) {
        },
        .meminit = nv04_devinit_meminit,
        .pll_set = nv04_devinit_pll_set,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c
index 4a19c10e5178..a2007a3efc4d 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c
@@ -136,4 +136,5 @@ nv05_devinit_oclass = &(struct nouveau_devinit_impl) {
        },
        .meminit = nv05_devinit_meminit,
        .pll_set = nv04_devinit_pll_set,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
index 3b8d657da279..178b46f79b50 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
@@ -107,4 +107,5 @@ nv10_devinit_oclass = &(struct nouveau_devinit_impl) {
        },
        .meminit = nv10_devinit_meminit,
        .pll_set = nv04_devinit_pll_set,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv1a.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv1a.c
index 526d0c6faacd..995dd97af3e9 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv1a.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv1a.c
@@ -34,4 +34,5 @@ nv1a_devinit_oclass = &(struct nouveau_devinit_impl) {
                .fini = nv04_devinit_fini,
        },
        .pll_set = nv04_devinit_pll_set,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c
index 04bc9732644c..915089fb46f7 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c
@@ -71,4 +71,5 @@ nv20_devinit_oclass = &(struct nouveau_devinit_impl) {
        },
        .meminit = nv20_devinit_meminit,
        .pll_set = nv04_devinit_pll_set,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.c
index b46c62a1d5d8..968334d1dca4 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.c
@@ -26,6 +26,7 @@
 #include <subdev/bios/dcb.h>
 #include <subdev/bios/disp.h>
 #include <subdev/bios/init.h>
+#include <subdev/ibus.h>
 #include <subdev/vga.h>
 #include "nv50.h"
@@ -91,6 +92,7 @@ int
 nv50_devinit_init(struct nouveau_object *object)
 {
        struct nouveau_bios *bios = nouveau_bios(object);
+        struct nouveau_ibus *ibus = nouveau_ibus(object);
        struct nv50_devinit_priv *priv = (void *)object;
        struct nvbios_outp info;
        struct dcb_output outp;
@@ -105,6 +107,13 @@ nv50_devinit_init(struct nouveau_object *object)
                }
        }
+        /* some boards appear to require certain priv register timeouts
+         * to be bumped before runing devinit scripts.  not a clue why
+         * the vbios engineers didn't make the scripts just work...
+         */
+        if (priv->base.post && ibus)
+                nv_ofuncs(ibus)->init(nv_object(ibus));
        ret = nouveau_devinit_init(&priv->base);
        if (ret)
                return ret;
@@ -160,4 +169,5 @@ nv50_devinit_oclass = &(struct nouveau_devinit_impl) {
        },
        .pll_set = nv50_devinit_pll_set,
        .disable = nv50_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h
index 51d5076333ec..f412bb7f780e 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h
@@ -18,4 +18,6 @@ int  nva3_devinit_pll_set(struct nouveau_devinit *, u32, u32);
 int  nvc0_devinit_pll_set(struct nouveau_devinit *, u32, u32);
+u64  gm107_devinit_disable(struct nouveau_devinit *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv84.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv84.c
index 787422505d87..a7c80ded77cd 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv84.c
@@ -60,4 +60,5 @@ nv84_devinit_oclass = &(struct nouveau_devinit_impl) {
        },
        .pll_set = nv50_devinit_pll_set,
        .disable = nv84_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv98.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv98.c
index 2b0e963fc6f0..a773253a17f6 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv98.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv98.c
@@ -59,4 +59,5 @@ nv98_devinit_oclass = &(struct nouveau_devinit_impl) {
        },
        .pll_set = nv50_devinit_pll_set,
        .disable = nv98_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c
index 006cf348bda7..b9cd9e53f760 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c
@@ -142,4 +142,5 @@ nva3_devinit_oclass = &(struct nouveau_devinit_impl) {
        .pll_set = nva3_devinit_pll_set,
        .disable = nva3_devinit_disable,
        .mmio    = nva3_devinit_mmio,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nvaf.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nvaf.c
index 4fc68d27eff3..3729846a8e5c 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nvaf.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nvaf.c
@@ -60,4 +60,5 @@ nvaf_devinit_oclass = &(struct nouveau_devinit_impl) {
        },
        .pll_set = nva3_devinit_pll_set,
        .disable = nvaf_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c
index 30c765747eea..80bd7f5eda3d 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c
@@ -115,4 +115,5 @@ nvc0_devinit_oclass = &(struct nouveau_devinit_impl) {
        },
        .pll_set = nvc0_devinit_pll_set,
        .disable = nvc0_devinit_disable,
+        .post = nvbios_init,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h b/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h
index f0e8683ad840..cbcd51852472 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h
@@ -3,6 +3,7 @@
 #include <subdev/bios.h>
 #include <subdev/bios/pll.h>
+#include <subdev/bios/init.h>
 #include <subdev/clock/pll.h>
 #include <subdev/devinit.h>
@@ -12,6 +13,7 @@ struct nouveau_devinit_impl {
        int  (*pll_set)(struct nouveau_devinit *, u32 type, u32 freq);
        u64  (*disable)(struct nouveau_devinit *);
        u32  (*mmio)(struct nouveau_devinit *, u32);
+        int  (*post)(struct nouveau_subdev *, bool);
 };
 #define nouveau_devinit_create(p,e,o,d)                                        \
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/base.c b/drivers/gpu/drm/nouveau/core/subdev/fb/base.c
index f009d8a39d9d..c866148c440f 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/base.c
@@ -23,37 +23,30 @@
 */
 #include <subdev/bios.h>
-#include <subdev/bios/bit.h>
+#include <subdev/bios/M0203.h>
 #include "priv.h"
 int
 nouveau_fb_bios_memtype(struct nouveau_bios *bios)
 {
-        struct bit_entry M;
+        const u8 ramcfg = (nv_rd32(bios, 0x101000) & 0x0000003c) >> 2;
-        u8 ramcfg;
+        struct nvbios_M0203E M0203E;
+        u8 ver, hdr;
-        ramcfg = (nv_rd32(bios, 0x101000) & 0x0000003c) >> 2;
-        if (!bit_entry(bios, 'M', &M) && M.version == 2 && M.length >= 5) {
+        if (nvbios_M0203Em(bios, ramcfg, &ver, &hdr, &M0203E)) {
-                u16 table   = nv_ro16(bios, M.offset + 3);
+                switch (M0203E.type) {
-                u8  version = nv_ro08(bios, table + 0);
+                case M0203E_TYPE_DDR2 : return NV_MEM_TYPE_DDR2;
-                u8  header  = nv_ro08(bios, table + 1);
+                case M0203E_TYPE_DDR3 : return NV_MEM_TYPE_DDR3;
-                u8  record  = nv_ro08(bios, table + 2);
+                case M0203E_TYPE_GDDR3: return NV_MEM_TYPE_GDDR3;
-                u8  entries = nv_ro08(bios, table + 3);
+                case M0203E_TYPE_GDDR5: return NV_MEM_TYPE_GDDR5;
-                if (table && version == 0x10 && ramcfg < entries) {
+                default:
-                        u16 entry = table + header + (ramcfg * record);
+                        nv_warn(bios, "M0203E type %02x\n", M0203E.type);
-                        switch (nv_ro08(bios, entry) & 0x0f) {
+                        return NV_MEM_TYPE_UNKNOWN;
-                        case 0: return NV_MEM_TYPE_DDR2;
-                        case 1: return NV_MEM_TYPE_DDR3;
-                        case 2: return NV_MEM_TYPE_GDDR3;
-                        case 3: return NV_MEM_TYPE_GDDR5;
-                        default:
-                                break;
-                        }
                }
        }
+        nv_warn(bios, "M0203E not matched!\n");
        return NV_MEM_TYPE_UNKNOWN;
 }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/gddr3.c b/drivers/gpu/drm/nouveau/core/subdev/fb/gddr3.c
new file mode 100644
index 000000000000..d85a25d027ee
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/gddr3.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ *          Roy Spliet <rspliet@eclipso.eu>
+ */
+#include <subdev/bios.h>
+#include "priv.h"
+struct ramxlat {
+        int id;
+        u8 enc;
+};
+static inline int
+ramxlat(const struct ramxlat *xlat, int id)
+{
+        while (xlat->id >= 0) {
+                if (xlat->id == id)
+                        return xlat->enc;
+                xlat++;
+        }
+        return -EINVAL;
+}
+static const struct ramxlat
+ramgddr3_cl_lo[] = {
+        { 7, 7 }, { 8, 0 }, { 9, 1 }, { 10, 2 }, { 11, 3 },
+        /* the below are mentioned in some, but not all, gddr3 docs */
+        { 12, 4 }, { 13, 5 }, { 14, 6 },
+        /* XXX: Per Samsung docs, are these used? They overlap with Qimonda */
+        /* { 4, 4 }, { 5, 5 }, { 6, 6 }, { 12, 8 }, { 13, 9 }, { 14, 10 },
+         * { 15, 11 }, */
+        { -1 }
+};
+static const struct ramxlat
+ramgddr3_cl_hi[] = {
+        { 10, 2 }, { 11, 3 }, { 12, 4 }, { 13, 5 }, { 14, 6 }, { 15, 7 },
+        { 16, 0 }, { 17, 1 },
+        { -1 }
+};
+static const struct ramxlat
+ramgddr3_wr_lo[] = {
+        { 5, 2 }, { 7, 4 }, { 8, 5 }, { 9, 6 }, { 10, 7 },
+        { 11, 0 },
+        /* the below are mentioned in some, but not all, gddr3 docs */
+        { 4, 1 }, { 6, 3 }, { 12, 1 }, { 13 , 2 },
+        { -1 }
+};
+int
+nouveau_gddr3_calc(struct nouveau_ram *ram)
+{
+        int CL, WR, CWL, DLL = 0, ODT = 0, hi;
+        switch (ram->next->bios.timing_ver) {
+        case 0x10:
+                CWL = ram->next->bios.timing_10_CWL;
+                CL  = ram->next->bios.timing_10_CL;
+                WR  = ram->next->bios.timing_10_WR;
+                DLL = !ram->next->bios.ramcfg_10_DLLoff;
+                ODT = ram->next->bios.timing_10_ODT;
+                break;
+        case 0x20:
+                CWL = (ram->next->bios.timing[1] & 0x00000f80) >> 7;
+                CL  = (ram->next->bios.timing[1] & 0x0000001f) >> 0;
+                WR  = (ram->next->bios.timing[2] & 0x007f0000) >> 16;
+                /* XXX: Get these values from the VBIOS instead */
+                DLL = !(ram->mr[1] & 0x1);
+                ODT =  (ram->mr[1] & 0x004) >> 2 |
+                       (ram->mr[1] & 0x040) >> 5 |
+                       (ram->mr[1] & 0x200) >> 7;
+                break;
+        default:
+                return -ENOSYS;
+        }
+        hi = ram->mr[2] & 0x1;
+        CL  = ramxlat(hi ? ramgddr3_cl_hi : ramgddr3_cl_lo, CL);
+        WR  = ramxlat(ramgddr3_wr_lo, WR);
+        if (CL < 0 || CWL < 1 || CWL > 7 || WR < 0)
+                return -EINVAL;
+        ram->mr[0] &= ~0xf74;
+        ram->mr[0] |= (CWL & 0x07) << 9;
+        ram->mr[0] |= (CL & 0x07) << 4;
+        ram->mr[0] |= (CL & 0x08) >> 1;
+        ram->mr[1] &= ~0x3fc;
+        ram->mr[1] |= (ODT & 0x03) << 2;
+        ram->mr[1] |= (ODT & 0x03) << 8;
+        ram->mr[1] |= (WR  & 0x03) << 4;
+        ram->mr[1] |= (WR  & 0x04) << 5;
+        ram->mr[1] |= !DLL << 6;
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
index 60322e906dd4..283863f7aa9b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
@@ -37,6 +37,7 @@ extern struct nouveau_oclass gm107_ram_oclass;
 int nouveau_sddr2_calc(struct nouveau_ram *ram);
 int nouveau_sddr3_calc(struct nouveau_ram *ram);
+int nouveau_gddr3_calc(struct nouveau_ram *ram);
 int nouveau_gddr5_calc(struct nouveau_ram *ram, bool nuts);
 #define nouveau_fb_create(p,e,c,d)                                             \
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramfuc.h b/drivers/gpu/drm/nouveau/core/subdev/fb/ramfuc.h
index d1fbbe4b00a2..0ac7256443bb 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramfuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramfuc.h
@@ -141,6 +141,20 @@ ramfuc_wait_vblank(struct ramfuc *ram)
 }
 static inline void
+ramfuc_train(struct ramfuc *ram)
+{
+        nouveau_memx_train(ram->memx);
+}
+static inline int
+ramfuc_train_result(struct nouveau_fb *pfb, u32 *result, u32 rsize)
+{
+        struct nouveau_pwr *ppwr = nouveau_pwr(pfb);
+        return nouveau_memx_train_result(ppwr, result, rsize);
+}
+static inline void
 ramfuc_block(struct ramfuc *ram)
 {
        nouveau_memx_block(ram->memx);
@@ -162,6 +176,8 @@ ramfuc_unblock(struct ramfuc *ram)
 #define ram_wait(s,r,m,d,n)  ramfuc_wait(&(s)->base, (r), (m), (d), (n))
 #define ram_nsec(s,n)        ramfuc_nsec(&(s)->base, (n))
 #define ram_wait_vblank(s)   ramfuc_wait_vblank(&(s)->base)
+#define ram_train(s)         ramfuc_train(&(s)->base)
+#define ram_train_result(s,r,l) ramfuc_train_result((s), (r), (l))
 #define ram_block(s)         ramfuc_block(&(s)->base)
 #define ram_unblock(s)       ramfuc_unblock(&(s)->base)
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c
index 3601deca0bd5..3b38a538845d 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c
@@ -20,86 +20,512 @@
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Ben Skeggs
+ *          Roy Spliet <rspliet@eclipso.eu>
 */
 #include <subdev/bios.h>
 #include <subdev/bios/bit.h>
 #include <subdev/bios/pll.h>
 #include <subdev/bios/rammap.h>
+#include <subdev/bios/M0205.h>
 #include <subdev/bios/timing.h>
 #include <subdev/clock/nva3.h>
 #include <subdev/clock/pll.h>
+#include <subdev/gpio.h>
+#include <subdev/timer.h>
+#include <engine/fifo.h>
 #include <core/option.h>
 #include "ramfuc.h"
 #include "nv50.h"
+/* XXX: Remove when memx gains GPIO support */
+extern int nv50_gpio_location(int line, u32 *reg, u32 *shift);
 struct nva3_ramfuc {
        struct ramfuc base;
+        struct ramfuc_reg r_0x001610;
+        struct ramfuc_reg r_0x001700;
+        struct ramfuc_reg r_0x002504;
        struct ramfuc_reg r_0x004000;
        struct ramfuc_reg r_0x004004;
        struct ramfuc_reg r_0x004018;
        struct ramfuc_reg r_0x004128;
        struct ramfuc_reg r_0x004168;
+        struct ramfuc_reg r_0x100080;
        struct ramfuc_reg r_0x100200;
        struct ramfuc_reg r_0x100210;
        struct ramfuc_reg r_0x100220[9];
+        struct ramfuc_reg r_0x100264;
        struct ramfuc_reg r_0x1002d0;
        struct ramfuc_reg r_0x1002d4;
        struct ramfuc_reg r_0x1002dc;
        struct ramfuc_reg r_0x10053c;
        struct ramfuc_reg r_0x1005a0;
        struct ramfuc_reg r_0x1005a4;
+        struct ramfuc_reg r_0x100700;
        struct ramfuc_reg r_0x100714;
        struct ramfuc_reg r_0x100718;
        struct ramfuc_reg r_0x10071c;
+        struct ramfuc_reg r_0x100720;
        struct ramfuc_reg r_0x100760;
        struct ramfuc_reg r_0x1007a0;
        struct ramfuc_reg r_0x1007e0;
+        struct ramfuc_reg r_0x100da0;
        struct ramfuc_reg r_0x10f804;
        struct ramfuc_reg r_0x1110e0;
        struct ramfuc_reg r_0x111100;
        struct ramfuc_reg r_0x111104;
+        struct ramfuc_reg r_0x1111e0;
+        struct ramfuc_reg r_0x111400;
        struct ramfuc_reg r_0x611200;
        struct ramfuc_reg r_mr[4];
+        struct ramfuc_reg r_gpioFBVREF;
+};
+struct nva3_ltrain {
+        enum {
+                NVA3_TRAIN_UNKNOWN,
+                NVA3_TRAIN_UNSUPPORTED,
+                NVA3_TRAIN_ONCE,
+                NVA3_TRAIN_EXEC,
+                NVA3_TRAIN_DONE
+        } state;
+        u32 r_100720;
+        u32 r_1111e0;
+        u32 r_111400;
+        struct nouveau_mem *mem;
 };
 struct nva3_ram {
        struct nouveau_ram base;
        struct nva3_ramfuc fuc;
+        struct nva3_ltrain ltrain;
 };
+void
+nva3_link_train_calc(u32 *vals, struct nva3_ltrain *train)
+{
+        int i, lo, hi;
+        u8 median[8], bins[4] = {0, 0, 0, 0}, bin = 0, qty = 0;
+        for (i = 0; i < 8; i++) {
+                for (lo = 0; lo < 0x40; lo++) {
+                        if (!(vals[lo] & 0x80000000))
+                                continue;
+                        if (vals[lo] & (0x101 << i))
+                                break;
+                }
+                if (lo == 0x40)
+                        return;
+                for (hi = lo + 1; hi < 0x40; hi++) {
+                        if (!(vals[lo] & 0x80000000))
+                                continue;
+                        if (!(vals[hi] & (0x101 << i))) {
+                                hi--;
+                                break;
+                        }
+                }
+                median[i] = ((hi - lo) >> 1) + lo;
+                bins[(median[i] & 0xf0) >> 4]++;
+                median[i] += 0x30;
+        }
+        /* Find the best value for 0x1111e0 */
+        for (i = 0; i < 4; i++) {
+                if (bins[i] > qty) {
+                        bin = i + 3;
+                        qty = bins[i];
+                }
+        }
+        train->r_100720 = 0;
+        for (i = 0; i < 8; i++) {
+                median[i] = max(median[i], (u8) (bin << 4));
+                median[i] = min(median[i], (u8) ((bin << 4) | 0xf));
+                train->r_100720 |= ((median[i] & 0x0f) << (i << 2));
+        }
+        train->r_1111e0 = 0x02000000 | (bin * 0x101);
+        train->r_111400 = 0x0;
+}
+/*
+ * Link training for (at least) DDR3
+ */
+int
+nva3_link_train(struct nouveau_fb *pfb)
+{
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nva3_ram *ram = (void *)pfb->ram;
+        struct nouveau_clock *clk = nouveau_clock(pfb);
+        struct nva3_ltrain *train = &ram->ltrain;
+        struct nouveau_device *device = nv_device(pfb);
+        struct nva3_ramfuc *fuc = &ram->fuc;
+        u32 *result, r1700;
+        int ret, i;
+        struct nvbios_M0205T M0205T = { 0 };
+        u8 ver, hdr, cnt, len, snr, ssz;
+        unsigned int clk_current;
+        unsigned long flags;
+        unsigned long *f = &flags;
+        if (nouveau_boolopt(device->cfgopt, "NvMemExec", true) != true)
+                return -ENOSYS;
+        /* XXX: Multiple partitions? */
+        result = kmalloc(64 * sizeof(u32), GFP_KERNEL);
+        if (!result)
+                return -ENOMEM;
+        train->state = NVA3_TRAIN_EXEC;
+        /* Clock speeds for training and back */
+        nvbios_M0205Tp(bios, &ver, &hdr, &cnt, &len, &snr, &ssz, &M0205T);
+        if (M0205T.freq == 0)
+                return -ENOENT;
+        clk_current = clk->read(clk, nv_clk_src_mem);
+        ret = nva3_clock_pre(clk, f);
+        if (ret)
+                goto out;
+        /* First: clock up/down */
+        ret = ram->base.calc(pfb, (u32) M0205T.freq * 1000);
+        if (ret)
+                goto out;
+        /* Do this *after* calc, eliminates write in script */
+        nv_wr32(pfb, 0x111400, 0x00000000);
+        /* XXX: Magic writes that improve train reliability? */
+        nv_mask(pfb, 0x100674, 0x0000ffff, 0x00000000);
+        nv_mask(pfb, 0x1005e4, 0x0000ffff, 0x00000000);
+        nv_mask(pfb, 0x100b0c, 0x000000ff, 0x00000000);
+        nv_wr32(pfb, 0x100c04, 0x00000400);
+        /* Now the training script */
+        r1700 = ram_rd32(fuc, 0x001700);
+        ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
+        ram_wr32(fuc, 0x611200, 0x3300);
+        ram_wait_vblank(fuc);
+        ram_wait(fuc, 0x611200, 0x00000003, 0x00000000, 500000);
+        ram_mask(fuc, 0x001610, 0x00000083, 0x00000003);
+        ram_mask(fuc, 0x100080, 0x00000020, 0x00000000);
+        ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);
+        ram_wr32(fuc, 0x001700, 0x00000000);
+        ram_train(fuc);
+        /* Reset */
+        ram_mask(fuc, 0x10f804, 0x80000000, 0x80000000);
+        ram_wr32(fuc, 0x10053c, 0x0);
+        ram_wr32(fuc, 0x100720, train->r_100720);
+        ram_wr32(fuc, 0x1111e0, train->r_1111e0);
+        ram_wr32(fuc, 0x111400, train->r_111400);
+        ram_nuke(fuc, 0x100080);
+        ram_mask(fuc, 0x100080, 0x00000020, 0x00000020);
+        ram_nsec(fuc, 1000);
+        ram_wr32(fuc, 0x001700, r1700);
+        ram_mask(fuc, 0x001610, 0x00000083, 0x00000080);
+        ram_wr32(fuc, 0x611200, 0x3330);
+        ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);
+        ram_exec(fuc, true);
+        ram->base.calc(pfb, clk_current);
+        ram_exec(fuc, true);
+        /* Post-processing, avoids flicker */
+        nv_mask(pfb, 0x616308, 0x10, 0x10);
+        nv_mask(pfb, 0x616b08, 0x10, 0x10);
+        nva3_clock_post(clk, f);
+        ram_train_result(pfb, result, 64);
+        for (i = 0; i < 64; i++)
+                nv_debug(pfb, "Train: %08x", result[i]);
+        nva3_link_train_calc(result, train);
+        nv_debug(pfb, "Train: %08x %08x %08x", train->r_100720,
+                        train->r_1111e0, train->r_111400);
+        kfree(result);
+        train->state = NVA3_TRAIN_DONE;
+        return ret;
+out:
+        if(ret == -EBUSY)
+                f = NULL;
+        train->state = NVA3_TRAIN_UNSUPPORTED;
+        nva3_clock_post(clk, f);
+        return ret;
+}
+int
+nva3_link_train_init(struct nouveau_fb *pfb)
+{
+        static const u32 pattern[16] = {
+                0xaaaaaaaa, 0xcccccccc, 0xdddddddd, 0xeeeeeeee,
+                0x00000000, 0x11111111, 0x44444444, 0xdddddddd,
+                0x33333333, 0x55555555, 0x77777777, 0x66666666,
+                0x99999999, 0x88888888, 0xeeeeeeee, 0xbbbbbbbb,
+        };
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nva3_ram *ram = (void *)pfb->ram;
+        struct nva3_ltrain *train = &ram->ltrain;
+        struct nouveau_mem *mem;
+        struct nvbios_M0205E M0205E;
+        u8 ver, hdr, cnt, len;
+        u32 r001700;
+        int ret, i = 0;
+        train->state = NVA3_TRAIN_UNSUPPORTED;
+        /* We support type "5"
+         * XXX: training pattern table appears to be unused for this routine */
+        if (!nvbios_M0205Ep(bios, i, &ver, &hdr, &cnt, &len, &M0205E))
+                return -ENOENT;
+        if (M0205E.type != 5)
+                return 0;
+        train->state = NVA3_TRAIN_ONCE;
+        ret = pfb->ram->get(pfb, 0x8000, 0x10000, 0, 0x800, &ram->ltrain.mem);
+        if (ret)
+                return ret;
+        mem = ram->ltrain.mem;
+        nv_wr32(pfb, 0x100538, 0x10000000 | (mem->offset >> 16));
+        nv_wr32(pfb, 0x1005a8, 0x0000ffff);
+        nv_mask(pfb, 0x10f800, 0x00000001, 0x00000001);
+        for (i = 0; i < 0x30; i++) {
+                nv_wr32(pfb, 0x10f8c0, (i << 8) | i);
+                nv_wr32(pfb, 0x10f900, pattern[i % 16]);
+        }
+        for (i = 0; i < 0x30; i++) {
+                nv_wr32(pfb, 0x10f8e0, (i << 8) | i);
+                nv_wr32(pfb, 0x10f920, pattern[i % 16]);
+        }
+        /* And upload the pattern */
+        r001700 = nv_rd32(pfb, 0x1700);
+        nv_wr32(pfb, 0x1700, mem->offset >> 16);
+        for (i = 0; i < 16; i++)
+                nv_wr32(pfb, 0x700000 + (i << 2), pattern[i]);
+        for (i = 0; i < 16; i++)
+                nv_wr32(pfb, 0x700100 + (i << 2), pattern[i]);
+        nv_wr32(pfb, 0x1700, r001700);
+        train->r_100720 = nv_rd32(pfb, 0x100720);
+        train->r_1111e0 = nv_rd32(pfb, 0x1111e0);
+        train->r_111400 = nv_rd32(pfb, 0x111400);
+        return 0;
+}
+void
+nva3_link_train_fini(struct nouveau_fb *pfb)
+{
+        struct nva3_ram *ram = (void *)pfb->ram;
+        if (ram->ltrain.mem)
+                pfb->ram->put(pfb, &ram->ltrain.mem);
+}
+/*
+ * RAM reclocking
+ */
+#define T(t) cfg->timing_10_##t
+static int
+nva3_ram_timing_calc(struct nouveau_fb *pfb, u32 *timing)
+{
+        struct nva3_ram *ram = (void *)pfb->ram;
+        struct nvbios_ramcfg *cfg = &ram->base.target.bios;
+        int tUNK_base, tUNK_40_0, prevCL;
+        u32 cur2, cur3, cur7, cur8;
+        cur2 = nv_rd32(pfb, 0x100228);
+        cur3 = nv_rd32(pfb, 0x10022c);
+        cur7 = nv_rd32(pfb, 0x10023c);
+        cur8 = nv_rd32(pfb, 0x100240);
+        switch ((!T(CWL)) * ram->base.type) {
+        case NV_MEM_TYPE_DDR2:
+                T(CWL) = T(CL) - 1;
+                break;
+        case NV_MEM_TYPE_GDDR3:
+                T(CWL) = ((cur2 & 0xff000000) >> 24) + 1;
+                break;
+        }
+        prevCL = (cur3 & 0x000000ff) + 1;
+        tUNK_base = ((cur7 & 0x00ff0000) >> 16) - prevCL;
+        timing[0] = (T(RP) << 24 | T(RAS) << 16 | T(RFC) << 8 | T(RC));
+        timing[1] = (T(WR) + 1 + T(CWL)) << 24 |
+                    max_t(u8,T(18), 1) << 16 |
+                    (T(WTR) + 1 + T(CWL)) << 8 |
+                    (5 + T(CL) - T(CWL));
+        timing[2] = (T(CWL) - 1) << 24 |
+                    (T(RRD) << 16) |
+                    (T(RCDWR) << 8) |
+                    T(RCDRD);
+        timing[3] = (cur3 & 0x00ff0000) |
+                    (0x30 + T(CL)) << 24 |
+                    (0xb + T(CL)) << 8 |
+                    (T(CL) - 1);
+        timing[4] = T(20) << 24 |
+                    T(21) << 16 |
+                    T(13) << 8 |
+                    T(13);
+        timing[5] = T(RFC) << 24 |
+                    max_t(u8,T(RCDRD), T(RCDWR)) << 16 |
+                    max_t(u8, (T(CWL) + 6), (T(CL) + 2)) << 8 |
+                    T(RP);
+        timing[6] = (0x5a + T(CL)) << 16 |
+                    max_t(u8, 1, (6 - T(CL) + T(CWL))) << 8 |
+                    (0x50 + T(CL) - T(CWL));
+        timing[7] = (cur7 & 0xff000000) |
+                    ((tUNK_base + T(CL)) << 16) |
+                    0x202;
+        timing[8] = cur8 & 0xffffff00;
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR2:
+        case NV_MEM_TYPE_GDDR3:
+                tUNK_40_0 = prevCL - (cur8 & 0xff);
+                if (tUNK_40_0 > 0)
+                        timing[8] |= T(CL);
+                break;
+        default:
+                break;
+        }
+        nv_debug(pfb, "Entry: 220: %08x %08x %08x %08x\n",
+                        timing[0], timing[1], timing[2], timing[3]);
+        nv_debug(pfb, "  230: %08x %08x %08x %08x\n",
+                        timing[4], timing[5], timing[6], timing[7]);
+        nv_debug(pfb, "  240: %08x\n", timing[8]);
+        return 0;
+}
+#undef T
+static void
+nouveau_sddr2_dll_reset(struct nva3_ramfuc *fuc)
+{
+        ram_mask(fuc, mr[0], 0x100, 0x100);
+        ram_nsec(fuc, 1000);
+        ram_mask(fuc, mr[0], 0x100, 0x000);
+        ram_nsec(fuc, 1000);
+}
+static void
+nouveau_sddr3_dll_disable(struct nva3_ramfuc *fuc, u32 *mr)
+{
+        u32 mr1_old = ram_rd32(fuc, mr[1]);
+        if (!(mr1_old & 0x1)) {
+                ram_wr32(fuc, 0x1002d4, 0x00000001);
+                ram_wr32(fuc, mr[1], mr[1]);
+                ram_nsec(fuc, 1000);
+        }
+}
+static void
+nouveau_gddr3_dll_disable(struct nva3_ramfuc *fuc, u32 *mr)
+{
+        u32 mr1_old = ram_rd32(fuc, mr[1]);
+        if (!(mr1_old & 0x40)) {
+                ram_wr32(fuc, mr[1], mr[1]);
+                ram_nsec(fuc, 1000);
+        }
+}
+static void
+nva3_ram_lock_pll(struct nva3_ramfuc *fuc, struct nva3_clock_info *mclk)
+{
+        ram_wr32(fuc, 0x004004, mclk->pll);
+        ram_mask(fuc, 0x004000, 0x00000001, 0x00000001);
+        ram_mask(fuc, 0x004000, 0x00000010, 0x00000000);
+        ram_wait(fuc, 0x004000, 0x00020000, 0x00020000, 64000);
+        ram_mask(fuc, 0x004000, 0x00000010, 0x00000010);
+}
+static void
+nva3_ram_fbvref(struct nva3_ramfuc *fuc, u32 val)
+{
+        struct nouveau_gpio *gpio = nouveau_gpio(fuc->base.pfb);
+        struct dcb_gpio_func func;
+        u32 reg, sh, gpio_val;
+        int ret;
+        if (gpio->get(gpio, 0, 0x2e, DCB_GPIO_UNUSED) != val) {
+                ret = gpio->find(gpio, 0, 0x2e, DCB_GPIO_UNUSED, &func);
+                if (ret)
+                        return;
+                nv50_gpio_location(func.line, &reg, &sh);
+                gpio_val = ram_rd32(fuc, gpioFBVREF);
+                if (gpio_val & (8 << sh))
+                        val = !val;
+                ram_mask(fuc, gpioFBVREF, (0x3 << sh), ((val | 0x2) << sh));
+                ram_nsec(fuc, 20000);
+        }
+}
 static int
 nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
 {
        struct nouveau_bios *bios = nouveau_bios(pfb);
        struct nva3_ram *ram = (void *)pfb->ram;
        struct nva3_ramfuc *fuc = &ram->fuc;
+        struct nva3_ltrain *train = &ram->ltrain;
        struct nva3_clock_info mclk;
        struct nouveau_ram_data *next;
        u8  ver, hdr, cnt, len, strap;
        u32 data;
-        u32 r004018, r100760, ctrl;
+        u32 r004018, r100760, r100da0, r111100, ctrl;
        u32 unk714, unk718, unk71c;
        int ret, i;
+        u32 timing[9];
+        bool pll2pll;
        next = &ram->base.target;
        next->freq = freq;
        ram->base.next = next;
+        if (ram->ltrain.state == NVA3_TRAIN_ONCE)
+                nva3_link_train(pfb);
        /* lookup memory config data relevant to the target frequency */
        i = 0;
-        while ((data = nvbios_rammapEp(bios, i++, &ver, &hdr, &cnt, &len,
+        data = nvbios_rammapEm(bios, freq / 1000, &ver, &hdr, &cnt, &len,
-                                      &next->bios))) {
+                                      &next->bios);
-                if (freq / 1000 >= next->bios.rammap_min &&
+        if (!data || ver != 0x10 || hdr < 0x05) {
-                    freq / 1000 <= next->bios.rammap_max)
-                        break;
-        }
-        if (!data || ver != 0x10 || hdr < 0x0e) {
                nv_error(pfb, "invalid/missing rammap entry\n");
                return -EINVAL;
        }
@@ -113,7 +539,7 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
        data = nvbios_rammapSp(bios, data, ver, hdr, cnt, len, strap,
                               &ver, &hdr, &next->bios);
-        if (!data || ver != 0x10 || hdr < 0x0e) {
+        if (!data || ver != 0x10 || hdr < 0x09) {
                nv_error(pfb, "invalid/missing ramcfg entry\n");
                return -EINVAL;
        }
@@ -123,7 +549,7 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
                data = nvbios_timingEp(bios, next->bios.ramcfg_timing,
                                       &ver, &hdr, &cnt, &len,
                                       &next->bios);
-                if (!data || ver != 0x10 || hdr < 0x19) {
+                if (!data || ver != 0x10 || hdr < 0x17) {
                        nv_error(pfb, "invalid/missing timing entry\n");
                        return -EINVAL;
                }
@@ -135,53 +561,99 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
                return ret;
        }
+        nva3_ram_timing_calc(pfb, timing);
        ret = ram_init(fuc, pfb);
        if (ret)
                return ret;
+        /* Determine ram-specific MR values */
+        ram->base.mr[0] = ram_rd32(fuc, mr[0]);
+        ram->base.mr[1] = ram_rd32(fuc, mr[1]);
+        ram->base.mr[2] = ram_rd32(fuc, mr[2]);
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR2:
+                ret = nouveau_sddr2_calc(&ram->base);
+                break;
+        case NV_MEM_TYPE_DDR3:
+                ret = nouveau_sddr3_calc(&ram->base);
+                break;
+        case NV_MEM_TYPE_GDDR3:
+                ret = nouveau_gddr3_calc(&ram->base);
+                break;
+        default:
+                ret = -ENOSYS;
+                break;
+        }
+        if (ret)
+                return ret;
        /* XXX: where the fuck does 750MHz come from? */
        if (freq <= 750000) {
                r004018 = 0x10000000;
                r100760 = 0x22222222;
+                r100da0 = 0x00000010;
        } else {
                r004018 = 0x00000000;
                r100760 = 0x00000000;
+                r100da0 = 0x00000000;
        }
+        if (!next->bios.ramcfg_10_DLLoff)
+                r004018 |= 0x00004000;
+        /* pll2pll requires to switch to a safe clock first */
        ctrl = ram_rd32(fuc, 0x004000);
-        if (ctrl & 0x00000008) {
+        pll2pll = (!(ctrl & 0x00000008)) && mclk.pll;
-                if (mclk.pll) {
-                        ram_mask(fuc, 0x004128, 0x00000101, 0x00000101);
-                        ram_wr32(fuc, 0x004004, mclk.pll);
-                        ram_wr32(fuc, 0x004000, (ctrl |= 0x00000001));
-                        ram_wr32(fuc, 0x004000, (ctrl &= 0xffffffef));
-                        ram_wait(fuc, 0x004000, 0x00020000, 0x00020000, 64000);
-                        ram_wr32(fuc, 0x004000, (ctrl |= 0x00000010));
-                        ram_wr32(fuc, 0x004018, 0x00005000 | r004018);
-                        ram_wr32(fuc, 0x004000, (ctrl |= 0x00000004));
-                }
-        } else {
-                u32 ssel = 0x00000101;
-                if (mclk.clk)
-                        ssel |= mclk.clk;
-                else
-                        ssel |= 0x00080000; /* 324MHz, shouldn't matter... */
-                ram_mask(fuc, 0x004168, 0x003f3141, ctrl);
-        }
+        /* Pre, NVIDIA does this outside the script */
        if (next->bios.ramcfg_10_02_10) {
                ram_mask(fuc, 0x111104, 0x00000600, 0x00000000);
        } else {
                ram_mask(fuc, 0x111100, 0x40000000, 0x40000000);
                ram_mask(fuc, 0x111104, 0x00000180, 0x00000000);
        }
+        /* Always disable this bit during reclock */
+        ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
+        /* If switching from non-pll to pll, lock before disabling FB */
+        if (mclk.pll && !pll2pll) {
+                ram_mask(fuc, 0x004128, 0x003f3141, mclk.clk | 0x00000101);
+                nva3_ram_lock_pll(fuc, &mclk);
+        }
+        /* Start with disabling some CRTCs and PFIFO? */
+        ram_wait_vblank(fuc);
+        ram_wr32(fuc, 0x611200, 0x3300);
+        ram_mask(fuc, 0x002504, 0x1, 0x1);
+        ram_nsec(fuc, 10000);
+        ram_wait(fuc, 0x002504, 0x10, 0x10, 20000); /* XXX: or longer? */
+        ram_block(fuc);
+        ram_nsec(fuc, 2000);
+        if (!next->bios.ramcfg_10_02_10) {
+                if (ram->base.type == NV_MEM_TYPE_GDDR3)
+                        ram_mask(fuc, 0x111100, 0x04020000, 0x00020000);
+                else
+                        ram_mask(fuc, 0x111100, 0x04020000, 0x04020000);
+        }
+        /* If we're disabling the DLL, do it now */
+        switch (next->bios.ramcfg_10_DLLoff * ram->base.type) {
+        case NV_MEM_TYPE_DDR3:
+                nouveau_sddr3_dll_disable(fuc, ram->base.mr);
+                break;
+        case NV_MEM_TYPE_GDDR3:
+                nouveau_gddr3_dll_disable(fuc, ram->base.mr);
+                break;
+        }
-        if (!next->bios.rammap_10_04_02)
+        if (fuc->r_gpioFBVREF.addr && next->bios.timing_10_ODT)
-                ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
+                nva3_ram_fbvref(fuc, 0);
-        ram_wr32(fuc, 0x611200, 0x00003300);
-        if (!next->bios.ramcfg_10_02_10)
-                ram_wr32(fuc, 0x111100, 0x4c020000); /*XXX*/
+        /* Brace RAM for impact */
        ram_wr32(fuc, 0x1002d4, 0x00000001);
        ram_wr32(fuc, 0x1002d0, 0x00000001);
        ram_wr32(fuc, 0x1002d0, 0x00000001);
@@ -189,24 +661,38 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
        ram_wr32(fuc, 0x1002dc, 0x00000001);
        ram_nsec(fuc, 2000);
-        ctrl = ram_rd32(fuc, 0x004000);
+        if (nv_device(pfb)->chipset == 0xa3 && freq <= 500000)
-        if (!(ctrl & 0x00000008) && mclk.pll) {
+                ram_mask(fuc, 0x100700, 0x00000006, 0x00000006);
-                ram_wr32(fuc, 0x004000, (ctrl |=  0x00000008));
+        /* Fiddle with clocks */
+        /* There's 4 scenario's
+         * pll->pll: first switch to a 324MHz clock, set up new PLL, switch
+         * clk->pll: Set up new PLL, switch
+         * pll->clk: Set up clock, switch
+         * clk->clk: Overwrite ctrl and other bits, switch */
+        /* Switch to regular clock - 324MHz */
+        if (pll2pll) {
+                ram_mask(fuc, 0x004000, 0x00000004, 0x00000004);
+                ram_mask(fuc, 0x004168, 0x003f3141, 0x00083101);
+                ram_mask(fuc, 0x004000, 0x00000008, 0x00000008);
                ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
                ram_wr32(fuc, 0x004018, 0x00001000);
-                ram_wr32(fuc, 0x004000, (ctrl &= ~0x00000001));
+                nva3_ram_lock_pll(fuc, &mclk);
-                ram_wr32(fuc, 0x004004, mclk.pll);
+        }
-                ram_wr32(fuc, 0x004000, (ctrl |=  0x00000001));
-                udelay(64);
+        if (mclk.pll) {
-                ram_wr32(fuc, 0x004018, 0x00005000 | r004018);
+                ram_mask(fuc, 0x004000, 0x00000105, 0x00000105);
-                udelay(20);
+                ram_wr32(fuc, 0x004018, 0x00001000 | r004018);
-        } else
+                ram_wr32(fuc, 0x100da0, r100da0);
-        if (!mclk.pll) {
+        } else {
-                ram_mask(fuc, 0x004168, 0x003f3040, mclk.clk);
+                ram_mask(fuc, 0x004168, 0x003f3141, mclk.clk | 0x00000101);
-                ram_wr32(fuc, 0x004000, (ctrl |= 0x00000008));
+                ram_mask(fuc, 0x004000, 0x00000108, 0x00000008);
                ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
-                ram_wr32(fuc, 0x004018, 0x0000d000 | r004018);
+                ram_wr32(fuc, 0x004018, 0x00009000 | r004018);
+                ram_wr32(fuc, 0x100da0, r100da0);
        }
+        ram_nsec(fuc, 20000);
        if (next->bios.rammap_10_04_08) {
                ram_wr32(fuc, 0x1005a0, next->bios.ramcfg_10_06 << 16 |
@@ -220,6 +706,12 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
                                        0x80000000);
                ram_mask(fuc, 0x10053c, 0x00001000, 0x00000000);
        } else {
+                if (train->state == NVA3_TRAIN_DONE) {
+                        ram_wr32(fuc, 0x100080, 0x1020);
+                        ram_mask(fuc, 0x111400, 0xffffffff, train->r_111400);
+                        ram_mask(fuc, 0x1111e0, 0xffffffff, train->r_1111e0);
+                        ram_mask(fuc, 0x100720, 0xffffffff, train->r_100720);
+                }
                ram_mask(fuc, 0x10053c, 0x00001000, 0x00001000);
                ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);
                ram_mask(fuc, 0x100760, 0x22222222, r100760);
@@ -227,65 +719,131 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
                ram_mask(fuc, 0x1007e0, 0x22222222, r100760);
        }
+        if (nv_device(pfb)->chipset == 0xa3 && freq > 500000) {
+                ram_mask(fuc, 0x100700, 0x00000006, 0x00000000);
+        }
+        /* Final switch */
        if (mclk.pll) {
                ram_mask(fuc, 0x1110e0, 0x00088000, 0x00011000);
-                ram_wr32(fuc, 0x004000, (ctrl &= ~0x00000008));
+                ram_mask(fuc, 0x004000, 0x00000008, 0x00000000);
        }
-        /*XXX: LEAVE */
        ram_wr32(fuc, 0x1002dc, 0x00000000);
        ram_wr32(fuc, 0x1002d4, 0x00000001);
        ram_wr32(fuc, 0x100210, 0x80000000);
-        ram_nsec(fuc, 1000);
+        ram_nsec(fuc, 2000);
-        ram_nsec(fuc, 1000);
-        ram_mask(fuc, mr[2], 0x00000000, 0x00000000);
+        /* Set RAM MR parameters and timings */
-        ram_nsec(fuc, 1000);
+        for (i = 2; i >= 0; i--) {
-        ram_nuke(fuc, mr[0]);
+                if (ram_rd32(fuc, mr[i]) != ram->base.mr[i]) {
-        ram_mask(fuc, mr[0], 0x00000000, 0x00000000);
+                        ram_wr32(fuc, mr[i], ram->base.mr[i]);
-        ram_nsec(fuc, 1000);
+                        ram_nsec(fuc, 1000);
+                }
+        }
-        ram_mask(fuc, 0x100220[3], 0x00000000, 0x00000000);
+        ram_wr32(fuc, 0x100220[3], timing[3]);
-        ram_mask(fuc, 0x100220[1], 0x00000000, 0x00000000);
+        ram_wr32(fuc, 0x100220[1], timing[1]);
-        ram_mask(fuc, 0x100220[6], 0x00000000, 0x00000000);
+        ram_wr32(fuc, 0x100220[6], timing[6]);
-        ram_mask(fuc, 0x100220[7], 0x00000000, 0x00000000);
+        ram_wr32(fuc, 0x100220[7], timing[7]);
-        ram_mask(fuc, 0x100220[2], 0x00000000, 0x00000000);
+        ram_wr32(fuc, 0x100220[2], timing[2]);
-        ram_mask(fuc, 0x100220[4], 0x00000000, 0x00000000);
+        ram_wr32(fuc, 0x100220[4], timing[4]);
-        ram_mask(fuc, 0x100220[5], 0x00000000, 0x00000000);
+        ram_wr32(fuc, 0x100220[5], timing[5]);
-        ram_mask(fuc, 0x100220[0], 0x00000000, 0x00000000);
+        ram_wr32(fuc, 0x100220[0], timing[0]);
-        ram_mask(fuc, 0x100220[8], 0x00000000, 0x00000000);
+        ram_wr32(fuc, 0x100220[8], timing[8]);
+        /* Misc */
        ram_mask(fuc, 0x100200, 0x00001000, !next->bios.ramcfg_10_02_08 << 12);
-        unk714 = ram_rd32(fuc, 0x100714) & ~0xf0000010;
+        /* XXX: A lot of "chipset"/"ram type" specific stuff...? */
-        unk718 = ram_rd32(fuc, 0x100718) & ~0x00000100;
+        unk714  = ram_rd32(fuc, 0x100714) & ~0xf0000130;
-        unk71c = ram_rd32(fuc, 0x10071c) & ~0x00000100;
+        unk718  = ram_rd32(fuc, 0x100718) & ~0x00000100;
+        unk71c  = ram_rd32(fuc, 0x10071c) & ~0x00000100;
+        r111100 = ram_rd32(fuc, 0x111100) & ~0x3a800000;
+        if (next->bios.ramcfg_10_02_04) {
+                switch (ram->base.type) {
+                case NV_MEM_TYPE_DDR3:
+                        if (nv_device(pfb)->chipset != 0xa8)
+                                r111100 |= 0x00000004;
+                        /* no break */
+                case NV_MEM_TYPE_DDR2:
+                        r111100 |= 0x08000000;
+                        break;
+                default:
+                        break;
+                }
+        } else {
+                switch (ram->base.type) {
+                case NV_MEM_TYPE_DDR2:
+                        r111100 |= 0x1a800000;
+                        unk714  |= 0x00000010;
+                        break;
+                case NV_MEM_TYPE_DDR3:
+                        if (nv_device(pfb)->chipset == 0xa8) {
+                                r111100 |=  0x08000000;
+                        } else {
+                                r111100 &= ~0x00000004;
+                                r111100 |=  0x12800000;
+                        }
+                        unk714  |= 0x00000010;
+                        break;
+                case NV_MEM_TYPE_GDDR3:
+                        r111100 |= 0x30000000;
+                        unk714  |= 0x00000020;
+                        break;
+                default:
+                        break;
+                }
+        }
+        unk714 |= (next->bios.ramcfg_10_04_01) << 8;
        if (next->bios.ramcfg_10_02_20)
                unk714 |= 0xf0000000;
-        if (!next->bios.ramcfg_10_02_04)
+        if (next->bios.ramcfg_10_02_02)
-                unk714 |= 0x00000010;
+                unk718 |= 0x00000100;
-        ram_wr32(fuc, 0x100714, unk714);
        if (next->bios.ramcfg_10_02_01)
                unk71c |= 0x00000100;
-        ram_wr32(fuc, 0x10071c, unk71c);
+        if (next->bios.timing_10_24 != 0xff) {
+                unk718 &= ~0xf0000000;
+                unk718 |= next->bios.timing_10_24 << 28;
+        }
+        if (next->bios.ramcfg_10_02_10)
+                r111100 &= ~0x04020000;
-        if (next->bios.ramcfg_10_02_02)
+        ram_mask(fuc, 0x100714, 0xffffffff, unk714);
-                unk718 |= 0x00000100;
+        ram_mask(fuc, 0x10071c, 0xffffffff, unk71c);
-        ram_wr32(fuc, 0x100718, unk718);
+        ram_mask(fuc, 0x100718, 0xffffffff, unk718);
+        ram_mask(fuc, 0x111100, 0xffffffff, r111100);
-        if (next->bios.ramcfg_10_02_10)
+        if (fuc->r_gpioFBVREF.addr && !next->bios.timing_10_ODT)
-                ram_wr32(fuc, 0x111100, 0x48000000); /*XXX*/
+                nva3_ram_fbvref(fuc, 1);
-        ram_mask(fuc, mr[0], 0x100, 0x100);
+        /* Reset DLL */
-        ram_nsec(fuc, 1000);
+        if (!next->bios.ramcfg_10_DLLoff)
-        ram_mask(fuc, mr[0], 0x100, 0x000);
+                nouveau_sddr2_dll_reset(fuc);
-        ram_nsec(fuc, 1000);
-        ram_nsec(fuc, 2000);
+        if (ram->base.type == NV_MEM_TYPE_GDDR3) {
-        ram_nsec(fuc, 12000);
+                ram_nsec(fuc, 31000);
+        } else {
+                ram_nsec(fuc, 14000);
+        }
+        if (ram->base.type == NV_MEM_TYPE_DDR3) {
+                ram_wr32(fuc, 0x100264, 0x1);
+                ram_nsec(fuc, 2000);
+        }
-        ram_wr32(fuc, 0x611200, 0x00003330);
+        ram_nuke(fuc, 0x100700);
+        ram_mask(fuc, 0x100700, 0x01000000, 0x01000000);
+        ram_mask(fuc, 0x100700, 0x01000000, 0x00000000);
+        /* Re-enable FB */
+        ram_unblock(fuc);
+        ram_wr32(fuc, 0x611200, 0x3330);
+        /* Post fiddlings */
        if (next->bios.rammap_10_04_02)
                ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);
        if (next->bios.ramcfg_10_02_10) {
@@ -313,7 +871,22 @@ nva3_ram_prog(struct nouveau_fb *pfb)
        struct nouveau_device *device = nv_device(pfb);
        struct nva3_ram *ram = (void *)pfb->ram;
        struct nva3_ramfuc *fuc = &ram->fuc;
-        ram_exec(fuc, nouveau_boolopt(device->cfgopt, "NvMemExec", true));
+        bool exec = nouveau_boolopt(device->cfgopt, "NvMemExec", true);
+        if (exec) {
+                nv_mask(pfb, 0x001534, 0x2, 0x2);
+                ram_exec(fuc, true);
+                /* Post-processing, avoids flicker */
+                nv_mask(pfb, 0x002504, 0x1, 0x0);
+                nv_mask(pfb, 0x001534, 0x2, 0x0);
+                nv_mask(pfb, 0x616308, 0x10, 0x10);
+                nv_mask(pfb, 0x616b08, 0x10, 0x10);
+        } else {
+                ram_exec(fuc, false);
+        }
        return 0;
 }
@@ -330,38 +903,24 @@ nva3_ram_init(struct nouveau_object *object)
 {
        struct nouveau_fb *pfb = (void *)object->parent;
        struct nva3_ram   *ram = (void *)object;
-        int ret, i;
+        int ret;
        ret = nouveau_ram_init(&ram->base);
        if (ret)
                return ret;
-        /* prepare for ddr link training, and load training patterns */
+        nva3_link_train_init(pfb);
-        switch (ram->base.type) {
-        case NV_MEM_TYPE_DDR3: {
+        return 0;
-                if (nv_device(pfb)->chipset == 0xa8) {
+}
-                        static const u32 pattern[16] = {
-                                0xaaaaaaaa, 0xcccccccc, 0xdddddddd, 0xeeeeeeee,
+static int
-                                0x00000000, 0x11111111, 0x44444444, 0xdddddddd,
+nva3_ram_fini(struct nouveau_object *object, bool suspend)
-                                0x33333333, 0x55555555, 0x77777777, 0x66666666,
+{
-                                0x99999999, 0x88888888, 0xeeeeeeee, 0xbbbbbbbb,
+        struct nouveau_fb *pfb = (void *)object->parent;
-                        };
+        if (!suspend)
-                        nv_wr32(pfb, 0x100538, 0x10001ff6); /*XXX*/
+                nva3_link_train_fini(pfb);
-                        nv_wr32(pfb, 0x1005a8, 0x0000ffff);
-                        nv_mask(pfb, 0x10f800, 0x00000001, 0x00000001);
-                        for (i = 0; i < 0x30; i++) {
-                                nv_wr32(pfb, 0x10f8c0, (i << 8) | i);
-                                nv_wr32(pfb, 0x10f8e0, (i << 8) | i);
-                                nv_wr32(pfb, 0x10f900, pattern[i % 16]);
-                                nv_wr32(pfb, 0x10f920, pattern[i % 16]);
-                        }
-                }
-        }
-                break;
-        default:
-                break;
-        }
        return 0;
 }
@@ -371,8 +930,12 @@ nva3_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
              struct nouveau_oclass *oclass, void *data, u32 datasize,
              struct nouveau_object **pobject)
 {
+        struct nouveau_fb *pfb = nouveau_fb(parent);
+        struct nouveau_gpio *gpio = nouveau_gpio(pfb);
+        struct dcb_gpio_func func;
        struct nva3_ram *ram;
        int ret, i;
+        u32 reg, shift;
        ret = nv50_ram_create(parent, engine, oclass, &ram);
        *pobject = nv_object(ram);
@@ -380,7 +943,9 @@ nva3_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                return ret;
        switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR2:
        case NV_MEM_TYPE_DDR3:
+        case NV_MEM_TYPE_GDDR3:
                ram->base.calc = nva3_ram_calc;
                ram->base.prog = nva3_ram_prog;
                ram->base.tidy = nva3_ram_tidy;
@@ -390,31 +955,41 @@ nva3_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                return 0;
        }
+        ram->fuc.r_0x001610 = ramfuc_reg(0x001610);
+        ram->fuc.r_0x001700 = ramfuc_reg(0x001700);
+        ram->fuc.r_0x002504 = ramfuc_reg(0x002504);
        ram->fuc.r_0x004000 = ramfuc_reg(0x004000);
        ram->fuc.r_0x004004 = ramfuc_reg(0x004004);
        ram->fuc.r_0x004018 = ramfuc_reg(0x004018);
        ram->fuc.r_0x004128 = ramfuc_reg(0x004128);
        ram->fuc.r_0x004168 = ramfuc_reg(0x004168);
+        ram->fuc.r_0x100080 = ramfuc_reg(0x100080);
        ram->fuc.r_0x100200 = ramfuc_reg(0x100200);
        ram->fuc.r_0x100210 = ramfuc_reg(0x100210);
        for (i = 0; i < 9; i++)
                ram->fuc.r_0x100220[i] = ramfuc_reg(0x100220 + (i * 4));
+        ram->fuc.r_0x100264 = ramfuc_reg(0x100264);
        ram->fuc.r_0x1002d0 = ramfuc_reg(0x1002d0);
        ram->fuc.r_0x1002d4 = ramfuc_reg(0x1002d4);
        ram->fuc.r_0x1002dc = ramfuc_reg(0x1002dc);
        ram->fuc.r_0x10053c = ramfuc_reg(0x10053c);
        ram->fuc.r_0x1005a0 = ramfuc_reg(0x1005a0);
        ram->fuc.r_0x1005a4 = ramfuc_reg(0x1005a4);
+        ram->fuc.r_0x100700 = ramfuc_reg(0x100700);
        ram->fuc.r_0x100714 = ramfuc_reg(0x100714);
        ram->fuc.r_0x100718 = ramfuc_reg(0x100718);
        ram->fuc.r_0x10071c = ramfuc_reg(0x10071c);
+        ram->fuc.r_0x100720 = ramfuc_reg(0x100720);
        ram->fuc.r_0x100760 = ramfuc_stride(0x100760, 4, ram->base.part_mask);
        ram->fuc.r_0x1007a0 = ramfuc_stride(0x1007a0, 4, ram->base.part_mask);
        ram->fuc.r_0x1007e0 = ramfuc_stride(0x1007e0, 4, ram->base.part_mask);
+        ram->fuc.r_0x100da0 = ramfuc_stride(0x100da0, 4, ram->base.part_mask);
        ram->fuc.r_0x10f804 = ramfuc_reg(0x10f804);
        ram->fuc.r_0x1110e0 = ramfuc_stride(0x1110e0, 4, ram->base.part_mask);
        ram->fuc.r_0x111100 = ramfuc_reg(0x111100);
        ram->fuc.r_0x111104 = ramfuc_reg(0x111104);
+        ram->fuc.r_0x1111e0 = ramfuc_reg(0x1111e0);
+        ram->fuc.r_0x111400 = ramfuc_reg(0x111400);
        ram->fuc.r_0x611200 = ramfuc_reg(0x611200);
        if (ram->base.ranks > 1) {
@@ -429,6 +1004,12 @@ nva3_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                ram->fuc.r_mr[3] = ramfuc_reg(0x1002e4);
        }
+        ret = gpio->find(gpio, 0, 0x2e, DCB_GPIO_UNUSED, &func);
+        if (ret == 0) {
+                nv50_gpio_location(func.line, &reg, &shift);
+                ram->fuc.r_gpioFBVREF = ramfuc_reg(reg);
+        }
        return 0;
 }
@@ -438,6 +1019,6 @@ nva3_ram_oclass = {
                .ctor = nva3_ram_ctor,
                .dtor = _nouveau_ram_dtor,
                .init = nva3_ram_init,
-                .fini = _nouveau_ram_fini,
+                .fini = nva3_ram_fini,
        },
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/sddr2.c b/drivers/gpu/drm/nouveau/core/subdev/fb/sddr2.c
index bb1eb8f3e639..252575f3aa29 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/sddr2.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/sddr2.c
@@ -66,7 +66,7 @@ nouveau_sddr2_calc(struct nouveau_ram *ram)
        case 0x10:
                CL  = ram->next->bios.timing_10_CL;
                WR  = ram->next->bios.timing_10_WR;
-                DLL = !ram->next->bios.ramcfg_10_02_40;
+                DLL = !ram->next->bios.ramcfg_10_DLLoff;
                ODT = ram->next->bios.timing_10_ODT & 3;
                break;
        case 0x20:
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/sddr3.c b/drivers/gpu/drm/nouveau/core/subdev/fb/sddr3.c
index 83949b11833a..a2dca4869e52 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/sddr3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/sddr3.c
@@ -80,7 +80,7 @@ nouveau_sddr3_calc(struct nouveau_ram *ram)
                CWL = ram->next->bios.timing_10_CWL;
                CL  = ram->next->bios.timing_10_CL;
                WR  = ram->next->bios.timing_10_WR;
-                DLL = !ram->next->bios.ramcfg_10_02_40;
+                DLL = !ram->next->bios.ramcfg_10_DLLoff;
                ODT = ram->next->bios.timing_10_ODT;
                break;
        case 0x20:
diff --git a/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c
index 1864fa98e6b1..2e30d5a62d6e 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c
@@ -54,7 +54,7 @@ nv50_gpio_reset(struct nouveau_gpio *gpio, u8 match)
        }
 }
-static int
+int
 nv50_gpio_location(int line, u32 *reg, u32 *shift)
 {
        const u32 nv50_gpio_reg[4] = { 0xe104, 0xe108, 0xe280, 0xe284 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
index 2b1bf545e488..0dc605db7ec8 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
@@ -473,18 +473,56 @@ nouveau_i2c_extdev_sclass[] = {
        nouveau_anx9805_sclass,
 };
+static void
+nouveau_i2c_create_port(struct nouveau_i2c *i2c, int index, u8 type,
+                        struct dcb_i2c_entry *info)
+{
+        const struct nouveau_i2c_impl *impl = (void *)nv_oclass(i2c);
+        struct nouveau_oclass *oclass;
+        struct nouveau_object *parent;
+        struct nouveau_object *object;
+        int ret, pad;
+        if (info->share != DCB_I2C_UNUSED) {
+                pad    = info->share;
+                oclass = impl->pad_s;
+        } else {
+                if (type != DCB_I2C_NVIO_AUX)
+                        pad = 0x100 + info->drive;
+                else
+                        pad = 0x100 + info->auxch;
+                oclass = impl->pad_x;
+        }
+        ret = nouveau_object_ctor(NULL, nv_object(i2c), oclass, NULL, pad,
+                                 &parent);
+        if (ret < 0)
+                return;
+        oclass = impl->sclass;
+        do {
+                ret = -EINVAL;
+                if (oclass->handle == type) {
+                        ret = nouveau_object_ctor(parent, nv_object(i2c),
+                                                  oclass, info, index,
+                                                 &object);
+                }
+        } while (ret && (++oclass)->handle);
+        nouveau_object_ref(NULL, &parent);
+}
 int
 nouveau_i2c_create_(struct nouveau_object *parent,
                    struct nouveau_object *engine,
                    struct nouveau_oclass *oclass,
                    int length, void **pobject)
 {
-        const struct nouveau_i2c_impl *impl = (void *)oclass;
        struct nouveau_bios *bios = nouveau_bios(parent);
        struct nouveau_i2c *i2c;
        struct nouveau_object *object;
        struct dcb_i2c_entry info;
-        int ret, i, j, index = -1, pad;
+        int ret, i, j, index = -1;
        struct dcb_output outp;
        u8  ver, hdr;
        u32 data;
@@ -507,43 +545,40 @@ nouveau_i2c_create_(struct nouveau_object *parent,
        INIT_LIST_HEAD(&i2c->ports);
        while (!dcb_i2c_parse(bios, ++index, &info)) {
-                if (info.type == DCB_I2C_UNUSED)
+                switch (info.type) {
+                case DCB_I2C_NV04_BIT:
+                case DCB_I2C_NV4E_BIT:
+                case DCB_I2C_NVIO_BIT:
+                        nouveau_i2c_create_port(i2c, NV_I2C_PORT(index),
+                                                info.type, &info);
+                        break;
+                case DCB_I2C_NVIO_AUX:
+                        nouveau_i2c_create_port(i2c, NV_I2C_AUX(index),
+                                                info.type, &info);
+                        break;
+                case DCB_I2C_PMGR:
+                        if (info.drive != DCB_I2C_UNUSED) {
+                                nouveau_i2c_create_port(i2c, NV_I2C_PORT(index),
+                                                        DCB_I2C_NVIO_BIT,
+                                                        &info);
+                        }
+                        if (info.auxch != DCB_I2C_UNUSED) {
+                                nouveau_i2c_create_port(i2c, NV_I2C_AUX(index),
+                                                        DCB_I2C_NVIO_AUX,
+                                                        &info);
+                        }
+                        break;
+                case DCB_I2C_UNUSED:
+                default:
                        continue;
-                if (info.share != DCB_I2C_UNUSED) {
-                        if (info.type == DCB_I2C_NVIO_AUX)
-                                pad = info.drive;
-                        else
-                                pad = info.share;
-                        oclass = impl->pad_s;
-                } else {
-                        pad = 0x100 + info.drive;
-                        oclass = impl->pad_x;
                }
-                ret = nouveau_object_ctor(NULL, *pobject, oclass,
-                                          NULL, pad, &parent);
-                if (ret < 0)
-                        continue;
-                oclass = impl->sclass;
-                do {
-                        ret = -EINVAL;
-                        if (oclass->handle == info.type) {
-                                ret = nouveau_object_ctor(parent, *pobject,
-                                                          oclass, &info,
-                                                          index, &object);
-                        }
-                } while (ret && (++oclass)->handle);
-                nouveau_object_ref(NULL, &parent);
        }
        /* in addition to the busses specified in the i2c table, there
         * may be ddc/aux channels hiding behind external tmds/dp/etc
         * transmitters.
         */
-        index = ((index + 0x0f) / 0x10) * 0x10;
+        index = NV_I2C_EXT(0);
        i = -1;
        while ((data = dcb_outp_parse(bios, ++i, &ver, &hdr, &outp))) {
                if (!outp.location || !outp.extdev)
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/gm204.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/gm204.c
new file mode 100644
index 000000000000..06a2b87ccbf1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/gm204.c
@@ -0,0 +1,221 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include "nv50.h"
+#define AUX_DBG(fmt, args...) nv_debug(aux, "AUXCH(%d): " fmt, ch, ##args)
+#define AUX_ERR(fmt, args...) nv_error(aux, "AUXCH(%d): " fmt, ch, ##args)
+static void
+auxch_fini(struct nouveau_i2c *aux, int ch)
+{
+        nv_mask(aux, 0x00d954 + (ch * 0x50), 0x00310000, 0x00000000);
+}
+static int
+auxch_init(struct nouveau_i2c *aux, int ch)
+{
+        const u32 unksel = 1; /* nfi which to use, or if it matters.. */
+        const u32 ureq = unksel ? 0x00100000 : 0x00200000;
+        const u32 urep = unksel ? 0x01000000 : 0x02000000;
+        u32 ctrl, timeout;
+        /* wait up to 1ms for any previous transaction to be done... */
+        timeout = 1000;
+        do {
+                ctrl = nv_rd32(aux, 0x00d954 + (ch * 0x50));
+                udelay(1);
+                if (!timeout--) {
+                        AUX_ERR("begin idle timeout 0x%08x\n", ctrl);
+                        return -EBUSY;
+                }
+        } while (ctrl & 0x03010000);
+        /* set some magic, and wait up to 1ms for it to appear */
+        nv_mask(aux, 0x00d954 + (ch * 0x50), 0x00300000, ureq);
+        timeout = 1000;
+        do {
+                ctrl = nv_rd32(aux, 0x00d954 + (ch * 0x50));
+                udelay(1);
+                if (!timeout--) {
+                        AUX_ERR("magic wait 0x%08x\n", ctrl);
+                        auxch_fini(aux, ch);
+                        return -EBUSY;
+                }
+        } while ((ctrl & 0x03000000) != urep);
+        return 0;
+}
+int
+gm204_aux(struct nouveau_i2c_port *base, bool retry,
+         u8 type, u32 addr, u8 *data, u8 size)
+{
+        struct nouveau_i2c *aux = nouveau_i2c(base);
+        struct nv50_i2c_port *port = (void *)base;
+        u32 ctrl, stat, timeout, retries;
+        u32 xbuf[4] = {};
+        int ch = port->addr;
+        int ret, i;
+        AUX_DBG("%d: 0x%08x %d\n", type, addr, size);
+        ret = auxch_init(aux, ch);
+        if (ret)
+                goto out;
+        stat = nv_rd32(aux, 0x00d958 + (ch * 0x50));
+        if (!(stat & 0x10000000)) {
+                AUX_DBG("sink not detected\n");
+                ret = -ENXIO;
+                goto out;
+        }
+        if (!(type & 1)) {
+                memcpy(xbuf, data, size);
+                for (i = 0; i < 16; i += 4) {
+                        AUX_DBG("wr 0x%08x\n", xbuf[i / 4]);
+                        nv_wr32(aux, 0x00d930 + (ch * 0x50) + i, xbuf[i / 4]);
+                }
+        }
+        ctrl  = nv_rd32(aux, 0x00d954 + (ch * 0x50));
+        ctrl &= ~0x0001f0ff;
+        ctrl |= type << 12;
+        ctrl |= size - 1;
+        nv_wr32(aux, 0x00d950 + (ch * 0x50), addr);
+        /* (maybe) retry transaction a number of times on failure... */
+        for (retries = 0; !ret && retries < 32; retries++) {
+                /* reset, and delay a while if this is a retry */
+                nv_wr32(aux, 0x00d954 + (ch * 0x50), 0x80000000 | ctrl);
+                nv_wr32(aux, 0x00d954 + (ch * 0x50), 0x00000000 | ctrl);
+                if (retries)
+                        udelay(400);
+                /* transaction request, wait up to 1ms for it to complete */
+                nv_wr32(aux, 0x00d954 + (ch * 0x50), 0x00010000 | ctrl);
+                timeout = 1000;
+                do {
+                        ctrl = nv_rd32(aux, 0x00d954 + (ch * 0x50));
+                        udelay(1);
+                        if (!timeout--) {
+                                AUX_ERR("tx req timeout 0x%08x\n", ctrl);
+                                ret = -EIO;
+                                goto out;
+                        }
+                } while (ctrl & 0x00010000);
+                ret = 1;
+                /* read status, and check if transaction completed ok */
+                stat = nv_mask(aux, 0x00d958 + (ch * 0x50), 0, 0);
+                if ((stat & 0x000f0000) == 0x00080000 ||
+                    (stat & 0x000f0000) == 0x00020000)
+                        ret = retry ? 0 : 1;
+                if ((stat & 0x00000100))
+                        ret = -ETIMEDOUT;
+                if ((stat & 0x00000e00))
+                        ret = -EIO;
+                AUX_DBG("%02d 0x%08x 0x%08x\n", retries, ctrl, stat);
+        }
+        if (type & 1) {
+                for (i = 0; i < 16; i += 4) {
+                        xbuf[i / 4] = nv_rd32(aux, 0x00d940 + (ch * 0x50) + i);
+                        AUX_DBG("rd 0x%08x\n", xbuf[i / 4]);
+                }
+                memcpy(data, xbuf, size);
+        }
+out:
+        auxch_fini(aux, ch);
+        return ret < 0 ? ret : (stat & 0x000f0000) >> 16;
+}
+static const struct nouveau_i2c_func
+gm204_aux_func = {
+        .aux       = gm204_aux,
+};
+int
+gm204_aux_port_ctor(struct nouveau_object *parent,
+                    struct nouveau_object *engine,
+                    struct nouveau_oclass *oclass, void *data, u32 index,
+                    struct nouveau_object **pobject)
+{
+        struct dcb_i2c_entry *info = data;
+        struct nv50_i2c_port *port;
+        int ret;
+        ret = nouveau_i2c_port_create(parent, engine, oclass, index,
+                                      &nouveau_i2c_aux_algo, &gm204_aux_func,
+                                      &port);
+        *pobject = nv_object(port);
+        if (ret)
+                return ret;
+        port->base.aux = info->auxch;
+        port->addr = info->auxch;
+        return 0;
+}
+struct nouveau_oclass
+gm204_i2c_sclass[] = {
+        { .handle = NV_I2C_TYPE_DCBI2C(DCB_I2C_NVIO_BIT),
+          .ofuncs = &(struct nouveau_ofuncs) {
+                  .ctor = nvd0_i2c_port_ctor,
+                  .dtor = _nouveau_i2c_port_dtor,
+                  .init = nv50_i2c_port_init,
+                  .fini = _nouveau_i2c_port_fini,
+          },
+        },
+        { .handle = NV_I2C_TYPE_DCBI2C(DCB_I2C_NVIO_AUX),
+          .ofuncs = &(struct nouveau_ofuncs) {
+                  .ctor = gm204_aux_port_ctor,
+                  .dtor = _nouveau_i2c_port_dtor,
+                  .init = _nouveau_i2c_port_init,
+                  .fini = _nouveau_i2c_port_fini,
+          },
+        },
+        {}
+};
+struct nouveau_oclass *
+gm204_i2c_oclass = &(struct nouveau_i2c_impl) {
+        .base.handle = NV_SUBDEV(I2C, 0x24),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = _nouveau_i2c_ctor,
+                .dtor = _nouveau_i2c_dtor,
+                .init = _nouveau_i2c_init,
+                .fini = _nouveau_i2c_fini,
+        },
+        .sclass = gm204_i2c_sclass,
+        .pad_x = &nv04_i2c_pad_oclass,
+        .pad_s = &gm204_i2c_pad_oclass,
+        .aux = 8,
+        .aux_stat = nve0_aux_stat,
+        .aux_mask = nve0_aux_mask,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/nv50.h b/drivers/gpu/drm/nouveau/core/subdev/i2c/nv50.h
index 5d2a77421c74..9ef965692fb1 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/nv50.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/nv50.h
@@ -10,8 +10,6 @@ struct nv50_i2c_priv {
 struct nv50_i2c_port {
        struct nouveau_i2c_port base;
        u32 addr;
-        u32 ctrl;
-        u32 data;
        u32 state;
 };
@@ -29,4 +27,8 @@ int  nv94_aux_port_ctor(struct nouveau_object *, struct nouveau_object *,
 void nv94_i2c_acquire(struct nouveau_i2c_port *);
 void nv94_i2c_release(struct nouveau_i2c_port *);
+int  nvd0_i2c_port_ctor(struct nouveau_object *, struct nouveau_object *,
+                        struct nouveau_oclass *, void *, u32,
+                        struct nouveau_object **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/nv94.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/nv94.c
index f59c3a255462..e383ee81f4d2 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/nv94.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/nv94.c
@@ -214,10 +214,6 @@ nv94_i2c_port_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        port->state = 7;
        port->addr = nv50_i2c_addr[info->drive];
-        if (info->share != DCB_I2C_UNUSED) {
-                port->ctrl = 0x00e500 + (info->share * 0x50);
-                port->data = 0x0000e001;
-        }
        return 0;
 }
@@ -242,13 +238,8 @@ nv94_aux_port_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        if (ret)
                return ret;
-        port->base.aux = info->drive;
+        port->base.aux = info->auxch;
-        port->addr = info->drive;
+        port->addr = info->auxch;
-        if (info->share != DCB_I2C_UNUSED) {
-                port->ctrl = 0x00e500 + (info->drive * 0x50);
-                port->data = 0x00002002;
-        }
        return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/nvd0.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/nvd0.c
index 364ddb1c5f03..fd99380502ec 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/nvd0.c
@@ -48,7 +48,7 @@ nvd0_i2c_func = {
        .sense_sda = nvd0_i2c_sense_sda,
 };
-static int
+int
 nvd0_i2c_port_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                   struct nouveau_oclass *oclass, void *data, u32 index,
                   struct nouveau_object **pobject)
@@ -66,10 +66,6 @@ nvd0_i2c_port_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
        port->state = 0x00000007;
        port->addr = 0x00d014 + (info->drive * 0x20);
-        if (info->share != DCB_I2C_UNUSED) {
-                port->ctrl = 0x00e500 + (info->share * 0x50);
-                port->data = 0x0000e001;
-        }
        return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/nve0.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/nve0.c
index cae77e1ad8dc..25fe5c2d110e 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/nve0.c
@@ -24,7 +24,7 @@
 #include "nv50.h"
-static void
+void
 nve0_aux_stat(struct nouveau_i2c *i2c, u32 *hi, u32 *lo, u32 *rq, u32 *tx)
 {
        u32 intr = nv_rd32(i2c, 0x00dc60);
@@ -38,7 +38,7 @@ nve0_aux_stat(struct nouveau_i2c *i2c, u32 *hi, u32 *lo, u32 *rq, u32 *tx)
        nv_wr32(i2c, 0x00dc60, intr);
 }
-static void
+void
 nve0_aux_mask(struct nouveau_i2c *i2c, u32 type, u32 mask, u32 data)
 {
        u32 temp = nv_rd32(i2c, 0x00dc68), i;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/padgm204.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/padgm204.c
new file mode 100644
index 000000000000..f0e6fbbaa8cd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/padgm204.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include "pad.h"
+struct gm204_i2c_pad {
+        struct nvkm_i2c_pad base;
+        int addr;
+};
+static int
+gm204_i2c_pad_fini(struct nouveau_object *object, bool suspend)
+{
+        struct nouveau_i2c *i2c = (void *)object->engine;
+        struct gm204_i2c_pad *pad = (void *)object;
+        nv_mask(i2c, 0x00d97c + pad->addr, 0x00000001, 0x00000001);
+        return nvkm_i2c_pad_fini(&pad->base, suspend);
+}
+static int
+gm204_i2c_pad_init(struct nouveau_object *object)
+{
+        struct nouveau_i2c *i2c = (void *)object->engine;
+        struct gm204_i2c_pad *pad = (void *)object;
+        switch (nv_oclass(pad->base.next)->handle) {
+        case NV_I2C_TYPE_DCBI2C(DCB_I2C_NVIO_AUX):
+                nv_mask(i2c, 0x00d970 + pad->addr, 0x0000c003, 0x00000002);
+                break;
+        case NV_I2C_TYPE_DCBI2C(DCB_I2C_NVIO_BIT):
+        default:
+                nv_mask(i2c, 0x00d970 + pad->addr, 0x0000c003, 0x0000c001);
+                break;
+        }
+        nv_mask(i2c, 0x00d97c + pad->addr, 0x00000001, 0x00000000);
+        return nvkm_i2c_pad_init(&pad->base);
+}
+static int
+gm204_i2c_pad_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                  struct nouveau_oclass *oclass, void *data, u32 index,
+                  struct nouveau_object **pobject)
+{
+        struct gm204_i2c_pad *pad;
+        int ret;
+        ret = nvkm_i2c_pad_create(parent, engine, oclass, index, &pad);
+        *pobject = nv_object(pad);
+        if (ret)
+                return ret;
+        pad->addr = index * 0x50;;
+        return 0;
+}
+struct nouveau_oclass
+gm204_i2c_pad_oclass = {
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gm204_i2c_pad_ctor,
+                .dtor = _nvkm_i2c_pad_dtor,
+                .init = gm204_i2c_pad_init,
+                .fini = gm204_i2c_pad_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/priv.h b/drivers/gpu/drm/nouveau/core/subdev/i2c/priv.h
index 780090b6425a..4fe7ae3fde4e 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/priv.h
@@ -5,6 +5,7 @@
 extern struct nouveau_oclass nv04_i2c_pad_oclass;
 extern struct nouveau_oclass nv94_i2c_pad_oclass;
+extern struct nouveau_oclass gm204_i2c_pad_oclass;
 #define nouveau_i2c_port_create(p,e,o,i,a,f,d)                                 \
        nouveau_i2c_port_create_((p), (e), (o), (i), (a), (f),                 \
@@ -82,4 +83,7 @@ struct nouveau_i2c_impl {
 void nv94_aux_stat(struct nouveau_i2c *, u32 *, u32 *, u32 *, u32 *);
 void nv94_aux_mask(struct nouveau_i2c *, u32, u32, u32);
+void nve0_aux_stat(struct nouveau_i2c *, u32 *, u32 *, u32 *, u32 *);
+void nve0_aux_mask(struct nouveau_i2c *, u32, u32, u32);
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/memx.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/memx.fuc
index e89789a53b80..ec03f9a4290b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/memx.fuc
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/memx.fuc
@@ -50,6 +50,7 @@ handler(WR32  , 0x0000, 0x0002, #memx_func_wr32)
 handler(WAIT  , 0x0004, 0x0000, #memx_func_wait)
 handler(DELAY , 0x0001, 0x0000, #memx_func_delay)
 handler(VBLANK, 0x0001, 0x0000, #memx_func_wait_vblank)
+handler(TRAIN , 0x0000, 0x0000, #memx_func_train)
 memx_func_tail:
 .equ #memx_func_size #memx_func_next - #memx_func_head
@@ -63,6 +64,10 @@ memx_ts_end:
 memx_data_head:
 .skip 0x0800
 memx_data_tail:
+memx_train_head:
+.skip 0x0100
+memx_train_tail:
 #endif
 /******************************************************************************
@@ -260,6 +265,101 @@ memx_func_delay:
 // description
 //
 // $r15 - current (memx)
+// $r4  - packet length
+// $r3  - opcode desciption
+// $r0  - zero
+memx_func_train:
+#if NVKM_PPWR_CHIPSET == GT215
+// $r5 - outer loop counter
+// $r6 - inner loop counter
+// $r7 - entry counter (#memx_train_head + $r7)
+        movw $r5 0x3
+        movw $r7 0x0
+// Read random memory to wake up... things
+        imm32($r9, 0x700000)
+        nv_rd32($r8,$r9)
+        movw $r14 0x2710
+        call(nsec)
+        memx_func_train_loop_outer:
+                mulu $r8 $r5 0x101
+                sethi $r8 0x02000000
+                imm32($r9, 0x1111e0)
+                nv_wr32($r9, $r8)
+                push $r5
+                movw $r6 0x0
+                memx_func_train_loop_inner:
+                        movw $r8 0x1111
+                        mulu $r9 $r6 $r8
+                        shl b32 $r8 $r9 0x10
+                        or $r8 $r9
+                        imm32($r9, 0x100720)
+                        nv_wr32($r9, $r8)
+                        imm32($r9, 0x100080)
+                        nv_rd32($r8, $r9)
+                        or $r8 $r8 0x20
+                        nv_wr32($r9, $r8)
+                        imm32($r9, 0x10053c)
+                        imm32($r8, 0x80003002)
+                        nv_wr32($r9, $r8)
+                        imm32($r14, 0x100560)
+                        imm32($r13, 0x80000000)
+                        add b32 $r12 $r13 0
+                        imm32($r11, 0x001e8480)
+                        call(wait)
+                        // $r5 - inner inner loop counter
+                        // $r9 - result
+                        movw $r5 0
+                        imm32($r9, 0x8300ffff)
+                        memx_func_train_loop_4x:
+                                imm32($r10, 0x100080)
+                                nv_rd32($r8, $r10)
+                                imm32($r11, 0xffffffdf)
+                                and $r8 $r11
+                                nv_wr32($r10, $r8)
+                                imm32($r10, 0x10053c)
+                                imm32($r8, 0x80003002)
+                                nv_wr32($r10, $r8)
+                                imm32($r14, 0x100560)
+                                imm32($r13, 0x80000000)
+                                mov b32 $r12 $r13
+                                imm32($r11, 0x00002710)
+                                call(wait)
+                                nv_rd32($r13, $r14)
+                                and $r9 $r9 $r13
+                                add b32 $r5 1
+                                cmp b16 $r5 0x4
+                                bra l #memx_func_train_loop_4x
+                        add b32 $r10 $r7 #memx_train_head
+                        st b32 D[$r10 + 0] $r9
+                        add b32 $r6 1
+                        add b32 $r7 4
+                        cmp b16 $r6 0x10
+                        bra l #memx_func_train_loop_inner
+                pop $r5
+                add b32 $r5 1
+                cmp b16 $r5 7
+                bra l #memx_func_train_loop_outer
+#endif
+        ret
+// description
+//
+// $r15 - current (memx)
 // $r14 - sender process name
 // $r13 - message (exec)
 // $r12 - head of script
@@ -307,8 +407,19 @@ memx_exec:
 // $r11 - data1
 // $r0  - zero
 memx_info:
+        cmp b16 $r12 0x1
+        bra e #memx_info_train
+        memx_info_data:
        mov $r12 #memx_data_head
        mov $r11 #memx_data_tail - #memx_data_head
+        bra #memx_info_send
+        memx_info_train:
+        mov $r12 #memx_train_head
+        mov $r11 #memx_train_tail - #memx_train_head
+        memx_info_send:
        call(send)
        ret
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h
index 4d278a96b2bb..713e11e2953d 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h
@@ -46,8 +46,8 @@ uint32_t nv108_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x584d454d,
-        0x0000061c,
+        0x0000062d,
-        0x0000060e,
+        0x0000061f,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -68,8 +68,8 @@ uint32_t nv108_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x46524550,
-        0x00000620,
+        0x00000631,
-        0x0000061e,
+        0x0000062f,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -90,8 +90,8 @@ uint32_t nv108_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x5f433249,
-        0x00000a24,
+        0x00000a35,
-        0x000008cb,
+        0x000008dc,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -112,8 +112,8 @@ uint32_t nv108_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x54534554,
-        0x00000a45,
+        0x00000a56,
-        0x00000a26,
+        0x00000a37,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -134,8 +134,8 @@ uint32_t nv108_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x454c4449,
-        0x00000a50,
+        0x00000a61,
-        0x00000a4e,
+        0x00000a5f,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -246,13 +246,15 @@ uint32_t nv108_pwr_data[] = {
        0x00010006,
        0x00000000,
        0x0000057b,
-/* 0x03b8: memx_func_tail */
+        0x00000007,
-/* 0x03b8: memx_ts_start */
        0x00000000,
-/* 0x03bc: memx_ts_end */
+        0x000005c3,
+/* 0x03c4: memx_func_tail */
+/* 0x03c4: memx_ts_start */
        0x00000000,
-/* 0x03c0: memx_data_head */
+/* 0x03c8: memx_ts_end */
        0x00000000,
+/* 0x03cc: memx_data_head */
        0x00000000,
        0x00000000,
        0x00000000,
@@ -764,8 +766,75 @@ uint32_t nv108_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x00000000,
-/* 0x0bc0: memx_data_tail */
+        0x00000000,
-/* 0x0bc0: i2c_scl_map */
+/* 0x0bcc: memx_data_tail */
+/* 0x0bcc: memx_train_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0ccc: memx_train_tail */
+/* 0x0ccc: i2c_scl_map */
        0x00000400,
        0x00000800,
        0x00001000,
@@ -776,7 +845,7 @@ uint32_t nv108_pwr_data[] = {
        0x00020000,
        0x00040000,
        0x00080000,
-/* 0x0be8: i2c_sda_map */
+/* 0x0cf4: i2c_sda_map */
        0x00100000,
        0x00200000,
        0x00400000,
@@ -844,9 +913,6 @@ uint32_t nv108_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
 };
 uint32_t nv108_pwr_code[] = {
@@ -1215,10 +1281,10 @@ uint32_t nv108_pwr_code[] = {
        0xf40464f0,
        0x2c06f70b,
        0xb50066cf,
-        0x00f8ee06,
+        0x00f8f106,
 /* 0x0500: memx_func_leave */
        0x66cf2c06,
-        0xef06b500,
+        0xf206b500,
        0xe4400406,
        0x0006f607,
 /* 0x0512: memx_func_leave_wait */
@@ -1270,370 +1336,374 @@ uint32_t nv108_pwr_code[] = {
        0x9800f800,
        0x10b6001e,
        0x005d7e04,
-/* 0x05c3: memx_exec */
+/* 0x05c3: memx_func_train */
-        0xf900f800,
+        0xf800f800,
-        0xb2d0f9e0,
+/* 0x05c5: memx_exec */
-/* 0x05cb: memx_exec_next */
+        0xf9e0f900,
-        0x98b2b2c1,
+        0xb2c1b2d0,
-        0x10b60013,
+/* 0x05cd: memx_exec_next */
-        0xf034e704,
+        0x001398b2,
-        0xe033e701,
+        0xe70410b6,
-        0x0132b601,
+        0xe701f034,
-        0x980c30f0,
+        0xb601e033,
-        0x55f9de35,
+        0x30f00132,
-        0x1ef412a6,
+        0xde35980c,
-        0xee0b98e5,
+        0x12a655f9,
-        0xbbef0c98,
+        0x98e51ef4,
-        0xc44b02cb,
+        0x0c98f10b,
-        0x00bbcf07,
+        0x02cbbbf2,
-        0xe0fcd0fc,
+        0xcf07c44b,
-        0x0002c27e,
+        0xd0fc00bb,
-/* 0x0602: memx_info */
+        0xc27ee0fc,
-        0xc04c00f8,
+        0x00f80002,
+/* 0x0604: memx_info */
+        0xf401c670,
+/* 0x060a: memx_info_data */
+        0xcc4c0c0b,
        0x08004b03,
-        0x0002c27e,
+/* 0x0613: memx_info_train */
-/* 0x060e: memx_recv */
+        0x4c090ef4,
-        0xd6b000f8,
+        0x004b0bcc,
-        0xb20bf401,
+/* 0x0619: memx_info_send */
-        0xf400d6b0,
+        0x02c27e01,
-        0x00f8eb0b,
+/* 0x061f: memx_recv */
-/* 0x061c: memx_init */
+        0xb000f800,
-/* 0x061e: perf_recv */
+        0x0bf401d6,
-        0x00f800f8,
+        0x00d6b0a3,
-/* 0x0620: perf_init */
+        0xf8dc0bf4,
-/* 0x0622: i2c_drive_scl */
+/* 0x062d: memx_init */
-        0x36b000f8,
+/* 0x062f: perf_recv */
-        0x0d0bf400,
+        0xf800f800,
-        0xf607e040,
+/* 0x0631: perf_init */
-        0x04bd0001,
+/* 0x0633: i2c_drive_scl */
-/* 0x0632: i2c_drive_scl_lo */
+        0xb000f800,
-        0xe44000f8,
+        0x0bf40036,
-        0x0001f607,
+        0x07e0400d,
-        0x00f804bd,
+        0xbd0001f6,
-/* 0x063c: i2c_drive_sda */
+/* 0x0643: i2c_drive_scl_lo */
-        0xf40036b0,
+        0x4000f804,
-        0xe0400d0b,
+        0x01f607e4,
-        0x0002f607,
+        0xf804bd00,
-        0x00f804bd,
+/* 0x064d: i2c_drive_sda */
-/* 0x064c: i2c_drive_sda_lo */
+        0x0036b000,
-        0xf607e440,
+        0x400d0bf4,
-        0x04bd0002,
+        0x02f607e0,
-/* 0x0656: i2c_sense_scl */
+        0xf804bd00,
-        0x32f400f8,
+/* 0x065d: i2c_drive_sda_lo */
-        0x07c44301,
+        0x07e44000,
-        0xfd0033cf,
+        0xbd0002f6,
-        0x0bf40431,
+/* 0x0667: i2c_sense_scl */
-        0x0131f406,
+        0xf400f804,
-/* 0x0668: i2c_sense_scl_done */
+        0xc4430132,
-/* 0x066a: i2c_sense_sda */
+        0x0033cf07,
-        0x32f400f8,
+        0xf40431fd,
-        0x07c44301,
+        0x31f4060b,
-        0xfd0033cf,
+/* 0x0679: i2c_sense_scl_done */
-        0x0bf40432,
+/* 0x067b: i2c_sense_sda */
-        0x0131f406,
+        0xf400f801,
-/* 0x067c: i2c_sense_sda_done */
+        0xc4430132,
-/* 0x067e: i2c_raise_scl */
+        0x0033cf07,
-        0x40f900f8,
+        0xf40432fd,
-        0x03089844,
+        0x31f4060b,
-        0x06227e01,
+/* 0x068d: i2c_sense_sda_done */
-/* 0x0689: i2c_raise_scl_wait */
+/* 0x068f: i2c_raise_scl */
-        0x03e84e00,
+        0xf900f801,
-        0x00005d7e,
+        0x08984440,
-        0x0006567e,
+        0x337e0103,
-        0xb60901f4,
+/* 0x069a: i2c_raise_scl_wait */
-        0x1bf40142,
+        0xe84e0006,
-/* 0x069d: i2c_raise_scl_done */
+        0x005d7e03,
-        0xf840fcef,
+        0x06677e00,
-/* 0x06a1: i2c_start */
+        0x0901f400,
-        0x06567e00,
+        0xf40142b6,
-        0x0d11f400,
+/* 0x06ae: i2c_raise_scl_done */
-        0x00066a7e,
+        0x40fcef1b,
-        0xf40611f4,
+/* 0x06b2: i2c_start */
-/* 0x06b2: i2c_start_rep */
+        0x677e00f8,
-        0x00032e0e,
+        0x11f40006,
-        0x0006227e,
+        0x067b7e0d,
-        0x3c7e0103,
+        0x0611f400,
-        0x76bb0006,
+/* 0x06c3: i2c_start_rep */
-        0x0465b600,
+        0x032e0ef4,
-        0x659450f9,
+        0x06337e00,
-        0x0256bb04,
-        0x75fd50bd,
-        0x7e50fc04,
-        0xb600067e,
-        0x11f40464,
-/* 0x06dd: i2c_start_send */
-        0x7e00031d,
-        0x4e00063c,
-        0x5d7e1388,
-        0x00030000,
-        0x0006227e,
-        0x7e13884e,
-/* 0x06f7: i2c_start_out */
-        0xf800005d,
-/* 0x06f9: i2c_stop */
-        0x7e000300,
-        0x03000622,
-        0x063c7e00,
-        0x03e84e00,
-        0x00005d7e,
-        0x227e0103,
-        0x884e0006,
-        0x005d7e13,
        0x7e010300,
-        0x4e00063c,
+        0xbb00064d,
-        0x5d7e1388,
-        0x00f80000,
-/* 0x0728: i2c_bitw */
-        0x00063c7e,
-        0x7e03e84e,
-        0xbb00005d,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x00067e7e,
+        0x00068f7e,
        0xf40464b6,
-        0x884e1711,
+/* 0x06ee: i2c_start_send */
-        0x005d7e13,
+        0x00031d11,
-        0x7e000300,
+        0x00064d7e,
-        0x4e000622,
+        0x7e13884e,
-        0x5d7e1388,
+        0x0300005d,
-/* 0x0766: i2c_bitw_out */
+        0x06337e00,
-        0x00f80000,
+        0x13884e00,
-/* 0x0768: i2c_bitr */
+        0x00005d7e,
-        0x3c7e0103,
+/* 0x0708: i2c_start_out */
+/* 0x070a: i2c_stop */
+        0x000300f8,
+        0x0006337e,
+        0x4d7e0003,
        0xe84e0006,
        0x005d7e03,
-        0x0076bb00,
+        0x7e010300,
-        0xf90465b6,
+        0x4e000633,
-        0x04659450,
+        0x5d7e1388,
-        0xbd0256bb,
+        0x01030000,
-        0x0475fd50,
+        0x00064d7e,
-        0x7e7e50fc,
+        0x7e13884e,
-        0x64b60006,
+        0xf800005d,
-        0x1a11f404,
+/* 0x0739: i2c_bitw */
-        0x00066a7e,
+        0x064d7e00,
-        0x227e0003,
+        0x03e84e00,
-        0x884e0006,
+        0x00005d7e,
-        0x005d7e13,
-        0x013cf000,
-/* 0x07ab: i2c_bitr_done */
-        0xf80131f4,
-/* 0x07ad: i2c_get_byte */
-        0x04000500,
-/* 0x07b1: i2c_get_byte_next */
-        0x0154b608,
        0xb60076bb,
        0x50f90465,
        0xbb046594,
        0x50bd0256,
        0xfc0475fd,
-        0x07687e50,
+        0x068f7e50,
        0x0464b600,
-        0xfd2a11f4,
+        0x4e1711f4,
-        0x42b60553,
+        0x5d7e1388,
-        0xd81bf401,
+        0x00030000,
-        0x76bb0103,
+        0x0006337e,
+        0x7e13884e,
+/* 0x0777: i2c_bitw_out */
+        0xf800005d,
+/* 0x0779: i2c_bitr */
+        0x7e010300,
+        0x4e00064d,
+        0x5d7e03e8,
+        0x76bb0000,
        0x0465b600,
        0x659450f9,
        0x0256bb04,
        0x75fd50bd,
        0x7e50fc04,
-        0xb6000728,
+        0xb600068f,
-/* 0x07fa: i2c_get_byte_done */
+        0x11f40464,
-        0x00f80464,
+        0x067b7e1a,
-/* 0x07fc: i2c_put_byte */
+        0x7e000300,
-/* 0x07fe: i2c_put_byte_next */
+        0x4e000633,
-        0x42b60804,
+        0x5d7e1388,
-        0x3854ff01,
+        0x3cf00000,
-        0xb60076bb,
+        0x0131f401,
-        0x50f90465,
+/* 0x07bc: i2c_bitr_done */
-        0xbb046594,
+/* 0x07be: i2c_get_byte */
-        0x50bd0256,
+        0x000500f8,
-        0xfc0475fd,
+/* 0x07c2: i2c_get_byte_next */
-        0x07287e50,
+        0x54b60804,
-        0x0464b600,
+        0x0076bb01,
-        0xb03411f4,
+        0xf90465b6,
-        0x1bf40046,
+        0x04659450,
-        0x0076bbd8,
+        0xbd0256bb,
+        0x0475fd50,
+        0x797e50fc,
+        0x64b60007,
+        0x2a11f404,
+        0xb60553fd,
+        0x1bf40142,
+        0xbb0103d8,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x0007397e,
+/* 0x080b: i2c_get_byte_done */
+        0xf80464b6,
+/* 0x080d: i2c_put_byte */
+/* 0x080f: i2c_put_byte_next */
+        0xb6080400,
+        0x54ff0142,
+        0x0076bb38,
        0xf90465b6,
        0x04659450,
        0xbd0256bb,
        0x0475fd50,
-        0x687e50fc,
+        0x397e50fc,
        0x64b60007,
-        0x0f11f404,
+        0x3411f404,
-        0xb00076bb,
+        0xf40046b0,
-        0x1bf40136,
+        0x76bbd81b,
-        0x0132f406,
-/* 0x0854: i2c_put_byte_done */
-/* 0x0856: i2c_addr */
-        0x76bb00f8,
        0x0465b600,
        0x659450f9,
        0x0256bb04,
        0x75fd50bd,
        0x7e50fc04,
-        0xb60006a1,
+        0xb6000779,
        0x11f40464,
-        0x2ec3e729,
+        0x0076bb0f,
-        0x0134b601,
+        0xf40136b0,
-        0xbb0553fd,
+        0x32f4061b,
+/* 0x0865: i2c_put_byte_done */
+/* 0x0867: i2c_addr */
+        0xbb00f801,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x0007fc7e,
+        0x0006b27e,
-/* 0x089b: i2c_addr_done */
+        0xf40464b6,
-        0xf80464b6,
+        0xc3e72911,
-/* 0x089d: i2c_acquire_addr */
+        0x34b6012e,
-        0xf8cec700,
+        0x0553fd01,
-        0xb705e4b6,
+        0xb60076bb,
-        0xf8d014e0,
+        0x50f90465,
-/* 0x08a9: i2c_acquire */
+        0xbb046594,
-        0x089d7e00,
+        0x50bd0256,
-        0x00047e00,
+        0xfc0475fd,
-        0x03d9f000,
+        0x080d7e50,
-        0x00002e7e,
+        0x0464b600,
-/* 0x08ba: i2c_release */
+/* 0x08ac: i2c_addr_done */
-        0x9d7e00f8,
+/* 0x08ae: i2c_acquire_addr */
+        0xcec700f8,
+        0x05e4b6f8,
+        0xd014e0b7,
+/* 0x08ba: i2c_acquire */
+        0xae7e00f8,
        0x047e0008,
-        0xdaf00000,
+        0xd9f00000,
        0x002e7e03,
-/* 0x08cb: i2c_recv */
+/* 0x08cb: i2c_release */
-        0xf400f800,
+        0x7e00f800,
-        0xc1c70132,
+        0x7e0008ae,
-        0x0214b6f8,
+        0xf0000004,
-        0xf52816b0,
+        0x2e7e03da,
-        0xb801371f,
+        0x00f80000,
-        0x000be813,
+/* 0x08dc: i2c_recv */
-        0xb8003298,
+        0xc70132f4,
-        0x000bc013,
+        0x14b6f8c1,
-        0xf4003198,
+        0x2816b002,
-        0xd0f90231,
+        0x01371ff5,
-        0xd0f9e0f9,
+        0x0cf413b8,
-        0x000067f1,
+        0x00329800,
-        0x100063f1,
+        0x0ccc13b8,
-        0xbb016792,
+        0x00319800,
+        0xf90231f4,
+        0xf9e0f9d0,
+        0x0067f1d0,
+        0x0063f100,
+        0x01679210,
+        0xb60076bb,
+        0x50f90465,
+        0xbb046594,
+        0x50bd0256,
+        0xfc0475fd,
+        0x08ba7e50,
+        0x0464b600,
+        0xd6b0d0fc,
+        0xb01bf500,
+        0xbb000500,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x0008a97e,
+        0x0008677e,
-        0xfc0464b6,
+        0xf50464b6,
-        0x00d6b0d0,
+        0xc700cc11,
-        0x00b01bf5,
+        0x76bbe0c5,
-        0x76bb0005,
        0x0465b600,
        0x659450f9,
        0x0256bb04,
        0x75fd50bd,
        0x7e50fc04,
-        0xb6000856,
+        0xb600080d,
        0x11f50464,
-        0xc5c700cc,
+        0x010500a9,
-        0x0076bbe0,
+        0xb60076bb,
-        0xf90465b6,
+        0x50f90465,
-        0x04659450,
+        0xbb046594,
-        0xbd0256bb,
+        0x50bd0256,
-        0x0475fd50,
+        0xfc0475fd,
-        0xfc7e50fc,
+        0x08677e50,
-        0x64b60007,
+        0x0464b600,
-        0xa911f504,
+        0x008711f5,
-        0xbb010500,
+        0xb60076bb,
-        0x65b60076,
+        0x50f90465,
-        0x9450f904,
+        0xbb046594,
-        0x56bb0465,
+        0x50bd0256,
-        0xfd50bd02,
+        0xfc0475fd,
-        0x50fc0475,
+        0x07be7e50,
-        0x0008567e,
+        0x0464b600,
-        0xf50464b6,
+        0xcb6711f4,
-        0xbb008711,
+        0x76bbe05b,
-        0x65b60076,
+        0x0465b600,
-        0x9450f904,
+        0x659450f9,
-        0x56bb0465,
+        0x0256bb04,
-        0xfd50bd02,
+        0x75fd50bd,
-        0x50fc0475,
+        0x7e50fc04,
-        0x0007ad7e,
+        0xb600070a,
-        0xf40464b6,
+        0x5bb20464,
-        0x5bcb6711,
+        0x0ef474bd,
-        0x0076bbe0,
+/* 0x09e1: i2c_recv_not_rd08 */
-        0xf90465b6,
+        0x01d6b041,
-        0x04659450,
+        0x053b1bf4,
-        0xbd0256bb,
+        0x08677e00,
-        0x0475fd50,
+        0x3211f400,
-        0xf97e50fc,
+        0x7ee0c5c7,
-        0x64b60006,
+        0xf400080d,
-        0xbd5bb204,
+        0x00052811,
-        0x410ef474,
+        0x0008677e,
-/* 0x09d0: i2c_recv_not_rd08 */
+        0xc71f11f4,
-        0xf401d6b0,
+        0x0d7ee0b5,
-        0x00053b1b,
+        0x11f40008,
-        0x0008567e,
+        0x070a7e15,
-        0xc73211f4,
+        0xc774bd00,
-        0xfc7ee0c5,
+        0x1bf408c5,
-        0x11f40007,
+        0x0232f409,
-        0x7e000528,
+/* 0x0a1f: i2c_recv_not_wr08 */
-        0xf4000856,
+/* 0x0a1f: i2c_recv_done */
-        0xb5c71f11,
+        0xc7030ef4,
-        0x07fc7ee0,
+        0xcb7ef8ce,
-        0x1511f400,
+        0xe0fc0008,
-        0x0006f97e,
+        0x12f4d0fc,
-        0xc5c774bd,
+        0x7e7cb209,
-        0x091bf408,
+/* 0x0a33: i2c_recv_exit */
-        0xf40232f4,
+        0xf80002c2,
-/* 0x0a0e: i2c_recv_not_wr08 */
+/* 0x0a35: i2c_init */
-/* 0x0a0e: i2c_recv_done */
+/* 0x0a37: test_recv */
-        0xcec7030e,
+        0x4100f800,
-        0x08ba7ef8,
+        0x11cf0458,
-        0xfce0fc00,
-        0x0912f4d0,
-        0xc27e7cb2,
-/* 0x0a22: i2c_recv_exit */
-        0x00f80002,
-/* 0x0a24: i2c_init */
-/* 0x0a26: test_recv */
-        0x584100f8,
-        0x0011cf04,
-        0x400110b6,
-        0x01f60458,
-        0xf104bd00,
-        0xf1d900e7,
-        0x7e134fe3,
-        0xf8000201,
-/* 0x0a45: test_init */
-        0x08004e00,
-        0x0002017e,
-/* 0x0a4e: idle_recv */
-        0x00f800f8,
-/* 0x0a50: idle */
-        0x410031f4,
-        0x11cf0454,
        0x0110b600,
-        0xf6045440,
+        0xf6045840,
        0x04bd0001,
-/* 0x0a64: idle_loop */
+        0xd900e7f1,
-        0x32f45801,
+        0x134fe3f1,
-/* 0x0a69: idle_proc */
+        0x0002017e,
-/* 0x0a69: idle_proc_exec */
+/* 0x0a56: test_init */
-        0xb210f902,
+        0x004e00f8,
-        0x02cb7e1e,
+        0x02017e08,
-        0xf410fc00,
+/* 0x0a5f: idle_recv */
-        0x31f40911,
+        0xf800f800,
-        0xf00ef402,
+/* 0x0a61: idle */
-/* 0x0a7c: idle_proc_next */
+        0x0031f400,
-        0xa65810b6,
+        0xcf045441,
-        0xe81bf41f,
+        0x10b60011,
-        0xf4e002f4,
+        0x04544001,
-        0x0ef40028,
+        0xbd0001f6,
-        0x000000c6,
+/* 0x0a75: idle_loop */
-        0x00000000,
+        0xf4580104,
-        0x00000000,
+/* 0x0a7a: idle_proc */
-        0x00000000,
+/* 0x0a7a: idle_proc_exec */
-        0x00000000,
+        0x10f90232,
+        0xcb7e1eb2,
+        0x10fc0002,
+        0xf40911f4,
+        0x0ef40231,
+/* 0x0a8d: idle_proc_next */
+        0x5810b6f0,
+        0x1bf41fa6,
+        0xe002f4e8,
+        0xf40028f4,
+        0x0000c60e,
        0x00000000,
        0x00000000,
        0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h
index 64e97baabc3c..d1f9b6cb66d7 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h
@@ -46,8 +46,8 @@ uint32_t nva3_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x584d454d,
-        0x000006e0,
+        0x00000842,
-        0x000006d2,
+        0x00000834,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -68,8 +68,8 @@ uint32_t nva3_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x46524550,
-        0x000006e4,
+        0x00000846,
-        0x000006e2,
+        0x00000844,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -90,8 +90,8 @@ uint32_t nva3_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x5f433249,
-        0x00000b14,
+        0x00000c76,
-        0x000009b7,
+        0x00000b19,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -112,8 +112,8 @@ uint32_t nva3_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x54534554,
-        0x00000b3d,
+        0x00000c9f,
-        0x00000b16,
+        0x00000c78,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -134,8 +134,8 @@ uint32_t nva3_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x454c4449,
-        0x00000b49,
+        0x00000cab,
-        0x00000b47,
+        0x00000ca9,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -246,13 +246,15 @@ uint32_t nva3_pwr_data[] = {
        0x00010006,
        0x00000000,
        0x000005f8,
-/* 0x03b8: memx_func_tail */
+        0x00000007,
-/* 0x03b8: memx_ts_start */
        0x00000000,
-/* 0x03bc: memx_ts_end */
+        0x0000067e,
+/* 0x03c4: memx_func_tail */
+/* 0x03c4: memx_ts_start */
        0x00000000,
-/* 0x03c0: memx_data_head */
+/* 0x03c8: memx_ts_end */
        0x00000000,
+/* 0x03cc: memx_data_head */
        0x00000000,
        0x00000000,
        0x00000000,
@@ -764,8 +766,75 @@ uint32_t nva3_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x00000000,
-/* 0x0bc0: memx_data_tail */
+        0x00000000,
-/* 0x0bc0: i2c_scl_map */
+/* 0x0bcc: memx_data_tail */
+/* 0x0bcc: memx_train_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0ccc: memx_train_tail */
+/* 0x0ccc: i2c_scl_map */
        0x00001000,
        0x00004000,
        0x00010000,
@@ -776,7 +845,7 @@ uint32_t nva3_pwr_data[] = {
        0x01000000,
        0x04000000,
        0x10000000,
-/* 0x0be8: i2c_sda_map */
+/* 0x0cf4: i2c_sda_map */
        0x00002000,
        0x00008000,
        0x00020000,
@@ -787,7 +856,7 @@ uint32_t nva3_pwr_data[] = {
        0x02000000,
        0x08000000,
        0x20000000,
-/* 0x0c10: i2c_ctrl */
+/* 0x0d1c: i2c_ctrl */
        0x0000e138,
        0x0000e150,
        0x0000e168,
@@ -845,9 +914,6 @@ uint32_t nva3_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
 };
 uint32_t nva3_pwr_code[] = {
@@ -1258,11 +1324,11 @@ uint32_t nva3_pwr_code[] = {
        0x67f0f30b,
        0x0664b62c,
        0x800066cf,
-        0x00f8ee06,
+        0x00f8f106,
 /* 0x05a8: memx_func_leave */
        0xb62c67f0,
        0x66cf0664,
-        0xef068000,
+        0xf2068000,
        0xf10467f0,
        0xb607e407,
        0x06d00604,
@@ -1323,408 +1389,479 @@ uint32_t nva3_pwr_code[] = {
        0x9800f8a4,
        0x10b6001e,
        0x7f21f404,
-/* 0x067e: memx_exec */
+/* 0x067e: memx_func_train */
-        0xe0f900f8,
+        0x57f100f8,
-        0xc1b9d0f9,
+        0x77f10003,
-        0x02b2b902,
+        0x97f10000,
-/* 0x0688: memx_exec_next */
+        0x93f00000,
-        0xb6001398,
+        0x029eb970,
-        0x34e70410,
+        0xb90421f4,
-        0x33e701f0,
+        0xe7f102d8,
-        0x32b601e0,
+        0x21f42710,
-        0x0c30f001,
+/* 0x069d: memx_func_train_loop_outer */
-        0xf9de3598,
+        0x0158e07f,
-        0x0612b855,
+        0x0083f101,
-        0x98e41ef4,
+        0xe097f102,
-        0x0c98ee0b,
+        0x1193f011,
-        0x02cbbbef,
+        0x80f990f9,
-        0x07c4b7f1,
+        0xe0fcd0fc,
-        0xcf06b4b6,
+        0xf93f21f4,
-        0xd0fc00bb,
+        0x0067f150,
-        0x21f5e0fc,
+/* 0x06bd: memx_func_train_loop_inner */
+        0x1187f100,
+        0x9068ff11,
+        0xfd109894,
+        0x97f10589,
+        0x93f00720,
+        0xf990f910,
+        0xfcd0fc80,
+        0x3f21f4e0,
+        0x008097f1,
+        0xb91093f0,
+        0x21f4029e,
+        0x02d8b904,
+        0xf92088c5,
+        0xfc80f990,
+        0xf4e0fcd0,
+        0x97f13f21,
+        0x93f0053c,
+        0x0287f110,
+        0x0083f130,
+        0xf990f980,
+        0xfcd0fc80,
+        0x3f21f4e0,
+        0x0560e7f1,
+        0xf110e3f0,
+        0xf10000d7,
+        0x908000d3,
+        0xb7f100dc,
+        0xb3f08480,
+        0xa421f41e,
+        0x000057f1,
+        0xffff97f1,
+        0x830093f1,
+/* 0x073c: memx_func_train_loop_4x */
+        0x0080a7f1,
+        0xb910a3f0,
+        0x21f402ae,
+        0x02d8b904,
+        0xffdfb7f1,
+        0xffffb3f1,
+        0xf9048bfd,
+        0xfc80f9a0,
+        0xf4e0fcd0,
+        0xa7f13f21,
+        0xa3f0053c,
+        0x0287f110,
+        0x0083f130,
+        0xf9a0f980,
+        0xfcd0fc80,
+        0x3f21f4e0,
+        0x0560e7f1,
+        0xf110e3f0,
+        0xf10000d7,
+        0xb98000d3,
+        0xb7f102dc,
+        0xb3f02710,
+        0xa421f400,
+        0xf402eeb9,
+        0xddb90421,
+        0x949dff02,
+        0x700150b6,
+        0x1ef40456,
+        0xcc7aa092,
+        0x00a9800b,
+        0xb60160b6,
+        0x66700470,
+        0x001ef510,
+        0xb650fcff,
+        0x56700150,
+        0xd41ef507,
+/* 0x07cf: memx_exec */
+        0xf900f8fe,
+        0xb9d0f9e0,
+        0xb2b902c1,
+/* 0x07d9: memx_exec_next */
+        0x00139802,
+        0xe70410b6,
+        0xe701f034,
+        0xb601e033,
+        0x30f00132,
+        0xde35980c,
+        0x12b855f9,
+        0xe41ef406,
+        0x98f10b98,
+        0xcbbbf20c,
+        0xc4b7f102,
+        0x06b4b607,
+        0xfc00bbcf,
+        0xf5e0fcd0,
+        0xf8034221,
+/* 0x0815: memx_info */
+        0x01c67000,
+/* 0x081b: memx_info_data */
+        0xf10e0bf4,
+        0xf103ccc7,
+        0xf40800b7,
+/* 0x0826: memx_info_train */
+        0xc7f10b0e,
+        0xb7f10bcc,
+/* 0x082e: memx_info_send */
+        0x21f50100,
        0x00f80342,
-/* 0x06c4: memx_info */
+/* 0x0834: memx_recv */
-        0x03c0c7f1,
+        0xf401d6b0,
-        0x0800b7f1,
+        0xd6b0980b,
-        0x034221f5,
+        0xd80bf400,
-/* 0x06d2: memx_recv */
+/* 0x0842: memx_init */
-        0xd6b000f8,
+        0x00f800f8,
-        0xa90bf401,
+/* 0x0844: perf_recv */
-        0xf400d6b0,
+/* 0x0846: perf_init */
-        0x00f8e90b,
-/* 0x06e0: memx_init */
-/* 0x06e2: perf_recv */
        0x00f800f8,
-/* 0x06e4: perf_init */
+/* 0x0848: i2c_drive_scl */
-/* 0x06e6: i2c_drive_scl */
+        0xf40036b0,
+        0x07f1110b,
+        0x04b607e0,
+        0x0001d006,
+        0x00f804bd,
+/* 0x085c: i2c_drive_scl_lo */
+        0x07e407f1,
+        0xd00604b6,
+        0x04bd0001,
+/* 0x086a: i2c_drive_sda */
        0x36b000f8,
        0x110bf400,
        0x07e007f1,
        0xd00604b6,
-        0x04bd0001,
+        0x04bd0002,
-/* 0x06fa: i2c_drive_scl_lo */
+/* 0x087e: i2c_drive_sda_lo */
        0x07f100f8,
        0x04b607e4,
-        0x0001d006,
-        0x00f804bd,
-/* 0x0708: i2c_drive_sda */
-        0xf40036b0,
-        0x07f1110b,
-        0x04b607e0,
        0x0002d006,
        0x00f804bd,
-/* 0x071c: i2c_drive_sda_lo */
+/* 0x088c: i2c_sense_scl */
-        0x07e407f1,
+        0xf10132f4,
-        0xd00604b6,
+        0xb607c437,
-        0x04bd0002,
+        0x33cf0634,
-/* 0x072a: i2c_sense_scl */
+        0x0431fd00,
-        0x32f400f8,
+        0xf4060bf4,
-        0xc437f101,
+/* 0x08a2: i2c_sense_scl_done */
-        0x0634b607,
+        0x00f80131,
-        0xfd0033cf,
+/* 0x08a4: i2c_sense_sda */
-        0x0bf40431,
+        0xf10132f4,
-        0x0131f406,
+        0xb607c437,
-/* 0x0740: i2c_sense_scl_done */
+        0x33cf0634,
-/* 0x0742: i2c_sense_sda */
+        0x0432fd00,
-        0x32f400f8,
+        0xf4060bf4,
-        0xc437f101,
+/* 0x08ba: i2c_sense_sda_done */
-        0x0634b607,
+        0x00f80131,
-        0xfd0033cf,
+/* 0x08bc: i2c_raise_scl */
-        0x0bf40432,
+        0x47f140f9,
-        0x0131f406,
+        0x37f00898,
-/* 0x0758: i2c_sense_sda_done */
+        0x4821f501,
-/* 0x075a: i2c_raise_scl */
+/* 0x08c9: i2c_raise_scl_wait */
-        0x40f900f8,
+        0xe8e7f108,
-        0x089847f1,
-        0xf50137f0,
-/* 0x0767: i2c_raise_scl_wait */
-        0xf106e621,
-        0xf403e8e7,
-        0x21f57f21,
-        0x01f4072a,
-        0x0142b609,
-/* 0x077b: i2c_raise_scl_done */
-        0xfcef1bf4,
-/* 0x077f: i2c_start */
-        0xf500f840,
-        0xf4072a21,
-        0x21f50d11,
-        0x11f40742,
-        0x300ef406,
-/* 0x0790: i2c_start_rep */
-        0xf50037f0,
-        0xf006e621,
-        0x21f50137,
-        0x76bb0708,
-        0x0465b600,
-        0x659450f9,
-        0x0256bb04,
-        0x75fd50bd,
-        0xf550fc04,
-        0xb6075a21,
-        0x11f40464,
-/* 0x07bd: i2c_start_send */
-        0x0037f01f,
-        0x070821f5,
-        0x1388e7f1,
-        0xf07f21f4,
-        0x21f50037,
-        0xe7f106e6,
-        0x21f41388,
-/* 0x07d9: i2c_start_out */
-/* 0x07db: i2c_stop */
-        0xf000f87f,
-        0x21f50037,
-        0x37f006e6,
-        0x0821f500,
-        0xe8e7f107,
        0x7f21f403,
-        0xf50137f0,
+        0x088c21f5,
-        0xf106e621,
+        0xb60901f4,
-        0xf41388e7,
+        0x1bf40142,
-        0x37f07f21,
+/* 0x08dd: i2c_raise_scl_done */
-        0x0821f501,
+        0xf840fcef,
-        0x88e7f107,
+/* 0x08e1: i2c_start */
-        0x7f21f413,
+        0x8c21f500,
-/* 0x080e: i2c_bitw */
+        0x0d11f408,
-        0x21f500f8,
+        0x08a421f5,
-        0xe7f10708,
+        0xf40611f4,
-        0x21f403e8,
+/* 0x08f2: i2c_start_rep */
-        0x0076bb7f,
+        0x37f0300e,
-        0xf90465b6,
+        0x4821f500,
-        0x04659450,
+        0x0137f008,
-        0xbd0256bb,
+        0x086a21f5,
-        0x0475fd50,
+        0xb60076bb,
-        0x21f550fc,
+        0x50f90465,
-        0x64b6075a,
+        0xbb046594,
-        0x1811f404,
+        0x50bd0256,
-        0x1388e7f1,
+        0xfc0475fd,
-        0xf07f21f4,
+        0xbc21f550,
+        0x0464b608,
+/* 0x091f: i2c_start_send */
+        0xf01f11f4,
        0x21f50037,
-        0xe7f106e6,
+        0xe7f1086a,
        0x21f41388,
-/* 0x084d: i2c_bitw_out */
+        0x0037f07f,
-/* 0x084f: i2c_bitr */
+        0x084821f5,
-        0xf000f87f,
+        0x1388e7f1,
-        0x21f50137,
+/* 0x093b: i2c_start_out */
-        0xe7f10708,
+        0xf87f21f4,
-        0x21f403e8,
+/* 0x093d: i2c_stop */
-        0x0076bb7f,
+        0x0037f000,
-        0xf90465b6,
+        0x084821f5,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0x21f550fc,
-        0x64b6075a,
-        0x1b11f404,
-        0x074221f5,
        0xf50037f0,
-        0xf106e621,
+        0xf1086a21,
+        0xf403e8e7,
+        0x37f07f21,
+        0x4821f501,
+        0x88e7f108,
+        0x7f21f413,
+        0xf50137f0,
+        0xf1086a21,
        0xf41388e7,
-        0x3cf07f21,
+        0x00f87f21,
-        0x0131f401,
+/* 0x0970: i2c_bitw */
-/* 0x0894: i2c_bitr_done */
+        0x086a21f5,
-/* 0x0896: i2c_get_byte */
+        0x03e8e7f1,
-        0x57f000f8,
+        0xbb7f21f4,
-        0x0847f000,
-/* 0x089c: i2c_get_byte_next */
-        0xbb0154b6,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x084f21f5,
+        0x08bc21f5,
        0xf40464b6,
-        0x53fd2b11,
+        0xe7f11811,
-        0x0142b605,
+        0x21f41388,
-        0xf0d81bf4,
+        0x0037f07f,
-        0x76bb0137,
+        0x084821f5,
-        0x0465b600,
+        0x1388e7f1,
-        0x659450f9,
+/* 0x09af: i2c_bitw_out */
-        0x0256bb04,
+        0xf87f21f4,
-        0x75fd50bd,
+/* 0x09b1: i2c_bitr */
-        0xf550fc04,
+        0x0137f000,
-        0xb6080e21,
+        0x086a21f5,
-/* 0x08e6: i2c_get_byte_done */
+        0x03e8e7f1,
-        0x00f80464,
+        0xbb7f21f4,
-/* 0x08e8: i2c_put_byte */
+        0x65b60076,
-/* 0x08eb: i2c_put_byte_next */
+        0x9450f904,
-        0xb60847f0,
+        0x56bb0465,
-        0x54ff0142,
+        0xfd50bd02,
-        0x0076bb38,
+        0x50fc0475,
+        0x08bc21f5,
+        0xf40464b6,
+        0x21f51b11,
+        0x37f008a4,
+        0x4821f500,
+        0x88e7f108,
+        0x7f21f413,
+        0xf4013cf0,
+/* 0x09f6: i2c_bitr_done */
+        0x00f80131,
+/* 0x09f8: i2c_get_byte */
+        0xf00057f0,
+/* 0x09fe: i2c_get_byte_next */
+        0x54b60847,
+        0x0076bb01,
        0xf90465b6,
        0x04659450,
        0xbd0256bb,
        0x0475fd50,
        0x21f550fc,
-        0x64b6080e,
+        0x64b609b1,
-        0x3411f404,
+        0x2b11f404,
-        0xf40046b0,
+        0xb60553fd,
-        0x76bbd81b,
+        0x1bf40142,
-        0x0465b600,
+        0x0137f0d8,
-        0x659450f9,
+        0xb60076bb,
-        0x0256bb04,
+        0x50f90465,
-        0x75fd50bd,
+        0xbb046594,
-        0xf550fc04,
+        0x50bd0256,
-        0xb6084f21,
+        0xfc0475fd,
-        0x11f40464,
+        0x7021f550,
-        0x0076bb0f,
+        0x0464b609,
-        0xf40136b0,
+/* 0x0a48: i2c_get_byte_done */
-        0x32f4061b,
+/* 0x0a4a: i2c_put_byte */
-/* 0x0941: i2c_put_byte_done */
+        0x47f000f8,
-/* 0x0943: i2c_addr */
+/* 0x0a4d: i2c_put_byte_next */
-        0xbb00f801,
+        0x0142b608,
+        0xbb3854ff,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x077f21f5,
+        0x097021f5,
        0xf40464b6,
-        0xc3e72911,
+        0x46b03411,
-        0x34b6012e,
+        0xd81bf400,
-        0x0553fd01,
        0xb60076bb,
        0x50f90465,
        0xbb046594,
        0x50bd0256,
        0xfc0475fd,
-        0xe821f550,
+        0xb121f550,
-        0x0464b608,
+        0x0464b609,
-/* 0x0988: i2c_addr_done */
+        0xbb0f11f4,
-/* 0x098a: i2c_acquire_addr */
+        0x36b00076,
-        0xcec700f8,
+        0x061bf401,
-        0x02e4b6f8,
+/* 0x0aa3: i2c_put_byte_done */
-        0x0c10e0b7,
+        0xf80132f4,
-        0xf800ee98,
+/* 0x0aa5: i2c_addr */
-/* 0x0999: i2c_acquire */
-        0x8a21f500,
-        0x0421f409,
-        0xf403d9f0,
-        0x00f83f21,
-/* 0x09a8: i2c_release */
-        0x098a21f5,
-        0xf00421f4,
-        0x21f403da,
-/* 0x09b7: i2c_recv */
-        0xf400f83f,
-        0xc1c70132,
-        0x0214b6f8,
-        0xf52816b0,
-        0xa0013a1f,
-        0x980be813,
-        0x13a00032,
-        0x31980bc0,
-        0x0231f400,
-        0xe0f9d0f9,
-        0x67f1d0f9,
-        0x63f10000,
-        0x67921000,
-        0x0076bb01,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0x21f550fc,
-        0x64b60999,
-        0xb0d0fc04,
-        0x1bf500d6,
-        0x57f000b3,
        0x0076bb00,
        0xf90465b6,
        0x04659450,
        0xbd0256bb,
        0x0475fd50,
        0x21f550fc,
-        0x64b60943,
+        0x64b608e1,
-        0xd011f504,
+        0x2911f404,
-        0xe0c5c700,
+        0x012ec3e7,
-        0xb60076bb,
+        0xfd0134b6,
-        0x50f90465,
+        0x76bb0553,
-        0xbb046594,
+        0x0465b600,
-        0x50bd0256,
+        0x659450f9,
-        0xfc0475fd,
+        0x0256bb04,
-        0xe821f550,
+        0x75fd50bd,
-        0x0464b608,
+        0xf550fc04,
-        0x00ad11f5,
+        0xb60a4a21,
-        0xbb0157f0,
+/* 0x0aea: i2c_addr_done */
+        0x00f80464,
+/* 0x0aec: i2c_acquire_addr */
+        0xb6f8cec7,
+        0xe0b702e4,
+        0xee980d1c,
+/* 0x0afb: i2c_acquire */
+        0xf500f800,
+        0xf40aec21,
+        0xd9f00421,
+        0x3f21f403,
+/* 0x0b0a: i2c_release */
+        0x21f500f8,
+        0x21f40aec,
+        0x03daf004,
+        0xf83f21f4,
+/* 0x0b19: i2c_recv */
+        0x0132f400,
+        0xb6f8c1c7,
+        0x16b00214,
+        0x3a1ff528,
+        0xf413a001,
+        0x0032980c,
+        0x0ccc13a0,
+        0xf4003198,
+        0xd0f90231,
+        0xd0f9e0f9,
+        0x000067f1,
+        0x100063f1,
+        0xbb016792,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x094321f5,
+        0x0afb21f5,
-        0xf50464b6,
+        0xfc0464b6,
-        0xbb008a11,
+        0x00d6b0d0,
+        0x00b31bf5,
+        0xbb0057f0,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x089621f5,
+        0x0aa521f5,
-        0xf40464b6,
+        0xf50464b6,
-        0x5bcb6a11,
+        0xc700d011,
-        0x0076bbe0,
+        0x76bbe0c5,
+        0x0465b600,
+        0x659450f9,
+        0x0256bb04,
+        0x75fd50bd,
+        0xf550fc04,
+        0xb60a4a21,
+        0x11f50464,
+        0x57f000ad,
+        0x0076bb01,
        0xf90465b6,
        0x04659450,
        0xbd0256bb,
        0x0475fd50,
        0x21f550fc,
-        0x64b607db,
+        0x64b60aa5,
-        0x025bb904,
+        0x8a11f504,
-        0x0ef474bd,
+        0x0076bb00,
-/* 0x0abd: i2c_recv_not_rd08 */
+        0xf90465b6,
-        0x01d6b043,
+        0x04659450,
-        0xf03d1bf4,
+        0xbd0256bb,
-        0x21f50057,
+        0x0475fd50,
-        0x11f40943,
+        0x21f550fc,
-        0xe0c5c733,
+        0x64b609f8,
-        0x08e821f5,
+        0x6a11f404,
-        0xf02911f4,
+        0xbbe05bcb,
-        0x21f50057,
+        0x65b60076,
-        0x11f40943,
+        0x9450f904,
-        0xe0b5c71f,
+        0x56bb0465,
-        0x08e821f5,
+        0xfd50bd02,
-        0xf51511f4,
+        0x50fc0475,
-        0xbd07db21,
+        0x093d21f5,
-        0x08c5c774,
+        0xb90464b6,
-        0xf4091bf4,
+        0x74bd025b,
-        0x0ef40232,
+/* 0x0c1f: i2c_recv_not_rd08 */
-/* 0x0afd: i2c_recv_not_wr08 */
+        0xb0430ef4,
-/* 0x0afd: i2c_recv_done */
+        0x1bf401d6,
-        0xf8cec703,
+        0x0057f03d,
-        0x09a821f5,
+        0x0aa521f5,
-        0xd0fce0fc,
+        0xc73311f4,
-        0xb90a12f4,
+        0x21f5e0c5,
-        0x21f5027c,
+        0x11f40a4a,
-/* 0x0b12: i2c_recv_exit */
+        0x0057f029,
-        0x00f80342,
+        0x0aa521f5,
-/* 0x0b14: i2c_init */
+        0xc71f11f4,
-/* 0x0b16: test_recv */
+        0x21f5e0b5,
-        0x17f100f8,
+        0x11f40a4a,
-        0x14b605d8,
+        0x3d21f515,
-        0x0011cf06,
+        0xc774bd09,
-        0xf10110b6,
+        0x1bf408c5,
-        0xb605d807,
+        0x0232f409,
-        0x01d00604,
+/* 0x0c5f: i2c_recv_not_wr08 */
-        0xf104bd00,
+/* 0x0c5f: i2c_recv_done */
-        0xf1d900e7,
+        0xc7030ef4,
-        0xf5134fe3,
+        0x21f5f8ce,
-        0xf8026221,
+        0xe0fc0b0a,
-/* 0x0b3d: test_init */
+        0x12f4d0fc,
-        0x00e7f100,
+        0x027cb90a,
-        0x6221f508,
+        0x034221f5,
-/* 0x0b47: idle_recv */
+/* 0x0c74: i2c_recv_exit */
-        0xf800f802,
+/* 0x0c76: i2c_init */
-/* 0x0b49: idle */
+        0x00f800f8,
-        0x0031f400,
+/* 0x0c78: test_recv */
-        0x05d417f1,
+        0x05d817f1,
        0xcf0614b6,
        0x10b60011,
-        0xd407f101,
+        0xd807f101,
        0x0604b605,
        0xbd0001d0,
-/* 0x0b65: idle_loop */
+        0x00e7f104,
-        0x5817f004,
+        0x4fe3f1d9,
-/* 0x0b6b: idle_proc */
+        0x6221f513,
-/* 0x0b6b: idle_proc_exec */
+/* 0x0c9f: test_init */
-        0xf90232f4,
+        0xf100f802,
-        0x021eb910,
+        0xf50800e7,
-        0x034b21f5,
+        0xf8026221,
-        0x11f410fc,
+/* 0x0ca9: idle_recv */
-        0x0231f409,
+/* 0x0cab: idle */
-/* 0x0b7f: idle_proc_next */
+        0xf400f800,
-        0xb6ef0ef4,
+        0x17f10031,
-        0x1fb85810,
+        0x14b605d4,
-        0xe61bf406,
+        0x0011cf06,
-        0xf4dd02f4,
+        0xf10110b6,
-        0x0ef40028,
+        0xb605d407,
-        0x000000bb,
+        0x01d00604,
-        0x00000000,
+/* 0x0cc7: idle_loop */
-        0x00000000,
+        0xf004bd00,
-        0x00000000,
+        0x32f45817,
-        0x00000000,
+/* 0x0ccd: idle_proc */
-        0x00000000,
+/* 0x0ccd: idle_proc_exec */
-        0x00000000,
+        0xb910f902,
-        0x00000000,
+        0x21f5021e,
-        0x00000000,
+        0x10fc034b,
-        0x00000000,
+        0xf40911f4,
-        0x00000000,
+        0x0ef40231,
-        0x00000000,
+/* 0x0ce1: idle_proc_next */
-        0x00000000,
+        0x5810b6ef,
-        0x00000000,
+        0xf4061fb8,
-        0x00000000,
+        0x02f4e61b,
-        0x00000000,
+        0x0028f4dd,
-        0x00000000,
+        0x00bb0ef4,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
        0x00000000,
        0x00000000,
        0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h
index ca30fa4011b5..90221d973f84 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h
@@ -46,8 +46,8 @@ uint32_t nvc0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x584d454d,
-        0x0000074b,
+        0x0000075e,
-        0x0000073d,
+        0x00000750,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -68,8 +68,8 @@ uint32_t nvc0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x46524550,
-        0x0000074f,
+        0x00000762,
-        0x0000074d,
+        0x00000760,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -90,8 +90,8 @@ uint32_t nvc0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x5f433249,
-        0x00000b7f,
+        0x00000b92,
-        0x00000a22,
+        0x00000a35,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -112,8 +112,8 @@ uint32_t nvc0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x54534554,
-        0x00000ba8,
+        0x00000bbb,
-        0x00000b81,
+        0x00000b94,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -134,8 +134,8 @@ uint32_t nvc0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x454c4449,
-        0x00000bb4,
+        0x00000bc7,
-        0x00000bb2,
+        0x00000bc5,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -246,13 +246,15 @@ uint32_t nvc0_pwr_data[] = {
        0x00010006,
        0x00000000,
        0x00000663,
-/* 0x03b8: memx_func_tail */
+        0x00000007,
-/* 0x03b8: memx_ts_start */
        0x00000000,
-/* 0x03bc: memx_ts_end */
+        0x000006e9,
+/* 0x03c4: memx_func_tail */
+/* 0x03c4: memx_ts_start */
        0x00000000,
-/* 0x03c0: memx_data_head */
+/* 0x03c8: memx_ts_end */
        0x00000000,
+/* 0x03cc: memx_data_head */
        0x00000000,
        0x00000000,
        0x00000000,
@@ -764,8 +766,75 @@ uint32_t nvc0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x00000000,
-/* 0x0bc0: memx_data_tail */
+        0x00000000,
-/* 0x0bc0: i2c_scl_map */
+/* 0x0bcc: memx_data_tail */
+/* 0x0bcc: memx_train_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0ccc: memx_train_tail */
+/* 0x0ccc: i2c_scl_map */
        0x00001000,
        0x00004000,
        0x00010000,
@@ -776,7 +845,7 @@ uint32_t nvc0_pwr_data[] = {
        0x01000000,
        0x04000000,
        0x10000000,
-/* 0x0be8: i2c_sda_map */
+/* 0x0cf4: i2c_sda_map */
        0x00002000,
        0x00008000,
        0x00020000,
@@ -787,7 +856,7 @@ uint32_t nvc0_pwr_data[] = {
        0x02000000,
        0x08000000,
        0x20000000,
-/* 0x0c10: i2c_ctrl */
+/* 0x0d1c: i2c_ctrl */
        0x0000e138,
        0x0000e150,
        0x0000e168,
@@ -845,9 +914,6 @@ uint32_t nvc0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
 };
 uint32_t nvc0_pwr_code[] = {
@@ -1272,10 +1338,10 @@ uint32_t nvc0_pwr_code[] = {
        0xcf0664b6,
        0x06800066,
 /* 0x05db: memx_func_leave */
-        0xf000f8ee,
+        0xf000f8f1,
        0x64b62c67,
        0x0066cf06,
-        0xf0ef0680,
+        0xf0f20680,
        0x07f10467,
        0x04b607e4,
        0x0006d006,
@@ -1350,382 +1416,450 @@ uint32_t nvc0_pwr_code[] = {
        0x1e9800f8,
        0x0410b600,
        0xf87f21f4,
-/* 0x06e9: memx_exec */
+/* 0x06e9: memx_func_train */
-        0xf9e0f900,
+/* 0x06eb: memx_exec */
-        0x02c1b9d0,
+        0xf900f800,
-/* 0x06f3: memx_exec_next */
+        0xb9d0f9e0,
-        0x9802b2b9,
+        0xb2b902c1,
-        0x10b60013,
+/* 0x06f5: memx_exec_next */
-        0xf034e704,
+        0x00139802,
-        0xe033e701,
+        0xe70410b6,
-        0x0132b601,
+        0xe701f034,
-        0x980c30f0,
+        0xb601e033,
-        0x55f9de35,
+        0x30f00132,
-        0xf40612b8,
+        0xde35980c,
-        0x0b98e41e,
+        0x12b855f9,
-        0xef0c98ee,
+        0xe41ef406,
-        0xf102cbbb,
+        0x98f10b98,
-        0xb607c4b7,
+        0xcbbbf20c,
-        0xbbcf06b4,
+        0xc4b7f102,
-        0xfcd0fc00,
+        0x06b4b607,
-        0x4221f5e0,
+        0xfc00bbcf,
-/* 0x072f: memx_info */
+        0xf5e0fcd0,
-        0xf100f803,
-        0xf103c0c7,
-        0xf50800b7,
        0xf8034221,
-/* 0x073d: memx_recv */
+/* 0x0731: memx_info */
-        0x01d6b000,
+        0x01c67000,
-        0xb0a90bf4,
+/* 0x0737: memx_info_data */
-        0x0bf400d6,
+        0xf10e0bf4,
-/* 0x074b: memx_init */
+        0xf103ccc7,
-        0xf800f8e9,
+        0xf40800b7,
-/* 0x074d: perf_recv */
+/* 0x0742: memx_info_train */
-/* 0x074f: perf_init */
+        0xc7f10b0e,
-        0xf800f800,
+        0xb7f10bcc,
-/* 0x0751: i2c_drive_scl */
+/* 0x074a: memx_info_send */
-        0x0036b000,
+        0x21f50100,
-        0xf1110bf4,
+        0x00f80342,
-        0xb607e007,
+/* 0x0750: memx_recv */
-        0x01d00604,
+        0xf401d6b0,
-        0xf804bd00,
+        0xd6b0980b,
-/* 0x0765: i2c_drive_scl_lo */
+        0xd80bf400,
-        0xe407f100,
+/* 0x075e: memx_init */
-        0x0604b607,
+        0x00f800f8,
-        0xbd0001d0,
+/* 0x0760: perf_recv */
-/* 0x0773: i2c_drive_sda */
+/* 0x0762: perf_init */
-        0xb000f804,
+        0x00f800f8,
-        0x0bf40036,
+/* 0x0764: i2c_drive_scl */
-        0xe007f111,
+        0xf40036b0,
-        0x0604b607,
+        0x07f1110b,
-        0xbd0002d0,
+        0x04b607e0,
-/* 0x0787: i2c_drive_sda_lo */
+        0x0001d006,
-        0xf100f804,
+        0x00f804bd,
-        0xb607e407,
+/* 0x0778: i2c_drive_scl_lo */
-        0x02d00604,
+        0x07e407f1,
-        0xf804bd00,
+        0xd00604b6,
-/* 0x0795: i2c_sense_scl */
+        0x04bd0001,
-        0x0132f400,
+/* 0x0786: i2c_drive_sda */
-        0x07c437f1,
+        0x36b000f8,
-        0xcf0634b6,
+        0x110bf400,
-        0x31fd0033,
+        0x07e007f1,
-        0x060bf404,
+        0xd00604b6,
-/* 0x07ab: i2c_sense_scl_done */
+        0x04bd0002,
-        0xf80131f4,
+/* 0x079a: i2c_drive_sda_lo */
-/* 0x07ad: i2c_sense_sda */
+        0x07f100f8,
-        0x0132f400,
+        0x04b607e4,
-        0x07c437f1,
+        0x0002d006,
-        0xcf0634b6,
+        0x00f804bd,
-        0x32fd0033,
+/* 0x07a8: i2c_sense_scl */
-        0x060bf404,
+        0xf10132f4,
-/* 0x07c3: i2c_sense_sda_done */
+        0xb607c437,
-        0xf80131f4,
+        0x33cf0634,
-/* 0x07c5: i2c_raise_scl */
+        0x0431fd00,
-        0xf140f900,
+        0xf4060bf4,
-        0xf0089847,
+/* 0x07be: i2c_sense_scl_done */
-        0x21f50137,
+        0x00f80131,
-/* 0x07d2: i2c_raise_scl_wait */
+/* 0x07c0: i2c_sense_sda */
-        0xe7f10751,
+        0xf10132f4,
-        0x21f403e8,
+        0xb607c437,
-        0x9521f57f,
+        0x33cf0634,
-        0x0901f407,
+        0x0432fd00,
-        0xf40142b6,
+        0xf4060bf4,
-/* 0x07e6: i2c_raise_scl_done */
+/* 0x07d6: i2c_sense_sda_done */
-        0x40fcef1b,
+        0x00f80131,
-/* 0x07ea: i2c_start */
+/* 0x07d8: i2c_raise_scl */
-        0x21f500f8,
+        0x47f140f9,
-        0x11f40795,
+        0x37f00898,
-        0xad21f50d,
+        0x6421f501,
-        0x0611f407,
+/* 0x07e5: i2c_raise_scl_wait */
-/* 0x07fb: i2c_start_rep */
-        0xf0300ef4,
-        0x21f50037,
-        0x37f00751,
-        0x7321f501,
-        0x0076bb07,
-        0xf90465b6,
-        0x04659450,
-        0xbd0256bb,
-        0x0475fd50,
-        0x21f550fc,
-        0x64b607c5,
-        0x1f11f404,
-/* 0x0828: i2c_start_send */
-        0xf50037f0,
-        0xf1077321,
-        0xf41388e7,
-        0x37f07f21,
-        0x5121f500,
-        0x88e7f107,
-        0x7f21f413,
-/* 0x0844: i2c_start_out */
-/* 0x0846: i2c_stop */
-        0x37f000f8,
-        0x5121f500,
-        0x0037f007,
-        0x077321f5,
-        0x03e8e7f1,
-        0xf07f21f4,
-        0x21f50137,
-        0xe7f10751,
-        0x21f41388,
-        0x0137f07f,
-        0x077321f5,
-        0x1388e7f1,
-        0xf87f21f4,
-/* 0x0879: i2c_bitw */
-        0x7321f500,
        0xe8e7f107,
        0x7f21f403,
+        0x07a821f5,
+        0xb60901f4,
+        0x1bf40142,
+/* 0x07f9: i2c_raise_scl_done */
+        0xf840fcef,
+/* 0x07fd: i2c_start */
+        0xa821f500,
+        0x0d11f407,
+        0x07c021f5,
+        0xf40611f4,
+/* 0x080e: i2c_start_rep */
+        0x37f0300e,
+        0x6421f500,
+        0x0137f007,
+        0x078621f5,
        0xb60076bb,
        0x50f90465,
        0xbb046594,
        0x50bd0256,
        0xfc0475fd,
-        0xc521f550,
+        0xd821f550,
        0x0464b607,
-        0xf11811f4,
+/* 0x083b: i2c_start_send */
-        0xf41388e7,
+        0xf01f11f4,
+        0x21f50037,
+        0xe7f10786,
+        0x21f41388,
+        0x0037f07f,
+        0x076421f5,
+        0x1388e7f1,
+/* 0x0857: i2c_start_out */
+        0xf87f21f4,
+/* 0x0859: i2c_stop */
+        0x0037f000,
+        0x076421f5,
+        0xf50037f0,
+        0xf1078621,
+        0xf403e8e7,
        0x37f07f21,
-        0x5121f500,
+        0x6421f501,
        0x88e7f107,
        0x7f21f413,
-/* 0x08b8: i2c_bitw_out */
+        0xf50137f0,
-/* 0x08ba: i2c_bitr */
+        0xf1078621,
-        0x37f000f8,
+        0xf41388e7,
-        0x7321f501,
+        0x00f87f21,
-        0xe8e7f107,
+/* 0x088c: i2c_bitw */
-        0x7f21f403,
+        0x078621f5,
-        0xb60076bb,
+        0x03e8e7f1,
-        0x50f90465,
+        0xbb7f21f4,
-        0xbb046594,
+        0x65b60076,
-        0x50bd0256,
+        0x9450f904,
-        0xfc0475fd,
+        0x56bb0465,
-        0xc521f550,
+        0xfd50bd02,
-        0x0464b607,
+        0x50fc0475,
-        0xf51b11f4,
+        0x07d821f5,
-        0xf007ad21,
+        0xf40464b6,
-        0x21f50037,
+        0xe7f11811,
-        0xe7f10751,
        0x21f41388,
-        0x013cf07f,
+        0x0037f07f,
-/* 0x08ff: i2c_bitr_done */
+        0x076421f5,
-        0xf80131f4,
+        0x1388e7f1,
-/* 0x0901: i2c_get_byte */
+/* 0x08cb: i2c_bitw_out */
-        0x0057f000,
+        0xf87f21f4,
-/* 0x0907: i2c_get_byte_next */
+/* 0x08cd: i2c_bitr */
-        0xb60847f0,
+        0x0137f000,
-        0x76bb0154,
+        0x078621f5,
-        0x0465b600,
+        0x03e8e7f1,
-        0x659450f9,
+        0xbb7f21f4,
-        0x0256bb04,
+        0x65b60076,
-        0x75fd50bd,
+        0x9450f904,
-        0xf550fc04,
+        0x56bb0465,
-        0xb608ba21,
+        0xfd50bd02,
-        0x11f40464,
+        0x50fc0475,
-        0x0553fd2b,
+        0x07d821f5,
-        0xf40142b6,
+        0xf40464b6,
-        0x37f0d81b,
+        0x21f51b11,
+        0x37f007c0,
+        0x6421f500,
+        0x88e7f107,
+        0x7f21f413,
+        0xf4013cf0,
+/* 0x0912: i2c_bitr_done */
+        0x00f80131,
+/* 0x0914: i2c_get_byte */
+        0xf00057f0,
+/* 0x091a: i2c_get_byte_next */
+        0x54b60847,
        0x0076bb01,
        0xf90465b6,
        0x04659450,
        0xbd0256bb,
        0x0475fd50,
        0x21f550fc,
-        0x64b60879,
+        0x64b608cd,
-/* 0x0951: i2c_get_byte_done */
+        0x2b11f404,
-/* 0x0953: i2c_put_byte */
+        0xb60553fd,
-        0xf000f804,
+        0x1bf40142,
-/* 0x0956: i2c_put_byte_next */
+        0x0137f0d8,
-        0x42b60847,
-        0x3854ff01,
        0xb60076bb,
        0x50f90465,
        0xbb046594,
        0x50bd0256,
        0xfc0475fd,
-        0x7921f550,
+        0x8c21f550,
        0x0464b608,
-        0xb03411f4,
+/* 0x0964: i2c_get_byte_done */
-        0x1bf40046,
+/* 0x0966: i2c_put_byte */
-        0x0076bbd8,
+        0x47f000f8,
+/* 0x0969: i2c_put_byte_next */
+        0x0142b608,
+        0xbb3854ff,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x088c21f5,
+        0xf40464b6,
+        0x46b03411,
+        0xd81bf400,
+        0xb60076bb,
+        0x50f90465,
+        0xbb046594,
+        0x50bd0256,
+        0xfc0475fd,
+        0xcd21f550,
+        0x0464b608,
+        0xbb0f11f4,
+        0x36b00076,
+        0x061bf401,
+/* 0x09bf: i2c_put_byte_done */
+        0xf80132f4,
+/* 0x09c1: i2c_addr */
+        0x0076bb00,
        0xf90465b6,
        0x04659450,
        0xbd0256bb,
        0x0475fd50,
        0x21f550fc,
-        0x64b608ba,
+        0x64b607fd,
-        0x0f11f404,
+        0x2911f404,
-        0xb00076bb,
+        0x012ec3e7,
-        0x1bf40136,
+        0xfd0134b6,
-        0x0132f406,
+        0x76bb0553,
-/* 0x09ac: i2c_put_byte_done */
-/* 0x09ae: i2c_addr */
-        0x76bb00f8,
        0x0465b600,
        0x659450f9,
        0x0256bb04,
        0x75fd50bd,
        0xf550fc04,
-        0xb607ea21,
+        0xb6096621,
-        0x11f40464,
+/* 0x0a06: i2c_addr_done */
-        0x2ec3e729,
+        0x00f80464,
-        0x0134b601,
+/* 0x0a08: i2c_acquire_addr */
-        0xbb0553fd,
+        0xb6f8cec7,
+        0xe0b702e4,
+        0xee980d1c,
+/* 0x0a17: i2c_acquire */
+        0xf500f800,
+        0xf40a0821,
+        0xd9f00421,
+        0x3f21f403,
+/* 0x0a26: i2c_release */
+        0x21f500f8,
+        0x21f40a08,
+        0x03daf004,
+        0xf83f21f4,
+/* 0x0a35: i2c_recv */
+        0x0132f400,
+        0xb6f8c1c7,
+        0x16b00214,
+        0x3a1ff528,
+        0xf413a001,
+        0x0032980c,
+        0x0ccc13a0,
+        0xf4003198,
+        0xd0f90231,
+        0xd0f9e0f9,
+        0x000067f1,
+        0x100063f1,
+        0xbb016792,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x095321f5,
+        0x0a1721f5,
-/* 0x09f3: i2c_addr_done */
+        0xfc0464b6,
-        0xf80464b6,
+        0x00d6b0d0,
-/* 0x09f5: i2c_acquire_addr */
+        0x00b31bf5,
-        0xf8cec700,
+        0xbb0057f0,
-        0xb702e4b6,
-        0x980c10e0,
-        0x00f800ee,
-/* 0x0a04: i2c_acquire */
-        0x09f521f5,
-        0xf00421f4,
-        0x21f403d9,
-/* 0x0a13: i2c_release */
-        0xf500f83f,
-        0xf409f521,
-        0xdaf00421,
-        0x3f21f403,
-/* 0x0a22: i2c_recv */
-        0x32f400f8,
-        0xf8c1c701,
-        0xb00214b6,
-        0x1ff52816,
-        0x13a0013a,
-        0x32980be8,
-        0xc013a000,
-        0x0031980b,
-        0xf90231f4,
-        0xf9e0f9d0,
-        0x0067f1d0,
-        0x0063f100,
-        0x01679210,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0x0421f550,
-        0x0464b60a,
-        0xd6b0d0fc,
-        0xb31bf500,
-        0x0057f000,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0xae21f550,
-        0x0464b609,
-        0x00d011f5,
-        0xbbe0c5c7,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x095321f5,
+        0x09c121f5,
        0xf50464b6,
-        0xf000ad11,
+        0xc700d011,
-        0x76bb0157,
+        0x76bbe0c5,
        0x0465b600,
        0x659450f9,
        0x0256bb04,
        0x75fd50bd,
        0xf550fc04,
-        0xb609ae21,
+        0xb6096621,
        0x11f50464,
-        0x76bb008a,
+        0x57f000ad,
-        0x0465b600,
+        0x0076bb01,
-        0x659450f9,
+        0xf90465b6,
-        0x0256bb04,
+        0x04659450,
-        0x75fd50bd,
+        0xbd0256bb,
-        0xf550fc04,
+        0x0475fd50,
-        0xb6090121,
+        0x21f550fc,
-        0x11f40464,
+        0x64b609c1,
-        0xe05bcb6a,
+        0x8a11f504,
-        0xb60076bb,
+        0x0076bb00,
-        0x50f90465,
+        0xf90465b6,
-        0xbb046594,
+        0x04659450,
-        0x50bd0256,
+        0xbd0256bb,
-        0xfc0475fd,
+        0x0475fd50,
-        0x4621f550,
+        0x21f550fc,
-        0x0464b608,
+        0x64b60914,
-        0xbd025bb9,
+        0x6a11f404,
-        0x430ef474,
+        0xbbe05bcb,
-/* 0x0b28: i2c_recv_not_rd08 */
+        0x65b60076,
-        0xf401d6b0,
+        0x9450f904,
-        0x57f03d1b,
+        0x56bb0465,
-        0xae21f500,
+        0xfd50bd02,
-        0x3311f409,
+        0x50fc0475,
-        0xf5e0c5c7,
+        0x085921f5,
-        0xf4095321,
+        0xb90464b6,
-        0x57f02911,
+        0x74bd025b,
-        0xae21f500,
+/* 0x0b3b: i2c_recv_not_rd08 */
-        0x1f11f409,
+        0xb0430ef4,
-        0xf5e0b5c7,
+        0x1bf401d6,
-        0xf4095321,
+        0x0057f03d,
-        0x21f51511,
+        0x09c121f5,
-        0x74bd0846,
+        0xc73311f4,
-        0xf408c5c7,
+        0x21f5e0c5,
-        0x32f4091b,
+        0x11f40966,
-        0x030ef402,
+        0x0057f029,
-/* 0x0b68: i2c_recv_not_wr08 */
+        0x09c121f5,
-/* 0x0b68: i2c_recv_done */
+        0xc71f11f4,
-        0xf5f8cec7,
+        0x21f5e0b5,
-        0xfc0a1321,
+        0x11f40966,
-        0xf4d0fce0,
+        0x5921f515,
-        0x7cb90a12,
+        0xc774bd08,
-        0x4221f502,
+        0x1bf408c5,
-/* 0x0b7d: i2c_recv_exit */
+        0x0232f409,
-/* 0x0b7f: i2c_init */
+/* 0x0b7b: i2c_recv_not_wr08 */
-        0xf800f803,
+/* 0x0b7b: i2c_recv_done */
-/* 0x0b81: test_recv */
+        0xc7030ef4,
-        0xd817f100,
+        0x21f5f8ce,
-        0x0614b605,
+        0xe0fc0a26,
-        0xb60011cf,
+        0x12f4d0fc,
-        0x07f10110,
+        0x027cb90a,
-        0x04b605d8,
+        0x034221f5,
-        0x0001d006,
+/* 0x0b90: i2c_recv_exit */
-        0xe7f104bd,
+/* 0x0b92: i2c_init */
-        0xe3f1d900,
-        0x21f5134f,
-        0x00f80262,
-/* 0x0ba8: test_init */
-        0x0800e7f1,
-        0x026221f5,
-/* 0x0bb2: idle_recv */
        0x00f800f8,
-/* 0x0bb4: idle */
+/* 0x0b94: test_recv */
-        0xf10031f4,
+        0x05d817f1,
-        0xb605d417,
+        0xcf0614b6,
-        0x11cf0614,
+        0x10b60011,
-        0x0110b600,
+        0xd807f101,
-        0x05d407f1,
+        0x0604b605,
-        0xd00604b6,
+        0xbd0001d0,
-        0x04bd0001,
+        0x00e7f104,
-/* 0x0bd0: idle_loop */
+        0x4fe3f1d9,
-        0xf45817f0,
+        0x6221f513,
-/* 0x0bd6: idle_proc */
+/* 0x0bbb: test_init */
-/* 0x0bd6: idle_proc_exec */
+        0xf100f802,
-        0x10f90232,
+        0xf50800e7,
-        0xf5021eb9,
+        0xf8026221,
-        0xfc034b21,
+/* 0x0bc5: idle_recv */
-        0x0911f410,
+/* 0x0bc7: idle */
-        0xf40231f4,
+        0xf400f800,
-/* 0x0bea: idle_proc_next */
+        0x17f10031,
-        0x10b6ef0e,
+        0x14b605d4,
-        0x061fb858,
+        0x0011cf06,
-        0xf4e61bf4,
+        0xf10110b6,
-        0x28f4dd02,
+        0xb605d407,
-        0xbb0ef400,
+        0x01d00604,
+/* 0x0be3: idle_loop */
+        0xf004bd00,
+        0x32f45817,
+/* 0x0be9: idle_proc */
+/* 0x0be9: idle_proc_exec */
+        0xb910f902,
+        0x21f5021e,
+        0x10fc034b,
+        0xf40911f4,
+        0x0ef40231,
+/* 0x0bfd: idle_proc_next */
+        0x5810b6ef,
+        0xf4061fb8,
+        0x02f4e61b,
+        0x0028f4dd,
+        0x00bb0ef4,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
        0x00000000,
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h
index 12d86f72ad10..7e16aab44d85 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h
@@ -46,8 +46,8 @@ uint32_t nvd0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x584d454d,
-        0x00000678,
+        0x0000068b,
-        0x0000066a,
+        0x0000067d,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -68,8 +68,8 @@ uint32_t nvd0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x46524550,
-        0x0000067c,
+        0x0000068f,
-        0x0000067a,
+        0x0000068d,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -90,8 +90,8 @@ uint32_t nvd0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x5f433249,
-        0x00000a97,
+        0x00000aaa,
-        0x0000093a,
+        0x0000094d,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -112,8 +112,8 @@ uint32_t nvd0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x54534554,
-        0x00000aba,
+        0x00000acd,
-        0x00000a99,
+        0x00000aac,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -134,8 +134,8 @@ uint32_t nvd0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x454c4449,
-        0x00000ac6,
+        0x00000ad9,
-        0x00000ac4,
+        0x00000ad7,
        0x00000000,
        0x00000000,
        0x00000000,
@@ -246,13 +246,15 @@ uint32_t nvd0_pwr_data[] = {
        0x00010006,
        0x00000000,
        0x000005d3,
-/* 0x03b8: memx_func_tail */
+        0x00000007,
-/* 0x03b8: memx_ts_start */
        0x00000000,
-/* 0x03bc: memx_ts_end */
+        0x00000619,
+/* 0x03c4: memx_func_tail */
+/* 0x03c4: memx_ts_start */
        0x00000000,
-/* 0x03c0: memx_data_head */
+/* 0x03c8: memx_ts_end */
        0x00000000,
+/* 0x03cc: memx_data_head */
        0x00000000,
        0x00000000,
        0x00000000,
@@ -764,8 +766,75 @@ uint32_t nvd0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x00000000,
-/* 0x0bc0: memx_data_tail */
+        0x00000000,
-/* 0x0bc0: i2c_scl_map */
+/* 0x0bcc: memx_data_tail */
+/* 0x0bcc: memx_train_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0ccc: memx_train_tail */
+/* 0x0ccc: i2c_scl_map */
        0x00000400,
        0x00000800,
        0x00001000,
@@ -776,7 +845,7 @@ uint32_t nvd0_pwr_data[] = {
        0x00020000,
        0x00040000,
        0x00080000,
-/* 0x0be8: i2c_sda_map */
+/* 0x0cf4: i2c_sda_map */
        0x00100000,
        0x00200000,
        0x00400000,
@@ -844,9 +913,6 @@ uint32_t nvd0_pwr_data[] = {
        0x00000000,
        0x00000000,
        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
 };
 uint32_t nvd0_pwr_code[] = {
@@ -1236,11 +1302,11 @@ uint32_t nvd0_pwr_code[] = {
        0x0bf40464,
        0x2c67f0f6,
        0x800066cf,
-        0x00f8ee06,
+        0x00f8f106,
 /* 0x0554: memx_func_leave */
        0xcf2c67f0,
        0x06800066,
-        0x0467f0ef,
+        0x0467f0f2,
        0x07e407f1,
        0xbd0006d0,
 /* 0x0569: memx_func_leave_wait */
@@ -1292,379 +1358,383 @@ uint32_t nvd0_pwr_code[] = {
        0x1e9800f8,
        0x0410b600,
        0xf86721f4,
-/* 0x0619: memx_exec */
+/* 0x0619: memx_func_train */
-        0xf9e0f900,
+/* 0x061b: memx_exec */
-        0x02c1b9d0,
+        0xf900f800,
-/* 0x0623: memx_exec_next */
+        0xb9d0f9e0,
-        0x9802b2b9,
+        0xb2b902c1,
-        0x10b60013,
+/* 0x0625: memx_exec_next */
-        0xf034e704,
+        0x00139802,
-        0xe033e701,
+        0xe70410b6,
-        0x0132b601,
+        0xe701f034,
-        0x980c30f0,
+        0xb601e033,
-        0x55f9de35,
+        0x30f00132,
-        0xf40612b8,
+        0xde35980c,
-        0x0b98e41e,
+        0x12b855f9,
-        0xef0c98ee,
+        0xe41ef406,
-        0xf102cbbb,
+        0x98f10b98,
-        0xcf07c4b7,
+        0xcbbbf20c,
-        0xd0fc00bb,
+        0xc4b7f102,
-        0x21f5e0fc,
+        0x00bbcf07,
-        0x00f802f1,
+        0xe0fcd0fc,
-/* 0x065c: memx_info */
-        0x03c0c7f1,
-        0x0800b7f1,
        0x02f121f5,
-/* 0x066a: memx_recv */
+/* 0x065e: memx_info */
-        0xd6b000f8,
+        0xc67000f8,
-        0xac0bf401,
+        0x0e0bf401,
-        0xf400d6b0,
+/* 0x0664: memx_info_data */
-        0x00f8e90b,
+        0x03ccc7f1,
-/* 0x0678: memx_init */
+        0x0800b7f1,
-/* 0x067a: perf_recv */
+/* 0x066f: memx_info_train */
-        0x00f800f8,
+        0xf10b0ef4,
-/* 0x067c: perf_init */
+        0xf10bccc7,
-/* 0x067e: i2c_drive_scl */
+/* 0x0677: memx_info_send */
-        0x36b000f8,
+        0xf50100b7,
-        0x0e0bf400,
+        0xf802f121,
-        0x07e007f1,
+/* 0x067d: memx_recv */
-        0xbd0001d0,
+        0x01d6b000,
-/* 0x068f: i2c_drive_scl_lo */
+        0xb09b0bf4,
-        0xf100f804,
+        0x0bf400d6,
-        0xd007e407,
+/* 0x068b: memx_init */
+        0xf800f8d8,
+/* 0x068d: perf_recv */
+/* 0x068f: perf_init */
+        0xf800f800,
+/* 0x0691: i2c_drive_scl */
+        0x0036b000,
+        0xf10e0bf4,
+        0xd007e007,
        0x04bd0001,
-/* 0x069a: i2c_drive_sda */
+/* 0x06a2: i2c_drive_scl_lo */
-        0x36b000f8,
+        0x07f100f8,
-        0x0e0bf400,
+        0x01d007e4,
-        0x07e007f1,
+        0xf804bd00,
-        0xbd0002d0,
+/* 0x06ad: i2c_drive_sda */
-/* 0x06ab: i2c_drive_sda_lo */
+        0x0036b000,
-        0xf100f804,
+        0xf10e0bf4,
-        0xd007e407,
+        0xd007e007,
        0x04bd0002,
-/* 0x06b6: i2c_sense_scl */
+/* 0x06be: i2c_drive_sda_lo */
+        0x07f100f8,
+        0x02d007e4,
+        0xf804bd00,
+/* 0x06c9: i2c_sense_scl */
+        0x0132f400,
+        0x07c437f1,
+        0xfd0033cf,
+        0x0bf40431,
+        0x0131f406,
+/* 0x06dc: i2c_sense_scl_done */
+/* 0x06de: i2c_sense_sda */
        0x32f400f8,
        0xc437f101,
        0x0033cf07,
-        0xf40431fd,
+        0xf40432fd,
        0x31f4060b,
-/* 0x06c9: i2c_sense_scl_done */
+/* 0x06f1: i2c_sense_sda_done */
-/* 0x06cb: i2c_sense_sda */
+/* 0x06f3: i2c_raise_scl */
-        0xf400f801,
+        0xf900f801,
-        0x37f10132,
+        0x9847f140,
-        0x33cf07c4,
+        0x0137f008,
-        0x0432fd00,
+        0x069121f5,
-        0xf4060bf4,
+/* 0x0700: i2c_raise_scl_wait */
-/* 0x06de: i2c_sense_sda_done */
-        0x00f80131,
-/* 0x06e0: i2c_raise_scl */
-        0x47f140f9,
-        0x37f00898,
-        0x7e21f501,
-/* 0x06ed: i2c_raise_scl_wait */
-        0xe8e7f106,
-        0x6721f403,
-        0x06b621f5,
-        0xb60901f4,
-        0x1bf40142,
-/* 0x0701: i2c_raise_scl_done */
-        0xf840fcef,
-/* 0x0705: i2c_start */
-        0xb621f500,
-        0x0d11f406,
-        0x06cb21f5,
-        0xf40611f4,
-/* 0x0716: i2c_start_rep */
-        0x37f0300e,
-        0x7e21f500,
-        0x0137f006,
-        0x069a21f5,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0xe021f550,
-        0x0464b606,
-/* 0x0743: i2c_start_send */
-        0xf01f11f4,
-        0x21f50037,
-        0xe7f1069a,
-        0x21f41388,
-        0x0037f067,
-        0x067e21f5,
-        0x1388e7f1,
-/* 0x075f: i2c_start_out */
-        0xf86721f4,
-/* 0x0761: i2c_stop */
-        0x0037f000,
-        0x067e21f5,
-        0xf50037f0,
-        0xf1069a21,
-        0xf403e8e7,
-        0x37f06721,
-        0x7e21f501,
-        0x88e7f106,
-        0x6721f413,
-        0xf50137f0,
-        0xf1069a21,
-        0xf41388e7,
-        0x00f86721,
-/* 0x0794: i2c_bitw */
-        0x069a21f5,
        0x03e8e7f1,
-        0xbb6721f4,
+        0xf56721f4,
-        0x65b60076,
+        0xf406c921,
-        0x9450f904,
+        0x42b60901,
-        0x56bb0465,
+        0xef1bf401,
-        0xfd50bd02,
+/* 0x0714: i2c_raise_scl_done */
-        0x50fc0475,
+        0x00f840fc,
-        0x06e021f5,
+/* 0x0718: i2c_start */
-        0xf40464b6,
+        0x06c921f5,
-        0xe7f11811,
+        0xf50d11f4,
-        0x21f41388,
+        0xf406de21,
-        0x0037f067,
+        0x0ef40611,
-        0x067e21f5,
+/* 0x0729: i2c_start_rep */
-        0x1388e7f1,
+        0x0037f030,
-/* 0x07d3: i2c_bitw_out */
+        0x069121f5,
-        0xf86721f4,
+        0xf50137f0,
-/* 0x07d5: i2c_bitr */
+        0xbb06ad21,
-        0x0137f000,
-        0x069a21f5,
-        0x03e8e7f1,
-        0xbb6721f4,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x06e021f5,
+        0x06f321f5,
        0xf40464b6,
-        0x21f51b11,
+/* 0x0756: i2c_start_send */
-        0x37f006cb,
+        0x37f01f11,
-        0x7e21f500,
+        0xad21f500,
        0x88e7f106,
        0x6721f413,
-        0xf4013cf0,
+        0xf50037f0,
-/* 0x081a: i2c_bitr_done */
+        0xf1069121,
-        0x00f80131,
+        0xf41388e7,
-/* 0x081c: i2c_get_byte */
+/* 0x0772: i2c_start_out */
-        0xf00057f0,
+        0x00f86721,
-/* 0x0822: i2c_get_byte_next */
+/* 0x0774: i2c_stop */
-        0x54b60847,
+        0xf50037f0,
-        0x0076bb01,
+        0xf0069121,
-        0xf90465b6,
+        0x21f50037,
-        0x04659450,
+        0xe7f106ad,
-        0xbd0256bb,
+        0x21f403e8,
-        0x0475fd50,
+        0x0137f067,
-        0x21f550fc,
+        0x069121f5,
-        0x64b607d5,
+        0x1388e7f1,
-        0x2b11f404,
+        0xf06721f4,
-        0xb60553fd,
+        0x21f50137,
-        0x1bf40142,
+        0xe7f106ad,
-        0x0137f0d8,
+        0x21f41388,
+/* 0x07a7: i2c_bitw */
+        0xf500f867,
+        0xf106ad21,
+        0xf403e8e7,
+        0x76bb6721,
+        0x0465b600,
+        0x659450f9,
+        0x0256bb04,
+        0x75fd50bd,
+        0xf550fc04,
+        0xb606f321,
+        0x11f40464,
+        0x88e7f118,
+        0x6721f413,
+        0xf50037f0,
+        0xf1069121,
+        0xf41388e7,
+/* 0x07e6: i2c_bitw_out */
+        0x00f86721,
+/* 0x07e8: i2c_bitr */
+        0xf50137f0,
+        0xf106ad21,
+        0xf403e8e7,
+        0x76bb6721,
+        0x0465b600,
+        0x659450f9,
+        0x0256bb04,
+        0x75fd50bd,
+        0xf550fc04,
+        0xb606f321,
+        0x11f40464,
+        0xde21f51b,
+        0x0037f006,
+        0x069121f5,
+        0x1388e7f1,
+        0xf06721f4,
+        0x31f4013c,
+/* 0x082d: i2c_bitr_done */
+/* 0x082f: i2c_get_byte */
+        0xf000f801,
+        0x47f00057,
+/* 0x0835: i2c_get_byte_next */
+        0x0154b608,
        0xb60076bb,
        0x50f90465,
        0xbb046594,
        0x50bd0256,
        0xfc0475fd,
-        0x9421f550,
+        0xe821f550,
        0x0464b607,
-/* 0x086c: i2c_get_byte_done */
+        0xfd2b11f4,
-/* 0x086e: i2c_put_byte */
+        0x42b60553,
-        0x47f000f8,
+        0xd81bf401,
-/* 0x0871: i2c_put_byte_next */
+        0xbb0137f0,
-        0x0142b608,
+        0x65b60076,
-        0xbb3854ff,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x07a721f5,
+/* 0x087f: i2c_get_byte_done */
+        0xf80464b6,
+/* 0x0881: i2c_put_byte */
+        0x0847f000,
+/* 0x0884: i2c_put_byte_next */
+        0xff0142b6,
+        0x76bb3854,
+        0x0465b600,
+        0x659450f9,
+        0x0256bb04,
+        0x75fd50bd,
+        0xf550fc04,
+        0xb607a721,
+        0x11f40464,
+        0x0046b034,
+        0xbbd81bf4,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x079421f5,
+        0x07e821f5,
        0xf40464b6,
-        0x46b03411,
+        0x76bb0f11,
-        0xd81bf400,
+        0x0136b000,
+        0xf4061bf4,
+/* 0x08da: i2c_put_byte_done */
+        0x00f80132,
+/* 0x08dc: i2c_addr */
        0xb60076bb,
        0x50f90465,
        0xbb046594,
        0x50bd0256,
        0xfc0475fd,
-        0xd521f550,
+        0x1821f550,
        0x0464b607,
-        0xbb0f11f4,
+        0xe72911f4,
-        0x36b00076,
+        0xb6012ec3,
-        0x061bf401,
+        0x53fd0134,
-/* 0x08c7: i2c_put_byte_done */
+        0x0076bb05,
-        0xf80132f4,
-/* 0x08c9: i2c_addr */
-        0x0076bb00,
        0xf90465b6,
        0x04659450,
        0xbd0256bb,
        0x0475fd50,
        0x21f550fc,
-        0x64b60705,
+        0x64b60881,
-        0x2911f404,
+/* 0x0921: i2c_addr_done */
-        0x012ec3e7,
+/* 0x0923: i2c_acquire_addr */
-        0xfd0134b6,
+        0xc700f804,
-        0x76bb0553,
+        0xe4b6f8ce,
-        0x0465b600,
+        0x14e0b705,
-        0x659450f9,
+/* 0x092f: i2c_acquire */
-        0x0256bb04,
+        0xf500f8d0,
-        0x75fd50bd,
+        0xf4092321,
-        0xf550fc04,
+        0xd9f00421,
-        0xb6086e21,
-/* 0x090e: i2c_addr_done */
-        0x00f80464,
-/* 0x0910: i2c_acquire_addr */
-        0xb6f8cec7,
-        0xe0b705e4,
-        0x00f8d014,
-/* 0x091c: i2c_acquire */
-        0x091021f5,
-        0xf00421f4,
-        0x21f403d9,
-/* 0x092b: i2c_release */
-        0xf500f833,
-        0xf4091021,
-        0xdaf00421,
        0x3321f403,
-/* 0x093a: i2c_recv */
+/* 0x093e: i2c_release */
-        0x32f400f8,
+        0x21f500f8,
-        0xf8c1c701,
+        0x21f40923,
-        0xb00214b6,
+        0x03daf004,
-        0x1ff52816,
+        0xf83321f4,
-        0x13a0013a,
+/* 0x094d: i2c_recv */
-        0x32980be8,
+        0x0132f400,
-        0xc013a000,
+        0xb6f8c1c7,
-        0x0031980b,
+        0x16b00214,
-        0xf90231f4,
+        0x3a1ff528,
-        0xf9e0f9d0,
+        0xf413a001,
-        0x0067f1d0,
+        0x0032980c,
-        0x0063f100,
+        0x0ccc13a0,
-        0x01679210,
+        0xf4003198,
-        0xb60076bb,
+        0xd0f90231,
-        0x50f90465,
+        0xd0f9e0f9,
-        0xbb046594,
+        0x000067f1,
-        0x50bd0256,
+        0x100063f1,
-        0xfc0475fd,
+        0xbb016792,
-        0x1c21f550,
-        0x0464b609,
-        0xd6b0d0fc,
-        0xb31bf500,
-        0x0057f000,
-        0xb60076bb,
-        0x50f90465,
-        0xbb046594,
-        0x50bd0256,
-        0xfc0475fd,
-        0xc921f550,
-        0x0464b608,
-        0x00d011f5,
-        0xbbe0c5c7,
        0x65b60076,
        0x9450f904,
        0x56bb0465,
        0xfd50bd02,
        0x50fc0475,
-        0x086e21f5,
+        0x092f21f5,
+        0xfc0464b6,
+        0x00d6b0d0,
+        0x00b31bf5,
+        0xbb0057f0,
+        0x65b60076,
+        0x9450f904,
+        0x56bb0465,
+        0xfd50bd02,
+        0x50fc0475,
+        0x08dc21f5,
        0xf50464b6,
-        0xf000ad11,
+        0xc700d011,
-        0x76bb0157,
+        0x76bbe0c5,
        0x0465b600,
        0x659450f9,
        0x0256bb04,
        0x75fd50bd,
        0xf550fc04,
-        0xb608c921,
+        0xb6088121,
        0x11f50464,
-        0x76bb008a,
+        0x57f000ad,
-        0x0465b600,
+        0x0076bb01,
-        0x659450f9,
+        0xf90465b6,
-        0x0256bb04,
+        0x04659450,
-        0x75fd50bd,
+        0xbd0256bb,
-        0xf550fc04,
+        0x0475fd50,
-        0xb6081c21,
+        0x21f550fc,
-        0x11f40464,
+        0x64b608dc,
-        0xe05bcb6a,
+        0x8a11f504,
-        0xb60076bb,
+        0x0076bb00,
-        0x50f90465,
+        0xf90465b6,
-        0xbb046594,
+        0x04659450,
-        0x50bd0256,
+        0xbd0256bb,
-        0xfc0475fd,
+        0x0475fd50,
-        0x6121f550,
+        0x21f550fc,
-        0x0464b607,
+        0x64b6082f,
-        0xbd025bb9,
+        0x6a11f404,
-        0x430ef474,
+        0xbbe05bcb,
-/* 0x0a40: i2c_recv_not_rd08 */
+        0x65b60076,
-        0xf401d6b0,
+        0x9450f904,
-        0x57f03d1b,
+        0x56bb0465,
-        0xc921f500,
+        0xfd50bd02,
-        0x3311f408,
+        0x50fc0475,
-        0xf5e0c5c7,
+        0x077421f5,
-        0xf4086e21,
+        0xb90464b6,
-        0x57f02911,
+        0x74bd025b,
-        0xc921f500,
+/* 0x0a53: i2c_recv_not_rd08 */
-        0x1f11f408,
+        0xb0430ef4,
-        0xf5e0b5c7,
+        0x1bf401d6,
-        0xf4086e21,
+        0x0057f03d,
-        0x21f51511,
+        0x08dc21f5,
-        0x74bd0761,
+        0xc73311f4,
-        0xf408c5c7,
+        0x21f5e0c5,
-        0x32f4091b,
+        0x11f40881,
-        0x030ef402,
+        0x0057f029,
-/* 0x0a80: i2c_recv_not_wr08 */
+        0x08dc21f5,
-/* 0x0a80: i2c_recv_done */
+        0xc71f11f4,
-        0xf5f8cec7,
+        0x21f5e0b5,
-        0xfc092b21,
+        0x11f40881,
-        0xf4d0fce0,
+        0x7421f515,
-        0x7cb90a12,
+        0xc774bd07,
-        0xf121f502,
+        0x1bf408c5,
-/* 0x0a95: i2c_recv_exit */
+        0x0232f409,
-/* 0x0a97: i2c_init */
+/* 0x0a93: i2c_recv_not_wr08 */
+/* 0x0a93: i2c_recv_done */
+        0xc7030ef4,
+        0x21f5f8ce,
+        0xe0fc093e,
+        0x12f4d0fc,
+        0x027cb90a,
+        0x02f121f5,
+/* 0x0aa8: i2c_recv_exit */
+/* 0x0aaa: i2c_init */
+        0x00f800f8,
+/* 0x0aac: test_recv */
+        0x05d817f1,
+        0xb60011cf,
+        0x07f10110,
+        0x01d005d8,
+        0xf104bd00,
+        0xf1d900e7,
+        0xf5134fe3,
+        0xf8022321,
+/* 0x0acd: test_init */
+        0x00e7f100,
+        0x2321f508,
+/* 0x0ad7: idle_recv */
        0xf800f802,
-/* 0x0a99: test_recv */
+/* 0x0ad9: idle */
-        0xd817f100,
+        0x0031f400,
-        0x0011cf05,
+        0x05d417f1,
-        0xf10110b6,
+        0xb60011cf,
-        0xd005d807,
+        0x07f10110,
-        0x04bd0001,
+        0x01d005d4,
-        0xd900e7f1,
+/* 0x0aef: idle_loop */
-        0x134fe3f1,
+        0xf004bd00,
-        0x022321f5,
+        0x32f45817,
-/* 0x0aba: test_init */
+/* 0x0af5: idle_proc */
-        0xe7f100f8,
+/* 0x0af5: idle_proc_exec */
-        0x21f50800,
+        0xb910f902,
-        0x00f80223,
+        0x21f5021e,
-/* 0x0ac4: idle_recv */
+        0x10fc02fa,
-/* 0x0ac6: idle */
+        0xf40911f4,
-        0x31f400f8,
+        0x0ef40231,
-        0xd417f100,
+/* 0x0b09: idle_proc_next */
-        0x0011cf05,
+        0x5810b6ef,
-        0xf10110b6,
+        0xf4061fb8,
-        0xd005d407,
+        0x02f4e61b,
-        0x04bd0001,
+        0x0028f4dd,
-/* 0x0adc: idle_loop */
+        0x00c10ef4,
-        0xf45817f0,
-/* 0x0ae2: idle_proc */
-/* 0x0ae2: idle_proc_exec */
-        0x10f90232,
-        0xf5021eb9,
-        0xfc02fa21,
-        0x0911f410,
-        0xf40231f4,
-/* 0x0af6: idle_proc_next */
-        0x10b6ef0e,
-        0x061fb858,
-        0xf4e61bf4,
-        0x28f4dd02,
-        0xc10ef400,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
        0x00000000,
        0x00000000,
        0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/os.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/os.h
index 522e3079f824..c8b06cb77e72 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/os.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/os.h
@@ -18,6 +18,10 @@
 #define MEMX_MSG_INFO 0
 #define MEMX_MSG_EXEC 1
+/* MEMX: info types */
+#define MEMX_INFO_DATA  0
+#define MEMX_INFO_TRAIN 1
 /* MEMX: script opcode definitions */
 #define MEMX_ENTER  1
 #define MEMX_LEAVE  2
@@ -25,6 +29,7 @@
 #define MEMX_WAIT   4
 #define MEMX_DELAY  5
 #define MEMX_VBLANK 6
+#define MEMX_TRAIN  7
 /* I2C_: message identifiers */
 #define I2C__MSG_RD08 0
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c
index 65eaa2546cad..7a9299d7159f 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c
@@ -47,7 +47,8 @@ nouveau_memx_init(struct nouveau_pwr *ppwr, struct nouveau_memx **pmemx)
        u32 reply[2];
        int ret;
-        ret = ppwr->message(ppwr, reply, PROC_MEMX, MEMX_MSG_INFO, 0, 0);
+        ret = ppwr->message(ppwr, reply, PROC_MEMX, MEMX_MSG_INFO,
+                                        MEMX_INFO_DATA, 0);
        if (ret)
                return ret;
@@ -106,7 +107,7 @@ nouveau_memx_wait(struct nouveau_memx *memx,
 {
        nv_debug(memx->ppwr, "R[%06x] & 0x%08x == 0x%08x, %d us\n",
                                addr, mask, data, nsec);
-        memx_cmd(memx, MEMX_WAIT, 4, (u32[]){ addr, ~mask, data, nsec });
+        memx_cmd(memx, MEMX_WAIT, 4, (u32[]){ addr, mask, data, nsec });
        memx_out(memx); /* fuc can't handle multiple */
 }
@@ -152,6 +153,38 @@ nouveau_memx_wait_vblank(struct nouveau_memx *memx)
 }
 void
+nouveau_memx_train(struct nouveau_memx *memx)
+{
+        nv_debug(memx->ppwr, "   MEM TRAIN\n");
+        memx_cmd(memx, MEMX_TRAIN, 0, NULL);
+}
+int
+nouveau_memx_train_result(struct nouveau_pwr *ppwr, u32 *res, int rsize)
+{
+        u32 reply[2], base, size, i;
+        int ret;
+        ret = ppwr->message(ppwr, reply, PROC_MEMX, MEMX_MSG_INFO,
+                                        MEMX_INFO_TRAIN, 0);
+        if (ret)
+                return ret;
+        base = reply[0];
+        size = reply[1] >> 2;
+        if (size > rsize)
+                return -ENOMEM;
+        /* read the packet */
+        nv_wr32(ppwr, 0x10a1c0, 0x02000000 | base);
+        for (i = 0; i < size; i++)
+                res[i] = nv_rd32(ppwr, 0x10a1c4);
+        return 0;
+}
+void
 nouveau_memx_block(struct nouveau_memx *memx)
 {
        nv_debug(memx->ppwr, "   HOST BLOCKED\n");
diff --git a/drivers/gpu/drm/nouveau/core/subdev/volt/base.c b/drivers/gpu/drm/nouveau/core/subdev/volt/base.c
index 32794a999106..26ccd8df193f 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/volt/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/volt/base.c
@@ -101,6 +101,41 @@ nouveau_volt_set_id(struct nouveau_volt *volt, u8 id, int condition)
        return ret;
 }
+static void nouveau_volt_parse_bios(struct nouveau_bios *bios,
+                struct nouveau_volt *volt)
+{
+        struct nvbios_volt_entry ivid;
+        struct nvbios_volt info;
+        u8  ver, hdr, cnt, len;
+        u16 data;
+        int i;
+        data = nvbios_volt_parse(bios, &ver, &hdr, &cnt, &len, &info);
+        if (data && info.vidmask && info.base && info.step) {
+                for (i = 0; i < info.vidmask + 1; i++) {
+                        if (info.base >= info.min &&
+                                info.base <= info.max) {
+                                volt->vid[volt->vid_nr].uv = info.base;
+                                volt->vid[volt->vid_nr].vid = i;
+                                volt->vid_nr++;
+                        }
+                        info.base += info.step;
+                }
+                volt->vid_mask = info.vidmask;
+        } else if (data && info.vidmask) {
+                for (i = 0; i < cnt; i++) {
+                        data = nvbios_volt_entry_parse(bios, i, &ver, &hdr,
+                                                          &ivid);
+                        if (data) {
+                                volt->vid[volt->vid_nr].uv = ivid.voltage;
+                                volt->vid[volt->vid_nr].vid = ivid.vid;
+                                volt->vid_nr++;
+                        }
+                }
+                volt->vid_mask = info.vidmask;
+        }
+}
 int
 _nouveau_volt_init(struct nouveau_object *object)
 {
@@ -136,10 +171,6 @@ nouveau_volt_create_(struct nouveau_object *parent,
 {
        struct nouveau_bios *bios = nouveau_bios(parent);
        struct nouveau_volt *volt;
-        struct nvbios_volt_entry ivid;
-        struct nvbios_volt info;
-        u8  ver, hdr, cnt, len;
-        u16 data;
        int ret, i;
        ret = nouveau_subdev_create_(parent, engine, oclass, 0, "VOLT",
@@ -152,31 +183,9 @@ nouveau_volt_create_(struct nouveau_object *parent,
        volt->set = nouveau_volt_set;
        volt->set_id = nouveau_volt_set_id;
-        data = nvbios_volt_parse(bios, &ver, &hdr, &cnt, &len, &info);
+        /* Assuming the non-bios device should build the voltage table later */
-        if (data && info.vidmask && info.base && info.step) {
+        if (bios)
-                for (i = 0; i < info.vidmask + 1; i++) {
+                nouveau_volt_parse_bios(bios, volt);
-                        if (info.base >= info.min &&
-                            info.base <= info.max) {
-                                volt->vid[volt->vid_nr].uv = info.base;
-                                volt->vid[volt->vid_nr].vid = i;
-                                volt->vid_nr++;
-                        }
-                        info.base += info.step;
-                }
-                volt->vid_mask = info.vidmask;
-        } else
-        if (data && info.vidmask) {
-                for (i = 0; i < cnt; i++) {
-                        data = nvbios_volt_entry_parse(bios, i, &ver, &hdr,
-                                                      &ivid);
-                        if (data) {
-                                volt->vid[volt->vid_nr].uv = ivid.voltage;
-                                volt->vid[volt->vid_nr].vid = ivid.vid;
-                                volt->vid_nr++;
-                        }
-                }
-                volt->vid_mask = info.vidmask;
-        }
        if (volt->vid_nr) {
                for (i = 0; i < volt->vid_nr; i++) {
diff --git a/drivers/gpu/drm/nouveau/core/subdev/volt/gk20a.c b/drivers/gpu/drm/nouveau/core/subdev/volt/gk20a.c
new file mode 100644
index 000000000000..717368ef31ac
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/volt/gk20a.c
@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifdef __KERNEL__
+#include <nouveau_platform.h>
+#endif
+#include <subdev/volt.h>
+struct cvb_coef {
+        int c0;
+        int c1;
+        int c2;
+        int c3;
+        int c4;
+        int c5;
+};
+struct gk20a_volt_priv {
+        struct nouveau_volt base;
+        struct regulator *vdd;
+};
+const struct cvb_coef gk20a_cvb_coef[] = {
+        /* MHz,        c0,     c1,   c2,    c3,     c4,   c5 */
+        /*  72 */ { 1209886, -36468,  515,   417, -13123,  203},
+        /* 108 */ { 1130804, -27659,  296,   298, -10834,  221},
+        /* 180 */ { 1162871, -27110,  247,   238, -10681,  268},
+        /* 252 */ { 1220458, -28654,  247,   179, -10376,  298},
+        /* 324 */ { 1280953, -30204,  247,   119,  -9766,  304},
+        /* 396 */ { 1344547, -31777,  247,   119,  -8545,  292},
+        /* 468 */ { 1420168, -34227,  269,    60,  -7172,  256},
+        /* 540 */ { 1490757, -35955,  274,    60,  -5188,  197},
+        /* 612 */ { 1599112, -42583,  398,     0,  -1831,  119},
+        /* 648 */ { 1366986, -16459, -274,     0,  -3204,   72},
+        /* 684 */ { 1391884, -17078, -274,   -60,  -1526,   30},
+        /* 708 */ { 1415522, -17497, -274,   -60,   -458,    0},
+        /* 756 */ { 1464061, -18331, -274,  -119,   1831,  -72},
+        /* 804 */ { 1524225, -20064, -254,  -119,   4272, -155},
+        /* 852 */ { 1608418, -21643, -269,     0,    763,  -48},
+};
+/**
+ * cvb_mv = ((c2 * speedo / s_scale + c1) * speedo / s_scale + c0)
+ */
+static inline int
+gk20a_volt_get_cvb_voltage(int speedo, int s_scale,
+                const struct cvb_coef *coef)
+{
+        int mv;
+        mv = DIV_ROUND_CLOSEST(coef->c2 * speedo, s_scale);
+        mv = DIV_ROUND_CLOSEST((mv + coef->c1) * speedo, s_scale) + coef->c0;
+        return mv;
+}
+/**
+ * cvb_t_mv =
+ * ((c2 * speedo / s_scale + c1) * speedo / s_scale + c0) +
+ * ((c3 * speedo / s_scale + c4 + c5 * T / t_scale) * T / t_scale)
+ */
+static inline int
+gk20a_volt_get_cvb_t_voltage(int speedo, int temp, int s_scale, int t_scale,
+                const struct cvb_coef *coef)
+{
+        int cvb_mv, mv;
+        cvb_mv = gk20a_volt_get_cvb_voltage(speedo, s_scale, coef);
+        mv = DIV_ROUND_CLOSEST(coef->c3 * speedo, s_scale) + coef->c4 +
+                DIV_ROUND_CLOSEST(coef->c5 * temp, t_scale);
+        mv = DIV_ROUND_CLOSEST(mv * temp, t_scale) + cvb_mv;
+        return mv;
+}
+static int
+gk20a_volt_calc_voltage(const struct cvb_coef *coef, int speedo)
+{
+        int mv;
+        mv = gk20a_volt_get_cvb_t_voltage(speedo, -10, 100, 10, coef);
+        mv = DIV_ROUND_UP(mv, 1000);
+        return mv * 1000;
+}
+static int
+gk20a_volt_vid_get(struct nouveau_volt *volt)
+{
+        struct gk20a_volt_priv *priv = (void *)volt;
+        int i, uv;
+        uv = regulator_get_voltage(priv->vdd);
+        for (i = 0; i < volt->vid_nr; i++)
+                if (volt->vid[i].uv >= uv)
+                        return i;
+        return -EINVAL;
+}
+static int
+gk20a_volt_vid_set(struct nouveau_volt *volt, u8 vid)
+{
+        struct gk20a_volt_priv *priv = (void *)volt;
+        nv_debug(volt, "set voltage as %duv\n", volt->vid[vid].uv);
+        return regulator_set_voltage(priv->vdd, volt->vid[vid].uv, 1200000);
+}
+static int
+gk20a_volt_set_id(struct nouveau_volt *volt, u8 id, int condition)
+{
+        struct gk20a_volt_priv *priv = (void *)volt;
+        int prev_uv = regulator_get_voltage(priv->vdd);
+        int target_uv = volt->vid[id].uv;
+        int ret;
+        nv_debug(volt, "prev=%d, target=%d, condition=%d\n",
+                        prev_uv, target_uv, condition);
+        if (!condition ||
+                (condition < 0 && target_uv < prev_uv) ||
+                (condition > 0 && target_uv > prev_uv)) {
+                ret = gk20a_volt_vid_set(volt, volt->vid[id].vid);
+        } else {
+                ret = 0;
+        }
+        return ret;
+}
+static int
+gk20a_volt_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+               struct nouveau_oclass *oclass, void *data, u32 size,
+               struct nouveau_object **pobject)
+{
+        struct gk20a_volt_priv *priv;
+        struct nouveau_volt *volt;
+        struct nouveau_platform_device *plat;
+        int i, ret, uv;
+        ret = nouveau_volt_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+        volt = &priv->base;
+        plat = nv_device_to_platform(nv_device(parent));
+        uv = regulator_get_voltage(plat->gpu->vdd);
+        nv_info(priv, "The default voltage is %duV\n", uv);
+        priv->vdd = plat->gpu->vdd;
+        priv->base.vid_get = gk20a_volt_vid_get;
+        priv->base.vid_set = gk20a_volt_vid_set;
+        priv->base.set_id = gk20a_volt_set_id;
+        volt->vid_nr = ARRAY_SIZE(gk20a_cvb_coef);
+        nv_debug(priv, "%s - vid_nr = %d\n", __func__, volt->vid_nr);
+        for (i = 0; i < volt->vid_nr; i++) {
+                volt->vid[i].vid = i;
+                volt->vid[i].uv = gk20a_volt_calc_voltage(&gk20a_cvb_coef[i],
+                                        plat->gpu_speedo);
+                nv_debug(priv, "%2d: vid=%d, uv=%d\n", i, volt->vid[i].vid,
+                                        volt->vid[i].uv);
+        }
+        return 0;
+}
+struct nouveau_oclass
+gk20a_volt_oclass = {
+        .handle = NV_SUBDEV(VOLT, 0xea),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gk20a_volt_ctor,
+                .dtor = _nouveau_volt_dtor,
+                .init = _nouveau_volt_init,
+                .fini = _nouveau_volt_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
index 2a03e77abef4..38402ade6835 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
@@ -614,7 +614,7 @@ nv_crtc_swap_fbs(struct drm_crtc *crtc, struct drm_framebuffer *old_fb)
        struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
        int ret;
-        ret = nouveau_bo_pin(nvfb->nvbo, TTM_PL_FLAG_VRAM);
+        ret = nouveau_bo_pin(nvfb->nvbo, TTM_PL_FLAG_VRAM, false);
        if (ret == 0) {
                if (disp->image[nv_crtc->index])
                        nouveau_bo_unpin(disp->image[nv_crtc->index]);
@@ -1130,7 +1130,7 @@ nv04_crtc_create(struct drm_device *dev, int crtc_num)
        ret = nouveau_bo_new(dev, 64*64*4, 0x100, TTM_PL_FLAG_VRAM,
                             0, 0x0000, NULL, NULL, &nv_crtc->cursor.nvbo);
        if (!ret) {
-                ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM);
+                ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM, false);
                if (!ret) {
                        ret = nouveau_bo_map(nv_crtc->cursor.nvbo);
                        if (ret)
diff --git a/drivers/gpu/drm/nouveau/dispnv04/overlay.c b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
index 1e9056a8df94..9f2498571d09 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/overlay.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
@@ -126,7 +126,7 @@ nv10_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
                        return -ERANGE;
        }
-        ret = nouveau_bo_pin(nv_fb->nvbo, TTM_PL_FLAG_VRAM);
+        ret = nouveau_bo_pin(nv_fb->nvbo, TTM_PL_FLAG_VRAM, false);
        if (ret)
                return ret;
@@ -373,7 +373,7 @@ nv04_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
        if (crtc_w < src_w || crtc_h < src_h)
                return -ERANGE;
-        ret = nouveau_bo_pin(nv_fb->nvbo, TTM_PL_FLAG_VRAM);
+        ret = nouveau_bo_pin(nv_fb->nvbo, TTM_PL_FLAG_VRAM, false);
        if (ret)
                return ret;
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index a24faa5e2a2a..d39a15000068 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -308,7 +308,7 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
        ret = nouveau_gem_new(dev, PAGE_SIZE, 0, NOUVEAU_GEM_DOMAIN_GART,
                              0, 0, &chan->ntfy);
        if (ret == 0)
-                ret = nouveau_bo_pin(chan->ntfy, TTM_PL_FLAG_TT);
+                ret = nouveau_bo_pin(chan->ntfy, TTM_PL_FLAG_TT, false);
        if (ret)
                goto done;
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index dae2c96deef8..7df6acc8bb34 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -1258,7 +1258,7 @@ olddcb_table(struct drm_device *dev)
                return NULL;
        }
-        if (dcb[0] >= 0x41) {
+        if (dcb[0] >= 0x42) {
                NV_WARN(drm, "DCB version 0x%02x unknown\n", dcb[0]);
                return NULL;
        } else
@@ -1481,18 +1481,22 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
                        entry->dpconf.link_bw = 540000;
                        break;
                }
-                switch ((conf & 0x0f000000) >> 24) {
+                entry->dpconf.link_nr = (conf & 0x0f000000) >> 24;
-                case 0xf:
+                if (dcb->version < 0x41) {
-                        entry->dpconf.link_nr = 4;
+                        switch (entry->dpconf.link_nr) {
-                        break;
+                        case 0xf:
-                case 0x3:
+                                entry->dpconf.link_nr = 4;
-                        entry->dpconf.link_nr = 2;
+                                break;
-                        break;
+                        case 0x3:
-                default:
+                                entry->dpconf.link_nr = 2;
-                        entry->dpconf.link_nr = 1;
+                                break;
-                        break;
+                        default:
+                                entry->dpconf.link_nr = 1;
+                                break;
+                        }
                }
                link = entry->dpconf.sor.link;
+                entry->i2c_index += NV_I2C_AUX(0);
                break;
        case DCB_OUTPUT_TMDS:
                if (dcb->version >= 0x40) {
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 3d474ac03f88..21ec561edc99 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -214,6 +214,9 @@ nouveau_bo_new(struct drm_device *dev, int size, int align,
        nvbo->tile_flags = tile_flags;
        nvbo->bo.bdev = &drm->ttm.bdev;
+        if (!nv_device_is_cpu_coherent(nvkm_device(&drm->device)))
+                nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;
        nvbo->page_shift = 12;
        if (drm->client.vm) {
                if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
@@ -291,8 +294,9 @@ void
 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
 {
        struct ttm_placement *pl = &nvbo->placement;
-        uint32_t flags = TTM_PL_MASK_CACHING |
+        uint32_t flags = (nvbo->force_coherent ? TTM_PL_FLAG_UNCACHED :
-                (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
+                                                 TTM_PL_MASK_CACHING) |
+                         (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
        pl->placement = nvbo->placements;
        set_placement_list(nvbo->placements, &pl->num_placement,
@@ -306,42 +310,75 @@ nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
 }
 int
-nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype)
+nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype, bool contig)
 {
        struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
        struct ttm_buffer_object *bo = &nvbo->bo;
+        bool force = false, evict = false;
        int ret;
        ret = ttm_bo_reserve(bo, false, false, false, NULL);
        if (ret)
-                goto out;
+                return ret;
-        if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) {
+        if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
-                NV_ERROR(drm, "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo,
+            memtype == TTM_PL_FLAG_VRAM && contig) {
-                         1 << bo->mem.mem_type, memtype);
+                if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG) {
-                ret = -EINVAL;
+                        if (bo->mem.mem_type == TTM_PL_VRAM) {
-                goto out;
+                                struct nouveau_mem *mem = bo->mem.mm_node;
+                                if (!list_is_singular(&mem->regions))
+                                        evict = true;
+                        }
+                        nvbo->tile_flags &= ~NOUVEAU_GEM_TILE_NONCONTIG;
+                        force = true;
+                }
        }
-        if (nvbo->pin_refcnt++)
+        if (nvbo->pin_refcnt) {
+                if (!(memtype & (1 << bo->mem.mem_type)) || evict) {
+                        NV_ERROR(drm, "bo %p pinned elsewhere: "
+                                      "0x%08x vs 0x%08x\n", bo,
+                                 1 << bo->mem.mem_type, memtype);
+                        ret = -EBUSY;
+                }
+                nvbo->pin_refcnt++;
                goto out;
+        }
+        if (evict) {
+                nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 0);
+                ret = nouveau_bo_validate(nvbo, false, false);
+                if (ret)
+                        goto out;
+        }
+        nvbo->pin_refcnt++;
        nouveau_bo_placement_set(nvbo, memtype, 0);
+        /* drop pin_refcnt temporarily, so we don't trip the assertion
+         * in nouveau_bo_move() that makes sure we're not trying to
+         * move a pinned buffer
+         */
+        nvbo->pin_refcnt--;
        ret = nouveau_bo_validate(nvbo, false, false);
-        if (ret == 0) {
+        if (ret)
-                switch (bo->mem.mem_type) {
+                goto out;
-                case TTM_PL_VRAM:
+        nvbo->pin_refcnt++;
-                        drm->gem.vram_available -= bo->mem.size;
-                        break;
+        switch (bo->mem.mem_type) {
-                case TTM_PL_TT:
+        case TTM_PL_VRAM:
-                        drm->gem.gart_available -= bo->mem.size;
+                drm->gem.vram_available -= bo->mem.size;
-                        break;
+                break;
-                default:
+        case TTM_PL_TT:
-                        break;
+                drm->gem.gart_available -= bo->mem.size;
-                }
+                break;
+        default:
+                break;
        }
 out:
+        if (force && ret)
+                nvbo->tile_flags |= NOUVEAU_GEM_TILE_NONCONTIG;
        ttm_bo_unreserve(bo);
        return ret;
 }
@@ -392,7 +429,14 @@ nouveau_bo_map(struct nouveau_bo *nvbo)
        if (ret)
                return ret;
-        ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);
+        /*
+         * TTM buffers allocated using the DMA API already have a mapping, let's
+         * use it instead.
+         */
+        if (!nvbo->force_coherent)
+                ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
+                                  &nvbo->kmap);
        ttm_bo_unreserve(&nvbo->bo);
        return ret;
 }
@@ -400,10 +444,57 @@ nouveau_bo_map(struct nouveau_bo *nvbo)
 void
 nouveau_bo_unmap(struct nouveau_bo *nvbo)
 {
-        if (nvbo)
+        if (!nvbo)
+                return;
+        /*
+         * TTM buffers allocated using the DMA API already had a coherent
+         * mapping which we used, no need to unmap.
+         */
+        if (!nvbo->force_coherent)
                ttm_bo_kunmap(&nvbo->kmap);
 }
+void
+nouveau_bo_sync_for_device(struct nouveau_bo *nvbo)
+{
+        struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
+        struct nouveau_device *device = nvkm_device(&drm->device);
+        struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
+        int i;
+        if (!ttm_dma)
+                return;
+        /* Don't waste time looping if the object is coherent */
+        if (nvbo->force_coherent)
+                return;
+        for (i = 0; i < ttm_dma->ttm.num_pages; i++)
+                dma_sync_single_for_device(nv_device_base(device),
+                        ttm_dma->dma_address[i], PAGE_SIZE, DMA_TO_DEVICE);
+}
+void
+nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo)
+{
+        struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
+        struct nouveau_device *device = nvkm_device(&drm->device);
+        struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
+        int i;
+        if (!ttm_dma)
+                return;
+        /* Don't waste time looping if the object is coherent */
+        if (nvbo->force_coherent)
+                return;
+        for (i = 0; i < ttm_dma->ttm.num_pages; i++)
+                dma_sync_single_for_cpu(nv_device_base(device),
+                        ttm_dma->dma_address[i], PAGE_SIZE, DMA_FROM_DEVICE);
+}
 int
 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
                    bool no_wait_gpu)
@@ -415,15 +506,41 @@ nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
        if (ret)
                return ret;
+        nouveau_bo_sync_for_device(nvbo);
        return 0;
 }
+static inline void *
+_nouveau_bo_mem_index(struct nouveau_bo *nvbo, unsigned index, void *mem, u8 sz)
+{
+        struct ttm_dma_tt *dma_tt;
+        u8 *m = mem;
+        index *= sz;
+        if (m) {
+                /* kmap'd address, return the corresponding offset */
+                m += index;
+        } else {
+                /* DMA-API mapping, lookup the right address */
+                dma_tt = (struct ttm_dma_tt *)nvbo->bo.ttm;
+                m = dma_tt->cpu_address[index / PAGE_SIZE];
+                m += index % PAGE_SIZE;
+        }
+        return m;
+}
+#define nouveau_bo_mem_index(o, i, m) _nouveau_bo_mem_index(o, i, m, sizeof(*m))
 u16
 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index)
 {
        bool is_iomem;
        u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
-        mem = &mem[index];
+        mem = nouveau_bo_mem_index(nvbo, index, mem);
        if (is_iomem)
                return ioread16_native((void __force __iomem *)mem);
        else
@@ -435,7 +552,9 @@ nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
 {
        bool is_iomem;
        u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
-        mem = &mem[index];
+        mem = nouveau_bo_mem_index(nvbo, index, mem);
        if (is_iomem)
                iowrite16_native(val, (void __force __iomem *)mem);
        else
@@ -447,7 +566,9 @@ nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
 {
        bool is_iomem;
        u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
-        mem = &mem[index];
+        mem = nouveau_bo_mem_index(nvbo, index, mem);
        if (is_iomem)
                return ioread32_native((void __force __iomem *)mem);
        else
@@ -459,7 +580,9 @@ nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
 {
        bool is_iomem;
        u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
-        mem = &mem[index];
+        mem = nouveau_bo_mem_index(nvbo, index, mem);
        if (is_iomem)
                iowrite32_native(val, (void __force __iomem *)mem);
        else
@@ -1184,6 +1307,9 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
        struct nouveau_drm_tile *new_tile = NULL;
        int ret = 0;
+        if (nvbo->pin_refcnt)
+                NV_WARN(drm, "Moving pinned object %p!\n", nvbo);
        if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
                ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
                if (ret)
@@ -1376,6 +1502,14 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm)
        dev = drm->dev;
        pdev = nv_device_base(device);
+        /*
+         * Objects matching this condition have been marked as force_coherent,
+         * so use the DMA API for them.
+         */
+        if (!nv_device_is_cpu_coherent(device) &&
+            ttm->caching_state == tt_uncached)
+                return ttm_dma_populate(ttm_dma, dev->dev);
 #if __OS_HAS_AGP
        if (drm->agp.stat == ENABLED) {
                return ttm_agp_tt_populate(ttm);
@@ -1433,6 +1567,14 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
        dev = drm->dev;
        pdev = nv_device_base(device);
+        /*
+         * Objects matching this condition have been marked as force_coherent,
+         * so use the DMA API for them.
+         */
+        if (!nv_device_is_cpu_coherent(device) &&
+            ttm->caching_state == tt_uncached)
+                ttm_dma_unpopulate(ttm_dma, dev->dev);
 #if __OS_HAS_AGP
        if (drm->agp.stat == ENABLED) {
                ttm_agp_tt_unpopulate(ttm);
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.h b/drivers/gpu/drm/nouveau/nouveau_bo.h
index 22d2c764d80b..072222efeeb7 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.h
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.h
@@ -13,6 +13,7 @@ struct nouveau_bo {
        u32 valid_domains;
        struct ttm_place placements[3];
        struct ttm_place busy_placements[3];
+        bool force_coherent;
        struct ttm_bo_kmap_obj kmap;
        struct list_head head;
@@ -72,7 +73,7 @@ int  nouveau_bo_new(struct drm_device *, int size, int align, u32 flags,
                    u32 tile_mode, u32 tile_flags, struct sg_table *sg,
                    struct reservation_object *robj,
                    struct nouveau_bo **);
-int  nouveau_bo_pin(struct nouveau_bo *, u32 flags);
+int  nouveau_bo_pin(struct nouveau_bo *, u32 flags, bool contig);
 int  nouveau_bo_unpin(struct nouveau_bo *);
 int  nouveau_bo_map(struct nouveau_bo *);
 void nouveau_bo_unmap(struct nouveau_bo *);
@@ -84,6 +85,8 @@ void nouveau_bo_wr32(struct nouveau_bo *, unsigned index, u32 val);
 void nouveau_bo_fence(struct nouveau_bo *, struct nouveau_fence *, bool exclusive);
 int  nouveau_bo_validate(struct nouveau_bo *, bool interruptible,
                         bool no_wait_gpu);
+void nouveau_bo_sync_for_device(struct nouveau_bo *nvbo);
+void nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo);
 struct nouveau_vma *
 nouveau_bo_vma_find(struct nouveau_bo *, struct nouveau_vm *);
diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c
index fd3dbd59d73e..aff9099aae6c 100644
--- a/drivers/gpu/drm/nouveau/nouveau_chan.c
+++ b/drivers/gpu/drm/nouveau/nouveau_chan.c
@@ -102,14 +102,14 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device,
        chan->drm = drm;
        /* allocate memory for dma push buffer */
-        target = TTM_PL_FLAG_TT;
+        target = TTM_PL_FLAG_TT | TTM_PL_FLAG_UNCACHED;
        if (nouveau_vram_pushbuf)
                target = TTM_PL_FLAG_VRAM;
        ret = nouveau_bo_new(drm->dev, size, 0, target, 0, 0, NULL, NULL,
                            &chan->push.buffer);
        if (ret == 0) {
-                ret = nouveau_bo_pin(chan->push.buffer, target);
+                ret = nouveau_bo_pin(chan->push.buffer, target, false);
                if (ret == 0)
                        ret = nouveau_bo_map(chan->push.buffer);
        }
@@ -285,7 +285,6 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart)
        struct nouveau_software_chan *swch;
        struct nv_dma_v0 args = {};
        int ret, i;
-        bool save;
        nvif_object_map(chan->object);
@@ -387,11 +386,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart)
        }
        /* initialise synchronisation */
-        save = cli->base.super;
+        return nouveau_fence(chan->drm)->context_new(chan);
-        cli->base.super = true; /* hack until fencenv50 fixed */
-        ret = nouveau_fence(chan->drm)->context_new(chan);
-        cli->base.super = save;
-        return ret;
 }
 int
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 2640fcfa5c37..5d93902a91ab 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -479,6 +479,7 @@ nouveau_display_create(struct drm_device *dev)
        if (nouveau_modeset != 2 && drm->vbios.dcb.entries) {
                static const u16 oclass[] = {
+                        GM204_DISP,
                        GM107_DISP,
                        GK110_DISP,
                        GK104_DISP,
@@ -568,9 +569,10 @@ nouveau_display_suspend(struct drm_device *dev, bool runtime)
        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+                if (nv_crtc->cursor.nvbo) {
-                nouveau_bo_unmap(nv_crtc->cursor.nvbo);
+                        nouveau_bo_unmap(nv_crtc->cursor.nvbo);
-                nouveau_bo_unpin(nv_crtc->cursor.nvbo);
+                        nouveau_bo_unpin(nv_crtc->cursor.nvbo);
+                }
        }
        return 0;
@@ -591,15 +593,17 @@ nouveau_display_resume(struct drm_device *dev, bool runtime)
                if (!nouveau_fb || !nouveau_fb->nvbo)
                        continue;
-                ret = nouveau_bo_pin(nouveau_fb->nvbo, TTM_PL_FLAG_VRAM);
+                ret = nouveau_bo_pin(nouveau_fb->nvbo, TTM_PL_FLAG_VRAM, true);
                if (ret)
                        NV_ERROR(drm, "Could not pin framebuffer\n");
        }
        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+                if (!nv_crtc->cursor.nvbo)
+                        continue;
-                ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM);
+                ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM, true);
                if (!ret)
                        ret = nouveau_bo_map(nv_crtc->cursor.nvbo);
                if (ret)
@@ -630,9 +634,10 @@ nouveau_display_resume(struct drm_device *dev, bool runtime)
        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-                u32 offset = nv_crtc->cursor.nvbo->bo.offset;
-                nv_crtc->cursor.set_offset(nv_crtc, offset);
+                if (!nv_crtc->cursor.nvbo)
+                        continue;
+                nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.nvbo->bo.offset);
                nv_crtc->cursor.set_pos(nv_crtc, nv_crtc->cursor_saved_x,
                                                 nv_crtc->cursor_saved_y);
        }
@@ -710,7 +715,7 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                return -ENOMEM;
        if (new_bo != old_bo) {
-                ret = nouveau_bo_pin(new_bo, TTM_PL_FLAG_VRAM);
+                ret = nouveau_bo_pin(new_bo, TTM_PL_FLAG_VRAM, true);
                if (ret)
                        goto fail_free;
        }
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 57238076049f..afb93bb72f97 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -613,27 +613,6 @@ fail_display:
        return ret;
 }
-int nouveau_pmops_suspend(struct device *dev)
-{
-        struct pci_dev *pdev = to_pci_dev(dev);
-        struct drm_device *drm_dev = pci_get_drvdata(pdev);
-        int ret;
-        if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF ||
-            drm_dev->switch_power_state == DRM_SWITCH_POWER_DYNAMIC_OFF)
-                return 0;
-        ret = nouveau_do_suspend(drm_dev, false);
-        if (ret)
-                return ret;
-        pci_save_state(pdev);
-        pci_disable_device(pdev);
-        pci_ignore_hotplug(pdev);
-        pci_set_power_state(pdev, PCI_D3hot);
-        return 0;
-}
 static int
 nouveau_do_resume(struct drm_device *dev, bool runtime)
 {
@@ -668,7 +647,30 @@ nouveau_do_resume(struct drm_device *dev, bool runtime)
        return 0;
 }
-int nouveau_pmops_resume(struct device *dev)
+int
+nouveau_pmops_suspend(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        int ret;
+        if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF ||
+            drm_dev->switch_power_state == DRM_SWITCH_POWER_DYNAMIC_OFF)
+                return 0;
+        ret = nouveau_do_suspend(drm_dev, false);
+        if (ret)
+                return ret;
+        pci_save_state(pdev);
+        pci_disable_device(pdev);
+        pci_ignore_hotplug(pdev);
+        pci_set_power_state(pdev, PCI_D3hot);
+        return 0;
+}
+int
+nouveau_pmops_resume(struct device *dev)
 {
        struct pci_dev *pdev = to_pci_dev(dev);
        struct drm_device *drm_dev = pci_get_drvdata(pdev);
@@ -688,20 +690,121 @@ int nouveau_pmops_resume(struct device *dev)
        return nouveau_do_resume(drm_dev, false);
 }
-static int nouveau_pmops_freeze(struct device *dev)
+static int
+nouveau_pmops_freeze(struct device *dev)
 {
        struct pci_dev *pdev = to_pci_dev(dev);
        struct drm_device *drm_dev = pci_get_drvdata(pdev);
        return nouveau_do_suspend(drm_dev, false);
 }
-static int nouveau_pmops_thaw(struct device *dev)
+static int
+nouveau_pmops_thaw(struct device *dev)
 {
        struct pci_dev *pdev = to_pci_dev(dev);
        struct drm_device *drm_dev = pci_get_drvdata(pdev);
        return nouveau_do_resume(drm_dev, false);
 }
+static int
+nouveau_pmops_runtime_suspend(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        int ret;
+        if (nouveau_runtime_pm == 0) {
+                pm_runtime_forbid(dev);
+                return -EBUSY;
+        }
+        /* are we optimus enabled? */
+        if (nouveau_runtime_pm == -1 && !nouveau_is_optimus() && !nouveau_is_v1_dsm()) {
+                DRM_DEBUG_DRIVER("failing to power off - not optimus\n");
+                pm_runtime_forbid(dev);
+                return -EBUSY;
+        }
+        nv_debug_level(SILENT);
+        drm_kms_helper_poll_disable(drm_dev);
+        vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_OFF);
+        nouveau_switcheroo_optimus_dsm();
+        ret = nouveau_do_suspend(drm_dev, true);
+        pci_save_state(pdev);
+        pci_disable_device(pdev);
+        pci_set_power_state(pdev, PCI_D3cold);
+        drm_dev->switch_power_state = DRM_SWITCH_POWER_DYNAMIC_OFF;
+        return ret;
+}
+static int
+nouveau_pmops_runtime_resume(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        struct nvif_device *device = &nouveau_drm(drm_dev)->device;
+        int ret;
+        if (nouveau_runtime_pm == 0)
+                return -EINVAL;
+        pci_set_power_state(pdev, PCI_D0);
+        pci_restore_state(pdev);
+        ret = pci_enable_device(pdev);
+        if (ret)
+                return ret;
+        pci_set_master(pdev);
+        ret = nouveau_do_resume(drm_dev, true);
+        drm_kms_helper_poll_enable(drm_dev);
+        /* do magic */
+        nvif_mask(device, 0x88488, (1 << 25), (1 << 25));
+        vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_ON);
+        drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
+        nv_debug_level(NORMAL);
+        return ret;
+}
+static int
+nouveau_pmops_runtime_idle(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        struct nouveau_drm *drm = nouveau_drm(drm_dev);
+        struct drm_crtc *crtc;
+        if (nouveau_runtime_pm == 0) {
+                pm_runtime_forbid(dev);
+                return -EBUSY;
+        }
+        /* are we optimus enabled? */
+        if (nouveau_runtime_pm == -1 && !nouveau_is_optimus() && !nouveau_is_v1_dsm()) {
+                DRM_DEBUG_DRIVER("failing to power off - not optimus\n");
+                pm_runtime_forbid(dev);
+                return -EBUSY;
+        }
+        /* if we have a hdmi audio device - make sure it has a driver loaded */
+        if (drm->hdmi_device) {
+                if (!drm->hdmi_device->driver) {
+                        DRM_DEBUG_DRIVER("failing to power off - no HDMI audio driver loaded\n");
+                        pm_runtime_mark_last_busy(dev);
+                        return -EBUSY;
+                }
+        }
+        list_for_each_entry(crtc, &drm->dev->mode_config.crtc_list, head) {
+                if (crtc->enabled) {
+                        DRM_DEBUG_DRIVER("failing to power off - crtc active\n");
+                        return -EBUSY;
+                }
+        }
+        pm_runtime_mark_last_busy(dev);
+        pm_runtime_autosuspend(dev);
+        /* we don't want the main rpm_idle to call suspend - we want to autosuspend */
+        return 1;
+}
 static int
 nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv)
@@ -908,103 +1011,6 @@ nouveau_drm_pci_table[] = {
        {}
 };
-static int nouveau_pmops_runtime_suspend(struct device *dev)
-{
-        struct pci_dev *pdev = to_pci_dev(dev);
-        struct drm_device *drm_dev = pci_get_drvdata(pdev);
-        int ret;
-        if (nouveau_runtime_pm == 0) {
-                pm_runtime_forbid(dev);
-                return -EBUSY;
-        }
-        /* are we optimus enabled? */
-        if (nouveau_runtime_pm == -1 && !nouveau_is_optimus() && !nouveau_is_v1_dsm()) {
-                DRM_DEBUG_DRIVER("failing to power off - not optimus\n");
-                pm_runtime_forbid(dev);
-                return -EBUSY;
-        }
-        nv_debug_level(SILENT);
-        drm_kms_helper_poll_disable(drm_dev);
-        vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_OFF);
-        nouveau_switcheroo_optimus_dsm();
-        ret = nouveau_do_suspend(drm_dev, true);
-        pci_save_state(pdev);
-        pci_disable_device(pdev);
-        pci_set_power_state(pdev, PCI_D3cold);
-        drm_dev->switch_power_state = DRM_SWITCH_POWER_DYNAMIC_OFF;
-        return ret;
-}
-static int nouveau_pmops_runtime_resume(struct device *dev)
-{
-        struct pci_dev *pdev = to_pci_dev(dev);
-        struct drm_device *drm_dev = pci_get_drvdata(pdev);
-        struct nvif_device *device = &nouveau_drm(drm_dev)->device;
-        int ret;
-        if (nouveau_runtime_pm == 0)
-                return -EINVAL;
-        pci_set_power_state(pdev, PCI_D0);
-        pci_restore_state(pdev);
-        ret = pci_enable_device(pdev);
-        if (ret)
-                return ret;
-        pci_set_master(pdev);
-        ret = nouveau_do_resume(drm_dev, true);
-        drm_kms_helper_poll_enable(drm_dev);
-        /* do magic */
-        nvif_mask(device, 0x88488, (1 << 25), (1 << 25));
-        vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_ON);
-        drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
-        nv_debug_level(NORMAL);
-        return ret;
-}
-static int nouveau_pmops_runtime_idle(struct device *dev)
-{
-        struct pci_dev *pdev = to_pci_dev(dev);
-        struct drm_device *drm_dev = pci_get_drvdata(pdev);
-        struct nouveau_drm *drm = nouveau_drm(drm_dev);
-        struct drm_crtc *crtc;
-        if (nouveau_runtime_pm == 0) {
-                pm_runtime_forbid(dev);
-                return -EBUSY;
-        }
-        /* are we optimus enabled? */
-        if (nouveau_runtime_pm == -1 && !nouveau_is_optimus() && !nouveau_is_v1_dsm()) {
-                DRM_DEBUG_DRIVER("failing to power off - not optimus\n");
-                pm_runtime_forbid(dev);
-                return -EBUSY;
-        }
-        /* if we have a hdmi audio device - make sure it has a driver loaded */
-        if (drm->hdmi_device) {
-                if (!drm->hdmi_device->driver) {
-                        DRM_DEBUG_DRIVER("failing to power off - no HDMI audio driver loaded\n");
-                        pm_runtime_mark_last_busy(dev);
-                        return -EBUSY;
-                }
-        }
-        list_for_each_entry(crtc, &drm->dev->mode_config.crtc_list, head) {
-                if (crtc->enabled) {
-                        DRM_DEBUG_DRIVER("failing to power off - crtc active\n");
-                        return -EBUSY;
-                }
-        }
-        pm_runtime_mark_last_busy(dev);
-        pm_runtime_autosuspend(dev);
-        /* we don't want the main rpm_idle to call suspend - we want to autosuspend */
-        return 1;
-}
 static void nouveau_display_options(void)
 {
        DRM_DEBUG_DRIVER("Loading Nouveau with parameters:\n");
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 593ef8a2a069..3ed12a8cfc91 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -341,7 +341,7 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
                goto out;
        }
-        ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_VRAM);
+        ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_VRAM, false);
        if (ret) {
                NV_ERROR(drm, "failed to pin fb: %d\n", ret);
                goto out_unref;
@@ -498,6 +498,23 @@ nouveau_fbcon_set_suspend_work(struct work_struct *work)
        console_unlock();
 }
+void
+nouveau_fbcon_set_suspend(struct drm_device *dev, int state)
+{
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        if (drm->fbcon) {
+                if (state == FBINFO_STATE_RUNNING) {
+                        schedule_work(&drm->fbcon->work);
+                        return;
+                }
+                flush_work(&drm->fbcon->work);
+                console_lock();
+                fb_set_suspend(drm->fbcon->helper.fbdev, state);
+                nouveau_fbcon_accel_save_disable(dev);
+                console_unlock();
+        }
+}
 int
 nouveau_fbcon_init(struct drm_device *dev)
 {
@@ -557,20 +574,3 @@ nouveau_fbcon_fini(struct drm_device *dev)
        kfree(drm->fbcon);
        drm->fbcon = NULL;
 }
-void
-nouveau_fbcon_set_suspend(struct drm_device *dev, int state)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        if (drm->fbcon) {
-                if (state == FBINFO_STATE_RUNNING) {
-                        schedule_work(&drm->fbcon->work);
-                        return;
-                }
-                flush_work(&drm->fbcon->work);
-                console_lock();
-                fb_set_suspend(drm->fbcon->helper.fbdev, state);
-                nouveau_fbcon_accel_save_disable(dev);
-                console_unlock();
-        }
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index ebba9deb0d04..28d51a22a4bf 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -870,6 +870,7 @@ nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
                else
                        ret = lret;
        }
+        nouveau_bo_sync_for_cpu(nvbo);
        drm_gem_object_unreference_unlocked(gem);
        return ret;
@@ -879,6 +880,17 @@ int
 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
                           struct drm_file *file_priv)
 {
+        struct drm_nouveau_gem_cpu_fini *req = data;
+        struct drm_gem_object *gem;
+        struct nouveau_bo *nvbo;
+        gem = drm_gem_object_lookup(dev, file_priv, req->handle);
+        if (!gem)
+                return -ENOENT;
+        nvbo = nouveau_gem_object(gem);
+        nouveau_bo_sync_for_device(nvbo);
+        drm_gem_object_unreference_unlocked(gem);
        return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_platform.c b/drivers/gpu/drm/nouveau/nouveau_platform.c
index 246a824c16ca..b307bbedd4c4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_platform.c
+++ b/drivers/gpu/drm/nouveau/nouveau_platform.c
@@ -27,6 +27,7 @@
 #include <linux/of.h>
 #include <linux/reset.h>
 #include <linux/regulator/consumer.h>
+#include <soc/tegra/fuse.h>
 #include <soc/tegra/pmc.h>
 #include "nouveau_drm.h"
@@ -128,6 +129,7 @@ static int nouveau_platform_probe(struct platform_device *pdev)
        }
        device->gpu = gpu;
+        device->gpu_speedo = tegra_sku_info.gpu_speedo_value;
        err = drm_dev_register(drm, 0);
        if (err < 0)
diff --git a/drivers/gpu/drm/nouveau/nouveau_platform.h b/drivers/gpu/drm/nouveau/nouveau_platform.h
index 91f66504900e..58c28b5653d5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_platform.h
+++ b/drivers/gpu/drm/nouveau/nouveau_platform.h
@@ -41,6 +41,8 @@ struct nouveau_platform_device {
        struct nouveau_device device;
        struct nouveau_platform_gpu *gpu;
+        int gpu_speedo;
 };
 #define nv_device_to_platform(d)                                               \
diff --git a/drivers/gpu/drm/nouveau/nouveau_prime.c b/drivers/gpu/drm/nouveau/nouveau_prime.c
index 228226ab27fc..dd32ad6db53d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_prime.c
+++ b/drivers/gpu/drm/nouveau/nouveau_prime.c
@@ -93,7 +93,7 @@ int nouveau_gem_prime_pin(struct drm_gem_object *obj)
        int ret;
        /* pin buffer into GTT */
-        ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_TT);
+        ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_TT, false);
        if (ret)
                return -EINVAL;
diff --git a/drivers/gpu/drm/nouveau/nv17_fence.c b/drivers/gpu/drm/nouveau/nv17_fence.c
index 40b461c7d5c5..57860cfa1de5 100644
--- a/drivers/gpu/drm/nouveau/nv17_fence.c
+++ b/drivers/gpu/drm/nouveau/nv17_fence.c
@@ -131,7 +131,7 @@ nv17_fence_create(struct nouveau_drm *drm)
        ret = nouveau_bo_new(drm->dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
                             0, 0x0000, NULL, NULL, &priv->bo);
        if (!ret) {
-                ret = nouveau_bo_pin(priv->bo, TTM_PL_FLAG_VRAM);
+                ret = nouveau_bo_pin(priv->bo, TTM_PL_FLAG_VRAM, false);
                if (!ret) {
                        ret = nouveau_bo_map(priv->bo);
                        if (ret)
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index 5beae7596f62..b1da7cd40db8 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -66,15 +66,29 @@ static int
 nv50_chan_create(struct nvif_object *disp, const u32 *oclass, u8 head,
                 void *data, u32 size, struct nv50_chan *chan)
 {
+        const u32 handle = (oclass[0] << 16) | head;
+        u32 sclass[8];
+        int ret, i;
+        ret = nvif_object_sclass(disp, sclass, ARRAY_SIZE(sclass));
+        WARN_ON(ret > ARRAY_SIZE(sclass));
+        if (ret < 0)
+                return ret;
        while (oclass[0]) {
-                int ret = nvif_object_init(disp, NULL, (oclass[0] << 16) | head,
+                for (i = 0; i < ARRAY_SIZE(sclass); i++) {
-                                           oclass[0], data, size,
+                        if (sclass[i] == oclass[0]) {
-                                          &chan->user);
+                                ret = nvif_object_init(disp, NULL, handle,
-                if (oclass++, ret == 0) {
+                                                       oclass[0], data, size,
-                        nvif_object_map(&chan->user);
+                                                       &chan->user);
-                        return ret;
+                                if (ret == 0)
+                                        nvif_object_map(&chan->user);
+                                return ret;
+                        }
                }
+                oclass++;
        }
        return -ENOSYS;
 }
@@ -111,6 +125,7 @@ nv50_pioc_create(struct nvif_object *disp, const u32 *oclass, u8 head,
 struct nv50_curs {
        struct nv50_pioc base;
+        struct nouveau_bo *image;
 };
 static int
@@ -266,6 +281,7 @@ nv50_core_create(struct nvif_object *disp, u64 syncbuf, struct nv50_mast *core)
                .pushbuf = 0xb0007d00,
        };
        static const u32 oclass[] = {
+                GM204_DISP_CORE_CHANNEL_DMA,
                GM107_DISP_CORE_CHANNEL_DMA,
                GK110_DISP_CORE_CHANNEL_DMA,
                GK104_DISP_CORE_CHANNEL_DMA,
@@ -425,8 +441,21 @@ evo_kick(u32 *push, void *evoc)
        mutex_unlock(&dmac->lock);
 }
+#if 1
 #define evo_mthd(p,m,s) *((p)++) = (((s) << 18) | (m))
 #define evo_data(p,d)   *((p)++) = (d)
+#else
+#define evo_mthd(p,m,s) do {                                                   \
+        const u32 _m = (m), _s = (s);                                          \
+        printk(KERN_ERR "%04x %d %s\n", _m, _s, __func__);                     \
+        *((p)++) = ((_s << 18) | _m);                                          \
+} while(0)
+#define evo_data(p,d) do {                                                     \
+        const u32 _d = (d);                                                    \
+        printk(KERN_ERR "\t%08x\n", _d);                                       \
+        *((p)++) = _d;                                                         \
+} while(0)
+#endif
 static bool
 evo_sync_wait(void *data)
@@ -888,23 +917,24 @@ static void
 nv50_crtc_cursor_show(struct nouveau_crtc *nv_crtc)
 {
        struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
+        struct nv50_curs *curs = nv50_curs(&nv_crtc->base);
        u32 *push = evo_wait(mast, 16);
        if (push) {
                if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) {
                        evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 2);
                        evo_data(push, 0x85000000);
-                        evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8);
+                        evo_data(push, curs->image->bo.offset >> 8);
                } else
                if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
                        evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 2);
                        evo_data(push, 0x85000000);
-                        evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8);
+                        evo_data(push, curs->image->bo.offset >> 8);
                        evo_mthd(push, 0x089c + (nv_crtc->index * 0x400), 1);
                        evo_data(push, mast->base.vram.handle);
                } else {
                        evo_mthd(push, 0x0480 + (nv_crtc->index * 0x300), 2);
                        evo_data(push, 0x85000000);
-                        evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8);
+                        evo_data(push, curs->image->bo.offset >> 8);
                        evo_mthd(push, 0x048c + (nv_crtc->index * 0x300), 1);
                        evo_data(push, mast->base.vram.handle);
                }
@@ -941,8 +971,9 @@ static void
 nv50_crtc_cursor_show_hide(struct nouveau_crtc *nv_crtc, bool show, bool update)
 {
        struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
+        struct nv50_curs *curs = nv50_curs(&nv_crtc->base);
-        if (show)
+        if (show && curs->image)
                nv50_crtc_cursor_show(nv_crtc);
        else
                nv50_crtc_cursor_hide(nv_crtc);
@@ -1042,7 +1073,7 @@ nv50_crtc_commit(struct drm_crtc *crtc)
                evo_kick(push, mast);
        }
-        nv50_crtc_cursor_show_hide(nv_crtc, nv_crtc->cursor.visible, true);
+        nv50_crtc_cursor_show_hide(nv_crtc, true, true);
        nv50_display_flip_next(crtc, crtc->primary->fb, NULL, 1);
 }
@@ -1061,7 +1092,7 @@ nv50_crtc_swap_fbs(struct drm_crtc *crtc, struct drm_framebuffer *old_fb)
        struct nv50_head *head = nv50_head(crtc);
        int ret;
-        ret = nouveau_bo_pin(nvfb->nvbo, TTM_PL_FLAG_VRAM);
+        ret = nouveau_bo_pin(nvfb->nvbo, TTM_PL_FLAG_VRAM, true);
        if (ret == 0) {
                if (head->image)
                        nouveau_bo_unpin(head->image);
@@ -1242,13 +1273,13 @@ nv50_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
                     uint32_t handle, uint32_t width, uint32_t height)
 {
        struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+        struct nv50_curs *curs = nv50_curs(crtc);
        struct drm_device *dev = crtc->dev;
-        struct drm_gem_object *gem;
+        struct drm_gem_object *gem = NULL;
-        struct nouveau_bo *nvbo;
+        struct nouveau_bo *nvbo = NULL;
-        bool visible = (handle != 0);
+        int ret = 0;
-        int i, ret = 0;
-        if (visible) {
+        if (handle) {
                if (width != 64 || height != 64)
                        return -EINVAL;
@@ -1257,23 +1288,17 @@ nv50_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
                        return -ENOENT;
                nvbo = nouveau_gem_object(gem);
-                ret = nouveau_bo_map(nvbo);
+                ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_VRAM, true);
-                if (ret == 0) {
-                        for (i = 0; i < 64 * 64; i++) {
-                                u32 v = nouveau_bo_rd32(nvbo, i);
-                                nouveau_bo_wr32(nv_crtc->cursor.nvbo, i, v);
-                        }
-                        nouveau_bo_unmap(nvbo);
-                }
-                drm_gem_object_unreference_unlocked(gem);
        }
-        if (visible != nv_crtc->cursor.visible) {
+        if (ret == 0) {
-                nv50_crtc_cursor_show_hide(nv_crtc, visible, true);
+                if (curs->image)
-                nv_crtc->cursor.visible = visible;
+                        nouveau_bo_unpin(curs->image);
+                nouveau_bo_ref(nvbo, &curs->image);
        }
+        drm_gem_object_unreference_unlocked(gem);
+        nv50_crtc_cursor_show_hide(nv_crtc, true, true);
        return ret;
 }
@@ -1328,10 +1353,10 @@ nv50_crtc_destroy(struct drm_crtc *crtc)
                nouveau_bo_unpin(head->image);
        nouveau_bo_ref(NULL, &head->image);
-        nouveau_bo_unmap(nv_crtc->cursor.nvbo);
+        /*XXX: ditto */
-        if (nv_crtc->cursor.nvbo)
+        if (head->curs.image)
-                nouveau_bo_unpin(nv_crtc->cursor.nvbo);
+                nouveau_bo_unpin(head->curs.image);
-        nouveau_bo_ref(NULL, &nv_crtc->cursor.nvbo);
+        nouveau_bo_ref(NULL, &head->curs.image);
        nouveau_bo_unmap(nv_crtc->lut.nvbo);
        if (nv_crtc->lut.nvbo)
@@ -1363,16 +1388,6 @@ static const struct drm_crtc_funcs nv50_crtc_func = {
        .page_flip = nouveau_crtc_page_flip,
 };
-static void
-nv50_cursor_set_pos(struct nouveau_crtc *nv_crtc, int x, int y)
-{
-}
-static void
-nv50_cursor_set_offset(struct nouveau_crtc *nv_crtc, uint32_t offset)
-{
-}
 static int
 nv50_crtc_create(struct drm_device *dev, int index)
 {
@@ -1391,8 +1406,6 @@ nv50_crtc_create(struct drm_device *dev, int index)
        head->base.set_color_vibrance = nv50_crtc_set_color_vibrance;
        head->base.color_vibrance = 50;
        head->base.vibrant_hue = 0;
-        head->base.cursor.set_offset = nv50_cursor_set_offset;
-        head->base.cursor.set_pos = nv50_cursor_set_pos;
        for (i = 0; i < 256; i++) {
                head->base.lut.r[i] = i << 8;
                head->base.lut.g[i] = i << 8;
@@ -1407,7 +1420,7 @@ nv50_crtc_create(struct drm_device *dev, int index)
        ret = nouveau_bo_new(dev, 8192, 0x100, TTM_PL_FLAG_VRAM,
                             0, 0x0000, NULL, NULL, &head->base.lut.nvbo);
        if (!ret) {
-                ret = nouveau_bo_pin(head->base.lut.nvbo, TTM_PL_FLAG_VRAM);
+                ret = nouveau_bo_pin(head->base.lut.nvbo, TTM_PL_FLAG_VRAM, true);
                if (!ret) {
                        ret = nouveau_bo_map(head->base.lut.nvbo);
                        if (ret)
@@ -1427,22 +1440,6 @@ nv50_crtc_create(struct drm_device *dev, int index)
        if (ret)
                goto out;
-        ret = nouveau_bo_new(dev, 64 * 64 * 4, 0x100, TTM_PL_FLAG_VRAM,
-                             0, 0x0000, NULL, NULL, &head->base.cursor.nvbo);
-        if (!ret) {
-                ret = nouveau_bo_pin(head->base.cursor.nvbo, TTM_PL_FLAG_VRAM);
-                if (!ret) {
-                        ret = nouveau_bo_map(head->base.cursor.nvbo);
-                        if (ret)
-                                nouveau_bo_unpin(head->base.lut.nvbo);
-                }
-                if (ret)
-                        nouveau_bo_ref(NULL, &head->base.cursor.nvbo);
-        }
-        if (ret)
-                goto out;
        /* allocate page flip / sync resources */
        ret = nv50_base_create(disp->disp, index, disp->sync->bo.offset,
                              &head->sync);
@@ -2374,11 +2371,6 @@ nv50_fb_ctor(struct drm_framebuffer *fb)
        u8 kind = nouveau_bo_tile_layout(nvbo) >> 8;
        u8 tile = nvbo->tile_mode;
-        if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG) {
-                NV_ERROR(drm, "framebuffer requires contiguous bo\n");
-                return -EINVAL;
-        }
        if (drm->device.info.chipset >= 0xc0)
                tile >>= 4; /* yep.. */
@@ -2492,7 +2484,7 @@ nv50_display_create(struct drm_device *dev)
        ret = nouveau_bo_new(dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
                             0, 0x0000, NULL, NULL, &disp->sync);
        if (!ret) {
-                ret = nouveau_bo_pin(disp->sync, TTM_PL_FLAG_VRAM);
+                ret = nouveau_bo_pin(disp->sync, TTM_PL_FLAG_VRAM, true);
                if (!ret) {
                        ret = nouveau_bo_map(disp->sync);
                        if (ret)
diff --git a/drivers/gpu/drm/nouveau/nv50_fence.c b/drivers/gpu/drm/nouveau/nv50_fence.c
index 22d242b37962..a82d9ea7c6fd 100644
--- a/drivers/gpu/drm/nouveau/nv50_fence.c
+++ b/drivers/gpu/drm/nouveau/nv50_fence.c
@@ -102,7 +102,7 @@ nv50_fence_create(struct nouveau_drm *drm)
        ret = nouveau_bo_new(drm->dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
                             0, 0x0000, NULL, NULL, &priv->bo);
        if (!ret) {
-                ret = nouveau_bo_pin(priv->bo, TTM_PL_FLAG_VRAM);
+                ret = nouveau_bo_pin(priv->bo, TTM_PL_FLAG_VRAM, false);
                if (!ret) {
                        ret = nouveau_bo_map(priv->bo);
                        if (ret)
diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c
index d6c6c87c3f07..cb5b88938d45 100644
--- a/drivers/gpu/drm/nouveau/nv84_fence.c
+++ b/drivers/gpu/drm/nouveau/nv84_fence.c
@@ -234,7 +234,7 @@ nv84_fence_create(struct nouveau_drm *drm)
        ret = nouveau_bo_new(drm->dev, 16 * priv->base.contexts, 0,
                             TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL, &priv->bo);
        if (ret == 0) {
-                ret = nouveau_bo_pin(priv->bo, TTM_PL_FLAG_VRAM);
+                ret = nouveau_bo_pin(priv->bo, TTM_PL_FLAG_VRAM, false);
                if (ret == 0) {
                        ret = nouveau_bo_map(priv->bo);
                        if (ret)
@@ -246,10 +246,10 @@ nv84_fence_create(struct nouveau_drm *drm)
        if (ret == 0)
                ret = nouveau_bo_new(drm->dev, 16 * priv->base.contexts, 0,
-                                     TTM_PL_FLAG_TT, 0, 0, NULL, NULL,
+                                     TTM_PL_FLAG_TT | TTM_PL_FLAG_UNCACHED, 0,
-                                     &priv->bo_gart);
+                                     0, NULL, NULL, &priv->bo_gart);
        if (ret == 0) {
-                ret = nouveau_bo_pin(priv->bo_gart, TTM_PL_FLAG_TT);
+                ret = nouveau_bo_pin(priv->bo_gart, TTM_PL_FLAG_TT, false);
                if (ret == 0) {
                        ret = nouveau_bo_map(priv->bo_gart);
                        if (ret)
diff --git a/drivers/gpu/drm/nouveau/nvif/class.h b/drivers/gpu/drm/nouveau/nvif/class.h
index e5a27df0672b..4e308eacb27a 100644
--- a/drivers/gpu/drm/nouveau/nvif/class.h
+++ b/drivers/gpu/drm/nouveau/nvif/class.h
@@ -35,6 +35,7 @@
 #define GK104_DISP                                                   0x00009170
 #define GK110_DISP                                                   0x00009270
 #define GM107_DISP                                                   0x00009470
+#define GM204_DISP                                                   0x00009570
 #define NV50_DISP_CURSOR                                             0x0000507a
 #define G82_DISP_CURSOR                                              0x0000827a
@@ -65,6 +66,7 @@
 #define GK104_DISP_CORE_CHANNEL_DMA                                  0x0000917d
 #define GK110_DISP_CORE_CHANNEL_DMA                                  0x0000927d
 #define GM107_DISP_CORE_CHANNEL_DMA                                  0x0000947d
+#define GM204_DISP_CORE_CHANNEL_DMA                                  0x0000957d
 #define NV50_DISP_OVERLAY_CHANNEL_DMA                                0x0000507e
 #define G82_DISP_OVERLAY_CHANNEL_DMA                                 0x0000827e
@@ -131,6 +133,7 @@ struct nv_device_v0 {
 #define NV_DEVICE_V0_DISABLE_COPY1                        0x0000010000000000ULL
 #define NV_DEVICE_V0_DISABLE_VIC                          0x0000020000000000ULL
 #define NV_DEVICE_V0_DISABLE_VENC                         0x0000040000000000ULL
+#define NV_DEVICE_V0_DISABLE_COPY2                        0x0000080000000000ULL
        __u64 disable;  /* disable particular subsystems */
        __u64 debug0;   /* as above, but *internal* ids, and *NOT* ABI */
 };
diff --git a/drivers/gpu/drm/nouveau/nvif/client.c b/drivers/gpu/drm/nouveau/nvif/client.c
index 3c4df1fc26dc..3f7ac5bc8e03 100644
--- a/drivers/gpu/drm/nouveau/nvif/client.c
+++ b/drivers/gpu/drm/nouveau/nvif/client.c
@@ -62,6 +62,7 @@ nvif_drivers[] = {
 #else
        &nvif_driver_drm,
        &nvif_driver_lib,
+        &nvif_driver_null,
 #endif
        NULL
 };
diff --git a/drivers/gpu/drm/nouveau/nvif/driver.h b/drivers/gpu/drm/nouveau/nvif/driver.h
index ac4bdb3ea506..8bd39e69229c 100644
--- a/drivers/gpu/drm/nouveau/nvif/driver.h
+++ b/drivers/gpu/drm/nouveau/nvif/driver.h
@@ -17,5 +17,6 @@ struct nvif_driver {
 extern const struct nvif_driver nvif_driver_nvkm;
 extern const struct nvif_driver nvif_driver_drm;
 extern const struct nvif_driver nvif_driver_lib;
+extern const struct nvif_driver nvif_driver_null;
 #endif
diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c
index 11a5043959dc..011a3363c265 100644
--- a/drivers/soc/tegra/fuse/fuse-tegra.c
+++ b/drivers/soc/tegra/fuse/fuse-tegra.c
@@ -31,6 +31,7 @@
 static u32 (*fuse_readl)(const unsigned int offset);
 static int fuse_size;
 struct tegra_sku_info tegra_sku_info;
+EXPORT_SYMBOL(tegra_sku_info);
 static const char *tegra_revision_name[TEGRA_REVISION_MAX] = {
        [TEGRA_REVISION_UNKNOWN] = "unknown",